Merge fef51f0 into 49afdc1

development.ini

@@ -0,0 +1,47 @@
+[server:main]
+use = egg:pyFF#pyffs
+
+[app:main]
+use = egg:pyFF#pyffapp
+
+[loggers]
+keys = root, pyff, xmlsec, pyff.pipes, pyff.store
+
+[handlers]
+keys = console
+
+[formatters]
+keys = generic
+
+[logger_root]
+level = DEBUG
+handlers = console
+
+[logger_pyff]
+level = DEBUG
+handlers =
+qualname = pyff 
+
+[logger_xmlsec]
+level = INFO
+handlers = 
+qualname = xmlsec
+
+[logger_pyff.pipes]
+level = INFO
+handlers =
+qualname = pyff.pipes
+
+[logger_pyff.store]
+level = DEBUG
+handlers =
+qualname = pyff.store
+
+[handler_console]
+class = StreamHandler
+args = (sys.stderr,)
+level = NOTSET
+formatter = generic
+
+[formatter_generic]
+format = %(asctime)s %(levelname)-5.5s [%(name)s:%(lineno)s][%(threadName)s] %(message)s

requirements.txt

@@ -23,3 +23,6 @@ multiprocess
 requests_cache
 requests_file
 whoosh
+accept_types
+pyramid
+six

setup.py

@@ -40,7 +40,9 @@
     'pyyaml',
     'multiprocess',
     'minify',
-    'whoosh'
+    'whoosh',
+    'pyramid',
+    'accept_types'
 ]
 
 python_implementation_str = python_implementation()
@@ -80,7 +82,13 @@
       zip_safe=False,
       install_requires=install_requires,
       entry_points={
-          'console_scripts': ['pyff=pyff.md:main', 'pyffd=pyff.mdx:main']
+          'console_scripts': ['pyff=pyff.md:main', 'pyffd=pyff.mdx:main'],
+          'paste.app_factory': [
+             'pyffapp=pyff.wsgi:app_factory'
+          ],
+          'paste.server_runner': [
+             'pyffs=pyff.wsgi:server_runner'
+          ],
       },
       message_extractors={'src': [
           ('**.py', 'python', None),

src/pyff/builtins.py

@@ -23,12 +23,13 @@
 from .utils import total_seconds, dumptree, safe_write, root, with_tree, duration2timedelta, xslt_transform, \
     validate_document, hash_id
 from .samlmd import sort_entities, iter_entities, annotate_entity, set_entity_attributes, \
-    discojson_t, set_pubinfo, set_reginfo, find_in_document, entitiesdescriptor, set_nodecountry
+    discojson_t, set_pubinfo, set_reginfo, find_in_document, entitiesdescriptor, set_nodecountry, resolve_entities
 from .fetch import Resource
 from six.moves.urllib_parse import urlparse
 from .exceptions import MetadataException
 from .store import make_store_instance
 import six
+import ipaddr
 
 __author__ = 'leifj'
 
@@ -484,7 +485,7 @@ def load(req, *opts):
             "Usage: load resource [as url] [[verify] verification] [via pipeline] [cleanup pipeline]")
 
         url = r.pop(0)
-        params = {"via": [],"cleanup": [],"verify": None, "as": url}
+        params = {"via": [], "cleanup": [], "verify": None, "as": url}
 
         while len(r) > 0:
             elt = r.pop(0)
@@ -603,7 +604,55 @@ def select(req, *opts):
         if opts[0] == 'as' and len(opts) == 2:
             name = opts[1]
 
-    ot = entitiesdescriptor(args, name, lookup_fn=req.md.store.select)
+    entities = resolve_entities(args, lookup_fn=req.md.store.select)
+
+    if req.state.get('match', None) is not None:  # TODO - allow this to be passed in via normal arguments
+
+        match = req.state['match']
+
+        if isinstance(match, six.string_types):
+            query = [match.lower()]
+
+        def _strings(elt):
+            lst = []
+            for attr in ['{%s}DisplayName' % NS['mdui'],
+                         '{%s}ServiceName' % NS['md'],
+                         '{%s}OrganizationDisplayName' % NS['md'],
+                         '{%s}OrganizationName' % NS['md'],
+                         '{%s}Keywords' % NS['mdui'],
+                         '{%s}Scope' % NS['shibmd']]:
+                lst.extend([s.text for s in elt.iter(attr)])
+            lst.append(elt.get('entityID'))
+            return [item for item in lst if item is not None]
+
+        def _ip_networks(elt):
+            return [ipaddr.IPNetwork(x.text) for x in elt.iter('{%s}IPHint' % NS['mdui'])]
+
+        def _match(qq, elt):
+            for q in qq:
+                q = q.strip()
+                if ':' in q or '.' in q:
+                    try:
+                        nets = _ip_networks(elt)
+                        for net in nets:
+                            if ':' in q and ipaddr.IPv6Address(q) in net:
+                                return net
+                            if '.' in q and ipaddr.IPv4Address(q) in net:
+                                return net
+                    except ValueError:
+                        pass
+
+                if q is not None and len(q) > 0:
+                    tokens = _strings(elt)
+                    for tstr in tokens:
+                        for tpart in tstr.split():
+                            if tpart.lower().startswith(q):
+                                return tstr
+            return None
+
+        entities = filter(lambda e: _match(match, e) is not None, entities)
+
+    ot = entitiesdescriptor(entities, name)
     if ot is None:
         raise PipeException("empty select - stop")
 
@@ -713,6 +762,12 @@ def _discojson(req, *opts):
     """
 Return a discojuice-compatible json representation of the tree
 
+.. code-block:: yaml
+  discojson:
+      - load_icons: True
+
+This would return a json representation of the active tree with data: URI version of the icons
+
 :param req: The request
 :param opts: Options (unusued)
 :return: returns a JSON array
@@ -721,7 +776,11 @@ def _discojson(req, *opts):
     if req.t is None:
         raise PipeException("Your pipeline is missing a select statement.")
 
-    res = discojson_t(req.t)
+    load_icons = False
+    if req.args:
+        load_icons = bool(req.args.get('load_icons', False))
+
+    res = discojson_t(req.t, load_icons=load_icons)
     res.sort(key=operator.itemgetter('title'))
 
     return json.dumps(res)

src/pyff/constants.py

@@ -4,6 +4,11 @@
 
 import pyconfig
 import logging
+import getopt
+import sys
+import os
+from . import __version__ as pyff_version
+
 
 __author__ = 'leifj'
 
@@ -75,6 +80,85 @@ class Config(object):
     redis_port = pyconfig.setting("pyff.redis_port", 6379)
     rq_queue = pyconfig.setting("pyff.rq_queue", "pyff")
     cache_chunks = pyconfig.setting("pyff.cache_chunks", 10)
+    pipeline = pyconfig.setting("pyff.pipeline", os.environ.get("PYFF_PIPELINE"))
 
 
 config = Config()
+
+
+def parse_options(program, docs, short_args, long_args):
+    try:
+        opts, args = getopt.getopt(sys.argv[1:], short_args, long_args)
+    except getopt.error as msg:
+        print(msg)
+        print(docs)
+        sys.exit(2)
+
+    if config.loglevel is None:
+        config.loglevel = logging.INFO
+
+    if config.aliases is None:
+        config.aliases = dict()
+
+    if config.modules is None:
+        config.modules = []
+
+    try:  # pragma: nocover
+        for o, a in opts:
+            if o in ('-h', '--help'):
+                print(docs)
+                sys.exit(0)
+            elif o == '--loglevel':
+                config.loglevel = getattr(logging, a.upper(), None)
+                if not isinstance(config.loglevel, int):
+                    raise ValueError('Invalid log level: %s' % config.loglevel)
+            elif o == '--logfile':
+                config.logfile = a
+            elif o in ('--log', '-l'):
+                config.error_log = a
+                config.access_log = a
+            elif o in '--error-log':
+                config.error_log = a
+            elif o in '--access-log':
+                config.access_log = a
+            elif o in ('--host', '-H'):
+                config.bind_address = a
+            elif o in ('--port', '-P'):
+                config.port = int(a)
+            elif o in ('--pidfile', '-p'):
+                config.pid_file = a
+            elif o in ('--no-caching', '-C'):
+                config.caching_enabled = False
+            elif o in ('--caching-delay', 'D'):
+                config.caching_delay = int(o)
+            elif o in ('--foreground', '-f'):
+                config.daemonize = False
+            elif o in ('--autoreload', '-a'):
+                config.autoreload = True
+            elif o in '--frequency':
+                config.update_frequency = int(a)
+            elif o in ('-A', '--alias'):
+                (a, colon, uri) = a.partition(':')
+                assert (colon == ':')
+                if a and uri:
+                    config.aliases[a] = uri
+            elif o in '--dir':
+                config.base_dir = a
+            elif o in '--proxy':
+                config.proxy = True
+            elif o in '--allow_shutdown':
+                config.allow_shutdown = True
+            elif o in ('-m', '--module'):
+                config.modules.append(a)
+            elif o in '--version':
+                print("{} version {}".format(program, pyff_version))
+                sys.exit(0)
+            else:
+                raise ValueError("Unknown option '%s'" % o)
+
+    except Exception as ex:
+        print(ex)
+        print(docs)
+        sys.exit(3)
+
+    return args

src/pyff/md.py

@@ -15,44 +15,14 @@
 from . import __version__
 from .samlmd import MDRepository
 from .pipes import plumbing
-from .constants import config
+from .constants import config, parse_options
 
 
 def main():
     """
     The main entrypoint for the pyFF cmdline tool.
     """
-
-    opts = None
-    args = None
-    try:
-        opts, args = getopt.getopt(sys.argv[1:], 'hRm', ['help', 'loglevel=', 'logfile=', 'version', 'module'])
-    except getopt.error as msg:
-        print(msg)
-        print(__doc__)
-        sys.exit(2)
-
-    if config.loglevel is None:
-        config.loglevel = logging.WARN
-
-    if config.modules is None:
-        config.modules = []
-
-    for o, a in opts:
-        if o in ('-h', '--help'):
-            print(__doc__)
-            sys.exit(0)
-        elif o in '--loglevel':
-            config.loglevel = getattr(logging, a.upper(), None)
-            if not isinstance(config.loglevel, int):
-                raise ValueError('Invalid log level: %s' % a)
-        elif o in '--logfile':
-            config.logfile = a
-        elif o in ('-m', '--module'):
-            config.modules.append(a)
-        elif o in '--version':
-            print("pyff version {}".format(__version__))
-            sys.exit(0)
+    args = parse_options("pyff", __doc__, 'hm', ['help', 'loglevel=', 'logfile=', 'version', 'module'])
 
     log_args = {'level': config.loglevel}
     if config.logfile is not None: