Add support for interfaces, lists, and variables to lift/walk/pivot/f…

…ilter syntax (SYN-6011, SYN-5837, SYN-1879, SYN-285, SYN-1875, SYN-6074) (#3334)

synapse/cortex.py

@@ -961,6 +961,10 @@ async def initServiceStorage(self):
 
         self.model = s_datamodel.Model()
 
+        await self._bumpCellVers('cortex:extmodel', (
+            (1, self._migrateTaxonomyIface),
+        ), nexs=False)
+
         # Perform module loading
         await self._loadCoreMods()
         await self._loadExtModel()
@@ -2960,6 +2964,27 @@ async def itemsStormVar(self):
     async def _cortexHealth(self, health):
         health.update('cortex', 'nominal')
 
+    async def _migrateTaxonomyIface(self):
+
+        extforms = await (await self.hive.open(('cortex', 'model', 'forms'))).dict()
+
+        for formname, basetype, typeopts, typeinfo in extforms.values():
+            try:
+                ifaces = typeinfo.get('interfaces')
+
+                if ifaces and 'taxonomy' in ifaces:
+                    logger.warning(f'Migrating taxonomy interface on form {formname} to meta:taxonomy.')
+
+                    ifaces = set(ifaces)
+                    ifaces.remove('taxonomy')
+                    ifaces.add('meta:taxonomy')
+                    typeinfo['interfaces'] = tuple(ifaces)
+
+                    await extforms.set(formname, (formname, basetype, typeopts, typeinfo))
+
+            except Exception as e:  # pragma: no cover
+                logger.exception(f'Taxonomy migration error for form: {formname} (skipped).')
+
     async def _loadExtModel(self):
 
         self.extforms = await (await self.hive.open(('cortex', 'model', 'forms'))).dict()
@@ -3166,6 +3191,17 @@ async def addForm(self, formname, basetype, typeopts, typeinfo):
 
     @s_nexus.Pusher.onPush('model:form:add')
     async def _addForm(self, formname, basetype, typeopts, typeinfo):
+
+        ifaces = typeinfo.get('interfaces')
+
+        if ifaces and 'taxonomy' in ifaces:
+            logger.warning(f'{formname} is using the deprecated taxonomy interface, updating to meta:taxonomy.')
+
+            ifaces = set(ifaces)
+            ifaces.remove('taxonomy')
+            ifaces.add('meta:taxonomy')
+            typeinfo['interfaces'] = tuple(ifaces)
+
         self.model.addType(formname, basetype, typeopts, typeinfo)
         self.model.addForm(formname, {}, ())
 

synapse/datamodel.py

@@ -11,6 +11,7 @@
 import synapse.common as s_common
 
 import synapse.lib.coro as s_coro
+import synapse.lib.cache as s_cache
 import synapse.lib.types as s_types
 import synapse.lib.dyndeps as s_dyndeps
 import synapse.lib.grammar as s_grammar
@@ -19,6 +20,8 @@
 
 hexre = regex.compile('^[0-9a-z]+$')
 
+PREFIX_CACHE_SIZE = 1000
+
 class TagProp:
 
     def __init__(self, model, name, tdef, info):
@@ -450,11 +453,13 @@ def __init__(self):
 
         self.propsbytype = collections.defaultdict(list)  # name: Prop()
         self.arraysbytype = collections.defaultdict(list)
-        # TODO use this for <nodes> -> foo:iface
+        self.ifaceprops = collections.defaultdict(list)
         self.formsbyiface = collections.defaultdict(list)
         self.edgesbyn1 = collections.defaultdict(list)
         self.edgesbyn2 = collections.defaultdict(list)
 
+        self.formprefixcache = s_cache.LruDict(PREFIX_CACHE_SIZE)
+
         self._type_pends = collections.defaultdict(list)
         self._modeldef = {
             'ctors': [],
@@ -571,22 +576,6 @@ def __init__(self):
             'doc': 'The time the node was created in the cortex.',
         })
 
-        self.addIface('taxonomy', {
-            'doc': 'Properties common to taxonomies.',
-            'props': (
-                ('title', ('str', {}), {'doc': 'A brief title of the definition.'}),
-                ('summary', ('str', {}), {
-                    'deprecated': True,
-                    'doc': 'Deprecated. Please use title/desc.',
-                    'disp': {'hint': 'text'}}),
-                ('desc', ('str', {}), {'doc': 'A definition of the taxonomy entry.', 'disp': {'hint': 'text'}}),
-                ('sort', ('int', {}), {'doc': 'A display sort order for siblings.', }),
-                ('base', ('taxon', {}), {'ro': True, 'doc': 'The base taxon.', }),
-                ('depth', ('int', {}), {'ro': True, 'doc': 'The depth indexed from 0.', }),
-                ('parent', ('$self', {}), {'ro': True, 'doc': 'The taxonomy parent.', }),
-            ),
-        })
-
     def getPropsByType(self, name):
         props = self.propsbytype.get(name, ())
         # TODO order props based on score...
@@ -600,6 +589,64 @@ def getProps(self):
         return [pobj for pname, pobj in self.props.items()
                 if not (isinstance(pname, tuple))]
 
+    def getFormsByPrefix(self, prefix):
+        forms = self.formprefixcache.get(prefix)
+        if forms is not None:
+            return forms
+
+        forms = []
+        for form in self.forms:
+            if form.startswith(prefix):
+                forms.append(form)
+
+        if forms:
+            forms.sort()
+            self.formprefixcache[prefix] = forms
+        return forms
+
+    def reqFormsByPrefix(self, prefix, extra=None):
+        forms = self.getFormsByPrefix(prefix)
+        if not forms:
+            mesg = f'No forms match prefix {prefix}.'
+            exc = s_exc.NoSuchForm(name=prefix, mesg=mesg)
+            if extra is not None:
+                exc = extra(exc)
+            raise exc
+
+        return forms
+
+    def reqFormsByLook(self, name, extra=None):
+        if (form := self.form(name)) is not None:
+            return (form.name,)
+
+        if (forms := self.formsbyiface.get(name)) is not None:
+            return forms
+
+        if name.endswith('*'):
+            return self.reqFormsByPrefix(name[:-1], extra=extra)
+
+        exc = s_exc.NoSuchForm.init(name)
+        if extra is not None:
+            exc = extra(exc)
+
+        raise exc
+
+    def reqPropsByLook(self, name, extra=None):
+        if (forms := self.formsbyiface.get(name)) is not None:
+            return forms
+
+        if (props := self.ifaceprops.get(name)) is not None:
+            return props
+
+        if name.endswith('*'):
+            return self.reqFormsByPrefix(name[:-1], extra=extra)
+
+        exc = s_exc.NoSuchProp.init(name)
+        if extra is not None:
+            exc = extra(exc)
+
+        raise exc
+
     def getTypeClone(self, typedef):
 
         base = self.types.get(typedef[0])
@@ -815,6 +862,8 @@ def addForm(self, formname, forminfo, propdefs):
         for ifname in form.type.info.get('interfaces', ()):
             self._addFormIface(form, ifname)
 
+        self.formprefixcache.clear()
+
         return form
 
     def delForm(self, formname):
@@ -833,11 +882,13 @@ def delForm(self, formname):
             self.arraysbytype[form.type.arraytype.name].remove(form)
 
         for ifname in form.ifaces.keys():
-            self.formsbyiface[ifname].remove(form)
+            self.formsbyiface[ifname].remove(formname)
 
         self.forms.pop(formname, None)
         self.props.pop(formname, None)
 
+        self.formprefixcache.clear()
+
     def addIface(self, name, info):
         # TODO should we add some meta-props here for queries?
         self.ifaces[name] = info
@@ -904,14 +955,18 @@ def _addFormIface(self, form, name):
             mesg = f'Form {form.name} depends on non-existant interface: {name}'
             raise s_exc.NoSuchName(mesg=mesg)
 
+        if iface.get('deprecated'):
+            mesg = f'Form {form.name} depends on deprecated interface {name} which will be removed in 3.0.0'
+            logger.warning(mesg)
+
         for propname, typedef, propinfo in iface.get('props', ()):
             if typedef[0] == '$self':
                 typedef = (form.name, typedef[1])
-            self._addFormProp(form, propname, typedef, propinfo)
+            prop = self._addFormProp(form, propname, typedef, propinfo)
+            self.ifaceprops[f'{name}:{propname}'].append(prop.full)
 
-        # TODO use this to allow storm: +foo:iface
         form.ifaces[name] = iface
-        self.formsbyiface[name].append(form)
+        self.formsbyiface[name].append(form.name)
 
         for ifname in iface.get('interfaces', ()):
             self._addFormIface(form, ifname)

synapse/exc.py

@@ -228,15 +228,15 @@ class NoSuchForm(SynErr):
     @classmethod
     def init(cls, name, mesg=None):
         if mesg is None:
-            mesg = f'No form named {name}'
+            mesg = f'No form named {name}.'
         return NoSuchForm(mesg=mesg, name=name)
 
 class NoSuchProp(SynErr):
 
     @classmethod
     def init(cls, name, mesg=None):
         if mesg is None:
-            mesg = f'No property named {name}'
+            mesg = f'No property named {name}.'
         return NoSuchProp(mesg=mesg, name=name)
 
 class NoSuchAbrv(SynErr): pass