Bug missing autoadd (SYN-2114) (SYN-2330)

synapse/lib/snap.py

@@ -601,12 +601,20 @@ async def _getadds(f, p, formnorm, forminfo, doaddnode=True):
                         subprop = self.core.model.prop(fullname)
                         if subprop is None:
                             continue
+                        if subprop.name in p:
+                            # Don't emit multiple EDIT_PROP_SET edits when one will be unconditionally made.
+                            continue
 
                         assert subprop.type.stortype is not None
 
-                        subnorm, subinfo = subprop.type.norm(subvalu)
-
-                        edits.append((s_layer.EDIT_PROP_SET, (subprop.name, subnorm, None, subprop.type.stortype), ()))
+                        subpropform = self.core.model.form(subprop.type.name)
+                        if subpropform:
+                            subnorm, subinfo = subprop.type.norm(subvalu)
+                            psubs = [x async for x in _getadds(subpropform, {}, subnorm, subinfo)]
+                            edits.append((s_layer.EDIT_PROP_SET, (subprop.name, subnorm, None, subprop.type.stortype), psubs))
+                        else:
+                            subnorm, _ = subprop.type.norm(subvalu)
+                            edits.append((s_layer.EDIT_PROP_SET, (subprop.name, subnorm, None, subprop.type.stortype), ()))
 
                 propform = self.core.model.form(prop.type.name)
                 if propform is not None:

synapse/tests/test_lib_stormlib_stix.py

@@ -51,7 +51,6 @@ def reqValidStix(self, item):
         resp = s_stix.validateStix(item)
         success = resp.get('ok')
         if not success:
-            print(resp)
             self.true(success)
 
     async def test_stormlib_libstix(self, conf=None):

synapse/tests/test_model_dns.py

@@ -1,7 +1,6 @@
 import synapse.common as s_common
 
 import synapse.tests.utils as s_t_utils
-from synapse.tests.utils import alist
 
 class DnsModelTest(s_t_utils.SynTest):
 
@@ -65,7 +64,10 @@ async def test_model_dns_request(self):
                     'server': 'udp://5.6.7.8:53',
                     'reply:code': 0,
                 }
-
+                expected_nodes = (
+                    ('inet:server', 'udp://5.6.7.8:53'),
+                    ('inet:fqdn', 'vertex.link'),
+                )
                 node = await snap.addNode('inet:dns:request', '*', props)
                 req_ndef = node.ndef
                 self.eq(node.get('time'), 1514764800000)
@@ -76,7 +78,7 @@ async def test_model_dns_request(self):
                 self.eq(node.get('query:name:fqdn'), 'vertex.link')
                 self.eq(node.get('query:type'), 255)
                 self.none(node.get('query:client'))
-                self.nn(await snap.getNodeByNdef(('inet:server', 'udp://5.6.7.8:53')))
+                await self.checkNodes(core, expected_nodes)
 
                 # Ensure some remaining inet:dns:query:name:* props are broken out
                 node = await snap.addNode('inet:dns:request', '*', {'query:name': '4.3.2.1.in-addr.arpa'})
@@ -138,6 +140,14 @@ async def test_model_dns_request(self):
                 self.eq(node.get('exe'), f'guid:{"a" * 32}')
                 self.eq(node.get('query:name'), 'notac2.someone.com')
 
+            nodes = await core.nodes('[inet:dns:request=(test,) :query:name="::ffff:8.7.6.5"]')
+            self.len(1, nodes)
+            expected_nodes = (
+                ('inet:ipv4', 0x08070605),
+                ('inet:ipv6', '::ffff:8.7.6.5'),
+            )
+            await self.checkNodes(core, expected_nodes)
+
             # DNS queries can be quite complex or awkward since the protocol
             # allows for nearly anything to be asked about. This can lead to
             # pivots with non-normable data.

synapse/tests/test_model_economic.py

@@ -104,6 +104,7 @@ async def test_model_econ(self):
             self.eq(perc.ndef[1], acqu.get('purchase'))
 
             self.len(1, await core.nodes('econ:acquired:item:form=inet:fqdn'))
+            self.len(1, await core.nodes('inet:fqdn=vertex.link'))
 
             self.eq(('inet:fqdn', 'vertex.link'), acqu.get('item'))
 

synapse/tests/test_model_files.py

@@ -201,6 +201,14 @@ async def test_model_file_types(self):
                 self.eq(node.get('path:base'), 'sup.exe')
                 self.eq(node.get('path:base:ext'), 'exe')
 
+            expected_nodes = (
+                ('file:path', 'c:/www/woah/really'),
+                ('file:path', 'c:/www/woah'),
+                ('file:path', 'c:/www'),
+                ('file:base', 'sup.exe'),
+            )
+            await self.checkNodes(core, expected_nodes)
+
     async def test_model_file_ismime(self):
 
         async with self.getTestCore() as core:

synapse/tests/test_model_inet.py

@@ -143,18 +143,23 @@ async def test_asn(self):
     async def test_asnet4(self):
         formname = 'inet:asnet4'
         async with self.getTestCore() as core:
+            valu = ('54959', ('1.2.3.4', '5.6.7.8'))
+            expected_ndef = (formname, (54959, (16909060, 84281096)))
+            expected_props = {
+                'net4:min': 16909060,
+                'net4': (16909060, 84281096),
+                'net4:max': 84281096,
+                'asn': 54959,
+            }
+            expected_nodes = (
+                ('inet:ipv4', 16909060),
+                ('inet:ipv4', 84281096),
+            )
             async with await core.snap() as snap:
 
-                valu = ('54959', ('1.2.3.4', '5.6.7.8'))
-                expected_ndef = (formname, (54959, (16909060, 84281096)))
-                expected_props = {
-                    'net4:min': 16909060,
-                    'net4': (16909060, 84281096),
-                    'net4:max': 84281096,
-                    'asn': 54959,
-                }
                 node = await snap.addNode(formname, valu)
                 self.checkNode(node, (expected_ndef, expected_props))
+            await self.checkNodes(core, expected_nodes)
 
     async def test_asnet6(self):
         async with self.getTestCore() as core:
@@ -165,6 +170,11 @@ async def test_asnet6(self):
             self.eq(('ff::', 'ff::100'), nodes[0].get('net6'))
             self.eq('ff::', nodes[0].get('net6:min'))
             self.eq('ff::100', nodes[0].get('net6:max'))
+            expected_nodes = (
+                ('inet:ipv6', 'ff::'),
+                ('inet:ipv6', 'ff::100'),
+            )
+            await self.checkNodes(core, expected_nodes)
 
     async def test_cidr4(self):
         formname = 'inet:cidr4'
@@ -1330,18 +1340,23 @@ async def test_url_mirror(self):
 
     async def test_urlredir(self):
         formname = 'inet:urlredir'
-        valu = ('https://vertex.link/idk', 'https://cool.vertex.link:443/something_else')
+        valu = ('https://vertex.link/idk', 'https://cool.vertex.newp:443/something_else')
         expected_props = {
             'src': 'https://vertex.link/idk',
             'src:fqdn': 'vertex.link',
-            'dst': 'https://cool.vertex.link:443/something_else',
-            'dst:fqdn': 'cool.vertex.link',
+            'dst': 'https://cool.vertex.newp:443/something_else',
+            'dst:fqdn': 'cool.vertex.newp',
         }
         expected_ndef = (formname, valu)
+        expected_nodes = (
+            ('inet:fqdn', 'vertex.link'),
+            ('inet:fqdn', 'cool.vertex.newp'),
+        )
         async with self.getTestCore() as core:
             async with await core.snap() as snap:
                 node = await snap.addNode(formname, valu)
                 self.checkNode(node, (expected_ndef, expected_props))
+            await self.checkNodes(core, expected_nodes)
 
     async def test_user(self):
         formname = 'inet:user'
@@ -1452,6 +1467,19 @@ async def test_web_action(self):
                 node = await snap.addNode(formname, valu, props=input_props)
                 self.checkNode(node, (expected_ndef, expected_props))
 
+            nodes = await core.nodes('inet:fqdn')
+            self.len(2, nodes)
+
+            expected_nodes = (
+                ('inet:ipv4', 0x08070605),
+                ('inet:ipv6', '::ffff:8.7.6.5'),
+                ('inet:fqdn', 'newp.com'),
+                ('inet:user', 'hehe'),
+            )
+            nodes = await core.nodes('[inet:web:action=(test,) :acct:user=hehe :acct:site=newp.com :client="tcp://::ffff:8.7.6.5"]')
+            self.len(1, nodes)
+            await self.checkNodes(core, expected_nodes)
+
     async def test_web_chprofile(self):
         formname = 'inet:web:chprofile'
         valu = 32 * 'a'
@@ -1477,6 +1505,15 @@ async def test_web_chprofile(self):
                 node = await snap.addNode(formname, valu, props=input_props)
                 self.checkNode(node, (expected_ndef, expected_props))
 
+            expected_nodes = (
+                ('inet:ipv4', 0x08070605),
+                ('inet:ipv6', '::ffff:8.7.6.5'),
+                ('inet:user', 'hehe'),
+            )
+            self.len(1, await core.nodes(
+                '[inet:web:chprofile=(test,) :acct:user=hehe :acct:site=newp.com :client="tcp://::ffff:8.7.6.5"]'))
+            await self.checkNodes(core, expected_nodes)
+
     async def test_web_file(self):
         formname = 'inet:web:file'
         valu = (('vertex.link', 'vertexmc'), 64 * 'f')
@@ -1495,11 +1532,18 @@ async def test_web_file(self):
             'client': 'tcp://::1',
             'client:ipv6': '::1'
         }
+        expected_nodes = (
+            ('inet:ipv6', '::1'),
+            ('inet:fqdn', 'vertex.link'),
+            ('file:bytes', 'sha256:' + 64 * 'f'),
+            ('inet:user', 'vertexmc'),
+        )
         expected_ndef = (formname, (valu[0], 'sha256:' + valu[1]))
         async with self.getTestCore() as core:
             async with await core.snap() as snap:
                 node = await snap.addNode(formname, valu, props=input_props)
                 self.checkNode(node, (expected_ndef, expected_props))
+            await self.checkNodes(core, expected_nodes)
 
     async def test_web_follows(self):
         formname = 'inet:web:follows'
@@ -1692,6 +1736,11 @@ async def test_web_post(self):
             'client:ipv6': '::1',
         }
 
+        expected_nodes = (
+            ('inet:fqdn', 'vertex.link'),
+            ('inet:group', 'ninjas'),
+        )
+
         expected_ndef = (formname, valu)
         async with self.getTestCore() as core:
             async with await core.snap() as snap:
@@ -1702,6 +1751,8 @@ async def test_web_post(self):
                 self.checkNode(node, (('inet:web:post', node2), expected2))
                 self.len(2, await core.nodes('inet:web:post -> inet:web:hashtag'))
 
+                await self.checkNodes(core, expected_nodes)
+
     async def test_whois_contact(self):
         formname = 'inet:whois:contact'
         valu = (('vertex.link', '@2015'), 'regiStrar')
@@ -1738,10 +1789,14 @@ async def test_whois_contact(self):
             'whois:fqdn': 'vertex.link'
         }
         expected_ndef = (formname, (('vertex.link', 1420070400000), 'registrar'))
+        expected_nodes = (
+            ('inet:fqdn', 'vertex.link'),
+        )
         async with self.getTestCore() as core:
             async with await core.snap() as snap:
                 node = await snap.addNode(formname, valu, props=input_props)
                 self.checkNode(node, (expected_ndef, expected_props))
+            await self.checkNodes(core, expected_nodes)
 
     async def test_whois_rar(self):
         formname = 'inet:whois:rar'
@@ -1982,8 +2037,20 @@ async def test_banner(self):
                 self.eq(443, node.get('server:port'))
                 self.eq(0x01020304, node.get('server:ipv4'))
 
-                strn = await snap.getNodeByNdef(('it:dev:str', 'Hi There'))
-                self.nn(strn)
+            expected_nodes = (
+                ('it:dev:str', 'Hi There'),
+                ('inet:ipv4', 0x01020304),
+            )
+            await self.checkNodes(core, expected_nodes)
+
+            node = await core.nodes('[inet:banner=("tcp://::ffff:8.7.6.5", sup)]')
+            self.len(1, node)
+            expected_nodes = (
+                ('it:dev:str', 'sup'),
+                ('inet:ipv4', 0x08070605),
+                ('inet:ipv6', '::ffff:8.7.6.5'),
+            )
+            await self.checkNodes(core, expected_nodes)
 
     async def test_search_query(self):
         async with self.getTestCore() as core:

synapse/tests/test_model_infotech.py

@@ -425,6 +425,7 @@ async def test_it_forms_prodsoft(self):
                 self.eq(node.get('sig'), (prod1, 'Bar.BAZ.faZ'.lower()))
                 self.eq(node.get('sig:name'), 'bar.baz.faz')
                 self.eq(node.get('sig:soft'), prod1)
+                await self.checkNodes(core, (('it:prod:soft', prod1),))
 
                 # Test 'vers' semver brute forcing
                 testvectors = [

synapse/tests/test_model_telco.py

@@ -77,6 +77,8 @@ async def test_telco_simple(self):
                 self.eq(node.get('latlong'), (0.0, 0.0))
                 self.eq(node.get('place'), place)
 
+                await self.checkNodes(core, (('tel:mob:mcc', '001'),))
+
                 # tel:mob:telem
                 guid = s_common.guid()
                 host = s_common.guid()

synapse/tests/utils.py

@@ -774,6 +774,11 @@ def checkNode(self, node, expected):
         if diff:
             logger.warning('form(%s): untested properties: %s', node.form.name, diff)
 
+    async def checkNodes(self, core, ndefs):
+        for ndef in ndefs:
+            node = await core.nodes('', opts={'ndefs': (ndef,)})
+            self.len(1, node)
+
     def worker(func, *args, **kwargs):
         '''
         Fire a worker thread to run the given func(*args,**kwargs)