Skip to content

Commit

Permalink
Bug missing autoadd (SYN-2114) (#2273)
Browse files Browse the repository at this point in the history 
* Allow us to skip subprop EDIT_PROP_SET nodeedits when we have the sub of sub already in p

* Add subedits to EDIT_PROP_SET edits when making subs of subs
  • Loading branch information
@vEpiphyte
vEpiphyte committed Jul 13, 2021
1 parent a857825 commit bbcebd4
Show file tree
Hide file tree
Showing 9 changed files with 121 additions and 20 deletions.
14 changes: 11 additions & 3 deletions synapse/lib/snap.py
View file
Expand Up @@ -601,12 +601,20 @@ async def _getadds(f, p, formnorm, forminfo, doaddnode=True):
subprop = self.core.model.prop(fullname)
if subprop is None:
continue
if subprop.name in p:
# Don't emit multiple EDIT_PROP_SET edits when one will be unconditionally made.
continue

assert subprop.type.stortype is not None

subnorm, subinfo = subprop.type.norm(subvalu)

edits.append((s_layer.EDIT_PROP_SET, (subprop.name, subnorm, None, subprop.type.stortype), ()))
subpropform = self.core.model.form(subprop.type.name)
if subpropform:
subnorm, subinfo = subprop.type.norm(subvalu)
psubs = [x async for x in _getadds(subpropform, {}, subnorm, subinfo)]
edits.append((s_layer.EDIT_PROP_SET, (subprop.name, subnorm, None, subprop.type.stortype), psubs))
else:
subnorm, _ = subprop.type.norm(subvalu)
edits.append((s_layer.EDIT_PROP_SET, (subprop.name, subnorm, None, subprop.type.stortype), ()))

propform = self.core.model.form(prop.type.name)
if propform is not None:
Expand Down
1 change: 0 additions & 1 deletion synapse/tests/test_lib_stormlib_stix.py
View file
Expand Up @@ -51,7 +51,6 @@ def reqValidStix(self, item):
resp = s_stix.validateStix(item)
success = resp.get('ok')
if not success:
print(resp)
self.true(success)

async def test_stormlib_libstix(self, conf=None):
Expand Down
16 changes: 13 additions & 3 deletions synapse/tests/test_model_dns.py
View file
@@ -1,7 +1,6 @@
import synapse.common as s_common

import synapse.tests.utils as s_t_utils
from synapse.tests.utils import alist

class DnsModelTest(s_t_utils.SynTest):

Expand Down Expand Up @@ -65,7 +64,10 @@ async def test_model_dns_request(self):
'server': 'udp://5.6.7.8:53',
'reply:code': 0,
}

expected_nodes = (
('inet:server', 'udp://5.6.7.8:53'),
('inet:fqdn', 'vertex.link'),
)
node = await snap.addNode('inet:dns:request', '*', props)
req_ndef = node.ndef
self.eq(node.get('time'), 1514764800000)
Expand All @@ -76,7 +78,7 @@ async def test_model_dns_request(self):
self.eq(node.get('query:name:fqdn'), 'vertex.link')
self.eq(node.get('query:type'), 255)
self.none(node.get('query:client'))
self.nn(await snap.getNodeByNdef(('inet:server', 'udp://5.6.7.8:53')))
await self.checkNodes(core, expected_nodes)

# Ensure some remaining inet:dns:query:name:* props are broken out
node = await snap.addNode('inet:dns:request', '*', {'query:name': '4.3.2.1.in-addr.arpa'})
Expand Down Expand Up @@ -138,6 +140,14 @@ async def test_model_dns_request(self):
self.eq(node.get('exe'), f'guid:{"a" * 32}')
self.eq(node.get('query:name'), 'notac2.someone.com')

nodes = await core.nodes('[inet:dns:request=(test,) :query:name="::ffff:8.7.6.5"]')
self.len(1, nodes)
expected_nodes = (
('inet:ipv4', 0x08070605),
('inet:ipv6', '::ffff:8.7.6.5'),
)
await self.checkNodes(core, expected_nodes)

# DNS queries can be quite complex or awkward since the protocol
# allows for nearly anything to be asked about. This can lead to
# pivots with non-normable data.
Expand Down
1 change: 1 addition & 0 deletions synapse/tests/test_model_economic.py
View file
Expand Up @@ -104,6 +104,7 @@ async def test_model_econ(self):
self.eq(perc.ndef[1], acqu.get('purchase'))

self.len(1, await core.nodes('econ:acquired:item:form=inet:fqdn'))
self.len(1, await core.nodes('inet:fqdn=vertex.link'))

self.eq(('inet:fqdn', 'vertex.link'), acqu.get('item'))

Expand Down
8 changes: 8 additions & 0 deletions synapse/tests/test_model_files.py
View file
Expand Up @@ -201,6 +201,14 @@ async def test_model_file_types(self):
self.eq(node.get('path:base'), 'sup.exe')
self.eq(node.get('path:base:ext'), 'exe')

expected_nodes = (
('file:path', 'c:/www/woah/really'),
('file:path', 'c:/www/woah'),
('file:path', 'c:/www'),
('file:base', 'sup.exe'),
)
await self.checkNodes(core, expected_nodes)

async def test_model_file_ismime(self):

async with self.getTestCore() as core:
Expand Down
93 changes: 80 additions & 13 deletions synapse/tests/test_model_inet.py
View file
Expand Up @@ -143,18 +143,23 @@ async def test_asn(self):
async def test_asnet4(self):
formname = 'inet:asnet4'
async with self.getTestCore() as core:
valu = ('54959', ('1.2.3.4', '5.6.7.8'))
expected_ndef = (formname, (54959, (16909060, 84281096)))
expected_props = {
'net4:min': 16909060,
'net4': (16909060, 84281096),
'net4:max': 84281096,
'asn': 54959,
}
expected_nodes = (
('inet:ipv4', 16909060),
('inet:ipv4', 84281096),
)
async with await core.snap() as snap:

valu = ('54959', ('1.2.3.4', '5.6.7.8'))
expected_ndef = (formname, (54959, (16909060, 84281096)))
expected_props = {
'net4:min': 16909060,
'net4': (16909060, 84281096),
'net4:max': 84281096,
'asn': 54959,
}
node = await snap.addNode(formname, valu)
self.checkNode(node, (expected_ndef, expected_props))
await self.checkNodes(core, expected_nodes)

async def test_asnet6(self):
async with self.getTestCore() as core:
Expand All @@ -165,6 +170,11 @@ async def test_asnet6(self):
self.eq(('ff::', 'ff::100'), nodes[0].get('net6'))
self.eq('ff::', nodes[0].get('net6:min'))
self.eq('ff::100', nodes[0].get('net6:max'))
expected_nodes = (
('inet:ipv6', 'ff::'),
('inet:ipv6', 'ff::100'),
)
await self.checkNodes(core, expected_nodes)

async def test_cidr4(self):
formname = 'inet:cidr4'
Expand Down Expand Up @@ -1330,18 +1340,23 @@ async def test_url_mirror(self):

async def test_urlredir(self):
formname = 'inet:urlredir'
valu = ('https://vertex.link/idk', 'https://cool.vertex.link:443/something_else')
valu = ('https://vertex.link/idk', 'https://cool.vertex.newp:443/something_else')
expected_props = {
'src': 'https://vertex.link/idk',
'src:fqdn': 'vertex.link',
'dst': 'https://cool.vertex.link:443/something_else',
'dst:fqdn': 'cool.vertex.link',
'dst': 'https://cool.vertex.newp:443/something_else',
'dst:fqdn': 'cool.vertex.newp',
}
expected_ndef = (formname, valu)
expected_nodes = (
('inet:fqdn', 'vertex.link'),
('inet:fqdn', 'cool.vertex.newp'),
)
async with self.getTestCore() as core:
async with await core.snap() as snap:
node = await snap.addNode(formname, valu)
self.checkNode(node, (expected_ndef, expected_props))
await self.checkNodes(core, expected_nodes)

async def test_user(self):
formname = 'inet:user'
Expand Down Expand Up @@ -1452,6 +1467,19 @@ async def test_web_action(self):
node = await snap.addNode(formname, valu, props=input_props)
self.checkNode(node, (expected_ndef, expected_props))

nodes = await core.nodes('inet:fqdn')
self.len(2, nodes)

expected_nodes = (
('inet:ipv4', 0x08070605),
('inet:ipv6', '::ffff:8.7.6.5'),
('inet:fqdn', 'newp.com'),
('inet:user', 'hehe'),
)
nodes = await core.nodes('[inet:web:action=(test,) :acct:user=hehe :acct:site=newp.com :client="tcp://::ffff:8.7.6.5"]')
self.len(1, nodes)
await self.checkNodes(core, expected_nodes)

async def test_web_chprofile(self):
formname = 'inet:web:chprofile'
valu = 32 * 'a'
Expand All @@ -1477,6 +1505,15 @@ async def test_web_chprofile(self):
node = await snap.addNode(formname, valu, props=input_props)
self.checkNode(node, (expected_ndef, expected_props))

expected_nodes = (
('inet:ipv4', 0x08070605),
('inet:ipv6', '::ffff:8.7.6.5'),
('inet:user', 'hehe'),
)
self.len(1, await core.nodes(
'[inet:web:chprofile=(test,) :acct:user=hehe :acct:site=newp.com :client="tcp://::ffff:8.7.6.5"]'))
await self.checkNodes(core, expected_nodes)

async def test_web_file(self):
formname = 'inet:web:file'
valu = (('vertex.link', 'vertexmc'), 64 * 'f')
Expand All @@ -1495,11 +1532,18 @@ async def test_web_file(self):
'client': 'tcp://::1',
'client:ipv6': '::1'
}
expected_nodes = (
('inet:ipv6', '::1'),
('inet:fqdn', 'vertex.link'),
('file:bytes', 'sha256:' + 64 * 'f'),
('inet:user', 'vertexmc'),
)
expected_ndef = (formname, (valu[0], 'sha256:' + valu[1]))
async with self.getTestCore() as core:
async with await core.snap() as snap:
node = await snap.addNode(formname, valu, props=input_props)
self.checkNode(node, (expected_ndef, expected_props))
await self.checkNodes(core, expected_nodes)

async def test_web_follows(self):
formname = 'inet:web:follows'
Expand Down Expand Up @@ -1692,6 +1736,11 @@ async def test_web_post(self):
'client:ipv6': '::1',
}

expected_nodes = (
('inet:fqdn', 'vertex.link'),
('inet:group', 'ninjas'),
)

expected_ndef = (formname, valu)
async with self.getTestCore() as core:
async with await core.snap() as snap:
Expand All @@ -1702,6 +1751,8 @@ async def test_web_post(self):
self.checkNode(node, (('inet:web:post', node2), expected2))
self.len(2, await core.nodes('inet:web:post -> inet:web:hashtag'))

await self.checkNodes(core, expected_nodes)

async def test_whois_contact(self):
formname = 'inet:whois:contact'
valu = (('vertex.link', '@2015'), 'regiStrar')
Expand Down Expand Up @@ -1738,10 +1789,14 @@ async def test_whois_contact(self):
'whois:fqdn': 'vertex.link'
}
expected_ndef = (formname, (('vertex.link', 1420070400000), 'registrar'))
expected_nodes = (
('inet:fqdn', 'vertex.link'),
)
async with self.getTestCore() as core:
async with await core.snap() as snap:
node = await snap.addNode(formname, valu, props=input_props)
self.checkNode(node, (expected_ndef, expected_props))
await self.checkNodes(core, expected_nodes)

async def test_whois_rar(self):
formname = 'inet:whois:rar'
Expand Down Expand Up @@ -1982,8 +2037,20 @@ async def test_banner(self):
self.eq(443, node.get('server:port'))
self.eq(0x01020304, node.get('server:ipv4'))

strn = await snap.getNodeByNdef(('it:dev:str', 'Hi There'))
self.nn(strn)
expected_nodes = (
('it:dev:str', 'Hi There'),
('inet:ipv4', 0x01020304),
)
await self.checkNodes(core, expected_nodes)

node = await core.nodes('[inet:banner=("tcp://::ffff:8.7.6.5", sup)]')
self.len(1, node)
expected_nodes = (
('it:dev:str', 'sup'),
('inet:ipv4', 0x08070605),
('inet:ipv6', '::ffff:8.7.6.5'),
)
await self.checkNodes(core, expected_nodes)

async def test_search_query(self):
async with self.getTestCore() as core:
Expand Down
1 change: 1 addition & 0 deletions synapse/tests/test_model_infotech.py
View file
Expand Up @@ -425,6 +425,7 @@ async def test_it_forms_prodsoft(self):
self.eq(node.get('sig'), (prod1, 'Bar.BAZ.faZ'.lower()))
self.eq(node.get('sig:name'), 'bar.baz.faz')
self.eq(node.get('sig:soft'), prod1)
await self.checkNodes(core, (('it:prod:soft', prod1),))

# Test 'vers' semver brute forcing
testvectors = [
Expand Down
2 changes: 2 additions & 0 deletions synapse/tests/test_model_telco.py
View file
Expand Up @@ -77,6 +77,8 @@ async def test_telco_simple(self):
self.eq(node.get('latlong'), (0.0, 0.0))
self.eq(node.get('place'), place)

await self.checkNodes(core, (('tel:mob:mcc', '001'),))

# tel:mob:telem
guid = s_common.guid()
host = s_common.guid()
Expand Down
5 changes: 5 additions & 0 deletions synapse/tests/utils.py
View file
Expand Up @@ -774,6 +774,11 @@ def checkNode(self, node, expected):
if diff:
logger.warning('form(%s): untested properties: %s', node.form.name, diff)

async def checkNodes(self, core, ndefs):
for ndef in ndefs:
node = await core.nodes('', opts={'ndefs': (ndef,)})
self.len(1, node)

def worker(func, *args, **kwargs):
'''
Fire a worker thread to run the given func(*args,**kwargs)
Expand Down

0 comments on commit bbcebd4

Please sign in to comment.