Implemented socks proxy support for storm/axon HTTP API

requirements.txt

@@ -6,6 +6,7 @@ tornado>=6.0.3,<7.0.0
 regex>=2020.5.14
 PyYAML>=5.2,<6.0
 aiohttp>=3.6.3,<4.0
+aiohttp-socks>=0.5.5,<0.6.0
 prompt-toolkit>=3.0.4,<3.1.0
 lark-parser>=0.9.0,<0.11.0
 Pygments>=2.6.0,<2.7.0

setup.py

@@ -55,6 +55,7 @@ def run(self):
         'regex>=2020.5.14',
         'PyYAML>=5.2,<6.0',
         'aiohttp>=3.6.3,<4.0',
+        'aiohttp-socks>=0.5.5,<0.6.0',
         'prompt-toolkit>=3.0.4,<3.1.0',
         'lark-parser>=0.9.0,<0.11.0',
         'Pygments>=2.6.0,<2.7.0',

synapse/axon.py

@@ -1,9 +1,11 @@
-import aiohttp
 import asyncio
 import hashlib
 import logging
 import tempfile
 
+import aiohttp
+import aiohttp_socks
+
 import synapse.exc as s_exc
 import synapse.common as s_common
 
@@ -236,7 +238,12 @@ class Axon(s_cell.Cell):
             'type': 'integer',
             'minimum': 1,
             'hidecmdl': True,
-        }
+        },
+        'http:proxy': {
+            'description': 'An aiohttp-socks compatible proxy URL to use in the wget API.',
+            'type': 'string',
+            'hidecmdl': True,
+        },
     }
 
     async def __anit__(self, dirn, conf=None):  # type: ignore
@@ -378,7 +385,12 @@ async def wget(self, url, params=None, headers=None, json=None, body=None, metho
         '''
         Stream a file download directly into the axon.
         '''
-        async with aiohttp.ClientSession() as sess:
+        connector = None
+        proxyurl = self.conf.get('http:proxy')
+        if proxyurl is not None:
+            connector = aiohttp_socks.ProxyConnector.from_url(proxyurl)
+
+        async with aiohttp.ClientSession(connector=connector) as sess:
 
             try:
 

synapse/cortex.py

@@ -834,6 +834,11 @@ class Cortex(s_cell.Cell):  # type: ignore
             'description': 'Logging log level to emit storm logs at.',
             'type': 'integer'
         },
+        'http:proxy': {
+            'description': 'An aiohttp-socks compatible proxy URL to use storm HTTP API.',
+            'type': 'string',
+            'hidecmdl': True,
+        },
     }
 
     cellapi = CoreApi
@@ -2091,7 +2096,13 @@ async def _initCoreAxon(self):
         turl = self.conf.get('axon')
         if turl is None:
             path = os.path.join(self.dirn, 'axon')
-            self.axon = await s_axon.Axon.anit(path)
+            conf = {}
+
+            proxyurl = self.conf.get('http:proxy')
+            if proxyurl is not None:
+                conf['http:proxy'] = proxyurl
+
+            self.axon = await s_axon.Axon.anit(path, conf=conf)
             self.axon.onfini(self.axready.clear)
             self.dynitems['axon'] = self.axon
             self.axready.set()

synapse/lib/stormhttp.py

@@ -1,8 +1,11 @@
 import json
+import asyncio
 
 import aiohttp
+import aiohttp_socks
 
 import synapse.exc as s_exc
+import synapse.common as s_common
 
 import synapse.lib.stormtypes as s_stormtypes
 
@@ -99,15 +102,24 @@ async def _httpRequest(self, meth, url, headers=None, json=None, body=None, ssl_
         if params:
             kwargs['params'] = params
 
-        async with aiohttp.ClientSession() as sess:
+        todo = s_common.todo('getConfOpt', 'http:proxy')
+        proxyurl = await self.runt.dyncall('cortex', todo)
+
+        connector = None
+        if proxyurl is not None:
+            connector = aiohttp_socks.ProxyConnector.from_url(proxyurl)
+
+        async with aiohttp.ClientSession(connector=connector) as sess:
             try:
                 async with sess.request(meth, url, headers=headers, json=json, data=body, **kwargs) as resp:
                     info = {
                         'code': resp.status,
                         'body': await resp.content.read()
                     }
                     return HttpResp(info)
-            except (TypeError, ValueError) as e:
+            except asyncio.CancelledError:  # pragma: no cover
+                raise
+            except Exception as e:
                 mesg = f'Error during http {meth} - {str(e)}'
                 raise s_exc.StormRuntimeError(mesg=mesg, headers=headers, json=json, body=body, params=params) from None
 

synapse/tests/test_axon.py

@@ -386,3 +386,9 @@ async def test_axon_wget(self):
 
                 resp = await proxy.wget(f'http://visi:secret@127.0.0.1:{port}/api/v1/axon/files/by/sha256/{sha2}')
                 self.false(resp['ok'])
+
+        conf = {'http:proxy': 'socks5://user:pass@127.0.0.1:1'}
+        async with self.getTestAxon(conf=conf) as axon:
+            async with axon.getLocalProxy() as proxy:
+                resp = await proxy.wget('http://vertex.link')
+                self.ne(-1, resp['mesg'].find('Can not connect to proxy 127.0.0.1:1'))

synapse/tests/test_lib_stormhttp.py

@@ -1,3 +1,4 @@
+import synapse.exc as s_exc
 import synapse.tests.utils as s_test
 
 class StormHttpTest(s_test.SynTest):
@@ -157,3 +158,11 @@ async def test_storm_http_post_file(self):
             err = errs[0]
             self.eq(err[0], 'StormRuntimeError')
             self.isin('Error during http POST - data and json parameters can not be used at the same time', err[1].get('mesg'))
+
+    async def test_storm_http_proxy(self):
+        conf = {'http:proxy': 'socks5://user:pass@127.0.0.1:1'}
+        async with self.getTestCore(conf=conf) as core:
+            resp = await core.callStorm('return($lib.axon.wget("http://vertex.link"))')
+            self.ne(-1, resp['mesg'].find('Can not connect to proxy 127.0.0.1:1'))
+            with self.raises(s_exc.StormRuntimeError):
+                await core.callStorm('return($lib.inet.http.get("http://vertex.link"))')