wget UA

synapse/lib/storm.py

@@ -1074,10 +1074,24 @@ def reqValidPkgdef(pkgdef):
             ('urls', {'nargs': '*', 'help': 'URLs to download.'}),
             ('--no-ssl-verify', {'default': False, 'action': 'store_true', 'help': 'Ignore SSL certificate validation errors.'}),
             ('--timeout', {'default': 300, 'type': 'int', 'help': 'Configure the timeout for the download operation.'}),
+            ('--params', {'default': None, 'help': 'Provide a dict containing url parameters.'}),
+            ('--headers', {
+                'default': {
+                    'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.212 Safari/537.36',
+                    'Accept': '*/*',
+                    'Accept-Encoding': 'gzip, deflate',
+                    'Accept-Language': 'en-US,en;q=0.9',
+                },
+                'help': 'Provide a dict containing custom request headers.'}),
+            ('--no-headers', {'default': False, 'action': 'store_true', 'help': 'Do NOT use any default headers.'}),
         ),
         'storm': '''
         init {
             $count = (0)
+
+            $params = $cmdopts.params
+            $headers = $cmdopts.headers
+            if $cmdopts.no_headers { $headers = $lib.null }
         }
 
         $ssl = (not $cmdopts.no_ssl_verify)
@@ -1095,13 +1109,13 @@ def reqValidPkgdef(pkgdef):
                 $urls = ($node.value(),)
             }
             for $url in $urls {
-                -> { yield $lib.axon.urlfile($url, ssl=$ssl, timeout=$timeout) }
+                -> { yield $lib.axon.urlfile($url, params=$params, headers=$headers, ssl=$ssl, timeout=$timeout) }
             }
         }
 
         if ($count = 0) {
             for $url in $cmdopts.urls {
-                yield $lib.axon.urlfile($url, ssl=$ssl, timeout=$timeout)
+                yield $lib.axon.urlfile($url, params=$params, headers=$headers, ssl=$ssl, timeout=$timeout)
             }
         }
         ''',

synapse/tests/test_lib_storm.py

@@ -1,3 +1,4 @@
+import json
 import asyncio
 import datetime
 import itertools
@@ -553,6 +554,70 @@ async def get(self, name):
                 nodes = [x for x in msgs if x[0] == 'node']
                 self.len(0, nodes)
 
+    async def test_storm_wget(self):
+
+        async def _getRespFromSha(core, mesgs):
+            for m in mesgs:
+                if m[0] == 'node' and m[1][0][0] == 'file:bytes':
+                    node = m[1]
+                    sha = node[1]['props']['sha256']
+
+            buf = b''
+            async for bytz in core.axon.get(s_common.uhex(sha)):
+                buf += bytz
+
+            resp = json.loads(buf.decode('utf8'))
+            return resp
+
+        async with self.getTestCore() as core:
+            addr, port = await core.addHttpsPort(0)
+            root = await core.auth.getUserByName('root')
+            await root.setPasswd('root')
+
+            core.addHttpApi('/api/v0/test', s_t_utils.HttpReflector, {'cell': core})
+            url = f'https://root:root@127.0.0.1:{port}/api/v0/test'
+            opts = {'vars': {'url': url}}
+
+            # Headers as list of tuples, params as dict
+            q = '''
+            $params=$lib.dict(key=valu, foo=bar)
+            $hdr = (
+                    ("User-Agent", "my fav ua"),
+            )|
+            wget $url --headers $hdr --params $params --no-ssl-verify | -> file:bytes $lib.print($node)
+            '''
+
+            mesgs = await alist(core.storm(q, opts=opts))
+
+            resp = await _getRespFromSha(core, mesgs)
+            data = resp.get('result')
+            self.eq(data.get('params'), {'key': ('valu',), 'foo': ('bar',)})
+            self.eq(data.get('headers').get('User-Agent'), 'my fav ua')
+
+            # no default headers(from wget command)
+            q = '''
+            $hdr = (
+                    ("User-Agent", "my fav ua"),
+            )|
+            wget $url --headers $hdr --no-headers --no-ssl-verify | -> file:bytes $lib.print($node)
+            '''
+            mesgs = await alist(core.storm(q, opts=opts))
+
+            resp = await _getRespFromSha(core, mesgs)
+            data = resp.get('result')
+            self.ne(data.get('headers').get('User-Agent'), 'my fav ua')
+
+            # params as list of key/value pairs
+            q = '''
+            $params=((foo, bar), (key, valu))
+            | wget $url --params $params --no-ssl-verify | -> file:bytes $lib.print($node)
+            '''
+            mesgs = await alist(core.storm(q, opts=opts))
+
+            resp = await _getRespFromSha(core, mesgs)
+            data = resp.get('result')
+            self.eq(data.get('params'), {'key': ('valu',), 'foo': ('bar',)})
+
     async def test_storm_vars_fini(self):
 
         async with self.getTestCore() as core: