splash.resource_timeout attribute; add support for errors in splash l…

…ua attributes

docs/scripting-ref.rst

@@ -1389,6 +1389,31 @@ result in an HTTP header:
 See also: :ref:`splash-set-result-status-code`,
 :ref:`splash-set-result-content-type`.
 
+.. _splash-resource-timeout:
+
+splash.resource_timeout
+-----------------------
+
+Set a default timeout for network requests, in seconds.
+
+**Signature:** ``splash.resource_timeout = value``
+
+Example - abort requests to remote resources if they take more than 10 seconds:
+
+.. code-block:: lua
+
+     function main(splash)
+         splash.resource_timeout = 10.0
+         assert(splash:go(splash.args.url))
+         return splash:png()
+     end
+
+Zero or nil value means "no timeout".
+
+Request timeouts set in :ref:`splash-on-request` using
+``request:set_timeout`` have a priority over :ref:`splash-resource-timeout`.
+
+
 .. _splash-images-enabled:
 
 splash.images_enabled
@@ -1628,7 +1653,7 @@ one of the ``request`` methods:
   See also: :ref:`splash-set-custom-headers`.
 * ``request:set_timeout(timeout)`` - set a timeout for this request,
   in seconds. If response is not fully received after the timeout,
-  request is aborted.
+  request is aborted. See also: :ref:`splash-resource-timeout`.
 
 A callback passed to :ref:`splash-on-request` can't call Splash
 async methods like :ref:`splash-wait` or :ref:`splash-go`.
@@ -1694,12 +1719,18 @@ request to Splash:
         }
     end)
 
-Example 6 - discard requests which take longer than 5 seconds to complete:
+Example 6 - discard requests which take longer than 5 seconds to complete,
+but allow up to 15 seconds for the first request:
 
 .. code-block:: lua
 
+    local first = true
+    splash.resource_timeout = 5
     splash:on_request(function(request)
-        request:set_timeout(5.0)
+        if first then
+            request:set_timeout(15.0)
+            first = false
+        end
     end)
 
 

splash/browser_tab.py

@@ -166,6 +166,10 @@ def set_resource_timeout(self, timeout):
         """ Set a default timeout for HTTP requests, in seconds. """
         self.web_page.resource_timeout = timeout
 
+    def get_resource_timeout(self):
+        """ Get a default timeout for HTTP requests, in seconds. """
+        return self.web_page.resource_timeout
+
     def set_images_enabled(self, enabled):
         self.web_page.settings().setAttribute(QWebSettings.AutoLoadImages,
                                               enabled)

splash/lua_modules/splash.lua

@@ -150,10 +150,10 @@ function Splash._create(py_splash)
 
   -- Set attribute handler functions
   for key, opts in pairs(py_splash.lua_properties) do
-    local func = drops_self_argument(py_splash[key])
-    self._getters[opts.name] = drops_self_argument(
-                                            py_splash[opts.getter_method])
-    self._setters[opts.name] = func
+    local setter = unwraps_errors(drops_self_argument(py_splash[key]))
+    local getter = unwraps_errors(drops_self_argument(py_splash[opts.getter_method]))
+    self._getters[opts.name] = getter
+    self._setters[opts.name] = setter
   end
 
   return self
@@ -234,22 +234,22 @@ function Response._create(py_reply)
     }
 
     setmetatable(self, Response)
-    
+
     -- convert har headers to something more convenient
     local _request_headers = {}
     for name, value in pairs(py_reply.request["headers"]) do
         _request_headers[value["name"]] = value["value"]
     end
     py_reply.request["headers"] = _request_headers
 
-    -- take some keys from py_reply.info 
+    -- take some keys from py_reply.info
     -- but not all (we don't want mess har headers with response headers)
     local keys_from_reply_info = {"status", "url", "ok"}
 
-    for key, value in pairs(keys_from_reply_info) do 
+    for key, value in pairs(keys_from_reply_info) do
         self[value] = py_reply.info[value]
     end
-    
+
     for key, opts in pairs(py_reply.commands) do
         local command = py_reply[key]
         command = drops_self_argument(command)
@@ -259,7 +259,7 @@ function Response._create(py_reply)
         end
         self[key] = command
     end
-    
+
     return self
 end
 

splash/qtrender_lua.py

@@ -312,10 +312,12 @@ def init_dispatcher(self, return_func):
         self._return = return_func
 
     @lua_property('js_enabled')
+    @command()
     def get_js_enabled(self):
         return self.tab.get_js_enabled()
 
     @get_js_enabled.lua_setter
+    @command()
     def set_js_enabled(self, value):
         self.tab.set_js_enabled(value)
 
@@ -611,6 +613,7 @@ def set_viewport_full(self):
         return tuple(self.tab.set_viewport('full'))
 
     @lua_property('images_enabled')
+    @command()
     def get_images_enabled(self):
         return self.tab.get_images_enabled()
 
@@ -620,6 +623,21 @@ def set_images_enabled(self, enabled):
         if enabled is not None:
             self.tab.set_images_enabled(int(enabled))
 
+    @lua_property('resource_timeout')
+    @command()
+    def get_resource_timeout(self):
+        return self.tab.get_resource_timeout()
+
+    @get_resource_timeout.lua_setter
+    @command()
+    def set_resource_timeout(self, timeout):
+        if timeout is None:
+            timeout = 0
+        timeout = float(timeout)
+        if timeout < 0:
+            raise ScriptError("splash.resource_timeout can't be negative")
+        self.tab.set_resource_timeout(timeout)
+
     @command()
     def status_code(self):
         return self.tab.last_http_status()
@@ -764,7 +782,10 @@ def set_header(self, name, value):
     @command()
     @_requires_request
     def set_timeout(self, timeout):
-        self.request.timeout = float(timeout)
+        timeout = float(timeout)
+        if timeout < 0:
+            raise ScriptError("request:set_timeout() argument can't be < 0")
+        self.request.timeout = timeout
 
 
 def _requires_response(meth):

splash/tests/lua_modules/emulation.lua

@@ -38,11 +38,7 @@ function Splash:go_and_wait(args)
   end
 
   -- set a resource timeout
-  if args.resource_timeout ~= nil then
-    self:on_request(function(request)
-      request:set_timeout(args.resource_timeout)
-    end)
-  end
+  self.resource_timeout = args.resource_timeout
 
   local ok, reason = self:go{url=url, baseurl=args.baseurl}
   if not ok then

splash/tests/test_execute.py

@@ -1482,6 +1482,9 @@ def test_set_custom_headers(self):
         self.assertIn("'Value 2'", data["res3"])
         self.assertNotIn("'Value 3'", data["res3"])
 
+
+class ResourceTimeoutTest(BaseLuaRenderTest):
+
     def test_resource_timeout_aborts_first(self):
         resp = self.request_lua("""
         function main(splash)
@@ -1493,6 +1496,67 @@ def test_resource_timeout_aborts_first(self):
         self.assertStatusCode(resp, 200)
         self.assertEqual(resp.json(), {'err': 'render_error'})
 
+    def test_resource_timeout_attribute(self):
+        # request should be cancelled
+        resp = self.request_lua("""
+        function main(splash)
+            splash.resource_timeout = 0.1
+            assert(splash:go(splash.args.url))
+        end
+        """, {"url": self.mockurl("slow.gif?n=4")})
+        self.assertStatusCode(resp, 400)
+        self.assertIn('render_error', resp.json()['message'])
+
+    def test_resource_timeout_attribute_priority(self):
+        # set_timeout should take a priority
+        resp = self.request_lua("""
+        function main(splash)
+            splash.resource_timeout = 0.1
+            splash:on_request(function(req) req:set_timeout(10) end)
+            assert(splash:go(splash.args.url))
+        end
+        """, {"url": self.mockurl("slow.gif?n=4")})
+        self.assertStatusCode(resp, 200)
+
+    def test_resource_timeout_read(self):
+        resp = self.request_lua("""
+        function main(splash)
+            local default = splash.resource_timeout
+            splash.resource_timeout = 0.1
+            local updated = splash.resource_timeout
+            return {default=default, updated=updated}
+        end
+        """)
+        self.assertStatusCode(resp, 200)
+        self.assertEqual(resp.json(), {"default": 0, "updated": 0.1})
+
+    def test_resource_timeout_zero(self):
+        resp = self.request_lua("""
+        function main(splash)
+            splash.resource_timeout = 0
+            assert(splash:go(splash.args.url))
+        end
+        """, {"url": self.mockurl("slow.gif?n=1")})
+        self.assertStatusCode(resp, 200)
+
+        resp = self.request_lua("""
+        function main(splash)
+            splash.resource_timeout = nil
+            assert(splash:go(splash.args.url))
+        end
+        """, {"url": self.mockurl("slow.gif?n=1")})
+        self.assertStatusCode(resp, 200)
+
+    def test_resource_timeout_negative(self):
+        resp = self.request_lua("""
+        function main(splash)
+            splash.resource_timeout = -1
+            assert(splash:go(splash.args.url))
+        end
+        """, {"url": self.mockurl("slow.gif?n=1")})
+        self.assertStatusCode(resp, 400)
+        self.assertIn('splash.resource_timeout', resp.json()['message'])
+
 
 class ResultStatusCodeTest(BaseLuaRenderTest):
     def test_set_result_status_code(self):