diff --git a/ckanext/resourceproxy/controller.py b/ckanext/resourceproxy/controller.py index 19ca06254b5..ebc9fdab828 100644 --- a/ckanext/resourceproxy/controller.py +++ b/ckanext/resourceproxy/controller.py @@ -7,6 +7,9 @@ log = getLogger(__name__) +MAX_FILE_SIZE = 1024 * 1024 * 2 # 2MB +CHUNK_SIZE = 256 + def proxy_resource(context, data_dict): resource_id = data_dict['resource_id'] @@ -14,9 +17,6 @@ def proxy_resource(context, data_dict): resource = logic.get_action('resource_show')(context, {'id': resource_id}) url = resource['url'] - MAX_FILE_SIZE = 1024 * 1024 * 2 # 2MB - CHUNK_SIZE = 256 - try: r = requests.get(url, prefetch=False) r.raise_for_status() diff --git a/ckanext/resourceproxy/tests/file_server.py b/ckanext/resourceproxy/tests/file_server.py index af13c896209..368b8a27843 100644 --- a/ckanext/resourceproxy/tests/file_server.py +++ b/ckanext/resourceproxy/tests/file_server.py @@ -7,6 +7,19 @@ PORT = 50001 +class StaticHandler(SimpleHTTPServer.SimpleHTTPRequestHandler): + def send_head(self): + if 'huge.json' in self.path: + f = open(self.translate_path(self.path), 'rb') + self.send_response(200) + self.send_header("Content-type", 'application/json') + self.send_header("Content-Length", '1000000000') + self.end_headers() + return f + else: + return SimpleHTTPServer.SimpleHTTPRequestHandler.send_head(self) + + def serve(port=PORT): '''Serves static test files over HTTP''' @@ -14,7 +27,7 @@ def serve(port=PORT): os.chdir(os.path.join(os.path.dirname(os.path.abspath(__file__)), 'static')) - Handler = SimpleHTTPServer.SimpleHTTPRequestHandler + Handler = StaticHandler class TestServer(SocketServer.TCPServer): allow_reuse_address = True diff --git a/ckanext/resourceproxy/tests/test_proxy.py b/ckanext/resourceproxy/tests/test_proxy.py index 71d3f677247..7d6d647a2b1 100644 --- a/ckanext/resourceproxy/tests/test_proxy.py +++ b/ckanext/resourceproxy/tests/test_proxy.py @@ -84,6 +84,14 @@ def test_resource_proxy_on_404(self): result = self.app.get(proxied_url, status='*') assert result.status == 404, result.status + def test_large_file(self): + self.set_resource_url('http://0.0.0.0:50001/huge.json') + + proxied_url = proxy.get_proxified_resource_url(self.data_dict) + result = self.app.get(proxied_url, status='*') + assert result.status == 500, result.status + assert 'too large' in result.body, result.body + def test_resource_proxy_non_existent(self): self.set_resource_url('http://foo.bar') @@ -95,4 +103,4 @@ def f1(): proxied_url = proxy.get_proxified_resource_url(self.data_dict) result = self.app.get(proxied_url, status='*') assert result.status == 500, result.status - assert 'Could not proxy resource' in result.body, result.body + assert 'connection error' in result.body, result.body