Test large files in resource proxy

ckan · Nov 25, 2012 · 4676d4d · 4676d4d
1 parent 61326d1
commit 4676d4d
Show file tree

Hide file tree

Showing 3 changed files with 26 additions and 5 deletions.
diff --git a/ckanext/resourceproxy/controller.py b/ckanext/resourceproxy/controller.py
@@ -7,16 +7,16 @@
 
 log = getLogger(__name__)
 
+MAX_FILE_SIZE = 1024 * 1024 * 2  # 2MB
+CHUNK_SIZE = 256
+
 
 def proxy_resource(context, data_dict):
         resource_id = data_dict['resource_id']
         log.info('Proxify resource {id}'.format(id=resource_id))
         resource = logic.get_action('resource_show')(context, {'id': resource_id})
         url = resource['url']
 
-        MAX_FILE_SIZE = 1024 * 1024 * 2  # 2MB
-        CHUNK_SIZE = 256
-
         try:
             r = requests.get(url, prefetch=False)
             r.raise_for_status()

diff --git a/ckanext/resourceproxy/tests/file_server.py b/ckanext/resourceproxy/tests/file_server.py
@@ -7,14 +7,27 @@
 PORT = 50001
 
 
+class StaticHandler(SimpleHTTPServer.SimpleHTTPRequestHandler):
+    def send_head(self):
+        if 'huge.json' in self.path:
+            f = open(self.translate_path(self.path), 'rb')
+            self.send_response(200)
+            self.send_header("Content-type", 'application/json')
+            self.send_header("Content-Length", '1000000000')
+            self.end_headers()
+            return f
+        else:
+            return SimpleHTTPServer.SimpleHTTPRequestHandler.send_head(self)
+
+
 def serve(port=PORT):
     '''Serves static test files over HTTP'''
 
     # Make sure we serve from the tests' static directory
     os.chdir(os.path.join(os.path.dirname(os.path.abspath(__file__)),
                           'static'))
 
-    Handler = SimpleHTTPServer.SimpleHTTPRequestHandler
+    Handler = StaticHandler
 
     class TestServer(SocketServer.TCPServer):
         allow_reuse_address = True

diff --git a/ckanext/resourceproxy/tests/test_proxy.py b/ckanext/resourceproxy/tests/test_proxy.py
@@ -84,6 +84,14 @@ def test_resource_proxy_on_404(self):
         result = self.app.get(proxied_url, status='*')
         assert result.status == 404, result.status
 
+    def test_large_file(self):
+        self.set_resource_url('http://0.0.0.0:50001/huge.json')
+
+        proxied_url = proxy.get_proxified_resource_url(self.data_dict)
+        result = self.app.get(proxied_url, status='*')
+        assert result.status == 500, result.status
+        assert 'too large' in result.body, result.body
+
     def test_resource_proxy_non_existent(self):
         self.set_resource_url('http://foo.bar')
 
@@ -95,4 +103,4 @@ def f1():
         proxied_url = proxy.get_proxified_resource_url(self.data_dict)
         result = self.app.get(proxied_url, status='*')
         assert result.status == 500, result.status
-        assert 'Could not proxy resource' in result.body, result.body
+        assert 'connection error' in result.body, result.body