Permalink
Browse files

[core] Increase upload chunks from 64KB to 64MB

Make it configurable
  • Loading branch information...
romainr committed Sep 19, 2012
1 parent 4fadd79 commit 2b44f592c9e564716b87547bea9a15454be1b7c9
@@ -59,6 +59,12 @@ def f():
root=os.path.join(os.path.dirname(__file__), '..', '..', 'credentials-merger', 'java-lib')),
private=True)
+UPLOAD_CHUNK_SIZE = Config(
+ key="upload_chunk_size",
+ help="Size, in bytes, of the 'chunks' Django should store into memory and feed into the handler. Default is 64MB.",
+ type=int,
+ default=1024 * 1024 * 64)
+
HDFS_CLUSTERS = UnspecifiedConfigSection(
"hdfs_clusters",
@@ -32,6 +32,7 @@
from django.core.files.uploadhandler import \
FileUploadHandler, StopFutureHandlers, StopUpload
import hadoop.cluster
+from hadoop.conf import UPLOAD_CHUNK_SIZE
UPLOAD_SUBDIR = 'hue-uploads'
LOG = logging.getLogger(__name__)
@@ -121,6 +122,8 @@ def __init__(self, request):
self._file = None
self._starttime = 0
self._activated = False
+ # Need to directly modify FileUploadHandler.chunk_size
+ FileUploadHandler.chunk_size = UPLOAD_CHUNK_SIZE.get()
def new_file(self, field_name, file_name, *args, **kwargs):
# Detect "HDFS" in the field name.
@@ -32,6 +32,7 @@
from hadoop.fs.hadoopfs import Hdfs
from hadoop.fs.exceptions import WebHdfsException
from hadoop.fs.webhdfs_types import WebHdfsStat, WebHdfsContentSummary
+from hadoop.conf import UPLOAD_CHUNK_SIZE
import hadoop.conf
@@ -420,11 +421,10 @@ def copyfile(self, src, dst):
if self.isdir(dst):
raise IOError(errno.INVAL, "Copy dst '%s' is a directory" % (dst,))
- CHUNK_SIZE = 65536
offset = 0
while True:
- data = self.read(src, offset, CHUNK_SIZE)
+ data = self.read(src, offset, UPLOAD_CHUNK_SIZE.get())
if offset == 0:
self.create(dst,
overwrite=True,

0 comments on commit 2b44f59

Please sign in to comment.