Buffer writes

Azure · Oct 18, 2019 · dc0cbc3 · dc0cbc3
1 parent 339b176
commit dc0cbc3
Show file tree

Hide file tree

Showing 2 changed files with 34 additions and 105 deletions.
diff --git a/Readme.md b/Readme.md
@@ -124,55 +124,6 @@ ADLDownloader(adl, '', 'my_temp_dir', 5, 2**24)
 ```
 # API
 
-|
-|
-|
-|
-|
-|
-|
-|
-|
-|
-|
-|
-|
-|
-|
-|
-|
-|
-|
-|
-|
-|
-|
-|
-|
-|
-|
-|
-|
-|
-|
-|
-|
-|
-|
-|
-|
-|
-|
-|
-|
-|
-|
-|
-|
-|
-|
-|
-
 #### class azure.datalake.store.core.AzureDLFileSystem(token=None, per_call_timeout_seconds=60, \*\*kwargs)
 Access Azure DataLake Store as if it were a file-system
 
@@ -211,46 +162,26 @@ Access Azure DataLake Store as if it were a file-system
 
 ### Methods
 
-|
-|
-|
-|
-|
-|
-|
-|
-|
-|
-|
-|
-|
-|
-|
-|
-|
-|
-|
-|
-|
-|
-|
-|
-|
-|
-|
-|
-|
-|
-|
-|
-|
-|
-|
-|
-|
-|
-|
-|
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
 <!-- !! processed by numpydoc !! -->
 
 #### access(self, path, invalidate_cache=True)
@@ -1235,12 +1166,8 @@ of files or a glob pattern.
 
 ### Methods
 
-|
-|
-|
-|
-|
-|
+
+
 <!-- !! processed by numpydoc !! -->
 
 #### active(self)
@@ -1397,12 +1324,9 @@ of files or a glob pattern.
 
 ### Methods
 
-|
-|
-|
-|
-|
-|
+
+
+
 <!-- !! processed by numpydoc !! -->
 
 #### active(self)

diff --git a/azure/datalake/store/core.py b/azure/datalake/store/core.py
@@ -1213,10 +1213,15 @@ def write(self, data):
         if self.closed:
             raise ValueError('I/O operation on closed file.')
 
-        out = self.buffer.write(ensure_writable(data))
-        self.loc += out
-        self.flush(syncFlag='DATA')
-        return out
+        # TODO Flush may be simplified
+        # Buffered writes so a very large buffer is not copied leading to very large memory consumption
+        bytes_written = 0
+        for i in range(0, len(data), self.blocksize):
+            out = self.buffer.write(ensure_writable(data[i:i + self.blocksize]))
+            self.loc += out
+            bytes_written += out
+            self.flush(syncFlag='DATA')
+        return bytes_written
 
     def flush(self, syncFlag='METADATA', force=False):
         """