Permalink
Browse files

file.h: limit maximum amount of file I/O by 1MB

The problem was observed when I tried to run
bdelta.exe --all-in-ram on 32-bit windows on
network-mounted files.

fread(size=170MB) failed with 'out of memory' there.

I think it is a result of network-attached drives is
implemented in userspace or calling process which leads
to massive memory overhead when reading/writing large
chunks of data.

Fixed it by limiting I/O on 1MB size. It should I/O patterns
slightly better for fuse-mounted linux filesystems as well.

Signed-off-by: Sergei Trofimovich <slyfox@gentoo.org>
  • Loading branch information...
1 parent f199354 commit 98945619d020e38586ae240cad13dfb3fe226a7e @trofi trofi committed Nov 1, 2012
Showing with 40 additions and 6 deletions.
  1. +40 −6 src/file.h
View
@@ -13,14 +13,48 @@
* Author: John Whitney <jjw@deltup.org>
*/
-void fread_fixed(FILE *f, void *buf, unsigned num_bytes) {
- if (fread(buf, 1, num_bytes, f) != num_bytes)
- throw "File read error.";
+#include <stdio.h>
+
+#define MAX_IO_BLOCK_SIZE (1024 * 1024)
+
+void fread_fixed(FILE *f, void * _buf, unsigned num_bytes) {
+ char * buf = (char *)_buf;
+
+ while (num_bytes != 0)
+ {
+ unsigned block_size = num_bytes;
+ if (block_size > MAX_IO_BLOCK_SIZE) block_size = MAX_IO_BLOCK_SIZE;
+
+ size_t r = fread(buf, 1, block_size, f);
+ if (r != block_size)
+ {
+ static char read_error_message[128];
+ sprintf (read_error_message, "read error: fread_fixed(block_size=%u) != %u", block_size, (unsigned)r);
+ throw read_error_message;
+ }
+ buf += block_size;
+ num_bytes -= block_size;
+ }
}
-void fwrite_fixed(FILE *f, const void *buf, unsigned num_bytes) {
- if (fwrite(buf, 1, num_bytes, f) != num_bytes)
- throw "File write error.";
+void fwrite_fixed(FILE *f, const void * _buf, unsigned num_bytes) {
+ char * buf = (char *)_buf;
+
+ while (num_bytes != 0)
+ {
+ unsigned block_size = num_bytes;
+ if (block_size > MAX_IO_BLOCK_SIZE) block_size = MAX_IO_BLOCK_SIZE;
+
+ size_t r = fwrite(buf, 1, block_size, f);
+ if (r != block_size)
+ {
+ static char write_error_message[128];
+ sprintf (write_error_message, "write error: fwrite_fixed(num_bytes=%u) != %u", block_size, (unsigned)r);
+ throw write_error_message;
+ }
+ buf += block_size;
+ num_bytes -= block_size;
+ }
}
unsigned read_word(FILE *f) {

0 comments on commit 9894561

Please sign in to comment.