Browse files

more large memory buffer fixes

  • Loading branch information...
1 parent f0d79db commit b6be1b0416199dd14cb93801aed3aa40a29354e4 @kholtman committed Feb 28, 2012
Showing with 67 additions and 7 deletions.
  1. +18 −5 HISTORY
  2. +33 −1 afio.c
  3. +16 −1 compfile.c
View
23 HISTORY
@@ -948,9 +948,22 @@ variable (causing compiler warning) in outflush() and compressfile().
Created afio home page on the web.
-Fixed bug in handling -s and -c options settings that create a >2GB
-memory buffer, by upgrading various variables to size_t size. [[[Did
-not test this extensively yet.]]] Based on bug report and patch by
-Yuri D'Elia.
-
+Fixed bug in handling buffer size calculations when -s and -c options
+settings that create a 2GB memory buffer, by upgrading various
+variables to size_t size. Based on bug report and patch by Yuri
+D'Elia. Tests have shown that (at least on i386 with 32 bit ints and
+size_t) this fix does not handle correctly buffer sizes >2GB, probably
+because size_t is 4 byte, not 8, and I still have doubts about 2GB
+too. Fixing everything and testing it is just too painful on my
+current 32 bit Linux install on a box that has only like 700M real
+memory.
+
+Therefore: added code to prevent memory buffer sizes larger than 1.5
+GB being used by -s and -c and by -M. For -M, probably a limit of
+3.5GB is still OK but I have not tested it so better safe than sorry.
+On native 64 bit installs, these limits may be over-cautious
+(especially if the compiler uses sizeof(int)==8, which is NOT the case
+for Linux amd64), but as I have no way to test this I'd rather get bug
+reports and patches from 64 bit platform people who run into this
+limit than have their backups silently fail.
View
34 afio.c
@@ -639,13 +639,29 @@ int main (int ac, char **av)
if(!compressprog) compressprog = PRG_COMPRESS;
compress_arg_list[0] = compressprog;
+ if(maxmem/1024L/1024L > (1024+512))
+ {
+ /* afio uses uints and ints and size_t at several places that
+ may make it (on architectures with 32 bit ints and size_t)
+ fail to do the right calculations when the buffer size is
+ >2GB. This 1.5 GB limit is somewhat conservative, but better
+ safe than sorry.
+
+ If you have an os/compile environment witrh 64 bit size_t
+ and int, things might be OK if you disable this test, but
+ I have not tested it.
+ If you disable here see also the code in memwrite().
+ */
+ fatal (arspec, "In-memory compression buffer size above 1.5GB not supported");
+ }
+
if (Fflag)
{
if ((buflen = (off_t) aruntil) == 0)
usage ();
}
else
- buflen = arbsize * group;
+ buflen = (off_t)arbsize * (off_t)group;
if( roundaruntil )
{
@@ -709,6 +725,22 @@ int main (int ac, char **av)
if (gflag && *arname != '/' && *arname != '!')
fatal (arspec, "Relative pathname");
VOID signal (SIGINT, goodbye);
+
+ if(buflen/1024L/1024L > (1024+512))
+ {
+ /* afio uses uints and ints at several places that make it
+ (on architectures with 32 bit ints and size_t) fail to do
+ the right calculations when the buffer size is >2GB.
+ This 1.5GB limit is somewhat conservative, but better
+ safe than sorry.
+
+ If you have an os/compile environment witrh 64 bit size_t
+ and int, things might be OK if you disable this test, but
+ I have not tested it.
+ */
+ fatal (arspec, "In-memory I/O buffer size above 1.5GB not supported");
+ }
+
/*
* +BLOCK is added to make sure we don't overrun buffer on a
* read (internal read(1) length is thus met)
View
17 compfile.c
@@ -101,7 +101,22 @@ void memwrite(char *buf, int count)
/* RWWH: don't increase by linear amount! grow exponentially! */
if (memsize+count>=allocsize) {
allocsize=1.25*(memsize+count);
- membank=(char *)realloc(membank,allocsize);
+#if 0
+ fprintf(stderr,"%d allocsize=%u\n",sizeof(size_t), allocsize);
+#endif
+ if(allocsize/1024L/1024L > 2000)
+ {
+ /* safety: to avoid possible errors due to use of 32 bit int
+ and size_t types here and there, prevent membank going over 2GB if
+ compression programm actually expands filesize.
+ Note that -M is capped at 1.5GB anyway.
+ */
+ membank=NULL; /* simulate realloc running out of memory */
+ }
+ else
+ {
+ membank=(char *)realloc(membank,allocsize);
+ }
if(membank==NULL)
{
warn_nocount("Memory","Low on virtual memory, zipping twice instead.");

0 comments on commit b6be1b0

Please sign in to comment.