Merge pull request #86 from till-varoquaux/zstr_fix

Zstr fix

configure.in

@@ -3,7 +3,7 @@ dnl
 dnl Process this file with autoconf to produce a configure script.
 dnl
 
-AC_INIT([okws], [3.1.32.0])
+AC_INIT([okws], [3.1.33.0])
 AM_CONFIG_HEADER([config.h])
 AC_CONFIG_AUX_DIR([.])
 AM_INIT_AUTOMAKE([subdir-objects])

libaok/ok.T

@@ -316,7 +316,6 @@ oksrvc_t::init (int argc, char *argv[])
     t->lookup ("gzip", &gztmp);
     t->lookup ("filtercgi", &ok_filter_cgi);
     t->lookup ("gziplev", &ok_gzip_compress_level);
-    t->lookup ("gzipnlev", &ok_gzip_naive_compress_level);
     t->lookup ("gzipcsl", &ok_gzip_cache_storelimit);
     t->lookup ("logtick", &ok_log_tick);
     t->lookup ("logprd", &ok_log_period);

libpub/okconst.C

@@ -37,8 +37,6 @@ u_int ok_gzip_cache_minstr = 0x0;            // smallest to cache
 u_int ok_gzip_cache_maxstr = 0x10000;        // largest to cache
 u_int ok_gzip_cache_storelimit = 0x1000000;  // 16 M
 u_int ok_gzip_mem_level = 9;                 // zlib max
-int   ok_gzip_naive_compress_level = 7;      // naive gzip compress level
-
 
 //
 // chunking tuning
@@ -416,12 +414,11 @@ ok_gzip_str_to_mode (const str &s, bool *okp)
   bool ok = true;
   if (!s) {
     /* nothing */
-  } else if (s == "1" || s == "on" || s == "smart") {
+  } else if (s == "1" || s == "on" || s == "smart" || s == "naive") {
+    // Naive is deprecated
     m = GZIP_SMART;
   } else if (s == "0" || s == "off") {
     m = GZIP_NONE;
-  } else if (s == "naive") {
-    m = GZIP_NAIVE;
   } else {
     ok = false;
   }

libpub/okconst.h

@@ -55,14 +55,14 @@ enum { XSSFILT_NONE = 0, XSSFILT_SOME = 1, XSSFILT_ALL = 2 };
 #define OK_SVC_FD_HIGH_WAT_UL 102400
 #define OK_SVC_FD_LOW_WAT_UL  100000
 
+// Naive is deprecated....
 typedef enum { GZIP_NONE = 0, GZIP_SMART = 1, GZIP_NAIVE = 2 } gzip_mode_t;
 
 //
 // gzip parameters (see zstr.h)
 //
 extern gzip_mode_t ok_gzip_mode;
 extern int     ok_gzip_compress_level;
-extern int     ok_gzip_naive_compress_level;
 extern u_int   ok_gzip_smallstr;
 extern u_int   ok_gzip_cache_minstr;
 extern u_int   ok_gzip_cache_maxstr;

libpub/pub3xdr.C

@@ -11,7 +11,12 @@ void
 zstr_to_xdr (const zstr &z, xpub3_zstr_t *x, int l)
 {
   x->s = z.to_str ();
-  if (l != Z_DISABLE) x->zs = z.compress (l);
+  if (l != Z_DISABLE && z.len() > 0) {
+      str compressed = z.compress (l);
+      if (compressed) {
+          x->zs = compressed;
+      }
+  }
   x->clev = l;
 }
 

libpub/zstr.C

@@ -33,14 +33,14 @@ int zlev;               // deflation level
 bool zinitted = false;  // protect against stupid bugs
 bool zdebug = false;    // debug messages
 
-static int
-zcompress (char *dest, uLong *dlenp, const char *src, uLong slen, int lev)
+str zcompress (const str &in, int lev)
 {
-  uLong dlen = *dlenp;
+  const char *src = in.cstr();
+  uLong sLen = in.len();
   zm.next_in = const_cast<Bytef *> (reinterpret_cast<const Bytef *> (src));
-  zm.avail_in = slen;
-  zm.next_out = reinterpret_cast<Bytef *> (dest);
-  zm.avail_out = dlen;
+  zm.avail_in = sLen;
+  zm.next_out = nullptr;
+  zm.avail_out = 0;
 
   if (!zinitted) {
      warn << "OKWS says: you forgot to call zinit()! i'm doing it for you\n";
@@ -58,34 +58,25 @@ zcompress (char *dest, uLong *dlenp, const char *src, uLong slen, int lev)
     sfs_profiler::exit_vomit_lib ();
     zlev = lev;
   }
+  //mstr res();
+
+  const uLong dLen = deflateBound(&zm, sLen);
+
+  mstr res(dLen);
+  zm.next_out = reinterpret_cast<Bytef *>(res.cstr());
+  zm.avail_out = dLen;
 
   sfs_profiler::enter_vomit_lib ();
-  int err = deflate (&zm, Z_FULL_FLUSH);
+  const int rc = deflate (&zm, Z_FULL_FLUSH);
   sfs_profiler::exit_vomit_lib ();
-  *dlenp = dlen - zm.avail_out;
-
-  return err;
-}
 
-size_t
-max_compressed_len (size_t in)
-{
-  return ((in * 1001) / 1000) + 16;
-}
-
-str
-zcompress (const str &in, int lev)
-{
-  uLong slen = in.len ();
-  uLong dlen = max_compressed_len (slen);
-  mstr m (dlen);
-  int rc = zcompress (m.cstr (), &dlen, in.cstr (), slen, lev);
   if (rc != Z_OK) {
     warn << "deflate returned error: " << rc << "\n";
-    return NULL;
+    return nullptr;
   }
-  m.setlen (dlen);
-  return m;
+
+  res.setlen(dLen - zm.avail_out);
+  return res;
 }
 
 static bool 
@@ -476,99 +467,6 @@ zbuf::to_zstr_vec (vec<zstr> *zs2)
 
 //-----------------------------------------------------------------------
 
-int
-zbuf::naive_compress (strbuf *b, int lev)
-{
-  if (!zinitted) {
-     warn << "OKWS says: you forgot to call zinit()! i'm doing it for you\n";
-     zinit (false);
-  }
-  assert (zinitted);
-  z_stream z;
-  z.zalloc = Z_NULL;
-  z.zfree = Z_NULL;
-  z.opaque = Z_NULL;
-  int rc = deflateInit2 (&z, lev, Z_DEFLATED, -MAX_WBITS, 
-			 ok_gzip_mem_level, Z_DEFAULT_STRATEGY);
-  if (rc != Z_OK)
-    return rc;
-
-  strbuf2zstr ();
-  size_t inlen = inflated_len ();
-  strbuf tmp;
-
-  // with little extra room for headers and footers
-  size_t outlen = max_compressed_len (inlen) + 64;
-
-  tmp << zhdr;
-
-  uLong crc = ::crc32 (0L, Z_NULL, 0);
-
-  mstr out (outlen);
-  char *outp = out.cstr ();
-  char *endp = out.cstr () + outlen;
-  z.avail_out = outlen;
-  deflateParams (&z, lev, Z_DEFAULT_STRATEGY);
-
-  for (size_t i = 0; rc == 0 && i <= zs.size (); i++) {
-
-    bool end = (i == zs.size ());
-    const char *src;
-    size_t srclen;
-
-    if (end) {
-      src = NULL;
-      srclen = 0;
-    } else {
-      src = zs[i].cstr ();
-      srclen = zs[i].len ();
-    }
-
-    if ((src && srclen) || end) {
-      z.next_out = reinterpret_cast<Bytef *> (outp);
-      size_t avail_out_pre = endp - outp;
-      z.avail_out = avail_out_pre;
-      z.next_in = const_cast<Bytef *> (reinterpret_cast<const Bytef *> (src));
-      z.avail_in = srclen;
-      if (!end) { crc = zs[i].crc32(crc); }
-      int drc = deflate (&z, end ? Z_FINISH : Z_NO_FLUSH);
-      if ((!end && drc != Z_OK) || (end && drc != Z_STREAM_END)) {
-	warn << "Compression error: " << drc << "\n";
-	rc = -1;
-      }
-      outp += (avail_out_pre - z.avail_out);
-    }
-  }
-
-  if (rc == 0) {
-    assert (outp <= endp);
-    out.setlen (outp - out.cstr ());
-    str out_s = out;
-    tmp << out_s;
-    b->hold_onto (out_s);
-
-    size_t trailer_size = 16;
-    mstr trailer (trailer_size);
-    char *p = trailer.cstr ();
-    p += uLong_to_buf (crc, p);
-    p += uLong_to_buf (inlen, p);
-    size_t bytes = p - trailer.cstr ();
-    assert (bytes < trailer_size);
-    trailer.setlen (bytes);
-    str trailer_s = trailer;
-
-    tmp << trailer_s;
-    b->hold_onto (trailer_s);
-    b->take (tmp);
-  }
-
-  deflateEnd (&z);
-
-  return rc;
-}
-
-//-----------------------------------------------------------------------
-
 int 
 zbuf::to_strbuf (strbuf *out, compressible_t::opts_t o)
 {
@@ -580,9 +478,6 @@ zbuf::to_strbuf (strbuf *out, compressible_t::opts_t o)
   case GZIP_SMART:
     compress (out, o);
     break;
-  case GZIP_NAIVE:
-    rc = naive_compress (out, o.lev);
-    break;
   default:
     warn << "unknown gzip mode given\n";
     rc = -1;

libpub/zstr.h

@@ -179,8 +179,6 @@ class zbuf : public compressible_t {
 
   size_t inflated_len () const;
 
-  int naive_compress (strbuf *b, int lev) ;
-
   void clear ();
   const strbuf &output () { output (&out); return out; }
   const strbuf &compress (compressible_t::opts_t o) 

okd/okld.T

@@ -820,8 +820,6 @@ okld_t::parse_file (const str &cf)
     .add ("Gzip", &gzip_tmp)
     .add ("GzipLevel", &ok_gzip_compress_level, Z_NO_COMPRESSION,
 	  Z_BEST_COMPRESSION)
-    .add ("GzipNaiveLevel", &ok_gzip_naive_compress_level, Z_NO_COMPRESSION,
-	  Z_BEST_COMPRESSION)
     .add ("GzipSmallStrLen", &ok_gzip_smallstr, 0, 0x8000)
     .add ("GzipCacheMin", &ok_gzip_cache_minstr, 0, 0x10000)
     .add ("GzipCacheMax", &ok_gzip_cache_maxstr, 0, 0x1000000)
@@ -1110,7 +1108,6 @@ okld_t::encode_env ()
     .insert ("svclog", int (logd_parms.svclog))
     .insert ("gzip", int (ok_gzip_mode))
     .insert ("gziplev", ok_gzip_compress_level)
-    .insert ("gzipnlev", ok_gzip_naive_compress_level)
     .insert ("gzipcsl", ok_gzip_cache_storelimit)
     .insert ("logtick", ok_log_tick)
     .insert ("logprd", ok_log_period)

test/unit/Makefile.am

@@ -93,7 +93,6 @@ noinst_PROGRAMS = \
 	gz \
 	srv \
 	newline \
-	gznaive \
 	json \
 	eztst \
 	dump_rpc_const \
@@ -110,7 +109,6 @@ pubmemtst_SOURCES = pubmemtst.C
 gz_SOURCES = gz.C
 srv_SOURCES = srv.C
 newline_SOURCES = newline.C
-gznaive_SOURCES = gznaive.C
 json_SOURCES = json.C
 eztst_SOURCES = eztst.C
 msgpack_SOURCES = msgpack.C