added bunch of code for predownloading

CMakeLists.txt

@@ -14,7 +14,7 @@ if(APPLE)
 endif()
 
 
-include_directories(src contrib contrib/epee/include "${CMAKE_BINARY_DIR}/version")
+include_directories(src contrib contrib/epee/include "${CMAKE_BINARY_DIR}/version" "${CMAKE_BINARY_DIR}/contrib/zlib")
 add_definitions(-DSTATICLIB)
 
 set(TESTNET FALSE CACHE BOOL "Compile for testnet")
@@ -30,6 +30,7 @@ if (UNIX AND NOT APPLE)
   find_package(Threads REQUIRED)
 endif()
 
+set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DHTTP_ENABLE_GZIP")
 
 if(MSVC)
   add_definitions("/bigobj /MP /W3 /GS- /D_CRT_SECURE_NO_WARNINGS /wd4996 /wd4267 /wd4345 /D_WIN32_WINNT=0x0600 /DWIN32_LEAN_AND_MEAN /DGTEST_HAS_TR1_TUPLE=0 /FIinline_c.h /D__SSE4_1__")

contrib/CMakeLists.txt

@@ -4,9 +4,11 @@ set(UPNPC_BUILD_TESTS OFF CACHE BOOL "Build test executables")
 
 add_subdirectory(miniupnpc)
 add_subdirectory(db)
+add_subdirectory(zlib)
 
 set_property(TARGET upnpc-static PROPERTY FOLDER "contrib")
 set_property(TARGET lmdb PROPERTY FOLDER "contrib")
+set_property(TARGET zlibstatic PROPERTY FOLDER "contrib")
 
 if(MSVC)
   set_property(TARGET upnpc-static APPEND_STRING PROPERTY COMPILE_FLAGS " -wd4244 -wd4267")

contrib/epee/include/gzip_encoding.h

@@ -49,7 +49,7 @@ namespace net_utils
 		*
 		*/
 		inline 
-		content_encoding_gzip(i_target_handler* powner_filter, bool is_deflate_mode = false):m_powner_filter(powner_filter), 
+      content_encoding_gzip(i_target_handler* powner_filter, bool is_deflate_mode = false, int compression_level = Z_DEFAULT_COMPRESSION) :m_powner_filter(powner_filter),
 			m_is_stream_ended(false), 
 			m_is_deflate_mode(is_deflate_mode),
 			m_is_first_update_in(true)
@@ -60,11 +60,11 @@ namespace net_utils
 			if(is_deflate_mode)
 			{
 				ret = inflateInit(&m_zstream_in);	
-				ret = deflateInit(&m_zstream_out, Z_DEFAULT_COMPRESSION);
+        ret = deflateInit(&m_zstream_out, compression_level);
 			}else
 			{
 				ret = inflateInit2(&m_zstream_in, 0x1F);
-				ret = deflateInit2(&m_zstream_out, Z_DEFAULT_COMPRESSION, Z_DEFLATED, 0x1F, 8, Z_DEFAULT_STRATEGY);
+        ret = deflateInit2(&m_zstream_out, compression_level, Z_DEFLATED, 0x1F, 8, Z_DEFAULT_STRATEGY);
 			}	
 		}
 		/*! \brief
@@ -219,6 +219,144 @@ namespace net_utils
 		*/
 		bool		m_is_first_update_in;
 	};
+
+
+  struct abstract_callback_base
+  {
+    virtual bool do_call(const std::string& piece_of_transfer) = 0;
+  };
+
+  template <typename callback_t>
+  struct abstract_callback : public abstract_callback_base
+  {
+    callback_t m_cb;
+
+    abstract_callback(callback_t cb) : m_cb(cb){}
+    virtual bool do_call(const std::string& piece_of_transfer)
+    {
+      return m_cb(piece_of_transfer);
+    }
+  };
+
+  class gzip_lambda_handler : public content_encoding_gzip,
+                              public i_target_handler
+  {
+    std::shared_ptr<abstract_callback_base> m_pcb;
+
+    virtual bool handle_target_data(std::string& piece_of_transfer)
+    {
+      bool r = m_pcb->do_call(piece_of_transfer);
+      piece_of_transfer.clear();
+      return r;
+    }
+  public: 
+    gzip_lambda_handler() : content_encoding_gzip(this, true, Z_BEST_COMPRESSION)
+    {}
+
+    template<class callback_t>
+    bool update_in(std::string& piece_of_transfer, callback_t cb)
+    {
+      m_pcb.reset(new abstract_callback<callback_t>(cb));
+      return content_encoding_gzip::update_in(piece_of_transfer);
+    }
+    template<class callback_t>
+    bool stop(callback_t cb)
+    {return true;}
+  };
+
+
+
+  class gzip_encoder_lyambda
+  {
+    bool m_initialized;
+    z_stream  m_zstream;
+  public: 
+
+
+    gzip_encoder_lyambda(int compression_level = Z_DEFAULT_COMPRESSION) :m_initialized(false), m_zstream(AUTO_VAL_INIT(m_zstream))
+    {
+      int ret = deflateInit(&m_zstream, compression_level);
+      if (ret == Z_OK)
+        m_initialized = true;
+    }
+    ~gzip_encoder_lyambda()
+    {
+      deflateEnd(&m_zstream);
+    }
+    template<typename callback_t>
+    bool update_in(const std::string& target, callback_t cb)
+    {
+      if (!m_initialized)
+      {        
+        return false;
+      }
+
+      if (!target.size())
+      {
+        return true;
+      }
+
+      std::string result_packed_buff;
+      //theoretically it supposed to be smaller
+      result_packed_buff.resize(target.size(), 'X');
+      while (true)
+      {
+        m_zstream.next_in = (Bytef*)target.data();
+        m_zstream.avail_in = (uInt)target.size();
+        m_zstream.next_out = (Bytef*)result_packed_buff.data();
+        m_zstream.avail_out = (uInt)result_packed_buff.size();
+
+        int ret = deflate(&m_zstream, Z_NO_FLUSH);
+        CHECK_AND_ASSERT_MES(ret >= 0, false, "Failed to deflate. err = " << ret);
+        if (m_zstream.avail_out == 0)
+        {
+          //twice bigger buffer
+          result_packed_buff.resize(result_packed_buff.size()*2);
+          continue;
+        }        
+        if (result_packed_buff.size() != m_zstream.avail_out)
+          result_packed_buff.resize(result_packed_buff.size() - m_zstream.avail_out);
+        break;
+      }
+
+      return cb(result_packed_buff);
+    }
+    template<typename callback_t>
+    bool stop(callback_t cb)
+    {
+      if (!m_initialized)
+      {
+        return false;
+      }
+
+      std::string result_packed_buff;
+      //theoretically it supposed to be smaller
+      result_packed_buff.resize(1000000, 'X');
+      while (true)
+      {
+        m_zstream.next_in =  nullptr;
+        m_zstream.avail_in = 0;
+        m_zstream.next_out = (Bytef*)result_packed_buff.data();
+        m_zstream.avail_out = (uInt)result_packed_buff.size();
+
+        int ret = deflate(&m_zstream, Z_FINISH);
+        CHECK_AND_ASSERT_MES(ret >= 0, false, "Failed to deflate at finish. err = " << ret);
+        if (ret != Z_STREAM_END)
+        {
+          //twice bigger buffer
+          result_packed_buff.resize(result_packed_buff.size() * 2);
+          continue;
+        }
+        if (result_packed_buff.size() != m_zstream.avail_out)
+          result_packed_buff.resize(result_packed_buff.size() - m_zstream.avail_out);
+        m_initialized = false;
+        break;
+      }
+
+      return cb(result_packed_buff);      
+    }
+
+  };
 }
 }
 

contrib/epee/include/md5_l.h

@@ -77,7 +77,7 @@ namespace md5
 	static void hmac_md5(const unsigned char* text, int text_len, const unsigned char* key, int key_len, unsigned char *digest);
 
 
-	inline bool md5( unsigned char *input, int ilen, unsigned char output[16] )
+	inline bool md5(const unsigned char *input, int ilen, unsigned char output[16] )
 	{
 		MD5_CTX ctx;
 

contrib/epee/include/net/http_client.h

@@ -62,42 +62,6 @@ namespace epee
 
     using namespace std;
 
-    /*struct url
-    {
-    public:
-    void parse(const std::string& url_s)
-    {
-    const string prot_end("://");
-    string::const_iterator prot_i = search(url_s.begin(), url_s.end(),
-    prot_end.begin(), prot_end.end());
-    protocol_.reserve(distance(url_s.begin(), prot_i));
-    transform(url_s.begin(), prot_i,
-    back_inserter(protocol_),
-    ptr_fun<int,int>(tolower)); // protocol is icase
-    if( prot_i == url_s.end() )
-    return;
-    advance(prot_i, prot_end.length());
-    string::const_iterator path_i = find(prot_i, url_s.end(), '/');
-    host_.reserve(distance(prot_i, path_i));
-    transform(prot_i, path_i,
-    back_inserter(host_),
-    ptr_fun<int,int>(tolower)); // host is icase
-    string::const_iterator query_i = find(path_i, url_s.end(), '?');
-    path_.assign(path_i, query_i);
-    if( query_i != url_s.end() )
-    ++query_i;
-    query_.assign(query_i, url_s.end());
-    }
-
-    std::string protocol_;
-    std::string host_;
-    std::string path_;
-    std::string query_;
-    };*/
-
-
-
-
     //---------------------------------------------------------------------------
     static inline const char* get_hex_vals()
     {
@@ -315,6 +279,10 @@ namespace epee
           req_buff += uri + " HTTP/1.1\r\n" +
             "Host: " + m_host_buff + "\r\n" + "Content-Length: " + boost::lexical_cast<std::string>(body.size()) + "\r\n";
 
+#ifdef HTTP_ENABLE_GZIP
+          req_buff += "Accept-Encoding: gzip, deflate\r\n";
+#endif
+
 
           //handle "additional_params"
           for (fields_list::const_iterator it = additional_params.begin(); it != additional_params.end(); it++)
@@ -957,6 +925,33 @@ namespace epee
           }
           auto local_cb = [&](const std::string& piece_of_data, uint64_t total_bytes, uint64_t received_bytes)
           {
+            fs.write(piece_of_data.data(), piece_of_data.size());
+            return cb(total_bytes, received_bytes);
+          };
+          bool r = this->invoke_cb(local_cb, url, timeout, method, body, additional_params);
+          fs.close();
+          return r;
+        }
+
+        //
+        template<typename callback_t>
+        bool download_and_unzip(callback_t cb, const std::string& path_for_file, const std::string& url, uint64_t timeout, const std::string& method = "GET", const std::string& body = std::string(), const fields_list& additional_params = fields_list())
+        {
+          std::ofstream fs;
+          fs.open(path_for_file, std::ios::binary | std::ios::out | std::ios::trunc);
+          if (!fs.is_open())
+          {
+            LOG_ERROR("Fsiled to open " << path_for_file);
+            return false;
+          }
+          std::string buff;
+          content_encoding_gzip zip_decoder(nullptr, true);
+          auto local_cb = [&](const std::string& piece_of_data, uint64_t total_bytes, uint64_t received_bytes)
+          {
+            buff += piece_of_data;
+            zip_decoder.update_in(buff);
+
+
             fs.write(piece_of_data.data(), piece_of_data.size());
             return cb(total_bytes, received_bytes);
           };

contrib/epee/include/zlib_helper.h

@@ -43,7 +43,6 @@ namespace zlib_helper
 		int ret = deflateInit(&zstream, Z_DEFAULT_COMPRESSION);
 		if(target.size())
 		{
-
 
 			result_packed_buff.resize(target.size()*2, 'X');