optimize compress speed by muti-thread;

Makefile

@@ -1,13 +1,14 @@
 # args
+MT       := 1
 STATIC_CPP := 0
 # used clang?
-CL  	 := 0
+CL  	   := 0
 # build with -m32?
 M32      := 0
 # build for out min size
 MINS     := 0
 ifeq ($(OS),Windows_NT) # mingw?
-  CC    := gcc
+  CC     := gcc
 endif
 
 
@@ -17,6 +18,13 @@ HDP_OBJ := \
     $(HDP_PATH)/libHDiffPatch/HPatch/patch.o \
     $(HDP_PATH)/file_for_patch.o
 
+ifeq ($(MT),0)
+else
+  HDP_OBJ += \
+    $(HDP_PATH)/libParallel/parallel_import.o \
+    $(HDP_PATH)/libParallel/parallel_channel.o
+endif
+
 TINY_OBJ := \
     decompress/tuz_dec.o \
 	compress/tuz_enc.o \
@@ -27,7 +35,15 @@ TINY_OBJ := \
 	$(HDP_OBJ)
 
 DEF_FLAGS := \
-    -O3 -DNDEBUG -D_LARGEFILE_SOURCE -D_FILE_OFFSET_BITS=64 -D_IS_USED_MULTITHREAD=0
+    -O3 -DNDEBUG -D_LARGEFILE_SOURCE -D_FILE_OFFSET_BITS=64
+
+ifeq ($(MT),0)
+  DEF_FLAGS += -D_IS_USED_MULTITHREAD=0
+else
+  DEF_FLAGS += \
+    -D_IS_USED_MULTITHREAD=1 \
+    -D_IS_USED_CPP11THREAD=1
+endif
 
 ifeq ($(M32),0)
 else
@@ -45,6 +61,10 @@ else
 endif
 
 TINY_LINK  := 
+ifeq ($(MT),0)
+else
+  TINY_LINK += -lpthread	# link pthread
+endif
 ifeq ($(M32),0)
 else
   TINY_LINK += -m32
@@ -64,7 +84,7 @@ else
 endif
 
 CFLAGS   += $(DEF_FLAGS) 
-CXXFLAGS += $(DEF_FLAGS)
+CXXFLAGS += $(DEF_FLAGS) -std=c++11
 
 .PHONY: all install clean
 

README.md

@@ -1,5 +1,5 @@
 # [tinyuz](https://github.com/sisong/tinyuz)
-[![release](https://img.shields.io/badge/release-v0.9.4-blue.svg)](https://github.com/sisong/tinyuz/releases) 
+[![release](https://img.shields.io/badge/release-v1.0.0-blue.svg)](https://github.com/sisong/tinyuz/releases) 
 [![license](https://img.shields.io/badge/license-MIT-blue.svg)](https://github.com/sisong/tinyuz/blob/master/LICENSE) 
 [![PRs Welcome](https://img.shields.io/badge/PRs-welcome-blue.svg)](https://github.com/sisong/tinyuz/pulls)
 [![+issue Welcome](https://img.shields.io/github/issues-raw/sisong/tinyuz?color=green&label=%2Bissue%20welcome)](https://github.com/sisong/tinyuz/issues)   

README_cn.md

@@ -1,5 +1,5 @@
 # [tinyuz](https://github.com/sisong/tinyuz)
-[![release](https://img.shields.io/badge/release-v0.9.4-blue.svg)](https://github.com/sisong/tinyuz/releases) 
+[![release](https://img.shields.io/badge/release-v1.0.0-blue.svg)](https://github.com/sisong/tinyuz/releases) 
 [![license](https://img.shields.io/badge/license-MIT-blue.svg)](https://github.com/sisong/tinyuz/blob/master/LICENSE) 
 [![PRs Welcome](https://img.shields.io/badge/PRs-welcome-blue.svg)](https://github.com/sisong/tinyuz/pulls)
 [![+issue Welcome](https://img.shields.io/github/issues-raw/sisong/tinyuz?color=green&label=%2Bissue%20welcome)](https://github.com/sisong/tinyuz/issues)   

builds/codeblocks/tinyuz.cbp

@@ -39,15 +39,19 @@
 			<Add option="-fexceptions" />
 			<Add option="-D_LARGEFILE_SOURCE" />
 			<Add option="-D_FILE_OFFSET_BITS=64" />
-			<Add option="-D_IS_USED_MULTITHREAD=0" />
 		</Compiler>
+		<Linker>
+			<Add library="pthread" />
+		</Linker>
 		<Unit filename="../../../HDiffPatch/file_for_patch.c">
 			<Option compilerVar="CC" />
 		</Unit>
 		<Unit filename="../../../HDiffPatch/libHDiffPatch/HDiff/private_diff/libdivsufsort/divsufsort.cpp" />
 		<Unit filename="../../../HDiffPatch/libHDiffPatch/HPatch/patch.c">
 			<Option compilerVar="CC" />
 		</Unit>
+		<Unit filename="../../../HDiffPatch/libParallel/parallel_channel.cpp" />
+		<Unit filename="../../../HDiffPatch/libParallel/parallel_import.cpp" />
 		<Unit filename="../../compress/tuz_enc.cpp" />
 		<Unit filename="../../compress/tuz_enc_private/tuz_enc_clip.cpp" />
 		<Unit filename="../../compress/tuz_enc_private/tuz_enc_code.cpp" />

builds/vc/libtinyuz.vcxproj

@@ -21,6 +21,8 @@
   <ItemGroup>
     <ClCompile Include="..\..\..\HDiffPatch\libHDiffPatch\HDiff\private_diff\libdivsufsort\divsufsort.cpp" />
     <ClCompile Include="..\..\..\HDiffPatch\libHDiffPatch\HPatch\patch.c" />
+    <ClCompile Include="..\..\..\HDiffPatch\libParallel\parallel_channel.cpp" />
+    <ClCompile Include="..\..\..\HDiffPatch\libParallel\parallel_import.cpp" />
     <ClCompile Include="..\..\compress\tuz_enc.cpp" />
     <ClCompile Include="..\..\compress\tuz_enc_private\tuz_enc_clip.cpp" />
     <ClCompile Include="..\..\compress\tuz_enc_private\tuz_enc_code.cpp" />
@@ -110,7 +112,7 @@
     <ClCompile>
       <WarningLevel>Level3</WarningLevel>
       <Optimization>Disabled</Optimization>
-      <PreprocessorDefinitions>WIN32;_IS_USED_MULTITHREAD=0;_DEBUG;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <PreprocessorDefinitions>WIN32;_DEBUG;%(PreprocessorDefinitions)</PreprocessorDefinitions>
       <PrecompiledHeader>NotUsing</PrecompiledHeader>
       <RuntimeLibrary>MultiThreadedDebug</RuntimeLibrary>
       <PrecompiledHeaderFile>
@@ -126,7 +128,7 @@
     <ClCompile>
       <WarningLevel>Level3</WarningLevel>
       <Optimization>Disabled</Optimization>
-      <PreprocessorDefinitions>WIN32;_IS_USED_MULTITHREAD=0;_DEBUG;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <PreprocessorDefinitions>WIN32;_DEBUG;%(PreprocessorDefinitions)</PreprocessorDefinitions>
       <PrecompiledHeader>NotUsing</PrecompiledHeader>
       <RuntimeLibrary>MultiThreadedDebug</RuntimeLibrary>
       <PrecompiledHeaderFile>
@@ -141,7 +143,7 @@
   <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
     <ClCompile>
       <WarningLevel>Level3</WarningLevel>
-      <PreprocessorDefinitions>WIN32;_IS_USED_MULTITHREAD=0;NDEBUG;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <PreprocessorDefinitions>WIN32;NDEBUG;%(PreprocessorDefinitions)</PreprocessorDefinitions>
       <PrecompiledHeader>NotUsing</PrecompiledHeader>
       <WholeProgramOptimization>true</WholeProgramOptimization>
       <RuntimeLibrary>MultiThreaded</RuntimeLibrary>
@@ -157,7 +159,7 @@
   <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
     <ClCompile>
       <WarningLevel>Level3</WarningLevel>
-      <PreprocessorDefinitions>WIN32;_IS_USED_MULTITHREAD=0;NDEBUG;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <PreprocessorDefinitions>WIN32;NDEBUG;%(PreprocessorDefinitions)</PreprocessorDefinitions>
       <PrecompiledHeader>NotUsing</PrecompiledHeader>
       <WholeProgramOptimization>true</WholeProgramOptimization>
       <RuntimeLibrary>MultiThreaded</RuntimeLibrary>

builds/xcode/libtinyuz.xcodeproj/project.pbxproj

@@ -9,6 +9,8 @@
 /* Begin PBXBuildFile section */
 		0D7FEA82282A676D0029772D /* patch.c in Sources */ = {isa = PBXBuildFile; fileRef = 0D7FEA81282A676D0029772D /* patch.c */; };
 		0D7FEA84282A67830029772D /* divsufsort.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 0D7FEA83282A67830029772D /* divsufsort.cpp */; };
+		0D7FEA86282A67850029772D /* parallel_channel.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 0D7FEA85282A67850029772D /* parallel_channel.cpp */; };
+		0D7FEA88282A67870029772D /* parallel_import.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 0D7FEA87282A67870029772D /* parallel_import.cpp */; };
 		D6F3B7C82427B30E0066B989 /* tuz_enc.cpp in Sources */ = {isa = PBXBuildFile; fileRef = D6F3B7BB2427B30E0066B989 /* tuz_enc.cpp */; };
 		D6F3B7C92427B30E0066B989 /* tuz_enc_types.h in Headers */ = {isa = PBXBuildFile; fileRef = D6F3B7BC2427B30E0066B989 /* tuz_enc_types.h */; };
 		D6F3B7CA2427B30E0066B989 /* tuz_enc.h in Headers */ = {isa = PBXBuildFile; fileRef = D6F3B7BD2427B30E0066B989 /* tuz_enc.h */; };
@@ -30,6 +32,8 @@
 /* Begin PBXFileReference section */
 		0D7FEA81282A676D0029772D /* patch.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; name = patch.c; path = ../../../HDiffPatch/libHDiffPatch/HPatch/patch.c; sourceTree = "<group>"; };
 		0D7FEA83282A67830029772D /* divsufsort.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; name = divsufsort.cpp; path = ../../../HDiffPatch/libHDiffPatch/HDiff/private_diff/libdivsufsort/divsufsort.cpp; sourceTree = "<group>"; };
+		0D7FEA85282A67850029772D /* parallel_channel.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; name = parallel_channel.cpp; path = ../../../HDiffPatch/libParallel/parallel_channel.cpp; sourceTree = "<group>"; };
+		0D7FEA87282A67870029772D /* parallel_import.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; name = parallel_import.cpp; path = ../../../HDiffPatch/libParallel/parallel_import.cpp; sourceTree = "<group>"; };
 		0D7FEA9C282A6BC60029772D /* tinyuz.xcworkspace */ = {isa = PBXFileReference; lastKnownFileType = wrapper.workspace; path = tinyuz.xcworkspace; sourceTree = "<group>"; };
 		D6F3B7B32427B2D20066B989 /* liblibtinyuz.a */ = {isa = PBXFileReference; explicitFileType = archive.ar; includeInIndex = 0; path = liblibtinyuz.a; sourceTree = BUILT_PRODUCTS_DIR; };
 		D6F3B7BB2427B30E0066B989 /* tuz_enc.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = tuz_enc.cpp; sourceTree = "<group>"; };
@@ -124,6 +128,8 @@
 			isa = PBXGroup;
 			children = (
 				0D7FEA83282A67830029772D /* divsufsort.cpp */,
+				0D7FEA85282A67850029772D /* parallel_channel.cpp */,
+				0D7FEA87282A67870029772D /* parallel_import.cpp */,
 				0D7FEA81282A676D0029772D /* patch.c */,
 			);
 			name = HDiffPatch;
@@ -213,6 +219,8 @@
 				D6F3B7D12427B30E0066B989 /* tuz_enc_match.cpp in Sources */,
 				D6F3B7CD2427B30E0066B989 /* tuz_enc_code.cpp in Sources */,
 				0D7FEA84282A67830029772D /* divsufsort.cpp in Sources */,
+				0D7FEA86282A67850029772D /* parallel_channel.cpp in Sources */,
+				0D7FEA88282A67870029772D /* parallel_import.cpp in Sources */,
 				D6F3B7C82427B30E0066B989 /* tuz_enc.cpp in Sources */,
 			);
 			runOnlyForDeploymentPostprocessing = 0;
@@ -226,6 +234,7 @@
 				ALWAYS_SEARCH_USER_PATHS = NO;
 				CLANG_ANALYZER_NONNULL = YES;
 				CLANG_ANALYZER_NUMBER_OBJECT_CONVERSION = YES_AGGRESSIVE;
+				CLANG_CXX_LANGUAGE_STANDARD = "c++0x";
 				CLANG_CXX_LIBRARY = "libc++";
 				CLANG_ENABLE_MODULES = YES;
 				CLANG_ENABLE_OBJC_ARC = YES;
@@ -264,7 +273,6 @@
 					"DEBUG=1",
 					"_LARGEFILE_SOURCE",
 					"_FILE_OFFSET_BITS=64",
-					"_IS_USED_MULTITHREAD=0",
 					"$(inherited)",
 				);
 				GCC_WARN_64_TO_32_BIT_CONVERSION = YES;
@@ -288,6 +296,7 @@
 				ALWAYS_SEARCH_USER_PATHS = NO;
 				CLANG_ANALYZER_NONNULL = YES;
 				CLANG_ANALYZER_NUMBER_OBJECT_CONVERSION = YES_AGGRESSIVE;
+				CLANG_CXX_LANGUAGE_STANDARD = "c++0x";
 				CLANG_CXX_LIBRARY = "libc++";
 				CLANG_ENABLE_MODULES = YES;
 				CLANG_ENABLE_OBJC_ARC = YES;
@@ -324,7 +333,6 @@
 					"NDEBUG",
 					"_LARGEFILE_SOURCE",
 					"_FILE_OFFSET_BITS=64",
-					"_IS_USED_MULTITHREAD=0",
 					"$(inherited)",
 				);
 				GCC_WARN_64_TO_32_BIT_CONVERSION = YES;

builds/xcode/speedTest.xcodeproj/project.pbxproj

@@ -142,7 +142,7 @@
 				ALWAYS_SEARCH_USER_PATHS = NO;
 				CLANG_ANALYZER_NONNULL = YES;
 				CLANG_ANALYZER_NUMBER_OBJECT_CONVERSION = YES_AGGRESSIVE;
-				CLANG_CXX_LANGUAGE_STANDARD = "gnu++14";
+				CLANG_CXX_LANGUAGE_STANDARD = "c++0x";
 				CLANG_CXX_LIBRARY = "libc++";
 				CLANG_ENABLE_MODULES = YES;
 				CLANG_ENABLE_OBJC_ARC = YES;
@@ -174,7 +174,6 @@
 				DEBUG_INFORMATION_FORMAT = dwarf;
 				ENABLE_STRICT_OBJC_MSGSEND = YES;
 				ENABLE_TESTABILITY = YES;
-				GCC_C_LANGUAGE_STANDARD = gnu11;
 				GCC_DYNAMIC_NO_PIC = NO;
 				GCC_NO_COMMON_BLOCKS = YES;
 				GCC_OPTIMIZATION_LEVEL = 0;
@@ -203,7 +202,7 @@
 				ALWAYS_SEARCH_USER_PATHS = NO;
 				CLANG_ANALYZER_NONNULL = YES;
 				CLANG_ANALYZER_NUMBER_OBJECT_CONVERSION = YES_AGGRESSIVE;
-				CLANG_CXX_LANGUAGE_STANDARD = "gnu++14";
+				CLANG_CXX_LANGUAGE_STANDARD = "c++0x";
 				CLANG_CXX_LIBRARY = "libc++";
 				CLANG_ENABLE_MODULES = YES;
 				CLANG_ENABLE_OBJC_ARC = YES;
@@ -235,7 +234,6 @@
 				DEBUG_INFORMATION_FORMAT = "dwarf-with-dsym";
 				ENABLE_NS_ASSERTIONS = NO;
 				ENABLE_STRICT_OBJC_MSGSEND = YES;
-				GCC_C_LANGUAGE_STANDARD = gnu11;
 				GCC_NO_COMMON_BLOCKS = YES;
 				GCC_WARN_64_TO_32_BIT_CONVERSION = YES;
 				GCC_WARN_ABOUT_RETURN_TYPE = YES_ERROR;

builds/xcode/tinyuz.xcodeproj/project.pbxproj

@@ -140,7 +140,7 @@
 				ALWAYS_SEARCH_USER_PATHS = NO;
 				CLANG_ANALYZER_NONNULL = YES;
 				CLANG_ANALYZER_NUMBER_OBJECT_CONVERSION = YES_AGGRESSIVE;
-				CLANG_CXX_LANGUAGE_STANDARD = "gnu++17";
+				CLANG_CXX_LANGUAGE_STANDARD = "c++0x";
 				CLANG_CXX_LIBRARY = "libc++";
 				CLANG_ENABLE_MODULES = YES;
 				CLANG_ENABLE_OBJC_ARC = YES;
@@ -171,7 +171,6 @@
 				DEBUG_INFORMATION_FORMAT = dwarf;
 				ENABLE_STRICT_OBJC_MSGSEND = YES;
 				ENABLE_TESTABILITY = YES;
-				GCC_C_LANGUAGE_STANDARD = gnu11;
 				GCC_DYNAMIC_NO_PIC = NO;
 				GCC_NO_COMMON_BLOCKS = YES;
 				GCC_OPTIMIZATION_LEVEL = 0;
@@ -199,7 +198,7 @@
 				ALWAYS_SEARCH_USER_PATHS = NO;
 				CLANG_ANALYZER_NONNULL = YES;
 				CLANG_ANALYZER_NUMBER_OBJECT_CONVERSION = YES_AGGRESSIVE;
-				CLANG_CXX_LANGUAGE_STANDARD = "gnu++17";
+				CLANG_CXX_LANGUAGE_STANDARD = "c++0x";
 				CLANG_CXX_LIBRARY = "libc++";
 				CLANG_ENABLE_MODULES = YES;
 				CLANG_ENABLE_OBJC_ARC = YES;
@@ -232,7 +231,6 @@
 				DEPLOYMENT_POSTPROCESSING = YES;
 				ENABLE_NS_ASSERTIONS = NO;
 				ENABLE_STRICT_OBJC_MSGSEND = YES;
-				GCC_C_LANGUAGE_STANDARD = gnu11;
 				GCC_NO_COMMON_BLOCKS = YES;
 				GCC_PREPROCESSOR_DEFINITIONS = NDEBUG;
 				GCC_WARN_64_TO_32_BIT_CONVERSION = YES;
@@ -241,6 +239,7 @@
 				GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE;
 				GCC_WARN_UNUSED_FUNCTION = YES;
 				GCC_WARN_UNUSED_VARIABLE = YES;
+				LLVM_LTO = YES;
 				MACOSX_DEPLOYMENT_TARGET = 10.9;
 				MTL_ENABLE_DEBUG_INFO = NO;
 				MTL_FAST_MATH = YES;

builds/xcode/unitTest.xcodeproj/project.pbxproj

@@ -138,7 +138,7 @@
 				ALWAYS_SEARCH_USER_PATHS = NO;
 				CLANG_ANALYZER_NONNULL = YES;
 				CLANG_ANALYZER_NUMBER_OBJECT_CONVERSION = YES_AGGRESSIVE;
-				CLANG_CXX_LANGUAGE_STANDARD = "gnu++14";
+				CLANG_CXX_LANGUAGE_STANDARD = "c++0x";
 				CLANG_CXX_LIBRARY = "libc++";
 				CLANG_ENABLE_MODULES = YES;
 				CLANG_ENABLE_OBJC_ARC = YES;
@@ -170,7 +170,6 @@
 				DEBUG_INFORMATION_FORMAT = dwarf;
 				ENABLE_STRICT_OBJC_MSGSEND = YES;
 				ENABLE_TESTABILITY = YES;
-				GCC_C_LANGUAGE_STANDARD = gnu11;
 				GCC_DYNAMIC_NO_PIC = NO;
 				GCC_NO_COMMON_BLOCKS = YES;
 				GCC_OPTIMIZATION_LEVEL = 0;
@@ -199,7 +198,7 @@
 				ALWAYS_SEARCH_USER_PATHS = NO;
 				CLANG_ANALYZER_NONNULL = YES;
 				CLANG_ANALYZER_NUMBER_OBJECT_CONVERSION = YES_AGGRESSIVE;
-				CLANG_CXX_LANGUAGE_STANDARD = "gnu++14";
+				CLANG_CXX_LANGUAGE_STANDARD = "c++0x";
 				CLANG_CXX_LIBRARY = "libc++";
 				CLANG_ENABLE_MODULES = YES;
 				CLANG_ENABLE_OBJC_ARC = YES;
@@ -231,7 +230,6 @@
 				DEBUG_INFORMATION_FORMAT = "dwarf-with-dsym";
 				ENABLE_NS_ASSERTIONS = NO;
 				ENABLE_STRICT_OBJC_MSGSEND = YES;
-				GCC_C_LANGUAGE_STANDARD = gnu11;
 				GCC_NO_COMMON_BLOCKS = YES;
 				GCC_WARN_64_TO_32_BIT_CONVERSION = YES;
 				GCC_WARN_ABOUT_RETURN_TYPE = YES_ERROR;