support data compression & delta-encoding of posting lists

.gitmodules

@@ -0,0 +1,4 @@
+[submodule "ThirdParty/zstd"]
+	path = ThirdParty/zstd
+	url = https://github.com/facebook/zstd
+	branch = release

AnnService/CMakeLists.txt

@@ -2,8 +2,10 @@
 # Licensed under the MIT License.
 
 set(AnnService ${PROJECT_SOURCE_DIR}/AnnService)
+set(Zstd ${PROJECT_SOURCE_DIR}/ThirdParty/zstd)
 
 include_directories(${AnnService})
+include_directories(${Zstd}/lib)
 
 file(GLOB_RECURSE HDR_FILES ${AnnService}/inc/Core/*.h  ${AnnService}/inc/Helper/*.h)
 file(GLOB_RECURSE SRC_FILES ${AnnService}/src/Core/*.cpp ${AnnService}/src/Helper/*.cpp)
@@ -32,9 +34,9 @@ if(${CMAKE_CXX_COMPILER_ID} STREQUAL "GNU")
 endif()
 
 add_library (SPTAGLib SHARED ${SRC_FILES} ${HDR_FILES})
-target_link_libraries (SPTAGLib DistanceUtils)
+target_link_libraries (SPTAGLib DistanceUtils libzstd_shared)
 add_library (SPTAGLibStatic STATIC ${SRC_FILES} ${HDR_FILES})
-target_link_libraries (SPTAGLibStatic DistanceUtils)
+target_link_libraries (SPTAGLibStatic DistanceUtils libzstd_static)
 if(${CMAKE_CXX_COMPILER_ID} STREQUAL "GNU")
     target_compile_options(SPTAGLibStatic PRIVATE -fPIC)
 endif()

AnnService/CoreLibrary.vcxproj

@@ -99,7 +99,11 @@
       <DebugInformationFormat>ProgramDatabase</DebugInformationFormat>
       <AdditionalOptions>/Zc:twoPhase- /Zc:__cplusplus %(AdditionalOptions)</AdditionalOptions>
       <LanguageStandard>stdcpp17</LanguageStandard>
+      <AdditionalIncludeDirectories>$(SolutionDir)\vcpkg\installed\x64-windows\include</AdditionalIncludeDirectories>
     </ClCompile>
+    <Lib>
+      <AdditionalDependencies>$(SolutionDir)\vcpkg\installed\x64-windows\lib\zstd.lib</AdditionalDependencies>
+    </Lib>
   </ItemDefinitionGroup>
   <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
     <ClCompile>
@@ -127,12 +131,19 @@
       <OpenMPSupport>true</OpenMPSupport>
       <PreprocessorDefinitions>_MBCS;_SCL_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
       <AdditionalOptions>/Zc:twoPhase- /Zc:__cplusplus %(AdditionalOptions)</AdditionalOptions>
-      <LanguageStandard>stdcpp17</LanguageStandard>
+      <LanguageStandard>Default</LanguageStandard>
+      <AdditionalIncludeDirectories>$(SolutionDir)\vcpkg\installed\x64-windows\include</AdditionalIncludeDirectories>
     </ClCompile>
     <Link>
       <EnableCOMDATFolding>true</EnableCOMDATFolding>
       <OptimizeReferences>true</OptimizeReferences>
     </Link>
+    <Lib>
+      <AdditionalDependencies>$(SolutionDir)\vcpkg\installed\x64-windows\lib\zstd.lib</AdditionalDependencies>
+    </Lib>
+    <PostBuildEvent>
+      <Command>XCOPY $(SolutionDir)\vcpkg\installed\x64-windows\bin "$(TargetDir)" /D /K /Y</Command>
+    </PostBuildEvent>
   </ItemDefinitionGroup>
   <ItemGroup>
     <ClInclude Include="inc\Core\Common\FineGrainedLock.h" />
@@ -160,6 +171,7 @@
     <ClInclude Include="inc\Core\MetadataSet.h" />
     <ClInclude Include="inc\Core\SearchQuery.h" />
     <ClInclude Include="inc\Core\SearchResult.h" />
+    <ClInclude Include="inc\Core\SPANN\Compressor.h" />
     <ClInclude Include="inc\Core\SPANN\ExtraFullGraphSearcher.h" />
     <ClInclude Include="inc\Core\SPANN\IExtraSearcher.h" />
     <ClInclude Include="inc\Core\SPANN\Index.h" />

AnnService/CoreLibrary.vcxproj.filters

@@ -214,6 +214,9 @@
     <ClInclude Include="inc\Core\Common\OPQQuantizer.h">
       <Filter>Header Files\Core\Common</Filter>
     </ClInclude>
+    <ClInclude Include="inc\Core\SPANN\Compressor.h">
+      <Filter>Header Files\Core\SPANN</Filter>
+    </ClInclude>
   </ItemGroup>
   <ItemGroup>
     <ClCompile Include="src\Core\VectorIndex.cpp">

AnnService/SSDServing.vcxproj

@@ -152,21 +152,25 @@
       <SDLCheck>true</SDLCheck>
       <PreprocessorDefinitions>_$(OutputType);_MBCS;_SCL_SECURE_NO_WARNINGS;_CRT_SECURE_NO_WARNINGS;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
       <ConformanceMode>true</ConformanceMode>
-      <AdditionalIncludeDirectories>%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
+      <AdditionalIncludeDirectories>$(SolutionDir)\vcpkg\installed\x64-windows\include;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
       <PrecompiledHeader>NotUsing</PrecompiledHeader>
       <PrecompiledHeaderFile>inc/SSDServing/Common/stdafx.h</PrecompiledHeaderFile>
       <OpenMPSupport>true</OpenMPSupport>
       <AdditionalOptions>/Zc:twoPhase- %(AdditionalOptions)</AdditionalOptions>
-      <LanguageStandard>stdcpp17</LanguageStandard>
+      <LanguageStandard>Default</LanguageStandard>
+      <Optimization>Disabled</Optimization>
     </ClCompile>
     <Link>
       <SubSystem>Console</SubSystem>
       <EnableCOMDATFolding>true</EnableCOMDATFolding>
       <OptimizeReferences>true</OptimizeReferences>
       <GenerateDebugInformation>true</GenerateDebugInformation>
       <AdditionalLibraryDirectories>%(AdditionalLibraryDirectories)</AdditionalLibraryDirectories>
-      <AdditionalDependencies>CoreLibrary.lib;%(AdditionalDependencies)</AdditionalDependencies>
+      <AdditionalDependencies>CoreLibrary.lib;$(SolutionDir)\vcpkg\installed\x64-windows\lib\zstd.lib;%(AdditionalDependencies)</AdditionalDependencies>
     </Link>
+    <PostBuildEvent>
+      <Command>XCOPY $(SolutionDir)\vcpkg\installed\x64-windows\bin "$(TargetDir)" /D /K /Y</Command>
+    </PostBuildEvent>
   </ItemDefinitionGroup>
   <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
   <ImportGroup Label="ExtensionTargets">

AnnService/inc/Core/SPANN/Compressor.h

@@ -0,0 +1,203 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+#ifndef _SPTAG_SPANN_COMPRESSOR_H_
+#define _SPTAG_SPANN_COMPRESSOR_H_
+
+#include <string>
+#include "zstd.h"
+#include "zdict.h"
+#include "../Common.h"
+
+namespace SPTAG
+{
+    namespace SPANN
+    {
+        class Compressor
+        {
+        private:
+            void CreateCDict()
+            {
+                cdict = ZSTD_createCDict((void *)dictBuffer.data(), dictBuffer.size(), compress_level);
+                if (cdict == NULL)
+                {
+                    LOG(Helper::LogLevel::LL_Error, "ZSTD_createCDict() failed! \n");
+                    exit(1);
+                }
+            }
+
+            void CreateDDict()
+            {
+                ddict = ZSTD_createDDict((void *)dictBuffer.data(), dictBuffer.size());
+                if (ddict == NULL)
+                {
+                    LOG(Helper::LogLevel::LL_Error, "ZSTD_createDDict() failed! \n");
+                    exit(1);
+                }
+            }
+
+            std::string CompressWithDict(const std::string &src)
+            {
+                size_t est_compress_size = ZSTD_compressBound(src.size());
+                std::string comp_buffer{};
+                comp_buffer.resize(est_compress_size);
+
+                ZSTD_CCtx *const cctx = ZSTD_createCCtx();
+                if (cctx == NULL)
+                {
+                    LOG(Helper::LogLevel::LL_Error, "ZSTD_createCCtx() failed! \n");
+                    exit(1);
+                }
+                size_t compressed_size = ZSTD_compress_usingCDict(cctx, (void *)comp_buffer.data(), est_compress_size, src.data(), src.size(), cdict);
+                if (ZSTD_isError(compressed_size))
+                {
+                    LOG(Helper::LogLevel::LL_Error, "ZSTD compress error %s, \n", ZSTD_getErrorName(compressed_size));
+                    exit(1);
+                }
+                ZSTD_freeCCtx(cctx);
+                comp_buffer.resize(compressed_size);
+                comp_buffer.shrink_to_fit();
+
+                return comp_buffer;
+            }
+
+            std::string DecompressWithDict(const char *src, size_t srcSize)
+            {
+                auto const est_decomp_size =
+                    ZSTD_getFrameContentSize(src, srcSize);
+
+                std::string decomp_buffer{};
+                decomp_buffer.resize(est_decomp_size);
+
+                ZSTD_DCtx *const dctx = ZSTD_createDCtx();
+                if (dctx == NULL)
+                {
+                    LOG(Helper::LogLevel::LL_Error, "ZSTD_createDCtx() failed! \n");
+                    exit(1);
+                }
+                size_t const decomp_size = ZSTD_decompress_usingDDict(dctx,
+                                                                      (void *)decomp_buffer.data(), est_decomp_size, src, srcSize, ddict);
+                if (ZSTD_isError(decomp_size))
+                {
+                    LOG(Helper::LogLevel::LL_Error, "ZSTD decompress error %s, \n", ZSTD_getErrorName(decomp_size));
+                    exit(1);
+                }
+
+                ZSTD_freeDCtx(dctx);
+                decomp_buffer.resize(decomp_size);
+                decomp_buffer.shrink_to_fit();
+                return decomp_buffer;
+            }
+
+            std::string CompressWithoutDict(const std::string &src)
+            {
+                size_t est_comp_size = ZSTD_compressBound(src.size());
+                std::string buffer{};
+                buffer.resize(est_comp_size);
+                size_t compressed_size = ZSTD_compress((void *)buffer.data(), est_comp_size,
+                                                       src.data(), src.size(), compress_level);
+                if (ZSTD_isError(compressed_size))
+                {
+                    LOG(Helper::LogLevel::LL_Error, "ZSTD compress error %s, \n", ZSTD_getErrorName(compressed_size));
+                    exit(1);
+                }
+                buffer.resize(compressed_size);
+                buffer.shrink_to_fit();
+
+                return buffer;
+            }
+
+            std::string DecompressWithoutDict(const char *src, size_t srcSize)
+            {
+                size_t est_decomp_size = ZSTD_getFrameContentSize(src, srcSize);
+                if (est_decomp_size == ZSTD_CONTENTSIZE_ERROR)
+                {
+                    LOG(Helper::LogLevel::LL_Error, "not compressed by zstd!\n");
+                    exit(1);
+                }
+                else if (est_decomp_size == ZSTD_CONTENTSIZE_UNKNOWN)
+                {
+                    LOG(Helper::LogLevel::LL_Error, "original size unknown!\n");
+                    exit(1);
+                }
+                std::string dst{};
+                dst.resize(est_decomp_size);
+                size_t const decomp_size = ZSTD_decompress(
+                    (void *)dst.data(), est_decomp_size, src, srcSize);
+                if (ZSTD_isError(decomp_size))
+                {
+                    LOG(Helper::LogLevel::LL_Error, "ZSTD decompress error %s, \n", ZSTD_getErrorName(decomp_size));
+                    exit(1);
+                }
+                dst.resize(decomp_size);
+                dst.shrink_to_fit();
+
+                return dst;
+            }
+
+        public:
+            Compressor(int level = 0, int bufferCapacity = 102400)
+            {
+                compress_level = level;
+                dictBufferCapacity = bufferCapacity;
+            }
+
+            virtual ~Compressor() {}
+
+            std::size_t TrainDict(std::string samplesBuffer, const size_t *samplesSizes, unsigned nbSamples)
+            {
+                dictBuffer.resize(dictBufferCapacity);
+                size_t dictSize = ZDICT_trainFromBuffer((void *)dictBuffer.data(), dictBufferCapacity, (void *)samplesBuffer.data(), &samplesSizes[0], nbSamples);
+                if (ZDICT_isError(dictSize))
+                {
+                    LOG(Helper::LogLevel::LL_Error, "ZDICT_trainFromBuffer() failed: %s \n", ZDICT_getErrorName(dictSize));
+                    exit(1);
+                }
+                dictBuffer.resize(dictSize);
+                dictBuffer.shrink_to_fit();
+
+                CreateCDict();
+
+                return dictSize;
+            }
+
+            std::string GetDictBuffer()
+            {
+                return dictBuffer;
+            }
+
+            void SetDictBuffer(std::string buffer)
+            {
+                dictBuffer = buffer;
+                CreateDDict();
+            }
+
+            std::string Compress(const std::string &src, const bool useDict)
+            {
+                return useDict ? CompressWithDict(src) : CompressWithoutDict(src);
+            }
+
+            std::string Decompress(const char *src, size_t srcSize, const bool useDict)
+            {
+                return useDict ? DecompressWithDict(src, srcSize) : DecompressWithoutDict(src, srcSize);
+            }
+
+            // return the compressed sie
+            size_t GetCompressedSize(const std::string &src, bool useDict)
+            {
+                std::string dst = Compress(src, useDict);
+                return dst.size();
+            }
+
+        private:
+            int compress_level;
+
+            std::string dictBuffer;
+            size_t dictBufferCapacity;
+            ZSTD_CDict *cdict;
+            ZSTD_DDict *ddict;
+        };
+    } // SPANN
+} // SPTAG
+
+#endif // _SPTAG_SPANN_COMPRESSOR_H_