Merge c1fb306 into merged_master (Bitcoin PR bitcoin/bitcoin#23114)

delta1 · May 24, 2023 · 81d5607 · 81d5607
2 parents e689ddb + c1fb306
commit 81d5607
Show file tree

Hide file tree

Showing 85 changed files with 8,945 additions and 22 deletions.
diff --git a/Makefile.am b/Makefile.am
@@ -206,6 +206,7 @@ LCOV_FILTER_PATTERN = \
 	-p "src/bench/" \
 	-p "src/univalue" \
 	-p "src/crypto/ctaes" \
+	-p "src/minisketch" \
 	-p "src/secp256k1" \
 	-p "depends"
 

diff --git a/build_msvc/bitcoin.sln b/build_msvc/bitcoin.sln
@@ -48,6 +48,8 @@ Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "libtest_util", "libtest_uti
 EndProject
 Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "test_bitcoin-qt", "test_bitcoin-qt\test_bitcoin-qt.vcxproj", "{51201D5E-D939-4854-AE9D-008F03FF518E}"
 EndProject
+Project("{542007E3-BE0D-4B0D-A6B0-AA8813E2558D}") = "libminisketch", "libminisketch\libminisketch.vcxproj", "{542007E3-BE0D-4B0D-A6B0-AA8813E2558D}"
+EndProject
 Global
     GlobalSection(SolutionConfigurationPlatforms) = preSolution
         Debug|x64 = Debug|x64
@@ -146,13 +148,15 @@ Global
         {51201D5E-D939-4854-AE9D-008F03FF518E}.Debug|x64.Build.0 = Debug|x64
         {51201D5E-D939-4854-AE9D-008F03FF518E}.Release|x64.ActiveCfg = Release|x64
         {51201D5E-D939-4854-AE9D-008F03FF518E}.Release|x64.Build.0 = Release|x64
+        {542007E3-BE0D-4B0D-A6B0-AA8813E2558D}.Debug|x64.ActiveCfg = Debug|x64
+        {542007E3-BE0D-4B0D-A6B0-AA8813E2558D}.Debug|x64.Build.0 = Debug|x64
+        {542007E3-BE0D-4B0D-A6B0-AA8813E2558D}.Release|x64.ActiveCfg = Release|x64
+        {542007E3-BE0D-4B0D-A6B0-AA8813E2558D}.Release|x64.Build.0 = Release|x64
     EndGlobalSection
     GlobalSection(SolutionProperties) = preSolution
         HideSolutionNode = FALSE
     EndGlobalSection
     GlobalSection(ExtensibilityGlobals) = postSolution
-                        SolutionGuid = {8AA72EDA-2CD4-4564-B1E4-688B760EEEE9}
-                SolutionGuid = {8607C0F4-F33D-41B8-8D51-18E366A0F8DF}
         SolutionGuid = {58AAB032-7274-49BD-845E-5EF4DBB69B70}
     EndGlobalSection
 EndGlobal
diff --git a/build_msvc/common.init.vcxproj b/build_msvc/common.init.vcxproj
@@ -91,7 +91,7 @@
       <DisableSpecificWarnings>4018;4244;4267;4334;4715;4805;4834</DisableSpecificWarnings>
       <TreatWarningAsError>true</TreatWarningAsError>
       <PreprocessorDefinitions>_SILENCE_CXX17_CODECVT_HEADER_DEPRECATION_WARNING;_SILENCE_CXX17_OLD_ALLOCATOR_MEMBERS_DEPRECATION_WARNING;ZMQ_STATIC;NOMINMAX;WIN32;HAVE_CONFIG_H;_CRT_SECURE_NO_WARNINGS;_SCL_SECURE_NO_WARNINGS;_CONSOLE;_WIN32_WINNT=0x0601;_WIN32_IE=0x0501;WIN32_LEAN_AND_MEAN;%(PreprocessorDefinitions)</PreprocessorDefinitions>
-      <AdditionalIncludeDirectories>..\..\src;..\..\src\univalue\include;..\..\src\secp256k1\include;..\..\src\leveldb\include;..\..\src\leveldb\helpers\memenv;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
+      <AdditionalIncludeDirectories>..\..\src;..\..\src\minisketch\include;..\..\src\univalue\include;..\..\src\secp256k1\include;..\..\src\leveldb\include;..\..\src\leveldb\helpers\memenv;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
     </ClCompile>
     <Link>
       <SubSystem>Console</SubSystem>

diff --git a/build_msvc/libminisketch/libminisketch.vcxproj b/build_msvc/libminisketch/libminisketch.vcxproj
@@ -0,0 +1,38 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <Import Project="..\common.init.vcxproj" />
+  <PropertyGroup Label="Globals">
+    <ProjectGuid>{542007E3-BE0D-4B0D-A6B0-AA8813E2558D}</ProjectGuid>
+  </PropertyGroup>
+  <PropertyGroup Label="Configuration">
+    <ConfigurationType>StaticLibrary</ConfigurationType>
+  </PropertyGroup>
+  <ItemGroup>
+    <ClCompile Include="..\..\src\minisketch\src\minisketch.cpp" />
+    <ClCompile Include="..\..\src\minisketch\src\fields\clmul_1byte.cpp" />
+    <ClCompile Include="..\..\src\minisketch\src\fields\clmul_2bytes.cpp" />
+    <ClCompile Include="..\..\src\minisketch\src\fields\clmul_3bytes.cpp" />
+    <ClCompile Include="..\..\src\minisketch\src\fields\clmul_4bytes.cpp" />
+    <ClCompile Include="..\..\src\minisketch\src\fields\clmul_5bytes.cpp" />
+    <ClCompile Include="..\..\src\minisketch\src\fields\clmul_6bytes.cpp" />
+    <ClCompile Include="..\..\src\minisketch\src\fields\clmul_7bytes.cpp" />
+    <ClCompile Include="..\..\src\minisketch\src\fields\clmul_8bytes.cpp" />
+    <ClCompile Include="..\..\src\minisketch\src\fields\generic_1byte.cpp" />
+    <ClCompile Include="..\..\src\minisketch\src\fields\generic_2bytes.cpp" />
+    <ClCompile Include="..\..\src\minisketch\src\fields\generic_3bytes.cpp" />
+    <ClCompile Include="..\..\src\minisketch\src\fields\generic_4bytes.cpp" />
+    <ClCompile Include="..\..\src\minisketch\src\fields\generic_5bytes.cpp" />
+    <ClCompile Include="..\..\src\minisketch\src\fields\generic_6bytes.cpp" />
+    <ClCompile Include="..\..\src\minisketch\src\fields\generic_7bytes.cpp" />
+    <ClCompile Include="..\..\src\minisketch\src\fields\generic_8bytes.cpp" />
+  </ItemGroup>
+  <ItemDefinitionGroup>
+    <ClCompile>
+      <DisableSpecificWarnings>4060;4065;4146;4244;4267;4554</DisableSpecificWarnings>
+      <PreprocessorDefinitions>HAVE_CLMUL;DISABLE_DEFAULT_FIELDS;ENABLE_FIELD_32;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+    </ClCompile>
+  </ItemDefinitionGroup>
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
+  <Import Project="..\common.vcxproj" />
+</Project>
diff --git a/build_msvc/test_bitcoin/test_bitcoin.vcxproj b/build_msvc/test_bitcoin/test_bitcoin.vcxproj
@@ -19,6 +19,9 @@
     <ClCompile Include="..\..\src\wallet\test\util.cpp" />
   </ItemGroup>
   <ItemGroup>
+    <ProjectReference Include="..\libminisketch\libminisketch.vcxproj">
+      <Project>{542007e3-be0d-4b0d-a6b0-aa8813e2558d}</Project>
+    </ProjectReference>
     <ProjectReference Include="..\libbitcoinconsensus\libbitcoinconsensus.vcxproj">
       <Project>{2b384fa8-9ee1-4544-93cb-0d733c25e8ce}</Project>
     </ProjectReference>

diff --git a/ci/lint/06_script.sh b/ci/lint/06_script.sh
@@ -17,6 +17,7 @@ export COMMIT_RANGE
 # check with -r to not have to fetch all the remotes.
 test/lint/git-subtree-check.sh src/crypto/ctaes
 test/lint/git-subtree-check.sh src/secp256k1
+test/lint/git-subtree-check.sh src/minisketch
 test/lint/git-subtree-check.sh src/univalue
 test/lint/git-subtree-check.sh src/leveldb
 test/lint/git-subtree-check.sh src/crc32c

diff --git a/ci/test/wrap-qemu.sh b/ci/test/wrap-qemu.sh
@@ -6,7 +6,7 @@
 
 export LC_ALL=C.UTF-8
 
-for b_name in {"${BASE_OUTDIR}/bin"/*,src/secp256k1/*tests,src/univalue/{no_nul,test_json,unitester,object}}; do
+for b_name in {"${BASE_OUTDIR}/bin"/*,src/secp256k1/*tests,src/minisketch/test{,-verify},src/univalue/{no_nul,test_json,unitester,object}}; do
     # shellcheck disable=SC2044
     for b in $(find "${BASE_ROOT_DIR}" -executable -type f -name $(basename $b_name)); do
       echo "Wrap $b ..."

diff --git a/ci/test/wrap-wine.sh b/ci/test/wrap-wine.sh
@@ -6,7 +6,7 @@
 
 export LC_ALL=C.UTF-8
 
-for b_name in {"${BASE_OUTDIR}/bin"/*,src/secp256k1/*tests,src/univalue/{no_nul,test_json,unitester,object}}.exe; do
+for b_name in {"${BASE_OUTDIR}/bin"/*,src/secp256k1/*tests,src/minisketch/test{,-verify},src/univalue/{no_nul,test_json,unitester,object}}.exe; do
     # shellcheck disable=SC2044
     for b in $(find "${BASE_ROOT_DIR}" -executable -type f -name "$(basename $b_name)"); do
       if (file "$b" | grep "Windows"); then

diff --git a/configure.ac b/configure.ac
@@ -479,6 +479,24 @@ AX_CHECK_COMPILE_FLAG([-msse4.1],[[SSE41_CXXFLAGS="-msse4.1"]],,[[$CXXFLAG_WERRO
 AX_CHECK_COMPILE_FLAG([-mavx -mavx2],[[AVX2_CXXFLAGS="-mavx -mavx2"]],,[[$CXXFLAG_WERROR]])
 AX_CHECK_COMPILE_FLAG([-msse4 -msha],[[SHANI_CXXFLAGS="-msse4 -msha"]],,[[$CXXFLAG_WERROR]])
 
+enable_clmul=
+AX_CHECK_COMPILE_FLAG([-mpclmul], [enable_clmul=yes], [], [$CXXFLAG_WERROR], [AC_LANG_PROGRAM([
+  #include <stdint.h>
+  #include <x86intrin.h>
+], [
+  __m128i a = _mm_cvtsi64_si128((uint64_t)7);
+  __m128i b = _mm_clmulepi64_si128(a, a, 37);
+  __m128i c = _mm_srli_epi64(b, 41);
+  __m128i d = _mm_xor_si128(b, c);
+  uint64_t e = _mm_cvtsi128_si64(d);
+  return e == 0;
+])])
+
+if test x$enable_clmul = xyes; then
+  CLMUL_CXXFLAGS="-mpclmul"
+  AC_DEFINE(HAVE_CLMUL, 1, [Define this symbol if clmul instructions can be used])
+fi
+
 TEMP_CXXFLAGS="$CXXFLAGS"
 CXXFLAGS="$CXXFLAGS $SSE42_CXXFLAGS"
 AC_MSG_CHECKING(for SSE4.2 intrinsics)
@@ -961,19 +979,20 @@ AC_CHECK_DECLS([bswap_16, bswap_32, bswap_64],,,
                  #endif])
 
 AC_MSG_CHECKING(for __builtin_clzl)
+
 AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ ]], [[
  (void) __builtin_clzl(0);
   ]])],
- [ AC_MSG_RESULT(yes); AC_DEFINE(HAVE_BUILTIN_CLZL, 1, [Define this symbol if you have __builtin_clzl])],
- [ AC_MSG_RESULT(no)]
+ [ AC_MSG_RESULT(yes); have_clzl=yes; AC_DEFINE(HAVE_BUILTIN_CLZL, 1, [Define this symbol if you have __builtin_clzl])],
+ [ AC_MSG_RESULT(no); have_clzl=no;]
 )
 
 AC_MSG_CHECKING(for __builtin_clzll)
 AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ ]], [[
   (void) __builtin_clzll(0);
   ]])],
- [ AC_MSG_RESULT(yes); AC_DEFINE(HAVE_BUILTIN_CLZLL, 1, [Define this symbol if you have __builtin_clzll])],
- [ AC_MSG_RESULT(no)]
+ [ AC_MSG_RESULT(yes); have_clzll=yes; AC_DEFINE(HAVE_BUILTIN_CLZLL, 1, [Define this symbol if you have __builtin_clzll])],
+ [ AC_MSG_RESULT(no); have_clzll=no;]
 )
 
 dnl Check for malloc_info (for memory statistics information in getmemoryinfo)
@@ -1774,6 +1793,10 @@ AM_CONDITIONAL([USE_NATPMP],[test x$use_natpmp = xyes])
 AM_CONDITIONAL([USE_UPNP],[test x$use_upnp = xyes])
 AM_CONDITIONAL([LIQUID],[test x$liquid_build = xyes])
 
+dnl for minisketch
+AM_CONDITIONAL([ENABLE_CLMUL],[test x$enable_clmul = xyes])
+AM_CONDITIONAL([HAVE_CLZ],[test x$have_clzl$have_clzll = xyesyes])
+
 AC_DEFINE(CLIENT_VERSION_MAJOR, _CLIENT_VERSION_MAJOR, [Major version])
 AC_DEFINE(CLIENT_VERSION_MINOR, _CLIENT_VERSION_MINOR, [Minor version])
 AC_DEFINE(CLIENT_VERSION_BUILD, _CLIENT_VERSION_BUILD, [Version Build])
@@ -1817,6 +1840,7 @@ AC_SUBST(SANITIZER_CXXFLAGS)
 AC_SUBST(SANITIZER_LDFLAGS)
 AC_SUBST(SSE42_CXXFLAGS)
 AC_SUBST(SSE41_CXXFLAGS)
+AC_SUBST(CLMUL_CXXFLAGS)
 AC_SUBST(AVX2_CXXFLAGS)
 AC_SUBST(SHANI_CXXFLAGS)
 AC_SUBST(ARM_CRC_CXXFLAGS)

diff --git a/contrib/devtools/copyright_header.py b/contrib/devtools/copyright_header.py
@@ -33,6 +33,7 @@
     # git subtrees
     "src/crypto/ctaes/",
     "src/leveldb/",
+    "src/minisketch",
     "src/secp256k1/",
     "src/univalue/",
     "src/crc32c/",

diff --git a/doc/developer-notes.md b/doc/developer-notes.md
@@ -1029,6 +1029,9 @@ Current subtrees include:
   - Subtree at https://github.com/bitcoin-core/univalue-subtree ; maintained by Core contributors.
   - Deviates from upstream https://github.com/jgarzik/univalue.
 
+- src/minisketch
+  - Upstream at https://github.com/sipa/minisketch ; maintained by Core contributors.
+
 Upgrading LevelDB
 ---------------------
 

diff --git a/src/Makefile.am b/src/Makefile.am
@@ -15,7 +15,7 @@ AM_LIBTOOLFLAGS = --preserve-dup-deps
 PTHREAD_FLAGS = $(PTHREAD_CFLAGS) $(PTHREAD_LIBS)
 EXTRA_LIBRARIES =
 
-BITCOIN_INCLUDES=-I$(builddir) -I$(srcdir)/secp256k1/include -I$(srcdir)/$(UNIVALUE_INCLUDE_DIR_INT) $(BDB_CPPFLAGS) $(BOOST_CPPFLAGS) $(LEVELDB_CPPFLAGS)
+BITCOIN_INCLUDES=-I$(builddir) -I$(srcdir)/$(MINISKETCH_INCLUDE_DIR_INT) -I$(srcdir)/secp256k1/include -I$(srcdir)/$(UNIVALUE_INCLUDE_DIR_INT) $(BDB_CPPFLAGS) $(BOOST_CPPFLAGS) $(LEVELDB_CPPFLAGS)
 
 LIBBITCOIN_SERVER=libbitcoin_server.a
 LIBBITCOIN_COMMON=libbitcoin_common.a
@@ -177,6 +177,7 @@ BITCOIN_CORE_H = \
   memusage.h \
   merkleblock.h \
   miner.h \
+  minisketchwrapper.h \
   net.h \
   net_permissions.h \
   net_processing.h \
@@ -352,6 +353,7 @@ libbitcoin_server_a_SOURCES = \
   mainchainrpc.cpp \
   mapport.cpp \
   miner.cpp \
+  minisketchwrapper.cpp \
   net.cpp \
   net_processing.cpp \
   node/blockstorage.cpp \
@@ -884,8 +886,11 @@ nodist_libbitcoin_ipc_a_SOURCES = $(libbitcoin_ipc_mpgen_output)
 CLEANFILES += $(libbitcoin_ipc_mpgen_output)
 endif
 
+include Makefile.minisketch.include
+
 include Makefile.crc32c.include
 include Makefile.leveldb.include
+
 include Makefile.test_util.include
 include Makefile.test_fuzz.include
 

diff --git a/src/Makefile.minisketch.include b/src/Makefile.minisketch.include
@@ -0,0 +1,43 @@
+include minisketch/sources.mk
+
+LIBMINISKETCH_CPPFLAGS=
+LIBMINISKETCH_CPPFLAGS += -DDISABLE_DEFAULT_FIELDS -DENABLE_FIELD_32
+
+LIBMINISKETCH = minisketch/libminisketch.a
+MINISKETCH_LIBS = $(LIBMINISKETCH)
+
+if ENABLE_CLMUL
+LIBMINISKETCH_CLMUL = minisketch/libminisketch_clmul.a
+LIBMINISKETCH_CPPFLAGS += -DHAVE_CLMUL
+MINISKETCH_LIBS += $(LIBMINISKETCH_CLMUL)
+endif
+
+if HAVE_CLZ
+LIBMINISKETCH_CPPFLAGS += -DHAVE_CLZ
+endif
+
+EXTRA_LIBRARIES += $(MINISKETCH_LIBS)
+
+minisketch_libminisketch_clmul_a_SOURCES = $(MINISKETCH_FIELD_CLMUL_SOURCES_INT) $(MINISKETCH_FIELD_CLMUL_HEADERS_INT)
+minisketch_libminisketch_clmul_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) $(CLMUL_CXXFLAGS)
+minisketch_libminisketch_clmul_a_CPPFLAGS = $(AM_CPPFLAGS) $(LIBMINISKETCH_CPPFLAGS)
+
+minisketch_libminisketch_a_SOURCES = $(MINISKETCH_FIELD_GENERIC_SOURCES_INT) $(MINISKETCH_LIB_SOURCES_INT)
+minisketch_libminisketch_a_SOURCES += $(MINISKETCH_FIELD_GENERIC_HEADERS_INT) $(MINISKETCH_LIB_HEADERS_INT) $(MINISKETCH_DIST_HEADERS_INT)
+minisketch_libminisketch_a_CPPFLAGS = $(AM_CPPFLAGS) $(LIBMINISKETCH_CPPFLAGS)
+minisketch_libminisketch_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS)
+
+if ENABLE_TESTS
+if !ENABLE_FUZZ
+MINISKETCH_TEST = minisketch/test
+TESTS += $(MINISKETCH_TEST)
+noinst_PROGRAMS += $(MINISKETCH_TEST)
+
+minisketch_test_SOURCES = $(MINISKETCH_TEST_SOURCES_INT)
+minisketch_test_CPPFLAGS = $(AM_CPPFLAGS) $(LIBMINISKETCH_CPPFLAGS)
+minisketch_test_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS)
+minisketch_test_LDADD = $(MINISKETCH_LIBS)
+minisketch_test_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) $(PTHREAD_FLAGS)
+
+endif
+endif
diff --git a/src/Makefile.test.include b/src/Makefile.test.include
@@ -104,6 +104,7 @@ BITCOIN_TESTS =\
   test/mempool_tests.cpp \
   test/merkle_tests.cpp \
   test/merkleblock_tests.cpp \
+  test/minisketch_tests.cpp \
   test/miner_tests.cpp \
   test/multisig_tests.cpp \
   test/net_peer_eviction_tests.cpp \
@@ -200,7 +201,7 @@ test_test_bitcoin_LDADD += $(LIBBITCOIN_WALLET)
 endif
 
 test_test_bitcoin_LDADD += $(LIBBITCOIN_SERVER) $(LIBBITCOIN_CLI) $(LIBBITCOIN_COMMON) $(LIBBITCOIN_UTIL) $(LIBBITCOIN_CONSENSUS) $(LIBBITCOIN_CRYPTO) $(LIBUNIVALUE) \
-  $(LIBLEVELDB) $(LIBLEVELDB_SSE42) $(LIBMEMENV) $(BOOST_LIBS) $(BOOST_UNIT_TEST_FRAMEWORK_LIB) $(LIBSECP256K1) $(EVENT_LIBS) $(EVENT_PTHREADS_LIBS)
+  $(LIBLEVELDB) $(LIBLEVELDB_SSE42) $(LIBMEMENV) $(BOOST_LIBS) $(BOOST_UNIT_TEST_FRAMEWORK_LIB) $(LIBSECP256K1) $(EVENT_LIBS) $(EVENT_PTHREADS_LIBS) $(MINISKETCH_LIBS)
 test_test_bitcoin_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS)
 
 test_test_bitcoin_LDADD += $(BDB_LIBS) $(MINIUPNPC_LIBS) $(NATPMP_LIBS) $(SQLITE_LIBS)