From 08c57a7a99044557d284143a1b2b478b952677d1 Mon Sep 17 00:00:00 2001 From: lbrynaut Date: Fri, 23 Feb 2018 11:05:12 -0600 Subject: [PATCH] Add lowercase and unicode normalization to claim name entries in the claim trie --- build-aux/m4/ax_boost_locale.m4 | 120 +++++++++ configure.ac | 14 +- reproducible_build.sh | 86 +++++-- src/Makefile.am | 8 +- src/Makefile.test.include | 2 +- src/claimtrie.cpp | 429 ++++++++++++++++++++------------ src/claimtrie.h | 48 +++- src/init.cpp | 16 ++ 8 files changed, 532 insertions(+), 191 deletions(-) create mode 100644 build-aux/m4/ax_boost_locale.m4 diff --git a/build-aux/m4/ax_boost_locale.m4 b/build-aux/m4/ax_boost_locale.m4 new file mode 100644 index 0000000000..4f9883b723 --- /dev/null +++ b/build-aux/m4/ax_boost_locale.m4 @@ -0,0 +1,120 @@ +# =========================================================================== +# https://www.gnu.org/software/autoconf-archive/ax_boost_locale.html +# =========================================================================== +# +# SYNOPSIS +# +# AX_BOOST_LOCALE +# +# DESCRIPTION +# +# Test for System library from the Boost C++ libraries. The macro requires +# a preceding call to AX_BOOST_BASE. Further documentation is available at +# . +# +# This macro calls: +# +# AC_SUBST(BOOST_LOCALE_LIB) +# +# And sets: +# +# HAVE_BOOST_LOCALE +# +# LICENSE +# +# Copyright (c) 2012 Xiyue Deng +# +# Copying and distribution of this file, with or without modification, are +# permitted in any medium without royalty provided the copyright notice +# and this notice are preserved. This file is offered as-is, without any +# warranty. + +#serial 2 + +AC_DEFUN([AX_BOOST_LOCALE], +[ + AC_ARG_WITH([boost-locale], + AS_HELP_STRING([--with-boost-locale@<:@=special-lib@:>@], + [use the Locale library from boost - it is possible to specify a certain library for the linker + e.g. --with-boost-locale=boost_locale-gcc-mt ]), + [ + if test "$withval" = "no"; then + want_boost="no" + elif test "$withval" = "yes"; then + want_boost="yes" + ax_boost_user_locale_lib="" + else + want_boost="yes" + ax_boost_user_locale_lib="$withval" + fi + ], + [want_boost="yes"] + ) + + if test "x$want_boost" = "xyes"; then + AC_REQUIRE([AC_PROG_CC]) + AC_REQUIRE([AC_CANONICAL_BUILD]) + CPPFLAGS_SAVED="$CPPFLAGS" + CPPFLAGS="$CPPFLAGS $BOOST_CPPFLAGS" + export CPPFLAGS + + LDFLAGS_SAVED="$LDFLAGS" + LDFLAGS="$LDFLAGS $BOOST_LDFLAGS" + export LDFLAGS + + AC_CACHE_CHECK(whether the Boost::Locale library is available, + ax_cv_boost_locale, + [AC_LANG_PUSH([C++]) + CXXFLAGS_SAVE=$CXXFLAGS + + AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[@%:@include ]], + [[boost::locale::generator gen; + std::locale::global(gen(""));]])], + ax_cv_boost_locale=yes, ax_cv_boost_locale=no) + CXXFLAGS=$CXXFLAGS_SAVE + AC_LANG_POP([C++]) + ]) + if test "x$ax_cv_boost_locale" = "xyes"; then + AC_SUBST(BOOST_CPPFLAGS) + + AC_DEFINE(HAVE_BOOST_LOCALE,,[define if the Boost::Locale library is available]) + BOOSTLIBDIR=`echo $BOOST_LDFLAGS | sed -e 's/@<:@^\/@:>@*//'` + + LDFLAGS_SAVE=$LDFLAGS + if test "x$ax_boost_user_locale_lib" = "x"; then + for libextension in `ls $BOOSTLIBDIR/libboost_locale*.so* $BOOSTLIBDIR/libboost_locale*.dylib* $BOOSTLIBDIR/libboost_locale*.a* 2>/dev/null | sed 's,.*/,,' | sed -e 's;^lib\(boost_locale.*\)\.so.*$;\1;' -e 's;^lib\(boost_locale.*\)\.dylib.*$;\1;' -e 's;^lib\(boost_locale.*\)\.a.*$;\1;'` ; do + ax_lib=${libextension} + AC_CHECK_LIB($ax_lib, exit, + [BOOST_LOCALE_LIB="-l$ax_lib"; AC_SUBST(BOOST_LOCALE_LIB) link_locale="yes"; break], + [link_locale="no"]) + done + if test "x$link_locale" != "xyes"; then + for libextension in `ls $BOOSTLIBDIR/boost_locale*.dll* $BOOSTLIBDIR/boost_locale*.a* 2>/dev/null | sed 's,.*/,,' | sed -e 's;^\(boost_locale.*\)\.dll.*$;\1;' -e 's;^\(boost_locale.*\)\.a.*$;\1;'` ; do + ax_lib=${libextension} + AC_CHECK_LIB($ax_lib, exit, + [BOOST_LOCALE_LIB="-l$ax_lib"; AC_SUBST(BOOST_LOCALE_LIB) link_locale="yes"; break], + [link_locale="no"]) + done + fi + + else + for ax_lib in $ax_boost_user_locale_lib boost_locale-$ax_boost_user_locale_lib; do + AC_CHECK_LIB($ax_lib, exit, + [BOOST_LOCALE_LIB="-l$ax_lib"; AC_SUBST(BOOST_LOCALE_LIB) link_locale="yes"; break], + [link_locale="no"]) + done + + fi + if test "x$ax_lib" = "x"; then + AC_MSG_ERROR(Could not find a version of the library!) + fi + if test "x$link_locale" = "xno"; then + AC_MSG_ERROR(Could not link against $ax_lib !) + fi + fi + + CPPFLAGS="$CPPFLAGS_SAVED" + LDFLAGS="$LDFLAGS_SAVED" + fi +]) + diff --git a/configure.ac b/configure.ac index 81a63e7ef2..df92a1aebc 100644 --- a/configure.ac +++ b/configure.ac @@ -583,6 +583,7 @@ AX_BOOST_FILESYSTEM AX_BOOST_PROGRAM_OPTIONS AX_BOOST_THREAD AX_BOOST_CHRONO +AX_BOOST_LOCALE if test x$use_reduce_exports = xyes; then @@ -645,7 +646,7 @@ fi if test x$use_boost = xyes; then -BOOST_LIBS="$BOOST_LDFLAGS $BOOST_SYSTEM_LIB $BOOST_FILESYSTEM_LIB $BOOST_PROGRAM_OPTIONS_LIB $BOOST_THREAD_LIB $BOOST_CHRONO_LIB" +BOOST_LIBS="$BOOST_LDFLAGS $BOOST_SYSTEM_LIB $BOOST_FILESYSTEM_LIB $BOOST_PROGRAM_OPTIONS_LIB $BOOST_THREAD_LIB $BOOST_CHRONO_LIB $BOOST_LOCALE_LIB" dnl If boost (prior to 1.57) was built without c++11, it emulated scoped enums @@ -746,8 +747,11 @@ if test x$use_pkgconfig = xyes; then m4_ifdef( [PKG_CHECK_MODULES], [ - PKG_CHECK_MODULES([SSL], [libssl],, [AC_MSG_ERROR(openssl not found.)]) - PKG_CHECK_MODULES([CRYPTO], [libcrypto],,[AC_MSG_ERROR(libcrypto not found.)]) + AC_CHECK_HEADER([unicode/errorcode.h],,AC_MSG_ERROR(libicu headers missing)) + AC_CHECK_LIB([icudata], [main],ICU_LIBS="-licui18n -licuuc -licudata -dl", AC_MSG_ERROR(libicu missing)) + + PKG_CHECK_MODULES([SSL], [libssl],, [AC_MSG_ERROR(openssl not found.)]) + PKG_CHECK_MODULES([CRYPTO], [libcrypto],,[AC_MSG_ERROR(libcrypto not found.)]) BITCOIN_QT_CHECK([PKG_CHECK_MODULES([PROTOBUF], [protobuf], [have_protobuf=yes], [BITCOIN_QT_FAIL(libprotobuf not found)])]) if test x$use_qr != xno; then BITCOIN_QT_CHECK([PKG_CHECK_MODULES([QR], [libqrencode], [have_qrencode=yes], [have_qrencode=no])]) @@ -771,6 +775,9 @@ if test x$use_pkgconfig = xyes; then ] ) else + AC_CHECK_HEADER([unicode/errorcode.h],,AC_MSG_ERROR(libicu headers missing)) + AC_CHECK_LIB([icudata], [main],ICU_LIBS="-licui18n -licuuc -licudata -dl", AC_MSG_ERROR(libicu missing)) + AC_CHECK_HEADER([openssl/crypto.h],,AC_MSG_ERROR(libcrypto headers missing)) AC_CHECK_LIB([crypto], [main],CRYPTO_LIBS=-lcrypto, AC_MSG_ERROR(libcrypto missing)) @@ -1039,6 +1046,7 @@ AC_SUBST(LIBTOOL_APP_LDFLAGS) AC_SUBST(USE_UPNP) AC_SUBST(USE_QRCODE) AC_SUBST(BOOST_LIBS) +AC_SUBST(ICU_LIBS) AC_SUBST(TESTDEFS) AC_SUBST(LEVELDB_TARGET_FLAGS) AC_SUBST(MINIUPNPC_CPPFLAGS) diff --git a/reproducible_build.sh b/reproducible_build.sh index 44c4bfcd0a..ed6470b6d9 100755 --- a/reproducible_build.sh +++ b/reproducible_build.sh @@ -1,5 +1,6 @@ #!/bin/bash +set -x set -euo pipefail function HELP { @@ -101,24 +102,23 @@ else # this file is created when the build starts START_TIME_FILE="$TRAVIS_BUILD_DIR/start_time" fi -if [ ! -f "${START_TIME_FILE}" ]; then - date +%s > "${START_TIME_FILE}" -fi +rm -f ${START_TIME_FILE} +date +%s > "${START_TIME_FILE}" NEXT_TIME=60 -function exit_at_40() { +function exit_at_60() { if [ -f "${START_TIME_FILE}" ]; then NOW=$(date +%s) START=$(cat "${START_TIME_FILE}") - TIMEOUT_SECS=2400 # 40 * 60 + TIMEOUT_SECS=3600 # 60 * 60 TIME=$((NOW - START)) if (( TIME > NEXT_TIME )); then echo "Build has taken $((TIME / 60)) minutes: $1" NEXT_TIME=$((TIME + 60)) fi if [ "$TIMEOUT" = true ] && (( TIME > TIMEOUT_SECS )); then - echo 'Exiting at 40 minutes to allow the cache to populate' + echo 'Exiting at 60 minutes to allow the cache to populate' OUTPUT_LOG=false exit 1 fi @@ -140,7 +140,7 @@ function wait_and_echo() { # loop until the process is no longer running # check every $SLEEP seconds, echoing a message every minute while (ps -p "${PID}" > /dev/null); do - exit_at_40 "$2" + exit_at_60 "$2" sleep "${SLEEP}" done # restore the xtrace setting @@ -226,10 +226,14 @@ function install_apt_packages() { ca-certificates automake bsdmainutils } +PARALLEL="" + function build_dependencies() { if [ "${OS_NAME}" = "osx" ]; then + PARALLEL="-j $(sysctl -n hw.ncpu)" install_brew_packages else + PARALLEL="-j $(grep -c processor /proc/cpuinfo)" install_apt_packages fi @@ -244,6 +248,7 @@ function build_dependencies() { # TODO: if the repo exists, make sure its clean: revert to head. mkdir -p "${LOG_DIR}" + build_dependency "${ICU_PREFIX}" "${LOG_DIR}/icu_build.log" build_icu build_dependency "${BDB_PREFIX}" "${LOG_DIR}/bdb_build.log" build_bdb build_dependency "${OPENSSL_PREFIX}" "${LOG_DIR}/openssl_build.log" build_openssl @@ -263,7 +268,7 @@ function build_bdb() { cd db-4.8.30.NC/build_unix echo "Building bdb. tail -f $BDB_LOG to see the details and monitor progress" ../dist/configure --prefix="${BDB_PREFIX}" --enable-cxx --disable-shared --with-pic > "${BDB_LOG}" - background make "${BDB_LOG}" "Waiting for bdb to finish building" + background "make ${PARALLEL}" "${BDB_LOG}" "Waiting for bdb to finish building" make install >> "${BDB_LOG}" 2>&1 } @@ -280,18 +285,46 @@ function build_openssl() { ./Configure --prefix="${OPENSSL_PREFIX}" --openssldir="${OPENSSL_PREFIX}/ssl" \ linux-x86_64 -fPIC -static no-shared no-dso > "${OPENSSL_LOG}" fi - background make "${OPENSSL_LOG}" "Waiting for openssl to finish building" + background "make ${PARALLEL}" "${OPENSSL_LOG}" "Waiting for openssl to finish building" make install >> "${OPENSSL_LOG}" 2>&1 } function build_boost() { BOOST_LOG="$1" cd boost_1_59_0 + + echo "int main() { return 0; }" > libs/regex/build/has_icu_test.cpp + echo "int main() { return 0; }" > libs/locale/build/has_icu_test.cpp + + export BOOST_ICU_LIBS="$(pkg-config icu-i18n --libs) -dl" + echo "BOOST_ICU_LIBS: $BOOST_ICU_LIBS" + echo "Building Boost. tail -f ${BOOST_LOG} to see the details and monitor progress" - ./bootstrap.sh --prefix="${BOOST_PREFIX}" > "${BOOST_LOG}" 2>&1 - background "./b2 link=static cxxflags=-fPIC install" \ - "${BOOST_LOG}" \ - "Waiting for boost to finish building" + + ./bootstrap.sh --prefix="${BOOST_PREFIX}" \ + "--with-icu=${ICU_PREFIX}" > "${BOOST_LOG}" 2>&1 + b2cmd="./b2 link=static cxxflags=-fPIC install boost.locale.iconv=off boost.locale.posix=off -sICU_PATH=$ICU_PREFIX -sICU_LINK=${BOOST_ICU_LIBS}" + + background "$b2cmd" "${BOOST_LOG}" "Waiting for boost to finish building" +} + +function build_icu() { + ICU_LOG="$1" + mkdir -p "${ICU_PREFIX}/icu" + wget http://download.icu-project.org/files/icu4c/55.1/icu4c-55_1-src.tgz + tar -xf icu4c-55_1-src.tgz + rm -f icu4c-55_1-src.tgz + pushd icu/source > /dev/null + echo "Building icu. tail -f $ICU_LOG to see the details and monitor progress" + ./configure --prefix="${ICU_PREFIX}" --enable-draft --enable-tools \ + --enable-static --disable-extras --disable-icuio --disable-layout \ + --disable-layoutex --disable-tests --disable-samples + TMP_TARGET="$TARGET" + unset TARGET + background "make ${PARALLEL} VERBOSE=1" "${ICU_LOG}" "Waiting for icu to finish building" + make install >> "${ICU_LOG}" 2>&1 + TARGET="$TMP_TARGET" + popd > /dev/null } function build_libevent() { @@ -305,7 +338,7 @@ function build_libevent() { ./configure --prefix="${LIBEVENT_PREFIX}" --enable-static --disable-shared --with-pic \ LDFLAGS="-L${OPENSSL_PREFIX}/lib/" \ CPPFLAGS="-I${OPENSSL_PREFIX}/include" >> "${LIBEVENT_LOG}" 2>&1 - background make "${LIBEVENT_LOG}" "Waiting for libevent to finish building" + background "make ${PARALLEL}" "${LIBEVENT_LOG}" "Waiting for libevent to finish building" make install >> "${LIBEVENT_LOG}" } @@ -323,16 +356,18 @@ function build_dependency() { } function build_lbrycrd() { - if [ "$CLONE" == true ]; then +# if [ "$CLONE" == true ]; then cd "${LBRYCRD_DEPENDENCIES}" - git clone https://github.com/lbryio/lbrycrd +# git clone https://github.com/lbryio/lbrycrd + git clone https://github.com/lbrynaut/lbrycrd cd lbrycrd - else - cd "${SOURCE_DIR}" - fi + git checkout -b normalized-name-fork origin/normalized-name-fork +# else +# cd "${SOURCE_DIR}" +# fi ./autogen.sh > "${LBRYCRD_LOG}" 2>&1 - LDFLAGS="-L${OPENSSL_PREFIX}/lib/ -L${BDB_PREFIX}/lib/ -L${LIBEVENT_PREFIX}/lib/ -static-libstdc++" - CPPFLAGS="-I${OPENSSL_PREFIX}/include -I${BDB_PREFIX}/include -I${LIBEVENT_PREFIX}/include/" + LDFLAGS="-L${ICU_PREFIX}/lib/ -L${OPENSSL_PREFIX}/lib/ -L${BDB_PREFIX}/lib/ -L${LIBEVENT_PREFIX}/lib/ -L${BOOST_PREFIX}/lib/ -static-libstdc++" + CPPFLAGS="-I${ICU_PREFIX}/include/ -I${OPENSSL_PREFIX}/include -I${BDB_PREFIX}/include -I${LIBEVENT_PREFIX}/include/ -I${BOOST_PREFIX}/include/" if [ "${OS_NAME}" = "osx" ]; then OPTIONS="--enable-cxx --enable-static --disable-shared --with-pic" else @@ -342,12 +377,14 @@ function build_lbrycrd() { --with-boost="${BOOST_PREFIX}" \ LDFLAGS="${LDFLAGS}" \ CPPFLAGS="${CPPFLAGS}" >> "${LBRYCRD_LOG}" 2>&1 - background make "${LBRYCRD_LOG}" "Waiting for lbrycrd to finish building" + background "make ${PARALLEL}" "${LBRYCRD_LOG}" "Waiting for lbrycrd to finish building" # tests don't work on OSX. Should definitely figure out why # that is but, for now, not letting that stop the rest # of the build if [ "${OS_NAME}" = "linux" ]; then - src/test/test_lbrycrd +# LD_LIBRARY_PATH="${ICU_PREFIX}/lib" make ${PARALLEL} check + LD_LIBRARY_PATH="${ICU_PREFIX}/lib" src/test/test_lbrycrd +# src/test/test_lbrycrd fi strip src/lbrycrdd strip src/lbrycrd-cli @@ -363,13 +400,14 @@ BDB_PREFIX="${OUTPUT_DIR}/bdb" OPENSSL_PREFIX="${OUTPUT_DIR}/openssl" BOOST_PREFIX="${OUTPUT_DIR}/boost" LIBEVENT_PREFIX="${OUTPUT_DIR}/libevent" +ICU_PREFIX="${OUTPUT_DIR}/icu" if [ "${BUILD_DEPENDENCIES}" = true ]; then build_dependencies fi set +u -export PKG_CONFIG_PATH="${PKG_CONFIG_PATH}:${OPENSSL_PREFIX}/lib/pkgconfig/:${LIBEVENT_PREFIX}/lib/pkgconfig/" +export PKG_CONFIG_PATH="${PKG_CONFIG_PATH}:${OPENSSL_PREFIX}/lib/pkgconfig/:${LIBEVENT_PREFIX}/lib/pkgconfig/:${ICU_PREFIX}/lib/pkgconfig/" set -u if [ "${BUILD_LBRYCRD}" = true ]; then diff --git a/src/Makefile.am b/src/Makefile.am index a9f0a949ce..3f038ca343 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -29,7 +29,7 @@ $(LIBLEVELDB) $(LIBMEMENV): endif BITCOIN_CONFIG_INCLUDES=-I$(builddir)/config -BITCOIN_INCLUDES=-I$(builddir) -I$(builddir)/obj $(BOOST_CPPFLAGS) $(LEVELDB_CPPFLAGS) $(CRYPTO_CFLAGS) $(SSL_CFLAGS) +BITCOIN_INCLUDES=-I$(builddir) -I$(builddir)/obj $(ICU_CPPFLAGS) $(BOOST_CPPFLAGS) $(LEVELDB_CPPFLAGS) $(CRYPTO_CFLAGS) $(SSL_CFLAGS) BITCOIN_INCLUDES += -I$(srcdir)/secp256k1/include BITCOIN_INCLUDES += $(UNIVALUE_CFLAGS) @@ -379,7 +379,7 @@ if ENABLE_WALLET lbrycrdd_LDADD += libbitcoin_wallet.a endif -lbrycrdd_LDADD += $(BOOST_LIBS) $(BDB_LIBS) $(SSL_LIBS) $(CRYPTO_LIBS) $(MINIUPNPC_LIBS) $(EVENT_PTHREADS_LIBS) $(EVENT_LIBS) +lbrycrdd_LDADD += $(ICU_LIBS) $(BOOST_LIBS) $(BDB_LIBS) $(SSL_LIBS) $(CRYPTO_LIBS) $(MINIUPNPC_LIBS) $(EVENT_PTHREADS_LIBS) $(EVENT_LIBS) #lbrycrd-cli binary # lbrycrd_cli_SOURCES = bitcoin-cli.cpp @@ -396,7 +396,7 @@ lbrycrd_cli_LDADD = \ $(LIBUNIVALUE) \ $(LIBBITCOIN_UTIL) -lbrycrd_cli_LDADD += $(BOOST_LIBS) $(SSL_LIBS) $(CRYPTO_LIBS) $(EVENT_LIBS) +lbrycrd_cli_LDADD += $(ICU_LIBS) $(BOOST_LIBS) $(SSL_LIBS) $(CRYPTO_LIBS) $(EVENT_LIBS) # # bitcoin-tx binary # @@ -417,7 +417,7 @@ lbrycrd_tx_LDADD = \ $(LIBBITCOIN_CRYPTO) \ $(LIBSECP256K1) -lbrycrd_tx_LDADD += $(BOOST_LIBS) $(CRYPTO_LIBS) +lbrycrd_tx_LDADD += $(ICU_LIBS) $(BOOST_LIBS) $(CRYPTO_LIBS) # # bitcoinconsensus library # diff --git a/src/Makefile.test.include b/src/Makefile.test.include index 5ae3077fc8..d4ec7070af 100644 --- a/src/Makefile.test.include +++ b/src/Makefile.test.include @@ -107,7 +107,7 @@ if ENABLE_WALLET test_test_lbrycrd_LDADD += $(LIBBITCOIN_WALLET) endif -test_test_lbrycrd_LDADD += $(LIBBITCOIN_CONSENSUS) $(BDB_LIBS) $(SSL_LIBS) $(CRYPTO_LIBS) $(MINIUPNPC_LIBS) +test_test_lbrycrd_LDADD += $(LIBBITCOIN_CONSENSUS) $(BDB_LIBS) $(SSL_LIBS) $(CRYPTO_LIBS) $(MINIUPNPC_LIBS) $(ICU_LIBS) test_test_lbrycrd_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) -static if ENABLE_ZMQ diff --git a/src/claimtrie.cpp b/src/claimtrie.cpp index 2560610b62..99d3425a54 100644 --- a/src/claimtrie.cpp +++ b/src/claimtrie.cpp @@ -1,6 +1,7 @@ #include "claimtrie.h" #include "coins.h" #include "hash.h" +#include "init.h" #include #include @@ -220,6 +221,11 @@ void CClaimTrie::setExpirationTime(int t) nExpirationTime = t; } +void CClaimTrie::setNormalize(bool normalize) +{ + normalizeNames = normalize; +} + void CClaimTrie::clear() { clear(&root); @@ -237,7 +243,8 @@ void CClaimTrie::clear(CClaimTrieNode* current) bool CClaimTrie::haveClaim(const std::string& name, const COutPoint& outPoint) const { const CClaimTrieNode* current = &root; - for (std::string::const_iterator itname = name.begin(); itname != name.end(); ++itname) + const std::string normalizedName = lbryNormalize(name, normalizeNames); + for (std::string::const_iterator itname = normalizedName.begin(); itname != normalizedName.end(); ++itname) { nodeMapType::const_iterator itchildren = current->children.find(*itname); if (itchildren == current->children.end()) @@ -250,7 +257,8 @@ bool CClaimTrie::haveClaim(const std::string& name, const COutPoint& outPoint) c bool CClaimTrie::haveSupport(const std::string& name, const COutPoint& outPoint) const { supportMapEntryType node; - if (!getSupportNode(name, node)) + const std::string normalizedName = lbryNormalize(name, normalizeNames); + if (!getSupportNode(normalizedName, node)) { return false; } @@ -282,29 +290,31 @@ bool CClaimTrie::haveClaimInQueue(const std::string& name, const COutPoint& outP { return false; } + const std::string normalizedName = lbryNormalize(name, CHECK_HEIGHT(nValidAtHeight)); claimQueueRowType row; if (getQueueRow(nValidAtHeight, row)) { for (claimQueueRowType::const_iterator itRow = row.begin(); itRow != row.end(); ++itRow) { - if (itRow->first == name && itRow->second.outPoint == outPoint) + if (itRow->first == normalizedName && itRow->second.outPoint == outPoint) { if (itRow->second.nValidAtHeight != nValidAtHeight) { - LogPrintf("%s: An inconsistency was found in the claim queue. Please report this to the developers:\nDifferent nValidAtHeight between named queue and height queue\n: name: %s, txid: %s, nOut: %d, nValidAtHeight in named queue: %d, nValidAtHeight in height queue: %d current height: %d\n", __func__, name, outPoint.hash.GetHex(), outPoint.n, nValidAtHeight, itRow->second.nValidAtHeight, nCurrentHeight); + LogPrintf("%s: An inconsistency was found in the claim queue. Please report this to the developers:\nDifferent nValidAtHeight between named queue and height queue\n: name: %s, txid: %s, nOut: %d, nValidAtHeight in named queue: %d, nValidAtHeight in height queue: %d current height: %d\n", __func__, normalizedName, outPoint.hash.GetHex(), outPoint.n, nValidAtHeight, itRow->second.nValidAtHeight, nCurrentHeight); } return true; } } } - LogPrintf("%s: An inconsistency was found in the claim queue. Please report this to the developers:\nFound in named queue but not in height queue: name: %s, txid: %s, nOut: %d, nValidAtHeight: %d, current height: %d\n", __func__, name, outPoint.hash.GetHex(), outPoint.n, nValidAtHeight, nCurrentHeight); + LogPrintf("%s: An inconsistency was found in the claim queue. Please report this to the developers:\nFound in named queue but not in height queue: name: %s, txid: %s, nOut: %d, nValidAtHeight: %d, current height: %d\n", __func__, normalizedName, outPoint.hash.GetHex(), outPoint.n, nValidAtHeight, nCurrentHeight); return false; } bool CClaimTrie::haveSupportInQueue(const std::string& name, const COutPoint& outPoint, int& nValidAtHeight) const { queueNameRowType nameRow; - if (!getSupportQueueNameRow(name, nameRow)) + const std::string normalizedName = lbryNormalize(name, normalizeNames); + if (!getSupportQueueNameRow(normalizedName, nameRow)) { return false; } @@ -326,17 +336,17 @@ bool CClaimTrie::haveSupportInQueue(const std::string& name, const COutPoint& ou { for (supportQueueRowType::const_iterator itRow = row.begin(); itRow != row.end(); ++itRow) { - if (itRow->first == name && itRow->second.outPoint == outPoint) + if (itRow->first == normalizedName && itRow->second.outPoint == outPoint) { if (itRow->second.nValidAtHeight != nValidAtHeight) { - LogPrintf("%s: An inconsistency was found in the support queue. Please report this to the developers:\nDifferent nValidAtHeight between named queue and height queue\n: name: %s, txid: %s, nOut: %d, nValidAtHeight in named queue: %d, nValidAtHeight in height queue: %d current height: %d\n", __func__, name, outPoint.hash.GetHex(), outPoint.n, nValidAtHeight, itRow->second.nValidAtHeight, nCurrentHeight); + LogPrintf("%s: An inconsistency was found in the support queue. Please report this to the developers:\nDifferent nValidAtHeight between named queue and height queue\n: name: %s, txid: %s, nOut: %d, nValidAtHeight in named queue: %d, nValidAtHeight in height queue: %d current height: %d\n", __func__, normalizedName, outPoint.hash.GetHex(), outPoint.n, nValidAtHeight, itRow->second.nValidAtHeight, nCurrentHeight); } return true; } } } - LogPrintf("%s: An inconsistency was found in the claim queue. Please report this to the developers:\nFound in named queue but not in height queue: name: %s, txid: %s, nOut: %d, nValidAtHeight: %d, current height: %d\n", __func__, name, outPoint.hash.GetHex(), outPoint.n, nValidAtHeight, nCurrentHeight); + LogPrintf("%s: An inconsistency was found in the claim queue. Please report this to the developers:\nFound in named queue but not in height queue: name: %s, txid: %s, nOut: %d, nValidAtHeight: %d, current height: %d\n", __func__, normalizedName, outPoint.hash.GetHex(), outPoint.n, nValidAtHeight, nCurrentHeight); return false; } @@ -404,12 +414,13 @@ CAmount CClaimTrie::getTotalValueOfClaimsRecursive(const CClaimTrieNode* current bool CClaimTrie::recursiveFlattenTrie(const std::string& name, const CClaimTrieNode* current, std::vector& nodes) const { - namedNodeType node(name, *current); + const std::string normalizedName = lbryNormalize(name, normalizeNames); + namedNodeType node(normalizedName, *current); nodes.push_back(node); for (nodeMapType::const_iterator it = current->children.begin(); it != current->children.end(); ++it) { std::stringstream ss; - ss << name << it->first; + ss << normalizedName << it->first; if (!recursiveFlattenTrie(ss.str(), it->second, nodes)) return false; } @@ -427,7 +438,8 @@ std::vector CClaimTrie::flattenTrie() const const CClaimTrieNode* CClaimTrie::getNodeForName(const std::string& name) const { const CClaimTrieNode* current = &root; - for (std::string::const_iterator itname = name.begin(); itname != name.end(); ++itname) + const std::string normalizedName = lbryNormalize(name, normalizeNames); + for (std::string::const_iterator itname = normalizedName.begin(); itname != normalizedName.end(); ++itname) { nodeMapType::const_iterator itchildren = current->children.find(*itname); if (itchildren == current->children.end()) @@ -439,7 +451,8 @@ const CClaimTrieNode* CClaimTrie::getNodeForName(const std::string& name) const bool CClaimTrie::getInfoForName(const std::string& name, CClaimValue& claim) const { - const CClaimTrieNode* current = getNodeForName(name); + const std::string normalizedName = lbryNormalize(name, normalizeNames); + const CClaimTrieNode* current = getNodeForName(normalizedName); if (current) { return current->getBestClaim(claim); @@ -449,7 +462,8 @@ bool CClaimTrie::getInfoForName(const std::string& name, CClaimValue& claim) con bool CClaimTrie::getLastTakeoverForName(const std::string& name, int& lastTakeoverHeight) const { - const CClaimTrieNode* current = getNodeForName(name); + const std::string normalizedName = lbryNormalize(name, normalizeNames); + const CClaimTrieNode* current = getNodeForName(normalizedName); if (current && !current->claims.empty()) { lastTakeoverHeight = current->nHeightOfLastTakeover; @@ -463,7 +477,8 @@ claimsForNameType CClaimTrie::getClaimsForName(const std::string& name) const std::vector claims; std::vector supports; int nLastTakeoverHeight = 0; - const CClaimTrieNode* current = getNodeForName(name); + const std::string normalizedName = lbryNormalize(name, normalizeNames); + const CClaimTrieNode* current = getNodeForName(normalizedName); if (current) { if (!current->claims.empty()) @@ -493,7 +508,7 @@ claimsForNameType CClaimTrie::getClaimsForName(const std::string& name) const { for (claimQueueRowType::const_iterator itClaimRow = claimRow.begin(); itClaimRow != claimRow.end(); ++itClaimRow) { - if (itClaimRow->first == name && itClaimRow->second.outPoint == itClaimsForName->outPoint) + if (itClaimRow->first == normalizedName && itClaimRow->second.outPoint == itClaimsForName->outPoint) { claims.push_back(itClaimRow->second); break; @@ -503,7 +518,7 @@ claimsForNameType CClaimTrie::getClaimsForName(const std::string& name) const } } queueNameRowType namedSupportRow; - if (getSupportQueueNameRow(name, namedSupportRow)) + if (getSupportQueueNameRow(normalizedName, namedSupportRow)) { for (queueNameRowType::const_iterator itSupportsForName = namedSupportRow.begin(); itSupportsForName != namedSupportRow.end(); ++itSupportsForName) { @@ -512,7 +527,7 @@ claimsForNameType CClaimTrie::getClaimsForName(const std::string& name) const { for (supportQueueRowType::const_iterator itSupportRow = supportRow.begin(); itSupportRow != supportRow.end(); ++itSupportRow) { - if (itSupportRow->first == name && itSupportRow->second.outPoint == itSupportsForName->outPoint) + if (itSupportRow->first == normalizedName && itSupportRow->second.outPoint == itSupportsForName->outPoint) { supports.push_back(itSupportRow->second); break; @@ -528,26 +543,26 @@ claimsForNameType CClaimTrie::getClaimsForName(const std::string& name) const //return effective amount form claim, retuns 0 if claim is not found CAmount CClaimTrie::getEffectiveAmountForClaim(const std::string& name, uint160 claimId) const { - claimsForNameType claims = getClaimsForName(name); - CAmount effectiveAmount = 0; - bool claim_found = false; - for (std::vector::iterator it=claims.claims.begin(); it!=claims.claims.end(); ++it) - { - if (it->claimId == claimId && it->nValidAtHeight < nCurrentHeight) - effectiveAmount += it->nAmount; - claim_found = true; - break; - } - if (!claim_found) - return effectiveAmount; - - for (std::vector::iterator it=claims.supports.begin(); it!=claims.supports.end(); ++it) - { - if (it->supportedClaimId == claimId && it->nValidAtHeight < nCurrentHeight) - effectiveAmount += it->nAmount; - } - return effectiveAmount; + const std::string normalizedName = lbryNormalize(name, normalizeNames); + claimsForNameType claims = getClaimsForName(normalizedName); + CAmount effectiveAmount = 0; + bool claim_found = false; + for (std::vector::iterator it=claims.claims.begin(); it!=claims.claims.end(); ++it) + { + if (it->claimId == claimId && it->nValidAtHeight < nCurrentHeight) + effectiveAmount += it->nAmount; + claim_found = true; + break; + } + if (!claim_found) + return effectiveAmount; + for (std::vector::iterator it=claims.supports.begin(); it!=claims.supports.end(); ++it) + { + if (it->supportedClaimId == claimId && it->nValidAtHeight < nCurrentHeight) + effectiveAmount += it->nAmount; + } + return effectiveAmount; } bool CClaimTrie::checkConsistency() const @@ -557,6 +572,14 @@ bool CClaimTrie::checkConsistency() const return recursiveCheckConsistency(&root); } +bool CClaimTrie::shouldNormalize(int nHeight) const +{ + if (nHeight == 0) + return normalizeNames; + + return nHeight >= LBRY_NORMALIZED_NAME_FORK_HEIGHT; +} + bool CClaimTrie::recursiveCheckConsistency(const CClaimTrieNode* node) const { std::vector vchToHash; @@ -602,13 +625,26 @@ bool CClaimTrie::getQueueRow(int nHeight, claimQueueRowType& row) const bool CClaimTrie::getQueueNameRow(const std::string& name, queueNameRowType& row) const { + // to properly bound around the fork height, we have to check both + // non-normalized and normalized names here queueNameType::const_iterator itQueueNameRow = dirtyQueueNameRows.find(name); if (itQueueNameRow != dirtyQueueNameRows.end()) { row = itQueueNameRow->second; return true; } - return db.Read(std::make_pair(CLAIM_QUEUE_NAME_ROW, name), row); + const std::string normalizedName = lbryNormalize(name, normalizeNames); + queueNameType::const_iterator itQueueNormalizedNameRow = dirtyQueueNameRows.find(normalizedName); + if (itQueueNormalizedNameRow != dirtyQueueNameRows.end()) + { + row = itQueueNameRow->second; + return true; + } + if (db.Read(std::make_pair(CLAIM_QUEUE_NAME_ROW, name), row)) + { + return true; + } + return db.Read(std::make_pair(CLAIM_QUEUE_NAME_ROW, normalizedName), row); } bool CClaimTrie::getExpirationQueueRow(int nHeight, expirationQueueRowType& row) const @@ -638,12 +674,13 @@ void CClaimTrie::updateQueueRow(int nHeight, claimQueueRowType& row) void CClaimTrie::updateQueueNameRow(const std::string& name, queueNameRowType& row) { - queueNameType::iterator itQueueRow = dirtyQueueNameRows.find(name); + const std::string normalizedName = lbryNormalize(name, normalizeNames); + queueNameType::iterator itQueueRow = dirtyQueueNameRows.find(normalizedName); if (itQueueRow == dirtyQueueNameRows.end()) { queueNameRowType newRow; std::pair ret; - ret = dirtyQueueNameRows.insert(std::pair(name, newRow)); + ret = dirtyQueueNameRows.insert(std::pair(normalizedName, newRow)); assert(ret.second); itQueueRow = ret.first; } @@ -666,12 +703,13 @@ void CClaimTrie::updateExpirationRow(int nHeight, expirationQueueRowType& row) void CClaimTrie::updateSupportMap(const std::string& name, supportMapEntryType& node) { - supportMapType::iterator itNode = dirtySupportNodes.find(name); + const std::string normalizedName = lbryNormalize(name, normalizeNames); + supportMapType::iterator itNode = dirtySupportNodes.find(normalizedName); if (itNode == dirtySupportNodes.end()) { supportMapEntryType newNode; std::pair ret; - ret = dirtySupportNodes.insert(std::pair(name, newNode)); + ret = dirtySupportNodes.insert(std::pair(normalizedName, newNode)); assert(ret.second); itNode = ret.first; } @@ -694,12 +732,13 @@ void CClaimTrie::updateSupportQueue(int nHeight, supportQueueRowType& row) void CClaimTrie::updateSupportNameQueue(const std::string& name, queueNameRowType& row) { - queueNameType::iterator itQueueRow = dirtySupportQueueNameRows.find(name); + const std::string normalizedName = lbryNormalize(name, normalizeNames); + queueNameType::iterator itQueueRow = dirtySupportQueueNameRows.find(normalizedName); if (itQueueRow == dirtySupportQueueNameRows.end()) { queueNameRowType newRow; std::pair ret; - ret = dirtySupportQueueNameRows.insert(std::pair(name, newRow)); + ret = dirtySupportQueueNameRows.insert(std::pair(normalizedName, newRow)); assert(ret.second); itQueueRow = ret.first; } @@ -722,13 +761,26 @@ void CClaimTrie::updateSupportExpirationQueue(int nHeight, expirationQueueRowTyp bool CClaimTrie::getSupportNode(std::string name, supportMapEntryType& node) const { + // to properly bound around the fork height, we have to check both + // non-normalized and normalized names here supportMapType::const_iterator itNode = dirtySupportNodes.find(name); if (itNode != dirtySupportNodes.end()) { node = itNode->second; return true; } - return db.Read(std::make_pair(SUPPORT, name), node); + const std::string normalizedName = lbryNormalize(name, normalizeNames); + supportMapType::const_iterator itNormalizedNode = dirtySupportNodes.find(normalizedName); + if (itNormalizedNode != dirtySupportNodes.end()) + { + node = itNode->second; + return true; + } + if (db.Read(std::make_pair(SUPPORT, name), node)) + { + return true; + } + return db.Read(std::make_pair(SUPPORT, normalizedName), node); } bool CClaimTrie::getSupportQueueRow(int nHeight, supportQueueRowType& row) const @@ -744,13 +796,26 @@ bool CClaimTrie::getSupportQueueRow(int nHeight, supportQueueRowType& row) const bool CClaimTrie::getSupportQueueNameRow(const std::string& name, queueNameRowType& row) const { + // to properly bound around the fork height, we have to check both + // non-normalized and normalized names here queueNameType::const_iterator itQueueNameRow = dirtySupportQueueNameRows.find(name); if (itQueueNameRow != dirtySupportQueueNameRows.end()) { row = itQueueNameRow->second; return true; } - return db.Read(std::make_pair(SUPPORT_QUEUE_NAME_ROW, name), row); + const std::string normalizedName = lbryNormalize(name, normalizeNames); + queueNameType::const_iterator itQueueNormalizedNameRow = dirtySupportQueueNameRows.find(normalizedName); + if (itQueueNormalizedNameRow != dirtySupportQueueNameRows.end()) + { + row = itQueueNameRow->second; + return true; + } + if (db.Read(std::make_pair(SUPPORT_QUEUE_NAME_ROW, name), row)) + { + return true; + } + return db.Read(std::make_pair(SUPPORT_QUEUE_NAME_ROW, normalizedName), row); } bool CClaimTrie::getSupportExpirationQueueRow(int nHeight, expirationQueueRowType& row) const @@ -820,6 +885,7 @@ bool CClaimTrie::update(nodeCacheType& cache, hashMapType& hashes, std::map= LBRY_NORMALIZED_NAME_FORK_HEIGHT); return true; } @@ -834,12 +900,13 @@ void CClaimTrie::markNodeDirty(const std::string &name, CClaimTrieNode* node) bool CClaimTrie::updateName(const std::string &name, CClaimTrieNode* updatedNode) { CClaimTrieNode* current = &root; - for (std::string::const_iterator itname = name.begin(); itname != name.end(); ++itname) + const std::string normalizedName = lbryNormalize(name, normalizeNames); + for (std::string::const_iterator itname = normalizedName.begin(); itname != normalizedName.end(); ++itname) { nodeMapType::iterator itchild = current->children.find(*itname); if (itchild == current->children.end()) { - if (itname + 1 == name.end()) + if (itname + 1 == normalizedName.end()) { CClaimTrieNode* newNode = new CClaimTrieNode(); current->children[*itname] = newNode; @@ -855,7 +922,7 @@ bool CClaimTrie::updateName(const std::string &name, CClaimTrieNode* updatedNode } assert(current != NULL); current->claims.swap(updatedNode->claims); - markNodeDirty(name, current); + markNodeDirty(normalizedName, current); for (nodeMapType::iterator itchild = current->children.begin(); itchild != current->children.end();) { nodeMapType::iterator itupdatechild = updatedNode->children.find(itchild->first); @@ -864,7 +931,7 @@ bool CClaimTrie::updateName(const std::string &name, CClaimTrieNode* updatedNode // This character has apparently been deleted, so delete // all descendents from this child. std::stringstream ss; - ss << name << itchild->first; + ss << normalizedName << itchild->first; std::string newName = ss.str(); if (!recursiveNullify(itchild->second, newName)) return false; @@ -879,16 +946,17 @@ bool CClaimTrie::updateName(const std::string &name, CClaimTrieNode* updatedNode bool CClaimTrie::recursiveNullify(CClaimTrieNode* node, std::string& name) { assert(node != NULL); + const std::string normalizedName = lbryNormalize(name, normalizeNames); for (nodeMapType::iterator itchild = node->children.begin(); itchild != node->children.end(); ++itchild) { std::stringstream ss; - ss << name << itchild->first; + ss << normalizedName << itchild->first; std::string newName = ss.str(); if (!recursiveNullify(itchild->second, newName)) return false; } node->children.clear(); - markNodeDirty(name, NULL); + markNodeDirty(normalizedName, NULL); delete node; return true; } @@ -896,7 +964,8 @@ bool CClaimTrie::recursiveNullify(CClaimTrieNode* node, std::string& name) bool CClaimTrie::updateHash(const std::string& name, uint256& hash) { CClaimTrieNode* current = &root; - for (std::string::const_iterator itname = name.begin(); itname != name.end(); ++itname) + const std::string normalizedName = lbryNormalize(name, normalizeNames); + for (std::string::const_iterator itname = normalizedName.begin(); itname != normalizedName.end(); ++itname) { nodeMapType::iterator itchild = current->children.find(*itname); if (itchild == current->children.end()) @@ -905,14 +974,15 @@ bool CClaimTrie::updateHash(const std::string& name, uint256& hash) } assert(current != NULL); current->hash = hash; - markNodeDirty(name, current); + markNodeDirty(normalizedName, current); return true; } bool CClaimTrie::updateTakeoverHeight(const std::string& name, int nTakeoverHeight) { CClaimTrieNode* current = &root; - for (std::string::const_iterator itname = name.begin(); itname != name.end(); ++itname) + const std::string normalizedName = lbryNormalize(name, CHECK_HEIGHT(nTakeoverHeight)); + for (std::string::const_iterator itname = normalizedName.begin(); itname != normalizedName.end(); ++itname) { nodeMapType::iterator itchild = current->children.find(*itname); if (itchild == current->children.end()) @@ -921,7 +991,7 @@ bool CClaimTrie::updateTakeoverHeight(const std::string& name, int nTakeoverHeig } assert(current != NULL); current->nHeightOfLastTakeover = nTakeoverHeight; - markNodeDirty(name, current); + markNodeDirty(normalizedName, current); return true; } @@ -930,11 +1000,12 @@ void CClaimTrie::BatchWriteNode(CDBBatch& batch, const std::string& name, const uint32_t num_claims = 0; if (pNode) num_claims = pNode->claims.size(); - LogPrintf("%s: Writing %s to disk with %d claims\n", __func__, name, num_claims); + const std::string normalizedName = lbryNormalize(name, normalizeNames); + LogPrintf("%s: Writing %s to disk with %d claims\n", __func__, normalizedName, num_claims); if (pNode) - batch.Write(std::make_pair(TRIE_NODE, name), *pNode); + batch.Write(std::make_pair(TRIE_NODE, normalizedName), *pNode); else - batch.Erase(std::make_pair(TRIE_NODE, name)); + batch.Erase(std::make_pair(TRIE_NODE, normalizedName)); } void CClaimTrie::BatchWriteQueueRows(CDBBatch& batch) @@ -1069,20 +1140,21 @@ bool CClaimTrie::WriteToDisk() bool CClaimTrie::InsertFromDisk(const std::string& name, CClaimTrieNode* node) { - if (name.size() == 0) + const std::string normalizedName = lbryNormalize(name, normalizeNames); + if (normalizedName.empty()) { root = *node; return true; } CClaimTrieNode* current = &root; - for (std::string::const_iterator itname = name.begin(); itname + 1 != name.end(); ++itname) + for (std::string::const_iterator itname = normalizedName.begin(); itname + 1 != normalizedName.end(); ++itname) { nodeMapType::iterator itchild = current->children.find(*itname); if (itchild == current->children.end()) return false; current = itchild->second; } - current->children[name[name.size()-1]] = node; + current->children[normalizedName[normalizedName.size()-1]] = node; return true; } @@ -1092,6 +1164,11 @@ bool CClaimTrie::ReadFromDisk(bool check) LogPrintf("%s: Couldn't read the best block's hash\n", __func__); if (!db.Read(CURRENT_HEIGHT, nCurrentHeight)) LogPrintf("%s: Couldn't read the current height\n", __func__); + + normalizeNames = nCurrentHeight >= LBRY_NORMALIZED_NAME_FORK_HEIGHT; + LogPrintf("%s: Current height %lu, normalization fork is %s\n", + __func__, nCurrentHeight, normalizeNames ? "ENABLED" : "DISABLED"); + boost::scoped_ptr pcursor(const_cast(&db)->NewIterator()); pcursor->SeekToFirst(); @@ -1142,7 +1219,6 @@ bool CClaimTrieCache::recursiveComputeMerkleHash(CClaimTrieNode* tnCurrent, std: std::vector vchToHash; nodeCacheType::iterator cachedNode; - for (nodeMapType::iterator it = tnCurrent->children.begin(); it != tnCurrent->children.end(); ++it) { std::stringstream ss; @@ -1245,10 +1321,11 @@ CClaimTrieNode* CClaimTrieCache::addNodeToCache(const std::string& position, CCl bool CClaimTrieCache::getOriginalInfoForName(const std::string& name, CClaimValue& claim) const { - nodeCacheType::const_iterator itOriginalCache = block_originals.find(name); + const std::string normalizedName = lbryNormalize(name, BASE_CHECK_HEIGHT(claim.nValidAtHeight)); + nodeCacheType::const_iterator itOriginalCache = block_originals.find(normalizedName); if (itOriginalCache == block_originals.end()) { - return base->getInfoForName(name, claim); + return base->getInfoForName(normalizedName, claim); } return itOriginalCache->second->getBestClaim(claim); } @@ -1261,10 +1338,11 @@ bool CClaimTrieCache::insertClaimIntoTrie(const std::string& name, CClaimValue c cachedNode = cache.find(""); if (cachedNode != cache.end()) currentNode = cachedNode->second; - for (std::string::const_iterator itCur = name.begin(); itCur != name.end(); ++itCur) + const std::string normalizedName = lbryNormalize(name, BASE_CHECK_HEIGHT(claim.nValidAtHeight)); + for (std::string::const_iterator itCur = normalizedName.begin(); itCur != normalizedName.end(); ++itCur) { - std::string sCurrentSubstring(name.begin(), itCur); - std::string sNextSubstring(name.begin(), itCur + 1); + std::string sCurrentSubstring(normalizedName.begin(), itCur); + std::string sNextSubstring(normalizedName.begin(), itCur + 1); cachedNode = cache.find(sNextSubstring); if (cachedNode != cache.end()) @@ -1302,14 +1380,14 @@ bool CClaimTrieCache::insertClaimIntoTrie(const std::string& name, CClaimValue c currentNode = newNode; } - cachedNode = cache.find(name); + cachedNode = cache.find(normalizedName); if (cachedNode != cache.end()) { assert(cachedNode->second == currentNode); } else { - currentNode = addNodeToCache(name, currentNode); + currentNode = addNodeToCache(normalizedName, currentNode); } bool fChanged = false; if (currentNode->claims.empty()) @@ -1322,21 +1400,21 @@ bool CClaimTrieCache::insertClaimIntoTrie(const std::string& name, CClaimValue c CClaimValue currentTop = currentNode->claims.front(); currentNode->insertClaim(claim); supportMapEntryType node; - getSupportsForName(name, node); + getSupportsForName(normalizedName, node); currentNode->reorderClaims(node); if (currentTop != currentNode->claims.front()) fChanged = true; } if (fChanged) { - for (std::string::const_iterator itCur = name.begin(); itCur != name.end(); ++itCur) + for (std::string::const_iterator itCur = normalizedName.begin(); itCur != normalizedName.end(); ++itCur) { - std::string sub(name.begin(), itCur); + std::string sub(normalizedName.begin(), itCur); dirtyHashes.insert(sub); } - dirtyHashes.insert(name); + dirtyHashes.insert(normalizedName); if (fCheckTakeover) - namesToCheckForTakeover.insert(name); + namesToCheckForTakeover.insert(normalizedName); } return true; } @@ -1350,10 +1428,11 @@ bool CClaimTrieCache::removeClaimFromTrie(const std::string& name, const COutPoi if (cachedNode != cache.end()) currentNode = cachedNode->second; assert(currentNode != NULL); // If there is no root in either the trie or the cache, how can there be any names to remove? - for (std::string::const_iterator itCur = name.begin(); itCur != name.end(); ++itCur) + const std::string normalizedName = lbryNormalize(name, BASE_CHECK_HEIGHT(claim.nValidAtHeight)); + for (std::string::const_iterator itCur = normalizedName.begin(); itCur != normalizedName.end(); ++itCur) { - std::string sCurrentSubstring(name.begin(), itCur); - std::string sNextSubstring(name.begin(), itCur + 1); + std::string sCurrentSubstring(normalizedName.begin(), itCur); + std::string sNextSubstring(normalizedName.begin(), itCur + 1); cachedNode = cache.find(sNextSubstring); if (cachedNode != cache.end()) @@ -1367,16 +1446,16 @@ bool CClaimTrieCache::removeClaimFromTrie(const std::string& name, const COutPoi currentNode = childNode->second; continue; } - LogPrintf("%s: The name %s does not exist in the trie\n", __func__, name.c_str()); + LogPrintf("%s: The name %s does not exist in the trie\n", __func__, normalizedName.c_str()); return false; } - cachedNode = cache.find(name); + cachedNode = cache.find(normalizedName); if (cachedNode != cache.end()) assert(cachedNode->second == currentNode); else { - currentNode = addNodeToCache(name, currentNode); + currentNode = addNodeToCache(normalizedName, currentNode); } bool fChanged = false; assert(currentNode != NULL); @@ -1394,7 +1473,7 @@ bool CClaimTrieCache::removeClaimFromTrie(const std::string& name, const COutPoi if (!currentNode->claims.empty()) { supportMapEntryType node; - getSupportsForName(name, node); + getSupportsForName(normalizedName, node); currentNode->reorderClaims(node); if (currentTop != currentNode->claims.front()) fChanged = true; @@ -1404,36 +1483,37 @@ bool CClaimTrieCache::removeClaimFromTrie(const std::string& name, const COutPoi if (!success) { - LogPrintf("%s: Removing a claim was unsuccessful. name = %s, txhash = %s, nOut = %d", __func__, name.c_str(), outPoint.hash.GetHex(), outPoint.n); + LogPrintf("%s: Removing a claim was unsuccessful. name = %s, txhash = %s, nOut = %d", __func__, normalizedName.c_str(), outPoint.hash.GetHex(), outPoint.n); return false; } if (fChanged) { - for (std::string::const_iterator itCur = name.begin(); itCur != name.end(); ++itCur) + for (std::string::const_iterator itCur = normalizedName.begin(); itCur != normalizedName.end(); ++itCur) { - std::string sub(name.begin(), itCur); + std::string sub(normalizedName.begin(), itCur); dirtyHashes.insert(sub); } - dirtyHashes.insert(name); + dirtyHashes.insert(normalizedName); if (fCheckTakeover) - namesToCheckForTakeover.insert(name); + namesToCheckForTakeover.insert(normalizedName); } CClaimTrieNode* rootNode = &(base->root); cachedNode = cache.find(""); if (cachedNode != cache.end()) rootNode = cachedNode->second; - return recursivePruneName(rootNode, 0, name); + return recursivePruneName(rootNode, 0, normalizedName); } bool CClaimTrieCache::recursivePruneName(CClaimTrieNode* tnCurrent, unsigned int nPos, std::string sName, bool* pfNullified) const { bool fNullified = false; - std::string sCurrentSubstring = sName.substr(0, nPos); - if (nPos < sName.size()) + const std::string normalizedName = lbryNormalize(sName, base->shouldNormalize()); + std::string sCurrentSubstring = normalizedName.substr(0, nPos); + if (nPos < normalizedName.size()) { - std::string sNextSubstring = sName.substr(0, nPos + 1); - unsigned char cNext = sName.at(nPos); + std::string sNextSubstring = normalizedName.substr(0, nPos + 1); + unsigned char cNext = normalizedName.at(nPos); CClaimTrieNode* tnNext = NULL; nodeCacheType::iterator cachedNode = cache.find(sNextSubstring); if (cachedNode != cache.end()) @@ -1447,7 +1527,7 @@ bool CClaimTrieCache::recursivePruneName(CClaimTrieNode* tnCurrent, unsigned int if (tnNext == NULL) return false; bool fChildNullified = false; - if (!recursivePruneName(tnNext, nPos + 1, sName, &fChildNullified)) + if (!recursivePruneName(tnNext, nPos + 1, normalizedName, &fChildNullified)) return false; if (fChildNullified) { @@ -1520,7 +1600,8 @@ claimQueueType::iterator CClaimTrieCache::getQueueCacheRow(int nHeight, bool cre queueNameType::iterator CClaimTrieCache::getQueueCacheNameRow(const std::string& name, bool createIfNotExists) const { - queueNameType::iterator itQueueNameRow = claimQueueNameCache.find(name); + const std::string normalizedName = lbryNormalize(name, base->shouldNormalize()); + queueNameType::iterator itQueueNameRow = claimQueueNameCache.find(normalizedName); if (itQueueNameRow == claimQueueNameCache.end()) { // Have to make a new name row and put it in the cache, if createIfNotExists is true @@ -1532,7 +1613,7 @@ queueNameType::iterator CClaimTrieCache::getQueueCacheNameRow(const std::string& return itQueueNameRow; // Stick the new row in the cache std::pair ret; - ret = claimQueueNameCache.insert(std::pair(name, queueNameRow)); + ret = claimQueueNameCache.insert(std::pair(normalizedName, queueNameRow)); assert(ret.second); itQueueNameRow = ret.first; } @@ -1541,58 +1622,70 @@ queueNameType::iterator CClaimTrieCache::getQueueCacheNameRow(const std::string& bool CClaimTrieCache::addClaim(const std::string& name, const COutPoint& outPoint, uint160 claimId, CAmount nAmount, int nHeight) const { - LogPrintf("%s: name: %s, txhash: %s, nOut: %d, claimId: %s, nAmount: %d, nHeight: %d, nCurrentHeight: %d\n", __func__, name, outPoint.hash.GetHex(), outPoint.n, claimId.GetHex(), nAmount, nHeight, nCurrentHeight); + const std::string normalizedName = lbryNormalize(name, BASE_CHECK_HEIGHT(nHeight)); + LogPrintf("%s: name: %s, txhash: %s, nOut: %d, claimId: %s, nAmount: %d, nHeight: %d, nCurrentHeight: %d\n", __func__, normalizedName, outPoint.hash.GetHex(), outPoint.n, claimId.GetHex(), nAmount, nHeight, nCurrentHeight); assert(nHeight == nCurrentHeight); CClaimValue currentClaim; int delayForClaim; - if (getOriginalInfoForName(name, currentClaim) && currentClaim.claimId == claimId) + if (getOriginalInfoForName(normalizedName, currentClaim) && currentClaim.claimId == claimId) { LogPrintf("%s: This is an update to a best claim.\n", __func__); delayForClaim = 0; } else { - delayForClaim = getDelayForName(name); + delayForClaim = getDelayForName(normalizedName); } CClaimValue newClaim(outPoint, claimId, nAmount, nHeight, nHeight + delayForClaim); - return addClaimToQueues(name, newClaim); + return addClaimToQueues(normalizedName, newClaim); } bool CClaimTrieCache::undoSpendClaim(const std::string& name, const COutPoint& outPoint, uint160 claimId, CAmount nAmount, int nHeight, int nValidAtHeight) const { - LogPrintf("%s: name: %s, txhash: %s, nOut: %d, claimId: %s, nAmount: %d, nHeight: %d, nValidAtHeight: %d, nCurrentHeight: %d\n", __func__, name, outPoint.hash.GetHex(), outPoint.n, claimId.GetHex(), nAmount, nHeight, nValidAtHeight, nCurrentHeight); + const std::string normalizedName = lbryNormalize(name, BASE_CHECK_HEIGHT(nValidAtHeight)); + LogPrintf("%s: name: %s, txhash: %s, nOut: %d, claimId: %s, nAmount: %d, nHeight: %d, nValidAtHeight: %d, nCurrentHeight: %d\n", __func__, normalizedName, outPoint.hash.GetHex(), outPoint.n, claimId.GetHex(), nAmount, nHeight, nValidAtHeight, nCurrentHeight); CClaimValue claim(outPoint, claimId, nAmount, nHeight, nValidAtHeight); if (nValidAtHeight < nCurrentHeight) { - nameOutPointType entry(name, claim.outPoint); + nameOutPointType entry(normalizedName, claim.outPoint); addToExpirationQueue(claim.nHeight + base->nExpirationTime, entry); - return insertClaimIntoTrie(name, claim, false); + return insertClaimIntoTrie(normalizedName, claim, false); } else { - return addClaimToQueues(name, claim); + return addClaimToQueues(normalizedName, claim); } } bool CClaimTrieCache::addClaimToQueues(const std::string& name, CClaimValue& claim) const { LogPrintf("%s: nValidAtHeight: %d\n", __func__, claim.nValidAtHeight); - claimQueueEntryType entry(name, claim); + const std::string normalizedName = lbryNormalize(name, BASE_CHECK_HEIGHT(claim.nValidAtHeight)); + claimQueueEntryType entry(normalizedName, claim); claimQueueType::iterator itQueueRow = getQueueCacheRow(claim.nValidAtHeight, true); - queueNameType::iterator itQueueNameRow = getQueueCacheNameRow(name, true); + queueNameType::iterator itQueueNameRow = getQueueCacheNameRow(normalizedName, true); itQueueRow->second.push_back(entry); itQueueNameRow->second.push_back(outPointHeightType(claim.outPoint, claim.nValidAtHeight)); - nameOutPointType expireEntry(name, claim.outPoint); + nameOutPointType expireEntry(normalizedName, claim.outPoint); addToExpirationQueue(claim.nHeight + base->nExpirationTime, expireEntry); return true; } bool CClaimTrieCache::removeClaimFromQueue(const std::string& name, const COutPoint& outPoint, CClaimValue& claim) const { + // to properly bound around the fork height, we have to check both + // non-normalized and normalized names here + bool normalizedName_found = false; + const std::string normalizedName = lbryNormalize(name, base->shouldNormalize()); queueNameType::iterator itQueueNameRow = getQueueCacheNameRow(name, false); if (itQueueNameRow == claimQueueNameCache.end()) { - return false; + itQueueNameRow = getQueueCacheNameRow(normalizedName, false); + if (itQueueNameRow == claimQueueNameCache.end()) + { + return false; + } + normalizedName_found = true; } queueNameRowType::iterator itQueueName; for (itQueueName = itQueueNameRow->second.begin(); itQueueName != itQueueNameRow->second.end(); ++itQueueName) @@ -1612,9 +1705,19 @@ bool CClaimTrieCache::removeClaimFromQueue(const std::string& name, const COutPo claimQueueRowType::iterator itQueue; for (itQueue = itQueueRow->second.begin(); itQueue != itQueueRow->second.end(); ++itQueue) { - if (name == itQueue->first && itQueue->second.outPoint == outPoint) + if (normalizedName_found) { - break; + if (normalizedName == itQueue->first && itQueue->second.outPoint == outPoint) + { + break; + } + } + else + { + if (name == itQueue->first && itQueue->second.outPoint == outPoint) + { + break; + } } } if (itQueue != itQueueRow->second.end()) @@ -1625,7 +1728,7 @@ bool CClaimTrieCache::removeClaimFromQueue(const std::string& name, const COutPo return true; } } - LogPrintf("%s: An inconsistency was found in the claim queue. Please report this to the developers:\nFound in named queue but not in height queue: name: %s, txid: %s, nOut: %d, nValidAtHeight: %d, current height: %d\n", __func__, name, outPoint.hash.GetHex(), outPoint.n, itQueueName->nHeight, nCurrentHeight); + LogPrintf("%s: An inconsistency was found in the claim queue. Please report this to the developers:\nFound in named queue but not in height queue: name: %s, txid: %s, nOut: %d, nValidAtHeight: %d, current height: %d\n", __func__, (normalizedName_found ? normalizedName : name), outPoint.hash.GetHex(), outPoint.n, itQueueName->nHeight, nCurrentHeight); return false; } @@ -1672,11 +1775,12 @@ void CClaimTrieCache::removeFromExpirationQueue(const std::string& name, const C int expirationHeight = nHeight + base->nExpirationTime; expirationQueueType::iterator itQueueRow = getExpirationQueueCacheRow(expirationHeight, false); expirationQueueRowType::iterator itQueue; + const std::string normalizedName = lbryNormalize(name, BASE_CHECK_HEIGHT(expirationHeight)); if (itQueueRow != expirationQueueCache.end()) { for (itQueue = itQueueRow->second.begin(); itQueue != itQueueRow->second.end(); ++itQueue) { - if (name == itQueue->name && outPoint == itQueue->outPoint) + if (normalizedName == itQueue->name && outPoint == itQueue->outPoint) break; } } @@ -1710,8 +1814,9 @@ expirationQueueType::iterator CClaimTrieCache::getExpirationQueueCacheRow(int nH bool CClaimTrieCache::reorderTrieNode(const std::string& name, bool fCheckTakeover) const { assert(base); + const std::string normalizedName = lbryNormalize(name, base->shouldNormalize()); nodeCacheType::iterator cachedNode; - cachedNode = cache.find(name); + cachedNode = cache.find(normalizedName); if (cachedNode == cache.end()) { CClaimTrieNode* currentNode; @@ -1720,10 +1825,10 @@ bool CClaimTrieCache::reorderTrieNode(const std::string& name, bool fCheckTakeov currentNode = &(base->root); else currentNode = cachedNode->second; - for (std::string::const_iterator itCur = name.begin(); itCur != name.end(); ++itCur) + for (std::string::const_iterator itCur = normalizedName.begin(); itCur != normalizedName.end(); ++itCur) { - std::string sCurrentSubstring(name.begin(), itCur); - std::string sNextSubstring(name.begin(), itCur + 1); + std::string sCurrentSubstring(normalizedName.begin(), itCur); + std::string sNextSubstring(normalizedName.begin(), itCur + 1); cachedNode = cache.find(sNextSubstring); if (cachedNode != cache.end()) @@ -1742,7 +1847,7 @@ bool CClaimTrieCache::reorderTrieNode(const std::string& name, bool fCheckTakeov } currentNode = new CClaimTrieNode(*currentNode); std::pair ret; - ret = cache.insert(std::pair(name, currentNode)); + ret = cache.insert(std::pair(normalizedName, currentNode)); assert(ret.second); cachedNode = ret.first; } @@ -1756,21 +1861,21 @@ bool CClaimTrieCache::reorderTrieNode(const std::string& name, bool fCheckTakeov { CClaimValue currentTop = cachedNode->second->claims.front(); supportMapEntryType node; - getSupportsForName(name, node); + getSupportsForName(normalizedName, node); cachedNode->second->reorderClaims(node); if (cachedNode->second->claims.front() != currentTop) fChanged = true; } if (fChanged) { - for (std::string::const_iterator itCur = name.begin(); itCur != name.end(); ++itCur) + for (std::string::const_iterator itCur = normalizedName.begin(); itCur != normalizedName.end(); ++itCur) { - std::string sub(name.begin(), itCur); + std::string sub(normalizedName.begin(), itCur); dirtyHashes.insert(sub); } - dirtyHashes.insert(name); + dirtyHashes.insert(normalizedName); if (fCheckTakeover) - namesToCheckForTakeover.insert(name); + namesToCheckForTakeover.insert(normalizedName); } return true; } @@ -1792,38 +1897,40 @@ bool CClaimTrieCache::getSupportsForName(const std::string& name, supportMapEntr bool CClaimTrieCache::insertSupportIntoMap(const std::string& name, CSupportValue support, bool fCheckTakeover) const { + const std::string normalizedName = lbryNormalize(name, base->shouldNormalize()); supportMapType::iterator cachedNode; // If this node is already in the cache, use that - cachedNode = supportCache.find(name); + cachedNode = supportCache.find(normalizedName); // If not, copy the one from base if it exists, and use that if (cachedNode == supportCache.end()) { supportMapEntryType node; - base->getSupportNode(name, node); + base->getSupportNode(normalizedName, node); std::pair ret; - ret = supportCache.insert(std::pair(name, node)); + ret = supportCache.insert(std::pair(normalizedName, node)); assert(ret.second); cachedNode = ret.first; } cachedNode->second.push_back(support); // See if this changed the biggest bid - return reorderTrieNode(name, fCheckTakeover); + return reorderTrieNode(normalizedName, fCheckTakeover); } bool CClaimTrieCache::removeSupportFromMap(const std::string& name, const COutPoint& outPoint, CSupportValue& support, bool fCheckTakeover) const { + const std::string normalizedName = lbryNormalize(name, base->shouldNormalize()); supportMapType::iterator cachedNode; - cachedNode = supportCache.find(name); + cachedNode = supportCache.find(normalizedName); if (cachedNode == supportCache.end()) { supportMapEntryType node; - if (!base->getSupportNode(name, node)) + if (!base->getSupportNode(normalizedName, node)) { // clearly, this support does not exist return false; } std::pair ret; - ret = supportCache.insert(std::pair(name, node)); + ret = supportCache.insert(std::pair(normalizedName, node)); assert(ret.second); cachedNode = ret.first; } @@ -1839,7 +1946,7 @@ bool CClaimTrieCache::removeSupportFromMap(const std::string& name, const COutPo { std::swap(support, *itSupport); cachedNode->second.erase(itSupport); - return reorderTrieNode(name, fCheckTakeover); + return reorderTrieNode(normalizedName, fCheckTakeover); } else { @@ -1869,17 +1976,18 @@ supportQueueType::iterator CClaimTrieCache::getSupportQueueCacheRow(int nHeight, queueNameType::iterator CClaimTrieCache::getSupportQueueCacheNameRow(const std::string& name, bool createIfNotExists) const { - queueNameType::iterator itQueueNameRow = supportQueueNameCache.find(name); + const std::string normalizedName = lbryNormalize(name, base->shouldNormalize()); + queueNameType::iterator itQueueNameRow = supportQueueNameCache.find(normalizedName); if (itQueueNameRow == supportQueueNameCache.end()) { queueNameRowType queueNameRow; - bool exists = base->getSupportQueueNameRow(name, queueNameRow); + bool exists = base->getSupportQueueNameRow(normalizedName, queueNameRow); if (!exists) if (!createIfNotExists) return itQueueNameRow; // Stick the new row in the name cache std::pair ret; - ret = supportQueueNameCache.insert(std::pair(name, queueNameRow)); + ret = supportQueueNameCache.insert(std::pair(normalizedName, queueNameRow)); assert(ret.second); itQueueNameRow = ret.first; } @@ -1889,19 +1997,21 @@ queueNameType::iterator CClaimTrieCache::getSupportQueueCacheNameRow(const std:: bool CClaimTrieCache::addSupportToQueues(const std::string& name, CSupportValue& support) const { LogPrintf("%s: nValidAtHeight: %d\n", __func__, support.nValidAtHeight); - supportQueueEntryType entry(name, support); + const std::string normalizedName = lbryNormalize(name, base->shouldNormalize(support.nValidAtHeight)); + supportQueueEntryType entry(normalizedName, support); supportQueueType::iterator itQueueRow = getSupportQueueCacheRow(support.nValidAtHeight, true); - queueNameType::iterator itQueueNameRow = getSupportQueueCacheNameRow(name, true); + queueNameType::iterator itQueueNameRow = getSupportQueueCacheNameRow(normalizedName, true); itQueueRow->second.push_back(entry); itQueueNameRow->second.push_back(outPointHeightType(support.outPoint, support.nValidAtHeight)); - nameOutPointType expireEntry(name, support.outPoint); + nameOutPointType expireEntry(normalizedName, support.outPoint); addSupportToExpirationQueue(support.nHeight + base->nExpirationTime, expireEntry); return true; } bool CClaimTrieCache::removeSupportFromQueue(const std::string& name, const COutPoint& outPoint, CSupportValue& support) const { - queueNameType::iterator itQueueNameRow = getSupportQueueCacheNameRow(name, false); + const std::string normalizedName = lbryNormalize(name, base->shouldNormalize()); + queueNameType::iterator itQueueNameRow = getSupportQueueCacheNameRow(normalizedName, false); if (itQueueNameRow == supportQueueNameCache.end()) { return false; @@ -1925,7 +2035,7 @@ bool CClaimTrieCache::removeSupportFromQueue(const std::string& name, const COut for (itQueue = itQueueRow->second.begin(); itQueue != itQueueRow->second.end(); ++itQueue) { CSupportValue& support = itQueue->second; - if (name == itQueue->first && support.outPoint == outPoint) + if (normalizedName == itQueue->first && support.outPoint == outPoint) { break; } @@ -1938,42 +2048,44 @@ bool CClaimTrieCache::removeSupportFromQueue(const std::string& name, const COut return true; } } - LogPrintf("%s: An inconsistency was found in the claim queue. Please report this to the developers:\nFound in named support queue but not in height support queue: name: %s, txid: %s, nOut: %d, nValidAtHeight: %d, current height: %d\n", __func__, name, outPoint.hash.GetHex(), outPoint.n, itQueueName->nHeight, nCurrentHeight); + LogPrintf("%s: An inconsistency was found in the claim queue. Please report this to the developers:\nFound in named support queue but not in height support queue: name: %s, txid: %s, nOut: %d, nValidAtHeight: %d, current height: %d\n", __func__, normalizedName, outPoint.hash.GetHex(), outPoint.n, itQueueName->nHeight, nCurrentHeight); return false; } bool CClaimTrieCache::addSupport(const std::string& name, const COutPoint& outPoint, CAmount nAmount, uint160 supportedClaimId, int nHeight) const { - LogPrintf("%s: name: %s, txhash: %s, nOut: %d, nAmount: %d, supportedClaimId: %s, nHeight: %d, nCurrentHeight: %d\n", __func__, name, outPoint.hash.GetHex(), outPoint.n, nAmount, supportedClaimId.GetHex(), nHeight, nCurrentHeight); + const std::string normalizedName = lbryNormalize(name, base->shouldNormalize()); + LogPrintf("%s: name: %s, txhash: %s, nOut: %d, nAmount: %d, supportedClaimId: %s, nHeight: %d, nCurrentHeight: %d\n", __func__, normalizedName, outPoint.hash.GetHex(), outPoint.n, nAmount, supportedClaimId.GetHex(), nHeight, nCurrentHeight); assert(nHeight == nCurrentHeight); CClaimValue claim; int delayForSupport; - if (getOriginalInfoForName(name, claim) && claim.claimId == supportedClaimId) + if (getOriginalInfoForName(normalizedName, claim) && claim.claimId == supportedClaimId) { LogPrintf("%s: This is a support to a best claim.\n", __func__); delayForSupport = 0; } else { - delayForSupport = getDelayForName(name); + delayForSupport = getDelayForName(normalizedName); } CSupportValue support(outPoint, supportedClaimId, nAmount, nHeight, nHeight + delayForSupport); - return addSupportToQueues(name, support); + return addSupportToQueues(normalizedName, support); } bool CClaimTrieCache::undoSpendSupport(const std::string& name, const COutPoint& outPoint, uint160 supportedClaimId, CAmount nAmount, int nHeight, int nValidAtHeight) const { - LogPrintf("%s: name: %s, txhash: %s, nOut: %d, nAmount: %d, supportedClaimId: %s, nHeight: %d, nCurrentHeight: %d\n", __func__, name, outPoint.hash.GetHex(), outPoint.n, nAmount, supportedClaimId.GetHex(), nHeight, nCurrentHeight); + const std::string normalizedName = lbryNormalize(name, base->shouldNormalize(nValidAtHeight)); + LogPrintf("%s: name: %s, txhash: %s, nOut: %d, nAmount: %d, supportedClaimId: %s, nHeight: %d, nCurrentHeight: %d\n", __func__, normalizedName, outPoint.hash.GetHex(), outPoint.n, nAmount, supportedClaimId.GetHex(), nHeight, nCurrentHeight); CSupportValue support(outPoint, supportedClaimId, nAmount, nHeight, nValidAtHeight); if (nValidAtHeight < nCurrentHeight) { - nameOutPointType entry(name, support.outPoint); + nameOutPointType entry(normalizedName, support.outPoint); addSupportToExpirationQueue(support.nHeight + base->nExpirationTime, entry); - return insertSupportIntoMap(name, support, false); + return insertSupportIntoMap(normalizedName, support, false); } else { - return addSupportToQueues(name, support); + return addSupportToQueues(normalizedName, support); } } @@ -1981,13 +2093,14 @@ bool CClaimTrieCache::removeSupport(const std::string& name, const COutPoint& ou { bool removed = false; CSupportValue support; - if (removeSupportFromQueue(name, outPoint, support)) + const std::string normalizedName = lbryNormalize(name, base->shouldNormalize(nHeight)); + if (removeSupportFromQueue(normalizedName, outPoint, support)) removed = true; - if (removed == false && removeSupportFromMap(name, outPoint, support, fCheckTakeover)) + if (removed == false && removeSupportFromMap(normalizedName, outPoint, support, fCheckTakeover)) removed = true; if (removed) { - removeSupportFromExpirationQueue(name, outPoint, nHeight); + removeSupportFromExpirationQueue(normalizedName, outPoint, nHeight); nValidAtHeight = support.nValidAtHeight; } return removed; @@ -2004,11 +2117,12 @@ void CClaimTrieCache::removeSupportFromExpirationQueue(const std::string& name, int expirationHeight = nHeight + base->nExpirationTime; expirationQueueType::iterator itQueueRow = getSupportExpirationQueueCacheRow(expirationHeight, false); expirationQueueRowType::iterator itQueue; + const std::string normalizedName = lbryNormalize(name, base->shouldNormalize()); if (itQueueRow != supportExpirationQueueCache.end()) { for (itQueue = itQueueRow->second.begin(); itQueue != itQueueRow->second.end(); ++itQueue) { - if (name == itQueue->name && outPoint == itQueue->outPoint) + if (normalizedName == itQueue->name && outPoint == itQueue->outPoint) break; } } @@ -2394,10 +2508,11 @@ bool CClaimTrieCache::getLastTakeoverForName(const std::string& name, int& nLast nLastTakeoverForName = 0; return true; } - std::map::iterator itHeights = cacheTakeoverHeights.find(name); + const std::string normalizedName = lbryNormalize(name, base->shouldNormalize()); + std::map::iterator itHeights = cacheTakeoverHeights.find(normalizedName); if (itHeights == cacheTakeoverHeights.end()) { - return base->getLastTakeoverForName(name, nLastTakeoverForName); + return base->getLastTakeoverForName(normalizedName, nLastTakeoverForName); } nLastTakeoverForName = itHeights->second; return true; @@ -2406,21 +2521,22 @@ bool CClaimTrieCache::getLastTakeoverForName(const std::string& name, int& nLast int CClaimTrieCache::getNumBlocksOfContinuousOwnership(const std::string& name) const { const CClaimTrieNode* node = NULL; - nodeCacheType::const_iterator itCache = cache.find(name); + const std::string normalizedName = lbryNormalize(name, base->shouldNormalize()); + nodeCacheType::const_iterator itCache = cache.find(normalizedName); if (itCache != cache.end()) { node = itCache->second; } if (!node) { - node = base->getNodeForName(name); + node = base->getNodeForName(normalizedName); } if (!node || node->claims.empty()) { return 0; } int nLastTakeoverHeight; - assert(getLastTakeoverForName(name, nLastTakeoverHeight)); + assert(getLastTakeoverForName(normalizedName, nLastTakeoverHeight)); return nCurrentHeight - nLastTakeoverHeight; } @@ -2430,7 +2546,8 @@ int CClaimTrieCache::getDelayForName(const std::string& name) const { return 0; } - int nBlocksOfContinuousOwnership = getNumBlocksOfContinuousOwnership(name); + const std::string normalizedName = lbryNormalize(name, base->shouldNormalize()); + int nBlocksOfContinuousOwnership = getNumBlocksOfContinuousOwnership(normalizedName); return std::min(nBlocksOfContinuousOwnership / base->nProportionalDelayFactor, 4032); } diff --git a/src/claimtrie.h b/src/claimtrie.h index 7b6ab254d7..1792dc49d5 100644 --- a/src/claimtrie.h +++ b/src/claimtrie.h @@ -11,6 +11,11 @@ #include #include +#include +#include +#include +#include + // leveldb keys #define HASH_BLOCK 'h' #define CURRENT_HEIGHT 't' @@ -23,6 +28,9 @@ #define SUPPORT_QUEUE_NAME_ROW 'p' #define SUPPORT_EXP_QUEUE_ROW 'x' +// hard fork height for claim name normalization +static const int LBRY_NORMALIZED_NAME_FORK_HEIGHT = 329320; + uint256 getValueHash(COutPoint outPoint, int nHeightOfLastTakeover); class CClaimValue @@ -277,23 +285,54 @@ struct claimsForNameType class CClaimTrieCache; +// lower-case and normalize any input string name +// see: https://unicode.org/reports/tr15/#Norm_Forms +inline const std::string lbryNormalize(const std::string& name, bool shouldNormalize) +{ + if (shouldNormalize) + { + boost::locale::localization_backend_manager manager = + boost::locale::localization_backend_manager::global(); + manager.select("icu"); + + boost::locale::generator curLocale(manager); + const std::locale utf8 = curLocale("en_US.UTF8"); + + const std::string normalized = boost::locale::normalize( + boost::locale::to_lower(name, utf8), boost::locale::norm_nfd, utf8); + + /* LogPrintf("%s: Normalized \"%s\" (%lu, %lu) to \"%s\" (%lu, %lu)\n", __func__, name, name.size(), sizeof(name), normalized, normalized.size(), sizeof(normalized)); */ + return normalized; + } + /* LogPrintf("%s: Non-Normalized \"%s\"\n", __func__, name); */ + return name; +} + +#define CHECK_HEIGHT(x) \ + (normalizeNames && (x >= LBRY_NORMALIZED_NAME_FORK_HEIGHT)) + +#define BASE_CHECK_HEIGHT(x) \ + (base->shouldNormalize() && (x >= LBRY_NORMALIZED_NAME_FORK_HEIGHT)) + class CClaimTrie { public: - CClaimTrie(bool fMemory = false, bool fWipe = false, int nProportionalDelayFactor = 32) + CClaimTrie(bool fMemory = false, bool fWipe = false, bool normalize = false, int nProportionalDelayFactor = 32) : db(GetDataDir() / "claimtrie", 100, fMemory, fWipe, false) , nCurrentHeight(0), nExpirationTime(262974) , nProportionalDelayFactor(nProportionalDelayFactor) , root(uint256S("0000000000000000000000000000000000000000000000000000000000000001")) + , normalizeNames(normalize) {} - + uint256 getMerkleHash(); bool empty() const; void clear(); bool checkConsistency() const; - + bool shouldNormalize(int nHeight = 0) const; + bool WriteToDisk(); bool ReadFromDisk(bool check = false); @@ -311,6 +350,7 @@ class CClaimTrie bool supportExpirationQueueEmpty() const; void setExpirationTime(int t); + void setNormalize(bool normalize); bool getQueueRow(int nHeight, claimQueueRowType& row) const; bool getQueueNameRow(const std::string& name, queueNameRowType& row) const; @@ -405,6 +445,8 @@ class CClaimTrie nodeCacheType dirtyNodes; supportMapType dirtySupportNodes; + + bool normalizeNames; }; class CClaimTrieProofNode diff --git a/src/init.cpp b/src/init.cpp index c064dd3348..28243ead52 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -1422,6 +1422,22 @@ bool AppInit2(boost::thread_group& threadGroup, CScheduler& scheduler) RandAddSeedPerfmon(); + if ((chainActive.Height() > LBRY_NORMALIZED_NAME_FORK_HEIGHT) && + !pclaimTrie->shouldNormalize()) + { + LogPrintf("Reloading ClaimTrie with normalization enabled\n"); + + delete pclaimTrie; + pclaimTrie = new CClaimTrie(false, fReindex, true); + + if (!pclaimTrie->ReadFromDisk(true)) + { + LogPrintf("Error re-loading the claim trie from disk"); + return false; + } + LogPrintf("Finished reloading ClaimTrie with normalization enabled\n"); + } + //// debug print LogPrintf("mapBlockIndex.size() = %u\n", mapBlockIndex.size()); LogPrintf("nBestHeight = %d\n", chainActive.Height());