Skip to content

Commit

Permalink
Add lowercase and unicode normalization to claim name entries in the
Browse files Browse the repository at this point in the history
claim trie
  • Loading branch information
lbrynaut committed Mar 12, 2018
1 parent 29bc408 commit 08c57a7
Show file tree
Hide file tree
Showing 8 changed files with 532 additions and 191 deletions.
120 changes: 120 additions & 0 deletions build-aux/m4/ax_boost_locale.m4
@@ -0,0 +1,120 @@
# ===========================================================================
# https://www.gnu.org/software/autoconf-archive/ax_boost_locale.html
# ===========================================================================
#
# SYNOPSIS
#
# AX_BOOST_LOCALE
#
# DESCRIPTION
#
# Test for System library from the Boost C++ libraries. The macro requires
# a preceding call to AX_BOOST_BASE. Further documentation is available at
# <http://randspringer.de/boost/index.html>.
#
# This macro calls:
#
# AC_SUBST(BOOST_LOCALE_LIB)
#
# And sets:
#
# HAVE_BOOST_LOCALE
#
# LICENSE
#
# Copyright (c) 2012 Xiyue Deng <manphiz@gmail.com>
#
# Copying and distribution of this file, with or without modification, are
# permitted in any medium without royalty provided the copyright notice
# and this notice are preserved. This file is offered as-is, without any
# warranty.

#serial 2

AC_DEFUN([AX_BOOST_LOCALE],
[
AC_ARG_WITH([boost-locale],
AS_HELP_STRING([--with-boost-locale@<:@=special-lib@:>@],
[use the Locale library from boost - it is possible to specify a certain library for the linker
e.g. --with-boost-locale=boost_locale-gcc-mt ]),
[
if test "$withval" = "no"; then
want_boost="no"
elif test "$withval" = "yes"; then
want_boost="yes"
ax_boost_user_locale_lib=""
else
want_boost="yes"
ax_boost_user_locale_lib="$withval"
fi
],
[want_boost="yes"]
)
if test "x$want_boost" = "xyes"; then
AC_REQUIRE([AC_PROG_CC])
AC_REQUIRE([AC_CANONICAL_BUILD])
CPPFLAGS_SAVED="$CPPFLAGS"
CPPFLAGS="$CPPFLAGS $BOOST_CPPFLAGS"
export CPPFLAGS
LDFLAGS_SAVED="$LDFLAGS"
LDFLAGS="$LDFLAGS $BOOST_LDFLAGS"
export LDFLAGS
AC_CACHE_CHECK(whether the Boost::Locale library is available,
ax_cv_boost_locale,
[AC_LANG_PUSH([C++])
CXXFLAGS_SAVE=$CXXFLAGS
AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[@%:@include <boost/locale.hpp>]],
[[boost::locale::generator gen;
std::locale::global(gen(""));]])],
ax_cv_boost_locale=yes, ax_cv_boost_locale=no)
CXXFLAGS=$CXXFLAGS_SAVE
AC_LANG_POP([C++])
])
if test "x$ax_cv_boost_locale" = "xyes"; then
AC_SUBST(BOOST_CPPFLAGS)
AC_DEFINE(HAVE_BOOST_LOCALE,,[define if the Boost::Locale library is available])
BOOSTLIBDIR=`echo $BOOST_LDFLAGS | sed -e 's/@<:@^\/@:>@*//'`
LDFLAGS_SAVE=$LDFLAGS
if test "x$ax_boost_user_locale_lib" = "x"; then
for libextension in `ls $BOOSTLIBDIR/libboost_locale*.so* $BOOSTLIBDIR/libboost_locale*.dylib* $BOOSTLIBDIR/libboost_locale*.a* 2>/dev/null | sed 's,.*/,,' | sed -e 's;^lib\(boost_locale.*\)\.so.*$;\1;' -e 's;^lib\(boost_locale.*\)\.dylib.*$;\1;' -e 's;^lib\(boost_locale.*\)\.a.*$;\1;'` ; do
ax_lib=${libextension}
AC_CHECK_LIB($ax_lib, exit,
[BOOST_LOCALE_LIB="-l$ax_lib"; AC_SUBST(BOOST_LOCALE_LIB) link_locale="yes"; break],
[link_locale="no"])
done
if test "x$link_locale" != "xyes"; then
for libextension in `ls $BOOSTLIBDIR/boost_locale*.dll* $BOOSTLIBDIR/boost_locale*.a* 2>/dev/null | sed 's,.*/,,' | sed -e 's;^\(boost_locale.*\)\.dll.*$;\1;' -e 's;^\(boost_locale.*\)\.a.*$;\1;'` ; do
ax_lib=${libextension}
AC_CHECK_LIB($ax_lib, exit,
[BOOST_LOCALE_LIB="-l$ax_lib"; AC_SUBST(BOOST_LOCALE_LIB) link_locale="yes"; break],
[link_locale="no"])
done
fi
else
for ax_lib in $ax_boost_user_locale_lib boost_locale-$ax_boost_user_locale_lib; do
AC_CHECK_LIB($ax_lib, exit,
[BOOST_LOCALE_LIB="-l$ax_lib"; AC_SUBST(BOOST_LOCALE_LIB) link_locale="yes"; break],
[link_locale="no"])
done
fi
if test "x$ax_lib" = "x"; then
AC_MSG_ERROR(Could not find a version of the library!)
fi
if test "x$link_locale" = "xno"; then
AC_MSG_ERROR(Could not link against $ax_lib !)
fi
fi
CPPFLAGS="$CPPFLAGS_SAVED"
LDFLAGS="$LDFLAGS_SAVED"
fi
])

14 changes: 11 additions & 3 deletions configure.ac
Expand Up @@ -583,6 +583,7 @@ AX_BOOST_FILESYSTEM
AX_BOOST_PROGRAM_OPTIONS
AX_BOOST_THREAD
AX_BOOST_CHRONO
AX_BOOST_LOCALE


if test x$use_reduce_exports = xyes; then
Expand Down Expand Up @@ -645,7 +646,7 @@ fi

if test x$use_boost = xyes; then

BOOST_LIBS="$BOOST_LDFLAGS $BOOST_SYSTEM_LIB $BOOST_FILESYSTEM_LIB $BOOST_PROGRAM_OPTIONS_LIB $BOOST_THREAD_LIB $BOOST_CHRONO_LIB"
BOOST_LIBS="$BOOST_LDFLAGS $BOOST_SYSTEM_LIB $BOOST_FILESYSTEM_LIB $BOOST_PROGRAM_OPTIONS_LIB $BOOST_THREAD_LIB $BOOST_CHRONO_LIB $BOOST_LOCALE_LIB"


dnl If boost (prior to 1.57) was built without c++11, it emulated scoped enums
Expand Down Expand Up @@ -746,8 +747,11 @@ if test x$use_pkgconfig = xyes; then
m4_ifdef(
[PKG_CHECK_MODULES],
[
PKG_CHECK_MODULES([SSL], [libssl],, [AC_MSG_ERROR(openssl not found.)])
PKG_CHECK_MODULES([CRYPTO], [libcrypto],,[AC_MSG_ERROR(libcrypto not found.)])
AC_CHECK_HEADER([unicode/errorcode.h],,AC_MSG_ERROR(libicu headers missing))
AC_CHECK_LIB([icudata], [main],ICU_LIBS="-licui18n -licuuc -licudata -dl", AC_MSG_ERROR(libicu missing))
PKG_CHECK_MODULES([SSL], [libssl],, [AC_MSG_ERROR(openssl not found.)])
PKG_CHECK_MODULES([CRYPTO], [libcrypto],,[AC_MSG_ERROR(libcrypto not found.)])
BITCOIN_QT_CHECK([PKG_CHECK_MODULES([PROTOBUF], [protobuf], [have_protobuf=yes], [BITCOIN_QT_FAIL(libprotobuf not found)])])
if test x$use_qr != xno; then
BITCOIN_QT_CHECK([PKG_CHECK_MODULES([QR], [libqrencode], [have_qrencode=yes], [have_qrencode=no])])
Expand All @@ -771,6 +775,9 @@ if test x$use_pkgconfig = xyes; then
]
)
else
AC_CHECK_HEADER([unicode/errorcode.h],,AC_MSG_ERROR(libicu headers missing))
AC_CHECK_LIB([icudata], [main],ICU_LIBS="-licui18n -licuuc -licudata -dl", AC_MSG_ERROR(libicu missing))

AC_CHECK_HEADER([openssl/crypto.h],,AC_MSG_ERROR(libcrypto headers missing))
AC_CHECK_LIB([crypto], [main],CRYPTO_LIBS=-lcrypto, AC_MSG_ERROR(libcrypto missing))

Expand Down Expand Up @@ -1039,6 +1046,7 @@ AC_SUBST(LIBTOOL_APP_LDFLAGS)
AC_SUBST(USE_UPNP)
AC_SUBST(USE_QRCODE)
AC_SUBST(BOOST_LIBS)
AC_SUBST(ICU_LIBS)
AC_SUBST(TESTDEFS)
AC_SUBST(LEVELDB_TARGET_FLAGS)
AC_SUBST(MINIUPNPC_CPPFLAGS)
Expand Down
86 changes: 62 additions & 24 deletions reproducible_build.sh
@@ -1,5 +1,6 @@
#!/bin/bash

set -x
set -euo pipefail

function HELP {
Expand Down Expand Up @@ -101,24 +102,23 @@ else
# this file is created when the build starts
START_TIME_FILE="$TRAVIS_BUILD_DIR/start_time"
fi
if [ ! -f "${START_TIME_FILE}" ]; then
date +%s > "${START_TIME_FILE}"
fi
rm -f ${START_TIME_FILE}
date +%s > "${START_TIME_FILE}"


NEXT_TIME=60
function exit_at_40() {
function exit_at_60() {
if [ -f "${START_TIME_FILE}" ]; then
NOW=$(date +%s)
START=$(cat "${START_TIME_FILE}")
TIMEOUT_SECS=2400 # 40 * 60
TIMEOUT_SECS=3600 # 60 * 60
TIME=$((NOW - START))
if (( TIME > NEXT_TIME )); then
echo "Build has taken $((TIME / 60)) minutes: $1"
NEXT_TIME=$((TIME + 60))
fi
if [ "$TIMEOUT" = true ] && (( TIME > TIMEOUT_SECS )); then
echo 'Exiting at 40 minutes to allow the cache to populate'
echo 'Exiting at 60 minutes to allow the cache to populate'
OUTPUT_LOG=false
exit 1
fi
Expand All @@ -140,7 +140,7 @@ function wait_and_echo() {
# loop until the process is no longer running
# check every $SLEEP seconds, echoing a message every minute
while (ps -p "${PID}" > /dev/null); do
exit_at_40 "$2"
exit_at_60 "$2"
sleep "${SLEEP}"
done
# restore the xtrace setting
Expand Down Expand Up @@ -226,10 +226,14 @@ function install_apt_packages() {
ca-certificates automake bsdmainutils
}

PARALLEL=""

function build_dependencies() {
if [ "${OS_NAME}" = "osx" ]; then
PARALLEL="-j $(sysctl -n hw.ncpu)"
install_brew_packages
else
PARALLEL="-j $(grep -c processor /proc/cpuinfo)"
install_apt_packages
fi

Expand All @@ -244,6 +248,7 @@ function build_dependencies() {
# TODO: if the repo exists, make sure its clean: revert to head.
mkdir -p "${LOG_DIR}"

build_dependency "${ICU_PREFIX}" "${LOG_DIR}/icu_build.log" build_icu
build_dependency "${BDB_PREFIX}" "${LOG_DIR}/bdb_build.log" build_bdb
build_dependency "${OPENSSL_PREFIX}" "${LOG_DIR}/openssl_build.log" build_openssl

Expand All @@ -263,7 +268,7 @@ function build_bdb() {
cd db-4.8.30.NC/build_unix
echo "Building bdb. tail -f $BDB_LOG to see the details and monitor progress"
../dist/configure --prefix="${BDB_PREFIX}" --enable-cxx --disable-shared --with-pic > "${BDB_LOG}"
background make "${BDB_LOG}" "Waiting for bdb to finish building"
background "make ${PARALLEL}" "${BDB_LOG}" "Waiting for bdb to finish building"
make install >> "${BDB_LOG}" 2>&1
}

Expand All @@ -280,18 +285,46 @@ function build_openssl() {
./Configure --prefix="${OPENSSL_PREFIX}" --openssldir="${OPENSSL_PREFIX}/ssl" \
linux-x86_64 -fPIC -static no-shared no-dso > "${OPENSSL_LOG}"
fi
background make "${OPENSSL_LOG}" "Waiting for openssl to finish building"
background "make ${PARALLEL}" "${OPENSSL_LOG}" "Waiting for openssl to finish building"
make install >> "${OPENSSL_LOG}" 2>&1
}

function build_boost() {
BOOST_LOG="$1"
cd boost_1_59_0

echo "int main() { return 0; }" > libs/regex/build/has_icu_test.cpp
echo "int main() { return 0; }" > libs/locale/build/has_icu_test.cpp

export BOOST_ICU_LIBS="$(pkg-config icu-i18n --libs) -dl"
echo "BOOST_ICU_LIBS: $BOOST_ICU_LIBS"

echo "Building Boost. tail -f ${BOOST_LOG} to see the details and monitor progress"
./bootstrap.sh --prefix="${BOOST_PREFIX}" > "${BOOST_LOG}" 2>&1
background "./b2 link=static cxxflags=-fPIC install" \
"${BOOST_LOG}" \
"Waiting for boost to finish building"

./bootstrap.sh --prefix="${BOOST_PREFIX}" \
"--with-icu=${ICU_PREFIX}" > "${BOOST_LOG}" 2>&1
b2cmd="./b2 link=static cxxflags=-fPIC install boost.locale.iconv=off boost.locale.posix=off -sICU_PATH=$ICU_PREFIX -sICU_LINK=${BOOST_ICU_LIBS}"

background "$b2cmd" "${BOOST_LOG}" "Waiting for boost to finish building"
}

function build_icu() {
ICU_LOG="$1"
mkdir -p "${ICU_PREFIX}/icu"
wget http://download.icu-project.org/files/icu4c/55.1/icu4c-55_1-src.tgz
tar -xf icu4c-55_1-src.tgz
rm -f icu4c-55_1-src.tgz
pushd icu/source > /dev/null
echo "Building icu. tail -f $ICU_LOG to see the details and monitor progress"
./configure --prefix="${ICU_PREFIX}" --enable-draft --enable-tools \
--enable-static --disable-extras --disable-icuio --disable-layout \
--disable-layoutex --disable-tests --disable-samples
TMP_TARGET="$TARGET"
unset TARGET
background "make ${PARALLEL} VERBOSE=1" "${ICU_LOG}" "Waiting for icu to finish building"
make install >> "${ICU_LOG}" 2>&1
TARGET="$TMP_TARGET"
popd > /dev/null
}

function build_libevent() {
Expand All @@ -305,7 +338,7 @@ function build_libevent() {
./configure --prefix="${LIBEVENT_PREFIX}" --enable-static --disable-shared --with-pic \
LDFLAGS="-L${OPENSSL_PREFIX}/lib/" \
CPPFLAGS="-I${OPENSSL_PREFIX}/include" >> "${LIBEVENT_LOG}" 2>&1
background make "${LIBEVENT_LOG}" "Waiting for libevent to finish building"
background "make ${PARALLEL}" "${LIBEVENT_LOG}" "Waiting for libevent to finish building"
make install >> "${LIBEVENT_LOG}"
}

Expand All @@ -323,16 +356,18 @@ function build_dependency() {
}

function build_lbrycrd() {
if [ "$CLONE" == true ]; then
# if [ "$CLONE" == true ]; then
cd "${LBRYCRD_DEPENDENCIES}"
git clone https://github.com/lbryio/lbrycrd
# git clone https://github.com/lbryio/lbrycrd
git clone https://github.com/lbrynaut/lbrycrd
cd lbrycrd
else
cd "${SOURCE_DIR}"
fi
git checkout -b normalized-name-fork origin/normalized-name-fork
# else
# cd "${SOURCE_DIR}"
# fi
./autogen.sh > "${LBRYCRD_LOG}" 2>&1
LDFLAGS="-L${OPENSSL_PREFIX}/lib/ -L${BDB_PREFIX}/lib/ -L${LIBEVENT_PREFIX}/lib/ -static-libstdc++"
CPPFLAGS="-I${OPENSSL_PREFIX}/include -I${BDB_PREFIX}/include -I${LIBEVENT_PREFIX}/include/"
LDFLAGS="-L${ICU_PREFIX}/lib/ -L${OPENSSL_PREFIX}/lib/ -L${BDB_PREFIX}/lib/ -L${LIBEVENT_PREFIX}/lib/ -L${BOOST_PREFIX}/lib/ -static-libstdc++"
CPPFLAGS="-I${ICU_PREFIX}/include/ -I${OPENSSL_PREFIX}/include -I${BDB_PREFIX}/include -I${LIBEVENT_PREFIX}/include/ -I${BOOST_PREFIX}/include/"
if [ "${OS_NAME}" = "osx" ]; then
OPTIONS="--enable-cxx --enable-static --disable-shared --with-pic"
else
Expand All @@ -342,12 +377,14 @@ function build_lbrycrd() {
--with-boost="${BOOST_PREFIX}" \
LDFLAGS="${LDFLAGS}" \
CPPFLAGS="${CPPFLAGS}" >> "${LBRYCRD_LOG}" 2>&1
background make "${LBRYCRD_LOG}" "Waiting for lbrycrd to finish building"
background "make ${PARALLEL}" "${LBRYCRD_LOG}" "Waiting for lbrycrd to finish building"
# tests don't work on OSX. Should definitely figure out why
# that is but, for now, not letting that stop the rest
# of the build
if [ "${OS_NAME}" = "linux" ]; then
src/test/test_lbrycrd
# LD_LIBRARY_PATH="${ICU_PREFIX}/lib" make ${PARALLEL} check
LD_LIBRARY_PATH="${ICU_PREFIX}/lib" src/test/test_lbrycrd
# src/test/test_lbrycrd
fi
strip src/lbrycrdd
strip src/lbrycrd-cli
Expand All @@ -363,13 +400,14 @@ BDB_PREFIX="${OUTPUT_DIR}/bdb"
OPENSSL_PREFIX="${OUTPUT_DIR}/openssl"
BOOST_PREFIX="${OUTPUT_DIR}/boost"
LIBEVENT_PREFIX="${OUTPUT_DIR}/libevent"
ICU_PREFIX="${OUTPUT_DIR}/icu"

if [ "${BUILD_DEPENDENCIES}" = true ]; then
build_dependencies
fi

set +u
export PKG_CONFIG_PATH="${PKG_CONFIG_PATH}:${OPENSSL_PREFIX}/lib/pkgconfig/:${LIBEVENT_PREFIX}/lib/pkgconfig/"
export PKG_CONFIG_PATH="${PKG_CONFIG_PATH}:${OPENSSL_PREFIX}/lib/pkgconfig/:${LIBEVENT_PREFIX}/lib/pkgconfig/:${ICU_PREFIX}/lib/pkgconfig/"
set -u

if [ "${BUILD_LBRYCRD}" = true ]; then
Expand Down

0 comments on commit 08c57a7

Please sign in to comment.