Skip to content

Commit 989665d

Browse files
denravonskadenravonska
authored
Merge pull request #1387 from jamescowens/integrated_scraper
New Scraper and NN
2 parents 27c1131 + f05c083 commit 989665d

File tree

279 files changed

+10584
-20932
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

279 files changed

+10584
-20932
lines changed

‎.travis.yml

100755100644
+9-4
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
sudo: required
2-
dist: trusty
2+
dist: bionic
33
os: linux
44
language: minimal
55
cache:
@@ -22,25 +22,30 @@ env:
2222
- WINEDEBUG=fixme-all
2323
matrix:
2424
# ARM
25-
- HOST=arm-linux-gnueabihf PACKAGES="g++-arm-linux-gnueabihf" DEP_OPTS="NO_QT=1" CHECK_DOC=1 GOAL="install" GRIDCOIN_CONFIG="--enable-glibc-back-compat --enable-reduce-exports RUN_TESTS=false"
25+
- HOST=arm-linux-gnueabihf PACKAGES="g++-arm-linux-gnueabihf zlib1g-dev" DEP_OPTS="NO_QT=1" CHECK_DOC=1 GOAL="install" GRIDCOIN_CONFIG="--enable-glibc-back-compat --disable-tests RUN_TESTS=false"
2626
# Win32
2727
- HOST=i686-w64-mingw32 DPKG_ADD_ARCH="i386" DEP_OPTS="NO_QT=1" PACKAGES="python3 nsis g++-mingw-w64-i686 wine1.6" RUN_TESTS=false GOAL="" GRIDCOIN_CONFIG="--enable-reduce-exports"
2828
# Qt5 & system libs
29-
- HOST=x86_64-unknown-linux-gnu PACKAGES="libqt5gui5 libqt5core5a qtbase5-dev libqt5dbus5 qttools5-dev qttools5-dev-tools libssl-dev libevent-dev bsdmainutils libboost-system-dev libboost-filesystem-dev libboost-chrono-dev libboost-program-options-dev libboost-test-dev libboost-thread-dev libdb5.1++-dev libminiupnpc-dev protobuf-compiler libqrencode-dev xvfb" NO_DEPENDS=1 NEED_XVFB=1 RUN_TESTS=true GOAL="install" GRIDCOIN_CONFIG=" --with-incompatible-bdb --enable-reduce-exports --with-gui=qt5"
29+
- HOST=x86_64-unknown-linux-gnu PACKAGES="libqt5gui5 libqt5core5a qtbase5-dev libqt5dbus5 qttools5-dev qttools5-dev-tools libssl-dev libevent-dev bsdmainutils libboost-system-dev libboost-filesystem-dev libboost-chrono-dev libboost-iostreams-dev libboost-program-options-dev libboost-test-dev libboost-thread-dev libdb5.3++-dev libminiupnpc-dev protobuf-compiler libqrencode-dev xvfb zlib1g-dev" NO_DEPENDS=1 NEED_XVFB=1 RUN_TESTS=true GOAL="install" GRIDCOIN_CONFIG=" --with-incompatible-bdb --enable-reduce-exports --with-gui=qt5"
3030
# 32-bit + dash
3131
- HOST=i686-pc-linux-gnu PACKAGES="g++-multilib" DEP_OPTS="NO_QT=1" RUN_TESTS=false GOAL="install" GRIDCOIN_CONFIG="--enable-glibc-back-compat --enable-reduce-exports LDFLAGS=-static-libstdc++" USE_SHELL="/bin/dash"
3232
# Win64
33-
- HOST=x86_64-w64-mingw32 DPKG_ADD_ARCH="i386" PACKAGES="python3 nsis g++-mingw-w64-x86-64 wine1.6 xvfb" DEP_OPTS="NO_QT=1" RUN_TESTS=false GOAL="" GRIDCOIN_CONFIG="--enable-reduce-exports" NEED_XVFB=1
33+
- HOST=x86_64-w64-mingw32 DPKG_ADD_ARCH="i386" PACKAGES="gcc-7 python3 nsis g++-mingw-w64-x86-64 wine1.6 xvfb" DEP_OPTS="NO_QT=1" RUN_TESTS=false GOAL="" GRIDCOIN_CONFIG="--enable-reduce-exports" NEED_XVFB=1
3434
# Cross-Mac - disabled for now, since we cannot compile QtSVG with the SDK
3535
#- HOST=x86_64-apple-darwin11 PACKAGES="cmake imagemagick libcap-dev librsvg2-bin libz-dev libbz2-dev libtiff-tools python-dev" GRIDCOIN_CONFIG="--enable-gui --enable-reduce-exports" OSX_SDK=10.11 RUN_TESTS=false GOAL="deploy"
3636

3737
before_install:
3838
- export PATH=$(echo $PATH | tr ':' "\n" | sed '/\/opt\/python/d' | tr "\n" ":" | sed "s|::|:|g")
39+
- sudo add-apt-repository ppa:ubuntu-toolchain-r/test -y
40+
- sudo apt-get update -q
41+
- sudo apt-get install gcc-7 -y
3942
install:
4043
- if [ -n "$DPKG_ADD_ARCH" ]; then sudo dpkg --add-architecture "$DPKG_ADD_ARCH" ; fi
4144
- if [ -n "$PACKAGES" ]; then travis_retry sudo apt-get update; fi
4245
- if [ -n "$PACKAGES" ]; then travis_retry sudo apt-get install --no-install-recommends --no-upgrade -qq $PACKAGES; fi
4346
before_script:
47+
- if [ "$HOST" == "i686-w64-mingw32" ]; then sudo update-alternatives --set i686-w64-mingw32-g++ /usr/bin/i686-w64-mingw32-g++-posix; fi
48+
- if [ "$HOST" == "x86_64-w64-mingw32" ]; then sudo update-alternatives --set x86_64-w64-mingw32-g++ /usr/bin/x86_64-w64-mingw32-g++-posix; fi
4449
- unset CC; unset CXX
4550
- mkdir -p depends/SDKs depends/sdk-sources
4651
- if [ -n "$OSX_SDK" -a ! -f depends/sdk-sources/MacOSX${OSX_SDK}.sdk.tar.gz ]; then curl --location --fail $SDK_URL/MacOSX${OSX_SDK}.sdk.tar.gz -o depends/sdk-sources/MacOSX${OSX_SDK}.sdk.tar.gz; fi

‎README.md

+1
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ These dependencies are required:
1313

1414
Library | Purpose | Description
1515
------------|------------------|----------------------------------------------------------------
16+
pkg-config | Build | Learn library inter-dependencies
1617
libssl | Crypto | Random Number Generation, Elliptic Curve Cryptography
1718
libboost | Utility | Library for threading, data structures, etc
1819
libevent | Networking | OS independent asynchronous networking

‎build-aux/m4/ax_boost_iostreams.m4

+116
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,116 @@
1+
# ===========================================================================
2+
# https://www.gnu.org/software/autoconf-archive/ax_boost_iostreams.html
3+
# ===========================================================================
4+
#
5+
# SYNOPSIS
6+
#
7+
# AX_BOOST_IOSTREAMS
8+
#
9+
# DESCRIPTION
10+
#
11+
# Test for IOStreams library from the Boost C++ libraries. The macro
12+
# requires a preceding call to AX_BOOST_BASE. Further documentation is
13+
# available at <http://randspringer.de/boost/index.html>.
14+
#
15+
# This macro calls:
16+
#
17+
# AC_SUBST(BOOST_IOSTREAMS_LIB)
18+
#
19+
# And sets:
20+
#
21+
# HAVE_BOOST_IOSTREAMS
22+
#
23+
# LICENSE
24+
#
25+
# Copyright (c) 2008 Thomas Porschberg <thomas@randspringer.de>
26+
#
27+
# Copying and distribution of this file, with or without modification, are
28+
# permitted in any medium without royalty provided the copyright notice
29+
# and this notice are preserved. This file is offered as-is, without any
30+
# warranty.
31+
32+
#serial 21
33+
34+
AC_DEFUN([AX_BOOST_IOSTREAMS],
35+
[
36+
AC_ARG_WITH([boost-iostreams],
37+
AS_HELP_STRING([--with-boost-iostreams@<:@=special-lib@:>@],
38+
[use the IOStreams library from boost - it is possible to specify a certain library for the linker
39+
e.g. --with-boost-iostreams=boost_iostreams-gcc-mt-d-1_33_1 ]),
40+
[
41+
if test "$withval" = "no"; then
42+
want_boost="no"
43+
elif test "$withval" = "yes"; then
44+
want_boost="yes"
45+
ax_boost_user_iostreams_lib=""
46+
else
47+
want_boost="yes"
48+
ax_boost_user_iostreams_lib="$withval"
49+
fi
50+
],
51+
[want_boost="yes"]
52+
)
53+
54+
if test "x$want_boost" = "xyes"; then
55+
AC_REQUIRE([AC_PROG_CC])
56+
CPPFLAGS_SAVED="$CPPFLAGS"
57+
CPPFLAGS="$CPPFLAGS $BOOST_CPPFLAGS"
58+
export CPPFLAGS
59+
60+
LDFLAGS_SAVED="$LDFLAGS"
61+
LDFLAGS="$LDFLAGS $BOOST_LDFLAGS"
62+
export LDFLAGS
63+
64+
AC_CACHE_CHECK(whether the Boost::IOStreams library is available,
65+
ax_cv_boost_iostreams,
66+
[AC_LANG_PUSH([C++])
67+
AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[@%:@include <boost/iostreams/filtering_stream.hpp>
68+
@%:@include <boost/range/iterator_range.hpp>
69+
]],
70+
[[std::string input = "Hello World!";
71+
namespace io = boost::iostreams;
72+
io::filtering_istream in(boost::make_iterator_range(input));
73+
return 0;
74+
]])],
75+
ax_cv_boost_iostreams=yes, ax_cv_boost_iostreams=no)
76+
AC_LANG_POP([C++])
77+
])
78+
if test "x$ax_cv_boost_iostreams" = "xyes"; then
79+
AC_DEFINE(HAVE_BOOST_IOSTREAMS,,[define if the Boost::IOStreams library is available])
80+
BOOSTLIBDIR=`echo $BOOST_LDFLAGS | sed -e 's/@<:@^\/@:>@*//'`
81+
if test "x$ax_boost_user_iostreams_lib" = "x"; then
82+
for libextension in `ls $BOOSTLIBDIR/libboost_iostreams*.so* $BOOSTLIBDIR/libboost_iostream*.dylib* $BOOSTLIBDIR/libboost_iostreams*.a* 2>/dev/null | sed 's,.*/,,' | sed -e 's;^lib\(boost_iostreams.*\)\.so.*$;\1;' -e 's;^lib\(boost_iostream.*\)\.dylib.*$;\1;' -e 's;^lib\(boost_iostreams.*\)\.a.*$;\1;'` ; do
83+
ax_lib=${libextension}
84+
AC_CHECK_LIB($ax_lib, exit,
85+
[BOOST_IOSTREAMS_LIB="-l$ax_lib"; AC_SUBST(BOOST_IOSTREAMS_LIB) link_iostreams="yes"; break],
86+
[link_iostreams="no"])
87+
done
88+
if test "x$link_iostreams" != "xyes"; then
89+
for libextension in `ls $BOOSTLIBDIR/boost_iostreams*.dll* $BOOSTLIBDIR/boost_iostreams*.a* 2>/dev/null | sed 's,.*/,,' | sed -e 's;^\(boost_iostreams.*\)\.dll.*$;\1;' -e 's;^\(boost_iostreams.*\)\.a.*$;\1;'` ; do
90+
ax_lib=${libextension}
91+
AC_CHECK_LIB($ax_lib, exit,
92+
[BOOST_IOSTREAMS_LIB="-l$ax_lib"; AC_SUBST(BOOST_IOSTREAMS_LIB) link_iostreams="yes"; break],
93+
[link_iostreams="no"])
94+
done
95+
fi
96+
97+
else
98+
for ax_lib in $ax_boost_user_iostreams_lib boost_iostreams-$ax_boost_user_iostreams_lib; do
99+
AC_CHECK_LIB($ax_lib, main,
100+
[BOOST_IOSTREAMS_LIB="-l$ax_lib"; AC_SUBST(BOOST_IOSTREAMS_LIB) link_iostreams="yes"; break],
101+
[link_iostreams="no"])
102+
done
103+
104+
fi
105+
if test "x$ax_lib" = "x"; then
106+
AC_MSG_ERROR(Could not find a version of the library!)
107+
fi
108+
if test "x$link_iostreams" != "xyes"; then
109+
AC_MSG_ERROR(Could not link against $ax_lib !)
110+
fi
111+
fi
112+
113+
CPPFLAGS="$CPPFLAGS_SAVED"
114+
LDFLAGS="$LDFLAGS_SAVED"
115+
fi
116+
])

‎build-aux/m4/ax_boost_zlib.m4

+90
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,90 @@
1+
# SYNOPSIS
2+
#
3+
# AX_BOOST_ZLIB
4+
#
5+
# DESCRIPTION
6+
#
7+
# Test for ZLib library from the Boost C++ libraries. The macro
8+
# requires a preceding call to AX_BOOST_BASE. Further documentation is
9+
# available at <http://randspringer.de/boost/index.html>.
10+
#
11+
# This macro calls:
12+
#
13+
# AC_SUBST(BOOST_ZLIB_LIB)
14+
#
15+
# And sets:
16+
#
17+
# HAVE_BOOST_ZLIB
18+
#
19+
# LICENSE
20+
#
21+
# Copyright (c) 2008 Thomas Porschberg <thomas@randspringer.de>
22+
#
23+
# Copying and distribution of this file, with or without modification, are
24+
# permitted in any medium without royalty provided the copyright notice
25+
# and this notice are preserved. This file is offered as-is, without any
26+
# warranty.
27+
28+
#serial 21
29+
30+
AC_DEFUN([AX_BOOST_ZLIB],
31+
[
32+
AC_ARG_WITH([boost-zlib],
33+
AS_HELP_STRING([--with-boost-zlib@<:@=special-lib@:>@],
34+
[use the ZLib library from boost - it is possible to specify a certain library for the linker
35+
e.g. --with-boost-zlib=boost_zlib-gcc-mt-d-1_33_1 ]),
36+
[
37+
if test "$withval" = "no"; then
38+
want_boost="no"
39+
elif test "$withval" = "yes"; then
40+
want_boost="yes"
41+
ax_boost_user_zlib_lib=""
42+
else
43+
want_boost="yes"
44+
ax_boost_user_zlib_lib="$withval"
45+
fi
46+
],
47+
[want_boost="yes"]
48+
)
49+
50+
if test "x$want_boost" = "xyes"; then
51+
AC_REQUIRE([AC_PROG_CC])
52+
CPPFLAGS_SAVED="$CPPFLAGS"
53+
CPPFLAGS="$CPPFLAGS $BOOST_CPPFLAGS"
54+
export CPPFLAGS
55+
56+
LDFLAGS_SAVED="$LDFLAGS"
57+
LDFLAGS="$LDFLAGS $BOOST_LDFLAGS"
58+
export LDFLAGS
59+
60+
AC_DEFINE(HAVE_BOOST_ZLIB,,[define if the Boost::zlib library is available])
61+
BOOSTLIBDIR=`echo $BOOST_LDFLAGS | sed -e 's/@<:@^\/@:>@*//'`
62+
if test "x$ax_boost_user_zlib_lib" = "x"; then
63+
for libextension in `ls $BOOSTLIBDIR/libboost_zlib*.so* $BOOSTLIBDIR/libboost_iostream*.dylib* $BOOSTLIBDIR/libboost_zlib*.a* 2>/dev/null | sed 's,.*/,,' | sed -e 's;^lib\(boost_zlib.*\)\.so.*$;\1;' -e 's;^lib\(boost_iostream.*\)\.dylib.*$;\1;' -e 's;^lib\(boost_zlib.*\)\.a.*$;\1;'` ; do
64+
ax_lib=${libextension}
65+
AC_CHECK_LIB($ax_lib, exit,
66+
[BOOST_ZLIB_LIB="-l$ax_lib"; AC_SUBST(BOOST_ZLIB_LIB) link_zlib="yes"; break],
67+
[link_zlib="no"])
68+
done
69+
if test "x$link_zlib" != "xyes"; then
70+
for libextension in `ls $BOOSTLIBDIR/boost_zlib*.dll* $BOOSTLIBDIR/boost_zlib*.a* 2>/dev/null | sed 's,.*/,,' | sed -e 's;^\(boost_zlib.*\)\.dll.*$;\1;' -e 's;^\(boost_zlib.*\)\.a.*$;\1;'` ; do
71+
ax_lib=${libextension}
72+
AC_CHECK_LIB($ax_lib, exit,
73+
[BOOST_ZLIB_LIB="-l$ax_lib"; AC_SUBST(BOOST_ZLIB_LIB) link_zlib="yes"; break],
74+
[link_zlib="no"])
75+
done
76+
fi
77+
78+
else
79+
for ax_lib in $ax_boost_user_zlib_lib boost_zlib-$ax_boost_user_zlib_lib; do
80+
AC_CHECK_LIB($ax_lib, main,
81+
[BOOST_ZLIB_LIB="-l$ax_lib"; AC_SUBST(BOOST_ZLIB_LIB) link_zlib="yes"; break],
82+
[link_zlib="no"])
83+
done
84+
85+
fi
86+
87+
CPPFLAGS="$CPPFLAGS_SAVED"
88+
LDFLAGS="$LDFLAGS_SAVED"
89+
fi
90+
])

‎build-aux/m4/m4_ax_boost_iostreams.m4

+116
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,116 @@
1+
# ===========================================================================
2+
# https://www.gnu.org/software/autoconf-archive/ax_boost_iostreams.html
3+
# ===========================================================================
4+
#
5+
# SYNOPSIS
6+
#
7+
# AX_BOOST_IOSTREAMS
8+
#
9+
# DESCRIPTION
10+
#
11+
# Test for IOStreams library from the Boost C++ libraries. The macro
12+
# requires a preceding call to AX_BOOST_BASE. Further documentation is
13+
# available at <http://randspringer.de/boost/index.html>.
14+
#
15+
# This macro calls:
16+
#
17+
# AC_SUBST(BOOST_IOSTREAMS_LIB)
18+
#
19+
# And sets:
20+
#
21+
# HAVE_BOOST_IOSTREAMS
22+
#
23+
# LICENSE
24+
#
25+
# Copyright (c) 2008 Thomas Porschberg <thomas@randspringer.de>
26+
#
27+
# Copying and distribution of this file, with or without modification, are
28+
# permitted in any medium without royalty provided the copyright notice
29+
# and this notice are preserved. This file is offered as-is, without any
30+
# warranty.
31+
32+
#serial 21
33+
34+
AC_DEFUN([AX_BOOST_IOSTREAMS],
35+
[
36+
AC_ARG_WITH([boost-iostreams],
37+
AS_HELP_STRING([--with-boost-iostreams@<:@=special-lib@:>@],
38+
[use the IOStreams library from boost - it is possible to specify a certain library for the linker
39+
e.g. --with-boost-iostreams=boost_iostreams-gcc-mt-d-1_33_1 ]),
40+
[
41+
if test "$withval" = "no"; then
42+
want_boost="no"
43+
elif test "$withval" = "yes"; then
44+
want_boost="yes"
45+
ax_boost_user_iostreams_lib=""
46+
else
47+
want_boost="yes"
48+
ax_boost_user_iostreams_lib="$withval"
49+
fi
50+
],
51+
[want_boost="yes"]
52+
)
53+
54+
if test "x$want_boost" = "xyes"; then
55+
AC_REQUIRE([AC_PROG_CC])
56+
CPPFLAGS_SAVED="$CPPFLAGS"
57+
CPPFLAGS="$CPPFLAGS $BOOST_CPPFLAGS"
58+
export CPPFLAGS
59+
60+
LDFLAGS_SAVED="$LDFLAGS"
61+
LDFLAGS="$LDFLAGS $BOOST_LDFLAGS"
62+
export LDFLAGS
63+
64+
AC_CACHE_CHECK(whether the Boost::IOStreams library is available,
65+
ax_cv_boost_iostreams,
66+
[AC_LANG_PUSH([C++])
67+
AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[@%:@include <boost/iostreams/filtering_stream.hpp>
68+
@%:@include <boost/range/iterator_range.hpp>
69+
]],
70+
[[std::string input = "Hello World!";
71+
namespace io = boost::iostreams;
72+
io::filtering_istream in(boost::make_iterator_range(input));
73+
return 0;
74+
]])],
75+
ax_cv_boost_iostreams=yes, ax_cv_boost_iostreams=no)
76+
AC_LANG_POP([C++])
77+
])
78+
if test "x$ax_cv_boost_iostreams" = "xyes"; then
79+
AC_DEFINE(HAVE_BOOST_IOSTREAMS,,[define if the Boost::IOStreams library is available])
80+
BOOSTLIBDIR=`echo $BOOST_LDFLAGS | sed -e 's/@<:@^\/@:>@*//'`
81+
if test "x$ax_boost_user_iostreams_lib" = "x"; then
82+
for libextension in `ls $BOOSTLIBDIR/libboost_iostreams*.so* $BOOSTLIBDIR/libboost_iostream*.dylib* $BOOSTLIBDIR/libboost_iostreams*.a* 2>/dev/null | sed 's,.*/,,' | sed -e 's;^lib\(boost_iostreams.*\)\.so.*$;\1;' -e 's;^lib\(boost_iostream.*\)\.dylib.*$;\1;' -e 's;^lib\(boost_iostreams.*\)\.a.*$;\1;'` ; do
83+
ax_lib=${libextension}
84+
AC_CHECK_LIB($ax_lib, exit,
85+
[BOOST_IOSTREAMS_LIB="-l$ax_lib"; AC_SUBST(BOOST_IOSTREAMS_LIB) link_iostreams="yes"; break],
86+
[link_iostreams="no"])
87+
done
88+
if test "x$link_iostreams" != "xyes"; then
89+
for libextension in `ls $BOOSTLIBDIR/boost_iostreams*.dll* $BOOSTLIBDIR/boost_iostreams*.a* 2>/dev/null | sed 's,.*/,,' | sed -e 's;^\(boost_iostreams.*\)\.dll.*$;\1;' -e 's;^\(boost_iostreams.*\)\.a.*$;\1;'` ; do
90+
ax_lib=${libextension}
91+
AC_CHECK_LIB($ax_lib, exit,
92+
[BOOST_IOSTREAMS_LIB="-l$ax_lib"; AC_SUBST(BOOST_IOSTREAMS_LIB) link_iostreams="yes"; break],
93+
[link_iostreams="no"])
94+
done
95+
fi
96+
97+
else
98+
for ax_lib in $ax_boost_user_iostreams_lib boost_iostreams-$ax_boost_user_iostreams_lib; do
99+
AC_CHECK_LIB($ax_lib, main,
100+
[BOOST_IOSTREAMS_LIB="-l$ax_lib"; AC_SUBST(BOOST_IOSTREAMS_LIB) link_iostreams="yes"; break],
101+
[link_iostreams="no"])
102+
done
103+
104+
fi
105+
if test "x$ax_lib" = "x"; then
106+
AC_MSG_ERROR(Could not find a version of the library!)
107+
fi
108+
if test "x$link_iostreams" != "xyes"; then
109+
AC_MSG_ERROR(Could not link against $ax_lib !)
110+
fi
111+
fi
112+
113+
CPPFLAGS="$CPPFLAGS_SAVED"
114+
LDFLAGS="$LDFLAGS_SAVED"
115+
fi
116+
])

0 commit comments

Comments
 (0)