Provisioning Bitcoin Network Crawler

Addy Yeow edited this page Aug 7, 2017 · 52 revisions

Launch Server

Hetzner PX91-SSD Dedicated Server 128GB Debian 8 64-bit (Nuremberg, DE)

Initial Login

ssh root@[IP_ADDRESS]
    [PASSWORD]

Change Password

passwd
    [PASSWORD]

Configure Hostname

vi /etc/hostname
    [HOSTNAME]

vi /etc/hosts
    127.0.0.1 localhost
    [IP_ADDRESS] [HOSTNAME]
    ::1 ip6-localhost ip6-loopback
    fe00::0 ip6-localnet
    ff00::0 ip6-mcastprefix
    ff02::1 ip6-allnodes
    ff02::2 ip6-allrouters
    ff02::3 ip6-allhosts
    [IPV6_ADDRESS] [HOSTNAME]

Configure Locales

dpkg-reconfigure locales
    en_US.UTF-8 UTF-8
    en_US.UTF-8

vi /etc/environment
    LC_CTYPE=en_US.UTF-8

Configure Timezone

dpkg-reconfigure tzdata
    None of the above
    UTC

Update Packages

apt-get update; apt-get upgrade

Install Packages

apt-get -y install htop sudo unzip build-essential python-dev python-virtualenv tcl subversion

Add User

adduser [USERNAME]
    [PASSWORD]
adduser [USERNAME] sudo
mkdir -p /home/[USERNAME]/.ssh; chmod 700 /home/[USERNAME]/.ssh
vi /home/[USERNAME]/.ssh/authorized_keys
    [PUBLICKEY]
chmod 600 /home/[USERNAME]/.ssh/authorized_keys; chown -R [USERNAME]:[USERNAME] /home/[USERNAME]/.ssh

Update /home/[USERNAME]/.bashrc

export REDIS_SOCKET=/tmp/redis.sock
export REDIS_PASSWORD=[PASSWORD]

Update /etc/ssh/sshd_config

Port [PORT]
LoginGraceTime 60
PermitRootLogin no
AllowUsers [USERNAME]
PasswordAuthentication no

Update /etc/sysctl.conf

net.ipv4.conf.default.rp_filter=1
net.ipv4.conf.all.rp_filter=1
net.ipv4.tcp_syncookies=1
net.ipv4.conf.all.accept_redirects=0
net.ipv6.conf.all.accept_redirects=0
net.ipv4.conf.all.send_redirects=0
net.ipv4.conf.all.accept_source_route=0
net.ipv6.conf.all.accept_source_route=0
net.ipv4.conf.all.log_martians=1
net.core.rmem_default=33554432
net.core.wmem_default=33554432
net.core.rmem_max=33554432
net.core.wmem_max=33554432
net.core.optmem_max=33554432
net.ipv4.tcp_rmem=10240 87380 33554432
net.ipv4.tcp_wmem=10240 87380 33554432
net.ipv4.ip_local_port_range=2000 65500
net.core.netdev_max_backlog=100000
net.ipv4.tcp_max_syn_backlog=80000
net.ipv4.tcp_max_tw_buckets=2000000
net.ipv4.tcp_tw_reuse=1
net.ipv4.tcp_fin_timeout=5
net.ipv4.tcp_slow_start_after_idle=0
net.core.somaxconn=60000
fs.file-max=1000000
vm.swappiness=10
vm.min_free_kbytes=1048576
vm.overcommit_memory=1

Update /etc/security/limits.conf

* soft nofile 1000000
* hard nofile 1000000

Update /etc/rc.local

echo never > /sys/kernel/mm/transparent_hugepage/enabled
echo never > /sys/kernel/mm/transparent_hugepage/defrag
/sbin/ifconfig eth0 txqueuelen 5000

Reboot Server

reboot

Normal User Login

ssh -p [PORT] -i [PRIVATEKEY] [USERNAME]@[IP_ADDRESS]

Install Redis

cd; wget http://download.redis.io/releases/redis-3.2.8.tar.gz
tar xzf redis-3.2.8.tar.gz; cd redis-3.2.8; make; make test
sudo make install
cd utils; sudo bash install_server.sh
sudo vi /etc/init.d/redis_0
*******************************************************************************
#!/bin/sh

##### BEGIN INIT INFO
# Provides:          redis_0
# Required-Start:    $network $local_fs $remote_fs
# Required-Stop:     $network $local_fs $remote_fs
# Default-Start:     2 3 4 5
# Default-Stop:      0 1 6
# Should-Start:      $syslog $named
# Should-Stop:       $syslog $named
# Short-Description: Starts redis_0
# Description:       Starts redis_0 using start-stop-daemon
##### END INIT INFO

EXEC=/usr/local/bin/redis-server
CLIEXEC=/usr/local/bin/redis-cli
PIDFILE=/var/run/redis_0.pid
CONF="/etc/redis/0.conf"
REDISSOCKET="/tmp/redis.sock"
REDISPASSWORD="[PASSWORD]"

case "$1" in
    start)
        if [ -f $PIDFILE ]
        then
            echo "$PIDFILE exists, process is already running or crashed"
        else
            echo "Starting Redis server..."
            $EXEC $CONF
        fi
        ;;
    stop)
        if [ ! -f $PIDFILE ]
        then
            echo "$PIDFILE does not exist, process is not running"
        else
            PID=$(cat $PIDFILE)
            echo "Stopping ..."
            $CLIEXEC -s $REDISSOCKET -a $REDISPASSWORD shutdown
            while [ -x /proc/${PID} ]
            do
                echo "Waiting for Redis to shutdown ..."
                sleep 1
            done
            echo "Redis stopped"
        fi
        ;;
    *)
        echo "Please use start or stop as first argument"
        ;;
esac
*******************************************************************************
sudo update-rc.d redis_0 defaults
sudo vi /etc/redis/0.conf
    unixsocket /tmp/redis.sock
    unixsocketperm 777
    #save 900 1
    #save 300 10
    #save 60 10000
    requirepass [PASSWORD]
    maxclients 50000
    maxmemory 68719476736
    maxmemory-policy volatile-ttl
    notify-keyspace-events K$z
    activerehashing no
    client-output-buffer-limit slave 512mb 256mb 300
    client-output-buffer-limit pubsub 512mb 256mb 300
    hz 20

Reboot Server

sudo reboot

Normal User Login

ssh -p [PORT] -i [PRIVATEKEY] [USERNAME]@[IP_ADDRESS]

Launch Crawler

cd; wget --no-check-certificate https://github.com/ayeowch/bitnodes/archive/master.zip
unzip master.zip
virtualenv ~/.virtualenvs/bitnodes
source ~/.virtualenvs/bitnodes/bin/activate
cd bitnodes-master
pip install -r requirements.txt
bash geoip/update.sh
bash start.sh
cd data/pcap/f9beb4d9
sudo rm *.pcap; sudo tcpdump -i eth0 -w %s.eth0.pcap -v -n -G 2 -B 65536 -Z [USERNAME] 'tcp and not src host [IP_ADDRESS] and not src host [IPV6_ADDRESS]' > eth0 2>&1 &
sudo tcpdump -i lo -w %s.lo.pcap -v -n -G 2 -B 65536 -Z [USERNAME] 'tcp and port 9050' > lo 2>&1 &
Clone this wiki locally
You can’t perform that action at this time.
You signed in with another tab or window. Reload to refresh your session. You signed out in another tab or window. Reload to refresh your session.
Press h to open a hovercard with more details.