bisq-network · ripcurlx · Feb 20, 2020 · Feb 17, 2020 · Feb 18, 2020 · Feb 20, 2020
diff --git a/monitor/collectd.conf b/monitor/collectd.conf
@@ -0,0 +1,136 @@
+Hostname "__ONION_ADDRESS__"
+Interval 30
+
+LoadPlugin syslog
+<Plugin syslog>
+	LogLevel info
+</Plugin>
+
+LoadPlugin cpu
+LoadPlugin df
+LoadPlugin disk
+LoadPlugin fhcount
+LoadPlugin interface
+LoadPlugin java
+LoadPlugin load
+LoadPlugin memory
+LoadPlugin processes
+LoadPlugin swap
+LoadPlugin write_graphite
+
+<Plugin cpu>
+	ReportByCpu true
+	ValuesPercentage true
+</Plugin>
+
+<Plugin df>
+	MountPoint "/"
+</Plugin>
+
+<Plugin disk>
+	Disk "/[hs]da/"
+</Plugin>
+
+<Plugin fhcount>
+	ValuesAbsolute false
+	ValuesPercentage true
+</Plugin>
+
+<Plugin interface>
+	Interface "eth0"
+</Plugin>
+
+<Plugin java>
+	JVMArg "-verbose:jni"
+	JVMArg "-Djava.class.path=/usr/share/collectd/java/collectd-api.jar:/usr/share/collectd/java/generic-jmx.jar"
+
+	LoadPlugin "org.collectd.java.GenericJMX"
+	<Plugin "GenericJMX">
+		# Generic heap/nonheap memory usage.
+		<MBean "memory">
+			ObjectName "java.lang:type=Memory"
+			#InstanceFrom ""
+			InstancePrefix "memory"
+
+			# Creates four values: committed, init, max, used
+			<Value>
+				Type "memory"
+				#InstancePrefix ""
+				#InstanceFrom ""
+				Table true
+				Attribute "HeapMemoryUsage"
+				InstancePrefix "heap-"
+			</Value>
+
+			# Creates four values: committed, init, max, used
+			<Value>
+				Type "memory"
+				#InstancePrefix ""
+				#InstanceFrom ""
+				Table true
+				Attribute "NonHeapMemoryUsage"
+				InstancePrefix "nonheap-"
+			</Value>
+		</MBean>
+
+		# Memory usage by memory pool.
+		<MBean "memory_pool">
+			ObjectName "java.lang:type=MemoryPool,*"
+			InstancePrefix "memory_pool-"
+			InstanceFrom "name"
+
+			<Value>
+				Type "memory"
+				#InstancePrefix ""
+				#InstanceFrom ""
+				Table true
+				Attribute "Usage"
+			</Value>
+		</MBean>
+
+		<Connection>
+			ServiceURL "service:jmx:rmi:///jndi/rmi://localhost:6969/jmxrmi"
+			Collect "memory_pool"
+			Collect "memory"
+		</Connection>
+
+		# See /usr/share/doc/collectd/examples/GenericJMX.conf
+		# for an example config.
+	</Plugin>
+</Plugin>
+
+#<Plugin load>
+#	ReportRelative true
+#</Plugin>
+
+#<Plugin memory>
+#	ValuesAbsolute true
+#	ValuesPercentage false
+#</Plugin>
+
+#<Plugin processes>
+#	Process "name"
+#	ProcessMatch "foobar" "/usr/bin/perl foobar\\.pl.*"
+#</Plugin>
+
+#<Plugin swap>
+#	ReportByDevice false
+#	ReportBytes true
+#</Plugin>
+
+<Plugin write_graphite>
+	<Node "node">
+		Host "127.0.0.1"
+		Port "2003"
+		Protocol "tcp"
+		ReconnectInterval 0
+		LogSendErrors false
+		Prefix "servers."
+		StoreRates true
+		AlwaysAppendDS false
+		EscapeCharacter "_"
+		SeparateInstances false
+		PreserveSeparator false
+		DropDuplicateFields false
+	</Node>
+</Plugin>
diff --git a/monitor/install_collectd_debian.sh b/monitor/install_collectd_debian.sh
@@ -0,0 +1,66 @@
+#!/bin/bash
+set -e
+
+echo "[*] Bisq Server Monitoring installation script"
+
+##### change paths if necessary for your system
+BISQ_REPO_URL=https://raw.githubusercontent.com/bisq-network/bisq
+BISQ_REPO_TAG=master
+ROOT_USER=root
+ROOT_GROUP=root
+ROOT_HOME=~root
+ROOT_PKG=(nginx collectd openssl)
+
+SYSTEMD_ENV_HOME=/etc/default
+
+#####
+
+echo "[*] Gathering information"
+read -p "Please provide the onion address of your service (eg. 3f3cu2yw7u457ztq): " onionaddress
+
+echo "[*] Updating apt repo sources"
+sudo -H -i -u "${ROOT_USER}" DEBIAN_FRONTEND=noninteractive apt-get update -q
+
+echo "[*] Upgrading OS packages"
+sudo -H -i -u "${ROOT_USER}" DEBIAN_FRONTEND=noninteractive apt-get upgrade -qq -y
+
+echo "[*] Installing base packages"
+sudo -H -i -u "${ROOT_USER}" DEBIAN_FRONTEND=noninteractive apt-get install -qq -y ${ROOT_PKG[@]}
+
+echo "[*] Preparing Bisq init script for monitoring"
+# remove stuff it it is there already
+sudo -H -i -u "${ROOT_USER}" sed -i -e 's/-Dcom.sun.management.jmxremote //g' -e 's/-Dcom.sun.management.jmxremote.local.only=true//g' -e 's/ -Dcom.sun.management.jmxremote.host=127.0.0.1//g' -e 's/ -Dcom.sun.management.jmxremote.port=6969//g' -e 's/ -Dcom.sun.management.jmxremote.rmi.port=6969//g' -e 's/ -Dcom.sun.management.jmxremote.ssl=false//g' -e 's/ -Dcom.sun.management.jmxremote.authenticate=false//g' "${SYSTEMD_ENV_HOME}/bisq.env"
+sudo -H -i -u "${ROOT_USER}" sed -i -e '/JAVA_OPTS/ s/"$/ -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.local.only=true -Dcom.sun.management.jmxremote.port=6969 -Dcom.sun.management.jmxremote.rmi.port=6969 -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false"/' "${SYSTEMD_ENV_HOME}/bisq.env"
+
+echo "[*] Seeding entropy from /dev/urandom"
+sudo -H -i -u "${ROOT_USER}" /bin/sh -c "head -1500 /dev/urandom > ${ROOT_HOME}/.rnd"
+echo "[*] Installing Nginx config"
+sudo -H -i -u "${ROOT_USER}" openssl req -x509 -nodes -newkey rsa:2048 -keyout /etc/nginx/cert.key -out /etc/nginx/cert.crt -subj="/O=Bisq/OU=Bisq Infrastructure/CN=$onionaddress"
+curl -s "${BISQ_REPO_URL}/${BISQ_REPO_TAG}/monitor/nginx.conf" > /tmp/nginx.conf
+sudo -H -i -u "${ROOT_USER}" install -c -o "${ROOT_USER}" -g "${ROOT_GROUP}" -m 644 /tmp/nginx.conf /etc/nginx/nginx.conf
+
+echo "[*] Installing collectd config"
+curl -s "${BISQ_REPO_URL}/${BISQ_REPO_TAG}/monitor/collectd.conf" > /tmp/collectd.conf
+sudo -H -i -u "${ROOT_USER}" install -c -o "${ROOT_USER}" -g "${ROOT_GROUP}" -m 644 /tmp/collectd.conf /etc/collectd/collectd.conf
+sudo -H -i -u "${ROOT_USER}" sed -i -e "s/__ONION_ADDRESS__/$onionaddress/" /etc/collectd/collectd.conf
+
+echo "[*] Updating systemd daemon configuration"
+sudo -H -i -u "${ROOT_USER}" systemctl daemon-reload
+sudo -H -i -u "${ROOT_USER}" systemctl enable nginx.service
+sudo -H -i -u "${ROOT_USER}" systemctl enable collectd.service
+
+echo "[*] Restarting services"
+sudo -H -i -u "${ROOT_USER}" systemctl restart bisq.service
+sudo -H -i -u "${ROOT_USER}" systemctl restart nginx.service
+sudo -H -i -u "${ROOT_USER}" systemctl restart collectd.service
+
+echo '[*] Done!'
+
+echo '  '
+echo '[*] Report this certificate to the monitoring team!'
+echo '----------------------------------------------------------------'
+echo "Server: $onionaddress"
+echo '  '
+cat /etc/nginx/cert.crt
+echo '----------------------------------------------------------------'
+echo '  '
diff --git a/monitor/nginx.conf b/monitor/nginx.conf
@@ -0,0 +1,30 @@
+load_module /usr/lib/nginx/modules/ngx_stream_module.so;
+
+worker_processes  1;
+
+events {
+    worker_connections  1024;
+}
+
+stream {
+
+	log_format basic '$remote_addr [$time_local] '
+	                 '$protocol Status $status Sent $bytes_sent Received $bytes_received '
+	                 'Time $session_time';
+
+	error_log syslog:server=unix:/dev/log;
+	access_log syslog:server=unix:/dev/log basic;
+
+	server {
+		listen 2003;
+		allow 127.0.0.1;
+		deny all;
+		proxy_pass monitor.bisq.network:2002;
+		proxy_ssl on;
+
+		proxy_ssl_certificate /etc/nginx/cert.crt;
+		proxy_ssl_certificate_key /etc/nginx/cert.key;
+
+		proxy_ssl_session_reuse on;
+	}
+}
diff --git a/monitor/uninstall_collectd_debian.sh b/monitor/uninstall_collectd_debian.sh
@@ -0,0 +1,11 @@
+#!/bin/sh
+echo "[*] Stopping Bisq Server monitoring utensils"
+echo '  '
+echo 'This script will not remove any configuration or binaries from the system. It just stops the services.'
+
+sleep 10
+sudo systemctl stop nginx
+sudo systemctl stop collectd
+sudo systemctl disable nginx
+sudo systemctl disable collectd
+echo "[*] Done!"
diff --git a/pricenode/collectd.conf.snippet b/pricenode/collectd.conf.snippet
@@ -0,0 +1,5 @@
+LoadPlugin exec
+
+<Plugin exec>
+	Exec "__USER_GROUP__" "__SCRAPERSCRIPT__"
+</Plugin>
diff --git a/pricenode/install_networksize_debian.sh b/pricenode/install_networksize_debian.sh
@@ -0,0 +1,42 @@
+#!/bin/sh
+set -e
+
+echo "[*] Network Size Monitoring installation script"
+
+##### change paths if necessary for your system
+
+ROOT_USER=root
+
+SCRAPER_HOME=/journalreader
+SCRAPER_USER=journalreader
+SCRAPER_GROUP=systemd-journal
+
+#####
+echo "[*] Checking environment..."
+if [ ! -f "/etc/collectd/collectd.conf" ]; then
+	echo 'Collectd is not installed. Did you do the install_monitoring_debian.sh?'
+	echo 'Exiting...'
+	exit
+fi
+
+echo "[*] Creating journal reader user"
+sudo -H -i -u "${ROOT_USER}" useradd -d "${SCRAPER_HOME}" -G "${SCRAPER_GROUP}" "${SCRAPER_USER}"
+sudo -H -i -u "${ROOT_USER}" mkdir -p "${SCRAPER_HOME}"
+sudo -H -i -u "${ROOT_USER}" chown "${SCRAPER_USER}":"${SCRAPER_GROUP}" ${SCRAPER_HOME}
+
+echo "[*] Installing journal parser script"
+curl -s https://raw.githubusercontent.com/bisq-network/bisq/master/pricenode/journalscraper.sh > /tmp/journalscraper.sh
+sudo -H -i -u "${ROOT_USER}" install -c -o "${SCRAPER_USER}" -g "${SCRAPER_GROUP}" -m 744 /tmp/journalscraper.sh "${SCRAPER_HOME}/scraperscript.sh"
+
+echo "[*] Installing collectd config"
+curl -s https://raw.githubusercontent.com/bisq-network/bisq/master/pricenode/collectd.conf.snippet > /tmp/collectd.conf.snippet
+sudo -H -i -u "${ROOT_USER}" /bin/sh -c "cat /tmp/collectd.conf.snippet >> /etc/collectd/collectd.conf"
+sudo -H -i -u "${ROOT_USER}" sed -i -e "s/__USER_GROUP__/${SCRAPER_USER}:${SCRAPER_GROUP}/" /etc/collectd/collectd.conf
+sudo -H -i -u "${ROOT_USER}" sed -i -e "s!__SCRAPERSCRIPT__!${SCRAPER_HOME}/scraperscript.sh!" /etc/collectd/collectd.conf
+
+sudo -H -i -u "${ROOT_USER}" systemctl enable collectd.service
+
+echo "[*] Restarting services"
+sudo -H -i -u "${ROOT_USER}" systemctl restart collectd.service
+
+echo '[*] Done!'
diff --git a/pricenode/journalscraper.sh b/pricenode/journalscraper.sh
@@ -0,0 +1,20 @@
+#!/bin/sh
+
+HOSTNAME="${COLLECTD_HOSTNAME:-localhost}"
+INTERVAL=750
+
+last=$(date +"%F %T" -d "$INTERVAL seconds ago")
+while true;
+do
+	now=$(date +"%F %T")
+
+	journalctl -u bisq --since="$last" --until="$now" | grep -Eo "getAllMarketPrices.*bisq/[0-9].[0-9].[0-9]" | cut -d / -f 2 | sort | uniq -c | while read -r line; do
+		number=$(echo "${line}" | cut -d ' ' -f 1);
+		version=$(echo "${line}" | cut -d \  -f 2);
+		version=${version//./_};
+		echo "PUTVAL $HOSTNAME/requestsPer750Seconds/gauge-v$version interval=$INTERVAL N:$number";
+	done
+	last=$now
+
+	sleep $INTERVAL
+done
diff --git a/seednode/README.md b/seednode/README.md
@@ -61,6 +61,15 @@ macOS:
 /Applications/Bisq.app/Contents/MacOS/Bisq --seedNodes=foo.onion:8000 --btcNodes=foo.onion:8333
 ```
 
+### Monitoring
+
+If you run a main seednode, you also are obliged to activate the monitoring feed by running
+
+```bash
+curl -s https://raw.githubusercontent.com/bisq-network/bisq/master/monitor/install_collectd_debian.sh | sudo bash
+```
+Follow the instruction given by the script and report your certificate to the seednode group!
+
 ### Upgrading
 
 To upgrade your seednode to a new tag, for example v1.2.5

diff --git a/seednode/bisq.env b/seednode/bisq.env
@@ -5,7 +5,7 @@
 JAVA_HOME=/usr/lib/jvm/openjdk-10.0.2
 
 # java memory and remote management options
-JAVA_OPTS="-Xms4096M -Xmx4096M -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.host=127.0.0.1 -Dcom.sun.management.jmxremote.port=6969 -Dcom.sun.management.jmxremote.rmi.port=6969 -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false"
+JAVA_OPTS="-Xms4096M -Xmx4096M"
 
 # bitcoin rpc credentials
 BITCOIN_RPC_USER=__BITCOIN_RPC_USER__