Poller service

.gitignore

@@ -10,3 +10,4 @@ nbproject
 .alerts.lock
 .ircbot.alert
 .metadata_never_index
+*.swp

discovery.php

@@ -106,7 +106,9 @@
 }
 
 foreach (dbFetch("SELECT * FROM `devices` WHERE status = 1 AND disabled = 0 $where ORDER BY device_id DESC") as $device) {
-    discover_device($device, $options);
+    if (dbGetLock('discovering.' . $device['device_id'])) {
+        discover_device($device, $options);
+    }
 }
 
 $end      = utime();

doc/Extensions/Poller-Service.md

@@ -0,0 +1,29 @@
+# Poller Service
+The Poller service is an alternative to polling and discovery cron jobs and provides support for distributed polling without memcache. It is multi-threaded and runs continuously discovering and polling devices with the oldest data attempting to honor the polling frequency configured in `config.php`. This service replaces all the required cron jobs except for `/opt/librenms/daily.sh` and `/opt/librenms/alerts.php`.
+
+Configure the maximum number of threads for the service in `$config['poller_service_workers']`. Configure the minimum desired polling frequency in `$config['poller_service_poll_frequency']` and the minimum desired discovery frequency in `$config['poller_service_discover_frequency']`. The service will not poll or discover devices which have data newer than this this configured age in seconds. Configure how frequently the service will attempt to poll devices which are down in `$config['poller_service_down_retry']`.
+
+The poller service is designed to gracefully degrade. If not all devices can be polled within the configured frequency, the service will continuously poll devices refreshing as frequently as possible using the configured number of threads.
+
+The service logs to syslog. A loglevel of INFO will print status updates every 5 minutes. Loglevel of DEBUG will print updates on every device as it is scanned.
+
+## Configuration
+```php
+// Poller-Service settings
+$config['poller_service_loglevel']                       = "INFO";
+$config['poller_service_workers']                        = 16;
+$config['poller_service_poll_frequency']                 = 300;
+$config['poller_service_discover_frequency']             = 21600;
+$config['poller_service_down_retry']                     = 60;
+```
+
+## Distributed Polling
+Distributed polling is possible, and uses the same configuration options as are described for traditional distributed polling, except that the memcached options are not necessary. The database must be acessable from the distributed pollers, and properly configured. Remote access to the RRD directory must also be configured as described in the Distributed Poller documentation. Memcache is not required. Concurrency is managed using mysql GET_LOCK to ensure that devices are only being polled by one device at at time. The poller service is compatible with poller groups.
+
+## Multi-Master MySQL considerations
+Because locks are not replicated in Multi-Master MySQL configurations, if you are using such a configuration, you will need to make sure that all pollers are using the same MySQL server.
+
+## Service Installation
+An upstart configuration `poller-service.conf` is provided. To install run `ln -s /opt/librenms/poller-service.conf /etc/init/poller-service.conf`. The service will start on boot and can be started manually by running `start poller-service`. If you recieve an error that the service does not exist, run `initctl reload-configuration`. The service is configured to run as the user `librenms` and will fail if that user does not exist.
+
+An LSB init script `poller-service.init` is also provided. To install run `ln -s /opt/librenms/poller-service.init /etc/init.d/poller-service && update-rc.d poller-service defaults`.

includes/dbFacile.php

@@ -59,6 +59,40 @@ function dbQuery($sql, $parameters=array()) {
 }//end dbQuery()
 
 
+/*
+ * Aquire a lock on a string
+ * */
+
+
+function dbGetLock($data, $timeout = 0) {
+    $sql = 'SELECT GET_LOCK(\'' . $data . '\',' . $timeout . ')';
+    $result = dbFetchCell($sql);
+    return $result;
+}
+
+/*
+ * Check a lock on a string
+ * */
+
+
+function dbCheckLock($data) {
+    $sql = 'SELECT IS_FREE_LOCK(\'' . $data . '\')';
+    $result = dbFetchCell($sql);
+    return $result;
+}
+
+/*
+ * Release a lock on a string
+ * */
+
+
+function dbReleaseLock($data) {
+    $sql = 'SELECT RELEASE_LOCK(\'' . $data . '\')';
+    $result = dbFetchCell($sql);
+    return $result;
+}
+
+
 /*
  * Passed an array and a table name, it attempts to insert the data into the table.
  * Check for boolean false to determine whether insert failed

includes/sql-schema/update.php

@@ -84,6 +84,22 @@
 
 asort($filelist);
 
+if (explode('.', max($filelist), 2)[0] <= $db_rev) {
+    if ($debug) {
+        echo "DB Schema already up to date.\n";
+    }
+    return;
+}
+
+if (!dbGetLock('schema_update')) {
+    echo "Schema update already in progress. Exiting\n";
+    exit(1);
+} //end if
+
+do {
+    sleep(1);
+} while (@dbFetchCell('SELECT COUNT(*) FROM `devices` WHERE NOT IS_FREE_LOCK(CONCAT("polling.", device_id)) OR NOT IS_FREE_LOCK(CONCAT("queued.", device_id)) OR NOT IS_FREE_LOCK(CONCAT("discovering.", device_id))') > 0);
+
 foreach ($filelist as $file) {
     list($filename,$extension) = explode('.', $file, 2);
     if ($filename > $db_rev) {
@@ -145,3 +161,5 @@
 
     echo "-- Done\n";
 }
+
+dbReleaseLock('schema_update');

poller-service.conf

@@ -0,0 +1,20 @@
+# poller-service - SNMP polling service for LibreNMS
+
+description "SNMP polling service for LibreNMS"
+author "Clint Armstrong <clint@clintarmstrong.net>"
+
+# When to start the service
+start on runlevel [2345]
+
+# When to stop the service
+stop on runlevel [016]
+
+# Automatically restart process if crashed
+respawn
+
+chdir /opt/librenms
+setuid librenms
+setgid librenms
+
+# Start the process
+exec /opt/librenms/poller-service.py

poller-service.init

@@ -0,0 +1,79 @@
+### BEGIN INIT INFO
+# Provides:          poller-service
+# Required-Start:    networking
+# Required-Stop:     networking
+# Default-Start:     2 3 4 5
+# Default-Stop:      0 1 6
+# Short-Description: The LibreNMS poller-service daemon
+# Description:       The LibreNMS poller-service daemon
+#                    This polls devices monitored by LibreNMS
+### END INIT INFO
+
+. /lib/lsb/init-functions
+
+NAME=poller-service
+
+DAEMON=/opt/librenms/poller-service.py
+
+USER=librenms
+
+PIDFILE=/var/run/poller-service.pid
+
+test -x $DAEMON || exit 5
+
+case $1 in
+
+ start)
+  # Checked the PID file exists and check the actual status of process
+  if [ -e $PIDFILE ]; then
+   status_of_proc -p $PIDFILE $DAEMON "$NAME process" && status="0" || status="$?"
+   # If the status is SUCCESS then don't need to start again.
+   if [ $status = "0" ]; then
+    exit # Exit
+   fi
+  fi
+  # Start the daemon.
+  log_daemon_msg "Starting the process" "$NAME"
+  # Start the daemon with the help of start-stop-daemon
+  # Log the message appropriately
+  if start-stop-daemon --start --quiet --oknodo --make-pidfile --pidfile $PIDFILE --exec $DAEMON --chuid $USER --background; then
+   log_end_msg 0
+  else
+   log_end_msg 1
+  fi
+  ;;
+
+ stop)
+  # Stop the daemon.
+  if [ -e $PIDFILE ]; then
+   status_of_proc -p $PIDFILE $DAEMON "Stoppping the $NAME process" && status="0" || status="$?"
+   if [ "$status" = 0 ]; then
+    start-stop-daemon --stop --quiet --oknodo --pidfile $PIDFILE
+    /bin/rm -rf $PIDFILE
+   fi
+  else
+   log_daemon_msg "$NAME process is not running"
+   log_end_msg 0
+  fi
+  ;;
+ restart)
+  # Restart the daemon.
+  $0 stop && sleep 2 && $0 start
+  ;;
+
+ status)
+  # Check the status of the process.
+  if [ -e $PIDFILE ]; then
+   status_of_proc -p $PIDFILE $DAEMON "$NAME process" && exit 0 || exit $?
+  else
+   log_daemon_msg "$NAME Process is not running"
+   log_end_msg 0
+  fi
+  ;;
+
+ *)
+  # For invalid arguments, print the usage message.
+  echo "Usage: $0 {start|stop|restart|reload|status}"
+  exit 2
+  ;;
+esac