From a65e679ee6101a0a859bea635ea9dfd09292af0a Mon Sep 17 00:00:00 2001 From: Todd Eddy Date: Fri, 9 Mar 2007 22:29:35 +0000 Subject: [PATCH] stores data in both mysql and rrd now reorganized file structure a bit --- buildrrd.py | 24 ++++++++ inc/RRD.py | 108 +++++++++++++++++++++++++++++++++++ inc/__init__.py | 1 + inc/temp05.py | 40 +++++++++++++ temp05.sql => sql/temp05.sql | 0 temp05.py | 14 ----- temp05listener.py | 63 +++++++++++++------- 7 files changed, 214 insertions(+), 36 deletions(-) create mode 100644 buildrrd.py create mode 100644 inc/RRD.py create mode 100644 inc/__init__.py create mode 100755 inc/temp05.py rename temp05.sql => sql/temp05.sql (100%) delete mode 100755 temp05.py diff --git a/buildrrd.py b/buildrrd.py new file mode 100644 index 0000000..e199269 --- /dev/null +++ b/buildrrd.py @@ -0,0 +1,24 @@ +#!/usr/bin/env python +"""Initialize a RRD database + +This will create the initial rrd database using settings you have defined +This should only be run the first time you setup everything + +@todo more error checks +""" + +import sys +from inc import temp05 +from inc.RRD import * + +rrd = RRD(temp05.rrdfile) + +# you need to manually type the names in here they should be in the same +# order as they are read from the console. The name cannot have spaces +values = ( + ('basement', 'GAUGE', 'U', 'U'), + ('todds_room', 'GAUGE', 'U', 'U'), + ('outside', 'GAUGE', 'U', 'U'), + ) + +rrd.create_rrd(60, values) \ No newline at end of file diff --git a/inc/RRD.py b/inc/RRD.py new file mode 100644 index 0000000..f75ebe2 --- /dev/null +++ b/inc/RRD.py @@ -0,0 +1,108 @@ + +#!/usr/bin/env python + +# Copyright (c) 2005, Corey Goldberg +# +# RRD.py is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. + +""" + +@author: Corey Goldberg +@copyright: (C) 2005 Corey Goldberg +@license: GNU General Public License (GPL) +""" + + +import os + + +class RRD: + + """Data access layer that provides a wrapper for RRDtool. + + RRDtool is a data logging and graphing application. RRD (Round Robin Database) is a system + to store and display time-series data. It stores the data in a very compact way that will + not expand over time. + + @ivar rrd_name: Name (and path) of the RRD + """ + + def __init__(self, rrd_name): + """ + + @param rrd_name: Name (and path) of the RRD + """ + self.rrd_name = rrd_name + + + def create_rrd(self, interval, data_sources): + """Create a new RRD. + + If an RRD of the same name already exists, it will be replaced with the new one. + + @param interval: Interval in seconds that data will be fed into the RRD + @param data_sources: Nested sequence (e.g. List of tuples) of data sources (DS) to store in the RRD. + Each data source is passed in as: (name, type, min_value, max_value). If you try to update + a data source with a value that is not within its accepted range, it will be ignored. + min_value and max_value can be replaced with a 'U' to accept unlimited values. + """ + interval = str(interval) + interval_mins = float(interval) / 60 + # heartbeat defines the maximum number of seconds that may pass between two updates of this + # data source before the value of the data source is assumed to be *UNKNOWN*. + heartbeat = str(int(interval) * 2) + + # unpack the tuples from the list and build the string of data sources used to create the RRD + ds_string = '' + for ds_name, ds_type, ds_min, ds_max in data_sources: + ds_string = ''.join((ds_string, ' DS:', str(ds_name), ':', str(ds_type), ':', heartbeat, ':', str(ds_min), ':', str(ds_max))) + + # build the command line to send to RRDtool + cmd_create = ''.join(( + 'rrdtool create ', self.rrd_name, ' --step ', interval, ds_string, + ' RRA:AVERAGE:0.5:1:', str(int(4000 / interval_mins)), + ' RRA:AVERAGE:0.5:', str(int(30 / interval_mins)), ':800', + ' RRA:AVERAGE:0.5:', str(int(120 / interval_mins)), ':800', + ' RRA:AVERAGE:0.5:', str(int(1440 / interval_mins)), ':800', + )) + + # execute the command as a subprocess and return file objects (child_stdin, child_stdout_and_stderr) + cmd = os.popen4(cmd_create) + # read contents of the file object (child_stdout_and_stderr) until EOF + cmd_output = cmd[1].read() + # close the handles + for fd in cmd: fd.close() + # check if anything comes back (the only output would be stderr) + if len(cmd_output) > 0: + raise RRDException, "Unable to create RRD: " + cmd_output + + + def update(self, *values): + """Update the RRD with a new set of values. + + Updates must supply a set of values that match the number of data sources (DS) containted in the RRD. + (i.e. You can not update an RRD with 1 value when the RRD contains 2 DS's) + + @param values: Values to be inserted into the RRD (arbitrary number of scalar arguments) + """ + # we need the values in a colon delimited list to add to our command line + # so we take the list of values, convert them to strings and append a colon to each, + # join the list into a string, and chop the last character to remove the trailing colon + values_args = ''.join([str(value) + ":" for value in values])[:-1] + # build the command line to send to RRDtool + cmd_update = ''.join(('rrdtool update ', self.rrd_name, ' N:',)) + values_args + # execute the command as a subprocess and return file objects (child_stdin, child_stdout_and_stderr) + cmd = os.popen4(cmd_update) + # read contents of the file object (child_stdout_and_stderr) until EOF + cmd_output = cmd[1].read() + # close the handles + for fd in cmd: fd.close() + # check if anything comes back (the only output would be stderr) + if len(cmd_output) > 0: + raise RRDException, "Unable to update the RRD: " + cmd_output + + +class RRDException(Exception): pass diff --git a/inc/__init__.py b/inc/__init__.py new file mode 100644 index 0000000..cdea1e4 --- /dev/null +++ b/inc/__init__.py @@ -0,0 +1 @@ +__all__ = ['temp05'] diff --git a/inc/temp05.py b/inc/temp05.py new file mode 100755 index 0000000..8afbb7d --- /dev/null +++ b/inc/temp05.py @@ -0,0 +1,40 @@ +"""Custom functions and initial config settings""" + +import sys + +# database settings +dbHost = "localhost" +dbUser = "temp05admin" +dbPasswd = "m13ZnbR0qIdDJMu0" +dbName = "temp05" + +# the number of sensors you have +sensorCount = 3 + +# the logfile to write to, setting to /dev/null effectively disables it +logfile = 'log.txt' + +# the path and filename of rrd database +rrdfile = 'temp05.rrd' + +# --- end configuration --- +fsock = open(logfile, 'w') +sys.stdout = fsock + + +def registerSerial(db, serial, num): + """Registers the serial number in the db if not set already + + todo: should write this to a variable so it's not polling the database as much + """ + cursor = db.cursor() + + # see if it's already in there + cursor.execute("SELECT COUNT(*) FROM sensors WHERE id = %s", (serial)); + recordCount = cursor.fetchone() + #print "the count:", recordCount[0] + if recordCount[0] == 0: + #they are not in there, register them + #print " Sensor now registered in database, creating now" + cursor.execute("INSERT INTO sensors (id, number_ref) VALUES (%s, %s)", (serial, int(num))) + # todo: error handling diff --git a/temp05.sql b/sql/temp05.sql similarity index 100% rename from temp05.sql rename to sql/temp05.sql diff --git a/temp05.py b/temp05.py deleted file mode 100755 index 963af21..0000000 --- a/temp05.py +++ /dev/null @@ -1,14 +0,0 @@ -# registers the serial number in db if not set already -# todo: should write this to a variable so it's not polling the database as much -def registerSerial(db, serial, num): - cursor = db.cursor() - - # see if it's already in there - cursor.execute("SELECT COUNT(*) FROM sensors WHERE id = %s", (serial)); - recordCount = cursor.fetchone() - #print "the count:", recordCount[0] - if recordCount[0] == 0: - #they are not in there, register them - #print " Sensor now registered in database, creating now" - cursor.execute("INSERT INTO sensors (id, number_ref) VALUES (%s, %s)", (serial, int(num))) - # todo: error handling diff --git a/temp05listener.py b/temp05listener.py index 4e5542d..e6a8ff5 100755 --- a/temp05listener.py +++ b/temp05listener.py @@ -1,44 +1,52 @@ #!/usr/bin/env python -### -# TEMP05 Reader -# -# This listens on the serial port and will place sensor information into a database -# this requires the pySerial and MySQLdb packages -# -# this should background it properly -# python temp05listner.py & -# -# TODO -# write to log file (can't print if the process is backgrounded) -# write the temperature data to the sensors table as well (since I can't figure out how -# to get it from the readings table +"""TEMP05 Reader + +This listens on the serial port and will place sensor information into a database +this requires the pySerial and MySQLdb packages + +this should background it properly +python temp05listner.py & + +TODO +write to log file (can't print if the process is backgrounded) +write the temperature data to the sensors table as well (since I can't figure out how + to get it from the readings table +""" # load everything we need -import temp05 # custom functions +import sys +from inc import temp05, RRD # custom functions and vars import serial import MySQLdb import time +import string + + +if string.find(sys.argv[0], 'pydoc') <> -1: + """if this is called from pydoc, just print a message and exit since pydoc + will hang if it runs""" + print "This file does not support pydoc, please open the file file to view comments" + sys.exit("Unsupported command") # ---------------- # Setup Everything # ---------------- -sensorCount = 3 # the number of sensors you have (used in lowering db overhead) # Serial ser = serial.Serial() # defaults to 9600 8,N,1 ser.port = 0 # set to first port # MySQL -dbHost = "localhost" -dbUser = "temp05admin" -dbPasswd = "m13ZnbR0qIdDJMu0" -dbName = "temp05" -db = MySQLdb.connect(host=dbHost, user=dbUser, passwd=dbPasswd, db=dbName) +db = MySQLdb.connect(host=temp05.dbHost, user=temp05.dbUser, + passwd=temp05.dbPasswd, db=temp05.dbName) dbCursor = db.cursor() +# RRD +rrd = RRD.RRD(temp05.rrdfile) # open port ser.open() #print "Using port:", ser.portstr count = 1 while True: + """Start infinite loop that processes serial content""" line = ser.readline() line = line.strip() if line[0:6] == "Sensor": @@ -49,18 +57,29 @@ num = line[8:10] serial = line[11:27] temperature = line[29:-1] # don't want the F or C at the end, but you then have to remember which it is - #print "Sensor", num, "with serial", serial, "has the temperature", temperature + print "Sensor", num, "with serial", serial, "has the temperature", temperature temp05.registerSerial(db, serial, num) + # there must be an easier way to do this + if num == '01': + tempValue1 = temperature + elif num == '02': + tempValue2 = temperature + elif num == '03': + tempValue3 = temperature + # insert temperature dbCursor.execute("INSERT INTO readings (sensor_id, temperature) VALUES (%s, %s)", (serial, float(temperature))) dbCursor.execute("UPDATE sensors SET latest_temperature = %s, latest_reading_at = NOW() WHERE id = %s", (float(temperature), serial)) # commit after one round of sensor readings - if count >= sensorCount: + if count >= temp05.sensorCount: db.commit() dbCursor.close() dbCursor = db.cursor() + + # update rrd + rrd.update(tempValue1, tempValue2, tempValue3) count = 1 else: count = count + 1