@@ -15,7 +15,7 @@ r = IPTTrafficRules('filter', graph_periods=(600,3600,86400),# time periods to graph graph_img_width=512, graph_img_height=256, - commit_interval=16, # only write data to disk every 16th step + commit_interval=4, # write data to disk every 4th step ) # Traffic and counter-specific graphing config @@ -55,7 +55,7 @@ r2 = IP6TTrafficRules('filter', ('eth+',), step=2, rrddb_base_filename='rrd_ip6/', graph_base_filename='ip6_', - commit_interval=16) + commit_interval=4) r2.rule_add('udp_other', ('-p udp',), color='#50FF50') r2.rule_add('tcp_other', ('-p tcp',), color='#c0c0c0') examples/teucrium.conf.example @@ -15,30 +15,85 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. +import logging import os -import rrdtool try: from gonium.linux.xtables import XTablesPoller except ImportError: pass +from gonium.fd_management import CHILD_REACT_KILL + from constants import CT_BYTES, CT_PACKETS from rrd_fn import RRDFileNamer class RRDTrafficCounter(RRDFileNamer): - def __init__(self, rrd_base_filename, rules2ds, chain2diriface, commit_interval): + logger = logging.getLogger('RRDTrafficCounter') + log = logger.log + def __init__(self, ed, rrd_base_filename, rules2ds, chain2diriface, commit_interval): + self.ed = ed self.rrd_base_filename = rrd_base_filename self.rules2ds = rules2ds self.chain2diriface = chain2diriface self.commit_interval = commit_interval self.commit_index = 0 self.output_cache = {} + self.rrd_child = None + self.rrd_line_cache = [] + + def rrd_child_spawn(self): + if not (self.rrd_child is None): + raise StandardError('I already have an active rrd_child: %r' % self.rrd_child) + self.rrd_child = self.ed.ChildRunnerPopen4(('rrdtool', '-'), + self.child_termination_process, finish=CHILD_REACT_KILL, + input_handler=self.child_input_process) + + def child_input_process(self, child, fd): + lines = child.buffers_input[fd].split('\n') + child.buffers_input[fd] = lines[-1] + del(lines[-1]) + idx_start = 0 + for i in range(len(lines)): + line = lines[i] + if (line.startswith('OK')): + if (self.rrd_line_cache): + del(self.rrd_line_cache[:]) + idx_start = i+1 + self.log(20, 'Sucessfully executed rrd command: %r' % (line,)) + continue + if (line.startswith('ERROR')): + error_lines = self.rrd_line_cache + lines[idx_start:i+1] + self.log(38, 'rrdtool error: %r' % '\n'.join(error_lines)) + idx_start = i+1 + if (self.rrd_line_cache): + del(self.rrd_line_cache[:]) + continue + if (line.startswith('For more information read the RRD manpages')): + self.log(40, 'Noticed rrdtool syntax error!') + + self.rrd_line_cache.extend(lines[idx_start:]) + + def child_termination_process(self, child, return_code, exit_status): + self.log(26, '%r notes termination of rrdtool child process RC: %r ES:' + '%r output: %r.' % (self,return_code, exit_status)) + self.rrd_child = None + self.rrd_line_cache = [] + + def rrdfiles_update(self, fn, val_seq): + if (self.rrd_child is None): + self.rrd_child_spawn() + + # Using full python string escape sequences isn't *exactly* correct, but + # as long as people don't try to deliberately mess up their own setup, + # it shouldn't cause any problems. + self.rrd_child.send_data('update %r -t %r %s\n' % (fn, self.DS_RAW, + ' '.join('%s:%s' % (tss, val) for (tss,val) in val_seq))) @classmethod def build_with_xtp(cls, ed, interval, xt, tables, *args, **kwargs): xtp = XTablesPoller(ed, interval, xt, tables) - self = cls(*args, **kwargs) + self = cls(ed, *args, **kwargs) self.xtp = xtp xtp.em_xtentries.EventListener(self.xtp_data_process) # Write buffered data on shutdown @@ -47,8 +102,7 @@ class RRDTrafficCounter(RRDFileNamer): def rrd_data_commit(self): for ((iface, dir_, ds, ct), val_list) in self.output_cache.items(): target_fn = self.rrd_fn_get(iface, dir_, ct, ds) - rrdtool.update(target_fn, '-t', self.DS_RAW, - *('%s:%s' % (tss, val) for (tss,val) in val_list)) + self.rrdfiles_update(target_fn, val_list) del(val_list[:]) def rrd_data_queue(self, iface, dir_, ds_l, tss, cbytes_l, cpackets_l): src/teucrium/rrd_tc.py 2d40654298b169a7c444df8e4bb235676c0e7e4f Sebastian Hagen sebastian_hagen@memespace.net http://github.com/sh01/teucrium/commit/ac330e4d6adf842100f0ff3bde7e28a67c0cbcea ac330e4d6adf842100f0ff3bde7e28a67c0cbcea 2009-03-05T06:17:50-08:00 2009-03-05T06:17:50-08:00 Change rrd update method to pipe-controlled rrdtool child processes, allowing non-blocking data commits. a2fd2a12e1bb9c09ddb0538814d7d9e54abd25a5 Sebastian Hagen sebastian_hagen@memespace.net