Makefile.PL bin/moonin-poller conf/moonin.conf extra/VeraMono.ttf lib/Moonin/Config.pm lib/Moonin/Config/Store.pm lib/Moonin/Domain.pm lib/Moonin/Exception.pm lib/Moonin/Graph.pm lib/Moonin/Node.pm lib/Moonin/Role/Exception.pm lib/Moonin/Web.pm lib/Moonin/Web/Controller/Graph.pm lib/Moonin/Web/Controller/Node.pm lib/Moonin/Web/Controller/Root.pm lib/Moonin/Web/Model/Config.pm lib/Moonin/Web/Model/Graph.pm lib/Moonin/Web/Model/Node.pm lib/Moonin/Web/View/TT.pm log/.emptyfile moonin_web.yml root/favicon.ico root/index.tt root/node/show.tt root/static/images/btn_120x50_built.png root/static/images/btn_120x50_built_shadow.png root/static/images/btn_120x50_powered.png root/static/images/btn_120x50_powered_shadow.png root/static/images/btn_88x31_built.png root/static/images/btn_88x31_built_shadow.png root/static/images/btn_88x31_powered.png root/static/images/btn_88x31_powered_shadow.png root/static/images/catalyst_logo.png rrds/.emptyfile script/moonin_web_cgi.pl script/moonin_web_create.pl script/moonin_web_fastcgi.pl script/moonin_web_server.pl script/moonin_web_test.pl t/01app.t t/02pod.t t/03podcoverage.t t/04moonin-graph.pm t/controller_Graph.t t/controller_Node.t t/model_ConfigStore.t t/view_TT.t tmp/.emptyfile @@ -24,7 +24,7 @@ has 'config' => ( is => 'rw', isa => 'HashRef', required => 0 ); has 'config_file' => ( is => 'rw', isa => 'Str', required => 1 ); has 'missingok' => ( is => 'rw', isa => 'Bool', required => 0, default => 0 ); has 'corruptok' => ( is => 'rw', isa => 'Bool', required => 0, default => 0 ); -has 'store' => ( is => 'ro', isa => 'Object', required => 0 ); +has 'store' => ( is => 'ro', isa => 'Object', required => 0 ); has 'legal' => ( is => 'ro', isa => 'ArrayRef', @@ -108,7 +108,7 @@ sub BUILD { $self->config->{'htmldir'} ||= "/usr/share/munin/html"; # Set up all the methods for our config object. - foreach my $attr (@{$self->legal}) { + foreach my $attr ( @{ $self->legal } ) { $self->meta->add_method( $attr => sub { my $obj = shift; @@ -116,13 +116,61 @@ sub BUILD { } ); } - - $self->log->debug("Adding " . $self->dbdir . "/moonin-config.db"); - $self->{store} = Moonin::Config::Store->new(file => $self->dbdir . "moonin-config.db"); - + + $self->log->debug( "Adding " . $self->dbdir . "/moonin-config.db" ); + $self->{store} = + Moonin::Config::Store->new( file => $self->dbdir . "moonin-config.db" ); + # $self->log->debug(dump($self->config)); } +sub get_domains { + my $self = shift; + my @result = sort( keys( %{ $self->domain } ) ); + return \@result; +} + +sub get_nodes { + my $self = shift; + my $domain = shift; + my @result = sort( keys( %{ $self->store->dbm->{node}->{$domain} } ) ); + return \@result; +} + +sub get_graph_categories { + my $self = shift; + my $domain = shift; + my $name = shift; + + my $clients = $self->store->dbm->{node}->{$domain}->{$name}->{client}; + + my @graph_categories; + foreach my $thing ( keys( %{$clients} ) ) { + if ( exists $clients->{$thing}->{'graph_category'} ) { + my $gc = ucfirst($clients->{$thing}->{'graph_category'}); + push( @graph_categories, $gc ) + unless ( grep /^$gc$/, @graph_categories ); + } + } + return \@graph_categories; +} + +sub get_graphs_by_category { + my $self = shift; + my $domain = shift; + my $name = shift; + + my $clients = $self->store->dbm->{node}->{$domain}->{$name}->{client}; + my $graphs = {}; + foreach my $thing ( keys( %{$clients} ) ) { + if ( exists $clients->{$thing}->{'graph_category'} ) { + my $gc = ucfirst($clients->{$thing}->{'graph_category'}); + push( @{$graphs->{$gc}}, { name => $thing, data => $clients->{$thing} }); + } + } + return $graphs; +} + sub _parse_config { my $self = shift; my $lines = shift; @@ -243,7 +291,7 @@ sub _set_var_path { } sub domain { - my ( $self ) = @_; + my ($self) = @_; return $self->config->{domain}; } lib/moonin/Config.pm @@ -27,8 +27,7 @@ sub BUILD { $self->{dbm} = DBM::Deep->new( file => $self->file, locking => 1, - autoflush => 1, - num_txns => 5 + autoflush => 1 ); $self; } lib/moonin/Config/Store.pm @@ -28,7 +28,7 @@ has 'config' => ( is => 'rw', isa => 'Object', required => 1 ); sub BUILD { my $self = shift; - $self->{domain} = $self->config->domain($self->name) + $self->{domain} = $self->config->domain->{$self->name}; } sub process { @@ -36,12 +36,12 @@ sub process { $self->log->debug("Processing domain " . $self->name); for my $key ( keys %{ $self->domain->{node} } ) { $self->log->debug("Processing node $key"); - # my $node = Moonin::Node->new( - # domain => $self->name, - # node => $key, - # config => $self->config - # ); - # $node->process; + my $node = Moonin::Node->new( + domain => $self->name, + node => $key, + config => $self->config + ); + $node->process; } } lib/moonin/Domain.pm @@ -29,8 +29,13 @@ has 'domain' => ( is => 'rw', isa => 'Str', required => 1 ); has 'config' => ( is => 'rw', isa => 'Object', required => 1 ); has 'node_config' => ( is => 'ro', isa => 'HashRef', required => 0 ); has 'timeout' => ( is => 'rw', isa => 'Int', required => 1, default => 180 ); -has 'limit_services' => - ( is => 'rw', required => 0, default => undef ); +has 'limit_services' => ( is => 'rw', required => 0, default => undef ); +has 'copy_fields' => ( + is => 'rw', + required => 0, + defualt => + sub { [ "label", "draw", "type", "rrdfile", "fieldname", "info" ] } +); sub BUILD { my $self = shift; @@ -98,7 +103,7 @@ sub process { } else { next unless ( $self->configure($socket) ); - $self->fetch( $socket ); + $self->fetch($socket); # Net::SSLeay::free ($tls) if ($tls); # Shut down TLS close $socket; @@ -124,7 +129,9 @@ sub fetch { next if ( exists( $node->{client}->{$service}->{update} ) and $node->{client}->{$service}->{update} ne "yes" ); - next if ( $self->limit_services and !grep ( /^$service$/, @{$self->limit_services} ) ); + next + if ( $self->limit_services + and !grep ( /^$service$/, @{ $self->limit_services } ) ); my $realservname = ( $node->{client}->{$service}->{realservname} || $service ); delete $node->{client}->{$service}->{realservname} @@ -186,7 +193,8 @@ sub fetch { sprintf( "%.2f", ( Time::HiRes::time - $servicefetch_time ) ); $self->log->debug( "Fetched service: $name -> $service ($servicefetch_time sec)"); - $self->config->store->dbm->{FS}->{$domain}->{$name}->{$service} = $servicefetch_time; + $self->config->store->dbm->{FS}->{$domain}->{$name}->{$service} = + $servicefetch_time; } $nodefetch_time = sprintf( "%.2f", ( Time::HiRes::time - $nodefetch_time ) ); @@ -202,7 +210,7 @@ sub configure { my $name = $self->name; my $node = $self->node_config; my $oldnode = $self->config->store->get("node-$domain-$name"); - + my $clientdomain = $self->_read_socket_single($socket); my $fetchdomain; chomp($clientdomain) if $clientdomain; @@ -368,9 +376,10 @@ sub configure { $node->{client}->{$servname}->{graph_order} = join( ' ', @graph_order ); } - $serviceconf_time = - sprintf( "%.2f", ( Time::HiRes::time - $serviceconf_time ) ); - $self->config->store->dbm->{'CS'}->{$domain}->{$name}->{$servname} = $serviceconf_time; + $serviceconf_time = + sprintf( "%.2f", ( Time::HiRes::time - $serviceconf_time ) ); + $self->config->store->dbm->{'CS'}->{$domain}->{$name}->{$servname} = + $serviceconf_time; $self->log->debug( "Configured service: $name -> $servname ($serviceconf_time sec)"); } @@ -382,6 +391,416 @@ sub configure { return 1; } +sub get_node_config { + my $self = shift; + return $self->config->store->dbm->{node}->{ $self->domain } + ->{ $self->name }; +} + +sub get_field_order { + my $self = shift; + my $service = shift; + my $node = $self->get_node_config; + my $config = $self->config; + my $domain = $self->domain; + my $result = []; + + if ( $node->{client}->{$service}->{graph_sources} ) { + foreach + my $gs ( split /\s+/, $node->{client}->{$service}->{'graph_sources'} ) { + push( @$result, "-" . $gs ); + } + } + if ( $node->{client}->{$service}->{graph_order} ) { + push( @$result, + split /\s+/, $node->{client}->{$service}->{'graph_order'} ); + } + + for my $key ( keys %{ $node->{client}->{$service} } ) { + my ( $client, $type ) = ""; + ( $client, $type ) = split /\./, $key; + if ( defined $type and $type eq "label" ) { + push @$result, $client if !grep /^\Q$client\E(?:=|$)/, @$result; + } + } + $self->log->debug( "Field order for $service: " . dump($result) ); + return $result; +} + +sub get_max_label_length { + my $self = shift; + my $node = $self->get_node_config; + my $service = shift; + my $order = shift; + my $result = 0; + + for my $field (@$order) { + my $path = undef; + ( my $f = $field ) =~ s/=.+//; + next + if (exists $node->{client}->{$service}->{ $f . ".process" } + and $node->{client}->{$service}->{ $f . ".process" } + and $node->{client}->{$service}->{ $f . ".process" } ne "yes" ); + next if ( exists $node->{client}->{$service}->{ $f . ".skipdraw" } ); + next + unless ( !exists $node->{client}->{$service}->{ $f . ".graph" } + or !$node->{client}->{$service}->{ $f . ".graph" } + or $node->{client}->{$service}->{ $f . ".graph" } eq "yes" ); + if ( $result < + length( $node->{client}->{$service}->{ $f . ".label" } || $f ) ) { + $result = + length( $node->{client}->{$service}->{ $f . ".label" } || $f ); + } + if ( exists $node->{client}->{$service}->{graph_total} + and length $node->{client}->{$service}->{graph_total} > $result ) { + $result = length $node->{client}->{$service}->{graph_total}; + } + } + return $result; +} + +sub process_field { + my $self = shift; + my $service = shift; + my $field = shift; + my $node = $self->get_node_config; + return ( $self->get_bool_val( $service, $field . ".process", 1 ) ); +} + +sub get_bool_val { + my $self = shift; + my $service = shift; + my $field_name = shift; + my $default = shift; + my $node = $self->get_node_config; + my $field = $node->{client}->{$service}->{$field_name}; + + if ( !defined $field ) { + if ( !defined $default ) { + return 0; + } else { + return $default; + } + } + + if ( $field =~ /^yes$/i + or $field =~ /^true$/i + or $field =~ /^on$/i + or $field =~ /^enable$/i + or $field =~ /^enabled$/i ) { + return 1; + } elsif ( $field =~ /^no$/i + or $field =~ /^false$/i + or $field =~ /^off$/i + or $field =~ /^disable$/i + or $field =~ /^disabled$/i ) { + return 0; + } elsif ( $field !~ /\D/ ) { + return $field; + } else { + return undef; + } +} + +sub get_stack_command { + my $self = shift; + my $service = shift; + my $field = shift; + my $node = $self->get_node_config; + + if ( defined $node->{client}->{$service}->{ $field . ".special_stack" } ) { + return $node->{client}->{$service}->{ $field . ".special_stack" }; + } elsif ( defined $node->{client}->{$service}->{ $field . ".stack" } ) { + return $node->{client}->{$service}->{ $field . ".stack" }; + } + + return undef; +} + +sub get_sum_command { + my $self = shift; + my $service = shift; + my $field = shift; + my $node = $self->get_node_config; + + if ( defined $node->{client}->{$service}->{ $field . ".special_sum" } ) { + return $node->{client}->{$service}->{ $field . ".special_sum" }; + } elsif ( defined $node->{client}->{$service}->{ $field . ".sum" } ) { + return $node->{client}->{$service}->{ $field . ".sum" }; + } + + return undef; +} + +sub get_filename { + my $self = shift; + my $config = $self->config; + my $domain = $self->domain; + my $node = $self->get_node_config; + my $service = shift; + my $field = shift; + + return ( + $config->dbdir . "/$domain/" . $self->name . "-$service-$field-" + . lc substr( + ( $node->{client}->{$service}->{ $field . ".type" } || "GAUGE" ), + 0, 1 ) + . ".rrd" + ); + +} + +sub get_rrd_filename { + my $self = shift; + my $node = $self->get_node_config; + my $config = $self->config; + my $domain = $self->domain; + my $name = $self->name; + my $service = shift; + my $field = shift; + my $path = shift; + my $result = "unknown"; + + if ( $node->{client}->{$service}->{ $field . ".filename" } ) { + $result = $node->{client}->{$service}->{ $field . ".filename" }; + } elsif ($path) { + if ( !defined( $node->{client}->{$service}->{ $field . ".label" } ) ) { + $self->log->debug("Setting label: $field\n"); + $node->{client}->{$service}->{ $field . ".label" } = $field; + } + + if ( $path =~ /^\s*([^:;]+)[:;]([^:]+):([^:\.]+)[:\.]([^:\.]+)\s*$/ ) { + $result = $self->get_filename( $config, $3, $4 ); + $self->log->debug("Expanding $path...\n"); + if ( !defined $node->{client}->{$service}->{ $field . "label" } ) { + for my $f ( @{ $self->copy_fields } ) { + if ( not exists $node->{client}->{$service}->{"$field.$f"} + and + exists $config->{'domain'}->{$1}->{'node'}->{$2}->{'client'}->{$3} + ->{"$4.$f"} ) { + $node->{client}->{$service}->{"$field.$f"} = + $config->{'domain'}->{$1}->{'node'}->{$2}->{'client'}->{$3} + ->{"$4.$f"}; + } + } + } + } elsif ( $path =~ /^\s*([^:]+):([^:\.]+)[:\.]([^:\.]+)\s*$/ ) { + $self->log->debug("Expanding $path...\n"); + $result = $self->get_filename( $2, $3 ); + for my $f ( @{ $self->copy_fields } ) { + if ( not exists $node->{client}->{$service}->{"$field.$f"} + and + exists $config->{'domain'}->{$domain}->{'node'}->{$1}->{'client'} + ->{$2}->{"$3.$f"} ) { + $self->log->debug("Copying $f...\n"); + $node->{client}->{$service}->{"$field.$f"} = + $config->{'domain'}->{$domain}->{'node'}->{$1}->{'client'}->{$2} + ->{"$3.$f"}; + } + } + } elsif ( $path =~ /^\s*([^:\.]+)[:\.]([^:\.]+)\s*$/ ) { + $self->log->debug("Expanding $path...\n"); + $result = $self->get_filename( $1, $2 ); + for my $f ( @{ $self->copy_fields } ) { + if ( not exists $node->{client}->{$service}->{"$field.$f"} + and exists $node->{client}->{$1}->{"$2.$f"} ) { + $node->{client}->{$service}->{"$field.$f"} = + $node->{client}->{$1}->{"$2.$f"}; + } + } + } elsif ( $path =~ /^\s*([^:\.]+)\s*$/ ) { + $self->log->debug("Expanding $path...\n"); + $result = $self->get_filename( $service, $1 ); + for my $f ( @{ $self->copy_fields } ) { + if ( not exists $node->{client}->{$service}->{"$field.$f"} + and exists $node->{client}->{$service}->{"$1.$f"} ) { + $node->{client}->{$service}->{"$field.$f"} = + $node->{client}->{$service}->{"$1.$f"}; + } + } + } + } else { + $self->log->debug("\nDEBUG5: Doing path...\n"); + $result = $self->get_filename( $service, $field ); + } + return $result; +} + +sub single_value { + my $self = shift; + my $node = $self->get_node_config; + my $service = shift; + my $field = shift; + my $order = shift; + + return 1 if @$order == 1; + return 1 + if (@$order == 2 + and $node->{client}->{$service}->{ $field . ".negative" } ); + + my $graphable = 0; + if ( !defined $node->{client}->{$service}->{"graphable"} ) { + + # foreach my $field (keys %{$node->{client}->{$service}}) + foreach my $field ( $self->get_field_order($service) ) { + $self->log->debug("single_value: Checking field \"$field\".\n"); + if ( $field =~ /^([^\.]+)\.label/ or $field =~ /=/ ) { + $graphable++ if $self->draw_field( $service, $1 ); + } + } + $node->{client}->{$service}->{"graphable"} = $graphable; + } + return 1 if ( $node->{client}->{$service}->{"graphable"} == 1 ); + + return 0; +} + +sub draw_field { + my $self = shift; + my $node = $self->get_node_config; + my $service = shift; + my $field = shift; + + $field =~ s/=.*//; + + $self->log->debug( "munin_draw_field: Checking $service -> $field: " + . $self->get_bool_val( $service, $field . ".graph", 1 ) + . ".\n" ); + return 0 + if ( exists $node->{client}->{$service}->{ $field . ".skipdraw" } ); + return ( $self->get_bool_val( $service, $field . ".graph", 1 ) ); +} + +sub expand_specials { + my $self = shift; + my $service = shift; + my $preproc = shift; + my $order = shift; + my $node = $self->get_node_config; + my $config = $self->config; + my $domain = $self->domain; + my $single = shift; + my $result = []; + + my $fieldnum = 0; + for my $field (@$order) { # Search for 'specials'... + + if ( $field =~ /^-(.+)$/ ) { + $field = $1; + unless ( defined $node->{client}->{$service}->{ $field . ".graph" } + or defined $node->{client}->{$service}->{ $field . ".skipdraw" } ) { + $node->{client}->{$service}->{ $field . ".graph" } = "no"; + } + } + + $fieldnum++; + my $tmp_field; + if ( + defined( $tmp_field = $self->get_stack_command( $service, $field ) ) ) { + $self->log->debug("Doing special_stack...\n"); + my @spc_stack = (); + foreach my $pre ( split( /\s+/, $tmp_field ) ) { + ( my $name = $pre ) =~ s/=.+//; + if ( !@spc_stack ) { + $node->{client}->{$service}->{ $name . ".draw" } = + $node->{client}->{$service}->{ $field . ".draw" }; + $node->{client}->{$service}->{ $field . ".process" } = "no"; + } else { + $node->{client}->{$service}->{ $name . ".draw" } = "STACK"; + } + push( @spc_stack, $name ); + push( @$preproc, $pre ); + push @$result, "$name.label"; + push @$result, "$name.draw"; + push @$result, "$name.cdef"; + + $node->{client}->{$service}->{ $name . ".label" } = $name; + $node->{client}->{$service}->{ $name . ".cdef" } = + "$name,UN,0,$name,IF"; + if ( exists $node->{client}->{$service}->{ $field . ".cdef" } + and ! + exists $node->{client}->{$service}->{ $name . ".onlynullcdef" } ) { + $self->log->debug("NotOnlynullcdef ($field)...\n"); + $node->{client}->{$service}->{ $name . ".cdef" } .= + "," . $node->{client}->{$service}->{ $field . ".cdef" }; + $node->{client}->{$service}->{ $name . ".cdef" } =~ + s/\b$field\b/$name/g; + } else { + $self->log->debug("Onlynullcdef ($field)...\n"); + $node->{client}->{$service}->{ $name . ".onlynullcdef" } = 1; + push @$result, "$name.onlynullcdef"; + } + } + } elsif ( + defined( $tmp_field = $self->get_sum_command( $service, $field ) ) ) { + my @spc_stack = (); + my $last_name = ""; + $self->log->debug("Doing special_sum...\n"); + + if ( @$order == 1 + or @$order == 2 + && $node->{client}->{$service}->{ $field . ".negative" } ) { + $single = 1; + } + + foreach my $pre ( split( /\s+/, $tmp_field ) ) { + ( my $path = $pre ) =~ s/.+=//; + my $name = "z" . $fieldnum . "_" . scalar(@spc_stack); + $last_name = $name; + + $node->{client}->{$service}->{ $name . ".cdef" } = + "$name,UN,0,$name,IF"; + $node->{client}->{$service}->{ $name . ".graph" } = "no"; + $node->{client}->{$service}->{ $name . ".label" } = $name; + push @$result, "$name.cdef"; + push @$result, "$name.graph"; + push @$result, "$name.label"; + + push( @spc_stack, $name ); + push( @$preproc, "$name=$pre" ); + } + $node->{client}->{$service}->{ $last_name . ".cdef" } .= + "," . join( ',+,', @spc_stack[ 0 .. @spc_stack - 2 ] ) . ',+'; + if ( exists $node->{client}->{$service}->{ $field . ".cdef" } + and length $node->{client}->{$service}->{ $field . ".cdef" } ) + { # Oh bugger... + my $tc = $node->{client}->{$service}->{ $field . ".cdef" }; + $self->log->debug("Oh bugger...($field)...\n"); + $tc =~ + s/\b$field\b/$node->{client}->{$service}->{$last_name.".cdef"}/; + $node->{client}->{$service}->{ $last_name . ".cdef" } = $tc; + } + $node->{client}->{$service}->{ $field . ".process" } = "no"; + $node->{client}->{$service}->{ $last_name . ".draw" } = + $node->{client}->{$service}->{ $field . ".draw" }; + $node->{client}->{$service}->{ $last_name . ".label" } = + $node->{client}->{$service}->{ $field . ".label" }; + if ( defined $node->{client}->{$service}->{ $field . ".graph" } ) { + $node->{client}->{$service}->{ $last_name . ".graph" } = + $node->{client}->{$service}->{ $field . ".graph" }; + } else { + $node->{client}->{$service}->{ $last_name . ".graph" } = "yes"; + } + if ( defined $node->{client}->{$service}->{ $field . ".negative" } ) { + $node->{client}->{$service}->{ $last_name . ".negative" } = + $node->{client}->{$service}->{ $field . ".negative" }; + } + $node->{client}->{$service}->{ $field . ".realname" } = $last_name; + $self->log->debug( +"Setting node->{client}->{$service}->{$field} -> realname = $last_name...\n" + ); + } elsif ( defined $node->{client}->{$service}->{ $field . ".negative" } ) + { + my $nf = $node->{client}->{$service}->{ $field . ".negative" }; + unless ( defined $node->{client}->{$service}->{ $nf . ".graph" } + or defined $node->{client}->{$service}->{ $nf . ".skipdraw" } ) { + $node->{client}->{$service}->{ $nf . ".graph" } = "no"; + } + } + } + return $result; +} + sub _sanitise_fieldname { my $self = shift; my $lname = shift; lib/moonin/Node.pm @@ -51,3 +51,4 @@ isa_ok( $mn, 'Moonin::Node' ); is($mn->node_config, $cfg->domain->{ "sfo.v2green.com" }->{node}->{ "ops1prod.sfo.v2green.com" }, "Node Config is correct" ); $mn->process; +is(ref($mn->get_field_order("cpu")), 'ARRAY', "CPU returns array") t/02moonin-node.pm 5ea61532076d0c5a43a4ab0bd256d054cfd4ac52 Adam Jacob adam@hjksolutions.com http://github.com/adamhjk/moonin/commit/83399f46aa421ca62097a773ca8d3dcd4834ad35 83399f46aa421ca62097a773ca8d3dcd4834ad35 2007-12-03T15:22:41-08:00 2007-12-03T15:22:41-08:00 First real pass at moonin 05dbb69eb101df395b46be1118282cec069c5615 Adam Jacob adam@hjksolutions.com