Skip to content
Browse files

Track server counts separately for IPv6

  • Loading branch information...
1 parent 1305c10 commit 840cdce2db436ca5c5cfb7c5af548b008d0e9334 @abh committed
Showing with 238 additions and 91 deletions.
  1. +184 −77 bin/zone_stats
  2. +2 −1 lib/NP/Model.pm
  3. +22 −5 lib/NP/Model/Zone.pm
  4. +12 −3 lib/NP/Model/ZoneServerCount.pm
  5. +5 −5 lib/NTPPool/Control/Zone.pm
  6. +13 −0 sql/ntppool.update
View
261 bin/zone_stats
@@ -39,46 +39,93 @@ if (@ARGV and $ARGV[0] eq "populate_all") {
}
while (my $zone = $zones->next) {
- my $count_active = $zone->server_count;
- my $count_all = $zone->server_count_all;
- my $netspeed_active = $zone->netspeed_active || 0;
- #printf "%-8s %3i %3i\n", $zone->name, $count_active, $count_all;
+
+ #next unless $zone->name eq 'europe';
+
my $now = time;
- $dbh->do(q[replace into zone_server_counts (zone_id,date,count_active,count_registered,netspeed_active)
- values (?,NOW(),?,?,?)],
- undef,
- $zone->id, $count_active, $count_all, $netspeed_active
- );
- create_rrd($zone->name);
- create_target_rrd($zone->name);
- update_target_rrd($zone->name);
- update_rrd($zone->name, $now, $count_active, $count_all, $netspeed_active);
- update_graph($zone->name);
+
+ create_rrd($zone);
+ create_target_rrd($zone);
+ update_target_rrd($zone);
+
+ my %data;
+
+ for my $ip_version (qw(v4 v6)) {
+
+ my $data = $data{$ip_version} = {};
+
+ $data->{count_active} = $zone->server_count($ip_version);
+ $data->{count_all} = $zone->server_count_all($ip_version);
+ $data->{netspeed_active} = $zone->netspeed_active($ip_version) || 0;
+
+ # printf "%-8s %s %3i %3i\n", $zone->name, $ip_version, $data->{count_active}, $data->{count_all};
+ $dbh->do(q[replace into zone_server_counts (zone_id,ip_version,date,count_active,count_registered,netspeed_active)
+ values (?,?,NOW(),?,?,?)],
+ undef,
+ $zone->id,
+ $ip_version,
+ $data->{count_active}, $data->{count_all}, $data->{netspeed_active}
+ );
+
+ }
+
+ update_rrd($zone, $now, \%data);
+
+ update_graph($zone, 'v4');
+ update_graph($zone, 'v6');
}
sub populate_all {
while (my $zone = $zones->next) {
- my $data = $dbh->selectall_arrayref(q[select unix_timestamp(date) as time, date,
+ my $data = $dbh->selectall_arrayref(q[select unix_timestamp(date) as time, date, ip_version,
count_active, count_registered, netspeed_active
from zone_server_counts
- where zone_id=? order by date],
+ where zone_id=? order by date],
{Columns=>{}}, $zone->id);
- create_rrd($zone->name);
+ create_rrd($zone);
my $i = 0;
+
+ my %days;
+
for my $d (@$data) {
- print $zone->name, " $d->{date} ($d->{count_active}, $d->{count_registered}, $d->{netspeed_active})\n";
- update_rrd($zone->name, $d->{time}, $d->{count_active}, $d->{count_registered}, $d->{netspeed_active});
+ $days{ $d->{date} }->{ $d->{ip_version} } = $d;
+ }
+
+ #use Data::Dump qw(pp);
+ #pp( \%days);
+
+ for my $day (sort keys %days) {
+
+ print $zone->name, " $day\n";
+ # $d->{date} ($d->{count_active}, $d->{count_registered}, $d->{netspeed_active})\n";
+
+ my $time = $days{$day}->{v4}->{time} || $days{$day}->{v6}->{time};
+
+ update_rrd($zone, $time, $days{$day});
+
}
+
}
}
sub update_rrd {
- my ($zone, $time, $active, $registered, $netspeed) = @_;
+ my ($zone, $time, $data) = @_;
+
+ #use Data::Dump qw(pp);
+ #print "update rrd ", pp( $data ), "\n";
+
my $path = rrd_path($zone);
- my @options = ('--template' => 'active:registered:netspeed',
- "$time:$active:$registered:$netspeed",
- );
+ my @options = (
+ '--template' => 'active_v4:registered_v4:netspeed_v4:active_v6:registered_v6:netspeed_v6',
+ "$time:" . join(
+ ":",
+ map {
+ my $v = $_;
+ map { $data->{$v}->{$_} || 'U' } qw(count_active count_registered netspeed_active)
+ } qw(v4 v6)
+ )
+ );
RRDs::update "$path", @options;
if (my $ERROR = RRDs::error) {
warn "$0: unable to update $path: $ERROR\nOptions: ", join " ", @options;
@@ -88,31 +135,52 @@ sub update_rrd {
sub rrd_path {
my $zone = shift;
- "$base_path/$zone.rrd";
+ my $name = $zone->name;
+ "$base_path/$name.rrd";
}
sub target_path {
my $zone = shift;
- "$target_path/$zone.rrd";
-}
-
-sub graph_path {
- my $zone = shift;
- "$base_path/graph/$zone.png";
+ my $name = $zone->name;
+ "$target_path/$name.rrd";
}
sub update_graph {
my $zone = shift;
+ my $ip_version = shift;
- my $title = "Servers in the $zone.pool.ntp.org zone";
+ my $fqdn = $zone->fqdn;
+ my $title = "Servers in the $fqdn zone ($ip_version)";
my $rrd = rrd_path($zone);
my $trrd = target_path($zone);
+
+ # if there's no data at all
+ my $start = 'end-1m';
+
+ # in other cases, start the graph when the first data started
+ if (my $first_stats = $zone->first_stats) {
+
+ my $date = $first_stats->date;
+ $date = $date->subtract(weeks => 2);
+
+ # except for ipv6; start those graphs no earlier than June 1, 2011
+ if ($ip_version eq 'v6' and $date->year < 2011) {
+ $date = DateTime->new(year => 2011, month => 6, day => 1);
+ }
+
+ $start = $date->epoch;
+
+ }
+
+ # print "Graph for $fqdn / $ip_version\n";
+ # print "START: ", $start, "\n";
+
my @options = (
#'--lazy',
'--end' => 'now',
- '--start' => 1123657000, # 2005-08-10, first data... 'end-14d',
+ '--start' => $start,
'--width' => 420,
'--height' => 130,
'--title' => $title,
@@ -121,9 +189,9 @@ sub update_graph {
'--alt-autoscale-max',
# '--no-gridfit',
'--slope-mode',
- qq[DEF:active=$rrd:active:AVERAGE],
- qq[DEF:registered=$rrd:registered:AVERAGE],
- qq[DEF:target=$trrd:target:AVERAGE],
+ qq[DEF:active=$rrd:active_${ip_version}:AVERAGE],
+ qq[DEF:registered=$rrd:registered_${ip_version}:AVERAGE],
+ qq[DEF:target=$trrd:target_${ip_version}:AVERAGE],
qq[CDEF:inactive=registered,active,-],
qq[LINE2:registered#00CCCC:registered servers],
#qq[LINE2:inactive#EE0000:inactive servers],
@@ -132,7 +200,7 @@ sub update_graph {
qq[LINE1:target#444444:growth target],
);
- my $path = graph_path($zone);
+ my $path = $zone->graph_path($ip_version);
RRDs::graph $path, @options;
my $ERROR = RRDs::error;
if ($ERROR) {
@@ -150,16 +218,23 @@ sub create_rrd {
my @graph = (
"--start", $START, "--step", 21600, # 6 hours interval
- "DS:active:GAUGE:180000:0:20000", # heartbeat of ~2 days, min value = 0, max = 20000
- "DS:registered:GAUGE:180000:0:20000",
- "DS:netspeed:GAUGE:180000:0:400000000",
+ "DS:active_v4:GAUGE:180000:0:30000", # heartbeat of ~2 days, min value = 0, max = 20000
+ "DS:registered_v4:GAUGE:180000:0:30000",
+ "DS:netspeed_v4:GAUGE:180000:0:400000000",
+
+ "DS:active_v6:GAUGE:180000:0:20000",
+ "DS:registered_v6:GAUGE:180000:0:20000",
+ "DS:netspeed_v6:GAUGE:180000:0:400000000",
+
"RRA:AVERAGE:0.5:1:1024", # 6 hours, 170 days
"RRA:AVERAGE:0.5:4:1024", # 1 day, ~3 years
"RRA:AVERAGE:0.5:12:1024", # 3 days, ~9 years
"RRA:AVERAGE:0.5:42:1024", # 1 week, ~19 years.
+
"RRA:MIN:0.5:4:1024", # 1 day, ~3 years
- "RRA:MAX:0.5:4:1024", # 1 day, ~3 years
"RRA:MIN:0.5:42:1024", # 1 week, ~19 years.
+
+ "RRA:MAX:0.5:4:1024", # 1 day, ~3 years
"RRA:MAX:0.5:42:1024", # 1 week, ~19 years.
);
@@ -177,47 +252,77 @@ sub update_target_rrd {
my $now = DateTime->from_epoch( epoch => time );
my $last = DateTime->from_epoch( epoch => (RRDs::last target_path($zone) || 0));
+ #print "updating targets for ", $zone->name, "\n";
+
+ use Data::Dump qw(pp);
+
if ($last < $now) {
- my $z = NP::Model->zone->fetch(name => $zone);
- my $first_stats = $z->first_stats;
- my $first_stats_date = $first_stats->date;
- my $start_year = ($first_stats_date->year < $last->year ? $last->year : $first_stats_date->year);
- for my $year ($start_year .. $now->year) {
- my $first_stats = $z->first_stats($year);
- next unless $first_stats;
- my $start_active = $first_stats->count_active;
- my $first_day = $first_stats->date->day_of_year;
-
- my $year_growth_goal = $start_active * ($growth_targets{$year} || $growth_targets{default});
- $year_growth_goal = 5 - $start_active if $year_growth_goal < 2;
-
- my $days_in_year = DateTime->new(year => $year)->is_leap_year ? 366 : 365;
-
- my $growth_per_day = $year_growth_goal / $days_in_year;
-
- my $day = DateTime->from_epoch(epoch => $first_stats->date->epoch);
- my $goal = $start_active;
- for ($first_day+1 .. $days_in_year) {
- $goal += $growth_per_day;
-
- my $time = $day->epoch;
- my $path = target_path($zone);
- my @options = ('--template' => 'target',
- "$time:$goal",
- );
- RRDs::update "$path", @options;
- if (my $ERROR = RRDs::error) {
- warn "$0: unable to update $path: $ERROR\nOptions: ", join " ", @options;
- }
+ my %data;
+
+ for my $ip_version (qw(v4 v6)) {
+
+ my $opening_stats = $zone->first_stats("", $ip_version);
+ my $opening_stats_date = $opening_stats ? $opening_stats->date : DateTime->now;
+
+ my $start_year = ($opening_stats_date->year < $last->year ? $last->year : $opening_stats_date->year);
+
+ for my $year ($start_year .. $now->year) {
+
+ my $days_in_year = DateTime->new(year => $year)->is_leap_year ? 366 : 365;
+
+ my $first_stats_year = $zone->first_stats($year, $ip_version);
+ my $start_active = $first_stats_year ? $first_stats_year->count_active : 0;
+ my $first_day = $first_stats_year ? $first_stats_year->date->day_of_year : 1;
+
+ #print "Year ", $year, " starts with ", $start_active, "\n";
+
+ my $year_growth_goal = $start_active * ($growth_targets{$year} || $growth_targets{default});
+ $year_growth_goal = 6 - $start_active if $year_growth_goal < 3;
+ my $growth_per_day = $year_growth_goal / $days_in_year;
+
+ my $day = DateTime->from_epoch(epoch => $first_stats_year->date->epoch);
+
+ my $goal = $start_active;
+ #print "starting goal: $goal\n";
+
+ my $end_of_year = DateTime->new(year => $year, month => 12, day => 31);
+
+ while ($day <= $end_of_year) {
+ my $time = $day->epoch;
+ $data{$time}->{$ip_version} = $goal;
+
+ my $step = 1;
+ $goal += ($growth_per_day * $step);
+ $day->add( days => $step );
+
+ }
+ #print "goal at end $goal\n";
- # update rrd
- $day->add( days => 1 );
}
+ }
+
+ #use Data::Dump qw(pp);
+ #print pp(\%data);
+
+ my $i = 0;
+ for my $time (sort keys %data) {
+
+ my $goal_v4 = $data{$time}->{v4} || 'U';
+ my $goal_v6 = $data{$time}->{v6} || 'U';
- my $growth_goal = ($days_in_year - $first_day) * $growth_per_day;
+ # print "adding for $time: $goal_v4 / $goal_v6\n";
+
+ my $path = target_path($zone);
+ my @options = ('--template' => 'target_v4:target_v6',
+ "$time:${goal_v4}:${goal_v6}",
+ );
+ #print join " ", @options, "\n";
+ RRDs::update "$path", @options;
+ if (my $ERROR = RRDs::error) {
+ warn "$0: unable to update $path: $ERROR\nOptions: ", join " ", @options;
+ }
- my $eoy_goal = $growth_goal + $start_active;
}
}
}
@@ -231,10 +336,12 @@ sub create_target_rrd {
my @graph = (
"--start", $START, "--step", 86400, # 24 hours interval
- "DS:target:GAUGE:180000:0:20000", # heartbeat of ~2 days, min value = 0, max = 20000
+ "DS:target_v4:GAUGE:2592000:0:20000", # heartbeat of ~30 days, min value = 0, max = 20000
+ "DS:target_v6:GAUGE:2592000:0:20000",
"RRA:AVERAGE:0.5:1:1024", # 1 day, ~3 years
"RRA:AVERAGE:0.5:3:1024", # 3 days, ~9 years
- "RRA:AVERAGE:0.5:7:1024", # 1 week, ~19 years.
+ "RRA:AVERAGE:0.5:7:512", # 1 week, ~10 years
+ "RRA:AVERAGE:0.5:30:512", # 1 month, ~40 years
);
View
3 lib/NP/Model.pm
@@ -909,6 +909,7 @@ __PACKAGE__->meta->setup(
columns => [
id => { type => 'serial', not_null => 1 },
zone_id => { type => 'integer', not_null => 1 },
+ ip_version => { type => 'enum', check_in => [ 'v4', 'v6' ], not_null => 1 },
date => { type => 'date', not_null => 1 },
count_active => { type => 'integer', not_null => 1 },
count_registered => { type => 'integer', not_null => 1 },
@@ -917,7 +918,7 @@ __PACKAGE__->meta->setup(
primary_key_columns => [ 'id' ],
- unique_key => [ 'zone_id', 'date' ],
+ unique_key => [ 'zone_id', 'date', 'ip_version' ],
foreign_keys => [
zone => {
View
27 lib/NP/Model/Zone.pm
@@ -34,7 +34,9 @@ sub rrd_path {
sub graph_path {
my $self = shift;
- my $file = $self->name . ".png";
+ my $ip_version = shift;
+ my $suffix = ($ip_version eq 'v6' ? '-v6' : '');
+ my $file = $self->name . "$suffix.png";
return "$rrd_path/graph/" . $file;
}
@@ -87,7 +89,7 @@ sub active_servers {
my $entries = $dbh->selectall_arrayref
(
- q[SELECT s.ip, s.netspeed
+ qq[SELECT s.ip, s.netspeed
FROM servers s, server_zones l
WHERE l.server_id = s.id AND l.zone_id = ? AND s.in_pool = 1 AND s.ip_version=?
AND s.score_raw > ?
@@ -103,10 +105,18 @@ sub active_servers {
}
+sub _ip_version_sql {
+ my $self = shift;
+ my $ip_version = shift or return "";
+ return "AND s.ip_version=" . $self->dbh->quote($ip_version);
+}
+
sub server_count {
my $self = shift;
+ my $ip_version_sql = $self->_ip_version_sql(shift);
+
my $dbh = $self->dbh;
- $dbh->selectrow_array(q[
+ $dbh->selectrow_array(qq[
select count(*) as count
from servers s
inner join server_zones l on(s.id=l.server_id)
@@ -115,13 +125,16 @@ sub server_count {
and s.score_raw > 10
and s.in_pool = 1
and (s.deletion_on IS NULL OR s.deletion_on > DATE_ADD(NOW(), interval ? day))
+ $ip_version_sql
], undef, $self->id, deletion_grace_days());
}
sub server_count_all {
my $self = shift;
+ my $ip_version_sql = $self->_ip_version_sql(shift);
+
my $dbh = $self->dbh;
- $dbh->selectrow_array(q[
+ $dbh->selectrow_array(qq[
select count(*) as count
from servers s
inner join server_zones l on(s.id=l.server_id)
@@ -130,13 +143,16 @@ sub server_count_all {
z.id=?
and s.in_pool = 1
and (s.deletion_on IS NULL OR s.deletion_on > DATE_ADD(NOW(), interval ? day))
+ $ip_version_sql
], undef, $self->id, deletion_grace_days());
}
sub netspeed_active {
my $self = shift;
+ my $ip_version_sql = $self->_ip_version_sql(shift);
+
my $dbh = $self->dbh;
- $dbh->selectrow_array(q[
+ $dbh->selectrow_array(qq[
select sum(s.netspeed) as netspeed
from servers s
inner join server_zones l on(s.id=l.server_id)
@@ -145,6 +161,7 @@ sub netspeed_active {
and s.score_raw > 10
and s.in_pool = 1
and (s.deletion_on IS NULL OR s.deletion_on > DATE_ADD(NOW(), interval ? day))
+ $ip_version_sql
], undef, $self->id, deletion_grace_days());
}
View
15 lib/NP/Model/ZoneServerCount.pm
@@ -11,21 +11,30 @@ package NP::Model::ZoneServerCount::Manager;
use strict;
sub first_stats {
- my ($class, $zone, $year) = @_;
+ my ($class, $zone, $year, $ip_version) = @_;
+
my $dbh = NP::Model->dbh;
+
+ my $ip_version_sql = $ip_version ? "AND ip_version=" . $dbh->quote($ip_version) : "";
+
my $id;
if ($year) {
- ($id) = $dbh->selectrow_array(q[select id from zone_server_counts where zone_id=? and year(date)=? order by date limit 1],
+ ($id) = $dbh->selectrow_array(qq[select id from zone_server_counts where zone_id=? and year(date)=?
+ $ip_version_sql
+ order by date limit 1],
undef,
$zone->id, $year,
);
}
else {
- ($id) = $dbh->selectrow_array(q[select id from zone_server_counts where zone_id=? order by date limit 1],
+ ($id) = $dbh->selectrow_array(qq[select id from zone_server_counts where zone_id=?
+ $ip_version_sql
+ order by date limit 1],
undef,
$zone->id,
);
}
+
return unless $id;
$class->fetch(id => $id);
}
View
10 lib/NTPPool/Control/Zone.pm
@@ -24,14 +24,15 @@ sub cache_info {
sub zone_name {
my $self = shift;
- my ($zone_name) = ($self->request->uri =~ m!^/zone/(?:graph/)?([^/]+?)(/|\.png)?$!);
+ my ($zone_name) = ($self->request->uri =~ m!^/zone/(?:graph/)?([^/]+?)(/|(-v6)?\.png)?$!);
$zone_name ||= '.';
$zone_name;
}
sub is_graph {
my $self = shift;
- return $self->request->path =~ m!^/zone/graph!;
+ return unless $self->request->path =~ m!^/zone/graph!;
+ return $self->request->path =~ m/-v6.png$/ ? 'v6' : 'v4';
}
# TODO: make the web interface actually do this
@@ -69,9 +70,8 @@ sub render {
return $self->redirect( $1, 301 );
}
-
- if ($self->is_graph) {
- my $path = $zone->graph_path;
+ if (my $ip_version = $self->is_graph) {
+ my $path = $zone->graph_path($ip_version);
open my $fh, $path
or warn "Could not open $path: $!" and return 403;
View
13 sql/ntppool.update
@@ -383,3 +383,16 @@ select now();
alter table server_scores
add created_on datetime not null,
add modified_on timestamp not null;
+
+#51
+alter table zone_server_counts
+ add ip_version enum ('v4', 'v6') not null default 'v4'
+ after zone_id;
+
+alter table zone_server_counts
+ modify ip_version enum ('v4', 'v6') not null;
+
+#52
+alter table zone_server_counts
+ drop key `zone`,
+ add unique key `zone_date` (`zone_id`, `date`, `ip_version`);

0 comments on commit 840cdce

Please sign in to comment.
Something went wrong with that request. Please try again.