Skip to content

Commit

Permalink
Store statistics into a sqlite3 database. This way, we can quickly fi…
Browse files Browse the repository at this point in the history 
…lter links on the top page, deprecating the linkusage page

Signed-off-by: jack <jack@k-net.pro>
  • Loading branch information
jack committed Feb 25, 2017
1 parent f8fe1b7 commit d6c4c0f
Show file tree
Hide file tree
Showing 7 changed files with 108 additions and 85 deletions.
51 changes: 20 additions & 31 deletions bin/rrd-extractstats.pl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@
my @links = values %knownlinks;

# walk through all RRD files in the given path and extract stats for all links
# from them; write the stats to a text file, sorted by total traffic
# from them; write the stats to an sqlite database

my @rrdfiles = File::Find::Rule->maxdepth(2)->file->in($rrdpath);

Expand All @@ -51,45 +51,34 @@
}
print "\n";

# now sort the keys in order of descending total traffic
my @asorder = sort {
my $total_a = 0;

foreach my $t (values %{$astraffic->{$a}}) {
$total_a += $t;
}
my $total_b = 0;
foreach my $t (values %{$astraffic->{$b}}) {
$total_b += $t;
}
return $total_b <=> $total_a;
} keys %$astraffic;

open(STATSFILE, ">$statsfile.tmp");

# print header line
print STATSFILE "as";
my $query = 'create table stats(asn int';
foreach my $link (@links) {
print STATSFILE "\t${link}_in\t${link}_out\t${link}_v6_in\t${link}_v6_out";
$query .= ", ${link}_in int, ${link}_out int, ${link}_v6_in int, ${link}_v6_out int";
}
print STATSFILE "\n";
$query .= ');';

use DBI;
my $db = DBI->connect("dbi:SQLite:dbname=$statsfile.tmp", '', '');
$db->do('PRAGMA synchronous = OFF');
$db->do('drop table if exists stats');
$db->do($query);

# print data
foreach my $as (@asorder) {
print STATSFILE "$as";
foreach my $as (keys %{ $astraffic }) {

$query = "insert into stats values('$as'";

foreach my $link (@links) {
print STATSFILE "\t" . undefaszero($astraffic->{$as}->{"${link}_in"});
print STATSFILE "\t" . undefaszero($astraffic->{$as}->{"${link}_out"});
print STATSFILE "\t" . undefaszero($astraffic->{$as}->{"${link}_v6_in"});
print STATSFILE "\t" . undefaszero($astraffic->{$as}->{"${link}_v6_out"});
$query .= ", '" . undefaszero($astraffic->{$as}->{"${link}_in"}) . "'";
$query .= ", '" . undefaszero($astraffic->{$as}->{"${link}_out"}) . "'";
$query .= ", '" . undefaszero($astraffic->{$as}->{"${link}_v6_in"}) . "'";
$query .= ", '" . undefaszero($astraffic->{$as}->{"${link}_v6_out"}) . "'";
}

print STATSFILE "\n";
$query .= ');';
$db->do($query);
}

close(STATSFILE);

$db->disconnect();
rename("$statsfile.tmp", $statsfile);

sub undefaszero {
Expand Down
1 change: 0 additions & 1 deletion www/config.inc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,6 @@ $show95th = true;
$ntop = 20;
$showv6 = true;
$showtitledetail = true;
$hidelinkusagename = true; # $showtitledetail will need to be true to allow this
$vertical_label = true; # vertical IN/OUT label in graph
$brighten_negative = true; # brighten the "negative" part of graphs

Expand Down
4 changes: 0 additions & 4 deletions www/config_defaults.inc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,10 +10,6 @@ $default_graph_height = 360;
$top_graph_width = 600;
$top_graph_height = 220;

/* Size of graphs on link usage page */
$linkusage_graph_width = 600;
$linkusage_graph_height = 480;

/* Size of graphs on AS-Set page */
$asset_graph_width = 600;
$asset_graph_height = 200;
Expand Down
83 changes: 47 additions & 36 deletions www/func.inc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -95,49 +95,52 @@ function getknownlinks() {
return $knownlinks;
}

function getasstats_top($ntop, $statfile) {
/* first step: walk the data for all ASes to determine the top 5 for the given link */
$fd = fopen($statfile, "r");
if (!$fd)
function getasstats_top($ntop, $statfile, $selected_links) {
try{
$db = new SQLite3($statfile);
}catch(Exception $e){
return array();
$cols = explode("\t", trim(fgets($fd)));

/* read in up to $ntop AS stats, sum up columns */
while (!feof($fd)) {
$line = trim(fgets($fd));
if (!$line)
continue;

$els = explode("\t", $line);

/* first element is the AS */
$as = $els[0];
}

if(sizeof($selected_links) == 0){
$selected_links = array();
foreach(getknownlinks() as $link)
$selected_links[] = $link['tag'];
}

$nlinks = 0;
$query_total = '0';
$query_links = '';
foreach($selected_links as $tag){
$query_links .= "${tag}_in, ${tag}_out, ${tag}_v6_in, ${tag}_v6_out, ";
$nlinks += 4;
$query_total .= " + ${tag}_in + ${tag}_out + ${tag}_v6_in + ${tag}_v6_out";
}
$query = "select asn, $query_links $query_total as total from stats order by total desc limit $ntop";

$asn = $db->query($query);
$asstats = array();
while($row = $asn->fetchArray()){
$tot_in = 0;
$tot_out = 0;
$tot_v6_in = 0;
$tot_v6_out = 0;

for ($i = 1; $i < count($els); $i++) {
if (strpos($cols[$i], "_in") !== false) {
if (strpos($cols[$i], "_v6_") !== false)
$tot_v6_in += $els[$i];
foreach($row as $key => $value){
if (strpos($key, '_in') !== false) {
if (strpos($key, '_v6_') !== false)
$tot_v6_in += $value;
else
$tot_in += $els[$i];
} else {
if (strpos($cols[$i], "_v6_") !== false)
$tot_v6_out += $els[$i];
$tot_in += $value;
} else if (strpos($key, '_out') !== false) {
if (strpos($key, '_v6_') !== false)
$tot_v6_out += $value;
else
$tot_out += $els[$i];
$tot_out += $value;
}
}

$asstats[$as] = array($tot_in, $tot_out, $tot_v6_in, $tot_v6_out);

if (count($asstats) >= $ntop)
break;

$asstats[$row['asn']] = array($tot_in, $tot_out, $tot_v6_in, $tot_v6_out);
}
fclose($fd);

return $asstats;
}

Expand Down Expand Up @@ -224,14 +227,14 @@ function clearCacheFileASSET($asset) {
}

# return the html used in top.php : <a href=blabla><img src=blabla/></url>
function getHTMLUrl($as, $ipversion, $desc, $start, $end, $peerusage){
$img = getHTMLImg($as, $ipversion, $desc, $start, $end, $peerusage, '', '', false);
function getHTMLUrl($as, $ipversion, $desc, $start, $end, $peerusage, $selected_links = array()){
$img = getHTMLImg($as, $ipversion, $desc, $start, $end, $peerusage, '', '', false, $selected_links);
$result = "<a href='history.php?as=$as&peerusage=$peerusage&v=$ipversion' target='_blank'>$img</a>";
return($result);
}

# return the html used in history.php (for example) : <img src=blabla/>
function getHTMLImg($as, $ipversion, $desc, $start, $end, $peerusage, $alt, $class, $history = false){
function getHTMLImg($as, $ipversion, $desc, $start, $end, $peerusage, $alt, $class, $history = false, $selected_links=array()){
global $top_graph_width;
global $top_graph_height;

Expand All @@ -240,6 +243,14 @@ function getHTMLImg($as, $ipversion, $desc, $start, $end, $peerusage, $alt, $cla
$result = "<img alt='$alt' class='$class' src='gengraph.php?v=$ipversion&as=$as&peerusage=$peerusage&dname=$dname&start=$start&end=$end";
if(!$history)
$result .= "&width=$top_graph_width&height=$top_graph_height&nolegend=1";

if(sizeof($selected_links) != 0){
$result .= "&selected_links=";
foreach($selected_links as $link)
$result .= "$link,";
$result = rtrim($result, ',');
}

$result .= "'";

if(!$history)
Expand Down
17 changes: 17 additions & 0 deletions www/gengraph.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,23 @@
$peerusage = 0;

$knownlinks = getknownlinks();

if(isset($_GET['selected_links'])){
$reverse = array();
foreach($knownlinks as $link)
$reverse[$link['tag']] = array('color' => $link['color'], 'descr' => $link['descr']);

$links = array();
foreach(explode(',', $_GET['selected_links']) as $tag){
$link = array('tag' => $tag,
'color' => $reverse[$tag]['color'],
'descr' => $reverse[$tag]['descr']);
$links[] = $link;
}

$knownlinks = $links;
}

$rrdfile = getRRDFileForAS($as, $peerusage);

if ($compat_rrdtool12) {
Expand Down
8 changes: 0 additions & 8 deletions www/headermenu.inc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,14 +17,6 @@ if($showpeeras == true){
endif;
}

echo "<span>Link usage:</span> ";
foreach ($top_intervals as $interval) {
echo '<a href="linkusage.php?numhours=' . $interval['hours'] . '"';
if ($dpagename == "linkusage" && @$_GET['numhours'] == $interval['hours'])
echo ' class="selected"';
echo '>' . $interval['label'] . '</a> | ';
}

if ($dpagename == "history"):
?><a href="history.php" class="selected">View an AS</a> | <?php
else:
Expand Down
29 changes: 24 additions & 5 deletions www/top.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,15 @@
$statsfile = statsFileForHours($hours);
}
$label = statsLabelForHours($hours);
$topas = getasstats_top($ntop, $statsfile);

$knownlinks = getknownlinks();
$selected_links = array();
foreach($knownlinks as $link){
if(isset($_GET["link_${link['tag']}"]))
$selected_links[] = $link['tag'];
}

$topas = getasstats_top($ntop, $statsfile, $selected_links);

$start = time() - $hours*3600;
$end = time();
Expand Down Expand Up @@ -98,9 +106,9 @@
</th>
<td>
<?php
echo getHTMLUrl($as, 4, $asinfo['descr'], $start, $end, $peerusage);
echo getHTMLUrl($as, 4, $asinfo['descr'], $start, $end, $peerusage, $selected_links);
if ($showv6)
echo getHTMLUrl($as, 6, $asinfo['descr'], $start, $end, $peerusage);
echo getHTMLUrl($as, 6, $asinfo['descr'], $start, $end, $peerusage, $selected_links);
?>
</td>
</tr>
Expand All @@ -109,11 +117,20 @@
</table>

<div id="legend">
<form method='get'>
<input type='hidden' name='numhours' value='<?php echo $hours; ?>'/>
<input type='hidden' name='n' value='<?php echo $ntop; ?>'/>
<table>
<?php
$knownlinks = getknownlinks();
foreach ($knownlinks as $link) {
echo "<tr><td style=\"border: 4px solid #fff;\">";
$tag = "link_${link['tag']}";

echo "<tr><td><input type='checkbox'";
if(isset($_GET[$tag]) && $_GET[$tag] == 'on')
echo " checked='checked'";

echo "name=\"$tag\" id=\"$tag\"/></td><td style=\"border: 4px solid #fff;\">";

echo "<table style=\"border-collapse: collapse; margin: 0; padding: 0\"><tr>";
if ($brighten_negative) {
Expand All @@ -124,10 +141,12 @@
}
echo "</tr></table>";

echo "</td><td>&nbsp;" . $link['descr'] . "</td></tr>\n";
echo "</td><td><label for=\"$tag\">&nbsp;${link['descr']}</label></td></tr>\n";
}
?>
</table>
<center><input type='submit' value='Filter'/></center>
</form>
</div>

<?php include('footer.inc'); ?>
Expand Down

2 comments on commit d6c4c0f

@falz
Copy link

@falz falz commented on d6c4c0f Mar 8, 2018

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why was linkusage deleted? It was quite useful. Was there a planned feature to replace this functionality?

@JackSlateur
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It was completely unreliable, by design
You can now filter out specific link, on the main page : this provides an exhaustive sorted AS list per link (or per group of link)

Do you have any more needs not filled up ?

Please sign in to comment.