Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse files

pulling in changes from production svn repo, including timeout/retry …

…enhancements
  • Loading branch information...
commit 3bba9a4da59a56d080e1567bec272ff8f382f220 1 parent deec8b8
Jake Maul authored
View
5 sentry/sentry-multi.pl
@@ -22,7 +22,7 @@
do "sentry.cfg";
my $dbh = DBI->connect( "DBI:mysql:$db:$host",$user,$pass) or die "Connecting : $dbi::errstr\n";
-$mirror_sql = qq{SELECT * FROM mirror_mirrors WHERE active='1' ORDER BY name};
+$mirror_sql = qq{SELECT * FROM mirror_mirrors WHERE active='1' ORDER BY rating DESC};
my $mirror_sth = $dbh->prepare($mirror_sql);
$mirror_sth->execute();
@@ -45,7 +45,8 @@
} else {
die "couldn't fork: $!\n";
}
- sleep 1; # wait a second between each spawn so we don't hose the system by spawning all children at once
+ # don't need to sleep anymore because of the $num_children check.
+ #sleep 1; # wait a second between each spawn so we don't hose the system by spawning all children at once
# if max. number was reached, wait before forking more children
if (++$forked_children >= $num_children) {
View
29 sentry/sentry.pl
@@ -7,13 +7,14 @@
use DBI;
use Data::Dumper;
use LWP;
-use LWP::UserAgent;
+use LWP::UserAgent::Determined;
use Net::DNS;
use URI;
use vars qw( $dbi::errstr );
my $start_timestamp = time;
-my $ua = LWP::UserAgent->new;
+my %ua_options = ('keep_alive' => 5);
+my $ua = LWP::UserAgent::Determined->new(%ua_options);
$ua->timeout(5);
$ua->agent("Mozilla Mirror Monitor/1.0");
@@ -77,6 +78,7 @@ sub log_this {
$getlog_sql = qq{SELECT mirror_rating, mirror_active FROM sentry_log WHERE mirror_id = ? ORDER BY log_date DESC LIMIT 4};
$updatelog_sql = qq{UPDATE sentry_log SET reason=? WHERE log_date=FROM_UNIXTIME(?) AND mirror_id=?};
$updaterating_sql = qq{UPDATE mirror_mirrors SET rating = ? WHERE id = ?};
+$logratingchange_sql = qq{INSERT INTO django_admin_log (action_time, user_id, content_type_id, object_id, object_repr, action_flag, change_message) VALUES (NOW(), 32, 16, ?, ?, 2, ?)};
my $location_sth = $dbh->prepare($location_sql);
my $mirror_sth = $dbh->prepare($mirror_sql);
@@ -86,6 +88,7 @@ sub log_this {
my $getlog_sth = $dbh->prepare($getlog_sql);
my $updatelog_sth = $dbh->prepare($updatelog_sql);
my $updaterating_sth = $dbh->prepare($updaterating_sql);
+my $logratingchange_sth = $dbh->prepare($logratingchange_sql);
# populate a product and os hash if we're debugging stuff
# this way we don't have to make too many selects against the DB
@@ -145,6 +148,13 @@ sub log_this {
next;
}
+ else {
+ my $answerpacket = $netres->query($domain);
+ my @answer = $answerpacket->answer;
+ foreach my $line (@answer) {
+ log_this $line->string . "\n";
+ }
+ }
# test the root of the domain, and mark the mirror as invalid on failure to find anything at root
# we do not allow simple_request because the root of a mirror could return a redirect
@@ -174,6 +184,7 @@ sub log_this {
$newweight = $mirror->{rating} - int($mirror->{rating} * 0.10);
log_this "**** $mirror->{baseurl} Weight change $mirror->{rating} -> $newweight\n";
$updaterating_sth->execute($newweight, $mirror->{id});
+ $logratingchange_sth->execute($mirror->{id}, $mirror->{name}, "Changed rating to $newweight.");
} else {
log_this "Pattern OK, leaving weight unchanged.\n";
}
@@ -200,6 +211,12 @@ sub log_this {
foreach my $location (@locations) {
+ if (($ARGV[0] eq 'checknow') && ((time - $start_timestamp) > 360)) {
+ log_this "*** $mirror->{baseurl} took longer than 6 minutes to execute file checks ... assuming host is overloaded and pulling it out.\n";
+ $failed_mirror_sth->execute($mirror->{id});
+ $log_sth->execute($start_timestamp, $mirror->{id}, '0', $mirror->{rating}, $output);
+ exit;
+ }
my $filepath = $location->{path};
if (($filepath =~ m!/firefox/!)
&& ($filepath !~ m!/namoroka/!)
@@ -219,7 +236,7 @@ sub log_this {
}
}
elsif ($filepath =~ m!/seamonkey/!) {
- if ($filepath =~ m!2\.0\.5!) {
+ if (($filepath =~ m!2\.0\.5!) || ($filepath =~ m!2\.0\.6!)) {
$filepath =~ s@:lang@zh-CN@;
}
else {
@@ -233,7 +250,11 @@ sub log_this {
}
log_this "Checking $filepath... ";
my $req = HTTP::Request->new(HEAD => $mirror->{baseurl} . $filepath);
- my $res = $ua->simple_request($req);
+
+ # allow 1 redirect
+ $ua->max_redirect(1);
+
+ my $res = $ua->request($req);
if (( $res->{_rc} == 200 ) && ( $res->{_headers}->{'content-type'} !~ /text\/html/ )) {
log_this "okay.\n";
View
1  sentry/sentryviewer/csv.cgi
@@ -7,6 +7,7 @@ dbuser = "";
dbpass = "";
print "Content-type: text/csv; charset=utf-8"
+print "Content-disposition: attachment; name=mozillamirrors.csv"
print
print "# mirror_id, mirror_name, mirror_baseurl, mirror_rating, mirror_active"
View
BIN  sentry/sentryviewer/favicon.ico
Binary file not shown
View
118 sentry/sentryviewer/generateuptake.pl
@@ -0,0 +1,118 @@
+#!/usr/bin/perl -wT
+
+use DBI;
+
+my $host = "";
+my $dbname = "";
+my $dbuser = "";
+my $dbpass = "";
+my $dsn = "dbi:mysql:host=$host;database=$dbname";
+my $dbh = DBI->connect($dsn, $dbuser, $dbpass, {});
+$dbh->do("SET NAMES utf8");
+
+# [ product name, product regexp, throttle warn level, throttle critical level ]
+my @productlist = (
+ ['Firefox 7.0b3', "^Firefox-7\\.0b3(-|\$)", 15000, 18000],
+ ['Firefox 6.0.1', "^Firefox-6\\.0\\.1(-|\$)", 45000, 35000],
+ ["Firefox 3.6.21", "^Firefox-3\\.6\\.21(-|\$)", 45000, 35000],
+ ["Thunderbird 7.0b3", "^Thunderbird-7\\.0b3(-|\$)", 15000, 18000],
+ ["Thunderbird 7.0", "^Thunderbird-7\\.0(-|\$)", 15000, 18000],
+ ["Thunderbird 3.1.15", "^Thunderbird-3\\.1\\.15(-|\$)", 30000, 18000],
+ ["SeaMonkey 2.3.2", "^Seamonkey-2\\.3\\.2(-|\$)", 30000, 18000],
+);
+
+my $uptakequery = "SELECT MIN(t.available) from (SELECT
+ SUM( m.rating ) as available,
+ (
+ SELECT SUM( rating )
+ FROM mirror_mirrors
+ WHERE active = '1'
+ ) as total,
+ ( 100 * SUM( m.rating )/
+ (
+ SELECT SUM( rating )
+ FROM mirror_mirrors
+ WHERE active = '1'
+ )
+ ) as percentage,
+ p.name as product_name,
+ o.name as os_name
+ FROM mirror_mirrors m
+ JOIN mirror_location_mirror_map lmm ON lmm.mirror_id = m.id
+ JOIN mirror_locations l ON l.id = lmm.location_id
+ JOIN mirror_products p ON p.id = l.product_id
+ JOIN mirror_os o ON o.id = l.os_id
+ WHERE lmm.active = '1' AND m.active = '1'
+ AND p.name REGEXP ?
+ AND o.name = 'win'
+ GROUP BY lmm.location_id) as t";
+my $uptakeqh = $dbh->prepare($uptakequery);
+
+my $regionsquery = "SELECT id, name, priority, throttle FROM geoip_regions ORDER BY name";
+my $regionsqh = $dbh->prepare($regionsquery);
+$regionsqh->execute();
+my $regions = $regionsqh->fetchall_hashref('name');
+
+my $uptakebyregionquery = "SELECT MIN(t.available) from (SELECT
+ SUM( m.rating ) as available,
+ (
+ SELECT SUM( rating )
+ FROM mirror_mirrors
+ WHERE active = '1'
+ ) as total,
+ ( 100 * SUM( m.rating )/
+ (
+ SELECT SUM( rating )
+ FROM mirror_mirrors
+ WHERE active = '1'
+ )
+ ) as percentage,
+ p.name as product_name,
+ o.name as os_name
+ FROM mirror_mirrors m
+ JOIN mirror_location_mirror_map lmm ON lmm.mirror_id = m.id
+ JOIN mirror_locations l ON l.id = lmm.location_id
+ JOIN mirror_products p ON p.id = l.product_id
+ JOIN mirror_os o ON o.id = l.os_id
+ JOIN geoip_mirror_region_map gmrm ON m.id = gmrm.mirror_id
+ WHERE lmm.active = '1' AND m.active = '1'
+ AND p.name REGEXP ?
+ AND o.name = 'win'
+ AND gmrm.region_id = ?
+ GROUP BY lmm.location_id) as t";
+my $uptakebyregionqh = $dbh->prepare($uptakebyregionquery);
+
+my $weightbyregionquery = "SELECT SUM(m.rating)
+ FROM mirror_mirrors m
+ JOIN geoip_mirror_region_map gmrm ON m.id = gmrm.mirror_id
+ WHERE active = '1' AND gmrm.region_id = ?";
+my $weightbyregionqh = $dbh->prepare($weightbyregionquery);
+
+foreach my $region (sort keys %$regions) {
+ $weightbyregionqh->execute($regions->{$region}->{'id'});
+ my ($weightbyregion) = $weightbyregionqh->fetchrow_array();
+ $regions->{$region}->{'activeweight'} = $weightbyregion;
+}
+
+use Data::Dumper;
+print Data::Dumper->Dump([\@productlist], [qw( productlist )]);
+print Data::Dumper->Dump([$regions], [qw( regions )]);
+
+my %uptake = ();
+for my $productitem (@productlist) {
+ my ($product, $productregexp, $throttlewarn, $throttlecrit) = @$productitem;
+ $uptakeqh->execute($productregexp);
+ $uptake{$product} = {};
+ my ($uptake) = $uptakeqh->fetchrow_array();
+ {
+ $uptake{$product}->{Global} = $uptake;
+ }
+ foreach my $region (sort keys %$regions) {
+ $uptakebyregionqh->execute($productregexp, $regions->{$region}->{'id'});
+ my ($uptakebyregion) = $uptakebyregionqh->fetchrow_array();
+ $uptake{$product}->{$region} = $uptakebyregion;
+ }
+}
+
+print Data::Dumper->Dump([\%uptake], [qw( uptake )]);
+
View
124 sentry/sentryviewer/index.cgi
@@ -1,4 +1,4 @@
-#!/usr/bin/perl -wT
+#!/usr/bin/perl -w
use DBI;
use Date::Format;
@@ -10,42 +10,22 @@ my $dbuser = "";
my $dbpass = "";
my $dsn = "dbi:mysql:host=$host;database=$dbname";
my $dbh = DBI->connect($dsn, $dbuser, $dbpass, {});
+$dbh->do("SET NAMES utf8");
# [ product name, product regexp, throttle warn level, throttle critical level ]
-my @productlist = (
- ["Firefox 3.6.3", "^Firefox-3\\.6\\.3(-|\$)", 15000, 8000],
- ["Firefox 3.5.9", "^Firefox-3\\.5\\.9(-|\$)", 45000, 35000],
- ["Firefox 3.0.19", "^Firefox-3\\.0\\.19(-|\$)", 45000, 35000],
- ["SeaMonkey 2.0.4", "^Seamonkey-2\\.0\\.4(-|\$)", 15000, 8000],
- ["Thunderbird 3.0.4", "^Thunderbird-3\\.0\\.4(-|\$)", 15000, 8000],
-);
-
-my $uptakequery = "SELECT MIN(t.available) from (SELECT
- SUM( m.rating ) as available,
- (
- SELECT SUM( rating )
- FROM mirror_mirrors
- WHERE active = '1'
- ) as total,
- ( 100 * SUM( m.rating )/
- (
- SELECT SUM( rating )
- FROM mirror_mirrors
- WHERE active = '1'
- )
- ) as percentage,
- p.name as product_name,
- o.name as os_name
- FROM mirror_mirrors m
- JOIN mirror_location_mirror_map lmm ON lmm.mirror_id = m.id
- JOIN mirror_locations l ON l.id = lmm.location_id
- JOIN mirror_products p ON p.id = l.product_id
- JOIN mirror_os o ON o.id = l.os_id
- WHERE lmm.active = '1' AND m.active = '1'
- AND p.name REGEXP ?
- AND o.name = 'win'
- GROUP BY lmm.location_id) as t";
-my $uptakeqh = $dbh->prepare($uptakequery);
+# PRODUCT LIST IS NOW IN THE generateuptake.pl FILE
+$::ENV{PATH} = '/usr/bin:/bin';
+our ($productlist, $regions, $uptake);
+my $uptakedata = `cat /usr/share/nagios/sentry/currentuptake.txt`;
+#($uptakedata) = $uptakedata =~ /^(.*)$/;
+#print "DEBUG: $uptakedata\n";
+eval $uptakedata;
+my @productlist = @$productlist;
+
+#use Data::Dumper;
+#print Data::Dumper::Dumper(\@productlist);
+#exit;
+
my $query = "SELECT FROM_UNIXTIME((UNIX_TIMESTAMP(log_date) - (UNIX_TIMESTAMP(log_date) % 300))) AS log_run,
check_time,
@@ -54,7 +34,7 @@ my $query = "SELECT FROM_UNIXTIME((UNIX_TIMESTAMP(log_date) - (UNIX_TIMESTAMP(lo
sentry_log.mirror_id,
sentry_log.mirror_active,
sentry_log.mirror_rating,
- LEFT(reason,20)
+ LEFT(reason,160)
FROM sentry_log
INNER JOIN mirror_mirrors ON sentry_log.mirror_id = mirror_mirrors.id
WHERE log_date > NOW() - INTERVAL 2 HOUR
@@ -103,11 +83,13 @@ Refresh: 150;
<html>
<head>
<title>Sentry log overview</title>
+<link rel="shortcut icon" type="image/png" href="favicon.ico" />
<style type="text/css"><!--
table { border: solid black 1px; padding: 0px; }
td { border: solid black 1px; font-family: sans-serif; font-size: xx-small; white-space: nowrap; }
td.active { background-color: #9f9; }
td.inactive { background-color: #f99; }
+td.redirected { background-color: #f9f; }
td.dnsfail { background-color: #99f; }
tr:hover td { border: solid yellow 1px; }
td.good { background-color: #9f9; }
@@ -118,11 +100,57 @@ tr.divider td { background-color: black; height: 4px; font-size: 1px; }
// --></style>
</head>
<body>
+<h3>Product Uptake</h3>
+
+<p>There are region overlaps, so the sum of all regions will be much higher than the global total. For example, the CDN-type mirrors each serve multiple regions.</p>
+<p>
+<span style="background-color: #faa">Definitely not enough</span>
+<span style="background-color: yellow">Should probably consider throttling</span>
+<span style="background-color: #afa">Good for release traffic</span>
+</p>
+
+<table>
+EOF
+
+print "<tr><td><b>Product</b></td><td><b>Global</b></td>";
+for my $region (sort keys %$regions) {
+ print "<td><b>$region</b></td>";
+}
+print "</tr>\n";
+print "<tr><td><b>Percent of traffic remains in-region:</b></tf><td></td>";
+for my $region (sort keys %$regions) {
+ print "<td><b>" . $regions->{$region}->{'throttle'} . "</b></td>";
+}
+print "<tr><td><b>Active and available aggregate weight:</b></td><td></td>";
+for my $region (sort keys %$regions) {
+ print "<td><b>" . $regions->{$region}->{'activeweight'} . "</b></td>";
+}
+print "</tr>\n";
+for my $productitem (@productlist) {
+ my ($product, $productregexp, $throttlewarn, $throttlecrit) = @$productitem;
+ my $thisuptake = $uptake->{$product}->{'Global'};
+ print "</tr>\n<tr><td>$product Uptake ($throttlewarn to stay unthrottled)</td>";
+ {
+ my $class = "good";
+ if ($thisuptake < $throttlewarn) { $class = "ok" };
+ if ($thisuptake < $throttlecrit) { $class = "poor" };
+ print qq{<td class="$class" title="$class">} . $thisuptake . "</td>";
+ }
+ foreach my $region (sort keys %$regions) {
+ my ($uptakebyregion) = $uptake->{$product}->{$region} || "";
+ print qq{<td>$uptakebyregion</td>};
+ }
+ print "</tr>\n";
+}
+
+print <<EOF;
+</table>
<h3>Sentry check stats for the last 2 hours</h3>
<p>Newest checks are on the left. Note that the most-recent column is often incomplete if checks are still running. The number inside the colored square is that server's weight at the time of the check.</p>
<p>
<span style="background-color: #faa">Response timeout or connection failure</span>
<span style="background-color: #aaf">DNS lookup failure</span>
+<span style="background-color: #faf">Server redirects us somewhere else</span>
<span style="background-color: #afa">Active and responsive</span>
</p>
<table>
@@ -148,24 +176,8 @@ foreach my $timestamp (reverse sort keys %seentime) {
if ($activeweight{$timestamp} < $productlist[0][3]) { $class = "poor" };
print qq{<td class="$class" title="$class">} . $activeweight{$timestamp} . "</td>";
}
-for my $productitem (@productlist) {
- my ($product, $productregexp, $throttlewarn, $throttlecrit) = @$productitem;
- $uptakeqh->execute($productregexp);
- my ($uptake) = $uptakeqh->fetchrow_array();
- print "</tr>\n<tr><td>$product Uptake ($throttlewarn to stay unthrottled)</td>";
- {
- my $class = "good";
- if ($uptake < $throttlewarn) { $class = "ok" };
- if ($uptake < $throttlecrit) { $class = "poor" };
- print qq{<td class="$class" title="$class">} . $uptake . "</td>";
- }
- my $colcount = scalar(keys %seentime);
- $colcount--;
- print "<td>&nbsp;</td>" x $colcount;
- print "</tr>\n";
-}
+
print qq{</tr>\n<tr class="divider"><td>&nbsp;</td>};
-print "<td>&nbsp;</td>" x scalar(keys %seentime);
print "</tr>\n";
foreach my $url (reverse sort { $weight{$a} <=> $weight{$b} } keys %byhost) {
@@ -178,6 +190,12 @@ foreach my $url (reverse sort { $weight{$a} <=> $weight{$b} } keys %byhost) {
my $reason = $byhost{$url}->{$timestamp}->{"reason"} || "Active and responsive";
my $rating = $byhost{$url}->{$timestamp}->{"rating"};
if ($byhost{$url}->{$timestamp}->{reason} =~ /DNS/i) { $class = "dnsfail" }
+ if ($byhost{$url}->{$timestamp}->{reason} =~ /returned error 3/im) {
+ $class = "redirected";
+ $reason =~ s/^.*?returned error 3\d\d \(//;
+ $reason =~ s/\).*/ - click for details/s;
+ $reason =~ s/redirect to/redirects to/;
+ }
if ($class eq 'active') {
$reason = "Click for full report";
}
View
2  sentry/sentryviewer/logentry.cgi
@@ -29,7 +29,7 @@ my $timestamp = $cgi->param("time");
$::ENV{PATH} = "/usr/bin:/bin"; # for `` from taint mode
my $timezone = `/bin/date +\%Z`;
chomp $timezone;
-my $mqh = $dbh->prepare("SELECT name, baseurl FROM mirror_mirrors WHERE mirror_id=?");
+my $mqh = $dbh->prepare("SELECT name, baseurl FROM mirror_mirrors WHERE id=?");
$mqh->execute($id);
my ($mirror_name, $baseurl) = $mqh->fetchrow_array();
my $qh = $dbh->prepare("SELECT reason FROM sentry_log WHERE mirror_id=? AND check_time=?");
Please sign in to comment.
Something went wrong with that request. Please try again.