Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP

Loading…

Changes from current production sentry svn repo #19

Merged
merged 3 commits into from

3 participants

@superawesome

Various things that are done in the current production sentry node, pulled from its SVN repo.

Of particular note is the changes around retrying failed products/mirrors.

sentry/sentry-multi.pl
@@ -45,7 +45,8 @@
} else {
die "couldn't fork: $!\n";
}
- sleep 1; # wait a second between each spawn so we don't hose the system by spawning all children at once
+ # don't need to sleep anymore because of the $num_children check.
@fwenzel Owner
fwenzel added a note

Just remove the line then :)

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
sentry/sentry.pl
@@ -219,7 +236,7 @@ sub log_this {
}
}
elsif ($filepath =~ m!/seamonkey/!) {
- if ($filepath =~ m!2\.0\.5!) {
+ if (($filepath =~ m!2\.0\.5!) || ($filepath =~ m!2\.0\.6!)) {
@fwenzel Owner
fwenzel added a note

uh, this indentation is off.

Also, I guess a comment wouldn't have hurt here.

@superawesome Owner

This chooses which locale should be tested. Historically we test the alphabetically last locale available, because rsync generally runs in alphabetical order... the thinking is that if the last locale exists, then the other do too. It's inherently a trade-off designed to avoid having to test every single locale individually.

I read this as "if seamonkey, and if it's version 2.0.5 or 2.0.6, check zh-CN. For all other versions, check tr".

Based on wandering around here: http://download.cdn.mozilla.net/pub/seamonkey/releases/ ... it appears that Seamonkey did indeed have zh-CN support for those two versions, and potentially a few others. However it was not consistent. 2.1 did not have it... 2.13 (current) has zh-CN and zh-TW. In many versions, "tr" was indeed the alphabetically-last locale.

However, for the most part it doesn't matter anymore. The lone mirrors are reverse-proxying CDNs, not rsync-based. The logic of checking the last locale no longer holds any real value... any locale is just as good as any other. We still need to check a locale, but it doesn't matter which one.

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
@fwenzel
Owner

This looks good now, thanks Jake. Looks like these are various tweaks presumably written by justdave for sentry and sentry viewer. Thanks for porting them!

@fwenzel
Owner

Do you want to merge this yourself? Otherwise, @brandonsavage or @Rik ?

@brandonsavage brandonsavage merged commit a053195 into from
@superawesome

Thanks!

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
Commits on Oct 16, 2012
  1. pulling in changes from production svn repo, including timeout/retry …

    Jake Maul authored
    …enhancements
Commits on Oct 17, 2012
  1. indentation and comment cleanup

    Jake Maul authored
  2. indentation cleanup... s/tab/space/

    Jake Maul authored
This page is out of date. Refresh to see the latest.
View
3  sentry/sentry-multi.pl
@@ -22,7 +22,7 @@
do "sentry.cfg";
my $dbh = DBI->connect( "DBI:mysql:$db:$host",$user,$pass) or die "Connecting : $dbi::errstr\n";
-$mirror_sql = qq{SELECT * FROM mirror_mirrors WHERE active='1' ORDER BY name};
+$mirror_sql = qq{SELECT * FROM mirror_mirrors WHERE active='1' ORDER BY rating DESC};
my $mirror_sth = $dbh->prepare($mirror_sql);
$mirror_sth->execute();
@@ -45,7 +45,6 @@
} else {
die "couldn't fork: $!\n";
}
- sleep 1; # wait a second between each spawn so we don't hose the system by spawning all children at once
# if max. number was reached, wait before forking more children
if (++$forked_children >= $num_children) {
View
39 sentry/sentry.pl
@@ -7,13 +7,14 @@
use DBI;
use Data::Dumper;
use LWP;
-use LWP::UserAgent;
+use LWP::UserAgent::Determined;
use Net::DNS;
use URI;
use vars qw( $dbi::errstr );
my $start_timestamp = time;
-my $ua = LWP::UserAgent->new;
+my %ua_options = ('keep_alive' => 5);
+my $ua = LWP::UserAgent::Determined->new(%ua_options);
$ua->timeout(5);
$ua->agent("Mozilla Mirror Monitor/1.0");
@@ -77,6 +78,7 @@ sub log_this {
$getlog_sql = qq{SELECT mirror_rating, mirror_active FROM sentry_log WHERE mirror_id = ? ORDER BY log_date DESC LIMIT 4};
$updatelog_sql = qq{UPDATE sentry_log SET reason=? WHERE log_date=FROM_UNIXTIME(?) AND mirror_id=?};
$updaterating_sql = qq{UPDATE mirror_mirrors SET rating = ? WHERE id = ?};
+$logratingchange_sql = qq{INSERT INTO django_admin_log (action_time, user_id, content_type_id, object_id, object_repr, action_flag, change_message) VALUES (NOW(), 32, 16, ?, ?, 2, ?)};
my $location_sth = $dbh->prepare($location_sql);
my $mirror_sth = $dbh->prepare($mirror_sql);
@@ -86,6 +88,7 @@ sub log_this {
my $getlog_sth = $dbh->prepare($getlog_sql);
my $updatelog_sth = $dbh->prepare($updatelog_sql);
my $updaterating_sth = $dbh->prepare($updaterating_sql);
+my $logratingchange_sth = $dbh->prepare($logratingchange_sql);
# populate a product and os hash if we're debugging stuff
# this way we don't have to make too many selects against the DB
@@ -145,6 +148,13 @@ sub log_this {
next;
}
+ else {
+ my $answerpacket = $netres->query($domain);
+ my @answer = $answerpacket->answer;
+ foreach my $line (@answer) {
+ log_this $line->string . "\n";
+ }
+ }
# test the root of the domain, and mark the mirror as invalid on failure to find anything at root
# we do not allow simple_request because the root of a mirror could return a redirect
@@ -174,6 +184,7 @@ sub log_this {
$newweight = $mirror->{rating} - int($mirror->{rating} * 0.10);
log_this "**** $mirror->{baseurl} Weight change $mirror->{rating} -> $newweight\n";
$updaterating_sth->execute($newweight, $mirror->{id});
+ $logratingchange_sth->execute($mirror->{id}, $mirror->{name}, "Changed rating to $newweight.");
} else {
log_this "Pattern OK, leaving weight unchanged.\n";
}
@@ -200,6 +211,12 @@ sub log_this {
foreach my $location (@locations) {
+ if (($ARGV[0] eq 'checknow') && ((time - $start_timestamp) > 360)) {
+ log_this "*** $mirror->{baseurl} took longer than 6 minutes to execute file checks ... assuming host is overloaded and pulling it out.\n";
+ $failed_mirror_sth->execute($mirror->{id});
+ $log_sth->execute($start_timestamp, $mirror->{id}, '0', $mirror->{rating}, $output);
+ exit;
+ }
my $filepath = $location->{path};
if (($filepath =~ m!/firefox/!)
&& ($filepath !~ m!/namoroka/!)
@@ -219,12 +236,12 @@ sub log_this {
}
}
elsif ($filepath =~ m!/seamonkey/!) {
- if ($filepath =~ m!2\.0\.5!) {
- $filepath =~ s@:lang@zh-CN@;
- }
- else {
- $filepath =~ s@:lang@tr@;
- }
+ if (($filepath =~ m!2\.0\.5!) || ($filepath =~ m!2\.0\.6!)) {
+ $filepath =~ s@:lang@zh-CN@;
+ }
+ else {
+ $filepath =~ s@:lang@tr@;
+ }
}
elsif ($filepath =~ m!-euballot/!i) {
$filepath =~ s@:lang@sv-SE@;
@@ -233,7 +250,11 @@ sub log_this {
}
log_this "Checking $filepath... ";
my $req = HTTP::Request->new(HEAD => $mirror->{baseurl} . $filepath);
- my $res = $ua->simple_request($req);
+
+ # allow 1 redirect
+ $ua->max_redirect(1);
+
+ my $res = $ua->request($req);
if (( $res->{_rc} == 200 ) && ( $res->{_headers}->{'content-type'} !~ /text\/html/ )) {
log_this "okay.\n";
View
1  sentry/sentryviewer/csv.cgi
@@ -7,6 +7,7 @@ dbuser = "";
dbpass = "";
print "Content-type: text/csv; charset=utf-8"
+print "Content-disposition: attachment; name=mozillamirrors.csv"
print
print "# mirror_id, mirror_name, mirror_baseurl, mirror_rating, mirror_active"
View
BIN  sentry/sentryviewer/favicon.ico
Binary file not shown
View
118 sentry/sentryviewer/generateuptake.pl
@@ -0,0 +1,118 @@
+#!/usr/bin/perl -wT
+
+use DBI;
+
+my $host = "";
+my $dbname = "";
+my $dbuser = "";
+my $dbpass = "";
+my $dsn = "dbi:mysql:host=$host;database=$dbname";
+my $dbh = DBI->connect($dsn, $dbuser, $dbpass, {});
+$dbh->do("SET NAMES utf8");
+
+# [ product name, product regexp, throttle warn level, throttle critical level ]
+my @productlist = (
+ ['Firefox 7.0b3', "^Firefox-7\\.0b3(-|\$)", 15000, 18000],
+ ['Firefox 6.0.1', "^Firefox-6\\.0\\.1(-|\$)", 45000, 35000],
+ ["Firefox 3.6.21", "^Firefox-3\\.6\\.21(-|\$)", 45000, 35000],
+ ["Thunderbird 7.0b3", "^Thunderbird-7\\.0b3(-|\$)", 15000, 18000],
+ ["Thunderbird 7.0", "^Thunderbird-7\\.0(-|\$)", 15000, 18000],
+ ["Thunderbird 3.1.15", "^Thunderbird-3\\.1\\.15(-|\$)", 30000, 18000],
+ ["SeaMonkey 2.3.2", "^Seamonkey-2\\.3\\.2(-|\$)", 30000, 18000],
+);
+
+my $uptakequery = "SELECT MIN(t.available) from (SELECT
+ SUM( m.rating ) as available,
+ (
+ SELECT SUM( rating )
+ FROM mirror_mirrors
+ WHERE active = '1'
+ ) as total,
+ ( 100 * SUM( m.rating )/
+ (
+ SELECT SUM( rating )
+ FROM mirror_mirrors
+ WHERE active = '1'
+ )
+ ) as percentage,
+ p.name as product_name,
+ o.name as os_name
+ FROM mirror_mirrors m
+ JOIN mirror_location_mirror_map lmm ON lmm.mirror_id = m.id
+ JOIN mirror_locations l ON l.id = lmm.location_id
+ JOIN mirror_products p ON p.id = l.product_id
+ JOIN mirror_os o ON o.id = l.os_id
+ WHERE lmm.active = '1' AND m.active = '1'
+ AND p.name REGEXP ?
+ AND o.name = 'win'
+ GROUP BY lmm.location_id) as t";
+my $uptakeqh = $dbh->prepare($uptakequery);
+
+my $regionsquery = "SELECT id, name, priority, throttle FROM geoip_regions ORDER BY name";
+my $regionsqh = $dbh->prepare($regionsquery);
+$regionsqh->execute();
+my $regions = $regionsqh->fetchall_hashref('name');
+
+my $uptakebyregionquery = "SELECT MIN(t.available) from (SELECT
+ SUM( m.rating ) as available,
+ (
+ SELECT SUM( rating )
+ FROM mirror_mirrors
+ WHERE active = '1'
+ ) as total,
+ ( 100 * SUM( m.rating )/
+ (
+ SELECT SUM( rating )
+ FROM mirror_mirrors
+ WHERE active = '1'
+ )
+ ) as percentage,
+ p.name as product_name,
+ o.name as os_name
+ FROM mirror_mirrors m
+ JOIN mirror_location_mirror_map lmm ON lmm.mirror_id = m.id
+ JOIN mirror_locations l ON l.id = lmm.location_id
+ JOIN mirror_products p ON p.id = l.product_id
+ JOIN mirror_os o ON o.id = l.os_id
+ JOIN geoip_mirror_region_map gmrm ON m.id = gmrm.mirror_id
+ WHERE lmm.active = '1' AND m.active = '1'
+ AND p.name REGEXP ?
+ AND o.name = 'win'
+ AND gmrm.region_id = ?
+ GROUP BY lmm.location_id) as t";
+my $uptakebyregionqh = $dbh->prepare($uptakebyregionquery);
+
+my $weightbyregionquery = "SELECT SUM(m.rating)
+ FROM mirror_mirrors m
+ JOIN geoip_mirror_region_map gmrm ON m.id = gmrm.mirror_id
+ WHERE active = '1' AND gmrm.region_id = ?";
+my $weightbyregionqh = $dbh->prepare($weightbyregionquery);
+
+foreach my $region (sort keys %$regions) {
+ $weightbyregionqh->execute($regions->{$region}->{'id'});
+ my ($weightbyregion) = $weightbyregionqh->fetchrow_array();
+ $regions->{$region}->{'activeweight'} = $weightbyregion;
+}
+
+use Data::Dumper;
+print Data::Dumper->Dump([\@productlist], [qw( productlist )]);
+print Data::Dumper->Dump([$regions], [qw( regions )]);
+
+my %uptake = ();
+for my $productitem (@productlist) {
+ my ($product, $productregexp, $throttlewarn, $throttlecrit) = @$productitem;
+ $uptakeqh->execute($productregexp);
+ $uptake{$product} = {};
+ my ($uptake) = $uptakeqh->fetchrow_array();
+ {
+ $uptake{$product}->{Global} = $uptake;
+ }
+ foreach my $region (sort keys %$regions) {
+ $uptakebyregionqh->execute($productregexp, $regions->{$region}->{'id'});
+ my ($uptakebyregion) = $uptakebyregionqh->fetchrow_array();
+ $uptake{$product}->{$region} = $uptakebyregion;
+ }
+}
+
+print Data::Dumper->Dump([\%uptake], [qw( uptake )]);
+
View
124 sentry/sentryviewer/index.cgi
@@ -1,4 +1,4 @@
-#!/usr/bin/perl -wT
+#!/usr/bin/perl -w
use DBI;
use Date::Format;
@@ -10,42 +10,22 @@ my $dbuser = "";
my $dbpass = "";
my $dsn = "dbi:mysql:host=$host;database=$dbname";
my $dbh = DBI->connect($dsn, $dbuser, $dbpass, {});
+$dbh->do("SET NAMES utf8");
# [ product name, product regexp, throttle warn level, throttle critical level ]
-my @productlist = (
- ["Firefox 3.6.3", "^Firefox-3\\.6\\.3(-|\$)", 15000, 8000],
- ["Firefox 3.5.9", "^Firefox-3\\.5\\.9(-|\$)", 45000, 35000],
- ["Firefox 3.0.19", "^Firefox-3\\.0\\.19(-|\$)", 45000, 35000],
- ["SeaMonkey 2.0.4", "^Seamonkey-2\\.0\\.4(-|\$)", 15000, 8000],
- ["Thunderbird 3.0.4", "^Thunderbird-3\\.0\\.4(-|\$)", 15000, 8000],
-);
-
-my $uptakequery = "SELECT MIN(t.available) from (SELECT
- SUM( m.rating ) as available,
- (
- SELECT SUM( rating )
- FROM mirror_mirrors
- WHERE active = '1'
- ) as total,
- ( 100 * SUM( m.rating )/
- (
- SELECT SUM( rating )
- FROM mirror_mirrors
- WHERE active = '1'
- )
- ) as percentage,
- p.name as product_name,
- o.name as os_name
- FROM mirror_mirrors m
- JOIN mirror_location_mirror_map lmm ON lmm.mirror_id = m.id
- JOIN mirror_locations l ON l.id = lmm.location_id
- JOIN mirror_products p ON p.id = l.product_id
- JOIN mirror_os o ON o.id = l.os_id
- WHERE lmm.active = '1' AND m.active = '1'
- AND p.name REGEXP ?
- AND o.name = 'win'
- GROUP BY lmm.location_id) as t";
-my $uptakeqh = $dbh->prepare($uptakequery);
+# PRODUCT LIST IS NOW IN THE generateuptake.pl FILE
+$::ENV{PATH} = '/usr/bin:/bin';
+our ($productlist, $regions, $uptake);
+my $uptakedata = `cat /usr/share/nagios/sentry/currentuptake.txt`;
+#($uptakedata) = $uptakedata =~ /^(.*)$/;
+#print "DEBUG: $uptakedata\n";
+eval $uptakedata;
+my @productlist = @$productlist;
+
+#use Data::Dumper;
+#print Data::Dumper::Dumper(\@productlist);
+#exit;
+
my $query = "SELECT FROM_UNIXTIME((UNIX_TIMESTAMP(log_date) - (UNIX_TIMESTAMP(log_date) % 300))) AS log_run,
check_time,
@@ -54,7 +34,7 @@ my $query = "SELECT FROM_UNIXTIME((UNIX_TIMESTAMP(log_date) - (UNIX_TIMESTAMP(lo
sentry_log.mirror_id,
sentry_log.mirror_active,
sentry_log.mirror_rating,
- LEFT(reason,20)
+ LEFT(reason,160)
FROM sentry_log
INNER JOIN mirror_mirrors ON sentry_log.mirror_id = mirror_mirrors.id
WHERE log_date > NOW() - INTERVAL 2 HOUR
@@ -103,11 +83,13 @@ Refresh: 150;
<html>
<head>
<title>Sentry log overview</title>
+<link rel="shortcut icon" type="image/png" href="favicon.ico" />
<style type="text/css"><!--
table { border: solid black 1px; padding: 0px; }
td { border: solid black 1px; font-family: sans-serif; font-size: xx-small; white-space: nowrap; }
td.active { background-color: #9f9; }
td.inactive { background-color: #f99; }
+td.redirected { background-color: #f9f; }
td.dnsfail { background-color: #99f; }
tr:hover td { border: solid yellow 1px; }
td.good { background-color: #9f9; }
@@ -118,11 +100,57 @@ tr.divider td { background-color: black; height: 4px; font-size: 1px; }
// --></style>
</head>
<body>
+<h3>Product Uptake</h3>
+
+<p>There are region overlaps, so the sum of all regions will be much higher than the global total. For example, the CDN-type mirrors each serve multiple regions.</p>
+<p>
+<span style="background-color: #faa">Definitely not enough</span>
+<span style="background-color: yellow">Should probably consider throttling</span>
+<span style="background-color: #afa">Good for release traffic</span>
+</p>
+
+<table>
+EOF
+
+print "<tr><td><b>Product</b></td><td><b>Global</b></td>";
+for my $region (sort keys %$regions) {
+ print "<td><b>$region</b></td>";
+}
+print "</tr>\n";
+print "<tr><td><b>Percent of traffic remains in-region:</b></tf><td></td>";
+for my $region (sort keys %$regions) {
+ print "<td><b>" . $regions->{$region}->{'throttle'} . "</b></td>";
+}
+print "<tr><td><b>Active and available aggregate weight:</b></td><td></td>";
+for my $region (sort keys %$regions) {
+ print "<td><b>" . $regions->{$region}->{'activeweight'} . "</b></td>";
+}
+print "</tr>\n";
+for my $productitem (@productlist) {
+ my ($product, $productregexp, $throttlewarn, $throttlecrit) = @$productitem;
+ my $thisuptake = $uptake->{$product}->{'Global'};
+ print "</tr>\n<tr><td>$product Uptake ($throttlewarn to stay unthrottled)</td>";
+ {
+ my $class = "good";
+ if ($thisuptake < $throttlewarn) { $class = "ok" };
+ if ($thisuptake < $throttlecrit) { $class = "poor" };
+ print qq{<td class="$class" title="$class">} . $thisuptake . "</td>";
+ }
+ foreach my $region (sort keys %$regions) {
+ my ($uptakebyregion) = $uptake->{$product}->{$region} || "";
+ print qq{<td>$uptakebyregion</td>};
+ }
+ print "</tr>\n";
+}
+
+print <<EOF;
+</table>
<h3>Sentry check stats for the last 2 hours</h3>
<p>Newest checks are on the left. Note that the most-recent column is often incomplete if checks are still running. The number inside the colored square is that server's weight at the time of the check.</p>
<p>
<span style="background-color: #faa">Response timeout or connection failure</span>
<span style="background-color: #aaf">DNS lookup failure</span>
+<span style="background-color: #faf">Server redirects us somewhere else</span>
<span style="background-color: #afa">Active and responsive</span>
</p>
<table>
@@ -148,24 +176,8 @@ foreach my $timestamp (reverse sort keys %seentime) {
if ($activeweight{$timestamp} < $productlist[0][3]) { $class = "poor" };
print qq{<td class="$class" title="$class">} . $activeweight{$timestamp} . "</td>";
}
-for my $productitem (@productlist) {
- my ($product, $productregexp, $throttlewarn, $throttlecrit) = @$productitem;
- $uptakeqh->execute($productregexp);
- my ($uptake) = $uptakeqh->fetchrow_array();
- print "</tr>\n<tr><td>$product Uptake ($throttlewarn to stay unthrottled)</td>";
- {
- my $class = "good";
- if ($uptake < $throttlewarn) { $class = "ok" };
- if ($uptake < $throttlecrit) { $class = "poor" };
- print qq{<td class="$class" title="$class">} . $uptake . "</td>";
- }
- my $colcount = scalar(keys %seentime);
- $colcount--;
- print "<td>&nbsp;</td>" x $colcount;
- print "</tr>\n";
-}
+
print qq{</tr>\n<tr class="divider"><td>&nbsp;</td>};
-print "<td>&nbsp;</td>" x scalar(keys %seentime);
print "</tr>\n";
foreach my $url (reverse sort { $weight{$a} <=> $weight{$b} } keys %byhost) {
@@ -178,6 +190,12 @@ foreach my $url (reverse sort { $weight{$a} <=> $weight{$b} } keys %byhost) {
my $reason = $byhost{$url}->{$timestamp}->{"reason"} || "Active and responsive";
my $rating = $byhost{$url}->{$timestamp}->{"rating"};
if ($byhost{$url}->{$timestamp}->{reason} =~ /DNS/i) { $class = "dnsfail" }
+ if ($byhost{$url}->{$timestamp}->{reason} =~ /returned error 3/im) {
+ $class = "redirected";
+ $reason =~ s/^.*?returned error 3\d\d \(//;
+ $reason =~ s/\).*/ - click for details/s;
+ $reason =~ s/redirect to/redirects to/;
+ }
if ($class eq 'active') {
$reason = "Click for full report";
}
View
2  sentry/sentryviewer/logentry.cgi
@@ -29,7 +29,7 @@ my $timestamp = $cgi->param("time");
$::ENV{PATH} = "/usr/bin:/bin"; # for `` from taint mode
my $timezone = `/bin/date +\%Z`;
chomp $timezone;
-my $mqh = $dbh->prepare("SELECT name, baseurl FROM mirror_mirrors WHERE mirror_id=?");
+my $mqh = $dbh->prepare("SELECT name, baseurl FROM mirror_mirrors WHERE id=?");
$mqh->execute($id);
my ($mirror_name, $baseurl) = $mqh->fetchrow_array();
my $qh = $dbh->prepare("SELECT reason FROM sentry_log WHERE mirror_id=? AND check_time=?");
Something went wrong with that request. Please try again.