Skip to content

Commit

Permalink
Write weight adjustments into dbs table according to how lagged/bogged
Browse files Browse the repository at this point in the history 
each reader DB is.  Log more info into dbs_readerstatus table.  Add
"dbs" comment to irc bot.
  • Loading branch information
@jamiemccarthy
jamiemccarthy committed Nov 24, 2004
1 parent 21746e9 commit 8ebdfc9
Show file tree
Hide file tree
Showing 7 changed files with 171 additions and 7 deletions.
16 changes: 16 additions & 0 deletions Slash/DB/Static/MySQL/MySQL.pm
View file
Expand Up @@ -232,6 +232,22 @@ sub deleteOldDBReaderStatus {
"ts < DATE_SUB(NOW(), INTERVAL $secs_back SECOND)");
}

########################################################
# For ircslash.pl
sub getDBsReaderStatus {
my($self, $secs_back) = @_;
$secs_back ||= 60;
return $self->sqlSelectAllHashref(
"dbid",
"dbid,
MIN(IF(was_alive='yes',1,0)) AS was_alive,
AVG(slave_lag_secs) AS lag,
AVG(query_bog_secs) AS bog",
"dbs_readerstatus",
"ts >= DATE_SUB(NOW(), INTERVAL $secs_back SECOND)",
"GROUP BY dbid");
}

########################################################
# For dailystuff
sub deleteRecycledComments {
Expand Down
8 changes: 8 additions & 0 deletions sql/mysql/defaults.sql
View file
Expand Up @@ -710,7 +710,15 @@ INSERT INTO vars (name, value, description) VALUES ('cur_performance_stats_weeks
INSERT INTO vars (name, value, description) VALUES ('currentqid',1,'The Current Question on the homepage pollbooth');
INSERT INTO vars (name, value, description) VALUES ('datadir','/usr/local/slash/www.example.com','What is the root of the install for Slash');
INSERT INTO vars (name, value, description) VALUES ('dbs_reader_adjust_delay','5','Number of seconds between each adjustment of reader DB weights');
INSERT INTO vars (name, value, description) VALUES ('dbs_reader_bog_secs_start','5','Number of seconds of reader DB bog at which balance_readers.pl should start to reduce its weight');
INSERT INTO vars (name, value, description) VALUES ('dbs_reader_bog_secs_end','60','Number of seconds of reader DB bog at which balance_readers.pl hits the minimum weight');
INSERT INTO vars (name, value, description) VALUES ('dbs_reader_bog_weight_min','0.2','The minimum weight to multiply a reader DB\'s base weight by once its bog hits dbs_reader_bog_secs_end');
INSERT INTO vars (name, value, description) VALUES ('dbs_reader_expire_secs', 86400 * 7,'Number of seconds worth of dbs_readerstatus log to keep around');
INSERT INTO vars (name, value, description) VALUES ('dbs_reader_lag_secs_start','5','Number of seconds of reader DB lag at which balance_readers.pl should start to reduce its weight');
INSERT INTO vars (name, value, description) VALUES ('dbs_reader_lag_secs_end','30','Number of seconds of reader DB lag at which balance_readers.pl hits the minimum weight');
INSERT INTO vars (name, value, description) VALUES ('dbs_reader_lag_weight_min','0.1','The minimum weight to multiply a reader DB\'s base weight by once its lag hits dbs_reader_lag_secs_end');
INSERT INTO vars (name, value, description) VALUES ('dbs_reader_weight_reduce_max','2.0','The maximum number of units per minute to reduce weight down to the minimum');
INSERT INTO vars (name, value, description) VALUES ('dbs_reader_weight_increase_max','1.0','The maximum number of units per minute to restore weight back up to 1');
INSERT INTO vars (name, value, description) VALUES ('dbs_revive_seconds','30','After a DB goes from isalive=no to yes, ramp up accesses to it over how many seconds?');
INSERT INTO vars (name, value, description) VALUES ('debug_db_cache','0','If set, then write debug info for the Slash::DB cache to STDERR');
INSERT INTO vars (name, value, description) VALUES ('debug_maintable_border','0','Border on the main table (for debugging purposes)');
Expand Down
5 changes: 3 additions & 2 deletions sql/mysql/slashschema_create.sql
View file
Expand Up @@ -343,8 +343,9 @@ CREATE TABLE dbs_readerstatus (
slave_lag_secs float DEFAULT '0',
query_bog_secs float DEFAULT '0',
bog_rsqid mediumint UNSIGNED DEFAULT NULL,
KEY ts_dbid (ts, dbid),
KEY ts_bog (ts, bog_rsqid, query_bog_secs)
had_weight tinyint UNSIGNED NOT NULL DEFAULT 1,
had_weight_adjust float UNSIGNED NOT NULL DEFAULT 1,
KEY ts_dbid (ts, dbid)
) TYPE=InnoDB;

#
Expand Down
12 changes: 12 additions & 0 deletions sql/mysql/upgrades
View file
Expand Up @@ -2769,3 +2769,15 @@ UPDATE vars SET value=300 WHERE name='freshenup_max_stories';

# End of T_2_5_0_38, Start of T_2_5_0_39 - 2004/11/23

ALTER TABLE dbs ADD UNIQUE type_vu (type, virtual_user);
ALTER TABLE dbs_readerstatus ADD COLUMN had_weight tinyint UNSIGNED NOT NULL DEFAULT 1, ADD COLUMN had_weight_adjust float UNSIGNED NOT NULL default 1, DROP INDEX ts_bog;

INSERT INTO vars (name, value, description) VALUES ('dbs_reader_bog_secs_start','5','Number of seconds of reader DB bog at which balance_readers.pl should start to reduce its weight');
INSERT INTO vars (name, value, description) VALUES ('dbs_reader_bog_secs_end','60','Number of seconds of reader DB bog at which balance_readers.pl hits the minimum weight');
INSERT INTO vars (name, value, description) VALUES ('dbs_reader_bog_weight_min','0.2','The minimum weight to multiply a reader DB\'s base weight by once its bog hits dbs_reader_bog_secs_end');
INSERT INTO vars (name, value, description) VALUES ('dbs_reader_lag_secs_start','5','Number of seconds of reader DB lag at which balance_readers.pl should start to reduce its weight');
INSERT INTO vars (name, value, description) VALUES ('dbs_reader_lag_secs_end','30','Number of seconds of reader DB lag at which balance_readers.pl hits the minimum weight');
INSERT INTO vars (name, value, description) VALUES ('dbs_reader_lag_weight_min','0.1','The minimum weight to multiply a reader DB\'s base weight by once its lag hits dbs_reader_lag_secs_end');
INSERT INTO vars (name, value, description) VALUES ('dbs_reader_weight_reduce_max','2.0','The maximum number of units per minute to reduce weight down to the minimum');
INSERT INTO vars (name, value, description) VALUES ('dbs_reader_weight_increase_max','1.0','The maximum number of units per minute to restore weight back up to 1');

99 changes: 96 additions & 3 deletions themes/slashcode/tasks/balance_readers.pl
View file
Expand Up @@ -95,8 +95,11 @@ sub check_readers {
my $reader_info = { };

# Weed out readers that are isalive='no'
my $vu_alive_hr = $slashdb->sqlSelectAllKeyValue(
"virtual_user, IF(isalive='yes',1,0)", "dbs", "type='reader'");
my $vu_hr = $slashdb->sqlSelectAllHashref(
"virtual_user",
"virtual_user, IF(isalive='yes',1,0) AS isalive, weight, weight_adjust",
"dbs", "type='reader'");
my $vu_alive_hr = {( map {( $_, $vu_hr->{$_}{isalive} )} keys %$vu_hr )};
my @alive_vus = grep { $vu_alive_hr->{$_} } sort keys %$readers;

# Note dead readers so they can be marked as dead when we return.
Expand All @@ -110,6 +113,10 @@ sub check_readers {
my %slave_sql_id = ( );
for my $vu (@alive_vus) {

# Copy across the data from the dbs table.
$reader_info->{$vu}{had_weight} = $vu_hr->{$vu}{weight};
$reader_info->{$vu}{had_weight_adjust} = $vu_hr->{$vu}{weight_adjust};

# Connect to this reader (this will also attempt to ping it
# and make sure it's reachable). If we can't, mark it as
# unreachable.
Expand Down Expand Up @@ -330,6 +337,8 @@ sub log_reader_info {
slave_lag_secs => $info->{slave_lag_secs},
query_bog_secs => $info->{query_bog_secs},
bog_rsqid => $bog_rsqid,
had_weight => $info->{had_weight},
had_weight_adjust => $info->{had_weight_adjust},
};
$slashdb->createDBReaderStatus($log_hr);
}
Expand All @@ -338,7 +347,91 @@ sub log_reader_info {

sub adjust_readers {
my($slashdb, $reader_info) = @_;
# Not coded yet.

my $constants = getCurrentStatic();
my $reduce_max = $constants->{dbs_reader_weight_reduce_max};
my $increase_max = $constants->{dbs_reader_weight_increase_max};
my $weight_adjust = { };

VU: for my $vu (sort keys %$reader_info) {

# If this DB is marked as isalive='no', it's not our
# responsibility, skip it.
next VU unless $reader_info->{was_alive};

# If this DB was not reachable, set its weight to 0
# immediately.
if (!$reader_info->{was_reachable}) {
set_reader_weight_adjust($slashdb, $vu, 0);
next VU;
}

# Get how lagged this slave DB's slave process is.
# If we did indeed reach the reader DB and found that
# its slave process was not running, that counts as a
# very high lag (which it will be, shortly, unless
# the slave process gets restarted!).
my $lag = $reader_info->{$vu}{slave_lag_secs};
$lag = 9999 if !$reader_info->{$vu}{was_running};

# Get the weight_adjust fraction for lag.
my $wa_lag = get_adjust_fraction($lag,
$constants->{dbs_reader_lag_secs_start},
$constants->{dbs_reader_lag_secs_end},
$constants->{dbs_reader_lag_weight_min});

# Get how bogged the DB is.
my $bog = $reader_info->{$vu}{query_bog_secs};

# Get the weight_adjust fraction for bog.
my $wa_bog = get_adjust_fraction($bog,
$constants->{dbs_reader_bog_secs_start},
$constants->{dbs_reader_bog_secs_end},
$constants->{dbs_reader_bog_weight_min});

# Decide what the total fraction we're going to use is.
# (For now, let's just do the min of those two. We may
# want to combine them in a synergistic way later or
# something.)
my $wa_total = $wa_lag;
$wa_total = $wa_bog if $wa_bog < $wa_total;
set_reader_weight_adjust($slashdb, $vu, $wa_total);
}
}

sub get_adjust_fraction {
my($secs, $secs_start, $secs_end, $weight_min) = @_;
if ($secs < $secs_start) {
# This one's as good as it gets :)
return 1;
}
if ($secs >= $secs_end) {
# This one's as bad as it gets :(
return $weight_min;
}
# This one's somewhere in the middle. Scale
# linearly to find out where.
return 1 - ($secs-$secs_start)*(1-$weight_min)/($secs_end-$secs_start);
}

# Set the weight for this virtual user to the new value given --
# but, do not increase it or reduce it more than the amount given,
# nor make it larger than 1 or smaller than 0. We do this with
# some clever SQL.
sub set_reader_weight_adjust {
my($slashdb, $vu, $new_wa) = @_;

my $dbid = get_reader_dbid($vu);
my $constants = getCurrentStatic();
my $delay = $constants->{dbs_reader_adjust_delay} || 5;
my $reduce_max = $constants->{dbs_reader_weight_reduce_max}*$delay/60;
my $increase_max = $constants->{dbs_reader_weight_increase_max}*$delay/60;

$slashdb->sqlUpdate("dbs",
{ -weight_adjust => "GREATEST(0, weight_adjust-$reduce_max,
LEAST(1, weight_adjust+$increase_max,
$new_wa))" },
"dbid=$dbid");
}

sub delete_old_logs {
Expand Down
24 changes: 23 additions & 1 deletion themes/slashcode/tasks/ircslash.pl
View file
Expand Up @@ -201,6 +201,7 @@ sub getIRCData {
whois => \&cmd_whois,
daddypants => \&cmd_daddypants,
slashd => \&cmd_slashd,
dbs => \&cmd_dbs,
quote => \&cmd_quote,
);
sub handle_cmd {
Expand Down Expand Up @@ -398,7 +399,6 @@ sub cmd_quote {
# Generate and emit the response.
my $response = getIRCData('quote_response', { stock => \%stock });
$self->privmsg($channel, $response);
slashdLog("stock: " . Dumper(\%stock));
slashdLog("quote: $response, cmd from $info->{event}{nick}");
}
} # end closure
Expand Down Expand Up @@ -453,6 +453,28 @@ sub check_slashd {
return (!$ok, $response);
}

sub cmd_dbs {
my($self, $info) = @_;
my $slashdb = getCurrentDB();
my $dbs = $slashdb->getDBs();
my $dbs_data = $slashdb->getDBsReaderStatus(60);
my $response;
if (%$dbs_data) {
for my $dbid (keys %$dbs_data) {
$dbs_data->{$dbid}{virtual_user} = $dbs->{$dbid}{virtual_user};
$dbs_data->{$dbid}{lag} = sprintf("%.1f", $dbs_data->{$dbid}{lag} || 0);
$dbs_data->{$dbid}{bog} = sprintf("%.1f", $dbs_data->{$dbid}{bog} || 0);
}
my @dbids =
sort { $dbs->{$a}{virtual_user} cmp $dbs->{$b}{virtual_user} }
keys %$dbs_data;
$response = getIRCData('dbs_response', { dbids => \@dbids, dbs => $dbs_data });
} else {
$response = getIRCData('dbs_nodata');
}
$self->privmsg($channel, $response);
}

############################################################

sub handle_remarks {
Expand Down
14 changes: 13 additions & 1 deletion themes/slashcode/templates/data;ircslash;default
View file
Expand Up @@ -20,7 +20,7 @@ __template__
[% SWITCH value %]

[% CASE 'help' %]
Commands: help, [un]hush, [un]ignore, slashd, ping, who, whois (uid|nick)[% IF plugins.daddypants %], daddypants [now|next|today|tomorrow|n days][% END %]
Commands: help, [un]hush, [un]ignore, slashd, ping, who, whois (uid|nick), quote (symbol), dbs, [% IF plugins.daddypants %], daddypants [now|next|today|tomorrow|n days][% END %]

[% CASE 'exiting' %]
Exiting (slashd should restart me within a minute)
Expand Down Expand Up @@ -62,6 +62,18 @@ __template__
Price for [% stock.name %] at [% stock.time %]: [% stock.last %] ([% stock.net %] from last close/open [% stock.close %]/[% stock.open %]) -- Today's lo-hi [% stock.low %]-[% stock.high %] -- Year lo-hi [% stock.year_low %]-[% stock.year_high %]
[% END %]

[% CASE 'dbs_response' %]
[% FOREACH dbid = dbids;
dbs.$dbid.virtual_user;
" alive="; dbs.$dbid.was_alive;
" lag="; dbs.$dbid.lag;
" bog="; dbs.$dbid.bog;
UNLESS loop.last(); " -- "; END;
END %]

[% CASE 'dbs_nodata' %]
Error: no dbs_readerstatus data found within the last minute.

[% CASE DEFAULT %]
-- MISSING USER MESSAGE SEGMENT -- [[% value %]] block not found.

Expand Down

0 comments on commit 8ebdfc9

Please sign in to comment.