From 8ebdfc9d6895fe611667c3627247acd07ded9152 Mon Sep 17 00:00:00 2001 From: Jamie McCarthy Date: Wed, 24 Nov 2004 06:31:14 +0000 Subject: [PATCH] Write weight adjustments into dbs table according to how lagged/bogged each reader DB is. Log more info into dbs_readerstatus table. Add "dbs" comment to irc bot. --- Slash/DB/Static/MySQL/MySQL.pm | 16 +++ sql/mysql/defaults.sql | 8 ++ sql/mysql/slashschema_create.sql | 5 +- sql/mysql/upgrades | 12 +++ themes/slashcode/tasks/balance_readers.pl | 99 ++++++++++++++++++- themes/slashcode/tasks/ircslash.pl | 24 ++++- .../slashcode/templates/data;ircslash;default | 14 ++- 7 files changed, 171 insertions(+), 7 deletions(-) diff --git a/Slash/DB/Static/MySQL/MySQL.pm b/Slash/DB/Static/MySQL/MySQL.pm index 4d7c47ca8..9c9e41de7 100644 --- a/Slash/DB/Static/MySQL/MySQL.pm +++ b/Slash/DB/Static/MySQL/MySQL.pm @@ -232,6 +232,22 @@ sub deleteOldDBReaderStatus { "ts < DATE_SUB(NOW(), INTERVAL $secs_back SECOND)"); } +######################################################## +# For ircslash.pl +sub getDBsReaderStatus { + my($self, $secs_back) = @_; + $secs_back ||= 60; + return $self->sqlSelectAllHashref( + "dbid", + "dbid, + MIN(IF(was_alive='yes',1,0)) AS was_alive, + AVG(slave_lag_secs) AS lag, + AVG(query_bog_secs) AS bog", + "dbs_readerstatus", + "ts >= DATE_SUB(NOW(), INTERVAL $secs_back SECOND)", + "GROUP BY dbid"); +} + ######################################################## # For dailystuff sub deleteRecycledComments { diff --git a/sql/mysql/defaults.sql b/sql/mysql/defaults.sql index f447125b1..59b160f2d 100644 --- a/sql/mysql/defaults.sql +++ b/sql/mysql/defaults.sql @@ -710,7 +710,15 @@ INSERT INTO vars (name, value, description) VALUES ('cur_performance_stats_weeks INSERT INTO vars (name, value, description) VALUES ('currentqid',1,'The Current Question on the homepage pollbooth'); INSERT INTO vars (name, value, description) VALUES ('datadir','/usr/local/slash/www.example.com','What is the root of the install for Slash'); INSERT INTO vars (name, value, description) VALUES ('dbs_reader_adjust_delay','5','Number of seconds between each adjustment of reader DB weights'); +INSERT INTO vars (name, value, description) VALUES ('dbs_reader_bog_secs_start','5','Number of seconds of reader DB bog at which balance_readers.pl should start to reduce its weight'); +INSERT INTO vars (name, value, description) VALUES ('dbs_reader_bog_secs_end','60','Number of seconds of reader DB bog at which balance_readers.pl hits the minimum weight'); +INSERT INTO vars (name, value, description) VALUES ('dbs_reader_bog_weight_min','0.2','The minimum weight to multiply a reader DB\'s base weight by once its bog hits dbs_reader_bog_secs_end'); INSERT INTO vars (name, value, description) VALUES ('dbs_reader_expire_secs', 86400 * 7,'Number of seconds worth of dbs_readerstatus log to keep around'); +INSERT INTO vars (name, value, description) VALUES ('dbs_reader_lag_secs_start','5','Number of seconds of reader DB lag at which balance_readers.pl should start to reduce its weight'); +INSERT INTO vars (name, value, description) VALUES ('dbs_reader_lag_secs_end','30','Number of seconds of reader DB lag at which balance_readers.pl hits the minimum weight'); +INSERT INTO vars (name, value, description) VALUES ('dbs_reader_lag_weight_min','0.1','The minimum weight to multiply a reader DB\'s base weight by once its lag hits dbs_reader_lag_secs_end'); +INSERT INTO vars (name, value, description) VALUES ('dbs_reader_weight_reduce_max','2.0','The maximum number of units per minute to reduce weight down to the minimum'); +INSERT INTO vars (name, value, description) VALUES ('dbs_reader_weight_increase_max','1.0','The maximum number of units per minute to restore weight back up to 1'); INSERT INTO vars (name, value, description) VALUES ('dbs_revive_seconds','30','After a DB goes from isalive=no to yes, ramp up accesses to it over how many seconds?'); INSERT INTO vars (name, value, description) VALUES ('debug_db_cache','0','If set, then write debug info for the Slash::DB cache to STDERR'); INSERT INTO vars (name, value, description) VALUES ('debug_maintable_border','0','Border on the main table (for debugging purposes)'); diff --git a/sql/mysql/slashschema_create.sql b/sql/mysql/slashschema_create.sql index 135eece51..4898b1987 100644 --- a/sql/mysql/slashschema_create.sql +++ b/sql/mysql/slashschema_create.sql @@ -343,8 +343,9 @@ CREATE TABLE dbs_readerstatus ( slave_lag_secs float DEFAULT '0', query_bog_secs float DEFAULT '0', bog_rsqid mediumint UNSIGNED DEFAULT NULL, - KEY ts_dbid (ts, dbid), - KEY ts_bog (ts, bog_rsqid, query_bog_secs) + had_weight tinyint UNSIGNED NOT NULL DEFAULT 1, + had_weight_adjust float UNSIGNED NOT NULL DEFAULT 1, + KEY ts_dbid (ts, dbid) ) TYPE=InnoDB; # diff --git a/sql/mysql/upgrades b/sql/mysql/upgrades index 0bb840414..bd05e8c9a 100644 --- a/sql/mysql/upgrades +++ b/sql/mysql/upgrades @@ -2769,3 +2769,15 @@ UPDATE vars SET value=300 WHERE name='freshenup_max_stories'; # End of T_2_5_0_38, Start of T_2_5_0_39 - 2004/11/23 +ALTER TABLE dbs ADD UNIQUE type_vu (type, virtual_user); +ALTER TABLE dbs_readerstatus ADD COLUMN had_weight tinyint UNSIGNED NOT NULL DEFAULT 1, ADD COLUMN had_weight_adjust float UNSIGNED NOT NULL default 1, DROP INDEX ts_bog; + +INSERT INTO vars (name, value, description) VALUES ('dbs_reader_bog_secs_start','5','Number of seconds of reader DB bog at which balance_readers.pl should start to reduce its weight'); +INSERT INTO vars (name, value, description) VALUES ('dbs_reader_bog_secs_end','60','Number of seconds of reader DB bog at which balance_readers.pl hits the minimum weight'); +INSERT INTO vars (name, value, description) VALUES ('dbs_reader_bog_weight_min','0.2','The minimum weight to multiply a reader DB\'s base weight by once its bog hits dbs_reader_bog_secs_end'); +INSERT INTO vars (name, value, description) VALUES ('dbs_reader_lag_secs_start','5','Number of seconds of reader DB lag at which balance_readers.pl should start to reduce its weight'); +INSERT INTO vars (name, value, description) VALUES ('dbs_reader_lag_secs_end','30','Number of seconds of reader DB lag at which balance_readers.pl hits the minimum weight'); +INSERT INTO vars (name, value, description) VALUES ('dbs_reader_lag_weight_min','0.1','The minimum weight to multiply a reader DB\'s base weight by once its lag hits dbs_reader_lag_secs_end'); +INSERT INTO vars (name, value, description) VALUES ('dbs_reader_weight_reduce_max','2.0','The maximum number of units per minute to reduce weight down to the minimum'); +INSERT INTO vars (name, value, description) VALUES ('dbs_reader_weight_increase_max','1.0','The maximum number of units per minute to restore weight back up to 1'); + diff --git a/themes/slashcode/tasks/balance_readers.pl b/themes/slashcode/tasks/balance_readers.pl index 98af70788..fd13313e2 100644 --- a/themes/slashcode/tasks/balance_readers.pl +++ b/themes/slashcode/tasks/balance_readers.pl @@ -95,8 +95,11 @@ sub check_readers { my $reader_info = { }; # Weed out readers that are isalive='no' - my $vu_alive_hr = $slashdb->sqlSelectAllKeyValue( - "virtual_user, IF(isalive='yes',1,0)", "dbs", "type='reader'"); + my $vu_hr = $slashdb->sqlSelectAllHashref( + "virtual_user", + "virtual_user, IF(isalive='yes',1,0) AS isalive, weight, weight_adjust", + "dbs", "type='reader'"); + my $vu_alive_hr = {( map {( $_, $vu_hr->{$_}{isalive} )} keys %$vu_hr )}; my @alive_vus = grep { $vu_alive_hr->{$_} } sort keys %$readers; # Note dead readers so they can be marked as dead when we return. @@ -110,6 +113,10 @@ sub check_readers { my %slave_sql_id = ( ); for my $vu (@alive_vus) { + # Copy across the data from the dbs table. + $reader_info->{$vu}{had_weight} = $vu_hr->{$vu}{weight}; + $reader_info->{$vu}{had_weight_adjust} = $vu_hr->{$vu}{weight_adjust}; + # Connect to this reader (this will also attempt to ping it # and make sure it's reachable). If we can't, mark it as # unreachable. @@ -330,6 +337,8 @@ sub log_reader_info { slave_lag_secs => $info->{slave_lag_secs}, query_bog_secs => $info->{query_bog_secs}, bog_rsqid => $bog_rsqid, + had_weight => $info->{had_weight}, + had_weight_adjust => $info->{had_weight_adjust}, }; $slashdb->createDBReaderStatus($log_hr); } @@ -338,7 +347,91 @@ sub log_reader_info { sub adjust_readers { my($slashdb, $reader_info) = @_; - # Not coded yet. + + my $constants = getCurrentStatic(); + my $reduce_max = $constants->{dbs_reader_weight_reduce_max}; + my $increase_max = $constants->{dbs_reader_weight_increase_max}; + my $weight_adjust = { }; + + VU: for my $vu (sort keys %$reader_info) { + + # If this DB is marked as isalive='no', it's not our + # responsibility, skip it. + next VU unless $reader_info->{was_alive}; + + # If this DB was not reachable, set its weight to 0 + # immediately. + if (!$reader_info->{was_reachable}) { + set_reader_weight_adjust($slashdb, $vu, 0); + next VU; + } + + # Get how lagged this slave DB's slave process is. + # If we did indeed reach the reader DB and found that + # its slave process was not running, that counts as a + # very high lag (which it will be, shortly, unless + # the slave process gets restarted!). + my $lag = $reader_info->{$vu}{slave_lag_secs}; + $lag = 9999 if !$reader_info->{$vu}{was_running}; + + # Get the weight_adjust fraction for lag. + my $wa_lag = get_adjust_fraction($lag, + $constants->{dbs_reader_lag_secs_start}, + $constants->{dbs_reader_lag_secs_end}, + $constants->{dbs_reader_lag_weight_min}); + + # Get how bogged the DB is. + my $bog = $reader_info->{$vu}{query_bog_secs}; + + # Get the weight_adjust fraction for bog. + my $wa_bog = get_adjust_fraction($bog, + $constants->{dbs_reader_bog_secs_start}, + $constants->{dbs_reader_bog_secs_end}, + $constants->{dbs_reader_bog_weight_min}); + + # Decide what the total fraction we're going to use is. + # (For now, let's just do the min of those two. We may + # want to combine them in a synergistic way later or + # something.) + my $wa_total = $wa_lag; + $wa_total = $wa_bog if $wa_bog < $wa_total; + set_reader_weight_adjust($slashdb, $vu, $wa_total); + } +} + +sub get_adjust_fraction { + my($secs, $secs_start, $secs_end, $weight_min) = @_; + if ($secs < $secs_start) { + # This one's as good as it gets :) + return 1; + } + if ($secs >= $secs_end) { + # This one's as bad as it gets :( + return $weight_min; + } + # This one's somewhere in the middle. Scale + # linearly to find out where. + return 1 - ($secs-$secs_start)*(1-$weight_min)/($secs_end-$secs_start); +} + +# Set the weight for this virtual user to the new value given -- +# but, do not increase it or reduce it more than the amount given, +# nor make it larger than 1 or smaller than 0. We do this with +# some clever SQL. +sub set_reader_weight_adjust { + my($slashdb, $vu, $new_wa) = @_; + + my $dbid = get_reader_dbid($vu); + my $constants = getCurrentStatic(); + my $delay = $constants->{dbs_reader_adjust_delay} || 5; + my $reduce_max = $constants->{dbs_reader_weight_reduce_max}*$delay/60; + my $increase_max = $constants->{dbs_reader_weight_increase_max}*$delay/60; + + $slashdb->sqlUpdate("dbs", + { -weight_adjust => "GREATEST(0, weight_adjust-$reduce_max, + LEAST(1, weight_adjust+$increase_max, + $new_wa))" }, + "dbid=$dbid"); } sub delete_old_logs { diff --git a/themes/slashcode/tasks/ircslash.pl b/themes/slashcode/tasks/ircslash.pl index 5a9dcda05..e85b5ab19 100644 --- a/themes/slashcode/tasks/ircslash.pl +++ b/themes/slashcode/tasks/ircslash.pl @@ -201,6 +201,7 @@ sub getIRCData { whois => \&cmd_whois, daddypants => \&cmd_daddypants, slashd => \&cmd_slashd, + dbs => \&cmd_dbs, quote => \&cmd_quote, ); sub handle_cmd { @@ -398,7 +399,6 @@ sub cmd_quote { # Generate and emit the response. my $response = getIRCData('quote_response', { stock => \%stock }); $self->privmsg($channel, $response); - slashdLog("stock: " . Dumper(\%stock)); slashdLog("quote: $response, cmd from $info->{event}{nick}"); } } # end closure @@ -453,6 +453,28 @@ sub check_slashd { return (!$ok, $response); } +sub cmd_dbs { + my($self, $info) = @_; + my $slashdb = getCurrentDB(); + my $dbs = $slashdb->getDBs(); + my $dbs_data = $slashdb->getDBsReaderStatus(60); + my $response; + if (%$dbs_data) { + for my $dbid (keys %$dbs_data) { + $dbs_data->{$dbid}{virtual_user} = $dbs->{$dbid}{virtual_user}; + $dbs_data->{$dbid}{lag} = sprintf("%.1f", $dbs_data->{$dbid}{lag} || 0); + $dbs_data->{$dbid}{bog} = sprintf("%.1f", $dbs_data->{$dbid}{bog} || 0); + } + my @dbids = + sort { $dbs->{$a}{virtual_user} cmp $dbs->{$b}{virtual_user} } + keys %$dbs_data; + $response = getIRCData('dbs_response', { dbids => \@dbids, dbs => $dbs_data }); + } else { + $response = getIRCData('dbs_nodata'); + } + $self->privmsg($channel, $response); +} + ############################################################ sub handle_remarks { diff --git a/themes/slashcode/templates/data;ircslash;default b/themes/slashcode/templates/data;ircslash;default index 110d9f577..11720142c 100644 --- a/themes/slashcode/templates/data;ircslash;default +++ b/themes/slashcode/templates/data;ircslash;default @@ -20,7 +20,7 @@ __template__ [% SWITCH value %] [% CASE 'help' %] - Commands: help, [un]hush, [un]ignore, slashd, ping, who, whois (uid|nick)[% IF plugins.daddypants %], daddypants [now|next|today|tomorrow|n days][% END %] + Commands: help, [un]hush, [un]ignore, slashd, ping, who, whois (uid|nick), quote (symbol), dbs, [% IF plugins.daddypants %], daddypants [now|next|today|tomorrow|n days][% END %] [% CASE 'exiting' %] Exiting (slashd should restart me within a minute) @@ -62,6 +62,18 @@ __template__ Price for [% stock.name %] at [% stock.time %]: [% stock.last %] ([% stock.net %] from last close/open [% stock.close %]/[% stock.open %]) -- Today's lo-hi [% stock.low %]-[% stock.high %] -- Year lo-hi [% stock.year_low %]-[% stock.year_high %] [% END %] +[% CASE 'dbs_response' %] + [% FOREACH dbid = dbids; + dbs.$dbid.virtual_user; + " alive="; dbs.$dbid.was_alive; + " lag="; dbs.$dbid.lag; + " bog="; dbs.$dbid.bog; + UNLESS loop.last(); " -- "; END; + END %] + +[% CASE 'dbs_nodata' %] + Error: no dbs_readerstatus data found within the last minute. + [% CASE DEFAULT %] -- MISSING USER MESSAGE SEGMENT -- [[% value %]] block not found.