diff --git a/Henzell/Config.pm b/Henzell/Config.pm index 85765964..712d7c28 100644 --- a/Henzell/Config.pm +++ b/Henzell/Config.pm @@ -6,6 +6,8 @@ use warnings; use base 'Exporter'; use Henzell::ServerConfig; +use YAML::Any; +use Cwd; our @EXPORT_OK = qw/read get %CONFIG %CMD %CMDPATH %PUBLIC_CMD @LOGS @MILESTONES/; @@ -16,9 +18,6 @@ my %DEFAULT_CONFIG = (use_pm => 0, irc_port => 6667, lock_name => 'henzell', - milestones => 'config/def.stones', - logs => 'config/def.logs', - # Does the bot respond to SQL queries (default: NO) sql_queries => 0, # Does the bot store logfiles and milestones in @@ -58,113 +57,9 @@ sub get() { \%CONFIG } -sub abbrev_load() { - %ABBRMAP = Henzell::ServerConfig::server_abbreviations(); - for my $key (keys %CONFIG) { - if ($key =~ /^abbr\.(\w+)$/) { - $ABBRMAP{$1} = $CONFIG{$key}; - } - } -} - -sub abbrev_expand($) { - my $abbr = shift; - # Expand if possible, else leave unchanged: - $ABBRMAP{$abbr} || "\$$abbr" -} - -sub abbrev_expand_all($) { - my $text = shift; - return $text unless defined $text; - s/\$(\w+)/abbrev_expand($1)/ge for $text; - $text -} - -sub parse_header_field($$) { - my ($hash, $piece) = @_; - my $host = $CONFIG{host} || ''; - if ($piece =~ /^local:(\w+)$/) { - if ($1 eq $host) { - $$hash{local} = 1; - $$hash{src} = $1; - } - } elsif ($piece =~ /^remote:(\w+)(?::(.*))?$/) { - if ($host ne $1) { - $$hash{remote} = 1; - $$hash{src} = $1; - $$hash{url} = abbrev_expand_all($2); - } - } elsif ($piece eq 'alpha') { - $$hash{alpha} = 1; - } else { - die "Unknown header field: $piece\n"; - } -} - -sub log_header_hash($) { - my $header = shift; - s/^\[//, s/\]$// for $header; - - my %hash; - for my $piece (split(/;/, $header)) { - s/^\s+//, s/\s+$// for $piece; - parse_header_field(\%hash, $piece); - } - - # Has to be either local or remote: - return undef unless $hash{local} || $hash{remote}; - - \%hash -} - -sub resolve_log_path($) { - my $path = shift; - return $path if $path =~ m{/}; - "server-xlogs/$path" -} - -sub log_path($$) { - my ($header, $path) = @_; - - my $log = log_header_hash($header); - return undef unless $log; - $$log{path} = resolve_log_path($path); - $log -} - -sub load_log_paths($$$) { - my ($paths, $file, $name) = @_; - - @$paths = (); - - open my $inf, '<', $file or return; - - my $header; - - while (<$inf>) { - chomp; - next unless /\S/ && !/^\s*#/; - - s/^\s+//; s/\s+$//; - - if (/^\[.*\]$/) { - $header = $_; - next; - } - - if ($header) { - my $path = $_; - my $logpath = log_path($header, $path); - push @$paths, $logpath if $logpath; - } - } - close $inf; -} - sub load_file_paths() { - abbrev_load(); - load_log_paths(\@LOGS, $CONFIG{logs}, "logfiles"); - load_log_paths(\@MILESTONES, $CONFIG{milestones}, "milestones"); + @LOGS = Henzell::ServerConfig::server_logfiles(); + @MILESTONES = Henzell::ServerConfig::server_milestones(); } sub load_public_commands($) { @@ -209,6 +104,7 @@ sub load_commands($$) { } sub setup_env() { + $ENV{HENZELL_ROOT} = getcwd(); if ($CONFIG{host}) { $ENV{HENZELL_HOST} = $CONFIG{host}; } else { @@ -230,18 +126,9 @@ sub read { %CONFIG = %DEFAULT_CONFIG; my $inf; - open $inf, '<', ($config || $CONFIG_FILE) or undef $inf; - if ($inf) { - while (<$inf>) { - s/^\s+//; s/\s+$//; - next unless /\S/; - if (/^([\w.]+)\s*=\s*(.*)/) { - $CONFIG{$1} = $2; - } - } - close $inf; - } + my $config_file = $config || $CONFIG_FILE; + %CONFIG = (%DEFAULT_CONFIG, %{YAML::Any::LoadFile($config_file)}); # So that users don't have to check for undef warnings. $CONFIG{host} ||= ''; diff --git a/Henzell/ServerConfig.pm b/Henzell/ServerConfig.pm index 91691d0f..d3cfc60e 100644 --- a/Henzell/ServerConfig.pm +++ b/Henzell/ServerConfig.pm @@ -6,9 +6,26 @@ package Henzell::ServerConfig; use base 'Exporter'; use YAML::Any qw/LoadFile/; -my $SERVER_CONFIG_FILE = 'config/servers.yml'; +use Henzell::SourceServer; + +my $SERVER_CONFIG_FILE = 'config/sources.yml'; my $SERVERCFG = LoadFile($SERVER_CONFIG_FILE); +my @SERVERS = map(Henzell::SourceServer->new($_), + @{$SERVERCFG->{sources}}); + +sub servers { + @SERVERS +} + +sub server_logfiles { + map($_->logfiles(), servers()) +} + +sub server_milestones { + map($_->milestones(), servers()) +} + sub source_hostname { my $source = shift; $$SERVERCFG{sources}{$source} diff --git a/Henzell/SourceServer.pm b/Henzell/SourceServer.pm new file mode 100644 index 00000000..30b4fdf1 --- /dev/null +++ b/Henzell/SourceServer.pm @@ -0,0 +1,79 @@ +# Object for a single server (CAO, etc.) and its logfiles / milestones +package Henzell::SourceServer; + +use strict; +use warnings; + +use Henzell::XlogSrc; + +sub new { + my ($cls, $yaml_config) = @_; + my $self = bless { _config => $yaml_config }, $cls; + $self->initialize(); + $self +} + +sub initialize { + my $self = shift; + $self->{logfiles} = [map { + Henzell::XlogSrc->new($_, 1, $self) + } $self->_expand_files('logfiles')]; + $self->{milestones} = [map { + Henzell::XlogSrc->new($_, 0, $self) + } $self->_expand_files('milestones')]; + my %duplicate; + for my $source (@{$self->{logfiles}}, @{$self->{milestones}}) { + my $cname = $source->canonical_name(); + if ($duplicate{$cname}) { + die("xlogfile collision: " . $source->source_name() . " and " . + $duplicate{$cname}->source_name() . " have same canonical name: " . + $cname); + } + $duplicate{$cname} = $source; + } +} + +sub name { + my $self = shift; + $self->{name} ||= $self->{_config}{name} +} + +sub local_base { + my $self = shift; + $self->{local_base} ||= $self->{_config}{local} +} + +sub http_base { + my $self = shift; + $self->{http_base} ||= $self->{_config}{base} +} + +sub logfiles { + my $self = shift; + @{$self->{logfiles}} +} + +sub milestones { + my $self = shift; + @{$self->{milestones}} +} + +sub split_files { + my $file = shift; + if (ref($file) eq 'HASH') { + my $key = (keys %$file)[0]; + my $tag = $file->{$key}; + return map(+{ $_ => $tag }, split_files($key)); + } + my $asterisked = $file =~ /\*/; + my $suffix = $asterisked ? '*' : ''; + $file =~ s/\*//g; + map($_ . $suffix, glob($file)) +} + +sub _expand_files { + my ($self, $key) = @_; + map(split_files($_), @{$self->{_config}{$key}}) +} + +1 diff --git a/Henzell/Utils.pm b/Henzell/Utils.pm index 2f22959d..f9d487ee 100644 --- a/Henzell/Utils.pm +++ b/Henzell/Utils.pm @@ -10,12 +10,35 @@ use POSIX; our @EXPORT = qw/tailed_handle/; our @EXPORT_OK = qw/lock_or_die lock daemonify/; +sub service_def { + my $service_def = shift; + die "Bad service def: must have name, command\n" unless ref($service_def) eq 'HASH'; + die "Service def without command\n" unless $service_def->{command}; + die "Service def without name\n" unless $service_def->{name}; + $service_def +} + +sub spawn_services { + my $service_defs = shift; + return unless $service_defs; + for my $service_def (@$service_defs) { + my $service = service_def($service_def); + spawn_service($service->{name}, $service->{directory}, $service->{command}); + } +} + sub spawn_service { - my ($service_name, $commandline) = @_; + my ($service_name, $directory, $commandline) = @_; + $service_name =~ tr/a-zA-Z 0-9//cd; + $service_name =~ tr/ /_/; + $service_name = lc $service_name; + print "Spawning service $service_name in $directory: $commandline\n"; my $pid = fork; return if $pid; - print "Spawning service $service_name: $commandline\n"; + if ($directory) { + chdir $directory or die "Couldn't cd $directory: $!\n"; + } my $log_file = "$service_name.log"; open my $logf, '>', $log_file or die "Can't write $log_file: $!\n"; $logf->autoflush; diff --git a/Henzell/XlogSrc.pm b/Henzell/XlogSrc.pm new file mode 100644 index 00000000..6b46f6a4 --- /dev/null +++ b/Henzell/XlogSrc.pm @@ -0,0 +1,179 @@ +package Henzell::XlogSrc; + +use strict; +use warnings; + +use File::Path; +use File::Spec; + +our $TARGET_BASE = 'server-xlogs'; + +use overload fallback => 1, '""' => sub { + my $xlog = shift; + $xlog->canonical_name() +}; + +sub new { + my ($cls, $name, $logfile, $server) = @_; + + my $tag; + if (ref($name) eq 'HASH') { + my $real_name = (keys %$name)[0]; + $tag = $name->{$real_name}; + $name = $real_name; + } + + bless { name => $name, + tag => $tag, + logfile => $logfile, + server => $server }, $cls +} + +sub canonical_name { + my $self = shift; + $self->{canonical_name} ||= $self->_make_canonical_name() +} + +sub canonical_version { + my $self = shift; + $self->{canonical_version} ||= $self->_find_canonical_version() +} + +sub source_name { + my $self = shift; + my $name = $self->{name}; + $name =~ tr/*//d; + $name +} + +sub tag { + shift()->{tag} +} + +sub alpha { + my $self = shift; + $self->canonical_version() eq 'git' +} + +sub server { + my $self = shift; + $self->{server} +} + +sub type { + my $self = shift; + $self->{logfile} ? 'logfile' : 'milestones' +} + +sub game_type { + my $self = shift; + my $name = $self->source_name(); + return 'sprint' if $name =~ /\bspr/i; + return 'zotdef' if $name =~ /\b(zd|zotdef)\b/i; + '' +} + +sub server_name { + my $self = shift; + $self->server()->name() +} + +sub http_url { + my $self = shift; + $self->{http_url} ||= $self->_find_http_url() +} + +sub local_filepath { + my $self = shift; + $self->{local_filepath} ||= $self->_find_local_filepath() +} + +sub local_source_exists { + my $self = shift; + $self->local_filepath() && -f($self->local_filepath()) +} + +sub target_exists { + my $self = shift; + -f($self->target_filepath()) +} + +# The (overwriteable) place to store downloaded files, or symlink local files. +# The parent directory is guaranteed to exist. +sub target_filepath { + my $self = shift; + File::Path::make_path($TARGET_BASE); + File::Spec->catfile($TARGET_BASE, $self->canonical_name()) +} + +# Returns true if we can expect this file to be actively updated on the server +sub is_live { + my $self = shift; + $self->{name} =~ /\*/ +} + +sub http_base { + my $self = shift; + $self->server()->http_base() +} + +sub local_base { + my $self = shift; + $self->server()->local_base() +} + +sub _find_http_url { + my $self = shift; + my $http_base = $self->http_base(); + return undef unless $http_base; + "$http_base/" . $self->source_name() +} + +sub _find_local_filepath { + my $self = shift; + my $local_base = $self->local_base(); + return undef unless $local_base; + File::Spec->catfile($local_base, $self->source_name()) +} + +sub _make_canonical_name { + my $self = shift; + my $canonical_version = $self->canonical_version(); + my $game_type = $self->game_type(); + my $tag = $self->tag(); + my $qualifier = $game_type ? "-$game_type" : ""; + $qualifier .= "-$tag" if $tag; + ("remote." . $self->server_name() . "-" . $self->type() . "-" . + $canonical_version . $qualifier) +} + +sub _canonicalize_version { + my ($version, $alpha) = @_; + return 'git' if $alpha && !$version; + if ($version) { + if ($version =~ /^\d{2}/) { + $version =~ s/^0+//; + return "0.$version"; + } + return $version if $version =~ /^\d+(?:[.]\d+)+$/; + if ($version =~ /^(\d)(\d+)$/) { + return "$1.$2"; + } + } + 'any' +} + +sub _find_canonical_version { + my $self = shift; + my $name = $self->source_name(); + my ($version) = $name =~ /(\d+(?:[.]\d+)+|\d{2,})/; + my $alpha = $name =~ /\b(?:git|svn|trunk)\b/; + my $canonical_version; + eval { + $canonical_version = _canonicalize_version($version, $alpha); + }; + die "Bad xlogfile: $name: $@" if $@; + $canonical_version +} + +1 diff --git a/commands/chars.rb b/commands/chars.rb index bfad90c4..8275229c 100755 --- a/commands/chars.rb +++ b/commands/chars.rb @@ -1,6 +1,5 @@ #! /usr/bin/env ruby -$:.push("src") require 'sqlhelper' require 'helper' diff --git a/commands/deathsby.rb b/commands/deathsby.rb index a86f2457..500bc1b1 100755 --- a/commands/deathsby.rb +++ b/commands/deathsby.rb @@ -1,6 +1,5 @@ #! /usr/bin/env ruby -$:.push("src") require 'helper' require 'sqlhelper' diff --git a/commands/deathsin.rb b/commands/deathsin.rb index 60904a4f..f9a904e1 100755 --- a/commands/deathsin.rb +++ b/commands/deathsin.rb @@ -1,6 +1,5 @@ #! /usr/bin/env ruby -$:.push("src") require 'sqlhelper' require 'helper' require 'query/query_string' diff --git a/commands/gamesby.rb b/commands/gamesby.rb index 52f0fe84..badb0ba7 100755 --- a/commands/gamesby.rb +++ b/commands/gamesby.rb @@ -1,6 +1,5 @@ #! /usr/bin/env ruby -$:.push("src") require 'helper.rb' require 'sqlhelper' require 'query/query_string' diff --git a/commands/gkills.rb b/commands/gkills.rb index 65036cd6..13038900 100755 --- a/commands/gkills.rb +++ b/commands/gkills.rb @@ -1,6 +1,5 @@ #! /usr/bin/env ruby -$:.push("src") require 'helper' require 'sqlhelper' require 'query/query_string' diff --git a/commands/helper.py b/commands/helper.py index 7566e1ec..850a691e 100644 --- a/commands/helper.py +++ b/commands/helper.py @@ -6,7 +6,7 @@ from datetime import datetime import time -CFGFILE = 'config/crawl-data.yml' +CFGFILE = os.path.join(os.environ['HENZELL_ROOT'], 'config/crawl-data.yml') CFG = yaml.load(open(CFGFILE).read()) www_rawdatapath = '/var/www/crawl/rawdata/' @@ -30,7 +30,7 @@ WHERE_DIRS = None NICK_ALIASES = { } -NICKMAP_FILE = 'dat/nicks.map' +NICKMAP_FILE = os.path.join(os.environ['HENZELL_ROOT'], 'dat/nicks.map') nick_aliases_loaded = False class Tournament (object): diff --git a/commands/hsn.rb b/commands/hsn.rb index 8f05d703..cdee22cb 100755 --- a/commands/hsn.rb +++ b/commands/hsn.rb @@ -1,6 +1,5 @@ #!/usr/bin/env ruby -$:.push("src") require 'helper' require 'sqlhelper' require 'libtv' diff --git a/commands/killsby.rb b/commands/killsby.rb index fef60ca6..bbef62b0 100755 --- a/commands/killsby.rb +++ b/commands/killsby.rb @@ -1,6 +1,5 @@ #! /usr/bin/env ruby -$:.push("src") require 'helper' require 'sqlhelper' require 'query/query_string' diff --git a/commands/lastlog.rb b/commands/lastlog.rb index ca54468c..b70aed1b 100755 --- a/commands/lastlog.rb +++ b/commands/lastlog.rb @@ -1,6 +1,5 @@ #!/usr/bin/env ruby -$:.push("src") require 'helper' require 'sqlhelper' diff --git a/commands/listgame.rb b/commands/listgame.rb index e5e652e3..230efb50 100755 --- a/commands/listgame.rb +++ b/commands/listgame.rb @@ -1,6 +1,5 @@ #!/usr/bin/env ruby -$:.push("src") require 'sqlhelper' require 'helper' require 'libtv' diff --git a/commands/milestone.rb b/commands/milestone.rb index 38a21c49..82e63daa 100755 --- a/commands/milestone.rb +++ b/commands/milestone.rb @@ -1,6 +1,5 @@ #! /usr/bin/env ruby -$:.push("src") require 'helper' require 'sqlhelper' require 'libtv' diff --git a/commands/nickmap.rb b/commands/nickmap.rb index ffa76cf5..424f79be 100755 --- a/commands/nickmap.rb +++ b/commands/nickmap.rb @@ -1,6 +1,5 @@ #! /usr/bin/env ruby -$:.push("src") require 'helper' require 'set' diff --git a/commands/send.rb b/commands/send.rb index d01a6e1f..84ba8dd8 100755 --- a/commands/send.rb +++ b/commands/send.rb @@ -1,6 +1,5 @@ #! /usr/bin/env ruby -$:.push("src") require 'helper' help("%CMD% : Will send to 's game (this is a campaign promise).") diff --git a/commands/streak.rb b/commands/streak.rb index 996efac0..11cf0e13 100755 --- a/commands/streak.rb +++ b/commands/streak.rb @@ -1,6 +1,5 @@ #! /usr/bin/env ruby -$:.push("src") require 'helper' require 'sqlhelper' require 'query/query_string' diff --git a/commands/ttr.rb b/commands/ttr.rb index 7f07e73c..2fe1cf11 100755 --- a/commands/ttr.rb +++ b/commands/ttr.rb @@ -1,6 +1,5 @@ #! /usr/bin/env ruby -$:.push("src") require 'helper' require 'sqlhelper' diff --git a/commands/tvdef.rb b/commands/tvdef.rb index 436e7a1b..d2d43b25 100755 --- a/commands/tvdef.rb +++ b/commands/tvdef.rb @@ -1,7 +1,5 @@ #! /usr/bin/env ruby -$:.push("src") - require 'helper' require 'libtv' require 'tv/channel_manager' diff --git a/commands/tvreq.rb b/commands/tvreq.rb index d3dbce5a..38b08e03 100755 --- a/commands/tvreq.rb +++ b/commands/tvreq.rb @@ -2,7 +2,6 @@ # Requests that C-SPLAT play the specified game. -$:.push("src") require 'helper' require 'sqlhelper' require 'libtv' diff --git a/commands/won.rb b/commands/won.rb index 8196b907..8be8070e 100755 --- a/commands/won.rb +++ b/commands/won.rb @@ -1,6 +1,5 @@ #!/usr/bin/env ruby -$:.push("src") require 'helper' require 'sqlhelper' require 'query/query_string' diff --git a/config.ru b/config.ru deleted file mode 100644 index 4a32eb49..00000000 --- a/config.ru +++ /dev/null @@ -1,4 +0,0 @@ -# -p 29880 - -require 'services/http_service' -run Sinatra::Application \ No newline at end of file diff --git a/config/def.logs b/config/def.logs index 8bf5a7b0..b2c48b3b 100644 --- a/config/def.logs +++ b/config/def.logs @@ -1,52 +1,3 @@ -[local:cao] -/var/www/crawl/allgames.txt - -[local:cao] -/home/crawl/chroot/var/games/crawl04/saves/logfile - -[local:cao] -/home/crawl/chroot/var/games/crawl05/saves/logfile - -[local:cao] -/home/crawl/chroot/var/games/crawl06/saves/logfile - -[local:cao] -/home/crawl/chroot/var/games/crawl07/saves/logfile - -[local:cao] -/home/crawl/chroot/var/games/crawl07/saves/logfile-sprint - -[local:cao] -/home/crawl/chroot/var/games/crawl08/saves/logfile - -[local:cao] -/home/crawl/chroot/var/games/crawl08/saves/logfile-sprint - -[local:cao] -/home/crawl/chroot/var/games/crawl09/saves/logfile - -[local:cao] -/home/crawl/chroot/var/games/crawl09/saves/logfile-sprint - -[local:cao] -/home/crawl/chroot/var/games/crawl10/saves/logfile - -[local:cao] -/home/crawl/chroot/var/games/crawl10/saves/logfile-sprint - -[local:cao] -/var/lib/dgamelaunch/crawl-master/crawl-git/saves/logfile - -[local:cao] -/var/lib/dgamelaunch/crawl-master/crawl-git/saves/logfile-sprint - -[local:cao] -/var/lib/dgamelaunch/crawl-master/crawl-git/saves/logfile-zotdef - -[local:cao] -/var/lib/dgamelaunch/var/games/crawl-lorcs/saves/logfile - - # http://$cao/allgames.txt [remote:cao] remote.cao-logfile-0.3 diff --git a/config/def.stones b/config/def.stones index 2ca3fd69..607b1cf8 100644 --- a/config/def.stones +++ b/config/def.stones @@ -1,56 +1,3 @@ -[local:cao] -/var/www/crawl/milestones02.txt - -[local:cao] -/home/crawl/chroot/var/games/crawl03/saves/milestones.txt - -[local:cao] -/home/crawl/chroot/var/games/crawl04/saves/milestones.txt - -[local:cao] -/home/crawl/chroot/var/games/crawl05/saves/milestones.txt - -[local:cao] -/home/crawl/chroot/var/games/crawl06/saves/milestones.txt - -[local:cao] -/home/crawl/chroot/var/games/crawl07/saves/milestones - -[local:cao] -/home/crawl/chroot/var/games/crawl07/saves/milestones-sprint - -[local:cao] -/home/crawl/chroot/var/games/crawl08/saves/milestones - -[local:cao] -/home/crawl/chroot/var/games/crawl08/saves/milestones-sprint - -[local:cao] -/home/crawl/chroot/var/games/crawl09/saves/milestones - -[local:cao] -/home/crawl/chroot/var/games/crawl09/saves/milestones-sprint - -[local:cao] -/home/crawl/chroot/var/games/crawl10/saves/milestones - -[local:cao] -/home/crawl/chroot/var/games/crawl10/saves/milestones-sprint - - -[local:cao] -/var/lib/dgamelaunch/crawl-master/crawl-git/saves/milestones - -[local:cao] -/var/lib/dgamelaunch/crawl-master/crawl-git/saves/milestones-sprint - -[local:cao] -/var/lib/dgamelaunch/crawl-master/crawl-git/saves/milestones-zotdef - -[local:cao] -/var/lib/dgamelaunch/var/games/crawl-lorcs/saves/milestones - - # http://$cao/milestones02.txt [remote:cao] remote.cao-milestones-0.2 diff --git a/config/servers.yml b/config/servers.yml deleted file mode 100644 index d9deee56..00000000 --- a/config/servers.yml +++ /dev/null @@ -1,47 +0,0 @@ -# Public server abbreviations -sources: - cao: crawl.akrasiac.org - cdo: crawl.develz.org - rhf: rl.heh.fi - csn: crawlus.somatika.net - cszo: dobrazupa.org - -morgue-paths: - - [ 'cao-.*', 'http://crawl.akrasiac.org/rawdata' ] - - [ 'cdo.*-0.4$', 'http://crawl.develz.org/morgues/0.4' ] - - [ 'cdo.*-0.5$', 'http://crawl.develz.org/morgues/0.5' ] - - [ 'cdo.*-0.6$', 'http://crawl.develz.org/morgues/0.6' ] - - [ 'cdo.*-0.7', 'http://crawl.develz.org/morgues/0.7' ] - - [ 'cdo.*-0.8', 'http://crawl.develz.org/morgues/0.8' ] - - [ 'cdo.*-0.10', 'http://crawl.develz.org/morgues/0.10' ] - - [ 'cdo.*-0.11', 'http://crawl.develz.org/morgues/0.11' ] - - [ 'cdo.*-svn$', 'http://crawl.develz.org/morgues/trunk' ] - - [ 'cdo.*-zd$', 'http://crawl.develz.org/morgues/trunk' ] - - [ 'cdo.*-spr$', 'http://crawl.develz.org/morgues/trunk' ] - - [ 'rhf.*-0.5$', 'http://rl.heh.fi/crawl/stuff' ] - - [ 'rhf.*-0.6$', 'http://rl.heh.fi/crawl-0.6/stuff' ] - - [ 'rhf.*-0.7$', 'http://rl.heh.fi/crawl-0.7/stuff' ] - - [ 'rhf.*-trunk$', 'http://rl.heh.fi/trunk/stuff' ] - - [ 'rhf.*-spr$', 'http://rl.heh.fi/sprint/stuff' ] - - [ 'csn.*', 'http://crawlus.somatika.net/dumps' ] - - [ 'cszo.*', 'http://dobrazupa.org/morgue' ] - -ttyrec-paths: - - [ 'cao-.*', ['http://termcast.develz.org/cao/ttyrecs', - 'http://crawl.akrasiac.org/rawdata'] ] - - [ 'cdo.*$', [ 'http://termcast.develz.org/ttyrecs', - 'http://crawl.develz.org/ttyrecs' ] ] - - [ 'rhf.*-0.5$', 'http://rl.heh.fi/crawl/stuff' ] - - [ 'rhf.*-0.6$', 'http://rl.heh.fi/crawl-0.6/stuff' ] - - [ 'rhf.*-0.7$', 'http://rl.heh.fi/crawl-0.7/stuff' ] - - [ 'rhf.*-trunk$', 'http://rl.heh.fi/trunk/stuff' ] - - [ 'rhf.*-spr$', 'http://rl.heh.fi/sprint/stuff' ] - - [ 'cszo.*', 'http://dobrazupa.org/ttyrec' ] - -server-timezones: - caoD: '-0400' - caoS: '-0500' - cdoD: '+0200' - cdoS: '+0100' - cszoD: '-0400' - cszoS: '-0500' diff --git a/config/sources.yml b/config/sources.yml new file mode 100644 index 00000000..ae520a89 --- /dev/null +++ b/config/sources.yml @@ -0,0 +1,132 @@ +sources: + - name: cao + base: http://crawl.akrasiac.org + + # If the file exists in this path, it will be linked into the data + # directory from the local path; otherwise it will be fetched + # using http. + local: /var/www + + # Timezones are used if this server had games prior to Crawl using + # only UTC in the logfile. + timezones: + D: '-0400' + S: '-0500' + + utc-epoch: '200808070330+0000' + + # Annotations: standard glob {} expansion is applied. Files + # flagged * are assumed to be logfiles that are currently being + # updated and will be refetched, always. Files without * are + # assumed to be dead versions, and will be fetched only if the + # local copy is missing. + logfiles: + - allgames.txt + - logfile04 + - logfile05 + - logfile06 + - logfile07{,-sprint} + - logfile08{,-sprint} + - logfile09 + - logfile10* + - logfile11{,-sprint,-zotdef}* + - logfile-git{,-sprint,-zotdef}* + - logfile-lorcs: lorcs + milestones: + - milestones02 + - milestones03 + - milestones04 + - milestones05 + - milestones06 + - milestones07{,-sprint} + - milestones08{,-sprint} + - milestones09 + - milestones10* + - milestones11{,-sprint,-zotdef}* + - milestones-git{,-sprint,-zotdef}* + - milestones-lorcs: lorcs + + # Regex -> location; player name is automatically appended. + morgues: + - http://crawl.akrasiac.org/rawdata + + ttyrecs: + - http://termcast.develz.org/cao/ttyrecs + - http://crawl.akrasiac.org/rawdata + + - name: cdo + base: http://crawl.develz.org + timezones: + D: '+0200' + S: '+0100' + logfiles: + - allgames-0.3.txt + - allgames-0.4.txt + - allgames-0.5.txt + - allgames-0.6.txt + - allgames{,-spr}-0.7.txt + - allgames{,-spr,-zd}-0.8.txt* + - allgames{,-spr,-zd}-0.10.txt* + - allgames{,-spr,-zd}-0.11.txt* + - allgames{,-spr,-zd}-svn.txt* + milestones: + - milestones-0.3.txt + - milestones-0.4.txt + - milestones-0.5.txt + - milestones-0.6.txt + - milestones{,-spr}-0.7.txt + - milestones{,-spr,-zd}-0.8.txt* + - milestones{,-spr,-zd}-0.10.txt* + - milestones{,-spr,-zd}-0.11.txt* + - milestones{,-spr,-zd}-svn.txt* + morgues: + - - time_gt: '20110819-1740' + version_match: '0.9' + - http://crawl.develz.org/morgues/0.9 + - ['cdo.*-(?:svn|git)', 'http://crawl.develz.org/morgues/trunk'] + - ['cdo.*-(\d+[.]\d+)$', 'http://crawl.develz.org/morgues/$1'] + ttyrecs: + - http://termcast.develz.org/ttyrecs + - http://crawl.develz.org/ttyrecs + + - name: cszo + base: http://dobrazupa.org + utc-epoch: '200808070330+0000' + logfiles: + - meta/git/logfile{,-sprint,-zotdef}* + - meta/0.11/logfile{,-sprint,-zotdef}* + - meta/0.10/logfile{,-sprint,-zotdef}* + milestones: + - meta/git/milestones{,-sprint,-zotdef}* + - meta/0.11/milestones{,-sprint,-zotdef}* + - meta/0.10/milestones{,-sprint,-zotdef}* + morgues: + - http://dobrazupa.org/morgue + ttyrecs: + - http://dobrazupa.org/ttyrec + + - name: csn + base: http://crawlus.somatika.net + logfiles: + - scoring/crawl-trunk/logfile{,-sprint,-zotdef}* + - scoring/crawl-0.11/logfile{,-sprint,-zotdef}* + milestones: + - scoring/crawl-trunk/milestones{,-sprint,-zotdef}* + - scoring/crawl-0.11/milestones{,-sprint,-zotdef}* + morgues: + - http://crawlus.somatika.net/dumps + + - name: rhf + local-only: true + logfiles: + - remote.rhf-logfile-0.5 + - remote.rhf-logfile-0.6 + - remote.rhf-logfile-0.7 + - remote.rhf-logfile-trunk + - remote.rhf-logfile-spr + milestones: + - remote.rhf-milestones-0.5 + - remote.rhf-milestones-0.6 + - remote.rhf-milestones-0.7 + - remote.rhf-milestones-spr + - remote.rhf-milestones-trunk diff --git a/henzell.pl b/henzell.pl index 7f0c9152..de22e550 100755 --- a/henzell.pl +++ b/henzell.pl @@ -8,6 +8,7 @@ use Henzell::Config qw/%CONFIG %CMD %PUBLIC_CMD/; use Henzell::Utils; use Getopt::Long; +use Cwd; END { kill TERM => -$$; @@ -21,7 +22,7 @@ END "rc=s" => \$config_file) or die "Invalid options\n"; $ENV{LC_ALL} = 'en_US.utf8'; -$ENV{RUBYOPT} = '-rubygems'; +$ENV{RUBYOPT} = "-rubygems -I" . File::Spec->catfile(getcwd(), 'src'); my $SERVER = 'cao'; # Local server. my $ALT_SERVER = 'cdo'; # Our 'alternative' server. @@ -79,8 +80,8 @@ END Henzell::Utils::lock(verbose => 1, lock_name => $CONFIG{lock_name}); -if ($irc && $CONFIG{http_services}) { - Henzell::Utils::spawn_service("http_service", "rackup -p 29880 config.ru"); +if ($CONFIG{startup_services}) { + Henzell::Utils::spawn_services($CONFIG{startup_services}); } # Daemonify. http://www.webreference.com/perl/tutorial/9/3.html diff --git a/rc/henzell.rc.cao b/rc/henzell.rc.cao index c0435877..89faf9bb 100644 --- a/rc/henzell.rc.cao +++ b/rc/henzell.rc.cao @@ -1,4 +1,4 @@ -sql_queries = 0 -sql_store = 0 -host = cao -announce = 1 \ No newline at end of file +sql_queries: 0 +sql_store: 0 +host: cao +announce: 1 \ No newline at end of file diff --git a/rc/henzell.rc.ircnet b/rc/henzell.rc.ircnet deleted file mode 100644 index b27be1a2..00000000 --- a/rc/henzell.rc.ircnet +++ /dev/null @@ -1,10 +0,0 @@ -irc_server = irc.fu-berlin.de -irc_port = 6666 -lock_name = ircnet -bot_nick = Sequell -sql_store = 0 -sql_queries = 1 -commands_file = commands/commands-sequell.txt -announce_channel = -seen_update = 0 -channels = #roguelike \ No newline at end of file diff --git a/rc/henzell.rc.sequell b/rc/henzell.rc.sequell index 70a47b3e..cda5cef3 100644 --- a/rc/henzell.rc.sequell +++ b/rc/henzell.rc.sequell @@ -1,7 +1,11 @@ -bot_nick = Sequell -sql_store = 1 -sql_queries = 1 -commands_file = commands/commands-sequell.txt -announce_channel = -seen_update = 0 -http_services = 1 \ No newline at end of file +bot_nick: Sequell +sql_store: 1 +sql_queries: 1 +commands_file: commands/commands-sequell.txt +announce_channel: +seen_update: + +startup_services: + - name: FooTV HTTP Service + command: rackup -p 29880 config.ru + directory: services/http-tv diff --git a/remote-fetch-logfile b/remote-fetch-logfile index c495c558..2459ba81 100755 --- a/remote-fetch-logfile +++ b/remote-fetch-logfile @@ -7,15 +7,45 @@ use Henzell::Config qw/%CONFIG @LOGS @MILESTONES/; use Henzell::Utils; my $verbose = -t STDIN; +my $show_missing = grep(/^--show-missing$/, @ARGV); +my $list_files = grep(/^--list-files$/, @ARGV); + +sub symlink_local_file($) { + my $file = shift; + my $local_source = $file->local_filepath(); + my $target_file = $file->target_filepath(); + print "Symlinking $local_source -> $target_file\n" if $verbose; + symlink($local_source, $target_file) or die "Couldn't symlink $local_source -> $target_file\n"; +} sub fetch_file($) { my $file = shift; - unless ($$file{url}) { - print "Skipping $$file{path}, no URL\n" if $verbose; + my $path = $file->target_filepath(); + my $target_exists = $file->target_exists(); + + if ($show_missing || $list_files) { + print "$path (" . $file->source_name() . ")\n" if !$target_exists || $list_files; + return; + } + + if ($target_exists && !$file->is_live()) { + print "$path already exists and is not live, skipping it\n" if $verbose; return; } - print "Fetching $$file{url} -> $$file{path}\n" if $verbose; - system(qq{wget --timeout=15 -q -c $$file{url} -O $$file{path} >/dev/null 2>&1}); + + if (!$target_exists && $file->local_source_exists()) { + symlink_local_file($file); + return; + } + + unless ($file->http_url()) { + print "Skipping $file, no URL\n" if $verbose; + return; + } + + my $url = $file->http_url(); + print "Fetching $url -> $path\n" if $verbose; + system(qq{wget --timeout=15 -q -c $url -O $path >/dev/null 2>&1}); } sub main() { diff --git a/runcmd.pl b/runcmd.pl index 26501c1a..08690370 100755 --- a/runcmd.pl +++ b/runcmd.pl @@ -9,7 +9,8 @@ $ENV{IRC_NICK_AUTHENTICATED} = 'y'; $ENV{HENZELL_SQL_QUERIES} = 'y'; -$ENV{RUBYOPT} = '-rubygems'; +$ENV{RUBYOPT} = '-rubygems -Isrc'; +$ENV{HENZELL_ROOT} = '.'; Henzell::Cmd::load_all_commands(); sub runcmd($) { diff --git a/scripts/rename-source-file b/scripts/rename-source-file new file mode 100755 index 00000000..b2890fb2 --- /dev/null +++ b/scripts/rename-source-file @@ -0,0 +1,65 @@ +#! /usr/bin/env perl +# +# Renames a source file from X to Y. Source and target files will be +# renamed in server-xlogs and in the Henzell DB. + +use strict; +use warnings; + +use DBI; + +my $XLOG_ROOT = 'server-xlogs'; +my $DBNAME = $ENV{HENZELL_DBNAME} || 'henzell'; +my $DBUSER = 'henzell'; +my $DBPASS = 'henzell'; + +sub resolve_file { + my $file = shift; + $file = "$XLOG_ROOT/$file" unless $file =~ m{/}; + $file +} + +sub db_url { + "dbi:Pg:dbname=" . $DBNAME +} + +sub connect_db { + my $dbh = DBI->connect(db_url(), $DBUSER, $DBPASS) or die "Could not connect to " . db_url() . ": $!\n"; + $dbh +} + +sub basename { + my $file = shift; + my ($basename) = $file =~ m{.*/(.*)}; + $basename || $file +} + +sub rename_db_file { + my ($source, $target) = @_; + $source = basename($source); + $target = basename($target); + my $dbh = connect_db(); + my $update_sql = 'UPDATE l_file SET file = ? WHERE file = ?'; + my @binds = ($target, $source); + my $res = $dbh->do($update_sql, undef, @binds); + if (!$res || $res <= 0) { + die "Could not update l_file with $update_sql (" . join(", ", @binds) . "): $!\n"; + } + print "Updated l_file from $source -> $target\n"; +} + +sub rename_fs_file { + my ($source, $target) = @_; + rename $source, $target or die "Could not rename $source -> $target: $!\n"; + print "Renamed $source -> $target\n"; +} + +sub main { + my ($source, $target) = map(resolve_file($_), @ARGV); + die "Source file $source does not exist\n" unless -f $source; + + rename_db_file($source, $target); + rename_fs_file($source, $target) +} + +main(); diff --git a/services/http-tv/Gemfile b/services/http-tv/Gemfile new file mode 100644 index 00000000..c9929fdd --- /dev/null +++ b/services/http-tv/Gemfile @@ -0,0 +1,10 @@ +source :rubygems + +gem 'sqlite3' +gem 'dbi' +gem 'dbd-pg' +gem 'sinatra' +gem 'rest-client' +gem 'rack-timeout' +gem 'json' +gem 'haml' diff --git a/services/http-tv/config.ru b/services/http-tv/config.ru new file mode 100644 index 00000000..ef911967 --- /dev/null +++ b/services/http-tv/config.ru @@ -0,0 +1,4 @@ +# -p 29880 + +require './http_service' +run Sinatra::Application \ No newline at end of file diff --git a/services/http_service.rb b/services/http-tv/http_service.rb similarity index 92% rename from services/http_service.rb rename to services/http-tv/http_service.rb index be3613de..39c02128 100644 --- a/services/http_service.rb +++ b/services/http-tv/http_service.rb @@ -1,4 +1,4 @@ -$:.push("src") +$:.push(File.join(ENV['HENZELL_ROOT'], 'src')) SERVICE_PORT = 29880 diff --git a/sqllog.pl b/sqllog.pl index cdf0b597..81ad5bad 100644 --- a/sqllog.pl +++ b/sqllog.pl @@ -8,7 +8,6 @@ use DBI; use Henzell::Crawl; use Henzell::DB; -use Henzell::ServerConfig; use Henzell::TableLoader; do 'game_parser.pl'; @@ -191,20 +190,20 @@ sub open_handles my @handles; for my $file (@files) { - my $path = $$file{path}; + my $path = $file->target_filepath(); open my $handle, '<', $path or do { warn "Unable to open $path for reading: $!"; next; }; seek($handle, 0, SEEK_END); # EOF - push @handles, { file => $$file{path}, + push @handles, { file => $file->target_filepath(), fref => $file, handle => $handle, pos => tell($handle), - server => $$file{src}, - src => $$file{src}, - alpha => $$file{alpha} }; + server => $file->server_name(), + src => $file->server_name(), + alpha => $file->alpha()}; } return @handles; } @@ -511,11 +510,7 @@ sub fixup_logfields { milestone_mangle($g); } else { - my $src = $g->{src}; - # Fixup src for interesting_game. - $g->{src} = "http://" . Henzell::ServerConfig::source_hostname($src) . "/"; $g->{splat} = ''; - $g->{src} = $src; } $g->{game_key} = "$$g{name}:$$g{src}:$$g{rstart}"; diff --git a/src/crawl/config.rb b/src/crawl/config.rb index 95c68475..58574f60 100644 --- a/src/crawl/config.rb +++ b/src/crawl/config.rb @@ -1,11 +1,16 @@ require 'yaml' +require 'henzell/config' module Crawl class Config CONFIG_FILE = 'config/crawl-data.yml' + def self.config_file + Henzell::Config.file_path(CONFIG_FILE) + end + def self.config - @config ||= read_config + @config ||= read_config(config_file) end def self.[](key) @@ -13,8 +18,8 @@ def self.[](key) end private - def self.read_config - YAML.load_file(CONFIG_FILE) + def self.read_config(config_file) + YAML.load_file(config_file) end end end diff --git a/src/helper.rb b/src/helper.rb index dac858a7..1645034a 100644 --- a/src/helper.rb +++ b/src/helper.rb @@ -6,9 +6,6 @@ require 'yaml' require 'fileutils' -SERVERS_FILE = 'config/servers.yml' -SERVER_CFG = YAML.load_file(SERVERS_FILE) - # Directory containing player directories that contain morgues. DGL_MORGUE_DIR = '/var/www/crawl/rawdata' @@ -25,25 +22,9 @@ def regex_paths(regex_path_mappings) } end -DGL_ALIEN_MORGUES = regex_paths(SERVER_CFG['morgue-paths']) -DGL_ALIEN_TTYRECS = regex_paths(SERVER_CFG['ttyrec-paths']) - -SERVER_TIMEZONE = SERVER_CFG['server-timezones'] - MORGUE_DATEFORMAT = '%Y%m%d-%H%M%S' SHORT_DATEFORMAT = '%Y%m%d%H%M%S' -begin - # The time (approximate) that Crawl switched from local time to UTC in - # logfiles. We'll have lamentable inaccuracy near this time, but that - # can't be helped. - LOCAL_UTC_EPOCH_DATETIME = DateTime.strptime('200808070330+0000', - '%Y%m%d%H%M%z') -rescue - # Ruby version differences :( - LOCAL_UTC_EPOCH_DATETIME = nil -end - NICK_ALIASES = { } NICKMAP_FILE = ENV['HENZELL_TEST'] ? 'dat/nicks-test.map' : 'dat/nicks.map' $nicks_loaded = false @@ -204,131 +185,16 @@ def morgue_time_dst?(e, key=nil) morgue_timestring(e, key) =~ /D$/ end -def game_ttyrec_datetime(e, key=nil) - time = morgue_time(e, key) - return nil if time.nil? - dt = DateTime.strptime(time + "+0000", MORGUE_DATEFORMAT + '%z') - if dt < LOCAL_UTC_EPOCH_DATETIME - dst = morgue_time_dst?(e, key) - src = e['src'] - src = src + (dst ? 'D' : 'S') - - tz = SERVER_TIMEZONE[src] - if tz - # Parse the time as the server's local TZ, and convert it to UTC. - dt = DateTime.strptime(time + tz, MORGUE_DATEFORMAT + '%z').new_offset(0) - end - end - dt -end - -def binary_search_alien_morgue(url, e) - require 'httplist' - user_url = url + "/" + e['name'] + "/" - mtime = morgue_time(e) - morgues = HttpList::find_files(user_url, /morgue-#{e['name']}.*?[.]txt/, - DateTime.strptime(mtime, MORGUE_DATEFORMAT)) - return nil if morgues.nil? - - short_mtime = mtime.sub(/\d{2}$/, '') - full_name = "morgue-#{e['name']}-#{mtime}.txt" - short_name = "morgue-#{e['name']}-#{short_mtime}.txt" - - # Look for exact match with the full time or short time - found = (morgues.find { |m| m == full_name } || - morgues.find { |m| m == short_name } || - binary_search(morgues, full_name)) - return found.url if found - - return nil -end - -def resolve_alien_morgue(url, e) - if Integer(/^0\.([0-9]+)/.match(e['v'])[1]) < 4 - return binary_search_alien_morgue(url, e) - else - return morgue_assemble_filename(url, e, morgue_time(e), '.txt') - end -end - -def find_alien_morgue(e) - for pair in DGL_ALIEN_MORGUES - if e['file'] =~ pair[0] - if e['cv'] == '0.9' && e['src'] == 'cdo' && e['end'] > '201107191740' - return resolve_alien_morgue('http://crawl.develz.org/morgues/0.9', e) - else - return resolve_alien_morgue(pair[1], e) - end - end - end - nil -end - def find_game_morgue(e) - return find_game_morgue_ext(e, ".txt", false) || - find_game_morgue_ext(e, ".txt.bz2", false) || - find_game_morgue_ext(e, ".txt.gz", false) || - find_game_morgue_ext(e, ".txt", true) -end - -def find_game_morgue_ext(e, ext, full_scan) - if e['src'] != ENV['HENZELL_HOST'] - return find_alien_morgue(e) - end - - fulltime = morgue_time(e) - - # Look for full timestamp - morgue = morgue_assemble_filename(DGL_MORGUE_DIR, e, fulltime, ext) - if File.exist?(morgue) - return morgue_assemble_filename(DGL_MORGUE_URL, e, fulltime, ext) - end - - parttime = fulltime.sub(/\d{2}$/, '') - morgue = morgue_assemble_filename(DGL_MORGUE_DIR, e, parttime, ext) - if File.exist?(morgue) - return morgue_assemble_filename(DGL_MORGUE_URL, e, parttime, ext) - end - - if full_scan - # We're in El Suck territory. Scan the directory listing. - morgue_list = game_morgues(e["name"]) - - # morgues are sorted. The morgue date should be greater than the - # full timestamp. - - found = binary_search(morgue_list, morgue) - if found then - found.sub!(/.*morgue-\w+-(.*)/, '\1') - return morgue_assemble_filename(DGL_MORGUE_URL, e, found, '') - end - end - - nil -end - -def crashdump_assemble_filename(urlbase, milestone) - (urlbase + '/' + milestone["name"] + - '/crash-' + milestone["name"] + '-' + - morgue_time(milestone) + ".txt") + require 'henzell/sources' + Henzell::Sources.instance.morgue_for(e) end def find_milestone_crash_dump(e) return nil if e['verb'] != 'crash' - # Check for cao crashes: - if e['src'] == ENV['HENZELL_HOST'] - return crashdump_assemble_filename(DGL_MORGUE_URL, e) - end - - # No cao? Look for alien crashes - for pair in DGL_ALIEN_MORGUES - if e['file'] =~ pair[0] - return crashdump_assemble_filename(pair[1], e) - end - end - - nil + require 'henzell/sources' + Henzell::Sources.instance.crash_dump_for(e) end def short_game_summary(g) @@ -368,18 +234,6 @@ def duration_str(dur) (dur % 3600) / 60, dur % 60) end -def game_user_url(game, urlbase) - urlbase + "/" + game['name'] + "/" -end - -def game_user_urls(game, urlbases) - if urlbases.is_a?(Array) - urlbases.map { |u| game_user_url(game, u) } - else - game_user_url(game, urlbases) - end -end - def ttyrec_list_string(game, ttyreclist) if !ttyreclist || ttyreclist.empty? return nil @@ -405,51 +259,6 @@ def ttyrec_list_string(game, ttyreclist) end end -def resolve_alien_ttyrecs_between(urlbase, game, tstart, tend) - require 'httplist' - user_url = game_user_urls(game, urlbase) - ttyrecs = HttpList::find_files(user_url, /[.]ttyrec/, tend) || [ ] - - sstart = tstart.strftime(SHORT_DATEFORMAT) - send = tend.strftime(SHORT_DATEFORMAT) - - first_ttyrec_before_start = nil - first_ttyrec_is_start = false - found = ttyrecs.find_all do |ttyrec| - filetime = ttyrec_filename_datetime_string(ttyrec.filename) - if (filetime && sstart && filetime < sstart) - first_ttyrec_before_start = ttyrec - end - - if (!first_ttyrec_is_start && filetime && sstart && filetime == sstart) - first_ttyrec_is_start = true - end - - filetime && (!sstart || filetime >= sstart) && filetime <= send - end - - if first_ttyrec_before_start && !first_ttyrec_is_start - found = [ first_ttyrec_before_start ] + found - end - - found -end - -def find_alien_ttyrecs(game) - tty_start = (game_ttyrec_datetime(game, 'start') || - game_ttyrec_datetime(game, 'rstart')) - tty_end = (game_ttyrec_datetime(game, 'end') || - game_ttyrec_datetime(game, 'time')) - - for pair in DGL_ALIEN_TTYRECS - if game['file'] =~ pair[0] - betw = resolve_alien_ttyrecs_between(pair[1], game, tty_start, tty_end) - return ttyrec_list_string(game, betw) - end - end - nil -end - def local_time(time) match = /^(\d{4})(\d{2})(\d{2})(\d{2})(\d{2})(\d{2})([SD])/.match(time) raise "Malformed local time: #{time}" unless match @@ -468,60 +277,17 @@ def local_time(time) time.utc end -def ttyrec_filename_datetime_string(filename) - if filename =~ /^(\d{4}-\d{2}-\d{2}\.\d{2}:\d{2}:\d{2})\.ttyrec/ - $1.gsub(/[-.:]/, '') - elsif filename =~ /(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2})[+]00:?00/ - $1.gsub(/[-:T]/, '') - elsif filename =~ /(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}[+]\d{2}:\d{2})/ - date = DateTime.strptime($1, '%Y-%m-%dT%H:%M:%S%Z') - date.new_offset(0).strftime('%Y%m%d%H%M%S') - else - nil - end -end - -def find_ttyrecs_between(game, s, e) - prefix = DGL_TTYREC_DIR + "/" + game['name'] + "/" - files = Dir[ prefix + "*.ttyrec*" ] - - s = s.strftime(SHORT_DATEFORMAT) - e = s.strftime(SHORT_DATEFORMAT) - bracketed = files.find_all do |rfile| - file = rfile.slice( prefix.length .. -1 ) - filetime = ttyrec_filename_datetime(file) - - next unless filetime - (!s || filetime >= s) and filetime <= e - end - bracketed.map { |f| f.slice( prefix.length .. -1 ) }.sort -end - -def find_cao_ttyrecs(game) - tty_start = game_ttyrec_datetime(game, 'start') - tty_end = game_ttyrec_datetime(game, 'end') - - betw = find_ttyrecs_between(game, tty_start, tty_end) - - unless betw.empty? - base_url = DGL_TTYREC_URL + "/" + game['name'] + "/" - spc = betw.length == 1 ? "" : " " - "#{base_url}#{spc}#{betw.join(" ")}" - end -end - def find_game_ttyrecs(game) - if game['src'] != ENV['HENZELL_HOST'] - find_alien_ttyrecs(game) - else - find_cao_ttyrecs(game) - end + require 'henzell/sources' + ttyrecs = Henzell::Sources.instance.ttyrecs_for(game) + ttyrec_list_string(game, ttyrecs) end def report_game_ttyrecs(n, game) puts(game_number_prefix(n) + short_game_summary(game) + ": " + (find_game_ttyrecs(game) || "Can't find ttyrec!")) rescue + raise puts(game_number_prefix(n) + short_game_summary(game) + ": " + $!) raise end diff --git a/src/henzell/commands.rb b/src/henzell/commands.rb index 55e8da91..c1694270 100644 --- a/src/henzell/commands.rb +++ b/src/henzell/commands.rb @@ -31,7 +31,7 @@ def execute(command_line, default_nick='???') raise StandardError, "Bad command: #{command_line}" end - command_script = "./commands/" + @commands[command] + command_script = File.join(Config.root, "commands", @commands[command]) target = default_nick if command_line =~ /^(\S+)\s+(\S+)/ target = $2 diff --git a/src/henzell/config.rb b/src/henzell/config.rb index 1edfa106..bfde3acc 100644 --- a/src/henzell/config.rb +++ b/src/henzell/config.rb @@ -2,9 +2,21 @@ module Henzell class Config - CONFIG_FILE = 'rc/henzell.rc' + CONFIG_FILEPATH = 'rc/henzell.rc' - def self.read(cfg=CONFIG_FILE) + def self.root + ENV['HENZELL_ROOT'] || '.' + end + + def self.file_path(name) + File.join(self.root, name) + end + + def self.config_file + file_path(CONFIG_FILEPATH) + end + + def self.read(cfg=self.config_file) config = self.new(cfg) config.load config @@ -16,23 +28,16 @@ def initialize(cfg) end def [](key) - @config[key] + @config[key.to_s] end def commands - @commands ||= Henzell::Commands.new(self[:commands_file]) + @commands ||= Henzell::Commands.new( + Henzell::Config.file_path(self[:commands_file])) end def load - File.open(@config_file, 'r') { |file| - file.each { |line| - next if line =~ /^\s*#/ - line = line.strip - if line =~ /^(\w+)\s*=\s*(.*)/ - @config[$1.downcase.to_sym] = $2 - end - } - } + @config = YAML.load_file(@config_file) end end end diff --git a/src/henzell/config_file.rb b/src/henzell/config_file.rb new file mode 100644 index 00000000..e6623783 --- /dev/null +++ b/src/henzell/config_file.rb @@ -0,0 +1,8 @@ +module Henzell + class ConfigFile + def self.root + en + def self.file(named) + File.join(ENV[ + end +end diff --git a/src/henzell/morgue_resolver.rb b/src/henzell/morgue_resolver.rb new file mode 100644 index 00000000..7788154c --- /dev/null +++ b/src/henzell/morgue_resolver.rb @@ -0,0 +1,26 @@ +module Henzell + class MorgueResolver + def initialize(sources, game) + @sources = sources + @source = @sources.source(game['src']) + @game = game + end + + def morgue(type='morgue') + morgue_paths.each { |morgue_path| + resolved_morgue = morgue_path.morgue_url(@game, type) + return resolved_morgue if resolved_morgue + } + nil + end + + def crash_dump + self.morgue('crash') + end + + private + def morgue_paths + @source.morgue_paths + end + end +end diff --git a/src/henzell/server/morgue_binary_search.rb b/src/henzell/server/morgue_binary_search.rb new file mode 100644 index 00000000..995242b7 --- /dev/null +++ b/src/henzell/server/morgue_binary_search.rb @@ -0,0 +1,75 @@ +module Henzell + module Server + class MorgueBinarySearch + def self.search(path, game) + require 'httplist' + self.new(path, game).search + end + + attr_reader :path, :game + + def initialize(path, game) + @path = path + @game = game + end + + def player_name + @game['name'] + end + + def user_url + @user_url ||= path + "/" + player_name + "/" + end + + def time + @time ||= morgue_time(@game) + end + + def short_time + @short_time ||= self.time.sub(/\d{2}$/, '') + end + + def full_morgue_name + @full_morgue_name ||= "morgue-#{self.player_name}-#{time}.txt" + end + + def short_morgue_name + @short_morgue_name ||= "morgue-#{self.player_name}-#{short_time}.txt" + end + + def morgues + @morgues ||= + HttpList::find_files(user_url, /morgue-#{player_name}.*?[.]txt/, + DateTime.strptime(time, MORGUE_DATEFORMAT)) + end + + def search + morgue = morgues.find { |m| m == self.full_morgue_name } || + morgues.find { |m| m == self.short_morgue_name } || + binary_search(morgues, self.full_morgue_name) + morgue.url if morgue + end + + private + def binary_search(morgues, name) + size = arr.size + if size == 1 + return what < arr[0] ? arr[0] : nil + end + s = 0 + e = size + while e - s > 1 + pivot = (s + e) / 2 + if arr[pivot] == what + return arr[pivot] + elsif arr[pivot] < what + s = pivot + else + e = pivot + end + end + e < size ? arr[e] : nil + end + end + end +end diff --git a/src/henzell/server/morgue_filename.rb b/src/henzell/server/morgue_filename.rb new file mode 100644 index 00000000..c265d885 --- /dev/null +++ b/src/henzell/server/morgue_filename.rb @@ -0,0 +1,63 @@ +module Henzell + module Server + class MorgueFilename + def self.name(path, game, prefix='morgue', extension='.txt') + self.new(path, game, extension, prefix).name + end + + attr_reader :path, :game, :extension, :prefix + + def initialize(path, game, extension='.txt', prefix='morgue') + @game = game + @path = resolve_path(path, game) + @extension = extension + @prefix = prefix + end + + def player + @game['name'] + end + + def time + morgue_time(@game) + end + + def name + @name ||= lookup_morgue + end + + def ancient_version? + require 'sql/version_number' + Sql::VersionNumber.version_numberize(@game['v']) < + Sql::VersionNumber.version_numberize('0.4') + end + + private + def resolve_path(path, game) + path.gsub(/:(\w+)/) { |match| + key = $1 + if key == 'prefix_v' + game['v'].sub(/^(\d+[.]\d+).*/, '\1') + else + game[key] || ":#{key}" + end + } + end + + def lookup_morgue + return binary_search if ancient_version? + simple_name + end + + def simple_name + (path + '/' + self.player + '/' + self.prefix + '-' + self.player + + '-' + self.time + self.extension) + end + + def binary_search + require 'henzell/server/morgue_binary_search' + MorgueBinarySearch.search(path, game) + end + end + end +end diff --git a/src/henzell/server/morgue_path.rb b/src/henzell/server/morgue_path.rb new file mode 100644 index 00000000..bb4c3f7c --- /dev/null +++ b/src/henzell/server/morgue_path.rb @@ -0,0 +1,74 @@ +require 'henzell/server/morgue_filename' +require 'henzell/server/morgue_predicate' + +module Henzell + module Server + class MorguePath + def initialize(server, path) + @server = server + @path = path + end + + def morgue_url(game, type='morgue') + morgue_matcher.morgue_url(game, type) + end + + def to_s + @path.inspect + end + + private + def morgue_matcher + @morgue_matcher ||= + if @path.is_a?(Array) + pattern = @path[0] + path = @path[1] + if pattern.is_a?(Hash) + PredicateMatcher.new(pattern, path) + else + FilePatternMatcher.new(pattern, path) + end + else + SimpleMorgueMatcher.new(@path) + end + end + end + + class SimpleMorgueMatcher + def initialize(path) + @path = path + end + + def morgue_url(game, type) + MorgueFilename.name(@path, game, type) + end + end + + class FilePatternMatcher + def initialize(pattern, path) + @pattern = Regexp.new(pattern) + @path = path + end + + def morgue_url(game, type) + match = @pattern.match(game['file']) + return nil unless match + url = @path.gsub(/\$(\d+)/) { |m| + match[$1.to_i] + } + MorgueFilename.name(url, game, type) + end + end + + class PredicateMatcher + def initialize(predicates, path) + @predicates = MorguePredicate.compile(predicates) + @path = path + end + + def morgue_url(game, type) + MorgueFilename.name(@path, game, type) if @predicates.matches?(game) + end + end + end +end diff --git a/src/henzell/server/morgue_predicate.rb b/src/henzell/server/morgue_predicate.rb new file mode 100644 index 00000000..ab0d0ed6 --- /dev/null +++ b/src/henzell/server/morgue_predicate.rb @@ -0,0 +1,70 @@ +module Henzell + module Server + class MorguePredicate + REGISTRY = { } + + def self.compile(predicates) + self.new(predicates) + end + + def self.register(key, cls) + REGISTRY[key] = cls + end + + def self.predicate_named(name) + REGISTRY[name] or die "Unknown predicate: #{name}" + end + + def initialize(predicates) + @clauses = predicates.map { |predicate, arguments| + compile_predicate(predicate, arguments) + } + end + + def matches?(game) + @clauses.all? { |clause| clause.matches?(game) } + end + + private + def compile_predicate(predicate, arguments) + arguments = [arguments] unless arguments.is_a?(Array) + self.class.predicate_named(predicate).new(predicate, *arguments) + end + end + + class TimeGt + MorguePredicate.register('time_gt', self) + + def initialize(name, value) + @value = value + end + + def matches?(game) + format_time(game_time(game)) > @value + end + + def game_time(game) + game['end'] || game['rend'] || game['time'] || game['rtime'] + end + + private + def format_time(time) + time.sub(/^(\d{4})(\d{2})(\d{2})(\d{4,}).*/) { |match| + $1 + sprintf('%02d', ($2.to_i + 1)) + $3 + '-' + $4 + } + end + end + + class VersionMatch + MorguePredicate.register('version_match', self) + + def initialize(name, version) + @version = version + end + + def matches?(game) + game['v'] =~ /^#{Regexp.quote(@version)}(?:[.-]|$)/ + end + end + end +end diff --git a/src/henzell/source.rb b/src/henzell/source.rb new file mode 100644 index 00000000..2cc44897 --- /dev/null +++ b/src/henzell/source.rb @@ -0,0 +1,32 @@ +module Henzell + class Source + def initialize(config) + @config = config + end + + def name + @name ||= @config['name'] + end + + def morgue_paths + require 'henzell/server/morgue_path' + @morgue_paths ||= @config['morgues'].map { |morgue_cfg| + Henzell::Server::MorguePath.new(self, morgue_cfg) + } + end + + def ttyrec_urls + @ttyrec_urls ||= @config['ttyrecs'] + end + + def utc_epoch + return nil unless @config['utc-epoch'] + @utc_epoch ||= DateTime.strptime(@config['utc-epoch'], '%Y%m%d%H%M%z') + end + + def timezone(type) + return nil unless @config['timezones'] + @config['timezones'][type] + end + end +end diff --git a/src/henzell/sources.rb b/src/henzell/sources.rb new file mode 100644 index 00000000..133e590b --- /dev/null +++ b/src/henzell/sources.rb @@ -0,0 +1,63 @@ +require 'henzell/config' +require 'yaml' + +module Henzell + class Sources + SOURCES_FILE = 'config/sources.yml' + + def self.sources_file + Henzell::Config.file_path(SOURCES_FILE) + end + + def self.instance + @instance ||= self.new(self.sources_file) + end + + def self.config + self.instance + end + + attr_reader :sources + + def initialize(config_file) + @config_file = @config_file + @config = YAML.load_file(config_file) + require 'henzell/source' + @sources ||= @config['sources'].map { |source_cfg| + Source.new(source_cfg) + } + @source_map = Hash[ @sources.map { |s| [s.name, s] } ] + end + + def source_abbreviations + self.sources.map(&:name) + end + + def source_names + self.sources.map(&:name) + end + + def source(name) + @source_map[name] or raise StandardError, "Unknown game source: #{name}" + end + + def source_for(game) + self.source(game['src']) + end + + def morgue_for(game) + require 'henzell/morgue_resolver' + MorgueResolver.new(self, game).morgue + end + + def crash_dump_for(game) + require 'henzell/morgue_resolver' + MorgueResolver.new(self, game).crash_dump + end + + def ttyrecs_for(game) + require 'henzell/ttyrec_search' + TtyrecSearch.ttyrecs(self.source_for(game), game) + end + end +end diff --git a/src/henzell/ttyrec_search.rb b/src/henzell/ttyrec_search.rb new file mode 100644 index 00000000..b6a36f4d --- /dev/null +++ b/src/henzell/ttyrec_search.rb @@ -0,0 +1,115 @@ +module Henzell + class TtyrecSearch + def self.ttyrecs(source, game) + self.new(source, game).ttyrecs + end + + attr_reader :source, :game + def initialize(source, game) + @source = source + @game = game + end + + def user + @game['name'] + end + + def utc_epoch + @source.utc_epoch + end + + def tty_start + @tty_start ||= (game_ttyrec_datetime(game, 'start') || + game_ttyrec_datetime(game, 'rstart')) + end + + def tty_end + @tty_end ||= (game_ttyrec_datetime(game, 'end') || + game_ttyrec_datetime(game, 'time')) + end + + def short_start_time + @short_start_time ||= self.tty_start.strftime(SHORT_DATEFORMAT) + end + + def short_end_time + @short_end_time ||= self.tty_end.strftime(SHORT_DATEFORMAT) + end + + def ttyrec_urls + self.source.ttyrec_urls + end + + def user_ttyrec_urls + @user_ttyrec_urls ||= self.ttyrec_urls.map { |ttyrec_url| + user_ttyrec_url(ttyrec_url) + } + end + + def ttyrecs + require 'httplist' + all_ttyrecs = + HttpList::find_files(self.user_ttyrec_urls, /[.]ttyrec/, tty_end) || + [ ] + + first_ttyrec_before_start = nil + first_ttyrec_is_start = false + found = all_ttyrecs.find_all do |ttyrec| + filetime = ttyrec_filename_datetime_string(ttyrec.filename) + if (filetime && short_start_time && filetime < short_start_time) + first_ttyrec_before_start = ttyrec + end + + if (!first_ttyrec_is_start && filetime && short_start_time && + filetime == short_start_time) + first_ttyrec_is_start = true + end + + filetime && (!short_start_time || filetime >= short_start_time) && + filetime <= short_end_time + end + + if first_ttyrec_before_start && !first_ttyrec_is_start + found = [ first_ttyrec_before_start ] + found + end + + found + end + + private + def user_ttyrec_url(ttyrec_url) + ttyrec_url + '/' + self.user + '/' + end + + def ttyrec_filename_datetime_string(filename) + if filename =~ /^(\d{4}-\d{2}-\d{2}\.\d{2}:\d{2}:\d{2})\.ttyrec/ + $1.gsub(/[-.:]/, '') + elsif filename =~ /(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2})[+]00:?00/ + $1.gsub(/[-:T]/, '') + elsif filename =~ /(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}[+]\d{2}:\d{2})/ + date = DateTime.strptime($1, '%Y-%m-%dT%H:%M:%S%Z') + date.new_offset(0).strftime('%Y%m%d%H%M%S') + else + nil + end + end + + def game_ttyrec_datetime(game, key=nil) + time = morgue_time(game, key) + return nil if time.nil? + dt = DateTime.strptime(time + "+0000", MORGUE_DATEFORMAT + '%z') + if self.utc_epoch && dt < self.utc_epoch + dst = morgue_time_dst?(game, key) + src = game['src'] + src = src + (dst ? 'D' : 'S') + + tz = source.timezone(src) + if tz + # Parse the time as the server's local TZ, and convert it to UTC. + dt = DateTime.strptime(time + tz, MORGUE_DATEFORMAT + '%z').new_offset(0) + end + end + dt + end + end +end diff --git a/src/henzell_config.rb b/src/henzell_config.rb index dafd8982..834529cd 100644 --- a/src/henzell_config.rb +++ b/src/henzell_config.rb @@ -1,12 +1,9 @@ module HenzellConfig require 'yaml' require 'set' + require 'crawl/config' - CONFIG_FILE = 'config/crawl-data.yml' - SERVER_CONFIG_FILE = 'config/servers.yml' - - CFG = YAML.load_file(CONFIG_FILE) - SERVER_CFG = YAML.load_file(SERVER_CONFIG_FILE) + CFG = Crawl::Config.config GAME_PREFIXES = CFG['game-type-prefixes'] GAME_TYPE_DEFAULT = CFG['default-game-type'] diff --git a/src/irc_auth.rb b/src/irc_auth.rb index 992df938..5e9c42ee 100644 --- a/src/irc_auth.rb +++ b/src/irc_auth.rb @@ -1,8 +1,12 @@ require 'yaml' +require 'henzell/config' class IrcAuth + AUTH_FILE = 'config/auth.yml' + def self.authorizations - @authorizations ||= YAML.load_file('config/auth.yml') + @authorizations ||= YAML.load_file( + Henzell::Config.file_path(AUTH_FILE)) end def self.acting_nick diff --git a/services/request_throttle.rb b/src/services/request_throttle.rb similarity index 100% rename from services/request_throttle.rb rename to src/services/request_throttle.rb diff --git a/src/sqlhelper.rb b/src/sqlhelper.rb index 43717d1d..3a913a64 100644 --- a/src/sqlhelper.rb +++ b/src/sqlhelper.rb @@ -5,8 +5,6 @@ end DEBUG_HENZELL = ENV['DEBUG_HENZELL'] -LG_CONFIG_FILE = 'config/crawl-data.yml' -LG_SERVERS_FILE = 'config/servers.yml' require 'dbi' require 'set' @@ -16,6 +14,7 @@ require 'tourney' require 'sql_connection' require 'henzell_config' +require 'henzell/sources' require 'sql/config' require 'sql/field_predicate' @@ -30,13 +29,13 @@ require 'query/summary_graph_builder' require 'crawl/branch_set' require 'crawl/gods' +require 'crawl/config' require 'string_fixup' include Tourney include HenzellConfig -CFG = YAML.load_file(LG_CONFIG_FILE) -LG_SERVER_CFG = YAML.load_file(LG_SERVERS_FILE) +CFG = Crawl::Config.config # Don't use more than this much memory (bytes) MAX_MEMORY_USED = 768 * 1024 * 1024 @@ -49,7 +48,7 @@ UNIQUES = Set.new(CFG['uniques'].map { |u| u.downcase }) GODS = Crawl::Gods.new(CFG['god']) -SOURCES = LG_SERVER_CFG['sources'].keys.sort +SOURCES = Henzell::Sources.instance.source_names CLASS_EXPANSIONS = Hash[CFG['classes'].map { |abbr, cls| [abbr, cls.sub('*', '')] }] @@ -196,9 +195,9 @@ def sql_show_game_with_extras(nick, other_args_string, extra_args = []) if opts[:tv] TV.request_game_verbosely(res.qualified_index, res.game, ARGV[1]) elsif logopts[:log] - report_game_log(res.n, res.game) + report_game_log(res.qualified_index, res.game) elsif logopts[:ttyrec] - report_game_ttyrecs(res.n, res.game) + report_game_ttyrecs(res.qualified_index, res.game) else print_game_result(res) end diff --git a/src/tourney.rb b/src/tourney.rb index 4e2dbeab..cc4b4f99 100644 --- a/src/tourney.rb +++ b/src/tourney.rb @@ -1,13 +1,12 @@ -require 'yaml' +require 'crawl/config' module Tourney - CFG = YAML.load_file(LG_CONFIG_FILE) + CFG = Crawl::Config.config TOURNEY_SPRINT_MAP = CFG['tournament-sprint-map'] TOURNEY_PREFIXES = CFG['tournament-prefixes'] TOURNEY_VERSIONS = CFG['tournament-versions'] TOURNEY_DATA = CFG['tournament-data'] - SPRINT_TOURNEY_DATES = TOURNEY_REGEXES = TOURNEY_PREFIXES.map do |p| %r/^(#{p})(\d*)([a-z]?)$/i diff --git a/src/tv/channel_manager.rb b/src/tv/channel_manager.rb index 0923b467..8cc9a4b1 100644 --- a/src/tv/channel_manager.rb +++ b/src/tv/channel_manager.rb @@ -1,5 +1,6 @@ require 'fileutils' require 'libtv' +require 'henzell/config' module TV CHANNEL_DIR = 'dat/tv' @@ -61,11 +62,15 @@ def save private def create_directory - FileUtils.mkdir_p(CHANNEL_DIR) + FileUtils.mkdir_p(channel_dir) + end + + def channel_dir + Henzell::Config.file_path(CHANNEL_DIR) end def channel_file - CHANNEL_FILE + Henzell::Config.file_path(CHANNEL_FILE) end def with_file(mode='a+')