Permalink
Browse files

Change schema to work with cnntp (perl.org 3.0 branch)

  • Loading branch information...
1 parent d267d56 commit ec6c11e4f57f18850ece0c861c59bad410b1c482 @abh committed Mar 13, 2009
Showing with 34 additions and 24 deletions.
  1. +18 −11 colobus-archive
  2. +15 −12 colobus.sql
  3. +1 −1 run
View
@@ -91,13 +91,13 @@ if (grep { !$groups{$_}->{num} } keys %groups) {
}
my $ins_header = $dbh->prepare(<<QUERY);
-INSERT INTO header SET
- grp = ?,
- art = ?,
+INSERT INTO articles SET
+ group_id = ?,
+ id = ?,
msgid = ?,
subjhash = ?,
fromhash = ?,
- thread = ?,
+ thread_id = ?,
parent = ?,
received = FROM_UNIXTIME(?),
h_date = ?,
@@ -114,7 +114,7 @@ for $group (@ARGV) {
print "$group: " if $TRACE;
- my ($indexed) = $dbh->selectrow_array("SELECT MAX(art) FROM header WHERE grp = ?", undef, $grp) || 0;
+ my ($indexed) = $dbh->selectrow_array("SELECT MAX(id) FROM articles WHERE group_id = ?", undef, $grp) || 0;
open NUM, '<', $groups{$group}->{'path'}."/num"
or die "unable to open num file: $!";
@@ -129,15 +129,21 @@ for $group (@ARGV) {
$xover->{'In-Reply-To'} && $xover->{'In-Reply-To'} =~ m/<(.+?)>/ && push @parents, $1;
my $parent;
- if ($parent = pop(@parents)) {
- $parent = $dbh->selectrow_hashref("SELECT art,thread FROM header WHERE msgid = ?", undef, md5_hex($parent));
+ while (my $parent_msg = pop(@parents)) {
+ $parent = $dbh->selectrow_hashref("SELECT id,thread_id FROM articles WHERE msgid = ?", undef, md5_hex($parent_msg));
+ last if $parent;
}
my $subj_hash = md5_hex(clean_subject($xover->{'Subject'} || ''));
# if no parent, but subject starts with 'Re:', try to find the parent/thread
if (!$parent && $xover->{'Subject'} && $xover->{'Subject'} =~ m/^(Re|An|Antwort|Aw)(\^\d+|\[\d+\]|\(\d+\))?:\s*/i) {
- $parent = $dbh->selectrow_hashref("SELECT thread FROM header WHERE subjhash = ? AND received BETWEEN FROM_UNIXTIME(?) - INTERVAL 14 DAY AND FROM_UNIXTIME(?) ORDER BY received DESC LIMIT 1", undef, $subj_hash, $xover->{'mtime'}, $xover->{'mtime'});
+ $parent = $dbh->selectrow_hashref("SELECT thread_id FROM articles WHERE subjhash = ? AND received BETWEEN FROM_UNIXTIME(?) - INTERVAL 14 DAY AND FROM_UNIXTIME(?) ORDER BY received DESC LIMIT 1", undef, $subj_hash, $xover->{'mtime'}, $xover->{'mtime'});
+ }
+
+ # if no parent, try grouping the message with other very recent messages with the same subject
+ if (!$parent && $xover->{'Subject'}) {
+ $parent = $dbh->selectrow_hashref("SELECT thread_id FROM articles WHERE subjhash = ? AND received BETWEEN FROM_UNIXTIME(?) - INTERVAL 3 DAY AND FROM_UNIXTIME(?) ORDER BY received DESC LIMIT 1", undef, $subj_hash, $xover->{'mtime'}, $xover->{'mtime'});
}
my ($message_id) = md5_hex($xover->{'Message-ID'} =~ m/<(.+?)>/);
@@ -149,8 +155,8 @@ for $group (@ARGV) {
$message_id,
$subj_hash,
$from_hash,
- $parent->{'thread'} || $num,
- $parent->{'art'} || 0,
+ $parent->{'thread_id'} || $num,
+ $parent->{'id'} || 0,
$xover->{'mtime'},
$xover->{'Date'} || "",
$xover->{'Message-ID'} || "",
@@ -167,7 +173,8 @@ for $group (@ARGV) {
sub clean_subject {
my $subj = shift;
- $subj =~ s/^(Re|An|Antwort|Aw)(\^\d+|\[\d+\]|\(\d+\))?:\s*//i;
+ my $i;
+ while ($subj =~ s/^(Re|An|Antwort|Aw)(\^\d+|\[\d+\]|\(\d+\))?:\s*//i) { last if $i++ > 5 };
$subj =~ s/\s//g;
return lc $subj;
}
View
@@ -1,18 +1,20 @@
+DROP TABLE IF EXISTS groups;
CREATE TABLE groups (
id smallint unsigned not null primary key,
name varchar(255) not null,
+ description varchar(255) not null,
unique key (name)
-) ENGINE=MyISAM;
+) ENGINE=InnoDB DEFAULT CHARSET=utf8;
-DROP TABLE IF EXISTS header;
-CREATE TABLE header (
- grp smallint(5) unsigned NOT NULL default '0',
- art int(10) unsigned NOT NULL default '0',
+DROP TABLE IF EXISTS articles;
+CREATE TABLE articles (
+ group_id smallint(5) unsigned NOT NULL default '0',
+ id int(10) unsigned NOT NULL default '0',
msgid varchar(32) NOT NULL default '',
subjhash varchar(32) NOT NULL default '',
fromhash varchar(32) NOT NULL default '',
- thread int(10) unsigned NOT NULL default '0',
+ thread_id int(10) unsigned NOT NULL default '0',
parent int(10) unsigned NOT NULL default '0',
received datetime NOT NULL default '0000-00-00 00:00:00',
h_date varchar(255) NOT NULL default '',
@@ -22,12 +24,13 @@ CREATE TABLE header (
h_references varchar(255) NOT NULL default '',
h_lines mediumint(8) unsigned NOT NULL default '0',
h_bytes int(10) unsigned NOT NULL default '0',
- PRIMARY KEY (grp,art),
+ PRIMARY KEY (group_id,id),
KEY msgid (msgid),
KEY fromhash (fromhash),
- KEY grp (grp,received),
- KEY grp_2 (grp,thread,parent),
- KEY grp_3 (grp,subjhash),
- KEY subjhash ( subjhash, received )
-) ENGINE=MyISAM DELAY_KEY_WRITE=1;
+ KEY grp (group_id,received),
+ KEY grp_2 (group_id,thread_id,parent),
+ KEY grp_3 (group_id,subjhash),
+ KEY subjhash ( subjhash, received ),
+ CONSTRAINT `articles_ibfk_1` FOREIGN KEY (`group_id`) REFERENCES `groups` (`id`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8;
View
@@ -1,3 +1,3 @@
#!/bin/sh
-exec softlimit -m 30000000 \
+exec softlimit -m 20000000 \
tcpserver -X -x rules.cdb -c 60 -vhR -u 515 -g 514 0 119 /pkg/bin/perl ./colobus 2>&1

0 comments on commit ec6c11e

Please sign in to comment.