Permalink
Browse files

Lots of all round changes to bring it more inline with the recent NGS…

… changes. Various minor bug fixes.
  • Loading branch information...
Elliot Chance
Elliot Chance committed Dec 10, 2010
1 parent 09952ae commit 1b8d0e7c10f6bfcd882cf3385676ccfe8487a275
Showing with 100 additions and 76 deletions.
  1. +1 −2 .gitignore
  2. +56 −25 backend/postgresql.pl
  3. +2 −6 plugins/livestats.pl
  4. +15 −7 plugins/pendinglog.pl
  5. +10 −21 settings.pl
  6. +1 −1 src/firstboot.pl
  7. +14 −13 src/functions.pl
  8. +1 −1 update.pl
View
@@ -4,5 +4,4 @@ cloc.pl
sloc.sh
mbdump*
replication/*
-settings.pl
-src/firstboot.pl
+._*
View
@@ -20,21 +20,22 @@ sub backend_postgresql_update_index {
# language or functions fail they will ultimatly be skipped.
# for PostgreSQL we need to try CREATE LANGUAGE
- if($g_db_rdbms eq 'postgresql') {
- mbz_do_sql("CREATE LANGUAGE plpgsql");
- }
+ backend_postgresql_create_plpgsql() if($g_db_rdbms eq 'postgresql');
open(SQL, "replication/CreateFunctions.sql");
chomp(my @lines = <SQL>);
my $full = "";
foreach my $line (@lines) {
# skip blank lines and single bracket lines
- next if($line eq "" || substr($line, 0, 2) eq "--" || substr($line, 0, 1) eq "\\");
+ next if($line eq "" || substr($line, 0, 2) eq "--" || substr($line, 0, 1) eq "\\" ||
+ $line eq "BEGIN;" || $line eq "COMMIT;");
$full .= "$line\n";
if(index($line, 'plpgsql') > 0) {
#print "$full\n";
- mbz_do_sql($full);
+ mbz_do_sql("begin");
+ mbz_do_sql($full, 'nodie');
+ mbz_do_sql("commit");
$full = "";
}
}
@@ -47,8 +48,14 @@ sub backend_postgresql_update_index {
next if($line eq "" || substr($line, 0, 2) eq "--" || substr($line, 0, 1) eq "\\" ||
substr($line, 0, 5) eq "BEGIN");
+ # no need to create the index if it already exists
+ my $pos_index = index($line, 'INDEX ');
+ my $index_name = mbz_trim(substr($line, $pos_index + 6, index($line, ' ', $pos_index + 7) -
+ $pos_index - 6));
+ next if(backend_postgresql_index_exists($index_name));
+
print "$line\n";
- mbz_do_sql($line);
+ mbz_do_sql($line, 'nodie');
}
close(SQL);
@@ -59,8 +66,14 @@ sub backend_postgresql_update_index {
next if($line eq "" || substr($line, 0, 2) eq "--" || substr($line, 0, 1) eq "\\" ||
substr($line, 0, 5) eq "BEGIN");
+ # no need to create the index if it already exists
+ my $pos_index = index($line, 'CONSTRAINT ');
+ my $index_name = mbz_trim(substr($line, $pos_index + 11, index($line, ' ', $pos_index + 12) -
+ $pos_index - 11));
+ next if(backend_postgresql_index_exists($index_name));
+
print "$line\n";
- mbz_do_sql($line);
+ mbz_do_sql($line, 'nodie');
}
close(SQL);
}
@@ -79,16 +92,16 @@ sub backend_postgresql_update_index {
# It would be nice if this subroutine had a makeover so that it would check items before attempting
# to create (and replace) them. This is just so all the error messages and so nasty.
# @return Always 1.
-sub backend_postgresql_update_schema {
- open(SQL, "replication/CreateTables.sql");
+sub backend_postgresql_update_schema_file {
+ open(SQL, $_[0]);
chomp(my @lines = <SQL>);
my $table = "";
foreach my $line (@lines) {
# skip blank lines and single bracket lines
next if($line eq "" || $line eq "(" || substr($line, 0, 1) eq "\\");
my $stmt = "";
- if(substr($line, 0, 6) eq "CREATE") {
+ if(substr($line, 0, 12) eq "CREATE TABLE") {
$table = mbz_remove_quotes(substr($line, 13, length($line)));
if(substr($table, length($table) - 1, 1) eq '(') {
$table = substr($table, 0, length($table) - 1);
@@ -108,7 +121,7 @@ sub backend_postgresql_update_schema {
# because the original MusicBrainz database is PostgreSQL we only need to make
# minimal changes to the SQL.
- if(substr($parts[$i], 0, 4) eq "CUBE" && !$g_contrib_cube) {
+ if(uc(substr($parts[$i], 0, 4)) eq "CUBE" && !$g_contrib_cube) {
$parts[$i] = "TEXT";
}
}
@@ -133,6 +146,12 @@ sub backend_postgresql_update_schema {
}
+sub backend_postgresql_update_schema {
+ backend_postgresql_update_schema_file("replication/CreateTables.sql");
+ backend_postgresql_update_schema_file("replication/ReplicationSetup.sql");
+}
+
+
# backend_postgresql_table_exists($tablename)
# Check if a table already exists.
# @note This must support searching for VIEWs as well. mbz_table_exists() is used for testing if
@@ -234,11 +253,13 @@ sub backend_postgresql_create_extra_tables {
# @param $index_name The name of the index to look for.
# @return 1 if the index exists, otherwise 0.
sub backend_postgresql_index_exists {
- my $index_name = $_[0];
-
- # TODO: incomplete
-
- return 0;
+ my $sth = $dbh->prepare("select count(*) from (".
+ "select constraint_name from information_schema.key_column_usage ".
+ "where constraint_name='$_[0]' union all select indexname from pg_indexes ".
+ "where indexname='$_[0]') as t");
+ $sth->execute();
+ my $result = $sth->fetchrow_hashref();
+ return $result->{'count'};
}
@@ -249,32 +270,34 @@ sub backend_postgresql_index_exists {
# @return Always 1.
sub backend_postgresql_load_pending {
$id = $_[0];
+ my $pending = mbz_escape_entity($g_pending);
+ my $pendingdata = mbz_escape_entity($g_pendingdata);
# make sure there are no pending transactions before cleanup
- $temp = $dbh->prepare("SELECT count(1) FROM $g_pending");
+ $temp = $dbh->prepare("SELECT count(1) FROM $pending");
$temp->execute;
@row = $temp->fetchrow_array();
$temp->finish;
return -1 if($row[0] ne '0');
# perform cleanup (makes sure there no left over records in the PendingData table)
- $dbh->do("DELETE FROM $g_pending");
+ $dbh->do("DELETE FROM $pending");
# load Pending and PendingData
print localtime() . ": Loading pending tables... ";
- open(TABLEDUMP, "replication/$id/mbdump/$g_pending")
- or warn("Error: cannot open file 'replication/$id/mbdump/$g_pending'\n");
- $dbh->do("COPY $g_pending FROM STDIN");
+ open(TABLEDUMP, "replication/$id/mbdump/$g_pendingfile")
+ or warn("Error: cannot open file 'replication/$id/mbdump/$g_pendingfile'\n");
+ $dbh->do("COPY $pending FROM STDIN");
while($readline = <TABLEDUMP>) {
$dbh->pg_putcopydata($readline);
}
close(TABLEDUMP);
$dbh->pg_putcopyend();
- open(TABLEDUMP, "replication/$id/mbdump/$g_pendingdata")
- or warn("Error: cannot open file 'replication/$id/mbdump/$g_pendingdata'\n");
- $dbh->do("COPY $g_pendingdata FROM STDIN");
+ open(TABLEDUMP, "replication/$id/mbdump/$g_pendingdatafile")
+ or warn("Error: cannot open file 'replication/$id/mbdump/$g_pendingdatafile'\n");
+ $dbh->do("COPY $pendingdata FROM STDIN");
while($readline = <TABLEDUMP>) {
$dbh->pg_putcopydata($readline);
}
@@ -296,11 +319,19 @@ sub backend_postgresql_load_pending {
# Wnen dealing with table and column names that contain upper and lowercase letters some databases
# require the table name to be encapsulated. PostgreSQL uses double-quotes.
# @return A new encapsulated entity.
-sub backend_mysql_escape_entity {
+sub backend_postgresql_escape_entity {
my $entity = $_[0];
return "\"$entity\"";
}
+sub backend_postgresql_create_plpgsql {
+ my $sth = $dbh->prepare("SELECT count(*) FROM pg_catalog.pg_language WHERE lanname='plpgsql'");
+ $sth->execute();
+ my @row = $sth->fetchrow_array();
+ mbz_do_sql("CREATE LANGUAGE plpgsql") if($row[0] == 0)
+}
+
+
# be nice
return 1;
View
@@ -20,9 +20,7 @@ sub livestats_init {
}
# for PostgreSQL we need to CREATE LANGUAGE
- if($g_db_rdbms eq 'postgresql') {
- mbz_do_sql("CREATE LANGUAGE plpgsql");
- }
+ backend_postgresql_create_plpgsql() if($g_db_rdbms eq 'postgresql');
# create tables
print "Creating livestats tables and views...";
@@ -82,9 +80,7 @@ sub livestats_init {
while(@result = $sth->fetchrow_array()) {
if($result[0] ne "livestats") {
print " Counting records for table $result[0]... ";
- $table = $result[0];
- $table = "\"$table\"" if($g_db_rdbms eq 'postgresql');
- $table = "`$table`" if($g_db_rdbms eq 'mysql');
+ $table = mbz_escape_entity($result[0]);
# create the key if it doesn't exist
my $sth2 = $dbh->prepare("select count(1) from livestats where name='count.$result[0]'");
View
@@ -38,18 +38,26 @@ sub pendinglog_init {
sub pendinglog_beforereplication {
my ($repID) = @_;
+ my $pending = mbz_escape_entity($g_pending);
+ my $pendingdata = mbz_escape_entity($g_pendingdata);
+ my $seqid = mbz_escape_entity("SeqId");
+ my $iskey = mbz_escape_entity("IsKey");
+ my $tablename = mbz_escape_entity("TableName");
+ my $op = mbz_escape_entity("Op");
+ my $xid = mbz_escape_entity("XID");
+ my $data = mbz_escape_entity("Data");
mbz_do_sql(qq|
INSERT INTO pendinglog
- SELECT $g_pending.SeqId, '$repID',
- substring(TableName from 11 for length(TableName) - 11) as TableName, Op, XID, P1.Data,
- P2.Data as keyclause
- FROM $g_pending
- LEFT JOIN $g_pendingdata as P1 on $g_pending.SeqId=P1.SeqId and P1.IsKey='f'
- LEFT JOIN $g_pendingdata as P2 on $g_pending.SeqId=P2.SeqId and P2.IsKey='t'
+ SELECT $pending.$seqid, '$repID',
+ substring($tablename from 11 for length($tablename) - 11) as $tablename, $op, $xid, P1.$data,
+ P2.$data as keyclause
+ FROM $pending
+ LEFT JOIN $pendingdata as P1 on $pending.$seqid=P1.$seqid and P1.$iskey='f'
+ LEFT JOIN $pendingdata as P2 on $pending.$seqid=P2.$seqid and P2.$iskey='t'
|);
mbz_do_sql(qq|
- UPDATE livestats set val=val+(select count(1) from $g_pending)
+ UPDATE livestats set val=val+(select count(1) from $pending)
where name='count.pendinglog'
|);
return 1;
View
@@ -7,16 +7,16 @@
#############################
# Must be 'mysql' or 'postgresql'
-$g_db_rdbms = 'mysql';
+$g_db_rdbms = 'postgresql';
# Database user name.
-$g_db_user = 'root';
+$g_db_user = 'ngsdb';
# Database user password.
-$g_db_pass = 'abcd1234';
+$g_db_pass = 'ngsdb';
# The name of the database to use.
-$g_db_name = 'mbzdb_ngs';
+$g_db_name = 'ngsdb';
# Use NGS (Next Generation Schema)?
$g_use_ngs = 1;
@@ -72,12 +72,8 @@
$g_func_url = "$schema_base;f=admin/sql/CreateFunctions.sql;hb=$hb";
$g_pending_url = "$schema_base;f=admin/sql/ReplicationSetup.sql;hb=$hb";
-# Replications URLs
-if($g_use_ngs) {
- $g_rep_url = "http://test.musicbrainz.org:82/pub/musicbrainz/data/replication";
-} else {
- $g_rep_url = "ftp://ftp.musicbrainz.org/pub/musicbrainz/data/replication";
-}
+# Replications URL
+$g_rep_url = "http://test.musicbrainz.org:82/pub/musicbrainz/data/replication";
# Kill the update script if a duplicate error (i.e. a duplicate unique key) occurs. It is
# recommended you leave this at 0.
@@ -104,16 +100,9 @@
$g_db_port = mbz_get_default_port($g_db_rdbms) if($g_db_port eq 'default');
-if($g_use_ngs) {
- $g_pending = 'dbmirror_Pending';
- $g_pendingdata = 'dbmirror_PendingData';
- $g_pendingfile = 'dbmirror_pending';
- $g_pendingdatafile = 'dbmirror_pendingdata';
-} else {
- $g_pending = 'Pending';
- $g_pendingdata = 'PendingData';
- $g_pendingfile = 'Pending';
- $g_pendingdatafile = 'PendingData';
-}
+$g_pending = 'dbmirror_Pending';
+$g_pendingdata = 'dbmirror_PendingData';
+$g_pendingfile = 'dbmirror_pending';
+$g_pendingdatafile = 'dbmirror_pendingdata';
return 1;
View
@@ -1,6 +1,6 @@
# First boot
$g_chosenlanguage = 0;
-$g_firstboot = 0;
+$g_firstboot = 1;
# Language
$g_language = 'English';
View
@@ -91,7 +91,9 @@ sub mbz_create_extra_tables {
# @param $sql The SQL statement to be executed.
# @return Passthru from $dbh::do().
sub mbz_do_sql {
- return $dbh->do($_[0]) or mbz_sql_error($dbh->errstr, $_[0]);
+ my $result = $dbh->do($_[0]);
+ $result or mbz_sql_error($dbh->errstr, $_[0]) if($_[1] ne 'nodie');
+ return $result;
}
@@ -521,11 +523,13 @@ sub mbz_round {
sub mbz_run_transactions {
my $pending = mbz_escape_entity($g_pending);
my $pendingdata = mbz_escape_entity($g_pendingdata);
+ my $seqid = mbz_escape_entity("SeqId");
+ my $iskey = mbz_escape_entity("IsKey");
my $rep_handle = $dbh->prepare(qq|
SELECT * from $pending
- LEFT JOIN $pendingdata ON $pending.SeqId=$pendingdata.SeqId
- ORDER BY $pending.SeqId, IsKey desc
+ LEFT JOIN $pendingdata ON $pending.$seqid=$pendingdata.$seqid
+ ORDER BY $pending.$seqid, $iskey desc
|);
$rep_handle->execute();
my $totalreps = mbz_get_count($g_pending);
@@ -535,16 +539,13 @@ sub mbz_run_transactions {
my ($key, $data);
for(my $rows = 1; @rep_row = $rep_handle->fetchrow_array(); ) {
# next if we are ignoring this table
- my $tableName = "";
- if($g_use_ngs) {
- $tableName = substr($rep_row[1], 15, length($rep_row[1]) - 16);
- } else {
- $tableName = substr($rep_row[1], 10, length($rep_row[1]) - 11);
- }
+ my $pos = index($rep_row[1], '.');
+ my $tableName = substr($rep_row[1], $pos + 2, length($rep_row[1]) - $pos - 3);
+
if(mbz_in_array(\@g_ignore_tables, $tableName)) {
++$rows if(($rep_row[5] eq '0' || $rep_row[5] eq 'f') || $rep_row[2] eq 'd');
- mbz_do_sql("DELETE FROM $pending WHERE SeqId='$rep_row[0]'");
- mbz_do_sql("DELETE FROM $pendingdata WHERE SeqId='$rep_row[0]'");
+ mbz_do_sql("DELETE FROM $pending WHERE $seqid='$rep_row[0]'");
+ mbz_do_sql("DELETE FROM $pendingdata WHERE $seqid='$rep_row[0]'");
next;
}
@@ -597,8 +598,8 @@ sub mbz_run_transactions {
}
# clear for next round
- mbz_do_sql("DELETE FROM $pending WHERE SeqId='$rep_row[0]'");
- mbz_do_sql("DELETE FROM $pendingdata WHERE SeqId='$rep_row[0]'");
+ mbz_do_sql("DELETE FROM $pending WHERE $seqid='$rep_row[0]'");
+ mbz_do_sql("DELETE FROM $pendingdata WHERE $seqid='$rep_row[0]'");
undef($key);
undef($data);
++$rows;
View
@@ -74,7 +74,7 @@
}
# FIND IF THERE ARE PENDING TRANSACTIONS
-$sth = $dbh->prepare("SELECT count(1) from $g_pending");
+$sth = $dbh->prepare("SELECT count(1) from " . mbz_escape_entity($g_pending));
$sth->execute();
@row = $sth->fetchrow_array();
if($f_info) {

0 comments on commit 1b8d0e7

Please sign in to comment.