Permalink
Browse files

Lots of changes to make it compatible with NGS replications, which sh…

…ould now be working. The schema is now built from ReplicationSetup.sql as well as CreateTables.sql. Update schema is a little smart in dodging CREATE INDEXs and also coping with primary and forign keys. The pending tables for NGS have been standardises as being called dbmiror_Pending and dbmirror_PendingData. NGS is now ready for testing (still only MySQL.)
  • Loading branch information...
1 parent 8dd4600 commit 8456864ea83f40f5bc85337178f79bb724cd17de Elliot Chance committed Jun 30, 2010
Showing with 68 additions and 45 deletions.
  1. +44 −29 backend/mysql.pl
  2. +0 −5 replication/.gitignore
  3. +0 −1 replication/blank.file
  4. +8 −3 settings.pl
  5. +16 −7 src/functions.pl
View
@@ -131,11 +131,11 @@ sub backend_mysql_load_pending {
# load Pending and PendingData
print localtime() . ": Loading pending tables... ";
mbz_do_sql(qq|
- LOAD DATA LOCAL INFILE 'replication/$id/mbdump/$g_pending'
+ LOAD DATA LOCAL INFILE 'replication/$id/mbdump/$g_pendingfile'
INTO TABLE `$g_pending`
|);
mbz_do_sql(qq|
- LOAD DATA LOCAL INFILE 'replication/$id/mbdump/$g_pendingdata'
+ LOAD DATA LOCAL INFILE 'replication/$id/mbdump/$g_pendingdatafile'
INTO TABLE `$g_pendingdata`
|);
print "Done\n";
@@ -266,34 +266,21 @@ sub backend_mysql_update_index {
}
-# backend_mysql_update_schema()
-# Attempt to update the scheme from the current version to a new version by creating a table with a
-# dummy field, altering the tables by adding one field at a time them removing the dummy field. The
-# idea is that given any schema and SQL file the new table fields will be added, the same fields
-# will result in an error and the table will be left unchanged and fields and tables that have been
-# removed from the new schema will not be removed from the current schema.
-# This is a crude way of doing it. The field order in each table after it's altered will not be
-# retained from the new schema however the field order should not have a big bearing on the usage
-# of the database because name based and column ID in scripts that use the database will remain the
-# same.
-# It would be nice if this subroutine had a makeover so that it would check items before attempting
-# to create (and replace) them. This is just so all the error messages and so nasty.
-# @return Always 1.
-sub backend_mysql_update_schema {
+sub backend_mysql_update_schema_from_file {
# TODO: this does not check for columns that have changed their type, as a column that already
# exists will be ignored. I'm not sure how important this is but its worth noting.
# this is where it has to translate PostgreSQL to MySQL as well as making any modifications
# needed.
- open(SQL, "replication/CreateTables.sql");
+ open(SQL, $_[0]);
chomp(my @lines = <SQL>);
my $table = "";
foreach my $line (@lines) {
# skip blank lines and single bracket lines
next if($line eq "" || $line eq "(" || substr($line, 0, 1) eq "\\");
my $stmt = "";
- if(substr($line, 0, 6) eq "CREATE") {
+ if(substr($line, 0, 6) eq "CREATE" && index($line, "INDEX") < 0) {
$table = mbz_remove_quotes(substr($line, 13, length($line)));
if(substr($table, length($table) - 1, 1) eq '(') {
$table = substr($table, 0, length($table) - 1);
@@ -318,25 +305,34 @@ sub backend_mysql_update_schema {
if(substr($parts[$i], length($parts[$i]) - 2, 2) eq "[]") {
$parts[$i] = "VARCHAR(255)";
}
- $parts[$i] = "INT NOT NULL" if(substr($parts[$i], 0, 6) eq "SERIAL");
- $parts[$i] = "CHAR(32)" if(substr($parts[$i], 0, 4) eq "UUID");
- $parts[$i] = "TEXT" if(substr($parts[$i], 0, 4) eq "CUBE");
- $parts[$i] = "CHAR(1)" if(substr($parts[$i], 0, 4) eq "BOOL");
- $parts[$i] = "VARCHAR(256)" if($parts[$i] eq "NAME");
- $parts[$i] = "0" if(substr($parts[$i], 0, 3) eq "NOW");
- $parts[$i] = "0" if(substr($parts[$i], 1, 1) eq "{");
- $parts[$i] = $parts[$i + 1] = $parts[$i + 2] = "" if($parts[$i] eq "WITH");
- if($parts[$i] eq "VARCHAR" && substr($parts[$i + 1], 0, 1) ne "(") {
+ if(uc(substr($parts[$i], 0, 7)) eq "VARCHAR" && index($line, '(') < 0) {
+ $parts[$i] = "TEXT";
+ }
+ $parts[$i] = "INT NOT NULL" if(uc(substr($parts[$i], 0, 6)) eq "SERIAL");
+ $parts[$i] = "CHAR(32)" if(uc(substr($parts[$i], 0, 4)) eq "UUID");
+ $parts[$i] = "TEXT" if(uc(substr($parts[$i], 0, 4)) eq "CUBE");
+ $parts[$i] = "CHAR(1)" if(uc(substr($parts[$i], 0, 4)) eq "BOOL");
+ #$parts[$i] = "VARCHAR(256)" if(uc($parts[$i]) eq "NAME");
+ $parts[$i] = "0" if(uc(substr($parts[$i], 0, 3)) eq "NOW");
+ $parts[$i] = "0" if(uc(substr($parts[$i], 1, 1)) eq "{");
+ $parts[$i] = $parts[$i + 1] = $parts[$i + 2] = "" if(uc($parts[$i]) eq "WITH");
+ if(uc($parts[$i]) eq "VARCHAR" && substr($parts[$i + 1], 0, 1) ne "(") {
$parts[$i] = "TEXT";
}
}
if(substr(reverse($parts[@parts - 1]), 0, 1) eq ",") {
$parts[@parts - 1] = substr($parts[@parts - 1], 0, length($parts[@parts - 1]) - 1);
}
- next if($parts[0] eq "CHECK" || $parts[0] eq "CONSTRAINT" || $parts[0] eq "");
+ next if(uc($parts[0]) eq "CHECK" || uc($parts[0]) eq "CONSTRAINT" || $parts[0] eq "");
$parts[0] = mbz_remove_quotes($parts[0]);
- $stmt = "ALTER TABLE `$table` ADD `$parts[0]` " .
+
+ if(uc($parts[0]) ne "PRIMARY" && uc($parts[0]) ne "FOREIGN") {
+ $new_col = "`$parts[0]`";
+ } else {
+ $new_col = $parts[0];
+ }
+ $stmt = "ALTER TABLE `$table` ADD $new_col " .
join(" ", @parts[1 .. @parts - 1]);
# no need to create the column if it already exists in the table
@@ -357,5 +353,24 @@ sub backend_mysql_update_schema {
}
+# backend_mysql_update_schema()
+# Attempt to update the scheme from the current version to a new version by creating a table with a
+# dummy field, altering the tables by adding one field at a time them removing the dummy field. The
+# idea is that given any schema and SQL file the new table fields will be added, the same fields
+# will result in an error and the table will be left unchanged and fields and tables that have been
+# removed from the new schema will not be removed from the current schema.
+# This is a crude way of doing it. The field order in each table after it's altered will not be
+# retained from the new schema however the field order should not have a big bearing on the usage
+# of the database because name based and column ID in scripts that use the database will remain the
+# same.
+# It would be nice if this subroutine had a makeover so that it would check items before attempting
+# to create (and replace) them. This is just so all the error messages and so nasty.
+# @return Always 1.
+sub backend_mysql_update_schema {
+ backend_mysql_update_schema_from_file("replication/CreateTables.sql");
+ backend_mysql_update_schema_from_file("replication/ReplicationSetup.sql");
+}
+
+
# be nice
return 1;
View
@@ -1,5 +0,0 @@
-.AppleDouble*
-:*
-!blank.file
-!.gitignore
-
View
@@ -1 +0,0 @@
-blank.file
View
@@ -16,10 +16,10 @@
$g_db_pass = 'abcd1234';
# The name of the database to use.
-$g_db_name = 'mbzdb_old';
+$g_db_name = 'mbzdb_ngs';
# Use NGS (Next Generation Schema)?
-$g_use_ngs = 0;
+$g_use_ngs = 1;
#############################
@@ -70,10 +70,11 @@
$g_index_url = "$schema_base;f=admin/sql/CreateIndexes.sql;hb=$hb";
$g_pk_url = "$schema_base;f=admin/sql/CreatePrimaryKeys.sql;hb=$hb";
$g_func_url = "$schema_base;f=admin/sql/CreateFunctions.sql;hb=$hb";
+$g_pending_url = "$schema_base;f=admin/sql/ReplicationSetup.sql;hb=$hb";
# Replications URLs
-# TODO: add the URLs for NGS when they are available.
if($g_use_ngs) {
+ $g_rep_url = "http://test.musicbrainz.org:82/pub/musicbrainz/data/replication";
} else {
$g_rep_url = "ftp://ftp.musicbrainz.org/pub/musicbrainz/data/replication";
}
@@ -106,9 +107,13 @@
if($g_use_ngs) {
$g_pending = 'dbmirror_Pending';
$g_pendingdata = 'dbmirror_PendingData';
+ $g_pendingfile = 'dbmirror_pending';
+ $g_pendingdatafile = 'dbmirror_pendingdata';
} else {
$g_pending = 'Pending';
$g_pendingdata = 'PendingData';
+ $g_pendingfile = 'Pending';
+ $g_pendingdatafile = 'PendingData';
}
return 1;
View
@@ -145,6 +145,8 @@ sub mbz_download_schema {
mbz_download_file($g_pk_url, "replication/CreatePrimaryKeys.sql");
unlink("replication/CreateFunctions.sql");
mbz_download_file($g_func_url, "replication/CreateFunctions.sql");
+ unlink("replication/ReplicationSetup.sql");
+ mbz_download_file($g_pending_url, "replication/ReplicationSetup.sql");
return 1;
}
@@ -386,19 +388,18 @@ sub mbz_raw_download {
or die "Can't change directory (ftp.musicbrainz.org): " . $ftp->message;
my @files = (
- # TODO: turn this back on when finished testing.
- #'mbdump-derived.tar.bz2',
- 'mbdump-stats.tar.bz2'
- #'mbdump.tar.bz2'
+ 'mbdump-derived.tar.bz2',
+ 'mbdump-stats.tar.bz2',
+ 'mbdump.tar.bz2'
);
} else {
# find out the latest fullexport
my $latest = "";
$ftp = Net::FTP->new('ftp.musicbrainz.org', Timeout => 60)
or die "Cannot contact ftp.musicbrainz.org: $!";
- $ftp->login('anonymous') or die "Can't login (ftp.musicbrainz.org): " . $ftp->message;
+ $ftp->login('anonymous') or die("Can't login (ftp.musicbrainz.org): " . $ftp->message);
$ftp->cwd('/pub/musicbrainz/data/fullexport/')
- or die "Can't change directory (ftp.musicbrainz.org): " . $ftp->message;
+ or die("Can't change directory (ftp.musicbrainz.org): " . $ftp->message);
my @ls = $ftp->ls('-lR');
foreach my $l (@ls) {
if(index($l, 'latest-is-') >= 0) {
@@ -527,13 +528,21 @@ sub mbz_run_transactions {
my ($key, $data);
for(my $rows = 1; @rep_row = $rep_handle->fetchrow_array(); ) {
# next if we are ignoring this table
- my $tableName = substr($rep_row[1], 10, length($rep_row[1]) - 11);
+ my $tableName = "";
+ if($g_use_ngs) {
+ $tableName = substr($rep_row[1], 15, length($rep_row[1]) - 16);
+ } else {
+ $tableName = substr($rep_row[1], 10, length($rep_row[1]) - 11);
+ }
if(mbz_in_array(\@g_ignore_tables, $tableName)) {
++$rows if(($rep_row[5] eq '0' || $rep_row[5] eq 'f') || $rep_row[2] eq 'd');
mbz_do_sql("DELETE FROM $pending WHERE SeqId='$rep_row[0]'");
mbz_do_sql("DELETE FROM $pendingdata WHERE SeqId='$rep_row[0]'");
next;
}
+
+ # also ignore any table that starts with "nz"
+ next if(substr($tableName, 0, 2) eq "nz");
# we use '1' and 't' for MySQL and PostgreSQL
$key = mbz_unpack_data($rep_row[6]) if($rep_row[5] eq '1' or $rep_row[5] eq 't');

0 comments on commit 8456864

Please sign in to comment.