Permalink
Browse files

Much more cleanup and MySQL can now build the NGS

  • Loading branch information...
1 parent f9af9de commit 3864746f4f08e1d6cb5fe51e26db5e4b36efee67 Elliot Chance committed Jun 18, 2010
Showing with 329 additions and 113 deletions.
  1. +2 −0 .gitignore
  2. +2 −0 backend/.gitignore
  3. +64 −5 backend/example.pl
  4. +114 −47 backend/mysql.pl
  5. +46 −7 backend/postgresql.pl
  6. +2 −0 bin/.gitignore
  7. +2 −1 init.pl
  8. +2 −0 languages/.gitignore
  9. +2 −0 mbdump/.gitignore
  10. +1 −0 mbdump/blank.file
  11. +2 −0 plugins/.gitignore
  12. +2 −0 replication/.gitignore
  13. +16 −22 settings.pl
  14. +2 −0 src/.gitignore
  15. +67 −30 src/functions.pl
  16. +3 −1 temp/.gitignore
View
@@ -0,0 +1,2 @@
+.AppleDouble*
+:*
View
@@ -0,0 +1,2 @@
+.AppleDouble*
+:*
View
@@ -6,25 +6,67 @@
# this can be anything you like, but the name of the file you choose is very important to the name
# of the subroutines you have in this file.
#
-# If you want to implement your own backend duplicate this file and make the changes where
-# appropriate - but all subroutines whether they are used or not must stay in the file.
+# === Implementing Your Own Backend ===
+# Duplicate this file and make the changes where appropriate - but all subroutines whether they are
+# used or not must stay in the file.
+#
+# * mbz_do_sql($sql) should be used for any SQL that needs to execute a statement that does not need
+# to return a handle, such as INSERT/DELETE/UPDATE etc.
+# * $dbh is the active database handle that you can use with your own error handling.
#
-# mbz_update_index()
-sub backend_NAME_update_index {
+# mbz_connect()
+# Make database connection. It will set the global $dbh and it will return it. When implmenting your
+# own backend you should only really need to change the $driver to the correct perl DBI driver.
+# $g_db_name, $g_db_host, $g_db_port, $g_db_user and $g_db_pass are supplied by settings.pl.
+# @return $dbh
+sub backend_NAME_connect {
+ $driver = 'mysql';
+ $dbh = DBI->connect("dbi:$driver:dbname=$g_db_name;host=$g_db_host;port=$g_db_port",
+ $g_db_user, $g_db_pass);
+ return $dbh;
}
# mbz_update_schema()
+# @return 1 on success, otherwise 0.
sub backend_NAME_update_schema {
+ return 1;
+}
+
+
+# mbz_update_index()
+# @return 1 on success, otherwise 0.
+sub backend_NAME_update_index {
+ return 1;
}
-# mbz_table_exists($tablename)
+# mbz_table_exists($table_name)
# Check if a table already exists.
+# @param $table_name The name of the table to look for.
# @return 1 if the table exists, otherwise 0.
sub backend_NAME_table_exists {
+ my $table_name = $_[0];
+
+ # your code here
+
+ return 0;
+}
+
+
+# mbz_table_column_exists($table_name, $col_name)
+# Check if a table already has a column.
+# @param $table_name The name of the table to look for.
+# @param $col_name The column name in the table.
+# @return 1 if the table column exists, otherwise 0.
+sub backend_NAME_table_column_exists {
+ my ($table_name, $col_name) = @_;
+
+ # your code here
+
+ return 0;
}
@@ -34,5 +76,22 @@ sub backend_NAME_load_data {
}
+# mbz_create_extra_tables()
+# The mbzdb plugins use a basic key-value table to hold information such as settings.
+# @see mbz_set_key(), mbz_get_key().
+# @return Passthru from $dbh::do().
+sub backend_NAME_create_extra_tables {
+ # no need to if the table already exists
+ return 1 if(mbz_table_exists("kv"));
+
+ $sql = "CREATE TABLE kv (" .
+ "name varchar(255) not null primary key," .
+ "value text" .
+ ")";
+ $sql .= " tablespace $g_tablespace" if($g_tablespace ne "");
+ return mbz_do_sql($sql);
+}
+
+
# be nice
return 1;
View
@@ -12,41 +12,15 @@
# must stay in the file.
#
-# We can't always use the CreateIndexes.sql script provided by MusicBrainz because it has
-# PostgreSQL specific functions. Instead we use a cardinality calculation to determine the need for
-# an index.
-sub backend_mysql_update_index {
- # go through each table
- $sth = $dbh->prepare('show tables');
- $sth->execute();
- $start = time();
- while(@result = $sth->fetchrow_array()) {
- next if($result[0] eq $g_pending || $result[0] eq $g_pendingdata);
-
- print "Indexing $result[0]\n";
- $sth2 = $dbh->prepare("\\d \"" . $result[0] . "\"");
- $sth2->execute();
- while(@result2 = $sth2->fetchrow_array()) {
- $start2 = time();
- if($result2[3] eq "" && $result2[1] ne "text") {
- print " Calculating cardinality of $result2[0]... ";
- $sth_card = $dbh->prepare("select count(1)/(select count(1) from \"$result[0]\") ".
- "from (select distinct \"$result2[0]\" from \"$result[0]\") as t");
- $sth_card->execute();
- my @card = $sth_card->fetchrow_array();
- if($card[0] >= 0.01) {
- print "$card[0] (Yes)\n";
- print " Adding index $result[0].$result2[0]...";
- mbz_do_sql("create index $result[0]_" . $result2[0] .
- " on \"$result[0]\"(\"$result2[0]\")");
- print " Done (", mbz_format_time(time() - $start2), ", ",
- mbz_format_time(time() - $start), " total)\n";
- } else {
- print "$card[0] (No)\n";
- }
- }
- }
- }
+
+# mbz_connect()
+# Make database connection. It will set the global $dbh and it will return it.
+# $g_db_name, $g_db_host, $g_db_port, $g_db_user and $g_db_pass are supplied by settings.pl.
+# @return $dbh
+sub backend_mysql_connect {
+ $dbh = DBI->connect("dbi:mysql:dbname=$g_db_name;host=$g_db_host;port=$g_db_port",
+ $g_db_user, $g_db_pass);
+ return $dbh;
}
@@ -64,12 +38,11 @@ sub backend_mysql_update_index {
# to create (and replace) them. This is just so all the error messages and so nasty.
# @return Always 1.
sub backend_mysql_update_schema {
- print $L{'downloadschema'};
- mbz_download_schema();
- print $L{'done'} . "\n";
+ # TODO: this does not check for columns that have changed their type, as a column that already
+ # exists will be ignored. I'm not sure how important this is but its worth noting.
- # this is where it has to translate PostgreSQL to MySQL
- # as well as making any modifications needed.
+ # this is where it has to translate PostgreSQL to MySQL as well as making any modifications
+ # needed.
open(SQL, "temp/CreateTables.sql");
chomp(my @lines = <SQL>);
my $table = "";
@@ -85,7 +58,12 @@ sub backend_mysql_update_schema {
}
$table = mbz_trim($table);
print $L{'table'} . " $table\n";
- $stmt = "CREATE TABLE \"$table\" (dummycolumn int) tablespace $g_tablespace";
+
+ # do not create the table if it already exists
+ if(!mbz_table_exists($table)) {
+ $stmt = "CREATE TABLE `$table` (dummycolumn int)";
+ $stmt .= " tablespace $g_tablespace" if($g_tablespace ne '');
+ }
} elsif(substr($line, 0, 1) eq " " || substr($line, 0, 1) eq "\t") {
my @parts = split(" ", $line);
for($i = 0; $i < @parts; ++$i) {
@@ -112,15 +90,21 @@ sub backend_mysql_update_schema {
if(substr(reverse($parts[@parts - 1]), 0, 1) eq ",") {
$parts[@parts - 1] = substr($parts[@parts - 1], 0, length($parts[@parts - 1]) - 1);
}
+
next if($parts[0] eq "CHECK" || $parts[0] eq "CONSTRAINT" || $parts[0] eq "");
$parts[0] = mbz_remove_quotes($parts[0]);
- $stmt = "ALTER TABLE \"$table\" ADD \"$parts[0]\" " .
+ $stmt = "ALTER TABLE `$table` ADD `$parts[0]` " .
join(" ", @parts[1 .. @parts - 1]);
+
+ # no need to create the column if it already exists in the table
+ $stmt = "" if(mbz_table_column_exists($table, $parts[0]));
} elsif(substr($line, 0, 2) eq ");") {
- $stmt = "ALTER TABLE \"$table\" DROP dummycolumn";
+ if(mbz_table_column_exists($table, "dummycolumn")) {
+ $stmt = "ALTER TABLE `$table` DROP dummycolumn";
+ }
}
- if($stmt ne "") {
- # if this statement fails its hopefully because the field exists
+
+ if(mbz_trim($stmt) ne "") {
$dbh->do($stmt) or print "";
}
}
@@ -130,11 +114,77 @@ sub backend_mysql_update_schema {
}
-# backend_mysql_table_exists($tablename)
+# We can't always use the CreateIndexes.sql script provided by MusicBrainz because it has
+# PostgreSQL specific functions. Instead we use a cardinality calculation to determine the need for
+# an index.
+sub backend_mysql_update_index {
+ # go through each table
+ $sth = $dbh->prepare('show tables');
+ $sth->execute();
+ $start = time();
+ while(@result = $sth->fetchrow_array()) {
+ next if($result[0] eq $g_pending || $result[0] eq $g_pendingdata);
+
+ print "Indexing $result[0]\n";
+ $sth2 = $dbh->prepare("\\d \"" . $result[0] . "\"");
+ $sth2->execute();
+ while(@result2 = $sth2->fetchrow_array()) {
+ $start2 = time();
+ if($result2[3] eq "" && $result2[1] ne "text") {
+ print " Calculating cardinality of $result2[0]... ";
+ $sth_card = $dbh->prepare("select count(1)/(select count(1) from \"$result[0]\") ".
+ "from (select distinct \"$result2[0]\" from \"$result[0]\") as t");
+ $sth_card->execute();
+ my @card = $sth_card->fetchrow_array();
+ if($card[0] >= 0.01) {
+ print "$card[0] (Yes)\n";
+ print " Adding index $result[0].$result2[0]...";
+ mbz_do_sql("create index $result[0]_" . $result2[0] .
+ " on \"$result[0]\"(\"$result2[0]\")");
+ print " Done (", mbz_format_time(time() - $start2), ", ",
+ mbz_format_time(time() - $start), " total)\n";
+ } else {
+ print "$card[0] (No)\n";
+ }
+ }
+ }
+ }
+}
+
+
+# backend_mysql_table_exists($table_name)
# Check if a table already exists.
+# @param $table_name The name of the table to look for.
# @return 1 if the table exists, otherwise 0.
sub backend_mysql_table_exists {
- # TODO: Incomplete
+ my $table_name = $_[0];
+
+ my $sth = $dbh->prepare('show tables');
+ $sth->execute();
+ while(@result = $sth->fetchrow_array()) {
+ return 1 if($result[0] eq $table_name);
+ }
+
+ # table was not found
+ return 0;
+}
+
+
+# mbz_table_column_exists($table_name, $col_name)
+# Check if a table already has a column.
+# @param $table_name The name of the table to look for.
+# @param $col_name The column name in the table.
+# @return 1 if the table column exists, otherwise 0.
+sub backend_mysql_table_column_exists {
+ my ($table_name, $col_name) = @_;
+
+ my $sth = $dbh->prepare("describe `$table_name`");
+ $sth->execute();
+ while(@result = $sth->fetchrow_array()) {
+ return 1 if($result[0] eq $col_name);
+ }
+
+ # table column was not found
return 0;
}
@@ -146,5 +196,22 @@ sub backend_mysql_load_data {
}
+# backend_mysql_create_extra_tables()
+# The mbzdb plugins use a basic key-value table to hold information such as settings.
+# @see mbz_set_key(), mbz_get_key().
+# @return Passthru from $dbh::do().
+sub backend_mysql_create_extra_tables {
+ # no need to if the table already exists
+ return 1 if(mbz_table_exists("kv"));
+
+ $sql = "CREATE TABLE kv (" .
+ "name varchar(255) not null primary key," .
+ "value text" .
+ ")";
+ $sql .= " tablespace $g_tablespace" if($g_tablespace ne "");
+ return mbz_do_sql($sql);
+}
+
+
# be nice
return 1;
Oops, something went wrong.

0 comments on commit 3864746

Please sign in to comment.