Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Move some common database import subroutines to Script::Utils #3200

Draft
wants to merge 2 commits into
base: master
Choose a base branch
from
Draft

Move some common database import subroutines to Script::Utils #3200

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
fe40684
Add `is_table_empty` to `Script::Utils`
mwiencek Feb 25, 2024
2b06d5d
Add `copy_table_from_file` to `Script::Utils`
mwiencek Feb 25, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
123 changes: 18 additions & 105 deletions admin/MBImport.pl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,14 +10,17 @@
use Getopt::Long;
use DBDefs;
use Sql;
use MusicBrainz::Script::Utils qw(
copy_table_from_file
is_table_empty
);
use MusicBrainz::Server::Replication qw( :replication_type );
use MusicBrainz::Server::Constants qw( @FULL_TABLE_LIST );

use aliased 'MusicBrainz::Server::DatabaseConnectionFactory' => 'Databases';

my ($fHelp, $fIgnoreErrors);
my $tmpdir = '/tmp';
my $fProgress = -t STDOUT;
my $fFixUTF8 = 0;
my $skip_ensure_editor = 0;
my $update_replication_control = 1;
Expand Down Expand Up @@ -214,115 +217,25 @@ sub usage

exit($errors ? 1 : 0);



sub ImportTable
{
my ($table, $file) = @_;

print localtime() . " : load $table\n";

my $rows = 0;

my $t1 = [gettimeofday];
my $interval;

my $size = -s($file)
or return 1;

my $p = sub {
my ($pre, $post) = @_;
no integer;
printf $pre.'%-30.30s %9d %3d%% %9d'.$post,
$table, $rows, int(100 * tell(LOAD) / $size),
$rows / ($interval||1);
};

$OUTPUT_AUTOFLUSH = 1;

eval
{
# open in :bytes mode (always keep byte octets), to allow fixing of invalid
# UTF-8 byte sequences in --fix-broken-utf8 mode.
# in default mode, the Pg driver will take care of the UTF-8 transformation
# and croak on any invalid UTF-8 character
open(LOAD, '<:bytes', $file) or die "open $file: $OS_ERROR";

# If you're looking at this code because your import failed, maybe
# with an error like this:
# ERROR: copy: line 1, Missing data for column "automodsaccepted"
# then the chances are it's because the data you're trying to load
# doesn't match the structure of the database you're trying to load it
# into. Please make sure you've got the right copy of the server
# code, as described in the INSTALL file.

$sql->begin;
$sql->do("DELETE FROM $table") if $delete_first;
my $dbh = $sql->dbh; # issues a ping, must be done before COPY
$sql->do("COPY $table FROM stdin");

$p->('', '') if $fProgress;
my $t;

use Encode;
while (<LOAD>)
{
$t = $_;
if ($fFixUTF8) {
# replaces any invalid UTF-8 character with special 0xFFFD codepoint
# and warn on any such occurence
$t = Encode::decode('UTF-8', $t, Encode::FB_DEFAULT | Encode::WARN_ON_ERR);
} else {
$t = Encode::decode('UTF-8', $t, Encode::FB_CROAK);
}
if (!$dbh->pg_putcopydata($t))
{
print 'ERROR while processing: ', $t;
die;
}

++$rows;
unless ($rows & 0xFFF)
{
$interval = tv_interval($t1);
$p->("\r", '') if $fProgress;
}
}
$dbh->pg_putcopyend() or die;
$interval = tv_interval($t1);
$p->(($fProgress ? "\r" : ''), sprintf(" %.2f sec\n", $interval));

close LOAD
or die $OS_ERROR;

$sql->commit;

die 'Error loading data'
if -f $file and empty($table);
my $rows = copy_table_from_file(
$sql, $table, $file,
delete_first => $delete_first,
fix_utf8 => $fFixUTF8,
ignore_errors => $fIgnoreErrors,
);

if ($rows) {
++$tables;
$totalrows += $rows;

1;
};

return 1 unless $EVAL_ERROR;
warn "Error loading $file: $EVAL_ERROR";
$sql->rollback;

++$errors, return 0 if $fIgnoreErrors;
exit 1;
}

sub empty
{
my $table = shift;

my $any = $sql->select_single_value(
"SELECT 1 FROM $table LIMIT 1",
);

not defined $any;
return 1;
} else {
++$errors;
return 0;
}
}

sub ImportAllTables
Expand All @@ -342,7 +255,7 @@ sub ImportAllTables
{
my $basetable = $1;

if (not empty($basetable) and not $delete_first)
if (not is_table_empty($sql, $basetable) and not $delete_first)
{
warn "$basetable table already contains data; skipping $table\n";
next;
Expand All @@ -352,7 +265,7 @@ sub ImportAllTables
ImportTable($basetable, $file) or next;

} else {
if (not empty($table) and not $delete_first)
if (not is_table_empty($sql, $table) and not $delete_first)
{
warn "$table already contains data; skipping\n";
next;
Expand Down
92 changes: 16 additions & 76 deletions admin/replication/ImportReplicationChanges
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,10 @@ use FindBin;
use lib "$FindBin::Bin/../../lib";

use Getopt::Long;
use MusicBrainz::Script::Utils qw(
copy_table_from_file
is_table_empty
);
use MusicBrainz::Server::Context;
use DBDefs;
use Sql;
Expand Down Expand Up @@ -104,83 +108,19 @@ sub ImportTable
{
my ($table, $file) = @_;

print localtime() . " : load $table\n";

my $rows = 0;

my $t1 = [gettimeofday];
my $interval;

my $size = -s($file) || 1;

my $p = sub {
my ($pre, $post) = @_;
no integer;
printf $pre.'%-30.30s %9d %3d%% %9d'.$post,
$table, $rows, int(100 * tell(LOAD) / $size),
$rows / ($interval||1);
};

$OUTPUT_AUTOFLUSH = 1;

eval
{
open(LOAD, '<:encoding(utf8)', $file) or die "open $file: $OS_ERROR";

$sql->begin;
my $dbh = $sql->dbh; # issues a ping, must be done before COPY
$sql->do("COPY $table FROM stdin");

$p->('', '');

while (<LOAD>)
{
$dbh->pg_putcopydata($_) or die;

++$rows;
unless ($rows & 0xFFF)
{
$interval = tv_interval($t1);
$p->("\r", '');
}
}

$dbh->pg_putcopyend() or die;

$interval = tv_interval($t1);
$p->("\r", sprintf(" %.2f sec\n", $interval));

close LOAD
or die $OS_ERROR;

$sql->commit;

die 'Error loading data'
if -f $file and empty($table);
my $rows = copy_table_from_file(
$sql, $table, $file,
ignore_errors => $fIgnoreErrors,
);

if ($rows) {
++$tables;
$totalrows += $rows;

1;
};

return 1 unless $EVAL_ERROR;
warn "Error loading $file: $EVAL_ERROR";
$sql->rollback;

++$errors, return 0 if $fIgnoreErrors;
exit 1;
}

sub empty
{
my $table = shift;

my $any = $sql->select_single_value(
"SELECT 1 FROM $table LIMIT 1",
);

not defined $any;
return 1;
} else {
++$errors;
return 0;
}
}

sub ImportReplicationTables
Expand All @@ -197,7 +137,7 @@ sub ImportReplicationTables
my $file = find_file($table);
$file or print("No data file found for '$table', skipping\n"), die;

if (not empty($table))
if (not is_table_empty($sql, $table))
{
die "$table already contains data; skipping\n";
next;
Expand Down Expand Up @@ -226,7 +166,7 @@ sub ImportDBMirror2ReplicationTables {
}

my $qualified_table = "dbmirror2.$table";
if (!empty($qualified_table)) {
if (!is_table_empty($sql, $qualified_table)) {
die "$qualified_table already contains data";
}

Expand Down
Loading