Multi-bug fix - SetMarcUnicodeFlag for records coming from Koha

This has bearing on bugs 2905, 2665, 2514 and other "wide character" crashes
related to diacritics and Unicode.  This should help open the door for reliable
input of diacriticals via acquisitions.

MARC21_utf8_flag_fix.pl diagnoses and fixes existing problems with MARC data
affected by the bug.

Adding SetMarcUnicodeFlag to TransformKohaToMarc prevents the bug from corrupting
further data.

Signed-off-by: Galen Charlton <galen.charlton@liblime.com>

C4/Biblio.pm

@@ -24,6 +24,7 @@ use MARC::Record;
 use MARC::File::USMARC;
 use MARC::File::XML;
 use ZOOM;
+use POSIX qw(strftime);
 
 use C4::Koha;
 use C4::Dates qw/format_date/;
@@ -1445,18 +1446,15 @@ sub GetPublisherNameFromIsbn($){
 =cut
 
 sub TransformKohaToMarc {
-
     my ( $hash ) = @_;
-    my $dbh = C4::Context->dbh;
-    my $sth =
-    $dbh->prepare(
+    my $sth = C4::Context->dbh->prepare(
         "SELECT tagfield,tagsubfield FROM marc_subfield_structure WHERE frameworkcode=? AND kohafield=?"
     );
     my $record = MARC::Record->new();
+    SetMarcUnicodeFlag($record, C4::Context->preference("marcflavour"));
     foreach (keys %{$hash}) {
-        &TransformKohaToMarcOneField( $sth, $record, $_,
-            $hash->{$_}, '' );
-        }
+        &TransformKohaToMarcOneField( $sth, $record, $_, $hash->{$_}, '' );
+    }
     return $record;
 }
 
@@ -1618,7 +1616,6 @@ sub TransformHtmlToXml {
     $xml .= "</datafield>\n" if @$tags > 0;
     if (C4::Context->preference('marcflavour') eq 'UNIMARC' and !$unimarc_and_100_exist) {
 #     warn "SETTING 100 for $auth_type";
-        use POSIX qw(strftime);
         my $string = strftime( "%Y%m%d", localtime(time) );
         # set 50 to position 26 is biblios, 13 if authorities
         my $pos=26;

C4/Charset.pm

@@ -25,7 +25,7 @@ use vars qw($VERSION @ISA @EXPORT @EXPORT_OK %EXPORT_TAGS);
 
 BEGIN {
     # set the version for version checking
-    $VERSION = 3.00;
+    $VERSION = 3.01;
     require Exporter;
     @ISA    = qw(Exporter);
     @EXPORT = qw(
@@ -228,7 +228,7 @@ any actual character conversion.
 
 sub SetMarcUnicodeFlag {
     my $marc_record = shift;
-    my $marc_flavour = shift;
+    my $marc_flavour = shift; # || C4::Context->preference("marcflavour");
 
     $marc_record->encoding('UTF-8');
     if ($marc_flavour eq 'MARC21') {
@@ -241,6 +241,8 @@ sub SetMarcUnicodeFlag {
             substr($sfa, 26, 4) = '5050';
             $field->update('a' => $sfa);
         }
+    } else {
+        warn "Unrecognized marcflavour: $marc_flavour";
     }
 }
 

acqui/addorder.pl

@@ -165,17 +165,12 @@ =head1 CGI PARAMETERS
 my $invoice       = $input->param('invoice');
 my $publishercode = $input->param('publishercode');
 my $suggestionid  = $input->param('suggestionid');
-my $user          = $input->remote_user;
-
-#warn "CREATEBIBITEM =  $input->param('createbibitem')";
-#warn Dumper $input->param('createbibitem');
-my $createbibitem = $input->param('createbibitem');
+my $biblionumber  = $input->param('biblionumber');
 
 # create, modify or delete biblio
 # create if $quantity>=0 and $existing='no'
 # modify if $quantity>=0 and $existing='yes'
 # delete if $quantity has been set to 0 by the librarian
-my $biblionumber  = $input->param('biblionumber');
 my $bibitemnum;
 if ( $quantity ne '0' ) {
     #check to see if biblio exists

misc/maintenance/leader_fix.pl → misc/maintenance/MARC21_utf8_flag_fix.pl

@@ -72,7 +72,8 @@ BEGIN
     $verbose++;
 }
 
-my $marcflavour = C4::Context->preference('marcflavour');
+my $marcflavour = C4::Context->preference('marcflavour') or die "No marcflavour (MARC21 or UNIMARC) set in syspref";
+($marcflavour eq 'MARC21') or die "marcflavour must be MARC21, not $marcflavour";
 
 my $all = C4::Context->dbh->prepare("SELECT COUNT(*) FROM biblioitems");
 $all->execute;
@@ -100,8 +101,6 @@ BEGIN
 $limit += $offset if $offset;           # increase limit for offset
 my $i = 0;
 
-$marcflavour or die "No marcflavour (MARC21 or UNIMARC) set in syspref";
-
 MARC::File::XML->default_record_format($marcflavour) or die "FAILED MARC::File::XML->default_record_format($marcflavour)";
 
 while ( my $row = $bad_recs->fetchrow_hashref() ) {
@@ -120,7 +119,7 @@ BEGIN
         next;
     }
     if ($fix) {
-        $record->encoding('UTF-8');
+        SetMarcUnicodeFlag($record, $marcflavour);
         if (ModBiblioMarc($record, $row->{biblionumber})) {
             printf "# %4d of %4d: biblionumber %s : <leader>%s</leader>\n", $i, $badcount, $row->{biblionumber}, $record->leader();
         } else {
@@ -134,13 +133,13 @@ BEGIN
 
 =head1 NAME
 
-leader_fix.pl - Repair missing leader position 9 value ("a" for MARC21 - UTF8).
+MARC21_utf8_flag_fix.pl - Repair missing leader position 9 value ("a" for MARC21 - UTF8).
 
 =head1 SYNOPSIS
 
-leader_fix.pl [ -h | -m ] [ -v ] [ -d ] [ -s ] [ -l N ] [ -o N ] [ -f ]
+MARC21_utf8_flag_fix.pl [ -h | -m ] [ -v ] [ -d ] [ -s ] [ -l N ] [ -o N ] [ -f ]
 
- Help Options:
+ Help Optionsc
    -h --help -?   Brief help message
    -m --man       Full documentation, same as --help --verbose
       --version   Prints version info
@@ -188,25 +187,27 @@ =head1 DESCRIPTION
 blank for MARC8 encoding.  This script is used to determine the extent of the 
 problem and to fix the affected leaders.
 
-Run leader_fix.pl the first time with no options, and assuming you agree that the leaders
+As the name suggests, this script is only useful for MARC21 and will die for marcflavour UNIMARC.
+
+Run MARC21_utf8_flag_fix.pl the first time with no options, and assuming you agree that the leaders
 presented need fixing, run it again with B<--fix>.  
 
 =head1 USAGE EXAMPLES
 
-B<leader_fix.pl>
+B<MARC21_utf8_flag_fix.pl>
 
 In the most basic form, displays summary of biblioitems examined
 and the leader from any found without /09 = a.
 
-B<leader_fix.pl --fix>
+B<MARC21_utf8_flag_fix.pl --fix>
 
 Fixes the same biblioitems, displaying summary and each leader before/after change.
 
-B<leader_fix.pl --limit=3 --offset=15 --nosummary --dump>
+B<MARC21_utf8_flag_fix.pl --limit=3 --offset=15 --nosummary --dump>
 
 Dumps MARCXML from the 16th, 17th and 18th bad records found.
 
-B<leader_fix.pl -l 3 -o 15 -s 0 -d>
+B<MARC21_utf8_flag_fix.pl -l 3 -o 15 -s 0 -d>
 
 Same thing as previous example in terse form.