From aa54faaea2219dccf61bcea3b5a36ab4cf953bb8 Mon Sep 17 00:00:00 2001 From: "K. Andrew Parker" Date: Sun, 15 May 2022 11:35:43 -0400 Subject: [PATCH 01/12] support multiple libraries in OPL_pgfiles --- bin/OPL-update | 27 +++++++++++-------- .../ContentGenerator/Instructor/SetMaker.pm | 9 ++++--- 2 files changed, 22 insertions(+), 14 deletions(-) diff --git a/bin/OPL-update b/bin/OPL-update index 57da124686..9c7dbed638 100755 --- a/bin/OPL-update +++ b/bin/OPL-update @@ -129,9 +129,12 @@ $dbh->prepare("SET NAMES '$character_set'")->execute(); print "using character set $character_set to build OPL database\n"; -my $libraryRoot = $ce->{problemLibrary}->{root}; +my $libraryRoot = $ce->{problemLibrary}{root}; +my $contribRoot = $ce->{problemLibrary}{root}; $libraryRoot =~ s|/+$||; +$contribRoot =~ s|/+$||; print "using libraryRoot $libraryRoot\n"; +print "using contribRoot $contribRoot\n"; print "WEBWORK_ROOT $ENV{WEBWORK_ROOT}\n"; my $libraryVersion = $ce->{problemLibrary}->{version}; my $db_storage_engine = $ce->{problemLibrary_db}->{storage_engine}; @@ -210,6 +213,7 @@ if($libraryVersion eq '2.5') { DBsection_id int(15) NOT NULL, author_id int(15), institution tinyblob, + libraryroot varchar(255) NOT NULL, path_id int(15) NOT NULL, filename varchar(255) NOT NULL, morelt_id int(127) DEFAULT 0 NOT NULL, @@ -377,8 +381,6 @@ sub isvalid { return 1; } -my ($name,$pgfile,$pgpath); - #### First read in textbook information if(open(IN, '<:encoding(UTF-8)', "$libraryRoot/Textbooks")) { @@ -572,7 +574,8 @@ print "Number of files processed:\n"; #### Now search for tagged problems #recursive search for all pg files -File::Find::find({ wanted => \&pgfiles, follow_fast=> 1}, $libraryRoot); +File::Find::find( { wanted => \&pgfiles, follow_fast => 1 }, $libraryRoot ); +File::Find::find( { wanted => \&pgfiles, follow_fast => 1 }, $contribRoot ); sub trim { my $str = shift; @@ -617,13 +620,16 @@ sub pgfiles { #print "\n$name"; if ($name =~ /\.pg$/) { - $pgfile = basename($name); - $pgpath = dirname($name); $cnt2++; printf("%6d", $cnt2) if(($cnt2 % 100) == 0); print "\n" if(($cnt2 % 1000) == 0); - $pgpath =~ s|^$libraryRoot/||; + my $pgfile = basename($name); + dirname($name) =~ m|^([^/]*)/(.*)|; + my ($pglib, $pgpath) = ($1, $2); + $pglib =~ s|^$libraryRoot|Library|; + $pglib =~ s|^$contribRoot|Contrib|; + my $tags = WeBWorK::Utils::Tags->new($name); if ($tags->istagged()) { @@ -717,12 +723,11 @@ sub pgfiles { for my $DBsection_id (@DBsection_ids) { my $pgfile_id = safe_get_id($tables{pgfile}, 'pgfile_id', - qq(WHERE filename = ? AND path_id = ? AND DBsection_id = ? ), [$pgfile, $path_id, $DBsection_id], 1, "", $DBsection_id, $author_id, $tags->{Institution}, $path_id, $pgfile, 0, $level, $lang, $static, $mathobj); + qq(WHERE filename = ? AND path_id = ? AND DBsection_id = ? AND libraryroot = ?), + [$pgfile, $path_id, $DBsection_id, $pglib], 1, "", $DBsection_id, $author_id, + $tags->{Institution}, $pglib, $path_id, $pgfile, 0, $level, $lang, $static, $mathobj); push @pgfile_ids, $pgfile_id; } -#if (scalar(@pgfile_ids)>1) { - # print "\npg ".join(', ', @pgfile_ids)."\n"; -#} # morelt table diff --git a/lib/WeBWorK/ContentGenerator/Instructor/SetMaker.pm b/lib/WeBWorK/ContentGenerator/Instructor/SetMaker.pm index 1d021e271f..2e9a3be042 100644 --- a/lib/WeBWorK/ContentGenerator/Instructor/SetMaker.pm +++ b/lib/WeBWorK/ContentGenerator/Instructor/SetMaker.pm @@ -1588,9 +1588,12 @@ sub process_search { my %mlt = (); my $mltind; for my $indx (0..$#dbsearch) { - $dbsearch[$indx]->{filepath} = "Library/".$dbsearch[$indx]->{path}."/".$dbsearch[$indx]->{filename}; -# For debugging -$dbsearch[$indx]->{oindex} = $indx; + $dbsearch[$indx]->{filepath} = + $dbsearch[$indx]->{libraryroot} || "Library" . "/" . + $dbsearch[$indx]->{path} . "/" . + $dbsearch[$indx]->{filename}; + # For debugging + $dbsearch[$indx]->{oindex} = $indx; if($mltind = $dbsearch[$indx]->{morelt}) { if(defined($mlt{$mltind})) { push @{$mlt{$mltind}}, $indx; From 7704f72d16e8c1c981518167334abd3bc16d2956 Mon Sep 17 00:00:00 2001 From: "K. Andrew Parker" Date: Sun, 15 May 2022 16:29:02 -0400 Subject: [PATCH 02/12] more support for libraryroot in pgfiles --- bin/OPL-update | 9 +- .../ContentGenerator/Instructor/SetMaker.pm | 16 +- lib/WeBWorK/Utils/ListingDB.pm | 150 ++++++------------ lib/WebworkWebservice/LibraryActions.pm | 3 +- 4 files changed, 72 insertions(+), 106 deletions(-) diff --git a/bin/OPL-update b/bin/OPL-update index 9c7dbed638..687efbc3c9 100755 --- a/bin/OPL-update +++ b/bin/OPL-update @@ -130,7 +130,7 @@ $dbh->prepare("SET NAMES '$character_set'")->execute(); print "using character set $character_set to build OPL database\n"; my $libraryRoot = $ce->{problemLibrary}{root}; -my $contribRoot = $ce->{problemLibrary}{root}; +my $contribRoot = $ce->{contribLibrary}{root}; $libraryRoot =~ s|/+$||; $contribRoot =~ s|/+$||; print "using libraryRoot $libraryRoot\n"; @@ -625,10 +625,11 @@ sub pgfiles { print "\n" if(($cnt2 % 1000) == 0); my $pgfile = basename($name); - dirname($name) =~ m|^([^/]*)/(.*)|; + my $pgpath = dirname($name); + $pgpath =~ s|^$libraryRoot|Library|; + $pgpath =~ s|^$contribRoot|Contrib|; + $pgpath =~ m|^([^/]*)/(.*)|; my ($pglib, $pgpath) = ($1, $2); - $pglib =~ s|^$libraryRoot|Library|; - $pglib =~ s|^$contribRoot|Contrib|; my $tags = WeBWorK::Utils::Tags->new($name); diff --git a/lib/WeBWorK/ContentGenerator/Instructor/SetMaker.pm b/lib/WeBWorK/ContentGenerator/Instructor/SetMaker.pm index 2e9a3be042..97b790f719 100644 --- a/lib/WeBWorK/ContentGenerator/Instructor/SetMaker.pm +++ b/lib/WeBWorK/ContentGenerator/Instructor/SetMaker.pm @@ -44,6 +44,7 @@ use Encode; require WeBWorK::Utils::ListingDB; # we use x to mark strings for maketext +use constant INCLUDE_CONTRIB_DEFAULT => 0; use constant SHOW_HINTS_DEFAULT => 0; use constant SHOW_SOLUTIONS_DEFAULT => 0; use constant MAX_SHOW_DEFAULT => 20; @@ -388,6 +389,16 @@ sub view_problems_line { # Option of whether to show hints and solutions $result .= CGI::div( { class => 'd-inline-block ms-2 mb-2' }, + CGI::div( + { class => 'form-check form-check-inline ms-2' }, + CGI::checkbox({ + name => "includeContrib", + checked => $r->param('includeContrib') || INCLUDE_CONTRIB_DEFAULT, + label => $r->maketext("Include Contrib"), + class => 'form-check-input me-1', + labelattributes => { class => 'form-check-label col-form-label-sm' } + }) + ), CGI::div( { class => 'form-check form-check-inline ms-2' }, CGI::checkbox({ @@ -1835,6 +1846,7 @@ sub pre_header_initialize { } elsif ($r->param('lib_view')) { @pg_files=(); + # TODO: deprecate OPLv1 -- replace getSectionListings with getDBListings($r,0) my @dbsearch = WeBWorK::Utils::ListingDB::getSectionListings($r); @pg_files = process_search($r, @dbsearch); $use_previous_problems=0; @@ -2057,8 +2069,8 @@ sub body { )); } - my $showHints = $r->param('showHints'); - my $showSolutions = $r->param('showSolutions'); + # my $showHints = $r->param('showHints'); + # my $showSolutions = $r->param('showSolutions'); ########## Extract information computed in pre_header_initialize diff --git a/lib/WeBWorK/Utils/ListingDB.pm b/lib/WeBWorK/Utils/ListingDB.pm index 2a04430b2d..7ace861115 100644 --- a/lib/WeBWorK/Utils/ListingDB.pm +++ b/lib/WeBWorK/Utils/ListingDB.pm @@ -23,12 +23,21 @@ use File::Basename; use WeBWorK::Debug; use constant LIBRARY_STRUCTURE => { - textbook => { select => 'tbk.textbook_id,tbk.title,tbk.author,tbk.edition', - name => 'library_textbook', where => 'tbk.textbook_id'}, - textchapter => { select => 'tc.number,tc.name', name=>'library_textchapter', - where => 'tc.name'}, - textsection => { select => 'ts.number,ts.name', name=>'library_textsection', - where => 'ts.name'}, + textbook => { + select => 'tbk.textbook_id,tbk.title,tbk.author,tbk.edition', + name => 'library_textbook', + where => 'tbk.textbook_id' + }, + textchapter => { + select => 'tc.number,tc.name', + name=>'library_textchapter', + where => 'tc.name' + }, + textsection => { + select => 'ts.number,ts.name', + name=>'library_textsection', + where => 'ts.name' + }, problem => { select => 'prob.name' }, }; @@ -193,7 +202,7 @@ $r is a Apache request object so we can get the right table names $path is the path to the file -Out put is an array reference: [MO, static] +Output is an array reference: [MO, static] =cut @@ -364,21 +373,14 @@ sub getAllDBchapters { my $subject = $r->param('library_subjects'); return () unless($subject); my $dbh = getDB($r->ce); -# my $query = "SELECT DISTINCT c.name, c.DBchapter_id -# FROM `$tables{dbchapter}` c, -# `$tables{dbsubject}` t -# WHERE c.DBsubject_id = t.DBsubject_id AND -# t.name = \"$subject\" ORDER BY c.DBchapter_id"; -# my $all_chaps_ref = $dbh->selectall_arrayref($query); - my $query = "SELECT DISTINCT c.name, c.DBchapter_id - FROM `$tables{dbchapter}` c, + + my $query = "SELECT DISTINCT c.name, c.DBchapter_id + FROM `$tables{dbchapter}` c, `$tables{dbsubject}` t WHERE c.DBsubject_id = t.DBsubject_id AND t.name = ? ORDER BY c.DBchapter_id"; - my $all_chaps_ref = $dbh->selectall_arrayref($query, {},$subject); - + my $all_chaps_ref = $dbh->selectall_arrayref($query, {}, $subject); my @results = map { $_->[0] } @{$all_chaps_ref}; - #@results = sortByName(undef, @results); return @results; } @@ -397,23 +399,15 @@ sub getAllDBsections { my $chapter = $r->param('library_chapters'); return () unless($chapter); my $dbh = getDB($r->ce); -# my $query = "SELECT DISTINCT s.name, s.DBsection_id -# FROM `$tables{dbsection}` s, -# `$tables{dbchapter}` c, `$tables{dbsubject}` t -# WHERE s.DBchapter_id = c.DBchapter_id AND -# c.DBsubject_id = t.DBsubject_id AND -# t.name = \"$subject\" AND c.name = \"$chapter\" ORDER BY s.DBsection_id"; -# my $all_sections_ref = $dbh->selectall_arrayref($query); + my $query = "SELECT DISTINCT s.name, s.DBsection_id FROM `$tables{dbsection}` s, `$tables{dbchapter}` c, `$tables{dbsubject}` t WHERE s.DBchapter_id = c.DBchapter_id AND c.DBsubject_id = t.DBsubject_id AND t.name = ? AND c.name = ? ORDER BY s.DBsection_id"; - my $all_sections_ref = $dbh->selectall_arrayref($query, {},$subject, $chapter); - + my $all_sections_ref = $dbh->selectall_arrayref($query, {}, $subject, $chapter); my @results = map { $_->[0] } @{$all_sections_ref}; - #@results = sortByName(undef, @results); return @results; } @@ -434,6 +428,7 @@ sub getDBListings { my $subj = $r->param('library_subjects') || ""; my $chap = $r->param('library_chapters') || ""; my $sec = $r->param('library_sections') || ""; + my $include_contrib = $r->param('includeContrib') // 0; # Make sure these strings are internally encoded in UTF-8 utf8::upgrade($subj); @@ -455,36 +450,29 @@ sub getDBListings { $kw1 = ", `$tables{keyword}` kw, `$tables{pgfile_keyword}` pgkey"; $kw2 = " AND kw.keyword_id=pgkey.keyword_id AND pgkey.pgfile_id=pgf.pgfile_id $keywordstring"; -# makeKeywordWhere($keywords) ; } my $dbh = getDB($ce); my $extrawhere = ''; my @select_parameters=(); - if($subj) { -# $subj =~ s/'/\\'/g; -# $extrawhere .= " AND dbsj.name=\"$subj\" "; + if ($subj) { $extrawhere .= " AND dbsj.name= ? "; push @select_parameters, $subj; } - if($chap) { -# $chap =~ s/'/\\'/g; -# $extrawhere .= " AND dbc.name=\"$chap\" "; + if ($chap) { $extrawhere .= " AND dbc.name= ? "; push @select_parameters, $chap; } - if($sec) { -# $sec =~ s/'/\\'/g; -# $extrawhere .= " AND dbsc.name=\"$sec\" "; + if ($sec) { $extrawhere .= " AND dbsc.name= ? "; push @select_parameters, $sec; } - if(scalar(@levels)) { -# $extrawhere .= " AND pgf.level IN (".join(',', @levels).") "; + if (scalar(@levels)) { $extrawhere .= " AND pgf.level IN ( ? ) "; push @select_parameters, join(',', @levels); } + $extrawhere .= " AND pgf.libraryroot = 'Library' " unless ($include_contrib); my $textextrawhere = ''; my $haveTextInfo=0; my @textInfo_parameters=(); @@ -500,15 +488,7 @@ sub getDBListings { } my $selectwhat = 'DISTINCT pgf.pgfile_id'; - $selectwhat = 'COUNT(' . $selectwhat . ')' if ($amcounter); - -# my $query = "SELECT $selectwhat from `$tables{pgfile}` pgf, -# `$tables{dbsection}` dbsc, `$tables{dbchapter}` dbc, `$tables{dbsubject}` dbsj $kw1 -# WHERE dbsj.DBsubject_id = dbc.DBsubject_id AND -# dbc.DBchapter_id = dbsc.DBchapter_id AND -# dbsc.DBsection_id = pgf.DBsection_id -# \n $extrawhere -# $kw2"; + $selectwhat = "COUNT($selectwhat)" if ($amcounter); my $pg_id_ref; @@ -525,47 +505,39 @@ sub getDBListings { pgp.problem_id = prob.problem_id AND tc.textbook_id = tbk.textbook_id AND ts.chapter_id = tc.chapter_id AND - prob.section_id = ts.section_id \n $extrawhere \n $textextrawhere - $kw2"; - - #$query =~ s/\n/ /g; - #warn "text info: ", $query; - #warn "params: ", join(" | ",@select_parameters, @textInfo_parameters,@keyword_params); - - $pg_id_ref = $dbh->selectall_arrayref($query, {},@select_parameters, @textInfo_parameters, @keyword_params); + prob.section_id = ts.section_id + $extrawhere $textextrawhere $kw2"; + $pg_id_ref = $dbh->selectall_arrayref($query, {}, @select_parameters, @textInfo_parameters, @keyword_params); } else { my $query = "SELECT $selectwhat from `$tables{pgfile}` pgf, `$tables{dbsection}` dbsc, `$tables{dbchapter}` dbc, `$tables{dbsubject}` dbsj $kw1 WHERE dbsj.DBsubject_id = dbc.DBsubject_id AND dbc.DBchapter_id = dbsc.DBchapter_id AND dbsc.DBsection_id = pgf.DBsection_id - \n $extrawhere - $kw2"; - - #$query =~ s/\n/ /g; - #warn "no text info: ", $query; - #warn "params: ", join(" | ",@select_parameters,@keyword_params); - - $pg_id_ref = $dbh->selectall_arrayref($query,{},@select_parameters,@keyword_params); - #$query =~ s/\n/ /g; + $extrawhere $kw2"; + $pg_id_ref = $dbh->selectall_arrayref($query, {}, @select_parameters, @keyword_params); } my @pg_ids = map { $_->[0] } @{$pg_id_ref}; - if($amcounter) { - return(@pg_ids[0]); - } - my @results=(); + return(@pg_ids[0]) if ($amcounter); + + my @results = (); for my $pgid (@pg_ids) { -# $query = "SELECT path, filename, morelt_id, pgfile_id, static, MO FROM `$tables{pgfile}` pgf, `$tables{path}` p -# WHERE p.path_id = pgf.path_id AND pgf.pgfile_id=\"$pgid\""; -# my $row = $dbh->selectrow_arrayref($query); - my $query = "SELECT path, filename, morelt_id, pgfile_id, static, MO FROM `$tables{pgfile}` pgf, `$tables{path}` p + my $query = "SELECT libraryroot, path, filename, morelt_id, pgfile_id, static, MO FROM `$tables{pgfile}` pgf, `$tables{path}` p WHERE p.path_id = pgf.path_id AND pgf.pgfile_id= ? "; - my $row = $dbh->selectrow_arrayref($query,{},$pgid); - - push @results, {'path' => $row->[0], 'filename' => $row->[1], 'morelt' => $row->[2], 'pgid'=> $row->[3], 'static' => $row->[4], 'MO' => $row->[5] }; + my $row = $dbh->selectrow_arrayref($query, {}, $pgid); + + push @results, { + 'libraryroot' => $row->[0], + 'path' => $row->[1], + 'filename' => $row->[2], + 'morelt' => $row->[3], + 'pgid' => $row->[4], + 'static' => $row->[5], + 'MO' => $row->[6], + }; } return @results; @@ -591,7 +563,7 @@ sub getMLTleader { # returns an array of hash references. # if section is omitted, get all from the chapter sub getSectionListings { - #print STDERR "ListingDB::getSectionListings(chapter,section)\n"; + # TODO: eliminate this subroutine after deprecating OPLv1 my $r = shift; my $ce = $r->ce; my $version = $ce->{problemLibrary}->{version} || 1; @@ -612,14 +584,6 @@ sub getSectionListings { } my @results; #returned -# my $query = "SELECT c.*, p.path -# FROM classify c, pgfiles p -# WHERE $chapstring $secstring c.pgfiles_id = p.pgfiles_id"; -# my $dbh = getDB($ce); -# my %tables = getTables($ce); -# my $sth = $dbh->prepare($query); -# -# $sth->execute(); my $query = "SELECT c.*, p.path FROM classify c, pgfiles p WHERE ? ? c.pgfiles_id = p.pgfiles_id"; @@ -629,18 +593,8 @@ sub getSectionListings { $sth->execute($chapstring,$secstring); - while (1) - { - my $row = $sth->fetchrow_hashref(); - if (!defined($row)) - { - last; - } - else - { - push @results, $row; - #print STDERR "ListingDB::getSectionListings $row\n"; - } + while (my $row = $sth->fetchrow_hashref) { + push @results, $row; } return @results; } diff --git a/lib/WebworkWebservice/LibraryActions.pm b/lib/WebworkWebservice/LibraryActions.pm index b6dc05dbea..4ea9da1e14 100644 --- a/lib/WebworkWebservice/LibraryActions.pm +++ b/lib/WebworkWebservice/LibraryActions.pm @@ -331,8 +331,7 @@ sub searchLib { #API for searching the NPL database $self->{library_textsection} = $rh->{library_textsection}; debug(to_json($rh)); my @listings = WeBWorK::Utils::ListingDB::getDBListings($self); - my @output = map {$templateDir."/Library/".$_->{path}."/".$_->{filename}} @listings; - #change the hard coding!!!....just saying + my @output = map {"$templateDir/".$_->{libraryroot}||"Library"."/".$_->{path}."/".$_->{filename}} @listings; $out->{ra_out} = \@output; return($out); }; From e5c5a5627f68d2c88d963ec0a52522044bf7564a Mon Sep 17 00:00:00 2001 From: "K. Andrew Parker" Date: Sun, 15 May 2022 16:46:49 -0400 Subject: [PATCH 03/12] remove libraryroot fallback, it didn't work anyhow --- lib/WeBWorK/ContentGenerator/Instructor/SetMaker.pm | 2 +- lib/WebworkWebservice/LibraryActions.pm | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/WeBWorK/ContentGenerator/Instructor/SetMaker.pm b/lib/WeBWorK/ContentGenerator/Instructor/SetMaker.pm index 97b790f719..537a32a6f6 100644 --- a/lib/WeBWorK/ContentGenerator/Instructor/SetMaker.pm +++ b/lib/WeBWorK/ContentGenerator/Instructor/SetMaker.pm @@ -1600,7 +1600,7 @@ sub process_search { my $mltind; for my $indx (0..$#dbsearch) { $dbsearch[$indx]->{filepath} = - $dbsearch[$indx]->{libraryroot} || "Library" . "/" . + $dbsearch[$indx]->{libraryroot} . "/" . $dbsearch[$indx]->{path} . "/" . $dbsearch[$indx]->{filename}; # For debugging diff --git a/lib/WebworkWebservice/LibraryActions.pm b/lib/WebworkWebservice/LibraryActions.pm index 4ea9da1e14..cbb021e118 100644 --- a/lib/WebworkWebservice/LibraryActions.pm +++ b/lib/WebworkWebservice/LibraryActions.pm @@ -331,7 +331,7 @@ sub searchLib { #API for searching the NPL database $self->{library_textsection} = $rh->{library_textsection}; debug(to_json($rh)); my @listings = WeBWorK::Utils::ListingDB::getDBListings($self); - my @output = map {"$templateDir/".$_->{libraryroot}||"Library"."/".$_->{path}."/".$_->{filename}} @listings; + my @output = map {"$templateDir/".$_->{libraryroot}."/".$_->{path}."/".$_->{filename}} @listings; $out->{ra_out} = \@output; return($out); }; From 8b12220318c6917001725ec1625971da95d5010f Mon Sep 17 00:00:00 2001 From: "K. Andrew Parker" Date: Fri, 27 May 2022 10:10:02 -0400 Subject: [PATCH 04/12] move OPL-update to legacy status --- bin/OPL-update | 911 +-------------------------------- bin/OPL-update-legacy | 942 +++++++++++++++++++++++++++++++++++ bin/update-OPL-statistics.pl | 4 +- 3 files changed, 957 insertions(+), 900 deletions(-) create mode 100755 bin/OPL-update-legacy diff --git a/bin/OPL-update b/bin/OPL-update index 687efbc3c9..e6e78c421a 100755 --- a/bin/OPL-update +++ b/bin/OPL-update @@ -14,919 +14,32 @@ # 3) Configuration for the OPL in site.conf needs to be # done (basically just setting the path to the OPL files). -#use strict; -use File::Find; -use File::Find::Rule; -use File::Basename; -use Cwd; -use DBI; - - #(maximum varchar length is 255 for mysql version < 5.0.3. - #You can increase path length to 4096 for mysql > 5.0.3) +use strict; +use warnings; BEGIN { die "WEBWORK_ROOT not found in environment.\n" unless exists $ENV{WEBWORK_ROOT}; # Unused variable, but define it to avoid an error message. $WeBWorK::Constants::WEBWORK_DIRECTORY = ''; -} - -# Taxonomy global variables -# Make a hash of hashes of hashes to record what is legal -# Also create list for json file -my $taxo={}; -#my $taxsubs = []; - - -### Data for creating the database tables - -my %OPLtables = ( - dbsubject => 'OPL_DBsubject', - dbchapter => 'OPL_DBchapter', - dbsection => 'OPL_DBsection', - author => 'OPL_author', - path => 'OPL_path', - pgfile => 'OPL_pgfile', - keyword => 'OPL_keyword', - pgfile_keyword => 'OPL_pgfile_keyword', - textbook => 'OPL_textbook', - chapter => 'OPL_chapter', - section => 'OPL_section', - problem => 'OPL_problem', - morelt => 'OPL_morelt', - pgfile_problem => 'OPL_pgfile_problem', -); - - -my %NPLtables = ( - dbsubject => 'NPL-DBsubject', - dbchapter => 'NPL-DBchapter', - dbsection => 'NPL-DBsection', - author => 'NPL-author', - path => 'NPL-path', - pgfile => 'NPL-pgfile', - keyword => 'NPL-keyword', - pgfile_keyword => 'NPL-pgfile-keyword', - textbook => 'NPL-textbook', - chapter => 'NPL-chapter', - section => 'NPL-section', - problem => 'NPL-problem', - morelt => 'NPL-morelt', - pgfile_problem => 'NPL-pgfile-problem', -); - -my $pg_dir; -BEGIN { die "WEBWORK_ROOT not found in environment.\n" unless exists $ENV{WEBWORK_ROOT}; - $pg_dir = $ENV{PG_ROOT} // "$ENV{WEBWORK_ROOT}/../pg"; - die "The pg directory must be defined in PG_ROOT" unless (-e $pg_dir); + $ENV{PG_ROOT} //= "$ENV{WEBWORK_ROOT}/../pg"; + die "The pg directory must be defined in PG_ROOT" unless (-e $ENV{PG_ROOT}); } # Get database connection use lib "$ENV{WEBWORK_ROOT}/lib"; -use lib "$pg_dir/lib"; +use lib "$ENV{PG_ROOT}/lib"; use lib "$ENV{WEBWORK_ROOT}/bin"; use WeBWorK::CourseEnvironment; -use WeBWorK::Utils::Tags; -use OPLUtils qw/build_library_directory_tree build_library_subject_tree build_library_textbook_tree writeJSONtoFile/; my $ce = new WeBWorK::CourseEnvironment({webwork_dir=>$ENV{WEBWORK_ROOT}}); -# decide whether the mysql installation can handle -# utf8mb4 and that should be used for the OPL - -my $ENABLE_UTF8MB4 = ($ce->{ENABLE_UTF8MB4})?1:0; -print "using utf8mb4 \n\n" if $ENABLE_UTF8MB4; - -# The DBD::MariaDB driver should not get the -# mysql_enable_utf8mb4 or mysql_enable_utf8 settings, -# but DBD::mysql should. -my %utf8_parameters = (); - -if ( $ce->{database_driver} =~ /^mysql$/i ) { - # Only needed for older DBI:mysql driver - if ( $ENABLE_UTF8MB4 ) { - $utf8_parameters{mysql_enable_utf8mb4} = 1; - } else { - $utf8_parameters{mysql_enable_utf8} = 1; - } -} - -my $dbh = DBI->connect( - $ce->{problemLibrary_db}->{dbsource}, - $ce->{problemLibrary_db}->{user}, - $ce->{problemLibrary_db}->{passwd}, - { - PrintError => 0, - RaiseError => 1, - %utf8_parameters, - }, -); - -my $character_set=''; -$character_set=($ENABLE_UTF8MB4)?"utf8mb4":"utf8"; -$dbh->prepare("SET NAMES '$character_set'")->execute(); - -print "using character set $character_set to build OPL database\n"; - -my $libraryRoot = $ce->{problemLibrary}{root}; -my $contribRoot = $ce->{contribLibrary}{root}; -$libraryRoot =~ s|/+$||; -$contribRoot =~ s|/+$||; -print "using libraryRoot $libraryRoot\n"; -print "using contribRoot $contribRoot\n"; -print "WEBWORK_ROOT $ENV{WEBWORK_ROOT}\n"; -my $libraryVersion = $ce->{problemLibrary}->{version}; -my $db_storage_engine = $ce->{problemLibrary_db}->{storage_engine}; - -my $verbose = 0; -my $cnt2 = 0; -# Can force library version -$libraryVersion = $ARGV[0] if(@ARGV); - -# auto flush printing -my $old_fh = select(STDOUT); -$| = 1; -select($old_fh); - -sub dbug { - my $msg = shift; - my $insignificance = shift || 2; - print $msg if($verbose>=$insignificance); -} - -##Figure out which set of tables to use - -my %tables; -if($libraryVersion eq '2.5') { - %tables = %OPLtables; - my $lib = 'OPL'; - warn "Library version is $libraryVersion; using OPLtables!\n"; -} else { - %tables = %NPLtables; - my $lib = 'NPL'; - print "Library version is $libraryVersion; NPLtables! \n"; -} - -@create_tables = ( -[$tables{dbsubject}, ' - DBsubject_id int(15) NOT NULL auto_increment, - name varchar(245) NOT NULL, - KEY DBsubject (name), - PRIMARY KEY (DBsubject_id) -'], -[$tables{dbchapter}, ' - DBchapter_id int(15) NOT NULL auto_increment, - name varchar(245) NOT NULL, - DBsubject_id int(15) DEFAULT 0 NOT NULL, - KEY DBchapter (name), - KEY (DBsubject_id), - PRIMARY KEY (DBchapter_id) -'], -[$tables{dbsection}, ' - DBsection_id int(15) NOT NULL auto_increment, - name varchar(245) NOT NULL, - DBchapter_id int(15) DEFAULT 0 NOT NULL, - KEY DBsection (name), - KEY (DBchapter_id), - PRIMARY KEY (DBsection_id) -'], -[$tables{author}, ' - author_id int (15) NOT NULL auto_increment, - institution tinyblob, - lastname varchar (255) NOT NULL, - firstname varchar (255) NOT NULL, - email varchar (255), - KEY author (lastname(100), firstname(100)), - PRIMARY KEY (author_id) -'], -[$tables{path}, ' - path_id int(15) NOT NULL auto_increment, - path varchar(245) NOT NULL, - machine varchar(255), - user varchar(255), - KEY (path), - PRIMARY KEY (path_id) -'], -[$tables{pgfile}, ' - pgfile_id int(15) NOT NULL auto_increment, - DBsection_id int(15) NOT NULL, - author_id int(15), - institution tinyblob, - libraryroot varchar(255) NOT NULL, - path_id int(15) NOT NULL, - filename varchar(255) NOT NULL, - morelt_id int(127) DEFAULT 0 NOT NULL, - level int(15), - language varchar(255), - static TINYINT, - MO TINYINT, - PRIMARY KEY (pgfile_id) -'], -[$tables{keyword}, ' - keyword_id int(15) NOT NULL auto_increment, - keyword varchar(245) NOT NULL, - KEY (keyword), - PRIMARY KEY (keyword_id) -'], -[$tables{pgfile_keyword}, ' - pgfile_id int(15) DEFAULT 0 NOT NULL, - keyword_id int(15) DEFAULT 0 NOT NULL, - KEY pgfile_keyword (keyword_id, pgfile_id), - KEY pgfile (pgfile_id) -'], -[$tables{textbook}, ' - textbook_id int (15) NOT NULL auto_increment, - title varchar (255) NOT NULL, - edition int (15) DEFAULT 0 NOT NULL, - author varchar (255) NOT NULL, - publisher varchar (255), - isbn char (15), - pubdate varchar (255), - PRIMARY KEY (textbook_id) -'], -[$tables{chapter}, ' - chapter_id int (15) NOT NULL auto_increment, - textbook_id int (15), - number int(3), - name varchar(245) NOT NULL, - page int(4), - KEY (textbook_id, name), - KEY (number), - PRIMARY KEY (chapter_id) -'], -[$tables{section}, ' - section_id int(15) NOT NULL auto_increment, - chapter_id int (15), - number int(3), - name varchar(245) NOT NULL, - page int(4), - KEY (chapter_id, name), - KEY (number), - PRIMARY KEY section (section_id) -'], -[$tables{problem}, ' - problem_id int(15) NOT NULL auto_increment, - section_id int(15), - number int(4) NOT NULL, - page int(4), - #KEY (page, number), - KEY (section_id), - PRIMARY KEY (problem_id) -'], -[$tables{morelt}, ' - morelt_id int(15) NOT NULL auto_increment, - name varchar(245) NOT NULL, - DBsection_id int(15), - leader int(15), # pgfile_id of the MLT leader - KEY (name), - PRIMARY KEY (morelt_id) -'], -[$tables{pgfile_problem}, ' - pgfile_id int(15) DEFAULT 0 NOT NULL, - problem_id int(15) DEFAULT 0 NOT NULL, - PRIMARY KEY (pgfile_id, problem_id) -']); - -### End of database data - -## Resetting the database tables. -# First take care of tables which are no longer used - -$dbh->do("DROP TABLE IF EXISTS `NPL-institution`"); -$dbh->do("DROP TABLE IF EXISTS `NPL-pgfile-institution`"); - -for my $tableinfo (@create_tables) { - my $tabname = $tableinfo->[0]; - my $tabinit = $tableinfo->[1]; - my $query = "DROP TABLE IF EXISTS `$tabname`"; - $dbh->do($query); - $query = "CREATE TABLE `$tabname` ( $tabinit ) ENGINE=$db_storage_engine CHARACTER SET $character_set"; - $dbh->do($query); - if($lib eq 'OPL') { - $old_tabname = $tabname; - $old_tabname =~ s/OPL/NPL/; - $old_tabname =~ s/_/-/g; - $query = "DROP TABLE IF EXISTS `$old_tabname`"; - $dbh -> do($query); - } -} - - -print "Mysql database reinitialized.\n"; - -# From pgfile -## DBchapter('Limits and Derivatives') -## DBsection('Calculating Limits using the Limit Laws') -## Date('6/3/2002') -## Author('Tangan Gao') -## Institution('csulb') -## TitleText1('Calculus Early Transcendentals') -## EditionText1('4') -## AuthorText1('Stewart') -## Section1('2.3') -## Problem1('7') - -# Get an id, and add entry to the database if needed -sub safe_get_id { - my $tablename = shift; - my $idname = shift; - my $whereclause = shift; - my $wherevalues = shift; - my $addifnew = shift; - my @insertvalues = @_; -#print "\nCalled with $tablename, $idname, $whereclause, [".join(',', @$wherevalues)."], (".join(',', @insertvalues).")\n"; - for my $j (0..$#insertvalues) { - $insertvalues[$j] =~ s/"/\\\"/g; - } - - my $query = "SELECT $idname FROM `$tablename` ".$whereclause; - my $sth = $dbh->prepare($query); - $sth->execute(@$wherevalues); - my $idvalue, @row; - unless(@row = $sth->fetchrow_array()) { - return 0 unless $addifnew; - for my $j (0..$#insertvalues) { - #print "Looking at ".$insertvalues[$j]."\n"; - if ($insertvalues[$j] ne "") { - $insertvalues[$j] = '"'.$insertvalues[$j].'"'; - } else { - $insertvalues[$j] = NULL; - } - } - $dbh->do("INSERT INTO `$tablename` VALUES(". join(',',@insertvalues) .")"); - dbug "INSERT INTO $tablename VALUES( ".join(',',@insertvalues).")\n"; - $sth = $dbh->prepare($query); - $sth->execute(@$wherevalues); - @row = $sth->fetchrow_array(); - } - $idvalue = $row[0]; - return($idvalue); -} - -sub isvalid { - my $tags = shift; - if(not defined $taxo->{$tags->{DBsubject}}) { - print "\nInvalid subject ".$tags->{DBsubject}."\n"; - return 0; - } - if(not ($tags->{DBchapter} eq 'Misc.') and not defined $taxo->{$tags->{DBsubject}}->{$tags->{DBchapter}}) { - print "\nInvalid chapter ".$tags->{DBchapter}."\n"; - return 0; - } - if(not ($tags->{DBsection} eq 'Misc.') and not defined $taxo->{$tags->{DBsubject}}->{$tags->{DBchapter}}->{$tags->{DBsection}}) { - print "\nInvalid section ".$tags->{DBsection}."\n"; - return 0; - } - return 1; -} - -#### First read in textbook information - -if(open(IN, '<:encoding(UTF-8)', "$libraryRoot/Textbooks")) { - print "Reading in textbook data from Textbooks in the library $libraryRoot.\n"; - my %textinfo = ( TitleText => '', EditionText =>'', AuthorText=>''); - my $bookid = undef; - while (my $line = ) { - $line =~ s|#*$||; - if($line =~ /^\s*(.*?)\s*>>>\s*(.*?)\s*$/) { # Should have chapter or section information - my $chapsec = $1; - my $title = $2; - if($chapsec=~ /(\d+)\.(\d+)/) { # We have a section - if(defined($bookid)) { - my $query = "SELECT chapter_id FROM `$tables{chapter}` WHERE textbook_id = \"$bookid\" AND number = \"$1\""; - my $chapid = $dbh->selectrow_array($query); - if(defined($chapid)) { - my $sectid = safe_get_id($tables{section}, 'section_id', - qq(WHERE chapter_id = ? and name = ?), [$chapid, $title], 1, "", $chapid, $2, $title, ""); - } else { - print "Cannot enter section $chapsec because textbook information is missing the chapter entry\n"; - } - } else { - print "Cannot enter section $chapsec because textbook information is incomplete\n"; - } - } else { # We have a chapter entry - if(defined($bookid)) { - my $chapid = safe_get_id($tables{chapter}, 'chapter_id', - qq(WHERE textbook_id = ? AND number = ?), [$bookid, $chapsec], 1, "", $bookid, $chapsec, $title, ""); - - # Add dummy section entry for problems tagged to the chapter - # without a section - $query = "SELECT section_id FROM `$tables{section}` WHERE chapter_id = \"$chapid\" AND number = -1"; - my $sectid = $dbh->selectrow_array($query); - if (!defined($sectid)) { - $dbh->do("INSERT INTO `$tables{section}` - VALUES( - NULL, - \"$chapid\", - \"-1\", - \"\", - NULL - )" - ); - dbug "INSERT INTO section VALUES(\"\", \"$chapid\", \"-1\", \"\", \"\" )\n"; - } - } else { - print "Cannot enter chapter $chapsec because textbook information is incomplete\n"; - } - } - } elsif($line =~ /^\s*(TitleText|EditionText|AuthorText)\(\s*'(.*?)'\s*\)/) { - # Textbook information, maybe new - my $type = $1; - if(defined($textinfo{$type})) { # signals new text - %textinfo = ( TitleText => undef, - EditionText =>undef, - AuthorText=> undef); - $textinfo{$type} = $2; - $bookid = undef; - } else { - $textinfo{$type} = $2; - if(defined($textinfo{TitleText}) and - defined($textinfo{AuthorText}) and - defined($textinfo{EditionText})) { - my $query = "SELECT textbook_id FROM `$tables{textbook}` WHERE title = \"$textinfo{TitleText}\" AND edition = \"$textinfo{EditionText}\" AND author=\"$textinfo{AuthorText}\""; - $bookid = $dbh->selectrow_array($query); - if (!defined($bookid)) { - $dbh->do("INSERT INTO `$tables{textbook}` - VALUES( - NULL, - \"$textinfo{TitleText}\", - \"$textinfo{EditionText}\", - \"$textinfo{AuthorText}\", - NULL, - NULL, - NULL - )" - ); - dbug "INSERT INTO textbook VALUES( \"\", \"$textinfo{TitleText}\", \"$textinfo{EditionText}\", \"$textinfo{AuthorText}\", \"\", \"\", \"\" )\n"; - $bookid = $dbh->selectrow_array($query); - } - } - } - } - } - close(IN); -} else { - print "Textbooks file was not found in library $libraryRoot. If the path to the problem library doesn't seem - correct, make modifications in webwork2/conf/site.conf (\$problemLibrary{root}). If that is correct then - updating from git should download the Textbooks file.\n"; -} -#### End of textbooks - -#### Next read in the taxonomy -my $clsep = '<<<'; -my $clinner = '__'; -my @cllist = (); -# Record full taxonomy for tagging menus (does not include cross-lists) -my $tagtaxo = []; -my ($chaplist, $seclist) = ([],[]); - -my $canopenfile = 0; -if(open(IN, '<:encoding(UTF-8)', "$libraryRoot/Taxonomy2")) { - print "Reading in OPL taxonomy from Taxonomy2 in the library $libraryRoot.\n"; - $canopenfile = 1; -} elsif(open(IN, '<:encoding(UTF-8)', "$libraryRoot/Taxonomy")) { - print "Reading in OPL taxonomy from Taxonomy in the library $libraryRoot.\n"; - $canopenfile = 1; -} else { - print "Taxonomy file was not found in library $libraryRoot. If the path to the problem library doesn't seem - correct, make modifications in webwork2/conf/site.conf (\$problemLibrary{root}). If that is correct then - updating from git should download the Taxonomy file.\n"; -} - -# Taxonomy is a subset of Taxonomy2, so we can use the same code either way -if($canopenfile) { - my ($cursub,$curchap); # these are strings - my ($subj, $chap, $sect); # these are indeces - while(my $line = ) { - $line =~ /^(\t*)/; - my $count = length($1); - my $oktag = 1; - chomp($line); - if($line =~ m/$clsep/) { - $oktag = 0; - my @cross = split $clsep, $line; - @cross = map(trim($_), @cross); - if(scalar(@cross) > 1) { - push @cllist, [join($clinner, ($cursub,$curchap,$cross[0])) ,$cross[1]]; - } - $line = $cross[0]; - } - $line = trim($line); - - # We put the line in the database in all cases - # but crosslists are not put in the heierarchy of legal tags - # instead they go in a list of crosslists to deal with after - # the full taxonomy is read in - if($count == 0) { #DBsubject - $cursub = $line; - if($oktag) { - $taxo->{$line} = {}; - ($chaplist, $seclist) = ([],[]); - push @{$tagtaxo}, {name=>$line, subfields=>$chaplist}; - } - $subj = safe_get_id($tables{dbsubject}, 'DBsubject_id', - qq(WHERE name = ?), [$line], 1, "", $line); - } elsif($count == 1) { #DBchapter - if($oktag) { - $taxo->{$cursub}->{$line} = {}; - $seclist=[]; - push @{$chaplist}, {name=>$line, subfields=>$seclist}; - } - $curchap = $line; - $chap = safe_get_id($tables{dbchapter}, 'DBchapter_id', - qq(WHERE name = ? and DBsubject_id = ?), [$line, $subj], 1, "", $line, $subj); - } else { #DBsection - if($oktag) { - $taxo->{$cursub}->{$curchap}->{$line} = []; - push @{$seclist}, {name=>$line}; - } - $sect = safe_get_id($tables{dbsection}, 'DBsection_id', - qq(WHERE name = ? and DBchapter_id = ?), [$line, $chap], 1, "", $line, $chap); - } - } - close(IN); -} -#### End of taxonomy/taxonomy2 +print "\nDownloading the latest OPL release.\n"; +do $ENV{WEBWORK_ROOT} . '/bin/download-OPL-metadata-release.pl'; -use JSON; - -#### Save the official taxonomy in json format -my $webwork_htdocs = $ce->{webworkDirs}{htdocs}; -my $file = "$webwork_htdocs/DATA/tagging-taxonomy.json"; - -writeJSONtoFile($tagtaxo,$file); -print "Saved taxonomy to $file.\n"; - -#### Now deal with cross-listed sections -for my $clinfo (@cllist) { - my @scs = split /$clinner/, $clinfo->[1]; - if(defined $taxo->{$scs[0]}->{$scs[1]}->{$scs[2]}) { - push @{$taxo->{$scs[0]}->{$scs[1]}->{$scs[2]}}, $clinfo->[0]; - } else { - print "Faulty cross-list: pointing to $scs[0] / $scs[1] / $scs[2]\n"; - } -} - -print "Converting data from tagged pgfiles into mysql.\n"; -print "Number of files processed:\n"; - -#### Now search for tagged problems -#recursive search for all pg files - -File::Find::find( { wanted => \&pgfiles, follow_fast => 1 }, $libraryRoot ); -File::Find::find( { wanted => \&pgfiles, follow_fast => 1 }, $contribRoot ); - -sub trim { - my $str = shift; - $str =~ s/^\s+//; - $str =~ s/\s+$//; - return $str; -} - -sub kwtidy { - my $s = shift; - $s =~ s/\W//g; - $s =~ s/_//g; - $s = lc($s); - return($s); -} - -sub keywordcleaner { - my $string = shift; - my @spl1 = split /,/, $string; - my @spl2 = map(kwtidy($_), @spl1); - return(@spl2); -} - -# Save on passing these values around -my %textinfo; - -# Initialize, if needed more text-info information; -sub maybenewtext { - my $textno = shift; - return if defined($textinfo{$textno}); - # So, not defined yet - $textinfo{$textno} = { title => '', author =>'', edition =>'', - section => '', chapter =>'', problems => [] }; -} - -# process each file returned by the find command. -sub pgfiles { - my $name = $File::Find::name; - my ($text, $edition, $textauthor, $textsection, $textproblem); - %textinfo=(); - my @textproblems = (-1); -#print "\n$name"; - - if ($name =~ /\.pg$/) { - $cnt2++; - printf("%6d", $cnt2) if(($cnt2 % 100) == 0); - print "\n" if(($cnt2 % 1000) == 0); - - my $pgfile = basename($name); - my $pgpath = dirname($name); - $pgpath =~ s|^$libraryRoot|Library|; - $pgpath =~ s|^$contribRoot|Contrib|; - $pgpath =~ m|^([^/]*)/(.*)|; - my ($pglib, $pgpath) = ($1, $2); - - my $tags = WeBWorK::Utils::Tags->new($name); - - if ($tags->istagged()) { - # Fill in missing data with Misc. instead of blank - print "\nNO SUBJECT $name\n" unless ($tags->{DBsubject} =~ /\S/); - - $tags->{DBchapter} = 'Misc.' unless $tags->{DBchapter} =~ /\S/; - $tags->{DBsection} = 'Misc.' unless $tags->{DBsection} =~ /\S/; - - # DBsubject table - - if(isvalid($tags)) { - my $DBsubject_id = safe_get_id($tables{dbsubject}, 'DBsubject_id', - qq(WHERE BINARY name = ?), [$tags->{DBsubject}], 1, "", $tags->{DBsubject}); - if(not $DBsubject_id) { - print "\nInvalid subject '$tags->{DBsubject}' in $name\n"; - next; - } - - # DBchapter table - - $DBchapter_id = safe_get_id($tables{dbchapter}, 'DBchapter_id', - qq(WHERE BINARY name = ? and DBsubject_id = ?), [$tags->{DBchapter}, $DBsubject_id], 1, "", $tags->{DBchapter}, $DBsubject_id); - if(not $DBchapter_id) { - print "\nInvalid chapter '$tags->{DBchapter}' in $name\n"; - next; - } - - # DBsection table - - $aDBsection_id = safe_get_id($tables{dbsection}, 'DBsection_id', - qq(WHERE BINARY name = ? and DBchapter_id = ?), [$tags->{DBsection}, $DBchapter_id], 1, "", $tags->{DBsection}, $DBchapter_id); - if(not $aDBsection_id) { - print "\nInvalid section '$tags->{DBsection}' in $name\n"; - next; - } - } else { # tags are not valid, error printed by validation part - print "File $name\n"; - next; - } - - my @DBsection_ids=($aDBsection_id); - # Now add crosslisted section - my @CL_array = @{$taxo->{$tags->{DBsubject}}->{$tags->{DBchapter}}->{$tags->{DBsection}}}; - for my $clsect (@CL_array) { - my @np = split /$clinner/, $clsect; - @np = map(trim($_), @np); - my $new_dbsubj_id = safe_get_id($tables{dbsubject}, 'DBsubject_id', - qq(WHERE name = ?), [$np[0]], 1, "", $np[0]); - my $new_dbchap_id = safe_get_id($tables{dbchapter}, 'DBchapter_id', - qq(WHERE name = ? and DBsubject_id = ?), [$np[1], $new_dbsubj_id], 1, "", $np[1], $new_dbsubj_id); - my $new_dbsect_id = safe_get_id($tables{dbsection}, 'DBsection_id', - qq(WHERE name = ? and DBchapter_id = ?), [$np[2], $new_dbchap_id], 1, "", $np[2], $new_dbchap_id); - push @DBsection_ids, $new_dbsect_id; - } - - # author table - - $tags->{Author} =~ /(.*?)\s(\w+)\s*$/; - my $firstname = $1; - my $lastname = $2; - #remove leading and trailing spaces from firstname, which includes any middle name too. - $firstname =~ s/^\s*//; - $firstname =~ s/\s*$//; - $lastname =~ s/^\s*//; - $lastname =~ s/\s*$//; - my $author_id = 0; - if($lastname) { - $author_id = safe_get_id($tables{author}, 'author_id', - qq(WHERE lastname = ? AND firstname = ?), [$lastname, $firstname], 1, "", $tags->{Institution}, $lastname, $firstname,""); - } - - # path table - - my $path_id = safe_get_id($tables{path}, 'path_id', - qq(WHERE path = ?), [$pgpath], 1, "", $pgpath, "", ""); - - # pgfile table -- set 4 defaults first - - ## TODO this is where we have to deal with crosslists, and pgfileid - ## will be an array of id's - ## Make an array of DBsection-id's above - - my $level = $tags->{Level} || 0; - # Default language is English - my $lang = $tags->{Language} || "en"; - my $mathobj = $tags->{MO} || 0; - my $static = $tags->{Static} || 0; - - my @pgfile_ids = (); - - for my $DBsection_id (@DBsection_ids) { - my $pgfile_id = safe_get_id($tables{pgfile}, 'pgfile_id', - qq(WHERE filename = ? AND path_id = ? AND DBsection_id = ? AND libraryroot = ?), - [$pgfile, $path_id, $DBsection_id, $pglib], 1, "", $DBsection_id, $author_id, - $tags->{Institution}, $pglib, $path_id, $pgfile, 0, $level, $lang, $static, $mathobj); - push @pgfile_ids, $pgfile_id; - } - - # morelt table - - my $morelt_id; - if($tags->{MLT}) { - for my $DBsection_id (@DBsection_ids) { - $morelt_id = safe_get_id($tables{morelt}, 'morelt_id', - qq(WHERE name = ?), [$tags->{MLT}], 1, "", $tags->{MLT}, $DBsection_id, ""); - - for my $pgfile_id (@pgfile_ids) { - $dbh->do("UPDATE `$tables{pgfile}` SET - morelt_id = \"$morelt_id\" WHERE pgfile_id = \"$pgfile_id\" "); - - dbug "UPDATE pgfile morelt_id for $pgfile_id to $morelt_id\n"; - if($tags->{MLTleader}) { - $dbh->do("UPDATE `$tables{morelt}` SET - leader = \"$pgfile_id\" WHERE morelt_id = \"$morelt_id\" "); - dbug "UPDATE morelt leader for $morelt_id to $pgfile_id\n"; - } - } - } - } - - # keyword table, and problem_keyword many-many table - - foreach my $keyword (@{$tags->{keywords}}) { - $keyword =~ s/[\'\"]//g; - $keyword = kwtidy($keyword); - # skip it if it is empty - next unless $keyword; - my $keyword_id = safe_get_id($tables{keyword}, 'keyword_id', - qq(WHERE keyword = ?), [$keyword], 1, "", $keyword); - - for my $pgfile_id (@pgfile_ids) { - $query = "SELECT pgfile_id FROM `$tables{pgfile_keyword}` WHERE keyword_id = \"$keyword_id\" and pgfile_id=\"$pgfile_id\""; - my $ok = $dbh->selectrow_array($query); - if (!defined($ok)) { - $dbh->do("INSERT INTO `$tables{pgfile_keyword}` - VALUES( - \"$pgfile_id\", - \"$keyword_id\" - )" - ); - dbug "INSERT INTO pgfile_keyword VALUES( \"$pgfile_id\", \"$keyword_id\" )\n"; - } - } - } #end foreach keyword - - # Textbook section - # problem table contains textbook problems - - for my $texthashref (@{$tags->{textinfo}}) { - - # textbook table - - $text = $texthashref->{TitleText}; - $edition = $texthashref->{EditionText} || 0; - $edition =~ s/[^\d\.]//g; - $textauthor = $texthashref->{AuthorText}; - next unless($text and $textauthor); - my $chapnum = $texthashref->{chapter} || -1; - my $secnum = $texthashref->{section} || -1; - $query = "SELECT textbook_id FROM `$tables{textbook}` WHERE title = \"$text\" AND edition = \"$edition\" AND author=\"$textauthor\""; - my $textbook_id = $dbh->selectrow_array($query); - if (!defined($textbook_id)) { - # make sure edition is an integer - $edition = 0 unless $edition; - $dbh->do("INSERT INTO `$tables{textbook}` - VALUES( - NULL, - \"$text\", - \"$edition\", - \"$textauthor\", - NULL, - NULL, - NULL - )" - ); - dbug "INSERT INTO textbook VALUES( \"\", \"$text\", \"$edition\", \"$textauthor\", \"\", \"\", \"\" )\n"; - dbug "\nLate add into $tables{textbook} \"$text\", \"$edition\", \"$textauthor\"\n", 1; - $textbook_id = $dbh->selectrow_array($query); - } - - # chapter weak table of textbook - # - $query = "SELECT chapter_id FROM `$tables{chapter}` WHERE textbook_id = \"$textbook_id\" AND number = \"$chapnum\""; - my $chapter_id = $dbh->selectrow_array($query); - if (!defined($chapter_id)) { - $dbh->do("INSERT INTO `$tables{chapter}` - VALUES( - NULL, - \"$textbook_id\", - \"".$chapnum."\", - \"$tags->{DBchapter}\", - NULL - )" - ); - dbug "\nLate add into $tables{chapter} \"$text\", \"$edition\", \"$textauthor\", $chapnum $tags->{chapter} from $name\n", 1; - dbug "INSERT INTO chapter VALUES(\"\", \"$textbook_id\", \"".$chapnum."\", \"$tags->{DBchapter}\", \"\" )\n"; - $chapter_id = $dbh->selectrow_array($query); - } - - # section weak table of textbook - # - $tags->{DBsection} = '' if ($secnum < 0); - $query = "SELECT section_id FROM `$tables{section}` WHERE chapter_id = \"$chapter_id\" AND number = \"$secnum\""; - my $section_id = $dbh->selectrow_array($query); - if (!defined($section_id)) { - $dbh->do("INSERT INTO `$tables{section}` - VALUES( - NULL, - \"$chapter_id\", - \"$secnum\", - \"$tags->{DBsection}\", - NULL - )" - ); - dbug "INSERT INTO section VALUES(\"\", \"$textbook_id\", \"$secnum\", \"$tags->{DBsection}\", \"\" )\n"; - dbug "\nLate add into $tables{section} \"$text\", \"$edition\", \"$textauthor\", $secnum $tags->{DBsection} from $name\n", 1; - $section_id = $dbh->selectrow_array($query); - } - - @textproblems = @{$texthashref->{problems}}; - for my $tp (@textproblems) { - $query = "SELECT problem_id FROM `$tables{problem}` WHERE section_id = \"$section_id\" AND number = \"$tp\""; - my $problem_id = $dbh->selectrow_array($query); - if (!defined($problem_id)) { - $dbh->do("INSERT INTO `$tables{problem}` - VALUES( - NULL, - \"$section_id\", - \"$tp\", - NULL - )" - ); - dbug "INSERT INTO problem VALUES( \"\", \"$section_id\", \"$tp\", \"\" )\n"; - $problem_id = $dbh->selectrow_array($query); - } - - # pgfile_problem table associates pgfiles with textbook problems - for my $pgfile_id (@pgfile_ids) { - $query = "SELECT problem_id FROM `$tables{pgfile_problem}` WHERE problem_id = \"$problem_id\" AND pgfile_id = \"$pgfile_id\""; - my $pg_problem_id = $dbh->selectrow_array($query); - if (!defined($pg_problem_id)) { - $dbh->do("INSERT INTO `$tables{pgfile_problem}` - VALUES( - \"$pgfile_id\", - \"$problem_id\" - )" - ); - dbug "INSERT INTO pgfile_problem VALUES( \"$pgfile_id\", \"$problem_id\" )\n"; - } - } - } - - #reset tag vars, they may not match the next text/file - $textauthor=""; $textsection=""; - } - } else { # This file was not tagged - # Message if not a pointer - # print STDERR "File $name is not tagged\n" if not $tags->isplaceholder(); - ; - } - } -} - -print "\n\n"; - -# Now prune away DBsection, etc, which do not appear in any files -#%my $query = "SELECT chapter_id FROM `$tables{chapter}` WHERE textbook_id = \"$bookid\" AND number = \"$1\""; -#%my $chapid = $dbh->selectrow_array($query); - -#select dbs.DBsection_id from OPL_DBsection dbs; -#select COUNT(*) from OPL_pgfile where DBsection_id=857; - -my $dbsects = $dbh->selectall_arrayref("SELECT DBsection_id from `$tables{dbsection}`"); -for my $sect (@{$dbsects}) { - $sect = $sect->[0]; - my $srar = $dbh->selectall_arrayref("SELECT * FROM `$tables{pgfile}` WHERE DBsection_id=$sect"); - if(scalar(@{$srar})==0) { - $dbh->do("DELETE FROM `$tables{dbsection}` WHERE DBsection_id=$sect"); - } -} - -my $dbchaps = $dbh->selectall_arrayref("SELECT DBchapter_id from `$tables{dbchapter}`"); -for my $chap (@{$dbchaps}) { - $chap = $chap->[0]; - my $srar = $dbh->selectall_arrayref("SELECT * FROM `$tables{dbsection}` WHERE DBchapter_id=$chap"); - if(scalar(@{$srar})==0) { - $dbh->do("DELETE FROM `$tables{dbchapter}` WHERE DBchapter_id=$chap"); - } -} - -# Note: this used to build some JSON versions of the textbooks, subjects and directory trees -# that could be used in the library browswer. It's functionality is now -# in the updateOPLextras.pl script. - -$dbh->disconnect; +# Generate set definition list files. +do $ENV{WEBWORK_ROOT} . '/bin/generate-OPL-set-def-lists.pl'; if ($ce->{problemLibrary}{showLibraryLocalStats} || $ce->{problemLibrary}{showLibraryGlobalStats}) { @@ -935,9 +48,9 @@ if ($ce->{problemLibrary}{showLibraryLocalStats} || print "\nLoading global statistics (if possible).\n"; do $ENV{WEBWORK_ROOT}.'/bin/load-OPL-global-statistics.pl'; -} -# Generate set definition list files. -do $ENV{WEBWORK_ROOT} . '/bin/generate-OPL-set-def-lists.pl'; + print "\nSharing aggregated statistics\n"; + do $ENV{WEBWORK_ROOT}.'/bin/upload-OPL-statistics.pl'; +} print "\nDone.\n"; diff --git a/bin/OPL-update-legacy b/bin/OPL-update-legacy new file mode 100755 index 0000000000..90e5177429 --- /dev/null +++ b/bin/OPL-update-legacy @@ -0,0 +1,942 @@ +#!/usr/bin/env perl + +# This is the script formerly known as loadDB2, and then known as NPL-update. + +# It is used to update +# the database when it comes to the WeBWorK Open Problem Library (OPL). +# This should be run after doing a git clone or pull for the OPL +# files. + +# In order for this script to work: +# 1) The OPL downloaded to your machine (the .pg files) +# 2) The environment variable WEBWORK_ROOT needs to be +# correctly defined (as with other scripts here). +# 3) Configuration for the OPL in site.conf needs to be +# done (basically just setting the path to the OPL files). + +#use strict; +use File::Find; +use File::Find::Rule; +use File::Basename; +use Cwd; +use DBI; + + #(maximum varchar length is 255 for mysql version < 5.0.3. + #You can increase path length to 4096 for mysql > 5.0.3) + +BEGIN { + die "WEBWORK_ROOT not found in environment.\n" + unless exists $ENV{WEBWORK_ROOT}; + # Unused variable, but define it to avoid an error message. + $WeBWorK::Constants::WEBWORK_DIRECTORY = ''; +} + +# Taxonomy global variables +# Make a hash of hashes of hashes to record what is legal +# Also create list for json file +my $taxo={}; +#my $taxsubs = []; + + +### Data for creating the database tables + +my %OPLtables = ( + dbsubject => 'OPL_DBsubject', + dbchapter => 'OPL_DBchapter', + dbsection => 'OPL_DBsection', + author => 'OPL_author', + path => 'OPL_path', + pgfile => 'OPL_pgfile', + keyword => 'OPL_keyword', + pgfile_keyword => 'OPL_pgfile_keyword', + textbook => 'OPL_textbook', + chapter => 'OPL_chapter', + section => 'OPL_section', + problem => 'OPL_problem', + morelt => 'OPL_morelt', + pgfile_problem => 'OPL_pgfile_problem', +); + + +my %NPLtables = ( + dbsubject => 'NPL-DBsubject', + dbchapter => 'NPL-DBchapter', + dbsection => 'NPL-DBsection', + author => 'NPL-author', + path => 'NPL-path', + pgfile => 'NPL-pgfile', + keyword => 'NPL-keyword', + pgfile_keyword => 'NPL-pgfile-keyword', + textbook => 'NPL-textbook', + chapter => 'NPL-chapter', + section => 'NPL-section', + problem => 'NPL-problem', + morelt => 'NPL-morelt', + pgfile_problem => 'NPL-pgfile-problem', +); + +my $pg_dir; +BEGIN { + die "WEBWORK_ROOT not found in environment.\n" unless exists $ENV{WEBWORK_ROOT}; + $pg_dir = $ENV{PG_ROOT} // "$ENV{WEBWORK_ROOT}/../pg"; + die "The pg directory must be defined in PG_ROOT" unless (-e $pg_dir); +} + +# Get database connection +use lib "$ENV{WEBWORK_ROOT}/lib"; +use lib "$pg_dir/lib"; +use lib "$ENV{WEBWORK_ROOT}/bin"; +use WeBWorK::CourseEnvironment; +use WeBWorK::Utils::Tags; +use OPLUtils qw/build_library_directory_tree build_library_subject_tree build_library_textbook_tree writeJSONtoFile/; + +my $ce = new WeBWorK::CourseEnvironment({webwork_dir=>$ENV{WEBWORK_ROOT}}); + +# decide whether the mysql installation can handle +# utf8mb4 and that should be used for the OPL + +my $ENABLE_UTF8MB4 = ($ce->{ENABLE_UTF8MB4})?1:0; +print "using utf8mb4 \n\n" if $ENABLE_UTF8MB4; + +# The DBD::MariaDB driver should not get the +# mysql_enable_utf8mb4 or mysql_enable_utf8 settings, +# but DBD::mysql should. +my %utf8_parameters = (); + +if ( $ce->{database_driver} =~ /^mysql$/i ) { + # Only needed for older DBI:mysql driver + if ( $ENABLE_UTF8MB4 ) { + $utf8_parameters{mysql_enable_utf8mb4} = 1; + } else { + $utf8_parameters{mysql_enable_utf8} = 1; + } +} + +my $dbh = DBI->connect( + $ce->{problemLibrary_db}->{dbsource}, + $ce->{problemLibrary_db}->{user}, + $ce->{problemLibrary_db}->{passwd}, + { + PrintError => 0, + RaiseError => 1, + %utf8_parameters, + }, +); + +my $character_set=''; +$character_set=($ENABLE_UTF8MB4)?"utf8mb4":"utf8"; +$dbh->prepare("SET NAMES '$character_set'")->execute(); + +print "using character set $character_set to build OPL database\n"; + +my $libraryRoot = $ce->{problemLibrary}{root}; +my $contribRoot = $ce->{contribLibrary}{root}; +$libraryRoot =~ s|/+$||; +$contribRoot =~ s|/+$||; +print "using libraryRoot $libraryRoot\n"; +print "using contribRoot $contribRoot\n"; +print "WEBWORK_ROOT $ENV{WEBWORK_ROOT}\n"; +my $libraryVersion = $ce->{problemLibrary}->{version}; +my $db_storage_engine = $ce->{problemLibrary_db}->{storage_engine}; + +my $verbose = 0; +my $cnt2 = 0; +# Can force library version +$libraryVersion = $ARGV[0] if(@ARGV); + +# auto flush printing +my $old_fh = select(STDOUT); +$| = 1; +select($old_fh); + +sub dbug { + my $msg = shift; + my $insignificance = shift || 2; + print $msg if($verbose>=$insignificance); +} + +##Figure out which set of tables to use + +my %tables; +if($libraryVersion eq '2.5') { + %tables = %OPLtables; + my $lib = 'OPL'; + warn "Library version is $libraryVersion; using OPLtables!\n"; +} else { + %tables = %NPLtables; + my $lib = 'NPL'; + print "Library version is $libraryVersion; NPLtables! \n"; +} + +@create_tables = ( +[$tables{dbsubject}, ' + DBsubject_id int(15) NOT NULL auto_increment, + name varchar(245) NOT NULL, + KEY DBsubject (name), + PRIMARY KEY (DBsubject_id) +'], +[$tables{dbchapter}, ' + DBchapter_id int(15) NOT NULL auto_increment, + name varchar(245) NOT NULL, + DBsubject_id int(15) DEFAULT 0 NOT NULL, + KEY DBchapter (name), + KEY (DBsubject_id), + PRIMARY KEY (DBchapter_id) +'], +[$tables{dbsection}, ' + DBsection_id int(15) NOT NULL auto_increment, + name varchar(245) NOT NULL, + DBchapter_id int(15) DEFAULT 0 NOT NULL, + KEY DBsection (name), + KEY (DBchapter_id), + PRIMARY KEY (DBsection_id) +'], +[$tables{author}, ' + author_id int (15) NOT NULL auto_increment, + institution tinyblob, + lastname varchar (255) NOT NULL, + firstname varchar (255) NOT NULL, + email varchar (255), + KEY author (lastname(100), firstname(100)), + PRIMARY KEY (author_id) +'], +[$tables{path}, ' + path_id int(15) NOT NULL auto_increment, + path varchar(245) NOT NULL, + machine varchar(255), + user varchar(255), + KEY (path), + PRIMARY KEY (path_id) +'], +[$tables{pgfile}, ' + pgfile_id int(15) NOT NULL auto_increment, + DBsection_id int(15) NOT NULL, + author_id int(15), + institution tinyblob, + libraryroot varchar(255) NOT NULL, + path_id int(15) NOT NULL, + filename varchar(255) NOT NULL, + morelt_id int(127) DEFAULT 0 NOT NULL, + level int(15), + language varchar(255), + static TINYINT, + MO TINYINT, + PRIMARY KEY (pgfile_id) +'], +[$tables{keyword}, ' + keyword_id int(15) NOT NULL auto_increment, + keyword varchar(245) NOT NULL, + KEY (keyword), + PRIMARY KEY (keyword_id) +'], +[$tables{pgfile_keyword}, ' + pgfile_id int(15) DEFAULT 0 NOT NULL, + keyword_id int(15) DEFAULT 0 NOT NULL, + KEY pgfile_keyword (keyword_id, pgfile_id), + KEY pgfile (pgfile_id) +'], +[$tables{textbook}, ' + textbook_id int (15) NOT NULL auto_increment, + title varchar (255) NOT NULL, + edition int (15) DEFAULT 0 NOT NULL, + author varchar (255) NOT NULL, + publisher varchar (255), + isbn char (15), + pubdate varchar (255), + PRIMARY KEY (textbook_id) +'], +[$tables{chapter}, ' + chapter_id int (15) NOT NULL auto_increment, + textbook_id int (15), + number int(3), + name varchar(245) NOT NULL, + page int(4), + KEY (textbook_id, name), + KEY (number), + PRIMARY KEY (chapter_id) +'], +[$tables{section}, ' + section_id int(15) NOT NULL auto_increment, + chapter_id int (15), + number int(3), + name varchar(245) NOT NULL, + page int(4), + KEY (chapter_id, name), + KEY (number), + PRIMARY KEY section (section_id) +'], +[$tables{problem}, ' + problem_id int(15) NOT NULL auto_increment, + section_id int(15), + number int(4) NOT NULL, + page int(4), + #KEY (page, number), + KEY (section_id), + PRIMARY KEY (problem_id) +'], +[$tables{morelt}, ' + morelt_id int(15) NOT NULL auto_increment, + name varchar(245) NOT NULL, + DBsection_id int(15), + leader int(15), # pgfile_id of the MLT leader + KEY (name), + PRIMARY KEY (morelt_id) +'], +[$tables{pgfile_problem}, ' + pgfile_id int(15) DEFAULT 0 NOT NULL, + problem_id int(15) DEFAULT 0 NOT NULL, + PRIMARY KEY (pgfile_id, problem_id) +']); + +### End of database data + +## Resetting the database tables. +# First take care of tables which are no longer used + +$dbh->do("DROP TABLE IF EXISTS `NPL-institution`"); +$dbh->do("DROP TABLE IF EXISTS `NPL-pgfile-institution`"); + +for my $tableinfo (@create_tables) { + my $tabname = $tableinfo->[0]; + my $tabinit = $tableinfo->[1]; + my $query = "DROP TABLE IF EXISTS `$tabname`"; + $dbh->do($query); + $query = "CREATE TABLE `$tabname` ( $tabinit ) ENGINE=$db_storage_engine CHARACTER SET $character_set"; + $dbh->do($query); + if($lib eq 'OPL') { + $old_tabname = $tabname; + $old_tabname =~ s/OPL/NPL/; + $old_tabname =~ s/_/-/g; + $query = "DROP TABLE IF EXISTS `$old_tabname`"; + $dbh -> do($query); + } +} + + +print "Mysql database reinitialized.\n"; + +# From pgfile +## DBchapter('Limits and Derivatives') +## DBsection('Calculating Limits using the Limit Laws') +## Date('6/3/2002') +## Author('Tangan Gao') +## Institution('csulb') +## TitleText1('Calculus Early Transcendentals') +## EditionText1('4') +## AuthorText1('Stewart') +## Section1('2.3') +## Problem1('7') + +# Get an id, and add entry to the database if needed +sub safe_get_id { + my $tablename = shift; + my $idname = shift; + my $whereclause = shift; + my $wherevalues = shift; + my $addifnew = shift; + my @insertvalues = @_; +#print "\nCalled with $tablename, $idname, $whereclause, [".join(',', @$wherevalues)."], (".join(',', @insertvalues).")\n"; + for my $j (0..$#insertvalues) { + $insertvalues[$j] =~ s/"/\\\"/g; + } + + my $query = "SELECT $idname FROM `$tablename` ".$whereclause; + my $sth = $dbh->prepare($query); + $sth->execute(@$wherevalues); + my $idvalue, @row; + unless(@row = $sth->fetchrow_array()) { + return 0 unless $addifnew; + for my $j (0..$#insertvalues) { + #print "Looking at ".$insertvalues[$j]."\n"; + if ($insertvalues[$j] ne "") { + $insertvalues[$j] = '"'.$insertvalues[$j].'"'; + } else { + $insertvalues[$j] = NULL; + } + } + $dbh->do("INSERT INTO `$tablename` VALUES(". join(',',@insertvalues) .")"); + dbug "INSERT INTO $tablename VALUES( ".join(',',@insertvalues).")\n"; + $sth = $dbh->prepare($query); + $sth->execute(@$wherevalues); + @row = $sth->fetchrow_array(); + } + $idvalue = $row[0]; + return($idvalue); +} + +sub isvalid { + my $tags = shift; + if(not defined $taxo->{$tags->{DBsubject}}) { + print "\nInvalid subject ".$tags->{DBsubject}."\n"; + return 0; + } + if(not ($tags->{DBchapter} eq 'Misc.') and not defined $taxo->{$tags->{DBsubject}}->{$tags->{DBchapter}}) { + print "\nInvalid chapter ".$tags->{DBchapter}."\n"; + return 0; + } + if(not ($tags->{DBsection} eq 'Misc.') and not defined $taxo->{$tags->{DBsubject}}->{$tags->{DBchapter}}->{$tags->{DBsection}}) { + print "\nInvalid section ".$tags->{DBsection}."\n"; + return 0; + } + return 1; +} + +#### First read in textbook information + +if(open(IN, '<:encoding(UTF-8)', "$libraryRoot/Textbooks")) { + print "Reading in textbook data from Textbooks in the library $libraryRoot.\n"; + my %textinfo = ( TitleText => '', EditionText =>'', AuthorText=>''); + my $bookid = undef; + while (my $line = ) { + $line =~ s|#*$||; + if($line =~ /^\s*(.*?)\s*>>>\s*(.*?)\s*$/) { # Should have chapter or section information + my $chapsec = $1; + my $title = $2; + if($chapsec=~ /(\d+)\.(\d+)/) { # We have a section + if(defined($bookid)) { + my $query = "SELECT chapter_id FROM `$tables{chapter}` WHERE textbook_id = \"$bookid\" AND number = \"$1\""; + my $chapid = $dbh->selectrow_array($query); + if(defined($chapid)) { + my $sectid = safe_get_id($tables{section}, 'section_id', + qq(WHERE chapter_id = ? and name = ?), [$chapid, $title], 1, "", $chapid, $2, $title, ""); + } else { + print "Cannot enter section $chapsec because textbook information is missing the chapter entry\n"; + } + } else { + print "Cannot enter section $chapsec because textbook information is incomplete\n"; + } + } else { # We have a chapter entry + if(defined($bookid)) { + my $chapid = safe_get_id($tables{chapter}, 'chapter_id', + qq(WHERE textbook_id = ? AND number = ?), [$bookid, $chapsec], 1, "", $bookid, $chapsec, $title, ""); + + # Add dummy section entry for problems tagged to the chapter + # without a section + $query = "SELECT section_id FROM `$tables{section}` WHERE chapter_id = \"$chapid\" AND number = -1"; + my $sectid = $dbh->selectrow_array($query); + if (!defined($sectid)) { + $dbh->do("INSERT INTO `$tables{section}` + VALUES( + NULL, + \"$chapid\", + \"-1\", + \"\", + NULL + )" + ); + dbug "INSERT INTO section VALUES(\"\", \"$chapid\", \"-1\", \"\", \"\" )\n"; + } + } else { + print "Cannot enter chapter $chapsec because textbook information is incomplete\n"; + } + } + } elsif($line =~ /^\s*(TitleText|EditionText|AuthorText)\(\s*'(.*?)'\s*\)/) { + # Textbook information, maybe new + my $type = $1; + if(defined($textinfo{$type})) { # signals new text + %textinfo = ( TitleText => undef, + EditionText =>undef, + AuthorText=> undef); + $textinfo{$type} = $2; + $bookid = undef; + } else { + $textinfo{$type} = $2; + if(defined($textinfo{TitleText}) and + defined($textinfo{AuthorText}) and + defined($textinfo{EditionText})) { + my $query = "SELECT textbook_id FROM `$tables{textbook}` WHERE title = \"$textinfo{TitleText}\" AND edition = \"$textinfo{EditionText}\" AND author=\"$textinfo{AuthorText}\""; + $bookid = $dbh->selectrow_array($query); + if (!defined($bookid)) { + $dbh->do("INSERT INTO `$tables{textbook}` + VALUES( + NULL, + \"$textinfo{TitleText}\", + \"$textinfo{EditionText}\", + \"$textinfo{AuthorText}\", + NULL, + NULL, + NULL + )" + ); + dbug "INSERT INTO textbook VALUES( \"\", \"$textinfo{TitleText}\", \"$textinfo{EditionText}\", \"$textinfo{AuthorText}\", \"\", \"\", \"\" )\n"; + $bookid = $dbh->selectrow_array($query); + } + } + } + } + } + close(IN); +} else { + print "Textbooks file was not found in library $libraryRoot. If the path to the problem library doesn't seem + correct, make modifications in webwork2/conf/site.conf (\$problemLibrary{root}). If that is correct then + updating from git should download the Textbooks file.\n"; +} +#### End of textbooks + +#### Next read in the taxonomy +my $clsep = '<<<'; +my $clinner = '__'; +my @cllist = (); +# Record full taxonomy for tagging menus (does not include cross-lists) +my $tagtaxo = []; +my ($chaplist, $seclist) = ([],[]); + +my $canopenfile = 0; +if(open(IN, '<:encoding(UTF-8)', "$libraryRoot/Taxonomy2")) { + print "Reading in OPL taxonomy from Taxonomy2 in the library $libraryRoot.\n"; + $canopenfile = 1; +} elsif(open(IN, '<:encoding(UTF-8)', "$libraryRoot/Taxonomy")) { + print "Reading in OPL taxonomy from Taxonomy in the library $libraryRoot.\n"; + $canopenfile = 1; +} else { + print "Taxonomy file was not found in library $libraryRoot. If the path to the problem library doesn't seem + correct, make modifications in webwork2/conf/site.conf (\$problemLibrary{root}). If that is correct then + updating from git should download the Taxonomy file.\n"; +} + +# Taxonomy is a subset of Taxonomy2, so we can use the same code either way +if($canopenfile) { + my ($cursub,$curchap); # these are strings + my ($subj, $chap, $sect); # these are indeces + while(my $line = ) { + $line =~ /^(\t*)/; + my $count = length($1); + my $oktag = 1; + chomp($line); + if($line =~ m/$clsep/) { + $oktag = 0; + my @cross = split $clsep, $line; + @cross = map(trim($_), @cross); + if(scalar(@cross) > 1) { + push @cllist, [join($clinner, ($cursub,$curchap,$cross[0])) ,$cross[1]]; + } + $line = $cross[0]; + } + $line = trim($line); + + # We put the line in the database in all cases + # but crosslists are not put in the heierarchy of legal tags + # instead they go in a list of crosslists to deal with after + # the full taxonomy is read in + if($count == 0) { #DBsubject + $cursub = $line; + if($oktag) { + $taxo->{$line} = {}; + ($chaplist, $seclist) = ([],[]); + push @{$tagtaxo}, {name=>$line, subfields=>$chaplist}; + } + $subj = safe_get_id($tables{dbsubject}, 'DBsubject_id', + qq(WHERE name = ?), [$line], 1, "", $line); + } elsif($count == 1) { #DBchapter + if($oktag) { + $taxo->{$cursub}->{$line} = {}; + $seclist=[]; + push @{$chaplist}, {name=>$line, subfields=>$seclist}; + } + $curchap = $line; + $chap = safe_get_id($tables{dbchapter}, 'DBchapter_id', + qq(WHERE name = ? and DBsubject_id = ?), [$line, $subj], 1, "", $line, $subj); + } else { #DBsection + if($oktag) { + $taxo->{$cursub}->{$curchap}->{$line} = []; + push @{$seclist}, {name=>$line}; + } + $sect = safe_get_id($tables{dbsection}, 'DBsection_id', + qq(WHERE name = ? and DBchapter_id = ?), [$line, $chap], 1, "", $line, $chap); + } + } + close(IN); +} +#### End of taxonomy/taxonomy2 + +use JSON; + +#### Save the official taxonomy in json format +my $webwork_htdocs = $ce->{webworkDirs}{htdocs}; +my $file = "$webwork_htdocs/DATA/tagging-taxonomy.json"; + +writeJSONtoFile($tagtaxo,$file); +print "Saved taxonomy to $file.\n"; + +#### Now deal with cross-listed sections +for my $clinfo (@cllist) { + my @scs = split /$clinner/, $clinfo->[1]; + if(defined $taxo->{$scs[0]}->{$scs[1]}->{$scs[2]}) { + push @{$taxo->{$scs[0]}->{$scs[1]}->{$scs[2]}}, $clinfo->[0]; + } else { + print "Faulty cross-list: pointing to $scs[0] / $scs[1] / $scs[2]\n"; + } +} + +print "Converting data from tagged pgfiles into mysql.\n"; +print "Number of files processed:\n"; + +#### Now search for tagged problems +#recursive search for all pg files + +File::Find::find( { wanted => \&pgfiles, follow_fast => 1 }, $libraryRoot ); +File::Find::find( { wanted => \&pgfiles, follow_fast => 1 }, $contribRoot ); + +sub trim { + my $str = shift; + $str =~ s/^\s+//; + $str =~ s/\s+$//; + return $str; +} + +sub kwtidy { + my $s = shift; + $s =~ s/\W//g; + $s =~ s/_//g; + $s = lc($s); + return($s); +} + +sub keywordcleaner { + my $string = shift; + my @spl1 = split /,/, $string; + my @spl2 = map(kwtidy($_), @spl1); + return(@spl2); +} + +# Save on passing these values around +my %textinfo; + +# Initialize, if needed more text-info information; +sub maybenewtext { + my $textno = shift; + return if defined($textinfo{$textno}); + # So, not defined yet + $textinfo{$textno} = { title => '', author =>'', edition =>'', + section => '', chapter =>'', problems => [] }; +} + +# process each file returned by the find command. +sub pgfiles { + my $name = $File::Find::name; + my ($text, $edition, $textauthor, $textsection, $textproblem); + %textinfo=(); + my @textproblems = (-1); +#print "\n$name"; + + if ($name =~ /\.pg$/) { + $cnt2++; + printf("%6d", $cnt2) if(($cnt2 % 100) == 0); + print "\n" if(($cnt2 % 1000) == 0); + + my $pgfile = basename($name); + my $pgpath = dirname($name); + $pgpath =~ s|^$libraryRoot|Library|; + $pgpath =~ s|^$contribRoot|Contrib|; + $pgpath =~ m|^([^/]*)/(.*)|; + my ($pglib, $pgpath) = ($1, $2); + + my $tags = WeBWorK::Utils::Tags->new($name); + + if ($tags->istagged()) { + # Fill in missing data with Misc. instead of blank + print "\nNO SUBJECT $name\n" unless ($tags->{DBsubject} =~ /\S/); + + $tags->{DBchapter} = 'Misc.' unless $tags->{DBchapter} =~ /\S/; + $tags->{DBsection} = 'Misc.' unless $tags->{DBsection} =~ /\S/; + + # DBsubject table + + if(isvalid($tags)) { + my $DBsubject_id = safe_get_id($tables{dbsubject}, 'DBsubject_id', + qq(WHERE BINARY name = ?), [$tags->{DBsubject}], 1, "", $tags->{DBsubject}); + if(not $DBsubject_id) { + print "\nInvalid subject '$tags->{DBsubject}' in $name\n"; + return; + } + + # DBchapter table + + $DBchapter_id = safe_get_id($tables{dbchapter}, 'DBchapter_id', + qq(WHERE BINARY name = ? and DBsubject_id = ?), [$tags->{DBchapter}, $DBsubject_id], 1, "", $tags->{DBchapter}, $DBsubject_id); + if(not $DBchapter_id) { + print "\nInvalid chapter '$tags->{DBchapter}' in $name\n"; + return; + } + + # DBsection table + + $aDBsection_id = safe_get_id($tables{dbsection}, 'DBsection_id', + qq(WHERE BINARY name = ? and DBchapter_id = ?), [$tags->{DBsection}, $DBchapter_id], 1, "", $tags->{DBsection}, $DBchapter_id); + if(not $aDBsection_id) { + print "\nInvalid section '$tags->{DBsection}' in $name\n"; + return; + } + } else { # tags are not valid, error printed by validation part + print "File $name\n"; + return; + } + + my @DBsection_ids=($aDBsection_id); + # Now add crosslisted section + my @CL_array = @{$taxo->{$tags->{DBsubject}}->{$tags->{DBchapter}}->{$tags->{DBsection}}}; + for my $clsect (@CL_array) { + my @np = split /$clinner/, $clsect; + @np = map(trim($_), @np); + my $new_dbsubj_id = safe_get_id($tables{dbsubject}, 'DBsubject_id', + qq(WHERE name = ?), [$np[0]], 1, "", $np[0]); + my $new_dbchap_id = safe_get_id($tables{dbchapter}, 'DBchapter_id', + qq(WHERE name = ? and DBsubject_id = ?), [$np[1], $new_dbsubj_id], 1, "", $np[1], $new_dbsubj_id); + my $new_dbsect_id = safe_get_id($tables{dbsection}, 'DBsection_id', + qq(WHERE name = ? and DBchapter_id = ?), [$np[2], $new_dbchap_id], 1, "", $np[2], $new_dbchap_id); + push @DBsection_ids, $new_dbsect_id; + } + + # author table + + $tags->{Author} =~ /(.*?)\s(\w+)\s*$/; + my $firstname = $1; + my $lastname = $2; + #remove leading and trailing spaces from firstname, which includes any middle name too. + $firstname =~ s/^\s*//; + $firstname =~ s/\s*$//; + $lastname =~ s/^\s*//; + $lastname =~ s/\s*$//; + my $author_id = 0; + if($lastname) { + $author_id = safe_get_id($tables{author}, 'author_id', + qq(WHERE lastname = ? AND firstname = ?), [$lastname, $firstname], 1, "", $tags->{Institution}, $lastname, $firstname,""); + } + + # path table + + my $path_id = safe_get_id($tables{path}, 'path_id', + qq(WHERE path = ?), [$pgpath], 1, "", $pgpath, "", ""); + + # pgfile table -- set 4 defaults first + + ## TODO this is where we have to deal with crosslists, and pgfileid + ## will be an array of id's + ## Make an array of DBsection-id's above + + my $level = $tags->{Level} || 0; + # Default language is English + my $lang = $tags->{Language} || "en"; + my $mathobj = $tags->{MO} || 0; + my $static = $tags->{Static} || 0; + + my @pgfile_ids = (); + + for my $DBsection_id (@DBsection_ids) { + my $pgfile_id = safe_get_id($tables{pgfile}, 'pgfile_id', + qq(WHERE filename = ? AND path_id = ? AND DBsection_id = ? AND libraryroot = ?), + [$pgfile, $path_id, $DBsection_id, $pglib], 1, "", $DBsection_id, $author_id, + $tags->{Institution}, $pglib, $path_id, $pgfile, 0, $level, $lang, $static, $mathobj); + push @pgfile_ids, $pgfile_id; + } + + # morelt table + + my $morelt_id; + if($tags->{MLT}) { + for my $DBsection_id (@DBsection_ids) { + $morelt_id = safe_get_id($tables{morelt}, 'morelt_id', + qq(WHERE name = ?), [$tags->{MLT}], 1, "", $tags->{MLT}, $DBsection_id, ""); + + for my $pgfile_id (@pgfile_ids) { + $dbh->do("UPDATE `$tables{pgfile}` SET + morelt_id = \"$morelt_id\" WHERE pgfile_id = \"$pgfile_id\" "); + + dbug "UPDATE pgfile morelt_id for $pgfile_id to $morelt_id\n"; + if($tags->{MLTleader}) { + $dbh->do("UPDATE `$tables{morelt}` SET + leader = \"$pgfile_id\" WHERE morelt_id = \"$morelt_id\" "); + dbug "UPDATE morelt leader for $morelt_id to $pgfile_id\n"; + } + } + } + } + + # keyword table, and problem_keyword many-many table + + foreach my $keyword (@{$tags->{keywords}}) { + $keyword =~ s/[\'\"]//g; + $keyword = kwtidy($keyword); + # skip it if it is empty + next unless $keyword; + my $keyword_id = safe_get_id($tables{keyword}, 'keyword_id', + qq(WHERE keyword = ?), [$keyword], 1, "", $keyword); + + for my $pgfile_id (@pgfile_ids) { + $query = "SELECT pgfile_id FROM `$tables{pgfile_keyword}` WHERE keyword_id = \"$keyword_id\" and pgfile_id=\"$pgfile_id\""; + my $ok = $dbh->selectrow_array($query); + if (!defined($ok)) { + $dbh->do("INSERT INTO `$tables{pgfile_keyword}` + VALUES( + \"$pgfile_id\", + \"$keyword_id\" + )" + ); + dbug "INSERT INTO pgfile_keyword VALUES( \"$pgfile_id\", \"$keyword_id\" )\n"; + } + } + } #end foreach keyword + + # Textbook section + # problem table contains textbook problems + + for my $texthashref (@{$tags->{textinfo}}) { + + # textbook table + + $text = $texthashref->{TitleText}; + $edition = $texthashref->{EditionText} || 0; + $edition =~ s/[^\d\.]//g; + $textauthor = $texthashref->{AuthorText}; + next unless($text and $textauthor); + my $chapnum = $texthashref->{chapter} || -1; + my $secnum = $texthashref->{section} || -1; + $query = "SELECT textbook_id FROM `$tables{textbook}` WHERE title = \"$text\" AND edition = \"$edition\" AND author=\"$textauthor\""; + my $textbook_id = $dbh->selectrow_array($query); + if (!defined($textbook_id)) { + # make sure edition is an integer + $edition = 0 unless $edition; + $dbh->do("INSERT INTO `$tables{textbook}` + VALUES( + NULL, + \"$text\", + \"$edition\", + \"$textauthor\", + NULL, + NULL, + NULL + )" + ); + dbug "INSERT INTO textbook VALUES( \"\", \"$text\", \"$edition\", \"$textauthor\", \"\", \"\", \"\" )\n"; + dbug "\nLate add into $tables{textbook} \"$text\", \"$edition\", \"$textauthor\"\n", 1; + $textbook_id = $dbh->selectrow_array($query); + } + + # chapter weak table of textbook + $query = "SELECT chapter_id FROM `$tables{chapter}` WHERE textbook_id = \"$textbook_id\" AND number = \"$chapnum\""; + my $chapter_id = $dbh->selectrow_array($query); + if (!defined($chapter_id) && defined($textbook_id)) { + $dbh->do("INSERT INTO `$tables{chapter}` + VALUES( + NULL, + \"$textbook_id\", + \"".$chapnum."\", + \"$tags->{DBchapter}\", + NULL + )" + ); + dbug "\nLate add into $tables{chapter} \"$text\", \"$edition\", \"$textauthor\", $chapnum $tags->{chapter} from $name\n", 1; + dbug "INSERT INTO chapter VALUES(\"\", \"$textbook_id\", \"".$chapnum."\", \"$tags->{DBchapter}\", \"\" )\n"; + $chapter_id = $dbh->selectrow_array($query); + } + + # section weak table of textbook + # + $tags->{DBsection} = '' if ($secnum < 0); + $query = "SELECT section_id FROM `$tables{section}` WHERE chapter_id = \"$chapter_id\" AND number = \"$secnum\""; + my $section_id = $dbh->selectrow_array($query); + if (!defined($section_id) && defined($chapter_id) && defined($textbook_id)) { + $dbh->do("INSERT INTO `$tables{section}` + VALUES( + NULL, + \"$chapter_id\", + \"$secnum\", + \"$tags->{DBsection}\", + NULL + )" + ); + dbug "INSERT INTO section VALUES(\"\", \"$textbook_id\", \"$secnum\", \"$tags->{DBsection}\", \"\" )\n"; + dbug "\nLate add into $tables{section} \"$text\", \"$edition\", \"$textauthor\", $secnum $tags->{DBsection} from $name\n", 1; + $section_id = $dbh->selectrow_array($query); + } + + @textproblems = @{$texthashref->{problems}}; + for my $tp (@textproblems) { + $query = "SELECT problem_id FROM `$tables{problem}` WHERE section_id = \"$section_id\" AND number = \"$tp\""; + my $problem_id = $dbh->selectrow_array($query); + if (!defined($problem_id)) { + $dbh->do("INSERT INTO `$tables{problem}` + VALUES( + NULL, + \"$section_id\", + \"$tp\", + NULL + )" + ); + dbug "INSERT INTO problem VALUES( \"\", \"$section_id\", \"$tp\", \"\" )\n"; + $problem_id = $dbh->selectrow_array($query); + } + + # pgfile_problem table associates pgfiles with textbook problems + for my $pgfile_id (@pgfile_ids) { + $query = "SELECT problem_id FROM `$tables{pgfile_problem}` WHERE problem_id = \"$problem_id\" AND pgfile_id = \"$pgfile_id\""; + my $pg_problem_id = $dbh->selectrow_array($query); + if (!defined($pg_problem_id)) { + $dbh->do("INSERT INTO `$tables{pgfile_problem}` + VALUES( + \"$pgfile_id\", + \"$problem_id\" + )" + ); + dbug "INSERT INTO pgfile_problem VALUES( \"$pgfile_id\", \"$problem_id\" )\n"; + } + } + } + + #reset tag vars, they may not match the next text/file + $textauthor=""; $textsection=""; + } + } else { # This file was not tagged + # Message if not a pointer + # print STDERR "File $name is not tagged\n" if not $tags->isplaceholder(); + ; + } + } +} + +print "\n\n"; + +# Now prune away DBsection, etc, which do not appear in any files +#%my $query = "SELECT chapter_id FROM `$tables{chapter}` WHERE textbook_id = \"$bookid\" AND number = \"$1\""; +#%my $chapid = $dbh->selectrow_array($query); + +#select dbs.DBsection_id from OPL_DBsection dbs; +#select COUNT(*) from OPL_pgfile where DBsection_id=857; + +my $dbsects = $dbh->selectall_arrayref("SELECT DBsection_id from `$tables{dbsection}`"); +for my $sect (@{$dbsects}) { + $sect = $sect->[0]; + my $srar = $dbh->selectall_arrayref("SELECT * FROM `$tables{pgfile}` WHERE DBsection_id=$sect"); + if(scalar(@{$srar})==0) { + $dbh->do("DELETE FROM `$tables{dbsection}` WHERE DBsection_id=$sect"); + } +} + +my $dbchaps = $dbh->selectall_arrayref("SELECT DBchapter_id from `$tables{dbchapter}`"); +for my $chap (@{$dbchaps}) { + $chap = $chap->[0]; + my $srar = $dbh->selectall_arrayref("SELECT * FROM `$tables{dbsection}` WHERE DBchapter_id=$chap"); + if(scalar(@{$srar})==0) { + $dbh->do("DELETE FROM `$tables{dbchapter}` WHERE DBchapter_id=$chap"); + } +} + +# Note: this used to build some JSON versions of the textbooks, subjects and directory trees +# that could be used in the library browswer. It's functionality is now +# in the updateOPLextras.pl script. + +$dbh->disconnect; + +if ($ce->{problemLibrary}{showLibraryLocalStats} || + $ce->{problemLibrary}{showLibraryGlobalStats}) { + print "\nUpdating Library Statistics.\n"; + do $ENV{WEBWORK_ROOT}.'/bin/update-OPL-statistics.pl'; + + print "\nLoading global statistics (if possible).\n"; + do $ENV{WEBWORK_ROOT}.'/bin/load-OPL-global-statistics.pl'; +} + +# Generate set definition list files. +do $ENV{WEBWORK_ROOT} . '/bin/generate-OPL-set-def-lists.pl'; + +print "\nDone.\n"; diff --git a/bin/update-OPL-statistics.pl b/bin/update-OPL-statistics.pl index 3c1e465260..9bf6ebafd3 100755 --- a/bin/update-OPL-statistics.pl +++ b/bin/update-OPL-statistics.pl @@ -155,7 +155,7 @@ BEGIN `${courseID}_problem`.set_id, `${courseID}_set`.due_date, `${courseID}_problem`.problem_id, - REPLACE(`${courseID}_problem`.source_file,'local/Library/','Library/'), + REPLACE(`${courseID}_problem`.source_file,'local/',''), `${courseID}_problem_user`.status, `${courseID}_problem_user`.attempted, `${courseID}_problem_user`.num_correct, @@ -168,6 +168,8 @@ BEGIN ON `${courseID}_problem_user`.set_id = `${courseID}_set`.set_id WHERE (`${courseID}_problem`.source_file LIKE 'Library/%' OR `${courseID}_problem`.source_file LIKE 'local/Library/%') + OR `${courseID}_problem`.source_file LIKE 'Contrib/%') + OR `${courseID}_problem`.source_file LIKE 'local/Contrib/%') AND `${courseID}_set`.due_date < $time; EOS From 2d598fc500b8fdf44b525fe10e5ea35862db62a9 Mon Sep 17 00:00:00 2001 From: "K. Andrew Parker" Date: Fri, 27 May 2022 10:36:30 -0400 Subject: [PATCH 05/12] clean up sloppy cut-and-paste job --- bin/update-OPL-statistics.pl | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/bin/update-OPL-statistics.pl b/bin/update-OPL-statistics.pl index 9bf6ebafd3..23bedbab65 100755 --- a/bin/update-OPL-statistics.pl +++ b/bin/update-OPL-statistics.pl @@ -37,8 +37,7 @@ BEGIN BEGIN{ my $ce = new WeBWorK::CourseEnvironment({ - webwork_dir => $ENV{WEBWORK_ROOT}, - }); + webwork_dir => $ENV{WEBWORK_ROOT}}); my $pg_dir = $ce->{pg_dir}; eval "use lib '$pg_dir/lib'"; @@ -54,7 +53,7 @@ BEGIN # get course environment and open up database my $ce = new WeBWorK::CourseEnvironment({ webwork_dir => $ENV{WEBWORK_ROOT}, - }); + }); # decide whether the mysql installation can handle # utf8mb4 and that should be used for the OPL @@ -63,14 +62,14 @@ BEGIN my $dbh = DBI->connect( - $ce->{problemLibrary_db}->{dbsource}, - $ce->{problemLibrary_db}->{user}, - $ce->{problemLibrary_db}->{passwd}, - { + $ce->{problemLibrary_db}->{dbsource}, + $ce->{problemLibrary_db}->{user}, + $ce->{problemLibrary_db}->{passwd}, + { AutoCommit => 0, - PrintError => 0, - RaiseError => 1, - }, + PrintError => 0, + RaiseError => 1, + }, ); # get course list @@ -167,8 +166,8 @@ BEGIN JOIN `${courseID}_set` ON `${courseID}_problem_user`.set_id = `${courseID}_set`.set_id WHERE (`${courseID}_problem`.source_file LIKE 'Library/%' - OR `${courseID}_problem`.source_file LIKE 'local/Library/%') - OR `${courseID}_problem`.source_file LIKE 'Contrib/%') + OR `${courseID}_problem`.source_file LIKE 'local/Library/%' + OR `${courseID}_problem`.source_file LIKE 'Contrib/%' OR `${courseID}_problem`.source_file LIKE 'local/Contrib/%') AND `${courseID}_set`.due_date < $time; EOS From e016977dfbe5c4b653b3a0a7c7c027913a20b70d Mon Sep 17 00:00:00 2001 From: "K. Andrew Parker" Date: Fri, 27 May 2022 12:56:35 -0400 Subject: [PATCH 06/12] check for link to Contrib in course templates --- bin/update-OPL-statistics.pl | 14 -------- .../ContentGenerator/Instructor/SetMaker.pm | 35 +++++++++++++------ lib/WeBWorK/Utils/ListingDB.pm | 2 ++ 3 files changed, 27 insertions(+), 24 deletions(-) diff --git a/bin/update-OPL-statistics.pl b/bin/update-OPL-statistics.pl index 23bedbab65..aa43bf98d4 100755 --- a/bin/update-OPL-statistics.pl +++ b/bin/update-OPL-statistics.pl @@ -30,20 +30,6 @@ BEGIN use WeBWorK::CourseEnvironment; use String::ShellQuote; -# hack to set version so that the script runs without warnings in -# earlier versions of WeBWorK, e.g. WW 2.7 - -BEGIN { $main::VERSION = "2.4"; } - -BEGIN{ - my $ce = new WeBWorK::CourseEnvironment({ - webwork_dir => $ENV{WEBWORK_ROOT}}); - - my $pg_dir = $ce->{pg_dir}; - eval "use lib '$pg_dir/lib'"; - die $@ if $@; -} - use DBI; use WeBWorK::Utils::CourseIntegrityCheck; use WeBWorK::Utils::CourseManagement qw/listCourses/; diff --git a/lib/WeBWorK/ContentGenerator/Instructor/SetMaker.pm b/lib/WeBWorK/ContentGenerator/Instructor/SetMaker.pm index 537a32a6f6..c8033e4d12 100644 --- a/lib/WeBWorK/ContentGenerator/Instructor/SetMaker.pm +++ b/lib/WeBWorK/ContentGenerator/Instructor/SetMaker.pm @@ -345,6 +345,7 @@ sub view_problems_line { my $internal_name = shift; my $label = shift; my $r = shift; # so we can get parameter values + my $contrib_exist = (-r $r->ce->{courseDirs}{templates}.'/Contrib') ? 1 : 0; my $result = CGI::start_div({ class => 'd-flex flex-wrap justify-content-center' }); @@ -386,19 +387,33 @@ sub view_problems_line { }) ); + my $maybe_OPL_box = ($contrib_exist) ? CGI::div( + { class => 'form-check form-check-inline ms-2' }, + CGI::checkbox({ + name => "includeOPL", + checked => $r->param('includeOPL') || 1, + label => $r->maketext("Include OPL"), + class => 'form-check-input me-1', + labelattributes => { class => 'form-check-label col-form-label-sm' } + }) + ) : {}; + + my $maybe_contrib_box = ($contrib_exist) ? CGI::div( + { class => 'form-check form-check-inline ms-2' }, + CGI::checkbox({ + name => "includeContrib", + checked => $r->param('includeContrib') || INCLUDE_CONTRIB_DEFAULT, + label => $r->maketext("Include Contrib"), + class => 'form-check-input me-1', + labelattributes => { class => 'form-check-label col-form-label-sm' } + }) + ) : {}; + # Option of whether to show hints and solutions $result .= CGI::div( { class => 'd-inline-block ms-2 mb-2' }, - CGI::div( - { class => 'form-check form-check-inline ms-2' }, - CGI::checkbox({ - name => "includeContrib", - checked => $r->param('includeContrib') || INCLUDE_CONTRIB_DEFAULT, - label => $r->maketext("Include Contrib"), - class => 'form-check-input me-1', - labelattributes => { class => 'form-check-label col-form-label-sm' } - }) - ), + $maybe_OPL_box, + $maybe_contrib_box, CGI::div( { class => 'form-check form-check-inline ms-2' }, CGI::checkbox({ diff --git a/lib/WeBWorK/Utils/ListingDB.pm b/lib/WeBWorK/Utils/ListingDB.pm index 7ace861115..23bcec2b3b 100644 --- a/lib/WeBWorK/Utils/ListingDB.pm +++ b/lib/WeBWorK/Utils/ListingDB.pm @@ -428,6 +428,7 @@ sub getDBListings { my $subj = $r->param('library_subjects') || ""; my $chap = $r->param('library_chapters') || ""; my $sec = $r->param('library_sections') || ""; + my $include_opl = $r->param('includeOPL') // 1; my $include_contrib = $r->param('includeContrib') // 0; # Make sure these strings are internally encoded in UTF-8 @@ -473,6 +474,7 @@ sub getDBListings { push @select_parameters, join(',', @levels); } $extrawhere .= " AND pgf.libraryroot = 'Library' " unless ($include_contrib); + $extrawhere .= " AND pgf.libraryroot = 'Contrib' " unless ($include_opl); my $textextrawhere = ''; my $haveTextInfo=0; my @textInfo_parameters=(); From d5fc20b26d93e935da44fba5480082396ed6d1f8 Mon Sep 17 00:00:00 2001 From: "K. Andrew Parker" Date: Fri, 27 May 2022 13:38:04 -0400 Subject: [PATCH 07/12] maintain checked/unchecked status for checkboxes --- lib/WeBWorK/ContentGenerator/Instructor/SetMaker.pm | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/lib/WeBWorK/ContentGenerator/Instructor/SetMaker.pm b/lib/WeBWorK/ContentGenerator/Instructor/SetMaker.pm index c8033e4d12..870548cd39 100644 --- a/lib/WeBWorK/ContentGenerator/Instructor/SetMaker.pm +++ b/lib/WeBWorK/ContentGenerator/Instructor/SetMaker.pm @@ -390,9 +390,9 @@ sub view_problems_line { my $maybe_OPL_box = ($contrib_exist) ? CGI::div( { class => 'form-check form-check-inline ms-2' }, CGI::checkbox({ - name => "includeOPL", - checked => $r->param('includeOPL') || 1, - label => $r->maketext("Include OPL"), + name => 'includeOPL', + checked => $r->param('includeOPL') // 1, + label => $r->maketext('Include OPL'), class => 'form-check-input me-1', labelattributes => { class => 'form-check-label col-form-label-sm' } }) @@ -401,9 +401,9 @@ sub view_problems_line { my $maybe_contrib_box = ($contrib_exist) ? CGI::div( { class => 'form-check form-check-inline ms-2' }, CGI::checkbox({ - name => "includeContrib", - checked => $r->param('includeContrib') || INCLUDE_CONTRIB_DEFAULT, - label => $r->maketext("Include Contrib"), + name => 'includeContrib', + checked => $r->param('includeContrib') // INCLUDE_CONTRIB_DEFAULT, + label => $r->maketext('Include Contrib'), class => 'form-check-input me-1', labelattributes => { class => 'form-check-label col-form-label-sm' } }) From 18eef1a08db712218bd073f8240e9897e67bad40 Mon Sep 17 00:00:00 2001 From: "K. Andrew Parker" Date: Fri, 27 May 2022 14:15:07 -0400 Subject: [PATCH 08/12] support contrib-only search --- lib/WeBWorK/ContentGenerator/Instructor/SetMaker.pm | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/lib/WeBWorK/ContentGenerator/Instructor/SetMaker.pm b/lib/WeBWorK/ContentGenerator/Instructor/SetMaker.pm index 870548cd39..23d41871cd 100644 --- a/lib/WeBWorK/ContentGenerator/Instructor/SetMaker.pm +++ b/lib/WeBWorK/ContentGenerator/Instructor/SetMaker.pm @@ -396,7 +396,7 @@ sub view_problems_line { class => 'form-check-input me-1', labelattributes => { class => 'form-check-label col-form-label-sm' } }) - ) : {}; + ) : (); my $maybe_contrib_box = ($contrib_exist) ? CGI::div( { class => 'form-check form-check-inline ms-2' }, @@ -407,7 +407,7 @@ sub view_problems_line { class => 'form-check-input me-1', labelattributes => { class => 'form-check-label col-form-label-sm' } }) - ) : {}; + ) : (); # Option of whether to show hints and solutions $result .= CGI::div( @@ -433,7 +433,8 @@ sub view_problems_line { class => 'form-check-input me-1', labelattributes => { class => 'form-check-label col-form-label-sm' } }) - ) + ), + CGI::hidden({name => 'includeOPL', default => 0}); ); $result .= CGI::end_div(); From 9ee1b8045b4bf2e030268bb6877a1e46f7e965cb Mon Sep 17 00:00:00 2001 From: "K. Andrew Parker" Date: Fri, 27 May 2022 14:16:11 -0400 Subject: [PATCH 09/12] stray ; --- lib/WeBWorK/ContentGenerator/Instructor/SetMaker.pm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/WeBWorK/ContentGenerator/Instructor/SetMaker.pm b/lib/WeBWorK/ContentGenerator/Instructor/SetMaker.pm index 23d41871cd..3341f5d3f1 100644 --- a/lib/WeBWorK/ContentGenerator/Instructor/SetMaker.pm +++ b/lib/WeBWorK/ContentGenerator/Instructor/SetMaker.pm @@ -434,7 +434,7 @@ sub view_problems_line { labelattributes => { class => 'form-check-label col-form-label-sm' } }) ), - CGI::hidden({name => 'includeOPL', default => 0}); + CGI::hidden({name => 'includeOPL', default => 0}) ); $result .= CGI::end_div(); From c7fe7cdfd63f4ea581c18b48008f3f63a2932375 Mon Sep 17 00:00:00 2001 From: "K. Andrew Parker" Date: Fri, 27 May 2022 14:26:31 -0400 Subject: [PATCH 10/12] fixing defaults for OPL/Contrib search --- lib/WeBWorK/ContentGenerator/Instructor/SetMaker.pm | 6 +++--- lib/WeBWorK/Utils/ListingDB.pm | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/lib/WeBWorK/ContentGenerator/Instructor/SetMaker.pm b/lib/WeBWorK/ContentGenerator/Instructor/SetMaker.pm index 3341f5d3f1..da6bd70622 100644 --- a/lib/WeBWorK/ContentGenerator/Instructor/SetMaker.pm +++ b/lib/WeBWorK/ContentGenerator/Instructor/SetMaker.pm @@ -387,7 +387,7 @@ sub view_problems_line { }) ); - my $maybe_OPL_box = ($contrib_exist) ? CGI::div( + my $maybe_OPL_box = $contrib_exist ? CGI::div( { class => 'form-check form-check-inline ms-2' }, CGI::checkbox({ name => 'includeOPL', @@ -398,7 +398,7 @@ sub view_problems_line { }) ) : (); - my $maybe_contrib_box = ($contrib_exist) ? CGI::div( + my $maybe_contrib_box = $contrib_exist ? CGI::div( { class => 'form-check form-check-inline ms-2' }, CGI::checkbox({ name => 'includeContrib', @@ -434,7 +434,7 @@ sub view_problems_line { labelattributes => { class => 'form-check-label col-form-label-sm' } }) ), - CGI::hidden({name => 'includeOPL', default => 0}) + CGI::hidden({name => 'includeOPL', default => $contrib_exist ? 0 : 1}) ); $result .= CGI::end_div(); diff --git a/lib/WeBWorK/Utils/ListingDB.pm b/lib/WeBWorK/Utils/ListingDB.pm index 23bcec2b3b..c973a6deac 100644 --- a/lib/WeBWorK/Utils/ListingDB.pm +++ b/lib/WeBWorK/Utils/ListingDB.pm @@ -473,8 +473,8 @@ sub getDBListings { $extrawhere .= " AND pgf.level IN ( ? ) "; push @select_parameters, join(',', @levels); } - $extrawhere .= " AND pgf.libraryroot = 'Library' " unless ($include_contrib); - $extrawhere .= " AND pgf.libraryroot = 'Contrib' " unless ($include_opl); + $extrawhere .= " AND pgf.libraryroot = 'Library' " unless $include_contrib; + $extrawhere .= " AND pgf.libraryroot = 'Contrib' " unless $include_opl; my $textextrawhere = ''; my $haveTextInfo=0; my @textInfo_parameters=(); From 52216ccfc4c849dfc14d78522a712752722060dc Mon Sep 17 00:00:00 2001 From: "K. Andrew Parker" Date: Fri, 27 May 2022 15:03:40 -0400 Subject: [PATCH 11/12] more protection against contrib tags --- bin/OPL-update-legacy | 52 ++++++++++++++++++++++--------------------- 1 file changed, 27 insertions(+), 25 deletions(-) diff --git a/bin/OPL-update-legacy b/bin/OPL-update-legacy index 90e5177429..895ade5085 100755 --- a/bin/OPL-update-legacy +++ b/bin/OPL-update-legacy @@ -851,34 +851,36 @@ sub pgfiles { } @textproblems = @{$texthashref->{problems}}; - for my $tp (@textproblems) { - $query = "SELECT problem_id FROM `$tables{problem}` WHERE section_id = \"$section_id\" AND number = \"$tp\""; - my $problem_id = $dbh->selectrow_array($query); - if (!defined($problem_id)) { - $dbh->do("INSERT INTO `$tables{problem}` - VALUES( - NULL, - \"$section_id\", - \"$tp\", - NULL - )" - ); - dbug "INSERT INTO problem VALUES( \"\", \"$section_id\", \"$tp\", \"\" )\n"; - $problem_id = $dbh->selectrow_array($query); - } - - # pgfile_problem table associates pgfiles with textbook problems - for my $pgfile_id (@pgfile_ids) { - $query = "SELECT problem_id FROM `$tables{pgfile_problem}` WHERE problem_id = \"$problem_id\" AND pgfile_id = \"$pgfile_id\""; - my $pg_problem_id = $dbh->selectrow_array($query); - if (!defined($pg_problem_id)) { - $dbh->do("INSERT INTO `$tables{pgfile_problem}` + if ($section_id) { + for my $tp (@textproblems) { + $query = "SELECT problem_id FROM `$tables{problem}` WHERE section_id = \"$section_id\" AND number = \"$tp\""; + my $problem_id = $dbh->selectrow_array($query); + if (!defined($problem_id)) { + $dbh->do("INSERT INTO `$tables{problem}` VALUES( - \"$pgfile_id\", - \"$problem_id\" + NULL, + \"$section_id\", + \"$tp\", + NULL )" ); - dbug "INSERT INTO pgfile_problem VALUES( \"$pgfile_id\", \"$problem_id\" )\n"; + dbug "INSERT INTO problem VALUES( \"\", \"$section_id\", \"$tp\", \"\" )\n"; + $problem_id = $dbh->selectrow_array($query); + } + + # pgfile_problem table associates pgfiles with textbook problems + for my $pgfile_id (@pgfile_ids) { + $query = "SELECT problem_id FROM `$tables{pgfile_problem}` WHERE problem_id = \"$problem_id\" AND pgfile_id = \"$pgfile_id\""; + my $pg_problem_id = $dbh->selectrow_array($query); + if (!defined($pg_problem_id)) { + $dbh->do("INSERT INTO `$tables{pgfile_problem}` + VALUES( + \"$pgfile_id\", + \"$problem_id\" + )" + ); + dbug "INSERT INTO pgfile_problem VALUES( \"$pgfile_id\", \"$problem_id\" )\n"; + } } } } From e003b3bce6e61d58b012a70f17cc638322d126c2 Mon Sep 17 00:00:00 2001 From: "K. Andrew Parker" Date: Fri, 27 May 2022 15:17:15 -0400 Subject: [PATCH 12/12] protect against non-integer 'Level' --- bin/OPL-update-legacy | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/bin/OPL-update-legacy b/bin/OPL-update-legacy index 895ade5085..210098b95d 100755 --- a/bin/OPL-update-legacy +++ b/bin/OPL-update-legacy @@ -710,21 +710,17 @@ sub pgfiles { # pgfile table -- set 4 defaults first - ## TODO this is where we have to deal with crosslists, and pgfileid - ## will be an array of id's - ## Make an array of DBsection-id's above - - my $level = $tags->{Level} || 0; - # Default language is English - my $lang = $tags->{Language} || "en"; - my $mathobj = $tags->{MO} || 0; - my $static = $tags->{Static} || 0; - + # pgfile table -- set 4 defaults first + my $level = ($tags->{Level} =~ /\d/) ? $tags->{Level} : 0; + my $lang = $tags->{Language} || 'en'; + my $mathobj = $tags->{MO} || 0; + my $static = $tags->{Static} || 0; + my @pgfile_ids = (); for my $DBsection_id (@DBsection_ids) { my $pgfile_id = safe_get_id($tables{pgfile}, 'pgfile_id', - qq(WHERE filename = ? AND path_id = ? AND DBsection_id = ? AND libraryroot = ?), + qq(WHERE filename = ? AND path_id = ? AND DBsection_id = ? AND libraryroot = ?), [$pgfile, $path_id, $DBsection_id, $pglib], 1, "", $DBsection_id, $author_id, $tags->{Institution}, $pglib, $path_id, $pgfile, 0, $level, $lang, $static, $mathobj); push @pgfile_ids, $pgfile_id;