diff --git a/Dockerfile b/Dockerfile index e6fb0506a7..d95e330b6b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -197,7 +197,6 @@ RUN apt-get update \ libjson-xs-perl \ libjson-maybexs-perl \ libcpanel-json-xs-perl \ - libmoox-options-perl \ make \ netpbm \ preview-latex-style \ diff --git a/bin/OPL-update b/bin/OPL-update index d9da002e66..ff2a41cf67 100755 --- a/bin/OPL-update +++ b/bin/OPL-update @@ -82,7 +82,7 @@ use lib "$ENV{WEBWORK_ROOT}/lib"; use lib "$ENV{WEBWORK_ROOT}/bin"; use WeBWorK::CourseEnvironment; use WeBWorK::Utils::Tags; -use OPLUtils qw/build_library_directory_tree build_library_subject_tree build_library_textbook_tree/; +use OPLUtils qw/build_library_directory_tree build_library_subject_tree build_library_textbook_tree writeJSONtoFile/; my $ce = new WeBWorK::CourseEnvironment({webwork_dir=>$ENV{WEBWORK_ROOT}}); @@ -527,13 +527,13 @@ if($canopenfile) { } #### End of taxonomy/taxonomy2 +use JSON; + #### Save the official taxonomy in json format my $webwork_htdocs = $ce->{webwork_dir}."/htdocs"; my $file = "$webwork_htdocs/DATA/tagging-taxonomy.json"; -open(OUTF, ">$file") or die "Cannot open $file"; -binmode(OUTF,':encoding(UTF-8)'); -print OUTF to_json($tagtaxo,{pretty=>1}) or die "Cannot write to $file"; -close(OUTF); + +writeJSONtoFile($tagtaxo,$file); print "Saved taxonomy to $file.\n"; #### Now deal with cross-listed sections @@ -901,84 +901,9 @@ for my $chap (@{$dbchaps}) { } } -# Now run the script build-library-tree -# This is used to create the file library-tree.json which can be used to -# load in subject-chapter-section information for the OPL - -use strict; -use warnings; -use JSON; - - -my $tree; # the library subject tree will be stored as arrays of objects. - -my $sth = $dbh->prepare("select * from OPL_DBsubject"); -$sth->execute; - -my @subjects = (); -my @subject_names = (); -while ( my @row = $sth->fetchrow_array ) { - push(@subjects,$row[0]); - push(@subject_names,$row[1]); - } - - -my @subject_tree; # array to store the individual library tree for each subject - -foreach my $i (0..$#subjects){ - - my $subject_row = $subjects[$i]; - my $subject_name = $subject_names[$i]; - - my $sth = $dbh->prepare("select * from OPL_DBchapter where DBsubject_id = $subject_row;"); - $sth->execute; - - my @chapters = (); - my @chapter_names = (); - while ( my @row = $sth->fetchrow_array ) { - push(@chapters,$row[0]); - push(@chapter_names,$row[1]); - } - - - my @chapter_tree; # array to store the individual library tree for each chapter - - foreach my $j (0..$#chapters) { - my $chapter_row = $chapters[$j]; - my $chapter_name = $chapter_names[$j]; - my $sth = $dbh->prepare("SELECT * FROM `$tables{dbsection}` WHERE DBchapter_id=$chapter_row"); - $sth->execute; - - my @subfields = (); - while ( my @row = $sth->fetchrow_array ) { - my $section_name; - $section_name->{name} = $row[1]; - my $clone = { %{ $section_name } }; # need to clone it before pushing into the @subfields array. - push(@subfields,$clone); - } - - my $chapter_tree; - $chapter_tree->{name} = $chapter_name; - $chapter_tree->{subfields} = \@subfields; - - my $clone = { %{ $chapter_tree } }; # need to clone it before pushing into the @chapter_tree array. - push(@chapter_tree,$clone); - - - - } - - my $subject_tree; - $subject_tree->{name} = $subject_name; - $subject_tree->{subfields} = \@chapter_tree; - - my $clone = { % {$subject_tree}}; - push (@subject_tree, $clone); -} - -build_library_directory_tree($ce); -build_library_subject_tree($ce,$dbh); -build_library_textbook_tree($ce,$dbh); +# Note: this used to build some JSON versions of the textbooks, subjects and directory trees +# that could be used in the library browswer. It's functionality is now +# in the updateOPLextras.pl script. $dbh->disconnect; diff --git a/bin/OPLUtils.pm b/bin/OPLUtils.pm index 0355322044..dcf6c322ab 100755 --- a/bin/OPLUtils.pm +++ b/bin/OPLUtils.pm @@ -3,27 +3,25 @@ package OPLUtils; use base qw(Exporter); -# This file contains the subroutines that build JSON files from the database to help speed up the client side. +# This file contains the subroutines that build JSON files from the database to help speed up the client side. # # The following files are created: # 1. $webwork_htdocs/DATA/library-directory-tree.json (the directory structure of the library) # 2. $webwork_htdocs/DATA/library-subject-tree.json (the subject/chapter/section struture of the library) -# 3. +# 3. $webwork_htdocs/DATA/textbook-tree.json (the subject/chapter/section struture of the library) -# This is used to create the file library-directory-tree.json which can be used to load in -# directory information for the OPL. It writes the file as a JSON of directories to be easily loaded. +# the above JSON files can be used to load and more quickly lookup OPL information +# use strict; use warnings; use File::Find::Rule; use File::Basename; use open qw/:std :utf8/; -# use Cwd; -# use DBI; use JSON; our @EXPORT = (); -our @EXPORT_OK = qw(build_library_directory_tree build_library_subject_tree build_library_textbook_tree); +our @EXPORT_OK = qw(build_library_directory_tree build_library_subject_tree build_library_textbook_tree writeJSONtoFile); ### Data for creating the database tables @@ -68,38 +66,21 @@ my %NPLtables = ( sub build_library_directory_tree { - my $ce = shift; + my ($ce,$verbose) = @_; - print "Creating the Directory Tree\n"; + print "Creating the Directory Tree\n" if $verbose; my $libraryRoot = $ce->{problemLibrary}->{root}; $libraryRoot =~ s|/+$||; - my $libraryVersion = $ce->{problemLibrary}->{version}; - - my($filename, $directories) = fileparse($libraryRoot); - my @dirArray = (); push(@dirArray,buildTree($libraryRoot)); my $webwork_htdocs = $ce->{webwork_dir}."/htdocs"; my $file = "$webwork_htdocs/DATA/library-directory-tree.json"; - # use a variable for the file handle - my $OUTFILE; - - # use the three arguments version of open - # and check for errors - open $OUTFILE, '>encoding(UTF-8)', $file or die "Cannot open $file"; - - # you can check for errors (e.g., if after opening the disk gets full) - print { $OUTFILE } to_json(\@dirArray) or die "Cannot write to $file"; - - # check for errors - close $OUTFILE or die "Cannot close $file"; - - - print "Wrote Library Directory Tree to $file\n"; + writeJSONtoFile(\@dirArray,$file); + print "Wrote Library Directory Tree to $file\n" if $verbose; } sub buildTree { @@ -123,10 +104,9 @@ sub buildTree { } else { $b = {}; $b->{name} = $dir; - + my @files = File::Find::Rule->file()->name("*.pg")->in($absoluteDir . "/" . $dir); - - #print $absoluteDir . "/" . $dir . " " . $b->{num_files} . "\n"; + if (scalar(@files)>0){ $b->{num_files} = scalar(@files); push(@branches,$b); @@ -138,58 +118,50 @@ sub buildTree { my @files = File::Find::Rule->file()->name("*.pg")->in($absoluteDir); $branch->{num_files} = scalar(@files); - return $branch; } sub build_library_subject_tree { - my ($ce,$dbh) = @_; + my ($ce,$dbh,$verbose) = @_; my $libraryRoot = $ce->{problemLibrary}->{root}; $libraryRoot =~ s|/+$||; my $libraryVersion = $ce->{problemLibrary}->{version}; - my %tables = ($libraryVersion eq '2.5')? %OPLtables : %NPLtables; + # query the database for all of the subject names + my $cmd = qq/select name from $tables{dbsubject};/; + my @subject_names = map { $_->[0]} $dbh->selectall_array($cmd); - my $selectClause = "select subj.name, ch.name, sect.name, path.path,pg.filename from `$tables{dbsection}` AS sect " - ."JOIN `$tables{dbchapter}` AS ch ON ch.DBchapter_id = sect.DBchapter_id " - ."JOIN `$tables{dbsubject}` AS subj ON subj.DBsubject_id = ch.DBsubject_id " - ."JOIN `$tables{pgfile}` AS pg ON sect.DBsection_id = pg.DBsection_id " - ."JOIN `$tables{path}` AS path ON pg.path_id = path.path_id "; - - my $tree; # the library subject tree will be stored as arrays of objects. - - my $results = $dbh->selectall_arrayref("select subj.name from `$tables{dbsubject}` AS subj"); - - my @subject_names = map { $_->[0]} @{$results}; + my $tree; # the library subject tree will be stored as arrays of objects. - my $i=0; # counter to print to the screen. + print "Building the subject-tree. There are " . scalar(@subject_names) . " subjects\n" if $verbose; - print "Building the subject-tree. There are " . scalar(@subject_names) . " subjects\n"; + my @subject_tree; # array to store the individual library tree for each subject - my @subject_tree; # array to store the individual library tree for each subject + my $selectClause = ""; for my $subj_name (@subject_names){ - + my $subj = $subj_name; - $subj =~ s/'/\'/g; + $subj =~ s/'/\'/g; # escape any single quotes; + print "subject: $subj_name is being processed.\n" if $verbose; - my $results = $dbh->selectall_arrayref("select ch.name from `$tables{dbsubject}` AS subj JOIN `$tables{dbchapter}` AS ch " - . " ON subj.DBsubject_id = ch.DBsubject_id WHERE subj.name='$subj';"); - my @chapter_names = map {$_->[0]} @{$results}; + my $cmd = qq/SELECT ch.name from $tables{dbchapter} AS ch + JOIN $tables{dbsubject} AS subj ON ch.DBsubject_id=subj.DBsubject_id + WHERE subj.name='$subj';/; + my @chapter_names = map { $_->[0] } $dbh->selectall_array($cmd); + my @chapter_tree; # array to store the individual library tree for each chapter - #print Dumper(\@chapter_names); - for my $ch_name (@chapter_names){ - + my $ch = $ch_name; - $ch =~ s/'/\'/g; + $ch =~ s/'/\'/g; # escape any single quotes; my $results = $dbh->selectall_arrayref("SELECT sect.name from `$tables{dbsubject}` AS subj " ."JOIN `$tables{dbchapter}` AS ch ON subj.DBsubject_id = ch.DBsubject_id " @@ -199,101 +171,71 @@ sub build_library_subject_tree { my @section_names = map { $_->[0]} @{$results}; my @subfields = (); - - for my $sect_name (@section_names){ - my $section_tree = {}; - $section_tree->{name} = $sect_name; - ## Determine the number of files that falls into each - my $sect = $section_tree->{name}; - $sect =~ s/'/\\'/g; + for my $sect_name (@section_names){ + my $section_tree = {name => $sect_name}; - my $whereClause ="WHERE sect.name='$sect' AND ch.name='$ch' AND subj.name='$subj'"; + ## Determine the number of files that falls into each - my $sth = $dbh->prepare($selectClause.$whereClause); - $sth->execute; - my $numFiles= scalar @{$sth->fetchall_arrayref()}; + my $sect = $sect_name; + $sect =~ s/'/\\'/g; # escape any single quotes - $section_tree->{num_files} = $numFiles; + my $cmd = qq/SELECT COUNT(*) from $tables{dbsection} AS sect + JOIN $tables{dbchapter} AS ch ON sect.DBchapter_id = ch.DBchapter_id + JOIN $tables{dbsubject} AS subj ON subj.DBsubject_id = ch.DBsubject_id + JOIN $tables{pgfile} AS pg ON sect.DBsection_id = pg.DBsection_id + where subj.name = '$subj' AND ch.name='$ch' AND sect.name='$sect';/; + $section_tree->{num_files} = $dbh->selectrow_array($cmd); my $clone = { %{ $section_tree } }; # need to clone it before pushing into the @subfield array. - push(@subfields,$clone); - } + push(@subfields,$clone); + } - my $chapter_tree; - $chapter_tree->{name} = $ch_name; - $chapter_tree->{subfields} = \@subfields; + my $chapter_tree = {name => $ch_name, subfields => \@subfields}; ## determine the number of files in each chapter - my $whereClause ="WHERE subj.name='$subj' AND ch.name='$ch'"; + my $cmd = qq/select COUNT(*) from $tables{dbsection} AS sect + JOIN $tables{dbchapter} AS ch ON sect.DBchapter_id = ch.DBchapter_id + JOIN $tables{dbsubject} AS subj ON subj.DBsubject_id = ch.DBsubject_id + JOIN $tables{pgfile} AS pg ON sect.DBsection_id = pg.DBsection_id + JOIN $tables{path} AS path ON pg.path_id = path.path_id + where ch.name = '$ch' AND subj.name = '$subj_name';/; - - my $sth = $dbh->prepare($selectClause.$whereClause); - $sth->execute; - my $numFiles = scalar @{$sth->fetchall_arrayref()}; - # my $allFiles = $sth->fetchall_arrayref; - $chapter_tree->{num_files} = $numFiles; + $chapter_tree->{num_files} = $dbh->selectrow_array($cmd); my $clone = { %{ $chapter_tree } }; # need to clone it before pushing into the @chapter_tree array. push(@chapter_tree,$clone); - - - } - my $subject_tree; - $subject_tree->{name} = $subj_name; - $subject_tree->{subfields} = \@chapter_tree; + my $subject_tree = {name => $subj_name, subfields => \@chapter_tree}; ## find the number of files on the subject level - my $whereClause ="WHERE subj.name='$subj'"; - - - my $sth = $dbh->prepare($selectClause.$whereClause); - $sth->execute; - my $numFiles = scalar @{$sth->fetchall_arrayref()}; - $subject_tree->{num_files} = $numFiles; - - $i++; + $cmd = qq/select COUNT(*) from $tables{dbsection} AS sect + JOIN $tables{dbchapter} AS ch ON sect.DBchapter_id = ch.DBchapter_id + JOIN $tables{dbsubject} AS subj ON subj.DBsubject_id = ch.DBsubject_id + JOIN $tables{pgfile} AS pg ON sect.DBsection_id = pg.DBsection_id + JOIN $tables{path} AS path ON pg.path_id = path.path_id + where subj.name = '$subj_name';/; - print sprintf("%3d", $i); - - if ($i%10 == 0) { - print "\n"; - } + $subject_tree->{num_files} = $dbh->selectrow_array($cmd); my $clone = { % {$subject_tree}}; push (@subject_tree, $clone); } - - print "\n"; - my $webwork_htdocs = $ce->{webwork_dir}."/htdocs"; my $file = "$webwork_htdocs/DATA/library-subject-tree.json"; - # use a variable for the file handle - my $OUTFILE; - - # use the three arguments version of open - # and check for errors - open $OUTFILE, '>encoding(UTF-8)', $file or die "Cannot open $file"; - - # you can check for errors (e.g., if after opening the disk gets full) - print { $OUTFILE } to_json(\@subject_tree,{pretty=>1}) or die "Cannot write to $file"; + writeJSONtoFile(\@subject_tree,$file); - # check for errors - close $OUTFILE or die "Cannot close $file"; - - - print "Wrote Library Subject Tree to $file\n"; + print "Wrote Library Subject Tree to $file\n" if $verbose; } sub build_library_textbook_tree { - my ($ce,$dbh) = @_; + my ($ce,$dbh,$verbose) = @_; my $libraryRoot = $ce->{problemLibrary}->{root}; $libraryRoot =~ s|/+$||; @@ -301,29 +243,30 @@ sub build_library_textbook_tree { my %tables = ($libraryVersion eq '2.5')? %OPLtables : %NPLtables; - - my $selectClause = "SELECT pg.pgfile_id from `$tables{path}` as path " - ."LEFT JOIN `$tables{pgfile}` AS pg ON pg.path_id=path.path_id " - ."LEFT JOIN `$tables{pgfile_problem}` AS pgprob ON pgprob.pgfile_id=pg.pgfile_id " - ."LEFT JOIN `$tables{problem}` AS prob ON prob.problem_id=pgprob.problem_id " - ."LEFT JOIN `$tables{section}` AS sect ON sect.section_id=prob.section_id " - ."LEFT JOIN `$tables{chapter}` AS ch ON ch.chapter_id=sect.chapter_id " - ."LEFT JOIN `$tables{textbook}` AS text ON text.textbook_id=ch.textbook_id "; + my $selectClause = "SELECT pg.pgfile_id from $tables{path} as path " + ."LEFT JOIN $tables{pgfile} AS pg ON pg.path_id=path.path_id " + ."LEFT JOIN $tables{pgfile_problem} AS pgprob ON pgprob.pgfile_id=pg.pgfile_id " + ."LEFT JOIN $tables{problem} AS prob ON prob.problem_id=pgprob.problem_id " + ."LEFT JOIN $tables{section} AS sect ON sect.section_id=prob.section_id " + ."LEFT JOIN $tables{chapter} AS ch ON ch.chapter_id=sect.chapter_id " + ."LEFT JOIN $tables{textbook} AS text ON text.textbook_id=ch.textbook_id "; my $results = $dbh->selectall_arrayref("select * from `$tables{textbook}` ORDER BY title;"); my @textbooks=map { {textbook_id=>$_->[0],title=>$_->[1],edition=>$_->[2], author=>$_->[3],publisher=>$_->[4],isbn=>$_->[5],pubdate=>$_->[6]}} @{$results}; + my @output = (); + my $i =0; ## index to alert user the length of the build - print "Building the Textbook Library Tree\n"; - print "There are ". $#textbooks ." textbooks to process.\n"; + print "Building the Textbook Library Tree\n" if $verbose; + print "There are ". $#textbooks ." textbooks to process.\n" if $verbose; for my $textbook (@textbooks){ $i++; - printf("%4d",$i); - print("\n") if ($i %10==0); + printf("%4d",$i) if $verbose; + print("\n") if ($i % 10==0 && $verbose); my $results = $dbh->selectall_arrayref("select ch.chapter_id,ch.name,ch.number " . " from `$tables{chapter}` AS ch JOIN `$tables{textbook}` AS text ON ch.textbook_id=text.textbook_id " @@ -331,6 +274,8 @@ sub build_library_textbook_tree { my @chapters=map { {chapter_id=>$_->[0],name=>$_->[1],number=>$_->[2]}} @{$results}; + my @chs = (); + for my $chapter (@chapters){ my $results = $dbh->selectall_arrayref("select sect.section_id,sect.name,sect.number " @@ -345,24 +290,36 @@ sub build_library_textbook_tree { for my $section (@sections){ - my $whereClause ="WHERE sect.section_id='". $section->{section_id} + my $whereClause ="WHERE sect.section_id='". $section->{section_id} ."' AND ch.chapter_id='". $chapter->{chapter_id}."' AND " ."text.textbook_id='".$textbook->{textbook_id}."'"; my $sth = $dbh->prepare($selectClause.$whereClause); $sth->execute; $section->{num_probs}=scalar @{$sth->fetchall_arrayref()}; - } my $whereClause ="WHERE ch.chapter_id='". $chapter->{chapter_id}."' AND " - ."text.textbook_id='".$textbook->{textbook_id}."'"; + ."text.textbook_id='".$textbook->{textbook_id}."'"; my $sth = $dbh->prepare($selectClause.$whereClause); $sth->execute; $chapter->{num_probs}=scalar @{$sth->fetchall_arrayref()}; $chapter->{sections}=\@sections; - + + my @sects = map {{ + name=>$_->{name}, + section_id => $_->{section_id}, + num_files=>$_->{num_probs} + }} @sections; + + push(@chs,{ + name=>$chapter->{name}, + chapter_id => $chapter->{chapter_id}, + num_files=>$chapter->{num_probs}, + subfields=>\@sects + }); + } my $whereClause ="WHERE text.textbook_id='".$textbook->{textbook_id}."'"; @@ -371,6 +328,13 @@ sub build_library_textbook_tree { $textbook->{num_probs}=scalar @{$sth->fetchall_arrayref()}; $textbook->{chapters}=\@chapters; + + push(@output,{ + name=>$textbook->{title}. " - " . $textbook->{author}, + textbook_id => $textbook->{textbook_id}, + subfields=>\@chs, + num_files=>$sth->rows + }); } print "\n"; @@ -378,23 +342,20 @@ sub build_library_textbook_tree { my $webwork_htdocs = $ce->{webwork_dir}."/htdocs"; my $file = "$webwork_htdocs/DATA/textbook-tree.json"; - # use a variable for the file handle - my $OUTFILE; - - # use the three arguments version of open - # and check for errors - open $OUTFILE, '>', $file or die "Cannot open $file"; + writeJSONtoFile(\@output,$file); - # you can check for errors (e.g., if after opening the disk gets full) - print { $OUTFILE } to_json(\@textbooks,{pretty=>1}) or die "Cannot write to $file"; - - # check for errors - close $OUTFILE or die "Cannot close $file"; + print "\n\nWrote Library Textbook Tree to $file\n" if $verbose; +} - print "Wrote Library Textbook Tree to $file\n"; +# this takes a hash created in the other subroutines and write the result to a file +sub writeJSONtoFile { + my ($data,$filename) = @_; + my $json = JSON->new->utf8->encode($data); + open my $fh, ">", $filename or die "Cannot open $filename"; + print $fh $json; + close $fh; } - 1; diff --git a/bin/check_modules.pl b/bin/check_modules.pl index 140752b39b..f57cd6516f 100755 --- a/bin/check_modules.pl +++ b/bin/check_modules.pl @@ -10,7 +10,7 @@ mv mysql tar - git + git gzip latex pdflatex @@ -87,8 +87,6 @@ Locale::Maketext::Simple LWP::Protocol::https MIME::Base64 - Moo - MooX::Options Net::IP Net::LDAPS Net::OAuth diff --git a/bin/updateOPLextras.pl b/bin/updateOPLextras.pl new file mode 100755 index 0000000000..73430eb880 --- /dev/null +++ b/bin/updateOPLextras.pl @@ -0,0 +1,94 @@ +#!/usr/bin/perl + +=head1 NAME + +updateOPLextras - re-build library trees + +=head1 SYNOPSIS + +updateOPLextras [options] + + Options: + -t --textbooks (rebuild textbook tree) + -s --subjects (rebuild subject tree) + -d --directories (rebuild directory tree) + -a --all (rebuild all trees) + -h --help (display this text) + +=head1 OPTIONS + +=over 8 + +=item B<-t> I<--textbooks> + +Rebuild the textbook tree and write to a JSON file. + +=item B<-s> I<--subjects> + +Rebuild the subject tree and write to a JSON file. + +=item B<-d> I<--directories> + +Rebuild the directory tree and write to a JSON file. + +=back + +=head1 DESCRIPTION + +B will rebuild the specified library trees +from the existing library contents in the database. + +=cut + +use strict; +use warnings; +use DBI; +use Getopt::Long; +use Pod::Usage; +Getopt::Long::Configure ("bundling"); + +my ($textbooks, $directories, $subjects, $verbose, $all); +GetOptions ( + 't|textbooks' => \$textbooks, + 'd|directories' => \$directories, + 's|subjects' => \$subjects, + 'a|all' => \$all, + 'v|verbose' => \$verbose +); +pod2usage(2) unless ($textbooks || $directories || $subjects || $all); + +##### +# +# This script allows to rerun a few scripts related to the OPL but doesn't require +# the entire OPLupdate script to be run. +# +#### + + +BEGIN { + die "WEBWORK_ROOT not found in environment.\n" unless exists $ENV{WEBWORK_ROOT}; +} + +use lib "$ENV{WEBWORK_ROOT}/bin"; +use lib "$ENV{WEBWORK_ROOT}/lib"; +use WeBWorK::CourseEnvironment; +use OPLUtils qw/build_library_directory_tree build_library_subject_tree build_library_textbook_tree/; + +my $ce = new WeBWorK::CourseEnvironment({webwork_dir=>$ENV{WEBWORK_ROOT}}); + +my $dbh = DBI->connect( + $ce->{problemLibrary_db}->{dbsource}, + $ce->{problemLibrary_db}->{user}, + $ce->{problemLibrary_db}->{passwd}, + { + PrintError => 0, + RaiseError => 1, + ($ce->{ENABLE_UTF8MB4})?(mysql_enable_utf8mb4 =>1):(mysql_enable_utf8 => 1), + }, +); + +build_library_textbook_tree($ce,$dbh,$verbose) if ($all || $textbooks); +build_library_directory_tree($ce,$verbose) if ($all || $directories); +build_library_subject_tree($ce,$dbh,$verbose) if ($all || $subjects); + +1;