From 4b76167e2dd2c94dfc57cd1737519692e6db83d5 Mon Sep 17 00:00:00 2001
From: Peter Staab <peter.staab@gmail.com>
Date: Wed, 26 Jun 2019 10:04:30 -0400
Subject: [PATCH 1/8] initial checkin of updateOPLextras script and some
 changes to the OPLutils.pm file.

---
 bin/OPLUtils.pm     | 58 ++++++++++++++++++++++----------
 bin/updateOPLextras | 80 +++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 121 insertions(+), 17 deletions(-)
 create mode 100644 bin/updateOPLextras

diff --git a/bin/OPLUtils.pm b/bin/OPLUtils.pm
index 0355322044..a3a7f3d23a 100755
--- a/bin/OPLUtils.pm
+++ b/bin/OPLUtils.pm
@@ -3,15 +3,15 @@
 package OPLUtils;
 use base qw(Exporter);
 
-# This file contains the subroutines that build JSON files from the database to help speed up the client side. 
+# This file contains the subroutines that build JSON files from the database to help speed up the client side.
 #
 #  The following files are created:
 #		1. $webwork_htdocs/DATA/library-directory-tree.json  (the directory structure of the library)
 #		2. $webwork_htdocs/DATA/library-subject-tree.json  (the subject/chapter/section struture of the library)
-#		3. 
+#		3.
 
-# This is used to create the file library-directory-tree.json which can be used to load in 
-# directory information for the OPL.  It writes the file as a JSON of directories to be easily loaded. 
+# This is used to create the file library-directory-tree.json which can be used to load in
+# directory information for the OPL.  It writes the file as a JSON of directories to be easily loaded.
 
 use strict;
 use warnings;
@@ -123,9 +123,9 @@ sub buildTree {
 		} else {
 			$b = {};
 			$b->{name} = $dir;
-			
+
 			my @files = File::Find::Rule->file()->name("*.pg")->in($absoluteDir . "/" . $dir);
-			
+
 			#print $absoluteDir . "/" . $dir . "   " . $b->{num_files} . "\n";
 			if (scalar(@files)>0){
 				$b->{num_files} = scalar(@files);
@@ -160,7 +160,7 @@ sub build_library_subject_tree {
 		."JOIN `$tables{pgfile}` AS pg ON sect.DBsection_id = pg.DBsection_id "
 		."JOIN `$tables{path}` AS path ON pg.path_id = path.path_id ";
 
-	my $tree;  # the library subject tree will be stored as arrays of objects. 
+	my $tree;  # the library subject tree will be stored as arrays of objects.
 
 	my $results = $dbh->selectall_arrayref("select subj.name from `$tables{dbsubject}` AS subj");
 
@@ -170,10 +170,10 @@ sub build_library_subject_tree {
 
 	print "Building the subject-tree.  There are " . scalar(@subject_names) . " subjects\n";
 
-	my @subject_tree;  # array to store the individual library tree for each subject 
+	my @subject_tree;  # array to store the individual library tree for each subject
 
 	for my $subj_name (@subject_names){
-		
+
 		my $subj = $subj_name;
 		$subj =~ s/'/\'/g;
 
@@ -187,7 +187,7 @@ sub build_library_subject_tree {
 		#print Dumper(\@chapter_names);
 
 		for my  $ch_name (@chapter_names){
-		
+
 			my $ch = $ch_name;
 			$ch =~ s/'/\'/g;
 
@@ -199,7 +199,7 @@ sub build_library_subject_tree {
 			my @section_names = map { $_->[0]} @{$results};
 
 			my @subfields = ();
-			
+
 			for my $sect_name (@section_names){
 				my $section_tree = {};
 				$section_tree->{name} = $sect_name;
@@ -243,9 +243,9 @@ sub build_library_subject_tree {
 
 		}
 
-		my $subject_tree; 
+		my $subject_tree;
 		$subject_tree->{name} = $subj_name;
-		$subject_tree->{subfields} = \@chapter_tree; 
+		$subject_tree->{subfields} = \@chapter_tree;
 
 		## find the number of files on the subject level
 
@@ -315,6 +315,8 @@ sub build_library_textbook_tree {
 	my @textbooks=map { {textbook_id=>$_->[0],title=>$_->[1],edition=>$_->[2],
 			author=>$_->[3],publisher=>$_->[4],isbn=>$_->[5],pubdate=>$_->[6]}} @{$results};
 
+	my @output = ();
+
 	my $i =0; ## index to alert user the length of the build
 
 	print "Building the Textbook Library Tree\n";
@@ -331,6 +333,8 @@ sub build_library_textbook_tree {
 
 		my @chapters=map { {chapter_id=>$_->[0],name=>$_->[1],number=>$_->[2]}} @{$results};
 
+		my @chs = ();
+
 		for my $chapter (@chapters){
 
 			my $results = $dbh->selectall_arrayref("select sect.section_id,sect.name,sect.number "
@@ -345,7 +349,7 @@ sub build_library_textbook_tree {
 
 			for my $section (@sections){
 
-		   		my $whereClause ="WHERE sect.section_id='". $section->{section_id} 
+		   		my $whereClause ="WHERE sect.section_id='". $section->{section_id}
 		   			."' AND ch.chapter_id='". $chapter->{chapter_id}."' AND "
 		   				."text.textbook_id='".$textbook->{textbook_id}."'";
 
@@ -362,7 +366,20 @@ sub build_library_textbook_tree {
 			$chapter->{num_probs}=scalar @{$sth->fetchall_arrayref()};
 
 			$chapter->{sections}=\@sections;
-		
+
+			my @sects = map {{
+				name=>$_->{name},
+				section_id => $_->{section_id},
+				num_files=>$_->{num_probs}
+			}} @sections;
+
+			push(@chs,{
+				name=>$chapter->{name},
+				chapter_id => $chapter->{chapter_id},
+				num_files=>$chapter->{num_probs},
+				subfields=>\@sects
+			});
+
 		}
 		my $whereClause ="WHERE text.textbook_id='".$textbook->{textbook_id}."'";
 
@@ -371,6 +388,13 @@ sub build_library_textbook_tree {
 		$textbook->{num_probs}=scalar @{$sth->fetchall_arrayref()};
 
 		$textbook->{chapters}=\@chapters;
+
+		push(@output,{
+			name=>$textbook->{title}. " - " . $textbook->{author},
+			textbook_id => $textbook->{textbook_id},
+			subfields=>\@chs,
+			num_files=>$sth->rows
+		});
 	}
 
 	print "\n";
@@ -386,13 +410,13 @@ sub build_library_textbook_tree {
 	open $OUTFILE, '>', $file  or die "Cannot open $file";
 
 	# you can check for errors (e.g., if after opening the disk gets full)
-	print { $OUTFILE } to_json(\@textbooks,{pretty=>1}) or die "Cannot write to $file";
+	print { $OUTFILE } to_json(\@output,{pretty=>1}) or die "Cannot write to $file";
 
 	# check for errors
 	close $OUTFILE or die "Cannot close $file";
 
 
-	print "Wrote Library Textbook Tree to $file\n";
+	print "\n\nWrote Library Textbook Tree to $file\n";
 
 }
 
diff --git a/bin/updateOPLextras b/bin/updateOPLextras
new file mode 100644
index 0000000000..abfc163948
--- /dev/null
+++ b/bin/updateOPLextras
@@ -0,0 +1,80 @@
+#!/usr/bin/perl
+
+#####
+#
+#  This script allows to rerun a few scripts related to the OPL but doesn't require
+#  the entire OPLupdate script to be run.
+#
+####
+
+
+use strict;
+use warnings;
+use feature 'say';
+use Moo;
+use MooX::Options;
+use Data::Dump qw/dd/;
+use DBI;
+
+
+option textbooks  => (
+  is => 'ro',
+  doc => 'run the script to update the OPL textbooks and write to a JSON file'
+);
+option directories => (
+  is => 'ro',
+  doc => 'run the script to update the OPL directories and write to a JSON file'
+);
+option subjects => (
+  is => 'ro',
+  doc => 'run the script to update the OPL subjects and write to a JSON file'
+);
+option all => (
+  is => 'ro',
+  doc => 'run all the scripts'
+);
+
+BEGIN {
+  die "WEBWORK_ROOT not found in environment.\n" unless exists $ENV{WEBWORK_ROOT};
+}
+
+use lib "$ENV{WEBWORK_ROOT}/bin";
+use lib "$ENV{WEBWORK_ROOT}/lib";
+use WeBWorK::CourseEnvironment;
+use OPLUtils qw/build_library_directory_tree build_library_subject_tree build_library_textbook_tree/;
+
+sub run {
+  my ($self) = @_;
+
+  my $ce = new WeBWorK::CourseEnvironment({webwork_dir=>$ENV{WEBWORK_ROOT}});
+
+  my $dbh = DBI->connect(
+          $ce->{problemLibrary_db}->{dbsource},
+          $ce->{problemLibrary_db}->{user},
+          $ce->{problemLibrary_db}->{passwd},
+          {
+              PrintError => 0,
+              RaiseError => 1,
+              ($ce->{ENABLE_UTF8MB4})?(mysql_enable_utf8mb4 =>1):(mysql_enable_utf8 => 1),
+          },
+  );
+
+  if (!$self->all && $self->textbooks){
+    build_library_textbook_tree($ce,$dbh);
+  }
+  if (!$self->all && $self->directories){
+    build_library_directory_tree($ce,$dbh);
+  }
+  if (!$self->all && $self->subjects){
+    build_library_subject_tree($ce,$dbh);
+  }
+  if ($self->all){
+    build_library_textbook_tree($ce,$dbh);
+    build_library_directory_tree($ce,$dbh);
+    build_library_subject_tree($ce,$dbh);
+  }
+}
+
+main->new_with_options->run;
+
+1;

From fbd526e323205172f96fbdba433ab7886fd4b85e Mon Sep 17 00:00:00 2001
From: Peter Staab <peter.staab@gmail.com>
Date: Sat, 6 Mar 2021 07:55:42 -0500
Subject: [PATCH 2/8] sped up the script and cleaned up the code.

---
 bin/OPLUtils.pm                             | 154 ++++++++------------
 bin/{updateOPLextras => updateOPLextras.pl} |  29 ++--
 2 files changed, 75 insertions(+), 108 deletions(-)
 rename bin/{updateOPLextras => updateOPLextras.pl} (75%)
 mode change 100644 => 100755

diff --git a/bin/OPLUtils.pm b/bin/OPLUtils.pm
index a3a7f3d23a..ccf682eb34 100755
--- a/bin/OPLUtils.pm
+++ b/bin/OPLUtils.pm
@@ -8,10 +8,10 @@ use base qw(Exporter);
 #  The following files are created:
 #		1. $webwork_htdocs/DATA/library-directory-tree.json  (the directory structure of the library)
 #		2. $webwork_htdocs/DATA/library-subject-tree.json  (the subject/chapter/section struture of the library)
-#		3.
+#		3. $webwork_htdocs/DATA/library-textbook-tree.json  (the subject/chapter/section struture of the library)
 
-# This is used to create the file library-directory-tree.json which can be used to load in
-# directory information for the OPL.  It writes the file as a JSON of directories to be easily loaded.
+# the above JSON files can be used to load and more quickly lookup OPL information
+# 
 
 use strict;
 use warnings;
@@ -21,6 +21,7 @@ use open qw/:std :utf8/;
 # use Cwd;
 # use DBI;
 use JSON;
+use Data::Dump qw/dd/;
 
 our @EXPORT    = ();
 our @EXPORT_OK = qw(build_library_directory_tree build_library_subject_tree build_library_textbook_tree);
@@ -68,9 +69,9 @@ my %NPLtables = (
 
 
 sub build_library_directory_tree {
-	my $ce = shift;
+	my ($ce,$verbose) = @_;
 
-	print "Creating the Directory Tree\n";
+	print "Creating the Directory Tree\n" if $verbose; 
 	my $libraryRoot = $ce->{problemLibrary}->{root};
 	$libraryRoot =~ s|/+$||;
 
@@ -97,9 +98,7 @@ sub build_library_directory_tree {
 	# check for errors
 	close $OUTFILE or die "Cannot close $file";
 
-
-	print "Wrote Library Directory Tree to $file\n";
-
+	print "Wrote Library Directory Tree to $file\n" if $verbose; 
 }
 
 sub buildTree {
@@ -126,7 +125,6 @@ sub buildTree {
 
 			my @files = File::Find::Rule->file()->name("*.pg")->in($absoluteDir . "/" . $dir);
 
-			#print $absoluteDir . "/" . $dir . "   " . $b->{num_files} . "\n";
 			if (scalar(@files)>0){
 				$b->{num_files} = scalar(@files);
 				push(@branches,$b);
@@ -138,58 +136,50 @@ sub buildTree {
 	my @files = File::Find::Rule->file()->name("*.pg")->in($absoluteDir);
 	$branch->{num_files} = scalar(@files);
 
-
 	return $branch;
 }
 
 
 sub build_library_subject_tree {
-	my ($ce,$dbh) = @_;
+	my ($ce,$dbh,$verbose) = @_;
 
 	my $libraryRoot = $ce->{problemLibrary}->{root};
 	$libraryRoot =~ s|/+$||;
 	my $libraryVersion = $ce->{problemLibrary}->{version};
 
-
 	my %tables = ($libraryVersion eq '2.5')? %OPLtables : %NPLtables;
 
-
-	my $selectClause = "select subj.name, ch.name, sect.name, path.path,pg.filename from `$tables{dbsection}` AS sect "
-		."JOIN `$tables{dbchapter}` AS ch ON ch.DBchapter_id = sect.DBchapter_id "
-		."JOIN `$tables{dbsubject}` AS subj ON subj.DBsubject_id = ch.DBsubject_id "
-		."JOIN `$tables{pgfile}` AS pg ON sect.DBsection_id = pg.DBsection_id "
-		."JOIN `$tables{path}` AS path ON pg.path_id = path.path_id ";
+	# query the database for all of the subject names
+	my $cmd = qq/select name from $tables{dbsubject};/;
+	my @subject_names = map { $_->[0]} $dbh->selectall_array($cmd);
 
 	my $tree;  # the library subject tree will be stored as arrays of objects.
 
-	my $results = $dbh->selectall_arrayref("select subj.name from `$tables{dbsubject}` AS subj");
-
-	my @subject_names = map { $_->[0]} @{$results};
-
-	my $i=0; # counter to print to the screen.
-
-	print "Building the subject-tree.  There are " . scalar(@subject_names) . " subjects\n";
+	print "Building the subject-tree.  There are " . scalar(@subject_names) . " subjects\n" if $verbose; 
 
 	my @subject_tree;  # array to store the individual library tree for each subject
 
+	my $selectClause = "";
+
 	for my $subj_name (@subject_names){
 
 		my $subj = $subj_name;
-		$subj =~ s/'/\'/g;
+		$subj =~ s/'/\'/g; # escape any single quotes; 
 
+    print "subject: $subj_name is being processed.\n" if $verbose; 
 
-		my $results = $dbh->selectall_arrayref("select ch.name from `$tables{dbsubject}` AS subj JOIN `$tables{dbchapter}` AS ch "
-				. " ON subj.DBsubject_id = ch.DBsubject_id WHERE subj.name='$subj';");
-		my @chapter_names = map {$_->[0]} @{$results};
+	  my $cmd = qq/SELECT ch.name from $tables{dbchapter} AS ch
+			JOIN $tables{dbsubject} AS subj ON ch.DBsubject_id=subj.DBsubject_id
+			WHERE subj.name='$subj';/;
 
+		my @chapter_names = map { $_->[0] } $dbh->selectall_array($cmd);
+	
 		my @chapter_tree; # array to store the individual library tree for each chapter
 
-		#print Dumper(\@chapter_names);
-
 		for my  $ch_name (@chapter_names){
 
 			my $ch = $ch_name;
-			$ch =~ s/'/\'/g;
+			$ch =~ s/'/\'/g; # escape any single quotes;
 
 			my $results = $dbh->selectall_arrayref("SELECT sect.name from `$tables{dbsubject}` AS subj "
 					."JOIN `$tables{dbchapter}` AS ch ON subj.DBsubject_id = ch.DBsubject_id "
@@ -201,69 +191,54 @@ sub build_library_subject_tree {
 			my @subfields = ();
 
 			for my $sect_name (@section_names){
-				my $section_tree = {};
-				$section_tree->{name} = $sect_name;
-				## Determine the number of files that falls into each
+				my $section_tree = {name => $sect_name};
 
-				my $sect = $section_tree->{name};
-				$sect =~ s/'/\\'/g;
-
-				my $whereClause ="WHERE sect.name='$sect' AND ch.name='$ch' AND subj.name='$subj'";
+				## Determine the number of files that falls into each
 
-				my $sth = $dbh->prepare($selectClause.$whereClause);
-				$sth->execute;
-				my $numFiles= scalar @{$sth->fetchall_arrayref()};
+				my $sect = $sect_name;
+				$sect =~ s/'/\\'/g; # escape any single quotes
 
-				$section_tree->{num_files} = $numFiles;
+				my $cmd = qq/SELECT COUNT(*) from $tables{dbsection} AS sect 
+					JOIN $tables{dbchapter} AS ch ON sect.DBchapter_id = ch.DBchapter_id
+					JOIN $tables{dbsubject} AS subj ON subj.DBsubject_id = ch.DBsubject_id
+					JOIN $tables{pgfile} AS pg ON sect.DBsection_id = pg.DBsection_id 
+					where subj.name = '$subj' AND ch.name='$ch' AND sect.name='$sect';/;
 
+				$section_tree->{num_files} = $dbh->selectrow_array($cmd); 
 				my $clone = { %{ $section_tree } };  # need to clone it before pushing into the @subfield array.
 
-			    push(@subfields,$clone);
-		    }
+				push(@subfields,$clone);
+			}
 
-			my $chapter_tree;
-			$chapter_tree->{name} = $ch_name;
-			$chapter_tree->{subfields} = \@subfields;
+			my $chapter_tree = {name => $ch_name, subfields => \@subfields};
 
 			## determine the number of files in each chapter
 
-			my $whereClause ="WHERE subj.name='$subj' AND ch.name='$ch'";
+			my $cmd = qq/select COUNT(*) from $tables{dbsection} AS sect 
+				JOIN $tables{dbchapter} AS ch ON sect.DBchapter_id = ch.DBchapter_id
+				JOIN $tables{dbsubject} AS subj ON subj.DBsubject_id = ch.DBsubject_id
+				JOIN $tables{pgfile} AS pg ON sect.DBsection_id = pg.DBsection_id 
+				JOIN $tables{path} AS path ON pg.path_id = path.path_id 
+				where ch.name = '$ch' AND subj.name = '$subj_name';/;
 
-
-			my $sth = $dbh->prepare($selectClause.$whereClause);
-			$sth->execute;
-			my $numFiles = scalar @{$sth->fetchall_arrayref()};
-			# my $allFiles = $sth->fetchall_arrayref;
-			 $chapter_tree->{num_files} = $numFiles;
+			$chapter_tree->{num_files} = $dbh->selectrow_array($cmd);
 
 			my $clone = { %{ $chapter_tree } };  # need to clone it before pushing into the @chapter_tree array.
 			push(@chapter_tree,$clone);
-
-
-
 		}
 
-		my $subject_tree;
-		$subject_tree->{name} = $subj_name;
-		$subject_tree->{subfields} = \@chapter_tree;
+		my $subject_tree = {name => $subj_name, subfields => \@chapter_tree};
 
 		## find the number of files on the subject level
 
-		my $whereClause ="WHERE subj.name='$subj'";
-
-
-		my $sth = $dbh->prepare($selectClause.$whereClause);
-		$sth->execute;
-		my $numFiles = scalar @{$sth->fetchall_arrayref()};
-		$subject_tree->{num_files} = $numFiles;
-
-		$i++;
-
-		print sprintf("%3d", $i);
+		$cmd = qq/select COUNT(*) from $tables{dbsection} AS sect 
+			JOIN $tables{dbchapter} AS ch ON sect.DBchapter_id = ch.DBchapter_id
+			JOIN $tables{dbsubject} AS subj ON subj.DBsubject_id = ch.DBsubject_id
+			JOIN $tables{pgfile} AS pg ON sect.DBsection_id = pg.DBsection_id 
+			JOIN $tables{path} AS path ON pg.path_id = path.path_id 
+			where subj.name = '$subj_name';/;
 
-		if ($i%10 == 0) {
-		    print "\n";
-		}
+		$subject_tree->{num_files} = $dbh->selectrow_array($cmd);
 
 		my $clone = { % {$subject_tree}};
 		push (@subject_tree, $clone);
@@ -293,7 +268,7 @@ sub build_library_subject_tree {
 
 sub build_library_textbook_tree {
 
-	my ($ce,$dbh) = @_;
+	my ($ce,$dbh,$verbose) = @_;
 
 	my $libraryRoot = $ce->{problemLibrary}->{root};
 	$libraryRoot =~ s|/+$||;
@@ -301,14 +276,13 @@ sub build_library_textbook_tree {
 
 	my %tables = ($libraryVersion eq '2.5')? %OPLtables : %NPLtables;
 
-
-	my $selectClause = "SELECT pg.pgfile_id from `$tables{path}` as path "
-		."LEFT JOIN `$tables{pgfile}` AS pg ON pg.path_id=path.path_id "
-		."LEFT JOIN `$tables{pgfile_problem}` AS pgprob ON pgprob.pgfile_id=pg.pgfile_id "
-		."LEFT JOIN `$tables{problem}` AS prob ON prob.problem_id=pgprob.problem_id "
-		."LEFT JOIN `$tables{section}` AS sect ON sect.section_id=prob.section_id "
-		."LEFT JOIN `$tables{chapter}` AS ch ON ch.chapter_id=sect.chapter_id "
-		."LEFT JOIN `$tables{textbook}` AS text ON text.textbook_id=ch.textbook_id ";
+	my $selectClause = "SELECT pg.pgfile_id from $tables{path} as path "
+		."LEFT JOIN $tables{pgfile} AS pg ON pg.path_id=path.path_id "
+		."LEFT JOIN $tables{pgfile_problem} AS pgprob ON pgprob.pgfile_id=pg.pgfile_id "
+		."LEFT JOIN $tables{problem} AS prob ON prob.problem_id=pgprob.problem_id "
+		."LEFT JOIN $tables{section} AS sect ON sect.section_id=prob.section_id "
+		."LEFT JOIN $tables{chapter} AS ch ON ch.chapter_id=sect.chapter_id "
+		."LEFT JOIN $tables{textbook} AS text ON text.textbook_id=ch.textbook_id ";
 
 	my $results = $dbh->selectall_arrayref("select * from `$tables{textbook}` ORDER BY title;");
 
@@ -319,13 +293,13 @@ sub build_library_textbook_tree {
 
 	my $i =0; ## index to alert user the length of the build
 
-	print "Building the Textbook Library Tree\n";
-	print "There are ". $#textbooks ." textbooks to process.\n";
+	print "Building the Textbook Library Tree\n" if $verbose;
+	print "There are ". $#textbooks ." textbooks to process.\n" if $verbose;
 
 	for my $textbook (@textbooks){
 		$i++;
-		printf("%4d",$i);
-		print("\n") if ($i %10==0);
+		printf("%4d",$i) if $verbose;
+		print("\n") if ($i % 10==0 && $verbose); 
 
 		my $results = $dbh->selectall_arrayref("select ch.chapter_id,ch.name,ch.number "
 			. " from `$tables{chapter}` AS ch JOIN `$tables{textbook}` AS text ON ch.textbook_id=text.textbook_id "
@@ -356,10 +330,9 @@ sub build_library_textbook_tree {
 				my $sth = $dbh->prepare($selectClause.$whereClause);
 				$sth->execute;
 				$section->{num_probs}=scalar @{$sth->fetchall_arrayref()};
-
 			}
 			my $whereClause ="WHERE ch.chapter_id='". $chapter->{chapter_id}."' AND "
-	   				."text.textbook_id='".$textbook->{textbook_id}."'";
+				."text.textbook_id='".$textbook->{textbook_id}."'";
 
 			my $sth = $dbh->prepare($selectClause.$whereClause);
 			$sth->execute;
@@ -420,5 +393,4 @@ sub build_library_textbook_tree {
 
 }
 
-
 1;
diff --git a/bin/updateOPLextras b/bin/updateOPLextras.pl
old mode 100644
new mode 100755
similarity index 75%
rename from bin/updateOPLextras
rename to bin/updateOPLextras.pl
index abfc163948..ed9f9d67c8
--- a/bin/updateOPLextras
+++ b/bin/updateOPLextras.pl
@@ -10,27 +10,33 @@
 
 use strict;
 use warnings;
-use feature 'say';
 use Moo;
 use MooX::Options;
-use Data::Dump qw/dd/;
 use DBI;
 
-
+option verbose => (
+  is => 'ro',
+  short => 'v',
+  doc => 'turn on verbosity'
+);
 option textbooks  => (
   is => 'ro',
+  short => 't',
   doc => 'run the script to update the OPL textbooks and write to a JSON file'
 );
 option directories => (
   is => 'ro',
+  short => 'd',
   doc => 'run the script to update the OPL directories and write to a JSON file'
 );
 option subjects => (
   is => 'ro',
+  short => 's',
   doc => 'run the script to update the OPL subjects and write to a JSON file'
 );
 option all => (
   is => 'ro',
+  short => 'a',
   doc => 'run all the scripts'
 );
 
@@ -59,20 +65,9 @@ sub run {
           },
   );
 
-  if (!$self->all && $self->textbooks){
-    build_library_textbook_tree($ce,$dbh);
-  }
-  if (!$self->all && $self->directories){
-    build_library_directory_tree($ce,$dbh);
-  }
-  if (!$self->all && $self->subjects){
-    build_library_subject_tree($ce,$dbh);
-  }
-  if ($self->all){
-    build_library_textbook_tree($ce,$dbh);
-    build_library_directory_tree($ce,$dbh);
-    build_library_subject_tree($ce,$dbh);
-  }
+  build_library_textbook_tree($ce,$dbh,$self->verbose) if ($self->all || $self->textbooks);
+  build_library_directory_tree($ce,$self->verbose) if ($self->all || $self->directories);
+  build_library_subject_tree($ce,$dbh,$self->verbose) if ($self->all || $self->subjects);
 }
 
 main->new_with_options->run;

From b2e84a44a4728ae7dfc36345454c52a4254f182d Mon Sep 17 00:00:00 2001
From: Peter Staab <peter.staab@gmail.com>
Date: Sun, 7 Mar 2021 08:49:39 -0500
Subject: [PATCH 3/8] remove functionality from OPL-update

---
 bin/OPL-update | 81 ++------------------------------------------------
 1 file changed, 3 insertions(+), 78 deletions(-)

diff --git a/bin/OPL-update b/bin/OPL-update
index d9da002e66..90d466c668 100755
--- a/bin/OPL-update
+++ b/bin/OPL-update
@@ -901,84 +901,9 @@ for my $chap (@{$dbchaps}) {
 	}
 }
 
-# Now run the script build-library-tree
-# This is used to create the file library-tree.json which can be used to 
-# load in subject-chapter-section information for the OPL
-
-use strict;
-use warnings;
-use JSON;
-
-
-my $tree;  # the library subject tree will be stored as arrays of objects. 
-
-my $sth = $dbh->prepare("select * from OPL_DBsubject");
-$sth->execute;
-
-my @subjects = ();
-my @subject_names = ();
-while ( my @row = $sth->fetchrow_array ) {
-	push(@subjects,$row[0]);
-	push(@subject_names,$row[1]);
-  }
-
-
-my @subject_tree;  # array to store the individual library tree for each subject 
-
-foreach my $i (0..$#subjects){
-
-	my $subject_row = $subjects[$i];
-	my $subject_name = $subject_names[$i];
-
-	my $sth = $dbh->prepare("select * from OPL_DBchapter where DBsubject_id = $subject_row;");
-	$sth->execute;
-
-	my @chapters = ();
-	my @chapter_names = ();
-	while ( my @row = $sth->fetchrow_array ) {
-		push(@chapters,$row[0]);
-		push(@chapter_names,$row[1]);
-	  }
-
-
-	my @chapter_tree; # array to store the individual library tree for each chapter
-
-	foreach my $j (0..$#chapters) {
-		my $chapter_row = $chapters[$j];
-		my $chapter_name = $chapter_names[$j];
-		my $sth = $dbh->prepare("SELECT * FROM `$tables{dbsection}` WHERE DBchapter_id=$chapter_row");
-		$sth->execute;
-
-		my @subfields = ();
-		while ( my @row = $sth->fetchrow_array ) {
-			my $section_name;
-			$section_name->{name} = $row[1];
-			my $clone = { %{ $section_name } };  # need to clone it before pushing into the @subfields array.
-		    push(@subfields,$clone);
-	    }
-
-		my $chapter_tree;
-		$chapter_tree->{name} = $chapter_name;
-		$chapter_tree->{subfields} = \@subfields;
-
-		my $clone = { %{ $chapter_tree } };  # need to clone it before pushing into the @chapter_tree array.
-		push(@chapter_tree,$clone);
-
-
-
-	}
-
-	my $subject_tree; 
-	$subject_tree->{name} = $subject_name;
-	$subject_tree->{subfields} = \@chapter_tree; 
-
-	my $clone = { % {$subject_tree}};
-	push (@subject_tree, $clone);
-}
-
-build_library_directory_tree($ce);
-build_library_subject_tree($ce,$dbh);
-build_library_textbook_tree($ce,$dbh);
+# Note: this used to build some JSON versions of the textbooks, subjects and directory trees 
+# that could be used in the library browswer. It's functionality is now 
+# in the updateOPLextras.pl script. 
 
 $dbh->disconnect;
 

From 7567f3db40533d0f26a248a48f24dfeb1f1e8612 Mon Sep 17 00:00:00 2001
From: Peter Staab <peter.staab@gmail.com>
Date: Sun, 7 Mar 2021 08:50:25 -0500
Subject: [PATCH 4/8] switch from MooX::Options to GetOpts::Long.

---
 bin/updateOPLextras.pl | 125 ++++++++++++++++++++++++-----------------
 1 file changed, 72 insertions(+), 53 deletions(-)

diff --git a/bin/updateOPLextras.pl b/bin/updateOPLextras.pl
index ed9f9d67c8..73430eb880 100755
--- a/bin/updateOPLextras.pl
+++ b/bin/updateOPLextras.pl
@@ -1,47 +1,72 @@
 #!/usr/bin/perl
 
-#####
-#
-#  This script allows to rerun a few scripts related to the OPL but doesn't require
-#  the entire OPLupdate script to be run.
-#
-####
+=head1 NAME
+ 
+updateOPLextras - re-build library trees
+ 
+=head1 SYNOPSIS
+ 
+updateOPLextras [options]
+ 
+ Options:
+   -t --textbooks        (rebuild textbook tree)
+   -s --subjects         (rebuild subject tree)
+   -d --directories      (rebuild directory tree)
+   -a --all              (rebuild all trees)
+   -h --help             (display this text)
+ 
+=head1 OPTIONS
+ 
+=over 8
+ 
+=item B<-t> I<--textbooks>
+ 
+Rebuild the textbook tree and write to a JSON file.
+
+=item B<-s> I<--subjects>
 
+Rebuild the subject tree and write to a JSON file.
+
+=item B<-d> I<--directories>
+
+Rebuild the directory tree and write to a JSON file.
+  
+=back
+ 
+=head1 DESCRIPTION
+ 
+B<This program> will rebuild the specified library trees
+from the existing library contents in the database.
+ 
+=cut
 
 use strict;
 use warnings;
-use Moo;
-use MooX::Options;
 use DBI;
+use Getopt::Long;
+use Pod::Usage;
+Getopt::Long::Configure ("bundling");
 
-option verbose => (
-  is => 'ro',
-  short => 'v',
-  doc => 'turn on verbosity'
-);
-option textbooks  => (
-  is => 'ro',
-  short => 't',
-  doc => 'run the script to update the OPL textbooks and write to a JSON file'
-);
-option directories => (
-  is => 'ro',
-  short => 'd',
-  doc => 'run the script to update the OPL directories and write to a JSON file'
-);
-option subjects => (
-  is => 'ro',
-  short => 's',
-  doc => 'run the script to update the OPL subjects and write to a JSON file'
-);
-option all => (
-  is => 'ro',
-  short => 'a',
-  doc => 'run all the scripts'
+my ($textbooks, $directories, $subjects, $verbose, $all);
+GetOptions (
+  't|textbooks'   => \$textbooks,
+  'd|directories' => \$directories,
+  's|subjects'    => \$subjects,
+  'a|all'         => \$all,
+	'v|verbose'     => \$verbose
 );
+pod2usage(2) unless ($textbooks || $directories || $subjects || $all);
+
+#####
+#
+#  This script allows to rerun a few scripts related to the OPL but doesn't require
+#  the entire OPLupdate script to be run.
+#
+####
+
 
 BEGIN {
-  die "WEBWORK_ROOT not found in environment.\n" unless exists $ENV{WEBWORK_ROOT};
+	die "WEBWORK_ROOT not found in environment.\n" unless exists $ENV{WEBWORK_ROOT};
 }
 
 use lib "$ENV{WEBWORK_ROOT}/bin";
@@ -49,27 +74,21 @@ BEGIN
 use WeBWorK::CourseEnvironment;
 use OPLUtils qw/build_library_directory_tree build_library_subject_tree build_library_textbook_tree/;
 
-sub run {
-  my ($self) = @_;
-
-  my $ce = new WeBWorK::CourseEnvironment({webwork_dir=>$ENV{WEBWORK_ROOT}});
+my $ce = new WeBWorK::CourseEnvironment({webwork_dir=>$ENV{WEBWORK_ROOT}});
 
-  my $dbh = DBI->connect(
-          $ce->{problemLibrary_db}->{dbsource},
-          $ce->{problemLibrary_db}->{user},
-          $ce->{problemLibrary_db}->{passwd},
-          {
-              PrintError => 0,
-              RaiseError => 1,
-              ($ce->{ENABLE_UTF8MB4})?(mysql_enable_utf8mb4 =>1):(mysql_enable_utf8 => 1),
-          },
-  );
-
-  build_library_textbook_tree($ce,$dbh,$self->verbose) if ($self->all || $self->textbooks);
-  build_library_directory_tree($ce,$self->verbose) if ($self->all || $self->directories);
-  build_library_subject_tree($ce,$dbh,$self->verbose) if ($self->all || $self->subjects);
-}
+my $dbh = DBI->connect(
+				$ce->{problemLibrary_db}->{dbsource},
+				$ce->{problemLibrary_db}->{user},
+				$ce->{problemLibrary_db}->{passwd},
+				{
+						PrintError => 0,
+						RaiseError => 1,
+						($ce->{ENABLE_UTF8MB4})?(mysql_enable_utf8mb4 =>1):(mysql_enable_utf8 => 1),
+				},
+);
 
-main->new_with_options->run;
+build_library_textbook_tree($ce,$dbh,$verbose) if ($all || $textbooks);
+build_library_directory_tree($ce,$verbose) if ($all || $directories);
+build_library_subject_tree($ce,$dbh,$verbose) if ($all || $subjects);
 
 1;

From 39688c63b58804a42a9d8609378c088656ffbe5d Mon Sep 17 00:00:00 2001
From: Peter Staab <peter.staab@gmail.com>
Date: Sun, 7 Mar 2021 08:50:41 -0500
Subject: [PATCH 5/8] Switching the way JSON is handled.

---
 bin/OPLUtils.pm | 54 ++++++++++++-------------------------------------
 1 file changed, 13 insertions(+), 41 deletions(-)

diff --git a/bin/OPLUtils.pm b/bin/OPLUtils.pm
index ccf682eb34..544f715822 100755
--- a/bin/OPLUtils.pm
+++ b/bin/OPLUtils.pm
@@ -85,18 +85,7 @@ sub build_library_directory_tree {
 	my $webwork_htdocs = $ce->{webwork_dir}."/htdocs";
 	my $file = "$webwork_htdocs/DATA/library-directory-tree.json";
 
-	# use a variable for the file handle
-	my $OUTFILE;
-
-	# use the three arguments version of open
-	# and check for errors
-	open $OUTFILE, '>encoding(UTF-8)', $file  or die "Cannot open $file";
-
-	# you can check for errors (e.g., if after opening the disk gets full)
-	print { $OUTFILE } to_json(\@dirArray) or die "Cannot write to $file";
-
-	# check for errors
-	close $OUTFILE or die "Cannot close $file";
+	writeJSONtoFile(\@dirArray,$file);
 
 	print "Wrote Library Directory Tree to $file\n" if $verbose; 
 }
@@ -243,27 +232,12 @@ sub build_library_subject_tree {
 		my $clone = { % {$subject_tree}};
 		push (@subject_tree, $clone);
 	}
-
-	print "\n";
-
 	my $webwork_htdocs = $ce->{webwork_dir}."/htdocs";
 	my $file = "$webwork_htdocs/DATA/library-subject-tree.json";
 
-	# use a variable for the file handle
-	my $OUTFILE;
-
-	# use the three arguments version of open
-	# and check for errors
-	open $OUTFILE, '>encoding(UTF-8)', $file  or die "Cannot open $file";
+	writeJSONtoFile(\@subject_tree,$file);
 
-	# you can check for errors (e.g., if after opening the disk gets full)
-	print { $OUTFILE } to_json(\@subject_tree,{pretty=>1}) or die "Cannot write to $file";
-
-	# check for errors
-	close $OUTFILE or die "Cannot close $file";
-
-
-	print "Wrote Library Subject Tree to $file\n";
+	print "Wrote Library Subject Tree to $file\n" if $verbose;
 }
 
 sub build_library_textbook_tree {
@@ -375,22 +349,20 @@ sub build_library_textbook_tree {
 	my $webwork_htdocs = $ce->{webwork_dir}."/htdocs";
 	my $file = "$webwork_htdocs/DATA/textbook-tree.json";
 
-	# use a variable for the file handle
-	my $OUTFILE;
+	writeJSONtoFile(\@output,$file);
 
-	# use the three arguments version of open
-	# and check for errors
-	open $OUTFILE, '>', $file  or die "Cannot open $file";
-
-	# you can check for errors (e.g., if after opening the disk gets full)
-	print { $OUTFILE } to_json(\@output,{pretty=>1}) or die "Cannot write to $file";
-
-	# check for errors
-	close $OUTFILE or die "Cannot close $file";
+	print "\n\nWrote Library Textbook Tree to $file\n" if $verbose; 
 
+}
 
-	print "\n\nWrote Library Textbook Tree to $file\n";
+# this takes a hash created in the other subroutines and write the result to a file
+sub writeJSONtoFile {
+	my ($data,$filename) = @_;
 
+	my $json = JSON->new->utf8->pretty(0)->encode($data);	
+	open my $fh, ">", $filename or die "Cannot open $filename";
+	print $fh $json; 
+	close $fh;
 }
 
 1;

From 8ca874021f9650de00720f135c78af7f95bc1719 Mon Sep 17 00:00:00 2001
From: Peter Staab <peter.staab@gmail.com>
Date: Sun, 7 Mar 2021 08:55:34 -0500
Subject: [PATCH 6/8] reinstated JSON package.

---
 bin/OPL-update | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/bin/OPL-update b/bin/OPL-update
index 90d466c668..5d5908dd6e 100755
--- a/bin/OPL-update
+++ b/bin/OPL-update
@@ -527,6 +527,8 @@ if($canopenfile) {
 }
 #### End of taxonomy/taxonomy2
 
+use JSON; 
+
 #### Save the official taxonomy in json format
 my $webwork_htdocs = $ce->{webwork_dir}."/htdocs";
 my $file = "$webwork_htdocs/DATA/tagging-taxonomy.json";

From a9c6b083820526ff25d91d44d10cdf4977778b50 Mon Sep 17 00:00:00 2001
From: Peter Staab <peter.staab@gmail.com>
Date: Tue, 9 Mar 2021 06:40:33 -0500
Subject: [PATCH 7/8] Cleanup of a few files.

---
 Dockerfile           |  1 -
 bin/OPL-update       |  8 +++-----
 bin/OPLUtils.pm      | 11 +++--------
 bin/check_modules.pl |  4 +---
 4 files changed, 7 insertions(+), 17 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index e6fb0506a7..d95e330b6b 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -197,7 +197,6 @@ RUN apt-get update \
 	libjson-xs-perl \
 	libjson-maybexs-perl \
 	libcpanel-json-xs-perl \
-	libmoox-options-perl \
 	make \
 	netpbm \
 	preview-latex-style \
diff --git a/bin/OPL-update b/bin/OPL-update
index 5d5908dd6e..ff2a41cf67 100755
--- a/bin/OPL-update
+++ b/bin/OPL-update
@@ -82,7 +82,7 @@ use lib "$ENV{WEBWORK_ROOT}/lib";
 use lib "$ENV{WEBWORK_ROOT}/bin";
 use WeBWorK::CourseEnvironment;
 use WeBWorK::Utils::Tags;
-use OPLUtils qw/build_library_directory_tree build_library_subject_tree build_library_textbook_tree/;
+use OPLUtils qw/build_library_directory_tree build_library_subject_tree build_library_textbook_tree writeJSONtoFile/;
 
 my $ce = new WeBWorK::CourseEnvironment({webwork_dir=>$ENV{WEBWORK_ROOT}});
 
@@ -532,10 +532,8 @@ use JSON;
 #### Save the official taxonomy in json format
 my $webwork_htdocs = $ce->{webwork_dir}."/htdocs";
 my $file = "$webwork_htdocs/DATA/tagging-taxonomy.json";
-open(OUTF, ">$file") or die "Cannot open $file";
-binmode(OUTF,':encoding(UTF-8)'); 
-print OUTF to_json($tagtaxo,{pretty=>1}) or die "Cannot write to $file";
-close(OUTF);
+
+writeJSONtoFile($tagtaxo,$file);
 print "Saved taxonomy to $file.\n";
 
 #### Now deal with cross-listed sections
diff --git a/bin/OPLUtils.pm b/bin/OPLUtils.pm
index 544f715822..319d39056d 100755
--- a/bin/OPLUtils.pm
+++ b/bin/OPLUtils.pm
@@ -8,7 +8,7 @@ use base qw(Exporter);
 #  The following files are created:
 #		1. $webwork_htdocs/DATA/library-directory-tree.json  (the directory structure of the library)
 #		2. $webwork_htdocs/DATA/library-subject-tree.json  (the subject/chapter/section struture of the library)
-#		3. $webwork_htdocs/DATA/library-textbook-tree.json  (the subject/chapter/section struture of the library)
+#		3. $webwork_htdocs/DATA/textbook-tree.json  (the subject/chapter/section struture of the library)
 
 # the above JSON files can be used to load and more quickly lookup OPL information
 # 
@@ -18,13 +18,10 @@ use warnings;
 use File::Find::Rule;
 use File::Basename;
 use open qw/:std :utf8/;
-# use Cwd;
-# use DBI;
 use JSON;
-use Data::Dump qw/dd/;
 
 our @EXPORT    = ();
-our @EXPORT_OK = qw(build_library_directory_tree build_library_subject_tree build_library_textbook_tree);
+our @EXPORT_OK = qw(build_library_directory_tree build_library_subject_tree build_library_textbook_tree writeJSONtoFile);
 
 ### Data for creating the database tables
 
@@ -75,8 +72,6 @@ sub build_library_directory_tree {
 	my $libraryRoot = $ce->{problemLibrary}->{root};
 	$libraryRoot =~ s|/+$||;
 
-	my $libraryVersion = $ce->{problemLibrary}->{version};
-
 	my($filename, $directories) = fileparse($libraryRoot);
 
 	my @dirArray = ();
@@ -359,7 +354,7 @@ sub build_library_textbook_tree {
 sub writeJSONtoFile {
 	my ($data,$filename) = @_;
 
-	my $json = JSON->new->utf8->pretty(0)->encode($data);	
+	my $json = JSON->new->utf8->encode($data);	
 	open my $fh, ">", $filename or die "Cannot open $filename";
 	print $fh $json; 
 	close $fh;
diff --git a/bin/check_modules.pl b/bin/check_modules.pl
index 140752b39b..f57cd6516f 100755
--- a/bin/check_modules.pl
+++ b/bin/check_modules.pl
@@ -10,7 +10,7 @@
 	mv
 	mysql
 	tar
-        git
+	git
 	gzip
 	latex
 	pdflatex
@@ -87,8 +87,6 @@
 	Locale::Maketext::Simple
 	LWP::Protocol::https
 	MIME::Base64
-	Moo
-	MooX::Options
 	Net::IP
 	Net::LDAPS
 	Net::OAuth

From ebc7c17f52d39bcde3398bb7a9d707737fcab709 Mon Sep 17 00:00:00 2001
From: Peter Staab <peter.staab@gmail.com>
Date: Tue, 9 Mar 2021 06:43:47 -0500
Subject: [PATCH 8/8] deleted one more unnecessary line.

---
 bin/OPLUtils.pm | 2 --
 1 file changed, 2 deletions(-)

diff --git a/bin/OPLUtils.pm b/bin/OPLUtils.pm
index 319d39056d..dcf6c322ab 100755
--- a/bin/OPLUtils.pm
+++ b/bin/OPLUtils.pm
@@ -72,8 +72,6 @@ sub build_library_directory_tree {
 	my $libraryRoot = $ce->{problemLibrary}->{root};
 	$libraryRoot =~ s|/+$||;
 
-	my($filename, $directories) = fileparse($libraryRoot);
-
 	my @dirArray = ();
 	push(@dirArray,buildTree($libraryRoot));