From c25ffe0fcfc41da91a5e9c421262fd8495ab6d7c Mon Sep 17 00:00:00 2001 From: pbailie Date: Tue, 16 Oct 2018 19:45:05 -0400 Subject: [PATCH 1/4] Changes to be committed: new file: sample_bin/registration_section_resync.php WIP: Registration re-sync tool for issue submitty/submitty#2960: "Registration Section data not always synced between master DB and course DB" --- sample_bin/registration_section_resync.php | 166 +++++++++++++++++++++ 1 file changed, 166 insertions(+) create mode 100755 sample_bin/registration_section_resync.php diff --git a/sample_bin/registration_section_resync.php b/sample_bin/registration_section_resync.php new file mode 100755 index 0000000..07b0bce --- /dev/null +++ b/sample_bin/registration_section_resync.php @@ -0,0 +1,166 @@ +#!/usr/bin/env php +init(); + + $this->process(); + + } + + public function __destruct() { + if (pg_connection_status(self::$db_master_conn) !== pgsql_connection_ok) { + pg_close(self::$db_master_conn); + } + + if (pg_connection_status(self::$db_course_conn) !== PGSQL_CONNECTION_OK) { + pg_close(self::$db_course_conn); + } + } + + private function init() { + //This script should be run as root. + if (posix_getuid() !== 0) { + exit(sprintf("This script must be run as root.%s", PHP_EOL)); + } + + //This script cannot be run as a webpage. + if (PHP_SAPI !== 'cli') { + exit(sprintf("This script must be run from the command line.%s", PHP_EOL)); + } + + //Get all academic terms to be processed (as cli arguments). + if ($_SERVER['argc'] < 2) { + exit(sprintf("Please specify at least one academic term code as a command line argument.%s", PHP_EOL)); + } + + self::$academic_terms = array_splice($_SERVER['argv'], 1); + + //Get DB connection config from Submitty + $json_data = file_get_contents(DB_CONFIG_PATH); + if ($json_data === false) { + exit(sprintf("Could not read database configuration at %s%s", DB_CONFIG_PATH, PHP_EOL)); + } + + $json_data = json_decode($json_data, true); + self::$db_host = $json_data['database_host']; + self::$db_user = $json_data['database_user']; + self::$db_password = $json_data['database_password']; + + //Connect to master DB. + self::$db_master_conn = pg_connect(sprintf("dbname=submitty host=%s user=%s password=%s sslmode=prefer", self::$db_host, self::$db_user, self::$db_password)); + if (pg_connection_status(self::$db_master_conn) !== PGSQL_CONNECTION_OK) { + exit(sprintf("ERROR: Could not establish connection to Submitty Master DB%sCheck configuration at %s%s", PHP_EOL, DB_CONFIG_PATH, PHP_EOL)); + } + } + + private function process() { + + //Loop through academic terms + foreach (self::$academic_terms as $term) { + //Get courses list for $term + $res = pg_query_params(self::$db_master_conn, "SELECT course FROM courses WHERE semester = $1", array($term)); + if ($res === false) { + exit(sprintf("Error reading course list for %s.%s", $term, PHP_EOL)); + } + + $courses = pg_fetch_all_columns($res, 0); + if (count($courses) < 1) { + fprintf(STDERR, "No courses registered for term code '%s'.%s", $term, PHP_EOL); + continue; + } + + //Now loop through all courses + foreach($courses as $course) { + //We need to compare registration sections in both master and course DBs + //Connect to course DB + $db_name = sprintf("submitty_%s_%s", $term, $course); + self::$db_course_conn = pg_connect(sprintf("dbname=%s host=%s user=%s password=%s sslmode=prefer", $db_name, self::$db_host, self::$db_user, self::$db_password)); + if (pg_connection_status(self::$db_course_conn) !== PGSQL_CONNECTION_OK) { + fprintf(STDERR, "ERROR: Could not establish connection to Submitty Course DB %s.%sSkipping %s %s.%s", $db_name, PHP_EOL, $term, $course, PHP_EOL); + continue; + } + + //First retrieve registration sections in master DB + $res = pg_query_params(self::$db_master_conn, "SELECT registration_section_id FROM courses_registration_sections WHERE semester=$1 AND course=$2", array($term, $coourse)); + if ($res === false) { + fprintf(STDERR, "Error reading registration sections from master DB: %s %s.%sSkipping %s %s.%s", $term, $course, PHP_EOL, $term, $course, PHP_EOL); + continue; + } + $master_registration_sections = pg_fetch_all_columns($res, 0); + + //Next retrieve registration sections in course DB + $res = pg_query(self::$db_course_conn, "SELECT sections_registration_id FROM sections_registration"); + if ($res === false) { + fprintf("Error reading registration sections from course DB: %s.%sSkipping %s %s.%s", $dbname, PHP_EOL, $term, $course, PHP_EOL); + continue; + } + $course_registration_sections = pg_fetch_all_columns($res, 0); + + //Get the differences of both lists (a registration section in either list, but not the other). + $sync_list = array_diff($course_registration_sections, $master_registration_section); + $sync_list = array_merge($sync_list, array_diff($master_registration_section, $course_registration_sections)); + + //INSERT $sync_list to master DB, ON CONFLICT DO NOTHING (prevents potential PK violations). We're using DB schema trigger to complete resync. + foreach($sync_list as $section) { + $res = pg_query_params($self::$db_master_conn, "INSERT INTO courses_registration_sections (semester, course, registration_section_id) VALUES ($1, $2, $3) ON CONFLICT DO NOTHING", array($term, $course, $section)); + if ($res === false) { + fprintf("Error during re-sync procedure: %s %s section %s.%s.This section could not be synced.%s", $term, $course, $section, PHP_EOL, PHP_EOL); + continue; + } + } + } + } + } + +} + + + + + + + + + + + + + + + + + + From 3d75debb54e99065b29bab7f0aa12c458cf57dce Mon Sep 17 00:00:00 2001 From: pbailie Date: Tue, 16 Oct 2018 20:09:53 -0400 Subject: [PATCH 2/4] Changes to be committed: modified: sample_bin/registration_section_resync.php WIP --- sample_bin/registration_section_resync.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sample_bin/registration_section_resync.php b/sample_bin/registration_section_resync.php index 07b0bce..d2f3292 100755 --- a/sample_bin/registration_section_resync.php +++ b/sample_bin/registration_section_resync.php @@ -124,7 +124,7 @@ private function process() { //Next retrieve registration sections in course DB $res = pg_query(self::$db_course_conn, "SELECT sections_registration_id FROM sections_registration"); if ($res === false) { - fprintf("Error reading registration sections from course DB: %s.%sSkipping %s %s.%s", $dbname, PHP_EOL, $term, $course, PHP_EOL); + fprintf(STDERR, "Error reading registration sections from course DB: %s.%sSkipping %s %s.%s", $dbname, PHP_EOL, $term, $course, PHP_EOL); continue; } $course_registration_sections = pg_fetch_all_columns($res, 0); @@ -137,7 +137,7 @@ private function process() { foreach($sync_list as $section) { $res = pg_query_params($self::$db_master_conn, "INSERT INTO courses_registration_sections (semester, course, registration_section_id) VALUES ($1, $2, $3) ON CONFLICT DO NOTHING", array($term, $course, $section)); if ($res === false) { - fprintf("Error during re-sync procedure: %s %s section %s.%s.This section could not be synced.%s", $term, $course, $section, PHP_EOL, PHP_EOL); + fprintf(STDERR, "Error during re-sync procedure: %s %s section %s.%s.This section could not be synced.%s", $term, $course, $section, PHP_EOL, PHP_EOL); continue; } } From fb60959789183dad22991174d98e0e0315509abc Mon Sep 17 00:00:00 2001 From: pbailie Date: Thu, 18 Oct 2018 18:41:20 -0400 Subject: [PATCH 3/4] Changes to be committed: modified: sample_bin/registration_section_resync.php WIP --- sample_bin/registration_section_resync.php | 32 ++++++---------------- 1 file changed, 9 insertions(+), 23 deletions(-) diff --git a/sample_bin/registration_section_resync.php b/sample_bin/registration_section_resync.php index d2f3292..1d6d3e0 100755 --- a/sample_bin/registration_section_resync.php +++ b/sample_bin/registration_section_resync.php @@ -22,6 +22,7 @@ //Run script new registration_section_resync(); +printf("Re-sync process finished.%s", PHP_EOL); exit(0); class registration_section_resync { @@ -114,7 +115,7 @@ private function process() { } //First retrieve registration sections in master DB - $res = pg_query_params(self::$db_master_conn, "SELECT registration_section_id FROM courses_registration_sections WHERE semester=$1 AND course=$2", array($term, $coourse)); + $res = pg_query_params(self::$db_master_conn, "SELECT registration_section_id FROM courses_registration_sections WHERE semester=$1 AND course=$2", array($term, $course)); if ($res === false) { fprintf(STDERR, "Error reading registration sections from master DB: %s %s.%sSkipping %s %s.%s", $term, $course, PHP_EOL, $term, $course, PHP_EOL); continue; @@ -130,37 +131,22 @@ private function process() { $course_registration_sections = pg_fetch_all_columns($res, 0); //Get the differences of both lists (a registration section in either list, but not the other). - $sync_list = array_diff($course_registration_sections, $master_registration_section); - $sync_list = array_merge($sync_list, array_diff($master_registration_section, $course_registration_sections)); + $sync_list = array_diff($course_registration_sections, $master_registration_sections); + $sync_list = array_merge($sync_list, array_diff($master_registration_sections, $course_registration_sections)); + + var_dump($sync_list); die; //INSERT $sync_list to master DB, ON CONFLICT DO NOTHING (prevents potential PK violations). We're using DB schema trigger to complete resync. foreach($sync_list as $section) { - $res = pg_query_params($self::$db_master_conn, "INSERT INTO courses_registration_sections (semester, course, registration_section_id) VALUES ($1, $2, $3) ON CONFLICT DO NOTHING", array($term, $course, $section)); + $res = pg_query_params($self::$db_master_conn, "INSERT INTO courses_registration_sections (semester, course, registration_section_id) VALUES ($1, $2, $3) ON CONFLICT ON CONSTRAINT courses_registration_sections_pkey DO UPDATE SET semester=$1, course=$2, registration_section_id=$3", array($term, $course, $section)); if ($res === false) { fprintf(STDERR, "Error during re-sync procedure: %s %s section %s.%s.This section could not be synced.%s", $term, $course, $section, PHP_EOL, PHP_EOL); continue; } + + printf("Sync complete for %s %s%s", $term, $course, PHP_EOL); } } } } - } - - - - - - - - - - - - - - - - - - From 13fca3e10d54b066f499e9ffaa6499a2c587305d Mon Sep 17 00:00:00 2001 From: pbailie Date: Fri, 19 Oct 2018 20:03:41 -0400 Subject: [PATCH 4/4] Changes to be committed: modified: sample_bin/registration_section_resync.php Ready for PR --- sample_bin/registration_section_resync.php | 65 ++++++++++++---------- 1 file changed, 37 insertions(+), 28 deletions(-) diff --git a/sample_bin/registration_section_resync.php b/sample_bin/registration_section_resync.php index 1d6d3e0..92c3db1 100755 --- a/sample_bin/registration_section_resync.php +++ b/sample_bin/registration_section_resync.php @@ -22,13 +22,11 @@ //Run script new registration_section_resync(); -printf("Re-sync process finished.%s", PHP_EOL); exit(0); class registration_section_resync { - private static $db_master_conn; - private static $db_course_conn; + private static $db_conn = array('master' => null, 'course' => null); private static $db_host; private static $db_user; private static $db_password; @@ -36,21 +34,16 @@ class registration_section_resync { public function __construct() { $this->init(); - $this->process(); - + printf("Re-sync process finished.%s", PHP_EOL); } public function __destruct() { - if (pg_connection_status(self::$db_master_conn) !== pgsql_connection_ok) { - pg_close(self::$db_master_conn); - } - - if (pg_connection_status(self::$db_course_conn) !== PGSQL_CONNECTION_OK) { - pg_close(self::$db_course_conn); - } + $this->close_db_conn('course'); + $this->close_db_conn('master'); } + /** Setup properties and open master DB connection */ private function init() { //This script should be run as root. if (posix_getuid() !== 0) { @@ -66,7 +59,6 @@ private function init() { if ($_SERVER['argc'] < 2) { exit(sprintf("Please specify at least one academic term code as a command line argument.%s", PHP_EOL)); } - self::$academic_terms = array_splice($_SERVER['argv'], 1); //Get DB connection config from Submitty @@ -81,25 +73,25 @@ private function init() { self::$db_password = $json_data['database_password']; //Connect to master DB. - self::$db_master_conn = pg_connect(sprintf("dbname=submitty host=%s user=%s password=%s sslmode=prefer", self::$db_host, self::$db_user, self::$db_password)); - if (pg_connection_status(self::$db_master_conn) !== PGSQL_CONNECTION_OK) { + self::$db_conn['master'] = pg_connect(sprintf("dbname=submitty host=%s user=%s password=%s sslmode=prefer", self::$db_host, self::$db_user, self::$db_password)); + if (pg_connection_status(self::$db_conn['master']) !== PGSQL_CONNECTION_OK) { exit(sprintf("ERROR: Could not establish connection to Submitty Master DB%sCheck configuration at %s%s", PHP_EOL, DB_CONFIG_PATH, PHP_EOL)); } } + /** Loop through academic terms and then loop through each course to sync registration sections */ private function process() { - //Loop through academic terms foreach (self::$academic_terms as $term) { //Get courses list for $term - $res = pg_query_params(self::$db_master_conn, "SELECT course FROM courses WHERE semester = $1", array($term)); + $res = pg_query_params(self::$db_conn['master'], "SELECT course FROM courses WHERE semester = $1", array($term)); if ($res === false) { exit(sprintf("Error reading course list for %s.%s", $term, PHP_EOL)); } $courses = pg_fetch_all_columns($res, 0); - if (count($courses) < 1) { - fprintf(STDERR, "No courses registered for term code '%s'.%s", $term, PHP_EOL); + if (empty($courses)) { + fprintf(STDERR, "No courses registered for %s.%s", $term, PHP_EOL); continue; } @@ -108,24 +100,26 @@ private function process() { //We need to compare registration sections in both master and course DBs //Connect to course DB $db_name = sprintf("submitty_%s_%s", $term, $course); - self::$db_course_conn = pg_connect(sprintf("dbname=%s host=%s user=%s password=%s sslmode=prefer", $db_name, self::$db_host, self::$db_user, self::$db_password)); - if (pg_connection_status(self::$db_course_conn) !== PGSQL_CONNECTION_OK) { + self::$db_conn['course'] = pg_connect(sprintf("dbname=%s host=%s user=%s password=%s sslmode=prefer", $db_name, self::$db_host, self::$db_user, self::$db_password)); + if (pg_connection_status(self::$db_conn['course']) !== PGSQL_CONNECTION_OK) { fprintf(STDERR, "ERROR: Could not establish connection to Submitty Course DB %s.%sSkipping %s %s.%s", $db_name, PHP_EOL, $term, $course, PHP_EOL); continue; } //First retrieve registration sections in master DB - $res = pg_query_params(self::$db_master_conn, "SELECT registration_section_id FROM courses_registration_sections WHERE semester=$1 AND course=$2", array($term, $course)); + $res = pg_query_params(self::$db_conn['master'], "SELECT registration_section_id FROM courses_registration_sections WHERE semester=$1 AND course=$2", array($term, $course)); if ($res === false) { fprintf(STDERR, "Error reading registration sections from master DB: %s %s.%sSkipping %s %s.%s", $term, $course, PHP_EOL, $term, $course, PHP_EOL); + $this->close_db_conn('course'); continue; } $master_registration_sections = pg_fetch_all_columns($res, 0); //Next retrieve registration sections in course DB - $res = pg_query(self::$db_course_conn, "SELECT sections_registration_id FROM sections_registration"); + $res = pg_query(self::$db_conn['course'], "SELECT sections_registration_id FROM sections_registration"); if ($res === false) { fprintf(STDERR, "Error reading registration sections from course DB: %s.%sSkipping %s %s.%s", $dbname, PHP_EOL, $term, $course, PHP_EOL); + $this->close_db_conn('course'); continue; } $course_registration_sections = pg_fetch_all_columns($res, 0); @@ -133,20 +127,35 @@ private function process() { //Get the differences of both lists (a registration section in either list, but not the other). $sync_list = array_diff($course_registration_sections, $master_registration_sections); $sync_list = array_merge($sync_list, array_diff($master_registration_sections, $course_registration_sections)); - - var_dump($sync_list); die; + if (empty($sync_list)) { + printf("No sync required for %s %s.%s", $term, $course, PHP_EOL); + $this->close_db_conn('course'); + continue; + } //INSERT $sync_list to master DB, ON CONFLICT DO NOTHING (prevents potential PK violations). We're using DB schema trigger to complete resync. foreach($sync_list as $section) { - $res = pg_query_params($self::$db_master_conn, "INSERT INTO courses_registration_sections (semester, course, registration_section_id) VALUES ($1, $2, $3) ON CONFLICT ON CONSTRAINT courses_registration_sections_pkey DO UPDATE SET semester=$1, course=$2, registration_section_id=$3", array($term, $course, $section)); + $res = pg_query_params(self::$db_conn['master'], "INSERT INTO courses_registration_sections (semester, course, registration_section_id) VALUES ($1, $2, $3) ON CONFLICT ON CONSTRAINT courses_registration_sections_pkey DO UPDATE SET semester=$1, course=$2, registration_section_id=$3", array($term, $course, $section)); if ($res === false) { fprintf(STDERR, "Error during re-sync procedure: %s %s section %s.%s.This section could not be synced.%s", $term, $course, $section, PHP_EOL, PHP_EOL); + $this->close_db_conn('course'); continue; } - - printf("Sync complete for %s %s%s", $term, $course, PHP_EOL); } + printf("Sync complete for %s %s.%s", $term, $course, PHP_EOL); + $this->close_db_conn('course'); } } } + + /** + * Close a given DB connection. + * + * @param string DB connection to close. + */ + private function close_db_conn($conn) { + if (pg_connection_status(self::$db_conn[$conn]) === PGSQL_CONNECTION_OK) { + pg_close(self::$db_conn[$conn]); + } + } }