From a89365be444ff18d5eed2fa9851d7d051b41377a Mon Sep 17 00:00:00 2001 From: Maxime Mulder Date: Fri, 6 Jun 2025 10:29:23 +0000 Subject: [PATCH 1/6] add initial redcap records import script fix typo add automatic session creation bugfixes from testing use record for session date simplify redcap client url parameter make RedcapHttpClient->getRecords public fetch record id in redcap records move record import to its own class fix yoda condition add import redcap records script add changes from cbig fix getVisitConfigEvent documentation --- modules/redcap/php/redcapmapper.class.inc | 37 ++++ .../redcap/tools/import_redcap_records.php | 197 ++++++++++++++++++ 2 files changed, 234 insertions(+) create mode 100644 modules/redcap/tools/import_redcap_records.php diff --git a/modules/redcap/php/redcapmapper.class.inc b/modules/redcap/php/redcapmapper.class.inc index 083e6fd909..32eeb8b49a 100644 --- a/modules/redcap/php/redcapmapper.class.inc +++ b/modules/redcap/php/redcapmapper.class.inc @@ -200,6 +200,43 @@ class RedcapMapper return $visit_config ? $visit_config : null; } + /** + * Get the REDCap event associated with the arm and event names of a REDCap + * visit mapping configuration node. + * + * @param RedcapConfigVisit $visit_config A REDCap visit mapping configuration + * node. + * + * @return RedcapEvent The REDCap event associated with the REDCap visit + * mapping configuration node. + */ + public function getVisitConfigEvent( + RedcapConfigVisit $visit_config, + ): RedcapEvent { + // Get the list of all the REDCap arms for this REDCap project. + $redcap_arms = $this->_redcap_client->getArms(); + + // Find the REDCap arm that matches the configuration arm name. + $redcap_arm = array_find( + $redcap_arms, + fn($redcap_arm) => $redcap_arm->name === $visit_config->redcap_arm_name, + ); + + // Get the list of all the REDCap events for this REDCap project. + $redcap_events = $this->_redcap_client->getEvents(); + + // Find the REDCap event that matches the arm and configuration event name. + $redcap_event = array_find( + $redcap_events, + fn($redcap_event) => ( + $redcap_event->arm_number === $redcap_arm->number + && $redcap_event->name === $visit_config->redcap_event_name + ), + ); + + return $redcap_event; + } + /** * Check that a session exists or create it using the REDCap module automatic * session creation configuration. Throw an exception if the session does not diff --git a/modules/redcap/tools/import_redcap_records.php b/modules/redcap/tools/import_redcap_records.php new file mode 100644 index 0000000000..ee4a6566e6 --- /dev/null +++ b/modules/redcap/tools/import_redcap_records.php @@ -0,0 +1,197 @@ +#!/usr/bin/env php +getModule('redcap')->registerAutoloader(); +} catch (\LorisNoSuchModuleException $th) { + error_log("[error] no 'redcap' module found."); + exit(1); +} + +use LORIS\redcap\config\RedcapConfigParser; +use LORIS\redcap\client\RedcapHttpClient; +use LORIS\redcap\RedcapMapper; +use LORIS\redcap\RedcapRecordImporter; + +// Get CLI arguments. + +$args = parseArgs( + getopt( + "", + [ + 'instance-url:', + 'project-id:', + 'simulate', + 'verbose', + ] + ) +); + +// Get CLI arguments. + +$redcap_instance_url = $args['instance-url']; +$redcap_project_id = $args['project-id']; +$simulate = $args['simulate']; +$verbose = $args['verbose']; + +// Display CLI arguments. + +$simulate_message = $simulate ? "enabled" : "disabled"; +$verbose_message = $verbose ? "enabled" : "disabled"; + +fprintf(STDOUT, "[args:instance-url] $redcap_instance_url\n"); +fprintf(STDOUT, "[args:project-id] $redcap_project_id\n"); +fprintf(STDOUT, "[args:simulate] $simulate_message\n"); +fprintf(STDOUT, "[args:verbose] $verbose_message\n"); + +// Get the REDCap module configuration for the relevant REDCap project. + +$redcap_config_parser = new RedcapConfigParser( + $lorisInstance, + $redcap_instance_url, + $redcap_project_id, +); + +$redcap_config = $redcap_config_parser->parse(); + +if ($redcap_config === null) { + fprintf( + STDERR, + "No REDCap configuration found for REDCap instance URL" + . "'{$redcap_instance_url}' and REDCap project ID " + . "{$redcap_project_id}.\n" + ); + + exit(1); +} + +$redcap_client = new RedcapHttpClient( + $redcap_config->redcap_instance_url, + $redcap_config->redcap_api_token, +); + +$redcap_mapper = new RedcapMapper($lorisInstance, $redcap_client, $redcap_config); + +// Fetch the REDCap unique event names that match the REDCap module project +// configuration. + +$unique_event_names = array_map( + fn ($visit_config) => $redcap_mapper + ->getVisitConfigEvent($visit_config) + ->unique_name, + $redcap_config->visits, +); + +$importable_instruments = $config->getSetting('redcap_importable_instrument'); + +// Fetch the REDCap records that match the REDCap importable instruments and unique +// event names. + +$records = $redcap_client->getRecords( + $importable_instruments, + $unique_event_names, + [], +); + +$record_importer = new RedcapRecordImporter( + $lorisInstance, + $redcap_client, + $redcap_config, +); + +$records_imported_count = 0; +$records_ignored_count = 0; +$records_failed_count = 0; + +foreach ($records as $record) { + fprintf( + STDOUT, + "Importing record {$record->unique_event_name} {$record->record_id}...\n", + ); + + // Do not actually import REDCap records into LORIS in simulation mode. + if ($simulate) { + $records_ignored_count += 1; + continue; + } + + try { + if ($record_importer->import($record)) { + $records_imported_count += 1; + fprintf(STDOUT, "Successfully imported record.\n"); + } else { + fprintf(STDOUT, "Skipped record import.\n"); + $records_ignored_count += 1; + } + } catch (\LorisException $exception) { + fprintf(STDOUT, "Failed record import:\n{$exception->getMessage()}.\n"); + $records_failed_count += 1; + } +} + +fprintf( + STDOUT, + "Successful records imports: $records_imported_count\n" + . "Skipped records imports: $records_ignored_count\n" + . "Failed records imports: $records_failed_count\n" +); + +/** + * Check the CLI arguments passed to the script and return them in an associative + * array. + * + * @param array $args The unstructured CLI arguments. + * + * @return array + */ +function parseArgs(array $args): array +{ + // Check the required CLI arguments. + if (!isset($args['instance-url']) || !isset($args['project-id'])) { + error_log("[error] Required arguments: --instance-url --project-id"); + showHelp(); + exit(1); + } + + return [ + 'instance-url' => $args['instance-url'], + 'project-id' => $args['project-id'], + 'simulate' => isset($args['simulate']), + 'verbose' => isset($args['verbose']), + ]; +} + +/** + * Displays help for this script. + * + * @return void + */ +function showHelp() : void +{ + fprintf( + STDERR, + "Usage:\n" + . " php import_redcap_records.php \n" + . " [--instance-url=URL]\n" + . " [--project-id=ID]\n" + . " [--simulate]\n" + . " [--verbose]\n\n" + . "Notes:\n" + . " --instance-url=URL (required) The URL of the REDCap instance from" + . " which to import records from.\n" + . " --project-id=ID (required) The ID of the REDCap project from" + . " which to import records from.\n" + . " --simulate (optional) Fetch records without importing them" + . " into LORIS. (default: false)\n" + . " --verbose (optional) Display verbose information. (default:" + . " false)\n\n" + ); +} From b55a6b318dde47e09730c254afeb41f6ed34d283 Mon Sep 17 00:00:00 2001 From: Maxime Mulder Date: Wed, 22 Oct 2025 09:04:43 +0000 Subject: [PATCH 2/6] refactor getInstrumentRecords --- .../php/client/redcaphttpclient.class.inc | 103 +++++++++++------- .../php/redcapnotificationhandler.class.inc | 2 +- .../redcap/tools/import_redcap_records.php | 17 ++- 3 files changed, 79 insertions(+), 43 deletions(-) diff --git a/modules/redcap/php/client/redcaphttpclient.class.inc b/modules/redcap/php/client/redcaphttpclient.class.inc index f14d43871d..5ab3518730 100644 --- a/modules/redcap/php/client/redcaphttpclient.class.inc +++ b/modules/redcap/php/client/redcaphttpclient.class.inc @@ -647,19 +647,19 @@ class RedcapHttpClient /** * Get all records for an single instrument. * - * @param string $instrument_name A REDCap instrument name. - * @param string $unique_event_name A REDCap unique event name. - * @param string $record_id A REDCap record ID. - * @param bool $completed_records_only Only return completed records. + * @param string $instrument_name A REDCap instrument name. + * @param string $unique_event_name A REDCap unique event name. + * @param ?string $record_id A REDCap record ID. + * @param bool $completed_records_only Only return completed records. * * @throws \LorisException * * @return RedcapRecord[] an array of records */ - public function getInstrumentRecord( - string $instrument_name, - string $unique_event_name, - string $record_id, + public function getInstrumentRecords( + string $instrument_name, + string $unique_event_name, + ?string $record_id, bool $completed_records_only = true ): array { if (empty($instrument_name)) { @@ -674,10 +674,6 @@ class RedcapHttpClient ); } - if (empty($record_id)) { - throw new \LorisException("[redcap] Error: required 'record_id'."); - } - // mapping check $mapping_instrument_event_exists = $this->hasMappingInstrumentEvent( $instrument_name, @@ -692,65 +688,98 @@ class RedcapHttpClient } // request - $records = $this->_getRecords( + $record_dicts = $this->_getRecords( [$instrument_name], [$unique_event_name], - [$record_id] + $record_id !== null ? [$record_id] : [], ); - if (empty($records)) { + if (empty($record_dicts)) { throw new \LorisException("[redcap] Error: no data found."); } // Only keep complete records if ($completed_records_only) { - $completed = array_filter( - $records, + $record_dicts = array_filter( + $record_dicts, fn ($record) => $record["{$instrument_name}_complete"] == 2 ); - if (count($completed) === 0) { + if (empty($record_dicts)) { throw new \LorisException( "[redcap] Error: no complete record found." ); } - } else { - // if not only completed records - $completed = $records; } - // Order the records by ${instrument_name}_dtt field value - usort( - $completed, - function ($a, $b) use ($instrument_name) { - $dttField = "{$instrument_name}_dtt"; - $a_date = new \DateTimeImmutable($a[$dttField]); - $b_date = new \DateTimeImmutable($b[$dttField]); - return $a_date <=> $b_date; - } - ); + $records = []; // is a repeating instrument? - $final = []; if ($this->getProjectInfo()->has_repeating_instruments && $this->hasRepeatingInstrumentEvent( $instrument_name, $unique_event_name ) ) { - foreach ($completed as $index => $record) { - $final[] = new RedcapRepeatingRecord( + // TODO: ${instrument_name}_dtt seems to be HBCD-specific. The code + // could probably be cleaner with a single timestamp abstraction. The + // problem is that the repeating index depends on the order in which the + // records were filled. However, this index might also be obtainable + // from a field in the record. Investigation needed. + // Order the records by ${instrument_name}_dtt field value + usort( + $record_dicts, + function ($a, $b) use ($instrument_name) { + $dttField = "{$instrument_name}_dtt"; + $a_date = new \DateTimeImmutable($a[$dttField]); + $b_date = new \DateTimeImmutable($b[$dttField]); + return $a_date <=> $b_date; + } + ); + + foreach ($record_dicts as $index => $record) { + $records[] = new RedcapRepeatingRecord( $instrument_name, $record, $index + 1 ); } } else { - // return the only record - $final[] = new RedcapRecord($instrument_name, $completed[0]); + // Transform the record dictionaries into record objects. + $records = array_map( + fn ($record_dict) => new RedcapRecord( + $instrument_name, + $record_dict, + ), + $record_dicts, + ); + + // Sort the records by datetime. + usort( + $records, + function ($a, $b) { + // If both records have a datetime, compare them. + if ($a->datetime !== null && $b->datetime !== null) { + return $a->datetime <=> $b->datetime; + } + + // If only record $a has datetime, it comes first. + if ($a->datetime !== null) { + return -1; + } + + // If only record $b has datetime, it comes first. + if ($b->datetime !== null) { + return 1; + } + + // If both records have no datetime, maintain the original order. + return 0; + }, + ); } - return $final; + return $records; } /** diff --git a/modules/redcap/php/redcapnotificationhandler.class.inc b/modules/redcap/php/redcapnotificationhandler.class.inc index 83300f8a8f..3bf84d64ce 100644 --- a/modules/redcap/php/redcapnotificationhandler.class.inc +++ b/modules/redcap/php/redcapnotificationhandler.class.inc @@ -126,7 +126,7 @@ class RedcapNotificationHandler $this->_acquireNotificationLock(); // get data from redcap - $records = $this->_redcap_client->getInstrumentRecord( + $records = $this->_redcap_client->getInstrumentRecords( $this->_redcap_notif->instrument_name, $this->_redcap_notif->unique_event_name, $this->_redcap_notif->record_id, diff --git a/modules/redcap/tools/import_redcap_records.php b/modules/redcap/tools/import_redcap_records.php index ee4a6566e6..69b1e05a35 100644 --- a/modules/redcap/tools/import_redcap_records.php +++ b/modules/redcap/tools/import_redcap_records.php @@ -94,12 +94,19 @@ // Fetch the REDCap records that match the REDCap importable instruments and unique // event names. +$records = []; -$records = $redcap_client->getRecords( - $importable_instruments, - $unique_event_names, - [], -); +foreach ($unique_event_names as $unique_event_name) { + foreach ($importable_instruments as $importable_instrument) { + $instrument_records = $redcap_client->getInstrumentRecords( + $importable_instrument, + $unique_event_name, + null, + ); + + $records = array_merge($records, $instrument_records); + } +} $record_importer = new RedcapRecordImporter( $lorisInstance, From 78d74a464515da69bd69e1a8ffd2a2fc1efbe53f Mon Sep 17 00:00:00 2001 From: Maxime Mulder Date: Sun, 26 Oct 2025 05:27:03 +0000 Subject: [PATCH 3/6] move script to tools directory --- {modules/redcap/tools => tools}/import_redcap_records.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename {modules/redcap/tools => tools}/import_redcap_records.php (98%) diff --git a/modules/redcap/tools/import_redcap_records.php b/tools/import_redcap_records.php similarity index 98% rename from modules/redcap/tools/import_redcap_records.php rename to tools/import_redcap_records.php index 69b1e05a35..25b3d542be 100644 --- a/modules/redcap/tools/import_redcap_records.php +++ b/tools/import_redcap_records.php @@ -5,7 +5,7 @@ * This script will fetch records from REDCap and import them into LORIS. */ -require_once __DIR__ . "/../../../tools/generic_includes.php"; +require_once __DIR__ . "/generic_includes.php"; // Load the REDCap module. From d397436fb1aca30d3a49e009c6a22397e881e3f1 Mon Sep 17 00:00:00 2001 From: Maxime Mulder Date: Sun, 26 Oct 2025 05:27:31 +0000 Subject: [PATCH 4/6] add executable permission --- tools/import_redcap_records.php | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 tools/import_redcap_records.php diff --git a/tools/import_redcap_records.php b/tools/import_redcap_records.php old mode 100644 new mode 100755 From 190ba48a1b6bac626c8dcb2f820a92e048b5d481 Mon Sep 17 00:00:00 2001 From: Maxime Mulder Date: Tue, 9 Dec 2025 13:43:55 +0000 Subject: [PATCH 5/6] print instrument name in import --- tools/import_redcap_records.php | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tools/import_redcap_records.php b/tools/import_redcap_records.php index 25b3d542be..1d5e0b15fd 100755 --- a/tools/import_redcap_records.php +++ b/tools/import_redcap_records.php @@ -121,7 +121,8 @@ foreach ($records as $record) { fprintf( STDOUT, - "Importing record {$record->unique_event_name} {$record->record_id}...\n", + "Importing record {$record->record_id} {$record->unique_event_name}" + . " {$record->getFormName()}...\n", ); // Do not actually import REDCap records into LORIS in simulation mode. From 0836dad8e668aad50f5a317e58c8b0c248981568 Mon Sep 17 00:00:00 2001 From: Maxime Mulder Date: Wed, 10 Dec 2025 07:08:26 +0000 Subject: [PATCH 6/6] address adam's comment --- modules/redcap/php/redcapmapper.class.inc | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/modules/redcap/php/redcapmapper.class.inc b/modules/redcap/php/redcapmapper.class.inc index 32eeb8b49a..c0dc96a8fd 100644 --- a/modules/redcap/php/redcapmapper.class.inc +++ b/modules/redcap/php/redcapmapper.class.inc @@ -201,14 +201,13 @@ class RedcapMapper } /** - * Get the REDCap event associated with the arm and event names of a REDCap - * visit mapping configuration node. + * Get the REDCap event associated with a REDCap visit mapping configuration + * node. * * @param RedcapConfigVisit $visit_config A REDCap visit mapping configuration * node. * - * @return RedcapEvent The REDCap event associated with the REDCap visit - * mapping configuration node. + * @return RedcapEvent The associated REDCap event. */ public function getVisitConfigEvent( RedcapConfigVisit $visit_config,