diff --git a/redcap_bridge/test_redcap/test_server_interface.py b/redcap_bridge/test_redcap/test_server_interface.py index 1fdc49fe..1c2b0fb6 100644 --- a/redcap_bridge/test_redcap/test_server_interface.py +++ b/redcap_bridge/test_redcap/test_server_interface.py @@ -63,9 +63,6 @@ def test_upload_datadict(clean_server, initialize_test_dir): def test_upload_records(clean_server, initialize_test_dir): """ Test upload of records to the server - - TODO: Finally this test should test the corresponding redcap_bridge - `upload_records` method instead of pycap itself """ upload_datadict(test_directory / 'testfiles' / 'metadata.csv', SERVER_CONFIG_YAML) res = upload_records(test_directory / 'testfiles' / 'record.csv', SERVER_CONFIG_YAML) diff --git a/redcap_bridge/test_redcap/testfiles/compression_test/expected_record.csv b/redcap_bridge/test_redcap/testfiles/compression_test/expected_record.csv index f9121b76..0f77beb4 100644 --- a/redcap_bridge/test_redcap/testfiles/compression_test/expected_record.csv +++ b/redcap_bridge/test_redcap/testfiles/compression_test/expected_record.csv @@ -1,9 +1,9 @@ -Record ID,My Comment Field,Modality,Single Mode Modality -1,blabla,"my first choice, my comment",yes -2,more blabla,"my first choice, my second choice",yes -3,,"my first choice, my third choice, no comment",yes -4,,"my first choice, my second choice, my third choice, minimal comment",yes -5,no bla,, -6,,my second choice, -7,,my third choice, -8,,"my second choice, my third choice", +Record ID,My Comment Field,Modality_compressed,Single Mode Modality,Other Modality (complex),Modality +1,blabla,"my first choice, my comment",yes,"A, B",My modality name +2,more blabla,"my first choice, my second choice",yes,"A, B",My modality name +3,,"my first choice, my third choice, no comment",yes,"A, B",My modality name +4,,"my first choice, my second choice, my third choice, minimal comment",yes,,My modality name +5,no bla,,,"A, B",My modality name +6,,my second choice,,,My modality name +7,,my third choice,,,My modality name +8,,"my second choice, my third choice",,,My modality name diff --git a/redcap_bridge/test_redcap/testfiles/compression_test/original_record.csv b/redcap_bridge/test_redcap/testfiles/compression_test/original_record.csv index e152051f..9f2f14d0 100644 --- a/redcap_bridge/test_redcap/testfiles/compression_test/original_record.csv +++ b/redcap_bridge/test_redcap/testfiles/compression_test/original_record.csv @@ -1,9 +1,9 @@ -Record ID,My Comment Field,Modality (choice=my first choice),Modality (choice=my second choice),Modality (choice=my third choice),Modality (choice=My custom choice {my_comment_field}),,Single Mode Modality (choice=yes) -1,blabla,my first choice,,,My custom choice {my_comment_field},my comment,yes -2,more blabla,my first choice,my second choice,,,,yes -3,,my first choice,,my third choice,My custom choice {my_comment_field},no comment,yes -4,,my first choice,my second choice,my third choice,My custom choice {my_comment_field},minimal comment,yes -5,no bla,,,,,, -6,,,my second choice,,,, -7,,,,my third choice,,, -8,,,my second choice,my third choice,,, +Record ID,My Comment Field,Modality (choice=my first choice),Modality (choice=my second choice),Modality (choice=my third choice),Modality (choice=My custom choice {my_comment_field}),,Single Mode Modality (choice=yes),Other Modality (complex) (choice=A),Other Modality (complex) (choice=B),Modality +1,blabla,my first choice,,,My custom choice {my_comment_field},my comment,yes,A,B,My modality name +2,more blabla,my first choice,my second choice,,,,yes,A,B,My modality name +3,,my first choice,,my third choice,My custom choice {my_comment_field},no comment,yes,A,B,My modality name +4,,my first choice,my second choice,my third choice,My custom choice {my_comment_field},minimal comment,yes,,,My modality name +5,no bla,,,,,,,A,B,My modality name +6,,,my second choice,,,,,,,My modality name +7,,,,my third choice,,,,,,My modality name +8,,,my second choice,my third choice,,,,,,My modality name diff --git a/redcap_bridge/utils.py b/redcap_bridge/utils.py index 03e39e41..40d32cfc 100644 --- a/redcap_bridge/utils.py +++ b/redcap_bridge/utils.py @@ -88,7 +88,19 @@ def merge_values(values): # merge multi-column fields names = set([c.split(' (choice=')[0] for c in df.filter(regex=r'. \(choice=.*\)').columns]) for name in names: - sub_columns = df.filter(regex=rf'^{name} \(choice=.').columns + regex_compatible_name = name + + # avoid duplicate column names + if name in df.columns: + new_name = name + '_compressed' + warnings.warn(f'Duplicate column name {name}. Creating new column with name ' + f'{new_name} instead.') + assert new_name not in df.columns + name = new_name + + for special_char in '\\.^$*+?{}[]|()': + regex_compatible_name = regex_compatible_name.replace(special_char, "\\" + special_char) + sub_columns = df.filter(regex=rf'^{regex_compatible_name} \(choice=.').columns sub_indexes = df.columns.get_indexer(sub_columns) # insert column with merged columns df.insert(loc=int(sub_indexes[0]),