Make compression robust against special characters in field names

redcap_bridge/test_redcap/test_server_interface.py

@@ -63,9 +63,6 @@ def test_upload_datadict(clean_server, initialize_test_dir):
 def test_upload_records(clean_server, initialize_test_dir):
     """
     Test upload of records to the server
-
-    TODO: Finally this test should test the corresponding redcap_bridge
-    `upload_records` method instead of pycap itself
     """
     upload_datadict(test_directory / 'testfiles' / 'metadata.csv', SERVER_CONFIG_YAML)
     res = upload_records(test_directory / 'testfiles' / 'record.csv', SERVER_CONFIG_YAML)

redcap_bridge/test_redcap/testfiles/compression_test/expected_record.csv

@@ -1,9 +1,9 @@
-Record ID,My Comment Field,Modality,Single Mode Modality
-1,blabla,"my first choice, my comment",yes
-2,more blabla,"my first choice, my second choice",yes
-3,,"my first choice, my third choice, no comment",yes
-4,,"my first choice, my second choice, my third choice, minimal comment",yes
-5,no bla,,
-6,,my second choice,
-7,,my third choice,
-8,,"my second choice, my third choice",
+Record ID,My Comment Field,Modality_compressed,Single Mode Modality,Other Modality (complex),Modality
+1,blabla,"my first choice, my comment",yes,"A, B",My modality name
+2,more blabla,"my first choice, my second choice",yes,"A, B",My modality name
+3,,"my first choice, my third choice, no comment",yes,"A, B",My modality name
+4,,"my first choice, my second choice, my third choice, minimal comment",yes,,My modality name
+5,no bla,,,"A, B",My modality name
+6,,my second choice,,,My modality name
+7,,my third choice,,,My modality name
+8,,"my second choice, my third choice",,,My modality name

redcap_bridge/test_redcap/testfiles/compression_test/original_record.csv

@@ -1,9 +1,9 @@
-Record ID,My Comment Field,Modality (choice=my first choice),Modality (choice=my second choice),Modality (choice=my third choice),Modality (choice=My custom choice {my_comment_field}),,Single Mode Modality (choice=yes)
-1,blabla,my first choice,,,My custom choice {my_comment_field},my comment,yes
-2,more blabla,my first choice,my second choice,,,,yes
-3,,my first choice,,my third choice,My custom choice {my_comment_field},no comment,yes
-4,,my first choice,my second choice,my third choice,My custom choice {my_comment_field},minimal comment,yes
-5,no bla,,,,,,
-6,,,my second choice,,,,
-7,,,,my third choice,,,
-8,,,my second choice,my third choice,,,
+Record ID,My Comment Field,Modality (choice=my first choice),Modality (choice=my second choice),Modality (choice=my third choice),Modality (choice=My custom choice {my_comment_field}),,Single Mode Modality (choice=yes),Other Modality (complex) (choice=A),Other Modality (complex) (choice=B),Modality
+1,blabla,my first choice,,,My custom choice {my_comment_field},my comment,yes,A,B,My modality name
+2,more blabla,my first choice,my second choice,,,,yes,A,B,My modality name
+3,,my first choice,,my third choice,My custom choice {my_comment_field},no comment,yes,A,B,My modality name
+4,,my first choice,my second choice,my third choice,My custom choice {my_comment_field},minimal comment,yes,,,My modality name
+5,no bla,,,,,,,A,B,My modality name
+6,,,my second choice,,,,,,,My modality name
+7,,,,my third choice,,,,,,My modality name
+8,,,my second choice,my third choice,,,,,,My modality name

redcap_bridge/utils.py

@@ -88,7 +88,19 @@ def merge_values(values):
     # merge multi-column fields
     names = set([c.split(' (choice=')[0] for c in df.filter(regex=r'. \(choice=.*\)').columns])
     for name in names:
-        sub_columns = df.filter(regex=rf'^{name} \(choice=.').columns
+        regex_compatible_name = name
+
+        # avoid duplicate column names
+        if name in df.columns:
+            new_name = name + '_compressed'
+            warnings.warn(f'Duplicate column name {name}. Creating new column with name '
+                          f'{new_name} instead.')
+            assert new_name not in df.columns
+            name = new_name
+
+        for special_char in '\\.^$*+?{}[]|()':
+            regex_compatible_name = regex_compatible_name.replace(special_char, "\\" + special_char)
+        sub_columns = df.filter(regex=rf'^{regex_compatible_name} \(choice=.').columns
         sub_indexes = df.columns.get_indexer(sub_columns)
         # insert column with merged columns
         df.insert(loc=int(sub_indexes[0]),