Merge branch 'NES-991' into dev

patientregistrationsystem/qdc/experiment/tests/test_forms.py

@@ -1990,62 +1990,37 @@ def test_SEP_Pause_Fix_Random_is_not_valid_number_of_uses_to_insert(self):
                          ["Certifique-se que este valor seja maior ou igual a 1."])
 
 
-class SEP_Questionnaire_FormTest(TestCase):
+class SepQuestionnaireFormTest(TestCase):
 
     @classmethod
     def setUp(cls):
-        cls.data = {
-            'name': 'Experimento TOC',
-            'description': 'Experimento TOC',
-        }
+        cls.data = {'name': 'Experimento TOC', 'description': 'Experimento TOC'}
 
         cls.research_project = ResearchProject.objects.create(
             title="Research project title", start_date=datetime.date.today(),
-            description="Research project description"
-        )
+            description="Research project description")
 
         cls.experiment = Experiment.objects.create(
-            research_project_id=cls.research_project.id,
-            title="Experimento-Update",
-            description="Descricao do Experimento-Update",
-            source_code_url="http://www.if.usp.br",
+            research_project_id=cls.research_project.id, title="Experimento-Update",
+            description="Descricao do Experimento-Update", source_code_url="http://www.if.usp.br",
             ethics_committee_project_url="http://www.fm.usp.br",
             ethics_committee_project_file="/users/celsovi/documents/unit_tests/links.rtf",
-            is_public=True,
-            data_acquisition_is_concluded=True)
+            is_public=True, data_acquisition_is_concluded=True)
 
-        cls.component = Component.objects.create(experiment=cls.experiment,
-                                                 identification="Identification",
-                                                 component_type="questionnaire"
-                                                 )
-
-        # Conecta no Lime Survey
-        cls.lime_survey = Questionnaires()
-
-        # Checa se conseguiu conectar no limeSurvey com as credenciais fornecidas no settings.py
-        cls.assertIsNotNone(cls.lime_survey.session_key,
-                            'Failed to connect LimeSurvey')
-
-        # Cria uma survey no Lime Survey
-        # TODO (NES-991): mock LimeSurvey
-        cls.survey_id = cls.lime_survey.add_survey(9999,
-                                                   'Questionario de teste - DjangoTests',
-                                                   'en', 'G')
-        # Deleta a survey gerada no Lime Survey
-        # TODO (NES-991): after mock above this is not necessary
-        status = cls.lime_survey.delete_survey(cls.survey_id)
+        cls.component = Component.objects.create(
+            experiment=cls.experiment, identification="Identification",
+            component_type="questionnaire")
 
     def test_SEP_Questionnaire_is_valid(self):
         duration_value = "5"
         identification = "Identification"
         experiment = self.experiment
 
-        # new_specific_component = Questionnaire()
-
-        sep_questionnaire_form = ComponentForm(data={'identification': identification,
-                                                     'duration_value': duration_value,
-                                                     'experiment': experiment
-                                                     })
+        sep_questionnaire_form = ComponentForm(
+            data={
+                'identification': identification, 'duration_value': duration_value,
+                'experiment': experiment
+            })
         sep_questionnaire_form.component_type = self.component.component_type  # jury-rig
         self.assertTrue(sep_questionnaire_form.is_valid())
 
@@ -2067,23 +2042,23 @@ def test_SEP_Questionnaire_is_not_valid_identification(self):
     def test_SEP_Questionnaire_Fix_Random_is_valid(self):
         number_of_uses_to_insert = 1  # valor inicial >=1
 
-        number_of_uses_SEP_Questionnaire_Fix_Random_form = NumberOfUsesToInsertForm(
+        number_of_uses_sep_questionnaire_fix_random_form = NumberOfUsesToInsertForm(
             data={'number_of_uses_to_insert': number_of_uses_to_insert}
         )
-        self.assertTrue(number_of_uses_SEP_Questionnaire_Fix_Random_form.is_valid())
+        self.assertTrue(number_of_uses_sep_questionnaire_fix_random_form.is_valid())
 
     def test_SEP_Questionnaire_Fix_Random_is_not_valid_number_of_uses_to_insert(self):
         number_of_uses_to_insert = 0  # valor inicial >=1
 
-        number_of_uses_SEP_Questionnaire_Fix_Random_form = NumberOfUsesToInsertForm(
+        number_of_uses_sep_questionnaire_fix_random_form = NumberOfUsesToInsertForm(
             data={'number_of_uses_to_insert': number_of_uses_to_insert}
         )
-        self.assertFalse(number_of_uses_SEP_Questionnaire_Fix_Random_form.is_valid())
-        self.assertEqual(number_of_uses_SEP_Questionnaire_Fix_Random_form.errors["number_of_uses_to_insert"],
+        self.assertFalse(number_of_uses_sep_questionnaire_fix_random_form.is_valid())
+        self.assertEqual(number_of_uses_sep_questionnaire_fix_random_form.errors["number_of_uses_to_insert"],
                          ["Certifique-se que este valor seja maior ou igual a 1."])
 
 
-class SEP_Stimulus_FormTest(TestCase):
+class SepStimulusFormTest(TestCase):
 
     @classmethod
     def setUp(cls):

patientregistrationsystem/qdc/export/export.py

@@ -223,12 +223,6 @@ def log_message(self, text, param1='', param2=''):
 
 class ExportExecution:
 
-    def get_username(self, request):
-        self.user_name = None
-        if request.user.is_authenticated():
-            self.user_name = request.user.username
-        return self.user_name
-
     def __init__(self, user_id, export_id):
         self.files_to_zip_list = []
         self.directory_base = ''
@@ -247,6 +241,33 @@ def __init__(self, user_id, export_id):
         self.per_group_data = {}
         self.questionnaire_utils = QuestionnaireUtils()
 
+    @staticmethod
+    def _temp_method_to_remove_undesirable_line(fields):
+        items = [item for item in fields if 'participant_code' in item]
+        for item in items:
+            fields.remove(item)
+
+    @staticmethod
+    def update_duplicates(fields):
+        """
+        Update duplicates in fields list by adding numbers for duplicates, started
+        in 1, then 2 etc.
+        :param fields: list
+        """
+        for field in fields:
+            duplicated_indexes = [i for i, duplicated_fields in enumerate(fields) if field == duplicated_fields]
+            if len(duplicated_indexes) > 1:
+                j = 1
+                for i in duplicated_indexes:
+                    fields[i] += str(j)
+                    j += 1
+
+    def get_username(self, request):
+        self.user_name = None
+        if request.user.is_authenticated():
+            self.user_name = request.user.username
+        return self.user_name
+
     def set_directory_base(self, user_id, export_id):
         self.directory_base = path.join(self.base_directory_name, str(user_id), str(export_id))
 
@@ -1001,18 +1022,18 @@ def merge_questionnaire_answer_list_per_participant(row_participant_data, answer
         return questionnaire_answer_list
 
     def merge_participants_data_per_questionnaire_process(self, fields_description, participant_list):
-        # get fields from patient
+        # Get fields from patient
         export_participant_row = self.process_participant_data(self.get_input_data('participants'), participant_list)
 
-        # merge fields from questionnaires and patient
+        # Merge fields from questionnaires and patient
         export_fields_list = []
-        # building the header
+        # Building the header
         export_row_list = fields_description[0][0:len(fields_description[0]) - 1]
         for field in export_participant_row[0]:
             export_row_list.append(field)
         export_fields_list.append(export_row_list)
 
-        # including the responses
+        # Including the responses
         for fields in fields_description[1:fields_description.__len__()]:
             participation_code = fields[len(fields) - 1]
             export_row_list = fields[0:len(fields) - 1]
@@ -1158,6 +1179,9 @@ def process_per_questionnaire(self, heading_type, plugin):
 
                 complete_filename = path.join(export_metadata_path, export_filename)
 
+                # At this point of the championship, fix it right here
+                self._temp_method_to_remove_undesirable_line(questionnaire_fields)
+
                 save_to_csv(complete_filename, questionnaire_fields, filesformat_type)
 
                 self.files_to_zip_list.append([
@@ -1303,6 +1327,10 @@ def process_per_entrance_questionnaire(self, heading_type):
                     questionnaire['prefix_filename_fields'], str(questionnaire_code), language)
                 # Path ex. NES_EXPORT/Participant_data/Questionnaire_metadata/Q123_aaa/Fields_Q123.csv'
                 complete_filename = path.join(export_metadata_path, export_filename + '.' + filesformat_type)
+
+                # At this point of the championship, fix it right here
+                self._temp_method_to_remove_undesirable_line(questionnaire_fields)
+
                 save_to_csv(complete_filename, questionnaire_fields, filesformat_type)
 
                 self.files_to_zip_list.append([
@@ -1574,7 +1602,6 @@ def process_per_experiment_questionnaire(self, heading_type, per_experiment_plug
                                     prefix_filename_fields, 'QS', language, filesformat_type)
 
                         complete_filename = path.join(complete_export_metadata_path, export_filename)
-
                         save_to_csv(complete_filename, questionnaire_fields, filesformat_type)
 
                         self.files_to_zip_list.append([
@@ -1892,7 +1919,6 @@ def process_per_participant_per_experiment(self, heading_type, per_experiment_pl
                             per_participant_rows = self.merge_questionnaire_answer_list_per_participant(
                                 export_rows_participants[1], answer_list[1: len(answer_list)])
 
-                            # TODO (NES-991): answer_list changes
                             field_type = 'fields' if heading_type == 'code' else 'header_questionnaire'
                             header = self.build_header_questionnaire_per_participant(
                                 export_rows_participants[0], answer_list[0][field_type])
@@ -2572,7 +2598,7 @@ def process_participant_data(self, participants_output_fields, participants, lan
 
         export_rows_participants = [headers]
 
-        # transform data
+        # Transform data
         for record in db_data:
             participant_rows = []
             for value in record:
@@ -2583,18 +2609,21 @@ def process_participant_data(self, participants_output_fields, participants, lan
 
         return export_rows_participants
 
-    def process_diagnosis_data(self, participants_output_fields, participants_list):
+    def process_diagnosis_data(self, participants_output_fields, participants_list, heading_type):
         headers, fields = self.get_headers_and_fields(participants_output_fields)
         model_to_export = getattr(modules['patient.models'], 'Patient')
         db_data = model_to_export.objects.filter(id__in=participants_list).values_list(*fields).extra(
             order_by=['id'])
 
         export_rows_participants = [headers]
 
-        # transform data
+        # Transform data
         for record in db_data:
             export_rows_participants.append([self.handle_exported_field(field) for field in record])
 
+        if heading_type == 'abbreviated':
+            self.update_duplicates(export_rows_participants[0])
+
         return export_rows_participants
 
     def get_participant_data_per_code(self, subject_code, questionnaire_response_fields):
@@ -2609,7 +2638,7 @@ def get_participant_data_per_code(self, subject_code, questionnaire_response_fie
 
         return questionnaire_response_fields
 
-    def build_participant_export_data(self, per_experiment):
+    def build_participant_export_data(self, per_experiment, heading_type):
         error_msg = ''
         participants_filtered_list = self.get_participants_filtered_data()
         # Process participants/diagnosis (Per_participant directory)
@@ -2633,11 +2662,11 @@ def build_participant_export_data(self, per_experiment):
             # TODO (NES-987): change this like in other places
             file_extension = 'tsv'
             separator = '\t'
-            export_filename = '%s.%s' % (self.get_input_data('participants')['output_filename'], file_extension)
         else:
             file_extension = 'csv'
             separator = ','
-            export_filename = '%s.%s' % (self.get_input_data('participants')['output_filename'], file_extension)
+
+        export_filename = '%s.%s' % (self.get_input_data('participants')['output_filename'], file_extension)
 
         complete_filename = path.join(base_export_directory, export_filename)
 
@@ -2668,14 +2697,14 @@ def build_participant_export_data(self, per_experiment):
 
         if diagnosis_input_data['output_list'] and participants_filtered_list:
             export_rows_diagnosis = self.process_diagnosis_data(
-                diagnosis_input_data['output_list'], participants_filtered_list)
+                diagnosis_input_data['output_list'], participants_filtered_list, heading_type)
 
             # TODO (NES-987): refactor this as in other places
             file_extension = 'tsv' if 'tsv' in self.get_input_data('filesformat_type') else 'csv'
             export_filename = ('%s.' + file_extension) % self.get_input_data('diagnosis')['output_filename']
             complete_filename = path.join(base_export_directory, export_filename)
 
-            diagnosis_headers, diagnosis_field_types = self._set_diagnosis_fields()
+            diagnosis_field_types = self._set_diagnosis_fields()
 
             self.files_to_zip_list.append([
                 complete_filename, base_directory,
@@ -2685,7 +2714,7 @@ def build_participant_export_data(self, per_experiment):
                     'format': file_extension, 'mediatype': 'text/%s' % file_extension, 'encoding': 'UTF-8',
                     'profile': 'tabular-data-resource',
                     'schema': {
-                        'fields': self._set_datapackage_table_schema(diagnosis_headers, diagnosis_field_types)
+                        'fields': self._set_datapackage_table_schema(export_rows_diagnosis[0], diagnosis_field_types)
                     }
                 }
             ])
@@ -2711,7 +2740,7 @@ def _set_datapackage_table_schema(self, headers, model_fields):
         fields = []
         for header, field in zip(headers, model_fields):
             field_info = {
-                'name': slugify(header), 'title': header, 'format': 'default', 'type': self._get_type(field)
+                'name': header, 'title': header, 'format': 'default', 'type': self._get_type(field)
             }
             fields.append(field_info)
 
@@ -2768,13 +2797,11 @@ def _set_diagnosis_fields(self):
             'medicalrecorddata__diagnosis__classification_of_diseases__abbreviated_description':
                 ClassificationOfDiseases._meta.get_field('abbreviated_description').__class__
         }
-        diagnosis_headers = []
         diagnosis_field_types = []
         for field in self.get_input_data('diagnosis')['output_list']:
-            diagnosis_headers.append(field['header'])
             diagnosis_field_types.append(field_types[field['field']])
 
-        return diagnosis_headers, diagnosis_field_types
+        return diagnosis_field_types
 
     @staticmethod
     def _set_questionnaire_metadata_fields():
@@ -2809,7 +2836,7 @@ def _set_questionnaire_response_fields(heading_type, participant_fields, questio
                 if abbreviated_data(item[key], heading_type) == participant_field)
             fields.append({
                 # str(field_info[key]) needed because of PATIENT_FIELDS 'description' keys are localized
-                'name': field_info['header'],
+                'name': abbreviated_data(str(field_info[key]), heading_type),
                 'title': abbreviated_data(str(field_info[key]), heading_type), 'type': field_info['json_data_type'],
                 'format': 'default'
             })
@@ -2822,8 +2849,10 @@ def _set_questionnaire_response_fields(heading_type, participant_fields, questio
             question = next(item for item in questions if item['title'] == question_cleared)
             type = QUESTION_TYPES[question['type']][1]
             title = question_header_questionnaire if heading_type != 'code' else question_field
-            fields.append({
-                'name': slugify(question_field), 'title': title, 'type': type, 'format': 'default'
+            # i + 2: currently in exportation, question headers are inserted between
+            # [participant_code, age] and the rest of participant fields when those exists
+            fields.insert(i + 2, {
+                'name': title, 'title': title, 'type': type, 'format': 'default'
             })
 
         return fields
@@ -3532,7 +3561,7 @@ def define_questionnaire(self, questionnaire, questionnaire_lime_survey, languag
         # Filter data (participants)
         questionnaire_responses = QuestionnaireResponse.objects.filter(survey__lime_survey_id=questionnaire_id)
 
-        # Include new filter that come from advanced search
+        # Include new filter that comes from advanced search
         filtered_data = self.get_participants_filtered_data()
         questionnaire_responses = questionnaire_responses.filter(patient_id__in=filtered_data)
 
@@ -3574,7 +3603,7 @@ def define_questionnaire(self, questionnaire, questionnaire_lime_survey, languag
                 if field in fill_list1[0]:
                     subscripts.append(fill_list1[0].index(field))
 
-            # if responses exits
+            # If responses exists
             if subscripts:
                 data_from_lime_survey = {}