168 see if the df to json can be improved using pivot tables similar to the df to download

src/regtech_data_validator/data_formatters.py

@@ -75,32 +75,53 @@ def df_to_table(df: pd.DataFrame) -> str:
 
 
 def df_to_json(df: pd.DataFrame) -> str:
-    findings_json = []
+    output_json = []
 
     if not df.empty:
-
-        for _, group_df in df.groupby(['validation_id']):
-            v_head = group_df.iloc[0]
-
-            finding_json = {
+        df.reset_index(drop=True, inplace=True)
+        findings_json = json.loads(df.to_json(orient='columns'))
+
+        grouped_data = {}
+        for i in range(len(findings_json['record_no'])):
+            validation_id = findings_json['validation_id'][str(i)]
+            if validation_id not in grouped_data:
+                grouped_data[validation_id] = []
+            grouped_data[validation_id].append(
+                {
+                    'record_no': findings_json['record_no'][str(i)],
+                    'uid': findings_json['uid'][str(i)],
+                    'field_name': findings_json['field_name'][str(i)],
+                    'field_value': findings_json['field_value'][str(i)],
+                }
+            )
+
+        for validation_id, records in grouped_data.items():
+            for key, value in findings_json['validation_id'].items():
+                if validation_id == value:
+                    validation_key = key
+                    break
+            validation_info = {
                 'validation': {
-                    'id': v_head['validation_id'],
-                    'name': v_head['validation_name'],
-                    'description': v_head['validation_desc'],
-                    'severity': v_head['validation_severity'],
-                    'scope': v_head['scope'],
-                    'fig_link': v_head['fig_link'],
+                    'id': validation_id,
+                    'name': findings_json['validation_name'][validation_key],
+                    'description': findings_json['validation_desc'][validation_key],
+                    'severity': findings_json['validation_severity'][validation_key],
+                    'scope': findings_json['scope'][validation_key],
+                    'fig_link': findings_json['fig_link'][validation_key],
                 },
                 'records': [],
             }
-            findings_json.append(finding_json)
-
-            for _, rec_df in group_df.groupby(by='record_no'):
-                rec = rec_df.iloc[0]
-                record_json = {'record_no': int(rec['record_no']), 'uid': rec['uid'], 'fields': []}
-                finding_json['records'].append(record_json)
-
-                fields = rec_df.iterrows() if v_head['validation_id'] in more_than_2_fields else rec_df[::-1].iterrows()
-                for _, field_data in fields:
-                    record_json['fields'].append({'name': field_data['field_name'], 'value': field_data['field_value']})
-    return json.dumps(findings_json, indent=4)
+            records_dict = {}
+            for record in records:
+                record_no = record['record_no']
+                if record_no not in records_dict:
+                    records_dict[record_no] = {'record_no': record['record_no'], 'uid': record['uid'], 'fields': []}
+                records_dict[record_no]['fields'].append({'name': record['field_name'], 'value': record['field_value']})
+            validation_info['records'] = list(records_dict.values())
+            for record in validation_info['records']:
+                if len(record['fields']) == 2:
+                    record['fields'][0], record['fields'][1] = record['fields'][1], record['fields'][0]
+            output_json.append(validation_info)
+
+        output_json = sorted(output_json, key=lambda x: x['validation']['id'])
+    return json.dumps(output_json, indent=4)