[#37] Update check_data.py

scripts/akvo-response-grouper-value-check/.gitignore

@@ -1,2 +1,3 @@
 *.csv
 *.xlsx
+*.xlsx#

scripts/akvo-response-grouper-value-check/check_data.py

@@ -5,12 +5,13 @@
     get_category,
 )
 
+version="1"
 
 with open("./check/category.json", "r") as category_json:
     categories = json.loads(category_json.read())
 
 
-data = pd.read_csv("./check/2018_data_export_new.csv")
+data = pd.read_csv(f"./check/2018_data_export_new_{version}.csv")
 
 # Remove columns containing '--other--' or '--OTHER--'
 columns_to_remove = [col for col in data.columns if '--other--' in col.lower()]
@@ -19,7 +20,8 @@
 data['id'] = np.arange(len(data)) + 1
 
 # save data without other
-pd.DataFrame(data).to_csv("./check/siwins-2018-data-without-other-new.csv", index=None)
+pd.DataFrame(data).to_csv(
+    f"./check/siwins-2018-data-without-other-new-{version}.csv", index=None)
 
 col_names = {d: d.split("|")[0] for d in list(data)}
 data = data.rename(columns=col_names)
@@ -49,7 +51,7 @@ def split_value(value):
         if ":" in value:
             return [v.split(":")[1].strip() if ":" in v else v.strip() for v in value.split("|")]
         return [v.strip() for v in value]
-    return []
+    return [value]
 
 
 for column in list(set(question_ids)):
@@ -70,8 +72,13 @@ def split_value(value):
             "name": category["name"]
         }
         ct = get_category(data=res)
+        if not ct:
+            print(ct)
+            print(d)
         result.update({category["name"]: ct})
     results.append(result)
 
 
-pd.DataFrame(results).to_csv("./check/results-siwins-2018-data-without-other.csv", index=None)
+pd.DataFrame(results).to_csv(
+    f"./check/results-siwins-2018-data-without-other-{version}.csv",
+    index=None)