Fix the possible exceptions caused by invalid date format

empower-ai · Sep 30, 2023 · 01fc26e · 01fc26e
1 parent 06550e1
commit 01fc26e
Show file tree

Hide file tree

Showing 3 changed files with 3 additions and 3 deletions.
diff --git a/backend/app/data_source/file/api.py b/backend/app/data_source/file/api.py
@@ -29,7 +29,7 @@ def load_schema(self):
             md5 = file_processor.save_file_with_md5(file_data, output_dir)
 
             schema = FileSource(md5).load_schema()
-            return orjson.dumps(schema), 200
+            return orjson.dumps(schema, option=orjson.OPT_NON_STR_KEYS), 200
         except FileExistsError as e:
             return build_error_response(str(e)), 409
         except Exception as e:

diff --git a/backend/app/data_source/file/file_source.py b/backend/app/data_source/file/file_source.py
@@ -77,7 +77,7 @@ def load_schema(self) -> FileSchema:
                     numDistinctValues=num_distinct_values,
                     minDate=min_date,
                     maxDate=max_date,
-                    numRowsByDate={row[column]: row["count"] for row in num_rows_by_date_df.rows(named=True)},
+                    numRowsByDate={row[column]: row["count"] for row in num_rows_by_date_df.rows(named=True) if row[column] is not None},
                     values=column_to_values[column]
                 ))
         return FileSchema(

diff --git a/backend/app/insight/api.py b/backend/app/insight/api.py
@@ -194,7 +194,7 @@ def get_insight(self):
         try:
             logger.info('Reading file')
             df = pl.read_csv(f'/tmp/dsensei/{file_id}') \
-                .with_columns(pl.col(date_column).str.slice(0, 10).str.to_date().alias("date"))
+                .with_columns(pl.col(date_column).str.slice(0, 10).str.to_date(strict=False).alias("date"))
 
             logger.info('File loaded')
             insight_builder = DFBasedInsightBuilder(