diff --git a/backend/app/data_source/file/api.py b/backend/app/data_source/file/api.py index 86477c6..8eb408d 100644 --- a/backend/app/data_source/file/api.py +++ b/backend/app/data_source/file/api.py @@ -29,7 +29,7 @@ def load_schema(self): md5 = file_processor.save_file_with_md5(file_data, output_dir) schema = FileSource(md5).load_schema() - return orjson.dumps(schema), 200 + return orjson.dumps(schema, option=orjson.OPT_NON_STR_KEYS), 200 except FileExistsError as e: return build_error_response(str(e)), 409 except Exception as e: diff --git a/backend/app/data_source/file/file_source.py b/backend/app/data_source/file/file_source.py index b9bdc6d..deca01f 100644 --- a/backend/app/data_source/file/file_source.py +++ b/backend/app/data_source/file/file_source.py @@ -77,7 +77,7 @@ def load_schema(self) -> FileSchema: numDistinctValues=num_distinct_values, minDate=min_date, maxDate=max_date, - numRowsByDate={row[column]: row["count"] for row in num_rows_by_date_df.rows(named=True)}, + numRowsByDate={row[column]: row["count"] for row in num_rows_by_date_df.rows(named=True) if row[column] is not None}, values=column_to_values[column] )) return FileSchema( diff --git a/backend/app/insight/api.py b/backend/app/insight/api.py index 98b6fb5..b35052b 100644 --- a/backend/app/insight/api.py +++ b/backend/app/insight/api.py @@ -194,7 +194,7 @@ def get_insight(self): try: logger.info('Reading file') df = pl.read_csv(f'/tmp/dsensei/{file_id}') \ - .with_columns(pl.col(date_column).str.slice(0, 10).str.to_date().alias("date")) + .with_columns(pl.col(date_column).str.slice(0, 10).str.to_date(strict=False).alias("date")) logger.info('File loaded') insight_builder = DFBasedInsightBuilder(