Added excel normalization code and associated tests

mathesar/imports/excel.py

@@ -28,10 +28,28 @@ def insert_records_from_dataframe(name, schema, column_names, engine, comment, d
     return table
 
 
+def remove_empty_rows_and_columns_from_dataframe(df):
+    if df.iloc[0].isna().any():
+
+        # drop rows with all NaN values
+        df.dropna(how='all', inplace=True)
+
+        # drop columns with all NaN values
+        df.dropna(axis=1, how='all', inplace=True)
+
+    if all(df.columns.str.startswith('Unnamed')):
+        df.columns = df.iloc[0]
+        df = df[1:]
+
+    return df
+
+
 def create_db_table_from_excel_data_file(data_file, name, schema, comment=None):
     db_name = schema.database.name
     engine = create_mathesar_engine(db_name)
-    dataframe = pandas.read_excel(data_file.file.path)
+    dataframe = remove_empty_rows_and_columns_from_dataframe(
+        pandas.read_excel(data_file.file.path)
+    )
     column_names = process_column_names(dataframe.columns)
     try:
         table = insert_records_from_dataframe(name, schema, column_names, engine, comment, dataframe)

mathesar/tests/api/test_table_api.py

@@ -29,6 +29,19 @@ def missing_keys_json_data_file():
     return data_file
 
 
+@pytest.fixture
+def misaligned_table_excel_data_file():
+    data_filepath = 'mathesar/tests/data/excel_parsing/misaligned_table.xlsx'
+    with open(data_filepath, "rb") as excel_file:
+        data_file = DataFile.objects.create(
+            file=File(excel_file),
+            created_from='file',
+            base_name='missaligned_table',
+            type='excel'
+        )
+    return data_file
+
+
 @pytest.fixture
 def schema_name():
     return 'table_tests'
@@ -1881,3 +1894,15 @@ def test_create_table_with_nested_json_objects(client, schema):
             client, table_name, table_name, datafile, schema, expected_data[index]["first_row"],
             expected_data[index]["column_names"], import_target_table=None
         )
+
+
+def test_create_table_and_normalize_excel_data_file(client, misaligned_table_excel_data_file, schema):
+    table_name = 'misaligned_table'
+    expt_name = get_expected_name(table_name, data_file=misaligned_table_excel_data_file)
+    first_row = (1, 'John', '25', 'Male')
+    column_names = ["Name", "Age", "Gender"]
+
+    check_create_table_response(
+        client, table_name, expt_name, misaligned_table_excel_data_file, schema, first_row,
+        column_names, import_target_table=None
+    )