From 156af9427af590a46891f200fc27093e03729fdb Mon Sep 17 00:00:00 2001 From: Sandip Date: Mon, 2 Oct 2023 01:05:34 +0530 Subject: [PATCH 1/3] fix: Modify 'load' behavior to append data to existing sheet Details: - Added logic to detect existing sheet and append data if present. - There is no direct way to append on the sheet, so I am reading the sheet to get the last row number and then writing after that row number. Resolves: #5472 --- openbb_terminal/helper_funcs.py | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/openbb_terminal/helper_funcs.py b/openbb_terminal/helper_funcs.py index 700ef0b7316a..84c128a0b7b9 100644 --- a/openbb_terminal/helper_funcs.py +++ b/openbb_terminal/helper_funcs.py @@ -1602,15 +1602,28 @@ def export_data( ) elif saved_path.exists(): + # Load the Excel file to get the existing data + existing_df = pd.read_excel(saved_path, sheet_name=sheet_name) + # Get the number of rows in the existing data + start_row = ( + existing_df.shape[0] + 1 + ) # Add 1 to start writing after the last row + + # Append data to the existing sheet with pd.ExcelWriter( saved_path, mode="a", - if_sheet_exists="new", + if_sheet_exists="overlay", engine="openpyxl", ) as writer: df.to_excel( - writer, sheet_name=sheet_name, index=True, header=True + writer, + sheet_name=sheet_name, + startrow=start_row, + index=True, + header=False, ) + else: with pd.ExcelWriter( saved_path, From e7378e58ee24fb0061bab91be2606dc5ec706bfe Mon Sep 17 00:00:00 2001 From: Sandip Date: Tue, 3 Oct 2023 01:14:02 +0530 Subject: [PATCH 2/3] Fixed 'worksheet not found' error during export to an existing sheet --- openbb_terminal/helper_funcs.py | 64 ++++++++++++++++++++------------- 1 file changed, 39 insertions(+), 25 deletions(-) diff --git a/openbb_terminal/helper_funcs.py b/openbb_terminal/helper_funcs.py index 84c128a0b7b9..234b5ff36bad 100644 --- a/openbb_terminal/helper_funcs.py +++ b/openbb_terminal/helper_funcs.py @@ -1602,36 +1602,50 @@ def export_data( ) elif saved_path.exists(): - # Load the Excel file to get the existing data - existing_df = pd.read_excel(saved_path, sheet_name=sheet_name) - # Get the number of rows in the existing data - start_row = ( - existing_df.shape[0] + 1 - ) # Add 1 to start writing after the last row - - # Append data to the existing sheet - with pd.ExcelWriter( - saved_path, - mode="a", - if_sheet_exists="overlay", - engine="openpyxl", - ) as writer: - df.to_excel( - writer, - sheet_name=sheet_name, - startrow=start_row, - index=True, - header=False, - ) + # check if the sheet already exists + with pd.ExcelFile(saved_path) as reader: + if sheet_name in reader.sheet_names: + # Load the Excel file to get the existing data + existing_df = pd.read_excel( + saved_path, sheet_name=sheet_name + ) + # Get the number of rows in the existing data + start_row = existing_df.shape[0] + 1 + # Append data to the existing sheet + with pd.ExcelWriter( + saved_path, + mode="a", + if_sheet_exists="overlay", + engine="openpyxl", + ) as writer: + df.to_excel( + writer, + sheet_name=sheet_name, + startrow=start_row, + index=True, + header=False, + ) + + else: + with pd.ExcelWriter( + saved_path, + mode="a", + if_sheet_exists="new", + engine="openpyxl", + ) as writer: + df.to_excel( + writer, + sheet_name=sheet_name, + index=True, + header=True, + ) else: - with pd.ExcelWriter( - saved_path, - engine="openpyxl", - ) as writer: + with pd.ExcelWriter(saved_path, engine="openpyxl") as writer: df.to_excel( writer, sheet_name=sheet_name, index=True, header=True ) + elif saved_path.suffix in [".jpg", ".pdf", ".png", ".svg"]: if figure is None: console.print("No plot to export.") From e055ba6c492c9349a760f10ddca43d2224378381 Mon Sep 17 00:00:00 2001 From: Sandip Date: Thu, 5 Oct 2023 10:39:54 +0530 Subject: [PATCH 3/3] Changes: * Now if the sheet exists in the xlsx file it will ask the user if he wants to replace/append/create new sheet --- openbb_terminal/helper_funcs.py | 94 ++++++++++++++++++--------------- 1 file changed, 50 insertions(+), 44 deletions(-) diff --git a/openbb_terminal/helper_funcs.py b/openbb_terminal/helper_funcs.py index 234b5ff36bad..c10062b467ac 100644 --- a/openbb_terminal/helper_funcs.py +++ b/openbb_terminal/helper_funcs.py @@ -1512,6 +1512,55 @@ def ask_file_overwrite(file_path: Path) -> Tuple[bool, bool]: return False, True +def save_to_excel(df, saved_path, sheet_name, start_row=0, index=True, header=True): + """Saves a Pandas DataFrame to an Excel file. + + Args: + df: A Pandas DataFrame. + saved_path: The path to the Excel file to save to. + sheet_name: The name of the sheet to save the DataFrame to. + start_row: The row number to start writing the DataFrame at. + index: Whether to write the DataFrame index to the Excel file. + header: Whether to write the DataFrame header to the Excel file. + """ + + overwrite_options = { + "o": "replace", + "a": "overlay", + "n": "new", + } + + if not saved_path.exists(): + with pd.ExcelWriter(saved_path, engine="openpyxl") as writer: + df.to_excel(writer, sheet_name=sheet_name, index=index, header=header) + + else: + with pd.ExcelFile(saved_path) as reader: + overwrite_option = "n" + if sheet_name in reader.sheet_names: + overwrite_option = input( + "\nSheet already exists. Overwrite/Append/New? [o/a/n]: " + ).lower() + start_row = 0 + if overwrite_option == "a": + existing_df = pd.read_excel(saved_path, sheet_name=sheet_name) + start_row = existing_df.shape[0] + 1 + + with pd.ExcelWriter( + saved_path, + mode="a", + if_sheet_exists=overwrite_options[overwrite_option], + engine="openpyxl", + ) as writer: + df.to_excel( + writer, + sheet_name=sheet_name, + startrow=start_row, + index=index, + header=False if overwrite_option == "a" else header, + ) + + # This is a false positive on pylint and being tracked in pylint #3060 # pylint: disable=abstract-class-instantiated def export_data( @@ -1600,51 +1649,8 @@ def export_data( index=True, header=True, ) - - elif saved_path.exists(): - # check if the sheet already exists - with pd.ExcelFile(saved_path) as reader: - if sheet_name in reader.sheet_names: - # Load the Excel file to get the existing data - existing_df = pd.read_excel( - saved_path, sheet_name=sheet_name - ) - # Get the number of rows in the existing data - start_row = existing_df.shape[0] + 1 - # Append data to the existing sheet - with pd.ExcelWriter( - saved_path, - mode="a", - if_sheet_exists="overlay", - engine="openpyxl", - ) as writer: - df.to_excel( - writer, - sheet_name=sheet_name, - startrow=start_row, - index=True, - header=False, - ) - - else: - with pd.ExcelWriter( - saved_path, - mode="a", - if_sheet_exists="new", - engine="openpyxl", - ) as writer: - df.to_excel( - writer, - sheet_name=sheet_name, - index=True, - header=True, - ) - else: - with pd.ExcelWriter(saved_path, engine="openpyxl") as writer: - df.to_excel( - writer, sheet_name=sheet_name, index=True, header=True - ) + save_to_excel(df, saved_path, sheet_name) elif saved_path.suffix in [".jpg", ".pdf", ".png", ".svg"]: if figure is None: