Merge pull request #95 from dihm/msg_replacement

Msg replacement

lyse/__main__.py

@@ -22,6 +22,7 @@
 import time
 import traceback
 import queue
+import warnings
 
 # 3rd party imports:
 splash.update_text('importing numpy')
@@ -41,6 +42,7 @@
 from labscript_utils.qtwidgets.headerview_with_widgets import HorizontalHeaderViewWithWidgets
 from labscript_utils.qtwidgets.outputbox import OutputBox
 import labscript_utils.shared_drive as shared_drive
+from labscript_utils import dedent
 
 from lyse.dataframe_utilities import (concat_with_padding,
                                       get_dataframe_from_shot,
@@ -1608,6 +1610,8 @@ def add_files(self, filepaths, new_row_data, done=False):
         # Update the Qt model:
         for filepath in to_add:
             self.update_row(filepath, dataframe_already_updated=True)
+
+        app.filebox.set_add_shots_progress(None, None, None)        
 
 
     @inmain_decorator()
@@ -2249,23 +2253,23 @@ def on_save_dataframe_triggered(self, choose_folder=True):
             for sequence in sequences:
                 sequence_df = pandas.DataFrame(df[df['sequence'] == sequence], columns=df.columns).dropna(axis=1, how='all')
                 labscript = sequence_df['labscript'].iloc[0]
-                filename = "dataframe_{}_{}.msg".format(sequence.to_pydatetime().strftime("%Y%m%dT%H%M%S"),labscript[:-3])
+                filename = "dataframe_{}_{}.pkl".format(sequence.to_pydatetime().strftime("%Y%m%dT%H%M%S"),labscript[:-3])
                 if not choose_folder:
                     save_path = os.path.dirname(sequence_df['filepath'].iloc[0])
                 sequence_df.infer_objects()
                 for col in sequence_df.columns :
                     if sequence_df[col].dtype == object:
                         sequence_df[col] = pandas.to_numeric(sequence_df[col], errors='ignore')
-                sequence_df.to_msgpack(os.path.join(save_path, filename))
+                sequence_df.to_pickle(os.path.join(save_path, filename))
         else:
             error_dialog('Dataframe is empty')
 
     def on_load_dataframe_triggered(self):
-        default = os.path.join(self.exp_config.get('paths', 'experiment_shot_storage'), 'dataframe.msg')
+        default = os.path.join(self.exp_config.get('paths', 'experiment_shot_storage'), 'dataframe.pkl')
         file = QtWidgets.QFileDialog.getOpenFileName(self.ui,
                         'Select dataframe file to load',
                         default,
-                        "dataframe files (*.msg)")
+                        "dataframe files (*.pkl *.msg)")
         if type(file) is tuple:
             file, _ = file
         if not file:
@@ -2274,7 +2278,22 @@ def on_load_dataframe_triggered(self):
         # Convert to standard platform specific path, otherwise Qt likes
         # forward slashes:
         file = os.path.abspath(file)
-        df = pandas.read_msgpack(file).sort_values("run time").reset_index()
+        if file.endswith('.msg'):
+            # try to read msgpack in case using older pandas
+            try:
+                df = pandas.read_msgpack(file).sort_values("run time").reset_index()
+                # raise a deprecation warning if this succeeds
+                msg = """msgpack support is being dropped by pandas >= 1.0.0.
+                Please resave this dataframe to use the new format."""
+                warnings.warn(dedent(msg),DeprecationWarning)
+            except AttributeError as err:
+                # using newer pandas that can't read msg
+                msg = """msgpack is no longer supported by pandas.
+                To read this dataframe, you must downgrade pandas to < 1.0.0.
+                You can then read this dataframe and resave it with the new format."""
+                raise DeprecationWarning(dedent(msg)) from err
+        else:
+            df = pandas.read_pickle(file).sort_values("run time").reset_index()
 
         # Check for changes in the shot files since the dataframe was exported
         def changed_since(filepath, time):
@@ -2285,7 +2304,7 @@ def changed_since(filepath, time):
 
         filepaths = df["filepath"].tolist()
         changetime_cache = os.path.getmtime(file)
-        need_updating = np.where(map(lambda x: changed_since(x, changetime_cache), filepaths))[0]
+        need_updating = np.where(list(map(lambda x: changed_since(x, changetime_cache), filepaths)))[0]
         need_updating = np.sort(need_updating)[::-1]  # sort in descending order to not remove the wrong items with pop
 
         # Reload the files where changes where made since exporting