Merge branch 'history-fix' into trunk

IPython/config/default/ipython_config.py

@@ -90,6 +90,8 @@
 
 # c.InteractiveShell.quiet = False
 
+# c.InteractiveShell.history_length = 10000
+
 # Readline 
 # c.InteractiveShell.readline_use = True
 

IPython/core/displayhook.py

@@ -173,7 +173,7 @@ def quiet(self):
         """Should we silence the display hook because of ';'?"""
         # do not print output if input ends in ';'
         try:
-            if self.shell.input_hist[self.prompt_count].endswith(';\n'):
+            if self.shell.history_manager.input_hist_parsed[self.prompt_count].endswith(';\n'):
                 return True
         except IndexError:
             # some uses of ipshellembed may fail here

IPython/core/history.py

@@ -14,13 +14,13 @@
 
 # Stdlib imports
 import fnmatch
+import json
 import os
 import sys
 
 # Our own packages
 import IPython.utils.io
 
-from IPython.core.inputlist import InputList
 from IPython.utils.pickleshare import PickleShareDB
 from IPython.utils.io import ask_yes_no
 from IPython.utils.warn import warn
@@ -36,9 +36,9 @@ class HistoryManager(object):
 
     # An instance of the IPython shell we are attached to
     shell = None
-    # An InputList instance to hold processed history
-    input_hist = None
-    # An InputList instance to hold raw history (as typed by user)
+    # A list to hold processed history
+    input_hist_parsed = None
+    # A list to hold raw history (as typed by user)
     input_hist_raw = None
     # A list of directories visited during session
     dir_hist = None
@@ -56,6 +56,11 @@ class HistoryManager(object):
     # history update, we populate the user's namespace with these, shifted as
     # necessary.
     _i00, _i, _ii, _iii = '','','',''
+
+    # A set with all forms of the exit command, so that we don't store them in
+    # the history (it's annoying to rewind the first entry and land on an exit
+    # call).
+    _exit_commands = None
 
     def __init__(self, shell):
         """Create a new history manager associated with a shell instance.
@@ -64,11 +69,11 @@ def __init__(self, shell):
         self.shell = shell
 
         # List of input with multi-line handling.
-        self.input_hist = InputList()
+        self.input_hist_parsed = []
         # This one will hold the 'raw' input history, without any
         # pre-processing.  This will allow users to retrieve the input just as
         # it was exactly typed in by the user, with %hist -r.
-        self.input_hist_raw = InputList()
+        self.input_hist_raw = []
 
         # list of visited directories
         try:
@@ -84,29 +89,21 @@ def __init__(self, shell):
             histfname = 'history-%s' % shell.profile
         else:
             histfname = 'history'
-        self.hist_file = os.path.join(shell.ipython_dir, histfname)
+        self.hist_file = os.path.join(shell.ipython_dir, histfname + '.json')
 
         # Objects related to shadow history management
         self._init_shadow_hist()
 
         self._i00, self._i, self._ii, self._iii = '','','',''
 
+        self._exit_commands = set(['Quit', 'quit', 'Exit', 'exit', '%Quit',
+                                   '%quit', '%Exit', '%exit'])
+
         # Object is fully initialized, we can now call methods on it.
 
         # Fill the history zero entry, user counter starts at 1
         self.store_inputs('\n', '\n')
 
-        # For backwards compatibility, we must put these back in the shell
-        # object, until we've removed all direct uses of the history objects in
-        # the shell itself.
-        shell.input_hist = self.input_hist
-        shell.input_hist_raw = self.input_hist_raw
-        shell.output_hist = self.output_hist
-        shell.dir_hist = self.dir_hist
-        shell.histfile = self.hist_file
-        shell.shadowhist = self.shadow_hist
-        shell.db = self.shadow_db
-
     def _init_shadow_hist(self):
         try:
             self.shadow_db = PickleShareDB(os.path.join(
@@ -119,24 +116,41 @@ def _init_shadow_hist(self):
             sys.exit()
         self.shadow_hist = ShadowHist(self.shadow_db, self.shell)
 
-    def save_hist(self):
-        """Save input history to a file (via readline library)."""
+    def populate_readline_history(self):
+        """Populate the readline history from the raw history.
 
-        try:
-            self.shell.readline.write_history_file(self.hist_file)
-        except:
-            print('Unable to save IPython command history to file: ' + 
-                  `self.hist_file`)
-
-    def reload_hist(self):
-        """Reload the input history from disk file."""
+        We only store one copy of the raw history, which is persisted to a json
+        file on disk.  The readline history is repopulated from the contents of
+        this file."""
 
         try:
             self.shell.readline.clear_history()
-            self.shell.readline.read_history_file(self.hist_file)
         except AttributeError:
             pass
+        else:
+            for h in self.input_hist_raw:
+                if not h.isspace():
+                    for line in h.splitlines():
+                        self.shell.readline.add_history(line)
 
+    def save_history(self):
+        """Save input history to a file (via readline library)."""
+        hist = dict(raw=self.input_hist_raw, #[-self.shell.history_length:],
+                    parsed=self.input_hist_parsed) #[-self.shell.history_length:])
+        with open(self.hist_file,'wt') as hfile:
+            json.dump(hist, hfile,
+                      sort_keys=True, indent=4)
+
+    def reload_history(self):
+        """Reload the input history from disk file."""
+
+        with open(self.hist_file,'rt') as hfile:
+            hist = json.load(hfile)
+            self.input_hist_parsed = hist['parsed']
+            self.input_hist_raw = hist['raw']
+            if self.shell.has_readline:
+                self.populate_readline_history()
+
     def get_history(self, index=None, raw=False, output=True):
         """Get the history list.
 
@@ -163,7 +177,7 @@ def get_history(self, index=None, raw=False, output=True):
         if raw:
             input_hist = self.input_hist_raw
         else:
-            input_hist = self.input_hist
+            input_hist = self.input_hist_parsed
         if output:
             output_hist = self.output_hist
         n = len(input_hist)
@@ -201,8 +215,13 @@ def store_inputs(self, source, source_raw=None):
         """
         if source_raw is None:
             source_raw = source
-        self.input_hist.append(source)
-        self.input_hist_raw.append(source_raw)
+
+        # do not store exit/quit commands
+        if source_raw.strip() in self._exit_commands:
+            return
+
+        self.input_hist_parsed.append(source.rstrip())
+        self.input_hist_raw.append(source_raw.rstrip())
         self.shadow_hist.add(source)
 
         # update the auto _i variables
@@ -221,12 +240,12 @@ def store_inputs(self, source, source_raw=None):
 
     def sync_inputs(self):
         """Ensure raw and translated histories have same length."""
-        if len(self.input_hist) != len (self.input_hist_raw):
-            self.input_hist_raw = InputList(self.input_hist)
+        if len(self.input_hist_parsed) != len (self.input_hist_raw):
+            self.input_hist_raw[:] = self.input_hist_parsed
 
     def reset(self):
         """Clear all histories managed by this object."""
-        self.input_hist[:] = []
+        self.input_hist_parsed[:] = []
         self.input_hist_raw[:] = []
         self.output_hist.clear()
         # The directory history can't be completely empty
@@ -299,12 +318,12 @@ def magic_history(self, parameter_s = ''):
         close_at_end = True
 
     if 't' in opts:
-        input_hist = self.shell.input_hist
+        input_hist = self.shell.history_manager.input_hist_parsed
     elif 'r' in opts:
-        input_hist = self.shell.input_hist_raw
+        input_hist = self.shell.history_manager.input_hist_raw
     else:
         # Raw history is the default
-        input_hist = self.shell.input_hist_raw
+        input_hist = self.shell.history_manager.input_hist_raw
 
     default_length = 40
     pattern = None
@@ -337,7 +356,7 @@ def magic_history(self, parameter_s = ''):
 
     found = False
     if pattern is not None:
-        sh = self.shell.shadowhist.all()
+        sh = self.shell.history_manager.shadowhist.all()
         for idx, s in sh:
             if fnmatch.fnmatch(s, pattern):
                 print("0%d: %s" %(idx, s.expandtabs(4)), file=outfile)
@@ -373,7 +392,7 @@ def magic_history(self, parameter_s = ''):
         else:
             print(inline, end='', file=outfile)
         if print_outputs:
-            output = self.shell.output_hist.get(in_num)
+            output = self.shell.history_manager.output_hist.get(in_num)
             if output is not None:
                 print(repr(output), file=outfile)
 

IPython/core/interactiveshell.py

@@ -45,7 +45,6 @@
 from IPython.core.extensions import ExtensionManager
 from IPython.core.fakemodule import FakeModule, init_fakemod_dict
 from IPython.core.history import HistoryManager
-from IPython.core.inputlist import InputList
 from IPython.core.inputsplitter import IPythonInputSplitter
 from IPython.core.logger import Logger
 from IPython.core.magic import Magic
@@ -176,6 +175,8 @@ class InteractiveShell(Configurable, Magic):
     prompts_pad_left = CBool(True, config=True)
     quiet = CBool(False, config=True)
 
+    history_length = Int(10000, config=True)
+
     # The readline stuff will eventually be moved to the terminal subclass
     # but for now, we can't do that as readline is welded in everywhere.
     readline_use = CBool(True, config=True)
@@ -293,6 +294,14 @@ def __init__(self, config=None, ipython_dir=None,
         self.hooks.late_startup_hook()
         atexit.register(self.atexit_operations)
 
+    # While we're trying to have each part of the code directly access what it
+    # needs without keeping redundant references to objects, we have too much
+    # legacy code that expects ip.db to exist, so let's make it a property that
+    # retrieves the underlying object from our new history manager.
+    @property
+    def db(self):
+        return self.history_manager.shadow_db
+
     @classmethod
     def instance(cls, *args, **kwargs):
         """Returns a global InteractiveShell instance."""
@@ -947,16 +956,16 @@ def init_user_ns(self):
             warn('help() not available - check site.py')
 
         # make global variables for user access to the histories
-        ns['_ih'] = self.input_hist
-        ns['_oh'] = self.output_hist
-        ns['_dh'] = self.dir_hist
+        ns['_ih'] = self.history_manager.input_hist_parsed
+        ns['_oh'] = self.history_manager.output_hist
+        ns['_dh'] = self.history_manager.dir_hist
 
         ns['_sh'] = shadowns
 
         # user aliases to input and output histories.  These shouldn't show up
         # in %who, as they can have very large reprs.
-        ns['In']  = self.input_hist
-        ns['Out'] = self.output_hist
+        ns['In']  = self.history_manager.input_hist_parsed
+        ns['Out'] = self.history_manager.output_hist
 
         # Store myself as the public api!!!
         ns['get_ipython'] = self.get_ipython
@@ -1228,19 +1237,13 @@ def object_inspect(self, oname):
     def init_history(self):
         self.history_manager = HistoryManager(shell=self)
 
-    def save_hist(self):
+    def save_history(self):
         """Save input history to a file (via readline library)."""
-        self.history_manager.save_hist()
+        self.history_manager.save_history()
 
-    # For backwards compatibility
-    savehist = save_hist
-
-    def reload_hist(self):
+    def reload_history(self):
         """Reload the input history from disk file."""
-        self.history_manager.reload_hist()
-
-    # For backwards compatibility
-    reloadhist = reload_hist
+        self.history_manager.reload_history()
 
     def history_saving_wrapper(self, func):
         """ Wrap func for readline history saving
@@ -1254,12 +1257,16 @@ def history_saving_wrapper(self, func):
             return func
 
         def wrapper():
-            self.save_hist()
+            self.save_history()
             try:
                 func()
             finally:
-                readline.read_history_file(self.histfile)
+                self.reload_history()
         return wrapper
+
+    def get_history(self, index=None, raw=False, output=True):
+        return self.history_manager.get_history(index, raw, output)
+
 
     #-------------------------------------------------------------------------
     # Things related to exception handling and tracebacks (not debugging)
@@ -1488,8 +1495,6 @@ def init_readline(self):
             self.has_readline = False
             self.readline = None
             # Set a number of methods that depend on readline to be no-op
-            self.save_hist = no_op
-            self.reload_hist = no_op
             self.set_readline_completer = no_op
             self.set_custom_completer = no_op
             self.set_completer_frame = no_op
@@ -1541,17 +1546,13 @@ def init_readline(self):
             delims = delims.replace(ESC_MAGIC, '')
             readline.set_completer_delims(delims)
             # otherwise we end up with a monster history after a while:
-            readline.set_history_length(1000)
+            readline.set_history_length(self.history_length)
             try:
                 #print '*** Reading readline history'  # dbg
-                readline.read_history_file(self.histfile)
+                self.reload_history() 
             except IOError:
                 pass  # It doesn't exist yet.
 
-            # If we have readline, we want our history saved upon ipython
-            # exiting. 
-            atexit.register(self.save_hist)
-
         # Configure auto-indent for all platforms
         self.set_autoindent(self.autoindent)
 
@@ -2109,7 +2110,8 @@ def myapp(self, val):  # dbg
                 list.append(self, val)
 
             import new
-            self.input_hist.append = types.MethodType(myapp, self.input_hist)
+            self.history_manager.input_hist_parsed.append = types.MethodType(myapp,
+                                                                            self.history_manager.input_hist_parsed)
         # End dbg
 
         # All user code execution must happen with our context managers active
@@ -2521,6 +2523,9 @@ def atexit_operations(self):
             except OSError:
                 pass
 
+
+        self.save_history()
+
         # Clear all user namespaces to release all references cleanly.
         self.reset()