Work around a bug in Python's shlex module with unicode input.

The problem appeared as a broken %popd, but ultimately the real bug is
in Python itself.  I added some more tests that exercise various parts
that were breaking due to the original problem, to at least improve
test coverage.

IPython/core/tests/test_magic.py

@@ -267,8 +267,35 @@ def doctest_time():
     Wall time: 0.00 s
     """
 
+
 def test_doctest_mode():
     "Toggle doctest_mode twice, it should be a no-op and run without error"
     _ip.magic('doctest_mode')
     _ip.magic('doctest_mode')
+
+
+def test_parse_options():
+    """Tests for basic options parsing in magics."""
+    # These are only the most minimal of tests, more should be added later.  At
+    # the very least we check that basic text/unicode calls work OK.
+    nt.assert_equal(_ip.parse_options('foo', '')[1], 'foo')
+    nt.assert_equal(_ip.parse_options(u'foo', '')[1], u'foo')
+
 
+def test_dirops():
+    """Test various directory handling operations."""
+    curpath = lambda :os.path.splitdrive(os.getcwd())[1].replace('\\','/')
+
+    startdir = os.getcwd()
+    ipdir = _ip.ipython_dir
+    try:
+        _ip.magic('cd "%s"' % ipdir)
+        nt.assert_equal(curpath(), ipdir)
+        _ip.magic('cd -')
+        nt.assert_equal(curpath(), startdir)
+        _ip.magic('pushd "%s"' % ipdir)
+        nt.assert_equal(curpath(), ipdir)
+        _ip.magic('popd')
+        nt.assert_equal(curpath(), startdir)
+    finally:
+        os.chdir(startdir)

IPython/utils/process.py

@@ -136,13 +136,12 @@ def arg_split(s, posix=False):
     function, but with a default of posix=False for splitting, so that quotes
     in inputs are respected."""
 
-    # XXX - there may be unicode-related problems here!!!  I'm not sure that
-    # shlex is truly unicode-safe, so it might be necessary to do
-    #
-    # s = s.encode(sys.stdin.encoding)
-    #
-    # first, to ensure that shlex gets a normal string.  Input from anyone who
-    # knows more about unicode and shlex than I would be good to have here...
+    # Unfortunately, python's shlex module is buggy with unicode input:
+    # http://bugs.python.org/issue1170
+    # At least encoding the input when it's unicode seems to help, but there
+    # may be more problems lurking.  Apparently this is fixed in python3.
+    if isinstance(s, unicode):
+        s = s.encode(sys.stdin.encoding)
     lex = shlex.shlex(s, posix=posix)
     lex.whitespace_split = True
     return list(lex)

IPython/utils/tests/test_process.py

@@ -18,7 +18,7 @@
 
 import nose.tools as nt
 
-from IPython.utils.process import find_cmd, FindCmdError
+from IPython.utils.process import find_cmd, FindCmdError, arg_split
 from IPython.testing import decorators as dec
 
 #-----------------------------------------------------------------------------
@@ -59,4 +59,10 @@ def test_find_cmd_fail():
     nt.assert_raises(FindCmdError,find_cmd,'asdfasdf')
 
 
-
+def test_arg_split():
+    """Ensure that argument lines are correctly split like in a shell."""
+    tests = [['hi', ['hi']],
+             [u'hi', [u'hi']],
+             ]
+    for argstr, argv in tests:
+        nt.assert_equal(arg_split(argstr), argv)