forked from ipython/ipython
-
Notifications
You must be signed in to change notification settings - Fork 1
/
test_inputsplitter.py
615 lines (515 loc) · 21.7 KB
/
test_inputsplitter.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
# -*- coding: utf-8 -*-
"""Tests for the inputsplitter module."""
from __future__ import print_function
# Copyright (c) IPython Development Team.
# Distributed under the terms of the Modified BSD License.
import unittest
import sys
import nose.tools as nt
from IPython.core import inputsplitter as isp
from IPython.core.inputtransformer import InputTransformer
from IPython.core.tests.test_inputtransformer import syntax, syntax_ml
from IPython.testing import tools as tt
from IPython.utils import py3compat
from IPython.utils.py3compat import string_types, input
#-----------------------------------------------------------------------------
# Semi-complete examples (also used as tests)
#-----------------------------------------------------------------------------
# Note: at the bottom, there's a slightly more complete version of this that
# can be useful during development of code here.
def mini_interactive_loop(input_func):
"""Minimal example of the logic of an interactive interpreter loop.
This serves as an example, and it is used by the test system with a fake
raw_input that simulates interactive input."""
from IPython.core.inputsplitter import InputSplitter
isp = InputSplitter()
# In practice, this input loop would be wrapped in an outside loop to read
# input indefinitely, until some exit/quit command was issued. Here we
# only illustrate the basic inner loop.
while isp.push_accepts_more():
indent = ' '*isp.indent_spaces
prompt = '>>> ' + indent
line = indent + input_func(prompt)
isp.push(line)
# Here we just return input so we can use it in a test suite, but a real
# interpreter would instead send it for execution somewhere.
src = isp.source_reset()
#print 'Input source was:\n', src # dbg
return src
#-----------------------------------------------------------------------------
# Test utilities, just for local use
#-----------------------------------------------------------------------------
def assemble(block):
"""Assemble a block into multi-line sub-blocks."""
return ['\n'.join(sub_block)+'\n' for sub_block in block]
def pseudo_input(lines):
"""Return a function that acts like raw_input but feeds the input list."""
ilines = iter(lines)
def raw_in(prompt):
try:
return next(ilines)
except StopIteration:
return ''
return raw_in
#-----------------------------------------------------------------------------
# Tests
#-----------------------------------------------------------------------------
def test_spaces():
tests = [('', 0),
(' ', 1),
('\n', 0),
(' \n', 1),
('x', 0),
(' x', 1),
(' x',2),
(' x',4),
# Note: tabs are counted as a single whitespace!
('\tx', 1),
('\t x', 2),
]
tt.check_pairs(isp.num_ini_spaces, tests)
def test_remove_comments():
tests = [('text', 'text'),
('text # comment', 'text '),
('text # comment\n', 'text \n'),
('text # comment \n', 'text \n'),
('line # c \nline\n','line \nline\n'),
('line # c \nline#c2 \nline\nline #c\n\n',
'line \nline\nline\nline \n\n'),
]
tt.check_pairs(isp.remove_comments, tests)
def test_get_input_encoding():
encoding = isp.get_input_encoding()
nt.assert_true(isinstance(encoding, string_types))
# simple-minded check that at least encoding a simple string works with the
# encoding we got.
nt.assert_equal(u'test'.encode(encoding), b'test')
class NoInputEncodingTestCase(unittest.TestCase):
def setUp(self):
self.old_stdin = sys.stdin
class X: pass
fake_stdin = X()
sys.stdin = fake_stdin
def test(self):
# Verify that if sys.stdin has no 'encoding' attribute we do the right
# thing
enc = isp.get_input_encoding()
self.assertEqual(enc, 'ascii')
def tearDown(self):
sys.stdin = self.old_stdin
class InputSplitterTestCase(unittest.TestCase):
def setUp(self):
self.isp = isp.InputSplitter()
def test_reset(self):
isp = self.isp
isp.push('x=1')
isp.reset()
self.assertEqual(isp._buffer, [])
self.assertEqual(isp.indent_spaces, 0)
self.assertEqual(isp.source, '')
self.assertEqual(isp.code, None)
self.assertEqual(isp._is_complete, False)
def test_source(self):
self.isp._store('1')
self.isp._store('2')
self.assertEqual(self.isp.source, '1\n2\n')
self.assertTrue(len(self.isp._buffer)>0)
self.assertEqual(self.isp.source_reset(), '1\n2\n')
self.assertEqual(self.isp._buffer, [])
self.assertEqual(self.isp.source, '')
def test_indent(self):
isp = self.isp # shorthand
isp.push('x=1')
self.assertEqual(isp.indent_spaces, 0)
isp.push('if 1:\n x=1')
self.assertEqual(isp.indent_spaces, 4)
isp.push('y=2\n')
self.assertEqual(isp.indent_spaces, 0)
def test_indent2(self):
isp = self.isp
isp.push('if 1:')
self.assertEqual(isp.indent_spaces, 4)
isp.push(' x=1')
self.assertEqual(isp.indent_spaces, 4)
# Blank lines shouldn't change the indent level
isp.push(' '*2)
self.assertEqual(isp.indent_spaces, 4)
def test_indent3(self):
isp = self.isp
# When a multiline statement contains parens or multiline strings, we
# shouldn't get confused.
isp.push("if 1:")
isp.push(" x = (1+\n 2)")
self.assertEqual(isp.indent_spaces, 4)
def test_indent4(self):
isp = self.isp
# whitespace after ':' should not screw up indent level
isp.push('if 1: \n x=1')
self.assertEqual(isp.indent_spaces, 4)
isp.push('y=2\n')
self.assertEqual(isp.indent_spaces, 0)
isp.push('if 1:\t\n x=1')
self.assertEqual(isp.indent_spaces, 4)
isp.push('y=2\n')
self.assertEqual(isp.indent_spaces, 0)
def test_dedent_pass(self):
isp = self.isp # shorthand
# should NOT cause dedent
isp.push('if 1:\n passes = 5')
self.assertEqual(isp.indent_spaces, 4)
isp.push('if 1:\n pass')
self.assertEqual(isp.indent_spaces, 0)
isp.push('if 1:\n pass ')
self.assertEqual(isp.indent_spaces, 0)
def test_dedent_break(self):
isp = self.isp # shorthand
# should NOT cause dedent
isp.push('while 1:\n breaks = 5')
self.assertEqual(isp.indent_spaces, 4)
isp.push('while 1:\n break')
self.assertEqual(isp.indent_spaces, 0)
isp.push('while 1:\n break ')
self.assertEqual(isp.indent_spaces, 0)
def test_dedent_continue(self):
isp = self.isp # shorthand
# should NOT cause dedent
isp.push('while 1:\n continues = 5')
self.assertEqual(isp.indent_spaces, 4)
isp.push('while 1:\n continue')
self.assertEqual(isp.indent_spaces, 0)
isp.push('while 1:\n continue ')
self.assertEqual(isp.indent_spaces, 0)
def test_dedent_raise(self):
isp = self.isp # shorthand
# should NOT cause dedent
isp.push('if 1:\n raised = 4')
self.assertEqual(isp.indent_spaces, 4)
isp.push('if 1:\n raise TypeError()')
self.assertEqual(isp.indent_spaces, 0)
isp.push('if 1:\n raise')
self.assertEqual(isp.indent_spaces, 0)
isp.push('if 1:\n raise ')
self.assertEqual(isp.indent_spaces, 0)
def test_dedent_return(self):
isp = self.isp # shorthand
# should NOT cause dedent
isp.push('if 1:\n returning = 4')
self.assertEqual(isp.indent_spaces, 4)
isp.push('if 1:\n return 5 + 493')
self.assertEqual(isp.indent_spaces, 0)
isp.push('if 1:\n return')
self.assertEqual(isp.indent_spaces, 0)
isp.push('if 1:\n return ')
self.assertEqual(isp.indent_spaces, 0)
isp.push('if 1:\n return(0)')
self.assertEqual(isp.indent_spaces, 0)
def test_push(self):
isp = self.isp
self.assertTrue(isp.push('x=1'))
def test_push2(self):
isp = self.isp
self.assertFalse(isp.push('if 1:'))
for line in [' x=1', '# a comment', ' y=2']:
print(line)
self.assertTrue(isp.push(line))
def test_push3(self):
isp = self.isp
isp.push('if True:')
isp.push(' a = 1')
self.assertFalse(isp.push('b = [1,'))
def test_push_accepts_more(self):
isp = self.isp
isp.push('x=1')
self.assertFalse(isp.push_accepts_more())
def test_push_accepts_more2(self):
isp = self.isp
isp.push('if 1:')
self.assertTrue(isp.push_accepts_more())
isp.push(' x=1')
self.assertTrue(isp.push_accepts_more())
isp.push('')
self.assertFalse(isp.push_accepts_more())
def test_push_accepts_more3(self):
isp = self.isp
isp.push("x = (2+\n3)")
self.assertFalse(isp.push_accepts_more())
def test_push_accepts_more4(self):
isp = self.isp
# When a multiline statement contains parens or multiline strings, we
# shouldn't get confused.
# FIXME: we should be able to better handle de-dents in statements like
# multiline strings and multiline expressions (continued with \ or
# parens). Right now we aren't handling the indentation tracking quite
# correctly with this, though in practice it may not be too much of a
# problem. We'll need to see.
isp.push("if 1:")
isp.push(" x = (2+")
isp.push(" 3)")
self.assertTrue(isp.push_accepts_more())
isp.push(" y = 3")
self.assertTrue(isp.push_accepts_more())
isp.push('')
self.assertFalse(isp.push_accepts_more())
def test_push_accepts_more5(self):
isp = self.isp
isp.push('try:')
isp.push(' a = 5')
isp.push('except:')
isp.push(' raise')
# We want to be able to add an else: block at this point, so it should
# wait for a blank line.
self.assertTrue(isp.push_accepts_more())
def test_continuation(self):
isp = self.isp
isp.push("import os, \\")
self.assertTrue(isp.push_accepts_more())
isp.push("sys")
self.assertFalse(isp.push_accepts_more())
def test_syntax_error(self):
isp = self.isp
# Syntax errors immediately produce a 'ready' block, so the invalid
# Python can be sent to the kernel for evaluation with possible ipython
# special-syntax conversion.
isp.push('run foo')
self.assertFalse(isp.push_accepts_more())
def test_unicode(self):
self.isp.push(u"Pérez")
self.isp.push(u'\xc3\xa9')
self.isp.push(u"u'\xc3\xa9'")
def test_line_continuation(self):
""" Test issue #2108."""
isp = self.isp
# A blank line after a line continuation should not accept more
isp.push("1 \\\n\n")
self.assertFalse(isp.push_accepts_more())
# Whitespace after a \ is a SyntaxError. The only way to test that
# here is to test that push doesn't accept more (as with
# test_syntax_error() above).
isp.push(r"1 \ ")
self.assertFalse(isp.push_accepts_more())
# Even if the line is continuable (c.f. the regular Python
# interpreter)
isp.push(r"(1 \ ")
self.assertFalse(isp.push_accepts_more())
def test_check_complete(self):
isp = self.isp
self.assertEqual(isp.check_complete("a = 1"), ('complete', None))
self.assertEqual(isp.check_complete("for a in range(5):"), ('incomplete', 4))
self.assertEqual(isp.check_complete("raise = 2"), ('invalid', None))
self.assertEqual(isp.check_complete("a = [1,\n2,"), ('incomplete', 0))
self.assertEqual(isp.check_complete("def a():\n x=1\n global x"), ('invalid', None))
class InteractiveLoopTestCase(unittest.TestCase):
"""Tests for an interactive loop like a python shell.
"""
def check_ns(self, lines, ns):
"""Validate that the given input lines produce the resulting namespace.
Note: the input lines are given exactly as they would be typed in an
auto-indenting environment, as mini_interactive_loop above already does
auto-indenting and prepends spaces to the input.
"""
src = mini_interactive_loop(pseudo_input(lines))
test_ns = {}
exec(src, test_ns)
# We can't check that the provided ns is identical to the test_ns,
# because Python fills test_ns with extra keys (copyright, etc). But
# we can check that the given dict is *contained* in test_ns
for k,v in ns.items():
self.assertEqual(test_ns[k], v)
def test_simple(self):
self.check_ns(['x=1'], dict(x=1))
def test_simple2(self):
self.check_ns(['if 1:', 'x=2'], dict(x=2))
def test_xy(self):
self.check_ns(['x=1; y=2'], dict(x=1, y=2))
def test_abc(self):
self.check_ns(['if 1:','a=1','b=2','c=3'], dict(a=1, b=2, c=3))
def test_multi(self):
self.check_ns(['x =(1+','1+','2)'], dict(x=4))
class IPythonInputTestCase(InputSplitterTestCase):
"""By just creating a new class whose .isp is a different instance, we
re-run the same test battery on the new input splitter.
In addition, this runs the tests over the syntax and syntax_ml dicts that
were tested by individual functions, as part of the OO interface.
It also makes some checks on the raw buffer storage.
"""
def setUp(self):
self.isp = isp.IPythonInputSplitter()
def test_syntax(self):
"""Call all single-line syntax tests from the main object"""
isp = self.isp
for example in syntax.values():
for raw, out_t in example:
if raw.startswith(' '):
continue
isp.push(raw+'\n')
out_raw = isp.source_raw
out = isp.source_reset()
self.assertEqual(out.rstrip(), out_t,
tt.pair_fail_msg.format("inputsplitter",raw, out_t, out))
self.assertEqual(out_raw.rstrip(), raw.rstrip())
def test_syntax_multiline(self):
isp = self.isp
for example in syntax_ml.values():
for line_pairs in example:
out_t_parts = []
raw_parts = []
for lraw, out_t_part in line_pairs:
if out_t_part is not None:
out_t_parts.append(out_t_part)
if lraw is not None:
isp.push(lraw)
raw_parts.append(lraw)
out_raw = isp.source_raw
out = isp.source_reset()
out_t = '\n'.join(out_t_parts).rstrip()
raw = '\n'.join(raw_parts).rstrip()
self.assertEqual(out.rstrip(), out_t)
self.assertEqual(out_raw.rstrip(), raw)
def test_syntax_multiline_cell(self):
isp = self.isp
for example in syntax_ml.values():
out_t_parts = []
for line_pairs in example:
raw = '\n'.join(r for r, _ in line_pairs if r is not None)
out_t = '\n'.join(t for _,t in line_pairs if t is not None)
out = isp.transform_cell(raw)
# Match ignoring trailing whitespace
self.assertEqual(out.rstrip(), out_t.rstrip())
def test_cellmagic_preempt(self):
isp = self.isp
for raw, name, line, cell in [
("%%cellm a\nIn[1]:", u'cellm', u'a', u'In[1]:'),
("%%cellm \nline\n>>> hi", u'cellm', u'', u'line\n>>> hi'),
(">>> %%cellm \nline\n>>> hi", u'cellm', u'', u'line\nhi'),
("%%cellm \n>>> hi", u'cellm', u'', u'>>> hi'),
("%%cellm \nline1\nline2", u'cellm', u'', u'line1\nline2'),
("%%cellm \nline1\\\\\nline2", u'cellm', u'', u'line1\\\\\nline2'),
]:
expected = "get_ipython().run_cell_magic(%r, %r, %r)" % (
name, line, cell
)
out = isp.transform_cell(raw)
self.assertEqual(out.rstrip(), expected.rstrip())
def test_multiline_passthrough(self):
isp = self.isp
class CommentTransformer(InputTransformer):
def __init__(self):
self._lines = []
def push(self, line):
self._lines.append(line + '#')
def reset(self):
text = '\n'.join(self._lines)
self._lines = []
return text
isp.physical_line_transforms.insert(0, CommentTransformer())
for raw, expected in [
("a=5", "a=5#"),
("%ls foo", "get_ipython().magic(%r)" % u'ls foo#'),
("!ls foo\n%ls bar", "get_ipython().system(%r)\nget_ipython().magic(%r)" % (
u'ls foo#', u'ls bar#'
)),
("1\n2\n3\n%ls foo\n4\n5", "1#\n2#\n3#\nget_ipython().magic(%r)\n4#\n5#" % u'ls foo#'),
]:
out = isp.transform_cell(raw)
self.assertEqual(out.rstrip(), expected.rstrip())
#-----------------------------------------------------------------------------
# Main - use as a script, mostly for developer experiments
#-----------------------------------------------------------------------------
if __name__ == '__main__':
# A simple demo for interactive experimentation. This code will not get
# picked up by any test suite.
from IPython.core.inputsplitter import IPythonInputSplitter
# configure here the syntax to use, prompt and whether to autoindent
#isp, start_prompt = InputSplitter(), '>>> '
isp, start_prompt = IPythonInputSplitter(), 'In> '
autoindent = True
#autoindent = False
try:
while True:
prompt = start_prompt
while isp.push_accepts_more():
indent = ' '*isp.indent_spaces
if autoindent:
line = indent + input(prompt+indent)
else:
line = input(prompt)
isp.push(line)
prompt = '... '
# Here we just return input so we can use it in a test suite, but a
# real interpreter would instead send it for execution somewhere.
#src = isp.source; raise EOFError # dbg
raw = isp.source_raw
src = isp.source_reset()
print('Input source was:\n', src)
print('Raw source was:\n', raw)
except EOFError:
print('Bye')
# Tests for cell magics support
def test_last_blank():
nt.assert_false(isp.last_blank(''))
nt.assert_false(isp.last_blank('abc'))
nt.assert_false(isp.last_blank('abc\n'))
nt.assert_false(isp.last_blank('abc\na'))
nt.assert_true(isp.last_blank('\n'))
nt.assert_true(isp.last_blank('\n '))
nt.assert_true(isp.last_blank('abc\n '))
nt.assert_true(isp.last_blank('abc\n\n'))
nt.assert_true(isp.last_blank('abc\nd\n\n'))
nt.assert_true(isp.last_blank('abc\nd\ne\n\n'))
nt.assert_true(isp.last_blank('abc \n \n \n\n'))
def test_last_two_blanks():
nt.assert_false(isp.last_two_blanks(''))
nt.assert_false(isp.last_two_blanks('abc'))
nt.assert_false(isp.last_two_blanks('abc\n'))
nt.assert_false(isp.last_two_blanks('abc\n\na'))
nt.assert_false(isp.last_two_blanks('abc\n \n'))
nt.assert_false(isp.last_two_blanks('abc\n\n'))
nt.assert_true(isp.last_two_blanks('\n\n'))
nt.assert_true(isp.last_two_blanks('\n\n '))
nt.assert_true(isp.last_two_blanks('\n \n'))
nt.assert_true(isp.last_two_blanks('abc\n\n '))
nt.assert_true(isp.last_two_blanks('abc\n\n\n'))
nt.assert_true(isp.last_two_blanks('abc\n\n \n'))
nt.assert_true(isp.last_two_blanks('abc\n\n \n '))
nt.assert_true(isp.last_two_blanks('abc\n\n \n \n'))
nt.assert_true(isp.last_two_blanks('abc\nd\n\n\n'))
nt.assert_true(isp.last_two_blanks('abc\nd\ne\nf\n\n\n'))
class CellMagicsCommon(object):
def test_whole_cell(self):
src = "%%cellm line\nbody\n"
out = self.sp.transform_cell(src)
ref = u"get_ipython().run_cell_magic({u}'cellm', {u}'line', {u}'body')\n"
nt.assert_equal(out, py3compat.u_format(ref))
def test_cellmagic_help(self):
self.sp.push('%%cellm?')
nt.assert_false(self.sp.push_accepts_more())
def tearDown(self):
self.sp.reset()
class CellModeCellMagics(CellMagicsCommon, unittest.TestCase):
sp = isp.IPythonInputSplitter(line_input_checker=False)
def test_incremental(self):
sp = self.sp
sp.push('%%cellm firstline\n')
nt.assert_true(sp.push_accepts_more()) #1
sp.push('line2\n')
nt.assert_true(sp.push_accepts_more()) #2
sp.push('\n')
# This should accept a blank line and carry on until the cell is reset
nt.assert_true(sp.push_accepts_more()) #3
def test_no_strip_coding(self):
src = '\n'.join([
'%%writefile foo.py',
'# coding: utf-8',
'print(u"üñîçø∂é")',
])
out = self.sp.transform_cell(src)
nt.assert_in('# coding: utf-8', out)
class LineModeCellMagics(CellMagicsCommon, unittest.TestCase):
sp = isp.IPythonInputSplitter(line_input_checker=True)
def test_incremental(self):
sp = self.sp
sp.push('%%cellm line2\n')
nt.assert_true(sp.push_accepts_more()) #1
sp.push('\n')
# In this case, a blank line should end the cell magic
nt.assert_false(sp.push_accepts_more()) #2