Assertion error in _PyPegen_raise_error_known_location() when running PyRun_SimpleFileExFlags()

[vstinner]

Script reproducing the bug:

import _testcapi
import ctypes.util

filename = ctypes.util.find_library('c')
libc = ctypes.cdll.LoadLibrary(filename)
pythonapi = ctypes.pythonapi
FILE_p = ctypes.c_void_p

fopen = libc.fopen
fopen.argtypes = (ctypes.c_char_p, ctypes.c_char_p)
fopen.restype = FILE_p

fclose = libc.fclose
fclose.argtypes = (FILE_p,)

PyRun_SimpleFileExFlags = pythonapi.PyRun_SimpleFileExFlags
PyRun_SimpleFileExFlags.argtypes = (FILE_p, ctypes.c_char_p, ctypes.c_int, ctypes.c_void_p)
PyRun_SimpleFileExFlags.restype = ctypes.c_int

def main():
    filename = 'demo.py'
    with open(filename, 'w') as fp:
        fp.write('import sys = 123')

    fp = fopen(filename.encode(), b"r")

    closeit = 0
    res = PyRun_SimpleFileExFlags(fp, b'abc', closeit, None)
    print(res)

    fclose(fp)

main()

I discovered the bug when writing C code. I converted it to Python to make it easier to reproduce.

gdb traceback:

python: Parser/pegen_errors.c:314: _PyPegen_raise_error_known_location: Assertion `p->tok->fp == NULL || p->tok->fp == stdin || p->tok->done == E_EOF' failed.

Program received signal SIGABRT, Aborted.
0x00007ffff7cffdcc in __pthread_kill_implementation () from /lib64/libc.so.6

(gdb) where
#0  0x00007ffff7cffdcc in __pthread_kill_implementation () from /lib64/libc.so.6
#1  0x00007ffff7ca4f8e in raise () from /lib64/libc.so.6
#2  0x00007ffff7c8c7b3 in abort () from /lib64/libc.so.6
#3  0x00007ffff7c8d804 in __libc_message_impl.cold () from /lib64/libc.so.6
#4  0x00007ffff7c9ce45 in __assert_fail () from /lib64/libc.so.6
#5  0x000000000040440d in _PyPegen_raise_error_known_location (p=0x7fffe9a979f0, 
    errtype=<type at remote 0x9e7740>, lineno=1, col_offset=12, end_lineno=1, end_col_offset=13, 
    errmsg=0x73b0ab "invalid syntax", va=0x7fffffffcaa8) at Parser/pegen_errors.c:314
#6  0x00000000004045b9 in RAISE_ERROR_KNOWN_LOCATION (p=p@entry=0x7fffe9a979f0, errtype=<optimized out>, 
    lineno=<optimized out>, col_offset=<optimized out>, end_lineno=<optimized out>, 
    end_col_offset=<optimized out>, errmsg=0x73b0ab "invalid syntax") at Parser/pegen.h:196
#7  0x0000000000404c90 in _Pypegen_set_syntax_error (p=p@entry=0x7fffe9a979f0, 
    last_token=last_token@entry=0x7fffe96d0ef0) at Parser/pegen_errors.c:404
#8  0x0000000000403d21 in _PyPegen_run_parser (p=p@entry=0x7fffe9a979f0) at Parser/pegen.c:965
#9  0x0000000000403ec1 in _PyPegen_run_parser_from_file_pointer (fp=fp@entry=0xaf55b0, 
    start_rule=start_rule@entry=257, filename_ob=filename_ob@entry='abc', enc=enc@entry=0x0, ps1=ps1@entry=0x0, 
    ps2=ps2@entry=0x0, flags=0x0, errcode=0x0, interactive_src=0x0, arena=0x7fffe96cefe0) at Parser/pegen.c:1032
#10 0x0000000000479815 in _PyParser_ASTFromFile (fp=fp@entry=0xaf55b0, filename_ob=filename_ob@entry='abc', 
    enc=enc@entry=0x0, mode=mode@entry=257, ps1=ps1@entry=0x0, ps2=ps2@entry=0x0, flags=0x0, errcode=0x0, 
    arena=0x7fffe96cefe0) at Parser/peg_api.c:27
#11 0x000000000066f7c7 in pyrun_file (fp=fp@entry=0xaf55b0, filename=filename@entry='abc', start=start@entry=257, 
    globals=globals@entry={'__name__': '__main__', '__doc__': None, '__package__': None, '__loader__': <SourceFileLoader(name='__main__', path='abc') at remote 0x7fffe96c5b10>, '__spec__': None, '__builtins__': <module at remote 0x7ffff7c385f0>, '__file__': '/home/vstinner/python/main/bug.py', '_testcapi': <module at remote 0x7fffe9bc3410>, 'ctypes': <module at remote 0x7fffe9a10ef0>, 'filename': 'libc.so.6', 'libc': <CDLL(_FuncPtr=<_ctypes.PyCFuncPtrType(__module__='ctypes', __firstlineno__=410, _flags_=1, _restype_=<_ctypes.PyCSimpleType(__module__='ctypes', __firstlineno__=186, _type_='i', __static_attributes__=(), __dict__=<getset_descriptor at remote 0x7ffff7c57f50>, __weakref__=<getset_descriptor at remote 0x7ffff7c553d0>, __doc__=None, __ctype_be__=<_ctypes.PyCSimpleType(__module__='ctypes', __firstlineno__=186, _type_='i', __static_attributes__=(...), __dict__=<getset_descriptor at remote 0x7ffff7c57f50>, __weakref__=<getset_descriptor at remote 0x7ffff7c553d0>, __doc__=None, __ctype_le__=<...>, __ctype_be__=<...>) at remote 0...(truncated), 
    locals=locals@entry={'__name__': '__main__', '__doc__': None, '__package__': None, '__loader__': <SourceFileLoader(name='__main__', path='abc') at remote 0x7fffe96c5b10>, '__spec__': None, '__builtins__': <module at remote 0x7ffff7c385f0>, '__file__': '/home/vstinner/python/main/bug.py', '_testcapi': <module at remote 0x7fffe9bc3410>, 'ctypes': <module at remote 0x7fffe9a10ef0>, 'filename': 'libc.so.6', 'libc': <CDLL(_FuncPtr=<_ctypes.PyCFuncPtrType(__module__='ctypes', __firstlineno__=410, _flags_=1, _restype_=<_ctypes.PyCSimpleType(__module__='ctypes', __firstlineno__=186, _type_='i', __static_attributes__=(), __dict__=<getset_descriptor at remote 0x7ffff7c57f50>, __weakref__=<getset_descriptor at remote 0x7ffff7c553d0>, __doc__=None, __ctype_be__=<_ctypes.PyCSimpleType(__module__='ctypes', __firstlineno__=186, _type_='i', __static_attributes__=(...), __dict__=<getset_descriptor at remote 0x7ffff7c57f50>, __weakref__=<getset_descriptor at remote 0x7ffff7c553d0>, __doc__=None, __ctype_le__=<...>, __ctype_be__=<...>) at remote 0...(truncated), closeit=closeit@entry=0, flags=0x0) at Python/pythonrun.c:1287
#12 0x0000000000670f32 in _PyRun_SimpleFileObject (fp=fp@entry=0xaf55b0, filename=filename@entry='abc', 
    closeit=closeit@entry=0, flags=flags@entry=0x0) at Python/pythonrun.c:518
#13 0x0000000000671017 in PyRun_SimpleFileExFlags (fp=0xaf55b0, filename=<optimized out>, closeit=0, flags=0x0)

(gdb) frame 5
#5  0x000000000040440d in _PyPegen_raise_error_known_location (...)

(gdb) p p->tok->fp
$5 = (FILE *) 0xaf55b0
(gdb) p p->tok->done   // 10 == E_OK
$6 = 10

[vstinner]

PyRun_SimpleFileExFlags(fp, b'abc', closeit, None)

The filename (abc) provides in the reproducer is not a valid filename. _PyErr_ProgramDecodedTextObject() fails to open it and so returns NULL. But this case is unexpected by the parser:

    if (!error_line) {
        /* PyErr_ProgramTextObject was not called or returned NULL. If it was not called,
           then we need to find the error line from some other source, because
           p->start_rule != Py_file_input. If it returned NULL, then it either unexpectedly
           failed or we're parsing from a string or the REPL. There's a third edge case where
           we're actually parsing from a file, which has an E_EOF SyntaxError and in that case
           `PyErr_ProgramTextObject` fails because lineno points to last_file_line + 1, which
           does not physically exist */
        assert(p->tok->fp == NULL || p->tok->fp == stdin || p->tok->done == E_EOF);