Merge pull request #2412 from Kodiologist/shebang-pointing

Skip shebangs in the reader, not outside it

NEWS.rst

@@ -11,6 +11,7 @@ Bug Fixes
 ------------------------------
 * Fixed an installation failure in some situations when version lookup
   fails.
+* Fixed traceback pointing in scripts with shebangs.
 
 New Features
 ------------------------------

hy/reader/__init__.py

@@ -30,16 +30,11 @@ def read_many(stream, filename="<string>", reader=None, skip_shebang=False):
     if isinstance(stream, str):
         stream = StringIO(stream)
     pos = stream.tell()
-    if skip_shebang:
-        if stream.read(2) == "#!":
-            stream.readline()
-            pos = stream.tell()
-        else:
-            stream.seek(pos)
     source = stream.read()
     stream.seek(pos)
 
-    m = hy.models.Lazy((reader or HyReader()).parse(stream, filename))
+    m = hy.models.Lazy((reader or HyReader()).parse(
+        stream, filename, skip_shebang))
     m.source = source
     m.filename = filename
     return m

hy/reader/hy_reader.py

@@ -1,5 +1,7 @@
 "Character reader for parsing Hy source."
 
+from itertools import islice
+
 import hy
 from hy.models import (
     Bytes,
@@ -140,7 +142,7 @@ def read_default(self, key):
             return self.prefixed_string('"', ident)
         return as_identifier(ident, reader=self)
 
-    def parse(self, stream, filename=None):
+    def parse(self, stream, filename=None, skip_shebang=False):
         """Yields all `hy.models.Object`'s in `source`
 
         Additionally exposes `self` as ``hy.&reader`` during read/compile time.
@@ -151,8 +153,16 @@ def parse(self, stream, filename=None):
             filename (str | None):
                 Filename to use for error messages. If `None` then previously
                 set filename is used.
+            skip_shebang:
+                Whether to detect a skip a shebang line at the start.
         """
         self._set_source(stream, filename)
+
+        if skip_shebang and "".join(islice(self.peeking(), len("#!"))) == "#!":
+            for c in self.chars():
+                if c == "\n":
+                    break
+
         rname = mangle("&reader")
         old_reader = getattr(hy, rname, None)
         setattr(hy, rname, self)

tests/test_bin.py

@@ -33,19 +33,18 @@ def run_cmd(
     else:
         env.pop("PYTHONDONTWRITEBYTECODE", None)
 
-    p = subprocess.Popen(
-        shlex.split(cmd),
-        stdin=subprocess.PIPE,
+    result = subprocess.run(
+        shlex.split(cmd) if isinstance(cmd, str) else cmd,
+        input=stdin_data,
         stdout=stdout,
         stderr=subprocess.PIPE,
         universal_newlines=True,
         shell=False,
         env=env,
         cwd=cwd,
     )
-    output = p.communicate(input=stdin_data)
-    assert p.wait() == expect
-    return output
+    assert result.returncode == expect
+    return (result.stdout, result.stderr)
 
 def rm(fpath):
     try:
@@ -351,7 +350,7 @@ def test_hyc():
     assert "usage" in output
 
     path = "tests/resources/argparse_ex.hy"
-    _, err = run_cmd("hyc " + path)
+    _, err = run_cmd(["hyc", path])
     assert "Compiling" in err
     assert os.path.exists(cache_from_source(path))
     rm(cache_from_source(path))
@@ -472,7 +471,7 @@ def testc_file_sys_path():
         rm(cache_from_source(test_file))
         assert not os.path.exists(cache_from_source(file_relative_path))
 
-        output, _ = run_cmd(f"{binary} {test_file}")
+        output, _ = run_cmd([binary, test_file])
         assert repr(file_relative_path) in output
 
 
@@ -500,12 +499,12 @@ def test_circular_macro_require():
     test_file = "tests/resources/bin/circular_macro_require.hy"
     rm(cache_from_source(test_file))
     assert not os.path.exists(cache_from_source(test_file))
-    output, _ = run_cmd("hy {}".format(test_file))
+    output, _ = run_cmd(["hy", test_file])
     assert output.strip() == "WOWIE"
 
     # Now, with bytecode
     assert os.path.exists(cache_from_source(test_file))
-    output, _ = run_cmd("hy {}".format(test_file))
+    output, _ = run_cmd(["hy", test_file])
     assert output.strip() == "WOWIE"
 
 
@@ -519,12 +518,12 @@ def test_macro_require():
     test_file = "tests/resources/bin/require_and_eval.hy"
     rm(cache_from_source(test_file))
     assert not os.path.exists(cache_from_source(test_file))
-    output, _ = run_cmd("hy {}".format(test_file))
+    output, _ = run_cmd(["hy", test_file])
     assert output.strip() == "abc"
 
     # Now, with bytecode
     assert os.path.exists(cache_from_source(test_file))
-    output, _ = run_cmd("hy {}".format(test_file))
+    output, _ = run_cmd(["hy", test_file])
     assert output.strip() == "abc"
 
 
@@ -611,6 +610,15 @@ def req_err(x):
     assert error_lines[-1].startswith("TypeError")
 
 
+def test_traceback_shebang(tmp_path):
+    # https://github.com/hylang/hy/issues/2405
+    (tmp_path / 'ex.hy').write_text('#!my cool shebang\n(/ 1 0)')
+    _, error = run_cmd(['hy', tmp_path / 'ex.hy'], expect = 1)
+    assert 'ZeroDivisionError'
+    assert 'my cool shebang' not in error
+    assert '(/ 1 0)' in error
+
+
 def test_hystartup():
     # spy == True and custom repl-output-fn
     os.environ["HYSTARTUP"] = "tests/resources/hystartup.hy"
@@ -651,11 +659,10 @@ def test_output_buffering(tmp_path):
         (import  sys  pathlib [Path])
         (print :file sys.stderr (.strip (.read-text (Path #[=[{tf}]=]))))
         (print "line 2")''')
-    pf = shlex.quote(str(pf))
 
-    for flag, expected in ("", ""), ("--unbuffered", "line 1"):
+    for flags, expected in ([], ""), (["--unbuffered"], "line 1"):
         with open(tf, "wb") as o:
-            _, stderr = run_cmd(f"hy {flag} {pf}", stdout=o)
+            _, stderr = run_cmd(["hy", *flags, pf], stdout=o)
         assert stderr.strip() == expected
         assert tf.read_text().splitlines() == ["line 1", "line 2"]
 
@@ -671,9 +678,9 @@ def test_uufileuu(tmp_path, monkeypatch):
 
     def file_is(arg, expected_py3_9):
         expected = expected_py3_9 if PY3_9 and not PYPY else Path(arg)
-        output, _ = run_cmd("python3 " + shlex.quote(arg + "pyex.py"))
+        output, _ = run_cmd(["python3", arg + "pyex.py"])
         assert output.rstrip() == str(expected / "pyex.py")
-        output, _ = run_cmd("hy " + shlex.quote(arg + "hyex.hy"))
+        output, _ = run_cmd(["hy", arg + "hyex.hy"])
         assert output.rstrip() == str(expected / "hyex.hy")
 
     monkeypatch.chdir(tmp_path)
@@ -731,15 +738,15 @@ def test_hy2py_recursive(tmp_path):
         (setv a 1)
         (setv b "hello world")""")
 
-    _, err = run_cmd(f"hy2py {(tmp_path / 'hy').as_posix()}", expect=1)
+    _, err = run_cmd(["hy2py", (tmp_path / 'hy')], expect=1)
     assert "ValueError" in err
 
-    run_cmd("hy2py " +
-        f"{(tmp_path / 'hy').as_posix()} " +
-        f"--output {(tmp_path / 'py').as_posix()}")
+    run_cmd(["hy2py",
+        (tmp_path / 'hy'),
+        "--output", (tmp_path / 'py')])
     assert set((tmp_path / 'py').rglob('*')) == {
         tmp_path / 'py' / p
         for p in ('first.py', 'folder', 'folder/second.py')}
 
-    output, _ = run_cmd(f"python3 first.py", cwd = tmp_path / 'py')
+    output, _ = run_cmd("python3 first.py", cwd = tmp_path / 'py')
     assert output == "1\nhello world\n"

tests/test_reader.py

@@ -23,8 +23,8 @@
 from hy.reader.exceptions import LexException, PrematureEndOfInput
 
 
-def tokenize(s):
-    return list(read_many(s))
+def tokenize(*args, **kwargs):
+    return list(read_many(*args, **kwargs))
 
 
 def peoi():
@@ -675,3 +675,13 @@ def test_read_error():
     assert "".join(traceback.format_exception_only(e.type, e.value)).startswith(
         '  File "<string>", line 1\n    (do (defn))\n         ^\n'
     )
+
+
+def test_shebang():
+    from hy.errors import HySyntaxError
+
+    with pytest.raises(HySyntaxError):
+        # By default, `read_many` doesn't allow a shebang.
+        assert tokenize('#!/usr/bin/env hy\n5')
+    assert (tokenize('#!/usr/bin/env hy\n5', skip_shebang = True) ==
+        [Integer(5)])