Python hS frontend

.dockerignore

@@ -2,3 +2,6 @@ report/benchmarks
 report/resources
 report/output
 report/archive
+.venv
+python_hs/.venv
+python_hs/report/

.gitignore

@@ -1,5 +1,6 @@
 scripts/vars.sh
 *__pycache__*
+*egg-info*
 .DS_Store
 *~
 \#*\#
@@ -9,3 +10,4 @@ report/output/
 test/output_bash/
 test/output_orch/
 test/results/
+build/

Dockerfile

@@ -3,24 +3,30 @@ FROM debian:12
 RUN mkdir -p /srv/hs
 WORKDIR /srv/hs
 SHELL ["/bin/bash", "-c"]
-RUN apt update
-RUN apt install -y vim sudo git python3 python3.11-venv strace wget make python3-cram file graphviz libtool python3-matplotlib libcap2-bin util-linux
-# pash distro deps
-RUN apt install -y bc curl graphviz bsdmainutils libffi-dev locales locales-all netcat-openbsd pkg-config procps python3-pip python3-setuptools python3-testresources wamerican-insane
-# try deps
-RUN apt install -y expect mergerfs attr
+
+# https://docs.docker.com/build/cache/optimize/#use-cache-mounts
+RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
+    --mount=type=cache,target=/var/lib/apt,sharing=locked \
+    apt update \
+    && apt install -y \
+        # hs deps
+        vim sudo git python3 python3.11-venv strace wget make python3-cram file graphviz libtool python3-matplotlib libcap2-bin util-linux \
+        # pash deps
+        curl graphviz bsdmainutils libffi-dev locales locales-all netcat-openbsd pkg-config procps python3-pip python3-setuptools python3-testresources wamerican-insane \
+        # try deps
+        expect mergerfs attr
 RUN git config --global --add safe.directory /srv
-COPY . .
-RUN python3 -m venv .venv
-RUN source .venv/bin/activate
 ENV PASH_SPEC_TOP=/srv/hs
 ENV PASH_TOP=/srv/hs/deps/pash
+# pash, try
+COPY deps/ deps/
 WORKDIR /srv/hs/deps/try
 RUN make -C utils
 RUN mv utils/try-commit /bin
 RUN mv utils/try-summary /bin
 WORKDIR /srv/hs/deps/pash
 RUN ./scripts/setup-pash.sh
 WORKDIR /srv/hs
-RUN chmod +x entrypoint.sh
+RUN python3 -m venv .venv
+COPY . .
 ENTRYPOINT ["/srv/hs/entrypoint.sh"]

parallel-orch/scheduler_server.py

@@ -11,12 +11,6 @@
 ## A scheduler server
 ##
 
-def handler(signum, frame):
-    logging.debug(f'Signal: {signum} caught')
-    shutdown()
-
-signal.signal(signal.SIGTERM, handler)
-
 def parse_args():
     parser = argparse.ArgumentParser(add_help=False)
     ## TODO: Import the arguments so that they are not duplicated here and in orch
@@ -80,6 +74,12 @@ def __init__(self, socket_file, window):
         self.waiting_for_response = {}
         self.partial_program_order = None
 
+        def handler(signum, frame):
+            logging.debug(f'Signal: {signum} caught')
+            self.shutdown()
+
+        signal.signal(signal.SIGTERM, handler)
+
     def handle_init(self, input_cmd: str):
         assert(input_cmd.startswith("Init"))
         partial_order_file = input_cmd.split(":")[1].rstrip()
@@ -215,6 +215,9 @@ def shutdown(self):
         self.terminate_pending_commands()
 
     def terminate_pending_commands(self):
+        if self.partial_program_order is None:
+            return
+
         for node in self.partial_program_order.get_executing_normal_and_spec_nodes():
             node.reset_to_ready()
             # proc.terminate()

pash-spec.sh

@@ -35,7 +35,12 @@ export PASH_SPEC_SCHEDULER_SOCKET="${PASH_SPEC_TMP_PREFIX}/scheduler_socket"
 
 ## TODO: Replace this with a call to pa.sh (which will start the scheduler on its own).
 # python3 "$PASH_SPEC_TOP/parallel-orch/orch.py" "$@"
-"$PASH_TOP/pa.sh" --speculative "$@"
+if [ "$1" = "--python" ]; then
+    shift
+    "$PASH_SPEC_TOP/python_hs/entrypoint.sh" "$@"
+else
+    "$PASH_TOP/pa.sh" --speculative "$@"
+fi
 EXITCODE=$?
 
 if [ -w /sys/fs/cgroup/ ]; then

python_hs/assumptions.md

@@ -0,0 +1,12 @@
+## Correctness
+
+- Ignores python file-state changes (ex: `open`)
+- Ignores stderr
+- Assumes sandboxed evaluation context cannot make changes to system.
+
+## Performance
+
+- Only straightline code plus for loops
+- Statically known arguments to commands
+- Only can speculate loops with statically known iterators
+- Only open() with statements that do a simple redirect of a command

python_hs/entrypoint.sh

@@ -0,0 +1,19 @@
+#!/bin/bash
+
+set -eu
+
+readonly FILE="$1"
+shift
+
+# point to the local downloaded folders
+export PYTHONPATH="$PASH_TOP/python_pkgs/:$PASH_SPEC_TOP/python_hs/:${PYTHONPATH:-}"
+# for start_server
+export PASH_TMP_PREFIX="$(mktemp -d /tmp/pash_XXXXXXX)/"
+
+source "$PASH_TOP/compiler/orchestrator_runtime/pash_init_setup.sh" --speculative
+# sets daemon_pid
+start_server "$@"
+trap "cleanup_server $daemon_pid" EXIT INT TERM
+
+python3 "$PASH_SPEC_TOP/python_hs/python_hs/entrypoint.py" "$FILE"
+

python_hs/pyproject.toml

@@ -0,0 +1,60 @@
+[build-system]
+requires = ["uv_build>=0.8.11,<0.9.0"]
+build-backend = "uv_build"
+
+[project]
+name = "python_hs"
+version = "0.1.0"
+description = "Python HS"
+authors = [{name = "The PaSh Authors"}]
+requires-python = ">=3.11"
+dependencies = [] # besides hS and the pash deps
+
+[project.scripts]
+preprocessor = "python_hs.preprocessor:main"
+
+[tool.uv.build-backend]
+module-name = "python_hs"
+module-root = ""
+
+[tool.ruff]
+line-length = 88
+target-version = "py311"
+
+[tool.ruff.lint]
+select = [
+    "E",   
+    "F",   
+    "I",   
+    "B",   
+    "UP",  
+    "C4",  
+    "SIM", 
+    "RUF", 
+]
+ignore = [
+    "B011",
+    "E501",
+]
+exclude = ["report/benchmarks/*/**.py"]
+
+[tool.ruff.format]
+docstring-code-format = true
+docstring-code-line-length = 72
+
+[tool.pyright]
+include = ["python_hs", "tests"]
+typeCheckingMode = "basic"
+pythonVersion = "3.11"
+
+[tool.pytest.ini_options]
+log_cli = true
+log_cli_level = "DEBUG"
+
+[dependency-groups]
+dev = [
+    "pyright>=1.1.406",
+    # only for the test runner; tests are written with unittest
+    "pytest>=8.4.2",
+    "ruff>=0.14.0",
+]

python_hs/python-hs

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+exec "$(dirname -- "${BASH_SOURCE[0]}")"/../pash-spec.sh --python "$@"

python_hs/python_hs/__init__.py

@@ -0,0 +1,3 @@
+from python_hs.runtime import hs_run
+
+__all__ = ["hs_run"]

python_hs/python_hs/constants.py

@@ -0,0 +1,21 @@
+import os
+from pathlib import Path
+
+
+def _get_path(env_var: str) -> Path | None:
+    try:
+        return Path(os.environ[env_var])
+    except KeyError:
+        return None
+
+
+PASH_SPEC_TOP = Path(__file__).parent.parent.parent
+assert PASH_SPEC_TOP.name == "hs"
+
+PASH_TOP = PASH_SPEC_TOP / "deps" / "pash"
+PYTHON_SPEC_TOP = PASH_SPEC_TOP / "python_hs"
+
+RUNTIME_DIR = PASH_TOP / "compiler" / "orchestrator_runtime"
+
+PASH_SPEC_SCHEDULER_SOCKET = _get_path("PASH_SPEC_SCHEDULER_SOCKET")
+PASH_SPEC_TMP_PREFIX = _get_path("PASH_SPEC_TMP_PREFIX")

python_hs/python_hs/entrypoint.py

@@ -0,0 +1,32 @@
+import argparse
+import runpy
+from pathlib import Path
+
+from python_hs import logging_
+
+
+def parse_args():
+    """Parse command line arguments."""
+    parser = argparse.ArgumentParser()
+    parser.add_argument("file", help="Input file to speculate", type=Path)
+    parser.add_argument("--debug", action="store_true", help="Enable debug logging")
+    return parser.parse_args()
+
+
+def main():
+    args = parse_args()
+    logging_.DEBUG = args.debug
+
+    from python_hs.constants import PASH_SPEC_TMP_PREFIX
+    from python_hs.preprocessor import preprocess_file
+    from python_hs.runtime import init_scheduler
+
+    assert PASH_SPEC_TMP_PREFIX is not None
+
+    preprocessed_file = preprocess_file(PASH_SPEC_TMP_PREFIX, args.file)
+    init_scheduler()
+    runpy.run_path(str(preprocessed_file))
+
+
+if __name__ == "__main__":
+    main()

python_hs/python_hs/logging_.py

@@ -0,0 +1,33 @@
+from __future__ import annotations
+
+import logging
+import sys
+
+DEBUG = True
+
+
+def setup_logger(name: str) -> logging.Logger:
+    """Setup and configure a logger with consistent formatting.
+
+    Args:
+        name: Logger name.
+    Returns:
+        Configured logger instance.
+    """
+    format_string = "%(asctime)s - %(name)s - %(levelname)s - %(message)s"
+    stream = sys.stderr
+    logger = logging.getLogger(name)
+
+    logger.handlers.clear()
+
+    handler = logging.StreamHandler(stream)
+    level = logging.DEBUG if DEBUG else logging.WARNING
+    handler.setLevel(level)
+    formatter = logging.Formatter(format_string)
+    handler.setFormatter(formatter)
+
+    logger.addHandler(handler)
+    logger.setLevel(level)
+    logger.debug("Logger initiated")
+
+    return logger