diff --git a/bin/local-dev/main.sh b/bin/local-dev/main.sh index 2874496c998..402d69a520b 100755 --- a/bin/local-dev/main.sh +++ b/bin/local-dev/main.sh @@ -40,6 +40,7 @@ # bin/local-dev.sh up [service] # [--fresh|--build|--skip-build] [--skip=svc1,svc2] [--json] # [--worktree=PATH | --branch=NAME] +# [--install-missing | --no-install] # Default: skip build if no source/lock # changes since last build. --build forces # incremental sbt dist + yarn/bun install. @@ -72,11 +73,22 @@ # branch and run its own local-dev.sh # instead (a warning is printed when such # drift is detected). +# TOOLCHAIN: a missing JDK 17 / Node / +# Python 3.12 is offered for install +# (distro package manager for the JDK, +# nvm for Node, pyenv for Python) after +# asking. --install-missing answers yes +# without asking, --no-install only ever +# prints the hint. Without a TTY the +# prompt is skipped entirely, so scripted +# and CI runs behave as before. Same flags +# on `auto`. # bin/local-dev.sh down [service] [--skip=svc1,svc2] [--json] # stop every non-skipped service, or just # (--json: summary JSON on # stdout). # bin/local-dev.sh auto [--skip=svc1,svc2] [--json] +# [--install-missing | --no-install] # rebuild + bounce only the services whose # source changed since the last build. # bin/local-dev.sh logs tail this service's log. @@ -288,6 +300,115 @@ amap_append() { eval "$_var=\"\${$_var:-}\$_suffix\"" } +# --------- toolchain versions + consented install --------- +# The versions this repo needs (AGENTS.md's table, and the CI matrices). +# Overridable so a contributor can try a newer runtime without editing this. +TEXERA_PYTHON_VERSION="${TEXERA_PYTHON_VERSION:-3.12}" +TEXERA_NODE_VERSION="${TEXERA_NODE_VERSION:-24}" + +# This block sits above the JDK probe on purpose: the probe is the first thing +# that can fail on a fresh machine, and it should be able to offer a fix. That +# also means no colour variables here — those are defined further down, with +# the rest of the TUI helpers. + +# The package manager we'd install with, or non-zero if we don't recognise one. +_pkg_manager() { + case "$(uname -s 2>/dev/null)" in + Darwin) + command -v brew >/dev/null 2>&1 && { printf 'brew\n'; return 0; } + ;; + Linux) + local m="" + for m in apt-get dnf yum pacman zypper; do + command -v "$m" >/dev/null 2>&1 && { printf '%s\n' "$m"; return 0; } + done + ;; + esac + return 1 +} + +# What we would run to install $1. Prints the command and never executes it — +# that split is what lets us show the user the exact command before asking, and +# lets CI assert on the decision without installing anything. +# +# Java prefers the distro package manager: it is system-wide and the distro +# patches it. SDKMAN is the fallback when there's no package manager (or no +# root). Node goes through nvm and Python through pyenv, because the versions +# this repo needs are newer than most distros ship — and both bootstrap +# themselves first if absent. +# +# docker and sbt are deliberately absent. Docker needs a daemon, group +# membership and a re-login; that is not something to do behind a y/n prompt. +_install_cmd_for() { + local tool="${1:-}" pm="" + pm=$(_pkg_manager) || pm="" + case "$tool" in + java) + case "$pm" in + apt-get) printf 'sudo apt-get install -y openjdk-17-jdk\n' ;; + dnf) printf 'sudo dnf install -y java-17-openjdk-devel\n' ;; + yum) printf 'sudo yum install -y java-17-openjdk-devel\n' ;; + pacman) printf 'sudo pacman -S --noconfirm jdk17-openjdk\n' ;; + zypper) printf 'sudo zypper install -y java-17-openjdk-devel\n' ;; + brew) printf 'brew install openjdk@17\n' ;; + *) printf 'curl -s https://get.sdkman.io | bash && sdk install java 17.0.13-tem\n' ;; + esac + ;; + node) + if command -v nvm >/dev/null 2>&1 || [[ -s "${NVM_DIR:-$HOME/.nvm}/nvm.sh" ]]; then + printf 'nvm install %s && nvm alias default %s\n' \ + "$TEXERA_NODE_VERSION" "$TEXERA_NODE_VERSION" + else + printf 'curl -fsSL https://raw.githubusercontent.com/nvm-sh/nvm/v0.40.3/install.sh | bash && . "$HOME/.nvm/nvm.sh" && nvm install %s && nvm alias default %s\n' \ + "$TEXERA_NODE_VERSION" "$TEXERA_NODE_VERSION" + fi + ;; + python) + if command -v pyenv >/dev/null 2>&1; then + printf 'pyenv install -s %s\n' "$TEXERA_PYTHON_VERSION" + else + printf 'curl -fsSL https://pyenv.run | bash && export PYENV_ROOT="$HOME/.pyenv" && export PATH="$PYENV_ROOT/bin:$PATH" && pyenv install -s %s\n' \ + "$TEXERA_PYTHON_VERSION" + fi + ;; + *) return 1 ;; + esac +} + +# Ask before installing anything. Never prompts without a TTY: `up` has to stay +# scriptable, and AGENTS.md points agents at the non-interactive subcommands, so +# a non-interactive run keeps the old behaviour of printing a hint and failing. +# --install-missing / TEXERA_INSTALL_MISSING=1 assume yes +# --no-install / TEXERA_INSTALL_MISSING=0 never install +_consent_to_install() { + local tool="${1:-}" cmd="${2:-}" reply="" + case "${TEXERA_INSTALL_MISSING:-ask}" in + 1|y|yes|true) return 0 ;; + 0|n|no|false) return 1 ;; + esac + [[ -t 0 && -t 1 ]] || return 1 + printf "\n %s is missing. I can run:\n\n" "$tool" >&2 + printf " %s\n\n" "$cmd" >&2 + printf " Run it now? [y/N] " >&2 + read -r reply || return 1 + [[ "$reply" == [Yy]* ]] +} + +# Offer to install $1, then report whether it's actually there afterwards. +# Runs through `bash -lc` so shell functions like nvm / pyenv / sdk resolve. +_offer_install() { + local tool="${1:-}" cmd="" + cmd=$(_install_cmd_for "$tool") || return 1 + _consent_to_install "$tool" "$cmd" || return 1 + printf " installing %s ...\n" "$tool" >&2 + if ! bash -lc "$cmd" >&2; then + printf " %s: install command failed\n" "$tool" >&2 + return 1 + fi + printf " %s: installed\n" "$tool" >&2 + return 0 +} + # --------- toolchain (JDK 17 + node) --------- # Detect a JDK 17 installation rather than pinning one path. We try, in # order: (1) caller-set $JAVA_HOME if it really is 17, (2) macOS's official @@ -423,7 +544,13 @@ _diagnose_jdk17() { echo "" >&2 } -JAVA_HOME_DETECTED="$(_find_jdk17)" || { +JAVA_HOME_DETECTED="$(_find_jdk17)" || JAVA_HOME_DETECTED="" +# Offer to install it rather than only describing how. Declines, and every +# non-interactive run, fall through to the hint-and-exit below unchanged. +if [[ -z "$JAVA_HOME_DETECTED" ]] && _offer_install java; then + JAVA_HOME_DETECTED="$(_find_jdk17)" || JAVA_HOME_DETECTED="" +fi +if [[ -z "$JAVA_HOME_DETECTED" ]]; then echo "FATAL: could not find a JDK 17 install." >&2 _diagnose_jdk17 echo " fix:" >&2 @@ -433,7 +560,7 @@ JAVA_HOME_DETECTED="$(_find_jdk17)" || { echo " or set JAVA_HOME=/path/to/jdk-17 explicitly" >&2 echo "" >&2 exit 1 -} +fi export JAVA_HOME="$JAVA_HOME_DETECTED" export PATH="$JAVA_HOME/bin:$PATH" @@ -565,7 +692,10 @@ export STORAGE_LAKEFS_ENDPOINT="${STORAGE_LAKEFS_ENDPOINT:-http://localhost:8000 export STORAGE_LAKEFS_AUTH_USERNAME="${STORAGE_LAKEFS_AUTH_USERNAME:-AKIAIOSFOLKFSSAMPLES}" export STORAGE_LAKEFS_AUTH_PASSWORD="${STORAGE_LAKEFS_AUTH_PASSWORD:-wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY}" export STORAGE_LAKEFS_AUTH_API_SECRET="${STORAGE_LAKEFS_AUTH_API_SECRET:-random_string_for_lakefs}" -export UDF_PYTHON_PATH="${UDF_PYTHON_PATH:-$(command -v python3 2>/dev/null || command -v python 2>/dev/null)}" +# UDF_PYTHON_PATH is resolved lazily by _require_udf_python (see below) rather +# than defaulted to `command -v python3` here: the system interpreter almost +# never has amber/requirements.txt installed, and picking it silently turned a +# toolchain problem into import errors inside a Python worker. export TEXERA_DASHBOARD_SERVICE_ENDPOINT="${TEXERA_DASHBOARD_SERVICE_ENDPOINT:-http://localhost:8080}" export WORKFLOW_COMPILING_SERVICE_ENDPOINT="${WORKFLOW_COMPILING_SERVICE_ENDPOINT:-http://localhost:9090}" export WORKFLOW_EXECUTION_SERVICE_ENDPOINT="${WORKFLOW_EXECUTION_SERVICE_ENDPOINT:-http://localhost:8085}" @@ -1243,6 +1373,140 @@ _diagnose_node() { printf "\n" } +# --------- python for UDF workers --------- +# "3.12" for an interpreter, or non-zero if it can't be run at all. +_python_version_of() { + local py="${1:-}" + [[ -n "$py" && -x "$py" ]] || return 1 + "$py" -c 'import sys; print("%d.%d" % sys.version_info[:2])' 2>/dev/null +} + +_python_version_ok() { + [[ "$(_python_version_of "${1:-}" 2>/dev/null)" == "$TEXERA_PYTHON_VERSION" ]] +} + +# Can this interpreter actually run a Python UDF worker? pyarrow, pandas and +# loguru are the heavy three from amber/requirements.txt — if they import, the +# environment was populated. +_python_deps_ok() { + local py="${1:-}" + [[ -n "$py" && -x "$py" ]] || return 1 + "$py" -c 'import pyarrow, pandas, loguru' >/dev/null 2>&1 +} + +# Interpreters that might run UDF workers, best first. Prints paths only; +# suitability is the caller's business. Deduplicated, because the sibling-venv +# and pyenv guesses overlap on some layouts. +_udf_python_candidates() { + local seen="" base="" root="" d="" + # Collapse `..` segments so the path we print and export is the one a + # contributor would recognise. `realpath` isn't portable to a stock macOS, + # and only the directory is resolved — the final component stays a symlink, + # which is what we want for a venv's bin/python. + _abspath() { + local dir="" leaf="" + dir=$(dirname "$1"); leaf=$(basename "$1") + [[ -d "$dir" ]] || { printf '%s\n' "$1"; return 0; } + printf '%s/%s\n' "$(cd "$dir" && pwd -P)" "$leaf" + } + _emit() { [[ -n "${1:-}" ]] || return 0; local p=""; p=$(_abspath "$1"); case "$seen" in *"|$p|"*) return 0 ;; esac; seen="$seen|$p|"; printf '%s\n' "$p"; } + # An activated venv is the most explicit statement of intent available. + [[ -n "${VIRTUAL_ENV:-}" ]] && _emit "$VIRTUAL_ENV/bin/python" + # AGENTS.md's layout: /{texera, texera-worktrees/, venv312} + for base in "$REPO_ROOT/.." "$REPO_ROOT/../.." "$SELF_ROOT/.." "$SELF_ROOT/../.."; do + _emit "$base/venv312/bin/python" + done + if command -v pyenv >/dev/null 2>&1; then + root=$(pyenv root 2>/dev/null) || root="$HOME/.pyenv" + while IFS= read -r d; do + [[ -n "$d" ]] && _emit "$d/bin/python" + done < <(shopt -s nullglob; printf '%s\n' "$root/versions/$TEXERA_PYTHON_VERSION"*) + fi + _emit "$(command -v "python$TEXERA_PYTHON_VERSION" 2>/dev/null || true)" + _emit "$(command -v python3 2>/dev/null || true)" +} + +# Lazy resolver, mirroring _require_host_lan_ip: only the subcommands that +# actually launch services pay for it, so `status` / `logs` / `--help` don't +# spawn an interpreter per candidate. +# +# Never fatal. The JVM services run fine without a Python toolchain; only +# Python UDFs don't. So this warns, offers a fix, and carries on. +_require_udf_python() { + [[ -n "${_UDF_PYTHON_RESOLVED:-}" ]] && return 0 + _UDF_PYTHON_RESOLVED=1 + # An explicit UDF_PYTHON_PATH always wins — it's the documented override — + # but say so if it can't import what a worker needs. + if [[ -n "${UDF_PYTHON_PATH:-}" ]]; then + if ! _python_deps_ok "$UDF_PYTHON_PATH"; then + tui_warn "python: UDF_PYTHON_PATH=$UDF_PYTHON_PATH can't import amber's deps" + tui_warn "python: Python UDFs will fail at worker launch" + fi + export UDF_PYTHON_PATH + return 0 + fi + local c="" no_deps="" + while IFS= read -r c; do + [[ -n "$c" ]] || continue + _python_version_ok "$c" || continue + if _python_deps_ok "$c"; then + export UDF_PYTHON_PATH="$c" + tui_ok "python: $c ${DIM}(runs Python UDFs)${RESET}" + return 0 + fi + [[ -z "$no_deps" ]] && no_deps="$c" + done < <(_udf_python_candidates) + # Distinguish "no 3.12 anywhere" from "3.12 without amber's deps": they need + # different fixes, and the old default silently picked the second one. + if [[ -n "$no_deps" ]]; then + tui_warn "python: $no_deps is $TEXERA_PYTHON_VERSION but can't import amber's deps" + printf " ${BOLD}populate it:${RESET}\n" + printf " %s -m pip install -r %s/amber/requirements.txt -r %s/amber/operator-requirements.txt\n" \ + "$no_deps" "$REPO_ROOT" "$REPO_ROOT" + export UDF_PYTHON_PATH="$no_deps" + return 0 + fi + tui_warn "python: no Python $TEXERA_PYTHON_VERSION found — Python UDFs will not run" + # Deliberately not `_install_hint python`: that hint is about the `-i` + # dashboard's textual dependency, which is a different interpreter and a + # different requirements file. + printf " ${BOLD}looked in:${RESET} \$VIRTUAL_ENV, /venv312, pyenv, PATH\n" + printf " ${BOLD}or point at one yourself:${RESET} export UDF_PYTHON_PATH=/path/to/python%s\n" \ + "$TEXERA_PYTHON_VERSION" + if _offer_install python; then + while IFS= read -r c; do + [[ -n "$c" ]] || continue + if _python_version_ok "$c"; then + export UDF_PYTHON_PATH="$c" + tui_ok "python: $c" + tui_warn "python: now install amber's deps into it:" + printf " %s -m pip install -r %s/amber/requirements.txt -r %s/amber/operator-requirements.txt\n" \ + "$c" "$REPO_ROOT" "$REPO_ROOT" + return 0 + fi + done < <(_udf_python_candidates) + fi + return 0 +} + +# Translate --install-missing / --no-install into TEXERA_INSTALL_MISSING. The +# flags beat the environment variable; the contradiction is refused rather than +# silently resolved one way. +_set_install_flag() { + local want="" + case "${1:-}" in + --install-missing) want=1 ;; + --no-install) want=0 ;; + *) return 0 ;; + esac + if [[ -n "${_INSTALL_FLAG_SEEN:-}" && "${_INSTALL_FLAG_SEEN}" != "$want" ]]; then + tui_err "--install-missing and --no-install are mutually exclusive" >&2 + exit 2 + fi + _INSTALL_FLAG_SEEN="$want" + export TEXERA_INSTALL_MISSING="$want" +} + # --------- helpers --------- # PID of whatever is listening on a TCP port, or nothing. Always exits 0 — # "empty output" is the contract for "nothing is listening", and callers @@ -2427,6 +2691,7 @@ cmd_up() { --skip-build) BUILD=no ;; --no-build) tui_err "--no-build was renamed to --skip-build" >&2; exit 2 ;; --json) JSON_OUT=true ;; + --install-missing|--no-install) _set_install_flag "$1" ;; # Deploy-target selectors are resolved at startup (they must precede # the build.sbt parse); accept and ignore them here. --worktree=*|--branch=*) selector_seen=true ;; @@ -2459,6 +2724,10 @@ cmd_up() { return $ec fi + # Resolve the interpreter the JVM services hand to Python UDF workers + # before anything is launched with that environment. + _require_udf_python + local n_skip=0 [[ -n "$SKIP_LIST" ]] && n_skip=$(echo "$SKIP_LIST" | tr ',' '\n' | wc -l | tr -d ' ') local skip_label="none" @@ -2619,6 +2888,7 @@ cmd_auto() { case "$1" in --skip=*) SKIP_LIST="${1#--skip=}" ;; --json) JSON_OUT=true ;; + --install-missing|--no-install) _set_install_flag "$1" ;; # Deploy-target selectors are resolved at startup; accept here. --worktree=*|--branch=*) ;; *) tui_err "unknown flag: $1" >&2; exit 2 ;; @@ -2628,6 +2898,8 @@ cmd_auto() { # See cmd_up: human progress to stderr, JSON summary on real stdout (fd 3). if $JSON_OUT; then exec 3>&1 1>&2; fi + _require_udf_python + tui_banner "Texera Local Dev — auto bounce" \ "rebuild + bounce only what changed since last build" if [[ "$REPO_ROOT" != "$SELF_ROOT" ]]; then @@ -2933,6 +3205,8 @@ cmd_up_one() { fi local type="" type=$(amap_get SVC_TYPE "$svc") + # Docker services never launch Python workers; don't pay for the probe. + [[ "$type" == "docker" ]] || _require_udf_python case "$type" in docker) # No source to build — --build degrades to a restart, everything diff --git a/bin/local-dev/tests/test_local_dev_sh.sh b/bin/local-dev/tests/test_local_dev_sh.sh index d1708c69b13..b5b330cbc68 100755 --- a/bin/local-dev/tests/test_local_dev_sh.sh +++ b/bin/local-dev/tests/test_local_dev_sh.sh @@ -839,5 +839,297 @@ else _fail "install hints with no Linux line:$hint_missing" fi +# -------------------------------------------------------------------------- +# Toolchain detection + consented install (#7066). +# Everything below goes through the *decision* helpers, which are pure: they +# print what would be done and never install anything. That separation is the +# reason this is testable in CI at all. +# -------------------------------------------------------------------------- +_extract_fn() { # $1=function name → its definition, or empty + awk -v fn="$1" 'index($0, fn "()") == 1 {f=1} f{print} f && /^}/{exit}' "$MAIN_SH" +} + +# 34) Which package manager we'd use. On Linux the first of apt-get/dnf/yum/ +# pacman/zypper on PATH wins; with none present it must refuse rather than +# emit a bogus name that a caller would then try to run. +pm_fn=$(_extract_fn _pkg_manager) +if [[ -z "$pm_fn" ]]; then + _fail "_pkg_manager helper missing" +else + _pm_dir=$(mktemp -d 2>/dev/null || mktemp -d -t ldpm) + _pm_check() { # $1=label $2=expected ("" = must refuse) $3..=fake tools + local label="$1" expect="$2"; shift 2 + rm -f "$_pm_dir"/* + local t="" + for t in "$@"; do printf '#!/bin/sh\nexit 0\n' > "$_pm_dir/$t"; chmod +x "$_pm_dir/$t"; done + # PATH is stripped to the fakes so the host's real apt/dnf can't answer, + # but _pkg_manager still needs `uname` to know the platform. + ln -sf "$(command -v uname)" "$_pm_dir/uname" + local got="" rc=0 + got=$(PATH="$_pm_dir"; eval "$pm_fn"; _pkg_manager) || rc=$? + if [[ -z "$expect" ]]; then + if (( rc != 0 )) && [[ -z "$got" ]]; then + _pass "pkg manager: $label (refuses)" + else + _fail "pkg manager: $label should refuse" "rc=$rc got='$got'" + fi + elif [[ "$got" == "$expect" ]]; then + _pass "pkg manager: $label → $got" + else + _fail "pkg manager: $label" "expected '$expect' got '$got'" + fi + } + if [[ "$(uname -s)" == "Linux" ]]; then + _pm_check "apt-get" "apt-get" apt-get + _pm_check "dnf only" "dnf" dnf + _pm_check "pacman only" "pacman" pacman + _pm_check "apt-get preferred over dnf" "apt-get" apt-get dnf + _pm_check "nothing installed" "" + else + _pass "skip: package-manager PATH probing is Linux-only on this host" + fi + rm -rf "$_pm_dir" + if [[ "$pm_fn" == *brew* && "$pm_fn" == *Darwin* ]]; then + _pass "_pkg_manager keeps the Darwin/brew branch" + else + _fail "_pkg_manager lost the Darwin/brew branch" + fi +fi + +# 35) The command we'd run per tool. Java goes through the distro package +# manager first with SDKMAN as the no-sudo fallback; node uses nvm; python +# uses pyenv and must pin 3.12, the version AGENTS.md and the CI matrix +# both specify. Unknown or deliberately-unsupported tools must refuse +# instead of printing something a caller would run. +cmd_fn=$(_extract_fn _install_cmd_for) +if [[ -z "$cmd_fn" || -z "$pm_fn" ]]; then + _fail "_install_cmd_for helper missing" +else + _ic_dir=$(mktemp -d 2>/dev/null || mktemp -d -t ldic) + _ic() { # $1=tool $2..=fake pkg managers on PATH → prints the command + local tool="$1"; shift + rm -f "$_ic_dir"/* + local t="" + for t in "$@"; do printf '#!/bin/sh\nexit 0\n' > "$_ic_dir/$t"; chmod +x "$_ic_dir/$t"; done + ln -sf "$(command -v uname)" "$_ic_dir/uname" + ( PATH="$_ic_dir" + TEXERA_NODE_VERSION=24 TEXERA_PYTHON_VERSION=3.12 + eval "$pm_fn"; eval "$cmd_fn"; _install_cmd_for "$tool" ) 2>/dev/null + } + _ic_expect() { # $1=label $2=needle $3=tool $4..=fake managers + local label="$1" needle="$2" tool="$3"; shift 3 + local got="" + got=$(_ic "$tool" "$@") + if [[ "$got" == *"$needle"* ]]; then + _pass "install cmd: $label" + else + _fail "install cmd: $label" "expected to contain '$needle', got '$got'" + fi + } + # Which manager answers is platform-dependent: _pkg_manager's Darwin branch + # only ever looks for brew, so a faked apt-get on PATH is correctly ignored + # there. Assert each platform's own branch. + if [[ "$(uname -s)" == "Linux" ]]; then + _ic_expect "java via apt names openjdk-17-jdk" "openjdk-17-jdk" java apt-get + _ic_expect "java via apt asks for sudo explicitly" "sudo" java apt-get + _ic_expect "java via dnf names java-17-openjdk-devel" "java-17-openjdk-devel" java dnf + else + _ic_expect "java via brew" "brew install openjdk@17" java brew + fi + # Platform-independent: with no manager at all we fall back to SDKMAN. + _ic_expect "java with no package manager falls back to SDKMAN" "sdk" java + _ic_expect "node uses nvm" "nvm install" node apt-get + _ic_expect "python uses pyenv" "pyenv install" python apt-get + _ic_expect "python pins 3.12" "3.12" python apt-get + for tool in docker sbt definitely-not-a-tool; do + got=""; rc=0 + got=$( PATH="$_ic_dir" + TEXERA_NODE_VERSION=24 TEXERA_PYTHON_VERSION=3.12 + eval "$pm_fn"; eval "$cmd_fn"; _install_cmd_for "$tool" 2>/dev/null ) || rc=$? + if (( rc != 0 )) && [[ -z "$got" ]]; then + _pass "install cmd: refuses '$tool'" + else + _fail "install cmd: should refuse '$tool'" "rc=$rc got='$got'" + fi + done + rm -rf "$_ic_dir" +fi + +# 36) Consent. The prompt is for humans; a non-interactive run (CI, cron, an +# agent following AGENTS.md's non-interactive subcommands) must keep the old +# behaviour and never block on a read. This is the most important test in +# this PR: a regression here hangs every automated `up`. +consent_fn=$(_extract_fn _consent_to_install) +if [[ -z "$consent_fn" ]]; then + _fail "_consent_to_install helper missing" +else + rc=0 + ( eval "$consent_fn"; _consent_to_install java "sudo apt-get install -y openjdk-17-jdk" ) \ + /dev/null 2>&1 || rc=$? + if (( rc != 0 )); then + _pass "consent: refuses without a TTY (never prompts)" + else + _fail "consent: granted install without a TTY" + fi + rc=0 + ( TEXERA_INSTALL_MISSING=1; eval "$consent_fn"; _consent_to_install java "cmd" ) \ + /dev/null 2>&1 || rc=$? + if (( rc == 0 )); then + _pass "consent: TEXERA_INSTALL_MISSING=1 assumes yes" + else + _fail "consent: TEXERA_INSTALL_MISSING=1 didn't assume yes" "rc=$rc" + fi + rc=0 + ( TEXERA_INSTALL_MISSING=0; eval "$consent_fn"; _consent_to_install java "cmd" ) \ + /dev/null 2>&1 || rc=$? + if (( rc != 0 )); then + _pass "consent: TEXERA_INSTALL_MISSING=0 refuses" + else + _fail "consent: TEXERA_INSTALL_MISSING=0 didn't refuse" + fi +fi + +# 37) Picking the interpreter that runs Python UDFs. The old default was +# `command -v python3` — the system interpreter, which has none of +# amber/requirements.txt — so UDFs died at worker launch on import errors +# that pointed nowhere near the interpreter choice. +ver_fn=$(_extract_fn _python_version_of) +vok_fn=$(_extract_fn _python_version_ok) +dok_fn=$(_extract_fn _python_deps_ok) +if [[ -z "$ver_fn" || -z "$vok_fn" || -z "$dok_fn" ]]; then + _fail "python probe helpers missing (_python_version_of/_python_version_ok/_python_deps_ok)" +else + _py_dir=$(mktemp -d 2>/dev/null || mktemp -d -t ldpy) + # `python -c ...` is only ever asked for the version string or an import + # check, so scripts that echo a version / set an exit code drive both probes. + printf '#!/bin/sh\necho 3.12\n' > "$_py_dir/py312" + printf '#!/bin/sh\necho 3.11\n' > "$_py_dir/py311" + printf '#!/bin/sh\nexit 0\n' > "$_py_dir/deps-ok" + printf '#!/bin/sh\nexit 1\n' > "$_py_dir/deps-missing" + chmod +x "$_py_dir"/* + _py_probe() { # $1=fn $2=path → rc + local rc=0 + ( TEXERA_PYTHON_VERSION=3.12 + eval "$ver_fn"; eval "$vok_fn"; eval "$dok_fn" + "$1" "$2" ) >/dev/null 2>&1 || rc=$? + return $rc + } + if _py_probe _python_version_ok "$_py_dir/py312"; then + _pass "python probe: 3.12 accepted" + else + _fail "python probe: 3.12 rejected" + fi + if ! _py_probe _python_version_ok "$_py_dir/py311"; then + _pass "python probe: 3.11 rejected" + else + _fail "python probe: 3.11 accepted (must pin 3.12)" + fi + if ! _py_probe _python_version_ok "$_py_dir/nonexistent"; then + _pass "python probe: missing interpreter rejected" + else + _fail "python probe: missing interpreter accepted" + fi + if _py_probe _python_deps_ok "$_py_dir/deps-ok"; then + _pass "python probe: importable deps accepted" + else + _fail "python probe: importable deps rejected" + fi + if ! _py_probe _python_deps_ok "$_py_dir/deps-missing"; then + _pass "python probe: missing deps rejected" + else + _fail "python probe: missing deps accepted" + fi + if ! _py_probe _python_deps_ok ""; then + _pass "python probe: empty path rejected" + else + _fail "python probe: empty path accepted" + fi + rm -rf "$_py_dir" +fi + +# 38) Candidate order. An activated venv must beat the sibling venv312 from +# AGENTS.md's layout, and both must beat whatever `python3` happens to be on +# PATH — that last one being the old, broken default. +cand_fn=$(_extract_fn _udf_python_candidates) +if [[ -z "$cand_fn" ]]; then + _fail "_udf_python_candidates helper missing" +else + _cd_dir=$(mktemp -d 2>/dev/null || mktemp -d -t ldcd) + mkdir -p "$_cd_dir/active/bin" "$_cd_dir/ws/venv312/bin" "$_cd_dir/ws/texera" + : > "$_cd_dir/active/bin/python"; chmod +x "$_cd_dir/active/bin/python" + : > "$_cd_dir/ws/venv312/bin/python"; chmod +x "$_cd_dir/ws/venv312/bin/python" + order=$( VIRTUAL_ENV="$_cd_dir/active" REPO_ROOT="$_cd_dir/ws/texera" \ + SELF_ROOT="$_cd_dir/ws/texera" TEXERA_PYTHON_VERSION=3.12 + eval "$cand_fn"; _udf_python_candidates 2>/dev/null ) + pos_active=$(printf '%s\n' "$order" | grep -n "active/bin/python" | head -1 | cut -d: -f1) + pos_venv=$(printf '%s\n' "$order" | grep -n "venv312/bin/python" | head -1 | cut -d: -f1) + pos_sys=$(printf '%s\n' "$order" | grep -nE "/python3$" | tail -1 | cut -d: -f1) + if [[ -n "$pos_active" && -n "$pos_venv" ]] && (( pos_active < pos_venv )); then + _pass "python candidates: \$VIRTUAL_ENV before sibling venv312" + else + _fail "python candidates: wrong order" \ + "active=$pos_active venv312=$pos_venv order=$(printf '%s' "$order" | tr '\n' '|')" + fi + if [[ -z "$pos_sys" ]] || { [[ -n "$pos_venv" ]] && (( pos_venv < pos_sys )); }; then + _pass "python candidates: venv312 before bare python3 on PATH" + else + _fail "python candidates: bare python3 outranks venv312" \ + "venv312=$pos_venv sys=$pos_sys order=$(printf '%s' "$order" | tr '\n' '|')" + fi + rm -rf "$_cd_dir" +fi + +# 39) The resolver must be wired into the paths that launch services, and must +# NOT run at source time — `status` / `--help` shouldn't pay for spawning +# interpreters. +if [[ -n "$(_extract_fn _require_udf_python)" ]]; then + _pass "_require_udf_python helper is defined" +else + _fail "_require_udf_python helper missing" +fi +for fn in cmd_up cmd_auto cmd_up_one; do + body=$(_extract_fn "$fn") + if [[ "$body" == *"_require_udf_python"* ]]; then + _pass "$fn resolves the UDF interpreter before launching" + else + _fail "$fn doesn't call _require_udf_python" + fi +done +# The UDF interpreter must not be advertised with the `-i` dashboard's hint: +# that one is about textual and amber/dev-requirements.txt, a different +# interpreter and a different requirements file. +# Comments are stripped: the code explains *why* it doesn't use that hint, and +# the explanation naturally names it. +udf_body=$(_extract_fn _require_udf_python | sed 's/^[[:space:]]*#.*$//') +if [[ "$udf_body" != *"_install_hint python"* ]] \ + && [[ "$udf_body" == *"UDF_PYTHON_PATH"* ]] \ + && [[ "$udf_body" == *"amber/requirements.txt"* ]]; then + _pass "_require_udf_python points at amber's requirements, not the TUI hint" +else + _fail "_require_udf_python reuses the TUI python hint or omits amber's requirements" +fi +if ! grep -qE '^export UDF_PYTHON_PATH="\$\{UDF_PYTHON_PATH:-\$\(command -v python3' "$MAIN_SH"; then + _pass "UDF_PYTHON_PATH no longer defaults to bare python3 at source time" +else + _fail "UDF_PYTHON_PATH still defaults to \`command -v python3\` at source time" +fi + +# 40) The new knobs are discoverable, and contradictory ones are refused rather +# than silently resolved one way. +help_out=$("$SCRIPT" --help 2>&1) +if [[ "$help_out" == *"--install-missing"* && "$help_out" == *"--no-install"* ]]; then + _pass "--help documents --install-missing / --no-install" +else + _fail "--help doesn't document the install flags" +fi +_im_state=$(mktemp -d 2>/dev/null || mktemp -d -t ldim) +out=$(TEXERA_LOCAL_DEV_DIR="$_im_state" "$SCRIPT" up --install-missing --no-install 2>&1); rc=$? +if (( rc == 2 )); then + _pass "up rejects --install-missing together with --no-install (rc=2)" +else + _fail "up accepted contradictory install flags" "rc=$rc out=$(echo "$out" | head -1)" +fi +rm -rf "$_im_state" + printf "\n%d passed, %d failed\n" "$PASS" "$FAIL" (( FAIL == 0 ))