Skip to content

feat(local-dev): pick a usable Python and offer to install a missing toolchain #7073

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
Yicong-Huang wants to merge 1 commit into from
+569 −3
Closed

feat(local-dev): pick a usable Python and offer to install a missing toolchain #7073

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
280 changes: 277 additions & 3 deletions bin/local-dev/main.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,7 @@
# bin/local-dev.sh up [service]
# [--fresh|--build|--skip-build] [--skip=svc1,svc2] [--json]
# [--worktree=PATH | --branch=NAME]
# [--install-missing | --no-install]
# Default: skip build if no source/lock
# changes since last build. --build forces
# incremental sbt dist + yarn/bun install.
Expand Down Expand Up @@ -72,11 +73,22 @@
# branch and run its own local-dev.sh
# instead (a warning is printed when such
# drift is detected).
# TOOLCHAIN: a missing JDK 17 / Node /
# Python 3.12 is offered for install
# (distro package manager for the JDK,
# nvm for Node, pyenv for Python) after
# asking. --install-missing answers yes
# without asking, --no-install only ever
# prints the hint. Without a TTY the
# prompt is skipped entirely, so scripted
# and CI runs behave as before. Same flags
# on `auto`.
# bin/local-dev.sh down [service] [--skip=svc1,svc2] [--json]
# stop every non-skipped service, or just
# <service> (--json: summary JSON on
# stdout).
# bin/local-dev.sh auto [--skip=svc1,svc2] [--json]
# [--install-missing | --no-install]
# rebuild + bounce only the services whose
# source changed since the last build.
# bin/local-dev.sh logs <service> tail this service's log.
Expand Down Expand Up @@ -288,6 +300,115 @@ amap_append() {
eval "$_var=\"\${$_var:-}\$_suffix\""
}

# --------- toolchain versions + consented install ---------
# The versions this repo needs (AGENTS.md's table, and the CI matrices).
# Overridable so a contributor can try a newer runtime without editing this.
TEXERA_PYTHON_VERSION="${TEXERA_PYTHON_VERSION:-3.12}"
TEXERA_NODE_VERSION="${TEXERA_NODE_VERSION:-24}"

# This block sits above the JDK probe on purpose: the probe is the first thing
# that can fail on a fresh machine, and it should be able to offer a fix. That
# also means no colour variables here — those are defined further down, with
# the rest of the TUI helpers.

# The package manager we'd install with, or non-zero if we don't recognise one.
_pkg_manager() {
case "$(uname -s 2>/dev/null)" in
Darwin)
command -v brew >/dev/null 2>&1 && { printf 'brew\n'; return 0; }
;;
Linux)
local m=""
for m in apt-get dnf yum pacman zypper; do
command -v "$m" >/dev/null 2>&1 && { printf '%s\n' "$m"; return 0; }
done
;;
esac
return 1
}

# What we would run to install $1. Prints the command and never executes it —
# that split is what lets us show the user the exact command before asking, and
# lets CI assert on the decision without installing anything.
#
# Java prefers the distro package manager: it is system-wide and the distro
# patches it. SDKMAN is the fallback when there's no package manager (or no
# root). Node goes through nvm and Python through pyenv, because the versions
# this repo needs are newer than most distros ship — and both bootstrap
# themselves first if absent.
#
# docker and sbt are deliberately absent. Docker needs a daemon, group
# membership and a re-login; that is not something to do behind a y/n prompt.
_install_cmd_for() {
local tool="${1:-}" pm=""
pm=$(_pkg_manager) || pm=""
case "$tool" in
java)
case "$pm" in
apt-get) printf 'sudo apt-get install -y openjdk-17-jdk\n' ;;
dnf) printf 'sudo dnf install -y java-17-openjdk-devel\n' ;;
yum) printf 'sudo yum install -y java-17-openjdk-devel\n' ;;
pacman) printf 'sudo pacman -S --noconfirm jdk17-openjdk\n' ;;
zypper) printf 'sudo zypper install -y java-17-openjdk-devel\n' ;;
brew) printf 'brew install openjdk@17\n' ;;
*) printf 'curl -s https://get.sdkman.io | bash && sdk install java 17.0.13-tem\n' ;;
esac
;;
node)
if command -v nvm >/dev/null 2>&1 || [[ -s "${NVM_DIR:-$HOME/.nvm}/nvm.sh" ]]; then
printf 'nvm install %s && nvm alias default %s\n' \
"$TEXERA_NODE_VERSION" "$TEXERA_NODE_VERSION"
else
printf 'curl -fsSL https://raw.githubusercontent.com/nvm-sh/nvm/v0.40.3/install.sh | bash && . "$HOME/.nvm/nvm.sh" && nvm install %s && nvm alias default %s\n' \
"$TEXERA_NODE_VERSION" "$TEXERA_NODE_VERSION"
fi
;;
python)
if command -v pyenv >/dev/null 2>&1; then
printf 'pyenv install -s %s\n' "$TEXERA_PYTHON_VERSION"
else
printf 'curl -fsSL https://pyenv.run | bash && export PYENV_ROOT="$HOME/.pyenv" && export PATH="$PYENV_ROOT/bin:$PATH" && pyenv install -s %s\n' \
"$TEXERA_PYTHON_VERSION"
fi
;;
*) return 1 ;;
esac
}

# Ask before installing anything. Never prompts without a TTY: `up` has to stay
# scriptable, and AGENTS.md points agents at the non-interactive subcommands, so
# a non-interactive run keeps the old behaviour of printing a hint and failing.
# --install-missing / TEXERA_INSTALL_MISSING=1 assume yes
# --no-install / TEXERA_INSTALL_MISSING=0 never install
_consent_to_install() {
local tool="${1:-}" cmd="${2:-}" reply=""
case "${TEXERA_INSTALL_MISSING:-ask}" in
1|y|yes|true) return 0 ;;
0|n|no|false) return 1 ;;
esac
[[ -t 0 && -t 1 ]] || return 1
printf "\n %s is missing. I can run:\n\n" "$tool" >&2
printf " %s\n\n" "$cmd" >&2
printf " Run it now? [y/N] " >&2
read -r reply || return 1
[[ "$reply" == [Yy]* ]]
}

# Offer to install $1, then report whether it's actually there afterwards.
# Runs through `bash -lc` so shell functions like nvm / pyenv / sdk resolve.
_offer_install() {
local tool="${1:-}" cmd=""
cmd=$(_install_cmd_for "$tool") || return 1
_consent_to_install "$tool" "$cmd" || return 1
printf " installing %s ...\n" "$tool" >&2
if ! bash -lc "$cmd" >&2; then
printf " %s: install command failed\n" "$tool" >&2
return 1
fi
printf " %s: installed\n" "$tool" >&2
return 0
}

# --------- toolchain (JDK 17 + node) ---------
# Detect a JDK 17 installation rather than pinning one path. We try, in
# order: (1) caller-set $JAVA_HOME if it really is 17, (2) macOS's official
Expand Down Expand Up @@ -423,7 +544,13 @@ _diagnose_jdk17() {
echo "" >&2
}

JAVA_HOME_DETECTED="$(_find_jdk17)" || {
JAVA_HOME_DETECTED="$(_find_jdk17)" || JAVA_HOME_DETECTED=""
# Offer to install it rather than only describing how. Declines, and every
# non-interactive run, fall through to the hint-and-exit below unchanged.
if [[ -z "$JAVA_HOME_DETECTED" ]] && _offer_install java; then
JAVA_HOME_DETECTED="$(_find_jdk17)" || JAVA_HOME_DETECTED=""
fi
if [[ -z "$JAVA_HOME_DETECTED" ]]; then
echo "FATAL: could not find a JDK 17 install." >&2
_diagnose_jdk17
echo " fix:" >&2
Expand All @@ -433,7 +560,7 @@ JAVA_HOME_DETECTED="$(_find_jdk17)" || {
echo " or set JAVA_HOME=/path/to/jdk-17 explicitly" >&2
echo "" >&2
exit 1
}
fi
export JAVA_HOME="$JAVA_HOME_DETECTED"
export PATH="$JAVA_HOME/bin:$PATH"

Expand Down Expand Up @@ -565,7 +692,10 @@ export STORAGE_LAKEFS_ENDPOINT="${STORAGE_LAKEFS_ENDPOINT:-http://localhost:8000
export STORAGE_LAKEFS_AUTH_USERNAME="${STORAGE_LAKEFS_AUTH_USERNAME:-AKIAIOSFOLKFSSAMPLES}"
export STORAGE_LAKEFS_AUTH_PASSWORD="${STORAGE_LAKEFS_AUTH_PASSWORD:-wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY}"
export STORAGE_LAKEFS_AUTH_API_SECRET="${STORAGE_LAKEFS_AUTH_API_SECRET:-random_string_for_lakefs}"
export UDF_PYTHON_PATH="${UDF_PYTHON_PATH:-$(command -v python3 2>/dev/null || command -v python 2>/dev/null)}"
# UDF_PYTHON_PATH is resolved lazily by _require_udf_python (see below) rather
# than defaulted to `command -v python3` here: the system interpreter almost
# never has amber/requirements.txt installed, and picking it silently turned a
# toolchain problem into import errors inside a Python worker.
export TEXERA_DASHBOARD_SERVICE_ENDPOINT="${TEXERA_DASHBOARD_SERVICE_ENDPOINT:-http://localhost:8080}"
export WORKFLOW_COMPILING_SERVICE_ENDPOINT="${WORKFLOW_COMPILING_SERVICE_ENDPOINT:-http://localhost:9090}"
export WORKFLOW_EXECUTION_SERVICE_ENDPOINT="${WORKFLOW_EXECUTION_SERVICE_ENDPOINT:-http://localhost:8085}"
Expand Down Expand Up @@ -1243,6 +1373,140 @@ _diagnose_node() {
printf "\n"
}

# --------- python for UDF workers ---------
# "3.12" for an interpreter, or non-zero if it can't be run at all.
_python_version_of() {
local py="${1:-}"
[[ -n "$py" && -x "$py" ]] || return 1
"$py" -c 'import sys; print("%d.%d" % sys.version_info[:2])' 2>/dev/null
}

_python_version_ok() {
[[ "$(_python_version_of "${1:-}" 2>/dev/null)" == "$TEXERA_PYTHON_VERSION" ]]
}

# Can this interpreter actually run a Python UDF worker? pyarrow, pandas and
# loguru are the heavy three from amber/requirements.txt — if they import, the
# environment was populated.
_python_deps_ok() {
local py="${1:-}"
[[ -n "$py" && -x "$py" ]] || return 1
"$py" -c 'import pyarrow, pandas, loguru' >/dev/null 2>&1
}

# Interpreters that might run UDF workers, best first. Prints paths only;
# suitability is the caller's business. Deduplicated, because the sibling-venv
# and pyenv guesses overlap on some layouts.
_udf_python_candidates() {
local seen="" base="" root="" d=""
# Collapse `..` segments so the path we print and export is the one a
# contributor would recognise. `realpath` isn't portable to a stock macOS,
# and only the directory is resolved — the final component stays a symlink,
# which is what we want for a venv's bin/python.
_abspath() {
local dir="" leaf=""
dir=$(dirname "$1"); leaf=$(basename "$1")
[[ -d "$dir" ]] || { printf '%s\n' "$1"; return 0; }
printf '%s/%s\n' "$(cd "$dir" && pwd -P)" "$leaf"
}
_emit() { [[ -n "${1:-}" ]] || return 0; local p=""; p=$(_abspath "$1"); case "$seen" in *"|$p|"*) return 0 ;; esac; seen="$seen|$p|"; printf '%s\n' "$p"; }
# An activated venv is the most explicit statement of intent available.
[[ -n "${VIRTUAL_ENV:-}" ]] && _emit "$VIRTUAL_ENV/bin/python"
# AGENTS.md's layout: <workspace>/{texera, texera-worktrees/<branch>, venv312}
for base in "$REPO_ROOT/.." "$REPO_ROOT/../.." "$SELF_ROOT/.." "$SELF_ROOT/../.."; do
_emit "$base/venv312/bin/python"
done
if command -v pyenv >/dev/null 2>&1; then
root=$(pyenv root 2>/dev/null) || root="$HOME/.pyenv"
while IFS= read -r d; do
[[ -n "$d" ]] && _emit "$d/bin/python"
done < <(shopt -s nullglob; printf '%s\n' "$root/versions/$TEXERA_PYTHON_VERSION"*)
fi
_emit "$(command -v "python$TEXERA_PYTHON_VERSION" 2>/dev/null || true)"
_emit "$(command -v python3 2>/dev/null || true)"
}

# Lazy resolver, mirroring _require_host_lan_ip: only the subcommands that
# actually launch services pay for it, so `status` / `logs` / `--help` don't
# spawn an interpreter per candidate.
#
# Never fatal. The JVM services run fine without a Python toolchain; only
# Python UDFs don't. So this warns, offers a fix, and carries on.
_require_udf_python() {
[[ -n "${_UDF_PYTHON_RESOLVED:-}" ]] && return 0
_UDF_PYTHON_RESOLVED=1
# An explicit UDF_PYTHON_PATH always wins — it's the documented override —
# but say so if it can't import what a worker needs.
if [[ -n "${UDF_PYTHON_PATH:-}" ]]; then
if ! _python_deps_ok "$UDF_PYTHON_PATH"; then
tui_warn "python: UDF_PYTHON_PATH=$UDF_PYTHON_PATH can't import amber's deps"
tui_warn "python: Python UDFs will fail at worker launch"
fi
export UDF_PYTHON_PATH
return 0
fi
local c="" no_deps=""
while IFS= read -r c; do
[[ -n "$c" ]] || continue
_python_version_ok "$c" || continue
if _python_deps_ok "$c"; then
export UDF_PYTHON_PATH="$c"
tui_ok "python: $c ${DIM}(runs Python UDFs)${RESET}"
return 0
fi
[[ -z "$no_deps" ]] && no_deps="$c"
done < <(_udf_python_candidates)
# Distinguish "no 3.12 anywhere" from "3.12 without amber's deps": they need
# different fixes, and the old default silently picked the second one.
if [[ -n "$no_deps" ]]; then
tui_warn "python: $no_deps is $TEXERA_PYTHON_VERSION but can't import amber's deps"
printf " ${BOLD}populate it:${RESET}\n"
printf " %s -m pip install -r %s/amber/requirements.txt -r %s/amber/operator-requirements.txt\n" \
"$no_deps" "$REPO_ROOT" "$REPO_ROOT"
export UDF_PYTHON_PATH="$no_deps"
return 0
fi
tui_warn "python: no Python $TEXERA_PYTHON_VERSION found — Python UDFs will not run"
# Deliberately not `_install_hint python`: that hint is about the `-i`
# dashboard's textual dependency, which is a different interpreter and a
# different requirements file.
printf " ${BOLD}looked in:${RESET} \$VIRTUAL_ENV, <workspace>/venv312, pyenv, PATH\n"
printf " ${BOLD}or point at one yourself:${RESET} export UDF_PYTHON_PATH=/path/to/python%s\n" \
"$TEXERA_PYTHON_VERSION"
if _offer_install python; then
while IFS= read -r c; do
[[ -n "$c" ]] || continue
if _python_version_ok "$c"; then
export UDF_PYTHON_PATH="$c"
tui_ok "python: $c"
tui_warn "python: now install amber's deps into it:"
printf " %s -m pip install -r %s/amber/requirements.txt -r %s/amber/operator-requirements.txt\n" \
"$c" "$REPO_ROOT" "$REPO_ROOT"
return 0
fi
done < <(_udf_python_candidates)
fi
return 0
}

# Translate --install-missing / --no-install into TEXERA_INSTALL_MISSING. The
# flags beat the environment variable; the contradiction is refused rather than
# silently resolved one way.
_set_install_flag() {
local want=""
case "${1:-}" in
--install-missing) want=1 ;;
--no-install) want=0 ;;
*) return 0 ;;
esac
if [[ -n "${_INSTALL_FLAG_SEEN:-}" && "${_INSTALL_FLAG_SEEN}" != "$want" ]]; then
tui_err "--install-missing and --no-install are mutually exclusive" >&2
exit 2
fi
_INSTALL_FLAG_SEEN="$want"
export TEXERA_INSTALL_MISSING="$want"
}

# --------- helpers ---------
# PID of whatever is listening on a TCP port, or nothing. Always exits 0 —
# "empty output" is the contract for "nothing is listening", and callers
Expand Down Expand Up @@ -2427,6 +2691,7 @@ cmd_up() {
--skip-build) BUILD=no ;;
--no-build) tui_err "--no-build was renamed to --skip-build" >&2; exit 2 ;;
--json) JSON_OUT=true ;;
--install-missing|--no-install) _set_install_flag "$1" ;;
# Deploy-target selectors are resolved at startup (they must precede
# the build.sbt parse); accept and ignore them here.
--worktree=*|--branch=*) selector_seen=true ;;
Expand Down Expand Up @@ -2459,6 +2724,10 @@ cmd_up() {
return $ec
fi

# Resolve the interpreter the JVM services hand to Python UDF workers
# before anything is launched with that environment.
_require_udf_python

local n_skip=0
[[ -n "$SKIP_LIST" ]] && n_skip=$(echo "$SKIP_LIST" | tr ',' '\n' | wc -l | tr -d ' ')
local skip_label="none"
Expand Down Expand Up @@ -2619,6 +2888,7 @@ cmd_auto() {
case "$1" in
--skip=*) SKIP_LIST="${1#--skip=}" ;;
--json) JSON_OUT=true ;;
--install-missing|--no-install) _set_install_flag "$1" ;;
# Deploy-target selectors are resolved at startup; accept here.
--worktree=*|--branch=*) ;;
*) tui_err "unknown flag: $1" >&2; exit 2 ;;
Expand All @@ -2628,6 +2898,8 @@ cmd_auto() {
# See cmd_up: human progress to stderr, JSON summary on real stdout (fd 3).
if $JSON_OUT; then exec 3>&1 1>&2; fi

_require_udf_python

tui_banner "Texera Local Dev — auto bounce" \
"rebuild + bounce only what changed since last build"
if [[ "$REPO_ROOT" != "$SELF_ROOT" ]]; then
Expand Down Expand Up @@ -2933,6 +3205,8 @@ cmd_up_one() {
fi
local type=""
type=$(amap_get SVC_TYPE "$svc")
# Docker services never launch Python workers; don't pay for the probe.
[[ "$type" == "docker" ]] || _require_udf_python
case "$type" in
docker)
# No source to build — --build degrades to a restart, everything
Expand Down
Loading
Loading