Handle shared installer venvs by Python/platform compatibility

.github/workflows/ai-pr-review.yml

@@ -72,10 +72,10 @@ jobs:
                   return text
 
               lines = text.splitlines()
-              if lines:
-                  lines = lines[1:]
-              if lines and lines[-1].strip() == "```":
-                  lines = lines[:-1]
+              if len(lines) < 2 or lines[-1].strip() != "```":
+                  return text
+
+              lines = lines[1:-1]
 
               return "\n".join(lines).strip()
 
@@ -143,26 +143,72 @@ jobs:
 
       - name: Post PR summary comment
         if: env.EMPTY_DIFF != 'true'
+        shell: bash
         run: |
+          if [ ! -f ai_output.json ] || ! jq -e . ai_output.json >/dev/null; then
+            echo "::warning::Skipping AI PR summary comment because ai_output.json is missing or invalid"
+            exit 0
+          fi
+
           SUMMARY=$(jq -r '.summary' ai_output.json)
+          if [ -z "$SUMMARY" ] || [ "$SUMMARY" = "null" ]; then
+            SUMMARY="AI review did not provide a summary."
+          fi
+          PR_SUMMARY_BODY=$(printf '%s\n\n%s' '## 🤖 AI Peer Review' "$SUMMARY")
+
           gh pr comment ${{ github.event.pull_request.number }} \
-            --body "## 🤖 AI Peer Review\n\n$SUMMARY"
+            --body "$PR_SUMMARY_BODY"
         env:
           GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Post inline review comments
         if: env.EMPTY_DIFF != 'true'
         continue-on-error: true
+        shell: bash
         run: |
-          jq -c '.comments[]?' ai_output.json | while read c; do
-            FILE=$(echo $c | jq -r '.file')
-            LINE=$(echo $c | jq -r '.line')
-            COMMENT=$(echo $c | jq -r '.comment')
+          decode_field() {
+            python3 -c 'import base64, sys; exec("try:\n sys.stdout.write(base64.b64decode(sys.argv[1]).decode())\nexcept Exception as exc:\n print(f\"decode_error:{exc}\", file=sys.stderr); sys.exit(1)")' "$1"
+          }
+
+          if [ ! -f ai_output.json ] || ! jq -e . ai_output.json >/dev/null; then
+            echo "::warning::Skipping AI inline review comments because ai_output.json is missing or invalid"
+            exit 0
+          fi
+
+          while IFS= read -r c; do
+            # Base64-encode fields during extraction so multiline markdown,
+            # quotes, and other special characters survive shell parsing.
+            if ! ENCODED_PAYLOAD=$(jq -r '[.file, (.line | tostring), (.comment // "")] | map(@base64)[]' <<<"$c"); then
+              echo "::warning::Skipping malformed AI review comment payload: jq extraction failed"
+              continue
+            fi
+            mapfile -t ENCODED_FIELDS <<<"$ENCODED_PAYLOAD"
+            if [ "${#ENCODED_FIELDS[@]}" -ne 3 ]; then
+              echo "::warning::Skipping malformed AI review comment payload: expected 3 fields, got ${#ENCODED_FIELDS[@]}"
+              continue
+            fi
+
+            if ! FILE=$(decode_field "${ENCODED_FIELDS[0]}"); then
+              echo "::warning::Skipping AI review comment payload: failed to decode file field"
+              continue
+            fi
+            if ! LINE=$(decode_field "${ENCODED_FIELDS[1]}"); then
+              echo "::warning::Skipping AI review comment payload: failed to decode line field"
+              continue
+            fi
+            if ! COMMENT=$(decode_field "${ENCODED_FIELDS[2]}"); then
+              echo "::warning::Skipping AI review comment payload: failed to decode comment field"
+              continue
+            fi
 
             if ! [[ "$LINE" =~ ^[0-9]+$ ]]; then
               echo "::warning::Skipping AI review comment with invalid line number for $FILE: $LINE"
               continue
             fi
+            if [ -z "$COMMENT" ] || [ "$COMMENT" = "null" ]; then
+              echo "::warning::Skipping AI review comment with empty body for $FILE:$LINE"
+              continue
+            fi
 
             gh api \
               repos/${{ github.repository }}/pulls/${{ github.event.pull_request.number }}/comments \
@@ -172,6 +218,6 @@ jobs:
               -F line="$LINE" \
               -f side="RIGHT" \
               -f subject_type="line" || echo "::warning::Failed to post AI inline review comment for $FILE:$LINE"
-          done
+          done < <(jq -c '.comments[]?' ai_output.json)
         env:
           GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/test-installer-curl.yml

@@ -61,13 +61,13 @@ jobs:
 
           DEFAULT_VENV_DIR="$HOME/mlcflow"
           VENV_DIR="$DEFAULT_VENV_DIR"
-          PY_MAJOR_MINOR="$(python3 -c 'import sys; print("{}.{}".format(*sys.version_info[:2]))')"
-          PY_ARCH="$(python3 -c 'import platform; print(platform.machine())')"
+          PY_MAJOR_MINOR="$(get_python_major_minor_version)"
+          PY_ARCH="$(normalize_architecture)"
 
           # Simulate an existing incompatible default venv and verify suffix fallback
           mkdir -p "$DEFAULT_VENV_DIR"
           resolve_default_venv_dir
-          EXPECTED_SUFFIX="${DEFAULT_VENV_DIR}_${PY_ARCH}_py${PY_MAJOR_MINOR}"
+          EXPECTED_SUFFIX="${DEFAULT_VENV_DIR}$(get_venv_suffix)"
           if [ "$VENV_DIR" != "$EXPECTED_SUFFIX" ]; then
             echo "Expected fallback venv path: $EXPECTED_SUFFIX"
             echo "Actual venv path: $VENV_DIR"

docs/install/README.md

@@ -257,6 +257,14 @@ deactivate
 - **Automation Repository**: `~/MLC/repos/mlcommons@mlperf-automations/`
 - **MLC Cache**: `~/MLC/repos/`
 
+If the requested virtual environment directory already exists but was created
+for a different platform or Python minor version, the installer will keep the
+existing directory untouched and create or reuse a compatible sibling such as
+`~/mlcflow_x86_64_py3.11` or `~/mlcflow_aarch64_py3.12`. The suffix always
+matches the current platform and Python minor version, so rerunning the
+installer on the same platform will reuse that sibling path when it is
+compatible.
+
 ## Troubleshooting
 
 ### "Python version < 3.7"

docs/install/mlcflow_unix_installer.sh

@@ -20,6 +20,7 @@ MIN_PYTHON_VERSION="3.7"
 DEFAULT_VENV_DIR="$HOME/mlcflow"
 DEFAULT_REPO="mlcommons@mlperf-automations"
 DEFAULT_BRANCH="dev"
+PYTHON_CMD="python3"
 
 UPGRADE=false
 ASSUME_YES=false
@@ -205,11 +206,11 @@ require_root_if_needed() {
 }
 
 have_pip_module() {
-    python3 -m pip --version >/dev/null 2>&1
+    "$PYTHON_CMD" -m pip --version >/dev/null 2>&1
 }
 
 have_venv_module() {
-    python3 -c 'import venv' >/dev/null 2>&1
+    "$PYTHON_CMD" -c 'import venv' >/dev/null 2>&1
 }
 
 check_missing_dependencies() {
@@ -264,19 +265,19 @@ version_ge() {
 }
 
 ensure_python() {
-    if ! command -v python3 >/dev/null 2>&1; then
+    if ! command -v "$PYTHON_CMD" >/dev/null 2>&1; then
         log_warn "Python3 not found."
         handle_python_install
     fi
 
-    if ! command -v python3 >/dev/null 2>&1; then
+    if ! command -v "$PYTHON_CMD" >/dev/null 2>&1; then
         log_error "Python3 is still unavailable after attempted installation."
         exit 1
     fi
 
-    PY_VERSION=$(python3 -c 'import sys; print(".".join(map(str, sys.version_info[:3])))')
-    PY_MAJOR_MINOR=$(python3 -c 'import sys; print("{}.{}".format(*sys.version_info[:2]))')
-    PY_ARCH=$(python3 -c 'import platform; print(platform.machine())')
+    PY_VERSION=$("$PYTHON_CMD" -c 'import sys; print(".".join(map(str, sys.version_info[:3])))')
+    PY_MAJOR_MINOR="$(get_python_major_minor_version)"
+    PY_ARCH="$(normalize_architecture "$("$PYTHON_CMD" -c 'import platform; print(platform.machine())')")"
     log_info "Detected Python version: $PY_VERSION"
     log_debug "Detected Python major.minor: $PY_MAJOR_MINOR"
     log_debug "Detected Python architecture: $PY_ARCH"
@@ -328,11 +329,42 @@ handle_python_install() {
 # Virtual Environment
 # ------------------------------------------------------------------------------
 
-is_venv_compatible() {
+normalize_architecture() {
+    local architecture="${1:-$(uname -m)}"
+
+    case "$architecture" in
+        x86_64|amd64)
+            echo "x86_64"
+            ;;
+        aarch64|arm64)
+            echo "aarch64"
+            ;;
+        *)
+            echo "$architecture"
+            ;;
+    esac
+}
+
+get_python_major_minor_version() {
+    "$PYTHON_CMD" -c 'import sys; print("{}.{}".format(sys.version_info[0], sys.version_info[1]))'
+}
+
+# $1 should be a Python executable (for example python3 or <venv>/bin/python).
+# Returns a compatibility signature in the format architecture|major.minor
+# (for example x86_64|3.11).
+get_python_compatibility_signature() {
+    "$1" -c 'import platform, sys; print("{}|{}.{}".format(platform.machine(), sys.version_info[0], sys.version_info[1]))' 2>/dev/null
+}
+
+get_venv_suffix() {
+    "$PYTHON_CMD" -c 'import platform, sys; print("_{}_py{}.{}".format(platform.machine(), sys.version_info[0], sys.version_info[1]))'
+}
+
+is_compatible_venv() {
     local venv_dir="$1"
-    local expected_py_major_minor="$2"
-    local expected_arch="$3"
     local venv_python="$venv_dir/bin/python3"
+    local current_signature
+    local venv_signature
 
     if [ ! -x "$venv_python" ]; then
         venv_python="$venv_dir/bin/python"
@@ -341,51 +373,83 @@ is_venv_compatible() {
         return 1
     fi
 
-    local venv_py_major_minor
-    local venv_arch
-    venv_py_major_minor=$("$venv_python" -c 'import sys; print("{}.{}".format(*sys.version_info[:2]))' 2>/dev/null) || return 1
-    venv_arch=$("$venv_python" -c 'import platform; print(platform.machine())' 2>/dev/null) || return 1
+    if ! current_signature="$(get_python_compatibility_signature "$PYTHON_CMD")"; then
+        log_debug "Failed to determine compatibility signature for $PYTHON_CMD."
+        return 1
+    fi
+    if ! venv_signature="$(get_python_compatibility_signature "$venv_python")"; then
+        log_debug "Failed to determine compatibility signature for $venv_python."
+        return 1
+    fi
 
-    [ "$venv_py_major_minor" = "$expected_py_major_minor" ] && [ "$venv_arch" = "$expected_arch" ]
+    [ "$venv_signature" = "$current_signature" ]
 }
 
-resolve_default_venv_dir() {
-    local shared_venv_dir="${DEFAULT_VENV_DIR}_${PY_ARCH}_py${PY_MAJOR_MINOR}"
+resolve_venv_dir() {
+    local requested_venv_dir="$1"
+    local resolved_venv_dir="$requested_venv_dir"
+    local compatible_venv_dir="${requested_venv_dir}$(get_venv_suffix)"
 
-    if [ -d "$DEFAULT_VENV_DIR" ]; then
-        if is_venv_compatible "$DEFAULT_VENV_DIR" "$PY_MAJOR_MINOR" "$PY_ARCH"; then
-            log_info "Reusing default virtual environment: $DEFAULT_VENV_DIR"
-            VENV_DIR="$DEFAULT_VENV_DIR"
+    if [ -d "$requested_venv_dir" ]; then
+        if is_compatible_venv "$requested_venv_dir"; then
+            echo "$resolved_venv_dir"
             return
         fi
 
-        log_warn "Default virtual environment is incompatible with Python ${PY_MAJOR_MINOR} (${PY_ARCH})."
-        VENV_DIR="$shared_venv_dir"
-        if [ -d "$shared_venv_dir" ] && ! is_venv_compatible "$shared_venv_dir" "$PY_MAJOR_MINOR" "$PY_ARCH"; then
-            log_warn "Removing stale/incompatible virtual environment: $shared_venv_dir"
-            rm -rf "$shared_venv_dir"
+        resolved_venv_dir="$compatible_venv_dir"
+        if [ -d "$resolved_venv_dir" ] && ! is_compatible_venv "$resolved_venv_dir"; then
+            log_warn "Removing stale/incompatible virtual environment: $resolved_venv_dir" >&2
+            rm -rf "$resolved_venv_dir"
         fi
-        log_info "Using platform/python-specific virtual environment: $VENV_DIR"
+        echo "$resolved_venv_dir"
         return
     fi
 
-    if [ -d "$shared_venv_dir" ] && is_venv_compatible "$shared_venv_dir" "$PY_MAJOR_MINOR" "$PY_ARCH"; then
-        VENV_DIR="$shared_venv_dir"
-        log_info "Reusing platform/python-specific virtual environment: $VENV_DIR"
+    if [ -d "$compatible_venv_dir" ] && is_compatible_venv "$compatible_venv_dir"; then
+        resolved_venv_dir="$compatible_venv_dir"
     fi
+
+    echo "$resolved_venv_dir"
 }
 
-setup_venv() {
-    if [ "$VENV_DIR" = "$DEFAULT_VENV_DIR" ]; then
-        resolve_default_venv_dir
+is_venv_compatible() {
+    is_compatible_venv "$1"
+}
+
+resolve_default_venv_dir() {
+    local shared_venv_dir="${DEFAULT_VENV_DIR}$(get_venv_suffix)"
+
+    VENV_DIR="$(resolve_venv_dir "$DEFAULT_VENV_DIR")"
+
+    if [ "$VENV_DIR" = "$DEFAULT_VENV_DIR" ] && [ -d "$DEFAULT_VENV_DIR" ]; then
+        log_info "Reusing default virtual environment: $DEFAULT_VENV_DIR"
+        return
     fi
 
+    if [ "$VENV_DIR" != "$DEFAULT_VENV_DIR" ]; then
+        if [ -d "$DEFAULT_VENV_DIR" ]; then
+            log_warn "Default virtual environment is incompatible with Python ${PY_MAJOR_MINOR} (${PY_ARCH})."
+        fi
+        if [ "$VENV_DIR" = "$shared_venv_dir" ] && [ -d "$shared_venv_dir" ]; then
+            log_info "Reusing platform/python-specific virtual environment: $VENV_DIR"
+        else
+            log_info "Using platform/python-specific virtual environment: $VENV_DIR"
+        fi
+    fi
+}
+
+setup_venv() {
+    local requested_venv_dir="$VENV_DIR"
+    VENV_DIR="$(resolve_venv_dir "$VENV_DIR")"
     log_info "Setting up virtual environment at: $VENV_DIR"
+    if [ "$VENV_DIR" != "$requested_venv_dir" ]; then
+        log_warn "Virtual environment at $requested_venv_dir is incompatible with current Python/platform. Using $VENV_DIR instead."
+    fi
 
-    if [ -d "$VENV_DIR" ]; then
+    if [ -d "$VENV_DIR" ] && is_compatible_venv "$VENV_DIR"; then
         log_info "Reusing existing virtual environment."
     else
-        python3 -m venv "$VENV_DIR"
+        "$PYTHON_CMD" -m venv "$VENV_DIR"
     fi
 
     # Activate venv
@@ -398,16 +462,16 @@ setup_venv() {
 # ------------------------------------------------------------------------------
 
 install_mlcflow() {
-    if python3 -m pip show mlcflow >/dev/null 2>&1; then
+    if "$PYTHON_CMD" -m pip show mlcflow >/dev/null 2>&1; then
         if $UPGRADE; then
             log_info "Upgrading mlcflow..."
-            python3 -m pip install --upgrade mlcflow
+            "$PYTHON_CMD" -m pip install --upgrade mlcflow
         else
             log_info "mlcflow already installed. Skipping."
         fi
     else
         log_info "Installing mlcflow..."
-        python3 -m pip install mlcflow
+        "$PYTHON_CMD" -m pip install mlcflow
     fi
 }