diff --git a/electron/main/python-bridge.ts b/electron/main/python-bridge.ts index 88b097b..dde2d66 100644 --- a/electron/main/python-bridge.ts +++ b/electron/main/python-bridge.ts @@ -5,6 +5,7 @@ import { existsSync, mkdirSync } from 'fs' import axios from 'axios' import { getSettings } from './settings-store' import { logger } from './logger' +import { cleanPythonEnv, getVenvPythonExe } from './python-setup' const API_PORT = 8765 const API_HOST = '127.0.0.1' @@ -21,12 +22,8 @@ export class PythonBridge { } async start(): Promise { - // Already fully ready if (this.ready) return - - // Startup already in progress — wait for the same promise instead of spawning again if (this.startPromise) return this.startPromise - this.startPromise = this._start() try { await this.startPromise @@ -37,7 +34,6 @@ export class PythonBridge { private async _start(): Promise { if (this.process) { - // Process spawned but not ready yet (e.g. second concurrent call) — just wait await this.waitUntilReady() return } @@ -53,9 +49,9 @@ export class PythonBridge { this.process = spawn(pythonExecutable, ['-m', 'uvicorn', 'main:app', '--host', API_HOST, '--port', String(API_PORT)], { cwd: apiDir, env: { - ...process.env, + ...cleanPythonEnv(), PYTHONUNBUFFERED: '1', - PYTHONPATH: this.resolveDependenciesDir(), + // No PYTHONPATH needed — the venv's Python has its own isolated site-packages MODELS_DIR: this.resolveModelsDir(), WORKSPACE_DIR: this.resolveWorkspaceDir(), EXTENSIONS_DIR: this.resolveExtensionsDir(), @@ -83,20 +79,18 @@ export class PythonBridge { this.ready = false this.process = null if (wasReady) { - // Server crashed while running — notify the renderer so it stops making API calls this.getWindow()?.webContents.send('python:crashed', { code }) } }) await this.waitUntilReady() - } // ← end of _start() + } async stop(): Promise { if (!this.process) return const proc = this.process this.process = null this.ready = false - // On Windows, taskkill /T kills the process AND all its children if (process.platform === 'win32') { const { execSync } = require('child_process') try { execSync(`taskkill /PID ${proc.pid} /T /F`) } catch {} @@ -107,20 +101,13 @@ export class PythonBridge { } private emitTqdmLog(raw: string): void { - // Skip uvicorn HTTP access logs and Python INFO logger lines if (/INFO/.test(raw)) return - // Skip empty lines if (!raw.trim()) return this.getWindow()?.webContents.send('python:log', raw.trim()) } - isReady(): boolean { - return this.ready - } - - getPort(): number { - return API_PORT - } + isReady(): boolean { return this.ready } + getPort(): number { return API_PORT } private async killProcessOnPort(): Promise { const { execSync } = require('child_process') @@ -130,43 +117,29 @@ export class PythonBridge { return } - // Retry loop — after a kill the port may take a few ms to be released for (let attempt = 0; attempt < 3; attempt++) { let output = '' try { - // ":8765 " with trailing space avoids matching :87650, :87651, etc. - output = execSync( - `netstat -ano | findstr ":${API_PORT} "`, - { encoding: 'utf8', shell: true } - ) as string - } catch { - break // findstr returns exit code 1 when no match — port is free - } + output = execSync(`netstat -ano | findstr ":${API_PORT} "`, { encoding: 'utf8', shell: true }) as string + } catch { break } const pids = new Set() for (const line of output.split('\n')) { const match = line.trim().match(/\s+(\d+)$/) if (match && match[1] !== '0') pids.add(match[1]) } - if (pids.size === 0) break for (const pid of pids) { - try { - execSync(`taskkill /PID ${pid} /T /F`, { shell: true }) - console.log(`[PythonBridge] Killed process tree PID ${pid} on port ${API_PORT}`) - } catch {} + try { execSync(`taskkill /PID ${pid} /T /F`, { shell: true }) } catch {} } - await new Promise((r) => setTimeout(r, 300)) } } private async waitUntilReady(maxRetries = 180, delayMs = 500): Promise { for (let i = 0; i < maxRetries; i++) { - if (!this.process) { - throw new Error('FastAPI process exited unexpectedly during startup') - } + if (!this.process) throw new Error('FastAPI process exited unexpectedly during startup') try { await axios.get(`${API_BASE_URL}/health`, { timeout: 2000 }) this.ready = true @@ -180,35 +153,31 @@ export class PythonBridge { } private resolvePythonExecutable(): string { - const apiDir = this.resolveApiDir() - const resourcesPath = app.isPackaged - ? process.resourcesPath - : join(app.getAppPath(), 'resources') const userData = app.getPath('userData') + const apiDir = this.resolveApiDir() - const candidates = [ - join(resourcesPath, 'python-embed', 'python.exe'), // Windows embedded (dev + packaged) - join(userData, 'venv', 'bin', 'python'), // Linux/macOS venv - join(apiDir, '.venv', 'Scripts', 'python.exe'), // legacy dev fallback - join(apiDir, '.venv', 'bin', 'python'), // legacy dev fallback - 'python3', - 'python', - ] + // Primary: venv created during setup (bundled Python → isolated venv) + const venvPython = getVenvPythonExe(userData) + if (existsSync(venvPython)) return venvPython - for (const candidate of candidates) { - if (existsSync(candidate)) { - return candidate - } + // Dev fallback: local .venv in the api directory + const devCandidates = [ + join(apiDir, '.venv', 'Scripts', 'python.exe'), + join(apiDir, '.venv', 'bin', 'python'), + ] + for (const c of devCandidates) { + if (existsSync(c)) return c } - return process.platform === 'win32' ? 'python' : 'python3' + // Never fall back to bare 'python' on Windows — it would be the user's system Python + if (process.platform === 'win32') { + throw new Error('Python venv not found. Please restart the application to re-run setup.') + } + return 'python3' } private resolveApiDir(): string { - if (app.isPackaged) { - return join(process.resourcesPath, 'api') - } - // In dev, app.getAppPath() returns the desktop/ folder + if (app.isPackaged) return join(process.resourcesPath, 'api') return join(app.getAppPath(), 'api') } @@ -229,11 +198,4 @@ export class PythonBridge { mkdirSync(s.extensionsDir, { recursive: true }) return s.extensionsDir } - - private resolveDependenciesDir(): string { - const s = getSettings(app.getPath('userData')) - mkdirSync(s.dependenciesDir, { recursive: true }) - return s.dependenciesDir - } - } diff --git a/electron/main/python-setup.ts b/electron/main/python-setup.ts index cd0d7ee..197a001 100644 --- a/electron/main/python-setup.ts +++ b/electron/main/python-setup.ts @@ -1,12 +1,11 @@ import { BrowserWindow, app } from 'electron' -import { existsSync, readFileSync, writeFileSync, readdirSync, mkdirSync } from 'fs' +import { existsSync, readFileSync, writeFileSync } from 'fs' import { join } from 'path' import { spawn, execSync } from 'child_process' import { createHash } from 'crypto' import { getSettings } from './settings-store' -const SETUP_VERSION = 2 -const TOTAL_PACKAGES = 20 +const SETUP_VERSION = 3 interface SetupJson { version: number @@ -28,13 +27,54 @@ function hashRequirements(): string { } } -// ─── Public helpers ────────────────────────────────────────────────────────── +// ─── Environment ───────────────────────────────────────────────────────────── + +/** + * Clean environment for spawning Python/pip. + * Strips vars that could redirect imports or installs to the user's system Python. + */ +export function cleanPythonEnv(): NodeJS.ProcessEnv { + const { + PYTHONHOME, PYTHONPATH, PYTHONSTARTUP, PYTHONUSERBASE, + PIP_USER, PIP_TARGET, PIP_PREFIX, PIP_REQUIRE_VIRTUALENV, + VIRTUAL_ENV, CONDA_PREFIX, + ...rest + } = process.env + void PYTHONHOME; void PYTHONPATH; void PYTHONSTARTUP; void PYTHONUSERBASE + void PIP_USER; void PIP_TARGET; void PIP_PREFIX; void PIP_REQUIRE_VIRTUALENV + void VIRTUAL_ENV; void CONDA_PREFIX + return rest +} + +// ─── Path helpers ───────────────────────────────────────────────────────────── + +export function getEmbeddedPythonDir(): string { + if (app.isPackaged) return join(process.resourcesPath, 'python-embed') + return join(app.getAppPath(), 'resources', 'python-embed') +} + +export function getEmbeddedPythonExe(): string { + const dir = getEmbeddedPythonDir() + return process.platform === 'win32' ? join(dir, 'python.exe') : join(dir, 'bin', 'python3') +} + +/** Venv lives inside dependenciesDir on Windows (user-configurable drive), userData on Linux. */ +export function getVenvDir(userData: string): string { + if (process.platform === 'win32') { + return join(getSettings(userData).dependenciesDir, 'venv') + } + return join(userData, 'venv') +} -function areSitePackagesInstalled(): boolean { - const { dependenciesDir } = getSettings(app.getPath('userData')) - return existsSync(join(dependenciesDir, 'uvicorn')) +export function getVenvPythonExe(userData: string): string { + const venvDir = getVenvDir(userData) + return process.platform === 'win32' + ? join(venvDir, 'Scripts', 'python.exe') + : join(venvDir, 'bin', 'python') } +// ─── Setup state ────────────────────────────────────────────────────────────── + export function checkSetupNeeded(userData: string): boolean { const jsonPath = join(userData, 'python_setup.json') if (!existsSync(jsonPath)) return true @@ -45,98 +85,53 @@ export function checkSetupNeeded(userData: string): boolean { } catch { return true } - // On Unix packaged: also verify the venv was created - if (process.platform !== 'win32' && app.isPackaged) { - if (!existsSync(join(userData, 'venv', 'bin', 'python'))) return true - } - // Verify packages are actually present (guards against reinstall without uninstall) - if (!areSitePackagesInstalled()) return true + if (!existsSync(getVenvPythonExe(userData))) return true return false } export function markSetupDone(userData: string): void { - const jsonPath = join(userData, 'python_setup.json') writeFileSync( - jsonPath, + join(userData, 'python_setup.json'), JSON.stringify({ version: SETUP_VERSION, requirementsHash: hashRequirements() }), 'utf-8' ) } -/** Path to the venv Python executable created during setup (packaged Unix). */ -export function getVenvPythonExe(userData: string): string { - return join(userData, 'venv', 'bin', 'python') -} - -// ─── Embedded Python helpers (all platforms) ───────────────────────────────── - -export function getEmbeddedPythonDir(): string { - if (app.isPackaged) return join(process.resourcesPath, 'python-embed') - return join(app.getAppPath(), 'resources', 'python-embed') -} - -export function getEmbeddedPythonExe(): string { - const dir = getEmbeddedPythonDir() - if (process.platform === 'win32') return join(dir, 'python.exe') - return join(dir, 'bin', 'python3') -} +// ─── Setup steps ───────────────────────────────────────────────────────────── -function enableSitePackages(pythonDir: string, dependenciesDir: string, win: BrowserWindow): void { - win.webContents.send('setup:progress', { step: 'enabling-site', percent: 5 }) - const files = readdirSync(pythonDir) as string[] - const pthFile = files.find((f) => f.match(/^python\d+\._pth$/)) - if (!pthFile) { - console.warn('[PythonSetup] No ._pth file found in', pythonDir) - return - } - const pthPath = join(pythonDir, pthFile) - let content = readFileSync(pthPath, 'utf-8') - content = content.replace(/^#import site/m, 'import site') - if (!content.includes('Lib\\site-packages')) { - content = content.trimEnd() + '\nLib\\site-packages\n' - } - // Add the user's dependencies dir so Python finds packages installed there. - // PYTHONPATH is ignored when a ._pth file is present (Python embed behavior). - if (!content.includes(dependenciesDir)) { - content = content.trimEnd() + `\n${dependenciesDir}\n` - } - writeFileSync(pthPath, content, 'utf-8') - console.log('[PythonSetup] Enabled site-packages in', pthFile) -} - -function installPip(pythonExe: string, resourcesPath: string, win: BrowserWindow): Promise { +function createVenv(pythonExe: string, venvDir: string, win: BrowserWindow): Promise { return new Promise((resolve, reject) => { - win.webContents.send('setup:progress', { step: 'pip', percent: 10 }) - const getPipPath = join(resourcesPath, 'get-pip.py') - console.log('[PythonSetup] Installing pip from', getPipPath) - const proc = spawn(pythonExe, [getPipPath, '--no-warn-script-location'], { + win.webContents.send('setup:progress', { step: 'venv', percent: 5 }) + console.log('[PythonSetup] Creating venv at', venvDir) + const proc = spawn(pythonExe, ['-m', 'venv', venvDir], { stdio: ['ignore', 'pipe', 'pipe'], + env: cleanPythonEnv(), }) - proc.stdout?.on('data', (d: Buffer) => console.log('[pip install]', d.toString().trim())) - proc.stderr?.on('data', (d: Buffer) => console.error('[pip install]', d.toString().trim())) + proc.stdout?.on('data', (d: Buffer) => console.log('[venv]', d.toString().trim())) + proc.stderr?.on('data', (d: Buffer) => console.error('[venv]', d.toString().trim())) proc.on('close', (code) => { - win.webContents.send('setup:progress', { step: 'pip', percent: 20 }) - if (code === 0) resolve() - else reject(new Error(`get-pip.py exited with code ${code}`)) + if (code === 0) { + win.webContents.send('setup:progress', { step: 'venv', percent: 20 }) + resolve() + } else { + reject(new Error(`python -m venv exited with code ${code}`)) + } }) }) } -// ─── Shared helper ─────────────────────────────────────────────────────────── - function installRequirements( pythonExe: string, requirementsPath: string, win: BrowserWindow ): Promise { - const dependenciesDir = getSettings(app.getPath('userData')).dependenciesDir - mkdirSync(dependenciesDir, { recursive: true }) + const TOTAL_PACKAGES = 20 return new Promise((resolve, reject) => { - console.log('[PythonSetup] Installing requirements into', dependenciesDir) + console.log('[PythonSetup] Installing requirements with', pythonExe) const proc = spawn( pythonExe, - ['-m', 'pip', 'install', '-r', requirementsPath, '--target', dependenciesDir, '--no-warn-script-location', '--progress-bar', 'off'], - { stdio: ['ignore', 'pipe', 'pipe'] } + ['-m', 'pip', 'install', '-r', requirementsPath, '--no-warn-script-location', '--progress-bar', 'off'], + { stdio: ['ignore', 'pipe', 'pipe'], env: cleanPythonEnv() } ) let packagesInstalled = 0 const onLine = (line: string) => { @@ -175,7 +170,7 @@ function installRequirements( }) } -// ─── Unix helpers (venv) ───────────────────────────────────────────────────── +// ─── Unix dev helper ───────────────────────────────────────────────────────── function findSystemPython(): string { const candidates = ['python3.12', 'python3.11', 'python3.10', 'python3', 'python'] @@ -186,7 +181,7 @@ function findSystemPython(): string { console.log(`[PythonSetup] Found system Python: ${cmd} → ${out}`) return cmd } - } catch { /* not found, try next */ } + } catch { /* not found */ } } throw new Error( 'Python 3 not found on your system.\n' + @@ -196,58 +191,34 @@ function findSystemPython(): string { ) } -function createVenv(python3: string, venvDir: string, win: BrowserWindow): Promise { - return new Promise((resolve, reject) => { - win.webContents.send('setup:progress', { step: 'venv', percent: 10 }) - console.log('[PythonSetup] Creating venv at', venvDir) - const proc = spawn(python3, ['-m', 'venv', venvDir], { stdio: ['ignore', 'pipe', 'pipe'] }) - proc.stdout?.on('data', (d: Buffer) => console.log('[venv]', d.toString().trim())) - proc.stderr?.on('data', (d: Buffer) => console.error('[venv]', d.toString().trim())) - proc.on('close', (code) => { - if (code === 0) { - win.webContents.send('setup:progress', { step: 'venv', percent: 20 }) - resolve() - } else { - reject(new Error(`python3 -m venv exited with code ${code}`)) - } - }) - }) -} - -// ─── Public orchestrator ───────────────────────────────────────────────────── +// ─── Public orchestrator ────────────────────────────────────────────────────── export async function runFullSetup(win: BrowserWindow, userData: string): Promise { try { const requirementsPath = getRequirementsPath() + const venvDir = getVenvDir(userData) - if (process.platform === 'win32') { - // Windows: use embedded Python bundled with the app - const pythonDir = getEmbeddedPythonDir() + if (process.platform === 'win32' || app.isPackaged) { + // Packaged (all platforms) + Windows dev: use bundled python-build-standalone. + // python-build-standalone is a full Python install → venv module works natively, + // DLLs come from the installer so SAC doesn't block them. const pythonExe = getEmbeddedPythonExe() - const resourcesPath = app.isPackaged - ? process.resourcesPath - : join(app.getAppPath(), 'resources') - - const dependenciesDir = getSettings(app.getPath('userData')).dependenciesDir - enableSitePackages(pythonDir, dependenciesDir, win) - await installPip(pythonExe, resourcesPath, win) - await installRequirements(pythonExe, requirementsPath, win) - } else if (app.isPackaged) { - // Linux / macOS packaged: use bundled Python to create a venv in userData - // (resources dir may be read-only inside .app bundle) - win.webContents.send('setup:progress', { step: 'venv', percent: 5 }) - const python3 = getEmbeddedPythonExe() - const venvDir = join(userData, 'venv') - await createVenv(python3, venvDir, win) - const venvPython = join(venvDir, 'bin', 'python') + if (!existsSync(pythonExe)) { + throw new Error( + 'Bundled Python runtime not found.\n' + + 'Please reinstall the application.\n' + + `(expected: ${pythonExe})` + ) + } + await createVenv(pythonExe, venvDir, win) + const venvPython = getVenvPythonExe(userData) await installRequirements(venvPython, requirementsPath, win) } else { - // Linux / macOS dev: create a venv using the system Python - win.webContents.send('setup:progress', { step: 'python', percent: 5 }) + // Linux / macOS dev: use system Python + win.webContents.send('setup:progress', { step: 'venv', percent: 5 }) const python3 = findSystemPython() - const venvDir = join(userData, 'venv') await createVenv(python3, venvDir, win) - const venvPython = join(venvDir, 'bin', 'python') + const venvPython = getVenvPythonExe(userData) await installRequirements(venvPython, requirementsPath, win) } diff --git a/package.json b/package.json index 27de181..3dfd22e 100644 --- a/package.json +++ b/package.json @@ -73,10 +73,6 @@ { "from": "resources/python-embed", "to": "python-embed" - }, - { - "from": "resources/get-pip.py", - "to": "get-pip.py" } ] }, @@ -86,7 +82,13 @@ }, "linux": { "target": "AppImage", - "icon": "resources/icons/icon.png" + "icon": "resources/icons/icon.png", + "extraResources": [ + { + "from": "resources/python-embed", + "to": "python-embed" + } + ] } } } diff --git a/scripts/download-python-embed.js b/scripts/download-python-embed.js index 68f10df..25cbd01 100644 --- a/scripts/download-python-embed.js +++ b/scripts/download-python-embed.js @@ -7,34 +7,32 @@ const { execSync } = require('child_process') const RESOURCES_DIR = path.join(__dirname, '..', 'resources') const EMBED_DIR = path.join(RESOURCES_DIR, 'python-embed') -// ── Windows: official embeddable package ───────────────────────────────────── -const WIN_PYTHON_VERSION = '3.11.9' -const WIN_ZIP_URL = `https://www.python.org/ftp/python/${WIN_PYTHON_VERSION}/python-${WIN_PYTHON_VERSION}-embed-amd64.zip` -const WIN_GET_PIP_URL = 'https://bootstrap.pypa.io/get-pip.py' - -// ── Linux / macOS: python-build-standalone ─────────────────────────────────── -const PBS_PYTHON_VERSION = '3.11.9' +// python-build-standalone provides a full Python installation (includes venv + pip) +// Used for ALL platforms — consistent behavior, no stripped-down embed issues. +const PBS_VERSION = '3.11.9' const PBS_RELEASE = '20240726' function getPbsUrl() { const arch = process.arch === 'arm64' ? 'aarch64' : 'x86_64' - const triple = process.platform === 'darwin' - ? `${arch}-apple-darwin` - : `${arch}-unknown-linux-gnu` + const triple = process.platform === 'win32' + ? `${arch}-pc-windows-msvc` + : process.platform === 'darwin' + ? `${arch}-apple-darwin` + : `${arch}-unknown-linux-gnu` return ( `https://github.com/indygreg/python-build-standalone/releases/download/` + - `${PBS_RELEASE}/cpython-${PBS_PYTHON_VERSION}+${PBS_RELEASE}-${triple}-install_only.tar.gz` + `${PBS_RELEASE}/cpython-${PBS_VERSION}+${PBS_RELEASE}-${triple}-install_only.tar.gz` ) } -// ── Shared helpers ──────────────────────────────────────────────────────────── +// ── Helpers ─────────────────────────────────────────────────────────────────── function download(url, dest) { return new Promise((resolve, reject) => { console.log(`Downloading ${url} → ${dest}`) const file = fs.createWriteStream(dest) const request = (u) => { - https.get(u, (res) => { + https.get(u, { headers: { 'User-Agent': 'modly-build' } }, (res) => { if (res.statusCode === 301 || res.statusCode === 302) { request(res.headers.location) return @@ -64,23 +62,11 @@ function download(url, dest) { }) } -// ── Platform-specific extraction ────────────────────────────────────────────── - -function extractZip(zipPath, destDir) { - console.log(`Extracting ${zipPath} → ${destDir}`) - fs.mkdirSync(destDir, { recursive: true }) - execSync( - `powershell -NoProfile -Command "Expand-Archive -Path '${zipPath}' -DestinationPath '${destDir}' -Force"`, - { stdio: 'inherit' } - ) -} - function extractTar(tarPath, destDir) { console.log(`Extracting ${tarPath} → ${destDir}`) fs.mkdirSync(destDir, { recursive: true }) // --strip-components=1 removes the top-level "python/" directory from the archive execSync(`tar -xzf "${tarPath}" --strip-components=1 -C "${destDir}"`, { stdio: 'inherit' }) - // On macOS, remove quarantine attribute so Gatekeeper doesn't block execution if (process.platform === 'darwin') { try { execSync(`xattr -cr "${destDir}"`) } catch {} } @@ -91,49 +77,21 @@ function extractTar(tarPath, destDir) { async function main() { fs.mkdirSync(RESOURCES_DIR, { recursive: true }) - if (process.platform === 'win32') { - const pythonExe = path.join(EMBED_DIR, 'python.exe') - const getPipDest = path.join(RESOURCES_DIR, 'get-pip.py') - const zipTmp = path.join(RESOURCES_DIR, 'python-embed.zip') - - if (fs.existsSync(pythonExe) && fs.existsSync(getPipDest)) { - console.log('python-embed (Windows) already present, skipping.') - return - } - - if (!fs.existsSync(pythonExe)) { - await download(WIN_ZIP_URL, zipTmp) - extractZip(zipTmp, EMBED_DIR) - fs.unlinkSync(zipTmp) - console.log('Python embeddable extracted.') - } else { - console.log('python.exe already present, skipping ZIP download.') - } - - if (!fs.existsSync(getPipDest)) { - await download(WIN_GET_PIP_URL, getPipDest) - console.log('get-pip.py downloaded.') - } else { - console.log('get-pip.py already present.') - } - } else { - // Linux / macOS: python-build-standalone (already includes pip) - const pythonExe = path.join(EMBED_DIR, 'bin', 'python3') - - if (fs.existsSync(pythonExe)) { - console.log('python-embed (Unix) already present, skipping.') - return - } + const pythonExe = process.platform === 'win32' + ? path.join(EMBED_DIR, 'python.exe') + : path.join(EMBED_DIR, 'bin', 'python3') - const tarUrl = getPbsUrl() - const tarTmp = path.join(RESOURCES_DIR, 'python-embed.tar.gz') - await download(tarUrl, tarTmp) - extractTar(tarTmp, EMBED_DIR) - fs.unlinkSync(tarTmp) - console.log('Python standalone extracted.') + if (fs.existsSync(pythonExe)) { + console.log('python-embed already present, skipping.') + return } - console.log('Done. Resources ready.') + const tarUrl = getPbsUrl() + const tarTmp = path.join(RESOURCES_DIR, 'python-embed.tar.gz') + await download(tarUrl, tarTmp) + extractTar(tarTmp, EMBED_DIR) + fs.unlinkSync(tarTmp) + console.log('Done. Python standalone extracted.') } main().catch((err) => {