Skip to content

fix: correctly return custom python invocation error messages #185

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 3 commits into from
Apr 24, 2025
Merged

fix: correctly return custom python invocation error messages #185

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
0a6ef23
fix: correctly return custom python invocation error messages
Strum355 Apr 24, 2025
e513a36
fix: pass deps individually, not as one big string
Strum355 Apr 24, 2025
71c7a02
fix: include cause in errors thrown
Strum355 Apr 24, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
178 changes: 75 additions & 103 deletions src/providers/python_controller.js
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,31 +1,26 @@
import {execSync} from "node:child_process";
import fs from "node:fs";
import path from 'node:path';
import os, {EOL} from "os";
import {environmentVariableIsPopulated,getCustom, handleSpacesInPath} from "../tools.js";

import {environmentVariableIsPopulated,getCustom, invokeCommand} from "../tools.js";

function getPipFreezeOutput() {
return environmentVariableIsPopulated("EXHORT_PIP_FREEZE") ? new Buffer(process.env["EXHORT_PIP_FREEZE"],'base64').toString('ascii') : execSync(`${handleSpacesInPath(this.pathToPipBin)} freeze --all`, err => {
if (err) {
throw new Error('fail invoking pip freeze to fetch all installed dependencies in environment --> ' + err.message)
}
}).toString();
try {
return environmentVariableIsPopulated("EXHORT_PIP_FREEZE") ? new Buffer.from(process.env["EXHORT_PIP_FREEZE"], 'base64').toString('ascii') : invokeCommand(this.pathToPipBin, ['freeze', '--all']).toString();
} catch (error) {
throw new Error('Failed invoking \'pip freeze\' to list all installed packages in environment', {cause: error})
}
}

function getPipShowOutput(depNames) {

return environmentVariableIsPopulated("EXHORT_PIP_SHOW") ? new Buffer(process.env["EXHORT_PIP_SHOW"],'base64').toString('ascii') : execSync(`${handleSpacesInPath(this.pathToPipBin)} show ${depNames}`, err => {
if (err) {
throw new Error('fail invoking pip show to fetch all installed dependencies metadata --> ' + err.message)
}
}).toString();
try {
return environmentVariableIsPopulated("EXHORT_PIP_SHOW") ? new Buffer.from(process.env["EXHORT_PIP_SHOW"], 'base64').toString('ascii') : invokeCommand(this.pathToPipBin, ['show', ...depNames]).toString();
} catch (error) {
throw new Error('fail invoking \'pip show\' to fetch metadata for all installed packages in environment', {cause: error})
}
}

/** @typedef {{name: string, version: string, dependencies: DependencyEntry[]}} DependencyEntry */



export default class Python_controller {

pythonEnvDir
Expand All @@ -51,26 +46,23 @@ export default class Python_controller {
this.pathToRequirements = pathToRequirements
this.options = options
}
prepareEnvironment()
{
prepareEnvironment() {
if(!this.realEnvironment) {
this.pythonEnvDir = path.join(path.sep,"tmp","exhort_env_js")
execSync(`${handleSpacesInPath(this.pathToPythonBin)} -m venv ${handleSpacesInPath(this.pythonEnvDir)} `, err => {
if (err) {
throw new Error('failed creating virtual python environment - ' + err.message)
}
})
if(this.pathToPythonBin.includes("python3"))
{
this.pythonEnvDir = path.join(path.sep, "tmp", "exhort_env_js")
try {
invokeCommand(this.pathToPythonBin, ['-m', 'venv', this.pythonEnvDir])
} catch (error) {
throw new Error('Failed creating virtual python environment', {cause: error})
}
if(this.pathToPythonBin.includes("python3")) {
this.pathToPipBin = path.join(path.sep,this.pythonEnvDir,os.platform() === 'win32' ? "Scripts" : "bin",this.#decideIfWindowsOrLinuxPath("pip3"))
this.pathToPythonBin = path.join(path.sep,this.pythonEnvDir,os.platform() === 'win32' ? "Scripts" : "bin",this.#decideIfWindowsOrLinuxPath("python3"))
if(os.platform() === 'win32') {
let driveLetter = path.parse(process.cwd()).root
this.pathToPythonBin = `${driveLetter}${this.pathToPythonBin.substring(1)}`
this.pathToPipBin = `${driveLetter}${this.pathToPipBin.substring(1)}`
}
}
else {
} else {
this.pathToPipBin = path.join(path.sep,this.pythonEnvDir,os.platform() === 'win32' ? "Scripts" : "bin",this.#decideIfWindowsOrLinuxPath("pip"));
this.pathToPythonBin = path.join(path.sep,this.pythonEnvDir,os.platform() === 'win32' ? "Scripts" : "bin",this.#decideIfWindowsOrLinuxPath("python"))
if(os.platform() === 'win32') {
Expand All @@ -80,18 +72,15 @@ export default class Python_controller {
}
}
// upgrade pip version to latest
execSync(`${handleSpacesInPath(this.pathToPythonBin)} -m pip install --upgrade pip `, err => {
if (err) {
throw new Error('failed upgrading pip version on virtual python environment - ' + err.message)
}
})
}
else{
try {
invokeCommand(this.pathToPythonBin, ['-m', 'pip', 'install', '--upgrade', 'pip'])
} catch (error) {
throw new Error('Failed upgrading pip version in virtual python environment', {cause: error})
}
} else {
if(this.pathToPythonBin.startsWith("python")) {
this.pythonEnvDir = process.cwd()
}
else
{
} else {
this.pythonEnvDir = path.dirname(this.pathToPythonBin)
}
}
Expand All @@ -100,8 +89,7 @@ export default class Python_controller {
#decideIfWindowsOrLinuxPath(fileName) {
if (os.platform() === "win32") {
return fileName + ".exe"
}
else {
} else {
return fileName
}
}
Expand All @@ -110,8 +98,7 @@ export default class Python_controller {
* @param {boolean} includeTransitive - whether to return include in returned object transitive dependencies or not
* @return {[DependencyEntry]}
*/
getDependencies(includeTransitive)
{
getDependencies(includeTransitive) {
let startingTime
let endingTime
if (process.env["EXHORT_DEBUG"] === "true") {
Expand All @@ -120,21 +107,19 @@ export default class Python_controller {
}
if(!this.realEnvironment) {
let installBestEfforts = getCustom("EXHORT_PYTHON_INSTALL_BEST_EFFORTS","false",this.options);
if(installBestEfforts === "false")
{
execSync(`${handleSpacesInPath(this.pathToPipBin)} install -r ${handleSpacesInPath(this.pathToRequirements)}`, err =>{
if (err) {
throw new Error('fail installing requirements.txt manifest in created virtual python environment --> ' + err.message)
}
})
if(installBestEfforts === "false") {
try {
invokeCommand(this.pathToPipBin, ['install', '-r', this.pathToRequirements])
} catch (error) {
throw new Error('Failed installing requirements.txt manifest in virtual python environment', {cause: error})
}
}
// make best efforts to install the requirements.txt on the virtual environment created from the python3 passed in.
// that means that it will install the packages without referring to the versions, but will let pip choose the version
// tailored for version of the python environment( and of pip package manager) for each package.
else {
let matchManifestVersions = getCustom("MATCH_MANIFEST_VERSIONS","true",this.options);
if(matchManifestVersions === "true")
{
if(matchManifestVersions === "true") {
throw new Error("Conflicting settings, EXHORT_PYTHON_INSTALL_BEST_EFFORTS=true can only work with MATCH_MANIFEST_VERSIONS=false")
}
this.#installingRequirementsOneByOne()
Expand All @@ -156,27 +141,26 @@ export default class Python_controller {
let requirementsRows = requirementsContent.toString().split(EOL);
requirementsRows.filter((line) => !line.trim().startsWith("#")).filter((line) => line.trim() !== "").forEach( (dependency) => {
let dependencyName = getDependencyName(dependency);
execSync(`${handleSpacesInPath(this.pathToPipBin)} install ${dependencyName}`, err =>{
if (err) {
throw new Error(`Best efforts process - failed installing ${dependencyName} in created virtual python environment --> error message: ` + err.message)
}
})
} )
try {
invokeCommand(this.pathToPipBin, ['install', dependencyName])
} catch (error) {
throw new Error(`Failed in best-effort installing ${dependencyName} in virtual python environment`, {cause: error})
}
})
}
/**
* @private
*/
#cleanEnvironment()
{
if(!this.realEnvironment)
{
execSync(`${handleSpacesInPath(this.pathToPipBin)} uninstall -y -r ${handleSpacesInPath(this.pathToRequirements)}`, err =>{
if (err) {
throw new Error('fail uninstalling requirements.txt in created virtual python environment --> ' + err.message)
}
})
#cleanEnvironment() {
if(!this.realEnvironment) {
try {
invokeCommand(this.pathToPipBin, ['uninstall', '-y', '-r', this.pathToRequirements])
} catch (error) {
throw new Error('Failed uninstalling requirements.txt in virtual python environment', {cause: error})
}
}
}

#getDependenciesImpl(includeTransitive) {
let dependencies = new Array()
let usePipDepTree = getCustom("EXHORT_PIP_USE_DEP_TREE","false",this.options);
Expand All @@ -189,7 +173,7 @@ export default class Python_controller {
if(usePipDepTree !== "true") {
freezeOutput = getPipFreezeOutput.call(this);
lines = freezeOutput.split(EOL)
depNames = lines.map( line => getDependencyName(line)).join(" ")
depNames = lines.map( line => getDependencyName(line))
}
else {
pipDepTreeJsonArrayOutput = getDependencyTreeJsonFromPipDepTree(this.pathToPipBin,this.pathToPythonBin)
Expand All @@ -198,7 +182,7 @@ export default class Python_controller {

if(usePipDepTree !== "true") {
pipShowOutput = getPipShowOutput.call(this, depNames);
allPipShowDeps = pipShowOutput.split( EOL +"---" + EOL);
allPipShowDeps = pipShowOutput.split( EOL + "---" + EOL);
}
//debug
// pipShowOutput = "alternative pip show output goes here for debugging"
Expand All @@ -213,8 +197,7 @@ export default class Python_controller {
CachedEnvironmentDeps[dependencyName.replace("-", "_")] = record
CachedEnvironmentDeps[dependencyName.replace("_", "-")] = record
})
}
else {
} else {
pipDepTreeJsonArrayOutput.forEach( depTreeEntry => {
let packageName = depTreeEntry["package"]["package_name"].toLowerCase()
let pipDepTreeEntryForCache = {
Expand All @@ -229,18 +212,15 @@ export default class Python_controller {
}
linesOfRequirements.forEach( (dep) => {
// if matchManifestVersions setting is turned on , then
if(matchManifestVersions === "true")
{
if(matchManifestVersions === "true") {
let dependencyName
let manifestVersion
let installedVersion
let doubleEqualSignPosition
if(dep.includes("=="))
{
if(dep.includes("==")) {
doubleEqualSignPosition = dep.indexOf("==")
manifestVersion = dep.substring(doubleEqualSignPosition + 2).trim()
if(manifestVersion.includes("#"))
{
if(manifestVersion.includes("#")) {
let hashCharIndex = manifestVersion.indexOf("#");
manifestVersion = manifestVersion.substring(0,hashCharIndex)
}
Expand All @@ -249,17 +229,15 @@ export default class Python_controller {
if(CachedEnvironmentDeps[dependencyName.toLowerCase()] !== undefined) {
if(usePipDepTree !== "true") {
installedVersion = getDependencyVersion(CachedEnvironmentDeps[dependencyName.toLowerCase()])
}
else {
} else {
installedVersion = CachedEnvironmentDeps[dependencyName.toLowerCase()].version
}
}
if(installedVersion) {
if (manifestVersion.trim() !== installedVersion.trim()) {
throw new Error(`Can't continue with analysis - versions mismatch for dependency name ${dependencyName}, manifest version=${manifestVersion}, installed Version=${installedVersion}, if you want to allow version mismatch for analysis between installed and requested packages, set environment variable/setting - MATCH_MANIFEST_VERSIONS=false`)
throw new Error(`Can't continue with analysis - versions mismatch for dependency name ${dependencyName} (manifest version=${manifestVersion}, installed version=${installedVersion}).If you want to allow version mismatch for analysis between installed and requested packages, set environment variable/setting MATCH_MANIFEST_VERSIONS=false`)
}
}

}
}
let path = new Array()
Expand All @@ -274,11 +252,11 @@ export default class Python_controller {
if(DEP1 < DEP2) {
return -1;
}
if(DEP1 > DEP2)
{
if(DEP1 > DEP2) {
return 1;
}
return 0;})
return 0;
})
return dependencies
}
}
Expand Down Expand Up @@ -321,8 +299,7 @@ function getDependencyName(depLine) {
const regex = /[\w\s-_.]+/g;
if(depLine.match(regex)) {
result = depLine.match(regex)[0]
}
else {
} else {
result = depLine
}
}
Expand Down Expand Up @@ -358,10 +335,7 @@ function bringAllDependencies(dependencies, dependencyName, cachedEnvironmentDep
}
let record = cachedEnvironmentDeps[dependencyName.toLowerCase()]
if(record === null || record === undefined) {
throw new Error(`Package name=>${dependencyName} is not installed in your python environment,
either install it ( better to install requirements.txt altogether) or set
the setting EXHORT_PYTHON_VIRTUAL_ENV to true to automatically install
it in virtual environment (please note that this may slow down the analysis) `)
throw new Error(`Package ${dependencyName} is not installed in your python environment, either install it (better to install requirements.txt altogether) or set the setting EXHORT_PYTHON_VIRTUAL_ENV=true to automatically install it in virtual environment (please note that this may slow down the analysis)`)
}
let depName
let version;
Expand All @@ -370,8 +344,7 @@ function bringAllDependencies(dependencies, dependencyName, cachedEnvironmentDep
depName = getDependencyNameShow(record)
version = getDependencyVersion(record);
directDeps = getDepsList(record)
}
else {
} else {
depName = record.name
version = record.version
directDeps = record.dependencies
Expand All @@ -398,11 +371,11 @@ function bringAllDependencies(dependencies, dependencyName, cachedEnvironmentDep
if(DEP1 < DEP2) {
return -1;
}
if(DEP1 > DEP2)
{
if(DEP1 > DEP2) {
return 1;
}
return 0;})
return 0;
})

entry["dependencies"] = targetDeps
})
Expand All @@ -418,20 +391,19 @@ function bringAllDependencies(dependencies, dependencyName, cachedEnvironmentDep
function getDependencyTreeJsonFromPipDepTree(pipPath,pythonPath) {
let dependencyTree
try {
execSync(`${handleSpacesInPath(pipPath)} install pipdeptree`)
} catch (e) {
throw new Error(`Couldn't install pipdeptree utility, reason: ${e.getMessage}`)
invokeCommand(pipPath, ['install', 'pipdeptree'])
} catch (error) {
throw new Error(`Failed installing pipdeptree utility`, {cause: error})
}

try {
if(pythonPath.startsWith("python")) {
dependencyTree = execSync(`pipdeptree --json`).toString()
}
else {
dependencyTree = execSync(`pipdeptree --json --python ${handleSpacesInPath(pythonPath)} `).toString()
dependencyTree = invokeCommand('pipdeptree', ['--json']).toString()
} else {
dependencyTree = invokeCommand('pipdeptree', ['--json', '--python', pythonPath]).toString()
}
} catch (e) {
throw new Error(`couldn't produce dependency tree using pipdeptree tool, stop analysis, message -> ${e.getMessage}`)
} catch (error) {
throw new Error(`Failed building dependency tree using pipdeptree tool, stopping analysis`, {cause: error})
}

return JSON.parse(dependencyTree)
Expand Down
Loading
Loading