diff --git a/graalpy-maven-plugin/src/main/java/org/graalvm/python/maven/plugin/AbstractGraalPyMojo.java b/graalpy-maven-plugin/src/main/java/org/graalvm/python/maven/plugin/AbstractGraalPyMojo.java index b02bf04..9c2d1c6 100644 --- a/graalpy-maven-plugin/src/main/java/org/graalvm/python/maven/plugin/AbstractGraalPyMojo.java +++ b/graalpy-maven-plugin/src/main/java/org/graalvm/python/maven/plugin/AbstractGraalPyMojo.java @@ -40,6 +40,7 @@ */ package org.graalvm.python.maven.plugin; +import java.util.ArrayList; import org.apache.maven.artifact.Artifact; import org.apache.maven.artifact.DefaultArtifact; import org.apache.maven.artifact.handler.DefaultArtifactHandler; @@ -110,6 +111,9 @@ public AbstractGraalPyMojo(ProjectBuilder projectBuilder) { @Parameter List packages; + @Parameter(property = "requirementsFile") + String requirementsFile; + @SuppressFBWarnings("UUF_UNUSED_FIELD") public static class PythonHome { @SuppressWarnings("unused") @@ -124,7 +128,7 @@ public static class PythonHome { @Parameter(defaultValue = "${session}", readonly = true, required = true) private MavenSession session; - private ProjectBuilder projectBuilder; + private final ProjectBuilder projectBuilder; private Set launcherClassPath; @@ -148,14 +152,25 @@ protected void listGraalPyResources() throws MojoExecutionException { } } - protected void preExec(boolean enableWarnings) throws MojoExecutionException { - pythonResourcesDirectory = normalizeEmpty(pythonResourcesDirectory); - externalDirectory = normalizeEmpty(externalDirectory); - resourceDirectory = normalizeEmpty(resourceDirectory); - graalPyLockFile = normalizeEmpty(graalPyLockFile); - packages = packages != null - ? packages.stream().filter(p -> p != null && !p.trim().isEmpty()).toList() - : List.of(); + protected void preExec(boolean enableWarnings) throws MojoExecutionException { + pythonResourcesDirectory = normalizeEmpty(pythonResourcesDirectory); + externalDirectory = normalizeEmpty(externalDirectory); + resourceDirectory = normalizeEmpty(resourceDirectory); + graalPyLockFile = normalizeEmpty(graalPyLockFile); + Path reqFilePath = resolveReqFile(); + if (reqFilePath != null) { + getLog().info("GraalPy requirements file: " + reqFilePath); + if (packages != null) { + throw new MojoExecutionException( + "Cannot use and at the same time. " + + "New option is a replacement for using with list of inline ."); + } + packages = new ArrayList<>(); + } else if (packages != null) { + packages = packages.stream() + .filter(p -> p != null && !p.trim().isEmpty()) + .toList(); + } if (pythonResourcesDirectory != null) { if (externalDirectory != null) { @@ -201,7 +216,29 @@ protected void preExec(boolean enableWarnings) throws MojoExecutionException { } } - protected void postExec() throws MojoExecutionException { + protected Path resolveReqFile() { + if (requirementsFile == null || requirementsFile.isBlank()) { + return null; + } + + Path path = Path.of(requirementsFile); + Path finalPath = path.isAbsolute() + ? path + : project.getBasedir().toPath().resolve(path).normalize(); + + if (Files.exists(finalPath)) { + return finalPath; + } + + Path defaultReq = project.getBasedir().toPath().resolve("requirements.txt").normalize(); + if (Files.exists(defaultReq)) { + return defaultReq; + } + + return null; + } + + protected void postExec() throws MojoExecutionException { for (Resource r : project.getBuild().getResources()) { if (Files.exists(Path.of(r.getDirectory(), resourceDirectory, "proj"))) { getLog().warn(String.format("usage of %s is deprecated, use %s instead", @@ -249,12 +286,11 @@ private static String normalizeEmpty(String s) { } protected Launcher createLauncher() { - Launcher launcherArg = new Launcher(getLauncherPath()) { - public Set computeClassPath() throws IOException { - return calculateLauncherClasspath(project); - } - }; - return launcherArg; + return new Launcher(getLauncherPath()) { + public Set computeClassPath() throws IOException { + return calculateLauncherClasspath(project); + } + }; } protected Path getLockFile() { @@ -293,7 +329,7 @@ protected static String getGraalPyVersion(MavenProject project) throws IOExcepti private static Artifact getGraalPyArtifact(MavenProject project) throws IOException { var projectArtifacts = resolveProjectDependencies(project); - Artifact graalPyArtifact = projectArtifacts.stream().filter(a -> isPythonArtifact(a)).findFirst().orElse(null); + Artifact graalPyArtifact = projectArtifacts.stream().filter(AbstractGraalPyMojo::isPythonArtifact).findFirst().orElse(null); return Optional.ofNullable(graalPyArtifact).orElseThrow(() -> new IOException( "Missing GraalPy dependency. Please add to your pom either %s:%s or %s:%s".formatted(POLYGLOT_GROUP_ID, PYTHON_COMMUNITY_ARTIFACT_ID, POLYGLOT_GROUP_ID, PYTHON_ARTIFACT_ID))); @@ -326,6 +362,7 @@ private Set calculateLauncherClasspath(MavenProject project) throws IOEx && PYTHON_LAUNCHER_ARTIFACT_ID.equals(a.getArtifactId())) .findFirst().orElse(null); // python-launcher artifact + assert graalPyLauncherArtifact != null; launcherClassPath.add(graalPyLauncherArtifact.getFile().getAbsolutePath()); // and transitively all its dependencies launcherClassPath.addAll(resolveDependencies(graalPyLauncherArtifact)); diff --git a/graalpy-maven-plugin/src/main/java/org/graalvm/python/maven/plugin/InstallPackagesMojo.java b/graalpy-maven-plugin/src/main/java/org/graalvm/python/maven/plugin/InstallPackagesMojo.java index 105db79..f929085 100644 --- a/graalpy-maven-plugin/src/main/java/org/graalvm/python/maven/plugin/InstallPackagesMojo.java +++ b/graalpy-maven-plugin/src/main/java/org/graalvm/python/maven/plugin/InstallPackagesMojo.java @@ -46,6 +46,7 @@ import org.apache.maven.plugins.annotations.ResolutionScope; import org.apache.maven.project.ProjectBuilder; import org.graalvm.python.embedding.tools.vfs.VFSUtils; +import org.graalvm.python.embedding.tools.vfs.VFSUtils.Launcher; import org.graalvm.python.embedding.tools.vfs.VFSUtils.PackagesChangedException; import javax.inject.Inject; @@ -98,9 +99,10 @@ private void manageVenv() throws MojoExecutionException { Path venvDirectory = getVenvDirectory(); MavenDelegateLog log = new MavenDelegateLog(getLog()); Path lockFile = getLockFile(); + Path reqFile = resolveReqFile(); try { VFSUtils.createVenv(venvDirectory, packages, lockFile, MISSING_LOCK_FILE_WARNING, createLauncher(), - getGraalPyVersion(project), log); + getGraalPyVersion(project), log, reqFile); } catch (PackagesChangedException pce) { String pluginPkgsString = pce.getPluginPackages().isEmpty() ? "None" diff --git a/graalpy-maven-plugin/src/main/java/org/graalvm/python/maven/plugin/LockPackagesMojo.java b/graalpy-maven-plugin/src/main/java/org/graalvm/python/maven/plugin/LockPackagesMojo.java index 99a036d..0464514 100644 --- a/graalpy-maven-plugin/src/main/java/org/graalvm/python/maven/plugin/LockPackagesMojo.java +++ b/graalpy-maven-plugin/src/main/java/org/graalvm/python/maven/plugin/LockPackagesMojo.java @@ -64,8 +64,50 @@ public class LockPackagesMojo extends AbstractGraalPyMojo { This file contains a list of all required Python packages with their specific versions, based on the packages defined in the plugin configuration and their dependencies. """; - - @Inject + public static final String MISSING_DEPENDENCY_CONFIGURATION_ERROR = """ + In order to run the lock-packages goal you must declare Python dependencies in the graalpy-maven-plugin configuration. + + You must configure Python dependencies in one of the following ways: + + Option 1: Use with inline versioned package entries: + + + org.graalvm.python + graalpy-maven-plugin + + + {package_name}=={package_version} + + ... + + + + Option 2: Use a pip-compatible requirements.txt file: + + + org.graalvm.python + graalpy-maven-plugin + + requirements.txt + ... + + + + IMPORTANT: + • The requirementsFile workflow follows pip's native behavior. + • GraalPy lock files are NOT used or generated when requirementsFile is specified. + • The 'lock-packages' goal is NOT supported with . + • Users are expected to manage locking / freezing themselves using pip conventions (e.g., pip freeze). + • Do not define both and at the same time. + • The section must be declared on the graalpy-maven-plugin itself, + not inside a specific execution. + + For more details, see: + https://github.com/oracle/graalpython/blob/master/docs/user/Embedding-Build-Tools.md + """; + + + @Inject public LockPackagesMojo(ProjectBuilder projectBuilder) { super(projectBuilder); } @@ -93,33 +135,15 @@ protected void manageVenv() throws MojoExecutionException { } } - private void checkEmptyPackages() throws MojoExecutionException { - if ((packages == null || packages.isEmpty())) { - getLog().error(""); - getLog().error( - "In order to run the lock-packages goal there have to be python packages declared in the graalpy-maven-plugin configuration."); - getLog().error(""); - getLog().error( - "NOTE that the section has to be declared for the whole graalpy-maven-plugin"); - getLog().error("and not specifically for the process-graalpy-resources execution goal."); - getLog().error(""); - getLog().error("Please add the section to your configuration as follows:"); - getLog().error(""); - getLog().error(" org.graalvm.python"); - getLog().error(" graalpy-maven-plugin"); - getLog().error(" "); - getLog().error(" "); - getLog().error(" {package_name}=={package_version}"); - getLog().error(" "); - getLog().error(" ..."); - getLog().error(" "); - getLog().error(""); - - getLog().error( - "For more information, please refer to https://github.com/oracle/graalpython/blob/master/docs/user/Embedding-Build-Tools.md"); - getLog().error(""); - - throw new MojoExecutionException("missing python packages in plugin configuration"); - } - } + private void checkEmptyPackages() throws MojoExecutionException { + Path reqFilePath = resolveReqFile(); + boolean emptyPackages = packages == null || packages.isEmpty(); + boolean requirementsExists = reqFilePath != null; + if (emptyPackages && !requirementsExists) { + getLog().error(""); + getLog().error(MISSING_DEPENDENCY_CONFIGURATION_ERROR); + getLog().error(""); + throw new MojoExecutionException("Missing Python dependency configuration."); + } + } } diff --git a/integration-tests/prepare_venv_requirements_pom.xml b/integration-tests/prepare_venv_requirements_pom.xml new file mode 100644 index 0000000..ea01535 --- /dev/null +++ b/integration-tests/prepare_venv_requirements_pom.xml @@ -0,0 +1,92 @@ + + + + 4.0.0 + + org.apache.maven.plugin.my.unit + project-prepare-venv + 1.0-SNAPSHOT + jar + Test MyMojo + + + + junit + junit + 3.8.1 + test + + + org.graalvm.polyglot + python + ${env.GRAALPY_VERSION} + pom + + + org.graalvm.python + python-launcher + ${env.GRAALPY_VERSION} + + + + + + + org.graalvm.python + graalpy-maven-plugin + ${env.GRAALPY_VERSION} + + + + process-graalpy-resources + + + + + requirements.txt + + + + + diff --git a/integration-tests/test_maven_plugin.py b/integration-tests/test_maven_plugin.py index 9b191e1..25c43cc 100644 --- a/integration-tests/test_maven_plugin.py +++ b/integration-tests/test_maven_plugin.py @@ -835,6 +835,32 @@ def test_multiple_namespaced_vfs(self): assert return_code == 0, log + def test_requirements_txt_packages(self): + with util.TemporaryTestDirectory() as dir: + target_name = "requirements_packages" + target_dir = os.path.join(str(dir), target_name) + pom_template = os.path.join( + os.path.dirname(__file__), + "prepare_venv_requirements_pom.xml", + ) + self.generate_app(dir, target_dir, target_name, pom_template) + + requirements_txt = os.path.join(target_dir, "requirements.txt") + if not os.path.exists(requirements_txt): + with open(requirements_txt, "w", encoding="utf-8") as f: + f.write("pyfiglet==1.0.2\n") + + mvnw_cmd = util.get_mvn_wrapper(target_dir, self.env) + + cmd = mvnw_cmd + ["process-resources"] + out, return_code = util.run_cmd(cmd, self.env, cwd=target_dir) + util.check_ouput("BUILD SUCCESS", out) + assert return_code == 0 + + lock_file = os.path.join(target_dir, "graalpy.lock") + assert not os.path.exists(lock_file) + + if __name__ == "__main__": run_path = os.path.join(os.path.abspath(__file__), 'run.py') print(f"Run this file using the run.py driver ({run_path})") diff --git a/org.graalvm.python.embedding.tools/src/main/java/org/graalvm/python/embedding/tools/vfs/VFSUtils.java b/org.graalvm.python.embedding.tools/src/main/java/org/graalvm/python/embedding/tools/vfs/VFSUtils.java index 7845f99..681c11b 100644 --- a/org.graalvm.python.embedding.tools/src/main/java/org/graalvm/python/embedding/tools/vfs/VFSUtils.java +++ b/org.graalvm.python.embedding.tools/src/main/java/org/graalvm/python/embedding/tools/vfs/VFSUtils.java @@ -40,6 +40,7 @@ */ package org.graalvm.python.embedding.tools.vfs; +import java.io.Serial; import org.graalvm.python.embedding.tools.exec.BuildToolLog; import org.graalvm.python.embedding.tools.exec.BuildToolLog.CollectOutputLog; import org.graalvm.python.embedding.tools.exec.GraalPyRunner; @@ -317,19 +318,17 @@ static InstalledPackages fromVenv(Path venvDirectory) throws IOException { return new InstalledPackages(venvDirectory, installed, pkgs); } - List freeze(BuildToolLog log) throws IOException { + void freeze(BuildToolLog log) throws IOException { CollectOutputLog collectOutputLog = new CollectOutputLog(log); runPip(venvDirectory, "freeze", collectOutputLog, "--local"); packages = new ArrayList<>(collectOutputLog.getOutput()); String toWrite = "# Generated by GraalPy Maven or Gradle plugin using pip freeze\n" + "# This file is used by GraalPy VirtualFileSystem\n" + String.join("\n", packages); - Files.write(installedFile, toWrite.getBytes(StandardCharsets.UTF_8), StandardOpenOption.CREATE, + Files.writeString(installedFile, toWrite, StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING); logDebug(log, packages, "VFSUtils venv packages after install %s:", installedFile); - - return packages; } } @@ -365,12 +364,10 @@ static VenvContents fromVenv(Path venvDirectory) throws IOException { if (lines.get(0).startsWith("version=")) { // this was created with version >= 25 Map m = new HashMap<>(); - Iterator it = lines.iterator(); - while (it.hasNext()) { - String l = it.next(); - int idx = l.indexOf("="); - m.put(l.substring(0, idx), l.substring(idx + 1)); - } + for (String l : lines) { + int idx = l.indexOf("="); + m.put(l.substring(0, idx), l.substring(idx + 1)); + } String graalPyVersion = m.get(KEY_VERSION); String platform = m.get(KEY_PLATFORM); String packagesLine = m.get(KEY_PACKAGES); @@ -405,118 +402,110 @@ void write(List pkgs) throws IOException { private static final String INPUT_PACKAGES_PREFIX = "# input-packages: "; private static final String INPUT_PACKAGES_DELIMITER = ","; - private static class LockFile { - - final Path path; - final List packages; - final List inputPackages; - - private LockFile(Path path, List inputPackages, List packages) { - this.path = path; - this.packages = packages; - this.inputPackages = inputPackages; - } - - static LockFile fromFile(Path file, BuildToolLog log) throws IOException { - List packages = new ArrayList<>(); - List inputPackages = null; - List lines; - try { - lines = Files.readAllLines(file); - } catch (IOException e) { - throw new IOException(String.format("Cannot read the lock file from '%s'", file), e); - } - if (lines.isEmpty()) { - throw wrongFormat(file, lines, log); - } - // format: - // 1.) a multiline header comment - // 2.) graalpy version - 1 line (starting with comment #) - // 2.) input packages - 1 line (starting with comment #) - // 3.) locked packages - 1 line each (as input for pip install) - // see also LockFile.write() - Iterator it = lines.iterator(); - try { - // graalpy version, we don't care about it for now, but with future versions the - // file format might change, and we will need to know to parse differently - String graalPyVersion = null; - while (it.hasNext()) { - String line = it.next(); - if (line.startsWith(GRAALPY_VERSION_PREFIX)) { - graalPyVersion = line.substring(GRAALPY_VERSION_PREFIX.length()).trim(); - if (graalPyVersion.isEmpty()) { - throw wrongFormat(file, lines, log); - } - break; - } - } - if (graalPyVersion == null) { - throw wrongFormat(file, lines, log); - } - // input packages - String line = it.next(); - if (!line.startsWith(INPUT_PACKAGES_PREFIX)) { - throw wrongFormat(file, lines, log); - } - String pkgs = line.substring(INPUT_PACKAGES_PREFIX.length()).trim(); - if (pkgs.isEmpty()) { - throw wrongFormat(file, lines, log); - } - inputPackages = Arrays.asList(pkgs.split(INPUT_PACKAGES_DELIMITER)); - // locked packages - while (it.hasNext()) { - packages.add(it.next()); - } - } catch (NoSuchElementException e) { - throw wrongFormat(file, lines, log); - } - return new LockFile(file, inputPackages, packages); - } - - private static IOException wrongFormat(Path file, List lines, BuildToolLog log) { - if (log.isDebugEnabled()) { - log.debug("wrong format of lock file " + file); - for (String l : lines) { - log.debug(l); - } - log.debug(""); - } - return new IOException(String.format( - "Cannot read the lock file from '%s'%n(turn on debug log level to see the contents)", file)); - } - - private static void write(Path venvDirectory, Path lockFile, String lockFileHeader, List inputPackages, - String graalPyVersion, BuildToolLog log) throws IOException { - Objects.requireNonNull(venvDirectory); - Objects.requireNonNull(lockFile); - Objects.requireNonNull(lockFileHeader); - Objects.requireNonNull(log); - - assert Files.exists(venvDirectory); - - InstalledPackages installedPackages = InstalledPackages.fromVenv(venvDirectory); - List header = getHeaderList(lockFileHeader); - header.add(GRAALPY_VERSION_PREFIX + graalPyVersion); - header.add(INPUT_PACKAGES_PREFIX + String.join(INPUT_PACKAGES_DELIMITER, inputPackages)); - Files.write(lockFile, header, StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING); - Files.write(lockFile, installedPackages.packages, StandardOpenOption.APPEND); - - lifecycle(log, "Created GraalPy lock file: %s", lockFile); - logDebug(log, installedPackages.packages, null); - } - - private static List getHeaderList(String lockFileHeader) { - List list = new ArrayList<>(); - String[] lines = lockFileHeader.split("\n"); - for (String l : lines) { - list.add("# " + l); - } - return list; - } - } + private record LockFile(Path path, List inputPackages, List packages) { + + static LockFile fromFile(Path file, BuildToolLog log) throws IOException { + List packages = new ArrayList<>(); + List inputPackages = null; + List lines; + try { + lines = Files.readAllLines(file); + } catch (IOException e) { + throw new IOException(String.format("Cannot read the lock file from '%s'", file), e); + } + if (lines.isEmpty()) { + throw wrongFormat(file, lines, log); + } + // format: + // 1.) a multiline header comment + // 2.) graalpy version - 1 line (starting with comment #) + // 2.) input packages - 1 line (starting with comment #) + // 3.) locked packages - 1 line each (as input for pip install) + // see also LockFile.write() + Iterator it = lines.iterator(); + try { + // graalpy version, we don't care about it for now, but with future versions the + // file format might change, and we will need to know to parse differently + String graalPyVersion = null; + while (it.hasNext()) { + String line = it.next(); + if (line.startsWith(GRAALPY_VERSION_PREFIX)) { + graalPyVersion = line.substring(GRAALPY_VERSION_PREFIX.length()).trim(); + if (graalPyVersion.isEmpty()) { + throw wrongFormat(file, lines, log); + } + break; + } + } + if (graalPyVersion == null) { + throw wrongFormat(file, lines, log); + } + // input packages + String line = it.next(); + if (!line.startsWith(INPUT_PACKAGES_PREFIX)) { + throw wrongFormat(file, lines, log); + } + String pkgs = line.substring(INPUT_PACKAGES_PREFIX.length()).trim(); + if (pkgs.isEmpty()) { + throw wrongFormat(file, lines, log); + } + inputPackages = Arrays.asList(pkgs.split(INPUT_PACKAGES_DELIMITER)); + // locked packages + while (it.hasNext()) { + packages.add(it.next()); + } + } catch (NoSuchElementException e) { + throw wrongFormat(file, lines, log); + } + return new LockFile(file, inputPackages, packages); + } + + private static IOException wrongFormat(Path file, List lines, BuildToolLog log) { + if (log.isDebugEnabled()) { + log.debug("wrong format of lock file " + file); + for (String l : lines) { + log.debug(l); + } + log.debug(""); + } + return new IOException(String.format( + "Cannot read the lock file from '%s'%n(turn on debug log level to see the contents)", file)); + } + + private static void write( + Path venvDirectory, Path lockFile, String lockFileHeader, List inputPackages, + String graalPyVersion, BuildToolLog log) throws IOException { + Objects.requireNonNull(venvDirectory); + Objects.requireNonNull(lockFile); + Objects.requireNonNull(lockFileHeader); + Objects.requireNonNull(log); + + assert Files.exists(venvDirectory); + + InstalledPackages installedPackages = InstalledPackages.fromVenv(venvDirectory); + List header = getHeaderList(lockFileHeader); + header.add(GRAALPY_VERSION_PREFIX + graalPyVersion); + header.add(INPUT_PACKAGES_PREFIX + String.join(INPUT_PACKAGES_DELIMITER, inputPackages)); + Files.write(lockFile, header, StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING); + Files.write(lockFile, installedPackages.packages, StandardOpenOption.APPEND); + + lifecycle(log, "Created GraalPy lock file: %s", lockFile); + logDebug(log, installedPackages.packages, null); + } + + private static List getHeaderList(String lockFileHeader) { + List list = new ArrayList<>(); + String[] lines = lockFileHeader.split("\n"); + for (String l : lines) { + list.add("# " + l); + } + return list; + } + } public static final class PackagesChangedException extends Exception { - private static final long serialVersionUID = 9162516912727973035L; + @Serial + private static final long serialVersionUID = 9162516912727973035L; private final transient List pluginPackages; private final transient List lockFilePackages; @@ -539,7 +528,7 @@ public List getLockFilePackages() { public static void createVenv(Path venvDirectory, List packagesArgs, Launcher launcherArgs, String graalPyVersion, BuildToolLog log) throws IOException { try { - createVenv(venvDirectory, packagesArgs, null, null, launcherArgs, graalPyVersion, log); + createVenv(venvDirectory, packagesArgs, null, null, launcherArgs, graalPyVersion, log, null); } catch (PackagesChangedException e) { // should not happen assert false; @@ -547,46 +536,85 @@ public static void createVenv(Path venvDirectory, List packagesArgs, Lau } } - public static void createVenv(Path venvDirectory, List packages, Path lockFilePath, - String missingLockFileWarning, Launcher launcher, String graalPyVersion, BuildToolLog log) - throws IOException, PackagesChangedException { - Objects.requireNonNull(venvDirectory); - Objects.requireNonNull(packages); - Objects.requireNonNull(launcher); - Objects.requireNonNull(graalPyVersion); - Objects.requireNonNull(log); + public static void createVenv(Path venvDirectory, List packages, Path lockFilePath, + String missingLockFileWarning, Launcher launcher, String graalPyVersion, BuildToolLog log, Path reqFile + ) throws IOException, PackagesChangedException { - logVenvArgs(venvDirectory, packages, lockFilePath, launcher, graalPyVersion, log); + Objects.requireNonNull(venvDirectory); + Objects.requireNonNull(launcher); + Objects.requireNonNull(graalPyVersion); + Objects.requireNonNull(log); - List pluginPackages = trim(packages); - LockFile lockFile = null; - if (lockFilePath != null && Files.exists(lockFilePath)) { - lockFile = LockFile.fromFile(lockFilePath, log); - } + if (installPackagesFromReqFile(venvDirectory, launcher, graalPyVersion, log, reqFile)) { + return; + } - if (!checkPackages(venvDirectory, pluginPackages, lockFile, log)) { - return; - } + installPackages(venvDirectory, packages, lockFilePath, missingLockFileWarning, launcher, graalPyVersion, log); + } - VenvContents venvContents = ensureVenv(venvDirectory, graalPyVersion, launcher, log); + static boolean installPackagesFromReqFile(Path venvDirectory, Launcher launcher, + String graalPyVersion, BuildToolLog log, Path reqFile) throws IOException { + if (reqFile != null) { + log.info("Using dependency mode."); + log.info("Installing Python dependencies from: " + reqFile); + log.warning("Lock file is ignored in mode."); + log.warning("The 'lock-packages' goal should not be used together with ."); - InstalledPackages installedPackages = InstalledPackages.fromVenv(venvDirectory); - boolean installed; - if (lockFile != null) { - installed = install(venvDirectory, installedPackages, lockFile, log); - } else { - installed = install(venvDirectory, pluginPackages, venvContents, log); - } - if (installed) { - venvContents.write(pluginPackages); - installedPackages.freeze(log); - } - if (lockFile == null) { - missingLockFileWarning(venvDirectory, pluginPackages, missingLockFileWarning, log); - } - } + ensureVenv(venvDirectory, graalPyVersion, launcher, log); + + runPip(venvDirectory, "install", log, "--compile", "-r", reqFile.toString()); + + InstalledPackages installed = InstalledPackages.fromVenv(venvDirectory); + installed.freeze(log); + + return true; + } + return false; + } + + static void installPackages(Path venvDirectory, List packages, Path lockFilePath, + String missingLockFileWarning, Launcher launcher, String graalPyVersion, + BuildToolLog log) throws IOException, PackagesChangedException { + Objects.requireNonNull(packages); + log.info("Using inline dependency mode."); + + validatePackagesOrLockFile(packages, lockFilePath); + logVenvArgs(venvDirectory, packages, lockFilePath, launcher, graalPyVersion, log); + + List pluginPackages = trim(packages); - private static boolean removedFromPluginPackages(Path venvDirectory, List pluginPackages) + LockFile lockFile = null; + if (lockFilePath != null && Files.exists(lockFilePath)) { + log.info("Lock-file detected: " + lockFilePath); + lockFile = LockFile.fromFile(lockFilePath, log); + } + + if (!checkPackages(venvDirectory, pluginPackages, lockFile, log)) { + return; + } + + VenvContents venvContents = ensureVenv(venvDirectory, graalPyVersion, launcher, log); + InstalledPackages installedPackages = InstalledPackages.fromVenv(venvDirectory); + + boolean installed; + if (lockFile != null) { + installed = install(venvDirectory, installedPackages, lockFile, log); + } else { + installed = install(venvDirectory, pluginPackages, venvContents, log); + } + + if (installed) { + venvContents.write(pluginPackages); + installedPackages.freeze(log); + } + + if (lockFile == null) { + missingLockFileWarning(venvDirectory, pluginPackages, missingLockFileWarning, log); + } + } + + + private static boolean removedFromPluginPackages(Path venvDirectory, List pluginPackages) throws IOException { if (Files.exists(venvDirectory)) { // compare with contents from prev install if such already present @@ -656,7 +684,19 @@ public static void lockPackages(Path venvDirectory, List packages, Path } } - private static void logVenvArgs(Path venvDirectory, List packages, Path lockFile, Launcher launcherArgs, + private static void validatePackagesOrLockFile(List packages, Path lockFilePath) { + boolean hasPackages = packages != null && !packages.isEmpty(); + boolean hasLockFile = lockFilePath != null; + + if (hasPackages == hasLockFile) { + throw new IllegalArgumentException( + "Invalid configuration: and lock-file cannot be used together. Provide exactly one." + ); + } + } + + + private static void logVenvArgs(Path venvDirectory, List packages, Path lockFile, Launcher launcherArgs, String graalPyVersion, BuildToolLog log) throws IOException { if (log.isDebugEnabled()) { // avoid computing classpath if not necessary @@ -716,15 +756,16 @@ private static void logPackages(List packages, Path lockFile, BuildToolL } } - private static List readPackagesFromFile(Path file) throws IOException { - return Files.readAllLines(file).stream().filter((s) -> { - if (s == null) { - return false; - } - String l = s.trim(); - return !l.startsWith("#") && !s.isEmpty(); - }).toList(); - } + private static List readPackagesFromFile(Path file) throws IOException { + return Files.readAllLines(file).stream() + .map(String::trim) + .filter(line -> !line.isEmpty() && !line.startsWith("#")) + .toList(); + } + + public static List requirementsPackages(Path requirementsFile) throws IOException { + return Files.exists(requirementsFile) ? readPackagesFromFile(requirementsFile) : Collections.emptyList(); + } private static VenvContents ensureVenv(Path venvDirectory, String graalPyVersion, Launcher launcher, BuildToolLog log) throws IOException { @@ -760,7 +801,17 @@ private static VenvContents ensureVenv(Path venvDirectory, String graalPyVersion return contents; } - private static boolean install(Path venvDirectory, InstalledPackages installedPackages, LockFile lockFile, + private static boolean install(Path venvDirectory, Path requirementsFile, BuildToolLog log) throws IOException { + if (!Files.exists(requirementsFile)) { + throw new IOException("Requirements file not found: " + requirementsFile); + } + info(log, "Installing Python dependencies from requirements file: " + requirementsFile); + runPip(venvDirectory, "install", log, "--compile", "-r", requirementsFile.toString()); + return true; + } + + + private static boolean install(Path venvDirectory, InstalledPackages installedPackages, LockFile lockFile, BuildToolLog log) throws IOException { if (installedPackages.packages.size() != lockFile.packages.size() || deleteUnwantedPackages(venvDirectory, lockFile.packages, installedPackages.packages, log)) { @@ -783,7 +834,7 @@ private static boolean install(Path venvDirectory, List newPackages, Ven private static void missingLockFileWarning(Path venvDirectory, List newPackages, String missingLockFileWarning, BuildToolLog log) throws IOException { if (missingLockFileWarning != null && !Boolean.getBoolean("graalpy.vfs.skipMissingLockFileWarning")) { - if (!newPackages.containsAll(InstalledPackages.fromVenv(venvDirectory).packages)) { + if (!new HashSet<>(newPackages).containsAll(InstalledPackages.fromVenv(venvDirectory).packages)) { if (log.isWarningEnabled()) { String txt = missingLockFileWarning + "\n"; for (String t : txt.split("\n")) { @@ -800,7 +851,7 @@ private static void missingLockFileWarning(Path venvDirectory, List newP private static void checkPluginPackagesInLockFile(List pluginPackages, LockFile lockFile) throws PackagesChangedException { if (pluginPackages.size() != lockFile.inputPackages.size() - || !pluginPackages.containsAll(lockFile.inputPackages)) { + || !new HashSet<>(pluginPackages).containsAll(lockFile.inputPackages)) { throw new PackagesChangedException(new ArrayList<>(pluginPackages), new ArrayList<>(lockFile.inputPackages)); } @@ -928,7 +979,7 @@ with open(pyvenvcfg, 'w', encoding='utf-8') as f: private static boolean installWantedPackages(Path venvDirectory, List packages, List installedPackages, BuildToolLog log) throws IOException { Set pkgsToInstall = new HashSet<>(packages); - pkgsToInstall.removeAll(installedPackages); + installedPackages.forEach(pkgsToInstall::remove); if (pkgsToInstall.isEmpty()) { return false; } @@ -948,7 +999,7 @@ private static boolean deleteUnwantedPackages(Path venvDirectory, List p } args.add(0, "-y"); - runPip(venvDirectory, "uninstall", log, args.toArray(new String[args.size()])); + runPip(venvDirectory, "uninstall", log, args.toArray(new String[0])); return true; } @@ -980,12 +1031,11 @@ private static void runVenvBin(Path venvDirectory, String bin, BuildToolLog log, private static List trim(List l) { Iterator it = l.iterator(); - while (it.hasNext()) { - String p = it.next(); - if (p == null || p.trim().isEmpty()) { - it.remove(); - } - } + for (String s : l) { + if (s == null || s.trim().isEmpty()) { + it.remove(); + } + } return l; } @@ -1000,9 +1050,9 @@ private static void info(BuildToolLog log, String txt, Object... args) { } } - private static void lifecycle(BuildToolLog log, String txt, Object... args) { + private static void lifecycle(BuildToolLog log, Object... args) { if (log.isLifecycleEnabled()) { - log.lifecycle(String.format(txt, args)); + log.lifecycle(String.format("Created GraalPy lock file: %s", args)); } } diff --git a/org.graalvm.python.embedding.tools/src/test/java/org/graalvm/python/embedding/tools/test/EmbeddingTestUtils.java b/org.graalvm.python.embedding.tools/src/test/java/org/graalvm/python/embedding/tools/test/EmbeddingTestUtils.java index ea8074a..66ab678 100644 --- a/org.graalvm.python.embedding.tools/src/test/java/org/graalvm/python/embedding/tools/test/EmbeddingTestUtils.java +++ b/org.graalvm.python.embedding.tools/src/test/java/org/graalvm/python/embedding/tools/test/EmbeddingTestUtils.java @@ -71,7 +71,7 @@ public static void createVenv(Path venvDir, String graalPyVersion, BuildToolLog Launcher launcher = createLauncher(venvDir); if (lockFile != null) { VFSUtils.createVenv(venvDir, Arrays.asList(packages), lockFile, missingLockFileWarning, launcher, - graalPyVersion, log); + graalPyVersion, log,null); } else { VFSUtils.createVenv(venvDir, Arrays.asList(packages), launcher, graalPyVersion, log); } diff --git a/org.graalvm.python.embedding.tools/src/test/java/org/graalvm/python/embedding/tools/test/VFSUtilsTest.java b/org.graalvm.python.embedding.tools/src/test/java/org/graalvm/python/embedding/tools/test/VFSUtilsTest.java index 2fef207..c76e718 100644 --- a/org.graalvm.python.embedding.tools/src/test/java/org/graalvm/python/embedding/tools/test/VFSUtilsTest.java +++ b/org.graalvm.python.embedding.tools/src/test/java/org/graalvm/python/embedding/tools/test/VFSUtilsTest.java @@ -108,12 +108,12 @@ private void clearOutput() { public void subProcessOut(String s) { println("[subout] ", s); - addLine(s.toString()); + addLine(s); } public void subProcessErr(String s) { println("[suberr] ", s); - addLine(s.toString()); + addLine(s); } public void info(String s) { @@ -185,7 +185,7 @@ private static boolean isVerbose() { /** * tests scenarios without lock file logic available, but not used - * + *

* - packages declared only in plugin config - lock file path is provided, but * does not exist */ @@ -203,7 +203,7 @@ public void withPackagesOnlyFromPluginConfig() throws IOException, PackagesChang Path lockFile = tmpDir.resolve("lockFile.txt"); Path contents = venvDir.resolve("contents"); - // no packages, lock file file does not exist - does nothing + // no packages, lock file does not exist - does nothing log.clearOutput(); createVenv(venvDir, "0.1", log, lockFile); assertFalse(Files.exists(venvDir)); @@ -324,23 +324,22 @@ public void lockFile() throws IOException { createWithLockFile(venvDir, lockFile, log); - List validLockFileHeader = createLockFileHeader("0.1", "pkg"); + List validLockFileHeader = createLockFileHeader("pkg"); List lockFileList; - int headerLineCount = LOCK_FILE_HEADER.split("\n").length; - // bogus graalPyVersion line - int graalpVersionLineIdx = headerLineCount; + // bogus graalPyVersion line + int graalpyVersionLineIdx = LOCK_FILE_HEADER.split("\n").length; lockFileList = new ArrayList<>(validLockFileHeader); - lockFileList.set(graalpVersionLineIdx, "test"); + lockFileList.set(graalpyVersionLineIdx, "test"); createWithLockFile(venvDir, lockFile, log, lockFileList); // empty graalPyVersion line lockFileList = new ArrayList<>(validLockFileHeader); - lockFileList.set(graalpVersionLineIdx, GRAALPY_VERSION_PREFIX); + lockFileList.set(graalpyVersionLineIdx, GRAALPY_VERSION_PREFIX); createWithLockFile(venvDir, lockFile, log, lockFileList); lockFileList = new ArrayList<>(validLockFileHeader); - lockFileList.set(graalpVersionLineIdx, GRAALPY_VERSION_PREFIX + " "); + lockFileList.set(graalpyVersionLineIdx, GRAALPY_VERSION_PREFIX + " "); createWithLockFile(venvDir, lockFile, log, lockFileList); // bogus input packages line @@ -379,12 +378,12 @@ private static void lock(Path venvDir, Path lockFile, TestLog log, String... pac "0.1", log); } - private static List createLockFileHeader(String graalPyVersion, String... packages) { + private static List createLockFileHeader(String... packages) { List lines = new ArrayList<>(); for (String s : LOCK_FILE_HEADER.split("\n")) { lines.add("# " + s); } - lines.add(GRAALPY_VERSION_PREFIX + graalPyVersion); + lines.add(GRAALPY_VERSION_PREFIX + "0.1"); lines.add(INPUT_PACKAGES_PREFIX + String.join(",", packages)); return lines; } @@ -594,30 +593,30 @@ public void packageRemoved() deleteDirOnShutdown(tmpDir); assertFalse(callPackageRemoved(Collections.emptyList(), Collections.emptyList(), Collections.emptyList())); - assertFalse(callPackageRemoved(Arrays.asList("pkg1"), Collections.emptyList(), Collections.emptyList())); - assertFalse(callPackageRemoved(Arrays.asList("pkg1"), Arrays.asList("pkg1"), Arrays.asList("pkg1==1"))); - assertFalse(callPackageRemoved(Arrays.asList("pkg1", "pkg2"), Arrays.asList("pkg1"), Arrays.asList("pkg1==1"))); + assertFalse(callPackageRemoved(Collections.singletonList("pkg1"), Collections.emptyList(), Collections.emptyList())); + assertFalse(callPackageRemoved(Collections.singletonList("pkg1"), Collections.singletonList("pkg1"), Collections.singletonList("pkg1==1"))); + assertFalse(callPackageRemoved(Arrays.asList("pkg1", "pkg2"), Collections.singletonList("pkg1"), Collections.singletonList("pkg1==1"))); assertFalse(callPackageRemoved(Arrays.asList("pkg1", "pkg2"), Arrays.asList("pkg1", "pkg2"), Arrays.asList("pkg1==1", "pkg2==1"))); - assertFalse(callPackageRemoved(Arrays.asList("pkg1=="), Arrays.asList("pkg1=="), Arrays.asList("pkg1==1"))); - assertFalse(callPackageRemoved(Arrays.asList("==pkg1"), Arrays.asList("==pkg1"), Arrays.asList("pkg1==1"))); - assertFalse(callPackageRemoved(Arrays.asList("pkg1==1"), Arrays.asList("pkg1"), Arrays.asList("pkg1==1"))); - assertFalse(callPackageRemoved(Arrays.asList("pkg1"), Arrays.asList("pkg1"), Arrays.asList("pkg1==1"))); + assertFalse(callPackageRemoved(Collections.singletonList("pkg1=="), Collections.singletonList("pkg1=="), Collections.singletonList("pkg1==1"))); + assertFalse(callPackageRemoved(Collections.singletonList("==pkg1"), Collections.singletonList("==pkg1"), Collections.singletonList("pkg1==1"))); + assertFalse(callPackageRemoved(Collections.singletonList("pkg1==1"), Collections.singletonList("pkg1"), Collections.singletonList("pkg1==1"))); + assertFalse(callPackageRemoved(Collections.singletonList("pkg1"), Collections.singletonList("pkg1"), Collections.singletonList("pkg1==1"))); - assertTrue(callPackageRemoved(Collections.emptyList(), Arrays.asList("pkg"), Arrays.asList("pkg==1"))); - assertTrue(callPackageRemoved(Arrays.asList("pkg2"), Arrays.asList("pkg1"), Arrays.asList("pkg1==1"))); - assertTrue(callPackageRemoved(Arrays.asList("pkg1"), Arrays.asList("pkg1", "pkg2"), + assertTrue(callPackageRemoved(Collections.emptyList(), Collections.singletonList("pkg"), Collections.singletonList("pkg==1"))); + assertTrue(callPackageRemoved(Collections.singletonList("pkg2"), Collections.singletonList("pkg1"), Collections.singletonList("pkg1==1"))); + assertTrue(callPackageRemoved(Collections.singletonList("pkg1"), Arrays.asList("pkg1", "pkg2"), Arrays.asList("pkg1==1", "pkg2==1"))); - assertTrue(callPackageRemoved(Arrays.asList("pkg1"), Arrays.asList("pkg1=="), Arrays.asList("pkg1==1"))); - assertTrue(callPackageRemoved(Arrays.asList("pkg1=="), Arrays.asList("pkg1"), Arrays.asList("pkg1==1"))); - assertTrue(callPackageRemoved(Arrays.asList("==pkg1"), Arrays.asList("pkg1"), Arrays.asList("pkg1==1"))); + assertTrue(callPackageRemoved(Collections.singletonList("pkg1"), Collections.singletonList("pkg1=="), Collections.singletonList("pkg1==1"))); + assertTrue(callPackageRemoved(Collections.singletonList("pkg1=="), Collections.singletonList("pkg1"), Collections.singletonList("pkg1==1"))); + assertTrue(callPackageRemoved(Collections.singletonList("==pkg1"), Collections.singletonList("pkg1"), Collections.singletonList("pkg1==1"))); - assertTrue(callPackageRemoved(Arrays.asList("pkg1==2"), Arrays.asList("pkg1==1"), Arrays.asList("pkg1==1"))); - assertTrue(callPackageRemoved(Arrays.asList("pkg1==2"), Arrays.asList("pkg1==1", "pkg2==1"), + assertTrue(callPackageRemoved(Collections.singletonList("pkg1==2"), Collections.singletonList("pkg1==1"), Collections.singletonList("pkg1==1"))); + assertTrue(callPackageRemoved(Collections.singletonList("pkg1==2"), Arrays.asList("pkg1==1", "pkg2==1"), Arrays.asList("pkg1==1", "pkg2==1"))); - assertTrue(callPackageRemoved(Arrays.asList("pkg1==2"), Arrays.asList("pkg1", "pkg2"), + assertTrue(callPackageRemoved(Collections.singletonList("pkg1==2"), Arrays.asList("pkg1", "pkg2"), Arrays.asList("pkg1==1", "pkg2==1"))); } @@ -714,7 +713,7 @@ private static void checkLockFile(Path lockFile, String[] inputPackages, String. throws IOException { assertTrue(Files.exists(lockFile)); List lines = Files.readAllLines(lockFile); - List header = createLockFileHeader("0.1", inputPackages); + List header = createLockFileHeader(inputPackages); assertTrue(lines.size() >= header.size()); for (int i = 0; i < header.size(); i++) { assertEquals(header.get(i), lines.get(i)); diff --git a/org.graalvm.python.gradle.plugin/src/main/java/org/graalvm/python/tasks/InstallPackagesTask.java b/org.graalvm.python.gradle.plugin/src/main/java/org/graalvm/python/tasks/InstallPackagesTask.java index 92f6e82..5eccd30 100644 --- a/org.graalvm.python.gradle.plugin/src/main/java/org/graalvm/python/tasks/InstallPackagesTask.java +++ b/org.graalvm.python.gradle.plugin/src/main/java/org/graalvm/python/tasks/InstallPackagesTask.java @@ -98,7 +98,7 @@ public void exec() throws GradleException { Path lockFilePath = getLockFilePath(); try { VFSUtils.createVenv(venvDirectory, getPackages().get(), lockFilePath, MISSING_LOCK_FILE_WARNING, - createLauncher(), getPolyglotVersion().get(), getLog()); + createLauncher(), getPolyglotVersion().get(), getLog(), null); } catch (PackagesChangedException pce) { String pluginPkgsString = pce.getPluginPackages().isEmpty() ? "None"