diff --git a/pkl-cli/src/main/kotlin/org/pkl/cli/CliProjectPackager.kt b/pkl-cli/src/main/kotlin/org/pkl/cli/CliProjectPackager.kt index 348960df3..8f6bfb485 100644 --- a/pkl-cli/src/main/kotlin/org/pkl/cli/CliProjectPackager.kt +++ b/pkl-cli/src/main/kotlin/org/pkl/cli/CliProjectPackager.kt @@ -21,17 +21,8 @@ import org.pkl.commons.cli.CliBaseOptions import org.pkl.commons.cli.CliException import org.pkl.commons.cli.CliTestException import org.pkl.commons.cli.CliTestOptions -import org.pkl.core.Loggers -import org.pkl.core.SecurityManagers -import org.pkl.core.StackFrameTransformers -import org.pkl.core.module.ModuleKeyFactories import org.pkl.core.project.Project import org.pkl.core.project.ProjectPackager -import org.pkl.core.resource.ResourceReaders -import org.pkl.core.runtime.ModuleResolver -import org.pkl.core.runtime.ResourceManager -import org.pkl.core.runtime.VmContext -import org.pkl.core.runtime.VmUtils import org.pkl.core.util.ErrorMessages class CliProjectPackager( @@ -85,38 +76,16 @@ class CliProjectPackager( } } } - // A VmContext is needed here because loading PklProject.deps.json uses resource readers - VmUtils.createContext { - val vmContext = VmContext.get(null) - vmContext.initialize( - VmContext.Holder( - StackFrameTransformers.defaultTransformer, - SecurityManagers.defaultManager, - httpClient, - ModuleResolver(listOf(ModuleKeyFactories.standardLibrary)), - ResourceManager(SecurityManagers.defaultManager, listOf(ResourceReaders.file())), - Loggers.noop(), - mapOf(), - mapOf(), - null, - null, - null, - null - ) - ) - - ProjectPackager( - projects, - cliOptions.normalizedWorkingDir, - outputPath, - stackFrameTransformer, - securityManager, - httpClient, - skipPublishCheck, - consoleWriter - ) - .createPackages() - } - .close() + ProjectPackager( + projects, + cliOptions.normalizedWorkingDir, + outputPath, + stackFrameTransformer, + securityManager, + httpClient, + skipPublishCheck, + consoleWriter + ) + .createPackages() } } diff --git a/pkl-core/src/main/java/org/pkl/core/EvaluatorImpl.java b/pkl-core/src/main/java/org/pkl/core/EvaluatorImpl.java index 9f9c26402..3bf4b9be9 100644 --- a/pkl-core/src/main/java/org/pkl/core/EvaluatorImpl.java +++ b/pkl-core/src/main/java/org/pkl/core/EvaluatorImpl.java @@ -105,7 +105,8 @@ public EvaluatorImpl( packageResolver, projectDependencies == null ? null - : new ProjectDependenciesManager(projectDependencies))); + : new ProjectDependenciesManager( + projectDependencies, moduleResolver, securityManager))); }); this.timeout = timeout; // NOTE: would probably make sense to share executor between evaluators diff --git a/pkl-core/src/main/java/org/pkl/core/ast/builder/AstBuilder.java b/pkl-core/src/main/java/org/pkl/core/ast/builder/AstBuilder.java index c4d15a7e4..352d71e94 100644 --- a/pkl-core/src/main/java/org/pkl/core/ast/builder/AstBuilder.java +++ b/pkl-core/src/main/java/org/pkl/core/ast/builder/AstBuilder.java @@ -1818,6 +1818,8 @@ private URI resolveImport(String importUri, StringConstantContext importUriCtx) } catch (VmException e) { throw exceptionBuilder() .evalError(e.getMessage(), e.getMessageArguments()) + .withCause(e.getCause()) + .withHint(e.getHint()) .withSourceSection(createSourceSection(importUriCtx)) .build(); } diff --git a/pkl-core/src/main/java/org/pkl/core/module/ModuleKeys.java b/pkl-core/src/main/java/org/pkl/core/module/ModuleKeys.java index 0a1b5cf4f..4886e3ca2 100644 --- a/pkl-core/src/main/java/org/pkl/core/module/ModuleKeys.java +++ b/pkl-core/src/main/java/org/pkl/core/module/ModuleKeys.java @@ -39,7 +39,6 @@ import org.pkl.core.util.HttpUtils; import org.pkl.core.util.IoUtils; import org.pkl.core.util.Nullable; -import org.pkl.core.util.Pair; /** Utilities for creating and using {@link ModuleKey}s. */ public final class ModuleKeys { @@ -288,16 +287,20 @@ public String loadSource() throws IOException { } } - private static class File extends DependencyAwareModuleKey { + private static class File implements ModuleKey { final URI uri; final Path path; File(URI uri, Path path) { - super(uri); this.uri = uri; this.path = path; } + @Override + public URI getUri() { + return uri; + } + @Override public boolean hasElement(SecurityManager securityManager, URI uri) throws SecurityManagerException { @@ -323,17 +326,18 @@ public ResolvedModuleKey resolve(SecurityManager securityManager) } @Override - protected Map getDependencies() { - var projectDepsManager = VmContext.get(null).getProjectDependenciesManager(); - if (projectDepsManager == null || !projectDepsManager.hasUri(uri)) { - throw new PackageLoadError("cannotResolveDependencyNoProject"); - } - return projectDepsManager.getDependencies(); + public boolean isGlobbable() { + return true; + } + + @Override + public boolean isLocal() { + return true; } @Override - protected PackageLoadError cannotFindDependency(String name) { - return new PackageLoadError("cannotFindDependencyInProject", name); + public boolean hasHierarchicalUris() { + return true; } } @@ -535,79 +539,67 @@ public ResolvedModuleKey resolve(SecurityManager securityManager) } } - /** Base implementation; knows how to resolve dependencies prefixed with @. */ - public abstract static class DependencyAwareModuleKey implements ModuleKey { + private abstract static class AbstractPackage implements ModuleKey { - protected final URI uri; + protected final PackageAssetUri packageAssetUri; - protected DependencyAwareModuleKey(URI uri) { - this.uri = uri; + AbstractPackage(PackageAssetUri packageAssetUri) { + this.packageAssetUri = packageAssetUri; } + protected abstract Map getDependencies() + throws IOException, SecurityManagerException; + @Override - public URI getUri() { - return uri; + public boolean hasHierarchicalUris() { + return true; } - protected Pair parseDependencyNotation(String importPath) { - var idx = importPath.indexOf('/'); - if (idx == -1) { - // treat named dependency without a subpath as the root path. - // i.e. resolve to `@foo` to `package://example.com/foo@1.0.0#/` - return Pair.of(importPath.substring(1), "/"); - } - return Pair.of(importPath.substring(1, idx), importPath.substring(idx)); + @Override + public boolean hasFragmentPaths() { + return true; } - protected abstract Map getDependencies() - throws IOException, SecurityManagerException; - @Override public boolean isLocal() { return true; } @Override - public boolean hasHierarchicalUris() { + public boolean isGlobbable() { return true; } @Override - public boolean isGlobbable() { - return true; + public URI getUri() { + return packageAssetUri.getUri(); } - private URI resolveDependencyNotation(String notation) - throws IOException, SecurityManagerException { - var parsed = parseDependencyNotation(notation); + @Override + public URI resolveUri(URI baseUri, URI importUri) throws IOException, SecurityManagerException { + var ssp = importUri.getSchemeSpecificPart(); + if (importUri.isAbsolute() || !ssp.startsWith("@")) { + return ModuleKey.super.resolveUri(baseUri, importUri); + } + var parsed = IoUtils.parseDependencyNotation(ssp); var name = parsed.getFirst(); var path = parsed.getSecond(); var dependency = getDependencies().get(name); if (dependency == null) { - throw cannotFindDependency(name); + throw new PackageLoadError( + "cannotFindDependencyInPackage", + name, + packageAssetUri.getPackageUri().getDisplayName()); } return dependency.getPackageUri().toPackageAssetUri(path).getUri(); } - - @Override - public URI resolveUri(URI baseUri, URI importUri) throws IOException, SecurityManagerException { - if (importUri.isAbsolute() || !importUri.getPath().startsWith("@")) { - return ModuleKey.super.resolveUri(baseUri, importUri); - } - return resolveDependencyNotation(importUri.getPath()); - } - - protected abstract PackageLoadError cannotFindDependency(String name); } /** Represents a module imported via the {@code package} scheme. */ - private static class Package extends DependencyAwareModuleKey { - - private final PackageAssetUri packageAssetUri; + private static class Package extends AbstractPackage { Package(PackageAssetUri packageAssetUri) { - super(packageAssetUri.getUri()); - this.packageAssetUri = packageAssetUri; + super(packageAssetUri); } private PackageResolver getPackageResolver() { @@ -619,6 +611,7 @@ private PackageResolver getPackageResolver() { @Override public ResolvedModuleKey resolve(SecurityManager securityManager) throws IOException, SecurityManagerException { + var uri = packageAssetUri.getUri(); securityManager.checkResolveModule(uri); var bytes = getPackageResolver() @@ -642,11 +635,6 @@ public boolean hasElement(SecurityManager securityManager, URI elementUri) return getPackageResolver().hasElement(assetUri, assetUri.getPackageUri().getChecksums()); } - @Override - public boolean hasFragmentPaths() { - return true; - } - @Override protected Map getDependencies() throws IOException, SecurityManagerException { @@ -655,12 +643,6 @@ public boolean hasFragmentPaths() { packageAssetUri.getPackageUri(), packageAssetUri.getPackageUri().getChecksums()) .getDependencies(); } - - @Override - protected PackageLoadError cannotFindDependency(String name) { - return new PackageLoadError( - "cannotFindDependencyInPackage", name, packageAssetUri.getPackageUri().getDisplayName()); - } } /** @@ -668,15 +650,11 @@ protected PackageLoadError cannotFindDependency(String name) { * *

The {@code projectpackage} scheme is what project-local dependencies resolve to when * imported using dependency notation (for example, {@code import "@foo/bar.pkl"}). This scheme is - * an internal implementation detail, and we do not expect a project to declare this. + * an internal implementation detail, and we do not expect a module to declare this. */ - private static class ProjectPackage extends DependencyAwareModuleKey { - - private final PackageAssetUri packageAssetUri; - + public static class ProjectPackage extends AbstractPackage { ProjectPackage(PackageAssetUri packageAssetUri) { - super(packageAssetUri.getUri()); - this.packageAssetUri = packageAssetUri; + super(packageAssetUri); } private PackageResolver getPackageResolver() { @@ -685,36 +663,36 @@ private PackageResolver getPackageResolver() { return packageResolver; } - private ProjectDependenciesManager getProjectDepsResolver() { + private ProjectDependenciesManager getProjectDependenciesManager() { var projectDepsManager = VmContext.get(null).getProjectDependenciesManager(); assert projectDepsManager != null; return projectDepsManager; } + private @Nullable URI getLocalUri(Dependency dependency) { + return getLocalUri(dependency, packageAssetUri); + } + private @Nullable URI getLocalUri(Dependency dependency, PackageAssetUri assetUri) { if (!(dependency instanceof LocalDependency localDependency)) { return null; } return localDependency.resolveAssetUri( - getProjectDepsResolver().getProjectBaseUri(), assetUri); - } - - private @Nullable Path getLocalPath(Dependency dependency) { - if (!(dependency instanceof LocalDependency)) { - return null; - } - return getLocalPath(dependency, packageAssetUri); + getProjectDependenciesManager().getProjectBaseUri(), assetUri); } @Override public ResolvedModuleKey resolve(SecurityManager securityManager) throws IOException, SecurityManagerException { - securityManager.checkResolveModule(packageAssetUri.getUri()); + var uri = packageAssetUri.getUri(); + securityManager.checkResolveModule(uri); var dependency = - getProjectDepsResolver().getResolvedDependency(packageAssetUri.getPackageUri()); - var local = getLocalUri(dependency, packageAssetUri); + getProjectDependenciesManager().getResolvedDependency(packageAssetUri.getPackageUri()); + var local = getLocalUri(dependency); if (local != null) { - return VmContext.get(null).getModuleResolver().resolve(local).resolve(securityManager); + var resolved = + VmContext.get(null).getModuleResolver().resolve(local).resolve(securityManager); + return ResolvedModuleKeys.delegated(resolved, this); } var dep = (Dependency.RemoteDependency) dependency; assert dep.getChecksums() != null; @@ -728,13 +706,15 @@ public List listElements(SecurityManager securityManager, URI baseU securityManager.checkResolveModule(baseUri); var packageAssetUri = PackageAssetUri.create(baseUri); var dependency = - getProjectDepsResolver().getResolvedDependency(packageAssetUri.getPackageUri()); + getProjectDependenciesManager().getResolvedDependency(packageAssetUri.getPackageUri()); var local = getLocalUri(dependency, packageAssetUri); if (local != null) { - return VmContext.get(null) - .getModuleResolver() - .resolve(local) - .listElements(securityManager, local); + var moduleKey = VmContext.get(null).getModuleResolver().resolve(local); + if (!moduleKey.isGlobbable()) { + throw new PackageLoadError( + "cannotResolveInLocalDependencyNotGlobbable", local.getScheme()); + } + return moduleKey.listElements(securityManager, local); } var dep = (Dependency.RemoteDependency) dependency; assert dep.getChecksums() != null; @@ -747,44 +727,36 @@ public boolean hasElement(SecurityManager securityManager, URI elementUri) securityManager.checkResolveModule(elementUri); var packageAssetUri = PackageAssetUri.create(elementUri); var dependency = - getProjectDepsResolver().getResolvedDependency(packageAssetUri.getPackageUri()); + getProjectDependenciesManager().getResolvedDependency(packageAssetUri.getPackageUri()); var local = getLocalUri(dependency, packageAssetUri); if (local != null) { - return VmContext.get(null) - .getModuleResolver() - .resolve(local) - .hasElement(securityManager, local); + var moduleKey = VmContext.get(null).getModuleResolver().resolve(local); + if (!moduleKey.isGlobbable() && !moduleKey.isLocal()) { + throw new PackageLoadError( + "cannotResolveInLocalDependencyNotGlobbableNorLocal", local.getScheme()); + } + return moduleKey.hasElement(securityManager, local); } var dep = (Dependency.RemoteDependency) dependency; assert dep.getChecksums() != null; return getPackageResolver().hasElement(packageAssetUri, dep.getChecksums()); } - @Override - public boolean hasFragmentPaths() { - return true; - } - @Override protected Map getDependencies() throws IOException, SecurityManagerException { var packageUri = packageAssetUri.getPackageUri(); - var projectResolver = getProjectDepsResolver(); + var projectResolver = getProjectDependenciesManager(); if (projectResolver.isLocalPackage(packageUri)) { return projectResolver.getLocalPackageDependencies(packageUri); } var dep = - (Dependency.RemoteDependency) getProjectDepsResolver().getResolvedDependency(packageUri); + (Dependency.RemoteDependency) + getProjectDependenciesManager().getResolvedDependency(packageUri); assert dep.getChecksums() != null; var dependencyMetadata = getPackageResolver().getDependencyMetadata(packageUri, dep.getChecksums()); return projectResolver.getResolvedDependenciesForPackage(packageUri, dependencyMetadata); } - - @Override - protected PackageLoadError cannotFindDependency(String name) { - return new PackageLoadError( - "cannotFindDependencyInPackage", name, packageAssetUri.getPackageUri().getDisplayName()); - } } } diff --git a/pkl-core/src/main/java/org/pkl/core/module/ProjectDependenciesManager.java b/pkl-core/src/main/java/org/pkl/core/module/ProjectDependenciesManager.java index 6e5855c35..50edc67aa 100644 --- a/pkl-core/src/main/java/org/pkl/core/module/ProjectDependenciesManager.java +++ b/pkl-core/src/main/java/org/pkl/core/module/ProjectDependenciesManager.java @@ -15,14 +15,16 @@ */ package org.pkl.core.module; +import java.io.IOException; import java.net.URI; -import java.net.URISyntaxException; import java.util.HashMap; import java.util.Map; import java.util.Objects; import javax.annotation.concurrent.GuardedBy; import org.graalvm.collections.EconomicMap; import org.pkl.core.PklBugException; +import org.pkl.core.SecurityManager; +import org.pkl.core.SecurityManagerException; import org.pkl.core.packages.Dependency; import org.pkl.core.packages.DependencyMetadata; import org.pkl.core.packages.PackageLoadError; @@ -30,10 +32,8 @@ import org.pkl.core.project.CanonicalPackageUri; import org.pkl.core.project.DeclaredDependencies; import org.pkl.core.project.ProjectDeps; -import org.pkl.core.resource.Resource; -import org.pkl.core.runtime.VmContext; +import org.pkl.core.runtime.ModuleResolver; import org.pkl.core.runtime.VmExceptionBuilder; -import org.pkl.core.runtime.VmTyped; import org.pkl.core.util.EconomicMaps; import org.pkl.core.util.json.Json.JsonParseException; @@ -44,6 +44,8 @@ public final class ProjectDependenciesManager { private final DeclaredDependencies declaredDependencies; private final URI projectBaseUri; + private final ModuleResolver moduleResolver; + private final SecurityManager securityManager; @GuardedBy("lock") private ProjectDeps projectDeps; @@ -61,10 +63,15 @@ public final class ProjectDependenciesManager { private final Object lock = new Object(); - public ProjectDependenciesManager(DeclaredDependencies declaredDependencies) { + public ProjectDependenciesManager( + DeclaredDependencies declaredDependencies, + ModuleResolver moduleResolver, + SecurityManager securityManager) { this.declaredDependencies = declaredDependencies; // new URI("scheme://host/a/b/c.txt").resolve(".") == new URI("scheme://host/a/b/") this.projectBaseUri = declaredDependencies.getProjectFileUri().resolve("."); + this.moduleResolver = moduleResolver; + this.securityManager = securityManager; } public boolean hasUri(URI uri) { @@ -207,39 +214,34 @@ public URI getProjectDepsFileUri() { return projectBaseUri.resolve(PKL_PROJECT_DEPS_FILENAME); } + public URI getProjectFileUri() { + return declaredDependencies.getProjectFileUri(); + } + private ProjectDeps getProjectDeps() { synchronized (lock) { if (projectDeps == null) { var depsUri = getProjectDepsFileUri(); - + var moduleKey = moduleResolver.resolve(depsUri); try { - var resource = VmContext.get(null).getResourceManager().read(depsUri, null); - if (resource.isEmpty()) { - throw new VmExceptionBuilder() - .evalError("missingProjectDepsJson", projectBaseUri) - .build(); - } - - var res = resource.get(); - if (res instanceof String) { - projectDeps = ProjectDeps.parse((String) res); - } else if (res instanceof VmTyped) { - var resInner = ((VmTyped) res).getExtraStorage(); - if (resInner instanceof Resource) { - projectDeps = ProjectDeps.parse(((Resource) resInner).getText()); - } else { - // ResourceManager.read() already catches this condition - throw PklBugException.unreachableCode(); - } + // treat PklProject.deps.json as a module read, rather than introduce a new API. + var depsJson = moduleKey.resolve(securityManager).loadSource(); + projectDeps = ProjectDeps.parse(depsJson); + } catch (IOException e) { + var exceptionBuilder = + new VmExceptionBuilder().evalError("cannotLoadProjectDepsJson", depsUri).withCause(e); + if (e.getMessage() != null) { + exceptionBuilder.withHint(e.getMessage()); } else { - // ResourceManager.read() already catches this condition - throw PklBugException.unreachableCode(); + exceptionBuilder.withHint("Encountered error: " + e); } - } catch (URISyntaxException | JsonParseException e) { + throw exceptionBuilder.build(); + } catch (JsonParseException e) { throw new VmExceptionBuilder() .evalError("invalidProjectDepsJson", depsUri, e.getMessage()) - .withCause(e) .build(); + } catch (SecurityManagerException e) { + throw PklBugException.unreachableCode(); } } return projectDeps; diff --git a/pkl-core/src/main/java/org/pkl/core/module/ResolvedModuleKeys.java b/pkl-core/src/main/java/org/pkl/core/module/ResolvedModuleKeys.java index 647bd9c3e..a0a863efd 100644 --- a/pkl-core/src/main/java/org/pkl/core/module/ResolvedModuleKeys.java +++ b/pkl-core/src/main/java/org/pkl/core/module/ResolvedModuleKeys.java @@ -52,6 +52,14 @@ public static ResolvedModuleKey virtual( return new Virtual(original, uri, sourceText, cached); } + /** + * Creates a resolved module key that behaves like {@code delegate}, except with {@code original} + * as its original module key. + */ + public static ResolvedModuleKey delegated(ResolvedModuleKey delegate, ModuleKey original) { + return new Delegated(delegate, original); + } + private static class File implements ResolvedModuleKey { final ModuleKey original; final URI uri; @@ -134,4 +142,30 @@ public String loadSource() { return sourceText; } } + + private static class Delegated implements ResolvedModuleKey { + + private final ResolvedModuleKey delegate; + private final ModuleKey original; + + Delegated(ResolvedModuleKey delegate, ModuleKey original) { + this.delegate = delegate; + this.original = original; + } + + @Override + public ModuleKey getOriginal() { + return original; + } + + @Override + public URI getUri() { + return delegate.getUri(); + } + + @Override + public String loadSource() throws IOException { + return delegate.loadSource(); + } + } } diff --git a/pkl-core/src/main/java/org/pkl/core/project/Project.java b/pkl-core/src/main/java/org/pkl/core/project/Project.java index 0cce2057a..c8e4c662d 100644 --- a/pkl-core/src/main/java/org/pkl/core/project/Project.java +++ b/pkl-core/src/main/java/org/pkl/core/project/Project.java @@ -28,6 +28,7 @@ import java.util.stream.Collectors; import org.pkl.core.Composite; import org.pkl.core.Duration; +import org.pkl.core.Evaluator; import org.pkl.core.EvaluatorBuilder; import org.pkl.core.ModuleSource; import org.pkl.core.PClassInfo; @@ -83,10 +84,7 @@ public static Project loadFromPath( .addEnvironmentVariables(envVars) .setTimeout(timeout) .build()) { - var output = evaluator.evaluateOutputValueAs(ModuleSource.path(path), PClassInfo.Project); - return Project.parseProject(output); - } catch (URISyntaxException e) { - throw new PklException(e.getMessage(), e); + return load(evaluator, ModuleSource.path(path)); } } @@ -105,6 +103,31 @@ public static Project loadFromPath(Path path) { return loadFromPath(path, SecurityManagers.defaultManager, null); } + /** Loads a project from the given source. */ + public static Project load(ModuleSource moduleSource) { + try (var evaluator = + EvaluatorBuilder.unconfigured() + .setSecurityManager(SecurityManagers.defaultManager) + .setStackFrameTransformer(StackFrameTransformers.defaultTransformer) + .addModuleKeyFactory(ModuleKeyFactories.standardLibrary) + .addModuleKeyFactory(ModuleKeyFactories.file) + .addModuleKeyFactory(ModuleKeyFactories.classPath(Project.class.getClassLoader())) + .addResourceReader(ResourceReaders.environmentVariable()) + .addResourceReader(ResourceReaders.file()) + .build()) { + return load(evaluator, moduleSource); + } + } + + public static Project load(Evaluator evaluator, ModuleSource moduleSource) { + try { + var output = evaluator.evaluateOutputValueAs(moduleSource, PClassInfo.Project); + return Project.parseProject(output); + } catch (URISyntaxException e) { + throw new PklException(e.getMessage(), e); + } + } + private static DeclaredDependencies parseDependencies( PObject module, URI projectFileUri, @Nullable PackageUri packageUri) throws URISyntaxException { @@ -266,8 +289,9 @@ private static T getProperty(PObject settings, String propertyName, Function } /** - * Resolve a path string against projectBaseUri Throws an exception if projectBaseUri is not a - * file: URI + * Resolve a path string against projectBaseUri. + * + * @throws PackageLoadError if projectBaseUri is not a {@code file:} URI. */ private static @Nullable Path resolveNullablePath( @Nullable String path, URI projectBaseUri, String propertyName) { @@ -401,11 +425,8 @@ public URI getProjectBaseUri() { } public Path getProjectDir() { - try { - return Path.of(projectBaseUri); - } catch (FileSystemNotFoundException e) { - throw new PackageLoadError("invalidUsageOfProjectFromNonFileUri", projectBaseUri); - } + assert projectBaseUri.getScheme().equalsIgnoreCase("file"); + return Path.of(projectBaseUri); } public static class EvaluatorSettings { diff --git a/pkl-core/src/main/java/org/pkl/core/project/ProjectPackager.java b/pkl-core/src/main/java/org/pkl/core/project/ProjectPackager.java index f1bffab39..9bfddf8b6 100644 --- a/pkl-core/src/main/java/org/pkl/core/project/ProjectPackager.java +++ b/pkl-core/src/main/java/org/pkl/core/project/ProjectPackager.java @@ -43,6 +43,7 @@ import org.pkl.core.StackFrameTransformer; import org.pkl.core.ast.builder.ImportsAndReadsParser; import org.pkl.core.http.HttpClient; +import org.pkl.core.module.ModuleKeyFactories; import org.pkl.core.module.ModuleKeys; import org.pkl.core.module.ProjectDependenciesManager; import org.pkl.core.module.ResolvedModuleKeys; @@ -53,6 +54,7 @@ import org.pkl.core.packages.PackageLoadError; import org.pkl.core.packages.PackageResolver; import org.pkl.core.packages.PackageUri; +import org.pkl.core.runtime.ModuleResolver; import org.pkl.core.runtime.VmExceptionBuilder; import org.pkl.core.util.ByteArrayUtils; import org.pkl.core.util.ErrorMessages; @@ -96,6 +98,7 @@ public final class ProjectPackager { private final Path workingDir; private final String outputPathPattern; private final StackFrameTransformer stackFrameTransformer; + private final SecurityManager securityManager; private final PackageResolver packageResolver; private final boolean skipPublishCheck; private final Writer outputWriter; @@ -113,6 +116,7 @@ public ProjectPackager( this.workingDir = workingDir; this.outputPathPattern = outputPathPattern; this.stackFrameTransformer = stackFrameTransformer; + this.securityManager = securityManager; // intentionally use InMemoryPackageResolver this.packageResolver = PackageResolver.getInstance(securityManager, httpClient, null); this.skipPublishCheck = skipPublishCheck; @@ -219,7 +223,12 @@ private Map buildDependencies(Project project) throws new HashMap( project.getDependencies().getLocalDependencies().size() + project.getDependencies().getRemoteDependencies().size()); - var projectDependenciesManager = new ProjectDependenciesManager(project.getDependencies()); + // module resolver is only used for reading PklProject.deps.json, so provide one that reads + // files. + var moduleResolver = new ModuleResolver(List.of(ModuleKeyFactories.file)); + var projectDependenciesManager = + new ProjectDependenciesManager( + project.getDependencies(), moduleResolver, this.securityManager); for (var entry : project.getDependencies().getRemoteDependencies().entrySet()) { var resolved = (RemoteDependency) diff --git a/pkl-core/src/main/java/org/pkl/core/repl/ReplServer.java b/pkl-core/src/main/java/org/pkl/core/repl/ReplServer.java index 711f1b5e9..a51220ac1 100644 --- a/pkl-core/src/main/java/org/pkl/core/repl/ReplServer.java +++ b/pkl-core/src/main/java/org/pkl/core/repl/ReplServer.java @@ -89,7 +89,9 @@ public ReplServer( var languageRef = new MutableReference(null); packageResolver = PackageResolver.getInstance(securityManager, httpClient, moduleCacheDir); projectDependenciesManager = - projectDependencies == null ? null : new ProjectDependenciesManager(projectDependencies); + projectDependencies == null + ? null + : new ProjectDependenciesManager(projectDependencies, moduleResolver, securityManager); polyglotContext = VmUtils.createContext( () -> { diff --git a/pkl-core/src/main/java/org/pkl/core/resource/ResourceReaders.java b/pkl-core/src/main/java/org/pkl/core/resource/ResourceReaders.java index c5a372837..856100570 100644 --- a/pkl-core/src/main/java/org/pkl/core/resource/ResourceReaders.java +++ b/pkl-core/src/main/java/org/pkl/core/resource/ResourceReaders.java @@ -38,6 +38,7 @@ import org.pkl.core.packages.PackageAssetUri; import org.pkl.core.packages.PackageResolver; import org.pkl.core.runtime.VmContext; +import org.pkl.core.runtime.VmExceptionBuilder; import org.pkl.core.util.ErrorMessages; import org.pkl.core.util.HttpUtils; import org.pkl.core.util.IoUtils; @@ -518,7 +519,13 @@ public List listElements(SecurityManager securityManager, URI baseU getProjectDepsResolver().getResolvedDependency(packageAssetUri.getPackageUri()); var local = getLocalUri(dependency, packageAssetUri); if (local != null) { - return VmContext.get(null).getResourceManager().listElements(local); + var reader = VmContext.get(null).getResourceManager().getResourceReader(local); + if (reader == null) { + throw new VmExceptionBuilder() + .evalError("noResourceReaderRegistered", local.getScheme()) + .build(); + } + return reader.listElements(securityManager, local); } var remoteDep = (Dependency.RemoteDependency) dependency; return getPackageResolver() @@ -534,7 +541,13 @@ public boolean hasElement(SecurityManager securityManager, URI elementUri) getProjectDepsResolver().getResolvedDependency(packageAssetUri.getPackageUri()); var local = getLocalUri(dependency, packageAssetUri); if (local != null) { - return VmContext.get(null).getResourceManager().hasElement(local); + var reader = VmContext.get(null).getResourceManager().getResourceReader(local); + if (reader == null) { + throw new VmExceptionBuilder() + .evalError("noResourceReaderRegistered", local.getScheme()) + .build(); + } + return reader.hasElement(securityManager, local); } var remoteDep = (Dependency.RemoteDependency) dependency; return getPackageResolver() diff --git a/pkl-core/src/main/java/org/pkl/core/runtime/ResourceManager.java b/pkl-core/src/main/java/org/pkl/core/runtime/ResourceManager.java index fcafee4b3..f83b27c43 100644 --- a/pkl-core/src/main/java/org/pkl/core/runtime/ResourceManager.java +++ b/pkl-core/src/main/java/org/pkl/core/runtime/ResourceManager.java @@ -29,7 +29,6 @@ import org.pkl.core.SecurityManagerException; import org.pkl.core.http.HttpClientInitException; import org.pkl.core.module.ModuleKey; -import org.pkl.core.module.PathElement; import org.pkl.core.packages.PackageLoadError; import org.pkl.core.resource.Resource; import org.pkl.core.resource.ResourceReader; @@ -37,6 +36,7 @@ import org.pkl.core.util.GlobResolver; import org.pkl.core.util.GlobResolver.InvalidGlobPatternException; import org.pkl.core.util.GlobResolver.ResolvedGlobElement; +import org.pkl.core.util.IoUtils; import org.pkl.core.util.Nullable; public final class ResourceManager { @@ -81,8 +81,8 @@ public List resolveGlob( var scheme = uri.getScheme(); URI resolvedUri; try { - resolvedUri = enclosingModuleKey.resolveUri(globUri); - } catch (SecurityManagerException | IOException e) { + resolvedUri = IoUtils.resolve(securityManager, enclosingModuleKey, globUri); + } catch (SecurityManagerException | URISyntaxException | IOException e) { throw new VmExceptionBuilder().withLocation(readNode).withCause(e).build(); } try { @@ -174,32 +174,10 @@ public Optional read(URI resourceUri, @Nullable Node readNode) { } /** - * Used by ResourceReaders.ProjectPackageResource to resolve resources from projects that may not - * be on the local filesystem + * Returns a {@link ResourceReader} registered to read the resource at {@code baseUri}, or {@code + * null} if there is none. */ - public List listElements(URI baseUri) throws IOException, SecurityManagerException { - var reader = resourceReaders.get(baseUri.getScheme()); - if (reader == null) { - throw new VmExceptionBuilder() - .evalError("noResourceReaderRegistered", baseUri.getScheme()) - .build(); - } - - return reader.listElements(securityManager, baseUri); - } - - /** - * Used by ResourceReaders.ProjectPackageResource to resolve resources from projects that may not - * be on the local filesystem - */ - public boolean hasElement(URI elementUri) throws IOException, SecurityManagerException { - var reader = resourceReaders.get(elementUri.getScheme()); - if (reader == null) { - throw new VmExceptionBuilder() - .evalError("noResourceReaderRegistered", elementUri.getScheme()) - .build(); - } - - return reader.hasElement(securityManager, elementUri); + public @Nullable ResourceReader getResourceReader(URI baseUri) { + return resourceReaders.get(baseUri.getScheme()); } } diff --git a/pkl-core/src/main/java/org/pkl/core/runtime/VmExceptionBuilder.java b/pkl-core/src/main/java/org/pkl/core/runtime/VmExceptionBuilder.java index 9f580183f..6fba20804 100644 --- a/pkl-core/src/main/java/org/pkl/core/runtime/VmExceptionBuilder.java +++ b/pkl-core/src/main/java/org/pkl/core/runtime/VmExceptionBuilder.java @@ -324,7 +324,7 @@ public VmExceptionBuilder withCause(Throwable cause) { return this; } - public VmExceptionBuilder withHint(String hint) { + public VmExceptionBuilder withHint(@Nullable String hint) { this.hint = hint; return this; } diff --git a/pkl-core/src/main/java/org/pkl/core/util/GlobResolver.java b/pkl-core/src/main/java/org/pkl/core/util/GlobResolver.java index 52b9b4b02..cda7e6ec5 100644 --- a/pkl-core/src/main/java/org/pkl/core/util/GlobResolver.java +++ b/pkl-core/src/main/java/org/pkl/core/util/GlobResolver.java @@ -27,6 +27,7 @@ import java.util.WeakHashMap; import java.util.regex.Pattern; import java.util.stream.Collectors; +import org.pkl.core.PklBugException; import org.pkl.core.SecurityManager; import org.pkl.core.SecurityManagerException; import org.pkl.core.module.ModuleKey; @@ -488,7 +489,14 @@ public static List resolveGlob( } return result; } - var baseUri = enclosingModuleKey.resolveUri(enclosingUri, URI.create(basePath)); + URI baseUri; + try { + baseUri = IoUtils.resolve(securityManager, enclosingModuleKey, URI.create(basePath)); + } catch (URISyntaxException e) { + // assertion: this is only thrown if the pattern starts with a triple-dot import. + // the language will throw an error if glob imports is combined with triple-dots. + throw new PklBugException(e); + } resolveHierarchicalGlob( securityManager, diff --git a/pkl-core/src/main/java/org/pkl/core/util/IoUtils.java b/pkl-core/src/main/java/org/pkl/core/util/IoUtils.java index a5da3df2d..e055fe197 100644 --- a/pkl-core/src/main/java/org/pkl/core/util/IoUtils.java +++ b/pkl-core/src/main/java/org/pkl/core/util/IoUtils.java @@ -38,7 +38,9 @@ import org.pkl.core.SecurityManager; import org.pkl.core.SecurityManagerException; import org.pkl.core.module.ModuleKey; +import org.pkl.core.packages.PackageLoadError; import org.pkl.core.runtime.ReaderBase; +import org.pkl.core.runtime.VmContext; import org.pkl.core.runtime.VmExceptionBuilder; public final class IoUtils { @@ -282,29 +284,15 @@ public static URI resolve(ReaderBase reader, URI baseUri, URI importUri) { return resolve(baseUri, importUri); } - /** - * Resolves {@code importUri} against the module key. - * - *

When {@code importUri} contains a triple-dot, it is resolved if the module key returns true - * for both {@link ModuleKey#isLocal()} and {@link ModuleKey#hasHierarchicalUris()}. Otherwise, an - * error is thrown. - */ - public static URI resolve(SecurityManager securityManager, ModuleKey moduleKey, URI importUri) - throws URISyntaxException, IOException, SecurityManagerException { - if (importUri.isAbsolute()) { - return moduleKey.resolveUri(importUri); - } + private static URI resolveTripleDotImport( + SecurityManager securityManager, ModuleKey moduleKey, String tripleDotPath) + throws IOException, SecurityManagerException { var moduleKeyUri = moduleKey.getUri(); - var tripleDotPath = parseTripleDotPath(importUri); - if (tripleDotPath == null) { - return moduleKey.resolveUri(importUri); - } if (!moduleKey.isLocal() || !moduleKey.hasHierarchicalUris()) { throw new VmExceptionBuilder() .evalError("cannotResolveTripleDotImports", moduleKeyUri) .build(); } - var currentPath = moduleKey.hasFragmentPaths() ? moduleKeyUri.getFragment() : moduleKeyUri.getPath(); var effectiveImportPath = @@ -334,6 +322,69 @@ public static URI resolve(SecurityManager securityManager, ModuleKey moduleKey, throw new FileNotFoundException(); } + public static Pair parseDependencyNotation(String importPath) { + var idx = importPath.indexOf('/'); + if (idx == -1) { + // treat named dependency without a subpath as the root path. + // i.e. resolve to `@foo` to `package://example.com/foo@1.0.0#/` + return Pair.of(importPath.substring(1), "/"); + } + return Pair.of(importPath.substring(1, idx), importPath.substring(idx)); + } + + private static URI resolveProjectDependency(ModuleKey moduleKey, String notation) { + var parsed = parseDependencyNotation(notation); + var name = parsed.getFirst(); + var path = parsed.getSecond(); + var projectDependenciesManager = VmContext.get(null).getProjectDependenciesManager(); + if (!moduleKey.hasHierarchicalUris() && projectDependenciesManager != null) { + throw new PackageLoadError( + "cannotResolveDependencyWithoutHierarchicalUris", + projectDependenciesManager.getProjectFileUri()); + } + if (projectDependenciesManager == null + || !projectDependenciesManager.hasUri(moduleKey.getUri())) { + throw new PackageLoadError("cannotResolveDependencyNoProject"); + } + var dependency = projectDependenciesManager.getDependencies().get(name); + if (dependency != null) { + return dependency.getPackageUri().toPackageAssetUri(path).getUri(); + } + throw new PackageLoadError("cannotFindDependencyInProject", name); + } + + /** + * Resolves {@code importUri} against the module key. + * + *

When {@code importUri} contains a triple-dot, it is resolved if the module key returns true + * for both {@link ModuleKey#isLocal()} and {@link ModuleKey#hasHierarchicalUris()}. Otherwise, an + * error is thrown. + * + *

When {@code importUri} starts with a {@code @}, it is resolved if the module key supports + * dependency notation () + */ + public static URI resolve(SecurityManager securityManager, ModuleKey moduleKey, URI importUri) + throws URISyntaxException, IOException, SecurityManagerException { + if (importUri.isAbsolute()) { + return moduleKey.resolveUri(importUri); + } + var tripleDotPath = parseTripleDotPath(importUri); + if (tripleDotPath != null) { + return resolveTripleDotImport(securityManager, moduleKey, tripleDotPath); + } + var moduleScheme = moduleKey.getUri().getScheme(); + var isPackage = + moduleScheme.equalsIgnoreCase("package") || moduleScheme.equalsIgnoreCase("projectpackage"); + var relativePart = importUri.getSchemeSpecificPart(); + // Special-case handling of project dependencies. + // We'll allow the Package and ProjectPackage module keys to resolve dependency notation on + // their own. + if (relativePart.startsWith("@") && !isPackage) { + return resolveProjectDependency(moduleKey, relativePart); + } + return moduleKey.resolveUri(importUri); + } + public static URI resolve(URI baseUri, URI newUri) { if (newUri.isAbsolute()) return newUri; diff --git a/pkl-core/src/main/resources/org/pkl/core/errorMessages.properties b/pkl-core/src/main/resources/org/pkl/core/errorMessages.properties index a6fd4f2f7..e5b106a7f 100644 --- a/pkl-core/src/main/resources/org/pkl/core/errorMessages.properties +++ b/pkl-core/src/main/resources/org/pkl/core/errorMessages.properties @@ -542,6 +542,12 @@ Cannot combine glob imports with triple-dot module URIs. cannotGlobUri=\ Cannot expand glob pattern `{0}` because scheme `{1}` is not globbable. +cannotResolveInLocalDependencyNotGlobbable=\ +Cannot resolve import in local dependency because scheme `{0}` is not globbable. + +cannotResolveInLocalDependencyNotGlobbableNorLocal=\ +Cannot resolve import in local dependency because scheme `{0}` is not globbable and is not local. + expectedAnnotationClass=\ Expected an annotation class. @@ -680,6 +686,11 @@ Cannot resolve a triple-dot import from module URI `{0}`.\n\ \n\ Triple-dot imports may only be resolved by module schemes that are considered local, and have hierarchical URIs. +moduleDoesNotSupportDependencies=\ +Module `{0}` does not support importing dependencies.\n\ +\n\ +Dependencies can only be imported in modules that belong to a project, or within a package. + cannotHaveRelativeImport=\ Module `{0}` cannot have a relative import URI. @@ -819,6 +830,9 @@ Only type unions can have a default marker (*). invalidModuleOutputValue=\ Expected `output.value` of module `{2}` to be of type `{0}`, but got type `{1}`. +cannotResolveDependencyWithoutHierarchicalUris=\ +Cannot import dependency because project URI `{0}` does not have a hierarchical path. + cannotResolveDependencyNoProject=\ Cannot import dependency because there is no project found.\n\ \n\ @@ -861,10 +875,10 @@ For example, `example.com` in URI `project://example.com/my/package@1.0.0`. unexpectedChecksumInPackageUri=\ Did not expect to find a checksum component in this package URI. -missingProjectDepsJson=\ -Cannot resolve dependency because file `PklProject.deps.json` is missing in project directory `{0}`.\n\ +cannotLoadProjectDepsJson=\ +Encountered an error when attempting to load PklProject.deps.json at `{0}`.\n\ \n\ -Run `pkl project resolve` to create a new set of dependencies. +Try running `pkl project resolve` within the project directory to create a new set of dependencies. invalidProjectDepsJson=\ Cannot resolve dependency because file `{0}` is malformed.\n\ diff --git a/pkl-core/src/test/files/LanguageSnippetTests/input/projects/project1/globbing.pkl b/pkl-core/src/test/files/LanguageSnippetTests/input/projects/project1/globbing.pkl index 59c46689e..82b880304 100644 --- a/pkl-core/src/test/files/LanguageSnippetTests/input/projects/project1/globbing.pkl +++ b/pkl-core/src/test/files/LanguageSnippetTests/input/projects/project1/globbing.pkl @@ -36,7 +36,7 @@ examples { } // https://github.com/apple/pkl/issues/166 -// ["glob-import behind local project import"] { -// import("@project6/children.pkl") -// } + ["glob-import behind local project import"] { + import("@project6/children.pkl") + } } diff --git a/pkl-core/src/test/files/LanguageSnippetTests/output/projects/badProjectDeps4/bug.err b/pkl-core/src/test/files/LanguageSnippetTests/output/projects/badProjectDeps4/bug.err index 0bbf5b7ab..fafb56001 100644 --- a/pkl-core/src/test/files/LanguageSnippetTests/output/projects/badProjectDeps4/bug.err +++ b/pkl-core/src/test/files/LanguageSnippetTests/output/projects/badProjectDeps4/bug.err @@ -1,9 +1,9 @@ –– Pkl Error –– -Expected value of type `* RemoteDependency|Project(isValidLoadDependency)`, but got a different `pkl.Project`. +Expected value of type `*RemoteDependency|Project(isValidLoadDependency)`, but got a different `pkl.Project`. Value: new ModuleClass { package = null; tests {}; dependencies {}; evaluatorSetting... -xxx | dependencies: Mapping - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +xxx | dependencies: Mapping + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ at pkl.Project#dependencies (pkl:Project) * Value is not of type `Project(isValidLoadDependency)` because: diff --git a/pkl-core/src/test/files/LanguageSnippetTests/output/projects/missingProjectDeps/bug.err b/pkl-core/src/test/files/LanguageSnippetTests/output/projects/missingProjectDeps/bug.err index f02231a5b..48d93906c 100644 --- a/pkl-core/src/test/files/LanguageSnippetTests/output/projects/missingProjectDeps/bug.err +++ b/pkl-core/src/test/files/LanguageSnippetTests/output/projects/missingProjectDeps/bug.err @@ -1,8 +1,9 @@ –– Pkl Error –– -Cannot resolve dependency because file `PklProject.deps.json` is missing in project directory `file:/$snippetsDir/input/projects/missingProjectDeps/`. +Encountered an error when attempting to load PklProject.deps.json at `file:/$snippetsDir/input/projects/missingProjectDeps/PklProject.deps.json`. +NoSuchFileException: /$snippetsDir/input/projects/missingProjectDeps/PklProject.deps.json x | import "@birds/Bird.pkl" ^^^^^^^^^^^^^^^^^ at bug (file:///$snippetsDir/input/projects/missingProjectDeps/bug.pkl) -Run `pkl project resolve` to create a new set of dependencies. +Try running `pkl project resolve` within the project directory to create a new set of dependencies. diff --git a/pkl-core/src/test/files/LanguageSnippetTests/output/projects/project1/globbing.pcf b/pkl-core/src/test/files/LanguageSnippetTests/output/projects/project1/globbing.pcf index 7fd20dcad..3f000af67 100644 --- a/pkl-core/src/test/files/LanguageSnippetTests/output/projects/project1/globbing.pcf +++ b/pkl-core/src/test/files/LanguageSnippetTests/output/projects/project1/globbing.pcf @@ -262,4 +262,19 @@ examples { } } } + ["glob-import behind local project import"] { + new { + children { + ["children/a.pkl"] { + name = "a" + } + ["children/b.pkl"] { + name = "b" + } + ["children/c.pkl"] { + name = "c" + } + } + } + } } diff --git a/pkl-core/src/test/kotlin/org/pkl/core/EvaluatorCustomReaders.kt b/pkl-core/src/test/kotlin/org/pkl/core/EvaluatorCustomReaders.kt deleted file mode 100644 index 4886ce2a3..000000000 --- a/pkl-core/src/test/kotlin/org/pkl/core/EvaluatorCustomReaders.kt +++ /dev/null @@ -1,140 +0,0 @@ -/** - * Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.pkl.core - -import org.pkl.core.module.* -import org.pkl.core.module.ModuleKeys.DependencyAwareModuleKey -import org.pkl.core.packages.Dependency -import org.pkl.core.packages.PackageLoadError -import org.pkl.core.resource.Resource -import org.pkl.core.resource.ResourceReader -import org.pkl.core.runtime.VmContext -import org.pkl.core.util.IoUtils -import java.io.FileNotFoundException -import java.net.URI -import java.nio.file.Files -import java.nio.file.Path -import java.util.* - -/** - * Provides Module- and Resource readers for use in tests - * Readers are provided a URI scheme and - */ -class EvaluatorCustomReaders { - - class CustomModuleKey(uri: URI, private val basePath: Path) : DependencyAwareModuleKey(uri) { - - override fun hasHierarchicalUris(): Boolean = true - override fun isGlobbable(): Boolean = true - - override fun hasElement(securityManager: SecurityManager, elementUri: URI): Boolean { - securityManager.checkResolveModule(elementUri); - val realPath = basePath.resolve(elementUri.path.removePrefix("/")).toRealPath() - if (!realPath.startsWith(basePath)) { - throw PklBugException("attempt to access path $realPath above base path $basePath") - } - return FileResolver.hasElement(realPath) - } - - override fun listElements(securityManager: SecurityManager, baseUri: URI): MutableList { - securityManager.checkResolveModule(baseUri) - val realPath = basePath.resolve(baseUri.path.removePrefix("/")).toRealPath() - if (!realPath.startsWith(basePath)) { - throw PklBugException("attempt to access path $realPath above base path $basePath") - } - return FileResolver.listElements(realPath) - } - - override fun resolve(securityManager: SecurityManager): ResolvedModuleKey { - securityManager.checkResolveModule(uri); - val realPath = basePath.toRealPath() - if (!realPath.startsWith(basePath) && realPath != basePath) { - throw PklBugException("attempt to access path $realPath above base path $basePath") - } - return ResolvedModuleKeys.file(this, realPath.toUri(), realPath) - } - - override fun getDependencies(): MutableMap { - val projectDepsManager = VmContext.get(null).projectDependenciesManager - if (projectDepsManager == null || !projectDepsManager.hasUri(uri)) { - throw PackageLoadError("cannotResolveDependencyNoProject") - } - return projectDepsManager.dependencies - } - - override fun cannotFindDependency(name: String): PackageLoadError { - return PackageLoadError("cannotFindDependencyInProject", name) - } - - } - - class CustomModuleKeyFactory(private val uriScheme: String, private val dir: Path) : ModuleKeyFactory { - - override fun create(uri: URI): Optional { - if (uri.scheme != uriScheme) { - return Optional.empty() - } - - val resolvedPath = dir.resolve(Path.of(uri.path.removePrefix("/"))).toRealPath() - return Optional.of(CustomModuleKey(uri, resolvedPath)) - } - - } - - class CustomResourceReader(private val customUriScheme: String, base: Path) : ResourceReader { - - private val basePath = base.toRealPath() - override fun hasHierarchicalUris(): Boolean = true - - override fun isGlobbable(): Boolean = true - - override fun getUriScheme(): String = customUriScheme - - override fun hasElement(securityManager: SecurityManager, elementUri: URI): Boolean { - securityManager.checkResolveModule(elementUri); - val realPath = basePath.resolve(elementUri.path.removePrefix("/")).toRealPath() - if (!realPath.startsWith(basePath)) { - throw PklBugException("attempt to access path $realPath above base path $basePath") - } - return FileResolver.hasElement(realPath) - } - - override fun listElements(securityManager: SecurityManager, baseUri: URI): MutableList { - securityManager.checkResolveModule(baseUri) - val realPath = basePath.resolve(baseUri.path.removePrefix("/")).toRealPath() - if (!realPath.startsWith(basePath)) { - throw PklBugException("attempt to access path $realPath above base path $basePath") - } - return FileResolver.listElements(realPath) - } - - override fun read(uri: URI): Optional { - val realPath = basePath.resolve(uri.path.removePrefix("/")).toRealPath() - if (!realPath.startsWith(basePath)) { - throw PklBugException("attempt to access path $realPath above base path $basePath") - } - - try { - val content = Files.readAllBytes(realPath) - return Optional.of(Resource(uri, content)) - } catch (e: FileNotFoundException) { - return Optional.empty() - } - } - - } - -} diff --git a/pkl-core/src/test/kotlin/org/pkl/core/EvaluatorCustomReadersTest.kt b/pkl-core/src/test/kotlin/org/pkl/core/EvaluatorCustomReadersTest.kt deleted file mode 100644 index df8e97f27..000000000 --- a/pkl-core/src/test/kotlin/org/pkl/core/EvaluatorCustomReadersTest.kt +++ /dev/null @@ -1,160 +0,0 @@ -/** - * Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.pkl.core - -import org.assertj.core.api.Assertions.assertThat -import org.junit.jupiter.api.Test -import org.junit.jupiter.api.io.TempDir -import org.pkl.commons.test.PackageServer -import org.pkl.core.module.ModuleKeyFactories -import org.pkl.core.project.Project -import org.pkl.core.resource.ResourceReaders -import java.nio.file.Path -import java.util.regex.Pattern -import kotlin.io.path.createDirectories -import kotlin.io.path.writeText - -class EvaluatorCustomReadersTest { - @Test - fun `evaluate with project dependencies from custom URI`(@TempDir tempDir: Path, @TempDir cacheDir: Path) { - PackageServer.populateCacheDir(cacheDir) - val libDir = tempDir.resolve("lib/").createDirectories() - libDir - .resolve("lib.pkl") - .writeText( - """ - text = "This is from lib" - """ - .trimIndent() - ) - libDir - .resolve("PklProject") - .writeText( - """ - amends "pkl:Project" - - package { - name = "lib" - baseUri = "package://localhost:12110/lib" - version = "5.0.0" - packageZipUrl = "https://localhost:12110/lib.zip" - } - """ - .trimIndent() - ) - val projectDir = tempDir.resolve("proj/").createDirectories() - val module = projectDir.resolve("mod.pkl") - module.writeText( - """ - import "@birds/Bird.pkl" - import "@lib/lib.pkl" - - res: Bird = new { - name = "Birdie" - favoriteFruit { name = "dragonfruit" } - } - - libContents = lib - """ - .trimIndent() - ) - projectDir - .resolve("PklProject") - .writeText( - """ - amends "pkl:Project" - - dependencies { - ["birds"] { - uri = "package://localhost:12110/birds@0.5.0" - } - ["lib"] = import("../lib/PklProject") - } - """ - .trimIndent() - ) - val dollar = '$' - projectDir - .resolve("PklProject.deps.json") - .writeText( - """ - { - "schemaVersion": 1, - "resolvedDependencies": { - "package://localhost:12110/birds@0": { - "type": "remote", - "uri": "projectpackage://localhost:12110/birds@0.5.0", - "checksums": { - "sha256": "${dollar}skipChecksumVerification" - } - }, - "package://localhost:12110/fruit@1": { - "type": "remote", - "uri": "projectpackage://localhost:12110/fruit@1.0.5", - "checksums": { - "sha256": "${dollar}skipChecksumVerification" - } - }, - "package://localhost:12110/lib@5": { - "type": "local", - "uri": "projectpackage://localhost:12110/lib@5.0.0", - "path": "../lib" - } - } - } - - """ - .trimIndent() - ) - - val evalBase = EvaluatorBuilder.unconfigured() - .setStackFrameTransformer(StackFrameTransformers.defaultTransformer) - .setAllowedModules(listOf(Pattern.compile("custom:"), Pattern.compile("pkl:"), Pattern.compile("projectpackage:"), Pattern.compile("package:"))) - .setAllowedResources(listOf(Pattern.compile("custom:"), Pattern.compile("prop:"))) - .addModuleKeyFactory(ModuleKeyFactories.standardLibrary) - .addModuleKeyFactory(ModuleKeyFactories.projectpackage) - .addModuleKeyFactory(ModuleKeyFactories.pkg) - .addModuleKeyFactory(EvaluatorCustomReaders.CustomModuleKeyFactory("custom", tempDir)) - .addResourceReader(ResourceReaders.externalProperty()) - .addResourceReader(EvaluatorCustomReaders.CustomResourceReader("custom", tempDir)) - - - val projEval = evalBase.build() - val projOutput = projEval.evaluateOutputValueAs(ModuleSource.uri("custom:///proj/PklProject"), PClassInfo.Project) - val proj = Project.parseProject(projOutput) - - val eval = evalBase.setProjectDependencies(proj.dependencies).build() - val output = eval.evaluateExpressionString(ModuleSource.uri("custom:///proj/mod.pkl"), "output.text") - assertThat(output) - .isEqualTo( - """ - res { - name = "Birdie" - favoriteFruit { - name = "dragonfruit" - } - } - libContents { - text = "This is from lib" - } - - """ - .trimIndent() - ) - - } - -} diff --git a/pkl-core/src/test/kotlin/org/pkl/core/EvaluatorTest.kt b/pkl-core/src/test/kotlin/org/pkl/core/EvaluatorTest.kt index 0d6e1c4db..9cd678124 100644 --- a/pkl-core/src/test/kotlin/org/pkl/core/EvaluatorTest.kt +++ b/pkl-core/src/test/kotlin/org/pkl/core/EvaluatorTest.kt @@ -6,6 +6,7 @@ import java.net.URI import java.nio.file.Files import java.nio.file.Path import org.assertj.core.api.Assertions.assertThat +import org.assertj.core.api.Assertions.assertThatCode import org.junit.jupiter.api.Test import org.junit.jupiter.api.assertThrows import org.junit.jupiter.api.io.TempDir @@ -15,7 +16,16 @@ import org.pkl.commons.writeString import org.pkl.core.ModuleSource.* import org.pkl.core.util.IoUtils import org.junit.jupiter.api.AfterAll +import org.pkl.commons.test.PackageServer +import org.pkl.core.module.ModuleKey +import org.pkl.core.module.ModuleKeyFactories +import org.pkl.core.module.ModuleKeyFactory +import org.pkl.core.module.ResolvedModuleKey +import org.pkl.core.project.Project +import java.nio.charset.StandardCharsets import java.nio.file.FileSystems +import java.util.* +import java.util.regex.Pattern import kotlin.io.path.writeText class EvaluatorTest { @@ -24,6 +34,28 @@ class EvaluatorTest { private const val sourceText = "name = \"pigeon\"; age = 10 + 20" + private object CustomModuleKeyFactory : ModuleKeyFactory { + override fun create(uri: URI): Optional { + return if (uri.scheme == "custom") Optional.of(CustomModuleKey(uri)) + else Optional.empty() + } + } + + private class CustomModuleKey(private val uri: URI) : ModuleKey, ResolvedModuleKey { + override fun hasHierarchicalUris(): Boolean = true + + override fun isGlobbable(): Boolean = false + + override fun getOriginal(): ModuleKey = this + + override fun getUri(): URI = uri + + override fun loadSource(): String = javaClass.classLoader.getResourceAsStream(uri.path.drop(1))!!.use { it.readAllBytes().toString( + StandardCharsets.UTF_8) } + + override fun resolve(securityManager: SecurityManager): ResolvedModuleKey = this + } + @AfterAll @JvmStatic fun afterAll() { @@ -289,6 +321,132 @@ class EvaluatorTest { assertThat(output["bar/../bark.yml"]?.text).isEqualTo("bark: bark bark") } + @Test + fun `project set from modulepath`(@TempDir cacheDir: Path) { + PackageServer.populateCacheDir(cacheDir) + val evaluatorBuilder = EvaluatorBuilder.preconfigured().setModuleCacheDir(cacheDir) + val project = Project.load(modulePath("/org/pkl/core/project/project5/PklProject")) + val result = evaluatorBuilder.setProjectDependencies(project.dependencies).build().use { evaluator -> + evaluator.evaluateOutputText(modulePath("/org/pkl/core/project/project5/main.pkl")) + } + assertThat(result).isEqualTo(""" + prop1 { + name = "Apple" + } + prop2 { + res = 1 + } + + """.trimIndent()) + } + + @Test + fun `project set from custom ModuleKeyFactory`(@TempDir cacheDir: Path) { + PackageServer.populateCacheDir(cacheDir) + val evaluatorBuilder = with(EvaluatorBuilder.preconfigured()) { + setAllowedModules(SecurityManagers.defaultAllowedModules + Pattern.compile("custom:")) + setAllowedResources(SecurityManagers.defaultAllowedResources + Pattern.compile("custom:")) + setModuleCacheDir(cacheDir) + setModuleKeyFactories( + listOf( + CustomModuleKeyFactory, + ModuleKeyFactories.standardLibrary, + ModuleKeyFactories.pkg, + ModuleKeyFactories.projectpackage, + ModuleKeyFactories.file + ) + ) + } + val project = evaluatorBuilder.build().use { Project.load(it, uri("custom:/org/pkl/core/project/project5/PklProject")) } + + val evaluator = evaluatorBuilder.setProjectDependencies(project.dependencies).build() + val output = evaluator.use { it.evaluateOutputText(uri("custom:/org/pkl/core/project/project5/main.pkl")) } + assertThat(output) + .isEqualTo( + """ + prop1 { + name = "Apple" + } + prop2 { + res = 1 + } + + """ + .trimIndent() + ) + } + + @Test + fun `project base path set to non-hierarchical scheme`() { + class FooBarModuleKey(val moduleUri: URI) : ModuleKey, ResolvedModuleKey { + override fun hasHierarchicalUris(): Boolean = false + override fun isGlobbable(): Boolean = false + override fun getOriginal(): ModuleKey = this + override fun getUri(): URI = moduleUri + override fun loadSource(): String = + if (uri.schemeSpecificPart.endsWith("PklProject")) { + """ + amends "pkl:Project" + """.trimIndent() + } else """ + birds = import("@birds/catalog/Ostritch.pkl") + """.trimIndent() + override fun resolve(securityManager: SecurityManager): ResolvedModuleKey { + return this + } + } + + val fooBayModuleKeyFactory = ModuleKeyFactory { uri -> + if (uri.scheme == "foobar") Optional.of(FooBarModuleKey(uri)) + else Optional.empty() + } + + val evaluatorBuilder = with(EvaluatorBuilder.preconfigured()) { + setAllowedModules(SecurityManagers.defaultAllowedModules + Pattern.compile("foobar:")) + setAllowedResources(SecurityManagers.defaultAllowedResources + Pattern.compile("foobar:")) + setModuleKeyFactories( + listOf( + fooBayModuleKeyFactory, + ModuleKeyFactories.standardLibrary, + ModuleKeyFactories.pkg, + ModuleKeyFactories.projectpackage, + ModuleKeyFactories.file + ) + ) + } + + val project = evaluatorBuilder.build().use { Project.load(it, uri("foobar:foo/PklProject")) } + val evaluator = evaluatorBuilder.setProjectDependencies(project.dependencies).build() + assertThatCode { evaluator.use { it.evaluateOutputText(uri("foobar:baz")) } } + .hasMessageContaining("Cannot import dependency because project URI `foobar:foo/PklProject` does not have a hierarchical path.") + } + + @Test + fun `cannot glob import in local dependency from modulepath`(@TempDir cacheDir: Path) { + PackageServer.populateCacheDir(cacheDir) + val evaluatorBuilder = EvaluatorBuilder.preconfigured().setModuleCacheDir(cacheDir) + val project = Project.load(modulePath("/org/pkl/core/project/project6/PklProject")) + evaluatorBuilder.setProjectDependencies(project.dependencies).build().use { evaluator -> + assertThatCode { + evaluator.evaluateOutputText(modulePath("/org/pkl/core/project/project6/globWithinDependency.pkl")) + }.hasMessageContaining(""" + Cannot resolve import in local dependency because scheme `modulepath` is not globbable. + + 1 | res = import*("*.pkl") + ^^^^^^^^^^^^^^^^ + """.trimIndent()) + assertThatCode { + evaluator.evaluateOutputText(modulePath("/org/pkl/core/project/project6/globIntoDependency.pkl")) + }.hasMessageContaining(""" + –– Pkl Error –– + Cannot resolve import in local dependency because scheme `modulepath` is not globbable. + + 1 | import* "@project7/*.pkl" as proj7Files + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + """.trimIndent()) + } + } + private fun checkModule(module: PModule) { assertThat(module.properties.size).isEqualTo(2) assertThat(module.getProperty("name")).isEqualTo("pigeon") diff --git a/pkl-core/src/test/kotlin/org/pkl/core/project/ProjectTest.kt b/pkl-core/src/test/kotlin/org/pkl/core/project/ProjectTest.kt index 450dc0881..61938c48f 100644 --- a/pkl-core/src/test/kotlin/org/pkl/core/project/ProjectTest.kt +++ b/pkl-core/src/test/kotlin/org/pkl/core/project/ProjectTest.kt @@ -1,16 +1,16 @@ package org.pkl.core.project -import org.pkl.commons.test.PackageServer -import org.pkl.commons.writeString -import org.pkl.core.* -import org.pkl.core.packages.PackageUri -import org.pkl.core.project.Project.EvaluatorSettings import org.assertj.core.api.Assertions.assertThat import org.assertj.core.api.Assertions.assertThatCode import org.junit.jupiter.api.Test import org.junit.jupiter.api.io.TempDir import org.pkl.commons.test.FileTestUtils +import org.pkl.commons.test.PackageServer +import org.pkl.commons.writeString +import org.pkl.core.* import org.pkl.core.http.HttpClient +import org.pkl.core.packages.PackageUri +import org.pkl.core.project.Project.EvaluatorSettings import java.net.URI import java.nio.file.Path import java.util.regex.Pattern diff --git a/pkl-core/src/test/resources/org/pkl/core/project/project5/PklProject b/pkl-core/src/test/resources/org/pkl/core/project/project5/PklProject new file mode 100644 index 000000000..771608add --- /dev/null +++ b/pkl-core/src/test/resources/org/pkl/core/project/project5/PklProject @@ -0,0 +1,8 @@ +amends "pkl:Project" + +dependencies { + ["fruit"] { + uri = "package://localhost:0/fruit@1.0.5" + } + ["project4"] = import("../project4/PklProject") +} diff --git a/pkl-core/src/test/resources/org/pkl/core/project/project5/PklProject.deps.json b/pkl-core/src/test/resources/org/pkl/core/project/project5/PklProject.deps.json new file mode 100644 index 000000000..42f11d3db --- /dev/null +++ b/pkl-core/src/test/resources/org/pkl/core/project/project5/PklProject.deps.json @@ -0,0 +1,17 @@ +{ + "schemaVersion": 1, + "resolvedDependencies": { + "package://localhost:0/fruit@1": { + "type": "remote", + "uri": "projectpackage://localhost:0/fruit@1.0.5", + "checksums": { + "sha256": "$skipChecksumVerification" + } + }, + "package://localhost:0/project4@1": { + "type": "local", + "uri": "projectpackage://localhost:0/project4@1.0.0", + "path": "../project4" + } + } +} diff --git a/pkl-core/src/test/resources/org/pkl/core/project/project5/main.pkl b/pkl-core/src/test/resources/org/pkl/core/project/project5/main.pkl new file mode 100644 index 000000000..476e3cb44 --- /dev/null +++ b/pkl-core/src/test/resources/org/pkl/core/project/project5/main.pkl @@ -0,0 +1,5 @@ +import "@fruit/catalog/apple.pkl" +import "@project4/module1.pkl" + +prop1 = apple +prop2 = module1 diff --git a/pkl-core/src/test/resources/org/pkl/core/project/project6/PklProject b/pkl-core/src/test/resources/org/pkl/core/project/project6/PklProject new file mode 100644 index 000000000..d2eec0c1a --- /dev/null +++ b/pkl-core/src/test/resources/org/pkl/core/project/project6/PklProject @@ -0,0 +1,5 @@ +amends "pkl:Project" + +dependencies { + ["project7"] = import("../project7/PklProject") +} diff --git a/pkl-core/src/test/resources/org/pkl/core/project/project6/PklProject.deps.json b/pkl-core/src/test/resources/org/pkl/core/project/project6/PklProject.deps.json new file mode 100644 index 000000000..6fc3cc8c0 --- /dev/null +++ b/pkl-core/src/test/resources/org/pkl/core/project/project6/PklProject.deps.json @@ -0,0 +1,10 @@ +{ + "schemaVersion": 1, + "resolvedDependencies": { + "package://localhost:0/project7@1": { + "type": "local", + "uri": "projectpackage://localhost:0/project7@1.0.0", + "path": "../project7" + } + } +} diff --git a/pkl-core/src/test/resources/org/pkl/core/project/project6/globIntoDependency.pkl b/pkl-core/src/test/resources/org/pkl/core/project/project6/globIntoDependency.pkl new file mode 100644 index 000000000..a3cfa7ddb --- /dev/null +++ b/pkl-core/src/test/resources/org/pkl/core/project/project6/globIntoDependency.pkl @@ -0,0 +1,3 @@ +import* "@project7/*.pkl" as proj7Files + +res = proj7Files diff --git a/pkl-core/src/test/resources/org/pkl/core/project/project6/globWithinDependency.pkl b/pkl-core/src/test/resources/org/pkl/core/project/project6/globWithinDependency.pkl new file mode 100644 index 000000000..df1445283 --- /dev/null +++ b/pkl-core/src/test/resources/org/pkl/core/project/project6/globWithinDependency.pkl @@ -0,0 +1,3 @@ +import "@project7/main.pkl" + +res = main.res diff --git a/pkl-core/src/test/resources/org/pkl/core/project/project7/PklProject b/pkl-core/src/test/resources/org/pkl/core/project/project7/PklProject new file mode 100644 index 000000000..76e474a05 --- /dev/null +++ b/pkl-core/src/test/resources/org/pkl/core/project/project7/PklProject @@ -0,0 +1,8 @@ +amends "pkl:Project" + +package { + name = "project7" + version = "1.0.0" + packageZipUrl = "https://bogus.value" + baseUri = "package://localhost:0/project7" +} diff --git a/pkl-core/src/test/resources/org/pkl/core/project/project7/main.pkl b/pkl-core/src/test/resources/org/pkl/core/project/project7/main.pkl new file mode 100644 index 000000000..6c85a2169 --- /dev/null +++ b/pkl-core/src/test/resources/org/pkl/core/project/project7/main.pkl @@ -0,0 +1 @@ +res = import*("*.pkl") diff --git a/pkl-core/src/test/resources/org/pkl/core/project/project7/moduleA.pkl b/pkl-core/src/test/resources/org/pkl/core/project/project7/moduleA.pkl new file mode 100644 index 000000000..e69de29bb diff --git a/pkl-core/src/test/resources/org/pkl/core/project/project7/moduleB.pkl b/pkl-core/src/test/resources/org/pkl/core/project/project7/moduleB.pkl new file mode 100644 index 000000000..e69de29bb diff --git a/pkl-server/src/main/kotlin/org/pkl/server/ClientModuleKeyFactory.kt b/pkl-server/src/main/kotlin/org/pkl/server/ClientModuleKeyFactory.kt index abcd46c47..90e70875d 100644 --- a/pkl-server/src/main/kotlin/org/pkl/server/ClientModuleKeyFactory.kt +++ b/pkl-server/src/main/kotlin/org/pkl/server/ClientModuleKeyFactory.kt @@ -17,16 +17,17 @@ package org.pkl.server import java.io.IOException import java.net.URI -import java.util.* +import java.util.Optional import java.util.concurrent.CompletableFuture import java.util.concurrent.ConcurrentHashMap import java.util.concurrent.Future import kotlin.random.Random import org.pkl.core.SecurityManager -import org.pkl.core.module.* -import org.pkl.core.packages.Dependency -import org.pkl.core.packages.PackageLoadError -import org.pkl.core.runtime.VmContext +import org.pkl.core.module.ModuleKey +import org.pkl.core.module.ModuleKeyFactory +import org.pkl.core.module.PathElement +import org.pkl.core.module.ResolvedModuleKey +import org.pkl.core.module.ResolvedModuleKeys internal class ClientModuleKeyFactory( private val readerSpecs: Collection, @@ -95,8 +96,6 @@ internal class ClientModuleKeyFactory( is ListModulesResponse -> { if (response.error != null) { completeExceptionally(IOException(response.error)) - } else if (response.pathElements != null) { - complete(response.pathElements) } else { complete(response.pathElements ?: emptyList()) } @@ -114,7 +113,7 @@ internal class ClientModuleKeyFactory( private val uri: URI, private val spec: ModuleReaderSpec, private val resolver: ClientModuleKeyResolver, - ) : ModuleKeys.DependencyAwareModuleKey(uri) { + ) : ModuleKey { override fun isLocal(): Boolean = spec.isLocal override fun hasHierarchicalUris(): Boolean = spec.hasHierarchicalUris @@ -134,22 +133,6 @@ internal class ClientModuleKeyFactory( override fun hasElement(securityManager: SecurityManager, uri: URI): Boolean { return resolver.hasElement(securityManager, uri) } - - override fun getDependencies(): Map { - if (!hasHierarchicalUris()) { - throw PackageLoadError("cannotResolveDependencyFromReaderWithOpaqueUris", spec.scheme) - } - - val projectDepsManager = VmContext.get(null).projectDependenciesManager - if (projectDepsManager == null || !projectDepsManager.hasUri(uri)) { - throw PackageLoadError("cannotResolveDependencyNoProject") - } - return projectDepsManager.dependencies - } - - override fun cannotFindDependency(name: String): PackageLoadError { - return PackageLoadError("cannotFindDependencyInProject", name) - } } } diff --git a/pkl-server/src/main/kotlin/org/pkl/server/ClientModuleKeyFactory.kt.orig b/pkl-server/src/main/kotlin/org/pkl/server/ClientModuleKeyFactory.kt.orig new file mode 100644 index 000000000..d1e001040 --- /dev/null +++ b/pkl-server/src/main/kotlin/org/pkl/server/ClientModuleKeyFactory.kt.orig @@ -0,0 +1,173 @@ +/** + * Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.pkl.server + +import java.io.IOException +import java.net.URI +import java.util.* +import java.util.concurrent.CompletableFuture +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.Future +import kotlin.random.Random +import org.pkl.core.SecurityManager +import org.pkl.core.module.* +import org.pkl.core.packages.Dependency +import org.pkl.core.packages.PackageLoadError +import org.pkl.core.runtime.VmContext + +internal class ClientModuleKeyFactory( + private val readerSpecs: Collection, + transport: MessageTransport, + evaluatorId: Long +) : ModuleKeyFactory { + companion object { + private class ClientModuleKeyResolver( + private val transport: MessageTransport, + private val evaluatorId: Long, + ) { + private val readResponses: MutableMap> = ConcurrentHashMap() + + private val listResponses: MutableMap>> = ConcurrentHashMap() + + fun listElements(securityManager: SecurityManager, uri: URI): List { + securityManager.checkResolveModule(uri) + return doListElements(uri) + } + + fun hasElement(securityManager: SecurityManager, uri: URI): Boolean { + securityManager.checkResolveModule(uri) + return try { + doReadModule(uri) + true + } catch (e: IOException) { + false + } + } + + fun resolveModule(securityManager: SecurityManager, uri: URI): String { + securityManager.checkResolveModule(uri) + return doReadModule(uri) + } + + private fun doReadModule(uri: URI): String = + readResponses + .computeIfAbsent(uri) { + CompletableFuture().apply { + val request = ReadModuleRequest(Random.nextLong(), evaluatorId, uri) + transport.send(request) { response -> + when (response) { + is ReadModuleResponse -> { + if (response.error != null) { + completeExceptionally(IOException(response.error)) + } else { + complete(response.contents!!) + } + } + else -> { + completeExceptionally(ProtocolException("unexpected response")) + } + } + } + } + } + .getUnderlying() + + private fun doListElements(uri: URI): List = + listResponses + .computeIfAbsent(uri) { + CompletableFuture>().apply { + val request = ListModulesRequest(Random.nextLong(), evaluatorId, uri) + transport.send(request) { response -> + when (response) { + is ListModulesResponse -> { + if (response.error != null) { + completeExceptionally(IOException(response.error)) + } else if (response.pathElements != null) { + complete(response.pathElements) + } else { +<<<<<<< HEAD + complete(response.pathElements ?: emptyList()) +======= + complete(emptyList()) +>>>>>>> d999b6ae0 (Support Projects loaded from arbitrary URIs) + } + } + else -> completeExceptionally(ProtocolException("unexpected response")) + } + } + } + } + .getUnderlying() + } + + /** [ModuleKey] that delegates module reads to the client. */ + private class ClientModuleKey( + private val uri: URI, + private val spec: ModuleReaderSpec, + private val resolver: ClientModuleKeyResolver, + ) : ModuleKeys.DependencyAwareModuleKey(uri) { + override fun isLocal(): Boolean = spec.isLocal + + override fun hasHierarchicalUris(): Boolean = spec.hasHierarchicalUris + + override fun isGlobbable(): Boolean = spec.isGlobbable + + override fun getUri(): URI = uri + + override fun listElements(securityManager: SecurityManager, baseUri: URI): List = + resolver.listElements(securityManager, baseUri) + + override fun resolve(securityManager: SecurityManager): ResolvedModuleKey { + val contents = resolver.resolveModule(securityManager, uri) + return ResolvedModuleKeys.virtual(this, uri, contents, true) + } + + override fun hasElement(securityManager: SecurityManager, uri: URI): Boolean { + return resolver.hasElement(securityManager, uri) + } + + override fun getDependencies(): Map { + if (!hasHierarchicalUris()) { + throw PackageLoadError("cannotResolveDependencyFromReaderWithOpaqueUris", spec.scheme) + } + + val projectDepsManager = VmContext.get(null).projectDependenciesManager + if (projectDepsManager == null || !projectDepsManager.hasUri(uri)) { + throw PackageLoadError("cannotResolveDependencyNoProject") + } + return projectDepsManager.dependencies + } + + override fun cannotFindDependency(name: String): PackageLoadError { + return PackageLoadError("cannotFindDependencyInProject", name) + } + } + } + + private val schemes = readerSpecs.map { it.scheme } + + private val resolver: ClientModuleKeyResolver = ClientModuleKeyResolver(transport, evaluatorId) + + override fun create(uri: URI): Optional = + when (uri.scheme) { + in schemes -> { + val readerSpec = readerSpecs.find { it.scheme == uri.scheme }!! + val moduleKey = ClientModuleKey(uri, readerSpec, resolver) + Optional.of(moduleKey) + } + else -> Optional.empty() + } +} diff --git a/pkl-server/src/main/kotlin/org/pkl/server/ClientResourceReader.kt.orig b/pkl-server/src/main/kotlin/org/pkl/server/ClientResourceReader.kt.orig new file mode 100644 index 000000000..2878bd788 --- /dev/null +++ b/pkl-server/src/main/kotlin/org/pkl/server/ClientResourceReader.kt.orig @@ -0,0 +1,112 @@ +/** + * Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.pkl.server + +import java.io.IOException +import java.net.URI +import java.util.Optional +import java.util.concurrent.CompletableFuture +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.Future +import kotlin.random.Random +import org.pkl.core.SecurityManager +import org.pkl.core.module.PathElement +import org.pkl.core.resource.Resource +import org.pkl.core.resource.ResourceReader + +/** Resource reader that delegates read logic to the client. */ +internal class ClientResourceReader( + private val transport: MessageTransport, + private val evaluatorId: Long, + private val readerSpec: ResourceReaderSpec, +) : ResourceReader { + private val readResponses: MutableMap> = ConcurrentHashMap() + + private val listResources: MutableMap>> = ConcurrentHashMap() + + override fun hasHierarchicalUris(): Boolean = readerSpec.hasHierarchicalUris + + override fun isGlobbable(): Boolean = readerSpec.isGlobbable + + override fun getUriScheme() = readerSpec.scheme + + override fun read(uri: URI): Optional = Optional.of(Resource(uri, doRead(uri))) + + override fun hasElement(securityManager: SecurityManager, elementUri: URI): Boolean { + securityManager.checkResolveResource(elementUri) + return try { + doRead(elementUri) + true + } catch (e: IOException) { + false + } + } + + override fun listElements(securityManager: SecurityManager, baseUri: URI): List { + securityManager.checkResolveResource(baseUri) + return doListElements(baseUri) + } + + private fun doListElements(baseUri: URI): List = + listResources + .computeIfAbsent(baseUri) { + CompletableFuture>().apply { + val request = ListResourcesRequest(Random.nextLong(), evaluatorId, baseUri) + transport.send(request) { response -> + when (response) { + is ListResourcesResponse -> + if (response.error != null) { + completeExceptionally(IOException(response.error)) +<<<<<<< HEAD + } else { + complete(response.pathElements ?: emptyList()) +======= + } else if (response.pathElements != null) { + complete(response.pathElements) + } else { + complete(emptyList()) +>>>>>>> d999b6ae0 (Support Projects loaded from arbitrary URIs) + } + else -> completeExceptionally(ProtocolException("Unexpected response")) + } + } + } + } + .getUnderlying() + + private fun doRead(uri: URI): ByteArray = + readResponses + .computeIfAbsent(uri) { + CompletableFuture().apply { + val request = ReadResourceRequest(Random.nextLong(), evaluatorId, uri) + transport.send(request) { response -> + when (response) { + is ReadResourceResponse -> { + if (response.error != null) { + completeExceptionally(IOException(response.error)) + } else { + complete(response.contents!!) + } + } + else -> { + completeExceptionally(ProtocolException("Unexpected response: $response")) + } + } + } + } + } + .getUnderlying() +} diff --git a/stdlib/Project.pkl b/stdlib/Project.pkl index ed90feafe..b312c422b 100644 --- a/stdlib/Project.pkl +++ b/stdlib/Project.pkl @@ -91,25 +91,15 @@ package: Package? /// ``` tests: Listing(isDistinct) -/// Tells if the project is a file-based module named `PklProject`, is not self, and has a [package] section +/// Tells if the project is a local module named `PklProject`, is not self, and has a [package] section local isValidLoadDependency = (it: Project) -> - isUriLocal(projectFileUri, it.projectFileUri) - && it.projectFileUri.endsWith("/PklProject") - && it != module - && it.package != null - -// this regex is an approximation but is good enough to handle URIs returned by `reflect` -const local uriRegex = Regex(#"^(?:([^:/?#]+):)?(?://([^/?#]*))?[^?#]*(?:\?[^#]*)?(?:#.*)?"#) + isUriLocal(projectFileUri, it.projectFileUri) + && it.projectFileUri.endsWith("/PklProject") + && it != module + && it.package != null const local function isUriLocal(uri1: String, uri2: String): Boolean = - let (uri1Match = uriRegex.matchEntire(uri1)?.ifNonNull((it) -> Pair(it.groups[1]?.value, it.groups[2]?.value))) - let (uri2Match = uriRegex.matchEntire(uri2).ifNonNull((it) -> Pair(it.groups[1]?.value, it.groups[2]?.value))) - uri1Match == uri2Match - -/// A local dependency is another [Project] that is local to the file system. -/// -/// To declare, use `import("path/to/PklProject")` -typealias LocalDependency = Project(isValidLoadDependency) + uri1.substring(0, uri1.indexOf(":")) == uri2.substring(0, uri2.indexOf(":")) /// The dependencies of this project. /// @@ -183,19 +173,32 @@ typealias LocalDependency = Project(isValidLoadDependency) /// 1. Gather all dependencies, both direct and transitive. /// 2. For each package's major version, determine the highest declared minor version. /// 3. Write each resolved dependency to sibling file `PklProject.deps.json`. -dependencies: Mapping +dependencies: Mapping + +local isFileBasedProject = projectFileUri.startsWith("file:") /// If set, controls the base evaluator settings when running the evaluator. /// /// These settings influence the behavior of the evaluator when running the `pkl eval`, `pkl test`, /// and `pkl repl` CLI commands. -/// Note that command line flags passed to the CLI will override any settings defined here. +/// Command line flags passed to the CLI will override any settings defined here. /// /// Other integrations can possibly ignore these evaluator settings. /// /// Evaluator settings do not get published as part of a package. /// It is not possible for a package dependency to influence the evaluator settings of a project. -evaluatorSettings: EvaluatorSettings +/// +/// Certain values can only be set if this is a file-based project. +/// These are: +/// +/// - [modulePath][EvaluatorSettings.modulePath] +/// - [rootDir][EvaluatorSettings.rootDir] +/// - [moduleCacheDir][EvaluatorSettings.moduleCacheDir] +evaluatorSettings: EvaluatorSettings( + (modulePath != null).implies(isFileBasedProject), + (rootDir != null).implies(isFileBasedProject), + (moduleCacheDir != null).implies(isFileBasedProject) +) /// The URI of the PklProject file. /// @@ -218,10 +221,10 @@ function newInstance(enclosingModule: Module): Project = new { } local hasVersion = (it: Uri) -> - let (versionSep = it.lastIndexOf("@")) - if (versionSep == -1) false - else let (version = it.drop(versionSep + 1)) - semver.parseOrNull(version) != null + let (versionSep = it.lastIndexOf("@")) + if (versionSep == -1) false + else let (version = it.drop(versionSep + 1)) + semver.parseOrNull(version) != null typealias PackageUri = Uri(startsWith("package:"), hasVersion)