From b4ba256208a9330a3e3a0af600ba86152ac09dd7 Mon Sep 17 00:00:00 2001 From: Hannes Wellmann Date: Sun, 3 Sep 2023 01:17:20 +0200 Subject: [PATCH] Suppress addition of unnecessary repo-refereces when assembling p2-repos --- RELEASE_NOTES.md | 8 +- .../DestinationRepositoryDescriptor.java | 11 +- .../facade/MirrorApplicationService.java | 9 +- .../p2tools/MirrorApplicationServiceImpl.java | 4 +- .../tycho/p2tools/TychoMirrorApplication.java | 123 +++++++++++++++--- .../p2tools/MirrorApplicationServiceTest.java | 18 +-- .../category.xml | 5 + .../pom.xml | 61 +++++++++ ...efLocationP2RepositoryIntegrationTest.java | 51 ++++++-- .../p2/repository/AssembleRepositoryMojo.java | 50 +++++-- 10 files changed, 282 insertions(+), 58 deletions(-) create mode 100644 tycho-its/projects/p2Repository.repositoryRef.filter.providing/category.xml create mode 100644 tycho-its/projects/p2Repository.repositoryRef.filter.providing/pom.xml diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index 3cc8f7fafc..143cdfa03c 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -6,8 +6,11 @@ This page describes the noteworthy improvements provided by each release of Ecli ### new option to filter added repository-references when assembling a p2-repository -The repository references automatically added to a assembled p2-repository (via `tycho-p2-repository-plugin`'s `addIUTargetRepositoryReferences` or `addPomRepositoryReferences`) -can now be filtered by their location using exclusion and inclusion patterns and therefore allows more fine-grained control which references are added. +If filtering provided artifacts is enabled, the repository references automatically added to a assembled p2-repository +(via `tycho-p2-repository-plugin`'s `addIUTargetRepositoryReferences` or `addPomRepositoryReferences`) can now be filtered by their location +using exclusion and inclusion patterns and therefore allows more fine-grained control which references are added. +Additionally the automatically added references can be filter based on if they provide any of the filtered units or not. +If `addOnlyProviding` is `true` repositories that don't provide any filtered unit are not added to the assembled repo. ```xml org.eclipse.tycho @@ -16,6 +19,7 @@ can now be filtered by their location using exclusion and inclusion patterns and ... other configuration options ... + true https://foo.bar.org/hidden/** %regex[http(s)?:\/\/foo\.bar\.org\/secret\/.*] diff --git a/tycho-core/src/main/java/org/eclipse/tycho/p2/tools/DestinationRepositoryDescriptor.java b/tycho-core/src/main/java/org/eclipse/tycho/p2/tools/DestinationRepositoryDescriptor.java index c434eefc7e..8edc208cd3 100644 --- a/tycho-core/src/main/java/org/eclipse/tycho/p2/tools/DestinationRepositoryDescriptor.java +++ b/tycho-core/src/main/java/org/eclipse/tycho/p2/tools/DestinationRepositoryDescriptor.java @@ -28,10 +28,12 @@ public class DestinationRepositoryDescriptor { private final boolean append; private final Map extraArtifactRepositoryProperties; private final List repositoryReferences; + private final List filterablRepositoryReferences; public DestinationRepositoryDescriptor(File location, String name, boolean compress, boolean xzCompress, boolean keepNonXzIndexFiles, boolean metaDataOnly, boolean append, - Map extraArtifactRepositoryProperties, List repositoryReferences) { + Map extraArtifactRepositoryProperties, List repositoryReferences, + List filterablRepositoryReferences) { this.location = location; this.name = name; this.compress = compress; @@ -41,12 +43,13 @@ public DestinationRepositoryDescriptor(File location, String name, boolean compr this.append = append; this.extraArtifactRepositoryProperties = extraArtifactRepositoryProperties; this.repositoryReferences = repositoryReferences; + this.filterablRepositoryReferences = filterablRepositoryReferences; } public DestinationRepositoryDescriptor(File location, String name, boolean compress, boolean xzCompress, boolean keepNonXzIndexFiles, boolean metaDataOnly, boolean append) { this(location, name, compress, xzCompress, keepNonXzIndexFiles, metaDataOnly, append, Collections.emptyMap(), - Collections.emptyList()); + Collections.emptyList(), Collections.emptyList()); } public DestinationRepositoryDescriptor(File location, String name) { @@ -88,4 +91,8 @@ public Map getExtraArtifactRepositoryProperties() { public List getRepositoryReferences() { return repositoryReferences == null ? Collections.emptyList() : repositoryReferences; } + + public List getFilterableRepositoryReferences() { + return filterablRepositoryReferences; + } } diff --git a/tycho-core/src/main/java/org/eclipse/tycho/p2/tools/mirroring/facade/MirrorApplicationService.java b/tycho-core/src/main/java/org/eclipse/tycho/p2/tools/mirroring/facade/MirrorApplicationService.java index 4bc7a855f4..5c6ec60935 100644 --- a/tycho-core/src/main/java/org/eclipse/tycho/p2/tools/mirroring/facade/MirrorApplicationService.java +++ b/tycho-core/src/main/java/org/eclipse/tycho/p2/tools/mirroring/facade/MirrorApplicationService.java @@ -54,7 +54,11 @@ public interface MirrorApplicationService { * Whether to include bundles mentioned in the require section of a feature * @param includeRequiredFeatures * Whether to include features mentioned in the require section of a feature - * @param filterProvided Whether to filter IU/artifacts that are already provided by a referenced repository + * @param filterProvided + * Whether to filter IU/artifacts that are already provided by a referenced + * repository + * @param addOnlyProvidingRepoReferences + * Whether to add only repository-references that provide any relevant IU * @param filterProperties * additional filter properties to be set in the p2 slicing options. May be * null @@ -64,7 +68,8 @@ public interface MirrorApplicationService { public void mirrorReactor(RepositoryReferences sources, DestinationRepositoryDescriptor destination, Collection seeds, BuildContext context, boolean includeAllDependencies, boolean includeAllSource, boolean includeRequiredBundles, boolean includeRequiredFeatures, - boolean filterProvided, Map filterProperties) throws FacadeException; + boolean filterProvided, boolean addOnlyProvidingRepoReferences, Map filterProperties) + throws FacadeException; /** * recreates the metadata of an existing repository e.g. to account for changes in the contained diff --git a/tycho-core/src/main/java/org/eclipse/tycho/p2tools/MirrorApplicationServiceImpl.java b/tycho-core/src/main/java/org/eclipse/tycho/p2tools/MirrorApplicationServiceImpl.java index f54ca5fae2..aeb833ffbb 100644 --- a/tycho-core/src/main/java/org/eclipse/tycho/p2tools/MirrorApplicationServiceImpl.java +++ b/tycho-core/src/main/java/org/eclipse/tycho/p2tools/MirrorApplicationServiceImpl.java @@ -152,7 +152,8 @@ private static IQuery createQuery(IUDescription iu) { public void mirrorReactor(RepositoryReferences sources, DestinationRepositoryDescriptor destination, Collection projectSeeds, BuildContext context, boolean includeAllDependencies, boolean includeAllSource, boolean includeRequiredBundles, boolean includeRequiredFeatures, - boolean filterProvided, Map filterProperties) throws FacadeException { + boolean filterProvided, boolean addOnlyProvidingRepoReferences, Map filterProperties) + throws FacadeException { final TychoMirrorApplication mirrorApp = createMirrorApplication(sources, destination, agent); // mirror scope: seed units... @@ -163,6 +164,7 @@ public void mirrorReactor(RepositoryReferences sources, DestinationRepositoryDes mirrorApp.setIncludeRequiredFeatures(includeRequiredFeatures); mirrorApp.setIncludePacked(false); // no way, Tycho do no longer support packed artifacts anyways mirrorApp.setFilterProvided(filterProvided); + mirrorApp.setAddOnlyProvidingRepoReferences(addOnlyProvidingRepoReferences); mirrorApp.setEnvironments(context.getEnvironments()); SlicingOptions options = new SlicingOptions(); options.considerStrictDependencyOnly(!includeAllDependencies); diff --git a/tycho-core/src/main/java/org/eclipse/tycho/p2tools/TychoMirrorApplication.java b/tycho-core/src/main/java/org/eclipse/tycho/p2tools/TychoMirrorApplication.java index f03a9a2430..9f0a40df15 100644 --- a/tycho-core/src/main/java/org/eclipse/tycho/p2tools/TychoMirrorApplication.java +++ b/tycho-core/src/main/java/org/eclipse/tycho/p2tools/TychoMirrorApplication.java @@ -10,19 +10,27 @@ * Contributors: * SAP SE - initial API and implementation * Hannes Wellmann - Assemble repository for all environments in one pass + * Hannes Wellmann - Implement user-defined filtering and filtering based on relevance for automatically added repo-references *******************************************************************************/ package org.eclipse.tycho.p2tools; import static java.util.stream.Collectors.groupingBy; +import static java.util.stream.Collectors.mapping; +import static java.util.stream.Collectors.toList; +import java.lang.reflect.Field; +import java.lang.reflect.Method; import java.net.URI; import java.net.URISyntaxException; import java.util.ArrayList; import java.util.Collection; +import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Map.Entry; import java.util.Set; +import java.util.stream.Collectors; import java.util.stream.Stream; import java.util.stream.StreamSupport; @@ -32,6 +40,7 @@ import org.eclipse.equinox.internal.p2.metadata.IRequiredCapability; import org.eclipse.equinox.internal.p2.metadata.InstallableUnit; import org.eclipse.equinox.internal.p2.metadata.RequiredCapability; +import org.eclipse.equinox.internal.p2.metadata.repository.LocalMetadataRepository; import org.eclipse.equinox.p2.core.IProvisioningAgent; import org.eclipse.equinox.p2.core.ProvisionException; import org.eclipse.equinox.p2.internal.repository.tools.RepositoryDescriptor; @@ -39,6 +48,7 @@ import org.eclipse.equinox.p2.metadata.IArtifactKey; import org.eclipse.equinox.p2.metadata.IInstallableUnit; import org.eclipse.equinox.p2.metadata.IRequirement; +import org.eclipse.equinox.p2.metadata.Version; import org.eclipse.equinox.p2.metadata.expression.IMatchExpression; import org.eclipse.equinox.p2.query.CollectionResult; import org.eclipse.equinox.p2.query.IQueryResult; @@ -46,6 +56,7 @@ import org.eclipse.equinox.p2.query.QueryUtil; import org.eclipse.equinox.p2.repository.IRepository; import org.eclipse.equinox.p2.repository.IRepositoryManager; +import org.eclipse.equinox.p2.repository.IRepositoryReference; import org.eclipse.equinox.p2.repository.artifact.IArtifactRepository; import org.eclipse.equinox.p2.repository.artifact.IArtifactRepositoryManager; import org.eclipse.equinox.p2.repository.metadata.IMetadataRepository; @@ -58,18 +69,17 @@ public class TychoMirrorApplication extends org.eclipse.tycho.p2tools.copiedfrom private static final String SOURCE_SUFFIX = ".source"; private static final String FEATURE_GROUP = ".feature.group"; - private final Map extraArtifactRepositoryProperties; - private final List repositoryReferences; + private final DestinationRepositoryDescriptor destination; private boolean includeAllSource; private boolean includeRequiredBundles; private boolean includeRequiredFeatures; private boolean filterProvided; + private boolean addOnlyProvidingRepoReferences; private TargetPlatform targetPlatform; public TychoMirrorApplication(IProvisioningAgent agent, DestinationRepositoryDescriptor destination) { super(agent); - this.extraArtifactRepositoryProperties = destination.getExtraArtifactRepositoryProperties(); - this.repositoryReferences = destination.getRepositoryReferences(); + this.destination = destination; this.removeAddedRepositories = false; } @@ -79,7 +89,7 @@ protected IArtifactRepository initializeDestination(RepositoryDescriptor toInit, IArtifactRepository result = super.initializeDestination(toInit, mgr); // simple.SimpleArtifactRepository.PUBLISH_PACK_FILES_AS_SIBLINGS is not public result.setProperty("publishPackFilesAsSiblings", "true"); - extraArtifactRepositoryProperties.forEach(result::setProperty); + destination.getExtraArtifactRepositoryProperties().forEach(result::setProperty); return result; } @@ -173,7 +183,6 @@ public IQueryable slice(IInstallableUnit[] ius, IProgressMonit } return slice; } - }; } @@ -181,7 +190,8 @@ public IQueryable slice(IInstallableUnit[] ius, IProgressMonit protected IMetadataRepository initializeDestination(RepositoryDescriptor toInit, IMetadataRepositoryManager mgr) throws ProvisionException { IMetadataRepository result = super.initializeDestination(toInit, mgr); - var refs = repositoryReferences.stream().flatMap(TychoMirrorApplication::toSpiRepositoryReferences).toList(); + var refs = Stream.of(destination.getRepositoryReferences(), destination.getFilterableRepositoryReferences()) + .flatMap(List::stream).flatMap(TychoMirrorApplication::toSpiRepositoryReferences).toList(); result.addReferences(refs); return result; } @@ -189,18 +199,25 @@ protected IMetadataRepository initializeDestination(RepositoryDescriptor toInit, private static Stream toSpiRepositoryReferences( RepositoryReference rr) { return Stream.of(IRepository.TYPE_METADATA, IRepository.TYPE_ARTIFACT).map(type -> { - URI location = URI.create(rr.getLocation()); + URI location = getNormalizedLocation(rr); int options = rr.isEnable() ? IRepository.ENABLED : IRepository.NONE; return new org.eclipse.equinox.p2.repository.spi.RepositoryReference(location, rr.getName(), type, options); }); } + private static URI getNormalizedLocation(RepositoryReference r) { + // P2 does the same before loading the repo and thus IRepository.getLocation() returns the normalized URL. + // In order to avoid stripping of slashes from URI instances do it now before URIs are created. + String location = r.getLocation(); + return URI.create(location.endsWith("/") ? location.substring(0, location.length() - 1) : location); + } + @Override protected List collectArtifactKeys(Collection ius, IProgressMonitor monitor) throws ProvisionException { List keys = super.collectArtifactKeys(ius, monitor); if (isFilterProvidedItems()) { - removeProvidedItems(keys, getArtifactRepositoryManager(), monitor); + removeProvidedItems(keys, getArtifactRepositoryManager(), IRepository.TYPE_ARTIFACT, monitor); } return keys; } @@ -210,28 +227,96 @@ protected Set collectUnits(IQueryable slice, throws ProvisionException { Set units = super.collectUnits(slice, monitor); if (isFilterProvidedItems()) { - removeProvidedItems(units, getMetadataRepositoryManager(), monitor); + Map> fullRepositoryContent = units.stream() + .collect(groupingBy(IInstallableUnit::getId, mapping(IInstallableUnit::getVersion, toList()))); + + List> metadataRepositories = removeProvidedItems(units, + getMetadataRepositoryManager(), IRepository.TYPE_METADATA, monitor); + + if (addOnlyProvidingRepoReferences) { + Set removableReferences = destination.getFilterableRepositoryReferences().stream() + .map(TychoMirrorApplication::getNormalizedLocation).collect(Collectors.toSet()); + destination.getRepositoryReferences().stream().map(TychoMirrorApplication::getNormalizedLocation) + .forEach(removableReferences::remove); // keep reference if explicitly added to the repository + if (!removableReferences.isEmpty()) { + // Assume that for all units that correspond to artifacts the metadata either has a co-located artifact repository or a references to to one that contains it. + removeNotProvidingReferences(fullRepositoryContent, metadataRepositories, removableReferences); + } + } } return units; } private boolean isFilterProvidedItems() { - return filterProvided && !repositoryReferences.isEmpty(); + return filterProvided && !destinationMetadataRepository.getReferences().isEmpty(); } - private void removeProvidedItems(Collection allElements, IRepositoryManager repoManager, - IProgressMonitor monitor) throws ProvisionException { + private List> removeProvidedItems(Collection allElements, IRepositoryManager repoManager, + int repositoryType, IProgressMonitor monitor) throws ProvisionException { List> referencedRepositories = new ArrayList<>(); - for (RepositoryReference reference : repositoryReferences) { + for (IRepositoryReference reference : destinationMetadataRepository.getReferences()) { + if (reference.getType() != repositoryType) { + continue; + } try { - URI location = new URI(reference.getLocation()); + URI location = reference.getLocation(); IRepository repository = loadRepository(repoManager, location, monitor); referencedRepositories.add(repository); - } catch (URISyntaxException e) { - throw new ProvisionException("Can't parse referenced URI!", e); + } catch (IllegalArgumentException e) { + if (e.getCause() instanceof URISyntaxException uriException) { + throw new ProvisionException("Can't parse referenced URI!", uriException); + } else { + throw e; + } } } allElements.removeIf(e -> referencedRepositories.stream().anyMatch(repo -> contains(repo, e))); + return referencedRepositories; + } + + private void removeNotProvidingReferences(Map> fullRepositoryContent, + List> metadataRepositories, Set removableReferenceURIs) { + Map> usedRepositoryItems = new HashMap<>(); + for (IRepository repo : metadataRepositories) { + IQueryResult allUnits = repo.query(QueryUtil.ALL_UNITS, null); + Set usedRepoContent = stream(allUnits) + .filter(a -> fullRepositoryContent.getOrDefault(a.getId(), List.of()).contains(a.getVersion())) + .collect(Collectors.toSet()); + usedRepositoryItems.put(repo.getLocation(), usedRepoContent); + } + // Remove filterable references that contribute nothing or whose relevant content is also provided by another repo + usedRepositoryItems.entrySet().removeIf(repo -> { + if (!removableReferenceURIs.contains(repo.getKey())) { + return false; + } + Set usedContent = repo.getValue(); + return usedContent.isEmpty() + || usedRepositoryItems.entrySet().stream().filter(e -> e != repo).map(Entry::getValue) + .anyMatch(other -> other.size() >= usedContent.size() && other.containsAll(usedContent)); + }); + IMetadataRepository repository = getDestinationMetadataRepository(); + List discardedReferences = repository.getReferences().stream() + .filter(rr -> !usedRepositoryItems.keySet().contains(rr.getLocation())).toList(); + removeRepositoryReferences(repository, discardedReferences); + } + + //TODO: Just call IMetadataRepository.removeReferences once available: https://github.com/eclipse-equinox/p2/pull/338 + private static void removeRepositoryReferences(IMetadataRepository metadataRepository, + Collection references) { + if (metadataRepository instanceof LocalMetadataRepository localRepo) { + try { + Field repositoriesField = LocalMetadataRepository.class.getDeclaredField("repositories"); + repositoriesField.trySetAccessible(); + Method save = LocalMetadataRepository.class.getDeclaredMethod("save"); + save.trySetAccessible(); + @SuppressWarnings("unchecked") + Set repositories = (Set) repositoriesField.get(localRepo); + repositories.removeAll(references); + save.invoke(localRepo); + } catch (ReflectiveOperationException e) { + throw new IllegalStateException("Failed to clean-up references from assembled repository", e); + } + } } //TODO: just call IRepositoryManager.loadRepository() once available: https://github.com/eclipse-equinox/p2/pull/311 @@ -278,4 +363,8 @@ public void setFilterProvided(boolean filterProvided) { this.filterProvided = filterProvided; } + public void setAddOnlyProvidingRepoReferences(boolean addOnlyProvidingRepoReferences) { + this.addOnlyProvidingRepoReferences = addOnlyProvidingRepoReferences; + } + } diff --git a/tycho-core/src/test/java/org/eclipse/tycho/p2tools/MirrorApplicationServiceTest.java b/tycho-core/src/test/java/org/eclipse/tycho/p2tools/MirrorApplicationServiceTest.java index fcfc787e5a..2774684bb4 100644 --- a/tycho-core/src/test/java/org/eclipse/tycho/p2tools/MirrorApplicationServiceTest.java +++ b/tycho-core/src/test/java/org/eclipse/tycho/p2tools/MirrorApplicationServiceTest.java @@ -101,13 +101,13 @@ public void testMirrorNothing() throws Exception { Collection noSeeds = Collections.emptyList(); subject.mirrorReactor(sourceRepos("patch", "e342"), destinationRepo, noSeeds, context, false, false, false, - false, false, null); + false, false, false, null); } @Test public void testMirrorFeatureWithContent() throws Exception { subject.mirrorReactor(sourceRepos("patch", "e342"), destinationRepo, seedFor(SIMPLE_FEATURE_IU), context, false, - false, false, false, false, null); + false, false, false, false, false, null); logVerifier.expectNoWarnings(); assertTrue(repoFile(destinationRepo, "plugins/org.eclipse.core.runtime_3.4.0.v20080512.jar").exists()); @@ -121,9 +121,10 @@ public void testExtraArtifactRepositoryProperties() throws Exception { extraArtifactRepositoryProperties.put("p2.mirrorsURL", "http://some.where.else"); extraArtifactRepositoryProperties.put("foo", "bar"); destinationRepo = new DestinationRepositoryDescriptor(tempFolder.newFolder("dest2"), DEFAULT_NAME, false, false, - false, false, true, extraArtifactRepositoryProperties, Collections.emptyList()); + false, false, true, extraArtifactRepositoryProperties, Collections.emptyList(), + Collections.emptyList()); subject.mirrorReactor(sourceRepos("patch", "e342"), destinationRepo, seedFor(SIMPLE_FEATURE_IU), context, false, - false, false, false, false, null); + false, false, false, false, false, null); logVerifier.expectNoWarnings(); File artifactsXml = repoFile(destinationRepo, "artifacts.xml"); @@ -146,7 +147,7 @@ public void testExtraArtifactRepositoryProperties() throws Exception { @Test public void testMirrorPatch() throws Exception { subject.mirrorReactor(sourceRepos("patch", "e352"), destinationRepo, seedFor(FEATURE_PATCH_IU), context, false, - false, false, false, false, null); + false, false, false, false, false, null); //TODO why mirror tool emits a warning here? logVerifier.expectNoWarnings(); assertTrue(repoFile(destinationRepo, "plugins/org.eclipse.core.runtime_3.5.0.v20090525.jar").exists()); @@ -156,7 +157,7 @@ public void testMirrorPatch() throws Exception { @Test public void testMirrorFeatureAndPatch() throws Exception { subject.mirrorReactor(sourceRepos("patch", "e352"), destinationRepo, - seedFor(SIMPLE_FEATURE_IU, FEATURE_PATCH_IU), context, false, false, false, false, false, null); + seedFor(SIMPLE_FEATURE_IU, FEATURE_PATCH_IU), context, false, false, false, false, false, false, null); assertTrue(repoFile(destinationRepo, "plugins/org.eclipse.core.runtime_3.5.0.v20090525.jar").exists()); assertTrue(repoFile(destinationRepo, "features/" + SIMPLE_FEATURE + "_1.0.0.jar").exists()); @@ -175,7 +176,7 @@ public void testMirrorWithMissingMandatoryContent() throws Exception { * warning is issued. */ subject.mirrorReactor(sourceRepos("patch"), destinationRepo, seedFor(SIMPLE_FEATURE_IU), context, false, false, - false, false, false, null); + false, false, false, false, null); logVerifier.expectWarning(not(is(""))); } @@ -189,7 +190,8 @@ public void testMirrorForSeedWithNullIU() throws Exception { List seeds = Collections .singletonList(new DependencySeed(null, "org.eclipse.core.runtime", null)); - subject.mirrorReactor(sourceRepos("e342"), destinationRepo, seeds, context, false, false, false, false, false, null); + subject.mirrorReactor(sourceRepos("e342"), destinationRepo, seeds, context, false, false, false, false, false, + false, null); assertTrue(repoFile(destinationRepo, "plugins/org.eclipse.core.runtime_3.4.0.v20080512.jar").exists()); } diff --git a/tycho-its/projects/p2Repository.repositoryRef.filter.providing/category.xml b/tycho-its/projects/p2Repository.repositoryRef.filter.providing/category.xml new file mode 100644 index 0000000000..6859b3b7b2 --- /dev/null +++ b/tycho-its/projects/p2Repository.repositoryRef.filter.providing/category.xml @@ -0,0 +1,5 @@ + + + + + diff --git a/tycho-its/projects/p2Repository.repositoryRef.filter.providing/pom.xml b/tycho-its/projects/p2Repository.repositoryRef.filter.providing/pom.xml new file mode 100644 index 0000000000..b740a56a31 --- /dev/null +++ b/tycho-its/projects/p2Repository.repositoryRef.filter.providing/pom.xml @@ -0,0 +1,61 @@ + + + 4.0.0 + + 1.0.0 + tycho-its-project.p2Repository.repositoryRef.location + repositoryRef.location + eclipse-repository + + + + + repo0 + https://download.eclipse.org/eclipse/updates/4.29/ + p2 + + + repo-provided-by-others + https://download.eclipse.org/modeling/emf/emf/builds/release/2.35.0 + p2 + + + an-unused-repo + https://download.eclipse.org/egit/updates-6.7/ + p2 + + + an-unused-repo-but-added-explicitly-in-category-xml + https://download.eclipse.org/cbi/updates/license + p2 + + + + + + + org.eclipse.tycho + tycho-maven-plugin + ${tycho-version} + true + + + org.eclipse.tycho + tycho-p2-repository-plugin + ${tycho-version} + + false + true + true + true + true + + true + + + + + + diff --git a/tycho-its/src/test/java/org/eclipse/tycho/test/p2Repository/RepoRefLocationP2RepositoryIntegrationTest.java b/tycho-its/src/test/java/org/eclipse/tycho/test/p2Repository/RepoRefLocationP2RepositoryIntegrationTest.java index 802954f6b7..867c0fb77b 100644 --- a/tycho-its/src/test/java/org/eclipse/tycho/test/p2Repository/RepoRefLocationP2RepositoryIntegrationTest.java +++ b/tycho-its/src/test/java/org/eclipse/tycho/test/p2Repository/RepoRefLocationP2RepositoryIntegrationTest.java @@ -22,7 +22,9 @@ import java.io.File; import java.util.List; +import java.util.function.Consumer; +import org.apache.maven.it.VerificationException; import org.apache.maven.it.Verifier; import org.eclipse.tycho.test.AbstractTychoIntegrationTest; import org.eclipse.tycho.test.util.P2RepositoryTool; @@ -34,13 +36,10 @@ public class RepoRefLocationP2RepositoryIntegrationTest extends AbstractTychoInt @Test public void testRefLocation() throws Exception { - Verifier verifier = getVerifier("/p2Repository.repositoryRef.location", false); - verifier.addCliOption("-Dtest-data-repo=" + ResourceUtil.P2Repositories.ECLIPSE_LATEST.toString()); - verifier.executeGoal("package"); - verifier.verifyErrorFreeLog(); - P2RepositoryTool p2Repo = P2RepositoryTool.forEclipseRepositoryModule(new File(verifier.getBasedir())); - List allRepositoryReferences = p2Repo.getAllRepositoryReferences(); + List allRepositoryReferences = buildAndGetRepositoryReferences( + "/p2Repository.repositoryRef.location", + v -> v.addCliOption("-Dtest-data-repo=" + ResourceUtil.P2Repositories.ECLIPSE_LATEST.toString())); assertEquals(4, allRepositoryReferences.size()); assertThat(allRepositoryReferences, @@ -56,13 +55,9 @@ public void testReferenceFiltering() throws Exception { // references, but it makes the test simple/faster instead of preparing a // target-definition with IU-location so that it can be added automatically, // which is the main use-case. - Verifier verifier = getVerifier("/p2Repository.repositoryRef.filter", false); - verifier.executeGoal("package"); - verifier.verifyErrorFreeLog(); - - P2RepositoryTool p2Repo = P2RepositoryTool.forEclipseRepositoryModule(new File(verifier.getBasedir())); - List allRepositoryReferences = p2Repo.getAllRepositoryReferences(); - + List allRepositoryReferences = buildAndGetRepositoryReferences( + "/p2Repository.repositoryRef.filter", c -> { + }); assertEquals(4, allRepositoryReferences.size()); assertThat(allRepositoryReferences, containsInAnyOrder( // new RepositoryReference("https://download.eclipse.org/tm4e/releases/0.8.1", TYPE_ARTIFACT, ENABLED), @@ -71,4 +66,34 @@ public void testReferenceFiltering() throws Exception { new RepositoryReference("https://some.where/from/category", TYPE_METADATA, ENABLED))); } + @Test + public void testAdditionOfOnlyProvidingRepos() throws Exception { + // Of course it is actually a bit pointless to filter explicitly specified + // references, but it makes the test simple/faster instead of preparing a + // target-definition with IU-location so that it can be added automatically, + // which is the main use-case. + List allRepositoryReferences = buildAndGetRepositoryReferences( + "/p2Repository.repositoryRef.filter.providing", c -> { + }); + + assertEquals(4, allRepositoryReferences.size()); + assertThat(allRepositoryReferences, containsInAnyOrder( // + new RepositoryReference("https://download.eclipse.org/eclipse/updates/4.29", TYPE_ARTIFACT, ENABLED), + new RepositoryReference("https://download.eclipse.org/eclipse/updates/4.29", TYPE_METADATA, ENABLED), + new RepositoryReference("https://download.eclipse.org/cbi/updates/license", TYPE_ARTIFACT, ENABLED), + new RepositoryReference("https://download.eclipse.org/cbi/updates/license", TYPE_METADATA, ENABLED))); + } + + private List buildAndGetRepositoryReferences(String buildRoot, Consumer setup) + throws Exception, VerificationException { + Verifier verifier = getVerifier(buildRoot, false); + setup.accept(verifier); + verifier.executeGoal("package"); + verifier.verifyErrorFreeLog(); + + P2RepositoryTool p2Repo = P2RepositoryTool.forEclipseRepositoryModule(new File(verifier.getBasedir())); + List allRepositoryReferences = p2Repo.getAllRepositoryReferences(); + return allRepositoryReferences; + } + } diff --git a/tycho-p2-repository-plugin/src/main/java/org/eclipse/tycho/plugins/p2/repository/AssembleRepositoryMojo.java b/tycho-p2-repository-plugin/src/main/java/org/eclipse/tycho/plugins/p2/repository/AssembleRepositoryMojo.java index 6461550af3..e4b45a21fa 100644 --- a/tycho-p2-repository-plugin/src/main/java/org/eclipse/tycho/plugins/p2/repository/AssembleRepositoryMojo.java +++ b/tycho-p2-repository-plugin/src/main/java/org/eclipse/tycho/plugins/p2/repository/AssembleRepositoryMojo.java @@ -21,7 +21,6 @@ import java.util.Map; import java.util.function.Predicate; import java.util.regex.Pattern; -import java.util.stream.Collectors; import org.apache.maven.plugin.MojoExecutionException; import org.apache.maven.plugin.MojoFailureException; @@ -75,8 +74,17 @@ public class AssembleRepositoryMojo extends AbstractRepositoryMojo { public static class RepositoryReferenceFilter { + /** + * If {@link #filterProvided} is {@code true} and repository references are added + * automatically via {@link #addIUTargetRepositoryReferences} or + * {@link #addPomRepositoryReferences}, then this property controls if from the + * automatically added ones only references to those repositories are added, that provide + * relevant content, which is not provided by any other referenced repositories. If this is + * set to {@code false} all automatically added references are added as they are available. + */ + public boolean addOnlyProviding = false; /** The list of location patterns that exclude matching repository references. */ - List exclude = List.of(); + public List exclude = List.of(); } private static final Object LOCK = new Object(); @@ -224,20 +232,32 @@ public static class RepositoryReferenceFilter { private boolean addIUTargetRepositoryReferences; /** - * A list of patterns to exclude automatically derived repository references from being added to - * the assembled repository. + * Filters to exclude automatically derived repository references from being added to the + * assembled repository. + * *

+ * Repository references can be filtered based on their location URI using a list of exclusion + * pattern:
* The location of a reference must not be matched by any pattern, in order to be eventually - * added to the assembled repository. An arbitrary number of patterns can be specified.
- * The specified filters are only applied to those repository references derived from the + * added to the assembled repository. An arbitrary number of patterns can be specified. + *

+ *

+ * If the sub-property {@code addOnlyProviding} is set to {@code true}, references to + * repositories that don't provide any relevant unit are excluded from being added to the + * assembled repository. + *

+ *

+ * All those filters are only applied to those repository references derived from the * target-definition or pom file, when {@link #addIUTargetRepositoryReferences} respectively - * {@link #addPomRepositoryReferences} is set to {@code true}. + * {@link #addPomRepositoryReferences} is set {@code true}. References explicitly listed in the + * repository file ({@code category.xml}) are always added. *

*

* Configuration example 1 * * 

      * <repositoryReferenceFilter>
+     *   <addOnlyProviding>true</addOnlyProviding>
      *   <exclude>https://foo.bar.org/hidden/**</exclude>
      * </repositoryReferenceFilter>
      *
@@ -246,6 +266,7 @@ public static class RepositoryReferenceFilter { * * 
      * <repositoryReferenceFilter>
+     *   <addOnlyProviding>false</addOnlyProviding>
      *   <exclude>
      *     <location>https://foo.bar.org/hidden/**</location>
      *     <location>%regex[http(s)?:\/\/foo\.bar\.org\/secret\/.*]</location>
@@ -259,7 +280,8 @@ public static class RepositoryReferenceFilter {
      * {@code %regex[]}). 

      * The third pattern is a negated (enclosed in {@code ![]}), which
      * effectively makes it an inclusion pattern that all references must match in order to
-     * be added.
+     * be added. Unlike in the first example, in the second example all references that pass the
+     * location filter are added, regardless of if the provide any unit or not.
      * 

      */
     @Parameter
@@ -312,14 +334,15 @@ public void execute() throws MojoExecutionException, MojoFailureException {
                         .map(Category::getRepositoryReferences)//
                         .flatMap(List::stream)//
                         .map(ref -> new RepositoryReference(ref.getName(), ref.getLocation(), ref.isEnabled()))//
-                        .collect(Collectors.toCollection(ArrayList::new));
+                        .toList();
                 Predicate autoReferencesFilter = buildRepositoryReferenceLocationFilter();
+                List autoRepositoryRefeferences = new ArrayList<>();
                 if (addPomRepositoryReferences) {
                     getProject().getRepositories().stream() //
                             .filter(pomRepo -> "p2".equals(pomRepo.getLayout()))
                             .filter(pomRepo -> autoReferencesFilter.test(pomRepo.getUrl()))
                             .map(pomRepo -> new RepositoryReference(pomRepo.getName(), pomRepo.getUrl(), true))
-                            .forEach(repositoryReferences::add);
+                            .forEach(autoRepositoryRefeferences::add);
                 }
                 if (addIUTargetRepositoryReferences) {
                     projectManager.getTargetPlatformConfiguration(getProject()).getTargets().stream()
@@ -328,14 +351,15 @@ public void execute() throws MojoExecutionException, MojoFailureException {
                             .flatMap(iu -> iu.getRepositories().stream())
                             .filter(iuRepo -> autoReferencesFilter.test(iuRepo.getLocation()))
                             .map(iuRepo -> new RepositoryReference(null, iuRepo.getLocation(), true))
-                            .forEach(repositoryReferences::add);
+                            .forEach(autoRepositoryRefeferences::add);
                 }
                 DestinationRepositoryDescriptor destinationRepoDescriptor = new DestinationRepositoryDescriptor(
                         destination, repositoryName, compress, xzCompress, keepNonXzIndexFiles,
-                        !createArtifactRepository, true, extraArtifactRepositoryProperties, repositoryReferences);
+                        !createArtifactRepository, true, extraArtifactRepositoryProperties, repositoryReferences,
+                        autoRepositoryRefeferences);
                 mirrorApp.mirrorReactor(sources, destinationRepoDescriptor, projectSeeds, getBuildContext(),
                         includeAllDependencies, includeAllSources, includeRequiredPlugins, includeRequiredFeatures,
-                        filterProvided, profileProperties);
+                        filterProvided, repositoryReferenceFilter.addOnlyProviding, profileProperties);
                 if (generateOSGiRepository) {
                     XMLResourceGenerator resourceGenerator = new XMLResourceGenerator();
                     resourceGenerator.name(repositoryName);