Skip to content

Commit

Permalink
Suppress addition of unnecessary repo-refereces when assembling p2-repos
Browse files Browse the repository at this point in the history
  • Loading branch information
@HannesWell
HannesWell committed Sep 29, 2023
1 parent 0c3be9b commit b4ba256
Show file tree
Hide file tree
Showing 10 changed files with 282 additions and 58 deletions.
8 changes: 6 additions & 2 deletions RELEASE_NOTES.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,8 +6,11 @@ This page describes the noteworthy improvements provided by each release of Ecli

### new option to filter added repository-references when assembling a p2-repository

The repository references automatically added to a assembled p2-repository (via `tycho-p2-repository-plugin`'s `addIUTargetRepositoryReferences` or `addPomRepositoryReferences`)
can now be filtered by their location using exclusion and inclusion patterns and therefore allows more fine-grained control which references are added.
If filtering provided artifacts is enabled, the repository references automatically added to a assembled p2-repository
(via `tycho-p2-repository-plugin`'s `addIUTargetRepositoryReferences` or `addPomRepositoryReferences`) can now be filtered by their location
using exclusion and inclusion patterns and therefore allows more fine-grained control which references are added.
Additionally the automatically added references can be filter based on if they provide any of the filtered units or not.
If `addOnlyProviding` is `true` repositories that don't provide any filtered unit are not added to the assembled repo.
```xml
<plugin>
<groupId>org.eclipse.tycho</groupId>
Expand All @@ -16,6 +19,7 @@ can now be filtered by their location using exclusion and inclusion patterns and
<configuration>
... other configuration options ...
<repositoryReferenceFilter>
<addOnlyProviding>true</addOnlyProviding>
<exclude>
<location>https://foo.bar.org/hidden/**</location>
<location>%regex[http(s)?:\/\/foo\.bar\.org\/secret\/.*]</location>
Expand Down
11 changes: 9 additions & 2 deletions tycho-core/src/main/java/org/eclipse/tycho/p2/tools/DestinationRepositoryDescriptor.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,10 +28,12 @@ public class DestinationRepositoryDescriptor {
private final boolean append;
private final Map<String, String> extraArtifactRepositoryProperties;
private final List<RepositoryReference> repositoryReferences;
private final List<RepositoryReference> filterablRepositoryReferences;

public DestinationRepositoryDescriptor(File location, String name, boolean compress, boolean xzCompress,
boolean keepNonXzIndexFiles, boolean metaDataOnly, boolean append,
Map<String, String> extraArtifactRepositoryProperties, List<RepositoryReference> repositoryReferences) {
Map<String, String> extraArtifactRepositoryProperties, List<RepositoryReference> repositoryReferences,
List<RepositoryReference> filterablRepositoryReferences) {
this.location = location;
this.name = name;
this.compress = compress;
Expand All @@ -41,12 +43,13 @@ public DestinationRepositoryDescriptor(File location, String name, boolean compr
this.append = append;
this.extraArtifactRepositoryProperties = extraArtifactRepositoryProperties;
this.repositoryReferences = repositoryReferences;
this.filterablRepositoryReferences = filterablRepositoryReferences;
}

public DestinationRepositoryDescriptor(File location, String name, boolean compress, boolean xzCompress,
boolean keepNonXzIndexFiles, boolean metaDataOnly, boolean append) {
this(location, name, compress, xzCompress, keepNonXzIndexFiles, metaDataOnly, append, Collections.emptyMap(),
Collections.emptyList());
Collections.emptyList(), Collections.emptyList());
}

public DestinationRepositoryDescriptor(File location, String name) {
Expand Down Expand Up @@ -88,4 +91,8 @@ public Map<String, String> getExtraArtifactRepositoryProperties() {
public List<RepositoryReference> getRepositoryReferences() {
return repositoryReferences == null ? Collections.emptyList() : repositoryReferences;
}

public List<RepositoryReference> getFilterableRepositoryReferences() {
return filterablRepositoryReferences;
}
}
9 changes: 7 additions & 2 deletions ...e/src/main/java/org/eclipse/tycho/p2/tools/mirroring/facade/MirrorApplicationService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,11 @@ public interface MirrorApplicationService {
* Whether to include bundles mentioned in the require section of a feature
* @param includeRequiredFeatures
* Whether to include features mentioned in the require section of a feature
* @param filterProvided Whether to filter IU/artifacts that are already provided by a referenced repository
* @param filterProvided
* Whether to filter IU/artifacts that are already provided by a referenced
* repository
* @param addOnlyProvidingRepoReferences
* Whether to add only repository-references that provide any relevant IU
* @param filterProperties
* additional filter properties to be set in the p2 slicing options. May be
* <code>null</code>
Expand All @@ -64,7 +68,8 @@ public interface MirrorApplicationService {
public void mirrorReactor(RepositoryReferences sources, DestinationRepositoryDescriptor destination,
Collection<DependencySeed> seeds, BuildContext context, boolean includeAllDependencies,
boolean includeAllSource, boolean includeRequiredBundles, boolean includeRequiredFeatures,
boolean filterProvided, Map<String, String> filterProperties) throws FacadeException;
boolean filterProvided, boolean addOnlyProvidingRepoReferences, Map<String, String> filterProperties)
throws FacadeException;

/**
* recreates the metadata of an existing repository e.g. to account for changes in the contained
Expand Down
4 changes: 3 additions & 1 deletion tycho-core/src/main/java/org/eclipse/tycho/p2tools/MirrorApplicationServiceImpl.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -152,7 +152,8 @@ private static IQuery<IInstallableUnit> createQuery(IUDescription iu) {
public void mirrorReactor(RepositoryReferences sources, DestinationRepositoryDescriptor destination,
Collection<DependencySeed> projectSeeds, BuildContext context, boolean includeAllDependencies,
boolean includeAllSource, boolean includeRequiredBundles, boolean includeRequiredFeatures,
boolean filterProvided, Map<String, String> filterProperties) throws FacadeException {
boolean filterProvided, boolean addOnlyProvidingRepoReferences, Map<String, String> filterProperties)
throws FacadeException {
final TychoMirrorApplication mirrorApp = createMirrorApplication(sources, destination, agent);

// mirror scope: seed units...
Expand All @@ -163,6 +164,7 @@ public void mirrorReactor(RepositoryReferences sources, DestinationRepositoryDes
mirrorApp.setIncludeRequiredFeatures(includeRequiredFeatures);
mirrorApp.setIncludePacked(false); // no way, Tycho do no longer support packed artifacts anyways
mirrorApp.setFilterProvided(filterProvided);
mirrorApp.setAddOnlyProvidingRepoReferences(addOnlyProvidingRepoReferences);
mirrorApp.setEnvironments(context.getEnvironments());
SlicingOptions options = new SlicingOptions();
options.considerStrictDependencyOnly(!includeAllDependencies);
Expand Down
123 changes: 106 additions & 17 deletions tycho-core/src/main/java/org/eclipse/tycho/p2tools/TychoMirrorApplication.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,19 +10,27 @@
* Contributors:
* SAP SE - initial API and implementation
* Hannes Wellmann - Assemble repository for all environments in one pass
* Hannes Wellmann - Implement user-defined filtering and filtering based on relevance for automatically added repo-references
*******************************************************************************/
package org.eclipse.tycho.p2tools;

import static java.util.stream.Collectors.groupingBy;
import static java.util.stream.Collectors.mapping;
import static java.util.stream.Collectors.toList;

import java.lang.reflect.Field;
import java.lang.reflect.Method;
import java.net.URI;
import java.net.URISyntaxException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import java.util.stream.StreamSupport;

Expand All @@ -32,20 +40,23 @@
import org.eclipse.equinox.internal.p2.metadata.IRequiredCapability;
import org.eclipse.equinox.internal.p2.metadata.InstallableUnit;
import org.eclipse.equinox.internal.p2.metadata.RequiredCapability;
import org.eclipse.equinox.internal.p2.metadata.repository.LocalMetadataRepository;
import org.eclipse.equinox.p2.core.IProvisioningAgent;
import org.eclipse.equinox.p2.core.ProvisionException;
import org.eclipse.equinox.p2.internal.repository.tools.RepositoryDescriptor;
import org.eclipse.equinox.p2.internal.repository.tools.SlicingOptions;
import org.eclipse.equinox.p2.metadata.IArtifactKey;
import org.eclipse.equinox.p2.metadata.IInstallableUnit;
import org.eclipse.equinox.p2.metadata.IRequirement;
import org.eclipse.equinox.p2.metadata.Version;
import org.eclipse.equinox.p2.metadata.expression.IMatchExpression;
import org.eclipse.equinox.p2.query.CollectionResult;
import org.eclipse.equinox.p2.query.IQueryResult;
import org.eclipse.equinox.p2.query.IQueryable;
import org.eclipse.equinox.p2.query.QueryUtil;
import org.eclipse.equinox.p2.repository.IRepository;
import org.eclipse.equinox.p2.repository.IRepositoryManager;
import org.eclipse.equinox.p2.repository.IRepositoryReference;
import org.eclipse.equinox.p2.repository.artifact.IArtifactRepository;
import org.eclipse.equinox.p2.repository.artifact.IArtifactRepositoryManager;
import org.eclipse.equinox.p2.repository.metadata.IMetadataRepository;
Expand All @@ -58,18 +69,17 @@ public class TychoMirrorApplication extends org.eclipse.tycho.p2tools.copiedfrom

private static final String SOURCE_SUFFIX = ".source";
private static final String FEATURE_GROUP = ".feature.group";
private final Map<String, String> extraArtifactRepositoryProperties;
private final List<RepositoryReference> repositoryReferences;
private final DestinationRepositoryDescriptor destination;
private boolean includeAllSource;
private boolean includeRequiredBundles;
private boolean includeRequiredFeatures;
private boolean filterProvided;
private boolean addOnlyProvidingRepoReferences;
private TargetPlatform targetPlatform;

public TychoMirrorApplication(IProvisioningAgent agent, DestinationRepositoryDescriptor destination) {
super(agent);
this.extraArtifactRepositoryProperties = destination.getExtraArtifactRepositoryProperties();
this.repositoryReferences = destination.getRepositoryReferences();
this.destination = destination;
this.removeAddedRepositories = false;
}

Expand All @@ -79,7 +89,7 @@ protected IArtifactRepository initializeDestination(RepositoryDescriptor toInit,
IArtifactRepository result = super.initializeDestination(toInit, mgr);
// simple.SimpleArtifactRepository.PUBLISH_PACK_FILES_AS_SIBLINGS is not public
result.setProperty("publishPackFilesAsSiblings", "true");
extraArtifactRepositoryProperties.forEach(result::setProperty);
destination.getExtraArtifactRepositoryProperties().forEach(result::setProperty);
return result;
}

Expand Down Expand Up @@ -173,34 +183,41 @@ public IQueryable<IInstallableUnit> slice(IInstallableUnit[] ius, IProgressMonit
}
return slice;
}

};
}

@Override
protected IMetadataRepository initializeDestination(RepositoryDescriptor toInit, IMetadataRepositoryManager mgr)
throws ProvisionException {
IMetadataRepository result = super.initializeDestination(toInit, mgr);
var refs = repositoryReferences.stream().flatMap(TychoMirrorApplication::toSpiRepositoryReferences).toList();
var refs = Stream.of(destination.getRepositoryReferences(), destination.getFilterableRepositoryReferences())
.flatMap(List::stream).flatMap(TychoMirrorApplication::toSpiRepositoryReferences).toList();
result.addReferences(refs);
return result;
}

private static Stream<org.eclipse.equinox.p2.repository.spi.RepositoryReference> toSpiRepositoryReferences(
RepositoryReference rr) {
return Stream.of(IRepository.TYPE_METADATA, IRepository.TYPE_ARTIFACT).map(type -> {
URI location = URI.create(rr.getLocation());
URI location = getNormalizedLocation(rr);
int options = rr.isEnable() ? IRepository.ENABLED : IRepository.NONE;
return new org.eclipse.equinox.p2.repository.spi.RepositoryReference(location, rr.getName(), type, options);
});
}

private static URI getNormalizedLocation(RepositoryReference r) {
// P2 does the same before loading the repo and thus IRepository.getLocation() returns the normalized URL.
// In order to avoid stripping of slashes from URI instances do it now before URIs are created.
String location = r.getLocation();
return URI.create(location.endsWith("/") ? location.substring(0, location.length() - 1) : location);
}

@Override
protected List<IArtifactKey> collectArtifactKeys(Collection<IInstallableUnit> ius, IProgressMonitor monitor)
throws ProvisionException {
List<IArtifactKey> keys = super.collectArtifactKeys(ius, monitor);
if (isFilterProvidedItems()) {
removeProvidedItems(keys, getArtifactRepositoryManager(), monitor);
removeProvidedItems(keys, getArtifactRepositoryManager(), IRepository.TYPE_ARTIFACT, monitor);
}
return keys;
}
Expand All @@ -210,28 +227,96 @@ protected Set<IInstallableUnit> collectUnits(IQueryable<IInstallableUnit> slice,
throws ProvisionException {
Set<IInstallableUnit> units = super.collectUnits(slice, monitor);
if (isFilterProvidedItems()) {
removeProvidedItems(units, getMetadataRepositoryManager(), monitor);
Map<String, List<Version>> fullRepositoryContent = units.stream()
.collect(groupingBy(IInstallableUnit::getId, mapping(IInstallableUnit::getVersion, toList())));

List<IRepository<IInstallableUnit>> metadataRepositories = removeProvidedItems(units,
getMetadataRepositoryManager(), IRepository.TYPE_METADATA, monitor);

if (addOnlyProvidingRepoReferences) {
Set<URI> removableReferences = destination.getFilterableRepositoryReferences().stream()
.map(TychoMirrorApplication::getNormalizedLocation).collect(Collectors.toSet());
destination.getRepositoryReferences().stream().map(TychoMirrorApplication::getNormalizedLocation)
.forEach(removableReferences::remove); // keep reference if explicitly added to the repository
if (!removableReferences.isEmpty()) {
// Assume that for all units that correspond to artifacts the metadata either has a co-located artifact repository or a references to to one that contains it.
removeNotProvidingReferences(fullRepositoryContent, metadataRepositories, removableReferences);
}
}
}
return units;
}

private boolean isFilterProvidedItems() {
return filterProvided && !repositoryReferences.isEmpty();
return filterProvided && !destinationMetadataRepository.getReferences().isEmpty();
}

private <T> void removeProvidedItems(Collection<T> allElements, IRepositoryManager<T> repoManager,
IProgressMonitor monitor) throws ProvisionException {
private <T> List<IRepository<T>> removeProvidedItems(Collection<T> allElements, IRepositoryManager<T> repoManager,
int repositoryType, IProgressMonitor monitor) throws ProvisionException {
List<IRepository<T>> referencedRepositories = new ArrayList<>();
for (RepositoryReference reference : repositoryReferences) {
for (IRepositoryReference reference : destinationMetadataRepository.getReferences()) {
if (reference.getType() != repositoryType) {
continue;
}
try {
URI location = new URI(reference.getLocation());
URI location = reference.getLocation();
IRepository<T> repository = loadRepository(repoManager, location, monitor);
referencedRepositories.add(repository);
} catch (URISyntaxException e) {
throw new ProvisionException("Can't parse referenced URI!", e);
} catch (IllegalArgumentException e) {
if (e.getCause() instanceof URISyntaxException uriException) {
throw new ProvisionException("Can't parse referenced URI!", uriException);
} else {
throw e;
}
}
}
allElements.removeIf(e -> referencedRepositories.stream().anyMatch(repo -> contains(repo, e)));
return referencedRepositories;
}

private void removeNotProvidingReferences(Map<String, List<Version>> fullRepositoryContent,
List<IRepository<IInstallableUnit>> metadataRepositories, Set<URI> removableReferenceURIs) {
Map<URI, Set<IInstallableUnit>> usedRepositoryItems = new HashMap<>();
for (IRepository<IInstallableUnit> repo : metadataRepositories) {
IQueryResult<IInstallableUnit> allUnits = repo.query(QueryUtil.ALL_UNITS, null);
Set<IInstallableUnit> usedRepoContent = stream(allUnits)
.filter(a -> fullRepositoryContent.getOrDefault(a.getId(), List.of()).contains(a.getVersion()))
.collect(Collectors.toSet());
usedRepositoryItems.put(repo.getLocation(), usedRepoContent);
}
// Remove filterable references that contribute nothing or whose relevant content is also provided by another repo
usedRepositoryItems.entrySet().removeIf(repo -> {
if (!removableReferenceURIs.contains(repo.getKey())) {
return false;
}
Set<IInstallableUnit> usedContent = repo.getValue();
return usedContent.isEmpty()
|| usedRepositoryItems.entrySet().stream().filter(e -> e != repo).map(Entry::getValue)
.anyMatch(other -> other.size() >= usedContent.size() && other.containsAll(usedContent));
});
IMetadataRepository repository = getDestinationMetadataRepository();
List<IRepositoryReference> discardedReferences = repository.getReferences().stream()
.filter(rr -> !usedRepositoryItems.keySet().contains(rr.getLocation())).toList();
removeRepositoryReferences(repository, discardedReferences);
}

//TODO: Just call IMetadataRepository.removeReferences once available: https://github.com/eclipse-equinox/p2/pull/338
private static void removeRepositoryReferences(IMetadataRepository metadataRepository,
Collection<? extends IRepositoryReference> references) {
if (metadataRepository instanceof LocalMetadataRepository localRepo) {
try {
Field repositoriesField = LocalMetadataRepository.class.getDeclaredField("repositories");
repositoriesField.trySetAccessible();
Method save = LocalMetadataRepository.class.getDeclaredMethod("save");
save.trySetAccessible();
@SuppressWarnings("unchecked")
Set<IRepositoryReference> repositories = (Set<IRepositoryReference>) repositoriesField.get(localRepo);
repositories.removeAll(references);
save.invoke(localRepo);
} catch (ReflectiveOperationException e) {
throw new IllegalStateException("Failed to clean-up references from assembled repository", e);
}
}
}

//TODO: just call IRepositoryManager.loadRepository() once available: https://github.com/eclipse-equinox/p2/pull/311
Expand Down Expand Up @@ -278,4 +363,8 @@ public void setFilterProvided(boolean filterProvided) {
this.filterProvided = filterProvided;
}

public void setAddOnlyProvidingRepoReferences(boolean addOnlyProvidingRepoReferences) {
this.addOnlyProvidingRepoReferences = addOnlyProvidingRepoReferences;
}

}
Loading

0 comments on commit b4ba256

Please sign in to comment.