Skip to content

Commit

Permalink
Merge pull request #3209 from matthiasblaesing/indexability_query
Browse files Browse the repository at this point in the history
Allow plugins to exclude files from being indexed
  • Loading branch information
matthiasblaesing committed Dec 14, 2021
2 parents 166180e + 81dc21a commit e991e94
Show file tree
Hide file tree
Showing 10 changed files with 1,360 additions and 623 deletions.
28 changes: 28 additions & 0 deletions ide/parsing.indexing/apichanges.xml
Expand Up @@ -87,6 +87,34 @@ is the proper place.
<!-- ACTUAL CHANGES BEGIN HERE: -->

<changes>
<change id="IndexabilityQuery">
<api name="IndexingAPI"/>
<summary>Allow plugins to exclude files from being indexed.</summary>
<version major="9" minor="24"/>
<date day="14" month="9" year="2021"/>
<author login="matthiasblaesing"/>
<compatibility source="compatible" binary="compatible" semantic="compatible" addition="yes"/>
<description>
<p>
There are situations where it is desireable to exclude files from the NetBeans indexing infrastructure. Implementors of
<a href="@TOP@/org/netbeans/modules/parsing/spi/indexing/IndexabilityQueryImplementation.html">IndexabilityQueryImplementation</a>
are queried whether an indexer should be invoked for a given file.
</p>
<p>
Indexing for a file can be rejected verbatim (<code>boolean preventIndexing(FileObject fo)</code>) or more specific by
indexer name, URL to the file to be indexed and the UTL of the indexing root.
<code>boolean preventIndexing(String indexerName, URL indexable, URL rootUrl)</code>.
</p>
<p>
One such example are Angular projects, where code assistence is provided by the typescript integration by the
language server. In these cases the <code>node_modules</code> folder often contains huge amounts of javascript
code, that the IDE user does not need to be indexed, but which take a lot of time to parse. A hypothetical
plugin could check if an <code>angular.json</code> file is present and then prevent the <code>js</code> indexer
from indexing files in the <code>node_modules</code> directory.
</p>
</description>
<class package="org.netbeans.modules.parsing.spi.indexing" name="IndexabilityQueryImplementation"/>
</change>
<change id="IndexingTask">
<api name="IndexingAPI"/>
<summary>Allow Parsers to adjust results for indexing</summary>
Expand Down
Expand Up @@ -20,10 +20,7 @@
package org.netbeans.modules.parsing.impl.indexing;

import org.netbeans.modules.parsing.spi.indexing.SuspendStatus;
import java.io.File;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.ArrayDeque;
import java.util.Collection;
import java.util.Comparator;
Expand Down Expand Up @@ -54,12 +51,13 @@ final class FileObjectCrawler extends Crawler {

private static final char SEPARATOR = '/'; //NOI18N
private static final Logger LOG = Logger.getLogger(FileObjectCrawler.class.getName());
/*test*/ static Map<Pair<FileObject,FileObject>,Boolean> mockLinkTypes;
@SuppressWarnings("PackageVisibleField") /*test*/
static Map<Pair<FileObject,FileObject>,Boolean> mockLinkTypes;

private final FileObject root;
private final ClassPath.Entry entry;
private final FileObject[] files;


FileObjectCrawler(
@NonNull final FileObject root,
Expand Down Expand Up @@ -176,7 +174,7 @@ protected boolean collectResources(Collection<Indexable> resources, Collection<I
StringBuilder relativePath = relPaths.get(parent);
if (relativePath != null) {
finished = collect(
cluster.toArray(new FileObject[cluster.size()]),
cluster.toArray(new FileObject[0]),
root,
resources,
allResources,
Expand Down Expand Up @@ -273,7 +271,7 @@ private boolean collect (
if (isCancelled()) {
return false;
}
if (!fo.isValid() || !isVisible(fo)) {
if (!fo.isValid() || !canBeIndexed(fo)) {
continue;
}

Expand Down Expand Up @@ -330,14 +328,15 @@ private StringBuilder getRelativePath(FileObject folder, FileObject fo) {
}
}

private boolean isVisible (final @NonNull FileObject fo) {
private boolean canBeIndexed (final @NonNull FileObject fo) {
try {
return VisibilityQuery.getDefault().isVisible(fo);
return VisibilityQuery.getDefault().isVisible(fo)
&& (! IndexabilityQuery.getInstance().preventIndexing(fo));
} finally {
setListenOnVisibility(true);
}
}

//Todo: Not exaclty correct. The correct implementation should find if whole root content
//is covered by files. But correct implementation will be very very slow and probably no one
//calls it with such params.
Expand Down Expand Up @@ -376,7 +375,7 @@ private static boolean isLink(
if (file.getNameExt().equals(pathElement.getNameExt())) {
try {
if (mockLinkTypes != null ?
mockLinkTypes.get(Pair.<FileObject,FileObject>of(pathElement, file)) :
mockLinkTypes.get(Pair.of(pathElement, file)) :
isSameFile(file, pathElement)) {
hasLink = true;
break;
Expand Down Expand Up @@ -438,28 +437,28 @@ private static PathRelation getFileRelation (
return PathRelation.EQUAL;
}
}

private static final class Stats {
public int filesCount;
public long linkCheckTime;
public int linkCount;
public Map<String, Integer> extensions = new HashMap<String, Integer>();
public Map<String, Integer> mimeTypes = new HashMap<String, Integer>();
public Map<String, Integer> extensions = new HashMap<>();
public Map<String, Integer> mimeTypes = new HashMap<>();
public static void inc(Map<String, Integer> m, String k) {
Integer i = m.get(k);
if (i == null) {
m.put(k, 1);
} else {
m.put(k, i.intValue() + 1);
m.put(k, i + 1);
}
}
public static void logHistogram(Level level, Map<String, Integer> data) {
Map<Integer, Set<String>> sortedMap = new TreeMap<Integer, Set<String>>(REVERSE);
Map<Integer, Set<String>> sortedMap = new TreeMap<>(REVERSE);
for(String item : data.keySet()) {
Integer freq = data.get(item);
Set<String> items = sortedMap.get(freq);
if (items == null) {
items = new TreeSet<String>();
items = new TreeSet<>();
sortedMap.put(freq, items);
}
items.add(item);
Expand All @@ -470,11 +469,7 @@ public static void logHistogram(Level level, Map<String, Integer> data) {
}
}
}
private static final Comparator<Integer> REVERSE = new Comparator<Integer>() {
public int compare(Integer o1, Integer o2) {
return -1 * o1.compareTo(o2);
}
};
private static final Comparator<Integer> REVERSE = (Integer o1, Integer o2) -> -1 * o1.compareTo(o2);
} // End of Stats class

private enum PathRelation {
Expand Down
@@ -0,0 +1,84 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.netbeans.modules.parsing.impl.indexing;

import java.util.Iterator;
import java.util.NoSuchElementException;
import java.util.function.Function;

/**
* Iterable wrapping another iterable, yielding an interator, that only offers
* values, for this the supplied filter returns a true value.
*/
class FilteringIterable<T> implements Iterable<T> {
private final Iterable<? extends T> delegate;
private final Function<T,Boolean> filter;

public FilteringIterable(Iterable<? extends T> delegate, Function<T,Boolean> filter) {
this.delegate = delegate;
this.filter = filter;
}

@Override
public Iterator<T> iterator() {
return new FilteringIterator(delegate.iterator(), filter);
}

private static class FilteringIterator<T> implements Iterator<T> {
private final Iterator<? extends T> delegate;
private final Function<T,Boolean> filter;

public FilteringIterator(Iterator<? extends T> delegate, Function<T,Boolean> filter) {
this.delegate = delegate;
this.filter = filter;
}

private T next;

@Override
public boolean hasNext() {
fillNextIfPossible();
return next != null;
}

@Override
public T next() {
fillNextIfPossible();
if(next == null) {
throw new NoSuchElementException();
}
T value = next;
next = null;
return value;
}

private void fillNextIfPossible() {
if (next == null) {
while (delegate.hasNext()) {
T nextCandiate = delegate.next();
if(filter.apply(nextCandiate)) {
next = nextCandiate;
break;
}
}
}
}
}
}

0 comments on commit e991e94

Please sign in to comment.