Skip to content

Commit

Permalink
Ensure changes to the IndexabilityQueries invalidate/update the indices
Browse files Browse the repository at this point in the history 
There are two different scenarios:

The first is changes of IndexabilityQueries at runtime. If these are
detected a full rescan is triggered. In this case it needs to be
considered, that changes to the supported/blocked indexers could have
happened and thus from the perspective of an affected indexer a removal
of the file has to be considered.

The second is changes of Indexability Queries between NetBeans restarts.
To support this, the timestamp state is coupled to the state of the
IndexabilityQueries. If the state of the IndexabilityQueries changes,
the timestamps will be invalidated and the corresponding files will be
rescanned.
  • Loading branch information
@matthiasblaesing
matthiasblaesing committed Dec 3, 2021
1 parent 8a3afff commit 35ab116
Show file tree
Hide file tree
Showing 7 changed files with 249 additions and 54 deletions.
2 changes: 1 addition & 1 deletion ide/parsing.indexing/src/org/netbeans/modules/parsing/impl/indexing/FileObjectCrawler.java
View file
Expand Up @@ -333,7 +333,7 @@ private StringBuilder getRelativePath(FileObject folder, FileObject fo) {
private boolean canBeIndexed (final @NonNull FileObject fo) {
try {
return VisibilityQuery.getDefault().isVisible(fo)
&& (! IndexabilityQuery.getDefault().preventIndexing(fo));
&& (! IndexabilityQuery.getInstance().preventIndexing(fo));
} finally {
setListenOnVisibility(true);
}
Expand Down
37 changes: 34 additions & 3 deletions ide/parsing.indexing/src/org/netbeans/modules/parsing/impl/indexing/IndexabilityQuery.java
View file
Expand Up @@ -21,9 +21,16 @@

import java.net.URL;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.logging.Level;
import java.util.logging.Logger;
import java.util.stream.Collectors;
import javax.swing.event.ChangeEvent;
import javax.swing.event.ChangeListener;
import org.netbeans.modules.parsing.spi.indexing.IndexabilityQueryImplementation;
Expand All @@ -40,6 +47,8 @@
* @see org.netbeans.modules.parsing.spi.indexing.IndexabilityQueryImplementation
*/
final class IndexabilityQuery {
private static final Logger LOG = Logger.getLogger(IndexabilityQuery.class.getName());

private static final IndexabilityQueryContextAccessor CONTEXT_CREATOR = IndexabilityQueryContextAccessor.getInstance();
private static final IndexabilityQuery INSTANCE = new IndexabilityQuery();

Expand All @@ -51,10 +60,10 @@ final class IndexabilityQuery {
private Lookup.Result<IndexabilityQueryImplementation> iqiResult = null;

/**
* Get default instance of IndexabilityQuery.
* Get instance of IndexabilityQuery.
* @return instance of IndexabilityQuery
*/
public static final IndexabilityQuery getDefault() {
public static final IndexabilityQuery getInstance() {
return INSTANCE;
}

Expand All @@ -81,6 +90,28 @@ public boolean preventIndexing (String indexerName, URL indexable, URL rootUrl)
return false;
}

public String getState() {
return getIqiInstances()
.stream()
.map(iqi -> iqi.getName() + "-" + iqi.getVersion() + "-" + iqi.getStateIdentifier())
.collect(Collectors.joining(","));
}

private Set<String> decodeState(String input) {
try {
return Arrays.stream(input.split(","))
.map(s -> s.trim())
.collect(Collectors.toSet());
} catch (Exception ex) {
LOG.log(Level.SEVERE, "Failed to parse IndexabilityQuery state from '" + input + "'", ex);
return Collections.EMPTY_SET;
}
}

public boolean isSameState(String reference) {
return Objects.equals(decodeState(reference), decodeState(getState()));
}

/**
* Add a listener to changes.
* @param l a listener to add
Expand Down Expand Up @@ -151,7 +182,7 @@ private class ResultListener implements LookupListener {
@Override
public void resultChanged(LookupEvent ev) {
setupChangeListeners(cachedIqiInstances, new ArrayList<>(iqiResult.allInstances()));
fireChange(new ChangeEvent(this));
fireChange(new ChangeEvent(IndexabilityQuery.this));
}
}

Expand Down
71 changes: 49 additions & 22 deletions ide/parsing.indexing/src/org/netbeans/modules/parsing/impl/indexing/IndexabilitySupport.java
View file
Expand Up @@ -53,7 +53,7 @@
*
* @author Tomas Zezula
*/
class IndexabilitySupport implements ChangeListener {
class IndexabilitySupport {

private static final int VISIBILITY_CHANGE_WINDOW = 1000;
private static final Logger LOGGER = Logger.getLogger(IndexabilitySupport.class.getName());
Expand All @@ -63,21 +63,45 @@ class IndexabilitySupport implements ChangeListener {
private final SlidingTask visibilityTask;
private final RequestProcessor.Task visibilityChanged;

private final ChangeListener visibilityListener;
private final ChangeListener indexabilityListener;

private IndexabilitySupport(
@NonNull final RepositoryUpdater ru,
@NonNull final RequestProcessor worker) {
this.visibilityTask = new SlidingTask(ru);
this.visibilityChanged = worker.create(this.visibilityTask);

visibilityListener = (ChangeEvent e) -> {
visibilityCache.clear();
if (Crawler.listenOnVisibility()) {
if (e instanceof VisibilityQueryChangeEvent) {
final FileObject[] affectedFiles = ((VisibilityQueryChangeEvent) e).getFileObjects();
visibilityTask.localChange(affectedFiles);
} else {
visibilityTask.globalChange();
}
visibilityChanged.schedule(VISIBILITY_CHANGE_WINDOW);
}
};

indexabilityListener = (ChangeEvent e) -> {
// Indexability could invalidate a subset of the indexer,
// so there is a valid index state for a file, but it might
// contain more/less data than intended after the change
visibilityTask.globalChangeFull();
visibilityChanged.schedule(VISIBILITY_CHANGE_WINDOW);
};
}

void start() {
VisibilityQuery.getDefault().addChangeListener(this);
IndexabilityQuery.getDefault().addChangeListener(this);
VisibilityQuery.getDefault().addChangeListener(visibilityListener);
IndexabilityQuery.getInstance().addChangeListener(indexabilityListener);
}

void stop() {
VisibilityQuery.getDefault().removeChangeListener(this);
IndexabilityQuery.getDefault().removeChangeListener(this);
VisibilityQuery.getDefault().removeChangeListener(visibilityListener);
IndexabilityQuery.getInstance().removeChangeListener(indexabilityListener);
}

boolean canIndex(
Expand All @@ -89,7 +113,7 @@ boolean canIndex(
}
try {
final VisibilityQuery vq = VisibilityQuery.getDefault();
final IndexabilityQuery iq = IndexabilityQuery.getDefault();
final IndexabilityQuery iq = IndexabilityQuery.getInstance();
final Deque<FileObject> fta = new ArrayDeque<>();
Boolean vote = null;
boolean folder = false;
Expand Down Expand Up @@ -130,26 +154,13 @@ boolean canIndex(
}
}

@Override
public void stateChanged(ChangeEvent e) {
visibilityCache.clear();
if (Crawler.listenOnVisibility()) {
if (e instanceof VisibilityQueryChangeEvent) {
final FileObject[] affectedFiles = ((VisibilityQueryChangeEvent)e).getFileObjects();
visibilityTask.localChange(affectedFiles);
} else {
visibilityTask.globalChange();
}
visibilityChanged.schedule(VISIBILITY_CHANGE_WINDOW);
}
}

private static class SlidingTask implements Runnable {

private final RepositoryUpdater ru;

//@GuardedBy("this")
private boolean globalChange;
private boolean fullScan;
private final Set</*@GuardedBy("this")*/FileObject> localChanges = new HashSet<>();
//@GuardedBy("this")
private LogContext visibilityLogCtx;
Expand All @@ -158,6 +169,15 @@ private static class SlidingTask implements Runnable {
this.ru = ru;
}

synchronized void globalChangeFull() {
globalChange = true;
fullScan = true;

if (visibilityLogCtx == null) {
visibilityLogCtx = LogContext.create(LogContext.EventType.FILE, null);
}
}

synchronized void globalChange() {
globalChange = true;

Expand All @@ -176,19 +196,26 @@ synchronized void localChange(final FileObject... onFiles) {
@Override
public void run() {
final boolean global;
final boolean full;
final Collection<FileObject> changedFiles;
final LogContext logCtx;
synchronized (this) {
logCtx = visibilityLogCtx;
visibilityLogCtx = null;
global = globalChange;
full = fullScan;
globalChange = false;
fullScan = false;
changedFiles = new ArrayList<FileObject>(localChanges);
localChanges.clear();
}
if (global) {
LOGGER.fine ("VisibilityQuery global changed, reindexing"); //NOI18N
ru.refreshAll(false, false, true, logCtx);
if(full) {
ru.refreshAll(true, false, true, logCtx);
} else {
ru.refreshAll(false, false, true, logCtx);
}
} else {
if (LOGGER.isLoggable(Level.FINE)) {
LOGGER.log (
Expand All @@ -201,7 +228,7 @@ public void run() {
final Set<URI> binChangedRoot = new HashSet<>();
final Map<URI,TimeStamps> tsPerRoot = new HashMap<>();
final VisibilityQuery vq = VisibilityQuery.getDefault();
final IndexabilityQuery iq = IndexabilityQuery.getDefault();
final IndexabilityQuery iq = IndexabilityQuery.getInstance();
for (FileObject chf : changedFiles) {
Pair<URL,FileObject> owner = ru.getOwningSourceRoot(chf);
if (owner != null) {
Expand Down
64 changes: 60 additions & 4 deletions ide/parsing.indexing/src/org/netbeans/modules/parsing/impl/indexing/RepositoryUpdater.java
View file
Expand Up @@ -2742,6 +2742,13 @@ private boolean doIndex(
return false;
}


Iterable<Indexable> notIndexables = new FilteringIterable(
new ProxyIterable<>(indexerIndexablesList),
notIndexableFilter(factory, value.second().getRootURI()));

factory.filesDeleted(notIndexables, value.second());

final CustomIndexer indexer = factory.createIndexer();
logStartIndexer(factory.getIndexerName());
SPIAccessor.getInstance().putProperty(value.second(), ClusteredIndexables.INDEX, usedCi);
Expand Down Expand Up @@ -3074,7 +3081,8 @@ protected final boolean indexBinary(
indexers.bifs);

for(BinaryIndexerFactory f : indexers.bifs) {
if(IndexabilityQuery.getDefault().preventIndexing(f.getIndexerName(), root, null)) {
if(IndexabilityQuery.getInstance().preventIndexing(f.getIndexerName(), root, null)) {
f.rootsRemoved(Collections.singleton(root));
continue;
}

Expand Down Expand Up @@ -3112,6 +3120,7 @@ protected final boolean indexBinary(
long et = System.currentTimeMillis();
logIndexerTime(f.getIndexerName(), (int)(et-st));
}

return !getCancelRequest().isRaised();
}

Expand All @@ -3124,15 +3133,17 @@ protected final boolean indexEmbedding(
final Map<Pair<String,Integer>,Pair<SourceIndexerFactory,Context>> transactionContexts,
final boolean sourceForBinaryRoot
) throws IOException {
IndexabilityQuery iq = IndexabilityQuery.getDefault();
IndexabilityQuery iq = IndexabilityQuery.getInstance();
for (final Indexable dirty : files) {
parkWhileSuspended();
if (getCancelRequest().isRaised()) {
return false;
}

Collection<? extends IndexerCache.IndexerInfo<EmbeddingIndexerFactory>> allIndexers =
getIndexerInfos(eifInfosMap, dirty.getMimeType());
Collection<? extends IndexerCache.IndexerInfo<EmbeddingIndexerFactory>> infos =
getIndexerInfos(eifInfosMap, dirty.getMimeType())
allIndexers
.stream()
.filter(i -> ! iq.preventIndexing(
i.getIndexerName(),
Expand All @@ -3142,6 +3153,34 @@ protected final boolean indexEmbedding(
)
.collect(Collectors.toList());

allIndexers
.stream()
.filter(i -> iq.preventIndexing(
i.getIndexerName(),
dirty.getURL(),
rootURL
)
)
.forEach(i -> {
try {
final Context context = SPIAccessor.getInstance().createContext(
cache,
rootURL,
i.getIndexerName(),
i.getIndexerVersion(),
null,
followUpJob,
checkEditor,
sourceForBinaryRoot,
getSuspendStatus(),
getCancelRequest(),
logCtx);
i.getIndexerFactory().filesDeleted(Collections.singleton(dirty), context);
} catch (IOException ex) {
LOG.log(Level.WARNING, null, ex);
}
});

if (infos != null && infos.size() > 0) {
final URL url = dirty.getURL();
if (url == null) {
Expand Down Expand Up @@ -3261,6 +3300,7 @@ public void run(ResultIterator resultIterator) throws Exception {

return !getCancelRequest().isRaised();
}
private static final Logger LOG = Logger.getLogger(Work.class.getName());

protected final boolean scanFiles(
@NonNull final URL root,
Expand Down Expand Up @@ -4162,6 +4202,13 @@ public Boolean call() throws Exception {
indexableFilter(factory, ctx.second().getRootURI()));

SPIAccessor.getInstance().setAllFilesJob(ctx.second(), true);

Iterable<Indexable> notIndexables = new FilteringIterable(
new ProxyIterable<>(indexerIndexablesList),
notIndexableFilter(factory, ctx.second().getRootURI()));

factory.filesDeleted(notIndexables, ctx.second());

final CustomIndexer indexer = factory.createIndexer();
if (LOGGER.isLoggable(Level.FINE)) {
StringBuilder sb = printMimeTypes(cifInfo.getMimeTypes(), new StringBuilder());
Expand Down Expand Up @@ -6705,7 +6752,16 @@ public boolean cancel() {

private static Function<Indexable, Boolean> indexableFilter(final CustomIndexerFactory factory, URL rootUrl) {
Function<Indexable, Boolean> canBeIndexed = indexable
-> !IndexabilityQuery.getDefault().preventIndexing(
-> !IndexabilityQuery.getInstance().preventIndexing(
factory.getIndexerName(),
indexable.getURL(),
rootUrl);
return canBeIndexed;
}

private static Function<Indexable, Boolean> notIndexableFilter(final CustomIndexerFactory factory, URL rootUrl) {
Function<Indexable, Boolean> canBeIndexed = indexable
-> IndexabilityQuery.getInstance().preventIndexing(
factory.getIndexerName(),
indexable.getURL(),
rootUrl);
Expand Down

0 comments on commit 35ab116

Please sign in to comment.