Allow plugins to exclude files from being indexed

This commit adds a query, that allows plugins to prevent files from
being indexed completely or by certain indexers.

One motivator for this is the JS indexer in combination with huge
node_modules directories. In the case of angular/typescript projects the
JS indexer does not help with development, as code assistence is
provided by the typescript LSP server. But the JS indexer is ran anyway
and created huge scan times and big indices. With these new hooks a
plugin could limit the indexing when an angular project is detected.

ide/parsing.indexing/apichanges.xml

@@ -87,6 +87,34 @@ is the proper place.
 <!-- ACTUAL CHANGES BEGIN HERE: -->
 
   <changes>
+      <change id="IndexabilityQuery">
+          <api name="IndexingAPI"/>
+          <summary>Allow plugins to exclude files from being indexed.</summary>
+          <version major="9" minor="22"/>
+          <date day="14" month="9" year="2021"/>
+          <author login="matthiasblaesing"/>
+          <compatibility source="compatible" binary="compatible" semantic="compatible" addition="yes"/>
+          <description>
+              <p>
+                  There are situations where it is desireable to exclude files from the NetBeans indexing infrastructure. Implementors of
+                  <a href="@TOP@/org/netbeans/modules/parsing/spi/indexing/IndexabilityQueryImplementation.html">IndexabilityQueryImplementation</a>
+                  are queried whether an indexer should be invoked for a given file.
+              </p>
+              <p>
+                  Indexing for a file can be rejected verbatim (<code>boolean preventIndexing(FileObject fo)</code>) or more specific by
+                  indexer name, indexer factory class name, the file to be indexed and the indexing root.
+                  <code>boolean preventIndexing(String indexerName, String factoryClassName, URL indexable, URL rootUrl)</code>.
+              </p>
+              <p>
+                  One such example are Angular projects, where code assistence is provided by the typescript integration by the
+                  language server. In these cases the <code>node_modules</code> folder often contains huge amounts of javascript
+                  code, that the IDE user does not need to be indexed, but which take a lot of time to parse. A hypothetical
+                  plugin could check if an <code>angular.json</code> file is present and then prevent the <code>js</code> indexer
+                  from indexing files in the <code>node_modules</code> directory.
+              </p>
+          </description>
+          <class package="org.netbeans.modules.parsing.spi.indexing" name="IndexabilityQueryImplementation"/>
+      </change>
       <change id="IndexingTask">
           <api name="IndexingAPI"/>
           <summary>Allow Parsers to adjust results for indexing</summary>

ide/parsing.indexing/src/org/netbeans/modules/parsing/impl/indexing/FileObjectCrawler.java

@@ -273,7 +273,7 @@ private boolean collect (
             if (isCancelled()) {
                 return false;
             }
-            if (!fo.isValid() || !isVisible(fo)) {
+            if (!fo.isValid() || !canBeIndexed(fo)) {
                 continue;
             }
 
@@ -330,9 +330,10 @@ private StringBuilder getRelativePath(FileObject folder, FileObject fo) {
         }
     }
 
-    private boolean isVisible (final @NonNull FileObject fo) {
+    private boolean canBeIndexed (final @NonNull FileObject fo) {
         try {
-            return VisibilityQuery.getDefault().isVisible(fo);
+            return VisibilityQuery.getDefault().isVisible(fo)
+                    && (! IndexabilityQuery.getDefault().preventIndexing(fo));
         } finally {
             setListenOnVisibility(true);
         }

ide/parsing.indexing/src/org/netbeans/modules/parsing/impl/indexing/FilteringIterable.java

@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.netbeans.modules.parsing.impl.indexing;
+
+import java.util.Iterator;
+import java.util.NoSuchElementException;
+import java.util.function.Function;
+
+/**
+ * Iterable wrapping another iterable, yielding an interator, that only offers
+ * values, for this the supplied filter returns a true value.
+ */
+public class  FilteringIterable<T> implements Iterable<T> {
+    private final Iterable<? extends T> delegate;
+    private final Function<T,Boolean> filter;
+
+    public FilteringIterable(Iterable<? extends T> delegate, Function<T,Boolean> filter) {
+        this.delegate = delegate;
+        this.filter = filter;
+    }
+
+    @Override
+    public Iterator<T> iterator() {
+        return new FilteringIterator(delegate.iterator(), filter);
+    }
+
+}
+
+class FilteringIterator<T> implements Iterator<T> {
+    private final Iterator<? extends T> delegate;
+    private final Function<T,Boolean> filter;
+
+    public FilteringIterator(Iterator<? extends T> delegate, Function<T,Boolean> filter) {
+        this.delegate = delegate;
+        this.filter = filter;
+    }
+
+    private T next;
+
+    @Override
+    public boolean hasNext() {
+        fillNextIfPossible();
+        return next != null;
+    }
+
+    @Override
+    public T next() {
+        fillNextIfPossible();
+        if(next == null) {
+            throw new NoSuchElementException();
+        }
+        T value = next;
+        next = null;
+        return value;
+    }
+
+    private void fillNextIfPossible() {
+        if (next == null) {
+            while (delegate.hasNext()) {
+                T nextCandiate = delegate.next();
+                if(filter.apply(nextCandiate)) {
+                    next = nextCandiate;
+                    break;
+                }
+            }
+        }
+    }
+}

ide/parsing.indexing/src/org/netbeans/modules/parsing/impl/indexing/IndexabilityQuery.java

@@ -0,0 +1,160 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.netbeans.modules.parsing.impl.indexing;
+
+import java.net.URL;
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import javax.swing.event.ChangeEvent;
+import javax.swing.event.ChangeListener;
+import org.netbeans.modules.parsing.spi.indexing.IndexabilityQueryImplementation;
+import org.openide.filesystems.FileObject;
+import org.openide.util.Exceptions;
+import org.openide.util.Lookup;
+import org.openide.util.LookupEvent;
+import org.openide.util.LookupListener;
+
+/**
+ * Determine whether files should be skipped by the netbeans indexing infrastructure.
+ * 
+ * @see org.netbeans.modules.parsing.spi.indexing.IndexabilityQueryImplementation
+ */
+public final class IndexabilityQuery {
+    private static final IndexabilityQuery INSTANCE = new IndexabilityQuery();
+
+    private final ResultListener resultListener = new ResultListener();
+    private final IqiChangedListener vqiListener = new IqiChangedListener();
+
+    private final List<ChangeListener> listeners = new ArrayList<>();
+    private volatile List<IndexabilityQueryImplementation> cachedIqiInstances = null;
+    private Lookup.Result<IndexabilityQueryImplementation> iqiResult = null;
+
+    /**
+     * Get default instance of IndexabilityQuery.
+     * @return instance of IndexabilityQuery
+     */
+    public static final IndexabilityQuery getDefault() {
+        return INSTANCE;
+    }
+
+    private IndexabilityQuery() {
+    }
+
+    boolean preventIndexing(FileObject fileObject) {
+        for (IndexabilityQueryImplementation iqi : getIqiInstances()) {
+            if (iqi.preventIndexing(fileObject)) {
+                return true;
+            }
+        }
+        return false;
+    }
+
+    boolean preventIndexing (String indexerName, String factoryClassName, URL indexable, URL rootUrl)  {
+        for (IndexabilityQueryImplementation iqi : getIqiInstances()) {
+            if (iqi.preventIndexing(indexerName, factoryClassName, indexable, rootUrl)) {
+                return true;
+            }
+        }
+        return false;
+    }
+
+    /**
+     * Add a listener to changes.
+     * @param l a listener to add
+     */
+    public void addChangeListener(ChangeListener l) {
+        synchronized (listeners) {
+            listeners.add(l);
+        }
+    }
+
+    /**
+     * Stop listening to changes.
+     * @param l a listener to remove
+     */
+    public void removeChangeListener(ChangeListener l) {
+        synchronized (listeners) {
+            listeners.remove(l);
+        }
+    }
+
+    private void fireChange(ChangeEvent event) {
+        assert event != null;
+        ArrayList<ChangeListener> lists;
+        synchronized (listeners) {
+            lists = new ArrayList<>(listeners);
+        }
+        for (ChangeListener listener : lists) {
+            try {
+                listener.stateChanged(event);
+            } catch (RuntimeException x) {
+                Exceptions.printStackTrace(x);
+            }
+        }
+    }
+
+    @SuppressWarnings("ReturnOfCollectionOrArrayField")
+    private synchronized List<IndexabilityQueryImplementation> getIqiInstances() {
+        if (cachedIqiInstances == null) {
+            iqiResult = Lookup.getDefault().lookupResult(IndexabilityQueryImplementation.class);
+            iqiResult.addLookupListener(resultListener);
+            setupChangeListeners(null, new ArrayList<>(iqiResult.allInstances()));
+        }
+        return cachedIqiInstances;
+    }
+
+    @SuppressWarnings("AssignmentToCollectionOrArrayFieldFromParameter")
+    private synchronized void setupChangeListeners(final List<IndexabilityQueryImplementation> oldVqiInstances, final List<IndexabilityQueryImplementation> newVqiInstances) {
+        if (oldVqiInstances != null) {
+            Set<IndexabilityQueryImplementation> removed = new HashSet<>(oldVqiInstances);
+            removed.removeAll(newVqiInstances);
+            for (IndexabilityQueryImplementation vqi : removed) {
+                vqi.removeChangeListener(vqiListener);
+            }
+        }
+
+        Set<IndexabilityQueryImplementation> added = new HashSet<>(newVqiInstances);
+        if (oldVqiInstances != null) {
+            added.removeAll(oldVqiInstances);
+        }
+        for (IndexabilityQueryImplementation vqi : added) {
+            vqi.addChangeListener(vqiListener);
+        }
+
+        cachedIqiInstances = newVqiInstances;
+    }
+
+    private class ResultListener implements LookupListener {
+        @Override
+        public void resultChanged(LookupEvent ev) {
+            setupChangeListeners(cachedIqiInstances, new ArrayList<>(iqiResult.allInstances()));
+            fireChange(new ChangeEvent(this));
+        }
+    }
+
+    private class IqiChangedListener implements ChangeListener {
+        @Override
+        public void stateChanged(ChangeEvent e) {
+            fireChange(e);
+        }
+    }
+}