Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse files

Merge pull request #19 from mads379/indexing-methods-using-lucene-API…

…-1001602

Stores occurrences of methods in Lucene
  • Loading branch information...
commit e2d1a105f802af1a0b747c87bbf45e0ded0bee95 2 parents ed12624 + bf07aab
@dotta dotta authored
Showing with 989 additions and 12 deletions.
  1. +2 −1  org.scala.tools.eclipse.search.tests/META-INF/MANIFEST.MF
  2. +31 −0 org.scala.tools.eclipse.search.tests/pom.xml
  3. +57 −0 org.scala.tools.eclipse.search.tests/src/org/scala/tools/eclipse/search/IndexTest.scala
  4. +6 −8 org.scala.tools.eclipse.search.tests/src/org/scala/tools/eclipse/search/LuceneIntegrationTest.scala
  5. +72 −0 org.scala.tools.eclipse.search.tests/src/org/scala/tools/eclipse/search/OccurrenceCollectorTest.scala
  6. +10 −0 org.scala.tools.eclipse.search.tests/src/org/scala/tools/eclipse/search/TestUtil.scala
  7. +4 −1 org.scala.tools.eclipse.search.tests/src/org/scala/tools/eclipse/search/TestsSuite.scala
  8. +68 −0 org.scala.tools.eclipse.search.tests/src/org/scala/tools/eclipse/search/UsingTest.scala
  9. +14 −0 org.scala.tools.eclipse.search.tests/test-workspace/aProject/src/org/example/InvocationAsArgument.scala
  10. +17 −0 org.scala.tools.eclipse.search.tests/test-workspace/aProject/src/org/example/MethodChaining.scala
  11. +8 −2 org.scala.tools.eclipse.search.tests/test-workspace/aProject/src/org/example/ScalaClass.scala
  12. +15 −0 org.scala.tools.eclipse.search.tests/test-workspace/aProject/src/org/example/SelectInApply.scala
  13. +7 −0 org.scala.tools.eclipse.search.tests/test-workspace/lucene_index_test_project/.classpath
  14. +18 −0 org.scala.tools.eclipse.search.tests/test-workspace/lucene_index_test_project/.project
  15. +15 −0 org.scala.tools.eclipse.search.tests/test-workspace/lucene_index_test_project/src/org/example/ScalaClass.scala
  16. +1 −0  org.scala.tools.eclipse.search/META-INF/MANIFEST.MF
  17. +5 −0 org.scala.tools.eclipse.search/plugin.xml
  18. +15 −0 org.scala.tools.eclipse.search/src/org/scala/tools/eclipse/search/EclipseImplicits.scala
  19. +74 −0 org.scala.tools.eclipse.search/src/org/scala/tools/eclipse/search/FileChangeObserver.scala
  20. +12 −0 org.scala.tools.eclipse.search/src/org/scala/tools/eclipse/search/Observing.scala
  21. +3 −0  org.scala.tools.eclipse.search/src/org/scala/tools/eclipse/search/ScalaSearchException.scala
  22. +70 −0 org.scala.tools.eclipse.search/src/org/scala/tools/eclipse/search/SearchPlugin.scala
  23. +17 −0 org.scala.tools.eclipse.search/src/org/scala/tools/eclipse/search/Startup.scala
  24. +19 −0 org.scala.tools.eclipse.search/src/org/scala/tools/eclipse/search/Util.scala
  25. +158 −0 org.scala.tools.eclipse.search/src/org/scala/tools/eclipse/search/indexing/Index.scala
  26. +42 −0 org.scala.tools.eclipse.search/src/org/scala/tools/eclipse/search/indexing/Occurrence.scala
  27. +63 −0 org.scala.tools.eclipse.search/src/org/scala/tools/eclipse/search/indexing/OccurrenceCollector.scala
  28. +47 −0 org.scala.tools.eclipse.search/src/org/scala/tools/eclipse/search/indexing/SourceIndexer.scala
  29. +108 −0 org.scala.tools.eclipse.search/src/org/scala/tools/eclipse/search/jobs/ProjectIndexJob.scala
  30. +11 −0 org.scala.tools.eclipse.search/src/org/scala/tools/eclipse/search/using.scala
View
3  org.scala.tools.eclipse.search.tests/META-INF/MANIFEST.MF
@@ -12,4 +12,5 @@ Require-Bundle: org.scala-ide.scala.library,
org.scala-ide.sdt.core
Import-Package: scala.tools.eclipse.testsetup,
org.aspectj.weaver.loadtime.definition
-Bundle-ClassPath: .
+Bundle-ClassPath: .,
+ target/lib/mockito-all-1.9.0.jar
View
31 org.scala.tools.eclipse.search.tests/pom.xml
@@ -8,6 +8,14 @@
<artifactId>org.scala.tools.eclipse.search.tests</artifactId>
<packaging>eclipse-test-plugin</packaging>
+ <dependencies>
+ <dependency>
+ <groupId>org.mockito</groupId>
+ <artifactId>mockito-all</artifactId>
+ <version>1.9.0</version>
+ </dependency>
+ </dependencies>
+
<build>
<sourceDirectory>src</sourceDirectory>
<plugins>
@@ -19,6 +27,29 @@
<testClass>org.scala.tools.eclipse.search.TestsSuite</testClass>
</configuration>
</plugin>
+ <plugin>
+ <!-- copy the mockito jar, so it can be used in eclipse -->
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-dependency-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>copy</id>
+ <phase>initialize</phase>
+ <goals>
+ <goal>copy</goal>
+ </goals>
+ <configuration>
+ <artifactItems>
+ <artifactItem>
+ <groupId>org.mockito</groupId>
+ <artifactId>mockito-all</artifactId>
+ <outputDirectory>${project.build.directory}/lib</outputDirectory>
+ </artifactItem>
+ </artifactItems>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
</plugins>
</build>
View
57 org.scala.tools.eclipse.search.tests/src/org/scala/tools/eclipse/search/IndexTest.scala
@@ -0,0 +1,57 @@
+package org.scala.tools.eclipse.search
+
+import java.io.File
+
+import scala.tools.eclipse.testsetup.TestProjectSetup
+
+import org.junit.Assert.assertEquals
+import org.junit.Test
+import org.scala.tools.eclipse.search.indexing.Declaration
+import org.scala.tools.eclipse.search.indexing.Index
+import org.scala.tools.eclipse.search.indexing.Occurrence
+import org.scala.tools.eclipse.search.indexing.Reference
+import org.scala.tools.eclipse.search.indexing.SourceIndexer
+
+import LuceneIndexTest.mkPath
+import LuceneIndexTest.scalaCompilationUnit
+
+/**
+ * Tests that the correct things are stored in the LuceneIndex. We shouldn't
+ * require that many tests for this as it is the responsibility of the OccurrenceCollector
+ * to record the correct information.
+ */
+class IndexTest {
+
+ import LuceneIndexTest._
+
+ @Test def storeAndRetrieve() {
+ val index = new Index(INDEX_DIR)
+ val indexer = new SourceIndexer(index)
+ val source = scalaCompilationUnit(mkPath("org","example","ScalaClass.scala"))
+ indexer.indexScalaFile(source)
+
+ val expected = List(
+ Occurrence("method", source, 46, Declaration),
+ Occurrence("methodOne", source, 78, Reference),
+ Occurrence("methodTwo", source, 101, Reference),
+ Occurrence("methodThree", source, 119, Reference),
+ Occurrence("methodOne", source, 172, Declaration),
+ Occurrence("methodTwo", source, 197, Declaration),
+ Occurrence("methodThree", source, 228, Declaration)
+ )
+
+ val interestingNames = List("method", "methodOne", "methodTwo", "methodThree")
+
+ val results = index.occurrencesInFile(source.file.file, source.getUnderlyingResource().getProject()).filter( x => interestingNames.contains(x.word))
+
+ assertEquals("Should be able to store and retrieve occurrences", expected, results)
+ }
+
+}
+
+object LuceneIndexTest extends TestProjectSetup("lucene_index_test_project", bundleName= "org.scala.tools.eclipse.search.tests")
+ with TestUtil {
+
+ val INDEX_DIR = new File(mkPath("target","lucene-index-test"))
+
+}
View
14 org.scala.tools.eclipse.search.tests/src/org/scala/tools/eclipse/search/LuceneIntegrationTest.scala
@@ -1,7 +1,6 @@
package org.scala.tools.eclipse.search
import java.io.File
-
import org.apache.lucene.analysis.core.SimpleAnalyzer
import org.apache.lucene.document.Document
import org.apache.lucene.document.Field
@@ -12,13 +11,7 @@ import org.apache.lucene.store.FSDirectory
import org.apache.lucene.util.Version
import org.junit.Assert._
import org.junit.Test
-
-object LuceneIntegrationTest {
- val INDEX_DIR = new File(path("target","lucene-test-index"))
-
- private def path(strings: String*) =
- strings.mkString(File.separator)
-}
+import scala.tools.eclipse.testsetup.TestProjectSetup
class LuceneIntegrationTest {
@@ -38,4 +31,9 @@ class LuceneIntegrationTest {
assertEquals("Should be able to store and read a document", doc.get("test"), d.get("test"))
}
+}
+
+object LuceneIntegrationTest extends TestUtil {
+ val INDEX_DIR = new File(mkPath("target","lucene-integration-test-index"))
+
}
View
72 org.scala.tools.eclipse.search.tests/src/org/scala/tools/eclipse/search/OccurrenceCollectorTest.scala
@@ -0,0 +1,72 @@
+package org.scala.tools.eclipse.search
+
+import org.junit.Test
+import org.junit.Assert._
+import scala.tools.eclipse.testsetup.TestProjectSetup
+import scala.tools.eclipse.javaelements.ScalaSourceFile
+import org.scala.tools.eclipse.search.indexing.OccurrenceCollector
+import org.scala.tools.eclipse.search.indexing.Occurrence
+import java.io.File
+
+object OccurrenceCollectorTest extends TestProjectSetup("aProject", bundleName= "org.scala.tools.eclipse.search.tests")
+ with TestUtil {
+
+ def occurrenceFor(word: String, occurrences: Seq[Occurrence]) = {
+ occurrences.filter( _.word == word )
+ }
+
+ def doWithOccurrencesInUnit(path: String*)(f: Seq[Occurrence] => Unit): Unit = {
+ val unit = scalaCompilationUnit(mkPath(path:_*))
+ val occurrences = OccurrenceCollector.findOccurrences(unit)
+ occurrences.fold(
+ error => fail(error),
+ occs => f(occs))
+ }
+
+}
+
+/**
+ * This tests the occurrence collector exclusively, this doesn't depend on any for of index.
+ */
+class OccurrenceCollectorTest {
+
+ import OccurrenceCollectorTest._
+
+ @Test
+ def numberOfMethods() {
+ doWithOccurrencesInUnit("org","example","ScalaClass.scala") { occurrences =>
+ val methodOne = occurrenceFor("methodOne", occurrences)
+ val methodTwo = occurrenceFor("methodTwo", occurrences)
+ val methodThree = occurrenceFor("methodThree", occurrences)
+ assertEquals("Should be two occurrences of methodOne %s".format(methodOne), 2, methodOne.size)
+ assertEquals("Should be two occurrences of methodTwo %s".format(methodTwo), 2, methodTwo.size)
+ assertEquals("Should be two occurrences of methodThree %s".format(methodThree), 2, methodThree.size)
+ }
+ }
+
+ @Test
+ def methodChaining() {
+ doWithOccurrencesInUnit("org","example","MethodChaining.scala") { occurrences =>
+ val foo = occurrenceFor("foo", occurrences)
+ val bar = occurrenceFor("bar", occurrences)
+ assertEquals("Should be two occurrences of foo %s".format(foo), 2, foo.size)
+ assertEquals("Should be two occurrences of bar %s".format(bar), 2, bar.size)
+ }
+ }
+
+ @Test def invocationAsArgument() {
+ doWithOccurrencesInUnit("org","example","InvocationAsArgument.scala") { occurrences =>
+ val m = occurrenceFor("methodTwo", occurrences)
+ assertEquals("Should be 3 occurrences of methodTwo %s".format(m), 3, m.size)
+ }
+ }
+
+ @Test def selectInApply() {
+ doWithOccurrencesInUnit("org","example","SelectInApply.scala") { occurrences =>
+ val x = occurrenceFor("x", occurrences)
+ assertEquals("Should be 2 occurrences of x %s".format(x), 2, x.size)
+ }
+ }
+
+
+}
View
10 org.scala.tools.eclipse.search.tests/src/org/scala/tools/eclipse/search/TestUtil.scala
@@ -0,0 +1,10 @@
+package org.scala.tools.eclipse.search
+
+import java.io.File
+
+trait TestUtil {
+
+ def mkPath(xs: String*): String = {
+ xs.mkString(File.separator)
+ }
+}
View
5 org.scala.tools.eclipse.search.tests/src/org/scala/tools/eclipse/search/TestsSuite.scala
@@ -6,6 +6,9 @@ import org.junit.runners.Suite
@RunWith(classOf[Suite])
@Suite.SuiteClasses(Array(
- classOf[LuceneIntegrationTest]
+ classOf[OccurrenceCollectorTest],
+ classOf[LuceneIntegrationTest],
+ classOf[IndexTest],
+ classOf[UsingTest]
))
class TestsSuite {}
View
68 org.scala.tools.eclipse.search.tests/src/org/scala/tools/eclipse/search/UsingTest.scala
@@ -0,0 +1,68 @@
+package org.scala.tools.eclipse.search
+
+import org.junit.Test
+import org.mockito.Mockito._
+import java.io.Closeable
+import org.junit.Assert._
+import scala.util.control.{ Exception => Ex }
+import java.io.IOException
+import scala.util.Try
+
+class UsingTest {
+
+ private class ExpectedException(msg: String) extends Exception(msg)
+
+ def anonymousResource: Closeable = mock(classOf[Closeable])
+
+ def exceptionThrowingResource: Closeable = {
+ val resource = mock(classOf[Closeable])
+ when(resource.close()).thenThrow(new IOException())
+ resource
+ }
+
+ // Make sure the exceptions are handled as expected
+
+ @Test def shouldSwallowExceptionsOnClose() {
+ using(exceptionThrowingResource) { _ => () }
+ }
+
+ @Test(expected = classOf[ExpectedException])
+ def shouldPropagateExceptionsOfBodyByDefault() {
+ using(anonymousResource) { _ =>
+ throw new ExpectedException("This should propagate")
+ }
+ }
+
+ @Test def canCatchExceptions() {
+ using(anonymousResource, handlers = Ex.ignoring(classOf[ExpectedException])) { _ =>
+ throw new ExpectedException("This shouldn't propagate")
+ }
+ }
+
+ // Make sure we always close the resource
+
+ @Test def shouldCloseOnSuccess() {
+ val resource = anonymousResource
+ using(resource) { _ => () }
+ verify(resource).close()
+ }
+
+ @Test def closesWhenCatches() {
+ val resource = anonymousResource
+ using(resource, handlers = Ex.ignoring(classOf[ExpectedException])) { _ =>
+ throw new ExpectedException("This shouldn't propagate")
+ }
+ verify(resource).close()
+ }
+
+ @Test def closesWhenPropagates() {
+ val resource = anonymousResource
+ Try {
+ using(resource) { _ =>
+ throw new ExpectedException("This should propagate")
+ }
+ }
+ verify(resource).close()
+ }
+
+}
View
14 org.scala.tools.eclipse.search.tests/test-workspace/aProject/src/org/example/InvocationAsArgument.scala
@@ -0,0 +1,14 @@
+package org.example
+
+class ScalaClass {
+ def method: String = {
+ val s2 = methodTwo(methodOne)
+ methodThree(s1)(methodTwo(s2))
+ }
+}
+
+object ScalaClass {
+ def methodOne = "Test"
+ def methodTwo(s: String) = s
+ def methodThree(s: String)(s2: String) = s + s2
+}
View
17 org.scala.tools.eclipse.search.tests/test-workspace/aProject/src/org/example/MethodChaining.scala
@@ -0,0 +1,17 @@
+package org.example
+
+class MethodChaining {
+
+ def foo() = this
+ def bar() = this
+
+}
+
+object MethodChaining {
+
+ def method() = {
+ val x new MethodChaining()
+ x.foo().bar()
+ }
+
+}
View
10 org.scala.tools.eclipse.search.tests/test-workspace/aProject/src/org/example/ScalaClass.scala
@@ -1,9 +1,15 @@
package org.example
class ScalaClass {
-
+ def method: String = {
+ val s1 = methodOne
+ val s2 = methodTwo(s1)
+ methodThree(s1)(s2)
+ }
}
object ScalaClass {
-
+ def methodOne = "Test"
+ def methodTwo(s: String) = s
+ def methodThree(s: String)(s2: String) = s + s2
}
View
15 org.scala.tools.eclipse.search.tests/test-workspace/aProject/src/org/example/SelectInApply.scala
@@ -0,0 +1,15 @@
+package org.example
+
+class Foo {
+ def bar(str: String) = str
+}
+
+class Bar {
+ def x = "test"
+}
+
+object ScalaClass {
+ val foo = new Foo
+ val bar = new Bar
+ foo.bar(bar.x)
+}
View
7 org.scala.tools.eclipse.search.tests/test-workspace/lucene_index_test_project/.classpath
@@ -0,0 +1,7 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<classpath>
+ <classpathentry kind="src" path="src"/>
+ <classpathentry kind="con" path="org.scala-ide.sdt.launching.SCALA_CONTAINER"/>
+ <classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-1.6"/>
+ <classpathentry kind="output" path="bin"/>
+</classpath>
View
18 org.scala.tools.eclipse.search.tests/test-workspace/lucene_index_test_project/.project
@@ -0,0 +1,18 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<projectDescription>
+ <name>aProject</name>
+ <comment></comment>
+ <projects>
+ </projects>
+ <buildSpec>
+ <buildCommand>
+ <name>org.scala-ide.sdt.core.scalabuilder</name>
+ <arguments>
+ </arguments>
+ </buildCommand>
+ </buildSpec>
+ <natures>
+ <nature>org.scala-ide.sdt.core.scalanature</nature>
+ <nature>org.eclipse.jdt.core.javanature</nature>
+ </natures>
+</projectDescription>
View
15 ...ala.tools.eclipse.search.tests/test-workspace/lucene_index_test_project/src/org/example/ScalaClass.scala
@@ -0,0 +1,15 @@
+package org.example
+
+class ScalaClass {
+ def method: String = {
+ val s1 = methodOne
+ val s2 = methodTwo(s1)
+ methodThree(s1)(s2)
+ }
+}
+
+object ScalaClass {
+ def methodOne = "Test"
+ def methodTwo(s: String) = s
+ def methodThree(s: String)(s2: String) = s + s2
+}
View
1  org.scala.tools.eclipse.search/META-INF/MANIFEST.MF
@@ -4,6 +4,7 @@ Bundle-Name: Scala Search
Bundle-SymbolicName: org.scala.tools.eclipse.search;singleton:=true
Bundle-Version: 0.1.0.qualifier
Bundle-Vendor: Scala IDE
+Bundle-Activator: org.scala.tools.eclipse.search.SearchPlugin
Bundle-ActivationPolicy: lazy
Bundle-Localization: plugin
Require-Bundle:
View
5 org.scala.tools.eclipse.search/plugin.xml
@@ -26,4 +26,9 @@
</viewPage>
</extension>
+ <!-- Force Eclipse to start the plugin when it starts. -->
+ <extension point="org.eclipse.ui.startup">
+ <startup class="org.scala.tools.eclipse.search.Startup"></startup>
+ </extension>
+
</plugin>
View
15 org.scala.tools.eclipse.search/src/org/scala/tools/eclipse/search/EclipseImplicits.scala
@@ -0,0 +1,15 @@
+package org.scala.tools.eclipse.search
+
+import org.eclipse.core.resources.IResourceDelta
+import org.eclipse.core.resources.IResourceDeltaVisitor
+import org.eclipse.core.resources.IResourceDeltaVisitor
+
+object EclipseImplicits {
+
+ implicit def resourceDeltaVisitor(f: IResourceDelta => Boolean): IResourceDeltaVisitor = new IResourceDeltaVisitor {
+ def visit(delta: IResourceDelta): Boolean = {
+ f(delta)
+ }
+ }
+
+}
View
74 org.scala.tools.eclipse.search/src/org/scala/tools/eclipse/search/FileChangeObserver.scala
@@ -0,0 +1,74 @@
+package org.scala.tools.eclipse.search
+
+import scala.tools.eclipse.logging.HasLogger
+
+import org.eclipse.core.resources.IFile
+import org.eclipse.core.resources.IProject
+import org.eclipse.core.resources.IResource
+import org.eclipse.core.resources.IResourceChangeEvent
+import org.eclipse.core.resources.IResourceChangeListener
+import org.eclipse.core.resources.IResourceDelta
+import org.eclipse.core.resources.IResourceDeltaVisitor
+import org.eclipse.core.resources.ResourcesPlugin
+
+/**
+ * Convenient way to react to changes that happen to
+ * files in eclipse projects in the workspace.
+ */
+object FileChangeObserver {
+
+ /**
+ * Starts an IResourceChangeListener that reacts to file changes. You
+ * are responsible for stopping the listener when appropriate using the
+ * stop method on Observing.
+ */
+ def apply(p: IProject)
+ (onChanged: IFile => Unit = _ => (),
+ onRemoved: IFile => Unit = _ => (),
+ onAdded: IFile => Unit = _ => ()): Observing = {
+
+ val observer = new FileChangeObserver.ChangeListener(p, onChanged, onRemoved, onAdded)
+
+ ResourcesPlugin.getWorkspace().addResourceChangeListener(
+ observer,
+ IResourceChangeEvent.POST_CHANGE)
+
+ new Observing(observer)
+
+ }
+
+ // Hiding the Eclipse implementation here as we don't want the methods that
+ // Eclipse needs to leak into the interface of FileChangeObserver.
+ private class ChangeListener(
+ project: IProject,
+ onChanged: IFile => Unit = _ => (),
+ onRemoved: IFile => Unit = _ => (),
+ onAdded: IFile => Unit = _ => ()) extends IResourceChangeListener with HasLogger {
+
+ import EclipseImplicits._
+
+ override def resourceChanged(event: IResourceChangeEvent): Unit = {
+ for {
+ ev <- Option(event)
+ delta <- Option(ev.getDelta())
+ } {
+ delta.accept { (delta: IResourceDelta) =>
+ val resource = delta.getResource
+ if (resource.getType == IResource.FILE) {
+ if (resource.isInstanceOf[IFile]) {
+ val file = resource.asInstanceOf[IFile]
+ if (file.getProject().equals(project)) {
+ delta.getKind match {
+ case IResourceDelta.ADDED => onAdded(file)
+ case IResourceDelta.REMOVED => onRemoved(file)
+ case IResourceDelta.CHANGED => onChanged(file)
+ }
+ }
+ } else eclipseLog.error(s"Expected resource to be of type IFile, found ${resource.getClass}")
+ false
+ } else true
+ }
+ }
+ }
+ }
+}
View
12 org.scala.tools.eclipse.search/src/org/scala/tools/eclipse/search/Observing.scala
@@ -0,0 +1,12 @@
+package org.scala.tools.eclipse.search
+
+import org.eclipse.core.resources.IResourceChangeListener
+import org.eclipse.core.resources.ResourcesPlugin
+
+class Observing(listener: IResourceChangeListener) {
+
+ def stop: Unit = {
+ ResourcesPlugin.getWorkspace().removeResourceChangeListener(listener)
+ }
+
+}
View
3  org.scala.tools.eclipse.search/src/org/scala/tools/eclipse/search/ScalaSearchException.scala
@@ -0,0 +1,3 @@
+package org.scala.tools.eclipse.search
+
+class ScalaSearchException(msg: String) extends Exception(msg) {}
View
70 org.scala.tools.eclipse.search/src/org/scala/tools/eclipse/search/SearchPlugin.scala
@@ -0,0 +1,70 @@
+package org.scala.tools.eclipse.search
+
+import java.io.File
+import scala.tools.eclipse.logging.HasLogger
+import org.eclipse.core.resources.IFile
+import org.eclipse.core.resources.ResourcesPlugin
+import org.eclipse.core.runtime.jobs.Job
+import org.eclipse.ui.plugin.AbstractUIPlugin
+import org.osgi.framework.BundleContext
+import org.scala.tools.eclipse.search.indexing.Index
+import org.scala.tools.eclipse.search.indexing.SourceIndexer
+import org.scala.tools.eclipse.search.jobs.ProjectIndexJob
+import scala.collection.concurrent.TrieMap
+import scala.collection.concurrent.Map
+import org.eclipse.core.resources.IProject
+import scala.tools.eclipse.ScalaProject
+import scala.tools.eclipse.ScalaPlugin
+
+object SearchPlugin extends HasLogger {
+
+ final val PLUGIN_ID = "org.scala.tools.eclipse.search"
+
+ @volatile var plugin: SearchPlugin = _
+
+ /**
+ * Checks if the `file` is a type we know how to index.
+ */
+ def isIndexable(file: IFile): Boolean = {
+ // TODO: https://scala-ide-portfolio.assembla.com/spaces/scala-ide/tickets/1001602
+ Option(file).map( _.getFileExtension() == "scala").getOrElse(false)
+ }
+}
+
+class SearchPlugin extends AbstractUIPlugin with HasLogger {
+
+ private final val INDEX_DIR_NAME = "lucene-indices"
+
+ @volatile private var indexLocation: File = _
+ @volatile private var index: Index = _
+ @volatile private var sourceIndexer: SourceIndexer = _
+
+ override def start(context: BundleContext) {
+ SearchPlugin.plugin = this
+ super.start(context)
+
+ indexLocation = getStateLocation().append(INDEX_DIR_NAME).toFile()
+ index = new Index(indexLocation)
+ sourceIndexer = new SourceIndexer(index)
+
+ val root = ResourcesPlugin.getWorkspace().getRoot()
+ root.getProjects().map(Option.apply).flatten.foreach { proj =>
+ ScalaPlugin.plugin.asScalaProject(proj).foreach { sp =>
+ ProjectIndexJob(sourceIndexer, sp)
+ }
+ }
+ }
+
+ override def stop(context: BundleContext) {
+ super.stop(context)
+ SearchPlugin.plugin = null
+ indexLocation = null
+ index = null
+ sourceIndexer = null
+ }
+
+ def indexLocationForProject(project: IProject) = {
+ new File(indexLocation.getAbsolutePath() + File.separator + project.getName())
+ }
+
+}
View
17 org.scala.tools.eclipse.search/src/org/scala/tools/eclipse/search/Startup.scala
@@ -0,0 +1,17 @@
+package org.scala.tools.eclipse.search
+
+import scala.tools.eclipse.logging.HasLogger
+
+import org.eclipse.ui.IStartup
+
+/**
+ * This class forces the plugin to be started at eclipse startup, i.e. it will create
+ * an instance of SearchPlugin and invoke start(context) on it. If this class was not
+ * here the plugin would be initialized lazily and we wouldn't be able to start
+ * indexing and observing changes before some of the classes in this plugin were used.
+ */
+class Startup extends IStartup with HasLogger {
+
+ override def earlyStartup() {}
+
+}
View
19 org.scala.tools.eclipse.search/src/org/scala/tools/eclipse/search/Util.scala
@@ -0,0 +1,19 @@
+package org.scala.tools.eclipse.search
+
+import scala.tools.eclipse.logging.HasLogger
+import scala.tools.eclipse.javaelements.ScalaSourceFile
+import org.eclipse.core.resources.IFile
+import org.eclipse.core.resources.ResourcesPlugin
+import org.eclipse.core.runtime.Path
+
+/**
+ * Object that contains various utility methods
+ */
+object Util extends HasLogger {
+
+ def scalaSourceFileFromIFile(file: IFile): Option[ScalaSourceFile] = {
+ val path = file.getFullPath().toOSString()
+ ScalaSourceFile.createFromPath(path)
+ }
+
+}
View
158 org.scala.tools.eclipse.search/src/org/scala/tools/eclipse/search/indexing/Index.scala
@@ -0,0 +1,158 @@
+package org.scala.tools.eclipse.search.indexing
+
+import java.io.File
+import scala.Array.fallbackCanBuildFrom
+import scala.collection.JavaConverters.asJavaIterableConverter
+import scala.tools.eclipse.logging.HasLogger
+import org.apache.lucene.analysis.core.SimpleAnalyzer
+import org.apache.lucene.index.DirectoryReader
+import org.apache.lucene.index.IndexWriter
+import org.apache.lucene.index.IndexWriterConfig
+import org.apache.lucene.index.Term
+import org.apache.lucene.search.IndexSearcher
+import org.apache.lucene.search.TermQuery
+import org.apache.lucene.store.FSDirectory
+import org.apache.lucene.util.Version
+import org.apache.lucene.document.Document
+import org.apache.lucene.document.Field
+import org.scala.tools.eclipse.search.Util
+import org.eclipse.core.resources.IProject
+import org.scala.tools.eclipse.search.SearchPlugin
+import org.scala.tools.eclipse.search.ScalaSearchException
+import scala.util.Try
+import org.scala.tools.eclipse.search.ScalaSearchException
+import scala.util.Failure
+import scala.util.Success
+import org.eclipse.core.resources.ResourcesPlugin
+import org.eclipse.core.runtime.Path
+import org.scala.tools.eclipse.search.using
+
+/**
+ * A Lucene based index of all occurrences of Scala entities recorded
+ * in the workspace. See `toDocument` for more information about what
+ * is stored in the index.
+ *
+ * A separate Lucene index is stored on disk for each project in the
+ * Workspace.
+ *
+ * This class assumes that the resources passed to the different
+ * methods exist and, in the case of a IProject, it's open.
+ *
+ * This class is thread-safe.
+ */
+class Index(indicesRoot: File) extends HasLogger {
+
+ private def config = {
+ val analyzer = new SimpleAnalyzer(Version.LUCENE_41)
+ new IndexWriterConfig(Version.LUCENE_41, analyzer)
+ }
+
+ private val MAX_POTENTIAL_MATCHES = 100000
+
+ /**
+ * Add all `occurrences` to the index of the specific project.
+ */
+ def addOccurrences(occurrences: Seq[Occurrence], project: IProject): Unit = {
+ doWithWriter(project) { writer =>
+ val docs = occurrences.map( toDocument )
+ writer.addDocuments(docs.toIterable.asJava)
+ }
+ }
+
+ /**
+ * Removed all occurrences from the index that are recorded in the
+ * given file
+ */
+ def removeOccurrencesFromFile(file: File, project: IProject): Unit = {
+ doWithWriter(project) { writer =>
+ val query = new TermQuery(TermQueries.fileTerm(file))
+ writer.deleteDocuments(query)
+ }
+ }
+
+ /**
+ * Returns all occurrences recorded in the index for the given file. Mostly useful
+ * for testing purposes.
+ */
+ def occurrencesInFile(file: File, project: IProject): Seq[Occurrence] = {
+ withSearcher(project) { searcher =>
+ val query = new TermQuery(TermQueries.fileTerm(file))
+ val hits = searcher.search(query, MAX_POTENTIAL_MATCHES).scoreDocs
+ hits.map { hit =>
+ val doc = searcher.doc(hit.doc)
+ fromDocument(doc)
+ }
+ }
+ }
+
+ /**
+ * ARM method for writing to the index.
+ */
+ private def doWithWriter(project: IProject)(f: IndexWriter => Unit): Unit = {
+ val loc = SearchPlugin.plugin.indexLocationForProject(project)
+ using(FSDirectory.open(loc)) { dir =>
+ using(new IndexWriter(dir, config)) { writer =>
+ f(writer)
+ }
+ }
+ }
+
+ /**
+ * ARM method for searching the index.
+ */
+ private def withSearcher[A](project: IProject)(f: IndexSearcher => A): A = {
+ val loc = SearchPlugin.plugin.indexLocationForProject(project)
+ using(FSDirectory.open(loc)) { dir =>
+ using(DirectoryReader.open(dir)) { reader =>
+ val searcher = new IndexSearcher(reader)
+ f(searcher)
+ }
+ }
+ }
+
+ /**
+ * Collection of Term's that are used in multiple queries.
+ */
+ private object TermQueries {
+ def fileTerm(file: File) = {
+ new Term(LuceneFields.FILE, file.getAbsolutePath())
+ }
+ }
+
+ /**
+ * Create a Lucene document based on the information stored in the
+ * occurrence.
+ */
+ private def toDocument(o: Occurrence): Document = {
+ import LuceneFields._
+ val doc = new Document
+ doc.add(new Field(WORD, o.word, Field.Store.YES, Field.Index.NOT_ANALYZED))
+ doc.add(new Field(FILE, o.file.file.file.getAbsolutePath, Field.Store.YES, Field.Index.NOT_ANALYZED))
+ doc.add(new Field(OFFSET, o.offset.toString, Field.Store.YES, Field.Index.NOT_ANALYZED))
+ doc.add(new Field(OCCURRENCE_KIND, o.occurrenceKind.toString, Field.Store.YES, Field.Index.NOT_ANALYZED))
+ doc
+ }
+
+ /**
+ * Converts a Lucene Document into an Occurrence. Will throw exception if
+ * there are things that can't be converted to the expected type.
+ */
+ def fromDocument(doc: Document): Occurrence = {
+ import LuceneFields._
+ (for {
+ word <- Option(doc.get(WORD))
+ path <- Option(doc.get(FILE))
+ offset <- Option(doc.get(OFFSET))
+ occurrenceKind <- Option(doc.get(OCCURRENCE_KIND))
+ } yield {
+ val workspace = ResourcesPlugin.getWorkspace()
+ val location= Path.fromOSString(path)
+ val ifile = Option(workspace.getRoot().getFileForLocation(location)) getOrElse (throw new Exception("Couldn't get ifile from path " + path))
+ val file = Util.scalaSourceFileFromIFile(ifile).getOrElse (throw new Exception("Wasn't able to create ScalaSourceFile from path " + path))
+ Occurrence(word, file, Integer.parseInt(offset), OccurrenceKind.fromString(occurrenceKind))
+ }) getOrElse {
+ throw new ScalaSearchException("Wasn't able to convert document to occurrence")
+ }
+ }
+
+}
View
42 org.scala.tools.eclipse.search/src/org/scala/tools/eclipse/search/indexing/Occurrence.scala
@@ -0,0 +1,42 @@
+package org.scala.tools.eclipse.search.indexing
+
+import scala.tools.eclipse.javaelements.ScalaSourceFile
+
+/**
+ * Represents the various kinds of occurrences that we deal with
+ */
+sealed abstract class OccurrenceKind
+object OccurrenceKind {
+ def fromString(str: String): OccurrenceKind = str match {
+ case "Declaration" => Declaration
+ case "Reference" => Reference
+ case x => throw new Exception("Unknown occurrence kind " + x)
+ }
+}
+case object Declaration extends OccurrenceKind
+case object Reference extends OccurrenceKind
+
+object LuceneFields {
+ val WORD = "word"
+ val FILE = "file"
+ val OFFSET = "offset"
+ val OCCURRENCE_KIND = "occurrenceKind"
+}
+
+/**
+ * Represents an occurrence of a word that we're interested in when
+ * indexing the parse trees.
+ */
+case class Occurrence(
+ word: String,
+ file: ScalaSourceFile,
+ offset: Int, // char offset from beginning of file
+ occurrenceKind: OccurrenceKind) {
+
+ override def toString = "%s in %s at char %s %s".format(
+ word,
+ file.file.file.getAbsolutePath(),
+ offset.toString,
+ occurrenceKind.toString)
+
+}
View
63 org.scala.tools.eclipse.search/src/org/scala/tools/eclipse/search/indexing/OccurrenceCollector.scala
@@ -0,0 +1,63 @@
+package org.scala.tools.eclipse.search.indexing
+
+import scala.tools.eclipse.ScalaPresentationCompiler
+import scala.tools.eclipse.javaelements.ScalaSourceFile
+import scala.tools.eclipse.logging.HasLogger
+
+/**
+ * Used to parse and traverse the parse tree of a compilation unit finding
+ * all the occurrence of Scala entities we're interested in.
+ */
+object OccurrenceCollector extends HasLogger {
+
+ /**
+ * Find all occurrences of words we're find interesting in a compilation unit. It
+ * will return Left if it wasn't able to access the source file.
+ */
+ def findOccurrences(file: ScalaSourceFile): Either[String, Seq[Occurrence]] = {
+ lazy val err = Left("Couldn't get source file for %".format(file.file.path.toString()))
+ file.withSourceFile( (source, pcompiler) => {
+ pcompiler.withParseTree(source) { tree =>
+ // withParseTree is invariant so we need state the exact type so it doesn't
+ // infer the type Right.
+ Right(findOccurrences(pcompiler)(file, tree)): Either[String, Seq[Occurrence]]
+ }
+ })(err)
+ }
+
+ private def findOccurrences(pc: ScalaPresentationCompiler)
+ (file: ScalaSourceFile, tree: pc.Tree): Seq[Occurrence] = {
+ import pc._
+
+ val occurrences = new scala.collection.mutable.ListBuffer[Occurrence]()
+ val traverser = new Traverser {
+ override def traverse(t: Tree) {
+ t match {
+
+ case Ident(fun) if !isSynthetic(pc)(t, fun.toString) =>
+ occurrences += Occurrence(fun.toString, file, t.pos.point, Reference)
+
+ case Select(rest,name) if !isSynthetic(pc)(t, name.toString) =>
+ occurrences += Occurrence(name.toString, file, t.pos.point, Reference)
+ super.traverse(rest) // recurse in the case of chained selects: foo.baz.bar
+
+ // Method definitions
+ case DefDef(_, name, _, _, _, body) if !isSynthetic(pc)(t, name.toString) =>
+ occurrences += Occurrence(name.toString, file, t.pos.point, Declaration)
+ super.traverse(body)
+
+ case _ => super.traverse(t)
+ }
+ }
+ }
+ traverser.apply(tree)
+ occurrences.toList
+ }
+
+ private def isSynthetic(pc: ScalaPresentationCompiler)
+ (tree: pc.Tree, name: String): Boolean = {
+ val syntheticNames = Set("<init>")
+ tree.pos == pc.NoPosition || syntheticNames.contains(name)
+ }
+
+}
View
47 org.scala.tools.eclipse.search/src/org/scala/tools/eclipse/search/indexing/SourceIndexer.scala
@@ -0,0 +1,47 @@
+package org.scala.tools.eclipse.search.indexing
+
+import scala.tools.eclipse.javaelements.ScalaSourceFile
+import scala.tools.eclipse.logging.HasLogger
+import org.eclipse.core.resources.IWorkspaceRoot
+import scala.tools.eclipse.ScalaPlugin
+import scala.tools.eclipse.ScalaProject
+import org.scala.tools.eclipse.search.Util._
+import org.eclipse.core.resources.IFile
+import scala.tools.eclipse.util.Utils
+
+/**
+ * Indexes Scala sources and add all occurrences to the `index`.
+ */
+class SourceIndexer(index: Index) extends HasLogger {
+
+ /**
+ * Indexes all sources in `project`. This removes all previous occurrences recorded in
+ * that project.
+ */
+ def indexProject(proj: ScalaProject): Unit = {
+ Utils.debugTimed("Indexing project %s".format(proj)) {
+ proj.allSourceFiles.foreach { indexIFile }
+ }
+ }
+
+ /**
+ * Indexes the parse tree of an IFile if the IFile is pointing to a Scala source file.
+ * This removes all previous occurrences recorded in that file.
+ */
+ def indexIFile(file: IFile): Unit = {
+ scalaSourceFileFromIFile(file).foreach { cu => indexScalaFile(cu) }
+ }
+
+ /**
+ * Indexes the occurrences in a Scala file. This removes all previous occurrences
+ * recorded in that file.
+ */
+ def indexScalaFile(sf: ScalaSourceFile): Unit = {
+ logger.debug(s"Indexing document: ${sf.file.path}")
+ index.removeOccurrencesFromFile(sf.file.file, sf.getUnderlyingResource().getProject())
+ OccurrenceCollector.findOccurrences(sf).fold(
+ fail => logger.debug(fail),
+ occurrences => index.addOccurrences(occurrences, sf.getUnderlyingResource().getProject()))
+ }
+
+}
View
108 org.scala.tools.eclipse.search/src/org/scala/tools/eclipse/search/jobs/ProjectIndexJob.scala
@@ -0,0 +1,108 @@
+package org.scala.tools.eclipse.search.jobs
+
+import scala.tools.eclipse.logging.HasLogger
+import org.eclipse.core.resources.WorkspaceJob
+import org.eclipse.core.runtime.IProgressMonitor
+import org.eclipse.core.runtime.IStatus
+import org.scala.tools.eclipse.search.indexing.SourceIndexer
+import org.eclipse.core.resources.IWorkspaceRoot
+import org.eclipse.core.runtime.Status
+import org.scala.tools.eclipse.search.Util._
+import org.eclipse.core.resources.IProject
+import org.scala.tools.eclipse.search.FileChangeObserver
+import org.eclipse.core.resources.IResource
+import org.eclipse.core.resources.IFile
+import scala.tools.eclipse.ScalaProject
+import org.scala.tools.eclipse.search.SearchPlugin
+import scala.collection.concurrent.TrieMap
+import scala.collection.concurrent.Map
+import java.util.concurrent.BlockingQueue
+import java.util.concurrent.LinkedBlockingQueue
+import scala.tools.eclipse.ScalaPlugin
+import org.eclipse.core.runtime.jobs.Job
+import org.eclipse.core.resources.ResourcesPlugin
+import org.eclipse.core.runtime.Platform
+
+/**
+ * Background jobs that indexes the source files of a given project.
+ *
+ * The continuously schedules itself for execution, and as such it will always
+ * be running as long as Eclipse is running.
+ *
+ * It uses a FileChangeObserver to keep track of files in the project that
+ * needs to be re-indexed.
+ */
+class ProjectIndexJob private (
+ indexer: SourceIndexer,
+ project: ScalaProject,
+ interval: Long) extends WorkspaceJob("Project Indexing Job: " + project.underlying.getName) with HasLogger {
+
+ private trait FileEvent
+ private case object Changed extends FileEvent
+ private case object Removed extends FileEvent
+
+ // Potentially changed by several threads. This job and the FileChangeObserver
+ private val changedResources: BlockingQueue[(IFile, FileEvent)] = new LinkedBlockingQueue[(IFile, FileEvent)]
+
+ private val changed = (f: IFile) => {
+ if (SearchPlugin.isIndexable(f)) {
+ changedResources.put(f, Changed)
+ }
+ }
+
+ private val removed = (f: IFile) => changedResources.put(f, Removed)
+
+ private val observer = FileChangeObserver(project.underlying)(
+ onChanged = changed,
+ onAdded = changed,
+ onRemoved = removed
+ )
+
+ private def openAndExists: Boolean = {
+ project.underlying.exists() && project.underlying.isOpen
+ }
+
+ override def runInWorkspace(monitor: IProgressMonitor): IStatus = {
+
+ if (monitor.isCanceled()) {
+ return Status.CANCEL_STATUS
+ }
+
+ if (!SearchPlugin.plugin.indexLocationForProject(project.underlying).exists) {
+ indexer.indexProject(project)
+ }
+
+ while( !changedResources.isEmpty && !monitor.isCanceled() && openAndExists) {
+ val (file, changed) = changedResources.poll()
+ monitor.subTask(file.getName())
+ changed match {
+ case Changed => indexer.indexIFile(file)
+ case Removed => // TODO: Remove from the index. This class needs access to the index for that.
+ }
+ monitor.worked(1)
+ }
+ monitor.done()
+
+ if (openAndExists) {
+ schedule(interval)
+ } else {
+ observer.stop
+ }
+ Status.OK_STATUS
+ }
+}
+
+object ProjectIndexJob extends HasLogger {
+
+ def apply(indexer: SourceIndexer, sp: ScalaProject): ProjectIndexJob = {
+
+ logger.debug("Started ProjectIndexJob for " + sp.underlying.getName)
+
+ val job = new ProjectIndexJob(indexer, sp, interval = 5000)
+ job.setRule(ResourcesPlugin.getWorkspace().getRuleFactory().modifyRule(sp.underlying))
+ job.setPriority(Job.LONG)
+ job.schedule()
+ job
+ }
+
+}
View
11 org.scala.tools.eclipse.search/src/org/scala/tools/eclipse/search/using.scala
@@ -0,0 +1,11 @@
+package org.scala.tools.eclipse.search
+
+import scala.util.control.Exception._
+
+object using {
+ def apply[T <: java.io.Closeable, R](resource: T, handlers: Catch[R] = noCatch)(body: T => R): R = (
+ handlers
+ andFinally (ignoring(classOf[Any]) { resource.close() })
+ apply body (resource)
+ )
+}
Please sign in to comment.
Something went wrong with that request. Please try again.