Permalink
Browse files

Stores occurrences of methods in Lucene

The plugin now keeps an Index (using Lucene) of all the places
methods are mentioned in the code-base. A separate Lucene index
is created on disk for each project.

The initial indices will be created upon start-up if no index
exists already. It will index the entire workspace and traverse
the parse-tree of all Scala sources.

For each project in the workspace we create a background job that
continuously update the index and re-index files that have changed.

There are two separate tests: OccurrenceCollectorTest contains test
that make sure we record the correct information from the parse trees
and IndexTest that makes sure the occurrence information can be stored
and retrieved from Lucene.

Fix #1001602
  • Loading branch information...
1 parent ed12624 commit bf07aabb4b277c92fb784cd850f42895be4608af @mads-hartmann mads-hartmann committed Feb 15, 2013
Showing with 989 additions and 12 deletions.
  1. +2 −1 org.scala.tools.eclipse.search.tests/META-INF/MANIFEST.MF
  2. +31 −0 org.scala.tools.eclipse.search.tests/pom.xml
  3. +57 −0 org.scala.tools.eclipse.search.tests/src/org/scala/tools/eclipse/search/IndexTest.scala
  4. +6 −8 org.scala.tools.eclipse.search.tests/src/org/scala/tools/eclipse/search/LuceneIntegrationTest.scala
  5. +72 −0 ...scala.tools.eclipse.search.tests/src/org/scala/tools/eclipse/search/OccurrenceCollectorTest.scala
  6. +10 −0 org.scala.tools.eclipse.search.tests/src/org/scala/tools/eclipse/search/TestUtil.scala
  7. +4 −1 org.scala.tools.eclipse.search.tests/src/org/scala/tools/eclipse/search/TestsSuite.scala
  8. +68 −0 org.scala.tools.eclipse.search.tests/src/org/scala/tools/eclipse/search/UsingTest.scala
  9. +14 −0 ...ala.tools.eclipse.search.tests/test-workspace/aProject/src/org/example/InvocationAsArgument.scala
  10. +17 −0 org.scala.tools.eclipse.search.tests/test-workspace/aProject/src/org/example/MethodChaining.scala
  11. +8 −2 org.scala.tools.eclipse.search.tests/test-workspace/aProject/src/org/example/ScalaClass.scala
  12. +15 −0 org.scala.tools.eclipse.search.tests/test-workspace/aProject/src/org/example/SelectInApply.scala
  13. +7 −0 org.scala.tools.eclipse.search.tests/test-workspace/lucene_index_test_project/.classpath
  14. +18 −0 org.scala.tools.eclipse.search.tests/test-workspace/lucene_index_test_project/.project
  15. +15 −0 ...ls.eclipse.search.tests/test-workspace/lucene_index_test_project/src/org/example/ScalaClass.scala
  16. +1 −0 org.scala.tools.eclipse.search/META-INF/MANIFEST.MF
  17. +5 −0 org.scala.tools.eclipse.search/plugin.xml
  18. +15 −0 org.scala.tools.eclipse.search/src/org/scala/tools/eclipse/search/EclipseImplicits.scala
  19. +74 −0 org.scala.tools.eclipse.search/src/org/scala/tools/eclipse/search/FileChangeObserver.scala
  20. +12 −0 org.scala.tools.eclipse.search/src/org/scala/tools/eclipse/search/Observing.scala
  21. +3 −0 org.scala.tools.eclipse.search/src/org/scala/tools/eclipse/search/ScalaSearchException.scala
  22. +70 −0 org.scala.tools.eclipse.search/src/org/scala/tools/eclipse/search/SearchPlugin.scala
  23. +17 −0 org.scala.tools.eclipse.search/src/org/scala/tools/eclipse/search/Startup.scala
  24. +19 −0 org.scala.tools.eclipse.search/src/org/scala/tools/eclipse/search/Util.scala
  25. +158 −0 org.scala.tools.eclipse.search/src/org/scala/tools/eclipse/search/indexing/Index.scala
  26. +42 −0 org.scala.tools.eclipse.search/src/org/scala/tools/eclipse/search/indexing/Occurrence.scala
  27. +63 −0 org.scala.tools.eclipse.search/src/org/scala/tools/eclipse/search/indexing/OccurrenceCollector.scala
  28. +47 −0 org.scala.tools.eclipse.search/src/org/scala/tools/eclipse/search/indexing/SourceIndexer.scala
  29. +108 −0 org.scala.tools.eclipse.search/src/org/scala/tools/eclipse/search/jobs/ProjectIndexJob.scala
  30. +11 −0 org.scala.tools.eclipse.search/src/org/scala/tools/eclipse/search/using.scala
@@ -12,4 +12,5 @@ Require-Bundle: org.scala-ide.scala.library,
org.scala-ide.sdt.core
Import-Package: scala.tools.eclipse.testsetup,
org.aspectj.weaver.loadtime.definition
-Bundle-ClassPath: .
+Bundle-ClassPath: .,
+ target/lib/mockito-all-1.9.0.jar
@@ -8,6 +8,14 @@
<artifactId>org.scala.tools.eclipse.search.tests</artifactId>
<packaging>eclipse-test-plugin</packaging>
+ <dependencies>
+ <dependency>
+ <groupId>org.mockito</groupId>
+ <artifactId>mockito-all</artifactId>
+ <version>1.9.0</version>
+ </dependency>
+ </dependencies>
+
<build>
<sourceDirectory>src</sourceDirectory>
<plugins>
@@ -19,6 +27,29 @@
<testClass>org.scala.tools.eclipse.search.TestsSuite</testClass>
</configuration>
</plugin>
+ <plugin>
+ <!-- copy the mockito jar, so it can be used in eclipse -->
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-dependency-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>copy</id>
+ <phase>initialize</phase>
+ <goals>
+ <goal>copy</goal>
+ </goals>
+ <configuration>
+ <artifactItems>
+ <artifactItem>
+ <groupId>org.mockito</groupId>
+ <artifactId>mockito-all</artifactId>
+ <outputDirectory>${project.build.directory}/lib</outputDirectory>
+ </artifactItem>
+ </artifactItems>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
</plugins>
</build>
@@ -0,0 +1,57 @@
+package org.scala.tools.eclipse.search
+
+import java.io.File
+
+import scala.tools.eclipse.testsetup.TestProjectSetup
+
+import org.junit.Assert.assertEquals
+import org.junit.Test
+import org.scala.tools.eclipse.search.indexing.Declaration
+import org.scala.tools.eclipse.search.indexing.Index
+import org.scala.tools.eclipse.search.indexing.Occurrence
+import org.scala.tools.eclipse.search.indexing.Reference
+import org.scala.tools.eclipse.search.indexing.SourceIndexer
+
+import LuceneIndexTest.mkPath
+import LuceneIndexTest.scalaCompilationUnit
+
+/**
+ * Tests that the correct things are stored in the LuceneIndex. We shouldn't
+ * require that many tests for this as it is the responsibility of the OccurrenceCollector
+ * to record the correct information.
+ */
+class IndexTest {
+
+ import LuceneIndexTest._
+
+ @Test def storeAndRetrieve() {
+ val index = new Index(INDEX_DIR)
+ val indexer = new SourceIndexer(index)
+ val source = scalaCompilationUnit(mkPath("org","example","ScalaClass.scala"))
+ indexer.indexScalaFile(source)
+
+ val expected = List(
+ Occurrence("method", source, 46, Declaration),
+ Occurrence("methodOne", source, 78, Reference),
+ Occurrence("methodTwo", source, 101, Reference),
+ Occurrence("methodThree", source, 119, Reference),
+ Occurrence("methodOne", source, 172, Declaration),
+ Occurrence("methodTwo", source, 197, Declaration),
+ Occurrence("methodThree", source, 228, Declaration)
+ )
+
+ val interestingNames = List("method", "methodOne", "methodTwo", "methodThree")
+
+ val results = index.occurrencesInFile(source.file.file, source.getUnderlyingResource().getProject()).filter( x => interestingNames.contains(x.word))
+
+ assertEquals("Should be able to store and retrieve occurrences", expected, results)
+ }
+
+}
+
+object LuceneIndexTest extends TestProjectSetup("lucene_index_test_project", bundleName= "org.scala.tools.eclipse.search.tests")
+ with TestUtil {
+
+ val INDEX_DIR = new File(mkPath("target","lucene-index-test"))
+
+}
@@ -1,7 +1,6 @@
package org.scala.tools.eclipse.search
import java.io.File
-
import org.apache.lucene.analysis.core.SimpleAnalyzer
import org.apache.lucene.document.Document
import org.apache.lucene.document.Field
@@ -12,13 +11,7 @@ import org.apache.lucene.store.FSDirectory
import org.apache.lucene.util.Version
import org.junit.Assert._
import org.junit.Test
-
-object LuceneIntegrationTest {
- val INDEX_DIR = new File(path("target","lucene-test-index"))
-
- private def path(strings: String*) =
- strings.mkString(File.separator)
-}
+import scala.tools.eclipse.testsetup.TestProjectSetup
class LuceneIntegrationTest {
@@ -38,4 +31,9 @@ class LuceneIntegrationTest {
assertEquals("Should be able to store and read a document", doc.get("test"), d.get("test"))
}
+}
+
+object LuceneIntegrationTest extends TestUtil {
+ val INDEX_DIR = new File(mkPath("target","lucene-integration-test-index"))
+
}
@@ -0,0 +1,72 @@
+package org.scala.tools.eclipse.search
+
+import org.junit.Test
+import org.junit.Assert._
+import scala.tools.eclipse.testsetup.TestProjectSetup
+import scala.tools.eclipse.javaelements.ScalaSourceFile
+import org.scala.tools.eclipse.search.indexing.OccurrenceCollector
+import org.scala.tools.eclipse.search.indexing.Occurrence
+import java.io.File
+
+object OccurrenceCollectorTest extends TestProjectSetup("aProject", bundleName= "org.scala.tools.eclipse.search.tests")
+ with TestUtil {
+
+ def occurrenceFor(word: String, occurrences: Seq[Occurrence]) = {
+ occurrences.filter( _.word == word )
+ }
+
+ def doWithOccurrencesInUnit(path: String*)(f: Seq[Occurrence] => Unit): Unit = {
+ val unit = scalaCompilationUnit(mkPath(path:_*))
+ val occurrences = OccurrenceCollector.findOccurrences(unit)
+ occurrences.fold(
+ error => fail(error),
+ occs => f(occs))
+ }
+
+}
+
+/**
+ * This tests the occurrence collector exclusively, this doesn't depend on any for of index.
+ */
+class OccurrenceCollectorTest {
+
+ import OccurrenceCollectorTest._
+
+ @Test
+ def numberOfMethods() {
+ doWithOccurrencesInUnit("org","example","ScalaClass.scala") { occurrences =>
+ val methodOne = occurrenceFor("methodOne", occurrences)
+ val methodTwo = occurrenceFor("methodTwo", occurrences)
+ val methodThree = occurrenceFor("methodThree", occurrences)
+ assertEquals("Should be two occurrences of methodOne %s".format(methodOne), 2, methodOne.size)
+ assertEquals("Should be two occurrences of methodTwo %s".format(methodTwo), 2, methodTwo.size)
+ assertEquals("Should be two occurrences of methodThree %s".format(methodThree), 2, methodThree.size)
+ }
+ }
+
+ @Test
+ def methodChaining() {
+ doWithOccurrencesInUnit("org","example","MethodChaining.scala") { occurrences =>
+ val foo = occurrenceFor("foo", occurrences)
+ val bar = occurrenceFor("bar", occurrences)
+ assertEquals("Should be two occurrences of foo %s".format(foo), 2, foo.size)
+ assertEquals("Should be two occurrences of bar %s".format(bar), 2, bar.size)
+ }
+ }
+
+ @Test def invocationAsArgument() {
+ doWithOccurrencesInUnit("org","example","InvocationAsArgument.scala") { occurrences =>
+ val m = occurrenceFor("methodTwo", occurrences)
+ assertEquals("Should be 3 occurrences of methodTwo %s".format(m), 3, m.size)
+ }
+ }
+
+ @Test def selectInApply() {
+ doWithOccurrencesInUnit("org","example","SelectInApply.scala") { occurrences =>
+ val x = occurrenceFor("x", occurrences)
+ assertEquals("Should be 2 occurrences of x %s".format(x), 2, x.size)
+ }
+ }
+
+
+}
@@ -0,0 +1,10 @@
+package org.scala.tools.eclipse.search
+
+import java.io.File
+
+trait TestUtil {
+
+ def mkPath(xs: String*): String = {
+ xs.mkString(File.separator)
+ }
+}
@@ -6,6 +6,9 @@ import org.junit.runners.Suite
@RunWith(classOf[Suite])
@Suite.SuiteClasses(Array(
- classOf[LuceneIntegrationTest]
+ classOf[OccurrenceCollectorTest],
+ classOf[LuceneIntegrationTest],
+ classOf[IndexTest],
+ classOf[UsingTest]
))
class TestsSuite {}
@@ -0,0 +1,68 @@
+package org.scala.tools.eclipse.search
+
+import org.junit.Test
+import org.mockito.Mockito._
+import java.io.Closeable
+import org.junit.Assert._
+import scala.util.control.{ Exception => Ex }
+import java.io.IOException
+import scala.util.Try
+
+class UsingTest {
+
+ private class ExpectedException(msg: String) extends Exception(msg)
+
+ def anonymousResource: Closeable = mock(classOf[Closeable])
+
+ def exceptionThrowingResource: Closeable = {
+ val resource = mock(classOf[Closeable])
+ when(resource.close()).thenThrow(new IOException())
+ resource
+ }
+
+ // Make sure the exceptions are handled as expected
+
+ @Test def shouldSwallowExceptionsOnClose() {
+ using(exceptionThrowingResource) { _ => () }
+ }
+
+ @Test(expected = classOf[ExpectedException])
+ def shouldPropagateExceptionsOfBodyByDefault() {
+ using(anonymousResource) { _ =>
+ throw new ExpectedException("This should propagate")
+ }
+ }
+
+ @Test def canCatchExceptions() {
+ using(anonymousResource, handlers = Ex.ignoring(classOf[ExpectedException])) { _ =>
+ throw new ExpectedException("This shouldn't propagate")
+ }
+ }
+
+ // Make sure we always close the resource
+
+ @Test def shouldCloseOnSuccess() {
+ val resource = anonymousResource
+ using(resource) { _ => () }
+ verify(resource).close()
+ }
+
+ @Test def closesWhenCatches() {
+ val resource = anonymousResource
+ using(resource, handlers = Ex.ignoring(classOf[ExpectedException])) { _ =>
+ throw new ExpectedException("This shouldn't propagate")
+ }
+ verify(resource).close()
+ }
+
+ @Test def closesWhenPropagates() {
+ val resource = anonymousResource
+ Try {
+ using(resource) { _ =>
+ throw new ExpectedException("This should propagate")
+ }
+ }
+ verify(resource).close()
+ }
+
+}
@@ -0,0 +1,14 @@
+package org.example
+
+class ScalaClass {
+ def method: String = {
+ val s2 = methodTwo(methodOne)
+ methodThree(s1)(methodTwo(s2))
+ }
+}
+
+object ScalaClass {
+ def methodOne = "Test"
+ def methodTwo(s: String) = s
+ def methodThree(s: String)(s2: String) = s + s2
+}
@@ -0,0 +1,17 @@
+package org.example
+
+class MethodChaining {
+
+ def foo() = this
+ def bar() = this
+
+}
+
+object MethodChaining {
+
+ def method() = {
+ val x new MethodChaining()
+ x.foo().bar()
+ }
+
+}
@@ -1,9 +1,15 @@
package org.example
class ScalaClass {
-
+ def method: String = {
+ val s1 = methodOne
+ val s2 = methodTwo(s1)
+ methodThree(s1)(s2)
+ }
}
object ScalaClass {
-
+ def methodOne = "Test"
+ def methodTwo(s: String) = s
+ def methodThree(s: String)(s2: String) = s + s2
}
@@ -0,0 +1,15 @@
+package org.example
+
+class Foo {
+ def bar(str: String) = str
+}
+
+class Bar {
+ def x = "test"
+}
+
+object ScalaClass {
+ val foo = new Foo
+ val bar = new Bar
+ foo.bar(bar.x)
+}
@@ -0,0 +1,7 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<classpath>
+ <classpathentry kind="src" path="src"/>
+ <classpathentry kind="con" path="org.scala-ide.sdt.launching.SCALA_CONTAINER"/>
+ <classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-1.6"/>
+ <classpathentry kind="output" path="bin"/>
+</classpath>
Oops, something went wrong.

0 comments on commit bf07aab

Please sign in to comment.