Integration Test framework and an initial test for Document Code

CONTRIBUTING.md

@@ -201,6 +201,19 @@ to use if you are uncertain which method to use for debugging. Option 2
 is especially useful when you need to set a breakpoint very early in
 the Agent startup.
 
+### Using VS Code to debug the Agent
+
+The setup described here also works with VS Code as your debugger and/or
+launcher. Both configurations work with IntelliJ and VS Code on the Cody
+side, and the VS Code Cody extension has both configurations available
+in its default run configurations.
+
+VS Code is usually the better choice for a debugger, because the JetBrains
+TypeScript plugin gets very confused over our TypeScript code base, and
+cannot search for files or symbols.
+
+Sometimes using JetBrains is more convenient, so we describe both options.
+
 ## How to set up Run Configurations
 
 Run configurations are basically IDEA's launcher scripts. You will need
@@ -381,3 +394,22 @@ see the configuration dropdown at the top.
   - Workaround is to exit the target gracefully by quitting each time,
     using the menus or hotkeys, rather than force-stopping it.
 
+# Integration Testing
+
+Run the integration tests at the command line with:
+
+```
+./gradlew integrationTest
+```
+
+If you pass in an access token, it will use the default production LLM
+rather than a mock LLM, which can be useful when updating the test if
+the protocol changes.
+
+```
+CODY_INTEGRATION_TEST_TOKEN=sgp_asdfasdfasdfasdfasdfasdfasdf
+```
+
+You can run and debug the integration tests, including the Agent node
+process, with the instructions above by making new run configurations
+for the test.

build.gradle.kts

@@ -8,9 +8,10 @@ import java.nio.file.Paths
 import java.nio.file.SimpleFileVisitor
 import java.nio.file.StandardCopyOption
 import java.nio.file.attribute.BasicFileAttributes
-import java.util.EnumSet
+import java.util.*
 import java.util.jar.JarFile
 import java.util.zip.ZipFile
+import kotlin.script.experimental.jvm.util.hasParentNamed
 import org.jetbrains.changelog.markdownToHTML
 import org.jetbrains.intellij.tasks.RunPluginVerifierTask.FailureLevel
 import org.jetbrains.kotlin.gradle.tasks.KotlinCompile
@@ -27,28 +28,25 @@ val isForceCodeSearchBuild = isForceBuild || properties("forceCodeSearchBuild")
 
 plugins {
   id("java")
+  id("jvm-test-suite")
   // Dependencies are locked at this version to work with JDK 11 on CI.
-  id("org.jetbrains.kotlin.jvm") version "1.9.22"
+  id("org.jetbrains.kotlin.jvm") version "1.9.22" // Also change in gradle.properties
   id("org.jetbrains.intellij") version "1.17.2"
   id("org.jetbrains.changelog") version "1.3.1"
   id("com.diffplug.spotless") version "6.25.0"
 }
 
+val kotlinVersion: String by project
+val platformVersion: String by project
+val javaVersion: String by project
+
 group = properties("pluginGroup")
 
 version = properties("pluginVersion")
 
-repositories { mavenCentral() }
-
-intellij {
-  pluginName.set(properties("pluginName"))
-  version.set(properties("platformVersion"))
-  type.set(properties("platformType"))
-
-  // Plugin Dependencies. Uses `platformPlugins` property from the gradle.properties file.
-  plugins.set(properties("platformPlugins").split(',').map(String::trim).filter(String::isNotEmpty))
-
-  updateSinceUntilBuild.set(false)
+repositories {
+  maven { url = uri("https://www.jetbrains.com/intellij-repository/releases") }
+  mavenCentral()
 }
 
 dependencies {
@@ -57,6 +55,10 @@ dependencies {
   implementation("org.eclipse.lsp4j:org.eclipse.lsp4j.jsonrpc:0.21.0")
   implementation("com.googlecode.java-diff-utils:diffutils:1.3.0")
   testImplementation("org.awaitility:awaitility-kotlin:4.2.0")
+  testImplementation("org.junit.jupiter:junit-jupiter:5.7.0")
+  testImplementation("org.jetbrains.kotlin:kotlin-test-junit:1.9.22")
+  implementation("org.mockito:mockito-all:1.10.19")
+  testImplementation("org.mockito.kotlin:mockito-kotlin:5.3.1")
 }
 
 spotless {
@@ -74,9 +76,21 @@ spotless {
     ktfmt()
     trimTrailingWhitespace()
     target("src/**/*.kt")
+    toggleOffOn()
   }
 }
 
+intellij {
+  pluginName.set(properties("pluginName"))
+  version.set(platformVersion)
+  type.set(properties("platformType"))
+
+  // Plugin Dependencies. Uses `platformPlugins` property from the gradle.properties file.
+  plugins.set(properties("platformPlugins").split(',').map(String::trim).filter(String::isNotEmpty))
+
+  updateSinceUntilBuild.set(false)
+}
+
 java {
   toolchain {
     // Always compile the codebase with Java 11 regardless of what Java
@@ -86,6 +100,8 @@ java {
   }
 }
 
+tasks.named("classpathIndexCleanup") { dependsOn("compileIntegrationTestKotlin") }
+
 fun download(url: String, output: File) {
   if (output.exists()) {
     println("Cached $output")
@@ -158,7 +174,7 @@ fun unzip(input: File, output: File, excludeMatcher: PathMatcher? = null) {
   }
 }
 
-val githubArchiveCache =
+val githubArchiveCache: File =
     Paths.get(System.getProperty("user.home"), ".sourcegraph", "caches", "jetbrains").toFile()
 
 tasks {
@@ -344,6 +360,7 @@ tasks {
     systemProperty(
         "cody.autocomplete.enableFormatting",
         project.property("cody.autocomplete.enableFormatting") ?: "true")
+    environment("CODY_JETBRAINS_FEATURES", "cody.feature.inline-edits=true")
 
     val platformRuntimeVersion = project.findProperty("platformRuntimeVersion")
     if (platformRuntimeVersion != null) {
@@ -401,4 +418,68 @@ tasks {
   }
 
   test { dependsOn(project.tasks.getByPath("buildCody")) }
+
+  configurations {
+    create("integrationTestImplementation") { extendsFrom(configurations.testImplementation.get()) }
+    create("integrationTestRuntimeClasspath") { extendsFrom(configurations.testRuntimeOnly.get()) }
+  }
+
+  sourceSets {
+    create("integrationTest") {
+      kotlin.srcDir("src/integrationTest/kotlin")
+      compileClasspath += main.get().output
+      runtimeClasspath += main.get().output
+    }
+  }
+
+  // Common configuration for integration tests.
+  val sharedIntegrationTestConfig: Test.() -> Unit = {
+    group = "verification"
+    testClassesDirs = sourceSets["integrationTest"].output.classesDirs
+    classpath = sourceSets["integrationTest"].runtimeClasspath
+
+    include { it.file.hasParentNamed("integrationTest") }
+
+    useJUnit()
+
+    systemProperty("cody.integration.testing", "true")
+    systemProperty(
+        "idea.test.execution.policy", // For now, should be used by all tests
+        "com.sourcegraph.cody.test.NonEdtIdeaTestExecutionPolicy")
+
+    environment("CODY_JETBRAINS_FEATURES", "cody.feature.inline-edits=true")
+    environment("CODY_RECORDING_MODE", "replay")
+    environment("CODY_RECORDING_DIRECTORY", "recordings")
+    environment("CODY_RECORD_IF_MISSING", "false") // Polly needs this to record at all.
+
+    dependsOn("buildCody")
+  }
+
+  register<Test>("integrationTest") {
+    description = "Runs the integration tests."
+    applySharedConfiguration(sharedIntegrationTestConfig)
+  }
+
+  // Make sure to set CODY_INTEGRATION_TEST_TOKEN when using this task.
+  register<Test>("passthroughIntegrationTest") {
+    description = "Runs the integration tests, passing everything through to the LLM."
+    applySharedConfiguration(sharedIntegrationTestConfig)
+    environment("CODY_RECORDING_MODE", "passthrough")
+  }
+
+  // Make sure to set CODY_INTEGRATION_TEST_TOKEN when using this task.
+  register<Test>("recordingIntegrationTest") {
+    description = "Runs the integration tests and records the responses."
+    applySharedConfiguration(sharedIntegrationTestConfig)
+
+    environment("CODY_RECORDING_MODE", "record")
+    environment("CODY_RECORDING_NAME", "integration-tests")
+    environment("CODY_RECORD_IF_MISSING", "true") // Polly needs this to record at all.
+  }
+
+  named("check") { dependsOn("integrationTest") }
+}
+
+fun Test.applySharedConfiguration(sharedConfig: Test.() -> Unit) {
+  sharedConfig()
 }

gradle.properties

@@ -18,6 +18,7 @@ platformPlugins=Git4Idea,PerforceDirectPlugin,java
 javaVersion=11
 # Gradle Releases -> https://github.com/gradle/gradle/releases
 gradleVersion=8.1.1
+kotlinVersion=1.9.22 // Also change this in build.gradle.kts plugins section
 # Opt-out flag for bundling Kotlin standard library.
 # See https://plugins.jetbrains.com/docs/intellij/kotlin.html#kotlin-standard-library for details.
 # suppress inspection "UnusedProperty"