Merge branch 'master' into standalone_tests_intellij

.editorconfig

@@ -0,0 +1,10 @@
+# EditorConfig: http://editorconfig.org/
+
+root = true
+
+[*.java]
+charset = utf-8
+indent_style = space
+indent_size = 4
+trim_trailing_whitespace = true
+insert_final_newline = true

TESTING.asciidoc

@@ -149,17 +149,23 @@ gradle test -Dtests.awaitsfix=[false] - known issue (@AwaitsFix)
 
 === Load balancing and caches.
 
-By default, the tests run sequentially on a single forked JVM.
-
-To run with more forked JVMs than the default use:
+By default the tests run on up to 4 JVMs based on the number of cores. If you
+want to explicitly specify the number of JVMs you can do so on the command
+line:
 
 ----------------------------
 gradle test -Dtests.jvms=8
 ----------------------------
 
-Don't count hypercores for CPU-intense tests and leave some slack
-for JVM-internal threads (like the garbage collector). Make sure there is
-enough RAM to handle child JVMs.
+Or in `~/.gradle/gradle.properties`:
+
+----------------------------
+systemProp.tests.jvms=8
+----------------------------
+
+Its difficult to pick the "right" number here. Hypercores don't count for CPU
+intensive tests and you should leave some slack for JVM-interal threads like
+the garbage collector. And you have to have enough RAM to handle each JVM.
 
 === Test compatibility.
 
@@ -280,11 +286,20 @@ The REST layer is tested through specific tests that are shared between all
 the elasticsearch official clients and consist of YAML files that describe the
 operations to be executed and the obtained results that need to be tested.
 
-The REST tests are run automatically when executing the maven test command. To run only the
+The REST tests are run automatically when executing the "gradle check" command. To run only the
 REST tests use the following command:
 
 ---------------------------------------------------------------------------
-gradle integTest -Dtests.filter="@Rest"
+gradle :distribution:tar:integTest   \
+  -Dtests.class=org.elasticsearch.test.rest.RestIT
+---------------------------------------------------------------------------
+
+A specific test case can be run with
+
+---------------------------------------------------------------------------
+gradle :distribution:tar:integTest \
+  -Dtests.class=org.elasticsearch.test.rest.RestIT \
+  -Dtests.method="test {p0=cat.shards/10_basic/Help}"
 ---------------------------------------------------------------------------
 
 `RestNIT` are the executable test classes that runs all the

build.gradle

@@ -45,7 +45,7 @@ subprojects {
           }
         }
       }
-    }  
+    }
     extraArchive {
       javadoc = true
       tests = false
@@ -86,8 +86,8 @@ subprojects {
     tasks.withType(Jar) {
       into('META-INF') {
         from project.rootProject.rootDir
-        include 'LICENSE.txt'   
-        include 'NOTICE.txt'   
+        include 'LICENSE.txt'
+        include 'NOTICE.txt'
       }
     }
     // ignore missing javadocs
@@ -101,12 +101,19 @@ subprojects {
     }
   }
 
+  /* Sets up the dependencies that we build as part of this project but
+    register as thought they were external to resolve internally. We register
+    them as external dependencies so the build plugin that we use can be used
+    to build elasticsearch plugins outside of the elasticsearch source tree. */
   ext.projectSubstitutions = [
     "org.elasticsearch:rest-api-spec:${version}": ':rest-api-spec',
     "org.elasticsearch:elasticsearch:${version}": ':core',
     "org.elasticsearch:test-framework:${version}": ':test-framework',
+    "org.elasticsearch.distribution.integ-test-zip:elasticsearch:${version}": ':distribution:integ-test-zip',
     "org.elasticsearch.distribution.zip:elasticsearch:${version}": ':distribution:zip',
-    "org.elasticsearch.distribution.tar:elasticsearch:${version}": ':distribution:tar'
+    "org.elasticsearch.distribution.tar:elasticsearch:${version}": ':distribution:tar',
+    "org.elasticsearch.distribution.rpm:elasticsearch:${version}": ':distribution:rpm',
+    "org.elasticsearch.distribution.deb:elasticsearch:${version}": ':distribution:deb',
   ]
   configurations.all {
     resolutionStrategy.dependencySubstitution { DependencySubstitutions subs ->
@@ -232,12 +239,11 @@ class Run extends DefaultTask {
   )
   public void setDebug(boolean enabled) {
     project.project(':distribution').run.clusterConfig.debug = enabled
-  } 
+  }
 }
 task run(type: Run) {
   dependsOn ':distribution:run'
   description = 'Runs elasticsearch in the foreground'
   group = 'Verification'
   impliesSubProjects = true
 }
-

buildSrc/build.gradle

@@ -80,3 +80,13 @@ eclipse {
     defaultOutputDir = new File(file('build'), 'eclipse')
   }
 }
+
+task copyEclipseSettings(type: Copy) {
+  from project.file('src/main/resources/eclipse.settings')
+  into '.settings'
+}
+// otherwise .settings is not nuked entirely
+tasks.cleanEclipse {
+  delete '.settings'
+}
+tasks.eclipse.dependsOn(cleanEclipse, copyEclipseSettings)

buildSrc/src/main/groovy/com/carrotsearch/gradle/junit4/TestLoggingConfiguration.groovy

@@ -1,5 +1,6 @@
 package com.carrotsearch.gradle.junit4
 
+import org.gradle.api.tasks.Input
 import org.gradle.util.ConfigureUtil
 
 class TestLoggingConfiguration {
@@ -20,6 +21,10 @@ class TestLoggingConfiguration {
     SlowTestsConfiguration slowTests = new SlowTestsConfiguration()
     StackTraceFiltersConfiguration stackTraceFilters = new StackTraceFiltersConfiguration()
 
+    /** Summarize the first N failures at the end of the test. */
+    @Input
+    int showNumFailuresAtEnd = 3 // match TextReport default
+
     void slowTests(Closure closure) {
         ConfigureUtil.configure(closure, slowTests)
     }
@@ -31,4 +36,8 @@ class TestLoggingConfiguration {
     void outputMode(String mode) {
         outputMode = mode.toUpperCase() as OutputMode
     }
+
+    void showNumFailuresAtEnd(int n) {
+        showNumFailuresAtEnd = n
+    }
 }

buildSrc/src/main/groovy/com/carrotsearch/gradle/junit4/TestReportLogger.groovy

@@ -48,9 +48,6 @@ class TestReportLogger extends TestsSummaryEventListener implements AggregatedEv
     /** Format line for JVM ID string. */
     String jvmIdFormat
 
-    /** Summarize the first N failures at the end. */
-    int showNumFailuresAtEnd = 3
-
     /** Output stream that logs messages to the given logger */
     LoggingOutputStream outStream
     LoggingOutputStream errStream
@@ -110,13 +107,13 @@ class TestReportLogger extends TestsSummaryEventListener implements AggregatedEv
 
     @Subscribe
     void onQuit(AggregatedQuitEvent e) throws IOException {
-        if (showNumFailuresAtEnd > 0 && !failedTests.isEmpty()) {
+        if (config.showNumFailuresAtEnd > 0 && !failedTests.isEmpty()) {
             List<Description> sublist = this.failedTests
             StringBuilder b = new StringBuilder()
             b.append('Tests with failures')
-            if (sublist.size() > showNumFailuresAtEnd) {
-                sublist = sublist.subList(0, showNumFailuresAtEnd)
-                b.append(" (first " + showNumFailuresAtEnd + " out of " + failedTests.size() + ")")
+            if (sublist.size() > config.showNumFailuresAtEnd) {
+                sublist = sublist.subList(0, config.showNumFailuresAtEnd)
+                b.append(" (first " + config.showNumFailuresAtEnd + " out of " + failedTests.size() + ")")
             }
             b.append(':\n')
             for (Description description : sublist) {

buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy

@@ -62,7 +62,7 @@ class BuildPlugin implements Plugin<Project> {
         configureCompile(project)
 
         configureTest(project)
-        PrecommitTasks.configure(project)
+        configurePrecommit(project)
     }
 
     /** Performs checks on the build environment and prints information about the build environment. */
@@ -283,6 +283,7 @@ class BuildPlugin implements Plugin<Project> {
 
     /** Adds compiler settings to the project */
     static void configureCompile(Project project) {
+        project.ext.compactProfile = 'compact3'
         project.afterEvaluate {
             // fail on all javac warnings
             project.tasks.withType(JavaCompile) {
@@ -295,6 +296,11 @@ class BuildPlugin implements Plugin<Project> {
                  */
                 // don't even think about passing args with -J-xxx, oracle will ask you to submit a bug report :)
                 options.compilerArgs << '-Werror' << '-Xlint:all,-path' << '-Xdoclint:all' << '-Xdoclint:-missing'
+                // compile with compact 3 profile by default
+                // NOTE: this is just a compile time check: does not replace testing with a compact3 JRE
+                if (project.compactProfile != 'full') {
+                    options.compilerArgs << '-profile' << project.compactProfile
+                }
                 options.encoding = 'UTF-8'
             }
         }
@@ -365,6 +371,7 @@ class BuildPlugin implements Plugin<Project> {
             enableSystemAssertions false
 
             testLogging {
+                showNumFailuresAtEnd 25
                 slowTests {
                     heartbeat 10
                     summarySize 5
@@ -409,4 +416,11 @@ class BuildPlugin implements Plugin<Project> {
         }
         return test
     }
+
+    private static configurePrecommit(Project project) {
+        Task precommit = PrecommitTasks.create(project, true)
+        project.check.dependsOn(precommit)
+        project.test.mustRunAfter(precommit)
+        project.dependencyLicenses.dependencies = project.configurations.runtime - project.configurations.provided
+    }
 }

buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginBuildPlugin.groovy

@@ -23,40 +23,41 @@ import org.elasticsearch.gradle.test.RestIntegTestTask
 import org.elasticsearch.gradle.test.RunTask
 import org.gradle.api.Project
 import org.gradle.api.Task
+import org.gradle.api.tasks.SourceSet
 import org.gradle.api.tasks.bundling.Zip
 
 /**
  * Encapsulates build configuration for an Elasticsearch plugin.
  */
-class PluginBuildPlugin extends BuildPlugin {
+public class PluginBuildPlugin extends BuildPlugin {
 
     @Override
-    void apply(Project project) {
+    public void apply(Project project) {
         super.apply(project)
         configureDependencies(project)
-        // this afterEvaluate must happen before the afterEvaluate added by integTest configure,
+        // this afterEvaluate must happen before the afterEvaluate added by integTest creation,
         // so that the file name resolution for installing the plugin will be setup
         project.afterEvaluate {
             String name = project.pluginProperties.extension.name
             project.jar.baseName = name
             project.bundlePlugin.baseName = name
+
             project.integTest.dependsOn(project.bundlePlugin)
-            project.integTest.clusterConfig.plugin(name, project.bundlePlugin.outputs.files)
             project.tasks.run.dependsOn(project.bundlePlugin)
-            project.tasks.run.clusterConfig.plugin(name, project.bundlePlugin.outputs.files)
-        }
-        RestIntegTestTask.configure(project)
-        RunTask.configure(project)
-        Task bundle = configureBundleTask(project)
-        project.configurations.archives.artifacts.removeAll { it.archiveTask.is project.jar }
-        project.configurations.getByName('default').extendsFrom = []
-        project.artifacts {
-            archives bundle
-            'default' bundle
+            if (project.path.startsWith(':modules:')) {
+                project.integTest.clusterConfig.module(project)
+                project.tasks.run.clusterConfig.module(project)
+            } else {
+                project.integTest.clusterConfig.plugin(name, project.bundlePlugin.outputs.files)
+                project.tasks.run.clusterConfig.plugin(name, project.bundlePlugin.outputs.files)
+            }
         }
+        createIntegTestTask(project)
+        createBundleTask(project)
+        project.tasks.create('run', RunTask) // allow running ES with this plugin in the foreground of a build
     }
 
-    static void configureDependencies(Project project) {
+    private static void configureDependencies(Project project) {
         project.dependencies {
             provided "org.elasticsearch:elasticsearch:${project.versions.elasticsearch}"
             testCompile "org.elasticsearch:test-framework:${project.versions.elasticsearch}"
@@ -72,21 +73,36 @@ class PluginBuildPlugin extends BuildPlugin {
         }
     }
 
-    static Task configureBundleTask(Project project) {
-        PluginPropertiesTask buildProperties = project.tasks.create(name: 'pluginProperties', type: PluginPropertiesTask)
-        File pluginMetadata = project.file("src/main/plugin-metadata")
-        project.sourceSets.test {
-            output.dir(buildProperties.generatedResourcesDir, builtBy: 'pluginProperties')
-            resources {
-                srcDir pluginMetadata
-            }
-        }
-        Task bundle = project.tasks.create(name: 'bundlePlugin', type: Zip, dependsOn: [project.jar, buildProperties])
-        bundle.configure {
-            from buildProperties
-            from pluginMetadata
-            from project.jar
-            from bundle.project.configurations.runtime - bundle.project.configurations.provided
+    /** Adds an integTest task which runs rest tests */
+    private static void createIntegTestTask(Project project) {
+        RestIntegTestTask integTest = project.tasks.create('integTest', RestIntegTestTask.class)
+        integTest.mustRunAfter(project.precommit, project.test)
+        project.check.dependsOn(integTest)
+    }
+
+    /**
+     * Adds a bundlePlugin task which builds the zip containing the plugin jars,
+     * metadata, properties, and packaging files
+     */
+    private static void createBundleTask(Project project) {
+        File pluginMetadata = project.file('src/main/plugin-metadata')
+
+        // create a task to build the properties file for this plugin
+        PluginPropertiesTask buildProperties = project.tasks.create('pluginProperties', PluginPropertiesTask.class)
+
+        // add the plugin properties and metadata to test resources, so unit tests can
+        // know about the plugin (used by test security code to statically initialize the plugin in unit tests)
+        SourceSet testSourceSet = project.sourceSets.test
+        testSourceSet.output.dir(buildProperties.generatedResourcesDir, builtBy: 'pluginProperties')
+        testSourceSet.resources.srcDir(pluginMetadata)
+
+        // create the actual bundle task, which zips up all the files for the plugin
+        Zip bundle = project.tasks.create(name: 'bundlePlugin', type: Zip, dependsOn: [project.jar, buildProperties]) {
+            from buildProperties // plugin properties file
+            from pluginMetadata // metadata (eg custom security policy)
+            from project.jar // this plugin's jar
+            from project.configurations.runtime - project.configurations.provided // the dep jars
+            // extra files for the plugin to go into the zip
             from('src/main/packaging') // TODO: move all config/bin/_size/etc into packaging
             from('src/main') {
                 include 'config/**'
@@ -97,6 +113,13 @@ class PluginBuildPlugin extends BuildPlugin {
             }
         }
         project.assemble.dependsOn(bundle)
-        return bundle
+
+        // remove jar from the archives (things that will be published), and set it to the zip
+        project.configurations.archives.artifacts.removeAll { it.archiveTask.is project.jar }
+        project.artifacts.add('archives', bundle)
+
+        // also make the zip the default artifact (used when depending on this project)
+        project.configurations.getByName('default').extendsFrom = []
+        project.artifacts.add('default', bundle)
     }
 }