From 96c7ecfc3b95404b21448a20311ddd415e4f8d7d Mon Sep 17 00:00:00 2001 From: Eric Ayers Date: Tue, 19 Apr 2016 18:30:07 -0700 Subject: [PATCH] Enhance parallel testing junit_tests Surfaces EXPERIMENTAL support for parallel testing of methods within a test class. - Adds a 'concurrency' parameter to set test concurrency to 'serial' 'parallel', or 'parallel_method' - Adds a 'threads' parameter to junit_tests to control concurrency. - Adds a --test-junit-parallel-methods to allow methods to run in parallel for the entire test run - Added a number of unit test cases and integration tests to exercise these features Note that there is a bug in that the @TestSerial annotation is not respected when the -parallel-methods flag is in use. See https://github.com/pantsbuild/pants/issues/3209 Followup work for this change is to add an `@TestParallelMethods` annotation and come up with a more rational way to pass concurrency options to the junit-runner backend. Testing Done: Integration test and additional unit tests added. CI is green at https://travis-ci.org/pantsbuild/pants/builds/123712609 Bugs closed: 3191, 3210 Reviewed at https://rbcommons.com/s/twitter/r/3707/ --- .../junit/annotations/TestSerial.java | 4 + .../tools/junit/impl/ConsoleRunnerImpl.java | 3 + .../pants/backend/jvm/targets/java_tests.py | 43 +++++- src/python/pants/backend/jvm/tasks/BUILD | 1 + .../pants/backend/jvm/tasks/junit_run.py | 53 +++++++- src/python/pants/util/BUILD | 5 + src/python/pants/util/argutil.py | 52 ++++++++ .../parallel/AnnotatedParallelTest1.java | 43 ++++++ .../parallel/AnnotatedParallelTest2.java | 18 +++ .../parallel/AnnotatedSerialTest1.java | 45 +++++++ .../parallel/AnnotatedSerialTest2.java | 18 +++ .../org/pantsbuild/testproject/parallel/BUILD | 53 ++++++++ .../testproject/parallel/ParallelTest1.java | 41 ++++++ .../testproject/parallel/ParallelTest2.java | 16 +++ .../testproject/parallelmethods/BUILD | 21 +++ .../ParallelMethodsDefaultParallelTest1.java | 48 +++++++ .../ParallelMethodsDefaultParallelTest2.java | 21 +++ .../tools/junit/impl/ConsoleRunnerTest.java | 21 +++ .../junit/lib/AnnotatedParallelTest1.java | 47 +++++++ .../junit/lib/AnnotatedParallelTest2.java | 18 +++ .../tools/junit/lib/AnnotatedSerialTest1.java | 49 +++++++ .../tools/junit/lib/AnnotatedSerialTest2.java | 18 +++ .../ParallelMethodsDefaultParallelTest1.java | 8 +- .../ParallelMethodsDefaultParallelTest2.java | 2 + .../tools/junit/lib/ParallelTest1.java | 43 ++++++ .../tools/junit/lib/ParallelTest2.java | 16 +++ .../pants_test/backend/jvm/targets/BUILD | 10 ++ .../backend/jvm/targets/test_java_tests.py | 49 +++++++ .../python/pants_test/backend/jvm/tasks/BUILD | 12 ++ ...est_junit_tests_concurrency_integration.py | 126 ++++++++++++++++++ .../projects/test_testprojects_integration.py | 1 + tests/python/pants_test/util/BUILD | 9 ++ tests/python/pants_test/util/test_argutil.py | 37 +++++ 33 files changed, 941 insertions(+), 10 deletions(-) create mode 100644 src/python/pants/util/argutil.py create mode 100644 testprojects/tests/java/org/pantsbuild/testproject/parallel/AnnotatedParallelTest1.java create mode 100644 testprojects/tests/java/org/pantsbuild/testproject/parallel/AnnotatedParallelTest2.java create mode 100644 testprojects/tests/java/org/pantsbuild/testproject/parallel/AnnotatedSerialTest1.java create mode 100644 testprojects/tests/java/org/pantsbuild/testproject/parallel/AnnotatedSerialTest2.java create mode 100644 testprojects/tests/java/org/pantsbuild/testproject/parallel/BUILD create mode 100644 testprojects/tests/java/org/pantsbuild/testproject/parallel/ParallelTest1.java create mode 100644 testprojects/tests/java/org/pantsbuild/testproject/parallel/ParallelTest2.java create mode 100644 testprojects/tests/java/org/pantsbuild/testproject/parallelmethods/BUILD create mode 100644 testprojects/tests/java/org/pantsbuild/testproject/parallelmethods/ParallelMethodsDefaultParallelTest1.java create mode 100644 testprojects/tests/java/org/pantsbuild/testproject/parallelmethods/ParallelMethodsDefaultParallelTest2.java create mode 100644 tests/java/org/pantsbuild/tools/junit/lib/AnnotatedParallelTest1.java create mode 100644 tests/java/org/pantsbuild/tools/junit/lib/AnnotatedParallelTest2.java create mode 100644 tests/java/org/pantsbuild/tools/junit/lib/AnnotatedSerialTest1.java create mode 100644 tests/java/org/pantsbuild/tools/junit/lib/AnnotatedSerialTest2.java create mode 100644 tests/java/org/pantsbuild/tools/junit/lib/ParallelTest1.java create mode 100644 tests/java/org/pantsbuild/tools/junit/lib/ParallelTest2.java create mode 100644 tests/python/pants_test/backend/jvm/targets/test_java_tests.py create mode 100644 tests/python/pants_test/backend/jvm/tasks/test_junit_tests_concurrency_integration.py create mode 100644 tests/python/pants_test/util/test_argutil.py diff --git a/src/java/org/pantsbuild/junit/annotations/TestSerial.java b/src/java/org/pantsbuild/junit/annotations/TestSerial.java index 78a2a34460be..851ecec24f4c 100644 --- a/src/java/org/pantsbuild/junit/annotations/TestSerial.java +++ b/src/java/org/pantsbuild/junit/annotations/TestSerial.java @@ -13,6 +13,10 @@ * Annotate that a test class must be run in serial. See usage note in * {@code org.pantsbuild.tools.junit.impl.ConsoleRunnerImpl}. This annotation takes precedence * over a {@link TestParallel} annotation if a class has both (including via inheritance). + *

+ * Note that this annotation is not currently compatible with the PARALLEL_METHODS default + * concurrency setting. See + * issue 3209 */ @Retention(RetentionPolicy.RUNTIME) @Inherited diff --git a/src/java/org/pantsbuild/tools/junit/impl/ConsoleRunnerImpl.java b/src/java/org/pantsbuild/tools/junit/impl/ConsoleRunnerImpl.java index 1270e36d1c82..e18e4ba96a80 100644 --- a/src/java/org/pantsbuild/tools/junit/impl/ConsoleRunnerImpl.java +++ b/src/java/org/pantsbuild/tools/junit/impl/ConsoleRunnerImpl.java @@ -657,6 +657,9 @@ class Options { usage = "Show a description of each test and timer for each test class.") private boolean perTestTimer; + // TODO(zundel): Combine -default-parallel and -paralel-methods together into a + // single argument: -default-concurrency {serial, parallel, parallel_methods} + // TODO(zundel): Also add a @TestParallelMethods annotation @Option(name = "-default-parallel", usage = "Whether to run test classes without @TestParallel or @TestSerial in parallel.") private boolean defaultParallel; diff --git a/src/python/pants/backend/jvm/targets/java_tests.py b/src/python/pants/backend/jvm/targets/java_tests.py index 910a29d39941..849e49842fac 100644 --- a/src/python/pants/backend/jvm/targets/java_tests.py +++ b/src/python/pants/backend/jvm/targets/java_tests.py @@ -7,6 +7,7 @@ from pants.backend.jvm.subsystems.jvm_platform import JvmPlatform from pants.backend.jvm.targets.jvm_target import JvmTarget +from pants.base.exceptions import TargetDefinitionException from pants.base.payload import Payload from pants.base.payload_field import PrimitiveField @@ -17,8 +18,11 @@ class JavaTests(JvmTarget): :API: public """ + _VALID_CONCURRENCY_OPTS = ['serial', 'parallel', 'parallel_methods'] + def __init__(self, cwd=None, test_platform=None, payload=None, timeout=None, - extra_jvm_options=None, extra_env_vars=None, **kwargs): + extra_jvm_options=None, extra_env_vars=None, concurrency=None, + threads=None, **kwargs): """ :param str cwd: working directory (relative to the build root) for the tests under this target. If unspecified (None), the working directory will be controlled by junit_run's --cwd. @@ -31,8 +35,12 @@ def __init__(self, cwd=None, test_platform=None, payload=None, timeout=None, tests. Example: ['-Dexample.property=1'] If unspecified, no extra jvm options will be added. :param dict extra_env_vars: A map of environment variables to set when running the tests, e.g. { 'FOOBAR': 12 }. Using `None` as the value will cause the variable to be unset. + :param string concurrency: One of 'serial', 'parallel', or 'parallel_methods'. Overrides + the setting of --test-junit-default-parallel or --test-junit-parallel-methods options. + :param int threads: Use the specified number of threads when running the test. Overrides + the setting of --test-junit-parallel-threads. """ - self.cwd = cwd + payload = payload or Payload() if extra_env_vars is None: @@ -43,12 +51,29 @@ def __init__(self, cwd=None, test_platform=None, payload=None, timeout=None, payload.add_fields({ 'test_platform': PrimitiveField(test_platform), + # TODO(zundel): Do extra_jvm_options and extra_env_vars really need to be fingerprinted? 'extra_jvm_options': PrimitiveField(tuple(extra_jvm_options or ())), 'extra_env_vars': PrimitiveField(tuple(extra_env_vars.items())), }) - self._timeout = timeout super(JavaTests, self).__init__(payload=payload, **kwargs) + # These parameters don't need to go into the fingerprint: + self._concurrency = concurrency + self._cwd = cwd + self._threads = None + self._timeout = timeout + + try: + if threads is not None: + self._threads = int(threads) + except ValueError: + raise TargetDefinitionException(self, + "The value for 'threads' must be an integer, got " + threads) + if concurrency and concurrency not in self._VALID_CONCURRENCY_OPTS: + raise TargetDefinitionException(self, + "The value for 'concurrency' must be one of " + + repr(self._VALID_CONCURRENCY_OPTS) + " got: " + concurrency) + # TODO(John Sirois): These could be scala, clojure, etc. 'jvm' and 'tests' are the only truly # applicable labels - fixup the 'java' misnomer. self.add_labels('java', 'tests') @@ -59,6 +84,18 @@ def test_platform(self): return JvmPlatform.global_instance().get_platform_by_name(self.payload.test_platform) return self.platform + @property + def concurrency(self): + return self._concurrency + + @property + def cwd(self): + return self._cwd + + @property + def threads(self): + return self._threads + @property def timeout(self): return self._timeout diff --git a/src/python/pants/backend/jvm/tasks/BUILD b/src/python/pants/backend/jvm/tasks/BUILD index 89425b2b3724..784b022f60cd 100644 --- a/src/python/pants/backend/jvm/tasks/BUILD +++ b/src/python/pants/backend/jvm/tasks/BUILD @@ -326,6 +326,7 @@ python_library( 'src/python/pants/build_graph', 'src/python/pants/java:util', 'src/python/pants/task', + 'src/python/pants/util:argutil', 'src/python/pants/util:contextutil', 'src/python/pants/util:dirutil', 'src/python/pants/util:process_handler', diff --git a/src/python/pants/backend/jvm/tasks/junit_run.py b/src/python/pants/backend/jvm/tasks/junit_run.py index c85bb97e3aaf..fa2e838eb772 100644 --- a/src/python/pants/backend/jvm/tasks/junit_run.py +++ b/src/python/pants/backend/jvm/tasks/junit_run.py @@ -31,6 +31,7 @@ from pants.java.distribution.distribution import DistributionLocator from pants.java.executor import SubprocessExecutor from pants.task.testrunner_task_mixin import TestRunnerTaskMixin +from pants.util.argutil import ensure_arg, remove_arg from pants.util.contextutil import environment_as from pants.util.strutil import pluralize from pants.util.xml_parser import XmlParser @@ -62,6 +63,14 @@ class JUnitRun(TestRunnerTaskMixin, JvmToolTaskMixin, JvmTask): """ :API: public """ + + _CONCURRENCY_PARALLEL = 'PARALLEL' + _CONCURRENCY_PARALLEL_METHODS = 'PARALLEL_METHODS' + _CONCURRENCY_SERIAL = 'SERIAL' + _CONCURRENCY_CHOICES = [ + _CONCURRENCY_PARALLEL, _CONCURRENCY_PARALLEL_METHODS, _CONCURRENCY_SERIAL + ] + _MAIN = 'org.pantsbuild.tools.junit.ConsoleRunner' @classmethod @@ -73,7 +82,12 @@ def register_options(cls, register): help='Force running of just these tests. Tests can be specified using any of: ' '[classname], [classname]#[methodname], [filename] or [filename]#[methodname]') register('--per-test-timer', type=bool, help='Show progress and timer for each test.') + register('--default-concurrency', advanced=True, + choices=cls._CONCURRENCY_CHOICES, default=cls._CONCURRENCY_SERIAL, + help='Set the default concurrency mode for running tests not annotated with' + + ' @TestParallel or @TestSerial.') register('--default-parallel', advanced=True, type=bool, + deprecated_hint='Use --concurrency instead.', deprecated_version='0.0.86', help='Run classes without @TestParallel or @TestSerial annotations in parallel.') register('--parallel-threads', advanced=True, type=int, default=0, help='Number of threads to run tests in parallel. 0 for autoset.') @@ -101,7 +115,7 @@ def register_options(cls, register): cls.register_jvm_tool(register, 'junit', classpath=[ - JarDependency(org='org.pantsbuild', name='junit-runner', rev='1.0.4'), + JarDependency(org='org.pantsbuild', name='junit-runner', rev='1.0.5'), ], main=JUnitRun._MAIN, # TODO(John Sirois): Investigate how much less we can get away with. @@ -174,11 +188,23 @@ def __init__(self, *args, **kwargs): self._args.append('-fail-fast') self._args.append('-outdir') self._args.append(self.workdir) - if options.per_test_timer: self._args.append('-per-test-timer') + + # TODO(zundel): Simply remove when --default_parallel finishes deprecation if options.default_parallel: self._args.append('-default-parallel') + + if options.default_concurrency == self._CONCURRENCY_PARALLEL_METHODS: + self._args.append('-default-parallel') + self._args.append('-parallel-methods') + elif options.default_concurrency == self._CONCURRENCY_PARALLEL: + self._args.append('-default-parallel') + elif options.default_concurrency == self._CONCURRENCY_SERIAL: + # TODO(zundel): we can't do anything here yet while the --default-parallel + # option is in deprecation mode. + pass + self._args.append('-parallel-threads') self._args.append(str(options.parallel_threads)) @@ -328,13 +354,16 @@ def _run_tests(self, tests_to_targets): lambda target: target.test_platform, lambda target: target.payload.extra_jvm_options, lambda target: target.payload.extra_env_vars, + lambda target: target.concurrency, + lambda target: target.threads ) # the below will be None if not set, and we'll default back to runtime_classpath classpath_product = self.context.products.get_data('instrument_classpath') result = 0 - for (workdir, platform, target_jvm_options, target_env_vars), tests in tests_by_properties.items(): + for properties, tests in tests_by_properties.items(): + (workdir, platform, target_jvm_options, target_env_vars, concurrency, threads) = properties for batch in self._partition(tests): # Batches of test classes will likely exist within the same targets: dedupe them. relevant_targets = set(map(tests_to_targets.get, batch)) @@ -345,6 +374,22 @@ def _run_tests(self, tests_to_targets): classpath_product=classpath_product)) complete_classpath.update(classpath_append) distribution = JvmPlatform.preferred_jvm_distribution([platform], self._strict_jvm_version) + + # Override cmdline args with values from junit_test() target that specify concurrency: + args = self._args + [u'-xmlreport'] + + # TODO(zundel): Combine these together into a single -concurrency choices style argument + if concurrency == 'serial': + args = remove_arg(args, '-default-parallel') + if concurrency == 'parallel': + args = ensure_arg(args, '-default-parallel') + if concurrency == 'parallel_methods': + args = ensure_arg(args, '-default-parallel') + args = ensure_arg(args, '-parallel-methods') + if threads is not None: + args = remove_arg(args, '-parallel-threads', has_param=True) + args += ['-parallel-threads', str(threads)] + with binary_util.safe_args(batch, self.get_options()) as batch_tests: self.context.log.debug('CWD = {}'.format(workdir)) self.context.log.debug('platform = {}'.format(platform)) @@ -355,7 +400,7 @@ def _run_tests(self, tests_to_targets): classpath=complete_classpath, main=JUnitRun._MAIN, jvm_options=self.jvm_options + extra_jvm_options + list(target_jvm_options), - args=self._args + batch_tests + [u'-xmlreport'], + args=args + batch_tests, workunit_factory=self.context.new_workunit, workunit_name='run', workunit_labels=[WorkUnitLabel.TEST], diff --git a/src/python/pants/util/BUILD b/src/python/pants/util/BUILD index 4b78f2801235..e5c889e18ec7 100644 --- a/src/python/pants/util/BUILD +++ b/src/python/pants/util/BUILD @@ -1,6 +1,11 @@ # Copyright 2014 Pants project contributors (see CONTRIBUTORS.md). # Licensed under the Apache License, Version 2.0 (see LICENSE). +python_library( + name = 'argutil', + sources = ['argutil.py'], +) + python_library( name = 'contextutil', sources = ['contextutil.py'], diff --git a/src/python/pants/util/argutil.py b/src/python/pants/util/argutil.py new file mode 100644 index 000000000000..1d1e1d30f191 --- /dev/null +++ b/src/python/pants/util/argutil.py @@ -0,0 +1,52 @@ +# coding=utf-8 +# Copyright 2016 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import (absolute_import, division, generators, nested_scopes, print_function, + unicode_literals, with_statement) + + +def ensure_arg(args, arg, param=None): + """Make sure the arg is present in the list of args. + + If arg is not present, adds the arg and the optional param. + If present and param != None, sets the parameter following the arg to param. + + :param list args: strings representing an argument list. + :param string arg: argument to make sure is present in the list. + :param string param: parameter to add or update after arg in the list. + :return: possibly modified list of args. + """ + found = False + for idx, found_arg in enumerate(args): + if found_arg == arg: + if param is not None: + args[idx + 1] = param + return args + + if not found: + args += [arg] + if param is not None: + args += [param] + return args + + +def remove_arg(args, arg, has_param=False): + """Removes the first instance of the specified arg from the list of args. + + If the arg is present and has_param is set, also removes the parameter that follows + the arg. + :param list args: strings representing an argument list. + :param staring arg: argument to remove from the list. + :param bool has_param: if true, also remove the parameter that follows arg in the list. + :return: possibly modified list of args. + """ + for idx, found_arg in enumerate(args): + if found_arg == arg: + if has_param: + slice_idx = idx + 2 + else: + slice_idx = idx + 1 + args = args[:idx] + args[slice_idx:] + break + return args diff --git a/testprojects/tests/java/org/pantsbuild/testproject/parallel/AnnotatedParallelTest1.java b/testprojects/tests/java/org/pantsbuild/testproject/parallel/AnnotatedParallelTest1.java new file mode 100644 index 000000000000..a8d612be24a4 --- /dev/null +++ b/testprojects/tests/java/org/pantsbuild/testproject/parallel/AnnotatedParallelTest1.java @@ -0,0 +1,43 @@ +// Copyright 2016 Pants project contributors (see CONTRIBUTORS.md). +// Licensed under the Apache License, Version 2.0 (see LICENSE). +package org.pantsbuild.testproject.parallel; + +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import org.junit.Test; +import org.pantsbuild.junit.annotations.TestParallel; + +import static org.junit.Assert.assertTrue; + +/** + * This test is designed to exercise the TestParallel annotation. + * A similar test runs in tests/java/... to exercise junit-runner standalone. + *

+ * For all methods in AnnotatedParallelTest1 and AnnotatedParallelTest2 + * to succeed, both test classes must be running at the same time with the flag: + * 

+ *  --test-junit-parallel-threads 2
+ * 
+ * when running with just these two classes as specs.
+ * 

+ * Runs in on the order of 10 milliseconds locally, but it may take longer on a CI machine to spin
+ * up 2 threads, so it has a generous timeout set.
+ */
+@TestParallel
+public class AnnotatedParallelTest1 {
+  private static final int NUM_CONCURRENT_TESTS = 2;
+  private static final int RETRY_TIMEOUT_MS = 3000;
+  private static CountDownLatch latch = new CountDownLatch(NUM_CONCURRENT_TESTS);
+
+  @Test
+  public void aptest1() throws Exception {
+    awaitLatch("aptest1");
+  }
+
+  static void awaitLatch(String methodName) throws Exception {
+    System.out.println("start " + methodName);
+    latch.countDown();
+    assertTrue(latch.await(RETRY_TIMEOUT_MS, TimeUnit.MILLISECONDS));
+    System.out.println("end " + methodName);
+  }
+}
diff --git a/testprojects/tests/java/org/pantsbuild/testproject/parallel/AnnotatedParallelTest2.java b/testprojects/tests/java/org/pantsbuild/testproject/parallel/AnnotatedParallelTest2.java
new file mode 100644
index 000000000000..0b709d311cf3
--- /dev/null
+++ b/testprojects/tests/java/org/pantsbuild/testproject/parallel/AnnotatedParallelTest2.java
@@ -0,0 +1,18 @@
+// Copyright 2016 Pants project contributors (see CONTRIBUTORS.md).
+// Licensed under the Apache License, Version 2.0 (see LICENSE).
+package org.pantsbuild.testproject.parallel;
+
+import org.junit.Test;
+import org.pantsbuild.junit.annotations.TestParallel;
+
+/**
+ * See {@link AnnotatedParallelTest1}
+ */
+@TestParallel
+public class AnnotatedParallelTest2 {
+
+  @Test
+  public void aptest2() throws Exception {
+    AnnotatedParallelTest1.awaitLatch("aptest2");
+  }
+}
diff --git a/testprojects/tests/java/org/pantsbuild/testproject/parallel/AnnotatedSerialTest1.java b/testprojects/tests/java/org/pantsbuild/testproject/parallel/AnnotatedSerialTest1.java
new file mode 100644
index 000000000000..92db47a0357e
--- /dev/null
+++ b/testprojects/tests/java/org/pantsbuild/testproject/parallel/AnnotatedSerialTest1.java
@@ -0,0 +1,45 @@
+// Copyright 2016 Pants project contributors (see CONTRIBUTORS.md).
+// Licensed under the Apache License, Version 2.0 (see LICENSE).
+package org.pantsbuild.testproject.parallel;
+
+import java.util.concurrent.atomic.AtomicBoolean;
+import org.junit.Test;
+import org.pantsbuild.junit.annotations.TestSerial;
+
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * This test is designed to exercise the TestSerial annotation when run under pants.
+ * A similar test runs in tests/java/... to exercise junit-runner standalone.
+ * 

+ * These tests are intended to show that the two classes will be run serially, even if
+ * parallel test running is on.
+ * To properly exercise this function, both test classes must be running at the same time with
+ * the flags:
+ * 
+ *  --test-junit-default-concurrency=PARALLEL --test-junit-parallel-threads 2
+ * 
+ * when running with just these two classes as specs.
+ * 

+ * Uses a timeout, so its not completely deterministic, but it gives 3 seconds to allow any
+ * concurrency to take place.
+ */
+@TestSerial
+public class AnnotatedSerialTest1 {
+  private static final int WAIT_TIMEOUT_MS = 3000;
+  private static AtomicBoolean waiting = new AtomicBoolean(false);
+
+  @Test
+  public void astest1() throws Exception {
+    awaitLatch("astest1");
+  }
+
+  static void awaitLatch(String methodName) throws Exception {
+    System.out.println("start " + methodName);
+    assertFalse(waiting.getAndSet(true));
+    Thread.sleep(WAIT_TIMEOUT_MS);
+    assertTrue(waiting.getAndSet(false));
+    System.out.println("end " + methodName);
+  }
+}
diff --git a/testprojects/tests/java/org/pantsbuild/testproject/parallel/AnnotatedSerialTest2.java b/testprojects/tests/java/org/pantsbuild/testproject/parallel/AnnotatedSerialTest2.java
new file mode 100644
index 000000000000..eeecbdfe0bab
--- /dev/null
+++ b/testprojects/tests/java/org/pantsbuild/testproject/parallel/AnnotatedSerialTest2.java
@@ -0,0 +1,18 @@
+// Copyright 2016 Pants project contributors (see CONTRIBUTORS.md).
+// Licensed under the Apache License, Version 2.0 (see LICENSE).
+package org.pantsbuild.testproject.parallel;
+
+import org.junit.Test;
+import org.pantsbuild.junit.annotations.TestSerial;
+
+/**
+ * See {@link AnnotatedSerialTest1}
+ */
+@TestSerial
+public class AnnotatedSerialTest2 {
+
+  @Test
+  public void astest2() throws Exception {
+    AnnotatedSerialTest1.awaitLatch("astest2");
+  }
+}
diff --git a/testprojects/tests/java/org/pantsbuild/testproject/parallel/BUILD b/testprojects/tests/java/org/pantsbuild/testproject/parallel/BUILD
new file mode 100644
index 000000000000..d6ccf5c9da72
--- /dev/null
+++ b/testprojects/tests/java/org/pantsbuild/testproject/parallel/BUILD
@@ -0,0 +1,53 @@
+# Copyright 2016 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+junit_tests(name='parallel',
+  sources=globs('ParallelTest*.java'),
+  dependencies=[
+    '3rdparty:junit',
+  ],
+  concurrency='parallel',
+  threads=2,
+)
+
+# This target runs the same tests as the one above, but doesn't have the concurrency settings.
+# Relies on the test.junit options being set as follows:
+#   --test-junit-default-parallel --test-junit-parallel-threads=2
+junit_tests(name='cmdline',
+  sources=globs('ParallelTest*.java'),
+  dependencies=[
+    '3rdparty:junit',
+  ],
+)
+
+# These tests are annotated with @TestParallel so should be able to run
+# in parallel even when --test-junit-default-concurrency=SERIAL is set.
+junit_tests(name='annotated-parallel',
+  sources=globs('AnnotatedParallelTest*.java'),
+  dependencies=[
+    '3rdparty:junit',
+    ':junit-runner-annotations'
+  ],
+  threads=2,
+)
+
+# Even though these tests are run with 'parallel' concurrency, they are annotated
+# with @TestSerial, so they should run serially, even when even when
+# --test-junit-default-concurrency={PARALLEL, PARALLEL_METHODS} is set
+# See: https://github.com/pantsbuild/pants/issues/3209
+junit_tests(name='annotated-serial',
+  sources=globs('AnnotatedSerialTest*.java'),
+  dependencies=[
+    '3rdparty:junit',
+    ':junit-runner-annotations'
+  ],
+  concurrency='parallel',
+  threads=2,
+)
+
+jar_library(
+  name='junit-runner-annotations',
+  jars=[
+    jar(org='org.pantsbuild', name='junit-runner-annotations', rev='0.0.11'),
+  ],
+)
diff --git a/testprojects/tests/java/org/pantsbuild/testproject/parallel/ParallelTest1.java b/testprojects/tests/java/org/pantsbuild/testproject/parallel/ParallelTest1.java
new file mode 100644
index 000000000000..9872ba167310
--- /dev/null
+++ b/testprojects/tests/java/org/pantsbuild/testproject/parallel/ParallelTest1.java
@@ -0,0 +1,41 @@
+// Copyright 2016 Pants project contributors (see CONTRIBUTORS.md).
+// Licensed under the Apache License, Version 2.0 (see LICENSE).
+package org.pantsbuild.testproject.parallel;
+
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import org.junit.Test;
+
+import static org.junit.Assert.assertTrue;
+
+/**
+ * This test is designed to exercise the test.junit runner --test-junit-default-parallel argument
+ * There is a similar test under tests/java/src/... to thest junit-runner standalone.
+ * 

+ * For all methods in ParallelTest1 and ParallelTest2
+ * to succeed, both test classes  must be running at the same time. Intended to test the flags
+ * 
+ * --test-junit-default-concurrency=PARALLEL --test-junit-parallel-threads=2
+ * 
+ * when running with just these two classes as specs.
+ * 

+ * Runs in on the order of 10 milliseconds locally, but it may take longer on a CI machine to spin
+ * up 2 threads, so it has a generous timeout set.
+ */
+public class ParallelTest1 {
+  private static final int NUM_CONCURRENT_TESTS = 2;
+  private static final int RETRY_TIMEOUT_MS = 3000;
+  private static CountDownLatch latch = new CountDownLatch(NUM_CONCURRENT_TESTS);
+
+  @Test
+  public void ptest1() throws Exception {
+    awaitLatch("ptest11");
+  }
+
+  static void awaitLatch(String methodName) throws Exception {
+    System.out.println("start " + methodName);
+    latch.countDown();
+    assertTrue(latch.await(RETRY_TIMEOUT_MS, TimeUnit.MILLISECONDS));
+    System.out.println("end " + methodName);
+  }
+}
diff --git a/testprojects/tests/java/org/pantsbuild/testproject/parallel/ParallelTest2.java b/testprojects/tests/java/org/pantsbuild/testproject/parallel/ParallelTest2.java
new file mode 100644
index 000000000000..0ab77cda60e0
--- /dev/null
+++ b/testprojects/tests/java/org/pantsbuild/testproject/parallel/ParallelTest2.java
@@ -0,0 +1,16 @@
+// Copyright 2016 Pants project contributors (see CONTRIBUTORS.md).
+// Licensed under the Apache License, Version 2.0 (see LICENSE).
+package org.pantsbuild.testproject.parallel;
+
+import org.junit.Test;
+
+/**
+ * See {@link ParallelTest1}
+ */
+public class ParallelTest2 {
+
+  @Test
+  public void ptest2() throws Exception {
+    ParallelTest1.awaitLatch("ptest2");
+  }
+}
diff --git a/testprojects/tests/java/org/pantsbuild/testproject/parallelmethods/BUILD b/testprojects/tests/java/org/pantsbuild/testproject/parallelmethods/BUILD
new file mode 100644
index 000000000000..3e3eff06a1a9
--- /dev/null
+++ b/testprojects/tests/java/org/pantsbuild/testproject/parallelmethods/BUILD
@@ -0,0 +1,21 @@
+# Copyright 2016 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+junit_tests(name='parallelmethods',
+  sources=globs('ParallelMethodsDefaultParallel*.java'),
+  dependencies=[
+    '3rdparty:junit',
+  ],
+  concurrency='parallel_methods',
+  threads=4,
+)
+
+# This target runs the same tests as the one above, but doesn't have the concurrency settings.
+# Relies on the test.junit options being set as follows:
+#   --test-junit-default-concurrency=PARALLEL_METHODS --test-junit-parallel-threads=4
+junit_tests(name='cmdline',
+  sources=globs('ParallelMethodsDefaultParallel*.java'),
+  dependencies=[
+    '3rdparty:junit',
+  ],
+)
diff --git a/testprojects/tests/java/org/pantsbuild/testproject/parallelmethods/ParallelMethodsDefaultParallelTest1.java b/testprojects/tests/java/org/pantsbuild/testproject/parallelmethods/ParallelMethodsDefaultParallelTest1.java
new file mode 100644
index 000000000000..c486fee4b19a
--- /dev/null
+++ b/testprojects/tests/java/org/pantsbuild/testproject/parallelmethods/ParallelMethodsDefaultParallelTest1.java
@@ -0,0 +1,48 @@
+// Copyright 2016 Pants project contributors (see CONTRIBUTORS.md).
+// Licensed under the Apache License, Version 2.0 (see LICENSE).
+package org.pantsbuild.testproject.parallelmethods;
+
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import org.junit.Test;
+
+import static org.junit.Assert.assertTrue;
+
+/**
+ * This test is designed to exercise the test.junit task argument:
+ * --test-junit-default-concurrency=PARALLEL_METHODS
+ * 

+ * There is a similar test under tests/java/ to test the junit-runner standalone.
+ * 

+ * For all methods in ParallelMethodsDefaultParallelTest1 and ParallelMethodsDefaultParallelTest2
+ * to succeed all of the test methods must be running at the same time. Intended to test the flags:
+ * 
+ * --test-junit-default-concurrency=PARALLEL_METHODS --test-junit-parallel-threads=4
+ * 
+ * when running with just these two classes as specs.
+ * 

+ * Runs in on the order of 10 milliseconds locally, but it may take longer on a CI machine to spin
+ * up 4 threads, so it has a generous timeout set.
+ */
+public class ParallelMethodsDefaultParallelTest1 {
+  private static final int NUM_CONCURRENT_TESTS = 4;
+  private static final int RETRY_TIMEOUT_MS = 3000;
+  private static CountDownLatch latch = new CountDownLatch(NUM_CONCURRENT_TESTS);
+
+  @Test
+  public void pmdptest11() throws Exception {
+    awaitLatch("pmdptest11");
+  }
+
+  @Test
+  public void pmdptest12() throws Exception {
+    awaitLatch("pmdptest12");
+  }
+
+  static void awaitLatch(String methodName) throws Exception {
+    System.out.println("start " + methodName);
+    latch.countDown();
+    assertTrue(latch.await(RETRY_TIMEOUT_MS, TimeUnit.MILLISECONDS));
+    System.out.println("end " + methodName);
+  }
+}
diff --git a/testprojects/tests/java/org/pantsbuild/testproject/parallelmethods/ParallelMethodsDefaultParallelTest2.java b/testprojects/tests/java/org/pantsbuild/testproject/parallelmethods/ParallelMethodsDefaultParallelTest2.java
new file mode 100644
index 000000000000..3f9cc26e64a7
--- /dev/null
+++ b/testprojects/tests/java/org/pantsbuild/testproject/parallelmethods/ParallelMethodsDefaultParallelTest2.java
@@ -0,0 +1,21 @@
+// Copyright 2016 Pants project contributors (see CONTRIBUTORS.md).
+// Licensed under the Apache License, Version 2.0 (see LICENSE).
+package org.pantsbuild.testproject.parallelmethods;
+
+import org.junit.Test;
+
+/**
+ * See {@link ParallelMethodsDefaultParallelTest1}
+ */
+public class ParallelMethodsDefaultParallelTest2 {
+
+  @Test
+  public void pmdptest21() throws Exception {
+    ParallelMethodsDefaultParallelTest1.awaitLatch("pmdptest21");
+  }
+
+  @Test
+  public void pmdptest22() throws Exception {
+    ParallelMethodsDefaultParallelTest1.awaitLatch("pmdptest22");
+  }
+}
diff --git a/tests/java/org/pantsbuild/tools/junit/impl/ConsoleRunnerTest.java b/tests/java/org/pantsbuild/tools/junit/impl/ConsoleRunnerTest.java
index 880eaa373e22..48eb7931dde2 100644
--- a/tests/java/org/pantsbuild/tools/junit/impl/ConsoleRunnerTest.java
+++ b/tests/java/org/pantsbuild/tools/junit/impl/ConsoleRunnerTest.java
@@ -146,6 +146,27 @@ public void testOutputDir() throws Exception {
     assertThat(output, CoreMatchers.containsString("end test42"));
   }
 
+  @Test
+  public void testParallelAnnotation() throws Exception {
+    ConsoleRunnerImpl.main(asArgsArray(
+      "AnnotatedParallelTest1 AnnotatedParallelTest2 -parallel-threads 2"));
+    assertEquals("aptest1 aptest2", TestRegistry.getCalledTests());
+  }
+
+  @Test
+  public void testSerialAnnotation() throws Exception {
+    ConsoleRunnerImpl.main(asArgsArray(
+        "AnnotatedSerialTest1 AnnotatedSerialTest2 -default-parallel -parallel-threads 2"));
+    assertEquals("astest1 astest2", TestRegistry.getCalledTests());
+  }
+
+  @Test
+  public void testParallelDefaultParallel() throws Exception {
+    ConsoleRunnerImpl.main(asArgsArray(
+        "ParallelTest1 ParallelTest2 -parallel-threads 2 -default-parallel"));
+    assertEquals("ptest1 ptest2", TestRegistry.getCalledTests());
+  }
+
   @Test
   public void testParallelMethodsDefaultParallel() throws Exception {
     ConsoleRunnerImpl.main(asArgsArray(
diff --git a/tests/java/org/pantsbuild/tools/junit/lib/AnnotatedParallelTest1.java b/tests/java/org/pantsbuild/tools/junit/lib/AnnotatedParallelTest1.java
new file mode 100644
index 000000000000..03852b750be3
--- /dev/null
+++ b/tests/java/org/pantsbuild/tools/junit/lib/AnnotatedParallelTest1.java
@@ -0,0 +1,47 @@
+// Copyright 2016 Pants project contributors (see CONTRIBUTORS.md).
+// Licensed under the Apache License, Version 2.0 (see LICENSE).
+package org.pantsbuild.tools.junit.lib;
+
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import org.junit.Test;
+import org.pantsbuild.junit.annotations.TestParallel;
+
+import static org.junit.Assert.assertTrue;
+
+/**
+ * This test is intentionally under a java_library() BUILD target so it will not be run
+ * on its own. It is run by the ConsoleRunnerTest suite to test ConsoleRunnerImpl.
+ * 

+ * Exercises the TestParallel annotation.
+ * 

+ * For all methods in AnnotatedParallelTest1 and AnnotatedParallelTest2
+ * to succeed, both test classes  must be running at the same time with the flag:
+ * 
+ *  -parallel-threads 2
+ * 
+ * when running with just these two classes as specs.
+ * 

+ * Runs in on the order of 10 milliseconds locally, but it may take longer on a CI machine to spin
+ * up 2 threads, so it has a generous timeout set.
+ * 

+ */
+@TestParallel
+public class AnnotatedParallelTest1 {
+  private static final int NUM_CONCURRENT_TESTS = 2;
+  private static final int RETRY_TIMEOUT_MS = 3000;
+  private static CountDownLatch latch = new CountDownLatch(NUM_CONCURRENT_TESTS);
+
+  @Test
+  public void aptest1() throws Exception {
+    awaitLatch("aptest1");
+  }
+
+  static void awaitLatch(String methodName) throws Exception {
+    TestRegistry.registerTestCall(methodName);
+    System.out.println("start " + methodName);
+    latch.countDown();
+    assertTrue(latch.await(RETRY_TIMEOUT_MS, TimeUnit.MILLISECONDS));
+    System.out.println("end " + methodName);
+  }
+}
diff --git a/tests/java/org/pantsbuild/tools/junit/lib/AnnotatedParallelTest2.java b/tests/java/org/pantsbuild/tools/junit/lib/AnnotatedParallelTest2.java
new file mode 100644
index 000000000000..8dca2c274351
--- /dev/null
+++ b/tests/java/org/pantsbuild/tools/junit/lib/AnnotatedParallelTest2.java
@@ -0,0 +1,18 @@
+// Copyright 2016 Pants project contributors (see CONTRIBUTORS.md).
+// Licensed under the Apache License, Version 2.0 (see LICENSE).
+package org.pantsbuild.tools.junit.lib;
+
+import org.junit.Test;
+import org.pantsbuild.junit.annotations.TestParallel;
+
+/**
+ * See {@link AnnotatedParallelTest1}
+ */
+@TestParallel
+public class AnnotatedParallelTest2 {
+
+  @Test
+  public void aptest2() throws Exception {
+    AnnotatedParallelTest1.awaitLatch("aptest2");
+  }
+}
diff --git a/tests/java/org/pantsbuild/tools/junit/lib/AnnotatedSerialTest1.java b/tests/java/org/pantsbuild/tools/junit/lib/AnnotatedSerialTest1.java
new file mode 100644
index 000000000000..cf6a62177e25
--- /dev/null
+++ b/tests/java/org/pantsbuild/tools/junit/lib/AnnotatedSerialTest1.java
@@ -0,0 +1,49 @@
+// Copyright 2016 Pants project contributors (see CONTRIBUTORS.md).
+// Licensed under the Apache License, Version 2.0 (see LICENSE).
+package org.pantsbuild.tools.junit.lib;
+
+import java.util.concurrent.atomic.AtomicBoolean;
+import org.junit.Test;
+import org.pantsbuild.junit.annotations.TestSerial;
+
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * This test is intentionally under a java_library() BUILD target so it will not be run
+ * on its own. It is run by the ConsoleRunnerTest suite to test ConsoleRunnerImpl.
+ * 

+ * Exercises the @TestSerial annotation.
+ * 

+ * These tests are intended to show that the two classes will be run serially, even if
+ * parallel test running is on.
+ * To properly exercise this function, both test classes must be running at the same time with
+ * the flags:
+ * 
+ *  -default-parallel -parallel-threads 2
+ * 
+ * when running with just these two classes as specs.
+ * 

+ * Uses a timeout, so its not completely deterministic, but it gives 3 seconds to allow any
+ * concurrency to take place.
+ * 

+ */
+@TestSerial
+public class AnnotatedSerialTest1 {
+  private static final int WAIT_TIMEOUT_MS = 3000;
+  private static AtomicBoolean waiting = new AtomicBoolean(false);
+
+  @Test
+  public void astest1() throws Exception {
+    awaitLatch("astest1");
+  }
+
+  static void awaitLatch(String methodName) throws Exception {
+    TestRegistry.registerTestCall(methodName);
+    System.out.println("start " + methodName);
+    assertFalse(waiting.getAndSet(true));
+    Thread.sleep(WAIT_TIMEOUT_MS);
+    assertTrue(waiting.getAndSet(false));
+    System.out.println("end " + methodName);
+  }
+}
diff --git a/tests/java/org/pantsbuild/tools/junit/lib/AnnotatedSerialTest2.java b/tests/java/org/pantsbuild/tools/junit/lib/AnnotatedSerialTest2.java
new file mode 100644
index 000000000000..227c14ec317d
--- /dev/null
+++ b/tests/java/org/pantsbuild/tools/junit/lib/AnnotatedSerialTest2.java
@@ -0,0 +1,18 @@
+// Copyright 2016 Pants project contributors (see CONTRIBUTORS.md).
+// Licensed under the Apache License, Version 2.0 (see LICENSE).
+package org.pantsbuild.tools.junit.lib;
+
+import org.junit.Test;
+import org.pantsbuild.junit.annotations.TestSerial;
+
+/**
+ * See {@link AnnotatedSerialTest1}
+ */
+@TestSerial
+public class AnnotatedSerialTest2 {
+
+  @Test
+  public void astest2() throws Exception {
+    AnnotatedSerialTest1.awaitLatch("astest2");
+  }
+}
diff --git a/tests/java/org/pantsbuild/tools/junit/lib/ParallelMethodsDefaultParallelTest1.java b/tests/java/org/pantsbuild/tools/junit/lib/ParallelMethodsDefaultParallelTest1.java
index 0cb06aaebf5f..2adbdf94046b 100644
--- a/tests/java/org/pantsbuild/tools/junit/lib/ParallelMethodsDefaultParallelTest1.java
+++ b/tests/java/org/pantsbuild/tools/junit/lib/ParallelMethodsDefaultParallelTest1.java
@@ -1,3 +1,5 @@
+// Copyright 2016 Pants project contributors (see CONTRIBUTORS.md).
+// Licensed under the Apache License, Version 2.0 (see LICENSE).
 package org.pantsbuild.tools.junit.lib;
 
 import java.util.concurrent.CountDownLatch;
@@ -9,8 +11,8 @@
 /**
  * This test is intentionally under a java_library() BUILD target so it will not be run
  * on its own. It is run by the ConsoleRunnerTest suite to test ConsoleRunnerImpl.
- *
- * This test is designed to exercise the junit runner -parallel-methods argument
+ *

+ * Exercises the junit runner -parallel-methods argument.
  * 

  * For all methods in ParallelMethodsDefaultParallelTest1 and ParallelMethodsDefaultParallelTest2
  * to succeed all of the test methods must be running at the same time. Intended to test the flags
@@ -25,7 +27,7 @@
  */
 public class ParallelMethodsDefaultParallelTest1 {
   private static final int NUM_CONCURRENT_TESTS = 4;
-  private static final int RETRY_TIMEOUT_MS = 10000;
+  private static final int RETRY_TIMEOUT_MS = 3000;
   private static CountDownLatch latch = new CountDownLatch(NUM_CONCURRENT_TESTS);
 
   @Test
diff --git a/tests/java/org/pantsbuild/tools/junit/lib/ParallelMethodsDefaultParallelTest2.java b/tests/java/org/pantsbuild/tools/junit/lib/ParallelMethodsDefaultParallelTest2.java
index c55026807716..065d438ce5e8 100644
--- a/tests/java/org/pantsbuild/tools/junit/lib/ParallelMethodsDefaultParallelTest2.java
+++ b/tests/java/org/pantsbuild/tools/junit/lib/ParallelMethodsDefaultParallelTest2.java
@@ -1,3 +1,5 @@
+// Copyright 2016 Pants project contributors (see CONTRIBUTORS.md).
+// Licensed under the Apache License, Version 2.0 (see LICENSE).
 package org.pantsbuild.tools.junit.lib;
 
 import org.junit.Test;
diff --git a/tests/java/org/pantsbuild/tools/junit/lib/ParallelTest1.java b/tests/java/org/pantsbuild/tools/junit/lib/ParallelTest1.java
new file mode 100644
index 000000000000..78d5e7d7a706
--- /dev/null
+++ b/tests/java/org/pantsbuild/tools/junit/lib/ParallelTest1.java
@@ -0,0 +1,43 @@
+// Copyright 2016 Pants project contributors (see CONTRIBUTORS.md).
+// Licensed under the Apache License, Version 2.0 (see LICENSE).
+package org.pantsbuild.tools.junit.lib;
+
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import org.junit.Test;
+
+import static org.junit.Assert.assertTrue;
+
+/**
+ * This test is intentionally under a java_library() BUILD target so it will not be run
+ * on its own. It is run by the ConsoleRunnerTest suite to test ConsoleRunnerImpl.
+ * 

+ * For all methods in ParallelTest1 and ParallelTest2
+ * to succeed, both test classes  must be running at the same time. Intended to test the flags
+ * 
+ * -default-parallel -parallel-threads 2
+ * 
+ * when running with just these two classes as specs.
+ * 

+ * Runs in on the order of 10 milliseconds locally, but it may take longer on a CI machine to spin
+ * up 2 threads, so it has a generous timeout set.
+ * 

+ */
+public class ParallelTest1 {
+  private static final int NUM_CONCURRENT_TESTS = 2;
+  private static final int RETRY_TIMEOUT_MS = 3000;
+  private static CountDownLatch latch = new CountDownLatch(NUM_CONCURRENT_TESTS);
+
+  @Test
+  public void ptest1() throws Exception {
+    awaitLatch("ptest1");
+  }
+
+  static void awaitLatch(String methodName) throws Exception {
+    TestRegistry.registerTestCall(methodName);
+    System.out.println("start " + methodName);
+    latch.countDown();
+    assertTrue(latch.await(RETRY_TIMEOUT_MS, TimeUnit.MILLISECONDS));
+    System.out.println("end " + methodName);
+  }
+}
diff --git a/tests/java/org/pantsbuild/tools/junit/lib/ParallelTest2.java b/tests/java/org/pantsbuild/tools/junit/lib/ParallelTest2.java
new file mode 100644
index 000000000000..fda5220d8fd2
--- /dev/null
+++ b/tests/java/org/pantsbuild/tools/junit/lib/ParallelTest2.java
@@ -0,0 +1,16 @@
+// Copyright 2016 Pants project contributors (see CONTRIBUTORS.md).
+// Licensed under the Apache License, Version 2.0 (see LICENSE).
+package org.pantsbuild.tools.junit.lib;
+
+import org.junit.Test;
+
+/**
+ * See {@link ParallelTest1}
+ */
+public class ParallelTest2 {
+
+  @Test
+  public void ptest2() throws Exception {
+    ParallelTest1.awaitLatch("ptest2");
+  }
+}
diff --git a/tests/python/pants_test/backend/jvm/targets/BUILD b/tests/python/pants_test/backend/jvm/targets/BUILD
index bfb33c240577..8d357c1737b1 100644
--- a/tests/python/pants_test/backend/jvm/targets/BUILD
+++ b/tests/python/pants_test/backend/jvm/targets/BUILD
@@ -29,6 +29,16 @@ python_tests(
   ]
 )
 
+python_tests(
+  name='java_tests',
+  sources=['test_java_tests.py'],
+  dependencies=[
+    'src/python/pants/backend/jvm/targets:jvm',
+    'src/python/pants/build_graph',
+    'tests/python/pants_test:base_test',
+  ],
+)
+
 python_tests(
   name='jvm_app',
   sources=['test_jvm_app.py'],
diff --git a/tests/python/pants_test/backend/jvm/targets/test_java_tests.py b/tests/python/pants_test/backend/jvm/targets/test_java_tests.py
new file mode 100644
index 000000000000..66526ad871de
--- /dev/null
+++ b/tests/python/pants_test/backend/jvm/targets/test_java_tests.py
@@ -0,0 +1,49 @@
+# coding=utf-8
+# Copyright 2016 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+from __future__ import (absolute_import, division, generators, nested_scopes, print_function,
+                        unicode_literals, with_statement)
+
+from pants.backend.jvm.targets.java_tests import JavaTests
+from pants.base.exceptions import TargetDefinitionException
+from pants.build_graph.target import Target
+from pants_test.base_test import BaseTest
+
+
+class JavaTestsTest(BaseTest):
+
+  def test_validation(self):
+    target = self.make_target('//:mybird', Target)
+    # A plain invocation with no frills
+    test1 = self.make_target('//:test1', JavaTests, sources=["Test.java"], dependencies=[target])
+    self.assertIsNone(test1.cwd)
+    self.assertIsNone(test1.concurrency)
+    self.assertIsNone(test1.threads)
+    self.assertIsNone(test1.timeout)
+
+    # cwd parameter
+    testcwd = self.make_target('//:testcwd1', JavaTests, sources=["Test.java"], concurrency='serial', cwd='/foo/bar')
+    self.assertEquals('/foo/bar', testcwd.cwd)
+
+    # concurrency parameter
+    tc1 = self.make_target('//:testconcurrency1', JavaTests, sources=["Test.java"], concurrency='serial')
+    self.assertEquals('serial', tc1.concurrency)
+    tc2 = self.make_target('//:testconcurrency2', JavaTests, sources=["Test.java"], concurrency='parallel')
+    self.assertEquals('parallel', tc2.concurrency)
+    tc3 = self.make_target('//:testconcurrency3', JavaTests, sources=["Test.java"], concurrency='parallel_methods')
+    self.assertEquals('parallel_methods', tc3.concurrency)
+    with self.assertRaisesRegexp(TargetDefinitionException, r'concurrency'):
+      self.make_target('//:testconcurrency4', JavaTests, sources=["Test.java"], concurrency='nonsense')
+
+    # threads parameter
+    tt1 = self.make_target('//:testthreads1', JavaTests, sources=["Test.java"], threads=99)
+    self.assertEquals(99, tt1.threads)
+    tt2 = self.make_target('//:testthreads2', JavaTests, sources=["Test.java"], threads="123")
+    self.assertEquals(123, tt2.threads)
+    with self.assertRaisesRegexp(TargetDefinitionException, r'threads'):
+      self.make_target('//:testthreads3', JavaTests, sources=["Test.java"], threads="abc")
+
+    # timeout parameter
+    timeout = self.make_target('//:testtimeout1', JavaTests, sources=["Test.java"], timeout=999)
+    self.assertEquals(999, timeout.timeout)
diff --git a/tests/python/pants_test/backend/jvm/tasks/BUILD b/tests/python/pants_test/backend/jvm/tasks/BUILD
index 0222348764ac..d8d656ac4fa7 100644
--- a/tests/python/pants_test/backend/jvm/tasks/BUILD
+++ b/tests/python/pants_test/backend/jvm/tasks/BUILD
@@ -359,6 +359,18 @@ python_tests(
     'tests/python/pants_test:int-test',
   ],
   tags = {'integration'},
+  timeout = 240,
+)
+
+python_tests(
+  name = 'junit_tests_concurrency_integration',
+  sources = ['test_junit_tests_concurrency_integration.py'],
+  dependencies = [
+    'src/python/pants/util:contextutil',
+    'tests/python/pants_test:int-test',
+  ],
+  tags = {'integration'},
+  timeout = 240,
 )
 
 python_library(
diff --git a/tests/python/pants_test/backend/jvm/tasks/test_junit_tests_concurrency_integration.py b/tests/python/pants_test/backend/jvm/tasks/test_junit_tests_concurrency_integration.py
new file mode 100644
index 000000000000..0c83c9134b2e
--- /dev/null
+++ b/tests/python/pants_test/backend/jvm/tasks/test_junit_tests_concurrency_integration.py
@@ -0,0 +1,126 @@
+# coding=utf-8
+# Copyright 2016 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+from __future__ import (absolute_import, division, generators, nested_scopes, print_function,
+                        unicode_literals, with_statement)
+
+import pytest
+
+from pants_test.pants_run_integration_test import PantsRunIntegrationTest
+
+
+class JunitTestsConcurrencyIntegrationTest(PantsRunIntegrationTest):
+
+  def test_parallel_target(self):
+    """Checks the 'concurrency=parallel' setting in the junit_tests() target"""
+    with self.temporary_workdir() as workdir:
+      pants_run = self.run_pants_with_workdir([
+        'test',
+        'testprojects/tests/java/org/pantsbuild/testproject/parallel'
+      ], workdir)
+      self.assert_success(pants_run)
+      self.assertIn("OK (2 tests)", pants_run.stdout_data)
+
+  def test_parallel_cmdline(self):
+    """Checks the --test-junit-default-concurrency=PARALLEL option"""
+    with self.temporary_workdir() as workdir:
+      pants_run = self.run_pants_with_workdir([
+        'test',
+        '--test-junit-default-concurrency=PARALLEL',
+        '--test-junit-parallel-threads=2',
+        'testprojects/tests/java/org/pantsbuild/testproject/parallel:cmdline'
+      ], workdir)
+      self.assert_success(pants_run)
+      self.assertIn("OK (2 tests)", pants_run.stdout_data)
+
+  def test_concurrency_serial_default(self):
+    """Checks the --test-junit-default-concurrency=SERIAL option"""
+    with self.temporary_workdir() as workdir:
+      # NB(zundel): the timeout for each test in ParallelMethodsDefaultParallel tests is
+      # currently set to 3 seconds making this test take about 2 seconds to run due
+      # to (1 timeout failure)
+      pants_run = self.run_pants_with_workdir([
+        'test',
+        '--test-junit-default-concurrency=SERIAL',
+        '--test-junit-parallel-threads=2',
+        'testprojects/tests/java/org/pantsbuild/testproject/parallel:cmdline'
+      ], workdir)
+      self.assert_failure(pants_run)
+      # Its not deterministic which test will fail, but one of them should timeout
+      self.assertIn("Tests run: 2,  Failures: 1", pants_run.stdout_data)
+
+  def test_parallel_annotated_test_parallel(self):
+    """Checks the @TestParallel annotation."""
+    with self.temporary_workdir() as workdir:
+      pants_run = self.run_pants_with_workdir([
+        'test',
+        '--test-junit-default-concurrency=SERIAL',
+        'testprojects/tests/java/org/pantsbuild/testproject/parallel:annotated-parallel'
+      ], workdir)
+      self.assert_success(pants_run)
+      self.assertIn("OK (2 tests)", pants_run.stdout_data)
+
+  def test_parallel_annotated_test_serial(self):
+    """Checks the @TestSerial annotation."""
+    with self.temporary_workdir() as workdir:
+      pants_run = self.run_pants_with_workdir([
+        'test',
+        '--test-junit-default-concurrency=PARALLEL',
+        '--test-junit-parallel-threads=2',
+        'testprojects/tests/java/org/pantsbuild/testproject/parallel:annotated-serial'
+      ], workdir)
+      self.assert_success(pants_run)
+      self.assertIn("OK (2 tests)", pants_run.stdout_data)
+
+  @pytest.mark.xfail
+  def test_concurrency_annotated_test_serial_parallel_methods(self):
+    """Checks the @TestSerial annotation with --test-junit-default-concurrency=PARALLEL_METHODS."""
+    with self.temporary_workdir() as workdir:
+      pants_run = self.run_pants_with_workdir([
+        'test',
+        '--test-junit-default-concurrency=PARALLEL_METHODS',
+        '--test-junit-parallel-threads=2',
+        'testprojects/tests/java/org/pantsbuild/testproject/parallel:annotated-serial'
+      ], workdir)
+      self.assert_success(pants_run)
+      self.assertIn("OK (2 tests)", pants_run.stdout_data)
+
+  def test_parallel_methods(self):
+    """Checks the concurency='parallel_methods' setting."""
+    with self.temporary_workdir() as workdir:
+      pants_run = self.run_pants_with_workdir([
+        'test',
+        '--test-junit-default-concurrency=SERIAL',
+        'testprojects/tests/java/org/pantsbuild/testproject/parallelmethods'
+      ], workdir)
+      self.assert_success(pants_run)
+      self.assertIn("OK (4 tests)", pants_run.stdout_data)
+
+  def test_parallel_methods_cmdline(self):
+    """Checks the --test-junit-parallel-methods setting."""
+    with self.temporary_workdir() as workdir:
+      pants_run = self.run_pants_with_workdir([
+         'test',
+        '--test-junit-default-concurrency=PARALLEL_METHODS',
+        '--test-junit-parallel-threads=4',
+        'testprojects/tests/java/org/pantsbuild/testproject/parallelmethods:cmdline'
+        ], workdir)
+      self.assert_success(pants_run)
+      self.assertIn("OK (4 tests)", pants_run.stdout_data)
+
+  def test_parallel_methods_serial_default(self):
+    """Checks the --no-test-junit-default-parallel setting."""
+    with self.temporary_workdir() as workdir:
+      # NB(zundel): the timeout for each test in ParallelMethodsDefaultParallel tests is
+      # currently set to 3 seconds making this test take about 9 seconds to run due
+      # to (3 timeout failures)
+      pants_run = self.run_pants_with_workdir([
+         'test',
+        '--test-junit-default-concurrency=SERIAL',
+        '--test-junit-parallel-threads=4',
+        'testprojects/tests/java/org/pantsbuild/testproject/parallelmethods:cmdline'
+        ], workdir)
+      self.assert_failure(pants_run)
+      # Its not deterministic which test will fail, but 3/4 of them should timeout
+      self.assertIn("Tests run: 4,  Failures: 3", pants_run.stdout_data)
diff --git a/tests/python/pants_test/projects/test_testprojects_integration.py b/tests/python/pants_test/projects/test_testprojects_integration.py
index c284569bd50a..14d8394d3043 100644
--- a/tests/python/pants_test/projects/test_testprojects_integration.py
+++ b/tests/python/pants_test/projects/test_testprojects_integration.py
@@ -53,6 +53,7 @@ def tests_testprojects(self):
       # These don't pass without special config.
       'testprojects/tests/java/org/pantsbuild/testproject/depman:new-tests',
       'testprojects/tests/java/org/pantsbuild/testproject/depman:old-tests',
+      'testprojects/tests/java/org/pantsbuild/testproject/parallel.*',
     ]
 
     # May not succeed without java8 installed
diff --git a/tests/python/pants_test/util/BUILD b/tests/python/pants_test/util/BUILD
index 848a4750f603..a8cd3384e7cf 100644
--- a/tests/python/pants_test/util/BUILD
+++ b/tests/python/pants_test/util/BUILD
@@ -1,6 +1,15 @@
 # Copyright 2014 Pants project contributors (see CONTRIBUTORS.md).
 # Licensed under the Apache License, Version 2.0 (see LICENSE).
 
+python_tests(
+  name = 'argutil',
+  sources = ['test_argutil.py'],
+  coverage = ['pants.util.argutil'],
+  dependencies = [
+    'src/python/pants/util:argutil',
+  ],
+)
+
 python_tests(
   name = 'contextutil',
   sources = ['test_contextutil.py'],
diff --git a/tests/python/pants_test/util/test_argutil.py b/tests/python/pants_test/util/test_argutil.py
new file mode 100644
index 000000000000..6070f2d62d1a
--- /dev/null
+++ b/tests/python/pants_test/util/test_argutil.py
@@ -0,0 +1,37 @@
+# coding=utf-8
+# Copyright 2016 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+from __future__ import (absolute_import, division, generators, nested_scopes, print_function,
+                        unicode_literals, with_statement)
+
+import unittest
+
+from pants.util.argutil import ensure_arg, remove_arg
+
+
+class ArgutilTest(unittest.TestCase):
+
+  def test_ensure_arg(self):
+    self.assertEquals(['foo'], ensure_arg([], 'foo'))
+    self.assertEquals(['foo'], ensure_arg(['foo'], 'foo'))
+    self.assertEquals(['bar', 'foo'], ensure_arg(['bar'], 'foo'))
+    self.assertEquals(['bar', 'foo'], ensure_arg(['bar', 'foo'], 'foo'))
+
+    self.assertEquals(['foo', 'baz'], ensure_arg([], 'foo', param='baz'))
+    self.assertEquals(['qux', 'foo', 'baz'], ensure_arg(['qux', 'foo', 'bar'], 'foo', param='baz'))
+    self.assertEquals(['foo', 'baz'], ensure_arg(['foo', 'bar'], 'foo', param='baz'))
+    self.assertEquals(['qux', 'foo', 'baz', 'foobar'], ensure_arg(['qux', 'foo', 'bar', 'foobar'], 'foo', param='baz'))
+
+  def test_remove_arg(self):
+    self.assertEquals([], remove_arg([], 'foo'))
+    self.assertEquals([], remove_arg(['foo'], 'foo'))
+    self.assertEquals(['bar'], remove_arg(['foo', 'bar'], 'foo'))
+    self.assertEquals(['bar'], remove_arg(['bar', 'foo'], 'foo'))
+    self.assertEquals(['bar', 'baz'], remove_arg(['bar', 'foo', 'baz'], 'foo'))
+
+    self.assertEquals([], remove_arg([], 'foo', has_param=True))
+    self.assertEquals([], remove_arg(['foo', 'bar'], 'foo', has_param=True))
+    self.assertEquals(['baz'], remove_arg(['baz', 'foo', 'bar'], 'foo', has_param=True))
+    self.assertEquals(['baz'], remove_arg(['foo', 'bar', 'baz'], 'foo', has_param=True))
+    self.assertEquals(['qux', 'foobar'], remove_arg(['qux', 'foo', 'bar', 'foobar'], 'foo', has_param='baz'))