Add tests for async observer executor shutdown (#118)

<!-- PR title should start with '[fix]', '[improvement]' or '[break]' if this PR would cause a patch, minor or major SemVer bump. Omit the prefix if this PR doesn't warrant a standalone release. -->

## Before this PR
<!-- Describe the problem you encountered with the current state of the world (or link to an issue) and why it's important to fix now. -->
There was no unit test coverage for cases where the async observer executor was shutdown before spans completed.

## After this PR
<!-- Describe at a high-level why this approach is better. -->

Additional test coverage for #66 & #67 for when the async observer executor is shutdown.

<!-- Reference any existing GitHub issues, e.g. 'fixes #000' or 'relevant to #000' -->

Author: schlosna

tracing/src/test/java/com/palantir/tracing/AsyncSlf4jSpanObserverTest.java

@@ -30,13 +30,16 @@
 import ch.qos.logback.classic.spi.ILoggingEvent;
 import ch.qos.logback.core.Appender;
 import com.google.common.collect.ImmutableMap;
+import com.google.common.util.concurrent.MoreExecutors;
+import com.palantir.tracing.api.OpenSpan;
 import com.palantir.tracing.api.Span;
 import com.palantir.tracing.api.SpanType;
 import java.net.Inet4Address;
 import java.net.Inet6Address;
 import java.net.InetAddress;
 import java.nio.charset.StandardCharsets;
 import java.util.Map;
+import java.util.concurrent.ExecutorService;
 import java.util.stream.Collectors;
 import org.jmock.lib.concurrent.DeterministicScheduler;
 import org.junit.After;
@@ -56,6 +59,7 @@
 
 public final class AsyncSlf4jSpanObserverTest {
 
+    private static final String TEST_OBSERVER = "test";
     private static final AsyncSlf4jSpanObserver.ZipkinCompatEndpoint DUMMY_ENDPOINT =
             ImmutableZipkinCompatEndpoint.builder()
                     .serviceName("")
@@ -80,17 +84,19 @@ public void before() {
 
         originalLevel = logger.getLevel();
         logger.setLevel(Level.TRACE);
+        Tracer.unsubscribe(TEST_OBSERVER);
     }
 
     @After
     public void after() {
         logger.setLevel(originalLevel);
+        Tracer.unsubscribe(TEST_OBSERVER);
     }
 
     @Test
     public void testJsonFormatToLog() throws Exception {
         DeterministicScheduler executor = new DeterministicScheduler();
-        Tracer.subscribe("", AsyncSlf4jSpanObserver.of(
+        Tracer.subscribe(TEST_OBSERVER, AsyncSlf4jSpanObserver.of(
                 "serviceName", Inet4Address.getLoopbackAddress(), logger, executor));
         Tracer.startSpan("operation");
         Span span = Tracer.completeSpan().get();
@@ -106,13 +112,12 @@ public void testJsonFormatToLog() throws Exception {
         assertThat(event.getValue().getFormattedMessage())
                 .isEqualTo(AsyncSlf4jSpanObserver.ZipkinCompatSpan.fromSpan(span, expectedEndpoint).toJson());
         verifyNoMoreInteractions(appender);
-        Tracer.unsubscribe("");
     }
 
     @Test
     public void testDefaultConstructorDeterminesIpAddress() throws Exception {
         DeterministicScheduler executor = new DeterministicScheduler();
-        Tracer.subscribe("", AsyncSlf4jSpanObserver.of("serviceName", executor));
+        Tracer.subscribe(TEST_OBSERVER, AsyncSlf4jSpanObserver.of("serviceName", executor));
         Tracer.startSpan("operation");
         Span span = Tracer.completeSpan().get();
 
@@ -131,7 +136,6 @@ public void testDefaultConstructorDeterminesIpAddress() throws Exception {
 
         assertThat(event.getValue().getFormattedMessage())
                 .isEqualTo(AsyncSlf4jSpanObserver.ZipkinCompatSpan.fromSpan(span, expectedEndpoint.build()).toJson());
-        Tracer.unsubscribe("");
     }
 
     @Test
@@ -219,6 +223,38 @@ public void testSpanMetadataConvertsToZipkinBinaryAnnotations() {
         assertThat(annotationMap).isEqualTo(spanMetadata);
     }
 
+    @Test
+    public void testCompleteSpanAfterExecutorShutdown() {
+        ExecutorService executor = MoreExecutors.newDirectExecutorService();
+        Tracer.subscribe(TEST_OBSERVER, AsyncSlf4jSpanObserver.of(
+                "serviceName", Inet4Address.getLoopbackAddress(), logger, executor));
+        OpenSpan span1 = Tracer.startSpan("operation");
+        assertThat(Tracer.completeSpan()).isPresent().get()
+                .extracting("spanId", "operation")
+                .contains(span1.getSpanId(), span1.getOperation());
+
+        OpenSpan span2 = Tracer.startSpan("operation");
+        executor.shutdown();
+        assertThat(Tracer.completeSpan()).isPresent().get()
+                .extracting("spanId", "operation")
+                .contains(span2.getSpanId(), span2.getOperation());
+    }
+
+    @Test
+    public void testFastCompleteSpanAfterExecutorShutdown() {
+        ExecutorService executor = MoreExecutors.newDirectExecutorService();
+        Tracer.subscribe(TEST_OBSERVER, AsyncSlf4jSpanObserver.of(
+                "serviceName", Inet4Address.getLoopbackAddress(), logger, executor));
+        OpenSpan span1 = Tracer.startSpan("operation");
+        assertThat(Tracer.completeSpan()).isPresent().get()
+                .extracting("spanId", "operation")
+                .contains(span1.getSpanId(), span1.getOperation());
+
+        Tracer.startSpan("operation");
+        executor.shutdown();
+        Tracer.fastCompleteSpan();
+    }
+
     private static AsyncSlf4jSpanObserver.ZipkinCompatSpan zipkinSpan(long start, long duration, SpanType type) {
         Span span = Span.builder()
                 .traceId("")