8357146: ForkJoinPool:schedule(*) does not throw RejectedExecutionException when pool is shutdown

Reviewed-by: alanb

Author: Doug Lea

src/java.base/share/classes/java/util/concurrent/ForkJoinPool.java

@@ -3452,22 +3452,30 @@ private DelayScheduler startDelayScheduler() {
             String name = poolName + "-delayScheduler";
             if (workerNamePrefix == null)
                 asyncCommonPool();  // override common parallelism zero
-            lockRunState();
+            long isShutdown = lockRunState() & SHUTDOWN;
             try {
-                if ((ds = delayScheduler) == null) {
+                if (isShutdown == 0L && (ds = delayScheduler) == null) {
                     ds = delayScheduler = new DelayScheduler(this, name);
                     start = true;
                 }
             } finally {
                 unlockRunState();
             }
             if (start) { // start outside of lock
-                // exceptions on start passed to (external) callers
                 SharedThreadContainer ctr;
-                if ((ctr = container) != null)
-                    ctr.start(ds);
-                else
-                    ds.start();
+                try {
+                    if ((ctr = container) != null)
+                        ctr.start(ds);
+                    else
+                        ds.start();
+                } catch (RuntimeException | Error ex) { // back out
+                    lockRunState();
+                    ds = delayScheduler = null;
+                    unlockRunState();
+                    tryTerminate(false, false);
+                    if (ex instanceof Error)
+                        throw ex;
+                }
             }
         }
         return ds;

test/jdk/java/util/concurrent/tck/ForkJoinPool20Test.java

@@ -644,4 +644,22 @@ public Boolean realCall() {
         }
     }
 
+    /**
+     * schedule throws RejectedExecutionException if shutdown before
+     * first delayed task is submitted
+     */
+    public void testInitialScheduleAfterShutdown() throws InterruptedException {
+        Runnable r = new NoOpRunnable();
+        boolean rje = false;
+        try (final ForkJoinPool p = new ForkJoinPool(1)) {
+            p.shutdown();
+            assertTrue(p.awaitTermination(LONG_DELAY_MS, MILLISECONDS));
+            try {
+                p.schedule(r, 1, MILLISECONDS);
+            } catch (RejectedExecutionException ok) {
+                rje = true;
+            }
+        }
+        assertTrue(rje);
+    }
 }