8235751: Assertion when triggering concurrent cycle during shutdown

Skip initial mark during shutdown and don't assert when that happens.

Reviewed-by: sjohanss, tschatzl

src/hotspot/share/gc/g1/g1CollectedHeap.cpp

@@ -1,5 +1,5 @@
- /*
- * Copyright (c) 2001, 2019, Oracle and/or its affiliates. All rights reserved.
+/*
+ * Copyright (c) 2001, 2020, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -2151,6 +2151,13 @@ bool G1CollectedHeap::try_collect_concurrently(GCCause::Cause cause,
       return op.gc_succeeded();
     }
 
+    // If VMOp skipped initiating concurrent marking cycle because
+    // we're terminating, then we're done.
+    if (op.terminating()) {
+      LOG_COLLECT_CONCURRENTLY(cause, "skipped: terminating");
+      return false;
+    }
+
     // Lock to get consistent set of values.
     uint old_marking_started_after;
     uint old_marking_completed_after;

src/hotspot/share/gc/g1/g1VMOperations.cpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2001, 2019, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2001, 2020, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -47,6 +47,7 @@ VM_G1TryInitiateConcMark::VM_G1TryInitiateConcMark(uint gc_count_before,
   _target_pause_time_ms(target_pause_time_ms),
   _transient_failure(false),
   _cycle_already_in_progress(false),
+  _terminating(false),
   _gc_succeeded(false)
 {}
 
@@ -66,7 +67,17 @@ void VM_G1TryInitiateConcMark::doit() {
   G1CollectedHeap* g1h = G1CollectedHeap::heap();
 
   GCCauseSetter x(g1h, _gc_cause);
-  if (!g1h->policy()->force_initial_mark_if_outside_cycle(_gc_cause)) {
+
+  // Record for handling by caller.
+  _terminating = g1h->_cm_thread->should_terminate();
+
+  if (_terminating && GCCause::is_user_requested_gc(_gc_cause)) {
+    // When terminating, the request to initiate a concurrent cycle will be
+    // ignored by do_collection_pause_at_safepoint; instead it will just do
+    // a young-only or mixed GC (depending on phase).  For a user request
+    // there's no point in even doing that much, so done.  For some non-user
+    // requests the alternative GC might still be needed.
+  } else if (!g1h->policy()->force_initial_mark_if_outside_cycle(_gc_cause)) {
     // Failure to force the next GC pause to be an initial mark indicates
     // there is already a concurrent marking cycle in progress.  Set flag
     // to notify the caller and return immediately.

src/hotspot/share/gc/g1/g1VMOperations.hpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2001, 2019, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2001, 2020, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -52,6 +52,7 @@ class VM_G1TryInitiateConcMark : public VM_GC_Operation {
   double _target_pause_time_ms;
   bool _transient_failure;
   bool _cycle_already_in_progress;
+  bool _terminating;
   bool _gc_succeeded;
 
 public:
@@ -63,6 +64,7 @@ class VM_G1TryInitiateConcMark : public VM_GC_Operation {
   virtual void doit();
   bool transient_failure() const { return _transient_failure; }
   bool cycle_already_in_progress() const { return _cycle_already_in_progress; }
+  bool terminating() const { return _terminating; }
   bool gc_succeeded() const { return _gc_succeeded; }
 };