8212218: [TESTBUG] runtime/ErrorHandling/TestHeapDumpOnOutOfMemoryErrorInMetaspace.java timed out

Reviewed-by: iklam, lmesnik

Author: tstuefe

test/hotspot/jtreg/runtime/ErrorHandling/TestHeapDumpOnOutOfMemoryError.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2018, 2020 Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -66,7 +66,19 @@ public static void main(String[] args) throws Exception {
     static void test(String type) throws Exception {
         String heapdumpFilename = type + ".hprof";
         ProcessBuilder pb = ProcessTools.createJavaProcessBuilder("-XX:+HeapDumpOnOutOfMemoryError",
-                "-XX:HeapDumpPath=" + heapdumpFilename, "-XX:MaxMetaspaceSize=64m",
+                "-XX:HeapDumpPath=" + heapdumpFilename,
+                // Note: When trying to provoke a metaspace OOM we may generate a lot of classes. In debug VMs this
+                //  can cause considerable wait times since:
+                // - Compiler Dependencies verification iterates the class tree
+                // - Before exit, the CLDG is checked.
+                // Both verifications show quadratic time or worse wrt to number of loaded classes. Therefore it
+                //  makes sense to switch one or both off and limit the metaspace size to something sensible.
+                // Example numbers on a slow ppc64 machine:
+                //  MaxMetaspaceSize=64M - ~60-70K classes - ~20min runtime with all verifications
+                //  MaxMetaspaceSize=16M - ~12-15K classes - ~12sec runtime with all verifications
+                //  MaxMetaspaceSize=16M - ~12-15K classes - VerifyDependencies off - ~3seconds on ppc
+                "-XX:MaxMetaspaceSize=16m",
+                "-XX:-VerifyDependencies",
                 TestHeapDumpOnOutOfMemoryError.class.getName(), type);
 
         OutputAnalyzer output = new OutputAnalyzer(pb.start());