8196751: Add jhsdb option to specify debug server RMI connector port

Reviewed-by: sspitsyn, ysuenaga

src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/HotSpotAgent.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2000, 2017, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2000, 2020, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -88,6 +88,7 @@
     private String javaExecutableName;
     private String coreFileName;
     private String debugServerID;
+    private int rmiPort;
 
     // All needed information for server side
     private String serverID;
@@ -200,18 +201,31 @@ public synchronized boolean detach() throws DebuggerException {
     /** This attaches to a process running on the local machine and
       starts a debug server, allowing remote machines to connect and
       examine this process. Uses specified name to uniquely identify a
-      specific debuggee on the server */
-    public synchronized void startServer(int processID, String uniqueID) {
+      specific debuggee on the server. Allows to specify the port number
+      to which the RMI connector is bound. If not specified a random
+      available port is used. */
+    public synchronized void startServer(int processID,
+                                         String uniqueID,
+                                         int rmiPort) {
         if (debugger != null) {
             throw new DebuggerException("Already attached");
         }
         pid = processID;
         startupMode = PROCESS_MODE;
         isServer = true;
         serverID = uniqueID;
+        this.rmiPort = rmiPort;
         go();
     }
 
+    /** This attaches to a process running on the local machine and
+     starts a debug server, allowing remote machines to connect and
+     examine this process. Uses specified name to uniquely identify a
+     specific debuggee on the server */
+    public synchronized void startServer(int processID, String uniqueID) {
+        startServer(processID, uniqueID, 0);
+    }
+
     /** This attaches to a process running on the local machine and
       starts a debug server, allowing remote machines to connect and
       examine this process. */
@@ -223,10 +237,12 @@ public synchronized void startServer(int processID)
     /** This opens a core file on the local machine and starts a debug
       server, allowing remote machines to connect and examine this
       core file. Uses supplied uniqueID to uniquely identify a specific
-      debugee */
+      debuggee. Allows to specify the port number to which the RMI connector
+      is bound. If not specified a random available port is used.  */
     public synchronized void startServer(String javaExecutableName,
-    String coreFileName,
-    String uniqueID) {
+                                         String coreFileName,
+                                         String uniqueID,
+                                         int rmiPort) {
         if (debugger != null) {
             throw new DebuggerException("Already attached");
         }
@@ -238,9 +254,20 @@ public synchronized void startServer(String javaExecutableName,
         startupMode = CORE_FILE_MODE;
         isServer = true;
         serverID = uniqueID;
+        this.rmiPort = rmiPort;
         go();
     }
 
+    /** This opens a core file on the local machine and starts a debug
+     server, allowing remote machines to connect and examine this
+     core file. Uses supplied uniqueID to uniquely identify a specific
+     debugee */
+    public synchronized void startServer(String javaExecutableName,
+                                         String coreFileName,
+                                         String uniqueID) {
+        startServer(javaExecutableName, coreFileName, uniqueID, 0);
+    }
+
     /** This opens a core file on the local machine and starts a debug
       server, allowing remote machines to connect and examine this
       core file. */
@@ -349,7 +376,7 @@ private void setupDebugger() {
             if (isServer) {
                 RemoteDebuggerServer remote = null;
                 try {
-                    remote = new RemoteDebuggerServer(debugger);
+                    remote = new RemoteDebuggerServer(debugger, rmiPort);
                 }
                 catch (RemoteException rem) {
                     throw new DebuggerException(rem);

src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/SALauncher.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2015, 2019, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2015, 2020, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -32,6 +32,7 @@
 import java.util.Map;
 import java.util.function.Consumer;
 
+import sun.jvm.hotspot.debugger.DebuggerException;
 import sun.jvm.hotspot.tools.JStack;
 import sun.jvm.hotspot.tools.JMap;
 import sun.jvm.hotspot.tools.JInfo;
@@ -94,6 +95,15 @@ private static boolean debugdHelp() {
         // [options] <pid> [server-id]
         // [options] <executable> <core> [server-id]
         System.out.println("    --serverid <id>         A unique identifier for this debug server.");
+        System.out.println("    --rmiport <port>        Sets the port number to which the RMI connector is bound." +
+                " If not specified a random available port is used.");
+        System.out.println("    --registryport <port>   Sets the RMI registry port." +
+                " This option overrides the system property 'sun.jvm.hotspot.rmi.port'. If not specified," +
+                " the system property is used. If the system property is not set, the default port 1099 is used.");
+        System.out.println("    --hostname <hostname>   Sets the hostname the RMI connector is bound. The value could" +
+                " be a hostname or an IPv4/IPv6 address. This option overrides the system property" +
+                " 'java.rmi.server.hostname'. If not specified, the system property is used. If the system" +
+                " property is not set, a system hostname is used.");
         return commonHelp("debugd");
     }
 
@@ -342,29 +352,96 @@ private static void runJSNAP(String[] oldArgs) {
         JSnap.main(buildAttachArgs(newArgMap, false));
     }
 
-    private static void runDEBUGD(String[] oldArgs) {
+    private static void runDEBUGD(String[] args) {
         // By default SA agent classes prefer Windows process debugger
         // to windbg debugger. SA expects special properties to be set
         // to choose other debuggers. We will set those here before
         // attaching to SA agent.
         System.setProperty("sun.jvm.hotspot.debugger.useWindbgDebugger", "true");
 
         Map<String, String> longOptsMap = Map.of("exe=", "exe",
-                                                 "core=", "core",
-                                                 "pid=", "pid",
-                                                 "serverid=", "serverid");
-        Map<String, String> newArgMap = parseOptions(oldArgs, longOptsMap);
-        var serverid = newArgMap.remove("serverid");
-        List<String> newArgArray = new ArrayList<>();
-        newArgArray.addAll(Arrays.asList(buildAttachArgs(newArgMap, false)));
+                "core=", "core",
+                "pid=", "pid",
+                "serverid=", "serverid",
+                "rmiport=", "rmiport",
+                "registryport=", "registryport",
+                "hostname=", "hostname");
+
+        Map<String, String> argMap = parseOptions(args, longOptsMap);
+
+        // Run the basic check for the options. If the check fails
+        // SAGetoptException will be thrown
+        buildAttachArgs(new HashMap<>(argMap), false);
+
+        String serverID = argMap.get("serverid");
+        String rmiPortString = argMap.get("rmiport");
+        String registryPort = argMap.get("registryport");
+        String host = argMap.get("hostname");
+        String javaExecutableName = argMap.get("exe");
+        String coreFileName = argMap.get("core");
+        String pidString = argMap.get("pid");
+
+        // Set RMI registry port, if specified
+        if (registryPort != null) {
+            try {
+                Integer.parseInt(registryPort);
+            } catch (NumberFormatException ex) {
+                throw new SAGetoptException("Invalid registry port: " + registryPort);
+            }
+            System.setProperty("sun.jvm.hotspot.rmi.port", registryPort);
+        }
+
+        // Set RMI connector hostname, if specified
+        if (host != null && !host.trim().isEmpty()) {
+            System.setProperty("java.rmi.server.hostname", host);
+        }
 
-        // `serverid` must be located at the tail.
-        if (serverid != null) {
-            newArgArray.add(serverid);
+        // Set RMI connector port, if specified
+        int rmiPort = 0;
+        if (rmiPortString != null) {
+            try {
+                rmiPort = Integer.parseInt(rmiPortString);
+            } catch (NumberFormatException ex) {
+                throw new SAGetoptException("Invalid RMI connector port: " + rmiPortString);
+            }
+        }
+
+        final HotSpotAgent agent = new HotSpotAgent();
+
+        if (pidString != null) {
+            int pid = 0;
+            try {
+                pid = Integer.parseInt(pidString);
+            } catch (NumberFormatException ex) {
+                throw new SAGetoptException("Invalid pid: " + pidString);
+            }
+            System.err.println("Attaching to process ID " + pid + " and starting RMI services," +
+                    " please wait...");
+            try {
+                agent.startServer(pid, serverID, rmiPort);
+            } catch (DebuggerException e) {
+                System.err.print("Error attaching to process or starting server: ");
+                e.printStackTrace();
+                System.exit(1);
+            } catch (NumberFormatException ex) {
+                throw new SAGetoptException("Invalid pid: " + pid);
+            }
+        } else if (javaExecutableName != null) {
+            System.err.println("Attaching to core " + coreFileName +
+                    " from executable " + javaExecutableName + " and starting RMI services, please wait...");
+            try {
+                agent.startServer(javaExecutableName, coreFileName, serverID, rmiPort);
+            } catch (DebuggerException e) {
+                System.err.print("Error attaching to core file or starting server: ");
+                e.printStackTrace();
+                System.exit(1);
+            }
         }
+        // shutdown hook to clean-up the server in case of forced exit.
+        Runtime.getRuntime().addShutdownHook(new java.lang.Thread(agent::shutdownServer));
+        System.err.println("Debugger attached and RMI services started." + ((rmiPortString != null) ?
+                (" RMI connector is bound to port " + rmiPort + ".") : ""));
 
-        // delegate to the actual SA debug server.
-        DebugServer.main(newArgArray.toArray(new String[0]));
     }
 
     // Key: tool name, Value: launcher method

src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/debugger/remote/RemoteDebuggerServer.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2002, 2009, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2002, 2020, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -43,12 +43,18 @@ public RemoteDebuggerServer() throws RemoteException {
   }
 
   /** This is the constructor used on the machine where the debuggee
-      process lies */
-  public RemoteDebuggerServer(Debugger debugger) throws RemoteException {
-    super();
+   process lies that accepts an RMI connector port */
+  public RemoteDebuggerServer(Debugger debugger, int port) throws RemoteException {
+    super(port);
     this.debugger = debugger;
   }
 
+  /** This is the constructor used on the machine where the debuggee
+   process lies */
+  public RemoteDebuggerServer(Debugger debugger) throws RemoteException {
+    this(debugger, 0);
+  }
+
   public String getOS() throws RemoteException {
     return debugger.getOS();
   }

test/hotspot/jtreg/serviceability/sa/sadebugd/SADebugDTest.java

@@ -23,8 +23,8 @@
 
 /**
  * @test
- * @bug 8163805 8224252
- * @summary Checks that the jshdb debugd utility sucessfully starts
+ * @bug 8163805 8224252 8196751
+ * @summary Checks that the jshdb debugd utility successfully starts
  *          and tries to attach to a running process
  * @requires vm.hasSA
  * @requires os.family != "windows"
@@ -39,13 +39,20 @@
 import jdk.test.lib.apps.LingeredApp;
 import jdk.test.lib.JDKToolLauncher;
 import jdk.test.lib.SA.SATestUtils;
-import static jdk.test.lib.process.ProcessTools.startProcess;
+import jdk.test.lib.Utils;
 
+import static jdk.test.lib.process.ProcessTools.startProcess;
 import jtreg.SkippedException;
 
 public class SADebugDTest {
 
     private static final String GOLDEN = "Debugger attached";
+    private static final String RMI_CONNECTOR_IS_BOUND = "RMI connector is bound to port ";
+    private static final String ADDRESS_ALREADY_IN_USE = "Address already in use";
+
+    private static final int REGISTRY_DEFAULT_PORT = 1099;
+    private static volatile boolean testResult = false;
+    private static volatile boolean portInUse = false;
 
     public static void main(String[] args) throws Exception {
         SATestUtils.skipIfCannotAttach(); // throws SkippedException if attach not expected to work.
@@ -61,31 +68,84 @@ public static void main(String[] args) throws Exception {
             // are not required.
             throw new SkippedException("Cannot run this test on OSX if adding privileges is required.");
         }
+        runTests();
+    }
+
+    private static void runTests() throws Exception {
+        boolean[] boolArray = {true, false};
+        for (boolean useRmiPort : boolArray) {
+            for (boolean useRegistryPort : boolArray) {
+                for (boolean useHostname : boolArray) {
+                    testWithPid(useRmiPort, useRegistryPort, useHostname);
+                }
+            }
+        }
+    }
+
 
+    private static void testWithPid(final boolean useRmiPort, final boolean useRegistryPort, final boolean useHostName) throws Exception {
         LingeredApp app = null;
 
         try {
             app = LingeredApp.startApp();
             System.out.println("Started LingeredApp with pid " + app.getPid());
 
-            JDKToolLauncher jhsdbLauncher = JDKToolLauncher.createUsingTestJDK("jhsdb");
-            jhsdbLauncher.addToolArg("debugd");
-            jhsdbLauncher.addToolArg("--pid");
-            jhsdbLauncher.addToolArg(Long.toString(app.getPid()));
-            ProcessBuilder pb = SATestUtils.createProcessBuilder(jhsdbLauncher);
+            do {
+                testResult = false;
+                portInUse = false;
+                JDKToolLauncher jhsdbLauncher = JDKToolLauncher.createUsingTestJDK("jhsdb");
+                jhsdbLauncher.addToolArg("debugd");
+                jhsdbLauncher.addToolArg("--pid");
+                jhsdbLauncher.addToolArg(Long.toString(app.getPid()));
 
-            // The startProcess will block untl the 'golden' string appears in either process' stdout or stderr
-            // In case of timeout startProcess kills the debugd process
-            Process debugd = startProcess("debugd", pb, null, l -> l.contains(GOLDEN), 20, TimeUnit.SECONDS);
+                int registryPort = REGISTRY_DEFAULT_PORT;
+                if (useRegistryPort) {
+                    registryPort = Utils.findUnreservedFreePort(REGISTRY_DEFAULT_PORT);
+                    jhsdbLauncher.addToolArg("--registryport");
+                    jhsdbLauncher.addToolArg(Integer.toString(registryPort));
+                }
 
-            // If we are here, this means we have received the golden line and the test has passed
-            // The debugd remains running, we have to kill it
-            debugd.destroy();
-            debugd.waitFor();
+                int rmiPort = -1;
+                if (useRmiPort) {
+                    rmiPort = Utils.findUnreservedFreePort(REGISTRY_DEFAULT_PORT, registryPort);
+                    jhsdbLauncher.addToolArg("--rmiport");
+                    jhsdbLauncher.addToolArg(Integer.toString(rmiPort));
+                }
+                if (useHostName) {
+                    jhsdbLauncher.addToolArg("--hostname");
+                    jhsdbLauncher.addToolArg("testhost");
+                }
+                ProcessBuilder pb = SATestUtils.createProcessBuilder(jhsdbLauncher);
+
+                final int finalRmiPort = rmiPort;
+
+                // The startProcess will block until the 'golden' string appears in either process' stdout or stderr
+                // In case of timeout startProcess kills the debugd process
+                Process debugd = startProcess("debugd", pb, null,
+                        l -> {
+                            if (!useRmiPort && l.contains(GOLDEN)) {
+                                testResult = true;
+                            } else if (useRmiPort && l.contains(RMI_CONNECTOR_IS_BOUND + finalRmiPort)) {
+                                testResult = true;
+                            } else if (l.contains(ADDRESS_ALREADY_IN_USE)) {
+                                portInUse = true;
+                            }
+                            return (l.contains(GOLDEN) || portInUse);
+                        }, 20, TimeUnit.SECONDS);
+
+                // If we are here, this means we have received the golden line and the test has passed
+                // The debugd remains running, we have to kill it
+                debugd.destroy();
+                debugd.waitFor();
+
+                if (!testResult) {
+                    throw new RuntimeException("Expected message \"" +
+                            RMI_CONNECTOR_IS_BOUND + rmiPort + "\" is not found in the output.");
+                }
+
+            } while (portInUse); // Repeat the test if the port is already in use
         } finally {
             LingeredApp.stopApp(app);
         }
-
     }
-
 }