[SYSTEMDS-3217] PythonAPI working with SYSTEMDS_ROOT set

.github/workflows/python.yml

@@ -27,7 +27,8 @@ on:
       - 'docs/**'
       - '*.md'
       - '*.html'
-      - 'src/main/python/docs/**'
+      - 'src/test/**'
+      - 'src/assembly/**'
       - 'dev/**'
     branches:
       - main
@@ -36,7 +37,7 @@ on:
       - 'docs/**'
       - '*.md'
       - '*.html'
-      - 'src/main/python/docs/**'
+      - 'src/test/**'
       - 'dev/**'
     branches:
       - main
@@ -94,7 +95,7 @@ jobs:
         architecture: 'x64'
 
     - name: Install pip Dependencies
-      run: pip install numpy py4j wheel scipy sklearn requests pandas
+      run: pip install numpy py4j wheel scipy sklearn requests pandas unittest-parallel
 
     - name: Build Python Package
       run: |
@@ -107,15 +108,16 @@ jobs:
         export PATH=$SYSTEMDS_ROOT/bin:$PATH
         export SYSDS_QUIET=1
         cd src/main/python
-        python -m unittest discover -s tests -p 'test_*.py'
+        unittest-parallel -t . -s tests --module-fixtures
+        # python -m unittest discover -s tests -p 'test_*.py'
         echo "Exit Status: " $?
 
-    # TODO debug and fix JDK11 environment 
-    #- name: Run all python tests no environment
-    #  run: |
-    #    cd src/main/python
-    #    python -m unittest discover -s tests -p 'test_*.py'
-    #    echo "Exit Status: " $?
+    - name: Run all python tests no environment
+      run: |
+        cd src/main/python
+        unittest-parallel -t . -s tests --module-fixtures
+        # python -m unittest discover -s tests -p 'test_*.py'
+        echo "Exit Status: " $?
 
     - name: Run Federated Python Tests
       run: |

src/main/java/org/apache/sysds/api/DMLOptions.java

@@ -71,6 +71,7 @@ public class DMLOptions {
 	public boolean              lineage_debugger = false;         // whether enable lineage debugger
 	public boolean              fedWorker     = false;
 	public int                  fedWorkerPort = -1;
+	public int                  pythonPort    = -1; 
 	public boolean              checkPrivacy  = false;            // Check which privacy constraints are loaded and checked during federated execution 
 	public boolean				federatedCompilation = false;     // Compile federated instructions based on input federation state and privacy constraints.
 
@@ -242,6 +243,10 @@ else if (lineageType.equalsIgnoreCase("debugger"))
 			}
 		}
 
+		if (line.hasOption("python")){
+			dmlOptions.pythonPort = Integer.parseInt(line.getOptionValue("python"));
+		}
+
 		// Named arguments map is created as ("$K, 123), ("$X", "X.csv"), etc
 		if (line.hasOption("nvargs")){
 			String varNameRegex = "^[a-zA-Z]([a-zA-Z0-9_])*$";
@@ -302,8 +307,8 @@ private static Options createCLIOptions() {
 			.hasOptionalArg().create("gpu");
 		Option debugOpt = OptionBuilder.withDescription("runs in debug mode; default off")
 			.create("debug");
-		Option pythonOpt = OptionBuilder.withDescription("parses Python-like DML")
-			.create("python");
+		Option pythonOpt = OptionBuilder.withDescription("Python Context start with port argument for communication to python")
+			.isRequired().hasArg().create("python");
 		Option fileOpt = OptionBuilder.withArgName("filename")
 			.withDescription("specifies dml/pydml file to execute; path can be local/hdfs/gpfs (prefixed with appropriate URI)")
 			.isRequired().hasArg().create("f");
@@ -332,7 +337,6 @@ private static Options createCLIOptions() {
 		options.addOption(execOpt);
 		options.addOption(gpuOpt);
 		options.addOption(debugOpt);
-		options.addOption(pythonOpt);
 		options.addOption(lineageOpt);
 		options.addOption(fedOpt);
 		options.addOption(checkPrivacy);
@@ -344,7 +348,8 @@ private static Options createCLIOptions() {
 			.addOption(fileOpt)
 			.addOption(cleanOpt)
 			.addOption(helpOpt)
-			.addOption(fedOpt);
+			.addOption(fedOpt)
+			.addOption(pythonOpt);
 		fileOrScriptOpt.setRequired(true);
 		options.addOptionGroup(fileOrScriptOpt);
 

src/main/java/org/apache/sysds/api/DMLScript.java

@@ -356,7 +356,7 @@ public static String readDMLScript( boolean isFile, String scriptOrFilename )
 	// (core compilation and execute)
 	////////
 
-	private static void loadConfiguration(String fnameOptConfig) throws IOException {
+	public static void loadConfiguration(String fnameOptConfig) throws IOException {
 		DMLConfig dmlconf = DMLConfig.readConfigurationFile(fnameOptConfig);
 		ConfigurationManager.setGlobalConfig(dmlconf);
 		CompilerConfig cconf = OptimizerUtils.constructCompilerConfig(dmlconf);

src/main/java/org/apache/sysds/api/PythonDMLScript.java

@@ -26,40 +26,29 @@
 
 import py4j.GatewayServer;
 import py4j.GatewayServerListener;
+import py4j.Py4JNetworkException;
 import py4j.Py4JServerConnection;
 
 public class PythonDMLScript {
-	private static final Log LOG = LogFactory.getLog(PythonDMLScript.class.getName());
+
 	private Connection _connection;
 
 	/**
 	 * Entry point for Python API.
 	 * 
-	 * The system returns with exit code 1, if the startup process fails, and 0 if the startup was successful.
-	 * 
 	 * @param args Command line arguments.
+	 * @throws Exception Throws exceptions if there is issues in startup or while running.
 	 */
-	public static void main(String[] args) {
-		if(args.length != 1) {
-			throw new IllegalArgumentException("Python DML Script should be initialized with a singe number argument");
-		}
-		else {
-			int port = Integer.parseInt(args[0]);
-			start(port);
-		}
+	public static void main(String[] args) throws Exception {
+		final DMLOptions dmlOptions = DMLOptions.parseCLArguments(args);
+		DMLScript.loadConfiguration(dmlOptions.configFile);
+		start(dmlOptions.pythonPort);
 	}
 
-	private static void start(int port) {
-		try {
-			// TODO Add argument parsing here.
-			GatewayServer GwS = new GatewayServer(new PythonDMLScript(), port);
-			GwS.addListener(new DMLGateWayListener());
-			GwS.start();
-		}
-		catch(py4j.Py4JNetworkException ex) {
-			LOG.error("Py4JNetworkException while executing the GateWay. Is a server instance already running?");
-			System.exit(-1);
-		}
+	private static void start(int port) throws Py4JNetworkException {
+		GatewayServer GwS = new GatewayServer(new PythonDMLScript(), port);
+		GwS.addListener(new DMLGateWayListener());
+		GwS.start();
 	}
 
 	private PythonDMLScript() {
@@ -79,50 +68,53 @@ private PythonDMLScript() {
 	public Connection getConnection() {
 		return _connection;
 	}
-}
-
-class DMLGateWayListener implements GatewayServerListener {
-	private static final Log LOG = LogFactory.getLog(DMLGateWayListener.class.getName());
-
-	@Override
-	public void connectionError(Exception e) {
-		LOG.warn("Connection error: " + e.getMessage());
-	}
-
-	@Override
-	public void connectionStarted(Py4JServerConnection gatewayConnection) {
-		LOG.debug("Connection Started: " + gatewayConnection.toString());
-	}
-
-	@Override
-	public void connectionStopped(Py4JServerConnection gatewayConnection) {
-		LOG.debug("Connection stopped: " + gatewayConnection.toString());
-	}
-
-	@Override
-	public void serverError(Exception e) {
-		LOG.error("Server Error " + e.getMessage());
-	}
-
-	@Override
-	public void serverPostShutdown() {
-		LOG.info("Shutdown done");
-		System.exit(0);
-	}
-
-	@Override
-	public void serverPreShutdown() {
-		LOG.info("Starting JVM shutdown");
-	}
-
-	@Override
-	public void serverStarted() {
-		// message the python interface that the JVM is ready.
-		System.out.println("GatewayServer Started");
-	}
-
-	@Override
-	public void serverStopped() {
-		System.out.println("GatewayServer Stopped");
+
+	protected static class DMLGateWayListener implements GatewayServerListener {
+		private static final Log LOG = LogFactory.getLog(DMLGateWayListener.class.getName());
+
+		@Override
+		public void connectionError(Exception e) {
+			LOG.warn("Connection error: " + e.getMessage());
+			System.exit(1);
+		}
+
+		@Override
+		public void connectionStarted(Py4JServerConnection gatewayConnection) {
+			LOG.debug("Connection Started: " + gatewayConnection.toString());
+		}
+
+		@Override
+		public void connectionStopped(Py4JServerConnection gatewayConnection) {
+			LOG.debug("Connection stopped: " + gatewayConnection.toString());
+		}
+
+		@Override
+		public void serverError(Exception e) {
+			LOG.error("Server Error " + e.getMessage());
+		}
+
+		@Override
+		public void serverPostShutdown() {
+			LOG.info("Shutdown done");
+			System.exit(0);
+		}
+
+		@Override
+		public void serverPreShutdown() {
+			LOG.info("Starting JVM shutdown");
+		}
+
+		@Override
+		public void serverStarted() {
+			// message the python interface that the JVM is ready.
+			System.out.println("GatewayServer Started");
+		}
+
+		@Override
+		public void serverStopped() {
+			System.out.println("GatewayServer Stopped");
+			System.exit(0);
+		}
 	}
 }
+

src/main/java/org/apache/sysds/runtime/io/IOUtilFunctions.java

@@ -93,15 +93,15 @@ public static FileSystem getFileSystem(Configuration conf) throws IOException {
 		try{
 			return FileSystem.get(conf);
 		} catch(NoClassDefFoundError err) {
-			throw new IOException(err.getMessage());
+			throw new IOException(err.getMessage(), err);
 		}
 	}
 
 	public static FileSystem getFileSystem(Path fname, Configuration conf) throws IOException {
 		try {
 			return FileSystem.get(fname.toUri(), conf);
 		} catch(NoClassDefFoundError err) {
-			throw new IOException(err.getMessage());
+			throw new IOException(err.getMessage(), err);
 		}
 	}