Skip to content

Commit

Permalink
[SYSTEMML-1325] Fixes formatting issues and warnings. Fixes bug causi…
Browse files Browse the repository at this point in the history 
…ng explain to sometimes not be printed.

Closes #838.
  • Loading branch information
@thomas9t
thomas9t authored and Niketan Pansare committed Nov 3, 2018
1 parent cc7ec76 commit bf4ba16
Show file tree
Hide file tree
Showing 8 changed files with 170 additions and 399 deletions.
104 changes: 52 additions & 52 deletions src/main/java/org/apache/sysml/api/DMLScript.java
View file
Expand Up @@ -376,56 +376,56 @@ else if (debug.equalsIgnoreCase("trace"))
// (core compilation and execute)
////////

/**
* The running body of DMLScript execution. This method should be called after execution properties have been correctly set,
* and customized parameters have been put into _argVals
*
* @param dmlScriptStr DML script string
* @param fnameOptConfig configuration file
* @param argVals map of argument values
* @param allArgs arguments
* @param scriptType type of script (DML or PyDML)
* @throws IOException if IOException occurs
*/
private static void execute(String dmlScriptStr, String fnameOptConfig, Map<String,String> argVals, String[] allArgs, ScriptType scriptType)
throws IOException
{
SCRIPT_TYPE = scriptType;

//print basic time and environment info
printStartExecInfo( dmlScriptStr );

//Step 1: parse configuration files & write any configuration specific global variables
DMLConfig dmlconf = DMLConfig.readConfigurationFile(fnameOptConfig);
ConfigurationManager.setGlobalConfig(dmlconf);
CompilerConfig cconf = OptimizerUtils.constructCompilerConfig(dmlconf);
ConfigurationManager.setGlobalConfig(cconf);
LOG.debug("\nDML config: \n" + dmlconf.getConfigInfo());

setGlobalFlags(dmlconf);
Program rtprog = ScriptExecutorUtils.compileRuntimeProgram(dmlScriptStr, argVals, allArgs,
scriptType, dmlconf, SystemMLAPI.DMLScript);
List<GPUContext> gCtxs = ConfigurationManager.getDMLOptions().gpu ? GPUContextPool.getAllGPUContexts() : null;

//double costs = CostEstimationWrapper.getTimeEstimate(rtprog, ExecutionContextFactory.createContext());
//System.out.println("Estimated costs: "+costs);

//Step 10: execute runtime program
ExecutionContext ec = null;
try {
ec = ScriptExecutorUtils.executeRuntimeProgram(
rtprog, dmlconf, ConfigurationManager.isStatistics() ?
ConfigurationManager.getDMLOptions().getStatisticsMaxHeavyHitters() : 0,
new LocalVariableMap(), null, SystemMLAPI.DMLScript, gCtxs);
}
finally {
if(ec != null && ec instanceof SparkExecutionContext)
((SparkExecutionContext) ec).close();
LOG.info("END DML run " + getDateTime() );
//cleanup scratch_space and all working dirs
cleanupHadoopExecution( dmlconf );
}
}
/**
* The running body of DMLScript execution. This method should be called after execution properties have been correctly set,
* and customized parameters have been put into _argVals
*
* @param dmlScriptStr DML script string
* @param fnameOptConfig configuration file
* @param argVals map of argument values
* @param allArgs arguments
* @param scriptType type of script (DML or PyDML)
* @throws IOException if IOException occurs
*/
private static void execute(String dmlScriptStr, String fnameOptConfig, Map<String,String> argVals, String[] allArgs, ScriptType scriptType)
throws IOException
{
SCRIPT_TYPE = scriptType;

//print basic time and environment info
printStartExecInfo( dmlScriptStr );

//Step 1: parse configuration files & write any configuration specific global variables
DMLConfig dmlconf = DMLConfig.readConfigurationFile(fnameOptConfig);
ConfigurationManager.setGlobalConfig(dmlconf);
CompilerConfig cconf = OptimizerUtils.constructCompilerConfig(dmlconf);
ConfigurationManager.setGlobalConfig(cconf);
LOG.debug("\nDML config: \n" + dmlconf.getConfigInfo());

setGlobalFlags(dmlconf);
Program rtprog = ScriptExecutorUtils.compileRuntimeProgram(dmlScriptStr, argVals, allArgs,
scriptType, dmlconf, SystemMLAPI.DMLScript);
List<GPUContext> gCtxs = ConfigurationManager.getDMLOptions().gpu ? GPUContextPool.getAllGPUContexts() : null;

//double costs = CostEstimationWrapper.getTimeEstimate(rtprog, ExecutionContextFactory.createContext());
//System.out.println("Estimated costs: "+costs);

//Step 10: execute runtime program
ExecutionContext ec = null;
try {
ec = ScriptExecutorUtils.executeRuntimeProgram(
rtprog, ConfigurationManager.isStatistics() ?
ConfigurationManager.getDMLOptions().getStatisticsMaxHeavyHitters() : 0,
new LocalVariableMap(), null, SystemMLAPI.DMLScript, gCtxs);
}
finally {
if(ec != null && ec instanceof SparkExecutionContext)
((SparkExecutionContext) ec).close();
LOG.info("END DML run " + getDateTime() );
//cleanup scratch_space and all working dirs
cleanupHadoopExecution( dmlconf );
}
}

/**
* Sets the global flags in DMLScript based on user provided configuration
Expand Down Expand Up @@ -547,8 +547,8 @@ private static void checkSecuritySetup(DMLConfig config)

//determine security states
boolean flagDiffUser = !( taskController.equals("org.apache.hadoop.mapred.LinuxTaskController") //runs map/reduce tasks as the current user
|| localMode // run in the same JVM anyway
|| groupNames.contains( ttGroupName) ); //user in task tracker group
|| localMode // run in the same JVM anyway
|| groupNames.contains( ttGroupName) ); //user in task tracker group
boolean flagLocalFS = fsURI==null || fsURI.getScheme().equals("file");
boolean flagSecurity = perm.equals("yes");

Expand Down
20 changes: 8 additions & 12 deletions src/main/java/org/apache/sysml/api/ScriptExecutorUtils.java
View file
Expand Up @@ -116,7 +116,7 @@ public static Program compileRuntimeProgram(String script, Map<String,String> ns
boolean init) {
DMLScript.SCRIPT_TYPE = scriptType;

Program rtprog = null;
Program rtprog;

if (ConfigurationManager.isGPU() && !IS_JCUDA_AVAILABLE)
throw new RuntimeException("Incorrect usage: Cannot use the GPU backend without JCuda libraries. Hint: Include systemml-*-extra.jar (compiled using mvn package -P distribution) into the classpath.");
Expand Down Expand Up @@ -161,7 +161,7 @@ else if (!ConfigurationManager.isGPU() && ConfigurationManager.isForcedGPU())

//init working directories (before usage by following compilation steps)
if(api != SystemMLAPI.JMLC)
if ((api == SystemMLAPI.MLContext && init) || api != SystemMLAPI.MLContext)
if (api != SystemMLAPI.MLContext || init)
DMLScript.initHadoopExecution( dmlconf );


Expand Down Expand Up @@ -222,8 +222,9 @@ else if(api == SystemMLAPI.MLContext) {
ExplainCounts counts = Explain.countDistributedOperations(rtprog);
Statistics.resetNoOfCompiledJobs( counts.numJobs );
//explain plan of program (hops or runtime)
if( DMLScript.EXPLAIN != ExplainType.NONE )
System.out.println(Explain.display(prog, rtprog, DMLScript.EXPLAIN, counts));
if( ConfigurationManager.getDMLOptions().explainType != ExplainType.NONE )
System.out.println(
Explain.display(prog, rtprog, ConfigurationManager.getDMLOptions().explainType, counts));

Statistics.stopCompileTimer();
}
Expand All @@ -232,9 +233,6 @@ else if(api == SystemMLAPI.MLContext) {
// don't chain ParseException (for cleaner error output)
throw pe;
}
catch(IOException ex) {
throw new DMLException(ex);
}
catch(Exception ex) {
throw new DMLException(ex);
}
Expand All @@ -248,8 +246,6 @@ else if(api == SystemMLAPI.MLContext) {
*
* @param rtprog
* runtime program
* @param dmlconf
* dml configuration
* @param statisticsMaxHeavyHitters
* maximum number of statistics to print
* @param symbolTable
Expand All @@ -262,7 +258,7 @@ else if(api == SystemMLAPI.MLContext) {
* list of GPU contexts
* @return execution context
*/
public static ExecutionContext executeRuntimeProgram(Program rtprog, DMLConfig dmlconf, int statisticsMaxHeavyHitters,
public static ExecutionContext executeRuntimeProgram(Program rtprog, int statisticsMaxHeavyHitters,
LocalVariableMap symbolTable, HashSet<String> outputVariables,
SystemMLAPI api, List<GPUContext> gCtxs) {
boolean exceptionThrown = false;
Expand Down Expand Up @@ -299,7 +295,7 @@ public static ExecutionContext executeRuntimeProgram(Program rtprog, DMLConfig d
if(outputVariables != null) {
for(String outVar : outputVariables) {
Data data = ec.getVariable(outVar);
if(data != null && data instanceof MatrixObject) {
if(data instanceof MatrixObject) {
for(GPUContext gCtx : ec.getGPUContexts()) {
GPUObject gpuObj = ((MatrixObject)data).getGPUObject(gCtx);
if(gpuObj != null && gpuObj.isDirty()) {
Expand Down Expand Up @@ -341,4 +337,4 @@ public static ExecutionContext executeRuntimeProgram(Program rtprog, DMLConfig d
return ec;
}

}
}
8 changes: 4 additions & 4 deletions src/main/java/org/apache/sysml/api/jmlc/PreparedScript.java
View file
Expand Up @@ -32,7 +32,6 @@
import org.apache.sysml.api.DMLException;
import org.apache.sysml.api.ScriptExecutorUtils;
import org.apache.sysml.api.ScriptExecutorUtils.SystemMLAPI;
import org.apache.sysml.api.DMLScript;
import org.apache.sysml.conf.CompilerConfig;
import org.apache.sysml.conf.ConfigurationManager;
import org.apache.sysml.conf.DMLConfig;
Expand All @@ -47,8 +46,6 @@
import org.apache.sysml.runtime.controlprogram.Program;
import org.apache.sysml.runtime.controlprogram.caching.FrameObject;
import org.apache.sysml.runtime.controlprogram.caching.MatrixObject;
import org.apache.sysml.runtime.controlprogram.context.ExecutionContext;
import org.apache.sysml.runtime.controlprogram.context.ExecutionContextFactory;
import org.apache.sysml.runtime.instructions.cp.BooleanObject;
import org.apache.sysml.runtime.instructions.cp.Data;
import org.apache.sysml.runtime.instructions.cp.DoubleObject;
Expand Down Expand Up @@ -100,6 +97,9 @@ private PreparedScript(PreparedScript that) {
_inVarReuse = new HashMap<>(that._inVarReuse);
_dmlconf = that._dmlconf;
_cconf = that._cconf;
_isStatisticsEnabled = that._isStatisticsEnabled;
_gatherMemStats = that._gatherMemStats;
_gpuCtx = that._gpuCtx;
}

/**
Expand Down Expand Up @@ -463,7 +463,7 @@ public ResultVariables executeScript() {

//create and populate execution context
ScriptExecutorUtils.executeRuntimeProgram(
_prog, _dmlconf, ConfigurationManager.isStatistics() ?
_prog, ConfigurationManager.isStatistics() ?
ConfigurationManager.getDMLOptions().getStatisticsMaxHeavyHitters() : 0,
_vars, _outVarnames, SystemMLAPI.JMLC, _gpuCtx);

Expand Down

0 comments on commit bf4ba16

Please sign in to comment.