Skip to content

Commit

Permalink
[MINOR] FileSystemConfig refinement
Browse files Browse the repository at this point in the history
This commit change the loading of the filesystem to return a future
to guarantee that we do not launch multiple filesystem requests at once.

Also included is a cleanup in pom.xml that fixes some space vs tabs
introduced.

Closes #1895
  • Loading branch information
Baunsgaard committed Sep 6, 2023
1 parent 4942601 commit 9028579
Show file tree
Hide file tree
Showing 2 changed files with 31 additions and 16 deletions.
28 changes: 17 additions & 11 deletions pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -235,9 +235,9 @@
<classpathPrefix>lib/</classpathPrefix>
<mainClass>org.apache.sysds.performance.Main</mainClass>
</manifest>
<manifestEntries>
<Class-Path>SystemDS.jar ${project.build.directory}/${project.artifactId}-${project.version}-tests.jar</Class-Path>
</manifestEntries>
<manifestEntries>
<Class-Path>SystemDS.jar ${project.build.directory}/${project.artifactId}-${project.version}-tests.jar</Class-Path>
</manifestEntries>
</archive>
</configuration>
</execution>
Expand Down Expand Up @@ -309,7 +309,7 @@
<artifactId>maven-compiler-plugin</artifactId>
<version>${maven-compiler-plugin.version}</version>
<configuration>
<compilerArgument>-Xlint:unchecked</compilerArgument>
<compilerArgument>-Xlint:unchecked</compilerArgument>
<source>${java.level}</source>
<target>${java.level}</target>
<release>${java.level}</release>
Expand Down Expand Up @@ -408,12 +408,12 @@
<output>file</output>
<destFile>${project.build.directory}/jacoco.exec</destFile>
<fileSets>
<fileSet>
<directory>${project.build.directory}</directory>
<includes>
<include>*/jacoco.exec</include>
</includes>
</fileSet>
<fileSet>
<directory>${project.build.directory}</directory>
<includes>
<include>*/jacoco.exec</include>
</includes>
</fileSet>
</fileSets>
</configuration>
<executions>
Expand Down Expand Up @@ -655,6 +655,7 @@
<exclude>src/test/scripts/functions/jmlc/tfmtd_example/dummycoded.column.names</exclude>
<exclude>src/test/scripts/functions/jmlc/tfmtd_example2/column.names</exclude>
<exclude>src/test/scripts/functions/jmlc/tfmtd_frame_example/tfmtd_frame</exclude>
<exclude>src/test/scripts/applications/entity_resolution/connected_components/expected/**</exclude>
<!-- csv test input not captured by *.csv -->
<exclude>src/test/scripts/functions/io/csv/in/**</exclude>
<!-- IOGEN input examples -->
Expand Down Expand Up @@ -1488,5 +1489,10 @@
<artifactId>log4j-core</artifactId>
<version>${log4j.version}</version>
</dependency>
<dependency>
<groupId>ch.randelshofer</groupId>
<artifactId>fastdoubleparser</artifactId>
<version>0.9.0</version>
</dependency>
</dependencies>
</project>
</project>
19 changes: 14 additions & 5 deletions src/main/java/org/apache/sysds/conf/ConfigurationManager.java
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
package org.apache.sysds.conf;

import java.util.concurrent.ExecutorService;
import java.util.concurrent.Future;

import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
Expand Down Expand Up @@ -56,37 +57,45 @@ public class ConfigurationManager{
/** Local compiler configuration for thead-local config updates */
private static ThreadLocalCompilerConfig _lcconf = new ThreadLocalCompilerConfig();

/** Indicate if the filesystem is loaded or not */
private static Future<?> loaded;

//global static initialization
static {
_rJob = new JobConf();

//initialization after job conf in order to prevent cyclic initialization issues
//ConfigManager -> OptimizerUtils -> InfrastructureAnalyer -> ConfigManager
//ConfigManager -> OptimizerUtils -> InfrastructureAnalyzer -> ConfigManager
_dmlconf = new DMLConfig();
_cconf = new CompilerConfig();

final ExecutorService pool = CommonThreadPool.get();
pool.submit(() ->{
loaded = pool.submit(() ->{
try{
IOUtilFunctions.getFileSystem(_rJob);
}
catch(Exception e){
LOG.warn(e.getMessage());
}
});
// pool.shutdown();
}


/**
* Returns a cached JobConf object, intended for global use by all operations
* with read-only access to job conf. This prevents to read the hadoop conf files
* over and over again from classpath. However,
* over and over again from classpath.
*
* @return the cached JobConf
*/
public static JobConf getCachedJobConf() {
return _rJob;
try{
loaded.get();
return _rJob;
}
catch(Exception e){
throw new RuntimeException(e);
}
}

public static void setCachedJobConf(JobConf job) {
Expand Down

0 comments on commit 9028579

Please sign in to comment.