Skip to content
This repository has been archived by the owner on Jul 3, 2023. It is now read-only.

Commit

Permalink
Merge pull request #116 from IMS94/master
Browse files Browse the repository at this point in the history 
[OODT-1031] Updated filemgr, workflow manager and resource manager starters to use XML RPC by default
  • Loading branch information
@chrismattmann
chrismattmann committed Dec 25, 2019
2 parents 2d38c8a + 16f2fe4 commit 771c580
Show file tree
Hide file tree
Showing 12 changed files with 87 additions and 53 deletions.
6 changes: 5 additions & 1 deletion filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/FileManagerServer.java
View file
Expand Up @@ -27,7 +27,11 @@
*
*/
public interface FileManagerServer {


String FILEMGR_PROPERTIES_FILE_SYSTEM_PROPERTY = "org.apache.oodt.cas.filemgr.properties";
String FILEMGR_SERVER_SYSTEM_PROPERTY = "filemgr.server";
String FILEMGR_CLIENT_SYSTEM_PROPERTY = "filemgr.client";

/**
*
* <p>Preparing and starting up the rpc server.</p>
Expand Down
20 changes: 11 additions & 9 deletions filemgr/src/main/java/org/apache/oodt/cas/filemgr/util/RpcCommunicationFactory.java
View file
Expand Up @@ -22,6 +22,8 @@
import org.apache.oodt.cas.filemgr.system.FileManagerServer;
import org.apache.oodt.cas.filemgr.system.rpc.FileManagerClientFactory;
import org.apache.oodt.cas.filemgr.system.rpc.FileManagerServerFactory;
import org.apache.oodt.cas.filemgr.system.rpc.XmlRpcFileManagerClientFactory;
import org.apache.oodt.cas.filemgr.system.rpc.XmlRpcFileManagerServerFactory;

import java.io.File;
import java.io.FileInputStream;
Expand All @@ -42,20 +44,20 @@ public class RpcCommunicationFactory {

private static Logger LOG = Logger.getLogger(RpcCommunicationFactory.class
.getName());

private static String getClientFactoryName(){
return System.getProperty("filemgr.client",
"org.apache.oodt.cas.filemgr.system.rpc.AvroFileManagerClientFactory");
}
private static String getClientFactoryName() {
return System.getProperty(FileManagerServer.FILEMGR_CLIENT_SYSTEM_PROPERTY,
XmlRpcFileManagerClientFactory.class.getName());
}

/**
* Set properties from filemgr.properties file
* to get name of RPC initialization class.
*/
private static void setPror(){
// set up the configuration, if there is any
if (System.getProperty("org.apache.oodt.cas.filemgr.properties") != null) {
String configFile = System.getProperty("org.apache.oodt.cas.filemgr.properties");
if (System.getProperty(FileManagerServer.FILEMGR_PROPERTIES_FILE_SYSTEM_PROPERTY) != null) {
String configFile = System.getProperty(FileManagerServer.FILEMGR_PROPERTIES_FILE_SYSTEM_PROPERTY);

LOG.log(Level.INFO, "Loading File Manager Configuration Properties from: [" + configFile + "]");

Expand Down Expand Up @@ -126,8 +128,8 @@ public static FileManagerClient createClient(URL filemgrUrl, boolean testConnect
public static FileManagerServer createServer(int port) throws IOException {
setPror();

String serverFactory = System.getProperty("filemgr.server",
"org.apache.oodt.cas.filemgr.system.rpc.AvroFileManagerServerFactory");
String serverFactory = System.getProperty(FileManagerServer.FILEMGR_SERVER_SYSTEM_PROPERTY,
XmlRpcFileManagerServerFactory.class.getName());

LOG.log(Level.INFO, "Init. server's factory class: " + serverFactory);

Expand Down
4 changes: 2 additions & 2 deletions filemgr/src/main/resources/filemgr.properties
View file
Expand Up @@ -18,8 +18,8 @@
# rpc configuration, uncomment the avro implementations to use AvroRPC
#filemgr.server=org.apache.oodt.cas.filemgr.system.rpc.XmlRpcFileManagerServerFactory
#filemgr.client=org.apache.oodt.cas.filemgr.system.rpc.XmlRpcFileManagerClientFactory
filemgr.server=org.apache.oodt.cas.filemgr.system.rpc.AvroFileManagerServerFactory
filemgr.client=org.apache.oodt.cas.filemgr.system.rpc.AvroFileManagerClientFactory
filemgr.server=org.apache.oodt.cas.filemgr.system.rpc.XmlRpcFileManagerServerFactory
filemgr.client=org.apache.oodt.cas.filemgr.system.rpc.XmlRpcFileManagerClientFactory

# repository factory
filemgr.repository.factory=org.apache.oodt.cas.filemgr.repository.XMLRepositoryManagerFactory
Expand Down
4 changes: 2 additions & 2 deletions resource/src/main/java/org/apache/oodt/cas/resource/system/AvroRpcResourceManager.java
View file
Expand Up @@ -72,8 +72,8 @@ public AvroRpcResourceManager(int port) {

List<String> propertiesFiles = new ArrayList<>();
// set up the configuration, if there is any
if (System.getProperty("org.apache.oodt.cas.resource.properties") != null) {
propertiesFiles.add(System.getProperty("org.apache.oodt.cas.resource.properties"));
if (System.getProperty(ResourceManager.RESMGR_PROPERTIES_FILE_SYSTEM_PROPERTY) != null) {
propertiesFiles.add(System.getProperty(ResourceManager.RESMGR_PROPERTIES_FILE_SYSTEM_PROPERTY));
}

configurationManager = ConfigurationManagerFactory
Expand Down
5 changes: 5 additions & 0 deletions resource/src/main/java/org/apache/oodt/cas/resource/system/ResourceManager.java
View file
Expand Up @@ -19,6 +19,11 @@

public interface ResourceManager {

String RESMGR_PROPERTIES_FILE_SYSTEM_PROPERTY = "org.apache.oodt.cas.resource.properties";
String RESMGR_SYSTEM_PROPERTY = "resmgr.manager";
String RESMGR_CLIENT_SYSTEM_PROPERTY = "resmgr.manager.client";


void startUp() throws Exception;

boolean isAlive();
Expand Down
4 changes: 2 additions & 2 deletions resource/src/main/java/org/apache/oodt/cas/resource/system/XmlRpcResourceManager.java
View file
Expand Up @@ -77,8 +77,8 @@ public XmlRpcResourceManager(int port) throws IOException {
this.port = port;
List<String> propertiesFiles = new ArrayList<>();
// set up the configuration, if there is any
if (System.getProperty("org.apache.oodt.cas.resource.properties") != null) {
propertiesFiles.add(System.getProperty("org.apache.oodt.cas.resource.properties"));
if (System.getProperty(ResourceManager.RESMGR_PROPERTIES_FILE_SYSTEM_PROPERTY) != null) {
propertiesFiles.add(System.getProperty(ResourceManager.RESMGR_PROPERTIES_FILE_SYSTEM_PROPERTY));
}

configurationManager = ConfigurationManagerFactory.getConfigurationManager(Component.RESOURCE_MANAGER, propertiesFiles);
Expand Down
21 changes: 12 additions & 9 deletions resource/src/main/java/org/apache/oodt/cas/resource/system/rpc/ResourceManagerFactory.java
View file
Expand Up @@ -19,36 +19,39 @@

import org.apache.oodt.cas.resource.system.ResourceManager;
import org.apache.oodt.cas.resource.system.ResourceManagerClient;
import org.apache.oodt.cas.resource.system.XmlRpcResourceManager;
import org.apache.oodt.cas.resource.system.XmlRpcResourceManagerClient;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.lang.reflect.Constructor;
import java.net.URL;

public class ResourceManagerFactory {

private static final Logger logger = LoggerFactory.getLogger(ResourceManagerFactory.class);

private static void loadProperties() {
// set up the configuration, if there is any
if (System.getProperty("org.apache.oodt.cas.resource.properties") != null) {
String configFile = System.getProperty("org.apache.oodt.cas.resource.properties");
if (System.getProperty(ResourceManager.RESMGR_PROPERTIES_FILE_SYSTEM_PROPERTY) != null) {
String configFile = System.getProperty(ResourceManager.RESMGR_PROPERTIES_FILE_SYSTEM_PROPERTY);

logger.info("Loading File Manager Configuration Properties from: [{}]", configFile);
logger.info("Loading resource manager configuration properties from: [{}]", configFile);
try {
System.getProperties().load(new FileInputStream(new File(configFile)));
} catch (Exception e) {
} catch (IOException e) {
logger.error("Error loading configuration properties from: [{}]", configFile);
}
}
}

public static ResourceManager getResourceManager(int port) throws Exception {
loadProperties();
String resourceManagerClass = System.getProperty("resmgr.manager",
"org.apache.oodt.cas.resource.system.AvroRpcResourceManager");
String resourceManagerClass = System.getProperty(ResourceManager.RESMGR_SYSTEM_PROPERTY,
XmlRpcResourceManager.class.getName());

logger.info("Creating resource manager {} at port: {}", resourceManagerClass, port);

Expand All @@ -66,8 +69,8 @@ public static ResourceManager getResourceManager(int port) throws Exception {

public static ResourceManagerClient getResourceManagerClient(URL url) throws Exception {
loadProperties();
String resMgrClientClass = System.getProperty("resmgr.manager.client",
"org.apache.oodt.cas.resource.system.AvroRpcResourceManagerClient");
String resMgrClientClass = System.getProperty(ResourceManager.RESMGR_CLIENT_SYSTEM_PROPERTY,
XmlRpcResourceManagerClient.class.getName());

logger.info("Creating resource manager client {}", resMgrClientClass);

Expand Down
4 changes: 2 additions & 2 deletions resource/src/main/resources/resource.properties
View file
Expand Up @@ -17,8 +17,8 @@
# Properties required to configure the Resource Manager

# Client and server classes to be used as resource managers
resmgr.manager=org.apache.oodt.cas.resource.system.AvroRpcResourceManager
resmgr.manager.client=org.apache.oodt.cas.resource.system.AvroRpcResourceManagerClient
resmgr.manager=org.apache.oodt.cas.resource.system.XmlRpcResourceManager
resmgr.manager.client=org.apache.oodt.cas.resource.system.XmlRpcResourceManagerClient

# resource spark master
resource.runner.spark.host = mesos://<ip>:5050
Expand Down
8 changes: 7 additions & 1 deletion workflow/src/main/java/org/apache/oodt/cas/workflow/system/AvroRpcWorkflowManager.java
View file
Expand Up @@ -83,7 +83,13 @@ public AvroRpcWorkflowManager(int port){
port, System.getProperty("user.name", "unknown"));

Preconditions.checkArgument(port > 0, "Must specify a port greater than 0");


try {
loadProperties();
} catch (IOException e) {
logger.error("Error occurred when loading properties", e);
}

logger.debug("Getting workflow engine");
engine = getWorkflowEngineFromProperty();
if(engine == null){
Expand Down
5 changes: 5 additions & 0 deletions workflow/src/main/java/org/apache/oodt/cas/workflow/system/WorkflowManager.java
View file
Expand Up @@ -8,7 +8,12 @@
public interface WorkflowManager {

String PROPERTIES_FILE_PROPERTY = "org.apache.oodt.cas.workflow.properties";
String DEFAULT_PROPERTIES_FILE = "workflow.properties";

String WORKFLOW_ENGINE_FACTORY_PROPERTY = "workflow.engine.factory";
String WORKFLOW_SERVER_FACTORY_PROPERTY = "workflow.server.factory";
String WORKFLOW_CLIENT_FACTORY_PROPERTY = "workflow.client.factory";

String ENGINE_RUNNER_FACTORY_PROPERTY = "workflow.engine.runner.factory";
String WORKFLOW_REPOSITORY_FACTORY_PROPERTY = "workflow.repo.factory";
int DEFAULT_WEB_SERVER_PORT = 9001;
Expand Down
55 changes: 32 additions & 23 deletions workflow/src/main/java/org/apache/oodt/cas/workflow/system/rpc/RpcCommunicationFactory.java
View file
Expand Up @@ -37,34 +37,20 @@ public class RpcCommunicationFactory {

private static final Logger logger = LoggerFactory.getLogger(RpcCommunicationFactory.class);

private static String getRpcServerClassName() {
InputStream prpFileStream = RpcCommunicationFactory.class.getResourceAsStream("/workflow.properties");
Properties properties = new Properties();
try {
properties.load(prpFileStream);
} catch (IOException e) {
e.printStackTrace();
}
return properties.getProperty("workflow.server.factory",
"org.apache.oodt.cas.workflow.system.rpc.AvroRpcWorkflowManagerFactory");
private static String getRpcServerFactoryClassName() {
return loadProperties().getProperty(WorkflowManager.WORKFLOW_SERVER_FACTORY_PROPERTY,
XmlRpcWorkflowManagerFactory.class.getName());
}

private static String getRpcClientClassName() {
InputStream prpFileStream = RpcCommunicationFactory.class.getResourceAsStream("/workflow.properties");
Properties properties = new Properties();
try {
properties.load(prpFileStream);
} catch (IOException e) {
logger.error("Unable to load properties", e);
}
return properties.getProperty("workflow.client.factory",
"org.apache.oodt.cas.workflow.system.rpc.AvroRpcWorkflowManagerFactory");

private static String getRpcClientFactoryClassName() {
return loadProperties().getProperty(WorkflowManager.WORKFLOW_CLIENT_FACTORY_PROPERTY,
XmlRpcWorkflowManagerFactory.class.getName());
}

public static WorkflowManager createServer(int port) {
try {
WorkflowManagerFactory workflowManagerFactory =
(WorkflowManagerFactory) Class.forName(getRpcServerClassName()).newInstance();
(WorkflowManagerFactory) Class.forName(getRpcServerFactoryClassName()).newInstance();
workflowManagerFactory.setPort(port);
logger.debug("Using workflow manager server factory : {}", workflowManagerFactory.getClass());
return workflowManagerFactory.createServer();
Expand All @@ -77,7 +63,7 @@ public static WorkflowManager createServer(int port) {
public static WorkflowManagerClient createClient(URL url) {
try {
WorkflowManagerFactory workflowManagerFactory =
(WorkflowManagerFactory) Class.forName(getRpcClientClassName()).newInstance();
(WorkflowManagerFactory) Class.forName(getRpcClientFactoryClassName()).newInstance();
workflowManagerFactory.setUrl(url);
logger.debug("Using workflow manager client factory : {}", workflowManagerFactory.getClass());
return workflowManagerFactory.createClient();
Expand All @@ -86,4 +72,27 @@ public static WorkflowManagerClient createClient(URL url) {
throw new IllegalStateException("Unable to create client", e);
}
}

/**
* Loads workflow manager properties
*
* @return
*/
private static Properties loadProperties() {
String propertiesFile = System.getProperty(WorkflowManager.PROPERTIES_FILE_PROPERTY, WorkflowManager.DEFAULT_PROPERTIES_FILE);
InputStream prpFileStream = RpcCommunicationFactory.class.getResourceAsStream(propertiesFile);
Properties properties = new Properties();

if (prpFileStream != null) {
try {
properties.load(prpFileStream);
} catch (IOException e) {
logger.error("An error occurred when loading properties file: {}", propertiesFile, e);
}
} else {
logger.warn("Properties file: '{}' could not be found. Skipped loading properties", propertiesFile);
}

return properties;
}
}
4 changes: 2 additions & 2 deletions workflow/src/main/resources/workflow.properties
View file
Expand Up @@ -16,8 +16,8 @@
# Properties required to configure the Workflow Manager

#Rpc workflow communication class
workflow.server.factory = org.apache.oodt.cas.workflow.system.rpc.AvroRpcWorkflowManagerFactory
workflow.client.factory = org.apache.oodt.cas.workflow.system.rpc.AvroRpcWorkflowManagerFactory
workflow.server.factory = org.apache.oodt.cas.workflow.system.rpc.XmlRpcWorkflowManagerFactory
workflow.client.factory = org.apache.oodt.cas.workflow.system.rpc.XmlRpcWorkflowManagerFactory

# workflow repository factory
workflow.repo.factory = org.apache.oodt.cas.workflow.repository.XMLWorkflowRepositoryFactory
Expand Down

0 comments on commit 771c580

Please sign in to comment.