Merge pull request #355 from poorva1209/master

Updated log manager and related docs

applications/.project

@@ -5,7 +5,13 @@
 	<projects>
 	</projects>
 	<buildSpec>
+		<buildCommand>
+			<name>org.python.pydev.PyDevBuilder</name>
+			<arguments>
+			</arguments>
+		</buildCommand>
 	</buildSpec>
 	<natures>
+		<nature>org.python.pydev.pythonNature</nature>
 	</natures>
 </projectDescription>

applications/vvo.config

@@ -1 +1,13 @@
-{"id":"vvo","description":"VVO app","creator":"pnnl","inputs":["/topic/goss.gridappsd.fncs.output"],"outputs":["/topic/goss.gridappsd.fncs.input"],"options":"SIMULATION_ID","execution_path":"app/vvoapp.py","type":"PYTHON","launch_on_startup":false,"prereqs":["fncs-goss-bridge"],"multiple_instances":true}
+{
+	"id":"vvo",
+	"description":"VVO app",
+	"creator":"pnnl",
+	"inputs":["/topic/goss.gridappsd.fncs.output"],
+	"outputs":["/topic/goss.gridappsd.fncs.input"]
+	,"options":"SIMULATION_ID",
+	"execution_path":"app/vvoapp.py",
+	"type":"PYTHON",
+	"launch_on_startup":false,
+	"prereqs":["fncs-goss-bridge"],
+	"multiple_instances":true
+}

docs/.project

@@ -0,0 +1,11 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<projectDescription>
+	<name>docs</name>
+	<comment></comment>
+	<projects>
+	</projects>
+	<buildSpec>
+	</buildSpec>
+	<natures>
+	</natures>
+</projectDescription>

docs/source/developer_resources/Developing_Apps.rst

@@ -0,0 +1,45 @@
+
+Supported Application Types
+^^^^^^^^^^^^^^^^^^^^^^^^^^^
+- Python
+- Java (Jar)
+
+Registering Application With Platform
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Assumptions: GOSS-GridAPPS-D repository (https://github.com/GRIDAPPSD/GOSS-GridAPPS-D.git) is cloned under [ROOT_DIR]
+
+1. Create a [app_name].config file in JSON format with keys and values as described below. where app_name should be unique for the application.
+
+::
+	
+	{
+		"id":"app_name",
+		"description":"This is desxription of the app",
+		"creator":"orgnization name",
+		"inputs":["topic.goss.gridappsd.input1", "topic.goss.gridappsd.input2", ..],
+		"outputs":["topic.goss.gridappsd.output1", "topic.goss.gridappsd.output2", ..],
+		"options":"space saperated command line input options",
+		"execution_path":"absolute/execution/path",
+		"type":"PYTHON|JAVA",
+		"launch_on_startup":true|false,
+		"prereqs":["other_app","other_service",..],
+		"multiple_instances":true|false
+	}
+
+2. Put [app_name].config file in applications folder under cloned repository location
+
+3. Put your application under applications/[app_name] folder under cloned repository location as shown below.
+
+::
+	
+	applications
+		[app_name]
+			app
+				Your application goes here
+			test
+				Test scripts for your application goes here.
+				
+
+See Using GridAPPSD section for details on starting a simulation from an application and communicating with plaform.
+It also has an example in Python and Java to start a simulation.

docs/source/developer_resources/Execution_Workflow.rst

@@ -14,3 +14,7 @@ Simulator - In this case GridLAB-D serves as the simulator.
 
 Hosted Application - Applications can be developed to use the data generated by the simulation and submit feedback and updates to the simulator.  Two examples of this have been developed in RC1, the VoltVar application and a vizualization application
 
+Log Manager - Process Manager recieves a log message. It retrieves the username associated with the message and forwards the message and username to Log Manager. Log Manager writes the message on a file and if store_to_db key is true in log message then log manager calls the data manager to store the log message in the database.
+
+
+

docs/source/developer_resources/index.rst

@@ -9,8 +9,7 @@ For developing application for GridAPPS-D platform see :ref:`using` .
 
 Developing Applications using GridAPPS-D
 ----------------------------------------
-
-This section is under development. Not available for current release of the platform.
+.. include:: Developing_Apps.rst
 
 Eclipse IDE Setup
 -----------------

docs/source/using_gridappsd/logging_status.rst

@@ -20,10 +20,12 @@ Message structure:
 .. code-block:: console
 
 	{
-		timestamp: "",
-		status: "[started|stopped|running|error|passed|failed]",
-		log_message: "",
-		log_level: "[info|debug|error]"
+		"process_id": ""
+		"timestamp": "",
+		"process_status": "[started|stopped|running|error|passed|failed]",
+		"log_message": "",
+		"log_level": "[info|debug|error]",
+		"store_to_db": [true|false]
 	}
 
 Receving multiple logs:

gov.pnnl.goss.gridappsd.test/src/gov/pnnl/goss/gridappsd/test/LogManagerTest.java

@@ -0,0 +1,67 @@
+package gov.pnnl.goss.gridappsd.test;
+
+import gov.pnnl.goss.gridappsd.dto.LogMessage;
+import gov.pnnl.goss.gridappsd.dto.LogMessage.LogLevel;
+import gov.pnnl.goss.gridappsd.dto.LogMessage.ProcessStatus;
+
+import java.io.Serializable;
+
+import javax.jms.JMSException;
+
+import static org.junit.Assert.assertNotNull;
+
+import org.apache.http.auth.Credentials;
+import org.apache.http.auth.UsernamePasswordCredentials;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.runners.MockitoJUnitRunner;
+
+import pnnl.goss.core.Client;
+import pnnl.goss.core.Client.PROTOCOL;
+import pnnl.goss.core.ClientFactory;
+import pnnl.goss.core.DataResponse;
+import pnnl.goss.core.GossResponseEvent;
+import pnnl.goss.core.client.ClientServiceFactory;
+
+@RunWith(MockitoJUnitRunner.class)
+public class LogManagerTest {
+
+	Client client; 
+
+	@Before
+	public void setup() throws Exception{
+		ClientFactory clientFactory = new ClientServiceFactory();
+		Credentials credentials = new UsernamePasswordCredentials(
+						"system", "manager");
+	 	client = clientFactory.create(PROTOCOL.STOMP, credentials);
+	}
+
+	@Test
+	public void sendLogMessage() throws JMSException{
+
+		String destination = "goss.gridappsd.process.log";
+
+		String process_id = "test";
+		long timestamp = System.currentTimeMillis();
+		String log_message = "this is a test";
+		LogLevel log_level = LogLevel.DEBUG;
+		ProcessStatus process_status = ProcessStatus.RUNNING;
+		Boolean storeToDB = true;
+		LogMessage logMessage = new LogMessage(process_id, timestamp, log_message, log_level, process_status, storeToDB);
+
+		String id = client.getResponse(logMessage, destination, null).toString();
+
+		client.subscribe("goss.gridappsd.response.data."+id, new GossResponseEvent() {
+
+			@Override
+			public void onMessage(Serializable message) {
+				DataResponse response = (DataResponse)message;
+				assertNotNull(response.getData());
+
+			}
+		});
+
+	}
+
+}

gov.pnnl.goss.gridappsd/conf/pnnl.goss.gridappsd.cfg

@@ -2,8 +2,8 @@ fncs.path = fncs_broker
 gridlabd.path = gridlabd
 gridappsd.temp.path = \tmp\gridappsd_tmp
 fncs.bridge.path = .\scripts\goss_fncs_bridge.py
-applications.path = applications
-services.path = services
+applications.path = ../applications
+services.path = ../services
 blazegraph.host.path = http://localhost:9999
 
 

gov.pnnl.goss.gridappsd/src/gov/pnnl/goss/gridappsd/api/AppManager.java

@@ -49,8 +49,7 @@
 
 public interface AppManager {
 
-	void process(StatusReporter statusReporter,
-			int processId, DataResponse event, Serializable message) throws Exception;
+	void process(int processId, DataResponse event, Serializable message) throws Exception;
 
 	void registerApp(AppInfo appInfo, byte[] appPackage) throws Exception;
 

gov.pnnl.goss.gridappsd/src/gov/pnnl/goss/gridappsd/api/LogDataManager.java

@@ -44,9 +44,9 @@
 
 public interface LogDataManager {
 
-	void store (String process_id, String username, long timestamp,
-			String log_message, LogLevel log_level, ProcessStatus process_status);
+	void store (String process_id, long timestamp,
+			String log_message, LogLevel log_level, ProcessStatus process_status, String username);
 
-	void query(String process_id, long timestamp, LogLevel log_level, ProcessStatus process_status, String username);
+	void query(String process_id, long timestamp, LogLevel log_level, ProcessStatus process_status, String username, String resultTopic, String logTopic);
 
 }

gov.pnnl.goss.gridappsd/src/gov/pnnl/goss/gridappsd/api/LogManager.java

@@ -43,10 +43,19 @@
 
 public interface LogManager {
 
+	/**
+	 * Implementation of this method should writes the message in log file. 
+	 * And calls LogDataManager to save the log  message in data store if 
+	 * store_to_db is true in  LogMessage object.  
+	 * @param message an Object of gov.pnnl.goss.gridappsd.dto.LogMessage
+	 * @param username username of the user logging the message
+	 */
+	void log(LogMessage message, String username);
 
-	void log(LogMessage message);
-
-
-	void get(LogMessage message);
+	/**
+	 * Implementation of this method should call an implementation of LogDataManager and get the log messages 
+	 * from data store based on the not null values in LogMessage object.
+	 */
+	void get(LogMessage message, String outputTopics, String LogTopic);
 
 }

gov.pnnl.goss.gridappsd/src/gov/pnnl/goss/gridappsd/app/AppManagerImpl.java

@@ -66,6 +66,7 @@
 import javax.jms.Destination;
 
 
+
 import org.apache.felix.dm.annotation.api.Component;
 import org.apache.felix.dm.annotation.api.ConfigurationDependency;
 import org.apache.felix.dm.annotation.api.ServiceDependency;
@@ -124,6 +125,8 @@ public class AppManagerImpl implements AppManager{
 
 	private HashMap<String, AppInstance> appInstances = new HashMap<String, AppInstance>();
 
+	private String username;
+
 	private Client client;
 
 	public AppManagerImpl() {
@@ -136,7 +139,12 @@ public AppManagerImpl(StatusReporter statusReporter, LogManager logManager, Clie
 	}
 
 	@Override
-	public void process(StatusReporter statusReporter, int processId, DataResponse event, Serializable message) throws Exception {
+	public void process(int processId, DataResponse event, Serializable message) throws Exception {
+
+
+		//TODO:Get username from message's metadata e.g. event.getUserName()
+		username  = GridAppsDConstants.username;
+
 		if(client==null){
 			Credentials credentials = new UsernamePasswordCredentials(
 					GridAppsDConstants.username, GridAppsDConstants.password);
@@ -214,7 +222,7 @@ public void start(){
 				"Starting "+this.getClass().getName(), 
 				LogLevel.INFO, 
 				ProcessStatus.RUNNING, 
-				true));
+				true),GridAppsDConstants.username);
 
 		scanForApps();
 
@@ -223,7 +231,7 @@ public void start(){
 				String.format("Found %s applications", apps.size()), 
 				LogLevel.INFO, 
 				ProcessStatus.RUNNING, 
-				true));
+				true),GridAppsDConstants.username);
 	}
 
 	protected void scanForApps(){
@@ -544,7 +552,7 @@ protected AppInfo parseAppInfo(File appConfigFile){
 			String appConfigStr = new String(Files.readAllBytes(appConfigFile.toPath()));
 			appInfo = AppInfo.parse(appConfigStr);
 		} catch (IOException e) {
-			logManager.log(new LogMessage("App Manager",new Date().getTime(), "Error while reading app config file: "+e.getMessage(), LogLevel.ERROR, ProcessStatus.ERROR, false));
+			logManager.log(new LogMessage("App Manager",new Date().getTime(), "Error while reading app config file: "+e.getMessage(), LogLevel.ERROR, ProcessStatus.ERROR, false),username);
 		}
 
 		return appInfo;
@@ -558,7 +566,7 @@ protected void writeAppInfo(AppInfo appInfo){
 		try {
 			Files.write(confFile.toPath(), appInfo.toString().getBytes());
 		} catch (IOException e) {
-			logManager.log(new LogMessage("App Manager", new Date().getTime(), "Error while writing app config file: "+e.getMessage(), LogLevel.ERROR, ProcessStatus.ERROR, false));
+			logManager.log(new LogMessage("App Manager", new Date().getTime(), "Error while writing app config file: "+e.getMessage(), LogLevel.ERROR, ProcessStatus.ERROR, false),username);
 		}
 	}
 
@@ -585,14 +593,14 @@ public void run() {
 	            try {
 	                while ((line = input.readLine()) != null) {
 	                	System.out.println("APPRECEIVED "+line);
-	                	logManager.log(new LogMessage(appInstance.getRequest_id(), new Date().getTime(), line, LogLevel.INFO, ProcessStatus.RUNNING, false));
+	                	logManager.log(new LogMessage(appInstance.getRequest_id(), new Date().getTime(), line, LogLevel.INFO, ProcessStatus.RUNNING, false), username);
 
 //	                    log.info(processName+": "+line);
 	                }
 	            } catch (IOException e) {
 	            	e.printStackTrace();
 //	                log.error("Error on process "+processName, e);
-                	logManager.log(new LogMessage(appInstance.getRequest_id(), new Date().getTime(), e.getMessage(), LogLevel.ERROR, ProcessStatus.ERROR, false));
+                	logManager.log(new LogMessage(appInstance.getRequest_id(), new Date().getTime(), e.getMessage(), LogLevel.ERROR, ProcessStatus.ERROR, false), username);
 
 	            }
 	        }

gov.pnnl.goss.gridappsd/src/gov/pnnl/goss/gridappsd/configuration/ConfigurationManagerImpl.java

@@ -135,7 +135,7 @@ public synchronized File getSimulationFile(int simulationId, RequestSimulation p
 
 		if(resp!=null && (resp instanceof DataResponse) && (((DataResponse)resp).getData())!=null && (((DataResponse)resp).getData() instanceof File)){
 			//Update simulation status after every step, for example:
-			statusReporter.reportStatus(GridAppsDConstants.topic_simulationStatus+simulationId, "Simulation files created");
+			statusReporter.reportStatus(GridAppsDConstants.topic_simulationLog+simulationId, "Simulation files created");
 			return (File)((DataResponse)resp).getData();
 		}