Documented how to start docker with external dbs

    - Added configuration parameters into setup system
  	- StoreOnlyLatestValue
    	- LocalAgentId
    	- EmbeddedAgent Registry db name
    - Optimized access to CouchDB for storing more than one ContextElement at time
    Implement bulk store data (both for historical and latest)
    Bug fixed on timewindow requests
    Fine grained XPath filter from attributeExpression
    First caching-enabled (only for lastValue) system
    Improve concurrency with FixedThreadPool
    Parameterized FixedThreadPool with external parameter
    Fixed problem with url encoding
    Refined some debugging log

IoTBroker-runner/iotbroker.conf.default

@@ -75,6 +75,9 @@ iotbroker_defaultthrottling=1000
 # Default subscription duration time
 iotbroker_defaultduration=31536000
 
+# System parameters
+iotbroker_updatethreadpoolsize=20
+
 # Ignore IoT Discovery failure (to be enabled if the IoT Discovery is not started)
 iotbroker_ignoreiotdiscoveryfailure=true
 
@@ -92,9 +95,12 @@ iotbroker_knowledgebaseport=8015
 ### Embedded agent configuration
 
 # CouchDB configurations
+iotbroker_embeddedagent_registrydbname="embeddedagentregistrydb"
 iotbroker_embeddedagent_couchdbname="historicalrepository"
 iotbroker_embeddedagent_couchdbcreatedb=true
 iotbroker_embeddedagent_couchdbprotocol="http"
 iotbroker_embeddedagent_couchdbhost="127.0.0.1"
 iotbroker_embeddedagent_couchdbport=5984
 iotbroker_embeddedagent_historicallyTrackQueryResponseAndNotifications=false
+iotbroker_embeddedagent_storeOnlyLatestValue=false
+iotbroker_embeddedagent_localagentid="embeddedagent1"

IoTBroker-runner/setup.sh

@@ -154,12 +154,17 @@ setPropertyIntoXML "timestampContextElement" "$iotbroker_timestampcontextelement
 setPropertyIntoXML "trackContextSource" "$iotbroker_trackcontextsource" "$iotbroker_configxml"
 setPropertyIntoXML "exposedAddress" "$iotbroker_exposedAddress" "$iotbroker_configxml"
 setPropertyIntoXML "historicallyTrackQueryResponseAndNotifications" "$iotbroker_embeddedagent_historicallyTrackQueryResponseAndNotifications" "$iotbroker_configxml"
+setPropertyIntoXML "updateThreadPoolSize" "$iotbroker_updatethreadpoolsize" "$iotbroker_configxml"
+setPropertyIntoXML "historicallyTrackQueryResponseAndNotifications" "$iotbroker_embeddedagent_historicallyTrackQueryResponseAndNotifications" "$iotbroker_configxml" 
 
 setPropertyIntoXML "couchdb_name" "$iotbroker_embeddedagent_couchdbname" "$iotbroker_embeddedagent_couchdbxml"
 setPropertyIntoXML "couchdb_createdb" "$iotbroker_embeddedagent_couchdbcreatedb" "$iotbroker_embeddedagent_couchdbxml"
 setPropertyIntoXML "couchdb_protocol" "$iotbroker_embeddedagent_couchdbprotocol" "$iotbroker_embeddedagent_couchdbxml"
 setPropertyIntoXML "couchdb_host" "$iotbroker_embeddedagent_couchdbhost" "$iotbroker_embeddedagent_couchdbxml"
 setPropertyIntoXML "couchdb_port" "$iotbroker_embeddedagent_couchdbport" "$iotbroker_embeddedagent_couchdbxml" 
+setPropertyIntoXML "storeOnlyLatestValue" "$iotbroker_embeddedagent_storeOnlyLatestValue" "$iotbroker_embeddedagent_couchdbxml" 
+setPropertyIntoXML "registrydb_name" "$iotbroker_embeddedagent_registrydbname" "$iotbroker_embeddedagent_couchdbxml" 
+setPropertyIntoXML "embeddedAgentId" "$iotbroker_embeddedagent_localagentid" "$iotbroker_embeddedagent_couchdbxml" 
 
 setPropertyIntoProperties "log4j.appender.ReportFileAppender.File" "$iotbroker_logfile" "$iotbroker_loggerproperties"
 setFirstPropertyValueOverMultipleValuesIntoProperties "log4j.rootLogger" "$iotbroker_loglevel" "$iotbroker_loggerproperties"

docker/README.md

@@ -25,4 +25,78 @@ docker run -t -p 8065:8065 -p 8060:8060 fiware/iotbroker:v5.4.3-standalone -p <i
 ```
 
 where *iotbroker_key* is one of the parameters available in the *IoTBroker_Runner/iotbroker.conf.default* and *confman_key* one of the parameters available in the *ConfMan_Runner/confman.conf.default*.
-Please note that such configurations are runtime properties and they will be forgotten the next time the docker is run.
+Please note that such configurations are runtime properties and they will be forgotten the next time the docker is run.
+
+Permanent Storage: CouchDB, PostgreSQL and HSQLDB
+---
+In order to handle permanent storage it is necessary to allow the docker image to access the CouchDB server (for the historical agent) and the PostgreSQL (for utility storage) into the docker host (or another machine).
+
+PLEASE NOTE: This configuration will expose postgres and couchdb to everybody, so please take care to fine tuning the configuration for an enhanced security.
+
+Configure **PostgreSQL**:
+
+Lets change the postgres user password.
+```
+sudo su
+su postgres
+psql
+ALTER USER postgres PASSWORD 'postgres'
+\q
+exit
+exit
+```
+Then we need to allow the docker image to access the postgreSQL database in the host machine. 
+First we need to check which is are the IP addresses of the virtual network created by the docker daemon:
+```
+$> ifconfig 
+docker0   Link encap:Ethernet  HWaddr 02:42:0e:f6:a6:67  
+          inet addr:172.17.0.1 Bcast:0.0.0.0  Mask:255.255.255.0
+          inet6 addr: fe80::42:eff:fef6:a667/64 Scope:Link
+          UP BROADCAST MULTICAST  MTU:1500  Metric:1
+          RX packets:157061 errors:0 dropped:0 overruns:0 frame:0
+          TX packets:202369 errors:0 dropped:0 overruns:0 carrier:0
+          collisions:0 txqueuelen:0 
+          RX bytes:10283839 (10.2 MB)  TX bytes:582227340 (582.2 MB)
+
+```
+Then we need to instruct postgreSQL to accept requests from this virtual network:
+```
+sudo echo -e 'host\tall\tall\t172.17.0.0/24\ttrust' >> /etc/postgresql/9.5/main/pg_hba.conf
+sudo sed -i "s/^port/listen_addresses = '\*'\nport/g" /etc/postgresql/9.5/main/postgresql.conf
+sudo service postgresql restart
+```
+
+Configure **CouchDB**:
+
+Similarly we need to allow docker to access couchdb:
+
+```
+sudo sed -i "s/;bind_address = 127.0.0.1/bind_address = 0\.0\.0\.0/g" "/etc/couchdb/local.ini"
+sudo service couchdb restart
+```
+
+Configure **HSQLDB**:
+
+First it is necessarry to copy the HSQLDB directory (https://github.com/Aeronbroker/Aeron/tree/master/SQL_database) that contains the database files somewhere in the host file system. We assume that it is located under /home/user/SQL_database.
+At this point we need to mount the HSQLDB folder into the docker container and instruct the IoT Broker in the container to access the mounted folder (with the -p option):
+```
+docker run -t -v /home/user/SQL_database/:/SQL_database nle/iotbroker:testing20170713 -p iotbroker_hsqldbdirectory="//SQL_database//database//linkDB"
+```
+
+Running docker with all permanent storages:
+
+```
+sudo docker run -t -p 8065:8065 -p 8060:8060  \
+-v /tmp/testhsqldbindocker:/testhsqldbindocker  \
+nle/iotbroker:testing20170713  \
+-p iotbroker_historicalagent="enabled" \
+-p iotbroker_embeddedagent_registrydbname="embeddedagentregistry" \
+-p iotbroker_embeddedagent_couchdbname="embeddedagenthistorical" \
+-p iotbroker_hsqldbdirectory="//SQL_database//database//linkDB"  \
+-p iotbroker_embeddedagent_couchdbhost="172.17.0.1" \
+-p confman_couchdbipandport="http://172.17.0.1:5984" \
+-p confman_couchdbregistercontextdbname="iotdiscoveryregistrations" \
+-p confman_couchdbsubscriptiondbname="iotdiscoverysubscriptions" \
+-p confman_postgresurl='//172.17.0.1/' \
+-p confman_postgresdbname="iotdiscoverypostgres" 
+```

eu.neclab.iotplatform.iotbroker.commons/src/main/java/eu/neclab/iotplatform/iotbroker/commons/FullHttpRequester.java

@@ -105,7 +105,7 @@ public static FullHttpResponse sendPost(URL url, String data,
 			}
 
 			if (logger.isDebugEnabled()) {
-				logger.info("\nSending 'POST' request to URL : " + url + "\n"
+				logger.debug("\nSending 'POST' request to URL : " + url + "\n"
 						+ "POST parameters : " + data + "\n");
 			}
 
@@ -422,10 +422,11 @@ public static FullHttpResponse sendPut(URL url, String data,
 
 				httpResponse.setBody(response.toString());
 
-				// logger.info("Response Code : " + responseCode);
-				logger.info("\nPUT to URL : " + url + "\nResponse Code : "
-						+ responseCode + "\n" + "Response : "
-						+ response.toString());
+				if (logger.isDebugEnabled()){
+					logger.debug("\nPUT to URL : " + url + "\nResponse Code : "
+							+ responseCode + "\n" + "Response : "
+							+ response.toString());
+				}
 
 				con.disconnect();
 				return httpResponse;
@@ -460,11 +461,12 @@ public static FullHttpResponse sendDelete(URL url, String xAuthToken)
 				con.setRequestProperty("X-Auth-Token", xAuthToken);
 			}
 
-			logger.info("\nSending 'DELETE' request to URL : " + url);
 
 			int responseCode = con.getResponseCode();
-			logger.info("\nDELETE to URL : " + url + "\nResponse Code : "
-					+ responseCode + "\n");
+			if (logger.isDebugEnabled()){
+				logger.debug("\nDELETE to URL : " + url + "\nResponse Code : "
+						+ responseCode + "\n");
+			}
 
 			httpResponse = new FullHttpResponse(HttpVersion.HTTP_1_0,
 					con.getResponseCode(), con.getResponseMessage());

eu.neclab.iotplatform.iotbroker.commons/src/main/java/eu/neclab/iotplatform/iotbroker/commons/interfaces/EmbeddedAgentStorageInterface.java

@@ -47,6 +47,7 @@
 import java.net.URI;
 import java.util.Date;
 import java.util.List;
+import java.util.Map;
 
 import eu.neclab.iotplatform.ngsi.api.datamodel.ContextElement;
 import eu.neclab.iotplatform.ngsi.api.datamodel.EntityId;
@@ -60,13 +61,15 @@ public interface EmbeddedAgentStorageInterface {
 	// @Override
 	public void setNgsi10Callback(Ngsi10Interface ngsi10Callback);
 
-	public void storeLatestData(ContextElement isolatedContextElement);
+	public Map<ContextElement, Boolean> storeData(List<ContextElement> isolatedLatestContextElement,
+			List<ContextElement> isolatedHistoricalContextElement,
+			Date defaultDate);
+
+	public boolean storeLatestData(ContextElement isolatedContextElement);
 
 	public void storeHistoricalData(ContextElement isolatedContextElement,
 			Date defaultDate);
 
-	// @Override
-
 	public List<ContextElement> getAllLatestValues();
 
 	public ContextElement getLatestValue(String id, URI type,

eu.neclab.iotplatform.iotbroker.commons/src/main/java/eu/neclab/iotplatform/iotbroker/commons/interfaces/IoTAgentInterface.java

@@ -52,7 +52,7 @@
 import eu.neclab.iotplatform.ngsi.api.datamodel.ContextRegistration;
 import eu.neclab.iotplatform.ngsi.api.datamodel.DiscoverContextAvailabilityResponse;
 import eu.neclab.iotplatform.ngsi.api.datamodel.EntityId;
-import eu.neclab.iotplatform.ngsi.api.datamodel.QueryContextRequest;
+import eu.neclab.iotplatform.ngsi.api.datamodel.StatusCode;
 import eu.neclab.iotplatform.ngsi.api.datamodel.SubscribeContextRequest;
 import eu.neclab.iotplatform.ngsi.api.ngsi10.Ngsi10Interface;
 
@@ -62,7 +62,7 @@
  */
 public interface IoTAgentInterface {
 
-	void storeData(List<ContextElement> contextElementList);
+	StatusCode storeData(List<ContextElement> contextElementList);
 
 	ContextElement getLatestValue(String id, URI type, String attributeName);
 

eu.neclab.iotplatform.iotbroker.commons/src/main/java/eu/neclab/iotplatform/iotbroker/commons/interfaces/KeyValueStoreInterface.java

@@ -46,6 +46,7 @@
 
 import java.util.Collection;
 import java.util.List;
+import java.util.Map;
 
 import com.google.common.collect.Multimap;
 
@@ -57,15 +58,23 @@ public interface KeyValueStoreInterface {
 
 	boolean storeValue(String key, ContextElement contextElement);
 
+	boolean storeValue(String key, ContextElement contextElement,
+			boolean cacheAfterStore);
+
+	public Map<String, Boolean> storeAndUpdateValues(
+			Map<String, ContextElement> keyValuesToStore,
+			Map<String, ContextElement> keyValuesToUpdate,
+			boolean cacheAfterStoring);
+
 	public Collection<String> getKeys(String startKey, String endKey);
 
 	public Multimap<String, String> getIdsByType();
 
 	ContextElement getValue(String latestValueDocumentKey);
 
 	ContextElement getValues(String startKey, String endKey);
-	
-	List<ContextElement> getValues(List<String> keys);
+
+	Collection<ContextElement> getValues(List<String> keys);
 
 	List<ContextElement> getAllValues(String startKey, String endKey);