runtime south: remove plugin subdirectory

calvin/Tools/csruntime.py

@@ -224,7 +224,7 @@ def set_config_from_args(args):
 
 def discover(timeout=2, retries=5):
     import struct
-    from calvin.runtime.south.plugins.storage.twistedimpl.dht.service_discovery_ssdp import SSDPServiceDiscovery,\
+    from calvin.runtime.south.storage.twistedimpl.dht.service_discovery_ssdp import SSDPServiceDiscovery,\
                                                                                             SERVICE_UUID,\
                                                                                             CA_SERVICE_UUID,\
                                                                                             SSDP_ADDR,\
@@ -266,7 +266,7 @@ def runtime_certificate(rt_attributes):
     from calvin.utilities import runtime_credentials
     from calvin.utilities import certificate
     from calvin.utilities import certificate_authority
-    from calvin.runtime.south.plugins.storage.twistedimpl.dht.service_discovery_ssdp import parse_http_response
+    from calvin.runtime.south.storage.twistedimpl.dht.service_discovery_ssdp import parse_http_response
     global _conf
     global _log
     _conf = calvinconfig.get()
@@ -374,11 +374,11 @@ def start_gui(interface4, port):
   extras_path = os.path.dirname(inspect.getfile(calvinextras))
   # build path to gui files
   gui_path = os.path.join(extras_path, "CalvinGUI", "Build", "GUI")
-  gui_config_path =  os.path.join(extras_path, "CalvinGUI", "calvin.conf") 
+  gui_config_path =  os.path.join(extras_path, "CalvinGUI", "calvin.conf")
   # Patch config
   _conf = calvinconfig.get()
   delta_config = _conf.config_at_path(gui_config_path)
-  _conf.update_config(delta_config)    
+  _conf.update_config(delta_config)
   # Add endpoint to twisted reactor
   resource = File(gui_path)
   factory = Site(resource)

calvin/Tools/run_test.sh

@@ -46,7 +46,7 @@ Options:
  Args:
   <ip:s>		Ip numbers to the host to start runtimes at.
 EOF
-} 
+}
 
 while [[ $# > 0 ]]
 do
@@ -66,7 +66,7 @@ case $key in
     GIT_COMMIT=$2
     shift # past argument
     shift # past argument
-    break 
+    break
     ;;
     --loglevel)
     LOG_LEVEL=$2
@@ -133,7 +133,7 @@ if [ "$MASTER" != "local" ]; then
 	pushgit $SSHKEY $MASTER $MRTEMPDIR
 	#pushfile $SSHKEY
 	# Execute us remote
-	[ ! -z "$RUN_TEST" ] && RUN_TEST="-t" 
+	[ ! -z "$RUN_TEST" ] && RUN_TEST="-t"
 	ssh -i $SSHKEY $MASTER "cd $MRTEMPDIR && bash $MRTEMPDIR/calvin/Tools/$SCRIPTNAME $RUN_TEST -s $(git_head) $IPS"
 	exit $?
 else
@@ -148,7 +148,7 @@ fi
 DEF_IF=$(netstat -nr | grep -E "^default|^0.0.0.0" | awk '{print $NF}' )
 [ -z "$MYIP" ] && MYIP=$(ifconfig $DEF_IF | grep 'inet ' | sed 's/addr://' | awk '{ print $2}')
 
-DEBUG_FLAGS="--loglevel=calvin.tests=DEBUG --loglevel=calvin.runtime.south.plugins.storage.twistedimpl.dht:DEBUG --loglevel=calvin.runtime.north.storage:DEBUG $LOG_LEVEL"
+DEBUG_FLAGS="--loglevel=calvin.tests=DEBUG --loglevel=calvin.runtime.south.storage.twistedimpl.dht:DEBUG --loglevel=calvin.runtime.north.storage:DEBUG $LOG_LEVEL"
 
 export CALVIN_TEST_UUID=$($UUID)
 export CALVIN_GLOBAL_DHT_NETWORK_FILTER=\"$($UUID)\"
@@ -163,7 +163,7 @@ for node in $IPS
 do
 	PORT=$(get_random_port)
 	PORT_DHT=$(get_random_port)
-	
+
 	# Do this in parallell and wait later
 	sh -c "$SCRIPTDIR/setup_node.sh $GIT_COMMIT $node $CALVIN_TEST_UUID $PORT $CALVIN_GLOBAL_DHT_NETWORK_FILTER $MYIP $SSHKEY $DEBUG_FLAGS 2>&1 | tee $node-log.txt > /dev/null" &
 	PID=$!
@@ -179,14 +179,14 @@ declare -a CALVIN_NODES
 for node in ${NODES[@]}
 do
 	IFS=';' read -a INFO <<< "$node"
-	
-	kill -0 ${INFO[2]} 2> /dev/null 
+
+	kill -0 ${INFO[2]} 2> /dev/null
 	if [ $? -ne 0 ]; then
 		echo "Calvin node at ${INFO[0]} did not start correctly"
 		RET=1
-		break 
+		break
 	fi
-	
+
 	# Wait for it to start
 	CALVIN_TEST_NODE=$(nc -l ${INFO[1]})
 	if [ "$CALVIN_TEST_NODE" == "FALSE" ]; then
@@ -204,7 +204,7 @@ if [ $RET -ne 0 ]; then
 	for node in ${NODES[@]}
 	do
 		IFS=';' read -a INFO <<< "$node"
-		kill ${INFO[2]} 2> /dev/null 
+		kill ${INFO[2]} 2> /dev/null
 	done
 	exit $RET
 

calvin/actor/actor.py

@@ -26,7 +26,7 @@
 # from calvin.runtime.north import calvincontrol
 from calvin.runtime.north.replicationmanager import ReplicationId
 import calvin.requests.calvinresponse as response
-from calvin.runtime.south.plugins.async import async
+from calvin.runtime.south.async import async
 from calvin.runtime.north.plugins.authorization_checks import check_authorization_plugin_list
 from calvin.utilities.calvin_callback import CalvinCB
 from calvin.csparser.port_property_syntax import get_port_property_capabilities, get_port_property_runtime
@@ -778,7 +778,7 @@ def requirements_get(self):
         else:
             capability_require = []
 
-        return (self._deployment_requirements + capability_require + 
+        return (self._deployment_requirements + capability_require +
                 capability_port + self._replication_id._placement_req)
 
     def _derive_port_property_capabilities(self):

calvin/runtime/north/actormanager.py

@@ -18,7 +18,7 @@
 from calvin.actorstore.store import ActorStore
 from calvin.utilities import dynops
 from calvin.utilities.requirement_matching import ReqMatch
-from calvin.runtime.south.plugins.async import async
+from calvin.runtime.south.async import async
 from calvin.utilities.calvinlogger import get_logger
 from calvin.utilities.calvin_callback import CalvinCB
 import calvin.requests.calvinresponse as response

calvin/runtime/north/appmanager.py

@@ -23,7 +23,7 @@
 import calvin.requests.calvinresponse as response
 from calvin.utilities import calvinuuid
 from calvin.actorstore.store import ActorStore, GlobalStore
-from calvin.runtime.south.plugins.async import async
+from calvin.runtime.south.async import async
 from calvin.utilities.security import Security
 from calvin.utilities.requirement_matching import ReqMatch
 
@@ -221,7 +221,7 @@ def _destroy(self, application, cb):
                     # Destroy the replicas
                     application.replication_ids.append(replication_id)
                     self._node.storage.get_replica(
-                        replication_id, 
+                        replication_id,
                         cb=CalvinCB(func=self._replicas_cb, replication_id=replication_id,
                                     master_id=self._node.am.actors[actor_id]._replication_id.original_actor_id,
                                     application=application))
@@ -686,11 +686,11 @@ def check_requirements_and_sec_policy(self, actor_name, info, actor_def=None, cb
         """
         try:
             if not 'shadow_actor' in info:
-                self.node.am.check_requirements_and_sec_policy(info['requires'], 
-                                                               security=self.sec, 
+                self.node.am.check_requirements_and_sec_policy(info['requires'],
+                                                               security=self.sec,
                                                                signer=info['signer'],
-                                                               callback=CalvinCB(self.instantiate, 
-                                                                                 actor_name, info, 
+                                                               callback=CalvinCB(self.instantiate,
+                                                                                 actor_name, info,
                                                                                  actor_def, cb=cb))
                 return
             self.instantiate(actor_name, info, cb=cb)
@@ -722,8 +722,8 @@ def instantiate(self, actor_name, info, actor_def=None, access_decision=None, cb
                 port_properties = None
             info['args']['name'] = actor_name
             # TODO add requirements should be part of actor_manager new
-            actor_id = self.node.am.new(actor_type=info['actor_type'], args=info['args'], signature=info['signature'], 
-                                        actor_def=actor_def, security=self.sec, access_decision=access_decision, 
+            actor_id = self.node.am.new(actor_type=info['actor_type'], args=info['args'], signature=info['signature'],
+                                        actor_def=actor_def, security=self.sec, access_decision=access_decision,
                                         shadow_actor='shadow_actor' in info, port_properties=port_properties)
             if not actor_id:
                 raise Exception("Could not instantiate actor %s" % actor_name)

calvin/runtime/north/calvin_network.py

@@ -22,15 +22,15 @@
 from calvin.utilities import calvinuuid
 from calvin.utilities.calvin_callback import CalvinCB
 import calvin.requests.calvinresponse as response
-from calvin.runtime.south.plugins.async import async
+from calvin.runtime.south.async import async
 from calvin.utilities import calvinlogger
 from calvin.utilities import calvinconfig
 _log = calvinlogger.get_logger(__name__)
 _conf = calvinconfig.get()
 
 # FIXME should be read from calvin config
-TRANSPORT_PLUGIN_PATH = os.path.join(os.path.dirname(os.path.dirname(__file__)), *['south', 'plugins', 'transports'])
-TRANSPORT_PLUGIN_NS = "calvin.runtime.south.plugins.transports"
+TRANSPORT_PLUGIN_PATH = os.path.join(os.path.dirname(os.path.dirname(__file__)), *['south', 'transports'])
+TRANSPORT_PLUGIN_NS = "calvin.runtime.south.transports"
 
 
 class CalvinBaseLink(object):

calvin/runtime/north/calvin_node.py

@@ -36,7 +36,7 @@
 from calvin.runtime.north.calvin_network import CalvinNetwork
 from calvin.runtime.north.calvin_proto import CalvinProto
 from calvin.runtime.north.portmanager import PortManager
-from calvin.runtime.south.plugins.async import async
+from calvin.runtime.south.async import async
 from calvin.utilities.attribute_resolver import AttributeResolver
 from calvin.utilities.calvin_callback import CalvinCB
 from calvin.utilities.security import security_modules_check

calvin/runtime/north/calvincontrol.py

@@ -19,7 +19,7 @@
 from urlparse import urlparse
 from calvin.utilities.calvinlogger import get_logger
 from calvin.utilities.calvin_callback import CalvinCB
-from calvin.runtime.south.plugins.async import server_connection
+from calvin.runtime.south.async import server_connection
 from calvin.requests import calvinresponse
 from calvin.utilities.security import Security
 from calvin.utilities import calvinuuid

calvin/runtime/north/control_apis/runtime_api.py

@@ -18,7 +18,7 @@
 from calvin.requests import calvinresponse
 from calvin.utilities.calvinlogger import get_logger
 from calvin.utilities.calvin_callback import CalvinCB
-from calvin.runtime.south.plugins.async import async
+from calvin.runtime.south.async import async
 from routes import handler, register
 from authentication import authentication_decorator
 from calvin.runtime.north.calvinsys import get_calvinsys

calvin/runtime/north/plugins/storage/proxy/__init__.py

@@ -15,7 +15,7 @@
 # limitations under the License.
 
 from calvin.runtime.north.plugins.storage.storage_base import StorageBase
-from calvin.runtime.south.plugins.async import async
+from calvin.runtime.south.async import async
 from calvin.utilities import calvinlogger
 from calvin.utilities import calvinconfig
 from calvin.utilities.calvin_callback import CalvinCB
@@ -60,15 +60,15 @@ def _got_link(self, master_id, org_cb):
         self.tunnel.register_tunnel_down(CalvinCB(self.tunnel_down, org_cb=org_cb))
         self.tunnel.register_tunnel_up(CalvinCB(self.tunnel_up, org_cb=org_cb))
         self.tunnel.register_recv(self.tunnel_recv_handler)
-        
+
     def _start_link_cb(self, status, uri, peer_node_id, org_cb):
         _log.analyze(self.node.id, "+", {'status': str(status)}, peer_node_id=peer_node_id)
 
         _log.info("status: {}, {}".format(status, str(status)))
 
         if status != 200:
             self.retries += 1
-                
+
             if self.max_retries - self.retries != 0:
                 delay = 0.5 * self.retries if self.retries < 20 else 10
                 _log.info("Link to proxy failed, retrying in {}".format(delay))

calvin/runtime/north/plugins/storage/storage_dict_local.py

@@ -14,7 +14,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from calvin.runtime.south.plugins.async import async
+from calvin.runtime.south.async import async
 from calvin.requests import calvinresponse
 import itertools
 

calvin/runtime/north/plugins/storage/storage_factory.py

@@ -16,7 +16,7 @@
 
 
 # Parsers
-from calvin.runtime.south.plugins.storage import dht, securedht, sql
+from calvin.runtime.south.storage import dht, securedht, sql
 from calvin.runtime.north.plugins.storage.proxy import StorageProxy
 from calvin.runtime.north.plugins.storage.storage_dict_local import StorageLocal
 

calvin/runtime/north/replicationmanager.py

@@ -25,7 +25,7 @@
 from calvin.utilities.requirement_matching import ReqMatch
 from calvin.utilities.replication_defs import REPLICATION_STATUS, PRE_CHECK
 from calvin.actorstore.store import GlobalStore
-from calvin.runtime.south.plugins.async import async
+from calvin.runtime.south.async import async
 from calvin.runtime.north.plugins.requirements import req_operations
 from calvin.actor.actorport import PortMeta
 from calvin.runtime.north.plugins.port import DISCONNECT
@@ -404,7 +404,7 @@ def _silent(*args, **kwargs):
                 status = kwargs.get("status", args[0])
                 _log.debug("destroy_replication_leader log cb %s status=%s" % (replication_id, status))
             try:
-                self.node.proto.leader_elected(peer_node_id=value['leader_node_id'], leader_type="replication", cmd="destroy", 
+                self.node.proto.leader_elected(peer_node_id=value['leader_node_id'], leader_type="replication", cmd="destroy",
                                                 data=replication_id, callback=_silent if cb is None else cb)
             except:
                 _log.exception("fail destroy_replication_leader _leader_node_cb %s" % replication_id)
@@ -744,7 +744,7 @@ def dereplicate(self, replication_id, callback, exhaust=False):
         if last_replica_id in self.node.am.actors:
             self.node.am.destroy_with_disconnect(last_replica_id, terminate=terminate,
                 callback=CalvinCB(self._dereplicated, replication_data=replication_data,
-                                    last_replica_id=last_replica_id, 
+                                    last_replica_id=last_replica_id,
                                     node_id=self.node.id, cb=cb_status))
         else:
             self.node.storage.get_actor(last_replica_id,
@@ -757,7 +757,7 @@ def _dereplicate_actor_cb(self, key, value, replication_data, terminate, cb):
         if calvinresponse.isnotfailresponse(value) and 'node_id' in value:
             # Use app destroy since it can remotely destroy actors
             self.node.proto.app_destroy(value['node_id'],
-                CalvinCB(self._dereplicated, replication_data=replication_data, last_replica_id=key, 
+                CalvinCB(self._dereplicated, replication_data=replication_data, last_replica_id=key,
                             node_id=value['node_id'], cb=cb),
                 None, [key], disconnect=terminate, replication_id=replication_data.id)
         else:
@@ -831,12 +831,12 @@ def replication_loop(self):
                 pre_check = PRE_CHECK.NO_OPERATION
             if pre_check == PRE_CHECK.SCALE_OUT:
                 _log.info("Auto-replicate")
-                self.replicate_by_requirements(replication_data, 
+                self.replicate_by_requirements(replication_data,
                     CalvinCB(self._replication_loop_log_cb, replication_id=replication_data.id))
                 replication_data._missing_replica_time = time.time() + 10  # Don't check missing for a while
             if pre_check == PRE_CHECK.SCALE_OUT_KNOWN:
                 _log.info("Auto-replicate known")
-                self.replicate_by_known_placement(replication_data, 
+                self.replicate_by_known_placement(replication_data,
                     CalvinCB(self._replication_loop_log_cb, replication_id=replication_data.id))
                 replication_data._missing_replica_time = time.time() + 10  # Don't check missing for a while
             elif pre_check == PRE_CHECK.SCALE_IN:

calvin/runtime/north/resource_monitor/cpu.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 
-from calvin.runtime.south.plugins.async import async
+from calvin.runtime.south.async import async
 from calvin.runtime.north.resource_monitor.helper import ResourceMonitorHelper
 from calvin.utilities.calvinlogger import get_logger
 

calvin/runtime/north/resource_monitor/helper.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 
-from calvin.runtime.south.plugins.async import async
+from calvin.runtime.south.async import async
 from calvin.utilities.calvin_callback import CalvinCB
 from calvin.utilities.attribute_resolver import AttributeResolver
 from calvin.utilities.calvinlogger import get_logger
@@ -44,7 +44,7 @@ def set(self, prefix, prefix_index, value, cb=None):
         prefix: String used in storage for attribute, e.g. nodeCpuAvail.
         prefix_index: String used in indexed_public structure for this field, e.g. cpuAvail.
         value: new value to set.
-        cb: callback to receive response. Signature: cb(value, True/False) 
+        cb: callback to receive response. Signature: cb(value, True/False)
         """
 
         # get old value to cleanup indexes

calvin/runtime/north/resource_monitor/memory.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 
-from calvin.runtime.south.plugins.async import async
+from calvin.runtime.south.async import async
 from calvin.runtime.north.resource_monitor.helper import ResourceMonitorHelper
 from calvin.utilities.calvinlogger import get_logger
 

calvin/runtime/north/scheduler.py

@@ -20,7 +20,7 @@
 import logging
 
 from monitor import Event_Monitor, VisualizingMonitor
-from calvin.runtime.south.plugins.async import async
+from calvin.runtime.south.async import async
 from calvin.utilities.calvin_callback import CalvinCB
 from calvin.utilities.calvinlogger import get_logger
 from calvin.utilities import calvinconfig
@@ -139,7 +139,7 @@ def _check_replication(self):
         tt = time.time() + self._replication_interval
         if not any([t[0] < tt for t in self._tasks if t[1] == self._check_replication]):
             self.insert_task(self._check_replication, self._replication_interval)
-        _log.debug("Next replication loop in %s %d %d" % (str([t[0] - time.time() for t in self._tasks if t[1] == self._check_replication]), 
+        _log.debug("Next replication loop in %s %d %d" % (str([t[0] - time.time() for t in self._tasks if t[1] == self._check_replication]),
                     [t[1] == self._check_replication for t in self._tasks].index(True), len(self._tasks)))
         self.insert_task(self.strategy, 0)
 
@@ -317,7 +317,7 @@ def _fire_actor_non_preemptive(self, actor):
 
         return actor_did_fire
 
-        
+
     def _fire_actor_once(self, actor):
         """
         Try to fire action on actor on this runtime.
@@ -418,7 +418,7 @@ def watchdog(self):
 # ROUND-ROBIN SCHEDULER
 ######################################################################
 class RoundRobinScheduler(SimpleScheduler):
-    
+
     def strategy(self):
         # Communicate
         list_of_endpoints = self.monitor.endpoints
@@ -436,7 +436,7 @@ def strategy(self):
 # NON-PREEMPTIVE SCHEDULER
 ######################################################################
 class NonPreemptiveScheduler(SimpleScheduler):
-    
+
     def strategy(self):
         # Communicate
         list_of_endpoints = self.monitor.endpoints