Bugfixing and updated on the MEC portion with ETSI IM (#97)

added ETSI MEC 010-2 missing fields, source reorganization, mec specific source folder, api modules are now standalone

etc/yaks.service

@@ -5,7 +5,7 @@ After=network.target
 
 [Service]
 Type=simple
-PIDFile=/var/yaks/yaksd.pid
+PIDFile=/var/fos/yaksd.pid
 ExecStart = /etc/fos/yaksd
 KillMode=process
 Restart=on-failure

fos-plugins/KVM/KVM_plugin

@@ -317,9 +317,22 @@ class KVM(RuntimePluginFDU):
                     else:
                         intf_id = 'kvm-{}-{}'.format(fdu.get_short_id(), i)
                         intf_data = self.connector.loc.actual.get_node_port(self.node, self.nm, intf.get('cp_id')).get('properties')
+                        #####
+                        def get_port_descriptor(cpid):
+                            parameters = {'cp_uuid': cpid}
+                            fname = 'get_port_info'
+                            return self.call_agent_function(fname, parameters)
+                        ####
+                        cp_data = get_port_descriptor(intf_data['cp_uuid'])
+                        net_data = self.connector.loc.actual.get_node_network(self.node, self.nm, cp_data['pair_id'])
                         #self.call_nw_plugin_function('create_virtual_interface', {'intf_id':intf_id, 'descriptor': intf})
-                        self.logger.info('configure_fdu()','KVM Plugin - Interface info {}'.format(intf_data))
-                        dev_name = intf_data.get('cp_name')
+
+                        #self.logger.info('configure_fdu()','KVM Plugin - Interface info {}'.format(intf_data))
+                        #dev_name = intf_data.get('cp_name')
+
+                        self.logger.info('configure_fdu()','KVM Plugin - Interface info {}'.format(net_data))
+                        dev_name = net_data['properties']['virtual_device']
+
                         net_cfg.append({'name':intf_name, 'mac':mac, 'dev':dev_name})
 
 

fos-plugins/LXD/LXD_plugin

@@ -78,7 +78,11 @@ class LXD(RuntimePluginFDU):
     def start_runtime(self):
         self.logger.info(
             'start_runtime()', ' LXD Plugin - Connecting to LXD')
-        self.conn = Client()
+        try:
+            self.conn = Client()
+        except:
+            self.conn = Client(endpoint='https://127.0.0.1:8443', verify=False)
+
         self.logger.info(
             'start_runtime()', '[ DONE ] LXD Plugin - Connecting to LXD')
 

fos-plugins/linuxbridge/Makefile

@@ -5,7 +5,7 @@ WD := $(shell dirname $(realpath $(lastword $(MAKEFILE_LIST))));
 LB_PLUGIN_DIR = /etc/fos/plugins/linuxbridge
 LB_PLUGIN_CONFFILE = /etc/fos/plugins/linuxbridge/linuxbridge_plugin.json
 all:
-	cd src/ocaml; make
+	echo "Nothing to do"
 
 install:
 	sudo pip3 install jinja2

fos-plugins/linuxbridge/linuxbridge_plugin

@@ -346,27 +346,56 @@ class LinuxBridge(NetworkPlugin):
         vdev_name = 'br-{}'.format(n_small_id)
         netns_name = 'fosns-{}'.format(n_small_id)
 
+        properties = {}
+        face = self.overlay_interface
+        if 'face' in manifest:
+            face = manifest['face']
 
+        vni = self.__get_vni(net_uuid)
+        if 'vni' in manifest:
+            vni = manifest['vni']
 
+        mcast_addr = self.__get_mcast_addr(net_uuid)
+        if 'mcast_addr' in manifest:
+            mcast_addr = manifest['mcast_addr']
 
+        vlan_id = self.__get_vlan_id(net_uuid, self.vlan_range[0], self.vlan_range[1])
+        if 'vlan_id' in manifest:
+            vlan_id = manifest['vlan_id']
 
-        properties = {}
-        if self.use_vlan:
-            properties.update({'vlan_id': self.__get_vlan_id(net_uuid, self.vlan_range[0], self.vlan_range[1])})
-            vlan_id = self.__get_vlan_id(net_uuid, self.vlan_range[0], self.vlan_range[1])
-            start_file = self.__generate_vnet_vlan_start_script(n_small_id, net_uuid)
-            shutdown_file = self.__generate_vnet_vlan_stop_script(n_small_id)
-            self.call_os_plugin_function('execute_command',{'command':os.path.join(self.BASE_DIR, start_file),'blocking':True, 'external':True})
-            properties.update({'vlan_id': vlan_id})
+        overlay = not self.use_vlan
+        if 'overlay' in manifest:
+            overlay = manifest['overlay']
 
-        else:
-            vxlan_id = self.__get_vni(net_uuid)
-            mcast_addr = self.__get_mcast_addr(net_uuid)
-            start_file  = self.__generate_vnet_start_script(n_small_id, net_uuid)
-            shutdown_file = self.__generate_vnet_stop_script(n_small_id)
+        if overlay:
+            start_file  = self.__generate_vnet_start_script2(n_small_id, net_uuid, vni, mcast_addr, face)
+            shutdown_file = self.__generate_vnet_stop_script2(n_small_id)
             self.call_os_plugin_function('execute_command',{'command':os.path.join(self.BASE_DIR, start_file),'blocking':True, 'external':True})
-            properties.update({'vxlan_id': vxlan_id})
+            properties.update({'vxlan_id': vni})
             properties.update({'mcast_addr': mcast_addr})
+        else:
+            properties.update({'vlan_id': vlan_id})
+            start_file = self.__generate_vnet_vlan_start_script2(n_small_id, net_uuid, vlan_id, face)
+            shutdown_file = self.__generate_vnet_vlan_stop_script2(n_small_id)
+            self.call_os_plugin_function('execute_command',{'command':os.path.join(self.BASE_DIR, start_file),'blocking':True, 'external':True})
+            properties.update({'vlan_id': vlan_id})
+
+
+        # if self.use_vlan:
+        #     properties.update({'vlan_id': self.__get_vlan_id(net_uuid, self.vlan_range[0], self.vlan_range[1])})
+        #     vlan_id = self.__get_vlan_id(net_uuid, self.vlan_range[0], self.vlan_range[1])
+        #     start_file = self.__generate_vnet_vlan_start_script(n_small_id, net_uuid)
+        #     shutdown_file = self.__generate_vnet_vlan_stop_script(n_small_id)
+        #     self.call_os_plugin_function('execute_command',{'command':os.path.join(self.BASE_DIR, start_file),'blocking':True, 'external':True})
+        #     properties.update({'vlan_id': vlan_id})
+        # else:
+        #     vxlan_id = self.__get_vni(net_uuid)
+        #     mcast_addr = self.__get_mcast_addr(net_uuid)
+        #     start_file  = self.__generate_vnet_start_script(n_small_id, net_uuid)
+        #     shutdown_file = self.__generate_vnet_stop_script(n_small_id)
+        #     self.call_os_plugin_function('execute_command',{'command':os.path.join(self.BASE_DIR, start_file),'blocking':True, 'external':True})
+        #     properties.update({'vxlan_id': vxlan_id})
+        #     properties.update({'mcast_addr': mcast_addr})
 
         properties.update({'virtual_device': vdev_name})
         properties.update({'net_ns': netns_name})
@@ -656,7 +685,62 @@ class LinuxBridge(NetworkPlugin):
         chmod_cmd = 'chmod +x {}'.format(os.path.join(self.BASE_DIR, f_name))
         self.call_os_plugin_function('execute_command',{'command':chmod_cmd,'blocking':True, 'external':False})
         return f_name
+# ########################## NEW SCRIPT CREATION FUNCTIONS:
+    def __generate_vnet_start_script2(self, netid, net_uuid, vxlan_id, mcast_addr, overlay_interface):
+
+        template_sh = self.call_os_plugin_function('read_file',{'file_path':os.path.join(self.DIR, 'templates', 'vnet_create.sh'), 'root':False})
+        net_sh = Environment().from_string(template_sh)
+        net_sh = net_sh.render(net_id=netid, group_id=vxlan_id, mcast_group_address=mcast_addr, wan=overlay_interface)
+        net_sh = binascii.hexlify(base64.b64encode(bytes(net_sh,'utf-8'))).decode()
+        f_name = 'start_{}.sh'.format(netid)
+        self.call_os_plugin_function('store_file',{'content':net_sh, 'file_path':self.BASE_DIR, 'filename':f_name})
+        chmod_cmd = 'chmod +x {}'.format(os.path.join(self.BASE_DIR, f_name))
+        self.call_os_plugin_function('execute_command',{'command':chmod_cmd,'blocking':True, 'external':False})
+        return f_name
+
+    def __generate_vnet_stop_script2(self, netid):
+        template_sh = self.call_os_plugin_function('read_file',{'file_path':os.path.join(self.DIR, 'templates', 'vnet_destroy.sh'), 'root':False})
+        net_sh = Environment().from_string(template_sh)
+        net_sh = net_sh.render(net_id=netid)
+        net_sh = binascii.hexlify(base64.b64encode(bytes(net_sh,'utf-8'))).decode()
+        f_name = 'stop_{}.sh'.format(netid)
+        self.call_os_plugin_function('store_file',{'content':net_sh, 'file_path':self.BASE_DIR, 'filename':f_name})
+        chmod_cmd = 'chmod +x {}'.format(os.path.join(self.BASE_DIR, f_name))
+        self.call_os_plugin_function('execute_command',{'command':chmod_cmd,'blocking':True, 'external':False})
+        return f_name
+
+    def __generate_vnet_vlan_start_script2(self, netid, net_uuid, vlan_id, vlan_face):
+
+        vintf = '{}.{}'.format(vlan_face, vlan_id)
+        if not self.check_if_interface_exists(vintf):
+            self.create_vlan_interface(vlan_face, vlan_id)
+
+        template_sh = self.call_os_plugin_function('read_file',{'file_path':os.path.join(self.DIR, 'templates', 'vnet_create_vlan.sh'), 'root':False})
+        net_sh = Environment().from_string(template_sh)
+
+        net_sh = net_sh.render(net_id=netid, vlan_intf=vintf)
+        net_sh = binascii.hexlify(base64.b64encode(bytes(net_sh,'utf-8'))).decode()
+        f_name = 'start_{}.sh'.format(netid)
+        self.call_os_plugin_function('store_file',{'content':net_sh, 'file_path':self.BASE_DIR, 'filename':f_name})
+        chmod_cmd = 'chmod +x {}'.format(os.path.join(self.BASE_DIR, f_name))
+        self.call_os_plugin_function('execute_command',{'command':chmod_cmd,'blocking':True, 'external':False})
+        return f_name
+
+    def __generate_vnet_vlan_stop_script2(self, netid):
+        template_sh = self.call_os_plugin_function('read_file',{'file_path':os.path.join(self.DIR, 'templates', 'vnet_destroy_vlan.sh'), 'root':False})
+        net_sh = Environment().from_string(template_sh)
+        net_sh = net_sh.render(net_id=netid)
+        net_sh = binascii.hexlify(base64.b64encode(bytes(net_sh,'utf-8'))).decode()
+        f_name = 'stop_{}.sh'.format(netid)
+        self.call_os_plugin_function('store_file',{'content':net_sh, 'file_path':self.BASE_DIR, 'filename':f_name})
+        chmod_cmd = 'chmod +x {}'.format(os.path.join(self.BASE_DIR, f_name))
+        self.call_os_plugin_function('execute_command',{'command':chmod_cmd,'blocking':True, 'external':False})
+        return f_name
+
+
+
 
+# ###########################
     def __generate_vnet_start_script(self, netid, net_uuid):
 
         vxlan_id = self.__get_vni(net_uuid)

fos_install.sh

@@ -13,7 +13,7 @@ MACHINE_TYPE=`uname -m`
 
 
 sudo apt update -qq
-sudo apt install libev4 libev-dev libssl1.0.0 python3-pip python3-dev curl -y
+sudo apt install libev4 libev-dev libssl1.0.0 python3-pip python3-dev curl jq -y
 sudo pip3 install jsonschema
 
 mkdir -p src/agent/_build/default/fos-agent

install_fos_lxd.sh

@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 
-git clone https://github.com/gabrik/fog05
+git clone https://github.com/eclipse/fog05
 cd fog05
 
 MACHINE_TYPE=`uname -m`

src/agent/fos-agent/fos_agent.ml

@@ -414,7 +414,7 @@ let agent verbose_flag debug_flag configuration =
        | (_,_) -> Lwt.return_unit)
 
   in
-  let cb_gd_net self (net:FTypes.virtual_network option) (is_remove:bool) (uuid:string option) =
+  let cb_gd_net self (net:FTypesRecord.virtual_network option) (is_remove:bool) (uuid:string option) =
     let%lwt net_p = get_network_plugin self in
     match is_remove with
     | false ->
@@ -423,9 +423,9 @@ let agent verbose_flag debug_flag configuration =
          MVar.read self >>= fun self ->
          let%lwt _ = Logs_lwt.debug (fun m -> m "[FOS-AGENT] - CB-GD-NET - ##############") in
          let%lwt _ = Logs_lwt.debug (fun m -> m "[FOS-AGENT] - CB-GD-NET - vNET Updated! Agent will update actual store and call the right plugin!") in
-         let%lwt _ = Yaks_connector.Global.Actual.add_network sys_id Yaks_connector.default_tenant_id net.uuid net self.yaks in
-         let record = FTypesRecord.{uuid = net.uuid; status = `CREATE; properties = None; ip_configuration = net.ip_configuration} in
-         Yaks_connector.Local.Desired.add_node_network (Apero.Option.get self.configuration.agent.uuid) net_p record.uuid record self.yaks
+         (* let%lwt _ = Yaks_connector.Global.Actual.add_network sys_id Yaks_connector.default_tenant_id net.uuid net self.yaks in *)
+         (* let record = FTypesRecord.{uuid = net.uuid; status = `CREATE; properties = None; ip_configuration = net.ip_configuration} in *)
+         Yaks_connector.Local.Desired.add_node_network (Apero.Option.get self.configuration.agent.uuid) net_p net.uuid net self.yaks
          >>= Lwt.return
        | None -> Lwt.return_unit)
     | true ->
@@ -696,7 +696,7 @@ let agent verbose_flag debug_flag configuration =
   let%lwt _ = Yaks_connector.Global.Desired.observe_node_plugins sys_id Yaks_connector.default_tenant_id uuid (cb_gd_plugin state) yaks in
   let%lwt _ = Yaks_connector.Global.Desired.observe_fdu sys_id Yaks_connector.default_tenant_id (cb_gd_fdu state) yaks in
   let%lwt _ = Yaks_connector.Global.Desired.observe_node_fdu sys_id Yaks_connector.default_tenant_id uuid (cb_gd_node_fdu state) yaks in
-  let%lwt _ = Yaks_connector.Global.Desired.observe_network sys_id Yaks_connector.default_tenant_id (cb_gd_net state) yaks in
+  let%lwt _ = Yaks_connector.Global.Desired.observe_node_network sys_id Yaks_connector.default_tenant_id uuid (cb_gd_net state) yaks in
   let%lwt _ = Yaks_connector.Global.Desired.observe_ports sys_id Yaks_connector.default_tenant_id (cb_gd_cp state) yaks in
   let%lwt _ = Yaks_connector.Global.Desired.observe_images sys_id Yaks_connector.default_tenant_id (cb_gd_image state) yaks in
   let%lwt _ = Yaks_connector.Global.Desired.observe_flavors sys_id Yaks_connector.default_tenant_id (cb_gd_flavor state) yaks in

src/api/ocaml/api/fos-fim-api/fos_fim_api.ml

@@ -12,7 +12,7 @@
 
 open Fos_im
 open Lwt.Infix
-open Fos_errors
+open Errors
 
 let ylocator = Apero.Option.get @@ Apero_net.Locator.of_string @@ Apero.Option.get_or_default (Sys.getenv_opt "FOS_YAKS_ENDPOINT") "tcp/127.0.0.1:7887"
 let ysystem = Apero.Option.get_or_default (Sys.getenv_opt "FOS_SYS_ID") "0"
@@ -74,8 +74,12 @@ module Network = struct
 
   let add_network (descriptor:FTypes.virtual_network) api =
     let netid = descriptor.uuid in
-    Yaks_connector.Global.Desired.add_network api.sysid api.tenantid netid descriptor api.yconnector
-    >>= fun _ -> Lwt.return true
+    let%lwt n = Yaks_connector.Global.Actual.get_network api.sysid api.tenantid netid api.yconnector in
+    match n with
+    | Some _ -> Lwt.fail @@ FException (`InternalError (`Msg (Printf.sprintf "Network with id %s already exists" netid)))
+    | None ->
+      Yaks_connector.Global.Actual.add_network api.sysid api.tenantid netid descriptor api.yconnector
+      >>= fun _ -> Lwt.return true
   let remove_network netid  api =
     Yaks_connector.Global.Desired.remove_network api.sysid api.tenantid netid api.yconnector
     >>= fun _ -> Lwt.return true
@@ -308,19 +312,24 @@ module Image = struct
 
 end
 
-(*
+
 module Flavor = struct
-  let add descriptor api =
-    ignore [descriptor; api];
-    Lwt.return true
-  let remove descriptor api =
-    ignore [descriptor; api];
-    Lwt.return true
+  let add (descriptor:Fdu.computational_requirements) api =
+    let flv_id = (match descriptor.uuid with
+        | Some id -> id
+        | None -> Apero.Uuid.to_string (Apero.Uuid.make ()) ) in
+    let descriptor = {descriptor with uuid = Some flv_id} in
+    Yaks_connector.Global.Desired.add_flavor api.sysid api.tenantid flv_id descriptor api.yconnector
+    >>= fun _ -> Lwt.return flv_id
+
+  let remove flv_id api =
+    Yaks_connector.Global.Desired.remove_flavor api.sysid api.tenantid flv_id api.yconnector
+    >>= fun _ -> Lwt.return flv_id
+
   let list api =
-    ignore api;
-    Lwt.return []
+    Yaks_connector.Global.Actual.get_all_flavors api.sysid api.tenantid api.yconnector
 
-end *)
+end
 
 module FIMAPI = struct
 

src/api/ocaml/api/fos-fim-api/fos_fim_api.mli

@@ -84,13 +84,13 @@ module Image : sig
 
 end
 
-(*
-   module Flavor : sig
-   val add : FTypes.computational_requirements_type -> api -> bool Lwt.t
-   val remove : string -> api -> bool Lwt.t
-   val list : api -> (FTypes.computational_requirements_type list) Lwt.t
 
-   end *)
+module Flavor : sig
+  val add : Fdu.computational_requirements -> api -> string Lwt.t
+  val remove : string -> api -> string Lwt.t
+  val list : api -> (Fdu.computational_requirements list) Lwt.t
+
+end
 
 module FIMAPI : sig
 

src/api/python/api/fog05/yaks_connector.py

@@ -494,7 +494,7 @@ def get_all_network_ports(self, sysid, tenantid):
         kvs = self.ws.get(p)
         d = []
         for k in kvs:
-            d.append(json.loads(kvs[0][1].get_value()))
+            d.append(json.loads(k[1].get_value()))
         return d
 
     def get_network(self, sysid, tenantid, netid):
@@ -509,7 +509,7 @@ def get_all_networks (self, sysid, tenantid):
         kvs = self.ws.get(p)
         d = []
         for k in kvs:
-            d.append(json.loads(kvs[0][1].get_value()))
+            d.append(json.loads(k[1].get_value()))
         return d
 
     def add_network(self, sysid, tenantid, netid, netinfo):

src/api/python/api/setup.py

@@ -27,7 +27,7 @@
     url='https://github.com/eclipse/fog05',
     authon_email='gabriele.baldoni@adlinktech.com',
     license='Apache 2.O or EPL 2.0',
-    install_requires=['yaks==0.2.7', 'jsonschema','mvar'],
+    install_requires=['yaks', 'jsonschema','mvar'],
     scripts=[],
     include_package_data=True
 )

src/core/ocaml/fos-core/yaks_connector.ml

@@ -1,9 +1,8 @@
 open Yaks_ocaml
 open Lwt.Infix
-open Fos_errors
 open Fos_core
 open Fos_im
-
+open Errors
 
 
 

src/im/ocaml/Makefile

@@ -11,20 +11,24 @@ atd:
 	atdgen -t fos-im/fos_records_types.atd
 	atdgen -j-std fos-im/fos_records_types.atd
 
-	atdgen -t fos-im/fdu.atd
+	# atdgen -t fos-im/fdu.atd
 	atdgen -j-std fos-im/fdu.atd
-	# atdgen -v fos-im/fos_records_types.atd
-	atdgen -t fos-im/im.atd
-	atdgen -j-std fos-im/im.atd
-	atdgen -v fos-im/im.atd
+
+	atdgen -j-std fos-im/nfv.atd
+
+	atdgen -j-std fos-im/mec.atd
+
+	atdgen -j-std fos-im/mec_interfaces.atd
+
 
 clean:
 	rm -rf fos-im/agent_types*.ml fos-im/agent_types*.mli
 	rm -rf fos-im/fos_types*.ml fos-im/fos_types*.mli
 	rm -rf fos-im/fos_records_types*.ml fos-im/fos_records_types*.mli
 	rm -rf fos-im/types_record*.ml fos-im/types_record*.mli
 	rm -rf fos-im/fdu*.ml fos-im/fdu*.mli
-	rm -rf fos-im/im*.ml*
+	rm -rf fos-im/nfv*.ml*
+	rm -rf fos-im/mec*.ml*
 	dune clean
 	rm -rf ./_build