REST Proxy work also as Image Service (#56)

Using rest api it is possible to upload an image file to the rest proxy, it will generate the uri that will be used when downloading image on the nodes for instantiation.

.gitignore

@@ -10,7 +10,9 @@ dist
 fog05.egg-info
 
 nohup.out
-
+.merlin
+*.install
+dune-project
 _build
 .idea/
 .vscode/

INSTALL.md

@@ -105,8 +105,8 @@ Open another shell (or use another machine that has the fog05 api installed)
 and execute:
 
     python3
-    >>> from fog05 import FIMAPIv2
-    >>> api = FIMAPIv2(locator='127.0.0.1') # or locator='IP of the YAKS server'
+    >>> from fog05 import FIMAPI
+    >>> api = FIMAPI(locator='127.0.0.1') # or locator='IP of the YAKS server'
     >>> api.node.list()
     >>> ['your node uuid',...]
     >>> api.close()

Makefile

@@ -96,12 +96,12 @@ uninstall:
 	sudo pip3 uninstall fog05 -y
 
 clean:
-	sudo rm -rf lldpd
 	opam remove fos-im -y
 	make -C src/im/ocaml clean
 	make -C src/core/ocaml clean
 	make -C src/agent clean
 	make -C src/api/ocaml/api clean
+	sudo rm -rf lldpd
 	sudo rm -rf src/pyhton/fog05.egg-info
 	sudo rm -rf src/pyhton/build
 	sudo rm -rf src/pyhton/dist

examples/im/vnet_dhcp.json

@@ -0,0 +1,14 @@
+{
+    "uuid": "6cc2aa30-1dcf-4c93-a57e-433fd0bd498e",
+    "name": "net1",
+    "net_type": "ELAN",
+    "is_mgmt": false,
+    "ip_configuration": {
+        "ip_version": "IPV4",
+        "subnet": "192.168.234.0/24",
+        "gateway": "192.168.234.1",
+        "dhcp_enable": true,
+        "dhcp_range": "192.168.234.2,192.168.234.100",
+        "dns": "8.8.8.8,8.8.4.4"
+    }
+}

fos-plugins/linuxbridge/linuxbridge_plugin

@@ -29,12 +29,15 @@ import socket
 import signal
 import base64
 import binascii
+import hashlib
 from jinja2 import Environment
 from fog05.interfaces.NetworkPlugin import *
 from fog05 import Yaks_Connector
 from mvar import MVar
 from fog05.DLogger import DLogger
 
+VNI_LIMIT = 16777215
+VLAN_LIMIT = 4095
 
 class LinuxBridge(NetworkPlugin):
 
@@ -50,8 +53,10 @@ class LinuxBridge(NetworkPlugin):
         self.configuration = manifest.get('configuration', {})
         self.pid = os.getpid()
         self.var = MVar()
-        self.agent_conf = \
-            self.connector.loc.actual.get_node_configuration(self.node)
+        self.agent_conf = self.connector.loc.actual.get_node_configuration(self.node)
+        self.use_vlan = self.configuration.get('use_vlan', False)
+        self.vlan_range = self.configuration.get('vlan_range', [0, VLAN_LIMIT])
+
         self.logger.info('__init__()', ' Hello from Linux Bridge Plugin')
 
         self.BASE_DIR = os.path.join(
@@ -314,15 +319,14 @@ class LinuxBridge(NetworkPlugin):
             return None
 
         n_small_id = net_uuid.split('-')[0]
-        start_file  = self.__generate_vnet_start_script(n_small_id)
+        start_file  = self.__generate_vnet_start_script(n_small_id, net_uuid)
         shutdown_file = self.__generate_vnet_stop_script(n_small_id)
         vdev_name = 'br-{}'.format(n_small_id)
         netns_name = 'fosns-{}'.format(n_small_id)
 
-        hx = bytes.fromhex(n_small_id)
-        vals = struct.unpack('<{}B'.format(len(hx)), hx)
-        vxlan_id = sum(vals)
-        mcast_addr = '239.245.{}.{}'.format(vals[0], vals[1])
+        vxlan_id = self.__get_vni(net_uuid)
+        mcast_addr = self.__get_mcast_addr(net_uuid)
+
 
         self.call_os_plugin_function('execute_command',{'command':os.path.join(self.BASE_DIR, start_file),'blocking':True, 'external':True})
         properties = {}
@@ -331,6 +335,8 @@ class LinuxBridge(NetworkPlugin):
         properties.update({'vxlan_id': vxlan_id})
         properties.update({'mcast_addr': mcast_addr})
         properties.update({'manager':self.uuid})
+        if self.use_vlan:
+            properties.update({'vlan_id': self.__get_vlan_id(net_uuid)})
 
 
         # self.agent.getOSPlugin().executeCommand(brcmd)
@@ -616,16 +622,18 @@ class LinuxBridge(NetworkPlugin):
         self.call_os_plugin_function('execute_command',{'command':chmod_cmd,'blocking':True, 'external':False})
         return f_name
 
-    def __generate_vnet_start_script(self, netid):
+    def __generate_vnet_start_script(self, netid, net_uuid):
 
-        hx = bytes.fromhex(netid)
-        vals = struct.unpack('<{}B'.format(len(hx)), hx)
-        vxlan_id = sum(vals)
-        mcast_addr = '239.245.{}.{}'.format(vals[0], vals[1])
+        vxlan_id = self.__get_vni(net_uuid)
+        mcast_addr = self.__get_mcast_addr(net_uuid)
 
         template_sh = self.call_os_plugin_function('read_file',{'file_path':os.path.join(self.DIR, 'templates', 'vnet_create.sh'), 'root':False})
         net_sh = Environment().from_string(template_sh)
-        net_sh = net_sh.render(net_id=netid, group_id=vxlan_id, mcast_group_address=mcast_addr, wan=self.overlay_interface)
+        if self.use_vlan:
+            overlay_interface = '{}.{}'.format(self.overlay_interface, self.__get_vlan_id(net_uuid))
+        else:
+            overlay_interface = self.overlay_interface
+        net_sh = net_sh.render(net_id=netid, group_id=vxlan_id, mcast_group_address=mcast_addr, wan=overlay_interface)
         net_sh = binascii.hexlify(base64.b64encode(bytes(net_sh,'utf-8'))).decode()
         f_name = 'start_{}.sh'.format(netid)
         self.call_os_plugin_function('store_file',{'content':net_sh, 'file_path':self.BASE_DIR, 'filename':f_name})
@@ -708,6 +716,25 @@ class LinuxBridge(NetworkPlugin):
         self.call_os_plugin_function('execute_command',{'command':chmod_cmd,'blocking':True, 'external':False})
         return f_name, vxlan_name, vxlan_id, mcast_addr
 
+    def __get_mcast_addr(self, net_id):
+        hashdata = hashlib.sha512(net_id.encode('utf8')).digest()
+        return '239.245.{}.{}'.format(hashdata[0],hashdata[-1])
+
+    def __get_vni(self, net_id):
+        hashdata = hashlib.sha512(net_id.encode('utf8')).digest()
+        vni = 0
+        for x in hashdata:
+            vni += x
+        return vni % VNI_LIMIT
+
+    def __get_vlan_id(self, net_id, min_id=0, max_id=VLAN_LIMIT)
+        hashdata = hashlib.sha512(net_id.encode('utf8')).digest()
+        vlid = 0
+        for x in hashdata:
+            vlid += x
+        return (vlid % max_id) + min_id
+
+
 def read_file(file_path):
     data = ''
     with open(file_path, 'r') as f:

fos-plugins/linuxbridge/linuxbridge_plugin.json

@@ -10,6 +10,11 @@
       "configuration": {
             "ylocator": "tcp/127.0.0.1:7887",
             "nodeid": "e0e442af51d14802a9bc71b5e634440e",
-            "dataplane_interface": "ens4"
+            "dataplane_interface": "ens4",
+            "use_vlan": false,
+            "vlan_range": [
+                  50,
+                  100
+            ]
       }
 }

src/agent/fos-agent/fos_agent.ml

@@ -367,7 +367,7 @@ let agent verbose_flag debug_flag configuration =
          let%lwt _ = Logs_lwt.debug (fun m -> m "[FOS-AGENT] - CB-GD-NET - ##############") in
          let%lwt _ = Logs_lwt.debug (fun m -> m "[FOS-AGENT] - CB-GD-NET - vNET Updated! Agent will update actual store and call the right plugin!") in
          let%lwt _ = Yaks_connector.Global.Actual.add_network sys_id Yaks_connector.default_tenant_id net.uuid net self.yaks in
-         let record = FTypesRecord.{uuid = net.uuid; status = `CREATE; properties = None} in
+         let record = FTypesRecord.{uuid = net.uuid; status = `CREATE; properties = None; ip_configuration = net.ip_configuration} in
          Yaks_connector.Local.Desired.add_node_network (Apero.Option.get self.configuration.agent.uuid) net_p record.uuid record self.yaks
          >>= Lwt.return
        | None -> Lwt.return_unit)

src/api/python/api/fog05/yaks_connector.py

@@ -498,7 +498,7 @@ def add_image(self, sysid, tenantid, imageid, imginfo):
         v = Value(json.dumps(imginfo), encoding=Encoding.STRING)
         return self.ws.put(p, v)
 
-    def remove_image(self, sysid, tenatid, imageid):
+    def remove_image(self, sysid, tenantid, imageid):
         p = self.get_image_info_path(sysid, tenantid, imageid)
         return self.ws.remove(p)
 

src/api/python/api/setup.py

@@ -23,6 +23,10 @@
     python_requires='>=3',
     author='ADLINK',
     packages=['fog05','fog05.interfaces'],
+    description='Python 3 API fog05 YAKS Client',
+    url='https://github.com/eclipse/fog05',
+    authon_email='gabriele.baldoni@adlinktech.com',
+    license='Apache 2.O or EPL 2.0',
     install_requires=['yaks==0.2.6', 'jsonschema','mvar'],
     scripts=[],
     include_package_data=True

src/api/python/rest_api/Makefile

@@ -2,11 +2,11 @@ all: echo "Nothing to do..."
 
 
 clean:
-	rm -rf build dist fogfimapi.egg-info
+	rm -rf build dist fog05rest.egg-info
 
 install:
 	python setup.py install
-	rm -rf build dist fogfimapi.egg-info
+	rm -rf build dist fog05rest.egg-info
 
 uninstall:
-	pip uninstall fogfimapi -y
+	pip uninstall fog05rest -y

src/api/python/rest_api/fog05rest/fimapi.py

@@ -14,6 +14,18 @@
 
 import requests
 import json
+import os
+import tempfile
+
+
+def save_file(content, filename):
+    full_path = os.path.join(tempfile.gettempdir(), filename)
+    f = open(full_path, 'w')
+    f.write(content)
+    f.flush()
+    f.close()
+    return full_path
+
 
 class FIMAPI(object):
     '''
@@ -190,9 +202,20 @@ class Image(object):
         def __init__(self, base_url):
             self.base_url = base_url
 
-        def add(self, descriptor):
+        def add(self, descriptor, image_path):
+            img_id = descriptor.get('uuid', None)
+            if  img_id is None:
+                img_id =  '{}'.format(uuid.uuid4())
+                descriptor.update({'uuid':img_id})
             url = '{}/image/add'.format(self.base_url)
-            return json.loads(str(requests.post(url, data=json.dumps(descriptor)).content))
+            desc_filename = '{}.json'.format(descriptor['uuid'])
+            temp_desc_file = save_file(json.dumps(descriptor),desc_filename)
+            files = {'descriptor': open(temp_desc_file, 'rb'), 'image': open(image_path, 'rb')}
+            res = json.loads(str(requests.post(url, files=files).content))
+            os.remove(temp_desc_file)
+            if res.get('result') == True:
+                return img_id
+            return res
 
         def get(self, image_uuid):
             url = '{}/image/{}'.format(self.base_url, image_uuid)

src/api/python/rest_api/setup.py

@@ -23,5 +23,17 @@
     author='ADLINK',
     packages=['fog05rest',],
     install_requires=['requests'],
-    include_package_data=True
+    include_package_data=True,
+    description='Python 2 API fog05 REST Client',
+    url='https://github.com/eclipse/fog05',
+    authon_email='gabriele.baldoni@adlinktech.com',
+    license='Apache 2.O or EPL 2.0',
+    classifiers=[
+          'Development Status :: 2 - Pre-Alpha',
+          'Intended Audience :: Developers',
+          'Intended Audience :: Telecommunications Industry',
+          'License :: OSI Approved :: Apache Software License',
+          'License :: OSI Approved :: Eclipse Public License 2.0 (EPL-2.0)',
+          'Programming Language :: Python :: 2'
+    ],
 )

src/im/ocaml/Makefile

@@ -10,7 +10,7 @@ atd:
 	atdgen -v fos-im/fos_types.atd
 	atdgen -t fos-im/fos_records_types.atd
 	atdgen -j-std fos-im/fos_records_types.atd
-	atdgen -v fos-im/fos_records_types.atd
+	# atdgen -v fos-im/fos_records_types.atd
 	atdgen -t fos-im/im.atd
 	atdgen -j-std fos-im/im.atd
 	atdgen -v fos-im/im.atd

src/im/ocaml/fos-im/fos_records_types.atd

@@ -14,7 +14,7 @@
 type json <ocaml module="Abs_json"> = abstract
 (* Atomit Entity Manifest *)
 
-
+type address_information <ocaml module="Fos_types"> = abstract
 
 
 type fdu_state = [
@@ -93,6 +93,7 @@ type virtual_network = {
   uuid : string;
   status: net_state;
   ?properties : json option;
+  ?ip_configuration : address_information option;
   }
 
 

src/im/ocaml/fos-im/fos_types.atd

@@ -174,11 +174,26 @@ type connection_point = {
 }
 
 
+type ip_kind = [
+  | IPV4
+  | IPV6
+]
+
+type address_information = {
+  ip_version : ip_kind;
+  subnet : string;   (*  X.X.X.X/Y*)
+  ?gateway : string option; (* X.X.X.X *)
+  dhcp_enable :  bool;
+  ?dhcp_range : string option; (* X.X.X.X,X.X.X.Z *)
+  ?dns : string option; (* X.X.X.X,X.X.X.Z,.... *)
+}
+
 type virtual_network = {
   uuid : string;
   name : string;
   net_type : vn_type;
-  is_mgmt : bool
+  is_mgmt : bool;
+  ?ip_configuration : address_information option;
 }