Skip to content

Commit

Permalink
Migration to latest YAKS APIs (#145)
Browse files Browse the repository at this point in the history 
* updated dependency to yaks 0.3.0.a0 in python APIs
* Various updates to support new YAKS API
* update to new default port for zenoh/yaks server
  • Loading branch information
@gabrik
gabrik committed Oct 3, 2019
1 parent 8c81b93 commit 2994c4c
Show file tree
Hide file tree
Showing 60 changed files with 276 additions and 205 deletions.
2 changes: 1 addition & 1 deletion INSTALL.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -137,6 +137,6 @@ It is also possible to install a CLI interface, just execute
On a node with the fos CLI interface installed declare the env variable `FOS_YAKS_ENDPOINT`
and verify the list of the nodes available

export FOS_YAKS_ENDPOINT="tcp/<address of yaks server>:7887"
export FOS_YAKS_ENDPOINT="tcp/<address of yaks server>:7447"
fos fim node list

18 changes: 13 additions & 5 deletions Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,10 +16,17 @@ all:
make -C src/api/ocaml/fimapi install
make -C src/api/ocaml/faemapi install
make -C src/api/ocaml/feoapi install
make -C src/agent/;
make -C src/agent/

install:

pip3 install pyangbind pyang
curl -L -o /usr/local/lib https://github.com/atolab/atobin/blob/master/zenoh-c/unstable/ubuntu/16.04/libzenohc.so
git clone https://github.com/atolab/zenoh-python
cd zenoh-python
python3 setup.py install
cd ..
git clone https://github.com/atolab/yaks-python
make -C yaks-python install
make -C src/im/python install
make -C src/api/python/api install

Expand All @@ -30,11 +37,11 @@ endif
sudo id -u fos >/dev/null 2>&1 || sudo useradd -r -s /bin/false fos
sudo usermod -aG sudo fos
ifeq ($(shell uname -m), x86_64)
curl -L -o /tmp/yaks.tar.gz https://www.dropbox.com/s/g4tnzvjwlx3zcr2/yaksd.tar.gz
curl -L -o /tmp/yaks.tar.gz https://www.dropbox.com/s/omlj6chql19g74s/yaks.tar.gz
else ifeq ($(shell uname -m), armv7l)
curl -L -o /tmp/yaks.tar.gz https://www.dropbox.com/s/lo4s72rc5uoowxw/yaksd_armv7l.tar.gz
curl -L -o /tmp/yaks.tar.gz https://www.dropbox.com/s/wi65knmjcj74pgg/yaks.tar.gz
else ifeq ($(shell uname -m), aarch64)
curl -L -o /tmp/yaks.tar.gz https://www.dropbox.com/s/j29g35zb9cy28ph/yaksd_arm64.tar.gz
curl -L -o /tmp/yaks.tar.gz https://www.dropbox.com/s/oj4z80c1jwofv2a/yaks.tar.gz
endif
tar -xzvf /tmp/yaks.tar.gz -C /etc/fos
rm -rf /tmp/yaks.tar.gz
Expand Down Expand Up @@ -100,6 +107,7 @@ uninstall:
clean:
opam remove fos-im -y
make -C src/im/ocaml clean
make -C src/im/python clean
make -C src/core/ocaml clean
make -C src/agent clean
make -C src/api/ocaml/fimapi clean
Expand Down
2 changes: 1 addition & 1 deletion README.md
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
[![Gitter](https://badges.gitter.im/adlink-labs/fog05.svg)](https://gitter.im/adlink-labs/fog05?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge)
[![Gitter](https://badges.gitter.im/atolab/fog05.svg)](https://gitter.im/atolab/fog05?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge)

# fog05

Expand Down
2 changes: 1 addition & 1 deletion etc/agent.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
"system": "0",
"pid_file": "/var/fos/agent.pid",
"expose": true,
"yaks": "tcp/127.0.0.1:7887",
"yaks": "tcp/127.0.0.1:7447",
"path": "/var/fos",
"enable_lldp": false,
"enable_spawner": false,
Expand Down
2 changes: 1 addition & 1 deletion fos-plugins/KVM/KVM_plugin.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@
],
"description": "kvm fdu plugin",
"configuration": {
"ylocator": "tcp/127.0.0.1:7887",
"ylocator": "tcp/127.0.0.1:7447",
"nodeid": "e0e442af51d14802a9bc71b5e634440e"
}
}
122 changes: 69 additions & 53 deletions fos-plugins/LXD/LXD_plugin
View file
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,11 @@ class LXD(RuntimePluginFDU):
self.conn = None
self.images = {}
self.flavors = {}
self.mon_th = {}

self.mon_instances = []

self.mon_th = None
self.mon_run = False


d = {
Expand Down Expand Up @@ -127,6 +131,11 @@ class LXD(RuntimePluginFDU):
self.manifest.update({'status': 'running'})
self.connector.loc.actual.add_node_plugin(self.node, self.uuid, self.manifest)

self.mon_run = True
mt = threading.Thread(target=self.__monitor_instance, daemon=True)
mt.start()
self.mon_th = mt

self.logger.info('start_runtime()', ' LXD Plugin - Started...')

r = self.var.get()
Expand Down Expand Up @@ -459,9 +468,9 @@ class LXD(RuntimePluginFDU):

# TODO: monitoring TBA
# this monitoring uses pylxd to get container information
mt = threading.Thread(target=self.__monitor_instance, args=(fdu_uuid, instance_uuid, fdu.name),daemon=True)
mt.start()
self.mon_th.update({instance_uuid:mt})

self.mon_instances.append((fdu_uuid, instance_uuid, fdu.name))

self.logger.info('run_fdu()', '[ DONE ] LXD Plugin - Starting a Monitoring of {}'.format(instance_uuid))

def clean_fdu(self, instance_uuid):
Expand Down Expand Up @@ -543,6 +552,7 @@ class LXD(RuntimePluginFDU):
fm.put('/etc/profile.d/99-fos', envs, mode="0644")
fdu.on_start()

self.mon_instances.remove((fdu_uuid, instance_uuid, fdu.name))

self.current_fdus.update({instance_uuid: fdu})
self.update_fdu_status(fdu_uuid, instance_uuid, 'RUN')
Expand Down Expand Up @@ -912,58 +922,57 @@ class LXD(RuntimePluginFDU):
return {'result':iface}
return {'error':503, 'error_msg': 'Unable to disconnect interface {} to cp'.format(iface)}

def __monitor_instance(self, fdu_uuid, instance_id, instance_name):
self.logger.info('__monitor_instance()','[ INFO ] LXD Plugin - Staring monitoring of Container uuid {}'.format(instance_id))
while True:
def __monitor_instance(self):
self.logger.info('__monitor_instance()','[ INFO ] LXD Plugin - Staring monitoring of FDUs')
while self.mon_run:
time.sleep(self.update_interval)
try:
record = self.connector.loc.actual.get_node_fdu(self.node, self.uuid, fdu_uuid, instance_id)
record = LXDFDU(record)
if record.get_status() not in ['LAND', 'TAKE_OFF','MIGRATE']:
self.logger.info('__monitor_instance()','[ INFO ] LXD Plugin - Updating status of {} - {}'.format(instance_id, instance_name))
c = self.conn.containers.get(instance_name)
cs = c.state()
detailed_state = {}

c_net = {}
for i in record.get_interfaces():
try:
ip = self.nm.get_address(i.get('mac_address',''))
except ValueError:
ip = ''
# Getting address from LXD if unable from network manager, eg. if connected to physical device/lxd bridge
if ip == '':
if cs.network is not None:
lxd_net_info_addrs = cs.network[i['vintf_name']]['addresses']
for a in lxd_net_info_addrs:
if a['family'] == 'inet':
ip = a['address']
c_net.update(
{
i.get('vintf_name'):
# (fdu_uuid, instance_uuid, fdu.name)
for fdu_id, iid, name in self.mon_instances:
try:
record = self.connector.loc.actual.get_node_fdu(self.node, self.uuid, fdu_id, iid)
record = LXDFDU(record)
if record.get_status() not in ['LAND', 'TAKE_OFF','MIGRATE']:
self.logger.info('__monitor_instance()','[ INFO ] LXD Plugin - Updating status of {} - {}'.format(iid, name))
c = self.conn.containers.get(name)
cs = c.state()
detailed_state = {}

c_net = {}
for i in record.get_interfaces():
try:
ip = self.nm.get_address(i.get('mac_address',''))
except ValueError:
ip = ''
# Getting address from LXD if unable from network manager, eg. if connected to physical device/lxd bridge
if ip == '':
if cs.network is not None:
lxd_net_info_addrs = cs.network[i['vintf_name']]['addresses']
for a in lxd_net_info_addrs:
if a['family'] == 'inet':
ip = a['address']
c_net.update(
{
'hwaddr': i.get('mac_address'),
'addresses': [
{'address': ip}
]
i.get('vintf_name'):
{
'hwaddr': i.get('mac_address'),
'addresses': [
{'address': ip}
]
}
}
}
)
)


detailed_state.update({'network': c_net})
detailed_state.update({'cpu': cs.cpu})
detailed_state.update({'memory': cs.memory})
detailed_state.update({'network': c_net})
detailed_state.update({'cpu': cs.cpu})
detailed_state.update({'memory': cs.memory})

record.set_hypervisor_info(detailed_state)
self.connector.loc.actual.add_node_fdu(self.node, self.uuid, fdu_uuid, instance_id, record.to_json())
record.set_hypervisor_info(detailed_state)
self.connector.loc.actual.add_node_fdu(self.node, self.uuid, fdu_id, iid, record.to_json())

# if c.status == 'Stopped':
# self.logger.info('__monitor_instance()','[ INFO ] LXD Plugin - Stopping monitoring of Container uuid {}'.format(fdu_uuid))
# return
except Exception as e:
self.logger.error('__monitor_instance()', '[ ERROR ] LXD Plugin - Stopping monitoring of Container uuid {} Error {}'.format(instance_id, e))
return
except Exception as e:
self.logger.error('__monitor_instance()', '[ ERROR ] LXD Plugin - Error on monitoring of Container uuid {} Error {}'.format(iid, e))
pass

def __generate_custom_profile_userdata_configuration(self, userdata, config):
config.update({'user.user-data': userdata})
Expand All @@ -987,7 +996,7 @@ class LXD(RuntimePluginFDU):

template_vintf = '{"name":"%s","type":"nic","parent":"%s","nictype":"physical","hwaddr":"%s"}'
template_value_phy_mac = '{"name":"%s","type":"nic","parent":"%s","nictype":"physical","hwaddr":"%s"}'
template_value_macvlan_mac = '{"name":"%s","type":"nic","parent":"%s","nictype":"macvlan",,"hwaddr":"%s"}'
template_value_macvlan_mac = '{"name":"%s","type":"nic","parent":"%s","nictype":"macvlan","hwaddr":"%s"}'
template_value_bridge_mac = '{"name":"%s","type":"nic","parent":"%s","nictype":"bridged","hwaddr":"%s"}'
'''
# Create tenant's storage pool
Expand All @@ -1006,22 +1015,29 @@ class LXD(RuntimePluginFDU):
for intf in fdu.get_interfaces():
intf_name = intf.get('vintf_name')
self.logger.info('__generate_custom_profile_devices_configuration()','Interface Info {}'.format(intf))
mac = intf.get('mac_address', self.__generate_random_mac())
intf.update({'mac_address': mac})

if intf.get('virtual_interface').get('intf_type').upper() in ['PHYSICAL','BRIDGED']:
# mac = intf.get('mac_address', self.__generate_random_mac())
# intf.update({'mac_address': mac})
real_intf_name = intf.get('phy_face', None)
if real_intf_name is None:
raise ValueError("phy_face cannot be none")
if self.os.get_intf_type(real_intf_name) in ['ethernet']:
mac = intf.get('mac_address', self.__generate_random_mac())
intf.update({'mac_address': mac})
nw_v = json.loads(str(template_value_macvlan_mac % (intf_name, real_intf_name, mac)))
self.os.set_interface_unaviable(real_intf_name)
elif self.os.get_intf_type(real_intf_name) in ['wireless']:
nw_v = json.loads(str(template_value_phy_mac % (intf_name, real_intf_name, mac)))
nw_v = json.loads(str(template_value_phy % (intf_name, real_intf_name)))
self.os.set_interface_unaviable(real_intf_name)
else:
mac = intf.get('mac_address', self.__generate_random_mac())
intf.update({'mac_address': mac})
nw_v = json.loads(str(template_value_bridge_mac % (intf_name, real_intf_name, mac)))
interfaces.append(intf)
else:
mac = intf.get('mac_address', self.__generate_random_mac())
intf.update({'mac_address': mac})
intf_id = 'lxd-{}-{}'.format(fdu.get_short_id(), i)
intf_data = self.nm.create_virtual_interface(intf_id, intf)
self.logger.info('__generate_custom_profile_userdata_configuration()','LXD Plugin - Interface info {}'.format(intf_data))
Expand Down
2 changes: 1 addition & 1 deletion fos-plugins/LXD/LXD_plugin.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@
"description": "lxd fdu plugin",
"configuration": {
"storage_pool":"default",
"ylocator": "tcp/127.0.0.1:7887",
"ylocator": "tcp/127.0.0.1:7447",
"nodeid": "e0e442af51d14802a9bc71b5e634440e",
"update_interval": 5
}
Expand Down
2 changes: 1 addition & 1 deletion fos-plugins/constraint/constraint.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@
],
"description": "native fdu plugin",
"configuration": {
"ylocator": "tcp/10.100.1.221:7887",
"ylocator": "tcp/10.100.1.221:7447",
"nodeid": "1",
"agent": "e0e442af51d14802a9bc71b5e634440e",
"path": "/var/fos/constraint"
Expand Down
9 changes: 8 additions & 1 deletion fos-plugins/linux/linux_plugin
View file
Original file line number Diff line number Diff line change
Expand Up @@ -471,9 +471,15 @@ class Linux(OSPlugin):
data = ''
if root:
file_path = 'sudo cat {}'.format(file_path)
self.logger.info('read_file()', 'sudo read: {}'.format(file_path))

p = psutil.Popen(file_path.split(), stdout=PIPE)
p.wait()
for line in p.stdout:
data = data + '{}'.format(line)
if isinstance(line, bytes):
data = data + line.decode()
else:
data = data + str(line)
else:
with open(file_path, 'r') as f:
data = f.read()
Expand Down Expand Up @@ -712,6 +718,7 @@ class Linux(OSPlugin):
else:
itype = 'unknown'

self.logger.info('get_intf_type()', 'OS Plugin Interface {} is of type {}'.format(name, itype))
return {'result':itype}

def __get_processor_name(self):
Expand Down
2 changes: 1 addition & 1 deletion fos-plugins/linux/linux_plugin.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
"type": "os",
"description": "linux os plugin",
"configuration": {
"ylocator": "tcp/127.0.0.1:7887",
"ylocator": "tcp/127.0.0.1:7447",
"nodeid": "e0e442af51d14802a9bc71b5e634440e",
"expose": true,
"update_interval": 5
Expand Down
12 changes: 8 additions & 4 deletions fos-plugins/linuxbridge/linuxbridge_plugin
View file
Original file line number Diff line number Diff line change
Expand Up @@ -198,13 +198,17 @@ class LinuxBridge(NetworkPlugin):
n_small_id = netid.split('-')[0]
path = os.path.join(self.BASE_DIR, self.DHCP_DIR)
lease_file = os.path.join(path,'net_{}_leases'.format(n_small_id))
if self.os.read_file(lease_file):
if self.os.file_exists(lease_file):
res = self.os.read_file(lease_file).split('\n')
for l in res:
tok = l.split(' ')
if tok[1] == mac_address:
self.logger.info('get_address()', 'Linux Bridge Plugin - IP address for {} is {}'.format(mac_address, tok[2]))
return {'result': tok[2]}
if len(tok) >= 2:
if tok[1] == mac_address:
self.logger.info('get_address()', 'Linux Bridge Plugin - IP address for {} is {}'.format(mac_address, tok[2]))
return {'result': tok[2]}
else:
self.logger.info('get_address()', 'Linux Bridge Plugin - Unable to get IP address for {} '.format(mac_address))
return {'result': ''}

self.logger.info('get_address()', 'Linux Bridge Plugin - Unable to get IP address for {} '.format(mac_address))
return {'result': ''}
Expand Down
2 changes: 1 addition & 1 deletion fos-plugins/linuxbridge/linuxbridge_plugin.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@
],
"description": "linux Bridge network plugin",
"configuration": {
"ylocator": "tcp/127.0.0.1:7887",
"ylocator": "tcp/127.0.0.1:7447",
"nodeid": "e0e442af51d14802a9bc71b5e634440e",
"dataplane_interface": "ens4",
"use_vlan": false,
Expand Down
Loading

0 comments on commit 2994c4c

Please sign in to comment.