diff --git a/cloudinit/apport.py b/cloudinit/apport.py index 9bded16c697..a3aec2d74ec 100644 --- a/cloudinit/apport.py +++ b/cloudinit/apport.py @@ -41,6 +41,7 @@ 'SmartOS', 'VMware', 'ZStack', + 'GPortal', 'Other' ] diff --git a/cloudinit/settings.py b/cloudinit/settings.py index ca4ffa8e681..e48dcda35fb 100644 --- a/cloudinit/settings.py +++ b/cloudinit/settings.py @@ -41,6 +41,7 @@ 'Oracle', 'Exoscale', 'RbxCloud', + 'GPortal', # At the end to act as a 'catch' when none of the above work... 'None', ], diff --git a/cloudinit/sources/DataSourceGPortal.py b/cloudinit/sources/DataSourceGPortal.py new file mode 100644 index 00000000000..0db6b166d95 --- /dev/null +++ b/cloudinit/sources/DataSourceGPortal.py @@ -0,0 +1,97 @@ +# Author: Alexander Birkner +# +# This file is part of cloud-init. See LICENSE file for license information. + +from cloudinit import log as logging +from cloudinit import sources +from cloudinit import util + +import cloudinit.sources.helpers.gportal as gportal_helper + +LOG = logging.getLogger(__name__) + +BUILTIN_DS_CONFIG = { + 'metadata_url': 'http://169.254.169.254/metadata/v1.json', +} + + +class DataSourceGPortal(sources.DataSource): + dsname = 'GPortal' + + # Available server types + ServerTypeBareMetal = 'BARE_METAL' + ServerTypeVirtual = 'VIRTUAL' + + def __init__(self, sys_cfg, distro, paths): + super(DataSourceGPortal, self).__init__(sys_cfg, distro, paths) + + self.ds_cfg = util.mergemanydict([self.ds_cfg, BUILTIN_DS_CONFIG]) + + self.metadata_address = self.ds_cfg['metadata_url'] + + self.retries = self.ds_cfg.get('retries', 30) + self.timeout = self.ds_cfg.get('timeout', 5) + self.wait_retry = self.ds_cfg.get('wait_retry', 2) + + self._network_config = None + self._server_type = None + + def _get_data(self): + LOG.info("Running on GPortal, downloading meta data now...") + + md = gportal_helper.load_metadata( + self.metadata_address, timeout=self.timeout, + sec_between=self.wait_retry, retries=self.retries) + + self.metadata_full = md + self.metadata['instance-id'] = md.get('id') + self.metadata['local-hostname'] = md.get('fqdn') + self.metadata['interfaces'] = md.get('interfaces') + self.metadata['routes'] = md.get('routes') + self.metadata['public-keys'] = md.get('public_keys') + self.metadata['availability_zone'] = md.get('region', 'unknown') + self.metadata['nameservers'] = md.get('dns', {}).get('nameservers', []) + self.vendordata_raw = md.get("vendor_data") + self.userdata_raw = md.get("user_data") + self._server_type = md.get('type', self.ServerTypeBareMetal) + + LOG.info("Detected GPortal server type: %s", self._server_type) + return True + + def check_instance_id(self, sys_cfg): + # Currently we don't have a way on bare metal nodes + # to detect if the instance-id matches. + if self._server_type == self.ServerTypeBareMetal: + return True + + return sources.instance_id_matches_system_uuid(self.get_instance_id()) + + @property + def network_config(self): + if self._network_config: + return self._network_config + + interfaces = self.metadata.get('interfaces') + + if not interfaces: + raise Exception("Unable to get meta-data from server....") + + nameservers = self.metadata.get('nameservers') + + routes = self.metadata.get('routes') + self._network_config = gportal_helper.convert_network_configuration( + interfaces, nameservers, routes) + return self._network_config + + +# Used to match classes to dependencies +datasources = [ + (DataSourceGPortal, (sources.DEP_FILESYSTEM, sources.DEP_NETWORK)), +] + + +# Return a list of data sources that match this set of dependencies +def get_datasource_list(depends): + return sources.list_from_depends(depends, datasources) + +# vi: ts=4 expandtab diff --git a/cloudinit/sources/helpers/gportal.py b/cloudinit/sources/helpers/gportal.py new file mode 100644 index 00000000000..5df8de6ddde --- /dev/null +++ b/cloudinit/sources/helpers/gportal.py @@ -0,0 +1,99 @@ +# Author: Alexander Birkner +# +# This file is part of cloud-init. See LICENSE file for license information. + +from cloudinit import log as logging +from cloudinit import net as cloudnet +from cloudinit import url_helper +from cloudinit import util + +LOG = logging.getLogger(__name__) + + +def load_metadata(url, timeout=2, sec_between=2, retries=30): + response = url_helper.readurl(url, timeout=timeout, + sec_between=sec_between, retries=retries) + if not response.ok(): + raise RuntimeError("unable to read metadata at %s" % url) + return util.load_json(response.contents.decode()) + + +def convert_network_configuration(config, dns_servers, routes): + """Convert the GPortal Network config into Cloud-Init's netconfig (v1) format. + See: https://cloudinit.readthedocs.io/en/latest/topics/network-config-format-v1.html + + Example JSON: + {'public': [ + {'mac': '40:9a:4c:8d:96:77', + 'ipv4': {'gateway': '176.57.186.5', + 'netmask': '255.255.255.0', + 'address': '176.57.186.1'}, + 'type': 'public' + ] + } + """ # noqa: E501 + + def _convert_subnet(cfg): + subpart = {'type': 'static', + 'control': 'auto', + 'address': cfg.get('address'), + 'gateway': cfg.get('gateway'), + 'netmask': cfg.get('netmask')} + + return subpart + + nic_configs = [] + + # Returns a map where the mac address is the key + # and the value is the interface name (e.g. eno3) + macs_to_nics = cloudnet.get_interfaces_by_mac() + LOG.debug("nic mapping: %s", macs_to_nics) + + for n in config: + nic = config[n][0] + + mac_address = nic.get('mac') + if mac_address not in macs_to_nics: + raise RuntimeError("Did not find network interface on system " + "with mac '%s'. Cannot apply configuration: %s" + % (mac_address, nic)) + + if_name = macs_to_nics.get(mac_address) + nic_type = nic.get('type', 'unknown') + + ncfg = {'type': 'physical', + 'mac_address': mac_address, + 'name': if_name} + + subnets = [] + raw_subnet = nic.get('ipv4', None) + if not raw_subnet: + continue + + # Convert subnet to ncfg subnet + sub_part = _convert_subnet(raw_subnet) + + # Only the public interface has a gateway address + if nic_type != "public": + del sub_part['gateway'] + + subnets.append(sub_part) + + ncfg['subnets'] = subnets + nic_configs.append(ncfg) + LOG.debug("nic '%s' configuration: %s", if_name, ncfg) + + if dns_servers: + LOG.debug("added dns servers: %s", dns_servers) + nic_configs.append({'type': 'nameserver', 'address': dns_servers}) + + if routes is not None: + for route in routes: + nic_configs.append({ + 'type': 'route', + 'destination': route['destination'], + 'gateway': route['gateway'], + 'metric': route['metric'], + }) + + return {'version': 1, 'config': nic_configs} diff --git a/doc/rtd/topics/availability.rst b/doc/rtd/topics/availability.rst index 8449046082a..3974070c8a7 100644 --- a/doc/rtd/topics/availability.rst +++ b/doc/rtd/topics/availability.rst @@ -54,6 +54,7 @@ environments in the public cloud: - CloudStack - AltCloud - SmartOS +- GPortal Additionally, cloud-init is supported on these private clouds: diff --git a/tests/unittests/test_datasource/test_common.py b/tests/unittests/test_datasource/test_common.py index 4ab5d471904..062c342d534 100644 --- a/tests/unittests/test_datasource/test_common.py +++ b/tests/unittests/test_datasource/test_common.py @@ -27,6 +27,7 @@ DataSourceRbxCloud as RbxCloud, DataSourceScaleway as Scaleway, DataSourceSmartOS as SmartOS, + DataSourceGPortal as GPortal, ) from cloudinit.sources import DataSourceNone as DSNone @@ -63,6 +64,7 @@ NoCloud.DataSourceNoCloudNet, OpenStack.DataSourceOpenStack, OVF.DataSourceOVFNet, + GPortal.DataSourceGPortal, ] diff --git a/tests/unittests/test_datasource/test_gportal.py b/tests/unittests/test_datasource/test_gportal.py new file mode 100644 index 00000000000..f6fc0358f67 --- /dev/null +++ b/tests/unittests/test_datasource/test_gportal.py @@ -0,0 +1,83 @@ +# Author: Alexander Birkner +# +# This file is part of cloud-init. See LICENSE file for license information. + +import json + +from cloudinit.sources import DataSourceGPortal +from cloudinit import settings +from cloudinit import helpers + +from cloudinit.tests.helpers import mock, CiTestCase + +METADATA = json.loads(""" +{ + "id": "3f657fa2-7b72-466e-bad4-f83a31fbe5cd", + "type": "BARE_METAL", + "fqdn": "server1", + "region": "FRA01", + "public_keys": [ + "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQCdggAqqRKDj8X9ckZzJ0tB/r/VF6pD5JqP6c2/BHL9ctSae5TQClyOpSJdbp395MCjF3xOe89uK2MeOzUsYNMsTrwYPpFfpndnyAmY8Dc8L/iniqtFnHBCFS+z5VAx1mZNHdRS3NEhTkzPrt7PdmcJ1+cyfrUo9+w3kJMLuc3iyj5ZsoAp7anGtQKLCCxIL5fFCAQcYZR8w/AvLIj4B8sCb6Mon4TF9QhAJB1KbUhEwe3PY3tP1IkjYve9mKM6D5JF/b27ylRvliLLiq92LD/tBjJWjAaw92BqC3fa+Yfx9O+bCrNyP2yFX15L3/nbkppxmlZkk7aHuovkc3W6I5FlcbQfXMUjmMafdA/5Kmss5mgoq0q0E5nWhUu4L2NMW5VAkeTv+lsWLhj1vCo9JC3408mgiNaUn0XNq+uC4J6nXF/qYFNq5XvJPDsaxcxDBJip9dl9a/5BGbo6gAfORMJ1NTzpumKlEIoWZjk/TYVP7Za81UppUkk/n5x8spN70ZxoSIjQ8mXsY/GJ0uAycmy8UYH2hnDmNXiFFEh2E3iDwHYxUaoCVy/kwKBZc7kf1Rc3Fnf+Bt0PU2+k3msqBbpej3ZIoHGMdAaHpI6yYj6c60HC6PYClskkFufbGqyAtN6DcnDD3BthHS096BaTvwpIRRpyrYcPBs4gAccDbX//qw==", + "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDEWZnqgDcJhyK1x7m5X6FNDIeBqM/ekFQbBUreIUW8b2pOhG1ldymCBXWONfhauuNonf9+tz1pEYNzSN7JBpHZYEraETjs32mV+X8ufHdLJhTxUM6KTjfFRRihYEHlD60URZtPFKxQStyue7s9YK02nVTALll1TyuV4wKfkP51V3AFMFjxw00i2swT19RhRTYNYzl8i7E8K14oMDEMFYlmLMeuOMpqJoORvZUX+lBKqn9eLsQmah4LssdVzP0bxMZXCGh+4j9XCk2eroyCgh7lO+obQlZxjltUIXLU/+UtOJIItpI3/2Ux/G8NCZ/FoZllIZwWuEJtB95mSX0g1uLDqSmgTbmvO56njQ8oC/KZMlX1IEEM56uUzPeLpQkclw6EEDsXLGhYn8S44u4PSYdC7DhprbRwNVbfgzfbpfnSSrqbnf1Z0ZugYT+0i8zx5ZyTHDiuN7dvvE8zVW/Oza+DZCctPsFW5RZIQiUENcJdf/NioCCgzJO8slh1RKc8lHpkxyN6YlbY6TbNv9x9Fpi5tHUI1vSeIQ75wTBrTIecIjNJNWZAkqQn2TVwdL6r2FG6Q/Avz2brCybUcNb0Pem3kn9BWrACbN1KPPRb9/iY+3wun8Bzq7CV9/JDrtgzCwFXM44fFiUnzYQgfvG9VpLvq8IVpIFhNlQAvkkNaLZuww==" + ], + "interfaces": { + "public": [ + { + "ipv4": { + "address": "176.57.186.5", + "netmask": "255.255.255.0", + "gateway": "176.57.186.1" + }, + "mac": "40:9a:4c:8d:96:77", + "type": "public" + } + ] + }, + "dns": { + "nameservers": [ + "8.8.8.8", + "1.1.1.1" + ] + } +} +""") # noqa: W291, E501 + + +def _mock_dmi(): + return True, METADATA.get('id') + + +class TestDataSourceGPortal(CiTestCase): + """ + Test reading the meta-data + """ + def setUp(self): + super(TestDataSourceGPortal, self).setUp() + self.tmp = self.tmp_dir() + + def get_ds(self, get_sysinfo=_mock_dmi): + ds = DataSourceGPortal.DataSourceGPortal( + settings.CFG_BUILTIN, None, helpers.Paths({'run_dir': self.tmp})) + if get_sysinfo is not None: + ds._get_sysinfo = get_sysinfo + return ds + + @mock.patch('cloudinit.sources.helpers.gportal.load_metadata') + def test_metadata(self, mock_loadmd): + mock_loadmd.return_value = METADATA.copy() + + ds = self.get_ds() + ret = ds.get_data() + self.assertTrue(ret) + + assert 0 != mock_loadmd.call_count + + self.assertEqual(METADATA.get('user_data'), ds.get_userdata_raw()) + self.assertEqual(METADATA.get('vendor_data'), ds.get_vendordata_raw()) + self.assertEqual(METADATA.get('region'), ds.availability_zone) + self.assertEqual(METADATA.get('id'), ds.get_instance_id()) + self.assertEqual(METADATA.get('fqdn'), ds.get_hostname()) + + self.assertIsInstance(ds.get_public_ssh_keys(), list) + self.assertEqual(METADATA.get('public_keys'), + ds.get_public_ssh_keys()) diff --git a/tools/ds-identify b/tools/ds-identify index 071cdc0cadd..0364e703ebf 100755 --- a/tools/ds-identify +++ b/tools/ds-identify @@ -124,7 +124,7 @@ DI_DSNAME="" # be searched if there is no setting found in config. DI_DSLIST_DEFAULT="MAAS ConfigDrive NoCloud AltCloud Azure Bigstep \ CloudSigma CloudStack DigitalOcean AliYun Ec2 GCE OpenNebula OpenStack \ -OVF SmartOS Scaleway Hetzner IBMCloud Oracle Exoscale RbxCloud" +OVF SmartOS Scaleway Hetzner IBMCloud Oracle Exoscale RbxCloud GPortal" DI_DSLIST="" DI_MODE="" DI_ON_FOUND="" @@ -1172,6 +1172,20 @@ dscheck_Oracle() { return ${DS_NOT_FOUND} } +dscheck_GPortal() { + dmi_sys_vendor_is GPortal && return ${DS_FOUND} + + case " ${DI_KERNEL_CMDLINE} " in + *\ ds=gportal\ *) return ${DS_FOUND};; + esac + + if [ -f "${PATH_ROOT}/var/lib/cloud/.gportal" ]; then + return ${DS_FOUND} + fi + + return ${DS_NOT_FOUND} +} + is_ibm_provisioning() { local pcfg="${PATH_ROOT}/root/provisioningConfiguration.cfg" local logf="${PATH_ROOT}/root/swinstall.log"