Added new Datasource GPortal to Cloud-Init

cloudinit/apport.py

@@ -41,6 +41,7 @@
     'SmartOS',
     'VMware',
     'ZStack',
+    'GPortal',
     'Other'
 ]
 

cloudinit/settings.py

@@ -41,6 +41,7 @@
         'Oracle',
         'Exoscale',
         'RbxCloud',
+        'GPortal',
         # At the end to act as a 'catch' when none of the above work...
         'None',
     ],

cloudinit/sources/DataSourceGPortal.py

@@ -0,0 +1,97 @@
+# Author: Alexander Birkner <alexander.birkner@g-portal.com>
+#
+# This file is part of cloud-init. See LICENSE file for license information.
+
+from cloudinit import log as logging
+from cloudinit import sources
+from cloudinit import util
+
+import cloudinit.sources.helpers.gportal as gportal_helper
+
+LOG = logging.getLogger(__name__)
+
+BUILTIN_DS_CONFIG = {
+    'metadata_url': 'http://169.254.169.254/metadata/v1.json',
+}
+
+
+class DataSourceGPortal(sources.DataSource):
+    dsname = 'GPortal'
+
+    # Available server types
+    ServerTypeBareMetal = 'BARE_METAL'
+    ServerTypeVirtual = 'VIRTUAL'
+
+    def __init__(self, sys_cfg, distro, paths):
+        super(DataSourceGPortal, self).__init__(sys_cfg, distro, paths)
+
+        self.ds_cfg = util.mergemanydict([self.ds_cfg, BUILTIN_DS_CONFIG])
+
+        self.metadata_address = self.ds_cfg['metadata_url']
+
+        self.retries = self.ds_cfg.get('retries', 30)
+        self.timeout = self.ds_cfg.get('timeout', 5)
+        self.wait_retry = self.ds_cfg.get('wait_retry', 2)
+
+        self._network_config = None
+        self._server_type = None
+
+    def _get_data(self):
+        LOG.info("Running on GPortal, downloading meta data now...")
+
+        md = gportal_helper.load_metadata(
+            self.metadata_address, timeout=self.timeout,
+            sec_between=self.wait_retry, retries=self.retries)
+
+        self.metadata_full = md
+        self.metadata['instance-id'] = md.get('id')
+        self.metadata['local-hostname'] = md.get('fqdn')
+        self.metadata['interfaces'] = md.get('interfaces')
+        self.metadata['routes'] = md.get('routes')
+        self.metadata['public-keys'] = md.get('public_keys')
+        self.metadata['availability_zone'] = md.get('region', 'unknown')
+        self.metadata['nameservers'] = md.get('dns', {}).get('nameservers', [])
+        self.vendordata_raw = md.get("vendor_data")
+        self.userdata_raw = md.get("user_data")
+        self._server_type = md.get('type', self.ServerTypeBareMetal)
+
+        LOG.info("Detected GPortal server type: %s", self._server_type)
+        return True
+
+    def check_instance_id(self, sys_cfg):
+        # Currently we don't have a way on bare metal nodes
+        # to detect if the instance-id matches.
+        if self._server_type == self.ServerTypeBareMetal:
+            return True
+
+        return sources.instance_id_matches_system_uuid(self.get_instance_id())
+
+    @property
+    def network_config(self):
+        if self._network_config:
+            return self._network_config
+
+        interfaces = self.metadata.get('interfaces')
+
+        if not interfaces:
+            raise Exception("Unable to get meta-data from server....")
+
+        nameservers = self.metadata.get('nameservers')
+
+        routes = self.metadata.get('routes')
+        self._network_config = gportal_helper.convert_network_configuration(
+            interfaces, nameservers, routes)
+        return self._network_config
+
+
+# Used to match classes to dependencies
+datasources = [
+    (DataSourceGPortal, (sources.DEP_FILESYSTEM, sources.DEP_NETWORK)),
+]
+
+
+# Return a list of data sources that match this set of dependencies
+def get_datasource_list(depends):
+    return sources.list_from_depends(depends, datasources)
+
+# vi: ts=4 expandtab

cloudinit/sources/helpers/gportal.py

@@ -0,0 +1,99 @@
+# Author: Alexander Birkner <alexander.birkner@g-portal.com>
+#
+# This file is part of cloud-init. See LICENSE file for license information.
+
+from cloudinit import log as logging
+from cloudinit import net as cloudnet
+from cloudinit import url_helper
+from cloudinit import util
+
+LOG = logging.getLogger(__name__)
+
+
+def load_metadata(url, timeout=2, sec_between=2, retries=30):
+    response = url_helper.readurl(url, timeout=timeout,
+                                  sec_between=sec_between, retries=retries)
+    if not response.ok():
+        raise RuntimeError("unable to read metadata at %s" % url)
+    return util.load_json(response.contents.decode())
+
+
+def convert_network_configuration(config, dns_servers, routes):
+    """Convert the GPortal Network config into Cloud-Init's netconfig (v1) format.
+       See: https://cloudinit.readthedocs.io/en/latest/topics/network-config-format-v1.html
+
+       Example JSON:
+        {'public': [
+              {'mac': '40:9a:4c:8d:96:77',
+               'ipv4': {'gateway': '176.57.186.5',
+                        'netmask': '255.255.255.0',
+                        'address': '176.57.186.1'},
+               'type': 'public'
+           ]
+        }
+    """  # noqa: E501
+
+    def _convert_subnet(cfg):
+        subpart = {'type': 'static',
+                   'control': 'auto',
+                   'address': cfg.get('address'),
+                   'gateway': cfg.get('gateway'),
+                   'netmask': cfg.get('netmask')}
+
+        return subpart
+
+    nic_configs = []
+
+    # Returns a map where the mac address is the key
+    # and the value is the interface name (e.g. eno3)
+    macs_to_nics = cloudnet.get_interfaces_by_mac()
+    LOG.debug("nic mapping: %s", macs_to_nics)
+
+    for n in config:
+        nic = config[n][0]
+
+        mac_address = nic.get('mac')
+        if mac_address not in macs_to_nics:
+            raise RuntimeError("Did not find network interface on system "
+                               "with mac '%s'. Cannot apply configuration: %s"
+                               % (mac_address, nic))
+
+        if_name = macs_to_nics.get(mac_address)
+        nic_type = nic.get('type', 'unknown')
+
+        ncfg = {'type': 'physical',
+                'mac_address': mac_address,
+                'name': if_name}
+
+        subnets = []
+        raw_subnet = nic.get('ipv4', None)
+        if not raw_subnet:
+            continue
+
+        # Convert subnet to ncfg subnet
+        sub_part = _convert_subnet(raw_subnet)
+
+        # Only the public interface has a gateway address
+        if nic_type != "public":
+            del sub_part['gateway']
+
+        subnets.append(sub_part)
+
+        ncfg['subnets'] = subnets
+        nic_configs.append(ncfg)
+        LOG.debug("nic '%s' configuration: %s", if_name, ncfg)
+
+    if dns_servers:
+        LOG.debug("added dns servers: %s", dns_servers)
+        nic_configs.append({'type': 'nameserver', 'address': dns_servers})
+
+    if routes is not None:
+        for route in routes:
+            nic_configs.append({
+                'type': 'route',
+                'destination': route['destination'],
+                'gateway': route['gateway'],
+                'metric': route['metric'],
+            })
+
+    return {'version': 1, 'config': nic_configs}

doc/rtd/topics/availability.rst

@@ -54,6 +54,7 @@ environments in the public cloud:
 - CloudStack
 - AltCloud
 - SmartOS
+- GPortal
 
 Additionally, cloud-init is supported on these private clouds:
 

tests/unittests/test_datasource/test_common.py

@@ -27,6 +27,7 @@
     DataSourceRbxCloud as RbxCloud,
     DataSourceScaleway as Scaleway,
     DataSourceSmartOS as SmartOS,
+    DataSourceGPortal as GPortal,
 )
 from cloudinit.sources import DataSourceNone as DSNone
 
@@ -63,6 +64,7 @@
     NoCloud.DataSourceNoCloudNet,
     OpenStack.DataSourceOpenStack,
     OVF.DataSourceOVFNet,
+    GPortal.DataSourceGPortal,
 ]
 
 

tests/unittests/test_datasource/test_gportal.py

@@ -0,0 +1,83 @@
+# Author: Alexander Birkner <alexander.birkner@g-portal.com>
+#
+# This file is part of cloud-init. See LICENSE file for license information.
+
+import json
+
+from cloudinit.sources import DataSourceGPortal
+from cloudinit import settings
+from cloudinit import helpers
+
+from cloudinit.tests.helpers import mock, CiTestCase
+
+METADATA = json.loads("""
+{
+  "id": "3f657fa2-7b72-466e-bad4-f83a31fbe5cd",
+  "type": "BARE_METAL",  
+  "fqdn": "server1",
+  "region": "FRA01",
+  "public_keys": [
+    "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQCdggAqqRKDj8X9ckZzJ0tB/r/VF6pD5JqP6c2/BHL9ctSae5TQClyOpSJdbp395MCjF3xOe89uK2MeOzUsYNMsTrwYPpFfpndnyAmY8Dc8L/iniqtFnHBCFS+z5VAx1mZNHdRS3NEhTkzPrt7PdmcJ1+cyfrUo9+w3kJMLuc3iyj5ZsoAp7anGtQKLCCxIL5fFCAQcYZR8w/AvLIj4B8sCb6Mon4TF9QhAJB1KbUhEwe3PY3tP1IkjYve9mKM6D5JF/b27ylRvliLLiq92LD/tBjJWjAaw92BqC3fa+Yfx9O+bCrNyP2yFX15L3/nbkppxmlZkk7aHuovkc3W6I5FlcbQfXMUjmMafdA/5Kmss5mgoq0q0E5nWhUu4L2NMW5VAkeTv+lsWLhj1vCo9JC3408mgiNaUn0XNq+uC4J6nXF/qYFNq5XvJPDsaxcxDBJip9dl9a/5BGbo6gAfORMJ1NTzpumKlEIoWZjk/TYVP7Za81UppUkk/n5x8spN70ZxoSIjQ8mXsY/GJ0uAycmy8UYH2hnDmNXiFFEh2E3iDwHYxUaoCVy/kwKBZc7kf1Rc3Fnf+Bt0PU2+k3msqBbpej3ZIoHGMdAaHpI6yYj6c60HC6PYClskkFufbGqyAtN6DcnDD3BthHS096BaTvwpIRRpyrYcPBs4gAccDbX//qw==",
+    "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDEWZnqgDcJhyK1x7m5X6FNDIeBqM/ekFQbBUreIUW8b2pOhG1ldymCBXWONfhauuNonf9+tz1pEYNzSN7JBpHZYEraETjs32mV+X8ufHdLJhTxUM6KTjfFRRihYEHlD60URZtPFKxQStyue7s9YK02nVTALll1TyuV4wKfkP51V3AFMFjxw00i2swT19RhRTYNYzl8i7E8K14oMDEMFYlmLMeuOMpqJoORvZUX+lBKqn9eLsQmah4LssdVzP0bxMZXCGh+4j9XCk2eroyCgh7lO+obQlZxjltUIXLU/+UtOJIItpI3/2Ux/G8NCZ/FoZllIZwWuEJtB95mSX0g1uLDqSmgTbmvO56njQ8oC/KZMlX1IEEM56uUzPeLpQkclw6EEDsXLGhYn8S44u4PSYdC7DhprbRwNVbfgzfbpfnSSrqbnf1Z0ZugYT+0i8zx5ZyTHDiuN7dvvE8zVW/Oza+DZCctPsFW5RZIQiUENcJdf/NioCCgzJO8slh1RKc8lHpkxyN6YlbY6TbNv9x9Fpi5tHUI1vSeIQ75wTBrTIecIjNJNWZAkqQn2TVwdL6r2FG6Q/Avz2brCybUcNb0Pem3kn9BWrACbN1KPPRb9/iY+3wun8Bzq7CV9/JDrtgzCwFXM44fFiUnzYQgfvG9VpLvq8IVpIFhNlQAvkkNaLZuww=="
+  ],
+  "interfaces": {
+    "public": [
+      {
+        "ipv4": {
+          "address": "176.57.186.5",
+          "netmask": "255.255.255.0",
+          "gateway": "176.57.186.1"
+        },
+        "mac": "40:9a:4c:8d:96:77",
+        "type": "public"
+      }
+    ]
+  },
+  "dns": {
+    "nameservers": [
+      "8.8.8.8",
+      "1.1.1.1"
+    ]
+  }
+}
+""")  # noqa: W291, E501
+
+
+def _mock_dmi():
+    return True, METADATA.get('id')
+
+
+class TestDataSourceGPortal(CiTestCase):
+    """
+    Test reading the meta-data
+    """
+    def setUp(self):
+        super(TestDataSourceGPortal, self).setUp()
+        self.tmp = self.tmp_dir()
+
+    def get_ds(self, get_sysinfo=_mock_dmi):
+        ds = DataSourceGPortal.DataSourceGPortal(
+            settings.CFG_BUILTIN, None, helpers.Paths({'run_dir': self.tmp}))
+        if get_sysinfo is not None:
+            ds._get_sysinfo = get_sysinfo
+        return ds
+
+    @mock.patch('cloudinit.sources.helpers.gportal.load_metadata')
+    def test_metadata(self, mock_loadmd):
+        mock_loadmd.return_value = METADATA.copy()
+
+        ds = self.get_ds()
+        ret = ds.get_data()
+        self.assertTrue(ret)
+
+        assert 0 != mock_loadmd.call_count
+
+        self.assertEqual(METADATA.get('user_data'), ds.get_userdata_raw())
+        self.assertEqual(METADATA.get('vendor_data'), ds.get_vendordata_raw())
+        self.assertEqual(METADATA.get('region'), ds.availability_zone)
+        self.assertEqual(METADATA.get('id'), ds.get_instance_id())
+        self.assertEqual(METADATA.get('fqdn'), ds.get_hostname())
+
+        self.assertIsInstance(ds.get_public_ssh_keys(), list)
+        self.assertEqual(METADATA.get('public_keys'),
+                         ds.get_public_ssh_keys())

tools/ds-identify

@@ -124,7 +124,7 @@ DI_DSNAME=""
 # be searched if there is no setting found in config.
 DI_DSLIST_DEFAULT="MAAS ConfigDrive NoCloud AltCloud Azure Bigstep \
 CloudSigma CloudStack DigitalOcean AliYun Ec2 GCE OpenNebula OpenStack \
-OVF SmartOS Scaleway Hetzner IBMCloud Oracle Exoscale RbxCloud"
+OVF SmartOS Scaleway Hetzner IBMCloud Oracle Exoscale RbxCloud GPortal"
 DI_DSLIST=""
 DI_MODE=""
 DI_ON_FOUND=""
@@ -1172,6 +1172,20 @@ dscheck_Oracle() {
     return ${DS_NOT_FOUND}
 }
 
+dscheck_GPortal() {
+    dmi_sys_vendor_is GPortal && return ${DS_FOUND}
+
+    case " ${DI_KERNEL_CMDLINE} " in
+        *\ ds=gportal\ *) return ${DS_FOUND};;
+    esac
+
+    if [ -f "${PATH_ROOT}/var/lib/cloud/.gportal" ]; then
+        return ${DS_FOUND}
+    fi
+
+    return ${DS_NOT_FOUND}
+}
+
 is_ibm_provisioning() {
     local pcfg="${PATH_ROOT}/root/provisioningConfiguration.cfg"
     local logf="${PATH_ROOT}/root/swinstall.log"