83956db
commit 9f83bb8e80806d3dd79ba426474dc3c696e19a41
83956db
Author: Ben Howard <bh@digitalocean.com>
83956db
Date:   Fri Aug 19 16:28:26 2016 -0600
83956db
83956db
    DigitalOcean: use meta-data for network configruation
83956db
    
83956db
    On DigitalOcean, Network information is provided via Meta-data.
83956db
    It changes the datasource to be a local datasource, meaning it
83956db
    will run before fallback networking is configured.
83956db
    
83956db
    The advantage of that is that before networking is configured it
83956db
    can bring up a network device with ipv4 link-local and hit the
83956db
    metadata service that lives at 169.254.169.254 to find its networking
83956db
    configuration.  It then takes down the link local address and lets
83956db
    cloud-init configure networking.
83956db
    
83956db
    The configuring of a network device to go looking for a metadata
83956db
    service is gated by a check of data in the smbios.  This guarantees
83956db
    that the code will not run on another system.
83956db
83956db
Index: cloud-init-0.7.8/cloudinit/sources/DataSourceDigitalOcean.py
83956db
===================================================================
83956db
--- cloud-init-0.7.8.orig/cloudinit/sources/DataSourceDigitalOcean.py
83956db
+++ cloud-init-0.7.8/cloudinit/sources/DataSourceDigitalOcean.py
83956db
@@ -18,13 +18,12 @@
83956db
 # DigitalOcean Droplet API:
83956db
 # https://developers.digitalocean.com/documentation/metadata/
83956db
 
83956db
-import json
83956db
-
83956db
 from cloudinit import log as logging
83956db
 from cloudinit import sources
83956db
-from cloudinit import url_helper
83956db
 from cloudinit import util
83956db
 
83956db
+import cloudinit.sources.helpers.digitalocean as do_helper
83956db
+
83956db
 LOG = logging.getLogger(__name__)
83956db
 
83956db
 BUILTIN_DS_CONFIG = {
83956db
@@ -36,11 +35,13 @@ BUILTIN_DS_CONFIG = {
83956db
 MD_RETRIES = 30
83956db
 MD_TIMEOUT = 2
83956db
 MD_WAIT_RETRY = 2
83956db
+MD_USE_IPV4LL = True
83956db
 
83956db
 
83956db
 class DataSourceDigitalOcean(sources.DataSource):
83956db
     def __init__(self, sys_cfg, distro, paths):
83956db
         sources.DataSource.__init__(self, sys_cfg, distro, paths)
83956db
+        self.distro = distro
83956db
         self.metadata = dict()
83956db
         self.ds_cfg = util.mergemanydict([
83956db
             util.get_cfg_by_path(sys_cfg, ["datasource", "DigitalOcean"], {}),
83956db
@@ -48,80 +49,72 @@ class DataSourceDigitalOcean(sources.Dat
83956db
         self.metadata_address = self.ds_cfg['metadata_url']
83956db
         self.retries = self.ds_cfg.get('retries', MD_RETRIES)
83956db
         self.timeout = self.ds_cfg.get('timeout', MD_TIMEOUT)
83956db
+        self.use_ip4LL = self.ds_cfg.get('use_ip4LL', MD_USE_IPV4LL)
83956db
         self.wait_retry = self.ds_cfg.get('wait_retry', MD_WAIT_RETRY)
83956db
+        self._network_config = None
83956db
 
83956db
     def _get_sysinfo(self):
83956db
-        # DigitalOcean embeds vendor ID and instance/droplet_id in the
83956db
-        # SMBIOS information
83956db
-
83956db
-        LOG.debug("checking if instance is a DigitalOcean droplet")
83956db
+        return do_helper.read_sysinfo()
83956db
 
83956db
-        # Detect if we are on DigitalOcean and return the Droplet's ID
83956db
-        vendor_name = util.read_dmi_data("system-manufacturer")
83956db
-        if vendor_name != "DigitalOcean":
83956db
-            return (False, None)
83956db
-
83956db
-        LOG.info("running on DigitalOcean")
83956db
-
83956db
-        droplet_id = util.read_dmi_data("system-serial-number")
83956db
-        if droplet_id:
83956db
-            LOG.debug(("system identified via SMBIOS as DigitalOcean Droplet"
83956db
-                       "{}").format(droplet_id))
83956db
-        else:
83956db
-            LOG.critical(("system identified via SMBIOS as a DigitalOcean "
83956db
-                          "Droplet, but did not provide an ID. Please file a "
83956db
-                          "support ticket at: "
83956db
-                          "https://cloud.digitalocean.com/support/tickets/"
83956db
-                          "new"))
83956db
-
83956db
-        return (True, droplet_id)
83956db
-
83956db
-    def get_data(self, apply_filter=False):
83956db
+    def get_data(self):
83956db
         (is_do, droplet_id) = self._get_sysinfo()
83956db
 
83956db
         # only proceed if we know we are on DigitalOcean
83956db
         if not is_do:
83956db
             return False
83956db
 
83956db
-        LOG.debug("reading metadata from {}".format(self.metadata_address))
83956db
-        response = url_helper.readurl(self.metadata_address,
83956db
-                                      timeout=self.timeout,
83956db
-                                      sec_between=self.wait_retry,
83956db
-                                      retries=self.retries)
83956db
-
83956db
-        contents = util.decode_binary(response.contents)
83956db
-        decoded = json.loads(contents)
83956db
-
83956db
-        self.metadata = decoded
83956db
-        self.metadata['instance-id'] = decoded.get('droplet_id', droplet_id)
83956db
-        self.metadata['local-hostname'] = decoded.get('hostname', droplet_id)
83956db
-        self.vendordata_raw = decoded.get("vendor_data", None)
83956db
-        self.userdata_raw = decoded.get("user_data", None)
83956db
-        return True
83956db
+        LOG.info("Running on digital ocean. droplet_id=%s" % droplet_id)
83956db
 
83956db
-    def get_public_ssh_keys(self):
83956db
-        public_keys = self.metadata.get('public_keys', [])
83956db
-        if isinstance(public_keys, list):
83956db
-            return public_keys
83956db
-        else:
83956db
-            return [public_keys]
83956db
+        ipv4LL_nic = None
83956db
+        if self.use_ip4LL:
83956db
+            ipv4LL_nic = do_helper.assign_ipv4_link_local()
83956db
+
83956db
+        md = do_helper.read_metadata(
83956db
+            self.metadata_address, timeout=self.timeout,
83956db
+            sec_between=self.wait_retry, retries=self.retries)
83956db
+
83956db
+        self.metadata_full = md
83956db
+        self.metadata['instance-id'] = md.get('droplet_id', droplet_id)
83956db
+        self.metadata['local-hostname'] = md.get('hostname', droplet_id)
83956db
+        self.metadata['interfaces'] = md.get('interfaces')
83956db
+        self.metadata['public-keys'] = md.get('public_keys')
83956db
+        self.metadata['availability_zone'] = md.get('region', 'default')
83956db
+        self.vendordata_raw = md.get("vendor_data", None)
83956db
+        self.userdata_raw = md.get("user_data", None)
83956db
 
83956db
-    @property
83956db
-    def availability_zone(self):
83956db
-        return self.metadata.get('region', 'default')
83956db
+        if ipv4LL_nic:
83956db
+            do_helper.del_ipv4_link_local(ipv4LL_nic)
83956db
 
83956db
-    @property
83956db
-    def launch_index(self):
83956db
-        return None
83956db
+        return True
83956db
 
83956db
     def check_instance_id(self, sys_cfg):
83956db
         return sources.instance_id_matches_system_uuid(
83956db
             self.get_instance_id(), 'system-serial-number')
83956db
 
83956db
+    @property
83956db
+    def network_config(self):
83956db
+        """Configure the networking. This needs to be done each boot, since
83956db
+           the IP information may have changed due to snapshot and/or
83956db
+           migration.
83956db
+        """
83956db
+
83956db
+        if self._network_config:
83956db
+            return self._network_config
83956db
+
83956db
+        interfaces = self.metadata.get('interfaces')
83956db
+        LOG.debug(interfaces)
83956db
+        if not interfaces:
83956db
+            raise Exception("Unable to get meta-data from server....")
83956db
+
83956db
+        nameservers = self.metadata_full['dns']['nameservers']
83956db
+        self._network_config = do_helper.convert_network_configuration(
83956db
+            interfaces, nameservers)
83956db
+        return self._network_config
83956db
+
83956db
 
83956db
 # Used to match classes to dependencies
83956db
 datasources = [
83956db
-    (DataSourceDigitalOcean, (sources.DEP_FILESYSTEM, sources.DEP_NETWORK)),
83956db
+    (DataSourceDigitalOcean, (sources.DEP_FILESYSTEM, )),
83956db
 ]
83956db
 
83956db
 
83956db
Index: cloud-init-0.7.8/cloudinit/sources/helpers/digitalocean.py
83956db
===================================================================
83956db
--- /dev/null
83956db
+++ cloud-init-0.7.8/cloudinit/sources/helpers/digitalocean.py
83956db
@@ -0,0 +1,218 @@
83956db
+# vi: ts=4 expandtab
83956db
+#
83956db
+#    Author: Ben Howard  <bh@digitalocean.com>
83956db
+
83956db
+#    This program is free software: you can redistribute it and/or modify
83956db
+#    it under the terms of the GNU General Public License version 3, as
83956db
+#    published by the Free Software Foundation.
83956db
+#
83956db
+#    This program is distributed in the hope that it will be useful,
83956db
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
83956db
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
83956db
+#    GNU General Public License for more details.
83956db
+#
83956db
+#    You should have received a copy of the GNU General Public License
83956db
+#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
83956db
+
83956db
+import json
83956db
+import random
83956db
+
83956db
+from cloudinit import log as logging
83956db
+from cloudinit import net as cloudnet
83956db
+from cloudinit import url_helper
83956db
+from cloudinit import util
83956db
+
83956db
+NIC_MAP = {'public': 'eth0', 'private': 'eth1'}
83956db
+
83956db
+LOG = logging.getLogger(__name__)
83956db
+
83956db
+
83956db
+def assign_ipv4_link_local(nic=None):
83956db
+    """Bring up NIC using an address using link-local (ip4LL) IPs. On
83956db
+       DigitalOcean, the link-local domain is per-droplet routed, so there
83956db
+       is no risk of collisions. However, to be more safe, the ip4LL
83956db
+       address is random.
83956db
+    """
83956db
+
83956db
+    if not nic:
83956db
+        for cdev in sorted(cloudnet.get_devicelist()):
83956db
+            if cloudnet.is_physical(cdev):
83956db
+                nic = cdev
83956db
+                LOG.debug("assigned nic '%s' for link-local discovery", nic)
83956db
+                break
83956db
+
83956db
+    if not nic:
83956db
+        raise RuntimeError("unable to find interfaces to access the"
83956db
+                           "meta-data server. This droplet is broken.")
83956db
+
83956db
+    addr = "169.254.{0}.{1}/16".format(random.randint(1, 168),
83956db
+                                       random.randint(0, 255))
83956db
+
83956db
+    ip_addr_cmd = ['ip', 'addr', 'add', addr, 'dev', nic]
83956db
+    ip_link_cmd = ['ip', 'link', 'set', 'dev', nic, 'up']
83956db
+
83956db
+    if not util.which('ip'):
83956db
+        raise RuntimeError("No 'ip' command available to configure ip4LL "
83956db
+                           "address")
83956db
+
83956db
+    try:
83956db
+        (result, _err) = util.subp(ip_addr_cmd)
83956db
+        LOG.debug("assigned ip4LL address '%s' to '%s'", addr, nic)
83956db
+
83956db
+        (result, _err) = util.subp(ip_link_cmd)
83956db
+        LOG.debug("brought device '%s' up", nic)
83956db
+    except Exception:
83956db
+        util.logexc(LOG, "ip4LL address assignment of '%s' to '%s' failed."
83956db
+                         " Droplet networking will be broken", addr, nic)
83956db
+        raise
83956db
+
83956db
+    return nic
83956db
+
83956db
+
83956db
+def del_ipv4_link_local(nic=None):
83956db
+    """Remove the ip4LL address. While this is not necessary, the ip4LL
83956db
+       address is extraneous and confusing to users.
83956db
+    """
83956db
+    if not nic:
83956db
+        LOG.debug("no link_local address interface defined, skipping link "
83956db
+                  "local address cleanup")
83956db
+        return
83956db
+
83956db
+    LOG.debug("cleaning up ipv4LL address")
83956db
+
83956db
+    ip_addr_cmd = ['ip', 'addr', 'flush', 'dev', nic]
83956db
+
83956db
+    try:
83956db
+        (result, _err) = util.subp(ip_addr_cmd)
83956db
+        LOG.debug("removed ip4LL addresses from %s", nic)
83956db
+
83956db
+    except Exception as e:
83956db
+        util.logexc(LOG, "failed to remove ip4LL address from '%s'.", nic, e)
83956db
+
83956db
+
83956db
+def convert_network_configuration(config, dns_servers):
83956db
+    """Convert the DigitalOcean Network description into Cloud-init's netconfig
83956db
+       format.
83956db
+
83956db
+       Example JSON:
83956db
+        {'public': [
83956db
+              {'mac': '04:01:58:27:7f:01',
83956db
+               'ipv4': {'gateway': '45.55.32.1',
83956db
+                        'netmask': '255.255.224.0',
83956db
+                        'ip_address': '45.55.50.93'},
83956db
+               'anchor_ipv4': {
83956db
+                        'gateway': '10.17.0.1',
83956db
+                        'netmask': '255.255.0.0',
83956db
+                        'ip_address': '10.17.0.9'},
83956db
+               'type': 'public',
83956db
+               'ipv6': {'gateway': '....',
83956db
+                        'ip_address': '....',
83956db
+                        'cidr': 64}}
83956db
+           ],
83956db
+          'private': [
83956db
+              {'mac': '04:01:58:27:7f:02',
83956db
+               'ipv4': {'gateway': '10.132.0.1',
83956db
+                        'netmask': '255.255.0.0',
83956db
+                        'ip_address': '10.132.75.35'},
83956db
+               'type': 'private'}
83956db
+           ]
83956db
+        }
83956db
+    """
83956db
+
83956db
+    def _get_subnet_part(pcfg, nameservers=None):
83956db
+        subpart = {'type': 'static',
83956db
+                   'control': 'auto',
83956db
+                   'address': pcfg.get('ip_address'),
83956db
+                   'gateway': pcfg.get('gateway')}
83956db
+
83956db
+        if nameservers:
83956db
+            subpart['dns_nameservers'] = nameservers
83956db
+
83956db
+        if ":" in pcfg.get('ip_address'):
83956db
+            subpart['address'] = "{0}/{1}".format(pcfg.get('ip_address'),
83956db
+                                                  pcfg.get('cidr'))
83956db
+        else:
83956db
+            subpart['netmask'] = pcfg.get('netmask')
83956db
+
83956db
+        return subpart
83956db
+
83956db
+    all_nics = []
83956db
+    for k in ('public', 'private'):
83956db
+        if k in config:
83956db
+            all_nics.extend(config[k])
83956db
+
83956db
+    macs_to_nics = cloudnet.get_interfaces_by_mac()
83956db
+    nic_configs = []
83956db
+
83956db
+    for nic in all_nics:
83956db
+
83956db
+        mac_address = nic.get('mac')
83956db
+        sysfs_name = macs_to_nics.get(mac_address)
83956db
+        nic_type = nic.get('type', 'unknown')
83956db
+        # Note: the entry 'public' above contains a list, but
83956db
+        # the list will only ever have one nic inside it per digital ocean.
83956db
+        # If it ever had more than one nic, then this code would
83956db
+        # assign all 'public' the same name.
83956db
+        if_name = NIC_MAP.get(nic_type, sysfs_name)
83956db
+
83956db
+        LOG.debug("mapped %s interface to %s, assigning name of %s",
83956db
+                  mac_address, sysfs_name, if_name)
83956db
+
83956db
+        ncfg = {'type': 'physical',
83956db
+                'mac_address': mac_address,
83956db
+                'name': if_name}
83956db
+
83956db
+        subnets = []
83956db
+        for netdef in ('ipv4', 'ipv6', 'anchor_ipv4', 'anchor_ipv6'):
83956db
+            raw_subnet = nic.get(netdef, None)
83956db
+            if not raw_subnet:
83956db
+                continue
83956db
+
83956db
+            sub_part = _get_subnet_part(raw_subnet)
83956db
+            if nic_type == 'public' and 'anchor' not in netdef:
83956db
+                # add DNS resolvers to the public interfaces only
83956db
+                sub_part = _get_subnet_part(raw_subnet, dns_servers)
83956db
+            else:
83956db
+                # remove the gateway any non-public interfaces
83956db
+                if 'gateway' in sub_part:
83956db
+                    del sub_part['gateway']
83956db
+
83956db
+            subnets.append(sub_part)
83956db
+
83956db
+        ncfg['subnets'] = subnets
83956db
+        nic_configs.append(ncfg)
83956db
+        LOG.debug("nic '%s' configuration: %s", if_name, ncfg)
83956db
+
83956db
+    return {'version': 1, 'config': nic_configs}
83956db
+
83956db
+
83956db
+def read_metadata(url, timeout=2, sec_between=2, retries=30):
83956db
+    response = url_helper.readurl(url, timeout=timeout,
83956db
+                                  sec_between=sec_between, retries=retries)
83956db
+    if not response.ok():
83956db
+        raise RuntimeError("unable to read metadata at %s" % url)
83956db
+    return json.loads(response.contents.decode())
83956db
+
83956db
+
83956db
+def read_sysinfo():
83956db
+    # DigitalOcean embeds vendor ID and instance/droplet_id in the
83956db
+    # SMBIOS information
83956db
+
83956db
+    # Detect if we are on DigitalOcean and return the Droplet's ID
83956db
+    vendor_name = util.read_dmi_data("system-manufacturer")
83956db
+    if vendor_name != "DigitalOcean":
83956db
+        return (False, None)
83956db
+
83956db
+    droplet_id = util.read_dmi_data("system-serial-number")
83956db
+    if droplet_id:
83956db
+        LOG.debug("system identified via SMBIOS as DigitalOcean Droplet: %s",
83956db
+                  droplet_id)
83956db
+    else:
83956db
+        msg = ("system identified via SMBIOS as a DigitalOcean "
83956db
+               "Droplet, but did not provide an ID. Please file a "
83956db
+               "support ticket at: "
83956db
+               "https://cloud.digitalocean.com/support/tickets/new")
83956db
+        LOG.critical(msg)
83956db
+        raise RuntimeError(msg)
83956db
+
83956db
+    return (True, droplet_id)
83956db
Index: cloud-init-0.7.8/tests/unittests/test_datasource/test_digitalocean.py
83956db
===================================================================
83956db
--- cloud-init-0.7.8.orig/tests/unittests/test_datasource/test_digitalocean.py
83956db
+++ cloud-init-0.7.8/tests/unittests/test_datasource/test_digitalocean.py
83956db
@@ -20,25 +20,123 @@ import json
83956db
 from cloudinit import helpers
83956db
 from cloudinit import settings
83956db
 from cloudinit.sources import DataSourceDigitalOcean
83956db
+from cloudinit.sources.helpers import digitalocean
83956db
 
83956db
-from .. import helpers as test_helpers
83956db
-from ..helpers import HttprettyTestCase
83956db
-
83956db
-httpretty = test_helpers.import_httpretty()
83956db
+from ..helpers import mock, TestCase
83956db
 
83956db
 DO_MULTIPLE_KEYS = ["ssh-rsa AAAAB3NzaC1yc2EAAAA... test1@do.co",
83956db
                     "ssh-rsa AAAAB3NzaC1yc2EAAAA... test2@do.co"]
83956db
 DO_SINGLE_KEY = "ssh-rsa AAAAB3NzaC1yc2EAAAA... test@do.co"
83956db
 
83956db
-DO_META = {
83956db
-    'user_data': 'user_data_here',
83956db
-    'vendor_data': 'vendor_data_here',
83956db
-    'public_keys': DO_SINGLE_KEY,
83956db
-    'region': 'nyc3',
83956db
-    'id': '2000000',
83956db
-    'hostname': 'cloudinit-test',
83956db
+# the following JSON was taken from droplet (that's why its a string)
83956db
+DO_META = json.loads("""
83956db
+{
83956db
+  "droplet_id": "22532410",
83956db
+  "hostname": "utl-96268",
83956db
+  "vendor_data": "vendordata goes here",
83956db
+  "user_data": "userdata goes here",
83956db
+  "public_keys": "",
83956db
+  "auth_key": "authorization_key",
83956db
+  "region": "nyc3",
83956db
+  "interfaces": {
83956db
+    "private": [
83956db
+      {
83956db
+        "ipv4": {
83956db
+          "ip_address": "10.132.6.205",
83956db
+          "netmask": "255.255.0.0",
83956db
+          "gateway": "10.132.0.1"
83956db
+        },
83956db
+        "mac": "04:01:57:d1:9e:02",
83956db
+        "type": "private"
83956db
+      }
83956db
+    ],
83956db
+    "public": [
83956db
+      {
83956db
+        "ipv4": {
83956db
+          "ip_address": "192.0.0.20",
83956db
+          "netmask": "255.255.255.0",
83956db
+          "gateway": "104.236.0.1"
83956db
+        },
83956db
+        "ipv6": {
83956db
+          "ip_address": "2604:A880:0800:0000:1000:0000:0000:0000",
83956db
+          "cidr": 64,
83956db
+          "gateway": "2604:A880:0800:0000:0000:0000:0000:0001"
83956db
+        },
83956db
+        "anchor_ipv4": {
83956db
+          "ip_address": "10.0.0.5",
83956db
+          "netmask": "255.255.0.0",
83956db
+          "gateway": "10.0.0.1"
83956db
+        },
83956db
+        "mac": "04:01:57:d1:9e:01",
83956db
+        "type": "public"
83956db
+      }
83956db
+    ]
83956db
+  },
83956db
+  "floating_ip": {
83956db
+    "ipv4": {
83956db
+      "active": false
83956db
+    }
83956db
+  },
83956db
+  "dns": {
83956db
+    "nameservers": [
83956db
+      "2001:4860:4860::8844",
83956db
+      "2001:4860:4860::8888",
83956db
+      "8.8.8.8"
83956db
+    ]
83956db
+  }
83956db
+}
83956db
+""")
83956db
+
83956db
+# This has no private interface
83956db
+DO_META_2 = {
83956db
+    "droplet_id": 27223699,
83956db
+    "hostname": "smtest1",
83956db
+    "vendor_data": "\n".join([
83956db
+        ('"Content-Type: multipart/mixed; '
83956db
+         'boundary=\"===============8645434374073493512==\"'),
83956db
+        'MIME-Version: 1.0',
83956db
+        '',
83956db
+        '--===============8645434374073493512==',
83956db
+        'MIME-Version: 1.0'
83956db
+        'Content-Type: text/cloud-config; charset="us-ascii"'
83956db
+        'Content-Transfer-Encoding: 7bit'
83956db
+        'Content-Disposition: attachment; filename="cloud-config"'
83956db
+        '',
83956db
+        '#cloud-config',
83956db
+        'disable_root: false',
83956db
+        'manage_etc_hosts: true',
83956db
+        '',
83956db
+        '',
83956db
+        '--===============8645434374073493512=='
83956db
+    ]),
83956db
+    "public_keys": [
83956db
+        "ssh-rsa AAAAB3NzaN...N3NtHw== smoser@brickies"
83956db
+    ],
83956db
+    "auth_key": "88888888888888888888888888888888",
83956db
+    "region": "nyc3",
83956db
+    "interfaces": {
83956db
+        "public": [{
83956db
+            "ipv4": {
83956db
+                "ip_address": "45.55.249.133",
83956db
+                "netmask": "255.255.192.0",
83956db
+                "gateway": "45.55.192.1"
83956db
+            },
83956db
+            "anchor_ipv4": {
83956db
+                "ip_address": "10.17.0.5",
83956db
+                "netmask": "255.255.0.0",
83956db
+                "gateway": "10.17.0.1"
83956db
+            },
83956db
+            "mac": "ae:cc:08:7c:88:00",
83956db
+            "type": "public"
83956db
+        }]
83956db
+    },
83956db
+    "floating_ip": {"ipv4": {"active": True, "ip_address": "138.197.59.92"}},
83956db
+    "dns": {"nameservers": ["8.8.8.8", "8.8.4.4"]},
83956db
+    "tags": None,
83956db
 }
83956db
 
83956db
+DO_META['public_keys'] = DO_SINGLE_KEY
83956db
+
83956db
 MD_URL = 'http://169.254.169.254/metadata/v1.json'
83956db
 
83956db
 
83956db
@@ -46,69 +144,189 @@ def _mock_dmi():
83956db
     return (True, DO_META.get('id'))
83956db
 
83956db
 
83956db
-def _request_callback(method, uri, headers):
83956db
-    return (200, headers, json.dumps(DO_META))
83956db
-
83956db
-
83956db
-class TestDataSourceDigitalOcean(HttprettyTestCase):
83956db
+class TestDataSourceDigitalOcean(TestCase):
83956db
     """
83956db
     Test reading the meta-data
83956db
     """
83956db
 
83956db
-    def setUp(self):
83956db
-        self.ds = DataSourceDigitalOcean.DataSourceDigitalOcean(
83956db
-            settings.CFG_BUILTIN, None,
83956db
-            helpers.Paths({}))
83956db
-        self.ds._get_sysinfo = _mock_dmi
83956db
-        super(TestDataSourceDigitalOcean, self).setUp()
83956db
-
83956db
-    @httpretty.activate
83956db
-    def test_connection(self):
83956db
-        httpretty.register_uri(
83956db
-            httpretty.GET, MD_URL,
83956db
-            body=json.dumps(DO_META))
83956db
-
83956db
-        success = self.ds.get_data()
83956db
-        self.assertTrue(success)
83956db
-
83956db
-    @httpretty.activate
83956db
-    def test_metadata(self):
83956db
-        httpretty.register_uri(
83956db
-            httpretty.GET, MD_URL,
83956db
-            body=_request_callback)
83956db
-        self.ds.get_data()
83956db
-
83956db
-        self.assertEqual(DO_META.get('user_data'),
83956db
-                         self.ds.get_userdata_raw())
83956db
-
83956db
-        self.assertEqual(DO_META.get('vendor_data'),
83956db
-                         self.ds.get_vendordata_raw())
83956db
-
83956db
-        self.assertEqual(DO_META.get('region'),
83956db
-                         self.ds.availability_zone)
83956db
-
83956db
-        self.assertEqual(DO_META.get('id'),
83956db
-                         self.ds.get_instance_id())
83956db
-
83956db
-        self.assertEqual(DO_META.get('hostname'),
83956db
-                         self.ds.get_hostname())
83956db
+    def get_ds(self, get_sysinfo=_mock_dmi):
83956db
+        ds = DataSourceDigitalOcean.DataSourceDigitalOcean(
83956db
+            settings.CFG_BUILTIN, None, helpers.Paths({}))
83956db
+        ds.use_ip4LL = False
83956db
+        if get_sysinfo is not None:
83956db
+            ds._get_sysinfo = get_sysinfo
83956db
+        return ds
83956db
+
83956db
+    @mock.patch('cloudinit.sources.helpers.digitalocean.read_sysinfo')
83956db
+    def test_returns_false_not_on_docean(self, m_read_sysinfo):
83956db
+        m_read_sysinfo.return_value = (False, None)
83956db
+        ds = self.get_ds(get_sysinfo=None)
83956db
+        self.assertEqual(False, ds.get_data())
83956db
+        m_read_sysinfo.assert_called()
83956db
+
83956db
+    @mock.patch('cloudinit.sources.helpers.digitalocean.read_metadata')
83956db
+    def test_metadata(self, mock_readmd):
83956db
+        mock_readmd.return_value = DO_META.copy()
83956db
+
83956db
+        ds = self.get_ds()
83956db
+        ret = ds.get_data()
83956db
+        self.assertTrue(ret)
83956db
+
83956db
+        mock_readmd.assert_called()
83956db
+
83956db
+        self.assertEqual(DO_META.get('user_data'), ds.get_userdata_raw())
83956db
+        self.assertEqual(DO_META.get('vendor_data'), ds.get_vendordata_raw())
83956db
+        self.assertEqual(DO_META.get('region'), ds.availability_zone)
83956db
+        self.assertEqual(DO_META.get('droplet_id'), ds.get_instance_id())
83956db
+        self.assertEqual(DO_META.get('hostname'), ds.get_hostname())
83956db
 
83956db
         # Single key
83956db
         self.assertEqual([DO_META.get('public_keys')],
83956db
-                         self.ds.get_public_ssh_keys())
83956db
+                         ds.get_public_ssh_keys())
83956db
+
83956db
+        self.assertIsInstance(ds.get_public_ssh_keys(), list)
83956db
 
83956db
-        self.assertIsInstance(self.ds.get_public_ssh_keys(), list)
83956db
+    @mock.patch('cloudinit.sources.helpers.digitalocean.read_metadata')
83956db
+    def test_multiple_ssh_keys(self, mock_readmd):
83956db
+        metadata = DO_META.copy()
83956db
+        metadata['public_keys'] = DO_MULTIPLE_KEYS
83956db
+        mock_readmd.return_value = metadata.copy()
83956db
 
83956db
-    @httpretty.activate
83956db
-    def test_multiple_ssh_keys(self):
83956db
-        DO_META['public_keys'] = DO_MULTIPLE_KEYS
83956db
-        httpretty.register_uri(
83956db
-            httpretty.GET, MD_URL,
83956db
-            body=_request_callback)
83956db
-        self.ds.get_data()
83956db
+        ds = self.get_ds()
83956db
+        ret = ds.get_data()
83956db
+        self.assertTrue(ret)
83956db
+
83956db
+        mock_readmd.assert_called()
83956db
 
83956db
         # Multiple keys
83956db
-        self.assertEqual(DO_META.get('public_keys'),
83956db
-                         self.ds.get_public_ssh_keys())
83956db
+        self.assertEqual(metadata['public_keys'], ds.get_public_ssh_keys())
83956db
+        self.assertIsInstance(ds.get_public_ssh_keys(), list)
83956db
+
83956db
+
83956db
+class TestNetworkConvert(TestCase):
83956db
 
83956db
-        self.assertIsInstance(self.ds.get_public_ssh_keys(), list)
83956db
+    def _get_networking(self):
83956db
+        netcfg = digitalocean.convert_network_configuration(
83956db
+            DO_META['interfaces'], DO_META['dns']['nameservers'])
83956db
+        self.assertIn('config', netcfg)
83956db
+        return netcfg
83956db
+
83956db
+    def test_networking_defined(self):
83956db
+        netcfg = self._get_networking()
83956db
+        self.assertIsNotNone(netcfg)
83956db
+
83956db
+        for nic_def in netcfg.get('config'):
83956db
+            print(json.dumps(nic_def, indent=3))
83956db
+            n_type = nic_def.get('type')
83956db
+            n_subnets = nic_def.get('type')
83956db
+            n_name = nic_def.get('name')
83956db
+            n_mac = nic_def.get('mac_address')
83956db
+
83956db
+            self.assertIsNotNone(n_type)
83956db
+            self.assertIsNotNone(n_subnets)
83956db
+            self.assertIsNotNone(n_name)
83956db
+            self.assertIsNotNone(n_mac)
83956db
+
83956db
+    def _get_nic_definition(self, int_type, expected_name):
83956db
+        """helper function to return if_type (i.e. public) and the expected
83956db
+           name used by cloud-init (i.e eth0)"""
83956db
+        netcfg = self._get_networking()
83956db
+        meta_def = (DO_META.get('interfaces')).get(int_type)[0]
83956db
+
83956db
+        self.assertEqual(int_type, meta_def.get('type'))
83956db
+
83956db
+        for nic_def in netcfg.get('config'):
83956db
+            print(nic_def)
83956db
+            if nic_def.get('name') == expected_name:
83956db
+                return nic_def, meta_def
83956db
+
83956db
+    def _get_match_subn(self, subnets, ip_addr):
83956db
+        """get the matching subnet definition based on ip address"""
83956db
+        for subn in subnets:
83956db
+            address = subn.get('address')
83956db
+            self.assertIsNotNone(address)
83956db
+
83956db
+            # equals won't work because of ipv6 addressing being in
83956db
+            # cidr notation, i.e fe00::1/64
83956db
+            if ip_addr in address:
83956db
+                print(json.dumps(subn, indent=3))
83956db
+                return subn
83956db
+
83956db
+    def test_public_interface_defined(self):
83956db
+        """test that the public interface is defined as eth0"""
83956db
+        (nic_def, meta_def) = self._get_nic_definition('public', 'eth0')
83956db
+        self.assertEqual('eth0', nic_def.get('name'))
83956db
+        self.assertEqual(meta_def.get('mac'), nic_def.get('mac_address'))
83956db
+        self.assertEqual('physical', nic_def.get('type'))
83956db
+
83956db
+    def test_private_interface_defined(self):
83956db
+        """test that the private interface is defined as eth1"""
83956db
+        (nic_def, meta_def) = self._get_nic_definition('private', 'eth1')
83956db
+        self.assertEqual('eth1', nic_def.get('name'))
83956db
+        self.assertEqual(meta_def.get('mac'), nic_def.get('mac_address'))
83956db
+        self.assertEqual('physical', nic_def.get('type'))
83956db
+
83956db
+    def _check_dns_nameservers(self, subn_def):
83956db
+        self.assertIn('dns_nameservers', subn_def)
83956db
+        expected_nameservers = DO_META['dns']['nameservers']
83956db
+        nic_nameservers = subn_def.get('dns_nameservers')
83956db
+        self.assertEqual(expected_nameservers, nic_nameservers)
83956db
+
83956db
+    def test_public_interface_ipv6(self):
83956db
+        """test public ipv6 addressing"""
83956db
+        (nic_def, meta_def) = self._get_nic_definition('public', 'eth0')
83956db
+        ipv6_def = meta_def.get('ipv6')
83956db
+        self.assertIsNotNone(ipv6_def)
83956db
+
83956db
+        subn_def = self._get_match_subn(nic_def.get('subnets'),
83956db
+                                        ipv6_def.get('ip_address'))
83956db
+
83956db
+        cidr_notated_address = "{0}/{1}".format(ipv6_def.get('ip_address'),
83956db
+                                                ipv6_def.get('cidr'))
83956db
+
83956db
+        self.assertEqual(cidr_notated_address, subn_def.get('address'))
83956db
+        self.assertEqual(ipv6_def.get('gateway'), subn_def.get('gateway'))
83956db
+        self._check_dns_nameservers(subn_def)
83956db
+
83956db
+    def test_public_interface_ipv4(self):
83956db
+        """test public ipv4 addressing"""
83956db
+        (nic_def, meta_def) = self._get_nic_definition('public', 'eth0')
83956db
+        ipv4_def = meta_def.get('ipv4')
83956db
+        self.assertIsNotNone(ipv4_def)
83956db
+
83956db
+        subn_def = self._get_match_subn(nic_def.get('subnets'),
83956db
+                                        ipv4_def.get('ip_address'))
83956db
+
83956db
+        self.assertEqual(ipv4_def.get('netmask'), subn_def.get('netmask'))
83956db
+        self.assertEqual(ipv4_def.get('gateway'), subn_def.get('gateway'))
83956db
+        self._check_dns_nameservers(subn_def)
83956db
+
83956db
+    def test_public_interface_anchor_ipv4(self):
83956db
+        """test public ipv4 addressing"""
83956db
+        (nic_def, meta_def) = self._get_nic_definition('public', 'eth0')
83956db
+        ipv4_def = meta_def.get('anchor_ipv4')
83956db
+        self.assertIsNotNone(ipv4_def)
83956db
+
83956db
+        subn_def = self._get_match_subn(nic_def.get('subnets'),
83956db
+                                        ipv4_def.get('ip_address'))
83956db
+
83956db
+        self.assertEqual(ipv4_def.get('netmask'), subn_def.get('netmask'))
83956db
+        self.assertNotIn('gateway', subn_def)
83956db
+
83956db
+    def test_convert_without_private(self):
83956db
+        netcfg = digitalocean.convert_network_configuration(
83956db
+            DO_META_2['interfaces'], DO_META_2['dns']['nameservers'])
83956db
+
83956db
+        byname = {}
83956db
+        for i in netcfg['config']:
83956db
+            if 'name' in i:
83956db
+                if i['name'] in byname:
83956db
+                    raise ValueError("name '%s' in config twice: %s" %
83956db
+                                     (i['name'], netcfg))
83956db
+                byname[i['name']] = i
83956db
+        self.assertTrue('eth0' in byname)
83956db
+        self.assertTrue('subnets' in byname['eth0'])
83956db
+        eth0 = byname['eth0']
83956db
+        self.assertEqual(
83956db
+            sorted(['45.55.249.133', '10.17.0.5']),
83956db
+            sorted([i['address'] for i in eth0['subnets']]))