Merge branch 'develop' of github.com:napalm-automation/napalm into re…

…corder

docs/support/index.rst

@@ -15,14 +15,14 @@ General support matrix
   **Driver Name**         eos         junos           iosxr        nxos          ios
   **Structured data**     Yes         Yes             No           Yes           No
   **Minimum version**     4.15.0F     12.1            5.1.0        6.1 [#g1]_    12.4(20)T
-  **Backend library**     `pyeapi`_   `junos-eznc`_   `pyIOSXR`_   `pycsco`_     `netmiko`_
+  **Backend library**     `pyeapi`_   `junos-eznc`_   `pyIOSXR`_   `pynxos`_     `netmiko`_
   **Caveats**             :doc:`eos`                               :doc:`nxos`   :doc:`ios`
   =====================   ==========  =============   ============ ============  ============
 
 .. _pyeapi: https://github.com/arista-eosplus/pyeapi
 .. _junos-eznc: https://github.com/Juniper/py-junos-eznc
 .. _pyIOSXR: https://github.com/fooelisa/pyiosxr
-.. _pycsco: https://github.com/jedelman8/pycsco
+.. _pynxos: https://github.com/networktocode/pynxos
 .. _netmiko: https://github.com/ktbyers/netmiko
 
 .. [#g1] NX-API support on the Nexus 5k, 6k and 7k families was introduced in version 7.2

docs/support/ios.rst

@@ -125,9 +125,17 @@ This can be controlled using the `auto_file_prompt` optional arguement:
 * `auto_file_prompt=False`: Disable the above automated behaviour. The managed device must have `file prompt quiet` in its running-config already,
   otherwise a `CommandErrorException` will be raised when file operations are attempted.
 
-Notes
+SCP File Transfers
 _____
 
-* Will automatically enable secure copy ('ip scp server enable') on the network device. This is a configuration change.
+The NAPALM-ios driver requires SCP to be enabled on the managed device. SCP
+server functionality is disabled in IOS by default, and is configured using
+`ip scp server enable`.
+
+If an operation requiring a file transfer is attempted, but the necessary
+configuration is not present, a `CommandErrorException` will be raised.
+
+Notes
+_____
 
 * The NAPALM-ios driver supports all Netmiko arguments as either standard arguments (hostname, username, password, timeout) or as optional_args (everything else).

napalm/base/__init__.py

@@ -75,24 +75,28 @@ def get_network_driver(module_name, prepend=True):
     if not (isinstance(module_name, py23_compat.string_types) and len(module_name) > 0):
         raise ModuleImportError('Please provide a valid driver name.')
 
-    try:
-        # Only lowercase allowed
-        module_name = module_name.lower()
-        # Try to not raise error when users requests IOS-XR for e.g.
-        module_install_name = module_name.replace('-', '')
-        # Can also request using napalm_[SOMETHING]
-        if 'napalm' not in module_install_name and prepend is True:
-            module_install_name = 'napalm.{name}'.format(name=module_install_name)
+    # Only lowercase allowed
+    module_name = module_name.lower()
+    # Try to not raise error when users requests IOS-XR for e.g.
+    module_install_name = module_name.replace('-', '')
+    community_install_name = "napalm_{name}".format(name=module_install_name)
+    custom_install_name = "custom_napalm.{name}".format(name=module_install_name)
+    # Can also request using napalm_[SOMETHING]
+    if 'napalm' not in module_install_name and prepend is True:
+        module_install_name = 'napalm.{name}'.format(name=module_install_name)
+    # Order is custom_napalm_os (local only) ->  napalm.os (core) ->  napalm_os (community)
+    for module_name in [custom_install_name, module_install_name, community_install_name]:
         try:
-            module = importlib.import_module("custom_" + module_install_name)
+            module = importlib.import_module(module_name)
+            break
         except ImportError:
-            module = importlib.import_module(module_install_name)
-    except ImportError:
+            pass
+    else:
         raise ModuleImportError(
-                'Cannot import "{install_name}". Is the library installed?'.format(
-                    install_name=module_install_name
-                )
+            'Cannot import "{install_name}". Is the library installed?'.format(
+                install_name=module_install_name
             )
+        )
 
     for name, obj in inspect.getmembers(module):
         if inspect.isclass(obj) and issubclass(obj, NetworkDriver):

napalm/eos/eos.py

@@ -571,10 +571,12 @@ def extract_temperature_data(data):
         # Matches either of
         # Mem:   3844356k total,  3763184k used,    81172k free,    16732k buffers ( 4.16 > )
         # KiB Mem:  32472080 total,  5697604 used, 26774476 free,   372052 buffers ( 4.16 < )
-        mem_regex = r'.*total,\s+(?P<used>\d+)[k\s]+used,\s+(?P<free>\d+)[k\s]+free,.*'
+        mem_regex = (r'[^\d]*(?P<total>\d+)[k\s]+total,'
+                     r'\s+(?P<used>\d+)[k\s]+used,'
+                     r'\s+(?P<free>\d+)[k\s]+free,.*')
         m = re.match(mem_regex, cpu_lines[3])
         environment_counters['memory'] = {
-            'available_ram': int(m.group('free')),
+            'available_ram': int(m.group('total')),
             'used_ram': int(m.group('used'))
         }
         return environment_counters

napalm/ios/ios.py

@@ -551,9 +551,10 @@ def _xfer_file(self, source_file=None, source_config=None, dest_file=None, file_
         elif source_config:
             kwargs = dict(ssh_conn=self.device, source_config=source_config, dest_file=dest_file,
                           direction='put', file_system=file_system)
-        enable_scp = True
+        use_scp = True
         if self.inline_transfer:
-            enable_scp = False
+            use_scp = False
+
         with TransferClass(**kwargs) as transfer:
 
             # Check if file already exists and has correct MD5
@@ -564,8 +565,14 @@ def _xfer_file(self, source_file=None, source_config=None, dest_file=None, file_
                 msg = "Insufficient space available on remote device"
                 return (False, msg)
 
-            if enable_scp:
-                transfer.enable_scp()
+            if use_scp:
+                cmd = 'ip scp server enable'
+                show_cmd = "show running-config | inc {}".format(cmd)
+                output = self.device.send_command_expect(show_cmd)
+                if cmd not in output:
+                    msg = "SCP file transfers are not enabled. " \
+                          "Configure 'ip scp server enable' on the device."
+                    raise CommandErrorException(msg)
 
             # Transfer file
             transfer.transfer_file()
@@ -1618,29 +1625,37 @@ def get_environment(self):
         output = self._send_command(mem_cmd)
         for line in output.splitlines():
             if 'Processor' in line:
-                _, _, _, proc_used_mem, proc_free_mem = line.split()[:5]
+                _, _, proc_total_mem, proc_used_mem, _ = line.split()[:5]
             elif 'I/O' in line or 'io' in line:
-                _, _, _, io_used_mem, io_free_mem = line.split()[:5]
+                _, _, io_total_mem, io_used_mem, _ = line.split()[:5]
+        total_mem = int(proc_total_mem) + int(io_total_mem)
         used_mem = int(proc_used_mem) + int(io_used_mem)
-        free_mem = int(proc_free_mem) + int(io_free_mem)
         environment.setdefault('memory', {})
         environment['memory']['used_ram'] = used_mem
-        environment['memory']['available_ram'] = free_mem
+        environment['memory']['available_ram'] = total_mem
 
         environment.setdefault('temperature', {})
+        re_temp_value = re.compile('(.*) Temperature Value')
         # The 'show env temperature status' is not ubiquitous in Cisco IOS
         output = self._send_command(temp_cmd)
         if '% Invalid' not in output:
             for line in output.splitlines():
-                if 'System Temperature Value' in line:
-                    system_temp = float(line.split(':')[1].split()[0])
+                m = re_temp_value.match(line)
+                if m is not None:
+                    temp_name = m.group(1).lower()
+                    temp_value = float(line.split(':')[1].split()[0])
+                    env_value = {'is_alert': False,
+                                 'is_critical': False,
+                                 'temperature': temp_value}
+                    environment['temperature'][temp_name] = env_value
                 elif 'Yellow Threshold' in line:
                     system_temp_alert = float(line.split(':')[1].split()[0])
+                    if temp_value > system_temp_alert:
+                        env_value['is_alert'] = True
                 elif 'Red Threshold' in line:
                     system_temp_crit = float(line.split(':')[1].split()[0])
-            env_value = {'is_alert': system_temp >= system_temp_alert,
-                         'is_critical': system_temp >= system_temp_crit, 'temperature': system_temp}
-            environment['temperature']['system'] = env_value
+                    if temp_value > system_temp_crit:
+                        env_value['is_critical'] = True
         else:
             env_value = {'is_alert': False, 'is_critical': False, 'temperature': -1.0}
             environment['temperature']['invalid'] = env_value

napalm/junos/junos.py

@@ -105,7 +105,6 @@ def __init__(self, hostname, username, password, timeout=60, optional_args=None)
         #                         recorder_options=self.recorder_options,
         #                         **connection_params)
         self.device = Device(**connection_params)
-
         self.profile = ["junos"]
 
     def open(self):
@@ -479,7 +478,7 @@ def get_environment(self):
         return environment_data
 
     @staticmethod
-    def _get_address_family(table):
+    def _get_address_family(table, instance):
         """
         Function to derive address family from a junos table name.
 
@@ -491,14 +490,17 @@ def _get_address_family(table):
             'inet6': 'ipv6',
             'inetflow': 'flow'
         }
-        family = table.split('.')[-2]
+        if instance == "master":
+            family = table.rsplit('.', 1)[-2]
+        else:
+            family = table.split('.')[-2]
         try:
             address_family = address_family_mapping[family]
         except KeyError:
-            address_family = family
+            address_family = None
         return address_family
 
-    def _parse_route_stats(self, neighbor):
+    def _parse_route_stats(self, neighbor, instance):
         data = {
             'ipv4': {
                 'received_prefixes': -1,
@@ -519,7 +521,12 @@ def _parse_route_stats(self, neighbor):
                 # is of type int. Therefore convert attribute to list
                 neighbor['sent_prefixes'] = [neighbor['sent_prefixes']]
             for idx, table in enumerate(neighbor['tables']):
-                family = self._get_address_family(table)
+                family = self._get_address_family(table, instance)
+                if family is None:
+                    # Need to remove counter from sent_prefixes list anyway
+                    if 'in sync' in neighbor['send-state'][idx]:
+                        neighbor['sent_prefixes'].pop(0)
+                    continue
                 data[family] = {}
                 data[family]['received_prefixes'] = neighbor['received_prefixes'][idx]
                 data[family]['accepted_prefixes'] = neighbor['accepted_prefixes'][idx]
@@ -528,11 +535,12 @@ def _parse_route_stats(self, neighbor):
                 else:
                     data[family]['sent_prefixes'] = 0
         else:
-            family = self._get_address_family(neighbor['tables'])
-            data[family] = {}
-            data[family]['received_prefixes'] = neighbor['received_prefixes']
-            data[family]['accepted_prefixes'] = neighbor['accepted_prefixes']
-            data[family]['sent_prefixes'] = neighbor['sent_prefixes']
+            family = self._get_address_family(neighbor['tables'], instance)
+            if family is not None:
+                data[family] = {}
+                data[family]['received_prefixes'] = neighbor['received_prefixes']
+                data[family]['accepted_prefixes'] = neighbor['accepted_prefixes']
+                data[family]['sent_prefixes'] = neighbor['sent_prefixes']
         return data
 
     @staticmethod
@@ -608,7 +616,7 @@ def _get_bgp_neighbors_core(neighbor_data, instance=None, uptime_table_items=Non
                 }
                 peer['local_as'] = napalm.base.helpers.as_number(peer['local_as'])
                 peer['remote_as'] = napalm.base.helpers.as_number(peer['remote_as'])
-                peer['address_family'] = self._parse_route_stats(neighbor_details)
+                peer['address_family'] = self._parse_route_stats(neighbor_details, instance)
                 if 'peers' not in bgp_neighbor_data[instance_name]:
                     bgp_neighbor_data[instance_name]['peers'] = {}
                 bgp_neighbor_data[instance_name]['peers'][peer_ip] = peer
@@ -701,12 +709,18 @@ def get_lldp_neighbors_detail(self, interface=''):
         # and rpc for M, MX, and T Series is get-lldp-interface-neighbors
         # ref1: https://apps.juniper.net/xmlapi/operTags.jsp  (Junos 13.1 and later)
         # ref2: https://www.juniper.net/documentation/en_US/junos12.3/information-products/topic-collections/junos-xml-ref-oper/index.html  (Junos 12.3) # noqa
+        # Exceptions:
+        # EX9208    personality = SWITCH    RPC: <get-lldp-interface-neighbors><interface-device>
         lldp_table.GET_RPC = 'get-lldp-interface-neighbors'
-        if self.device.facts.get('personality') not in ('MX', 'M', 'T'):
+        if 'EX9208' in self.device.facts.get('model'):
+            pass
+        elif self.device.facts.get('personality') not in ('MX', 'M', 'T'):
             lldp_table.GET_RPC = 'get-lldp-interface-neighbors-information'
 
         for interface in interfaces:
-            if self.device.facts.get('personality') not in ('MX', 'M', 'T'):
+            if 'EX9208' in self.device.facts.get('model'):
+                lldp_table.get(interface_device=interface)
+            elif self.device.facts.get('personality') not in ('MX', 'M', 'T'):
                 lldp_table.get(interface_name=interface)
             else:
                 lldp_table.get(interface_device=interface)
@@ -787,6 +801,17 @@ def _match(txt, pattern):
             ]
             return '\n'.join(matched)
 
+        def _find(txt, pattern):
+            '''
+            Search for first occurrence of pattern.
+            '''
+            rgx = '^.*({pattern})(.*)$'.format(pattern=pattern)
+            match = re.search(rgx, txt, re.I | re.M | re.DOTALL)
+            if match:
+                return '{pattern}{rest}'.format(pattern=pattern, rest=match.group(2))
+            else:
+                return '\nPattern not found'
+
         def _process_pipe(cmd, txt):
             '''
             Process CLI output from Juniper device that
@@ -800,6 +825,7 @@ def _process_pipe(cmd, txt):
             _OF_MAP['last'] = _last
             _OF_MAP['trim'] = _trim
             _OF_MAP['count'] = _count
+            _OF_MAP['find'] = _find
             # the operations order matter in this case!
             exploded_cmd = cmd.split('|')
             pipe_oper_args = {}

napalm/junos/utils/junos_views.yml

@@ -310,7 +310,7 @@ junos_bgp_config_peers_view:
     local_as: {local-as/as-number: int}
     cluster: cluster
     remote_as: {peer-as: int}
-    authentication_key: {authentication_key: unicode}
+    authentication_key: {authentication-key: unicode}
     inet_unicast_limit_prefix_limit: {family/inet/unicast/prefix-limit/maximum: int}
     inet_unicast_teardown_threshold_prefix_limit: {family/inet/unicast/prefix-limit/teardown/limit-threshold: int}
     inet_unicast_teardown_timeout_prefix_limit: {family/inet/unicast/prefix-limit/teardown/idle-timeout/timeout: int}

napalm/nxos_ssh/nxos_ssh.py

@@ -1311,8 +1311,7 @@ def process_mac_fields(vlan, mac, mac_type, interface):
 
         for line in output.splitlines():
 
-            # Every 500 Mac's Legend is reprinted, regardless of term len.
-            # Above split will not help in this scenario
+            # Every 500 Mac's Legend is reprinted, regardless of terminal length
             if re.search(r'^Legend', line):
                 continue
             elif re.search(r'^\s+\* \- primary entry', line):
@@ -1328,24 +1327,23 @@ def process_mac_fields(vlan, mac, mac_type, interface):
 
             for pattern in [RE_MACTABLE_FORMAT1, RE_MACTABLE_FORMAT2, RE_MACTABLE_FORMAT3]:
                 if re.search(pattern, line):
-                    split = line.split()
-                    if len(split) >= 7:
-                        vlan, mac, mac_type, _, _, _, interface = split[:7]
+                    fields = line.split()
+                    if len(fields) >= 7:
+                        vlan, mac, mac_type, _, _, _, interface = fields[:7]
                         mac_address_table.append(process_mac_fields(vlan, mac, mac_type,
                                                                     interface))
 
-                        # if there is multiples interfaces for the mac address on the same line
-                        for i in range(7, len(split)):
+                        # there can be multiples interfaces for the same MAC on the same line
+                        for interface in fields[7:]:
                             mac_address_table.append(process_mac_fields(vlan, mac, mac_type,
-                                                                        split[i]))
+                                                                        interface))
                         break
 
-                    # if there is multiples interfaces for the mac address on next lines
-                    # we reuse the old variable 'vlan' 'mac' and 'mac_type'
-                    elif len(split) < 7:
-                        for i in range(0, len(split)):
+                    # interfaces can overhang to the next line (line only contains interfaces)
+                    elif len(fields) < 7:
+                        for interface in fields:
                             mac_address_table.append(process_mac_fields(vlan, mac, mac_type,
-                                                                        split[i]))
+                                                                        interface))
                         break
             else:
                 raise ValueError("Unexpected output from: {}".format(repr(line)))

test/eos/mocked_data/test_get_environment/issue80_stacktrace_for_get_environment/expected_result.json

@@ -359,7 +359,7 @@
     }
   },
   "memory": {
-    "available_ram": 26775428,
+    "available_ram": 32472080,
     "used_ram": 5696652
   },
   "cpu": {

test/eos/mocked_data/test_get_environment/issue90_veos/expected_result.json

@@ -1 +1 @@
-{"fans": {}, "cpu": {"0": {"%usage": 14.6}}, "temperature": {}, "power": {}, "memory": {"available_ram": 65936, "used_ram": 1831660}}
+{"fans": {}, "cpu": {"0": {"%usage": 14.6}}, "temperature": {}, "power": {}, "memory": {"available_ram": 1897596, "used_ram": 1831660}}

test/eos/mocked_data/test_get_environment/normal/expected_result.json

@@ -359,7 +359,7 @@
     }
   },
   "memory": {
-    "available_ram": 11431608,
+    "available_ram": 16012300,
     "used_ram": 4580692
   },
   "cpu": {