Merge pull request #381 from kellyjonbrazil/dev

Dev v1.23.1

CHANGELOG

@@ -1,5 +1,13 @@
 jc changelog
 
+20230323 v1.23.1
+- Fix `zpool-status` command parser for lines that start with tab
+- Fix `timedatectl` command parser when RTC set to local
+- Fix to ensure `py.typed` file is included in the package wheel
+- Fix `lsusb` command parser to support CDC MBIM and CDC MBIM Extended fields
+- Add support for the `timesync-status` for the `timedatectl` command parser
+- Fix to ignore non-parser-plugins in the parser plugin directory
+
 20230227 v1.23.0
 - Add input slicing as a `jc` command-line option
 - Add `ssh` configuration file parser

README.md

@@ -533,20 +533,22 @@ for item in result:
     print(item["filename"])
 ```
 
-### Custom Parsers
-Custom local parser plugins may be placed in a `jc/jcparsers` folder in your
-local **"App data directory"**:
+### Parser Plugins
+Parser plugins may be placed in a `jc/jcparsers` folder in your local
+**"App data directory"**:
 
 - Linux/unix: `$HOME/.local/share/jc/jcparsers`
 - macOS: `$HOME/Library/Application Support/jc/jcparsers`
 - Windows: `$LOCALAPPDATA\jc\jc\jcparsers`
 
-Local parser plugins are standard python module files. Use the
+Parser plugins are standard python module files. Use the
 [`jc/parsers/foo.py`](https://github.com/kellyjonbrazil/jc/blob/master/jc/parsers/foo.py)
 or [`jc/parsers/foo_s.py (streaming)`](https://github.com/kellyjonbrazil/jc/blob/master/jc/parsers/foo_s.py)
 parser as a template and simply place a `.py` file in the `jcparsers` subfolder.
+Any dependencies can be placed in the `jc` folder above `jcparsers` and can
+be imported in the parser code.
 
-Local plugin filenames must be valid python module names and therefore must
+Parser plugin filenames must be valid python module names and therefore must
 start with a letter and consist entirely of alphanumerics and underscores.
 Local plugins may override default parsers.
 
@@ -604,7 +606,7 @@ they are run on an unsupported platform. To see all parser information,
 including compatibility, run `jc -ap`.
 
 You may still use a parser on an unsupported platform - for example, you may
-want to parse a file with linux `lsof` output on an macOS or Windows laptop. In
+want to parse a file with linux `lsof` output on a macOS or Windows laptop. In
 that case you can suppress the warning message with the `-q` cli option or the
 `quiet=True` function parameter in `parse()`:
 

docs/parsers/lsusb.md

@@ -102,6 +102,24 @@ Schema:
                     ]
                   }
                 },
+                "cdc_mbim": {
+                  "<item>": {
+                    "value":                  string,
+                    "description":            string,
+                    "attributes": [
+                                              string
+                    ]
+                  }
+                },
+                "cdc_mbim_extended": {
+                  "<item>": {
+                    "value":                  string,
+                    "description":            string,
+                    "attributes": [
+                                              string
+                    ]
+                  }
+                },
                 "videocontrol_descriptors": [
                   {
                     "<item>": {
@@ -312,4 +330,4 @@ Returns:
 ### Parser Information
 Compatibility:  linux
 
-Version 1.3 by Kelly Brazil (kellyjonbrazil@gmail.com)
+Version 1.4 by Kelly Brazil (kellyjonbrazil@gmail.com)

docs/parsers/timedatectl.md

@@ -5,6 +5,8 @@
 
 jc - JSON Convert `timedatectl` command output parser
 
+Also supports the `timesync-status` option.
+
 The `epoch_utc` calculated timestamp field is timezone-aware and is only
 available if the `universal_time` field is available.
 
@@ -34,7 +36,24 @@ Schema:
       "system_clock_synchronized":         boolean,
       "systemd-timesyncd.service_active":  boolean,
       "rtc_in_local_tz":                   boolean,
-      "dst_active":                        boolean
+      "dst_active":                        boolean,
+      "server":                            string,
+      "poll_interval":                     string,
+      "leap":                              string,
+      "version":                           integer,
+      "stratum":                           integer,
+      "reference":                         string,
+      "precision":                         string,
+      "root_distance":                     string,
+      "offset":                            float,
+      "offset_unit":                       string,
+      "delay":                             float,
+      "delay_unit":                        string,
+      "jitter":                            float,
+      "jitter_unit":                       string,
+      "packet_count":                      integer,
+      "frequency":                         float,
+      "frequency_unit":                    string
     }
 
 Examples:
@@ -87,4 +106,4 @@ Returns:
 ### Parser Information
 Compatibility:  linux
 
-Version 1.7 by Kelly Brazil (kellyjonbrazil@gmail.com)
+Version 1.8 by Kelly Brazil (kellyjonbrazil@gmail.com)

docs/parsers/zpool_status.md

@@ -160,4 +160,4 @@ Returns:
 ### Parser Information
 Compatibility:  linux, darwin, freebsd
 
-Version 1.0 by Kelly Brazil (kellyjonbrazil@gmail.com)
+Version 1.1 by Kelly Brazil (kellyjonbrazil@gmail.com)

jc/__init__.py

@@ -124,6 +124,14 @@
 `parse()`, `parser_info()`, and `get_help()`. This list is a subset of
 `parser_mod_list()`.
 """
-from .lib import (__version__, parse, parser_mod_list, plugin_parser_mod_list,
-                  standard_parser_mod_list, streaming_parser_mod_list,
-                  parser_info, all_parser_info, get_help)
+from .lib import (
+    __version__ as __version__,
+    parse as parse,
+    parser_mod_list as parser_mod_list,
+    plugin_parser_mod_list as plugin_parser_mod_list,
+    standard_parser_mod_list as standard_parser_mod_list,
+    streaming_parser_mod_list as streaming_parser_mod_list,
+    parser_info as parser_info,
+    all_parser_info as all_parser_info,
+    get_help as get_help
+)

jc/cli.py

@@ -215,14 +215,14 @@ def parser_categories_text(self) -> str:
         category_text: str = ''
         padding_char: str = ' '
         all_parsers = all_parser_info(show_hidden=True, show_deprecated=False)
-        generic = [{'arg': x['argument'], 'desc': x['description']} for x in all_parsers if 'generic' in x['tags']]
-        standard = [{'arg': x['argument'], 'desc': x['description']} for x in all_parsers if 'standard' in x['tags']]
-        command = [{'arg': x['argument'], 'desc': x['description']} for x in all_parsers if 'command' in x['tags']]
+        generic = [{'arg': x['argument'], 'desc': x['description']} for x in all_parsers if 'generic' in x.get('tags', [])]
+        standard = [{'arg': x['argument'], 'desc': x['description']} for x in all_parsers if 'standard' in x.get('tags', [])]
+        command = [{'arg': x['argument'], 'desc': x['description']} for x in all_parsers if 'command' in x.get('tags', [])]
         file_str_bin = [
             {'arg': x['argument'], 'desc': x['description']} for x in all_parsers
-                if 'file' in x['tags'] or
-                'string' in x['tags'] or
-                'binary' in x['tags']
+                if 'file' in x.get('tags', []) or
+                'string' in x.get('tags', []) or
+                'binary' in x.get('tags', [])
         ]
         streaming = [{'arg': x['argument'], 'desc': x['description']} for x in all_parsers if x.get('streaming')]
         categories: Dict = {

jc/lib.py

@@ -9,7 +9,7 @@
 from jc import appdirs
 
 
-__version__ = '1.23.0'
+__version__ = '1.23.1'
 
 parsers: List[str] = [
     'acpi',
@@ -213,6 +213,19 @@ def _modname_to_cliname(parser_mod_name: str) -> str:
     """Return module's cli name (underscores converted to dashes)"""
     return parser_mod_name.replace('_', '-')
 
+def _is_valid_parser_plugin(name: str, local_parsers_dir: str) -> bool:
+    if re.match(r'\w+\.py$', name) and os.path.isfile(os.path.join(local_parsers_dir, name)):
+        try:
+            parser_mod_name = _cliname_to_modname(name)[0:-3]
+            modpath = 'jcparsers.'
+            plugin =  importlib.import_module(f'{modpath}{parser_mod_name}')
+            if hasattr(plugin, 'info') and hasattr(plugin, 'parse'):
+                del plugin
+                return True
+        except Exception:
+            return False
+    return False
+
 # Create the local_parsers list. This is a list of custom or
 # override parsers from <user_data_dir>/jc/jcparsers/*.py.
 # Once this list is created, extend the parsers list with it.
@@ -222,7 +235,7 @@ def _modname_to_cliname(parser_mod_name: str) -> str:
 if os.path.isdir(local_parsers_dir):
     sys.path.append(data_dir)
     for name in os.listdir(local_parsers_dir):
-        if re.match(r'\w+\.py$', name) and os.path.isfile(os.path.join(local_parsers_dir, name)):
+        if _is_valid_parser_plugin(name, local_parsers_dir):
             plugin_name = name[0:-3]
             local_parsers.append(_modname_to_cliname(plugin_name))
             if plugin_name not in parsers:

jc/parsers/lsusb.py

@@ -97,6 +97,24 @@
                     ]
                   }
                 },
+                "cdc_mbim": {
+                  "<item>": {
+                    "value":                  string,
+                    "description":            string,
+                    "attributes": [
+                                              string
+                    ]
+                  }
+                },
+                "cdc_mbim_extended": {
+                  "<item>": {
+                    "value":                  string,
+                    "description":            string,
+                    "attributes": [
+                                              string
+                    ]
+                  }
+                },
                 "videocontrol_descriptors": [
                   {
                     "<item>": {
@@ -291,7 +309,7 @@
 
 class info():
     """Provides parser metadata (version, author, etc.)"""
-    version = '1.3'
+    version = '1.4'
     description = '`lsusb` command parser'
     author = 'Kelly Brazil'
     author_email = 'kellyjonbrazil@gmail.com'
@@ -500,6 +518,8 @@ def __init__(self):
         self.cdc_call_management = _descriptor_obj('cdc_call_management')
         self.cdc_acm = _descriptor_obj('cdc_acm')
         self.cdc_union = _descriptor_obj('cdc_union')
+        self.cdc_mbim = _descriptor_obj('cdc_mbim')
+        self.cdc_mbim_extended = _descriptor_obj('cdc_mbim_extended')
         self.endpoint_descriptors = _descriptor_list('endpoint_descriptor')
         self.videocontrol_interface_descriptors = _descriptor_list('videocontrol_interface_descriptor')
         self.videostreaming_interface_descriptors = _descriptor_list('videostreaming_interface_descriptor')
@@ -538,7 +558,8 @@ def _add_attributes(self, line):
         section_header = self.normal_section_header
 
         if self.section == 'videocontrol_interface_descriptor' \
-            or self.section == 'videostreaming_interface_descriptor':
+            or self.section == 'videostreaming_interface_descriptor' \
+            or self.section == 'cdc_mbim_extended':
 
             section_header = self.larger_section_header
 
@@ -689,6 +710,8 @@ def _set_sections(self, line):
             '      CDC Union:': 'cdc_union',
             '        HID Device Descriptor:': 'hid_device_descriptor',
             '         Report Descriptors:': 'report_descriptors',
+            '      CDC MBIM:': 'cdc_mbim',
+            '      CDC MBIM Extended:': 'cdc_mbim_extended',
             'Hub Descriptor:': 'hub_descriptor',
             ' Hub Port Status:': 'hub_port_status',
             'Device Qualifier (for other device speed):': 'device_qualifier',
@@ -713,6 +736,8 @@ def _populate_lists(self, line):
             'cdc_call_management': self.cdc_call_management.list,
             'cdc_acm': self.cdc_acm.list,
             'cdc_union': self.cdc_union.list,
+            'cdc_mbim': self.cdc_mbim.list,
+            'cdc_mbim_extended': self.cdc_mbim_extended.list,
             'hid_device_descriptor': self.hid_device_descriptor.list,
             # 'report_descriptors': self.report_descriptors_list,         # not implemented
             'videocontrol_interface_descriptor': self.videocontrol_interface_descriptors.list,
@@ -757,6 +782,8 @@ def _populate_schema(self):
         ['device_descriptor']['configuration_descriptor']['interface_descriptors'][0]['cdc_call_management'] = {}
         ['device_descriptor']['configuration_descriptor']['interface_descriptors'][0]['cdc_acm'] = {}
         ['device_descriptor']['configuration_descriptor']['interface_descriptors'][0]['cdc_union'] = {}
+        ['device_descriptor']['configuration_descriptor']['interface_descriptors'][0]['cdc_mbim'] = {}
+        ['device_descriptor']['configuration_descriptor']['interface_descriptors'][0]['cdc_mbim_extended'] = {}
         ['device_descriptor']['configuration_descriptor']['interface_descriptors'][0]['hid_device_descriptor'] = {}
         ['device_descriptor']['configuration_descriptor']['interface_descriptors'][0]['endpoint_descriptors'] = []
         ['device_descriptor']['configuration_descriptor']['interface_descriptors'][0]['endpoint_descriptors'][0] = {}
@@ -847,6 +874,12 @@ def _populate_schema(self):
                     if self.cdc_union._entries_for_this_bus_and_interface_idx_exist(idx, iface_idx):
                         self.cdc_union._update_output(idx, iface_idx, i_desc_obj)
 
+                    if self.cdc_mbim._entries_for_this_bus_and_interface_idx_exist(idx, iface_idx):
+                        self.cdc_mbim._update_output(idx, iface_idx, i_desc_obj)
+
+                    if self.cdc_mbim_extended._entries_for_this_bus_and_interface_idx_exist(idx, iface_idx):
+                        self.cdc_mbim_extended._update_output(idx, iface_idx, i_desc_obj)
+
                     if self.hid_device_descriptor._entries_for_this_bus_and_interface_idx_exist(idx, iface_idx):
                         self.hid_device_descriptor._update_output(idx, iface_idx, i_desc_obj)
 
@@ -923,6 +956,10 @@ def parse(data, raw=False, quiet=False):
     lsusb = _LsUsb()
 
     if jc.utils.has_data(data):
+
+        # fix known too-long field names
+        data = data.replace('bmNetworkCapabilities', 'bmNetworkCapabilit   ')
+
         for line in data.splitlines():
             # only -v option or no options are supported
             if line.startswith('/'):

jc/parsers/proc.py

@@ -204,7 +204,7 @@ def parse(
         pid_smaps_p = re.compile(r'^[0-9a-f]{12}-[0-9a-f]{12} [rwxsp\-]{4} [0-9a-f]{8} [0-9a-f]{2}:[0-9a-f]{2} \d+ [^\n]+\nSize:\s+\d+ \S\S\n')
         pid_stat_p = re.compile(r'^\d+ \(.+\) \S \d+ \d+ \d+ \d+ -?\d+ (?:\d+ ){43}\d+$', re.DOTALL)
         pid_statm_p = re.compile(r'^\d+ \d+ \d+\s\d+\s\d+\s\d+\s\d+$')
-        pid_status_p = re.compile(r'^Name:\t.+\nUmask:\t\d+\nState:\t.+\nTgid:\t\d+\n')
+        pid_status_p = re.compile(r'^Name:\t.+\n(?:Umask:\t\d+\n)?State:\t.+\nTgid:\t\d+\n')
 
         # scsi_device_info = re.compile(r"^'\w+' '.+' 0x\d+")
         # scsi_scsi_p = re.compile(r'^Attached devices:\nHost: \w+ ')