From 59662a150028718b1657f69eebc5dc10d849ef75 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Sun, 5 Mar 2023 10:04:00 -0800 Subject: [PATCH 01/18] fix for lines that start with tab --- jc/parsers/zpool_status.py | 4 ++-- tests/fixtures/generic/zpool-status-v4.json | 1 + tests/fixtures/generic/zpool-status-v4.out | 21 +++++++++++++++++++++ tests/test_zpool_status.py | 14 +++++++++++++- 4 files changed, 37 insertions(+), 3 deletions(-) create mode 100644 tests/fixtures/generic/zpool-status-v4.json create mode 100644 tests/fixtures/generic/zpool-status-v4.out diff --git a/jc/parsers/zpool_status.py b/jc/parsers/zpool_status.py index 27c831b19..bce6c0018 100644 --- a/jc/parsers/zpool_status.py +++ b/jc/parsers/zpool_status.py @@ -138,7 +138,7 @@ class info(): """Provides parser metadata (version, author, etc.)""" - version = '1.0' + version = '1.1' description = '`zpool status` command parser' author = 'Kelly Brazil' author_email = 'kellyjonbrazil@gmail.com' @@ -233,7 +233,7 @@ def parse( continue # preserve indentation in continuation lines - if line.startswith(' '): + if line.startswith(' ') or line.startswith('\t'): pool_str += line + '\n' continue diff --git a/tests/fixtures/generic/zpool-status-v4.json b/tests/fixtures/generic/zpool-status-v4.json new file mode 100644 index 000000000..69635305e --- /dev/null +++ b/tests/fixtures/generic/zpool-status-v4.json @@ -0,0 +1 @@ +[{"pool":"pool1","state":"ONLINE","status":"Some supported and requested features are not enabled on the pool.\nThe pool can still be used, but some features are unavailable.","action":"Enable all features using 'zpool upgrade'. Once this is done,\nthe pool may no longer be accessible by software that does not support\nthe features. See zpool-features(7) for details.","scan":"scrub repaired 0B in 11:16:03 with 0 errors on Sun Feb 12 11:40:04 2023","config":[{"name":"pool1","state":"ONLINE","read":0,"write":0,"checksum":0},{"name":"mirror-0","state":"ONLINE","read":0,"write":0,"checksum":0},{"name":"wwn-0x5000c500c65ac66f","state":"ONLINE","read":0,"write":0,"checksum":0},{"name":"wwn-0x5000c500c5eee542","state":"ONLINE","read":0,"write":0,"checksum":0},{"name":"mirror-1","state":"ONLINE","read":0,"write":0,"checksum":0},{"name":"wwn-0x5000c500e39e8af6","state":"ONLINE","read":0,"write":0,"checksum":0},{"name":"wwn-0x5000c500e3b3a41e","state":"ONLINE","read":0,"write":0,"checksum":0}],"errors":"No known data errors"}] diff --git a/tests/fixtures/generic/zpool-status-v4.out b/tests/fixtures/generic/zpool-status-v4.out new file mode 100644 index 000000000..01a2f8078 --- /dev/null +++ b/tests/fixtures/generic/zpool-status-v4.out @@ -0,0 +1,21 @@ + pool: pool1 + state: ONLINE +status: Some supported and requested features are not enabled on the pool. + The pool can still be used, but some features are unavailable. +action: Enable all features using 'zpool upgrade'. Once this is done, + the pool may no longer be accessible by software that does not support + the features. See zpool-features(7) for details. + scan: scrub repaired 0B in 11:16:03 with 0 errors on Sun Feb 12 11:40:04 2023 +config: + + NAME STATE READ WRITE CKSUM + pool1 ONLINE 0 0 0 + mirror-0 ONLINE 0 0 0 + wwn-0x5000c500c65ac66f ONLINE 0 0 0 + wwn-0x5000c500c5eee542 ONLINE 0 0 0 + mirror-1 ONLINE 0 0 0 + wwn-0x5000c500e39e8af6 ONLINE 0 0 0 + wwn-0x5000c500e3b3a41e ONLINE 0 0 0 + +errors: No known data errors + diff --git a/tests/test_zpool_status.py b/tests/test_zpool_status.py index e31c4c4ce..122ac3ef8 100644 --- a/tests/test_zpool_status.py +++ b/tests/test_zpool_status.py @@ -22,7 +22,10 @@ def setUpClass(cls): 'fixtures/generic/zpool-status-v2.json'), 'zpool_status3': ( 'fixtures/generic/zpool-status-v3.out', - 'fixtures/generic/zpool-status-v3.json') + 'fixtures/generic/zpool-status-v3.json'), + 'zpool_status4': ( + 'fixtures/generic/zpool-status-v4.out', + 'fixtures/generic/zpool-status-v4.json') } for file, filepaths in fixtures.items(): @@ -66,6 +69,15 @@ def test_zpool_status_v_3(self): self.f_json['zpool_status3'] ) + def test_zpool_status_v_with_tabs(self): + """ + Test 'zpool status -v' with tabs instead of spaces + """ + self.assertEqual( + parse(self.f_in['zpool_status4'], quiet=True), + self.f_json['zpool_status4'] + ) + if __name__ == '__main__': unittest.main() From 92ad2068db1e930622bdfddabfdb1c754c8ff586 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Sun, 5 Mar 2023 10:06:54 -0800 Subject: [PATCH 02/18] version bump --- CHANGELOG | 3 +++ docs/parsers/zpool_status.md | 2 +- jc/lib.py | 2 +- man/jc.1 | 2 +- setup.py | 2 +- 5 files changed, 7 insertions(+), 4 deletions(-) diff --git a/CHANGELOG b/CHANGELOG index 201df8518..65474600f 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,5 +1,8 @@ jc changelog +20230305 v1.23.1 +- Fix `zpool-status` command parser for lines that start with tab + 20230227 v1.23.0 - Add input slicing as a `jc` command-line option - Add `ssh` configuration file parser diff --git a/docs/parsers/zpool_status.md b/docs/parsers/zpool_status.md index b0b84a522..9fc005af3 100644 --- a/docs/parsers/zpool_status.md +++ b/docs/parsers/zpool_status.md @@ -160,4 +160,4 @@ Returns: ### Parser Information Compatibility: linux, darwin, freebsd -Version 1.0 by Kelly Brazil (kellyjonbrazil@gmail.com) +Version 1.1 by Kelly Brazil (kellyjonbrazil@gmail.com) diff --git a/jc/lib.py b/jc/lib.py index bc97d64ea..4621b9070 100644 --- a/jc/lib.py +++ b/jc/lib.py @@ -9,7 +9,7 @@ from jc import appdirs -__version__ = '1.23.0' +__version__ = '1.23.1' parsers: List[str] = [ 'acpi', diff --git a/man/jc.1 b/man/jc.1 index f2b1bf873..9d1f44740 100644 --- a/man/jc.1 +++ b/man/jc.1 @@ -1,4 +1,4 @@ -.TH jc 1 2023-02-27 1.23.0 "JSON Convert" +.TH jc 1 2023-03-05 1.23.0 "JSON Convert" .SH NAME \fBjc\fP \- JSON Convert JSONifies the output of many CLI tools, file-types, and strings diff --git a/setup.py b/setup.py index 3485138cc..72e4615ae 100755 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ setuptools.setup( name='jc', - version='1.23.0', + version='1.23.1', author='Kelly Brazil', author_email='kellyjonbrazil@gmail.com', description='Converts the output of popular command-line tools and file-types to JSON.', From 894946b2070660083266d028b87ef713289570b2 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Sun, 5 Mar 2023 10:07:23 -0800 Subject: [PATCH 03/18] doc update --- man/jc.1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/man/jc.1 b/man/jc.1 index 9d1f44740..ec81bea10 100644 --- a/man/jc.1 +++ b/man/jc.1 @@ -1,4 +1,4 @@ -.TH jc 1 2023-03-05 1.23.0 "JSON Convert" +.TH jc 1 2023-03-05 1.23.1 "JSON Convert" .SH NAME \fBjc\fP \- JSON Convert JSONifies the output of many CLI tools, file-types, and strings From 9b5c25cb5ba6a1866362496e53dae8beed17275d Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Mon, 13 Mar 2023 19:47:38 -0700 Subject: [PATCH 04/18] fix for rtc configured as UTC --- jc/parsers/timedatectl.py | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/jc/parsers/timedatectl.py b/jc/parsers/timedatectl.py index 5a7d4e989..cfb78b4f2 100644 --- a/jc/parsers/timedatectl.py +++ b/jc/parsers/timedatectl.py @@ -64,7 +64,7 @@ class info(): """Provides parser metadata (version, author, etc.)""" - version = '1.7' + version = '1.8' description = '`timedatectl status` command parser' author = 'Kelly Brazil' author_email = 'kellyjonbrazil@gmail.com' @@ -120,15 +120,25 @@ def parse(data, raw=False, quiet=False): jc.utils.input_type_check(data) raw_output = {} + valid_fields = [ + 'local time', 'universal time', 'rtc time', 'time zone', 'ntp enabled', + 'ntp synchronized', 'rtc in local tz', 'dst active', 'system clock synchronized', + 'ntp service', 'systemd-timesyncd.service active' + ] if jc.utils.has_data(data): for line in filter(None, data.splitlines()): - linedata = line.split(':', maxsplit=1) - raw_output[linedata[0].strip().lower().replace(' ', '_')] = linedata[1].strip() - - if linedata[0].strip() == 'DST active': - break + try: + key, val = line.split(':', maxsplit=1) + key = key.lower().strip() + val = val.strip() + except ValueError: + continue + + if key in valid_fields: + keyname = key.replace(' ', '_') + raw_output[keyname] = val if raw: return raw_output From 7e134a63bd42a8c44c13c249df89901a1054f353 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Wed, 15 Mar 2023 06:58:31 -0700 Subject: [PATCH 05/18] add support for the timesync-status option --- jc/parsers/timedatectl.py | 33 ++++++++++++++++++++++++--------- 1 file changed, 24 insertions(+), 9 deletions(-) diff --git a/jc/parsers/timedatectl.py b/jc/parsers/timedatectl.py index cfb78b4f2..f834fcb9a 100644 --- a/jc/parsers/timedatectl.py +++ b/jc/parsers/timedatectl.py @@ -1,5 +1,7 @@ """jc - JSON Convert `timedatectl` command output parser +Also supports the `timesync-status` option. + The `epoch_utc` calculated timestamp field is timezone-aware and is only available if the `universal_time` field is available. @@ -29,7 +31,20 @@ "system_clock_synchronized": boolean, "systemd-timesyncd.service_active": boolean, "rtc_in_local_tz": boolean, - "dst_active": boolean + "dst_active": boolean, + "server": string, + "poll_interval": string, + "leap": string, + "version": string, + "stratum": string, + "reference": string, + "precision": string, + "root_distance": string, + "offset": string, + "delay": string, + "jitter": string, + "packet_count": string, + "frequency": string } Examples: @@ -120,11 +135,14 @@ def parse(data, raw=False, quiet=False): jc.utils.input_type_check(data) raw_output = {} - valid_fields = [ + valid_fields = { 'local time', 'universal time', 'rtc time', 'time zone', 'ntp enabled', - 'ntp synchronized', 'rtc in local tz', 'dst active', 'system clock synchronized', - 'ntp service', 'systemd-timesyncd.service active' - ] + 'ntp synchronized', 'rtc in local tz', 'dst active', + 'system clock synchronized', 'ntp service', + 'systemd-timesyncd.service active', 'server', 'poll interval', 'leap', + 'version', 'stratum', 'reference', 'precision', 'root distance', + 'offset', 'delay', 'jitter', 'packet count', 'frequency' + } if jc.utils.has_data(data): @@ -140,7 +158,4 @@ def parse(data, raw=False, quiet=False): keyname = key.replace(' ', '_') raw_output[keyname] = val - if raw: - return raw_output - else: - return _process(raw_output) + return raw_output if raw else _process(raw_output) From 8f9d650f6c6eb10a86bd4a735fbb1e0070f2f144 Mon Sep 17 00:00:00 2001 From: Jakub Wilk Date: Sat, 18 Mar 2023 10:28:05 +0100 Subject: [PATCH 06/18] make umask optional in /proc/PID/status Zombie processes don't have the umask field available. --- jc/parsers/proc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jc/parsers/proc.py b/jc/parsers/proc.py index 2330c6abf..628578e63 100644 --- a/jc/parsers/proc.py +++ b/jc/parsers/proc.py @@ -204,7 +204,7 @@ def parse( pid_smaps_p = re.compile(r'^[0-9a-f]{12}-[0-9a-f]{12} [rwxsp\-]{4} [0-9a-f]{8} [0-9a-f]{2}:[0-9a-f]{2} \d+ [^\n]+\nSize:\s+\d+ \S\S\n') pid_stat_p = re.compile(r'^\d+ \(.+\) \S \d+ \d+ \d+ \d+ -?\d+ (?:\d+ ){43}\d+$', re.DOTALL) pid_statm_p = re.compile(r'^\d+ \d+ \d+\s\d+\s\d+\s\d+\s\d+$') - pid_status_p = re.compile(r'^Name:\t.+\nUmask:\t\d+\nState:\t.+\nTgid:\t\d+\n') + pid_status_p = re.compile(r'^Name:\t.+\n(?:Umask:\t\d+\n)?State:\t.+\nTgid:\t\d+\n') # scsi_device_info = re.compile(r"^'\w+' '.+' 0x\d+") # scsi_scsi_p = re.compile(r'^Attached devices:\nHost: \w+ ') From 22ef4897951fdf6b6d2f4b961aa65886a40d621c Mon Sep 17 00:00:00 2001 From: Jakub Wilk Date: Sat, 18 Mar 2023 11:07:15 +0100 Subject: [PATCH 07/18] fix typo --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 571a2c85f..9789fa65c 100644 --- a/README.md +++ b/README.md @@ -604,7 +604,7 @@ they are run on an unsupported platform. To see all parser information, including compatibility, run `jc -ap`. You may still use a parser on an unsupported platform - for example, you may -want to parse a file with linux `lsof` output on an macOS or Windows laptop. In +want to parse a file with linux `lsof` output on a macOS or Windows laptop. In that case you can suppress the warning message with the `-q` cli option or the `quiet=True` function parameter in `parse()`: From 164294ecb7361e6bc4d52256ecb2f9185ad85b04 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Sat, 18 Mar 2023 15:14:59 -0700 Subject: [PATCH 08/18] add integer and float conversions --- CHANGELOG | 2 ++ jc/parsers/timedatectl.py | 33 +++++++++++++++---- .../generic/timedatectl-rtc-local.json | 1 + .../generic/timedatectl-rtc-local.out | 14 ++++++++ .../generic/timedatectl-timesync-status.json | 1 + .../generic/timedatectl-timesync-status.out | 13 ++++++++ tests/test_timedatectl.py | 24 ++++++++++++++ 7 files changed, 81 insertions(+), 7 deletions(-) create mode 100644 tests/fixtures/generic/timedatectl-rtc-local.json create mode 100644 tests/fixtures/generic/timedatectl-rtc-local.out create mode 100644 tests/fixtures/generic/timedatectl-timesync-status.json create mode 100644 tests/fixtures/generic/timedatectl-timesync-status.out diff --git a/CHANGELOG b/CHANGELOG index 65474600f..1f2364e73 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -2,6 +2,8 @@ jc changelog 20230305 v1.23.1 - Fix `zpool-status` command parser for lines that start with tab +- Fix `timedatectl` command parser when RTC set to local +- Add support for the `timesync-status` for the `timedatectl` command parser 20230227 v1.23.0 - Add input slicing as a `jc` command-line option diff --git a/jc/parsers/timedatectl.py b/jc/parsers/timedatectl.py index f834fcb9a..684bc93b5 100644 --- a/jc/parsers/timedatectl.py +++ b/jc/parsers/timedatectl.py @@ -35,16 +35,20 @@ "server": string, "poll_interval": string, "leap": string, - "version": string, - "stratum": string, + "version": integer, + "stratum": integer, "reference": string, "precision": string, "root_distance": string, - "offset": string, - "delay": string, - "jitter": string, - "packet_count": string, - "frequency": string + "offset": float, + "offset_unit": string, + "delay": float, + "delay_unit": string, + "jitter": float, + "jitter_unit": string, + "packet_count": integer, + "frequency": float, + "frequency_unit": string } Examples: @@ -105,11 +109,26 @@ def _process(proc_data): """ bool_list = {'ntp_enabled', 'ntp_synchronized', 'rtc_in_local_tz', 'dst_active', 'system_clock_synchronized', 'systemd-timesyncd.service_active'} + int_list = {'version', 'stratum', 'packet_count'} + float_list = {'offset', 'delay', 'jitter', 'frequency'} + + for key in ['offset', 'delay', 'jitter']: + if key in proc_data: + proc_data[key + '_unit'] = proc_data[key][-2:] + + if 'frequency' in proc_data: + proc_data['frequency_unit'] = proc_data['frequency'][-3:] for key in proc_data: if key in bool_list: proc_data[key] = jc.utils.convert_to_bool(proc_data[key]) + if key in int_list: + proc_data[key] = jc.utils.convert_to_int(proc_data[key]) + + if key in float_list: + proc_data[key] = jc.utils.convert_to_float(proc_data[key]) + if 'universal_time' in proc_data: ts = jc.utils.timestamp(proc_data['universal_time'], format_hint=(7300,)) proc_data['epoch_utc'] = ts.utc diff --git a/tests/fixtures/generic/timedatectl-rtc-local.json b/tests/fixtures/generic/timedatectl-rtc-local.json new file mode 100644 index 000000000..ead6eff22 --- /dev/null +++ b/tests/fixtures/generic/timedatectl-rtc-local.json @@ -0,0 +1 @@ +{"local_time":"Mon 2023-03-13 14:17:38 EET","universal_time":"Mon 2023-03-13 12:17:38 UTC","rtc_time":"Mon 2023-03-13 12:17:38","time_zone":"Europe/Athens (EET, +0200)","system_clock_synchronized":true,"ntp_service":"active","rtc_in_local_tz":true,"epoch_utc":1678709858} diff --git a/tests/fixtures/generic/timedatectl-rtc-local.out b/tests/fixtures/generic/timedatectl-rtc-local.out new file mode 100644 index 000000000..5e9cd587c --- /dev/null +++ b/tests/fixtures/generic/timedatectl-rtc-local.out @@ -0,0 +1,14 @@ + Local time: Mon 2023-03-13 14:17:38 EET + Universal time: Mon 2023-03-13 12:17:38 UTC + RTC time: Mon 2023-03-13 12:17:38 + Time zone: Europe/Athens (EET, +0200) +System clock synchronized: yes + NTP service: active + RTC in local TZ: yes + +Warning: The system is configured to read the RTC time in the local time zone. + This mode cannot be fully supported. It will create various problems + with time zone changes and daylight saving time adjustments. The RTC + time is never updated, it relies on external facilities to maintain it. + If at all possible, use RTC in UTC by calling + 'timedatectl set-local-rtc 0'. diff --git a/tests/fixtures/generic/timedatectl-timesync-status.json b/tests/fixtures/generic/timedatectl-timesync-status.json new file mode 100644 index 000000000..25b45f332 --- /dev/null +++ b/tests/fixtures/generic/timedatectl-timesync-status.json @@ -0,0 +1 @@ +{"server":"216.239.35.8 (time.google.com)","poll_interval":"8min 32s (min: 32s; max 34min 8s)","leap":"normal","version":4,"stratum":1,"reference":"GOOG","precision":"1us (-20)","root_distance":"91us (max: 5s)","offset":-5.224,"delay":65.884,"jitter":5.386,"packet_count":5,"frequency":27.071,"offset_unit":"ms","delay_unit":"ms","jitter_unit":"ms","frequency_unit":"ppm"} diff --git a/tests/fixtures/generic/timedatectl-timesync-status.out b/tests/fixtures/generic/timedatectl-timesync-status.out new file mode 100644 index 000000000..1b6802668 --- /dev/null +++ b/tests/fixtures/generic/timedatectl-timesync-status.out @@ -0,0 +1,13 @@ + Server: 216.239.35.8 (time.google.com) +Poll interval: 8min 32s (min: 32s; max 34min 8s) + Leap: normal + Version: 4 + Stratum: 1 + Reference: GOOG + Precision: 1us (-20) +Root distance: 91us (max: 5s) + Offset: -5.224ms + Delay: 65.884ms + Jitter: 5.386ms + Packet count: 5 + Frequency: +27.071ppm diff --git a/tests/test_timedatectl.py b/tests/test_timedatectl.py index fe4b12042..eb2cbb523 100644 --- a/tests/test_timedatectl.py +++ b/tests/test_timedatectl.py @@ -15,6 +15,12 @@ class MyTests(unittest.TestCase): with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/ubuntu-18.04/timedatectl.out'), 'r', encoding='utf-8') as f: ubuntu_18_4_timedatectl = f.read() + with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/generic/timedatectl-rtc-local.out'), 'r', encoding='utf-8') as f: + timedatectl_rtc_local = f.read() + + with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/generic/timedatectl-timesync-status.out'), 'r', encoding='utf-8') as f: + timedatectl_timesync_status = f.read() + # output with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/centos-7.7/timedatectl.json'), 'r', encoding='utf-8') as f: centos_7_7_timedatectl_json = json.loads(f.read()) @@ -22,6 +28,12 @@ class MyTests(unittest.TestCase): with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/ubuntu-18.04/timedatectl.json'), 'r', encoding='utf-8') as f: ubuntu_18_4_timedatectl_json = json.loads(f.read()) + with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/generic/timedatectl-rtc-local.json'), 'r', encoding='utf-8') as f: + timedatectl_rtc_local_json = json.loads(f.read()) + + with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/generic/timedatectl-timesync-status.json'), 'r', encoding='utf-8') as f: + timedatectl_timesync_status_json = json.loads(f.read()) + def test_timedatectl_nodata(self): """ @@ -41,6 +53,18 @@ def test_timedatectl_ubuntu_18_4(self): """ self.assertEqual(jc.parsers.timedatectl.parse(self.ubuntu_18_4_timedatectl, quiet=True), self.ubuntu_18_4_timedatectl_json) + def test_timedatectl_rtc_local(self): + """ + Test 'timedatectl' with RTC set to local + """ + self.assertEqual(jc.parsers.timedatectl.parse(self.timedatectl_rtc_local, quiet=True), self.timedatectl_rtc_local_json) + + def test_timedatectl_timesync_status(self): + """ + Test 'timedatectl timesync-status' + """ + self.assertEqual(jc.parsers.timedatectl.parse(self.timedatectl_timesync_status, quiet=True), self.timedatectl_timesync_status_json) + if __name__ == '__main__': unittest.main() From 049e93707c11333d16c3ba1958a9474f71fcfbee Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Sat, 18 Mar 2023 15:16:02 -0700 Subject: [PATCH 09/18] doc update --- docs/parsers/timedatectl.md | 23 +++++++++++++++++++++-- man/jc.1 | 2 +- 2 files changed, 22 insertions(+), 3 deletions(-) diff --git a/docs/parsers/timedatectl.md b/docs/parsers/timedatectl.md index 080767e84..b6fe49e79 100644 --- a/docs/parsers/timedatectl.md +++ b/docs/parsers/timedatectl.md @@ -5,6 +5,8 @@ jc - JSON Convert `timedatectl` command output parser +Also supports the `timesync-status` option. + The `epoch_utc` calculated timestamp field is timezone-aware and is only available if the `universal_time` field is available. @@ -34,7 +36,24 @@ Schema: "system_clock_synchronized": boolean, "systemd-timesyncd.service_active": boolean, "rtc_in_local_tz": boolean, - "dst_active": boolean + "dst_active": boolean, + "server": string, + "poll_interval": string, + "leap": string, + "version": integer, + "stratum": integer, + "reference": string, + "precision": string, + "root_distance": string, + "offset": float, + "offset_unit": string, + "delay": float, + "delay_unit": string, + "jitter": float, + "jitter_unit": string, + "packet_count": integer, + "frequency": float, + "frequency_unit": string } Examples: @@ -87,4 +106,4 @@ Returns: ### Parser Information Compatibility: linux -Version 1.7 by Kelly Brazil (kellyjonbrazil@gmail.com) +Version 1.8 by Kelly Brazil (kellyjonbrazil@gmail.com) diff --git a/man/jc.1 b/man/jc.1 index ec81bea10..42149b69f 100644 --- a/man/jc.1 +++ b/man/jc.1 @@ -1,4 +1,4 @@ -.TH jc 1 2023-03-05 1.23.1 "JSON Convert" +.TH jc 1 2023-03-18 1.23.1 "JSON Convert" .SH NAME \fBjc\fP \- JSON Convert JSONifies the output of many CLI tools, file-types, and strings From c68bf674a168035f01bd7c6644af92dd55531246 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Sat, 18 Mar 2023 15:43:22 -0700 Subject: [PATCH 10/18] add py.typed file to wheel --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index 72e4615ae..e235b0ad5 100755 --- a/setup.py +++ b/setup.py @@ -20,6 +20,7 @@ python_requires='>=3.6', url='https://github.com/kellyjonbrazil/jc', packages=setuptools.find_packages(exclude=['*.tests', '*.tests.*', 'tests.*', 'tests']), + package_data={'jc': ['py.typed']}, entry_points={ 'console_scripts': [ 'jc=jc.cli:main' From 11e94b686cb62c67d9f8574b3358726c995a5f63 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Sat, 18 Mar 2023 15:47:19 -0700 Subject: [PATCH 11/18] mypy fix --- jc/__init__.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/jc/__init__.py b/jc/__init__.py index fd297d591..5c26c5a71 100644 --- a/jc/__init__.py +++ b/jc/__init__.py @@ -124,6 +124,14 @@ `parse()`, `parser_info()`, and `get_help()`. This list is a subset of `parser_mod_list()`. """ -from .lib import (__version__, parse, parser_mod_list, plugin_parser_mod_list, - standard_parser_mod_list, streaming_parser_mod_list, - parser_info, all_parser_info, get_help) +from .lib import ( + __version__ as __version__, + parse as parse, + parser_mod_list as parser_mod_list, + plugin_parser_mod_list as plugin_parser_mod_list, + standard_parser_mod_list as standard_parser_mod_list, + streaming_parser_mod_list as streaming_parser_mod_list, + parser_info as parser_info, + all_parser_info as all_parser_info, + get_help as get_help +) From bf07973d9059eed47751b9cc770c5ca58e59852b Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Sun, 19 Mar 2023 12:39:28 -0700 Subject: [PATCH 12/18] doc update --- man/jc.1 | 2 +- templates/readme_template | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/man/jc.1 b/man/jc.1 index 42149b69f..69922fcd4 100644 --- a/man/jc.1 +++ b/man/jc.1 @@ -1,4 +1,4 @@ -.TH jc 1 2023-03-18 1.23.1 "JSON Convert" +.TH jc 1 2023-03-19 1.23.1 "JSON Convert" .SH NAME \fBjc\fP \- JSON Convert JSONifies the output of many CLI tools, file-types, and strings diff --git a/templates/readme_template b/templates/readme_template index d6325585a..16ca2d6aa 100644 --- a/templates/readme_template +++ b/templates/readme_template @@ -464,7 +464,7 @@ they are run on an unsupported platform. To see all parser information, including compatibility, run `jc -ap`. You may still use a parser on an unsupported platform - for example, you may -want to parse a file with linux `lsof` output on an macOS or Windows laptop. In +want to parse a file with linux `lsof` output on a macOS or Windows laptop. In that case you can suppress the warning message with the `-q` cli option or the `quiet=True` function parameter in `parse()`: From 44f83d800f1dd2aa282ef59ef3f4a5ccc09c6751 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Sun, 19 Mar 2023 12:49:25 -0700 Subject: [PATCH 13/18] doc update --- CHANGELOG | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG b/CHANGELOG index 1f2364e73..eced6b70c 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,8 +1,9 @@ jc changelog -20230305 v1.23.1 +20230319 v1.23.1 - Fix `zpool-status` command parser for lines that start with tab - Fix `timedatectl` command parser when RTC set to local +- Fix to ensure `py.typed` file is included in the package wheel - Add support for the `timesync-status` for the `timedatectl` command parser 20230227 v1.23.0 From be51304c9c2c990668a64f0a78751c9281cd2da5 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Wed, 22 Mar 2023 16:43:57 -0700 Subject: [PATCH 14/18] add support for CDC MBIM and CDC MBIM Extended --- jc/parsers/lsusb.py | 39 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 38 insertions(+), 1 deletion(-) diff --git a/jc/parsers/lsusb.py b/jc/parsers/lsusb.py index 652a9fbc7..0d707754c 100644 --- a/jc/parsers/lsusb.py +++ b/jc/parsers/lsusb.py @@ -97,6 +97,24 @@ ] } }, + "cdc_mbim": { + "": { + "value": string, + "description": string, + "attributes": [ + string + ] + } + }, + "cdc_mbim_extended": { + "": { + "value": string, + "description": string, + "attributes": [ + string + ] + } + }, "videocontrol_descriptors": [ { "": { @@ -500,6 +518,8 @@ def __init__(self): self.cdc_call_management = _descriptor_obj('cdc_call_management') self.cdc_acm = _descriptor_obj('cdc_acm') self.cdc_union = _descriptor_obj('cdc_union') + self.cdc_mbim = _descriptor_obj('cdc_mbim') + self.cdc_mbim_extended = _descriptor_obj('cdc_mbim_extended') self.endpoint_descriptors = _descriptor_list('endpoint_descriptor') self.videocontrol_interface_descriptors = _descriptor_list('videocontrol_interface_descriptor') self.videostreaming_interface_descriptors = _descriptor_list('videostreaming_interface_descriptor') @@ -538,7 +558,8 @@ def _add_attributes(self, line): section_header = self.normal_section_header if self.section == 'videocontrol_interface_descriptor' \ - or self.section == 'videostreaming_interface_descriptor': + or self.section == 'videostreaming_interface_descriptor' \ + or self.section == 'cdc_mbim_extended': section_header = self.larger_section_header @@ -689,6 +710,8 @@ def _set_sections(self, line): ' CDC Union:': 'cdc_union', ' HID Device Descriptor:': 'hid_device_descriptor', ' Report Descriptors:': 'report_descriptors', + ' CDC MBIM:': 'cdc_mbim', + ' CDC MBIM Extended:': 'cdc_mbim_extended', 'Hub Descriptor:': 'hub_descriptor', ' Hub Port Status:': 'hub_port_status', 'Device Qualifier (for other device speed):': 'device_qualifier', @@ -713,6 +736,8 @@ def _populate_lists(self, line): 'cdc_call_management': self.cdc_call_management.list, 'cdc_acm': self.cdc_acm.list, 'cdc_union': self.cdc_union.list, + 'cdc_mbim': self.cdc_mbim.list, + 'cdc_mbim_extended': self.cdc_mbim_extended.list, 'hid_device_descriptor': self.hid_device_descriptor.list, # 'report_descriptors': self.report_descriptors_list, # not implemented 'videocontrol_interface_descriptor': self.videocontrol_interface_descriptors.list, @@ -757,6 +782,8 @@ def _populate_schema(self): ['device_descriptor']['configuration_descriptor']['interface_descriptors'][0]['cdc_call_management'] = {} ['device_descriptor']['configuration_descriptor']['interface_descriptors'][0]['cdc_acm'] = {} ['device_descriptor']['configuration_descriptor']['interface_descriptors'][0]['cdc_union'] = {} + ['device_descriptor']['configuration_descriptor']['interface_descriptors'][0]['cdc_mbim'] = {} + ['device_descriptor']['configuration_descriptor']['interface_descriptors'][0]['cdc_mbim_extended'] = {} ['device_descriptor']['configuration_descriptor']['interface_descriptors'][0]['hid_device_descriptor'] = {} ['device_descriptor']['configuration_descriptor']['interface_descriptors'][0]['endpoint_descriptors'] = [] ['device_descriptor']['configuration_descriptor']['interface_descriptors'][0]['endpoint_descriptors'][0] = {} @@ -847,6 +874,12 @@ def _populate_schema(self): if self.cdc_union._entries_for_this_bus_and_interface_idx_exist(idx, iface_idx): self.cdc_union._update_output(idx, iface_idx, i_desc_obj) + if self.cdc_mbim._entries_for_this_bus_and_interface_idx_exist(idx, iface_idx): + self.cdc_mbim._update_output(idx, iface_idx, i_desc_obj) + + if self.cdc_mbim_extended._entries_for_this_bus_and_interface_idx_exist(idx, iface_idx): + self.cdc_mbim_extended._update_output(idx, iface_idx, i_desc_obj) + if self.hid_device_descriptor._entries_for_this_bus_and_interface_idx_exist(idx, iface_idx): self.hid_device_descriptor._update_output(idx, iface_idx, i_desc_obj) @@ -923,6 +956,10 @@ def parse(data, raw=False, quiet=False): lsusb = _LsUsb() if jc.utils.has_data(data): + + # fix known too-long field names + data = data.replace('bmNetworkCapabilities', 'bmNetworkCapabilit ') + for line in data.splitlines(): # only -v option or no options are supported if line.startswith('/'): From 22afb695739150e03d8649b7fac975590ea68561 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Wed, 22 Mar 2023 16:47:45 -0700 Subject: [PATCH 15/18] doc update --- CHANGELOG | 1 + docs/parsers/lsusb.md | 20 +++++++++++++++++++- jc/parsers/lsusb.py | 2 +- man/jc.1 | 2 +- 4 files changed, 22 insertions(+), 3 deletions(-) diff --git a/CHANGELOG b/CHANGELOG index eced6b70c..92ae4afa3 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -4,6 +4,7 @@ jc changelog - Fix `zpool-status` command parser for lines that start with tab - Fix `timedatectl` command parser when RTC set to local - Fix to ensure `py.typed` file is included in the package wheel +- Fix `lsusb` command parser to support CDC MBIM and CDC MBIM Extended fields - Add support for the `timesync-status` for the `timedatectl` command parser 20230227 v1.23.0 diff --git a/docs/parsers/lsusb.md b/docs/parsers/lsusb.md index f9168da5b..158345130 100644 --- a/docs/parsers/lsusb.md +++ b/docs/parsers/lsusb.md @@ -102,6 +102,24 @@ Schema: ] } }, + "cdc_mbim": { + "": { + "value": string, + "description": string, + "attributes": [ + string + ] + } + }, + "cdc_mbim_extended": { + "": { + "value": string, + "description": string, + "attributes": [ + string + ] + } + }, "videocontrol_descriptors": [ { "": { @@ -312,4 +330,4 @@ Returns: ### Parser Information Compatibility: linux -Version 1.3 by Kelly Brazil (kellyjonbrazil@gmail.com) +Version 1.4 by Kelly Brazil (kellyjonbrazil@gmail.com) diff --git a/jc/parsers/lsusb.py b/jc/parsers/lsusb.py index 0d707754c..075c7fc0a 100644 --- a/jc/parsers/lsusb.py +++ b/jc/parsers/lsusb.py @@ -309,7 +309,7 @@ class info(): """Provides parser metadata (version, author, etc.)""" - version = '1.3' + version = '1.4' description = '`lsusb` command parser' author = 'Kelly Brazil' author_email = 'kellyjonbrazil@gmail.com' diff --git a/man/jc.1 b/man/jc.1 index 69922fcd4..5aa93e782 100644 --- a/man/jc.1 +++ b/man/jc.1 @@ -1,4 +1,4 @@ -.TH jc 1 2023-03-19 1.23.1 "JSON Convert" +.TH jc 1 2023-03-22 1.23.1 "JSON Convert" .SH NAME \fBjc\fP \- JSON Convert JSONifies the output of many CLI tools, file-types, and strings From fb7c39050684d1b9fe4ffb5b72caa3a3cfe97531 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Thu, 23 Mar 2023 06:43:06 -0700 Subject: [PATCH 16/18] add lsusb tests --- tests/fixtures/generic/lsusb-cdc-mbim.json | 1 + tests/fixtures/generic/lsusb-cdc-mbim.out | 286 +++++++++++++++++++++ tests/test_lsusb.py | 12 + 3 files changed, 299 insertions(+) create mode 100644 tests/fixtures/generic/lsusb-cdc-mbim.json create mode 100644 tests/fixtures/generic/lsusb-cdc-mbim.out diff --git a/tests/fixtures/generic/lsusb-cdc-mbim.json b/tests/fixtures/generic/lsusb-cdc-mbim.json new file mode 100644 index 000000000..e88bdf02f --- /dev/null +++ b/tests/fixtures/generic/lsusb-cdc-mbim.json @@ -0,0 +1 @@ +[{"bus":"001","device":"003","id":"2c7c:0125","description":"Quectel Wireless Solutions Co., Ltd. EC25 LTE modem","device_descriptor":{"bLength":{"value":"18"},"bDescriptorType":{"value":"1"},"bcdUSB":{"value":"2.00"},"bDeviceClass":{"value":"239","description":"Miscellaneous Device"},"bDeviceSubClass":{"value":"2"},"bDeviceProtocol":{"value":"1","description":"Interface Association"},"bMaxPacketSize0":{"value":"64"},"idVendor":{"value":"0x2c7c","description":"Quectel Wireless Solutions Co., Ltd."},"idProduct":{"value":"0x0125","description":"EC25 LTE modem"},"bcdDevice":{"value":"3.18"},"iManufacturer":{"value":"1","description":"Quectel Incorporated"},"iProduct":{"value":"2","description":"LTE Module"},"iSerial":{"value":"0"},"bNumConfigurations":{"value":"1"},"configuration_descriptor":{"bLength":{"value":"9"},"bDescriptorType":{"value":"2"},"wTotalLength":{"value":"0x0109"},"bNumInterfaces":{"value":"6"},"bConfigurationValue":{"value":"1"},"iConfiguration":{"value":"0"},"bmAttributes":{"value":"0xa0","attributes":["(Bus Powered)","Remote Wakeup"]},"MaxPower":{"description":"500mA"},"interface_association":{"bLength":{"value":"8"},"bDescriptorType":{"value":"11"},"bFirstInterface":{"value":"4"},"bInterfaceCount":{"value":"2"},"bFunctionClass":{"value":"2","description":"Communications"},"bFunctionSubClass":{"value":"14"},"bFunctionProtocol":{"value":"0"},"iFunction":{"value":"0"}},"interface_descriptors":[{"bLength":{"value":"9"},"bDescriptorType":{"value":"4"},"bInterfaceNumber":{"value":"0"},"bAlternateSetting":{"value":"0"},"bNumEndpoints":{"value":"2"},"bInterfaceClass":{"value":"255","description":"Vendor Specific Class"},"bInterfaceSubClass":{"value":"255","description":"Vendor Specific Subclass"},"bInterfaceProtocol":{"value":"255","description":"Vendor Specific Protocol"},"iInterface":{"value":"0"},"endpoint_descriptors":[{"bLength":{"value":"7"},"bDescriptorType":{"value":"5"},"bEndpointAddress":{"value":"0x81","description":"EP 1 IN"},"bmAttributes":{"value":"2","attributes":["Transfer Type Bulk","Synch Type None","Usage Type Data"]},"wMaxPacketSize":{"value":"0x0200","description":"1x 512 bytes"},"bInterval":{"value":"0"}},{"bLength":{"value":"7"},"bDescriptorType":{"value":"5"},"bEndpointAddress":{"value":"0x01","description":"EP 1 OUT"},"bmAttributes":{"value":"2","attributes":["Transfer Type Bulk","Synch Type None","Usage Type Data"]},"wMaxPacketSize":{"value":"0x0200","description":"1x 512 bytes"},"bInterval":{"value":"0"}}]},{"bLength":{"value":"9"},"bDescriptorType":{"value":"4"},"bInterfaceNumber":{"value":"1"},"bAlternateSetting":{"value":"0"},"bNumEndpoints":{"value":"3"},"bInterfaceClass":{"value":"255","description":"Vendor Specific Class"},"bInterfaceSubClass":{"value":"0"},"bInterfaceProtocol":{"value":"0"},"iInterface":{"value":"0"},"** UNRECOGNIZED: 05":{"value":"24","description":"06 00 00"},"** UNRECOGNIZED: 04":{"value":"24","description":"02 02"},"endpoint_descriptors":[{"bLength":{"value":"7"},"bDescriptorType":{"value":"5"},"bEndpointAddress":{"value":"0x83","description":"EP 3 IN"},"bmAttributes":{"value":"3","attributes":["Transfer Type Interrupt","Synch Type None","Usage Type Data"]},"wMaxPacketSize":{"value":"0x000a","description":"1x 10 bytes"},"bInterval":{"value":"9"}},{"bLength":{"value":"7"},"bDescriptorType":{"value":"5"},"bEndpointAddress":{"value":"0x82","description":"EP 2 IN"},"bmAttributes":{"value":"2","attributes":["Transfer Type Bulk","Synch Type None","Usage Type Data"]},"wMaxPacketSize":{"value":"0x0200","description":"1x 512 bytes"},"bInterval":{"value":"0"}},{"bLength":{"value":"7"},"bDescriptorType":{"value":"5"},"bEndpointAddress":{"value":"0x02","description":"EP 2 OUT"},"bmAttributes":{"value":"2","attributes":["Transfer Type Bulk","Synch Type None","Usage Type Data"]},"wMaxPacketSize":{"value":"0x0200","description":"1x 512 bytes"},"bInterval":{"value":"0"}}]},{"bLength":{"value":"9"},"bDescriptorType":{"value":"4"},"bInterfaceNumber":{"value":"2"},"bAlternateSetting":{"value":"0"},"bNumEndpoints":{"value":"3"},"bInterfaceClass":{"value":"255","description":"Vendor Specific Class"},"bInterfaceSubClass":{"value":"0"},"bInterfaceProtocol":{"value":"0"},"iInterface":{"value":"0"},"** UNRECOGNIZED: 05":{"value":"24","description":"06 00 00"},"** UNRECOGNIZED: 04":{"value":"24","description":"02 02"},"endpoint_descriptors":[{"bLength":{"value":"7"},"bDescriptorType":{"value":"5"},"bEndpointAddress":{"value":"0x85","description":"EP 5 IN"},"bmAttributes":{"value":"3","attributes":["Transfer Type Interrupt","Synch Type None","Usage Type Data"]},"wMaxPacketSize":{"value":"0x000a","description":"1x 10 bytes"},"bInterval":{"value":"9"}},{"bLength":{"value":"7"},"bDescriptorType":{"value":"5"},"bEndpointAddress":{"value":"0x84","description":"EP 4 IN"},"bmAttributes":{"value":"2","attributes":["Transfer Type Bulk","Synch Type None","Usage Type Data"]},"wMaxPacketSize":{"value":"0x0200","description":"1x 512 bytes"},"bInterval":{"value":"0"}},{"bLength":{"value":"7"},"bDescriptorType":{"value":"5"},"bEndpointAddress":{"value":"0x03","description":"EP 3 OUT"},"bmAttributes":{"value":"2","attributes":["Transfer Type Bulk","Synch Type None","Usage Type Data"]},"wMaxPacketSize":{"value":"0x0200","description":"1x 512 bytes"},"bInterval":{"value":"0"}}]},{"bLength":{"value":"9"},"bDescriptorType":{"value":"4"},"bInterfaceNumber":{"value":"3"},"bAlternateSetting":{"value":"0"},"bNumEndpoints":{"value":"3"},"bInterfaceClass":{"value":"255","description":"Vendor Specific Class"},"bInterfaceSubClass":{"value":"0"},"bInterfaceProtocol":{"value":"0"},"iInterface":{"value":"0"},"** UNRECOGNIZED: 05":{"value":"24","description":"06 00 00"},"** UNRECOGNIZED: 04":{"value":"24","description":"02 02"},"endpoint_descriptors":[{"bLength":{"value":"7"},"bDescriptorType":{"value":"5"},"bEndpointAddress":{"value":"0x87","description":"EP 7 IN"},"bmAttributes":{"value":"3","attributes":["Transfer Type Interrupt","Synch Type None","Usage Type Data"]},"wMaxPacketSize":{"value":"0x000a","description":"1x 10 bytes"},"bInterval":{"value":"9"}},{"bLength":{"value":"7"},"bDescriptorType":{"value":"5"},"bEndpointAddress":{"value":"0x86","description":"EP 6 IN"},"bmAttributes":{"value":"2","attributes":["Transfer Type Bulk","Synch Type None","Usage Type Data"]},"wMaxPacketSize":{"value":"0x0200","description":"1x 512 bytes"},"bInterval":{"value":"0"}},{"bLength":{"value":"7"},"bDescriptorType":{"value":"5"},"bEndpointAddress":{"value":"0x04","description":"EP 4 OUT"},"bmAttributes":{"value":"2","attributes":["Transfer Type Bulk","Synch Type None","Usage Type Data"]},"wMaxPacketSize":{"value":"0x0200","description":"1x 512 bytes"},"bInterval":{"value":"0"}}]},{"bLength":{"value":"9"},"bDescriptorType":{"value":"4"},"bInterfaceNumber":{"value":"4"},"bAlternateSetting":{"value":"0"},"bNumEndpoints":{"value":"1"},"bInterfaceClass":{"value":"2","description":"Communications"},"bInterfaceSubClass":{"value":"14"},"bInterfaceProtocol":{"value":"0"},"iInterface":{"value":"6","description":"LTE Module"},"cdc_header":{"bcdCDC":{"value":"1.10"}},"cdc_union":{"bMasterInterface":{"value":"4"},"bSlaveInterface":{"value":"5"}},"cdc_mbim":{"bcdMBIMVersion":{"value":"1.00"},"wMaxControlMessage":{"value":"4096"},"bNumberFilters":{"value":"32"},"bMaxFilterSize":{"value":"128"},"wMaxSegmentSize":{"value":"2048"},"bmNetworkCapabilit":{"description":"0x20","attributes":["8-byte ntb input size"]}},"cdc_mbim_extended":{"bcdMBIMExtendedVersion":{"value":"1.00"},"bMaxOutstandingCommandMessages":{"value":"64"},"wMTU":{"value":"1500"}},"endpoint_descriptors":[{"bLength":{"value":"7"},"bDescriptorType":{"value":"5"},"bEndpointAddress":{"value":"0x89","description":"EP 9 IN"},"bmAttributes":{"value":"3","attributes":["Transfer Type Interrupt","Synch Type None","Usage Type Data"]},"wMaxPacketSize":{"value":"0x0040","description":"1x 64 bytes"},"bInterval":{"value":"9"}}]},{"bLength":{"value":"9"},"bDescriptorType":{"value":"4"},"bInterfaceNumber":{"value":"5"},"bAlternateSetting":{"value":"0"},"bNumEndpoints":{"value":"0"},"bInterfaceClass":{"value":"10","description":"CDC Data"},"bInterfaceSubClass":{"value":"0"},"bInterfaceProtocol":{"value":"2"},"iInterface":{"value":"7","description":"MBIM Data"}},{"bLength":{"value":"9"},"bDescriptorType":{"value":"4"},"bInterfaceNumber":{"value":"5"},"bAlternateSetting":{"value":"1"},"bNumEndpoints":{"value":"2"},"bInterfaceClass":{"value":"10","description":"CDC Data"},"bInterfaceSubClass":{"value":"0"},"bInterfaceProtocol":{"value":"2"},"iInterface":{"value":"7","description":"MBIM Data"},"endpoint_descriptors":[{"bLength":{"value":"7"},"bDescriptorType":{"value":"5"},"bEndpointAddress":{"value":"0x88","description":"EP 8 IN"},"bmAttributes":{"value":"2","attributes":["Transfer Type Bulk","Synch Type None","Usage Type Data"]},"wMaxPacketSize":{"value":"0x0200","description":"1x 512 bytes"},"bInterval":{"value":"0"}},{"bLength":{"value":"7"},"bDescriptorType":{"value":"5"},"bEndpointAddress":{"value":"0x05","description":"EP 5 OUT"},"bmAttributes":{"value":"2","attributes":["Transfer Type Bulk","Synch Type None","Usage Type Data"]},"wMaxPacketSize":{"value":"0x0200","description":"1x 512 bytes"},"bInterval":{"value":"0"}}]}]}},"device_qualifier":{"bLength":{"value":"10"},"bDescriptorType":{"value":"6"},"bcdUSB":{"value":"2.00"},"bDeviceClass":{"value":"239","description":"Miscellaneous Device"},"bDeviceSubClass":{"value":"2"},"bDeviceProtocol":{"value":"1","description":"Interface Association"},"bMaxPacketSize0":{"value":"64"},"bNumConfigurations":{"value":"1"}},"device_status":{"value":"0x0000","description":"(Bus Powered)"}}] diff --git a/tests/fixtures/generic/lsusb-cdc-mbim.out b/tests/fixtures/generic/lsusb-cdc-mbim.out new file mode 100644 index 000000000..5ec336aed --- /dev/null +++ b/tests/fixtures/generic/lsusb-cdc-mbim.out @@ -0,0 +1,286 @@ +Bus 001 Device 003: ID 2c7c:0125 Quectel Wireless Solutions Co., Ltd. EC25 LTE modem +Device Descriptor: + bLength 18 + bDescriptorType 1 + bcdUSB 2.00 + bDeviceClass 239 Miscellaneous Device + bDeviceSubClass 2 + bDeviceProtocol 1 Interface Association + bMaxPacketSize0 64 + idVendor 0x2c7c Quectel Wireless Solutions Co., Ltd. + idProduct 0x0125 EC25 LTE modem + bcdDevice 3.18 + iManufacturer 1 Quectel Incorporated + iProduct 2 LTE Module + iSerial 0 + bNumConfigurations 1 + Configuration Descriptor: + bLength 9 + bDescriptorType 2 + wTotalLength 0x0109 + bNumInterfaces 6 + bConfigurationValue 1 + iConfiguration 0 + bmAttributes 0xa0 + (Bus Powered) + Remote Wakeup + MaxPower 500mA + Interface Descriptor: + bLength 9 + bDescriptorType 4 + bInterfaceNumber 0 + bAlternateSetting 0 + bNumEndpoints 2 + bInterfaceClass 255 Vendor Specific Class + bInterfaceSubClass 255 Vendor Specific Subclass + bInterfaceProtocol 255 Vendor Specific Protocol + iInterface 0 + Endpoint Descriptor: + bLength 7 + bDescriptorType 5 + bEndpointAddress 0x81 EP 1 IN + bmAttributes 2 + Transfer Type Bulk + Synch Type None + Usage Type Data + wMaxPacketSize 0x0200 1x 512 bytes + bInterval 0 + Endpoint Descriptor: + bLength 7 + bDescriptorType 5 + bEndpointAddress 0x01 EP 1 OUT + bmAttributes 2 + Transfer Type Bulk + Synch Type None + Usage Type Data + wMaxPacketSize 0x0200 1x 512 bytes + bInterval 0 + Interface Descriptor: + bLength 9 + bDescriptorType 4 + bInterfaceNumber 1 + bAlternateSetting 0 + bNumEndpoints 3 + bInterfaceClass 255 Vendor Specific Class + bInterfaceSubClass 0 + bInterfaceProtocol 0 + iInterface 0 + ** UNRECOGNIZED: 05 24 00 10 01 + ** UNRECOGNIZED: 05 24 01 00 00 + ** UNRECOGNIZED: 04 24 02 02 + ** UNRECOGNIZED: 05 24 06 00 00 + Endpoint Descriptor: + bLength 7 + bDescriptorType 5 + bEndpointAddress 0x83 EP 3 IN + bmAttributes 3 + Transfer Type Interrupt + Synch Type None + Usage Type Data + wMaxPacketSize 0x000a 1x 10 bytes + bInterval 9 + Endpoint Descriptor: + bLength 7 + bDescriptorType 5 + bEndpointAddress 0x82 EP 2 IN + bmAttributes 2 + Transfer Type Bulk + Synch Type None + Usage Type Data + wMaxPacketSize 0x0200 1x 512 bytes + bInterval 0 + Endpoint Descriptor: + bLength 7 + bDescriptorType 5 + bEndpointAddress 0x02 EP 2 OUT + bmAttributes 2 + Transfer Type Bulk + Synch Type None + Usage Type Data + wMaxPacketSize 0x0200 1x 512 bytes + bInterval 0 + Interface Descriptor: + bLength 9 + bDescriptorType 4 + bInterfaceNumber 2 + bAlternateSetting 0 + bNumEndpoints 3 + bInterfaceClass 255 Vendor Specific Class + bInterfaceSubClass 0 + bInterfaceProtocol 0 + iInterface 0 + ** UNRECOGNIZED: 05 24 00 10 01 + ** UNRECOGNIZED: 05 24 01 00 00 + ** UNRECOGNIZED: 04 24 02 02 + ** UNRECOGNIZED: 05 24 06 00 00 + Endpoint Descriptor: + bLength 7 + bDescriptorType 5 + bEndpointAddress 0x85 EP 5 IN + bmAttributes 3 + Transfer Type Interrupt + Synch Type None + Usage Type Data + wMaxPacketSize 0x000a 1x 10 bytes + bInterval 9 + Endpoint Descriptor: + bLength 7 + bDescriptorType 5 + bEndpointAddress 0x84 EP 4 IN + bmAttributes 2 + Transfer Type Bulk + Synch Type None + Usage Type Data + wMaxPacketSize 0x0200 1x 512 bytes + bInterval 0 + Endpoint Descriptor: + bLength 7 + bDescriptorType 5 + bEndpointAddress 0x03 EP 3 OUT + bmAttributes 2 + Transfer Type Bulk + Synch Type None + Usage Type Data + wMaxPacketSize 0x0200 1x 512 bytes + bInterval 0 + Interface Descriptor: + bLength 9 + bDescriptorType 4 + bInterfaceNumber 3 + bAlternateSetting 0 + bNumEndpoints 3 + bInterfaceClass 255 Vendor Specific Class + bInterfaceSubClass 0 + bInterfaceProtocol 0 + iInterface 0 + ** UNRECOGNIZED: 05 24 00 10 01 + ** UNRECOGNIZED: 05 24 01 00 00 + ** UNRECOGNIZED: 04 24 02 02 + ** UNRECOGNIZED: 05 24 06 00 00 + Endpoint Descriptor: + bLength 7 + bDescriptorType 5 + bEndpointAddress 0x87 EP 7 IN + bmAttributes 3 + Transfer Type Interrupt + Synch Type None + Usage Type Data + wMaxPacketSize 0x000a 1x 10 bytes + bInterval 9 + Endpoint Descriptor: + bLength 7 + bDescriptorType 5 + bEndpointAddress 0x86 EP 6 IN + bmAttributes 2 + Transfer Type Bulk + Synch Type None + Usage Type Data + wMaxPacketSize 0x0200 1x 512 bytes + bInterval 0 + Endpoint Descriptor: + bLength 7 + bDescriptorType 5 + bEndpointAddress 0x04 EP 4 OUT + bmAttributes 2 + Transfer Type Bulk + Synch Type None + Usage Type Data + wMaxPacketSize 0x0200 1x 512 bytes + bInterval 0 + Interface Association: + bLength 8 + bDescriptorType 11 + bFirstInterface 4 + bInterfaceCount 2 + bFunctionClass 2 Communications + bFunctionSubClass 14 + bFunctionProtocol 0 + iFunction 0 + Interface Descriptor: + bLength 9 + bDescriptorType 4 + bInterfaceNumber 4 + bAlternateSetting 0 + bNumEndpoints 1 + bInterfaceClass 2 Communications + bInterfaceSubClass 14 + bInterfaceProtocol 0 + iInterface 6 LTE Module + CDC Header: + bcdCDC 1.10 + CDC Union: + bMasterInterface 4 + bSlaveInterface 5 + CDC MBIM: + bcdMBIMVersion 1.00 + wMaxControlMessage 4096 + bNumberFilters 32 + bMaxFilterSize 128 + wMaxSegmentSize 2048 + bmNetworkCapabilities 0x20 + 8-byte ntb input size + CDC MBIM Extended: + bcdMBIMExtendedVersion 1.00 + bMaxOutstandingCommandMessages 64 + wMTU 1500 + Endpoint Descriptor: + bLength 7 + bDescriptorType 5 + bEndpointAddress 0x89 EP 9 IN + bmAttributes 3 + Transfer Type Interrupt + Synch Type None + Usage Type Data + wMaxPacketSize 0x0040 1x 64 bytes + bInterval 9 + Interface Descriptor: + bLength 9 + bDescriptorType 4 + bInterfaceNumber 5 + bAlternateSetting 0 + bNumEndpoints 0 + bInterfaceClass 10 CDC Data + bInterfaceSubClass 0 + bInterfaceProtocol 2 + iInterface 7 MBIM Data + Interface Descriptor: + bLength 9 + bDescriptorType 4 + bInterfaceNumber 5 + bAlternateSetting 1 + bNumEndpoints 2 + bInterfaceClass 10 CDC Data + bInterfaceSubClass 0 + bInterfaceProtocol 2 + iInterface 7 MBIM Data + Endpoint Descriptor: + bLength 7 + bDescriptorType 5 + bEndpointAddress 0x88 EP 8 IN + bmAttributes 2 + Transfer Type Bulk + Synch Type None + Usage Type Data + wMaxPacketSize 0x0200 1x 512 bytes + bInterval 0 + Endpoint Descriptor: + bLength 7 + bDescriptorType 5 + bEndpointAddress 0x05 EP 5 OUT + bmAttributes 2 + Transfer Type Bulk + Synch Type None + Usage Type Data + wMaxPacketSize 0x0200 1x 512 bytes + bInterval 0 +Device Qualifier (for other device speed): + bLength 10 + bDescriptorType 6 + bcdUSB 2.00 + bDeviceClass 239 Miscellaneous Device + bDeviceSubClass 2 + bDeviceProtocol 1 Interface Association + bMaxPacketSize0 64 + bNumConfigurations 1 +Device Status: 0x0000 + (Bus Powered) diff --git a/tests/test_lsusb.py b/tests/test_lsusb.py index 091228f12..6bfddacb4 100644 --- a/tests/test_lsusb.py +++ b/tests/test_lsusb.py @@ -37,6 +37,9 @@ class MyTests(unittest.TestCase): with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/generic/lsusb-extra-hub-port-status-info.out'), 'r', encoding='utf-8') as f: generic_lsusb_extra_hub_port_status_info = f.read() + with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/generic/lsusb-cdc-mbim.out'), 'r', encoding='utf-8') as f: + generic_lsusb_cdc_mbim = f.read() + # output with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/centos-7.7/lsusb.json'), 'r', encoding='utf-8') as f: centos_7_7_lsusb_json = json.loads(f.read()) @@ -62,6 +65,9 @@ class MyTests(unittest.TestCase): with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/generic/lsusb-extra-hub-port-status-info.json'), 'r', encoding='utf-8') as f: generic_lsusb_extra_hub_port_status_info_json = json.loads(f.read()) + with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/generic/lsusb-cdc-mbim.json'), 'r', encoding='utf-8') as f: + generic_lsusb_cdc_mbim_json = json.loads(f.read()) + def test_lsusb_nodata(self): """ @@ -123,6 +129,12 @@ def test_lsusb_extra_hub_port_status_info(self): """ self.assertEqual(jc.parsers.lsusb.parse(self.generic_lsusb_extra_hub_port_status_info, quiet=True), self.generic_lsusb_extra_hub_port_status_info_json) + def test_lsusb_cdc_mbim(self): + """ + Test 'lsusb -v' with CDC MBIM and CDC MBIM Extended fields + """ + self.assertEqual(jc.parsers.lsusb.parse(self.generic_lsusb_cdc_mbim, quiet=True), self.generic_lsusb_cdc_mbim_json) + if __name__ == '__main__': unittest.main() From c60d899f31f59310838e3d16fbfd940d059919d5 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Thu, 23 Mar 2023 16:40:13 -0700 Subject: [PATCH 17/18] ignore non-parser-plugin python files in plugin directory --- README.md | 12 +++++++----- jc/cli.py | 12 ++++++------ jc/lib.py | 15 ++++++++++++++- man/jc.1 | 12 +++++++----- templates/manpage_template | 10 ++++++---- templates/readme_template | 12 +++++++----- 6 files changed, 47 insertions(+), 26 deletions(-) diff --git a/README.md b/README.md index 9789fa65c..f16c34031 100644 --- a/README.md +++ b/README.md @@ -533,20 +533,22 @@ for item in result: print(item["filename"]) ``` -### Custom Parsers -Custom local parser plugins may be placed in a `jc/jcparsers` folder in your -local **"App data directory"**: +### Parser Plugins +Parser plugins may be placed in a `jc/jcparsers` folder in your local +**"App data directory"**: - Linux/unix: `$HOME/.local/share/jc/jcparsers` - macOS: `$HOME/Library/Application Support/jc/jcparsers` - Windows: `$LOCALAPPDATA\jc\jc\jcparsers` -Local parser plugins are standard python module files. Use the +Parser plugins are standard python module files. Use the [`jc/parsers/foo.py`](https://github.com/kellyjonbrazil/jc/blob/master/jc/parsers/foo.py) or [`jc/parsers/foo_s.py (streaming)`](https://github.com/kellyjonbrazil/jc/blob/master/jc/parsers/foo_s.py) parser as a template and simply place a `.py` file in the `jcparsers` subfolder. +Any dependencies can be placed in the `jc` folder above `jcparsers` and can +be imported in the parser code. -Local plugin filenames must be valid python module names and therefore must +Parser plugin filenames must be valid python module names and therefore must start with a letter and consist entirely of alphanumerics and underscores. Local plugins may override default parsers. diff --git a/jc/cli.py b/jc/cli.py index 66b808c94..dd175b59f 100644 --- a/jc/cli.py +++ b/jc/cli.py @@ -215,14 +215,14 @@ def parser_categories_text(self) -> str: category_text: str = '' padding_char: str = ' ' all_parsers = all_parser_info(show_hidden=True, show_deprecated=False) - generic = [{'arg': x['argument'], 'desc': x['description']} for x in all_parsers if 'generic' in x['tags']] - standard = [{'arg': x['argument'], 'desc': x['description']} for x in all_parsers if 'standard' in x['tags']] - command = [{'arg': x['argument'], 'desc': x['description']} for x in all_parsers if 'command' in x['tags']] + generic = [{'arg': x['argument'], 'desc': x['description']} for x in all_parsers if 'generic' in x.get('tags', [])] + standard = [{'arg': x['argument'], 'desc': x['description']} for x in all_parsers if 'standard' in x.get('tags', [])] + command = [{'arg': x['argument'], 'desc': x['description']} for x in all_parsers if 'command' in x.get('tags', [])] file_str_bin = [ {'arg': x['argument'], 'desc': x['description']} for x in all_parsers - if 'file' in x['tags'] or - 'string' in x['tags'] or - 'binary' in x['tags'] + if 'file' in x.get('tags', []) or + 'string' in x.get('tags', []) or + 'binary' in x.get('tags', []) ] streaming = [{'arg': x['argument'], 'desc': x['description']} for x in all_parsers if x.get('streaming')] categories: Dict = { diff --git a/jc/lib.py b/jc/lib.py index 4621b9070..54f234364 100644 --- a/jc/lib.py +++ b/jc/lib.py @@ -213,6 +213,19 @@ def _modname_to_cliname(parser_mod_name: str) -> str: """Return module's cli name (underscores converted to dashes)""" return parser_mod_name.replace('_', '-') +def _is_valid_parser_plugin(name: str, local_parsers_dir: str) -> bool: + if re.match(r'\w+\.py$', name) and os.path.isfile(os.path.join(local_parsers_dir, name)): + try: + parser_mod_name = _cliname_to_modname(name)[0:-3] + modpath = 'jcparsers.' + plugin = importlib.import_module(f'{modpath}{parser_mod_name}') + if hasattr(plugin, 'info') and hasattr(plugin, 'parse'): + del plugin + return True + except Exception: + return False + return False + # Create the local_parsers list. This is a list of custom or # override parsers from /jc/jcparsers/*.py. # Once this list is created, extend the parsers list with it. @@ -222,7 +235,7 @@ def _modname_to_cliname(parser_mod_name: str) -> str: if os.path.isdir(local_parsers_dir): sys.path.append(data_dir) for name in os.listdir(local_parsers_dir): - if re.match(r'\w+\.py$', name) and os.path.isfile(os.path.join(local_parsers_dir, name)): + if _is_valid_parser_plugin(name, local_parsers_dir): plugin_name = name[0:-3] local_parsers.append(_modname_to_cliname(plugin_name)) if plugin_name not in parsers: diff --git a/man/jc.1 b/man/jc.1 index 5aa93e782..0dd56e5d3 100644 --- a/man/jc.1 +++ b/man/jc.1 @@ -1,4 +1,4 @@ -.TH jc 1 2023-03-22 1.23.1 "JSON Convert" +.TH jc 1 2023-03-23 1.23.1 "JSON Convert" .SH NAME \fBjc\fP \- JSON Convert JSONifies the output of many CLI tools, file-types, and strings @@ -1316,8 +1316,8 @@ etc... Note: Unbuffered output can be slower for large data streams. .RE -.SH CUSTOM PARSERS -Custom local parser plugins may be placed in a \fBjc/jcparsers\fP folder in your +.SH PARSER PLUGINS +Parser plugins may be placed in a \fBjc/jcparsers\fP folder in your local "App data directory": .RS @@ -1328,11 +1328,13 @@ local "App data directory": .fi .RE -Local parser plugins are standard python module files. Use the +Parser plugins are standard python module files. Use the \fBjc/parsers/foo.py\fP or \fBjc/parsers/foo_s.py\fP (streaming) parser as a template and simply place a \fB.py\fP file in the \fBjcparsers\fP subfolder. +Any dependencies can be placed in the \fBjc\fP folder above \fBjcparsers\fP +and can be imported in the parser code. -Local plugin filenames must be valid python module names and therefore must +Parser plugin filenames must be valid python module names and therefore must start with a letter and consist entirely of alphanumerics and underscores. Local plugins may override default parsers. diff --git a/templates/manpage_template b/templates/manpage_template index 9630e8cbe..26e8a0c21 100644 --- a/templates/manpage_template +++ b/templates/manpage_template @@ -366,8 +366,8 @@ etc... Note: Unbuffered output can be slower for large data streams. .RE -.SH CUSTOM PARSERS -Custom local parser plugins may be placed in a \fBjc/jcparsers\fP folder in your +.SH PARSER PLUGINS +Parser plugins may be placed in a \fBjc/jcparsers\fP folder in your local "App data directory": .RS @@ -378,11 +378,13 @@ local "App data directory": .fi .RE -Local parser plugins are standard python module files. Use the +Parser plugins are standard python module files. Use the \fBjc/parsers/foo.py\fP or \fBjc/parsers/foo_s.py\fP (streaming) parser as a template and simply place a \fB.py\fP file in the \fBjcparsers\fP subfolder. +Any dependencies can be placed in the \fBjc\fP folder above \fBjcparsers\fP +and can be imported in the parser code. -Local plugin filenames must be valid python module names and therefore must +Parser plugin filenames must be valid python module names and therefore must start with a letter and consist entirely of alphanumerics and underscores. Local plugins may override default parsers. diff --git a/templates/readme_template b/templates/readme_template index 16ca2d6aa..92b767bf2 100644 --- a/templates/readme_template +++ b/templates/readme_template @@ -393,20 +393,22 @@ for item in result: print(item["filename"]) ``` -### Custom Parsers -Custom local parser plugins may be placed in a `jc/jcparsers` folder in your -local **"App data directory"**: +### Parser Plugins +Parser plugins may be placed in a `jc/jcparsers` folder in your local +**"App data directory"**: - Linux/unix: `$HOME/.local/share/jc/jcparsers` - macOS: `$HOME/Library/Application Support/jc/jcparsers` - Windows: `$LOCALAPPDATA\jc\jc\jcparsers` -Local parser plugins are standard python module files. Use the +Parser plugins are standard python module files. Use the [`jc/parsers/foo.py`](https://github.com/kellyjonbrazil/jc/blob/master/jc/parsers/foo.py) or [`jc/parsers/foo_s.py (streaming)`](https://github.com/kellyjonbrazil/jc/blob/master/jc/parsers/foo_s.py) parser as a template and simply place a `.py` file in the `jcparsers` subfolder. +Any dependencies can be placed in the `jc` folder above `jcparsers` and can +be imported in the parser code. -Local plugin filenames must be valid python module names and therefore must +Parser plugin filenames must be valid python module names and therefore must start with a letter and consist entirely of alphanumerics and underscores. Local plugins may override default parsers. From bff065daf3c58028aedf7bc2f7e121f00676fc3f Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Thu, 23 Mar 2023 16:41:50 -0700 Subject: [PATCH 18/18] doc update --- CHANGELOG | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG b/CHANGELOG index 92ae4afa3..417c15966 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,11 +1,12 @@ jc changelog -20230319 v1.23.1 +20230323 v1.23.1 - Fix `zpool-status` command parser for lines that start with tab - Fix `timedatectl` command parser when RTC set to local - Fix to ensure `py.typed` file is included in the package wheel - Fix `lsusb` command parser to support CDC MBIM and CDC MBIM Extended fields - Add support for the `timesync-status` for the `timedatectl` command parser +- Fix to ignore non-parser-plugins in the parser plugin directory 20230227 v1.23.0 - Add input slicing as a `jc` command-line option