From eb1894fc0b67a7eb9abe1f5c4c98fa1dc98798c0 Mon Sep 17 00:00:00 2001 From: Przemek Wirkus Date: Mon, 4 Jul 2016 15:16:46 +0100 Subject: [PATCH 1/5] Add global $HOME/.mbed-ls/.mbedls-mock storage for mock file For backward compatibilty we will for now read first local .mbedls-mock file and skip one in $HOME --- mbed_lstools/lstools_base.py | 72 ++++++++++++++++++++++++++++-------- setup.py | 4 +- 2 files changed, 59 insertions(+), 17 deletions(-) diff --git a/mbed_lstools/lstools_base.py b/mbed_lstools/lstools_base.py index 9a44c76..4c608c9 100644 --- a/mbed_lstools/lstools_base.py +++ b/mbed_lstools/lstools_base.py @@ -17,7 +17,9 @@ import re import os +from os.path import expanduser import json +import lockfile from os import listdir from os.path import isfile, join @@ -38,6 +40,9 @@ def __init__(self): for mid in mock_ids: self.manufacture_ids[mid] = mock_ids[mid] + # Create in HOME direcotry place for mbed-ls to store information + self.mbedls_home_dir_init() + # Which OSs are supported by this module # Note: more than one OS can be supported by mbed-lstools_* module os_supported = [] @@ -190,11 +195,27 @@ def __init__(self): "RIOT": "RIOT", } + # Directory where we will store global (OS user specific mocking) + HOME_DIR = expanduser("~") + MBEDLS_HOME_DIR = '.mbed-ls' MOCK_FILE_NAME = '.mbedls-mock' + MBEDLS_GLOBAL_LOCK = 'mbedls-lock' + MOCK_HOME_FILE_NAME = os.path.join(HOME_DIR, MBEDLS_HOME_DIR, MOCK_FILE_NAME) RETARGET_FILE_NAME = 'mbedls.json' DETAILS_TXT_NAME = 'DETAILS.TXT' MBED_HTM_NAME = 'mbed.htm' + def mbedls_home_dir_init(self): + """ Initialize data in home directory for locking features + """ + if not os.path.isdir(os.path.join(self.HOME_DIR, self.MBEDLS_HOME_DIR)): + os.mkdir(os.path.join(self.HOME_DIR, self.MBEDLS_HOME_DIR)) + + def mbedls_get_global_lock(self): + file_path = os.path.join(self.HOME_DIR, self.MBEDLS_HOME_DIR, self.MBEDLS_GLOBAL_LOCK) + lock = lockfile.LockFile(file_path) + return lock + def list_manufacture_ids(self): from prettytable import PrettyTable @@ -213,35 +234,54 @@ def mock_read(self): """! Load mocking data from local file @return Curent mocking configuration (dictionary) """ - if isfile(self.MOCK_FILE_NAME): + + def read_mock_file(filename): if self.DEBUG_FLAG: - self.debug(self.mock_read.__name__, "reading mock file %s"% self.MOCK_FILE_NAME) + self.debug(self.mock_read.__name__, "reading mock file '%s'"% filename) try: - with open(self.MOCK_FILE_NAME, "r") as f: + with open(filename, "r") as f: return json.load(f) except IOError as e: - self.err("reading file '%s' failed: %s"% (os.path.abspath(self.MOCK_FILE_NAME), + self.err("reading file '%s' failed: %s"% (os.path.abspath(filename), str(e))) except ValueError as e: - self.err("reading file '%s' content failed: %s"% (os.path.abspath(self.MOCK_FILE_NAME), + self.err("reading file '%s' content failed: %s"% (os.path.abspath(filename), str(e))) + return {} + + with self.mbedls_get_global_lock(): + # This read is for backward compatibility + # When user already have on its system local mock-up it will work + # overwriting global one + if isfile(self.MOCK_FILE_NAME): + return read_mock_file(self.MOCK_FILE_NAME) + + if isfile(self.MOCK_HOME_FILE_NAME): + return read_mock_file(self.MOCK_HOME_FILE_NAME) return {} def mock_write(self, mock_ids): """! Write current mocking structure @param mock_ids JSON mock data to dump to file """ - if self.DEBUG_FLAG: - self.debug(self.mock_write.__name__, "writing %s"% self.MOCK_FILE_NAME) - try: - with open(self.MOCK_FILE_NAME, "w") as f: - f.write(json.dumps(mock_ids, indent=4)) - except IOError as e: - self.err("reading file '%s' failed: %s"% (os.path.abspath(self.MOCK_FILE_NAME), - str(e))) - except ValueError as e: - self.err("reading file '%s' content failed: %s"% (os.path.abspath(self.MOCK_FILE_NAME), - str(e))) + def write_mock_file(filename, mock_ids): + if self.DEBUG_FLAG: + self.debug(self.mock_write.__name__, "writing mock file '%s'"% filename) + try: + with open(filename, "w") as f: + f.write(json.dumps(mock_ids, indent=4)) + return True + except IOError as e: + self.err("writing file '%s' failed: %s"% (os.path.abspath(filename), + str(e))) + except ValueError as e: + self.err("writing file '%s' content failed: %s"% (os.path.abspath(filename), + str(e))) + return False + + with self.mbedls_get_global_lock(): + return write_mock_file(self.MOCK_HOME_FILE_NAME, mock_ids) + return False def retarget_read(self): """! Load retarget data from local file diff --git a/setup.py b/setup.py index 751e46a..0a6beae 100644 --- a/setup.py +++ b/setup.py @@ -49,4 +49,6 @@ def read(fname): "mbedls=mbed_lstools:mbedls_main", ], }, - install_requires=["PrettyTable>=0.7.2"]) + install_requires=[ + "PrettyTable>=0.7.2", + "lockfile"]) From 9054a5f3fb2de2e256efd6f10ceb1a8d98b6c34c Mon Sep 17 00:00:00 2001 From: Przemek Wirkus Date: Mon, 4 Jul 2016 15:26:24 +0100 Subject: [PATCH 2/5] Add try/catch for filelock operations --- mbed_lstools/lstools_base.py | 30 ++++++++++++++++++------------ 1 file changed, 18 insertions(+), 12 deletions(-) diff --git a/mbed_lstools/lstools_base.py b/mbed_lstools/lstools_base.py index 4c608c9..de3022c 100644 --- a/mbed_lstools/lstools_base.py +++ b/mbed_lstools/lstools_base.py @@ -40,7 +40,7 @@ def __init__(self): for mid in mock_ids: self.manufacture_ids[mid] = mock_ids[mid] - # Create in HOME direcotry place for mbed-ls to store information + # Create in HOME directory place for mbed-ls to store information self.mbedls_home_dir_init() # Which OSs are supported by this module @@ -249,15 +249,18 @@ def read_mock_file(filename): str(e))) return {} - with self.mbedls_get_global_lock(): - # This read is for backward compatibility - # When user already have on its system local mock-up it will work - # overwriting global one - if isfile(self.MOCK_FILE_NAME): - return read_mock_file(self.MOCK_FILE_NAME) - - if isfile(self.MOCK_HOME_FILE_NAME): - return read_mock_file(self.MOCK_HOME_FILE_NAME) + try: + with self.mbedls_get_global_lock(): + # This read is for backward compatibility + # When user already have on its system local mock-up it will work + # overwriting global one + if isfile(self.MOCK_FILE_NAME): + return read_mock_file(self.MOCK_FILE_NAME) + + if isfile(self.MOCK_HOME_FILE_NAME): + return read_mock_file(self.MOCK_HOME_FILE_NAME) + except LockFailed as e: + self.err(str(e)) return {} def mock_write(self, mock_ids): @@ -279,8 +282,11 @@ def write_mock_file(filename, mock_ids): str(e))) return False - with self.mbedls_get_global_lock(): - return write_mock_file(self.MOCK_HOME_FILE_NAME, mock_ids) + try: + with self.mbedls_get_global_lock(): + return write_mock_file(self.MOCK_HOME_FILE_NAME, mock_ids) + except LockFailed as e: + self.err(str(e)) return False def retarget_read(self): From 2b1c4302cd9d0ebca48b1b48291e52f5233c1e4f Mon Sep 17 00:00:00 2001 From: Przemek Wirkus Date: Mon, 4 Jul 2016 15:30:15 +0100 Subject: [PATCH 3/5] Moved debug flag check inside debug function for cleaner source code --- mbed_lstools/lstools_base.py | 48 ++++++++++----------------- mbed_lstools/lstools_linux_generic.py | 9 ++--- mbed_lstools/lstools_win7.py | 15 +++------ 3 files changed, 25 insertions(+), 47 deletions(-) diff --git a/mbed_lstools/lstools_base.py b/mbed_lstools/lstools_base.py index de3022c..aaea33a 100644 --- a/mbed_lstools/lstools_base.py +++ b/mbed_lstools/lstools_base.py @@ -236,8 +236,7 @@ def mock_read(self): """ def read_mock_file(filename): - if self.DEBUG_FLAG: - self.debug(self.mock_read.__name__, "reading mock file '%s'"% filename) + self.debug(self.mock_read.__name__, "reading mock file '%s'"% filename) try: with open(filename, "r") as f: return json.load(f) @@ -268,8 +267,7 @@ def mock_write(self, mock_ids): @param mock_ids JSON mock data to dump to file """ def write_mock_file(filename, mock_ids): - if self.DEBUG_FLAG: - self.debug(self.mock_write.__name__, "writing mock file '%s'"% filename) + self.debug(self.mock_write.__name__, "writing mock file '%s'"% filename) try: with open(filename, "w") as f: f.write(json.dumps(mock_ids, indent=4)) @@ -294,8 +292,7 @@ def retarget_read(self): @return Curent retarget configuration (dictionary) """ if os.path.isfile(self.RETARGET_FILE_NAME): - if self.DEBUG_FLAG: - self.debug(self.retarget_read.__name__, "reading retarget file %s"% self.RETARGET_FILE_NAME) + self.debug(self.retarget_read.__name__, "reading retarget file %s"% self.RETARGET_FILE_NAME) try: with open(self.RETARGET_FILE_NAME, "r") as f: return json.load(f) @@ -324,17 +321,14 @@ def mock_manufacture_ids(self, mid, platform_name, oper='+'): # Operations on mocked structure if oper == '+': mock_ids[mid] = platform_name - if self.DEBUG_FLAG: - self.debug(self.mock_manufacture_ids.__name__, "mock_ids['%s'] = '%s'"% (mid, platform_name)) + self.debug(self.mock_manufacture_ids.__name__, "mock_ids['%s'] = '%s'"% (mid, platform_name)) elif oper in ['-', '!']: if mid in mock_ids: mock_ids.pop(mid) - if self.DEBUG_FLAG: - self.debug(self.mock_manufacture_ids.__name__, "removing '%s' mock"% mid) + self.debug(self.mock_manufacture_ids.__name__, "removing '%s' mock"% mid) elif mid == '*': mock_ids = {} # Zero mocking set - if self.DEBUG_FLAG: - self.debug(self.mock_manufacture_ids.__name__, "zero mocking set") + self.debug(self.mock_manufacture_ids.__name__, "zero mocking set") self.mock_write(mock_ids) return mock_ids @@ -381,8 +375,7 @@ def list_mbeds_ext(self): target_id = val['target_id'] if target_id in self.retarget_data: mbeds[i].update(self.retarget_data[target_id]) - if self.DEBUG_FLAG: - self.debug(self.list_mbeds_ext.__name__, ("retargeting", target_id, mbeds[i])) + self.debug(self.list_mbeds_ext.__name__, ("retargeting", target_id, mbeds[i])) # Add interface chip meta data to mbed structure details_txt = self.get_details_txt(val['mount_point']) @@ -399,8 +392,7 @@ def list_mbeds_ext(self): if field_name not in mbeds[i]: mbeds[i][field_name] = mbed_htm[field] - if self.DEBUG_FLAG: - self.debug(self.list_mbeds_ext.__name__, (mbeds[i]['platform_name_unique'], val['target_id'])) + self.debug(self.list_mbeds_ext.__name__, (mbeds[i]['platform_name_unique'], val['target_id'])) return mbeds def list_platforms(self): @@ -462,7 +454,8 @@ def debug(self, name, text): @param text Text to be included in debug message @details Function prints directly on console """ - print 'debug @%s.%s: %s'% (self.__class__.__name__, name, text) + if self.DEBUG_FLAG: + print 'debug @%s.%s: %s'% (self.__class__.__name__, name, text) def __str__(self): """! Object to string casting @@ -584,11 +577,9 @@ def get_mbed_htm_lines(self, mount_point): with open(mbed_htm_path, 'r') as f: result = f.readlines() except IOError: - if self.DEBUG_FLAG: - self.debug(self.get_mbed_htm_target_id.__name__, ('Failed to open file', mbed_htm_path)) + self.debug(self.get_mbed_htm_target_id.__name__, ('Failed to open file', mbed_htm_path)) except OSError: - if self.DEBUG_FLAG: - self.debug(self.get_mbed_htm_target_id.__name__, ('Failed to list mount point', mount_point)) + self.debug(self.get_mbed_htm_target_id.__name__, ('Failed to list mount point', mount_point)) return result @@ -622,8 +613,7 @@ def get_details_txt(self, mount_point): with open(path_to_details_txt, 'r') as f: result = self.parse_details_txt(f.readlines()) except IOError: - if self.DEBUG_FLAG: - self.debug(self.get_mbed_fw_version.get_details_txt.__name__, ('Failed to open file', path_to_details_txt)) + self.debug(self.get_mbed_fw_version.get_details_txt.__name__, ('Failed to open file', path_to_details_txt)) return result if result else None def parse_details_txt(self, lines): @@ -649,20 +639,16 @@ def scan_html_line_for_target_id(self, line): m = re.search('\?code=([a-fA-F0-9]+)', line) if m: result = m.groups()[0] - if self.DEBUG_FLAG: - self.debug(self.scan_html_line_for_target_id.__name__, line.strip()) - if self.DEBUG_FLAG: - self.debug(self.scan_html_line_for_target_id.__name__, (m.groups(), result)) + self.debug(self.scan_html_line_for_target_id.__name__, line.strip()) + self.debug(self.scan_html_line_for_target_id.__name__, (m.groups(), result)) return result # Last resort, we can try to see if old mbed.htm format is there else: m = re.search('\?auth=([a-fA-F0-9]+)', line) if m: result = m.groups()[0] - if self.DEBUG_FLAG: - self.debug(self.scan_html_line_for_target_id.__name__, line.strip()) - if self.DEBUG_FLAG: - self.debug(self.scan_html_line_for_target_id.__name__, (m.groups(), result)) + self.debug(self.scan_html_line_for_target_id.__name__, line.strip()) + self.debug(self.scan_html_line_for_target_id.__name__, (m.groups(), result)) return result return None diff --git a/mbed_lstools/lstools_linux_generic.py b/mbed_lstools/lstools_linux_generic.py index 225463f..4877075 100644 --- a/mbed_lstools/lstools_linux_generic.py +++ b/mbed_lstools/lstools_linux_generic.py @@ -138,8 +138,7 @@ def get_dev_by_id_process(self, lines, retval): line = line.rstrip() if not line.lower().startswith('total '): # total 0 result.append(line) - if self.DEBUG_FLAG: - self.debug(self.get_dev_by_id_process.__name__, line) + self.debug(self.get_dev_by_id_process.__name__, line) return result def get_dev_by_id(self, subdir): @@ -157,8 +156,7 @@ def get_mounts(self): result = [] cmd = 'mount | grep vfat' - if self.DEBUG_FLAG: - self.debug(self.get_mounts.__name__, cmd) + self.debug(self.get_mounts.__name__, cmd) _stdout, _, retval = self.run_cli_process(cmd) @@ -166,8 +164,7 @@ def get_mounts(self): for line in _stdout.splitlines(): line = line.rstrip() result.append(line) - if self.DEBUG_FLAG: - self.debug(self.get_mounts.__name__, line) + self.debug(self.get_mounts.__name__, line) return result def get_disk_hex_ids(self, disk_list): diff --git a/mbed_lstools/lstools_win7.py b/mbed_lstools/lstools_win7.py index 7b877ad..10024d5 100644 --- a/mbed_lstools/lstools_win7.py +++ b/mbed_lstools/lstools_win7.py @@ -89,8 +89,7 @@ def get_mbed_com_port(self, tid): self.winreg.Enum = self.winreg.OpenKey(self.winreg.HKEY_LOCAL_MACHINE, r'SYSTEM\CurrentControlSet\Enum') usb_devs = self.winreg.OpenKey(self.winreg.Enum, 'USB') - if self.DEBUG_FLAG: - self.debug(self.get_mbed_com_port.__name__, 'ID: ' + tid) + self.debug(self.get_mbed_com_port.__name__, 'ID: ' + tid) # first try to find all devs keys (by tid) dev_keys = [] @@ -105,8 +104,7 @@ def get_mbed_com_port(self, tid): try: param = self.winreg.OpenKey(key, "Device Parameters") port = self.winreg.QueryValueEx(param, 'PortName')[0] - if self.DEBUG_FLAG: - self.debug(self.get_mbed_com_port.__name__, port) + self.debug(self.get_mbed_com_port.__name__, port) return port except: pass @@ -122,8 +120,7 @@ def get_mbed_com_port(self, tid): ports += [self.get_mbed_com_port(dev)] for port in ports: if port: - if self.DEBUG_FLAG: - self.debug(self.get_mbed_com_port.__name__, port) + self.debug(self.get_mbed_com_port.__name__, port) return port except: pass @@ -148,8 +145,7 @@ def get_mbeds(self): # TargetID is a hex string with 10-48 chars tid = re.search('[0-9A-Fa-f]{10,48}', mbed[1]).group(0) mbeds += [(mountpoint, tid)] - if self.DEBUG_FLAG: - self.debug(self.get_mbeds.__name__, (mountpoint, tid)) + self.debug(self.get_mbeds.__name__, (mountpoint, tid)) return mbeds # =============================== Registry ==================================== @@ -183,8 +179,7 @@ def get_mbed_devices(self): result += [d for d in self.get_dos_devices() if ven.upper() in d[1].upper()] for r in result: - if self.DEBUG_FLAG: - self.debug(self.get_mbed_devices.__name__, r) + self.debug(self.get_mbed_devices.__name__, r) return result def get_dos_devices(self): From b7154d87074290c56afa48b50beee830857e3a38 Mon Sep 17 00:00:00 2001 From: Przemek Wirkus Date: Mon, 4 Jul 2016 15:33:18 +0100 Subject: [PATCH 4/5] Add missing import for lockfile exception --- mbed_lstools/lstools_base.py | 1 + 1 file changed, 1 insertion(+) diff --git a/mbed_lstools/lstools_base.py b/mbed_lstools/lstools_base.py index aaea33a..43a7c2c 100644 --- a/mbed_lstools/lstools_base.py +++ b/mbed_lstools/lstools_base.py @@ -22,6 +22,7 @@ import lockfile from os import listdir from os.path import isfile, join +from lockfile import LockFailed class MbedLsToolsBase: """ Base class for mbed-lstools, defines mbed-ls tools interface for mbed-enabled devices detection for various hosts From 79f44cced683f5ecf67de0a48cc287fe68e055f0 Mon Sep 17 00:00:00 2001 From: Przemek Wirkus Date: Tue, 5 Jul 2016 11:01:58 +0100 Subject: [PATCH 5/5] Add changes to README.md --- README.md | 27 ++++++++++++--------------- 1 file changed, 12 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index dc3c399..ddba3d6 100644 --- a/README.md +++ b/README.md @@ -441,7 +441,7 @@ Let's say we want change ```serial_port```'s value to other COM port. For exampl To do so we would have to create a new file called ```mbedls.json``` in directory where want to use this modification. File content could look like this: a JSON file where keys are ```target_id```'s and values are dictionaries with new values: ``` -$ cat mbedls.ls +$ cat mbedls.json { "0240022648cb1e77000000000000000000000000b512e3cf" : { "serial_port" : "MyComPort01" @@ -461,24 +461,21 @@ $ mbedls # Mocking new or existing target to custom platform name Command line switch ```--mock``` provide simple manufacturers ID masking with new platform name. -Users should be able to add locally new ```MID``` -> ```platform_name``` mapping when e.g. prototyping. +Users should be able to add temporarily new ```MID``` -> ```platform_name``` mapping when e.g. prototyping. -Mock configuration will be stored in directory where ```mbedls --mock``` command was issues, in local file ```.mbedls-mock```. +Mock configuration will be stored in `$HOME/.mbed-ls/` directory, in local file ```.mbedls-mock```. -**Note***: ```MID```: "manufacturers ID", first 4 characters of ```target_id```. Example: If ```target_id``` is ```02400221A0811E505D5FE3E8```, corresponding manufacturers ID is ```0240```. +**Note***: ```MID``` stands for "manufacturers ID". `MID` is first four (4) characters of ```target_id``` string. Example: If ```target_id``` is ```02400221A0811E505D5FE3E8```, corresponding manufacturers ID is ```0240```. ## Mock command line examples -* Add new command line parameter ```--mock``` (switch -m) -* Add new / mask existing mapping ```MID``` -> ```platform_name``` and assign MID +* Mock command line parameter: `--mock` or (switch `-m`) +* Add new / mask existing mapping ```MID``` -> ```platform_name``` and assign `MID`: * ```$ mbedls --mock MID:PLATFORM_NAME``` or * ```$ mbedls --mock MID1:PLATFORM_NAME1,MID2:PLATFORM_NAME2``` -* Mask existing manufacturers ID with new platform name -* Remove masking with '!' prefix - * ```$ mbedls --mock !MID``` -* Remove all maskings using !* notation - * ```$ mbedls --mock !*``` -* Combine above using comma (```,```) separator: - * ```$ mbedls --mock MID1:PLATFORM_NAME1,!MID2``` + * Example: `$ mbedls --mock 0818:NUCLEO_F767ZI` +* Remove masking with '!' prefix: `$ mbedls --mock !MID` +* Remove all maskings using !* notation: `$ mbedls --mock !*` +* Combine above using comma (`,`) separator: `$ mbedls --mock MID1:PLATFORM_NAME1,!MID2` ## Mocking example with Freescale K64F platform Initial setup with 1 x Freescale ```K64F``` board: @@ -513,7 +510,7 @@ $ mbedls +--------------+---------------------+------------+------------+-------------------------+ ``` -* We can remove mapping ```1234``` -> Anythying using ```!``` wildcard. +* We can remove mapping ```1234``` -> Anythying using ```!``` wild-card. Note: We are using flag ```-json``` to get JSON format output of the ```--mock``` operation. ``` $ mbedls --mock !1234 --json @@ -539,7 +536,7 @@ $ mbedls --mock !* We can verify our mapping is reset: ``` -$ cat .mbedls-mock +$ cat $HOME/.mbed-ls/.mbedls-mock {} ```