From f92b44bc4ff86d0bf13d75231b61404d07f68d9a Mon Sep 17 00:00:00 2001 From: "Malcolm Jones (bossjones/Tony Dark)" Date: Mon, 6 Aug 2018 21:18:36 -0400 Subject: [PATCH] Feature: feature-fix-flatpak-env-and-config-usage (#298) --- Makefile | 31 + hacking/flatpak/enter-env | 4 +- requirements_dev.in | 1 + scarlett_os/common/configure/default.yaml | 22 +- scarlett_os/common/configure/ruamel_config.py | 361 +++++++----- scarlett_os/config.py | 433 +++++++------- scarlett_os/internal/debugger.py | 22 +- scarlett_os/listener.py | 117 +++- scarlett_os/scripts/cli.py | 242 +++++--- scarlett_os/scripts/config.py | 62 +- scarlett_os/utility/yaml.py | 536 ++++++++--------- scripts/check_mock_path_autospec | 15 + scripts/monkeytype_run | 25 + setup.py | 25 +- tests/__init__.py | 548 ++++++++++++++++++ tests/conftest.py | 76 ++- .../test_integration_listener.py | 13 +- tests/unittests/test_cli.py | 21 +- tests/unittests/utility/test_yaml.py | 522 ++++++++--------- 19 files changed, 1997 insertions(+), 1079 deletions(-) create mode 100755 scripts/check_mock_path_autospec create mode 100755 scripts/monkeytype_run diff --git a/Makefile b/Makefile index 2ab87938..1908520f 100644 --- a/Makefile +++ b/Makefile @@ -1053,3 +1053,34 @@ run-black-check: .PHONY: run-black run-black: black --verbose . + + +.PHONY: reinstall-deps +reinstall-deps: + pip install --force-reinstall -r requirements.txt + +.PHONY: install-deps-all +reinstall-deps-all: + pip install --force-reinstall -r requirements.txt; \ + pip install --force-reinstall -r requirements-dev.txt; \ + +.PHONY: install-deps-osx +install-deps-osx: + env ARCHFLAGS="-arch x86_64" LDFLAGS="-L/usr/local/opt/openssl/lib" CFLAGS="-I/usr/local/opt/openssl/include" pip install -r requirements.txt + +.PHONY: install-deps-all-osx +install-deps-all-osx: + env ARCHFLAGS="-arch x86_64" LDFLAGS="-L/usr/local/opt/openssl/lib" CFLAGS="-I/usr/local/opt/openssl/include" pip install -r requirements.txt; \ + env ARCHFLAGS="-arch x86_64" LDFLAGS="-L/usr/local/opt/openssl/lib" CFLAGS="-I/usr/local/opt/openssl/include" pip install -r requirements_dev.txt; \ + + + +# NOTE: This assumes that all of your repos live in the same workspace! +# link_roles: +# # add aliases for dotfiles +# @for file in $(shell find $(CURDIR)/.. -name "*ansible-role*" -type d -print); do \ +# echo $$file; \ +# f=$$(basename $$file); \ +# ln -sfn $$file $(CURDIR)/roles/$f; \ +# done; \ +# ls -lta $(CURDIR)/roles/; \ diff --git a/hacking/flatpak/enter-env b/hacking/flatpak/enter-env index 81acb470..221fe481 100755 --- a/hacking/flatpak/enter-env +++ b/hacking/flatpak/enter-env @@ -3,13 +3,13 @@ SCARLETT_DIR=$(realpath $(dirname $0)/../../) function generate_path_and_completion_calls { - echo "export PATH=$SCARLETT_DIR/bin/:/app/bin/:/usr/bin/:\$PATH" + echo "export PATH=$SCARLETT_DIR/bin:/app/bin:/usr/bin:\$PATH" } RCFILE=$SCARLETT_DIR/.bashrc cp ~/.bashrc $RCFILE -echo "export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/app/lib/" >> $RCFILE +echo "export LD_LIBRARY_PATH=\$LD_LIBRARY_PATH:/app/lib/" >> $RCFILE echo "export PS1=[scarlettOS]\ \$PS1" >> $RCFILE generate_path_and_completion_calls >> $RCFILE diff --git a/requirements_dev.in b/requirements_dev.in index 87a6ec2e..b21bfa18 100644 --- a/requirements_dev.in +++ b/requirements_dev.in @@ -58,3 +58,4 @@ yapf # Update packages in a requirements.txt file to latest versions. pur +rope diff --git a/scarlett_os/common/configure/default.yaml b/scarlett_os/common/configure/default.yaml index 34bfe02a..d8b8e504 100644 --- a/scarlett_os/common/configure/default.yaml +++ b/scarlett_os/common/configure/default.yaml @@ -6,14 +6,32 @@ latitude: 40.7056308 longitude: -73.9780034 +# pocketsphinx config options source: https://github.com/cmusphinx/pocketsphinx/blob/1fdc9ccb637836d45d40956e745477a2bd3b470a/doc/pocketsphinx_continuous.1 pocketsphinx: hmm: /home/pi/.virtualenvs/scarlett_os/share/pocketsphinx/model/en-us/en-us lm: /home/pi/dev/bossjones-github/scarlett_os/static/speech/lm/1473.lm dict: /home/pi/dev/bossjones-github/scarlett_os/static/speech/dict/1473.dic + # Silence word transition probability silprob: 0.1 - wip: 1e-4 - bestpath: 0 + # ******************************************************** + # FIXME: ????? THIS IS THE ORIG VALUE, do we need too set it back? 8/5/2018 # wip: 1e-4 + # Enable Graph Search | Boolean. Default: true + # ******************************************************** + # Word insertion penalty + wip: 0.0001 device: plughw:CARD=Device,DEV=0 + # ******************************************************** + # FIXME: ????? THIS IS THE ORIG VALUE, do we need too set it back? 8/5/2018 # bestpath: 0 + # Enable Graph Search | Boolean. Default: true + # ******************************************************** + bestpath: True + # Enable Flat Lexicon Search | Default: true + fwdflat: True + # Evaluate acoustic model every N frames | Integer. Range: 1 - 10 Default: 1 + dsratio: 1 + # Maximum number of HMMs searched per frame | Integer. Range: 1 - 100000 Default: 30000 + maxhmmpf: 3000 + # Impacts weather/sunrise data elevation: 665 diff --git a/scarlett_os/common/configure/ruamel_config.py b/scarlett_os/common/configure/ruamel_config.py index 177e4633..66a2b4b4 100644 --- a/scarlett_os/common/configure/ruamel_config.py +++ b/scarlett_os/common/configure/ruamel_config.py @@ -227,9 +227,27 @@ def dump(self, data, stream=None, **kw): hmm: /home/pi/.virtualenvs/scarlett_os/share/pocketsphinx/model/en-us/en-us lm: /home/pi/dev/bossjones-github/scarlett_os/static/speech/lm/1473.lm dict: /home/pi/dev/bossjones-github/scarlett_os/static/speech/dict/1473.dic + # Silence word transition probability silprob: 0.1 - wip: 1e-4 - bestpath: 0 + # ******************************************************** + # FIXME: ????? THIS IS THE ORIG VALUE, do we need too set it back? 8/5/2018 # wip: 1e-4 + # Enable Graph Search | Boolean. Default: true + # ******************************************************** + # Word insertion penalty + wip: 0.0001 + device: plughw:CARD=Device,DEV=0 + # ******************************************************** + # FIXME: ????? THIS IS THE ORIG VALUE, do we need too set it back? 8/5/2018 # bestpath: 0 + # Enable Graph Search | Boolean. Default: true + # ******************************************************** + bestpath: True + # Enable Flat Lexicon Search | Default: true + fwdflat: True + # Evaluate acoustic model every N frames | Integer. Range: 1 - 10 Default: 1 + dsratio: 1 + # Maximum number of HMMs searched per frame | Integer. Range: 1 - 100000 Default: 30000 + maxhmmpf: 3000 + # Impacts weather/sunrise data elevation: 665 @@ -252,6 +270,8 @@ def dump(self, data, stream=None, **kw): features: - time + +graphviz_debug_dir: /home/pi/dev/bossjones-github/scarlett_os/_debug """ RUSSIAN_CONFIG = """ @@ -763,33 +783,34 @@ def _dump_in_memory_config_to_stdout(data, stream=None): ) -def _insert_key_to_commented_map(data, position, key_name, key_value, comment=None): - # type: (Any, Any, Any, Optional[Any]) -> ruamel.yaml.comments.CommentedMap - """[Insert a key into a CommentedMap at a particular position, while optionally adding a comment] +# NOTE: YAGNI at the moment +# def _insert_key_to_commented_map(data, position, key_name, key_value, comment=None): +# # type: (Any, Any, Any, Optional[Any]) -> ruamel.yaml.comments.CommentedMap +# """[Insert a key into a CommentedMap at a particular position, while optionally adding a comment] - Arguments: - data {[ruamel.yaml.comments.CommentedMap]} -- [CommentedMap returned via a roundtrip load] - position {[int]} -- [int providing position where to insert value, eg 1] - key_name {[str]} -- [string value for key value, eg 'Full Name'] - key_value {[str]} -- [string value for key value, eg 'Malcolm Jones'] +# Arguments: +# data {[ruamel.yaml.comments.CommentedMap]} -- [CommentedMap returned via a roundtrip load] +# position {[int]} -- [int providing position where to insert value, eg 1] +# key_name {[str]} -- [string value for key value, eg 'Full Name'] +# key_value {[str]} -- [string value for key value, eg 'Malcolm Jones'] - Keyword Arguments: - comment {[ANY,str]} -- [Optional inline comment] (default: {None}) +# Keyword Arguments: +# comment {[ANY,str]} -- [Optional inline comment] (default: {None}) - Returns: - [ruamel.yaml.comments.CommentedMap] -- [Modified ruamel.yaml.comments.CommentedMap] - """ +# Returns: +# [ruamel.yaml.comments.CommentedMap] -- [Modified ruamel.yaml.comments.CommentedMap] +# """ - # TODO: Validation - # 1. assert position is valid number - # 2. assert key_name in string format - # 3. assert key_value in string format - # 4. assert comment is kwarg with value in string format - data.insert(position, key_name, key_value, comment=comment) +# # TODO: Validation +# # 1. assert position is valid number +# # 2. assert key_name in string format +# # 3. assert key_value in string format +# # 4. assert comment is kwarg with value in string format +# data.insert(position, key_name, key_value, comment=comment) - # EXAMPLE: taken directly from ruamel docs - # data.insert(1, 'last name', 'Vandelay', comment="new key") - return data +# # EXAMPLE: taken directly from ruamel docs +# # data.insert(1, 'last name', 'Vandelay', comment="new key") +# return data # INFO: ruamel: Indentation of block sequences @@ -820,65 +841,65 @@ def _insert_key_to_commented_map(data, position, key_name, key_value, comment=No # assert data.mlget(['a', 1, 'd', 'f'], list_ok=True) == 196 ########################################################################## +# NOTE: YAGNI for the moment +# def get_config_key_position(data, key_name): +# """Return position value of a key in a ruamel CommentedMap -def get_config_key_position(data, key_name): - """Return position value of a key in a ruamel CommentedMap - - Arguments: - data {[ruamel.yaml.comments.CommentedMap]} -- [CommentedMap returned via a roundtrip load] - key_name {[str]} -- [string value for key value, eg 'Full Name'] - - Returns: - [ANY, tuple] -- [Return a tuple of where node object is located in yaml document or return None] - """ +# Arguments: +# data {[ruamel.yaml.comments.CommentedMap]} -- [CommentedMap returned via a roundtrip load] +# key_name {[str]} -- [string value for key value, eg 'Full Name'] - # EXAMPLE: Output - # pdb> in_memory_config.lc.key('pocketsphinx') - # (9, 0) - # pdb> in_memory_config.lc.key('pocketsphinxa') - # *** KeyError: 'pocketsphinxa' - # pdb> - try: - pos = data.lc.key(key_name) - except KeyError: - logger.error( - "Tried to find {} but failed! Setting position return value to None".format( - key_name - ) - ) - pos = None +# Returns: +# [ANY, tuple] -- [Return a tuple of where node object is located in yaml document or return None] +# """ - return pos +# # EXAMPLE: Output +# # pdb> in_memory_config.lc.key('pocketsphinx') +# # (9, 0) +# # pdb> in_memory_config.lc.key('pocketsphinxa') +# # *** KeyError: 'pocketsphinxa' +# # pdb> +# try: +# pos = data.lc.key(key_name) +# except KeyError: +# logger.error( +# "Tried to find {} but failed! Setting position return value to None".format( +# key_name +# ) +# ) +# pos = None +# return pos -def get_config_value_position(data, key_name): - """Return position value of a key in a ruamel CommentedMap +# NOTE: YAGNI ( for the moment ) +# def get_config_value_position(data, key_name): +# """Return position value of a key in a ruamel CommentedMap - Arguments: - data {[ruamel.yaml.comments.CommentedMap]} -- [CommentedMap returned via a roundtrip load] - key_name {[str]} -- [string value for key value, eg 'Full Name'] +# Arguments: +# data {[ruamel.yaml.comments.CommentedMap]} -- [CommentedMap returned via a roundtrip load] +# key_name {[str]} -- [string value for key value, eg 'Full Name'] - Returns: - [ANY, tuple] -- [Return a tuple of where node object is located in yaml document or return None] - """ +# Returns: +# [ANY, tuple] -- [Return a tuple of where node object is located in yaml document or return None] +# """ - # EXAMPLE: Output - # pdb> in_memory_config.lc.value('pocketsphinx') - # (10, 4) - # pdb> in_memory_config.lc.value('pocketsphinxa') - # *** KeyError: 'pocketsphinxa' - # pdb> - try: - pos = data.lc.value(key_name) - except KeyError: - logger.error( - "Tried to find {} but failed! Setting position return value to None".format( - key_name - ) - ) - pos = None +# # EXAMPLE: Output +# # pdb> in_memory_config.lc.value('pocketsphinx') +# # (10, 4) +# # pdb> in_memory_config.lc.value('pocketsphinxa') +# # *** KeyError: 'pocketsphinxa' +# # pdb> +# try: +# pos = data.lc.value(key_name) +# except KeyError: +# logger.error( +# "Tried to find {} but failed! Setting position return value to None".format( +# key_name +# ) +# ) +# pos = None - return pos +# return pos # FIXME: YANGNI @@ -943,74 +964,75 @@ def ensure_config_dir_path(config_dir: str) -> None: # ruamel.yaml.representer.RoundTripRepresenter.add_representer(type(None), represent_none) # ------------------------------------------------------------- +# NOTE: YAGNI # source: chamberlain -def prep_default_config(homedir=None): - """[setup config.yaml defaults] - - Keyword Arguments: - homedir {[str]} -- [path to sub directory containing config.yaml file, eg $HOME/.config/scarlett/config.yaml] (default: {None}) - - - Returns: - [str] -- [home, eg $HOME/.config/scarlett] - [str] -- [default_cfg, eg $HOME/.config/scarlett/config.yaml] - """ - - # ---------------------------------------------- - # DEFAULT CONFIG SETUP - START - # ---------------------------------------------- - # Step 1. loead - default_yaml = os.path.join(os.path.abspath(__file__), "default.yaml") - default_yaml_in_memory_config = load_config(default_yaml) - # Step 2. Check environment variables, if they exist, override them - if os.environ.get("SCARLETT_OS_CONFIG_LATITUDE"): - default_yaml_in_memory_config["latitude"] = os.environ.get( - "SCARLETT_OS_CONFIG_LATITUDE" - ) - if os.environ.get("SCARLETT_OS_CONFIG_LONGITUDE"): - default_yaml_in_memory_config["longitude"] = os.environ.get( - "SCARLETT_OS_CONFIG_LONGITUDE" - ) - if os.environ.get("SCARLETT_OS_CONFIG_POCKETSPHINX_HMM"): - default_yaml_in_memory_config["pocketsphinx"]["hmm"] = os.environ.get( - "SCARLETT_OS_CONFIG_POCKETSPHINX_HMM" - ) - if os.environ.get("SCARLETT_OS_CONFIG_POCKETSPHINX_LM"): - default_yaml_in_memory_config["pocketsphinx"]["lm"] = os.environ.get( - "SCARLETT_OS_CONFIG_POCKETSPHINX_LM" - ) - if os.environ.get("SCARLETT_OS_CONFIG_POCKETSPHINX_DICT"): - default_yaml_in_memory_config["pocketsphinx"]["dict"] = os.environ.get( - "SCARLETT_OS_CONFIG_POCKETSPHINX_DICT" - ) +# def prep_default_config(homedir=None): +# """[setup config.yaml defaults] - # ---------------------------------------------- - # DEFAULT CONFIG SETUP - END - # ---------------------------------------------- +# Keyword Arguments: +# homedir {[str]} -- [path to sub directory containing config.yaml file, eg $HOME/.config/scarlett/config.yaml] (default: {None}) - # Step 1. Get sub directory path for config - if homedir is None: - home = get_config_sub_dir_path() - else: - # override for things like tests - home = homedir - - # Step 2. ensure sub directory actually exists - ensure_config_dir_path(home) - - # Step 3. Set location of config.yaml file - cfg = os.path.join(home, "config.yaml") - # Step 4. check if config file exists, if it doesnt, create a default config - if not os.path.exists(cfg): - # Write merged config - with open(cfg, "wb") as f: - yaml.dump(default_yaml_in_memory_config, f) - - # Load the newly merged configure file - in_memory_cfg = load_config(cfg) +# Returns: +# [str] -- [home, eg $HOME/.config/scarlett] +# [str] -- [default_cfg, eg $HOME/.config/scarlett/config.yaml] +# """ - return home, cfg, in_memory_cfg +# # ---------------------------------------------- +# # DEFAULT CONFIG SETUP - START +# # ---------------------------------------------- +# # Step 1. loead +# default_yaml = os.path.join(os.path.abspath(__file__), "default.yaml") +# default_yaml_in_memory_config = load_config(default_yaml) +# # Step 2. Check environment variables, if they exist, override them +# if os.environ.get("SCARLETT_OS_CONFIG_LATITUDE"): +# default_yaml_in_memory_config["latitude"] = os.environ.get( +# "SCARLETT_OS_CONFIG_LATITUDE" +# ) +# if os.environ.get("SCARLETT_OS_CONFIG_LONGITUDE"): +# default_yaml_in_memory_config["longitude"] = os.environ.get( +# "SCARLETT_OS_CONFIG_LONGITUDE" +# ) +# if os.environ.get("SCARLETT_OS_CONFIG_POCKETSPHINX_HMM"): +# default_yaml_in_memory_config["pocketsphinx"]["hmm"] = os.environ.get( +# "SCARLETT_OS_CONFIG_POCKETSPHINX_HMM" +# ) +# if os.environ.get("SCARLETT_OS_CONFIG_POCKETSPHINX_LM"): +# default_yaml_in_memory_config["pocketsphinx"]["lm"] = os.environ.get( +# "SCARLETT_OS_CONFIG_POCKETSPHINX_LM" +# ) +# if os.environ.get("SCARLETT_OS_CONFIG_POCKETSPHINX_DICT"): +# default_yaml_in_memory_config["pocketsphinx"]["dict"] = os.environ.get( +# "SCARLETT_OS_CONFIG_POCKETSPHINX_DICT" +# ) + +# # ---------------------------------------------- +# # DEFAULT CONFIG SETUP - END +# # ---------------------------------------------- + +# # Step 1. Get sub directory path for config +# if homedir is None: +# home = get_config_sub_dir_path() +# else: +# # override for things like tests +# home = homedir + +# # Step 2. ensure sub directory actually exists +# ensure_config_dir_path(home) + +# # Step 3. Set location of config.yaml file +# cfg = os.path.join(home, "config.yaml") + +# # Step 4. check if config file exists, if it doesnt, create a default config +# if not os.path.exists(cfg): +# # Write merged config +# with open(cfg, "wb") as f: +# yaml.dump(default_yaml_in_memory_config, f) + +# # Load the newly merged configure file +# in_memory_cfg = load_config(cfg) + +# return home, cfg, in_memory_cfg class ConfigManager(object): @@ -1022,36 +1044,75 @@ class ConfigManager(object): os.path.dirname(os.path.abspath(__file__)), "default.yaml" ) - def __init__(self): - self._config = None + def __init__(self, config_path=None): + self.cfg = None + self._config_path = config_path if config_path else ConfigManager.CONFIG_PATH + self._config_path_base = None + self._config_path_cache = None + + @property + def config_path_base(self): + if self._config_path_base is None: + self._config_path_base = os.path.dirname(self._config_path) + return self._config_path_base + + @config_path_base.setter + def config_path_base(self, config_path_base): + self._config_path_base = config_path_base def check_folder_structure(self): - mkdir_if_does_not_exist(os.path.dirname(ConfigManager.CONFIG_PATH)) + mkdir_if_does_not_exist(os.path.dirname(self._config_path)) - def load(self, path_override=None): + def load(self): self.check_folder_structure() - if path_override: - self._config = load_config(path_override) - else: - self._config = load_config(ConfigManager.CONFIG_PATH) + self.cfg = load_config(self._config_path) + self.check_environment_overrides() def check_environment_overrides(self): if os.environ.get("SCARLETT_OS_CONFIG_LATITUDE"): - self._config["latitude"] = os.environ.get("SCARLETT_OS_CONFIG_LATITUDE") + self.cfg["latitude"] = os.environ.get("SCARLETT_OS_CONFIG_LATITUDE") if os.environ.get("SCARLETT_OS_CONFIG_LONGITUDE"): - self._config["longitude"] = os.environ.get("SCARLETT_OS_CONFIG_LONGITUDE") + self.cfg["longitude"] = os.environ.get("SCARLETT_OS_CONFIG_LONGITUDE") if os.environ.get("SCARLETT_OS_CONFIG_POCKETSPHINX_HMM"): - self._config["pocketsphinx"]["hmm"] = os.environ.get( + self.cfg["pocketsphinx"]["hmm"] = os.environ.get( "SCARLETT_OS_CONFIG_POCKETSPHINX_HMM" ) if os.environ.get("SCARLETT_OS_CONFIG_POCKETSPHINX_LM"): - self._config["pocketsphinx"]["lm"] = os.environ.get( + self.cfg["pocketsphinx"]["lm"] = os.environ.get( "SCARLETT_OS_CONFIG_POCKETSPHINX_LM" ) if os.environ.get("SCARLETT_OS_CONFIG_POCKETSPHINX_DICT"): - self._config["pocketsphinx"]["dict"] = os.environ.get( + self.cfg["pocketsphinx"]["dict"] = os.environ.get( "SCARLETT_OS_CONFIG_POCKETSPHINX_DICT" ) + if os.environ.get("SCARLETT_OS_CONFIG_DEVICE"): + self.cfg["pocketsphinx"]["device"] = os.environ.get( + "SCARLETT_OS_CONFIG_DEVICE" + ) + + def as_dict(self): + """Return the attributes for this object as a dictionary. + + This is equivalent to calling:: + + json.loads(obj.as_json()) + + :returns: this object's attributes serialized to a dictionary + :rtype: dict + """ + return self.cfg + + def as_yaml_str(self): + """Return the yaml data for this object. + + This is equivalent to calling:: + + json.dumps(obj.as_dict()) + + :returns: this object's attributes as a JSON string + :rtype: str + """ + return dump_in_memory_config_to_var(self.cfg) def prep_default_config(self): """setup config.yaml defaults.""" diff --git a/scarlett_os/config.py b/scarlett_os/config.py index 8836fc14..10b2a63b 100644 --- a/scarlett_os/config.py +++ b/scarlett_os/config.py @@ -2,6 +2,8 @@ import logging import os import shutil + +# NOTE: Eventually, we want to just delete this whole file. from types import MappingProxyType # pylint: disable=unused-import @@ -37,226 +39,227 @@ VERSION_FILE = ".SC_OS_VERSION" CONFIG_DIR_NAME = ".scarlett_os" -DEFAULT_CORE_CONFIG = ( - # Tuples (attribute, default, auto detect property, description) - (CONF_NAME, "Home", None, "Name of the location where ScarlettOS is " "running"), - ( - CONF_LATITUDE, - 0, - "latitude", - "Location required to calculate the time" " the sun rises and sets", - ), - (CONF_LONGITUDE, 0, "longitude", None), - ( - CONF_ELEVATION, - 0, - None, - "Impacts weather/sunrise data" " (altitude above sea level in meters)", - ), - ( - CONF_UNIT_SYSTEM, - CONF_UNIT_SYSTEM_METRIC, - None, - "{} for Metric, {} for Imperial".format( - CONF_UNIT_SYSTEM_METRIC, CONF_UNIT_SYSTEM_IMPERIAL - ), - ), - ( - CONF_TIME_ZONE, - "UTC", - "time_zone", - "Pick yours from here: http://en.wiki" - "pedia.org/wiki/List_of_tz_database_time_zones", - ), -) # type: Tuple[Tuple[str, Any, Any, str], ...] -DEFAULT_CONFIG = """ -# Show links to resources in log and frontend -introduction: - -# Enables the frontend -frontend: - -http: - # Uncomment this to add a password (recommended!) - # api_password: PASSWORD - -# Checks for available updates -updater: - -# Discover some devices automatically -discovery: - -# Allows you to issue voice commands from the frontend in enabled browsers -conversation: - -# Enables support for tracking state changes over time. -history: - -# View all events in a logbook -logbook: - -# Track the sun -sun: - -# Weather Prediction -sensor: - platform: yr -""" - - -def _valid_customize(value): - """Config validator for customize.""" - if not isinstance(value, dict): - raise vol.Invalid("Expected dictionary") - - for key, val in value.items(): - if not valid_entity_id(key): - raise vol.Invalid("Invalid entity ID: {}".format(key)) - - if not isinstance(val, dict): - raise vol.Invalid("Value of {} is not a dictionary".format(key)) - - return value - - -CORE_CONFIG_SCHEMA = vol.Schema( - { - CONF_NAME: vol.Coerce(str), - CONF_LATITUDE: cv.latitude, - CONF_LONGITUDE: cv.longitude, - CONF_ELEVATION: vol.Coerce(int), - vol.Optional(CONF_TEMPERATURE_UNIT): cv.temperature_unit, - CONF_UNIT_SYSTEM: cv.unit_system, - CONF_TIME_ZONE: cv.time_zone, - vol.Required(CONF_CUSTOMIZE, default=MappingProxyType({})): _valid_customize, - } -) - - -def get_default_config_dir() -> str: - """Put together the default configuration directory based on OS.""" - data_dir = os.getenv("APPDATA") if os.name == "nt" else os.path.expanduser("~") - return os.path.join(data_dir, CONFIG_DIR_NAME) - - -def ensure_config_exists(config_dir: str, detect_location: bool = True) -> str: - """Ensure a config file exists in given configuration directory. - - Creating a default one if needed. - Return path to the config file. - """ - config_path = find_config_file(config_dir) - - if config_path is None: - print("Unable to find configuration. Creating default one in", config_dir) - config_path = create_default_config(config_dir, detect_location) - - return config_path - - -def create_default_config(config_dir, detect_location=True): - """Create a default configuration file in given configuration directory. - - Return path to new config file if success, None if failed. - This method needs to run in an executor. - """ - config_path = os.path.join(config_dir, YAML_CONFIG_FILE) - version_path = os.path.join(config_dir, VERSION_FILE) - - info = {attr: default for attr, default, _, _ in DEFAULT_CORE_CONFIG} - - location_info = detect_location and loc_utility.detect_location_info() - - if location_info: - if location_info.use_metric: - info[CONF_UNIT_SYSTEM] = CONF_UNIT_SYSTEM_METRIC - else: - info[CONF_UNIT_SYSTEM] = CONF_UNIT_SYSTEM_IMPERIAL - - for attr, default, prop, _ in DEFAULT_CORE_CONFIG: - if prop is None: - continue - info[attr] = getattr(location_info, prop) or default - - if location_info.latitude and location_info.longitude: - info[CONF_ELEVATION] = loc_utility.elevation( - location_info.latitude, location_info.longitude - ) - - # Writing files with YAML does not create the most human readable results - # So we're hard coding a YAML template. - try: - with open(config_path, "w") as config_file: - config_file.write("scarlett_os:\n") - - for attr, _, _, description in DEFAULT_CORE_CONFIG: - if info[attr] is None: - continue - elif description: - config_file.write(" # {}\n".format(description)) - config_file.write(" {}: {}\n".format(attr, info[attr])) - - config_file.write(DEFAULT_CONFIG) - - with open(version_path, "wt") as version_file: - version_file.write(__version__) - - return config_path - - except IOError: - print("Unable to create default configuration file", config_path) - return None - - -def find_config_file(config_dir): - """Look in given directory for supported configuration files. - - Async friendly. - """ - config_path = os.path.join(config_dir, YAML_CONFIG_FILE) - - return config_path if os.path.isfile(config_path) else None - - -def load_yaml_config_file(config_path): - """Parse a YAML configuration file. - - This method needs to run in an executor. - """ - conf_dict = load_yaml(config_path) - - if not isinstance(conf_dict, dict): - msg = "The configuration file {} does not contain a dictionary".format( - os.path.basename(config_path) - ) - logger.error(msg) - raise ScarlettError(msg) - - return conf_dict +# NOTE: YAGNI 8/5/2018 +# DEFAULT_CORE_CONFIG = ( +# # Tuples (attribute, default, auto detect property, description) +# (CONF_NAME, "Home", None, "Name of the location where ScarlettOS is " "running"), +# ( +# CONF_LATITUDE, +# 0, +# "latitude", +# "Location required to calculate the time" " the sun rises and sets", +# ), +# (CONF_LONGITUDE, 0, "longitude", None), +# ( +# CONF_ELEVATION, +# 0, +# None, +# "Impacts weather/sunrise data" " (altitude above sea level in meters)", +# ), +# ( +# CONF_UNIT_SYSTEM, +# CONF_UNIT_SYSTEM_METRIC, +# None, +# "{} for Metric, {} for Imperial".format( +# CONF_UNIT_SYSTEM_METRIC, CONF_UNIT_SYSTEM_IMPERIAL +# ), +# ), +# ( +# CONF_TIME_ZONE, +# "UTC", +# "time_zone", +# "Pick yours from here: http://en.wiki" +# "pedia.org/wiki/List_of_tz_database_time_zones", +# ), +# ) # type: Tuple[Tuple[str, Any, Any, str], ...] +# DEFAULT_CONFIG = """ +# # Show links to resources in log and frontend +# introduction: + +# # Enables the frontend +# frontend: + +# http: +# # Uncomment this to add a password (recommended!) +# # api_password: PASSWORD + +# # Checks for available updates +# updater: + +# # Discover some devices automatically +# discovery: + +# # Allows you to issue voice commands from the frontend in enabled browsers +# conversation: + +# # Enables support for tracking state changes over time. +# history: + +# # View all events in a logbook +# logbook: + +# # Track the sun +# sun: + +# # Weather Prediction +# sensor: +# platform: yr +# """ + +# NOTE: YAGNI 8/5/2018 +# def _valid_customize(value): +# """Config validator for customize.""" +# if not isinstance(value, dict): +# raise vol.Invalid("Expected dictionary") + +# for key, val in value.items(): +# if not valid_entity_id(key): +# raise vol.Invalid("Invalid entity ID: {}".format(key)) + +# if not isinstance(val, dict): +# raise vol.Invalid("Value of {} is not a dictionary".format(key)) + +# return value + +# NOTE: YAGNI 8/5/2018 +# CORE_CONFIG_SCHEMA = vol.Schema( +# { +# CONF_NAME: vol.Coerce(str), +# CONF_LATITUDE: cv.latitude, +# CONF_LONGITUDE: cv.longitude, +# CONF_ELEVATION: vol.Coerce(int), +# vol.Optional(CONF_TEMPERATURE_UNIT): cv.temperature_unit, +# CONF_UNIT_SYSTEM: cv.unit_system, +# CONF_TIME_ZONE: cv.time_zone, +# vol.Required(CONF_CUSTOMIZE, default=MappingProxyType({})): _valid_customize, +# } +# ) + +# NOTE: YAGNI 8/5/2018 +# def get_default_config_dir() -> str: +# """Put together the default configuration directory based on OS.""" +# data_dir = os.getenv("APPDATA") if os.name == "nt" else os.path.expanduser("~") +# return os.path.join(data_dir, CONFIG_DIR_NAME) + +# NOTE: YAGNI 8/5/2018 +# def ensure_config_exists(config_dir: str, detect_location: bool = True) -> str: +# """Ensure a config file exists in given configuration directory. + +# Creating a default one if needed. +# Return path to the config file. +# """ +# config_path = find_config_file(config_dir) + +# if config_path is None: +# print("Unable to find configuration. Creating default one in", config_dir) +# config_path = create_default_config(config_dir, detect_location) + +# return config_path + +# NOTE: YAGNI 8/5/2018 +# def create_default_config(config_dir, detect_location=True): +# """Create a default configuration file in given configuration directory. + +# Return path to new config file if success, None if failed. +# This method needs to run in an executor. +# """ +# config_path = os.path.join(config_dir, YAML_CONFIG_FILE) +# version_path = os.path.join(config_dir, VERSION_FILE) + +# info = {attr: default for attr, default, _, _ in DEFAULT_CORE_CONFIG} + +# location_info = detect_location and loc_utility.detect_location_info() + +# if location_info: +# if location_info.use_metric: +# info[CONF_UNIT_SYSTEM] = CONF_UNIT_SYSTEM_METRIC +# else: +# info[CONF_UNIT_SYSTEM] = CONF_UNIT_SYSTEM_IMPERIAL + +# for attr, default, prop, _ in DEFAULT_CORE_CONFIG: +# if prop is None: +# continue +# info[attr] = getattr(location_info, prop) or default + +# if location_info.latitude and location_info.longitude: +# info[CONF_ELEVATION] = loc_utility.elevation( +# location_info.latitude, location_info.longitude +# ) + +# # Writing files with YAML does not create the most human readable results +# # So we're hard coding a YAML template. +# try: +# with open(config_path, "w") as config_file: +# config_file.write("scarlett_os:\n") + +# for attr, _, _, description in DEFAULT_CORE_CONFIG: +# if info[attr] is None: +# continue +# elif description: +# config_file.write(" # {}\n".format(description)) +# config_file.write(" {}: {}\n".format(attr, info[attr])) + +# config_file.write(DEFAULT_CONFIG) + +# with open(version_path, "wt") as version_file: +# version_file.write(__version__) + +# return config_path + +# except IOError: +# print("Unable to create default configuration file", config_path) +# return None + +# NOTE: YAGNI 8/5/2018 +# def find_config_file(config_dir): +# """Look in given directory for supported configuration files. + +# Async friendly. +# """ +# config_path = os.path.join(config_dir, YAML_CONFIG_FILE) + +# return config_path if os.path.isfile(config_path) else None + +# NOTE: YAGNI 8/5/2018 +# def load_yaml_config_file(config_path): +# """Parse a YAML configuration file. + +# This method needs to run in an executor. +# """ +# conf_dict = load_yaml(config_path) + +# if not isinstance(conf_dict, dict): +# msg = "The configuration file {} does not contain a dictionary".format( +# os.path.basename(config_path) +# ) +# logger.error(msg) +# raise ScarlettError(msg) +# return conf_dict -def process_ha_config_upgrade(ss): - """Upgrade config if necessary. +# NOTE: YAGNI 8/5/2018 +# def process_ha_config_upgrade(ss): +# """Upgrade config if necessary. - This method needs to run in an executor. - """ - version_path = ss.config.path(VERSION_FILE) +# This method needs to run in an executor. +# """ +# version_path = ss.config.path(VERSION_FILE) - try: - with open(version_path, "rt") as inp: - conf_version = inp.readline().strip() - except FileNotFoundError: - # Last version to not have this file - conf_version = "0.7.7" +# try: +# with open(version_path, "rt") as inp: +# conf_version = inp.readline().strip() +# except FileNotFoundError: +# # Last version to not have this file +# conf_version = "0.7.7" - if conf_version == __version__: - return +# if conf_version == __version__: +# return - logger.info("Upgrading config directory from %s to %s", conf_version, __version__) +# logger.info("Upgrading config directory from %s to %s", conf_version, __version__) - lib_path = ss.config.path("deps") - if os.path.isdir(lib_path): - shutil.rmtree(lib_path) +# lib_path = ss.config.path("deps") +# if os.path.isdir(lib_path): +# shutil.rmtree(lib_path) - with open(version_path, "wt") as outp: - outp.write(__version__) +# with open(version_path, "wt") as outp: +# outp.write(__version__) diff --git a/scarlett_os/internal/debugger.py b/scarlett_os/internal/debugger.py index 9564acca..10801696 100644 --- a/scarlett_os/internal/debugger.py +++ b/scarlett_os/internal/debugger.py @@ -29,15 +29,20 @@ # NOTE: # enable path to dump dot files # this *must* be done before 'import Gst' because it reads the env var on startup. ( Double check if this is still true or not ) -def set_gst_grapviz_tracing(enabled=True): +def set_gst_grapviz_tracing(enabled=True, path_to_dir=None): if enabled: # FIXME: If this breaks, you have to use a string explicitly for it to work. - os.environ[ - "GST_DEBUG_DUMP_DOT_DIR" - ] = "/home/pi/dev/bossjones-github/scarlett_os/_debug" # noqa - os.putenv( - "GST_DEBUG_DUMP_DIR_DIR", "/home/pi/dev/bossjones-github/scarlett_os/_debug" - ) + if path_to_dir: + os.environ["GST_DEBUG_DUMP_DOT_DIR"] = path_to_dir # noqa + os.putenv("GST_DEBUG_DUMP_DIR_DIR", path_to_dir) + else: + os.environ[ + "GST_DEBUG_DUMP_DOT_DIR" + ] = "/home/pi/dev/bossjones-github/scarlett_os/_debug" # noqa + os.putenv( + "GST_DEBUG_DUMP_DIR_DIR", + "/home/pi/dev/bossjones-github/scarlett_os/_debug", + ) else: if os.environ.get("GST_DEBUG_DUMP_DOT_DIR"): del os.environ["GST_DEBUG_DUMP_DOT_DIR"] @@ -210,3 +215,6 @@ def pprint_color(obj): from pprint import pformat print(highlight(pformat(obj), PythonLexer(), Terminal256Formatter())) + + +__all__ = ("pprint_color", "get_pprint", "dump_magic", "dump_color", "dump") diff --git a/scarlett_os/listener.py b/scarlett_os/listener.py index 5ad79a40..4635dc16 100644 --- a/scarlett_os/listener.py +++ b/scarlett_os/listener.py @@ -330,7 +330,7 @@ class ouside the dbus server. # __dr = None __instance = None - def __init__(self, name, *args): + def __init__(self, name, config_manager, *args): threading.Thread.__init__(self) _IdleObject.__init__(self) @@ -339,6 +339,30 @@ def __init__(self, name, *args): self.ready_sem = threading.Semaphore(SEMAPHORE_NUM) self.queue = queue.Queue(QUEUE_SIZE) + # Load in config object, and set default device information + self._config_manager = config_manager + self._graphviz_debug_dir = self._config_manager.cfg["graphviz_debug_dir"] + + self._device = self._config_manager.cfg["pocketsphinx"]["device"] + self._hmm = self._config_manager.cfg["pocketsphinx"]["hmm"] + self._lm = self._config_manager.cfg["pocketsphinx"]["lm"] + self._dic = self._config_manager.cfg["pocketsphinx"]["dict"] + self._fwdflat = bool(self._config_manager.cfg["pocketsphinx"]["fwdflat"]) + self._bestpath = bool(self._config_manager.cfg["pocketsphinx"]["bestpath"]) + self._dsratio = int(self._config_manager.cfg["pocketsphinx"]["dsratio"]) + self._maxhmmpf = int(self._config_manager.cfg["pocketsphinx"]["maxhmmpf"]) + self._bestpath = bool(self._config_manager.cfg["pocketsphinx"]["bestpath"]) + self._silprob = float(self._config_manager.cfg["pocketsphinx"]["silprob"]) + self._wip = float(self._config_manager.cfg["pocketsphinx"]["wip"]) + + # dotfile setup + self._dotfile_listener = os.path.join( + self._graphviz_debug_dir, "generator-listener.dot" + ) + self._pngfile_listener = os.path.join( + self._graphviz_debug_dir, "generator-listener-pipeline.png" + ) + # self._handler = DbusSignalHandler() # Get a dbus proxy and check if theres a service registered called 'org.scarlett.Listener' @@ -492,7 +516,7 @@ def get_pocketsphinx_definition(self, override=False): # TODO: Add audio levels, see the following # SOURCE: http://stackoverflow.com/questions/5686424/detecting-blowing-on-a-microphone-with-gstreamer-or-another-library _gst_launch = [ - "alsasrc device=" + ScarlettListenerI.device, + "alsasrc device=" + self.device, # source: https://github.com/walterbender/story/blob/master/grecord.py # without a buffer here, gstreamer struggles at the start of the # recording and then the A/V sync is bad for the whole video @@ -561,10 +585,13 @@ def _connect_to_dbus(self): def on_debug_activate(self): # FIXME: This needs to use dynamic paths, it's possible that we're having issues because of order of operations # FIXME: STATIC PATH 7/3/2018 - dotfile = ( - "/home/pi/dev/bossjones-github/scarlett_os/_debug/generator-listener.dot" - ) - pngfile = "/home/pi/dev/bossjones-github/scarlett_os/_debug/generator-listener-pipeline.png" # NOQA + # dotfile = ( + # "/home/pi/dev/bossjones-github/scarlett_os/_debug/generator-listener.dot" + # ) + # pngfile = "/home/pi/dev/bossjones-github/scarlett_os/_debug/generator-listener-pipeline.png" # NOQA + dotfile = self._dotfile_listener + pngfile = self._pngfile_listener + if os.access(dotfile, os.F_OK): os.remove(dotfile) if os.access(pngfile, os.F_OK): @@ -664,30 +691,57 @@ def init_gst(self): self.elements_stack.append(appsink) - # get gst pipeline element pocketsphinx and set properties + # ************************************************************ + # get gst pipeline element pocketsphinx and set properties - BEGIN + # ************************************************************ pocketsphinx = pipeline.get_by_name("asr") - if ScarlettListenerI.hmm: - pocketsphinx.set_property("hmm", ScarlettListenerI.hmm) - if ScarlettListenerI.lm: - pocketsphinx.set_property("lm", ScarlettListenerI.lm) - if ScarlettListenerI.dic: - pocketsphinx.set_property("dict", ScarlettListenerI.dic) - - pocketsphinx.set_property( - "fwdflat", True - ) # Enable Flat Lexicon Search | Default: true - pocketsphinx.set_property( - "bestpath", True - ) # Enable Graph Search | Boolean. Default: true - pocketsphinx.set_property( - "dsratio", 1 - ) # Evaluate acoustic model every N frames | Integer. Range: 1 - 10 Default: 1 - pocketsphinx.set_property( - "maxhmmpf", 3000 - ) # Maximum number of HMMs searched per frame | Integer. Range: 1 - 100000 Default: 30000 - pocketsphinx.set_property( - "bestpath", True - ) # Enable Graph Search | Boolean. Default: true + if self._hmm: + pocketsphinx.set_property("hmm", self._hmm) + if self._lm: + pocketsphinx.set_property("lm", self._lm) + if self._dic: + pocketsphinx.set_property("dict", self._dic) + + if self._fwdflat: + pocketsphinx.set_property("fwdflat", self._fwdflat) + + if self._bestpath: + pocketsphinx.set_property("bestpath", self._bestpath) + + if self._dsratio: + pocketsphinx.set_property("dsratio", self._dsratio) + + if self._maxhmmpf: + pocketsphinx.set_property("maxhmmpf", self._maxhmmpf) + + if self._bestpath: + pocketsphinx.set_property("bestpath", self._bestpath) + + if self._silprob: + pocketsphinx.set_property("silprob", self._silprob) + + if self._wip: + pocketsphinx.set_property("wip", self._wip) + # ************************************************************ + # get gst pipeline element pocketsphinx and set properties - END + # ************************************************************ + + # NOTE: Old way of setting pocketsphinx properties. 8/5/2018 + # pocketsphinx.set_property( + # "fwdflat", True + # ) # Enable Flat Lexicon Search | Default: true + # pocketsphinx.set_property( + # "bestpath", True + # ) # Enable Graph Search | Boolean. Default: true + # pocketsphinx.set_property( + # "dsratio", 1 + # ) # Evaluate acoustic model every N frames | Integer. Range: 1 - 10 Default: 1 + # pocketsphinx.set_property( + # "maxhmmpf", 3000 + # ) # Maximum number of HMMs searched per frame | Integer. Range: 1 - 100000 Default: 30000 + # pocketsphinx.set_property( + # "bestpath", True + # ) # Enable Graph Search | Boolean. Default: true # pocketsphinx.set_property('maxwpf', -1) # # pocketsphinx.set_property('maxwpf', 20) # Maximum number of words # searched per frame | Range: 1 - 100000 Default: -1 @@ -941,12 +995,17 @@ def stop_threads(self, *args): def add_thread(self): # NOTE: if we do this via a gobject connect we need def add_thread(self, sender): # make a thread and start it + + # Load in config object, and set default device information + config_manager = ConfigManager() + config_manager.load() name = "Thread #{}".format(random.randint(0, 1000)) self.manager.make_thread( self.thread_finished, # completedCb self.thread_progress, # progressCb ScarlettListenerI, # threadclass name, + config_manager, ) # args[1] <- verify that this is value is correct def thread_finished(self, thread): diff --git a/scarlett_os/scripts/cli.py b/scarlett_os/scripts/cli.py index db08e02a..fcd9cb1e 100644 --- a/scarlett_os/scripts/cli.py +++ b/scarlett_os/scripts/cli.py @@ -9,84 +9,127 @@ import click from click_plugins import with_plugins -from pkg_resources import iter_entry_points + +# from pkg_resources import iter_entry_points import scarlett_os -from scarlett_os.compat import configparser +from scarlett_os.common.configure.ruamel_config import ConfigManager + +# from scarlett_os.compat import configparser + + +# def configure_logging(verbosity): +# log_level = max(10, 30 - 10 * verbosity) +# logging.basicConfig(stream=sys.stderr, level=log_level) + + +# def read_config(cfg): +# parser = configparser.ConfigParser() +# parser.read(cfg) +# rv = {} +# for section in parser.sections(): +# for key, value in parser.items(section): +# rv["{0}.{1}".format(section, key)] = value +# return rv + +# @with_plugins(ep for ep in list(iter_entry_points("scarlett_os.scarlett_os_commands"))) +# @click.group() +# @click.version_option(version=scarlett_os.__version__, message="%(version)s") +# @click.option( +# "--name", +# "-n", +# help="Name ScarlettOS process explicitly.", +# metavar="NAME", +# default="scarlett_system", +# ) +# @click.option( +# "--daemon", +# "-d", +# is_flag=True, +# help="Daemon mode, background process.", +# default=False, +# ) +# @click.option( +# "--mode", +# "-m", +# type=click.Choice(["dbus_server", "listener", "tasker", "check_all_services"]), +# help="ScarlettOS type", +# default="check_all_services", +# ) +# @click.option( +# "--master", +# "-m", +# is_flag=True, +# help="Run ScarlettOS process as a Master", +# default=False, +# ) +# @click.option( +# "--slave", +# "-s", +# is_flag=True, +# help="Run ScarlettOS process as a Slave", +# default=False, +# ) +# @click.option("--etcd-host", help="Etcd Host for distributed mode.", default=False) +# @click.option( +# "--quiet", +# "-q", +# is_flag=True, +# help="Limit output to errors and warnings.", +# default=False, +# ) +# @click.option("--verbose", "-V", is_flag=True, help="Be verbose.", default=False) +# @click.option( +# "--config", +# "-c", +# type=click.Path(exists=True, resolve_path=True), +# help="Config file", +# ) +# @click.pass_context +# def main_group( +# ctx, name, daemon, mode, master, slave, etcd_host, quiet, verbose, config +# ): +# """This is the command line interface to ScarlettOS. +# """ +# # NOTE: ctx +# # Most public functions are actually methods of a 'context' object which +# # is passed as the first parameter (ctx). The context object stores the +# # precision, cached data, and a few other things. It also defines +# # conversions so that the same high-level code can be used for several +# # different base types (mpf, mpfs in Sage, intervals, Python floats) by +# # switching contexts. +# # +# # The default context is called 'mp'. You can call most functions as +# # mpmath.mp.foo(). The top-level function mpmath.foo() is just an alias +# # for this. -def configure_logging(verbosity): - log_level = max(10, 30 - 10 * verbosity) - logging.basicConfig(stream=sys.stderr, level=log_level) +# ctx.obj = {} +# config = config or os.path.join(click.get_app_dir("scarlett_os"), "scarlett_os.ini") +# cfg = read_config(config) +# if cfg: +# ctx.obj["config_file"] = config +# ctx.obj["cfg"] = cfg +# ctx.default_map = cfg +# verbosity = ( +# os.environ.get("SCARLETTOS_VERBOSE") +# or ctx.lookup_default("scarlett_os.verbosity") +# or 0 +# ) +# if verbose or quiet: +# verbosity = verbose - quiet +# verbosity = int(verbosity) +# configure_logging(verbosity) -def read_config(cfg): - parser = configparser.ConfigParser() - parser.read(cfg) - rv = {} - for section in parser.sections(): - for key, value in parser.items(section): - rv["{0}.{1}".format(section, key)] = value - return rv +# ctx.obj["verbosity"] = verbosity -@with_plugins(ep for ep in list(iter_entry_points("scarlett_os.scarlett_os_commands"))) @click.group() @click.version_option(version=scarlett_os.__version__, message="%(version)s") -@click.option( - "--name", - "-n", - help="Name ScarlettOS process explicitly.", - metavar="NAME", - default="scarlett_system", -) -@click.option( - "--daemon", - "-d", - is_flag=True, - help="Daemon mode, background process.", - default=False, -) -@click.option( - "--mode", - "-m", - type=click.Choice(["dbus_server", "listener", "tasker", "check_all_services"]), - help="ScarlettOS type", - default="check_all_services", -) -@click.option( - "--master", - "-m", - is_flag=True, - help="Run ScarlettOS process as a Master", - default=False, -) -@click.option( - "--slave", - "-s", - is_flag=True, - help="Run ScarlettOS process as a Slave", - default=False, -) -@click.option("--etcd-host", help="Etcd Host for distributed mode.", default=False) -@click.option( - "--quiet", - "-q", - is_flag=True, - help="Limit output to errors and warnings.", - default=False, -) -@click.option("--verbose", "-V", is_flag=True, help="Be verbose.", default=False) -@click.option( - "--config", - "-c", - type=click.Path(exists=True, resolve_path=True), - help="Config file", -) +@click.option("--debug/--no-debug", default=False) @click.pass_context -def main_group( - ctx, name, daemon, mode, master, slave, etcd_host, quiet, verbose, config -): +def main_group(ctx, debug): """This is the command line interface to ScarlettOS. """ # NOTE: ctx @@ -100,23 +143,54 @@ def main_group( # The default context is called 'mp'. You can call most functions as # mpmath.mp.foo(). The top-level function mpmath.foo() is just an alias # for this. - ctx.obj = {} - config = config or os.path.join(click.get_app_dir("scarlett_os"), "scarlett_os.ini") - cfg = read_config(config) - if cfg: - ctx.obj["config_file"] = config - ctx.obj["cfg"] = cfg - ctx.default_map = cfg - - verbosity = ( - os.environ.get("SCARLETTOS_VERBOSE") - or ctx.lookup_default("scarlett_os.verbosity") - or 0 + + ctx.obj["DEBUG"] = debug + + click.echo("Debug mode is %s" % ("on" if debug else "off")) + # click.echo('Debug is %s' % (ctx.obj['DEBUG'] and 'on' or 'off')) + + +@click.command() +@click.pass_context +def config_init(ctx): + """ + Setup default configuration file + """ + + cfg = ConfigManager() + cfg.prep_default_config() + click.echo( + "Default configuration created. Please don't forget to update your locations if you don't like the default values!" ) - if verbose or quiet: - verbosity = verbose - quiet - verbosity = int(verbosity) - configure_logging(verbosity) - ctx.obj["verbosity"] = verbosity + +@click.command() +@click.option( + "--mode", + "-m", + type=click.Choice(["dbus_server", "listener", "tasker", "check_all_services"]), + help="ScarlettOS type", + default="check_all_services", +) +@click.pass_context +def run(ctx): + """ + Run a particular Scarlett Module. Options: [dbus_server|listener|tasker|check_all_services] + """ + pass + + +@click.command() +@click.pass_context +def dummy(ctx): + """ + Dummy command, doesn't do anything. + """ + + click.echo("Dummy command, doesn't do anything.") + + +main_group.add_command(config_init) +main_group.add_command(dummy) +main_group.add_command(run) diff --git a/scarlett_os/scripts/config.py b/scarlett_os/scripts/config.py index 41c69a5b..6b3296ec 100644 --- a/scarlett_os/scripts/config.py +++ b/scarlett_os/scripts/config.py @@ -1,37 +1,37 @@ -import os +# import os -import click +# import click -@click.command(short_help="Show all config settings.") -@click.pass_context -def config(ctx): - """Show access token and other configuration settings. +# @click.command(short_help="Show all config settings.") +# @click.pass_context +# def config(ctx): +# """Show access token and other configuration settings. - The access token and command verbosity level can be set on the - command line, as environment variables, and in mapbox.ini config - files. - """ - ctx.default_map = ctx.obj["cfg"] - click.echo("CLI:") - # click.echo("access-token = {0}".format(ctx.obj['access_token'])) - # click.echo("verbosity = {0}".format(ctx.obj['verbosity'])) - click.echo("") +# The access token and command verbosity level can be set on the +# command line, as environment variables, and in mapbox.ini config +# files. +# """ +# ctx.default_map = ctx.obj["cfg"] +# click.echo("CLI:") +# # click.echo("access-token = {0}".format(ctx.obj['access_token'])) +# # click.echo("verbosity = {0}".format(ctx.obj['verbosity'])) +# click.echo("") - click.echo("Environment:") - # if 'MAPBOX_ACCESS_TOKEN' in os.environ: - # click.echo("MAPBOX_ACCESS_TOKEN = {0}".format( - # os.environ['MAPBOX_ACCESS_TOKEN'])) - # if 'MapboxAccessToken' in os.environ: - # click.echo("MapboxAccessToken = {0}".format( - # os.environ['MapboxAccessToken'])) - # if 'MAPBOX_VERBOSE' in os.environ: - # click.echo("MAPBOX_VERBOSE = {0}".format( - # os.environ['MAPBOX_VERBOSE'])) - click.echo("") +# click.echo("Environment:") +# # if 'MAPBOX_ACCESS_TOKEN' in os.environ: +# # click.echo("MAPBOX_ACCESS_TOKEN = {0}".format( +# # os.environ['MAPBOX_ACCESS_TOKEN'])) +# # if 'MapboxAccessToken' in os.environ: +# # click.echo("MapboxAccessToken = {0}".format( +# # os.environ['MapboxAccessToken'])) +# # if 'MAPBOX_VERBOSE' in os.environ: +# # click.echo("MAPBOX_VERBOSE = {0}".format( +# # os.environ['MAPBOX_VERBOSE'])) +# click.echo("") - if "config_file" in ctx.obj: - click.echo("Config file {0}:".format(ctx.obj["config_file"])) - for key, value in ctx.default_map.items(): - click.echo("{0} = {1}".format(key, value)) - click.echo("") +# if "config_file" in ctx.obj: +# click.echo("Config file {0}:".format(ctx.obj["config_file"])) +# for key, value in ctx.default_map.items(): +# click.echo("{0} = {1}".format(key, value)) +# click.echo("") diff --git a/scarlett_os/utility/yaml.py b/scarlett_os/utility/yaml.py index 69eb4c7f..a53e22ae 100644 --- a/scarlett_os/utility/yaml.py +++ b/scarlett_os/utility/yaml.py @@ -1,269 +1,269 @@ -"""YAML utility functions.""" -import logging -import os -import sys -import fnmatch -from collections import OrderedDict -from typing import Union, List, Dict - -import yaml - -try: - import keyring -except ImportError: - keyring = None - -from scarlett_os.exceptions import ScarlettError - -logger = logging.getLogger(__name__) -_SECRET_NAMESPACE = "scarlett_os" -_SECRET_YAML = "secrets.yaml" -__SECRET_CACHE = {} # type: Dict - - -# pylint: disable=too-many-ancestors -class SafeLineLoader(yaml.SafeLoader): - """Loader class that keeps track of line numbers.""" - - def compose_node(self, parent: yaml.nodes.Node, index) -> yaml.nodes.Node: - """Annotate a node with the first line it was seen.""" - last_line = self.line # type: int - node = super(SafeLineLoader, self).compose_node( - parent, index - ) # type: yaml.nodes.Node - node.__line__ = last_line + 1 - return node - - -def load_yaml(fname: str) -> Union[List, Dict]: - """Load a YAML file.""" - try: - with open(fname, encoding="utf-8") as conf_file: - # If configuration file is empty YAML returns None - # We convert that to an empty dict - return yaml.load(conf_file, Loader=SafeLineLoader) or {} - except yaml.YAMLError as exc: - logger.error(exc) - raise ScarlettError(exc) - except UnicodeDecodeError as exc: - logger.error("Unable to read file %s: %s", fname, exc) - raise ScarlettError(exc) - - -# def clear_secret_cache() -> None: -# """Clear the secret cache. -# -# Async friendly. +# """YAML utility functions.""" +# import logging +# import os +# import sys +# import fnmatch +# from collections import OrderedDict +# from typing import Union, List, Dict + +# import yaml + +# try: +# import keyring +# except ImportError: +# keyring = None + +# from scarlett_os.exceptions import ScarlettError + +# logger = logging.getLogger(__name__) +# _SECRET_NAMESPACE = "scarlett_os" +# _SECRET_YAML = "secrets.yaml" +# __SECRET_CACHE = {} # type: Dict + + +# # pylint: disable=too-many-ancestors +# class SafeLineLoader(yaml.SafeLoader): +# """Loader class that keeps track of line numbers.""" + +# def compose_node(self, parent: yaml.nodes.Node, index) -> yaml.nodes.Node: +# """Annotate a node with the first line it was seen.""" +# last_line = self.line # type: int +# node = super(SafeLineLoader, self).compose_node( +# parent, index +# ) # type: yaml.nodes.Node +# node.__line__ = last_line + 1 +# return node + + +# def load_yaml(fname: str) -> Union[List, Dict]: +# """Load a YAML file.""" +# try: +# with open(fname, encoding="utf-8") as conf_file: +# # If configuration file is empty YAML returns None +# # We convert that to an empty dict +# return yaml.load(conf_file, Loader=SafeLineLoader) or {} +# except yaml.YAMLError as exc: +# logger.error(exc) +# raise ScarlettError(exc) +# except UnicodeDecodeError as exc: +# logger.error("Unable to read file %s: %s", fname, exc) +# raise ScarlettError(exc) + + +# # def clear_secret_cache() -> None: +# # """Clear the secret cache. +# # +# # Async friendly. +# # """ +# # __SECRET_CACHE.clear() + + +# def _include_yaml(loader: SafeLineLoader, node: yaml.nodes.Node) -> Union[List, Dict]: +# """Load another YAML file and embeds it using the !include tag. + +# Example: +# device_tracker: !include device_tracker.yaml # """ -# __SECRET_CACHE.clear() - - -def _include_yaml(loader: SafeLineLoader, node: yaml.nodes.Node) -> Union[List, Dict]: - """Load another YAML file and embeds it using the !include tag. - - Example: - device_tracker: !include device_tracker.yaml - """ - fname = os.path.join(os.path.dirname(loader.name), node.value) - return load_yaml(fname) - - -def _is_file_valid(name: str) -> bool: - """Decide if a file is valid.""" - return not name.startswith(".") - - -def _find_files(directory: str, pattern: str): - """Recursively load files in a directory.""" - for root, dirs, files in os.walk(directory, topdown=True): - dirs[:] = [d for d in dirs if _is_file_valid(d)] - for basename in files: - if _is_file_valid(basename) and fnmatch.fnmatch(basename, pattern): - filename = os.path.join(root, basename) - yield filename - - -def _include_dir_named_yaml( - loader: SafeLineLoader, node: yaml.nodes.Node -) -> OrderedDict: - """Load multiple files from directory as a dictionary.""" - mapping = OrderedDict() # type: OrderedDict - loc = os.path.join(os.path.dirname(loader.name), node.value) - for fname in _find_files(loc, "*.yaml"): - filename = os.path.splitext(os.path.basename(fname))[0] - mapping[filename] = load_yaml(fname) - return mapping - - -def _include_dir_merge_named_yaml( - loader: SafeLineLoader, node: yaml.nodes.Node -) -> OrderedDict: - """Load multiple files from directory as a merged dictionary.""" - mapping = OrderedDict() # type: OrderedDict - loc = os.path.join(os.path.dirname(loader.name), node.value) - for fname in _find_files(loc, "*.yaml"): - if os.path.basename(fname) == _SECRET_YAML: - continue - loaded_yaml = load_yaml(fname) - if isinstance(loaded_yaml, dict): - mapping.update(loaded_yaml) - return mapping - - -def _include_dir_list_yaml(loader: SafeLineLoader, node: yaml.nodes.Node): - """Load multiple files from directory as a list.""" - loc = os.path.join(os.path.dirname(loader.name), node.value) - return [ - load_yaml(f) - for f in _find_files(loc, "*.yaml") - if os.path.basename(f) != _SECRET_YAML - ] - - -def _include_dir_merge_list_yaml(loader: SafeLineLoader, node: yaml.nodes.Node): - """Load multiple files from directory as a merged list.""" - loc = os.path.join(os.path.dirname(loader.name), node.value) # type: str - merged_list = [] # type: List - for fname in _find_files(loc, "*.yaml"): - if os.path.basename(fname) == _SECRET_YAML: - continue - loaded_yaml = load_yaml(fname) - if isinstance(loaded_yaml, list): - merged_list.extend(loaded_yaml) - return merged_list - - -def _ordered_dict(loader: SafeLineLoader, node: yaml.nodes.MappingNode) -> OrderedDict: - """Load YAML mappings into an ordered dictionary to preserve key order.""" - loader.flatten_mapping(node) - nodes = loader.construct_pairs(node) - - seen = {} # type: Dict - for (key, _), (child_node, _) in zip(nodes, node.value): - line = child_node.start_mark.line - - try: - hash(key) - except TypeError: - fname = getattr(loader.stream, "name", "") - raise yaml.MarkedYAMLError( - context='invalid key: "{}"'.format(key), - context_mark=yaml.Mark(fname, 0, line, -1, None, None), - ) - - if key in seen: - fname = getattr(loader.stream, "name", "") - first_mark = yaml.Mark(fname, 0, seen[key], -1, None, None) - second_mark = yaml.Mark(fname, 0, line, -1, None, None) - raise yaml.MarkedYAMLError( - context='duplicate key: "{}"'.format(key), - context_mark=first_mark, - problem_mark=second_mark, - ) - seen[key] = line - - processed = OrderedDict(nodes) - setattr(processed, "__config_file__", loader.name) - setattr(processed, "__line__", node.start_mark.line) - return processed - - -def _construct_seq(loader: SafeLineLoader, node: yaml.nodes.Node): - """Add line number and file name to Load YAML sequence.""" - obj, = loader.construct_yaml_seq(node) - - class NodeClass(list): - """Wrapper class to be able to add attributes on a list.""" - - pass - - processed = NodeClass(obj) - setattr(processed, "__config_file__", loader.name) - setattr(processed, "__line__", node.start_mark.line) - return processed - - -def _env_var_yaml(loader: SafeLineLoader, node: yaml.nodes.Node): - """Load environment variables and embed it into the configuration YAML.""" - if node.value in os.environ: - return os.environ[node.value] - else: - logger.error("Environment variable %s not defined.", node.value) - raise ScarlettError(node.value) - - -# pylint: disable=no-member -def _load_secret_yaml(secret_path: str) -> Dict: - """Load the secrets yaml from path.""" - secret_path = os.path.join(secret_path, _SECRET_YAML) - if secret_path in __SECRET_CACHE: - return __SECRET_CACHE[secret_path] - - # FIXME: Getting the error message below which doesn't make sense to me? - logger.debug("Loading %s", secret_path) # pylint: disable=used-before-assignment - try: - secrets = load_yaml(secret_path) - if "logger" in secrets: - logger = str(secrets["logger"]).lower() - if logger == "debug": - logger.setLevel(logging.DEBUG) # pylint: disable=no-member - else: - logger.error( - "secrets.yaml: 'logger: debug' expected," " but 'logger: %s' found", - logger, - ) # pylint: disable=no-member - del secrets["logger"] - except FileNotFoundError: - secrets = {} - __SECRET_CACHE[secret_path] = secrets - return secrets - - -# pylint: disable=protected-access -def _secret_yaml(loader: SafeLineLoader, node: yaml.nodes.Node): - """Load secrets and embed it into the configuration YAML.""" - secret_path = os.path.dirname(loader.name) - while True: - secrets = _load_secret_yaml(secret_path) - - if node.value in secrets: - logger.debug( - "Secret %s retrieved from secrets.yaml in " "folder %s", - node.value, - secret_path, - ) - return secrets[node.value] - - if secret_path == os.path.dirname(sys.path[0]): - break # sys.path[0] set to config/deps folder by bootstrap - - secret_path = os.path.dirname(secret_path) - if not os.path.exists(secret_path) or len(secret_path) < 5: - break # Somehow we got past the .scarlett_os config folder - - if keyring: - # do some keyring stuff - pwd = keyring.get_password(_SECRET_NAMESPACE, node.value) - if pwd: - logger.debug("Secret %s retrieved from keyring.", node.value) - return pwd - - logger.error("Secret %s not defined.", node.value) - raise ScarlettError(node.value) - - -yaml.SafeLoader.add_constructor("!include", _include_yaml) -yaml.SafeLoader.add_constructor( - yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG, _ordered_dict -) -yaml.SafeLoader.add_constructor( - yaml.resolver.BaseResolver.DEFAULT_SEQUENCE_TAG, _construct_seq -) -yaml.SafeLoader.add_constructor("!env_var", _env_var_yaml) -yaml.SafeLoader.add_constructor("!secret", _secret_yaml) -yaml.SafeLoader.add_constructor("!include_dir_list", _include_dir_list_yaml) -yaml.SafeLoader.add_constructor("!include_dir_merge_list", _include_dir_merge_list_yaml) -yaml.SafeLoader.add_constructor("!include_dir_named", _include_dir_named_yaml) -yaml.SafeLoader.add_constructor( - "!include_dir_merge_named", _include_dir_merge_named_yaml -) +# fname = os.path.join(os.path.dirname(loader.name), node.value) +# return load_yaml(fname) + + +# def _is_file_valid(name: str) -> bool: +# """Decide if a file is valid.""" +# return not name.startswith(".") + + +# def _find_files(directory: str, pattern: str): +# """Recursively load files in a directory.""" +# for root, dirs, files in os.walk(directory, topdown=True): +# dirs[:] = [d for d in dirs if _is_file_valid(d)] +# for basename in files: +# if _is_file_valid(basename) and fnmatch.fnmatch(basename, pattern): +# filename = os.path.join(root, basename) +# yield filename + + +# def _include_dir_named_yaml( +# loader: SafeLineLoader, node: yaml.nodes.Node +# ) -> OrderedDict: +# """Load multiple files from directory as a dictionary.""" +# mapping = OrderedDict() # type: OrderedDict +# loc = os.path.join(os.path.dirname(loader.name), node.value) +# for fname in _find_files(loc, "*.yaml"): +# filename = os.path.splitext(os.path.basename(fname))[0] +# mapping[filename] = load_yaml(fname) +# return mapping + + +# def _include_dir_merge_named_yaml( +# loader: SafeLineLoader, node: yaml.nodes.Node +# ) -> OrderedDict: +# """Load multiple files from directory as a merged dictionary.""" +# mapping = OrderedDict() # type: OrderedDict +# loc = os.path.join(os.path.dirname(loader.name), node.value) +# for fname in _find_files(loc, "*.yaml"): +# if os.path.basename(fname) == _SECRET_YAML: +# continue +# loaded_yaml = load_yaml(fname) +# if isinstance(loaded_yaml, dict): +# mapping.update(loaded_yaml) +# return mapping + + +# def _include_dir_list_yaml(loader: SafeLineLoader, node: yaml.nodes.Node): +# """Load multiple files from directory as a list.""" +# loc = os.path.join(os.path.dirname(loader.name), node.value) +# return [ +# load_yaml(f) +# for f in _find_files(loc, "*.yaml") +# if os.path.basename(f) != _SECRET_YAML +# ] + + +# def _include_dir_merge_list_yaml(loader: SafeLineLoader, node: yaml.nodes.Node): +# """Load multiple files from directory as a merged list.""" +# loc = os.path.join(os.path.dirname(loader.name), node.value) # type: str +# merged_list = [] # type: List +# for fname in _find_files(loc, "*.yaml"): +# if os.path.basename(fname) == _SECRET_YAML: +# continue +# loaded_yaml = load_yaml(fname) +# if isinstance(loaded_yaml, list): +# merged_list.extend(loaded_yaml) +# return merged_list + + +# def _ordered_dict(loader: SafeLineLoader, node: yaml.nodes.MappingNode) -> OrderedDict: +# """Load YAML mappings into an ordered dictionary to preserve key order.""" +# loader.flatten_mapping(node) +# nodes = loader.construct_pairs(node) + +# seen = {} # type: Dict +# for (key, _), (child_node, _) in zip(nodes, node.value): +# line = child_node.start_mark.line + +# try: +# hash(key) +# except TypeError: +# fname = getattr(loader.stream, "name", "") +# raise yaml.MarkedYAMLError( +# context='invalid key: "{}"'.format(key), +# context_mark=yaml.Mark(fname, 0, line, -1, None, None), +# ) + +# if key in seen: +# fname = getattr(loader.stream, "name", "") +# first_mark = yaml.Mark(fname, 0, seen[key], -1, None, None) +# second_mark = yaml.Mark(fname, 0, line, -1, None, None) +# raise yaml.MarkedYAMLError( +# context='duplicate key: "{}"'.format(key), +# context_mark=first_mark, +# problem_mark=second_mark, +# ) +# seen[key] = line + +# processed = OrderedDict(nodes) +# setattr(processed, "__config_file__", loader.name) +# setattr(processed, "__line__", node.start_mark.line) +# return processed + + +# def _construct_seq(loader: SafeLineLoader, node: yaml.nodes.Node): +# """Add line number and file name to Load YAML sequence.""" +# obj, = loader.construct_yaml_seq(node) + +# class NodeClass(list): +# """Wrapper class to be able to add attributes on a list.""" + +# pass + +# processed = NodeClass(obj) +# setattr(processed, "__config_file__", loader.name) +# setattr(processed, "__line__", node.start_mark.line) +# return processed + + +# def _env_var_yaml(loader: SafeLineLoader, node: yaml.nodes.Node): +# """Load environment variables and embed it into the configuration YAML.""" +# if node.value in os.environ: +# return os.environ[node.value] +# else: +# logger.error("Environment variable %s not defined.", node.value) +# raise ScarlettError(node.value) + + +# # pylint: disable=no-member +# def _load_secret_yaml(secret_path: str) -> Dict: +# """Load the secrets yaml from path.""" +# secret_path = os.path.join(secret_path, _SECRET_YAML) +# if secret_path in __SECRET_CACHE: +# return __SECRET_CACHE[secret_path] + +# # FIXME: Getting the error message below which doesn't make sense to me? +# logger.debug("Loading %s", secret_path) # pylint: disable=used-before-assignment +# try: +# secrets = load_yaml(secret_path) +# if "logger" in secrets: +# logger = str(secrets["logger"]).lower() +# if logger == "debug": +# logger.setLevel(logging.DEBUG) # pylint: disable=no-member +# else: +# logger.error( +# "secrets.yaml: 'logger: debug' expected," " but 'logger: %s' found", +# logger, +# ) # pylint: disable=no-member +# del secrets["logger"] +# except FileNotFoundError: +# secrets = {} +# __SECRET_CACHE[secret_path] = secrets +# return secrets + + +# # pylint: disable=protected-access +# def _secret_yaml(loader: SafeLineLoader, node: yaml.nodes.Node): +# """Load secrets and embed it into the configuration YAML.""" +# secret_path = os.path.dirname(loader.name) +# while True: +# secrets = _load_secret_yaml(secret_path) + +# if node.value in secrets: +# logger.debug( +# "Secret %s retrieved from secrets.yaml in " "folder %s", +# node.value, +# secret_path, +# ) +# return secrets[node.value] + +# if secret_path == os.path.dirname(sys.path[0]): +# break # sys.path[0] set to config/deps folder by bootstrap + +# secret_path = os.path.dirname(secret_path) +# if not os.path.exists(secret_path) or len(secret_path) < 5: +# break # Somehow we got past the .scarlett_os config folder + +# if keyring: +# # do some keyring stuff +# pwd = keyring.get_password(_SECRET_NAMESPACE, node.value) +# if pwd: +# logger.debug("Secret %s retrieved from keyring.", node.value) +# return pwd + +# logger.error("Secret %s not defined.", node.value) +# raise ScarlettError(node.value) + + +# yaml.SafeLoader.add_constructor("!include", _include_yaml) +# yaml.SafeLoader.add_constructor( +# yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG, _ordered_dict +# ) +# yaml.SafeLoader.add_constructor( +# yaml.resolver.BaseResolver.DEFAULT_SEQUENCE_TAG, _construct_seq +# ) +# yaml.SafeLoader.add_constructor("!env_var", _env_var_yaml) +# yaml.SafeLoader.add_constructor("!secret", _secret_yaml) +# yaml.SafeLoader.add_constructor("!include_dir_list", _include_dir_list_yaml) +# yaml.SafeLoader.add_constructor("!include_dir_merge_list", _include_dir_merge_list_yaml) +# yaml.SafeLoader.add_constructor("!include_dir_named", _include_dir_named_yaml) +# yaml.SafeLoader.add_constructor( +# "!include_dir_merge_named", _include_dir_merge_named_yaml +# ) diff --git a/scripts/check_mock_path_autospec b/scripts/check_mock_path_autospec new file mode 100755 index 00000000..44e7f9d5 --- /dev/null +++ b/scripts/check_mock_path_autospec @@ -0,0 +1,15 @@ +#!/bin/sh +# Run mock_patch_checker on test suite. + +# Stop on errors +set -e + +cd "$(dirname "$0")/.." + +for pyfile in $(find `pwd`/tests -name "*.py" -type f -print); do + echo $pyfile + f=$(basename $pyfile) + echo $f + echo '[cmd] `pwd`/scripts/contrib/mock_patch_checker.py $pyfile' + `pwd`/scripts/contrib/mock_patch_checker.py $pyfile +done diff --git a/scripts/monkeytype_run b/scripts/monkeytype_run new file mode 100755 index 00000000..dc1894c9 --- /dev/null +++ b/scripts/monkeytype_run @@ -0,0 +1,25 @@ +#!/bin/sh +# Run monkeytype on test suite or optionally on a test module or directory. + +# Stop on errors +set -e + +cd "$(dirname "$0")/.." + +command -v pytest >/dev/null 2>&1 || { + echo >&2 "This script requires pytest but it's not installed." \ + "Aborting. Try: pip install pytest"; exit 1; } + +command -v monkeytype >/dev/null 2>&1 || { + echo >&2 "This script requires monkeytype but it's not installed." \ + "Aborting. Try: pip install monkeytype"; exit 1; } + +if [ $# -eq 0 ] + then + echo "Run monkeytype on test suite" + monkeytype run "`command -v pytest`" + exit +fi + +echo "Run monkeytype on tests in $1" +monkeytype run "`command -v pytest`" "$1" diff --git a/setup.py b/setup.py index 2f407aa2..f1bdce6c 100644 --- a/setup.py +++ b/setup.py @@ -142,40 +142,17 @@ def run_tests(self): download_url=DOWNLOAD_URL, packages=["scarlett_os"], package_dir={"scarlett_os": "scarlett_os"}, - # , - # entry_points={ - # 'console_scripts': [ - # 'ss = scarlett_os.__main__:main' - # ] - # }, - # entry_points={ - # 'console_scripts': [ - # 'scarlett_os=scarlett_os.cli:main' - # ] - # }, - # source: mapbox-cli-py entry_points=""" [console_scripts] - scarlett_os=scarlett_os.scripts.cli:main_group - - [scarlett_os.scarlett_os_commands] - config=scarlett_os.scripts.config:config + scarlett-cli=scarlett_os.scripts.cli:main_group """, # geocoding=mapboxcli.scripts.geocoding:geocoding - # directions=mapboxcli.scripts.directions:directions - # distance=mapboxcli.scripts.distance:distance - # mapmatching=mapboxcli.scripts.mapmatching:match - # upload=mapboxcli.scripts.uploads:upload - # staticmap=mapboxcli.scripts.static:staticmap - # surface=mapboxcli.scripts.surface:surface - # dataset=mapboxcli.scripts.datasets:datasets extras_require={ "test": requirements_test, "experimental": requirements_test_experimental, "dev": requirements_dev, }, include_package_data=True, - # data_files= [('data/sounds', ['data/sounds/*.wav'])], install_requires=requirements, license=PROJECT_LICENSE, zip_safe=False, diff --git a/tests/__init__.py b/tests/__init__.py index 2665c7ae..b68296e7 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -73,3 +73,551 @@ def setup(): # TODO: Comment this in after we figure out best way to get some of this working # if not setup(): # raise ImportError("Could not setup testsuite") + +TEST_LM_DATA = """ +Language model created by QuickLM on Sat Jan 2 12:51:32 EST 2016 +Copyright (c) 1996-2010 Carnegie Mellon University and Alexander I. Rudnicky + +The model is in standard ARPA format, designed by Doug Paul while he was at MITRE. + +The code that was used to produce this language model is available in Open Source. +Please visit http://www.speech.cs.cmu.edu/tools/ for more information + +The (fixed) discount mass is 0.5. The backoffs are computed using the ratio method. +This model based on a corpus of 70 sentences and 80 words + +\data\ +ngram 1=80 +ngram 2=155 +ngram 3=128 + +\1-grams: +-0.9301 -0.3010 +-0.9301 -0.2393 +-2.7752 ALL -0.2892 +-2.7752 APPLE -0.2468 +-2.7752 ARE -0.2915 +-2.7752 BACKWARD -0.2468 +-2.4742 BATHROOM -0.2334 +-2.7752 BLUE -0.2468 +-2.7752 BRIGHTER -0.2468 +-2.7752 BUTTON -0.2468 +-2.7752 CANCEL -0.2468 +-1.9971 CHANGE -0.2915 +-2.4742 CHANNEL -0.2944 +-2.7752 CIRCLE -0.3003 +-2.7752 CLOSE -0.2915 +-2.7752 DARKER -0.2468 +-2.1732 DOWN -0.2468 +-2.7752 EIGHT -0.2468 +-2.7752 ENTER -0.2468 +-2.4742 FAMILY -0.2996 +-2.7752 FIVE -0.2468 +-2.7752 FORWARD -0.2468 +-2.7752 FOUR -0.2468 +-2.7752 FRIZZY -0.2468 +-2.4742 GET -0.2988 +-2.7752 GIVE -0.3003 +-2.7752 GO -0.2468 +-2.7752 GREEN -0.2468 +-2.7752 HALLWAY -0.2468 +-2.7752 HBO -0.2468 +-2.7752 IDS -0.2468 +-2.2981 IN -0.2974 +-2.7752 INPUT -0.2468 +-2.7752 IS -0.3003 +-2.7752 IT -0.2468 +-2.2981 LEFT -0.2468 +-2.2981 LIGHT -0.2981 +-1.5711 LIGHTS -0.2393 +-2.7752 ME -0.2988 +-2.7752 MENU -0.2468 +-2.7752 MTV -0.2468 +-2.7752 MUTE -0.2468 +-2.7752 MY -0.2892 +-2.2981 NAMES -0.2459 +-2.7752 NEGATIVE -0.2468 +-2.7752 NINE -0.2468 +-2.7752 OF -0.3003 +-2.7752 OFF -0.2915 +-1.8722 ON -0.2747 +-2.7752 ONE -0.2468 +-2.7752 PAUSE -0.2468 +-2.7752 PLAY -0.2468 +-2.7752 POWER -0.2468 +-2.7752 RECALL -0.2468 +-2.7752 RED -0.2468 +-2.2981 RIGHT -0.2468 +-2.4742 ROOM -0.2334 +-2.7752 SCARLETT -0.2468 +-2.1732 SELECT -0.2922 +-2.7752 SEVEN -0.2468 +-2.7752 SEXY -0.2996 +-2.7752 SHADES -0.2468 +-2.7752 SIX -0.2468 +-2.7752 STOP -0.2468 +-1.6613 THE -0.2832 +-2.7752 THREE -0.2468 +-2.4742 TIME -0.2459 +-2.4742 TO -0.2996 +-2.7752 TOSHIBA -0.2468 +-1.7339 TURN -0.2847 +-2.2981 TV -0.2451 +-2.7752 TWO -0.2468 +-2.0763 UP -0.2468 +-2.4742 VOLUME -0.2944 +-2.7752 WEATHER -0.2468 +-2.4742 WHAT -0.2988 +-2.7752 WHATS -0.2915 +-2.7752 WHITE -0.2468 +-2.7752 WINDOW -0.2892 +-2.7752 ZERO -0.2468 + +\2-grams: +-2.1461 APPLE 0.0000 +-2.1461 BACKWARD 0.0000 +-2.1461 CANCEL 0.0000 +-1.3680 CHANGE 0.0000 +-1.8451 CHANNEL 0.0000 +-2.1461 CIRCLE 0.0000 +-2.1461 CLOSE 0.0000 +-1.8451 DOWN 0.0000 +-2.1461 EIGHT 0.0000 +-2.1461 FIVE 0.0000 +-2.1461 FORWARD 0.0000 +-2.1461 FOUR 0.0000 +-2.1461 FRIZZY 0.0000 +-1.8451 GET 0.0000 +-2.1461 GIVE 0.0000 +-2.1461 GO 0.0000 +-2.1461 INPUT 0.0000 +-1.8451 LEFT 0.0000 +-2.1461 MENU 0.0000 +-2.1461 MUTE 0.0000 +-2.1461 NEGATIVE 0.0000 +-2.1461 NINE 0.0000 +-2.1461 ONE 0.0000 +-2.1461 PAUSE 0.0000 +-2.1461 PLAY 0.0000 +-2.1461 POWER 0.0000 +-2.1461 RECALL 0.0000 +-1.8451 RIGHT 0.0000 +-2.1461 SCARLETT 0.0000 +-1.5441 SELECT 0.0000 +-2.1461 SEVEN 0.0000 +-2.1461 SEXY 0.0000 +-2.1461 SIX 0.0000 +-2.1461 STOP 0.0000 +-2.1461 THREE 0.0000 +-2.1461 TOSHIBA 0.0000 +-1.1047 TURN 0.0000 +-2.1461 TV -0.2218 +-2.1461 TWO 0.0000 +-1.8451 UP 0.0000 +-1.8451 VOLUME 0.0000 +-1.8451 WHAT 0.0000 +-2.1461 WHATS 0.0000 +-2.1461 ZERO 0.0000 +-0.3010 ALL LIGHTS -0.1938 +-0.3010 APPLE -0.3010 +-0.3010 ARE THE -0.2840 +-0.3010 BACKWARD -0.3010 +-0.6021 BATHROOM -0.3010 +-0.6021 BATHROOM LIGHTS -0.1938 +-0.3010 BLUE -0.3010 +-0.3010 BRIGHTER -0.3010 +-0.3010 BUTTON -0.3010 +-0.3010 CANCEL -0.3010 +-0.3010 CHANGE THE -0.0902 +-0.6021 CHANNEL DOWN 0.0000 +-0.6021 CHANNEL UP 0.0000 +-0.3010 CIRCLE BUTTON 0.0000 +-0.3010 CLOSE THE -0.2840 +-0.3010 DARKER -0.3010 +-0.3010 DOWN -0.3010 +-0.3010 EIGHT -0.3010 +-0.3010 ENTER -0.3010 +-0.3010 FAMILY ROOM 0.0000 +-0.3010 FIVE -0.3010 +-0.3010 FORWARD -0.3010 +-0.3010 FOUR -0.3010 +-0.3010 FRIZZY -0.3010 +-0.3010 GET LIGHT 0.0000 +-0.3010 GIVE ME 0.0000 +-0.3010 GO -0.3010 +-0.3010 GREEN -0.3010 +-0.3010 HALLWAY -0.3010 +-0.3010 HBO -0.3010 +-0.3010 IDS -0.3010 +-0.7782 IN BATHROOM -0.1761 +-0.7782 IN FAMILY 0.0000 +-0.7782 IN HALLWAY 0.0000 +-0.3010 INPUT -0.3010 +-0.3010 IS IT 0.0000 +-0.3010 IT -0.3010 +-0.3010 LEFT -0.3010 +-0.7782 LIGHT IDS 0.0000 +-0.4771 LIGHT NAMES -0.1249 +-0.6601 LIGHTS -0.3010 +-1.5051 LIGHTS BLUE 0.0000 +-1.5051 LIGHTS BRIGHTER 0.0000 +-1.5051 LIGHTS DARKER 0.0000 +-1.5051 LIGHTS GREEN 0.0000 +-1.0280 LIGHTS IN 0.0000 +-1.5051 LIGHTS RED 0.0000 +-1.5051 LIGHTS WHITE 0.0000 +-0.3010 ME LIGHT -0.1249 +-0.3010 MENU -0.3010 +-0.3010 MTV -0.3010 +-0.3010 MUTE -0.3010 +-0.3010 MY LIGHTS -0.1938 +-0.4771 NAMES -0.3010 +-0.7782 NAMES OF 0.0000 +-0.3010 NEGATIVE -0.3010 +-0.3010 NINE -0.3010 +-0.3010 OF MY 0.0000 +-0.3010 OFF THE -0.1413 +-1.2041 ON ALL 0.0000 +-1.2041 ON BATHROOM -0.1761 +-1.2041 ON FAMILY 0.0000 +-0.7270 ON LIGHTS -0.2583 +-1.2041 ON THE -0.1413 +-1.2041 ON WINDOW 0.0000 +-0.3010 ONE -0.3010 +-0.3010 PAUSE -0.3010 +-0.3010 PLAY -0.3010 +-0.3010 POWER -0.3010 +-0.3010 RECALL -0.3010 +-0.3010 RED -0.3010 +-0.3010 RIGHT -0.3010 +-0.6021 ROOM -0.3010 +-0.6021 ROOM LIGHTS -0.1938 +-0.3010 SCARLETT -0.3010 +-0.9031 SELECT ENTER 0.0000 +-0.9031 SELECT LEFT 0.0000 +-0.9031 SELECT RIGHT 0.0000 +-0.9031 SELECT UP 0.0000 +-0.3010 SEVEN -0.3010 +-0.3010 SEXY TIME -0.1761 +-0.3010 SHADES -0.3010 +-0.3010 SIX -0.3010 +-0.3010 STOP -0.3010 +-0.5119 THE LIGHTS -0.0746 +-1.4150 THE NAMES -0.2218 +-1.4150 THE SHADES 0.0000 +-1.1139 THE TV -0.1249 +-1.4150 THE WEATHER 0.0000 +-0.3010 THREE -0.3010 +-0.6021 TIME -0.3010 +-0.6021 TIME IS 0.0000 +-0.6021 TO HBO 0.0000 +-0.6021 TO MTV 0.0000 +-0.3010 TOSHIBA -0.3010 +-1.3424 TURN OFF 0.0000 +-0.4393 TURN ON 0.0000 +-1.0414 TURN THE -0.1413 +-0.7782 TV -0.3010 +-0.4771 TV TO 0.0000 +-0.3010 TWO -0.3010 +-0.3010 UP -0.3010 +-0.6021 VOLUME DOWN 0.0000 +-0.6021 VOLUME UP 0.0000 +-0.3010 WEATHER -0.3010 +-0.6021 WHAT ARE 0.0000 +-0.6021 WHAT TIME -0.1761 +-0.3010 WHATS THE -0.2840 +-0.3010 WHITE -0.3010 +-0.3010 WINDOW LIGHTS -0.1938 +-0.3010 ZERO -0.3010 + +\3-grams: +-0.3010 APPLE +-0.3010 BACKWARD +-0.3010 CANCEL +-0.3010 CHANGE THE +-0.6021 CHANNEL DOWN +-0.6021 CHANNEL UP +-0.3010 CIRCLE BUTTON +-0.3010 CLOSE THE +-0.3010 DOWN +-0.3010 EIGHT +-0.3010 FIVE +-0.3010 FORWARD +-0.3010 FOUR +-0.3010 FRIZZY +-0.3010 GET LIGHT +-0.3010 GIVE ME +-0.3010 GO +-0.3010 INPUT +-0.3010 LEFT +-0.3010 MENU +-0.3010 MUTE +-0.3010 NEGATIVE +-0.3010 NINE +-0.3010 ONE +-0.3010 PAUSE +-0.3010 PLAY +-0.3010 POWER +-0.3010 RECALL +-0.3010 RIGHT +-0.3010 SCARLETT +-0.9031 SELECT ENTER +-0.9031 SELECT LEFT +-0.9031 SELECT RIGHT +-0.9031 SELECT UP +-0.3010 SEVEN +-0.3010 SEXY TIME +-0.3010 SIX +-0.3010 STOP +-0.3010 THREE +-0.3010 TOSHIBA +-1.3424 TURN OFF +-0.4393 TURN ON +-1.0414 TURN THE +-0.3010 TV +-0.3010 TWO +-0.3010 UP +-0.6021 VOLUME DOWN +-0.6021 VOLUME UP +-0.6021 WHAT ARE +-0.6021 WHAT TIME +-0.3010 WHATS THE +-0.3010 ZERO +-0.3010 ALL LIGHTS +-0.3010 ARE THE NAMES +-0.3010 BATHROOM LIGHTS +-0.4771 CHANGE THE LIGHTS +-0.7782 CHANGE THE TV +-0.3010 CHANNEL DOWN +-0.3010 CHANNEL UP +-0.3010 CIRCLE BUTTON +-0.3010 CLOSE THE SHADES +-0.6021 FAMILY ROOM +-0.6021 FAMILY ROOM LIGHTS +-0.6021 GET LIGHT IDS +-0.6021 GET LIGHT NAMES +-0.3010 GIVE ME LIGHT +-0.3010 IN BATHROOM +-0.3010 IN FAMILY ROOM +-0.3010 IN HALLWAY +-0.3010 IS IT +-0.3010 LIGHT IDS +-0.3010 LIGHT NAMES +-0.3010 LIGHTS BLUE +-0.3010 LIGHTS BRIGHTER +-0.3010 LIGHTS DARKER +-0.3010 LIGHTS GREEN +-0.7782 LIGHTS IN BATHROOM +-0.7782 LIGHTS IN FAMILY +-0.7782 LIGHTS IN HALLWAY +-0.3010 LIGHTS RED +-0.3010 LIGHTS WHITE +-0.3010 ME LIGHT NAMES +-0.3010 MY LIGHTS +-0.3010 NAMES OF MY +-0.3010 OF MY LIGHTS +-0.3010 OFF THE LIGHTS +-0.3010 ON ALL LIGHTS +-0.3010 ON BATHROOM LIGHTS +-0.3010 ON FAMILY ROOM +-0.3010 ON LIGHTS IN +-0.3010 ON THE LIGHTS +-0.3010 ON WINDOW LIGHTS +-0.3010 ROOM LIGHTS +-0.3010 SELECT ENTER +-0.3010 SELECT LEFT +-0.3010 SELECT RIGHT +-0.3010 SELECT UP +-0.3010 SEXY TIME +-0.9031 THE LIGHTS +-1.2041 THE LIGHTS BLUE +-1.2041 THE LIGHTS BRIGHTER +-1.2041 THE LIGHTS DARKER +-1.2041 THE LIGHTS GREEN +-1.2041 THE LIGHTS RED +-1.2041 THE LIGHTS WHITE +-0.3010 THE NAMES OF +-0.3010 THE SHADES +-0.3010 THE TV TO +-0.3010 THE WEATHER +-0.3010 TIME IS IT +-0.3010 TO HBO +-0.3010 TO MTV +-0.3010 TURN OFF THE +-1.2041 TURN ON ALL +-1.2041 TURN ON BATHROOM +-1.2041 TURN ON FAMILY +-0.7270 TURN ON LIGHTS +-1.2041 TURN ON THE +-1.2041 TURN ON WINDOW +-0.3010 TURN THE LIGHTS +-0.6021 TV TO HBO +-0.6021 TV TO MTV +-0.3010 VOLUME DOWN +-0.3010 VOLUME UP +-0.3010 WHAT ARE THE +-0.3010 WHAT TIME IS +-0.3010 WHATS THE WEATHER +-0.3010 WINDOW LIGHTS + +\end\ +""" + + +TEST_DIC_DATA = """ +ALL AO L +APPLE AE P AH L +ARE AA R +ARE(2) ER +BACKWARD B AE K W ER D +BATHROOM B AE TH R UW M +BLUE B L UW +BRIGHTER B R AY T ER +BUTTON B AH T AH N +CANCEL K AE N S AH L +CHANGE CH EY N JH +CHANNEL CH AE N AH L +CIRCLE S ER K AH L +CLOSE K L OW S +CLOSE(2) K L OW Z +DARKER D AA R K ER +DOWN D AW N +EIGHT EY T +ENTER EH N T ER +ENTER(2) EH N ER +FAMILY F AE M AH L IY +FAMILY(2) F AE M L IY +FIVE F AY V +FORWARD F AO R W ER D +FOUR F AO R +FRIZZY F R IH Z IY +GET G EH T +GET(2) G IH T +GIVE G IH V +GO G OW +GREEN G R IY N +HALLWAY HH AO L W EY +HBO EY CH B IY OW +IDS AY D IY Z +IDS(2) IH D Z +IN IH N +INPUT IH N P UH T +IS IH Z +IT IH T +LEFT L EH F T +LIGHT L AY T +LIGHTS L AY T S +ME M IY +MENU M EH N Y UW +MTV EH M T IY V IY +MUTE M Y UW T +MY M AY +NAMES N EY M Z +NEGATIVE N EH G AH T IH V +NINE N AY N +OF AH V +OFF AO F +ON AA N +ON(2) AO N +ONE W AH N +ONE(2) HH W AH N +PAUSE P AO Z +PLAY P L EY +POWER P AW ER +RECALL R IY K AO L +RECALL(2) R IH K AO L +RED R EH D +RIGHT R AY T +ROOM R UW M +SCARLETT S K AA R L IH T +SELECT S AH L EH K T +SEVEN S EH V AH N +SEXY S EH K S IY +SHADES SH EY D Z +SIX S IH K S +STOP S T AA P +THE DH AH +THE(2) DH IY +THREE TH R IY +TIME T AY M +TO T UW +TO(2) T IH +TO(3) T AH +TOSHIBA T OW SH IY B AH +TURN T ER N +TV T IY V IY +TV(2) T EH L AH V IH ZH AH N +TWO T UW +UP AH P +VOLUME V AA L Y UW M +WEATHER W EH DH ER +WHAT W AH T +WHAT(2) HH W AH T +WHATS W AH T S +WHATS(2) HH W AH T S +WHITE W AY T +WHITE(2) HH W AY T +WINDOW W IH N D OW +ZERO Z IY R OW +""" + + +COMMON_MOCKED_CONFIG = """ +# Omitted values in this section will be auto detected using freegeoip.io + +# Location required to calculate the time the sun rises and sets. +# Coordinates are also used for location for weather related automations. +# Google Maps can be used to determine more precise GPS coordinates. +latitude: 40.7056308 +longitude: -73.9780034 + +pocketsphinx: + hmm: /home/pi/.virtualenvs/scarlett_os/share/pocketsphinx/model/en-us/en-us + lm: /home/pi/dev/bossjones-github/scarlett_os/static/speech/lm/1473.lm + dict: /home/pi/dev/bossjones-github/scarlett_os/static/speech/dict/1473.dic + # Silence word transition probability + silprob: 0.1 + # ******************************************************** + # FIXME: ????? THIS IS THE ORIG VALUE, do we need too set it back? 8/5/2018 # wip: 1e-4 + # Enable Graph Search | Boolean. Default: true + # ******************************************************** + # Word insertion penalty + wip: 0.0001 + device: plughw:CARD=Device,DEV=0 + # ******************************************************** + # FIXME: ????? THIS IS THE ORIG VALUE, do we need too set it back? 8/5/2018 # bestpath: 0 + # Enable Graph Search | Boolean. Default: true + # ******************************************************** + bestpath: True + # Enable Flat Lexicon Search | Default: true + fwdflat: True + # Evaluate acoustic model every N frames | Integer. Range: 1 - 10 Default: 1 + dsratio: 1 + # Maximum number of HMMs searched per frame | Integer. Range: 1 - 100000 Default: 30000 + maxhmmpf: 3000 + + +# Impacts weather/sunrise data +elevation: 665 + +# 'metric' for Metric System, 'imperial' for imperial system +unit_system: metric + +# Pick yours from here: +# http://en.wikipedia.org/wiki/List_of_tz_database_time_zones +time_zone: America/New_York + +# Name of the location where ScarlettOS Assistant is running +name: ScarlettOS + +owner: "Hair Ron Jones" + +keywords_list: +- 'scarlett' +- 'SCARLETT' + +features: +- time + +graphviz_debug_dir: /home/pi/dev/bossjones-github/scarlett_os/_debug +""" diff --git a/tests/conftest.py b/tests/conftest.py index bb4cc0cd..1f2d145e 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -18,7 +18,10 @@ import pytest from _pytest.monkeypatch import MonkeyPatch -from tests import PROJECT_ROOT +from tests import PROJECT_ROOT, TEST_LM_DATA, TEST_DIC_DATA, COMMON_MOCKED_CONFIG + +from ruamel.yaml import YAML +import shutil """ Component test fixtures. This module makes the following assumptions: @@ -99,6 +102,77 @@ def get_environment(): yield setup_environment() +@pytest.fixture(scope="function") +def fake_temp_data_pocketsphinx_lm(tmpdir_factory): + """Create a temporary config file.""" + lm_data = TEST_LM_DATA + # you should start with making a directory + # a_dir acts like the object returned from the tmpdir fixture + a_dir = tmpdir_factory.mktemp("fake_data_pocketsphinx_lm") + + # base_temp will be the parent dir of 'mydir' + # you don't have to use getbasetemp() + # using it here just to show that it's available + base_temp = tmpdir_factory.getbasetemp() + print("base:", base_temp) + a_file = a_dir.join("fake.lm") + print("file:{}".format(str(a_file))) + with a_file.open("wt") as f: + f.write(lm_data) + + return a_file + + +@pytest.fixture(scope="function") +def fake_temp_data_pocketsphinx_dic(tmpdir_factory): + """Create a temporary config file.""" + dic_data = TEST_DIC_DATA + # you should start with making a directory + # a_dir acts like the object returned from the tmpdir fixture + a_dir = tmpdir_factory.mktemp("fake_data_pocketsphinx_dic") + + # base_temp will be the parent dir of 'mydir' + # you don't have to use getbasetemp() + # using it here just to show that it's available + base_temp = tmpdir_factory.getbasetemp() + print("base:", base_temp) + a_file = a_dir.join("fake.dic") + print("file:{}".format(str(a_file))) + with a_file.open("wt") as f: + f.write(dic_data) + + return a_file + + +@pytest.fixture(scope="function") +def mocked_config_file_path( + fake_temp_data_pocketsphinx_dic, fake_temp_data_pocketsphinx_lm, tmpdir_factory +): + path_to_pocketsphix_dic = os.path.join( + str(fake_temp_data_pocketsphinx_dic), "fake.dic" + ) + path_to_pocketsphix_lm = os.path.join( + str(fake_temp_data_pocketsphinx_lm), "fake.lm" + ) + # config part + base = tempfile.mkdtemp() + config_file = os.path.join(base, "config.yaml") + + yaml = YAML() + + m_cfg = yaml.load(COMMON_MOCKED_CONFIG) + m_cfg["pocketsphinx"]["dic"] = path_to_pocketsphix_dic + m_cfg["pocketsphinx"]["lm"] = path_to_pocketsphix_lm + + with open(config_file, "w", encoding="utf-8") as fp: + yaml.dump(m_cfg, fp) + + yield config_file + + shutil.rmtree(base) + + +# TEST_DICT_DATA ######################################################################## # NOTE: unix sockets, abstract ( eg 'unix:abstract=' ) ######################################################################## diff --git a/tests/integrationtests/test_integration_listener.py b/tests/integrationtests/test_integration_listener.py index ec3e8646..ec9cbfd5 100644 --- a/tests/integrationtests/test_integration_listener.py +++ b/tests/integrationtests/test_integration_listener.py @@ -41,6 +41,7 @@ from scarlett_os import listener from scarlett_os.utility import threadmanager +from scarlett_os.common.configure.ruamel_config import ConfigManager ########################################### @@ -212,10 +213,18 @@ def quit(*args): # GLib.timeout_add_seconds(10, quit) +# fake_temp_data_pocketsphinx_dic +# fake_temp_data_pocketsphinx_lm + + class TestScarlettListener(object): - def test_ScarlettListenerI_init(self, listener_mocker_stopall): + def test_ScarlettListenerI_init( + self, listener_mocker_stopall, mocked_config_file_path + ): + fake_config_manager = ConfigManager(config_path=mocked_config_file_path) + fake_config_manager.load() - sl = listener.ScarlettListenerI("scarlett_listener") + sl = listener.ScarlettListenerI("scarlett_listener", fake_config_manager) assert sl.running is False assert sl.finished is False diff --git a/tests/unittests/test_cli.py b/tests/unittests/test_cli.py index 9a53ae64..7dc42bb0 100644 --- a/tests/unittests/test_cli.py +++ b/tests/unittests/test_cli.py @@ -27,14 +27,18 @@ pp = pprint.PrettyPrinter(indent=4) +@pytest.fixture +def runner(): + return CliRunner() + + # pylint: disable=C0111 # pylint: disable=R0201 # pylint: disable=C0103 @pytest.mark.scarlettonly @pytest.mark.scarlettonlyunittest class TestScarlettCli(object): - def test_command_line_interface_help(self): - runner = CliRunner() + def test_command_line_interface_help(self, runner): result = runner.invoke(main_group) assert result.exit_code == 0 assert "main_group" in result.output @@ -42,4 +46,15 @@ def test_command_line_interface_help(self): help_result = runner.invoke(main_group, ["--help"]) assert help_result.exit_code == 0 print(help_result.output) - assert "dbus_server|listener|tasker|check_all_services" in help_result.output + + +# pylint: disable=C0111 +# pylint: disable=R0201 +# pylint: disable=C0103 +@pytest.mark.scarlettonly +@pytest.mark.scarlettonlyunittest +def test_cli_dummy(runner): + result = runner.invoke(main_group, ["dummy"]) + assert result.exit_code == 0 + assert not result.exception + assert "Dummy command, doesn't do anything" in result.output diff --git a/tests/unittests/utility/test_yaml.py b/tests/unittests/utility/test_yaml.py index affe7399..8b9ac7a8 100644 --- a/tests/unittests/utility/test_yaml.py +++ b/tests/unittests/utility/test_yaml.py @@ -1,267 +1,267 @@ """Test ScarlettOS yaml loader.""" -import io -import os -import unittest -from unittest.mock import patch +# import io +# import os +# import unittest +# from unittest.mock import patch -from scarlett_os.config import YAML_CONFIG_FILE, load_yaml_config_file -from scarlett_os.exceptions import ScarlettError -from scarlett_os.utility import yaml -from tests.common import get_test_config_dir, patch_yaml_files +# from scarlett_os.config import YAML_CONFIG_FILE, load_yaml_config_file +# from scarlett_os.exceptions import ScarlettError +# from scarlett_os.utility import yaml +# from tests.common import get_test_config_dir, patch_yaml_files # FIXME: Convert to pytest # FIXME: 5/10/2017 -class TestYaml(unittest.TestCase): - """Test utility.yaml loader.""" - - # pylint: disable=no-self-use,invalid-name - - def test_simple_list(self): - """Test simple list.""" - conf = "config:\n - simple\n - list" - with io.StringIO(conf) as file: - doc = yaml.yaml.safe_load(file) - assert doc["config"] == ["simple", "list"] - - def test_simple_dict(self): - """Test simple dict.""" - conf = "key: value" - with io.StringIO(conf) as file: - doc = yaml.yaml.safe_load(file) - assert doc["key"] == "value" - - def test_duplicate_key(self): - """Test duplicate dict keys.""" - files = {YAML_CONFIG_FILE: "key: thing1\nkey: thing2"} - with self.assertRaises(ScarlettError): - with patch_yaml_files(files): - load_yaml_config_file(YAML_CONFIG_FILE) - - def test_unhashable_key(self): - """Test an unhasable key.""" - files = {YAML_CONFIG_FILE: "message:\n {{ states.state }}"} - with self.assertRaises(ScarlettError), patch_yaml_files(files): - load_yaml_config_file(YAML_CONFIG_FILE) - - def test_no_key(self): - """Test item without an key.""" - files = {YAML_CONFIG_FILE: "a: a\nnokeyhere"} - with self.assertRaises(ScarlettError), patch_yaml_files(files): - yaml.load_yaml(YAML_CONFIG_FILE) - - def test_enviroment_variable(self): - """Test config file with enviroment variable.""" - os.environ["PASSWORD"] = "secret_password" - conf = "password: !env_var PASSWORD" - with io.StringIO(conf) as file: - doc = yaml.yaml.safe_load(file) - assert doc["password"] == "secret_password" - del os.environ["PASSWORD"] - - def test_invalid_enviroment_variable(self): - """Test config file with no enviroment variable sat.""" - conf = "password: !env_var PASSWORD" - with self.assertRaises(ScarlettError): - with io.StringIO(conf) as file: - yaml.yaml.safe_load(file) - - def test_include_yaml(self): - """Test include yaml.""" - with patch_yaml_files({"test.yaml": "value"}): - conf = "key: !include test.yaml" - with io.StringIO(conf) as file: - doc = yaml.yaml.safe_load(file) - assert doc["key"] == "value" - - @patch("scarlett_os.utility.yaml.os.walk") - def test_include_dir_list(self, mock_walk): - """Test include dir list yaml.""" - mock_walk.return_value = [["/tmp", [], ["one.yaml", "two.yaml"]]] - - with patch_yaml_files({"/tmp/one.yaml": "one", "/tmp/two.yaml": "two"}): - conf = "key: !include_dir_list /tmp" - with io.StringIO(conf) as file: - doc = yaml.yaml.safe_load(file) - assert sorted(doc["key"]) == sorted(["one", "two"]) - - @patch("scarlett_os.utility.yaml.os.walk") - def test_include_dir_list_recursive(self, mock_walk): - """Test include dir recursive list yaml.""" - mock_walk.return_value = [ - ["/tmp", ["tmp2", ".ignore", "ignore"], ["zero.yaml"]], - ["/tmp/tmp2", [], ["one.yaml", "two.yaml"]], - ["/tmp/ignore", [], [".ignore.yaml"]], - ] - - with patch_yaml_files( - { - "/tmp/zero.yaml": "zero", - "/tmp/tmp2/one.yaml": "one", - "/tmp/tmp2/two.yaml": "two", - } - ): - conf = "key: !include_dir_list /tmp" - with io.StringIO(conf) as file: - assert ( - ".ignore" in mock_walk.return_value[0][1] - ), "Expecting .ignore in here" - doc = yaml.yaml.safe_load(file) - assert "tmp2" in mock_walk.return_value[0][1] - assert ".ignore" not in mock_walk.return_value[0][1] - assert sorted(doc["key"]) == sorted(["zero", "one", "two"]) - - @patch("scarlett_os.utility.yaml.os.walk") - def test_include_dir_named(self, mock_walk): - """Test include dir named yaml.""" - mock_walk.return_value = [["/tmp", [], ["first.yaml", "second.yaml"]]] - - with patch_yaml_files({"/tmp/first.yaml": "one", "/tmp/second.yaml": "two"}): - conf = "key: !include_dir_named /tmp" - correct = {"first": "one", "second": "two"} - with io.StringIO(conf) as file: - doc = yaml.yaml.safe_load(file) - assert doc["key"] == correct - - @patch("scarlett_os.utility.yaml.os.walk") - def test_include_dir_named_recursive(self, mock_walk): - """Test include dir named yaml.""" - mock_walk.return_value = [ - ["/tmp", ["tmp2", ".ignore", "ignore"], ["first.yaml"]], - ["/tmp/tmp2", [], ["second.yaml", "third.yaml"]], - ["/tmp/ignore", [], [".ignore.yaml"]], - ] - - with patch_yaml_files( - { - "/tmp/first.yaml": "one", - "/tmp/tmp2/second.yaml": "two", - "/tmp/tmp2/third.yaml": "three", - } - ): - conf = "key: !include_dir_named /tmp" - correct = {"first": "one", "second": "two", "third": "three"} - with io.StringIO(conf) as file: - assert ( - ".ignore" in mock_walk.return_value[0][1] - ), "Expecting .ignore in here" - doc = yaml.yaml.safe_load(file) - assert "tmp2" in mock_walk.return_value[0][1] - assert ".ignore" not in mock_walk.return_value[0][1] - assert doc["key"] == correct - - @patch("scarlett_os.utility.yaml.os.walk") - def test_include_dir_merge_list(self, mock_walk): - """Test include dir merge list yaml.""" - mock_walk.return_value = [["/tmp", [], ["first.yaml", "second.yaml"]]] - - with patch_yaml_files( - {"/tmp/first.yaml": "- one", "/tmp/second.yaml": "- two\n- three"} - ): - conf = "key: !include_dir_merge_list /tmp" - with io.StringIO(conf) as file: - doc = yaml.yaml.safe_load(file) - assert sorted(doc["key"]) == sorted(["one", "two", "three"]) - - @patch("scarlett_os.utility.yaml.os.walk") - def test_include_dir_merge_list_recursive(self, mock_walk): - """Test include dir merge list yaml.""" - mock_walk.return_value = [ - ["/tmp", ["tmp2", ".ignore", "ignore"], ["first.yaml"]], - ["/tmp/tmp2", [], ["second.yaml", "third.yaml"]], - ["/tmp/ignore", [], [".ignore.yaml"]], - ] - - with patch_yaml_files( - { - "/tmp/first.yaml": "- one", - "/tmp/tmp2/second.yaml": "- two", - "/tmp/tmp2/third.yaml": "- three\n- four", - } - ): - conf = "key: !include_dir_merge_list /tmp" - with io.StringIO(conf) as file: - assert ( - ".ignore" in mock_walk.return_value[0][1] - ), "Expecting .ignore in here" - doc = yaml.yaml.safe_load(file) - assert "tmp2" in mock_walk.return_value[0][1] - assert ".ignore" not in mock_walk.return_value[0][1] - assert sorted(doc["key"]) == sorted(["one", "two", "three", "four"]) - - @patch("scarlett_os.utility.yaml.os.walk") - def test_include_dir_merge_named(self, mock_walk): - """Test include dir merge named yaml.""" - mock_walk.return_value = [["/tmp", [], ["first.yaml", "second.yaml"]]] - - with patch_yaml_files( - { - "/tmp/first.yaml": "key1: one", - "/tmp/second.yaml": "key2: two\nkey3: three", - } - ): - conf = "key: !include_dir_merge_named /tmp" - with io.StringIO(conf) as file: - doc = yaml.yaml.safe_load(file) - assert doc["key"] == {"key1": "one", "key2": "two", "key3": "three"} - - @patch("scarlett_os.utility.yaml.os.walk") - def test_include_dir_merge_named_recursive(self, mock_walk): - """Test include dir merge named yaml.""" - mock_walk.return_value = [ - ["/tmp", ["tmp2", ".ignore", "ignore"], ["first.yaml"]], - ["/tmp/tmp2", [], ["second.yaml", "third.yaml"]], - ["/tmp/ignore", [], [".ignore.yaml"]], - ] - - with patch_yaml_files( - { - "/tmp/first.yaml": "key1: one", - "/tmp/tmp2/second.yaml": "key2: two", - "/tmp/tmp2/third.yaml": "key3: three\nkey4: four", - } - ): - conf = "key: !include_dir_merge_named /tmp" - with io.StringIO(conf) as file: - assert ( - ".ignore" in mock_walk.return_value[0][1] - ), "Expecting .ignore in here" - doc = yaml.yaml.safe_load(file) - assert "tmp2" in mock_walk.return_value[0][1] - assert ".ignore" not in mock_walk.return_value[0][1] - assert doc["key"] == { - "key1": "one", - "key2": "two", - "key3": "three", - "key4": "four", - } - - @patch("scarlett_os.utility.yaml.open", create=True) - def test_load_yaml_encoding_error(self, mock_open): - """Test raising a UnicodeDecodeError.""" - mock_open.side_effect = UnicodeDecodeError("", b"", 1, 0, "") - self.assertRaises(ScarlettError, yaml.load_yaml, "test") - - -FILES = {} - - -def load_yaml(fname, string): - """Write a string to file and return the parsed yaml.""" - FILES[fname] = string - with patch_yaml_files(FILES): - return load_yaml_config_file(fname) - - -class FakeKeyring: # pylint: disable=too-few-public-methods - """Fake a keyring class.""" - - def __init__(self, secrets_dict): - """Store keyring dictionary.""" - self._secrets = secrets_dict - - # pylint: disable=protected-access - def get_password(self, domain, name): - """Retrieve password.""" - assert domain == yaml._SECRET_NAMESPACE - return self._secrets.get(name) +# class TestYaml(unittest.TestCase): +# """Test utility.yaml loader.""" + +# # pylint: disable=no-self-use,invalid-name + +# def test_simple_list(self): +# """Test simple list.""" +# conf = "config:\n - simple\n - list" +# with io.StringIO(conf) as file: +# doc = yaml.yaml.safe_load(file) +# assert doc["config"] == ["simple", "list"] + +# def test_simple_dict(self): +# """Test simple dict.""" +# conf = "key: value" +# with io.StringIO(conf) as file: +# doc = yaml.yaml.safe_load(file) +# assert doc["key"] == "value" + +# def test_duplicate_key(self): +# """Test duplicate dict keys.""" +# files = {YAML_CONFIG_FILE: "key: thing1\nkey: thing2"} +# with self.assertRaises(ScarlettError): +# with patch_yaml_files(files): +# load_yaml_config_file(YAML_CONFIG_FILE) + +# # def test_unhashable_key(self): +# # """Test an unhasable key.""" +# # files = {YAML_CONFIG_FILE: "message:\n {{ states.state }}"} +# # with self.assertRaises(ScarlettError), patch_yaml_files(files): +# # load_yaml_config_file(YAML_CONFIG_FILE) + +# def test_no_key(self): +# """Test item without an key.""" +# files = {YAML_CONFIG_FILE: "a: a\nnokeyhere"} +# with self.assertRaises(ScarlettError), patch_yaml_files(files): +# yaml.load_yaml(YAML_CONFIG_FILE) + +# def test_enviroment_variable(self): +# """Test config file with enviroment variable.""" +# os.environ["PASSWORD"] = "secret_password" +# conf = "password: !env_var PASSWORD" +# with io.StringIO(conf) as file: +# doc = yaml.yaml.safe_load(file) +# assert doc["password"] == "secret_password" +# del os.environ["PASSWORD"] + +# def test_invalid_enviroment_variable(self): +# """Test config file with no enviroment variable sat.""" +# conf = "password: !env_var PASSWORD" +# with self.assertRaises(ScarlettError): +# with io.StringIO(conf) as file: +# yaml.yaml.safe_load(file) + +# def test_include_yaml(self): +# """Test include yaml.""" +# with patch_yaml_files({"test.yaml": "value"}): +# conf = "key: !include test.yaml" +# with io.StringIO(conf) as file: +# doc = yaml.yaml.safe_load(file) +# assert doc["key"] == "value" + +# @patch("scarlett_os.utility.yaml.os.walk") +# def test_include_dir_list(self, mock_walk): +# """Test include dir list yaml.""" +# mock_walk.return_value = [["/tmp", [], ["one.yaml", "two.yaml"]]] + +# with patch_yaml_files({"/tmp/one.yaml": "one", "/tmp/two.yaml": "two"}): +# conf = "key: !include_dir_list /tmp" +# with io.StringIO(conf) as file: +# doc = yaml.yaml.safe_load(file) +# assert sorted(doc["key"]) == sorted(["one", "two"]) + +# @patch("scarlett_os.utility.yaml.os.walk") +# def test_include_dir_list_recursive(self, mock_walk): +# """Test include dir recursive list yaml.""" +# mock_walk.return_value = [ +# ["/tmp", ["tmp2", ".ignore", "ignore"], ["zero.yaml"]], +# ["/tmp/tmp2", [], ["one.yaml", "two.yaml"]], +# ["/tmp/ignore", [], [".ignore.yaml"]], +# ] + +# with patch_yaml_files( +# { +# "/tmp/zero.yaml": "zero", +# "/tmp/tmp2/one.yaml": "one", +# "/tmp/tmp2/two.yaml": "two", +# } +# ): +# conf = "key: !include_dir_list /tmp" +# with io.StringIO(conf) as file: +# assert ( +# ".ignore" in mock_walk.return_value[0][1] +# ), "Expecting .ignore in here" +# doc = yaml.yaml.safe_load(file) +# assert "tmp2" in mock_walk.return_value[0][1] +# assert ".ignore" not in mock_walk.return_value[0][1] +# assert sorted(doc["key"]) == sorted(["zero", "one", "two"]) + +# @patch("scarlett_os.utility.yaml.os.walk") +# def test_include_dir_named(self, mock_walk): +# """Test include dir named yaml.""" +# mock_walk.return_value = [["/tmp", [], ["first.yaml", "second.yaml"]]] + +# with patch_yaml_files({"/tmp/first.yaml": "one", "/tmp/second.yaml": "two"}): +# conf = "key: !include_dir_named /tmp" +# correct = {"first": "one", "second": "two"} +# with io.StringIO(conf) as file: +# doc = yaml.yaml.safe_load(file) +# assert doc["key"] == correct + +# @patch("scarlett_os.utility.yaml.os.walk") +# def test_include_dir_named_recursive(self, mock_walk): +# """Test include dir named yaml.""" +# mock_walk.return_value = [ +# ["/tmp", ["tmp2", ".ignore", "ignore"], ["first.yaml"]], +# ["/tmp/tmp2", [], ["second.yaml", "third.yaml"]], +# ["/tmp/ignore", [], [".ignore.yaml"]], +# ] + +# with patch_yaml_files( +# { +# "/tmp/first.yaml": "one", +# "/tmp/tmp2/second.yaml": "two", +# "/tmp/tmp2/third.yaml": "three", +# } +# ): +# conf = "key: !include_dir_named /tmp" +# correct = {"first": "one", "second": "two", "third": "three"} +# with io.StringIO(conf) as file: +# assert ( +# ".ignore" in mock_walk.return_value[0][1] +# ), "Expecting .ignore in here" +# doc = yaml.yaml.safe_load(file) +# assert "tmp2" in mock_walk.return_value[0][1] +# assert ".ignore" not in mock_walk.return_value[0][1] +# assert doc["key"] == correct + +# @patch("scarlett_os.utility.yaml.os.walk") +# def test_include_dir_merge_list(self, mock_walk): +# """Test include dir merge list yaml.""" +# mock_walk.return_value = [["/tmp", [], ["first.yaml", "second.yaml"]]] + +# with patch_yaml_files( +# {"/tmp/first.yaml": "- one", "/tmp/second.yaml": "- two\n- three"} +# ): +# conf = "key: !include_dir_merge_list /tmp" +# with io.StringIO(conf) as file: +# doc = yaml.yaml.safe_load(file) +# assert sorted(doc["key"]) == sorted(["one", "two", "three"]) + +# @patch("scarlett_os.utility.yaml.os.walk") +# def test_include_dir_merge_list_recursive(self, mock_walk): +# """Test include dir merge list yaml.""" +# mock_walk.return_value = [ +# ["/tmp", ["tmp2", ".ignore", "ignore"], ["first.yaml"]], +# ["/tmp/tmp2", [], ["second.yaml", "third.yaml"]], +# ["/tmp/ignore", [], [".ignore.yaml"]], +# ] + +# with patch_yaml_files( +# { +# "/tmp/first.yaml": "- one", +# "/tmp/tmp2/second.yaml": "- two", +# "/tmp/tmp2/third.yaml": "- three\n- four", +# } +# ): +# conf = "key: !include_dir_merge_list /tmp" +# with io.StringIO(conf) as file: +# assert ( +# ".ignore" in mock_walk.return_value[0][1] +# ), "Expecting .ignore in here" +# doc = yaml.yaml.safe_load(file) +# assert "tmp2" in mock_walk.return_value[0][1] +# assert ".ignore" not in mock_walk.return_value[0][1] +# assert sorted(doc["key"]) == sorted(["one", "two", "three", "four"]) + +# @patch("scarlett_os.utility.yaml.os.walk") +# def test_include_dir_merge_named(self, mock_walk): +# """Test include dir merge named yaml.""" +# mock_walk.return_value = [["/tmp", [], ["first.yaml", "second.yaml"]]] + +# with patch_yaml_files( +# { +# "/tmp/first.yaml": "key1: one", +# "/tmp/second.yaml": "key2: two\nkey3: three", +# } +# ): +# conf = "key: !include_dir_merge_named /tmp" +# with io.StringIO(conf) as file: +# doc = yaml.yaml.safe_load(file) +# assert doc["key"] == {"key1": "one", "key2": "two", "key3": "three"} + +# @patch("scarlett_os.utility.yaml.os.walk") +# def test_include_dir_merge_named_recursive(self, mock_walk): +# """Test include dir merge named yaml.""" +# mock_walk.return_value = [ +# ["/tmp", ["tmp2", ".ignore", "ignore"], ["first.yaml"]], +# ["/tmp/tmp2", [], ["second.yaml", "third.yaml"]], +# ["/tmp/ignore", [], [".ignore.yaml"]], +# ] + +# with patch_yaml_files( +# { +# "/tmp/first.yaml": "key1: one", +# "/tmp/tmp2/second.yaml": "key2: two", +# "/tmp/tmp2/third.yaml": "key3: three\nkey4: four", +# } +# ): +# conf = "key: !include_dir_merge_named /tmp" +# with io.StringIO(conf) as file: +# assert ( +# ".ignore" in mock_walk.return_value[0][1] +# ), "Expecting .ignore in here" +# doc = yaml.yaml.safe_load(file) +# assert "tmp2" in mock_walk.return_value[0][1] +# assert ".ignore" not in mock_walk.return_value[0][1] +# assert doc["key"] == { +# "key1": "one", +# "key2": "two", +# "key3": "three", +# "key4": "four", +# } + +# @patch("scarlett_os.utility.yaml.open", create=True) +# def test_load_yaml_encoding_error(self, mock_open): +# """Test raising a UnicodeDecodeError.""" +# mock_open.side_effect = UnicodeDecodeError("", b"", 1, 0, "") +# self.assertRaises(ScarlettError, yaml.load_yaml, "test") + + +# FILES = {} + + +# def load_yaml(fname, string): +# """Write a string to file and return the parsed yaml.""" +# FILES[fname] = string +# with patch_yaml_files(FILES): +# return load_yaml_config_file(fname) + + +# class FakeKeyring: # pylint: disable=too-few-public-methods +# """Fake a keyring class.""" + +# def __init__(self, secrets_dict): +# """Store keyring dictionary.""" +# self._secrets = secrets_dict + +# # pylint: disable=protected-access +# def get_password(self, domain, name): +# """Retrieve password.""" +# assert domain == yaml._SECRET_NAMESPACE +# return self._secrets.get(name)