[enh] settings.yml: add use_default_settings option (2nd version)

Makefile

@@ -266,19 +266,4 @@ test.clean:
 travis.codecov:
 	$(Q)$(PY_ENV_BIN)/python -m pip install codecov
 
-
-# user-settings
-# -------------
-
-PHONY += user-settings.create user-settings.update
-
-user-settings.update:  pyenvinstall
-	$(Q)$(PY_ENV_ACT); pip install ruamel.yaml
-	$(Q)$(PY_ENV_ACT); python utils/update_user_settings.py ${SEARX_SETTINGS_PATH}
-
-user-settings.update.engines:  pyenvinstall
-	$(Q)$(PY_ENV_ACT); pip install ruamel.yaml
-	$(Q)$(PY_ENV_ACT); python utils/update_user_settings.py --add-engines ${SEARX_SETTINGS_PATH}
-
-
-.PHONY: $(PHONY)
+.PHONY: $(PHONY)

docs/admin/settings.rst

@@ -235,68 +235,51 @@ In the following example, the actual settings are the default settings defined i
 
 .. code-block:: yaml
 
-  use_default_settings: true
+  use_default_settings: True
   server:
       secret_key: "uvys6bRhKHUdFF5CqbJonSDSRN8H0sCBziNSrDGNVdpz7IeZhveVart3yvghoKHA"
-  server:
       bind_address: "0.0.0.0"
 
-With ``use_default_settings: True``, each settings can be override in a similar way with one exception, the ``engines`` section:
-
-* If the ``engines`` section is not defined in the user settings, searx uses the engines from the default setttings (the above example).
-* If the ``engines`` section is defined then:
-
-   * searx loads only the engines declare in the user setttings.
-   * searx merges the configuration according to the engine name.
+With ``use_default_settings: True``, each settings can be override in a similar way, the ``engines`` section is merged according to the engine ``name``.
 
-In the following example, only three engines are available. Each engine configuration is merged with the default configuration.
+In this example, searx will load all the engine and the arch linux wiki engine has a :ref:`token<private engines>`:
 
 .. code-block:: yaml
 
-  use_default_settings: true
+  use_default_settings: True
   server:
       secret_key: "uvys6bRhKHUdFF5CqbJonSDSRN8H0sCBziNSrDGNVdpz7IeZhveVart3yvghoKHA"
   engines:
-    - name: wikipedia
-    - name: wikidata
-    - name: ddg definitions
+    - name: arch linux wiki
+      tokens: ['$ecretValue']
 
-Another example where four engines are available. The arch linux wiki engine has a :ref:`token<private engines>`.
+It is possible to remove some engines from the default settings. The following example is similar to the above one, but searx doesn't load the the google engine:
 
 .. code-block:: yaml
 
-  use_default_settings: true
+  use_default_settings:
+      engines:
+         remove:
+           - google
   server:
       secret_key: "uvys6bRhKHUdFF5CqbJonSDSRN8H0sCBziNSrDGNVdpz7IeZhveVart3yvghoKHA"
   engines:
     - name: arch linux wiki
       tokens: ['$ecretValue']
-    - name: wikipedia
-    - name: wikidata
-    - name: ddg definitions
-
-automatic update
-----------------
-
-The following comand creates or updates a minimal user settings (a secret key is defined if it is not already the case):
-
-.. code-block:: sh
-
-  make SEARX_SETTINGS_PATH=/etc/searx/settings.yml user-settings.update
-
-Set ``SEARX_SETTINGS_PATH`` to your user settings path.
 
-As soon the user settings contains an ``engines`` section, it becomes difficult to keep the engine list updated.
-The following command creates or updates the user settings including the ``engines`` section:
+As an alternative, it is possible to specify the engines to keep. In the following example, searx has only two engines:
 
-.. code-block:: sh
-
-  make SEARX_SETTINGS_PATH=/etc/searx/settings.yml user-settings.update.engines
-
-After that ``/etc/searx/settings.yml``
-
-* has a ``secret key``
-* has a ``engine`` section if it is not already the case, moreover the command:
+.. code-block:: yaml
 
-  * has deleted engines that do not exist in the default settings.
-  * has added engines that exist in the default settings but are not declare in the user settings.
+  use_default_settings:
+      engines:
+         keep_only:
+           - google
+           - duckduckgo
+  server:
+      secret_key: "uvys6bRhKHUdFF5CqbJonSDSRN8H0sCBziNSrDGNVdpz7IeZhveVart3yvghoKHA"
+  engines:
+    - name: google
+      tokens: ['$ecretValue']
+    - name: duckduckgo
+      tokens: ['$ecretValue']

searx/__init__.py

@@ -16,15 +16,15 @@
 '''
 
 import logging
-import searx.settings
+import searx.settings_loader
 from os import environ
 from os.path import realpath, dirname, join, abspath, isfile
 
 
 searx_dir = abspath(dirname(__file__))
 engine_dir = dirname(realpath(__file__))
 static_path = abspath(join(dirname(__file__), 'static'))
-settings, settings_load_message = searx.settings.load_settings()
+settings, settings_load_message = searx.settings_loader.load_settings()
 
 if settings['ui']['static_path']:
     static_path = settings['ui']['static_path']

searx/settings_loader.py

@@ -0,0 +1,129 @@
+# SPDX-License-Identifier: AGPL-3.0-or-later
+
+from os import environ
+from os.path import dirname, join, abspath, isfile
+from collections.abc import Mapping
+from itertools import filterfalse
+
+import yaml
+
+from searx.exceptions import SearxSettingsException
+
+
+searx_dir = abspath(dirname(__file__))
+
+
+def check_settings_yml(file_name):
+    if isfile(file_name):
+        return file_name
+    return None
+
+
+def load_yaml(file_name):
+    try:
+        with open(file_name, 'r', encoding='utf-8') as settings_yaml:
+            return yaml.safe_load(settings_yaml)
+    except IOError as e:
+        raise SearxSettingsException(e, file_name)
+    except yaml.YAMLError as e:
+        raise SearxSettingsException(e, file_name)
+
+
+def get_default_settings_path():
+    return check_settings_yml(join(searx_dir, 'settings.yml'))
+
+
+def get_user_settings_path():
+    # find location of settings.yml
+    if 'SEARX_SETTINGS_PATH' in environ:
+        # if possible set path to settings using the
+        # enviroment variable SEARX_SETTINGS_PATH
+        return check_settings_yml(environ['SEARX_SETTINGS_PATH'])
+
+    # if not, get it from searx code base or last solution from /etc/searx
+    return check_settings_yml('/etc/searx/settings.yml')
+
+
+def update_dict(default_dict, user_dict):
+    for k, v in user_dict.items():
+        if isinstance(v, Mapping):
+            default_dict[k] = update_dict(default_dict.get(k, {}), v)
+        else:
+            default_dict[k] = v
+    return default_dict
+
+
+def update_settings(default_settings, user_settings):
+    # merge everything except the engines
+    for k, v in user_settings.items():
+        if k not in ('use_default_settings', 'engines'):
+            update_dict(default_settings[k], v)
+
+    # parse the engines
+    remove_engines = None
+    keep_only_engines = None
+    use_default_settings = user_settings.get('use_default_settings')
+    if isinstance(use_default_settings, dict):
+        remove_engines = use_default_settings.get('engines', {}).get('remove')
+        keep_only_engines = use_default_settings.get('engines', {}).get('keep_only')
+
+    if 'engines' in user_settings or remove_engines is not None or keep_only_engines is not None:
+        engines = default_settings['engines']
+
+        # parse "use_default_settings.engines.remove"
+        if remove_engines is not None:
+            engines = list(filterfalse(lambda engine: (engine.get('name')) in remove_engines, engines))
+
+        # parse "use_default_settings.engines.keep_only"
+        if keep_only_engines is not None:
+            engines = list(filter(lambda engine: (engine.get('name')) in keep_only_engines, engines))
+
+        # parse "engines"
+        user_engines = user_settings.get('engines')
+        if user_engines:
+            engines_dict = dict((definition['name'], definition) for definition in engines)
+            for user_engine in user_engines:
+                default_engine = engines_dict.get(user_engine['name'])
+                if default_engine:
+                    update_dict(default_engine, user_engine)
+                else:
+                    engines.append(user_engine)
+
+        # store the result
+        default_settings['engines'] = engines
+
+    return default_settings
+
+
+def is_use_default_settings(user_settings):
+    use_default_settings = user_settings.get('use_default_settings')
+    if use_default_settings is True:
+        return True
+    if isinstance(use_default_settings, dict):
+        return True
+    if use_default_settings is False or use_default_settings is None:
+        return False
+    raise ValueError('Invalid value for use_default_settings')
+
+
+def load_settings(load_user_setttings=True):
+    default_settings_path = get_default_settings_path()
+    user_settings_path = get_user_settings_path()
+    if user_settings_path is None or not load_user_setttings:
+        # no user settings
+        return (load_yaml(default_settings_path),
+                'load the default settings from {}'.format(default_settings_path))
+
+    # user settings
+    user_settings = load_yaml(user_settings_path)
+    if is_use_default_settings(user_settings):
+        # the user settings are merged with the default configuration
+        default_settings = load_yaml(default_settings_path)
+        update_settings(default_settings, user_settings)
+        return (default_settings,
+                'merge the default settings ( {} ) and the user setttings ( {} )'
+                .format(default_settings_path, user_settings_path))
+
+    # the user settings, fully replace the default configuration
+    return (user_settings,
+            'load the user settings from {}'.format(user_settings_path))

tests/unit/settings/syntaxerror_settings.yml

@@ -0,0 +1,2 @@
+Test:
+  **********