Merge pull request #1475 from kvch/add-all-language-option-again

Revert "remove 'all' option from search languages"

searx/engines/archlinux.py

@@ -36,7 +36,7 @@ def locale_to_lang_code(locale):
 # wikis for some languages were moved off from the main site, we need to make
 # requests to correct URLs to be able to get results in those languages
 lang_urls = {
-    'en': {
+    'all': {
         'base': 'https://wiki.archlinux.org',
         'search': '/index.php?title=Special:Search&offset={offset}&{query}'
     },
@@ -67,7 +67,7 @@ def locale_to_lang_code(locale):
 def get_lang_urls(language):
     if language in lang_urls:
         return lang_urls[language]
-    return lang_urls['en']
+    return lang_urls['all']
 
 
 # Language names to build search requests for

searx/engines/bing.py

@@ -34,7 +34,10 @@
 def request(query, params):
     offset = (params['pageno'] - 1) * 10 + 1
 
-    lang = match_language(params['language'], supported_languages, language_aliases)
+    if params['language'] == 'all':
+        lang = 'EN'
+    else:
+        lang = match_language(params['language'], supported_languages, language_aliases)
 
     query = u'language:{} {}'.format(lang.split('-')[0].upper(), query.decode('utf-8')).encode('utf-8')
 

searx/engines/bing_news.py

@@ -71,7 +71,10 @@ def request(query, params):
 
     offset = (params['pageno'] - 1) * 10 + 1
 
-    language = match_language(params['language'], supported_languages, language_aliases)
+    if params['language'] == 'all':
+        language = 'en-US'
+    else:
+        language = match_language(params['language'], supported_languages, language_aliases)
 
     params['url'] = _get_url(query, language, offset, params['time_range'])
 

searx/engines/dailymotion.py

@@ -33,7 +33,10 @@
 
 # do search-request
 def request(query, params):
-    locale = match_language(params['language'], supported_languages)
+    if params['language'] == 'all':
+        locale = 'en-US'
+    else:
+        locale = match_language(params['language'], supported_languages)
 
     params['url'] = search_url.format(
         query=urlencode({'search': query, 'localization': locale}),

searx/engines/duckduckgo.py

@@ -54,23 +54,29 @@
 
 # match query's language to a region code that duckduckgo will accept
 def get_region_code(lang, lang_list=[]):
+    if lang == 'all':
+        return None
+
     lang_code = match_language(lang, lang_list, language_aliases, 'wt-WT')
     lang_parts = lang_code.split('-')
 
     # country code goes first
     return lang_parts[1].lower() + '-' + lang_parts[0].lower()
 
 
-# do search-request
 def request(query, params):
     if params['time_range'] and params['time_range'] not in time_range_dict:
         return params
 
     offset = (params['pageno'] - 1) * 30
 
     region_code = get_region_code(params['language'], supported_languages)
-    params['url'] = url.format(
-        query=urlencode({'q': query, 'kl': region_code}), offset=offset, dc_param=offset)
+    if region_code:
+        params['url'] = url.format(
+            query=urlencode({'q': query, 'kl': region_code}), offset=offset, dc_param=offset)
+    else:
+        params['url'] = url.format(
+            query=urlencode({'q': query}), offset=offset, dc_param=offset)
 
     if params['time_range'] in time_range_dict:
         params['url'] += time_range_url.format(range=time_range_dict[params['time_range']])

searx/engines/duckduckgo_images.py

@@ -56,8 +56,12 @@ def request(query, params):
     safesearch = params['safesearch'] - 1
 
     region_code = get_region_code(params['language'], lang_list=supported_languages)
-    params['url'] = images_url.format(
-        query=urlencode({'q': query, 'l': region_code}), offset=offset, safesearch=safesearch, vqd=vqd)
+    if region_code:
+        params['url'] = images_url.format(
+            query=urlencode({'q': query, 'l': region_code}), offset=offset, safesearch=safesearch, vqd=vqd)
+    else:
+        params['url'] = images_url.format(
+            query=urlencode({'q': query}), offset=offset, safesearch=safesearch, vqd=vqd)
 
     return params
 

searx/engines/faroo.py

@@ -40,7 +40,10 @@ def request(query, params):
     offset = (params['pageno'] - 1) * number_of_results + 1
     categorie = search_category.get(params['category'], 'web')
 
-    language = params['language'].split('-')[0]
+    if params['language'] == 'all':
+        language = 'en'
+    else:
+        language = params['language'].split('-')[0]
 
     # if language is not supported, put it in english
     if language != 'en' and\

searx/engines/gigablast.py

@@ -50,9 +50,12 @@
 def request(query, params):
     offset = (params['pageno'] - 1) * number_of_results
 
-    language = params['language'].replace('-', '_').lower()
-    if language.split('-')[0] != 'zh':
-        language = language.split('-')[0]
+    if params['language'] == 'all':
+        language = 'xx'
+    else:
+        language = params['language'].replace('-', '_').lower()
+        if language.split('-')[0] != 'zh':
+            language = language.split('-')[0]
 
     if params['safesearch'] >= 1:
         safesearch = 1

searx/engines/google.py

@@ -166,7 +166,11 @@ def extract_text_from_dom(result, xpath):
 def request(query, params):
     offset = (params['pageno'] - 1) * 10
 
-    language = match_language(params['language'], supported_languages, language_aliases)
+    if params['language'] == 'all' or params['language'] == 'en-US':
+        language = 'en-GB'
+    else:
+        language = match_language(params['language'], supported_languages, language_aliases)
+
     language_array = language.split('-')
     if params['language'].find('-') > 0:
         country = params['language'].split('-')[1]

searx/engines/google_news.py

@@ -51,9 +51,10 @@ def request(query, params):
     params['url'] = search_url.format(query=urlencode({'q': query}),
                                       search_options=urlencode(search_options))
 
-    language = match_language(params['language'], supported_languages, language_aliases).split('-')[0]
-    if language:
-        params['url'] += '&lr=lang_' + language
+    if params['language'] != 'all':
+        language = match_language(params['language'], supported_languages, language_aliases).split('-')[0]
+        if language:
+            params['url'] += '&lr=lang_' + language
 
     return params
 

searx/engines/mediawiki.py

@@ -45,7 +45,10 @@ def request(query, params):
 
     format_strings = list(Formatter().parse(base_url))
 
-    language = params['language'].split('-')[0]
+    if params['language'] == 'all':
+        language = 'en'
+    else:
+        language = params['language'].split('-')[0]
 
     # format_string [('https://', 'language', '', None), ('.wikipedia.org/', None, None, None)]
     if any(x[1] == 'language' for x in format_strings):

searx/engines/photon.py

@@ -35,9 +35,10 @@ def request(query, params):
         search_string.format(query=urlencode({'q': query}),
                              limit=number_of_results)
 
-    language = params['language'].split('-')[0]
-    if language in supported_languages:
-        params['url'] = params['url'] + "&lang=" + language
+    if params['language'] != 'all':
+        language = params['language'].split('_')[0]
+        if language in supported_languages:
+            params['url'] = params['url'] + "&lang=" + language
 
     # using searx User-Agent
     params['headers']['User-Agent'] = searx_useragent()

searx/engines/qwant.py

@@ -46,8 +46,9 @@ def request(query, params):
                                    offset=offset)
 
     # add language tag
-    language = match_language(params['language'], supported_languages, language_aliases)
-    params['url'] += '&locale=' + language.replace('-', '_').lower()
+    if params['language'] != 'all':
+        language = match_language(params['language'], supported_languages, language_aliases)
+        params['url'] += '&locale=' + language.replace('-', '_').lower()
 
     return params
 

searx/engines/startpage.py

@@ -46,8 +46,9 @@ def request(query, params):
     params['data'] = {'query': query,
                       'startat': offset}
 
-    # set language
-    params['data']['with_language'] = ('lang_' + params['language'].split('-')[0])
+    # set language if specified
+    if params['language'] != 'all':
+        params['data']['with_language'] = ('lang_' + params['language'].split('-')[0])
 
     return params
 

searx/engines/subtitleseeker.py

@@ -48,7 +48,7 @@ def response(resp):
         search_lang = 'Farsi'
     elif resp.search_params['language'] == 'pt-BR':
         search_lang = 'Brazilian'
-    else:
+    elif resp.search_params['language'] != 'all':
         search_lang = [lc[3]
                        for lc in language_codes
                        if lc[0].split('-')[0] == resp.search_params['language'].split('-')[0]]

searx/engines/swisscows.py

@@ -36,8 +36,12 @@
 
 # do search-request
 def request(query, params):
-    region = match_language(params['language'], supported_languages, language_aliases)
-    ui_language = region.split('-')[0]
+    if params['language'] == 'all':
+        ui_language = 'browser'
+        region = 'browser'
+    else:
+        region = match_language(params['language'], supported_languages, language_aliases)
+        ui_language = region.split('-')[0]
 
     search_path = search_string.format(
         query=urlencode({'query': query, 'uiLanguage': ui_language, 'region': region}),

searx/engines/twitter.py

@@ -37,7 +37,12 @@
 # do search-request
 def request(query, params):
     params['url'] = search_url + urlencode({'q': query})
-    params['cookies']['lang'] = params['language'].split('-')[0]
+
+    # set language if specified
+    if params['language'] != 'all':
+        params['cookies']['lang'] = params['language'].split('-')[0]
+    else:
+        params['cookies']['lang'] = 'en'
 
     return params
 

searx/engines/wikidata.py

@@ -68,7 +68,10 @@ def response(resp):
     html = fromstring(resp.text)
     search_results = html.xpath(wikidata_ids_xpath)
 
-    language = match_language(resp.search_params['language'], supported_languages, language_aliases).split('-')[0]
+    if resp.search_params['language'].split('-')[0] == 'all':
+        language = 'en'
+    else:
+        language = match_language(resp.search_params['language'], supported_languages, language_aliases).split('-')[0]
 
     # TODO: make requests asynchronous to avoid timeout when result_count > 1
     for search_result in search_results[:result_count]:

searx/engines/wikipedia.py

@@ -31,6 +31,9 @@
 
 # set language in base_url
 def url_lang(lang):
+    lang_pre = lang.split('-')[0]
+    if lang_pre == 'all' or lang_pre not in supported_languages and lang_pre not in language_aliases:
+        return 'en'
     return match_language(lang, supported_languages, language_aliases).split('-')[0]
 
 

searx/engines/yacy.py

@@ -51,7 +51,9 @@ def request(query, params):
                           limit=number_of_results,
                           search_type=search_type)
 
-    params['url'] += '&lr=lang_' + params['language'].split('-')[0]
+    # add language tag if specified
+    if params['language'] != 'all':
+        params['url'] += '&lr=lang_' + params['language'].split('-')[0]
 
     return params
 

searx/engines/yahoo.py

@@ -73,16 +73,25 @@ def _get_url(query, offset, language, time_range):
                                         lang=language)
 
 
+def _get_language(params):
+    if params['language'] == 'all':
+        return 'en'
+
+    language = match_language(params['language'], supported_languages, language_aliases)
+    if language not in language_aliases.values():
+        language = language.split('-')[0]
+    language = language.replace('-', '_').lower()
+
+    return language
+
+
 # do search-request
 def request(query, params):
     if params['time_range'] and params['time_range'] not in time_range_dict:
         return params
 
     offset = (params['pageno'] - 1) * 10 + 1
-    language = match_language(params['language'], supported_languages, language_aliases)
-    if language not in language_aliases.values():
-        language = language.split('-')[0]
-    language = language.replace('-', '_').lower()
+    language = _get_language(params)
 
     params['url'] = _get_url(query, offset, language, params['time_range'])
 

searx/engines/yahoo_news.py

@@ -41,7 +41,10 @@
 def request(query, params):
     offset = (params['pageno'] - 1) * 10 + 1
 
-    language = match_language(params['language'], supported_languages, language_aliases).split('-')[0]
+    if params['language'] == 'all':
+        language = 'en'
+    else:
+        language = match_language(params['language'], supported_languages, language_aliases).split('-')[0]
 
     params['url'] = search_url.format(offset=offset,
                                       query=urlencode({'p': query}),

searx/engines/youtube_api.py

@@ -34,7 +34,9 @@ def request(query, params):
     params['url'] = search_url.format(query=urlencode({'q': query}),
                                       api_key=api_key)
 
-    params['url'] += '&relevanceLanguage=' + params['language'].split('-')[0]
+    # add language tag if specified
+    if params['language'] != 'all':
+        params['url'] += '&relevanceLanguage=' + params['language'].split('-')[0]
 
     return params
 

searx/preferences.py

@@ -12,6 +12,7 @@
 
 COOKIE_MAX_AGE = 60 * 60 * 24 * 365 * 5  # 5 years
 LANGUAGE_CODES = [l[0] for l in languages]
+LANGUAGE_CODES.append('all')
 DISABLED = 0
 ENABLED = 1
 DOI_RESOLVERS = list(settings['doi_resolvers'])

searx/search.py

@@ -24,7 +24,7 @@
 import requests.exceptions
 import searx.poolrequests as requests_lib
 from searx.engines import (
-    categories, engines, settings
+    categories, engines
 )
 from searx.answerers import ask
 from searx.utils import gen_useragent
@@ -221,10 +221,6 @@ def get_search_query_from_webapp(preferences, form):
     else:
         query_lang = preferences.get_value('language')
 
-    # provides backwards compatibility for requests using old language default
-    if query_lang == 'all':
-        query_lang = settings['search']['language']
-
     # check language
     if not VALID_LANGUAGE_CODE.match(query_lang):
         raise SearxParameterException('language', query_lang)

searx/settings_robot.yml

@@ -5,7 +5,7 @@ general:
 search:
     safe_search : 0
     autocomplete : ""
-    language: "en-US"
+    language: "all"
 
 server:
     port : 11111

searx/templates/courgette/preferences.html

@@ -13,6 +13,7 @@ <h2>{{ _('Preferences') }}</h2>
         <legend>{{ _('Search language') }}</legend>
         <p>
             <select name='language'>
+                <option value="all" {% if current_language == 'all' %}selected="selected"{% endif %}>{{ _('Default language') }}</option>
                 {% for lang_id,lang_name,country_name,english_name in language_codes | sort(attribute=1) %}
                 <option value="{{ lang_id }}" {% if lang_id == current_language %}selected="selected"{% endif %}>{{ lang_name }} {% if country_name %}({{ country_name }}) {% endif %}- {{ lang_id }}</option>
                 {% endfor %}

searx/templates/legacy/preferences.html

@@ -14,6 +14,7 @@ <h2>{{ _('Preferences') }}</h2>
         <legend>{{ _('Search language') }}</legend>
         <p>
         <select name='language'>
+            <option value="all" {% if current_language == 'all' %}selected="selected"{% endif %}>{{ _('Default language') }}</option>
             {% for lang_id,lang_name,country_name,english_name in language_codes | sort(attribute=1) %}
             <option value="{{ lang_id }}" {% if lang_id == current_language %}selected="selected"{% endif %}>{{ lang_name }} {% if country_name %}({{ country_name }}) {% endif %}- {{ lang_id }}</option>
             {% endfor %}

searx/templates/oscar/languages.html

@@ -3,9 +3,10 @@
 {% else %}
 <select class="time_range custom-select form-control" id='language' name='language'>
 {% endif %}
-	{% for lang_id,lang_name,country_name,english_name in language_codes | sort(attribute=1) %}
-	<option value="{{ lang_id }}" {% if lang_id == current_language %}selected="selected"{% endif %}>
-		{{ lang_name }} {% if country_name %}({{ country_name }}) {% endif %}- {{ lang_id }}
-	</option>
-	{% endfor %}
+	<option value="all" {% if current_language == 'all' %}selected="selected"{% endif %}>{{ _('Default language') }}</option>
+		{% for lang_id,lang_name,country_name,english_name in language_codes | sort(attribute=1) %}
+		<option value="{{ lang_id }}" {% if lang_id == current_language %}selected="selected"{% endif %}>
+			{{ lang_name }} {% if country_name %}({{ country_name }}) {% endif %}- {{ lang_id }}
+		</option>
+		{% endfor %}
 </select>