[enh] self-report: check if an engine has returned some results since…

… the instance has started.

Check "Number of results" in the /stats page.
The /stats page is loaded multiple time because of possible use of uwsgi
see searx/searx#199

html/index.html

@@ -258,6 +258,13 @@ <h2>Working engines of {{instances_filtered.length}} instance(s) (on total {{ins
                     </tr>
                 </tbody>
             </table>
+            <h4 id="help-http-grade">Icons:</h4>
+            <ul>
+              <li>? - No self-checking, the engine hasn't returned results since the searx instance has started. Perhaps the engine has been used.</li>
+              <li>🟡 - No self-checking, the engine has returned results since the searx instance has started. Maybe it doesn't work anymore.</li>
+              <li>❌ - After self-check, the engine does not return a result.</li>
+              <li>✔️ - After a self-check, the engine returns results.</li>
+            </ul>
           </div>
           <div v-if="selected_tab === 'about'">
             <h2>About</h2>

html/main.css

@@ -214,28 +214,27 @@ tbody .column-responsetime {
   margin: 0;
 }
 
-.item-check {
-  display: inline-block;
-  transform: rotate(45deg);
-  height: 1rem;
-  width: 0.5rem;
-  border-bottom: 2px solid #78b13f;
-  border-right: 2px solid #78b13f;
-  text-indent: 100%;
-  white-space: nowrap;
-  overflow: hidden;
-  margin: 0.2rem;
+.column-engine-status {
+  padding: 0.9rem;
 }
 
-.item-nocheck  {
-  display: none;
+.item-check, .item-unknow, .item-maybe {
+  display: inline-block;
+  height: 2rem;
+  width: 2rem;
+  margin: 0;
+  padding: 0;
+  text-align: center;
 }
 
 .item-unknow {
-  display: inline-block;
   color: lightgray;
 }
 
+.item-nocheck  {
+  display: none;
+}
+
 .table-striped-column td, .table-striped-column th {
   position: relative;
 }

html/main.js

@@ -337,10 +337,14 @@ Vue.component('engine-component', {
                     return h('span', {
                         staticClass: 'item-check'
                     }, '✔️');
-                } if (engine.status === false) {
+                } else if (engine.status === false) {
                     return createTooltip(h, h('span', {
                         staticClass: 'item-uncheck'
                     }, '❌'), engine.error);
+                } else if (engine.stats === true) {
+                    return createTooltip(h, h('span', {
+                        staticClass: 'item-maybe'
+                    }, '🟡'), engine.error);
                 } else {
                     return h('span', {
                         staticClass: 'item-unknow'

searxstats/fetcher/selfreport.py

@@ -1,13 +1,21 @@
 # pylint: disable=invalid-name
 import json
 from urllib.parse import urljoin
+from lxml import etree
 from searxstats.common.utils import dict_merge
 from searxstats.common.foreach import for_each
 from searxstats.common.http import new_session, get, get_network_type
+from searxstats.common.html import html_fromstring, extract_text
 from searxstats.common.memoize import MemoizeToDisk
 from searxstats.model import SearxStatisticsResult
 
 
+STATS_ENGINES_XPATH = etree.XPath(
+    "//div[@class='col-xs-12 col-sm-12 col-md-6'][3]//div[@class='row']")
+STATS_ENGINES_NAME_XPATH = etree.XPath("div[@class='col-sm-4 col-md-4'][1]")
+STATS_ENGINES_COUNT_XPATH = etree.XPath("div[@class='col-sm-8 col-md-8'][1]")
+
+
 # pylint: disable=unused-argument
 def get_usable_engines_key(_, instance_url):
     return instance_url
@@ -32,6 +40,43 @@ async def get_status(session, instance_url):
     return result
 
 
+async def get_stats(session, instance_url):
+    result = set()
+    response, error = await get(session, urljoin(instance_url, 'stats'), timeout=5)
+    if response is not None and error is None:
+        html = await html_fromstring(response.text)
+        for e in STATS_ENGINES_XPATH(html):
+            engine_name = extract_text(STATS_ENGINES_NAME_XPATH(e))
+            result_count = extract_text(STATS_ENGINES_COUNT_XPATH(e))
+            if result_count not in ['', '0.00'] and engine_name is not None:
+                result.add(engine_name)
+    return result
+
+
+@MemoizeToDisk(func_key=get_usable_engines_key)
+async def get_stats_multi(session, instance_url):
+    result = set()
+    # fetch the stats four times because of uwsgi
+    # may be not enough to get the statistics from all the uwsgi processes
+    # still better than only once
+    # see https://github.com/asciimoo/searx/issues/162
+    # and https://github.com/asciimoo/searx/issues/199
+    for _ in range(4):
+        result = result.union(await get_stats(session, instance_url))
+    return result
+
+
+def get_status_from_stats(stats):
+    if len(stats) == 0:
+        return None
+    else:
+        status = {}
+        for engine_name in stats:
+            engine_status = status.setdefault(engine_name, {})
+            engine_status['stats'] = True
+        return status
+
+
 @MemoizeToDisk(func_key=get_usable_engines_key)
 async def get_config(session, instance_url):
     result_config = None
@@ -75,6 +120,9 @@ async def fetch_one(searx_stats_result: SearxStatisticsResult, url: str, detail)
         # get config and config
         result_status = await get_status(session, url)
         result_config, result_instance = await get_config(session, url)
+        if result_status is None:
+            result_stats = await get_stats_multi(session, url)
+            result_status = get_status_from_stats(result_stats)
 
         # update config and status for the instance
         detail_engines = detail.setdefault('engines', dict())