searx

commit: 10947536aa1e6e08c54f075b227ba592f1553cb0
parent: 17b0c9f74a17d76add28ccf515c2b8092c455d8a
Author: Adam Tauber <asciimoo@gmail.com>
Date:   Sat, 20 Feb 2016 00:21:56 +0100

[enh] suspend engines after error

The duration is based on the number of continuous errors, but maximized in one minute

Diffstat:
M
searx/engines/__init__.py
4
+++-
M
searx/search.py
17
++++++++++++++---

2 files changed, 17 insertions(+), 4 deletions(-)
diff --git a/searx/engines/__init__.py b/searx/engines/__init__.py
@@ -40,7 +40,9 @@ engine_default_args = {'paging': False,
                        'safesearch': False,
                        'timeout': settings['outgoing']['request_timeout'],
                        'shortcut': '-',
-                       'disabled': False}
+                       'disabled': False,
+                       'suspend_end_time': 0,
+                       'continuous_errors': 0}
 
 
 def load_module(filename):
diff --git a/searx/search.py b/searx/search.py
@@ -34,16 +34,23 @@ number_of_searches = 0
 
 
 def search_request_wrapper(fn, url, engine_name, **kwargs):
+    ret = None
+    engine = engines[engine_name]
     try:
-        return fn(url, **kwargs)
+        ret = fn(url, **kwargs)
+        with threading.RLock():
+            engine.continuous_errors = 0
+            engine.suspend_end_time = 0
     except:
         # increase errors stats
         with threading.RLock():
-            engines[engine_name].stats['errors'] += 1
+            engine.stats['errors'] += 1
+            engine.continuous_errors += 1
+            engine.suspend_end_time = time() + min(60, engine.continuous_errors)
 
         # print engine name and specific error message
         logger.exception('engine crash: {0}'.format(engine_name))
-        return
+    return ret
 
 
 def threaded_requests(requests):
@@ -241,6 +248,10 @@ class Search(object):
                                     for engine in categories[categ]
                                     if (engine.name, categ) not in self.blocked_engines)
 
+        # remove suspended engines
+        self.engines = [e for e in self.engines
+                        if engines[e['name']].suspend_end_time <= time()]
+
     # do search-request
     def search(self, request):
         global number_of_searches