logo

searx

Unnamed repository; edit this file 'description' to name the repository.
commit: 10947536aa1e6e08c54f075b227ba592f1553cb0
parent: 17b0c9f74a17d76add28ccf515c2b8092c455d8a
Author: Adam Tauber <asciimoo@gmail.com>
Date:   Sat, 20 Feb 2016 00:21:56 +0100

[enh] suspend engines after error

The duration is based on the number of continuous errors, but maximized in one minute

Diffstat:

searx/engines/__init__.py | 4+++-
searx/search.py | 17++++++++++++++---
2 files changed, 17 insertions(+), 4 deletions(-)

diff --git a/searx/engines/__init__.py b/searx/engines/__init__.py @@ -40,7 +40,9 @@ engine_default_args = {'paging': False, 'safesearch': False, 'timeout': settings['outgoing']['request_timeout'], 'shortcut': '-', - 'disabled': False} + 'disabled': False, + 'suspend_end_time': 0, + 'continuous_errors': 0} def load_module(filename): diff --git a/searx/search.py b/searx/search.py @@ -34,16 +34,23 @@ number_of_searches = 0 def search_request_wrapper(fn, url, engine_name, **kwargs): + ret = None + engine = engines[engine_name] try: - return fn(url, **kwargs) + ret = fn(url, **kwargs) + with threading.RLock(): + engine.continuous_errors = 0 + engine.suspend_end_time = 0 except: # increase errors stats with threading.RLock(): - engines[engine_name].stats['errors'] += 1 + engine.stats['errors'] += 1 + engine.continuous_errors += 1 + engine.suspend_end_time = time() + min(60, engine.continuous_errors) # print engine name and specific error message logger.exception('engine crash: {0}'.format(engine_name)) - return + return ret def threaded_requests(requests): @@ -241,6 +248,10 @@ class Search(object): for engine in categories[categ] if (engine.name, categ) not in self.blocked_engines) + # remove suspended engines + self.engines = [e for e in self.engines + if engines[e['name']].suspend_end_time <= time()] + # do search-request def search(self, request): global number_of_searches