searx

commit: 1c969186bfdabbc5d6c804fa5f1f2fc068492b5f
parent: 8c05c00762a6c4735763d0590844dfeb80db7227
Author: Adam Tauber <asciimoo@gmail.com>
Date:   Fri, 19 Dec 2014 20:01:01 +0100

[mod] better search request exception handling

Diffstat:
M
searx/search.py
28
++++++++++++++++------------

1 file changed, 16 insertions(+), 12 deletions(-)
diff --git a/searx/search.py b/searx/search.py
@@ -34,14 +34,27 @@ from searx.query import Query
 number_of_searches = 0
 
 
+def search_request_wrapper(fn, url, engine_name, **kwargs):
+    try:
+        return fn(url, **kwargs)
+    except Exception, e:
+        # increase errors stats
+        engines[engine_name].stats['errors'] += 1
+
+        # print engine name and specific error message
+        print('[E] Error with engine "{0}":\n\t{1}'.format(
+            engine_name, str(e)))
+        return
+
+
 def threaded_requests(requests):
     timeout_limit = max(r[2]['timeout'] for r in requests)
     search_start = time()
     for fn, url, request_args, engine_name in requests:
         request_args['timeout'] = timeout_limit
         th = threading.Thread(
-            target=fn,
-            args=(url,),
+            target=search_request_wrapper,
+            args=(fn, url, engine_name),
             kwargs=request_args,
             name='search_request',
         )
@@ -79,16 +92,7 @@ def make_callback(engine_name, results_queue, callback, params):
             return
 
         # callback
-        try:
-            search_results = callback(response)
-        except Exception, e:
-            # increase errors stats
-            engines[engine_name].stats['errors'] += 1
-
-            # print engine name and specific error message
-            print '[E] Error with engine "{0}":\n\t{1}'.format(
-                engine_name, str(e))
-            return
+        search_results = callback(response)
 
         # add results
         for result in search_results: