searx

commit: a5324d9d820fbe03c0898117cac3857477ed71ef
parent: f5aec98cbd023a63ffd6bbee1c18134036b3a7bc
Author: Adam Tauber <asciimoo@gmail.com>
Date:   Sun, 14 Dec 2014 01:18:01 +0100

[fix] proper search timeout handling

Diffstat:
M
searx/search.py
25
+++++++++++++++++++------

1 file changed, 19 insertions(+), 6 deletions(-)
diff --git a/searx/search.py b/searx/search.py
@@ -21,6 +21,8 @@ import re
 from itertools import izip_longest, chain
 from datetime import datetime
 from operator import itemgetter
+from Queue import Queue
+from time import time
 from urlparse import urlparse, unquote
 from searx.engines import (
     categories, engines
@@ -34,6 +36,8 @@ number_of_searches = 0
 
 
 def threaded_requests(requests):
+    timeout_limit = max(r[2]['timeout'] for r in requests)
+    search_start = time()
     for fn, url, request_args in requests:
         th = threading.Thread(
             target=fn,
@@ -45,7 +49,11 @@ def threaded_requests(requests):
 
     for th in threading.enumerate():
         if th.name == 'search_request':
-            th.join()
+            remaining_time = max(0.0, timeout_limit - (time() - search_start))
+            th.join(remaining_time)
+            if th.isAlive():
+                print('engine timeout')
+
 
 
 # get default reqest parameter
@@ -56,7 +64,7 @@ def default_request_params():
 
 # create a callback wrapper for the search engine results
 def make_callback(engine_name,
-                  results,
+                  results_queue,
                   suggestions,
                   answers,
                   infoboxes,
@@ -74,7 +82,7 @@ def make_callback(engine_name,
         except Exception, e:
             # increase errors stats
             engines[engine_name].stats['errors'] += 1
-            results[engine_name] = cb_res
+            results_queue.put_nowait((engine_name, cb_res))
 
             # print engine name and specific error message
             print '[E] Error with engine "{0}":\n\t{1}'.format(
@@ -104,7 +112,7 @@ def make_callback(engine_name,
             # append result
             cb_res.append(result)
 
-        results[engine_name] = cb_res
+        results_queue.put_nowait((engine_name, cb_res))
 
         # update stats with current page-load-time
         engines[engine_name].stats['page_load_time'] += \
@@ -420,7 +428,7 @@ class Search(object):
 
         # init vars
         requests = []
-        results = {}
+        results_queue = Queue()
         suggestions = set()
         answers = set()
         infoboxes = []
@@ -468,7 +476,7 @@ class Search(object):
             # create a callback wrapper for the search engine results
             callback = make_callback(
                 selected_engine['name'],
-                results,
+                results_queue,
                 suggestions,
                 answers,
                 infoboxes,
@@ -502,6 +510,11 @@ class Search(object):
         # send all search-request
         threaded_requests(requests)
 
+        results = {}
+        while not results_queue.empty():
+            engine_name, engine_results = results_queue.get_nowait()
+            results[engine_name] = engine_results
+
         # update engine-specific stats
         for engine_name, engine_results in results.items():
             engines[engine_name].stats['search_count'] += 1