searx

commit: d959cb1c059008984554c129cb6e17b6c5394bfc
parent: b1b0b82a2a1d6aaa46799ad65cf75849895b8f07
Author: Adam Tauber <asciimoo@gmail.com>
Date:   Fri,  5 Dec 2014 19:24:11 +0100

[enh] gevent/grequests changed to the built-in threading lib

Diffstat:
M
requirements.txt
2
+-
M
searx/search.py
23
++++++++++++++++++-----
M
searx/webapp.py
4
----
M
setup.py
2
+-
M
versions.cfg
12
------------

5 files changed, 20 insertions(+), 23 deletions(-)
diff --git a/requirements.txt b/requirements.txt
@@ -1,6 +1,6 @@
 flask
 flask-babel
-grequests
+requests
 lxml
 pyyaml
 python-dateutil
diff --git a/searx/search.py b/searx/search.py
@@ -15,7 +15,8 @@ along with searx. If not, see < http://www.gnu.org/licenses/ >.
 (C) 2013- by Adam Tauber, <asciimoo@gmail.com>
 '''
 
-import grequests
+import requests as requests_lib
+import threading
 import re
 from itertools import izip_longest, chain
 from datetime import datetime
@@ -32,6 +33,18 @@ from searx.query import Query
 number_of_searches = 0
 
 
+def threaded_requests(requests):
+    for fn, url, request_args in requests:
+        th = threading.Thread(
+            target=fn, args=(url,), kwargs=request_args, name=url,
+        )
+        th.start()
+
+    for th in threading.enumerate():
+        if th.name.startswith('http'):
+            th.join()
+
+
 # get default reqest parameter
 def default_request_params():
     return {
@@ -471,9 +484,9 @@ class Search(object):
 
             # specific type of request (GET or POST)
             if request_params['method'] == 'GET':
-                req = grequests.get
+                req = requests_lib.get
             else:
-                req = grequests.post
+                req = requests_lib.post
                 request_args['data'] = request_params['data']
 
             # ignoring empty urls
@@ -481,10 +494,10 @@ class Search(object):
                 continue
 
             # append request to list
-            requests.append(req(request_params['url'], **request_args))
+            requests.append((req, request_params['url'], request_args))
 
         # send all search-request
-        grequests.map(requests)
+        threaded_requests(requests)
 
         # update engine-specific stats
         for engine_name, engine_results in results.items():
diff --git a/searx/webapp.py b/searx/webapp.py
@@ -17,10 +17,6 @@ along with searx. If not, see < http://www.gnu.org/licenses/ >.
 (C) 2013- by Adam Tauber, <asciimoo@gmail.com>
 '''
 
-from gevent import monkey
-monkey.patch_all()
-
-
 if __name__ == '__main__':
     from sys import path
     from os.path import realpath, dirname
diff --git a/setup.py b/setup.py
@@ -41,7 +41,7 @@ setup(
     install_requires=[
         'flask',
         'flask-babel',
-        'grequests',
+        'requests',
         'lxml',
         'pyyaml',
         'setuptools',
diff --git a/versions.cfg b/versions.cfg
@@ -37,18 +37,6 @@ zc.recipe.testrunner = 2.0.0
 beautifulsoup4 = 4.3.2
 
 # Required by:
-# grequests==0.2.0
-gevent = 1.0
-
-# Required by:
-# gevent==1.0
-greenlet = 0.4.2
-
-# Required by:
-# searx==0.1
-grequests = 0.2.0
-
-# Required by:
 # robotframework-httplibrary==0.4.2
 jsonpatch = 1.3