logo

searx

Unnamed repository; edit this file 'description' to name the repository.
commit: d959cb1c059008984554c129cb6e17b6c5394bfc
parent: b1b0b82a2a1d6aaa46799ad65cf75849895b8f07
Author: Adam Tauber <asciimoo@gmail.com>
Date:   Fri,  5 Dec 2014 19:24:11 +0100

[enh] gevent/grequests changed to the built-in threading lib

Diffstat:

requirements.txt | 2+-
searx/search.py | 23++++++++++++++++++-----
searx/webapp.py | 4----
setup.py | 2+-
versions.cfg | 12------------
5 files changed, 20 insertions(+), 23 deletions(-)

diff --git a/requirements.txt b/requirements.txt @@ -1,6 +1,6 @@ flask flask-babel -grequests +requests lxml pyyaml python-dateutil diff --git a/searx/search.py b/searx/search.py @@ -15,7 +15,8 @@ along with searx. If not, see < http://www.gnu.org/licenses/ >. (C) 2013- by Adam Tauber, <asciimoo@gmail.com> ''' -import grequests +import requests as requests_lib +import threading import re from itertools import izip_longest, chain from datetime import datetime @@ -32,6 +33,18 @@ from searx.query import Query number_of_searches = 0 +def threaded_requests(requests): + for fn, url, request_args in requests: + th = threading.Thread( + target=fn, args=(url,), kwargs=request_args, name=url, + ) + th.start() + + for th in threading.enumerate(): + if th.name.startswith('http'): + th.join() + + # get default reqest parameter def default_request_params(): return { @@ -471,9 +484,9 @@ class Search(object): # specific type of request (GET or POST) if request_params['method'] == 'GET': - req = grequests.get + req = requests_lib.get else: - req = grequests.post + req = requests_lib.post request_args['data'] = request_params['data'] # ignoring empty urls @@ -481,10 +494,10 @@ class Search(object): continue # append request to list - requests.append(req(request_params['url'], **request_args)) + requests.append((req, request_params['url'], request_args)) # send all search-request - grequests.map(requests) + threaded_requests(requests) # update engine-specific stats for engine_name, engine_results in results.items(): diff --git a/searx/webapp.py b/searx/webapp.py @@ -17,10 +17,6 @@ along with searx. If not, see < http://www.gnu.org/licenses/ >. (C) 2013- by Adam Tauber, <asciimoo@gmail.com> ''' -from gevent import monkey -monkey.patch_all() - - if __name__ == '__main__': from sys import path from os.path import realpath, dirname diff --git a/setup.py b/setup.py @@ -41,7 +41,7 @@ setup( install_requires=[ 'flask', 'flask-babel', - 'grequests', + 'requests', 'lxml', 'pyyaml', 'setuptools', diff --git a/versions.cfg b/versions.cfg @@ -37,18 +37,6 @@ zc.recipe.testrunner = 2.0.0 beautifulsoup4 = 4.3.2 # Required by: -# grequests==0.2.0 -gevent = 1.0 - -# Required by: -# gevent==1.0 -greenlet = 0.4.2 - -# Required by: -# searx==0.1 -grequests = 0.2.0 - -# Required by: # robotframework-httplibrary==0.4.2 jsonpatch = 1.3