logo

searx

Unnamed repository; edit this file 'description' to name the repository.
commit: 10891bdeab9cc1ffb4b490d2d4897aca0f15c921
parent: 549dcac588e810090e98cb753fde2828bef66325
Author: Adam Tauber <asciimoo@gmail.com>
Date:   Wed, 21 Jan 2015 19:44:20 +0100

Merge pull request #192 from dalf/connection-pool

[enh] improve response time. close #100

Diffstat:

searx/autocomplete.py | 2+-
searx/engines/wikidata.py | 2+-
searx/poolrequests.py | 61+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
searx/search.py | 3+--
searx/webapp.py | 2+-
5 files changed, 65 insertions(+), 5 deletions(-)

diff --git a/searx/autocomplete.py b/searx/autocomplete.py @@ -17,13 +17,13 @@ along with searx. If not, see < http://www.gnu.org/licenses/ >. from lxml import etree -from requests import get from json import loads from urllib import urlencode from searx.languages import language_codes from searx.engines import ( categories, engines, engine_shortcuts ) +from searx.poolrequests import get def searx_bang(full_query): diff --git a/searx/engines/wikidata.py b/searx/engines/wikidata.py @@ -1,6 +1,6 @@ import json -from requests import get from urllib import urlencode +from searx.poolrequests import get from searx.utils import format_date_by_locale result_count = 1 diff --git a/searx/poolrequests.py b/searx/poolrequests.py @@ -0,0 +1,61 @@ +import requests + + +the_http_adapter = requests.adapters.HTTPAdapter(pool_connections=100) +the_https_adapter = requests.adapters.HTTPAdapter(pool_connections=100) + + +class SessionSinglePool(requests.Session): + + def __init__(self): + global the_https_adapter, the_http_adapter + super(SessionSinglePool, self).__init__() + + # reuse the same adapters + self.adapters.clear() + self.mount('https://', the_https_adapter) + self.mount('http://', the_http_adapter) + + def close(self): + """Call super, but clear adapters since there are managed globaly""" + self.adapters.clear() + super(SessionSinglePool, self).close() + + +def request(method, url, **kwargs): + """same as requests/requests/api.py request(...) except it use SessionSinglePool""" + session = SessionSinglePool() + response = session.request(method=method, url=url, **kwargs) + session.close() + return response + + +def get(url, **kwargs): + kwargs.setdefault('allow_redirects', True) + return request('get', url, **kwargs) + + +def options(url, **kwargs): + kwargs.setdefault('allow_redirects', True) + return request('options', url, **kwargs) + + +def head(url, **kwargs): + kwargs.setdefault('allow_redirects', False) + return request('head', url, **kwargs) + + +def post(url, data=None, json=None, **kwargs): + return request('post', url, data=data, json=json, **kwargs) + + +def put(url, data=None, **kwargs): + return request('put', url, data=data, **kwargs) + + +def patch(url, data=None, **kwargs): + return request('patch', url, data=data, **kwargs) + + +def delete(url, **kwargs): + return request('delete', url, **kwargs) diff --git a/searx/search.py b/searx/search.py @@ -15,9 +15,9 @@ along with searx. If not, see < http://www.gnu.org/licenses/ >. (C) 2013- by Adam Tauber, <asciimoo@gmail.com> ''' -import requests as requests_lib import threading import re +import searx.poolrequests as requests_lib from itertools import izip_longest, chain from operator import itemgetter from Queue import Queue @@ -31,7 +31,6 @@ from searx.utils import gen_useragent from searx.query import Query from searx import logger - logger = logger.getChild('search') number_of_searches = 0 diff --git a/searx/webapp.py b/searx/webapp.py @@ -28,7 +28,6 @@ import os import hashlib from datetime import datetime, timedelta -from requests import get as http_get from itertools import chain from urllib import urlencode from flask import ( @@ -37,6 +36,7 @@ from flask import ( ) from flask.ext.babel import Babel, gettext, format_date from searx import settings, searx_dir +from searx.poolrequests import get as http_get from searx.engines import ( categories, engines, get_engines_stats, engine_shortcuts )