commit: d07cfd9089d05a6a81109518f03ba82660a4aef7
parent a865e6672fa2a289937ff378e8893c19f71c5e0c
Author: dalf <alex@al-f.net>
Date: Wed, 21 Jan 2015 11:33:16 +0100
[enh] use one single http connection pool : improve response time. close #100
Diffstat:
5 files changed, 65 insertions(+), 5 deletions(-)
diff --git a/searx/autocomplete.py b/searx/autocomplete.py
@@ -17,13 +17,13 @@ along with searx. If not, see < http://www.gnu.org/licenses/ >.
from lxml import etree
-from requests import get
from json import loads
from urllib import urlencode
from searx.languages import language_codes
from searx.engines import (
categories, engines, engine_shortcuts
)
+from searx.poolrequests import get
def searx_bang(full_query):
diff --git a/searx/engines/wikidata.py b/searx/engines/wikidata.py
@@ -1,6 +1,6 @@
import json
-from requests import get
from urllib import urlencode
+from searx.poolrequests import get
from searx.utils import format_date_by_locale
result_count = 1
diff --git a/searx/poolrequests.py b/searx/poolrequests.py
@@ -0,0 +1,61 @@
+import requests
+
+
+the_http_adapter = requests.adapters.HTTPAdapter(pool_connections=100)
+the_https_adapter = requests.adapters.HTTPAdapter(pool_connections=100)
+
+
+class SessionSinglePool(requests.Session):
+
+ def __init__(self):
+ global the_https_adapter, the_http_adapter
+ super(SessionSinglePool, self).__init__()
+
+ # reuse the same adapters
+ self.adapters.clear()
+ self.mount('https://', the_https_adapter)
+ self.mount('http://', the_http_adapter)
+
+ def close(self):
+ """Call super, but clear adapters since there are managed globaly"""
+ self.adapters.clear()
+ super(SessionSinglePool, self).close()
+
+
+def request(method, url, **kwargs):
+ """same as requests/requests/api.py request(...) except it use SessionSinglePool"""
+ session = SessionSinglePool()
+ response = session.request(method=method, url=url, **kwargs)
+ session.close()
+ return response
+
+
+def get(url, **kwargs):
+ kwargs.setdefault('allow_redirects', True)
+ return request('get', url, **kwargs)
+
+
+def options(url, **kwargs):
+ kwargs.setdefault('allow_redirects', True)
+ return request('options', url, **kwargs)
+
+
+def head(url, **kwargs):
+ kwargs.setdefault('allow_redirects', False)
+ return request('head', url, **kwargs)
+
+
+def post(url, data=None, json=None, **kwargs):
+ return request('post', url, data=data, json=json, **kwargs)
+
+
+def put(url, data=None, **kwargs):
+ return request('put', url, data=data, **kwargs)
+
+
+def patch(url, data=None, **kwargs):
+ return request('patch', url, data=data, **kwargs)
+
+
+def delete(url, **kwargs):
+ return request('delete', url, **kwargs)
diff --git a/searx/search.py b/searx/search.py
@@ -15,9 +15,9 @@ along with searx. If not, see < http://www.gnu.org/licenses/ >.
(C) 2013- by Adam Tauber, <asciimoo@gmail.com>
'''
-import requests as requests_lib
import threading
import re
+import searx.poolrequests as requests_lib
from itertools import izip_longest, chain
from operator import itemgetter
from Queue import Queue
@@ -31,7 +31,6 @@ from searx.utils import gen_useragent
from searx.query import Query
from searx import logger
-
logger = logger.getChild('search')
number_of_searches = 0
diff --git a/searx/webapp.py b/searx/webapp.py
@@ -27,7 +27,6 @@ import cStringIO
import os
from datetime import datetime, timedelta
-from requests import get as http_get
from itertools import chain
from urllib import urlencode
from flask import (
@@ -36,6 +35,7 @@ from flask import (
)
from flask.ext.babel import Babel, gettext, format_date
from searx import settings, searx_dir
+from searx.poolrequests import get as http_get
from searx.engines import (
categories, engines, get_engines_stats, engine_shortcuts
)
