logo

searx

Unnamed repository; edit this file 'description' to name the repository.
commit: 03579c225ff34406879bbf3ec4d362815960b59b
parent: 7f7f10bb6f7746c0891c2795b36261286b52a76a
Author: Adam Tauber <asciimoo@gmail.com>
Date:   Sun, 22 Feb 2015 23:24:49 +0100

[enh] support of multiple outgoing IPs

Diffstat:

searx/poolrequests.py | 53++++++++++++++++++++++++++++++++++++++++++++++++-----
searx/settings.yml | 6++++++
2 files changed, 54 insertions(+), 5 deletions(-)

diff --git a/searx/poolrequests.py b/searx/poolrequests.py @@ -1,20 +1,63 @@ import requests +from itertools import cycle +from searx import settings -the_http_adapter = requests.adapters.HTTPAdapter(pool_connections=100) -the_https_adapter = requests.adapters.HTTPAdapter(pool_connections=100) +class HTTPAdapterWithConnParams(requests.adapters.HTTPAdapter): + + def __init__(self, pool_connections=requests.adapters.DEFAULT_POOLSIZE, + pool_maxsize=requests.adapters.DEFAULT_POOLSIZE, + max_retries=requests.adapters.DEFAULT_RETRIES, + pool_block=requests.adapters.DEFAULT_POOLBLOCK, + **conn_params): + if max_retries == requests.adapters.DEFAULT_RETRIES: + self.max_retries = requests.adapters.Retry(0, read=False) + else: + self.max_retries = requests.adapters.Retry.from_int(max_retries) + self.config = {} + self.proxy_manager = {} + + super(requests.adapters.HTTPAdapter, self).__init__() + + self._pool_connections = pool_connections + self._pool_maxsize = pool_maxsize + self._pool_block = pool_block + self._conn_params = conn_params + + self.init_poolmanager(pool_connections, pool_maxsize, block=pool_block, **conn_params) + + def __setstate__(self, state): + # Can't handle by adding 'proxy_manager' to self.__attrs__ because + # because self.poolmanager uses a lambda function, which isn't pickleable. + self.proxy_manager = {} + self.config = {} + + for attr, value in state.items(): + setattr(self, attr, value) + + self.init_poolmanager(self._pool_connections, self._pool_maxsize, + block=self._pool_block, **self._conn_params) + + +if settings.get('source_ips'): + http_adapters = cycle(HTTPAdapterWithConnParams(pool_connections=100, source_address=(source_ip, 0)) + for source_ip in settings['source_ips']) + https_adapters = cycle(HTTPAdapterWithConnParams(pool_connections=100, source_address=(source_ip, 0)) + for source_ip in settings['source_ips']) +else: + http_adapters = cycle((HTTPAdapterWithConnParams(pool_connections=100), )) + https_adapters = cycle((HTTPAdapterWithConnParams(pool_connections=100), )) class SessionSinglePool(requests.Session): def __init__(self): - global the_https_adapter, the_http_adapter super(SessionSinglePool, self).__init__() # reuse the same adapters self.adapters.clear() - self.mount('https://', the_https_adapter) - self.mount('http://', the_http_adapter) + self.mount('https://', next(https_adapters)) + self.mount('http://', next(http_adapters)) def close(self): """Call super, but clear adapters since there are managed globaly""" diff --git a/searx/settings.yml b/searx/settings.yml @@ -11,6 +11,12 @@ server: image_proxy : False # Proxying image results through searx default_locale : "" # Default interface locale - leave blank to detect from browser information or use codes from the 'locales' config section +# uncomment below section only if you have more than one network interface +# which can be the source of outgoing search requests +#source_ips: +# - 1.1.1.1 +# - 1.1.1.2 + engines: - name : wikipedia engine : mediawiki