searx

commit: f18a3a77ef894d6563543fe449321e52b16e93ad
parent 5cbe4c53329a1fd2b949660fda25ff7a4ce6f254
Author: Guilhem Bonnefille <guilhem.bonnefille@gmail.com>
Date:   Thu, 28 Apr 2016 13:59:41 +0200

Use new settings to fine tune the connection pool

In some circumstances, it is necessary to increase size of pool.

Diffstat:
M
searx/poolrequests.py
12
++++++++----

1 file changed, 8 insertions(+), 4 deletions(-)
diff --git a/searx/poolrequests.py b/searx/poolrequests.py
@@ -41,14 +41,18 @@ class HTTPAdapterWithConnParams(requests.adapters.HTTPAdapter):
                               block=self._pool_block, **self._conn_params)
 
 
+connect=settings['outgoing'].get('pool_connections', 100) # Magic number kept from previous code
+maxsize=settings['outgoing'].get('pool_maxsize', requests.adapters.DEFAULT_POOLSIZE) # Picked from constructor
 if settings['outgoing'].get('source_ips'):
-    http_adapters = cycle(HTTPAdapterWithConnParams(pool_connections=100, source_address=(source_ip, 0))
+    http_adapters = cycle(HTTPAdapterWithConnParams(pool_connections=connect, pool_maxsize=maxsize,
+                                                    source_address=(source_ip, 0))
                           for source_ip in settings['outgoing']['source_ips'])
-    https_adapters = cycle(HTTPAdapterWithConnParams(pool_connections=100, source_address=(source_ip, 0))
+    https_adapters = cycle(HTTPAdapterWithConnParams(pool_connections=connect, pool_maxsize=maxsize,
+                                                    source_address=(source_ip, 0))
                            for source_ip in settings['outgoing']['source_ips'])
 else:
-    http_adapters = cycle((HTTPAdapterWithConnParams(pool_connections=100), ))
-    https_adapters = cycle((HTTPAdapterWithConnParams(pool_connections=100), ))
+    http_adapters = cycle((HTTPAdapterWithConnParams(pool_connections=connect, pool_maxsize=maxsize), ))
+    https_adapters = cycle((HTTPAdapterWithConnParams(pool_connections=connect, pool_maxsize=maxsize), ))
 
 
 class SessionSinglePool(requests.Session):