searx

commit: 5cea4f9445e2b08fe91f9b964cc768920aac7dc8
parent: 029291eca18f08061500194b4e1369c43797739c
Author: Adam Tauber <asciimoo@gmail.com>
Date:   Tue, 22 Dec 2015 20:05:42 +0100

[fix] prevent google engine to redirect

nid/pref cookies are also removed

Diffstat:
M
searx/engines/google.py
30
+-----------------------------
M
searx/tests/engines/test_google.py
6
+-----

2 files changed, 2 insertions(+), 34 deletions(-)
diff --git a/searx/engines/google.py b/searx/engines/google.py
@@ -13,7 +13,6 @@ from cgi import escape
 from urllib import urlencode
 from urlparse import urlparse, parse_qsl
 from lxml import html, etree
-from searx.poolrequests import get
 from searx.engines.xpath import extract_text, extract_url
 from searx.search import logger
 
@@ -91,7 +90,7 @@ url_map = 'https://www.openstreetmap.org/'\
 search_path = '/search'
 search_url = ('https://{hostname}' +
               search_path +
-              '?{query}&start={offset}&gbv=1')
+              '?{query}&start={offset}&gbv=1&gws_rd=cr')
 
 # other URLs
 map_hostname_start = 'maps.google.'
@@ -129,27 +128,6 @@ image_img_src_xpath = './img/@src'
 property_address = "Address"
 property_phone = "Phone number"
 
-# cookies
-pref_cookie = ''
-nid_cookie = {}
-
-
-# see https://support.google.com/websearch/answer/873?hl=en
-def get_google_pref_cookie():
-    global pref_cookie
-    if pref_cookie == '':
-        resp = get('https://www.google.com/ncr', allow_redirects=False)
-        pref_cookie = resp.cookies["PREF"]
-    return pref_cookie
-
-
-def get_google_nid_cookie(google_hostname):
-    global nid_cookie
-    if google_hostname not in nid_cookie:
-        resp = get('https://' + google_hostname)
-        nid_cookie[google_hostname] = resp.cookies.get("NID", None)
-    return nid_cookie[google_hostname]
-
 
 # remove google-specific tracking-url
 def parse_url(url_string, google_hostname):
@@ -201,12 +179,6 @@ def request(query, params):
 
     params['headers']['Accept-Language'] = language
     params['headers']['Accept'] = 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8'
-    if google_hostname == default_hostname:
-        try:
-            params['cookies']['PREF'] = get_google_pref_cookie()
-        except:
-            logger.warning('cannot fetch PREF cookie')
-    params['cookies']['NID'] = get_google_nid_cookie(google_hostname)
 
     params['google_hostname'] = google_hostname
 
diff --git a/searx/tests/engines/test_google.py b/searx/tests/engines/test_google.py
@@ -9,7 +9,7 @@ from searx.testing import SearxTestCase
 class TestGoogleEngine(SearxTestCase):
 
     def mock_response(self, text):
-        response = mock.Mock(text=text, url='https://www.google.com/search?q=test&start=0&gbv=1')
+        response = mock.Mock(text=text, url='https://www.google.com/search?q=test&start=0&gbv=1&gws_rd=cr')
         response.search_params = mock.Mock()
         response.search_params.get = mock.Mock(return_value='www.google.com')
         return response
@@ -23,16 +23,12 @@ class TestGoogleEngine(SearxTestCase):
         self.assertIn('url', params)
         self.assertIn(query, params['url'])
         self.assertIn('google.fr', params['url'])
-        self.assertNotIn('PREF', params['cookies'])
-        self.assertIn('NID', params['cookies'])
         self.assertIn('fr', params['headers']['Accept-Language'])
 
         dicto['language'] = 'all'
         params = google.request(query, dicto)
         self.assertIn('google.com', params['url'])
         self.assertIn('en', params['headers']['Accept-Language'])
-        # self.assertIn('PREF', params['cookies'])
-        self.assertIn('NID', params['cookies'])
 
     def test_response(self):
         self.assertRaises(AttributeError, google.response, None)