logo

searx

My custom branche(s) on searx, a meta-search engine
commit: 5cea4f9445e2b08fe91f9b964cc768920aac7dc8
parent: 029291eca18f08061500194b4e1369c43797739c
Author: Adam Tauber <asciimoo@gmail.com>
Date:   Tue, 22 Dec 2015 20:05:42 +0100

[fix] prevent google engine to redirect

nid/pref cookies are also removed

Diffstat:

Msearx/engines/google.py30+-----------------------------
Msearx/tests/engines/test_google.py6+-----
2 files changed, 2 insertions(+), 34 deletions(-)

diff --git a/searx/engines/google.py b/searx/engines/google.py @@ -13,7 +13,6 @@ from cgi import escape from urllib import urlencode from urlparse import urlparse, parse_qsl from lxml import html, etree -from searx.poolrequests import get from searx.engines.xpath import extract_text, extract_url from searx.search import logger @@ -91,7 +90,7 @@ url_map = 'https://www.openstreetmap.org/'\ search_path = '/search' search_url = ('https://{hostname}' + search_path + - '?{query}&start={offset}&gbv=1') + '?{query}&start={offset}&gbv=1&gws_rd=cr') # other URLs map_hostname_start = 'maps.google.' @@ -129,27 +128,6 @@ image_img_src_xpath = './img/@src' property_address = "Address" property_phone = "Phone number" -# cookies -pref_cookie = '' -nid_cookie = {} - - -# see https://support.google.com/websearch/answer/873?hl=en -def get_google_pref_cookie(): - global pref_cookie - if pref_cookie == '': - resp = get('https://www.google.com/ncr', allow_redirects=False) - pref_cookie = resp.cookies["PREF"] - return pref_cookie - - -def get_google_nid_cookie(google_hostname): - global nid_cookie - if google_hostname not in nid_cookie: - resp = get('https://' + google_hostname) - nid_cookie[google_hostname] = resp.cookies.get("NID", None) - return nid_cookie[google_hostname] - # remove google-specific tracking-url def parse_url(url_string, google_hostname): @@ -201,12 +179,6 @@ def request(query, params): params['headers']['Accept-Language'] = language params['headers']['Accept'] = 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8' - if google_hostname == default_hostname: - try: - params['cookies']['PREF'] = get_google_pref_cookie() - except: - logger.warning('cannot fetch PREF cookie') - params['cookies']['NID'] = get_google_nid_cookie(google_hostname) params['google_hostname'] = google_hostname diff --git a/searx/tests/engines/test_google.py b/searx/tests/engines/test_google.py @@ -9,7 +9,7 @@ from searx.testing import SearxTestCase class TestGoogleEngine(SearxTestCase): def mock_response(self, text): - response = mock.Mock(text=text, url='https://www.google.com/search?q=test&start=0&gbv=1') + response = mock.Mock(text=text, url='https://www.google.com/search?q=test&start=0&gbv=1&gws_rd=cr') response.search_params = mock.Mock() response.search_params.get = mock.Mock(return_value='www.google.com') return response @@ -23,16 +23,12 @@ class TestGoogleEngine(SearxTestCase): self.assertIn('url', params) self.assertIn(query, params['url']) self.assertIn('google.fr', params['url']) - self.assertNotIn('PREF', params['cookies']) - self.assertIn('NID', params['cookies']) self.assertIn('fr', params['headers']['Accept-Language']) dicto['language'] = 'all' params = google.request(query, dicto) self.assertIn('google.com', params['url']) self.assertIn('en', params['headers']['Accept-Language']) - # self.assertIn('PREF', params['cookies']) - self.assertIn('NID', params['cookies']) def test_response(self): self.assertRaises(AttributeError, google.response, None)