logo

searx

My custom branche(s) on searx, a meta-search engine git clone https://hacktivis.me/git/searx.git

test_google.py (9959B)

    

  1. # -*- coding: utf-8 -*-

  2. from collections import defaultdict

  3. import mock

  4. import lxml

  5. from searx.engines import google

  6. from searx.testing import SearxTestCase

  7. 

  8. 

  9. class TestGoogleEngine(SearxTestCase):

  10. 

  11.     def mock_response(self, text):

  12.         response = mock.Mock(text=text, url='https://www.google.com/search?q=test&start=0&gbv=1&gws_rd=cr')

  13.         response.search_params = mock.Mock()

  14.         response.search_params.get = mock.Mock(return_value='www.google.com')

  15.         return response

  16. 

  17.     def test_request(self):

  18.         google.supported_languages = ['en', 'fr', 'zh-CN']

  19. 

  20.         query = 'test_query'

  21.         dicto = defaultdict(dict)

  22.         dicto['pageno'] = 1

  23.         dicto['language'] = 'fr-FR'

  24.         dicto['time_range'] = ''

  25.         params = google.request(query, dicto)

  26.         self.assertIn('url', params)

  27.         self.assertIn(query, params['url'])

  28.         self.assertIn('google.fr', params['url'])

  29.         self.assertIn('fr', params['url'])

  30.         self.assertIn('fr', params['headers']['Accept-Language'])

  31. 

  32.         dicto['language'] = 'en-US'

  33.         params = google.request(query, dicto)

  34.         self.assertIn('google.com', params['url'])

  35.         self.assertIn('en', params['url'])

  36.         self.assertIn('en', params['headers']['Accept-Language'])

  37. 

  38.         dicto['language'] = 'zh'

  39.         params = google.request(query, dicto)

  40.         self.assertIn('google.com', params['url'])

  41.         self.assertIn('zh-CN', params['url'])

  42.         self.assertIn('zh-CN', params['headers']['Accept-Language'])

  43. 

  44.     def test_response(self):

  45.         self.assertRaises(AttributeError, google.response, None)

  46.         self.assertRaises(AttributeError, google.response, [])

  47.         self.assertRaises(AttributeError, google.response, '')

  48.         self.assertRaises(AttributeError, google.response, '[]')

  49. 

  50.         response = self.mock_response('<html></html>')

  51.         self.assertEqual(google.response(response), [])

  52. 

  53.         html = """

  54.         <div class="g">

  55.             <h3 class="r">

  56.                 <a href="http://this.should.be.the.link/">

  57.                     <b>This</b> is <b>the</b> title

  58.                 </a>

  59.             </h3>

  60.             <div class="s">

  61.                 <div class="kv" style="margin-bottom:2px">

  62.                     <cite>

  63.                         <b>test</b>.psychologies.com/

  64.                     </cite>

  65.                     <div class="_nBb">‎

  66.                         <div style="display:inline" onclick="google.sham(this);" aria-expanded="false"

  67.                             aria-haspopup="true" tabindex="0" data-ved="0CBUQ7B0wAA">

  68.                             <span class="_O0">

  69.                             </span>

  70.                         </div>

  71.                         <div style="display:none" class="am-dropdown-menu" role="menu" tabindex="-1">

  72.                             <ul>

  73.                                 <li class="_Ykb">

  74.                                     <a class="_Zkb" href="http://www.google.fr/url?url=http://webcache.googleusercontent

  75.                                         .com/search%3Fcache:R1Z_4pGXjuIJ:http://test.psychologies.com/">

  76.                                         En cache

  77.                                     </a>

  78.                                 </li>

  79.                                 <li class="_Ykb">

  80.                                     <a class="_Zkb" href="/search?safe=off&amp;q=related:test.psy.com/">

  81.                                         Pages similaires

  82.                                     </a>

  83.                                 </li>

  84.                             </ul>

  85.                         </div>

  86.                     </div>

  87.                 </div>

  88.                 <span class="st">

  89.                     This should be the content.

  90.                 </span>

  91.                 <br>

  92.                 <div class="osl">‎

  93.                     <a href="http://www.google.fr/url?url=http://test.psychologies.com/tests/">

  94.                         Test Personnalité

  95.                     </a> - ‎

  96.                     <a href="http://www.google.fr/url?url=http://test.psychologies.com/test/">

  97.                         Tests - Moi

  98.                     </a> - ‎

  99.                     <a href="http://www.google.fr/url?url=http://test.psychologies.com/test/tests-couple">

  100.                         Test Couple

  101.                     </a>

  102.                     - ‎

  103.                     <a href="http://www.google.fr/url?url=http://test.psychologies.com/tests/tests-amour">

  104.                         Test Amour

  105.                     </a>

  106.                 </div>

  107.             </div>

  108.         </div>

  109.         <div class="g">

  110.             <h3 class="r">

  111.                 <a href="http://www.google.com/images?q=toto">

  112.                     <b>This</b>

  113.                 </a>

  114.             </h3>

  115.         </div>

  116.         <div class="g">

  117.             <h3 class="r">

  118.                 <a href="http://www.google.com/search?q=toto">

  119.                     <b>This</b> is

  120.                 </a>

  121.             </h3>

  122.         </div>

  123.         <div class="g">

  124.             <h3 class="r">

  125.                 <a href="€">

  126.                     <b>This</b> is <b>the</b>

  127.                 </a>

  128.             </h3>

  129.         </div>

  130.         <div class="g">

  131.             <h3 class="r">

  132.                 <a href="/url?q=url">

  133.                     <b>This</b> is <b>the</b>

  134.                 </a>

  135.             </h3>

  136.         </div>

  137.         <p class="_Bmc" style="margin:3px 8px">

  138.             <a href="/search?num=20&amp;safe=off&amp;q=t&amp;revid=1754833769&amp;sa=X&amp;ei=-&amp;ved=">

  139.                 suggestion <b>title</b>

  140.             </a>

  141.         </p>

  142.         """

  143.         response = self.mock_response(html)

  144.         results = google.response(response)

  145.         self.assertEqual(type(results), list)

  146.         self.assertEqual(len(results), 2)

  147.         self.assertEqual(results[0]['title'], 'This is the title')

  148.         self.assertEqual(results[0]['url'], 'http://this.should.be.the.link/')

  149.         self.assertEqual(results[0]['content'], 'This should be the content.')

  150.         self.assertEqual(results[1]['suggestion'], 'suggestion title')

  151. 

  152.         html = """

  153.         <li class="b_algo" u="0|5109|4755453613245655|UAGjXgIrPH5yh-o5oNHRx_3Zta87f_QO">

  154.         </li>

  155.         """

  156.         response = self.mock_response(html)

  157.         results = google.response(response)

  158.         self.assertEqual(type(results), list)

  159.         self.assertEqual(len(results), 0)

  160. 

  161.         response = mock.Mock(text='<html></html>', url='https://sorry.google.com')

  162.         response.search_params = mock.Mock()

  163.         response.search_params.get = mock.Mock(return_value='www.google.com')

  164.         self.assertRaises(RuntimeWarning, google.response, response)

  165. 

  166.         response = mock.Mock(text='<html></html>', url='https://www.google.com/sorry/IndexRedirect')

  167.         response.search_params = mock.Mock()

  168.         response.search_params.get = mock.Mock(return_value='www.google.com')

  169.         self.assertRaises(RuntimeWarning, google.response, response)

  170. 

  171.     def test_parse_images(self):

  172.         html = """

  173.         <li>

  174.             <div>

  175.                 <a href="http://www.google.com/url?q=http://this.is.the.url/">

  176.                     <img style="margin:3px 0;margin-right:6px;padding:0" height="90"

  177.                         src="https://this.is.the.image/image.jpg" width="60" align="middle" alt="" border="0">

  178.                 </a>

  179.             </div>

  180.         </li>

  181.         """

  182.         dom = lxml.html.fromstring(html)

  183.         results = google.parse_images(dom, 'www.google.com')

  184.         self.assertEqual(type(results), list)

  185.         self.assertEqual(len(results), 1)

  186.         self.assertEqual(results[0]['url'], 'http://this.is.the.url/')

  187.         self.assertEqual(results[0]['title'], '')

  188.         self.assertEqual(results[0]['content'], '')

  189.         self.assertEqual(results[0]['img_src'], 'https://this.is.the.image/image.jpg')

  190. 

  191.     def test_fetch_supported_languages(self):

  192.         html = """<html></html>"""

  193.         response = mock.Mock(text=html)

  194.         languages = google._fetch_supported_languages(response)

  195.         self.assertEqual(type(languages), dict)

  196.         self.assertEqual(len(languages), 0)

  197. 

  198.         html = u"""

  199.         <html>

  200.             <body>

  201.                 <table>

  202.                     <tbody>

  203.                         <tr>

  204.                             <td>

  205.                                 <font>

  206.                                     <label>

  207.                                         <span id="ten">English</span>

  208.                                     </label>

  209.                                 </font>

  210.                             </td>

  211.                             <td>

  212.                                 <font>

  213.                                     <label>

  214.                                         <span id="tzh-CN">中文 (简体)</span>

  215.                                     </label>

  216.                                     <label>

  217.                                         <span id="tzh-TW">中文 (繁體)</span>

  218.                                     </label>

  219.                                 </font>

  220.                             </td>

  221.                         </tr>

  222.                     </tbody>

  223.                 </table>

  224.             </body>

  225.         </html>

  226.         """

  227.         response = mock.Mock(text=html)

  228.         languages = google._fetch_supported_languages(response)

  229.         self.assertEqual(type(languages), dict)

  230.         self.assertEqual(len(languages), 3)

  231. 

  232.         self.assertIn('en', languages)

  233.         self.assertIn('zh-CN', languages)

  234.         self.assertIn('zh-TW', languages)

  235. 

  236.         self.assertEquals(type(languages['en']), dict)

  237.         self.assertEquals(type(languages['zh-CN']), dict)

  238.         self.assertEquals(type(languages['zh-TW']), dict)

  239. 

  240.         self.assertIn('name', languages['en'])

  241.         self.assertIn('name', languages['zh-CN'])

  242.         self.assertIn('name', languages['zh-TW'])

  243. 

  244.         self.assertEquals(languages['en']['name'], 'English')

  245.         self.assertEquals(languages['zh-CN']['name'], u'中文 (简体)')

  246.         self.assertEquals(languages['zh-TW']['name'], u'中文 (繁體)')