logo

searx

My custom branche(s) on searx, a meta-search engine git clone https://anongit.hacktivis.me/git/searx.git/

test_google.py (9959B)

    

  1. # -*- coding: utf-8 -*-
  2. from collections import defaultdict
  3. import mock
  4. import lxml
  5. from searx.engines import google
  6. from searx.testing import SearxTestCase
  9. class TestGoogleEngine(SearxTestCase):
  11.     def mock_response(self, text):
  12.         response = mock.Mock(text=text, url='https://www.google.com/search?q=test&start=0&gbv=1&gws_rd=cr')
  13.         response.search_params = mock.Mock()
  14.         response.search_params.get = mock.Mock(return_value='www.google.com')
  15.         return response
  17.     def test_request(self):
  18.         google.supported_languages = ['en', 'fr', 'zh-CN']
  20.         query = 'test_query'
  21.         dicto = defaultdict(dict)
  22.         dicto['pageno'] = 1
  23.         dicto['language'] = 'fr-FR'
  24.         dicto['time_range'] = ''
  25.         params = google.request(query, dicto)
  26.         self.assertIn('url', params)
  27.         self.assertIn(query, params['url'])
  28.         self.assertIn('google.fr', params['url'])
  29.         self.assertIn('fr', params['url'])
  30.         self.assertIn('fr', params['headers']['Accept-Language'])
  32.         dicto['language'] = 'en-US'
  33.         params = google.request(query, dicto)
  34.         self.assertIn('google.com', params['url'])
  35.         self.assertIn('en', params['url'])
  36.         self.assertIn('en', params['headers']['Accept-Language'])
  38.         dicto['language'] = 'zh'
  39.         params = google.request(query, dicto)
  40.         self.assertIn('google.com', params['url'])
  41.         self.assertIn('zh-CN', params['url'])
  42.         self.assertIn('zh-CN', params['headers']['Accept-Language'])
  44.     def test_response(self):
  45.         self.assertRaises(AttributeError, google.response, None)
  46.         self.assertRaises(AttributeError, google.response, [])
  47.         self.assertRaises(AttributeError, google.response, '')
  48.         self.assertRaises(AttributeError, google.response, '[]')
  50.         response = self.mock_response('<html></html>')
  51.         self.assertEqual(google.response(response), [])
  53.         html = """
  54.         <div class="g">
  55.             <h3 class="r">
  56.                 <a href="http://this.should.be.the.link/">
  57.                     <b>This</b> is <b>the</b> title
  58.                 </a>
  59.             </h3>
  60.             <div class="s">
  61.                 <div class="kv" style="margin-bottom:2px">
  62.                     <cite>
  63.                         <b>test</b>.psychologies.com/
  64.                     </cite>
  65.                     <div class="_nBb">‎
  66.                         <div style="display:inline" onclick="google.sham(this);" aria-expanded="false"
  67.                             aria-haspopup="true" tabindex="0" data-ved="0CBUQ7B0wAA">
  68.                             <span class="_O0">
  69.                             </span>
  70.                         </div>
  71.                         <div style="display:none" class="am-dropdown-menu" role="menu" tabindex="-1">
  72.                             <ul>
  73.                                 <li class="_Ykb">
  74.                                     <a class="_Zkb" href="http://www.google.fr/url?url=http://webcache.googleusercontent
  75.                                         .com/search%3Fcache:R1Z_4pGXjuIJ:http://test.psychologies.com/">
  76.                                         En cache
  77.                                     </a>
  78.                                 </li>
  79.                                 <li class="_Ykb">
  80.                                     <a class="_Zkb" href="/search?safe=off&amp;q=related:test.psy.com/">
  81.                                         Pages similaires
  82.                                     </a>
  83.                                 </li>
  84.                             </ul>
  85.                         </div>
  86.                     </div>
  87.                 </div>
  88.                 <span class="st">
  89.                     This should be the content.
  90.                 </span>
  91.                 <br>
  92.                 <div class="osl">‎
  93.                     <a href="http://www.google.fr/url?url=http://test.psychologies.com/tests/">
  94.                         Test Personnalité
  95.                     </a> - ‎
  96.                     <a href="http://www.google.fr/url?url=http://test.psychologies.com/test/">
  97.                         Tests - Moi
  98.                     </a> - ‎
  99.                     <a href="http://www.google.fr/url?url=http://test.psychologies.com/test/tests-couple">
  100.                         Test Couple
  101.                     </a>
  102.                     - ‎
  103.                     <a href="http://www.google.fr/url?url=http://test.psychologies.com/tests/tests-amour">
  104.                         Test Amour
  105.                     </a>
  106.                 </div>
  107.             </div>
  108.         </div>
  109.         <div class="g">
  110.             <h3 class="r">
  111.                 <a href="http://www.google.com/images?q=toto">
  112.                     <b>This</b>
  113.                 </a>
  114.             </h3>
  115.         </div>
  116.         <div class="g">
  117.             <h3 class="r">
  118.                 <a href="http://www.google.com/search?q=toto">
  119.                     <b>This</b> is
  120.                 </a>
  121.             </h3>
  122.         </div>
  123.         <div class="g">
  124.             <h3 class="r">
  125.                 <a href="€">
  126.                     <b>This</b> is <b>the</b>
  127.                 </a>
  128.             </h3>
  129.         </div>
  130.         <div class="g">
  131.             <h3 class="r">
  132.                 <a href="/url?q=url">
  133.                     <b>This</b> is <b>the</b>
  134.                 </a>
  135.             </h3>
  136.         </div>
  137.         <p class="_Bmc" style="margin:3px 8px">
  138.             <a href="/search?num=20&amp;safe=off&amp;q=t&amp;revid=1754833769&amp;sa=X&amp;ei=-&amp;ved=">
  139.                 suggestion <b>title</b>
  140.             </a>
  141.         </p>
  142.         """
  143.         response = self.mock_response(html)
  144.         results = google.response(response)
  145.         self.assertEqual(type(results), list)
  146.         self.assertEqual(len(results), 2)
  147.         self.assertEqual(results[0]['title'], 'This is the title')
  148.         self.assertEqual(results[0]['url'], 'http://this.should.be.the.link/')
  149.         self.assertEqual(results[0]['content'], 'This should be the content.')
  150.         self.assertEqual(results[1]['suggestion'], 'suggestion title')
  152.         html = """
  153.         <li class="b_algo" u="0|5109|4755453613245655|UAGjXgIrPH5yh-o5oNHRx_3Zta87f_QO">
  154.         </li>
  155.         """
  156.         response = self.mock_response(html)
  157.         results = google.response(response)
  158.         self.assertEqual(type(results), list)
  159.         self.assertEqual(len(results), 0)
  161.         response = mock.Mock(text='<html></html>', url='https://sorry.google.com')
  162.         response.search_params = mock.Mock()
  163.         response.search_params.get = mock.Mock(return_value='www.google.com')
  164.         self.assertRaises(RuntimeWarning, google.response, response)
  166.         response = mock.Mock(text='<html></html>', url='https://www.google.com/sorry/IndexRedirect')
  167.         response.search_params = mock.Mock()
  168.         response.search_params.get = mock.Mock(return_value='www.google.com')
  169.         self.assertRaises(RuntimeWarning, google.response, response)
  171.     def test_parse_images(self):
  172.         html = """
  173.         <li>
  174.             <div>
  175.                 <a href="http://www.google.com/url?q=http://this.is.the.url/">
  176.                     <img style="margin:3px 0;margin-right:6px;padding:0" height="90"
  177.                         src="https://this.is.the.image/image.jpg" width="60" align="middle" alt="" border="0">
  178.                 </a>
  179.             </div>
  180.         </li>
  181.         """
  182.         dom = lxml.html.fromstring(html)
  183.         results = google.parse_images(dom, 'www.google.com')
  184.         self.assertEqual(type(results), list)
  185.         self.assertEqual(len(results), 1)
  186.         self.assertEqual(results[0]['url'], 'http://this.is.the.url/')
  187.         self.assertEqual(results[0]['title'], '')
  188.         self.assertEqual(results[0]['content'], '')
  189.         self.assertEqual(results[0]['img_src'], 'https://this.is.the.image/image.jpg')
  191.     def test_fetch_supported_languages(self):
  192.         html = """<html></html>"""
  193.         response = mock.Mock(text=html)
  194.         languages = google._fetch_supported_languages(response)
  195.         self.assertEqual(type(languages), dict)
  196.         self.assertEqual(len(languages), 0)
  198.         html = u"""
  199.         <html>
  200.             <body>
  201.                 <table>
  202.                     <tbody>
  203.                         <tr>
  204.                             <td>
  205.                                 <font>
  206.                                     <label>
  207.                                         <span id="ten">English</span>
  208.                                     </label>
  209.                                 </font>
  210.                             </td>
  211.                             <td>
  212.                                 <font>
  213.                                     <label>
  214.                                         <span id="tzh-CN">中文 (简体)</span>
  215.                                     </label>
  216.                                     <label>
  217.                                         <span id="tzh-TW">中文 (繁體)</span>
  218.                                     </label>
  219.                                 </font>
  220.                             </td>
  221.                         </tr>
  222.                     </tbody>
  223.                 </table>
  224.             </body>
  225.         </html>
  226.         """
  227.         response = mock.Mock(text=html)
  228.         languages = google._fetch_supported_languages(response)
  229.         self.assertEqual(type(languages), dict)
  230.         self.assertEqual(len(languages), 3)
  232.         self.assertIn('en', languages)
  233.         self.assertIn('zh-CN', languages)
  234.         self.assertIn('zh-TW', languages)
  236.         self.assertEquals(type(languages['en']), dict)
  237.         self.assertEquals(type(languages['zh-CN']), dict)
  238.         self.assertEquals(type(languages['zh-TW']), dict)
  240.         self.assertIn('name', languages['en'])
  241.         self.assertIn('name', languages['zh-CN'])
  242.         self.assertIn('name', languages['zh-TW'])
  244.         self.assertEquals(languages['en']['name'], 'English')
  245.         self.assertEquals(languages['zh-CN']['name'], u'中文 (简体)')
  246.         self.assertEquals(languages['zh-TW']['name'], u'中文 (繁體)')