logo

searx

My custom branche(s) on searx, a meta-search engine git clone https://anongit.hacktivis.me/git/searx.git/

test_wikipedia.py (10255B)

    

  1. # -*- coding: utf-8 -*-
  2. from collections import defaultdict
  3. import mock
  4. from searx.engines import wikipedia
  5. from searx.testing import SearxTestCase
  8. class TestWikipediaEngine(SearxTestCase):
  10.     def test_request(self):
  11.         wikipedia.supported_languages = ['fr', 'en']
  13.         query = 'test_query'
  14.         dicto = defaultdict(dict)
  15.         dicto['language'] = 'fr-FR'
  16.         params = wikipedia.request(query.encode('utf-8'), dicto)
  17.         self.assertIn('url', params)
  18.         self.assertIn(query, params['url'])
  19.         self.assertIn('test_query', params['url'])
  20.         self.assertIn('Test_Query', params['url'])
  21.         self.assertIn('fr.wikipedia.org', params['url'])
  23.         query = u'Test_Query'
  24.         params = wikipedia.request(query.encode('utf-8'), dicto)
  25.         self.assertIn('Test_Query', params['url'])
  26.         self.assertNotIn('test_query', params['url'])
  28.         dicto['language'] = 'xx'
  29.         params = wikipedia.request(query, dicto)
  30.         self.assertIn('en', params['url'])
  32.     def test_response(self):
  33.         dicto = defaultdict(dict)
  34.         dicto['language'] = 'fr'
  36.         self.assertRaises(AttributeError, wikipedia.response, None)
  37.         self.assertRaises(AttributeError, wikipedia.response, [])
  38.         self.assertRaises(AttributeError, wikipedia.response, '')
  39.         self.assertRaises(AttributeError, wikipedia.response, '[]')
  41.         # page not found
  42.         json = """
  43.         {
  44.             "batchcomplete": "",
  45.             "query": {
  46.                 "normalized": [],
  47.                 "pages": {
  48.                     "-1": {
  49.                         "ns": 0,
  50.                         "title": "",
  51.                         "missing": ""
  52.                     }
  53.                 }
  54.             }
  55.         }"""
  56.         response = mock.Mock(text=json, search_params=dicto)
  57.         self.assertEqual(wikipedia.response(response), [])
  59.         # normal case
  60.         json = """
  61.         {
  62.             "batchcomplete": "",
  63.             "query": {
  64.                 "normalized": [],
  65.                 "pages": {
  66.                     "12345": {
  67.                         "pageid": 12345,
  68.                         "ns": 0,
  69.                         "title": "The Title",
  70.                         "extract": "The Title is...",
  71.                         "thumbnail": {
  72.                             "source": "img_src.jpg"
  73.                         },
  74.                         "pageimage": "img_name.jpg"
  75.                     }
  76.                 }
  77.             }
  78.         }"""
  79.         response = mock.Mock(text=json, search_params=dicto)
  80.         results = wikipedia.response(response)
  81.         self.assertEqual(type(results), list)
  82.         self.assertEqual(len(results), 2)
  83.         self.assertEqual(results[0]['title'], u'The Title')
  84.         self.assertIn('fr.wikipedia.org/wiki/The_Title', results[0]['url'])
  85.         self.assertEqual(results[1]['infobox'], u'The Title')
  86.         self.assertIn('fr.wikipedia.org/wiki/The_Title', results[1]['id'])
  87.         self.assertIn('The Title is...', results[1]['content'])
  88.         self.assertEqual(results[1]['img_src'], 'img_src.jpg')
  90.         # disambiguation page
  91.         json = """
  92.         {
  93.             "batchcomplete": "",
  94.             "query": {
  95.                 "normalized": [],
  96.                 "pages": {
  97.                     "12345": {
  98.                         "pageid": 12345,
  99.                         "ns": 0,
  100.                         "title": "The Title",
  101.                         "extract": "The Title can be:\\nThe Title 1\\nThe Title 2\\nThe Title 3\\nThe Title 4......................................................................................................................................." """  # noqa
  102.         json += """
  103.                     }
  104.                 }
  105.             }
  106.         }"""
  107.         response = mock.Mock(text=json, search_params=dicto)
  108.         results = wikipedia.response(response)
  109.         self.assertEqual(type(results), list)
  110.         self.assertEqual(len(results), 2)
  112.         # no image
  113.         json = """
  114.         {
  115.             "batchcomplete": "",
  116.             "query": {
  117.                 "normalized": [],
  118.                 "pages": {
  119.                     "12345": {
  120.                         "pageid": 12345,
  121.                         "ns": 0,
  122.                         "title": "The Title",
  123.                         "extract": "The Title is......................................................................................................................................................................................." """  # noqa
  124.         json += """
  125.                     }
  126.                 }
  127.             }
  128.         }"""
  129.         response = mock.Mock(text=json, search_params=dicto)
  130.         results = wikipedia.response(response)
  131.         self.assertEqual(type(results), list)
  132.         self.assertEqual(len(results), 2)
  133.         self.assertIn('The Title is...', results[1]['content'])
  134.         self.assertEqual(results[1]['img_src'], None)
  136.         # title not in first paragraph
  137.         json = u"""
  138.         {
  139.             "batchcomplete": "",
  140.             "query": {
  141.                 "normalized": [],
  142.                 "pages": {
  143.                     "12345": {
  144.                         "pageid": 12345,
  145.                         "ns": 0,
  146.                         "title": "披頭四樂隊",
  147.                         "extract": "披头士乐队....................................................................................................................................................................................................\\n披頭四樂隊...", """  # noqa
  148.         json += """
  149.                         "thumbnail": {
  150.                             "source": "img_src.jpg"
  151.                         },
  152.                         "pageimage": "img_name.jpg"
  153.                     }
  154.                 }
  155.             }
  156.         }"""
  157.         response = mock.Mock(text=json, search_params=dicto)
  158.         results = wikipedia.response(response)
  159.         self.assertEqual(type(results), list)
  160.         self.assertEqual(len(results), 2)
  161.         self.assertEqual(results[1]['infobox'], u'披頭四樂隊')
  162.         self.assertIn(u'披头士乐队...', results[1]['content'])
  164.     def test_fetch_supported_languages(self):
  165.         html = u"""<html></html>"""
  166.         response = mock.Mock(text=html)
  167.         languages = wikipedia._fetch_supported_languages(response)
  168.         self.assertEqual(type(languages), dict)
  169.         self.assertEqual(len(languages), 0)
  171.         html = u"""
  172.         <html>
  173.             <body>
  174.                 <div>
  175.                     <div>
  176.                         <h3>Table header</h3>
  177.                         <table class="sortable jquery-tablesorter">
  178.                             <thead>
  179.                                 <tr>
  180.                                     <th>N</th>
  181.                                     <th>Language</th>
  182.                                     <th>Language (local)</th>
  183.                                     <th>Wiki</th>
  184.                                     <th>Articles</th>
  185.                                 </tr>
  186.                             </thead>
  187.                             <tbody>
  188.                                 <tr>
  189.                                     <td>2</td>
  190.                                     <td><a>Swedish</a></td>
  191.                                     <td><a>Svenska</a></td>
  192.                                     <td><a>sv</a></td>
  193.                                     <td><a><b>3000000</b></a></td>
  194.                                 </tr>
  195.                                 <tr>
  196.                                     <td>3</td>
  197.                                     <td><a>Cebuano</a></td>
  198.                                     <td><a>Sinugboanong Binisaya</a></td>
  199.                                     <td><a>ceb</a></td>
  200.                                     <td><a><b>3000000</b></a></td>
  201.                                 </tr>
  202.                             </tbody>
  203.                         </table>
  204.                         <h3>Table header</h3>
  205.                         <table class="sortable jquery-tablesorter">
  206.                             <thead>
  207.                                 <tr>
  208.                                     <th>N</th>
  209.                                     <th>Language</th>
  210.                                     <th>Language (local)</th>
  211.                                     <th>Wiki</th>
  212.                                     <th>Articles</th>
  213.                                 </tr>
  214.                             </thead>
  215.                             <tbody>
  216.                                 <tr>
  217.                                     <td>2</td>
  218.                                     <td><a>Norwegian (Bokmål)</a></td>
  219.                                     <td><a>Norsk (Bokmål)</a></td>
  220.                                     <td><a>no</a></td>
  221.                                     <td><a><b>100000</b></a></td>
  222.                                 </tr>
  223.                             </tbody>
  224.                         </table>
  225.                     </div>
  226.                 </div>
  227.             </body>
  228.         </html>
  229.         """
  230.         response = mock.Mock(text=html)
  231.         languages = wikipedia._fetch_supported_languages(response)
  232.         self.assertEqual(type(languages), dict)
  233.         self.assertEqual(len(languages), 3)
  235.         self.assertIn('sv', languages)
  236.         self.assertIn('ceb', languages)
  237.         self.assertIn('no', languages)
  239.         self.assertEqual(type(languages['sv']), dict)
  240.         self.assertEqual(type(languages['ceb']), dict)
  241.         self.assertEqual(type(languages['no']), dict)
  243.         self.assertIn('name', languages['sv'])
  244.         self.assertIn('english_name', languages['sv'])
  245.         self.assertIn('articles', languages['sv'])
  247.         self.assertEqual(languages['sv']['name'], 'Svenska')
  248.         self.assertEqual(languages['sv']['english_name'], 'Swedish')
  249.         self.assertEqual(languages['sv']['articles'], 3000000)
  250.         self.assertEqual(languages['ceb']['name'], 'Sinugboanong Binisaya')
  251.         self.assertEqual(languages['ceb']['english_name'], 'Cebuano')
  252.         self.assertEqual(languages['ceb']['articles'], 3000000)
  253.         self.assertEqual(languages['no']['name'], u'Norsk (Bokmål)')
  254.         self.assertEqual(languages['no']['english_name'], u'Norwegian (Bokmål)')
  255.         self.assertEqual(languages['no']['articles'], 100000)