logo

searx

My custom branche(s) on searx, a meta-search engine git clone https://hacktivis.me/git/searx.git

test_wikipedia.py (10255B)


  1. # -*- coding: utf-8 -*-
  2. from collections import defaultdict
  3. import mock
  4. from searx.engines import wikipedia
  5. from searx.testing import SearxTestCase
  6. class TestWikipediaEngine(SearxTestCase):
  7. def test_request(self):
  8. wikipedia.supported_languages = ['fr', 'en']
  9. query = 'test_query'
  10. dicto = defaultdict(dict)
  11. dicto['language'] = 'fr-FR'
  12. params = wikipedia.request(query.encode('utf-8'), dicto)
  13. self.assertIn('url', params)
  14. self.assertIn(query, params['url'])
  15. self.assertIn('test_query', params['url'])
  16. self.assertIn('Test_Query', params['url'])
  17. self.assertIn('fr.wikipedia.org', params['url'])
  18. query = u'Test_Query'
  19. params = wikipedia.request(query.encode('utf-8'), dicto)
  20. self.assertIn('Test_Query', params['url'])
  21. self.assertNotIn('test_query', params['url'])
  22. dicto['language'] = 'xx'
  23. params = wikipedia.request(query, dicto)
  24. self.assertIn('en', params['url'])
  25. def test_response(self):
  26. dicto = defaultdict(dict)
  27. dicto['language'] = 'fr'
  28. self.assertRaises(AttributeError, wikipedia.response, None)
  29. self.assertRaises(AttributeError, wikipedia.response, [])
  30. self.assertRaises(AttributeError, wikipedia.response, '')
  31. self.assertRaises(AttributeError, wikipedia.response, '[]')
  32. # page not found
  33. json = """
  34. {
  35. "batchcomplete": "",
  36. "query": {
  37. "normalized": [],
  38. "pages": {
  39. "-1": {
  40. "ns": 0,
  41. "title": "",
  42. "missing": ""
  43. }
  44. }
  45. }
  46. }"""
  47. response = mock.Mock(text=json, search_params=dicto)
  48. self.assertEqual(wikipedia.response(response), [])
  49. # normal case
  50. json = """
  51. {
  52. "batchcomplete": "",
  53. "query": {
  54. "normalized": [],
  55. "pages": {
  56. "12345": {
  57. "pageid": 12345,
  58. "ns": 0,
  59. "title": "The Title",
  60. "extract": "The Title is...",
  61. "thumbnail": {
  62. "source": "img_src.jpg"
  63. },
  64. "pageimage": "img_name.jpg"
  65. }
  66. }
  67. }
  68. }"""
  69. response = mock.Mock(text=json, search_params=dicto)
  70. results = wikipedia.response(response)
  71. self.assertEqual(type(results), list)
  72. self.assertEqual(len(results), 2)
  73. self.assertEqual(results[0]['title'], u'The Title')
  74. self.assertIn('fr.wikipedia.org/wiki/The_Title', results[0]['url'])
  75. self.assertEqual(results[1]['infobox'], u'The Title')
  76. self.assertIn('fr.wikipedia.org/wiki/The_Title', results[1]['id'])
  77. self.assertIn('The Title is...', results[1]['content'])
  78. self.assertEqual(results[1]['img_src'], 'img_src.jpg')
  79. # disambiguation page
  80. json = """
  81. {
  82. "batchcomplete": "",
  83. "query": {
  84. "normalized": [],
  85. "pages": {
  86. "12345": {
  87. "pageid": 12345,
  88. "ns": 0,
  89. "title": "The Title",
  90. "extract": "The Title can be:\\nThe Title 1\\nThe Title 2\\nThe Title 3\\nThe Title 4......................................................................................................................................." """ # noqa
  91. json += """
  92. }
  93. }
  94. }
  95. }"""
  96. response = mock.Mock(text=json, search_params=dicto)
  97. results = wikipedia.response(response)
  98. self.assertEqual(type(results), list)
  99. self.assertEqual(len(results), 2)
  100. # no image
  101. json = """
  102. {
  103. "batchcomplete": "",
  104. "query": {
  105. "normalized": [],
  106. "pages": {
  107. "12345": {
  108. "pageid": 12345,
  109. "ns": 0,
  110. "title": "The Title",
  111. "extract": "The Title is......................................................................................................................................................................................." """ # noqa
  112. json += """
  113. }
  114. }
  115. }
  116. }"""
  117. response = mock.Mock(text=json, search_params=dicto)
  118. results = wikipedia.response(response)
  119. self.assertEqual(type(results), list)
  120. self.assertEqual(len(results), 2)
  121. self.assertIn('The Title is...', results[1]['content'])
  122. self.assertEqual(results[1]['img_src'], None)
  123. # title not in first paragraph
  124. json = u"""
  125. {
  126. "batchcomplete": "",
  127. "query": {
  128. "normalized": [],
  129. "pages": {
  130. "12345": {
  131. "pageid": 12345,
  132. "ns": 0,
  133. "title": "披頭四樂隊",
  134. "extract": "披头士乐队....................................................................................................................................................................................................\\n披頭四樂隊...", """ # noqa
  135. json += """
  136. "thumbnail": {
  137. "source": "img_src.jpg"
  138. },
  139. "pageimage": "img_name.jpg"
  140. }
  141. }
  142. }
  143. }"""
  144. response = mock.Mock(text=json, search_params=dicto)
  145. results = wikipedia.response(response)
  146. self.assertEqual(type(results), list)
  147. self.assertEqual(len(results), 2)
  148. self.assertEqual(results[1]['infobox'], u'披頭四樂隊')
  149. self.assertIn(u'披头士乐队...', results[1]['content'])
  150. def test_fetch_supported_languages(self):
  151. html = u"""<html></html>"""
  152. response = mock.Mock(text=html)
  153. languages = wikipedia._fetch_supported_languages(response)
  154. self.assertEqual(type(languages), dict)
  155. self.assertEqual(len(languages), 0)
  156. html = u"""
  157. <html>
  158. <body>
  159. <div>
  160. <div>
  161. <h3>Table header</h3>
  162. <table class="sortable jquery-tablesorter">
  163. <thead>
  164. <tr>
  165. <th>N</th>
  166. <th>Language</th>
  167. <th>Language (local)</th>
  168. <th>Wiki</th>
  169. <th>Articles</th>
  170. </tr>
  171. </thead>
  172. <tbody>
  173. <tr>
  174. <td>2</td>
  175. <td><a>Swedish</a></td>
  176. <td><a>Svenska</a></td>
  177. <td><a>sv</a></td>
  178. <td><a><b>3000000</b></a></td>
  179. </tr>
  180. <tr>
  181. <td>3</td>
  182. <td><a>Cebuano</a></td>
  183. <td><a>Sinugboanong Binisaya</a></td>
  184. <td><a>ceb</a></td>
  185. <td><a><b>3000000</b></a></td>
  186. </tr>
  187. </tbody>
  188. </table>
  189. <h3>Table header</h3>
  190. <table class="sortable jquery-tablesorter">
  191. <thead>
  192. <tr>
  193. <th>N</th>
  194. <th>Language</th>
  195. <th>Language (local)</th>
  196. <th>Wiki</th>
  197. <th>Articles</th>
  198. </tr>
  199. </thead>
  200. <tbody>
  201. <tr>
  202. <td>2</td>
  203. <td><a>Norwegian (Bokmål)</a></td>
  204. <td><a>Norsk (Bokmål)</a></td>
  205. <td><a>no</a></td>
  206. <td><a><b>100000</b></a></td>
  207. </tr>
  208. </tbody>
  209. </table>
  210. </div>
  211. </div>
  212. </body>
  213. </html>
  214. """
  215. response = mock.Mock(text=html)
  216. languages = wikipedia._fetch_supported_languages(response)
  217. self.assertEqual(type(languages), dict)
  218. self.assertEqual(len(languages), 3)
  219. self.assertIn('sv', languages)
  220. self.assertIn('ceb', languages)
  221. self.assertIn('no', languages)
  222. self.assertEqual(type(languages['sv']), dict)
  223. self.assertEqual(type(languages['ceb']), dict)
  224. self.assertEqual(type(languages['no']), dict)
  225. self.assertIn('name', languages['sv'])
  226. self.assertIn('english_name', languages['sv'])
  227. self.assertIn('articles', languages['sv'])
  228. self.assertEqual(languages['sv']['name'], 'Svenska')
  229. self.assertEqual(languages['sv']['english_name'], 'Swedish')
  230. self.assertEqual(languages['sv']['articles'], 3000000)
  231. self.assertEqual(languages['ceb']['name'], 'Sinugboanong Binisaya')
  232. self.assertEqual(languages['ceb']['english_name'], 'Cebuano')
  233. self.assertEqual(languages['ceb']['articles'], 3000000)
  234. self.assertEqual(languages['no']['name'], u'Norsk (Bokmål)')
  235. self.assertEqual(languages['no']['english_name'], u'Norwegian (Bokmål)')
  236. self.assertEqual(languages['no']['articles'], 100000)