logo

searx

My custom branche(s) on searx, a meta-search engine git clone https://hacktivis.me/git/searx.git

test_wikidata.py (21170B)

    

  1. # -*- coding: utf-8 -*-

  2. from lxml.html import fromstring

  3. from collections import defaultdict

  4. import mock

  5. from searx.engines import wikidata

  6. from searx.testing import SearxTestCase

  7. 

  8. 

  9. class TestWikidataEngine(SearxTestCase):

  10. 

  11.     def test_request(self):

  12.         query = 'test_query'

  13.         dicto = defaultdict(dict)

  14.         params = wikidata.request(query, dicto)

  15.         self.assertIn('url', params)

  16.         self.assertIn(query, params['url'])

  17.         self.assertIn('wikidata.org', params['url'])

  18. 

  19.         params = wikidata.request(query, dicto)

  20.         self.assertIn(query, params['url'])

  21. 

  22.     # successful cases are not tested here to avoid sending additional requests

  23.     def test_response(self):

  24.         self.assertRaises(AttributeError, wikidata.response, None)

  25.         self.assertRaises(AttributeError, wikidata.response, [])

  26.         self.assertRaises(AttributeError, wikidata.response, '')

  27.         self.assertRaises(AttributeError, wikidata.response, '[]')

  28. 

  29.         wikidata.supported_languages = ['en', 'es']

  30.         response = mock.Mock(text='<html></html>', search_params={"language": "en"})

  31.         self.assertEqual(wikidata.response(response), [])

  32. 

  33.     def test_getDetail(self):

  34.         response = {}

  35.         results = wikidata.getDetail(response, "Q123", "en", "en-US")

  36.         self.assertEqual(results, [])

  37. 

  38.         title_html = '<div><div class="wikibase-title-label">Test</div></div>'

  39.         html = """

  40.         <div>

  41.             <div class="wikibase-entitytermsview-heading-description">

  42.             </div>

  43.             <div>

  44.                 <ul class="wikibase-sitelinklistview-listview">

  45.                     <li data-wb-siteid="enwiki"><a href="http://en.wikipedia.org/wiki/Test">Test</a></li>

  46.                 </ul>

  47.             </div>

  48.         </div>

  49.         """

  50.         response = {"parse": {"displaytitle": title_html, "text": html}}

  51. 

  52.         results = wikidata.getDetail(response, "Q123", "en", "en-US")

  53.         self.assertEqual(len(results), 1)

  54.         self.assertEqual(results[0]['url'], 'https://en.wikipedia.org/wiki/Test')

  55. 

  56.         title_html = """

  57.         <div>

  58.             <div class="wikibase-title-label">

  59.                 <span lang="en">Test</span>

  60.                 <sup class="wb-language-fallback-indicator">English</sup>

  61.             </div>

  62.         </div>

  63.         """

  64.         html = """

  65.         <div>

  66.             <div class="wikibase-entitytermsview-heading-description">

  67.                 <span lang="en">Description</span>

  68.                 <sup class="wb-language-fallback-indicator">English</sup>

  69.             </div>

  70.             <div id="P856">

  71.                 <div class="wikibase-statementgroupview-property-label">

  72.                     <a href="/wiki/Property:P856">

  73.                         <span lang="en">official website</span>

  74.                         <sup class="wb-language-fallback-indicator">English</sup>

  75.                     </a>

  76.                 </div>

  77.                 <div class="wikibase-statementview-mainsnak">

  78.                     <a class="external free" href="https://officialsite.com">

  79.                         https://officialsite.com

  80.                     </a>

  81.                 </div>

  82.             </div>

  83.             <div>

  84.                 <ul class="wikibase-sitelinklistview-listview">

  85.                     <li data-wb-siteid="enwiki"><a href="http://en.wikipedia.org/wiki/Test">Test</a></li>

  86.                 </ul>

  87.             </div>

  88.         </div>

  89.         """

  90.         response = {"parse": {"displaytitle": title_html, "text": html}}

  91. 

  92.         results = wikidata.getDetail(response, "Q123", "yua", "yua_MX")

  93.         self.assertEqual(len(results), 2)

  94.         self.assertEqual(results[0]['title'], 'Official website')

  95.         self.assertEqual(results[0]['url'], 'https://officialsite.com')

  96. 

  97.         self.assertEqual(results[1]['infobox'], 'Test')

  98.         self.assertEqual(results[1]['id'], None)

  99.         self.assertEqual(results[1]['content'], 'Description')

  100.         self.assertEqual(results[1]['attributes'], [])

  101.         self.assertEqual(results[1]['urls'][0]['title'], 'Official website')

  102.         self.assertEqual(results[1]['urls'][0]['url'], 'https://officialsite.com')

  103.         self.assertEqual(results[1]['urls'][1]['title'], 'Wikipedia (en)')

  104.         self.assertEqual(results[1]['urls'][1]['url'], 'https://en.wikipedia.org/wiki/Test')

  105. 

  106.     def test_add_image(self):

  107.         image_src = wikidata.add_image(fromstring("<div></div>"))

  108.         self.assertEqual(image_src, None)

  109. 

  110.         html = u"""

  111.         <div>

  112.             <div id="P18">

  113.                 <div class="wikibase-statementgroupview-property-label">

  114.                     <a href="/wiki/Property:P18">

  115.                         image

  116.                     </a>

  117.                 </div>

  118.                 <div class="wikibase-statementlistview">

  119.                     <div class="wikibase-statementview listview-item">

  120.                         <div class="wikibase-statementview-rankselector">

  121.                             <span class="wikibase-rankselector-normal"></span>

  122.                         </div>

  123.                         <div class="wikibase-statementview-mainsnak">

  124.                             <div>

  125.                                 <div class="wikibase-snakview-value">

  126.                                     <div class="commons-media-caption">

  127.                                         <a href="https://commons.wikimedia.org/wiki/File:image.png">image.png</a>

  128.                                         <br/>2,687 &#215; 3,356; 1.22 MB

  129.                                     </div>

  130.                                 </div>

  131.                             </div>

  132.                         </div>

  133.                     </div>

  134.                 </div>

  135.             </div>

  136.         </div>

  137.         """

  138.         html_etree = fromstring(html)

  139. 

  140.         image_src = wikidata.add_image(html_etree)

  141.         self.assertEqual(image_src,

  142.                          "https://commons.wikimedia.org/wiki/Special:FilePath/image.png?width=500&height=400")

  143. 

  144.         html = u"""

  145.         <div>

  146.             <div id="P2910">

  147.                 <div class="wikibase-statementgroupview-property-label">

  148.                     <a href="/wiki/Property:P2910">

  149.                         icon

  150.                     </a>

  151.                 </div>

  152.                 <div class="wikibase-statementlistview">

  153.                     <div class="wikibase-statementview listview-item">

  154.                         <div class="wikibase-statementview-rankselector">

  155.                             <span class="wikibase-rankselector-normal"></span>

  156.                         </div>

  157.                         <div class="wikibase-statementview-mainsnak">

  158.                             <div>

  159.                                 <div class="wikibase-snakview-value">

  160.                                     <div class="commons-media-caption">

  161.                                         <a href="https://commons.wikimedia.org/wiki/File:icon.png">icon.png</a>

  162.                                         <br/>671 &#215; 671; 18 KB</div>

  163.                                     </div>

  164.                                 </div>

  165.                             </div>

  166.                         </div>

  167.                     </div>

  168.                 </div>

  169.             </div>

  170.             <div id="P154">

  171.                 <div class="wikibase-statementgroupview-property-label">

  172.                     <a href="/wiki/Property:P154">

  173.                         logo

  174.                     </a>

  175.                 </div>

  176.                 <div class="wikibase-statementlistview">

  177.                     <div class="wikibase-statementview listview-item">

  178.                         <div class="wikibase-statementview-rankselector">

  179.                             <span class="wikibase-rankselector-normal"></span>

  180.                         </div>

  181.                         <div class="wikibase-statementview-mainsnak">

  182.                             <div>

  183.                                 <div class="wikibase-snakview-value">

  184.                                     <div class="commons-media-caption">

  185.                                         <a href="https://commons.wikimedia.org/wiki/File:logo.png">logo.png</a>

  186.                                         <br/>170 &#215; 170; 1 KB

  187.                                     </div>

  188.                                 </div>

  189.                             </div>

  190.                         </div>

  191.                     </div>

  192.                 </div>

  193.             </div>

  194.         </div>

  195.         """

  196.         html_etree = fromstring(html)

  197. 

  198.         image_src = wikidata.add_image(html_etree)

  199.         self.assertEqual(image_src,

  200.                          "https://commons.wikimedia.org/wiki/Special:FilePath/logo.png?width=500&height=400")

  201. 

  202.     def test_add_attribute(self):

  203.         html = u"""

  204.         <div>

  205.             <div id="P27">

  206.                 <div class="wikibase-statementgroupview-property-label">

  207.                     <a href="/wiki/Property:P27">

  208.                         country of citizenship

  209.                     </a>

  210.                 </div>

  211.                 <div class="wikibase-statementlistview">

  212.                     <div class="wikibase-statementview listview-item">

  213.                         <div class="wikibase-statementview-rankselector">

  214.                             <span class="wikibase-rankselector-normal"></span>

  215.                         </div>

  216.                         <div class="wikibase-statementview-mainsnak">

  217.                             <div>

  218.                                 <div class="wikibase-snakview-value">

  219.                                     <a href="/wiki/Q145">

  220.                                         United Kingdom

  221.                                     </a>

  222.                                 </div>

  223.                             </div>

  224.                         </div>

  225.                     </div>

  226.                 </div>

  227.             </div>

  228.         </div>

  229.         """

  230.         attributes = []

  231.         html_etree = fromstring(html)

  232. 

  233.         wikidata.add_attribute(attributes, html_etree, "Fail")

  234.         self.assertEqual(attributes, [])

  235. 

  236.         wikidata.add_attribute(attributes, html_etree, "P27")

  237.         self.assertEqual(len(attributes), 1)

  238.         self.assertEqual(attributes[0]["label"], "Country of citizenship")

  239.         self.assertEqual(attributes[0]["value"], "United Kingdom")

  240. 

  241.         html = u"""

  242.         <div>

  243.             <div id="P569">

  244.                 <div class="wikibase-statementgroupview-property-label">

  245.                     <a href="/wiki/Property:P569">

  246.                         date of birth

  247.                     </a>

  248.                 </div>

  249.                 <div class="wikibase-statementlistview">

  250.                     <div class="wikibase-statementview listview-item">

  251.                         <div class="wikibase-statementview-rankselector">

  252.                             <span class="wikibase-rankselector-normal"></span>

  253.                         </div>

  254.                         <div class="wikibase-statementview-mainsnak">

  255.                             <div>

  256.                                 <div class="wikibase-snakview-value">

  257.                                     27 January 1832

  258.                                     <sup class="wb-calendar-name">

  259.                                         Gregorian

  260.                                     </sup>

  261.                                 </div>

  262.                             </div>

  263.                         </div>

  264.                     </div>

  265.                 </div>

  266.             </div>

  267.         </div>

  268.         """

  269.         attributes = []

  270.         html_etree = fromstring(html)

  271.         wikidata.add_attribute(attributes, html_etree, "P569", date=True)

  272.         self.assertEqual(len(attributes), 1)

  273.         self.assertEqual(attributes[0]["label"], "Date of birth")

  274.         self.assertEqual(attributes[0]["value"], "27 January 1832")

  275. 

  276.         html = u"""

  277.         <div>

  278.             <div id="P6">

  279.                 <div class="wikibase-statementgroupview-property-label">

  280.                     <a href="/wiki/Property:P27">

  281.                         head of government

  282.                     </a>

  283.                 </div>

  284.                 <div class="wikibase-statementlistview">

  285.                     <div class="wikibase-statementview listview-item">

  286.                         <div class="wikibase-statementview-rankselector">

  287.                             <span class="wikibase-rankselector-normal"></span>

  288.                         </div>

  289.                         <div class="wikibase-statementview-mainsnak">

  290.                             <div>

  291.                                 <div class="wikibase-snakview-value">

  292.                                     <a href="/wiki/Q206">

  293.                                         Old Prime Minister

  294.                                     </a>

  295.                                 </div>

  296.                             </div>

  297.                         </div>

  298.                     </div>

  299.                     <div class="wikibase-statementview listview-item">

  300.                         <div class="wikibase-statementview-rankselector">

  301.                             <span class="wikibase-rankselector-preferred"></span>

  302.                         </div>

  303.                         <div class="wikibase-statementview-mainsnak">

  304.                             <div>

  305.                                 <div class="wikibase-snakview-value">

  306.                                     <a href="/wiki/Q3099714">

  307.                                         Actual Prime Minister

  308.                                     </a>

  309.                                 </div>

  310.                             </div>

  311.                         </div>

  312.                     </div>

  313.                 </div>

  314.             </div>

  315.         </div>

  316.         """

  317.         attributes = []

  318.         html_etree = fromstring(html)

  319.         wikidata.add_attribute(attributes, html_etree, "P6")

  320.         self.assertEqual(len(attributes), 1)

  321.         self.assertEqual(attributes[0]["label"], "Head of government")

  322.         self.assertEqual(attributes[0]["value"], "Old Prime Minister, Actual Prime Minister")

  323. 

  324.         attributes = []

  325.         html_etree = fromstring(html)

  326.         wikidata.add_attribute(attributes, html_etree, "P6", trim=True)

  327.         self.assertEqual(len(attributes), 1)

  328.         self.assertEqual(attributes[0]["value"], "Actual Prime Minister")

  329. 

  330.     def test_add_url(self):

  331.         html = u"""

  332.         <div>

  333.             <div id="P856">

  334.                 <div class="wikibase-statementgroupview-property-label">

  335.                     <a href="/wiki/Property:P856">

  336.                         official website

  337.                     </a>

  338.                 </div>

  339.                 <div class="wikibase-statementlistview">

  340.                     <div class="wikibase-statementview listview-item">

  341.                         <div class="wikibase-statementview-mainsnak">

  342.                             <div>

  343.                                 <div class="wikibase-snakview-value">

  344.                                     <a class="external free" href="https://searx.me">

  345.                                         https://searx.me/

  346.                                     </a>

  347.                                 </div>

  348.                             </div>

  349.                         </div>

  350.                     </div>

  351.                 </div>

  352.             </div>

  353.         </div>

  354.         """

  355.         urls = []

  356.         html_etree = fromstring(html)

  357.         wikidata.add_url(urls, html_etree, 'P856')

  358.         self.assertEquals(len(urls), 1)

  359.         self.assertIn({'title': 'Official website', 'url': 'https://searx.me/'}, urls)

  360.         urls = []

  361.         results = []

  362.         wikidata.add_url(urls, html_etree, 'P856', 'custom label', results=results)

  363.         self.assertEquals(len(urls), 1)

  364.         self.assertEquals(len(results), 1)

  365.         self.assertIn({'title': 'custom label', 'url': 'https://searx.me/'}, urls)

  366.         self.assertIn({'title': 'custom label', 'url': 'https://searx.me/'}, results)

  367. 

  368.         html = u"""

  369.         <div>

  370.             <div id="P856">

  371.                 <div class="wikibase-statementgroupview-property-label">

  372.                     <a href="/wiki/Property:P856">

  373.                         official website

  374.                     </a>

  375.                 </div>

  376.                 <div class="wikibase-statementlistview">

  377.                     <div class="wikibase-statementview listview-item">

  378.                         <div class="wikibase-statementview-mainsnak">

  379.                             <div>

  380.                                 <div class="wikibase-snakview-value">

  381.                                     <a class="external free" href="http://www.worldofwarcraft.com">

  382.                                         http://www.worldofwarcraft.com

  383.                                     </a>

  384.                                 </div>

  385.                             </div>

  386.                         </div>

  387.                     </div>

  388.                     <div class="wikibase-statementview listview-item">

  389.                         <div class="wikibase-statementview-mainsnak">

  390.                             <div>

  391.                                 <div class="wikibase-snakview-value">

  392.                                     <a class="external free" href="http://eu.battle.net/wow/en/">

  393.                                         http://eu.battle.net/wow/en/

  394.                                     </a>

  395.                                 </div>

  396.                             </div>

  397.                         </div>

  398.                     </div>

  399.                 </div>

  400.             </div>

  401.         </div>

  402.         """

  403.         urls = []

  404.         html_etree = fromstring(html)

  405.         wikidata.add_url(urls, html_etree, 'P856')

  406.         self.assertEquals(len(urls), 2)

  407.         self.assertIn({'title': 'Official website', 'url': 'http://www.worldofwarcraft.com'}, urls)

  408.         self.assertIn({'title': 'Official website', 'url': 'http://eu.battle.net/wow/en/'}, urls)

  409. 

  410.     def test_get_imdblink(self):

  411.         html = u"""

  412.         <div>

  413.             <div class="wikibase-statementview-mainsnak">

  414.                 <div>

  415.                     <div class="wikibase-snakview-value">

  416.                         <a class="wb-external-id" href="http://www.imdb.com/tt0433664">

  417.                             tt0433664

  418.                         </a>

  419.                     </div>

  420.                 </div>

  421.             </div>

  422.         </div>

  423.         """

  424.         html_etree = fromstring(html)

  425.         imdblink = wikidata.get_imdblink(html_etree, 'https://www.imdb.com/')

  426. 

  427.         html = u"""

  428.         <div>

  429.             <div class="wikibase-statementview-mainsnak">

  430.                 <div>

  431.                     <div class="wikibase-snakview-value">

  432.                         <a class="wb-external-id"

  433.                            href="href="http://tools.wmflabs.org/...http://www.imdb.com/&id=nm4915994"">

  434.                             nm4915994

  435.                         </a>

  436.                     </div>

  437.                 </div>

  438.             </div>

  439.         </div>

  440.         """

  441.         html_etree = fromstring(html)

  442.         imdblink = wikidata.get_imdblink(html_etree, 'https://www.imdb.com/')

  443.         self.assertIn('https://www.imdb.com/name/nm4915994', imdblink)

  444. 

  445.     def test_get_geolink(self):

  446.         html = u"""

  447.         <div>

  448.             <div class="wikibase-statementview-mainsnak">

  449.                 <div>

  450.                     <div class="wikibase-snakview-value">

  451.                         60°N, 40°E

  452.                     </div>

  453.                 </div>

  454.             </div>

  455.         </div>

  456.         """

  457.         html_etree = fromstring(html)

  458.         geolink = wikidata.get_geolink(html_etree)

  459.         self.assertIn('https://www.openstreetmap.org/', geolink)

  460.         self.assertIn('lat=60&lon=40', geolink)

  461. 

  462.         html = u"""

  463.         <div>

  464.             <div class="wikibase-statementview-mainsnak">

  465.                 <div>

  466.                     <div class="wikibase-snakview-value">

  467.                         34°35'59"S, 58°22'55"W

  468.                     </div>

  469.                 </div>

  470.             </div>

  471.         </div>

  472.         """

  473.         html_etree = fromstring(html)

  474.         geolink = wikidata.get_geolink(html_etree)

  475.         self.assertIn('https://www.openstreetmap.org/', geolink)

  476.         self.assertIn('lat=-34.59', geolink)

  477.         self.assertIn('lon=-58.38', geolink)

  478. 

  479.     def test_get_wikilink(self):

  480.         html = """

  481.         <div>

  482.             <div>

  483.                 <ul class="wikibase-sitelinklistview-listview">

  484.                     <li data-wb-siteid="arwiki"><a href="http://ar.wikipedia.org/wiki/Test">Test</a></li>

  485.                     <li data-wb-siteid="enwiki"><a href="http://en.wikipedia.org/wiki/Test">Test</a></li>

  486.                 </ul>

  487.             </div>

  488.             <div>

  489.                 <ul class="wikibase-sitelinklistview-listview">

  490.                     <li data-wb-siteid="enwikiquote"><a href="https://en.wikiquote.org/wiki/Test">Test</a></li>

  491.                 </ul>

  492.             </div>

  493.         </div>

  494.         """

  495.         html_etree = fromstring(html)

  496.         wikilink = wikidata.get_wikilink(html_etree, 'nowiki')

  497.         self.assertEqual(wikilink, None)

  498.         wikilink = wikidata.get_wikilink(html_etree, 'enwiki')

  499.         self.assertEqual(wikilink, 'https://en.wikipedia.org/wiki/Test')

  500.         wikilink = wikidata.get_wikilink(html_etree, 'arwiki')

  501.         self.assertEqual(wikilink, 'https://ar.wikipedia.org/wiki/Test')

  502.         wikilink = wikidata.get_wikilink(html_etree, 'enwikiquote')

  503.         self.assertEqual(wikilink, 'https://en.wikiquote.org/wiki/Test')