logo

searx

My custom branche(s) on searx, a meta-search engine git clone https://anongit.hacktivis.me/git/searx.git/

test_wikidata.py (21170B)

    

  1. # -*- coding: utf-8 -*-
  2. from lxml.html import fromstring
  3. from collections import defaultdict
  4. import mock
  5. from searx.engines import wikidata
  6. from searx.testing import SearxTestCase
  9. class TestWikidataEngine(SearxTestCase):
  11.     def test_request(self):
  12.         query = 'test_query'
  13.         dicto = defaultdict(dict)
  14.         params = wikidata.request(query, dicto)
  15.         self.assertIn('url', params)
  16.         self.assertIn(query, params['url'])
  17.         self.assertIn('wikidata.org', params['url'])
  19.         params = wikidata.request(query, dicto)
  20.         self.assertIn(query, params['url'])
  22.     # successful cases are not tested here to avoid sending additional requests
  23.     def test_response(self):
  24.         self.assertRaises(AttributeError, wikidata.response, None)
  25.         self.assertRaises(AttributeError, wikidata.response, [])
  26.         self.assertRaises(AttributeError, wikidata.response, '')
  27.         self.assertRaises(AttributeError, wikidata.response, '[]')
  29.         wikidata.supported_languages = ['en', 'es']
  30.         response = mock.Mock(text='<html></html>', search_params={"language": "en"})
  31.         self.assertEqual(wikidata.response(response), [])
  33.     def test_getDetail(self):
  34.         response = {}
  35.         results = wikidata.getDetail(response, "Q123", "en", "en-US")
  36.         self.assertEqual(results, [])
  38.         title_html = '<div><div class="wikibase-title-label">Test</div></div>'
  39.         html = """
  40.         <div>
  41.             <div class="wikibase-entitytermsview-heading-description">
  42.             </div>
  43.             <div>
  44.                 <ul class="wikibase-sitelinklistview-listview">
  45.                     <li data-wb-siteid="enwiki"><a href="http://en.wikipedia.org/wiki/Test">Test</a></li>
  46.                 </ul>
  47.             </div>
  48.         </div>
  49.         """
  50.         response = {"parse": {"displaytitle": title_html, "text": html}}
  52.         results = wikidata.getDetail(response, "Q123", "en", "en-US")
  53.         self.assertEqual(len(results), 1)
  54.         self.assertEqual(results[0]['url'], 'https://en.wikipedia.org/wiki/Test')
  56.         title_html = """
  57.         <div>
  58.             <div class="wikibase-title-label">
  59.                 <span lang="en">Test</span>
  60.                 <sup class="wb-language-fallback-indicator">English</sup>
  61.             </div>
  62.         </div>
  63.         """
  64.         html = """
  65.         <div>
  66.             <div class="wikibase-entitytermsview-heading-description">
  67.                 <span lang="en">Description</span>
  68.                 <sup class="wb-language-fallback-indicator">English</sup>
  69.             </div>
  70.             <div id="P856">
  71.                 <div class="wikibase-statementgroupview-property-label">
  72.                     <a href="/wiki/Property:P856">
  73.                         <span lang="en">official website</span>
  74.                         <sup class="wb-language-fallback-indicator">English</sup>
  75.                     </a>
  76.                 </div>
  77.                 <div class="wikibase-statementview-mainsnak">
  78.                     <a class="external free" href="https://officialsite.com">
  79.                         https://officialsite.com
  80.                     </a>
  81.                 </div>
  82.             </div>
  83.             <div>
  84.                 <ul class="wikibase-sitelinklistview-listview">
  85.                     <li data-wb-siteid="enwiki"><a href="http://en.wikipedia.org/wiki/Test">Test</a></li>
  86.                 </ul>
  87.             </div>
  88.         </div>
  89.         """
  90.         response = {"parse": {"displaytitle": title_html, "text": html}}
  92.         results = wikidata.getDetail(response, "Q123", "yua", "yua_MX")
  93.         self.assertEqual(len(results), 2)
  94.         self.assertEqual(results[0]['title'], 'Official website')
  95.         self.assertEqual(results[0]['url'], 'https://officialsite.com')
  97.         self.assertEqual(results[1]['infobox'], 'Test')
  98.         self.assertEqual(results[1]['id'], None)
  99.         self.assertEqual(results[1]['content'], 'Description')
  100.         self.assertEqual(results[1]['attributes'], [])
  101.         self.assertEqual(results[1]['urls'][0]['title'], 'Official website')
  102.         self.assertEqual(results[1]['urls'][0]['url'], 'https://officialsite.com')
  103.         self.assertEqual(results[1]['urls'][1]['title'], 'Wikipedia (en)')
  104.         self.assertEqual(results[1]['urls'][1]['url'], 'https://en.wikipedia.org/wiki/Test')
  106.     def test_add_image(self):
  107.         image_src = wikidata.add_image(fromstring("<div></div>"))
  108.         self.assertEqual(image_src, None)
  110.         html = u"""
  111.         <div>
  112.             <div id="P18">
  113.                 <div class="wikibase-statementgroupview-property-label">
  114.                     <a href="/wiki/Property:P18">
  115.                         image
  116.                     </a>
  117.                 </div>
  118.                 <div class="wikibase-statementlistview">
  119.                     <div class="wikibase-statementview listview-item">
  120.                         <div class="wikibase-statementview-rankselector">
  121.                             <span class="wikibase-rankselector-normal"></span>
  122.                         </div>
  123.                         <div class="wikibase-statementview-mainsnak">
  124.                             <div>
  125.                                 <div class="wikibase-snakview-value">
  126.                                     <div class="commons-media-caption">
  127.                                         <a href="https://commons.wikimedia.org/wiki/File:image.png">image.png</a>
  128.                                         <br/>2,687 &#215; 3,356; 1.22 MB
  129.                                     </div>
  130.                                 </div>
  131.                             </div>
  132.                         </div>
  133.                     </div>
  134.                 </div>
  135.             </div>
  136.         </div>
  137.         """
  138.         html_etree = fromstring(html)
  140.         image_src = wikidata.add_image(html_etree)
  141.         self.assertEqual(image_src,
  142.                          "https://commons.wikimedia.org/wiki/Special:FilePath/image.png?width=500&height=400")
  144.         html = u"""
  145.         <div>
  146.             <div id="P2910">
  147.                 <div class="wikibase-statementgroupview-property-label">
  148.                     <a href="/wiki/Property:P2910">
  149.                         icon
  150.                     </a>
  151.                 </div>
  152.                 <div class="wikibase-statementlistview">
  153.                     <div class="wikibase-statementview listview-item">
  154.                         <div class="wikibase-statementview-rankselector">
  155.                             <span class="wikibase-rankselector-normal"></span>
  156.                         </div>
  157.                         <div class="wikibase-statementview-mainsnak">
  158.                             <div>
  159.                                 <div class="wikibase-snakview-value">
  160.                                     <div class="commons-media-caption">
  161.                                         <a href="https://commons.wikimedia.org/wiki/File:icon.png">icon.png</a>
  162.                                         <br/>671 &#215; 671; 18 KB</div>
  163.                                     </div>
  164.                                 </div>
  165.                             </div>
  166.                         </div>
  167.                     </div>
  168.                 </div>
  169.             </div>
  170.             <div id="P154">
  171.                 <div class="wikibase-statementgroupview-property-label">
  172.                     <a href="/wiki/Property:P154">
  173.                         logo
  174.                     </a>
  175.                 </div>
  176.                 <div class="wikibase-statementlistview">
  177.                     <div class="wikibase-statementview listview-item">
  178.                         <div class="wikibase-statementview-rankselector">
  179.                             <span class="wikibase-rankselector-normal"></span>
  180.                         </div>
  181.                         <div class="wikibase-statementview-mainsnak">
  182.                             <div>
  183.                                 <div class="wikibase-snakview-value">
  184.                                     <div class="commons-media-caption">
  185.                                         <a href="https://commons.wikimedia.org/wiki/File:logo.png">logo.png</a>
  186.                                         <br/>170 &#215; 170; 1 KB
  187.                                     </div>
  188.                                 </div>
  189.                             </div>
  190.                         </div>
  191.                     </div>
  192.                 </div>
  193.             </div>
  194.         </div>
  195.         """
  196.         html_etree = fromstring(html)
  198.         image_src = wikidata.add_image(html_etree)
  199.         self.assertEqual(image_src,
  200.                          "https://commons.wikimedia.org/wiki/Special:FilePath/logo.png?width=500&height=400")
  202.     def test_add_attribute(self):
  203.         html = u"""
  204.         <div>
  205.             <div id="P27">
  206.                 <div class="wikibase-statementgroupview-property-label">
  207.                     <a href="/wiki/Property:P27">
  208.                         country of citizenship
  209.                     </a>
  210.                 </div>
  211.                 <div class="wikibase-statementlistview">
  212.                     <div class="wikibase-statementview listview-item">
  213.                         <div class="wikibase-statementview-rankselector">
  214.                             <span class="wikibase-rankselector-normal"></span>
  215.                         </div>
  216.                         <div class="wikibase-statementview-mainsnak">
  217.                             <div>
  218.                                 <div class="wikibase-snakview-value">
  219.                                     <a href="/wiki/Q145">
  220.                                         United Kingdom
  221.                                     </a>
  222.                                 </div>
  223.                             </div>
  224.                         </div>
  225.                     </div>
  226.                 </div>
  227.             </div>
  228.         </div>
  229.         """
  230.         attributes = []
  231.         html_etree = fromstring(html)
  233.         wikidata.add_attribute(attributes, html_etree, "Fail")
  234.         self.assertEqual(attributes, [])
  236.         wikidata.add_attribute(attributes, html_etree, "P27")
  237.         self.assertEqual(len(attributes), 1)
  238.         self.assertEqual(attributes[0]["label"], "Country of citizenship")
  239.         self.assertEqual(attributes[0]["value"], "United Kingdom")
  241.         html = u"""
  242.         <div>
  243.             <div id="P569">
  244.                 <div class="wikibase-statementgroupview-property-label">
  245.                     <a href="/wiki/Property:P569">
  246.                         date of birth
  247.                     </a>
  248.                 </div>
  249.                 <div class="wikibase-statementlistview">
  250.                     <div class="wikibase-statementview listview-item">
  251.                         <div class="wikibase-statementview-rankselector">
  252.                             <span class="wikibase-rankselector-normal"></span>
  253.                         </div>
  254.                         <div class="wikibase-statementview-mainsnak">
  255.                             <div>
  256.                                 <div class="wikibase-snakview-value">
  257.                                     27 January 1832
  258.                                     <sup class="wb-calendar-name">
  259.                                         Gregorian
  260.                                     </sup>
  261.                                 </div>
  262.                             </div>
  263.                         </div>
  264.                     </div>
  265.                 </div>
  266.             </div>
  267.         </div>
  268.         """
  269.         attributes = []
  270.         html_etree = fromstring(html)
  271.         wikidata.add_attribute(attributes, html_etree, "P569", date=True)
  272.         self.assertEqual(len(attributes), 1)
  273.         self.assertEqual(attributes[0]["label"], "Date of birth")
  274.         self.assertEqual(attributes[0]["value"], "27 January 1832")
  276.         html = u"""
  277.         <div>
  278.             <div id="P6">
  279.                 <div class="wikibase-statementgroupview-property-label">
  280.                     <a href="/wiki/Property:P27">
  281.                         head of government
  282.                     </a>
  283.                 </div>
  284.                 <div class="wikibase-statementlistview">
  285.                     <div class="wikibase-statementview listview-item">
  286.                         <div class="wikibase-statementview-rankselector">
  287.                             <span class="wikibase-rankselector-normal"></span>
  288.                         </div>
  289.                         <div class="wikibase-statementview-mainsnak">
  290.                             <div>
  291.                                 <div class="wikibase-snakview-value">
  292.                                     <a href="/wiki/Q206">
  293.                                         Old Prime Minister
  294.                                     </a>
  295.                                 </div>
  296.                             </div>
  297.                         </div>
  298.                     </div>
  299.                     <div class="wikibase-statementview listview-item">
  300.                         <div class="wikibase-statementview-rankselector">
  301.                             <span class="wikibase-rankselector-preferred"></span>
  302.                         </div>
  303.                         <div class="wikibase-statementview-mainsnak">
  304.                             <div>
  305.                                 <div class="wikibase-snakview-value">
  306.                                     <a href="/wiki/Q3099714">
  307.                                         Actual Prime Minister
  308.                                     </a>
  309.                                 </div>
  310.                             </div>
  311.                         </div>
  312.                     </div>
  313.                 </div>
  314.             </div>
  315.         </div>
  316.         """
  317.         attributes = []
  318.         html_etree = fromstring(html)
  319.         wikidata.add_attribute(attributes, html_etree, "P6")
  320.         self.assertEqual(len(attributes), 1)
  321.         self.assertEqual(attributes[0]["label"], "Head of government")
  322.         self.assertEqual(attributes[0]["value"], "Old Prime Minister, Actual Prime Minister")
  324.         attributes = []
  325.         html_etree = fromstring(html)
  326.         wikidata.add_attribute(attributes, html_etree, "P6", trim=True)
  327.         self.assertEqual(len(attributes), 1)
  328.         self.assertEqual(attributes[0]["value"], "Actual Prime Minister")
  330.     def test_add_url(self):
  331.         html = u"""
  332.         <div>
  333.             <div id="P856">
  334.                 <div class="wikibase-statementgroupview-property-label">
  335.                     <a href="/wiki/Property:P856">
  336.                         official website
  337.                     </a>
  338.                 </div>
  339.                 <div class="wikibase-statementlistview">
  340.                     <div class="wikibase-statementview listview-item">
  341.                         <div class="wikibase-statementview-mainsnak">
  342.                             <div>
  343.                                 <div class="wikibase-snakview-value">
  344.                                     <a class="external free" href="https://searx.me">
  345.                                         https://searx.me/
  346.                                     </a>
  347.                                 </div>
  348.                             </div>
  349.                         </div>
  350.                     </div>
  351.                 </div>
  352.             </div>
  353.         </div>
  354.         """
  355.         urls = []
  356.         html_etree = fromstring(html)
  357.         wikidata.add_url(urls, html_etree, 'P856')
  358.         self.assertEquals(len(urls), 1)
  359.         self.assertIn({'title': 'Official website', 'url': 'https://searx.me/'}, urls)
  360.         urls = []
  361.         results = []
  362.         wikidata.add_url(urls, html_etree, 'P856', 'custom label', results=results)
  363.         self.assertEquals(len(urls), 1)
  364.         self.assertEquals(len(results), 1)
  365.         self.assertIn({'title': 'custom label', 'url': 'https://searx.me/'}, urls)
  366.         self.assertIn({'title': 'custom label', 'url': 'https://searx.me/'}, results)
  368.         html = u"""
  369.         <div>
  370.             <div id="P856">
  371.                 <div class="wikibase-statementgroupview-property-label">
  372.                     <a href="/wiki/Property:P856">
  373.                         official website
  374.                     </a>
  375.                 </div>
  376.                 <div class="wikibase-statementlistview">
  377.                     <div class="wikibase-statementview listview-item">
  378.                         <div class="wikibase-statementview-mainsnak">
  379.                             <div>
  380.                                 <div class="wikibase-snakview-value">
  381.                                     <a class="external free" href="http://www.worldofwarcraft.com">
  382.                                         http://www.worldofwarcraft.com
  383.                                     </a>
  384.                                 </div>
  385.                             </div>
  386.                         </div>
  387.                     </div>
  388.                     <div class="wikibase-statementview listview-item">
  389.                         <div class="wikibase-statementview-mainsnak">
  390.                             <div>
  391.                                 <div class="wikibase-snakview-value">
  392.                                     <a class="external free" href="http://eu.battle.net/wow/en/">
  393.                                         http://eu.battle.net/wow/en/
  394.                                     </a>
  395.                                 </div>
  396.                             </div>
  397.                         </div>
  398.                     </div>
  399.                 </div>
  400.             </div>
  401.         </div>
  402.         """
  403.         urls = []
  404.         html_etree = fromstring(html)
  405.         wikidata.add_url(urls, html_etree, 'P856')
  406.         self.assertEquals(len(urls), 2)
  407.         self.assertIn({'title': 'Official website', 'url': 'http://www.worldofwarcraft.com'}, urls)
  408.         self.assertIn({'title': 'Official website', 'url': 'http://eu.battle.net/wow/en/'}, urls)
  410.     def test_get_imdblink(self):
  411.         html = u"""
  412.         <div>
  413.             <div class="wikibase-statementview-mainsnak">
  414.                 <div>
  415.                     <div class="wikibase-snakview-value">
  416.                         <a class="wb-external-id" href="http://www.imdb.com/tt0433664">
  417.                             tt0433664
  418.                         </a>
  419.                     </div>
  420.                 </div>
  421.             </div>
  422.         </div>
  423.         """
  424.         html_etree = fromstring(html)
  425.         imdblink = wikidata.get_imdblink(html_etree, 'https://www.imdb.com/')
  427.         html = u"""
  428.         <div>
  429.             <div class="wikibase-statementview-mainsnak">
  430.                 <div>
  431.                     <div class="wikibase-snakview-value">
  432.                         <a class="wb-external-id"
  433.                            href="href="http://tools.wmflabs.org/...http://www.imdb.com/&id=nm4915994"">
  434.                             nm4915994
  435.                         </a>
  436.                     </div>
  437.                 </div>
  438.             </div>
  439.         </div>
  440.         """
  441.         html_etree = fromstring(html)
  442.         imdblink = wikidata.get_imdblink(html_etree, 'https://www.imdb.com/')
  443.         self.assertIn('https://www.imdb.com/name/nm4915994', imdblink)
  445.     def test_get_geolink(self):
  446.         html = u"""
  447.         <div>
  448.             <div class="wikibase-statementview-mainsnak">
  449.                 <div>
  450.                     <div class="wikibase-snakview-value">
  451.                         60°N, 40°E
  452.                     </div>
  453.                 </div>
  454.             </div>
  455.         </div>
  456.         """
  457.         html_etree = fromstring(html)
  458.         geolink = wikidata.get_geolink(html_etree)
  459.         self.assertIn('https://www.openstreetmap.org/', geolink)
  460.         self.assertIn('lat=60&lon=40', geolink)
  462.         html = u"""
  463.         <div>
  464.             <div class="wikibase-statementview-mainsnak">
  465.                 <div>
  466.                     <div class="wikibase-snakview-value">
  467.                         34°35'59"S, 58°22'55"W
  468.                     </div>
  469.                 </div>
  470.             </div>
  471.         </div>
  472.         """
  473.         html_etree = fromstring(html)
  474.         geolink = wikidata.get_geolink(html_etree)
  475.         self.assertIn('https://www.openstreetmap.org/', geolink)
  476.         self.assertIn('lat=-34.59', geolink)
  477.         self.assertIn('lon=-58.38', geolink)
  479.     def test_get_wikilink(self):
  480.         html = """
  481.         <div>
  482.             <div>
  483.                 <ul class="wikibase-sitelinklistview-listview">
  484.                     <li data-wb-siteid="arwiki"><a href="http://ar.wikipedia.org/wiki/Test">Test</a></li>
  485.                     <li data-wb-siteid="enwiki"><a href="http://en.wikipedia.org/wiki/Test">Test</a></li>
  486.                 </ul>
  487.             </div>
  488.             <div>
  489.                 <ul class="wikibase-sitelinklistview-listview">
  490.                     <li data-wb-siteid="enwikiquote"><a href="https://en.wikiquote.org/wiki/Test">Test</a></li>
  491.                 </ul>
  492.             </div>
  493.         </div>
  494.         """
  495.         html_etree = fromstring(html)
  496.         wikilink = wikidata.get_wikilink(html_etree, 'nowiki')
  497.         self.assertEqual(wikilink, None)
  498.         wikilink = wikidata.get_wikilink(html_etree, 'enwiki')
  499.         self.assertEqual(wikilink, 'https://en.wikipedia.org/wiki/Test')
  500.         wikilink = wikidata.get_wikilink(html_etree, 'arwiki')
  501.         self.assertEqual(wikilink, 'https://ar.wikipedia.org/wiki/Test')
  502.         wikilink = wikidata.get_wikilink(html_etree, 'enwikiquote')
  503.         self.assertEqual(wikilink, 'https://en.wikiquote.org/wiki/Test')