logo

searx

My custom branche(s) on searx, a meta-search engine git clone https://hacktivis.me/git/searx.git

wikidata.py (17152B)

    

  1. # -*- coding: utf-8 -*-

  2. """

  3.  Wikidata

  4. 

  5.  @website     https://wikidata.org

  6.  @provide-api yes (https://wikidata.org/w/api.php)

  7. 

  8.  @using-api   partially (most things require scraping)

  9.  @results     JSON, HTML

  10.  @stable      no (html can change)

  11.  @parse       url, infobox

  12. """

  13. 

  14. from searx import logger

  15. from searx.poolrequests import get

  16. from searx.engines.xpath import extract_text

  17. from searx.engines.wikipedia import _fetch_supported_languages, supported_languages_url

  18. from searx.url_utils import urlencode

  19. from searx.utils import match_language

  20. 

  21. from json import loads

  22. from lxml.html import fromstring

  23. 

  24. logger = logger.getChild('wikidata')

  25. result_count = 1

  26. 

  27. # urls

  28. wikidata_host = 'https://www.wikidata.org'

  29. url_search = wikidata_host \

  30.     + '/w/index.php?{query}'

  31. 

  32. wikidata_api = wikidata_host + '/w/api.php'

  33. url_detail = wikidata_api\

  34.     + '?action=parse&format=json&{query}'\

  35.     + '&redirects=1&prop=text%7Cdisplaytitle%7Clanglinks%7Crevid'\

  36.     + '&disableeditsection=1&disabletidy=1&preview=1&sectionpreview=1&disabletoc=1&utf8=1&formatversion=2'

  37. 

  38. url_map = 'https://www.openstreetmap.org/'\

  39.     + '?lat={latitude}&lon={longitude}&zoom={zoom}&layers=M'

  40. url_image = 'https://commons.wikimedia.org/wiki/Special:FilePath/{filename}?width=500&height=400'

  41. 

  42. # xpaths

  43. wikidata_ids_xpath = '//ul[@class="mw-search-results"]/li//a/@href'

  44. title_xpath = '//*[contains(@class,"wikibase-title-label")]'

  45. description_xpath = '//div[contains(@class,"wikibase-entitytermsview-heading-description")]'

  46. property_xpath = '//div[@id="{propertyid}"]'

  47. label_xpath = './/div[contains(@class,"wikibase-statementgroupview-property-label")]/a'

  48. url_xpath = './/a[contains(@class,"external free") or contains(@class, "wb-external-id")]'

  49. wikilink_xpath = './/ul[contains(@class,"wikibase-sitelinklistview-listview")]'\

  50.     + '/li[contains(@data-wb-siteid,"{wikiid}")]//a/@href'

  51. property_row_xpath = './/div[contains(@class,"wikibase-statementview")]'

  52. preferred_rank_xpath = './/span[contains(@class,"wikibase-rankselector-preferred")]'

  53. value_xpath = './/div[contains(@class,"wikibase-statementview-mainsnak")]'\

  54.     + '/*/div[contains(@class,"wikibase-snakview-value")]'

  55. language_fallback_xpath = '//sup[contains(@class,"wb-language-fallback-indicator")]'

  56. calendar_name_xpath = './/sup[contains(@class,"wb-calendar-name")]'

  57. media_xpath = value_xpath + '//div[contains(@class,"commons-media-caption")]//a'

  58. 

  59. 

  60. def request(query, params):

  61.     params['url'] = url_search.format(

  62.         query=urlencode({'search': query}))

  63.     return params

  64. 

  65. 

  66. def response(resp):

  67.     results = []

  68.     html = fromstring(resp.text)

  69.     search_results = html.xpath(wikidata_ids_xpath)

  70. 

  71.     language = match_language(resp.search_params['language'], supported_languages).split('-')[0]

  72. 

  73.     # TODO: make requests asynchronous to avoid timeout when result_count > 1

  74.     for search_result in search_results[:result_count]:

  75.         wikidata_id = search_result.split('/')[-1]

  76.         url = url_detail.format(query=urlencode({'page': wikidata_id, 'uselang': language}))

  77.         htmlresponse = get(url)

  78.         jsonresponse = loads(htmlresponse.text)

  79.         results += getDetail(jsonresponse, wikidata_id, language, resp.search_params['language'])

  80. 

  81.     return results

  82. 

  83. 

  84. def getDetail(jsonresponse, wikidata_id, language, locale):

  85.     results = []

  86.     urls = []

  87.     attributes = []

  88. 

  89.     title = jsonresponse.get('parse', {}).get('displaytitle', {})

  90.     result = jsonresponse.get('parse', {}).get('text', {})

  91. 

  92.     if not title or not result:

  93.         return results

  94. 

  95.     title = fromstring(title)

  96.     for elem in title.xpath(language_fallback_xpath):

  97.         elem.getparent().remove(elem)

  98.     title = extract_text(title.xpath(title_xpath))

  99. 

  100.     result = fromstring(result)

  101.     for elem in result.xpath(language_fallback_xpath):

  102.         elem.getparent().remove(elem)

  103. 

  104.     description = extract_text(result.xpath(description_xpath))

  105. 

  106.     # URLS

  107. 

  108.     # official website

  109.     add_url(urls, result, 'P856', results=results)

  110. 

  111.     # wikipedia

  112.     wikipedia_link_count = 0

  113.     wikipedia_link = get_wikilink(result, language + 'wiki')

  114.     if wikipedia_link:

  115.         wikipedia_link_count += 1

  116.         urls.append({'title': 'Wikipedia (' + language + ')',

  117.                      'url': wikipedia_link})

  118. 

  119.     if language != 'en':

  120.         wikipedia_en_link = get_wikilink(result, 'enwiki')

  121.         if wikipedia_en_link:

  122.             wikipedia_link_count += 1

  123.             urls.append({'title': 'Wikipedia (en)',

  124.                          'url': wikipedia_en_link})

  125. 

  126.     # TODO: get_wiki_firstlanguage

  127.     # if wikipedia_link_count == 0:

  128. 

  129.     # more wikis

  130.     add_url(urls, result, default_label='Wikivoyage (' + language + ')', link_type=language + 'wikivoyage')

  131.     add_url(urls, result, default_label='Wikiquote (' + language + ')', link_type=language + 'wikiquote')

  132.     add_url(urls, result, default_label='Wikimedia Commons', link_type='commonswiki')

  133. 

  134.     add_url(urls, result, 'P625', 'OpenStreetMap', link_type='geo')

  135. 

  136.     # musicbrainz

  137.     add_url(urls, result, 'P434', 'MusicBrainz', 'http://musicbrainz.org/artist/')

  138.     add_url(urls, result, 'P435', 'MusicBrainz', 'http://musicbrainz.org/work/')

  139.     add_url(urls, result, 'P436', 'MusicBrainz', 'http://musicbrainz.org/release-group/')

  140.     add_url(urls, result, 'P966', 'MusicBrainz', 'http://musicbrainz.org/label/')

  141. 

  142.     # IMDb

  143.     add_url(urls, result, 'P345', 'IMDb', 'https://www.imdb.com/', link_type='imdb')

  144.     # source code repository

  145.     add_url(urls, result, 'P1324')

  146.     # blog

  147.     add_url(urls, result, 'P1581')

  148.     # social media links

  149.     add_url(urls, result, 'P2397', 'YouTube', 'https://www.youtube.com/channel/')

  150.     add_url(urls, result, 'P1651', 'YouTube', 'https://www.youtube.com/watch?v=')

  151.     add_url(urls, result, 'P2002', 'Twitter', 'https://twitter.com/')

  152.     add_url(urls, result, 'P2013', 'Facebook', 'https://facebook.com/')

  153.     add_url(urls, result, 'P2003', 'Instagram', 'https://instagram.com/')

  154. 

  155.     urls.append({'title': 'Wikidata',

  156.                  'url': 'https://www.wikidata.org/wiki/'

  157.                  + wikidata_id + '?uselang=' + language})

  158. 

  159.     # INFOBOX ATTRIBUTES (ROWS)

  160. 

  161.     # DATES

  162.     # inception date

  163.     add_attribute(attributes, result, 'P571', date=True)

  164.     # dissolution date

  165.     add_attribute(attributes, result, 'P576', date=True)

  166.     # start date

  167.     add_attribute(attributes, result, 'P580', date=True)

  168.     # end date

  169.     add_attribute(attributes, result, 'P582', date=True)

  170.     # date of birth

  171.     add_attribute(attributes, result, 'P569', date=True)

  172.     # date of death

  173.     add_attribute(attributes, result, 'P570', date=True)

  174.     # date of spacecraft launch

  175.     add_attribute(attributes, result, 'P619', date=True)

  176.     # date of spacecraft landing

  177.     add_attribute(attributes, result, 'P620', date=True)

  178. 

  179.     # nationality

  180.     add_attribute(attributes, result, 'P27')

  181.     # country of origin

  182.     add_attribute(attributes, result, 'P495')

  183.     # country

  184.     add_attribute(attributes, result, 'P17')

  185.     # headquarters

  186.     add_attribute(attributes, result, 'Q180')

  187. 

  188.     # PLACES

  189.     # capital

  190.     add_attribute(attributes, result, 'P36', trim=True)

  191.     # head of state

  192.     add_attribute(attributes, result, 'P35', trim=True)

  193.     # head of government

  194.     add_attribute(attributes, result, 'P6', trim=True)

  195.     # type of government

  196.     add_attribute(attributes, result, 'P122')

  197.     # official language

  198.     add_attribute(attributes, result, 'P37')

  199.     # population

  200.     add_attribute(attributes, result, 'P1082', trim=True)

  201.     # area

  202.     add_attribute(attributes, result, 'P2046')

  203.     # currency

  204.     add_attribute(attributes, result, 'P38', trim=True)

  205.     # heigth (building)

  206.     add_attribute(attributes, result, 'P2048')

  207. 

  208.     # MEDIA

  209.     # platform (videogames)

  210.     add_attribute(attributes, result, 'P400')

  211.     # author

  212.     add_attribute(attributes, result, 'P50')

  213.     # creator

  214.     add_attribute(attributes, result, 'P170')

  215.     # director

  216.     add_attribute(attributes, result, 'P57')

  217.     # performer

  218.     add_attribute(attributes, result, 'P175')

  219.     # developer

  220.     add_attribute(attributes, result, 'P178')

  221.     # producer

  222.     add_attribute(attributes, result, 'P162')

  223.     # manufacturer

  224.     add_attribute(attributes, result, 'P176')

  225.     # screenwriter

  226.     add_attribute(attributes, result, 'P58')

  227.     # production company

  228.     add_attribute(attributes, result, 'P272')

  229.     # record label

  230.     add_attribute(attributes, result, 'P264')

  231.     # publisher

  232.     add_attribute(attributes, result, 'P123')

  233.     # original network

  234.     add_attribute(attributes, result, 'P449')

  235.     # distributor

  236.     add_attribute(attributes, result, 'P750')

  237.     # composer

  238.     add_attribute(attributes, result, 'P86')

  239.     # publication date

  240.     add_attribute(attributes, result, 'P577', date=True)

  241.     # genre

  242.     add_attribute(attributes, result, 'P136')

  243.     # original language

  244.     add_attribute(attributes, result, 'P364')

  245.     # isbn

  246.     add_attribute(attributes, result, 'Q33057')

  247.     # software license

  248.     add_attribute(attributes, result, 'P275')

  249.     # programming language

  250.     add_attribute(attributes, result, 'P277')

  251.     # version

  252.     add_attribute(attributes, result, 'P348', trim=True)

  253.     # narrative location

  254.     add_attribute(attributes, result, 'P840')

  255. 

  256.     # LANGUAGES

  257.     # number of speakers

  258.     add_attribute(attributes, result, 'P1098')

  259.     # writing system

  260.     add_attribute(attributes, result, 'P282')

  261.     # regulatory body

  262.     add_attribute(attributes, result, 'P1018')

  263.     # language code

  264.     add_attribute(attributes, result, 'P218')

  265. 

  266.     # OTHER

  267.     # ceo

  268.     add_attribute(attributes, result, 'P169', trim=True)

  269.     # founder

  270.     add_attribute(attributes, result, 'P112')

  271.     # legal form (company/organization)

  272.     add_attribute(attributes, result, 'P1454')

  273.     # operator

  274.     add_attribute(attributes, result, 'P137')

  275.     # crew members (tripulation)

  276.     add_attribute(attributes, result, 'P1029')

  277.     # taxon

  278.     add_attribute(attributes, result, 'P225')

  279.     # chemical formula

  280.     add_attribute(attributes, result, 'P274')

  281.     # winner (sports/contests)

  282.     add_attribute(attributes, result, 'P1346')

  283.     # number of deaths

  284.     add_attribute(attributes, result, 'P1120')

  285.     # currency code

  286.     add_attribute(attributes, result, 'P498')

  287. 

  288.     image = add_image(result)

  289. 

  290.     if len(attributes) == 0 and len(urls) == 2 and len(description) == 0:

  291.         results.append({

  292.                        'url': urls[0]['url'],

  293.                        'title': title,

  294.                        'content': description

  295.                        })

  296.     else:

  297.         results.append({

  298.                        'infobox': title,

  299.                        'id': wikipedia_link,

  300.                        'content': description,

  301.                        'img_src': image,

  302.                        'attributes': attributes,

  303.                        'urls': urls

  304.                        })

  305. 

  306.     return results

  307. 

  308. 

  309. # only returns first match

  310. def add_image(result):

  311.     # P15: route map, P242: locator map, P154: logo, P18: image, P242: map, P41: flag, P2716: collage, P2910: icon

  312.     property_ids = ['P15', 'P242', 'P154', 'P18', 'P242', 'P41', 'P2716', 'P2910']

  313. 

  314.     for property_id in property_ids:

  315.         image = result.xpath(property_xpath.replace('{propertyid}', property_id))

  316.         if image:

  317.             image_name = image[0].xpath(media_xpath)

  318.             image_src = url_image.replace('{filename}', extract_text(image_name[0]))

  319.             return image_src

  320. 

  321. 

  322. # setting trim will only returned high ranked rows OR the first row

  323. def add_attribute(attributes, result, property_id, default_label=None, date=False, trim=False):

  324.     attribute = result.xpath(property_xpath.replace('{propertyid}', property_id))

  325.     if attribute:

  326. 

  327.         if default_label:

  328.             label = default_label

  329.         else:

  330.             label = extract_text(attribute[0].xpath(label_xpath))

  331.             label = label[0].upper() + label[1:]

  332. 

  333.         if date:

  334.             trim = True

  335.             # remove calendar name

  336.             calendar_name = attribute[0].xpath(calendar_name_xpath)

  337.             for calendar in calendar_name:

  338.                 calendar.getparent().remove(calendar)

  339. 

  340.         concat_values = ""

  341.         values = []

  342.         first_value = None

  343.         for row in attribute[0].xpath(property_row_xpath):

  344.             if not first_value or not trim or row.xpath(preferred_rank_xpath):

  345. 

  346.                 value = row.xpath(value_xpath)

  347.                 if not value:

  348.                     continue

  349.                 value = extract_text(value)

  350. 

  351.                 # save first value in case no ranked row is found

  352.                 if trim and not first_value:

  353.                     first_value = value

  354.                 else:

  355.                     # to avoid duplicate values

  356.                     if value not in values:

  357.                         concat_values += value + ", "

  358.                         values.append(value)

  359. 

  360.         if trim and not values:

  361.             attributes.append({'label': label,

  362.                                'value': first_value})

  363.         else:

  364.             attributes.append({'label': label,

  365.                                'value': concat_values[:-2]})

  366. 

  367. 

  368. # requires property_id unless it's a wiki link (defined in link_type)

  369. def add_url(urls, result, property_id=None, default_label=None, url_prefix=None, results=None, link_type=None):

  370.     links = []

  371. 

  372.     # wiki links don't have property in wikidata page

  373.     if link_type and 'wiki' in link_type:

  374.             links.append(get_wikilink(result, link_type))

  375.     else:

  376.         dom_element = result.xpath(property_xpath.replace('{propertyid}', property_id))

  377.         if dom_element:

  378.             dom_element = dom_element[0]

  379.             if not default_label:

  380.                 label = extract_text(dom_element.xpath(label_xpath))

  381.                 label = label[0].upper() + label[1:]

  382. 

  383.             if link_type == 'geo':

  384.                 links.append(get_geolink(dom_element))

  385. 

  386.             elif link_type == 'imdb':

  387.                 links.append(get_imdblink(dom_element, url_prefix))

  388. 

  389.             else:

  390.                 url_results = dom_element.xpath(url_xpath)

  391.                 for link in url_results:

  392.                     if link is not None:

  393.                         if url_prefix:

  394.                             link = url_prefix + extract_text(link)

  395.                         else:

  396.                             link = extract_text(link)

  397.                         links.append(link)

  398. 

  399.     # append urls

  400.     for url in links:

  401.         if url is not None:

  402.             urls.append({'title': default_label or label,

  403.                          'url': url})

  404.             if results is not None:

  405.                 results.append({'title': default_label or label,

  406.                                 'url': url})

  407. 

  408. 

  409. def get_imdblink(result, url_prefix):

  410.     imdb_id = result.xpath(value_xpath)

  411.     if imdb_id:

  412.         imdb_id = extract_text(imdb_id)

  413.         id_prefix = imdb_id[:2]

  414.         if id_prefix == 'tt':

  415.             url = url_prefix + 'title/' + imdb_id

  416.         elif id_prefix == 'nm':

  417.             url = url_prefix + 'name/' + imdb_id

  418.         elif id_prefix == 'ch':

  419.             url = url_prefix + 'character/' + imdb_id

  420.         elif id_prefix == 'co':

  421.             url = url_prefix + 'company/' + imdb_id

  422.         elif id_prefix == 'ev':

  423.             url = url_prefix + 'event/' + imdb_id

  424.         else:

  425.             url = None

  426.         return url

  427. 

  428. 

  429. def get_geolink(result):

  430.     coordinates = result.xpath(value_xpath)

  431.     if not coordinates:

  432.         return None

  433.     coordinates = extract_text(coordinates[0])

  434.     latitude, longitude = coordinates.split(',')

  435. 

  436.     # convert to decimal

  437.     lat = int(latitude[:latitude.find(u'°')])

  438.     if latitude.find('\'') >= 0:

  439.         lat += int(latitude[latitude.find(u'°') + 1:latitude.find('\'')] or 0) / 60.0

  440.     if latitude.find('"') >= 0:

  441.         lat += float(latitude[latitude.find('\'') + 1:latitude.find('"')] or 0) / 3600.0

  442.     if latitude.find('S') >= 0:

  443.         lat *= -1

  444.     lon = int(longitude[:longitude.find(u'°')])

  445.     if longitude.find('\'') >= 0:

  446.         lon += int(longitude[longitude.find(u'°') + 1:longitude.find('\'')] or 0) / 60.0

  447.     if longitude.find('"') >= 0:

  448.         lon += float(longitude[longitude.find('\'') + 1:longitude.find('"')] or 0) / 3600.0

  449.     if longitude.find('W') >= 0:

  450.         lon *= -1

  451. 

  452.     # TODO: get precision

  453.     precision = 0.0002

  454.     # there is no zoom information, deduce from precision (error prone)

  455.     # samples :

  456.     # 13 --> 5

  457.     # 1 --> 6

  458.     # 0.016666666666667 --> 9

  459.     # 0.00027777777777778 --> 19

  460.     # wolframalpha :

  461.     # quadratic fit { {13, 5}, {1, 6}, {0.0166666, 9}, {0.0002777777,19}}

  462.     # 14.1186-8.8322 x+0.625447 x^2

  463.     if precision < 0.0003:

  464.         zoom = 19

  465.     else:

  466.         zoom = int(15 - precision * 8.8322 + precision * precision * 0.625447)

  467. 

  468.     url = url_map\

  469.         .replace('{latitude}', str(lat))\

  470.         .replace('{longitude}', str(lon))\

  471.         .replace('{zoom}', str(zoom))

  472. 

  473.     return url

  474. 

  475. 

  476. def get_wikilink(result, wikiid):

  477.     url = result.xpath(wikilink_xpath.replace('{wikiid}', wikiid))

  478.     if not url:

  479.         return None

  480.     url = url[0]

  481.     if url.startswith('http://'):

  482.         url = url.replace('http://', 'https://')

  483.     elif url.startswith('//'):

  484.         url = 'https:' + url

  485.     return url