logo

searx

My custom branche(s) on searx, a meta-search engine git clone https://anongit.hacktivis.me/git/searx.git/

bing.py (2955B)

    

  1. """
  2.  Bing (Web)
  4.  @website     https://www.bing.com
  5.  @provide-api yes (http://datamarket.azure.com/dataset/bing/search),
  6.               max. 5000 query/month
  8.  @using-api   no (because of query limit)
  9.  @results     HTML (using search portal)
  10.  @stable      no (HTML can change)
  11.  @parse       url, title, content
  13.  @todo        publishedDate
  14. """
  16. from lxml import html
  17. from searx.engines.xpath import extract_text
  18. from searx.url_utils import urlencode
  19. from searx.utils import match_language, gen_useragent
  21. # engine dependent config
  22. categories = ['general']
  23. paging = True
  24. language_support = True
  25. supported_languages_url = 'https://www.bing.com/account/general'
  26. language_aliases = {'zh-CN': 'zh-CHS', 'zh-TW': 'zh-CHT', 'zh-HK': 'zh-CHT'}
  28. # search-url
  29. base_url = 'https://www.bing.com/'
  30. search_string = 'search?{query}&first={offset}'
  33. # do search-request
  34. def request(query, params):
  35.     offset = (params['pageno'] - 1) * 10 + 1
  37.     lang = match_language(params['language'], supported_languages, language_aliases)
  39.     query = u'language:{} {}'.format(lang.split('-')[0].upper(), query.decode('utf-8')).encode('utf-8')
  41.     search_path = search_string.format(
  42.         query=urlencode({'q': query}),
  43.         offset=offset)
  45.     params['url'] = base_url + search_path
  47.     params['headers']['User-Agent'] = gen_useragent('Windows NT 6.3; WOW64')
  49.     return params
  52. # get response from search-request
  53. def response(resp):
  54.     results = []
  56.     dom = html.fromstring(resp.text)
  58.     try:
  59.         results.append({'number_of_results': int(dom.xpath('//span[@class="sb_count"]/text()')[0]
  60.                                                  .split()[0].replace(',', ''))})
  61.     except:
  62.         pass
  64.     # parse results
  65.     for result in dom.xpath('//div[@class="sa_cc"]'):
  66.         link = result.xpath('.//h3/a')[0]
  67.         url = link.attrib.get('href')
  68.         title = extract_text(link)
  69.         content = extract_text(result.xpath('.//p'))
  71.         # append result
  72.         results.append({'url': url,
  73.                         'title': title,
  74.                         'content': content})
  76.     # parse results again if nothing is found yet
  77.     for result in dom.xpath('//li[@class="b_algo"]'):
  78.         link = result.xpath('.//h2/a')[0]
  79.         url = link.attrib.get('href')
  80.         title = extract_text(link)
  81.         content = extract_text(result.xpath('.//p'))
  83.         # append result
  84.         results.append({'url': url,
  85.                         'title': title,
  86.                         'content': content})
  88.     # return results
  89.     return results
  92. # get supported languages from their site
  93. def _fetch_supported_languages(resp):
  94.     supported_languages = []
  95.     dom = html.fromstring(resp.text)
  96.     options = dom.xpath('//div[@id="limit-languages"]//input')
  97.     for option in options:
  98.         code = option.xpath('./@id')[0].replace('_', '-')
  99.         if code == 'nb':
  100.             code = 'no'
  101.         supported_languages.append(code)
  103.     return supported_languages