logo

searx

My custom branche(s) on searx, a meta-search engine git clone https://anongit.hacktivis.me/git/searx.git/

duckduckgo.py (3314B)

    

  1. """
  2.  DuckDuckGo (Web)
  4.  @website     https://duckduckgo.com/
  5.  @provide-api yes (https://duckduckgo.com/api),
  6.               but not all results from search-site
  8.  @using-api   no
  9.  @results     HTML (using search portal)
  10.  @stable      no (HTML can change)
  11.  @parse       url, title, content
  13.  @todo        rewrite to api
  14. """
  16. from lxml.html import fromstring
  17. from json import loads
  18. from searx.engines.xpath import extract_text
  19. from searx.poolrequests import get
  20. from searx.url_utils import urlencode
  21. from searx.utils import match_language
  23. # engine dependent config
  24. categories = ['general']
  25. paging = True
  26. language_support = True
  27. supported_languages_url = 'https://duckduckgo.com/util/u172.js'
  28. time_range_support = True
  30. language_aliases = {
  31.     'ar-SA': 'ar-XA',
  32.     'es-419': 'es-XL',
  33.     'ja': 'jp-JP',
  34.     'ko': 'kr-KR',
  35.     'sl-SI': 'sl-SL',
  36.     'zh-TW': 'tzh-TW',
  37.     'zh-HK': 'tzh-HK'
  38. }
  40. # search-url
  41. url = 'https://duckduckgo.com/html?{query}&s={offset}&dc={dc_param}'
  42. time_range_url = '&df={range}'
  44. time_range_dict = {'day': 'd',
  45.                    'week': 'w',
  46.                    'month': 'm'}
  48. # specific xpath variables
  49. result_xpath = '//div[@class="result results_links results_links_deep web-result "]'  # noqa
  50. url_xpath = './/a[@class="result__a"]/@href'
  51. title_xpath = './/a[@class="result__a"]'
  52. content_xpath = './/a[@class="result__snippet"]'
  55. # match query's language to a region code that duckduckgo will accept
  56. def get_region_code(lang, lang_list=[]):
  57.     lang_code = match_language(lang, lang_list, language_aliases, 'wt-WT')
  58.     lang_parts = lang_code.split('-')
  60.     # country code goes first
  61.     return lang_parts[1].lower() + '-' + lang_parts[0].lower()
  64. # do search-request
  65. def request(query, params):
  66.     if params['time_range'] and params['time_range'] not in time_range_dict:
  67.         return params
  69.     offset = (params['pageno'] - 1) * 30
  71.     region_code = get_region_code(params['language'], supported_languages)
  72.     params['url'] = url.format(
  73.         query=urlencode({'q': query, 'kl': region_code}), offset=offset, dc_param=offset)
  75.     if params['time_range'] in time_range_dict:
  76.         params['url'] += time_range_url.format(range=time_range_dict[params['time_range']])
  78.     return params
  81. # get response from search-request
  82. def response(resp):
  83.     results = []
  85.     doc = fromstring(resp.text)
  87.     # parse results
  88.     for r in doc.xpath(result_xpath):
  89.         try:
  90.             res_url = r.xpath(url_xpath)[-1]
  91.         except:
  92.             continue
  94.         if not res_url:
  95.             continue
  97.         title = extract_text(r.xpath(title_xpath))
  98.         content = extract_text(r.xpath(content_xpath))
  100.         # append result
  101.         results.append({'title': title,
  102.                         'content': content,
  103.                         'url': res_url})
  105.     # return results
  106.     return results
  109. # get supported languages from their site
  110. def _fetch_supported_languages(resp):
  112.     # response is a js file with regions as an embedded object
  113.     response_page = resp.text
  114.     response_page = response_page[response_page.find('regions:{') + 8:]
  115.     response_page = response_page[:response_page.find('}') + 1]
  117.     regions_json = loads(response_page)
  118.     supported_languages = map((lambda x: x[3:] + '-' + x[:2].upper()), regions_json.keys())
  120.     return list(supported_languages)