logo

searx

My custom branche(s) on searx, a meta-search engine git clone https://hacktivis.me/git/searx.git

yacy.py (3054B)

    

  1. # Yacy (Web, Images, Videos, Music, Files)
  2. #
  3. # @website     http://yacy.net
  4. # @provide-api yes
  5. #              (http://www.yacy-websuche.de/wiki/index.php/Dev:APIyacysearch)
  6. #
  7. # @using-api   yes
  8. # @results     JSON
  9. # @stable      yes
  10. # @parse       (general)    url, title, content, publishedDate
  11. # @parse       (images)     url, title, img_src
  12. #
  13. # @todo        parse video, audio and file results
  15. from json import loads
  16. from dateutil import parser
  17. from searx.url_utils import urlencode
  19. from searx.utils import html_to_text
  21. # engine dependent config
  22. categories = ['general', 'images']  # TODO , 'music', 'videos', 'files'
  23. paging = True
  24. language_support = True
  25. number_of_results = 5
  27. # search-url
  28. base_url = 'http://localhost:8090'
  29. search_url = '/yacysearch.json?{query}'\
  30.              '&startRecord={offset}'\
  31.              '&maximumRecords={limit}'\
  32.              '&contentdom={search_type}'\
  33.              '&resource=global'
  35. # yacy specific type-definitions
  36. search_types = {'general': 'text',
  37.                 'images': 'image',
  38.                 'files': 'app',
  39.                 'music': 'audio',
  40.                 'videos': 'video'}
  43. # do search-request
  44. def request(query, params):
  45.     offset = (params['pageno'] - 1) * number_of_results
  46.     search_type = search_types.get(params.get('category'), '0')
  48.     params['url'] = base_url +\
  49.         search_url.format(query=urlencode({'query': query}),
  50.                           offset=offset,
  51.                           limit=number_of_results,
  52.                           search_type=search_type)
  54.     params['url'] += '&lr=lang_' + params['language'].split('-')[0]
  56.     return params
  59. # get response from search-request
  60. def response(resp):
  61.     results = []
  63.     raw_search_results = loads(resp.text)
  65.     # return empty array if there are no results
  66.     if not raw_search_results:
  67.         return []
  69.     search_results = raw_search_results.get('channels', [])
  71.     if len(search_results) == 0:
  72.         return []
  74.     for result in search_results[0].get('items', []):
  75.         # parse image results
  76.         if result.get('image'):
  78.             result_url = ''
  79.             if 'url' in result:
  80.                 result_url = result['url']
  81.             elif 'link' in result:
  82.                 result_url = result['link']
  83.             else:
  84.                 continue
  86.             # append result
  87.             results.append({'url': result_url,
  88.                             'title': result['title'],
  89.                             'content': '',
  90.                             'img_src': result['image'],
  91.                             'template': 'images.html'})
  93.         # parse general results
  94.         else:
  95.             publishedDate = parser.parse(result['pubDate'])
  97.             # append result
  98.             results.append({'url': result['link'],
  99.                             'title': result['title'],
  100.                             'content': html_to_text(result['description']),
  101.                             'publishedDate': publishedDate})
  103.         # TODO parse video, audio and file results
  105.     # return results
  106.     return results