logo

searx

My custom branche(s) on searx, a meta-search engine git clone https://hacktivis.me/git/searx.git

swisscows.py (3764B)


  1. """
  2. Swisscows (Web, Images)
  3. @website https://swisscows.ch
  4. @provide-api no
  5. @using-api no
  6. @results HTML (using search portal)
  7. @stable no (HTML can change)
  8. @parse url, title, content
  9. """
  10. from json import loads
  11. import re
  12. from lxml.html import fromstring
  13. from searx.url_utils import unquote, urlencode
  14. from searx.utils import match_language
  15. # engine dependent config
  16. categories = ['general', 'images']
  17. paging = True
  18. language_support = True
  19. # search-url
  20. base_url = 'https://swisscows.ch/'
  21. search_string = '?{query}&page={page}'
  22. supported_languages_url = base_url
  23. # regex
  24. regex_json = re.compile(b'initialData: {"Request":(.|\n)*},\s*environment')
  25. regex_json_remove_start = re.compile(b'^initialData:\s*')
  26. regex_json_remove_end = re.compile(b',\s*environment$')
  27. regex_img_url_remove_start = re.compile(b'^https?://i\.swisscows\.ch/\?link=')
  28. # do search-request
  29. def request(query, params):
  30. region = match_language(params['language'], supported_languages)
  31. ui_language = region.split('-')[0]
  32. search_path = search_string.format(
  33. query=urlencode({'query': query, 'uiLanguage': ui_language, 'region': region}),
  34. page=params['pageno']
  35. )
  36. # image search query is something like 'image?{query}&page={page}'
  37. if params['category'] == 'images':
  38. search_path = 'image' + search_path
  39. params['url'] = base_url + search_path
  40. return params
  41. # get response from search-request
  42. def response(resp):
  43. results = []
  44. json_regex = regex_json.search(resp.text)
  45. # check if results are returned
  46. if not json_regex:
  47. return []
  48. json_raw = regex_json_remove_end.sub(b'', regex_json_remove_start.sub(b'', json_regex.group()))
  49. json = loads(json_raw.decode('utf-8'))
  50. # parse results
  51. for result in json['Results'].get('items', []):
  52. result_title = result['Title'].replace(u'\uE000', '').replace(u'\uE001', '')
  53. # parse image results
  54. if result.get('ContentType', '').startswith('image'):
  55. img_url = unquote(regex_img_url_remove_start.sub(b'', result['Url'].encode('utf-8')).decode('utf-8'))
  56. # append result
  57. results.append({'url': result['SourceUrl'],
  58. 'title': result['Title'],
  59. 'content': '',
  60. 'img_src': img_url,
  61. 'template': 'images.html'})
  62. # parse general results
  63. else:
  64. result_url = result['Url'].replace(u'\uE000', '').replace(u'\uE001', '')
  65. result_content = result['Description'].replace(u'\uE000', '').replace(u'\uE001', '')
  66. # append result
  67. results.append({'url': result_url,
  68. 'title': result_title,
  69. 'content': result_content})
  70. # parse images
  71. for result in json.get('Images', []):
  72. # decode image url
  73. img_url = unquote(regex_img_url_remove_start.sub(b'', result['Url'].encode('utf-8')).decode('utf-8'))
  74. # append result
  75. results.append({'url': result['SourceUrl'],
  76. 'title': result['Title'],
  77. 'content': '',
  78. 'img_src': img_url,
  79. 'template': 'images.html'})
  80. # return results
  81. return results
  82. # get supported languages from their site
  83. def _fetch_supported_languages(resp):
  84. supported_languages = []
  85. dom = fromstring(resp.text)
  86. options = dom.xpath('//div[@id="regions-popup"]//ul/li/a')
  87. for option in options:
  88. code = option.xpath('./@data-search-language')[0]
  89. if code.startswith('nb-'):
  90. code = code.replace('nb', 'no', 1)
  91. supported_languages.append(code)
  92. return supported_languages