logo

searx

My custom branche(s) on searx, a meta-search engine git clone https://hacktivis.me/git/searx.git

fetch_languages.py (8083B)


  1. # -*- coding: utf-8 -*-
  2. # This script generates languages.py from intersecting each engine's supported languages.
  3. #
  4. # Output files (engines_languages.json and languages.py)
  5. # are written in current directory to avoid overwriting in case something goes wrong.
  6. from json import dump
  7. import io
  8. from sys import path
  9. from babel import Locale, UnknownLocaleError
  10. from babel.languages import get_global
  11. path.append('../searx') # noqa
  12. from searx import settings
  13. from searx.engines import initialize_engines, engines
  14. # Output files.
  15. engines_languages_file = 'engines_languages.json'
  16. languages_file = 'languages.py'
  17. # Fetchs supported languages for each engine and writes json file with those.
  18. def fetch_supported_languages():
  19. engines_languages = {}
  20. for engine_name in engines:
  21. if hasattr(engines[engine_name], 'fetch_supported_languages'):
  22. try:
  23. engines_languages[engine_name] = engines[engine_name].fetch_supported_languages()
  24. if type(engines_languages[engine_name]) == list:
  25. engines_languages[engine_name] = sorted(engines_languages[engine_name])
  26. except Exception as e:
  27. print(e)
  28. # write json file
  29. with io.open(engines_languages_file, "w", encoding="utf-8") as f:
  30. dump(engines_languages, f, ensure_ascii=False, indent=4, separators=(',', ': '))
  31. return engines_languages
  32. # Get babel Locale object from lang_code if possible.
  33. def get_locale(lang_code):
  34. try:
  35. locale = Locale.parse(lang_code, sep='-')
  36. return locale
  37. except (UnknownLocaleError, ValueError):
  38. return None
  39. # Append engine_name to list of engines that support locale.
  40. def add_engine_counter(lang_code, engine_name, languages):
  41. if lang_code in languages:
  42. if 'counter' not in languages[lang_code]:
  43. languages[lang_code]['counter'] = [engine_name]
  44. elif engine_name not in languages[lang_code]['counter']:
  45. languages[lang_code]['counter'].append(engine_name)
  46. # Join all language lists.
  47. # TODO: Add language names from engine's language list if name not known by babel.
  48. def join_language_lists(engines_languages):
  49. language_list = {}
  50. for engine_name in engines_languages:
  51. for lang_code in engines_languages[engine_name]:
  52. # apply custom fixes if necessary
  53. if lang_code in getattr(engines[engine_name], 'language_aliases', {}).values():
  54. lang_code = next(lc for lc, alias in engines[engine_name].language_aliases.items()
  55. if lang_code == alias)
  56. locale = get_locale(lang_code)
  57. # ensure that lang_code uses standard language and country codes
  58. if locale and locale.territory:
  59. lang_code = locale.language + '-' + locale.territory
  60. # add locale if it's not in list
  61. if lang_code not in language_list:
  62. if locale:
  63. language_list[lang_code] = {'name': locale.get_language_name().title(),
  64. 'english_name': locale.english_name,
  65. 'country': locale.get_territory_name() or ''}
  66. # also add language without country
  67. if locale.language not in language_list:
  68. language_list[locale.language] = {'name': locale.get_language_name().title(),
  69. 'english_name': locale.english_name}
  70. else:
  71. language_list[lang_code] = {}
  72. # count engine for both language_country combination and language alone
  73. add_engine_counter(lang_code, engine_name, language_list)
  74. add_engine_counter(lang_code.split('-')[0], engine_name, language_list)
  75. return language_list
  76. # Filter language list so it only includes the most supported languages and countries.
  77. def filter_language_list(all_languages):
  78. min_supported_engines = 10
  79. main_engines = [engine_name for engine_name in engines.keys()
  80. if 'general' in engines[engine_name].categories and
  81. engines[engine_name].supported_languages and
  82. not engines[engine_name].disabled]
  83. # filter list to include only languages supported by most engines or all default general engines
  84. filtered_languages = {code: lang for code, lang
  85. in all_languages.items()
  86. if (len(lang.get('counter', [])) >= min_supported_engines or
  87. all(main_engine in lang.get('counter', [])
  88. for main_engine in main_engines))}
  89. return filtered_languages
  90. # Add country codes to languages without one and filter out language codes.
  91. def assign_country_codes(filtered_languages, all_languages):
  92. sorted_languages = sorted(all_languages,
  93. key=lambda lang: len(all_languages[lang].get('counter', [])),
  94. reverse=True)
  95. previous_lang = None
  96. previous_code = None
  97. countries = 0
  98. for current_code in sorted(filtered_languages):
  99. current_lang = current_code.split('-')[0]
  100. # count country codes per language
  101. if current_lang == previous_lang:
  102. countries += 1
  103. else:
  104. if previous_lang is not None:
  105. # if language has no single country code
  106. if countries == 0:
  107. # try to get country code with most supported engines
  108. for l in sorted_languages:
  109. l_parts = l.split('-')
  110. if len(l_parts) == 2 and l_parts[0] == previous_lang:
  111. filtered_languages[l] = all_languages[l]
  112. filtered_languages[l]['country'] = ''
  113. countries = 1
  114. break
  115. if countries == 0:
  116. # get most likely country code from babel
  117. subtags = get_global('likely_subtags').get(previous_lang)
  118. if subtags:
  119. subtag_parts = subtags.split('_')
  120. new_code = subtag_parts[0] + '-' + subtag_parts[-1]
  121. filtered_languages[new_code] = all_languages[previous_lang]
  122. countries = 1
  123. if countries == 1:
  124. # remove countryless version of language if there's only one country
  125. del filtered_languages[previous_lang]
  126. if previous_code in filtered_languages:
  127. filtered_languages[previous_code]['country'] = ''
  128. countries = 0
  129. previous_lang = current_lang
  130. previous_code = current_code
  131. # Write languages.py.
  132. def write_languages_file(languages):
  133. new_file = open(languages_file, 'wb')
  134. file_content = '# -*- coding: utf-8 -*-\n'\
  135. + '# list of language codes\n'\
  136. + '# this file is generated automatically by utils/update_search_languages.py\n'\
  137. + '\nlanguage_codes = ('
  138. for code in sorted(languages):
  139. file_content += '\n (u"' + code + '"'\
  140. + ', u"' + languages[code]['name'].split(' (')[0] + '"'\
  141. + ', u"' + languages[code].get('country', '') + '"'\
  142. + ', u"' + languages[code].get('english_name', '').split(' (')[0] + '"),'
  143. # remove last comma
  144. file_content = file_content[:-1]
  145. file_content += '\n)\n'
  146. new_file.write(file_content.encode('utf8'))
  147. new_file.close()
  148. if __name__ == "__main__":
  149. initialize_engines(settings['engines'])
  150. engines_languages = fetch_supported_languages()
  151. all_languages = join_language_lists(engines_languages)
  152. filtered_languages = filter_language_list(all_languages)
  153. assign_country_codes(filtered_languages, all_languages)
  154. write_languages_file(filtered_languages)