commit: e48f07a367e55bf8aa881902b977bd7ce1cd2bb6 parent 219f047bf359ce94397241b875639f3aaddb0fe5 Author: Alexandre Flament <alex@al-f.net> Date: Fri, 9 Dec 2016 23:11:45 +0100 Merge branch 'master' into searchpy2Diffstat:
63 files changed, 392 insertions(+), 202 deletions(-)diff --git a/searx/answerers/__init__.py b/searx/answerers/__init__.py@@ -0,0 +1,46 @@ +from os import listdir +from os.path import realpath, dirname, join, isdir +from searx.utils import load_module +from collections import defaultdict + + +answerers_dir = dirname(realpath(__file__)) + + +def load_answerers(): + answerers = [] + for filename in listdir(answerers_dir): + if not isdir(join(answerers_dir, filename)): + continue + module = load_module('answerer.py', join(answerers_dir, filename)) + if not hasattr(module, 'keywords') or not isinstance(module.keywords, tuple) or not len(module.keywords): + exit(2) + answerers.append(module) + return answerers + + +def get_answerers_by_keywords(answerers): + by_keyword = defaultdict(list) + for answerer in answerers: + for keyword in answerer.keywords: + for keyword in answerer.keywords: + by_keyword[keyword].append(answerer.answer) + return by_keyword + + +def ask(query): + results = [] + query_parts = filter(None, query.query.split()) + + if query_parts[0] not in answerers_by_keywords: + return results + + for answerer in answerers_by_keywords[query_parts[0]]: + result = answerer(query) + if result: + results.append(result) + return results + + +answerers = load_answerers() +answerers_by_keywords = get_answerers_by_keywords(answerers)diff --git a/searx/answerers/random/answerer.py b/searx/answerers/random/answerer.py@@ -0,0 +1,50 @@ +import random +import string +from flask_babel import gettext + +# required answerer attribute +# specifies which search query keywords triggers this answerer +keywords = ('random',) + +random_int_max = 2**31 + +random_string_letters = string.lowercase + string.digits + string.uppercase + + +def random_string(): + return u''.join(random.choice(random_string_letters) + for _ in range(random.randint(8, 32))) + + +def random_float(): + return unicode(random.random()) + + +def random_int(): + return unicode(random.randint(-random_int_max, random_int_max)) + + +random_types = {u'string': random_string, + u'int': random_int, + u'float': random_float} + + +# required answerer function +# can return a list of results (any result type) for a given query +def answer(query): + parts = query.query.split() + if len(parts) != 2: + return [] + + if parts[1] not in random_types: + return [] + + return [{'answer': random_types[parts[1]]()}] + + +# required answerer function +# returns information about the answerer +def self_info(): + return {'name': gettext('Random value generator'), + 'description': gettext('Generate different random values'), + 'examples': [u'random {}'.format(x) for x in random_types]}diff --git a/searx/answerers/statistics/answerer.py b/searx/answerers/statistics/answerer.py@@ -0,0 +1,51 @@ +from functools import reduce +from operator import mul + +from flask_babel import gettext + +keywords = ('min', + 'max', + 'avg', + 'sum', + 'prod') + + +# required answerer function +# can return a list of results (any result type) for a given query +def answer(query): + parts = query.query.split() + + if len(parts) < 2: + return [] + + try: + args = map(float, parts[1:]) + except: + return [] + + func = parts[0] + answer = None + + if func == 'min': + answer = min(args) + elif func == 'max': + answer = max(args) + elif func == 'avg': + answer = sum(args) / len(args) + elif func == 'sum': + answer = sum(args) + elif func == 'prod': + answer = reduce(mul, args, 1) + + if answer is None: + return [] + + return [{'answer': unicode(answer)}] + + +# required answerer function +# returns information about the answerer +def self_info(): + return {'name': gettext('Statistics functions'), + 'description': gettext('Compute {functions} of the arguments').format(functions='/'.join(keywords)), + 'examples': ['avg 123 548 2.04 24.2']}diff --git a/searx/engines/__init__.py b/searx/engines/__init__.py@@ -16,13 +16,13 @@ along with searx. If not, see < http://www.gnu.org/licenses/ >. (C) 2013- by Adam Tauber, <asciimoo@gmail.com> ''' -from os.path import realpath, dirname, splitext, join +from os.path import realpath, dirname import sys -from imp import load_source from flask_babel import gettext from operator import itemgetter from searx import settings from searx import logger +from searx.utils import load_module logger = logger.getChild('engines') @@ -32,6 +32,7 @@ engine_dir = dirname(realpath(__file__)) engines = {} categories = {'general': []} +_initialized = False engine_shortcuts = {} engine_default_args = {'paging': False, @@ -46,16 +47,6 @@ engine_default_args = {'paging': False, 'time_range_support': False} -def load_module(filename): - modname = splitext(filename)[0] - if modname in sys.modules: - del sys.modules[modname] - filepath = join(engine_dir, filename) - module = load_source(modname, filepath) - module.name = modname - return module - - def load_engine(engine_data): if '_' in engine_data['name']: @@ -65,7 +56,7 @@ def load_engine(engine_data): engine_module = engine_data['engine'] try: - engine = load_module(engine_module + '.py') + engine = load_module(engine_module + '.py', engine_dir) except: logger.exception('Cannot load engine "{}"'.format(engine_module)) return Nonediff --git a/searx/engines/archlinux.py b/searx/engines/archlinux.py@@ -12,7 +12,6 @@ """ from urlparse import urljoin -from cgi import escape from urllib import urlencode from lxml import html from searx.engines.xpath import extract_text @@ -135,7 +134,7 @@ def response(resp): for result in dom.xpath(xpath_results): link = result.xpath(xpath_link)[0] href = urljoin(base_url, link.attrib.get('href')) - title = escape(extract_text(link)) + title = extract_text(link) results.append({'url': href, 'title': title})diff --git a/searx/engines/base.py b/searx/engines/base.py@@ -16,7 +16,6 @@ from lxml import etree from urllib import urlencode from searx.utils import searx_useragent -from cgi import escape from datetime import datetime import re @@ -94,7 +93,7 @@ def response(resp): url = item.text elif item.attrib["name"] == "dcdescription": - content = escape(item.text[:300]) + content = item.text[:300] if len(item.text) > 300: content += "..."diff --git a/searx/engines/bing.py b/searx/engines/bing.py@@ -14,7 +14,6 @@ """ from urllib import urlencode -from cgi import escape from lxml import html from searx.engines.xpath import extract_text @@ -32,18 +31,14 @@ search_string = 'search?{query}&first={offset}' def request(query, params): offset = (params['pageno'] - 1) * 10 + 1 - if params['language'] == 'all': - language = 'en-US' - else: - language = params['language'].replace('_', '-') + if params['language'] != 'all': + query = u'language:{} {}'.format(params['language'].split('_')[0].upper(), + query.decode('utf-8')).encode('utf-8') search_path = search_string.format( - query=urlencode({'q': query, 'setmkt': language}), + query=urlencode({'q': query}), offset=offset) - params['cookies']['SRCHHPGUSR'] = \ - 'NEWWND=0&NRSLT=-1&SRCHLANG=' + language.split('-')[0] - params['url'] = base_url + search_path return params @@ -65,7 +60,7 @@ def response(resp): link = result.xpath('.//h3/a')[0] url = link.attrib.get('href') title = extract_text(link) - content = escape(extract_text(result.xpath('.//p'))) + content = extract_text(result.xpath('.//p')) # append result results.append({'url': url, @@ -77,7 +72,7 @@ def response(resp): link = result.xpath('.//h2/a')[0] url = link.attrib.get('href') title = extract_text(link) - content = escape(extract_text(result.xpath('.//p'))) + content = extract_text(result.xpath('.//p')) # append result results.append({'url': url,diff --git a/searx/engines/btdigg.py b/searx/engines/btdigg.py@@ -11,7 +11,6 @@ """ from urlparse import urljoin -from cgi import escape from urllib import quote from lxml import html from operator import itemgetter @@ -51,8 +50,8 @@ def response(resp): for result in search_res: link = result.xpath('.//td[@class="torrent_name"]//a')[0] href = urljoin(url, link.attrib.get('href')) - title = escape(extract_text(link)) - content = escape(extract_text(result.xpath('.//pre[@class="snippet"]')[0])) + title = extract_text(link) + content = extract_text(result.xpath('.//pre[@class="snippet"]')[0]) content = "<br />".join(content.split("\n")) filesize = result.xpath('.//span[@class="attr_val"]/text()')[0].split()[0]diff --git a/searx/engines/dailymotion.py b/searx/engines/dailymotion.py@@ -14,7 +14,6 @@ from urllib import urlencode from json import loads -from cgi import escape from datetime import datetime # engine dependent config @@ -57,7 +56,7 @@ def response(resp): for res in search_res['list']: title = res['title'] url = res['url'] - content = escape(res['description']) + content = res['description'] thumbnail = res['thumbnail_360_url'] publishedDate = datetime.fromtimestamp(res['created_time'], None) embedded = embedded_url.format(videoid=res['id'])diff --git a/searx/engines/deezer.py b/searx/engines/deezer.py@@ -51,10 +51,11 @@ def response(resp): if url.startswith('http://'): url = 'https' + url[4:] - content = result['artist']['name'] +\ - " • " +\ - result['album']['title'] +\ - " • " + result['title'] + content = '{} - {} - {}'.format( + result['artist']['name'], + result['album']['title'], + result['title']) + embedded = embedded_url.format(audioid=result['id']) # append resultdiff --git a/searx/engines/dictzone.py b/searx/engines/dictzone.py@@ -12,7 +12,6 @@ import re from urlparse import urljoin from lxml import html -from cgi import escape from searx.utils import is_valid_lang categories = ['general'] @@ -62,8 +61,8 @@ def response(resp): results.append({ 'url': urljoin(resp.url, '?%d' % k), - 'title': escape(from_result.text_content()), - 'content': escape('; '.join(to_results)) + 'title': from_result.text_content(), + 'content': '; '.join(to_results) }) return resultsdiff --git a/searx/engines/digg.py b/searx/engines/digg.py@@ -13,7 +13,6 @@ from urllib import quote_plus from json import loads from lxml import html -from cgi import escape from dateutil import parser # engine dependent config @@ -56,7 +55,7 @@ def response(resp): url = result.attrib.get('data-contenturl') thumbnail = result.xpath('.//img')[0].attrib.get('src') title = ''.join(result.xpath(title_xpath)) - content = escape(''.join(result.xpath(content_xpath))) + content = ''.join(result.xpath(content_xpath)) pubdate = result.xpath(pubdate_xpath)[0].attrib.get('datetime') publishedDate = parser.parse(pubdate)diff --git a/searx/engines/fdroid.py b/searx/engines/fdroid.py@@ -9,7 +9,6 @@ @parse url, title, content """ -from cgi import escape from urllib import urlencode from searx.engines.xpath import extract_text from lxml import html @@ -43,7 +42,7 @@ def response(resp): img_src = app.xpath('.//img/@src')[0] content = extract_text(app.xpath('./p')[0]) - content = escape(content.replace(title, '', 1).strip()) + content = content.replace(title, '', 1).strip() results.append({'url': url, 'title': title,diff --git a/searx/engines/flickr.py b/searx/engines/flickr.py@@ -77,21 +77,13 @@ def response(resp): url = build_flickr_url(photo['owner'], photo['id']) - title = photo['title'] - - content = '<span class="photo-author">' +\ - photo['ownername'] +\ - '</span><br />' +\ - '<span class="description">' +\ - photo['description']['_content'] +\ - '</span>' - # append result results.append({'url': url, - 'title': title, + 'title': photo['title'], 'img_src': img_src, 'thumbnail_src': thumbnail_src, - 'content': content, + 'content': photo['description']['_content'], + 'author': photo['ownername'], 'template': 'images.html'}) # return resultsdiff --git a/searx/engines/flickr_noapi.py b/searx/engines/flickr_noapi.py@@ -102,16 +102,15 @@ def response(resp): title = photo.get('title', '') - content = '<span class="photo-author">' +\ - photo['username'] +\ - '</span><br />' + author = photo['username'] # append result results.append({'url': url, 'title': title, 'img_src': img_src, 'thumbnail_src': thumbnail_src, - 'content': content, + 'content': '', + 'author': author, 'template': 'images.html'}) return resultsdiff --git a/searx/engines/gigablast.py b/searx/engines/gigablast.py@@ -10,7 +10,6 @@ @parse url, title, content """ -from cgi import escape from json import loads from random import randint from time import time @@ -78,8 +77,8 @@ def response(resp): for result in response_json['results']: # append result results.append({'url': result['url'], - 'title': escape(result['title']), - 'content': escape(result['sum'])}) + 'title': result['title'], + 'content': result['sum']}) # return results return resultsdiff --git a/searx/engines/github.py b/searx/engines/github.py@@ -12,7 +12,6 @@ from urllib import urlencode from json import loads -from cgi import escape # engine dependent config categories = ['it'] @@ -48,7 +47,7 @@ def response(resp): url = res['html_url'] if res['description']: - content = escape(res['description'][:500]) + content = res['description'][:500] else: content = ''diff --git a/searx/engines/google.py b/searx/engines/google.py@@ -9,7 +9,6 @@ # @parse url, title, content, suggestion import re -from cgi import escape from urllib import urlencode from urlparse import urlparse, parse_qsl from lxml import html, etree @@ -155,7 +154,7 @@ def parse_url(url_string, google_hostname): def extract_text_from_dom(result, xpath): r = result.xpath(xpath) if len(r) > 0: - return escape(extract_text(r[0])) + return extract_text(r[0]) return None @@ -264,7 +263,7 @@ def response(resp): # parse suggestion for suggestion in dom.xpath(suggestion_xpath): # append suggestion - results.append({'suggestion': escape(extract_text(suggestion))}) + results.append({'suggestion': extract_text(suggestion)}) # return results return resultsdiff --git a/searx/engines/kickass.py b/searx/engines/kickass.py@@ -11,7 +11,6 @@ """ from urlparse import urljoin -from cgi import escape from urllib import quote from lxml import html from operator import itemgetter @@ -57,7 +56,7 @@ def response(resp): link = result.xpath('.//a[@class="cellMainLink"]')[0] href = urljoin(url, link.attrib['href']) title = extract_text(link) - content = escape(extract_text(result.xpath(content_xpath))) + content = extract_text(result.xpath(content_xpath)) seed = extract_text(result.xpath('.//td[contains(@class, "green")]')) leech = extract_text(result.xpath('.//td[contains(@class, "red")]')) filesize_info = extract_text(result.xpath('.//td[contains(@class, "nobr")]'))diff --git a/searx/engines/nyaa.py b/searx/engines/nyaa.py@@ -9,7 +9,6 @@ @parse url, title, content, seed, leech, torrentfile """ -from cgi import escape from urllib import urlencode from lxml import html from searx.engines.xpath import extract_text @@ -78,7 +77,7 @@ def response(resp): # torrent title page_a = result.xpath(xpath_title)[0] - title = escape(extract_text(page_a)) + title = extract_text(page_a) # link to the page href = page_a.attrib.get('href') @@ -90,7 +89,7 @@ def response(resp): try: file_size, suffix = result.xpath(xpath_filesize)[0].split(' ') file_size = int(float(file_size) * get_filesize_mul(suffix)) - except Exception as e: + except: file_size = None # seed count @@ -105,7 +104,6 @@ def response(resp): # content string contains all information not included into template content = 'Category: "{category}". Downloaded {downloads} times.' content = content.format(category=category, downloads=downloads) - content = escape(content) results.append({'url': href, 'title': title,diff --git a/searx/engines/openstreetmap.py b/searx/engines/openstreetmap.py@@ -43,7 +43,7 @@ def response(resp): if 'display_name' not in r: continue - title = r['display_name'] + title = r['display_name'] or u'' osm_type = r.get('osm_type', r.get('type')) url = result_base_url.format(osm_type=osm_type, osm_id=r['osm_id'])diff --git a/searx/engines/piratebay.py b/searx/engines/piratebay.py@@ -9,7 +9,6 @@ # @parse url, title, content, seed, leech, magnetlink from urlparse import urljoin -from cgi import escape from urllib import quote from lxml import html from operator import itemgetter @@ -62,7 +61,7 @@ def response(resp): link = result.xpath('.//div[@class="detName"]//a')[0] href = urljoin(url, link.attrib.get('href')) title = extract_text(link) - content = escape(extract_text(result.xpath(content_xpath))) + content = extract_text(result.xpath(content_xpath)) seed, leech = result.xpath('.//td[@align="right"]/text()')[:2] # convert seed to int if possiblediff --git a/searx/engines/reddit.py b/searx/engines/reddit.py@@ -11,7 +11,6 @@ """ import json -from cgi import escape from urllib import urlencode from urlparse import urlparse, urljoin from datetime import datetime @@ -68,7 +67,7 @@ def response(resp): img_results.append(params) else: created = datetime.fromtimestamp(data['created_utc']) - content = escape(data['selftext']) + content = data['selftext'] if len(content) > 500: content = content[:500] + '...' params['content'] = contentdiff --git a/searx/engines/searchcode_doc.py b/searx/engines/searchcode_doc.py@@ -44,20 +44,12 @@ def response(resp): # parse results for result in search_results.get('results', []): href = result['url'] - title = "[" + result['type'] + "] " +\ - result['namespace'] +\ - " " + result['name'] - content = '<span class="highlight">[' +\ - result['type'] + "] " +\ - result['name'] + " " +\ - result['synopsis'] +\ - "</span><br />" +\ - result['description'] + title = "[{}] {} {}".format(result['type'], result['namespace'], result['name']) # append result results.append({'url': href, 'title': title, - 'content': content}) + 'content': result['description']}) # return results return resultsdiff --git a/searx/engines/seedpeer.py b/searx/engines/seedpeer.py@@ -9,7 +9,6 @@ # @parse url, title, content, seed, leech, magnetlink from urlparse import urljoin -from cgi import escape from urllib import quote from lxml import html from operator import itemgetterdiff --git a/searx/engines/spotify.py b/searx/engines/spotify.py@@ -46,10 +46,11 @@ def response(resp): if result['type'] == 'track': title = result['name'] url = result['external_urls']['spotify'] - content = result['artists'][0]['name'] +\ - " • " +\ - result['album']['name'] +\ - " • " + result['name'] + content = '{} - {} - {}'.format( + result['artists'][0]['name'], + result['album']['name'], + result['name']) + embedded = embedded_url.format(audioid=result['id']) # append resultdiff --git a/searx/engines/stackoverflow.py b/searx/engines/stackoverflow.py@@ -11,7 +11,6 @@ """ from urlparse import urljoin -from cgi import escape from urllib import urlencode from lxml import html from searx.engines.xpath import extract_text @@ -48,8 +47,8 @@ def response(resp): for result in dom.xpath(results_xpath): link = result.xpath(link_xpath)[0] href = urljoin(url, link.attrib.get('href')) - title = escape(extract_text(link)) - content = escape(extract_text(result.xpath(content_xpath))) + title = extract_text(link) + content = extract_text(result.xpath(content_xpath)) # append result results.append({'url': href,diff --git a/searx/engines/startpage.py b/searx/engines/startpage.py@@ -11,7 +11,6 @@ # @todo paging from lxml import html -from cgi import escape from dateutil import parser from datetime import datetime, timedelta import re @@ -79,10 +78,10 @@ def response(resp): if re.match(r"^http(s|)://(www\.)?ixquick\.com/do/search\?.*$", url): continue - title = escape(extract_text(link)) + title = extract_text(link) if result.xpath('./p[@class="desc clk"]'): - content = escape(extract_text(result.xpath('./p[@class="desc clk"]'))) + content = extract_text(result.xpath('./p[@class="desc clk"]')) else: content = ''diff --git a/searx/engines/subtitleseeker.py b/searx/engines/subtitleseeker.py@@ -10,7 +10,6 @@ @parse url, title, content """ -from cgi import escape from urllib import quote_plus from lxml import html from searx.languages import language_codes @@ -59,7 +58,7 @@ def response(resp): elif search_lang: href = href + search_lang + '/' - title = escape(extract_text(link)) + title = extract_text(link) content = extract_text(result.xpath('.//div[contains(@class,"red")]')) content = content + " - " @@ -75,7 +74,7 @@ def response(resp): # append result results.append({'url': href, 'title': title, - 'content': escape(content)}) + 'content': content}) # return results return resultsdiff --git a/searx/engines/swisscows.py b/searx/engines/swisscows.py@@ -10,7 +10,6 @@ @parse url, title, content """ -from cgi import escape from json import loads from urllib import urlencode, unquote import re @@ -78,7 +77,7 @@ def response(resp): # append result results.append({'url': result['SourceUrl'], - 'title': escape(result['Title']), + 'title': result['Title'], 'content': '', 'img_src': img_url, 'template': 'images.html'}) @@ -90,8 +89,8 @@ def response(resp): # append result results.append({'url': result_url, - 'title': escape(result_title), - 'content': escape(result_content)}) + 'title': result_title, + 'content': result_content}) # parse images for result in json.get('Images', []): @@ -100,7 +99,7 @@ def response(resp): # append result results.append({'url': result['SourceUrl'], - 'title': escape(result['Title']), + 'title': result['Title'], 'content': '', 'img_src': img_url, 'template': 'images.html'})diff --git a/searx/engines/tokyotoshokan.py b/searx/engines/tokyotoshokan.py@@ -11,7 +11,6 @@ """ import re -from cgi import escape from urllib import urlencode from lxml import html from searx.engines.xpath import extract_textdiff --git a/searx/engines/torrentz.py b/searx/engines/torrentz.py@@ -12,7 +12,6 @@ """ import re -from cgi import escape from urllib import urlencode from lxml import html from searx.engines.xpath import extract_textdiff --git a/searx/engines/translated.py b/searx/engines/translated.py@@ -9,7 +9,6 @@ @parse url, title, content """ import re -from cgi import escape from searx.utils import is_valid_lang categories = ['general'] @@ -52,14 +51,14 @@ def request(query, params): def response(resp): results = [] results.append({ - 'url': escape(web_url.format( + 'url': web_url.format( from_lang=resp.search_params['from_lang'][2], to_lang=resp.search_params['to_lang'][2], - query=resp.search_params['query'])), - 'title': escape('[{0}-{1}] {2}'.format( + query=resp.search_params['query']), + 'title': '[{0}-{1}] {2}'.format( resp.search_params['from_lang'][1], resp.search_params['to_lang'][1], - resp.search_params['query'])), - 'content': escape(resp.json()['responseData']['translatedText']) + resp.search_params['query']), + 'content': resp.json()['responseData']['translatedText'] }) return resultsdiff --git a/searx/engines/wolframalpha_noapi.py b/searx/engines/wolframalpha_noapi.py@@ -8,7 +8,6 @@ # @stable no # @parse url, infobox -from cgi import escape from json import loads from time import time from urllib import urlencodediff --git a/searx/engines/yandex.py b/searx/engines/yandex.py@@ -9,7 +9,6 @@ @parse url, title, content """ -from cgi import escape from urllib import urlencode from lxml import html from searx.search import logger @@ -52,8 +51,8 @@ def response(resp): for result in dom.xpath(results_xpath): try: res = {'url': result.xpath(url_xpath)[0], - 'title': escape(''.join(result.xpath(title_xpath))), - 'content': escape(''.join(result.xpath(content_xpath)))} + 'title': ''.join(result.xpath(title_xpath)), + 'content': ''.join(result.xpath(content_xpath))} except: logger.exception('yandex parse crash') continuediff --git a/searx/plugins/doai_rewrite.py b/searx/plugins/doai_rewrite.py@@ -27,5 +27,5 @@ def on_result(request, search, result): if doi.endswith(suffix): doi = doi[:-len(suffix)] result['url'] = 'http://doai.io/' + doi - result['parsed_url'] = urlparse(ctx['result']['url']) + result['parsed_url'] = urlparse(result['url']) return Truediff --git a/searx/preferences.py b/searx/preferences.py@@ -49,28 +49,32 @@ class StringSetting(Setting): class EnumStringSetting(Setting): """Setting of a value which can only come from the given choices""" + def _validate_selection(self, selection): + if selection not in self.choices: + raise ValidationException('Invalid value: "{0}"'.format(selection)) + def _post_init(self): if not hasattr(self, 'choices'): raise MissingArgumentException('Missing argument: choices') - - if self.value != '' and self.value not in self.choices: - raise ValidationException('Invalid default value: {0}'.format(self.value)) + self._validate_selection(self.value) def parse(self, data): - if data not in self.choices and data != self.value: - raise ValidationException('Invalid choice: {0}'.format(data)) + self._validate_selection(data) self.value = data class MultipleChoiceSetting(EnumStringSetting): """Setting of values which can only come from the given choices""" + def _validate_selections(self, selections): + for item in selections: + if item not in self.choices: + raise ValidationException('Invalid value: "{0}"'.format(selections)) + def _post_init(self): if not hasattr(self, 'choices'): raise MissingArgumentException('Missing argument: choices') - for item in self.value: - if item not in self.choices: - raise ValidationException('Invalid default value: {0}'.format(self.value)) + self._validate_selections(self.value) def parse(self, data): if data == '': @@ -78,9 +82,7 @@ class MultipleChoiceSetting(EnumStringSetting): return elements = data.split(',') - for item in elements: - if item not in self.choices: - raise ValidationException('Invalid choice: {0}'.format(item)) + self._validate_selections(elements) self.value = elements def parse_form(self, data): @@ -214,11 +216,12 @@ class Preferences(object): super(Preferences, self).__init__() self.key_value_settings = {'categories': MultipleChoiceSetting(['general'], choices=categories), - 'language': EnumStringSetting('all', choices=LANGUAGE_CODES), + 'language': EnumStringSetting(settings['search']['language'], + choices=LANGUAGE_CODES), 'locale': EnumStringSetting(settings['ui']['default_locale'], - choices=settings['locales'].keys()), + choices=settings['locales'].keys() + ['']), 'autocomplete': EnumStringSetting(settings['search']['autocomplete'], - choices=autocomplete.backends.keys()), + choices=autocomplete.backends.keys() + ['']), 'image_proxy': MapSetting(settings['server']['image_proxy'], map={'': settings['server']['image_proxy'], '0': False,diff --git a/searx/results.py b/searx/results.py@@ -146,16 +146,17 @@ class ResultContainer(object): self._number_of_results.append(result['number_of_results']) results.remove(result) - with RLock(): - engines[engine_name].stats['search_count'] += 1 - engines[engine_name].stats['result_count'] += len(results) + if engine_name in engines: + with RLock(): + engines[engine_name].stats['search_count'] += 1 + engines[engine_name].stats['result_count'] += len(results) if not results: return self.results[engine_name].extend(results) - if not self.paging and engines[engine_name].paging: + if not self.paging and engine_name in engines and engines[engine_name].paging: self.paging = True for i, result in enumerate(results):diff --git a/searx/search.py b/searx/search.py@@ -24,6 +24,7 @@ import searx.poolrequests as requests_lib from searx.engines import ( categories, engines ) +from searx.answerers import ask from searx.utils import gen_useragent from searx.query import RawTextQuery, SearchQuery from searx.results import ResultContainer @@ -300,6 +301,14 @@ class Search(object): # start time start_time = time() + # answeres ? + answerers_results = ask(self.search_query) + + if answerers_results: + for results in answerers_results: + self.result_container.extend('answer', results) + return self.result_container + # init vars requests = []diff --git a/searx/settings.yml b/searx/settings.yml@@ -5,6 +5,7 @@ general: search: safe_search : 0 # Filter results. 0: None, 1: Moderate, 2: Strict autocomplete : "" # Existing autocomplete backends: "dbpedia", "duckduckgo", "google", "startpage", "wikipedia" - leave blank to turn it off by default + language : "all" server: port : 8888diff --git a/searx/settings_robot.yml b/searx/settings_robot.yml@@ -5,6 +5,7 @@ general: search: safe_search : 0 autocomplete : "" + language: "all" server: port : 11111diff --git a/searx/static/plugins/js/infinite_scroll.js b/searx/static/plugins/js/infinite_scroll.js@@ -5,7 +5,7 @@ $(document).ready(function() { var formData = $('#pagination form:last').serialize(); if (formData) { $('#pagination').html('<div class="loading-spinner"></div>'); - $.post('/', formData, function (data) { + $.post('./', formData, function (data) { var body = $(data); $('#pagination').remove(); $('#main_results').append('<hr/>');diff --git a/searx/templates/courgette/opensearch_response_rss.xml b/searx/templates/courgette/opensearch_response_rss.xml@@ -3,14 +3,14 @@ xmlns:opensearch="http://a9.com/-/spec/opensearch/1.1/" xmlns:atom="http://www.w3.org/2005/Atom"> <channel> - <title>Searx search: {{ q }}</title> - <link>{{ base_url }}?q={{ q }}</link> - <description>Search results for "{{ q }}" - searx</description> + <title>Searx search: {{ q|e }}</title> + <link>{{ base_url }}?q={{ q|e }}</link> + <description>Search results for "{{ q|e }}" - searx</description> <opensearch:totalResults>{{ number_of_results }}</opensearch:totalResults> <opensearch:startIndex>1</opensearch:startIndex> <opensearch:itemsPerPage>{{ number_of_results }}</opensearch:itemsPerPage> <atom:link rel="search" type="application/opensearchdescription+xml" href="{{ base_url }}opensearch.xml"/> - <opensearch:Query role="request" searchTerms="{{ q }}" startPage="1" /> + <opensearch:Query role="request" searchTerms="{{ q|e }}" startPage="1" /> {% for r in results %} <item> <title>{{ r.title }}</title>diff --git a/searx/templates/courgette/results.html b/searx/templates/courgette/results.html@@ -1,6 +1,6 @@ {% extends "courgette/base.html" %} -{% block title %}{{ q }} - {% endblock %} -{% block meta %}<link rel="alternate" type="application/rss+xml" title="Searx search: {{ q }}" href="{{ url_for('index') }}?q={{ q|urlencode }}&format=rss&{% for category in selected_categories %}category_{{ category }}=1&{% endfor %}pageno={{ pageno }}">{% endblock %} +{% block title %}{{ q|e }} - {% endblock %} +{% block meta %}<link rel="alternate" type="application/rss+xml" title="Searx search: {{ q|e }}" href="{{ url_for('index') }}?q={{ q|urlencode }}&format=rss&{% for category in selected_categories %}category_{{ category }}=1&{% endfor %}pageno={{ pageno }}">{% endblock %} {% block content %} <div class="right"><a href="{{ url_for('preferences') }}" id="preferences"><span>{{ _('preferences') }}</span></a></div> <div class="small search center"> @@ -17,7 +17,7 @@ {% for output_type in ('csv', 'json', 'rss') %} <form method="{{ method or 'POST' }}" action="{{ url_for('index') }}"> <div class="left"> - <input type="hidden" name="q" value="{{ q }}" /> + <input type="hidden" name="q" value="{{ q|e }}" /> <input type="hidden" name="format" value="{{ output_type }}" /> {% for category in selected_categories %} <input type="hidden" name="category_{{ category }}" value="1"/> @@ -62,7 +62,7 @@ {% if pageno > 1 %} <form method="{{ method or 'POST' }}" action="{{ url_for('index') }}"> <div class="left"> - <input type="hidden" name="q" value="{{ q }}" /> + <input type="hidden" name="q" value="{{ q|e }}" /> {% for category in selected_categories %} <input type="hidden" name="category_{{ category }}" value="1"/> {% endfor %} @@ -76,7 +76,7 @@ {% for category in selected_categories %} <input type="hidden" name="category_{{ category }}" value="1"/> {% endfor %} - <input type="hidden" name="q" value="{{ q }}" /> + <input type="hidden" name="q" value="{{ q|e }}" /> <input type="hidden" name="pageno" value="{{ pageno+1 }}" /> <input type="submit" value="{{ _('next page') }} >>" /> </div>diff --git a/searx/templates/legacy/opensearch_response_rss.xml b/searx/templates/legacy/opensearch_response_rss.xml@@ -3,14 +3,14 @@ xmlns:opensearch="http://a9.com/-/spec/opensearch/1.1/" xmlns:atom="http://www.w3.org/2005/Atom"> <channel> - <title>Searx search: {{ q }}</title> - <link>{{ base_url }}?q={{ q }}</link> - <description>Search results for "{{ q }}" - searx</description> + <title>Searx search: {{ q|e }}</title> + <link>{{ base_url }}?q={{ q|e }}</link> + <description>Search results for "{{ q|e }}" - searx</description> <opensearch:totalResults>{{ number_of_results }}</opensearch:totalResults> <opensearch:startIndex>1</opensearch:startIndex> <opensearch:itemsPerPage>{{ number_of_results }}</opensearch:itemsPerPage> <atom:link rel="search" type="application/opensearchdescription+xml" href="{{ base_url }}opensearch.xml"/> - <opensearch:Query role="request" searchTerms="{{ q }}" startPage="1" /> + <opensearch:Query role="request" searchTerms="{{ q|e }}" startPage="1" /> {% for r in results %} <item> <title>{{ r.title }}</title>diff --git a/searx/templates/legacy/results.html b/searx/templates/legacy/results.html@@ -1,6 +1,6 @@ {% extends "legacy/base.html" %} -{% block title %}{{ q }} - {% endblock %} -{% block meta %}<link rel="alternate" type="application/rss+xml" title="Searx search: {{ q }}" href="{{ url_for('index') }}?q={{ q|urlencode }}&format=rss&{% for category in selected_categories %}category_{{ category }}=1&{% endfor %}pageno={{ pageno }}">{% endblock %} +{% block title %}{{ q|e }} - {% endblock %} +{% block meta %}<link rel="alternate" type="application/rss+xml" title="Searx search: {{ q|e }}" href="{{ url_for('index') }}?q={{ q|urlencode }}&format=rss&{% for category in selected_categories %}category_{{ category }}=1&{% endfor %}pageno={{ pageno }}">{% endblock %} {% block content %} <div class="preferences_container right"><a href="{{ url_for('preferences') }}" id="preferences"><span>preferences</span></a></div> <div class="small search center"> @@ -18,7 +18,7 @@ {% for output_type in ('csv', 'json', 'rss') %} <form method="{{ method or 'POST' }}" action="{{ url_for('index') }}"> <div class="left"> - <input type="hidden" name="q" value="{{ q }}" /> + <input type="hidden" name="q" value="{{ q|e }}" /> <input type="hidden" name="format" value="{{ output_type }}" /> {% for category in selected_categories %} <input type="hidden" name="category_{{ category }}" value="1"/> @@ -73,7 +73,7 @@ {% if pageno > 1 %} <form method="{{ method or 'POST' }}" action="{{ url_for('index') }}"> <div class="{% if rtl %}right{% else %}left{% endif %}"> - <input type="hidden" name="q" value="{{ q }}" /> + <input type="hidden" name="q" value="{{ q|e }}" /> {% for category in selected_categories %} <input type="hidden" name="category_{{ category }}" value="1"/> {% endfor %} @@ -87,7 +87,7 @@ {% for category in selected_categories %} <input type="hidden" name="category_{{ category }}" value="1"/> {% endfor %} - <input type="hidden" name="q" value="{{ q }}" /> + <input type="hidden" name="q" value="{{ q|e }}" /> <input type="hidden" name="pageno" value="{{ pageno+1 }}" /> <input type="submit" value="{{ _('next page') }} >>" /> </div>diff --git a/searx/templates/oscar/base.html b/searx/templates/oscar/base.html@@ -1,3 +1,4 @@ +{% from 'oscar/macros.html' import icon %} <!DOCTYPE html> <html xmlns="http://www.w3.org/1999/xhtml" lang="en" xml:lang="en"{% if rtl %} dir="rtl"{% endif %}> <head> @@ -54,6 +55,20 @@ <body> {% include 'oscar/navbar.html' %} <div class="container"> + {% if errors %} + <div class="alert alert-danger fade in" role="alert"> + <button class="close" data-dismiss="alert" type="button"> + <span aria-hidden="true">×</span> + <span class="sr-only">{{ _('Close') }}</span> + </button> + <strong class="lead">{{ icon('info-sign') }} {{ _('Error!') }}</strong> + <ul> + {% for message in errors %} + <li>{{ message }}</li> + {% endfor %} + </ul> + </div> + {% endif %} {% block site_alert_error %} {% endblock %}diff --git a/searx/templates/oscar/opensearch_response_rss.xml b/searx/templates/oscar/opensearch_response_rss.xml@@ -3,14 +3,14 @@ xmlns:opensearch="http://a9.com/-/spec/opensearch/1.1/" xmlns:atom="http://www.w3.org/2005/Atom"> <channel> - <title>Searx search: {{ q }}</title> - <link>{{ base_url }}?q={{ q }}</link> - <description>Search results for "{{ q }}" - searx</description> + <title>Searx search: {{ q|e }}</title> + <link>{{ base_url }}?q={{ q|e }}</link> + <description>Search results for "{{ q|e }}" - searx</description> <opensearch:totalResults>{{ number_of_results }}</opensearch:totalResults> <opensearch:startIndex>1</opensearch:startIndex> <opensearch:itemsPerPage>{{ number_of_results }}</opensearch:itemsPerPage> <atom:link rel="search" type="application/opensearchdescription+xml" href="{{ base_url }}opensearch.xml"/> - <opensearch:Query role="request" searchTerms="{{ q }}" startPage="1" /> + <opensearch:Query role="request" searchTerms="{{ q|e }}" startPage="1" /> {% for r in results %} <item> <title>{{ r.title }}</title>diff --git a/searx/templates/oscar/preferences.html b/searx/templates/oscar/preferences.html@@ -12,6 +12,7 @@ <li class="active"><a href="#tab_general" role="tab" data-toggle="tab">{{ _('General') }}</a></li> <li><a href="#tab_engine" role="tab" data-toggle="tab">{{ _('Engines') }}</a></li> <li><a href="#tab_plugins" role="tab" data-toggle="tab">{{ _('Plugins') }}</a></li> + {% if answerers %}<li><a href="#tab_answerers" role="tab" data-toggle="tab">{{ _('Answerers') }}</a></li>{% endif %} <li><a href="#tab_cookies" role="tab" data-toggle="tab">{{ _('Cookies') }}</a></li> </ul> @@ -224,6 +225,34 @@ </fieldset> </div> + {% if answerers %} + <div class="tab-pane active_if_nojs" id="tab_answerers"> + <noscript> + <h3>{{ _('Answerers') }}</h3> + </noscript> + <p class="text-muted" style="margin:20px 0;"> + {{ _('This is the list of searx\'s instant answering modules.') }} + </p> + <table class="table table-striped"> + <tr> + <th class="text-muted">{{ _('Name') }}</th> + <th class="text-muted">{{ _('Keywords') }}</th> + <th class="text-muted">{{ _('Description') }}</th> + <th class="text-muted">{{ _('Examples') }}</th> + </tr> + + {% for answerer in answerers %} + <tr> + <td class="text-muted">{{ answerer.info.name }}</td> + <td class="text-muted">{{ answerer.keywords|join(', ') }}</td> + <td class="text-muted">{{ answerer.info.description }}</td> + <td class="text-muted">{{ answerer.info.examples|join(', ') }}</td> + </tr> + {% endfor %} + </table> + </div> + {% endif %} + <div class="tab-pane active_if_nojs" id="tab_cookies"> <noscript> <h3>{{ _('Cookies') }}</h3>diff --git a/searx/templates/oscar/result_templates/images.html b/searx/templates/oscar/result_templates/images.html@@ -13,7 +13,12 @@ </div> <div class="modal-body"> <img class="img-responsive center-block" src="{% if result.thumbnail_src %}{{ image_proxify(result.thumbnail_src) }}{% else %}{{ image_proxify(result.img_src) }}{% endif %}" alt="{{ result.title|striptags }}"> - {% if result.content %}<p class="result-content">{{ result.content|safe }}</p>{% endif %} + {% if result.author %}<span class="photo-author">{{ result.author }}</span><br />{% endif %} + {% if result.content %} + <p class="result-content"> + {{ result.content }} + </p> + {% endif %} </div> <div class="modal-footer"> <div class="clearfix"></div>diff --git a/searx/templates/oscar/results.html b/searx/templates/oscar/results.html@@ -1,6 +1,6 @@ {% extends "oscar/base.html" %} -{% block title %}{{ q }} - {% endblock %} -{% block meta %}<link rel="alternate" type="application/rss+xml" title="Searx search: {{ q }}" href="{{ url_for('index') }}?q={{ q|urlencode }}&format=rss&{% for category in selected_categories %}category_{{ category }}=1&{% endfor %}pageno={{ pageno }}&time_range={{ time_range }}">{% endblock %} +{% block title %}{{ q|e }} - {% endblock %} +{% block meta %}<link rel="alternate" type="application/rss+xml" title="Searx search: {{ q|e }}" href="{{ url_for('index') }}?q={{ q|urlencode }}&format=rss&{% for category in selected_categories %}category_{{ category }}=1&{% endfor %}pageno={{ pageno }}&time_range={{ time_range }}">{% endblock %} {% block content %} <div class="row"> <div class="col-sm-8" id="main_results"> @@ -37,9 +37,9 @@ <div id="pagination"> <div class="pull-left"> <form method="{{ method or 'POST' }}" action="{{ url_for('index') }}" class="pull-left"> - <input type="hidden" name="q" value="{{ q }}" /> + <input type="hidden" name="q" value="{{ q|e }}" /> {% for category in selected_categories %}<input type="hidden" name="category_{{ category }}" value="1"/>{% endfor %} - <input type="hidden" name="q" value="{{ q }}" /> + <input type="hidden" name="q" value="{{ q|e }}" /> <input type="hidden" name="pageno" value="{{ pageno+1 }}" /> <input type="hidden" name="time_range" value="{{ time_range }}" /> <button type="submit" class="btn btn-default"><span class="glyphicon glyphicon-backward"></span> {{ _('next page') }}</button> @@ -59,7 +59,7 @@ <div id="pagination"> <div class="pull-left"> <form method="{{ method or 'POST' }}" action="{{ url_for('index') }}" class="pull-left"> - <input type="hidden" name="q" value="{{ q }}" /> + <input type="hidden" name="q" value="{{ q|e }}" /> {% for category in selected_categories %}<input type="hidden" name="category_{{ category }}" value="1"/>{% endfor %} <input type="hidden" name="pageno" value="{{ pageno-1 }}" /> <input type="hidden" name="time_range" value="{{ time_range }}" /> @@ -69,7 +69,7 @@ <div class="pull-right"> <form method="{{ method or 'POST' }}" action="{{ url_for('index') }}" class="pull-left"> {% for category in selected_categories %}<input type="hidden" name="category_{{ category }}" value="1"/>{% endfor %} - <input type="hidden" name="q" value="{{ q }}" /> + <input type="hidden" name="q" value="{{ q|e }}" /> <input type="hidden" name="pageno" value="{{ pageno+1 }}" /> <input type="hidden" name="time_range" value="{{ time_range }}" /> <button type="submit" class="btn btn-default"><span class="glyphicon glyphicon-forward"></span> {{ _('next page') }}</button> @@ -130,7 +130,7 @@ <div class="clearfix"></div> {% for output_type in ('csv', 'json', 'rss') %} <form method="{{ method or 'POST' }}" action="{{ url_for('index') }}" class="form-inline pull-{% if rtl %}right{% else %}left{% endif %} result_download"> - <input type="hidden" name="q" value="{{ q }}"> + <input type="hidden" name="q" value="{{ q|e }}"> <input type="hidden" name="format" value="{{ output_type }}"> {% for category in selected_categories %}<input type="hidden" name="category_{{ category }}" value="1">{% endfor %} <input type="hidden" name="pageno" value="{{ pageno }}">diff --git a/searx/templates/pix-art/results.html b/searx/templates/pix-art/results.html@@ -5,7 +5,7 @@ {% endfor %} {% else %} {% extends "pix-art/base.html" %} -{% block title %}{{ q }} - {% endblock %} +{% block title %}{{ q|e }} - {% endblock %} {% block meta %}{% endblock %} {% block content %} <div id="logo"><a href="./"><img src="{{ url_for('static', filename='img/searx-pixel-small.png') }}" alt="searx Logo"/></a></div> @@ -25,8 +25,8 @@ </span> <div id="pagination"> <br /> - <input type="button" onclick="load_more('{{ q }}', {{ pageno+1 }})" id="load_more" value="{{ _('Load more...') }}" /> + <input type="button" onclick="load_more('{{ q|e }}', {{ pageno+1 }})" id="load_more" value="{{ _('Load more...') }}" /> </div> </div> {% endblock %} -{% endif %} -\ No newline at end of file +{% endif %}diff --git a/searx/utils.py b/searx/utils.py@@ -6,7 +6,10 @@ import re from babel.dates import format_date from codecs import getincrementalencoder from HTMLParser import HTMLParser +from imp import load_source +from os.path import splitext, join from random import choice +import sys from searx.version import VERSION_STRING from searx.languages import language_codes @@ -285,3 +288,13 @@ def is_valid_lang(lang): if l[1].lower() == lang.lower(): return (True, l[0][:2], l[1].lower()) return False + + +def load_module(filename, module_dir): + modname = splitext(filename)[0] + if modname in sys.modules: + del sys.modules[modname] + filepath = join(module_dir, filename) + module = load_source(modname, filepath) + module.name = modname + return modulediff --git a/searx/webapp.py b/searx/webapp.py@@ -40,7 +40,7 @@ except: logger.critical("cannot import dependency: pygments") from sys import exit exit(1) - +from cgi import escape from datetime import datetime, timedelta from urllib import urlencode from urlparse import urlparse, urljoin @@ -62,11 +62,12 @@ from searx.utils import ( ) from searx.version import VERSION_STRING from searx.languages import language_codes -from searx.search import Search, SearchWithPlugins, get_search_query_from_webapp -from searx.query import RawTextQuery, SearchQuery +from searx.search import SearchWithPlugins, get_search_query_from_webapp +from searx.query import RawTextQuery from searx.autocomplete import searx_bang, backends as autocomplete_backends from searx.plugins import plugins from searx.preferences import Preferences, ValidationException +from searx.answerers import answerers # check if the pyopenssl, ndg-httpsclient, pyasn1 packages are installed. # They are needed for SSL connection without trouble, see #298 @@ -344,6 +345,8 @@ def render(template_name, override_theme=None, **kwargs): kwargs['cookies'] = request.cookies + kwargs['errors'] = request.errors + kwargs['instance_name'] = settings['general']['instance_name'] kwargs['results_on_new_tab'] = request.preferences.get_value('results_on_new_tab') @@ -364,15 +367,16 @@ def render(template_name, override_theme=None, **kwargs): @app.before_request def pre_request(): - # merge GET, POST vars + request.errors = [] + preferences = Preferences(themes, categories.keys(), engines, plugins) + request.preferences = preferences try: preferences.parse_cookies(request.cookies) except: - # TODO throw error message to the user - logger.warning('Invalid config') - request.preferences = preferences + request.errors.append(gettext('Invalid settings, please edit your preferences')) + # merge GET, POST vars # request.form request.form = dict(request.form.items()) for k, v in request.args.items(): @@ -397,7 +401,7 @@ def index(): Supported outputs: html, json, csv, rss. """ - if not request.args and not request.form: + if request.form.get('q') is None: return render( 'index.html', ) @@ -411,6 +415,8 @@ def index(): search = SearchWithPlugins(search_query, request) result_container = search.search() except: + request.errors.append(gettext('search error')) + logger.exception('search error') return render( 'index.html', ) @@ -427,8 +433,10 @@ def index(): for result in results: if output_format == 'html': if 'content' in result and result['content']: - result['content'] = highlight_content(result['content'][:1024], search_query.query.encode('utf-8')) - result['title'] = highlight_content(result['title'], search_query.query.encode('utf-8')) + result['content'] = highlight_content(escape(result['content'][:1024]), + search_query.query.encode('utf-8')) + result['title'] = highlight_content(escape(result['title'] or u''), + search_query.query.encode('utf-8')) else: if result.get('content'): result['content'] = html_to_text(result['content']).strip() @@ -572,7 +580,7 @@ def preferences(): try: request.preferences.parse_form(request.form) except ValidationException: - # TODO use flash feature of flask + request.errors.append(gettext('Invalid settings, please edit your preferences')) return resp return request.preferences.save(resp) @@ -609,6 +617,7 @@ def preferences(): language_codes=language_codes, engines_by_category=categories, stats=stats, + answerers=[{'info': a.self_info(), 'keywords': a.keywords} for a in answerers], disabled_engines=disabled_engines, autocomplete_backends=autocomplete_backends, shortcuts={y: x for x, y in engine_shortcuts.items()},diff --git a/tests/unit/engines/test_bing.py b/tests/unit/engines/test_bing.py@@ -14,14 +14,12 @@ class TestBingEngine(SearxTestCase): params = bing.request(query, dicto) self.assertTrue('url' in params) self.assertTrue(query in params['url']) + self.assertTrue('language%3AFR' in params['url']) self.assertTrue('bing.com' in params['url']) - self.assertTrue('SRCHHPGUSR' in params['cookies']) - self.assertTrue('fr' in params['cookies']['SRCHHPGUSR']) dicto['language'] = 'all' params = bing.request(query, dicto) - self.assertTrue('SRCHHPGUSR' in params['cookies']) - self.assertTrue('en' in params['cookies']['SRCHHPGUSR']) + self.assertTrue('language' not in params['url']) def test_response(self): self.assertRaises(AttributeError, bing.response, None)diff --git a/tests/unit/engines/test_deezer.py b/tests/unit/engines/test_deezer.py@@ -42,7 +42,7 @@ class TestDeezerEngine(SearxTestCase): self.assertEqual(len(results), 1) self.assertEqual(results[0]['title'], 'Title of track') self.assertEqual(results[0]['url'], 'https://www.deezer.com/track/1094042') - self.assertEqual(results[0]['content'], 'Artist Name • Album Title • Title of track') + self.assertEqual(results[0]['content'], 'Artist Name - Album Title - Title of track') self.assertTrue('100' in results[0]['embedded']) json = r"""diff --git a/tests/unit/engines/test_flickr.py b/tests/unit/engines/test_flickr.py@@ -52,7 +52,7 @@ class TestFlickrEngine(SearxTestCase): self.assertEqual(results[0]['url'], 'https://www.flickr.com/photos/66847915@N08/15751017054') self.assertTrue('o.jpg' in results[0]['img_src']) self.assertTrue('n.jpg' in results[0]['thumbnail_src']) - self.assertTrue('Owner' in results[0]['content']) + self.assertTrue('Owner' in results[0]['author']) self.assertTrue('Description' in results[0]['content']) json = r""" @@ -76,7 +76,7 @@ class TestFlickrEngine(SearxTestCase): self.assertEqual(results[0]['url'], 'https://www.flickr.com/photos/66847915@N08/15751017054') self.assertTrue('z.jpg' in results[0]['img_src']) self.assertTrue('z.jpg' in results[0]['thumbnail_src']) - self.assertTrue('Owner' in results[0]['content']) + self.assertTrue('Owner' in results[0]['author']) self.assertTrue('Description' in results[0]['content']) json = r""" @@ -100,7 +100,7 @@ class TestFlickrEngine(SearxTestCase): self.assertEqual(results[0]['url'], 'https://www.flickr.com/photos/66847915@N08/15751017054') self.assertTrue('o.jpg' in results[0]['img_src']) self.assertTrue('o.jpg' in results[0]['thumbnail_src']) - self.assertTrue('Owner' in results[0]['content']) + self.assertTrue('Owner' in results[0]['author']) self.assertTrue('Description' in results[0]['content']) json = r"""diff --git a/tests/unit/engines/test_flickr_noapi.py b/tests/unit/engines/test_flickr_noapi.py@@ -145,7 +145,7 @@ class TestFlickrNoapiEngine(SearxTestCase): self.assertEqual(results[0]['url'], 'https://www.flickr.com/photos/59729010@N00/14001294434') self.assertIn('k.jpg', results[0]['img_src']) self.assertIn('n.jpg', results[0]['thumbnail_src']) - self.assertIn('Owner', results[0]['content']) + self.assertIn('Owner', results[0]['author']) # no n size, only the z size json = """ @@ -188,7 +188,7 @@ class TestFlickrNoapiEngine(SearxTestCase): self.assertEqual(results[0]['url'], 'https://www.flickr.com/photos/59729010@N00/14001294434') self.assertIn('z.jpg', results[0]['img_src']) self.assertIn('z.jpg', results[0]['thumbnail_src']) - self.assertIn('Owner', results[0]['content']) + self.assertIn('Owner', results[0]['author']) # no z or n size json = """ @@ -231,7 +231,7 @@ class TestFlickrNoapiEngine(SearxTestCase): self.assertEqual(results[0]['url'], 'https://www.flickr.com/photos/59729010@N00/14001294434') self.assertIn('o.jpg', results[0]['img_src']) self.assertIn('o.jpg', results[0]['thumbnail_src']) - self.assertIn('Owner', results[0]['content']) + self.assertIn('Owner', results[0]['author']) # no image test json = """diff --git a/tests/unit/engines/test_kickass.py b/tests/unit/engines/test_kickass.py@@ -98,7 +98,7 @@ class TestKickassEngine(SearxTestCase): self.assertEqual(len(results), 1) self.assertEqual(results[0]['title'], 'This should be the title') self.assertEqual(results[0]['url'], 'https://kickass.cd/url.html') - self.assertEqual(results[0]['content'], 'Posted by riri in Other > Unsorted') + self.assertEqual(results[0]['content'], 'Posted by riri in Other > Unsorted') self.assertEqual(results[0]['seed'], 10) self.assertEqual(results[0]['leech'], 1) self.assertEqual(results[0]['filesize'], 449) @@ -381,7 +381,7 @@ class TestKickassEngine(SearxTestCase): self.assertEqual(len(results), 5) self.assertEqual(results[0]['title'], 'This should be the title') self.assertEqual(results[0]['url'], 'https://kickass.cd/url.html') - self.assertEqual(results[0]['content'], 'Posted by riri in Other > Unsorted') + self.assertEqual(results[0]['content'], 'Posted by riri in Other > Unsorted') self.assertEqual(results[0]['seed'], 10) self.assertEqual(results[0]['leech'], 1) self.assertEqual(results[0]['files'], 4)diff --git a/tests/unit/engines/test_searchcode_doc.py b/tests/unit/engines/test_searchcode_doc.py@@ -56,9 +56,6 @@ class TestSearchcodeDocEngine(SearxTestCase): self.assertEqual(len(results), 1) self.assertEqual(results[0]['title'], '[Type] Namespace test') self.assertEqual(results[0]['url'], 'http://url') - self.assertIn('Synopsis', results[0]['content']) - self.assertIn('Type', results[0]['content']) - self.assertIn('test', results[0]['content']) self.assertIn('Description', results[0]['content']) json = r"""diff --git a/tests/unit/engines/test_spotify.py b/tests/unit/engines/test_spotify.py@@ -90,7 +90,7 @@ class TestSpotifyEngine(SearxTestCase): self.assertEqual(len(results), 1) self.assertEqual(results[0]['title'], 'Title of track') self.assertEqual(results[0]['url'], 'https://open.spotify.com/track/2GzvFiedqW8hgqUpWcASZa') - self.assertEqual(results[0]['content'], 'Artist Name • Album Title • Title of track') + self.assertEqual(results[0]['content'], 'Artist Name - Album Title - Title of track') self.assertIn('1000', results[0]['embedded']) json = """diff --git a/tests/unit/test_answerers.py b/tests/unit/test_answerers.py@@ -0,0 +1,16 @@ +# -*- coding: utf-8 -*- + +from mock import Mock + +from searx.answerers import answerers +from searx.testing import SearxTestCase + + +class AnswererTest(SearxTestCase): + + def test_unicode_input(self): + query = Mock() + unicode_payload = u'árvíztűrő tükörfúrógép' + for answerer in answerers: + query.query = u'{} {}'.format(answerer.keywords[0], unicode_payload) + self.assertTrue(isinstance(answerer.answer(query), list))diff --git a/tests/unit/test_webapp.py b/tests/unit/test_webapp.py@@ -5,6 +5,7 @@ from mock import Mock from urlparse import ParseResult from searx import webapp from searx.testing import SearxTestCase +from searx.search import Search class ViewsTestCase(SearxTestCase): @@ -41,7 +42,7 @@ class ViewsTestCase(SearxTestCase): results_number=lambda: 3, results_length=lambda: len(self.test_results)) - webapp.Search.search = search_mock + Search.search = search_mock def get_current_theme_name_mock(override=None): return 'legacy'