logo

searx

My custom branche(s) on searx, a meta-search engine
commit: b0ba367a1a575fbbd3d847bdbe39b6920d2f46e4
parent: a07b2b514c38cff031e0e36b99878a6041873842
Author: Adam Tauber <asciimoo@gmail.com>
Date:   Mon,  7 Jul 2014 13:59:27 +0200

[enh][mod] search refactor

Diffstat:

Msearx/engines/__init__.py163+------------------------------------------------------------------------------
Msearx/search.py163+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Msearx/tests/test_webapp.py8++++----
Msearx/webapp.py10++--------
4 files changed, 170 insertions(+), 174 deletions(-)

diff --git a/searx/engines/__init__.py b/searx/engines/__init__.py @@ -19,19 +19,12 @@ along with searx. If not, see < http://www.gnu.org/licenses/ >. from os.path import realpath, dirname, splitext, join import sys from imp import load_source -from itertools import izip_longest, chain -from operator import itemgetter -from urlparse import urlparse, unquote -from datetime import datetime -import grequests from flask.ext.babel import gettext +from operator import itemgetter from searx import settings -from searx.utils import gen_useragent engine_dir = dirname(realpath(__file__)) -number_of_searches = 0 - engines = {} categories = {'general': []} @@ -114,160 +107,6 @@ for engine_data in settings['engines']: engine_shortcuts[engine.shortcut] = engine.name -def default_request_params(): - return { - 'method': 'GET', 'headers': {}, 'data': {}, 'url': '', 'cookies': {}} - - -def make_callback(engine_name, results, suggestions, callback, params): - # creating a callback wrapper for the search engine results - def process_callback(response, **kwargs): - cb_res = [] - response.search_params = params - engines[engine_name].stats['page_load_time'] += \ - (datetime.now() - params['started']).total_seconds() - try: - search_results = callback(response) - except Exception, e: - engines[engine_name].stats['errors'] += 1 - results[engine_name] = cb_res - print '[E] Error with engine "{0}":\n\t{1}'.format( - engine_name, str(e)) - return - for result in search_results: - result['engine'] = engine_name - if 'suggestion' in result: - # TODO type checks - suggestions.add(result['suggestion']) - continue - cb_res.append(result) - results[engine_name] = cb_res - return process_callback - - -def score_results(results): - flat_res = filter( - None, chain.from_iterable(izip_longest(*results.values()))) - flat_len = len(flat_res) - engines_len = len(results) - results = [] - # deduplication + scoring - for i, res in enumerate(flat_res): - - res['parsed_url'] = urlparse(res['url']) - - res['host'] = res['parsed_url'].netloc - - if res['host'].startswith('www.'): - res['host'] = res['host'].replace('www.', '', 1) - - res['engines'] = [res['engine']] - weight = 1.0 - - if hasattr(engines[res['engine']], 'weight'): - weight = float(engines[res['engine']].weight) - - score = int((flat_len - i) / engines_len) * weight + 1 - duplicated = False - - for new_res in results: - p1 = res['parsed_url'].path[:-1] if res['parsed_url'].path.endswith('/') else res['parsed_url'].path # noqa - p2 = new_res['parsed_url'].path[:-1] if new_res['parsed_url'].path.endswith('/') else new_res['parsed_url'].path # noqa - if res['host'] == new_res['host'] and\ - unquote(p1) == unquote(p2) and\ - res['parsed_url'].query == new_res['parsed_url'].query and\ - res.get('template') == new_res.get('template'): - duplicated = new_res - break - if duplicated: - if res.get('content') > duplicated.get('content'): - duplicated['content'] = res['content'] - duplicated['score'] += score - duplicated['engines'].append(res['engine']) - if duplicated['parsed_url'].scheme == 'https': - continue - elif res['parsed_url'].scheme == 'https': - duplicated['url'] = res['parsed_url'].geturl() - duplicated['parsed_url'] = res['parsed_url'] - else: - res['score'] = score - results.append(res) - return sorted(results, key=itemgetter('score'), reverse=True) - - -def search(query, request, selected_engines, pageno=1, lang='all'): - global engines, categories, number_of_searches - requests = [] - results = {} - suggestions = set() - number_of_searches += 1 - #user_agent = request.headers.get('User-Agent', '') - user_agent = gen_useragent() - - for selected_engine in selected_engines: - if selected_engine['name'] not in engines: - continue - - engine = engines[selected_engine['name']] - - if pageno > 1 and not engine.paging: - continue - - if lang != 'all' and not engine.language_support: - continue - - request_params = default_request_params() - request_params['headers']['User-Agent'] = user_agent - request_params['category'] = selected_engine['category'] - request_params['started'] = datetime.now() - request_params['pageno'] = pageno - request_params['language'] = lang - request_params = engine.request(query.encode('utf-8'), request_params) - - if request_params['url'] is None: - # TODO add support of offline engines - pass - - callback = make_callback( - selected_engine['name'], - results, - suggestions, - engine.response, - request_params - ) - - request_args = dict( - headers=request_params['headers'], - hooks=dict(response=callback), - cookies=request_params['cookies'], - timeout=engine.timeout - ) - - if request_params['method'] == 'GET': - req = grequests.get - else: - req = grequests.post - request_args['data'] = request_params['data'] - - # ignoring empty urls - if not request_params['url']: - continue - - requests.append(req(request_params['url'], **request_args)) - grequests.map(requests) - for engine_name, engine_results in results.items(): - engines[engine_name].stats['search_count'] += 1 - engines[engine_name].stats['result_count'] += len(engine_results) - - results = score_results(results) - - for result in results: - for res_engine in result['engines']: - engines[result['engine']].stats['score_count'] += result['score'] - - return results, suggestions - - def get_engines_stats(): # TODO refactor pageloads = [] diff --git a/searx/search.py b/searx/search.py @@ -1,7 +1,96 @@ +import grequests +from itertools import izip_longest, chain +from datetime import datetime +from operator import itemgetter +from urlparse import urlparse, unquote from searx.engines import ( categories, engines, engine_shortcuts ) from searx.languages import language_codes +from searx.utils import gen_useragent + +number_of_searches = 0 + + +def default_request_params(): + return { + 'method': 'GET', 'headers': {}, 'data': {}, 'url': '', 'cookies': {}} + + +def make_callback(engine_name, results, suggestions, callback, params): + # creating a callback wrapper for the search engine results + def process_callback(response, **kwargs): + cb_res = [] + response.search_params = params + engines[engine_name].stats['page_load_time'] += \ + (datetime.now() - params['started']).total_seconds() + try: + search_results = callback(response) + except Exception, e: + engines[engine_name].stats['errors'] += 1 + results[engine_name] = cb_res + print '[E] Error with engine "{0}":\n\t{1}'.format( + engine_name, str(e)) + return + for result in search_results: + result['engine'] = engine_name + if 'suggestion' in result: + # TODO type checks + suggestions.add(result['suggestion']) + continue + cb_res.append(result) + results[engine_name] = cb_res + return process_callback + + +def score_results(results): + flat_res = filter( + None, chain.from_iterable(izip_longest(*results.values()))) + flat_len = len(flat_res) + engines_len = len(results) + results = [] + # deduplication + scoring + for i, res in enumerate(flat_res): + + res['parsed_url'] = urlparse(res['url']) + + res['host'] = res['parsed_url'].netloc + + if res['host'].startswith('www.'): + res['host'] = res['host'].replace('www.', '', 1) + + res['engines'] = [res['engine']] + weight = 1.0 + + if hasattr(engines[res['engine']], 'weight'): + weight = float(engines[res['engine']].weight) + + score = int((flat_len - i) / engines_len) * weight + 1 + duplicated = False + + for new_res in results: + p1 = res['parsed_url'].path[:-1] if res['parsed_url'].path.endswith('/') else res['parsed_url'].path # noqa + p2 = new_res['parsed_url'].path[:-1] if new_res['parsed_url'].path.endswith('/') else new_res['parsed_url'].path # noqa + if res['host'] == new_res['host'] and\ + unquote(p1) == unquote(p2) and\ + res['parsed_url'].query == new_res['parsed_url'].query and\ + res.get('template') == new_res.get('template'): + duplicated = new_res + break + if duplicated: + if res.get('content') > duplicated.get('content'): + duplicated['content'] = res['content'] + duplicated['score'] += score + duplicated['engines'].append(res['engine']) + if duplicated['parsed_url'].scheme == 'https': + continue + elif res['parsed_url'].scheme == 'https': + duplicated['url'] = res['parsed_url'].geturl() + duplicated['parsed_url'] = res['parsed_url'] + else: + res['score'] = score + results.append(res) + return sorted(results, key=itemgetter('score'), reverse=True) class Search(object): @@ -112,3 +201,77 @@ class Search(object): if modified: self.query = self.query.replace(query_parts[0], '', 1).strip() self.parse_query() + + def search(self, request): + global number_of_searches + requests = [] + results = {} + suggestions = set() + number_of_searches += 1 + #user_agent = request.headers.get('User-Agent', '') + user_agent = gen_useragent() + + for selected_engine in self.engines: + if selected_engine['name'] not in engines: + continue + + engine = engines[selected_engine['name']] + + if self.pageno > 1 and not engine.paging: + continue + + if self.lang != 'all' and not engine.language_support: + continue + + request_params = default_request_params() + request_params['headers']['User-Agent'] = user_agent + request_params['category'] = selected_engine['category'] + request_params['started'] = datetime.now() + request_params['pageno'] = self.pageno + request_params['language'] = self.lang + request_params = engine.request(self.query.encode('utf-8'), + request_params) + + if request_params['url'] is None: + # TODO add support of offline engines + pass + + callback = make_callback( + selected_engine['name'], + results, + suggestions, + engine.response, + request_params + ) + + request_args = dict( + headers=request_params['headers'], + hooks=dict(response=callback), + cookies=request_params['cookies'], + timeout=engine.timeout + ) + + if request_params['method'] == 'GET': + req = grequests.get + else: + req = grequests.post + request_args['data'] = request_params['data'] + + # ignoring empty urls + if not request_params['url']: + continue + + requests.append(req(request_params['url'], **request_args)) + grequests.map(requests) + for engine_name, engine_results in results.items(): + engines[engine_name].stats['search_count'] += 1 + engines[engine_name].stats['result_count'] += len(engine_results) + + results = score_results(results) + + for result in results: + for res_engine in result['engines']: + engines[result['engine']]\ + .stats['score_count'] += result['score'] + + return results, suggestions diff --git a/searx/tests/test_webapp.py b/searx/tests/test_webapp.py @@ -39,7 +39,7 @@ class ViewsTestCase(SearxTestCase): self.assertEqual(result.status_code, 200) self.assertIn('<div class="title"><h1>searx</h1></div>', result.data) - @patch('searx.webapp.do_search') + @patch('searx.search.Search.search') def test_index_html(self, search): search.return_value = ( self.test_results, @@ -55,7 +55,7 @@ class ViewsTestCase(SearxTestCase): result.data ) - @patch('searx.webapp.do_search') + @patch('searx.search.Search.search') def test_index_json(self, search): search.return_value = ( self.test_results, @@ -71,7 +71,7 @@ class ViewsTestCase(SearxTestCase): self.assertEqual( result_dict['results'][0]['url'], 'http://first.test.xyz') - @patch('searx.webapp.do_search') + @patch('searx.search.Search.search') def test_index_csv(self, search): search.return_value = ( self.test_results, @@ -86,7 +86,7 @@ class ViewsTestCase(SearxTestCase): result.data ) - @patch('searx.webapp.do_search') + @patch('searx.search.Search.search') def test_index_rss(self, search): search.return_value = ( self.test_results, diff --git a/searx/webapp.py b/searx/webapp.py @@ -39,8 +39,7 @@ from flask import ( from flask.ext.babel import Babel, gettext, format_date from searx import settings, searx_dir from searx.engines import ( - search as do_search, categories, engines, get_engines_stats, - engine_shortcuts + categories, engines, get_engines_stats, engine_shortcuts ) from searx.utils import ( UnicodeWriter, highlight_content, html_to_text, get_themes @@ -191,12 +190,7 @@ def index(): 'index.html', ) - # TODO moar refactor - do_search integration into Search class - search.results, search.suggestions = do_search(search.query, - request, - search.engines, - search.pageno, - search.lang) + search.results, search.suggestions = search.search(request) for result in search.results: