logo

searx

Unnamed repository; edit this file 'description' to name the repository.
commit: a07b2b514c38cff031e0e36b99878a6041873842
parent: cebf5868b33659d9b66f949c9c78f4101bd3b925
Author: Adam Tauber <asciimoo@gmail.com>
Date:   Sat,  5 Jul 2014 17:33:19 +0200

[fix] url path unquoted check to avoid duplications

Diffstat:

searx/engines/__init__.py | 10++++++++--
1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/searx/engines/__init__.py b/searx/engines/__init__.py @@ -21,7 +21,7 @@ import sys from imp import load_source from itertools import izip_longest, chain from operator import itemgetter -from urlparse import urlparse +from urlparse import urlparse, unquote from datetime import datetime import grequests from flask.ext.babel import gettext @@ -153,7 +153,9 @@ def score_results(results): results = [] # deduplication + scoring for i, res in enumerate(flat_res): + res['parsed_url'] = urlparse(res['url']) + res['host'] = res['parsed_url'].netloc if res['host'].startswith('www.'): @@ -172,7 +174,7 @@ def score_results(results): p1 = res['parsed_url'].path[:-1] if res['parsed_url'].path.endswith('/') else res['parsed_url'].path # noqa p2 = new_res['parsed_url'].path[:-1] if new_res['parsed_url'].path.endswith('/') else new_res['parsed_url'].path # noqa if res['host'] == new_res['host'] and\ - p1 == p2 and\ + unquote(p1) == unquote(p2) and\ res['parsed_url'].query == new_res['parsed_url'].query and\ res.get('template') == new_res.get('template'): duplicated = new_res @@ -222,6 +224,10 @@ def search(query, request, selected_engines, pageno=1, lang='all'): request_params['language'] = lang request_params = engine.request(query.encode('utf-8'), request_params) + if request_params['url'] is None: + # TODO add support of offline engines + pass + callback = make_callback( selected_engine['name'], results,