commit: fa9c9e090b6975987fac187f519dbf66f29eeed4
parent: a1d15c3076154b2971d1030aa679ff0c088d625d
Author: asciimoo <asciimoo@gmail.com>
Date: Wed, 16 Oct 2013 23:03:26 +0200
[enh] result ordering and deduplication
Diffstat:
1 file changed, 21 insertions(+), 1 deletion(-)
diff --git a/searx/engines/__init__.py b/searx/engines/__init__.py
@@ -4,6 +4,7 @@ from os import listdir
from imp import load_source
import grequests
from itertools import izip_longest, chain
+from operator import itemgetter
engine_dir = dirname(realpath(__file__))
@@ -56,4 +57,23 @@ def search(query, request, selected_engines):
)
requests.append(req)
grequests.map(requests)
- return list(filter(None, chain(*izip_longest(*results.values()))))
+ flat_res = list(filter(None, chain(*izip_longest(*results.values()))))
+ flat_len = len(flat_res)
+ results = []
+ # deduplication + scoring
+ for i,res in enumerate(flat_res):
+ score = flat_len - i
+ duplicated = False
+ for new_res in results:
+ if res['url'] == new_res['url']:
+ duplicated = new_res
+ break
+ if duplicated:
+ if len(res['content']) > len(duplicated):
+ duplicated['content'] = res['content']
+ duplicated['score'] += score
+ else:
+ res['score'] = score
+ results.append(res)
+
+ return sorted(results, key=itemgetter('score'), reverse=True)