searx

commit: fa9c9e090b6975987fac187f519dbf66f29eeed4
parent: a1d15c3076154b2971d1030aa679ff0c088d625d
Author: asciimoo <asciimoo@gmail.com>
Date:   Wed, 16 Oct 2013 23:03:26 +0200

[enh] result ordering and deduplication

Diffstat:
M
searx/engines/__init__.py
22
+++++++++++++++++++++-

1 file changed, 21 insertions(+), 1 deletion(-)
diff --git a/searx/engines/__init__.py b/searx/engines/__init__.py
@@ -4,6 +4,7 @@ from os import listdir
 from imp import load_source
 import grequests
 from itertools import izip_longest, chain
+from operator import itemgetter
 
 engine_dir = dirname(realpath(__file__))
 
@@ -56,4 +57,23 @@ def search(query, request, selected_engines):
                                 )
         requests.append(req)
     grequests.map(requests)
-    return list(filter(None, chain(*izip_longest(*results.values()))))
+    flat_res = list(filter(None, chain(*izip_longest(*results.values()))))
+    flat_len = len(flat_res)
+    results = []
+    # deduplication + scoring
+    for i,res in enumerate(flat_res):
+        score = flat_len - i
+        duplicated = False
+        for new_res in results:
+            if res['url'] == new_res['url']:
+                duplicated = new_res
+                break
+        if duplicated:
+            if len(res['content']) > len(duplicated):
+                duplicated['content'] = res['content']
+            duplicated['score'] += score
+        else:
+            res['score'] = score
+            results.append(res)
+
+    return sorted(results, key=itemgetter('score'), reverse=True)