searx

commit: c38a743c6f22433b64323ac4a2e74a3293b55fee
parent: ed925cf561b9bbbb9ba75fc2458e1393a70bf086
Author: Adam Tauber <asciimoo@gmail.com>
Date:   Thu,  4 Sep 2014 19:31:47 +0200

Merge pull request #90 from pointhi/engines

using generic mediawiki engines
Diffstat:
M
searx/engines/mediawiki.py
63
++++++++++++++++++++++++++++++++++++++++++++++++++++++---------
D
searx/engines/wikipedia.py
67
-------------------------------------------------------------------
M
searx/settings.yml
9
+++++----

3 files changed, 59 insertions(+), 80 deletions(-)
diff --git a/searx/engines/mediawiki.py b/searx/engines/mediawiki.py
@@ -1,22 +1,67 @@
+## general mediawiki-engine (Web)
+# 
+# @website     websites built on mediawiki (https://www.mediawiki.org)
+# @provide-api yes (http://www.mediawiki.org/wiki/API:Search)
+# 
+# @using-api   yes
+# @results     JSON
+# @stable      yes
+# @parse       url, title 
+#
+# @todo        content
+
 from json import loads
 from urllib import urlencode, quote
 
-url = 'https://en.wikipedia.org/'
+# engine dependent config
+categories = ['general']
+language_support = True
+paging = True
+number_of_results = 1
+    
+# search-url
+base_url = 'https://{language}.wikipedia.org/'
+search_url = base_url + 'w/api.php?action=query&list=search&{query}&srprop=timestamp&format=json&sroffset={offset}&srlimit={limit}'  # noqa
 
-search_url = url + 'w/api.php?action=query&list=search&{query}&srprop=timestamp&format=json&sroffset={offset}'  # noqa
 
-number_of_results = 10
+# do search-request
+def request(query, params):
+    offset = (params['pageno'] - 1) * number_of_results
 
+    if params['language'] == 'all':
+        language = 'en'
+    else:
+        language = params['language'].split('_')[0]
+    
+    # write search-language back to params, required in response
+    params['language'] = language
 
-def request(query, params):
-    offset = (params['pageno'] - 1) * 10
     params['url'] = search_url.format(query=urlencode({'srsearch': query}),
-                                      offset=offset)
+                                      offset=offset,
+                                      limit=number_of_results,
+                                      language=language)
+
     return params
 
 
+# get response from search-request
 def response(resp):
+    results = []
+
     search_results = loads(resp.text)
-    res = search_results.get('query', {}).get('search', [])
-    return [{'url': url + 'wiki/' + quote(result['title'].replace(' ', '_').encode('utf-8')),  # noqa
-        'title': result['title']} for result in res[:int(number_of_results)]]
+
+    # return empty array if there are no results
+    if not search_results.get('query', {}).get('search'):
+        return []
+
+    # parse results
+    for result in search_results['query']['search']:
+        url = base_url.format(language=resp.search_params['language']) + 'wiki/' + quote(result['title'].replace(' ', '_').encode('utf-8'))
+        
+        # append result
+        results.append({'url': url,
+                        'title': result['title'],
+                        'content': ''})
+
+    # return results
+    return results
diff --git a/searx/engines/wikipedia.py b/searx/engines/wikipedia.py
@@ -1,67 +0,0 @@
-## Wikipedia (Web)
-# 
-# @website     http://www.wikipedia.org
-# @provide-api yes (http://www.mediawiki.org/wiki/API:Search)
-# 
-# @using-api   yes
-# @results     JSON
-# @stable      yes
-# @parse       url, title 
-#
-# @todo        content
-
-from json import loads
-from urllib import urlencode, quote
-
-# engine dependent config
-categories = ['general']
-language_support = True
-paging = True
-number_of_results = 1
-    
-# search-url
-url = 'https://{language}.wikipedia.org/'
-search_url = url + 'w/api.php?action=query&list=search&{query}&srprop=timestamp&format=json&sroffset={offset}&srlimit={limit}'  # noqa
-
-
-# do search-request
-def request(query, params):
-    offset = (params['pageno'] - 1) * number_of_results
-
-    if params['language'] == 'all':
-        language = 'en'
-    else:
-        language = params['language'].split('_')[0]
-    
-    # write search-language back to params, required in response
-    params['language'] = language
-
-    params['url'] = search_url.format(query=urlencode({'srsearch': query}),
-                                      offset=offset,
-                                      limit=number_of_results,
-                                      language=language)
-
-    return params
-
-
-# get response from search-request
-def response(resp):
-    results = []
-
-    search_results = loads(resp.text)
-
-    # return empty array if there are no results
-    if not search_results.get('query', {}).get('search'):
-        return []
-
-    # parse results
-    for result in search_results['query']['search']:
-        res_url = url.format(language=resp.search_params['language']) + 'wiki/' + quote(result['title'].replace(' ', '_').encode('utf-8'))
-        
-        # append result
-        results.append({'url': res_url,
-                        'title': result['title'],
-                        'content': ''})
-
-    # return results
-    return results
diff --git a/searx/settings.yml b/searx/settings.yml
@@ -10,9 +10,10 @@ server:
 
 engines:
   - name : wikipedia
-    engine : wikipedia
+    engine : mediawiki
     shortcut : wp
-#    number_of_results : 1 # default is 1
+    base_url : 'https://{language}.wikipedia.org/'
+    number_of_results : 1
 
   - name : bing
     engine : bing
@@ -108,9 +109,9 @@ engines:
 # maybe in a fun category
 #  - name : uncyclopedia
 #    engine : mediawiki
-#    categories : general
 #    shortcut : unc
-#    url : https://uncyclopedia.wikia.com/
+#    base_url : https://uncyclopedia.wikia.com/
+#    number_of_results : 5
 
 # tmp suspended - too slow, too many errors
 #  - name : urbandictionary