commit: 29afa9b4a1c1d79673f5e1b8bc0dda58fe737cff
parent: cb8acbd2c3601def5fb5e07bd0f484e8a53adc1d
Author: Adam Tauber <asciimoo@gmail.com>
Date: Thu, 4 Sep 2014 21:50:02 +0200
Merge pull request #91 from pointhi/engines
fix yacy-engine and add comments
Diffstat:
2 files changed, 74 insertions(+), 18 deletions(-)
diff --git a/searx/engines/yacy.py b/searx/engines/yacy.py
@@ -1,40 +1,89 @@
+## Yacy (Web, Images, Videos, Music, Files)
+#
+# @website http://yacy.net
+# @provide-api yes (http://www.yacy-websuche.de/wiki/index.php/Dev:APIyacysearch)
+#
+# @using-api yes
+# @results JSON
+# @stable yes
+# @parse (general) url, title, content, publishedDate
+# @parse (images) url, title, img_src
+#
+# @todo parse video, audio and file results
+
from json import loads
from urllib import urlencode
+from dateutil import parser
+
+# engine dependent config
+categories = ['general', 'images'] #TODO , 'music', 'videos', 'files'
+paging = True
+language_support = True
+number_of_results = 5
+
+# search-url
+base_url = 'http://localhost:8090'
+search_url = '/yacysearch.json?{query}&startRecord={offset}&maximumRecords={limit}&contentdom={search_type}&resource=global'
-url = 'http://localhost:8090'
-search_url = '/yacysearch.json?{query}&maximumRecords=10'
+# yacy specific type-definitions
+search_types = {'general': 'text',
+ 'images': 'image',
+ 'files': 'app',
+ 'music': 'audio',
+ 'videos': 'video'}
+# do search-request
def request(query, params):
- params['url'] = url + search_url.format(query=urlencode({'query': query}))
+ offset = (params['pageno'] - 1) * number_of_results
+ search_type = search_types.get(params['category'], '0')
+
+ params['url'] = base_url + search_url.format(query=urlencode({'query': query}),
+ offset=offset,
+ limit=number_of_results,
+ search_type=search_type)
+
+ # add language tag if specified
+ if params['language'] != 'all':
+ params['url'] += '&lr=lang_' + params['language'].split('_')[0]
+
return params
+# get response from search-request
def response(resp):
+ results = []
+
raw_search_results = loads(resp.text)
+ # return empty array if there are no results
if not raw_search_results:
return []
search_results = raw_search_results.get('channels', {})[0].get('items', [])
- results = []
-
- for result in search_results:
- tmp_result = {}
- tmp_result['title'] = result['title']
- tmp_result['url'] = result['link']
- tmp_result['content'] = ''
-
- if result['description']:
- tmp_result['content'] += result['description'] + "<br/>"
+ if resp.search_params['category'] == 'general':
+ # parse general results
+ for result in search_results:
+ publishedDate = parser.parse(result['pubDate'])
- if result['pubDate']:
- tmp_result['content'] += result['pubDate'] + "<br/>"
+ # append result
+ results.append({'url': result['link'],
+ 'title': result['title'],
+ 'content': result['description'],
+ 'publishedDate': publishedDate})
- if result['size'] != '-1':
- tmp_result['content'] += result['sizename']
+ elif resp.search_params['category'] == 'images':
+ # parse image results
+ for result in search_results:
+ # append result
+ results.append({'url': result['url'],
+ 'title': result['title'],
+ 'content': '',
+ 'img_src': result['image'],
+ 'template': 'images.html'})
- results.append(tmp_result)
+ #TODO parse video, audio and file results
+ # return results
return results
diff --git a/searx/settings.yml b/searx/settings.yml
@@ -143,6 +143,13 @@ engines:
locale : en-US
shortcut : vm
+# - name : yacy
+# engine : yacy
+# shortcut : ya
+# base_url : 'http://localhost:8090'
+# number_of_results : 5
+# timeout: 3.0
+
locales:
en : English
de : Deutsch
