logo

searx

Unnamed repository; edit this file 'description' to name the repository.
commit: 29afa9b4a1c1d79673f5e1b8bc0dda58fe737cff
parent: cb8acbd2c3601def5fb5e07bd0f484e8a53adc1d
Author: Adam Tauber <asciimoo@gmail.com>
Date:   Thu,  4 Sep 2014 21:50:02 +0200

Merge pull request #91 from pointhi/engines

fix yacy-engine and add comments

Diffstat:

searx/engines/yacy.py | 85++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++-----------------
searx/settings.yml | 7+++++++
2 files changed, 74 insertions(+), 18 deletions(-)

diff --git a/searx/engines/yacy.py b/searx/engines/yacy.py @@ -1,40 +1,89 @@ +## Yacy (Web, Images, Videos, Music, Files) +# +# @website http://yacy.net +# @provide-api yes (http://www.yacy-websuche.de/wiki/index.php/Dev:APIyacysearch) +# +# @using-api yes +# @results JSON +# @stable yes +# @parse (general) url, title, content, publishedDate +# @parse (images) url, title, img_src +# +# @todo parse video, audio and file results + from json import loads from urllib import urlencode +from dateutil import parser + +# engine dependent config +categories = ['general', 'images'] #TODO , 'music', 'videos', 'files' +paging = True +language_support = True +number_of_results = 5 + +# search-url +base_url = 'http://localhost:8090' +search_url = '/yacysearch.json?{query}&startRecord={offset}&maximumRecords={limit}&contentdom={search_type}&resource=global' -url = 'http://localhost:8090' -search_url = '/yacysearch.json?{query}&maximumRecords=10' +# yacy specific type-definitions +search_types = {'general': 'text', + 'images': 'image', + 'files': 'app', + 'music': 'audio', + 'videos': 'video'} +# do search-request def request(query, params): - params['url'] = url + search_url.format(query=urlencode({'query': query})) + offset = (params['pageno'] - 1) * number_of_results + search_type = search_types.get(params['category'], '0') + + params['url'] = base_url + search_url.format(query=urlencode({'query': query}), + offset=offset, + limit=number_of_results, + search_type=search_type) + + # add language tag if specified + if params['language'] != 'all': + params['url'] += '&lr=lang_' + params['language'].split('_')[0] + return params +# get response from search-request def response(resp): + results = [] + raw_search_results = loads(resp.text) + # return empty array if there are no results if not raw_search_results: return [] search_results = raw_search_results.get('channels', {})[0].get('items', []) - results = [] - - for result in search_results: - tmp_result = {} - tmp_result['title'] = result['title'] - tmp_result['url'] = result['link'] - tmp_result['content'] = '' - - if result['description']: - tmp_result['content'] += result['description'] + "<br/>" + if resp.search_params['category'] == 'general': + # parse general results + for result in search_results: + publishedDate = parser.parse(result['pubDate']) - if result['pubDate']: - tmp_result['content'] += result['pubDate'] + "<br/>" + # append result + results.append({'url': result['link'], + 'title': result['title'], + 'content': result['description'], + 'publishedDate': publishedDate}) - if result['size'] != '-1': - tmp_result['content'] += result['sizename'] + elif resp.search_params['category'] == 'images': + # parse image results + for result in search_results: + # append result + results.append({'url': result['url'], + 'title': result['title'], + 'content': '', + 'img_src': result['image'], + 'template': 'images.html'}) - results.append(tmp_result) + #TODO parse video, audio and file results + # return results return results diff --git a/searx/settings.yml b/searx/settings.yml @@ -143,6 +143,13 @@ engines: locale : en-US shortcut : vm +# - name : yacy +# engine : yacy +# shortcut : ya +# base_url : 'http://localhost:8090' +# number_of_results : 5 +# timeout: 3.0 + locales: en : English de : Deutsch