logo

searx

Unnamed repository; edit this file 'description' to name the repository.
commit: 8b0cb686d5fa1fe34667e313b86cadfbb235ca79
parent: 96c8b20a045f62205a3b9a03113086f0fcfbc579
Author: Adam Tauber <asciimoo@gmail.com>
Date:   Fri, 27 Jun 2014 17:25:16 +0200

[enh] general-file.com engine added

Diffstat:

searx/engines/generalfile.py | 35+++++++++++++++++++++++++++++++++++
1 file changed, 35 insertions(+), 0 deletions(-)

diff --git a/searx/engines/generalfile.py b/searx/engines/generalfile.py @@ -0,0 +1,35 @@ +from lxml import html + + +base_url = 'http://www.general-file.com' +search_url = base_url + '/files-{letter}/{query}/{pageno}' + +result_xpath = '//table[@class="block-file"]' +title_xpath = './/h2/a//text()' +url_xpath = './/h2/a/@href' +content_xpath = './/p//text()' + +paging = True + + +def request(query, params): + params['url'] = search_url.format(query=query, + letter=query[0], + pageno=params['pageno']) + return params + + +def response(resp): + + results = [] + dom = html.fromstring(resp.text) + for result in dom.xpath(result_xpath): + url = result.xpath(url_xpath)[0] + # skip fast download links + if not url.startswith('/'): + continue + results.append({'url': base_url + url, + 'title': ''.join(result.xpath(title_xpath)), + 'content': ''.join(result.xpath(content_xpath))}) + + return results