searx

commit: 8b0cb686d5fa1fe34667e313b86cadfbb235ca79
parent: 96c8b20a045f62205a3b9a03113086f0fcfbc579
Author: Adam Tauber <asciimoo@gmail.com>
Date:   Fri, 27 Jun 2014 17:25:16 +0200

[enh] general-file.com engine added

Diffstat:
A
searx/engines/generalfile.py
35
+++++++++++++++++++++++++++++++++++

1 file changed, 35 insertions(+), 0 deletions(-)
diff --git a/searx/engines/generalfile.py b/searx/engines/generalfile.py
@@ -0,0 +1,35 @@
+from lxml import html
+
+
+base_url = 'http://www.general-file.com'
+search_url = base_url + '/files-{letter}/{query}/{pageno}'
+
+result_xpath = '//table[@class="block-file"]'
+title_xpath = './/h2/a//text()'
+url_xpath = './/h2/a/@href'
+content_xpath = './/p//text()'
+
+paging = True
+
+
+def request(query, params):
+    params['url'] = search_url.format(query=query,
+                                      letter=query[0],
+                                      pageno=params['pageno'])
+    return params
+
+
+def response(resp):
+
+    results = []
+    dom = html.fromstring(resp.text)
+    for result in dom.xpath(result_xpath):
+        url = result.xpath(url_xpath)[0]
+        # skip fast download links
+        if not url.startswith('/'):
+            continue
+        results.append({'url': base_url + url,
+                        'title': ''.join(result.xpath(title_xpath)),
+                        'content': ''.join(result.xpath(content_xpath))})
+
+    return results