searx

commit: fafc564874c63e2c8fb5082c944779c1572e2f15
parent: 357fc47811a1de5a273622881574dbde23c7cb59
Author: Adam Tauber <asciimoo@gmail.com>
Date:   Sat, 31 Oct 2015 15:27:23 +0100

[enh] yandex engine added

Diffstat:
A
searx/engines/yandex.py
55
+++++++++++++++++++++++++++++++++++++++++++++++++++++++
M
searx/settings.yml
5
+++++

2 files changed, 60 insertions(+), 0 deletions(-)
diff --git a/searx/engines/yandex.py b/searx/engines/yandex.py
@@ -0,0 +1,55 @@
+"""
+ Yahoo (Web)
+
+ @website     https://yandex.ru/
+ @provide-api ?
+ @using-api   no
+ @results     HTML (using search portal)
+ @stable      no (HTML can change)
+ @parse       url, title, content
+"""
+
+from urllib import urlencode
+from lxml import html
+from searx.search import logger
+
+logger = logger.getChild('yandex engine')
+
+# engine dependent config
+categories = ['general']
+paging = True
+language_support = True  # TODO
+
+# search-url
+base_url = 'https://yandex.ru/'
+search_url = 'search/?{query}&p={page}'
+
+results_xpath = '//div[@class="serp-item serp-item_plain_yes clearfix i-bem"]'
+url_xpath = './/h2/a/@href'
+title_xpath = './/h2/a//text()'
+content_xpath = './/div[@class="serp-item__text"]//text()'
+
+
+def request(query, params):
+    params['url'] = base_url + search_url.format(page=params['pageno']-1,
+                                                 query=urlencode({'text': query}))
+    return params
+
+
+# get response from search-request
+def response(resp):
+    dom = html.fromstring(resp.text)
+    results = []
+
+    for result in dom.xpath(results_xpath):
+        try:
+            res = {'url': result.xpath(url_xpath)[0],
+                   'title': ''.join(result.xpath(title_xpath)),
+                   'content': ''.join(result.xpath(content_xpath))}
+        except:
+            logger.exception('yandex parse crash')
+            continue
+
+        results.append(res)
+
+    return results
diff --git a/searx/settings.yml b/searx/settings.yml
@@ -274,6 +274,11 @@ engines:
     engine : yahoo
     shortcut : yh
 
+  - name : yandex
+    engine : yandex
+    shortcut : ya
+    disabled : True
+
   - name : yahoo news
     engine : yahoo_news
     shortcut : yhn