commit: d748b8419ad1ef875f34783bbbcf773ebc4cfb5e
parent 24ea39d04603cb28436af77450c233c0d53d63ae
Author: Kirill Isakov <ukwt@ya.ru>
Date: Thu, 24 Mar 2016 01:57:27 +0600
Add Arch Linux Wiki search engine
Diffstat:
4 files changed, 260 insertions(+), 0 deletions(-)
diff --git a/AUTHORS.rst b/AUTHORS.rst
@@ -41,3 +41,4 @@ generally made searx better:
- @GreenLunar
- Noemi Vanyi
- Kang-min Liu
+- Kirill Isakov
diff --git a/searx/engines/archlinux.py b/searx/engines/archlinux.py
@@ -0,0 +1,144 @@
+# -*- coding: utf-8 -*-
+
+"""
+ Arch Linux Wiki
+
+ @website https://wiki.archlinux.org
+ @provide-api no (Mediawiki provides API, but Arch Wiki blocks access to it
+ @using-api no
+ @results HTML
+ @stable no (HTML can change)
+ @parse url, title, content
+"""
+
+from urlparse import urljoin
+from cgi import escape
+from urllib import urlencode
+from lxml import html
+from searx.engines.xpath import extract_text
+
+# engine dependent config
+categories = ['it']
+language_support = True
+paging = True
+base_url = 'https://wiki.archlinux.org'
+
+# xpath queries
+xpath_results = '//ul[@class="mw-search-results"]/li'
+xpath_link = './/div[@class="mw-search-result-heading"]/a'
+xpath_content = './/div[@class="searchresult"]'
+
+
+# cut 'en' from 'en_US', 'de' from 'de_CH', and so on
+def locale_to_lang_code(locale):
+ if locale.find('_') >= 0:
+ locale = locale.split('_')[0]
+ return locale
+
+# wikis for some languages were moved off from the main site, we need to make
+# requests to correct URLs to be able to get results in those languages
+lang_urls = {
+ 'all': {
+ 'base': 'https://wiki.archlinux.org',
+ 'search': '/index.php?title=Special:Search&offset={offset}&{query}'
+ },
+ 'de': {
+ 'base': 'https://wiki.archlinux.de',
+ 'search': '/index.php?title=Spezial:Suche&offset={offset}&{query}'
+ },
+ 'fr': {
+ 'base': 'https://wiki.archlinux.fr',
+ 'search': '/index.php?title=Spécial:Recherche&offset={offset}&{query}'
+ },
+ 'ja': {
+ 'base': 'https://wiki.archlinuxjp.org',
+ 'search': '/index.php?title=特別:検索&offset={offset}&{query}'
+ },
+ 'ro': {
+ 'base': 'http://wiki.archlinux.ro',
+ 'search': '/index.php?title=Special:Căutare&offset={offset}&{query}'
+ },
+ 'tr': {
+ 'base': 'http://archtr.org/wiki',
+ 'search': '/index.php?title=Özel:Ara&offset={offset}&{query}'
+ }
+}
+
+
+# get base & search URLs for selected language
+def get_lang_urls(language):
+ if language in lang_urls:
+ return lang_urls[language]
+ return lang_urls['all']
+
+# Language names to build search requests for
+# those languages which are hosted on the main site.
+main_langs = {
+ 'ar': 'العربية',
+ 'bg': 'Български',
+ 'cs': 'Česky',
+ 'da': 'Dansk',
+ 'el': 'Ελληνικά',
+ 'es': 'Español',
+ 'he': 'עברית',
+ 'hr': 'Hrvatski',
+ 'hu': 'Magyar',
+ 'it': 'Italiano',
+ 'ko': '한국어',
+ 'lt': 'Lietuviškai',
+ 'nl': 'Nederlands',
+ 'pl': 'Polski',
+ 'pt': 'Português',
+ 'ru': 'Русский',
+ 'sl': 'Slovenský',
+ 'th': 'ไทย',
+ 'uk': 'Українська',
+ 'zh': '简体中文'
+}
+
+
+# do search-request
+def request(query, params):
+ # translate the locale (e.g. 'en_US') to language code ('en')
+ language = locale_to_lang_code(params['language'])
+
+ # if our language is hosted on the main site, we need to add its name
+ # to the query in order to narrow the results to that language
+ if language in main_langs:
+ query += '(' + main_langs[language] + ')'
+
+ # prepare the request parameters
+ query = urlencode({'search': query})
+ offset = (params['pageno'] - 1) * 20
+
+ # get request URLs for our language of choice
+ urls = get_lang_urls(language)
+ search_url = urls['base'] + urls['search']
+
+ params['url'] = search_url.format(query=query, offset=offset)
+
+ return params
+
+
+# get response from search-request
+def response(resp):
+ # get the base URL for the language in which request was made
+ language = locale_to_lang_code(resp.search_params['language'])
+ base_url = get_lang_urls(language)['base']
+
+ results = []
+
+ dom = html.fromstring(resp.text)
+
+ # parse results
+ for result in dom.xpath(xpath_results):
+ link = result.xpath(xpath_link)[0]
+ href = urljoin(base_url, link.attrib.get('href'))
+ title = escape(extract_text(link))
+ content = escape(extract_text(result.xpath(xpath_content)))
+
+ results.append({'url': href,
+ 'title': title,
+ 'content': content})
+
+ return results
diff --git a/searx/settings.yml b/searx/settings.yml
@@ -34,6 +34,10 @@ outgoing: # communication with search engines
# - 1.1.1.2
engines:
+ - name : arch linux wiki
+ engine : archlinux
+ shortcut : al
+
- name : wikipedia
engine : mediawiki
shortcut : wp
diff --git a/tests/unit/engines/test_archlinux.py b/tests/unit/engines/test_archlinux.py
@@ -0,0 +1,111 @@
+from collections import defaultdict
+import mock
+from searx.engines import archlinux
+from searx.testing import SearxTestCase
+
+domains = {
+ 'all': 'https://wiki.archlinux.org',
+ 'de': 'https://wiki.archlinux.de',
+ 'fr': 'https://wiki.archlinux.fr',
+ 'ja': 'https://wiki.archlinuxjp.org',
+ 'ro': 'http://wiki.archlinux.ro',
+ 'tr': 'http://archtr.org/wiki'
+}
+
+
+class TestArchLinuxEngine(SearxTestCase):
+
+ def test_request(self):
+ query = 'test_query'
+ dic = defaultdict(dict)
+ dic['pageno'] = 0
+ dic['language'] = 'en_US'
+ params = archlinux.request(query, dic)
+ self.assertTrue('url' in params)
+ self.assertTrue(query in params['url'])
+ self.assertTrue('wiki.archlinux.org' in params['url'])
+
+ for lang, domain in domains.iteritems():
+ dic['language'] = lang
+ params = archlinux.request(query, dic)
+ self.assertTrue(domain in params['url'])
+
+ def test_response(self):
+ response = mock.Mock(text='<html></html>')
+ response.search_params = {
+ 'language': 'en_US'
+ }
+ self.assertEqual(archlinux.response(response), [])
+
+ html = """
+ <ul class="mw-search-results">
+ <li>
+ <div class="mw-search-result-heading">
+ <a href="/index.php/ATI" title="ATI">ATI</a>
+ </div>
+ <div class="searchresult">
+ Lorem ipsum dolor sit amet
+ </div>
+ <div class="mw-search-result-data">
+ 30 KB (4,630 words) - 19:04, 17 March 2016</div>
+ </li>
+ <li>
+ <div class="mw-search-result-heading">
+ <a href="/index.php/Frequently_asked_questions" title="Frequently asked questions">
+ Frequently asked questions
+ </a>
+ </div>
+ <div class="searchresult">
+ CPUs with AMDs instruction set "AMD64"
+ </div>
+ <div class="mw-search-result-data">
+ 17 KB (2,722 words) - 20:13, 21 March 2016
+ </div>
+ </li>
+ <li>
+ <div class="mw-search-result-heading">
+ <a href="/index.php/CPU_frequency_scaling" title="CPU frequency scaling">CPU frequency scaling</a>
+ </div>
+ <div class="searchresult">
+ ondemand for AMD and older Intel CPU
+ </div>
+ <div class="mw-search-result-data">
+ 15 KB (2,319 words) - 23:46, 16 March 2016
+ </div>
+ </li>
+ </ul>
+ """
+
+ expected = [
+ {
+ 'title': 'ATI',
+ 'url': 'https://wiki.archlinux.org/index.php/ATI',
+ 'content': 'Lorem ipsum dolor sit amet'
+ },
+ {
+ 'title': 'Frequently asked questions',
+ 'url': 'https://wiki.archlinux.org/index.php/Frequently_asked_questions',
+ 'content': 'CPUs with AMDs instruction set "AMD64"'
+ },
+ {
+ 'title': 'CPU frequency scaling',
+ 'url': 'https://wiki.archlinux.org/index.php/CPU_frequency_scaling',
+ 'content': 'ondemand for AMD and older Intel CPU'
+ }
+ ]
+
+ response = mock.Mock(text=html)
+ response.search_params = {
+ 'language': 'en_US'
+ }
+ results = archlinux.response(response)
+
+ self.assertEqual(type(results), list)
+ self.assertEqual(len(results), len(expected))
+
+ i = 0
+ for exp in expected:
+ res = results[i]
+ i += 1
+ for key, value in exp.iteritems():
+ self.assertEqual(res[key], value)
