commit: 80df181575d3dac663434c8ed290dc3e4eb3dcae
parent: ddd5c2632ff92947d80e530e93ba65c62537a298
Author: Adam Tauber <asciimoo@gmail.com>
Date: Thu, 23 Mar 2017 21:25:52 +0100
Merge pull request #881 from mmuman/framalibre
Add framalibre engine
Diffstat:
3 files changed, 180 insertions(+), 0 deletions(-)
diff --git a/searx/engines/framalibre.py b/searx/engines/framalibre.py
@@ -0,0 +1,72 @@
+"""
+ FramaLibre (It)
+
+ @website https://framalibre.org/
+ @provide-api no
+
+ @using-api no
+ @results HTML
+ @stable no (HTML can change)
+ @parse url, title, content, thumbnail, img_src
+"""
+
+from urlparse import urljoin
+from cgi import escape
+from urllib import urlencode
+from lxml import html
+from searx.engines.xpath import extract_text
+from dateutil import parser
+
+# engine dependent config
+categories = ['it']
+paging = True
+
+# search-url
+base_url = 'https://framalibre.org/'
+search_url = base_url + 'recherche-par-crit-res?{query}&page={offset}'
+
+# specific xpath variables
+results_xpath = '//div[@class="nodes-list-row"]/div[contains(@typeof,"sioc:Item")]'
+link_xpath = './/h3[@class="node-title"]/a[@href]'
+thumbnail_xpath = './/img[@class="media-object img-responsive"]/@src'
+content_xpath = './/div[@class="content"]//p'
+
+
+# do search-request
+def request(query, params):
+ offset = (params['pageno'] - 1)
+ params['url'] = search_url.format(query=urlencode({'keys': query}),
+ offset=offset)
+
+ return params
+
+
+# get response from search-request
+def response(resp):
+ results = []
+
+ dom = html.fromstring(resp.text)
+
+ # parse results
+ for result in dom.xpath(results_xpath):
+ link = result.xpath(link_xpath)[0]
+ href = urljoin(base_url, link.attrib.get('href'))
+ # there's also a span (class="rdf-meta element-hidden" property="dc:title")'s content property for this...
+ title = escape(extract_text(link))
+ thumbnail_tags = result.xpath(thumbnail_xpath)
+ thumbnail = None
+ if len(thumbnail_tags) > 0:
+ thumbnail = extract_text(thumbnail_tags[0])
+ if thumbnail[0] == '/':
+ thumbnail = base_url + thumbnail
+ content = escape(extract_text(result.xpath(content_xpath)))
+
+ # append result
+ results.append({'url': href,
+ 'title': title,
+ 'thumbnail': thumbnail,
+ 'img_src': thumbnail,
+ 'content': content})
+
+ # return results
+ return results
diff --git a/searx/settings.yml b/searx/settings.yml
@@ -465,6 +465,11 @@ engines:
shortcut : scc
disabled : True
+ - name : framalibre
+ engine : framalibre
+ shortcut : frl
+ disabled : True
+
# - name : searx
# engine : searx_engine
# shortcut : se
diff --git a/tests/unit/engines/test_framalibre.py b/tests/unit/engines/test_framalibre.py
@@ -0,0 +1,103 @@
+# -*- coding: utf-8 -*-
+from collections import defaultdict
+import mock
+from searx.engines import framalibre
+from searx.testing import SearxTestCase
+
+
+class TestFramalibreEngine(SearxTestCase):
+
+ def test_request(self):
+ query = 'test_query'
+ dicto = defaultdict(dict)
+ dicto['pageno'] = 0
+ params = framalibre.request(query, dicto)
+ self.assertTrue('url' in params)
+ self.assertTrue(query in params['url'])
+ self.assertTrue('framalibre.org' in params['url'])
+
+ def test_response(self):
+ self.assertRaises(AttributeError, framalibre.response, None)
+ self.assertRaises(AttributeError, framalibre.response, [])
+ self.assertRaises(AttributeError, framalibre.response, '')
+ self.assertRaises(AttributeError, framalibre.response, '[]')
+
+ response = mock.Mock(text='{}')
+ self.assertEqual(framalibre.response(response), [])
+
+ response = mock.Mock(text='{"data": []}')
+ self.assertEqual(framalibre.response(response), [])
+
+ html = u"""
+ <div class="nodes-list-row">
+ <div id="node-431"
+ class="node node-logiciel-annuaires node-promoted node-teaser node-teaser node-sheet clearfix nodes-list"
+ about="/content/gogs" typeof="sioc:Item foaf:Document">
+ <header class="media">
+ <div class="media-left">
+ <div class="field field-name-field-logo field-type-image field-label-hidden">
+ <div class="field-items">
+ <div class="field-item even">
+ <a href="/content/gogs">
+ <img class="media-object img-responsive" typeof="foaf:Image"
+ src="https://framalibre.org/sites/default/files/styles/teaser_logo/public/leslogos/gogs-lg.png?itok=rrCxKKBy"
+ width="70" height="70" alt="" />
+ </a>
+ </div>
+ </div>
+ </div>
+ </div>
+ <div class="media-body">
+ <h3 class="node-title"><a href="/content/gogs">Gogs</a></h3>
+ <span property="dc:title" content="Gogs" class="rdf-meta element-hidden"></span>
+ <div class="field field-name-field-annuaires field-type-taxonomy-term-reference field-label-hidden">
+ <div class="field-items">
+ <div class="field-item even">
+ <a href="/annuaires/cloudwebapps"
+ typeof="skos:Concept" property="rdfs:label skos:prefLabel"
+ datatype="" class="label label-primary">Cloud/webApps</a>
+ </div>
+ </div>
+ </div>
+ </div>
+ </header>
+ <div class="content">
+ <div class="field field-name-field-votre-appr-ciation field-type-fivestar field-label-hidden">
+ <div class="field-items">
+ <div class="field-item even">
+ </div>
+ </div>
+ </div>
+ <div class="field field-name-body field-type-text-with-summary field-label-hidden">
+ <div class="field-items">
+ <div class="field-item even" property="content:encoded">
+ <p>Gogs est une interface web basée sur git et une bonne alternative à GitHub.</p>
+ </div>
+ </div>
+ </div>
+ </div>
+ <footer>
+ <a href="/content/gogs" class="read-more btn btn-default btn-sm">Voir la notice</a>
+ <div class="field field-name-field-lien-officiel field-type-link-field field-label-hidden">
+ <div class="field-items">
+ <div class="field-item even">
+ <a href="https://gogs.io/" target="_blank" title="Voir le site officiel">
+ <span class="glyphicon glyphicon-globe"></span>
+ <span class="sr-only">Lien officiel</span>
+ </a>
+ </div>
+ </div>
+ </div>
+ </footer>
+ </div>
+ </div>
+ """
+ response = mock.Mock(text=html)
+ results = framalibre.response(response)
+ self.assertEqual(type(results), list)
+ self.assertEqual(len(results), 1)
+ self.assertEqual(results[0]['title'], 'Gogs')
+ self.assertEqual(results[0]['url'],
+ 'https://framalibre.org/content/gogs')
+ self.assertEqual(results[0]['content'],
+ u"Gogs est une interface web basée sur git et une bonne alternative à GitHub.")
