logo

searx

Unnamed repository; edit this file 'description' to name the repository.
commit: 678b87f9d5969668a07c36f9b80be5a5b2d08587
parent: fcc3264f8086e96eba84ff6b8b63400952ac546c
Author: Adam Tauber <asciimoo@gmail.com>
Date:   Sat,  9 Jul 2016 22:16:59 +0200

Merge pull request #613 from mmuman/ina

Add INA search engine

Diffstat:

searx/engines/ina.py | 83+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
searx/settings.yml | 6++++++
tests/unit/engines/test_ina.py | 64++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
3 files changed, 153 insertions(+), 0 deletions(-)

diff --git a/searx/engines/ina.py b/searx/engines/ina.py @@ -0,0 +1,83 @@ +# INA (Videos) +# +# @website https://www.ina.fr/ +# @provide-api no +# +# @using-api no +# @results HTML (using search portal) +# @stable no (HTML can change) +# @parse url, title, content, publishedDate, thumbnail +# +# @todo set content-parameter with correct data +# @todo embedded (needs some md5 from video page) + +from json import loads +from urllib import urlencode +from lxml import html +from HTMLParser import HTMLParser +from searx.engines.xpath import extract_text +from dateutil import parser + +# engine dependent config +categories = ['videos'] +paging = True +page_size = 48 + +# search-url +base_url = 'https://www.ina.fr' +search_url = base_url + '/layout/set/ajax/recherche/result?autopromote=&hf={ps}&b={start}&type=Video&r=&{query}' + +# specific xpath variables +results_xpath = '//div[contains(@class,"search-results--list")]/div[@class="media"]' +url_xpath = './/a/@href' +title_xpath = './/h3[@class="h3--title media-heading"]' +thumbnail_xpath = './/img/@src' +publishedDate_xpath = './/span[@class="broadcast"]' +content_xpath = './/p[@class="media-body__summary"]' + + +# do search-request +def request(query, params): + params['url'] = search_url.format(ps=page_size, + start=params['pageno'] * page_size, + query=urlencode({'q': query})) + + return params + + +# get response from search-request +def response(resp): + results = [] + + # we get html in a JSON container... + response = loads(resp.text) + if "content" not in response: + return [] + dom = html.fromstring(response["content"]) + p = HTMLParser() + + # parse results + for result in dom.xpath(results_xpath): + videoid = result.xpath(url_xpath)[0] + url = base_url + videoid + title = p.unescape(extract_text(result.xpath(title_xpath))) + thumbnail = extract_text(result.xpath(thumbnail_xpath)[0]) + if thumbnail[0] == '/': + thumbnail = base_url + thumbnail + d = extract_text(result.xpath(publishedDate_xpath)[0]) + d = d.split('/') + # force ISO date to avoid wrong parsing + d = "%s-%s-%s" % (d[2], d[1], d[0]) + publishedDate = parser.parse(d) + content = extract_text(result.xpath(content_xpath)) + + # append result + results.append({'url': url, + 'title': title, + 'content': content, + 'template': 'videos.html', + 'publishedDate': publishedDate, + 'thumbnail': thumbnail}) + + # return results + return results diff --git a/searx/settings.yml b/searx/settings.yml @@ -254,6 +254,12 @@ engines: disabled : True shortcut : habr + - name : ina + engine : ina + shortcut : in + timeout : 6.0 + disabled : True + - name : mixcloud engine : mixcloud shortcut : mc diff --git a/tests/unit/engines/test_ina.py b/tests/unit/engines/test_ina.py @@ -0,0 +1,64 @@ +from collections import defaultdict +import mock +from searx.engines import ina +from searx.testing import SearxTestCase + + +class TestInaEngine(SearxTestCase): + + def test_request(self): + query = 'test_query' + dicto = defaultdict(dict) + dicto['pageno'] = 0 + params = ina.request(query, dicto) + self.assertTrue('url' in params) + self.assertTrue(query in params['url']) + self.assertTrue('ina.fr' in params['url']) + + def test_response(self): + self.assertRaises(AttributeError, ina.response, None) + self.assertRaises(AttributeError, ina.response, []) + self.assertRaises(AttributeError, ina.response, '') + self.assertRaises(AttributeError, ina.response, '[]') + + response = mock.Mock(text='{}') + self.assertEqual(ina.response(response), []) + + response = mock.Mock(text='{"data": []}') + self.assertEqual(ina.response(response), []) + + json = """ + {"content":"\\t<div class=\\"container\\">\\n\\t\\n\ + <!-- DEBUT CONTENU PRINCIPAL -->\\n<div class=\\"row\\">\\n\ + <div class=\\"search-results--list\\"><div class=\\"media\\">\\n\ + \\t\\t\\t\\t<a class=\\"media-left media-video premium xiti_click_action\\" \ + data-xiti-params=\\"recherche_v4::resultats_conference_de_presse_du_general_de_gaulle::N\\" \ + href=\\"\/video\/CAF89035682\/conference-de-presse-du-general-de-gaulle-video.html\\">\\n\ + <img src=\\"https:\/\/www.ina.fr\/images_v2\/140x105\/CAF89035682.jpeg\\" \ + alt=\\"Conf\\u00e9rence de presse du G\\u00e9n\\u00e9ral de Gaulle \\">\\n\ + \\t\\t\\t\\t\\t<\/a>\\n\ + \\t\\t\\t\\t\\t<div class=\\"media-body\\">\\n\\t\\t\\t\\t\\t\\t<h3 class=\\"h3--title media-heading\\">\\n\ + \\t\\t\\t\\t\\t\\t\\t<a class=\\"xiti_click_action\\" \ + data-xiti-params=\\"recherche_v4::resultats_conference_de_presse_du_general_de_gaulle::N\\" \ + href=\\"\/video\/CAF89035682\/conference-de-presse-du-general-de-gaulle-video.html\\">\ + Conf\\u00e9rence de presse du G\\u00e9n\\u00e9ral de Gaulle <\/a>\\n\ + <\/h3>\\n\ + <div class=\\"media-body__info\\">\\n<span class=\\"broadcast\\">27\/11\/1967<\/span>\\n\ + <span class=\\"views\\">29321 vues<\/span>\\n\ + <span class=\\"duration\\">01h 33m 07s<\/span>\\n\ + <\/div>\\n\ + <p class=\\"media-body__summary\\">VERSION INTEGRALE DE LA CONFERENCE DE PRESSE DU GENERAL DE GAULLE . \ + - PA le Pr\\u00e9sident DE GAULLE : il ouvre les bras et s'assied. DP journalis...<\/p>\\n\ + <\/div>\\n<\/div><!-- \/.media -->\\n" + } + """ + response = mock.Mock(text=json) + results = ina.response(response) + self.assertEqual(type(results), list) + self.assertEqual(len(results), 1) + self.assertEqual(results[0]['title'], u'Conf\xe9rence de presse du G\xe9n\xe9ral de Gaulle') + self.assertEqual(results[0]['url'], + 'https://www.ina.fr/video/CAF89035682/conference-de-presse-du-general-de-gaulle-video.html') + self.assertEqual(results[0]['content'], + u"VERSION INTEGRALE DE LA CONFERENCE DE PRESSE DU GENERAL DE GAULLE ." + u" - PA le Pr\u00e9sident DE GAULLE : il ouvre les bras et s'assied. DP journalis...")