logo

searx

Unnamed repository; edit this file 'description' to name the repository.
commit: c9b8c7f8deffce93b920273171a4f8f9d6339ea7
parent: 1b77befe1fb1dbf82f99db69f857d21a7f0fdd6e
Author: Adam Tauber <asciimoo@gmail.com>
Date:   Mon, 15 Jun 2015 04:43:15 -0400

Merge pull request #370 from Cqoicebordel/fix_vimeo

Fix vimeo

Diffstat:

searx/engines/vimeo.py | 12++++++------
searx/tests/engines/test_swisscows.py | 4++++
searx/tests/engines/test_vimeo.py | 53+++++++++++++++++++++++++++++++++++------------------
searx/tests/engines/test_yahoo_news.py | 19++++++++++++++++++-
searx/tests/engines/test_youtube_noapi.py | 51+++++++++++++++++++++++++++++++++++++++++++++++++++
5 files changed, 114 insertions(+), 25 deletions(-)

diff --git a/searx/engines/vimeo.py b/searx/engines/vimeo.py @@ -27,11 +27,11 @@ base_url = 'https://vimeo.com' search_url = base_url + '/search/page:{pageno}?{query}' # specific xpath variables -results_xpath = '//div[@id="browse_content"]/ol/li' -url_xpath = './a/@href' -title_xpath = './a/div[@class="data"]/p[@class="title"]' -content_xpath = './a/img/@src' -publishedDate_xpath = './/p[@class="meta"]//attribute::datetime' +results_xpath = '//div[contains(@class,"results_grid")]/ul/li' +url_xpath = './/a/@href' +title_xpath = './/span[@class="title"]' +thumbnail_xpath = './/img[@class="js-clip_thumbnail_image"]/@src' +publishedDate_xpath = './/time/attribute::datetime' embedded_url = '<iframe data-src="//player.vimeo.com/video{videoid}" ' +\ 'width="540" height="304" frameborder="0" ' +\ @@ -58,7 +58,7 @@ def response(resp): videoid = result.xpath(url_xpath)[0] url = base_url + videoid title = p.unescape(extract_text(result.xpath(title_xpath))) - thumbnail = extract_text(result.xpath(content_xpath)[0]) + thumbnail = extract_text(result.xpath(thumbnail_xpath)[0]) publishedDate = parser.parse(extract_text(result.xpath(publishedDate_xpath)[0])) embedded = embedded_url.format(videoid=videoid) diff --git a/searx/tests/engines/test_swisscows.py b/searx/tests/engines/test_swisscows.py @@ -23,6 +23,10 @@ class TestSwisscowsEngine(SearxTestCase): self.assertTrue('uiLanguage=browser' in params['url']) self.assertTrue('region=browser' in params['url']) + dicto['category'] = 'images' + params = swisscows.request(query, dicto) + self.assertIn('image', params['url']) + def test_response(self): self.assertRaises(AttributeError, swisscows.response, None) self.assertRaises(AttributeError, swisscows.response, []) diff --git a/searx/tests/engines/test_vimeo.py b/searx/tests/engines/test_vimeo.py @@ -1,3 +1,4 @@ +# -*- coding: utf-8 -*- from collections import defaultdict import mock from searx.engines import vimeo @@ -25,26 +26,42 @@ class TestVimeoEngine(SearxTestCase): self.assertEqual(vimeo.response(response), []) html = """ - <div id="browse_content" class="" data-search-id="696d5f8366914ec4ffec33cf7652de384976d4f4"> - <ol class="js-browse_list clearfix browse browse_videos browse_videos_thumbnails kane" + <div id="browse_content" class="results_grid" data-search-id="696d5f8366914ec4ffec33cf7652de384976d4f4"> + <ul class="js-browse_list clearfix browse browse_videos browse_videos_thumbnails kane" data-stream="c2VhcmNoOjo6ZGVzYzp7InF1ZXJ5IjoidGVzdCJ9"> - <li id="clip_100785455" data-start-page="/search/page:1/sort:relevant/" data-position="1"> - <a href="/videoid" title="Futurama 3d (test shot)"> - <img src="http://image.url.webp" - srcset="http://i.vimeocdn.com/video/482375085_590x332.webp 2x" alt="" - class="thumbnail thumbnail_lg_wide"> - <div class="data"> - <p class="title"> - This is the title - </p> - <p class="meta"> - <time datetime="2014-07-15T04:16:27-04:00" - title="mardi 15 juillet 2014 04:16">Il y a 6 mois</time> - </p> - </div> - </a> + <li data-position="7" data-result-id="clip_79600943"> + <div class="clip_thumbnail"> + <a href="/videoid" class="js-result_url"> + <div class="thumbnail_wrapper"> + <img src="http://image.url.webp" class="js-clip_thumbnail_image"> + <div class="overlay overlay_clip_meta"> + <div class="meta_data_footer"> + <span class="clip_upload_date"> + <time datetime="2013-11-17T08:49:09-05:00" + title="dimanche 17 novembre 2013 08:49">Il y a 1 an</time> + </span> + <span class="clip_likes"> + <img src="https://f.vimeocdn.com/images_v6/svg/heart-icon.svg">2 215 + </span> + <span class="clip_comments"> + <img src="https://f.vimeocdn.com/images_v6/svg/comment-icon.svg">75 + </span> + <span class="overlay meta_data_footer clip_duration">01:12</span> + </div> + </div> + </div> + <span class="title">This is the title</span> + </a> + </div> + <div class="clip_thumbnail_attribution"> + <a href="/fedorshmidt"> + <img src="https://i.vimeocdn.com/portrait/6628061_100x100.jpg" class="avatar"> + <span class="display_name">Fedor Shmidt</span> + </a> + <span class="plays">2,1M lectures</span> + </div> </li> - </ol> + </ul> </div> """ response = mock.Mock(text=html) diff --git a/searx/tests/engines/test_yahoo_news.py b/searx/tests/engines/test_yahoo_news.py @@ -29,6 +29,13 @@ class TestYahooNewsEngine(SearxTestCase): self.assertIn('en', params['cookies']['sB']) self.assertIn('en', params['url']) + def test_sanitize_url(self): + url = "test.url" + self.assertEqual(url, yahoo_news.sanitize_url(url)) + + url = "www.yahoo.com/;_ylt=test" + self.assertEqual("www.yahoo.com/", yahoo_news.sanitize_url(url)) + def test_response(self): self.assertRaises(AttributeError, yahoo_news.response, None) self.assertRaises(AttributeError, yahoo_news.response, []) @@ -57,7 +64,17 @@ class TestYahooNewsEngine(SearxTestCase): This is the content </div> </li> - </div> + <li class="first"> + <div class="compTitle"> + <h3> + <a class="yschttl spt" target="_blank"> + </a> + </h3> + </div> + <div class="compText"> + </div> + </li> + </ol> """ response = mock.Mock(text=html) results = yahoo_news.response(response) diff --git a/searx/tests/engines/test_youtube_noapi.py b/searx/tests/engines/test_youtube_noapi.py @@ -94,6 +94,57 @@ class TestYoutubeNoAPIEngine(SearxTestCase): html = """ <ol id="item-section-063864" class="item-section"> <li> + <div class="yt-lockup yt-lockup-tile yt-lockup-video vve-check clearfix yt-uix-tile" + data-context-item-id="DIVZCPfAOeM" + data-visibility-tracking="CBgQ3DAYACITCPGXnYau6sUCFZEIHAod-VQASCj0JECx_-GK5uqMpcIB"> + <div class="yt-lockup-dismissable"><div class="yt-lockup-thumbnail contains-addto"> + <a aria-hidden="true" href="/watch?v=DIVZCPfAOeM" class=" yt-uix-sessionlink pf-link" + data-sessionlink="itct=CBgQ3DAYACITCPGXnYau6sUCFZEIHAod-VQASCj0JFIEdGVzdA"> + <div class="yt-thumb video-thumb"><img src="//i.ytimg.com/vi/DIVZCPfAOeM/mqdefault.jpg" + width="196" height="110"/></div><span class="video-time" aria-hidden="true">11:35</span></a> + <span class="thumb-menu dark-overflow-action-menu video-actions"> + </span> + </div> + <div class="yt-lockup-content"> + <h3 class="yt-lockup-title"> + <span class="accessible-description" id="description-id-259079"> - Durée : 11:35.</span> + </h3> + <div class="yt-lockup-byline">de + <a href="/user/mheejapan" class=" yt-uix-sessionlink spf-link g-hovercard" + data-sessionlink="itct=CBgQ3DAYACITCPGXnYau6sUCFZEIHAod-VQASCj0JA" data-ytid="UCzEesu54Hjs0uRKmpy66qeA" + data-name="">MEHAY SUPERBIKE</a></div><div class="yt-lockup-meta"> + <ul class="yt-lockup-meta-info"> + <li>il y a 20 heures</li> + <li>8 424 vues</li> + </ul> + </div> + <div class="yt-lockup-badges"> + <ul class="yt-badge-list "> + <li class="yt-badge-item" > + <span class="yt-badge">Nouveauté</span> + </li> + <li class="yt-badge-item" ><span class="yt-badge " >HD</span></li> + </ul> + </div> + <div class="yt-lockup-action-menu yt-uix-menu-container"> + <div class="yt-uix-menu yt-uix-videoactionmenu hide-until-delayloaded" + data-video-id="DIVZCPfAOeM" data-menu-content-id="yt-uix-videoactionmenu-menu"> + </div> + </div> + </div> + </div> + </div> + </li> + </ol> + """ + response = mock.Mock(text=html) + results = youtube_noapi.response(response) + self.assertEqual(type(results), list) + self.assertEqual(len(results), 1) + + html = """ + <ol id="item-section-063864" class="item-section"> + <li> </li> </ol> """