commit: 360f8fab97a364e2f5e7fc8c5329cccb1ad5654a
parent: ccc6955f0c3796347cc32755b82afdf7a1c53a79
Author: Adam Tauber <asciimoo@gmail.com>
Date: Sun, 11 Feb 2018 01:01:01 +0100
Merge pull request #1186 from kvch/fix-bing-videos
Fix Bing videos engine
Diffstat:
2 files changed, 38 insertions(+), 83 deletions(-)
diff --git a/searx/engines/bing_videos.py b/searx/engines/bing_videos.py
@@ -69,22 +69,11 @@ def response(resp):
dom = html.fromstring(resp.text)
for result in dom.xpath('//div[@class="dg_u"]'):
-
- # try to extract the url
- url_container = result.xpath('.//div[@class="sa_wrapper"]/@data-eventpayload')
- if len(url_container) > 0:
- url = loads(url_container[0])['purl']
- else:
- url = result.xpath('./a/@href')[0]
-
- # discard results that do not return an external url
- # very recent results sometimes don't return the video's url
- if url.startswith('/videos/search?'):
- continue
-
- title = extract_text(result.xpath('./a//div[@class="tl"]'))
- content = extract_text(result.xpath('.//div[@class="pubInfo"]'))
- thumbnail = result.xpath('.//div[@class="vthumb"]/img/@src')[0]
+ url = result.xpath('./div[@class="mc_vtvc"]/a/@href')[0]
+ url = 'https://bing.com' + url
+ title = extract_text(result.xpath('./div/a/div/div[@class="mc_vtvc_title"]/@title'))
+ content = extract_text(result.xpath('./div/a/div/div/div/div/text()'))
+ thumbnail = result.xpath('./div/a/div/div/img/@src')[0]
results.append({'url': url,
'title': title,
@@ -92,7 +81,6 @@ def response(resp):
'thumbnail': thumbnail,
'template': 'videos.html'})
- # first page ignores requested number of results
if len(results) >= number_of_results:
break
diff --git a/tests/unit/engines/test_bing_videos.py b/tests/unit/engines/test_bing_videos.py
@@ -47,87 +47,54 @@ class TestBingVideosEngine(SearxTestCase):
self.assertEqual(bing_videos.response(response), [])
html = """
- <div>
- <div class="dg_u">
- <a class="dv_i" href="/videos/search?abcde">
- <div class="vthblock">
- <div class="vthumb">
+ <div class="dg_u">
+ <div id="mc_vtvc_1" class="mc_vtvc">
+ <a class="mc_vtvc_link" href="/video">
+ <div class="mc_vtvc_th">
+ <div class="cico">
<img src="thumb_1.jpg" />
</div>
- <div>
- <div class="tl">
- Title 1
+ <div class="mc_vtvc_ban_lo">
+ <div class="vtbc">
+ <div class="mc_bc_w b_smText">
+ <div class="mc_bc pivot bpi_2">
+ <span title="">
+ <span class="mv_vtvc_play cipg "></span>
+ </span>
+ </div>
+ <div class="mc_bc items">10:06</div>
+ </div>
</div>
</div>
- </div>
- <div class="videoInfoPanel">
- <div class="pubInfo">
- <div>Content 1</div>
</div>
- </div>
- </a>
- <div class="sa_wrapper"
- data-eventpayload="{"purl": "https://url.com/1"}">
- </div>
- </div>
- </div>
- """
- response = mock.Mock(text=html)
- results = bing_videos.response(response)
- self.assertEqual(type(results), list)
- self.assertEqual(len(results), 1)
- self.assertEqual(results[0]['title'], 'Title 1')
- self.assertEqual(results[0]['url'], 'https://url.com/1')
- self.assertEqual(results[0]['content'], 'Content 1')
- self.assertEqual(results[0]['thumbnail'], 'thumb_1.jpg')
-
- html = """
- <div>
- <div class="dg_u">
- <a class="dv_i" href="https://url.com/1">
- <div class="vthblock">
- <div class="vthumb">
- <img src="thumb_1.jpg" />
- </div>
- <div>
- <div class="tl">
- Title 1
- </div>
- </div>
- </div>
- <div class="videoInfoPanel">
- <div class="pubInfo">
- <div>Content 1</div>
- </div>
- </div>
- </a>
- </div>
- <div class="dg_u">
- <a class="dv_i" href="/videos/search?abcde">
- <div class="vthblock">
- <div class="vthumb">
- <img src="thumb_2.jpg" />
- </div>
- <div>
- <div class="tl">
- Title 2
+ <div class="mc_vtvc_meta">
+ <div class="mc_vtvc_title" title="Title 1"></div>
+ <div class="mc_vtvc_meta_block_area">
+ <div class="mc_vtvc_meta_block">
+ <div class="mc_vtvc_meta_row">
+ <span>65,696,000+ views</span>
+ <span>1 year ago</span>
</div>
+ <div class="mc_vtvc_meta_row mc_vtvc_meta_row_channel">Content 1</div>
+ <div class="mc_vtvc_meta_row"><span>
+ <div class="cico mc_vtvc_src_ico">
+ <div></div>
+ </div>
+ <span>YouTube</span>
+ </span></div>
</div>
- </div>
- <div class="videoInfoPanel">
- <div class="pubInfo">
- <div>Content 2</div>
</div>
</div>
- </a>
+ <div class="vrhdata"></div>
+ </a>
+ </div>
</div>
- </div>
"""
response = mock.Mock(text=html)
results = bing_videos.response(response)
self.assertEqual(type(results), list)
self.assertEqual(len(results), 1)
self.assertEqual(results[0]['title'], 'Title 1')
- self.assertEqual(results[0]['url'], 'https://url.com/1')
+ self.assertEqual(results[0]['url'], 'https://bing.com/video')
self.assertEqual(results[0]['content'], 'Content 1')
self.assertEqual(results[0]['thumbnail'], 'thumb_1.jpg')
