logo

searx

Unnamed repository; edit this file 'description' to name the repository.
commit: e34166168ccb3700d6e7b9b3c28870039d070e46
parent: 55e94d439b138d1d9467a5f150715af84db343e1
Author: Adam Tauber <asciimoo@gmail.com>
Date:   Sun,  3 May 2015 18:45:56 -0400

Merge pull request #310 from dalf/yahoo_news

[fix] yahoo_news engine

Diffstat:

searx/engines/yahoo_news.py | 25++++++++++++++++++-------
searx/tests/engines/test_yahoo_news.py | 153+++++++++++++++++++++++++++++++++++++------------------------------------------
2 files changed, 89 insertions(+), 89 deletions(-)

diff --git a/searx/engines/yahoo_news.py b/searx/engines/yahoo_news.py @@ -23,15 +23,15 @@ paging = True language_support = True # search-url -search_url = 'https://news.search.yahoo.com/search?{query}&b={offset}&fl=1&vl=lang_{lang}' # noqa +search_url = 'https://news.search.yahoo.com/search?{query}&b={offset}&{lang}=uh3_news_web_gs_1&pz=10&xargs=0&vl=lang_{lang}' # noqa # specific xpath variables -results_xpath = '//div[@class="res"]' +results_xpath = '//ol[contains(@class,"searchCenterMiddle")]//li' url_xpath = './/h3/a/@href' title_xpath = './/h3/a' -content_xpath = './/div[@class="abstr"]' -publishedDate_xpath = './/span[@class="timestamp"]' -suggestion_xpath = '//div[@id="satat"]//a' +content_xpath = './/div[@class="compText"]' +publishedDate_xpath = './/span[contains(@class,"tri")]' +suggestion_xpath = '//div[contains(@class,"VerALSOTRY")]//a' # do search-request @@ -48,11 +48,18 @@ def request(query, params): lang=language) # TODO required? - params['cookies']['sB'] = 'fl=1&vl=lang_{lang}&sh=1&rw=new&v=1'\ + params['cookies']['sB'] = '"v=1&vm=p&fl=1&vl=lang_{lang}&sh=1&pn=10&rw=new'\ .format(lang=language) return params +def sanitize_url(url): + if ".yahoo.com/" in url: + return re.sub(u"\;\_ylt\=.+$", "", url) + else: + return url + + # get response from search-request def response(resp): results = [] @@ -61,13 +68,17 @@ def response(resp): # parse results for result in dom.xpath(results_xpath): - url = parse_url(extract_url(result.xpath(url_xpath), search_url)) + urls = result.xpath(url_xpath) + if len(urls) != 1: + continue + url = sanitize_url(parse_url(extract_url(urls, search_url))) title = extract_text(result.xpath(title_xpath)[0]) content = extract_text(result.xpath(content_xpath)[0]) # parse publishedDate publishedDate = extract_text(result.xpath(publishedDate_xpath)[0]) + # still useful ? if re.match("^[0-9]+ minute(s|) ago$", publishedDate): publishedDate = datetime.now() - timedelta(minutes=int(re.match(r'\d+', publishedDate).group())) # noqa else: diff --git a/searx/tests/engines/test_yahoo_news.py b/searx/tests/engines/test_yahoo_news.py @@ -39,19 +39,24 @@ class TestYahooNewsEngine(SearxTestCase): self.assertEqual(yahoo_news.response(response), []) html = """ - <div class="res"> - <div> - <h3> - <a class="yschttl spt" href="http://this.is.the.url" target="_blank"> - This is - the <b>title</b>... - </a> - </h3> - </div> - <span class="url">Business via Yahoo! Finance</span> &nbsp; <span class="timestamp">Feb 03 09:45am</span> - <div class="abstr"> - This is the content - </div> + <ol class=" reg searchCenterMiddle"> + <li class="first"> + <div class="compTitle"> + <h3> + <a class="yschttl spt" href="http://this.is.the.url" target="_blank"> + This is + the <b>title</b>... + </a> + </h3> + </div> + <div> + <span class="cite">Business via Yahoo!</span> + <span class="tri fc-2nd ml-10">May 01 10:00 AM</span> + </div> + <div class="compText"> + This is the content + </div> + </li> </div> """ response = mock.Mock(text=html) @@ -63,48 +68,59 @@ class TestYahooNewsEngine(SearxTestCase): self.assertEqual(results[0]['content'], 'This is the content') html = """ - <div class="res"> - <div> - <h3> - <a class="yschttl spt" href="http://this.is.the.url" target="_blank"> - This is - the <b>title</b>... - </a> - </h3> - </div> - <span class="url">Business via Yahoo!</span> &nbsp; <span class="timestamp">2 hours, 22 minutes ago</span> - <div class="abstr"> - This is the content - </div> - </div> - <div class="res"> - <div> - <h3> - <a class="yschttl spt" href="http://this.is.the.url" target="_blank"> - This is - the <b>title</b>... - </a> - </h3> - </div> - <span class="url">Business via Yahoo!</span> &nbsp; <span class="timestamp">22 minutes ago</span> - <div class="abstr"> - This is the content - </div> - </div> - <div class="res"> - <div> - <h3> - <a class="yschttl spt" href="http://this.is.the.url" target="_blank"> - This is - the <b>title</b>... - </a> - </h3> - </div> - <span class="url">Business via Yahoo!</span> &nbsp; <span class="timestamp">Feb 03 09:45am 1900</span> - <div class="abstr"> - This is the content - </div> - </div> + <ol class=" reg searchCenterMiddle"> + <li class="first"> + <div class="compTitle"> + <h3> + <a class="yschttl spt" href="http://this.is.the.url" target="_blank"> + This is + the <b>title</b>... + </a> + </h3> + </div> + <div> + <span class="cite">Business via Yahoo!</span> + <span class="tri fc-2nd ml-10">2 hours, 22 minutes ago</span> + </div> + <div class="compText"> + This is the content + </div> + </li> + <li> + <div class="compTitle"> + <h3> + <a class="yschttl spt" href="http://this.is.the.url" target="_blank"> + This is + the <b>title</b>... + </a> + </h3> + </div> + <div> + <span class="cite">Business via Yahoo!</span> + <span class="tri fc-2nd ml-10">22 minutes ago</span> + </div> + <div class="compText"> + This is the content + </div> + </li> + <li> + <div class="compTitle"> + <h3> + <a class="yschttl spt" href="http://this.is.the.url" target="_blank"> + This is + the <b>title</b>... + </a> + </h3> + </div> + <div> + <span class="cite">Business via Yahoo!</span> + <span class="tri fc-2nd ml-10">Feb 03 09:45AM 1900</span> + </div> + <div class="compText"> + This is the content + </div> + </li> + </ol> """ response = mock.Mock(text=html) results = yahoo_news.response(response) @@ -114,30 +130,3 @@ class TestYahooNewsEngine(SearxTestCase): self.assertEqual(results[0]['url'], 'http://this.is.the.url/') self.assertEqual(results[0]['content'], 'This is the content') self.assertEqual(results[2]['publishedDate'].year, datetime.now().year) - - html = """ - <li class="b_algo" u="0|5109|4755453613245655|UAGjXgIrPH5yh-o5oNHRx_3Zta87f_QO"> - <div Class="sa_mc"> - <div class="sb_tlst"> - <h2> - <a href="http://this.should.be.the.link/" h="ID=SERP,5124.1"> - <strong>This</strong> should be the title</a> - </h2> - </div> - <div class="sb_meta"> - <cite> - <strong>this</strong>.meta.com</cite> - <span class="c_tlbxTrg"> - <span class="c_tlbxH" H="BASE:CACHEDPAGEDEFAULT" K="SERP,5125.1"> - </span> - </span> - </div> - <p> - <strong>This</strong> should be the content.</p> - </div> - </li> - """ - response = mock.Mock(text=html) - results = yahoo_news.response(response) - self.assertEqual(type(results), list) - self.assertEqual(len(results), 0)