logo

searx

Unnamed repository; edit this file 'description' to name the repository.
commit: 0e6f8393ab8b29b2e85d1fafdc7442455767f753
parent: 03137eebd9fdfaa57452cb364c1bc9f31b243f67
Author: Adam Tauber <adam.tauber@balabit.com>
Date:   Mon,  2 Feb 2015 09:37:12 +0100

Merge branch 'Cqoicebordel-unit-tests'

Diffstat:

.gitignore | 1+
searx/engines/bing.py | 9+++++----
searx/engines/bing_images.py | 5++++-
searx/engines/bing_news.py | 31++++++++++++-------------------
searx/engines/btdigg.py | 11+++--------
searx/engines/deviantart.py | 7++++---
searx/engines/digg.py | 2+-
searx/engines/flickr_noapi.py | 2+-
searx/engines/google_images.py | 6+++---
searx/engines/google_news.py | 6++++--
searx/engines/kickass.py | 8++++----
searx/engines/piratebay.py | 12++++++++----
searx/engines/searchcode_code.py | 2+-
searx/engines/searchcode_doc.py | 2+-
searx/engines/stackoverflow.py | 8++++----
searx/engines/vimeo.py | 3+--
searx/engines/www500px.py | 11++++++-----
searx/engines/xpath.py | 4++--
searx/engines/youtube.py | 4++--
searx/settings.yml | 6+++---
searx/tests/engines/test_bing.py | 90+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
searx/tests/engines/test_bing_images.py | 268+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
searx/tests/engines/test_bing_news.py | 236+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
searx/tests/engines/test_btdigg.py | 384+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
searx/tests/engines/test_dailymotion.py | 74++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
searx/tests/engines/test_deezer.py | 57+++++++++++++++++++++++++++++++++++++++++++++++++++++++++
searx/tests/engines/test_deviantart.py | 118+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
searx/tests/engines/test_digg.py | 101+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
searx/tests/engines/test_flickr.py | 142+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
searx/tests/engines/test_flickr_noapi.py | 442+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
searx/tests/engines/test_google_images.py | 108+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
searx/tests/engines/test_google_news.py | 136+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
searx/tests/engines/test_kickass.py | 398+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
searx/tests/engines/test_mixcloud.py | 67+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
searx/tests/engines/test_piratebay.py | 137+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
searx/tests/engines/test_searchcode_code.py | 75+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
searx/tests/engines/test_searchcode_doc.py | 73+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
searx/tests/engines/test_soundcloud.py | 192+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
searx/tests/engines/test_stackoverflow.py | 106+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
searx/tests/engines/test_vimeo.py | 84+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
searx/tests/engines/test_www500px.py | 83+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
searx/tests/engines/test_youtube.py | 204+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
searx/tests/test_engines.py | 22++++++++++++++++++++++
searx/tests/test_utils.py | 22++++++++++++++++++++++
searx/utils.py | 4+++-
45 files changed, 3692 insertions(+), 71 deletions(-)

diff --git a/.gitignore b/.gitignore @@ -23,3 +23,4 @@ local/ parts/ searx.egg-info/ var/ +node_modules/ diff --git a/searx/engines/bing.py b/searx/engines/bing.py @@ -14,6 +14,7 @@ from urllib import urlencode from cgi import escape from lxml import html +from searx.engines.xpath import extract_text # engine dependent config categories = ['general'] @@ -55,8 +56,8 @@ def response(resp): for result in dom.xpath('//div[@class="sa_cc"]'): link = result.xpath('.//h3/a')[0] url = link.attrib.get('href') - title = ' '.join(link.xpath('.//text()')) - content = escape(' '.join(result.xpath('.//p//text()'))) + title = extract_text(link) + content = escape(extract_text(result.xpath('.//p'))) # append result results.append({'url': url, @@ -71,8 +72,8 @@ def response(resp): for result in dom.xpath('//li[@class="b_algo"]'): link = result.xpath('.//h2/a')[0] url = link.attrib.get('href') - title = ' '.join(link.xpath('.//text()')) - content = escape(' '.join(result.xpath('.//p//text()'))) + title = extract_text(link) + content = escape(extract_text(result.xpath('.//p'))) # append result results.append({'url': url, diff --git a/searx/engines/bing_images.py b/searx/engines/bing_images.py @@ -33,7 +33,10 @@ def request(query, params): offset = (params['pageno'] - 1) * 10 + 1 # required for cookie - language = 'en-US' + if params['language'] == 'all': + language = 'en-US' + else: + language = params['language'].replace('_', '-') search_path = search_string.format( query=urlencode({'q': query}), diff --git a/searx/engines/bing_news.py b/searx/engines/bing_news.py @@ -15,6 +15,7 @@ from lxml import html from datetime import datetime, timedelta from dateutil import parser import re +from searx.engines.xpath import extract_text # engine dependent config categories = ['news'] @@ -42,6 +43,7 @@ def request(query, params): params['cookies']['_FP'] = "ui=en-US" params['url'] = base_url + search_path + return params @@ -55,44 +57,35 @@ def response(resp): for result in dom.xpath('//div[@class="sn_r"]'): link = result.xpath('.//div[@class="newstitle"]/a')[0] url = link.attrib.get('href') - title = ' '.join(link.xpath('.//text()')) - contentXPath = result.xpath('.//div[@class="sn_txt"]/div' - '//span[@class="sn_snip"]//text()') - if contentXPath is not None: - content = escape(' '.join(contentXPath)) + title = extract_text(link) + contentXPath = result.xpath('.//div[@class="sn_txt"]/div//span[@class="sn_snip"]') + content = escape(extract_text(contentXPath)) # parse publishedDate publishedDateXPath = result.xpath('.//div[@class="sn_txt"]/div' '//span[contains(@class,"sn_ST")]' - '//span[contains(@class,"sn_tm")]' - '//text()') - if publishedDateXPath is not None: - publishedDate = escape(' '.join(publishedDateXPath)) + '//span[contains(@class,"sn_tm")]') + + publishedDate = escape(extract_text(publishedDateXPath)) if re.match("^[0-9]+ minute(s|) ago$", publishedDate): timeNumbers = re.findall(r'\d+', publishedDate) - publishedDate = datetime.now()\ - - timedelta(minutes=int(timeNumbers[0])) + publishedDate = datetime.now() - timedelta(minutes=int(timeNumbers[0])) elif re.match("^[0-9]+ hour(s|) ago$", publishedDate): timeNumbers = re.findall(r'\d+', publishedDate) - publishedDate = datetime.now()\ - - timedelta(hours=int(timeNumbers[0])) - elif re.match("^[0-9]+ hour(s|)," - " [0-9]+ minute(s|) ago$", publishedDate): + publishedDate = datetime.now() - timedelta(hours=int(timeNumbers[0])) + elif re.match("^[0-9]+ hour(s|), [0-9]+ minute(s|) ago$", publishedDate): timeNumbers = re.findall(r'\d+', publishedDate) publishedDate = datetime.now()\ - timedelta(hours=int(timeNumbers[0]))\ - timedelta(minutes=int(timeNumbers[1])) elif re.match("^[0-9]+ day(s|) ago$", publishedDate): timeNumbers = re.findall(r'\d+', publishedDate) - publishedDate = datetime.now()\ - - timedelta(days=int(timeNumbers[0])) + publishedDate = datetime.now() - timedelta(days=int(timeNumbers[0])) else: try: - # FIXME use params['language'] to parse either mm/dd or dd/mm publishedDate = parser.parse(publishedDate, dayfirst=False) except TypeError: - # FIXME publishedDate = datetime.now() # append result diff --git a/searx/engines/btdigg.py b/searx/engines/btdigg.py @@ -23,11 +23,6 @@ paging = True url = 'https://btdigg.org' search_url = url + '/search?q={search_term}&p={pageno}' -# specific xpath variables -magnet_xpath = './/a[@title="Torrent magnet link"]' -torrent_xpath = './/a[@title="Download torrent file"]' -content_xpath = './/span[@class="font11px lightgrey block"]' - # do search-request def request(query, params): @@ -52,8 +47,8 @@ def response(resp): # parse results for result in search_res: link = result.xpath('.//td[@class="torrent_name"]//a')[0] - href = urljoin(url, link.attrib['href']) - title = escape(extract_text(link.xpath('.//text()'))) + href = urljoin(url, link.attrib.get('href')) + title = escape(extract_text(link)) content = escape(extract_text(result.xpath('.//pre[@class="snippet"]')[0])) content = "<br />".join(content.split("\n")) @@ -81,7 +76,7 @@ def response(resp): filesize = int(filesize * 1024 * 1024 * 1024) elif filesize_multiplier == 'MB': filesize = int(filesize * 1024 * 1024) - elif filesize_multiplier == 'kb': + elif filesize_multiplier == 'KB': filesize = int(filesize * 1024) except: filesize = None diff --git a/searx/engines/deviantart.py b/searx/engines/deviantart.py @@ -14,6 +14,7 @@ from urllib import urlencode from urlparse import urljoin from lxml import html import re +from searx.engines.xpath import extract_text # engine dependent config categories = ['images'] @@ -50,9 +51,9 @@ def response(resp): for result in dom.xpath('//div[contains(@class, "tt-a tt-fh")]'): link = result.xpath('.//a[contains(@class, "thumb")]')[0] url = urljoin(base_url, link.attrib.get('href')) - title_links = result.xpath('.//span[@class="details"]//a[contains(@class, "t")]') # noqa - title = ''.join(title_links[0].xpath('.//text()')) - thumbnail_src = link.xpath('.//img')[0].attrib['src'] + title_links = result.xpath('.//span[@class="details"]//a[contains(@class, "t")]') + title = extract_text(title_links[0]) + thumbnail_src = link.xpath('.//img')[0].attrib.get('src') img_src = regex.sub('/', thumbnail_src) # append result diff --git a/searx/engines/digg.py b/searx/engines/digg.py @@ -44,7 +44,7 @@ def response(resp): search_result = loads(resp.text) - if search_result['html'] == '': + if 'html' not in search_result or search_result['html'] == '': return results dom = html.fromstring(search_result['html']) diff --git a/searx/engines/flickr_noapi.py b/searx/engines/flickr_noapi.py @@ -21,7 +21,7 @@ logger = logger.getChild('flickr-noapi') categories = ['images'] url = 'https://secure.flickr.com/' -search_url = url+'search/?{query}&page={page}' +search_url = url + 'search/?{query}&page={page}' photo_url = 'https://www.flickr.com/photos/{userid}/{photoid}' regex = re.compile(r"\"search-photos-models\",\"photos\":(.*}),\"totalItems\":", re.DOTALL) image_sizes = ('o', 'k', 'h', 'b', 'c', 'z', 'n', 'm', 't', 'q', 's') diff --git a/searx/engines/google_images.py b/searx/engines/google_images.py @@ -18,7 +18,7 @@ paging = True # search-url url = 'https://ajax.googleapis.com/' -search_url = url + 'ajax/services/search/images?v=1.0&start={offset}&rsz=large&safe=off&filter=off&{query}' # noqa +search_url = url + 'ajax/services/search/images?v=1.0&start={offset}&rsz=large&safe=off&filter=off&{query}' # do search-request @@ -45,14 +45,14 @@ def response(resp): for result in search_res['responseData']['results']: href = result['originalContextUrl'] title = result['title'] - if not result['url']: + if 'url' not in result: continue thumbnail_src = result['tbUrl'] # append result results.append({'url': href, 'title': title, - 'content': '', + 'content': result['content'], 'thumbnail_src': thumbnail_src, 'img_src': unquote(result['url']), 'template': 'images.html'}) diff --git a/searx/engines/google_news.py b/searx/engines/google_news.py @@ -20,7 +20,7 @@ language_support = True # engine dependent config url = 'https://ajax.googleapis.com/' -search_url = url + 'ajax/services/search/news?v=2.0&start={offset}&rsz=large&safe=off&filter=off&{query}&hl={language}' # noqa +search_url = url + 'ajax/services/search/news?v=2.0&start={offset}&rsz=large&safe=off&filter=off&{query}&hl={lang}' # do search-request @@ -33,7 +33,7 @@ def request(query, params): params['url'] = search_url.format(offset=offset, query=urlencode({'q': query}), - language=language) + lang=language) return params @@ -52,6 +52,8 @@ def response(resp): for result in search_res['responseData']['results']: # parse publishedDate publishedDate = parser.parse(result['publishedDate']) + if 'url' not in result: + continue # append result results.append({'url': result['unescapedUrl'], diff --git a/searx/engines/kickass.py b/searx/engines/kickass.py @@ -13,6 +13,7 @@ from cgi import escape from urllib import quote from lxml import html from operator import itemgetter +from searx.engines.xpath import extract_text # engine dependent config categories = ['videos', 'music', 'files'] @@ -56,9 +57,8 @@ def response(resp): for result in search_res[1:]: link = result.xpath('.//a[@class="cellMainLink"]')[0] href = urljoin(url, link.attrib['href']) - title = ' '.join(link.xpath('.//text()')) - content = escape(html.tostring(result.xpath(content_xpath)[0], - method="text")) + title = extract_text(link) + content = escape(extract_text(result.xpath(content_xpath))) seed = result.xpath('.//td[contains(@class, "green")]/text()')[0] leech = result.xpath('.//td[contains(@class, "red")]/text()')[0] filesize = result.xpath('.//td[contains(@class, "nobr")]/text()')[0] @@ -88,7 +88,7 @@ def response(resp): filesize = int(filesize * 1024 * 1024 * 1024) elif filesize_multiplier == 'MB': filesize = int(filesize * 1024 * 1024) - elif filesize_multiplier == 'kb': + elif filesize_multiplier == 'KB': filesize = int(filesize * 1024) except: filesize = None diff --git a/searx/engines/piratebay.py b/searx/engines/piratebay.py @@ -13,6 +13,7 @@ from cgi import escape from urllib import quote from lxml import html from operator import itemgetter +from searx.engines.xpath import extract_text # engine dependent config categories = ['videos', 'music', 'files'] @@ -29,7 +30,8 @@ search_types = {'files': '0', # specific xpath variables magnet_xpath = './/a[@title="Download this torrent using magnet"]' -content_xpath = './/font[@class="detDesc"]//text()' +torrent_xpath = './/a[@title="Download this torrent"]' +content_xpath = './/font[@class="detDesc"]' # do search-request @@ -59,8 +61,8 @@ def response(resp): for result in search_res[1:]: link = result.xpath('.//div[@class="detName"]//a')[0] href = urljoin(url, link.attrib.get('href')) - title = ' '.join(link.xpath('.//text()')) - content = escape(' '.join(result.xpath(content_xpath))) + title = extract_text(link) + content = escape(extract_text(result.xpath(content_xpath))) seed, leech = result.xpath('.//td[@align="right"]/text()')[:2] # convert seed to int if possible @@ -76,6 +78,7 @@ def response(resp): leech = 0 magnetlink = result.xpath(magnet_xpath)[0] + torrentfile = result.xpath(torrent_xpath)[0] # append result results.append({'url': href, @@ -83,7 +86,8 @@ def response(resp): 'content': content, 'seed': seed, 'leech': leech, - 'magnetlink': magnetlink.attrib['href'], + 'magnetlink': magnetlink.attrib.get('href'), + 'torrentfile': torrentfile.attrib.get('href'), 'template': 'torrent.html'}) # return results sorted by seeder diff --git a/searx/engines/searchcode_code.py b/searx/engines/searchcode_code.py @@ -42,7 +42,7 @@ def response(resp): search_results = loads(resp.text) # parse results - for result in search_results['results']: + for result in search_results.get('results', []): href = result['url'] title = "" + result['name'] + " - " + result['filename'] repo = result['repo'] diff --git a/searx/engines/searchcode_doc.py b/searx/engines/searchcode_doc.py @@ -35,7 +35,7 @@ def response(resp): search_results = loads(resp.text) # parse results - for result in search_results['results']: + for result in search_results.get('results', []): href = result['url'] title = "[" + result['type'] + "] " +\ result['namespace'] +\ diff --git a/searx/engines/stackoverflow.py b/searx/engines/stackoverflow.py @@ -12,6 +12,7 @@ from urlparse import urljoin from cgi import escape from urllib import urlencode from lxml import html +from searx.engines.xpath import extract_text # engine dependent config categories = ['it'] @@ -24,8 +25,7 @@ search_url = url+'search?{query}&page={pageno}' # specific xpath variables results_xpath = '//div[contains(@class,"question-summary")]' link_xpath = './/div[@class="result-link"]//a|.//div[@class="summary"]//h3//a' -title_xpath = './/text()' -content_xpath = './/div[@class="excerpt"]//text()' +content_xpath = './/div[@class="excerpt"]' # do search-request @@ -46,8 +46,8 @@ def response(resp): for result in dom.xpath(results_xpath): link = result.xpath(link_xpath)[0] href = urljoin(url, link.attrib.get('href')) - title = escape(' '.join(link.xpath(title_xpath))) - content = escape(' '.join(result.xpath(content_xpath))) + title = escape(extract_text(link)) + content = escape(extract_text(result.xpath(content_xpath))) # append result results.append({'url': href, diff --git a/searx/engines/vimeo.py b/searx/engines/vimeo.py @@ -59,8 +59,7 @@ def response(resp): url = base_url + videoid title = p.unescape(extract_text(result.xpath(title_xpath))) thumbnail = extract_text(result.xpath(content_xpath)[0]) - publishedDate = parser.parse(extract_text( - result.xpath(publishedDate_xpath)[0])) + publishedDate = parser.parse(extract_text(result.xpath(publishedDate_xpath)[0])) embedded = embedded_url.format(videoid=videoid) # append result diff --git a/searx/engines/www500px.py b/searx/engines/www500px.py @@ -15,6 +15,7 @@ from urllib import urlencode from urlparse import urljoin from lxml import html import re +from searx.engines.xpath import extract_text # engine dependent config categories = ['images'] @@ -22,7 +23,7 @@ paging = True # search-url base_url = 'https://500px.com' -search_url = base_url+'/search?search?page={pageno}&type=photos&{query}' +search_url = base_url + '/search?search?page={pageno}&type=photos&{query}' # do search-request @@ -44,11 +45,11 @@ def response(resp): for result in dom.xpath('//div[@class="photo"]'): link = result.xpath('.//a')[0] url = urljoin(base_url, link.attrib.get('href')) - title = result.xpath('.//div[@class="title"]//text()')[0] - thumbnail_src = link.xpath('.//img')[0].attrib['src'] + title = extract_text(result.xpath('.//div[@class="title"]')) + thumbnail_src = link.xpath('.//img')[0].attrib.get('src') # To have a bigger thumbnail, uncomment the next line - #thumbnail_src = regex.sub('4.jpg', thumbnail_src) - content = result.xpath('.//div[@class="info"]//text()')[0] + # thumbnail_src = regex.sub('4.jpg', thumbnail_src) + content = extract_text(result.xpath('.//div[@class="info"]')) img_src = regex.sub('2048.jpg', thumbnail_src) # append result diff --git a/searx/engines/xpath.py b/searx/engines/xpath.py @@ -28,13 +28,13 @@ def extract_text(xpath_results): result = '' for e in xpath_results: result = result + extract_text(e) - return result + return result.strip() elif type(xpath_results) in [_ElementStringResult, _ElementUnicodeResult]: # it's a string return ''.join(xpath_results) else: # it's a element - return html_to_text(xpath_results.text_content()) + return html_to_text(xpath_results.text_content()).strip() def extract_url(xpath_results, search_url): diff --git a/searx/engines/youtube.py b/searx/engines/youtube.py @@ -57,7 +57,7 @@ def response(resp): url = [x['href'] for x in result['link'] if x['type'] == 'text/html'] if not url: - return + continue # remove tracking url = url[0].replace('feature=youtube_gdata', '') @@ -73,7 +73,7 @@ def response(resp): pubdate = result['published']['$t'] publishedDate = parser.parse(pubdate) - if result['media$group']['media$thumbnail']: + if 'media$thumbnail' in result['media$group']: thumbnail = result['media$group']['media$thumbnail'][0]['url'] content = result['content']['$t'] diff --git a/searx/settings.yml b/searx/settings.yml @@ -161,9 +161,9 @@ engines: engine : photon shortcut : ph -# - name : piratebay -# engine : piratebay -# shortcut : tpb + - name : piratebay + engine : piratebay + shortcut : tpb - name : kickass engine : kickass diff --git a/searx/tests/engines/test_bing.py b/searx/tests/engines/test_bing.py @@ -0,0 +1,90 @@ +from collections import defaultdict +import mock +from searx.engines import bing +from searx.testing import SearxTestCase + + +class TestBingEngine(SearxTestCase): + + def test_request(self): + query = 'test_query' + dicto = defaultdict(dict) + dicto['pageno'] = 0 + dicto['language'] = 'fr_FR' + params = bing.request(query, dicto) + self.assertTrue('url' in params) + self.assertTrue(query in params['url']) + self.assertTrue('bing.com' in params['url']) + self.assertTrue('SRCHHPGUSR' in params['cookies']) + self.assertTrue('fr' in params['cookies']['SRCHHPGUSR']) + + dicto['language'] = 'all' + params = bing.request(query, dicto) + self.assertTrue('SRCHHPGUSR' in params['cookies']) + self.assertTrue('en' in params['cookies']['SRCHHPGUSR']) + + def test_response(self): + self.assertRaises(AttributeError, bing.response, None) + self.assertRaises(AttributeError, bing.response, []) + self.assertRaises(AttributeError, bing.response, '') + self.assertRaises(AttributeError, bing.response, '[]') + + response = mock.Mock(content='<html></html>') + self.assertEqual(bing.response(response), []) + + response = mock.Mock(content='<html></html>') + self.assertEqual(bing.response(response), []) + + html = """ + <div class="sa_cc" u="0|5109|4755453613245655|UAGjXgIrPH5yh-o5oNHRx_3Zta87f_QO"> + <div Class="sa_mc"> + <div class="sb_tlst"> + <h3> + <a href="http://this.should.be.the.link/" h="ID=SERP,5124.1"> + <strong>This</strong> should be the title</a> + </h3> + </div> + <div class="sb_meta"><cite><strong>this</strong>.meta.com</cite> + <span class="c_tlbxTrg"> + <span class="c_tlbxH" H="BASE:CACHEDPAGEDEFAULT" K="SERP,5125.1"> + </span> + </span> + </div> + <p><strong>This</strong> should be the content.</p> + </div> + </div> + """ + response = mock.Mock(content=html) + results = bing.response(response) + self.assertEqual(type(results), list) + self.assertEqual(len(results), 1) + self.assertEqual(results[0]['title'], 'This should be the title') + self.assertEqual(results[0]['url'], 'http://this.should.be.the.link/') + self.assertEqual(results[0]['content'], 'This should be the content.') + + html = """ + <li class="b_algo" u="0|5109|4755453613245655|UAGjXgIrPH5yh-o5oNHRx_3Zta87f_QO"> + <div Class="sa_mc"> + <div class="sb_tlst"> + <h2> + <a href="http://this.should.be.the.link/" h="ID=SERP,5124.1"> + <strong>This</strong> should be the title</a> + </h2> + </div> + <div class="sb_meta"><cite><strong>this</strong>.meta.com</cite> + <span class="c_tlbxTrg"> + <span class="c_tlbxH" H="BASE:CACHEDPAGEDEFAULT" K="SERP,5125.1"> + </span> + </span> + </div> + <p><strong>This</strong> should be the content.</p> + </div> + </li> + """ + response = mock.Mock(content=html) + results = bing.response(response) + self.assertEqual(type(results), list) + self.assertEqual(len(results), 1) + self.assertEqual(results[0]['title'], 'This should be the title') + self.assertEqual(results[0]['url'], 'http://this.should.be.the.link/') + self.assertEqual(results[0]['content'], 'This should be the content.') diff --git a/searx/tests/engines/test_bing_images.py b/searx/tests/engines/test_bing_images.py @@ -0,0 +1,268 @@ +# -*- coding: utf-8 -*- +from collections import defaultdict +import mock +from searx.engines import bing_images +from searx.testing import SearxTestCase + + +class TestBingImagesEngine(SearxTestCase): + + def test_request(self): + query = 'test_query' + dicto = defaultdict(dict) + dicto['pageno'] = 1 + dicto['language'] = 'fr_FR' + params = bing_images.request(query, dicto) + self.assertTrue('url' in params) + self.assertTrue(query in params['url']) + self.assertTrue('bing.com' in params['url']) + self.assertTrue('SRCHHPGUSR' in params['cookies']) + self.assertTrue('fr' in params['cookies']['SRCHHPGUSR']) + + dicto['language'] = 'all' + params = bing_images.request(query, dicto) + self.assertIn('SRCHHPGUSR', params['cookies']) + self.assertIn('en', params['cookies']['SRCHHPGUSR']) + + def test_response(self): + self.assertRaises(AttributeError, bing_images.response, None) + self.assertRaises(AttributeError, bing_images.response, []) + self.assertRaises(AttributeError, bing_images.response, '') + self.assertRaises(AttributeError, bing_images.response, '[]') + + response = mock.Mock(content='<html></html>') + self.assertEqual(bing_images.response(response), []) + + response = mock.Mock(content='<html></html>') + self.assertEqual(bing_images.response(response), []) + + html = """ + <div class="dg_u" style="width:178px;height:144px;left:17px;top:0px"> + <a href="#" ihk="HN.608003696942779811" + m="{ns:&quot;images&quot;,k:&quot;5045&quot;, +mid:&quot;659EB92C317974F34517A1CCAEBEF76A578E08DEE&quot;, +surl:&quot;http://www.page.url/&quot;,imgurl:&quot;http://test.url/Test%20Query.jpg&quot;, +oh:&quot;238&quot;,tft:&quot;0&quot;,oi:&quot;http://www.image.url/Images/Test%20Query.jpg&quot;}" + mid="59EB92C317974F34517A1CCAEBEF76A578E08DEE" onclick="return false;" + t1="Test Query" t2="650 x 517 · 31 kB · jpeg" t3="www.short.url" h="ID=images,5045.1"> + <img src="https://tse4.mm.bing.net/th?id=HN.608003696942779811&amp;o=4&amp;pid=1.7" + style="height:144px;" width="178" height="144"/> + </a> + </div> + """ + html = html.replace('\r\n', '').replace('\n', '').replace('\r', '') + response = mock.Mock(content=html) + results = bing_images.response(response) + self.assertEqual(type(results), list) + self.assertEqual(len(results), 1) + self.assertEqual(results[0]['title'], 'Test Query') + self.assertEqual(results[0]['url'], 'http://www.page.url/') + self.assertEqual(results[0]['content'], '') + self.assertEqual(results[0]['thumbnail_src'], 'http://ts1.mm.bing.net/th?id=HN.608003696942779811') + self.assertEqual(results[0]['img_src'], 'http://test.url/Test%20Query.jpg') + + html = """ + <a href="#" ihk="HN.608003696942779811" + m="{ns:&quot;images&quot;,k:&quot;5045&quot;, + mid:&quot;59EB92C317974F34517A1CCAEBEF76A578E08DEE&quot;, + surl:&quot;http://www.page.url/&quot;, + imgurl:&quot;http://test.url/Test%20Query.jpg&quot;,oh:&quot;238&quot;, + tft:&quot;0&quot;,oi:&quot;http://www.image.url/Images/Test%20Query.jpg&quot;}" + mid="59EB92C317974F34517A1CCAEBEF76A578E08DEE" onclick="return false;" + t1="Test Query" t2="650 x 517 · 31 kB · jpeg" t3="www.short.url" h="ID=images,5045.1"> + <img src="https://tse4.mm.bing.net/th?id=HN.608003696942779811&amp;o=4&amp;pid=1.7" + style="height:144px;" width="178" height="144"/> + </a> + """ + response = mock.Mock(content=html) + results = bing_images.response(response) + self.assertEqual(type(results), list) + self.assertEqual(len(results), 0) + + html = """ + <div class="dg_u" style="width:178px;height:144px;left:17px;top:0px"> + <a href="#" ihk="HN.608003696942779811" + m="{ns:&quot;images&quot;,k:&quot;5045&quot;, +mid:&quot;659EB92C317974F34517A1CCAEBEF76A578E08DEE&quot;, +surl:&quot;http://www.page.url/&quot;,imgurl:&quot;http://test.url/Test%20Query.jpg&quot;, +oh:&quot;238&quot;,tft:&quot;0&quot;,oi:&quot;http://www.image.url/Images/Test%20Query.jpg&quot;}" + mid="59EB92C317974F34517A1CCAEBEF76A578E08DEE" onclick="return false;" + t1="Test Query" t2="650 x 517 · 31 kB · jpeg" t3="www.short.url" h="ID=images,5045.1"> + <img src="https://tse4.mm.bing.net/th?id=HN.608003696942779811&amp;o=4&amp;pid=1.7" + style="height:144px;" width="178" height="144"/> + </a> + </div> + <div class="dg_u" style="width:178px;height:144px;left:17px;top:0px"> + <a href="#" ihk="HN.608003696942779811" + m="{ns:&quot;images&quot;,k:&quot;5045&quot;, +mid:&quot;659EB92C317974F34517A1CCAEBEF76A578E08DEE&quot;, +surl:&quot;http://www.page.url/&quot;,imgurl:&quot;http://test.url/Test%20Query.jpg&quot;, +oh:&quot;238&quot;,tft:&quot;0&quot;,oi:&quot;http://www.image.url/Images/Test%20Query.jpg&quot;}" + mid="59EB92C317974F34517A1CCAEBEF76A578E08DEE" onclick="return false;" + t1="Test Query" t2="650 x 517 · 31 kB · jpeg" t3="www.short.url" h="ID=images,5045.1"> + <img src="https://tse4.mm.bing.net/th?id=HN.608003696942779811&amp;o=4&amp;pid=1.7" + style="height:144px;" width="178" height="144"/> + </a> + </div> + <div class="dg_u" style="width:178px;height:144px;left:17px;top:0px"> + <a href="#" ihk="HN.608003696942779811" + m="{ns:&quot;images&quot;,k:&quot;5045&quot;, +mid:&quot;659EB92C317974F34517A1CCAEBEF76A578E08DEE&quot;, +surl:&quot;http://www.page.url/&quot;,imgurl:&quot;http://test.url/Test%20Query.jpg&quot;, +oh:&quot;238&quot;,tft:&quot;0&quot;,oi:&quot;http://www.image.url/Images/Test%20Query.jpg&quot;}" + mid="59EB92C317974F34517A1CCAEBEF76A578E08DEE" onclick="return false;" + t1="Test Query" t2="650 x 517 · 31 kB · jpeg" t3="www.short.url" h="ID=images,5045.1"> + <img src="https://tse4.mm.bing.net/th?id=HN.608003696942779811&amp;o=4&amp;pid=1.7" + style="height:144px;" width="178" height="144"/> + </a> + </div> + <div class="dg_u" style="width:178px;height:144px;left:17px;top:0px"> + <a href="#" ihk="HN.608003696942779811" + m="{ns:&quot;images&quot;,k:&quot;5045&quot;, +mid:&quot;659EB92C317974F34517A1CCAEBEF76A578E08DEE&quot;, +surl:&quot;http://www.page.url/&quot;,imgurl:&quot;http://test.url/Test%20Query.jpg&quot;, +oh:&quot;238&quot;,tft:&quot;0&quot;,oi:&quot;http://www.image.url/Images/Test%20Query.jpg&quot;}" + mid="59EB92C317974F34517A1CCAEBEF76A578E08DEE" onclick="return false;" + t1="Test Query" t2="650 x 517 · 31 kB · jpeg" t3="www.short.url" h="ID=images,5045.1"> + <img src="https://tse4.mm.bing.net/th?id=HN.608003696942779811&amp;o=4&amp;pid=1.7" + style="height:144px;" width="178" height="144"/> + </a> + </div> + <div class="dg_u" style="width:178px;height:144px;left:17px;top:0px"> + <a href="#" ihk="HN.608003696942779811" + m="{ns:&quot;images&quot;,k:&quot;5045&quot;, +mid:&quot;659EB92C317974F34517A1CCAEBEF76A578E08DEE&quot;, +surl:&quot;http://www.page.url/&quot;,imgurl:&quot;http://test.url/Test%20Query.jpg&quot;, +oh:&quot;238&quot;,tft:&quot;0&quot;,oi:&quot;http://www.image.url/Images/Test%20Query.jpg&quot;}" + mid="59EB92C317974F34517A1CCAEBEF76A578E08DEE" onclick="return false;" + t1="Test Query" t2="650 x 517 · 31 kB · jpeg" t3="www.short.url" h="ID=images,5045.1"> + <img src="https://tse4.mm.bing.net/th?id=HN.608003696942779811&amp;o=4&amp;pid=1.7" + style="height:144px;" width="178" height="144"/> + </a> + </div> + <div class="dg_u" style="width:178px;height:144px;left:17px;top:0px"> + <a href="#" ihk="HN.608003696942779811" + m="{ns:&quot;images&quot;,k:&quot;5045&quot;, +mid:&quot;659EB92C317974F34517A1CCAEBEF76A578E08DEE&quot;, +surl:&quot;http://www.page.url/&quot;,imgurl:&quot;http://test.url/Test%20Query.jpg&quot;, +oh:&quot;238&quot;,tft:&quot;0&quot;,oi:&quot;http://www.image.url/Images/Test%20Query.jpg&quot;}" + mid="59EB92C317974F34517A1CCAEBEF76A578E08DEE" onclick="return false;" + t1="Test Query" t2="650 x 517 · 31 kB · jpeg" t3="www.short.url" h="ID=images,5045.1"> + <img src="https://tse4.mm.bing.net/th?id=HN.608003696942779811&amp;o=4&amp;pid=1.7" + style="height:144px;" width="178" height="144"/> + </a> + </div> + <div class="dg_u" style="width:178px;height:144px;left:17px;top:0px"> + <a href="#" ihk="HN.608003696942779811" + m="{ns:&quot;images&quot;,k:&quot;5045&quot;, +mid:&quot;659EB92C317974F34517A1CCAEBEF76A578E08DEE&quot;, +surl:&quot;http://www.page.url/&quot;,imgurl:&quot;http://test.url/Test%20Query.jpg&quot;, +oh:&quot;238&quot;,tft:&quot;0&quot;,oi:&quot;http://www.image.url/Images/Test%20Query.jpg&quot;}" + mid="59EB92C317974F34517A1CCAEBEF76A578E08DEE" onclick="return false;" + t1="Test Query" t2="650 x 517 · 31 kB · jpeg" t3="www.short.url" h="ID=images,5045.1"> + <img src="https://tse4.mm.bing.net/th?id=HN.608003696942779811&amp;o=4&amp;pid=1.7" + style="height:144px;" width="178" height="144"/> + </a> + </div> + <div class="dg_u" style="width:178px;height:144px;left:17px;top:0px"> + <a href="#" ihk="HN.608003696942779811" + m="{ns:&quot;images&quot;,k:&quot;5045&quot;, +mid:&quot;659EB92C317974F34517A1CCAEBEF76A578E08DEE&quot;, +surl:&quot;http://www.page.url/&quot;,imgurl:&quot;http://test.url/Test%20Query.jpg&quot;, +oh:&quot;238&quot;,tft:&quot;0&quot;,oi:&quot;http://www.image.url/Images/Test%20Query.jpg&quot;}" + mid="59EB92C317974F34517A1CCAEBEF76A578E08DEE" onclick="return false;" + t1="Test Query" t2="650 x 517 · 31 kB · jpeg" t3="www.short.url" h="ID=images,5045.1"> + <img src="https://tse4.mm.bing.net/th?id=HN.608003696942779811&amp;o=4&amp;pid=1.7" + style="height:144px;" width="178" height="144"/> + </a> + </div> + <div class="dg_u" style="width:178px;height:144px;left:17px;top:0px"> + <a href="#" ihk="HN.608003696942779811" + m="{ns:&quot;images&quot;,k:&quot;5045&quot;, +mid:&quot;659EB92C317974F34517A1CCAEBEF76A578E08DEE&quot;, +surl:&quot;http://www.page.url/&quot;,imgurl:&quot;http://test.url/Test%20Query.jpg&quot;, +oh:&quot;238&quot;,tft:&quot;0&quot;,oi:&quot;http://www.image.url/Images/Test%20Query.jpg&quot;}" + mid="59EB92C317974F34517A1CCAEBEF76A578E08DEE" onclick="return false;" + t1="Test Query" t2="650 x 517 · 31 kB · jpeg" t3="www.short.url" h="ID=images,5045.1"> + <img src="https://tse4.mm.bing.net/th?id=HN.608003696942779811&amp;o=4&amp;pid=1.7" + style="height:144px;" width="178" height="144"/> + </a> + </div> + <div class="dg_u" style="width:178px;height:144px;left:17px;top:0px"> + <a href="#" ihk="HN.608003696942779811" + m="{ns:&quot;images&quot;,k:&quot;5045&quot;, +mid:&quot;659EB92C317974F34517A1CCAEBEF76A578E08DEE&quot;, +surl:&quot;http://www.page.url/&quot;,imgurl:&quot;http://test.url/Test%20Query.jpg&quot;, +oh:&quot;238&quot;,tft:&quot;0&quot;,oi:&quot;http://www.image.url/Images/Test%20Query.jpg&quot;}" + mid="59EB92C317974F34517A1CCAEBEF76A578E08DEE" onclick="return false;" + t1="Test Query" t2="650 x 517 · 31 kB · jpeg" t3="www.short.url" h="ID=images,5045.1"> + <img src="https://tse4.mm.bing.net/th?id=HN.608003696942779811&amp;o=4&amp;pid=1.7" + style="height:144px;" width="178" height="144"/> + </a> + </div> + <div class="dg_u" style="width:178px;height:144px;left:17px;top:0px"> + <a href="#" ihk="HN.608003696942779811" + m="{ns:&quot;images&quot;,k:&quot;5045&quot;, +mid:&quot;659EB92C317974F34517A1CCAEBEF76A578E08DEE&quot;, +surl:&quot;http://www.page.url/&quot;,imgurl:&quot;http://test.url/Test%20Query.jpg&quot;, +oh:&quot;238&quot;,tft:&quot;0&quot;,oi:&quot;http://www.image.url/Images/Test%20Query.jpg&quot;}" + mid="59EB92C317974F34517A1CCAEBEF76A578E08DEE" onclick="return false;" + t1="Test Query" t2="650 x 517 · 31 kB · jpeg" t3="www.short.url" h="ID=images,5045.1"> + <img src="https://tse4.mm.bing.net/th?id=HN.608003696942779811&amp;o=4&amp;pid=1.7" + style="height:144px;" width="178" height="144"/> + </a> + </div> + <div class="dg_u" style="width:178px;height:144px;left:17px;top:0px"> + <a href="#" ihk="HN.608003696942779811" + m="{ns:&quot;images&quot;,k:&quot;5045&quot;, +mid:&quot;659EB92C317974F34517A1CCAEBEF76A578E08DEE&quot;, +surl:&quot;http://www.page.url/&quot;,imgurl:&quot;http://test.url/Test%20Query.jpg&quot;, +oh:&quot;238&quot;,tft:&quot;0&quot;,oi:&quot;http://www.image.url/Images/Test%20Query.jpg&quot;}" + mid="59EB92C317974F34517A1CCAEBEF76A578E08DEE" onclick="return false;" + t1="Test Query" t2="650 x 517 · 31 kB · jpeg" t3="www.short.url" h="ID=images,5045.1"> + <img src="https://tse4.mm.bing.net/th?id=HN.608003696942779811&amp;o=4&amp;pid=1.7" + style="height:144px;" width="178" height="144"/> + </a> + </div> + <div class="dg_u" style="width:178px;height:144px;left:17px;top:0px"> + <a href="#" ihk="HN.608003696942779811" + m="{ns:&quot;images&quot;,k:&quot;5045&quot;, +mid:&quot;659EB92C317974F34517A1CCAEBEF76A578E08DEE&quot;, +surl:&quot;http://www.page.url/&quot;,imgurl:&quot;http://test.url/Test%20Query.jpg&quot;, +oh:&quot;238&quot;,tft:&quot;0&quot;,oi:&quot;http://www.image.url/Images/Test%20Query.jpg&quot;}" + mid="59EB92C317974F34517A1CCAEBEF76A578E08DEE" onclick="return false;" + t1="Test Query" t2="650 x 517 · 31 kB · jpeg" t3="www.short.url" h="ID=images,5045.1"> + <img src="https://tse4.mm.bing.net/th?id=HN.608003696942779811&amp;o=4&amp;pid=1.7" + style="height:144px;" width="178" height="144"/> + </a> + </div> + <div class="dg_u" style="width:178px;height:144px;left:17px;top:0px"> + <a href="#" ihk="HN.608003696942779811" + m="{ns:&quot;images&quot;,k:&quot;5045&quot;, +mid:&quot;659EB92C317974F34517A1CCAEBEF76A578E08DEE&quot;, +surl:&quot;http://www.page.url/&quot;,imgurl:&quot;http://test.url/Test%20Query.jpg&quot;, +oh:&quot;238&quot;,tft:&quot;0&quot;,oi:&quot;http://www.image.url/Images/Test%20Query.jpg&quot;}" + mid="59EB92C317974F34517A1CCAEBEF76A578E08DEE" onclick="return false;" + t1="Test Query" t2="650 x 517 · 31 kB · jpeg" t3="www.short.url" h="ID=images,5045.1"> + <img src="https://tse4.mm.bing.net/th?id=HN.608003696942779811&amp;o=4&amp;pid=1.7" + style="height:144px;" width="178" height="144"/> + </a> + </div> + <div class="dg_u" style="width:178px;height:144px;left:17px;top:0px"> + <a href="#" ihk="HN.608003696942779811" + m="{ns:&quot;images&quot;,k:&quot;5045&quot;, +mid:&quot;659EB92C317974F34517A1CCAEBEF76A578E08DEE&quot;, +surl:&quot;http://www.page.url/&quot;,imgurl:&quot;http://test.url/Test%20Query.jpg&quot;, +oh:&quot;238&quot;,tft:&quot;0&quot;,oi:&quot;http://www.image.url/Images/Test%20Query.jpg&quot;}" + mid="59EB92C317974F34517A1CCAEBEF76A578E08DEE" onclick="return false;" + t1="Test Query" t2="650 x 517 · 31 kB · jpeg" t3="www.short.url" h="ID=images,5045.1"> + <img src="https://tse4.mm.bing.net/th?id=HN.608003696942779811&amp;o=4&amp;pid=1.7" + style="height:144px;" width="178" height="144"/> + </a> + </div> + """ + html = html.replace('\r\n', '').replace('\n', '').replace('\r', '') + response = mock.Mock(content=html) + results = bing_images.response(response) + self.assertEqual(type(results), list) + self.assertEqual(len(results), 10) diff --git a/searx/tests/engines/test_bing_news.py b/searx/tests/engines/test_bing_news.py @@ -0,0 +1,236 @@ +from collections import defaultdict +import mock +from searx.engines import bing_news +from searx.testing import SearxTestCase + + +class TestBingNewsEngine(SearxTestCase): + + def test_request(self): + query = 'test_query' + dicto = defaultdict(dict) + dicto['pageno'] = 1 + dicto['language'] = 'fr_FR' + params = bing_news.request(query, dicto) + self.assertIn('url', params) + self.assertIn(query, params['url']) + self.assertIn('bing.com', params['url']) + self.assertIn('fr', params['url']) + self.assertIn('_FP', params['cookies']) + self.assertIn('en', params['cookies']['_FP']) + + dicto['language'] = 'all' + params = bing_news.request(query, dicto) + self.assertIn('en', params['url']) + self.assertIn('_FP', params['cookies']) + self.assertIn('en', params['cookies']['_FP']) + + def test_response(self): + self.assertRaises(AttributeError, bing_news.response, None) + self.assertRaises(AttributeError, bing_news.response, []) + self.assertRaises(AttributeError, bing_news.response, '') + self.assertRaises(AttributeError, bing_news.response, '[]') + + response = mock.Mock(content='<html></html>') + self.assertEqual(bing_news.response(response), []) + + response = mock.Mock(content='<html></html>') + self.assertEqual(bing_news.response(response), []) + + html = """ + <div class="sn_r"> + <div class="newstitle"> + <a href="http://url.of.article/" target="_blank" h="ID=news,5022.1"> + Title + </a> + </div> + <div class="sn_img"> + <a href="http://url.of.article2/" target="_blank" h="ID=news,5024.1"> + <img class="rms_img" height="80" id="emb1" src="/image.src" title="Title" width="80" /> + </a> + </div> + <div class="sn_txt"> + <div class="sn_oi"> + <span class="sn_snip">Article Content</span> + <span class="sn_ST"> + <cite class="sn_src">metronews.fr</cite> + &nbsp;&#0183;&#32; + <span class="sn_tm">44 minutes ago</span> + </span> + </div> + </div> + </div> + """ + response = mock.Mock(content=html) + results = bing_news.response(response) + self.assertEqual(type(results), list) + self.assertEqual(len(results), 1) + self.assertEqual(results[0]['title'], 'Title') + self.assertEqual(results[0]['url'], 'http://url.of.article/') + self.assertEqual(results[0]['content'], 'Article Content') + + html = """ + <div class="sn_r"> + <div class="newstitle"> + <a href="http://url.of.article/" target="_blank" h="ID=news,5022.1"> + Title + </a> + </div> + <div class="sn_img"> + <a href="http://url.of.article2/" target="_blank" h="ID=news,5024.1"> + <img class="rms_img" height="80" id="emb1" src="/image.src" title="Title" width="80" /> + </a> + </div> + <div class="sn_txt"> + <div class="sn_oi"> + <span class="sn_snip">Article Content</span> + <span class="sn_ST"> + <cite class="sn_src">metronews.fr</cite> + &nbsp;&#0183;&#32; + <span class="sn_tm">44 minutes ago</span> + </span> + </div> + </div> + </div> + <div class="sn_r"> + <div class="newstitle"> + <a href="http://url.of.article/" target="_blank" h="ID=news,5022.1"> + Title + </a> + </div> + <div class="sn_img"> + <a href="http://url.of.article2/" target="_blank" h="ID=news,5024.1"> + <img class="rms_img" height="80" id="emb1" src="/image.src" title="Title" width="80" /> + </a> + </div> + <div class="sn_txt"> + <div class="sn_oi"> + <span class="sn_snip">Article Content</span> + <span class="sn_ST"> + <cite class="sn_src">metronews.fr</cite> + &nbsp;&#0183;&#32; + <span class="sn_tm">3 hours, 44 minutes ago</span> + </span> + </div> + </div> + </div> + <div class="sn_r"> + <div class="newstitle"> + <a href="http://url.of.article/" target="_blank" h="ID=news,5022.1"> + Title + </a> + </div> + <div class="sn_img"> + <a href="http://url.of.article2/" target="_blank" h="ID=news,5024.1"> + <img class="rms_img" height="80" id="emb1" src="/image.src" title="Title" width="80" /> + </a> + </div> + <div class="sn_txt"> + <div class="sn_oi"> + <span class="sn_snip">Article Content</span> + <span class="sn_ST"> + <cite class="sn_src">metronews.fr</cite> + &nbsp;&#0183;&#32; + <span class="sn_tm">44 hours ago</span> + </span> + </div> + </div> + </div> + <div class="sn_r"> + <div class="newstitle"> + <a href="http://url.of.article/" target="_blank" h="ID=news,5022.1"> + Title + </a> + </div> + <div class="sn_img"> + <a href="http://url.of.article2/" target="_blank" h="ID=news,5024.1"> + <img class="rms_img" height="80" id="emb1" src="/image.src" title="Title" width="80" /> + </a> + </div> + <div class="sn_txt"> + <div class="sn_oi"> + <span class="sn_snip">Article Content</span> + <span class="sn_ST"> + <cite class="sn_src">metronews.fr</cite> + &nbsp;&#0183;&#32; + <span class="sn_tm">2 days ago</span> + </span> + </div> + </div> + </div> + <div class="sn_r"> + <div class="newstitle"> + <a href="http://url.of.article/" target="_blank" h="ID=news,5022.1"> + Title + </a> + </div> + <div class="sn_img"> + <a href="http://url.of.article2/" target="_blank" h="ID=news,5024.1"> + <img class="rms_img" height="80" id="emb1" src="/image.src" title="Title" width="80" /> + </a> + </div> + <div class="sn_txt"> + <div class="sn_oi"> + <span class="sn_snip">Article Content</span> + <span class="sn_ST"> + <cite class="sn_src">metronews.fr</cite> + &nbsp;&#0183;&#32; + <span class="sn_tm">27/01/2015</span> + </span> + </div> + </div> + </div> + <div class="sn_r"> + <div class="newstitle"> + <a href="http://url.of.article/" target="_blank" h="ID=news,5022.1"> + Title + </a> + </div> + <div class="sn_img"> + <a href="http://url.of.article2/" target="_blank" h="ID=news,5024.1"> + <img class="rms_img" height="80" id="emb1" src="/image.src" title="Title" width="80" /> + </a> + </div> + <div class="sn_txt"> + <div class="sn_oi"> + <span class="sn_snip">Article Content</span> + <span class="sn_ST"> + <cite class="sn_src">metronews.fr</cite> + &nbsp;&#0183;&#32; + <span class="sn_tm">Il y a 3 heures</span> + </span> + </div> + </div> + </div> + """ + response = mock.Mock(content=html) + results = bing_news.response(response) + self.assertEqual(type(results), list) + self.assertEqual(len(results), 6) + + html = """ + <div class="newstitle"> + <a href="http://url.of.article/" target="_blank" h="ID=news,5022.1"> + Title + </a> + </div> + <div class="sn_img"> + <a href="http://url.of.article2/" target="_blank" h="ID=news,5024.1"> + <img class="rms_img" height="80" id="emb1" src="/image.src" title="Title" width="80" /> + </a> + </div> + <div class="sn_txt"> + <div class="sn_oi"> + <span class="sn_snip">Article Content</span> + <span class="sn_ST"> + <cite class="sn_src">metronews.fr</cite> + &nbsp;&#0183;&#32; + <span class="sn_tm">44 minutes ago</span> + </span> + </div> + </div> + """ + response = mock.Mock(content=html) + results = bing_news.response(response) + self.assertEqual(type(results), list) + self.assertEqual(len(results), 0) diff --git a/searx/tests/engines/test_btdigg.py b/searx/tests/engines/test_btdigg.py @@ -0,0 +1,384 @@ +# -*- coding: utf-8 -*- +from collections import defaultdict +import mock +from searx.engines import btdigg +from searx.testing import SearxTestCase + + +class TestBtdiggEngine(SearxTestCase): + + def test_request(self): + query = 'test_query' + dicto = defaultdict(dict) + dicto['pageno'] = 0 + params = btdigg.request(query, dicto) + self.assertIn('url', params) + self.assertIn(query, params['url']) + self.assertIn('btdigg.org', params['url']) + + def test_response(self): + self.assertRaises(AttributeError, btdigg.response, None) + self.assertRaises(AttributeError, btdigg.response, []) + self.assertRaises(AttributeError, btdigg.response, '') + self.assertRaises(AttributeError, btdigg.response, '[]') + + response = mock.Mock(text='<html></html>') + self.assertEqual(btdigg.response(response), []) + + html = """ + <div id="search_res"> + <table> + <tr> + <td class="idx">1</td> + <td> + <table class="torrent_name_tbl"> + <tr> + <td class="torrent_name"> + <a href="/url">Should be the title</a> + </td> + </tr> + </table> + <table class="torrent_name_tbl"> + <tr> + <td class="ttth"> + <a onclick="fclck(this.href)" href="magnet:?xt=urn:btih:magnet&amp;dn=Test" + title="Télécharger des liens Magnet">[magnet]</a> + </td> + <td class="ttth"> + <a href="https://btcloud.io/manager?cmd=add&amp;info_hash=hash" + target="_blank" title="Ajouter à BTCloud">[cloud]</a> + </td> + <td> + <span class="attr_name">Taille:</span> + <span class="attr_val">8 B</span> + </td> + <td> + <span class="attr_name">Fichiers:</span> + <span class="attr_val">710</span> + </td> + <td> + <span class="attr_name">Téléchargements:</span> + <span class="attr_val">5</span> + </td> + <td> + <span class="attr_name">Temps:</span> + <span class="attr_val">417.8&nbsp;jours</span> + </td> + <td> + <span class="attr_name">Dernière&nbsp;mise&nbsp;à&nbsp;jour:</span> + <span class="attr_val">5.3&nbsp;jours</span> + </td> + <td> + <span class="attr_name">Faux:</span> + <span class="attr_val">Aucun</span> + </td> + </tr> + </table> + <pre class="snippet"> + Content + </pre> + </td> + </tr> + </table> + </div> + """ + response = mock.Mock(text=html) + results = btdigg.response(response) + self.assertEqual(type(results), list) + self.assertEqual(len(results), 1) + self.assertEqual(results[0]['title'], 'Should be the title') + self.assertEqual(results[0]['url'], 'https://btdigg.org/url') + self.assertEqual(results[0]['content'], 'Content') + self.assertEqual(results[0]['seed'], 5) + self.assertEqual(results[0]['leech'], 0) + self.assertEqual(results[0]['filesize'], 8) + self.assertEqual(results[0]['files'], 710) + self.assertEqual(results[0]['magnetlink'], 'magnet:?xt=urn:btih:magnet&dn=Test') + + html = """ + <div id="search_res"> + <table> + </table> + </div> + """ + response = mock.Mock(text=html) + results = btdigg.response(response) + self.assertEqual(type(results), list) + self.assertEqual(len(results), 0) + + html = """ + <div id="search_res"> + <table> + <tr> + <td class="idx">1</td> + <td> + <table class="torrent_name_tbl"> + <tr> + <td class="torrent_name"> + <a href="/url">Should be the title</a> + </td> + </tr> + </table> + <table class="torrent_name_tbl"> + <tr> + <td class="ttth"> + <a onclick="fclck(this.href)" href="magnet:?xt=urn:btih:magnet&amp;dn=Test" + title="Télécharger des liens Magnet">[magnet]</a> + </td> + <td class="ttth"> + <a href="https://btcloud.io/manager?cmd=add&amp;info_hash=hash" + target="_blank" title="Ajouter à BTCloud">[cloud]</a> + </td> + <td> + <span class="attr_name">Taille:</span> + <span class="attr_val">1 KB</span> + </td> + <td> + <span class="attr_name">Fichiers:</span> + <span class="attr_val">710</span> + </td> + <td> + <span class="attr_name">Téléchargements:</span> + <span class="attr_val">5</span> + </td> + <td> + <span class="attr_name">Temps:</span> + <span class="attr_val">417.8&nbsp;jours</span> + </td> + <td> + <span class="attr_name">Dernière&nbsp;mise&nbsp;à&nbsp;jour:</span> + <span class="attr_val">5.3&nbsp;jours</span> + </td> + <td> + <span class="attr_name">Faux:</span> + <span class="attr_val">Aucun</span> + </td> + </tr> + </table> + <pre class="snippet"> + Content + </pre> + </td> + </tr> + <tr> + <td class="idx">1</td> + <td> + <table class="torrent_name_tbl"> + <tr> + <td class="torrent_name"> + <a href="/url">Should be the title</a> + </td> + </tr> + </table> + <table class="torrent_name_tbl"> + <tr> + <td class="ttth"> + <a onclick="fclck(this.href)" href="magnet:?xt=urn:btih:magnet&amp;dn=Test" + title="Télécharger des liens Magnet">[magnet]</a> + </td> + <td class="ttth"> + <a href="https://btcloud.io/manager?cmd=add&amp;info_hash=hash" + target="_blank" title="Ajouter à BTCloud">[cloud]</a> + </td> + <td> + <span class="attr_name">Taille:</span> + <span class="attr_val">1 MB</span> + </td> + <td> + <span class="attr_name">Fichiers:</span> + <span class="attr_val">a</span> + </td> + <td> + <span class="attr_name">Téléchargements:</span> + <span class="attr_val">4</span> + </td> + <td> + <span class="attr_name">Temps:</span> + <span class="attr_val">417.8&nbsp;jours</span> + </td> + <td> + <span class="attr_name">Dernière&nbsp;mise&nbsp;à&nbsp;jour:</span> + <span class="attr_val">5.3&nbsp;jours</span> + </td> + <td> + <span class="attr_name">Faux:</span> + <span class="attr_val">Aucun</span> + </td> + </tr> + </table> + <pre class="snippet"> + Content + </pre> + </td> + </tr> + <tr> + <td class="idx">1</td> + <td> + <table class="torrent_name_tbl"> + <tr> + <td class="torrent_name"> + <a href="/url">Should be the title</a> + </td> + </tr> + </table> + <table class="torrent_name_tbl"> + <tr> + <td class="ttth"> + <a onclick="fclck(this.href)" href="magnet:?xt=urn:btih:magnet&amp;dn=Test" + title="Télécharger des liens Magnet">[magnet]</a> + </td> + <td class="ttth"> + <a href="https://btcloud.io/manager?cmd=add&amp;info_hash=hash" + target="_blank" title="Ajouter à BTCloud">[cloud]</a> + </td> + <td> + <span class="attr_name">Taille:</span> + <span class="attr_val">1 GB</span> + </td> + <td> + <span class="attr_name">Fichiers:</span> + <span class="attr_val">710</span> + </td> + <td> + <span class="attr_name">Téléchargements:</span> + <span class="attr_val">3</span> + </td> + <td> + <span class="attr_name">Temps:</span> + <span class="attr_val">417.8&nbsp;jours</span> + </td> + <td> + <span class="attr_name">Dernière&nbsp;mise&nbsp;à&nbsp;jour:</span> + <span class="attr_val">5.3&nbsp;jours</span> + </td> + <td> + <span class="attr_name">Faux:</span> + <span class="attr_val">Aucun</span> + </td> + </tr> + </table> + <pre class="snippet"> + Content + </pre> + </td> + </tr> + <tr> + <td class="idx">1</td> + <td> + <table class="torrent_name_tbl"> + <tr> + <td class="torrent_name"> + <a href="/url">Should be the title</a> + </td> + </tr> + </table> + <table class="torrent_name_tbl"> + <tr> + <td class="ttth"> + <a onclick="fclck(this.href)" href="magnet:?xt=urn:btih:magnet&amp;dn=Test" + title="Télécharger des liens Magnet">[magnet]</a> + </td> + <td class="ttth"> + <a href="https://btcloud.io/manager?cmd=add&amp;info_hash=hash" + target="_blank" title="Ajouter à BTCloud">[cloud]</a> + </td> + <td> + <span class="attr_name">Taille:</span> + <span class="attr_val">1 TB</span> + </td> + <td> + <span class="attr_name">Fichiers:</span> + <span class="attr_val">710</span> + </td> + <td> + <span class="attr_name">Téléchargements:</span> + <span class="attr_val">2</span> + </td> + <td> + <span class="attr_name">Temps:</span> + <span class="attr_val">417.8&nbsp;jours</span> + </td> + <td> + <span class="attr_name">Dernière&nbsp;mise&nbsp;à&nbsp;jour:</span> + <span class="attr_val">5.3&nbsp;jours</span> + </td> + <td> + <span class="attr_name">Faux:</span> + <span class="attr_val">Aucun</span> + </td> + </tr> + </table> + <pre class="snippet"> + Content + </pre> + </td> + </tr> + <tr> + <td class="idx">1</td> + <td> + <table class="torrent_name_tbl"> + <tr> + <td class="torrent_name"> + <a href="/url">Should be the title</a> + </td> + </tr> + </table> + <table class="torrent_name_tbl"> + <tr> + <td class="ttth"> + <a onclick="fclck(this.href)" href="magnet:?xt=urn:btih:magnet&amp;dn=Test" + title="Télécharger des liens Magnet">[magnet]</a> + </td> + <td class="ttth"> + <a href="https://btcloud.io/manager?cmd=add&amp;info_hash=hash" + target="_blank" title="Ajouter à BTCloud">[cloud]</a> + </td> + <td> + <span class="attr_name">Taille:</span> + <span class="attr_val">a TB</span> + </td> + <td> + <span class="attr_name">Fichiers:</span> + <span class="attr_val">710</span> + </td> + <td> + <span class="attr_name">Téléchargements:</span> + <span class="attr_val">z</span> + </td> + <td> + <span class="attr_name">Temps:</span> + <span class="attr_val">417.8&nbsp;jours</span> + </td> + <td> + <span class="attr_name">Dernière&nbsp;mise&nbsp;à&nbsp;jour:</span> + <span class="attr_val">5.3&nbsp;jours</span> + </td> + <td> + <span class="attr_name">Faux:</span> + <span class="attr_val">Aucun</span> + </td> + </tr> + </table> + <pre class="snippet"> + Content + </pre> + </td> + </tr> + </table> + </div> + """ + response = mock.Mock(text=html) + results = btdigg.response(response) + self.assertEqual(type(results), list) + self.assertEqual(len(results), 5) + self.assertEqual(results[0]['title'], 'Should be the title') + self.assertEqual(results[0]['url'], 'https://btdigg.org/url') + self.assertEqual(results[0]['content'], 'Content') + self.assertEqual(results[0]['seed'], 5) + self.assertEqual(results[0]['leech'], 0) + self.assertEqual(results[0]['files'], 710) + self.assertEqual(results[0]['magnetlink'], 'magnet:?xt=urn:btih:magnet&dn=Test') + self.assertEqual(results[0]['filesize'], 1024) + self.assertEqual(results[1]['filesize'], 1048576) + self.assertEqual(results[2]['filesize'], 1073741824) + self.assertEqual(results[3]['filesize'], 1099511627776) diff --git a/searx/tests/engines/test_dailymotion.py b/searx/tests/engines/test_dailymotion.py @@ -0,0 +1,74 @@ +from collections import defaultdict +import mock +from searx.engines import dailymotion +from searx.testing import SearxTestCase + + +class TestDailymotionEngine(SearxTestCase): + + def test_request(self): + query = 'test_query' + dicto = defaultdict(dict) + dicto['pageno'] = 0 + dicto['language'] = 'fr_FR' + params = dailymotion.request(query, dicto) + self.assertTrue('url' in params) + self.assertTrue(query in params['url']) + self.assertTrue('dailymotion.com' in params['url']) + self.assertTrue('fr' in params['url']) + + dicto['language'] = 'all' + params = dailymotion.request(query, dicto) + self.assertTrue('en' in params['url']) + + def test_response(self): + self.assertRaises(AttributeError, dailymotion.response, None) + self.assertRaises(AttributeError, dailymotion.response, []) + self.assertRaises(AttributeError, dailymotion.response, '') + self.assertRaises(AttributeError, dailymotion.response, '[]') + + response = mock.Mock(text='{}') + self.assertEqual(dailymotion.response(response), []) + + response = mock.Mock(text='{"data": []}') + self.assertEqual(dailymotion.response(response), []) + + json = """ + { + "page": 1, + "limit": 5, + "explicit": false, + "total": 289487, + "has_more": true, + "list": [ + { + "created_time": 1422173451, + "title": "Title", + "description": "Description", + "duration": 81, + "url": "http://www.url", + "thumbnail_360_url": "http://thumbnail", + "id": "x2fit7q" + } + ] + } + """ + response = mock.Mock(text=json) + results = dailymotion.response(response) + self.assertEqual(type(results), list) + self.assertEqual(len(results), 1) + self.assertEqual(results[0]['title'], 'Title') + self.assertEqual(results[0]['url'], 'http://www.url') + self.assertEqual(results[0]['content'], 'Description') + self.assertIn('x2fit7q', results[0]['embedded']) + + json = """ + {"toto":[ + {"id":200,"name":"Artist Name", + "link":"http:\/\/www.dailymotion.com\/artist\/1217","type":"artist"} + ]} + """ + response = mock.Mock(text=json) + results = dailymotion.response(response) + self.assertEqual(type(results), list) + self.assertEqual(len(results), 0) diff --git a/searx/tests/engines/test_deezer.py b/searx/tests/engines/test_deezer.py @@ -0,0 +1,57 @@ +from collections import defaultdict +import mock +from searx.engines import deezer +from searx.testing import SearxTestCase + + +class TestDeezerEngine(SearxTestCase): + + def test_request(self): + query = 'test_query' + dicto = defaultdict(dict) + dicto['pageno'] = 0 + params = deezer.request(query, dicto) + self.assertTrue('url' in params) + self.assertTrue(query in params['url']) + self.assertTrue('deezer.com' in params['url']) + + def test_response(self): + self.assertRaises(AttributeError, deezer.response, None) + self.assertRaises(AttributeError, deezer.response, []) + self.assertRaises(AttributeError, deezer.response, '') + self.assertRaises(AttributeError, deezer.response, '[]') + + response = mock.Mock(text='{}') + self.assertEqual(deezer.response(response), []) + + response = mock.Mock(text='{"data": []}') + self.assertEqual(deezer.response(response), []) + + json = """ + {"data":[ + {"id":100, "title":"Title of track", + "link":"http:\/\/www.deezer.com\/track\/1094042","duration":232, + "artist":{"id":200,"name":"Artist Name", + "link":"http:\/\/www.deezer.com\/artist\/1217","type":"artist"}, + "album":{"id":118106,"title":"Album Title","type":"album"},"type":"track"} + ]} + """ + response = mock.Mock(text=json) + results = deezer.response(response) + self.assertEqual(type(results), list) + self.assertEqual(len(results), 1) + self.assertEqual(results[0]['title'], 'Title of track') + self.assertEqual(results[0]['url'], 'http://www.deezer.com/track/1094042') + self.assertEqual(results[0]['content'], 'Artist Name &bull; Album Title &bull; Title of track') + self.assertTrue('100' in results[0]['embedded']) + + json = """ + {"data":[ + {"id":200,"name":"Artist Name", + "link":"http:\/\/www.deezer.com\/artist\/1217","type":"artist"} + ]} + """ + response = mock.Mock(text=json) + results = deezer.response(response) + self.assertEqual(type(results), list) + self.assertEqual(len(results), 0) diff --git a/searx/tests/engines/test_deviantart.py b/searx/tests/engines/test_deviantart.py @@ -0,0 +1,118 @@ +from collections import defaultdict +import mock +from searx.engines import deviantart +from searx.testing import SearxTestCase + + +class TestDeviantartEngine(SearxTestCase): + + def test_request(self): + query = 'test_query' + dicto = defaultdict(dict) + dicto['pageno'] = 0 + params = deviantart.request(query, dicto) + self.assertTrue('url' in params) + self.assertTrue(query in params['url']) + self.assertTrue('deviantart.com' in params['url']) + + def test_response(self): + self.assertRaises(AttributeError, deviantart.response, None) + self.assertRaises(AttributeError, deviantart.response, []) + self.assertRaises(AttributeError, deviantart.response, '') + self.assertRaises(AttributeError, deviantart.response, '[]') + + response = mock.Mock(text='<html></html>') + self.assertEqual(deviantart.response(response), []) + + response = mock.Mock(status_code=302) + self.assertEqual(deviantart.response(response), []) + + html = """ + <div class="tt-a tt-fh tt-boxed" collect_rid="1:149167425" + usericon="http://a.deviantart.net/avatars/t/e/test-0.gif" userid="233301" + username="test-0" symbol="~" category="digitalart/animation"> + <span class="tt-w" style="width: auto; max-width: 277px;"> + <span class="tt-fh-tc" style="width: 202px;"> + <span class="tt-bb" style="width: 202px;"> + </span> + <span class="shadow"> + <a class="thumb" href="http://url.of.result/2nd.part.of.url" + title="Behoimi BE Animation Test by test-0, Jan 4, + 2010 in Digital Art &gt; Animation"> <i></i> + <img width="200" height="200" alt="Test" + src="http://url.of.thumbnail" data-src="http://th08.deviantart.net/test.jpg"> + </a> + </span> + <!-- ^TTT --> + </span> + <span class="details"> + <a href="http://test-0.deviantart.com/art/Test" class="t" + title="Behoimi BE Animation Test by test-0, Jan 4, 2010"> + <span class="tt-fh-oe">Title of image</span> </a> + <small> + <span class="category"> + <span class="age"> + 5 years ago + </span> + in <a title="Behoimi BE Animation Test by test-0, Jan 4, 2010" + href="http://www.deviantart.com/browse/all/digitalart/animation/">Animation</a> + </span> + <div class="commentcount"> + <a href="http://test-0.deviantart.com/art/Test#comments"> + <span class="iconcommentsstats"></span>9 Comments</a> + </div> + <a class="mlt-link" href="http://www.deviantart.com/morelikethis/149167425"> + <span class="mlt-icon"></span> <span class="mlt-text">More Like This</span> </a> + </span> + </small> <!-- TTT$ --> + </span> + </div> + """ + response = mock.Mock(text=html) + results = deviantart.response(response) + self.assertEqual(type(results), list) + self.assertEqual(len(results), 1) + self.assertEqual(results[0]['title'], 'Title of image') + self.assertEqual(results[0]['url'], 'http://url.of.result/2nd.part.of.url') + self.assertNotIn('content', results[0]) + self.assertEqual(results[0]['thumbnail_src'], 'http://url.of.thumbnail') + + html = """ + <span class="tt-fh-tc" style="width: 202px;"> + <span class="tt-bb" style="width: 202px;"> + </span> + <span class="shadow"> + <a class="thumb" href="http://url.of.result/2nd.part.of.url" + title="Behoimi BE Animation Test by test-0, Jan 4, + 2010 in Digital Art &gt; Animation"> <i></i> + <img width="200" height="200" alt="Test" + src="http://url.of.thumbnail" data-src="http://th08.deviantart.net/test.jpg"> + </a> + </span> + <!-- ^TTT --> + </span> + <span class="details"> + <a href="http://test-0.deviantart.com/art/Test" class="t" + title="Behoimi BE Animation Test by test-0, Jan 4, 2010"> + <span class="tt-fh-oe">Title of image</span> </a> + <small> + <span class="category"> + <span class="age"> + 5 years ago + </span> + in <a title="Behoimi BE Animation Test by test-0, Jan 4, 2010" + href="http://www.deviantart.com/browse/all/digitalart/animation/">Animation</a> + </span> + <div class="commentcount"> + <a href="http://test-0.deviantart.com/art/Test#comments"> + <span class="iconcommentsstats"></span>9 Comments</a> + </div> + <a class="mlt-link" href="http://www.deviantart.com/morelikethis/149167425"> + <span class="mlt-icon"></span> <span class="mlt-text">More Like This</span> </a> + </span> + </small> <!-- TTT$ --> + """ + response = mock.Mock(text=html) + results = deviantart.response(response) + self.assertEqual(type(results), list) + self.assertEqual(len(results), 0) diff --git a/searx/tests/engines/test_digg.py b/searx/tests/engines/test_digg.py @@ -0,0 +1,101 @@ +from collections import defaultdict +import mock +from searx.engines import digg +from searx.testing import SearxTestCase + + +class TestDiggEngine(SearxTestCase): + + def test_request(self): + query = 'test_query' + dicto = defaultdict(dict) + dicto['pageno'] = 1 + params = digg.request(query, dicto) + self.assertIn('url', params) + self.assertIn(query, params['url']) + self.assertIn('digg.com', params['url']) + + def test_response(self): + self.assertRaises(AttributeError, digg.response, None) + self.assertRaises(AttributeError, digg.response, []) + self.assertRaises(AttributeError, digg.response, '') + self.assertRaises(AttributeError, digg.response, '[]') + + response = mock.Mock(text='{}') + self.assertEqual(digg.response(response), []) + + response = mock.Mock(text='{"data": []}') + self.assertEqual(digg.response(response), []) + + json = """ + { + "status": "ok", + "num": 10, + "next_position": 20, + "html": "<article itemscope itemtype=\\"http://schema.org/Article\\" + class=\\"story-container digg-story-el hentry entry story-1sRANah col-1\\" + data-content-id=\\"1sRANah\\" data-contenturl=\\"http://url.of.link\\" + data-position=\\"0\\" data-diggs=\\"24\\" data-tweets=\\"69\\" + data-digg-score=\\"1190\\"> <div class=\\"story-image story-image-thumb\\"> + <a data-position=\\"0\\" data-content-id=\\"1sRANah\\" + class=\\"story-link\\" href=\\"http://www.thedailybeast.com/\\" + target=\\"_blank\\"><img class=\\"story-image-img\\" + src=\\"http://url.of.image.jpeg\\" width=\\"312\\" height=\\"170\\" + alt=\\"\\" /> </a> </div> <div class=\\"story-content\\"><header + class=\\"story-header\\"> <div itemprop=\\"alternativeHeadline\\" + class=\\"story-kicker\\" >Kicker</div> <h2 itemprop=\\"headline\\" + class=\\"story-title entry-title\\"><a class=\\"story-title-link story-link\\" + rel=\\"bookmark\\" itemprop=\\"url\\" href=\\"http://www.thedailybeast.com/\\" + target=\\"_blank\\">Title of article</h2> <div class=\\"story-meta\\"> + <div class=\\"story-score \\"> + <div class=\\"story-score-diggscore diggscore-1sRANah\\">1190</div> + <div class=\\"story-score-details\\"> <div class=\\"arrow\\"></div> + <ul class=\\"story-score-details-list\\"> <li + class=\\"story-score-detail story-score-diggs\\"><span + class=\\"label\\">Diggs:</span> <span class=\\"count diggs-1sRANah\\">24</span> + </li> <li class=\\"story-score-detail story-score-twitter\\"><span + class=\\"label\\">Tweets:</span> <span class=\\"count tweets-1sRANah\\">69</span> + </li> <li class=\\"story-score-detail story-score-facebook\\"><span + class=\\"label\\">Facebook Shares:</span> <span + class=\\"count fb_shares-1sRANah\\">1097</span></li> </ul> </div> </div> + <span class=\\"story-meta-item story-source\\"> <a + itemprop=\\"publisher copyrightHolder sourceOrganization provider\\" + class=\\"story-meta-item-link story-source-link\\" + href=\\"/source/thedailybeast.com\\">The Daily Beast </a> </span> + <span class=\\"story-meta-item story-tag first-tag\\"> <a + itemprop=\\"keywords\\" rel=\\"tag\\" + class=\\"story-meta-item-link story-tag-link\\" href=\\"/tag/news\\">News</a> + </span> <abbr class=\\"published story-meta-item story-timestamp\\" + title=\\"2014-10-18 14:53:45\\"> <time datetime=\\"2014-10-18 14:53:45\\">18 Oct 2014</time> + </abbr> </div> </header> </div> <ul class=\\"story-actions\\"> <li + class=\\"story-action story-action-digg btn-story-action-container\\"> + <a class=\\"target digg-1sRANah\\" href=\\"#\\">Digg</a></li> <li + class=\\"story-action story-action-save btn-story-action-container\\"> + <a class=\\"target save-1sRANah\\" href=\\"#\\">Save</a></li> <li + class=\\"story-action story-action-share\\"><a + class=\\"target share-facebook\\" href=\\"https://www.facebook.com/\\">Facebook</a></li> + <li class=\\"story-action story-action-share\\"><a class=\\"target share-twitter\\" + href=\\"https://twitter.com/\\">Twitter</a></li> </ul> </article>" + } + """ + json = json.replace('\r\n', '').replace('\n', '').replace('\r', '') + response = mock.Mock(text=json) + results = digg.response(response) + self.assertEqual(type(results), list) + self.assertEqual(len(results), 1) + self.assertEqual(results[0]['title'], 'Title of article') + self.assertEqual(results[0]['url'], 'http://url.of.link') + self.assertEqual(results[0]['thumbnail'], 'http://url.of.image.jpeg') + self.assertEqual(results[0]['content'], '') + + json = """ + { + "status": "error", + "num": 10, + "next_position": 20 + } + """ + response = mock.Mock(text=json) + results = digg.response(response) + self.assertEqual(type(results), list) + self.assertEqual(len(results), 0) diff --git a/searx/tests/engines/test_flickr.py b/searx/tests/engines/test_flickr.py @@ -0,0 +1,142 @@ +from collections import defaultdict +import mock +from searx.engines import flickr +from searx.testing import SearxTestCase + + +class TestFlickrEngine(SearxTestCase): + + def test_request(self): + query = 'test_query' + dicto = defaultdict(dict) + dicto['pageno'] = 0 + params = flickr.request(query, dicto) + self.assertTrue('url' in params) + self.assertTrue(query in params['url']) + self.assertTrue('flickr.com' in params['url']) + + def test_response(self): + self.assertRaises(AttributeError, flickr.response, None) + self.assertRaises(AttributeError, flickr.response, []) + self.assertRaises(AttributeError, flickr.response, '') + self.assertRaises(AttributeError, flickr.response, '[]') + + response = mock.Mock(text='{}') + self.assertEqual(flickr.response(response), []) + + response = mock.Mock(text='{"data": []}') + self.assertEqual(flickr.response(response), []) + + json = """ + { "photos": { "page": 1, "pages": "41001", "perpage": 100, "total": "4100032", + "photo": [ + { "id": "15751017054", "owner": "66847915@N08", + "secret": "69c22afc40", "server": "7285", "farm": 8, + "title": "Photo title", "ispublic": 1, + "isfriend": 0, "isfamily": 0, + "description": { "_content": "Description" }, + "ownername": "Owner", + "url_o": "https:\/\/farm8.staticflickr.com\/7285\/15751017054_9178e0f963_o.jpg", + "height_o": "2100", "width_o": "2653", + "url_n": "https:\/\/farm8.staticflickr.com\/7285\/15751017054_69c22afc40_n.jpg", + "height_n": "253", "width_n": "320", + "url_z": "https:\/\/farm8.staticflickr.com\/7285\/15751017054_69c22afc40_z.jpg", + "height_z": "507", "width_z": "640" } + ] }, "stat": "ok" } + """ + response = mock.Mock(text=json) + results = flickr.response(response) + self.assertEqual(type(results), list) + self.assertEqual(len(results), 1) + self.assertEqual(results[0]['title'], 'Photo title') + self.assertEqual(results[0]['url'], 'https://www.flickr.com/photos/66847915@N08/15751017054') + self.assertTrue('o.jpg' in results[0]['img_src']) + self.assertTrue('n.jpg' in results[0]['thumbnail_src']) + self.assertTrue('Owner' in results[0]['content']) + self.assertTrue('Description' in results[0]['content']) + + json = """ + { "photos": { "page": 1, "pages": "41001", "perpage": 100, "total": "4100032", + "photo": [ + { "id": "15751017054", "owner": "66847915@N08", + "secret": "69c22afc40", "server": "7285", "farm": 8, + "title": "Photo title", "ispublic": 1, + "isfriend": 0, "isfamily": 0, + "description": { "_content": "Description" }, + "ownername": "Owner", + "url_z": "https:\/\/farm8.staticflickr.com\/7285\/15751017054_69c22afc40_z.jpg", + "height_z": "507", "width_z": "640" } + ] }, "stat": "ok" } + """ + response = mock.Mock(text=json) + results = flickr.response(response) + self.assertEqual(type(results), list) + self.assertEqual(len(results), 1) + self.assertEqual(results[0]['title'], 'Photo title') + self.assertEqual(results[0]['url'], 'https://www.flickr.com/photos/66847915@N08/15751017054') + self.assertTrue('z.jpg' in results[0]['img_src']) + self.assertTrue('z.jpg' in results[0]['thumbnail_src']) + self.assertTrue('Owner' in results[0]['content']) + self.assertTrue('Description' in results[0]['content']) + + json = """ + { "photos": { "page": 1, "pages": "41001", "perpage": 100, "total": "4100032", + "photo": [ + { "id": "15751017054", "owner": "66847915@N08", + "secret": "69c22afc40", "server": "7285", "farm": 8, + "title": "Photo title", "ispublic": 1, + "isfriend": 0, "isfamily": 0, + "description": { "_content": "Description" }, + "ownername": "Owner", + "url_o": "https:\/\/farm8.staticflickr.com\/7285\/15751017054_9178e0f963_o.jpg", + "height_o": "2100", "width_o": "2653" } + ] }, "stat": "ok" } + """ + response = mock.Mock(text=json) + results = flickr.response(response) + self.assertEqual(type(results), list) + self.assertEqual(len(results), 1) + self.assertEqual(results[0]['title'], 'Photo title') + self.assertEqual(results[0]['url'], 'https://www.flickr.com/photos/66847915@N08/15751017054') + self.assertTrue('o.jpg' in results[0]['img_src']) + self.assertTrue('o.jpg' in results[0]['thumbnail_src']) + self.assertTrue('Owner' in results[0]['content']) + self.assertTrue('Description' in results[0]['content']) + + json = """ + { "photos": { "page": 1, "pages": "41001", "perpage": 100, "total": "4100032", + "photo": [ + { "id": "15751017054", "owner": "66847915@N08", + "secret": "69c22afc40", "server": "7285", "farm": 8, + "title": "Photo title", "ispublic": 1, + "isfriend": 0, "isfamily": 0, + "description": { "_content": "Description" }, + "ownername": "Owner", + "url_n": "https:\/\/farm8.staticflickr.com\/7285\/15751017054_69c22afc40_n.jpg", + "height_n": "253", "width_n": "320" } + ] }, "stat": "ok" } + """ + response = mock.Mock(text=json) + results = flickr.response(response) + self.assertEqual(type(results), list) + self.assertEqual(len(results), 0) + + json = """ + { "photos": { "page": 1, "pages": "41001", "perpage": 100, "total": "4100032", + "toto": [] }, "stat": "ok" } + """ + response = mock.Mock(text=json) + results = flickr.response(response) + self.assertEqual(type(results), list) + self.assertEqual(len(results), 0) + + json = """ + {"toto":[ + {"id":200,"name":"Artist Name", + "link":"http:\/\/www.flickr.com\/artist\/1217","type":"artist"} + ]} + """ + response = mock.Mock(text=json) + results = flickr.response(response) + self.assertEqual(type(results), list) + self.assertEqual(len(results), 0) diff --git a/searx/tests/engines/test_flickr_noapi.py b/searx/tests/engines/test_flickr_noapi.py @@ -0,0 +1,442 @@ +from collections import defaultdict +import mock +from searx.engines import flickr_noapi +from searx.testing import SearxTestCase + + +class TestFlickrNoapiEngine(SearxTestCase): + + def test_build_flickr_url(self): + url = flickr_noapi.build_flickr_url("uid", "pid") + self.assertIn("uid", url) + self.assertIn("pid", url) + + def test_request(self): + query = 'test_query' + dicto = defaultdict(dict) + dicto['pageno'] = 1 + params = flickr_noapi.request(query, dicto) + self.assertIn('url', params) + self.assertIn(query, params['url']) + self.assertIn('flickr.com', params['url']) + + def test_response(self): + self.assertRaises(AttributeError, flickr_noapi.response, None) + self.assertRaises(AttributeError, flickr_noapi.response, []) + self.assertRaises(AttributeError, flickr_noapi.response, '') + self.assertRaises(AttributeError, flickr_noapi.response, '[]') + + response = mock.Mock(text='"search-photos-models","photos":{},"totalItems":') + self.assertEqual(flickr_noapi.response(response), []) + + response = mock.Mock(text='search-photos-models","photos":{"data": []},"totalItems":') + self.assertEqual(flickr_noapi.response(response), []) + + json = """ + "search-photos-models","photos": + { + "_data": [ + { + "_flickrModelRegistry": "photo-models", + "title": "This is the title", + "sizes": { + "c": { + "displayUrl": "//farm8.staticflickr.com/7246/14001294434_410f653777_c.jpg", + "width": 541, + "height": 800, + "url": "//c4.staticflickr.com/8/7246/14001294434_410f653777_c.jpg", + "key": "c" + }, + "h": { + "displayUrl": "//farm8.staticflickr.com/7246/14001294434_761d32237a_h.jpg", + "width": 1081, + "height": 1600, + "url": "//c4.staticflickr.com/8/7246/14001294434_761d32237a_h.jpg", + "key": "h" + }, + "k": { + "displayUrl": "//farm8.staticflickr.com/7246/14001294434_f145a2c11a_k.jpg", + "width": 1383, + "height": 2048, + "url": "//c4.staticflickr.com/8/7246/14001294434_f145a2c11a_k.jpg", + "key": "k" + }, + "l": { + "displayUrl": "//farm8.staticflickr.com/7246/14001294434_410f653777_b.jpg", + "width": 692, + "height": 1024, + "url": "//c4.staticflickr.com/8/7246/14001294434_410f653777_b.jpg", + "key": "l" + }, + "m": { + "displayUrl": "//farm8.staticflickr.com/7246/14001294434_410f653777.jpg", + "width": 338, + "height": 500, + "url": "//c4.staticflickr.com/8/7246/14001294434_410f653777.jpg", + "key": "m" + }, + "n": { + "displayUrl": "//farm8.staticflickr.com/7246/14001294434_410f653777_n.jpg", + "width": 216, + "height": 320, + "url": "//c4.staticflickr.com/8/7246/14001294434_410f653777_n.jpg", + "key": "n" + }, + "q": { + "displayUrl": "//farm8.staticflickr.com/7246/14001294434_410f653777_q.jpg", + "width": 150, + "height": 150, + "url": "//c4.staticflickr.com/8/7246/14001294434_410f653777_q.jpg", + "key": "q" + }, + "s": { + "displayUrl": "//farm8.staticflickr.com/7246/14001294434_410f653777_m.jpg", + "width": 162, + "height": 240, + "url": "//c4.staticflickr.com/8/7246/14001294434_410f653777_m.jpg", + "key": "s" + }, + "sq": { + "displayUrl": "//farm8.staticflickr.com/7246/14001294434_410f653777_s.jpg", + "width": 75, + "height": 75, + "url": "//c4.staticflickr.com/8/7246/14001294434_410f653777_s.jpg", + "key": "sq" + }, + "t": { + "displayUrl": "//farm8.staticflickr.com/7246/14001294434_410f653777_t.jpg", + "width": 68, + "height": 100, + "url": "//c4.staticflickr.com/8/7246/14001294434_410f653777_t.jpg", + "key": "t" + }, + "z": { + "displayUrl": "//farm8.staticflickr.com/7246/14001294434_410f653777_z.jpg", + "width": 433, + "height": 640, + "url": "//c4.staticflickr.com/8/7246/14001294434_410f653777_z.jpg", + "key": "z" + } + }, + "canComment": false, + "rotation": 0, + "owner": { + "_flickrModelRegistry": "person-models", + "pathAlias": "klink692", + "username": "Owner", + "buddyicon": { + "retina": null, + "large": null, + "medium": null, + "small": null, + "default": "//c1.staticflickr.com/9/8108/buddyicons/59729010@N00.jpg?1361642376#59729010@N00" + }, + "isPro": true, + "id": "59729010@N00" + }, + "engagement": { + "_flickrModelRegistry": "photo-engagement-models", + "ownerNsid": "59729010@N00", + "faveCount": 21, + "commentCount": 14, + "viewCount": 10160, + "id": "14001294434" + }, + "description": "Description", + "isHD": false, + "secret": "410f653777", + "canAddMeta": false, + "license": 0, + "oWidth": 1803, + "oHeight": 2669, + "safetyLevel": 0, + "id": "14001294434" + } + ], + "fetchedStart": true, + "fetchedEnd": false, + "totalItems": "4386039" + },"totalItems": + """ + json = json.replace('\r\n', '').replace('\n', '').replace('\r', '') + response = mock.Mock(text=json) + results = flickr_noapi.response(response) + self.assertEqual(type(results), list) + self.assertEqual(len(results), 1) + self.assertEqual(results[0]['title'], 'This is the title') + self.assertEqual(results[0]['url'], 'https://www.flickr.com/photos/59729010@N00/14001294434') + self.assertIn('k.jpg', results[0]['img_src']) + self.assertIn('n.jpg', results[0]['thumbnail_src']) + self.assertIn('Owner', results[0]['content']) + self.assertIn('Description', results[0]['content']) + + json = """ + "search-photos-models","photos": + { + "_data": [ + { + "_flickrModelRegistry": "photo-models", + "title": "This is the title", + "sizes": { + "z": { + "displayUrl": "//farm8.staticflickr.com/7246/14001294434_410f653777_z.jpg", + "width": 433, + "height": 640, + "url": "//c4.staticflickr.com/8/7246/14001294434_410f653777_z.jpg", + "key": "z" + } + }, + "canComment": false, + "rotation": 0, + "owner": { + "_flickrModelRegistry": "person-models", + "pathAlias": "klink692", + "username": "Owner", + "buddyicon": { + "retina": null, + "large": null, + "medium": null, + "small": null, + "default": "//c1.staticflickr.com/9/8108/buddyicons/59729010@N00.jpg?1361642376#59729010@N00" + }, + "isPro": true, + "id": "59729010@N00" + }, + "engagement": { + "_flickrModelRegistry": "photo-engagement-models", + "ownerNsid": "59729010@N00", + "faveCount": 21, + "commentCount": 14, + "viewCount": 10160, + "id": "14001294434" + }, + "description": "Description", + "isHD": false, + "secret": "410f653777", + "canAddMeta": false, + "license": 0, + "oWidth": 1803, + "oHeight": 2669, + "safetyLevel": 0, + "id": "14001294434" + } + ], + "fetchedStart": true, + "fetchedEnd": false, + "totalItems": "4386039" + },"totalItems": + """ + response = mock.Mock(text=json) + results = flickr_noapi.response(response) + self.assertEqual(type(results), list) + self.assertEqual(len(results), 1) + self.assertEqual(results[0]['title'], 'This is the title') + self.assertEqual(results[0]['url'], 'https://www.flickr.com/photos/59729010@N00/14001294434') + self.assertIn('z.jpg', results[0]['img_src']) + self.assertIn('z.jpg', results[0]['thumbnail_src']) + self.assertIn('Owner', results[0]['content']) + self.assertIn('Description', results[0]['content']) + + json = """ + "search-photos-models","photos": + { + "_data": [ + { + "_flickrModelRegistry": "photo-models", + "title": "This is the title", + "sizes": { + "o": { + "displayUrl": "//farm8.staticflickr.com/7246/14001294434_410f653777_o.jpg", + "width": 433, + "height": 640, + "url": "//c4.staticflickr.com/8/7246/14001294434_410f653777_o.jpg", + "key": "o" + } + }, + "canComment": false, + "rotation": 0, + "owner": { + "_flickrModelRegistry": "person-models", + "pathAlias": "klink692", + "username": "Owner", + "buddyicon": { + "retina": null, + "large": null, + "medium": null, + "small": null, + "default": "//c1.staticflickr.com/9/8108/buddyicons/59729010@N00.jpg?1361642376#59729010@N00" + }, + "isPro": true, + "id": "59729010@N00" + }, + "engagement": { + "_flickrModelRegistry": "photo-engagement-models", + "ownerNsid": "59729010@N00", + "faveCount": 21, + "commentCount": 14, + "viewCount": 10160, + "id": "14001294434" + }, + "isHD": false, + "secret": "410f653777", + "canAddMeta": false, + "license": 0, + "oWidth": 1803, + "oHeight": 2669, + "safetyLevel": 0, + "id": "14001294434" + } + ], + "fetchedStart": true, + "fetchedEnd": false, + "totalItems": "4386039" + },"totalItems": + """ + response = mock.Mock(text=json) + results = flickr_noapi.response(response) + self.assertEqual(type(results), list) + self.assertEqual(len(results), 1) + self.assertEqual(results[0]['title'], 'This is the title') + self.assertEqual(results[0]['url'], 'https://www.flickr.com/photos/59729010@N00/14001294434') + self.assertIn('o.jpg', results[0]['img_src']) + self.assertIn('o.jpg', results[0]['thumbnail_src']) + self.assertIn('Owner', results[0]['content']) + + json = """ + "search-photos-models","photos": + { + "_data": [ + { + "_flickrModelRegistry": "photo-models", + "title": "This is the title", + "sizes": { + }, + "canComment": false, + "rotation": 0, + "owner": { + "_flickrModelRegistry": "person-models", + "pathAlias": "klink692", + "username": "Owner", + "buddyicon": { + "retina": null, + "large": null, + "medium": null, + "small": null, + "default": "//c1.staticflickr.com/9/8108/buddyicons/59729010@N00.jpg?1361642376#59729010@N00" + }, + "isPro": true, + "id": "59729010@N00" + }, + "engagement": { + "_flickrModelRegistry": "photo-engagement-models", + "ownerNsid": "59729010@N00", + "faveCount": 21, + "commentCount": 14, + "viewCount": 10160, + "id": "14001294434" + }, + "description": "Description", + "isHD": false, + "secret": "410f653777", + "canAddMeta": false, + "license": 0, + "oWidth": 1803, + "oHeight": 2669, + "safetyLevel": 0, + "id": "14001294434" + } + ], + "fetchedStart": true, + "fetchedEnd": false, + "totalItems": "4386039" + },"totalItems": + """ + response = mock.Mock(text=json) + results = flickr_noapi.response(response) + self.assertEqual(type(results), list) + self.assertEqual(len(results), 0) + + json = """ + "search-photos-models","photos": + { + "_data": [null], + "fetchedStart": true, + "fetchedEnd": false, + "totalItems": "4386039" + },"totalItems": + """ + response = mock.Mock(text=json) + results = flickr_noapi.response(response) + self.assertEqual(type(results), list) + self.assertEqual(len(results), 0) + + json = """ + "search-photos-models","photos": + { + "_data": [ + { + "_flickrModelRegistry": "photo-models", + "title": "This is the title", + "sizes": { + "o": { + "displayUrl": "//farm8.staticflickr.com/7246/14001294434_410f653777_o.jpg", + "width": 433, + "height": 640, + "url": "//c4.staticflickr.com/8/7246/14001294434_410f653777_o.jpg", + "key": "o" + } + }, + "canComment": false, + "rotation": 0, + "owner": { + "_flickrModelRegistry": "person-models", + "pathAlias": "klink692", + "username": "Owner", + "buddyicon": { + "retina": null, + "large": null, + "medium": null, + "small": null, + "default": "//c1.staticflickr.com/9/8108/buddyicons/59729010@N00.jpg?1361642376#59729010@N00" + }, + "isPro": true + }, + "engagement": { + "_flickrModelRegistry": "photo-engagement-models", + "ownerNsid": "59729010@N00", + "faveCount": 21, + "commentCount": 14, + "viewCount": 10160, + "id": "14001294434" + }, + "description": "Description", + "isHD": false, + "secret": "410f653777", + "canAddMeta": false, + "license": 0, + "oWidth": 1803, + "oHeight": 2669, + "safetyLevel": 0, + "id": "14001294434" + } + ], + "fetchedStart": true, + "fetchedEnd": false, + "totalItems": "4386039" + },"totalItems": + """ + response = mock.Mock(text=json) + results = flickr_noapi.response(response) + self.assertEqual(type(results), list) + self.assertEqual(len(results), 0) + + json = """ + {"toto":[ + {"id":200,"name":"Artist Name", + "link":"http:\/\/www.flickr.com\/artist\/1217","type":"artist"} + ]} + """ + response = mock.Mock(text=json) + results = flickr_noapi.response(response) + self.assertEqual(type(results), list) + self.assertEqual(len(results), 0) diff --git a/searx/tests/engines/test_google_images.py b/searx/tests/engines/test_google_images.py @@ -0,0 +1,108 @@ +from collections import defaultdict +import mock +from searx.engines import google_images +from searx.testing import SearxTestCase + + +class TestGoogleImagesEngine(SearxTestCase): + + def test_request(self): + query = 'test_query' + dicto = defaultdict(dict) + dicto['pageno'] = 1 + params = google_images.request(query, dicto) + self.assertTrue('url' in params) + self.assertTrue(query in params['url']) + self.assertTrue('googleapis.com' in params['url']) + + def test_response(self): + self.assertRaises(AttributeError, google_images.response, None) + self.assertRaises(AttributeError, google_images.response, []) + self.assertRaises(AttributeError, google_images.response, '') + self.assertRaises(AttributeError, google_images.response, '[]') + + response = mock.Mock(text='{}') + self.assertEqual(google_images.response(response), []) + + response = mock.Mock(text='{"data": []}') + self.assertEqual(google_images.response(response), []) + + json = """ + { + "responseData": { + "results": [ + { + "GsearchResultClass": "GimageSearch", + "width": "400", + "height": "400", + "imageId": "ANd9GcQbYb9FJuAbG_hT4i8FeC0O0x-P--EHdzgRIF9ao97nHLl7C2mREn6qTQ", + "tbWidth": "124", + "tbHeight": "124", + "unescapedUrl": "http://unescaped.url.jpg", + "url": "http://image.url.jpg", + "visibleUrl": "insolitebuzz.fr", + "title": "This is the title", + "titleNoFormatting": "Petit test sympa qui rend fou tout le monde ! A faire", + "originalContextUrl": "http://this.is.the.url", + "content": "<b>test</b>", + "contentNoFormatting": "test", + "tbUrl": "http://thumbnail.url" + } + ] + }, + "responseDetails": null, + "responseStatus": 200 + } + """ + response = mock.Mock(text=json) + results = google_images.response(response) + self.assertEqual(type(results), list) + self.assertEqual(len(results), 1) + self.assertEqual(results[0]['title'], 'This is the title') + self.assertEqual(results[0]['url'], 'http://this.is.the.url') + self.assertEqual(results[0]['thumbnail_src'], 'http://thumbnail.url') + self.assertEqual(results[0]['img_src'], 'http://image.url.jpg') + self.assertEqual(results[0]['content'], '<b>test</b>') + + json = """ + { + "responseData": { + "results": [ + { + "GsearchResultClass": "GimageSearch", + "width": "400", + "height": "400", + "imageId": "ANd9GcQbYb9FJuAbG_hT4i8FeC0O0x-P--EHdzgRIF9ao97nHLl7C2mREn6qTQ", + "tbWidth": "124", + "tbHeight": "124", + "unescapedUrl": "http://unescaped.url.jpg", + "visibleUrl": "insolitebuzz.fr", + "title": "This is the title", + "titleNoFormatting": "Petit test sympa qui rend fou tout le monde ! A faire", + "originalContextUrl": "http://this.is.the.url", + "content": "<b>test</b>", + "contentNoFormatting": "test", + "tbUrl": "http://thumbnail.url" + } + ] + }, + "responseDetails": null, + "responseStatus": 200 + } + """ + response = mock.Mock(text=json) + results = google_images.response(response) + self.assertEqual(type(results), list) + self.assertEqual(len(results), 0) + + json = """ + { + "responseData": {}, + "responseDetails": null, + "responseStatus": 200 + } + """ + response = mock.Mock(text=json) + results = google_images.response(response) + self.assertEqual(type(results), list) + self.assertEqual(len(results), 0) diff --git a/searx/tests/engines/test_google_news.py b/searx/tests/engines/test_google_news.py @@ -0,0 +1,136 @@ +from collections import defaultdict +import mock +from searx.engines import google_news +from searx.testing import SearxTestCase + + +class TestGoogleNewsEngine(SearxTestCase): + + def test_request(self): + query = 'test_query' + dicto = defaultdict(dict) + dicto['pageno'] = 1 + dicto['language'] = 'fr_FR' + params = google_news.request(query, dicto) + self.assertIn('url', params) + self.assertIn(query, params['url']) + self.assertIn('googleapis.com', params['url']) + self.assertIn('fr', params['url']) + + dicto['language'] = 'all' + params = google_news.request(query, dicto) + self.assertIn('url', params) + self.assertIn('en', params['url']) + + def test_response(self): + self.assertRaises(AttributeError, google_news.response, None) + self.assertRaises(AttributeError, google_news.response, []) + self.assertRaises(AttributeError, google_news.response, '') + self.assertRaises(AttributeError, google_news.response, '[]') + + response = mock.Mock(text='{}') + self.assertEqual(google_news.response(response), []) + + response = mock.Mock(text='{"data": []}') + self.assertEqual(google_news.response(response), []) + + json = """ + { + "responseData": { + "results": [ + { + "GsearchResultClass": "GnewsSearch", + "clusterUrl": "http://news.google.com/news/story?ncl=d2d3t1LMDpNIj2MPPhdTT0ycN4sWM&hl=fr&ned=fr", + "content": "This is the content", + "unescapedUrl": "http://this.is.the.url", + "url": "http://this.is.the.url", + "title": "This is the title", + "titleNoFormatting": "This is the title", + "location": "", + "publisher": "Jeux Actu", + "publishedDate": "Fri, 30 Jan 2015 11:00:25 -0800", + "signedRedirectUrl": "http://news.google.com/", + "language": "fr", + "image": { + "url": "http://i.jeuxactus.com/datas/jeux/d/y/dying-light/vu/dying-light-54cc080b568fb.jpg", + "tbUrl": "http://t1.gstatic.com/images?q=tbn:ANd9GcSF4yYrs9Ycw23DGiOSAZ-5SEPXYwG3LNs", + "originalContextUrl": "http://www.jeuxactu.com/test-dying-light-sur-ps4-97208.htm", + "publisher": "Jeux Actu", + "tbWidth": 80, + "tbHeight": 30 + }, + "relatedStories": [ + { + "unescapedUrl": "http://www.jeuxvideo.com/test/415823/dying-light.htm", + "url": "http%3A%2F%2Fwww.jeuxvideo.com%2Ftest%2F415823%2Fdying-light.htm", + "title": "<b>Test</b> du jeu Dying Light - jeuxvideo.com", + "titleNoFormatting": "Test du jeu Dying Light - jeuxvideo.com", + "location": "", + "publisher": "JeuxVideo.com", + "publishedDate": "Fri, 30 Jan 2015 08:52:30 -0800", + "signedRedirectUrl": "http://news.google.com/news/url?sa=T&", + "language": "fr" + } + ] + } + ] + }, + "responseDetails": null, + "responseStatus": 200 + } + """ + response = mock.Mock(text=json) + results = google_news.response(response) + self.assertEqual(type(results), list) + self.assertEqual(len(results), 1) + self.assertEqual(results[0]['title'], 'This is the title') + self.assertEqual(results[0]['url'], 'http://this.is.the.url') + self.assertEqual(results[0]['content'], 'This is the content') + + json = """ + { + "responseData": { + "results": [ + { + "GsearchResultClass": "GnewsSearch", + "clusterUrl": "http://news.google.com/news/story?ncl=d2d3t1LMDpNIj2MPPhdTT0ycN4sWM&hl=fr&ned=fr", + "content": "This is the content", + "unescapedUrl": "http://this.is.the.url", + "title": "This is the title", + "titleNoFormatting": "This is the title", + "location": "", + "publisher": "Jeux Actu", + "publishedDate": "Fri, 30 Jan 2015 11:00:25 -0800", + "signedRedirectUrl": "http://news.google.com/news/", + "language": "fr", + "image": { + "url": "http://i.jeuxactus.com/datas/jeux/d/y/dying-light/vu/dying-light-54cc080b568fb.jpg", + "tbUrl": "http://t1.gstatic.com/images?q=tbn:b_6f-OSAZ-5SEPXYwG3LNs", + "originalContextUrl": "http://www.jeuxactu.com/test-dying-light-sur-ps4-97208.htm", + "publisher": "Jeux Actu", + "tbWidth": 80, + "tbHeight": 30 + } + } + ] + }, + "responseDetails": null, + "responseStatus": 200 + } + """ + response = mock.Mock(text=json) + results = google_news.response(response) + self.assertEqual(type(results), list) + self.assertEqual(len(results), 0) + + json = """ + { + "responseData": {}, + "responseDetails": null, + "responseStatus": 200 + } + """ + response = mock.Mock(text=json) + results = google_news.response(response) + self.assertEqual(type(results), list) + self.assertEqual(len(results), 0) diff --git a/searx/tests/engines/test_kickass.py b/searx/tests/engines/test_kickass.py @@ -0,0 +1,398 @@ +# -*- coding: utf-8 -*- +from collections import defaultdict +import mock +from searx.engines import kickass +from searx.testing import SearxTestCase + + +class TestKickassEngine(SearxTestCase): + + def test_request(self): + query = 'test_query' + dicto = defaultdict(dict) + dicto['pageno'] = 1 + params = kickass.request(query, dicto) + self.assertIn('url', params) + self.assertIn(query, params['url']) + self.assertIn('kickass.so', params['url']) + self.assertIn('verify', params) + self.assertFalse(params['verify']) + + def test_response(self): + self.assertRaises(AttributeError, kickass.response, None) + self.assertRaises(AttributeError, kickass.response, []) + self.assertRaises(AttributeError, kickass.response, '') + self.assertRaises(AttributeError, kickass.response, '[]') + + response = mock.Mock(text='<html></html>') + self.assertEqual(kickass.response(response), []) + + html = """ + <table cellpadding="0" cellspacing="0" class="data" style="width: 100%"> + <tr class="firstr"> + <th class="width100perc nopad">torrent name</th> + <th class="center"> + <a href="/search/test/?field=size&sorder=desc" rel="nofollow">size</a> + </th> + <th class="center"><span class="files"> + <a href="/search/test/?field=files_count&sorder=desc" rel="nofollow">files</a></span> + </th> + <th class="center"><span> + <a href="/search/test/?field=time_add&sorder=desc" rel="nofollow">age</a></span> + </th> + <th class="center"><span class="seed"> + <a href="/search/test/?field=seeders&sorder=desc" rel="nofollow">seed</a></span> + </th> + <th class="lasttd nobr center"> + <a href="/search/test/?field=leechers&sorder=desc" rel="nofollow">leech</a> + </th> + </tr> + <tr class="even" id="torrent_test6478745"> + <td> + <div class="iaconbox center floatright"> + <a rel="6478745,0" class="icommentjs icon16" href="/test-t6478745.html#comment"> + <em style="font-size: 12px; margin: 0 4px 0 4px;" class="iconvalue">3</em> + <i class="ka ka-comment"></i> + </a> + <a class="iverify icon16" href="/test-t6478745.html" title="Verified Torrent"> + <i class="ka ka16 ka-verify ka-green"></i> + </a> + <a href="#" onclick="_scq.push([]); return false;" class="partner1Button idownload icon16"> + <i class="ka ka16 ka-arrow-down partner1Button"></i> + </a> + <a title="Torrent magnet link" + href="magnet:?xt=urn:btih:MAGNETURL&dn=test" class="imagnet icon16"> + <i class="ka ka16 ka-magnet"></i> + </a> + <a title="Download torrent file" + href="http://torcache.net/torrent/53917.torrent?title=test" class="idownload icon16"> + <i class="ka ka16 ka-arrow-down"></i> + </a> + </div> + <div class="torrentname"> + <a href="/test-t6478745.html" class="torType txtType"></a> + <a href="/test-t6478745.html" class="normalgrey font12px plain bold"></a> + <div class="markeredBlock torType txtType"> + <a href="/url.html" class="cellMainLink"> + <strong class="red">This should be the title</strong> + </a> + <span class="font11px lightgrey block"> + Posted by <i class="ka ka-verify" style="font-size: 16px;color:orange;"></i> + <a class="plain" href="/user/riri/">riri</a> in + <span id="cat_6478745"> + <strong><a href="/other/">Other</a> > <a href="/unsorted/">Unsorted</a></strong> + </span> + </span> + </div> + </td> + <td class="nobr center">449 <span>bytes</span></td> + <td class="center">4</td> + <td class="center">2&nbsp;years</td> + <td class="green center">10</td> + <td class="red lasttd center">1</td> + </tr> + </table> + """ + response = mock.Mock(text=html) + results = kickass.response(response) + self.assertEqual(type(results), list) + self.assertEqual(len(results), 1) + self.assertEqual(results[0]['title'], 'This should be the title') + self.assertEqual(results[0]['url'], 'https://kickass.so/url.html') + self.assertEqual(results[0]['content'], 'Posted by riri in Other &gt; Unsorted') + self.assertEqual(results[0]['seed'], 10) + self.assertEqual(results[0]['leech'], 1) + self.assertEqual(results[0]['filesize'], 449) + self.assertEqual(results[0]['files'], 4) + self.assertEqual(results[0]['magnetlink'], 'magnet:?xt=urn:btih:MAGNETURL&dn=test') + self.assertEqual(results[0]['torrentfile'], 'http://torcache.net/torrent/53917.torrent?title=test') + + html = """ + <table cellpadding="0" cellspacing="0" class="data" style="width: 100%"> + <tr class="firstr"> + <th class="width100perc nopad">torrent name</th> + <th class="center"> + <a href="/search/test/?field=size&sorder=desc" rel="nofollow">size</a> + </th> + <th class="center"><span class="files"> + <a href="/search/test/?field=files_count&sorder=desc" rel="nofollow">files</a></span> + </th> + <th class="center"><span> + <a href="/search/test/?field=time_add&sorder=desc" rel="nofollow">age</a></span> + </th> + <th class="center"><span class="seed"> + <a href="/search/test/?field=seeders&sorder=desc" rel="nofollow">seed</a></span> + </th> + <th class="lasttd nobr center"> + <a href="/search/test/?field=leechers&sorder=desc" rel="nofollow">leech</a> + </th> + </tr> + </table> + """ + response = mock.Mock(text=html) + results = kickass.response(response) + self.assertEqual(type(results), list) + self.assertEqual(len(results), 0) + + html = """ + <table cellpadding="0" cellspacing="0" class="data" style="width: 100%"> + <tr class="firstr"> + <th class="width100perc nopad">torrent name</th> + <th class="center"> + <a href="/search/test/?field=size&sorder=desc" rel="nofollow">size</a> + </th> + <th class="center"><span class="files"> + <a href="/search/test/?field=files_count&sorder=desc" rel="nofollow">files</a></span> + </th> + <th class="center"><span> + <a href="/search/test/?field=time_add&sorder=desc" rel="nofollow">age</a></span> + </th> + <th class="center"><span class="seed"> + <a href="/search/test/?field=seeders&sorder=desc" rel="nofollow">seed</a></span> + </th> + <th class="lasttd nobr center"> + <a href="/search/test/?field=leechers&sorder=desc" rel="nofollow">leech</a> + </th> + </tr> + <tr class="even" id="torrent_test6478745"> + <td> + <div class="iaconbox center floatright"> + <a rel="6478745,0" class="icommentjs icon16" href="/test-t6478745.html#comment"> + <em style="font-size: 12px; margin: 0 4px 0 4px;" class="iconvalue">3</em> + <i class="ka ka-comment"></i> + </a> + <a class="iverify icon16" href="/test-t6478745.html" title="Verified Torrent"> + <i class="ka ka16 ka-verify ka-green"></i> + </a> + <a href="#" onclick="_scq.push([]); return false;" class="partner1Button idownload icon16"> + <i class="ka ka16 ka-arrow-down partner1Button"></i> + </a> + <a title="Torrent magnet link" + href="magnet:?xt=urn:btih:MAGNETURL&dn=test" class="imagnet icon16"> + <i class="ka ka16 ka-magnet"></i> + </a> + <a title="Download torrent file" + href="http://torcache.net/torrent/53917.torrent?title=test" class="idownload icon16"> + <i class="ka ka16 ka-arrow-down"></i> + </a> + </div> + <div class="torrentname"> + <a href="/test-t6478745.html" class="torType txtType"></a> + <a href="/test-t6478745.html" class="normalgrey font12px plain bold"></a> + <div class="markeredBlock torType txtType"> + <a href="/url.html" class="cellMainLink"> + <strong class="red">This should be the title</strong> + </a> + <span class="font11px lightgrey block"> + Posted by <i class="ka ka-verify" style="font-size: 16px;color:orange;"></i> + <a class="plain" href="/user/riri/">riri</a> in + <span id="cat_6478745"> + <strong><a href="/other/">Other</a> > <a href="/unsorted/">Unsorted</a></strong> + </span> + </span> + </div> + </td> + <td class="nobr center">1 <span>KB</span></td> + <td class="center">4</td> + <td class="center">2&nbsp;years</td> + <td class="green center">10</td> + <td class="red lasttd center">1</td> + </tr> + <tr class="even" id="torrent_test6478745"> + <td> + <div class="iaconbox center floatright"> + <a rel="6478745,0" class="icommentjs icon16" href="/test-t6478745.html#comment"> + <em style="font-size: 12px; margin: 0 4px 0 4px;" class="iconvalue">3</em> + <i class="ka ka-comment"></i> + </a> + <a class="iverify icon16" href="/test-t6478745.html" title="Verified Torrent"> + <i class="ka ka16 ka-verify ka-green"></i> + </a> + <a href="#" onclick="_scq.push([]); return false;" class="partner1Button idownload icon16"> + <i class="ka ka16 ka-arrow-down partner1Button"></i> + </a> + <a title="Torrent magnet link" + href="magnet:?xt=urn:btih:MAGNETURL&dn=test" class="imagnet icon16"> + <i class="ka ka16 ka-magnet"></i> + </a> + <a title="Download torrent file" + href="http://torcache.net/torrent/53917.torrent?title=test" class="idownload icon16"> + <i class="ka ka16 ka-arrow-down"></i> + </a> + </div> + <div class="torrentname"> + <a href="/test-t6478745.html" class="torType txtType"></a> + <a href="/test-t6478745.html" class="normalgrey font12px plain bold"></a> + <div class="markeredBlock torType txtType"> + <a href="/url.html" class="cellMainLink"> + <strong class="red">This should be the title</strong> + </a> + <span class="font11px lightgrey block"> + Posted by <i class="ka ka-verify" style="font-size: 16px;color:orange;"></i> + <a class="plain" href="/user/riri/">riri</a> in + <span id="cat_6478745"> + <strong><a href="/other/">Other</a> > <a href="/unsorted/">Unsorted</a></strong> + </span> + </span> + </div> + </td> + <td class="nobr center">1 <span>MB</span></td> + <td class="center">4</td> + <td class="center">2&nbsp;years</td> + <td class="green center">9</td> + <td class="red lasttd center">1</td> + </tr> + <tr class="even" id="torrent_test6478745"> + <td> + <div class="iaconbox center floatright"> + <a rel="6478745,0" class="icommentjs icon16" href="/test-t6478745.html#comment"> + <em style="font-size: 12px; margin: 0 4px 0 4px;" class="iconvalue">3</em> + <i class="ka ka-comment"></i> + </a> + <a class="iverify icon16" href="/test-t6478745.html" title="Verified Torrent"> + <i class="ka ka16 ka-verify ka-green"></i> + </a> + <a href="#" onclick="_scq.push([]); return false;" class="partner1Button idownload icon16"> + <i class="ka ka16 ka-arrow-down partner1Button"></i> + </a> + <a title="Torrent magnet link" + href="magnet:?xt=urn:btih:MAGNETURL&dn=test" class="imagnet icon16"> + <i class="ka ka16 ka-magnet"></i> + </a> + <a title="Download torrent file" + href="http://torcache.net/torrent/53917.torrent?title=test" class="idownload icon16"> + <i class="ka ka16 ka-arrow-down"></i> + </a> + </div> + <div class="torrentname"> + <a href="/test-t6478745.html" class="torType txtType"></a> + <a href="/test-t6478745.html" class="normalgrey font12px plain bold"></a> + <div class="markeredBlock torType txtType"> + <a href="/url.html" class="cellMainLink"> + <strong class="red">This should be the title</strong> + </a> + <span class="font11px lightgrey block"> + Posted by <i class="ka ka-verify" style="font-size: 16px;color:orange;"></i> + <a class="plain" href="/user/riri/">riri</a> in + <span id="cat_6478745"> + <strong><a href="/other/">Other</a> > <a href="/unsorted/">Unsorted</a></strong> + </span> + </span> + </div> + </td> + <td class="nobr center">1 <span>GB</span></td> + <td class="center">4</td> + <td class="center">2&nbsp;years</td> + <td class="green center">8</td> + <td class="red lasttd center">1</td> + </tr> + <tr class="even" id="torrent_test6478745"> + <td> + <div class="iaconbox center floatright"> + <a rel="6478745,0" class="icommentjs icon16" href="/test-t6478745.html#comment"> + <em style="font-size: 12px; margin: 0 4px 0 4px;" class="iconvalue">3</em> + <i class="ka ka-comment"></i> + </a> + <a class="iverify icon16" href="/test-t6478745.html" title="Verified Torrent"> + <i class="ka ka16 ka-verify ka-green"></i> + </a> + <a href="#" onclick="_scq.push([]); return false;" class="partner1Button idownload icon16"> + <i class="ka ka16 ka-arrow-down partner1Button"></i> + </a> + <a title="Torrent magnet link" + href="magnet:?xt=urn:btih:MAGNETURL&dn=test" class="imagnet icon16"> + <i class="ka ka16 ka-magnet"></i> + </a> + <a title="Download torrent file" + href="http://torcache.net/torrent/53917.torrent?title=test" class="idownload icon16"> + <i class="ka ka16 ka-arrow-down"></i> + </a> + </div> + <div class="torrentname"> + <a href="/test-t6478745.html" class="torType txtType"></a> + <a href="/test-t6478745.html" class="normalgrey font12px plain bold"></a> + <div class="markeredBlock torType txtType"> + <a href="/url.html" class="cellMainLink"> + <strong class="red">This should be the title</strong> + </a> + <span class="font11px lightgrey block"> + Posted by <i class="ka ka-verify" style="font-size: 16px;color:orange;"></i> + <a class="plain" href="/user/riri/">riri</a> in + <span id="cat_6478745"> + <strong><a href="/other/">Other</a> > <a href="/unsorted/">Unsorted</a></strong> + </span> + </span> + </div> + </td> + <td class="nobr center">1 <span>TB</span></td> + <td class="center">4</td> + <td class="center">2&nbsp;years</td> + <td class="green center">7</td> + <td class="red lasttd center">1</td> + </tr> + <tr class="even" id="torrent_test6478745"> + <td> + <div class="iaconbox center floatright"> + <a rel="6478745,0" class="icommentjs icon16" href="/test-t6478745.html#comment"> + <em style="font-size: 12px; margin: 0 4px 0 4px;" class="iconvalue">3</em> + <i class="ka ka-comment"></i> + </a> + <a class="iverify icon16" href="/test-t6478745.html" title="Verified Torrent"> + <i class="ka ka16 ka-verify ka-green"></i> + </a> + <a href="#" onclick="_scq.push([]); return false;" class="partner1Button idownload icon16"> + <i class="ka ka16 ka-arrow-down partner1Button"></i> + </a> + <a title="Torrent magnet link" + href="magnet:?xt=urn:btih:MAGNETURL&dn=test" class="imagnet icon16"> + <i class="ka ka16 ka-magnet"></i> + </a> + <a title="Download torrent file" + href="http://torcache.net/torrent/53917.torrent?title=test" class="idownload icon16"> + <i class="ka ka16 ka-arrow-down"></i> + </a> + </div> + <div class="torrentname"> + <a href="/test-t6478745.html" class="torType txtType"></a> + <a href="/test-t6478745.html" class="normalgrey font12px plain bold"></a> + <div class="markeredBlock torType txtType"> + <a href="/url.html" class="cellMainLink"> + <strong class="red">This should be the title</strong> + </a> + <span class="font11px lightgrey block"> + Posted by <i class="ka ka-verify" style="font-size: 16px;color:orange;"></i> + <a class="plain" href="/user/riri/">riri</a> in + <span id="cat_6478745"> + <strong><a href="/other/">Other</a> > <a href="/unsorted/">Unsorted</a></strong> + </span> + </span> + </div> + </td> + <td class="nobr center">z <span>bytes</span></td> + <td class="center">r</td> + <td class="center">2&nbsp;years</td> + <td class="green center">a</td> + <td class="red lasttd center">t</td> + </tr> + </table> + """ + response = mock.Mock(text=html) + results = kickass.response(response) + self.assertEqual(type(results), list) + self.assertEqual(len(results), 5) + self.assertEqual(results[0]['title'], 'This should be the title') + self.assertEqual(results[0]['url'], 'https://kickass.so/url.html') + self.assertEqual(results[0]['content'], 'Posted by riri in Other &gt; Unsorted') + self.assertEqual(results[0]['seed'], 10) + self.assertEqual(results[0]['leech'], 1) + self.assertEqual(results[0]['files'], 4) + self.assertEqual(results[0]['magnetlink'], 'magnet:?xt=urn:btih:MAGNETURL&dn=test') + self.assertEqual(results[0]['torrentfile'], 'http://torcache.net/torrent/53917.torrent?title=test') + self.assertEqual(results[0]['filesize'], 1024) + self.assertEqual(results[1]['filesize'], 1048576) + self.assertEqual(results[2]['filesize'], 1073741824) + self.assertEqual(results[3]['filesize'], 1099511627776) + self.assertEqual(results[4]['seed'], 0) + self.assertEqual(results[4]['leech'], 0) + self.assertEqual(results[4]['files'], None) + self.assertEqual(results[4]['filesize'], None) diff --git a/searx/tests/engines/test_mixcloud.py b/searx/tests/engines/test_mixcloud.py @@ -0,0 +1,67 @@ +from collections import defaultdict +import mock +from searx.engines import mixcloud +from searx.testing import SearxTestCase + + +class TestMixcloudEngine(SearxTestCase): + + def test_request(self): + query = 'test_query' + dicto = defaultdict(dict) + dicto['pageno'] = 0 + params = mixcloud.request(query, dicto) + self.assertTrue('url' in params) + self.assertTrue(query in params['url']) + self.assertTrue('mixcloud.com' in params['url']) + + def test_response(self): + self.assertRaises(AttributeError, mixcloud.response, None) + self.assertRaises(AttributeError, mixcloud.response, []) + self.assertRaises(AttributeError, mixcloud.response, '') + self.assertRaises(AttributeError, mixcloud.response, '[]') + + response = mock.Mock(text='{}') + self.assertEqual(mixcloud.response(response), []) + + response = mock.Mock(text='{"data": []}') + self.assertEqual(mixcloud.response(response), []) + + json = """ + {"data":[ + { + "user": { + "url": "http://www.mixcloud.com/user/", + "username": "user", + "name": "User", + "key": "/user/" + }, + "key": "/user/this-is-the-url/", + "created_time": "2014-11-14T13:30:02Z", + "audio_length": 3728, + "slug": "this-is-the-url", + "name": "Title of track", + "url": "http://www.mixcloud.com/user/this-is-the-url/", + "updated_time": "2014-11-14T13:14:10Z" + } + ]} + """ + response = mock.Mock(text=json) + results = mixcloud.response(response) + self.assertEqual(type(results), list) + self.assertEqual(len(results), 1) + self.assertEqual(results[0]['title'], 'Title of track') + self.assertEqual(results[0]['url'], 'http://www.mixcloud.com/user/this-is-the-url/') + self.assertEqual(results[0]['content'], 'User') + self.assertTrue('http://www.mixcloud.com/user/this-is-the-url/' in results[0]['embedded']) + + json = """ + {"toto":[ + {"id":200,"name":"Artist Name", + "link":"http:\/\/www.mixcloud.com\/artist\/1217","type":"artist"} + ]} + """ + response = mock.Mock(text=json) + results = mixcloud.response(response) + self.assertEqual(type(results), list) + self.assertEqual(len(results), 0) diff --git a/searx/tests/engines/test_piratebay.py b/searx/tests/engines/test_piratebay.py @@ -0,0 +1,137 @@ +# -*- coding: utf-8 -*- +from collections import defaultdict +import mock +from searx.engines import piratebay +from searx.testing import SearxTestCase + + +class TestPiratebayEngine(SearxTestCase): + + def test_request(self): + query = 'test_query' + dicto = defaultdict(dict) + dicto['pageno'] = 1 + dicto['category'] = 'Toto' + params = piratebay.request(query, dicto) + self.assertIn('url', params) + self.assertIn(query, params['url']) + self.assertIn('piratebay.cr', params['url']) + self.assertIn('0', params['url']) + + dicto['category'] = 'music' + params = piratebay.request(query, dicto) + self.assertIn('100', params['url']) + + def test_response(self): + self.assertRaises(AttributeError, piratebay.response, None) + self.assertRaises(AttributeError, piratebay.response, []) + self.assertRaises(AttributeError, piratebay.response, '') + self.assertRaises(AttributeError, piratebay.response, '[]') + + response = mock.Mock(text='<html></html>') + self.assertEqual(piratebay.response(response), []) + + html = """ + <table id="searchResult"> + <tr> + </tr> + <tr> + <td class="vertTh"> + <center> + <a href="#" title="More from this category">Anime</a><br/> + (<a href="#" title="More from this category">Anime</a>) + </center> + </td> + <td> + <div class="detName"> + <a href="/this.is.the.link" class="detLink" title="Title"> + This is the title + </a> + </div> + <a href="magnet:?xt=urn:btih:MAGNETLINK" title="Download this torrent using magnet"> + <img src="/static/img/icon-magnet.gif" alt="Magnet link"/> + </a> + <a href="http://torcache.net/torrent/TORRENTFILE.torrent" title="Download this torrent"> + <img src="/static/img/dl.gif" class="dl" alt="Download"/> + </a> + <a href="/user/HorribleSubs"> + <img src="/static/img/vip.gif" alt="VIP" title="VIP" style="width:11px;" border='0'/> + </a> + <img src="/static/img/11x11p.png"/> + <font class="detDesc"> + This is the content <span>and should be</span> OK + </font> + </td> + <td align="right">13</td> + <td align="right">334</td> + </tr> + </table> + """ + response = mock.Mock(text=html) + results = piratebay.response(response) + self.assertEqual(type(results), list) + self.assertEqual(len(results), 1) + self.assertEqual(results[0]['title'], 'This is the title') + self.assertEqual(results[0]['url'], 'https://thepiratebay.cr/this.is.the.link') + self.assertEqual(results[0]['content'], 'This is the content and should be OK') + self.assertEqual(results[0]['seed'], 13) + self.assertEqual(results[0]['leech'], 334) + self.assertEqual(results[0]['magnetlink'], 'magnet:?xt=urn:btih:MAGNETLINK') + self.assertEqual(results[0]['torrentfile'], 'http://torcache.net/torrent/TORRENTFILE.torrent') + + html = """ + <table id="searchResult"> + <tr> + </tr> + <tr> + <td class="vertTh"> + <center> + <a href="#" title="More from this category">Anime</a><br/> + (<a href="#" title="More from this category">Anime</a>) + </center> + </td> + <td> + <div class="detName"> + <a href="/this.is.the.link" class="detLink" title="Title"> + This is the title + </a> + </div> + <a href="magnet:?xt=urn:btih:MAGNETLINK" title="Download this torrent using magnet"> + <img src="/static/img/icon-magnet.gif" alt="Magnet link"/> + </a> + <a href="http://torcache.net/torrent/TORRENTFILE.torrent" title="Download this torrent"> + <img src="/static/img/dl.gif" class="dl" alt="Download"/> + </a> + <a href="/user/HorribleSubs"> + <img src="/static/img/vip.gif" alt="VIP" title="VIP" style="width:11px;" border='0'/> + </a> + <img src="/static/img/11x11p.png"/> + <font class="detDesc"> + This is the content <span>and should be</span> OK + </font> + </td> + <td align="right">s</td> + <td align="right">d</td> + </tr> + </table> + """ + response = mock.Mock(text=html) + results = piratebay.response(response) + self.assertEqual(type(results), list) + self.assertEqual(len(results), 1) + self.assertEqual(results[0]['title'], 'This is the title') + self.assertEqual(results[0]['url'], 'https://thepiratebay.cr/this.is.the.link') + self.assertEqual(results[0]['content'], 'This is the content and should be OK') + self.assertEqual(results[0]['seed'], 0) + self.assertEqual(results[0]['leech'], 0) + self.assertEqual(results[0]['magnetlink'], 'magnet:?xt=urn:btih:MAGNETLINK') + self.assertEqual(results[0]['torrentfile'], 'http://torcache.net/torrent/TORRENTFILE.torrent') + + html = """ + <table id="searchResult"> + </table> + """ + response = mock.Mock(text=html) + results = piratebay.response(response) + self.assertEqual(type(results), list) + self.assertEqual(len(results), 0) diff --git a/searx/tests/engines/test_searchcode_code.py b/searx/tests/engines/test_searchcode_code.py @@ -0,0 +1,75 @@ +from collections import defaultdict +import mock +from searx.engines import searchcode_code +from searx.testing import SearxTestCase + + +class TestSearchcodeCodeEngine(SearxTestCase): + + def test_request(self): + query = 'test_query' + dicto = defaultdict(dict) + dicto['pageno'] = 0 + params = searchcode_code.request(query, dicto) + self.assertIn('url', params) + self.assertIn(query, params['url']) + self.assertIn('searchcode.com', params['url']) + + def test_response(self): + self.assertRaises(AttributeError, searchcode_code.response, None) + self.assertRaises(AttributeError, searchcode_code.response, []) + self.assertRaises(AttributeError, searchcode_code.response, '') + self.assertRaises(AttributeError, searchcode_code.response, '[]') + + response = mock.Mock(text='{}') + self.assertEqual(searchcode_code.response(response), []) + + response = mock.Mock(text='{"data": []}') + self.assertEqual(searchcode_code.response(response), []) + + json = """ + { + "matchterm": "test", + "previouspage": null, + "searchterm": "test", + "query": "test", + "total": 1000, + "page": 0, + "nextpage": 1, + "results": [ + { + "repo": "https://repo", + "linescount": 1044, + "location": "/tests", + "name": "Name", + "url": "https://url", + "md5hash": "ecac6e479edd2b9406c9e08603cec655", + "lines": { + "1": "// Test 011", + "2": "// Source: " + }, + "id": 51223527, + "filename": "File.CPP" + } + ] + } + """ + response = mock.Mock(text=json) + results = searchcode_code.response(response) + self.assertEqual(type(results), list) + self.assertEqual(len(results), 1) + self.assertEqual(results[0]['title'], 'Name - File.CPP') + self.assertEqual(results[0]['url'], 'https://url') + self.assertEqual(results[0]['repository'], 'https://repo') + self.assertEqual(results[0]['code_language'], 'cpp') + + json = """ + {"toto":[ + {"id":200,"name":"Artist Name", + "link":"http:\/\/www.searchcode_code.com\/artist\/1217","type":"artist"} + ]} + """ + response = mock.Mock(text=json) + results = searchcode_code.response(response) + self.assertEqual(type(results), list) + self.assertEqual(len(results), 0) diff --git a/searx/tests/engines/test_searchcode_doc.py b/searx/tests/engines/test_searchcode_doc.py @@ -0,0 +1,73 @@ +from collections import defaultdict +import mock +from searx.engines import searchcode_doc +from searx.testing import SearxTestCase + + +class TestSearchcodeDocEngine(SearxTestCase): + + def test_request(self): + query = 'test_query' + dicto = defaultdict(dict) + dicto['pageno'] = 0 + params = searchcode_doc.request(query, dicto) + self.assertIn('url', params) + self.assertIn(query, params['url']) + self.assertIn('searchcode.com', params['url']) + + def test_response(self): + self.assertRaises(AttributeError, searchcode_doc.response, None) + self.assertRaises(AttributeError, searchcode_doc.response, []) + self.assertRaises(AttributeError, searchcode_doc.response, '') + self.assertRaises(AttributeError, searchcode_doc.response, '[]') + + response = mock.Mock(text='{}') + self.assertEqual(searchcode_doc.response(response), []) + + response = mock.Mock(text='{"data": []}') + self.assertEqual(searchcode_doc.response(response), []) + + json = """ + { + "matchterm": "test", + "previouspage": null, + "searchterm": "test", + "query": "test", + "total": 60, + "page": 0, + "nextpage": 1, + "results": [ + { + "synopsis": "Synopsis", + "displayname": null, + "name": "test", + "url": "http://url", + "type": "Type", + "icon": null, + "namespace": "Namespace", + "description": "Description" + } + ] + } + """ + response = mock.Mock(text=json) + results = searchcode_doc.response(response) + self.assertEqual(type(results), list) + self.assertEqual(len(results), 1) + self.assertEqual(results[0]['title'], '[Type] Namespace test') + self.assertEqual(results[0]['url'], 'http://url') + self.assertIn('Synopsis', results[0]['content']) + self.assertIn('Type', results[0]['content']) + self.assertIn('test', results[0]['content']) + self.assertIn('Description', results[0]['content']) + + json = """ + {"toto":[ + {"id":200,"name":"Artist Name", + "link":"http:\/\/www.searchcode_doc.com\/artist\/1217","type":"artist"} + ]} + """ + response = mock.Mock(text=json) + results = searchcode_doc.response(response) + self.assertEqual(type(results), list) + self.assertEqual(len(results), 0) diff --git a/searx/tests/engines/test_soundcloud.py b/searx/tests/engines/test_soundcloud.py @@ -0,0 +1,192 @@ +from collections import defaultdict +import mock +from searx.engines import soundcloud +from searx.testing import SearxTestCase +from urllib import quote_plus + + +class TestSoundcloudEngine(SearxTestCase): + + def test_request(self): + query = 'test_query' + dicto = defaultdict(dict) + dicto['pageno'] = 1 + params = soundcloud.request(query, dicto) + self.assertIn('url', params) + self.assertIn(query, params['url']) + self.assertIn('soundcloud.com', params['url']) + + def test_response(self): + self.assertRaises(AttributeError, soundcloud.response, None) + self.assertRaises(AttributeError, soundcloud.response, []) + self.assertRaises(AttributeError, soundcloud.response, '') + self.assertRaises(AttributeError, soundcloud.response, '[]') + + response = mock.Mock(text='{}') + self.assertEqual(soundcloud.response(response), []) + + response = mock.Mock(text='{"data": []}') + self.assertEqual(soundcloud.response(response), []) + + json = """ + { + "collection": [ + { + "kind": "track", + "id": 159723640, + "created_at": "2014/07/22 00:51:21 +0000", + "user_id": 2976616, + "duration": 303780, + "commentable": true, + "state": "finished", + "original_content_size": 13236349, + "last_modified": "2015/01/31 15:14:50 +0000", + "sharing": "public", + "tag_list": "seekae flume", + "permalink": "seekae-test-recognise-flume-re-work", + "streamable": true, + "embeddable_by": "all", + "downloadable": true, + "purchase_url": "http://www.facebook.com/seekaemusic", + "label_id": null, + "purchase_title": "Seekae", + "genre": "freedownload", + "title": "This is the title", + "description": "This is the content", + "label_name": "Future Classic", + "release": "", + "track_type": "remix", + "key_signature": "", + "isrc": "", + "video_url": null, + "bpm": null, + "release_year": 2014, + "release_month": 7, + "release_day": 22, + "original_format": "mp3", + "license": "all-rights-reserved", + "uri": "https://api.soundcloud.com/tracks/159723640", + "user": { + "id": 2976616, + "kind": "user", + "permalink": "flume", + "username": "Flume", + "last_modified": "2014/11/24 19:21:29 +0000", + "uri": "https://api.soundcloud.com/users/2976616", + "permalink_url": "http://soundcloud.com/flume", + "avatar_url": "https://i1.sndcdn.com/avatars-000044475439-4zi7ii-large.jpg" + }, + "permalink_url": "http://soundcloud.com/this.is.the.url", + "artwork_url": "https://i1.sndcdn.com/artworks-000085857162-xdxy5c-large.jpg", + "waveform_url": "https://w1.sndcdn.com/DWrL1lAN8BkP_m.png", + "stream_url": "https://api.soundcloud.com/tracks/159723640/stream", + "download_url": "https://api.soundcloud.com/tracks/159723640/download", + "playback_count": 2190687, + "download_count": 54856, + "favoritings_count": 49061, + "comment_count": 826, + "likes_count": 49061, + "reposts_count": 15910, + "attachments_uri": "https://api.soundcloud.com/tracks/159723640/attachments", + "policy": "ALLOW" + } + ], + "total_results": 375750, + "next_href": "https://api.soundcloud.com/search?&q=test", + "tx_id": "" + } + """ + response = mock.Mock(text=json) + results = soundcloud.response(response) + self.assertEqual(type(results), list) + self.assertEqual(len(results), 1) + self.assertEqual(results[0]['title'], 'This is the title') + self.assertEqual(results[0]['url'], 'http://soundcloud.com/this.is.the.url') + self.assertEqual(results[0]['content'], 'This is the content') + self.assertIn(quote_plus('https://api.soundcloud.com/tracks/159723640'), results[0]['embedded']) + + json = """ + { + "collection": [ + { + "kind": "user", + "id": 159723640, + "created_at": "2014/07/22 00:51:21 +0000", + "user_id": 2976616, + "duration": 303780, + "commentable": true, + "state": "finished", + "original_content_size": 13236349, + "last_modified": "2015/01/31 15:14:50 +0000", + "sharing": "public", + "tag_list": "seekae flume", + "permalink": "seekae-test-recognise-flume-re-work", + "streamable": true, + "embeddable_by": "all", + "downloadable": true, + "purchase_url": "http://www.facebook.com/seekaemusic", + "label_id": null, + "purchase_title": "Seekae", + "genre": "freedownload", + "title": "This is the title", + "description": "This is the content", + "label_name": "Future Classic", + "release": "", + "track_type": "remix", + "key_signature": "", + "isrc": "", + "video_url": null, + "bpm": null, + "release_year": 2014, + "release_month": 7, + "release_day": 22, + "original_format": "mp3", + "license": "all-rights-reserved", + "uri": "https://api.soundcloud.com/tracks/159723640", + "user": { + "id": 2976616, + "kind": "user", + "permalink": "flume", + "username": "Flume", + "last_modified": "2014/11/24 19:21:29 +0000", + "uri": "https://api.soundcloud.com/users/2976616", + "permalink_url": "http://soundcloud.com/flume", + "avatar_url": "https://i1.sndcdn.com/avatars-000044475439-4zi7ii-large.jpg" + }, + "permalink_url": "http://soundcloud.com/this.is.the.url", + "artwork_url": "https://i1.sndcdn.com/artworks-000085857162-xdxy5c-large.jpg", + "waveform_url": "https://w1.sndcdn.com/DWrL1lAN8BkP_m.png", + "stream_url": "https://api.soundcloud.com/tracks/159723640/stream", + "download_url": "https://api.soundcloud.com/tracks/159723640/download", + "playback_count": 2190687, + "download_count": 54856, + "favoritings_count": 49061, + "comment_count": 826, + "likes_count": 49061, + "reposts_count": 15910, + "attachments_uri": "https://api.soundcloud.com/tracks/159723640/attachments", + "policy": "ALLOW" + } + ], + "total_results": 375750, + "next_href": "https://api.soundcloud.com/search?&q=test", + "tx_id": "" + } + """ + response = mock.Mock(text=json) + results = soundcloud.response(response) + self.assertEqual(type(results), list) + self.assertEqual(len(results), 0) + + json = """ + { + "collection": [], + "total_results": 375750, + "next_href": "https://api.soundcloud.com/search?&q=test", + "tx_id": "" + } + """ + response = mock.Mock(text=json) + results = soundcloud.response(response) + self.assertEqual(type(results), list) + self.assertEqual(len(results), 0) diff --git a/searx/tests/engines/test_stackoverflow.py b/searx/tests/engines/test_stackoverflow.py @@ -0,0 +1,106 @@ +from collections import defaultdict +import mock +from searx.engines import stackoverflow +from searx.testing import SearxTestCase + + +class TestStackoverflowEngine(SearxTestCase): + + def test_request(self): + query = 'test_query' + dicto = defaultdict(dict) + dicto['pageno'] = 0 + params = stackoverflow.request(query, dicto) + self.assertTrue('url' in params) + self.assertTrue(query in params['url']) + self.assertTrue('stackoverflow.com' in params['url']) + + def test_response(self): + self.assertRaises(AttributeError, stackoverflow.response, None) + self.assertRaises(AttributeError, stackoverflow.response, []) + self.assertRaises(AttributeError, stackoverflow.response, '') + self.assertRaises(AttributeError, stackoverflow.response, '[]') + + response = mock.Mock(text='<html></html>') + self.assertEqual(stackoverflow.response(response), []) + + html = """ + <div class="question-summary search-result" id="answer-id-1783426"> + <div class="statscontainer"> + <div class="statsarrow"></div> + <div class="stats"> + <div class="vote"> + <div class="votes answered"> + <span class="vote-count-post "><strong>2583</strong></span> + <div class="viewcount">votes</div> + </div> + </div> + </div> + </div> + <div class="summary"> + <div class="result-link"> + <span> + <a href="/questions/this.is.the.url" + data-searchsession="/questions" + title="Checkout remote Git branch"> + This is the title + </a> + </span> + </div> + <div class="excerpt"> + This is the content + </div> + <div class="tags user-tags t-git t-git-checkout t-remote-branch"> + </div> + <div class="started fr"> + answered <span title="2009-11-23 14:26:08Z" class="relativetime">nov 23 '09</span> by + <a href="/users/214090/hallski">hallski</a> + </div> + </div> + </div> + """ + response = mock.Mock(text=html) + results = stackoverflow.response(response) + self.assertEqual(type(results), list) + self.assertEqual(len(results), 1) + self.assertEqual(results[0]['title'], 'This is the title') + self.assertEqual(results[0]['url'], 'http://stackoverflow.com/questions/this.is.the.url') + self.assertEqual(results[0]['content'], 'This is the content') + + html = """ + <div class="statscontainer"> + <div class="statsarrow"></div> + <div class="stats"> + <div class="vote"> + <div class="votes answered"> + <span class="vote-count-post "><strong>2583</strong></span> + <div class="viewcount">votes</div> + </div> + </div> + </div> + </div> + <div class="summary"> + <div class="result-link"> + <span> + <a href="/questions/this.is.the.url" + data-searchsession="/questions" + title="Checkout remote Git branch"> + This is the title + </a> + </span> + </div> + <div class="excerpt"> + This is the content + </div> + <div class="tags user-tags t-git t-git-checkout t-remote-branch"> + </div> + <div class="started fr"> + answered <span title="2009-11-23 14:26:08Z" class="relativetime">nov 23 '09</span> by + <a href="/users/214090/hallski">hallski</a> + </div> + </div> + """ + response = mock.Mock(text=html) + results = stackoverflow.response(response) + self.assertEqual(type(results), list) + self.assertEqual(len(results), 0) diff --git a/searx/tests/engines/test_vimeo.py b/searx/tests/engines/test_vimeo.py @@ -0,0 +1,84 @@ +from collections import defaultdict +import mock +from searx.engines import vimeo +from searx.testing import SearxTestCase + + +class TestVimeoEngine(SearxTestCase): + + def test_request(self): + query = 'test_query' + dicto = defaultdict(dict) + dicto['pageno'] = 0 + params = vimeo.request(query, dicto) + self.assertTrue('url' in params) + self.assertTrue(query in params['url']) + self.assertTrue('vimeo.com' in params['url']) + + def test_response(self): + self.assertRaises(AttributeError, vimeo.response, None) + self.assertRaises(AttributeError, vimeo.response, []) + self.assertRaises(AttributeError, vimeo.response, '') + self.assertRaises(AttributeError, vimeo.response, '[]') + + response = mock.Mock(text='<html></html>') + self.assertEqual(vimeo.response(response), []) + + html = """ + <div id="browse_content" class="" data-search-id="696d5f8366914ec4ffec33cf7652de384976d4f4"> + <ol class="js-browse_list clearfix browse browse_videos browse_videos_thumbnails kane" + data-stream="c2VhcmNoOjo6ZGVzYzp7InF1ZXJ5IjoidGVzdCJ9"> + <li id="clip_100785455" data-start-page="/search/page:1/sort:relevant/" data-position="1"> + <a href="/videoid" title="Futurama 3d (test shot)"> + <img src="http://image.url.webp" + srcset="http://i.vimeocdn.com/video/482375085_590x332.webp 2x" alt="" + class="thumbnail thumbnail_lg_wide"> + <div class="data"> + <p class="title"> + This is the title + </p> + <p class="meta"> + <time datetime="2014-07-15T04:16:27-04:00" + title="mardi 15 juillet 2014 04:16">Il y a 6 mois</time> + </p> + </div> + </a> + </li> + </ol> + </div> + """ + response = mock.Mock(text=html) + results = vimeo.response(response) + self.assertEqual(type(results), list) + self.assertEqual(len(results), 1) + self.assertEqual(results[0]['title'], 'This is the title') + self.assertEqual(results[0]['url'], 'http://vimeo.com/videoid') + self.assertEqual(results[0]['content'], '') + self.assertEqual(results[0]['thumbnail'], 'http://image.url.webp') + self.assertIn('/videoid', results[0]['embedded']) + + html = """ + <ol class="js-browse_list clearfix browse browse_videos browse_videos_thumbnails kane" + data-stream="c2VhcmNoOjo6ZGVzYzp7InF1ZXJ5IjoidGVzdCJ9"> + <li id="clip_100785455" data-start-page="/search/page:1/sort:relevant/" data-position="1"> + <a href="/videoid" title="Futurama 3d (test shot)"> + <img src="http://image.url.webp" + srcset="http://i.vimeocdn.com/video/482375085_590x332.webp 2x" alt="" + class="thumbnail thumbnail_lg_wide"> + <div class="data"> + <p class="title"> + This is the title + </p> + <p class="meta"> + <time datetime="2014-07-15T04:16:27-04:00" + title="mardi 15 juillet 2014 04:16">Il y a 6 mois</time> + </p> + </div> + </a> + </li> + </ol> + """ + response = mock.Mock(text=html) + results = vimeo.response(response) + self.assertEqual(type(results), list) + self.assertEqual(len(results), 0) diff --git a/searx/tests/engines/test_www500px.py b/searx/tests/engines/test_www500px.py @@ -0,0 +1,83 @@ +# -*- coding: utf-8 -*- +from collections import defaultdict +import mock +from searx.engines import www500px +from searx.testing import SearxTestCase + + +class TestWww500pxImagesEngine(SearxTestCase): + + def test_request(self): + query = 'test_query' + dicto = defaultdict(dict) + dicto['pageno'] = 1 + params = www500px.request(query, dicto) + self.assertTrue('url' in params) + self.assertTrue(query in params['url']) + self.assertTrue('500px.com' in params['url']) + + def test_response(self): + self.assertRaises(AttributeError, www500px.response, None) + self.assertRaises(AttributeError, www500px.response, []) + self.assertRaises(AttributeError, www500px.response, '') + self.assertRaises(AttributeError, www500px.response, '[]') + + response = mock.Mock(text='<html></html>') + self.assertEqual(www500px.response(response), []) + + html = """ + <div class="photo"> + <a href="/this.should.be.the.url" data-ga-category="Photo Thumbnail" data-ga-action="Title"> + <img src="https://image.url/3.jpg?v=0" /> + </a> + <div class="details"> + <div class="inside"> + <div class="title"> + <a href="/photo/64312705/branch-out-by-oliver-turpin?feature="> + This is the title + </a> + </div> + <div class="info"> + <a href="/ChronicleUK" data-ga-action="Image" data-ga-category="Photo Thumbnail"> + This is the content + </a> + </div> + <div class="rating">44.8</div> + </div> + </div> + </div> + """ + response = mock.Mock(text=html) + results = www500px.response(response) + self.assertEqual(type(results), list) + self.assertEqual(len(results), 1) + self.assertEqual(results[0]['title'], 'This is the title') + self.assertEqual(results[0]['url'], 'https://500px.com/this.should.be.the.url') + self.assertEqual(results[0]['content'], 'This is the content') + self.assertEqual(results[0]['thumbnail_src'], 'https://image.url/3.jpg?v=0') + self.assertEqual(results[0]['img_src'], 'https://image.url/2048.jpg') + + html = """ + <a href="/this.should.be.the.url" data-ga-category="Photo Thumbnail" data-ga-action="Title"> + <img src="https://image.url/3.jpg?v=0" /> + </a> + <div class="details"> + <div class="inside"> + <div class="title"> + <a href="/photo/64312705/branch-out-by-oliver-turpin?feature="> + This is the title + </a> + </div> + <div class="info"> + <a href="/ChronicleUK" data-ga-action="Image" data-ga-category="Photo Thumbnail"> + Oliver Turpin + </a> + </div> + <div class="rating">44.8</div> + </div> + </div> + """ + response = mock.Mock(text=html) + results = www500px.response(response) + self.assertEqual(type(results), list) + self.assertEqual(len(results), 0) diff --git a/searx/tests/engines/test_youtube.py b/searx/tests/engines/test_youtube.py @@ -0,0 +1,204 @@ +from collections import defaultdict +import mock +from searx.engines import youtube +from searx.testing import SearxTestCase + + +class TestYoutubeEngine(SearxTestCase): + + def test_request(self): + query = 'test_query' + dicto = defaultdict(dict) + dicto['pageno'] = 0 + dicto['language'] = 'fr_FR' + params = youtube.request(query, dicto) + self.assertTrue('url' in params) + self.assertTrue(query in params['url']) + self.assertTrue('youtube.com' in params['url']) + self.assertTrue('fr' in params['url']) + + dicto['language'] = 'all' + params = youtube.request(query, dicto) + self.assertFalse('fr' in params['url']) + + def test_response(self): + self.assertRaises(AttributeError, youtube.response, None) + self.assertRaises(AttributeError, youtube.response, []) + self.assertRaises(AttributeError, youtube.response, '') + self.assertRaises(AttributeError, youtube.response, '[]') + + response = mock.Mock(text='{}') + self.assertEqual(youtube.response(response), []) + + response = mock.Mock(text='{"data": []}') + self.assertEqual(youtube.response(response), []) + + json = """ + {"feed":{"entry":[{ + "id":{"$t":"http://gdata.youtube.com/feeds/api/videos/DIVZCPfAOeM"}, + "published":{"$t":"2015-01-23T21:25:00.000Z"}, + "updated":{"$t":"2015-01-26T14:38:15.000Z"}, + "title":{"$t":"Title", + "type":"text"},"content":{"$t":"Description","type":"text"}, + "link":[{"rel":"alternate","type":"text/html", + "href":"https://www.youtube.com/watch?v=DIVZCPfAOeM&feature=youtube_gdata"}, + {"rel":"http://gdata.youtube.com/schemas/2007#video.related", + "type":"application/atom+xml", + "href":"https://gdata.youtube.com/feeds/api/videos/DIVZCPfAOeM/related"}, + {"rel":"http://gdata.youtube.com/schemas/2007#mobile","type":"text/html", + "href":"https://m.youtube.com/details?v=DIVZCPfAOeM"}, + {"rel":"self","type":"application/atom+xml", + "href":"https://gdata.youtube.com/feeds/api/videos/DIVZCPfAOeM"}], + "author":[{"name":{"$t":"Cauet"}, + "uri":{"$t":"https://gdata.youtube.com/feeds/api/users/cauetofficiel"} }], + "gd$comments":{"gd$feedLink":{"rel":"http://gdata.youtube.com/schemas/2007#comments", + "href":"https://gdata.youtube.com/feeds/api/videos/DIVZCPfAOeM/comments", + "countHint":8} }, + "media$group":{"media$category":[{"$t":"Comedy","label":"Comedy", + "scheme":"http://gdata.youtube.com/schemas/2007/categories.cat"}], + "media$content":[{"url":"https://www.youtube.com/v/DIVZCPfAOeM?version=3&f=videos&app=youtube_gdata", + "type":"application/x-shockwave-flash","medium":"video", + "isDefault":"true","expression":"full","duration":354,"yt$format":5}, + {"url":"rtsp://r1---sn-cg07luel.c.youtube.com/CiILENy73wIaGQnjOcD3CFmFDBMYDSANFEgGUgZ2aWRlb3MM/0/0/0/video.3gp", + "type":"video/3gpp","medium":"video","expression":"full","duration":354, + "yt$format":1}, + {"url":"rtsp://r1---sn-cg07luel.c.youtube.com/CiILENy73wIaGQnjOcD3CFmFDBMYESARFEgGUgZ2aWRlb3MM/0/0/0/video.3gp", + "type":"video/3gpp","medium":"video","expression":"full","duration":354,"yt$format":6}], + "media$description":{"$t":"Desc","type":"plain"}, + "media$keywords":{}, + "media$player":[{"url":"https://www.youtube.com/watch?v=DIVZCPfAOeM&feature=youtube_gdata_player"}], + "media$thumbnail":[{"url":"https://i.ytimg.com/vi/DIVZCPfAOeM/0.jpg", + "height":360,"width":480,"time":"00:02:57"}, + {"url":"https://i.ytimg.com/vi/DIVZCPfAOeM/1.jpg","height":90,"width":120,"time":"00:01:28.500"}, + {"url":"https://i.ytimg.com/vi/DIVZCPfAOeM/2.jpg","height":90,"width":120,"time":"00:02:57"}, + {"url":"https://i.ytimg.com/vi/DIVZCPfAOeM/3.jpg","height":90,"width":120,"time":"00:04:25.500"}], + "media$title":{"$t":"Title","type":"plain"}, + "yt$duration":{"seconds":"354"} }, + "gd$rating":{"average":4.932159,"max":5,"min":1,"numRaters":1533, + "rel":"http://schemas.google.com/g/2005#overall"}, + "yt$statistics":{"favoriteCount":"0","viewCount":"92464"} } + ] + } + } + """ + response = mock.Mock(text=json) + results = youtube.response(response) + self.assertEqual(type(results), list) + self.assertEqual(len(results), 1) + self.assertEqual(results[0]['title'], 'Title') + self.assertEqual(results[0]['url'], 'https://www.youtube.com/watch?v=DIVZCPfAOeM') + self.assertEqual(results[0]['content'], 'Description') + self.assertEqual(results[0]['thumbnail'], 'https://i.ytimg.com/vi/DIVZCPfAOeM/0.jpg') + self.assertTrue('DIVZCPfAOeM' in results[0]['embedded']) + + json = """ + {"feed":{"entry":[{ + "id":{"$t":"http://gdata.youtube.com/feeds/api/videos/DIVZCPfAOeM"}, + "published":{"$t":"2015-01-23T21:25:00.000Z"}, + "updated":{"$t":"2015-01-26T14:38:15.000Z"}, + "title":{"$t":"Title", + "type":"text"},"content":{"$t":"Description","type":"text"}, + "link":[{"rel":"http://gdata.youtube.com/schemas/2007#video.related", + "type":"application/atom+xml", + "href":"https://gdata.youtube.com/feeds/api/videos/DIVZCPfAOeM/related"}, + {"rel":"self","type":"application/atom+xml", + "href":"https://gdata.youtube.com/feeds/api/videos/DIVZCPfAOeM"}], + "author":[{"name":{"$t":"Cauet"}, + "uri":{"$t":"https://gdata.youtube.com/feeds/api/users/cauetofficiel"} }], + "gd$comments":{"gd$feedLink":{"rel":"http://gdata.youtube.com/schemas/2007#comments", + "href":"https://gdata.youtube.com/feeds/api/videos/DIVZCPfAOeM/comments", + "countHint":8} }, + "media$group":{"media$category":[{"$t":"Comedy","label":"Comedy", + "scheme":"http://gdata.youtube.com/schemas/2007/categories.cat"}], + "media$content":[{"url":"https://www.youtube.com/v/DIVZCPfAOeM?version=3&f=videos&app=youtube_gdata", + "type":"application/x-shockwave-flash","medium":"video", + "isDefault":"true","expression":"full","duration":354,"yt$format":5}, + {"url":"rtsp://r1---sn-cg07luel.c.youtube.com/CiILENy73wIaGQnjOcD3CFmFDBMYDSANFEgGUgZ2aWRlb3MM/0/0/0/video.3gp", + "type":"video/3gpp","medium":"video","expression":"full","duration":354, + "yt$format":1}, + {"url":"rtsp://r1---sn-cg07luel.c.youtube.com/CiILENy73wIaGQnjOcD3CFmFDBMYESARFEgGUgZ2aWRlb3MM/0/0/0/video.3gp", + "type":"video/3gpp","medium":"video","expression":"full","duration":354,"yt$format":6}], + "media$description":{"$t":"Desc","type":"plain"}, + "media$keywords":{}, + "media$player":[{"url":"https://www.youtube.com/watch?v=DIVZCPfAOeM&feature=youtube_gdata_player"}], + "media$thumbnail":[{"url":"https://i.ytimg.com/vi/DIVZCPfAOeM/0.jpg", + "height":360,"width":480,"time":"00:02:57"}, + {"url":"https://i.ytimg.com/vi/DIVZCPfAOeM/1.jpg","height":90,"width":120,"time":"00:01:28.500"}, + {"url":"https://i.ytimg.com/vi/DIVZCPfAOeM/2.jpg","height":90,"width":120,"time":"00:02:57"}, + {"url":"https://i.ytimg.com/vi/DIVZCPfAOeM/3.jpg","height":90,"width":120,"time":"00:04:25.500"}], + "media$title":{"$t":"Title","type":"plain"}, + "yt$duration":{"seconds":"354"} }, + "gd$rating":{"average":4.932159,"max":5,"min":1,"numRaters":1533, + "rel":"http://schemas.google.com/g/2005#overall"}, + "yt$statistics":{"favoriteCount":"0","viewCount":"92464"} } + ] + } + } + """ + response = mock.Mock(text=json) + results = youtube.response(response) + self.assertEqual(type(results), list) + self.assertEqual(len(results), 0) + + json = """ + {"feed":{"entry":[{ + "id":{"$t":"http://gdata.youtube.com/feeds/api/videos/DIVZCPfAOeM"}, + "published":{"$t":"2015-01-23T21:25:00.000Z"}, + "updated":{"$t":"2015-01-26T14:38:15.000Z"}, + "title":{"$t":"Title", + "type":"text"},"content":{"$t":"Description","type":"text"}, + "link":[{"rel":"alternate","type":"text/html", + "href":"https://www.youtube.com/watch?v=DIVZCPfAOeM"}, + {"rel":"http://gdata.youtube.com/schemas/2007#video.related", + "type":"application/atom+xml", + "href":"https://gdata.youtube.com/feeds/api/videos/DIVZCPfAOeM/related"}, + {"rel":"http://gdata.youtube.com/schemas/2007#mobile","type":"text/html", + "href":"https://m.youtube.com/details?v=DIVZCPfAOeM"}, + {"rel":"self","type":"application/atom+xml", + "href":"https://gdata.youtube.com/feeds/api/videos/DIVZCPfAOeM"}], + "author":[{"name":{"$t":"Cauet"}, + "uri":{"$t":"https://gdata.youtube.com/feeds/api/users/cauetofficiel"} }], + "gd$comments":{"gd$feedLink":{"rel":"http://gdata.youtube.com/schemas/2007#comments", + "href":"https://gdata.youtube.com/feeds/api/videos/DIVZCPfAOeM/comments", + "countHint":8} }, + "media$group":{"media$category":[{"$t":"Comedy","label":"Comedy", + "scheme":"http://gdata.youtube.com/schemas/2007/categories.cat"}], + "media$content":[{"url":"https://www.youtube.com/v/DIVZCPfAOeM?version=3&f=videos&app=youtube_gdata", + "type":"application/x-shockwave-flash","medium":"video", + "isDefault":"true","expression":"full","duration":354,"yt$format":5}, + {"url":"rtsp://r1---sn-cg07luel.c.youtube.com/CiILENy73wIaGQnjOcD3CFmFDBMYDSANFEgGUgZ2aWRlb3MM/0/0/0/video.3gp", + "type":"video/3gpp","medium":"video","expression":"full","duration":354, + "yt$format":1}, + {"url":"rtsp://r1---sn-cg07luel.c.youtube.com/CiILENy73wIaGQnjOcD3CFmFDBMYESARFEgGUgZ2aWRlb3MM/0/0/0/video.3gp", + "type":"video/3gpp","medium":"video","expression":"full","duration":354,"yt$format":6}], + "media$description":{"$t":"Desc","type":"plain"}, + "media$keywords":{}, + "media$player":[{"url":"https://www.youtube.com/watch?v=DIVZCPfAOeM&feature=youtube_gdata_player"}], + "media$title":{"$t":"Title","type":"plain"}, + "yt$duration":{"seconds":"354"} }, + "gd$rating":{"average":4.932159,"max":5,"min":1,"numRaters":1533, + "rel":"http://schemas.google.com/g/2005#overall"}, + "yt$statistics":{"favoriteCount":"0","viewCount":"92464"} } + ] + } + } + """ + response = mock.Mock(text=json) + results = youtube.response(response) + self.assertEqual(type(results), list) + self.assertEqual(len(results), 1) + self.assertEqual(results[0]['title'], 'Title') + self.assertEqual(results[0]['url'], 'https://www.youtube.com/watch?v=DIVZCPfAOeM') + self.assertEqual(results[0]['content'], 'Description') + self.assertEqual(results[0]['thumbnail'], '') + self.assertTrue('DIVZCPfAOeM' in results[0]['embedded']) + + json = """ + {"toto":{"entry":[] + } + } + """ + response = mock.Mock(text=json) + results = youtube.response(response) + self.assertEqual(type(results), list) + self.assertEqual(len(results), 0) diff --git a/searx/tests/test_engines.py b/searx/tests/test_engines.py @@ -1,3 +1,25 @@ +from searx.tests.engines.test_bing import * # noqa +from searx.tests.engines.test_bing_images import * # noqa +from searx.tests.engines.test_bing_news import * # noqa +from searx.tests.engines.test_btdigg import * # noqa +from searx.tests.engines.test_dailymotion import * # noqa +from searx.tests.engines.test_deezer import * # noqa +from searx.tests.engines.test_deviantart import * # noqa +from searx.tests.engines.test_digg import * # noqa from searx.tests.engines.test_dummy import * # noqa +from searx.tests.engines.test_flickr import * # noqa +from searx.tests.engines.test_flickr_noapi import * # noqa from searx.tests.engines.test_github import * # noqa from searx.tests.engines.test_www1x import * # noqa +from searx.tests.engines.test_google_images import * # noqa +from searx.tests.engines.test_google_news import * # noqa +from searx.tests.engines.test_kickass import * # noqa +from searx.tests.engines.test_mixcloud import * # noqa +from searx.tests.engines.test_piratebay import * # noqa +from searx.tests.engines.test_searchcode_code import * # noqa +from searx.tests.engines.test_searchcode_doc import * # noqa +from searx.tests.engines.test_soundcloud import * # noqa +from searx.tests.engines.test_stackoverflow import * # noqa +from searx.tests.engines.test_vimeo import * # noqa +from searx.tests.engines.test_www500px import * # noqa +from searx.tests.engines.test_youtube import * # noqa diff --git a/searx/tests/test_utils.py b/searx/tests/test_utils.py @@ -10,6 +10,11 @@ class TestUtils(SearxTestCase): self.assertIsNotNone(utils.gen_useragent()) self.assertTrue(utils.gen_useragent().startswith('Mozilla')) + def test_searx_useragent(self): + self.assertIsInstance(utils.searx_useragent(), str) + self.assertIsNotNone(utils.searx_useragent()) + self.assertTrue(utils.searx_useragent().startswith('searx')) + def test_highlight_content(self): self.assertEqual(utils.highlight_content(0, None), None) self.assertEqual(utils.highlight_content(None, None), None) @@ -29,6 +34,23 @@ class TestUtils(SearxTestCase): query = 'a test' self.assertEqual(utils.highlight_content(content, query), content) + def test_html_to_text(self): + html = """ + <a href="/testlink" class="link_access_account"> + <span class="toto"> + <span> + <img src="test.jpg" /> + </span> + </span> + <span class="titi"> + Test text + </span> + </a> + """ + self.assertIsInstance(utils.html_to_text(html), unicode) + self.assertIsNotNone(utils.html_to_text(html)) + self.assertEqual(utils.html_to_text(html), "Test text") + class TestHTMLTextExtractor(SearxTestCase): diff --git a/searx/utils.py b/searx/utils.py @@ -115,10 +115,12 @@ class HTMLTextExtractor(HTMLParser): self.result.append(name) def get_text(self): - return u''.join(self.result) + return u''.join(self.result).strip() def html_to_text(html): + html = html.replace('\n', ' ') + html = ' '.join(html.split()) s = HTMLTextExtractor() s.feed(html) return s.get_text()