logo

searx

Unnamed repository; edit this file 'description' to name the repository.
commit: e58949b76fac7aa93341523ff0e2f35e0a03e057
parent: 6b6007fc78f47099d84099c6d6c23e2cd213b82f
Author: Adam Tauber <asciimoo@gmail.com>
Date:   Mon, 12 Dec 2016 10:06:43 +0100

Merge pull request #783 from kvch/time-range-search-year

add year support to engines which support it

Diffstat:

searx/engines/bing_images.py | 3++-
searx/engines/bing_news.py | 3+++
searx/engines/deviantart.py | 3+++
searx/engines/duckduckgo.py | 3+++
searx/engines/flickr_noapi.py | 3++-
searx/engines/google.py | 3++-
searx/engines/google_images.py | 10+++++++++-
searx/engines/google_news.py | 3++-
searx/engines/yahoo.py | 3+++
searx/engines/youtube_noapi.py | 3++-
searx/templates/oscar/time-range.html | 3+++
tests/unit/engines/test_bing_news.py | 7+++++++
tests/unit/engines/test_deviantart.py | 9++++++++-
tests/unit/engines/test_duckduckgo.py | 7+++++++
tests/unit/engines/test_yahoo.py | 7+++++++
tests/unit/engines/test_youtube_noapi.py | 19+++++++++++++++++++
16 files changed, 82 insertions(+), 7 deletions(-)

diff --git a/searx/engines/bing_images.py b/searx/engines/bing_images.py @@ -33,7 +33,8 @@ time_range_string = '&qft=+filterui:age-lt{interval}' thumb_url = "https://www.bing.com/th?id={ihk}" time_range_dict = {'day': '1440', 'week': '10080', - 'month': '43200'} + 'month': '43200', + 'year': '525600'} # safesearch definitions safesearch_types = {2: 'STRICT', diff --git a/searx/engines/bing_news.py b/searx/engines/bing_news.py @@ -66,6 +66,9 @@ def _get_url(query, language, offset, time_range): # do search-request def request(query, params): + if params['time_range'] and params['time_range'] not in time_range_dict: + return params + offset = (params['pageno'] - 1) * 10 + 1 if params['language'] == 'all': diff --git a/searx/engines/deviantart.py b/searx/engines/deviantart.py @@ -34,6 +34,9 @@ time_range_dict = {'day': 11, # do search-request def request(query, params): + if params['time_range'] and params['time_range'] not in time_range_dict: + return params + offset = (params['pageno'] - 1) * 24 params['url'] = search_url.format(offset=offset, diff --git a/searx/engines/duckduckgo.py b/searx/engines/duckduckgo.py @@ -41,6 +41,9 @@ content_xpath = './/a[@class="result__snippet"]' # do search-request def request(query, params): + if params['time_range'] and params['time_range'] not in time_range_dict: + return params + offset = (params['pageno'] - 1) * 30 if params['language'] == 'all': diff --git a/searx/engines/flickr_noapi.py b/searx/engines/flickr_noapi.py @@ -34,7 +34,8 @@ paging = True time_range_support = True time_range_dict = {'day': 60 * 60 * 24, 'week': 60 * 60 * 24 * 7, - 'month': 60 * 60 * 24 * 7 * 4} + 'month': 60 * 60 * 24 * 7 * 4, + 'year': 60 * 60 * 24 * 7 * 52} def build_flickr_url(user_id, photo_id): diff --git a/searx/engines/google.py b/searx/engines/google.py @@ -95,7 +95,8 @@ search_url = ('https://{hostname}' + time_range_search = "&tbs=qdr:{range}" time_range_dict = {'day': 'd', 'week': 'w', - 'month': 'm'} + 'month': 'm', + 'year': 'y'} # other URLs map_hostname_start = 'maps.google.' diff --git a/searx/engines/google_images.py b/searx/engines/google_images.py @@ -10,10 +10,12 @@ @parse url, title, img_src """ +from datetime import date, timedelta from urllib import urlencode from json import loads from lxml import html + # engine dependent config categories = ['images'] paging = True @@ -29,6 +31,7 @@ search_url = 'https://www.google.com/search'\ '&yv=2'\ '&{search_options}' time_range_attr = "qdr:{range}" +time_range_custom_attr = "cdr:1,cd_min:{start},cd_max{end}" time_range_dict = {'day': 'd', 'week': 'w', 'month': 'm'} @@ -36,7 +39,6 @@ time_range_dict = {'day': 'd', # do search-request def request(query, params): - search_options = { 'ijn': params['pageno'] - 1, 'start': (params['pageno'] - 1) * number_of_results @@ -44,6 +46,12 @@ def request(query, params): if params['time_range'] in time_range_dict: search_options['tbs'] = time_range_attr.format(range=time_range_dict[params['time_range']]) + elif params['time_range'] == 'year': + now = date.today() + then = now - timedelta(days=365) + start = then.strftime('%m/%d/%Y') + end = now.strftime('%m/%d/%Y') + search_options['tbs'] = time_range_custom_attr.format(start=start, end=end) if safesearch and params['safesearch']: search_options['safe'] = 'on' diff --git a/searx/engines/google_news.py b/searx/engines/google_news.py @@ -29,7 +29,8 @@ search_url = 'https://www.google.com/search'\ time_range_attr = "qdr:{range}" time_range_dict = {'day': 'd', 'week': 'w', - 'month': 'm'} + 'month': 'm', + 'year': 'y'} # do search-request diff --git a/searx/engines/yahoo.py b/searx/engines/yahoo.py @@ -77,6 +77,9 @@ def _get_language(params): # do search-request def request(query, params): + if params['time_range'] and params['time_range'] not in time_range_dict: + return params + offset = (params['pageno'] - 1) * 10 + 1 language = _get_language(params) diff --git a/searx/engines/youtube_noapi.py b/searx/engines/youtube_noapi.py @@ -25,7 +25,8 @@ search_url = base_url + '?search_query={query}&page={page}' time_range_url = '&sp=EgII{time_range}%253D%253D' time_range_dict = {'day': 'Ag', 'week': 'Aw', - 'month': 'BA'} + 'month': 'BA', + 'year': 'BQ'} embedded_url = '<iframe width="540" height="304" ' +\ 'data-src="//www.youtube-nocookie.com/embed/{videoid}" ' +\ diff --git a/searx/templates/oscar/time-range.html b/searx/templates/oscar/time-range.html @@ -11,4 +11,7 @@ <option id="time-range-month" value="month" {{ "selected" if time_range=="month" else ""}}> {{ _('Last month') }} </option> + <option id="time-range-year" value="year" {{ "selected" if time_range=="year" else ""}}> + {{ _('Last year') }} + </option> </select> diff --git a/tests/unit/engines/test_bing_news.py b/tests/unit/engines/test_bing_news.py @@ -23,6 +23,13 @@ class TestBingNewsEngine(SearxTestCase): params = bing_news.request(query, dicto) self.assertIn('en', params['url']) + def test_no_url_in_request_year_time_range(self): + dicto = defaultdict(dict) + query = 'test_query' + dicto['time_range'] = 'year' + params = bing_news.request(query, dicto) + self.assertEqual({}, params['url']) + def test_response(self): self.assertRaises(AttributeError, bing_news.response, None) self.assertRaises(AttributeError, bing_news.response, []) diff --git a/tests/unit/engines/test_deviantart.py b/tests/unit/engines/test_deviantart.py @@ -7,8 +7,8 @@ from searx.testing import SearxTestCase class TestDeviantartEngine(SearxTestCase): def test_request(self): - query = 'test_query' dicto = defaultdict(dict) + query = 'test_query' dicto['pageno'] = 0 dicto['time_range'] = '' params = deviantart.request(query, dicto) @@ -16,6 +16,13 @@ class TestDeviantartEngine(SearxTestCase): self.assertTrue(query in params['url']) self.assertTrue('deviantart.com' in params['url']) + def test_no_url_in_request_year_time_range(self): + dicto = defaultdict(dict) + query = 'test_query' + dicto['time_range'] = 'year' + params = deviantart.request(query, dicto) + self.assertEqual({}, params['url']) + def test_response(self): self.assertRaises(AttributeError, deviantart.response, None) self.assertRaises(AttributeError, deviantart.response, []) diff --git a/tests/unit/engines/test_duckduckgo.py b/tests/unit/engines/test_duckduckgo.py @@ -19,6 +19,13 @@ class TestDuckduckgoEngine(SearxTestCase): self.assertIn('duckduckgo.com', params['url']) self.assertIn('ch-de', params['url']) + def test_no_url_in_request_year_time_range(self): + dicto = defaultdict(dict) + query = 'test_query' + dicto['time_range'] = 'year' + params = duckduckgo.request(query, dicto) + self.assertEqual({}, params['url']) + def test_response(self): self.assertRaises(AttributeError, duckduckgo.response, None) self.assertRaises(AttributeError, duckduckgo.response, []) diff --git a/tests/unit/engines/test_yahoo.py b/tests/unit/engines/test_yahoo.py @@ -46,6 +46,13 @@ class TestYahooEngine(SearxTestCase): self.assertIn('en', params['cookies']['sB']) self.assertIn('en', params['url']) + def test_no_url_in_request_year_time_range(self): + dicto = defaultdict(dict) + query = 'test_query' + dicto['time_range'] = 'year' + params = yahoo.request(query, dicto) + self.assertEqual({}, params['url']) + def test_response(self): self.assertRaises(AttributeError, yahoo.response, None) self.assertRaises(AttributeError, yahoo.response, []) diff --git a/tests/unit/engines/test_youtube_noapi.py b/tests/unit/engines/test_youtube_noapi.py @@ -17,6 +17,25 @@ class TestYoutubeNoAPIEngine(SearxTestCase): self.assertIn(query, params['url']) self.assertIn('youtube.com', params['url']) + def test_time_range_search(self): + dicto = defaultdict(dict) + query = 'test_query' + dicto['time_range'] = 'year' + params = youtube_noapi.request(query, dicto) + self.assertIn('&sp=EgIIBQ%253D%253D', params['url']) + + dicto['time_range'] = 'month' + params = youtube_noapi.request(query, dicto) + self.assertIn('&sp=EgIIBA%253D%253D', params['url']) + + dicto['time_range'] = 'week' + params = youtube_noapi.request(query, dicto) + self.assertIn('&sp=EgIIAw%253D%253D', params['url']) + + dicto['time_range'] = 'day' + params = youtube_noapi.request(query, dicto) + self.assertIn('&sp=EgIIAg%253D%253D', params['url']) + def test_response(self): self.assertRaises(AttributeError, youtube_noapi.response, None) self.assertRaises(AttributeError, youtube_noapi.response, [])