logo

searx

Unnamed repository; edit this file 'description' to name the repository.
commit: c65a409f0d2728ba5a0c3ffa1a0cb05659033a71
parent: e60e98156f9aeeb40ca7272c883704c095d9f7d4
Author: marc <a01200356@itesm.mx>
Date:   Sat, 20 May 2017 22:33:08 -0500

add duckduckgo images engine

Diffstat:

searx/engines/duckduckgo.py | 65+++++++++++++++++++++++++++++++++++------------------------------
searx/engines/duckduckgo_images.py | 91+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
searx/settings.yml | 6++++++
tests/unit/engines/test_duckduckgo_images.py | 72++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
4 files changed, 204 insertions(+), 30 deletions(-)

diff --git a/searx/engines/duckduckgo.py b/searx/engines/duckduckgo.py @@ -41,46 +41,51 @@ title_xpath = './/a[@class="result__a"]' content_xpath = './/a[@class="result__snippet"]' -# do search-request -def request(query, params): - if params['time_range'] and params['time_range'] not in time_range_dict: - return params - - offset = 30 + (params['pageno'] - 1) * 50 - dc_param = offset + 1 - +# match query's language to a region code that duckduckgo will accept +def get_region_code(lang): # custom fixes for languages - if params['language'] == 'all': - locale = None - elif params['language'][:2] == 'ja': - locale = 'jp-jp' - elif params['language'][:2] == 'sl': - locale = 'sl-sl' - elif params['language'] == 'zh-TW': - locale = 'tw-tzh' - elif params['language'] == 'zh-HK': - locale = 'hk-tzh' - elif params['language'][-2:] == 'SA': - locale = 'xa-' + params['language'].split('-')[0] - elif params['language'][-2:] == 'GB': - locale = 'uk-' + params['language'].split('-')[0] + if lang == 'all': + region_code = None + elif lang[:2] == 'ja': + region_code = 'jp-jp' + elif lang[:2] == 'sl': + region_code = 'sl-sl' + elif lang == 'zh-TW': + region_code = 'tw-tzh' + elif lang == 'zh-HK': + region_code = 'hk-tzh' + elif lang[-2:] == 'SA': + region_code = 'xa-' + lang.split('-')[0] + elif lang[-2:] == 'GB': + region_code = 'uk-' + lang.split('-')[0] else: - locale = params['language'].split('-') - if len(locale) == 2: + region_code = lang.split('-') + if len(region_code) == 2: # country code goes first - locale = locale[1].lower() + '-' + locale[0].lower() + region_code = region_code[1].lower() + '-' + region_code[0].lower() else: # tries to get a country code from language - locale = locale[0].lower() + region_code = region_code[0].lower() for lc in supported_languages: lc = lc.split('-') - if locale == lc[0]: - locale = lc[1].lower() + '-' + lc[0].lower() + if region_code == lc[0]: + region_code = lc[1].lower() + '-' + lc[0].lower() break + return region_code + + +# do search-request +def request(query, params): + if params['time_range'] and params['time_range'] not in time_range_dict: + return params + + offset = 30 + (params['pageno'] - 1) * 50 + dc_param = offset + 1 - if locale: + region_code = get_region_code(params['language']) + if region_code: params['url'] = url.format( - query=urlencode({'q': query, 'kl': locale}), offset=offset, dc_param=dc_param) + query=urlencode({'q': query, 'kl': region_code}), offset=offset, dc_param=dc_param) else: params['url'] = url.format( query=urlencode({'q': query}), offset=offset, dc_param=dc_param) diff --git a/searx/engines/duckduckgo_images.py b/searx/engines/duckduckgo_images.py @@ -0,0 +1,91 @@ +""" + DuckDuckGo (Images) + + @website https://duckduckgo.com/ + @provide-api yes (https://duckduckgo.com/api), + but images are not supported + + @using-api no + @results JSON (site requires js to get images) + @stable no (JSON can change) + @parse url, title, img_src + + @todo avoid extra request +""" + +from requests import get +from json import loads +from searx.engines.xpath import extract_text +from searx.engines.duckduckgo import _fetch_supported_languages, supported_languages_url, get_region_code +from searx.url_utils import urlencode + +# engine dependent config +categories = ['images'] +paging = True +language_support = True +safesearch = True + +# search-url +images_url = 'https://duckduckgo.com/i.js?{query}&s={offset}&p={safesearch}&o=json&vqd={vqd}' +site_url = 'https://duckduckgo.com/?{query}&iar=images&iax=1&ia=images' + + +# run query in site to get vqd number needed for requesting images +# TODO: find a way to get this number without an extra request (is it a hash of the query?) +def get_vqd(query): + res = get(site_url.format(query=urlencode({'q': query}))) + content = res.text + vqd = content[content.find('vqd=\'') + 5:] + vqd = vqd[:vqd.find('\'')] + return vqd + + +# do search-request +def request(query, params): + # to avoid running actual external requests when testing + if 'is_test' not in params: + vqd = get_vqd(query) + else: + vqd = '12345' + + offset = (params['pageno'] - 1) * 50 + + safesearch = params['safesearch'] - 1 + + region_code = get_region_code(params['language']) + if region_code: + params['url'] = images_url.format( + query=urlencode({'q': query, 'l': region_code}), offset=offset, safesearch=safesearch, vqd=vqd) + else: + params['url'] = images_url.format( + query=urlencode({'q': query}), offset=offset, safesearch=safesearch, vqd=vqd) + + return params + + +# get response from search-request +def response(resp): + results = [] + + content = resp.text + try: + res_json = loads(content) + except: + return [] + + # parse results + for result in res_json['results']: + title = result['title'] + url = result['url'] + thumbnail = result['thumbnail'] + image = result['image'] + + # append result + results.append({'template': 'images.html', + 'title': title, + 'content': '', + 'thumbnail_src': thumbnail, + 'img_src': image, + 'url': url}) + + return results diff --git a/searx/settings.yml b/searx/settings.yml @@ -167,6 +167,12 @@ engines: shortcut : ddg disabled : True + - name : duckduckgo images + engine : duckduckgo_images + shortcut : ddi + timeout: 3.0 + disabled : True + - name : etymonline engine : xpath paging : True diff --git a/tests/unit/engines/test_duckduckgo_images.py b/tests/unit/engines/test_duckduckgo_images.py @@ -0,0 +1,72 @@ +# -*- coding: utf-8 -*- +from collections import defaultdict +import mock +from searx.engines import duckduckgo_images +from searx.testing import SearxTestCase + + +class TestDuckduckgoImagesEngine(SearxTestCase): + + def test_request(self): + query = 'test_query' + dicto = defaultdict(dict) + dicto['is_test'] = True + dicto['pageno'] = 1 + dicto['safesearch'] = 0 + dicto['language'] = 'all' + params = duckduckgo_images.request(query, dicto) + self.assertIn('url', params) + self.assertIn(query, params['url']) + self.assertIn('duckduckgo.com', params['url']) + self.assertIn('s=0', params['url']) + self.assertIn('p=-1', params['url']) + self.assertIn('vqd=12345', params['url']) + + # test paging and safe search + dicto['pageno'] = 2 + dicto['safesearch'] = 2 + params = duckduckgo_images.request(query, dicto) + self.assertIn('url', params) + self.assertIn(query, params['url']) + self.assertIn('s=50', params['url']) + self.assertIn('p=1', params['url']) + + def test_response(self): + self.assertRaises(AttributeError, duckduckgo_images.response, None) + self.assertRaises(AttributeError, duckduckgo_images.response, []) + self.assertRaises(AttributeError, duckduckgo_images.response, '') + self.assertRaises(AttributeError, duckduckgo_images.response, '[]') + + response = mock.Mock(text='If this error persists, please let us know: ops@duckduckgo.com') + self.assertEqual(duckduckgo_images.response(response), []) + + json = u""" + { + "query": "test_query", + "results": [ + { + "title": "Result 1", + "url": "https://site1.url", + "thumbnail": "https://thumb1.nail", + "image": "https://image1" + }, + { + "title": "Result 2", + "url": "https://site2.url", + "thumbnail": "https://thumb2.nail", + "image": "https://image2" + } + ] + } + """ + response = mock.Mock(text=json) + results = duckduckgo_images.response(response) + self.assertEqual(len(results), 2) + self.assertEqual(results[0]['title'], 'Result 1') + self.assertEqual(results[0]['url'], 'https://site1.url') + self.assertEqual(results[0]['thumbnail_src'], 'https://thumb1.nail') + self.assertEqual(results[0]['img_src'], 'https://image1') + self.assertEqual(results[1]['title'], 'Result 2') + self.assertEqual(results[1]['url'], 'https://site2.url') + self.assertEqual(results[1]['thumbnail_src'], 'https://thumb2.nail') + self.assertEqual(results[1]['img_src'], 'https://image2')