commit: 78d3f3d6b116e0aca8af8b939d5a98ea74bb8de3
parent 45b5073f3e10cad68ddda6a1a83d3273a3d734fc
Author: a01200356 <a01200356@itesm.mx>
Date: Sat, 27 Feb 2016 19:06:44 -0600
[enh] infobox for wolframalpha
TODO:
- infobox styles
- unit tests
ISSUES:
- no_api version needs to re-call server for additional pods, such
as plots. therefore, it's even slower than before. comment out the
part that calls get_async_pod if requests reach timeout or increase
timeout in settings.yml.
Diffstat:
3 files changed, 150 insertions(+), 46 deletions(-)
diff --git a/searx/engines/wolframalpha_api.py b/searx/engines/wolframalpha_api.py
@@ -1,40 +1,56 @@
-# Wolfram Alpha (Maths)
+# Wolfram Alpha (Science)
#
-# @website http://www.wolframalpha.com
-# @provide-api yes (http://api.wolframalpha.com/v2/)
+# @website https://www.wolframalpha.com
+# @provide-api yes (https://api.wolframalpha.com/v2/)
#
# @using-api yes
# @results XML
# @stable yes
-# @parse result
+# @parse url, infobox
from urllib import urlencode
from lxml import etree
-from re import search
# search-url
-base_url = 'http://api.wolframalpha.com/v2/query'
-search_url = base_url + '?appid={api_key}&{query}&format=plaintext'
-site_url = 'http://www.wolframalpha.com/input/?{query}'
+search_url = 'https://api.wolframalpha.com/v2/query?appid={api_key}&{query}'
+site_url = 'https://www.wolframalpha.com/input/?{query}'
api_key = '' # defined in settings.yml
# xpath variables
failure_xpath = '/queryresult[attribute::success="false"]'
answer_xpath = '//pod[attribute::primary="true"]/subpod/plaintext'
input_xpath = '//pod[starts-with(attribute::title, "Input")]/subpod/plaintext'
+pods_xpath = '//pod'
+subpods_xpath = './subpod'
+pod_title_xpath = './@title'
+plaintext_xpath = './plaintext'
+image_xpath = './img'
+img_src_xpath = './@src'
+img_alt_xpath = './@alt'
+
+# pods to display as image in infobox
+# this pods do return a plaintext, but they look better and are more useful as images
+image_pods = {'Visual representation',
+ 'Manipulatives illustration'}
# do search-request
def request(query, params):
params['url'] = search_url.format(query=urlencode({'input': query}),
api_key=api_key)
+ params['headers']['Referer'] = site_url.format(query=urlencode({'i': query}))
return params
# replace private user area characters to make text legible
def replace_pua_chars(text):
- pua_chars = {u'\uf74c': 'd',
+ pua_chars = {u'\uf522': u'\u2192',
+ u'\uf7b1': u'\u2115',
+ u'\uf7b4': u'\u211a',
+ u'\uf7b5': u'\u211d',
+ u'\uf7bd': u'\u2124',
+ u'\uf74c': 'd',
u'\uf74d': u'\u212f',
u'\uf74e': 'i',
u'\uf7d9': '='}
@@ -55,23 +71,45 @@ def response(resp):
if search_results.xpath(failure_xpath):
return []
- # parse answers
- answers = search_results.xpath(answer_xpath)
- if answers:
- for answer in answers:
- answer = replace_pua_chars(answer.text)
+ infobox_title = search_results.xpath(input_xpath)
+ if infobox_title:
+ infobox_title = replace_pua_chars(infobox_title[0].text)
+
+ pods = search_results.xpath(pods_xpath)
+ result_chunks = []
+ for pod in pods:
+ pod_title = replace_pua_chars(pod.xpath(pod_title_xpath)[0])
+
+ subpods = pod.xpath(subpods_xpath)
+ if not subpods:
+ continue
+
+ for subpod in subpods:
+ content = subpod.xpath(plaintext_xpath)[0].text
+ image = subpod.xpath(image_xpath)
+ if content and pod_title not in image_pods:
+ content = replace_pua_chars(content)
+ result_chunks.append({'label': pod_title, 'value': content})
- results.append({'answer': answer})
+ # if there's no input pod, infobox_title is content of first pod
+ if not infobox_title:
+ infobox_title = content
+
+ elif image:
+ result_chunks.append({'label': pod_title,
+ 'image': {'src': image[0].xpath(img_src_xpath)[0],
+ 'alt': image[0].xpath(img_alt_xpath)[0]}})
+
+ if not result_chunks:
+ return []
- # if there's no input section in search_results, check if answer has the input embedded (before their "=" sign)
- try:
- query_input = search_results.xpath(input_xpath)[0].text
- except IndexError:
- query_input = search(u'([^\uf7d9]+)', answers[0].text).group(1)
+ results.append({'infobox': infobox_title,
+ 'attributes': result_chunks,
+ 'urls': [{'title': 'Wolfram|Alpha', 'url': resp.request.headers['Referer']}]})
# append link to site
- result_url = site_url.format(query=urlencode({'i': query_input.encode('utf-8')}))
- results.append({'url': result_url,
- 'title': query_input + " - Wolfram|Alpha"})
+ results.append({'url': resp.request.headers['Referer'],
+ 'title': 'Wolfram|Alpha',
+ 'content': infobox_title})
return results
diff --git a/searx/engines/wolframalpha_noapi.py b/searx/engines/wolframalpha_noapi.py
@@ -1,23 +1,23 @@
-# WolframAlpha (Maths)
+# Wolfram|Alpha (Science)
#
-# @website http://www.wolframalpha.com/
-# @provide-api yes (http://api.wolframalpha.com/v2/)
+# @website https://www.wolframalpha.com/
+# @provide-api yes (https://api.wolframalpha.com/v2/)
#
# @using-api no
-# @results HTML
+# @results JSON
# @stable no
-# @parse answer
+# @parse url, infobox
from cgi import escape
from json import loads
from time import time
from urllib import urlencode
+from lxml.etree import XML
from searx.poolrequests import get as http_get
# search-url
url = 'https://www.wolframalpha.com/'
-search_url = url + 'input/?{query}'
search_url = url + 'input/json.jsp'\
'?async=true'\
@@ -33,13 +33,25 @@ search_url = url + 'input/json.jsp'\
'&sponsorcategories=true'\
'&statemethod=deploybutton'
-# xpath variables
-scripts_xpath = '//script'
-title_xpath = '//title'
-failure_xpath = '//p[attribute::class="pfail"]'
+referer_url = url + 'input/?{query}'
+
token = {'value': '',
'last_updated': None}
+# xpath variables
+success_xpath = '/pod[attribute::error="false"]'
+plaintext_xpath = './plaintext'
+title_xpath = './@title'
+image_xpath = './img'
+img_src_xpath = './img/@src'
+img_alt_xpath = './img/@alt'
+
+# pods to display as image in infobox
+# this pods do return a plaintext, but they look better and are more useful as images
+image_pods = {'Visual representation',
+ 'Manipulatives illustration',
+ 'Symbol'}
+
# seems, wolframalpha resets its token in every hour
def obtain_token():
@@ -62,13 +74,42 @@ def request(query, params):
if time() - token['last_updated'] > 3600:
obtain_token()
params['url'] = search_url.format(query=urlencode({'input': query}), token=token['value'])
- params['headers']['Referer'] = 'https://www.wolframalpha.com/input/?i=' + query
+ params['headers']['Referer'] = referer_url.format(query=urlencode({'i': query}))
return params
+# get additional pod
+# NOTE: this makes an additional requests to server, so the response will take longer and might reach timeout
+def get_async_pod(url):
+ pod = {'subpods': []}
+
+ try:
+ resp = http_get(url, timeout=2.0)
+
+ resp_pod = XML(resp.content)
+ if resp_pod.xpath(success_xpath):
+
+ for subpod in resp_pod:
+ plaintext = subpod.xpath(plaintext_xpath)[0].text
+ if plaintext:
+ pod['subpods'].append({'title': subpod.xpath(title_xpath)[0],
+ 'plaintext': plaintext})
+ elif subpod.xpath(image_xpath):
+ pod['subpods'].append({'title': subpod.xpath(title_xpath)[0],
+ 'plaintext': '',
+ 'img': {'src': subpod.xpath(img_src_xpath)[0],
+ 'alt': subpod.xpath(img_alt_xpath)[0]}})
+ except:
+ pass
+
+ return pod
+
+
# get response from search-request
def response(resp):
+ results = []
+
resp_json = loads(resp.text)
if not resp_json['queryresult']['success']:
@@ -76,20 +117,45 @@ def response(resp):
# TODO handle resp_json['queryresult']['assumptions']
result_chunks = []
+ infobox_title = None
for pod in resp_json['queryresult']['pods']:
pod_title = pod.get('title', '')
+
if 'subpods' not in pod:
- continue
+ # comment this section if your requests always reach timeout
+ if pod['async']:
+ result = get_async_pod(pod['async'])
+ if result:
+ pod = result
+ else:
+ continue
+
+ # infobox title is input or text content on first pod
+ if pod_title.startswith('Input') or not infobox_title:
+ try:
+ infobox_title = pod['subpods'][0]['plaintext']
+ except:
+ infobox_title = ''
+ pass
+
for subpod in pod['subpods']:
- if 'img' in subpod:
- result_chunks.append(u'<p>{0}<br /><img src="{1}" alt="{2}" /></p>'
- .format(escape(pod_title or subpod['img']['alt']),
- escape(subpod['img']['src']),
- escape(subpod['img']['alt'])))
+ if subpod['plaintext'] != '' and pod_title not in image_pods:
+ # append unless it's not an actual answer
+ if subpod['plaintext'] != '(requires interactivity)':
+ result_chunks.append({'label': pod_title, 'value': subpod['plaintext']})
+
+ elif 'img' in subpod:
+ result_chunks.append({'label': pod_title, 'image': subpod['img']})
if not result_chunks:
return []
- return [{'url': resp.request.headers['Referer'].decode('utf-8'),
- 'title': 'Wolframalpha',
- 'content': ''.join(result_chunks)}]
+ results.append({'infobox': infobox_title,
+ 'attributes': result_chunks,
+ 'urls': [{'title': 'Wolfram|Alpha', 'url': resp.request.headers['Referer']}]})
+
+ results.append({'url': resp.request.headers['Referer'],
+ 'title': 'Wolfram|Alpha',
+ 'content': infobox_title})
+
+ return results
diff --git a/searx/settings.yml b/searx/settings.yml
@@ -310,10 +310,10 @@ engines:
shortcut : wa
# You can use the engine using the official stable API, but you need an API key
# See : http://products.wolframalpha.com/api/
- # engine : wolframalpha_api
- # api_key: 'apikey' # required!
+ # engine : wolframalpha_api
+ # api_key: '5952JX-X52L3VKWT8' # required!
engine : wolframalpha_noapi
- timeout: 6.0
+ timeout: 10.0
categories : science
#The blekko technology and team have joined IBM Watson! -> https://blekko.com/
