commit: 7c13d630e4531630ce3c392a7d60752715742291
parent ffcec383b7355c6ca8b60da8579a43019d7d7e6b
Author: dalf <alex@al-f.net>
Date: Sun, 7 Dec 2014 16:37:56 +0100
[fix] pep8 : engines (errors E121, E127, E128 and E501 still exist)
Diffstat:
22 files changed, 109 insertions(+), 97 deletions(-)
diff --git a/searx/engines/bing.py b/searx/engines/bing.py
@@ -1,8 +1,9 @@
## Bing (Web)
-#
+#
# @website https://www.bing.com
-# @provide-api yes (http://datamarket.azure.com/dataset/bing/search), max. 5000 query/month
-#
+# @provide-api yes (http://datamarket.azure.com/dataset/bing/search),
+# max. 5000 query/month
+#
# @using-api no (because of query limit)
# @results HTML (using search portal)
# @stable no (HTML can change)
@@ -58,8 +59,8 @@ def response(resp):
content = escape(' '.join(result.xpath('.//p//text()')))
# append result
- results.append({'url': url,
- 'title': title,
+ results.append({'url': url,
+ 'title': title,
'content': content})
# return results if something is found
@@ -74,8 +75,8 @@ def response(resp):
content = escape(' '.join(result.xpath('.//p//text()')))
# append result
- results.append({'url': url,
- 'title': title,
+ results.append({'url': url,
+ 'title': title,
'content': content})
# return results
diff --git a/searx/engines/bing_images.py b/searx/engines/bing_images.py
@@ -1,17 +1,19 @@
## Bing (Images)
-#
+#
# @website https://www.bing.com/images
-# @provide-api yes (http://datamarket.azure.com/dataset/bing/search), max. 5000 query/month
-#
+# @provide-api yes (http://datamarket.azure.com/dataset/bing/search),
+# max. 5000 query/month
+#
# @using-api no (because of query limit)
# @results HTML (using search portal)
# @stable no (HTML can change)
# @parse url, title, img_src
#
-# @todo currently there are up to 35 images receive per page, because bing does not parse count=10. limited response to 10 images
+# @todo currently there are up to 35 images receive per page,
+# because bing does not parse count=10.
+# limited response to 10 images
from urllib import urlencode
-from cgi import escape
from lxml import html
from yaml import load
import re
@@ -51,15 +53,15 @@ def response(resp):
dom = html.fromstring(resp.content)
# init regex for yaml-parsing
- p = re.compile( '({|,)([a-z]+):(")')
+ p = re.compile('({|,)([a-z]+):(")')
# parse results
for result in dom.xpath('//div[@class="dg_u"]'):
link = result.xpath('./a')[0]
# parse yaml-data (it is required to add a space, to make it parsable)
- yaml_data = load(p.sub( r'\1\2: \3', link.attrib.get('m')))
-
+ yaml_data = load(p.sub(r'\1\2: \3', link.attrib.get('m')))
+
title = link.attrib.get('t1')
#url = 'http://' + link.attrib.get('t3')
url = yaml_data.get('surl')
@@ -69,7 +71,7 @@ def response(resp):
results.append({'template': 'images.html',
'url': url,
'title': title,
- 'content': '',
+ 'content': '',
'img_src': img_src})
# TODO stop parsing if 10 images are found
diff --git a/searx/engines/bing_news.py b/searx/engines/bing_news.py
@@ -1,8 +1,9 @@
## Bing (News)
-#
+#
# @website https://www.bing.com/news
-# @provide-api yes (http://datamarket.azure.com/dataset/bing/search), max. 5000 query/month
-#
+# @provide-api yes (http://datamarket.azure.com/dataset/bing/search),
+# max. 5000 query/month
+#
# @using-api no (because of query limit)
# @results HTML (using search portal)
# @stable no (HTML can change)
@@ -57,12 +58,12 @@ def response(resp):
url = link.attrib.get('href')
title = ' '.join(link.xpath('.//text()'))
contentXPath = result.xpath('.//div[@class="sn_txt"]/div//span[@class="sn_snip"]//text()')
- if contentXPath != None:
+ if contentXPath is not None:
content = escape(' '.join(contentXPath))
-
+
# parse publishedDate
publishedDateXPath = result.xpath('.//div[@class="sn_txt"]/div//span[contains(@class,"sn_ST")]//span[contains(@class,"sn_tm")]//text()')
- if publishedDateXPath != None:
+ if publishedDateXPath is not None:
publishedDate = escape(' '.join(publishedDateXPath))
if re.match("^[0-9]+ minute(s|) ago$", publishedDate):
@@ -89,10 +90,10 @@ def response(resp):
except TypeError:
# FIXME
publishedDate = datetime.now()
-
+
# append result
- results.append({'url': url,
- 'title': title,
+ results.append({'url': url,
+ 'title': title,
'publishedDate': publishedDate,
'content': content})
diff --git a/searx/engines/currency_convert.py b/searx/engines/currency_convert.py
@@ -55,6 +55,6 @@ def response(resp):
resp.search_params['to'].lower()
)
- results.append({'answer' : answer, 'url': url})
+ results.append({'answer': answer, 'url': url})
return results
diff --git a/searx/engines/dailymotion.py b/searx/engines/dailymotion.py
@@ -1,8 +1,8 @@
## Dailymotion (Videos)
-#
+#
# @website https://www.dailymotion.com
# @provide-api yes (http://www.dailymotion.com/developer)
-#
+#
# @using-api yes
# @results JSON
# @stable yes
@@ -12,7 +12,6 @@
from urllib import urlencode
from json import loads
-from lxml import html
# engine dependent config
categories = ['videos']
diff --git a/searx/engines/deviantart.py b/searx/engines/deviantart.py
@@ -1,8 +1,8 @@
## Deviantart (Images)
-#
+#
# @website https://www.deviantart.com/
# @provide-api yes (https://www.deviantart.com/developers/) (RSS)
-#
+#
# @using-api no (TODO, rewrite to api)
# @results HTML
# @stable no (HTML can change)
diff --git a/searx/engines/duckduckgo.py b/searx/engines/duckduckgo.py
@@ -1,15 +1,17 @@
## DuckDuckGo (Web)
-#
+#
# @website https://duckduckgo.com/
-# @provide-api yes (https://duckduckgo.com/api), but not all results from search-site
-#
+# @provide-api yes (https://duckduckgo.com/api),
+# but not all results from search-site
+#
# @using-api no
# @results HTML (using search portal)
# @stable no (HTML can change)
# @parse url, title, content
#
# @todo rewrite to api
-# @todo language support (the current used site does not support language-change)
+# @todo language support
+# (the current used site does not support language-change)
from urllib import urlencode
from lxml.html import fromstring
@@ -37,7 +39,7 @@ def request(query, params):
if params['language'] == 'all':
locale = 'en-us'
else:
- locale = params['language'].replace('_','-').lower()
+ locale = params['language'].replace('_', '-').lower()
params['url'] = url.format(
query=urlencode({'q': query, 'kl': locale}),
diff --git a/searx/engines/dummy.py b/searx/engines/dummy.py
@@ -1,5 +1,5 @@
## Dummy
-#
+#
# @results empty array
# @stable yes
diff --git a/searx/engines/faroo.py b/searx/engines/faroo.py
@@ -1,8 +1,8 @@
## Faroo (Web, News)
-#
+#
# @website http://www.faroo.com
# @provide-api yes (http://www.faroo.com/hp/api/api.html), require API-key
-#
+#
# @using-api yes
# @results JSON
# @stable yes
@@ -24,9 +24,10 @@ api_key = None
url = 'http://www.faroo.com/'
search_url = url + 'api?{query}&start={offset}&length={number_of_results}&l={language}&src={categorie}&i=false&f=json&key={api_key}'
-search_category = {'general': 'web',
+search_category = {'general': 'web',
'news': 'news'}
+
# do search-request
def request(query, params):
offset = (params['pageno']-1) * number_of_results + 1
@@ -48,7 +49,7 @@ def request(query, params):
query=urlencode({'q': query}),
language=language,
categorie=categorie,
- api_key=api_key )
+ api_key=api_key)
# using searx User-Agent
params['headers']['User-Agent'] = searx_useragent()
@@ -101,7 +102,7 @@ def response(resp):
results.append({'template': 'images.html',
'url': result['url'],
'title': result['title'],
- 'content': result['kwic'],
+ 'content': result['kwic'],
'img_src': result['iurl']})
# return results
diff --git a/searx/engines/generalfile.py b/searx/engines/generalfile.py
@@ -1,8 +1,8 @@
## General Files (Files)
-#
+#
# @website http://www.general-files.org
# @provide-api no (nothing found)
-#
+#
# @using-api no (because nothing found)
# @results HTML (using search portal)
# @stable no (HTML can change)
diff --git a/searx/engines/github.py b/searx/engines/github.py
@@ -1,8 +1,8 @@
## Github (It)
-#
+#
# @website https://github.com/
# @provide-api yes (https://developer.github.com/v3/)
-#
+#
# @using-api yes
# @results JSON
# @stable yes (using api)
diff --git a/searx/engines/google_images.py b/searx/engines/google_images.py
@@ -1,8 +1,9 @@
## Google (Images)
-#
+#
# @website https://www.google.com
-# @provide-api yes (https://developers.google.com/web-search/docs/), deprecated!
-#
+# @provide-api yes (https://developers.google.com/web-search/docs/),
+# deprecated!
+#
# @using-api yes
# @results JSON
# @stable yes (but deprecated)
diff --git a/searx/engines/google_news.py b/searx/engines/google_news.py
@@ -1,8 +1,9 @@
## Google (News)
-#
+#
# @website https://www.google.com
-# @provide-api yes (https://developers.google.com/web-search/docs/), deprecated!
-#
+# @provide-api yes (https://developers.google.com/web-search/docs/),
+# deprecated!
+#
# @using-api yes
# @results JSON
# @stable yes (but deprecated)
diff --git a/searx/engines/openstreetmap.py b/searx/engines/openstreetmap.py
@@ -39,16 +39,16 @@ def response(resp):
url = result_base_url.format(osm_type=osm_type,
osm_id=r['osm_id'])
- osm = {'type':osm_type,
- 'id':r['osm_id']}
+ osm = {'type': osm_type,
+ 'id': r['osm_id']}
- geojson = r.get('geojson')
+ geojson = r.get('geojson')
# if no geojson is found and osm_type is a node, add geojson Point
if not geojson and\
osm_type == 'node':
- geojson = {u'type':u'Point',
- u'coordinates':[r['lon'],r['lat']]}
+ geojson = {u'type': u'Point',
+ u'coordinates': [r['lon'], r['lat']]}
address_raw = r.get('address')
address = {}
@@ -59,20 +59,20 @@ def response(resp):
r['class'] == 'tourism' or\
r['class'] == 'leisure':
if address_raw.get('address29'):
- address = {'name':address_raw.get('address29')}
+ address = {'name': address_raw.get('address29')}
else:
- address = {'name':address_raw.get(r['type'])}
+ address = {'name': address_raw.get(r['type'])}
# add rest of adressdata, if something is already found
if address.get('name'):
- address.update({'house_number':address_raw.get('house_number'),
- 'road':address_raw.get('road'),
- 'locality':address_raw.get('city',
- address_raw.get('town',
- address_raw.get('village'))),
- 'postcode':address_raw.get('postcode'),
- 'country':address_raw.get('country'),
- 'country_code':address_raw.get('country_code')})
+ address.update({'house_number': address_raw.get('house_number'),
+ 'road': address_raw.get('road'),
+ 'locality': address_raw.get('city',
+ address_raw.get('town',
+ address_raw.get('village'))),
+ 'postcode': address_raw.get('postcode'),
+ 'country': address_raw.get('country'),
+ 'country_code': address_raw.get('country_code')})
else:
address = None
diff --git a/searx/engines/piratebay.py b/searx/engines/piratebay.py
@@ -1,8 +1,8 @@
## Piratebay (Videos, Music, Files)
-#
+#
# @website https://thepiratebay.se
# @provide-api no (nothing found)
-#
+#
# @using-api no
# @results HTML (using search portal)
# @stable yes (HTML can change)
@@ -23,7 +23,7 @@ url = 'https://thepiratebay.se/'
search_url = url + 'search/{search_term}/{pageno}/99/{search_type}'
# piratebay specific type-definitions
-search_types = {'files': '0',
+search_types = {'files': '0',
'music': '100',
'videos': '200'}
diff --git a/searx/engines/soundcloud.py b/searx/engines/soundcloud.py
@@ -1,8 +1,8 @@
## Soundcloud (Music)
-#
+#
# @website https://soundcloud.com
# @provide-api yes (https://developers.soundcloud.com/)
-#
+#
# @using-api yes
# @results JSON
# @stable yes
diff --git a/searx/engines/stackoverflow.py b/searx/engines/stackoverflow.py
@@ -1,8 +1,8 @@
## Stackoverflow (It)
-#
+#
# @website https://stackoverflow.com/
# @provide-api not clear (https://api.stackexchange.com/docs/advanced-search)
-#
+#
# @using-api no
# @results HTML
# @stable no (HTML can change)
@@ -50,8 +50,8 @@ def response(resp):
content = escape(' '.join(result.xpath(content_xpath)))
# append result
- results.append({'url': href,
- 'title': title,
+ results.append({'url': href,
+ 'title': title,
'content': content})
# return results
diff --git a/searx/engines/twitter.py b/searx/engines/twitter.py
@@ -1,8 +1,8 @@
## Twitter (Social media)
-#
+#
# @website https://www.bing.com/news
# @provide-api yes (https://dev.twitter.com/docs/using-search)
-#
+#
# @using-api no
# @results HTML (using search portal)
# @stable no (HTML can change)
diff --git a/searx/engines/vimeo.py b/searx/engines/vimeo.py
@@ -1,8 +1,9 @@
## Vimeo (Videos)
-#
+#
# @website https://vimeo.com/
-# @provide-api yes (http://developer.vimeo.com/api), they have a maximum count of queries/hour
-#
+# @provide-api yes (http://developer.vimeo.com/api),
+# they have a maximum count of queries/hour
+#
# @using-api no (TODO, rewrite to api)
# @results HTML (using search portal)
# @stable no (HTML can change)
@@ -35,11 +36,12 @@ publishedDate_xpath = './/p[@class="meta"]//attribute::datetime'
# do search-request
def request(query, params):
- params['url'] = search_url.format(pageno=params['pageno'] ,
+ params['url'] = search_url.format(pageno=params['pageno'],
query=urlencode({'q': query}))
# TODO required?
- params['cookies']['__utma'] = '00000000.000#0000000.0000000000.0000000000.0000000000.0'
+ params['cookies']['__utma'] =\
+ '00000000.000#0000000.0000000000.0000000000.0000000000.0'
return params
diff --git a/searx/engines/yacy.py b/searx/engines/yacy.py
@@ -1,8 +1,9 @@
## Yacy (Web, Images, Videos, Music, Files)
-#
+#
# @website http://yacy.net
-# @provide-api yes (http://www.yacy-websuche.de/wiki/index.php/Dev:APIyacysearch)
-#
+# @provide-api yes
+# (http://www.yacy-websuche.de/wiki/index.php/Dev:APIyacysearch)
+#
# @using-api yes
# @results JSON
# @stable yes
@@ -16,7 +17,7 @@ from urllib import urlencode
from dateutil import parser
# engine dependent config
-categories = ['general', 'images'] #TODO , 'music', 'videos', 'files'
+categories = ['general', 'images'] # TODO , 'music', 'videos', 'files'
paging = True
language_support = True
number_of_results = 5
@@ -28,7 +29,7 @@ search_url = '/yacysearch.json?{query}&startRecord={offset}&maximumRecords={limi
# yacy specific type-definitions
search_types = {'general': 'text',
'images': 'image',
- 'files': 'app',
+ 'files': 'app',
'music': 'audio',
'videos': 'video'}
diff --git a/searx/engines/yahoo.py b/searx/engines/yahoo.py
@@ -1,8 +1,9 @@
## Yahoo (Web)
-#
+#
# @website https://search.yahoo.com/web
-# @provide-api yes (https://developer.yahoo.com/boss/search/), $0.80/1000 queries
-#
+# @provide-api yes (https://developer.yahoo.com/boss/search/),
+# $0.80/1000 queries
+#
# @using-api no (because pricing)
# @results HTML (using search portal)
# @stable no (HTML can change)
@@ -40,8 +41,8 @@ def parse_url(url_string):
if endpos > -1:
endpositions.append(endpos)
- if start==0 or len(endpositions) == 0:
- return url_string
+ if start == 0 or len(endpositions) == 0:
+ return url_string
else:
end = min(endpositions)
return unquote(url_string[start:end])
@@ -84,8 +85,8 @@ def response(resp):
content = extract_text(result.xpath(content_xpath)[0])
# append result
- results.append({'url': url,
- 'title': title,
+ results.append({'url': url,
+ 'title': title,
'content': content})
# if no suggestion found, return results
diff --git a/searx/engines/youtube.py b/searx/engines/youtube.py
@@ -1,8 +1,8 @@
## Youtube (Videos)
-#
+#
# @website https://www.youtube.com/
# @provide-api yes (http://gdata-samples-youtube-search-py.appspot.com/)
-#
+#
# @using-api yes
# @results JSON
# @stable yes
