commit: d5ec0f43e465e1bd016f1c841122fc3c774fc718
parent: 2440e74b1eb75832e3a187e4130b4df212a4dc06
Author: asciimoo <asciimoo@gmail.com>
Date: Tue, 15 Oct 2013 18:19:06 +0200
[mod] search refactor
Diffstat:
2 files changed, 40 insertions(+), 34 deletions(-)
diff --git a/searx/engines/__init__.py b/searx/engines/__init__.py
@@ -2,6 +2,7 @@
from os.path import realpath, dirname, splitext, join
from os import listdir
from imp import load_source
+import grequests
engine_dir = dirname(realpath(__file__))
@@ -12,4 +13,40 @@ for filename in listdir(engine_dir):
if filename.startswith('_') or not filename.endswith('.py'):
continue
filepath = join(engine_dir, filename)
- engines.append(load_source(modname, filepath))
+ engine = load_source(modname, filepath)
+ if not hasattr(engine, 'request') or not hasattr(engine, 'response'):
+ continue
+ engines.append(engine)
+
+def default_request_params():
+ return {'method': 'GET', 'headers': {}, 'data': {}, 'url': ''}
+
+def make_callback(results, callback):
+ def process_callback(response, **kwargs):
+ results.extend(callback(response))
+ return process_callback
+
+def search(query, request):
+ global engines
+ requests = []
+ results = []
+ user_agent = request.headers.get('User-Agent', '')
+ for engine in engines:
+ headers = default_request_params()
+ headers['User-Agent'] = user_agent
+ request_params = engine.request(query, headers)
+ callback = make_callback(results, engine.response)
+ if request_params['method'] == 'GET':
+ req = grequests.get(request_params['url']
+ ,headers=headers
+ ,hooks=dict(response=callback)
+ )
+ else:
+ req = grequests.post(request_params['url']
+ ,data=request_params['data']
+ ,headers=headers
+ ,hooks=dict(response=callback)
+ )
+ requests.append(req)
+ grequests.map(requests)
+ return results
diff --git a/searx/webapp.py b/searx/webapp.py
@@ -25,8 +25,7 @@ if __name__ == "__main__":
from flask import Flask, request, flash, render_template
import ConfigParser
from os import getenv
-from searx.engines import engines
-import grequests
+from searx.engines import search
cfg = ConfigParser.SafeConfigParser()
cfg.read('/etc/searx.conf')
@@ -38,14 +37,6 @@ cfg.read('searx.conf')
app = Flask(__name__)
app.secret_key = cfg.get('app', 'secret_key')
-def default_request_params():
- return {'method': 'GET', 'headers': {}, 'data': {}, 'url': ''}
-
-def make_callback(results, callback):
- def process_callback(response, **kwargs):
- results.extend(callback(response))
- return process_callback
-
@app.route('/', methods=['GET', 'POST'])
def index():
if request.method=='POST':
@@ -53,30 +44,8 @@ def index():
flash('Wrong post data')
return render_template('index.html')
query = request.form['q']
- requests = []
- results = []
- user_agent = request.headers.get('User-Agent', '')
- for engine in engines:
- headers = default_request_params()
- headers['User-Agent'] = user_agent
- request_params = engine.request(query, headers)
- callback = make_callback(results, engine.response)
- if request_params['method'] == 'GET':
- req = grequests.get(request_params['url']
- ,headers=headers
- ,hooks=dict(response=callback)
- )
- else:
- req = grequests.post(request_params['url']
- ,data=request_params['data']
- ,headers=headers
- ,hooks=dict(response=callback)
- )
- requests.append(req)
- grequests.map(requests)
+ results = search(query, request)
return render_template('results.html', results=results, q=query)
-
-
return render_template('index.html')
if __name__ == "__main__":