commit: bd92b43449410fe1c598600b16b0f23c4f6c256e
parent: 4cc79533e7452e4c08a2654f1ee1022307d227b0
Author: Adam Tauber <asciimoo@gmail.com>
Date: Sun, 15 Mar 2015 11:38:07 +0100
Merge branch 'plugins'
Diffstat:
10 files changed, 238 insertions(+), 74 deletions(-)
diff --git a/searx/plugins/__init__.py b/searx/plugins/__init__.py
@@ -0,0 +1,48 @@
+from searx.plugins import self_ip
+from searx import logger
+from sys import exit
+
+logger = logger.getChild('plugins')
+
+required_attrs = (('name', str),
+ ('description', str),
+ ('default_on', bool))
+
+
+class Plugin():
+ default_on = False
+ name = 'Default plugin'
+ description = 'Default plugin description'
+
+
+class PluginStore():
+
+ def __init__(self):
+ self.plugins = []
+
+ def __iter__(self):
+ for plugin in self.plugins:
+ yield plugin
+
+ def register(self, *plugins):
+ for plugin in plugins:
+ for plugin_attr, plugin_attr_type in required_attrs:
+ if not hasattr(plugin, plugin_attr) or not isinstance(getattr(plugin, plugin_attr), plugin_attr_type):
+ logger.critical('missing attribute "{0}", cannot load plugin: {1}'.format(plugin_attr, plugin))
+ exit(3)
+ plugin.id = plugin.name.replace(' ', '_')
+ self.plugins.append(plugin)
+
+ def call(self, plugin_type, request, *args, **kwargs):
+ ret = True
+ for plugin in request.user_plugins:
+ if hasattr(plugin, plugin_type):
+ ret = getattr(plugin, plugin_type)(request, *args, **kwargs)
+ if not ret:
+ break
+
+ return ret
+
+
+plugins = PluginStore()
+plugins.register(self_ip)
diff --git a/searx/plugins/self_ip.py b/searx/plugins/self_ip.py
@@ -0,0 +1,21 @@
+from flask.ext.babel import gettext
+name = "Self IP"
+description = gettext('Display your source IP address if the query expression is "ip"')
+default_on = True
+
+
+# attach callback to the pre search hook
+# request: flask request object
+# ctx: the whole local context of the pre search hook
+def pre_search(request, ctx):
+ if ctx['search'].query == 'ip':
+ x_forwarded_for = request.headers.getlist("X-Forwarded-For")
+ if x_forwarded_for:
+ ip = x_forwarded_for[0]
+ else:
+ ip = request.remote_addr
+ ctx['search'].answers.clear()
+ ctx['search'].answers.add(ip)
+ # return False prevents exeecution of the original block
+ return False
+ return True
diff --git a/searx/search.py b/searx/search.py
@@ -329,8 +329,8 @@ class Search(object):
self.blocked_engines = get_blocked_engines(engines, request.cookies)
self.results = []
- self.suggestions = []
- self.answers = []
+ self.suggestions = set()
+ self.answers = set()
self.infoboxes = []
self.request_data = {}
@@ -429,9 +429,6 @@ class Search(object):
requests = []
results_queue = Queue()
results = {}
- suggestions = set()
- answers = set()
- infoboxes = []
# increase number of searches
number_of_searches += 1
@@ -511,7 +508,7 @@ class Search(object):
selected_engine['name']))
if not requests:
- return results, suggestions, answers, infoboxes
+ return self
# send all search-request
threaded_requests(requests)
@@ -519,19 +516,19 @@ class Search(object):
engine_name, engine_results = results_queue.get_nowait()
# TODO type checks
- [suggestions.add(x['suggestion'])
+ [self.suggestions.add(x['suggestion'])
for x in list(engine_results)
if 'suggestion' in x
and engine_results.remove(x) is None]
- [answers.add(x['answer'])
+ [self.answers.add(x['answer'])
for x in list(engine_results)
if 'answer' in x
and engine_results.remove(x) is None]
- infoboxes.extend(x for x in list(engine_results)
- if 'infobox' in x
- and engine_results.remove(x) is None)
+ self.infoboxes.extend(x for x in list(engine_results)
+ if 'infobox' in x
+ and engine_results.remove(x) is None)
results[engine_name] = engine_results
@@ -541,16 +538,16 @@ class Search(object):
engines[engine_name].stats['result_count'] += len(engine_results)
# score results and remove duplications
- results = score_results(results)
+ self.results = score_results(results)
# merge infoboxes according to their ids
- infoboxes = merge_infoboxes(infoboxes)
+ self.infoboxes = merge_infoboxes(self.infoboxes)
# update engine stats, using calculated score
- for result in results:
+ for result in self.results:
for res_engine in result['engines']:
engines[result['engine']]\
.stats['score_count'] += result['score']
# return results, suggestions, answers and infoboxes
- return results, suggestions, answers, infoboxes
+ return self
diff --git a/searx/settings.yml b/searx/settings.yml
@@ -106,6 +106,7 @@ engines:
- name : gigablast
engine : gigablast
shortcut : gb
+ disabled: True
- name : github
engine : github
diff --git a/searx/templates/oscar/macros.html b/searx/templates/oscar/macros.html
@@ -59,3 +59,11 @@
</div>
{% endif %}
{%- endmacro %}
+
+{% macro checkbox_toggle(id, blocked) -%}
+ <div class="checkbox">
+ <input class="hidden" type="checkbox" id="{{ id }}" name="{{ id }}"{% if blocked %} checked="checked"{% endif %} />
+ <label class="btn btn-success label_hide_if_checked" for="{{ id }}">{{ _('Block') }}</label>
+ <label class="btn btn-danger label_hide_if_not_checked" for="{{ id }}">{{ _('Allow') }}</label>
+ </div>
+{%- endmacro %}
diff --git a/searx/templates/oscar/preferences.html b/searx/templates/oscar/preferences.html
@@ -1,4 +1,4 @@
-{% from 'oscar/macros.html' import preferences_item_header, preferences_item_header_rtl, preferences_item_footer, preferences_item_footer_rtl %}
+{% from 'oscar/macros.html' import preferences_item_header, preferences_item_header_rtl, preferences_item_footer, preferences_item_footer_rtl, checkbox_toggle %}
{% extends "oscar/base.html" %}
{% block title %}{{ _('preferences') }} - {% endblock %}
{% block site_alert_warning_nojs %}
@@ -16,6 +16,7 @@
<ul class="nav nav-tabs nav-justified hide_if_nojs" role="tablist" style="margin-bottom:20px;">
<li class="active"><a href="#tab_general" role="tab" data-toggle="tab">{{ _('General') }}</a></li>
<li><a href="#tab_engine" role="tab" data-toggle="tab">{{ _('Engines') }}</a></li>
+ <li><a href="#tab_plugins" role="tab" data-toggle="tab">{{ _('Plugins') }}</a></li>
</ul>
<!-- Tab panes -->
@@ -139,11 +140,7 @@
<div class="col-xs-6 col-sm-4 col-md-4">{{ search_engine.name }} ({{ shortcuts[search_engine.name] }})</div>
{% endif %}
<div class="col-xs-6 col-sm-4 col-md-4">
- <div class="checkbox">
- <input class="hidden" type="checkbox" id="engine_{{ categ|replace(' ', '_') }}_{{ search_engine.name|replace(' ', '_') }}" name="engine_{{ search_engine.name }}__{{ categ }}"{% if (search_engine.name, categ) in blocked_engines %} checked="checked"{% endif %} />
- <label class="btn btn-success label_hide_if_checked" for="engine_{{ categ|replace(' ', '_') }}_{{ search_engine.name|replace(' ', '_') }}">{{ _('Block') }}</label>
- <label class="btn btn-danger label_hide_if_not_checked" for="engine_{{ categ|replace(' ', '_') }}_{{ search_engine.name|replace(' ', '_') }}">{{ _('Allow') }}</label>
- </div>
+ {{ checkbox_toggle('engine_' + search_engine.name|replace(' ', '_') + '__' + categ|replace(' ', '_'), (search_engine.name, categ) in blocked_engines) }}
</div>
{% if rtl %}
<div class="col-xs-6 col-sm-4 col-md-4">{{ search_engine.name }} ({{ shortcuts[search_engine.name] }})‎</div>
@@ -157,6 +154,28 @@
{% endfor %}
</div>
</div>
+ <div class="tab-pane active_if_nojs" id="tab_plugins">
+ <noscript>
+ <h3>{{ _('Plugins') }}</h3>
+ </noscript>
+ <fieldset>
+ <div class="container-fluid">
+ {% for plugin in plugins %}
+ <div class="panel panel-default">
+ <div class="panel-heading">
+ <h3 class="panel-title">{{ plugin.name }}</h3>
+ </div>
+ <div class="panel-body">
+ <div class="col-xs-6 col-sm-4 col-md-6">{{ plugin.description }}</div>
+ <div class="col-xs-6 col-sm-4 col-md-6">
+ {{ checkbox_toggle('plugin_' + plugin.id, plugin.id not in allowed_plugins) }}
+ </div>
+ </div>
+ </div>
+ {% endfor %}
+ </div>
+ </fieldset>
+ </div>
</div>
<p class="text-muted" style="margin:20px 0;">{{ _('These settings are stored in your cookies, this allows us not to store this data about you.') }}
<br />
diff --git a/searx/templates/oscar/results.html b/searx/templates/oscar/results.html
@@ -25,8 +25,8 @@
{% endif %}
</div>
{% endfor %}
-
- {% if not results %}
+
+ {% if not results and not answers %}
{% include 'oscar/messages/no_results.html' %}
{% endif %}
@@ -82,7 +82,7 @@
{% for infobox in infoboxes %}
{% include 'oscar/infobox.html' %}
{% endfor %}
- {% endif %}
+ {% endif %}
{% if suggestions %}
<div class="panel panel-default">
@@ -111,7 +111,7 @@
<input id="search_url" type="url" class="form-control select-all-on-click cursor-text" name="search_url" value="{{ base_url }}?q={{ q|urlencode }}&pageno={{ pageno }}{% if selected_categories %}&category_{{ selected_categories|join("&category_")|replace(' ','+') }}{% endif %}" readonly>
</div>
</form>
-
+
<label>{{ _('Download results') }}</label>
<div class="clearfix"></div>
{% for output_type in ('csv', 'json', 'rss') %}
@@ -122,7 +122,7 @@
<input type="hidden" name="pageno" value="{{ pageno }}">
<button type="submit" class="btn btn-default">{{ output_type }}</button>
</form>
- {% endfor %}
+ {% endfor %}
<div class="clearfix"></div>
</div>
</div>
diff --git a/searx/tests/test_plugins.py b/searx/tests/test_plugins.py
@@ -0,0 +1,51 @@
+# -*- coding: utf-8 -*-
+
+from searx.testing import SearxTestCase
+from searx import plugins
+from mock import Mock
+
+
+class PluginStoreTest(SearxTestCase):
+
+ def test_PluginStore_init(self):
+ store = plugins.PluginStore()
+ self.assertTrue(isinstance(store.plugins, list) and len(store.plugins) == 0)
+
+ def test_PluginStore_register(self):
+ store = plugins.PluginStore()
+ testplugin = plugins.Plugin()
+ store.register(testplugin)
+
+ self.assertTrue(len(store.plugins) == 1)
+
+ def test_PluginStore_call(self):
+ store = plugins.PluginStore()
+ testplugin = plugins.Plugin()
+ store.register(testplugin)
+ setattr(testplugin, 'asdf', Mock())
+ request = Mock(user_plugins=[])
+ store.call('asdf', request, Mock())
+
+ self.assertFalse(testplugin.asdf.called)
+
+ request.user_plugins.append(testplugin)
+ store.call('asdf', request, Mock())
+
+ self.assertTrue(testplugin.asdf.called)
+
+
+class SelfIPTest(SearxTestCase):
+
+ def test_PluginStore_init(self):
+ store = plugins.PluginStore()
+ store.register(plugins.self_ip)
+
+ self.assertTrue(len(store.plugins) == 1)
+
+ request = Mock(user_plugins=store.plugins,
+ remote_addr='127.0.0.1')
+ request.headers.getlist.return_value = []
+ ctx = {'search': Mock(answers=set(),
+ query='ip')}
+ store.call('pre_search', request, ctx)
+ self.assertTrue('127.0.0.1' in ctx['search'].answers)
diff --git a/searx/tests/test_webapp.py b/searx/tests/test_webapp.py
@@ -2,7 +2,6 @@
import json
from urlparse import ParseResult
-from mock import patch
from searx import webapp
from searx.testing import SearxTestCase
@@ -33,6 +32,11 @@ class ViewsTestCase(SearxTestCase):
},
]
+ def search_mock(search_self, *args):
+ search_self.results = self.test_results
+
+ webapp.Search.search = search_mock
+
self.maxDiff = None # to see full diffs
def test_index_empty(self):
@@ -40,14 +44,7 @@ class ViewsTestCase(SearxTestCase):
self.assertEqual(result.status_code, 200)
self.assertIn('<div class="title"><h1>searx</h1></div>', result.data)
- @patch('searx.search.Search.search')
- def test_index_html(self, search):
- search.return_value = (
- self.test_results,
- set(),
- set(),
- set()
- )
+ def test_index_html(self):
result = self.app.post('/', data={'q': 'test'})
self.assertIn(
'<h3 class="result_title"><img width="14" height="14" class="favicon" src="/static/themes/default/img/icons/icon_youtube.ico" alt="youtube" /><a href="http://second.test.xyz">Second <span class="highlight">Test</span></a></h3>', # noqa
@@ -58,14 +55,7 @@ class ViewsTestCase(SearxTestCase):
result.data
)
- @patch('searx.search.Search.search')
- def test_index_json(self, search):
- search.return_value = (
- self.test_results,
- set(),
- set(),
- set()
- )
+ def test_index_json(self):
result = self.app.post('/', data={'q': 'test', 'format': 'json'})
result_dict = json.loads(result.data)
@@ -76,14 +66,7 @@ class ViewsTestCase(SearxTestCase):
self.assertEqual(
result_dict['results'][0]['url'], 'http://first.test.xyz')
- @patch('searx.search.Search.search')
- def test_index_csv(self, search):
- search.return_value = (
- self.test_results,
- set(),
- set(),
- set()
- )
+ def test_index_csv(self):
result = self.app.post('/', data={'q': 'test', 'format': 'csv'})
self.assertEqual(
@@ -93,14 +76,7 @@ class ViewsTestCase(SearxTestCase):
result.data
)
- @patch('searx.search.Search.search')
- def test_index_rss(self, search):
- search.return_value = (
- self.test_results,
- set(),
- set(),
- set()
- )
+ def test_index_rss(self):
result = self.app.post('/', data={'q': 'test', 'format': 'rss'})
self.assertIn(
diff --git a/searx/webapp.py b/searx/webapp.py
@@ -27,6 +27,18 @@ import cStringIO
import os
import hashlib
+from searx import logger
+logger = logger.getChild('webapp')
+
+try:
+ from pygments import highlight
+ from pygments.lexers import get_lexer_by_name
+ from pygments.formatters import HtmlFormatter
+except:
+ logger.critical("cannot import dependency: pygments")
+ from sys import exit
+ exit(1)
+
from datetime import datetime, timedelta
from urllib import urlencode
from werkzeug.contrib.fixers import ProxyFix
@@ -51,18 +63,8 @@ from searx.https_rewrite import https_url_rewrite
from searx.search import Search
from searx.query import Query
from searx.autocomplete import searx_bang, backends as autocomplete_backends
-from searx import logger
-try:
- from pygments import highlight
- from pygments.lexers import get_lexer_by_name
- from pygments.formatters import HtmlFormatter
-except:
- logger.critical("cannot import dependency: pygments")
- from sys import exit
- exit(1)
-
+from searx.plugins import plugins
-logger = logger.getChild('webapp')
static_path, templates_path, themes =\
get_themes(settings['themes_path']
@@ -303,6 +305,23 @@ def render(template_name, override_theme=None, **kwargs):
'{}/{}'.format(kwargs['theme'], template_name), **kwargs)
+@app.before_request
+def pre_request():
+ # merge GET, POST vars
+ request.form = dict(request.form.items())
+ for k, v in request.args:
+ if k not in request.form:
+ request.form[k] = v
+
+ request.user_plugins = []
+ allowed_plugins = request.cookies.get('allowed_plugins', '').split(',')
+ disabled_plugins = request.cookies.get('disabled_plugins', '').split(',')
+ for plugin in plugins:
+ if ((plugin.default_on and plugin.id not in disabled_plugins)
+ or plugin.id in allowed_plugins):
+ request.user_plugins.append(plugin)
+
+
@app.route('/search', methods=['GET', 'POST'])
@app.route('/', methods=['GET', 'POST'])
def index():
@@ -323,8 +342,10 @@ def index():
'index.html',
)
- search.results, search.suggestions,\
- search.answers, search.infoboxes = search.search(request)
+ if plugins.call('pre_search', request, locals()):
+ search.search(request)
+
+ plugins.call('post_search', request, locals())
for result in search.results:
@@ -487,11 +508,11 @@ def preferences():
blocked_engines = get_blocked_engines(engines, request.cookies)
else: # on save
selected_categories = []
+ post_disabled_plugins = []
locale = None
autocomplete = ''
method = 'POST'
safesearch = '1'
-
for pd_name, pd in request.form.items():
if pd_name.startswith('category_'):
category = pd_name[9:]
@@ -514,14 +535,34 @@ def preferences():
safesearch = pd
elif pd_name.startswith('engine_'):
if pd_name.find('__') > -1:
- engine_name, category = pd_name.replace('engine_', '', 1).split('__', 1)
+ # TODO fix underscore vs space
+ engine_name, category = [x.replace('_', ' ') for x in
+ pd_name.replace('engine_', '', 1).split('__', 1)]
if engine_name in engines and category in engines[engine_name].categories:
blocked_engines.append((engine_name, category))
elif pd_name == 'theme':
theme = pd if pd in themes else default_theme
+ elif pd_name.startswith('plugin_'):
+ plugin_id = pd_name.replace('plugin_', '', 1)
+ if not any(plugin.id == plugin_id for plugin in plugins):
+ continue
+ post_disabled_plugins.append(plugin_id)
else:
resp.set_cookie(pd_name, pd, max_age=cookie_max_age)
+ disabled_plugins = []
+ allowed_plugins = []
+ for plugin in plugins:
+ if plugin.default_on:
+ if plugin.id in post_disabled_plugins:
+ disabled_plugins.append(plugin.id)
+ elif plugin.id not in post_disabled_plugins:
+ allowed_plugins.append(plugin.id)
+
+ resp.set_cookie('disabled_plugins', ','.join(disabled_plugins), max_age=cookie_max_age)
+
+ resp.set_cookie('allowed_plugins', ','.join(allowed_plugins), max_age=cookie_max_age)
+
resp.set_cookie(
'blocked_engines', ','.join('__'.join(e) for e in blocked_engines),
max_age=cookie_max_age
@@ -571,6 +612,8 @@ def preferences():
autocomplete_backends=autocomplete_backends,
shortcuts={y: x for x, y in engine_shortcuts.items()},
themes=themes,
+ plugins=plugins,
+ allowed_plugins=[plugin.id for plugin in request.user_plugins],
theme=get_current_theme_name())
