logo

searx

My custom branche(s) on searx, a meta-search engine
commit: bd92b43449410fe1c598600b16b0f23c4f6c256e
parent: 4cc79533e7452e4c08a2654f1ee1022307d227b0
Author: Adam Tauber <asciimoo@gmail.com>
Date:   Sun, 15 Mar 2015 11:38:07 +0100

Merge branch 'plugins'

Diffstat:

Asearx/plugins/__init__.py48++++++++++++++++++++++++++++++++++++++++++++++++
Asearx/plugins/self_ip.py21+++++++++++++++++++++
Msearx/search.py27++++++++++++---------------
Msearx/settings.yml1+
Msearx/templates/oscar/macros.html8++++++++
Msearx/templates/oscar/preferences.html31+++++++++++++++++++++++++------
Msearx/templates/oscar/results.html10+++++-----
Asearx/tests/test_plugins.py51+++++++++++++++++++++++++++++++++++++++++++++++++++
Msearx/tests/test_webapp.py42+++++++++---------------------------------
Msearx/webapp.py73++++++++++++++++++++++++++++++++++++++++++++++++++++++++++---------------
10 files changed, 238 insertions(+), 74 deletions(-)

diff --git a/searx/plugins/__init__.py b/searx/plugins/__init__.py @@ -0,0 +1,48 @@ +from searx.plugins import self_ip +from searx import logger +from sys import exit + +logger = logger.getChild('plugins') + +required_attrs = (('name', str), + ('description', str), + ('default_on', bool)) + + +class Plugin(): + default_on = False + name = 'Default plugin' + description = 'Default plugin description' + + +class PluginStore(): + + def __init__(self): + self.plugins = [] + + def __iter__(self): + for plugin in self.plugins: + yield plugin + + def register(self, *plugins): + for plugin in plugins: + for plugin_attr, plugin_attr_type in required_attrs: + if not hasattr(plugin, plugin_attr) or not isinstance(getattr(plugin, plugin_attr), plugin_attr_type): + logger.critical('missing attribute "{0}", cannot load plugin: {1}'.format(plugin_attr, plugin)) + exit(3) + plugin.id = plugin.name.replace(' ', '_') + self.plugins.append(plugin) + + def call(self, plugin_type, request, *args, **kwargs): + ret = True + for plugin in request.user_plugins: + if hasattr(plugin, plugin_type): + ret = getattr(plugin, plugin_type)(request, *args, **kwargs) + if not ret: + break + + return ret + + +plugins = PluginStore() +plugins.register(self_ip) diff --git a/searx/plugins/self_ip.py b/searx/plugins/self_ip.py @@ -0,0 +1,21 @@ +from flask.ext.babel import gettext +name = "Self IP" +description = gettext('Display your source IP address if the query expression is "ip"') +default_on = True + + +# attach callback to the pre search hook +# request: flask request object +# ctx: the whole local context of the pre search hook +def pre_search(request, ctx): + if ctx['search'].query == 'ip': + x_forwarded_for = request.headers.getlist("X-Forwarded-For") + if x_forwarded_for: + ip = x_forwarded_for[0] + else: + ip = request.remote_addr + ctx['search'].answers.clear() + ctx['search'].answers.add(ip) + # return False prevents exeecution of the original block + return False + return True diff --git a/searx/search.py b/searx/search.py @@ -329,8 +329,8 @@ class Search(object): self.blocked_engines = get_blocked_engines(engines, request.cookies) self.results = [] - self.suggestions = [] - self.answers = [] + self.suggestions = set() + self.answers = set() self.infoboxes = [] self.request_data = {} @@ -429,9 +429,6 @@ class Search(object): requests = [] results_queue = Queue() results = {} - suggestions = set() - answers = set() - infoboxes = [] # increase number of searches number_of_searches += 1 @@ -511,7 +508,7 @@ class Search(object): selected_engine['name'])) if not requests: - return results, suggestions, answers, infoboxes + return self # send all search-request threaded_requests(requests) @@ -519,19 +516,19 @@ class Search(object): engine_name, engine_results = results_queue.get_nowait() # TODO type checks - [suggestions.add(x['suggestion']) + [self.suggestions.add(x['suggestion']) for x in list(engine_results) if 'suggestion' in x and engine_results.remove(x) is None] - [answers.add(x['answer']) + [self.answers.add(x['answer']) for x in list(engine_results) if 'answer' in x and engine_results.remove(x) is None] - infoboxes.extend(x for x in list(engine_results) - if 'infobox' in x - and engine_results.remove(x) is None) + self.infoboxes.extend(x for x in list(engine_results) + if 'infobox' in x + and engine_results.remove(x) is None) results[engine_name] = engine_results @@ -541,16 +538,16 @@ class Search(object): engines[engine_name].stats['result_count'] += len(engine_results) # score results and remove duplications - results = score_results(results) + self.results = score_results(results) # merge infoboxes according to their ids - infoboxes = merge_infoboxes(infoboxes) + self.infoboxes = merge_infoboxes(self.infoboxes) # update engine stats, using calculated score - for result in results: + for result in self.results: for res_engine in result['engines']: engines[result['engine']]\ .stats['score_count'] += result['score'] # return results, suggestions, answers and infoboxes - return results, suggestions, answers, infoboxes + return self diff --git a/searx/settings.yml b/searx/settings.yml @@ -106,6 +106,7 @@ engines: - name : gigablast engine : gigablast shortcut : gb + disabled: True - name : github engine : github diff --git a/searx/templates/oscar/macros.html b/searx/templates/oscar/macros.html @@ -59,3 +59,11 @@ </div> {% endif %} {%- endmacro %} + +{% macro checkbox_toggle(id, blocked) -%} + <div class="checkbox"> + <input class="hidden" type="checkbox" id="{{ id }}" name="{{ id }}"{% if blocked %} checked="checked"{% endif %} /> + <label class="btn btn-success label_hide_if_checked" for="{{ id }}">{{ _('Block') }}</label> + <label class="btn btn-danger label_hide_if_not_checked" for="{{ id }}">{{ _('Allow') }}</label> + </div> +{%- endmacro %} diff --git a/searx/templates/oscar/preferences.html b/searx/templates/oscar/preferences.html @@ -1,4 +1,4 @@ -{% from 'oscar/macros.html' import preferences_item_header, preferences_item_header_rtl, preferences_item_footer, preferences_item_footer_rtl %} +{% from 'oscar/macros.html' import preferences_item_header, preferences_item_header_rtl, preferences_item_footer, preferences_item_footer_rtl, checkbox_toggle %} {% extends "oscar/base.html" %} {% block title %}{{ _('preferences') }} - {% endblock %} {% block site_alert_warning_nojs %} @@ -16,6 +16,7 @@ <ul class="nav nav-tabs nav-justified hide_if_nojs" role="tablist" style="margin-bottom:20px;"> <li class="active"><a href="#tab_general" role="tab" data-toggle="tab">{{ _('General') }}</a></li> <li><a href="#tab_engine" role="tab" data-toggle="tab">{{ _('Engines') }}</a></li> + <li><a href="#tab_plugins" role="tab" data-toggle="tab">{{ _('Plugins') }}</a></li> </ul> <!-- Tab panes --> @@ -139,11 +140,7 @@ <div class="col-xs-6 col-sm-4 col-md-4">{{ search_engine.name }} ({{ shortcuts[search_engine.name] }})</div> {% endif %} <div class="col-xs-6 col-sm-4 col-md-4"> - <div class="checkbox"> - <input class="hidden" type="checkbox" id="engine_{{ categ|replace(' ', '_') }}_{{ search_engine.name|replace(' ', '_') }}" name="engine_{{ search_engine.name }}__{{ categ }}"{% if (search_engine.name, categ) in blocked_engines %} checked="checked"{% endif %} /> - <label class="btn btn-success label_hide_if_checked" for="engine_{{ categ|replace(' ', '_') }}_{{ search_engine.name|replace(' ', '_') }}">{{ _('Block') }}</label> - <label class="btn btn-danger label_hide_if_not_checked" for="engine_{{ categ|replace(' ', '_') }}_{{ search_engine.name|replace(' ', '_') }}">{{ _('Allow') }}</label> - </div> + {{ checkbox_toggle('engine_' + search_engine.name|replace(' ', '_') + '__' + categ|replace(' ', '_'), (search_engine.name, categ) in blocked_engines) }} </div> {% if rtl %} <div class="col-xs-6 col-sm-4 col-md-4">{{ search_engine.name }} ({{ shortcuts[search_engine.name] }})&lrm;</div> @@ -157,6 +154,28 @@ {% endfor %} </div> </div> + <div class="tab-pane active_if_nojs" id="tab_plugins"> + <noscript> + <h3>{{ _('Plugins') }}</h3> + </noscript> + <fieldset> + <div class="container-fluid"> + {% for plugin in plugins %} + <div class="panel panel-default"> + <div class="panel-heading"> + <h3 class="panel-title">{{ plugin.name }}</h3> + </div> + <div class="panel-body"> + <div class="col-xs-6 col-sm-4 col-md-6">{{ plugin.description }}</div> + <div class="col-xs-6 col-sm-4 col-md-6"> + {{ checkbox_toggle('plugin_' + plugin.id, plugin.id not in allowed_plugins) }} + </div> + </div> + </div> + {% endfor %} + </div> + </fieldset> + </div> </div> <p class="text-muted" style="margin:20px 0;">{{ _('These settings are stored in your cookies, this allows us not to store this data about you.') }} <br /> diff --git a/searx/templates/oscar/results.html b/searx/templates/oscar/results.html @@ -25,8 +25,8 @@ {% endif %} </div> {% endfor %} - - {% if not results %} + + {% if not results and not answers %} {% include 'oscar/messages/no_results.html' %} {% endif %} @@ -82,7 +82,7 @@ {% for infobox in infoboxes %} {% include 'oscar/infobox.html' %} {% endfor %} - {% endif %} + {% endif %} {% if suggestions %} <div class="panel panel-default"> @@ -111,7 +111,7 @@ <input id="search_url" type="url" class="form-control select-all-on-click cursor-text" name="search_url" value="{{ base_url }}?q={{ q|urlencode }}&amp;pageno={{ pageno }}{% if selected_categories %}&amp;category_{{ selected_categories|join("&category_")|replace(' ','+') }}{% endif %}" readonly> </div> </form> - + <label>{{ _('Download results') }}</label> <div class="clearfix"></div> {% for output_type in ('csv', 'json', 'rss') %} @@ -122,7 +122,7 @@ <input type="hidden" name="pageno" value="{{ pageno }}"> <button type="submit" class="btn btn-default">{{ output_type }}</button> </form> - {% endfor %} + {% endfor %} <div class="clearfix"></div> </div> </div> diff --git a/searx/tests/test_plugins.py b/searx/tests/test_plugins.py @@ -0,0 +1,51 @@ +# -*- coding: utf-8 -*- + +from searx.testing import SearxTestCase +from searx import plugins +from mock import Mock + + +class PluginStoreTest(SearxTestCase): + + def test_PluginStore_init(self): + store = plugins.PluginStore() + self.assertTrue(isinstance(store.plugins, list) and len(store.plugins) == 0) + + def test_PluginStore_register(self): + store = plugins.PluginStore() + testplugin = plugins.Plugin() + store.register(testplugin) + + self.assertTrue(len(store.plugins) == 1) + + def test_PluginStore_call(self): + store = plugins.PluginStore() + testplugin = plugins.Plugin() + store.register(testplugin) + setattr(testplugin, 'asdf', Mock()) + request = Mock(user_plugins=[]) + store.call('asdf', request, Mock()) + + self.assertFalse(testplugin.asdf.called) + + request.user_plugins.append(testplugin) + store.call('asdf', request, Mock()) + + self.assertTrue(testplugin.asdf.called) + + +class SelfIPTest(SearxTestCase): + + def test_PluginStore_init(self): + store = plugins.PluginStore() + store.register(plugins.self_ip) + + self.assertTrue(len(store.plugins) == 1) + + request = Mock(user_plugins=store.plugins, + remote_addr='127.0.0.1') + request.headers.getlist.return_value = [] + ctx = {'search': Mock(answers=set(), + query='ip')} + store.call('pre_search', request, ctx) + self.assertTrue('127.0.0.1' in ctx['search'].answers) diff --git a/searx/tests/test_webapp.py b/searx/tests/test_webapp.py @@ -2,7 +2,6 @@ import json from urlparse import ParseResult -from mock import patch from searx import webapp from searx.testing import SearxTestCase @@ -33,6 +32,11 @@ class ViewsTestCase(SearxTestCase): }, ] + def search_mock(search_self, *args): + search_self.results = self.test_results + + webapp.Search.search = search_mock + self.maxDiff = None # to see full diffs def test_index_empty(self): @@ -40,14 +44,7 @@ class ViewsTestCase(SearxTestCase): self.assertEqual(result.status_code, 200) self.assertIn('<div class="title"><h1>searx</h1></div>', result.data) - @patch('searx.search.Search.search') - def test_index_html(self, search): - search.return_value = ( - self.test_results, - set(), - set(), - set() - ) + def test_index_html(self): result = self.app.post('/', data={'q': 'test'}) self.assertIn( '<h3 class="result_title"><img width="14" height="14" class="favicon" src="/static/themes/default/img/icons/icon_youtube.ico" alt="youtube" /><a href="http://second.test.xyz">Second <span class="highlight">Test</span></a></h3>', # noqa @@ -58,14 +55,7 @@ class ViewsTestCase(SearxTestCase): result.data ) - @patch('searx.search.Search.search') - def test_index_json(self, search): - search.return_value = ( - self.test_results, - set(), - set(), - set() - ) + def test_index_json(self): result = self.app.post('/', data={'q': 'test', 'format': 'json'}) result_dict = json.loads(result.data) @@ -76,14 +66,7 @@ class ViewsTestCase(SearxTestCase): self.assertEqual( result_dict['results'][0]['url'], 'http://first.test.xyz') - @patch('searx.search.Search.search') - def test_index_csv(self, search): - search.return_value = ( - self.test_results, - set(), - set(), - set() - ) + def test_index_csv(self): result = self.app.post('/', data={'q': 'test', 'format': 'csv'}) self.assertEqual( @@ -93,14 +76,7 @@ class ViewsTestCase(SearxTestCase): result.data ) - @patch('searx.search.Search.search') - def test_index_rss(self, search): - search.return_value = ( - self.test_results, - set(), - set(), - set() - ) + def test_index_rss(self): result = self.app.post('/', data={'q': 'test', 'format': 'rss'}) self.assertIn( diff --git a/searx/webapp.py b/searx/webapp.py @@ -27,6 +27,18 @@ import cStringIO import os import hashlib +from searx import logger +logger = logger.getChild('webapp') + +try: + from pygments import highlight + from pygments.lexers import get_lexer_by_name + from pygments.formatters import HtmlFormatter +except: + logger.critical("cannot import dependency: pygments") + from sys import exit + exit(1) + from datetime import datetime, timedelta from urllib import urlencode from werkzeug.contrib.fixers import ProxyFix @@ -51,18 +63,8 @@ from searx.https_rewrite import https_url_rewrite from searx.search import Search from searx.query import Query from searx.autocomplete import searx_bang, backends as autocomplete_backends -from searx import logger -try: - from pygments import highlight - from pygments.lexers import get_lexer_by_name - from pygments.formatters import HtmlFormatter -except: - logger.critical("cannot import dependency: pygments") - from sys import exit - exit(1) - +from searx.plugins import plugins -logger = logger.getChild('webapp') static_path, templates_path, themes =\ get_themes(settings['themes_path'] @@ -303,6 +305,23 @@ def render(template_name, override_theme=None, **kwargs): '{}/{}'.format(kwargs['theme'], template_name), **kwargs) +@app.before_request +def pre_request(): + # merge GET, POST vars + request.form = dict(request.form.items()) + for k, v in request.args: + if k not in request.form: + request.form[k] = v + + request.user_plugins = [] + allowed_plugins = request.cookies.get('allowed_plugins', '').split(',') + disabled_plugins = request.cookies.get('disabled_plugins', '').split(',') + for plugin in plugins: + if ((plugin.default_on and plugin.id not in disabled_plugins) + or plugin.id in allowed_plugins): + request.user_plugins.append(plugin) + + @app.route('/search', methods=['GET', 'POST']) @app.route('/', methods=['GET', 'POST']) def index(): @@ -323,8 +342,10 @@ def index(): 'index.html', ) - search.results, search.suggestions,\ - search.answers, search.infoboxes = search.search(request) + if plugins.call('pre_search', request, locals()): + search.search(request) + + plugins.call('post_search', request, locals()) for result in search.results: @@ -487,11 +508,11 @@ def preferences(): blocked_engines = get_blocked_engines(engines, request.cookies) else: # on save selected_categories = [] + post_disabled_plugins = [] locale = None autocomplete = '' method = 'POST' safesearch = '1' - for pd_name, pd in request.form.items(): if pd_name.startswith('category_'): category = pd_name[9:] @@ -514,14 +535,34 @@ def preferences(): safesearch = pd elif pd_name.startswith('engine_'): if pd_name.find('__') > -1: - engine_name, category = pd_name.replace('engine_', '', 1).split('__', 1) + # TODO fix underscore vs space + engine_name, category = [x.replace('_', ' ') for x in + pd_name.replace('engine_', '', 1).split('__', 1)] if engine_name in engines and category in engines[engine_name].categories: blocked_engines.append((engine_name, category)) elif pd_name == 'theme': theme = pd if pd in themes else default_theme + elif pd_name.startswith('plugin_'): + plugin_id = pd_name.replace('plugin_', '', 1) + if not any(plugin.id == plugin_id for plugin in plugins): + continue + post_disabled_plugins.append(plugin_id) else: resp.set_cookie(pd_name, pd, max_age=cookie_max_age) + disabled_plugins = [] + allowed_plugins = [] + for plugin in plugins: + if plugin.default_on: + if plugin.id in post_disabled_plugins: + disabled_plugins.append(plugin.id) + elif plugin.id not in post_disabled_plugins: + allowed_plugins.append(plugin.id) + + resp.set_cookie('disabled_plugins', ','.join(disabled_plugins), max_age=cookie_max_age) + + resp.set_cookie('allowed_plugins', ','.join(allowed_plugins), max_age=cookie_max_age) + resp.set_cookie( 'blocked_engines', ','.join('__'.join(e) for e in blocked_engines), max_age=cookie_max_age @@ -571,6 +612,8 @@ def preferences(): autocomplete_backends=autocomplete_backends, shortcuts={y: x for x, y in engine_shortcuts.items()}, themes=themes, + plugins=plugins, + allowed_plugins=[plugin.id for plugin in request.user_plugins], theme=get_current_theme_name())