logo

searx

My custom branche(s) on searx, a meta-search engine
commit: 8ef709ea1f86413d04051eea0876c20a48c8ae90
parent: aff33ea8969a477e9e02ef4be00872685ba1ca72
Author: Adam Tauber <asciimoo@gmail.com>
Date:   Sun, 11 Jan 2015 01:20:46 +0100

Merge pull request #170 from pointhi/little_fixes

Little fixes, add searx-autocompletion, more informations about torrents

Diffstat:

Msearx/autocomplete.py79+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Msearx/engines/kickass.py31+++++++++++++++++++++++++++++++
Msearx/query.py2+-
Msearx/templates/oscar/base.html1+
Msearx/templates/oscar/result_templates/default.html4+---
Msearx/templates/oscar/result_templates/torrent.html17++++++++++++++---
Msearx/templates/oscar/result_templates/videos.html4+---
Msearx/templates/oscar/results.html1+
Msearx/webapp.py11++++++++---
9 files changed, 137 insertions(+), 13 deletions(-)

diff --git a/searx/autocomplete.py b/searx/autocomplete.py @@ -20,6 +20,85 @@ from lxml import etree from requests import get from json import loads from urllib import urlencode +from searx.languages import language_codes +from searx.engines import ( + categories, engines, engine_shortcuts +) + + +def searx_bang(full_query): + '''check if the searchQuery contain a bang, and create fitting autocompleter results''' + # check if there is a query which can be parsed + if len(full_query.getSearchQuery()) == 0: + return [] + + results = [] + + # check if current query stats with !bang + if full_query.getSearchQuery()[0] == '!': + if len(full_query.getSearchQuery()) == 1: + # show some example queries + # TODO, check if engine is not avaliable + results.append("!images") + results.append("!wikipedia") + results.append("!osm") + else: + engine_query = full_query.getSearchQuery()[1:] + + # check if query starts with categorie name + for categorie in categories: + if categorie.startswith(engine_query): + results.append('!{categorie}'.format(categorie=categorie)) + + # check if query starts with engine name + for engine in engines: + if engine.startswith(engine_query.replace('_', ' ')): + results.append('!{engine}'.format(engine=engine.replace(' ', '_'))) + + # check if query starts with engine shortcut + for engine_shortcut in engine_shortcuts: + if engine_shortcut.startswith(engine_query): + results.append('!{engine_shortcut}'.format(engine_shortcut=engine_shortcut)) + + # check if current query stats with :bang + elif full_query.getSearchQuery()[0] == ':': + if len(full_query.getSearchQuery()) == 1: + # show some example queries + results.append(":en") + results.append(":en_us") + results.append(":english") + results.append(":united_kingdom") + else: + engine_query = full_query.getSearchQuery()[1:] + + for lc in language_codes: + lang_id, lang_name, country = map(str.lower, lc) + + # check if query starts with language-id + if lang_id.startswith(engine_query): + if len(engine_query) <= 2: + results.append(':{lang_id}'.format(lang_id=lang_id.split('_')[0])) + else: + results.append(':{lang_id}'.format(lang_id=lang_id)) + + # check if query starts with language name + if lang_name.startswith(engine_query): + results.append(':{lang_name}'.format(lang_name=lang_name)) + + # check if query starts with country + if country.startswith(engine_query.replace('_', ' ')): + results.append(':{country}'.format(country=country.replace(' ', '_'))) + + # remove duplicates + result_set = set(results) + + # remove results which are already contained in the query + for query_part in full_query.query_parts: + if query_part in result_set: + result_set.remove(query_part) + + # convert result_set back to list + return list(result_set) def dbpedia(query): diff --git a/searx/engines/kickass.py b/searx/engines/kickass.py @@ -24,6 +24,7 @@ search_url = url + 'search/{search_term}/{pageno}/' # specific xpath variables magnet_xpath = './/a[@title="Torrent magnet link"]' +torrent_xpath = './/a[@title="Download torrent file"]' content_xpath = './/span[@class="font11px lightgrey block"]' @@ -60,6 +61,9 @@ def response(resp): method="text")) seed = result.xpath('.//td[contains(@class, "green")]/text()')[0] leech = result.xpath('.//td[contains(@class, "red")]/text()')[0] + filesize = result.xpath('.//td[contains(@class, "nobr")]/text()')[0] + filesize_multiplier = result.xpath('.//td[contains(@class, "nobr")]//span/text()')[0] + files = result.xpath('.//td[contains(@class, "center")][2]/text()')[0] # convert seed to int if possible if seed.isdigit(): @@ -73,15 +77,42 @@ def response(resp): else: leech = 0 + # convert filesize to byte if possible + try: + filesize = float(filesize) + + # convert filesize to byte + if filesize_multiplier == 'TB': + filesize = int(filesize * 1024 * 1024 * 1024 * 1024) + elif filesize_multiplier == 'GB': + filesize = int(filesize * 1024 * 1024 * 1024) + elif filesize_multiplier == 'MB': + filesize = int(filesize * 1024 * 1024) + elif filesize_multiplier == 'kb': + filesize = int(filesize * 1024) + except: + filesize = None + + # convert files to int if possible + if files.isdigit(): + files = int(files) + else: + files = None + magnetlink = result.xpath(magnet_xpath)[0].attrib['href'] + torrentfile = result.xpath(torrent_xpath)[0].attrib['href'] + # append result results.append({'url': href, 'title': title, 'content': content, 'seed': seed, 'leech': leech, + 'filesize': filesize, + 'files': files, 'magnetlink': magnetlink, + 'torrentfile': torrentfile, 'template': 'torrent.html'}) # return results sorted by seeder diff --git a/searx/query.py b/searx/query.py @@ -77,7 +77,7 @@ class Query(object): if lang == lang_id\ or lang_id.startswith(lang)\ or lang == lang_name\ - or lang == country: + or lang.replace('_', ' ') == country: parse_next = True self.languages.append(lang) break diff --git a/searx/templates/oscar/base.html b/searx/templates/oscar/base.html @@ -7,6 +7,7 @@ <meta http-equiv="X-UA-Compatible" content="IE=edge"> <meta name="generator" content="searx/{{ searx_version }}"> <meta name="viewport" content="width=device-width, initial-scale=1 , maximum-scale=1.0, user-scalable=1" /> + {% block meta %}{% endblock %} <title>{% block title %}{% endblock %}searx</title> <link rel="stylesheet" href="{{ url_for('static', filename='css/bootstrap.min.css') }}" type="text/css" /> diff --git a/searx/templates/oscar/result_templates/default.html b/searx/templates/oscar/result_templates/default.html @@ -11,9 +11,7 @@ {% if result.embedded %} <div id="result-media-{{ index }}" class="collapse"> -{% autoescape false %} - {{ result.embedded }} -{% endautoescape %} + {{ result.embedded|safe }} </div> {% endif %} diff --git a/searx/templates/oscar/result_templates/torrent.html b/searx/templates/oscar/result_templates/torrent.html @@ -5,9 +5,20 @@ {% if result.publishedDate %}<time class="text-muted" datetime="{{ result.pubdate }}" >{{ result.publishedDate }}</time>{% endif %} <small><a class="text-info" href="https://web.archive.org/web/{{ result.url }}">{{ icon('link') }} {{ _('cached') }}</a></small> -<p class="result-content">{{ icon('transfer') }} {{ _('Seeder') }} <span class="badge">{{ result.seed }}</span>, {{ _('Leecher') }} <span class="badge">{{ result.leech }}</span> -<br/> -<a href="{{ result.magnetlink }}" class="magnetlink">{{ icon('magnet') }} magnet link</a></p> +<p class="result-content">{{ icon('transfer') }} {{ _('Seeder') }} <span class="badge">{{ result.seed }}</span> &bull; {{ _('Leecher') }} <span class="badge">{{ result.leech }}</span> +{% if result.filesize %}</br>{{ icon('floppy-disk') }} {{ _('Filesize') }} + <span class="badge"> + {% if result.filesize < 1024 %}{{ result.filesize }} Byte + {% elif result.filesize < 1024*1024 %}{{ '{0:0.2f}'.format(result.filesize/1024) }} kb + {% elif result.filesize < 1024*1024*1024 %}{{ '{0:0.2f}'.format(result.filesize/1024/1024) }} MB + {% elif result.filesize < 1024*1024*1024*1024 %}{{ '{0:0.2f}'.format(result.filesize/1024/1024/1024) }} GB + {% else %}{{ '{0:0.2f}'.format(result.filesize/1024/1024/1024/1024) }} TB{% endif %} + </span>{% endif %} +{% if result.files %}</br>{{ icon('file') }} {{ _('Number of Files') }} <span class="badge">{{ result.files }}</span>{% endif %}</p> +<p class="result-content"> + <a href="{{ result.magnetlink }}" class="magnetlink">{{ icon('magnet') }} {{ _('magnet link') }}</a> + {% if result.torrentfile %}</br><a href="{{ result.torrentfile }}" class="torrentfile">{{ icon('download-alt') }} {{ _('torrent file') }}</a>{% endif %} +</p> {% if result.content %}<p class="result-content">{{ result.content|safe }}</p>{% endif %} diff --git a/searx/templates/oscar/result_templates/videos.html b/searx/templates/oscar/result_templates/videos.html @@ -11,9 +11,7 @@ {% if result.embedded %} <div id="result-video-{{ index }}" class="collapse"> -{% autoescape false %} - {{ result.embedded }} -{% endautoescape %} + {{ result.embedded|safe }} </div> {% endif %} diff --git a/searx/templates/oscar/results.html b/searx/templates/oscar/results.html @@ -1,5 +1,6 @@ {% extends "oscar/base.html" %} {% block title %}{{ q }} - {% endblock %} +{% block meta %}<link rel="alternate" type="application/rss+xml" title="Searx search: {{ q }}" href="{{ url_for('index') }}?q={{ q }}&format=rss&{% for category in selected_categories %}category_{{ category }}=1&{% endfor %}pageno={{ pageno+1 }}">{% endblock %} {% block content %} <div class="row"> <div class="col-sm-8" id="main_results"> diff --git a/searx/webapp.py b/searx/webapp.py @@ -46,7 +46,7 @@ from searx.languages import language_codes from searx.https_rewrite import https_url_rewrite from searx.search import Search from searx.query import Query -from searx.autocomplete import backends as autocomplete_backends +from searx.autocomplete import searx_bang, backends as autocomplete_backends from searx import logger @@ -352,8 +352,13 @@ def autocompleter(): if not completer: return '', 400 - # run autocompletion - raw_results = completer(query.getSearchQuery()) + # parse searx specific autocompleter results like !bang + raw_results = searx_bang(query) + + # normal autocompletion results only appear if max 3. searx results returned + if len(raw_results) <= 3: + # run autocompletion + raw_results.extend(completer(query.getSearchQuery())) # parse results (write :language and !engine back to result string) results = []