commit: 4130b3dfd640d4e4f88937e79648cfe7929dca43 parent 3bbdb23fd86a65d9c982aa34ee42326fa3dd9bc1 Author: a01200356 <a01200356@itesm.mx> Date: Sun, 10 Jan 2016 19:29:05 -0600 Merge branch 'master' of https://github.com/asciimoo/searx Conflicts: searx/tests/test_engines.pyDiffstat:
74 files changed, 330 insertions(+), 898 deletions(-)diff --git a/.gitignore b/.gitignore@@ -5,24 +5,13 @@ env robot_log.html robot_output.xml robot_report.html +test_basic/ setup.cfg *.pyc */*.pyc *~ -bin/ -build/ -coverage/ -develop-eggs/ -dist/ -eggs/ -include/ -lib/ -local/ -parts/ -searx.egg-info/ -var/ node_modules/ .tx/diff --git a/.travis.yml b/.travis.yml@@ -11,16 +11,16 @@ before_install: - "export DISPLAY=:99.0" - "sh -e /etc/init.d/xvfb start" - npm install -g less grunt-cli - - ( cd searx/static/themes/oscar;npm install ) + - ( cd searx/static/themes/oscar;npm install; cd - ) install: - - "make" + - ./manage.sh update_dev_packages - pip install coveralls script: - - "make flake8" - - "make robot" - - "make styles" - - "make grunt" - - make coverage + - ./manage.sh pep8_check + - ./manage.sh styles + - ./manage.sh grunt_build + - ./manage.sh py_test_coverage + - ./manage.sh robot_tests after_success: coveralls notifications:diff --git a/Makefile b/Makefile@@ -1,66 +0,0 @@ -# convenience makefile to boostrap & run buildout -# use `make options=-v` to run buildout with extra options - -version = 2.7 -python = bin/python -options = - -all: .installed.cfg - -.installed.cfg: bin/buildout buildout.cfg setup.py - bin/buildout $(options) - -bin/buildout: $(python) buildout.cfg bootstrap.py - $(python) bootstrap.py - @touch $@ - -$(python): - virtualenv -p python$(version) --no-site-packages . - @touch $@ - -robot: .installed.cfg - @bin/robot - -flake8: .installed.cfg - @bin/flake8 setup.py - @bin/flake8 ./searx/ - -tests: .installed.cfg flake8 - @bin/test - @grunt test --gruntfile searx/static/themes/oscar/gruntfile.js - -coverage: .installed.cfg - @bin/coverage run bin/test - @bin/coverage report - @bin/coverage html - -production: bin/buildout production.cfg setup.py - bin/buildout -c production.cfg $(options) - @echo "* Please modify `readlink --canonicalize-missing ./searx/settings.py`" - @echo "* Hint 1: on production, disable debug mode and change secret_key" - @echo "* Hint 2: searx will be executed at server startup by crontab" - @echo "* Hint 3: to run immediatley, execute 'bin/supervisord'" - -minimal: bin/buildout minimal.cfg setup.py - bin/buildout -c minimal.cfg $(options) - -styles: - @lessc -x searx/static/themes/default/less/style.less > searx/static/themes/default/css/style.css - @lessc -x searx/static/themes/default/less/style-rtl.less > searx/static/themes/default/css/style-rtl.css - @lessc -x searx/static/themes/courgette/less/style.less > searx/static/themes/courgette/css/style.css - @lessc -x searx/static/themes/courgette/less/style-rtl.less > searx/static/themes/courgette/css/style-rtl.css - @lessc -x searx/static/less/bootstrap/bootstrap.less > searx/static/css/bootstrap.min.css - @lessc -x searx/static/themes/oscar/less/oscar/oscar.less > searx/static/themes/oscar/css/oscar.min.css - @lessc -x searx/static/themes/pix-art/less/style.less > searx/static/themes/pix-art/css/style.css - -grunt: - @grunt --gruntfile searx/static/themes/oscar/gruntfile.js - -locales: - @pybabel compile -d searx/translations - -clean: - @rm -rf .installed.cfg .mr.developer.cfg bin parts develop-eggs eggs \ - searx.egg-info lib include .coverage coverage - -.PHONY: all tests robot flake8 coverage production minimal styles locales cleandiff --git a/README.rst b/README.rst@@ -7,31 +7,16 @@ engine <https://en.wikipedia.org/wiki/Metasearch_engine>`__. List of `running instances <https://github.com/asciimoo/searx/wiki/Searx-instances>`__. -See the `wiki <https://github.com/asciimoo/searx/wiki>`__ for more information. +See the `documentation <https://asciimoo.github.io/searx>`__ and the `wiki <https://github.com/asciimoo/searx/wiki>`__ for more information. |Flattr searx| -Features -~~~~~~~~ - -- Tracking free -- Supports multiple output formats - - - json ``curl https://searx.me/?format=json&q=[query]`` - - csv ``curl https://searx.me/?format=csv&q=[query]`` - - opensearch/rss ``curl https://searx.me/?format=rss&q=[query]`` -- Opensearch support (you can set as default search engine) -- Configurable search engines/categories -- Different search languages -- Duckduckgo like !bang functionality with engine shortcuts -- Parallel queries - relatively fast - Installation ~~~~~~~~~~~~ - clone source: ``git clone git@github.com:asciimoo/searx.git && cd searx`` -- install dependencies: ``pip install -r requirements.txt`` +- install dependencies: ``./manage.sh update_packages`` - edit your `settings.yml <https://github.com/asciimoo/searx/blob/master/settings.yml>`__ (set your ``secret_key``!) @@ -40,104 +25,6 @@ Installation For all the details, follow this `step by step installation <https://github.com/asciimoo/searx/wiki/Installation>`__ -Alternative (Recommended) Installation -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -- clone source: - ``git clone git@github.com:asciimoo/searx.git && cd searx`` -- build in current folder: ``make minimal`` -- run ``bin/searx-run`` to start the application - -Development -~~~~~~~~~~~ - -Just run ``make``. Versions of dependencies are pinned down inside -``versions.cfg`` to produce most stable build. Also remember, NO make -command should be run as root, not even ``make production`` - -Deployment -~~~~~~~~~~ - -- clone source: - ``git clone git@github.com:asciimoo/searx.git && cd searx`` -- build in current folder: ``make production`` -- run ``bin/supervisord`` to start the application - -Upgrading -~~~~~~~~~ - -- inside previously cloned searx directory run: ``git stash`` to - temporarily save any changes you have made -- pull source: ``git pull origin master`` -- re-build in current folder: ``make production`` -- run ``bin/supervisorctl stop searx`` to stop searx, if it does not, - then run ``fuser -k 8888/tcp`` -- run ``bin/supervisorctl reload`` to re-read supervisor config and - start searx - -Command make -~~~~~~~~~~~~ - -``make`` -'''''''' - -Builds development environment with testing support. - -``make tests`` -'''''''''''''' - -Runs tests. You can write tests -`here <https://github.com/asciimoo/searx/tree/master/searx/tests>`__ and -remember 'untested code is broken code'. - -``make robot`` -'''''''''''''' - -Runs robot (Selenium) tests, you must have ``firefox`` installed because -this functional tests actually run the browser and perform operations on -it. Also searx is executed with -`settings\_robot <https://github.com/asciimoo/searx/blob/master/searx/settings_robot.yml>`__. - -``make flake8`` -''''''''''''''' - -'pep8 is a tool to check your Python code against some of the style -conventions in `PEP 8 <http://www.python.org/dev/peps/pep-0008/>`__.' - -``make coverage`` -''''''''''''''''' - -Checks coverage of tests, after running this, execute this: -``firefox ./coverage/index.html`` - -``make production`` -''''''''''''''''''' - -Used to make co-called production environment - without tests (you -should ran tests before deploying searx on the server). This installs -supervisord, so if searx crashes, it will try to pick itself up again. -And crontab entry is added to start supervisord at server boot. - -``make minimal`` -'''''''''''''''' - -Minimal build - without test frameworks, the quickest build option. - -``make clean`` -'''''''''''''' - -Deletes several folders and files (see ``Makefile`` for more), so that -next time you run any other ``make`` command it will rebuild everithing. - -TODO -~~~~ - -- Moar engines -- Better ui -- Browser integration -- Documentation -- Tests - Bugs ~~~~diff --git a/base.cfg b/base.cfg@@ -1,17 +0,0 @@ -[buildout] -extends = versions.cfg -unzip = true -newest = false -prefer-final = true -develop = . - -eggs = - searx - -parts = - omelette - - -[omelette] -recipe = collective.recipe.omelette -eggs = ${buildout:eggs}diff --git a/bootstrap.py b/bootstrap.py@@ -1,210 +0,0 @@ -############################################################################## -# -# Copyright (c) 2006 Zope Foundation and Contributors. -# All Rights Reserved. -# -# This software is subject to the provisions of the Zope Public License, -# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution. -# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED -# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS -# FOR A PARTICULAR PURPOSE. -# -############################################################################## -"""Bootstrap a buildout-based project - -Simply run this script in a directory containing a buildout.cfg. -The script accepts buildout command-line options, so you can -use the -c option to specify an alternate configuration file. -""" - -import os -import shutil -import sys -import tempfile - -from optparse import OptionParser - -__version__ = '2015-07-01' -# See zc.buildout's changelog if this version is up to date. - -tmpeggs = tempfile.mkdtemp(prefix='bootstrap-') - -usage = '''\ -[DESIRED PYTHON FOR BUILDOUT] bootstrap.py [options] - -Bootstraps a buildout-based project. - -Simply run this script in a directory containing a buildout.cfg, using the -Python that you want bin/buildout to use. - -Note that by using --find-links to point to local resources, you can keep -this script from going over the network. -''' - -parser = OptionParser(usage=usage) -parser.add_option("--version", - action="store_true", default=False, - help=("Return bootstrap.py version.")) -parser.add_option("-t", "--accept-buildout-test-releases", - dest='accept_buildout_test_releases', - action="store_true", default=False, - help=("Normally, if you do not specify a --version, the " - "bootstrap script and buildout gets the newest " - "*final* versions of zc.buildout and its recipes and " - "extensions for you. If you use this flag, " - "bootstrap and buildout will get the newest releases " - "even if they are alphas or betas.")) -parser.add_option("-c", "--config-file", - help=("Specify the path to the buildout configuration " - "file to be used.")) -parser.add_option("-f", "--find-links", - help=("Specify a URL to search for buildout releases")) -parser.add_option("--allow-site-packages", - action="store_true", default=False, - help=("Let bootstrap.py use existing site packages")) -parser.add_option("--buildout-version", - help="Use a specific zc.buildout version") -parser.add_option("--setuptools-version", - help="Use a specific setuptools version") -parser.add_option("--setuptools-to-dir", - help=("Allow for re-use of existing directory of " - "setuptools versions")) - -options, args = parser.parse_args() -if options.version: - print("bootstrap.py version %s" % __version__) - sys.exit(0) - - -###################################################################### -# load/install setuptools - -try: - from urllib.request import urlopen -except ImportError: - from urllib2 import urlopen - -ez = {} -if os.path.exists('ez_setup.py'): - exec(open('ez_setup.py').read(), ez) -else: - exec(urlopen('https://bootstrap.pypa.io/ez_setup.py').read(), ez) - -if not options.allow_site_packages: - # ez_setup imports site, which adds site packages - # this will remove them from the path to ensure that incompatible versions - # of setuptools are not in the path - import site - # inside a virtualenv, there is no 'getsitepackages'. - # We can't remove these reliably - if hasattr(site, 'getsitepackages'): - for sitepackage_path in site.getsitepackages(): - # Strip all site-packages directories from sys.path that - # are not sys.prefix; this is because on Windows - # sys.prefix is a site-package directory. - if sitepackage_path != sys.prefix: - sys.path[:] = [x for x in sys.path - if sitepackage_path not in x] - -setup_args = dict(to_dir=tmpeggs, download_delay=0) - -if options.setuptools_version is not None: - setup_args['version'] = options.setuptools_version -if options.setuptools_to_dir is not None: - setup_args['to_dir'] = options.setuptools_to_dir - -ez['use_setuptools'](**setup_args) -import setuptools -import pkg_resources - -# This does not (always?) update the default working set. We will -# do it. -for path in sys.path: - if path not in pkg_resources.working_set.entries: - pkg_resources.working_set.add_entry(path) - -###################################################################### -# Install buildout - -ws = pkg_resources.working_set - -setuptools_path = ws.find( - pkg_resources.Requirement.parse('setuptools')).location - -# Fix sys.path here as easy_install.pth added before PYTHONPATH -cmd = [sys.executable, '-c', - 'import sys; sys.path[0:0] = [%r]; ' % setuptools_path + - 'from setuptools.command.easy_install import main; main()', - '-mZqNxd', tmpeggs] - -find_links = os.environ.get( - 'bootstrap-testing-find-links', - options.find_links or - ('http://downloads.buildout.org/' - if options.accept_buildout_test_releases else None) - ) -if find_links: - cmd.extend(['-f', find_links]) - -requirement = 'zc.buildout' -version = options.buildout_version -if version is None and not options.accept_buildout_test_releases: - # Figure out the most recent final version of zc.buildout. - import setuptools.package_index - _final_parts = '*final-', '*final' - - def _final_version(parsed_version): - try: - return not parsed_version.is_prerelease - except AttributeError: - # Older setuptools - for part in parsed_version: - if (part[:1] == '*') and (part not in _final_parts): - return False - return True - - index = setuptools.package_index.PackageIndex( - search_path=[setuptools_path]) - if find_links: - index.add_find_links((find_links,)) - req = pkg_resources.Requirement.parse(requirement) - if index.obtain(req) is not None: - best = [] - bestv = None - for dist in index[req.project_name]: - distv = dist.parsed_version - if _final_version(distv): - if bestv is None or distv > bestv: - best = [dist] - bestv = distv - elif distv == bestv: - best.append(dist) - if best: - best.sort() - version = best[-1].version -if version: - requirement = '=='.join((requirement, version)) -cmd.append(requirement) - -import subprocess -if subprocess.call(cmd) != 0: - raise Exception( - "Failed to execute command:\n%s" % repr(cmd)[1:-1]) - -###################################################################### -# Import and run buildout - -ws.add_entry(tmpeggs) -ws.require(requirement) -import zc.buildout.buildout - -if not [a for a in args if '=' not in a]: - args.append('bootstrap') - -# if -c was provided, we push it back into args for buildout' main function -if options.config_file is not None: - args[0:0] = ['-c', options.config_file] - -zc.buildout.buildout.main(args) -shutil.rmtree(tmpeggs)diff --git a/buildout.cfg b/buildout.cfg@@ -1,30 +0,0 @@ -[buildout] -extends = base.cfg -develop = . - -eggs = - searx [test] - -parts += - pyscripts - robot - test - - -[pyscripts] -recipe = zc.recipe.egg:script -eggs = ${buildout:eggs} -interpreter = py -dependent-scripts = true - - -[robot] -recipe = zc.recipe.testrunner -eggs = ${buildout:eggs} -defaults = ['--color', '--auto-progress', '--layer', 'SearxRobotLayer'] - - -[test] -recipe = zc.recipe.testrunner -eggs = ${buildout:eggs} -defaults = ['--color', '--auto-progress', '--layer', 'SearxTestLayer', '--layer', '!SearxRobotLayer']diff --git a/manage.sh b/manage.sh@@ -0,0 +1,94 @@ +#!/bin/sh + +BASE_DIR=$(dirname `readlink -f $0`) +PYTHONPATH=$BASE_DIR +SEARX_DIR="$BASE_DIR/searx" +ACTION=$1 + +update_packages() { + pip install --upgrade -r "$BASE_DIR/requirements.txt" +} + +update_dev_packages() { + update_packages + pip install --upgrade -r "$BASE_DIR/requirements-dev.txt" +} + +pep8_check() { + echo '[!] Running pep8 check' + pep8 --max-line-length=120 "$SEARX_DIR" "$BASE_DIR/tests" +} + +unit_tests() { + echo '[!] Running unit tests' + python -m nose2 -s "$BASE_DIR/tests/unit" +} + +py_test_coverage() { + echo '[!] Running python test coverage' + PYTHONPATH=`pwd` python -m nose2 -C --coverage "$SEARX_DIR" -s "$BASE_DIR/tests/unit" + coverage report + coverage html +} + +robot_tests() { + echo '[!] Running robot tests' + PYTHONPATH=`pwd` python "$SEARX_DIR/testing.py" robot +} + +tests() { + set -e + pep8_check + unit_tests + robot_tests + set +e +} + +build_style() { + lessc -x "$BASE_DIR/searx/static/$1" "$BASE_DIR/searx/static/$2" +} + +styles() { + echo '[!] Building styles' + build_style themes/default/less/style.less themes/default/css/style.css + build_style themes/default/less/style-rtl.less themes/default/css/style-rtl.css + build_style themes/courgette/less/style.less themes/courgette/css/style.css + build_style themes/courgette/less/style-rtl.less themes/courgette/css/style-rtl.css + build_style less/bootstrap/bootstrap.less css/bootstrap.min.css + build_style themes/oscar/less/oscar/oscar.less themes/oscar/css/oscar.min.css + build_style themes/pix-art/less/style.less themes/pix-art/css/style.css +} + +grunt_build() { + grunt --gruntfile "$SEARX_DIR/static/themes/oscar/gruntfile.js" +} + +locales() { + pybabel compile -d "$SEARX_DIR/translations" +} + +help() { + [ -z "$1" ] || echo "Error: $1\n" + echo "Searx manage.sh help + +Commands +======== + grunt_build - Build js files + help - This text + locales - Compile locales + pep8_check - Pep8 validation + py_test_coverage - Unit test coverage + robot_tests - Run selenium tests + styles - Build less files + tests - Run all python tests (pep8, unit, robot) + unit_tests - Run unit tests + update_dev_packages - Check & update development and production dependency changes + update_packages - Check & update dependency changes +" +} + +if type $ACTION 1>/dev/null; then + $ACTION +else + help "action not found" +fidiff --git a/minimal.cfg b/minimal.cfg@@ -1,15 +0,0 @@ -[buildout] -extends = base.cfg -develop = . - -eggs = - searx - -parts += - pyscripts - - -[pyscripts] -recipe = zc.recipe.egg:script -eggs = ${buildout:eggs} -interpreter = pydiff --git a/production.cfg b/production.cfg@@ -1,34 +0,0 @@ -[buildout] -extends = base.cfg -develop = . - -eggs = - searx - -parts += - pyscripts - supervisor - crontab_reboot - - -[pyscripts] -recipe = zc.recipe.egg:script -eggs = ${buildout:eggs} -interpreter = py - - -[supervisor] -recipe = collective.recipe.supervisor -http-socket = unix -user = searxer -password = ohpleasedochangeme -file = /tmp/supervisor.sock -chmod = 0700 -programs = - 50 searx ${buildout:bin-directory}/searx-run - - -[crontab_reboot] -recipe = z3c.recipe.usercrontab -times = @reboot -command = ${buildout:bin-directory}/supervisorddiff --git a/requirements-dev.txt b/requirements-dev.txt@@ -0,0 +1,10 @@ +babel==2.2.0 +flake8==2.5.1 +mock==1.0.1 +nose2[coverage-plugin] +plone.testing==4.0.15 +robotframework-selenium2library==1.7.4 +robotsuite==1.7.0 +transifex-client==0.11 +unittest2==1.1.0 +zope.testrunner==4.4.10diff --git a/requirements.txt b/requirements.txt@@ -1,12 +1,12 @@ -flask -flask-babel -requests -lxml -pyyaml -pygments -python-dateutil -ndg-httpsclient -pyopenssl -pyasn1 -pyasn1-modules -certifi +certifi==2015.11.20.1 +flask==0.10.1 +flask-babel==0.9 +lxml==3.5.0 +ndg-httpsclient==0.4.0 +pyasn1==0.1.9 +pyasn1-modules==0.0.8 +pygments==2.0.2 +pyopenssl==0.15.1 +python-dateutil==2.4.2 +pyyaml==3.11 +requests==2.9.1diff --git a/searx/engines/google.py b/searx/engines/google.py@@ -90,7 +90,7 @@ url_map = 'https://www.openstreetmap.org/'\ search_path = '/search' search_url = ('https://{hostname}' + search_path + - '?{query}&start={offset}&gbv=1&gws_rd=cr') + '?{query}&start={offset}&gbv=1&gws_rd=ssl') # other URLs map_hostname_start = 'maps.google.' @@ -99,7 +99,7 @@ redirect_path = '/url' images_path = '/images' # specific xpath variables -results_xpath = '//li[@class="g"]' +results_xpath = '//div[@class="g"]' url_xpath = './/h3/a/@href' title_xpath = './/h3' content_xpath = './/span[@class="st"]'diff --git a/searx/testing.py b/searx/testing.py@@ -3,6 +3,7 @@ from plone.testing import Layer from unittest2 import TestCase +from os.path import dirname, join, abspath import os @@ -42,11 +43,11 @@ class SearxRobotLayer(Layer): os.path.abspath(os.path.dirname(os.path.realpath(__file__))), 'webapp.py' ) - exe = os.path.abspath(os.path.dirname(__file__) + '/../bin/py') + exe = 'python' # set robot settings path - os.environ['SEARX_SETTINGS_PATH'] = os.path.abspath( - os.path.dirname(__file__) + '/settings_robot.yml') + os.environ['SEARX_SETTINGS_PATH'] = abspath( + dirname(__file__) + '/settings_robot.yml') # run the server self.server = subprocess.Popen( @@ -68,3 +69,16 @@ class SearxTestCase(TestCase): """Base test case for non-robot tests.""" layer = SearxTestLayer + + +if __name__ == '__main__': + from tests.test_robot import test_suite + import sys + from zope.testrunner.runner import Runner + + base_dir = abspath(join(dirname(__file__), '../tests')) + if sys.argv[1] == 'robot': + Runner(['--color', + '--auto-progress', + '--path', base_dir], + found_suites=[test_suite()]).run()diff --git a/searx/tests/engines/test_google.py b/searx/tests/engines/test_google.py@@ -1,178 +0,0 @@ -# -*- coding: utf-8 -*- -from collections import defaultdict -import mock -import lxml -from searx.engines import google -from searx.testing import SearxTestCase - - -class TestGoogleEngine(SearxTestCase): - - def mock_response(self, text): - response = mock.Mock(text=text, url='https://www.google.com/search?q=test&start=0&gbv=1&gws_rd=cr') - response.search_params = mock.Mock() - response.search_params.get = mock.Mock(return_value='www.google.com') - return response - - def test_request(self): - query = 'test_query' - dicto = defaultdict(dict) - dicto['pageno'] = 1 - dicto['language'] = 'fr_FR' - params = google.request(query, dicto) - self.assertIn('url', params) - self.assertIn(query, params['url']) - self.assertIn('google.fr', params['url']) - self.assertIn('fr', params['headers']['Accept-Language']) - - dicto['language'] = 'all' - params = google.request(query, dicto) - self.assertIn('google.com', params['url']) - self.assertIn('en', params['headers']['Accept-Language']) - - def test_response(self): - self.assertRaises(AttributeError, google.response, None) - self.assertRaises(AttributeError, google.response, []) - self.assertRaises(AttributeError, google.response, '') - self.assertRaises(AttributeError, google.response, '[]') - - response = self.mock_response('<html></html>') - self.assertEqual(google.response(response), []) - - html = """ - <li class="g"> - <h3 class="r"> - <a href="http://this.should.be.the.link/"> - <b>This</b> is <b>the</b> title - </a> - </h3> - <div class="s"> - <div class="kv" style="margin-bottom:2px"> - <cite> - <b>test</b>.psychologies.com/ - </cite> - <div class="_nBb"> - <div style="display:inline" onclick="google.sham(this);" aria-expanded="false" - aria-haspopup="true" tabindex="0" data-ved="0CBUQ7B0wAA"> - <span class="_O0"> - </span> - </div> - <div style="display:none" class="am-dropdown-menu" role="menu" tabindex="-1"> - <ul> - <li class="_Ykb"> - <a class="_Zkb" href="http://www.google.fr/url?url=http://webcache.googleusercontent - .com/search%3Fcache:R1Z_4pGXjuIJ:http://test.psychologies.com/"> - En cache - </a> - </li> - <li class="_Ykb"> - <a class="_Zkb" href="/search?safe=off&q=related:test.psy.com/"> - Pages similaires - </a> - </li> - </ul> - </div> - </div> - </div> - <span class="st"> - This should be the content. - </span> - <br> - <div class="osl"> - <a href="http://www.google.fr/url?url=http://test.psychologies.com/tests/"> - Test Personnalité - </a> - - <a href="http://www.google.fr/url?url=http://test.psychologies.com/test/"> - Tests - Moi - </a> - - <a href="http://www.google.fr/url?url=http://test.psychologies.com/test/tests-couple"> - Test Couple - </a> - - - <a href="http://www.google.fr/url?url=http://test.psychologies.com/tests/tests-amour"> - Test Amour - </a> - </div> - </div> - </li> - <li class="g"> - <h3 class="r"> - <a href="http://www.google.com/images?q=toto"> - <b>This</b> - </a> - </h3> - </li> - <li class="g"> - <h3 class="r"> - <a href="http://www.google.com/search?q=toto"> - <b>This</b> is - </a> - </h3> - </li> - <li class="g"> - <h3 class="r"> - <a href="€"> - <b>This</b> is <b>the</b> - </a> - </h3> - </li> - <li class="g"> - <h3 class="r"> - <a href="/url?q=url"> - <b>This</b> is <b>the</b> - </a> - </h3> - </li> - <p class="_Bmc" style="margin:3px 8px"> - <a href="/search?num=20&safe=off&q=t&revid=1754833769&sa=X&ei=-&ved="> - suggestion <b>title</b> - </a> - </p> - """ - response = self.mock_response(html) - results = google.response(response) - self.assertEqual(type(results), list) - self.assertEqual(len(results), 2) - self.assertEqual(results[0]['title'], 'This is the title') - self.assertEqual(results[0]['url'], 'http://this.should.be.the.link/') - self.assertEqual(results[0]['content'], 'This should be the content.') - self.assertEqual(results[1]['suggestion'], 'suggestion title') - - html = """ - <li class="b_algo" u="0|5109|4755453613245655|UAGjXgIrPH5yh-o5oNHRx_3Zta87f_QO"> - </li> - """ - response = self.mock_response(html) - results = google.response(response) - self.assertEqual(type(results), list) - self.assertEqual(len(results), 0) - - response = mock.Mock(text='<html></html>', url='https://sorry.google.com') - response.search_params = mock.Mock() - response.search_params.get = mock.Mock(return_value='www.google.com') - self.assertRaises(RuntimeWarning, google.response, response) - - response = mock.Mock(text='<html></html>', url='https://www.google.com/sorry/IndexRedirect') - response.search_params = mock.Mock() - response.search_params.get = mock.Mock(return_value='www.google.com') - self.assertRaises(RuntimeWarning, google.response, response) - - def test_parse_images(self): - html = """ - <li> - <div> - <a href="http://www.google.com/url?q=http://this.is.the.url/"> - <img style="margin:3px 0;margin-right:6px;padding:0" height="90" - src="https://this.is.the.image/image.jpg" width="60" align="middle" alt="" border="0"> - </a> - </div> - </li> - """ - dom = lxml.html.fromstring(html) - results = google.parse_images(dom, 'www.google.com') - self.assertEqual(type(results), list) - self.assertEqual(len(results), 1) - self.assertEqual(results[0]['url'], 'http://this.is.the.url/') - self.assertEqual(results[0]['title'], '') - self.assertEqual(results[0]['content'], '') - self.assertEqual(results[0]['img_src'], 'https://this.is.the.image/image.jpg')diff --git a/searx/tests/test_engines.py b/searx/tests/test_engines.py@@ -1,47 +0,0 @@ -from searx.tests.engines.test_bing import * # noqa -from searx.tests.engines.test_bing_images import * # noqa -from searx.tests.engines.test_bing_news import * # noqa -from searx.tests.engines.test_blekko_images import * # noqa -from searx.tests.engines.test_btdigg import * # noqa -from searx.tests.engines.test_currency_convert import * # noqa -from searx.tests.engines.test_dailymotion import * # noqa -from searx.tests.engines.test_deezer import * # noqa -from searx.tests.engines.test_deviantart import * # noqa -from searx.tests.engines.test_digg import * # noqa -from searx.tests.engines.test_duckduckgo import * # noqa -from searx.tests.engines.test_duckduckgo_definitions import * # noqa -from searx.tests.engines.test_dummy import * # noqa -from searx.tests.engines.test_faroo import * # noqa -from searx.tests.engines.test_flickr import * # noqa -from searx.tests.engines.test_flickr_noapi import * # noqa -from searx.tests.engines.test_gigablast import * # noqa -from searx.tests.engines.test_github import * # noqa -from searx.tests.engines.test_google import * # noqa -from searx.tests.engines.test_google_images import * # noqa -from searx.tests.engines.test_google_news import * # noqa -from searx.tests.engines.test_kickass import * # noqa -from searx.tests.engines.test_mediawiki import * # noqa -from searx.tests.engines.test_mixcloud import * # noqa -from searx.tests.engines.test_openstreetmap import * # noqa -from searx.tests.engines.test_photon import * # noqa -from searx.tests.engines.test_piratebay import * # noqa -from searx.tests.engines.test_qwant import * # noqa -from searx.tests.engines.test_searchcode_code import * # noqa -from searx.tests.engines.test_searchcode_doc import * # noqa -from searx.tests.engines.test_soundcloud import * # noqa -from searx.tests.engines.test_spotify import * # noqa -from searx.tests.engines.test_stackoverflow import * # noqa -from searx.tests.engines.test_startpage import * # noqa -from searx.tests.engines.test_subtitleseeker import * # noqa -from searx.tests.engines.test_swisscows import * # noqa -from searx.tests.engines.test_twitter import * # noqa -from searx.tests.engines.test_vimeo import * # noqa -from searx.tests.engines.test_wolframalpha_api import * # noqa -from searx.tests.engines.test_wolframalpha_noapi import * # noqa -from searx.tests.engines.test_www1x import * # noqa -from searx.tests.engines.test_www500px import * # noqa -from searx.tests.engines.test_yacy import * # noqa -from searx.tests.engines.test_yahoo import * # noqa -from searx.tests.engines.test_youtube_api import * # noqa -from searx.tests.engines.test_youtube_noapi import * # noqa -from searx.tests.engines.test_yahoo_news import * # noqadiff --git a/setup.py b/setup.py@@ -17,6 +17,8 @@ def read(*rnames): long_description = read('README.rst') +requirements = map(str.strip, open('requirements.txt').readlines()) +dev_requirements = map(str.strip, open('requirements-dev.txt').readlines()) setup( name='searx', @@ -38,35 +40,9 @@ setup( license='GNU Affero General Public License', packages=find_packages('.'), zip_safe=False, - install_requires=[ - 'flask', - 'flask-babel', - 'requests', - 'lxml', - 'pyyaml', - 'pygments', - 'setuptools', - 'python-dateutil', - 'pyopenssl', - 'ndg-httpsclient', - 'pyasn1', - 'pyasn1-modules', - 'certifi' - ], + install_requires=requirements, extras_require={ - 'test': [ - 'coverage', - 'flake8', - 'mock', - 'plone.testing', - 'robotframework', - 'robotframework-debuglibrary', - 'robotframework-httplibrary', - 'robotframework-selenium2library', - 'robotsuite', - 'unittest2', - 'zope.testrunner', - ] + 'test': dev_requirements }, entry_points={ 'console_scripts': [ @@ -86,6 +62,9 @@ setup( 'static/*/*/*/*/*.*', 'templates/*/*.*', 'templates/*/*/*.*', + 'tests/*', + 'tests/*/*', + 'tests/*/*/*', 'translations/*/*/*' ], },diff --git a/searx/tests/__init__.py b/tests/__init__.pydiff --git a/searx/tests/engines/__init__.py b/tests/robot/__init__.pydiff --git a/searx/tests/robot/test_basic.robot b/tests/robot/test_basic.robotdiff --git a/searx/tests/test_robot.py b/tests/test_robot.pydiff --git a/searx/tests/robot/__init__.py b/tests/unit/__init__.pydiff --git a/searx/tests/__init__.py b/tests/unit/engines/__init__.pydiff --git a/searx/tests/engines/test_bing.py b/tests/unit/engines/test_bing.pydiff --git a/searx/tests/engines/test_bing_images.py b/tests/unit/engines/test_bing_images.pydiff --git a/searx/tests/engines/test_bing_news.py b/tests/unit/engines/test_bing_news.pydiff --git a/searx/tests/engines/test_blekko_images.py b/tests/unit/engines/test_blekko_images.pydiff --git a/searx/tests/engines/test_btdigg.py b/tests/unit/engines/test_btdigg.pydiff --git a/searx/tests/engines/test_currency_convert.py b/tests/unit/engines/test_currency_convert.pydiff --git a/searx/tests/engines/test_dailymotion.py b/tests/unit/engines/test_dailymotion.pydiff --git a/searx/tests/engines/test_deezer.py b/tests/unit/engines/test_deezer.pydiff --git a/searx/tests/engines/test_deviantart.py b/tests/unit/engines/test_deviantart.pydiff --git a/searx/tests/engines/test_digg.py b/tests/unit/engines/test_digg.pydiff --git a/searx/tests/engines/test_duckduckgo.py b/tests/unit/engines/test_duckduckgo.pydiff --git a/searx/tests/engines/test_duckduckgo_definitions.py b/tests/unit/engines/test_duckduckgo_definitions.pydiff --git a/searx/tests/engines/test_dummy.py b/tests/unit/engines/test_dummy.pydiff --git a/searx/tests/engines/test_faroo.py b/tests/unit/engines/test_faroo.pydiff --git a/searx/tests/engines/test_flickr.py b/tests/unit/engines/test_flickr.pydiff --git a/searx/tests/engines/test_flickr_noapi.py b/tests/unit/engines/test_flickr_noapi.pydiff --git a/searx/tests/engines/test_gigablast.py b/tests/unit/engines/test_gigablast.pydiff --git a/searx/tests/engines/test_github.py b/tests/unit/engines/test_github.pydiff --git a/tests/unit/engines/test_google.py b/tests/unit/engines/test_google.py@@ -0,0 +1,178 @@ +# -*- coding: utf-8 -*- +from collections import defaultdict +import mock +import lxml +from searx.engines import google +from searx.testing import SearxTestCase + + +class TestGoogleEngine(SearxTestCase): + + def mock_response(self, text): + response = mock.Mock(text=text, url='https://www.google.com/search?q=test&start=0&gbv=1&gws_rd=cr') + response.search_params = mock.Mock() + response.search_params.get = mock.Mock(return_value='www.google.com') + return response + + def test_request(self): + query = 'test_query' + dicto = defaultdict(dict) + dicto['pageno'] = 1 + dicto['language'] = 'fr_FR' + params = google.request(query, dicto) + self.assertIn('url', params) + self.assertIn(query, params['url']) + self.assertIn('google.fr', params['url']) + self.assertIn('fr', params['headers']['Accept-Language']) + + dicto['language'] = 'all' + params = google.request(query, dicto) + self.assertIn('google.com', params['url']) + self.assertIn('en', params['headers']['Accept-Language']) + + def test_response(self): + self.assertRaises(AttributeError, google.response, None) + self.assertRaises(AttributeError, google.response, []) + self.assertRaises(AttributeError, google.response, '') + self.assertRaises(AttributeError, google.response, '[]') + + response = self.mock_response('<html></html>') + self.assertEqual(google.response(response), []) + + html = """ + <div class="g"> + <h3 class="r"> + <a href="http://this.should.be.the.link/"> + <b>This</b> is <b>the</b> title + </a> + </h3> + <div class="s"> + <div class="kv" style="margin-bottom:2px"> + <cite> + <b>test</b>.psychologies.com/ + </cite> + <div class="_nBb"> + <div style="display:inline" onclick="google.sham(this);" aria-expanded="false" + aria-haspopup="true" tabindex="0" data-ved="0CBUQ7B0wAA"> + <span class="_O0"> + </span> + </div> + <div style="display:none" class="am-dropdown-menu" role="menu" tabindex="-1"> + <ul> + <li class="_Ykb"> + <a class="_Zkb" href="http://www.google.fr/url?url=http://webcache.googleusercontent + .com/search%3Fcache:R1Z_4pGXjuIJ:http://test.psychologies.com/"> + En cache + </a> + </li> + <li class="_Ykb"> + <a class="_Zkb" href="/search?safe=off&q=related:test.psy.com/"> + Pages similaires + </a> + </li> + </ul> + </div> + </div> + </div> + <span class="st"> + This should be the content. + </span> + <br> + <div class="osl"> + <a href="http://www.google.fr/url?url=http://test.psychologies.com/tests/"> + Test Personnalité + </a> - + <a href="http://www.google.fr/url?url=http://test.psychologies.com/test/"> + Tests - Moi + </a> - + <a href="http://www.google.fr/url?url=http://test.psychologies.com/test/tests-couple"> + Test Couple + </a> + - + <a href="http://www.google.fr/url?url=http://test.psychologies.com/tests/tests-amour"> + Test Amour + </a> + </div> + </div> + </div> + <div class="g"> + <h3 class="r"> + <a href="http://www.google.com/images?q=toto"> + <b>This</b> + </a> + </h3> + </div> + <div class="g"> + <h3 class="r"> + <a href="http://www.google.com/search?q=toto"> + <b>This</b> is + </a> + </h3> + </div> + <div class="g"> + <h3 class="r"> + <a href="€"> + <b>This</b> is <b>the</b> + </a> + </h3> + </div> + <div class="g"> + <h3 class="r"> + <a href="/url?q=url"> + <b>This</b> is <b>the</b> + </a> + </h3> + </div> + <p class="_Bmc" style="margin:3px 8px"> + <a href="/search?num=20&safe=off&q=t&revid=1754833769&sa=X&ei=-&ved="> + suggestion <b>title</b> + </a> + </p> + """ + response = self.mock_response(html) + results = google.response(response) + self.assertEqual(type(results), list) + self.assertEqual(len(results), 2) + self.assertEqual(results[0]['title'], 'This is the title') + self.assertEqual(results[0]['url'], 'http://this.should.be.the.link/') + self.assertEqual(results[0]['content'], 'This should be the content.') + self.assertEqual(results[1]['suggestion'], 'suggestion title') + + html = """ + <li class="b_algo" u="0|5109|4755453613245655|UAGjXgIrPH5yh-o5oNHRx_3Zta87f_QO"> + </li> + """ + response = self.mock_response(html) + results = google.response(response) + self.assertEqual(type(results), list) + self.assertEqual(len(results), 0) + + response = mock.Mock(text='<html></html>', url='https://sorry.google.com') + response.search_params = mock.Mock() + response.search_params.get = mock.Mock(return_value='www.google.com') + self.assertRaises(RuntimeWarning, google.response, response) + + response = mock.Mock(text='<html></html>', url='https://www.google.com/sorry/IndexRedirect') + response.search_params = mock.Mock() + response.search_params.get = mock.Mock(return_value='www.google.com') + self.assertRaises(RuntimeWarning, google.response, response) + + def test_parse_images(self): + html = """ + <li> + <div> + <a href="http://www.google.com/url?q=http://this.is.the.url/"> + <img style="margin:3px 0;margin-right:6px;padding:0" height="90" + src="https://this.is.the.image/image.jpg" width="60" align="middle" alt="" border="0"> + </a> + </div> + </li> + """ + dom = lxml.html.fromstring(html) + results = google.parse_images(dom, 'www.google.com') + self.assertEqual(type(results), list) + self.assertEqual(len(results), 1) + self.assertEqual(results[0]['url'], 'http://this.is.the.url/') + self.assertEqual(results[0]['title'], '') + self.assertEqual(results[0]['content'], '') + self.assertEqual(results[0]['img_src'], 'https://this.is.the.image/image.jpg')diff --git a/searx/tests/engines/test_google_images.py b/tests/unit/engines/test_google_images.pydiff --git a/searx/tests/engines/test_google_news.py b/tests/unit/engines/test_google_news.pydiff --git a/searx/tests/engines/test_kickass.py b/tests/unit/engines/test_kickass.pydiff --git a/searx/tests/engines/test_mediawiki.py b/tests/unit/engines/test_mediawiki.pydiff --git a/searx/tests/engines/test_mixcloud.py b/tests/unit/engines/test_mixcloud.pydiff --git a/searx/tests/engines/test_openstreetmap.py b/tests/unit/engines/test_openstreetmap.pydiff --git a/searx/tests/engines/test_photon.py b/tests/unit/engines/test_photon.pydiff --git a/searx/tests/engines/test_piratebay.py b/tests/unit/engines/test_piratebay.pydiff --git a/searx/tests/engines/test_qwant.py b/tests/unit/engines/test_qwant.pydiff --git a/searx/tests/engines/test_searchcode_code.py b/tests/unit/engines/test_searchcode_code.pydiff --git a/searx/tests/engines/test_searchcode_doc.py b/tests/unit/engines/test_searchcode_doc.pydiff --git a/searx/tests/engines/test_soundcloud.py b/tests/unit/engines/test_soundcloud.pydiff --git a/searx/tests/engines/test_spotify.py b/tests/unit/engines/test_spotify.pydiff --git a/searx/tests/engines/test_stackoverflow.py b/tests/unit/engines/test_stackoverflow.pydiff --git a/searx/tests/engines/test_startpage.py b/tests/unit/engines/test_startpage.pydiff --git a/searx/tests/engines/test_subtitleseeker.py b/tests/unit/engines/test_subtitleseeker.pydiff --git a/searx/tests/engines/test_swisscows.py b/tests/unit/engines/test_swisscows.pydiff --git a/searx/tests/engines/test_twitter.py b/tests/unit/engines/test_twitter.pydiff --git a/searx/tests/engines/test_vimeo.py b/tests/unit/engines/test_vimeo.pydiff --git a/searx/tests/engines/test_www1x.py b/tests/unit/engines/test_www1x.pydiff --git a/searx/tests/engines/test_www500px.py b/tests/unit/engines/test_www500px.pydiff --git a/searx/tests/engines/test_yacy.py b/tests/unit/engines/test_yacy.pydiff --git a/searx/tests/engines/test_yahoo.py b/tests/unit/engines/test_yahoo.pydiff --git a/searx/tests/engines/test_yahoo_news.py b/tests/unit/engines/test_yahoo_news.pydiff --git a/searx/tests/engines/test_youtube_api.py b/tests/unit/engines/test_youtube_api.pydiff --git a/searx/tests/engines/test_youtube_noapi.py b/tests/unit/engines/test_youtube_noapi.pydiff --git a/searx/tests/test_plugins.py b/tests/unit/test_plugins.pydiff --git a/searx/tests/test_results.py b/tests/unit/test_results.pydiff --git a/searx/tests/test_search.py b/tests/unit/test_search.pydiff --git a/searx/tests/test_utils.py b/tests/unit/test_utils.pydiff --git a/searx/tests/test_webapp.py b/tests/unit/test_webapp.pydiff --git a/versions.cfg b/versions.cfg@@ -1,122 +0,0 @@ -[versions] -Babel = 1.3 -Flask = 0.10.1 -Flask-Babel = 0.9 -Jinja2 = 2.7.3 -MarkupSafe = 0.23 -Pygments = 2.0.2 -WebOb = 1.4.1 -WebTest = 2.0.18 -Werkzeug = 0.10.4 -collective.recipe.omelette = 0.16 -coverage = 3.7.1 -decorator = 3.4.2 -docutils = 0.12 -flake8 = 2.4.1 -itsdangerous = 0.24 -mccabe = 0.3.1 -mock = 1.0.1 -pep8 = 1.5.7 -plone.testing = 4.0.13 -pyflakes = 0.8.1 -pytz = 2015.4 -pyyaml = 3.11 -requests = 2.7.0 -robotframework-debuglibrary = 0.3 -robotframework-httplibrary = 0.4.2 -robotframework-selenium2library = 1.7.1 -robotsuite = 1.6.1 -selenium = 2.46.0 -speaklater = 1.3 -unittest2 = 1.0.1 -waitress = 0.8.9 -zc.recipe.testrunner = 2.0.0 -pyopenssl = 0.15.1 -ndg-httpsclient = 0.4.0 -pyasn1 = 0.1.8 -pyasn1-modules = 0.0.6 -certifi = 2015.11.20.1 - -cffi = 1.1.2 -cryptography = 0.9.1 - -# Required by: -# robotsuite==1.6.1 -# searx==0.7.0 -lxml = 3.4.4 - -# Required by: -# searx==0.7.0 -python-dateutil = 2.4.2 - -# Required by: -# searx==0.7.0 -# zope.exceptions==4.0.7 -# zope.interface==4.1.2 -# zope.testrunner==4.4.9 -setuptools = 18.0.1 - -# Required by: -# WebTest==2.0.18 -beautifulsoup4 = 4.3.2 - -# Required by: -# cryptography==0.9.1 -enum34 = 1.0.4 - -# Required by: -# cryptography==0.9.1 -idna = 2.0 - -# Required by: -# cryptography==0.9.1 -ipaddress = 1.0.7 - -# Required by: -# robotframework-httplibrary==0.4.2 -jsonpatch = 1.11 - -# Required by: -# robotframework-httplibrary==0.4.2 -jsonpointer = 1.9 - -# Required by: -# traceback2==1.4.0 -linecache2 = 1.0.0 - -# Required by: -# cffi==1.1.2 -pycparser = 2.12 - -# Required by: -# robotframework-httplibrary==0.4.2 -robotframework = 2.8.7 - -# Required by: -# robotsuite==1.6.1 -# zope.testrunner==4.4.9 -six = 1.9.0 - -# Required by: -# unittest2==1.0.1 -traceback2 = 1.4.0 - -# Required by: -# collective.recipe.omelette==0.16 -zc.recipe.egg = 2.0.1 - -# Required by: -# zope.testrunner==4.4.9 -zope.exceptions = 4.0.7 - -# Required by: -# zope.testrunner==4.4.9 -zope.interface = 4.1.2 - -# Required by: -# plone.testing==4.0.13 -zope.testing = 4.2.0 - -# Required by: -# zc.recipe.testrunner==2.0.0 -zope.testrunner = 4.4.9