logo

searx

My custom branche(s) on searx, a meta-search engine
commit: e740c8a8ea7dc4bbda7dab9a63ec476167c209ef
parent: 348187cff9bc3c55c1327f3d7177e6d1a61b8430
Author: Matej Cotman <cotman.matej@gmail.com>
Date:   Sun, 12 Jan 2014 12:40:27 +0100

tests and robot tests framework, build overhaul

Diffstat:

M.gitignore12++++++++++--
AMakefile50++++++++++++++++++++++++++++++++++++++++++++++++++
MREADME.md44++++++++++++++++++++++++++++++++++++++++++++
Abase.cfg23+++++++++++++++++++++++
Abootstrap.py277+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Abuildout.cfg32++++++++++++++++++++++++++++++++
Aproduction.cfg17+++++++++++++++++
Asearx/settings_robot.py16++++++++++++++++
Asearx/testing.py59+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Asearx/tests/__init__.py0
Asearx/tests/robot/__init__.py0
Asearx/tests/robot/test_basic.robot11+++++++++++
Asearx/tests/test_robot.py24++++++++++++++++++++++++
Asearx/tests/test_unit.py10++++++++++
Msearx/webapp.py19+++++++++++++++----
Asetup.py51+++++++++++++++++++++++++++++++++++++++++++++++++++
Aversions.cfg93+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
17 files changed, 732 insertions(+), 6 deletions(-)

diff --git a/.gitignore b/.gitignore @@ -1,5 +1,14 @@ env engines.cfg +.installed.cfg +setup.cfg *.pyc -*/*.pyc- \ No newline at end of file +*/*.pyc + +bin/ +include/ +lib/ +build/ +develop-eggs/ +parts/ diff --git a/Makefile b/Makefile @@ -0,0 +1,50 @@ +# convenience makefile to boostrap & run buildout +# use `make options=-v` to run buildout with extra options + +version = 2.7 +python = bin/python +options = + +all: .installed.cfg + +.installed.cfg: bin/buildout buildout.cfg setup.py + bin/buildout $(options) + +bin/buildout: $(python) buildout.cfg bootstrap.py + $(python) bootstrap.py + @touch $@ + +$(python): + virtualenv -p python$(version) --no-site-packages . + @touch $@ + +tests: .installed.cfg + @bin/test + +enginescfg: + @test -f ./engines.cfg || echo "Copying engines.cfg ..." + @cp --no-clobber engines.cfg_sample engines.cfg + +robot: .installed.cfg enginescfg + @bin/robot + +flake8: .installed.cfg + @bin/flake8 setup.py + @bin/flake8 ./searx/ + +coverage: .installed.cfg + @bin/coverage run --source=./searx/ --branch bin/test + @bin/coverage report --show-missing + @bin/coverage html --directory ./coverage + +minimal: bin/buildout production.cfg setup.py enginescfg + bin/buildout -c production.cfg $(options) + @echo "* Please modify `readlink --canonicalize-missing ./searx/settings.py`" + @echo "* Hint 1: on production, disable debug mode and change secret_key" + @echo "* Hint 2: to run server execute 'bin/searx-run'" + +clean: + @rm -rf .installed.cfg .mr.developer.cfg bin parts develop-eggs \ + searx.egg-info lib include .coverage coverage + +.PHONY: all tests enginescfg robot flake8 coverage minimal clean diff --git a/README.md b/README.md @@ -29,6 +29,48 @@ List of [running instances](https://github.com/asciimoo/searx/wiki/Searx-instanc For all the details, follow this [step by step installation](https://github.com/asciimoo/searx/wiki/Installation) +### Alternative (Recommended) Installation + +* clone source: `git clone git@github.com:asciimoo/searx.git && cd searx` +* build in current folder: `make minimal` +* run `bin/searx-run` to start the application + + +### Development + +Just run `make`. Versions of dependencies are pinned down inside `versions.cfg` to produce most stable build. + +#### Command make + +##### `make` + +Builds development environment with testing support. + +##### `make tests` + +Runs tests. You can write tests [here](https://github.com/asciimoo/searx/tree/master/searx/tests) and remember 'untested code is broken code'. + +##### `make robot` + +Runs robot (Selenium) tests, you must have `firefox` installed because this functional tests actually run the browser and perform operations on it. Also searx is executed with [settings_robot](https://github.com/asciimoo/searx/blob/master/searx/settings_robot.py). + +##### `make flake8` + +'pep8 is a tool to check your Python code against some of the style conventions in [PEP 8](http://www.python.org/dev/peps/pep-0008/).' + +##### `make coverage` + +Checks coverage of tests, after running this, execute this: `firefox ./coverage/index.html` + +##### `make minimal` + +Used to make co-called production environment - without tests (you should ran tests before deploying searx on the server). + +##### `make clean` + +Deletes several folders and files (see `Makefile` for more), so that next time you run any other `make` command it will rebuild everithing. + + ### TODO * Moar engines @@ -36,7 +78,9 @@ For all the details, follow this [step by step installation](https://github.com/ * Language support * Documentation * Pagination +* Fix `flake8` errors, `make flake8` will be merged into `make tests` when it does not fail anymore * Tests +* When we have more tests, we can integrate Travis-CI ### Bugs diff --git a/base.cfg b/base.cfg @@ -0,0 +1,23 @@ +[buildout] +extends = versions.cfg +versions = versions +unzip = true +newest = false +extends = versions.cfg +versions = versions +prefer-final = true +develop = . + +extensions = + buildout_versions + +eggs = + searx + +parts = + omelette + + +[omelette] +recipe = collective.recipe.omelette +eggs = ${buildout:eggs} diff --git a/bootstrap.py b/bootstrap.py @@ -0,0 +1,277 @@ +############################################################################## +# +# Copyright (c) 2006 Zope Foundation and Contributors. +# All Rights Reserved. +# +# This software is subject to the provisions of the Zope Public License, +# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution. +# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED +# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS +# FOR A PARTICULAR PURPOSE. +# +############################################################################## +"""Bootstrap a buildout-based project + +Simply run this script in a directory containing a buildout.cfg. +The script accepts buildout command-line options, so you can +use the -c option to specify an alternate configuration file. +""" + +import os, shutil, sys, tempfile, urllib, urllib2, subprocess +from optparse import OptionParser + +if sys.platform == 'win32': + def quote(c): + if ' ' in c: + return '"%s"' % c # work around spawn lamosity on windows + else: + return c +else: + quote = str + +# See zc.buildout.easy_install._has_broken_dash_S for motivation and comments. +stdout, stderr = subprocess.Popen( + [sys.executable, '-Sc', + 'try:\n' + ' import ConfigParser\n' + 'except ImportError:\n' + ' print 1\n' + 'else:\n' + ' print 0\n'], + stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate() +has_broken_dash_S = bool(int(stdout.strip())) + +# In order to be more robust in the face of system Pythons, we want to +# run without site-packages loaded. This is somewhat tricky, in +# particular because Python 2.6's distutils imports site, so starting +# with the -S flag is not sufficient. However, we'll start with that: +if not has_broken_dash_S and 'site' in sys.modules: + # We will restart with python -S. + args = sys.argv[:] + args[0:0] = [sys.executable, '-S'] + args = map(quote, args) + os.execv(sys.executable, args) +# Now we are running with -S. We'll get the clean sys.path, import site +# because distutils will do it later, and then reset the path and clean +# out any namespace packages from site-packages that might have been +# loaded by .pth files. +clean_path = sys.path[:] +import site # imported because of its side effects +sys.path[:] = clean_path +for k, v in sys.modules.items(): + if k in ('setuptools', 'pkg_resources') or ( + hasattr(v, '__path__') and + len(v.__path__) == 1 and + not os.path.exists(os.path.join(v.__path__[0], '__init__.py'))): + # This is a namespace package. Remove it. + sys.modules.pop(k) + +is_jython = sys.platform.startswith('java') + +setuptools_source = 'http://peak.telecommunity.com/dist/ez_setup.py' +distribute_source = 'http://python-distribute.org/distribute_setup.py' +distribute_source = 'https://bitbucket.org/pypa/setuptools/raw/f657df1f1ed46596d236376649c99a470662b4ba/distribute_setup.py' + +# parsing arguments +def normalize_to_url(option, opt_str, value, parser): + if value: + if '://' not in value: # It doesn't smell like a URL. + value = 'file://%s' % ( + urllib.pathname2url( + os.path.abspath(os.path.expanduser(value))),) + if opt_str == '--download-base' and not value.endswith('/'): + # Download base needs a trailing slash to make the world happy. + value += '/' + else: + value = None + name = opt_str[2:].replace('-', '_') + setattr(parser.values, name, value) + +usage = '''\ +[DESIRED PYTHON FOR BUILDOUT] bootstrap.py [options] + +Bootstraps a buildout-based project. + +Simply run this script in a directory containing a buildout.cfg, using the +Python that you want bin/buildout to use. + +Note that by using --setup-source and --download-base to point to +local resources, you can keep this script from going over the network. +''' + +parser = OptionParser(usage=usage) +parser.add_option("-v", "--version", dest="version", + help="use a specific zc.buildout version") +parser.add_option("-d", "--distribute", + action="store_true", dest="use_distribute", default=False, + help="Use Distribute rather than Setuptools.") +parser.add_option("--setup-source", action="callback", dest="setup_source", + callback=normalize_to_url, nargs=1, type="string", + help=("Specify a URL or file location for the setup file. " + "If you use Setuptools, this will default to " + + setuptools_source + "; if you use Distribute, this " + "will default to " + distribute_source + ".")) +parser.add_option("--download-base", action="callback", dest="download_base", + callback=normalize_to_url, nargs=1, type="string", + help=("Specify a URL or directory for downloading " + "zc.buildout and either Setuptools or Distribute. " + "Defaults to PyPI.")) +parser.add_option("--eggs", + help=("Specify a directory for storing eggs. Defaults to " + "a temporary directory that is deleted when the " + "bootstrap script completes.")) +parser.add_option("-t", "--accept-buildout-test-releases", + dest='accept_buildout_test_releases', + action="store_true", default=False, + help=("Normally, if you do not specify a --version, the " + "bootstrap script and buildout gets the newest " + "*final* versions of zc.buildout and its recipes and " + "extensions for you. If you use this flag, " + "bootstrap and buildout will get the newest releases " + "even if they are alphas or betas.")) +parser.add_option("-c", None, action="store", dest="config_file", + help=("Specify the path to the buildout configuration " + "file to be used.")) + +options, args = parser.parse_args() + +if options.eggs: + eggs_dir = os.path.abspath(os.path.expanduser(options.eggs)) +else: + eggs_dir = tempfile.mkdtemp() + +if options.setup_source is None: + if options.use_distribute: + options.setup_source = distribute_source + else: + options.setup_source = setuptools_source + +if options.accept_buildout_test_releases: + args.insert(0, 'buildout:accept-buildout-test-releases=true') + +try: + import pkg_resources + import setuptools # A flag. Sometimes pkg_resources is installed alone. + if not hasattr(pkg_resources, '_distribute'): + raise ImportError +except ImportError: + ez_code = urllib2.urlopen( + options.setup_source).read().replace('\r\n', '\n') + ez = {} + exec ez_code in ez + setup_args = dict(to_dir=eggs_dir, download_delay=0) + if options.download_base: + setup_args['download_base'] = options.download_base + if options.use_distribute: + setup_args['no_fake'] = True + if sys.version_info[:2] == (2, 4): + setup_args['version'] = '0.6.32' + ez['use_setuptools'](**setup_args) + if 'pkg_resources' in sys.modules: + reload(sys.modules['pkg_resources']) + import pkg_resources + # This does not (always?) update the default working set. We will + # do it. + for path in sys.path: + if path not in pkg_resources.working_set.entries: + pkg_resources.working_set.add_entry(path) + +cmd = [quote(sys.executable), + '-c', + quote('from setuptools.command.easy_install import main; main()'), + '-mqNxd', + quote(eggs_dir)] + +if not has_broken_dash_S: + cmd.insert(1, '-S') + +find_links = options.download_base +if not find_links: + find_links = os.environ.get('bootstrap-testing-find-links') +if not find_links and options.accept_buildout_test_releases: + find_links = 'http://downloads.buildout.org/' +if find_links: + cmd.extend(['-f', quote(find_links)]) + +if options.use_distribute: + setup_requirement = 'distribute' +else: + setup_requirement = 'setuptools' +ws = pkg_resources.working_set +setup_requirement_path = ws.find( + pkg_resources.Requirement.parse(setup_requirement)).location +env = dict( + os.environ, + PYTHONPATH=setup_requirement_path) + +requirement = 'zc.buildout' +version = options.version +if version is None and not options.accept_buildout_test_releases: + # Figure out the most recent final version of zc.buildout. + import setuptools.package_index + _final_parts = '*final-', '*final' + + def _final_version(parsed_version): + for part in parsed_version: + if (part[:1] == '*') and (part not in _final_parts): + return False + return True + index = setuptools.package_index.PackageIndex( + search_path=[setup_requirement_path]) + if find_links: + index.add_find_links((find_links,)) + req = pkg_resources.Requirement.parse(requirement) + if index.obtain(req) is not None: + best = [] + bestv = None + for dist in index[req.project_name]: + distv = dist.parsed_version + if distv >= pkg_resources.parse_version('2dev'): + continue + if _final_version(distv): + if bestv is None or distv > bestv: + best = [dist] + bestv = distv + elif distv == bestv: + best.append(dist) + if best: + best.sort() + version = best[-1].version + +if version: + requirement += '=='+version +else: + requirement += '<2dev' + +cmd.append(requirement) + +if is_jython: + import subprocess + exitcode = subprocess.Popen(cmd, env=env).wait() +else: # Windows prefers this, apparently; otherwise we would prefer subprocess + exitcode = os.spawnle(*([os.P_WAIT, sys.executable] + cmd + [env])) +if exitcode != 0: + sys.stdout.flush() + sys.stderr.flush() + print ("An error occurred when trying to install zc.buildout. " + "Look above this message for any errors that " + "were output by easy_install.") + sys.exit(exitcode) + +ws.add_entry(eggs_dir) +ws.require(requirement) +import zc.buildout.buildout + +# If there isn't already a command in the args, add bootstrap +if not [a for a in args if '=' not in a]: + args.append('bootstrap') + + +# if -c was provided, we push it back into args for buildout's main function +if options.config_file is not None: + args[0:0] = ['-c', options.config_file] + +zc.buildout.buildout.main(args) +if not options.eggs: # clean up temporary egg directory + shutil.rmtree(eggs_dir) diff --git a/buildout.cfg b/buildout.cfg @@ -0,0 +1,32 @@ +[buildout] +extends = base.cfg +develop = . + +eggs = + searx [test] + +parts += + pyscripts + robot + test + + +[pyscripts] +recipe = zc.recipe.egg:script +eggs = ${buildout:eggs} +interpreter = py +dependent-scripts = true +entry-points = + searx-run=searx.webapp:run + + +[robot] +recipe = zc.recipe.testrunner +eggs = ${buildout:eggs} +defaults = ['--color', '--auto-progress', '--layer', 'SearxRobotLayer'] + + +[test] +recipe = zc.recipe.testrunner +eggs = ${buildout:eggs} +defaults = ['--color', '--auto-progress', '--layer', 'SearxTestLayer', '--layer', '!SearxRobotLayer'] diff --git a/production.cfg b/production.cfg @@ -0,0 +1,17 @@ +[buildout] +extends = base.cfg +develop = . + +eggs = + searx + +parts += + pyscripts + + +[pyscripts] +recipe = zc.recipe.egg:script +eggs = ${buildout:eggs} +interpreter = py +entry-points = + searx-run=searx.webapp:run diff --git a/searx/settings_robot.py b/searx/settings_robot.py @@ -0,0 +1,16 @@ + +port = 11111 + +secret_key = "ultrasecretkey" # change this! + +debug = False + +request_timeout = 5.0 # seconds + +weights = {} # 'search_engine_name': float(weight) | default is 1.0 + +blacklist = [] # search engine blacklist + +categories = {} # custom search engine categories + +base_url = None # "https://your.domain.tld/" or None (to use request parameters) diff --git a/searx/testing.py b/searx/testing.py @@ -0,0 +1,59 @@ +# -*- coding: utf-8 -*- +"""Shared testing code.""" + +from plone.testing import Layer +from unittest2 import TestCase + + +import os +import subprocess +import sys + + +class SearxTestLayer: + + __name__ = u'SearxTestLayer' + + def setUp(cls): + pass + setUp = classmethod(setUp) + + def tearDown(cls): + pass + tearDown = classmethod(tearDown) + + def testSetUp(cls): + pass + testSetUp = classmethod(testSetUp) + + def testTearDown(cls): + pass + testTearDown = classmethod(testTearDown) + + +class SearxRobotLayer(Layer): + """Searx Robot Test Layer""" + + def setUp(self): + os.setpgrp() # create new process group, become its leader + webapp = os.path.join( + os.path.abspath(os.path.dirname(os.path.realpath(__file__))), + 'webapp.py' + ) + exe = os.path.abspath(os.path.dirname(__file__) + '/../bin/py') + self.server = subprocess.Popen( + [exe, webapp, 'settings_robot'], + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT + ) + + def tearDown(self): + # TERM all processes in my group + os.killpg(os.getpgid(self.server.pid), 15) + + +SEARXROBOTLAYER = SearxRobotLayer() + + +class SearxTestCase(TestCase): + layer = SearxTestLayer diff --git a/searx/tests/__init__.py b/searx/tests/__init__.py diff --git a/searx/tests/robot/__init__.py b/searx/tests/robot/__init__.py diff --git a/searx/tests/robot/test_basic.robot b/searx/tests/robot/test_basic.robot @@ -0,0 +1,11 @@ +*** Settings *** +Library Selenium2Library timeout=10 implicit_wait=0.5 +Test Setup Open Browser http://localhost:11111/ +Test Teardown Close All Browsers + + +*** Test Cases *** +Front page + Page Should Contain about + Page Should Contain preferences + diff --git a/searx/tests/test_robot.py b/searx/tests/test_robot.py @@ -0,0 +1,24 @@ +# -*- coding: utf-8 -*- + +from plone.testing import layered +from robotsuite import RobotTestSuite +from searx.testing import SEARXROBOTLAYER + +import os +import unittest2 as unittest + + +def test_suite(): + suite = unittest.TestSuite() + current_dir = os.path.abspath(os.path.dirname(__file__)) + robot_dir = os.path.join(current_dir, 'robot') + tests = [ + os.path.join('robot', f) for f in + os.listdir(robot_dir) if f.endswith('.robot') and + f.startswith('test_') + ] + for test in tests: + suite.addTests([ + layered(RobotTestSuite(test), layer=SEARXROBOTLAYER), + ]) + return suite diff --git a/searx/tests/test_unit.py b/searx/tests/test_unit.py @@ -0,0 +1,10 @@ +# -*- coding: utf-8 -*- + +from searx.testing import SearxTestCase + + +class UnitTestCase(SearxTestCase): + + def test_flask(self): + import flask + self.assertIn('Flask', dir(flask)) diff --git a/searx/webapp.py b/searx/webapp.py @@ -18,13 +18,20 @@ along with searx. If not, see < http://www.gnu.org/licenses/ >. ''' import os +import sys if __name__ == "__main__": - from sys import path - path.append(os.path.realpath(os.path.dirname(os.path.realpath(__file__))+'/../')) + sys.path.append(os.path.realpath(os.path.dirname(os.path.realpath(__file__))+'/../')) + +# first argument is for specifying settings module, used mostly by robot tests +from sys import argv +if len(argv) == 2: + from importlib import import_module + settings = import_module('searx.' + argv[1]) +else: + from searx import settings from flask import Flask, request, render_template, url_for, Response, make_response, redirect from searx.engines import search, categories, engines, get_engines_stats -from searx import settings import json import cStringIO from searx.utils import UnicodeWriter @@ -226,7 +233,7 @@ def favicon(): 'favicon.png', mimetype='image/vnd.microsoft.icon') -if __name__ == "__main__": +def run(): from gevent import monkey monkey.patch_all() @@ -234,3 +241,7 @@ if __name__ == "__main__": ,use_debugger = settings.debug ,port = settings.port ) + + +if __name__ == "__main__": + run() diff --git a/setup.py b/setup.py @@ -0,0 +1,51 @@ +# -*- coding: utf-8 -*- +"""Installer for Searx package.""" + +from setuptools import setup +from setuptools import find_packages + +import os + + +def read(*rnames): + return open(os.path.join(os.path.dirname(__file__), *rnames)).read() + + +long_description = read('README.md') + +setup( + name='searx', + version="0.1", + description="", + long_description=long_description, + classifiers=[ + "Programming Language :: Python", + ], + keywords='meta search engine', + author='Adam Tauber', + author_email='asciimoo@gmail.com', + url='https://github.com/asciimoo/searx', + license='GNU Affero General Public License', + packages=find_packages('.'), + zip_safe=False, + install_requires=[ + 'flask', + 'grequests', + 'lxml', + 'setuptools', + ], + extras_require={ + 'test': [ + 'coverage', + 'flake8', + 'plone.testing', + 'robotframework', + 'robotframework-debuglibrary', + 'robotframework-httplibrary', + 'robotframework-selenium2library', + 'robotsuite', + 'unittest2', + 'zope.testrunner', + ] + }, +) diff --git a/versions.cfg b/versions.cfg @@ -0,0 +1,93 @@ +[versions] +Flask = 0.10.1 +Jinja2 = 2.7.2 +MarkupSafe = 0.18 +WebOb = 1.3.1 +WebTest = 2.0.11 +Werkzeug = 0.9.4 +buildout-versions = 1.7 +collective.recipe.omelette = 0.16 +coverage = 3.7.1 +decorator = 3.4.0 +docutils = 0.11 +flake8 = 2.1.0 +itsdangerous = 0.23 +mccabe = 0.2.1 +pep8 = 1.4.6 +plone.testing = 4.0.8 +pyflakes = 0.7.3 +requests = 2.2.0 +robotframework-debuglibrary = 0.3 +robotframework-httplibrary = 0.4.2 +robotframework-selenium2library = 1.5.0 +robotsuite = 1.4.2 +selenium = 2.39.0 +unittest2 = 0.5.1 +waitress = 0.8.8 +zc.recipe.testrunner = 2.0.0 + +# Required by: +# WebTest==2.0.11 +beautifulsoup4 = 4.3.2 + +# Required by: +# grequests==0.2.0 +gevent = 1.0 + +# Required by: +# gevent==1.0 +greenlet = 0.4.2 + +# Required by: +# searx==0.1 +grequests = 0.2.0 + +# Required by: +# robotframework-httplibrary==0.4.2 +jsonpatch = 1.3 + +# Required by: +# robotframework-httplibrary==0.4.2 +jsonpointer = 1.1 + +# Required by: +# robotsuite==1.4.2 +# searx==0.1 +lxml = 3.2.5 + +# Required by: +# robotframework-httplibrary==0.4.2 +robotframework = 2.8.3 + +# Required by: +# plone.testing==4.0.8 +# robotsuite==1.4.2 +# searx==0.1 +# zope.exceptions==4.0.6 +# zope.interface==4.0.5 +# zope.testrunner==4.4.1 +setuptools = 2.1 + +# Required by: +# zope.testrunner==4.4.1 +six = 1.5.2 + +# Required by: +# collective.recipe.omelette==0.16 +zc.recipe.egg = 2.0.1 + +# Required by: +# zope.testrunner==4.4.1 +zope.exceptions = 4.0.6 + +# Required by: +# zope.testrunner==4.4.1 +zope.interface = 4.0.5 + +# Required by: +# plone.testing==4.0.8 +zope.testing = 4.1.2 + +# Required by: +# zc.recipe.testrunner==2.0.0 +zope.testrunner = 4.4.1