commit: 822bf2f36d715531d40cbd84cd825c5bd4da3a7a
parent: d50414275c9dce9dd6084f1a1ca1b5ce08a987ab
Author: asciimoo <asciimoo@gmail.com>
Date: Sun, 1 Dec 2013 16:10:38 +0100
[enh] robots.txt added
Diffstat:
1 file changed, 8 insertions(+), 0 deletions(-)
diff --git a/searx/webapp.py b/searx/webapp.py
@@ -148,6 +148,14 @@ def stats():
stats = get_engines_stats()
return render('stats.html', stats=stats)
+@app.route('/robots.txt', methods=['GET'])
+def robots():
+ return Response("""User-agent: *
+Allow: /
+Allow: /about
+Disallow: /stats
+""", mimetype='text/plain')
+
@app.route('/opensearch.xml', methods=['GET'])
def opensearch():
global opensearch_xml