logo

searx

Unnamed repository; edit this file 'description' to name the repository.
commit: 85c7237a4f26cea523d5c3b8a863058e459ca07a
parent: 218857999e41c86732738a6733cb1fe3ea6b33ca
Author: Adam Tauber <asciimoo@gmail.com>
Date:   Sat, 22 Oct 2016 05:40:28 +0200

Merge pull request #728 from kvch/fix-kickass-torrents

fix kickass torrents engine

Diffstat:

searx/engines/kickass.py | 46+++++++++++-----------------------------------
searx/settings.yml | 6++++++
searx/utils.py | 15+++++++++++++++
tests/unit/engines/test_kickass.py | 26+++++++++++++-------------
4 files changed, 45 insertions(+), 48 deletions(-)

diff --git a/searx/engines/kickass.py b/searx/engines/kickass.py @@ -16,13 +16,14 @@ from urllib import quote from lxml import html from operator import itemgetter from searx.engines.xpath import extract_text +from searx.utils import get_torrent_size, convert_str_to_int # engine dependent config categories = ['videos', 'music', 'files'] paging = True # search-url -url = 'https://kickass.to/' +url = 'https://kickass.cd/' search_url = url + 'search/{search_term}/{pageno}/' # specific xpath variables @@ -57,41 +58,16 @@ def response(resp): href = urljoin(url, link.attrib['href']) title = extract_text(link) content = escape(extract_text(result.xpath(content_xpath))) - seed = result.xpath('.//td[contains(@class, "green")]/text()')[0] - leech = result.xpath('.//td[contains(@class, "red")]/text()')[0] - filesize = result.xpath('.//td[contains(@class, "nobr")]/text()')[0] - filesize_multiplier = result.xpath('.//td[contains(@class, "nobr")]//span/text()')[0] - files = result.xpath('.//td[contains(@class, "center")][2]/text()')[0] - - # convert seed to int if possible - if seed.isdigit(): - seed = int(seed) - else: - seed = 0 + seed = extract_text(result.xpath('.//td[contains(@class, "green")]')) + leech = extract_text(result.xpath('.//td[contains(@class, "red")]')) + filesize_info = extract_text(result.xpath('.//td[contains(@class, "nobr")]')) + files = extract_text(result.xpath('.//td[contains(@class, "center")][2]')) - # convert leech to int if possible - if leech.isdigit(): - leech = int(leech) - else: - leech = 0 - - # convert filesize to byte if possible - try: - filesize = float(filesize) - - # convert filesize to byte - if filesize_multiplier == 'TB': - filesize = int(filesize * 1024 * 1024 * 1024 * 1024) - elif filesize_multiplier == 'GB': - filesize = int(filesize * 1024 * 1024 * 1024) - elif filesize_multiplier == 'MB': - filesize = int(filesize * 1024 * 1024) - elif filesize_multiplier == 'KB': - filesize = int(filesize * 1024) - except: - filesize = None - - # convert files to int if possible + seed = convert_str_to_int(seed) + leech = convert_str_to_int(leech) + + filesize, filesize_multiplier = filesize_info.split() + filesize = get_torrent_size(filesize, filesize_multiplier) if files.isdigit(): files = int(files) else: diff --git a/searx/settings.yml b/searx/settings.yml @@ -307,6 +307,12 @@ engines: timeout : 6.0 disabled : True + - name: kickass + engine : kickass + shortcut : kc + timeout : 4.0 + disabled : True + - name : microsoft academic engine : json_engine paging : True diff --git a/searx/utils.py b/searx/utils.py @@ -252,12 +252,27 @@ def get_torrent_size(filesize, filesize_multiplier): filesize = int(filesize * 1024 * 1024) elif filesize_multiplier == 'KB': filesize = int(filesize * 1024) + elif filesize_multiplier == 'TiB': + filesize = int(filesize * 1000 * 1000 * 1000 * 1000) + elif filesize_multiplier == 'GiB': + filesize = int(filesize * 1000 * 1000 * 1000) + elif filesize_multiplier == 'MiB': + filesize = int(filesize * 1000 * 1000) + elif filesize_multiplier == 'KiB': + filesize = int(filesize * 1000) except: filesize = None return filesize +def convert_str_to_int(number_str): + if number_str.isdigit(): + return int(number_str) + else: + return 0 + + def is_valid_lang(lang): is_abbr = (len(lang) == 2) if is_abbr: diff --git a/tests/unit/engines/test_kickass.py b/tests/unit/engines/test_kickass.py @@ -14,7 +14,7 @@ class TestKickassEngine(SearxTestCase): params = kickass.request(query, dicto) self.assertIn('url', params) self.assertIn(query, params['url']) - self.assertIn('kickass.to', params['url']) + self.assertIn('kickass.cd', params['url']) self.assertFalse(params['verify']) def test_response(self): @@ -84,7 +84,7 @@ class TestKickassEngine(SearxTestCase): </span> </div> </td> - <td class="nobr center">449 <span>bytes</span></td> + <td class="nobr center">449 bytes</td> <td class="center">4</td> <td class="center">2&nbsp;years</td> <td class="green center">10</td> @@ -97,7 +97,7 @@ class TestKickassEngine(SearxTestCase): self.assertEqual(type(results), list) self.assertEqual(len(results), 1) self.assertEqual(results[0]['title'], 'This should be the title') - self.assertEqual(results[0]['url'], 'https://kickass.to/url.html') + self.assertEqual(results[0]['url'], 'https://kickass.cd/url.html') self.assertEqual(results[0]['content'], 'Posted by riri in Other &gt; Unsorted') self.assertEqual(results[0]['seed'], 10) self.assertEqual(results[0]['leech'], 1) @@ -191,7 +191,7 @@ class TestKickassEngine(SearxTestCase): </span> </div> </td> - <td class="nobr center">1 <span>KB</span></td> + <td class="nobr center">1 KiB</td> <td class="center">4</td> <td class="center">2&nbsp;years</td> <td class="green center">10</td> @@ -235,7 +235,7 @@ class TestKickassEngine(SearxTestCase): </span> </div> </td> - <td class="nobr center">1 <span>MB</span></td> + <td class="nobr center">1 MiB</td> <td class="center">4</td> <td class="center">2&nbsp;years</td> <td class="green center">9</td> @@ -279,7 +279,7 @@ class TestKickassEngine(SearxTestCase): </span> </div> </td> - <td class="nobr center">1 <span>GB</span></td> + <td class="nobr center">1 GiB</td> <td class="center">4</td> <td class="center">2&nbsp;years</td> <td class="green center">8</td> @@ -323,7 +323,7 @@ class TestKickassEngine(SearxTestCase): </span> </div> </td> - <td class="nobr center">1 <span>TB</span></td> + <td class="nobr center">1 TiB</td> <td class="center">4</td> <td class="center">2&nbsp;years</td> <td class="green center">7</td> @@ -367,7 +367,7 @@ class TestKickassEngine(SearxTestCase): </span> </div> </td> - <td class="nobr center">z <span>bytes</span></td> + <td class="nobr center">z bytes</td> <td class="center">r</td> <td class="center">2&nbsp;years</td> <td class="green center">a</td> @@ -380,17 +380,17 @@ class TestKickassEngine(SearxTestCase): self.assertEqual(type(results), list) self.assertEqual(len(results), 5) self.assertEqual(results[0]['title'], 'This should be the title') - self.assertEqual(results[0]['url'], 'https://kickass.to/url.html') + self.assertEqual(results[0]['url'], 'https://kickass.cd/url.html') self.assertEqual(results[0]['content'], 'Posted by riri in Other &gt; Unsorted') self.assertEqual(results[0]['seed'], 10) self.assertEqual(results[0]['leech'], 1) self.assertEqual(results[0]['files'], 4) self.assertEqual(results[0]['magnetlink'], 'magnet:?xt=urn:btih:MAGNETURL&dn=test') self.assertEqual(results[0]['torrentfile'], 'http://torcache.net/torrent/53917.torrent?title=test') - self.assertEqual(results[0]['filesize'], 1024) - self.assertEqual(results[1]['filesize'], 1048576) - self.assertEqual(results[2]['filesize'], 1073741824) - self.assertEqual(results[3]['filesize'], 1099511627776) + self.assertEqual(results[0]['filesize'], 1000) + self.assertEqual(results[1]['filesize'], 1000000) + self.assertEqual(results[2]['filesize'], 1000000000) + self.assertEqual(results[3]['filesize'], 1000000000000) self.assertEqual(results[4]['seed'], 0) self.assertEqual(results[4]['leech'], 0) self.assertEqual(results[4]['files'], None)