commit: e7d005c621a0ce08d3fcdfc6122d27c809a5dff3 parent: dceb9031140a12972d000849ea8819a6d383739a Author: Adam Tauber <asciimoo@gmail.com> Date: Mon, 3 Oct 2016 13:21:12 +0200 Merge pull request #677 from pydo/feature/seedpeer-engine-integration Feature/seedpeer engine integrationDiffstat:
| M | AUTHORS.rst | 1 | + |
| A | searx/engines/seedpeer.py | 78 | ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ |
| M | searx/settings.yml | 6 | ++++++ |
| A | tests/unit/engines/seedpeer_fixture.html | 111 | +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ |
| A | tests/unit/engines/test_seedpeer.py | 51 | +++++++++++++++++++++++++++++++++++++++++++++++++++ |
5 files changed, 247 insertions(+), 0 deletions(-)
diff --git a/AUTHORS.rst b/AUTHORS.rst @@ -58,3 +58,4 @@ generally made searx better: - marc @a01200356 - Harry Wood @harry-wood - Thomas Renard @threnard +- Pydo `<https://github.com/pydo>`_ diff --git a/searx/engines/seedpeer.py b/searx/engines/seedpeer.py @@ -0,0 +1,78 @@ +# Seedpeer (Videos, Music, Files) +# +# @website http://seedpeer.eu +# @provide-api no (nothing found) +# +# @using-api no +# @results HTML (using search portal) +# @stable yes (HTML can change) +# @parse url, title, content, seed, leech, magnetlink + +from urlparse import urljoin +from cgi import escape +from urllib import quote +from lxml import html +from operator import itemgetter +from searx.engines.xpath import extract_text + + +url = 'http://www.seedpeer.eu/' +search_url = url + 'search/{search_term}/7/{page_no}.html' +# specific xpath variables +torrent_xpath = '//*[@id="body"]/center/center/table[2]/tr/td/a' +alternative_torrent_xpath = '//*[@id="body"]/center/center/table[1]/tr/td/a' +title_xpath = '//*[@id="body"]/center/center/table[2]/tr/td/a/text()' +alternative_title_xpath = '//*[@id="body"]/center/center/table/tr/td/a' +seeds_xpath = '//*[@id="body"]/center/center/table[2]/tr/td[4]/font/text()' +alternative_seeds_xpath = '//*[@id="body"]/center/center/table/tr/td[4]/font/text()' +peers_xpath = '//*[@id="body"]/center/center/table[2]/tr/td[5]/font/text()' +alternative_peers_xpath = '//*[@id="body"]/center/center/table/tr/td[5]/font/text()' +age_xpath = '//*[@id="body"]/center/center/table[2]/tr/td[2]/text()' +alternative_age_xpath = '//*[@id="body"]/center/center/table/tr/td[2]/text()' +size_xpath = '//*[@id="body"]/center/center/table[2]/tr/td[3]/text()' +alternative_size_xpath = '//*[@id="body"]/center/center/table/tr/td[3]/text()' + + +# do search-request +def request(query, params): + params['url'] = search_url.format(search_term=quote(query), + page_no=params['pageno'] - 1) + return params + + +# get response from search-request +def response(resp): + results = [] + dom = html.fromstring(resp.text) + torrent_links = dom.xpath(torrent_xpath) + if len(torrent_links) > 0: + seeds = dom.xpath(seeds_xpath) + peers = dom.xpath(peers_xpath) + titles = dom.xpath(title_xpath) + sizes = dom.xpath(size_xpath) + ages = dom.xpath(age_xpath) + else: # under ~5 results uses a different xpath + torrent_links = dom.xpath(alternative_torrent_xpath) + seeds = dom.xpath(alternative_seeds_xpath) + peers = dom.xpath(alternative_peers_xpath) + titles = dom.xpath(alternative_title_xpath) + sizes = dom.xpath(alternative_size_xpath) + ages = dom.xpath(alternative_age_xpath) + # return empty array if nothing is found + if not torrent_links: + return [] + + # parse results + for index, result in enumerate(torrent_links): + link = result.attrib.get('href') + href = urljoin(url, link) + results.append({'url': href, + 'title': titles[index].text_content(), + 'content': '{}, {}'.format(sizes[index], ages[index]), + 'seed': seeds[index], + 'leech': peers[index], + + 'template': 'torrent.html'}) + + # return results sorted by seeder + return sorted(results, key=itemgetter('seed'), reverse=True) diff --git a/searx/settings.yml b/searx/settings.yml @@ -495,6 +495,12 @@ engines: timeout: 6.0 categories : science + - name : seedpeer + engine : seedpeer + shortcut: speu + categories: files, music, videos + disabled: True + - name : dictzone engine : dictzone shortcut : dc diff --git a/tests/unit/engines/seedpeer_fixture.html b/tests/unit/engines/seedpeer_fixture.html @@ -0,0 +1,110 @@ +<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"> + <html xmlns="http://www.w3.org/1999/xhtml"> + <head> + </head> + <body> + <div id="header"> + <div id="whoIsYou"> + <a href="/lang.php"><small>SeedPeer in your own language?</small></a> <a href="http://www.seedpeer.eu"><img src="/images/flags/uk.gif" width="16px" alt="Torrents EN" /></a> <a href="http://spanish.seedpeer.eu"><img src="/images/flags/es.gif" width="16px" alt="Torrents ES" /></a> <a href="http://german.seedpeer.eu"><img src="/images/flags/de.gif" width="16px" alt="Torrents DE" /></a> <a href="http://french.seedpeer.eu"><img src="/images/flags/fr.gif" width="16px" alt="Torrents FR" /></a> <a href="http://portuguese.seedpeer.eu"><img src="/images/flags/pt.gif" width="16px" alt="Torrents Portuguese" /></a> <a href="http://swedish.seedpeer.eu"><img src="/images/flags/se.gif" width="16px" alt="Torrents Sweden" /></a> + </div> + + <script type="text/javascript"> + whoIsYou(); + </script> + <div id="search"> + <form action="/search.php" method="get"> + <input id="topsearchbar" name="search" value="narcos season 2" /> + <input type="submit" class="searchbutton" value="Torrents" /> + <input style="color:#000" type="submit" class="searchbutton" name="usenet" value="Usenet Binaries" /> + </form> + <div id="suggestion"></div> + </div> + <div id="logo"><a href="/"><img src="/images/logo2.gif" alt="Seedpeer homepage" width="415" height="143" /></a></div> + <div id="subtext"><a href="/">Home</a> > <a href="/search.html">Torrent search</a> > Narcos season 2 | page 1</div> + </div> + <div id="nav"> + <ul> + <!-- + <li><font style="color:red;font-size:9px;font-weight:bold;">NEW</font><a title="Download TOP Games for FREE" rel="nofollow" href="http://www.bigrebelads.com/affiliate/index?ref=9301" target="_blank">FREE Games</a></li> + + --> + <li style="border-left:none" id="categories"><a title="Browse Torrent Categories" href="/browse.html">Categories</a> + <ul> + <li><a title="Browse Anime Torrents" href="/browse.html#6">Anime</a></li> + <li><a title="Browse Game Torrents" href="/browse.html#4">Games</a></li> + <li><a title="Browse Movie Torrents" href="/browse.html#1">Movies</a></li> + <li><a title="Browse Music Torrents" href="/browse.html#3">Music</a></li> + <li><a title="Browse Software Torrents" href="/browse.html#5">Software</a></li> + <li><a title="Browse TV Torrents" href="/browse.html#2">TV Shows</a></li> + <li><a title="Browse Other Torrents" href="/browse.html#7">Others</a></li> + </ul> + </li> + <li><a title="Upload A Torrents" href="/upload.html">Upload torrent</a></li> + <li id="verified"><a title="Verified Torrents" href="/verified.html">Verified</a></li> + <li id="searchoptions"><a title="Search Torrents" href="/search.html">Torrent search</a></li> + <li id="newsgroups"><a style="color:#212b3e" title="News Groups" href="/usenet.html">Usenet Binaries</a></li> + <li id="about" style="border-right:none"><a rel="nofollow" href="/faq.html">About Us</a> + <ul> + <li><a title="SeedPeer Statistics" href="/stats.html">Statistics</a></li> + <li><a title="Contact Us" href="/contact.html">Contact</a></li> + <li><a title="Frequently Asked Questions" href="/faq.html">FAQ</a></li> + <li><a title="SeedPeer API" href="http://api.seedpeer.eu">Our API</a></li> + <li><a title="SeedPeer Blog" href="/blog">Blog</a></li> + </ul> + </li> + <!--<li><a href="/toolbar.php">Our Toolbar</a></li>--> + </ul> + <div class="clear"></div> + </div> + <div id="body"><div id="pageTop"></div> + <div id="headerbox"><h1>Verified <font class="colored">Narcos season 2</font> torrents</h1></div><table width="100%"><tr><th> + <span style="float:right"> + <a href="/search/narcos-season-2/8/1.html"><img style="vertical-align:middle" src="/images/comments.gif" alt="comments" /></a> | + <a href="/search/narcos-season-2/7/1.html"><img style="vertical-align:middle" src="/images/ver.gif" alt="verified" /></a> + </span> + <a href="/search/narcos-season-2/1/1.html">Torrent name</a></th><th class="right"><a href="/search/narcos-season-2/2/1.html">Age</a></th><th class="right"><a href="/search/narcos-season-2/3/1.html">Size</a></th><th class="right"><a href="/search/narcos-season-2/4/1.html">Seeds</a></th><th class="right"><a href="/search/narcos-season-2/5/1.html">Peers</a></th><th class="center"><a href="/search/narcos-season-2/6/1.html">Health</a></th><td class="tableAd" rowspan="6"><iframe src="http://creative.wwwpromoter.com/13689?d=300x250" width="300" height="250" style="border: none;" frameborder="0" scrolling="no"></iframe></td></tr><tr class=""><td><a class="pblink" id="pblink_table_item_1" href="" data-tad="431726" data-last-search="narcos+season+2" target="_blank" rel="nofollow"><strong class='colored'>Narcos season 2</strong> Full Version</a></td><td class="right">20 hours</td><td class="right">681.3 MB</td><td class="right"><font color="green">28</font> </td><td class="right"><font color="navy">654</font> </td><td class="center"><img src="/images/h5.gif" alt="Health" /></td></tr><tr class="tdark"><td><a class="pblink" id="pblink_table_item_2" href="" data-tad="431727" data-url="narcos+season+2" target="_blank" rel="nofollow"><strong class='colored'>Narcos season 2</strong> Trusted Source</a></td><td class="right">12 hours</td><td class="right">787.1 MB</td><td class="right"><font color="green">64</font> </td><td class="right"><font color="navy">220</font> </td><td class="center"><img src="/images/h5.gif" alt="Health" /></td></tr><tr class=""><td><a class="pblink" id="pblink_table_item_3" href="" data-tad="431729" data-last-search="narcos+season+2" target="_blank" rel="nofollow"><strong class='colored'>Full Narcos season 2 Download</strong></a> <small><a class="pblink" id="pblink_table_item_4" href="" data-tad="431729" data-last-search="narcos+season+2" target="_blank" rel="nofollow">Usenet</a></small></td><td class="right">24 hours</td><td class="right">775.5 MB</td><td class="right"><font color="green">60</font> </td><td class="right"><font color="navy">236</font> </td><td class="center"><img src="/images/h5.gif" alt="Health" /></td></tr><tr class="tdark"><td><a class="pblink" id="pblink_table_item_5" href="" data-tad="431730" data-last-search="narcos+season+2" target="_blank" rel="nofollow"><strong class='colored'>Narcos season 2</strong> 2014 - DIRECT STREAMING</a> <small><a class="pblink" id="pblink_table_item_6" href="" data-tad="431729" data-last-search="narcos+season+2" target="_blank" rel="nofollow">Movies</a></small></td><td class="right">17 hours</td><td class="right">654.1 MB</td><td class="right"><font color="green">2</font> </td><td class="right"><font color="navy">391</font> </td><td class="center"><img src="/images/h5.gif" alt="Health" /></td></tr><tr class=""><td><a class="pblink" id="pblink_table_item_7" href="" data-tad="431731" data-last-search="narcos+season+2" target="_blank" rel="nofollow"><strong class='colored'>Narcos season 2</strong> 2014</a> <small><a class="pblink" id="pblink_table_item_8" href="" data-tad="431729" data-last-search="narcos+season+2" target="_blank" rel="nofollow">Movies</a></small></td><td class="right">20 hours</td><td class="right">754.5 MB</td><td class="right"><font color="green">21</font> </td><td class="right"><font color="navy">919</font> </td><td class="center"><img src="/images/h5.gif" alt="Health" /></td></tr></table><br /><br /><center><iframe src='http://creative.wwwpromoter.com/13689?d=728x90' width='728' height='90' style='border: none;' frameborder='0' scrolling='no'></iframe><center><span style="float:right;margin:1em .2em 0 0"><a title="Download at the speed of your connection" href="/usenet.php?search=narcos+season+2"><img src="/images/dlf.gif" alt="Search Binaries" /></a></span><div style="margin-bottom:1em;margin-right:290px" id="headerbox"><h1><a href="/searchfeed/narcos+season+2.xml" target="_blank" title="SeedPeer RSS Torrent Search Feed fornarcos season 2"><img src="/images/feedIcon.png" border="0" /></a> 2 <font class="colored">Narcos season 2</font> Torrents were found</h1></div><table width="100%"><tr><th> + <span style="float:right"> + <a href="/search/narcos-season-2/8/1.html"><img style="vertical-align:middle" src="/images/comments.gif" alt="comments" /></a> | + <a href="/search/narcos-season-2/7/1.html"><img style="vertical-align:middle" src="/images/ver.gif" alt="verified" /></a> + </span> + <a href="/search/narcos-season-2/1/1.html">Torrent name</a></th><th class="right"><a href="/search/narcos-season-2/2/1.html">Age</a></th><th class="right"><a href="/search/narcos-season-2/3/1.html">Size</a></th><th class="right"><a href="/search/narcos-season-2/4/1.html">Seeds</a></th><th class="right"><a href="/search/narcos-season-2/5/1.html">Peers</a></th><th class="center"><a href="/search/narcos-season-2/6/1.html">Health</a></th></tr><tr class=""><td><small class="comments"><a href="http://www.facebook.com/sharer.php?t=Download%20<strong class='colored'>Narcos</strong> <strong class='colored'>Season</strong> <strong class='colored'>2</strong> Complete 7<strong class='colored'>2</strong>0p WebRip EN-SUB x<strong class='colored'>2</strong>64-[MULVAcoded] S0<strong class='colored'>2</strong>%20 torrent&u=http://seedpeer.seedpeer.eu/details/11686840/Narcos-Season-2-Complete-720p-WebRip-EN-SUB-x264-[MULVAcoded]-S02.html"><img src="/images/facebook.png" alt="Add to Facebook" width="14" height="14" /></a></small><a href="/details/11686840/Narcos-Season-2-Complete-720p-WebRip-EN-SUB-x264-[MULVAcoded]-S02.html"><strong class='colored'>Narcos</strong> <strong class='colored'>Season</strong> <strong class='colored'>2</strong> Complete 7<strong class='colored'>2</strong>0p WebRip EN-SUB x<strong class='colored'>2</strong>64-[MULVAcoded] S0<strong class='colored'>2</strong> <small><a href="/browse.html#11686840"></a></small></a></td><td class="right">19 hours</td><td class="right">4.39 GB</td><td class="right"><font color="green">715</font> </td><td class="right"><font color="navy">183</font> </td><td class="center"><img src="/images/h5.gif" alt="Health" width="40" height="11" /></td></tr><tr class="tdark"><td><small class="comments"><a href="http://www.facebook.com/sharer.php?t=Download%20<strong class='colored'>Narcos</strong> - <strong class='colored'>Season</strong> <strong class='colored'>2</strong> - 7<strong class='colored'>2</strong>0p WEBRiP - x<strong class='colored'>2</strong>65 HEVC - ShAaNiG%20 torrent&u=http://seedpeer.seedpeer.eu/details/11685972/Narcos---Season-2---720p-WEBRiP---x265-HEVC---ShAaNiG.html"><img src="/images/facebook.png" alt="Add to Facebook" width="14" height="14" /></a></small><a href="/details/11685972/Narcos---Season-2---720p-WEBRiP---x265-HEVC---ShAaNiG.html"><strong class='colored'>Narcos</strong> - <strong class='colored'>Season</strong> <strong class='colored'>2</strong> - 7<strong class='colored'>2</strong>0p WEBRiP - x<strong class='colored'>2</strong>65 HEVC - ShAaNiG <small><a href="/browse.html#11685972"></a></small></a></td><td class="right">1 day</td><td class="right">2.48 GB</td><td class="right"><font color="green">861</font> </td><td class="right"><font color="navy">332</font> </td><td class="center"><img src="/images/h5.gif" alt="Health" width="40" height="11" /></td></tr></table><div id="headerbox"><h1>Related searches for: <font class="colored">Narcos season 2</font></h1></div><div id="search_suggestions"><br />Other suggested searches: </div><br /><a href="http://torrentz2.eu/search?f=narcos-season-2">Search for "narcos-season-2" on Torrentz2.eu</a><br /><a href="http://torrent-finder.info/show.php?q=narcos-season-2">Search for "narcos-season-2" on Torrent-Finder</a><br /><center><iframe src='http://creative.wwwpromoter.com/13689?d=300x250' width='300' height='250' style='border: none;' frameborder='0' scrolling='no'></iframe> <iframe src='http://creative.wwwpromoter.com/13689?d=300x250' width='300' height='250' style='border: none;' frameborder='0' scrolling='no'></iframe> <iframe src='http://creative.wwwpromoter.com/13689?d=300x250' width='300' height='250' style='border: none;' frameborder='0' scrolling='no'></iframe></center><div id="footer"> + <table width="100%"> + <tr> + <td width="30%"> + <h2>Torrents Download</h2> + <a href="/">Torrent search</a><br /> + <a href="/browse.html">Browse categories</a><br /> + <a href="/verified.html">Verified Torrents</a><br /> + <a href="/order-date.html">Today's torrents</a><br /> + <a href="/yesterday.html">Yesterday's torrents</a><br /> + <a href="/stats.html">Statistics</a><br /> + <br /> + <a href="/faq.html#copyright"><strong>Copyright & Removal</strong></a> + </td> + <td width="30%"><h2>Cool Stuff</h2> + <a href="/promotional.php">Promotional</a><br /> + <a href="/contact.html">Advertising Information</a><br /> + <strong><a href="/plugins.php" title="Add a search plugin to Firefox or Internet Explorer">Search Plugin <span style="color:red">*</span></a></strong><br /> + <a href="http://www.utorrent.com">µTorrent Client</a><br /> + <a href="/blog">Seedpeer Blog</a><br /> + </td> + <td width="30%"><h2>Links</h2> + <a href="http://www.sumotorrent.com" target="_blank"><strong>SumoTorrent</strong></a><br /> + <a href="http://www.torrent-finder.info" target="_blank"><strong>Torrent Finder</strong></a><br /> + <a href="http://www.torrentpond.com" target="_blank"><strong>TorrentPond</strong></a><br /> + <a href="https://www.limetorrents.cc" target="_blank">LimeTorrents.cc</a><br /> + <a href="http://www.torrents.to/" target="_blank">Torrents.to</a><br /> + <a href="http://www.torrentfunk.com" target="_blank">TorrentFunk</a><br /> + <a href="https://monova.org" target="_blank">Monova</a><br /> + <a href="http://www.torrentroom.com" target="_blank">TorrentRoom</a><br /> + <a href="http://www.katcr.co/" target="_blank">Kickass Torrents Community</a><br /> + </td> + <td width="10%"><div id="bottomlogo"></div></td> + </tr> + </table> + <br /> + <br /> + </div> + </div> + </body> + </html>+ \ No newline at end of file diff --git a/tests/unit/engines/test_seedpeer.py b/tests/unit/engines/test_seedpeer.py @@ -0,0 +1,51 @@ +import mock +from collections import defaultdict +from searx.engines import seedpeer +from searx.testing import SearxTestCase +from datetime import datetime + + +class TestSeedPeerEngine(SearxTestCase): + + html = '' + with open('./tests/unit/engines/seedpeer_fixture.html') as fixture: + html += fixture.read() + + def test_request(self): + query = 'test_query' + dicto = defaultdict(dict) + dicto['pageno'] = 1 + params = seedpeer.request(query, dicto) + self.assertIn('url', params) + self.assertIn(query, params['url']) + self.assertIn('seedpeer.eu', params['url']) + + def test_response_raises_attr_error_on_empty_response(self): + self.assertRaises(AttributeError, seedpeer.response, None) + self.assertRaises(AttributeError, seedpeer.response, []) + self.assertRaises(AttributeError, seedpeer.response, '') + self.assertRaises(AttributeError, seedpeer.response, '[]') + + def test_response_returns_empty_list(self): + response = mock.Mock(text='<html></html>') + self.assertEqual(seedpeer.response(response), []) + + def test_response_returns_all_results(self): + response = mock.Mock(text=self.html) + results = seedpeer.response(response) + self.assertTrue(isinstance(results, list)) + self.assertEqual(len(results), 2) + + def test_response_returns_correct_results(self): + response = mock.Mock(text=self.html) + results = seedpeer.response(response) + self.assertEqual( + results[0]['title'], 'Narcos - Season 2 - 720p WEBRiP - x265 HEVC - ShAaNiG ' + ) + self.assertEqual( + results[0]['url'], + 'http://www.seedpeer.eu/details/11685972/Narcos---Season-2---720p-WEBRiP---x265-HEVC---ShAaNiG.html' + ) + self.assertEqual(results[0]['content'], '2.48 GB, 1 day') + self.assertEqual(results[0]['seed'], '861') + self.assertEqual(results[0]['leech'], '332')