logo

searx

My custom branche(s) on searx, a meta-search engine git clone https://hacktivis.me/git/searx.git

tokyotoshokan.py (3405B)


  1. """
  2. Tokyo Toshokan (A BitTorrent Library for Japanese Media)
  3. @website https://www.tokyotosho.info/
  4. @provide-api no
  5. @using-api no
  6. @results HTML
  7. @stable no (HTML can change)
  8. @parse url, title, publishedDate, seed, leech,
  9. filesize, magnetlink, content
  10. """
  11. import re
  12. from lxml import html
  13. from searx.engines.xpath import extract_text
  14. from datetime import datetime
  15. from searx.url_utils import urlencode
  16. from searx.utils import get_torrent_size, int_or_zero
  17. # engine dependent config
  18. categories = ['files', 'videos', 'music']
  19. paging = True
  20. # search-url
  21. base_url = 'https://www.tokyotosho.info/'
  22. search_url = base_url + 'search.php?{query}'
  23. # do search-request
  24. def request(query, params):
  25. query = urlencode({'page': params['pageno'], 'terms': query})
  26. params['url'] = search_url.format(query=query)
  27. return params
  28. # get response from search-request
  29. def response(resp):
  30. results = []
  31. dom = html.fromstring(resp.text)
  32. rows = dom.xpath('//table[@class="listing"]//tr[contains(@class, "category_0")]')
  33. # check if there are no results or page layout was changed so we cannot parse it
  34. # currently there are two rows for each result, so total count must be even
  35. if len(rows) == 0 or len(rows) % 2 != 0:
  36. return []
  37. # regular expression for parsing torrent size strings
  38. size_re = re.compile(r'Size:\s*([\d.]+)(TB|GB|MB|B)', re.IGNORECASE)
  39. # processing the results, two rows at a time
  40. for i in range(0, len(rows), 2):
  41. # parse the first row
  42. name_row = rows[i]
  43. links = name_row.xpath('./td[@class="desc-top"]/a')
  44. params = {
  45. 'template': 'torrent.html',
  46. 'url': links[-1].attrib.get('href'),
  47. 'title': extract_text(links[-1])
  48. }
  49. # I have not yet seen any torrents without magnet links, but
  50. # it's better to be prepared to stumble upon one some day
  51. if len(links) == 2:
  52. magnet = links[0].attrib.get('href')
  53. if magnet.startswith('magnet'):
  54. # okay, we have a valid magnet link, let's add it to the result
  55. params['magnetlink'] = magnet
  56. # no more info in the first row, start parsing the second one
  57. info_row = rows[i + 1]
  58. desc = extract_text(info_row.xpath('./td[@class="desc-bot"]')[0])
  59. for item in desc.split('|'):
  60. item = item.strip()
  61. if item.startswith('Size:'):
  62. try:
  63. # ('1.228', 'GB')
  64. groups = size_re.match(item).groups()
  65. params['filesize'] = get_torrent_size(groups[0], groups[1])
  66. except:
  67. pass
  68. elif item.startswith('Date:'):
  69. try:
  70. # Date: 2016-02-21 21:44 UTC
  71. date = datetime.strptime(item, 'Date: %Y-%m-%d %H:%M UTC')
  72. params['publishedDate'] = date
  73. except:
  74. pass
  75. elif item.startswith('Comment:'):
  76. params['content'] = item
  77. stats = info_row.xpath('./td[@class="stats"]/span')
  78. # has the layout not changed yet?
  79. if len(stats) == 3:
  80. params['seed'] = int_or_zero(extract_text(stats[0]))
  81. params['leech'] = int_or_zero(extract_text(stats[1]))
  82. results.append(params)
  83. return results