logo

youtube-dl

[mirror] Download/Watch videos from video hostersgit clone https://hacktivis.me/git/mirror/youtube-dl.git

youporn.py (27267B)

    

  1. # coding: utf-8
  2. from __future__ import unicode_literals
  4. import itertools
  5. import re
  6. from time import sleep
  8. from .common import InfoExtractor
  9. from ..utils import (
  10.     clean_html,
  11.     extract_attributes,
  12.     ExtractorError,
  13.     get_element_by_class,
  14.     get_element_by_id,
  15.     int_or_none,
  16.     merge_dicts,
  17.     parse_count,
  18.     parse_qs,
  19.     T,
  20.     traverse_obj,
  21.     unified_strdate,
  22.     url_or_none,
  23.     urljoin,
  24. )
  27. class YouPornIE(InfoExtractor):
  28.     _VALID_URL = (
  29.         r'youporn:(?P<id>\d+)',
  30.         r'''(?x)
  31.             https?://(?:www\.)?youporn\.com/(?:watch|embed)/(?P<id>\d+)
  32.             (?:/(?:(?P<display_id>[^/?#&]+)/?)?)?(?:[#?]|$)
  33.     '''
  34.     )
  35.     _EMBED_REGEX = [r'<iframe[^>]+\bsrc=["\'](?P<url>(?:https?:)?//(?:www\.)?youporn\.com/embed/\d+)']
  36.     _TESTS = [{
  37.         'url': 'http://www.youporn.com/watch/505835/sex-ed-is-it-safe-to-masturbate-daily/',
  38.         'md5': '3744d24c50438cf5b6f6d59feb5055c2',
  39.         'info_dict': {
  40.             'id': '505835',
  41.             'display_id': 'sex-ed-is-it-safe-to-masturbate-daily',
  42.             'ext': 'mp4',
  43.             'title': 'Sex Ed: Is It Safe To Masturbate Daily?',
  44.             'description': 'Love & Sex Answers: http://bit.ly/DanAndJenn -- Is It Unhealthy To Masturbate Daily?',
  45.             'thumbnail': r're:^https?://.*\.jpg$',
  46.             'duration': 210,
  47.             'uploader': 'Ask Dan And Jennifer',
  48.             'upload_date': '20101217',
  49.             'average_rating': int,
  50.             'view_count': int,
  51.             'categories': list,
  52.             'tags': list,
  53.             'age_limit': 18,
  54.         },
  55.         'skip': 'This video has been deactivated',
  56.     }, {
  57.         # Unknown uploader
  58.         'url': 'http://www.youporn.com/watch/561726/big-tits-awesome-brunette-on-amazing-webcam-show/?from=related3&al=2&from_id=561726&pos=4',
  59.         'info_dict': {
  60.             'id': '561726',
  61.             'display_id': 'big-tits-awesome-brunette-on-amazing-webcam-show',
  62.             'ext': 'mp4',
  63.             'title': 'Big Tits Awesome Brunette On amazing webcam show',
  64.             'description': 'http://sweetlivegirls.com Big Tits Awesome Brunette On amazing webcam show.mp4',
  65.             'thumbnail': r're:^https?://.*\.jpg$',
  66.             'uploader': 'Unknown',
  67.             'upload_date': '20110418',
  68.             'average_rating': int,
  69.             'view_count': int,
  70.             'categories': list,
  71.             'tags': list,
  72.             'age_limit': 18,
  73.         },
  74.         'params': {
  75.             'skip_download': True,
  76.         },
  77.         'skip': '404',
  78.     }, {
  79.         'url': 'https://www.youporn.com/embed/505835/sex-ed-is-it-safe-to-masturbate-daily/',
  80.         'only_matching': True,
  81.     }, {
  82.         'url': 'http://www.youporn.com/watch/505835',
  83.         'only_matching': True,
  84.     }, {
  85.         'url': 'https://www.youporn.com/watch/13922959/femdom-principal/',
  86.         'only_matching': True,
  87.     }, {
  88.         'url': 'https://www.youporn.com/watch/16290308/tinderspecial-trailer1/',
  89.         'info_dict': {
  90.             'id': '16290308',
  91.             'age_limit': 18,
  92.             'categories': [],
  93.             'description': None,  # SEO spam using title removed
  94.             'display_id': 'tinderspecial-trailer1',
  95.             'duration': 298.0,
  96.             'ext': 'mp4',
  97.             'upload_date': '20201123',
  98.             'uploader': 'Ersties',
  99.             'tags': [],
  100.             'thumbnail': 'https://fi1.ypncdn.com/m=eaSaaTbWx/202011/23/16290308/original/3.jpg',
  101.             'timestamp': 1606147564,
  102.             'title': 'Tinder In Real Life',
  103.             'view_count': int,
  104.         }
  105.     }]
  107.     @classmethod
  108.     def _extract_urls(cls, webpage):
  109.         def yield_urls():
  110.             for p in cls._EMBED_REGEX:
  111.                 for m in re.finditer(p, webpage):
  112.                     yield m.group('url')
  114.         return list(yield_urls())
  116.     def _real_extract(self, url):
  117.         # A different video ID (data-video-id) is hidden in the page but
  118.         # never seems to be used
  119.         video_id, display_id = self._match_valid_url(url).group('id', 'display_id')
  120.         url = 'http://www.youporn.com/watch/%s' % (video_id,)
  121.         webpage = self._download_webpage(
  122.             url, video_id, headers={'Cookie': 'age_verified=1'})
  124.         watchable = self._search_regex(
  125.             r'''(<div\s[^>]*\bid\s*=\s*('|")?watch-container(?(2)\2|(?!-)\b)[^>]*>)''',
  126.             webpage, 'watchability', default=None)
  127.         if not watchable:
  128.             msg = re.split(r'\s{4}', clean_html(get_element_by_id(
  129.                 'mainContent', webpage)) or '')[0]
  130.             raise ExtractorError(
  131.                 ('%s says: %s' % (self.IE_NAME, msg))
  132.                 if msg else 'Video unavailable: no reason found',
  133.                 expected=True)
  134.         # internal ID ?
  135.         # video_id = extract_attributes(watchable).get('data-video-id')
  137.         playervars = self._search_json(
  138.             r'\bplayervars\s*:', webpage, 'playervars', video_id)
  140.         def get_fmt(x):
  141.             v_url = url_or_none(x.get('videoUrl'))
  142.             if v_url:
  143.                 x['videoUrl'] = v_url
  144.                 return (x['format'], x)
  146.         defs_by_format = dict(traverse_obj(playervars, (
  147.             'mediaDefinitions', lambda _, v: v.get('format'), T(get_fmt))))
  149.         def get_format_data(f):
  150.             if f not in defs_by_format:
  151.                 return []
  152.             return self._download_json(
  153.                 defs_by_format[f]['videoUrl'], video_id, '{0}-formats'.format(f))
  155.         formats = []
  156.         # Try to extract only the actual master m3u8 first, avoiding the duplicate single resolution "master" m3u8s
  157.         for hls_url in traverse_obj(
  158.                 get_format_data('hls'),
  159.                 (lambda _, v: not isinstance(v['defaultQuality'], bool), 'videoUrl'),
  160.                 (Ellipsis, 'videoUrl')):
  161.             formats.extend(self._extract_m3u8_formats(
  162.                 hls_url, video_id, 'mp4', fatal=False, m3u8_id='hls',
  163.                 entry_protocol='m3u8_native'))
  165.         for f in traverse_obj(get_format_data('mp4'), (
  166.                 lambda _, v: v.get('videoUrl'), {
  167.                     'url': ('videoUrl', T(url_or_none)),
  168.                     'filesize': ('videoSize', T(int_or_none)),
  169.                     'height': ('quality', T(int_or_none)),
  170.                 }, T(lambda x: x.get('videoUrl') and x))):
  171.             # Video URL's path looks like this:
  172.             #  /201012/17/505835/720p_1500k_505835/YouPorn%20-%20Sex%20Ed%20Is%20It%20Safe%20To%20Masturbate%20Daily.mp4
  173.             #  /201012/17/505835/vl_240p_240k_505835/YouPorn%20-%20Sex%20Ed%20Is%20It%20Safe%20To%20Masturbate%20Daily.mp4
  174.             #  /videos/201703/11/109285532/1080P_4000K_109285532.mp4
  175.             # We will benefit from it by extracting some metadata
  176.             mobj = re.search(r'(?P<height>\d{3,4})[pP]_(?P<bitrate>\d+)[kK]_\d+', f['videoUrl'])
  177.             if mobj:
  178.                 if not f.get('height'):
  179.                     f['height'] = int(mobj.group('height'))
  180.                 f['tbr'] = int(mobj.group('bitrate'))
  181.                 f['format_id'] = '%dp-%dk' % (f['height'], f['tbr'])
  182.             formats.append(f)
  183.         self._sort_formats(formats)
  185.         title = self._html_search_regex(
  186.             r'(?s)<div[^>]+class=["\']watchVideoTitle[^>]+>(.+?)</div>',
  187.             webpage, 'title', default=None) or self._og_search_title(
  188.             webpage, default=None) or self._html_search_meta(
  189.             'title', webpage, fatal=True)
  191.         description = self._html_search_regex(
  192.             r'(?s)<div[^>]+\bid=["\']description["\'][^>]*>(.+?)</div>',
  193.             webpage, 'description',
  194.             default=None) or self._og_search_description(
  195.             webpage, default=None)
  196.         thumbnail = self._search_regex(
  197.             r'(?:imageurl\s*=|poster\s*:)\s*(["\'])(?P<thumbnail>.+?)\1',
  198.             webpage, 'thumbnail', fatal=False, group='thumbnail')
  199.         duration = traverse_obj(playervars, ('duration', T(int_or_none)))
  200.         if duration is None:
  201.             duration = int_or_none(self._html_search_meta(
  202.                 'video:duration', webpage, 'duration', fatal=False))
  204.         uploader = self._html_search_regex(
  205.             r'(?s)<div[^>]+class=["\']submitByLink["\'][^>]*>(.+?)</div>',
  206.             webpage, 'uploader', fatal=False)
  207.         upload_date = unified_strdate(self._html_search_regex(
  208.             (r'UPLOADED:\s*<span>([^<]+)',
  209.              r'Date\s+[Aa]dded:\s*<span>([^<]+)',
  210.              r'''(?s)<div[^>]+class=["']videoInfo(?:Date|Time)\b[^>]*>(.+?)</div>''',
  211.              r'(?s)<label\b[^>]*>Uploaded[^<]*</label>\s*<span\b[^>]*>(.+?)</span>'),
  212.             webpage, 'upload date', fatal=False))
  214.         age_limit = self._rta_search(webpage)
  216.         view_count = None
  217.         views = self._search_regex(
  218.             r'(<div\s[^>]*\bdata-value\s*=[^>]+>)\s*<label>Views:</label>',
  219.             webpage, 'views', default=None)
  220.         if views:
  221.             view_count = parse_count(extract_attributes(views).get('data-value'))
  222.         comment_count = parse_count(self._search_regex(
  223.             r'>All [Cc]omments? \(([\d,.]+)\)',
  224.             webpage, 'comment count', default=None))
  226.         def extract_tag_box(regex, title):
  227.             tag_box = self._search_regex(regex, webpage, title, default=None)
  228.             if not tag_box:
  229.                 return []
  230.             return re.findall(r'<a[^>]+href=[^>]+>([^<]+)', tag_box)
  232.         categories = extract_tag_box(
  233.             r'(?s)Categories:.*?</[^>]+>(.+?)</div>', 'categories')
  234.         tags = extract_tag_box(
  235.             r'(?s)Tags:.*?</div>\s*<div[^>]+class=["\']tagBoxContent["\'][^>]*>(.+?)</div>',
  236.             'tags')
  238.         data = self._search_json_ld(webpage, video_id, expected_type='VideoObject', fatal=False) or {}
  239.         data.pop('url', None)
  241.         result = merge_dicts(data, {
  242.             'id': video_id,
  243.             'display_id': display_id,
  244.             'title': title,
  245.             'description': description,
  246.             'thumbnail': thumbnail,
  247.             'duration': duration,
  248.             'uploader': uploader,
  249.             'upload_date': upload_date,
  250.             'view_count': view_count,
  251.             'comment_count': comment_count,
  252.             'categories': categories,
  253.             'tags': tags,
  254.             'age_limit': age_limit,
  255.             'formats': formats,
  256.         })
  257.         # Remove promotional non-description
  258.         if result.get('description', '').startswith(
  259.                 'Watch %s online' % (result['title'],)):
  260.             del result['description']
  261.         return result
  264. class YouPornListBase(InfoExtractor):
  265.     # pattern in '.title-text' element of page section containing videos
  266.     _PLAYLIST_TITLEBAR_RE = r'\s+[Vv]ideos\s*$'
  267.     _PAGE_RETRY_COUNT = 0  # ie, no retry
  268.     _PAGE_RETRY_DELAY = 2  # seconds
  270.     def _get_next_url(self, url, pl_id, html):
  271.         return urljoin(url, self._search_regex(
  272.             r'''<a\s[^>]*?\bhref\s*=\s*("|')(?P<url>(?:(?!\1)[^>])+)\1''',
  273.             get_element_by_id('next', html) or '', 'next page',
  274.             group='url', default=None))
  276.     @classmethod
  277.     def _get_title_from_slug(cls, title_slug):
  278.         return re.sub(r'[_-]', ' ', title_slug)
  280.     def _entries(self, url, pl_id, html=None, page_num=None):
  282.         # separates page sections
  283.         PLAYLIST_SECTION_RE = (
  284.             r'''<div\s[^>]*\bclass\s*=\s*('|")(?:[\w$-]+\s+|\s)*?title-bar(?:\s+[\w$-]+|\s)*\1[^>]*>'''
  285.         )
  286.         # contains video link
  287.         VIDEO_URL_RE = r'''(?x)
  288.             <div\s[^>]*\bdata-video-id\s*=\s*('|")\d+\1[^>]*>\s*
  289.             (?:<div\b[\s\S]+?</div>\s*)*
  290.             <a\s[^>]*\bhref\s*=\s*('|")(?P<url>(?:(?!\2)[^>])+)\2
  291.         '''
  293.         def yield_pages(url, html=html, page_num=page_num):
  294.             fatal = not html
  295.             for pnum in itertools.count(start=page_num or 1):
  296.                 if not html:
  297.                     html = self._download_webpage(
  298.                         url, pl_id, note='Downloading page %d' % pnum,
  299.                         fatal=fatal)
  300.                 if not html:
  301.                     break
  302.                 fatal = False
  303.                 yield (url, html, pnum)
  304.                 # explicit page: extract just that page
  305.                 if page_num is not None:
  306.                     break
  307.                 next_url = self._get_next_url(url, pl_id, html)
  308.                 if not next_url or next_url == url:
  309.                     break
  310.                 url, html = next_url, None
  312.         def retry_page(msg, tries_left, page_data):
  313.             if tries_left <= 0:
  314.                 return
  315.             self.report_warning(msg, pl_id)
  316.             sleep(self._PAGE_RETRY_DELAY)
  317.             return next(
  318.                 yield_pages(page_data[0], page_num=page_data[2]), None)
  320.         def yield_entries(html):
  321.             for frag in re.split(PLAYLIST_SECTION_RE, html):
  322.                 if not frag:
  323.                     continue
  324.                 t_text = get_element_by_class('title-text', frag or '')
  325.                 if not (t_text and re.search(self._PLAYLIST_TITLEBAR_RE, t_text)):
  326.                     continue
  327.                 for m in re.finditer(VIDEO_URL_RE, frag):
  328.                     video_url = urljoin(url, m.group('url'))
  329.                     if video_url:
  330.                         yield self.url_result(video_url)
  332.         last_first_url = None
  333.         for page_data in yield_pages(url, html=html, page_num=page_num):
  334.             # page_data: url, html, page_num
  335.             first_url = None
  336.             tries_left = self._PAGE_RETRY_COUNT + 1
  337.             while tries_left > 0:
  338.                 tries_left -= 1
  339.                 for from_ in yield_entries(page_data[1]):
  340.                     # may get the same page twice instead of empty page
  341.                     # or (site bug) intead of actual next page
  342.                     if not first_url:
  343.                         first_url = from_['url']
  344.                         if first_url == last_first_url:
  345.                             # sometimes (/porntags/) the site serves the previous page
  346.                             # instead but may provide the correct page after a delay
  347.                             page_data = retry_page(
  348.                                 'Retrying duplicate page...', tries_left, page_data)
  349.                             if page_data:
  350.                                 first_url = None
  351.                                 break
  352.                             continue
  353.                     yield from_
  354.                 else:
  355.                     if not first_url and 'no-result-paragarph1' in page_data[1]:
  356.                         page_data = retry_page(
  357.                             'Retrying empty page...', tries_left, page_data)
  358.                         if page_data:
  359.                             continue
  360.                     else:
  361.                         # success/failure
  362.                         break
  363.             # may get an infinite (?) sequence of empty pages
  364.             if not first_url:
  365.                 break
  366.             last_first_url = first_url
  368.     def _real_extract(self, url, html=None):
  369.         # exceptionally, id may be None
  370.         m_dict = self._match_valid_url(url).groupdict()
  371.         pl_id, page_type, sort = (m_dict.get(k) for k in ('id', 'type', 'sort'))
  373.         qs = parse_qs(url)
  374.         for q, v in qs.items():
  375.             if v:
  376.                 qs[q] = v[-1]
  377.             else:
  378.                 del qs[q]
  380.         base_id = pl_id or 'YouPorn'
  381.         title = self._get_title_from_slug(base_id)
  382.         if page_type:
  383.             title = '%s %s' % (page_type.capitalize(), title)
  384.         base_id = [base_id.lower()]
  385.         if sort is None:
  386.             title += ' videos'
  387.         else:
  388.             title = '%s videos by %s' % (title, re.sub(r'[_-]', ' ', sort))
  389.             base_id.append(sort)
  390.         if qs:
  391.             ps = ['%s=%s' % item for item in sorted(qs.items())]
  392.             title += ' (%s)' % ','.join(ps)
  393.             base_id.extend(ps)
  394.         pl_id = '/'.join(base_id)
  396.         return self.playlist_result(
  397.             self._entries(url, pl_id, html=html,
  398.                           page_num=int_or_none(qs.get('page'))),
  399.             playlist_id=pl_id, playlist_title=title)
  402. class YouPornCategoryIE(YouPornListBase):
  403.     IE_DESC = 'YouPorn category, with sorting, filtering and pagination'
  404.     _VALID_URL = r'''(?x)
  405.         https?://(?:www\.)?youporn\.com/
  406.         (?P<type>category)/(?P<id>[^/?#&]+)
  407.         (?:/(?P<sort>popular|views|rating|time|duration))?/?(?:[#?]|$)
  408.     '''
  409.     _TESTS = [{
  410.         'note': 'Full list with pagination',
  411.         'url': 'https://www.youporn.com/category/lingerie/popular/',
  412.         'info_dict': {
  413.             'id': 'lingerie/popular',
  414.             'title': 'Category lingerie videos by popular',
  415.         },
  416.         'playlist_mincount': 39,
  417.     }, {
  418.         'note': 'Filtered paginated list with single page result',
  419.         'url': 'https://www.youporn.com/category/lingerie/duration/?min_minutes=10',
  420.         'info_dict': {
  421.             'id': 'lingerie/duration/min_minutes=10',
  422.             'title': 'Category lingerie videos by duration (min_minutes=10)',
  423.         },
  424.         'playlist_maxcount': 30,
  425.     }, {
  426.         'note': 'Single page of full list',
  427.         'url': 'https://www.youporn.com/category/lingerie/popular?page=1',
  428.         'info_dict': {
  429.             'id': 'lingerie/popular/page=1',
  430.             'title': 'Category lingerie videos by popular (page=1)',
  431.         },
  432.         'playlist_count': 30,
  433.     }]
  436. class YouPornChannelIE(YouPornListBase):
  437.     IE_DESC = 'YouPorn channel, with sorting and pagination'
  438.     _VALID_URL = r'''(?x)
  439.         https?://(?:www\.)?youporn\.com/
  440.         (?P<type>channel)/(?P<id>[^/?#&]+)
  441.         (?:/(?P<sort>rating|views|duration))?/?(?:[#?]|$)
  442.     '''
  443.     _TESTS = [{
  444.         'note': 'Full list with pagination',
  445.         'url': 'https://www.youporn.com/channel/x-feeds/',
  446.         'info_dict': {
  447.             'id': 'x-feeds',
  448.             'title': 'Channel X-Feeds videos',
  449.         },
  450.         'playlist_mincount': 37,
  451.     }, {
  452.         'note': 'Single page of full list (no filters here)',
  453.         'url': 'https://www.youporn.com/channel/x-feeds/duration?page=1',
  454.         'info_dict': {
  455.             'id': 'x-feeds/duration/page=1',
  456.             'title': 'Channel X-Feeds videos by duration (page=1)',
  457.         },
  458.         'playlist_count': 24,
  459.     }]
  461.     @staticmethod
  462.     def _get_title_from_slug(title_slug):
  463.         return re.sub(r'_', ' ', title_slug).title()
  466. class YouPornCollectionIE(YouPornListBase):
  467.     IE_DESC = 'YouPorn collection (user playlist), with sorting and pagination'
  468.     _VALID_URL = r'''(?x)
  469.         https?://(?:www\.)?youporn\.com/
  470.         (?P<type>collection)s/videos/(?P<id>\d+)
  471.         (?:/(?P<sort>rating|views|time|duration))?/?(?:[#?]|$)
  472.     '''
  473.     _PLAYLIST_TITLEBAR_RE = r'^\s*Videos\s+in\s'
  474.     _TESTS = [{
  475.         'note': 'Full list with pagination',
  476.         'url': 'https://www.youporn.com/collections/videos/33044251/',
  477.         'info_dict': {
  478.             'id': '33044251',
  479.             'title': 'Collection Sexy Lips videos',
  480.             'uploader': 'ph-littlewillyb',
  481.         },
  482.         'playlist_mincount': 50,
  483.     }, {
  484.         'note': 'Single page of full list (no filters here)',
  485.         'url': 'https://www.youporn.com/collections/videos/33044251/time?page=1',
  486.         'info_dict': {
  487.             'id': '33044251/time/page=1',
  488.             'title': 'Collection Sexy Lips videos by time (page=1)',
  489.             'uploader': 'ph-littlewillyb',
  490.         },
  491.         'playlist_count': 20,
  492.     }]
  494.     def _real_extract(self, url):
  495.         pl_id = self._match_id(url)
  496.         html = self._download_webpage(url, pl_id)
  497.         playlist = super(YouPornCollectionIE, self)._real_extract(url, html=html)
  498.         infos = re.sub(r'\s+', ' ', clean_html(get_element_by_class(
  499.             'collection-infos', html)) or '')
  500.         title, uploader = self._search_regex(
  501.             r'^\s*Collection: (?P<title>.+?) \d+ VIDEOS \d+ VIEWS \d+ days LAST UPDATED From: (?P<uploader>[\w_-]+)',
  502.             infos, 'title/uploader', group=('title', 'uploader'), default=(None, None))
  504.         return merge_dicts({
  505.             'title': playlist['title'].replace(playlist['id'].split('/')[0], title),
  506.             'uploader': uploader,
  507.         }, playlist) if title else playlist
  510. class YouPornTagIE(YouPornListBase):
  511.     IE_DESC = 'YouPorn tag (porntags), with sorting, filtering and pagination'
  512.     _VALID_URL = r'''(?x)
  513.         https?://(?:www\.)?youporn\.com/
  514.         porn(?P<type>tag)s/(?P<id>[^/?#&]+)
  515.         (?:/(?P<sort>views|rating|time|duration))?/?(?:[#?]|$)
  516.     '''
  517.     _PLAYLIST_TITLEBAR_RE = r'^\s*Videos\s+tagged\s'
  518.     _PAGE_RETRY_COUNT = 1
  519.     _TESTS = [{
  520.         'note': 'Full list with pagination',
  521.         'url': 'https://www.youporn.com/porntags/austrian',
  522.         'info_dict': {
  523.             'id': 'austrian',
  524.             'title': 'Tag austrian videos',
  525.         },
  526.         'playlist_mincount': 35,
  527.         'expected_warnings': ['Retrying duplicate page'],
  528.     }, {
  529.         'note': 'Filtered paginated list with single page result',
  530.         'url': 'https://www.youporn.com/porntags/austrian/duration/?min_minutes=10',
  531.         'info_dict': {
  532.             'id': 'austrian/duration/min_minutes=10',
  533.             'title': 'Tag austrian videos by duration (min_minutes=10)',
  534.         },
  535.         # number of videos per page is (row x col) 2x3 + 6x4 + 2, or + 3,
  536.         # or more, varying with number of ads; let's set max as 9x4
  537.         # NB col 1 may not be shown in non-JS page with site CSS and zoom 100%
  538.         'playlist_maxcount': 32,
  539.         'expected_warnings': ['Retrying duplicate page', 'Retrying empty page'],
  540.     }, {
  541.         'note': 'Single page of full list',
  542.         'url': 'https://www.youporn.com/porntags/austrian/?page=1',
  543.         'info_dict': {
  544.             'id': 'austrian/page=1',
  545.             'title': 'Tag austrian videos (page=1)',
  546.         },
  547.         'playlist_mincount': 32,
  548.         'playlist_maxcount': 34,
  549.         'expected_warnings': ['Retrying duplicate page', 'Retrying empty page'],
  550.     }]
  552.     # YP tag navigation is broken, loses sort
  553.     def _get_next_url(self, url, pl_id, html):
  554.         next_url = super(YouPornTagIE, self)._get_next_url(url, pl_id, html)
  555.         if next_url:
  556.             n = self._match_valid_url(next_url)
  557.             if n:
  558.                 s = n.groupdict().get('sort')
  559.             if s:
  560.                 u = self._match_valid_url(url)
  561.                 if u:
  562.                     u = u.groupdict().get('sort')
  563.                     if s and not u:
  564.                         n = n.end('sort')
  565.                         next_url = next_url[:n] + '/' + u + next_url[n:]
  566.         return next_url
  569. class YouPornStarIE(YouPornListBase):
  570.     IE_DESC = 'YouPorn Pornstar, with description, sorting and pagination'
  571.     _VALID_URL = r'''(?x)
  572.         https?://(?:www\.)?youporn\.com/
  573.         (?P<type>pornstar)/(?P<id>[^/?#&]+)
  574.         (?:/(?P<sort>rating|views|duration))?/?(?:[#?]|$)
  575.     '''
  576.     _PLAYLIST_TITLEBAR_RE = r'^\s*Videos\s+[fF]eaturing\s'
  577.     _TESTS = [{
  578.         'note': 'Full list with pagination',
  579.         'url': 'https://www.youporn.com/pornstar/daynia/',
  580.         'info_dict': {
  581.             'id': 'daynia',
  582.             'title': 'Pornstar Daynia videos',
  583.             'description': r're:Daynia Rank \d+ Videos \d+ Views [\d,.]+ .+ Subscribers \d+',
  584.         },
  585.         'playlist_mincount': 45,
  586.     }, {
  587.         'note': 'Single page of full list (no filters here)',
  588.         'url': 'https://www.youporn.com/pornstar/daynia/?page=1',
  589.         'info_dict': {
  590.             'id': 'daynia/page=1',
  591.             'title': 'Pornstar Daynia videos (page=1)',
  592.             'description': 're:.{180,}',
  593.         },
  594.         'playlist_count': 26,
  595.     }]
  597.     @staticmethod
  598.     def _get_title_from_slug(title_slug):
  599.         return re.sub(r'_', ' ', title_slug).title()
  601.     def _real_extract(self, url):
  602.         pl_id = self._match_id(url)
  603.         html = self._download_webpage(url, pl_id)
  604.         playlist = super(YouPornStarIE, self)._real_extract(url, html=html)
  605.         INFO_ELEMENT_RE = r'''(?x)
  606.             <div\s[^>]*\bclass\s*=\s*('|")(?:[\w$-]+\s+|\s)*?pornstar-info-wrapper(?:\s+[\w$-]+|\s)*\1[^>]*>
  607.             (?P<info>[\s\S]+?)(?:</div>\s*){6,}
  608.         '''
  610.         infos = self._search_regex(INFO_ELEMENT_RE, html, 'infos', group='info', default='')
  611.         if infos:
  612.             infos = re.sub(
  613.                 r'(?:\s*nl=nl)+\s*', ' ',
  614.                 re.sub(r'(?u)\s+', ' ', clean_html(
  615.                     re.sub('\n', 'nl=nl', infos)))).replace('ribe Subsc', '')
  617.         return merge_dicts({
  618.             'description': infos.strip() or None,
  619.         }, playlist)
  622. class YouPornVideosIE(YouPornListBase):
  623.     IE_DESC = 'YouPorn video (browse) playlists, with sorting, filtering and pagination'
  624.     _VALID_URL = r'''(?x)
  625.         https?://(?:www\.)?youporn\.com/
  626.             (?:(?P<id>browse)/)?
  627.             (?P<sort>(?(id)
  628.                 (?:duration|rating|time|views)|
  629.                 (?:most_(?:favou?rit|view)ed|recommended|top_rated)?))
  630.             (?:[/#?]|$)
  631.     '''
  632.     _PLAYLIST_TITLEBAR_RE = r'\s+(?:[Vv]ideos|VIDEOS)\s*$'
  633.     _TESTS = [{
  634.         'note': 'Full list with pagination (too long for test)',
  635.         'url': 'https://www.youporn.com/',
  636.         'info_dict': {
  637.             'id': 'youporn',
  638.             'title': 'YouPorn videos',
  639.         },
  640.         'only_matching': True,
  641.     }, {
  642.         'note': 'Full list with pagination (too long for test)',
  643.         'url': 'https://www.youporn.com/recommended',
  644.         'info_dict': {
  645.             'id': 'youporn/recommended',
  646.             'title': 'YouPorn videos by recommended',
  647.         },
  648.         'only_matching': True,
  649.     }, {
  650.         'note': 'Full list with pagination (too long for test)',
  651.         'url': 'https://www.youporn.com/top_rated',
  652.         'info_dict': {
  653.             'id': 'youporn/top_rated',
  654.             'title': 'YouPorn videos by top rated',
  655.         },
  656.         'only_matching': True,
  657.     }, {
  658.         'note': 'Full list with pagination (too long for test)',
  659.         'url': 'https://www.youporn.com/browse/time',
  660.         'info_dict': {
  661.             'id': 'browse/time',
  662.             'title': 'YouPorn videos by time',
  663.         },
  664.         'only_matching': True,
  665.     }, {
  666.         'note': 'Filtered paginated list with single page result',
  667.         'url': 'https://www.youporn.com/most_favorited/?res=VR&max_minutes=2',
  668.         'info_dict': {
  669.             'id': 'youporn/most_favorited/max_minutes=2/res=VR',
  670.             'title': 'YouPorn videos by most favorited (max_minutes=2,res=VR)',
  671.         },
  672.         'playlist_mincount': 10,
  673.         'playlist_maxcount': 28,
  674.     }, {
  675.         'note': 'Filtered paginated list with several pages',
  676.         'url': 'https://www.youporn.com/most_favorited/?res=VR&max_minutes=5',
  677.         'info_dict': {
  678.             'id': 'youporn/most_favorited/max_minutes=5/res=VR',
  679.             'title': 'YouPorn videos by most favorited (max_minutes=5,res=VR)',
  680.         },
  681.         'playlist_mincount': 45,
  682.     }, {
  683.         'note': 'Single page of full list',
  684.         'url': 'https://www.youporn.com/browse/time?page=1',
  685.         'info_dict': {
  686.             'id': 'browse/time/page=1',
  687.             'title': 'YouPorn videos by time (page=1)',
  688.         },
  689.         'playlist_count': 36,
  690.     }]
  692.     @staticmethod
  693.     def _get_title_from_slug(title_slug):
  694.         return 'YouPorn' if title_slug == 'browse' else title_slug