logo

youtube-dl

[mirror] Download/Watch videos from video hostersgit clone https://hacktivis.me/git/mirror/youtube-dl.git

imgur.py (13298B)

    

  1. # coding: utf-8
  2. from __future__ import unicode_literals
  4. import re
  6. from .common import InfoExtractor
  7. from ..utils import (
  8.     determine_ext,
  9.     ExtractorError,
  10.     float_or_none,
  11.     int_or_none,
  12.     js_to_json,
  13.     merge_dicts,
  14.     mimetype2ext,
  15.     parse_iso8601,
  16.     T,
  17.     traverse_obj,
  18.     txt_or_none,
  19.     url_or_none,
  20. )
  23. class ImgurBaseIE(InfoExtractor):
  24.     # hard-coded value, as also used by ArchiveTeam
  25.     _CLIENT_ID = '546c25a59c58ad7'
  27.     @classmethod
  28.     def _imgur_result(cls, item_id):
  29.         return cls.url_result('imgur:%s' % item_id, ImgurIE.ie_key(), item_id)
  31.     def _call_api(self, endpoint, video_id, **kwargs):
  32.         return self._download_json(
  33.             'https://api.imgur.com/post/v1/%s/%s?client_id=%s&include=media,account' % (endpoint, video_id, self._CLIENT_ID),
  34.             video_id, **kwargs)
  36.     @staticmethod
  37.     def get_description(s):
  38.         if 'Discover the magic of the internet at Imgur' in s:
  39.             return None
  40.         return txt_or_none(s)
  43. class ImgurIE(ImgurBaseIE):
  44.     _VALID_URL = r'''(?x)
  45.         (?:
  46.             https?://(?:i\.)?imgur\.com/(?!(?:a|gallery|t|topic|r)/)|
  47.             imgur:
  48.         )(?P<id>[a-zA-Z0-9]+)
  49.     '''
  51.     _TESTS = [{
  52.         'url': 'https://imgur.com/A61SaA1',
  53.         'info_dict': {
  54.             'id': 'A61SaA1',
  55.             'ext': 'mp4',
  56.             'title': 're:Imgur GIF$|MRW gifv is up and running without any bugs$',
  57.             'timestamp': 1416446068,
  58.             'upload_date': '20141120',
  59.         },
  60.     }, {
  61.         'url': 'https://i.imgur.com/A61SaA1.gifv',
  62.         'only_matching': True,
  63.     }, {
  64.         'url': 'https://i.imgur.com/crGpqCV.mp4',
  65.         'only_matching': True,
  66.     }, {
  67.         # previously, no title
  68.         'url': 'https://i.imgur.com/jxBXAMC.gifv',
  69.         'info_dict': {
  70.             'id': 'jxBXAMC',
  71.             'ext': 'mp4',
  72.             'title': 'Fahaka puffer feeding',
  73.             'timestamp': 1533835503,
  74.             'upload_date': '20180809',
  75.         },
  76.     }]
  78.     def _extract_twitter_formats(self, html, tw_id='twitter', **kwargs):
  79.         fatal = kwargs.pop('fatal', False)
  80.         tw_stream = self._html_search_meta('twitter:player:stream', html, fatal=fatal, **kwargs)
  81.         if not tw_stream:
  82.             return []
  83.         ext = mimetype2ext(self._html_search_meta(
  84.             'twitter:player:stream:content_type', html, default=None))
  85.         width, height = (int_or_none(self._html_search_meta('twitter:player:' + v, html, default=None))
  86.                          for v in ('width', 'height'))
  87.         return [{
  88.             'format_id': tw_id,
  89.             'url': tw_stream,
  90.             'ext': ext or determine_ext(tw_stream),
  91.             'width': width,
  92.             'height': height,
  93.         }]
  95.     def _real_extract(self, url):
  96.         video_id = self._match_id(url)
  97.         data = self._call_api('media', video_id, fatal=False, expected_status=404)
  98.         webpage = self._download_webpage(
  99.             'https://i.imgur.com/{id}.gifv'.format(id=video_id), video_id, fatal=not data) or ''
  101.         if not traverse_obj(data, ('media', 0, (
  102.                 ('type', T(lambda t: t == 'video' or None)),
  103.                 ('metadata', 'is_animated'))), get_all=False):
  104.             raise ExtractorError(
  105.                 '%s is not a video or animated image' % video_id,
  106.                 expected=True)
  108.         media_fmt = traverse_obj(data, ('media', 0, {
  109.             'url': ('url', T(url_or_none)),
  110.             'ext': 'ext',
  111.             'width': ('width', T(int_or_none)),
  112.             'height': ('height', T(int_or_none)),
  113.             'filesize': ('size', T(int_or_none)),
  114.             'acodec': ('metadata', 'has_sound', T(lambda b: None if b else 'none')),
  115.         }))
  117.         media_url = traverse_obj(media_fmt, 'url')
  118.         if media_url:
  119.             if not media_fmt.get('ext'):
  120.                 media_fmt['ext'] = mimetype2ext(traverse_obj(
  121.                     data, ('media', 0, 'mime_type'))) or determine_ext(media_url)
  122.             if traverse_obj(data, ('media', 0, 'type')) == 'image':
  123.                 media_fmt['acodec'] = 'none'
  124.                 media_fmt.setdefault('preference', -10)
  126.         tw_formats = self._extract_twitter_formats(webpage)
  127.         if traverse_obj(tw_formats, (0, 'url')) == media_url:
  128.             tw_formats = []
  129.         else:
  130.             # maybe this isn't an animated image/video?
  131.             self._check_formats(tw_formats, video_id)
  133.         video_elements = self._search_regex(
  134.             r'(?s)<div class="video-elements">(.*?)</div>',
  135.             webpage, 'video elements', default=None)
  136.         if not (video_elements or tw_formats or media_url):
  137.             raise ExtractorError(
  138.                 'No sources found for video %s. Maybe a plain image?' % video_id,
  139.                 expected=True)
  141.         def mung_format(fmt, *extra):
  142.             fmt.update({
  143.                 'http_headers': {
  144.                     'User-Agent': 'youtube-dl (like wget)',
  145.                 },
  146.             })
  147.             for d in extra:
  148.                 fmt.update(d)
  149.             return fmt
  151.         if video_elements:
  152.             def og_get_size(media_type):
  153.                 return dict((p, int_or_none(self._og_search_property(
  154.                     ':'.join((media_type, p)), webpage, default=None)))
  155.                     for p in ('width', 'height'))
  157.             size = og_get_size('video')
  158.             if all(v is None for v in size.values()):
  159.                 size = og_get_size('image')
  161.             formats = traverse_obj(
  162.                 re.finditer(r'<source\s+src="(?P<src>[^"]+)"\s+type="(?P<type>[^"]+)"', video_elements),
  163.                 (Ellipsis, {
  164.                     'format_id': ('type', T(lambda s: s.partition('/')[2])),
  165.                     'url': ('src', T(self._proto_relative_url)),
  166.                     'ext': ('type', T(mimetype2ext)),
  167.                 }, T(lambda f: mung_format(f, size))))
  169.             gif_json = self._search_regex(
  170.                 r'(?s)var\s+videoItem\s*=\s*(\{.*?\})',
  171.                 webpage, 'GIF code', fatal=False)
  172.             MUST_BRANCH = (None, T(lambda _: None))
  173.             formats.extend(traverse_obj(gif_json, (
  174.                 T(lambda j: self._parse_json(
  175.                     j, video_id, transform_source=js_to_json, fatal=False)), {
  176.                         'url': ('gifUrl', T(self._proto_relative_url)),
  177.                         'filesize': ('size', T(int_or_none)),
  178.                 }, T(lambda f: mung_format(f, size, {
  179.                     'format_id': 'gif',
  180.                     'preference': -10,  # gifs are worse than videos
  181.                     'ext': 'gif',
  182.                     'acodec': 'none',
  183.                     'vcodec': 'gif',
  184.                     'container': 'gif',
  185.                 })), MUST_BRANCH)))
  186.         else:
  187.             formats = []
  189.         # maybe add formats from JSON or page Twitter metadata
  190.         if not any((u == media_url) for u in traverse_obj(formats, (Ellipsis, 'url'))):
  191.             formats.append(mung_format(media_fmt))
  192.         tw_url = traverse_obj(tw_formats, (0, 'url'))
  193.         if not any((u == tw_url) for u in traverse_obj(formats, (Ellipsis, 'url'))):
  194.             formats.extend(mung_format(f) for f in tw_formats)
  196.         self._sort_formats(formats)
  198.         return merge_dicts(traverse_obj(data, {
  199.             'uploader_id': ('account_id', T(txt_or_none),
  200.                             T(lambda a: a if int_or_none(a) != 0 else None)),
  201.             'uploader': ('account', 'username', T(txt_or_none)),
  202.             'uploader_url': ('account', 'avatar_url', T(url_or_none)),
  203.             'like_count': ('upvote_count', T(int_or_none)),
  204.             'dislike_count': ('downvote_count', T(int_or_none)),
  205.             'comment_count': ('comment_count', T(int_or_none)),
  206.             'age_limit': ('is_mature', T(lambda x: 18 if x else None)),
  207.             'timestamp': (('updated_at', 'created_at'), T(parse_iso8601)),
  208.             'release_timestamp': ('created_at', T(parse_iso8601)),
  209.         }, get_all=False), traverse_obj(data, ('media', 0, 'metadata', {
  210.             'title': ('title', T(txt_or_none)),
  211.             'description': ('description', T(self.get_description)),
  212.             'duration': ('duration', T(float_or_none)),
  213.             'timestamp': (('updated_at', 'created_at'), T(parse_iso8601)),
  214.             'release_timestamp': ('created_at', T(parse_iso8601)),
  215.         })), {
  216.             'id': video_id,
  217.             'formats': formats,
  218.             'title': self._og_search_title(webpage, default='Imgur video ' + video_id),
  219.             'description': self.get_description(self._og_search_description(webpage)),
  220.             'thumbnail': url_or_none(self._html_search_meta('thumbnailUrl', webpage, default=None)),
  221.         })
  224. class ImgurGalleryBaseIE(ImgurBaseIE):
  225.     _GALLERY = True
  227.     def _real_extract(self, url):
  228.         gallery_id = self._match_id(url)
  230.         data = self._call_api('albums', gallery_id, fatal=False, expected_status=404)
  232.         info = traverse_obj(data, {
  233.             'title': ('title', T(txt_or_none)),
  234.             'description': ('description', T(self.get_description)),
  235.         })
  237.         if traverse_obj(data, 'is_album'):
  239.             def yield_media_ids():
  240.                 for m_id in traverse_obj(data, (
  241.                         'media', lambda _, v: v.get('type') == 'video' or v['metadata']['is_animated'],
  242.                         'id', T(txt_or_none))):
  243.                     yield m_id
  245.             # if a gallery with exactly one video, apply album metadata to video
  246.             media_id = (
  247.                 self._GALLERY
  248.                 and traverse_obj(data, ('image_count', T(lambda c: c == 1)))
  249.                 and next(yield_media_ids(), None))
  251.             if not media_id:
  252.                 result = self.playlist_result(
  253.                     map(self._imgur_result, yield_media_ids()), gallery_id)
  254.                 result.update(info)
  255.                 return result
  256.             gallery_id = media_id
  258.         result = self._imgur_result(gallery_id)
  259.         info['_type'] = 'url_transparent'
  260.         result.update(info)
  261.         return result
  264. class ImgurGalleryIE(ImgurGalleryBaseIE):
  265.     IE_NAME = 'imgur:gallery'
  266.     _VALID_URL = r'https?://(?:i\.)?imgur\.com/(?:gallery|(?:t(?:opic)?|r)/[^/]+)/(?P<id>[a-zA-Z0-9]+)'
  268.     _TESTS = [{
  269.         'url': 'http://imgur.com/gallery/Q95ko',
  270.         'info_dict': {
  271.             'id': 'Q95ko',
  272.             'title': 'Adding faces make every GIF better',
  273.         },
  274.         'playlist_count': 25,
  275.         'skip': 'Zoinks! You\'ve taken a wrong turn.',
  276.     }, {
  277.         # TODO: static images - replace with animated/video gallery
  278.         'url': 'http://imgur.com/topic/Aww/ll5Vk',
  279.         'only_matching': True,
  280.     }, {
  281.         'url': 'https://imgur.com/gallery/YcAQlkx',
  282.         'add_ies': ['Imgur'],
  283.         'info_dict': {
  284.             'id': 'YcAQlkx',
  285.             'ext': 'mp4',
  286.             'title': 'Classic Steve Carell gif...cracks me up everytime....damn the repost downvotes....',
  287.             'timestamp': 1358554297,
  288.             'upload_date': '20130119',
  289.             'uploader_id': '1648642',
  290.             'uploader': 'wittyusernamehere',
  291.         },
  292.     }, {
  293.         # TODO: static image - replace with animated/video gallery
  294.         'url': 'http://imgur.com/topic/Funny/N8rOudd',
  295.         'only_matching': True,
  296.     }, {
  297.         'url': 'http://imgur.com/r/aww/VQcQPhM',
  298.         'add_ies': ['Imgur'],
  299.         'info_dict': {
  300.             'id': 'VQcQPhM',
  301.             'ext': 'mp4',
  302.             'title': 'The boss is here',
  303.             'timestamp': 1476494751,
  304.             'upload_date': '20161015',
  305.             'uploader_id': '19138530',
  306.             'uploader': 'thematrixcam',
  307.         },
  308.     },
  309.         # from PR #16674
  310.         {
  311.         'url': 'https://imgur.com/t/unmuted/6lAn9VQ',
  312.         'info_dict': {
  313.             'id': '6lAn9VQ',
  314.             'title': 'Penguins !',
  315.         },
  316.         'playlist_count': 3,
  317.     }, {
  318.         'url': 'https://imgur.com/t/unmuted/kx2uD3C',
  319.         'add_ies': ['Imgur'],
  320.         'info_dict': {
  321.             'id': 'ZVMv45i',
  322.             'ext': 'mp4',
  323.             'title': 'Intruder',
  324.             'timestamp': 1528129683,
  325.             'upload_date': '20180604',
  326.         },
  327.     }, {
  328.         'url': 'https://imgur.com/t/unmuted/wXSK0YH',
  329.         'add_ies': ['Imgur'],
  330.         'info_dict': {
  331.             'id': 'JCAP4io',
  332.             'ext': 'mp4',
  333.             'title': 're:I got the blues$',
  334.             'description': 'Luka’s vocal stylings.\n\nFP edit: don’t encourage me. I’ll never stop posting Luka and friends.',
  335.             'timestamp': 1527809525,
  336.             'upload_date': '20180531',
  337.         },
  338.     }]
  341. class ImgurAlbumIE(ImgurGalleryBaseIE):
  342.     IE_NAME = 'imgur:album'
  343.     _VALID_URL = r'https?://(?:i\.)?imgur\.com/a/(?P<id>[a-zA-Z0-9]+)'
  344.     _GALLERY = False
  345.     _TESTS = [{
  346.         # TODO: only static images - replace with animated/video gallery
  347.         'url': 'http://imgur.com/a/j6Orj',
  348.         'only_matching': True,
  349.     },
  350.         # from PR #21693
  351.         {
  352.         'url': 'https://imgur.com/a/iX265HX',
  353.         'info_dict': {
  354.             'id': 'iX265HX',
  355.             'title': 'enen-no-shouboutai'
  356.         },
  357.         'playlist_count': 2,
  358.     }, {
  359.         'url': 'https://imgur.com/a/8pih2Ed',
  360.         'info_dict': {
  361.             'id': '8pih2Ed'
  362.         },
  363.         'playlist_mincount': 1,
  364.     }]