logo

youtube-dl

[mirror] Download/Watch videos from video hostersgit clone https://hacktivis.me/git/mirror/youtube-dl.git

mixcloud.py (12454B)

    

  1. # coding: utf-8
  2. from __future__ import unicode_literals
  4. import itertools
  5. import re
  7. from .common import InfoExtractor
  8. from ..compat import (
  9.     compat_b64decode,
  10.     compat_chr,
  11.     compat_ord,
  12.     compat_str,
  13.     compat_urllib_parse_unquote,
  14.     compat_zip as zip,
  15. )
  16. from ..utils import (
  17.     int_or_none,
  18.     parse_iso8601,
  19.     strip_or_none,
  20.     try_get,
  21. )
  24. class MixcloudBaseIE(InfoExtractor):
  25.     def _call_api(self, object_type, object_fields, display_id, username, slug=None):
  26.         lookup_key = object_type + 'Lookup'
  27.         return self._download_json(
  28.             'https://app.mixcloud.com/graphql', display_id, query={
  29.                 'query': '''{
  30.   %s(lookup: {username: "%s"%s}) {
  31.     %s
  32.   }
  33. }''' % (lookup_key, username, ', slug: "%s"' % slug if slug else '', object_fields)
  34.             })['data'][lookup_key]
  37. class MixcloudIE(MixcloudBaseIE):
  38.     _VALID_URL = r'https?://(?:(?:www|beta|m)\.)?mixcloud\.com/([^/]+)/(?!stream|uploads|favorites|listens|playlists)([^/]+)'
  39.     IE_NAME = 'mixcloud'
  41.     _TESTS = [{
  42.         'url': 'http://www.mixcloud.com/dholbach/cryptkeeper/',
  43.         'info_dict': {
  44.             'id': 'dholbach_cryptkeeper',
  45.             'ext': 'm4a',
  46.             'title': 'Cryptkeeper',
  47.             'description': 'After quite a long silence from myself, finally another Drum\'n\'Bass mix with my favourite current dance floor bangers.',
  48.             'uploader': 'dholbach',  # was: 'Daniel Holbach',
  49.             'uploader_id': 'dholbach',
  50.             'thumbnail': r're:https?://.*\.jpg',
  51.             'view_count': int,
  52.             'timestamp': 1321359578,
  53.             'upload_date': '20111115',
  54.         },
  55.     }, {
  56.         'url': 'http://www.mixcloud.com/gillespeterson/caribou-7-inch-vinyl-mix-chat/',
  57.         'info_dict': {
  58.             'id': 'gillespeterson_caribou-7-inch-vinyl-mix-chat',
  59.             'ext': 'mp3',
  60.             'title': 'Caribou 7 inch Vinyl Mix & Chat',
  61.             'description': r're:Last week Dan Snaith aka Caribou swung by the Brownswood.{136}',
  62.             'uploader': 'Gilles Peterson Worldwide',
  63.             'uploader_id': 'gillespeterson',
  64.             'thumbnail': 're:https?://.*',
  65.             'view_count': int,
  66.             'timestamp': 1422987057,
  67.             'upload_date': '20150203',
  68.         },
  69.         'params': {
  70.             'skip_download': '404 not found',
  71.         },
  72.     }, {
  73.         'url': 'https://www.mixcloud.com/gillespeterson/carnival-m%C3%BAsica-popular-brasileira-mix/',
  74.         'info_dict': {
  75.             'id': 'gillespeterson_carnival-música-popular-brasileira-mix',
  76.             'ext': 'm4a',
  77.             'title': 'Carnival Música Popular Brasileira Mix',
  78.             'description': r're:Gilles was recently in Brazil to play at Boiler Room.{208}',
  79.             'timestamp': 1454347174,
  80.             'upload_date': '20160201',
  81.             'uploader': 'Gilles Peterson Worldwide',
  82.             'uploader_id': 'gillespeterson',
  83.             'thumbnail': 're:https?://.*',
  84.             'view_count': int,
  85.         },
  86.     }, {
  87.         'url': 'https://beta.mixcloud.com/RedLightRadio/nosedrip-15-red-light-radio-01-18-2016/',
  88.         'only_matching': True,
  89.     }]
  90.     _DECRYPTION_KEY = 'IFYOUWANTTHEARTISTSTOGETPAIDDONOTDOWNLOADFROMMIXCLOUD'
  92.     @staticmethod
  93.     def _decrypt_xor_cipher(key, ciphertext):
  94.         """Encrypt/Decrypt XOR cipher. Both ways are possible because it's XOR."""
  95.         return ''.join([
  96.             compat_chr(compat_ord(ch) ^ compat_ord(k))
  97.             for ch, k in zip(ciphertext, itertools.cycle(key))])
  99.     def _real_extract(self, url):
  100.         username, slug = self._match_valid_url(url).groups()
  101.         username, slug = compat_urllib_parse_unquote(username), compat_urllib_parse_unquote(slug)
  102.         track_id = '%s_%s' % (username, slug)
  104.         cloudcast = self._call_api('cloudcast', '''audioLength
  105.     comments(first: 100) {
  106.       edges {
  107.         node {
  108.           comment
  109.           created
  110.           user {
  111.             displayName
  112.             username
  113.           }
  114.         }
  115.       }
  116.       totalCount
  117.     }
  118.     description
  119.     favorites {
  120.       totalCount
  121.     }
  122.     featuringArtistList
  123.     isExclusive
  124.     name
  125.     owner {
  126.       displayName
  127.       url
  128.       username
  129.     }
  130.     picture(width: 1024, height: 1024) {
  131.         url
  132.     }
  133.     plays
  134.     publishDate
  135.     reposts {
  136.       totalCount
  137.     }
  138.     streamInfo {
  139.       dashUrl
  140.       hlsUrl
  141.       url
  142.     }
  143.     tags {
  144.       tag {
  145.         name
  146.       }
  147.     }''', track_id, username, slug)
  149.         title = cloudcast['name']
  151.         stream_info = cloudcast['streamInfo']
  152.         formats = []
  154.         for url_key in ('url', 'hlsUrl', 'dashUrl'):
  155.             format_url = stream_info.get(url_key)
  156.             if not format_url:
  157.                 continue
  158.             decrypted = self._decrypt_xor_cipher(
  159.                 self._DECRYPTION_KEY, compat_b64decode(format_url))
  160.             if url_key == 'hlsUrl':
  161.                 formats.extend(self._extract_m3u8_formats(
  162.                     decrypted, track_id, 'mp4', entry_protocol='m3u8_native',
  163.                     m3u8_id='hls', fatal=False))
  164.             elif url_key == 'dashUrl':
  165.                 formats.extend(self._extract_mpd_formats(
  166.                     decrypted, track_id, mpd_id='dash', fatal=False))
  167.             else:
  168.                 formats.append({
  169.                     'format_id': 'http',
  170.                     'url': decrypted,
  171.                     'downloader_options': {
  172.                         # Mixcloud starts throttling at >~5M
  173.                         'http_chunk_size': 5242880,
  174.                     },
  175.                 })
  177.         if not formats and cloudcast.get('isExclusive'):
  178.             self.raise_login_required()
  180.         self._sort_formats(formats)
  182.         comments = []
  183.         for edge in (try_get(cloudcast, lambda x: x['comments']['edges']) or []):
  184.             node = edge.get('node') or {}
  185.             text = strip_or_none(node.get('comment'))
  186.             if not text:
  187.                 continue
  188.             user = node.get('user') or {}
  189.             comments.append({
  190.                 'author': user.get('displayName'),
  191.                 'author_id': user.get('username'),
  192.                 'text': text,
  193.                 'timestamp': parse_iso8601(node.get('created')),
  194.             })
  196.         tags = []
  197.         for t in cloudcast.get('tags'):
  198.             tag = try_get(t, lambda x: x['tag']['name'], compat_str)
  199.             if not tag:
  200.                 tags.append(tag)
  202.         get_count = lambda x: int_or_none(try_get(cloudcast, lambda y: y[x]['totalCount']))
  204.         owner = cloudcast.get('owner') or {}
  206.         return {
  207.             'id': track_id,
  208.             'title': title,
  209.             'formats': formats,
  210.             'description': cloudcast.get('description'),
  211.             'thumbnail': try_get(cloudcast, lambda x: x['picture']['url'], compat_str),
  212.             'uploader': owner.get('displayName'),
  213.             'timestamp': parse_iso8601(cloudcast.get('publishDate')),
  214.             'uploader_id': owner.get('username'),
  215.             'uploader_url': owner.get('url'),
  216.             'duration': int_or_none(cloudcast.get('audioLength')),
  217.             'view_count': int_or_none(cloudcast.get('plays')),
  218.             'like_count': get_count('favorites'),
  219.             'repost_count': get_count('reposts'),
  220.             'comment_count': get_count('comments'),
  221.             'comments': comments,
  222.             'tags': tags,
  223.             'artist': ', '.join(cloudcast.get('featuringArtistList') or []) or None,
  224.         }
  227. class MixcloudPlaylistBaseIE(MixcloudBaseIE):
  228.     def _get_cloudcast(self, node):
  229.         return node
  231.     def _get_playlist_title(self, title, slug):
  232.         return title
  234.     def _real_extract(self, url):
  235.         username, slug = re.match(self._VALID_URL, url).groups()
  236.         username = compat_urllib_parse_unquote(username)
  237.         if not slug:
  238.             slug = 'uploads'
  239.         else:
  240.             slug = compat_urllib_parse_unquote(slug)
  241.         playlist_id = '%s_%s' % (username, slug)
  243.         is_playlist_type = self._ROOT_TYPE == 'playlist'
  244.         playlist_type = 'items' if is_playlist_type else slug
  245.         list_filter = ''
  247.         has_next_page = True
  248.         entries = []
  249.         while has_next_page:
  250.             playlist = self._call_api(
  251.                 self._ROOT_TYPE, '''%s
  252.     %s
  253.     %s(first: 100%s) {
  254.       edges {
  255.         node {
  256.           %s
  257.         }
  258.       }
  259.       pageInfo {
  260.         endCursor
  261.         hasNextPage
  262.       }
  263.     }''' % (self._TITLE_KEY, self._DESCRIPTION_KEY, playlist_type, list_filter, self._NODE_TEMPLATE),
  264.                 playlist_id, username, slug if is_playlist_type else None)
  266.             items = playlist.get(playlist_type) or {}
  267.             for edge in items.get('edges', []):
  268.                 cloudcast = self._get_cloudcast(edge.get('node') or {})
  269.                 cloudcast_url = cloudcast.get('url')
  270.                 if not cloudcast_url:
  271.                     continue
  272.                 slug = try_get(cloudcast, lambda x: x['slug'], compat_str)
  273.                 owner_username = try_get(cloudcast, lambda x: x['owner']['username'], compat_str)
  274.                 video_id = '%s_%s' % (owner_username, slug) if slug and owner_username else None
  275.                 entries.append(self.url_result(
  276.                     cloudcast_url, MixcloudIE.ie_key(), video_id))
  278.             page_info = items['pageInfo']
  279.             has_next_page = page_info['hasNextPage']
  280.             list_filter = ', after: "%s"' % page_info['endCursor']
  282.         return self.playlist_result(
  283.             entries, playlist_id,
  284.             self._get_playlist_title(playlist[self._TITLE_KEY], slug),
  285.             playlist.get(self._DESCRIPTION_KEY))
  288. class MixcloudUserIE(MixcloudPlaylistBaseIE):
  289.     _VALID_URL = r'https?://(?:www\.)?mixcloud\.com/(?P<id>[^/]+)/(?P<type>uploads|favorites|listens|stream)?/?$'
  290.     IE_NAME = 'mixcloud:user'
  292.     _TESTS = [{
  293.         'url': 'http://www.mixcloud.com/dholbach/',
  294.         'info_dict': {
  295.             'id': 'dholbach_uploads',
  296.             'title': 'Daniel Holbach (uploads)',
  297.             'description': 'md5:b60d776f0bab534c5dabe0a34e47a789',
  298.         },
  299.         'playlist_mincount': 36,
  300.     }, {
  301.         'url': 'http://www.mixcloud.com/dholbach/uploads/',
  302.         'info_dict': {
  303.             'id': 'dholbach_uploads',
  304.             'title': 'Daniel Holbach (uploads)',
  305.             'description': 'md5:b60d776f0bab534c5dabe0a34e47a789',
  306.         },
  307.         'playlist_mincount': 36,
  308.     }, {
  309.         'url': 'http://www.mixcloud.com/dholbach/favorites/',
  310.         'info_dict': {
  311.             'id': 'dholbach_favorites',
  312.             'title': 'Daniel Holbach (favorites)',
  313.             'description': 'md5:b60d776f0bab534c5dabe0a34e47a789',
  314.         },
  315.         # 'params': {
  316.         #     'playlist_items': '1-100',
  317.         # },
  318.         'playlist_mincount': 396,
  319.     }, {
  320.         'url': 'http://www.mixcloud.com/dholbach/listens/',
  321.         'info_dict': {
  322.             'id': 'dholbach_listens',
  323.             'title': 'Daniel Holbach (listens)',
  324.             'description': 'md5:b60d776f0bab534c5dabe0a34e47a789',
  325.         },
  326.         # 'params': {
  327.         #     'playlist_items': '1-100',
  328.         # },
  329.         'playlist_mincount': 1623,
  330.         'skip': 'Large list',
  331.     }, {
  332.         'url': 'https://www.mixcloud.com/FirstEar/stream/',
  333.         'info_dict': {
  334.             'id': 'FirstEar_stream',
  335.             'title': 'First Ear (stream)',
  336.             'description': 'Curators of good music\r\n\r\nfirstearmusic.com',
  337.         },
  338.         'playlist_mincount': 271,
  339.     }]
  341.     _TITLE_KEY = 'displayName'
  342.     _DESCRIPTION_KEY = 'biog'
  343.     _ROOT_TYPE = 'user'
  344.     _NODE_TEMPLATE = '''slug
  345.           url
  346.           owner { username }'''
  348.     def _get_playlist_title(self, title, slug):
  349.         return '%s (%s)' % (title, slug)
  352. class MixcloudPlaylistIE(MixcloudPlaylistBaseIE):
  353.     _VALID_URL = r'https?://(?:www\.)?mixcloud\.com/(?P<user>[^/]+)/playlists/(?P<playlist>[^/]+)/?$'
  354.     IE_NAME = 'mixcloud:playlist'
  356.     _TESTS = [{
  357.         'url': 'https://www.mixcloud.com/maxvibes/playlists/jazzcat-on-ness-radio/',
  358.         'info_dict': {
  359.             'id': 'maxvibes_jazzcat-on-ness-radio',
  360.             'title': 'Ness Radio sessions',
  361.         },
  362.         'playlist_mincount': 59,
  363.     }]
  364.     _TITLE_KEY = 'name'
  365.     _DESCRIPTION_KEY = 'description'
  366.     _ROOT_TYPE = 'playlist'
  367.     _NODE_TEMPLATE = '''cloudcast {
  368.             slug
  369.             url
  370.             owner { username }
  371.           }'''
  373.     def _get_cloudcast(self, node):
  374.         return node.get('cloudcast') or {}