logo

youtube-dl

[mirror] Download/Watch videos from video hostersgit clone https://hacktivis.me/git/mirror/youtube-dl.git

mixcloud.py (11661B)

    

  1. from __future__ import unicode_literals
  3. import itertools
  4. import re
  6. from .common import InfoExtractor
  7. from ..compat import (
  8.     compat_b64decode,
  9.     compat_chr,
  10.     compat_ord,
  11.     compat_str,
  12.     compat_urllib_parse_unquote,
  13.     compat_zip
  14. )
  15. from ..utils import (
  16.     int_or_none,
  17.     parse_iso8601,
  18.     strip_or_none,
  19.     try_get,
  20. )
  23. class MixcloudBaseIE(InfoExtractor):
  24.     def _call_api(self, object_type, object_fields, display_id, username, slug=None):
  25.         lookup_key = object_type + 'Lookup'
  26.         return self._download_json(
  27.             'https://www.mixcloud.com/graphql', display_id, query={
  28.                 'query': '''{
  29.   %s(lookup: {username: "%s"%s}) {
  30.     %s
  31.   }
  32. }''' % (lookup_key, username, ', slug: "%s"' % slug if slug else '', object_fields)
  33.             })['data'][lookup_key]
  36. class MixcloudIE(MixcloudBaseIE):
  37.     _VALID_URL = r'https?://(?:(?:www|beta|m)\.)?mixcloud\.com/([^/]+)/(?!stream|uploads|favorites|listens|playlists)([^/]+)'
  38.     IE_NAME = 'mixcloud'
  40.     _TESTS = [{
  41.         'url': 'http://www.mixcloud.com/dholbach/cryptkeeper/',
  42.         'info_dict': {
  43.             'id': 'dholbach_cryptkeeper',
  44.             'ext': 'm4a',
  45.             'title': 'Cryptkeeper',
  46.             'description': 'After quite a long silence from myself, finally another Drum\'n\'Bass mix with my favourite current dance floor bangers.',
  47.             'uploader': 'Daniel Holbach',
  48.             'uploader_id': 'dholbach',
  49.             'thumbnail': r're:https?://.*\.jpg',
  50.             'view_count': int,
  51.             'timestamp': 1321359578,
  52.             'upload_date': '20111115',
  53.         },
  54.     }, {
  55.         'url': 'http://www.mixcloud.com/gillespeterson/caribou-7-inch-vinyl-mix-chat/',
  56.         'info_dict': {
  57.             'id': 'gillespeterson_caribou-7-inch-vinyl-mix-chat',
  58.             'ext': 'mp3',
  59.             'title': 'Caribou 7 inch Vinyl Mix & Chat',
  60.             'description': 'md5:2b8aec6adce69f9d41724647c65875e8',
  61.             'uploader': 'Gilles Peterson Worldwide',
  62.             'uploader_id': 'gillespeterson',
  63.             'thumbnail': 're:https?://.*',
  64.             'view_count': int,
  65.             'timestamp': 1422987057,
  66.             'upload_date': '20150203',
  67.         },
  68.     }, {
  69.         'url': 'https://beta.mixcloud.com/RedLightRadio/nosedrip-15-red-light-radio-01-18-2016/',
  70.         'only_matching': True,
  71.     }]
  72.     _DECRYPTION_KEY = 'IFYOUWANTTHEARTISTSTOGETPAIDDONOTDOWNLOADFROMMIXCLOUD'
  74.     @staticmethod
  75.     def _decrypt_xor_cipher(key, ciphertext):
  76.         """Encrypt/Decrypt XOR cipher. Both ways are possible because it's XOR."""
  77.         return ''.join([
  78.             compat_chr(compat_ord(ch) ^ compat_ord(k))
  79.             for ch, k in compat_zip(ciphertext, itertools.cycle(key))])
  81.     def _real_extract(self, url):
  82.         username, slug = re.match(self._VALID_URL, url).groups()
  83.         username, slug = compat_urllib_parse_unquote(username), compat_urllib_parse_unquote(slug)
  84.         track_id = '%s_%s' % (username, slug)
  86.         cloudcast = self._call_api('cloudcast', '''audioLength
  87.     comments(first: 100) {
  88.       edges {
  89.         node {
  90.           comment
  91.           created
  92.           user {
  93.             displayName
  94.             username
  95.           }
  96.         }
  97.       }
  98.       totalCount
  99.     }
  100.     description
  101.     favorites {
  102.       totalCount
  103.     }
  104.     featuringArtistList
  105.     isExclusive
  106.     name
  107.     owner {
  108.       displayName
  109.       url
  110.       username
  111.     }
  112.     picture(width: 1024, height: 1024) {
  113.         url
  114.     }
  115.     plays
  116.     publishDate
  117.     reposts {
  118.       totalCount
  119.     }
  120.     streamInfo {
  121.       dashUrl
  122.       hlsUrl
  123.       url
  124.     }
  125.     tags {
  126.       tag {
  127.         name
  128.       }
  129.     }''', track_id, username, slug)
  131.         title = cloudcast['name']
  133.         stream_info = cloudcast['streamInfo']
  134.         formats = []
  136.         for url_key in ('url', 'hlsUrl', 'dashUrl'):
  137.             format_url = stream_info.get(url_key)
  138.             if not format_url:
  139.                 continue
  140.             decrypted = self._decrypt_xor_cipher(
  141.                 self._DECRYPTION_KEY, compat_b64decode(format_url))
  142.             if url_key == 'hlsUrl':
  143.                 formats.extend(self._extract_m3u8_formats(
  144.                     decrypted, track_id, 'mp4', entry_protocol='m3u8_native',
  145.                     m3u8_id='hls', fatal=False))
  146.             elif url_key == 'dashUrl':
  147.                 formats.extend(self._extract_mpd_formats(
  148.                     decrypted, track_id, mpd_id='dash', fatal=False))
  149.             else:
  150.                 formats.append({
  151.                     'format_id': 'http',
  152.                     'url': decrypted,
  153.                     'downloader_options': {
  154.                         # Mixcloud starts throttling at >~5M
  155.                         'http_chunk_size': 5242880,
  156.                     },
  157.                 })
  159.         if not formats and cloudcast.get('isExclusive'):
  160.             self.raise_login_required()
  162.         self._sort_formats(formats)
  164.         comments = []
  165.         for edge in (try_get(cloudcast, lambda x: x['comments']['edges']) or []):
  166.             node = edge.get('node') or {}
  167.             text = strip_or_none(node.get('comment'))
  168.             if not text:
  169.                 continue
  170.             user = node.get('user') or {}
  171.             comments.append({
  172.                 'author': user.get('displayName'),
  173.                 'author_id': user.get('username'),
  174.                 'text': text,
  175.                 'timestamp': parse_iso8601(node.get('created')),
  176.             })
  178.         tags = []
  179.         for t in cloudcast.get('tags'):
  180.             tag = try_get(t, lambda x: x['tag']['name'], compat_str)
  181.             if not tag:
  182.                 tags.append(tag)
  184.         get_count = lambda x: int_or_none(try_get(cloudcast, lambda y: y[x]['totalCount']))
  186.         owner = cloudcast.get('owner') or {}
  188.         return {
  189.             'id': track_id,
  190.             'title': title,
  191.             'formats': formats,
  192.             'description': cloudcast.get('description'),
  193.             'thumbnail': try_get(cloudcast, lambda x: x['picture']['url'], compat_str),
  194.             'uploader': owner.get('displayName'),
  195.             'timestamp': parse_iso8601(cloudcast.get('publishDate')),
  196.             'uploader_id': owner.get('username'),
  197.             'uploader_url': owner.get('url'),
  198.             'duration': int_or_none(cloudcast.get('audioLength')),
  199.             'view_count': int_or_none(cloudcast.get('plays')),
  200.             'like_count': get_count('favorites'),
  201.             'repost_count': get_count('reposts'),
  202.             'comment_count': get_count('comments'),
  203.             'comments': comments,
  204.             'tags': tags,
  205.             'artist': ', '.join(cloudcast.get('featuringArtistList') or []) or None,
  206.         }
  209. class MixcloudPlaylistBaseIE(MixcloudBaseIE):
  210.     def _get_cloudcast(self, node):
  211.         return node
  213.     def _get_playlist_title(self, title, slug):
  214.         return title
  216.     def _real_extract(self, url):
  217.         username, slug = re.match(self._VALID_URL, url).groups()
  218.         username = compat_urllib_parse_unquote(username)
  219.         if not slug:
  220.             slug = 'uploads'
  221.         else:
  222.             slug = compat_urllib_parse_unquote(slug)
  223.         playlist_id = '%s_%s' % (username, slug)
  225.         is_playlist_type = self._ROOT_TYPE == 'playlist'
  226.         playlist_type = 'items' if is_playlist_type else slug
  227.         list_filter = ''
  229.         has_next_page = True
  230.         entries = []
  231.         while has_next_page:
  232.             playlist = self._call_api(
  233.                 self._ROOT_TYPE, '''%s
  234.     %s
  235.     %s(first: 100%s) {
  236.       edges {
  237.         node {
  238.           %s
  239.         }
  240.       }
  241.       pageInfo {
  242.         endCursor
  243.         hasNextPage
  244.       }
  245.     }''' % (self._TITLE_KEY, self._DESCRIPTION_KEY, playlist_type, list_filter, self._NODE_TEMPLATE),
  246.                 playlist_id, username, slug if is_playlist_type else None)
  248.             items = playlist.get(playlist_type) or {}
  249.             for edge in items.get('edges', []):
  250.                 cloudcast = self._get_cloudcast(edge.get('node') or {})
  251.                 cloudcast_url = cloudcast.get('url')
  252.                 if not cloudcast_url:
  253.                     continue
  254.                 slug = try_get(cloudcast, lambda x: x['slug'], compat_str)
  255.                 owner_username = try_get(cloudcast, lambda x: x['owner']['username'], compat_str)
  256.                 video_id = '%s_%s' % (owner_username, slug) if slug and owner_username else None
  257.                 entries.append(self.url_result(
  258.                     cloudcast_url, MixcloudIE.ie_key(), video_id))
  260.             page_info = items['pageInfo']
  261.             has_next_page = page_info['hasNextPage']
  262.             list_filter = ', after: "%s"' % page_info['endCursor']
  264.         return self.playlist_result(
  265.             entries, playlist_id,
  266.             self._get_playlist_title(playlist[self._TITLE_KEY], slug),
  267.             playlist.get(self._DESCRIPTION_KEY))
  270. class MixcloudUserIE(MixcloudPlaylistBaseIE):
  271.     _VALID_URL = r'https?://(?:www\.)?mixcloud\.com/(?P<id>[^/]+)/(?P<type>uploads|favorites|listens|stream)?/?$'
  272.     IE_NAME = 'mixcloud:user'
  274.     _TESTS = [{
  275.         'url': 'http://www.mixcloud.com/dholbach/',
  276.         'info_dict': {
  277.             'id': 'dholbach_uploads',
  278.             'title': 'Daniel Holbach (uploads)',
  279.             'description': 'md5:b60d776f0bab534c5dabe0a34e47a789',
  280.         },
  281.         'playlist_mincount': 36,
  282.     }, {
  283.         'url': 'http://www.mixcloud.com/dholbach/uploads/',
  284.         'info_dict': {
  285.             'id': 'dholbach_uploads',
  286.             'title': 'Daniel Holbach (uploads)',
  287.             'description': 'md5:b60d776f0bab534c5dabe0a34e47a789',
  288.         },
  289.         'playlist_mincount': 36,
  290.     }, {
  291.         'url': 'http://www.mixcloud.com/dholbach/favorites/',
  292.         'info_dict': {
  293.             'id': 'dholbach_favorites',
  294.             'title': 'Daniel Holbach (favorites)',
  295.             'description': 'md5:b60d776f0bab534c5dabe0a34e47a789',
  296.         },
  297.         # 'params': {
  298.         #     'playlist_items': '1-100',
  299.         # },
  300.         'playlist_mincount': 396,
  301.     }, {
  302.         'url': 'http://www.mixcloud.com/dholbach/listens/',
  303.         'info_dict': {
  304.             'id': 'dholbach_listens',
  305.             'title': 'Daniel Holbach (listens)',
  306.             'description': 'md5:b60d776f0bab534c5dabe0a34e47a789',
  307.         },
  308.         # 'params': {
  309.         #     'playlist_items': '1-100',
  310.         # },
  311.         'playlist_mincount': 1623,
  312.         'skip': 'Large list',
  313.     }, {
  314.         'url': 'https://www.mixcloud.com/FirstEar/stream/',
  315.         'info_dict': {
  316.             'id': 'FirstEar_stream',
  317.             'title': 'First Ear (stream)',
  318.             'description': 'Curators of good music\r\n\r\nfirstearmusic.com',
  319.         },
  320.         'playlist_mincount': 271,
  321.     }]
  323.     _TITLE_KEY = 'displayName'
  324.     _DESCRIPTION_KEY = 'biog'
  325.     _ROOT_TYPE = 'user'
  326.     _NODE_TEMPLATE = '''slug
  327.           url
  328.           owner { username }'''
  330.     def _get_playlist_title(self, title, slug):
  331.         return '%s (%s)' % (title, slug)
  334. class MixcloudPlaylistIE(MixcloudPlaylistBaseIE):
  335.     _VALID_URL = r'https?://(?:www\.)?mixcloud\.com/(?P<user>[^/]+)/playlists/(?P<playlist>[^/]+)/?$'
  336.     IE_NAME = 'mixcloud:playlist'
  338.     _TESTS = [{
  339.         'url': 'https://www.mixcloud.com/maxvibes/playlists/jazzcat-on-ness-radio/',
  340.         'info_dict': {
  341.             'id': 'maxvibes_jazzcat-on-ness-radio',
  342.             'title': 'Ness Radio sessions',
  343.         },
  344.         'playlist_mincount': 59,
  345.     }]
  346.     _TITLE_KEY = 'name'
  347.     _DESCRIPTION_KEY = 'description'
  348.     _ROOT_TYPE = 'playlist'
  349.     _NODE_TEMPLATE = '''cloudcast {
  350.             slug
  351.             url
  352.             owner { username }
  353.           }'''
  355.     def _get_cloudcast(self, node):
  356.         return node.get('cloudcast') or {}