logo

oasis-root

Compiled tree of Oasis Linux based on own branch at <https://hacktivis.me/git/oasis/> git clone https://anongit.hacktivis.me/git/oasis-root.git

soundcloud.py (40811B)

    

  1. import functools
  2. import itertools
  3. import json
  4. import re
  6. from .common import InfoExtractor, SearchInfoExtractor
  7. from ..networking import HEADRequest
  8. from ..networking.exceptions import HTTPError
  9. from ..utils import (
  10.     ExtractorError,
  11.     float_or_none,
  12.     int_or_none,
  13.     join_nonempty,
  14.     mimetype2ext,
  15.     parse_qs,
  16.     str_or_none,
  17.     try_call,
  18.     unified_timestamp,
  19.     update_url_query,
  20.     url_or_none,
  21.     urlhandle_detect_ext,
  22. )
  23. from ..utils.traversal import traverse_obj
  26. class SoundcloudEmbedIE(InfoExtractor):
  27.     _VALID_URL = r'https?://(?:w|player|p)\.soundcloud\.com/player/?.*?\burl=(?P<id>.+)'
  28.     _EMBED_REGEX = [r'<iframe[^>]+src=(["\'])(?P<url>(?:https?://)?(?:w\.)?soundcloud\.com/player.+?)\1']
  29.     _TEST = {
  30.         # from https://www.soundi.fi/uutiset/ennakkokuuntelussa-timo-kaukolammen-station-to-station-to-station-julkaisua-juhlitaan-tanaan-g-livelabissa/
  31.         'url': 'https://w.soundcloud.com/player/?visual=true&url=https%3A%2F%2Fapi.soundcloud.com%2Fplaylists%2F922213810&show_artwork=true&maxwidth=640&maxheight=960&dnt=1&secret_token=s-ziYey',
  32.         'only_matching': True,
  33.     }
  35.     def _real_extract(self, url):
  36.         query = parse_qs(url)
  37.         api_url = query['url'][0]
  38.         secret_token = query.get('secret_token')
  39.         if secret_token:
  40.             api_url = update_url_query(api_url, {'secret_token': secret_token[0]})
  41.         return self.url_result(api_url)
  44. class SoundcloudBaseIE(InfoExtractor):
  45.     _NETRC_MACHINE = 'soundcloud'
  47.     _API_V2_BASE = 'https://api-v2.soundcloud.com/'
  48.     _BASE_URL = 'https://soundcloud.com/'
  49.     _USER_AGENT = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.105 Safari/537.36'
  50.     _API_AUTH_QUERY_TEMPLATE = '?client_id=%s'
  51.     _API_AUTH_URL_PW = 'https://api-auth.soundcloud.com/web-auth/sign-in/password%s'
  52.     _API_VERIFY_AUTH_TOKEN = 'https://api-auth.soundcloud.com/connect/session%s'
  53.     _HEADERS = {}
  55.     _IMAGE_REPL_RE = r'-([0-9a-z]+)\.jpg'
  57.     _ARTWORK_MAP = {
  58.         'mini': 16,
  59.         'tiny': 20,
  60.         'small': 32,
  61.         'badge': 47,
  62.         't67x67': 67,
  63.         'large': 100,
  64.         't300x300': 300,
  65.         'crop': 400,
  66.         't500x500': 500,
  67.         'original': 0,
  68.     }
  70.     _DEFAULT_FORMATS = ['http_aac', 'hls_aac', 'http_opus', 'hls_opus', 'http_mp3', 'hls_mp3']
  72.     @functools.cached_property
  73.     def _is_requested(self):
  74.         return re.compile(r'|'.join(set(
  75.             re.escape(pattern).replace(r'\*', r'.*') if pattern != 'default'
  76.             else '|'.join(map(re.escape, self._DEFAULT_FORMATS))
  77.             for pattern in self._configuration_arg('formats', ['default'], ie_key=SoundcloudIE)
  78.         ))).fullmatch
  80.     def _store_client_id(self, client_id):
  81.         self.cache.store('soundcloud', 'client_id', client_id)
  83.     def _update_client_id(self):
  84.         webpage = self._download_webpage('https://soundcloud.com/', None)
  85.         for src in reversed(re.findall(r'<script[^>]+src="([^"]+)"', webpage)):
  86.             script = self._download_webpage(src, None, fatal=False)
  87.             if script:
  88.                 client_id = self._search_regex(
  89.                     r'client_id\s*:\s*"([0-9a-zA-Z]{32})"',
  90.                     script, 'client id', default=None)
  91.                 if client_id:
  92.                     self._CLIENT_ID = client_id
  93.                     self._store_client_id(client_id)
  94.                     return
  95.         raise ExtractorError('Unable to extract client id')
  97.     def _call_api(self, *args, **kwargs):
  98.         non_fatal = kwargs.get('fatal') is False
  99.         if non_fatal:
  100.             del kwargs['fatal']
  101.         query = kwargs.get('query', {}).copy()
  102.         for _ in range(2):
  103.             query['client_id'] = self._CLIENT_ID
  104.             kwargs['query'] = query
  105.             try:
  106.                 return self._download_json(*args, **kwargs)
  107.             except ExtractorError as e:
  108.                 if isinstance(e.cause, HTTPError) and e.cause.status in (401, 403):
  109.                     self._store_client_id(None)
  110.                     self._update_client_id()
  111.                     continue
  112.                 elif non_fatal:
  113.                     self.report_warning(str(e))
  114.                     return False
  115.                 raise
  117.     def _initialize_pre_login(self):
  118.         self._CLIENT_ID = self.cache.load('soundcloud', 'client_id') or 'a3e059563d7fd3372b49b37f00a00bcf'
  120.     def _verify_oauth_token(self, token):
  121.         if self._request_webpage(
  122.                 self._API_VERIFY_AUTH_TOKEN % (self._API_AUTH_QUERY_TEMPLATE % self._CLIENT_ID),
  123.                 None, note='Verifying login token...', fatal=False,
  124.                 data=json.dumps({'session': {'access_token': token}}).encode()):
  125.             self._HEADERS['Authorization'] = f'OAuth {token}'
  126.             self.report_login()
  127.         else:
  128.             self.report_warning('Provided authorization token is invalid. Continuing as guest')
  130.     def _real_initialize(self):
  131.         if self._HEADERS:
  132.             return
  133.         if token := try_call(lambda: self._get_cookies(self._BASE_URL)['oauth_token'].value):
  134.             self._verify_oauth_token(token)
  136.     def _perform_login(self, username, password):
  137.         if username != 'oauth':
  138.             raise ExtractorError(
  139.                 'Login using username and password is not currently supported. '
  140.                 'Use "--username oauth --password <oauth_token>" to login using an oauth token, '
  141.                 f'or else {self._login_hint(method="cookies")}', expected=True)
  142.         if self._HEADERS:
  143.             return
  144.         self._verify_oauth_token(password)
  146.         r'''
  147.         def genDevId():
  148.             def genNumBlock():
  149.                 return ''.join([str(random.randrange(10)) for i in range(6)])
  150.             return '-'.join([genNumBlock() for i in range(4)])
  152.         payload = {
  153.             'client_id': self._CLIENT_ID,
  154.             'recaptcha_pubkey': 'null',
  155.             'recaptcha_response': 'null',
  156.             'credentials': {
  157.                 'identifier': username,
  158.                 'password': password
  159.             },
  160.             'signature': self.sign(username, password, self._CLIENT_ID),
  161.             'device_id': genDevId(),
  162.             'user_agent': self._USER_AGENT
  163.         }
  165.         response = self._call_api(
  166.             self._API_AUTH_URL_PW % (self._API_AUTH_QUERY_TEMPLATE % self._CLIENT_ID),
  167.             None, note='Verifying login token...', fatal=False,
  168.             data=json.dumps(payload).encode())
  170.         if token := traverse_obj(response, ('session', 'access_token', {str})):
  171.             self._HEADERS['Authorization'] = f'OAuth {token}'
  172.             self.report_login()
  173.             return
  175.         raise ExtractorError('Unable to get access token, login may have failed', expected=True)
  176.         '''
  178.     # signature generation
  179.     def sign(self, user, pw, clid):
  180.         a = 33
  181.         i = 1
  182.         s = 440123
  183.         w = 117
  184.         u = 1800000
  185.         l = 1042
  186.         b = 37
  187.         k = 37
  188.         c = 5
  189.         n = '0763ed7314c69015fd4a0dc16bbf4b90'  # _KEY
  190.         y = '8'  # _REV
  191.         r = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.105 Safari/537.36'  # _USER_AGENT
  192.         e = user  # _USERNAME
  193.         t = clid  # _CLIENT_ID
  195.         d = '-'.join([str(mInt) for mInt in [a, i, s, w, u, l, b, k]])
  196.         h = n + y + d + r + e + t + d + n
  198.         m = 8011470
  200.         for f in range(len(h)):
  201.             m = (m >> 1) + ((1 & m) << 23)
  202.             m += ord(h[f])
  203.             m &= 16777215
  205.         # c is not even needed
  206.         return f'{y}:{d}:{m:x}:{c}'
  208.     def _extract_info_dict(self, info, full_title=None, secret_token=None, extract_flat=False):
  209.         track_id = str(info['id'])
  211.         format_urls = set()
  212.         formats = []
  213.         has_drm = False
  214.         query = {'client_id': self._CLIENT_ID}
  215.         if secret_token:
  216.             query['secret_token'] = secret_token
  218.         if not extract_flat and info.get('downloadable') and info.get('has_downloads_left'):
  219.             try:
  220.                 # Do not use _call_api(); HTTP Error codes have different meanings for this request
  221.                 download_data = self._download_json(
  222.                     f'{self._API_V2_BASE}tracks/{track_id}/download', track_id,
  223.                     'Downloading original download format info JSON', query=query, headers=self._HEADERS)
  224.             except ExtractorError as e:
  225.                 if isinstance(e.cause, HTTPError) and e.cause.status == 401:
  226.                     self.report_warning(
  227.                         'Original download format is only available '
  228.                         f'for registered users. {self._login_hint()}')
  229.                 elif isinstance(e.cause, HTTPError) and e.cause.status == 403:
  230.                     self.write_debug('Original download format is not available for this client')
  231.                 else:
  232.                     self.report_warning(e.msg)
  233.                 download_data = None
  235.             if redirect_url := traverse_obj(download_data, ('redirectUri', {url_or_none})):
  236.                 urlh = self._request_webpage(
  237.                     HEADRequest(redirect_url), track_id, 'Checking original download format availability',
  238.                     'Original download format is not available', fatal=False)
  239.                 if urlh:
  240.                     format_url = urlh.url
  241.                     format_urls.add(format_url)
  242.                     formats.append({
  243.                         'format_id': 'download',
  244.                         'ext': urlhandle_detect_ext(urlh, default='mp3'),
  245.                         'filesize': int_or_none(urlh.headers.get('Content-Length')),
  246.                         'url': format_url,
  247.                         'quality': 10,
  248.                         'format_note': 'Original',
  249.                         'vcodec': 'none',
  250.                     })
  252.         def invalid_url(url):
  253.             return not url or url in format_urls
  255.         # New API
  256.         for t in traverse_obj(info, ('media', 'transcodings', lambda _, v: url_or_none(v['url']) and v['preset'])):
  257.             if extract_flat:
  258.                 break
  259.             format_url = t['url']
  260.             preset = t['preset']
  261.             preset_base = preset.partition('_')[0]
  263.             protocol = traverse_obj(t, ('format', 'protocol', {str})) or 'http'
  264.             if protocol.startswith(('ctr-', 'cbc-')):
  265.                 has_drm = True
  266.                 continue
  267.             if protocol == 'progressive':
  268.                 protocol = 'http'
  269.             if protocol != 'hls' and '/hls' in format_url:
  270.                 protocol = 'hls'
  271.             if protocol == 'encrypted-hls' or '/encrypted-hls' in format_url:
  272.                 protocol = 'hls-aes'
  274.             short_identifier = f'{protocol}_{preset_base}'
  275.             if preset_base == 'abr':
  276.                 self.write_debug(f'Skipping broken "{short_identifier}" format')
  277.                 continue
  278.             if not self._is_requested(short_identifier):
  279.                 self.write_debug(f'"{short_identifier}" is not a requested format, skipping')
  280.                 continue
  282.             # XXX: if not extract_flat, 429 error must be caught where _extract_info_dict is called
  283.             stream_url = traverse_obj(self._call_api(
  284.                 format_url, track_id, f'Downloading {short_identifier} format info JSON',
  285.                 query=query, headers=self._HEADERS), ('url', {url_or_none}))
  286.             if invalid_url(stream_url):
  287.                 continue
  288.             format_urls.add(stream_url)
  290.             mime_type = traverse_obj(t, ('format', 'mime_type', {str}))
  291.             codec = self._search_regex(r'codecs="([^"]+)"', mime_type, 'codec', default=None)
  292.             ext = {
  293.                 'mp4a': 'm4a',
  294.                 'opus': 'opus',
  295.             }.get(codec[:4] if codec else None) or mimetype2ext(mime_type, default=None)
  296.             if not ext or ext == 'm3u8':
  297.                 ext = preset_base
  299.             is_premium = t.get('quality') == 'hq'
  300.             abr = int_or_none(
  301.                 self._search_regex(r'(\d+)k$', preset, 'abr', default=None)
  302.                 or self._search_regex(r'\.(\d+)\.(?:opus|mp3)[/?]', stream_url, 'abr', default=None)
  303.                 or (256 if (is_premium and 'aac' in preset) else None))
  305.             is_preview = (t.get('snipped')
  306.                           or '/preview/' in format_url
  307.                           or re.search(r'/(?:preview|playlist)/0/30/', stream_url))
  309.             formats.append({
  310.                 'format_id': join_nonempty(protocol, preset, is_preview and 'preview', delim='_'),
  311.                 'url': stream_url,
  312.                 'ext': ext,
  313.                 'acodec': codec,
  314.                 'vcodec': 'none',
  315.                 'abr': abr,
  316.                 'protocol': 'm3u8_native' if protocol in ('hls', 'hls-aes') else 'http',
  317.                 'container': 'm4a_dash' if ext == 'm4a' else None,
  318.                 'quality': 5 if is_premium else 0 if (abr and abr >= 160) else -1,
  319.                 'format_note': 'Premium' if is_premium else None,
  320.                 'preference': -10 if is_preview else None,
  321.             })
  323.         if not formats:
  324.             if has_drm:
  325.                 self.report_drm(track_id)
  326.             if info.get('policy') == 'BLOCK':
  327.                 self.raise_geo_restricted(metadata_available=True)
  329.         user = info.get('user') or {}
  331.         thumbnails = []
  332.         artwork_url = info.get('artwork_url')
  333.         thumbnail = artwork_url or user.get('avatar_url')
  334.         if isinstance(thumbnail, str):
  335.             if re.search(self._IMAGE_REPL_RE, thumbnail):
  336.                 for image_id, size in self._ARTWORK_MAP.items():
  337.                     i = {
  338.                         'id': image_id,
  339.                         'url': re.sub(self._IMAGE_REPL_RE, f'-{image_id}.jpg', thumbnail),
  340.                     }
  341.                     if image_id == 'tiny' and not artwork_url:
  342.                         size = 18
  343.                     elif image_id == 'original':
  344.                         i['preference'] = 10
  345.                     if size:
  346.                         i.update({
  347.                             'width': size,
  348.                             'height': size,
  349.                         })
  350.                     thumbnails.append(i)
  351.             else:
  352.                 thumbnails = [{'url': thumbnail}]
  354.         def extract_count(key):
  355.             return int_or_none(info.get(f'{key}_count'))
  357.         return {
  358.             'id': track_id,
  359.             'uploader': user.get('username'),
  360.             'uploader_id': str_or_none(user.get('id')) or user.get('permalink'),
  361.             'uploader_url': user.get('permalink_url'),
  362.             'timestamp': unified_timestamp(info.get('created_at')),
  363.             'title': info.get('title'),
  364.             'description': info.get('description'),
  365.             'thumbnails': thumbnails,
  366.             'duration': float_or_none(info.get('duration'), 1000),
  367.             'webpage_url': info.get('permalink_url'),
  368.             'license': info.get('license'),
  369.             'view_count': extract_count('playback'),
  370.             'like_count': extract_count('favoritings') or extract_count('likes'),
  371.             'comment_count': extract_count('comment'),
  372.             'repost_count': extract_count('reposts'),
  373.             'genres': traverse_obj(info, ('genre', {str}, filter, all, filter)),
  374.             'artists': traverse_obj(info, ('publisher_metadata', 'artist', {str}, filter, all, filter)),
  375.             'formats': formats if not extract_flat else None,
  376.         }
  378.     @classmethod
  379.     def _resolv_url(cls, url):
  380.         return cls._API_V2_BASE + 'resolve?url=' + url
  383. class SoundcloudIE(SoundcloudBaseIE):
  384.     """Information extractor for soundcloud.com
  385.        To access the media, the uid of the song and a stream token
  386.        must be extracted from the page source and the script must make
  387.        a request to media.soundcloud.com/crossdomain.xml. Then
  388.        the media can be grabbed by requesting from an url composed
  389.        of the stream token and uid
  390.      """
  392.     _VALID_URL = r'''(?x)^(?:https?://)?
  393.                     (?:(?:(?:www\.|m\.)?soundcloud\.com/
  394.                             (?!stations/track)
  395.                             (?P<uploader>[\w\d-]+)/
  396.                             (?!(?:tracks|albums|sets(?:/.+?)?|reposts|likes|spotlight)/?(?:$|[?#]))
  397.                             (?P<title>[\w\d-]+)
  398.                             (?:/(?P<token>(?!(?:albums|sets|recommended))[^?]+?))?
  399.                             (?:[?].*)?$)
  400.                        |(?:api(?:-v2)?\.soundcloud\.com/tracks/(?P<track_id>\d+)
  401.                           (?:/?\?secret_token=(?P<secret_token>[^&]+))?)
  402.                     )
  403.                     '''
  404.     IE_NAME = 'soundcloud'
  405.     _TESTS = [
  406.         {
  407.             'url': 'http://soundcloud.com/ethmusic/lostin-powers-she-so-heavy',
  408.             'md5': 'de9bac153e7427a7333b4b0c1b6a18d2',
  409.             'info_dict': {
  410.                 'id': '62986583',
  411.                 'ext': 'opus',
  412.                 'title': 'Lostin Powers - She so Heavy (SneakPreview) Adrian Ackers Blueprint 1',
  413.                 'description': 'No Downloads untill we record the finished version this weekend, i was too pumped n i had to post it , earl is prolly gonna b hella p.o\'d',
  414.                 'uploader': 'E.T. ExTerrestrial Music',
  415.                 'uploader_id': '1571244',
  416.                 'timestamp': 1349920598,
  417.                 'upload_date': '20121011',
  418.                 'duration': 143.216,
  419.                 'license': 'all-rights-reserved',
  420.                 'view_count': int,
  421.                 'like_count': int,
  422.                 'comment_count': int,
  423.                 'repost_count': int,
  424.                 'thumbnail': 'https://i1.sndcdn.com/artworks-000031955188-rwb18x-original.jpg',
  425.                 'uploader_url': 'https://soundcloud.com/ethmusic',
  426.             },
  427.         },
  428.         # geo-restricted
  429.         {
  430.             'url': 'https://soundcloud.com/the-concept-band/goldrushed-mastered?in=the-concept-band/sets/the-royal-concept-ep',
  431.             'info_dict': {
  432.                 'id': '47127627',
  433.                 'ext': 'opus',
  434.                 'title': 'Goldrushed',
  435.                 'description': 'From Stockholm Sweden\r\nPovel / Magnus / Filip / David\r\nwww.theroyalconcept.com',
  436.                 'uploader': 'The Royal Concept',
  437.                 'uploader_id': '9615865',
  438.                 'timestamp': 1337635207,
  439.                 'upload_date': '20120521',
  440.                 'duration': 227.155,
  441.                 'license': 'all-rights-reserved',
  442.                 'view_count': int,
  443.                 'like_count': int,
  444.                 'comment_count': int,
  445.                 'repost_count': int,
  446.                 'uploader_url': 'https://soundcloud.com/the-concept-band',
  447.                 'thumbnail': 'https://i1.sndcdn.com/artworks-v8bFHhXm7Au6-0-original.jpg',
  448.                 'genres': ['Alternative'],
  449.                 'artists': ['The Royal Concept'],
  450.             },
  451.         },
  452.         # private link
  453.         {
  454.             'url': 'https://soundcloud.com/jaimemf/youtube-dl-test-video-a-y-baw/s-8Pjrp',
  455.             'md5': 'aa0dd32bfea9b0c5ef4f02aacd080604',
  456.             'info_dict': {
  457.                 'id': '123998367',
  458.                 'ext': 'mp3',
  459.                 'title': 'Youtube - Dl Test Video \'\' Ä↭',
  460.                 'description': 'test chars:  "\'/\\ä↭',
  461.                 'uploader': 'jaimeMF',
  462.                 'uploader_id': '69767071',
  463.                 'timestamp': 1386604920,
  464.                 'upload_date': '20131209',
  465.                 'duration': 9.927,
  466.                 'license': 'all-rights-reserved',
  467.                 'view_count': int,
  468.                 'like_count': int,
  469.                 'comment_count': int,
  470.                 'repost_count': int,
  471.                 'uploader_url': 'https://soundcloud.com/jaimemf',
  472.                 'thumbnail': 'https://a1.sndcdn.com/images/default_avatar_large.png',
  473.                 'genres': ['youtubedl'],
  474.             },
  475.         },
  476.         # private link (alt format)
  477.         {
  478.             'url': 'https://api.soundcloud.com/tracks/123998367?secret_token=s-8Pjrp',
  479.             'md5': 'aa0dd32bfea9b0c5ef4f02aacd080604',
  480.             'info_dict': {
  481.                 'id': '123998367',
  482.                 'ext': 'mp3',
  483.                 'title': 'Youtube - Dl Test Video \'\' Ä↭',
  484.                 'description': 'test chars:  "\'/\\ä↭',
  485.                 'uploader': 'jaimeMF',
  486.                 'uploader_id': '69767071',
  487.                 'timestamp': 1386604920,
  488.                 'upload_date': '20131209',
  489.                 'duration': 9.927,
  490.                 'license': 'all-rights-reserved',
  491.                 'view_count': int,
  492.                 'like_count': int,
  493.                 'comment_count': int,
  494.                 'repost_count': int,
  495.                 'uploader_url': 'https://soundcloud.com/jaimemf',
  496.                 'thumbnail': 'https://a1.sndcdn.com/images/default_avatar_large.png',
  497.                 'genres': ['youtubedl'],
  498.             },
  499.         },
  500.         # downloadable song
  501.         {
  502.             'url': 'https://soundcloud.com/the80m/the-following',
  503.             'md5': '9ffcddb08c87d74fb5808a3c183a1d04',
  504.             'info_dict': {
  505.                 'id': '343609555',
  506.                 'ext': 'wav',
  507.                 'title': 'The Following',
  508.                 'description': '',
  509.                 'uploader': '80M',
  510.                 'uploader_id': '312384765',
  511.                 'uploader_url': 'https://soundcloud.com/the80m',
  512.                 'upload_date': '20170922',
  513.                 'timestamp': 1506120436,
  514.                 'duration': 397.228,
  515.                 'thumbnail': 'https://i1.sndcdn.com/artworks-000243916348-ktoo7d-original.jpg',
  516.                 'license': 'all-rights-reserved',
  517.                 'like_count': int,
  518.                 'comment_count': int,
  519.                 'repost_count': int,
  520.                 'view_count': int,
  521.                 'genres': ['Dance & EDM'],
  522.                 'artists': ['80M'],
  523.             },
  524.         },
  525.         # private link, downloadable format
  526.         {
  527.             'url': 'https://soundcloud.com/oriuplift/uponly-238-no-talking-wav/s-AyZUd',
  528.             'md5': '64a60b16e617d41d0bef032b7f55441e',
  529.             'info_dict': {
  530.                 'id': '340344461',
  531.                 'ext': 'wav',
  532.                 'title': 'Uplifting Only 238 [No Talking] (incl. Alex Feed Guestmix) (Aug 31, 2017) [wav]',
  533.                 'description': 'md5:fa20ee0fca76a3d6df8c7e57f3715366',
  534.                 'uploader': 'Ori Uplift Music',
  535.                 'uploader_id': '12563093',
  536.                 'timestamp': 1504206263,
  537.                 'upload_date': '20170831',
  538.                 'duration': 7449.096,
  539.                 'license': 'all-rights-reserved',
  540.                 'view_count': int,
  541.                 'like_count': int,
  542.                 'comment_count': int,
  543.                 'repost_count': int,
  544.                 'thumbnail': 'https://i1.sndcdn.com/artworks-000240712245-kedn4p-original.jpg',
  545.                 'uploader_url': 'https://soundcloud.com/oriuplift',
  546.                 'genres': ['Trance'],
  547.                 'artists': ['Ori Uplift'],
  548.             },
  549.         },
  550.         # no album art, use avatar pic for thumbnail
  551.         {
  552.             'url': 'https://soundcloud.com/garyvee/sideways-prod-mad-real',
  553.             'md5': '59c7872bc44e5d99b7211891664760c2',
  554.             'info_dict': {
  555.                 'id': '309699954',
  556.                 'ext': 'mp3',
  557.                 'title': 'Sideways (Prod. Mad Real)',
  558.                 'description': 'md5:d41d8cd98f00b204e9800998ecf8427e',
  559.                 'uploader': 'garyvee',
  560.                 'uploader_id': '2366352',
  561.                 'timestamp': 1488152409,
  562.                 'upload_date': '20170226',
  563.                 'duration': 207.012,
  564.                 'thumbnail': r're:https?://.*\.jpg',
  565.                 'license': 'all-rights-reserved',
  566.                 'view_count': int,
  567.                 'like_count': int,
  568.                 'comment_count': int,
  569.                 'repost_count': int,
  570.                 'uploader_url': 'https://soundcloud.com/garyvee',
  571.                 'artists': ['MadReal'],
  572.             },
  573.             'params': {
  574.                 'skip_download': True,
  575.             },
  576.         },
  577.         {
  578.             'url': 'https://soundcloud.com/giovannisarani/mezzo-valzer',
  579.             'md5': '8227c3473a4264df6b02ad7e5b7527ac',
  580.             'info_dict': {
  581.                 'id': '583011102',
  582.                 'ext': 'opus',
  583.                 'title': 'Mezzo Valzer',
  584.                 'description': 'md5:f4d5f39d52e0ccc2b4f665326428901a',
  585.                 'uploader': 'Giovanni Sarani',
  586.                 'uploader_id': '3352531',
  587.                 'timestamp': 1551394171,
  588.                 'upload_date': '20190228',
  589.                 'duration': 180.157,
  590.                 'thumbnail': r're:https?://.*\.jpg',
  591.                 'license': 'all-rights-reserved',
  592.                 'view_count': int,
  593.                 'like_count': int,
  594.                 'comment_count': int,
  595.                 'repost_count': int,
  596.                 'genres': ['Piano'],
  597.                 'uploader_url': 'https://soundcloud.com/giovannisarani',
  598.             },
  599.         },
  600.         {
  601.             # AAC HQ format available (account with active subscription needed)
  602.             'url': 'https://soundcloud.com/wandw/the-chainsmokers-ft-daya-dont-let-me-down-ww-remix-1',
  603.             'only_matching': True,
  604.         },
  605.         {
  606.             # Go+ (account with active subscription needed)
  607.             'url': 'https://soundcloud.com/taylorswiftofficial/look-what-you-made-me-do',
  608.             'only_matching': True,
  609.         },
  610.     ]
  612.     def _real_extract(self, url):
  613.         mobj = self._match_valid_url(url)
  615.         track_id = mobj.group('track_id')
  617.         query = {}
  618.         if track_id:
  619.             info_json_url = self._API_V2_BASE + 'tracks/' + track_id
  620.             full_title = track_id
  621.             token = mobj.group('secret_token')
  622.             if token:
  623.                 query['secret_token'] = token
  624.         else:
  625.             full_title = resolve_title = '{}/{}'.format(*mobj.group('uploader', 'title'))
  626.             token = mobj.group('token')
  627.             if token:
  628.                 resolve_title += f'/{token}'
  629.             info_json_url = self._resolv_url(self._BASE_URL + resolve_title)
  631.         info = self._call_api(
  632.             info_json_url, full_title, 'Downloading info JSON', query=query, headers=self._HEADERS)
  634.         for retry in self.RetryManager():
  635.             try:
  636.                 return self._extract_info_dict(info, full_title, token)
  637.             except ExtractorError as e:
  638.                 if not isinstance(e.cause, HTTPError) or not e.cause.status == 429:
  639.                     raise
  640.                 self.report_warning(
  641.                     'You have reached the API rate limit, which is ~600 requests per '
  642.                     '10 minutes. Use the --extractor-retries and --retry-sleep options '
  643.                     'to configure an appropriate retry count and wait time', only_once=True)
  644.                 retry.error = e.cause
  647. class SoundcloudPlaylistBaseIE(SoundcloudBaseIE):
  648.     def _extract_set(self, playlist, token=None):
  649.         playlist_id = str(playlist['id'])
  650.         tracks = playlist.get('tracks') or []
  651.         if not all(t.get('permalink_url') for t in tracks) and token:
  652.             tracks = self._call_api(
  653.                 self._API_V2_BASE + 'tracks', playlist_id,
  654.                 'Downloading tracks', query={
  655.                     'ids': ','.join([str(t['id']) for t in tracks]),
  656.                     'playlistId': playlist_id,
  657.                     'playlistSecretToken': token,
  658.                 }, headers=self._HEADERS)
  659.         entries = []
  660.         for track in tracks:
  661.             track_id = str_or_none(track.get('id'))
  662.             url = track.get('permalink_url')
  663.             if not url:
  664.                 if not track_id:
  665.                     continue
  666.                 url = self._API_V2_BASE + 'tracks/' + track_id
  667.                 if token:
  668.                     url += '?secret_token=' + token
  669.             entries.append(self.url_result(
  670.                 url, SoundcloudIE.ie_key(), track_id))
  671.         return self.playlist_result(
  672.             entries, playlist_id,
  673.             playlist.get('title'),
  674.             playlist.get('description'))
  677. class SoundcloudSetIE(SoundcloudPlaylistBaseIE):
  678.     _VALID_URL = r'https?://(?:(?:www|m)\.)?soundcloud\.com/(?P<uploader>[\w\d-]+)/sets/(?P<slug_title>[:\w\d-]+)(?:/(?P<token>[^?/]+))?'
  679.     IE_NAME = 'soundcloud:set'
  680.     _TESTS = [{
  681.         'url': 'https://soundcloud.com/the-concept-band/sets/the-royal-concept-ep',
  682.         'info_dict': {
  683.             'id': '2284613',
  684.             'title': 'The Royal Concept EP',
  685.             'description': 'md5:71d07087c7a449e8941a70a29e34671e',
  686.         },
  687.         'playlist_mincount': 5,
  688.     }, {
  689.         'url': 'https://soundcloud.com/the-concept-band/sets/the-royal-concept-ep/token',
  690.         'only_matching': True,
  691.     }, {
  692.         'url': 'https://soundcloud.com/discover/sets/weekly::flacmatic',
  693.         'only_matching': True,
  694.     }, {
  695.         'url': 'https://soundcloud.com/discover/sets/charts-top:all-music:de',
  696.         'only_matching': True,
  697.     }, {
  698.         'url': 'https://soundcloud.com/discover/sets/charts-top:hiphoprap:kr',
  699.         'only_matching': True,
  700.     }]
  702.     def _real_extract(self, url):
  703.         mobj = self._match_valid_url(url)
  705.         full_title = '{}/sets/{}'.format(*mobj.group('uploader', 'slug_title'))
  706.         token = mobj.group('token')
  707.         if token:
  708.             full_title += '/' + token
  710.         info = self._call_api(self._resolv_url(
  711.             self._BASE_URL + full_title), full_title, headers=self._HEADERS)
  713.         if 'errors' in info:
  714.             msgs = (str(err['error_message']) for err in info['errors'])
  715.             raise ExtractorError('unable to download video webpage: {}'.format(','.join(msgs)))
  717.         return self._extract_set(info, token)
  720. class SoundcloudPagedPlaylistBaseIE(SoundcloudBaseIE):
  721.     def _extract_playlist(self, base_url, playlist_id, playlist_title):
  722.         return {
  723.             '_type': 'playlist',
  724.             'id': playlist_id,
  725.             'title': playlist_title,
  726.             'entries': self._entries(base_url, playlist_id),
  727.         }
  729.     def _entries(self, url, playlist_id):
  730.         # Per the SoundCloud documentation, the maximum limit for a linked partitioning query is 200.
  731.         # https://developers.soundcloud.com/blog/offset-pagination-deprecated
  732.         query = {
  733.             'limit': 200,
  734.             'linked_partitioning': '1',
  735.             'offset': 0,
  736.         }
  738.         for i in itertools.count():
  739.             for retry in self.RetryManager():
  740.                 try:
  741.                     response = self._call_api(
  742.                         url, playlist_id, query=query, headers=self._HEADERS,
  743.                         note=f'Downloading track page {i + 1}')
  744.                     break
  745.                 except ExtractorError as e:
  746.                     # Downloading page may result in intermittent 502 HTTP error
  747.                     # See https://github.com/yt-dlp/yt-dlp/issues/872
  748.                     if not isinstance(e.cause, HTTPError) or e.cause.status != 502:
  749.                         raise
  750.                     retry.error = e
  751.                     continue
  753.             def resolve_entry(*candidates):
  754.                 for cand in candidates:
  755.                     if not isinstance(cand, dict):
  756.                         continue
  757.                     permalink_url = url_or_none(cand.get('permalink_url'))
  758.                     if permalink_url:
  759.                         return self.url_result(
  760.                             permalink_url,
  761.                             SoundcloudIE.ie_key() if SoundcloudIE.suitable(permalink_url) else None,
  762.                             str_or_none(cand.get('id')), cand.get('title'))
  764.             for e in response['collection'] or []:
  765.                 yield resolve_entry(e, e.get('track'), e.get('playlist'))
  767.             url = response.get('next_href')
  768.             if not url:
  769.                 break
  770.             query.pop('offset', None)
  773. class SoundcloudUserIE(SoundcloudPagedPlaylistBaseIE):
  774.     _VALID_URL = r'''(?x)
  775.                         https?://
  776.                             (?:(?:www|m)\.)?soundcloud\.com/
  777.                             (?P<user>[^/]+)
  778.                             (?:/
  779.                                 (?P<rsrc>tracks|albums|sets|reposts|likes|spotlight)
  780.                             )?
  781.                             /?(?:[?#].*)?$
  782.                     '''
  783.     IE_NAME = 'soundcloud:user'
  784.     _TESTS = [{
  785.         'url': 'https://soundcloud.com/soft-cell-official',
  786.         'info_dict': {
  787.             'id': '207965082',
  788.             'title': 'Soft Cell (All)',
  789.         },
  790.         'playlist_mincount': 28,
  791.     }, {
  792.         'url': 'https://soundcloud.com/soft-cell-official/tracks',
  793.         'info_dict': {
  794.             'id': '207965082',
  795.             'title': 'Soft Cell (Tracks)',
  796.         },
  797.         'playlist_mincount': 27,
  798.     }, {
  799.         'url': 'https://soundcloud.com/soft-cell-official/albums',
  800.         'info_dict': {
  801.             'id': '207965082',
  802.             'title': 'Soft Cell (Albums)',
  803.         },
  804.         'playlist_mincount': 1,
  805.     }, {
  806.         'url': 'https://soundcloud.com/jcv246/sets',
  807.         'info_dict': {
  808.             'id': '12982173',
  809.             'title': 'Jordi / cv (Sets)',
  810.         },
  811.         'playlist_mincount': 2,
  812.     }, {
  813.         'url': 'https://soundcloud.com/jcv246/reposts',
  814.         'info_dict': {
  815.             'id': '12982173',
  816.             'title': 'Jordi / cv (Reposts)',
  817.         },
  818.         'playlist_mincount': 6,
  819.     }, {
  820.         'url': 'https://soundcloud.com/clalberg/likes',
  821.         'info_dict': {
  822.             'id': '11817582',
  823.             'title': 'clalberg (Likes)',
  824.         },
  825.         'playlist_mincount': 5,
  826.     }, {
  827.         'url': 'https://soundcloud.com/grynpyret/spotlight',
  828.         'info_dict': {
  829.             'id': '7098329',
  830.             'title': 'Grynpyret (Spotlight)',
  831.         },
  832.         'playlist_mincount': 1,
  833.     }]
  835.     _BASE_URL_MAP = {
  836.         'all': 'stream/users/%s',
  837.         'tracks': 'users/%s/tracks',
  838.         'albums': 'users/%s/albums',
  839.         'sets': 'users/%s/playlists',
  840.         'reposts': 'stream/users/%s/reposts',
  841.         'likes': 'users/%s/likes',
  842.         'spotlight': 'users/%s/spotlight',
  843.     }
  845.     def _real_extract(self, url):
  846.         mobj = self._match_valid_url(url)
  847.         uploader = mobj.group('user')
  849.         user = self._call_api(
  850.             self._resolv_url(self._BASE_URL + uploader),
  851.             uploader, 'Downloading user info', headers=self._HEADERS)
  853.         resource = mobj.group('rsrc') or 'all'
  855.         return self._extract_playlist(
  856.             self._API_V2_BASE + self._BASE_URL_MAP[resource] % user['id'],
  857.             str_or_none(user.get('id')),
  858.             '{} ({})'.format(user['username'], resource.capitalize()))
  861. class SoundcloudUserPermalinkIE(SoundcloudPagedPlaylistBaseIE):
  862.     _VALID_URL = r'https?://api\.soundcloud\.com/users/(?P<id>\d+)'
  863.     IE_NAME = 'soundcloud:user:permalink'
  864.     _TESTS = [{
  865.         'url': 'https://api.soundcloud.com/users/30909869',
  866.         'info_dict': {
  867.             'id': '30909869',
  868.             'title': 'neilcic',
  869.         },
  870.         'playlist_mincount': 22,
  871.     }]
  873.     def _real_extract(self, url):
  874.         user_id = self._match_id(url)
  875.         user = self._call_api(
  876.             self._resolv_url(url), user_id, 'Downloading user info', headers=self._HEADERS)
  878.         return self._extract_playlist(
  879.             f'{self._API_V2_BASE}users/{user["id"]}/tracks', str(user['id']), user.get('username'))
  882. class SoundcloudTrackStationIE(SoundcloudPagedPlaylistBaseIE):
  883.     _VALID_URL = r'https?://(?:(?:www|m)\.)?soundcloud\.com/stations/track/[^/]+/(?P<id>[^/?#&]+)'
  884.     IE_NAME = 'soundcloud:trackstation'
  885.     _TESTS = [{
  886.         'url': 'https://soundcloud.com/stations/track/officialsundial/your-text',
  887.         'info_dict': {
  888.             'id': '286017854',
  889.             'title': 'Track station: your text',
  890.         },
  891.         'playlist_mincount': 47,
  892.     }]
  894.     def _real_extract(self, url):
  895.         track_name = self._match_id(url)
  897.         track = self._call_api(self._resolv_url(url), track_name, headers=self._HEADERS)
  898.         track_id = self._search_regex(
  899.             r'soundcloud:track-stations:(\d+)', track['id'], 'track id')
  901.         return self._extract_playlist(
  902.             self._API_V2_BASE + 'stations/{}/tracks'.format(track['id']),
  903.             track_id, 'Track station: {}'.format(track['title']))
  906. class SoundcloudRelatedIE(SoundcloudPagedPlaylistBaseIE):
  907.     _VALID_URL = r'https?://(?:(?:www|m)\.)?soundcloud\.com/(?P<slug>[\w\d-]+/[\w\d-]+)/(?P<relation>albums|sets|recommended)'
  908.     IE_NAME = 'soundcloud:related'
  909.     _TESTS = [{
  910.         'url': 'https://soundcloud.com/wajang/sexapil-pingers-5/recommended',
  911.         'info_dict': {
  912.             'id': '1084577272',
  913.             'title': 'Sexapil - Pingers 5 (Recommended)',
  914.         },
  915.         'playlist_mincount': 50,
  916.     }, {
  917.         'url': 'https://soundcloud.com/wajang/sexapil-pingers-5/albums',
  918.         'info_dict': {
  919.             'id': '1084577272',
  920.             'title': 'Sexapil - Pingers 5 (Albums)',
  921.         },
  922.         'playlist_mincount': 1,
  923.     }, {
  924.         'url': 'https://soundcloud.com/wajang/sexapil-pingers-5/sets',
  925.         'info_dict': {
  926.             'id': '1084577272',
  927.             'title': 'Sexapil - Pingers 5 (Sets)',
  928.         },
  929.         'playlist_mincount': 4,
  930.     }]
  932.     _BASE_URL_MAP = {
  933.         'albums': 'tracks/%s/albums',
  934.         'sets': 'tracks/%s/playlists_without_albums',
  935.         'recommended': 'tracks/%s/related',
  936.     }
  938.     def _real_extract(self, url):
  939.         slug, relation = self._match_valid_url(url).group('slug', 'relation')
  941.         track = self._call_api(
  942.             self._resolv_url(self._BASE_URL + slug),
  943.             slug, 'Downloading track info', headers=self._HEADERS)
  945.         if track.get('errors'):
  946.             raise ExtractorError(f'{self.IE_NAME} said: %s' % ','.join(
  947.                 str(err['error_message']) for err in track['errors']), expected=True)
  949.         return self._extract_playlist(
  950.             self._API_V2_BASE + self._BASE_URL_MAP[relation] % track['id'], str(track['id']),
  951.             '{} ({})'.format(track.get('title') or slug, relation.capitalize()))
  954. class SoundcloudPlaylistIE(SoundcloudPlaylistBaseIE):
  955.     _VALID_URL = r'https?://api(?:-v2)?\.soundcloud\.com/playlists/(?P<id>[0-9]+)(?:/?\?secret_token=(?P<token>[^&]+?))?$'
  956.     IE_NAME = 'soundcloud:playlist'
  957.     _TESTS = [{
  958.         'url': 'https://api.soundcloud.com/playlists/4110309',
  959.         'info_dict': {
  960.             'id': '4110309',
  961.             'title': 'TILT Brass - Bowery Poetry Club, August \'03 [Non-Site SCR 02]',
  962.             'description': 're:.*?TILT Brass - Bowery Poetry Club',
  963.         },
  964.         'playlist_count': 6,
  965.     }]
  967.     def _real_extract(self, url):
  968.         mobj = self._match_valid_url(url)
  969.         playlist_id = mobj.group('id')
  971.         query = {}
  972.         token = mobj.group('token')
  973.         if token:
  974.             query['secret_token'] = token
  976.         data = self._call_api(
  977.             self._API_V2_BASE + 'playlists/' + playlist_id,
  978.             playlist_id, 'Downloading playlist', query=query, headers=self._HEADERS)
  980.         return self._extract_set(data, token)
  983. class SoundcloudSearchIE(SoundcloudBaseIE, SearchInfoExtractor):
  984.     IE_NAME = 'soundcloud:search'
  985.     IE_DESC = 'Soundcloud search'
  986.     _SEARCH_KEY = 'scsearch'
  987.     _TESTS = [{
  988.         'url': 'scsearch15:post-avant jazzcore',
  989.         'info_dict': {
  990.             'id': 'post-avant jazzcore',
  991.             'title': 'post-avant jazzcore',
  992.         },
  993.         'playlist_count': 15,
  994.     }]
  996.     _MAX_RESULTS_PER_PAGE = 200
  997.     _DEFAULT_RESULTS_PER_PAGE = 50
  999.     def _get_collection(self, endpoint, collection_id, **query):
  1000.         limit = min(
  1001.             query.get('limit', self._DEFAULT_RESULTS_PER_PAGE),
  1002.             self._MAX_RESULTS_PER_PAGE)
  1003.         query.update({
  1004.             'limit': limit,
  1005.             'linked_partitioning': 1,
  1006.             'offset': 0,
  1007.         })
  1008.         next_url = update_url_query(self._API_V2_BASE + endpoint, query)
  1010.         for i in itertools.count(1):
  1011.             response = self._call_api(
  1012.                 next_url, collection_id, f'Downloading page {i}',
  1013.                 'Unable to download API page', headers=self._HEADERS)
  1015.             for item in response.get('collection') or []:
  1016.                 if item:
  1017.                     yield self.url_result(
  1018.                         item['uri'], SoundcloudIE.ie_key(), **self._extract_info_dict(item, extract_flat=True))
  1020.             next_url = response.get('next_href')
  1021.             if not next_url:
  1022.                 break
  1024.     def _get_n_results(self, query, n):
  1025.         return self.playlist_result(itertools.islice(
  1026.             self._get_collection('search/tracks', query, limit=n, q=query),
  1027.             0, None if n == float('inf') else n), query, query)