logo

youtube-dl

[mirror] Download/Watch videos from video hostersgit clone https://hacktivis.me/git/mirror/youtube-dl.git

vevo.py (14131B)


  1. from __future__ import unicode_literals
  2. import re
  3. import json
  4. from .common import InfoExtractor
  5. from ..compat import (
  6. compat_str,
  7. compat_urlparse,
  8. compat_HTTPError,
  9. )
  10. from ..utils import (
  11. ExtractorError,
  12. int_or_none,
  13. parse_iso8601,
  14. )
  15. class VevoBaseIE(InfoExtractor):
  16. def _extract_json(self, webpage, video_id):
  17. return self._parse_json(
  18. self._search_regex(
  19. r'window\.__INITIAL_STORE__\s*=\s*({.+?});\s*</script>',
  20. webpage, 'initial store'),
  21. video_id)
  22. class VevoIE(VevoBaseIE):
  23. '''
  24. Accepts urls from vevo.com or in the format 'vevo:{id}'
  25. (currently used by MTVIE and MySpaceIE)
  26. '''
  27. _VALID_URL = r'''(?x)
  28. (?:https?://(?:www\.)?vevo\.com/watch/(?!playlist|genre)(?:[^/]+/(?:[^/]+/)?)?|
  29. https?://cache\.vevo\.com/m/html/embed\.html\?video=|
  30. https?://videoplayer\.vevo\.com/embed/embedded\?videoId=|
  31. https?://embed\.vevo\.com/.*?[?&]isrc=|
  32. vevo:)
  33. (?P<id>[^&?#]+)'''
  34. _TESTS = [{
  35. 'url': 'http://www.vevo.com/watch/hurts/somebody-to-die-for/GB1101300280',
  36. 'md5': '95ee28ee45e70130e3ab02b0f579ae23',
  37. 'info_dict': {
  38. 'id': 'GB1101300280',
  39. 'ext': 'mp4',
  40. 'title': 'Hurts - Somebody to Die For',
  41. 'timestamp': 1372057200,
  42. 'upload_date': '20130624',
  43. 'uploader': 'Hurts',
  44. 'track': 'Somebody to Die For',
  45. 'artist': 'Hurts',
  46. 'genre': 'Pop',
  47. },
  48. 'expected_warnings': ['Unable to download SMIL file', 'Unable to download info'],
  49. }, {
  50. 'note': 'v3 SMIL format',
  51. 'url': 'http://www.vevo.com/watch/cassadee-pope/i-wish-i-could-break-your-heart/USUV71302923',
  52. 'md5': 'f6ab09b034f8c22969020b042e5ac7fc',
  53. 'info_dict': {
  54. 'id': 'USUV71302923',
  55. 'ext': 'mp4',
  56. 'title': 'Cassadee Pope - I Wish I Could Break Your Heart',
  57. 'timestamp': 1392796919,
  58. 'upload_date': '20140219',
  59. 'uploader': 'Cassadee Pope',
  60. 'track': 'I Wish I Could Break Your Heart',
  61. 'artist': 'Cassadee Pope',
  62. 'genre': 'Country',
  63. },
  64. 'expected_warnings': ['Unable to download SMIL file', 'Unable to download info'],
  65. }, {
  66. 'note': 'Age-limited video',
  67. 'url': 'https://www.vevo.com/watch/justin-timberlake/tunnel-vision-explicit/USRV81300282',
  68. 'info_dict': {
  69. 'id': 'USRV81300282',
  70. 'ext': 'mp4',
  71. 'title': 'Justin Timberlake - Tunnel Vision (Explicit)',
  72. 'age_limit': 18,
  73. 'timestamp': 1372888800,
  74. 'upload_date': '20130703',
  75. 'uploader': 'Justin Timberlake',
  76. 'track': 'Tunnel Vision (Explicit)',
  77. 'artist': 'Justin Timberlake',
  78. 'genre': 'Pop',
  79. },
  80. 'expected_warnings': ['Unable to download SMIL file', 'Unable to download info'],
  81. }, {
  82. 'note': 'No video_info',
  83. 'url': 'http://www.vevo.com/watch/k-camp-1/Till-I-Die/USUV71503000',
  84. 'md5': '8b83cc492d72fc9cf74a02acee7dc1b0',
  85. 'info_dict': {
  86. 'id': 'USUV71503000',
  87. 'ext': 'mp4',
  88. 'title': 'K Camp ft. T.I. - Till I Die',
  89. 'age_limit': 18,
  90. 'timestamp': 1449468000,
  91. 'upload_date': '20151207',
  92. 'uploader': 'K Camp',
  93. 'track': 'Till I Die',
  94. 'artist': 'K Camp',
  95. 'genre': 'Hip-Hop',
  96. },
  97. 'expected_warnings': ['Unable to download SMIL file', 'Unable to download info'],
  98. }, {
  99. 'note': 'Featured test',
  100. 'url': 'https://www.vevo.com/watch/lemaitre/Wait/USUV71402190',
  101. 'md5': 'd28675e5e8805035d949dc5cf161071d',
  102. 'info_dict': {
  103. 'id': 'USUV71402190',
  104. 'ext': 'mp4',
  105. 'title': 'Lemaitre ft. LoLo - Wait',
  106. 'age_limit': 0,
  107. 'timestamp': 1413432000,
  108. 'upload_date': '20141016',
  109. 'uploader': 'Lemaitre',
  110. 'track': 'Wait',
  111. 'artist': 'Lemaitre',
  112. 'genre': 'Electronic',
  113. },
  114. 'expected_warnings': ['Unable to download SMIL file', 'Unable to download info'],
  115. }, {
  116. 'note': 'Only available via webpage',
  117. 'url': 'http://www.vevo.com/watch/GBUV71600656',
  118. 'md5': '67e79210613865b66a47c33baa5e37fe',
  119. 'info_dict': {
  120. 'id': 'GBUV71600656',
  121. 'ext': 'mp4',
  122. 'title': 'ABC - Viva Love',
  123. 'age_limit': 0,
  124. 'timestamp': 1461830400,
  125. 'upload_date': '20160428',
  126. 'uploader': 'ABC',
  127. 'track': 'Viva Love',
  128. 'artist': 'ABC',
  129. 'genre': 'Pop',
  130. },
  131. 'expected_warnings': ['Failed to download video versions info'],
  132. }, {
  133. # no genres available
  134. 'url': 'http://www.vevo.com/watch/INS171400764',
  135. 'only_matching': True,
  136. }, {
  137. # Another case available only via the webpage; using streams/streamsV3 formats
  138. # Geo-restricted to Netherlands/Germany
  139. 'url': 'http://www.vevo.com/watch/boostee/pop-corn-clip-officiel/FR1A91600909',
  140. 'only_matching': True,
  141. }, {
  142. 'url': 'https://embed.vevo.com/?isrc=USH5V1923499&partnerId=4d61b777-8023-4191-9ede-497ed6c24647&partnerAdCode=',
  143. 'only_matching': True,
  144. }]
  145. _VERSIONS = {
  146. 0: 'youtube', # only in AuthenticateVideo videoVersions
  147. 1: 'level3',
  148. 2: 'akamai',
  149. 3: 'level3',
  150. 4: 'amazon',
  151. }
  152. def _initialize_api(self, video_id):
  153. webpage = self._download_webpage(
  154. 'https://accounts.vevo.com/token', None,
  155. note='Retrieving oauth token',
  156. errnote='Unable to retrieve oauth token',
  157. data=json.dumps({
  158. 'client_id': 'SPupX1tvqFEopQ1YS6SS',
  159. 'grant_type': 'urn:vevo:params:oauth:grant-type:anonymous',
  160. }).encode('utf-8'),
  161. headers={
  162. 'Content-Type': 'application/json',
  163. })
  164. if re.search(r'(?i)THIS PAGE IS CURRENTLY UNAVAILABLE IN YOUR REGION', webpage):
  165. self.raise_geo_restricted(
  166. '%s said: This page is currently unavailable in your region' % self.IE_NAME)
  167. auth_info = self._parse_json(webpage, video_id)
  168. self._api_url_template = self.http_scheme() + '//apiv2.vevo.com/%s?token=' + auth_info['legacy_token']
  169. def _call_api(self, path, *args, **kwargs):
  170. try:
  171. data = self._download_json(self._api_url_template % path, *args, **kwargs)
  172. except ExtractorError as e:
  173. if isinstance(e.cause, compat_HTTPError):
  174. errors = self._parse_json(e.cause.read().decode(), None)['errors']
  175. error_message = ', '.join([error['message'] for error in errors])
  176. raise ExtractorError('%s said: %s' % (self.IE_NAME, error_message), expected=True)
  177. raise
  178. return data
  179. def _real_extract(self, url):
  180. video_id = self._match_id(url)
  181. self._initialize_api(video_id)
  182. video_info = self._call_api(
  183. 'video/%s' % video_id, video_id, 'Downloading api video info',
  184. 'Failed to download video info')
  185. video_versions = self._call_api(
  186. 'video/%s/streams' % video_id, video_id,
  187. 'Downloading video versions info',
  188. 'Failed to download video versions info',
  189. fatal=False)
  190. # Some videos are only available via webpage (e.g.
  191. # https://github.com/ytdl-org/youtube-dl/issues/9366)
  192. if not video_versions:
  193. webpage = self._download_webpage(url, video_id)
  194. json_data = self._extract_json(webpage, video_id)
  195. if 'streams' in json_data.get('default', {}):
  196. video_versions = json_data['default']['streams'][video_id][0]
  197. else:
  198. video_versions = [
  199. value
  200. for key, value in json_data['apollo']['data'].items()
  201. if key.startswith('%s.streams' % video_id)]
  202. uploader = None
  203. artist = None
  204. featured_artist = None
  205. artists = video_info.get('artists')
  206. for curr_artist in artists:
  207. if curr_artist.get('role') == 'Featured':
  208. featured_artist = curr_artist['name']
  209. else:
  210. artist = uploader = curr_artist['name']
  211. formats = []
  212. for video_version in video_versions:
  213. version = self._VERSIONS.get(video_version.get('version'), 'generic')
  214. version_url = video_version.get('url')
  215. if not version_url:
  216. continue
  217. if '.ism' in version_url:
  218. continue
  219. elif '.mpd' in version_url:
  220. formats.extend(self._extract_mpd_formats(
  221. version_url, video_id, mpd_id='dash-%s' % version,
  222. note='Downloading %s MPD information' % version,
  223. errnote='Failed to download %s MPD information' % version,
  224. fatal=False))
  225. elif '.m3u8' in version_url:
  226. formats.extend(self._extract_m3u8_formats(
  227. version_url, video_id, 'mp4', 'm3u8_native',
  228. m3u8_id='hls-%s' % version,
  229. note='Downloading %s m3u8 information' % version,
  230. errnote='Failed to download %s m3u8 information' % version,
  231. fatal=False))
  232. else:
  233. m = re.search(r'''(?xi)
  234. _(?P<width>[0-9]+)x(?P<height>[0-9]+)
  235. _(?P<vcodec>[a-z0-9]+)
  236. _(?P<vbr>[0-9]+)
  237. _(?P<acodec>[a-z0-9]+)
  238. _(?P<abr>[0-9]+)
  239. \.(?P<ext>[a-z0-9]+)''', version_url)
  240. if not m:
  241. continue
  242. formats.append({
  243. 'url': version_url,
  244. 'format_id': 'http-%s-%s' % (version, video_version['quality']),
  245. 'vcodec': m.group('vcodec'),
  246. 'acodec': m.group('acodec'),
  247. 'vbr': int(m.group('vbr')),
  248. 'abr': int(m.group('abr')),
  249. 'ext': m.group('ext'),
  250. 'width': int(m.group('width')),
  251. 'height': int(m.group('height')),
  252. })
  253. self._sort_formats(formats)
  254. track = video_info['title']
  255. if featured_artist:
  256. artist = '%s ft. %s' % (artist, featured_artist)
  257. title = '%s - %s' % (artist, track) if artist else track
  258. genres = video_info.get('genres')
  259. genre = (
  260. genres[0] if genres and isinstance(genres, list)
  261. and isinstance(genres[0], compat_str) else None)
  262. is_explicit = video_info.get('isExplicit')
  263. if is_explicit is True:
  264. age_limit = 18
  265. elif is_explicit is False:
  266. age_limit = 0
  267. else:
  268. age_limit = None
  269. return {
  270. 'id': video_id,
  271. 'title': title,
  272. 'formats': formats,
  273. 'thumbnail': video_info.get('imageUrl') or video_info.get('thumbnailUrl'),
  274. 'timestamp': parse_iso8601(video_info.get('releaseDate')),
  275. 'uploader': uploader,
  276. 'duration': int_or_none(video_info.get('duration')),
  277. 'view_count': int_or_none(video_info.get('views', {}).get('total')),
  278. 'age_limit': age_limit,
  279. 'track': track,
  280. 'artist': uploader,
  281. 'genre': genre,
  282. }
  283. class VevoPlaylistIE(VevoBaseIE):
  284. _VALID_URL = r'https?://(?:www\.)?vevo\.com/watch/(?P<kind>playlist|genre)/(?P<id>[^/?#&]+)'
  285. _TESTS = [{
  286. 'url': 'http://www.vevo.com/watch/playlist/dadbf4e7-b99f-4184-9670-6f0e547b6a29',
  287. 'info_dict': {
  288. 'id': 'dadbf4e7-b99f-4184-9670-6f0e547b6a29',
  289. 'title': 'Best-Of: Birdman',
  290. },
  291. 'playlist_count': 10,
  292. }, {
  293. 'url': 'http://www.vevo.com/watch/genre/rock',
  294. 'info_dict': {
  295. 'id': 'rock',
  296. 'title': 'Rock',
  297. },
  298. 'playlist_count': 20,
  299. }, {
  300. 'url': 'http://www.vevo.com/watch/playlist/dadbf4e7-b99f-4184-9670-6f0e547b6a29?index=0',
  301. 'md5': '32dcdfddddf9ec6917fc88ca26d36282',
  302. 'info_dict': {
  303. 'id': 'USCMV1100073',
  304. 'ext': 'mp4',
  305. 'title': 'Birdman - Y.U. MAD',
  306. 'timestamp': 1323417600,
  307. 'upload_date': '20111209',
  308. 'uploader': 'Birdman',
  309. 'track': 'Y.U. MAD',
  310. 'artist': 'Birdman',
  311. 'genre': 'Rap/Hip-Hop',
  312. },
  313. 'expected_warnings': ['Unable to download SMIL file'],
  314. }, {
  315. 'url': 'http://www.vevo.com/watch/genre/rock?index=0',
  316. 'only_matching': True,
  317. }]
  318. def _real_extract(self, url):
  319. mobj = re.match(self._VALID_URL, url)
  320. playlist_id = mobj.group('id')
  321. playlist_kind = mobj.group('kind')
  322. webpage = self._download_webpage(url, playlist_id)
  323. qs = compat_urlparse.parse_qs(compat_urlparse.urlparse(url).query)
  324. index = qs.get('index', [None])[0]
  325. if index:
  326. video_id = self._search_regex(
  327. r'<meta[^>]+content=(["\'])vevo://video/(?P<id>.+?)\1[^>]*>',
  328. webpage, 'video id', default=None, group='id')
  329. if video_id:
  330. return self.url_result('vevo:%s' % video_id, VevoIE.ie_key())
  331. playlists = self._extract_json(webpage, playlist_id)['default']['%ss' % playlist_kind]
  332. playlist = (list(playlists.values())[0]
  333. if playlist_kind == 'playlist' else playlists[playlist_id])
  334. entries = [
  335. self.url_result('vevo:%s' % src, VevoIE.ie_key())
  336. for src in playlist['isrcs']]
  337. return self.playlist_result(
  338. entries, playlist.get('playlistId') or playlist_id,
  339. playlist.get('name'), playlist.get('description'))