logo

youtube-dl

[mirror] Download/Watch videos from video hostersgit clone https://hacktivis.me/git/mirror/youtube-dl.git

bandcamp.py (14460B)

    

  1. # coding: utf-8
  2. from __future__ import unicode_literals
  4. import random
  5. import re
  6. import time
  8. from .common import InfoExtractor
  9. from ..compat import compat_str
  10. from ..utils import (
  11.     ExtractorError,
  12.     float_or_none,
  13.     int_or_none,
  14.     KNOWN_EXTENSIONS,
  15.     parse_filesize,
  16.     str_or_none,
  17.     try_get,
  18.     update_url_query,
  19.     unified_strdate,
  20.     unified_timestamp,
  21.     url_or_none,
  22.     urljoin,
  23. )
  26. class BandcampIE(InfoExtractor):
  27.     _VALID_URL = r'https?://[^/]+\.bandcamp\.com/track/(?P<id>[^/?#&]+)'
  28.     _TESTS = [{
  29.         'url': 'http://youtube-dl.bandcamp.com/track/youtube-dl-test-song',
  30.         'md5': 'c557841d5e50261777a6585648adf439',
  31.         'info_dict': {
  32.             'id': '1812978515',
  33.             'ext': 'mp3',
  34.             'title': "youtube-dl  \"'/\\ä↭ - youtube-dl  \"'/\\ä↭ - youtube-dl test song \"'/\\ä↭",
  35.             'duration': 9.8485,
  36.             'uploader': 'youtube-dl  "\'/\\ä↭',
  37.             'upload_date': '20121129',
  38.             'timestamp': 1354224127,
  39.         },
  40.         '_skip': 'There is a limit of 200 free downloads / month for the test song'
  41.     }, {
  42.         # free download
  43.         'url': 'http://benprunty.bandcamp.com/track/lanius-battle',
  44.         'info_dict': {
  45.             'id': '2650410135',
  46.             'ext': 'aiff',
  47.             'title': 'Ben Prunty - Lanius (Battle)',
  48.             'thumbnail': r're:^https?://.*\.jpg$',
  49.             'uploader': 'Ben Prunty',
  50.             'timestamp': 1396508491,
  51.             'upload_date': '20140403',
  52.             'release_timestamp': 1396483200,
  53.             'release_date': '20140403',
  54.             'duration': 260.877,
  55.             'track': 'Lanius (Battle)',
  56.             'track_number': 1,
  57.             'track_id': '2650410135',
  58.             'artist': 'Ben Prunty',
  59.             'album': 'FTL: Advanced Edition Soundtrack',
  60.         },
  61.     }, {
  62.         # no free download, mp3 128
  63.         'url': 'https://relapsealumni.bandcamp.com/track/hail-to-fire',
  64.         'md5': 'fec12ff55e804bb7f7ebeb77a800c8b7',
  65.         'info_dict': {
  66.             'id': '2584466013',
  67.             'ext': 'mp3',
  68.             'title': 'Mastodon - Hail to Fire',
  69.             'thumbnail': r're:^https?://.*\.jpg$',
  70.             'uploader': 'Mastodon',
  71.             'timestamp': 1322005399,
  72.             'upload_date': '20111122',
  73.             'release_timestamp': 1076112000,
  74.             'release_date': '20040207',
  75.             'duration': 120.79,
  76.             'track': 'Hail to Fire',
  77.             'track_number': 5,
  78.             'track_id': '2584466013',
  79.             'artist': 'Mastodon',
  80.             'album': 'Call of the Mastodon',
  81.         },
  82.     }]
  84.     def _extract_data_attr(self, webpage, video_id, attr='tralbum', fatal=True):
  85.         return self._parse_json(self._html_search_regex(
  86.             r'data-%s=(["\'])({.+?})\1' % attr, webpage,
  87.             attr + ' data', group=2), video_id, fatal=fatal)
  89.     def _real_extract(self, url):
  90.         title = self._match_id(url)
  91.         webpage = self._download_webpage(url, title)
  92.         tralbum = self._extract_data_attr(webpage, title)
  93.         thumbnail = self._og_search_thumbnail(webpage)
  95.         track_id = None
  96.         track = None
  97.         track_number = None
  98.         duration = None
  100.         formats = []
  101.         track_info = try_get(tralbum, lambda x: x['trackinfo'][0], dict)
  102.         if track_info:
  103.             file_ = track_info.get('file')
  104.             if isinstance(file_, dict):
  105.                 for format_id, format_url in file_.items():
  106.                     if not url_or_none(format_url):
  107.                         continue
  108.                     ext, abr_str = format_id.split('-', 1)
  109.                     formats.append({
  110.                         'format_id': format_id,
  111.                         'url': self._proto_relative_url(format_url, 'http:'),
  112.                         'ext': ext,
  113.                         'vcodec': 'none',
  114.                         'acodec': ext,
  115.                         'abr': int_or_none(abr_str),
  116.                     })
  117.             track = track_info.get('title')
  118.             track_id = str_or_none(
  119.                 track_info.get('track_id') or track_info.get('id'))
  120.             track_number = int_or_none(track_info.get('track_num'))
  121.             duration = float_or_none(track_info.get('duration'))
  123.         embed = self._extract_data_attr(webpage, title, 'embed', False)
  124.         current = tralbum.get('current') or {}
  125.         artist = embed.get('artist') or current.get('artist') or tralbum.get('artist')
  126.         timestamp = unified_timestamp(
  127.             current.get('publish_date') or tralbum.get('album_publish_date'))
  129.         download_link = tralbum.get('freeDownloadPage')
  130.         if download_link:
  131.             track_id = compat_str(tralbum['id'])
  133.             download_webpage = self._download_webpage(
  134.                 download_link, track_id, 'Downloading free downloads page')
  136.             blob = self._extract_data_attr(download_webpage, track_id, 'blob')
  138.             info = try_get(
  139.                 blob, (lambda x: x['digital_items'][0],
  140.                        lambda x: x['download_items'][0]), dict)
  141.             if info:
  142.                 downloads = info.get('downloads')
  143.                 if isinstance(downloads, dict):
  144.                     if not track:
  145.                         track = info.get('title')
  146.                     if not artist:
  147.                         artist = info.get('artist')
  148.                     if not thumbnail:
  149.                         thumbnail = info.get('thumb_url')
  151.                     download_formats = {}
  152.                     download_formats_list = blob.get('download_formats')
  153.                     if isinstance(download_formats_list, list):
  154.                         for f in blob['download_formats']:
  155.                             name, ext = f.get('name'), f.get('file_extension')
  156.                             if all(isinstance(x, compat_str) for x in (name, ext)):
  157.                                 download_formats[name] = ext.strip('.')
  159.                     for format_id, f in downloads.items():
  160.                         format_url = f.get('url')
  161.                         if not format_url:
  162.                             continue
  163.                         # Stat URL generation algorithm is reverse engineered from
  164.                         # download_*_bundle_*.js
  165.                         stat_url = update_url_query(
  166.                             format_url.replace('/download/', '/statdownload/'), {
  167.                                 '.rand': int(time.time() * 1000 * random.random()),
  168.                             })
  169.                         format_id = f.get('encoding_name') or format_id
  170.                         stat = self._download_json(
  171.                             stat_url, track_id, 'Downloading %s JSON' % format_id,
  172.                             transform_source=lambda s: s[s.index('{'):s.rindex('}') + 1],
  173.                             fatal=False)
  174.                         if not stat:
  175.                             continue
  176.                         retry_url = url_or_none(stat.get('retry_url'))
  177.                         if not retry_url:
  178.                             continue
  179.                         formats.append({
  180.                             'url': self._proto_relative_url(retry_url, 'http:'),
  181.                             'ext': download_formats.get(format_id),
  182.                             'format_id': format_id,
  183.                             'format_note': f.get('description'),
  184.                             'filesize': parse_filesize(f.get('size_mb')),
  185.                             'vcodec': 'none',
  186.                         })
  188.         self._sort_formats(formats)
  190.         title = '%s - %s' % (artist, track) if artist else track
  192.         if not duration:
  193.             duration = float_or_none(self._html_search_meta(
  194.                 'duration', webpage, default=None))
  196.         return {
  197.             'id': track_id,
  198.             'title': title,
  199.             'thumbnail': thumbnail,
  200.             'uploader': artist,
  201.             'timestamp': timestamp,
  202.             'release_timestamp': unified_timestamp(tralbum.get('album_release_date')),
  203.             'duration': duration,
  204.             'track': track,
  205.             'track_number': track_number,
  206.             'track_id': track_id,
  207.             'artist': artist,
  208.             'album': embed.get('album_title'),
  209.             'formats': formats,
  210.         }
  213. class BandcampAlbumIE(BandcampIE):
  214.     IE_NAME = 'Bandcamp:album'
  215.     _VALID_URL = r'https?://(?:(?P<subdomain>[^.]+)\.)?bandcamp\.com(?:/album/(?P<id>[^/?#&]+))?'
  217.     _TESTS = [{
  218.         'url': 'http://blazo.bandcamp.com/album/jazz-format-mixtape-vol-1',
  219.         'playlist': [
  220.             {
  221.                 'md5': '39bc1eded3476e927c724321ddf116cf',
  222.                 'info_dict': {
  223.                     'id': '1353101989',
  224.                     'ext': 'mp3',
  225.                     'title': 'Blazo - Intro',
  226.                     'timestamp': 1311756226,
  227.                     'upload_date': '20110727',
  228.                     'uploader': 'Blazo',
  229.                 }
  230.             },
  231.             {
  232.                 'md5': '1a2c32e2691474643e912cc6cd4bffaa',
  233.                 'info_dict': {
  234.                     'id': '38097443',
  235.                     'ext': 'mp3',
  236.                     'title': 'Blazo - Kero One - Keep It Alive (Blazo remix)',
  237.                     'timestamp': 1311757238,
  238.                     'upload_date': '20110727',
  239.                     'uploader': 'Blazo',
  240.                 }
  241.             },
  242.         ],
  243.         'info_dict': {
  244.             'title': 'Jazz Format Mixtape vol.1',
  245.             'id': 'jazz-format-mixtape-vol-1',
  246.             'uploader_id': 'blazo',
  247.         },
  248.         'params': {
  249.             'playlistend': 2
  250.         },
  251.         'skip': 'Bandcamp imposes download limits.'
  252.     }, {
  253.         'url': 'http://nightbringer.bandcamp.com/album/hierophany-of-the-open-grave',
  254.         'info_dict': {
  255.             'title': 'Hierophany of the Open Grave',
  256.             'uploader_id': 'nightbringer',
  257.             'id': 'hierophany-of-the-open-grave',
  258.         },
  259.         'playlist_mincount': 9,
  260.     }, {
  261.         'url': 'http://dotscale.bandcamp.com',
  262.         'info_dict': {
  263.             'title': 'Loom',
  264.             'id': 'dotscale',
  265.             'uploader_id': 'dotscale',
  266.         },
  267.         'playlist_mincount': 7,
  268.     }, {
  269.         # with escaped quote in title
  270.         'url': 'https://jstrecords.bandcamp.com/album/entropy-ep',
  271.         'info_dict': {
  272.             'title': '"Entropy" EP',
  273.             'uploader_id': 'jstrecords',
  274.             'id': 'entropy-ep',
  275.             'description': 'md5:0ff22959c943622972596062f2f366a5',
  276.         },
  277.         'playlist_mincount': 3,
  278.     }, {
  279.         # not all tracks have songs
  280.         'url': 'https://insulters.bandcamp.com/album/we-are-the-plague',
  281.         'info_dict': {
  282.             'id': 'we-are-the-plague',
  283.             'title': 'WE ARE THE PLAGUE',
  284.             'uploader_id': 'insulters',
  285.             'description': 'md5:b3cf845ee41b2b1141dc7bde9237255f',
  286.         },
  287.         'playlist_count': 2,
  288.     }]
  290.     @classmethod
  291.     def suitable(cls, url):
  292.         return (False
  293.                 if BandcampWeeklyIE.suitable(url) or BandcampIE.suitable(url)
  294.                 else super(BandcampAlbumIE, cls).suitable(url))
  296.     def _real_extract(self, url):
  297.         uploader_id, album_id = re.match(self._VALID_URL, url).groups()
  298.         playlist_id = album_id or uploader_id
  299.         webpage = self._download_webpage(url, playlist_id)
  300.         tralbum = self._extract_data_attr(webpage, playlist_id)
  301.         track_info = tralbum.get('trackinfo')
  302.         if not track_info:
  303.             raise ExtractorError('The page doesn\'t contain any tracks')
  304.         # Only tracks with duration info have songs
  305.         entries = [
  306.             self.url_result(
  307.                 urljoin(url, t['title_link']), BandcampIE.ie_key(),
  308.                 str_or_none(t.get('track_id') or t.get('id')), t.get('title'))
  309.             for t in track_info
  310.             if t.get('duration')]
  312.         current = tralbum.get('current') or {}
  314.         return {
  315.             '_type': 'playlist',
  316.             'uploader_id': uploader_id,
  317.             'id': playlist_id,
  318.             'title': current.get('title'),
  319.             'description': current.get('about'),
  320.             'entries': entries,
  321.         }
  324. class BandcampWeeklyIE(BandcampIE):
  325.     IE_NAME = 'Bandcamp:weekly'
  326.     _VALID_URL = r'https?://(?:www\.)?bandcamp\.com/?\?(?:.*?&)?show=(?P<id>\d+)'
  327.     _TESTS = [{
  328.         'url': 'https://bandcamp.com/?show=224',
  329.         'md5': 'b00df799c733cf7e0c567ed187dea0fd',
  330.         'info_dict': {
  331.             'id': '224',
  332.             'ext': 'opus',
  333.             'title': 'BC Weekly April 4th 2017 - Magic Moments',
  334.             'description': 'md5:5d48150916e8e02d030623a48512c874',
  335.             'duration': 5829.77,
  336.             'release_date': '20170404',
  337.             'series': 'Bandcamp Weekly',
  338.             'episode': 'Magic Moments',
  339.             'episode_id': '224',
  340.         },
  341.         'params': {
  342.             'format': 'opus-lo',
  343.         },
  344.     }, {
  345.         'url': 'https://bandcamp.com/?blah/blah@&show=228',
  346.         'only_matching': True
  347.     }]
  349.     def _real_extract(self, url):
  350.         show_id = self._match_id(url)
  351.         webpage = self._download_webpage(url, show_id)
  353.         blob = self._extract_data_attr(webpage, show_id, 'blob')
  355.         show = blob['bcw_data'][show_id]
  357.         formats = []
  358.         for format_id, format_url in show['audio_stream'].items():
  359.             if not url_or_none(format_url):
  360.                 continue
  361.             for known_ext in KNOWN_EXTENSIONS:
  362.                 if known_ext in format_id:
  363.                     ext = known_ext
  364.                     break
  365.             else:
  366.                 ext = None
  367.             formats.append({
  368.                 'format_id': format_id,
  369.                 'url': format_url,
  370.                 'ext': ext,
  371.                 'vcodec': 'none',
  372.             })
  373.         self._sort_formats(formats)
  375.         title = show.get('audio_title') or 'Bandcamp Weekly'
  376.         subtitle = show.get('subtitle')
  377.         if subtitle:
  378.             title += ' - %s' % subtitle
  380.         return {
  381.             'id': show_id,
  382.             'title': title,
  383.             'description': show.get('desc') or show.get('short_desc'),
  384.             'duration': float_or_none(show.get('audio_duration')),
  385.             'is_live': False,
  386.             'release_date': unified_strdate(show.get('published_date')),
  387.             'series': 'Bandcamp Weekly',
  388.             'episode': show.get('subtitle'),
  389.             'episode_id': show_id,
  390.             'formats': formats
  391.         }