commit: 69fcdb845b9744125161f514cb4166becbae2959
parent 6868d272e5c4a85ba4143bacbb7269dac099c55d
Author: Sergey M․ <dstftw@gmail.com>
Date: Sun, 8 Jul 2018 00:48:23 +0700
[frontendmasters] Fix issues and improve extraction (closes #3661, closes #16328)
Diffstat:
3 files changed, 266 insertions(+), 274 deletions(-)
diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
@@ -390,9 +390,10 @@ from .francetv import (
from .freesound import FreesoundIE
from .freespeech import FreespeechIE
from .freshlive import FreshLiveIE
-from .frontendmaster import (
- FrontEndMasterIE,
- FrontEndMasterCourseIE
+from .frontendmasters import (
+ FrontendMastersIE,
+ FrontendMastersLessonIE,
+ FrontendMastersCourseIE
)
from .funimation import FunimationIE
from .funk import (
diff --git a/youtube_dl/extractor/frontendmaster.py b/youtube_dl/extractor/frontendmaster.py
@@ -1,271 +0,0 @@
-# coding: utf-8
-from __future__ import unicode_literals
-
-import collections
-import re
-
-from .common import InfoExtractor
-from ..compat import (
- compat_urlparse)
-from ..utils import (
- ExtractorError,
- urlencode_postdata,
- qualities, unescapeHTML)
-
-
-class FrontEndMasterBaseIE(InfoExtractor):
- _API_BASE = 'https://api.frontendmasters.com/v1/kabuki/courses'
- _VIDEO_BASE = 'http://www.frontendmasters.com/courses'
- _CAPTIONS_BASE = 'https://api.frontendmasters.com/v1/kabuki/transcripts'
- _COOKIES_BASE = 'https://api.frontendmasters.com'
- _LOGIN_URL = 'https://frontendmasters.com/login/'
-
- _QUALITIES_PREFERENCE = ('low', 'medium', 'high')
- _QUALITIES = {
- 'low': {'width': 480, 'height': 360},
- 'medium': {'width': 1280, 'height': 720},
- 'high': {'width': 1920, 'height': 1080}
- }
-
- AllowedQuality = collections.namedtuple('AllowedQuality',
- ['ext', 'qualities'])
- _ALLOWED_QUALITIES = [
- AllowedQuality('webm', ['low', 'medium', 'high']),
- AllowedQuality('mp4', ['low', 'medium', 'high'])
- ]
-
- def _real_initialize(self):
- self._login()
-
- def _login(self):
- (username, password) = self._get_login_info()
- if username is None:
- return
-
- login_page = self._download_webpage(
- self._LOGIN_URL, None, 'Downloading login page')
-
- login_form = self._hidden_inputs(login_page)
-
- login_form.update({
- 'username': username,
- 'password': password
- })
-
- post_url = self._search_regex(
- r'<form[^>]+action=(["\'])(?P<url>.+?)\1', login_page,
- 'post_url', default=self._LOGIN_URL, group='url')
-
- if not post_url.startswith('http'):
- post_url = compat_urlparse.urljoin(self._LOGIN_URL, post_url)
-
- response = self._download_webpage(
- post_url, None, 'Logging in',
- data=urlencode_postdata(login_form),
- headers={'Content-Type': 'application/x-www-form-urlencoded'}
- )
-
- error = self._search_regex(
- r'<div[^>]+class=["\']Message MessageAlert["\'][^>]*>'
- r'([^<]+)'
- r'</div>',
- response, 'error message', default=None)
-
- if error:
- raise ExtractorError('Unable to login: %s' % unescapeHTML(error),
- expected=True)
-
- def _download_course(self, course_id, url):
- response = self._download_json(
- '%s/%s' % (self._API_BASE, course_id), course_id,
- 'Downloading course JSON',
- headers={
- 'Content-Type': 'application/json;charset=utf-8',
- 'Referer': url,
- })
- return response
-
- @staticmethod
- def _pair_section_video_element(lesson_elements):
- sections = {}
- current_section = None
- current_section_number = 0
- for elem in lesson_elements:
- if not isinstance(elem, int):
- elem_name = elem
- if not isinstance(elem_name, str):
- # convert unicode to str
- elem_name = elem.encode('utf-8')
- (current_section, current_section_number) = \
- (elem_name, current_section_number + 1)
- else:
- if current_section:
- sections[elem] = (current_section, current_section_number)
-
- return sections
-
-
-class FrontEndMasterIE(FrontEndMasterBaseIE):
- IE_NAME = 'frontend-masters'
- _VALID_URL = r'https?://(?:www\.)?frontendmasters\.com/courses/' \
- r'(?P<courseid>[a-z\-]+)/' \
- r'(?P<id>[a-z\-]+)'
-
- _NETRC_MACHINE = 'frontendmasters'
-
- _TEST = {
- 'url': 'https://frontendmasters.com/courses/web-development/tools',
- 'md5': '7f161159710d6b7016a4f4af6fcb05e2',
- 'info_dict': {
- 'id': 'tools',
- 'title': 'Tools',
- 'display_id': 'tools',
- 'description': 'md5:82c1ea6472e88ed5acd1829fe992e4f7',
- 'ext': 'mp4'
- },
- 'skip': 'Requires FrontendMasters account credentials',
- }
-
- def _get_subtitles(self, video_hash, video_id):
- captions = self._download_webpage(
- '%s/%s.vtt' % (self._CAPTIONS_BASE, video_hash), video_id,
- fatal=False)
- if captions:
- return {
- 'en': [{
- 'ext': 'vtt',
- 'data': captions
- }]
- }
-
- def _real_extract(self, url):
- mobj = re.match(self._VALID_URL, url)
- video_id = mobj.group('id')
- course_id = mobj.group('courseid')
-
- course_json_content = self._download_course(course_id=course_id,
- url=url)
-
- # Necessary to get mandatory informations like title and video_url
- lesson_index = course_json_content.get('lessonSlugs').index(video_id)
- lesson_hash = course_json_content.get('lessonHashes')[lesson_index]
- lesson_data = course_json_content.get('lessonData')[lesson_hash]
- # This is necessary to get the link for the video
- lesson_source_base = lesson_data['sourceBase']
-
- lesson_title = lesson_data['title']
-
- # Some optional fields
- lesson_description = lesson_data.get('description')
- lesson_index = lesson_data.get('index')
- lesson_slug = lesson_data.get('slug')
- lesson_thumbnail_url = lesson_data.get('thumbnail')
- lesson_section_elements = course_json_content.get('lessonElements')
-
- try:
- course_sections_pairing = self._pair_section_video_element(
- lesson_section_elements)
-
- lesson_section = \
- course_sections_pairing.get(lesson_index)[0]
-
- lesson_section_number = \
- course_sections_pairing.get(lesson_index)[1]
- except Exception:
- lesson_section = None
- lesson_section_number = None
-
- video_request_url = '%s/source'
- video_request_headers = {
- 'origin': 'https://frontendmasters.com',
- 'referer': lesson_source_base,
- }
-
- quality_key = qualities(self._QUALITIES_PREFERENCE)
-
- formats = []
- for ext, qualities_ in self._ALLOWED_QUALITIES:
- for quality in qualities_:
- f = self._QUALITIES[quality].copy()
- video_request_params = {
- 'r': f['height'],
- 'f': ext
- }
- video_response = self._download_json(
- video_request_url % lesson_source_base, video_id,
- query=video_request_params, headers=video_request_headers)
-
- video_url = video_response.get('url')
- clip_f = f.copy()
- clip_f.update({
- 'url': video_url,
- 'ext': ext,
- 'format_id': '%s-%s' % (ext, quality),
- 'quality': quality_key(quality),
- 'height': f['height']
- })
- formats.append(clip_f)
-
- self._sort_formats(formats)
-
- subtitles = self.extract_subtitles(lesson_hash, video_id)
-
- return {
- 'id': video_id,
- 'display_id': lesson_slug,
- 'title': lesson_title,
- 'description': lesson_description,
- 'chapter': lesson_section,
- 'chapter_number': lesson_section_number,
- 'thumbnail': lesson_thumbnail_url,
- 'formats': formats,
- 'subtitles': subtitles
- }
-
-
-class FrontEndMasterCourseIE(FrontEndMasterBaseIE):
- IE_NAME = 'frontend-masters:course'
- _VALID_URL = r'https?://(?:www\.)?frontendmasters\.com/courses/(?P<courseid>[a-z\-]+)/?$'
-
- _NETRC_MACHINE = 'frontendmasters'
-
- _TEST = {
- 'url': 'https://frontendmasters.com/courses/javascript-basics/',
- 'info_dict': {
- 'id': 'javascript-basics',
- 'title': 'Introduction to JavaScript Programming',
- 'description': 'md5:269412fbb76d86954761599ad8e4cbc9'
- },
- 'playlist_count': 19,
- 'skip': 'Requires FrontendMasters account credentials'
- }
-
- @classmethod
- def suitable(cls, url):
- return False if FrontEndMasterIE.suitable(url) else super(FrontEndMasterBaseIE, cls).suitable(url)
-
- def _real_extract(self, url):
- mobj = re.match(self._VALID_URL, url)
- course_id = mobj.group('courseid')
- course_json_content = self._download_course(course_id=course_id,
- url=url)
-
- title = course_json_content.get('title')
- description = course_json_content.get('description')
- course_display_id = course_json_content.get('slug')
-
- videos_data = course_json_content.get('lessonData').values()
- videos_data = sorted(videos_data, key=lambda video: video.get('index'))
-
- entries = []
- for video in videos_data:
- video_slug = video.get('slug')
- clip_url = '%s/%s/%s' % (
- self._VIDEO_BASE, course_display_id, video_slug)
- entries.append({
- '_type': 'url_transparent',
- 'url': clip_url,
- 'ie_key': FrontEndMasterIE.ie_key()
- })
-
- return self.playlist_result(entries, course_id, title, description)
diff --git a/youtube_dl/extractor/frontendmasters.py b/youtube_dl/extractor/frontendmasters.py
@@ -0,0 +1,262 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..compat import (
+ compat_str,
+ compat_urlparse,
+)
+from ..utils import (
+ ExtractorError,
+ parse_duration,
+ urlencode_postdata,
+)
+
+
+class FrontendMastersBaseIE(InfoExtractor):
+ _API_BASE = 'https://api.frontendmasters.com/v1/kabuki'
+ _LOGIN_URL = 'https://frontendmasters.com/login/'
+
+ _NETRC_MACHINE = 'frontendmasters'
+
+ _QUALITIES = {
+ 'low': {'width': 480, 'height': 360},
+ 'mid': {'width': 1280, 'height': 720},
+ 'high': {'width': 1920, 'height': 1080}
+ }
+
+ def _real_initialize(self):
+ self._login()
+
+ def _login(self):
+ (username, password) = self._get_login_info()
+ if username is None:
+ return
+
+ login_page = self._download_webpage(
+ self._LOGIN_URL, None, 'Downloading login page')
+
+ login_form = self._hidden_inputs(login_page)
+
+ login_form.update({
+ 'username': username,
+ 'password': password
+ })
+
+ post_url = self._search_regex(
+ r'<form[^>]+action=(["\'])(?P<url>.+?)\1', login_page,
+ 'post_url', default=self._LOGIN_URL, group='url')
+
+ if not post_url.startswith('http'):
+ post_url = compat_urlparse.urljoin(self._LOGIN_URL, post_url)
+
+ response = self._download_webpage(
+ post_url, None, 'Logging in', data=urlencode_postdata(login_form),
+ headers={'Content-Type': 'application/x-www-form-urlencoded'})
+
+ # Successful login
+ if any(p in response for p in (
+ 'wp-login.php?action=logout', '>Logout')):
+ return
+
+ error = self._html_search_regex(
+ r'class=(["\'])(?:(?!\1).)*\bMessageAlert\b(?:(?!\1).)*\1[^>]*>(?P<error>[^<]+)<',
+ response, 'error message', default=None, group='error')
+ if error:
+ raise ExtractorError('Unable to login: %s' % error, expected=True)
+ raise ExtractorError('Unable to log in')
+
+
+class FrontendMastersPageBaseIE(FrontendMastersBaseIE):
+ def _download_course(self, course_name, url):
+ return self._download_json(
+ '%s/courses/%s' % (self._API_BASE, course_name), course_name,
+ 'Downloading course JSON', headers={'Referer': url})
+
+ @staticmethod
+ def _extract_chapters(course):
+ chapters = []
+ lesson_elements = course.get('lessonElements')
+ if isinstance(lesson_elements, list):
+ chapters = [e for e in lesson_elements if isinstance(e, compat_str)]
+ return chapters
+
+ @staticmethod
+ def _extract_lesson(chapters, lesson_id, lesson):
+ title = lesson.get('title') or lesson_id
+ display_id = lesson.get('slug')
+ description = lesson.get('description')
+ thumbnail = lesson.get('thumbnail')
+
+ chapter_number = None
+ index = lesson.get('index')
+ element_index = lesson.get('elementIndex')
+ if (isinstance(index, int) and isinstance(element_index, int) and
+ index < element_index):
+ chapter_number = element_index - index
+ chapter = (chapters[chapter_number - 1]
+ if chapter_number - 1 < len(chapters) else None)
+
+ duration = None
+ timestamp = lesson.get('timestamp')
+ if isinstance(timestamp, compat_str):
+ mobj = re.search(
+ r'(?P<start>\d{1,2}:\d{1,2}:\d{1,2})\s*-(?P<end>\s*\d{1,2}:\d{1,2}:\d{1,2})',
+ timestamp)
+ if mobj:
+ duration = parse_duration(mobj.group('end')) - parse_duration(
+ mobj.group('start'))
+
+ return {
+ '_type': 'url_transparent',
+ 'url': 'frontendmasters:%s' % lesson_id,
+ 'ie_key': FrontendMastersIE.ie_key(),
+ 'id': lesson_id,
+ 'display_id': display_id,
+ 'title': title,
+ 'description': description,
+ 'thumbnail': thumbnail,
+ 'duration': duration,
+ 'chapter': chapter,
+ 'chapter_number': chapter_number,
+ }
+
+
+class FrontendMastersIE(FrontendMastersBaseIE):
+ _VALID_URL = r'(?:frontendmasters:|https?://api\.frontendmasters\.com/v\d+/kabuki/video/)(?P<id>[^/]+)'
+ _TESTS = [{
+ 'url': 'https://api.frontendmasters.com/v1/kabuki/video/a2qogef6ba',
+ 'md5': '7f161159710d6b7016a4f4af6fcb05e2',
+ 'info_dict': {
+ 'id': 'a2qogef6ba',
+ 'ext': 'mp4',
+ 'title': 'a2qogef6ba',
+ },
+ 'skip': 'Requires FrontendMasters account credentials',
+ }, {
+ 'url': 'frontendmasters:a2qogef6ba',
+ 'only_matching': True,
+ }]
+
+ def _real_extract(self, url):
+ lesson_id = self._match_id(url)
+
+ source_url = '%s/video/%s/source' % (self._API_BASE, lesson_id)
+
+ formats = []
+ for ext in ('webm', 'mp4'):
+ for quality in ('low', 'mid', 'high'):
+ resolution = self._QUALITIES[quality].copy()
+ format_id = '%s-%s' % (ext, quality)
+ format_url = self._download_json(
+ source_url, lesson_id,
+ 'Downloading %s source JSON' % format_id, query={
+ 'f': ext,
+ 'r': resolution['height'],
+ }, headers={
+ 'Referer': url,
+ }, fatal=False)['url']
+
+ if not format_url:
+ continue
+
+ f = resolution.copy()
+ f.update({
+ 'url': format_url,
+ 'ext': ext,
+ 'format_id': format_id,
+ })
+ formats.append(f)
+ self._sort_formats(formats)
+
+ subtitles = {
+ 'en': [{
+ 'url': '%s/transcripts/%s.vtt' % (self._API_BASE, lesson_id),
+ }]
+ }
+
+ return {
+ 'id': lesson_id,
+ 'title': lesson_id,
+ 'formats': formats,
+ 'subtitles': subtitles
+ }
+
+
+class FrontendMastersLessonIE(FrontendMastersPageBaseIE):
+ _VALID_URL = r'https?://(?:www\.)?frontendmasters\.com/courses/(?P<course_name>[^/]+)/(?P<lesson_name>[^/]+)'
+ _TEST = {
+ 'url': 'https://frontendmasters.com/courses/web-development/tools',
+ 'info_dict': {
+ 'id': 'a2qogef6ba',
+ 'display_id': 'tools',
+ 'ext': 'mp4',
+ 'title': 'Tools',
+ 'description': 'md5:82c1ea6472e88ed5acd1829fe992e4f7',
+ 'thumbnail': r're:^https?://.*\.jpg$',
+ 'chapter': 'Introduction',
+ 'chapter_number': 1,
+ },
+ 'params': {
+ 'skip_download': True,
+ },
+ 'skip': 'Requires FrontendMasters account credentials',
+ }
+
+ def _real_extract(self, url):
+ mobj = re.match(self._VALID_URL, url)
+ course_name, lesson_name = mobj.group('course_name', 'lesson_name')
+
+ course = self._download_course(course_name, url)
+
+ lesson_id, lesson = next(
+ (video_id, data)
+ for video_id, data in course['lessonData'].items()
+ if data.get('slug') == lesson_name)
+
+ chapters = self._extract_chapters(course)
+ return self._extract_lesson(chapters, lesson_id, lesson)
+
+
+class FrontendMastersCourseIE(FrontendMastersPageBaseIE):
+ _VALID_URL = r'https?://(?:www\.)?frontendmasters\.com/courses/(?P<id>[^/]+)'
+ _TEST = {
+ 'url': 'https://frontendmasters.com/courses/web-development/',
+ 'info_dict': {
+ 'id': 'web-development',
+ 'title': 'Introduction to Web Development',
+ 'description': 'md5:9317e6e842098bf725d62360e52d49a6',
+ },
+ 'playlist_count': 81,
+ 'skip': 'Requires FrontendMasters account credentials',
+ }
+
+ @classmethod
+ def suitable(cls, url):
+ return False if FrontendMastersLessonIE.suitable(url) else super(
+ FrontendMastersBaseIE, cls).suitable(url)
+
+ def _real_extract(self, url):
+ course_name = self._match_id(url)
+
+ course = self._download_course(course_name, url)
+
+ chapters = self._extract_chapters(course)
+
+ lessons = sorted(
+ course['lessonData'].values(), key=lambda data: data['index'])
+
+ entries = []
+ for lesson in lessons:
+ lesson_name = lesson.get('slug')
+ if not lesson_name:
+ continue
+ lesson_id = lesson.get('hash') or lesson.get('statsId')
+ entries.append(self._extract_lesson(chapters, lesson_id, lesson))
+
+ title = course.get('title')
+ description = course.get('description')
+
+ return self.playlist_result(entries, course_name, title, description)
