logo

youtube-dl

[mirror] Download/Watch videos from video hostersgit clone https://hacktivis.me/git/mirror/youtube-dl.git

safari.py (9746B)

    

  1. # coding: utf-8
  2. from __future__ import unicode_literals
  4. import json
  5. import re
  7. from .common import InfoExtractor
  9. from ..compat import (
  10.     compat_parse_qs,
  11.     compat_urlparse,
  12. )
  13. from ..utils import (
  14.     ExtractorError,
  15.     update_url_query,
  16. )
  19. class SafariBaseIE(InfoExtractor):
  20.     _LOGIN_URL = 'https://learning.oreilly.com/accounts/login/'
  21.     _NETRC_MACHINE = 'safari'
  23.     _API_BASE = 'https://learning.oreilly.com/api/v1'
  24.     _API_FORMAT = 'json'
  26.     LOGGED_IN = False
  28.     def _real_initialize(self):
  29.         self._login()
  31.     def _login(self):
  32.         username, password = self._get_login_info()
  33.         if username is None:
  34.             return
  36.         _, urlh = self._download_webpage_handle(
  37.             'https://learning.oreilly.com/accounts/login-check/', None,
  38.             'Downloading login page')
  40.         def is_logged(urlh):
  41.             return 'learning.oreilly.com/home/' in urlh.geturl()
  43.         if is_logged(urlh):
  44.             self.LOGGED_IN = True
  45.             return
  47.         redirect_url = urlh.geturl()
  48.         parsed_url = compat_urlparse.urlparse(redirect_url)
  49.         qs = compat_parse_qs(parsed_url.query)
  50.         next_uri = compat_urlparse.urljoin(
  51.             'https://api.oreilly.com', qs['next'][0])
  53.         auth, urlh = self._download_json_handle(
  54.             'https://www.oreilly.com/member/auth/login/', None, 'Logging in',
  55.             data=json.dumps({
  56.                 'email': username,
  57.                 'password': password,
  58.                 'redirect_uri': next_uri,
  59.             }).encode(), headers={
  60.                 'Content-Type': 'application/json',
  61.                 'Referer': redirect_url,
  62.             }, expected_status=400)
  64.         credentials = auth.get('credentials')
  65.         if (not auth.get('logged_in') and not auth.get('redirect_uri')
  66.                 and credentials):
  67.             raise ExtractorError(
  68.                 'Unable to login: %s' % credentials, expected=True)
  70.         # oreilly serves two same instances of the following cookies
  71.         # in Set-Cookie header and expects first one to be actually set
  72.         for cookie in ('groot_sessionid', 'orm-jwt', 'orm-rt'):
  73.             self._apply_first_set_cookie_header(urlh, cookie)
  75.         _, urlh = self._download_webpage_handle(
  76.             auth.get('redirect_uri') or next_uri, None, 'Completing login',)
  78.         if is_logged(urlh):
  79.             self.LOGGED_IN = True
  80.             return
  82.         raise ExtractorError('Unable to log in')
  85. class SafariIE(SafariBaseIE):
  86.     IE_NAME = 'safari'
  87.     IE_DESC = 'safaribooksonline.com online video'
  88.     _VALID_URL = r'''(?x)
  89.                         https?://
  90.                             (?:www\.)?(?:safaribooksonline|(?:learning\.)?oreilly)\.com/
  91.                             (?:
  92.                                 library/view/[^/]+/(?P<course_id>[^/]+)/(?P<part>[^/?\#&]+)\.html|
  93.                                 videos/[^/]+/[^/]+/(?P<reference_id>[^-]+-[^/?\#&]+)
  94.                             )
  95.                     '''
  97.     _TESTS = [{
  98.         'url': 'https://www.safaribooksonline.com/library/view/hadoop-fundamentals-livelessons/9780133392838/part00.html',
  99.         'md5': 'dcc5a425e79f2564148652616af1f2a3',
  100.         'info_dict': {
  101.             'id': '0_qbqx90ic',
  102.             'ext': 'mp4',
  103.             'title': 'Introduction to Hadoop Fundamentals LiveLessons',
  104.             'timestamp': 1437758058,
  105.             'upload_date': '20150724',
  106.             'uploader_id': 'stork',
  107.         },
  108.     }, {
  109.         # non-digits in course id
  110.         'url': 'https://www.safaribooksonline.com/library/view/create-a-nodejs/100000006A0210/part00.html',
  111.         'only_matching': True,
  112.     }, {
  113.         'url': 'https://www.safaribooksonline.com/library/view/learning-path-red/9780134664057/RHCE_Introduction.html',
  114.         'only_matching': True,
  115.     }, {
  116.         'url': 'https://www.safaribooksonline.com/videos/python-programming-language/9780134217314/9780134217314-PYMC_13_00',
  117.         'only_matching': True,
  118.     }, {
  119.         'url': 'https://learning.oreilly.com/videos/hadoop-fundamentals-livelessons/9780133392838/9780133392838-00_SeriesIntro',
  120.         'only_matching': True,
  121.     }, {
  122.         'url': 'https://www.oreilly.com/library/view/hadoop-fundamentals-livelessons/9780133392838/00_SeriesIntro.html',
  123.         'only_matching': True,
  124.     }]
  126.     _PARTNER_ID = '1926081'
  127.     _UICONF_ID = '29375172'
  129.     def _real_extract(self, url):
  130.         mobj = re.match(self._VALID_URL, url)
  132.         reference_id = mobj.group('reference_id')
  133.         if reference_id:
  134.             video_id = reference_id
  135.             partner_id = self._PARTNER_ID
  136.             ui_id = self._UICONF_ID
  137.         else:
  138.             video_id = '%s-%s' % (mobj.group('course_id'), mobj.group('part'))
  140.             webpage, urlh = self._download_webpage_handle(url, video_id)
  142.             mobj = re.match(self._VALID_URL, urlh.geturl())
  143.             reference_id = mobj.group('reference_id')
  144.             if not reference_id:
  145.                 reference_id = self._search_regex(
  146.                     r'data-reference-id=(["\'])(?P<id>(?:(?!\1).)+)\1',
  147.                     webpage, 'kaltura reference id', group='id')
  148.             partner_id = self._search_regex(
  149.                 r'data-partner-id=(["\'])(?P<id>(?:(?!\1).)+)\1',
  150.                 webpage, 'kaltura widget id', default=self._PARTNER_ID,
  151.                 group='id')
  152.             ui_id = self._search_regex(
  153.                 r'data-ui-id=(["\'])(?P<id>(?:(?!\1).)+)\1',
  154.                 webpage, 'kaltura uiconf id', default=self._UICONF_ID,
  155.                 group='id')
  157.         query = {
  158.             'wid': '_%s' % partner_id,
  159.             'uiconf_id': ui_id,
  160.             'flashvars[referenceId]': reference_id,
  161.         }
  163.         if self.LOGGED_IN:
  164.             kaltura_session = self._download_json(
  165.                 '%s/player/kaltura_session/?reference_id=%s' % (self._API_BASE, reference_id),
  166.                 video_id, 'Downloading kaltura session JSON',
  167.                 'Unable to download kaltura session JSON', fatal=False,
  168.                 headers={'Accept': 'application/json'})
  169.             if kaltura_session:
  170.                 session = kaltura_session.get('session')
  171.                 if session:
  172.                     query['flashvars[ks]'] = session
  174.         return self.url_result(update_url_query(
  175.             'https://cdnapisec.kaltura.com/html5/html5lib/v2.37.1/mwEmbedFrame.php', query),
  176.             'Kaltura')
  179. class SafariApiIE(SafariBaseIE):
  180.     IE_NAME = 'safari:api'
  181.     _VALID_URL = r'https?://(?:www\.)?(?:safaribooksonline|(?:learning\.)?oreilly)\.com/api/v1/book/(?P<course_id>[^/]+)/chapter(?:-content)?/(?P<part>[^/?#&]+)\.html'
  183.     _TESTS = [{
  184.         'url': 'https://www.safaribooksonline.com/api/v1/book/9780133392838/chapter/part00.html',
  185.         'only_matching': True,
  186.     }, {
  187.         'url': 'https://www.safaribooksonline.com/api/v1/book/9780134664057/chapter/RHCE_Introduction.html',
  188.         'only_matching': True,
  189.     }]
  191.     def _real_extract(self, url):
  192.         mobj = re.match(self._VALID_URL, url)
  193.         part = self._download_json(
  194.             url, '%s/%s' % (mobj.group('course_id'), mobj.group('part')),
  195.             'Downloading part JSON')
  196.         return self.url_result(part['web_url'], SafariIE.ie_key())
  199. class SafariCourseIE(SafariBaseIE):
  200.     IE_NAME = 'safari:course'
  201.     IE_DESC = 'safaribooksonline.com online courses'
  203.     _VALID_URL = r'''(?x)
  204.                     https?://
  205.                         (?:
  206.                             (?:www\.)?(?:safaribooksonline|(?:learning\.)?oreilly)\.com/
  207.                             (?:
  208.                                 library/view/[^/]+|
  209.                                 api/v1/book|
  210.                                 videos/[^/]+
  211.                             )|
  212.                             techbus\.safaribooksonline\.com
  213.                         )
  214.                         /(?P<id>[^/]+)
  215.                     '''
  217.     _TESTS = [{
  218.         'url': 'https://www.safaribooksonline.com/library/view/hadoop-fundamentals-livelessons/9780133392838/',
  219.         'info_dict': {
  220.             'id': '9780133392838',
  221.             'title': 'Hadoop Fundamentals LiveLessons',
  222.         },
  223.         'playlist_count': 22,
  224.         'skip': 'Requires safaribooksonline account credentials',
  225.     }, {
  226.         'url': 'https://www.safaribooksonline.com/api/v1/book/9781449396459/?override_format=json',
  227.         'only_matching': True,
  228.     }, {
  229.         'url': 'http://techbus.safaribooksonline.com/9780134426365',
  230.         'only_matching': True,
  231.     }, {
  232.         'url': 'https://www.safaribooksonline.com/videos/python-programming-language/9780134217314',
  233.         'only_matching': True,
  234.     }, {
  235.         'url': 'https://learning.oreilly.com/videos/hadoop-fundamentals-livelessons/9780133392838',
  236.         'only_matching': True,
  237.     }, {
  238.         'url': 'https://www.oreilly.com/library/view/hadoop-fundamentals-livelessons/9780133392838/',
  239.         'only_matching': True,
  240.     }]
  242.     @classmethod
  243.     def suitable(cls, url):
  244.         return (False if SafariIE.suitable(url) or SafariApiIE.suitable(url)
  245.                 else super(SafariCourseIE, cls).suitable(url))
  247.     def _real_extract(self, url):
  248.         course_id = self._match_id(url)
  250.         course_json = self._download_json(
  251.             '%s/book/%s/?override_format=%s' % (self._API_BASE, course_id, self._API_FORMAT),
  252.             course_id, 'Downloading course JSON')
  254.         if 'chapters' not in course_json:
  255.             raise ExtractorError(
  256.                 'No chapters found for course %s' % course_id, expected=True)
  258.         entries = [
  259.             self.url_result(chapter, SafariApiIE.ie_key())
  260.             for chapter in course_json['chapters']]
  262.         course_title = course_json['title']
  264.         return self.playlist_result(entries, course_id, course_title)