youtube-dl

commit: 22f5f5c6fcd2d7f0c9f1ff3019fe0b957b771f44
parent 49bd993fd9adbcf6b5c11a7ec11c2b4a552e49c5
Author: Ales Jirasek <schunkac@gmail.com>
Date:   Wed, 10 Oct 2018 23:47:21 +0200

[malltv] Add extractor (closes #18058)

Diffstat:
M
test/test_InfoExtractor.py
2
++
M
youtube_dl/extractor/common.py
2
+-
M
youtube_dl/extractor/extractors.py
1
+
A
youtube_dl/extractor/malltv.py
58
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
M
youtube_dl/utils.py
2
+-

5 files changed, 63 insertions(+), 2 deletions(-)
diff --git a/test/test_InfoExtractor.py b/test/test_InfoExtractor.py
@@ -61,6 +61,7 @@ class TestInfoExtractor(unittest.TestCase):
             <meta content='Foo' property=og:foobar>
             <meta name="og:test1" content='foo > < bar'/>
             <meta name="og:test2" content="foo >//< bar"/>
+            <meta property=og-test3 content='Ill-formatted opengraph'/>
             '''
         self.assertEqual(ie._og_search_title(html), 'Foo')
         self.assertEqual(ie._og_search_description(html), 'Some video\'s description ')
@@ -69,6 +70,7 @@ class TestInfoExtractor(unittest.TestCase):
         self.assertEqual(ie._og_search_property('foobar', html), 'Foo')
         self.assertEqual(ie._og_search_property('test1', html), 'foo > < bar')
         self.assertEqual(ie._og_search_property('test2', html), 'foo >//< bar')
+        self.assertEqual(ie._og_search_property('test3', html), 'Ill-formatted opengraph')
         self.assertEqual(ie._og_search_property(('test0', 'test1'), html), 'foo > < bar')
         self.assertRaises(RegexNotFoundError, ie._og_search_property, 'test0', html, None, fatal=True)
         self.assertRaises(RegexNotFoundError, ie._og_search_property, ('test0', 'test00'), html, None, fatal=True)
diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
@@ -1058,7 +1058,7 @@ class InfoExtractor(object):
     @staticmethod
     def _og_regexes(prop):
         content_re = r'content=(?:"([^"]+?)"|\'([^\']+?)\'|\s*([^\s"\'=<>`]+?))'
-        property_re = (r'(?:name|property)=(?:\'og:%(prop)s\'|"og:%(prop)s"|\s*og:%(prop)s\b)'
+        property_re = (r'(?:name|property)=(?:\'og[:-]%(prop)s\'|"og[:-]%(prop)s"|\s*og[:-]%(prop)s\b)'
                        % {'prop': re.escape(prop)})
         template = r'<meta[^>]+?%s[^>]+?%s'
         return [
diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
@@ -619,6 +619,7 @@ from .mailru import (
     MailRuMusicSearchIE,
 )
 from .makertv import MakerTVIE
+from .malltv import MallTVIE
 from .mangomolo import (
     MangomoloVideoIE,
     MangomoloLiveIE,
diff --git a/youtube_dl/extractor/malltv.py b/youtube_dl/extractor/malltv.py
@@ -0,0 +1,58 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+from .common import InfoExtractor
+from ..utils import parse_duration, merge_dicts
+
+
+class MallTVIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?mall\.tv/(?:.+/)?(?P<id>.+)(?:\?.*$|$)'
+    _TESTS = [
+        {
+            'url': 'https://www.mall.tv/18-miliard-pro-neziskovky-opravdu-jsou-sportovci-nebo-clovek-v-tisni-pijavice',
+            'md5': '9ced0de056534410837077e23bfba796',
+            'info_dict': {
+                'id': 't0zzt0',
+                'ext': 'mp4',
+                'title': '18 miliard pro neziskovky. Opravdu jsou sportovci nebo Člověk v tísni pijavice?',
+                'description': 'Pokud někdo hospodaří s penězmi daňových poplatníků, pak logicky chceme vědět, jak s nimi nakládá. Objem dotací pro neziskovky roste, ale opravdu jsou tyto organizace „pijavice", jak o nich hovoří And',
+                'upload_date': '20181007',
+                'timestamp': 1538870400
+                }
+        },
+        {
+            'url': 'https://www.mall.tv/kdo-to-plati/18-miliard-pro-neziskovky-opravdu-jsou-sportovci-nebo-clovek-v-tisni-pijavice',
+            'md5': '9ced0de056534410837077e23bfba796',
+            'only_matching': 1,
+            'info_dict': {
+                'id': 't0zzt0',
+                'ext': 'mp4',
+                'title': '18 miliard pro neziskovky. Opravdu jsou sportovci nebo Člověk v tísni pijavice?',
+                'description': 'Pokud někdo hospodaří s penězmi daňových poplatníků, pak logicky chceme vědět, jak s nimi nakládá. Objem dotací pro neziskovky roste, ale opravdu jsou tyto organizace „pijavice", jak o nich hovoří And',
+                'upload_date': '20181007',
+                'timestamp': 1538870400
+                }
+        }
+    ]
+
+    def _real_extract(self, url):
+        display_id = self._match_id(url)
+        webpage = self._download_webpage(url, display_id)
+        src_id_regex = r'(?P<src><source src=([\"\'])?.+?/(?P<id>\w{6,}?)/index)(?P<after>\1?[^>]*?>)'
+        video_id = self._search_regex(src_id_regex, webpage, 'ID',
+                                           group='id')
+        info = self._search_json_ld(webpage, video_id, default={})
+        html = re.sub(src_id_regex, r'\g<src>.m3u8\g<after>', webpage)
+        media = self._parse_html5_media_entries(url, html, video_id)
+        thumbnail = info.get('thumbnail', self._og_search_thumbnail(webpage))
+        duration = parse_duration(info.get('duration'))
+        result = {
+            'id': video_id,
+            'title': info.get('title', self._og_search_title(webpage)),
+            'description': self._og_search_description(webpage)
+        }
+        result.update({'thumbnail': thumbnail})
+        result.update({'duration': duration})
+
+        return merge_dicts(media[0], info, result)
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
@@ -184,7 +184,7 @@ DATE_FORMATS_MONTH_FIRST.extend([
 ])
 
 PACKED_CODES_RE = r"}\('(.+)',(\d+),(\d+),'([^']+)'\.split\('\|'\)"
-JSON_LD_RE = r'(?is)<script[^>]+type=(["\'])application/ld\+json\1[^>]*>(?P<json_ld>.+?)</script>'
+JSON_LD_RE = r'(?is)<script[^>]+type=(["\']?)application/ld\+json\1[^>]*>(?P<json_ld>.+?)</script>'
 
 
 def preferredencoding():