Merge 44f73880fe into 0153b387e5

* [VidLii] Add HD support  (yt-dlp backport-ish)

* Also fix a bug with the view count

---------

Co-authored-by: dirkf <fieldhouse@gmx.net>

test/helper.py

@@ -5,9 +5,9 @@ import hashlib
 import json
 import os.path
 import re
-import types
 import ssl
 import sys
+import types
 import unittest
 
 import youtube_dl.extractor
@@ -181,18 +181,18 @@ def expect_value(self, got, expected, field):
             op, _, expected_num = expected.partition(':')
             expected_num = int(expected_num)
             if op == 'mincount':
-                assert_func = assertGreaterEqual
+                assert_func = self.assertGreaterEqual
                 msg_tmpl = 'Expected %d items in field %s, but only got %d'
             elif op == 'maxcount':
-                assert_func = assertLessEqual
+                assert_func = self.assertLessEqual
                 msg_tmpl = 'Expected maximum %d items in field %s, but got %d'
             elif op == 'count':
-                assert_func = assertEqual
+                assert_func = self.assertEqual
                 msg_tmpl = 'Expected exactly %d items in field %s, but got %d'
             else:
                 assert False
             assert_func(
-                self, len(got), expected_num,
+                len(got), expected_num,
                 msg_tmpl % (expected_num, field, len(got)))
             return
         self.assertEqual(
@@ -262,27 +262,6 @@ def assertRegexpMatches(self, text, regexp, msg=None):
             self.assertTrue(m, msg)
 
 
-def assertGreaterEqual(self, got, expected, msg=None):
-    if not (got >= expected):
-        if msg is None:
-            msg = '%r not greater than or equal to %r' % (got, expected)
-        self.assertTrue(got >= expected, msg)
-
-
-def assertLessEqual(self, got, expected, msg=None):
-    if not (got <= expected):
-        if msg is None:
-            msg = '%r not less than or equal to %r' % (got, expected)
-        self.assertTrue(got <= expected, msg)
-
-
-def assertEqual(self, got, expected, msg=None):
-    if not (got == expected):
-        if msg is None:
-            msg = '%r not equal to %r' % (got, expected)
-        self.assertTrue(got == expected, msg)
-
-
 def expect_warnings(ydl, warnings_re):
     real_warning = ydl.report_warning
 

test/test_download.py

@@ -9,8 +9,6 @@ import unittest
 sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
 from test.helper import (
-    assertGreaterEqual,
-    assertLessEqual,
     expect_warnings,
     get_params,
     gettestcases,
@@ -36,12 +34,20 @@ from youtube_dl.utils import (
     ExtractorError,
     error_to_compat_str,
     format_bytes,
+    IDENTITY,
+    preferredencoding,
     UnavailableVideoError,
 )
 from youtube_dl.extractor import get_info_extractor
 
 RETRIES = 3
 
+# Some unittest APIs require actual str
+if not isinstance('TEST', str):
+    _encode_str = lambda s: s.encode(preferredencoding())
+else:
+    _encode_str = IDENTITY
+
 
 class YoutubeDL(youtube_dl.YoutubeDL):
     def __init__(self, *args, **kwargs):
@@ -102,7 +108,7 @@ def generator(test_case, tname):
 
         def print_skipping(reason):
             print('Skipping %s: %s' % (test_case['name'], reason))
-            self.skipTest(reason)
+            self.skipTest(_encode_str(reason))
 
         if not ie.working():
             print_skipping('IE marked as not _WORKING')
@@ -187,16 +193,14 @@ def generator(test_case, tname):
                 expect_info_dict(self, res_dict, test_case.get('info_dict', {}))
 
             if 'playlist_mincount' in test_case:
-                assertGreaterEqual(
-                    self,
+                self.assertGreaterEqual(
                     len(res_dict['entries']),
                     test_case['playlist_mincount'],
                     'Expected at least %d in playlist %s, but got only %d' % (
                         test_case['playlist_mincount'], test_case['url'],
                         len(res_dict['entries'])))
             if 'playlist_maxcount' in test_case:
-                assertLessEqual(
-                    self,
+                self.assertLessEqual(
                     len(res_dict['entries']),
                     test_case['playlist_maxcount'],
                     'Expected at most %d in playlist %s, but got %d' % (
@@ -243,8 +247,8 @@ def generator(test_case, tname):
                         if params.get('test'):
                             expected_minsize = max(expected_minsize, 10000)
                         got_fsize = os.path.getsize(tc_filename)
-                        assertGreaterEqual(
-                            self, got_fsize, expected_minsize,
+                        self.assertGreaterEqual(
+                            got_fsize, expected_minsize,
                             'Expected %s to be at least %s, but it\'s only %s ' %
                             (tc_filename, format_bytes(expected_minsize),
                                 format_bytes(got_fsize)))

youtube_dl/YoutubeDL.py

@@ -1039,8 +1039,8 @@ class YoutubeDL(object):
         elif result_type in ('playlist', 'multi_video'):
             # Protect from infinite recursion due to recursively nested playlists
             # (see https://github.com/ytdl-org/youtube-dl/issues/27833)
-            webpage_url = ie_result['webpage_url']
-            if webpage_url in self._playlist_urls:
+            webpage_url = ie_result.get('webpage_url')  # not all pl/mv have this
+            if webpage_url and webpage_url in self._playlist_urls:
                 self.to_screen(
                     '[download] Skipping already downloaded playlist: %s'
                     % ie_result.get('title') or ie_result.get('id'))
@@ -1048,6 +1048,10 @@ class YoutubeDL(object):
 
             self._playlist_level += 1
             self._playlist_urls.add(webpage_url)
+            new_result = dict((k, v) for k, v in extra_info.items() if k not in ie_result)
+            if new_result:
+                new_result.update(ie_result)
+                ie_result = new_result
             try:
                 return self.__process_playlist(ie_result, download)
             finally:
@@ -1593,6 +1597,28 @@ class YoutubeDL(object):
         self.cookiejar.add_cookie_header(pr)
         return pr.get_header('Cookie')
 
+    def _fill_common_fields(self, info_dict, final=True):
+
+        for ts_key, date_key in (
+                ('timestamp', 'upload_date'),
+                ('release_timestamp', 'release_date'),
+        ):
+            if info_dict.get(date_key) is None and info_dict.get(ts_key) is not None:
+                # Working around out-of-range timestamp values (e.g. negative ones on Windows,
+                # see http://bugs.python.org/issue1646728)
+                try:
+                    upload_date = datetime.datetime.utcfromtimestamp(info_dict[ts_key])
+                    info_dict[date_key] = compat_str(upload_date.strftime('%Y%m%d'))
+                except (ValueError, OverflowError, OSError):
+                    pass
+
+        # Auto generate title fields corresponding to the *_number fields when missing
+        # in order to always have clean titles. This is very common for TV series.
+        if final:
+            for field in ('chapter', 'season', 'episode'):
+                if info_dict.get('%s_number' % field) is not None and not info_dict.get(field):
+                    info_dict[field] = '%s %d' % (field.capitalize(), info_dict['%s_number' % field])
+
     def process_video_result(self, info_dict, download=True):
         assert info_dict.get('_type', 'video') == 'video'
 
@@ -1660,24 +1686,7 @@ class YoutubeDL(object):
         if 'display_id' not in info_dict and 'id' in info_dict:
             info_dict['display_id'] = info_dict['id']
 
-        for ts_key, date_key in (
-                ('timestamp', 'upload_date'),
-                ('release_timestamp', 'release_date'),
-        ):
-            if info_dict.get(date_key) is None and info_dict.get(ts_key) is not None:
-                # Working around out-of-range timestamp values (e.g. negative ones on Windows,
-                # see http://bugs.python.org/issue1646728)
-                try:
-                    upload_date = datetime.datetime.utcfromtimestamp(info_dict[ts_key])
-                    info_dict[date_key] = compat_str(upload_date.strftime('%Y%m%d'))
-                except (ValueError, OverflowError, OSError):
-                    pass
-
-        # Auto generate title fields corresponding to the *_number fields when missing
-        # in order to always have clean titles. This is very common for TV series.
-        for field in ('chapter', 'season', 'episode'):
-            if info_dict.get('%s_number' % field) is not None and not info_dict.get(field):
-                info_dict[field] = '%s %d' % (field.capitalize(), info_dict['%s_number' % field])
+        self._fill_common_fields(info_dict)
 
         for cc_kind in ('subtitles', 'automatic_captions'):
             cc = info_dict.get(cc_kind)

youtube_dl/extractor/extractors.py

@@ -898,21 +898,13 @@ from .ooyala import (
 )
 from .ora import OraTVIE
 from .orf import (
-    ORFTVthekIE,
-    ORFFM4IE,
+    ORFONIE,
+    ORFONLiveIE,
     ORFFM4StoryIE,
-    ORFOE1IE,
-    ORFOE3IE,
-    ORFNOEIE,
-    ORFWIEIE,
-    ORFBGLIE,
-    ORFOOEIE,
-    ORFSTMIE,
-    ORFKTNIE,
-    ORFSBGIE,
-    ORFTIRIE,
-    ORFVBGIE,
     ORFIPTVIE,
+    ORFPodcastIE,
+    ORFRadioIE,
+    ORFRadioCollectionIE,
 )
 from .outsidetv import OutsideTVIE
 from .packtpub import (

youtube_dl/extractor/mixcloud.py

@@ -1,3 +1,4 @@
+# coding: utf-8
 from __future__ import unicode_literals
 
 import itertools
@@ -10,7 +11,7 @@ from ..compat import (
     compat_ord,
     compat_str,
     compat_urllib_parse_unquote,
-    compat_zip
+    compat_zip as zip,
 )
 from ..utils import (
     int_or_none,
@@ -24,7 +25,7 @@ class MixcloudBaseIE(InfoExtractor):
     def _call_api(self, object_type, object_fields, display_id, username, slug=None):
         lookup_key = object_type + 'Lookup'
         return self._download_json(
-            'https://www.mixcloud.com/graphql', display_id, query={
+            'https://app.mixcloud.com/graphql', display_id, query={
                 'query': '''{
   %s(lookup: {username: "%s"%s}) {
     %s
@@ -44,7 +45,7 @@ class MixcloudIE(MixcloudBaseIE):
             'ext': 'm4a',
             'title': 'Cryptkeeper',
             'description': 'After quite a long silence from myself, finally another Drum\'n\'Bass mix with my favourite current dance floor bangers.',
-            'uploader': 'Daniel Holbach',
+            'uploader': 'dholbach',  # was: 'Daniel Holbach',
             'uploader_id': 'dholbach',
             'thumbnail': r're:https?://.*\.jpg',
             'view_count': int,
@@ -57,7 +58,7 @@ class MixcloudIE(MixcloudBaseIE):
             'id': 'gillespeterson_caribou-7-inch-vinyl-mix-chat',
             'ext': 'mp3',
             'title': 'Caribou 7 inch Vinyl Mix & Chat',
-            'description': 'md5:2b8aec6adce69f9d41724647c65875e8',
+            'description': r're:Last week Dan Snaith aka Caribou swung by the Brownswood.{136}',
             'uploader': 'Gilles Peterson Worldwide',
             'uploader_id': 'gillespeterson',
             'thumbnail': 're:https?://.*',
@@ -65,6 +66,23 @@ class MixcloudIE(MixcloudBaseIE):
             'timestamp': 1422987057,
             'upload_date': '20150203',
         },
+        'params': {
+            'skip_download': '404 not found',
+        },
+    }, {
+        'url': 'https://www.mixcloud.com/gillespeterson/carnival-m%C3%BAsica-popular-brasileira-mix/',
+        'info_dict': {
+            'id': 'gillespeterson_carnival-música-popular-brasileira-mix',
+            'ext': 'm4a',
+            'title': 'Carnival Música Popular Brasileira Mix',
+            'description': r're:Gilles was recently in Brazil to play at Boiler Room.{208}',
+            'timestamp': 1454347174,
+            'upload_date': '20160201',
+            'uploader': 'Gilles Peterson Worldwide',
+            'uploader_id': 'gillespeterson',
+            'thumbnail': 're:https?://.*',
+            'view_count': int,
+        },
     }, {
         'url': 'https://beta.mixcloud.com/RedLightRadio/nosedrip-15-red-light-radio-01-18-2016/',
         'only_matching': True,
@@ -76,10 +94,10 @@ class MixcloudIE(MixcloudBaseIE):
         """Encrypt/Decrypt XOR cipher. Both ways are possible because it's XOR."""
         return ''.join([
             compat_chr(compat_ord(ch) ^ compat_ord(k))
-            for ch, k in compat_zip(ciphertext, itertools.cycle(key))])
+            for ch, k in zip(ciphertext, itertools.cycle(key))])
 
     def _real_extract(self, url):
-        username, slug = re.match(self._VALID_URL, url).groups()
+        username, slug = self._match_valid_url(url).groups()
         username, slug = compat_urllib_parse_unquote(username), compat_urllib_parse_unquote(slug)
         track_id = '%s_%s' % (username, slug)
 

youtube_dl/extractor/mtv.py

@@ -157,7 +157,7 @@ class MTVServicesInfoExtractor(InfoExtractor):
 
         description = strip_or_none(xpath_text(itemdoc, 'description'))
 
-        timestamp = timeconvert(xpath_text(itemdoc, 'pubDate'))
+        timestamp = timeconvert(xpath_text(itemdoc, 'pubDate') or xpath_text(itemdoc, 'airDate'))
 
         title_el = None
         if title_el is None:
@@ -290,7 +290,17 @@ class MTVServicesInfoExtractor(InfoExtractor):
             main_container = self._extract_child_with_type(data, 'MainContainer')
             ab_testing = self._extract_child_with_type(main_container, 'ABTesting')
             video_player = self._extract_child_with_type(ab_testing or main_container, 'VideoPlayer')
-            mgid = video_player['props']['media']['video']['config']['uri']
+            if video_player:
+                mgid = video_player['props']['media']['video']['config']['uri']
+            else:
+                flex_wrapper = self._extract_child_with_type(ab_testing or main_container, 'FlexWrapper')
+                auth_suite_wrapper = self._extract_child_with_type(flex_wrapper, 'AuthSuiteWrapper')
+                player = self._extract_child_with_type(auth_suite_wrapper or flex_wrapper, 'Player')
+                if player:
+                    mgid = player['props']['videoDetail']['mgid']
+
+        if not mgid:
+            raise ExtractorError('Could not extract mgid')
 
         return mgid
 

youtube_dl/extractor/southpark.py

@@ -6,19 +6,19 @@ from .mtv import MTVServicesInfoExtractor
 
 class SouthParkIE(MTVServicesInfoExtractor):
     IE_NAME = 'southpark.cc.com'
-    _VALID_URL = r'https?://(?:www\.)?(?P<url>southpark(?:\.cc|studios)\.com/(?:clips|(?:full-)?episodes|collections)/(?P<id>.+?)(\?|#|$))'
+    _VALID_URL = r'https?://(?:www\.)?(?P<url>southpark(?:\.cc|studios)\.com/((?:video-)?clips|(?:full-)?episodes|collections)/(?P<id>.+?)(\?|#|$))'
 
     _FEED_URL = 'http://feeds.mtvnservices.com/od/feed/intl-mrss-player-feed'
 
     _TESTS = [{
-        'url': 'http://southpark.cc.com/clips/104437/bat-daded#tab=featured',
+        'url': 'https://southpark.cc.com/video-clips/d7wr06/south-park-you-all-agreed-to-counseling',
         'info_dict': {
-            'id': 'a7bff6c2-ed00-11e0-aca6-0026b9414f30',
+            'id': '31929ad5-8269-11eb-8774-70df2f866ace',
             'ext': 'mp4',
-            'title': 'South Park|Bat Daded',
-            'description': 'Randy disqualifies South Park by getting into a fight with Bat Dad.',
-            'timestamp': 1112760000,
-            'upload_date': '20050406',
+            'title': 'You All Agreed to Counseling',
+            'description': 'Kenny, Cartman, Stan, and Kyle visit Mr. Mackey and ask for his help getting Mrs. Nelson to come back. Mr. Mackey reveals the only way to get things back to normal is to get the teachers vaccinated.',
+            'timestamp': 1615377600,
+            'upload_date': '20210310',
         },
     }, {
         'url': 'http://southpark.cc.com/collections/7758/fan-favorites/1',
@@ -40,11 +40,11 @@ class SouthParkIE(MTVServicesInfoExtractor):
 
 class SouthParkEsIE(SouthParkIE):
     IE_NAME = 'southpark.cc.com:español'
-    _VALID_URL = r'https?://(?:www\.)?(?P<url>southpark\.cc\.com/episodios-en-espanol/(?P<id>.+?)(\?|#|$))'
+    _VALID_URL = r'https?://(?:www\.)?(?P<url>southpark\.cc\.com/es/episodios/(?P<id>.+?)(\?|#|$))'
     _LANG = 'es'
 
     _TESTS = [{
-        'url': 'http://southpark.cc.com/episodios-en-espanol/s01e01-cartman-consigue-una-sonda-anal#source=351c1323-0b96-402d-a8b9-40d01b2e9bde&position=1&sort=!airdate',
+        'url': 'http://southpark.cc.com/es/episodios/s01e01-cartman-consigue-una-sonda-anal#source=351c1323-0b96-402d-a8b9-40d01b2e9bde&position=1&sort=!airdate',
         'info_dict': {
             'title': 'Cartman Consigue Una Sonda Anal',
             'description': 'Cartman Consigue Una Sonda Anal',
@@ -69,6 +69,7 @@ class SouthParkDeIE(SouthParkIE):
             'timestamp': 1380160800,
             'upload_date': '20130926',
         },
+        'skip': 'Geo-restricted',
     }, {
         # non-ASCII characters in initial URL
         'url': 'http://www.southpark.de/alle-episoden/s18e09-hashtag-aufwärmen',
@@ -77,6 +78,7 @@ class SouthParkDeIE(SouthParkIE):
             'description': 'Kyle will mit seinem kleinen Bruder Ike Videospiele spielen. Als der nicht mehr mit ihm spielen will, hat Kyle Angst, dass er die Kids von heute nicht mehr versteht.',
         },
         'playlist_count': 3,
+        'skip': 'Geo-restricted',
     }, {
         # non-ASCII characters in redirect URL
         'url': 'http://www.southpark.de/alle-episoden/s18e09',
@@ -85,6 +87,7 @@ class SouthParkDeIE(SouthParkIE):
             'description': 'Kyle will mit seinem kleinen Bruder Ike Videospiele spielen. Als der nicht mehr mit ihm spielen will, hat Kyle Angst, dass er die Kids von heute nicht mehr versteht.',
         },
         'playlist_count': 3,
+        'skip': 'Geo-restricted',
     }, {
         'url': 'http://www.southpark.de/collections/2476/superhero-showdown/1',
         'only_matching': True,
@@ -103,6 +106,7 @@ class SouthParkNlIE(SouthParkIE):
             'description': 'Stan is addicted to the new Terrance and Phillip mobile game.',
         },
         'playlist_mincount': 3,
+        'skip': 'Geo-restricted',
     }]
 
 
@@ -118,6 +122,7 @@ class SouthParkDkIE(SouthParkIE):
             'description': 'Butters is convinced he\'s living in a virtual reality.',
         },
         'playlist_mincount': 3,
+        'skip': 'Geo-restricted',
     }, {
         'url': 'http://www.southparkstudios.dk/collections/2476/superhero-showdown/1',
         'only_matching': True,

youtube_dl/extractor/vidlii.py

@@ -4,6 +4,7 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
+
 from ..utils import (
     float_or_none,
     get_element_by_id,
@@ -11,6 +12,7 @@ from ..utils import (
     strip_or_none,
     unified_strdate,
     urljoin,
+    str_to_int,
 )
 
 
@@ -35,6 +37,26 @@ class VidLiiIE(InfoExtractor):
             'categories': ['News & Politics'],
             'tags': ['Vidlii', 'Jan', 'Videogames'],
         }
+    }, {
+        # HD
+        'url': 'https://www.vidlii.com/watch?v=2Ng8Abj2Fkl',
+        'md5': '450e7da379c884788c3a4fa02a3ce1a4',
+        'info_dict': {
+            'id': '2Ng8Abj2Fkl',
+            'ext': 'mp4',
+            'title': 'test',
+            'description': 'md5:cc55a86032a7b6b3cbfd0f6b155b52e9',
+            'thumbnail': 'https://www.vidlii.com/usfi/thmp/2Ng8Abj2Fkl.jpg',
+            'uploader': 'VidLii',
+            'uploader_url': 'https://www.vidlii.com/user/VidLii',
+            'upload_date': '20200927',
+            'duration': 5,
+            'view_count': int,
+            'comment_count': int,
+            'average_rating': float,
+            'categories': ['Film & Animation'],
+            'tags': list,
+        },
     }, {
         'url': 'https://www.vidlii.com/embed?v=tJluaH4BJ3v&a=0',
         'only_matching': True,
@@ -46,11 +68,32 @@ class VidLiiIE(InfoExtractor):
         webpage = self._download_webpage(
             'https://www.vidlii.com/watch?v=%s' % video_id, video_id)
 
-        video_url = self._search_regex(
-            r'src\s*:\s*(["\'])(?P<url>(?:https?://)?(?:(?!\1).)+)\1', webpage,
-            'video url', group='url')
+        formats = []
 
-        title = self._search_regex(
+        def add_format(format_url, height=None):
+            height = int(self._search_regex(r'(\d+)\.mp4',
+                         format_url, 'height', default=360))
+
+            formats.append({
+                'url': format_url,
+                'format_id': '%dp' % height if height else None,
+                'height': height,
+            })
+
+        sources = re.findall(
+            r'src\s*:\s*(["\'])(?P<url>(?:https?://)?(?:(?!\1).)+)\1',
+            webpage)
+
+        formats = []
+        if len(sources) > 1:
+            add_format(sources[1][1])
+            self._check_formats(formats, video_id)
+        if len(sources) > 0:
+            add_format(sources[0][1])
+
+        self._sort_formats(formats)
+
+        title = self._html_search_regex(
             (r'<h1>([^<]+)</h1>', r'<title>([^<]+) - VidLii<'), webpage,
             'title')
 
@@ -82,9 +125,9 @@ class VidLiiIE(InfoExtractor):
             default=None) or self._search_regex(
             r'duration\s*:\s*(\d+)', webpage, 'duration', fatal=False))
 
-        view_count = int_or_none(self._search_regex(
-            (r'<strong>(\d+)</strong> views',
-             r'Views\s*:\s*<strong>(\d+)</strong>'),
+        view_count = str_to_int(self._html_search_regex(
+            (r'<strong>([\d,.]+)</strong> views',
+             r'Views\s*:\s*<strong>([\d,.]+)</strong>'),
             webpage, 'view count', fatal=False))
 
         comment_count = int_or_none(self._search_regex(
@@ -109,7 +152,7 @@ class VidLiiIE(InfoExtractor):
 
         return {
             'id': video_id,
-            'url': video_url,
+            'formats': formats,
             'title': title,
             'description': description,
             'thumbnail': thumbnail,

youtube_dl/utils.py

@@ -2094,7 +2094,7 @@ def timeconvert(timestr):
     timetuple = email.utils.parsedate_tz(timestr)
     if timetuple is not None:
         timestamp = email.utils.mktime_tz(timetuple)
-    return timestamp
+    return int(timestamp) if timestamp is not None else timestamp
 
 
 def sanitize_filename(s, restricted=False, is_id=False):