From 3f209889955c90f465b14b8de24b4da86602e732 Mon Sep 17 00:00:00 2001 From: df Date: Mon, 11 Oct 2021 16:58:57 +0100 Subject: [PATCH] Back-port Paramount+ extractor (etc) from yt-dlp With supporting changes to CBS and ThePlatform extractors --- youtube_dl/extractor/cbs.py | 131 ++++++++++++++++------- youtube_dl/extractor/extractors.py | 4 + youtube_dl/extractor/paramountplus.py | 145 ++++++++++++++++++++++++++ youtube_dl/extractor/theplatform.py | 21 +++- 4 files changed, 259 insertions(+), 42 deletions(-) create mode 100644 youtube_dl/extractor/paramountplus.py diff --git a/youtube_dl/extractor/cbs.py b/youtube_dl/extractor/cbs.py index c79e55a75..8ff2b2240 100644 --- a/youtube_dl/extractor/cbs.py +++ b/youtube_dl/extractor/cbs.py @@ -8,6 +8,7 @@ from ..utils import ( xpath_element, xpath_text, update_url_query, + url_or_none, ) @@ -25,10 +26,62 @@ class CBSBaseIE(ThePlatformFeedIE): }) return subtitles + def _extract_common_video_info(self, content_id, asset_types, mpx_acc, extra_info): + tp_path = 'dJ5BDC/media/guid/%d/%s' % (mpx_acc, content_id) + tp_release_url = 'http://link.theplatform.com/s/' + tp_path + info = self._extract_theplatform_metadata(tp_path, content_id) + + formats, subtitles = [], {} + last_e = None + for asset_type, query in asset_types.items(): + try: + tp_formats, tp_subtitles = self._extract_theplatform_smil( + update_url_query(tp_release_url, query), content_id, + 'Downloading %s SMIL data' % asset_type) + except ExtractorError as e: + last_e = e + if asset_type != 'fallback': + continue + query['formats'] = '' # blank query to check if expired + try: + tp_formats, tp_subtitles = self._extract_theplatform_smil( + update_url_query(tp_release_url, query), content_id, + 'Downloading %s SMIL data, trying again with another format' % asset_type) + except ExtractorError as e: + last_e = e + continue + formats.extend(tp_formats) + subtitles = self._merge_subtitles(subtitles, tp_subtitles) + if last_e and not formats: + raise last_e + self._sort_formats(formats) + + extra_info.update({ + 'id': content_id, + 'formats': formats, + 'subtitles': subtitles, + }) + info.update({k: v for k, v in extra_info.items() if v is not None}) + return info + + def _extract_video_info(self, *args, **kwargs): + # Extract assets + metadata and call _extract_common_video_info + raise NotImplementedError('This method must be implemented by subclasses') + + def _real_extract(self, url): + return self._extract_video_info(self._match_id(url)) + class CBSIE(CBSBaseIE): - _VALID_URL = r'(?:cbs:|https?://(?:www\.)?(?:(?:cbs|paramountplus)\.com/shows/[^/]+/video|colbertlateshow\.com/(?:video|podcasts))/)(?P[\w-]+)' + _VALID_URL = r'''(?x) + (?: + cbs:| + https?://(?:www\.)?(?: + cbs\.com/(?:shows/[^/]+/video|movies/[^/]+)/| + colbertlateshow\.com/(?:video|podcasts)/) + )(?P[\w-]+)''' + # All tests are blocked outside US _TESTS = [{ 'url': 'http://www.cbs.com/shows/garth-brooks/video/_u7W953k6la293J7EPTd9oHkSPs6Xn6_/connect-chat-feat-garth-brooks/', 'info_dict': { @@ -46,70 +99,70 @@ class CBSIE(CBSBaseIE): 'skip_download': True, }, '_skip': 'Blocked outside the US', + }, { + 'url': 'https://www.cbs.com/shows/the-late-show-with-stephen-colbert/video/60icOhMb9NcjbcWnF_gub9XXHdeBcNk2/the-late-show-6-23-21-christine-baranski-joy-oladokun-', + 'info_dict': { + 'id': '60icOhMb9NcjbcWnF_gub9XXHdeBcNk2', + 'title': 'The Late Show - 6/23/21 (Christine Baranski, Joy Oladokun)', + 'timestamp': 1624507140, + 'description': 'md5:e01af24e95c74d55e8775aef86117b95', + 'uploader': 'CBSI-NEW', + 'upload_date': '20210624', + }, + 'params': { + 'ignore_no_formats_error': True, + 'skip_download': True, + }, + 'expected_warnings': [ + 'This content expired on', 'No video formats found', 'Requested format is not available'], }, { 'url': 'http://colbertlateshow.com/video/8GmB0oY0McANFvp2aEffk9jZZZ2YyXxy/the-colbeard/', 'only_matching': True, }, { 'url': 'http://www.colbertlateshow.com/podcasts/dYSwjqPs_X1tvbV_P2FcPWRa_qT6akTC/in-the-bad-room-with-stephen/', 'only_matching': True, - }, { - 'url': 'https://www.paramountplus.com/shows/all-rise/video/QmR1WhNkh1a_IrdHZrbcRklm176X_rVc/all-rise-space/', - 'only_matching': True, }] def _extract_video_info(self, content_id, site='cbs', mpx_acc=2198311517): items_data = self._download_xml( - 'http://can.cbs.com/thunder/player/videoPlayerService.php', + 'https://can.cbs.com/thunder/player/videoPlayerService.php', content_id, query={'partner': site, 'contentId': content_id}) video_data = xpath_element(items_data, './/item') - title = xpath_text(video_data, 'videoTitle', 'title', True) - tp_path = 'dJ5BDC/media/guid/%d/%s' % (mpx_acc, content_id) - tp_release_url = 'http://link.theplatform.com/s/' + tp_path + title = xpath_text(video_data, 'videoTitle', 'title') or xpath_text(video_data, 'videotitle', 'title') - asset_types = [] - subtitles = {} - formats = [] - last_e = None + asset_types = {} + has_drm = False for item in items_data.findall('.//item'): asset_type = xpath_text(item, 'assetType') - if not asset_type or asset_type in asset_types or 'HLS_FPS' in asset_type or 'DASH_CENC' in asset_type: - continue - asset_types.append(asset_type) query = { 'mbr': 'true', 'assetTypes': asset_type, } - if asset_type.startswith('HLS') or asset_type in ('OnceURL', 'StreamPack'): + if not asset_type: + # fallback for content_ids that videoPlayerService doesn't return anything for + asset_type = 'fallback' + query['formats'] = 'M3U+none,MPEG4,M3U+appleHlsEncryption,MP3' + del query['assetTypes'] + if asset_type in asset_types: + continue + elif any(excluded in asset_type for excluded in ('HLS_FPS', 'DASH_CENC', 'OnceURL')): + if 'DASH_CENC' in asset_type: + has_drm = True + continue + if asset_type.startswith('HLS') or 'StreamPack' in asset_type: query['formats'] = 'MPEG4,M3U' elif asset_type in ('RTMP', 'WIFI', '3G'): query['formats'] = 'MPEG4,FLV' - try: - tp_formats, tp_subtitles = self._extract_theplatform_smil( - update_url_query(tp_release_url, query), content_id, - 'Downloading %s SMIL data' % asset_type) - except ExtractorError as e: - last_e = e - continue - formats.extend(tp_formats) - subtitles = self._merge_subtitles(subtitles, tp_subtitles) - if last_e and not formats: - raise last_e - self._sort_formats(formats) + asset_types[asset_type] = query - info = self._extract_theplatform_metadata(tp_path, content_id) - info.update({ - 'id': content_id, + if not asset_types and has_drm: + raise ExtractorError('Only DRM formats found', video_id=content_id, expected=True) + + return self._extract_common_video_info(content_id, asset_types, mpx_acc, extra_info={ 'title': title, 'series': xpath_text(video_data, 'seriesTitle'), 'season_number': int_or_none(xpath_text(video_data, 'seasonNumber')), 'episode_number': int_or_none(xpath_text(video_data, 'episodeNumber')), 'duration': int_or_none(xpath_text(video_data, 'videoLength'), 1000), - 'thumbnail': xpath_text(video_data, 'previewImageURL'), - 'formats': formats, - 'subtitles': subtitles, + 'thumbnail': url_or_none(xpath_text(video_data, 'previewImageURL')), }) - return info - - def _real_extract(self, url): - content_id = self._match_id(url) - return self._extract_video_info(content_id) diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py index 6e8fc3961..da3c7a828 100644 --- a/youtube_dl/extractor/extractors.py +++ b/youtube_dl/extractor/extractors.py @@ -889,6 +889,10 @@ from .palcomp3 import ( PalcoMP3VideoIE, ) from .pandoratv import PandoraTVIE +from .paramountplus import ( + ParamountPlusIE, + ParamountPlusSeriesIE, +) from .parliamentliveuk import ParliamentLiveUKIE from .patreon import PatreonIE from .pbs import PBSIE diff --git a/youtube_dl/extractor/paramountplus.py b/youtube_dl/extractor/paramountplus.py new file mode 100644 index 000000000..86c07f61f --- /dev/null +++ b/youtube_dl/extractor/paramountplus.py @@ -0,0 +1,145 @@ +from __future__ import unicode_literals + +from .common import InfoExtractor +from .cbs import CBSBaseIE +from ..utils import ( + int_or_none, + url_or_none, +) + + +class ParamountPlusIE(CBSBaseIE): + _VALID_URL = r'''(?x) + (?: + paramountplus:| + https?://(?:www\.)?(?: + paramountplus\.com/(?:shows/[^/]+/video|movies/[^/]+)/ + )(?P[\w-]+))''' + + # All tests are blocked outside US + _TESTS = [{ + 'url': 'https://www.paramountplus.com/shows/catdog/video/Oe44g5_NrlgiZE3aQVONleD6vXc8kP0k/catdog-climb-every-catdog-the-canine-mutiny/', + 'info_dict': { + 'id': 'Oe44g5_NrlgiZE3aQVONleD6vXc8kP0k', + 'ext': 'mp4', + 'title': 'CatDog - Climb Every CatDog/The Canine Mutiny', + 'description': 'md5:7ac835000645a69933df226940e3c859', + 'duration': 1426, + 'timestamp': 920264400, + 'upload_date': '19990301', + 'uploader': 'CBSI-NEW', + }, + 'params': { + 'skip_download': 'm3u8', + }, + }, { + 'url': 'https://www.paramountplus.com/shows/tooning-out-the-news/video/6hSWYWRrR9EUTz7IEe5fJKBhYvSUfexd/7-23-21-week-in-review-rep-jahana-hayes-howard-fineman-sen-michael-bennet-sheera-frenkel-cecilia-kang-/', + 'info_dict': { + 'id': '6hSWYWRrR9EUTz7IEe5fJKBhYvSUfexd', + 'ext': 'mp4', + 'title': '7/23/21 WEEK IN REVIEW (Rep. Jahana Hayes/Howard Fineman/Sen. Michael Bennet/Sheera Frenkel & Cecilia Kang)', + 'description': 'md5:f4adcea3e8b106192022e121f1565bae', + 'duration': 2506, + 'timestamp': 1627063200, + 'upload_date': '20210723', + 'uploader': 'CBSI-NEW', + }, + 'params': { + 'skip_download': 'm3u8', + }, + }, { + 'url': 'https://www.paramountplus.com/movies/daddys-home/vM2vm0kE6vsS2U41VhMRKTOVHyQAr6pC', + 'info_dict': { + 'id': 'vM2vm0kE6vsS2U41VhMRKTOVHyQAr6pC', + 'ext': 'mp4', + 'title': 'Daddy\'s Home', + 'upload_date': '20151225', + 'description': 'md5:9a6300c504d5e12000e8707f20c54745', + 'uploader': 'CBSI-NEW', + 'timestamp': 1451030400, + }, + 'params': { + 'skip_download': 'm3u8', + 'format': 'bestvideo', + }, + 'expected_warnings': ['Ignoring subtitle tracks'], # TODO: Investigate this + }, { + 'url': 'https://www.paramountplus.com/movies/sonic-the-hedgehog/5EKDXPOzdVf9voUqW6oRuocyAEeJGbEc', + 'info_dict': { + 'id': '5EKDXPOzdVf9voUqW6oRuocyAEeJGbEc', + 'ext': 'mp4', + 'uploader': 'CBSI-NEW', + 'description': 'md5:bc7b6fea84ba631ef77a9bda9f2ff911', + 'timestamp': 1577865600, + 'title': 'Sonic the Hedgehog', + 'upload_date': '20200101', + }, + 'params': { + 'skip_download': 'm3u8', + 'format': 'bestvideo', + }, + 'expected_warnings': ['Ignoring subtitle tracks'], + }, { + 'url': 'https://www.paramountplus.com/shows/all-rise/video/QmR1WhNkh1a_IrdHZrbcRklm176X_rVc/all-rise-space/', + 'only_matching': True, + }, { + 'url': 'https://www.paramountplus.com/movies/million-dollar-american-princesses-meghan-and-harry/C0LpgNwXYeB8txxycdWdR9TjxpJOsdCq', + 'only_matching': True, + }] + + def _extract_video_info(self, content_id, mpx_acc=2198311517): + items_data = self._download_json( + 'https://www.paramountplus.com/apps-api/v2.0/androidtv/video/cid/%s.json' % content_id, + content_id, query={'locale': 'en-us', 'at': 'ABCqWNNSwhIqINWIIAG+DFzcFUvF8/vcN6cNyXFFfNzWAIvXuoVgX+fK4naOC7V8MLI='}, headers=self.geo_verification_headers()) + + asset_types = { + item.get('assetType'): { + 'format': 'SMIL', + 'formats': 'MPEG4,M3U', + } for item in items_data['itemList'] + } + item = items_data['itemList'][-1] + return self._extract_common_video_info(content_id, asset_types, mpx_acc, extra_info={ + 'title': item.get('title'), + 'series': item.get('seriesTitle'), + 'season_number': int_or_none(item.get('seasonNum')), + 'episode_number': int_or_none(item.get('episodeNum')), + 'duration': int_or_none(item.get('duration')), + 'thumbnail': url_or_none(item.get('thumbnail')), + }) + + +class ParamountPlusSeriesIE(InfoExtractor): + _VALID_URL = r'https?://(?:www\.)?paramountplus\.com/shows/(?P[a-zA-Z0-9-_]+)/?(?:[#?]|$)' + _TESTS = [{ + 'url': 'https://www.paramountplus.com/shows/drake-josh', + 'playlist_mincount': 45, + 'info_dict': { + 'id': 'drake-josh', + } + }, { + 'url': 'https://www.paramountplus.com/shows/hawaii_five_0/', + 'playlist_mincount': 240, + 'info_dict': { + 'id': 'hawaii_five_0', + } + }, { + 'url': 'https://www.paramountplus.com/shows/spongebob-squarepants/', + 'playlist_mincount': 248, + 'info_dict': { + 'id': 'spongebob-squarepants', + } + }] + _API_URL = 'https://www.paramountplus.com/shows/{}/xhr/episodes/page/0/size/100000/xs/0/season/0/' + + def _entries(self, show_name): + show_json = self._download_json(self._API_URL.format(show_name), video_id=show_name) + if show_json.get('success'): + for episode in show_json['result']['data']: + yield self.url_result( + 'https://www.paramountplus.com%s' % episode['url'], + ie=ParamountPlusIE.ie_key(), video_id=episode['content_id']) + + def _real_extract(self, url): + show_name = self._match_id(url) + return self.playlist_result(self._entries(show_name), playlist_id=show_name) diff --git a/youtube_dl/extractor/theplatform.py b/youtube_dl/extractor/theplatform.py index adfe11e31..65d583ce6 100644 --- a/youtube_dl/extractor/theplatform.py +++ b/youtube_dl/extractor/theplatform.py @@ -20,6 +20,7 @@ from ..utils import ( float_or_none, int_or_none, sanitized_Request, + try_get, unsmuggle_url, update_url_query, xpath_with_ns, @@ -34,6 +35,15 @@ _x = lambda p: xpath_with_ns(p, {'smil': default_ns}) class ThePlatformBaseIE(OnceIE): _TP_TLD = 'com' + @classmethod + def _match_valid_url(cls, url): + # This does not use has/getattr intentionally - we want to know whether + # we have cached the regexp for *this* class, whereas getattr would also + # match the superclass + if '_VALID_URL_RE' not in cls.__dict__: + cls._VALID_URL_RE = re.compile(cls._VALID_URL) + return cls._VALID_URL_RE.match(url) + def _extract_theplatform_smil(self, smil_url, video_id, note='Downloading SMIL data'): meta = self._download_xml( smil_url, video_id, note=note, query={'format': 'SMIL'}, @@ -238,7 +248,7 @@ class ThePlatformIE(ThePlatformBaseIE, AdobePassIE): 'countries': smuggled_data.get('geo_countries'), }) - mobj = re.match(self._VALID_URL, url) + mobj = self._match_valid_url(url) provider_id = mobj.group('provider_id') video_id = mobj.group('id') @@ -338,6 +348,7 @@ class ThePlatformFeedIE(ThePlatformBaseIE): 'categories': ['MSNBC/Issues/Democrats', 'MSNBC/Issues/Elections/Election 2016'], 'uploader': 'NBCU-NEWS', }, + 'expected_warnings': ('Empty metadata',), }, { 'url': 'http://feed.theplatform.com/f/2E2eJC/nnd_NBCNews?byGuid=nn_netcast_180306.Copy.01', 'only_matching': True, @@ -345,7 +356,11 @@ class ThePlatformFeedIE(ThePlatformBaseIE): def _extract_feed_info(self, provider_id, feed_id, filter_query, video_id, custom_fields=None, asset_types_query={}, account_id=None): real_url = self._URL_TEMPLATE % (self.http_scheme(), provider_id, feed_id, filter_query) - entry = self._download_json(real_url, video_id)['entries'][0] + entry = self._download_json(real_url, video_id) + entry = try_get(entry, lambda x: x['entries'][0], dict) + if not entry: + self.report_warning('Empty metadata', video_id) + return None main_smil_url = 'http://link.theplatform.com/s/%s/media/guid/%d/%s' % (provider_id, account_id, entry['guid']) if account_id else entry.get('plmedia$publicUrl') formats = [] @@ -404,7 +419,7 @@ class ThePlatformFeedIE(ThePlatformBaseIE): return ret def _real_extract(self, url): - mobj = re.match(self._VALID_URL, url) + mobj = self._match_valid_url(url) video_id = mobj.group('id') provider_id = mobj.group('provider_id')