From c84890f7082237b02a5fc61657b2ea513836a85e Mon Sep 17 00:00:00 2001 From: Adrian Kretz <adriankretz@gmail.com> Date: Sat, 6 Dec 2014 18:28:21 +0100 Subject: [PATCH 1/4] [prosiebensat1] Add support for playlists (fixes #4357) --- youtube_dl/extractor/prosiebensat1.py | 47 +++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/youtube_dl/extractor/prosiebensat1.py b/youtube_dl/extractor/prosiebensat1.py index 32d747ede..c6e539e5a 100644 --- a/youtube_dl/extractor/prosiebensat1.py +++ b/youtube_dl/extractor/prosiebensat1.py @@ -8,6 +8,7 @@ from .common import InfoExtractor from ..utils import ( compat_urllib_parse, unified_strdate, + ExtractorError, ) @@ -152,6 +153,15 @@ class ProSiebenSat1IE(InfoExtractor): 'skip_download': True, }, }, + { + 'url': 'http://www.prosieben.de/tv/joko-gegen-klaas/videos/playlists/episode-8-ganze-folge-playlist', + 'info_dict': { + 'id': '439664', + 'title': 'Episode 8 - Ganze Folge - Playlist', + 'description': 'Das finale und härteste Duell aller Zeiten ist vorbei! Der Weltmeister für dieses Jahr steht! Alle packenden Duelle der achten Episode von "Joko gegen Klaas - das Duell um die Welt" seht ihr hier noch einmal in voller Länge!', + }, + 'playlist_count': 2, + }, ] _CLIPID_REGEXES = [ @@ -178,11 +188,48 @@ class ProSiebenSat1IE(InfoExtractor): r'<span style="padding-left: 4px;line-height:20px; color:#404040">(\d{2}\.\d{2}\.\d{4})</span>', r'(\d{2}\.\d{2}\.\d{4}) \| \d{2}:\d{2} Min<br/>', ] + _ITEM_TYPE_REGEXES = [ + r"'itemType'\s*:\s*'([^']*)'", + ] + _ITEM_ID_REGEXES = [ + r"'itemId'\s*:\s*'([^']*)'", + ] + _PLAYLIST_CLIPS_REGEXES = [ + r'data-qvt=.+?<a href="([^"]+)"', + ] def _real_extract(self, url): video_id = self._match_id(url) webpage = self._download_webpage(url, video_id) + item_type = self._html_search_regex(self._ITEM_TYPE_REGEXES, webpage, 'item type', default='CLIP') + if item_type == 'CLIP': + return self._clip_extract(url, webpage) + elif item_type == 'PLAYLIST': + playlist_id = self._html_search_regex(self._ITEM_ID_REGEXES, webpage, 'playlist id') + + for regex in self._PLAYLIST_CLIPS_REGEXES: + playlist_clips = re.findall(regex, webpage, re.DOTALL) + if playlist_clips: + title = self._html_search_regex(self._TITLE_REGEXES, webpage, 'title') + description = self._html_search_regex(self._DESCRIPTION_REGEXES, webpage, 'description', fatal=False) + root_url = re.match('(.+?//.+?)/', url).group(1) + + return { + '_type': 'playlist', + 'id': playlist_id, + 'title': title, + 'description': description, + 'entries': [self._clip_extract(root_url + clip_path) for clip_path in playlist_clips] + } + else: + raise ExtractorError('Unknown item type "%s"' % item_type) + + def _clip_extract(self, url, webpage=None): + if webpage is None: + video_id = self._match_id(url) + webpage = self._download_webpage(url, video_id) + clip_id = self._html_search_regex(self._CLIPID_REGEXES, webpage, 'clip id') access_token = 'testclient' From 8d1c8cae9ce9027058d90b246b9fa30ea88602ef Mon Sep 17 00:00:00 2001 From: Adrian Kretz <adriankretz@gmail.com> Date: Sat, 6 Dec 2014 19:21:05 +0100 Subject: [PATCH 2/4] [prosiebensat1] Fix broken tests --- youtube_dl/extractor/prosiebensat1.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/youtube_dl/extractor/prosiebensat1.py b/youtube_dl/extractor/prosiebensat1.py index c6e539e5a..8c9451f40 100644 --- a/youtube_dl/extractor/prosiebensat1.py +++ b/youtube_dl/extractor/prosiebensat1.py @@ -86,7 +86,7 @@ class ProSiebenSat1IE(InfoExtractor): 'ext': 'mp4', 'title': 'Im Interview: Kai Wiesinger', 'description': 'md5:e4e5370652ec63b95023e914190b4eb9', - 'upload_date': '20140225', + 'upload_date': '20140203', 'duration': 522.56, }, 'params': { @@ -101,7 +101,7 @@ class ProSiebenSat1IE(InfoExtractor): 'ext': 'mp4', 'title': 'Jagd auf Fertigkost im Elsthal - Teil 2', 'description': 'md5:2669cde3febe9bce13904f701e774eb6', - 'upload_date': '20140225', + 'upload_date': '20141014', 'duration': 2410.44, }, 'params': { @@ -168,6 +168,7 @@ class ProSiebenSat1IE(InfoExtractor): r'"clip_id"\s*:\s+"(\d+)"', r'clipid: "(\d+)"', r'clip[iI]d=(\d+)', + r"'itemImageUrl'\s*:\s*'/dynamic/thumbnails/full/\d+/(\d+)", ] _TITLE_REGEXES = [ r'<h2 class="subtitle" itemprop="name">\s*(.+?)</h2>', From acf5cbfe93ec4f828fef75486fa8b4e587a64baa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com> Date: Sun, 7 Dec 2014 01:46:30 +0600 Subject: [PATCH 3/4] [extractor/common] Add description to playlist_result --- youtube_dl/extractor/common.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py index cb6081dd0..2faaf6226 100644 --- a/youtube_dl/extractor/common.py +++ b/youtube_dl/extractor/common.py @@ -439,7 +439,7 @@ class InfoExtractor(object): return video_info @staticmethod - def playlist_result(entries, playlist_id=None, playlist_title=None): + def playlist_result(entries, playlist_id=None, playlist_title=None, playlist_description=None): """Returns a playlist""" video_info = {'_type': 'playlist', 'entries': entries} @@ -447,6 +447,8 @@ class InfoExtractor(object): video_info['id'] = playlist_id if playlist_title: video_info['title'] = playlist_title + if playlist_description: + video_info['description'] = playlist_description return video_info def _search_regex(self, pattern, string, name, default=_NO_DEFAULT, fatal=True, flags=0, group=None): From 6a52eed80edbb0f1e65a74def32d84236ac6e640 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com> Date: Sun, 7 Dec 2014 01:46:44 +0600 Subject: [PATCH 4/4] [prosiebensat1] Improve and simplify --- youtube_dl/extractor/prosiebensat1.py | 74 +++++++++++++-------------- 1 file changed, 36 insertions(+), 38 deletions(-) diff --git a/youtube_dl/extractor/prosiebensat1.py b/youtube_dl/extractor/prosiebensat1.py index 8c9451f40..1262793c8 100644 --- a/youtube_dl/extractor/prosiebensat1.py +++ b/youtube_dl/extractor/prosiebensat1.py @@ -8,7 +8,6 @@ from .common import InfoExtractor from ..utils import ( compat_urllib_parse, unified_strdate, - ExtractorError, ) @@ -158,7 +157,7 @@ class ProSiebenSat1IE(InfoExtractor): 'info_dict': { 'id': '439664', 'title': 'Episode 8 - Ganze Folge - Playlist', - 'description': 'Das finale und härteste Duell aller Zeiten ist vorbei! Der Weltmeister für dieses Jahr steht! Alle packenden Duelle der achten Episode von "Joko gegen Klaas - das Duell um die Welt" seht ihr hier noch einmal in voller Länge!', + 'description': 'md5:63b8963e71f481782aeea877658dec84', }, 'playlist_count': 2, }, @@ -189,48 +188,19 @@ class ProSiebenSat1IE(InfoExtractor): r'<span style="padding-left: 4px;line-height:20px; color:#404040">(\d{2}\.\d{2}\.\d{4})</span>', r'(\d{2}\.\d{2}\.\d{4}) \| \d{2}:\d{2} Min<br/>', ] - _ITEM_TYPE_REGEXES = [ + _PAGE_TYPE_REGEXES = [ + r'<meta name="page_type" content="([^"]+)">', r"'itemType'\s*:\s*'([^']*)'", ] - _ITEM_ID_REGEXES = [ + _PLAYLIST_ID_REGEXES = [ + r'content[iI]d=(\d+)', r"'itemId'\s*:\s*'([^']*)'", ] - _PLAYLIST_CLIPS_REGEXES = [ - r'data-qvt=.+?<a href="([^"]+)"', + _PLAYLIST_CLIP_REGEXES = [ + r'(?s)data-qvt=.+?<a href="([^"]+)"', ] - def _real_extract(self, url): - video_id = self._match_id(url) - webpage = self._download_webpage(url, video_id) - - item_type = self._html_search_regex(self._ITEM_TYPE_REGEXES, webpage, 'item type', default='CLIP') - if item_type == 'CLIP': - return self._clip_extract(url, webpage) - elif item_type == 'PLAYLIST': - playlist_id = self._html_search_regex(self._ITEM_ID_REGEXES, webpage, 'playlist id') - - for regex in self._PLAYLIST_CLIPS_REGEXES: - playlist_clips = re.findall(regex, webpage, re.DOTALL) - if playlist_clips: - title = self._html_search_regex(self._TITLE_REGEXES, webpage, 'title') - description = self._html_search_regex(self._DESCRIPTION_REGEXES, webpage, 'description', fatal=False) - root_url = re.match('(.+?//.+?)/', url).group(1) - - return { - '_type': 'playlist', - 'id': playlist_id, - 'title': title, - 'description': description, - 'entries': [self._clip_extract(root_url + clip_path) for clip_path in playlist_clips] - } - else: - raise ExtractorError('Unknown item type "%s"' % item_type) - - def _clip_extract(self, url, webpage=None): - if webpage is None: - video_id = self._match_id(url) - webpage = self._download_webpage(url, video_id) - + def _extract_clip(self, url, webpage): clip_id = self._html_search_regex(self._CLIPID_REGEXES, webpage, 'clip id') access_token = 'testclient' @@ -329,3 +299,31 @@ class ProSiebenSat1IE(InfoExtractor): 'duration': duration, 'formats': formats, } + + def _extract_playlist(self, url, webpage): + playlist_id = self._html_search_regex( + self._PLAYLIST_ID_REGEXES, webpage, 'playlist id') + for regex in self._PLAYLIST_CLIP_REGEXES: + playlist_clips = re.findall(regex, webpage) + if playlist_clips: + title = self._html_search_regex( + self._TITLE_REGEXES, webpage, 'title') + description = self._html_search_regex( + self._DESCRIPTION_REGEXES, webpage, 'description', fatal=False) + entries = [ + self.url_result( + re.match('(.+?//.+?)/', url).group(1) + clip_path, + 'ProSiebenSat1') + for clip_path in playlist_clips] + return self.playlist_result(entries, playlist_id, title, description) + + def _real_extract(self, url): + video_id = self._match_id(url) + webpage = self._download_webpage(url, video_id) + page_type = self._search_regex( + self._PAGE_TYPE_REGEXES, webpage, + 'page type', default='clip').lower() + if page_type == 'clip': + return self._extract_clip(url, webpage) + elif page_type == 'playlist': + return self._extract_playlist(url, webpage)