From 7b4137c351222a94f46f854bf490a299e4124acc Mon Sep 17 00:00:00 2001 From: remitamine Date: Wed, 9 Sep 2015 10:42:47 +0100 Subject: [PATCH 1/7] [fktv] fix info extraction --- youtube_dl/extractor/__init__.py | 5 +- youtube_dl/extractor/fktv.py | 78 ++++++++------------------------ 2 files changed, 20 insertions(+), 63 deletions(-) diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py index 57f55b479..f8d4c8462 100644 --- a/youtube_dl/extractor/__init__.py +++ b/youtube_dl/extractor/__init__.py @@ -170,10 +170,7 @@ from .firstpost import FirstpostIE from .firsttv import FirstTVIE from .fivemin import FiveMinIE from .fivetv import FiveTVIE -from .fktv import ( - FKTVIE, - FKTVPosteckeIE, -) +from .fktv import FKTVIE from .flickr import FlickrIE from .folketinget import FolketingetIE from .footyroom import FootyRoomIE diff --git a/youtube_dl/extractor/fktv.py b/youtube_dl/extractor/fktv.py index 190d9f9ad..c2aa23aa2 100644 --- a/youtube_dl/extractor/fktv.py +++ b/youtube_dl/extractor/fktv.py @@ -1,13 +1,11 @@ from __future__ import unicode_literals import re -import random -import json from .common import InfoExtractor from ..utils import ( - get_element_by_id, clean_html, + determine_ext, ) @@ -17,66 +15,28 @@ class FKTVIE(InfoExtractor): _TEST = { 'url': 'http://fernsehkritik.tv/folge-1', + 'md5': '21f0b0c99bce7d5b524eb1b17b1c6d79', 'info_dict': { - 'id': '00011', - 'ext': 'flv', + 'id': '1', + 'ext': 'mp4', 'title': 'Folge 1 vom 10. April 2007', - 'description': 'md5:fb4818139c7cfe6907d4b83412a6864f', }, } def _real_extract(self, url): - episode = int(self._match_id(url)) + episode = self._match_id(url) - video_thumbnail = 'http://fernsehkritik.tv/images/magazin/folge%s.jpg' % episode - start_webpage = self._download_webpage('http://fernsehkritik.tv/folge-%s/Start' % episode, - episode) - playlist = self._search_regex(r'playlist = (\[.*?\]);', start_webpage, - 'playlist', flags=re.DOTALL) - files = json.loads(re.sub('{[^{}]*?}', '{}', playlist)) - - videos = [] - for i, _ in enumerate(files, 1): - video_id = '%04d%d' % (episode, i) - video_url = 'http://fernsehkritik.tv/js/directme.php?file=%s%s.flv' % (episode, '' if i == 1 else '-%d' % i) - videos.append({ - 'ext': 'flv', - 'id': video_id, - 'url': video_url, - 'title': clean_html(get_element_by_id('eptitle', start_webpage)), - 'description': clean_html(get_element_by_id('contentlist', start_webpage)), - 'thumbnail': video_thumbnail - }) - return { - '_type': 'multi_video', - 'entries': videos, - 'id': 'folge-%s' % episode, - } - - -class FKTVPosteckeIE(InfoExtractor): - IE_NAME = 'fernsehkritik.tv:postecke' - _VALID_URL = r'http://(?:www\.)?fernsehkritik\.tv/inline-video/postecke\.php\?(.*&)?ep=(?P[0-9]+)(&|$)' - _TEST = { - 'url': 'http://fernsehkritik.tv/inline-video/postecke.php?iframe=true&width=625&height=440&ep=120', - 'md5': '262f0adbac80317412f7e57b4808e5c4', - 'info_dict': { - 'id': '0120', - 'ext': 'flv', - 'title': 'Postecke 120', - } - } - - def _real_extract(self, url): - mobj = re.match(self._VALID_URL, url) - episode = int(mobj.group('ep')) - - server = random.randint(2, 4) - video_id = '%04d' % episode - video_url = 'http://dl%d.fernsehkritik.tv/postecke/postecke%d.flv' % (server, episode) - video_title = 'Postecke %d' % episode - return { - 'id': video_id, - 'url': video_url, - 'title': video_title, - } + webpage = self._download_webpage('http://fernsehkritik.tv/folge-%s/play' % episode, episode) + title = clean_html(self._html_search_regex('

([^<]+?)

', webpage, 'title')) + matchs = re.search(r'(?s)]*poster="([^"]+)"[^>]*>(.*?)', webpage) + if matchs: + poster, sources = matchs.groups() + urls = re.findall(r'(?s)]*src="([^"]+)"[^>]*>', sources) + if sources: + formats = [{'url': url, 'format_id': determine_ext(url)} for url in urls] + return { + 'id': episode, + 'title': title, + 'formats': formats, + 'thumbnail': poster, + } From 2e40a12225fbe64b84b3975b3063a676df0f4522 Mon Sep 17 00:00:00 2001 From: Yen Chi Hsuan Date: Fri, 25 Sep 2015 17:24:35 +0800 Subject: [PATCH 2/7] [fktv] Correct spellings --- youtube_dl/extractor/fktv.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/youtube_dl/extractor/fktv.py b/youtube_dl/extractor/fktv.py index c2aa23aa2..93c4fd641 100644 --- a/youtube_dl/extractor/fktv.py +++ b/youtube_dl/extractor/fktv.py @@ -28,9 +28,9 @@ class FKTVIE(InfoExtractor): webpage = self._download_webpage('http://fernsehkritik.tv/folge-%s/play' % episode, episode) title = clean_html(self._html_search_regex('

([^<]+?)

', webpage, 'title')) - matchs = re.search(r'(?s)]*poster="([^"]+)"[^>]*>(.*?)', webpage) - if matchs: - poster, sources = matchs.groups() + matches = re.search(r'(?s)]*poster="([^"]+)"[^>]*>(.*?)', webpage) + if matches: + poster, sources = matches.groups() urls = re.findall(r'(?s)]*src="([^"]+)"[^>]*>', sources) if sources: formats = [{'url': url, 'format_id': determine_ext(url)} for url in urls] From 8ddf48d59f2d04b7411202eb6bf02c6eaa387035 Mon Sep 17 00:00:00 2001 From: Yen Chi Hsuan Date: Fri, 25 Sep 2015 17:48:51 +0800 Subject: [PATCH 3/7] [fktv] Raise an error is no videos found --- youtube_dl/extractor/fktv.py | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/youtube_dl/extractor/fktv.py b/youtube_dl/extractor/fktv.py index 93c4fd641..74c6cf866 100644 --- a/youtube_dl/extractor/fktv.py +++ b/youtube_dl/extractor/fktv.py @@ -6,6 +6,7 @@ from .common import InfoExtractor from ..utils import ( clean_html, determine_ext, + ExtractorError, ) @@ -29,14 +30,15 @@ class FKTVIE(InfoExtractor): webpage = self._download_webpage('http://fernsehkritik.tv/folge-%s/play' % episode, episode) title = clean_html(self._html_search_regex('

([^<]+?)

', webpage, 'title')) matches = re.search(r'(?s)]*poster="([^"]+)"[^>]*>(.*?)', webpage) - if matches: - poster, sources = matches.groups() - urls = re.findall(r'(?s)]*src="([^"]+)"[^>]*>', sources) - if sources: - formats = [{'url': url, 'format_id': determine_ext(url)} for url in urls] - return { - 'id': episode, - 'title': title, - 'formats': formats, - 'thumbnail': poster, - } + if matches is None: + raise ExtractorError('Unable to extract the video') + + poster, sources = matches.groups() + urls = re.findall(r'(?s)]*src="([^"]+)"[^>]*>', sources) + formats = [{'url': url, 'format_id': determine_ext(url)} for url in urls] + return { + 'id': episode, + 'title': title, + 'formats': formats, + 'thumbnail': poster, + } From 140359fc2cfe7e9cbfecfd2fd625c6407232fe0f Mon Sep 17 00:00:00 2001 From: Yen Chi Hsuan Date: Fri, 25 Sep 2015 17:51:48 +0800 Subject: [PATCH 4/7] [fktv] Correct and improve some regexs --- youtube_dl/extractor/fktv.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/youtube_dl/extractor/fktv.py b/youtube_dl/extractor/fktv.py index 74c6cf866..b081eb535 100644 --- a/youtube_dl/extractor/fktv.py +++ b/youtube_dl/extractor/fktv.py @@ -28,13 +28,13 @@ class FKTVIE(InfoExtractor): episode = self._match_id(url) webpage = self._download_webpage('http://fernsehkritik.tv/folge-%s/play' % episode, episode) - title = clean_html(self._html_search_regex('

([^<]+?)

', webpage, 'title')) - matches = re.search(r'(?s)]*poster="([^"]+)"[^>]*>(.*?)', webpage) + title = clean_html(self._html_search_regex('

([^<]+)

', webpage, 'title')) + matches = re.search(r'(?s)]+poster="([^"]+)"[^>]*>(.*)', webpage) if matches is None: raise ExtractorError('Unable to extract the video') poster, sources = matches.groups() - urls = re.findall(r'(?s)]*src="([^"]+)"[^>]*>', sources) + urls = re.findall(r']+src="([^"]+)"', sources) formats = [{'url': url, 'format_id': determine_ext(url)} for url in urls] return { 'id': episode, From 577380396171ba096240bfb3101f8151e32b587a Mon Sep 17 00:00:00 2001 From: Yen Chi Hsuan Date: Fri, 25 Sep 2015 17:58:44 +0800 Subject: [PATCH 5/7] [fktv] Correct thumbnail extraction and add the test --- youtube_dl/extractor/fktv.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/youtube_dl/extractor/fktv.py b/youtube_dl/extractor/fktv.py index b081eb535..289cbb8c8 100644 --- a/youtube_dl/extractor/fktv.py +++ b/youtube_dl/extractor/fktv.py @@ -21,6 +21,7 @@ class FKTVIE(InfoExtractor): 'id': '1', 'ext': 'mp4', 'title': 'Folge 1 vom 10. April 2007', + 'thumbnail': 're:^https?://.*\.jpg$', }, } @@ -29,11 +30,14 @@ class FKTVIE(InfoExtractor): webpage = self._download_webpage('http://fernsehkritik.tv/folge-%s/play' % episode, episode) title = clean_html(self._html_search_regex('

([^<]+)

', webpage, 'title')) - matches = re.search(r'(?s)]+poster="([^"]+)"[^>]*>(.*)', webpage) + matches = re.search(r'(?s)]+(?:poster="([^"]+)")?[^>]*>(.*)', webpage) if matches is None: raise ExtractorError('Unable to extract the video') poster, sources = matches.groups() + if poster is None: + self.report_warning('unable to extract thumbnail') + urls = re.findall(r']+src="([^"]+)"', sources) formats = [{'url': url, 'format_id': determine_ext(url)} for url in urls] return { From 711762f0b70b65f2f28f2cd8023497f50fccd81a Mon Sep 17 00:00:00 2001 From: Yen Chi Hsuan Date: Fri, 25 Sep 2015 18:01:08 +0800 Subject: [PATCH 6/7] [fktv] Coding style --- youtube_dl/extractor/fktv.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/youtube_dl/extractor/fktv.py b/youtube_dl/extractor/fktv.py index 289cbb8c8..0c14834f9 100644 --- a/youtube_dl/extractor/fktv.py +++ b/youtube_dl/extractor/fktv.py @@ -28,9 +28,12 @@ class FKTVIE(InfoExtractor): def _real_extract(self, url): episode = self._match_id(url) - webpage = self._download_webpage('http://fernsehkritik.tv/folge-%s/play' % episode, episode) - title = clean_html(self._html_search_regex('

([^<]+)

', webpage, 'title')) - matches = re.search(r'(?s)]+(?:poster="([^"]+)")?[^>]*>(.*)', webpage) + webpage = self._download_webpage( + 'http://fernsehkritik.tv/folge-%s/play' % episode, episode) + title = clean_html(self._html_search_regex( + '

([^<]+)

', webpage, 'title')) + matches = re.search( + r'(?s)]+(?:poster="([^"]+)")?[^>]*>(.*)', webpage) if matches is None: raise ExtractorError('Unable to extract the video') @@ -39,7 +42,10 @@ class FKTVIE(InfoExtractor): self.report_warning('unable to extract thumbnail') urls = re.findall(r']+src="([^"]+)"', sources) - formats = [{'url': url, 'format_id': determine_ext(url)} for url in urls] + formats = [{ + 'url': url, + 'format_id': determine_ext(url), + } for url in urls] return { 'id': episode, 'title': title, From 8de28761c41ca0451e3bbe4b9ee236d6651cca44 Mon Sep 17 00:00:00 2001 From: Yen Chi Hsuan Date: Fri, 25 Sep 2015 18:17:48 +0800 Subject: [PATCH 7/7] [fktv] Fix a regex --- youtube_dl/extractor/fktv.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/youtube_dl/extractor/fktv.py b/youtube_dl/extractor/fktv.py index 0c14834f9..d9fc9952d 100644 --- a/youtube_dl/extractor/fktv.py +++ b/youtube_dl/extractor/fktv.py @@ -33,7 +33,8 @@ class FKTVIE(InfoExtractor): title = clean_html(self._html_search_regex( '

([^<]+)

', webpage, 'title')) matches = re.search( - r'(?s)]+(?:poster="([^"]+)")?[^>]*>(.*)', webpage) + r'(?s)])+(?:poster="([^"]+)")?[^>]*>(.*)', + webpage) if matches is None: raise ExtractorError('Unable to extract the video')