From 3e49222a74f4cbb84395af720ee4e0849202a2d4 Mon Sep 17 00:00:00 2001 From: dirkf Date: Wed, 16 Jun 2021 15:08:11 +0100 Subject: [PATCH 1/6] Handle additional indirection in Redtube playlists Fixes issue #29305 --- youtube_dl/extractor/redtube.py | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/youtube_dl/extractor/redtube.py b/youtube_dl/extractor/redtube.py index a1ca791ca..6871e1d3a 100644 --- a/youtube_dl/extractor/redtube.py +++ b/youtube_dl/extractor/redtube.py @@ -84,15 +84,25 @@ class RedTubeIE(InfoExtractor): r'mediaDefinition["\']?\s*:\s*(\[.+?}\s*\])', webpage, 'media definitions', default='{}'), video_id, fatal=False) - if medias and isinstance(medias, list): - for media in medias: + for media in medias if isinstance(medias, list) else []: + format_url = url_or_none(media.get('videoUrl')) + if not format_url: + continue + format_id = media.get('format') + quality = media.get('quality') + if format_id == 'hls' or (format_id == 'mp4' and not quality): + more_media = self._download_json(format_url, video_id, fatal=False) + else: + more_media = [media] + for media in more_media if isinstance(more_media, list) else []: format_url = url_or_none(media.get('videoUrl')) if not format_url: continue - if media.get('format') == 'hls' or determine_ext(format_url) == 'm3u8': + format_id = media.get('format') + if format_id == 'hls' or determine_ext(format_url) == 'm3u8': formats.extend(self._extract_m3u8_formats( format_url, video_id, 'mp4', - entry_protocol='m3u8_native', m3u8_id='hls', + entry_protocol='m3u8_native', m3u8_id=format_id or 'hls', fatal=False)) continue format_id = media.get('quality') From d42b7b6c5e927a8e4e82be96a551b5054e445974 Mon Sep 17 00:00:00 2001 From: df Date: Mon, 21 Jun 2021 12:52:33 +0100 Subject: [PATCH 2/6] Restore coding style deprecated by @dstftw --- youtube_dl/extractor/redtube.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/youtube_dl/extractor/redtube.py b/youtube_dl/extractor/redtube.py index 6871e1d3a..138f333b4 100644 --- a/youtube_dl/extractor/redtube.py +++ b/youtube_dl/extractor/redtube.py @@ -104,13 +104,13 @@ class RedTubeIE(InfoExtractor): format_url, video_id, 'mp4', entry_protocol='m3u8_native', m3u8_id=format_id or 'hls', fatal=False)) - continue - format_id = media.get('quality') - formats.append({ - 'url': format_url, - 'format_id': format_id, - 'height': int_or_none(format_id), - }) + else: + format_id = media.get('quality') + formats.append({ + 'url': format_url, + 'format_id': format_id, + 'height': int_or_none(format_id), + }) if not formats: video_url = self._html_search_regex( r'', webpage, 'video URL') From 1e490159e6fa65c29c4f9442aca006127127477a Mon Sep 17 00:00:00 2001 From: df Date: Sun, 26 Sep 2021 14:29:08 +0100 Subject: [PATCH 3/6] New test video to supplant test that became 'private' --- youtube_dl/extractor/redtube.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/youtube_dl/extractor/redtube.py b/youtube_dl/extractor/redtube.py index 138f333b4..cd5374b31 100644 --- a/youtube_dl/extractor/redtube.py +++ b/youtube_dl/extractor/redtube.py @@ -27,7 +27,23 @@ class RedTubeIE(InfoExtractor): 'duration': 596, 'view_count': int, 'age_limit': 18, - } + }, + 'skip': 'private video', + }, { + 'url': 'https://www.redtube.com/38864951', + 'md5': '4fba70cbca3aefd25767ab4b523c9878', + 'info_dict': { + 'id': '38864951', + 'ext': 'mp4', + 'title': 'Public Sex on the Balcony in Freezing Paris! Amateur Couple LeoLulu', + 'description': 'Watch video Public Sex on the Balcony in Freezing Paris! Amateur Couple LeoLulu on Redtube, home of free Blowjob porn videos and Blonde sex movies online. Video length: (10:46) - Uploaded by leolulu - Verified User - Starring Pornstar: Leolulu', + 'upload_date': '20210111', + 'timestamp': 1610343109, + 'duration': 646, + 'view_count': int, + 'age_limit': 18, + 'thumbnail': r're:https://\wi-ph\.rdtcdn\.com/videos/.+/.+\.jpg', + }, }, { 'url': 'http://embed.redtube.com/?bgcolor=000000&id=1443286', 'only_matching': True, From 4d4aefdb9cbad434aeea3a54f75a9883d8f287f5 Mon Sep 17 00:00:00 2001 From: dirkf Date: Fri, 21 Jul 2023 19:32:02 +0100 Subject: [PATCH 4/6] [RedTube] Update for 2023 * URLs need host://domain default * use traverse_obj() --- youtube_dl/extractor/redtube.py | 105 ++++++++++++++++---------------- 1 file changed, 51 insertions(+), 54 deletions(-) diff --git a/youtube_dl/extractor/redtube.py b/youtube_dl/extractor/redtube.py index cd5374b31..17bc12aa5 100644 --- a/youtube_dl/extractor/redtube.py +++ b/youtube_dl/extractor/redtube.py @@ -1,42 +1,36 @@ +# coding: utf-8 from __future__ import unicode_literals import re from .common import InfoExtractor +from ..compat import compat_str from ..utils import ( determine_ext, ExtractorError, int_or_none, + join_nonempty, merge_dicts, str_to_int, + T, + traverse_obj, unified_strdate, url_or_none, + urljoin, ) class RedTubeIE(InfoExtractor): _VALID_URL = r'https?://(?:(?:\w+\.)?redtube\.com/|embed\.redtube\.com/\?.*?\bid=)(?P[0-9]+)' + _EMBED_REGEX = [r']+?src=["\'](?P(?:https?:)?//embed\.redtube\.com/\?.*?\bid=\d+)'] _TESTS = [{ - 'url': 'http://www.redtube.com/66418', - 'md5': 'fc08071233725f26b8f014dba9590005', - 'info_dict': { - 'id': '66418', - 'ext': 'mp4', - 'title': 'Sucked on a toilet', - 'upload_date': '20110811', - 'duration': 596, - 'view_count': int, - 'age_limit': 18, - }, - 'skip': 'private video', - }, { 'url': 'https://www.redtube.com/38864951', 'md5': '4fba70cbca3aefd25767ab4b523c9878', 'info_dict': { 'id': '38864951', 'ext': 'mp4', 'title': 'Public Sex on the Balcony in Freezing Paris! Amateur Couple LeoLulu', - 'description': 'Watch video Public Sex on the Balcony in Freezing Paris! Amateur Couple LeoLulu on Redtube, home of free Blowjob porn videos and Blonde sex movies online. Video length: (10:46) - Uploaded by leolulu - Verified User - Starring Pornstar: Leolulu', + 'description': 'Watch video Public Sex on the Balcony in Freezing Paris! Amateur Couple LeoLulu on Redtube, home of free Blowjob porn videos and Blonde sex movies online. Video length: (10:46) - Uploaded by leolulu - Verified User - Starring Pornstar: LeoLulu', 'upload_date': '20210111', 'timestamp': 1610343109, 'duration': 646, @@ -52,11 +46,11 @@ class RedTubeIE(InfoExtractor): 'only_matching': True, }] - @staticmethod - def _extract_urls(webpage): - return re.findall( - r']+?src=["\'](?P(?:https?:)?//embed\.redtube\.com/\?.*?\bid=\d+)', - webpage) + @classmethod + def _extract_urls(cls, webpage): + for embed_re in cls._EMBED_REGEX: + for from_ in re.findall(embed_re, webpage): + yield from_ def _real_extract(self, url): video_id = self._match_id(url) @@ -87,50 +81,53 @@ class RedTubeIE(InfoExtractor): self._search_regex( r'sources\s*:\s*({.+?})', webpage, 'source', default='{}'), video_id, fatal=False) - if sources and isinstance(sources, dict): - for format_id, format_url in sources.items(): - if format_url: - formats.append({ - 'url': format_url, - 'format_id': format_id, - 'height': int_or_none(format_id), - }) - medias = self._parse_json( - self._search_regex( - r'mediaDefinition["\']?\s*:\s*(\[.+?}\s*\])', webpage, - 'media definitions', default='{}'), - video_id, fatal=False) - for media in medias if isinstance(medias, list) else []: - format_url = url_or_none(media.get('videoUrl')) + + def full_url(u): + return urljoin(url, u) + + for fmt in traverse_obj(sources, (T(dict.items), { + 'url': (1, T(full_url)), + 'format_id': (2, T(compat_str)), + 'height': (2, T(int_or_none)), })): + if 'url' in fmt: + formats.append(fmt) + + medias = self._search_regex( + r'''mediaDefinitions?["']?\s*:\s*(\[[\s\S]+?}\s*\])''', webpage, + 'media definitions', default='{}') + medias = self._parse_json(medias, video_id, fatal=False) + for fmt in traverse_obj(medias, (Ellipsis, T(dict))): + format_url = full_url(fmt.get('videoUrl')) if not format_url: continue - format_id = media.get('format') - quality = media.get('quality') - if format_id == 'hls' or (format_id == 'mp4' and not quality): + more_media = None + if fmt['format'] == 'hls' or (fmt['format'] == 'mp4' and not fmt.get('quality')): more_media = self._download_json(format_url, video_id, fatal=False) - else: - more_media = [media] - for media in more_media if isinstance(more_media, list) else []: - format_url = url_or_none(media.get('videoUrl')) + if more_media is None: + more_media = [fmt] + for fmt in traverse_obj(more_media, (Ellipsis, { + 'url': ('videoUrl', T(full_url)), + 'ext': ('format', T(compat_str)), + 'format_id': ('quality', T(compat_str)), })): + format_url = fmt.get('url') if not format_url: continue - format_id = media.get('format') - if format_id == 'hls' or determine_ext(format_url) == 'm3u8': + if fmt.get('ext') == 'hls' or determine_ext(format_url) == 'm3u8': formats.extend(self._extract_m3u8_formats( format_url, video_id, 'mp4', - entry_protocol='m3u8_native', m3u8_id=format_id or 'hls', + entry_protocol='m3u8_native', m3u8_id='hls', fatal=False)) - else: - format_id = media.get('quality') - formats.append({ - 'url': format_url, - 'format_id': format_id, - 'height': int_or_none(format_id), - }) + continue + fmt['height'] = int_or_none(fmt.get('format_id')) + fmt['format_id'] = join_nonempty('ext', 'format_id', from_dict=fmt) + formats.append(fmt) if not formats: - video_url = self._html_search_regex( - r'', webpage, 'video URL') - formats.append({'url': video_url}) + video_url = url_or_none(self._html_search_regex( + r'', webpage, 'video URL')) + if video_url: + formats.append({'url': video_url}) + + self._check_formats(formats, video_id) self._sort_formats(formats) thumbnail = self._og_search_thumbnail(webpage) From af80da38ba3778c4c01ec000a56d066cd13dc458 Mon Sep 17 00:00:00 2001 From: dirkf Date: Sat, 22 Jul 2023 14:10:55 +0100 Subject: [PATCH 5/6] Make download test pass --- youtube_dl/extractor/redtube.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/youtube_dl/extractor/redtube.py b/youtube_dl/extractor/redtube.py index 17bc12aa5..4825e7ce5 100644 --- a/youtube_dl/extractor/redtube.py +++ b/youtube_dl/extractor/redtube.py @@ -25,7 +25,7 @@ class RedTubeIE(InfoExtractor): _EMBED_REGEX = [r']+?src=["\'](?P(?:https?:)?//embed\.redtube\.com/\?.*?\bid=\d+)'] _TESTS = [{ 'url': 'https://www.redtube.com/38864951', - 'md5': '4fba70cbca3aefd25767ab4b523c9878', + 'md5': 'd7de9cb32e8adb3f6379f1a30f655fae', 'info_dict': { 'id': '38864951', 'ext': 'mp4', @@ -38,6 +38,11 @@ class RedTubeIE(InfoExtractor): 'age_limit': 18, 'thumbnail': r're:https://\wi-ph\.rdtcdn\.com/videos/.+/.+\.jpg', }, + 'expected_warnings': [ + 'Failed to download m3u8 information: HTTP Error 404', + ], + 'params': { + 'format': '[format_id !^= hls]', }, { 'url': 'http://embed.redtube.com/?bgcolor=000000&id=1443286', 'only_matching': True, From d2d6b49a5aec42b5b53a050ba18a4873b77dca40 Mon Sep 17 00:00:00 2001 From: dirkf Date: Sat, 22 Jul 2023 15:36:08 +0100 Subject: [PATCH 6/6] Update youtube_dl/extractor/redtube.py --- youtube_dl/extractor/redtube.py | 1 + 1 file changed, 1 insertion(+) diff --git a/youtube_dl/extractor/redtube.py b/youtube_dl/extractor/redtube.py index 4825e7ce5..0245ce2fa 100644 --- a/youtube_dl/extractor/redtube.py +++ b/youtube_dl/extractor/redtube.py @@ -43,6 +43,7 @@ class RedTubeIE(InfoExtractor): ], 'params': { 'format': '[format_id !^= hls]', + }, }, { 'url': 'http://embed.redtube.com/?bgcolor=000000&id=1443286', 'only_matching': True,