From 7561727db6490308205d487a3609e2581e9bced3 Mon Sep 17 00:00:00 2001 From: marieell Date: Thu, 10 Feb 2022 21:49:36 +0100 Subject: [PATCH] Remove extractors for dead pages They all redirect to youporn.com. --- youtube_dl/extractor/extractors.py | 6 -- youtube_dl/extractor/extremetube.py | 50 ----------- youtube_dl/extractor/generic.py | 6 -- youtube_dl/extractor/keezmovies.py | 133 ---------------------------- youtube_dl/extractor/mofosex.py | 79 ----------------- youtube_dl/extractor/tube8.py | 60 ++++++++++++- 6 files changed, 58 insertions(+), 276 deletions(-) delete mode 100644 youtube_dl/extractor/extremetube.py delete mode 100644 youtube_dl/extractor/keezmovies.py delete mode 100644 youtube_dl/extractor/mofosex.py diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py index 50b7cb4a0..88c9b0756 100644 --- a/youtube_dl/extractor/extractors.py +++ b/youtube_dl/extractor/extractors.py @@ -356,7 +356,6 @@ from .esri import EsriVideoIE from .europa import EuropaIE from .expotv import ExpoTVIE from .expressen import ExpressenIE -from .extremetube import ExtremeTubeIE from .eyedotv import EyedoTVIE from .facebook import ( FacebookIE, @@ -537,7 +536,6 @@ from .kaltura import KalturaIE from .kankan import KankanIE from .karaoketv import KaraoketvIE from .karrierevideos import KarriereVideosIE -from .keezmovies import KeezMoviesIE from .ketnet import KetnetIE from .khanacademy import ( KhanAcademyIE, @@ -686,10 +684,6 @@ from .mlb import ( ) from .mnet import MnetIE from .moevideo import MoeVideoIE -from .mofosex import ( - MofosexIE, - MofosexEmbedIE, -) from .mojvideo import MojvideoIE from .morningstar import MorningstarIE from .motherless import ( diff --git a/youtube_dl/extractor/extremetube.py b/youtube_dl/extractor/extremetube.py deleted file mode 100644 index acd4090fa..000000000 --- a/youtube_dl/extractor/extremetube.py +++ /dev/null @@ -1,50 +0,0 @@ -from __future__ import unicode_literals - -from ..utils import str_to_int -from .keezmovies import KeezMoviesIE - - -class ExtremeTubeIE(KeezMoviesIE): - _VALID_URL = r'https?://(?:www\.)?extremetube\.com/(?:[^/]+/)?video/(?P[^/#?&]+)' - _TESTS = [{ - 'url': 'http://www.extremetube.com/video/music-video-14-british-euro-brit-european-cumshots-swallow-652431', - 'md5': '92feaafa4b58e82f261e5419f39c60cb', - 'info_dict': { - 'id': 'music-video-14-british-euro-brit-european-cumshots-swallow-652431', - 'ext': 'mp4', - 'title': 'Music Video 14 british euro brit european cumshots swallow', - 'uploader': 'anonim', - 'view_count': int, - 'age_limit': 18, - } - }, { - 'url': 'http://www.extremetube.com/gay/video/abcde-1234', - 'only_matching': True, - }, { - 'url': 'http://www.extremetube.com/video/latina-slut-fucked-by-fat-black-dick', - 'only_matching': True, - }, { - 'url': 'http://www.extremetube.com/video/652431', - 'only_matching': True, - }] - - def _real_extract(self, url): - webpage, info = self._extract_info(url) - - if not info['title']: - info['title'] = self._search_regex( - r']+title="([^"]+)"[^>]*>', webpage, 'title') - - uploader = self._html_search_regex( - r'Uploaded by:\s*]+>\s*]+>(.+?)', - webpage, 'uploader', fatal=False) - view_count = str_to_int(self._search_regex( - r'Views:\s*]+>\s*<[^>]+>([\d,\.]+)[^/]+)-)?(?P\d+)' - _TESTS = [{ - 'url': 'https://www.keezmovies.com/video/arab-wife-want-it-so-bad-i-see-she-thirsty-and-has-tiny-money-18070681', - 'md5': '2ac69cdb882055f71d82db4311732a1a', - 'info_dict': { - 'id': '18070681', - 'display_id': 'arab-wife-want-it-so-bad-i-see-she-thirsty-and-has-tiny-money', - 'ext': 'mp4', - 'title': 'Arab wife want it so bad I see she thirsty and has tiny money.', - 'thumbnail': None, - 'view_count': int, - 'age_limit': 18, - } - }, { - 'url': 'http://www.keezmovies.com/video/18070681', - 'only_matching': True, - }] - - def _extract_info(self, url, fatal=True): - mobj = re.match(self._VALID_URL, url) - video_id = mobj.group('id') - display_id = (mobj.group('display_id') - if 'display_id' in mobj.groupdict() - else None) or mobj.group('id') - - webpage = self._download_webpage( - url, display_id, headers={'Cookie': 'age_verified=1'}) - - formats = [] - format_urls = set() - - title = None - thumbnail = None - duration = None - encrypted = False - - def extract_format(format_url, height=None): - format_url = url_or_none(format_url) - if not format_url or not format_url.startswith(('http', '//')): - return - if format_url in format_urls: - return - format_urls.add(format_url) - tbr = int_or_none(self._search_regex( - r'[/_](\d+)[kK][/_]', format_url, 'tbr', default=None)) - if not height: - height = int_or_none(self._search_regex( - r'[/_](\d+)[pP][/_]', format_url, 'height', default=None)) - if encrypted: - format_url = aes_decrypt_text( - video_url, title, 32).decode('utf-8') - formats.append({ - 'url': format_url, - 'format_id': '%dp' % height if height else None, - 'height': height, - 'tbr': tbr, - }) - - flashvars = self._parse_json( - self._search_regex( - r'flashvars\s*=\s*({.+?});', webpage, - 'flashvars', default='{}'), - display_id, fatal=False) - - if flashvars: - title = flashvars.get('video_title') - thumbnail = flashvars.get('image_url') - duration = int_or_none(flashvars.get('video_duration')) - encrypted = flashvars.get('encrypted') is True - for key, value in flashvars.items(): - mobj = re.search(r'quality_(\d+)[pP]', key) - if mobj: - extract_format(value, int(mobj.group(1))) - video_url = flashvars.get('video_url') - if video_url and determine_ext(video_url, None): - extract_format(video_url) - - video_url = self._html_search_regex( - r'flashvars\.video_url\s*=\s*(["\'])(?Phttp.+?)\1', - webpage, 'video url', default=None, group='url') - if video_url: - extract_format(compat_urllib_parse_unquote(video_url)) - - if not formats: - if 'title="This video is no longer available"' in webpage: - raise ExtractorError( - 'Video %s is no longer available' % video_id, expected=True) - - try: - self._sort_formats(formats) - except ExtractorError: - if fatal: - raise - - if not title: - title = self._html_search_regex( - r']*>([^<]+)', webpage, 'title') - - return webpage, { - 'id': video_id, - 'display_id': display_id, - 'title': strip_or_none(title), - 'thumbnail': thumbnail, - 'duration': duration, - 'age_limit': 18, - 'formats': formats, - } - - def _real_extract(self, url): - webpage, info = self._extract_info(url, fatal=False) - if not info['formats']: - return self.url_result(url, 'Generic') - info['view_count'] = str_to_int(self._search_regex( - r'([\d,.]+) Views?', webpage, 'view count', fatal=False)) - return info diff --git a/youtube_dl/extractor/mofosex.py b/youtube_dl/extractor/mofosex.py deleted file mode 100644 index 5234cac02..000000000 --- a/youtube_dl/extractor/mofosex.py +++ /dev/null @@ -1,79 +0,0 @@ -from __future__ import unicode_literals - -import re - -from .common import InfoExtractor -from ..utils import ( - int_or_none, - str_to_int, - unified_strdate, -) -from .keezmovies import KeezMoviesIE - - -class MofosexIE(KeezMoviesIE): - _VALID_URL = r'https?://(?:www\.)?mofosex\.com/videos/(?P\d+)/(?P[^/?#&.]+)\.html' - _TESTS = [{ - 'url': 'http://www.mofosex.com/videos/318131/amateur-teen-playing-and-masturbating-318131.html', - 'md5': '558fcdafbb63a87c019218d6e49daf8a', - 'info_dict': { - 'id': '318131', - 'display_id': 'amateur-teen-playing-and-masturbating-318131', - 'ext': 'mp4', - 'title': 'amateur teen playing and masturbating', - 'thumbnail': r're:^https?://.*\.jpg$', - 'upload_date': '20121114', - 'view_count': int, - 'like_count': int, - 'dislike_count': int, - 'age_limit': 18, - } - }, { - # This video is no longer available - 'url': 'http://www.mofosex.com/videos/5018/japanese-teen-music-video.html', - 'only_matching': True, - }] - - def _real_extract(self, url): - webpage, info = self._extract_info(url) - - view_count = str_to_int(self._search_regex( - r'VIEWS:\s*([\d,.]+)', webpage, 'view count', fatal=False)) - like_count = int_or_none(self._search_regex( - r'id=["\']amountLikes["\'][^>]*>(\d+)', webpage, - 'like count', fatal=False)) - dislike_count = int_or_none(self._search_regex( - r'id=["\']amountDislikes["\'][^>]*>(\d+)', webpage, - 'like count', fatal=False)) - upload_date = unified_strdate(self._html_search_regex( - r'Added:([^<]+)', webpage, 'upload date', fatal=False)) - - info.update({ - 'view_count': view_count, - 'like_count': like_count, - 'dislike_count': dislike_count, - 'upload_date': upload_date, - 'thumbnail': self._og_search_thumbnail(webpage), - }) - - return info - - -class MofosexEmbedIE(InfoExtractor): - _VALID_URL = r'https?://(?:www\.)?mofosex\.com/embed/?\?.*?\bvideoid=(?P\d+)' - _TESTS = [{ - 'url': 'https://www.mofosex.com/embed/?videoid=318131&referrer=KM', - 'only_matching': True, - }] - - @staticmethod - def _extract_urls(webpage): - return re.findall( - r']+\bsrc=["\']((?:https?:)?//(?:www\.)?mofosex\.com/embed/?\?.*?\bvideoid=\d+)', - webpage) - - def _real_extract(self, url): - video_id = self._match_id(url) - return self.url_result( - 'http://www.mofosex.com/videos/{0}/{0}.html'.format(video_id), - ie=MofosexIE.ie_key(), video_id=video_id) diff --git a/youtube_dl/extractor/tube8.py b/youtube_dl/extractor/tube8.py index db93b0182..0c22fae36 100644 --- a/youtube_dl/extractor/tube8.py +++ b/youtube_dl/extractor/tube8.py @@ -3,13 +3,19 @@ from __future__ import unicode_literals import re from ..utils import ( + determine_ext, + ExtractorError, int_or_none, str_to_int, + strip_or_none, + url_or_none, ) -from .keezmovies import KeezMoviesIE +from .common import InfoExtractor +from ..aes import aes_decrypt_text +from ..compat import compat_urllib_parse_unquote -class Tube8IE(KeezMoviesIE): +class Tube8IE(InfoExtractor): _VALID_URL = r'https?://(?:www\.)?tube8\.com/(?:[^/]+/)+(?P[^/]+)/(?P\d+)' _TESTS = [{ 'url': 'http://www.tube8.com/teen/kasia-music-video/229795/', @@ -37,6 +43,56 @@ class Tube8IE(KeezMoviesIE): r']+\bsrc=["\']((?:https?:)?//(?:www\.)?tube8\.com/embed/(?:[^/]+/)+\d+)', webpage) + def _extract_info(self, url, fatal=True): + mobj = re.match(self._VALID_URL, url) + video_id = mobj.group('id') + display_id = (mobj.group('display_id') + if 'display_id' in mobj.groupdict() + else None) or mobj.group('id') + + webpage = self._download_webpage( + url, display_id, headers={'Cookie': 'age_verified=1'}) + + formats = [] + format_urls = set() + + title = None + thumbnail = None + duration = None + encrypted = False + + def extract_format(format_url, height=None): + format_url = url_or_none(format_url) + if not format_url or not format_url.startswith(('http', '//')): + return + if format_url in format_urls: + return + format_urls.add(format_url) + tbr = int_or_none(self._search_regex( + r'[/_](\d+)[kK][/_]', format_url, 'tbr', default=None)) + if not height: + height = int_or_none(self._search_regex( + r'[/_](\d+)[pP][/_]', format_url, 'height', default=None)) + if encrypted: + format_url = aes_decrypt_text( + video_url, title, 32).decode('utf-8') + formats.append({ + 'url': format_url, + 'format_id': '%dp' % height if height else None, + 'height': height, + 'tbr': tbr, + }) + + flashvars = self._parse_json( + self._search_regex( + r'flashvars\s*=\s*({.+?});', webpage, + 'flashvars', default='{}'), + display_id, fatal=False) + + if flashvars: + title = flashvars.get('video_title') + thumbnail = flashvars.get('image_url') + def _real_extract(self, url): webpage, info = self._extract_info(url)