From 6b7441ed64458adbb8a1a4cb223db37a4e1193cb Mon Sep 17 00:00:00 2001 From: tabjy Date: Sun, 2 Oct 2022 03:11:53 -0400 Subject: [PATCH 01/13] [VXXX] Implement extractor for vxxx.com --- youtube_dl/extractor/extractors.py | 1 + youtube_dl/extractor/vxxx.py | 115 +++++++++++++++++++++++++++++ 2 files changed, 116 insertions(+) create mode 100644 youtube_dl/extractor/vxxx.py diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py index 751fc38b6..a567225a5 100644 --- a/youtube_dl/extractor/extractors.py +++ b/youtube_dl/extractor/extractors.py @@ -1518,6 +1518,7 @@ from .vvvvid import ( VVVVIDIE, VVVVIDShowIE, ) +from .vxxx import VXXXIE from .vyborymos import VyboryMosIE from .vzaar import VzaarIE from .wakanim import WakanimIE diff --git a/youtube_dl/extractor/vxxx.py b/youtube_dl/extractor/vxxx.py new file mode 100644 index 000000000..67571059a --- /dev/null +++ b/youtube_dl/extractor/vxxx.py @@ -0,0 +1,115 @@ +# coding: utf-8 +from __future__ import unicode_literals + +import re + +from .common import InfoExtractor +from ..utils import unified_timestamp, parse_duration + + +class VXXXIE(InfoExtractor): + _VALID_URL = r'https?://vxxx\.com/video-(?P\d+)' + _TESTS = [{ + 'url': 'https://vxxx.com/video-80747', + 'md5': '4736e868b0e008b4ff9dc09585c26c52', + 'info_dict': { + 'id': '80747', + 'ext': 'mp4', + 'title': 'Monica Aka Selina', + 'display_id': 'monica-aka-selina', + 'thumbnail': 'https://tn.vxxx.com/contents/videos_screenshots/80000/80747/420x236/1.jpg', + 'description': '', + 'timestamp': 1607167706, + 'upload_date': '20201205', + 'duration': 2373.0, + 'view_count': 1071, + 'like_count': 1, + 'dislike_count': 0, + 'average_rating': 5.0, + 'categories': ['Anal', 'Asian', 'BDSM', 'Brunette', 'Toys', + 'Fetish', 'HD', 'Interracial', 'MILF'], + }}] + + def _download_info_object(self, video_id): + return self._download_json( + 'https://vxxx.com/api/json/video/86400/0/{}/{}.json'.format( + int(video_id) // 10000 * 10000, + video_id, + ), video_id, headers={'Referer': 'https://vxxx.com'})['video'] + + def _download_format_object(self, video_id): + return self._download_json( + 'https://vxxx.com/api/videofile.php?video_id={}'.format(video_id), + video_id, + headers={'Referer': 'https://vxxx.com'} + ) + + def _get_video_host(self): + return 'vxxx.com' + + def _decode_base164(self, text): + alphabet = [*'АВСDЕFGHIJKLМNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789.,~'] + bit_str = '' + text_str = '' + + for char in text: + if char in alphabet: + bin_char = bin(alphabet.index(char)).lstrip("0b") + bin_char = bin_char.zfill(6) + bit_str += bin_char + + brackets = [bit_str[x:x + 8] for x in range(0, len(bit_str), 8)] + + for bracket in brackets: + text_str += chr(int(bracket, 2)) + + return text_str + + def _extract_info(self, url): + mobj = re.match(self._VALID_URL, url) + id = mobj.group('id') + + info_object = self._download_info_object(id) + + info = { + 'id': id, + 'title': info_object['title'], + 'display_id': info_object['dir'], + 'thumbnail': info_object['thumb'], + 'description': info_object['description'], + 'timestamp': unified_timestamp(info_object['post_date']), + 'duration': parse_duration(info_object['duration']), + 'view_count': int(info_object['statistics']['viewed']), + 'like_count': int(info_object['statistics']['likes']), + 'dislike_count': int(info_object['statistics']['dislikes']), + 'average_rating': float(info_object['statistics']['rating']), + 'categories': [category['title'] for category in info_object['categories'].values()], + 'formats': None + } + + qualities = { + '_hd.mp4': -1, + '_sd.mp4': -2 + } + + format_object = self._download_format_object(id) + formats = list(map(lambda f: { + 'url': "https://{}{}".format( + self._get_video_host(), + self._decode_base164(f['video_url']) + ), + 'format_id': f['format'], + 'quality': qualities.get(f['format'], -1) + }, format_object)) + self._sort_formats(formats) + + info['formats'] = formats + return info + + def _real_extract(self, url): + info = self._extract_info(url) + + if not info['formats']: + return self.url_result(url, 'Generic') + + return info From c0bda232e96548ddfe0628574eff7fd130f25fa9 Mon Sep 17 00:00:00 2001 From: tabjy Date: Sun, 2 Oct 2022 05:08:43 -0400 Subject: [PATCH 02/13] [VXXX] Fix the non-standard base164 encoding --- youtube_dl/extractor/vxxx.py | 54 +++++++++++++++++++----------------- 1 file changed, 29 insertions(+), 25 deletions(-) diff --git a/youtube_dl/extractor/vxxx.py b/youtube_dl/extractor/vxxx.py index 67571059a..6d7dc7bc6 100644 --- a/youtube_dl/extractor/vxxx.py +++ b/youtube_dl/extractor/vxxx.py @@ -1,6 +1,7 @@ # coding: utf-8 from __future__ import unicode_literals +import base64 import re from .common import InfoExtractor @@ -10,7 +11,7 @@ from ..utils import unified_timestamp, parse_duration class VXXXIE(InfoExtractor): _VALID_URL = r'https?://vxxx\.com/video-(?P\d+)' _TESTS = [{ - 'url': 'https://vxxx.com/video-80747', + 'url': 'https://vxxx.com/video-80747/', 'md5': '4736e868b0e008b4ff9dc09585c26c52', 'info_dict': { 'id': '80747', @@ -33,7 +34,7 @@ class VXXXIE(InfoExtractor): def _download_info_object(self, video_id): return self._download_json( 'https://vxxx.com/api/json/video/86400/0/{}/{}.json'.format( - int(video_id) // 10000 * 10000, + int(video_id) // 1000 * 1000, video_id, ), video_id, headers={'Referer': 'https://vxxx.com'})['video'] @@ -47,32 +48,34 @@ class VXXXIE(InfoExtractor): def _get_video_host(self): return 'vxxx.com' - def _decode_base164(self, text): - alphabet = [*'АВСDЕFGHIJKLМNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789.,~'] - bit_str = '' - text_str = '' + def _decode_base164(self, e): + """ + Some non-standard encoding called "base164" in the JavaScript code. It + is similar to base 64 with some alphabets replaced: + - "АВСЕМ" are Cyrillic letters instead of uppercase English letters + - "." is used instead of "+"; "," is used instead of "/" + - "~" is used for padding instead of "=" + """ - for char in text: - if char in alphabet: - bin_char = bin(alphabet.index(char)).lstrip("0b") - bin_char = bin_char.zfill(6) - bit_str += bin_char - - brackets = [bit_str[x:x + 8] for x in range(0, len(bit_str), 8)] - - for bracket in brackets: - text_str += chr(int(bracket, 2)) - - return text_str + return base64.b64decode(e + .replace("А", "A") + .replace("В", "B") + .replace("С", "C") + .replace("Е", "E") + .replace("М", "M") + .replace(".", "+") + .replace(",", "/") + .replace("~", "=") + ).decode() def _extract_info(self, url): - mobj = re.match(self._VALID_URL, url) - id = mobj.group('id') + matches = re.match(self._VALID_URL, url) + video_id = matches.group('id') - info_object = self._download_info_object(id) + info_object = self._download_info_object(video_id) info = { - 'id': id, + 'id': video_id, 'title': info_object['title'], 'display_id': info_object['dir'], 'thumbnail': info_object['thumb'], @@ -88,11 +91,12 @@ class VXXXIE(InfoExtractor): } qualities = { - '_hd.mp4': -1, - '_sd.mp4': -2 + '_fhd.mp4': -1, + '_hd.mp4': -2, + '_sd.mp4': -3 } - format_object = self._download_format_object(id) + format_object = self._download_format_object(video_id) formats = list(map(lambda f: { 'url': "https://{}{}".format( self._get_video_host(), From a59f77ebabc3ed4d7f6f2d0a43ba687ed0c76ef4 Mon Sep 17 00:00:00 2001 From: tabjy Date: Sun, 2 Oct 2022 05:26:06 -0400 Subject: [PATCH 03/13] [VXXX] Support "friend" site: bdsmx.tube --- youtube_dl/extractor/bdsmxtube.py | 39 ++++++++++++++++++++++++++++++ youtube_dl/extractor/extractors.py | 1 + youtube_dl/extractor/vxxx.py | 16 ++++++------ 3 files changed, 47 insertions(+), 9 deletions(-) create mode 100644 youtube_dl/extractor/bdsmxtube.py diff --git a/youtube_dl/extractor/bdsmxtube.py b/youtube_dl/extractor/bdsmxtube.py new file mode 100644 index 000000000..a7c5299ed --- /dev/null +++ b/youtube_dl/extractor/bdsmxtube.py @@ -0,0 +1,39 @@ +from .vxxx import VXXXIE + + +class BdsmxTubeIE(VXXXIE): + _VALID_URL = r'https?://bdsmx\.tube/video/(?P\d+)' + _TESTS = [{ + 'url': 'https://bdsmx.tube/video/127583/latex-puppy-leashed/', + 'md5': '06b6000c19207cb068bc0009f243345d', + 'info_dict': { + 'id': '127583', + 'ext': 'mp4', + 'title': 'Latex Puppy Leashed', + 'display_id': 'latex-puppy-leashed', + 'thumbnail': 'https://tn.bdsmx-porn.com/contents/videos_screenshots/127000/127583/480x270/1.jpg', + 'description': '', + 'timestamp': 1651003323, + 'upload_date': '20220426', + 'duration': 68.0, + 'categories': ['Asian', 'Brunette', 'Cosplay', 'Fetish', + 'Fuck Machine', 'Gagging', 'Japanese', + 'JAV Uncensored', 'Latex', 'Leather', 'POV']} + }] + + def _download_info_object(self, video_id): + return self._download_json( + 'https://bdsmx.tube/api/json/video/86400/0/{}/{}.json'.format( + int(video_id) // 1000 * 1000, + video_id, + ), video_id, headers={'Referer': 'https://bdsmx.tube'})['video'] + + def _download_format_object(self, video_id): + return self._download_json( + 'https://bdsmx.tube/api/videofile.php?video_id={}'.format(video_id), + video_id, + headers={'Referer': 'https://bdsmx.tube'} + ) + + def _get_video_host(self): + return 'bdsmx.tube' diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py index a567225a5..3eb9a11c8 100644 --- a/youtube_dl/extractor/extractors.py +++ b/youtube_dl/extractor/extractors.py @@ -106,6 +106,7 @@ from .bbc import ( BBCCoUkPlaylistIE, BBCIE, ) +from .bdsmxtube import BdsmxTubeIE from .beeg import BeegIE from .behindkink import BehindKinkIE from .bellmedia import BellMediaIE diff --git a/youtube_dl/extractor/vxxx.py b/youtube_dl/extractor/vxxx.py index 6d7dc7bc6..bcde6555f 100644 --- a/youtube_dl/extractor/vxxx.py +++ b/youtube_dl/extractor/vxxx.py @@ -23,10 +23,6 @@ class VXXXIE(InfoExtractor): 'timestamp': 1607167706, 'upload_date': '20201205', 'duration': 2373.0, - 'view_count': 1071, - 'like_count': 1, - 'dislike_count': 0, - 'average_rating': 5.0, 'categories': ['Anal', 'Asian', 'BDSM', 'Brunette', 'Toys', 'Fetish', 'HD', 'Interracial', 'MILF'], }}] @@ -50,8 +46,8 @@ class VXXXIE(InfoExtractor): def _decode_base164(self, e): """ - Some non-standard encoding called "base164" in the JavaScript code. It - is similar to base 64 with some alphabets replaced: + Some non-standard encoding called "base164" in the JavaScript code. It's + similar to the regular base64 with a slightly different alphabet: - "АВСЕМ" are Cyrillic letters instead of uppercase English letters - "." is used instead of "+"; "," is used instead of "/" - "~" is used for padding instead of "=" @@ -91,9 +87,11 @@ class VXXXIE(InfoExtractor): } qualities = { - '_fhd.mp4': -1, - '_hd.mp4': -2, - '_sd.mp4': -3 + '_fhd.mp4': -1, # 1080p + '_hd.mp4': -2, # 720p + '_hq.mp4': -2, # 720p + '_sd.mp4': -3, # 480p + '_lq.mp4': -3 # 480p } format_object = self._download_format_object(video_id) From ba4c5b3a2e466a6c8f881b27a1a35ca12ef01f10 Mon Sep 17 00:00:00 2001 From: tabjy Date: Sun, 2 Oct 2022 05:48:44 -0400 Subject: [PATCH 04/13] [VXXX] Support "friend" site: inporn.com --- youtube_dl/extractor/bdsmxtube.py | 3 +++ youtube_dl/extractor/extractors.py | 1 + youtube_dl/extractor/inporn.py | 41 ++++++++++++++++++++++++++++++ 3 files changed, 45 insertions(+) create mode 100644 youtube_dl/extractor/inporn.py diff --git a/youtube_dl/extractor/bdsmxtube.py b/youtube_dl/extractor/bdsmxtube.py index a7c5299ed..590e2f0a1 100644 --- a/youtube_dl/extractor/bdsmxtube.py +++ b/youtube_dl/extractor/bdsmxtube.py @@ -1,3 +1,6 @@ +# coding: utf-8 +from __future__ import unicode_literals + from .vxxx import VXXXIE diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py index 3eb9a11c8..a6be263bf 100644 --- a/youtube_dl/extractor/extractors.py +++ b/youtube_dl/extractor/extractors.py @@ -521,6 +521,7 @@ from .instagram import ( ) from .internazionale import InternazionaleIE from .internetvideoarchive import InternetVideoArchiveIE +from .inporn import InPornIE from .iprima import IPrimaIE from .iqiyi import IqiyiIE from .ir90tv import Ir90TvIE diff --git a/youtube_dl/extractor/inporn.py b/youtube_dl/extractor/inporn.py new file mode 100644 index 000000000..4bc5e052e --- /dev/null +++ b/youtube_dl/extractor/inporn.py @@ -0,0 +1,41 @@ +# coding: utf-8 +from __future__ import unicode_literals + +from .vxxx import VXXXIE + + +class InPornIE(VXXXIE): + _VALID_URL = r'https?://(?:www\.)?inporn\.com/video/(?P\d+)' + _TESTS = [{ + 'url': 'https://inporn.com/video/533613/2k-t-2nd-season-parm-151/', + 'md5': '111e5c4680b1fa5995144e101c521a4f', + 'info_dict': { + 'id': '533613', + 'ext': 'mp4', + 'title': '2k 美月まい - ガーリー系アパレルモt゙ルの挑発パンチラ 2nd Season [parm-151]', + 'display_id': '2k-t-2nd-season-parm-151', + 'thumbnail': 'https://tn.inporn.com/media/tn/533613_1.jpg', + 'description': '', + 'timestamp': 1664571262, + 'upload_date': '20220930', + 'duration': 480.0, + 'categories': ['Asian', 'Brunette', 'Casting', 'HD', 'Japanese', + 'JAV Uncensored']} + }] + + def _download_info_object(self, video_id): + return self._download_json( + 'https://inporn.com/api/json/video/86400/0/{}/{}.json'.format( + int(video_id) // 1000 * 1000, + video_id, + ), video_id, headers={'Referer': 'https://inporn.com'})['video'] + + def _download_format_object(self, video_id): + return self._download_json( + 'https://inporn.com/api/videofile.php?video_id={}'.format(video_id), + video_id, + headers={'Referer': 'https://inporn.com'} + ) + + def _get_video_host(self): + return 'inporn.com' From aaafaa2cb8a32362e50f9f541b3cf7e17cfb6986 Mon Sep 17 00:00:00 2001 From: tabjy Date: Sun, 2 Oct 2022 06:17:24 -0400 Subject: [PATCH 05/13] [VXXX] Support "friend" site: xmilf.com --- youtube_dl/extractor/extractors.py | 1 + youtube_dl/extractor/xmilf.py | 41 ++++++++++++++++++++++++++++++ 2 files changed, 42 insertions(+) create mode 100644 youtube_dl/extractor/xmilf.py diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py index a6be263bf..e797c76ec 100644 --- a/youtube_dl/extractor/extractors.py +++ b/youtube_dl/extractor/extractors.py @@ -1580,6 +1580,7 @@ from .ximalaya import ( XimalayaAlbumIE ) from .xminus import XMinusIE +from .xmilf import XMilfIE from .xnxx import XNXXIE from .xstream import XstreamIE from .xtube import XTubeUserIE, XTubeIE diff --git a/youtube_dl/extractor/xmilf.py b/youtube_dl/extractor/xmilf.py new file mode 100644 index 000000000..74c193437 --- /dev/null +++ b/youtube_dl/extractor/xmilf.py @@ -0,0 +1,41 @@ +# coding: utf-8 +from __future__ import unicode_literals + +from .vxxx import VXXXIE + + +class XMilfIE(VXXXIE): + _VALID_URL = r'https?://xmilf\.com/video/(?P\d+)' + _TESTS = [{ + 'url': 'https://xmilf.com/video/143777/big-boob-brunette-masturbates3/', + 'md5': 'a93d43a83042cb6e42103053d981de81', + 'info_dict': { + 'id': '143777', + 'ext': 'mp4', + 'title': 'Big Boob Brunette Masturbates', + 'display_id': 'big-boob-brunette-masturbates3', + 'thumbnail': 'https://tn.xmilf.com/contents/videos_screenshots/143000/143777/480x270/1.jpg', + 'description': '', + 'timestamp': 1662465481, + 'upload_date': '20220906', + 'duration': 480.0, + 'categories': ['Amateur', 'Big Tits', 'Brunette', 'Fetish', 'HD', 'Lingerie', 'MILF', 'Webcam'], + } + }] + + def _download_info_object(self, video_id): + return self._download_json( + 'https://xmilf.com/api/json/video/86400/0/{}/{}.json'.format( + int(video_id) // 1000 * 1000, + video_id, + ), video_id, headers={'Referer': 'https://xmilf.com'})['video'] + + def _download_format_object(self, video_id): + return self._download_json( + 'https://xmilf.com/api/videofile.php?video_id={}'.format(video_id), + video_id, + headers={'Referer': 'https://xmilf.com'} + ) + + def _get_video_host(self): + return 'xmilf.com' From a6a1c149d63def606d170cb6a58904acbdb7e505 Mon Sep 17 00:00:00 2001 From: tabjy Date: Sun, 2 Oct 2022 06:27:18 -0400 Subject: [PATCH 06/13] [VXXX] Support "friend" site: blackporn.tube --- youtube_dl/extractor/blackporntube.py | 43 +++++++++++++++++++++++++++ youtube_dl/extractor/extractors.py | 1 + youtube_dl/extractor/xmilf.py | 3 +- 3 files changed, 46 insertions(+), 1 deletion(-) create mode 100644 youtube_dl/extractor/blackporntube.py diff --git a/youtube_dl/extractor/blackporntube.py b/youtube_dl/extractor/blackporntube.py new file mode 100644 index 000000000..0cdff8233 --- /dev/null +++ b/youtube_dl/extractor/blackporntube.py @@ -0,0 +1,43 @@ +# coding: utf-8 +from __future__ import unicode_literals + +from .vxxx import VXXXIE + + +class BlackPornTubeIE(VXXXIE): + _VALID_URL = r'https?://blackporn\.tube/video/(?P\d+)' + _TESTS = [{ + 'url': 'https://blackporn.tube/video/10043813/young-ebony-babe-gets-super-wet/', + 'md5': 'f5c2652f686e66d453f6fede3bdba054', + 'info_dict': { + 'id': '10043813', + 'ext': 'mp4', + 'title': 'Young Ebony Babe Gets Super Wet', + 'display_id': 'young-ebony-babe-gets-super-wet', + 'thumbnail': 'https://tn.blackporn.tube/contents/videos_screenshots/10043000/10043813/480x270/1.jpg', + 'description': '', + 'timestamp': 1654806141, + 'upload_date': '20220609', + 'duration': 193.0, + 'categories': ['BDSM', 'Bondage', 'Celebrity', 'Ebony', 'Fetish', + 'Shibari Bondage', 'Solo Female', + 'Tattoo'] + } + }] + + def _download_info_object(self, video_id): + return self._download_json( + 'https://blackporn.tube/api/json/video/86400/0/{}/{}.json'.format( + int(video_id) // 1000 * 1000, + video_id, + ), video_id, headers={'Referer': 'https://blackporn.tube'})['video'] + + def _download_format_object(self, video_id): + return self._download_json( + 'https://blackporn.tube/api/videofile.php?video_id={}'.format(video_id), + video_id, + headers={'Referer': 'https://blackporn.tube'} + ) + + def _get_video_host(self): + return 'blackporn.tube' diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py index e797c76ec..f06fdf60a 100644 --- a/youtube_dl/extractor/extractors.py +++ b/youtube_dl/extractor/extractors.py @@ -139,6 +139,7 @@ from .bleacherreport import ( BleacherReportIE, BleacherReportCMSIE, ) +from .blackporntube import BlackPornTubeIE from .bloomberg import BloombergIE from .bokecc import BokeCCIE from .bongacams import BongaCamsIE diff --git a/youtube_dl/extractor/xmilf.py b/youtube_dl/extractor/xmilf.py index 74c193437..a85788bb0 100644 --- a/youtube_dl/extractor/xmilf.py +++ b/youtube_dl/extractor/xmilf.py @@ -19,7 +19,8 @@ class XMilfIE(VXXXIE): 'timestamp': 1662465481, 'upload_date': '20220906', 'duration': 480.0, - 'categories': ['Amateur', 'Big Tits', 'Brunette', 'Fetish', 'HD', 'Lingerie', 'MILF', 'Webcam'], + 'categories': ['Amateur', 'Big Tits', 'Brunette', 'Fetish', 'HD', + 'Lingerie', 'MILF', 'Webcam'], } }] From f2398c00702c2266b5824c41069b39ea821b4789 Mon Sep 17 00:00:00 2001 From: tabjy Date: Sun, 2 Oct 2022 06:41:08 -0400 Subject: [PATCH 07/13] [VXXX] Support "friend" site: mrgay.com --- youtube_dl/extractor/extractors.py | 1 + youtube_dl/extractor/mrgay.py | 42 ++++++++++++++++++++++++++++++ 2 files changed, 43 insertions(+) create mode 100644 youtube_dl/extractor/mrgay.py diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py index f06fdf60a..95cd463d0 100644 --- a/youtube_dl/extractor/extractors.py +++ b/youtube_dl/extractor/extractors.py @@ -675,6 +675,7 @@ from .metacritic import MetacriticIE from .mgoon import MgoonIE from .mgtv import MGTVIE from .miaopai import MiaoPaiIE +from .mrgay import MrGayIE from .microsoftvirtualacademy import ( MicrosoftVirtualAcademyIE, MicrosoftVirtualAcademyCourseIE, diff --git a/youtube_dl/extractor/mrgay.py b/youtube_dl/extractor/mrgay.py new file mode 100644 index 000000000..8d44463ab --- /dev/null +++ b/youtube_dl/extractor/mrgay.py @@ -0,0 +1,42 @@ +# coding: utf-8 +from __future__ import unicode_literals + +from .vxxx import VXXXIE + + +class MrGayIE(VXXXIE): + _VALID_URL = r'https?://mrgay\.com/video/(?P\d+)' + _TESTS = [{ + 'url': 'https://mrgay.com/video/10169199/jpn-crossdresser-6/', + 'md5': 'b2ff401f8a168007702f3f5cbecd7bc2', + 'info_dict': { + 'id': '10169199', + 'ext': 'mp4', + 'title': 'Jpn Crossdresser 6', + 'display_id': 'jpn-crossdresser-6', + 'thumbnail': 'https://tn.mrgay.com/media/tn/10169199_1.jpg', + 'description': '', + 'timestamp': 1651066888, + 'upload_date': '20220427', + 'duration': 834.0, + 'categories': ['Amateur', 'Asian', 'Brunette', 'Crossdressing', + 'Japanese', 'Webcam'], + } + }] + + def _download_info_object(self, video_id): + return self._download_json( + 'https://mrgay.com/api/json/video/86400/0/{}/{}.json'.format( + int(video_id) // 1000 * 1000, + video_id, + ), video_id, headers={'Referer': 'https://mrgay.com'})['video'] + + def _download_format_object(self, video_id): + return self._download_json( + 'https://mrgay.com/api/videofile.php?video_id={}'.format(video_id), + video_id, + headers={'Referer': 'https://mrgay.com'} + ) + + def _get_video_host(self): + return 'mrgay.com' From 9c5c7787925d008150ef24354145c37c70da51e1 Mon Sep 17 00:00:00 2001 From: tabjy Date: Sun, 2 Oct 2022 06:50:56 -0400 Subject: [PATCH 08/13] [VXXX] Explicitly set age_limit to 18 --- youtube_dl/extractor/bdsmxtube.py | 4 +++- youtube_dl/extractor/blackporntube.py | 3 ++- youtube_dl/extractor/inporn.py | 4 +++- youtube_dl/extractor/mrgay.py | 1 + youtube_dl/extractor/vxxx.py | 5 ++++- youtube_dl/extractor/xmilf.py | 1 + 6 files changed, 14 insertions(+), 4 deletions(-) diff --git a/youtube_dl/extractor/bdsmxtube.py b/youtube_dl/extractor/bdsmxtube.py index 590e2f0a1..cebb57edd 100644 --- a/youtube_dl/extractor/bdsmxtube.py +++ b/youtube_dl/extractor/bdsmxtube.py @@ -21,7 +21,9 @@ class BdsmxTubeIE(VXXXIE): 'duration': 68.0, 'categories': ['Asian', 'Brunette', 'Cosplay', 'Fetish', 'Fuck Machine', 'Gagging', 'Japanese', - 'JAV Uncensored', 'Latex', 'Leather', 'POV']} + 'JAV Uncensored', 'Latex', 'Leather', 'POV'], + 'age_limit': 18, + } }] def _download_info_object(self, video_id): diff --git a/youtube_dl/extractor/blackporntube.py b/youtube_dl/extractor/blackporntube.py index 0cdff8233..48780f310 100644 --- a/youtube_dl/extractor/blackporntube.py +++ b/youtube_dl/extractor/blackporntube.py @@ -21,7 +21,8 @@ class BlackPornTubeIE(VXXXIE): 'duration': 193.0, 'categories': ['BDSM', 'Bondage', 'Celebrity', 'Ebony', 'Fetish', 'Shibari Bondage', 'Solo Female', - 'Tattoo'] + 'Tattoo'], + 'age_limit': 18, } }] diff --git a/youtube_dl/extractor/inporn.py b/youtube_dl/extractor/inporn.py index 4bc5e052e..5b92d9ddb 100644 --- a/youtube_dl/extractor/inporn.py +++ b/youtube_dl/extractor/inporn.py @@ -20,7 +20,9 @@ class InPornIE(VXXXIE): 'upload_date': '20220930', 'duration': 480.0, 'categories': ['Asian', 'Brunette', 'Casting', 'HD', 'Japanese', - 'JAV Uncensored']} + 'JAV Uncensored'], + 'age_limit': 18, + }, }] def _download_info_object(self, video_id): diff --git a/youtube_dl/extractor/mrgay.py b/youtube_dl/extractor/mrgay.py index 8d44463ab..badc4553e 100644 --- a/youtube_dl/extractor/mrgay.py +++ b/youtube_dl/extractor/mrgay.py @@ -21,6 +21,7 @@ class MrGayIE(VXXXIE): 'duration': 834.0, 'categories': ['Amateur', 'Asian', 'Brunette', 'Crossdressing', 'Japanese', 'Webcam'], + 'age_limit': 18, } }] diff --git a/youtube_dl/extractor/vxxx.py b/youtube_dl/extractor/vxxx.py index bcde6555f..e53420a66 100644 --- a/youtube_dl/extractor/vxxx.py +++ b/youtube_dl/extractor/vxxx.py @@ -25,7 +25,9 @@ class VXXXIE(InfoExtractor): 'duration': 2373.0, 'categories': ['Anal', 'Asian', 'BDSM', 'Brunette', 'Toys', 'Fetish', 'HD', 'Interracial', 'MILF'], - }}] + 'age_limit': 18, + } + }] def _download_info_object(self, video_id): return self._download_json( @@ -83,6 +85,7 @@ class VXXXIE(InfoExtractor): 'dislike_count': int(info_object['statistics']['dislikes']), 'average_rating': float(info_object['statistics']['rating']), 'categories': [category['title'] for category in info_object['categories'].values()], + 'age_limit': 18, 'formats': None } diff --git a/youtube_dl/extractor/xmilf.py b/youtube_dl/extractor/xmilf.py index a85788bb0..2ee849d14 100644 --- a/youtube_dl/extractor/xmilf.py +++ b/youtube_dl/extractor/xmilf.py @@ -21,6 +21,7 @@ class XMilfIE(VXXXIE): 'duration': 480.0, 'categories': ['Amateur', 'Big Tits', 'Brunette', 'Fetish', 'HD', 'Lingerie', 'MILF', 'Webcam'], + 'age_limit': 18, } }] From 1e522505be842c55b187581e89d51c09f2f9421e Mon Sep 17 00:00:00 2001 From: tabjy Date: Mon, 3 Oct 2022 02:27:04 -0400 Subject: [PATCH 09/13] [VXXX] Switch to HSL for much faster downloads --- youtube_dl/extractor/bdsmxtube.py | 2 +- youtube_dl/extractor/blackporntube.py | 2 +- youtube_dl/extractor/inporn.py | 2 +- youtube_dl/extractor/mrgay.py | 2 +- youtube_dl/extractor/vxxx.py | 24 +++++++----------------- youtube_dl/extractor/xmilf.py | 2 +- 6 files changed, 12 insertions(+), 22 deletions(-) diff --git a/youtube_dl/extractor/bdsmxtube.py b/youtube_dl/extractor/bdsmxtube.py index cebb57edd..0507e95cd 100644 --- a/youtube_dl/extractor/bdsmxtube.py +++ b/youtube_dl/extractor/bdsmxtube.py @@ -8,7 +8,7 @@ class BdsmxTubeIE(VXXXIE): _VALID_URL = r'https?://bdsmx\.tube/video/(?P\d+)' _TESTS = [{ 'url': 'https://bdsmx.tube/video/127583/latex-puppy-leashed/', - 'md5': '06b6000c19207cb068bc0009f243345d', + 'md5': '79751d4ed75668afe07a660c4bcb2f1b', 'info_dict': { 'id': '127583', 'ext': 'mp4', diff --git a/youtube_dl/extractor/blackporntube.py b/youtube_dl/extractor/blackporntube.py index 48780f310..12168fcb9 100644 --- a/youtube_dl/extractor/blackporntube.py +++ b/youtube_dl/extractor/blackporntube.py @@ -8,7 +8,7 @@ class BlackPornTubeIE(VXXXIE): _VALID_URL = r'https?://blackporn\.tube/video/(?P\d+)' _TESTS = [{ 'url': 'https://blackporn.tube/video/10043813/young-ebony-babe-gets-super-wet/', - 'md5': 'f5c2652f686e66d453f6fede3bdba054', + 'md5': '4a4c126970f2f1453b8b2050947fc870', 'info_dict': { 'id': '10043813', 'ext': 'mp4', diff --git a/youtube_dl/extractor/inporn.py b/youtube_dl/extractor/inporn.py index 5b92d9ddb..ab68c9f02 100644 --- a/youtube_dl/extractor/inporn.py +++ b/youtube_dl/extractor/inporn.py @@ -8,7 +8,7 @@ class InPornIE(VXXXIE): _VALID_URL = r'https?://(?:www\.)?inporn\.com/video/(?P\d+)' _TESTS = [{ 'url': 'https://inporn.com/video/533613/2k-t-2nd-season-parm-151/', - 'md5': '111e5c4680b1fa5995144e101c521a4f', + 'md5': 'c358d1da6b451ebe7cfb00dd89741607', 'info_dict': { 'id': '533613', 'ext': 'mp4', diff --git a/youtube_dl/extractor/mrgay.py b/youtube_dl/extractor/mrgay.py index badc4553e..ad827f422 100644 --- a/youtube_dl/extractor/mrgay.py +++ b/youtube_dl/extractor/mrgay.py @@ -8,7 +8,7 @@ class MrGayIE(VXXXIE): _VALID_URL = r'https?://mrgay\.com/video/(?P\d+)' _TESTS = [{ 'url': 'https://mrgay.com/video/10169199/jpn-crossdresser-6/', - 'md5': 'b2ff401f8a168007702f3f5cbecd7bc2', + 'md5': 'b5780a9437c205b4bc87eb939b23e8ef', 'info_dict': { 'id': '10169199', 'ext': 'mp4', diff --git a/youtube_dl/extractor/vxxx.py b/youtube_dl/extractor/vxxx.py index e53420a66..9dd5cc8d8 100644 --- a/youtube_dl/extractor/vxxx.py +++ b/youtube_dl/extractor/vxxx.py @@ -12,7 +12,7 @@ class VXXXIE(InfoExtractor): _VALID_URL = r'https?://vxxx\.com/video-(?P\d+)' _TESTS = [{ 'url': 'https://vxxx.com/video-80747/', - 'md5': '4736e868b0e008b4ff9dc09585c26c52', + 'md5': '2f4bfd829b682ff9e3da1bda71b81b81', 'info_dict': { 'id': '80747', 'ext': 'mp4', @@ -89,26 +89,16 @@ class VXXXIE(InfoExtractor): 'formats': None } - qualities = { - '_fhd.mp4': -1, # 1080p - '_hd.mp4': -2, # 720p - '_hq.mp4': -2, # 720p - '_sd.mp4': -3, # 480p - '_lq.mp4': -3 # 480p - } - format_object = self._download_format_object(video_id) - formats = list(map(lambda f: { - 'url': "https://{}{}".format( + m3u8_formats = self._extract_m3u8_formats( + "https://{}{}&f=video.m3u8".format( self._get_video_host(), - self._decode_base164(f['video_url']) + self._decode_base164(format_object[0]['video_url']) ), - 'format_id': f['format'], - 'quality': qualities.get(f['format'], -1) - }, format_object)) - self._sort_formats(formats) + video_id, 'mp4') + self._sort_formats(m3u8_formats) + info['formats'] = m3u8_formats - info['formats'] = formats return info def _real_extract(self, url): diff --git a/youtube_dl/extractor/xmilf.py b/youtube_dl/extractor/xmilf.py index 2ee849d14..d3a749402 100644 --- a/youtube_dl/extractor/xmilf.py +++ b/youtube_dl/extractor/xmilf.py @@ -8,7 +8,7 @@ class XMilfIE(VXXXIE): _VALID_URL = r'https?://xmilf\.com/video/(?P\d+)' _TESTS = [{ 'url': 'https://xmilf.com/video/143777/big-boob-brunette-masturbates3/', - 'md5': 'a93d43a83042cb6e42103053d981de81', + 'md5': 'a196fe8daebe194a758754c81e9232ad', 'info_dict': { 'id': '143777', 'ext': 'mp4', From 8414d8d8f5f26a1f1a610e43fc40d53e83dfebc9 Mon Sep 17 00:00:00 2001 From: Kangcheng Xu <8033899+tabjy@users.noreply.github.com> Date: Sat, 29 Oct 2022 01:54:55 -0400 Subject: [PATCH 10/13] Apply suggestions from code review Co-authored-by: dirkf --- youtube_dl/extractor/bdsmxtube.py | 4 +- youtube_dl/extractor/extractors.py | 9 +++- youtube_dl/extractor/vxxx.py | 73 +++++++++++++++++------------- 3 files changed, 51 insertions(+), 35 deletions(-) diff --git a/youtube_dl/extractor/bdsmxtube.py b/youtube_dl/extractor/bdsmxtube.py index 0507e95cd..ef0769c03 100644 --- a/youtube_dl/extractor/bdsmxtube.py +++ b/youtube_dl/extractor/bdsmxtube.py @@ -28,14 +28,14 @@ class BdsmxTubeIE(VXXXIE): def _download_info_object(self, video_id): return self._download_json( - 'https://bdsmx.tube/api/json/video/86400/0/{}/{}.json'.format( + 'https://bdsmx.tube/api/json/video/86400/0/{0}/{1}.json'.format( int(video_id) // 1000 * 1000, video_id, ), video_id, headers={'Referer': 'https://bdsmx.tube'})['video'] def _download_format_object(self, video_id): return self._download_json( - 'https://bdsmx.tube/api/videofile.php?video_id={}'.format(video_id), + 'https://bdsmx.tube/api/videofile.php?video_id={0}'.format(video_id), video_id, headers={'Referer': 'https://bdsmx.tube'} ) diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py index 95cd463d0..df68e2f62 100644 --- a/youtube_dl/extractor/extractors.py +++ b/youtube_dl/extractor/extractors.py @@ -1522,7 +1522,14 @@ from .vvvvid import ( VVVVIDIE, VVVVIDShowIE, ) -from .vxxx import VXXXIE +from .vxxx import ( + BdsmxTubeIE, + BlackPornTubeIE, + InPornIE, + MrGayIE, + VXXXIE, + XMilfIE, +) from .vyborymos import VyboryMosIE from .vzaar import VzaarIE from .wakanim import WakanimIE diff --git a/youtube_dl/extractor/vxxx.py b/youtube_dl/extractor/vxxx.py index 9dd5cc8d8..014ae808e 100644 --- a/youtube_dl/extractor/vxxx.py +++ b/youtube_dl/extractor/vxxx.py @@ -5,7 +5,10 @@ import base64 import re from .common import InfoExtractor -from ..utils import unified_timestamp, parse_duration +from ..utils import ( + parse_duration, + unified_timestamp, +) class VXXXIE(InfoExtractor): @@ -31,67 +34,73 @@ class VXXXIE(InfoExtractor): def _download_info_object(self, video_id): return self._download_json( - 'https://vxxx.com/api/json/video/86400/0/{}/{}.json'.format( + self._INFO_OBJECT_URL_TMPL.format( + self._BASE_URL, int(video_id) // 1000 * 1000, video_id, - ), video_id, headers={'Referer': 'https://vxxx.com'})['video'] + ), video_id, headers={'Referer': self._BASE_URL})['video'] def _download_format_object(self, video_id): return self._download_json( - 'https://vxxx.com/api/videofile.php?video_id={}'.format(video_id), + self._FORMAT_OBJECT_URL_TMPL.format(self._BASE_URL, video_id), video_id, - headers={'Referer': 'https://vxxx.com'} + headers={'Referer': self._BASE_URL} ) - def _get_video_host(self): - return 'vxxx.com' + @classmethod + def _get_video_host(cls): + # or use the proper Python URL parsing functions + return cls._BASE_URL.split('//')[-1] def _decode_base164(self, e): """ Some non-standard encoding called "base164" in the JavaScript code. It's similar to the regular base64 with a slightly different alphabet: - - "АВСЕМ" are Cyrillic letters instead of uppercase English letters + - "АВСЕМ" are Cyrillic letters instead of uppercase Latin letters - "." is used instead of "+"; "," is used instead of "/" - "~" is used for padding instead of "=" """ - return base64.b64decode(e - .replace("А", "A") - .replace("В", "B") - .replace("С", "C") - .replace("Е", "E") - .replace("М", "M") - .replace(".", "+") - .replace(",", "/") - .replace("~", "=") + # using the kwarg to memoise the result + def get_trans_tbl(from_, to, tbl={}): + k = (from_, to) + if not tbl.get(k): + tbl[k] = string.maketrans(from_, to) + return tbl[k] + + # maybe for the 2nd arg: + # import unicodedata and + # ''.join((unicodedata.lookup('CYRILLIC CAPITAL LETTER ' + x) for x in ('A', 'BE', 'ES', 'IE', 'EM'))) + '+/=' + trans_tbl = get_trans_tbl('АBCEM.,~', 'ABCEM+/=') + return base64.b64decode(e.translate(trans_tbl) ).decode() def _extract_info(self, url): - matches = re.match(self._VALID_URL, url) - video_id = matches.group('id') + video_id = self._match_id(url) info_object = self._download_info_object(video_id) + title = info_object['title'] + stats = info_object.get('statistics') or {} info = { 'id': video_id, - 'title': info_object['title'], - 'display_id': info_object['dir'], - 'thumbnail': info_object['thumb'], - 'description': info_object['description'], - 'timestamp': unified_timestamp(info_object['post_date']), - 'duration': parse_duration(info_object['duration']), - 'view_count': int(info_object['statistics']['viewed']), - 'like_count': int(info_object['statistics']['likes']), - 'dislike_count': int(info_object['statistics']['dislikes']), - 'average_rating': float(info_object['statistics']['rating']), - 'categories': [category['title'] for category in info_object['categories'].values()], + 'title': title, + 'display_id': info_object.get('dir'), + 'thumbnail': url_or_none(info_object.get('thumb')), + 'description': strip_or_none(info_object('description')) or None, + 'timestamp': unified_timestamp(info_object.get('post_date')), + 'duration': parse_duration(info_object.get('duration')), + 'view_count': int_or_none(stats.get('viewed')), + 'like_count': int_or_none(stats.get('likes')), + 'dislike_count': int_or_none(stats.get('dislikes')), + 'average_rating': float_or_none(stats.get('rating')), + 'categories': [category['title'] for category in (info_object.get('categories') or {}).values() if category.get('title')], 'age_limit': 18, - 'formats': None } format_object = self._download_format_object(video_id) m3u8_formats = self._extract_m3u8_formats( - "https://{}{}&f=video.m3u8".format( + 'https://{0}{1}&f=video.m3u8'.format( self._get_video_host(), self._decode_base164(format_object[0]['video_url']) ), From 9d2b2f9b1e48db58a2293b8b9a55a19b4d293ed4 Mon Sep 17 00:00:00 2001 From: tabjy Date: Sat, 29 Oct 2022 02:35:12 -0400 Subject: [PATCH 11/13] [VXXX] Refactor and apply further code review suggestions --- youtube_dl/extractor/bdsmxtube.py | 44 ------- youtube_dl/extractor/blackporntube.py | 44 ------- youtube_dl/extractor/extractors.py | 5 - youtube_dl/extractor/inporn.py | 43 ------- youtube_dl/extractor/mrgay.py | 43 ------- youtube_dl/extractor/vxxx.py | 173 +++++++++++++++++++++----- youtube_dl/extractor/xmilf.py | 43 ------- 7 files changed, 144 insertions(+), 251 deletions(-) delete mode 100644 youtube_dl/extractor/bdsmxtube.py delete mode 100644 youtube_dl/extractor/blackporntube.py delete mode 100644 youtube_dl/extractor/inporn.py delete mode 100644 youtube_dl/extractor/mrgay.py delete mode 100644 youtube_dl/extractor/xmilf.py diff --git a/youtube_dl/extractor/bdsmxtube.py b/youtube_dl/extractor/bdsmxtube.py deleted file mode 100644 index ef0769c03..000000000 --- a/youtube_dl/extractor/bdsmxtube.py +++ /dev/null @@ -1,44 +0,0 @@ -# coding: utf-8 -from __future__ import unicode_literals - -from .vxxx import VXXXIE - - -class BdsmxTubeIE(VXXXIE): - _VALID_URL = r'https?://bdsmx\.tube/video/(?P\d+)' - _TESTS = [{ - 'url': 'https://bdsmx.tube/video/127583/latex-puppy-leashed/', - 'md5': '79751d4ed75668afe07a660c4bcb2f1b', - 'info_dict': { - 'id': '127583', - 'ext': 'mp4', - 'title': 'Latex Puppy Leashed', - 'display_id': 'latex-puppy-leashed', - 'thumbnail': 'https://tn.bdsmx-porn.com/contents/videos_screenshots/127000/127583/480x270/1.jpg', - 'description': '', - 'timestamp': 1651003323, - 'upload_date': '20220426', - 'duration': 68.0, - 'categories': ['Asian', 'Brunette', 'Cosplay', 'Fetish', - 'Fuck Machine', 'Gagging', 'Japanese', - 'JAV Uncensored', 'Latex', 'Leather', 'POV'], - 'age_limit': 18, - } - }] - - def _download_info_object(self, video_id): - return self._download_json( - 'https://bdsmx.tube/api/json/video/86400/0/{0}/{1}.json'.format( - int(video_id) // 1000 * 1000, - video_id, - ), video_id, headers={'Referer': 'https://bdsmx.tube'})['video'] - - def _download_format_object(self, video_id): - return self._download_json( - 'https://bdsmx.tube/api/videofile.php?video_id={0}'.format(video_id), - video_id, - headers={'Referer': 'https://bdsmx.tube'} - ) - - def _get_video_host(self): - return 'bdsmx.tube' diff --git a/youtube_dl/extractor/blackporntube.py b/youtube_dl/extractor/blackporntube.py deleted file mode 100644 index 12168fcb9..000000000 --- a/youtube_dl/extractor/blackporntube.py +++ /dev/null @@ -1,44 +0,0 @@ -# coding: utf-8 -from __future__ import unicode_literals - -from .vxxx import VXXXIE - - -class BlackPornTubeIE(VXXXIE): - _VALID_URL = r'https?://blackporn\.tube/video/(?P\d+)' - _TESTS = [{ - 'url': 'https://blackporn.tube/video/10043813/young-ebony-babe-gets-super-wet/', - 'md5': '4a4c126970f2f1453b8b2050947fc870', - 'info_dict': { - 'id': '10043813', - 'ext': 'mp4', - 'title': 'Young Ebony Babe Gets Super Wet', - 'display_id': 'young-ebony-babe-gets-super-wet', - 'thumbnail': 'https://tn.blackporn.tube/contents/videos_screenshots/10043000/10043813/480x270/1.jpg', - 'description': '', - 'timestamp': 1654806141, - 'upload_date': '20220609', - 'duration': 193.0, - 'categories': ['BDSM', 'Bondage', 'Celebrity', 'Ebony', 'Fetish', - 'Shibari Bondage', 'Solo Female', - 'Tattoo'], - 'age_limit': 18, - } - }] - - def _download_info_object(self, video_id): - return self._download_json( - 'https://blackporn.tube/api/json/video/86400/0/{}/{}.json'.format( - int(video_id) // 1000 * 1000, - video_id, - ), video_id, headers={'Referer': 'https://blackporn.tube'})['video'] - - def _download_format_object(self, video_id): - return self._download_json( - 'https://blackporn.tube/api/videofile.php?video_id={}'.format(video_id), - video_id, - headers={'Referer': 'https://blackporn.tube'} - ) - - def _get_video_host(self): - return 'blackporn.tube' diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py index df68e2f62..d74169aad 100644 --- a/youtube_dl/extractor/extractors.py +++ b/youtube_dl/extractor/extractors.py @@ -106,7 +106,6 @@ from .bbc import ( BBCCoUkPlaylistIE, BBCIE, ) -from .bdsmxtube import BdsmxTubeIE from .beeg import BeegIE from .behindkink import BehindKinkIE from .bellmedia import BellMediaIE @@ -139,7 +138,6 @@ from .bleacherreport import ( BleacherReportIE, BleacherReportCMSIE, ) -from .blackporntube import BlackPornTubeIE from .bloomberg import BloombergIE from .bokecc import BokeCCIE from .bongacams import BongaCamsIE @@ -522,7 +520,6 @@ from .instagram import ( ) from .internazionale import InternazionaleIE from .internetvideoarchive import InternetVideoArchiveIE -from .inporn import InPornIE from .iprima import IPrimaIE from .iqiyi import IqiyiIE from .ir90tv import Ir90TvIE @@ -675,7 +672,6 @@ from .metacritic import MetacriticIE from .mgoon import MgoonIE from .mgtv import MGTVIE from .miaopai import MiaoPaiIE -from .mrgay import MrGayIE from .microsoftvirtualacademy import ( MicrosoftVirtualAcademyIE, MicrosoftVirtualAcademyCourseIE, @@ -1589,7 +1585,6 @@ from .ximalaya import ( XimalayaAlbumIE ) from .xminus import XMinusIE -from .xmilf import XMilfIE from .xnxx import XNXXIE from .xstream import XstreamIE from .xtube import XTubeUserIE, XTubeIE diff --git a/youtube_dl/extractor/inporn.py b/youtube_dl/extractor/inporn.py deleted file mode 100644 index ab68c9f02..000000000 --- a/youtube_dl/extractor/inporn.py +++ /dev/null @@ -1,43 +0,0 @@ -# coding: utf-8 -from __future__ import unicode_literals - -from .vxxx import VXXXIE - - -class InPornIE(VXXXIE): - _VALID_URL = r'https?://(?:www\.)?inporn\.com/video/(?P\d+)' - _TESTS = [{ - 'url': 'https://inporn.com/video/533613/2k-t-2nd-season-parm-151/', - 'md5': 'c358d1da6b451ebe7cfb00dd89741607', - 'info_dict': { - 'id': '533613', - 'ext': 'mp4', - 'title': '2k 美月まい - ガーリー系アパレルモt゙ルの挑発パンチラ 2nd Season [parm-151]', - 'display_id': '2k-t-2nd-season-parm-151', - 'thumbnail': 'https://tn.inporn.com/media/tn/533613_1.jpg', - 'description': '', - 'timestamp': 1664571262, - 'upload_date': '20220930', - 'duration': 480.0, - 'categories': ['Asian', 'Brunette', 'Casting', 'HD', 'Japanese', - 'JAV Uncensored'], - 'age_limit': 18, - }, - }] - - def _download_info_object(self, video_id): - return self._download_json( - 'https://inporn.com/api/json/video/86400/0/{}/{}.json'.format( - int(video_id) // 1000 * 1000, - video_id, - ), video_id, headers={'Referer': 'https://inporn.com'})['video'] - - def _download_format_object(self, video_id): - return self._download_json( - 'https://inporn.com/api/videofile.php?video_id={}'.format(video_id), - video_id, - headers={'Referer': 'https://inporn.com'} - ) - - def _get_video_host(self): - return 'inporn.com' diff --git a/youtube_dl/extractor/mrgay.py b/youtube_dl/extractor/mrgay.py deleted file mode 100644 index ad827f422..000000000 --- a/youtube_dl/extractor/mrgay.py +++ /dev/null @@ -1,43 +0,0 @@ -# coding: utf-8 -from __future__ import unicode_literals - -from .vxxx import VXXXIE - - -class MrGayIE(VXXXIE): - _VALID_URL = r'https?://mrgay\.com/video/(?P\d+)' - _TESTS = [{ - 'url': 'https://mrgay.com/video/10169199/jpn-crossdresser-6/', - 'md5': 'b5780a9437c205b4bc87eb939b23e8ef', - 'info_dict': { - 'id': '10169199', - 'ext': 'mp4', - 'title': 'Jpn Crossdresser 6', - 'display_id': 'jpn-crossdresser-6', - 'thumbnail': 'https://tn.mrgay.com/media/tn/10169199_1.jpg', - 'description': '', - 'timestamp': 1651066888, - 'upload_date': '20220427', - 'duration': 834.0, - 'categories': ['Amateur', 'Asian', 'Brunette', 'Crossdressing', - 'Japanese', 'Webcam'], - 'age_limit': 18, - } - }] - - def _download_info_object(self, video_id): - return self._download_json( - 'https://mrgay.com/api/json/video/86400/0/{}/{}.json'.format( - int(video_id) // 1000 * 1000, - video_id, - ), video_id, headers={'Referer': 'https://mrgay.com'})['video'] - - def _download_format_object(self, video_id): - return self._download_json( - 'https://mrgay.com/api/videofile.php?video_id={}'.format(video_id), - video_id, - headers={'Referer': 'https://mrgay.com'} - ) - - def _get_video_host(self): - return 'mrgay.com' diff --git a/youtube_dl/extractor/vxxx.py b/youtube_dl/extractor/vxxx.py index 014ae808e..6b37c8128 100644 --- a/youtube_dl/extractor/vxxx.py +++ b/youtube_dl/extractor/vxxx.py @@ -2,12 +2,15 @@ from __future__ import unicode_literals import base64 -import re from .common import InfoExtractor from ..utils import ( + float_or_none, + int_or_none, parse_duration, + strip_or_none, unified_timestamp, + url_or_none, ) @@ -22,7 +25,7 @@ class VXXXIE(InfoExtractor): 'title': 'Monica Aka Selina', 'display_id': 'monica-aka-selina', 'thumbnail': 'https://tn.vxxx.com/contents/videos_screenshots/80000/80747/420x236/1.jpg', - 'description': '', + 'description': None, 'timestamp': 1607167706, 'upload_date': '20201205', 'duration': 2373.0, @@ -32,6 +35,10 @@ class VXXXIE(InfoExtractor): } }] + _BASE_URL = 'https://vxxx.com' + _INFO_OBJECT_URL_TMPL = '{0}/api/json/video/86400/0/{1}/{2}.json' + _FORMAT_OBJECT_URL_TMPL = '{0}/api/videofile.php?video_id={1}' + def _download_info_object(self, video_id): return self._download_json( self._INFO_OBJECT_URL_TMPL.format( @@ -47,11 +54,6 @@ class VXXXIE(InfoExtractor): headers={'Referer': self._BASE_URL} ) - @classmethod - def _get_video_host(cls): - # or use the proper Python URL parsing functions - return cls._BASE_URL.split('//')[-1] - def _decode_base164(self, e): """ Some non-standard encoding called "base164" in the JavaScript code. It's @@ -61,21 +63,19 @@ class VXXXIE(InfoExtractor): - "~" is used for padding instead of "=" """ - # using the kwarg to memoise the result - def get_trans_tbl(from_, to, tbl={}): - k = (from_, to) - if not tbl.get(k): - tbl[k] = string.maketrans(from_, to) - return tbl[k] + # using the kwarg to memoise the result + def get_trans_tbl(from_, to, tbl={}): + k = (from_, to) + if not tbl.get(k): + tbl[k] = str.maketrans(from_, to) + return tbl[k] - # maybe for the 2nd arg: - # import unicodedata and - # ''.join((unicodedata.lookup('CYRILLIC CAPITAL LETTER ' + x) for x in ('A', 'BE', 'ES', 'IE', 'EM'))) + '+/=' - trans_tbl = get_trans_tbl('АBCEM.,~', 'ABCEM+/=') - return base64.b64decode(e.translate(trans_tbl) - ).decode() + trans_tbl = get_trans_tbl( + '\u0410\u0412\u0421\u0415\u041c.,~', + 'ABCEM+/=') + return base64.b64decode(e.translate(trans_tbl)).decode() - def _extract_info(self, url): + def _real_extract(self, url): video_id = self._match_id(url) info_object = self._download_info_object(video_id) @@ -87,21 +87,22 @@ class VXXXIE(InfoExtractor): 'title': title, 'display_id': info_object.get('dir'), 'thumbnail': url_or_none(info_object.get('thumb')), - 'description': strip_or_none(info_object('description')) or None, + 'description': strip_or_none(info_object.get('description')) or None, 'timestamp': unified_timestamp(info_object.get('post_date')), 'duration': parse_duration(info_object.get('duration')), 'view_count': int_or_none(stats.get('viewed')), 'like_count': int_or_none(stats.get('likes')), 'dislike_count': int_or_none(stats.get('dislikes')), 'average_rating': float_or_none(stats.get('rating')), - 'categories': [category['title'] for category in (info_object.get('categories') or {}).values() if category.get('title')], + 'categories': [category['title'] for category in (info_object.get('categories') or {}).values() + if category.get('title')], 'age_limit': 18, } format_object = self._download_format_object(video_id) m3u8_formats = self._extract_m3u8_formats( - 'https://{0}{1}&f=video.m3u8'.format( - self._get_video_host(), + '{0}/{1}&f=video.m3u8'.format( + self._BASE_URL, self._decode_base164(format_object[0]['video_url']) ), video_id, 'mp4') @@ -110,10 +111,124 @@ class VXXXIE(InfoExtractor): return info - def _real_extract(self, url): - info = self._extract_info(url) - if not info['formats']: - return self.url_result(url, 'Generic') +class BdsmxTubeIE(VXXXIE): + _VALID_URL = r'https?://bdsmx\.tube/video/(?P\d+)' + _TESTS = [{ + 'url': 'https://bdsmx.tube/video/127583/latex-puppy-leashed/', + 'md5': '79751d4ed75668afe07a660c4bcb2f1b', + 'info_dict': { + 'id': '127583', + 'ext': 'mp4', + 'title': 'Latex Puppy Leashed', + 'display_id': 'latex-puppy-leashed', + 'thumbnail': 'https://tn.bdsmx-porn.com/contents/videos_screenshots/127000/127583/480x270/1.jpg', + 'description': None, + 'timestamp': 1651003323, + 'upload_date': '20220426', + 'duration': 68.0, + 'categories': ['Asian', 'Brunette', 'Cosplay', 'Fetish', + 'Fuck Machine', 'Gagging', 'Japanese', + 'JAV Uncensored', 'Latex', 'Leather', 'POV'], + 'age_limit': 18, + } + }] - return info + _BASE_URL = 'https://bdsmx.tube' + + +class BlackPornTubeIE(VXXXIE): + _VALID_URL = r'https?://blackporn\.tube/video/(?P\d+)' + _TESTS = [{ + 'url': 'https://blackporn.tube/video/10043813/young-ebony-babe-gets-super-wet/', + 'md5': '4a4c126970f2f1453b8b2050947fc870', + 'info_dict': { + 'id': '10043813', + 'ext': 'mp4', + 'title': 'Young Ebony Babe Gets Super Wet', + 'display_id': 'young-ebony-babe-gets-super-wet', + 'thumbnail': 'https://tn.blackporn.tube/contents/videos_screenshots/10043000/10043813/480x270/1.jpg', + 'description': None, + 'timestamp': 1654806141, + 'upload_date': '20220609', + 'duration': 193.0, + 'categories': ['BDSM', 'Bondage', 'Celebrity', 'Ebony', 'Fetish', + 'Shibari Bondage', 'Solo Female', + 'Tattoo'], + 'age_limit': 18, + } + }] + + _BASE_URL = 'https://blackporn.tube' + + +class InPornIE(VXXXIE): + _VALID_URL = r'https?://(?:www\.)?inporn\.com/video/(?P\d+)' + _TESTS = [{ + 'url': 'https://inporn.com/video/533613/2k-t-2nd-season-parm-151/', + 'md5': 'c358d1da6b451ebe7cfb00dd89741607', + 'info_dict': { + 'id': '533613', + 'ext': 'mp4', + 'title': '2k 美月まい - ガーリー系アパレルモt゙ルの挑発パンチラ 2nd Season [parm-151]', + 'display_id': '2k-t-2nd-season-parm-151', + 'thumbnail': 'https://tn.inporn.com/media/tn/533613_1.jpg', + 'description': None, + 'timestamp': 1664571262, + 'upload_date': '20220930', + 'duration': 480.0, + 'categories': ['Asian', 'Brunette', 'Casting', 'HD', 'Japanese', + 'JAV Uncensored'], + 'age_limit': 18, + }, + }] + + _BASE_URL = 'https://inporn.com' + + +class MrGayIE(VXXXIE): + _VALID_URL = r'https?://mrgay\.com/video/(?P\d+)' + _TESTS = [{ + 'url': 'https://mrgay.com/video/10169199/jpn-crossdresser-6/', + 'md5': 'b5780a9437c205b4bc87eb939b23e8ef', + 'info_dict': { + 'id': '10169199', + 'ext': 'mp4', + 'title': 'Jpn Crossdresser 6', + 'display_id': 'jpn-crossdresser-6', + 'thumbnail': 'https://tn.mrgay.com/media/tn/10169199_1.jpg', + 'description': None, + 'timestamp': 1651066888, + 'upload_date': '20220427', + 'duration': 834.0, + 'categories': ['Amateur', 'Asian', 'Brunette', 'Crossdressing', + 'Japanese', 'Webcam'], + 'age_limit': 18, + } + }] + + _BASE_URL = 'https://mrgay.com' + + +class XMilfIE(VXXXIE): + _VALID_URL = r'https?://xmilf\.com/video/(?P\d+)' + _TESTS = [{ + 'url': 'https://xmilf.com/video/143777/big-boob-brunette-masturbates3/', + 'md5': 'a196fe8daebe194a758754c81e9232ad', + 'info_dict': { + 'id': '143777', + 'ext': 'mp4', + 'title': 'Big Boob Brunette Masturbates', + 'display_id': 'big-boob-brunette-masturbates3', + 'thumbnail': 'https://tn.xmilf.com/contents/videos_screenshots/143000/143777/480x270/1.jpg', + 'description': None, + 'timestamp': 1662465481, + 'upload_date': '20220906', + 'duration': 480.0, + 'categories': ['Amateur', 'Big Tits', 'Brunette', 'Fetish', 'HD', + 'Lingerie', 'MILF', 'Webcam'], + 'age_limit': 18, + } + }] + + _BASE_URL = 'https://xmilf.com' diff --git a/youtube_dl/extractor/xmilf.py b/youtube_dl/extractor/xmilf.py deleted file mode 100644 index d3a749402..000000000 --- a/youtube_dl/extractor/xmilf.py +++ /dev/null @@ -1,43 +0,0 @@ -# coding: utf-8 -from __future__ import unicode_literals - -from .vxxx import VXXXIE - - -class XMilfIE(VXXXIE): - _VALID_URL = r'https?://xmilf\.com/video/(?P\d+)' - _TESTS = [{ - 'url': 'https://xmilf.com/video/143777/big-boob-brunette-masturbates3/', - 'md5': 'a196fe8daebe194a758754c81e9232ad', - 'info_dict': { - 'id': '143777', - 'ext': 'mp4', - 'title': 'Big Boob Brunette Masturbates', - 'display_id': 'big-boob-brunette-masturbates3', - 'thumbnail': 'https://tn.xmilf.com/contents/videos_screenshots/143000/143777/480x270/1.jpg', - 'description': '', - 'timestamp': 1662465481, - 'upload_date': '20220906', - 'duration': 480.0, - 'categories': ['Amateur', 'Big Tits', 'Brunette', 'Fetish', 'HD', - 'Lingerie', 'MILF', 'Webcam'], - 'age_limit': 18, - } - }] - - def _download_info_object(self, video_id): - return self._download_json( - 'https://xmilf.com/api/json/video/86400/0/{}/{}.json'.format( - int(video_id) // 1000 * 1000, - video_id, - ), video_id, headers={'Referer': 'https://xmilf.com'})['video'] - - def _download_format_object(self, video_id): - return self._download_json( - 'https://xmilf.com/api/videofile.php?video_id={}'.format(video_id), - video_id, - headers={'Referer': 'https://xmilf.com'} - ) - - def _get_video_host(self): - return 'xmilf.com' From 191d1d0a854ef6af05c8cec02e4c5a7048d72c62 Mon Sep 17 00:00:00 2001 From: tabjy Date: Wed, 2 Nov 2022 14:26:06 -0400 Subject: [PATCH 12/13] [VXXX] Remove supports for site missing DMCA notices --- youtube_dl/extractor/extractors.py | 3 - youtube_dl/extractor/vxxx.py | 105 +++++++++++++++-------------- 2 files changed, 55 insertions(+), 53 deletions(-) diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py index d74169aad..b1edba582 100644 --- a/youtube_dl/extractor/extractors.py +++ b/youtube_dl/extractor/extractors.py @@ -1519,12 +1519,9 @@ from .vvvvid import ( VVVVIDShowIE, ) from .vxxx import ( - BdsmxTubeIE, - BlackPornTubeIE, InPornIE, MrGayIE, VXXXIE, - XMilfIE, ) from .vyborymos import VyboryMosIE from .vzaar import VzaarIE diff --git a/youtube_dl/extractor/vxxx.py b/youtube_dl/extractor/vxxx.py index 6b37c8128..1c4bcb6ce 100644 --- a/youtube_dl/extractor/vxxx.py +++ b/youtube_dl/extractor/vxxx.py @@ -112,56 +112,6 @@ class VXXXIE(InfoExtractor): return info -class BdsmxTubeIE(VXXXIE): - _VALID_URL = r'https?://bdsmx\.tube/video/(?P\d+)' - _TESTS = [{ - 'url': 'https://bdsmx.tube/video/127583/latex-puppy-leashed/', - 'md5': '79751d4ed75668afe07a660c4bcb2f1b', - 'info_dict': { - 'id': '127583', - 'ext': 'mp4', - 'title': 'Latex Puppy Leashed', - 'display_id': 'latex-puppy-leashed', - 'thumbnail': 'https://tn.bdsmx-porn.com/contents/videos_screenshots/127000/127583/480x270/1.jpg', - 'description': None, - 'timestamp': 1651003323, - 'upload_date': '20220426', - 'duration': 68.0, - 'categories': ['Asian', 'Brunette', 'Cosplay', 'Fetish', - 'Fuck Machine', 'Gagging', 'Japanese', - 'JAV Uncensored', 'Latex', 'Leather', 'POV'], - 'age_limit': 18, - } - }] - - _BASE_URL = 'https://bdsmx.tube' - - -class BlackPornTubeIE(VXXXIE): - _VALID_URL = r'https?://blackporn\.tube/video/(?P\d+)' - _TESTS = [{ - 'url': 'https://blackporn.tube/video/10043813/young-ebony-babe-gets-super-wet/', - 'md5': '4a4c126970f2f1453b8b2050947fc870', - 'info_dict': { - 'id': '10043813', - 'ext': 'mp4', - 'title': 'Young Ebony Babe Gets Super Wet', - 'display_id': 'young-ebony-babe-gets-super-wet', - 'thumbnail': 'https://tn.blackporn.tube/contents/videos_screenshots/10043000/10043813/480x270/1.jpg', - 'description': None, - 'timestamp': 1654806141, - 'upload_date': '20220609', - 'duration': 193.0, - 'categories': ['BDSM', 'Bondage', 'Celebrity', 'Ebony', 'Fetish', - 'Shibari Bondage', 'Solo Female', - 'Tattoo'], - 'age_limit': 18, - } - }] - - _BASE_URL = 'https://blackporn.tube' - - class InPornIE(VXXXIE): _VALID_URL = r'https?://(?:www\.)?inporn\.com/video/(?P\d+)' _TESTS = [{ @@ -209,6 +159,60 @@ class MrGayIE(VXXXIE): _BASE_URL = 'https://mrgay.com' +# The following three extractors are for "friend" sites whose videos could be +# extracted in the same way, but unsupported by youtube-dl due to missing proper +# DMCA notices. Consider re-enable them if their DMCA pages become available. +class BdsmxTubeIE(VXXXIE): + _VALID_URL = r'https?://bdsmx\.tube/video/(?P\d+)' + _TESTS = [{ + 'url': 'https://bdsmx.tube/video/127583/latex-puppy-leashed/', + 'md5': '79751d4ed75668afe07a660c4bcb2f1b', + 'info_dict': { + 'id': '127583', + 'ext': 'mp4', + 'title': 'Latex Puppy Leashed', + 'display_id': 'latex-puppy-leashed', + 'thumbnail': 'https://tn.bdsmx-porn.com/contents/videos_screenshots/127000/127583/480x270/1.jpg', + 'description': None, + 'timestamp': 1651003323, + 'upload_date': '20220426', + 'duration': 68.0, + 'categories': ['Asian', 'Brunette', 'Cosplay', 'Fetish', + 'Fuck Machine', 'Gagging', 'Japanese', + 'JAV Uncensored', 'Latex', 'Leather', 'POV'], + 'age_limit': 18, + } + }] + _WORKING = False + + _BASE_URL = 'https://bdsmx.tube' + + +class BlackPornTubeIE(VXXXIE): + _VALID_URL = r'https?://blackporn\.tube/video/(?P\d+)' + _TESTS = [{ + 'url': 'https://blackporn.tube/video/10043813/young-ebony-babe-gets-super-wet/', + 'md5': '4a4c126970f2f1453b8b2050947fc870', + 'info_dict': { + 'id': '10043813', + 'ext': 'mp4', + 'title': 'Young Ebony Babe Gets Super Wet', + 'display_id': 'young-ebony-babe-gets-super-wet', + 'thumbnail': 'https://tn.blackporn.tube/contents/videos_screenshots/10043000/10043813/480x270/1.jpg', + 'description': None, + 'timestamp': 1654806141, + 'upload_date': '20220609', + 'duration': 193.0, + 'categories': ['BDSM', 'Bondage', 'Celebrity', 'Ebony', 'Fetish', + 'Shibari Bondage', 'Solo Female', + 'Tattoo'], + 'age_limit': 18, + } + }] + _WORKING = False + + _BASE_URL = 'https://blackporn.tube' + class XMilfIE(VXXXIE): _VALID_URL = r'https?://xmilf\.com/video/(?P\d+)' @@ -230,5 +234,6 @@ class XMilfIE(VXXXIE): 'age_limit': 18, } }] + _WORKING = False _BASE_URL = 'https://xmilf.com' From 76738e4832fda5c644362a0bd3dd29c874a35ce3 Mon Sep 17 00:00:00 2001 From: tabjy Date: Fri, 11 Nov 2022 16:06:28 -0500 Subject: [PATCH 13/13] [VXXX] fix liting --- youtube_dl/extractor/vxxx.py | 1 + 1 file changed, 1 insertion(+) diff --git a/youtube_dl/extractor/vxxx.py b/youtube_dl/extractor/vxxx.py index 1c4bcb6ce..9359d8d57 100644 --- a/youtube_dl/extractor/vxxx.py +++ b/youtube_dl/extractor/vxxx.py @@ -159,6 +159,7 @@ class MrGayIE(VXXXIE): _BASE_URL = 'https://mrgay.com' + # The following three extractors are for "friend" sites whose videos could be # extracted in the same way, but unsupported by youtube-dl due to missing proper # DMCA notices. Consider re-enable them if their DMCA pages become available.