From c278f31d62ec63ad110a934bcbffd9131af207a8 Mon Sep 17 00:00:00 2001 From: dirkf Date: Sun, 10 Apr 2022 14:19:35 +0100 Subject: [PATCH 1/2] [Porn91] Update extractor --- youtube_dl/extractor/porn91.py | 57 +++++++++++++++++++++++++--------- 1 file changed, 42 insertions(+), 15 deletions(-) diff --git a/youtube_dl/extractor/porn91.py b/youtube_dl/extractor/porn91.py index 20eac647a..b299c7b9a 100644 --- a/youtube_dl/extractor/porn91.py +++ b/youtube_dl/extractor/porn91.py @@ -1,11 +1,21 @@ # coding: utf-8 from __future__ import unicode_literals +import re + from .common import InfoExtractor +from ..compat import ( + compat_urllib_parse_unquote, +) from ..utils import ( + clean_html, + extract_attributes, + get_elements_by_class, parse_duration, - int_or_none, ExtractorError, + str_to_int, + strip_or_none, + unified_strdate, ) @@ -15,13 +25,15 @@ class Porn91IE(InfoExtractor): _TEST = { 'url': 'http://91porn.com/view_video.php?viewkey=7e42283b4f5ab36da134', - 'md5': '7fcdb5349354f40d41689bd0fa8db05a', + 'md5': 'd869db281402e0ef4ddef3c38b866f86', 'info_dict': { 'id': '7e42283b4f5ab36da134', 'title': '18岁大一漂亮学妹，水嫩性感，再爽一次！', 'ext': 'mp4', 'duration': 431, 'age_limit': 18, + 'upload_date': '20150520', + 'uploader': '千岁九王爷', } } @@ -35,29 +47,44 @@ class Porn91IE(InfoExtractor): if '作为游客，你每天只可观看10个视频' in webpage: raise ExtractorError('91 Porn says: Daily limit 10 videos exceeded', expected=True) - title = self._search_regex( - r'
([^<]+)
', webpage, 'title') - title = title.replace('\n', '') + title = self._html_search_regex( + r'(?s)]*>\s*(.+?)\s*(?:Chinese\s+homemade\s+video\s*)?]+id=["\']fm-video_link[^>]+>([^<]+)', - webpage, 'video link') - videopage = self._download_webpage(video_link_url, video_id) + r'''document\s*\.\s*write\s*\(\s*strencode2\s*\((?P"|')(?P[%\da-fA-F]+)(?P=q)\s*\)''', + webpage, 'video link', group='enc_str') + video_link_url = compat_urllib_parse_unquote(video_link_url) - info_dict = self._parse_html5_media_entries(url, videopage, video_id)[0] + info_dict = self._parse_html5_media_entries(url, '' % (video_link_url, ), video_id)[0] - duration = parse_duration(self._search_regex( - r'时长:\s*\s*(\d+:\d+)', webpage, 'duration', fatal=False)) + FIELD_MAP = { + '时长': ('duration', parse_duration, 'Runtime', ), + '查看': ('view_count', str_to_int, 'Views', ), + '留言': ('comment_count', str_to_int, 'Comments', ), + '收藏': ('like_count', str_to_int, 'Favorites', ), + '添加时间': ('upload_date', unified_strdate, 'Added', ), + # same as title for en, not description for cn_CN + '__ignore__': ('description', strip_or_none, 'Description', ), + '作者': ('uploader', strip_or_none, 'From', ), + } + for elt in get_elements_by_class('info', re.sub(r'\s*]*?>', '', webpage)) or []: + elt = re.split(r':\s*', clean_html(elt), 1) + if len(elt) != 2 or elt[1] == '': + continue + parm = FIELD_MAP.get(elt[0].strip()) + if parm and elt[1] is not None: + info_dict[parm[0]] = parm[1](elt[1]) if parm[1] else elt[1] - comment_count = int_or_none(self._search_regex( - r'留言:\s*\s*(\d+)', webpage, 'comment count', fatal=False)) + thumbnail = extract_attributes( + self._search_regex( + r'''(]*\bid\s*=\s*(?P"|')player_one(?P=q)[^>]*>)''', + webpage, 'poster', default='')).get('poster') info_dict.update({ 'id': video_id, 'title': title, - 'duration': duration, - 'comment_count': comment_count, 'age_limit': self._rta_search(webpage), + 'thumbnail': thumbnail, }) return info_dict From e84b4d1c9ea50e11b687df93f3407bfabba9b729 Mon Sep 17 00:00:00 2001 From: dirkf Date: Tue, 19 Apr 2022 04:05:23 +0100 Subject: [PATCH 2/2] Sanitise info HTML for yt-dlp's get_elements_by_class() --- youtube_dl/extractor/porn91.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/youtube_dl/extractor/porn91.py b/youtube_dl/extractor/porn91.py index b299c7b9a..340903f45 100644 --- a/youtube_dl/extractor/porn91.py +++ b/youtube_dl/extractor/porn91.py @@ -39,6 +39,7 @@ class Porn91IE(InfoExtractor): def _real_extract(self, url): video_id = self._match_id(url) + # set language for page to be extracted self._set_cookie('91porn.com', 'language', 'cn_CN') webpage = self._download_webpage( @@ -57,7 +58,9 @@ class Porn91IE(InfoExtractor): info_dict = self._parse_html5_media_entries(url, '' % (video_link_url, ), video_id)[0] + # extract various fields in Name: value FIELD_MAP = { + # cn_CN name: (yt-dl key, value parser, en name) '时长': ('duration', parse_duration, 'Runtime', ), '查看': ('view_count', str_to_int, 'Views', ), '留言': ('comment_count', str_to_int, 'Comments', ), @@ -67,7 +70,14 @@ class Porn91IE(InfoExtractor): '__ignore__': ('description', strip_or_none, 'Description', ), '作者': ('uploader', strip_or_none, 'From', ), } - for elt in get_elements_by_class('info', re.sub(r'\s*]*?>', '', webpage)) or []: + # yt-dl's original implementation of get_elements_by_class() uses regex + # yt-dlp uses an actual HTML parser, and can be confused by bad HTML fragments + for elt in get_elements_by_class( + 'info', + # concatenate s ... + re.sub(r'(?i)\s*]*?>', '', + # ... and strip out possibly unbalanced for yt-dlp + re.sub(r'(?i)(?:]*?>|)', '', webpage))) or []: elt = re.split(r':\s*', clean_html(elt), 1) if len(elt) != 2 or elt[1] == '': continue