mirror of
https://github.com/ytdl-org/youtube-dl.git
synced 2024-11-25 19:52:11 +00:00
[SendtoNews] Improve _VALID_URL, fix handling API result, add/fix tests
Media links now come in `configuration.sources.src` of playlist item
This commit is contained in:
dirkf
parent
b6a24ad93d
commit
ebf0fcd916
1 changed files with 94 additions and 39 deletions
|
|
@ -5,43 +5,45 @@ import re
|
|||
|
||||
from .common import InfoExtractor
|
||||
from ..utils import (
|
||||
dict_get,
|
||||
ExtractorError,
|
||||
float_or_none,
|
||||
parse_iso8601,
|
||||
update_url_query,
|
||||
int_or_none,
|
||||
determine_ext,
|
||||
determine_protocol,
|
||||
strip_or_none,
|
||||
try_get,
|
||||
unescapeHTML,
|
||||
urljoin,
|
||||
)
|
||||
|
||||
|
||||
class SendtoNewsIE(InfoExtractor):
|
||||
_VALID_URL = r'https?://embed\.sendtonews\.com/player2/embedplayer\.php\?.*\bSC=(?P<id>[0-9A-Za-z-]+)'
|
||||
|
||||
_TEST = {
|
||||
# TODO handle items with ?fk=XXXX6789&cid=1234 -> SC=XXXX6789-???????-1234
|
||||
_VALID_URL = r'https?://embed\.sendtonews\.com/(?:player\d/embed(?:player|code)\.(?:php|js)|oembed/?)\?.*\bSC=(?P<id>[\w-]+)'
|
||||
_TESTS = [{
|
||||
# From http://cleveland.cbslocal.com/2016/05/16/indians-score-season-high-15-runs-in-blowout-win-over-reds-rapid-reaction/
|
||||
'url': 'http://embed.sendtonews.com/player2/embedplayer.php?SC=GxfCe0Zo7D-175909-5588&type=single&autoplay=on&sound=YES',
|
||||
'info_dict': {
|
||||
'id': 'GxfCe0Zo7D-175909-5588'
|
||||
},
|
||||
'playlist_count': 8,
|
||||
# test the first video only to prevent lengthy tests
|
||||
'playlist': [{
|
||||
'info_dict': {
|
||||
'id': '240385',
|
||||
'ext': 'mp4',
|
||||
'title': 'Indians introduce Encarnacion',
|
||||
'description': 'Indians president of baseball operations Chris Antonetti and Edwin Encarnacion discuss the slugger\'s three-year contract with Cleveland',
|
||||
'duration': 137.898,
|
||||
'thumbnail': r're:https?://.*\.jpg$',
|
||||
'upload_date': '20170105',
|
||||
'timestamp': 1483649762,
|
||||
},
|
||||
}],
|
||||
'params': {
|
||||
# m3u8 download
|
||||
'skip_download': True,
|
||||
'playlist_count': 10,
|
||||
}, {
|
||||
'url': 'https://embed.sendtonews.com/player4/embedplayer.php?SC=mq3wIKSb68-1206898-8402&type=single',
|
||||
'info_dict': {
|
||||
'id': '1752278',
|
||||
'ext': 'mp4',
|
||||
'title': 'Las vegas homebuilders had banner sales year in 2021, and other top stories from January 24, 2022.',
|
||||
'description': 'LAS VEGAS HOMEBUILDERS HAD BANNER SALES YEAR IN 2021., and other top stories from January 24, 2022.',
|
||||
'timestamp': 1643063702,
|
||||
'upload_date': '20220124',
|
||||
'thumbnail': r're:https?://.*\.(?:png|jpg)$',
|
||||
'categories': ['Business'],
|
||||
'tags': list,
|
||||
},
|
||||
}
|
||||
}]
|
||||
|
||||
_URL_TEMPLATE = '//embed.sendtonews.com/player2/embedplayer.php?SC=%s'
|
||||
|
||||
|
|
@ -59,17 +61,62 @@ class SendtoNewsIE(InfoExtractor):
|
|||
playlist_id = self._match_id(url)
|
||||
|
||||
data_url = update_url_query(
|
||||
url.replace('embedplayer.php', 'data_read.php'),
|
||||
{'cmd': 'loadInitial'})
|
||||
re.sub(
|
||||
r'(?P<player>player\d)?(?<!/)/(?P<embed>embed.+?|oembed/)\?',
|
||||
lambda m: '/%s/data_read.php?' % ((m.group('player') or 'player4'), ),
|
||||
url),
|
||||
{'cmd': 'loadInitial', 'type': 'single', })
|
||||
playlist_data = self._download_json(data_url, playlist_id)
|
||||
playlist = try_get(playlist_data, lambda x: x['playlistData'][0], (dict, list)) or {}
|
||||
if isinstance(playlist, dict):
|
||||
err = playlist.get('error', 'No or invalid data returned from API')
|
||||
raise ExtractorError(err)
|
||||
|
||||
entries = []
|
||||
for video in playlist_data['playlistData'][0]:
|
||||
info_dict = self._parse_jwplayer_data(
|
||||
video['jwconfiguration'],
|
||||
require_title=False, m3u8_id='hls', rtmp_params={'no_resume': True})
|
||||
info_dict = {}
|
||||
for video in playlist:
|
||||
try:
|
||||
err = video.get('error')
|
||||
if err and video.get('S_ID') is not None:
|
||||
e = ExtractorError(err)
|
||||
e.msg = err
|
||||
raise e
|
||||
except AttributeError:
|
||||
continue
|
||||
except ExtractorError as e:
|
||||
self.report_warning(e.msg, playlist_id)
|
||||
continue
|
||||
if 'jwconfiguration' in video:
|
||||
info_dict.update(self._parse_jwplayer_data(
|
||||
video['jwconfiguration'],
|
||||
require_title=False, m3u8_id='hls', rtmp_params={'no_resume': True}))
|
||||
elif 'configuration' not in video:
|
||||
continue
|
||||
else:
|
||||
fmt_url = urljoin(
|
||||
url,
|
||||
try_get(video, lambda x: x['configuration']['sources']['src']))
|
||||
if not fmt_url:
|
||||
continue
|
||||
video_id = strip_or_none(video.get('SM_ID') or video['configuration']['mediaid'])
|
||||
title = strip_or_none(video.get('S_headLine') or video['configuration']['title'])
|
||||
if not video_id or not title:
|
||||
continue
|
||||
ext = determine_ext(fmt_url)
|
||||
if ext == 'm3u8':
|
||||
formats = self._extract_m3u8_formats(
|
||||
fmt_url, playlist_id, 'mp4', entry_protocol='m3u8_native',
|
||||
m3u8_id='hls', fatal=False)
|
||||
else:
|
||||
formats = [{
|
||||
'url': fmt_url,
|
||||
'ext': ext,
|
||||
'width': int_or_none(video.get('SM_M_VIDEO_WIDTH')),
|
||||
'height': int_or_none(video.get('SM_M_VIDEO_HEIGHT')),
|
||||
}]
|
||||
info_dict['formats'] = formats
|
||||
|
||||
for f in info_dict['formats']:
|
||||
for f in info_dict.get('formats') or []:
|
||||
if f.get('tbr'):
|
||||
continue
|
||||
tbr = int_or_none(self._search_regex(
|
||||
|
|
@ -83,23 +130,31 @@ class SendtoNewsIE(InfoExtractor):
|
|||
self._sort_formats(info_dict['formats'], ('tbr', 'height', 'width', 'format_id'))
|
||||
|
||||
thumbnails = []
|
||||
if video.get('thumbnailUrl'):
|
||||
for tn_id, tn in (('poster', video['configuration'].get('poster')),
|
||||
('normal', video.get('thumbnailUrl')),
|
||||
('small', video.get('smThumbnailUrl'))):
|
||||
tn = urljoin(url, tn)
|
||||
if not tn:
|
||||
continue
|
||||
thumbnails.append({
|
||||
'id': 'normal',
|
||||
'url': video['thumbnailUrl'],
|
||||
})
|
||||
if video.get('smThumbnailUrl'):
|
||||
thumbnails.append({
|
||||
'id': 'small',
|
||||
'url': video['smThumbnailUrl'],
|
||||
'id': tn_id,
|
||||
'url': tn,
|
||||
})
|
||||
info_dict.update({
|
||||
'title': video['S_headLine'].strip(),
|
||||
'description': unescapeHTML(video.get('S_fullStory')),
|
||||
'id': video_id,
|
||||
'title': title,
|
||||
'description': unescapeHTML(dict_get(video, ('S_fullStory', 'S_shortSummary'))),
|
||||
'thumbnails': thumbnails,
|
||||
'duration': float_or_none(video.get('SM_length')),
|
||||
'duration': float_or_none(
|
||||
dict_get(video, ('SM_length', 'SM_M_LENGTH'))
|
||||
or video['configuration'].get('duration')),
|
||||
'timestamp': parse_iso8601(video.get('S_sysDate'), delimiter=' '),
|
||||
'tags': [t for t in video.get('S_tags', '').split(',') if t],
|
||||
'categories': [c for c in video.get('S_category', '').split(',') if c],
|
||||
})
|
||||
entries.append(info_dict)
|
||||
|
||||
if len(entries) == 1:
|
||||
entries[0]['display_id'] = playlist_id
|
||||
return entries[0]
|
||||
return self.playlist_result(entries, playlist_id)
|
||||
|
|
|
|||
Loading…
Reference in a new issue