depau/youtube-dl
1
0
Fork 0
mirror of https://github.com/ytdl-org/youtube-dl.git synced 2024-11-26 04:02:11 +00:00
Code Issues Releases Wiki Activity

[downloader/dash:hls] Respect --fragment-retries and --skip-unavailable-fragments (Closes #10165, closes #10448)

Browse source
This commit is contained in:
Sergey M․ 2016-08-27 04:55:55 +07:00
parent 9603b66012
commit 25afc2a783
No known key found for this signature in database
GPG key ID: 2C393E0F18A9236D
2 changed files with 41 additions and 12 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

12
youtube_dl/downloader/dash.py
View file

@ -38,6 +38,7 @@ class DashSegmentsFD(FragmentFD):
segments_filenames = [] segments_filenames = []
fragment_retries = self.params.get('fragment_retries', 0) fragment_retries = self.params.get('fragment_retries', 0)
skip_unavailable_fragments = self.params.get('skip_unavailable_fragments', True)
def append_url_to_file(target_url, tmp_filename, segment_name): def append_url_to_file(target_url, tmp_filename, segment_name):
target_filename = '%s-%s' % (tmp_filename, segment_name) target_filename = '%s-%s' % (tmp_filename, segment_name)
@ -52,19 +53,20 @@ class DashSegmentsFD(FragmentFD):
down.close() down.close()
segments_filenames.append(target_sanitized) segments_filenames.append(target_sanitized)
break break
except (compat_urllib_error.HTTPError, ) as err: except compat_urllib_error.HTTPError:
# YouTube may often return 404 HTTP error for a fragment causing the # YouTube may often return 404 HTTP error for a fragment causing the
# whole download to fail. However if the same fragment is immediately # whole download to fail. However if the same fragment is immediately
# retried with the same request data this usually succeeds (1-2 attemps # retried with the same request data this usually succeeds (1-2 attemps
# is usually enough) thus allowing to download the whole file successfully. # is usually enough) thus allowing to download the whole file successfully.
# So, we will retry all fragments that fail with 404 HTTP error for now. # To be future-proof we will retry all fragments that fail with any
if err.code != 404: # HTTP error.
raise
# Retry fragment
count += 1 count += 1
if count <= fragment_retries: if count <= fragment_retries:
self.report_retry_fragment(segment_name, count, fragment_retries) self.report_retry_fragment(segment_name, count, fragment_retries)
if count > fragment_retries: if count > fragment_retries:
if skip_unavailable_fragments:
self.report_skip_fragment(segment_name)
return
self.report_error('giving up after %s fragment retries' % fragment_retries) self.report_error('giving up after %s fragment retries' % fragment_retries)
return False return False

31
youtube_dl/downloader/hls.py
View file

@ -13,6 +13,7 @@ from .fragment import FragmentFD
from .external import FFmpegFD from .external import FFmpegFD
from ..compat import ( from ..compat import (
compat_urllib_error,
compat_urlparse, compat_urlparse,
compat_struct_pack, compat_struct_pack,
) )
@ -83,6 +84,10 @@ class HlsFD(FragmentFD):
self._prepare_and_start_frag_download(ctx) self._prepare_and_start_frag_download(ctx)
fragment_retries = self.params.get('fragment_retries', 0)
skip_unavailable_fragments = self.params.get('skip_unavailable_fragments', True)
test = self.params.get('test', False)
extra_query = None extra_query = None
extra_param_to_segment_url = info_dict.get('extra_param_to_segment_url') extra_param_to_segment_url = info_dict.get('extra_param_to_segment_url')
if extra_param_to_segment_url: if extra_param_to_segment_url:
@ -99,15 +104,37 @@ class HlsFD(FragmentFD):
line line
if re.match(r'^https?://', line) if re.match(r'^https?://', line)
else compat_urlparse.urljoin(man_url, line)) else compat_urlparse.urljoin(man_url, line))
frag_filename = '%s-Frag%d' % (ctx['tmpfilename'], i) frag_name = 'Frag%d' % i
frag_filename = '%s-%s' % (ctx['tmpfilename'], frag_name)
if extra_query: if extra_query:
frag_url = update_url_query(frag_url, extra_query) frag_url = update_url_query(frag_url, extra_query)
count = 0
while count <= fragment_retries:
try:
success = ctx['dl'].download(frag_filename, {'url': frag_url}) success = ctx['dl'].download(frag_filename, {'url': frag_url})
if not success: if not success:
return False return False
down, frag_sanitized = sanitize_open(frag_filename, 'rb') down, frag_sanitized = sanitize_open(frag_filename, 'rb')
frag_content = down.read() frag_content = down.read()
down.close() down.close()
break
except compat_urllib_error.HTTPError:
# Unavailable (possibly temporary) fragments may be served.
# First we try to retry then either skip or abort.
# See https://github.com/rg3/youtube-dl/issues/10165,
# https://github.com/rg3/youtube-dl/issues/10448).
count += 1
if count <= fragment_retries:
self.report_retry_fragment(frag_name, count, fragment_retries)
if count > fragment_retries:
if skip_unavailable_fragments:
i += 1
media_sequence += 1
self.report_skip_fragment(frag_name)
continue
self.report_error(
'giving up after %s fragment retries' % fragment_retries)
return False
if decrypt_info['METHOD'] == 'AES-128': if decrypt_info['METHOD'] == 'AES-128':
iv = decrypt_info.get('IV') or compat_struct_pack('>8xq', media_sequence) iv = decrypt_info.get('IV') or compat_struct_pack('>8xq', media_sequence)
frag_content = AES.new( frag_content = AES.new(
@ -115,7 +142,7 @@ class HlsFD(FragmentFD):
ctx['dest_stream'].write(frag_content) ctx['dest_stream'].write(frag_content)
frags_filenames.append(frag_sanitized) frags_filenames.append(frag_sanitized)
# We only download the first fragment during the test # We only download the first fragment during the test
if self.params.get('test', False): if test:
break break
i += 1 i += 1
media_sequence += 1 media_sequence += 1