1
0
Fork 0
mirror of https://github.com/ytdl-org/youtube-dl.git synced 2024-11-22 18:22:21 +00:00

accept playlist filename as commandline argument; refractor threading code

This commit is contained in:
Ravi 2011-07-12 20:52:57 -04:00
parent 23d09bfa5d
commit 167b0a9025

View file

@ -49,7 +49,7 @@ std_headers = {
simple_title_chars = string.ascii_letters.decode('ascii') + string.digits.decode('ascii') simple_title_chars = string.ascii_letters.decode('ascii') + string.digits.decode('ascii')
downloadqueue=Queue.Queue() downloadqueue = Queue.Queue()
def preferredencoding(): def preferredencoding():
"""Get preferred encoding. """Get preferred encoding.
@ -307,7 +307,6 @@ class FileDownloader(object):
self._num_downloads = 0 self._num_downloads = 0
self._screen_file = [sys.stdout, sys.stderr][params.get('logtostderr', False)] self._screen_file = [sys.stdout, sys.stderr][params.get('logtostderr', False)]
self.params = params self.params = params
self.queue=Queue.Queue
@staticmethod @staticmethod
def pmkdir(filename): def pmkdir(filename):
@ -607,11 +606,10 @@ class FileDownloader(object):
# Extract information from URL and process it # Extract information from URL and process it
ie.extract(url) ie.extract(url)
#parallel downloader needs dummy at the end to signal end of queue
#parallel downloader needs dummy at the end to signal end of queue #for the thread to exit
#for the thread to exit
for i in xrange(self.params.get('parallel')): for i in xrange(self.params.get('parallel')):
downloadqueue.put({'filename':None } ) downloadqueue.put({'filename':None })
# Suitable InfoExtractor had been found; go to next URL # Suitable InfoExtractor had been found; go to next URL
break break
@ -663,19 +661,15 @@ class FileDownloader(object):
return False return False
def _do_download(self, filename, url, player_url): def _do_download(self, filename, url, player_url):
if ( self.params.get('playlistfile') != None ): if (self.params.get('playlistfile') != None):
self.params.get('playlistfile').write(filename+"\n") self.params.get('playlistfile').write(filename+"\n")
self.params.get('playlistfile').flush() self.params.get('playlistfile').flush()
if self.params.get('parallel') > 0: if self.params.get('parallel') > 0:
downloadqueue.put({'filename':filename,'url':url,'player_url':player_url,'params':self.params}) downloadqueue.put({'filename':filename,'url':url,'player_url':player_url,'params':self.params})
return False return False
else: else:
self._do_real_download(filename, url, player_url) self._do_real_download(filename, url, player_url)
def _do_real_download(self, filename, url, player_url): def _do_real_download(self, filename, url, player_url):
# Check file already present # Check file already present
if self.params.get('continuedl', False) and os.path.isfile(filename) and not self.params.get('nopart', False): if self.params.get('continuedl', False) and os.path.isfile(filename) and not self.params.get('nopart', False):
@ -809,26 +803,19 @@ class FileDownloader(object):
return True return True
class FileDownloadHelper(threading.Thread): def threadedFileDownloader():
"""File Downloader that does threaded download if needed. """File Downloader that does threaded download if needed.
Download parameters are added to downloadqueue in FileDownloader class, Download parameters are added to downloadqueue in FileDownloader class,
which each thread waits on and calls FileDownloader._do_real_download which each thread waits on and calls FileDownloader._do_real_download
Individual threads are created in main function. Individual threads are created in main function.
""" """
while True:
def __init__(self): d = downloadqueue.get()
threading.Thread.__init__(self) if (d['filename'] == None):
break
fd=FileDownloader(d['params'])
def run(self): fd._do_real_download(d['filename'],d['url'],d['player_url'])
while True: downloadqueue.task_done()
d=downloadqueue.get()
if ( d['filename'] == None):
break
self.params=d['params']
fd=FileDownloader(d['params'])
fd._do_real_download(d['filename'],d['url'],d['player_url'])
downloadqueue.task_done()
class InfoExtractor(object): class InfoExtractor(object):
@ -2796,7 +2783,7 @@ if __name__ == '__main__':
parser.add_option('-P','--parallel', parser.add_option('-P','--parallel',
type="int",dest='parallel',help='Number of parallel downloads',default=0) type="int",dest='parallel',help='Number of parallel downloads',default=0)
parser.add_option('-s', '--save-playlist', parser.add_option('-s', '--save-playlist',
action='store_true', dest='saveplaylist', help='Save file list to a playlist file') action='store', dest='saveplaylist', help='Save file list to a playlist file')
@ -2966,9 +2953,13 @@ if __name__ == '__main__':
facebook_ie = FacebookIE() facebook_ie = FacebookIE()
generic_ie = GenericIE() generic_ie = GenericIE()
playlistfile=None playlistfile = None
if ( opts.saveplaylist): if (opts.saveplaylist != None):
playlistfile=open("playlist.m3u","w") if(opts.saveplaylist.find(".") == -1 ):
playlist_filename = opts.saveplaylist + ".m3u"
else:
playlist_filename = opts.saveplaylist
playlistfile=open(playlist_filename,"w")
# File downloader # File downloader
fd = FileDownloader({ fd = FileDownloader({
@ -3007,7 +2998,7 @@ if __name__ == '__main__':
'nopart': opts.nopart, 'nopart': opts.nopart,
'updatetime': opts.updatetime, 'updatetime': opts.updatetime,
'parallel': opts.parallel, 'parallel': opts.parallel,
'playlistfile':playlistfile 'playlistfile': playlistfile
}) })
fd.add_info_extractor(youtube_search_ie) fd.add_info_extractor(youtube_search_ie)
fd.add_info_extractor(youtube_pl_ie) fd.add_info_extractor(youtube_pl_ie)
@ -3036,14 +3027,14 @@ if __name__ == '__main__':
update_self(fd, sys.argv[0]) update_self(fd, sys.argv[0])
#create downloader threads that wait for URLs #create downloader threads that wait for URLs
downloadparallel=opts.parallel downloadparallel = opts.parallel
threads=[] threads = []
if downloadparallel > 0: if downloadparallel > 0:
for threadcount in xrange(downloadparallel): for threadcount in xrange(downloadparallel):
d=FileDownloadHelper() t = threading.Thread(target=threadedFileDownloader)
d.setDaemon(True) t.setDaemon(True)
d.start() t.start()
threads.append(d) threads.append(t)
# Maybe do nothing # Maybe do nothing
if len(all_urls) < 1: if len(all_urls) < 1:
@ -3055,14 +3046,8 @@ if __name__ == '__main__':
#wait for download threads to terminate #wait for download threads to terminate
if downloadparallel > 0: if downloadparallel > 0:
for threadcount in xrange(downloadparallel): for t in threads:
while True: t.join(2**32)
if( not threads[threadcount].isAlive()):
break
time.sleep(1)
for threadcount in xrange(downloadparallel):
threads[threadcount].join()
# Dump cookie jar if requested # Dump cookie jar if requested
if opts.cookiefile is not None: if opts.cookiefile is not None: