From df8db1aa2107f204fa14c157d7a536e45ceb65c4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?= Date: Tue, 26 Feb 2013 23:33:58 +0100 Subject: [PATCH 01/14] Create extract_info method --- youtube_dl/FileDownloader.py | 85 +++++++++++++++++++----------------- 1 file changed, 44 insertions(+), 41 deletions(-) diff --git a/youtube_dl/FileDownloader.py b/youtube_dl/FileDownloader.py index 192ad37d2..b26c34729 100644 --- a/youtube_dl/FileDownloader.py +++ b/youtube_dl/FileDownloader.py @@ -377,6 +377,44 @@ class FileDownloader(object): if re.search(rejecttitle, title, re.IGNORECASE): return u'"' + title + '" title matched reject pattern "' + rejecttitle + '"' return None + + def extract_info(self, url): + ''' + Returns a list with a dictionary for each video we find. + ''' + suitable_found = False + for ie in self._ies: + # Go to next InfoExtractor if not suitable + if not ie.suitable(url): + continue + + # Warn if the _WORKING attribute is False + if not ie.working(): + self.to_stderr(u'WARNING: the program functionality for this site has been marked as broken, ' + u'and will probably not work. If you want to go on, use the -i option.') + + # Suitable InfoExtractor found + suitable_found = True + + # Extract information from URL and process it + try: + videos = ie.extract(url) + for video in videos or []: + if not 'extractor' in video: + #The extractor has already been set somewher else + video['extractor'] = ie.IE_NAME + return videos + except ExtractorError as de: # An error we somewhat expected + self.trouble(u'ERROR: ' + compat_str(de), de.format_traceback()) + break + except Exception as e: + if self.params.get('ignoreerrors', False): + self.trouble(u'ERROR: ' + compat_str(e), tb=compat_str(traceback.format_exc())) + break + else: + raise + if not suitable_found: + self.trouble(u'ERROR: no suitable InfoExtractor: %s' % url) def process_info(self, info_dict): """Process a single dictionary returned by an InfoExtractor.""" @@ -488,49 +526,14 @@ class FileDownloader(object): raise SameFileError(self.params['outtmpl']) for url in url_list: - suitable_found = False - for ie in self._ies: - # Go to next InfoExtractor if not suitable - if not ie.suitable(url): - continue + videos = self.extract_info(url) - # Warn if the _WORKING attribute is False - if not ie.working(): - self.to_stderr(u'WARNING: the program functionality for this site has been marked as broken, ' - u'and will probably not work. If you want to go on, use the -i option.') - - # Suitable InfoExtractor found - suitable_found = True - - # Extract information from URL and process it + for video in videos or []: try: - videos = ie.extract(url) - except ExtractorError as de: # An error we somewhat expected - self.trouble(u'ERROR: ' + compat_str(de), de.format_traceback()) - break - except Exception as e: - if self.params.get('ignoreerrors', False): - self.trouble(u'ERROR: ' + compat_str(e), tb=compat_str(traceback.format_exc())) - break - else: - raise - - if len(videos or []) > 1 and self.fixed_template(): - raise SameFileError(self.params['outtmpl']) - - for video in videos or []: - video['extractor'] = ie.IE_NAME - try: - self.increment_downloads() - self.process_info(video) - except UnavailableVideoError: - self.trouble(u'\nERROR: unable to download video') - - # Suitable InfoExtractor had been found; go to next URL - break - - if not suitable_found: - self.trouble(u'ERROR: no suitable InfoExtractor: %s' % url) + self.increment_downloads() + self.process_info(video) + except UnavailableVideoError: + self.trouble(u'\nERROR: unable to download video') return self._download_retcode From 631f73978c0ee851950ac697dfd73f9092abd3c3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?= Date: Mon, 4 Mar 2013 22:16:42 +0100 Subject: [PATCH 02/14] Add a method for extracting info from a list of urls --- youtube_dl/FileDownloader.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/youtube_dl/FileDownloader.py b/youtube_dl/FileDownloader.py index b26c34729..f668b362b 100644 --- a/youtube_dl/FileDownloader.py +++ b/youtube_dl/FileDownloader.py @@ -415,6 +415,14 @@ class FileDownloader(object): raise if not suitable_found: self.trouble(u'ERROR: no suitable InfoExtractor: %s' % url) + def extract_info_iterable(self, urls): + ''' + Return the videos founded for the urls + ''' + results = [] + for url in urls: + results.extend(self.extract_info(url)) + return results def process_info(self, info_dict): """Process a single dictionary returned by an InfoExtractor.""" From 597cc8a45536aa4207c5ffc3e421fcebf2e08fe6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?= Date: Tue, 5 Mar 2013 11:58:01 +0100 Subject: [PATCH 03/14] Use extract_info in YoutubePlaylist and YoutubeSearch --- test/test_youtube_lists.py | 16 +++++++++------- youtube_dl/InfoExtractors.py | 8 +++----- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/test/test_youtube_lists.py b/test/test_youtube_lists.py index f4705bc5b..055bf69c8 100644 --- a/test/test_youtube_lists.py +++ b/test/test_youtube_lists.py @@ -10,6 +10,7 @@ sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) from youtube_dl.InfoExtractors import YoutubeUserIE, YoutubePlaylistIE, YoutubeIE from youtube_dl.utils import * +from youtube_dl.FileDownloader import FileDownloader PARAMETERS_FILE = os.path.join(os.path.dirname(os.path.abspath(__file__)), "parameters.json") with io.open(PARAMETERS_FILE, encoding='utf-8') as pf: @@ -22,7 +23,7 @@ proxy_handler = compat_urllib_request.ProxyHandler() opener = compat_urllib_request.build_opener(proxy_handler, cookie_processor, YoutubeDLHandler()) compat_urllib_request.install_opener(opener) -class FakeDownloader(object): +class FakeDownloader(FileDownloader): def __init__(self): self.result = [] self.params = parameters @@ -30,15 +31,16 @@ class FakeDownloader(object): print(s) def trouble(self, s): raise Exception(s) - def download(self, x): - self.result.append(x) + def extract_info(self, url): + self.result.append(url) + return url class TestYoutubeLists(unittest.TestCase): def test_youtube_playlist(self): dl = FakeDownloader() ie = YoutubePlaylistIE(dl) ie.extract('https://www.youtube.com/playlist?list=PLwiyx1dc3P2JR9N8gQaQN_BCvlSlap7re') - ytie_results = [YoutubeIE()._extract_id(r[0]) for r in dl.result] + ytie_results = [YoutubeIE()._extract_id(url) for url in dl.result] self.assertEqual(ytie_results, [ 'bV9L5Ht9LgY', 'FXxLjLQi3Fg', 'tU3Bgo5qJZE']) def test_issue_673(self): @@ -58,7 +60,7 @@ class TestYoutubeLists(unittest.TestCase): dl = FakeDownloader() ie = YoutubePlaylistIE(dl) ie.extract('https://www.youtube.com/playlist?list=PLwP_SiAcdui0KVebT0mU9Apz359a4ubsC') - ytie_results = [YoutubeIE()._extract_id(r[0]) for r in dl.result] + ytie_results = [YoutubeIE()._extract_id(url) for url in dl.result] self.assertFalse('pElCt5oNDuI' in ytie_results) self.assertFalse('KdPEApIVdWM' in ytie_results) @@ -67,9 +69,9 @@ class TestYoutubeLists(unittest.TestCase): ie = YoutubePlaylistIE(dl) # TODO find a > 100 (paginating?) videos course ie.extract('https://www.youtube.com/course?list=ECUl4u3cNGP61MdtwGTqZA0MreSaDybji8') - self.assertEqual(YoutubeIE()._extract_id(dl.result[0][0]), 'j9WZyLZCBzs') + self.assertEqual(YoutubeIE()._extract_id(dl.result[0]), 'j9WZyLZCBzs') self.assertEqual(len(dl.result), 25) - self.assertEqual(YoutubeIE()._extract_id(dl.result[-1][0]), 'rYefUsYuEp0') + self.assertEqual(YoutubeIE()._extract_id(dl.result[-1]), 'rYefUsYuEp0') def test_youtube_channel(self): # I give up, please find a channel that does paginate and test this like test_youtube_playlist_long diff --git a/youtube_dl/InfoExtractors.py b/youtube_dl/InfoExtractors.py index 7ce84fe79..8a7694a76 100755 --- a/youtube_dl/InfoExtractors.py +++ b/youtube_dl/InfoExtractors.py @@ -1756,9 +1756,7 @@ class YoutubePlaylistIE(InfoExtractor): else: self._downloader.to_screen(u'[youtube] PL %s: Found %i videos, downloading %i' % (playlist_id, total, len(videos))) - for video in videos: - self._downloader.download([video]) - return + return self._downloader.extract_info_iterable(videos) class YoutubeChannelIE(InfoExtractor): @@ -1892,8 +1890,8 @@ class YoutubeUserIE(InfoExtractor): self._downloader.to_screen(u"[youtube] user %s: Collected %d video ids (downloading %d of them)" % (username, all_ids_count, len(video_ids))) - for video_id in video_ids: - self._downloader.download(['http://www.youtube.com/watch?v=%s' % video_id]) + urls = ['http://www.youtube.com/watch?v=%s' % video_id for video_id in video_ids] + return self._downloader.extract_info_iterable(urls) class BlipTVUserIE(InfoExtractor): From f6e6da9525150487476d4990693eedf73acffab1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?= Date: Tue, 5 Mar 2013 12:26:18 +0100 Subject: [PATCH 04/14] Use extract_info in BlipTV User and Youtube Channel --- youtube_dl/InfoExtractors.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/youtube_dl/InfoExtractors.py b/youtube_dl/InfoExtractors.py index 8a7694a76..d79f6068f 100755 --- a/youtube_dl/InfoExtractors.py +++ b/youtube_dl/InfoExtractors.py @@ -1806,9 +1806,8 @@ class YoutubeChannelIE(InfoExtractor): self._downloader.to_screen(u'[youtube] Channel %s: Found %i videos' % (channel_id, len(video_ids))) - for id in video_ids: - self._downloader.download(['http://www.youtube.com/watch?v=%s' % id]) - return + urls = ['http://www.youtube.com/watch?v=%s' % id for id in video_ids] + return self._downloader.extract_info_iterable(urls) class YoutubeUserIE(InfoExtractor): @@ -1981,8 +1980,8 @@ class BlipTVUserIE(InfoExtractor): self._downloader.to_screen(u"[%s] user %s: Collected %d video ids (downloading %d of them)" % (self.IE_NAME, username, all_ids_count, len(video_ids))) - for video_id in video_ids: - self._downloader.download([u'http://blip.tv/'+video_id]) + urls = [u'http://blip.tv/%s' % video_id for video_id in video_ids] + return self._downloader.extract_info_iterable(urls) class DepositFilesIE(InfoExtractor): From 6ac7f082c469b3b2153735ae8475e1d0fc8b5439 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?= Date: Tue, 5 Mar 2013 20:14:32 +0100 Subject: [PATCH 05/14] `extract_info` now expects `ie.extract` to return a list in the format proposed in issue 608. Each element should have a '_type' key specifying if it's a video, an url or a playlist. `extract_info` will process each element to get the full info --- youtube_dl/FileDownloader.py | 32 ++++++++++++++++++++++++++------ 1 file changed, 26 insertions(+), 6 deletions(-) diff --git a/youtube_dl/FileDownloader.py b/youtube_dl/FileDownloader.py index 9b630c123..68fad11bc 100644 --- a/youtube_dl/FileDownloader.py +++ b/youtube_dl/FileDownloader.py @@ -410,12 +410,9 @@ class FileDownloader(object): # Extract information from URL and process it try: - videos = ie.extract(url) - for video in videos or []: - if not 'extractor' in video: - #The extractor has already been set somewher else - video['extractor'] = ie.IE_NAME - return videos + ie_results = ie.extract(url) + results = self.process_ie_results(ie_results, ie) + return results except ExtractorError as de: # An error we somewhat expected self.trouble(u'ERROR: ' + compat_str(de), de.format_traceback()) break @@ -435,6 +432,29 @@ class FileDownloader(object): for url in urls: results.extend(self.extract_info(url)) return results + + def process_ie_results(self, ie_results, ie): + """ + Take the results of the ie and return a list of videos. + For url elements it will seartch the suitable ie and get the videos + For playlist elements it will process each of the elements of the 'entries' key + """ + results = [] + for result in ie_results or []: + result_type = result.get('_type', 'video') #If not given we suppose it's a video, support the dafault old system + if result_type == 'video': + if not 'extractor' in result: + #The extractor has already been set somewhere else + result['extractor'] = ie.IE_NAME + results.append(result) + elif result_type == 'url': + #We get the videos pointed by the url + results.extend(self.extract_info(result['url'])) + elif result_type == 'playlist': + #We process each entry in the playlist + entries_result = self.process_ie_results(result['entries'], ie) + results.extend(entries_result) + return results def process_info(self, info_dict): """Process a single dictionary returned by an InfoExtractor.""" From 8a38a194fb08a253986cdbafa02cf699ef76c9a5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?= Date: Tue, 5 Mar 2013 20:55:48 +0100 Subject: [PATCH 06/14] Add auxiliary methods to InfoExtractor to set the '_type' key and use them for some playlist IEs --- test/test_youtube_lists.py | 35 +++++++++++++++++++++-------------- youtube_dl/InfoExtractors.py | 30 ++++++++++++++++++++++++++---- 2 files changed, 47 insertions(+), 18 deletions(-) diff --git a/test/test_youtube_lists.py b/test/test_youtube_lists.py index 055bf69c8..9e91484f8 100644 --- a/test/test_youtube_lists.py +++ b/test/test_youtube_lists.py @@ -36,31 +36,37 @@ class FakeDownloader(FileDownloader): return url class TestYoutubeLists(unittest.TestCase): + def assertIsPlaylist(self,info): + """Make sure the info has '_type' set to 'playlist'""" + self.assertEqual(info['_type'], 'playlist') + def test_youtube_playlist(self): dl = FakeDownloader() ie = YoutubePlaylistIE(dl) - ie.extract('https://www.youtube.com/playlist?list=PLwiyx1dc3P2JR9N8gQaQN_BCvlSlap7re') - ytie_results = [YoutubeIE()._extract_id(url) for url in dl.result] + result = ie.extract('https://www.youtube.com/playlist?list=PLwiyx1dc3P2JR9N8gQaQN_BCvlSlap7re')[0] + self.assertIsPlaylist(result) + ytie_results = [YoutubeIE()._extract_id(url['url']) for url in result['entries']] self.assertEqual(ytie_results, [ 'bV9L5Ht9LgY', 'FXxLjLQi3Fg', 'tU3Bgo5qJZE']) def test_issue_673(self): dl = FakeDownloader() ie = YoutubePlaylistIE(dl) - ie.extract('PLBB231211A4F62143') - self.assertTrue(len(dl.result) > 40) + result = ie.extract('PLBB231211A4F62143')[0] + self.assertTrue(len(result['entries']) > 40) def test_youtube_playlist_long(self): dl = FakeDownloader() ie = YoutubePlaylistIE(dl) - ie.extract('https://www.youtube.com/playlist?list=UUBABnxM4Ar9ten8Mdjj1j0Q') - self.assertTrue(len(dl.result) >= 799) + result = ie.extract('https://www.youtube.com/playlist?list=UUBABnxM4Ar9ten8Mdjj1j0Q')[0] + self.assertIsPlaylist(result) + self.assertTrue(len(result['entries']) >= 799) def test_youtube_playlist_with_deleted(self): #651 dl = FakeDownloader() ie = YoutubePlaylistIE(dl) - ie.extract('https://www.youtube.com/playlist?list=PLwP_SiAcdui0KVebT0mU9Apz359a4ubsC') - ytie_results = [YoutubeIE()._extract_id(url) for url in dl.result] + result = ie.extract('https://www.youtube.com/playlist?list=PLwP_SiAcdui0KVebT0mU9Apz359a4ubsC')[0] + ytie_results = [YoutubeIE()._extract_id(url['url']) for url in result['entries']] self.assertFalse('pElCt5oNDuI' in ytie_results) self.assertFalse('KdPEApIVdWM' in ytie_results) @@ -68,10 +74,11 @@ class TestYoutubeLists(unittest.TestCase): dl = FakeDownloader() ie = YoutubePlaylistIE(dl) # TODO find a > 100 (paginating?) videos course - ie.extract('https://www.youtube.com/course?list=ECUl4u3cNGP61MdtwGTqZA0MreSaDybji8') - self.assertEqual(YoutubeIE()._extract_id(dl.result[0]), 'j9WZyLZCBzs') - self.assertEqual(len(dl.result), 25) - self.assertEqual(YoutubeIE()._extract_id(dl.result[-1]), 'rYefUsYuEp0') + result = ie.extract('https://www.youtube.com/course?list=ECUl4u3cNGP61MdtwGTqZA0MreSaDybji8')[0] + entries = result['entries'] + self.assertEqual(YoutubeIE()._extract_id(entries[0]['url']), 'j9WZyLZCBzs') + self.assertEqual(len(entries), 25) + self.assertEqual(YoutubeIE()._extract_id(entries[-1]['url']), 'rYefUsYuEp0') def test_youtube_channel(self): # I give up, please find a channel that does paginate and test this like test_youtube_playlist_long @@ -80,8 +87,8 @@ class TestYoutubeLists(unittest.TestCase): def test_youtube_user(self): dl = FakeDownloader() ie = YoutubeUserIE(dl) - ie.extract('https://www.youtube.com/user/TheLinuxFoundation') - self.assertTrue(len(dl.result) >= 320) + result = ie.extract('https://www.youtube.com/user/TheLinuxFoundation')[0] + self.assertTrue(len(result['entries']) >= 320) if __name__ == '__main__': unittest.main() diff --git a/youtube_dl/InfoExtractors.py b/youtube_dl/InfoExtractors.py index d79f6068f..895658f49 100755 --- a/youtube_dl/InfoExtractors.py +++ b/youtube_dl/InfoExtractors.py @@ -128,6 +128,24 @@ class InfoExtractor(object): urlh = self._request_webpage(url_or_request, video_id, note, errnote) webpage_bytes = urlh.read() return webpage_bytes.decode('utf-8', 'replace') + + #Methods for following #608 + #They set the correct value of the '_type' key + def video_result(self, video_info): + """Returns a video""" + video_info['_type'] = 'video' + return video_info + def url_result(self, url, ie=None): + """Returns a url that points to a page that should be processed""" + #TODO: ie should be the class used for getting the info + video_info = {'_type': 'url', + 'url': url} + return video_info + def playlist_result(self, entries): + """Returns a playlist""" + video_info = {'_type': 'playlist', + 'entries': entries} + return video_info class YoutubeIE(InfoExtractor): @@ -1756,7 +1774,8 @@ class YoutubePlaylistIE(InfoExtractor): else: self._downloader.to_screen(u'[youtube] PL %s: Found %i videos, downloading %i' % (playlist_id, total, len(videos))) - return self._downloader.extract_info_iterable(videos) + url_results = [self.url_result(url) for url in videos] + return [self.playlist_result(url_results)] class YoutubeChannelIE(InfoExtractor): @@ -1807,7 +1826,8 @@ class YoutubeChannelIE(InfoExtractor): self._downloader.to_screen(u'[youtube] Channel %s: Found %i videos' % (channel_id, len(video_ids))) urls = ['http://www.youtube.com/watch?v=%s' % id for id in video_ids] - return self._downloader.extract_info_iterable(urls) + url_entries = [self.url_result(url) for url in urls] + return [self.playlist_result(url_entries)] class YoutubeUserIE(InfoExtractor): @@ -1890,7 +1910,8 @@ class YoutubeUserIE(InfoExtractor): (username, all_ids_count, len(video_ids))) urls = ['http://www.youtube.com/watch?v=%s' % video_id for video_id in video_ids] - return self._downloader.extract_info_iterable(urls) + url_results = [self.url_result(url) for url in urls] + return [self.playlist_result(url_results)] class BlipTVUserIE(InfoExtractor): @@ -1981,7 +2002,8 @@ class BlipTVUserIE(InfoExtractor): (self.IE_NAME, username, all_ids_count, len(video_ids))) urls = [u'http://blip.tv/%s' % video_id for video_id in video_ids] - return self._downloader.extract_info_iterable(urls) + url_entries = [self.url_result(url) for url in urls] + return [self.playlist_result(url_entries)] class DepositFilesIE(InfoExtractor): From a0d6fe7b924697c089ed7ae37df0ca590ac38a96 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?= Date: Tue, 5 Mar 2013 22:33:32 +0100 Subject: [PATCH 07/14] When a redirect is found return the new url using the new style --- youtube_dl/InfoExtractors.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/youtube_dl/InfoExtractors.py b/youtube_dl/InfoExtractors.py index 895658f49..e714fa6b0 100755 --- a/youtube_dl/InfoExtractors.py +++ b/youtube_dl/InfoExtractors.py @@ -1311,7 +1311,7 @@ class GenericIE(InfoExtractor): self._downloader.to_screen(u'[redirect] Following redirect to %s' % new_url) def _test_redirect(self, url): - """Check if it is a redirect, like url shorteners, in case restart chain.""" + """Check if it is a redirect, like url shorteners, in case return the new url.""" class HeadRequest(compat_urllib_request.Request): def get_method(self): return "HEAD" @@ -1362,11 +1362,11 @@ class GenericIE(InfoExtractor): return False self.report_following_redirect(new_url) - self._downloader.download([new_url]) - return True + return new_url def _real_extract(self, url): - if self._test_redirect(url): return + new_url = self._test_redirect(url) + if new_url: return [self.url_result(new_url)] video_id = url.split('/')[-1] request = compat_urllib_request.Request(url) From d2c690828a8297c014d8053fbdee4e26fe11586a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?= Date: Thu, 28 Mar 2013 13:39:00 +0100 Subject: [PATCH 08/14] Add title and id to playlist results Not all IE give both. They are not used yet. --- youtube_dl/InfoExtractors.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/youtube_dl/InfoExtractors.py b/youtube_dl/InfoExtractors.py index dd4a776e4..6053d14ec 100755 --- a/youtube_dl/InfoExtractors.py +++ b/youtube_dl/InfoExtractors.py @@ -147,10 +147,14 @@ class InfoExtractor(object): video_info = {'_type': 'url', 'url': url} return video_info - def playlist_result(self, entries): + def playlist_result(self, entries, playlist_id=None, playlist_title=None): """Returns a playlist""" video_info = {'_type': 'playlist', 'entries': entries} + if playlist_id: + video_info['id'] = playlist_id + if playlist_title: + video_info['title'] = playlist_title return video_info @@ -1808,7 +1812,7 @@ class YoutubePlaylistIE(InfoExtractor): self._downloader.to_screen(u'[youtube] PL %s: Found %i videos, downloading %i' % (playlist_id, total, len(videos))) url_results = [self.url_result(url) for url in videos] - return [self.playlist_result(url_results)] + return [self.playlist_result(url_results, playlist_id)] class YoutubeChannelIE(InfoExtractor): @@ -1860,7 +1864,7 @@ class YoutubeChannelIE(InfoExtractor): urls = ['http://www.youtube.com/watch?v=%s' % id for id in video_ids] url_entries = [self.url_result(url) for url in urls] - return [self.playlist_result(url_entries)] + return [self.playlist_result(url_entries, channel_id)] class YoutubeUserIE(InfoExtractor): @@ -1944,7 +1948,7 @@ class YoutubeUserIE(InfoExtractor): urls = ['http://www.youtube.com/watch?v=%s' % video_id for video_id in video_ids] url_results = [self.url_result(url) for url in urls] - return [self.playlist_result(url_results)] + return [self.playlist_result(url_results, playlist_title = username)] class BlipTVUserIE(InfoExtractor): @@ -2036,7 +2040,7 @@ class BlipTVUserIE(InfoExtractor): urls = [u'http://blip.tv/%s' % video_id for video_id in video_ids] url_entries = [self.url_result(url) for url in urls] - return [self.playlist_result(url_entries)] + return [self.playlist_result(url_entries, playlist_title = username)] class DepositFilesIE(InfoExtractor): From 7eab8dc7504cf1f5f1dd03eb62e266ce24948b93 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?= Date: Fri, 29 Mar 2013 12:32:42 +0100 Subject: [PATCH 09/14] Pass the playlist info_dict to process_info the playlist value can be used in the output template --- README.md | 1 + youtube_dl/FileDownloader.py | 11 ++++++++++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 7c09d0c0d..1f3422ef8 100644 --- a/README.md +++ b/README.md @@ -138,6 +138,7 @@ The `-o` option allows users to indicate a template for the output file names. T - `ext`: The sequence will be replaced by the appropriate extension (like flv or mp4). - `epoch`: The sequence will be replaced by the Unix epoch when creating the file. - `autonumber`: The sequence will be replaced by a five-digit number that will be increased with each download, starting at zero. + - `playlist`: The name or the id of the playlist that contains the video. The current default template is `%(id)s.%(ext)s`, but that will be switchted to `%(title)s-%(id)s.%(ext)s` (which can be requested with `-t` at the moment). diff --git a/youtube_dl/FileDownloader.py b/youtube_dl/FileDownloader.py index 6af2acbee..d2b9be9ef 100644 --- a/youtube_dl/FileDownloader.py +++ b/youtube_dl/FileDownloader.py @@ -460,12 +460,21 @@ class FileDownloader(object): elif result_type == 'playlist': #We process each entry in the playlist entries_result = self.process_ie_results(result['entries'], ie) - results.extend(entries_result) + result['entries'] = entries_result + results.extend([result]) return results def process_info(self, info_dict): """Process a single dictionary returned by an InfoExtractor.""" + if info_dict.get('_type','video') == 'playlist': + playlist = info_dict.get('title', None) or info_dict.get('id', None) + self.to_screen(u'[download] Downloading playlist: %s' % playlist) + for video in info_dict['entries']: + video['playlist'] = playlist + self.process_info(video) + return + # Keep for backwards compatibility info_dict['stitle'] = info_dict['title'] From d39919c03e45b3e8f804c23f78fae33cb4adc7df Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?= Date: Fri, 5 Apr 2013 13:01:59 +0200 Subject: [PATCH 10/14] Add progress counter for playlists Closes #276 --- youtube_dl/FileDownloader.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/youtube_dl/FileDownloader.py b/youtube_dl/FileDownloader.py index 2237d355d..ba3277577 100644 --- a/youtube_dl/FileDownloader.py +++ b/youtube_dl/FileDownloader.py @@ -492,8 +492,10 @@ class FileDownloader(object): if info_dict.get('_type','video') == 'playlist': playlist = info_dict.get('title', None) or info_dict.get('id', None) self.to_screen(u'[download] Downloading playlist: %s' % playlist) - for video in info_dict['entries']: + n_videos = len(info_dict['entries']) + for i,video in enumerate(info_dict['entries'],1): video['playlist'] = playlist + self.to_screen(u'[download] Downloading video #%s of %s' %(i, n_videos)) self.process_info(video) return From 146c12a2dafdb9ff0e5138aa0f9da38bddca6c8a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?= Date: Wed, 10 Apr 2013 00:05:04 +0200 Subject: [PATCH 11/14] Change the order for extracting/downloading Now it gets a video info and directly downloads it, the it pass to the next video founded. --- youtube_dl/FileDownloader.py | 103 ++++++++++++++++++----------------- 1 file changed, 53 insertions(+), 50 deletions(-) diff --git a/youtube_dl/FileDownloader.py b/youtube_dl/FileDownloader.py index ba3277577..58be5caee 100644 --- a/youtube_dl/FileDownloader.py +++ b/youtube_dl/FileDownloader.py @@ -419,9 +419,10 @@ class FileDownloader(object): return u'"' + title + '" title matched reject pattern "' + rejecttitle + '"' return None - def extract_info(self, url): + def extract_info(self, url, download = True): ''' Returns a list with a dictionary for each video we find. + If 'download', also downloads the videos. ''' suitable_found = False for ie in self._ies: @@ -440,7 +441,12 @@ class FileDownloader(object): # Extract information from URL and process it try: ie_results = ie.extract(url) - results = self.process_ie_results(ie_results, ie) + results = [] + for ie_result in ie_results: + if not 'extractor' in ie_result: + #The extractor has already been set somewhere else + ie_result['extractor'] = ie.IE_NAME + results.append(self.process_ie_result(ie_result, download)) return results except ExtractorError as de: # An error we somewhat expected self.trouble(u'ERROR: ' + compat_str(de), de.format_traceback()) @@ -453,51 +459,51 @@ class FileDownloader(object): raise if not suitable_found: self.trouble(u'ERROR: no suitable InfoExtractor: %s' % url) - def extract_info_iterable(self, urls): - ''' - Return the videos founded for the urls - ''' - results = [] - for url in urls: - results.extend(self.extract_info(url)) - return results - def process_ie_results(self, ie_results, ie): + def process_ie_result(self, ie_result, download = True): """ - Take the results of the ie and return a list of videos. - For url elements it will seartch the suitable ie and get the videos + Take the result of the ie and return a list of videos. + For url elements it will search the suitable ie and get the videos For playlist elements it will process each of the elements of the 'entries' key + + It will also download the videos if 'download'. """ - results = [] - for result in ie_results or []: - result_type = result.get('_type', 'video') #If not given we suppose it's a video, support the dafault old system - if result_type == 'video': - if not 'extractor' in result: - #The extractor has already been set somewhere else - result['extractor'] = ie.IE_NAME - results.append(result) - elif result_type == 'url': - #We get the videos pointed by the url - results.extend(self.extract_info(result['url'])) - elif result_type == 'playlist': - #We process each entry in the playlist - entries_result = self.process_ie_results(result['entries'], ie) - result['entries'] = entries_result - results.extend([result]) - return results + result_type = ie_result.get('_type', 'video') #If not given we suppose it's a video, support the dafault old system + if result_type == 'video': + if 'playlist' not in ie_result: + #It isn't part of a playlist + ie_result['playlist'] = None + if download: + #Do the download: + self.process_info(ie_result) + return ie_result + elif result_type == 'url': + #We get the video pointed by the url + result = self.extract_info(ie_result['url'], download)[0] + return result + elif result_type == 'playlist': + #We process each entry in the playlist + playlist = ie_result.get('title', None) or ie_result.get('id', None) + self.to_screen(u'[download] Downloading playlist: %s' % playlist) + n_videos = len(ie_result['entries']) + playlist_results = [] + for i,entry in enumerate(ie_result['entries'],1): + self.to_screen(u'[download] Downloading video #%s of %s' %(i, n_videos)) + entry_result = self.process_ie_result(entry, False) + entry_result['playlist'] = playlist + #We must do the download here to correctly set the 'playlist' key + if download: + self.process_info(entry_result) + playlist_results.append(entry_result) + result = ie_result.copy() + result['entries'] = playlist_results + return result def process_info(self, info_dict): """Process a single dictionary returned by an InfoExtractor.""" - if info_dict.get('_type','video') == 'playlist': - playlist = info_dict.get('title', None) or info_dict.get('id', None) - self.to_screen(u'[download] Downloading playlist: %s' % playlist) - n_videos = len(info_dict['entries']) - for i,video in enumerate(info_dict['entries'],1): - video['playlist'] = playlist - self.to_screen(u'[download] Downloading video #%s of %s' %(i, n_videos)) - self.process_info(video) - return + #We increment the download the download count here to match the previous behaviour. + self.increment_downloads() # Keep for backwards compatibility info_dict['stitle'] = info_dict['title'] @@ -633,17 +639,14 @@ class FileDownloader(object): raise SameFileError(self.params['outtmpl']) for url in url_list: - videos = self.extract_info(url) - - for video in videos or []: - try: - self.increment_downloads() - self.process_info(video) - except UnavailableVideoError: - self.trouble(u'\nERROR: unable to download video') - except MaxDownloadsReached: - self.to_screen(u'[info] Maximum number of downloaded files reached.') - raise + try: + #It also downloads the videos + videos = self.extract_info(url) + except UnavailableVideoError: + self.trouble(u'\nERROR: unable to download video') + except MaxDownloadsReached: + self.to_screen(u'[info] Maximum number of downloaded files reached.') + raise return self._download_retcode From 532d797824a1ec48480f1d10075e66a90aa53449 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?= Date: Wed, 10 Apr 2013 00:06:03 +0200 Subject: [PATCH 12/14] In MetacafeIE return a url if YoutubeIE should do the job --- youtube_dl/InfoExtractors.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/youtube_dl/InfoExtractors.py b/youtube_dl/InfoExtractors.py index 81eaddc72..b7371365a 100755 --- a/youtube_dl/InfoExtractors.py +++ b/youtube_dl/InfoExtractors.py @@ -723,8 +723,7 @@ class MetacafeIE(InfoExtractor): # Check if video comes from YouTube mobj2 = re.match(r'^yt-(.*)$', video_id) if mobj2 is not None: - self._downloader.download(['http://www.youtube.com/watch?v=%s' % mobj2.group(1)]) - return + return [self.url_result('http://www.youtube.com/watch?v=%s' % mobj2.group(1))] # Retrieve video webpage to extract further information request = compat_urllib_request.Request('http://www.metacafe.com/watch/%s/' % video_id) From bce878a7c1678ac698ecd556b2c77a1e2f2306df Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?= Date: Wed, 10 Apr 2013 14:32:03 +0200 Subject: [PATCH 13/14] Implement the playlist/start options in FileDownloader It makes it available for all the InfoExtractors --- youtube_dl/FileDownloader.py | 21 ++++++++++++++++++--- youtube_dl/InfoExtractors.py | 34 ---------------------------------- 2 files changed, 18 insertions(+), 37 deletions(-) diff --git a/youtube_dl/FileDownloader.py b/youtube_dl/FileDownloader.py index 58be5caee..5a5141ba5 100644 --- a/youtube_dl/FileDownloader.py +++ b/youtube_dl/FileDownloader.py @@ -485,10 +485,25 @@ class FileDownloader(object): #We process each entry in the playlist playlist = ie_result.get('title', None) or ie_result.get('id', None) self.to_screen(u'[download] Downloading playlist: %s' % playlist) - n_videos = len(ie_result['entries']) + playlist_results = [] - for i,entry in enumerate(ie_result['entries'],1): - self.to_screen(u'[download] Downloading video #%s of %s' %(i, n_videos)) + + n_all_entries = len(ie_result['entries']) + playliststart = self.params.get('playliststart', 1) - 1 + playlistend = self.params.get('playlistend', -1) + + if playlistend == -1: + entries = ie_result['entries'][playliststart:] + else: + entries = ie_result['entries'][playliststart:playlistend] + + n_entries = len(entries) + + self.to_screen(u"[%s] playlist '%s': Collected %d video ids (downloading %d of them)" % + (ie_result['extractor'], playlist, n_all_entries, n_entries)) + + for i,entry in enumerate(entries,1): + self.to_screen(u'[download] Downloading video #%s of %s' %(i, n_entries)) entry_result = self.process_ie_result(entry, False) entry_result['playlist'] = playlist #We must do the download here to correctly set the 'playlist' key diff --git a/youtube_dl/InfoExtractors.py b/youtube_dl/InfoExtractors.py index b7371365a..a7fdf1607 100755 --- a/youtube_dl/InfoExtractors.py +++ b/youtube_dl/InfoExtractors.py @@ -1806,19 +1806,6 @@ class YoutubePlaylistIE(InfoExtractor): page_num += 1 videos = [v[1] for v in sorted(videos)] - total = len(videos) - - playliststart = self._downloader.params.get('playliststart', 1) - 1 - playlistend = self._downloader.params.get('playlistend', -1) - if playlistend == -1: - videos = videos[playliststart:] - else: - videos = videos[playliststart:playlistend] - - if len(videos) == total: - self._downloader.to_screen(u'[youtube] PL %s: Found %i videos' % (playlist_id, total)) - else: - self._downloader.to_screen(u'[youtube] PL %s: Found %i videos, downloading %i' % (playlist_id, total, len(videos))) url_results = [self.url_result(url) for url in videos] return [self.playlist_result(url_results, playlist_id)] @@ -1943,18 +1930,6 @@ class YoutubeUserIE(InfoExtractor): pagenum += 1 - all_ids_count = len(video_ids) - playliststart = self._downloader.params.get('playliststart', 1) - 1 - playlistend = self._downloader.params.get('playlistend', -1) - - if playlistend == -1: - video_ids = video_ids[playliststart:] - else: - video_ids = video_ids[playliststart:playlistend] - - self._downloader.to_screen(u"[youtube] user %s: Collected %d video ids (downloading %d of them)" % - (username, all_ids_count, len(video_ids))) - urls = ['http://www.youtube.com/watch?v=%s' % video_id for video_id in video_ids] url_results = [self.url_result(url) for url in urls] return [self.playlist_result(url_results, playlist_title = username)] @@ -2035,15 +2010,6 @@ class BlipTVUserIE(InfoExtractor): pagenum += 1 - all_ids_count = len(video_ids) - playliststart = self._downloader.params.get('playliststart', 1) - 1 - playlistend = self._downloader.params.get('playlistend', -1) - - if playlistend == -1: - video_ids = video_ids[playliststart:] - else: - video_ids = video_ids[playliststart:playlistend] - self._downloader.to_screen(u"[%s] user %s: Collected %d video ids (downloading %d of them)" % (self.IE_NAME, username, all_ids_count, len(video_ids))) From d281274bf250065f876bb4f75fb6f711e1a26eba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?= Date: Tue, 16 Apr 2013 15:13:29 +0200 Subject: [PATCH 14/14] Add a playlist_index key to the info_dict, can be used in the output template --- README.md | 1 + youtube_dl/FileDownloader.py | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/README.md b/README.md index c8d28db3c..e2958a9b0 100644 --- a/README.md +++ b/README.md @@ -145,6 +145,7 @@ The `-o` option allows users to indicate a template for the output file names. T - `epoch`: The sequence will be replaced by the Unix epoch when creating the file. - `autonumber`: The sequence will be replaced by a five-digit number that will be increased with each download, starting at zero. - `playlist`: The name or the id of the playlist that contains the video. + - `playlist_index`: The index of the video in the playlist, a five-digit number. The current default template is `%(id)s.%(ext)s`, but that will be switchted to `%(title)s-%(id)s.%(ext)s` (which can be requested with `-t` at the moment). diff --git a/youtube_dl/FileDownloader.py b/youtube_dl/FileDownloader.py index 5a5141ba5..4dabbb440 100644 --- a/youtube_dl/FileDownloader.py +++ b/youtube_dl/FileDownloader.py @@ -389,6 +389,8 @@ class FileDownloader(object): template_dict['epoch'] = int(time.time()) template_dict['autonumber'] = u'%05d' % self._num_downloads + if template_dict['playlist_index'] is not None: + template_dict['playlist_index'] = u'%05d' % template_dict['playlist_index'] sanitize = lambda k,v: sanitize_filename( u'NA' if v is None else compat_str(v), @@ -473,6 +475,7 @@ class FileDownloader(object): if 'playlist' not in ie_result: #It isn't part of a playlist ie_result['playlist'] = None + ie_result['playlist_index'] = None if download: #Do the download: self.process_info(ie_result) @@ -506,6 +509,7 @@ class FileDownloader(object): self.to_screen(u'[download] Downloading video #%s of %s' %(i, n_entries)) entry_result = self.process_ie_result(entry, False) entry_result['playlist'] = playlist + entry_result['playlist_index'] = i + playliststart #We must do the download here to correctly set the 'playlist' key if download: self.process_info(entry_result)