From 90e16aa42c8b1ad2df05d8ae6a7430c341530705 Mon Sep 17 00:00:00 2001 From: Alexander Nartov Date: Sat, 19 Sep 2020 22:30:12 +1000 Subject: [PATCH 1/3] [rutube] Add RuTube LiST support --- youtube_dl/extractor/rutube.py | 39 ++++++++++++++++++++++++++++++++-- 1 file changed, 37 insertions(+), 2 deletions(-) diff --git a/youtube_dl/extractor/rutube.py b/youtube_dl/extractor/rutube.py index 8f54d5675..f82448f9b 100644 --- a/youtube_dl/extractor/rutube.py +++ b/youtube_dl/extractor/rutube.py @@ -1,6 +1,7 @@ # coding: utf-8 from __future__ import unicode_literals +import json import re import itertools @@ -21,6 +22,8 @@ from ..utils import ( class RutubeBaseIE(InfoExtractor): + _NETRC_MACHINE = 'rutube' + def _download_api_info(self, video_id, query=None): if not query: query = {} @@ -55,6 +58,7 @@ class RutubeBaseIE(InfoExtractor): 'view_count': int_or_none(video.get('hits')), 'comment_count': int_or_none(video.get('comments_count')), 'is_live': bool_or_none(video.get('is_livestream')), + 'is_club': bool_or_none(video.get('is_club')), } def _download_and_extract_info(self, video_id, query=None): @@ -66,7 +70,7 @@ class RutubeBaseIE(InfoExtractor): query = {} query['format'] = 'json' return self._download_json( - 'http://rutube.ru/api/play/options/%s/' % video_id, + 'https://rutube.ru/api/play/options/%s/' % video_id, video_id, 'Downloading options JSON', 'Unable to download options JSON', headers=self.geo_verification_headers(), query=query) @@ -141,7 +145,38 @@ class RutubeIE(RutubeBaseIE): def _real_extract(self, url): video_id = self._match_id(url) info = self._download_and_extract_info(video_id) - info['formats'] = self._download_and_extract_formats(video_id) + query = {} + + if info['is_club']: + username, password = self._get_login_info() + if username is None or password is None: + self.raise_login_required() + + login = self._download_json( + 'https://pass.rutube.ru/api/accounts/phone/login/', video_id, data=json.dumps({ + 'phone': username, + 'password': password, + }).encode(), + headers={'Content-Type': 'application/json', 'Accept': 'application/json'} + ) + if not login['success']: + self.raise_login_required('Invalid login or password') + + self._download_webpage( + 'https://rutube.ru/social/auth/rupass/?callback_path=/social/login/rupass/', video_id) + + visitor = self._download_json( + 'https://rutube.ru/api/accounts/visitor/', video_id, 'Downloading visitor JSON', + 'Unable to download visitor JSON') + clubParams = visitor['club_params_encrypted'] + ad = self._download_json( + 'https://mtr.rutube.ru/api/v3/interactive?' + clubParams + '&video_id=' + video_id, video_id, + 'Downloading AD JSON', 'Unable to download AD JSON') + clubToken = ad['award'] + query['club_token'] = clubToken + query['no_404'] = 'true' + + info['formats'] = self._download_and_extract_formats(video_id, query) return info From f91625cd122d913eddcb8ad7860644bc17717ce5 Mon Sep 17 00:00:00 2001 From: Alexander Nartov Date: Mon, 21 Sep 2020 03:17:02 +1000 Subject: [PATCH 2/3] [rutube] Better login and error handling, code style fixes --- youtube_dl/extractor/rutube.py | 76 +++++++++++++++++++++++----------- 1 file changed, 51 insertions(+), 25 deletions(-) diff --git a/youtube_dl/extractor/rutube.py b/youtube_dl/extractor/rutube.py index f82448f9b..fdda8c4c9 100644 --- a/youtube_dl/extractor/rutube.py +++ b/youtube_dl/extractor/rutube.py @@ -18,7 +18,7 @@ from ..utils import ( try_get, unified_timestamp, url_or_none, -) + ExtractorError) class RutubeBaseIE(InfoExtractor): @@ -61,6 +61,39 @@ class RutubeBaseIE(InfoExtractor): 'is_club': bool_or_none(video.get('is_club')), } + def _real_initialize(self): + self._login() + + def _login(self): + username, password = self._get_login_info() + if username is None: + return + + login = self._download_json( + 'https://pass.rutube.ru/api/accounts/phone/login/', None, + 'Logging in', 'Unable to log in', + data=json.dumps({ + 'phone': username, + 'password': password, + }).encode(), + headers={ + 'Content-Type': 'application/json', + 'Accept': 'application/json' + }, + expected_status=400 + ) + if not login.get('success'): + msg = login.get('message') + raise ExtractorError( + 'Unable to log in. %s said: %s' % (self.IE_NAME, msg), + expected=True + ) + + self._download_webpage( + 'https://rutube.ru/social/auth/rupass/?callback_path=/social/login/rupass/', + None, False + ) + def _download_and_extract_info(self, video_id, query=None): return self._extract_info( self._download_api_info(video_id, query=query), video_id) @@ -148,33 +181,26 @@ class RutubeIE(RutubeBaseIE): query = {} if info['is_club']: - username, password = self._get_login_info() - if username is None or password is None: + visitor_json, urlh = self._download_webpage_handle( + 'https://rutube.ru/api/accounts/visitor/', video_id, + 'Downloading visitor JSON', 'Unable to download visitor JSON' + ) + if not visitor_json: self.raise_login_required() - login = self._download_json( - 'https://pass.rutube.ru/api/accounts/phone/login/', video_id, data=json.dumps({ - 'phone': username, - 'password': password, - }).encode(), - headers={'Content-Type': 'application/json', 'Accept': 'application/json'} - ) - if not login['success']: - self.raise_login_required('Invalid login or password') - - self._download_webpage( - 'https://rutube.ru/social/auth/rupass/?callback_path=/social/login/rupass/', video_id) - - visitor = self._download_json( - 'https://rutube.ru/api/accounts/visitor/', video_id, 'Downloading visitor JSON', - 'Unable to download visitor JSON') - clubParams = visitor['club_params_encrypted'] + visitor = self._parse_json(visitor_json, video_id) + club_params = visitor['club_params_encrypted'] ad = self._download_json( - 'https://mtr.rutube.ru/api/v3/interactive?' + clubParams + '&video_id=' + video_id, video_id, - 'Downloading AD JSON', 'Unable to download AD JSON') - clubToken = ad['award'] - query['club_token'] = clubToken - query['no_404'] = 'true' + 'https://mtr.rutube.ru/api/v3/interactive?%s&video_id=%s' % + (club_params, video_id), + video_id, 'Downloading AD JSON', 'Unable to download AD JSON' + ) + club_token = ad['award'] + + query.update({ + 'club_token': club_token, + 'no_404': 'true', + }) info['formats'] = self._download_and_extract_formats(video_id, query) return info From eba856318a506f6a4fabd08323bdbe0b32937df1 Mon Sep 17 00:00:00 2001 From: Alexander Nartov Date: Mon, 21 Sep 2020 04:08:57 +1000 Subject: [PATCH 3/3] [rutube] Fix code style --- youtube_dl/extractor/rutube.py | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/youtube_dl/extractor/rutube.py b/youtube_dl/extractor/rutube.py index fdda8c4c9..7a8767473 100644 --- a/youtube_dl/extractor/rutube.py +++ b/youtube_dl/extractor/rutube.py @@ -80,19 +80,16 @@ class RutubeBaseIE(InfoExtractor): 'Content-Type': 'application/json', 'Accept': 'application/json' }, - expected_status=400 - ) + expected_status=400) if not login.get('success'): msg = login.get('message') raise ExtractorError( 'Unable to log in. %s said: %s' % (self.IE_NAME, msg), - expected=True - ) + expected=True) self._download_webpage( 'https://rutube.ru/social/auth/rupass/?callback_path=/social/login/rupass/', - None, False - ) + None, False) def _download_and_extract_info(self, video_id, query=None): return self._extract_info( @@ -183,8 +180,7 @@ class RutubeIE(RutubeBaseIE): if info['is_club']: visitor_json, urlh = self._download_webpage_handle( 'https://rutube.ru/api/accounts/visitor/', video_id, - 'Downloading visitor JSON', 'Unable to download visitor JSON' - ) + 'Downloading visitor JSON', 'Unable to download visitor JSON') if not visitor_json: self.raise_login_required() @@ -193,8 +189,7 @@ class RutubeIE(RutubeBaseIE): ad = self._download_json( 'https://mtr.rutube.ru/api/v3/interactive?%s&video_id=%s' % (club_params, video_id), - video_id, 'Downloading AD JSON', 'Unable to download AD JSON' - ) + video_id, 'Downloading AD JSON', 'Unable to download AD JSON') club_token = ad['award'] query.update({