mirror of
https://github.com/ytdl-org/youtube-dl.git
synced 2024-11-23 18:51:51 +00:00
[cache] Add cache validation by program version, based on yt-dlp
This commit is contained in:
parent
55c823634d
commit
218c423bc0
2 changed files with 37 additions and 7 deletions
|
@ -3,17 +3,18 @@
|
|||
|
||||
from __future__ import unicode_literals
|
||||
|
||||
import shutil
|
||||
|
||||
# Allow direct execution
|
||||
import os
|
||||
import sys
|
||||
import unittest
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
|
||||
import shutil
|
||||
|
||||
from test.helper import FakeYDL
|
||||
from youtube_dl.cache import Cache
|
||||
from youtube_dl.utils import version_tuple
|
||||
from youtube_dl.version import __version__
|
||||
|
||||
|
||||
def _is_empty(d):
|
||||
|
@ -54,6 +55,17 @@ class TestCache(unittest.TestCase):
|
|||
self.assertFalse(os.path.exists(self.test_dir))
|
||||
self.assertEqual(c.load('test_cache', 'k.'), None)
|
||||
|
||||
def test_cache_validation(self):
|
||||
ydl = FakeYDL({
|
||||
'cachedir': self.test_dir,
|
||||
})
|
||||
c = Cache(ydl)
|
||||
obj = {'x': 1, 'y': ['ä', '\\a', True]}
|
||||
c.store('test_cache', 'k.', obj)
|
||||
self.assertEqual(c.load('test_cache', 'k.', min_ver='1970.01.01'), obj)
|
||||
new_version = '.'.join(('%d' % ((v + 1) if i == 0 else v, )) for i, v in enumerate(version_tuple(__version__)))
|
||||
self.assertIs(c.load('test_cache', 'k.', min_ver=new_version), None)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
|
|
|
@ -10,12 +10,21 @@ import traceback
|
|||
|
||||
from .compat import compat_getenv
|
||||
from .utils import (
|
||||
error_to_compat_str,
|
||||
expand_path,
|
||||
is_outdated_version,
|
||||
try_get,
|
||||
write_json_file,
|
||||
)
|
||||
from .version import __version__
|
||||
|
||||
|
||||
class Cache(object):
|
||||
|
||||
_YTDL_DIR = 'youtube-dl'
|
||||
_VERSION_KEY = _YTDL_DIR + '_version'
|
||||
_DEFAULT_VERSION = '2021.12.17'
|
||||
|
||||
def __init__(self, ydl):
|
||||
self._ydl = ydl
|
||||
|
||||
|
@ -23,7 +32,7 @@ class Cache(object):
|
|||
res = self._ydl.params.get('cachedir')
|
||||
if res is None:
|
||||
cache_root = compat_getenv('XDG_CACHE_HOME', '~/.cache')
|
||||
res = os.path.join(cache_root, 'youtube-dl')
|
||||
res = os.path.join(cache_root, self._YTDL_DIR)
|
||||
return expand_path(res)
|
||||
|
||||
def _get_cache_fn(self, section, key, dtype):
|
||||
|
@ -50,13 +59,22 @@ class Cache(object):
|
|||
except OSError as ose:
|
||||
if ose.errno != errno.EEXIST:
|
||||
raise
|
||||
write_json_file(data, fn)
|
||||
write_json_file({self._VERSION_KEY: __version__, 'data': data}, fn)
|
||||
except Exception:
|
||||
tb = traceback.format_exc()
|
||||
self._ydl.report_warning(
|
||||
'Writing cache to %r failed: %s' % (fn, tb))
|
||||
|
||||
def load(self, section, key, dtype='json', default=None):
|
||||
def _validate(self, data, min_ver):
|
||||
version = try_get(data, lambda x: x[self._VERSION_KEY])
|
||||
if not version: # Backward compatibility
|
||||
data, version = {'data': data}, self._DEFAULT_VERSION
|
||||
if not is_outdated_version(version, min_ver or '0', assume_new=False):
|
||||
return data['data']
|
||||
self._ydl.to_screen(
|
||||
'Discarding old cache from version {version} (needs {min_ver})'.format(**locals()))
|
||||
|
||||
def load(self, section, key, dtype='json', default=None, min_ver=None):
|
||||
assert dtype in ('json',)
|
||||
|
||||
if not self.enabled:
|
||||
|
@ -66,12 +84,12 @@ class Cache(object):
|
|||
try:
|
||||
try:
|
||||
with io.open(cache_fn, 'r', encoding='utf-8') as cachef:
|
||||
return json.load(cachef)
|
||||
return self._validate(json.load(cachef), min_ver)
|
||||
except ValueError:
|
||||
try:
|
||||
file_size = os.path.getsize(cache_fn)
|
||||
except (OSError, IOError) as oe:
|
||||
file_size = str(oe)
|
||||
file_size = error_to_compat_str(oe)
|
||||
self._ydl.report_warning(
|
||||
'Cache retrieval from %s failed (%s)' % (cache_fn, file_size))
|
||||
except IOError:
|
||||
|
|
Loading…
Reference in a new issue