mirror of
https://github.com/ytdl-org/youtube-dl.git
synced 2024-09-28 06:15:27 +00:00
Compare commits
1267 commits
2020.03.06
...
master
Author | SHA1 | Date | |
---|---|---|---|
|
c5098961b0 | ||
|
dbc08fba83 | ||
|
71223bff39 | ||
|
e1b3fa242c | ||
|
451046d62a | ||
|
16f5bbc464 | ||
|
d35ce6ce95 | ||
|
76ac69917e | ||
|
756f6b45c7 | ||
|
43a74c5fa5 | ||
|
a452f9437c | ||
|
36801c62df | ||
|
f4b47754d9 | ||
|
37cea84f77 | ||
|
4652109643 | ||
|
3c466186a8 | ||
|
4d05f84325 | ||
|
e0094e63c3 | ||
|
fd8242e3ef | ||
|
ad01fa6cca | ||
|
2eac0fa379 | ||
|
0153b387e5 | ||
|
a48fe7491d | ||
|
e20ca543f0 | ||
|
e39466051f | ||
|
d95c0d203f | ||
|
3bde6a5752 | ||
|
50f6c5668a | ||
|
b4ff08bd2d | ||
|
88bd8b9f87 | ||
|
21924742f7 | ||
|
768ccccd9b | ||
|
eee9a247eb | ||
|
34484e49f5 | ||
|
06da64ee51 | ||
|
a08f2b7e45 | ||
|
668332b973 | ||
|
0b2ce3685e | ||
|
c2766cb80e | ||
|
eb38665438 | ||
|
e0727e4ab6 | ||
|
4ea59c6107 | ||
|
21792b88b7 | ||
|
d8f134a664 | ||
|
31a15a7c8d | ||
|
19dc10b986 | ||
|
182f63e82a | ||
|
71211e7db7 | ||
|
a96a45b2cd | ||
|
820fae3b3a | ||
|
aef24d97e9 | ||
|
f7b30e3f73 | ||
|
f66372403f | ||
|
7216fa2ac4 | ||
|
acc383b9e3 | ||
|
f0812d7848 | ||
|
40bd5c1815 | ||
|
70f230f9cf | ||
|
48ddab1f3a | ||
|
7687389f08 | ||
|
4416f82c80 | ||
|
bdda6b81df | ||
|
1fd8f802b8 | ||
|
4eaeb9b2c6 | ||
|
bec9180e89 | ||
|
c58b655a9e | ||
|
dc512e3a8a | ||
|
f8b0135850 | ||
|
640d39f03a | ||
|
6651871416 | ||
|
be008e657d | ||
|
b1bbc1e502 | ||
|
55a442adae | ||
|
c62936a5f2 | ||
|
427472351c | ||
|
c6538ed323 | ||
|
8d227cb97b | ||
|
4e115e18cb | ||
|
b7fca0fab3 | ||
|
00ef748cc0 | ||
|
66ab0814c4 | ||
|
bbd3e7e999 | ||
|
21caaf2380 | ||
|
31f50c8194 | ||
|
7d58f0769a | ||
|
86e3cf5e58 | ||
|
2efc8de4d2 | ||
|
e4178b5af3 | ||
|
2d2a4bc832 | ||
|
7d965e6b65 | ||
|
abef53466d | ||
|
e7926ae9f4 | ||
|
87e578c9b8 | ||
|
0861812d72 | ||
|
b870181229 | ||
|
a25e9f3c84 | ||
|
aac33155e4 | ||
|
2b7dd3b2a2 | ||
|
44faa71b19 | ||
|
7bce2ad441 | ||
|
ca71e56c48 | ||
|
2a4e9faa77 | ||
|
74eef6bb5e | ||
|
1fa8b86f0b | ||
|
b2ba24bb02 | ||
|
a190b55964 | ||
|
b2741f2654 | ||
|
8465222041 | ||
|
4339910df3 | ||
|
eaaf4c6736 | ||
|
4566e6e53e | ||
|
1e8ccdd2eb | ||
|
cb9366eda5 | ||
|
d9d07a9581 | ||
|
825a40744b | ||
|
47214e46d8 | ||
|
1d8d5a93f7 | ||
|
1634b1d61e | ||
|
21438a4194 | ||
|
8334ec961b | ||
|
3801d36416 | ||
|
b383be9887 | ||
|
46fde7caee | ||
|
648dc5304c | ||
|
1720c04dc5 | ||
|
d5ef405c5d | ||
|
f47fdb9564 | ||
|
b6dff4073d | ||
|
f24bc9272e | ||
|
b08a580906 | ||
|
2500300c2a | ||
|
58fc5bde47 | ||
|
fa7f0effbe | ||
|
ebdc82c586 | ||
|
9112e668a5 | ||
|
07af47960f | ||
|
ae8ba2c319 | ||
|
d6433cbb2c | ||
|
ff75c300f5 | ||
|
a2534f7b88 | ||
|
b8a86dcf1a | ||
|
2389c7cbd3 | ||
|
ee731f3d00 | ||
|
1f7c6f8b2b | ||
|
d89c2137ba | ||
|
d1c6c5c4d6 | ||
|
6ed3433828 | ||
|
a85a875fef | ||
|
11cc3f3ad0 | ||
|
64d6dd64c8 | ||
|
211cbfd5d4 | ||
|
26035bde46 | ||
|
2da3fa04a6 | ||
|
735e87adfc | ||
|
fe7e13066c | ||
|
213d1d91bf | ||
|
f8253a5289 | ||
|
d6ae3b77cd | ||
|
9f4d83ff42 | ||
|
25124bd640 | ||
|
78da22489b | ||
|
557dbac173 | ||
|
cdf40b6aa6 | ||
|
3f6d2bd76f | ||
|
88f28f620b | ||
|
f35b757c82 | ||
|
45495228b7 | ||
|
6fece0a96b | ||
|
70ff013910 | ||
|
e8de54bce5 | ||
|
baa6c5e95c | ||
|
5c985d4f81 | ||
|
8c86fd33dc | ||
|
27d41d7365 | ||
|
0402710227 | ||
|
3e92c60fcd | ||
|
3da17834a4 | ||
|
f7ce98a21e | ||
|
e67e52a8f8 | ||
|
1d3751c3fe | ||
|
6067451e43 | ||
|
57802e632f | ||
|
2dd6c6edd8 | ||
|
dd9aa74bee | ||
|
42b098dd79 | ||
|
6f8c2635a5 | ||
|
de48105dd8 | ||
|
822f19f05d | ||
|
33db85c571 | ||
|
f33923cba7 | ||
|
e8198c517b | ||
|
bafb6dec72 | ||
|
4e04f10499 | ||
|
90c9f789d9 | ||
|
249f2b6316 | ||
|
d6b14ba316 | ||
|
30e986b834 | ||
|
58988c1421 | ||
|
e19ec52322 | ||
|
f2f90887ca | ||
|
cd987e6fca | ||
|
d947ffe8e3 | ||
|
384f632e8a | ||
|
9d17948b5a | ||
|
f316f5d4e3 | ||
|
bc6f94e459 | ||
|
be3392a0d4 | ||
|
6d829d8119 | ||
|
98b0cf1cd0 | ||
|
e9611a2a36 | ||
|
807e593a32 | ||
|
297fbff23b | ||
|
37cbdfa0e7 | ||
|
295736c9cb | ||
|
14ef89a8da | ||
|
195f22f679 | ||
|
fc2beab0e7 | ||
|
1a4fbe8462 | ||
|
c2f9be3e63 | ||
|
604762a9f8 | ||
|
47e70fff8b | ||
|
de39d1281c | ||
|
27ed77aabb | ||
|
c4b19a8816 | ||
|
087ddc2371 | ||
|
65ccb0dd4e | ||
|
a874871801 | ||
|
b7c25959f0 | ||
|
f102e3dc4e | ||
|
a19855f0f5 | ||
|
ce5d36486e | ||
|
d25cf62086 | ||
|
502cefa41f | ||
|
0faa45d6c0 | ||
|
447edc48e6 | ||
|
ee8560d01e | ||
|
7135277fec | ||
|
7bbd5b13d4 | ||
|
c91cbf6072 | ||
|
11b284c81f | ||
|
c94a459a24 | ||
|
6e2626f092 | ||
|
c282e5f8d7 | ||
|
2ced5a7912 | ||
|
82e4eca711 | ||
|
1b1442887e | ||
|
22127b271c | ||
|
d35557a75d | ||
|
9493ffdb8b | ||
|
7009bb9f31 | ||
|
218c423bc0 | ||
|
55c823634d | ||
|
4050e10a4c | ||
|
ed5c44e7b7 | ||
|
0f6422590e | ||
|
4c6fba3765 | ||
|
d619dd712f | ||
|
573b13410e | ||
|
66e58dccc2 | ||
|
556862bc91 | ||
|
a8d5316aaf | ||
|
fd3f3bebd0 | ||
|
46b8ae2f52 | ||
|
538ec65ba7 | ||
|
b0a60ce203 | ||
|
e52e8b8111 | ||
|
d231b56717 | ||
|
e6a836d54c | ||
|
deee741fb1 | ||
|
adb5294177 | ||
|
5f5c127ece | ||
|
090acd58c1 | ||
|
a03b9775d5 | ||
|
8a158a936c | ||
|
11665dd236 | ||
|
cc179df346 | ||
|
0700fde640 | ||
|
811c480f7b | ||
|
3aa94d7945 | ||
|
ef044be34b | ||
|
530f4582d0 | ||
|
1baa0f5f66 | ||
|
9aa8e5340f | ||
|
04fd3289d3 | ||
|
52c3751df7 | ||
|
187a48aee2 | ||
|
be35e5343a | ||
|
c3deca86ae | ||
|
c7965b9fc2 | ||
|
e988fa4523 | ||
|
e27d8d819f | ||
|
ebc627847c | ||
|
a0068bd6be | ||
|
b764dbe773 | ||
|
871645a4a4 | ||
|
1f50a07771 | ||
|
9e5ca66f16 | ||
|
17d295a1ec | ||
|
49c5293014 | ||
|
6508688e88 | ||
|
4194d253c0 | ||
|
f8e543c906 | ||
|
c4d1738316 | ||
|
1f13ccfd7f | ||
|
923292ba64 | ||
|
782bfd26db | ||
|
3472227074 | ||
|
bf23bc0489 | ||
|
85bf26c1d0 | ||
|
d8adca1b66 | ||
|
d02064218b | ||
|
b1297308fb | ||
|
8088ce036a | ||
|
29f7bfc4d7 | ||
|
74f8cc48af | ||
|
8ff961d10f | ||
|
266b6ef185 | ||
|
825d3426c5 | ||
|
47b0c8697a | ||
|
734dfbb4e3 | ||
|
ddc080a562 | ||
|
16a3fe2ba6 | ||
|
c820a284a2 | ||
|
58babe9af7 | ||
|
6d4932f023 | ||
|
92d73ef393 | ||
|
91278f4b6b | ||
|
73e1ab6125 | ||
|
584715a803 | ||
|
e00b0eab1e | ||
|
005339d637 | ||
|
23ad6402a6 | ||
|
9642344965 | ||
|
568c7005d5 | ||
|
5cb4833f40 | ||
|
5197336de6 | ||
|
01824d275b | ||
|
39a98b09a2 | ||
|
f0a05a55c2 | ||
|
4186e81777 | ||
|
b494824286 | ||
|
8248133e5e | ||
|
27dbf6f0ab | ||
|
61d791726f | ||
|
0c0876f790 | ||
|
7a497f1405 | ||
|
5add3f4373 | ||
|
78ce962f4f | ||
|
41f0043983 | ||
|
34c06b16f5 | ||
|
1e677567cd | ||
|
af9e72507e | ||
|
6ca7b77696 | ||
|
9d142109f4 | ||
|
1ca673bd98 | ||
|
e1eae16b56 | ||
|
96f87aaa3b | ||
|
5f5de51a49 | ||
|
39ca35e765 | ||
|
d76d59d99d | ||
|
2c2c2bd348 | ||
|
46e0a729b2 | ||
|
57044eaceb | ||
|
a3373da70c | ||
|
2c4cb134a9 | ||
|
bfe72723d8 | ||
|
ed99d68bdd | ||
|
5014bd67c2 | ||
|
e418823350 | ||
|
b5242da7d2 | ||
|
a803582717 | ||
|
7fb9564420 | ||
|
379f52a495 | ||
|
cb668eb973 | ||
|
751c9ae39a | ||
|
da32828208 | ||
|
2ccee8db74 | ||
|
47f2f2fbe9 | ||
|
03ab02730f | ||
|
4c77a2e538 | ||
|
4131703001 | ||
|
cc21aebe90 | ||
|
57b9a4b4c6 | ||
|
3a7ef27cf3 | ||
|
a7f61feab2 | ||
|
8fe5d54eb7 | ||
|
d156bc8d59 | ||
|
c2350cac24 | ||
|
b224cf39d5 | ||
|
5f85eb820c | ||
|
bb7ac1ed66 | ||
|
fdf91c52a8 | ||
|
943070af4a | ||
|
82f3993ba3 | ||
|
d495292852 | ||
|
2ee6c7f110 | ||
|
6511b8e8d7 | ||
|
f3cd1d9cec | ||
|
e13a01061d | ||
|
24297a42ef | ||
|
1980ff4550 | ||
|
dfbbe2902f | ||
|
e1a9d0ef78 | ||
|
f47627a1c9 | ||
|
efeb9e0fbf | ||
|
e90a890f01 | ||
|
199c645bee | ||
|
503a3744ad | ||
|
ef03721f47 | ||
|
1e8aaa1d15 | ||
|
6423d7054e | ||
|
eb5080286a | ||
|
286e01ce30 | ||
|
8536dcafd8 | ||
|
552b139911 | ||
|
2202cef0e4 | ||
|
a726009987 | ||
|
03afef7538 | ||
|
b797c1cc75 | ||
|
04be55307a | ||
|
504e4d804d | ||
|
1786cd3fe4 | ||
|
b8645c1f58 | ||
|
fe05191b8c | ||
|
0204838163 | ||
|
a0df8a0617 | ||
|
d1b9a5e2ef | ||
|
ff04d43c46 | ||
|
d2f72c40db | ||
|
e33dfb445c | ||
|
94520568b3 | ||
|
273964d190 | ||
|
346dd3b5e8 | ||
|
f5c2c06231 | ||
|
57eaaff5cf | ||
|
999329cf6b | ||
|
c6ab792990 | ||
|
0db79d8181 | ||
|
7e8b3f9439 | ||
|
ac19c3ac80 | ||
|
c4a451bcdd | ||
|
5ad69d3d0e | ||
|
32290307a4 | ||
|
dab83a2597 | ||
|
41920fc80e | ||
|
9f6c03a006 | ||
|
596b26606c | ||
|
f20b505b46 | ||
|
cfee2dfe83 | ||
|
30a3a4c70f | ||
|
a00a7e0cad | ||
|
54558e0baa | ||
|
7c52395479 | ||
|
ea87ed8394 | ||
|
d01e261a15 | ||
|
79e4ccfc4b | ||
|
06159135ef | ||
|
4fb25ff5a3 | ||
|
1b0a13f33c | ||
|
27e5a4464d | ||
|
545d6cb9d0 | ||
|
006eea564d | ||
|
281b8e3443 | ||
|
c0c5134c57 | ||
|
72a2c0a9ed | ||
|
445db582a2 | ||
|
6b116f0c03 | ||
|
70d0d4f9be | ||
|
6b315d96bc | ||
|
25b1287323 | ||
|
760c911299 | ||
|
162bf9e10a | ||
|
6beb1ac65b | ||
|
3ae9c0f410 | ||
|
e165f5641f | ||
|
aee6feb02a | ||
|
654b4f4ff2 | ||
|
1df2596f81 | ||
|
04d4a3b136 | ||
|
392c467f95 | ||
|
c5aa8f36bf | ||
|
3748863070 | ||
|
ca304beb15 | ||
|
e789bb1aa4 | ||
|
14f29f087e | ||
|
b97fb2edac | ||
|
28bab774a0 | ||
|
8f493de9fb | ||
|
207bc35d34 | ||
|
955894e72f | ||
|
287e50b56b | ||
|
da762c4e32 | ||
|
87a8bde777 | ||
|
49fc0a567f | ||
|
cc777dcaa0 | ||
|
c785911870 | ||
|
605e7b5e47 | ||
|
8562218350 | ||
|
76da1c954a | ||
|
c2fbfb49da | ||
|
d1069d33b4 | ||
|
eafcadea26 | ||
|
a40002444e | ||
|
5208ae92fc | ||
|
8117d613ac | ||
|
00b4d72d1e | ||
|
21ccd0d7f4 | ||
|
7e79ba7dd6 | ||
|
fa6bf0a711 | ||
|
f912d6c8cf | ||
|
357bfe251d | ||
|
3be098010f | ||
|
9955bb4a27 | ||
|
ebfd66c4b1 | ||
|
b509d24b2f | ||
|
1860d0f41c | ||
|
60845121ca | ||
|
1182f9567b | ||
|
ef414343e5 | ||
|
43d986acd8 | ||
|
9c644a6419 | ||
|
fc2c6d5323 | ||
|
64ed3af328 | ||
|
bae7dbf78b | ||
|
15c24b0346 | ||
|
477bff6906 | ||
|
1a1ccd9a6e | ||
|
7dc513487f | ||
|
c6a14755bb | ||
|
7f064d50db | ||
|
b8b622fbeb | ||
|
ec64ec9651 | ||
|
f68692b004 | ||
|
8c9766f4bf | ||
|
061c030133 | ||
|
8f56907afa | ||
|
e1adb3ed4f | ||
|
e465b25c1f | ||
|
7c06216abf | ||
|
0002888627 | ||
|
3fb14cd214 | ||
|
bee6182680 | ||
|
38fe5e239a | ||
|
678d46f6bb | ||
|
3c58f9e0b9 | ||
|
ef28e33249 | ||
|
9662e4964b | ||
|
44603290e5 | ||
|
1631fca1ee | ||
|
295860ff00 | ||
|
8cb4b71909 | ||
|
d81421af4b | ||
|
7422a2194f | ||
|
2090dbdc8c | ||
|
0a04e03a02 | ||
|
44b2d5f5fc | ||
|
aa9118a373 | ||
|
36abc16c3c | ||
|
919d764600 | ||
|
696183e133 | ||
|
f90d825a6b | ||
|
3037ab00c7 | ||
|
21e872b19a | ||
|
cf2dbec630 | ||
|
b92bb0e02a | ||
|
40edffae3d | ||
|
9fc5eafb8e | ||
|
08c2fbb844 | ||
|
3997efb65e | ||
|
a7356dffe9 | ||
|
e20ec43094 | ||
|
70baa7bfae | ||
|
8980f53b42 | ||
|
a363fb5d28 | ||
|
646052e416 | ||
|
844e4cbc54 | ||
|
56c63c8c02 | ||
|
07eb8f1916 | ||
|
4b5410c5c8 | ||
|
be2e9b76ee | ||
|
d8085580f6 | ||
|
6d32c6c6d3 | ||
|
f94d764993 | ||
|
f28f1b4d6e | ||
|
360d5f0daa | ||
|
cd493c5adc | ||
|
a4c7ed6b1e | ||
|
7f8b8bc418 | ||
|
311ebdd9a5 | ||
|
99c68db0a8 | ||
|
5fc53690cb | ||
|
7a9161578e | ||
|
2405854705 | ||
|
0cf09c2b41 | ||
|
0156ce95c5 | ||
|
1641b13232 | ||
|
a4bdc3112b | ||
|
c7d407bca2 | ||
|
7215691ab7 | ||
|
fc88e8f0e3 | ||
|
cfefb7d854 | ||
|
3c07d007ca | ||
|
89c5a7d5aa | ||
|
2adc0c51cd | ||
|
1f0910bc27 | ||
|
e22ff4e356 | ||
|
83031d749b | ||
|
1b731ebcaa | ||
|
ab25f3f431 | ||
|
07f7aad81c | ||
|
1e2575df87 | ||
|
b111a64135 | ||
|
0e3a968479 | ||
|
c11f7cf9bd | ||
|
8fa7cc387d | ||
|
65eee5a745 | ||
|
efef4ddf51 | ||
|
159a3d48df | ||
|
b46483a6ec | ||
|
9c724601ba | ||
|
67299f23d8 | ||
|
8bf9591a70 | ||
|
a800838f5a | ||
|
ba15b2fee6 | ||
|
56a7ee9033 | ||
|
0b4f03a563 | ||
|
7b8fa658f8 | ||
|
fd95fc33b1 | ||
|
c669554ef5 | ||
|
11b68df7a4 | ||
|
d18f4419a7 | ||
|
0f7d413d5b | ||
|
286e5d6724 | ||
|
395981288b | ||
|
55bb3556c8 | ||
|
57f2488bbe | ||
|
ea399a53eb | ||
|
811a183eb6 | ||
|
b63981e850 | ||
|
186cbaffb9 | ||
|
dbf3fa8af6 | ||
|
f08c31cf33 | ||
|
d8dab85419 | ||
|
5519bba3e1 | ||
|
142c584063 | ||
|
4542e3e555 | ||
|
fa8f6d8580 | ||
|
3bb7769c40 | ||
|
8d286bd5b6 | ||
|
cff72b4cc0 | ||
|
657221c81d | ||
|
62acf5fa2c | ||
|
b79977fb6b | ||
|
bc7c8f3d4e | ||
|
015e19b350 | ||
|
54856480d7 | ||
|
1dd12708c2 | ||
|
f9201cef58 | ||
|
26499ba823 | ||
|
58f6c2112d | ||
|
de026a6acd | ||
|
d4564afc70 | ||
|
360a5e0f60 | ||
|
55a3ca16d3 | ||
|
ef50cb3fda | ||
|
8673f4344c | ||
|
f1487d4fca | ||
|
0cd4c402f0 | ||
|
9c9b458145 | ||
|
9d50f86232 | ||
|
7e92f9015e | ||
|
aa860b8016 | ||
|
b484097b01 | ||
|
ab9001dab5 | ||
|
879866a230 | ||
|
8e5477d036 | ||
|
1e8e5d5238 | ||
|
d81a213cfb | ||
|
7c2d18a13f | ||
|
2408e6d26a | ||
|
cf862771d7 | ||
|
a938f111ed | ||
|
4759543f6e | ||
|
d0fc289f45 | ||
|
70f572585d | ||
|
c2d06aef60 | ||
|
ff1e765400 | ||
|
170e1c1995 | ||
|
61e669acff | ||
|
2c337f4e85 | ||
|
bf6a74c620 | ||
|
38a967c98e | ||
|
3a61e6d360 | ||
|
3d8e32dcc0 | ||
|
8f29b2dd38 | ||
|
a29e340efa | ||
|
b13f29098f | ||
|
430c4bc9d0 | ||
|
4ae243fc6c | ||
|
8f20ad36dc | ||
|
799c794947 | ||
|
1ae7ae0b96 | ||
|
ccc7112291 | ||
|
5b24f8f505 | ||
|
fcd90d2583 | ||
|
8f757c7353 | ||
|
be1a3f2d11 | ||
|
ecae54a98d | ||
|
f318882955 | ||
|
c3399cac19 | ||
|
9237aaa77f | ||
|
766fcdd0fa | ||
|
f6ea29e24b | ||
|
8a3797a4ab | ||
|
745db8899d | ||
|
83db801cbf | ||
|
964a8eb754 | ||
|
ac61f2e058 | ||
|
8487e8b98a | ||
|
9c484c0019 | ||
|
0e96b4b5ce | ||
|
a563c97c5c | ||
|
e88c9ef62a | ||
|
0889eb33e0 | ||
|
0021a2b9a1 | ||
|
19ec468635 | ||
|
491ee7efe4 | ||
|
8522bcd97c | ||
|
ac71fd5919 | ||
|
8e953dcbb1 | ||
|
f4afb9a6a8 | ||
|
d5b8cf093c | ||
|
5c6e84c0ff | ||
|
1aaee908b9 | ||
|
b2d9fd9c9f | ||
|
bc2f83b95e | ||
|
85de33b04e | ||
|
7dfd966848 | ||
|
a25d03d7cb | ||
|
cabfd4b1f0 | ||
|
7b643d4cd0 | ||
|
1f1d01d498 | ||
|
21a42e2588 | ||
|
2df93a0c4a | ||
|
75972e200d | ||
|
d0d838638c | ||
|
8c17afc471 | ||
|
40d66e07df | ||
|
ab89a8678b | ||
|
4d7d056909 | ||
|
c35bc82606 | ||
|
2f56caf083 | ||
|
4066945919 | ||
|
2a84694b1e | ||
|
4046ffe1e1 | ||
|
d1d0612160 | ||
|
7b0f04ed1f | ||
|
2e21b06ea2 | ||
|
a6f75e6e89 | ||
|
bd18824c2a | ||
|
bdd044e67b | ||
|
f7e95fb2a0 | ||
|
9dd674e1d2 | ||
|
9c1e164e0c | ||
|
c706fbe9fe | ||
|
ebdcf70b0d | ||
|
5966095e65 | ||
|
9ee984fc76 | ||
|
53528e1d23 | ||
|
c931c4b8dd | ||
|
7acd042bbb | ||
|
bcfe485e01 | ||
|
479cc6d5a1 | ||
|
38286ee729 | ||
|
1a95953867 | ||
|
71febd1c52 | ||
|
f1bc56c99b | ||
|
64e419bd73 | ||
|
782ea947b4 | ||
|
f27224d57b | ||
|
c007188598 | ||
|
af93ecfd88 | ||
|
794771a164 | ||
|
6f2eaaf73d | ||
|
4c7a4dbc4d | ||
|
f86b299d0e | ||
|
e474996541 | ||
|
aed617e311 | ||
|
0fa67c1d68 | ||
|
365b3cc72d | ||
|
a272fe21a8 | ||
|
cec1c2f211 | ||
|
12053450dc | ||
|
46cffb0c47 | ||
|
c32a059f52 | ||
|
6911312e53 | ||
|
f22b5a6b96 | ||
|
58e55198c1 | ||
|
d61ed9f2f1 | ||
|
8bc4c6350e | ||
|
cfa4ffa23b | ||
|
4f1dc1463d | ||
|
17e0f41d34 | ||
|
b57b27ff8f | ||
|
bbe8cc6662 | ||
|
98106accb6 | ||
|
af1312bfc3 | ||
|
4c7d7215cd | ||
|
0370d9eb3d | ||
|
1434651d20 | ||
|
2c312ab84a | ||
|
0ee78d62d5 | ||
|
7f3c90ab25 | ||
|
1d3cd29730 | ||
|
4ef1fc9707 | ||
|
f9e6aa1dcf | ||
|
f83db9064b | ||
|
2da9a86399 | ||
|
ecaa535cf4 | ||
|
79dd92b1fe | ||
|
bd3844c9c2 | ||
|
7bf5e3a84a | ||
|
08a17dae5b | ||
|
924ea66ade | ||
|
5b72f5b74f | ||
|
bfa345744d | ||
|
f966461476 | ||
|
b8aea53682 | ||
|
c0d9eb7043 | ||
|
3ba6aabd25 | ||
|
a8b31505ed | ||
|
90a271e914 | ||
|
172d270607 | ||
|
22feed08a1 | ||
|
942b8ca3be | ||
|
3729c52f9d | ||
|
71679eaee8 | ||
|
76fe4ba3b2 | ||
|
164a4a5756 | ||
|
455951985b | ||
|
c29500e412 | ||
|
1bc1520adc | ||
|
022e05dc1f | ||
|
b34c9551aa | ||
|
84f19c026f | ||
|
6bde5492b6 | ||
|
6086df4d6a | ||
|
c98052c5da | ||
|
ab62bc5838 | ||
|
bc87ba8424 | ||
|
b79df1b68d | ||
|
2797c7be45 | ||
|
755f186e21 | ||
|
2240a1dc4d | ||
|
03d3af9768 | ||
|
5ce9527e16 | ||
|
c527f5ada0 | ||
|
ace52668f0 | ||
|
9c33eb027e | ||
|
679b711395 | ||
|
1727541315 | ||
|
45b0a0d11b | ||
|
e665fcd4da | ||
|
aae737d4af | ||
|
92a6de861e | ||
|
5ff881aee6 | ||
|
eae19a4473 | ||
|
f70c263ce5 | ||
|
92d135921f | ||
|
d8008dee4f | ||
|
bb38a12157 | ||
|
bcc8ef0a5a | ||
|
3cb1a5dc73 | ||
|
ed40c02c9b | ||
|
b5fdceb4d1 | ||
|
1e6f7f3b45 | ||
|
469ff02f4e | ||
|
56f1c5ad38 | ||
|
3a6b8f4edf | ||
|
3ae02d4a62 | ||
|
59fea63dc2 | ||
|
22933e02d4 | ||
|
98a62d7cbd | ||
|
d6c5fdef34 | ||
|
4b9051cf39 | ||
|
00f5068908 | ||
|
432c6b0f48 | ||
|
ad5e5788ff | ||
|
dc65041c22 | ||
|
9f88b07945 | ||
|
225646c4ca | ||
|
5c6e9f0b6c | ||
|
6c370bc149 | ||
|
37fd242342 | ||
|
aee1f87168 | ||
|
b69bb1ed11 | ||
|
772cefef8c | ||
|
842654b6d0 | ||
|
df5e50954b | ||
|
a4a2fa8754 | ||
|
9da0504a09 | ||
|
470cf496f5 | ||
|
e029da9add | ||
|
e00b8f60d4 | ||
|
644c3ef886 | ||
|
9d8d0f8b4a | ||
|
5a1fbbf8b7 | ||
|
e2bdf8bf4f | ||
|
c368dc98e0 | ||
|
e7eff914cd | ||
|
07333d0062 | ||
|
5bd7ad2e81 | ||
|
3ded751985 | ||
|
6956db3606 | ||
|
17b01228f8 | ||
|
4f1ecca58d | ||
|
2717036489 | ||
|
d9482c0083 | ||
|
791b743765 | ||
|
fa604d9083 | ||
|
2bf0634d16 | ||
|
dccf4932e1 | ||
|
91dd25fe1e | ||
|
06bf2ac20f | ||
|
6ad0d8781e | ||
|
f2c704e112 | ||
|
5e822c2526 | ||
|
cc017e07ca | ||
|
082da36416 | ||
|
6bf95b15ee | ||
|
4c93b2fd15 | ||
|
1b26bfd425 | ||
|
13ec444a98 | ||
|
51579d87e4 | ||
|
e147619669 | ||
|
e7f93fbd85 | ||
|
58f7ada235 | ||
|
c67b33888f | ||
|
e8c0af04b7 | ||
|
5d769860c3 | ||
|
02b04785ee | ||
|
41c92b8d02 | ||
|
fe0c28f956 | ||
|
957c65b9ea | ||
|
5e95e18ce9 | ||
|
e91df0c550 | ||
|
c5636e9bca | ||
|
2e47264235 | ||
|
1c78cb118c | ||
|
beab2f88c9 | ||
|
0025447369 | ||
|
da4304609d | ||
|
ea89680aea | ||
|
664dd8ba85 | ||
|
64554c12e1 | ||
|
4ded9c0f00 | ||
|
c0820dd52a | ||
|
2bb70750a9 | ||
|
09d923f2eb | ||
|
37d979ad33 | ||
|
95ac4de229 | ||
|
d3e142b3fa | ||
|
132aece1ed | ||
|
3e4e338133 | ||
|
be19ae11fd | ||
|
59d63d8d4a | ||
|
cfeba5d17f | ||
|
6da0e5e7a2 | ||
|
d6ce649f15 | ||
|
b449b73dcc | ||
|
16c822e91e | ||
|
4318170779 | ||
|
fb626c0586 | ||
|
717d1d2d5a | ||
|
9585b376db | ||
|
f04cfe24e0 | ||
|
20c50c6556 | ||
|
f9f9699f2f | ||
|
a3cf22e590 | ||
|
99de2f38d3 | ||
|
9fe50837c3 | ||
|
4dc545553f | ||
|
686e898fde | ||
|
3a78198a96 | ||
|
836c810716 | ||
|
97c5be383c | ||
|
a7ea88537a | ||
|
6c35de4c6b | ||
|
579d43951d | ||
|
e24ebeafd1 | ||
|
01c92973dd | ||
|
f4415faa46 | ||
|
a86ce9d7a1 | ||
|
37258c644f | ||
|
d0512ac4c5 | ||
|
da4eaa15a2 | ||
|
191286265d | ||
|
3234272818 | ||
|
9d2c90354f | ||
|
316b10855a | ||
|
484fe78737 | ||
|
2cd43a00d1 | ||
|
dd0f524c69 | ||
|
c84f9475b8 | ||
|
15f2734791 | ||
|
cb6e24f946 | ||
|
9d531aa291 | ||
|
e9cbb98a0f | ||
|
193422e12a | ||
|
c4cabf040e | ||
|
f23eceebbf | ||
|
650bd8f623 | ||
|
5d8cb4367d | ||
|
82abc13aed | ||
|
1e72660c9b | ||
|
049f224248 | ||
|
0ada1b90b8 | ||
|
71ddc222ad | ||
|
21292c0649 | ||
|
46a265a2da | ||
|
e2096776b9 | ||
|
a1c88c4819 | ||
|
51ca93d751 | ||
|
a7e0531999 | ||
|
ab0eda99e1 | ||
|
ec99f47108 | ||
|
b31b5f4434 | ||
|
86f2fa1590 | ||
|
7bc7fbce23 | ||
|
a78e530c14 | ||
|
af7bb684c0 | ||
|
dd9e0f58f3 | ||
|
59e583f7e8 | ||
|
daa25d4142 | ||
|
25a35cb38a | ||
|
2cf8003638 | ||
|
cf1a8668e8 | ||
|
2dbb45ae82 | ||
|
4fe190df70 | ||
|
039e715b30 | ||
|
32152bab7a | ||
|
b1347a5881 | ||
|
91e954587f | ||
|
9b505185da | ||
|
444a68e0ec | ||
|
c7178f0f7a | ||
|
5c3f7014ef | ||
|
d65628ef03 | ||
|
8a6c5b0806 | ||
|
9360936f26 | ||
|
bb2b89e077 | ||
|
2d7a29081c | ||
|
2864179293 | ||
|
284f8306df | ||
|
aa613ef7e1 | ||
|
cb2b9a22a5 | ||
|
5b867c15a8 | ||
|
f8c749f12c | ||
|
7d509c613b | ||
|
b92e95aa01 | ||
|
e3cad6bd99 | ||
|
6699b6ce41 | ||
|
91dcde8a38 | ||
|
11f3471c4b | ||
|
f4093b34f6 | ||
|
a80b23c373 | ||
|
2e7fa18bb9 | ||
|
fe07e788bf | ||
|
6d3bdcf217 | ||
|
ec2a2ab441 | ||
|
9448a20312 | ||
|
3f1748b944 | ||
|
fe13087cd1 | ||
|
f22fa82d7f | ||
|
059fa9aa81 | ||
|
650aec4a98 | ||
|
2ea9c97432 | ||
|
efc589b865 | ||
|
1737ea69b9 | ||
|
1fb034d029 | ||
|
28f9568a84 | ||
|
5bbdadd5f8 | ||
|
2de2ca6659 | ||
|
051071203c | ||
|
34299510bb | ||
|
b9bceba37c | ||
|
6d4733ce7b | ||
|
416da574ec | ||
|
48c5663c5f | ||
|
7d740e7dc7 | ||
|
4eda10499e | ||
|
605535776a | ||
|
1050e0d09f | ||
|
d65d89183f | ||
|
0c92f1e96b | ||
|
adae9e844b | ||
|
c5764b3f89 | ||
|
0837992a22 | ||
|
b55715934b | ||
|
bbc3b5b4bb | ||
|
1ca5f821c8 | ||
|
defc820b70 | ||
|
82ef02e936 | ||
|
b856b3997c | ||
|
cd85a1bb8b | ||
|
ce5b904050 | ||
|
ad06b99dd4 | ||
|
540b9f5164 | ||
|
6e65a2a67e | ||
|
f8c7bed133 | ||
|
cdc55e666f | ||
|
86b7c00adc | ||
|
e8c5d40bc8 | ||
|
ca7ebc4e5e | ||
|
bff857a8af | ||
|
a31a022efd | ||
|
45f6362464 | ||
|
97f34a48d7 | ||
|
ea74e00b3a | ||
|
06cd4cdb25 | ||
|
da2069fb22 | ||
|
95c9810015 | ||
|
b03eebdb6a | ||
|
1f7675451c | ||
|
aa27253556 | ||
|
d51e23d9fc | ||
|
6cd452acff | ||
|
50e9fcc1fd | ||
|
16ee69c1b7 | ||
|
67171ed7e9 | ||
|
1d9bf655e6 | ||
|
62ae19ff76 | ||
|
5ed05f26ad | ||
|
841b683804 | ||
|
f5863a3ea0 | ||
|
10709fc7c6 | ||
|
a7e348556a | ||
|
6cb30ea5ed | ||
|
a4ed50bb84 | ||
|
570611955f | ||
|
e450f6cb63 | ||
|
a115e07594 | ||
|
718393c632 | ||
|
07af16b92e | ||
|
e942cfd1a7 | ||
|
9a7e5cb88a | ||
|
2391941f28 | ||
|
9ff6165a81 | ||
|
1c748722f9 | ||
|
ee0b726cd7 | ||
|
dbeafce5d5 | ||
|
ed604ce7bc | ||
|
7adc7ca547 | ||
|
a6211d237b | ||
|
7b16239a49 | ||
|
37357d21a9 | ||
|
b477fc1314 | ||
|
d84b21b427 | ||
|
48bd042ce7 | ||
|
84213ea8d4 | ||
|
562de77f41 | ||
|
e1723c4bac | ||
|
607d204551 | ||
|
a5b6102ea8 | ||
|
b77888228d | ||
|
0b1eaec3bc | ||
|
b37e47a3f9 | ||
|
ce3735df02 | ||
|
a0455d0ffd | ||
|
c8b232cc48 | ||
|
b4eb0bc7bd | ||
|
d5147b65ac | ||
|
7b0b53ea69 | ||
|
7016e24ebe | ||
|
bef4688c72 | ||
|
228c1d685b | ||
|
efd72b05d2 | ||
|
fe515e5c75 | ||
|
1db5ab6b34 | ||
|
2791e80b60 | ||
|
8f841fafcd | ||
|
a54c5f83c0 | ||
|
cd13343ad8 | ||
|
9cd5f54e31 | ||
|
9a269547f2 | ||
|
bf097a5077 | ||
|
52c50a10af | ||
|
b334732709 | ||
|
384bf91f88 | ||
|
fae11394f0 | ||
|
adc13b0748 | ||
|
327593257c | ||
|
9d8f3a12a6 | ||
|
b002bc433a | ||
|
b74896dad1 | ||
|
fa3db38333 | ||
|
30fa5c6087 | ||
|
6c907eb33f | ||
|
f7b42518dc | ||
|
ce7db64bf1 | ||
|
1328305851 | ||
|
6c22cee673 | ||
|
6d874fee2a | ||
|
676723e0da | ||
|
c380cc28c4 | ||
|
f7f304910d | ||
|
00a41ca4c3 | ||
|
66f32ca0e1 | ||
|
6ffc3cf74a | ||
|
4433bb0245 | ||
|
e40c758c2a | ||
|
011e75e641 | ||
|
2468a6fa64 | ||
|
700265bfcf | ||
|
c97f5e934f | ||
|
38db9a405a | ||
|
2cdfe977d7 | ||
|
46d0baf941 | ||
|
00eb865b3c | ||
|
2f19835726 | ||
|
533f3e3557 | ||
|
75294a5ed0 | ||
|
b9e5f87291 | ||
|
6b09401b0b | ||
|
5caf88ccb4 | ||
|
dcc8522fdb | ||
|
c9595ee780 | ||
|
91bd3bd019 | ||
|
13b08034b5 | ||
|
6a6e1a0cd8 | ||
|
4e7b5bba5f | ||
|
52c4c51556 | ||
|
8fae1a04eb | ||
|
d44a707fdd | ||
|
049c0486bb | ||
|
30b5121a1c | ||
|
b439634f0e | ||
|
6e47200b6e | ||
|
38fa761a45 | ||
|
08a27407c4 | ||
|
be7dacf9cf | ||
|
4560adc820 | ||
|
63dce3094b | ||
|
b4eb08bb03 | ||
|
2e20cb3636 | ||
|
a6c5859d6b | ||
|
c76cdf2382 | ||
|
787c360467 | ||
|
73453430c1 | ||
|
158bc5ac03 | ||
|
4568a11802 | ||
|
4cbce88f8b | ||
|
541fe3eaff | ||
|
9bfe088594 | ||
|
fcaf4d7a06 | ||
|
40b6495d40 | ||
|
f1a8511f7b | ||
|
042b664933 | ||
|
68fa15155f | ||
|
434f573046 | ||
|
cff99c91d1 | ||
|
fa9b8c6628 | ||
|
ea782aca52 | ||
|
43ebf77df3 | ||
|
d332ec725d | ||
|
f93abcf1da | ||
|
0ec9d4e565 |
399 changed files with 39653 additions and 16162 deletions
6
.github/ISSUE_TEMPLATE/1_broken_site.md
vendored
6
.github/ISSUE_TEMPLATE/1_broken_site.md
vendored
|
@ -18,7 +18,7 @@ title: ''
|
||||||
|
|
||||||
<!--
|
<!--
|
||||||
Carefully read and work through this check list in order to prevent the most common mistakes and misuse of youtube-dl:
|
Carefully read and work through this check list in order to prevent the most common mistakes and misuse of youtube-dl:
|
||||||
- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2020.03.06. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
|
- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.12.17. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
|
||||||
- Make sure that all provided video/audio/playlist URLs (if any) are alive and playable in a browser.
|
- Make sure that all provided video/audio/playlist URLs (if any) are alive and playable in a browser.
|
||||||
- Make sure that all URLs and arguments with special characters are properly quoted or escaped as explained in http://yt-dl.org/escape.
|
- Make sure that all URLs and arguments with special characters are properly quoted or escaped as explained in http://yt-dl.org/escape.
|
||||||
- Search the bugtracker for similar issues: http://yt-dl.org/search-issues. DO NOT post duplicates.
|
- Search the bugtracker for similar issues: http://yt-dl.org/search-issues. DO NOT post duplicates.
|
||||||
|
@ -26,7 +26,7 @@ Carefully read and work through this check list in order to prevent the most com
|
||||||
-->
|
-->
|
||||||
|
|
||||||
- [ ] I'm reporting a broken site support
|
- [ ] I'm reporting a broken site support
|
||||||
- [ ] I've verified that I'm running youtube-dl version **2020.03.06**
|
- [ ] I've verified that I'm running youtube-dl version **2021.12.17**
|
||||||
- [ ] I've checked that all provided URLs are alive and playable in a browser
|
- [ ] I've checked that all provided URLs are alive and playable in a browser
|
||||||
- [ ] I've checked that all URLs and arguments with special characters are properly quoted or escaped
|
- [ ] I've checked that all URLs and arguments with special characters are properly quoted or escaped
|
||||||
- [ ] I've searched the bugtracker for similar issues including closed ones
|
- [ ] I've searched the bugtracker for similar issues including closed ones
|
||||||
|
@ -41,7 +41,7 @@ Add the `-v` flag to your command line you run youtube-dl with (`youtube-dl -v <
|
||||||
[debug] User config: []
|
[debug] User config: []
|
||||||
[debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
|
[debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
|
||||||
[debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
|
[debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
|
||||||
[debug] youtube-dl version 2020.03.06
|
[debug] youtube-dl version 2021.12.17
|
||||||
[debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
|
[debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
|
||||||
[debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
|
[debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
|
||||||
[debug] Proxy map: {}
|
[debug] Proxy map: {}
|
||||||
|
|
|
@ -19,7 +19,7 @@ labels: 'site-support-request'
|
||||||
|
|
||||||
<!--
|
<!--
|
||||||
Carefully read and work through this check list in order to prevent the most common mistakes and misuse of youtube-dl:
|
Carefully read and work through this check list in order to prevent the most common mistakes and misuse of youtube-dl:
|
||||||
- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2020.03.06. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
|
- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.12.17. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
|
||||||
- Make sure that all provided video/audio/playlist URLs (if any) are alive and playable in a browser.
|
- Make sure that all provided video/audio/playlist URLs (if any) are alive and playable in a browser.
|
||||||
- Make sure that site you are requesting is not dedicated to copyright infringement, see https://yt-dl.org/copyright-infringement. youtube-dl does not support such sites. In order for site support request to be accepted all provided example URLs should not violate any copyrights.
|
- Make sure that site you are requesting is not dedicated to copyright infringement, see https://yt-dl.org/copyright-infringement. youtube-dl does not support such sites. In order for site support request to be accepted all provided example URLs should not violate any copyrights.
|
||||||
- Search the bugtracker for similar site support requests: http://yt-dl.org/search-issues. DO NOT post duplicates.
|
- Search the bugtracker for similar site support requests: http://yt-dl.org/search-issues. DO NOT post duplicates.
|
||||||
|
@ -27,7 +27,7 @@ Carefully read and work through this check list in order to prevent the most com
|
||||||
-->
|
-->
|
||||||
|
|
||||||
- [ ] I'm reporting a new site support request
|
- [ ] I'm reporting a new site support request
|
||||||
- [ ] I've verified that I'm running youtube-dl version **2020.03.06**
|
- [ ] I've verified that I'm running youtube-dl version **2021.12.17**
|
||||||
- [ ] I've checked that all provided URLs are alive and playable in a browser
|
- [ ] I've checked that all provided URLs are alive and playable in a browser
|
||||||
- [ ] I've checked that none of provided URLs violate any copyrights
|
- [ ] I've checked that none of provided URLs violate any copyrights
|
||||||
- [ ] I've searched the bugtracker for similar site support requests including closed ones
|
- [ ] I've searched the bugtracker for similar site support requests including closed ones
|
||||||
|
|
|
@ -18,13 +18,13 @@ title: ''
|
||||||
|
|
||||||
<!--
|
<!--
|
||||||
Carefully read and work through this check list in order to prevent the most common mistakes and misuse of youtube-dl:
|
Carefully read and work through this check list in order to prevent the most common mistakes and misuse of youtube-dl:
|
||||||
- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2020.03.06. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
|
- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.12.17. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
|
||||||
- Search the bugtracker for similar site feature requests: http://yt-dl.org/search-issues. DO NOT post duplicates.
|
- Search the bugtracker for similar site feature requests: http://yt-dl.org/search-issues. DO NOT post duplicates.
|
||||||
- Finally, put x into all relevant boxes (like this [x])
|
- Finally, put x into all relevant boxes (like this [x])
|
||||||
-->
|
-->
|
||||||
|
|
||||||
- [ ] I'm reporting a site feature request
|
- [ ] I'm reporting a site feature request
|
||||||
- [ ] I've verified that I'm running youtube-dl version **2020.03.06**
|
- [ ] I've verified that I'm running youtube-dl version **2021.12.17**
|
||||||
- [ ] I've searched the bugtracker for similar site feature requests including closed ones
|
- [ ] I've searched the bugtracker for similar site feature requests including closed ones
|
||||||
|
|
||||||
|
|
||||||
|
|
6
.github/ISSUE_TEMPLATE/4_bug_report.md
vendored
6
.github/ISSUE_TEMPLATE/4_bug_report.md
vendored
|
@ -18,7 +18,7 @@ title: ''
|
||||||
|
|
||||||
<!--
|
<!--
|
||||||
Carefully read and work through this check list in order to prevent the most common mistakes and misuse of youtube-dl:
|
Carefully read and work through this check list in order to prevent the most common mistakes and misuse of youtube-dl:
|
||||||
- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2020.03.06. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
|
- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.12.17. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
|
||||||
- Make sure that all provided video/audio/playlist URLs (if any) are alive and playable in a browser.
|
- Make sure that all provided video/audio/playlist URLs (if any) are alive and playable in a browser.
|
||||||
- Make sure that all URLs and arguments with special characters are properly quoted or escaped as explained in http://yt-dl.org/escape.
|
- Make sure that all URLs and arguments with special characters are properly quoted or escaped as explained in http://yt-dl.org/escape.
|
||||||
- Search the bugtracker for similar issues: http://yt-dl.org/search-issues. DO NOT post duplicates.
|
- Search the bugtracker for similar issues: http://yt-dl.org/search-issues. DO NOT post duplicates.
|
||||||
|
@ -27,7 +27,7 @@ Carefully read and work through this check list in order to prevent the most com
|
||||||
-->
|
-->
|
||||||
|
|
||||||
- [ ] I'm reporting a broken site support issue
|
- [ ] I'm reporting a broken site support issue
|
||||||
- [ ] I've verified that I'm running youtube-dl version **2020.03.06**
|
- [ ] I've verified that I'm running youtube-dl version **2021.12.17**
|
||||||
- [ ] I've checked that all provided URLs are alive and playable in a browser
|
- [ ] I've checked that all provided URLs are alive and playable in a browser
|
||||||
- [ ] I've checked that all URLs and arguments with special characters are properly quoted or escaped
|
- [ ] I've checked that all URLs and arguments with special characters are properly quoted or escaped
|
||||||
- [ ] I've searched the bugtracker for similar bug reports including closed ones
|
- [ ] I've searched the bugtracker for similar bug reports including closed ones
|
||||||
|
@ -43,7 +43,7 @@ Add the `-v` flag to your command line you run youtube-dl with (`youtube-dl -v <
|
||||||
[debug] User config: []
|
[debug] User config: []
|
||||||
[debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
|
[debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
|
||||||
[debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
|
[debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
|
||||||
[debug] youtube-dl version 2020.03.06
|
[debug] youtube-dl version 2021.12.17
|
||||||
[debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
|
[debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
|
||||||
[debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
|
[debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
|
||||||
[debug] Proxy map: {}
|
[debug] Proxy map: {}
|
||||||
|
|
4
.github/ISSUE_TEMPLATE/5_feature_request.md
vendored
4
.github/ISSUE_TEMPLATE/5_feature_request.md
vendored
|
@ -19,13 +19,13 @@ labels: 'request'
|
||||||
|
|
||||||
<!--
|
<!--
|
||||||
Carefully read and work through this check list in order to prevent the most common mistakes and misuse of youtube-dl:
|
Carefully read and work through this check list in order to prevent the most common mistakes and misuse of youtube-dl:
|
||||||
- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2020.03.06. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
|
- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.12.17. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
|
||||||
- Search the bugtracker for similar feature requests: http://yt-dl.org/search-issues. DO NOT post duplicates.
|
- Search the bugtracker for similar feature requests: http://yt-dl.org/search-issues. DO NOT post duplicates.
|
||||||
- Finally, put x into all relevant boxes (like this [x])
|
- Finally, put x into all relevant boxes (like this [x])
|
||||||
-->
|
-->
|
||||||
|
|
||||||
- [ ] I'm reporting a feature request
|
- [ ] I'm reporting a feature request
|
||||||
- [ ] I've verified that I'm running youtube-dl version **2020.03.06**
|
- [ ] I've verified that I'm running youtube-dl version **2021.12.17**
|
||||||
- [ ] I've searched the bugtracker for similar feature requests including closed ones
|
- [ ] I've searched the bugtracker for similar feature requests including closed ones
|
||||||
|
|
||||||
|
|
||||||
|
|
1
.github/ISSUE_TEMPLATE/config.yml
vendored
Normal file
1
.github/ISSUE_TEMPLATE/config.yml
vendored
Normal file
|
@ -0,0 +1 @@
|
||||||
|
blank_issues_enabled: false
|
4
.github/PULL_REQUEST_TEMPLATE.md
vendored
4
.github/PULL_REQUEST_TEMPLATE.md
vendored
|
@ -7,8 +7,10 @@
|
||||||
---
|
---
|
||||||
|
|
||||||
### Before submitting a *pull request* make sure you have:
|
### Before submitting a *pull request* make sure you have:
|
||||||
- [ ] At least skimmed through [adding new extractor tutorial](https://github.com/ytdl-org/youtube-dl#adding-support-for-a-new-site) and [youtube-dl coding conventions](https://github.com/ytdl-org/youtube-dl#youtube-dl-coding-conventions) sections
|
|
||||||
- [ ] [Searched](https://github.com/ytdl-org/youtube-dl/search?q=is%3Apr&type=Issues) the bugtracker for similar pull requests
|
- [ ] [Searched](https://github.com/ytdl-org/youtube-dl/search?q=is%3Apr&type=Issues) the bugtracker for similar pull requests
|
||||||
|
- [ ] Read [adding new extractor tutorial](https://github.com/ytdl-org/youtube-dl#adding-support-for-a-new-site)
|
||||||
|
- [ ] Read [youtube-dl coding conventions](https://github.com/ytdl-org/youtube-dl#youtube-dl-coding-conventions) and adjusted the code to meet them
|
||||||
|
- [ ] Covered the code with tests (note that PRs without tests will be REJECTED)
|
||||||
- [ ] Checked the code with [flake8](https://pypi.python.org/pypi/flake8)
|
- [ ] Checked the code with [flake8](https://pypi.python.org/pypi/flake8)
|
||||||
|
|
||||||
### In order to be accepted and merged into youtube-dl each piece of code must be in public domain or released under [Unlicense](http://unlicense.org/). Check one of the following options:
|
### In order to be accepted and merged into youtube-dl each piece of code must be in public domain or released under [Unlicense](http://unlicense.org/). Check one of the following options:
|
||||||
|
|
479
.github/workflows/ci.yml
vendored
Normal file
479
.github/workflows/ci.yml
vendored
Normal file
|
@ -0,0 +1,479 @@
|
||||||
|
name: CI
|
||||||
|
|
||||||
|
env:
|
||||||
|
all-cpython-versions: 2.6, 2.7, 3.2, 3.3, 3.4, 3.5, 3.6, 3.7, 3.8, 3.9, 3.10, 3.11, 3.12
|
||||||
|
main-cpython-versions: 2.7, 3.2, 3.5, 3.9, 3.11
|
||||||
|
pypy-versions: pypy-2.7, pypy-3.6, pypy-3.7
|
||||||
|
cpython-versions: main
|
||||||
|
test-set: core
|
||||||
|
# Python beta version to be built using pyenv before setup-python support
|
||||||
|
# Must also be included in all-cpython-versions
|
||||||
|
next: 3.13
|
||||||
|
|
||||||
|
on:
|
||||||
|
push:
|
||||||
|
# push inputs aren't known to GitHub
|
||||||
|
inputs:
|
||||||
|
cpython-versions:
|
||||||
|
type: string
|
||||||
|
default: all
|
||||||
|
test-set:
|
||||||
|
type: string
|
||||||
|
default: core
|
||||||
|
pull_request:
|
||||||
|
# pull_request inputs aren't known to GitHub
|
||||||
|
inputs:
|
||||||
|
cpython-versions:
|
||||||
|
type: string
|
||||||
|
default: main
|
||||||
|
test-set:
|
||||||
|
type: string
|
||||||
|
default: both
|
||||||
|
workflow_dispatch:
|
||||||
|
inputs:
|
||||||
|
cpython-versions:
|
||||||
|
type: choice
|
||||||
|
description: CPython versions (main = 2.7, 3.2, 3.5, 3.9, 3.11)
|
||||||
|
options:
|
||||||
|
- all
|
||||||
|
- main
|
||||||
|
required: true
|
||||||
|
default: main
|
||||||
|
test-set:
|
||||||
|
type: choice
|
||||||
|
description: core, download
|
||||||
|
options:
|
||||||
|
- both
|
||||||
|
- core
|
||||||
|
- download
|
||||||
|
required: true
|
||||||
|
default: both
|
||||||
|
|
||||||
|
permissions:
|
||||||
|
contents: read
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
select:
|
||||||
|
name: Select tests from inputs
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
outputs:
|
||||||
|
cpython-versions: ${{ steps.run.outputs.cpython-versions }}
|
||||||
|
test-set: ${{ steps.run.outputs.test-set }}
|
||||||
|
own-pip-versions: ${{ steps.run.outputs.own-pip-versions }}
|
||||||
|
steps:
|
||||||
|
# push and pull_request inputs aren't known to GitHub (pt3)
|
||||||
|
- name: Set push defaults
|
||||||
|
if: ${{ github.event_name == 'push' }}
|
||||||
|
env:
|
||||||
|
cpython-versions: all
|
||||||
|
test-set: core
|
||||||
|
run: |
|
||||||
|
echo "cpython-versions=${{env.cpython-versions}}" >> "$GITHUB_ENV"
|
||||||
|
echo "test_set=${{env.test_set}}" >> "$GITHUB_ENV"
|
||||||
|
- name: Get pull_request inputs
|
||||||
|
if: ${{ github.event_name == 'pull_request' }}
|
||||||
|
env:
|
||||||
|
cpython-versions: main
|
||||||
|
test-set: both
|
||||||
|
run: |
|
||||||
|
echo "cpython-versions=${{env.cpython-versions}}" >> "$GITHUB_ENV"
|
||||||
|
echo "test_set=${{env.test_set}}" >> "$GITHUB_ENV"
|
||||||
|
- name: Make version array
|
||||||
|
id: run
|
||||||
|
run: |
|
||||||
|
# Make a JSON Array from comma/space-separated string (no extra escaping)
|
||||||
|
json_list() { \
|
||||||
|
ret=""; IFS="${IFS},"; set -- $*; \
|
||||||
|
for a in "$@"; do \
|
||||||
|
ret=$(printf '%s"%s"' "${ret}${ret:+, }" "$a"); \
|
||||||
|
done; \
|
||||||
|
printf '[%s]' "$ret"; }
|
||||||
|
tests="${{ inputs.test-set || env.test-set }}"
|
||||||
|
[ $tests = both ] && tests="core download"
|
||||||
|
printf 'test-set=%s\n' "$(json_list $tests)" >> "$GITHUB_OUTPUT"
|
||||||
|
versions="${{ inputs.cpython-versions || env.cpython-versions }}"
|
||||||
|
if [ "$versions" = all ]; then \
|
||||||
|
versions="${{ env.all-cpython-versions }}"; else \
|
||||||
|
versions="${{ env.main-cpython-versions }}"; \
|
||||||
|
fi
|
||||||
|
printf 'cpython-versions=%s\n' \
|
||||||
|
"$(json_list ${versions}${versions:+, }${{ env.pypy-versions }})" >> "$GITHUB_OUTPUT"
|
||||||
|
# versions with a special get-pip.py in a per-version subdirectory
|
||||||
|
printf 'own-pip-versions=%s\n' \
|
||||||
|
"$(json_list 2.6, 2.7, 3.2, 3.3, 3.4, 3.5, 3.6)" >> "$GITHUB_OUTPUT"
|
||||||
|
|
||||||
|
tests:
|
||||||
|
name: Run tests
|
||||||
|
needs: select
|
||||||
|
permissions:
|
||||||
|
contents: read
|
||||||
|
packages: write
|
||||||
|
runs-on: ${{ matrix.os }}
|
||||||
|
env:
|
||||||
|
PIP: python -m pip
|
||||||
|
PIP_DISABLE_PIP_VERSION_CHECK: true
|
||||||
|
PIP_NO_PYTHON_VERSION_WARNING: true
|
||||||
|
strategy:
|
||||||
|
fail-fast: true
|
||||||
|
matrix:
|
||||||
|
os: [ubuntu-20.04]
|
||||||
|
python-version: ${{ fromJSON(needs.select.outputs.cpython-versions) }}
|
||||||
|
python-impl: [cpython]
|
||||||
|
ytdl-test-set: ${{ fromJSON(needs.select.outputs.test-set) }}
|
||||||
|
run-tests-ext: [sh]
|
||||||
|
include:
|
||||||
|
- os: windows-2019
|
||||||
|
python-version: 3.4
|
||||||
|
python-impl: cpython
|
||||||
|
ytdl-test-set: ${{ contains(needs.select.outputs.test-set, 'core') && 'core' || 'nocore' }}
|
||||||
|
run-tests-ext: bat
|
||||||
|
- os: windows-2019
|
||||||
|
python-version: 3.4
|
||||||
|
python-impl: cpython
|
||||||
|
ytdl-test-set: ${{ contains(needs.select.outputs.test-set, 'download') && 'download' || 'nodownload' }}
|
||||||
|
run-tests-ext: bat
|
||||||
|
# jython
|
||||||
|
- os: ubuntu-20.04
|
||||||
|
python-version: 2.7
|
||||||
|
python-impl: jython
|
||||||
|
ytdl-test-set: ${{ contains(needs.select.outputs.test-set, 'core') && 'core' || 'nocore' }}
|
||||||
|
run-tests-ext: sh
|
||||||
|
- os: ubuntu-20.04
|
||||||
|
python-version: 2.7
|
||||||
|
python-impl: jython
|
||||||
|
ytdl-test-set: ${{ contains(needs.select.outputs.test-set, 'download') && 'download' || 'nodownload' }}
|
||||||
|
run-tests-ext: sh
|
||||||
|
steps:
|
||||||
|
- name: Prepare Linux
|
||||||
|
if: ${{ startswith(matrix.os, 'ubuntu') }}
|
||||||
|
shell: bash
|
||||||
|
run: |
|
||||||
|
# apt in runner, if needed, may not be up-to-date
|
||||||
|
sudo apt-get update
|
||||||
|
- name: Checkout
|
||||||
|
uses: actions/checkout@v3
|
||||||
|
#-------- Python 3 -----
|
||||||
|
- name: Set up supported Python ${{ matrix.python-version }}
|
||||||
|
id: setup-python
|
||||||
|
if: ${{ matrix.python-impl == 'cpython' && matrix.python-version != '2.6' && matrix.python-version != '2.7' && matrix.python-version != env.next }}
|
||||||
|
# wrap broken actions/setup-python@v4
|
||||||
|
# NB may run apt-get install in Linux
|
||||||
|
uses: ytdl-org/setup-python@v1
|
||||||
|
env:
|
||||||
|
# Temporary workaround for Python 3.5 failures - May 2024
|
||||||
|
PIP_TRUSTED_HOST: "pypi.python.org pypi.org files.pythonhosted.org"
|
||||||
|
with:
|
||||||
|
python-version: ${{ matrix.python-version }}
|
||||||
|
cache-build: true
|
||||||
|
allow-build: info
|
||||||
|
- name: Locate supported Python ${{ matrix.python-version }}
|
||||||
|
if: ${{ env.pythonLocation }}
|
||||||
|
shell: bash
|
||||||
|
run: |
|
||||||
|
echo "PYTHONHOME=${pythonLocation}" >> "$GITHUB_ENV"
|
||||||
|
export expected="${{ steps.setup-python.outputs.python-path }}"
|
||||||
|
dirname() { printf '%s\n' \
|
||||||
|
'import os, sys' \
|
||||||
|
'print(os.path.dirname(sys.argv[1]))' \
|
||||||
|
| ${expected} - "$1"; }
|
||||||
|
expd="$(dirname "$expected")"
|
||||||
|
export python="$(command -v python)"
|
||||||
|
[ "$expd" = "$(dirname "$python")" ] || echo "PATH=$expd:${PATH}" >> "$GITHUB_ENV"
|
||||||
|
[ -x "$python" ] || printf '%s\n' \
|
||||||
|
'import os' \
|
||||||
|
'exp = os.environ["expected"]' \
|
||||||
|
'python = os.environ["python"]' \
|
||||||
|
'exps = os.path.split(exp)' \
|
||||||
|
'if python and (os.path.dirname(python) == exp[0]):' \
|
||||||
|
' exit(0)' \
|
||||||
|
'exps[1] = "python" + os.path.splitext(exps[1])[1]' \
|
||||||
|
'python = os.path.join(*exps)' \
|
||||||
|
'try:' \
|
||||||
|
' os.symlink(exp, python)' \
|
||||||
|
'except AttributeError:' \
|
||||||
|
' os.rename(exp, python)' \
|
||||||
|
| ${expected} -
|
||||||
|
printf '%s\n' \
|
||||||
|
'import sys' \
|
||||||
|
'print(sys.path)' \
|
||||||
|
| ${expected} -
|
||||||
|
#-------- Python next (was 3.12) -
|
||||||
|
- name: Set up CPython 3.next environment
|
||||||
|
if: ${{ matrix.python-impl == 'cpython' && matrix.python-version == env.next }}
|
||||||
|
shell: bash
|
||||||
|
run: |
|
||||||
|
PYENV_ROOT=$HOME/.local/share/pyenv
|
||||||
|
echo "PYENV_ROOT=${PYENV_ROOT}" >> "$GITHUB_ENV"
|
||||||
|
- name: Cache Python 3.next
|
||||||
|
id: cachenext
|
||||||
|
if: ${{ matrix.python-impl == 'cpython' && matrix.python-version == env.next }}
|
||||||
|
uses: actions/cache@v3
|
||||||
|
with:
|
||||||
|
key: python-${{ env.next }}
|
||||||
|
path: |
|
||||||
|
${{ env.PYENV_ROOT }}
|
||||||
|
- name: Build and set up Python 3.next
|
||||||
|
if: ${{ matrix.python-impl == 'cpython' && matrix.python-version == env.next && ! steps.cachenext.outputs.cache-hit }}
|
||||||
|
# dl and build locally
|
||||||
|
shell: bash
|
||||||
|
run: |
|
||||||
|
# Install build environment
|
||||||
|
sudo apt-get install -y build-essential llvm libssl-dev tk-dev \
|
||||||
|
libncursesw5-dev libreadline-dev libsqlite3-dev \
|
||||||
|
libffi-dev xz-utils zlib1g-dev libbz2-dev liblzma-dev
|
||||||
|
# Download PyEnv from its GitHub repository.
|
||||||
|
export PYENV_ROOT=${{ env.PYENV_ROOT }}
|
||||||
|
export PATH=$PYENV_ROOT/bin:$PATH
|
||||||
|
git clone "https://github.com/pyenv/pyenv.git" "$PYENV_ROOT"
|
||||||
|
pyenv install ${{ env.next }}
|
||||||
|
- name: Locate Python 3.next
|
||||||
|
if: ${{ matrix.python-impl == 'cpython' && matrix.python-version == env.next }}
|
||||||
|
shell: bash
|
||||||
|
run: |
|
||||||
|
PYTHONHOME="$(echo "${{ env.PYENV_ROOT }}/versions/${{ env.next }}."*)"
|
||||||
|
test -n "$PYTHONHOME"
|
||||||
|
echo "PYTHONHOME=$PYTHONHOME" >> "$GITHUB_ENV"
|
||||||
|
echo "PATH=${PYTHONHOME}/bin:$PATH" >> "$GITHUB_ENV"
|
||||||
|
#-------- Python 2.7 --
|
||||||
|
- name: Set up Python 2.7
|
||||||
|
if: ${{ matrix.python-impl == 'cpython' && matrix.python-version == '2.7' }}
|
||||||
|
# install 2.7
|
||||||
|
shell: bash
|
||||||
|
run: |
|
||||||
|
sudo apt-get install -y python2 python-is-python2
|
||||||
|
echo "PYTHONHOME=/usr" >> "$GITHUB_ENV"
|
||||||
|
#-------- Python 2.6 --
|
||||||
|
- name: Set up Python 2.6 environment
|
||||||
|
if: ${{ matrix.python-impl == 'cpython' && matrix.python-version == '2.6' }}
|
||||||
|
shell: bash
|
||||||
|
run: |
|
||||||
|
openssl_name=openssl-1.0.2u
|
||||||
|
echo "openssl_name=${openssl_name}" >> "$GITHUB_ENV"
|
||||||
|
openssl_dir=$HOME/.local/opt/$openssl_name
|
||||||
|
echo "openssl_dir=${openssl_dir}" >> "$GITHUB_ENV"
|
||||||
|
PYENV_ROOT=$HOME/.local/share/pyenv
|
||||||
|
echo "PYENV_ROOT=${PYENV_ROOT}" >> "$GITHUB_ENV"
|
||||||
|
sudo apt-get install -y openssl ca-certificates
|
||||||
|
- name: Cache Python 2.6
|
||||||
|
id: cache26
|
||||||
|
if: ${{ matrix.python-version == '2.6' }}
|
||||||
|
uses: actions/cache@v3
|
||||||
|
with:
|
||||||
|
key: python-2.6.9
|
||||||
|
path: |
|
||||||
|
${{ env.openssl_dir }}
|
||||||
|
${{ env.PYENV_ROOT }}
|
||||||
|
- name: Build and set up Python 2.6
|
||||||
|
if: ${{ matrix.python-impl == 'cpython' && matrix.python-version == '2.6' && ! steps.cache26.outputs.cache-hit }}
|
||||||
|
# dl and build locally
|
||||||
|
shell: bash
|
||||||
|
run: |
|
||||||
|
# Install build environment
|
||||||
|
sudo apt-get install -y build-essential llvm libssl-dev tk-dev \
|
||||||
|
libncursesw5-dev libreadline-dev libsqlite3-dev \
|
||||||
|
libffi-dev xz-utils zlib1g-dev libbz2-dev liblzma-dev
|
||||||
|
# Download and install OpenSSL 1.0.2, back in time
|
||||||
|
openssl_name=${{ env.openssl_name }}
|
||||||
|
openssl_targz=${openssl_name}.tar.gz
|
||||||
|
openssl_dir=${{ env.openssl_dir }}
|
||||||
|
openssl_inc=$openssl_dir/include
|
||||||
|
openssl_lib=$openssl_dir/lib
|
||||||
|
openssl_ssl=$openssl_dir/ssl
|
||||||
|
curl -L "https://www.openssl.org/source/$openssl_targz" -o $openssl_targz
|
||||||
|
tar -xf $openssl_targz
|
||||||
|
( cd $openssl_name; \
|
||||||
|
./config --prefix=$openssl_dir --openssldir=${openssl_dir}/ssl \
|
||||||
|
--libdir=lib -Wl,-rpath=${openssl_dir}/lib shared zlib-dynamic && \
|
||||||
|
make && \
|
||||||
|
make install )
|
||||||
|
rm -rf $openssl_name
|
||||||
|
rmdir $openssl_ssl/certs && ln -s /etc/ssl/certs $openssl_ssl/certs
|
||||||
|
# Download PyEnv from its GitHub repository.
|
||||||
|
export PYENV_ROOT=${{ env.PYENV_ROOT }}
|
||||||
|
export PATH=$PYENV_ROOT/bin:$PATH
|
||||||
|
git clone "https://github.com/pyenv/pyenv.git" "$PYENV_ROOT"
|
||||||
|
# Prevent pyenv build trying (and failing) to update pip
|
||||||
|
export GET_PIP=get-pip-2.6.py
|
||||||
|
echo 'import sys; sys.exit(0)' > ${GET_PIP}
|
||||||
|
GET_PIP=$(realpath $GET_PIP)
|
||||||
|
# Build and install Python
|
||||||
|
export CFLAGS="-I$openssl_inc"
|
||||||
|
export LDFLAGS="-L$openssl_lib"
|
||||||
|
export LD_LIBRARY_PATH="$openssl_lib"
|
||||||
|
pyenv install 2.6.9
|
||||||
|
- name: Locate Python 2.6
|
||||||
|
if: ${{ matrix.python-impl == 'cpython' && matrix.python-version == '2.6' }}
|
||||||
|
shell: bash
|
||||||
|
run: |
|
||||||
|
PYTHONHOME="${{ env.PYENV_ROOT }}/versions/2.6.9"
|
||||||
|
echo "PYTHONHOME=$PYTHONHOME" >> "$GITHUB_ENV"
|
||||||
|
echo "PATH=${PYTHONHOME}/bin:$PATH" >> "$GITHUB_ENV"
|
||||||
|
echo "LD_LIBRARY_PATH=${{ env.openssl_dir }}/lib${LD_LIBRARY_PATH:+:}${LD_LIBRARY_PATH}" >> "$GITHUB_ENV"
|
||||||
|
#-------- Jython ------
|
||||||
|
- name: Set up Java 8
|
||||||
|
if: ${{ matrix.python-impl == 'jython' }}
|
||||||
|
uses: actions/setup-java@v3
|
||||||
|
with:
|
||||||
|
java-version: 8
|
||||||
|
distribution: 'zulu'
|
||||||
|
- name: Setup Jython environment
|
||||||
|
if: ${{ matrix.python-impl == 'jython' }}
|
||||||
|
shell: bash
|
||||||
|
run: |
|
||||||
|
echo "JYTHON_ROOT=${HOME}/jython" >> "$GITHUB_ENV"
|
||||||
|
echo "PIP=pip" >> "$GITHUB_ENV"
|
||||||
|
- name: Cache Jython
|
||||||
|
id: cachejy
|
||||||
|
if: ${{ matrix.python-impl == 'jython' && matrix.python-version == '2.7' }}
|
||||||
|
uses: actions/cache@v3
|
||||||
|
with:
|
||||||
|
# 2.7.3 now available, may solve SNI issue
|
||||||
|
key: jython-2.7.1
|
||||||
|
path: |
|
||||||
|
${{ env.JYTHON_ROOT }}
|
||||||
|
- name: Install Jython
|
||||||
|
if: ${{ matrix.python-impl == 'jython' && matrix.python-version == '2.7' && ! steps.cachejy.outputs.cache-hit }}
|
||||||
|
shell: bash
|
||||||
|
run: |
|
||||||
|
JYTHON_ROOT="${{ env.JYTHON_ROOT }}"
|
||||||
|
curl -L "https://repo1.maven.org/maven2/org/python/jython-installer/2.7.1/jython-installer-2.7.1.jar" -o jython-installer.jar
|
||||||
|
java -jar jython-installer.jar -s -d "${JYTHON_ROOT}"
|
||||||
|
echo "${JYTHON_ROOT}/bin" >> "$GITHUB_PATH"
|
||||||
|
- name: Set up cached Jython
|
||||||
|
if: ${{ steps.cachejy.outputs.cache-hit }}
|
||||||
|
shell: bash
|
||||||
|
run: |
|
||||||
|
JYTHON_ROOT="${{ env.JYTHON_ROOT }}"
|
||||||
|
echo "${JYTHON_ROOT}/bin" >> $GITHUB_PATH
|
||||||
|
- name: Install supporting Python 2.7 if possible
|
||||||
|
if: ${{ steps.cachejy.outputs.cache-hit }}
|
||||||
|
shell: bash
|
||||||
|
run: |
|
||||||
|
sudo apt-get install -y python2.7 || true
|
||||||
|
#-------- pip ---------
|
||||||
|
- name: Set up supported Python ${{ matrix.python-version }} pip
|
||||||
|
if: ${{ (matrix.python-version != '3.2' && steps.setup-python.outputs.python-path) || matrix.python-version == '2.7' }}
|
||||||
|
# This step may run in either Linux or Windows
|
||||||
|
shell: bash
|
||||||
|
run: |
|
||||||
|
echo "$PATH"
|
||||||
|
echo "$PYTHONHOME"
|
||||||
|
# curl is available on both Windows and Linux, -L follows redirects, -O gets name
|
||||||
|
python -m ensurepip || python -m pip --version || { \
|
||||||
|
get_pip="${{ contains(needs.select.outputs.own-pip-versions, matrix.python-version) && format('{0}/', matrix.python-version) || '' }}"; \
|
||||||
|
curl -L -O "https://bootstrap.pypa.io/pip/${get_pip}get-pip.py"; \
|
||||||
|
python get-pip.py; }
|
||||||
|
- name: Set up Python 2.6 pip
|
||||||
|
if: ${{ matrix.python-version == '2.6' }}
|
||||||
|
shell: bash
|
||||||
|
run: |
|
||||||
|
python -m pip --version || { \
|
||||||
|
curl -L -O "https://bootstrap.pypa.io/pip/2.6/get-pip.py"; \
|
||||||
|
curl -L -O "https://files.pythonhosted.org/packages/ac/95/a05b56bb975efa78d3557efa36acaf9cf5d2fd0ee0062060493687432e03/pip-9.0.3-py2.py3-none-any.whl"; \
|
||||||
|
python get-pip.py --no-setuptools --no-wheel pip-9.0.3-py2.py3-none-any.whl; }
|
||||||
|
# work-around to invoke pip module on 2.6: https://bugs.python.org/issue2751
|
||||||
|
echo "PIP=python -m pip.__main__" >> "$GITHUB_ENV"
|
||||||
|
- name: Set up other Python ${{ matrix.python-version }} pip
|
||||||
|
if: ${{ matrix.python-version == '3.2' && steps.setup-python.outputs.python-path }}
|
||||||
|
shell: bash
|
||||||
|
run: |
|
||||||
|
python -m pip --version || { \
|
||||||
|
curl -L -O "https://bootstrap.pypa.io/pip/3.2/get-pip.py"; \
|
||||||
|
curl -L -O "https://files.pythonhosted.org/packages/b2/d0/cd115fe345dd6f07ec1c780020a7dfe74966fceeb171e0f20d1d4905b0b7/pip-7.1.2-py2.py3-none-any.whl"; \
|
||||||
|
python get-pip.py --no-setuptools --no-wheel pip-7.1.2-py2.py3-none-any.whl; }
|
||||||
|
#-------- unittest ----
|
||||||
|
- name: Upgrade Unittest for Python 2.6
|
||||||
|
if: ${{ matrix.python-version == '2.6' }}
|
||||||
|
shell: bash
|
||||||
|
run: |
|
||||||
|
# Work around deprecation of support for non-SNI clients at PyPI CDN (see https://status.python.org/incidents/hzmjhqsdjqgb)
|
||||||
|
$PIP -qq show unittest2 || { \
|
||||||
|
for u in "65/26/32b8464df2a97e6dd1b656ed26b2c194606c16fe163c695a992b36c11cdf/six-1.13.0-py2.py3-none-any.whl" \
|
||||||
|
"f2/94/3af39d34be01a24a6e65433d19e107099374224905f1e0cc6bbe1fd22a2f/argparse-1.4.0-py2.py3-none-any.whl" \
|
||||||
|
"c7/a3/c5da2a44c85bfbb6eebcfc1dde24933f8704441b98fdde6528f4831757a6/linecache2-1.0.0-py2.py3-none-any.whl" \
|
||||||
|
"17/0a/6ac05a3723017a967193456a2efa0aa9ac4b51456891af1e2353bb9de21e/traceback2-1.4.0-py2.py3-none-any.whl" \
|
||||||
|
"72/20/7f0f433060a962200b7272b8c12ba90ef5b903e218174301d0abfd523813/unittest2-1.1.0-py2.py3-none-any.whl"; do \
|
||||||
|
curl -L -O "https://files.pythonhosted.org/packages/${u}"; \
|
||||||
|
$PIP install ${u##*/}; \
|
||||||
|
done; }
|
||||||
|
# make tests use unittest2
|
||||||
|
for test in ./test/test_*.py ./test/helper.py; do
|
||||||
|
sed -r -i -e '/^import unittest$/s/test/test2 as unittest/' "$test"
|
||||||
|
done
|
||||||
|
#-------- nose --------
|
||||||
|
- name: Install nose for Python ${{ matrix.python-version }}
|
||||||
|
if: ${{ (matrix.python-version != '3.2' && steps.setup-python.outputs.python-path) || (matrix.python-impl == 'cpython' && (matrix.python-version == '2.7' || matrix.python-version == env.next)) }}
|
||||||
|
shell: bash
|
||||||
|
run: |
|
||||||
|
echo "$PATH"
|
||||||
|
echo "$PYTHONHOME"
|
||||||
|
# Use PyNose for recent Pythons instead of Nose
|
||||||
|
py3ver="${{ matrix.python-version }}"
|
||||||
|
py3ver=${py3ver#3.}
|
||||||
|
[ "$py3ver" != "${{ matrix.python-version }}" ] && py3ver=${py3ver%.*} || py3ver=0
|
||||||
|
[ "$py3ver" -ge 9 ] && nose=pynose || nose=nose
|
||||||
|
$PIP -qq show $nose || $PIP install $nose
|
||||||
|
- name: Install nose for other Python 2
|
||||||
|
if: ${{ matrix.python-impl == 'jython' || (matrix.python-impl == 'cpython' && matrix.python-version == '2.6') }}
|
||||||
|
shell: bash
|
||||||
|
run: |
|
||||||
|
# Work around deprecation of support for non-SNI clients at PyPI CDN (see https://status.python.org/incidents/hzmjhqsdjqgb)
|
||||||
|
$PIP -qq show nose || { \
|
||||||
|
curl -L -O "https://files.pythonhosted.org/packages/99/4f/13fb671119e65c4dce97c60e67d3fd9e6f7f809f2b307e2611f4701205cb/nose-1.3.7-py2-none-any.whl"; \
|
||||||
|
$PIP install nose-1.3.7-py2-none-any.whl; }
|
||||||
|
- name: Install nose for other Python 3
|
||||||
|
if: ${{ matrix.python-version == '3.2' && steps.setup-python.outputs.python-path }}
|
||||||
|
shell: bash
|
||||||
|
run: |
|
||||||
|
$PIP -qq show nose || { \
|
||||||
|
curl -L -O "https://files.pythonhosted.org/packages/15/d8/dd071918c040f50fa1cf80da16423af51ff8ce4a0f2399b7bf8de45ac3d9/nose-1.3.7-py3-none-any.whl"; \
|
||||||
|
$PIP install nose-1.3.7-py3-none-any.whl; }
|
||||||
|
- name: Set up nosetest test
|
||||||
|
if: ${{ contains(needs.select.outputs.test-set, matrix.ytdl-test-set ) }}
|
||||||
|
shell: bash
|
||||||
|
run: |
|
||||||
|
# set PYTHON_VER
|
||||||
|
PYTHON_VER=${{ matrix.python-version }}
|
||||||
|
[ "${PYTHON_VER#*-}" != "$PYTHON_VER" ] || PYTHON_VER="${{ matrix.python-impl }}-${PYTHON_VER}"
|
||||||
|
echo "PYTHON_VER=$PYTHON_VER" >> "$GITHUB_ENV"
|
||||||
|
echo "PYTHON_IMPL=${{ matrix.python-impl }}" >> "$GITHUB_ENV"
|
||||||
|
# define a test to validate the Python version used by nosetests
|
||||||
|
printf '%s\n' \
|
||||||
|
'from __future__ import unicode_literals' \
|
||||||
|
'import sys, os, platform' \
|
||||||
|
'try:' \
|
||||||
|
' import unittest2 as unittest' \
|
||||||
|
'except ImportError:' \
|
||||||
|
' import unittest' \
|
||||||
|
'class TestPython(unittest.TestCase):' \
|
||||||
|
' def setUp(self):' \
|
||||||
|
' self.ver = os.environ["PYTHON_VER"].split("-")' \
|
||||||
|
' def test_python_ver(self):' \
|
||||||
|
' self.assertEqual(["%d" % v for v in sys.version_info[:2]], self.ver[-1].split(".")[:2])' \
|
||||||
|
' self.assertTrue(sys.version.startswith(self.ver[-1]))' \
|
||||||
|
' self.assertIn(self.ver[0], ",".join((sys.version, platform.python_implementation())).lower())' \
|
||||||
|
' def test_python_impl(self):' \
|
||||||
|
' self.assertIn(platform.python_implementation().lower(), (os.environ["PYTHON_IMPL"], self.ver[0]))' \
|
||||||
|
> test/test_python.py
|
||||||
|
#-------- TESTS -------
|
||||||
|
- name: Run tests
|
||||||
|
if: ${{ contains(needs.select.outputs.test-set, matrix.ytdl-test-set ) }}
|
||||||
|
continue-on-error: ${{ matrix.ytdl-test-set == 'download' || matrix.python-impl == 'jython' }}
|
||||||
|
env:
|
||||||
|
YTDL_TEST_SET: ${{ matrix.ytdl-test-set }}
|
||||||
|
run: |
|
||||||
|
./devscripts/run_tests.${{ matrix.run-tests-ext }}
|
||||||
|
flake8:
|
||||||
|
name: Linter
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
steps:
|
||||||
|
- uses: actions/checkout@v3
|
||||||
|
- name: Set up Python
|
||||||
|
uses: actions/setup-python@v4
|
||||||
|
with:
|
||||||
|
python-version: 3.9
|
||||||
|
- name: Install flake8
|
||||||
|
run: pip install flake8
|
||||||
|
- name: Run flake8
|
||||||
|
run: flake8 .
|
||||||
|
|
50
.travis.yml
50
.travis.yml
|
@ -1,50 +0,0 @@
|
||||||
language: python
|
|
||||||
python:
|
|
||||||
- "2.6"
|
|
||||||
- "2.7"
|
|
||||||
- "3.2"
|
|
||||||
- "3.3"
|
|
||||||
- "3.4"
|
|
||||||
- "3.5"
|
|
||||||
- "3.6"
|
|
||||||
- "pypy"
|
|
||||||
- "pypy3"
|
|
||||||
dist: trusty
|
|
||||||
env:
|
|
||||||
- YTDL_TEST_SET=core
|
|
||||||
- YTDL_TEST_SET=download
|
|
||||||
jobs:
|
|
||||||
include:
|
|
||||||
- python: 3.7
|
|
||||||
dist: xenial
|
|
||||||
env: YTDL_TEST_SET=core
|
|
||||||
- python: 3.7
|
|
||||||
dist: xenial
|
|
||||||
env: YTDL_TEST_SET=download
|
|
||||||
- python: 3.8
|
|
||||||
dist: xenial
|
|
||||||
env: YTDL_TEST_SET=core
|
|
||||||
- python: 3.8
|
|
||||||
dist: xenial
|
|
||||||
env: YTDL_TEST_SET=download
|
|
||||||
- python: 3.8-dev
|
|
||||||
dist: xenial
|
|
||||||
env: YTDL_TEST_SET=core
|
|
||||||
- python: 3.8-dev
|
|
||||||
dist: xenial
|
|
||||||
env: YTDL_TEST_SET=download
|
|
||||||
- env: JYTHON=true; YTDL_TEST_SET=core
|
|
||||||
- env: JYTHON=true; YTDL_TEST_SET=download
|
|
||||||
- name: flake8
|
|
||||||
python: 3.8
|
|
||||||
dist: xenial
|
|
||||||
install: pip install flake8
|
|
||||||
script: flake8 .
|
|
||||||
fast_finish: true
|
|
||||||
allow_failures:
|
|
||||||
- env: YTDL_TEST_SET=download
|
|
||||||
- env: JYTHON=true; YTDL_TEST_SET=core
|
|
||||||
- env: JYTHON=true; YTDL_TEST_SET=download
|
|
||||||
before_install:
|
|
||||||
- if [ "$JYTHON" == "true" ]; then ./devscripts/install_jython.sh; export PATH="$HOME/jython/bin:$PATH"; fi
|
|
||||||
script: ./devscripts/run_tests.sh
|
|
1
AUTHORS
1
AUTHORS
|
@ -246,3 +246,4 @@ Enes Solak
|
||||||
Nathan Rossi
|
Nathan Rossi
|
||||||
Thomas van der Berg
|
Thomas van der Berg
|
||||||
Luca Cherubin
|
Luca Cherubin
|
||||||
|
Adrian Heine
|
|
@ -150,10 +150,10 @@ After you have ensured this site is distributing its content legally, you can fo
|
||||||
# TODO more properties (see youtube_dl/extractor/common.py)
|
# TODO more properties (see youtube_dl/extractor/common.py)
|
||||||
}
|
}
|
||||||
```
|
```
|
||||||
5. Add an import in [`youtube_dl/extractor/extractors.py`](https://github.com/ytdl-org/youtube-dl/blob/master/youtube_dl/extractor/extractors.py).
|
5. Add an import in [`youtube_dl/extractor/extractors.py`](https://github.com/ytdl-org/youtube-dl/blob/master/youtube_dl/extractor/extractors.py). This makes the extractor available for use, as long as the class ends with `IE`.
|
||||||
6. Run `python test/test_download.py TestDownload.test_YourExtractor`. This *should fail* at first, but you can continually re-run it until you're done. If you decide to add more than one test, then rename ``_TEST`` to ``_TESTS`` and make it into a list of dictionaries. The tests will then be named `TestDownload.test_YourExtractor`, `TestDownload.test_YourExtractor_1`, `TestDownload.test_YourExtractor_2`, etc. Note that tests with `only_matching` key in test's dict are not counted in.
|
6. Run `python test/test_download.py TestDownload.test_YourExtractor`. This *should fail* at first, but you can continually re-run it until you're done. If you decide to add more than one test, then rename ``_TEST`` to ``_TESTS`` and make it into a list of dictionaries. The tests will then be named `TestDownload.test_YourExtractor`, `TestDownload.test_YourExtractor_1`, `TestDownload.test_YourExtractor_2`, etc. Note that tests with `only_matching` key in test's dict are not counted in.
|
||||||
7. Have a look at [`youtube_dl/extractor/common.py`](https://github.com/ytdl-org/youtube-dl/blob/master/youtube_dl/extractor/common.py) for possible helper methods and a [detailed description of what your extractor should and may return](https://github.com/ytdl-org/youtube-dl/blob/7f41a598b3fba1bcab2817de64a08941200aa3c8/youtube_dl/extractor/common.py#L94-L303). Add tests and code for as many as you want.
|
7. Have a look at [`youtube_dl/extractor/common.py`](https://github.com/ytdl-org/youtube-dl/blob/master/youtube_dl/extractor/common.py) for possible helper methods and a [detailed description of what your extractor should and may return](https://github.com/ytdl-org/youtube-dl/blob/7f41a598b3fba1bcab2817de64a08941200aa3c8/youtube_dl/extractor/common.py#L94-L303). Add tests and code for as many as you want.
|
||||||
8. Make sure your code follows [youtube-dl coding conventions](#youtube-dl-coding-conventions) and check the code with [flake8](http://flake8.pycqa.org/en/latest/index.html#quickstart):
|
8. Make sure your code follows [youtube-dl coding conventions](#youtube-dl-coding-conventions) and check the code with [flake8](https://flake8.pycqa.org/en/latest/index.html#quickstart):
|
||||||
|
|
||||||
$ flake8 youtube_dl/extractor/yourextractor.py
|
$ flake8 youtube_dl/extractor/yourextractor.py
|
||||||
|
|
||||||
|
|
1
devscripts/__init__.py
Normal file
1
devscripts/__init__.py
Normal file
|
@ -0,0 +1 @@
|
||||||
|
# Empty file needed to make devscripts.utils properly importable from outside
|
|
@ -5,8 +5,12 @@ import os
|
||||||
from os.path import dirname as dirn
|
from os.path import dirname as dirn
|
||||||
import sys
|
import sys
|
||||||
|
|
||||||
sys.path.insert(0, dirn(dirn((os.path.abspath(__file__)))))
|
sys.path.insert(0, dirn(dirn(os.path.abspath(__file__))))
|
||||||
|
|
||||||
import youtube_dl
|
import youtube_dl
|
||||||
|
from youtube_dl.compat import compat_open as open
|
||||||
|
|
||||||
|
from utils import read_file
|
||||||
|
|
||||||
BASH_COMPLETION_FILE = "youtube-dl.bash-completion"
|
BASH_COMPLETION_FILE = "youtube-dl.bash-completion"
|
||||||
BASH_COMPLETION_TEMPLATE = "devscripts/bash-completion.in"
|
BASH_COMPLETION_TEMPLATE = "devscripts/bash-completion.in"
|
||||||
|
@ -18,9 +22,8 @@ def build_completion(opt_parser):
|
||||||
for option in group.option_list:
|
for option in group.option_list:
|
||||||
# for every long flag
|
# for every long flag
|
||||||
opts_flag.append(option.get_opt_string())
|
opts_flag.append(option.get_opt_string())
|
||||||
with open(BASH_COMPLETION_TEMPLATE) as f:
|
template = read_file(BASH_COMPLETION_TEMPLATE)
|
||||||
template = f.read()
|
with open(BASH_COMPLETION_FILE, "w", encoding='utf-8') as f:
|
||||||
with open(BASH_COMPLETION_FILE, "w") as f:
|
|
||||||
# just using the special char
|
# just using the special char
|
||||||
filled_template = template.replace("{{flags}}", " ".join(opts_flag))
|
filled_template = template.replace("{{flags}}", " ".join(opts_flag))
|
||||||
f.write(filled_template)
|
f.write(filled_template)
|
||||||
|
|
83
devscripts/cli_to_api.py
Executable file
83
devscripts/cli_to_api.py
Executable file
|
@ -0,0 +1,83 @@
|
||||||
|
#!/usr/bin/env python
|
||||||
|
# coding: utf-8
|
||||||
|
|
||||||
|
from __future__ import unicode_literals
|
||||||
|
|
||||||
|
"""
|
||||||
|
This script displays the API parameters corresponding to a yt-dl command line
|
||||||
|
|
||||||
|
Example:
|
||||||
|
$ ./cli_to_api.py -f best
|
||||||
|
{u'format': 'best'}
|
||||||
|
$
|
||||||
|
"""
|
||||||
|
|
||||||
|
# Allow direct execution
|
||||||
|
import os
|
||||||
|
import sys
|
||||||
|
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||||
|
|
||||||
|
import youtube_dl
|
||||||
|
from types import MethodType
|
||||||
|
|
||||||
|
|
||||||
|
def cli_to_api(*opts):
|
||||||
|
YDL = youtube_dl.YoutubeDL
|
||||||
|
|
||||||
|
# to extract the parsed options, break out of YoutubeDL instantiation
|
||||||
|
|
||||||
|
# return options via this Exception
|
||||||
|
class ParseYTDLResult(Exception):
|
||||||
|
def __init__(self, result):
|
||||||
|
super(ParseYTDLResult, self).__init__('result')
|
||||||
|
self.opts = result
|
||||||
|
|
||||||
|
# replacement constructor that raises ParseYTDLResult
|
||||||
|
def ytdl_init(ydl, ydl_opts):
|
||||||
|
super(YDL, ydl).__init__(ydl_opts)
|
||||||
|
raise ParseYTDLResult(ydl_opts)
|
||||||
|
|
||||||
|
# patch in the constructor
|
||||||
|
YDL.__init__ = MethodType(ytdl_init, YDL)
|
||||||
|
|
||||||
|
# core parser
|
||||||
|
def parsed_options(argv):
|
||||||
|
try:
|
||||||
|
youtube_dl._real_main(list(argv))
|
||||||
|
except ParseYTDLResult as result:
|
||||||
|
return result.opts
|
||||||
|
|
||||||
|
# from https://github.com/yt-dlp/yt-dlp/issues/5859#issuecomment-1363938900
|
||||||
|
default = parsed_options([])
|
||||||
|
|
||||||
|
def neq_opt(a, b):
|
||||||
|
if a == b:
|
||||||
|
return False
|
||||||
|
if a is None and repr(type(object)).endswith(".utils.DateRange'>"):
|
||||||
|
return '0001-01-01 - 9999-12-31' != '{0}'.format(b)
|
||||||
|
return a != b
|
||||||
|
|
||||||
|
diff = dict((k, v) for k, v in parsed_options(opts).items() if neq_opt(default[k], v))
|
||||||
|
if 'postprocessors' in diff:
|
||||||
|
diff['postprocessors'] = [pp for pp in diff['postprocessors'] if pp not in default['postprocessors']]
|
||||||
|
return diff
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
from pprint import PrettyPrinter
|
||||||
|
|
||||||
|
pprint = PrettyPrinter()
|
||||||
|
super_format = pprint.format
|
||||||
|
|
||||||
|
def format(object, context, maxlevels, level):
|
||||||
|
if repr(type(object)).endswith(".utils.DateRange'>"):
|
||||||
|
return '{0}: {1}>'.format(repr(object)[:-2], object), True, False
|
||||||
|
return super_format(object, context, maxlevels, level)
|
||||||
|
|
||||||
|
pprint.format = format
|
||||||
|
|
||||||
|
pprint.pprint(cli_to_api(*sys.argv))
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
main()
|
|
@ -1,7 +1,6 @@
|
||||||
#!/usr/bin/env python
|
#!/usr/bin/env python
|
||||||
from __future__ import unicode_literals
|
from __future__ import unicode_literals
|
||||||
|
|
||||||
import io
|
|
||||||
import json
|
import json
|
||||||
import mimetypes
|
import mimetypes
|
||||||
import netrc
|
import netrc
|
||||||
|
@ -10,7 +9,9 @@ import os
|
||||||
import re
|
import re
|
||||||
import sys
|
import sys
|
||||||
|
|
||||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
dirn = os.path.dirname
|
||||||
|
|
||||||
|
sys.path.insert(0, dirn(dirn(os.path.abspath(__file__))))
|
||||||
|
|
||||||
from youtube_dl.compat import (
|
from youtube_dl.compat import (
|
||||||
compat_basestring,
|
compat_basestring,
|
||||||
|
@ -22,6 +23,7 @@ from youtube_dl.utils import (
|
||||||
make_HTTPS_handler,
|
make_HTTPS_handler,
|
||||||
sanitized_Request,
|
sanitized_Request,
|
||||||
)
|
)
|
||||||
|
from utils import read_file
|
||||||
|
|
||||||
|
|
||||||
class GitHubReleaser(object):
|
class GitHubReleaser(object):
|
||||||
|
@ -89,8 +91,7 @@ def main():
|
||||||
|
|
||||||
changelog_file, version, build_path = args
|
changelog_file, version, build_path = args
|
||||||
|
|
||||||
with io.open(changelog_file, encoding='utf-8') as inf:
|
changelog = read_file(changelog_file)
|
||||||
changelog = inf.read()
|
|
||||||
|
|
||||||
mobj = re.search(r'(?s)version %s\n{2}(.+?)\n{3}' % version, changelog)
|
mobj = re.search(r'(?s)version %s\n{2}(.+?)\n{3}' % version, changelog)
|
||||||
body = mobj.group(1) if mobj else ''
|
body = mobj.group(1) if mobj else ''
|
||||||
|
|
|
@ -6,10 +6,13 @@ import os
|
||||||
from os.path import dirname as dirn
|
from os.path import dirname as dirn
|
||||||
import sys
|
import sys
|
||||||
|
|
||||||
sys.path.insert(0, dirn(dirn((os.path.abspath(__file__)))))
|
sys.path.insert(0, dirn(dirn(os.path.abspath(__file__))))
|
||||||
|
|
||||||
import youtube_dl
|
import youtube_dl
|
||||||
from youtube_dl.utils import shell_quote
|
from youtube_dl.utils import shell_quote
|
||||||
|
|
||||||
|
from utils import read_file, write_file
|
||||||
|
|
||||||
FISH_COMPLETION_FILE = 'youtube-dl.fish'
|
FISH_COMPLETION_FILE = 'youtube-dl.fish'
|
||||||
FISH_COMPLETION_TEMPLATE = 'devscripts/fish-completion.in'
|
FISH_COMPLETION_TEMPLATE = 'devscripts/fish-completion.in'
|
||||||
|
|
||||||
|
@ -38,11 +41,9 @@ def build_completion(opt_parser):
|
||||||
complete_cmd.extend(EXTRA_ARGS.get(long_option, []))
|
complete_cmd.extend(EXTRA_ARGS.get(long_option, []))
|
||||||
commands.append(shell_quote(complete_cmd))
|
commands.append(shell_quote(complete_cmd))
|
||||||
|
|
||||||
with open(FISH_COMPLETION_TEMPLATE) as f:
|
template = read_file(FISH_COMPLETION_TEMPLATE)
|
||||||
template = f.read()
|
|
||||||
filled_template = template.replace('{{commands}}', '\n'.join(commands))
|
filled_template = template.replace('{{commands}}', '\n'.join(commands))
|
||||||
with open(FISH_COMPLETION_FILE, 'w') as f:
|
write_file(FISH_COMPLETION_FILE, filled_template)
|
||||||
f.write(filled_template)
|
|
||||||
|
|
||||||
|
|
||||||
parser = youtube_dl.parseOpts()[0]
|
parser = youtube_dl.parseOpts()[0]
|
||||||
|
|
|
@ -6,16 +6,21 @@ import sys
|
||||||
import hashlib
|
import hashlib
|
||||||
import os.path
|
import os.path
|
||||||
|
|
||||||
|
dirn = os.path.dirname
|
||||||
|
|
||||||
|
sys.path.insert(0, dirn(dirn(dirn(os.path.abspath(__file__)))))
|
||||||
|
|
||||||
|
from devscripts.utils import read_file, write_file
|
||||||
|
from youtube_dl.compat import compat_open as open
|
||||||
|
|
||||||
if len(sys.argv) <= 1:
|
if len(sys.argv) <= 1:
|
||||||
print('Specify the version number as parameter')
|
print('Specify the version number as parameter')
|
||||||
sys.exit()
|
sys.exit()
|
||||||
version = sys.argv[1]
|
version = sys.argv[1]
|
||||||
|
|
||||||
with open('update/LATEST_VERSION', 'w') as f:
|
write_file('update/LATEST_VERSION', version)
|
||||||
f.write(version)
|
|
||||||
|
|
||||||
versions_info = json.load(open('update/versions.json'))
|
versions_info = json.loads(read_file('update/versions.json'))
|
||||||
if 'signature' in versions_info:
|
if 'signature' in versions_info:
|
||||||
del versions_info['signature']
|
del versions_info['signature']
|
||||||
|
|
||||||
|
@ -39,5 +44,5 @@ for key, filename in filenames.items():
|
||||||
versions_info['versions'][version] = new_version
|
versions_info['versions'][version] = new_version
|
||||||
versions_info['latest'] = version
|
versions_info['latest'] = version
|
||||||
|
|
||||||
with open('update/versions.json', 'w') as jsonf:
|
with open('update/versions.json', 'w', encoding='utf-8') as jsonf:
|
||||||
json.dump(versions_info, jsonf, indent=4, sort_keys=True)
|
json.dumps(versions_info, jsonf, indent=4, sort_keys=True)
|
||||||
|
|
|
@ -2,14 +2,21 @@
|
||||||
from __future__ import unicode_literals
|
from __future__ import unicode_literals
|
||||||
|
|
||||||
import json
|
import json
|
||||||
|
import os.path
|
||||||
|
import sys
|
||||||
|
|
||||||
versions_info = json.load(open('update/versions.json'))
|
dirn = os.path.dirname
|
||||||
|
|
||||||
|
sys.path.insert(0, dirn(dirn((os.path.abspath(__file__)))))
|
||||||
|
|
||||||
|
from utils import read_file, write_file
|
||||||
|
|
||||||
|
versions_info = json.loads(read_file('update/versions.json'))
|
||||||
version = versions_info['latest']
|
version = versions_info['latest']
|
||||||
version_dict = versions_info['versions'][version]
|
version_dict = versions_info['versions'][version]
|
||||||
|
|
||||||
# Read template page
|
# Read template page
|
||||||
with open('download.html.in', 'r', encoding='utf-8') as tmplf:
|
template = read_file('download.html.in')
|
||||||
template = tmplf.read()
|
|
||||||
|
|
||||||
template = template.replace('@PROGRAM_VERSION@', version)
|
template = template.replace('@PROGRAM_VERSION@', version)
|
||||||
template = template.replace('@PROGRAM_URL@', version_dict['bin'][0])
|
template = template.replace('@PROGRAM_URL@', version_dict['bin'][0])
|
||||||
|
@ -18,5 +25,5 @@ template = template.replace('@EXE_URL@', version_dict['exe'][0])
|
||||||
template = template.replace('@EXE_SHA256SUM@', version_dict['exe'][1])
|
template = template.replace('@EXE_SHA256SUM@', version_dict['exe'][1])
|
||||||
template = template.replace('@TAR_URL@', version_dict['tar'][0])
|
template = template.replace('@TAR_URL@', version_dict['tar'][0])
|
||||||
template = template.replace('@TAR_SHA256SUM@', version_dict['tar'][1])
|
template = template.replace('@TAR_SHA256SUM@', version_dict['tar'][1])
|
||||||
with open('download.html', 'w', encoding='utf-8') as dlf:
|
|
||||||
dlf.write(template)
|
write_file('download.html', template)
|
||||||
|
|
|
@ -5,17 +5,22 @@ from __future__ import with_statement, unicode_literals
|
||||||
|
|
||||||
import datetime
|
import datetime
|
||||||
import glob
|
import glob
|
||||||
import io # For Python 2 compatibility
|
|
||||||
import os
|
import os
|
||||||
import re
|
import re
|
||||||
|
import sys
|
||||||
|
|
||||||
year = str(datetime.datetime.now().year)
|
dirn = os.path.dirname
|
||||||
|
|
||||||
|
sys.path.insert(0, dirn(dirn(dirn(os.path.abspath(__file__)))))
|
||||||
|
|
||||||
|
from devscripts.utils import read_file, write_file
|
||||||
|
from youtube_dl import compat_str
|
||||||
|
|
||||||
|
year = compat_str(datetime.datetime.now().year)
|
||||||
for fn in glob.glob('*.html*'):
|
for fn in glob.glob('*.html*'):
|
||||||
with io.open(fn, encoding='utf-8') as f:
|
content = read_file(fn)
|
||||||
content = f.read()
|
|
||||||
newc = re.sub(r'(?P<copyright>Copyright © 2011-)(?P<year>[0-9]{4})', 'Copyright © 2011-' + year, content)
|
newc = re.sub(r'(?P<copyright>Copyright © 2011-)(?P<year>[0-9]{4})', 'Copyright © 2011-' + year, content)
|
||||||
if content != newc:
|
if content != newc:
|
||||||
tmpFn = fn + '.part'
|
tmpFn = fn + '.part'
|
||||||
with io.open(tmpFn, 'wt', encoding='utf-8') as outf:
|
write_file(tmpFn, newc)
|
||||||
outf.write(newc)
|
|
||||||
os.rename(tmpFn, fn)
|
os.rename(tmpFn, fn)
|
||||||
|
|
|
@ -2,10 +2,16 @@
|
||||||
from __future__ import unicode_literals
|
from __future__ import unicode_literals
|
||||||
|
|
||||||
import datetime
|
import datetime
|
||||||
import io
|
|
||||||
import json
|
import json
|
||||||
|
import os.path
|
||||||
import textwrap
|
import textwrap
|
||||||
|
import sys
|
||||||
|
|
||||||
|
dirn = os.path.dirname
|
||||||
|
|
||||||
|
sys.path.insert(0, dirn(dirn(os.path.abspath(__file__))))
|
||||||
|
|
||||||
|
from utils import write_file
|
||||||
|
|
||||||
atom_template = textwrap.dedent("""\
|
atom_template = textwrap.dedent("""\
|
||||||
<?xml version="1.0" encoding="utf-8"?>
|
<?xml version="1.0" encoding="utf-8"?>
|
||||||
|
@ -72,5 +78,4 @@ for v in versions:
|
||||||
entries_str = textwrap.indent(''.join(entries), '\t')
|
entries_str = textwrap.indent(''.join(entries), '\t')
|
||||||
atom_template = atom_template.replace('@ENTRIES@', entries_str)
|
atom_template = atom_template.replace('@ENTRIES@', entries_str)
|
||||||
|
|
||||||
with io.open('update/releases.atom', 'w', encoding='utf-8') as atom_file:
|
write_file('update/releases.atom', atom_template)
|
||||||
atom_file.write(atom_template)
|
|
||||||
|
|
|
@ -5,15 +5,17 @@ import sys
|
||||||
import os
|
import os
|
||||||
import textwrap
|
import textwrap
|
||||||
|
|
||||||
|
dirn = os.path.dirname
|
||||||
|
|
||||||
# We must be able to import youtube_dl
|
# We must be able to import youtube_dl
|
||||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))
|
sys.path.insert(0, dirn(dirn(dirn(os.path.abspath(__file__)))))
|
||||||
|
|
||||||
import youtube_dl
|
import youtube_dl
|
||||||
|
from devscripts.utils import read_file, write_file
|
||||||
|
|
||||||
|
|
||||||
def main():
|
def main():
|
||||||
with open('supportedsites.html.in', 'r', encoding='utf-8') as tmplf:
|
template = read_file('supportedsites.html.in')
|
||||||
template = tmplf.read()
|
|
||||||
|
|
||||||
ie_htmls = []
|
ie_htmls = []
|
||||||
for ie in youtube_dl.list_extractors(age_limit=None):
|
for ie in youtube_dl.list_extractors(age_limit=None):
|
||||||
|
@ -29,8 +31,7 @@ def main():
|
||||||
|
|
||||||
template = template.replace('@SITES@', textwrap.indent('\n'.join(ie_htmls), '\t'))
|
template = template.replace('@SITES@', textwrap.indent('\n'.join(ie_htmls), '\t'))
|
||||||
|
|
||||||
with open('supportedsites.html', 'w', encoding='utf-8') as sitesf:
|
write_file('supportedsites.html', template)
|
||||||
sitesf.write(template)
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == '__main__':
|
if __name__ == '__main__':
|
||||||
|
|
|
@ -1,5 +0,0 @@
|
||||||
#!/bin/bash
|
|
||||||
|
|
||||||
wget http://central.maven.org/maven2/org/python/jython-installer/2.7.1/jython-installer-2.7.1.jar
|
|
||||||
java -jar jython-installer-2.7.1.jar -s -d "$HOME/jython"
|
|
||||||
$HOME/jython/bin/jython -m pip install nose
|
|
|
@ -1,10 +1,11 @@
|
||||||
#!/usr/bin/env python
|
#!/usr/bin/env python
|
||||||
from __future__ import unicode_literals
|
from __future__ import unicode_literals
|
||||||
|
|
||||||
import io
|
|
||||||
import optparse
|
import optparse
|
||||||
import re
|
import re
|
||||||
|
|
||||||
|
from utils import read_file, write_file
|
||||||
|
|
||||||
|
|
||||||
def main():
|
def main():
|
||||||
parser = optparse.OptionParser(usage='%prog INFILE OUTFILE')
|
parser = optparse.OptionParser(usage='%prog INFILE OUTFILE')
|
||||||
|
@ -14,8 +15,7 @@ def main():
|
||||||
|
|
||||||
infile, outfile = args
|
infile, outfile = args
|
||||||
|
|
||||||
with io.open(infile, encoding='utf-8') as inf:
|
readme = read_file(infile)
|
||||||
readme = inf.read()
|
|
||||||
|
|
||||||
bug_text = re.search(
|
bug_text = re.search(
|
||||||
r'(?s)#\s*BUGS\s*[^\n]*\s*(.*?)#\s*COPYRIGHT', readme).group(1)
|
r'(?s)#\s*BUGS\s*[^\n]*\s*(.*?)#\s*COPYRIGHT', readme).group(1)
|
||||||
|
@ -25,8 +25,7 @@ def main():
|
||||||
|
|
||||||
out = bug_text + dev_text
|
out = bug_text + dev_text
|
||||||
|
|
||||||
with io.open(outfile, 'w', encoding='utf-8') as outf:
|
write_file(outfile, out)
|
||||||
outf.write(out)
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == '__main__':
|
if __name__ == '__main__':
|
||||||
|
|
|
@ -1,8 +1,11 @@
|
||||||
#!/usr/bin/env python
|
#!/usr/bin/env python
|
||||||
from __future__ import unicode_literals
|
from __future__ import unicode_literals
|
||||||
|
|
||||||
import io
|
|
||||||
import optparse
|
import optparse
|
||||||
|
import os.path
|
||||||
|
import sys
|
||||||
|
|
||||||
|
from utils import read_file, read_version, write_file
|
||||||
|
|
||||||
|
|
||||||
def main():
|
def main():
|
||||||
|
@ -13,17 +16,11 @@ def main():
|
||||||
|
|
||||||
infile, outfile = args
|
infile, outfile = args
|
||||||
|
|
||||||
with io.open(infile, encoding='utf-8') as inf:
|
issue_template_tmpl = read_file(infile)
|
||||||
issue_template_tmpl = inf.read()
|
|
||||||
|
|
||||||
# Get the version from youtube_dl/version.py without importing the package
|
out = issue_template_tmpl % {'version': read_version()}
|
||||||
exec(compile(open('youtube_dl/version.py').read(),
|
|
||||||
'youtube_dl/version.py', 'exec'))
|
|
||||||
|
|
||||||
out = issue_template_tmpl % {'version': locals()['__version__']}
|
write_file(outfile, out)
|
||||||
|
|
||||||
with io.open(outfile, 'w', encoding='utf-8') as outf:
|
|
||||||
outf.write(out)
|
|
||||||
|
|
||||||
if __name__ == '__main__':
|
if __name__ == '__main__':
|
||||||
main()
|
main()
|
||||||
|
|
|
@ -1,28 +1,49 @@
|
||||||
from __future__ import unicode_literals, print_function
|
from __future__ import unicode_literals, print_function
|
||||||
|
|
||||||
from inspect import getsource
|
from inspect import getsource
|
||||||
import io
|
|
||||||
import os
|
import os
|
||||||
from os.path import dirname as dirn
|
from os.path import dirname as dirn
|
||||||
|
import re
|
||||||
import sys
|
import sys
|
||||||
|
|
||||||
print('WARNING: Lazy loading extractors is an experimental feature that may not always work', file=sys.stderr)
|
print('WARNING: Lazy loading extractors is an experimental feature that may not always work', file=sys.stderr)
|
||||||
|
|
||||||
sys.path.insert(0, dirn(dirn((os.path.abspath(__file__)))))
|
sys.path.insert(0, dirn(dirn(os.path.abspath(__file__))))
|
||||||
|
|
||||||
lazy_extractors_filename = sys.argv[1]
|
lazy_extractors_filename = sys.argv[1]
|
||||||
if os.path.exists(lazy_extractors_filename):
|
if os.path.exists(lazy_extractors_filename):
|
||||||
os.remove(lazy_extractors_filename)
|
os.remove(lazy_extractors_filename)
|
||||||
|
# Py2: may be confused by leftover lazy_extractors.pyc
|
||||||
|
if sys.version_info[0] < 3:
|
||||||
|
for c in ('c', 'o'):
|
||||||
|
try:
|
||||||
|
os.remove(lazy_extractors_filename + 'c')
|
||||||
|
except OSError:
|
||||||
|
pass
|
||||||
|
|
||||||
|
from devscripts.utils import read_file, write_file
|
||||||
|
from youtube_dl.compat import compat_register_utf8
|
||||||
|
|
||||||
|
compat_register_utf8()
|
||||||
|
|
||||||
from youtube_dl.extractor import _ALL_CLASSES
|
from youtube_dl.extractor import _ALL_CLASSES
|
||||||
from youtube_dl.extractor.common import InfoExtractor, SearchInfoExtractor
|
from youtube_dl.extractor.common import InfoExtractor, SearchInfoExtractor
|
||||||
|
|
||||||
with open('devscripts/lazy_load_template.py', 'rt') as f:
|
module_template = read_file('devscripts/lazy_load_template.py')
|
||||||
module_template = f.read()
|
|
||||||
|
|
||||||
|
def get_source(m):
|
||||||
|
return re.sub(r'(?m)^\s*#.*\n', '', getsource(m))
|
||||||
|
|
||||||
|
|
||||||
module_contents = [
|
module_contents = [
|
||||||
module_template + '\n' + getsource(InfoExtractor.suitable) + '\n',
|
module_template,
|
||||||
'class LazyLoadSearchExtractor(LazyLoadExtractor):\n pass\n']
|
get_source(InfoExtractor.suitable),
|
||||||
|
get_source(InfoExtractor._match_valid_url) + '\n',
|
||||||
|
'class LazyLoadSearchExtractor(LazyLoadExtractor):\n pass\n',
|
||||||
|
# needed for suitable() methods of Youtube extractor (see #28780)
|
||||||
|
'from youtube_dl.utils import parse_qs, variadic\n',
|
||||||
|
]
|
||||||
|
|
||||||
ie_template = '''
|
ie_template = '''
|
||||||
class {name}({bases}):
|
class {name}({bases}):
|
||||||
|
@ -54,14 +75,14 @@ def build_lazy_ie(ie, name):
|
||||||
valid_url=valid_url,
|
valid_url=valid_url,
|
||||||
module=ie.__module__)
|
module=ie.__module__)
|
||||||
if ie.suitable.__func__ is not InfoExtractor.suitable.__func__:
|
if ie.suitable.__func__ is not InfoExtractor.suitable.__func__:
|
||||||
s += '\n' + getsource(ie.suitable)
|
s += '\n' + get_source(ie.suitable)
|
||||||
if hasattr(ie, '_make_valid_url'):
|
if hasattr(ie, '_make_valid_url'):
|
||||||
# search extractors
|
# search extractors
|
||||||
s += make_valid_template.format(valid_url=ie._make_valid_url())
|
s += make_valid_template.format(valid_url=ie._make_valid_url())
|
||||||
return s
|
return s
|
||||||
|
|
||||||
|
|
||||||
# find the correct sorting and add the required base classes so that sublcasses
|
# find the correct sorting and add the required base classes so that subclasses
|
||||||
# can be correctly created
|
# can be correctly created
|
||||||
classes = _ALL_CLASSES[:-1]
|
classes = _ALL_CLASSES[:-1]
|
||||||
ordered_cls = []
|
ordered_cls = []
|
||||||
|
@ -94,7 +115,17 @@ for ie in ordered_cls:
|
||||||
module_contents.append(
|
module_contents.append(
|
||||||
'_ALL_CLASSES = [{0}]'.format(', '.join(names)))
|
'_ALL_CLASSES = [{0}]'.format(', '.join(names)))
|
||||||
|
|
||||||
module_src = '\n'.join(module_contents) + '\n'
|
module_src = '\n'.join(module_contents)
|
||||||
|
|
||||||
with io.open(lazy_extractors_filename, 'wt', encoding='utf-8') as f:
|
write_file(lazy_extractors_filename, module_src + '\n')
|
||||||
f.write(module_src)
|
|
||||||
|
# work around JVM byte code module limit in Jython
|
||||||
|
if sys.platform.startswith('java') and sys.version_info[:2] == (2, 7):
|
||||||
|
import subprocess
|
||||||
|
from youtube_dl.compat import compat_subprocess_get_DEVNULL
|
||||||
|
# if Python 2.7 is available, use it to compile the module for Jython
|
||||||
|
try:
|
||||||
|
# if Python 2.7 is available, use it to compile the module for Jython
|
||||||
|
subprocess.check_call(['python2.7', '-m', 'py_compile', lazy_extractors_filename], stdout=compat_subprocess_get_DEVNULL())
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
|
|
@ -1,8 +1,14 @@
|
||||||
from __future__ import unicode_literals
|
from __future__ import unicode_literals
|
||||||
|
|
||||||
import io
|
import os.path
|
||||||
import sys
|
|
||||||
import re
|
import re
|
||||||
|
import sys
|
||||||
|
dirn = os.path.dirname
|
||||||
|
|
||||||
|
sys.path.insert(0, dirn(dirn(os.path.abspath(__file__))))
|
||||||
|
|
||||||
|
from utils import read_file
|
||||||
|
from youtube_dl.compat import compat_open as open
|
||||||
|
|
||||||
README_FILE = 'README.md'
|
README_FILE = 'README.md'
|
||||||
helptext = sys.stdin.read()
|
helptext = sys.stdin.read()
|
||||||
|
@ -10,8 +16,7 @@ helptext = sys.stdin.read()
|
||||||
if isinstance(helptext, bytes):
|
if isinstance(helptext, bytes):
|
||||||
helptext = helptext.decode('utf-8')
|
helptext = helptext.decode('utf-8')
|
||||||
|
|
||||||
with io.open(README_FILE, encoding='utf-8') as f:
|
oldreadme = read_file(README_FILE)
|
||||||
oldreadme = f.read()
|
|
||||||
|
|
||||||
header = oldreadme[:oldreadme.index('# OPTIONS')]
|
header = oldreadme[:oldreadme.index('# OPTIONS')]
|
||||||
footer = oldreadme[oldreadme.index('# CONFIGURATION'):]
|
footer = oldreadme[oldreadme.index('# CONFIGURATION'):]
|
||||||
|
@ -20,7 +25,7 @@ options = helptext[helptext.index(' General Options:') + 19:]
|
||||||
options = re.sub(r'(?m)^ (\w.+)$', r'## \1', options)
|
options = re.sub(r'(?m)^ (\w.+)$', r'## \1', options)
|
||||||
options = '# OPTIONS\n' + options + '\n'
|
options = '# OPTIONS\n' + options + '\n'
|
||||||
|
|
||||||
with io.open(README_FILE, 'w', encoding='utf-8') as f:
|
with open(README_FILE, 'w', encoding='utf-8') as f:
|
||||||
f.write(header)
|
f.write(header)
|
||||||
f.write(options)
|
f.write(options)
|
||||||
f.write(footer)
|
f.write(footer)
|
||||||
|
|
|
@ -1,17 +1,19 @@
|
||||||
#!/usr/bin/env python
|
#!/usr/bin/env python
|
||||||
from __future__ import unicode_literals
|
from __future__ import unicode_literals
|
||||||
|
|
||||||
import io
|
|
||||||
import optparse
|
import optparse
|
||||||
import os
|
import os.path
|
||||||
import sys
|
import sys
|
||||||
|
|
||||||
|
|
||||||
# Import youtube_dl
|
# Import youtube_dl
|
||||||
ROOT_DIR = os.path.join(os.path.dirname(__file__), '..')
|
dirn = os.path.dirname
|
||||||
sys.path.insert(0, ROOT_DIR)
|
|
||||||
|
sys.path.insert(0, dirn(dirn(os.path.abspath(__file__))))
|
||||||
|
|
||||||
import youtube_dl
|
import youtube_dl
|
||||||
|
|
||||||
|
from utils import write_file
|
||||||
|
|
||||||
|
|
||||||
def main():
|
def main():
|
||||||
parser = optparse.OptionParser(usage='%prog OUTFILE.md')
|
parser = optparse.OptionParser(usage='%prog OUTFILE.md')
|
||||||
|
@ -38,8 +40,7 @@ def main():
|
||||||
' - ' + md + '\n'
|
' - ' + md + '\n'
|
||||||
for md in gen_ies_md(ies))
|
for md in gen_ies_md(ies))
|
||||||
|
|
||||||
with io.open(outfile, 'w', encoding='utf-8') as outf:
|
write_file(outfile, out)
|
||||||
outf.write(out)
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == '__main__':
|
if __name__ == '__main__':
|
||||||
|
|
|
@ -1,13 +1,13 @@
|
||||||
from __future__ import unicode_literals
|
from __future__ import unicode_literals
|
||||||
|
|
||||||
import io
|
|
||||||
import optparse
|
import optparse
|
||||||
import os.path
|
import os.path
|
||||||
import re
|
import re
|
||||||
|
|
||||||
|
from utils import read_file, write_file
|
||||||
|
|
||||||
ROOT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
ROOT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||||
README_FILE = os.path.join(ROOT_DIR, 'README.md')
|
README_FILE = os.path.join(ROOT_DIR, 'README.md')
|
||||||
|
|
||||||
PREFIX = r'''%YOUTUBE-DL(1)
|
PREFIX = r'''%YOUTUBE-DL(1)
|
||||||
|
|
||||||
# NAME
|
# NAME
|
||||||
|
@ -29,8 +29,7 @@ def main():
|
||||||
|
|
||||||
outfile, = args
|
outfile, = args
|
||||||
|
|
||||||
with io.open(README_FILE, encoding='utf-8') as f:
|
readme = read_file(README_FILE)
|
||||||
readme = f.read()
|
|
||||||
|
|
||||||
readme = re.sub(r'(?s)^.*?(?=# DESCRIPTION)', '', readme)
|
readme = re.sub(r'(?s)^.*?(?=# DESCRIPTION)', '', readme)
|
||||||
readme = re.sub(r'\s+youtube-dl \[OPTIONS\] URL \[URL\.\.\.\]', '', readme)
|
readme = re.sub(r'\s+youtube-dl \[OPTIONS\] URL \[URL\.\.\.\]', '', readme)
|
||||||
|
@ -38,8 +37,7 @@ def main():
|
||||||
|
|
||||||
readme = filter_options(readme)
|
readme = filter_options(readme)
|
||||||
|
|
||||||
with io.open(outfile, 'w', encoding='utf-8') as outf:
|
write_file(outfile, readme)
|
||||||
outf.write(readme)
|
|
||||||
|
|
||||||
|
|
||||||
def filter_options(readme):
|
def filter_options(readme):
|
||||||
|
|
17
devscripts/run_tests.bat
Normal file
17
devscripts/run_tests.bat
Normal file
|
@ -0,0 +1,17 @@
|
||||||
|
@echo off
|
||||||
|
|
||||||
|
rem Keep this list in sync with the `offlinetest` target in Makefile
|
||||||
|
set DOWNLOAD_TESTS="age_restriction^|download^|iqiyi_sdk_interpreter^|socks^|subtitles^|write_annotations^|youtube_lists^|youtube_signature"
|
||||||
|
|
||||||
|
if "%YTDL_TEST_SET%" == "core" (
|
||||||
|
set test_set="-I test_("%DOWNLOAD_TESTS%")\.py"
|
||||||
|
set multiprocess_args=""
|
||||||
|
) else if "%YTDL_TEST_SET%" == "download" (
|
||||||
|
set test_set="-I test_(?!"%DOWNLOAD_TESTS%").+\.py"
|
||||||
|
set multiprocess_args="--processes=4 --process-timeout=540"
|
||||||
|
) else (
|
||||||
|
echo YTDL_TEST_SET is not set or invalid
|
||||||
|
exit /b 1
|
||||||
|
)
|
||||||
|
|
||||||
|
nosetests test --verbose %test_set:"=% %multiprocess_args:"=%
|
62
devscripts/utils.py
Normal file
62
devscripts/utils.py
Normal file
|
@ -0,0 +1,62 @@
|
||||||
|
# coding: utf-8
|
||||||
|
from __future__ import unicode_literals
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import functools
|
||||||
|
import os.path
|
||||||
|
import subprocess
|
||||||
|
import sys
|
||||||
|
|
||||||
|
dirn = os.path.dirname
|
||||||
|
|
||||||
|
sys.path.insert(0, dirn(dirn(os.path.abspath(__file__))))
|
||||||
|
|
||||||
|
from youtube_dl.compat import (
|
||||||
|
compat_kwargs,
|
||||||
|
compat_open as open,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def read_file(fname):
|
||||||
|
with open(fname, encoding='utf-8') as f:
|
||||||
|
return f.read()
|
||||||
|
|
||||||
|
|
||||||
|
def write_file(fname, content, mode='w'):
|
||||||
|
with open(fname, mode, encoding='utf-8') as f:
|
||||||
|
return f.write(content)
|
||||||
|
|
||||||
|
|
||||||
|
def read_version(fname='youtube_dl/version.py'):
|
||||||
|
"""Get the version without importing the package"""
|
||||||
|
exec(compile(read_file(fname), fname, 'exec'))
|
||||||
|
return locals()['__version__']
|
||||||
|
|
||||||
|
|
||||||
|
def get_filename_args(has_infile=False, default_outfile=None):
|
||||||
|
parser = argparse.ArgumentParser()
|
||||||
|
if has_infile:
|
||||||
|
parser.add_argument('infile', help='Input file')
|
||||||
|
kwargs = {'nargs': '?', 'default': default_outfile} if default_outfile else {}
|
||||||
|
kwargs['help'] = 'Output file'
|
||||||
|
parser.add_argument('outfile', **compat_kwargs(kwargs))
|
||||||
|
|
||||||
|
opts = parser.parse_args()
|
||||||
|
if has_infile:
|
||||||
|
return opts.infile, opts.outfile
|
||||||
|
return opts.outfile
|
||||||
|
|
||||||
|
|
||||||
|
def compose_functions(*functions):
|
||||||
|
return lambda x: functools.reduce(lambda y, f: f(y), functions, x)
|
||||||
|
|
||||||
|
|
||||||
|
def run_process(*args, **kwargs):
|
||||||
|
kwargs.setdefault('text', True)
|
||||||
|
kwargs.setdefault('check', True)
|
||||||
|
kwargs.setdefault('capture_output', True)
|
||||||
|
if kwargs['text']:
|
||||||
|
kwargs.setdefault('encoding', 'utf-8')
|
||||||
|
kwargs.setdefault('errors', 'replace')
|
||||||
|
kwargs = compat_kwargs(kwargs)
|
||||||
|
return subprocess.run(args, **kwargs)
|
|
@ -7,6 +7,8 @@ import sys
|
||||||
|
|
||||||
sys.path.insert(0, dirn(dirn((os.path.abspath(__file__)))))
|
sys.path.insert(0, dirn(dirn((os.path.abspath(__file__)))))
|
||||||
import youtube_dl
|
import youtube_dl
|
||||||
|
from utils import read_file, write_file
|
||||||
|
|
||||||
|
|
||||||
ZSH_COMPLETION_FILE = "youtube-dl.zsh"
|
ZSH_COMPLETION_FILE = "youtube-dl.zsh"
|
||||||
ZSH_COMPLETION_TEMPLATE = "devscripts/zsh-completion.in"
|
ZSH_COMPLETION_TEMPLATE = "devscripts/zsh-completion.in"
|
||||||
|
@ -34,15 +36,13 @@ def build_completion(opt_parser):
|
||||||
|
|
||||||
flags = [opt.get_opt_string() for opt in opts]
|
flags = [opt.get_opt_string() for opt in opts]
|
||||||
|
|
||||||
with open(ZSH_COMPLETION_TEMPLATE) as f:
|
template = read_file(ZSH_COMPLETION_TEMPLATE)
|
||||||
template = f.read()
|
|
||||||
|
|
||||||
template = template.replace("{{fileopts}}", "|".join(fileopts))
|
template = template.replace("{{fileopts}}", "|".join(fileopts))
|
||||||
template = template.replace("{{diropts}}", "|".join(diropts))
|
template = template.replace("{{diropts}}", "|".join(diropts))
|
||||||
template = template.replace("{{flags}}", " ".join(flags))
|
template = template.replace("{{flags}}", " ".join(flags))
|
||||||
|
|
||||||
with open(ZSH_COMPLETION_FILE, "w") as f:
|
write_file(ZSH_COMPLETION_FILE, template)
|
||||||
f.write(template)
|
|
||||||
|
|
||||||
|
|
||||||
parser = youtube_dl.parseOpts()[0]
|
parser = youtube_dl.parseOpts()[0]
|
||||||
|
|
|
@ -1,9 +1,9 @@
|
||||||
# Supported sites
|
# Supported sites
|
||||||
- **1tv**: Первый канал
|
- **1tv**: Первый канал
|
||||||
- **1up.com**
|
|
||||||
- **20min**
|
- **20min**
|
||||||
- **220.ro**
|
- **220.ro**
|
||||||
- **23video**
|
- **23video**
|
||||||
|
- **247sports**
|
||||||
- **24video**
|
- **24video**
|
||||||
- **3qsdn**: 3Q SDN
|
- **3qsdn**: 3Q SDN
|
||||||
- **3sat**
|
- **3sat**
|
||||||
|
@ -35,32 +35,39 @@
|
||||||
- **adobetv:video**
|
- **adobetv:video**
|
||||||
- **AdultSwim**
|
- **AdultSwim**
|
||||||
- **aenetworks**: A+E Networks: A&E, Lifetime, History.com, FYI Network and History Vault
|
- **aenetworks**: A+E Networks: A&E, Lifetime, History.com, FYI Network and History Vault
|
||||||
|
- **aenetworks:collection**
|
||||||
|
- **aenetworks:show**
|
||||||
- **afreecatv**: afreecatv.com
|
- **afreecatv**: afreecatv.com
|
||||||
- **AirMozilla**
|
- **AirMozilla**
|
||||||
- **AliExpressLive**
|
- **AliExpressLive**
|
||||||
- **AlJazeera**
|
- **AlJazeera**
|
||||||
- **Allocine**
|
- **Allocine**
|
||||||
- **AlphaPorno**
|
- **AlphaPorno**
|
||||||
|
- **Amara**
|
||||||
- **AMCNetworks**
|
- **AMCNetworks**
|
||||||
- **AmericasTestKitchen**
|
- **AmericasTestKitchen**
|
||||||
|
- **AmericasTestKitchenSeason**
|
||||||
- **anderetijden**: npo.nl, ntr.nl, omroepwnl.nl, zapp.nl and npo3.nl
|
- **anderetijden**: npo.nl, ntr.nl, omroepwnl.nl, zapp.nl and npo3.nl
|
||||||
- **AnimeOnDemand**
|
- **AnimeOnDemand**
|
||||||
- **Anvato**
|
- **Anvato**
|
||||||
- **aol.com**
|
- **aol.com**: Yahoo screen and movies
|
||||||
- **APA**
|
- **APA**
|
||||||
- **Aparat**
|
- **Aparat**
|
||||||
- **AppleConnect**
|
- **AppleConnect**
|
||||||
- **AppleDaily**: 臺灣蘋果日報
|
- **AppleDaily**: 臺灣蘋果日報
|
||||||
|
- **ApplePodcasts**
|
||||||
- **appletrailers**
|
- **appletrailers**
|
||||||
- **appletrailers:section**
|
- **appletrailers:section**
|
||||||
- **archive.org**: archive.org videos
|
- **archive.org**: archive.org videos
|
||||||
|
- **ArcPublishing**
|
||||||
- **ARD**
|
- **ARD**
|
||||||
- **ARD:mediathek**
|
- **ARD:mediathek**
|
||||||
- **ARDBetaMediathek**
|
- **ARDBetaMediathek**
|
||||||
- **Arkena**
|
- **Arkena**
|
||||||
- **arte.tv:+7**
|
- **arte.sky.it**
|
||||||
- **arte.tv:embed**
|
- **ArteTV**
|
||||||
- **arte.tv:playlist**
|
- **ArteTVEmbed**
|
||||||
|
- **ArteTVPlaylist**
|
||||||
- **AsianCrush**
|
- **AsianCrush**
|
||||||
- **AsianCrushPlaylist**
|
- **AsianCrushPlaylist**
|
||||||
- **AtresPlayer**
|
- **AtresPlayer**
|
||||||
|
@ -76,6 +83,7 @@
|
||||||
- **awaan:video**
|
- **awaan:video**
|
||||||
- **AZMedien**: AZ Medien videos
|
- **AZMedien**: AZ Medien videos
|
||||||
- **BaiduVideo**: 百度视频
|
- **BaiduVideo**: 百度视频
|
||||||
|
- **bandaichannel**
|
||||||
- **Bandcamp**
|
- **Bandcamp**
|
||||||
- **Bandcamp:album**
|
- **Bandcamp:album**
|
||||||
- **Bandcamp:weekly**
|
- **Bandcamp:weekly**
|
||||||
|
@ -83,7 +91,8 @@
|
||||||
- **bbc**: BBC
|
- **bbc**: BBC
|
||||||
- **bbc.co.uk**: BBC iPlayer
|
- **bbc.co.uk**: BBC iPlayer
|
||||||
- **bbc.co.uk:article**: BBC articles
|
- **bbc.co.uk:article**: BBC articles
|
||||||
- **bbc.co.uk:iplayer:playlist**
|
- **bbc.co.uk:iplayer:episodes**
|
||||||
|
- **bbc.co.uk:iplayer:group**
|
||||||
- **bbc.co.uk:playlist**
|
- **bbc.co.uk:playlist**
|
||||||
- **BBVTV**
|
- **BBVTV**
|
||||||
- **Beatport**
|
- **Beatport**
|
||||||
|
@ -93,21 +102,28 @@
|
||||||
- **BellMedia**
|
- **BellMedia**
|
||||||
- **Bet**
|
- **Bet**
|
||||||
- **bfi:player**
|
- **bfi:player**
|
||||||
|
- **bfmtv**
|
||||||
|
- **bfmtv:article**
|
||||||
|
- **bfmtv:live**
|
||||||
|
- **BibelTV**
|
||||||
- **Bigflix**
|
- **Bigflix**
|
||||||
- **Bild**: Bild.de
|
- **Bild**: Bild.de
|
||||||
- **BiliBili**
|
- **BiliBili**
|
||||||
- **BilibiliAudio**
|
- **BilibiliAudio**
|
||||||
- **BilibiliAudioAlbum**
|
- **BilibiliAudioAlbum**
|
||||||
|
- **BiliBiliPlayer**
|
||||||
- **BioBioChileTV**
|
- **BioBioChileTV**
|
||||||
|
- **Biography**
|
||||||
- **BIQLE**
|
- **BIQLE**
|
||||||
- **BitChute**
|
- **BitChute**
|
||||||
- **BitChuteChannel**
|
- **BitChuteChannel**
|
||||||
- **BleacherReport**
|
- **BleacherReport**
|
||||||
- **BleacherReportCMS**
|
- **BleacherReportCMS**
|
||||||
- **blinkx**
|
|
||||||
- **Bloomberg**
|
- **Bloomberg**
|
||||||
- **BokeCC**
|
- **BokeCC**
|
||||||
|
- **BongaCams**
|
||||||
- **BostonGlobe**
|
- **BostonGlobe**
|
||||||
|
- **Box**
|
||||||
- **Bpb**: Bundeszentrale für politische Bildung
|
- **Bpb**: Bundeszentrale für politische Bildung
|
||||||
- **BR**: Bayerischer Rundfunk
|
- **BR**: Bayerischer Rundfunk
|
||||||
- **BravoTV**
|
- **BravoTV**
|
||||||
|
@ -140,10 +156,12 @@
|
||||||
- **CBS**
|
- **CBS**
|
||||||
- **CBSInteractive**
|
- **CBSInteractive**
|
||||||
- **CBSLocal**
|
- **CBSLocal**
|
||||||
|
- **CBSLocalArticle**
|
||||||
- **cbsnews**: CBS News
|
- **cbsnews**: CBS News
|
||||||
- **cbsnews:embed**
|
- **cbsnews:embed**
|
||||||
- **cbsnews:livevideo**: CBS News Live Videos
|
- **cbsnews:livevideo**: CBS News Live Videos
|
||||||
- **CBSSports**
|
- **cbssports**
|
||||||
|
- **cbssports:embed**
|
||||||
- **CCMA**
|
- **CCMA**
|
||||||
- **CCTV**: 央视网
|
- **CCTV**: 央视网
|
||||||
- **CDA**
|
- **CDA**
|
||||||
|
@ -155,6 +173,7 @@
|
||||||
- **Chilloutzone**
|
- **Chilloutzone**
|
||||||
- **chirbit**
|
- **chirbit**
|
||||||
- **chirbit:profile**
|
- **chirbit:profile**
|
||||||
|
- **cielotv.it**
|
||||||
- **Cinchcast**
|
- **Cinchcast**
|
||||||
- **Cinemax**
|
- **Cinemax**
|
||||||
- **CiscoLiveSearch**
|
- **CiscoLiveSearch**
|
||||||
|
@ -176,8 +195,6 @@
|
||||||
- **CNNArticle**
|
- **CNNArticle**
|
||||||
- **CNNBlogs**
|
- **CNNBlogs**
|
||||||
- **ComedyCentral**
|
- **ComedyCentral**
|
||||||
- **ComedyCentralFullEpisodes**
|
|
||||||
- **ComedyCentralShortname**
|
|
||||||
- **ComedyCentralTV**
|
- **ComedyCentralTV**
|
||||||
- **CondeNast**: Condé Nast media group: Allure, Architectural Digest, Ars Technica, Bon Appétit, Brides, Condé Nast, Condé Nast Traveler, Details, Epicurious, GQ, Glamour, Golf Digest, SELF, Teen Vogue, The New Yorker, Vanity Fair, Vogue, W Magazine, WIRED
|
- **CondeNast**: Condé Nast media group: Allure, Architectural Digest, Ars Technica, Bon Appétit, Brides, Condé Nast, Condé Nast Traveler, Details, Epicurious, GQ, Glamour, Golf Digest, SELF, Teen Vogue, The New Yorker, Vanity Fair, Vogue, W Magazine, WIRED
|
||||||
- **CONtv**
|
- **CONtv**
|
||||||
|
@ -188,9 +205,9 @@
|
||||||
- **CrooksAndLiars**
|
- **CrooksAndLiars**
|
||||||
- **crunchyroll**
|
- **crunchyroll**
|
||||||
- **crunchyroll:playlist**
|
- **crunchyroll:playlist**
|
||||||
- **CSNNE**
|
|
||||||
- **CSpan**: C-SPAN
|
- **CSpan**: C-SPAN
|
||||||
- **CtsNews**: 華視新聞
|
- **CtsNews**: 華視新聞
|
||||||
|
- **CTV**
|
||||||
- **CTVNews**
|
- **CTVNews**
|
||||||
- **cu.ntv.co.jp**: Nippon Television Network
|
- **cu.ntv.co.jp**: Nippon Television Network
|
||||||
- **Culturebox**
|
- **Culturebox**
|
||||||
|
@ -198,6 +215,7 @@
|
||||||
- **curiositystream**
|
- **curiositystream**
|
||||||
- **curiositystream:collection**
|
- **curiositystream:collection**
|
||||||
- **CWTV**
|
- **CWTV**
|
||||||
|
- **DagelijkseKost**: dagelijksekost.een.be
|
||||||
- **DailyMail**
|
- **DailyMail**
|
||||||
- **dailymotion**
|
- **dailymotion**
|
||||||
- **dailymotion:playlist**
|
- **dailymotion:playlist**
|
||||||
|
@ -219,6 +237,7 @@
|
||||||
- **DiscoveryGo**
|
- **DiscoveryGo**
|
||||||
- **DiscoveryGoPlaylist**
|
- **DiscoveryGoPlaylist**
|
||||||
- **DiscoveryNetworksDe**
|
- **DiscoveryNetworksDe**
|
||||||
|
- **DiscoveryPlus**
|
||||||
- **DiscoveryVR**
|
- **DiscoveryVR**
|
||||||
- **Disney**
|
- **Disney**
|
||||||
- **dlive:stream**
|
- **dlive:stream**
|
||||||
|
@ -261,7 +280,6 @@
|
||||||
- **ESPNArticle**
|
- **ESPNArticle**
|
||||||
- **EsriVideo**
|
- **EsriVideo**
|
||||||
- **Europa**
|
- **Europa**
|
||||||
- **EveryonesMixtape**
|
|
||||||
- **EWETV**
|
- **EWETV**
|
||||||
- **ExpoTV**
|
- **ExpoTV**
|
||||||
- **Expressen**
|
- **Expressen**
|
||||||
|
@ -303,11 +321,11 @@
|
||||||
- **FrontendMasters**
|
- **FrontendMasters**
|
||||||
- **FrontendMastersCourse**
|
- **FrontendMastersCourse**
|
||||||
- **FrontendMastersLesson**
|
- **FrontendMastersLesson**
|
||||||
|
- **FujiTVFODPlus7**
|
||||||
- **Funimation**
|
- **Funimation**
|
||||||
- **Funk**
|
- **Funk**
|
||||||
- **Fusion**
|
- **Fusion**
|
||||||
- **Fux**
|
- **Fux**
|
||||||
- **FXNetworks**
|
|
||||||
- **Gaia**
|
- **Gaia**
|
||||||
- **GameInformer**
|
- **GameInformer**
|
||||||
- **GameSpot**
|
- **GameSpot**
|
||||||
|
@ -315,6 +333,7 @@
|
||||||
- **Gaskrank**
|
- **Gaskrank**
|
||||||
- **Gazeta**
|
- **Gazeta**
|
||||||
- **GDCVault**
|
- **GDCVault**
|
||||||
|
- **GediDigital**
|
||||||
- **generic**: Generic downloader that works on some sites
|
- **generic**: Generic downloader that works on some sites
|
||||||
- **Gfycat**
|
- **Gfycat**
|
||||||
- **GiantBomb**
|
- **GiantBomb**
|
||||||
|
@ -326,6 +345,8 @@
|
||||||
- **Go**
|
- **Go**
|
||||||
- **GodTube**
|
- **GodTube**
|
||||||
- **Golem**
|
- **Golem**
|
||||||
|
- **google:podcasts**
|
||||||
|
- **google:podcasts:feed**
|
||||||
- **GoogleDrive**
|
- **GoogleDrive**
|
||||||
- **Goshgay**
|
- **Goshgay**
|
||||||
- **GPUTechConf**
|
- **GPUTechConf**
|
||||||
|
@ -338,8 +359,10 @@
|
||||||
- **HentaiStigma**
|
- **HentaiStigma**
|
||||||
- **hetklokhuis**
|
- **hetklokhuis**
|
||||||
- **hgtv.com:show**
|
- **hgtv.com:show**
|
||||||
|
- **HGTVDe**
|
||||||
- **HiDive**
|
- **HiDive**
|
||||||
- **HistoricFilms**
|
- **HistoricFilms**
|
||||||
|
- **history:player**
|
||||||
- **history:topic**: History.com Topic
|
- **history:topic**: History.com Topic
|
||||||
- **hitbox**
|
- **hitbox**
|
||||||
- **hitbox:live**
|
- **hitbox:live**
|
||||||
|
@ -359,6 +382,10 @@
|
||||||
- **HungamaSong**
|
- **HungamaSong**
|
||||||
- **Hypem**
|
- **Hypem**
|
||||||
- **ign.com**
|
- **ign.com**
|
||||||
|
- **IGNArticle**
|
||||||
|
- **IGNVideo**
|
||||||
|
- **IHeartRadio**
|
||||||
|
- **iheartradio:podcast**
|
||||||
- **imdb**: Internet Movie Database trailers
|
- **imdb**: Internet Movie Database trailers
|
||||||
- **imdb:list**: Internet Movie Database lists
|
- **imdb:list**: Internet Movie Database lists
|
||||||
- **Imgur**
|
- **Imgur**
|
||||||
|
@ -392,14 +419,14 @@
|
||||||
- **JWPlatform**
|
- **JWPlatform**
|
||||||
- **Kakao**
|
- **Kakao**
|
||||||
- **Kaltura**
|
- **Kaltura**
|
||||||
- **KanalPlay**: Kanal 5/9/11 Play
|
|
||||||
- **Kankan**
|
- **Kankan**
|
||||||
- **Karaoketv**
|
- **Karaoketv**
|
||||||
- **KarriereVideos**
|
- **KarriereVideos**
|
||||||
- **Katsomo**
|
- **Katsomo**
|
||||||
- **KeezMovies**
|
- **KeezMovies**
|
||||||
- **Ketnet**
|
- **Ketnet**
|
||||||
- **KhanAcademy**
|
- **khanacademy**
|
||||||
|
- **khanacademy:unit**
|
||||||
- **KickStarter**
|
- **KickStarter**
|
||||||
- **KinjaEmbed**
|
- **KinjaEmbed**
|
||||||
- **KinoPoisk**
|
- **KinoPoisk**
|
||||||
|
@ -416,6 +443,8 @@
|
||||||
- **la7.it**
|
- **la7.it**
|
||||||
- **laola1tv**
|
- **laola1tv**
|
||||||
- **laola1tv:embed**
|
- **laola1tv:embed**
|
||||||
|
- **lbry**
|
||||||
|
- **lbry:channel**
|
||||||
- **LCI**
|
- **LCI**
|
||||||
- **Lcp**
|
- **Lcp**
|
||||||
- **LcpPlay**
|
- **LcpPlay**
|
||||||
|
@ -435,14 +464,14 @@
|
||||||
- **limelight**
|
- **limelight**
|
||||||
- **limelight:channel**
|
- **limelight:channel**
|
||||||
- **limelight:channel_list**
|
- **limelight:channel_list**
|
||||||
|
- **LineLive**
|
||||||
|
- **LineLiveChannel**
|
||||||
- **LineTV**
|
- **LineTV**
|
||||||
- **linkedin:learning**
|
- **linkedin:learning**
|
||||||
- **linkedin:learning:course**
|
- **linkedin:learning:course**
|
||||||
- **LinuxAcademy**
|
- **LinuxAcademy**
|
||||||
- **LiTV**
|
- **LiTV**
|
||||||
- **LiveJournal**
|
- **LiveJournal**
|
||||||
- **LiveLeak**
|
|
||||||
- **LiveLeakEmbed**
|
|
||||||
- **livestream**
|
- **livestream**
|
||||||
- **livestream:original**
|
- **livestream:original**
|
||||||
- **LnkGo**
|
- **LnkGo**
|
||||||
|
@ -460,11 +489,13 @@
|
||||||
- **mangomolo:live**
|
- **mangomolo:live**
|
||||||
- **mangomolo:video**
|
- **mangomolo:video**
|
||||||
- **ManyVids**
|
- **ManyVids**
|
||||||
|
- **MaoriTV**
|
||||||
- **Markiza**
|
- **Markiza**
|
||||||
- **MarkizaPage**
|
- **MarkizaPage**
|
||||||
- **massengeschmack.tv**
|
- **massengeschmack.tv**
|
||||||
- **MatchTV**
|
- **MatchTV**
|
||||||
- **MDR**: MDR.DE and KiKA
|
- **MDR**: MDR.DE and KiKA
|
||||||
|
- **MedalTV**
|
||||||
- **media.ccc.de**
|
- **media.ccc.de**
|
||||||
- **media.ccc.de:lists**
|
- **media.ccc.de:lists**
|
||||||
- **Medialaan**
|
- **Medialaan**
|
||||||
|
@ -479,9 +510,13 @@
|
||||||
- **META**
|
- **META**
|
||||||
- **metacafe**
|
- **metacafe**
|
||||||
- **Metacritic**
|
- **Metacritic**
|
||||||
|
- **mewatch**
|
||||||
- **Mgoon**
|
- **Mgoon**
|
||||||
- **MGTV**: 芒果TV
|
- **MGTV**: 芒果TV
|
||||||
- **MiaoPai**
|
- **MiaoPai**
|
||||||
|
- **minds**
|
||||||
|
- **minds:channel**
|
||||||
|
- **minds:group**
|
||||||
- **MinistryGrid**
|
- **MinistryGrid**
|
||||||
- **Minoto**
|
- **Minoto**
|
||||||
- **miomio.tv**
|
- **miomio.tv**
|
||||||
|
@ -489,13 +524,13 @@
|
||||||
- **mixcloud**
|
- **mixcloud**
|
||||||
- **mixcloud:playlist**
|
- **mixcloud:playlist**
|
||||||
- **mixcloud:user**
|
- **mixcloud:user**
|
||||||
- **Mixer:live**
|
|
||||||
- **Mixer:vod**
|
|
||||||
- **MLB**
|
- **MLB**
|
||||||
|
- **MLBVideo**
|
||||||
- **Mnet**
|
- **Mnet**
|
||||||
- **MNetTV**
|
- **MNetTV**
|
||||||
- **MoeVideo**: LetitBit video services: moevideo.net, playreplay.net and videochart.net
|
- **MoeVideo**: LetitBit video services: moevideo.net, playreplay.net and videochart.net
|
||||||
- **Mofosex**
|
- **Mofosex**
|
||||||
|
- **MofosexEmbed**
|
||||||
- **Mojvideo**
|
- **Mojvideo**
|
||||||
- **Morningstar**: morningstar.com
|
- **Morningstar**: morningstar.com
|
||||||
- **Motherless**
|
- **Motherless**
|
||||||
|
@ -512,6 +547,7 @@
|
||||||
- **mtv:video**
|
- **mtv:video**
|
||||||
- **mtvjapan**
|
- **mtvjapan**
|
||||||
- **mtvservices:embedded**
|
- **mtvservices:embedded**
|
||||||
|
- **MTVUutisetArticle**
|
||||||
- **MuenchenTV**: münchen.tv
|
- **MuenchenTV**: münchen.tv
|
||||||
- **mva**: Microsoft Virtual Academy videos
|
- **mva**: Microsoft Virtual Academy videos
|
||||||
- **mva:course**: Microsoft Virtual Academy courses
|
- **mva:course**: Microsoft Virtual Academy courses
|
||||||
|
@ -530,6 +566,11 @@
|
||||||
- **NationalGeographicTV**
|
- **NationalGeographicTV**
|
||||||
- **Naver**
|
- **Naver**
|
||||||
- **NBA**
|
- **NBA**
|
||||||
|
- **nba:watch**
|
||||||
|
- **nba:watch:collection**
|
||||||
|
- **NBAChannel**
|
||||||
|
- **NBAEmbed**
|
||||||
|
- **NBAWatchEmbed**
|
||||||
- **NBC**
|
- **NBC**
|
||||||
- **NBCNews**
|
- **NBCNews**
|
||||||
- **nbcolympics**
|
- **nbcolympics**
|
||||||
|
@ -559,8 +600,10 @@
|
||||||
- **NextTV**: 壹電視
|
- **NextTV**: 壹電視
|
||||||
- **Nexx**
|
- **Nexx**
|
||||||
- **NexxEmbed**
|
- **NexxEmbed**
|
||||||
- **nfl.com**
|
- **nfl.com** (Currently broken)
|
||||||
|
- **nfl.com:article** (Currently broken)
|
||||||
- **NhkVod**
|
- **NhkVod**
|
||||||
|
- **NhkVodProgram**
|
||||||
- **nhl.com**
|
- **nhl.com**
|
||||||
- **nick.com**
|
- **nick.com**
|
||||||
- **nick.de**
|
- **nick.de**
|
||||||
|
@ -574,7 +617,6 @@
|
||||||
- **njoy:embed**
|
- **njoy:embed**
|
||||||
- **NJPWWorld**: 新日本プロレスワールド
|
- **NJPWWorld**: 新日本プロレスワールド
|
||||||
- **NobelPrize**
|
- **NobelPrize**
|
||||||
- **Noco**
|
|
||||||
- **NonkTube**
|
- **NonkTube**
|
||||||
- **Noovo**
|
- **Noovo**
|
||||||
- **Normalboots**
|
- **Normalboots**
|
||||||
|
@ -592,6 +634,7 @@
|
||||||
- **Npr**
|
- **Npr**
|
||||||
- **NRK**
|
- **NRK**
|
||||||
- **NRKPlaylist**
|
- **NRKPlaylist**
|
||||||
|
- **NRKRadioPodkast**
|
||||||
- **NRKSkole**: NRK Skole
|
- **NRKSkole**: NRK Skole
|
||||||
- **NRKTV**: NRK TV and NRK Radio
|
- **NRKTV**: NRK TV and NRK Radio
|
||||||
- **NRKTVDirekte**: NRK TV Direkte and NRK Radio Direkte
|
- **NRKTVDirekte**: NRK TV Direkte and NRK Radio Direkte
|
||||||
|
@ -604,6 +647,7 @@
|
||||||
- **Nuvid**
|
- **Nuvid**
|
||||||
- **NYTimes**
|
- **NYTimes**
|
||||||
- **NYTimesArticle**
|
- **NYTimesArticle**
|
||||||
|
- **NYTimesCooking**
|
||||||
- **NZZ**
|
- **NZZ**
|
||||||
- **ocw.mit.edu**
|
- **ocw.mit.edu**
|
||||||
- **OdaTV**
|
- **OdaTV**
|
||||||
|
@ -618,21 +662,33 @@
|
||||||
- **Ooyala**
|
- **Ooyala**
|
||||||
- **OoyalaExternal**
|
- **OoyalaExternal**
|
||||||
- **OraTV**
|
- **OraTV**
|
||||||
|
- **orf:burgenland**: Radio Burgenland
|
||||||
- **orf:fm4**: radio FM4
|
- **orf:fm4**: radio FM4
|
||||||
- **orf:fm4:story**: fm4.orf.at stories
|
- **orf:fm4:story**: fm4.orf.at stories
|
||||||
- **orf:iptv**: iptv.ORF.at
|
- **orf:iptv**: iptv.ORF.at
|
||||||
|
- **orf:kaernten**: Radio Kärnten
|
||||||
|
- **orf:noe**: Radio Niederösterreich
|
||||||
|
- **orf:oberoesterreich**: Radio Oberösterreich
|
||||||
- **orf:oe1**: Radio Österreich 1
|
- **orf:oe1**: Radio Österreich 1
|
||||||
|
- **orf:oe3**: Radio Österreich 3
|
||||||
|
- **orf:salzburg**: Radio Salzburg
|
||||||
|
- **orf:steiermark**: Radio Steiermark
|
||||||
|
- **orf:tirol**: Radio Tirol
|
||||||
- **orf:tvthek**: ORF TVthek
|
- **orf:tvthek**: ORF TVthek
|
||||||
|
- **orf:vorarlberg**: Radio Vorarlberg
|
||||||
|
- **orf:wien**: Radio Wien
|
||||||
- **OsnatelTV**
|
- **OsnatelTV**
|
||||||
- **OutsideTV**
|
- **OutsideTV**
|
||||||
- **PacktPub**
|
- **PacktPub**
|
||||||
- **PacktPubCourse**
|
- **PacktPubCourse**
|
||||||
|
- **PalcoMP3:artist**
|
||||||
|
- **PalcoMP3:song**
|
||||||
|
- **PalcoMP3:video**
|
||||||
- **pandora.tv**: 판도라TV
|
- **pandora.tv**: 판도라TV
|
||||||
- **ParamountNetwork**
|
- **ParamountNetwork**
|
||||||
- **parliamentlive.tv**: UK parliament videos
|
- **parliamentlive.tv**: UK parliament videos
|
||||||
- **Patreon**
|
- **Patreon**
|
||||||
- **pbs**: Public Broadcasting Service (PBS) and member stations: PBS: Public Broadcasting Service, APT - Alabama Public Television (WBIQ), GPB/Georgia Public Broadcasting (WGTV), Mississippi Public Broadcasting (WMPN), Nashville Public Television (WNPT), WFSU-TV (WFSU), WSRE (WSRE), WTCI (WTCI), WPBA/Channel 30 (WPBA), Alaska Public Media (KAKM), Arizona PBS (KAET), KNME-TV/Channel 5 (KNME), Vegas PBS (KLVX), AETN/ARKANSAS ETV NETWORK (KETS), KET (WKLE), WKNO/Channel 10 (WKNO), LPB/LOUISIANA PUBLIC BROADCASTING (WLPB), OETA (KETA), Ozarks Public Television (KOZK), WSIU Public Broadcasting (WSIU), KEET TV (KEET), KIXE/Channel 9 (KIXE), KPBS San Diego (KPBS), KQED (KQED), KVIE Public Television (KVIE), PBS SoCal/KOCE (KOCE), ValleyPBS (KVPT), CONNECTICUT PUBLIC TELEVISION (WEDH), KNPB Channel 5 (KNPB), SOPTV (KSYS), Rocky Mountain PBS (KRMA), KENW-TV3 (KENW), KUED Channel 7 (KUED), Wyoming PBS (KCWC), Colorado Public Television / KBDI 12 (KBDI), KBYU-TV (KBYU), Thirteen/WNET New York (WNET), WGBH/Channel 2 (WGBH), WGBY (WGBY), NJTV Public Media NJ (WNJT), WLIW21 (WLIW), mpt/Maryland Public Television (WMPB), WETA Television and Radio (WETA), WHYY (WHYY), PBS 39 (WLVT), WVPT - Your Source for PBS and More! (WVPT), Howard University Television (WHUT), WEDU PBS (WEDU), WGCU Public Media (WGCU), WPBT2 (WPBT), WUCF TV (WUCF), WUFT/Channel 5 (WUFT), WXEL/Channel 42 (WXEL), WLRN/Channel 17 (WLRN), WUSF Public Broadcasting (WUSF), ETV (WRLK), UNC-TV (WUNC), PBS Hawaii - Oceanic Cable Channel 10 (KHET), Idaho Public Television (KAID), KSPS (KSPS), OPB (KOPB), KWSU/Channel 10 & KTNW/Channel 31 (KWSU), WILL-TV (WILL), Network Knowledge - WSEC/Springfield (WSEC), WTTW11 (WTTW), Iowa Public Television/IPTV (KDIN), Nine Network (KETC), PBS39 Fort Wayne (WFWA), WFYI Indianapolis (WFYI), Milwaukee Public Television (WMVS), WNIN (WNIN), WNIT Public Television (WNIT), WPT (WPNE), WVUT/Channel 22 (WVUT), WEIU/Channel 51 (WEIU), WQPT-TV (WQPT), WYCC PBS Chicago (WYCC), WIPB-TV (WIPB), WTIU (WTIU), CET (WCET), ThinkTVNetwork (WPTD), WBGU-TV (WBGU), WGVU TV (WGVU), NET1 (KUON), Pioneer Public Television (KWCM), SDPB Television (KUSD), TPT (KTCA), KSMQ (KSMQ), KPTS/Channel 8 (KPTS), KTWU/Channel 11 (KTWU), East Tennessee PBS (WSJK), WCTE-TV (WCTE), WLJT, Channel 11 (WLJT), WOSU TV (WOSU), WOUB/WOUC (WOUB), WVPB (WVPB), WKYU-PBS (WKYU), KERA 13 (KERA), MPBN (WCBB), Mountain Lake PBS (WCFE), NHPTV (WENH), Vermont PBS (WETK), witf (WITF), WQED Multimedia (WQED), WMHT Educational Telecommunications (WMHT), Q-TV (WDCQ), WTVS Detroit Public TV (WTVS), CMU Public Television (WCMU), WKAR-TV (WKAR), WNMU-TV Public TV 13 (WNMU), WDSE - WRPT (WDSE), WGTE TV (WGTE), Lakeland Public Television (KAWE), KMOS-TV - Channels 6.1, 6.2 and 6.3 (KMOS), MontanaPBS (KUSM), KRWG/Channel 22 (KRWG), KACV (KACV), KCOS/Channel 13 (KCOS), WCNY/Channel 24 (WCNY), WNED (WNED), WPBS (WPBS), WSKG Public TV (WSKG), WXXI (WXXI), WPSU (WPSU), WVIA Public Media Studios (WVIA), WTVI (WTVI), Western Reserve PBS (WNEO), WVIZ/PBS ideastream (WVIZ), KCTS 9 (KCTS), Basin PBS (KPBT), KUHT / Channel 8 (KUHT), KLRN (KLRN), KLRU (KLRU), WTJX Channel 12 (WTJX), WCVE PBS (WCVE), KBTC Public Television (KBTC)
|
- **pbs**: Public Broadcasting Service (PBS) and member stations: PBS: Public Broadcasting Service, APT - Alabama Public Television (WBIQ), GPB/Georgia Public Broadcasting (WGTV), Mississippi Public Broadcasting (WMPN), Nashville Public Television (WNPT), WFSU-TV (WFSU), WSRE (WSRE), WTCI (WTCI), WPBA/Channel 30 (WPBA), Alaska Public Media (KAKM), Arizona PBS (KAET), KNME-TV/Channel 5 (KNME), Vegas PBS (KLVX), AETN/ARKANSAS ETV NETWORK (KETS), KET (WKLE), WKNO/Channel 10 (WKNO), LPB/LOUISIANA PUBLIC BROADCASTING (WLPB), OETA (KETA), Ozarks Public Television (KOZK), WSIU Public Broadcasting (WSIU), KEET TV (KEET), KIXE/Channel 9 (KIXE), KPBS San Diego (KPBS), KQED (KQED), KVIE Public Television (KVIE), PBS SoCal/KOCE (KOCE), ValleyPBS (KVPT), CONNECTICUT PUBLIC TELEVISION (WEDH), KNPB Channel 5 (KNPB), SOPTV (KSYS), Rocky Mountain PBS (KRMA), KENW-TV3 (KENW), KUED Channel 7 (KUED), Wyoming PBS (KCWC), Colorado Public Television / KBDI 12 (KBDI), KBYU-TV (KBYU), Thirteen/WNET New York (WNET), WGBH/Channel 2 (WGBH), WGBY (WGBY), NJTV Public Media NJ (WNJT), WLIW21 (WLIW), mpt/Maryland Public Television (WMPB), WETA Television and Radio (WETA), WHYY (WHYY), PBS 39 (WLVT), WVPT - Your Source for PBS and More! (WVPT), Howard University Television (WHUT), WEDU PBS (WEDU), WGCU Public Media (WGCU), WPBT2 (WPBT), WUCF TV (WUCF), WUFT/Channel 5 (WUFT), WXEL/Channel 42 (WXEL), WLRN/Channel 17 (WLRN), WUSF Public Broadcasting (WUSF), ETV (WRLK), UNC-TV (WUNC), PBS Hawaii - Oceanic Cable Channel 10 (KHET), Idaho Public Television (KAID), KSPS (KSPS), OPB (KOPB), KWSU/Channel 10 & KTNW/Channel 31 (KWSU), WILL-TV (WILL), Network Knowledge - WSEC/Springfield (WSEC), WTTW11 (WTTW), Iowa Public Television/IPTV (KDIN), Nine Network (KETC), PBS39 Fort Wayne (WFWA), WFYI Indianapolis (WFYI), Milwaukee Public Television (WMVS), WNIN (WNIN), WNIT Public Television (WNIT), WPT (WPNE), WVUT/Channel 22 (WVUT), WEIU/Channel 51 (WEIU), WQPT-TV (WQPT), WYCC PBS Chicago (WYCC), WIPB-TV (WIPB), WTIU (WTIU), CET (WCET), ThinkTVNetwork (WPTD), WBGU-TV (WBGU), WGVU TV (WGVU), NET1 (KUON), Pioneer Public Television (KWCM), SDPB Television (KUSD), TPT (KTCA), KSMQ (KSMQ), KPTS/Channel 8 (KPTS), KTWU/Channel 11 (KTWU), East Tennessee PBS (WSJK), WCTE-TV (WCTE), WLJT, Channel 11 (WLJT), WOSU TV (WOSU), WOUB/WOUC (WOUB), WVPB (WVPB), WKYU-PBS (WKYU), KERA 13 (KERA), MPBN (WCBB), Mountain Lake PBS (WCFE), NHPTV (WENH), Vermont PBS (WETK), witf (WITF), WQED Multimedia (WQED), WMHT Educational Telecommunications (WMHT), Q-TV (WDCQ), WTVS Detroit Public TV (WTVS), CMU Public Television (WCMU), WKAR-TV (WKAR), WNMU-TV Public TV 13 (WNMU), WDSE - WRPT (WDSE), WGTE TV (WGTE), Lakeland Public Television (KAWE), KMOS-TV - Channels 6.1, 6.2 and 6.3 (KMOS), MontanaPBS (KUSM), KRWG/Channel 22 (KRWG), KACV (KACV), KCOS/Channel 13 (KCOS), WCNY/Channel 24 (WCNY), WNED (WNED), WPBS (WPBS), WSKG Public TV (WSKG), WXXI (WXXI), WPSU (WPSU), WVIA Public Media Studios (WVIA), WTVI (WTVI), Western Reserve PBS (WNEO), WVIZ/PBS ideastream (WVIZ), KCTS 9 (KCTS), Basin PBS (KPBT), KUHT / Channel 8 (KUHT), KLRN (KLRN), KLRU (KLRU), WTJX Channel 12 (WTJX), WCVE PBS (WCVE), KBTC Public Television (KBTC)
|
||||||
- **pcmag**
|
|
||||||
- **PearVideo**
|
- **PearVideo**
|
||||||
- **PeerTube**
|
- **PeerTube**
|
||||||
- **People**
|
- **People**
|
||||||
|
@ -646,18 +702,21 @@
|
||||||
- **PicartoVod**
|
- **PicartoVod**
|
||||||
- **Piksel**
|
- **Piksel**
|
||||||
- **Pinkbike**
|
- **Pinkbike**
|
||||||
|
- **Pinterest**
|
||||||
|
- **PinterestCollection**
|
||||||
- **Pladform**
|
- **Pladform**
|
||||||
- **Platzi**
|
- **Platzi**
|
||||||
- **PlatziCourse**
|
- **PlatziCourse**
|
||||||
- **play.fm**
|
- **play.fm**
|
||||||
|
- **player.sky.it**
|
||||||
- **PlayPlusTV**
|
- **PlayPlusTV**
|
||||||
|
- **PlayStuff**
|
||||||
- **PlaysTV**
|
- **PlaysTV**
|
||||||
- **Playtvak**: Playtvak.cz, iDNES.cz and Lidovky.cz
|
- **Playtvak**: Playtvak.cz, iDNES.cz and Lidovky.cz
|
||||||
- **Playvid**
|
- **Playvid**
|
||||||
- **Playwire**
|
- **Playwire**
|
||||||
- **pluralsight**
|
- **pluralsight**
|
||||||
- **pluralsight:course**
|
- **pluralsight:course**
|
||||||
- **plus.google**: Google Plus
|
|
||||||
- **podomatic**
|
- **podomatic**
|
||||||
- **Pokemon**
|
- **Pokemon**
|
||||||
- **PolskieRadio**
|
- **PolskieRadio**
|
||||||
|
@ -687,6 +746,7 @@
|
||||||
- **qqmusic:singer**: QQ音乐 - 歌手
|
- **qqmusic:singer**: QQ音乐 - 歌手
|
||||||
- **qqmusic:toplist**: QQ音乐 - 排行榜
|
- **qqmusic:toplist**: QQ音乐 - 排行榜
|
||||||
- **QuantumTV**
|
- **QuantumTV**
|
||||||
|
- **Qub**
|
||||||
- **Quickline**
|
- **Quickline**
|
||||||
- **QuicklineLive**
|
- **QuicklineLive**
|
||||||
- **R7**
|
- **R7**
|
||||||
|
@ -705,6 +765,8 @@
|
||||||
- **RayWenderlichCourse**
|
- **RayWenderlichCourse**
|
||||||
- **RBMARadio**
|
- **RBMARadio**
|
||||||
- **RDS**: RDS.ca
|
- **RDS**: RDS.ca
|
||||||
|
- **RedBull**
|
||||||
|
- **RedBullEmbed**
|
||||||
- **RedBullTV**
|
- **RedBullTV**
|
||||||
- **RedBullTVRrnContent**
|
- **RedBullTVRrnContent**
|
||||||
- **Reddit**
|
- **Reddit**
|
||||||
|
@ -739,6 +801,7 @@
|
||||||
- **RTVNH**
|
- **RTVNH**
|
||||||
- **RTVS**
|
- **RTVS**
|
||||||
- **RUHD**
|
- **RUHD**
|
||||||
|
- **RumbleEmbed**
|
||||||
- **rutube**: Rutube videos
|
- **rutube**: Rutube videos
|
||||||
- **rutube:channel**: Rutube channels
|
- **rutube:channel**: Rutube channels
|
||||||
- **rutube:embed**: Rutube embedded videos
|
- **rutube:embed**: Rutube embedded videos
|
||||||
|
@ -753,6 +816,7 @@
|
||||||
- **safari:course**: safaribooksonline.com online courses
|
- **safari:course**: safaribooksonline.com online courses
|
||||||
- **SAKTV**
|
- **SAKTV**
|
||||||
- **SaltTV**
|
- **SaltTV**
|
||||||
|
- **SampleFocus**
|
||||||
- **Sapo**: SAPO Vídeos
|
- **Sapo**: SAPO Vídeos
|
||||||
- **savefrom.net**
|
- **savefrom.net**
|
||||||
- **SBS**: sbs.com.au
|
- **SBS**: sbs.com.au
|
||||||
|
@ -775,19 +839,21 @@
|
||||||
- **ShahidShow**
|
- **ShahidShow**
|
||||||
- **Shared**: shared.sx
|
- **Shared**: shared.sx
|
||||||
- **ShowRoomLive**
|
- **ShowRoomLive**
|
||||||
|
- **simplecast**
|
||||||
|
- **simplecast:episode**
|
||||||
|
- **simplecast:podcast**
|
||||||
- **Sina**
|
- **Sina**
|
||||||
|
- **sky.it**
|
||||||
|
- **sky:news**
|
||||||
|
- **sky:sports**
|
||||||
|
- **sky:sports:news**
|
||||||
|
- **skyacademy.it**
|
||||||
- **SkylineWebcams**
|
- **SkylineWebcams**
|
||||||
- **SkyNews**
|
|
||||||
- **skynewsarabia:article**
|
- **skynewsarabia:article**
|
||||||
- **skynewsarabia:video**
|
- **skynewsarabia:video**
|
||||||
- **SkySports**
|
|
||||||
- **Slideshare**
|
- **Slideshare**
|
||||||
- **SlidesLive**
|
- **SlidesLive**
|
||||||
- **Slutload**
|
- **Slutload**
|
||||||
- **smotri**: Smotri.com
|
|
||||||
- **smotri:broadcast**: Smotri.com broadcasts
|
|
||||||
- **smotri:community**: Smotri.com community videos
|
|
||||||
- **smotri:user**: Smotri.com user videos
|
|
||||||
- **Snotr**
|
- **Snotr**
|
||||||
- **Sohu**
|
- **Sohu**
|
||||||
- **SonyLIV**
|
- **SonyLIV**
|
||||||
|
@ -809,12 +875,16 @@
|
||||||
- **SpankBangPlaylist**
|
- **SpankBangPlaylist**
|
||||||
- **Spankwire**
|
- **Spankwire**
|
||||||
- **Spiegel**
|
- **Spiegel**
|
||||||
- **Spiegel:Article**: Articles on spiegel.de
|
|
||||||
- **Spiegeltv**
|
|
||||||
- **sport.francetvinfo.fr**
|
- **sport.francetvinfo.fr**
|
||||||
- **Sport5**
|
- **Sport5**
|
||||||
- **SportBox**
|
- **SportBox**
|
||||||
- **SportDeutschland**
|
- **SportDeutschland**
|
||||||
|
- **spotify**
|
||||||
|
- **spotify:show**
|
||||||
|
- **Spreaker**
|
||||||
|
- **SpreakerPage**
|
||||||
|
- **SpreakerShow**
|
||||||
|
- **SpreakerShowPage**
|
||||||
- **SpringboardPlatform**
|
- **SpringboardPlatform**
|
||||||
- **Sprout**
|
- **Sprout**
|
||||||
- **sr:mediathek**: Saarländischer Rundfunk
|
- **sr:mediathek**: Saarländischer Rundfunk
|
||||||
|
@ -823,6 +893,10 @@
|
||||||
- **stanfordoc**: Stanford Open ClassRoom
|
- **stanfordoc**: Stanford Open ClassRoom
|
||||||
- **Steam**
|
- **Steam**
|
||||||
- **Stitcher**
|
- **Stitcher**
|
||||||
|
- **StitcherShow**
|
||||||
|
- **StoryFire**
|
||||||
|
- **StoryFireSeries**
|
||||||
|
- **StoryFireUser**
|
||||||
- **Streamable**
|
- **Streamable**
|
||||||
- **streamcloud.eu**
|
- **streamcloud.eu**
|
||||||
- **StreamCZ**
|
- **StreamCZ**
|
||||||
|
@ -843,7 +917,6 @@
|
||||||
- **Tagesschau**
|
- **Tagesschau**
|
||||||
- **tagesschau:player**
|
- **tagesschau:player**
|
||||||
- **Tass**
|
- **Tass**
|
||||||
- **TastyTrade**
|
|
||||||
- **TBS**
|
- **TBS**
|
||||||
- **TDSLifeway**
|
- **TDSLifeway**
|
||||||
- **Teachable**
|
- **Teachable**
|
||||||
|
@ -866,6 +939,7 @@
|
||||||
- **TeleQuebecEmission**
|
- **TeleQuebecEmission**
|
||||||
- **TeleQuebecLive**
|
- **TeleQuebecLive**
|
||||||
- **TeleQuebecSquat**
|
- **TeleQuebecSquat**
|
||||||
|
- **TeleQuebecVideo**
|
||||||
- **TeleTask**
|
- **TeleTask**
|
||||||
- **Telewebion**
|
- **Telewebion**
|
||||||
- **TennisTV**
|
- **TennisTV**
|
||||||
|
@ -883,7 +957,7 @@
|
||||||
- **ThisAV**
|
- **ThisAV**
|
||||||
- **ThisOldHouse**
|
- **ThisOldHouse**
|
||||||
- **TikTok**
|
- **TikTok**
|
||||||
- **TikTokUser**
|
- **TikTokUser** (Currently broken)
|
||||||
- **tinypic**: tinypic.com videos
|
- **tinypic**: tinypic.com videos
|
||||||
- **TMZ**
|
- **TMZ**
|
||||||
- **TMZArticle**
|
- **TMZArticle**
|
||||||
|
@ -891,12 +965,13 @@
|
||||||
- **TNAFlixNetworkEmbed**
|
- **TNAFlixNetworkEmbed**
|
||||||
- **toggle**
|
- **toggle**
|
||||||
- **ToonGoggles**
|
- **ToonGoggles**
|
||||||
- **Tosh**: Tosh.0
|
|
||||||
- **tou.tv**
|
- **tou.tv**
|
||||||
- **Toypics**: Toypics video
|
- **Toypics**: Toypics video
|
||||||
- **ToypicsUser**: Toypics user profile
|
- **ToypicsUser**: Toypics user profile
|
||||||
- **TrailerAddict** (Currently broken)
|
- **TrailerAddict** (Currently broken)
|
||||||
- **Trilulilu**
|
- **Trilulilu**
|
||||||
|
- **Trovo**
|
||||||
|
- **TrovoVod**
|
||||||
- **TruNews**
|
- **TruNews**
|
||||||
- **TruTV**
|
- **TruTV**
|
||||||
- **Tube8**
|
- **Tube8**
|
||||||
|
@ -916,11 +991,15 @@
|
||||||
- **TV2DKBornholmPlay**
|
- **TV2DKBornholmPlay**
|
||||||
- **TV4**: tv4.se and tv4play.se
|
- **TV4**: tv4.se and tv4play.se
|
||||||
- **TV5MondePlus**: TV5MONDE+
|
- **TV5MondePlus**: TV5MONDE+
|
||||||
|
- **tv5unis**
|
||||||
|
- **tv5unis:video**
|
||||||
|
- **tv8.it**
|
||||||
- **TVA**
|
- **TVA**
|
||||||
- **TVANouvelles**
|
- **TVANouvelles**
|
||||||
- **TVANouvellesArticle**
|
- **TVANouvellesArticle**
|
||||||
- **TVC**
|
- **TVC**
|
||||||
- **TVCArticle**
|
- **TVCArticle**
|
||||||
|
- **TVer**
|
||||||
- **tvigle**: Интернет-телевидение Tvigle.ru
|
- **tvigle**: Интернет-телевидение Tvigle.ru
|
||||||
- **tvland.com**
|
- **tvland.com**
|
||||||
- **TVN24**
|
- **TVN24**
|
||||||
|
@ -938,16 +1017,13 @@
|
||||||
- **TVPlayHome**
|
- **TVPlayHome**
|
||||||
- **Tweakers**
|
- **Tweakers**
|
||||||
- **TwitCasting**
|
- **TwitCasting**
|
||||||
- **twitch:chapter**
|
|
||||||
- **twitch:clips**
|
- **twitch:clips**
|
||||||
- **twitch:profile**
|
|
||||||
- **twitch:stream**
|
- **twitch:stream**
|
||||||
- **twitch:video**
|
|
||||||
- **twitch:videos:all**
|
|
||||||
- **twitch:videos:highlights**
|
|
||||||
- **twitch:videos:past-broadcasts**
|
|
||||||
- **twitch:videos:uploads**
|
|
||||||
- **twitch:vod**
|
- **twitch:vod**
|
||||||
|
- **TwitchCollection**
|
||||||
|
- **TwitchVideos**
|
||||||
|
- **TwitchVideosClips**
|
||||||
|
- **TwitchVideosCollections**
|
||||||
- **twitter**
|
- **twitter**
|
||||||
- **twitter:amplify**
|
- **twitter:amplify**
|
||||||
- **twitter:broadcast**
|
- **twitter:broadcast**
|
||||||
|
@ -989,7 +1065,10 @@
|
||||||
- **Vidbit**
|
- **Vidbit**
|
||||||
- **Viddler**
|
- **Viddler**
|
||||||
- **Videa**
|
- **Videa**
|
||||||
|
- **video.arnes.si**: Arnes Video
|
||||||
- **video.google:search**: Google Video search
|
- **video.google:search**: Google Video search
|
||||||
|
- **video.sky.it**
|
||||||
|
- **video.sky.it:live**
|
||||||
- **VideoDetective**
|
- **VideoDetective**
|
||||||
- **videofy.me**
|
- **videofy.me**
|
||||||
- **videomore**
|
- **videomore**
|
||||||
|
@ -1001,7 +1080,6 @@
|
||||||
- **vidme**
|
- **vidme**
|
||||||
- **vidme:user**
|
- **vidme:user**
|
||||||
- **vidme:user:likes**
|
- **vidme:user:likes**
|
||||||
- **Vidzi**
|
|
||||||
- **vier**: vier.be and vijf.be
|
- **vier**: vier.be and vijf.be
|
||||||
- **vier:videos**
|
- **vier:videos**
|
||||||
- **viewlift**
|
- **viewlift**
|
||||||
|
@ -1031,7 +1109,7 @@
|
||||||
- **vk:wallpost**
|
- **vk:wallpost**
|
||||||
- **vlive**
|
- **vlive**
|
||||||
- **vlive:channel**
|
- **vlive:channel**
|
||||||
- **vlive:playlist**
|
- **vlive:post**
|
||||||
- **Vodlocker**
|
- **Vodlocker**
|
||||||
- **VODPl**
|
- **VODPl**
|
||||||
- **VODPlatform**
|
- **VODPlatform**
|
||||||
|
@ -1046,10 +1124,12 @@
|
||||||
- **vrv**
|
- **vrv**
|
||||||
- **vrv:series**
|
- **vrv:series**
|
||||||
- **VShare**
|
- **VShare**
|
||||||
|
- **VTM**
|
||||||
- **VTXTV**
|
- **VTXTV**
|
||||||
- **vube**: Vube.com
|
- **vube**: Vube.com
|
||||||
- **VuClip**
|
- **VuClip**
|
||||||
- **VVVVID**
|
- **VVVVID**
|
||||||
|
- **VVVVIDShow**
|
||||||
- **VyboryMos**
|
- **VyboryMos**
|
||||||
- **Vzaar**
|
- **Vzaar**
|
||||||
- **Wakanim**
|
- **Wakanim**
|
||||||
|
@ -1072,6 +1152,7 @@
|
||||||
- **WeiboMobile**
|
- **WeiboMobile**
|
||||||
- **WeiqiTV**: WQTV
|
- **WeiqiTV**: WQTV
|
||||||
- **Wistia**
|
- **Wistia**
|
||||||
|
- **WistiaPlaylist**
|
||||||
- **wnl**: npo.nl, ntr.nl, omroepwnl.nl, zapp.nl and npo3.nl
|
- **wnl**: npo.nl, ntr.nl, omroepwnl.nl, zapp.nl and npo3.nl
|
||||||
- **WorldStarHipHop**
|
- **WorldStarHipHop**
|
||||||
- **WSJ**: Wall Street Journal
|
- **WSJ**: Wall Street Journal
|
||||||
|
@ -1079,7 +1160,7 @@
|
||||||
- **WWE**
|
- **WWE**
|
||||||
- **XBef**
|
- **XBef**
|
||||||
- **XboxClips**
|
- **XboxClips**
|
||||||
- **XFileShare**: XFileShare based sites: ClipWatching, GoUnlimited, GoVid, HolaVid, Streamty, TheVideoBee, Uqload, VidBom, vidlo, VidLocker, VidShare, VUp, XVideoSharing
|
- **XFileShare**: XFileShare based sites: Aparat, ClipWatching, GoUnlimited, GoVid, HolaVid, Streamty, TheVideoBee, Uqload, VidBom, vidlo, VidLocker, VidShare, VUp, WolfStream, XVideoSharing
|
||||||
- **XHamster**
|
- **XHamster**
|
||||||
- **XHamsterEmbed**
|
- **XHamsterEmbed**
|
||||||
- **XHamsterUser**
|
- **XHamsterUser**
|
||||||
|
@ -1103,6 +1184,8 @@
|
||||||
- **yahoo:japannews**: Yahoo! Japan News
|
- **yahoo:japannews**: Yahoo! Japan News
|
||||||
- **YandexDisk**
|
- **YandexDisk**
|
||||||
- **yandexmusic:album**: Яндекс.Музыка - Альбом
|
- **yandexmusic:album**: Яндекс.Музыка - Альбом
|
||||||
|
- **yandexmusic:artist:albums**: Яндекс.Музыка - Артист - Альбомы
|
||||||
|
- **yandexmusic:artist:tracks**: Яндекс.Музыка - Артист - Треки
|
||||||
- **yandexmusic:playlist**: Яндекс.Музыка - Плейлист
|
- **yandexmusic:playlist**: Яндекс.Музыка - Плейлист
|
||||||
- **yandexmusic:track**: Яндекс.Музыка - Трек
|
- **yandexmusic:track**: Яндекс.Музыка - Трек
|
||||||
- **YandexVideo**
|
- **YandexVideo**
|
||||||
|
@ -1120,25 +1203,24 @@
|
||||||
- **YourPorn**
|
- **YourPorn**
|
||||||
- **YourUpload**
|
- **YourUpload**
|
||||||
- **youtube**: YouTube.com
|
- **youtube**: YouTube.com
|
||||||
- **youtube:channel**: YouTube.com channels
|
|
||||||
- **youtube:favorites**: YouTube.com favourite videos, ":ytfav" for short (requires authentication)
|
- **youtube:favorites**: YouTube.com favourite videos, ":ytfav" for short (requires authentication)
|
||||||
- **youtube:history**: Youtube watch history, ":ythistory" for short (requires authentication)
|
- **youtube:history**: Youtube watch history, ":ythistory" for short (requires authentication)
|
||||||
- **youtube:live**: YouTube.com live streams
|
|
||||||
- **youtube:playlist**: YouTube.com playlists
|
- **youtube:playlist**: YouTube.com playlists
|
||||||
- **youtube:playlists**: YouTube.com user/channel playlists
|
|
||||||
- **youtube:recommended**: YouTube.com recommended videos, ":ytrec" for short (requires authentication)
|
- **youtube:recommended**: YouTube.com recommended videos, ":ytrec" for short (requires authentication)
|
||||||
- **youtube:search**: YouTube.com searches
|
- **youtube:search**: YouTube.com searches
|
||||||
- **youtube:search:date**: YouTube.com searches, newest videos first
|
- **youtube:search:date**: YouTube.com searches, newest videos first
|
||||||
- **youtube:search_url**: YouTube.com search URLs
|
|
||||||
- **youtube:show**: YouTube.com (multi-season) shows
|
|
||||||
- **youtube:subscriptions**: YouTube.com subscriptions feed, "ytsubs" keyword (requires authentication)
|
- **youtube:subscriptions**: YouTube.com subscriptions feed, "ytsubs" keyword (requires authentication)
|
||||||
- **youtube:user**: YouTube.com user videos (URL or "ytuser" keyword)
|
- **youtube:tab**: YouTube.com tab
|
||||||
- **youtube:watchlater**: Youtube watch later list, ":ytwatchlater" for short (requires authentication)
|
- **youtube:watchlater**: Youtube watch later list, ":ytwatchlater" for short (requires authentication)
|
||||||
|
- **YoutubeYtBe**
|
||||||
|
- **YoutubeYtUser**
|
||||||
- **Zapiks**
|
- **Zapiks**
|
||||||
- **Zaq1**
|
|
||||||
- **Zattoo**
|
- **Zattoo**
|
||||||
- **ZattooLive**
|
- **ZattooLive**
|
||||||
- **ZDF**
|
- **ZDF**
|
||||||
- **ZDFChannel**
|
- **ZDFChannel**
|
||||||
|
- **Zhihu**
|
||||||
- **zingmp3**: mp3.zing.vn
|
- **zingmp3**: mp3.zing.vn
|
||||||
|
- **zingmp3:album**
|
||||||
|
- **zoom**
|
||||||
- **Zype**
|
- **Zype**
|
||||||
|
|
|
@ -1,22 +1,24 @@
|
||||||
from __future__ import unicode_literals
|
from __future__ import unicode_literals
|
||||||
|
|
||||||
import errno
|
import errno
|
||||||
import io
|
|
||||||
import hashlib
|
import hashlib
|
||||||
import json
|
import json
|
||||||
import os.path
|
import os.path
|
||||||
import re
|
import re
|
||||||
import types
|
|
||||||
import ssl
|
import ssl
|
||||||
import sys
|
import sys
|
||||||
|
import types
|
||||||
|
import unittest
|
||||||
|
|
||||||
import youtube_dl.extractor
|
import youtube_dl.extractor
|
||||||
from youtube_dl import YoutubeDL
|
from youtube_dl import YoutubeDL
|
||||||
from youtube_dl.compat import (
|
from youtube_dl.compat import (
|
||||||
|
compat_open as open,
|
||||||
compat_os_name,
|
compat_os_name,
|
||||||
compat_str,
|
compat_str,
|
||||||
)
|
)
|
||||||
from youtube_dl.utils import (
|
from youtube_dl.utils import (
|
||||||
|
IDENTITY,
|
||||||
preferredencoding,
|
preferredencoding,
|
||||||
write_string,
|
write_string,
|
||||||
)
|
)
|
||||||
|
@ -27,10 +29,10 @@ def get_params(override=None):
|
||||||
"parameters.json")
|
"parameters.json")
|
||||||
LOCAL_PARAMETERS_FILE = os.path.join(os.path.dirname(os.path.abspath(__file__)),
|
LOCAL_PARAMETERS_FILE = os.path.join(os.path.dirname(os.path.abspath(__file__)),
|
||||||
"local_parameters.json")
|
"local_parameters.json")
|
||||||
with io.open(PARAMETERS_FILE, encoding='utf-8') as pf:
|
with open(PARAMETERS_FILE, encoding='utf-8') as pf:
|
||||||
parameters = json.load(pf)
|
parameters = json.load(pf)
|
||||||
if os.path.exists(LOCAL_PARAMETERS_FILE):
|
if os.path.exists(LOCAL_PARAMETERS_FILE):
|
||||||
with io.open(LOCAL_PARAMETERS_FILE, encoding='utf-8') as pf:
|
with open(LOCAL_PARAMETERS_FILE, encoding='utf-8') as pf:
|
||||||
parameters.update(json.load(pf))
|
parameters.update(json.load(pf))
|
||||||
if override:
|
if override:
|
||||||
parameters.update(override)
|
parameters.update(override)
|
||||||
|
@ -72,7 +74,8 @@ class FakeYDL(YoutubeDL):
|
||||||
def to_screen(self, s, skip_eol=None):
|
def to_screen(self, s, skip_eol=None):
|
||||||
print(s)
|
print(s)
|
||||||
|
|
||||||
def trouble(self, s, tb=None):
|
def trouble(self, *args, **kwargs):
|
||||||
|
s = args[0] if len(args) > 0 else kwargs.get('message', 'Missing message')
|
||||||
raise Exception(s)
|
raise Exception(s)
|
||||||
|
|
||||||
def download(self, x):
|
def download(self, x):
|
||||||
|
@ -89,6 +92,17 @@ class FakeYDL(YoutubeDL):
|
||||||
self.report_warning = types.MethodType(report_warning, self)
|
self.report_warning = types.MethodType(report_warning, self)
|
||||||
|
|
||||||
|
|
||||||
|
class FakeLogger(object):
|
||||||
|
def debug(self, msg):
|
||||||
|
pass
|
||||||
|
|
||||||
|
def warning(self, msg):
|
||||||
|
pass
|
||||||
|
|
||||||
|
def error(self, msg):
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
def gettestcases(include_onlymatching=False):
|
def gettestcases(include_onlymatching=False):
|
||||||
for ie in youtube_dl.extractor.gen_extractors():
|
for ie in youtube_dl.extractor.gen_extractors():
|
||||||
for tc in ie.get_testcases(include_onlymatching):
|
for tc in ie.get_testcases(include_onlymatching):
|
||||||
|
@ -128,6 +142,12 @@ def expect_value(self, got, expected, field):
|
||||||
self.assertTrue(
|
self.assertTrue(
|
||||||
contains_str in got,
|
contains_str in got,
|
||||||
'field %s (value: %r) should contain %r' % (field, got, contains_str))
|
'field %s (value: %r) should contain %r' % (field, got, contains_str))
|
||||||
|
elif isinstance(expected, compat_str) and re.match(r'lambda \w+:', expected):
|
||||||
|
fn = eval(expected)
|
||||||
|
suite = expected.split(':', 1)[1].strip()
|
||||||
|
self.assertTrue(
|
||||||
|
fn(got),
|
||||||
|
'Expected field %s to meet condition %s, but value %r failed ' % (field, suite, got))
|
||||||
elif isinstance(expected, type):
|
elif isinstance(expected, type):
|
||||||
self.assertTrue(
|
self.assertTrue(
|
||||||
isinstance(got, expected),
|
isinstance(got, expected),
|
||||||
|
@ -137,7 +157,7 @@ def expect_value(self, got, expected, field):
|
||||||
elif isinstance(expected, list) and isinstance(got, list):
|
elif isinstance(expected, list) and isinstance(got, list):
|
||||||
self.assertEqual(
|
self.assertEqual(
|
||||||
len(expected), len(got),
|
len(expected), len(got),
|
||||||
'Expect a list of length %d, but got a list of length %d for field %s' % (
|
'Expected a list of length %d, but got a list of length %d for field %s' % (
|
||||||
len(expected), len(got), field))
|
len(expected), len(got), field))
|
||||||
for index, (item_got, item_expected) in enumerate(zip(got, expected)):
|
for index, (item_got, item_expected) in enumerate(zip(got, expected)):
|
||||||
type_got = type(item_got)
|
type_got = type(item_got)
|
||||||
|
@ -161,18 +181,18 @@ def expect_value(self, got, expected, field):
|
||||||
op, _, expected_num = expected.partition(':')
|
op, _, expected_num = expected.partition(':')
|
||||||
expected_num = int(expected_num)
|
expected_num = int(expected_num)
|
||||||
if op == 'mincount':
|
if op == 'mincount':
|
||||||
assert_func = assertGreaterEqual
|
assert_func = self.assertGreaterEqual
|
||||||
msg_tmpl = 'Expected %d items in field %s, but only got %d'
|
msg_tmpl = 'Expected %d items in field %s, but only got %d'
|
||||||
elif op == 'maxcount':
|
elif op == 'maxcount':
|
||||||
assert_func = assertLessEqual
|
assert_func = self.assertLessEqual
|
||||||
msg_tmpl = 'Expected maximum %d items in field %s, but got %d'
|
msg_tmpl = 'Expected maximum %d items in field %s, but got %d'
|
||||||
elif op == 'count':
|
elif op == 'count':
|
||||||
assert_func = assertEqual
|
assert_func = self.assertEqual
|
||||||
msg_tmpl = 'Expected exactly %d items in field %s, but got %d'
|
msg_tmpl = 'Expected exactly %d items in field %s, but got %d'
|
||||||
else:
|
else:
|
||||||
assert False
|
assert False
|
||||||
assert_func(
|
assert_func(
|
||||||
self, len(got), expected_num,
|
len(got), expected_num,
|
||||||
msg_tmpl % (expected_num, field, len(got)))
|
msg_tmpl % (expected_num, field, len(got)))
|
||||||
return
|
return
|
||||||
self.assertEqual(
|
self.assertEqual(
|
||||||
|
@ -242,27 +262,6 @@ def assertRegexpMatches(self, text, regexp, msg=None):
|
||||||
self.assertTrue(m, msg)
|
self.assertTrue(m, msg)
|
||||||
|
|
||||||
|
|
||||||
def assertGreaterEqual(self, got, expected, msg=None):
|
|
||||||
if not (got >= expected):
|
|
||||||
if msg is None:
|
|
||||||
msg = '%r not greater than or equal to %r' % (got, expected)
|
|
||||||
self.assertTrue(got >= expected, msg)
|
|
||||||
|
|
||||||
|
|
||||||
def assertLessEqual(self, got, expected, msg=None):
|
|
||||||
if not (got <= expected):
|
|
||||||
if msg is None:
|
|
||||||
msg = '%r not less than or equal to %r' % (got, expected)
|
|
||||||
self.assertTrue(got <= expected, msg)
|
|
||||||
|
|
||||||
|
|
||||||
def assertEqual(self, got, expected, msg=None):
|
|
||||||
if not (got == expected):
|
|
||||||
if msg is None:
|
|
||||||
msg = '%r not equal to %r' % (got, expected)
|
|
||||||
self.assertTrue(got == expected, msg)
|
|
||||||
|
|
||||||
|
|
||||||
def expect_warnings(ydl, warnings_re):
|
def expect_warnings(ydl, warnings_re):
|
||||||
real_warning = ydl.report_warning
|
real_warning = ydl.report_warning
|
||||||
|
|
||||||
|
@ -280,3 +279,7 @@ def http_server_port(httpd):
|
||||||
else:
|
else:
|
||||||
sock = httpd.socket
|
sock = httpd.socket
|
||||||
return sock.getsockname()[1]
|
return sock.getsockname()[1]
|
||||||
|
|
||||||
|
|
||||||
|
def expectedFailureIf(cond):
|
||||||
|
return unittest.expectedFailure if cond else IDENTITY
|
||||||
|
|
|
@ -18,7 +18,6 @@
|
||||||
"noprogress": false,
|
"noprogress": false,
|
||||||
"outtmpl": "%(id)s.%(ext)s",
|
"outtmpl": "%(id)s.%(ext)s",
|
||||||
"password": null,
|
"password": null,
|
||||||
"playlistend": -1,
|
|
||||||
"playliststart": 1,
|
"playliststart": 1,
|
||||||
"prefer_free_formats": false,
|
"prefer_free_formats": false,
|
||||||
"quiet": false,
|
"quiet": false,
|
||||||
|
@ -37,7 +36,7 @@
|
||||||
"writeinfojson": true,
|
"writeinfojson": true,
|
||||||
"writesubtitles": false,
|
"writesubtitles": false,
|
||||||
"allsubtitles": false,
|
"allsubtitles": false,
|
||||||
"listssubtitles": false,
|
"listsubtitles": false,
|
||||||
"socket_timeout": 20,
|
"socket_timeout": 20,
|
||||||
"fixup": "never"
|
"fixup": "never"
|
||||||
}
|
}
|
||||||
|
|
|
@ -3,19 +3,37 @@
|
||||||
from __future__ import unicode_literals
|
from __future__ import unicode_literals
|
||||||
|
|
||||||
# Allow direct execution
|
# Allow direct execution
|
||||||
import io
|
|
||||||
import os
|
import os
|
||||||
import sys
|
import sys
|
||||||
import unittest
|
import unittest
|
||||||
|
|
||||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||||
|
|
||||||
from test.helper import FakeYDL, expect_dict, expect_value, http_server_port
|
|
||||||
from youtube_dl.compat import compat_etree_fromstring, compat_http_server
|
|
||||||
from youtube_dl.extractor.common import InfoExtractor
|
|
||||||
from youtube_dl.extractor import YoutubeIE, get_info_extractor
|
|
||||||
from youtube_dl.utils import encode_data_uri, strip_jsonp, ExtractorError, RegexNotFoundError
|
|
||||||
import threading
|
import threading
|
||||||
|
|
||||||
|
from test.helper import (
|
||||||
|
expect_dict,
|
||||||
|
expect_value,
|
||||||
|
FakeYDL,
|
||||||
|
http_server_port,
|
||||||
|
)
|
||||||
|
from youtube_dl.compat import (
|
||||||
|
compat_etree_fromstring,
|
||||||
|
compat_http_server,
|
||||||
|
compat_open as open,
|
||||||
|
)
|
||||||
|
from youtube_dl.extractor.common import InfoExtractor
|
||||||
|
from youtube_dl.extractor import (
|
||||||
|
get_info_extractor,
|
||||||
|
YoutubeIE,
|
||||||
|
)
|
||||||
|
from youtube_dl.utils import (
|
||||||
|
encode_data_uri,
|
||||||
|
ExtractorError,
|
||||||
|
RegexNotFoundError,
|
||||||
|
strip_jsonp,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
TEAPOT_RESPONSE_STATUS = 418
|
TEAPOT_RESPONSE_STATUS = 418
|
||||||
TEAPOT_RESPONSE_BODY = "<h1>418 I'm a teapot</h1>"
|
TEAPOT_RESPONSE_BODY = "<h1>418 I'm a teapot</h1>"
|
||||||
|
@ -35,13 +53,13 @@ class InfoExtractorTestRequestHandler(compat_http_server.BaseHTTPRequestHandler)
|
||||||
assert False
|
assert False
|
||||||
|
|
||||||
|
|
||||||
class TestIE(InfoExtractor):
|
class DummyIE(InfoExtractor):
|
||||||
pass
|
pass
|
||||||
|
|
||||||
|
|
||||||
class TestInfoExtractor(unittest.TestCase):
|
class TestInfoExtractor(unittest.TestCase):
|
||||||
def setUp(self):
|
def setUp(self):
|
||||||
self.ie = TestIE(FakeYDL())
|
self.ie = DummyIE(FakeYDL())
|
||||||
|
|
||||||
def test_ie_key(self):
|
def test_ie_key(self):
|
||||||
self.assertEqual(get_info_extractor(YoutubeIE.ie_key()), YoutubeIE)
|
self.assertEqual(get_info_extractor(YoutubeIE.ie_key()), YoutubeIE)
|
||||||
|
@ -62,6 +80,7 @@ class TestInfoExtractor(unittest.TestCase):
|
||||||
<meta name="og:test1" content='foo > < bar'/>
|
<meta name="og:test1" content='foo > < bar'/>
|
||||||
<meta name="og:test2" content="foo >//< bar"/>
|
<meta name="og:test2" content="foo >//< bar"/>
|
||||||
<meta property=og-test3 content='Ill-formatted opengraph'/>
|
<meta property=og-test3 content='Ill-formatted opengraph'/>
|
||||||
|
<meta property=og:test4 content=unquoted-value/>
|
||||||
'''
|
'''
|
||||||
self.assertEqual(ie._og_search_title(html), 'Foo')
|
self.assertEqual(ie._og_search_title(html), 'Foo')
|
||||||
self.assertEqual(ie._og_search_description(html), 'Some video\'s description ')
|
self.assertEqual(ie._og_search_description(html), 'Some video\'s description ')
|
||||||
|
@ -74,6 +93,7 @@ class TestInfoExtractor(unittest.TestCase):
|
||||||
self.assertEqual(ie._og_search_property(('test0', 'test1'), html), 'foo > < bar')
|
self.assertEqual(ie._og_search_property(('test0', 'test1'), html), 'foo > < bar')
|
||||||
self.assertRaises(RegexNotFoundError, ie._og_search_property, 'test0', html, None, fatal=True)
|
self.assertRaises(RegexNotFoundError, ie._og_search_property, 'test0', html, None, fatal=True)
|
||||||
self.assertRaises(RegexNotFoundError, ie._og_search_property, ('test0', 'test00'), html, None, fatal=True)
|
self.assertRaises(RegexNotFoundError, ie._og_search_property, ('test0', 'test00'), html, None, fatal=True)
|
||||||
|
self.assertEqual(ie._og_search_property('test4', html), 'unquoted-value')
|
||||||
|
|
||||||
def test_html_search_meta(self):
|
def test_html_search_meta(self):
|
||||||
ie = self.ie
|
ie = self.ie
|
||||||
|
@ -98,6 +118,123 @@ class TestInfoExtractor(unittest.TestCase):
|
||||||
self.assertRaises(RegexNotFoundError, ie._html_search_meta, 'z', html, None, fatal=True)
|
self.assertRaises(RegexNotFoundError, ie._html_search_meta, 'z', html, None, fatal=True)
|
||||||
self.assertRaises(RegexNotFoundError, ie._html_search_meta, ('z', 'x'), html, None, fatal=True)
|
self.assertRaises(RegexNotFoundError, ie._html_search_meta, ('z', 'x'), html, None, fatal=True)
|
||||||
|
|
||||||
|
def test_search_nextjs_data(self):
|
||||||
|
html = '''
|
||||||
|
<!DOCTYPE html>
|
||||||
|
<html>
|
||||||
|
<head>
|
||||||
|
<meta http-equiv="content-type" content=
|
||||||
|
"text/html; charset=utf-8">
|
||||||
|
<meta name="viewport" content="width=device-width">
|
||||||
|
<title>Test _search_nextjs_data()</title>
|
||||||
|
</head>
|
||||||
|
<body>
|
||||||
|
<div id="__next">
|
||||||
|
<div style="background-color:#17171E" class="FU" dir="ltr">
|
||||||
|
<div class="sc-93de261d-0 dyzzYE">
|
||||||
|
<div>
|
||||||
|
<header class="HD"></header>
|
||||||
|
<main class="MN">
|
||||||
|
<div style="height:0" class="HT0">
|
||||||
|
<div style="width:NaN%" data-testid=
|
||||||
|
"stream-container" class="WDN"></div>
|
||||||
|
</div>
|
||||||
|
</main>
|
||||||
|
</div>
|
||||||
|
<footer class="sc-6e5faf91-0 dEGaHS"></footer>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<script id="__NEXT_DATA__" type="application/json">
|
||||||
|
{"props":{"pageProps":{"video":{"id":"testid"}}}}
|
||||||
|
</script>
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
|
'''
|
||||||
|
search = self.ie._search_nextjs_data(html, 'testID')
|
||||||
|
self.assertEqual(search['props']['pageProps']['video']['id'], 'testid')
|
||||||
|
search = self.ie._search_nextjs_data(
|
||||||
|
'no next.js data here, move along', 'testID', default={'status': 0})
|
||||||
|
self.assertEqual(search['status'], 0)
|
||||||
|
|
||||||
|
def test_search_nuxt_data(self):
|
||||||
|
html = '''
|
||||||
|
<!DOCTYPE html>
|
||||||
|
<html>
|
||||||
|
<head>
|
||||||
|
<meta http-equiv="content-type" content=
|
||||||
|
"text/html; charset=utf-8">
|
||||||
|
<title>Nuxt.js Test Page</title>
|
||||||
|
<meta name="viewport" content=
|
||||||
|
"width=device-width, initial-scale=1">
|
||||||
|
<meta data-hid="robots" name="robots" content="all">
|
||||||
|
</head>
|
||||||
|
<body class="BD">
|
||||||
|
<div id="__layout">
|
||||||
|
<h1 class="H1">Example heading</h1>
|
||||||
|
<div class="IN">
|
||||||
|
<p>Decoy text</p>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<script>
|
||||||
|
window.__NUXT__=(function(a,b,c,d,e,f,g,h){return {decoy:" default",data:[{track:{id:f,title:g}}]}}(null,null,"c",null,null,"testid","Nuxt.js title",null));
|
||||||
|
</script>
|
||||||
|
<script src="/_nuxt/a12345b.js" defer="defer"></script>
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
|
'''
|
||||||
|
search = self.ie._search_nuxt_data(html, 'testID')
|
||||||
|
self.assertEqual(search['track']['id'], 'testid')
|
||||||
|
|
||||||
|
def test_search_json_ld_realworld(self):
|
||||||
|
# https://github.com/ytdl-org/youtube-dl/issues/23306
|
||||||
|
expect_dict(
|
||||||
|
self,
|
||||||
|
self.ie._search_json_ld(r'''<script type="application/ld+json">
|
||||||
|
{
|
||||||
|
"@context": "http://schema.org/",
|
||||||
|
"@type": "VideoObject",
|
||||||
|
"name": "1 On 1 With Kleio",
|
||||||
|
"url": "https://www.eporner.com/hd-porn/xN49A1cT3eB/1-On-1-With-Kleio/",
|
||||||
|
"duration": "PT0H12M23S",
|
||||||
|
"thumbnailUrl": ["https://static-eu-cdn.eporner.com/thumbs/static4/7/78/780/780814/9_360.jpg", "https://imggen.eporner.com/780814/1920/1080/9.jpg"],
|
||||||
|
"contentUrl": "https://gvideo.eporner.com/xN49A1cT3eB/xN49A1cT3eB.mp4",
|
||||||
|
"embedUrl": "https://www.eporner.com/embed/xN49A1cT3eB/1-On-1-With-Kleio/",
|
||||||
|
"image": "https://static-eu-cdn.eporner.com/thumbs/static4/7/78/780/780814/9_360.jpg",
|
||||||
|
"width": "1920",
|
||||||
|
"height": "1080",
|
||||||
|
"encodingFormat": "mp4",
|
||||||
|
"bitrate": "6617kbps",
|
||||||
|
"isFamilyFriendly": "False",
|
||||||
|
"description": "Kleio Valentien",
|
||||||
|
"uploadDate": "2015-12-05T21:24:35+01:00",
|
||||||
|
"interactionStatistic": {
|
||||||
|
"@type": "InteractionCounter",
|
||||||
|
"interactionType": { "@type": "http://schema.org/WatchAction" },
|
||||||
|
"userInteractionCount": 1120958
|
||||||
|
}, "aggregateRating": {
|
||||||
|
"@type": "AggregateRating",
|
||||||
|
"ratingValue": "88",
|
||||||
|
"ratingCount": "630",
|
||||||
|
"bestRating": "100",
|
||||||
|
"worstRating": "0"
|
||||||
|
}, "actor": [{
|
||||||
|
"@type": "Person",
|
||||||
|
"name": "Kleio Valentien",
|
||||||
|
"url": "https://www.eporner.com/pornstar/kleio-valentien/"
|
||||||
|
}]}
|
||||||
|
</script>''', None),
|
||||||
|
{
|
||||||
|
'title': '1 On 1 With Kleio',
|
||||||
|
'description': 'Kleio Valentien',
|
||||||
|
'url': 'https://gvideo.eporner.com/xN49A1cT3eB/xN49A1cT3eB.mp4',
|
||||||
|
'timestamp': 1449347075,
|
||||||
|
'duration': 743.0,
|
||||||
|
'view_count': 1120958,
|
||||||
|
'width': 1920,
|
||||||
|
'height': 1080,
|
||||||
|
})
|
||||||
|
|
||||||
def test_download_json(self):
|
def test_download_json(self):
|
||||||
uri = encode_data_uri(b'{"foo": "blah"}', 'application/json')
|
uri = encode_data_uri(b'{"foo": "blah"}', 'application/json')
|
||||||
self.assertEqual(self.ie._download_json(uri, None), {'foo': 'blah'})
|
self.assertEqual(self.ie._download_json(uri, None), {'foo': 'blah'})
|
||||||
|
@ -108,6 +245,18 @@ class TestInfoExtractor(unittest.TestCase):
|
||||||
self.assertEqual(self.ie._download_json(uri, None, fatal=False), None)
|
self.assertEqual(self.ie._download_json(uri, None, fatal=False), None)
|
||||||
|
|
||||||
def test_parse_html5_media_entries(self):
|
def test_parse_html5_media_entries(self):
|
||||||
|
# inline video tag
|
||||||
|
expect_dict(
|
||||||
|
self,
|
||||||
|
self.ie._parse_html5_media_entries(
|
||||||
|
'https://127.0.0.1/video.html',
|
||||||
|
r'<html><video src="/vid.mp4" /></html>', None)[0],
|
||||||
|
{
|
||||||
|
'formats': [{
|
||||||
|
'url': 'https://127.0.0.1/vid.mp4',
|
||||||
|
}],
|
||||||
|
})
|
||||||
|
|
||||||
# from https://www.r18.com/
|
# from https://www.r18.com/
|
||||||
# with kpbs in label
|
# with kpbs in label
|
||||||
expect_dict(
|
expect_dict(
|
||||||
|
@ -285,6 +434,24 @@ class TestInfoExtractor(unittest.TestCase):
|
||||||
}],
|
}],
|
||||||
})
|
})
|
||||||
|
|
||||||
|
# from https://0000.studio/
|
||||||
|
# with type attribute but without extension in URL
|
||||||
|
expect_dict(
|
||||||
|
self,
|
||||||
|
self.ie._parse_html5_media_entries(
|
||||||
|
'https://0000.studio',
|
||||||
|
r'''
|
||||||
|
<video src="https://d1ggyt9m8pwf3g.cloudfront.net/protected/ap-northeast-1:1864af40-28d5-492b-b739-b32314b1a527/archive/clip/838db6a7-8973-4cd6-840d-8517e4093c92"
|
||||||
|
controls="controls" type="video/mp4" preload="metadata" autoplay="autoplay" playsinline class="object-contain">
|
||||||
|
</video>
|
||||||
|
''', None)[0],
|
||||||
|
{
|
||||||
|
'formats': [{
|
||||||
|
'url': 'https://d1ggyt9m8pwf3g.cloudfront.net/protected/ap-northeast-1:1864af40-28d5-492b-b739-b32314b1a527/archive/clip/838db6a7-8973-4cd6-840d-8517e4093c92',
|
||||||
|
'ext': 'mp4',
|
||||||
|
}],
|
||||||
|
})
|
||||||
|
|
||||||
def test_extract_jwplayer_data_realworld(self):
|
def test_extract_jwplayer_data_realworld(self):
|
||||||
# from http://www.suffolk.edu/sjc/
|
# from http://www.suffolk.edu/sjc/
|
||||||
expect_dict(
|
expect_dict(
|
||||||
|
@ -738,8 +905,8 @@ jwplayer("mediaplayer").setup({"abouttext":"Visit Indie DB","aboutlink":"http:\/
|
||||||
]
|
]
|
||||||
|
|
||||||
for m3u8_file, m3u8_url, expected_formats in _TEST_CASES:
|
for m3u8_file, m3u8_url, expected_formats in _TEST_CASES:
|
||||||
with io.open('./test/testdata/m3u8/%s.m3u8' % m3u8_file,
|
with open('./test/testdata/m3u8/%s.m3u8' % m3u8_file,
|
||||||
mode='r', encoding='utf-8') as f:
|
mode='r', encoding='utf-8') as f:
|
||||||
formats = self.ie._parse_m3u8_formats(
|
formats = self.ie._parse_m3u8_formats(
|
||||||
f.read(), m3u8_url, ext='mp4')
|
f.read(), m3u8_url, ext='mp4')
|
||||||
self.ie._sort_formats(formats)
|
self.ie._sort_formats(formats)
|
||||||
|
@ -829,7 +996,8 @@ jwplayer("mediaplayer").setup({"abouttext":"Visit Indie DB","aboutlink":"http:\/
|
||||||
'tbr': 5997.485,
|
'tbr': 5997.485,
|
||||||
'width': 1920,
|
'width': 1920,
|
||||||
'height': 1080,
|
'height': 1080,
|
||||||
}]
|
}],
|
||||||
|
{},
|
||||||
), (
|
), (
|
||||||
# https://github.com/ytdl-org/youtube-dl/pull/14844
|
# https://github.com/ytdl-org/youtube-dl/pull/14844
|
||||||
'urls_only',
|
'urls_only',
|
||||||
|
@ -912,7 +1080,8 @@ jwplayer("mediaplayer").setup({"abouttext":"Visit Indie DB","aboutlink":"http:\/
|
||||||
'tbr': 4400,
|
'tbr': 4400,
|
||||||
'width': 1920,
|
'width': 1920,
|
||||||
'height': 1080,
|
'height': 1080,
|
||||||
}]
|
}],
|
||||||
|
{},
|
||||||
), (
|
), (
|
||||||
# https://github.com/ytdl-org/youtube-dl/issues/20346
|
# https://github.com/ytdl-org/youtube-dl/issues/20346
|
||||||
# Media considered unfragmented even though it contains
|
# Media considered unfragmented even though it contains
|
||||||
|
@ -958,18 +1127,185 @@ jwplayer("mediaplayer").setup({"abouttext":"Visit Indie DB","aboutlink":"http:\/
|
||||||
'width': 360,
|
'width': 360,
|
||||||
'height': 360,
|
'height': 360,
|
||||||
'fps': 30,
|
'fps': 30,
|
||||||
}]
|
}],
|
||||||
|
{},
|
||||||
|
), (
|
||||||
|
# https://github.com/ytdl-org/youtube-dl/issues/30235
|
||||||
|
# Bento4 generated test mpd
|
||||||
|
# mp4dash --mpd-name=manifest.mpd --no-split --use-segment-list mediafiles
|
||||||
|
'url_and_range',
|
||||||
|
'http://unknown/manifest.mpd', # mpd_url
|
||||||
|
'http://unknown/', # mpd_base_url
|
||||||
|
[{
|
||||||
|
'manifest_url': 'http://unknown/manifest.mpd',
|
||||||
|
'fragment_base_url': 'http://unknown/',
|
||||||
|
'ext': 'm4a',
|
||||||
|
'format_id': 'audio-und-mp4a.40.2',
|
||||||
|
'format_note': 'DASH audio',
|
||||||
|
'container': 'm4a_dash',
|
||||||
|
'protocol': 'http_dash_segments',
|
||||||
|
'acodec': 'mp4a.40.2',
|
||||||
|
'vcodec': 'none',
|
||||||
|
'tbr': 98.808,
|
||||||
|
}, {
|
||||||
|
'manifest_url': 'http://unknown/manifest.mpd',
|
||||||
|
'fragment_base_url': 'http://unknown/',
|
||||||
|
'ext': 'mp4',
|
||||||
|
'format_id': 'video-avc1',
|
||||||
|
'format_note': 'DASH video',
|
||||||
|
'container': 'mp4_dash',
|
||||||
|
'protocol': 'http_dash_segments',
|
||||||
|
'acodec': 'none',
|
||||||
|
'vcodec': 'avc1.4D401E',
|
||||||
|
'tbr': 699.597,
|
||||||
|
'width': 768,
|
||||||
|
'height': 432
|
||||||
|
}],
|
||||||
|
{},
|
||||||
|
), (
|
||||||
|
# https://github.com/ytdl-org/youtube-dl/issues/27575
|
||||||
|
# GPAC generated test mpd
|
||||||
|
# MP4Box -dash 10000 -single-file -out manifest.mpd mediafiles
|
||||||
|
'range_only',
|
||||||
|
'http://unknown/manifest.mpd', # mpd_url
|
||||||
|
'http://unknown/', # mpd_base_url
|
||||||
|
[{
|
||||||
|
'manifest_url': 'http://unknown/manifest.mpd',
|
||||||
|
'fragment_base_url': 'http://unknown/audio_dashinit.mp4',
|
||||||
|
'ext': 'm4a',
|
||||||
|
'format_id': '2',
|
||||||
|
'format_note': 'DASH audio',
|
||||||
|
'container': 'm4a_dash',
|
||||||
|
'protocol': 'http_dash_segments',
|
||||||
|
'acodec': 'mp4a.40.2',
|
||||||
|
'vcodec': 'none',
|
||||||
|
'tbr': 98.096,
|
||||||
|
}, {
|
||||||
|
'manifest_url': 'http://unknown/manifest.mpd',
|
||||||
|
'fragment_base_url': 'http://unknown/video_dashinit.mp4',
|
||||||
|
'ext': 'mp4',
|
||||||
|
'format_id': '1',
|
||||||
|
'format_note': 'DASH video',
|
||||||
|
'container': 'mp4_dash',
|
||||||
|
'protocol': 'http_dash_segments',
|
||||||
|
'acodec': 'none',
|
||||||
|
'vcodec': 'avc1.4D401E',
|
||||||
|
'tbr': 526.987,
|
||||||
|
'width': 768,
|
||||||
|
'height': 432
|
||||||
|
}],
|
||||||
|
{},
|
||||||
|
), (
|
||||||
|
'subtitles',
|
||||||
|
'https://sdn-global-streaming-cache-3qsdn.akamaized.net/stream/3144/files/17/07/672975/3144-kZT4LWMQw6Rh7Kpd.ism/manifest.mpd',
|
||||||
|
'https://sdn-global-streaming-cache-3qsdn.akamaized.net/stream/3144/files/17/07/672975/3144-kZT4LWMQw6Rh7Kpd.ism/',
|
||||||
|
[{
|
||||||
|
'format_id': 'audio=128001',
|
||||||
|
'manifest_url': 'https://sdn-global-streaming-cache-3qsdn.akamaized.net/stream/3144/files/17/07/672975/3144-kZT4LWMQw6Rh7Kpd.ism/manifest.mpd',
|
||||||
|
'ext': 'm4a',
|
||||||
|
'tbr': 128.001,
|
||||||
|
'asr': 48000,
|
||||||
|
'format_note': 'DASH audio',
|
||||||
|
'container': 'm4a_dash',
|
||||||
|
'vcodec': 'none',
|
||||||
|
'acodec': 'mp4a.40.2',
|
||||||
|
'url': 'https://sdn-global-streaming-cache-3qsdn.akamaized.net/stream/3144/files/17/07/672975/3144-kZT4LWMQw6Rh7Kpd.ism/manifest.mpd',
|
||||||
|
'fragment_base_url': 'https://sdn-global-streaming-cache-3qsdn.akamaized.net/stream/3144/files/17/07/672975/3144-kZT4LWMQw6Rh7Kpd.ism/dash/',
|
||||||
|
'protocol': 'http_dash_segments',
|
||||||
|
}, {
|
||||||
|
'format_id': 'video=100000',
|
||||||
|
'manifest_url': 'https://sdn-global-streaming-cache-3qsdn.akamaized.net/stream/3144/files/17/07/672975/3144-kZT4LWMQw6Rh7Kpd.ism/manifest.mpd',
|
||||||
|
'ext': 'mp4',
|
||||||
|
'width': 336,
|
||||||
|
'height': 144,
|
||||||
|
'tbr': 100,
|
||||||
|
'format_note': 'DASH video',
|
||||||
|
'container': 'mp4_dash',
|
||||||
|
'vcodec': 'avc1.4D401F',
|
||||||
|
'acodec': 'none',
|
||||||
|
'url': 'https://sdn-global-streaming-cache-3qsdn.akamaized.net/stream/3144/files/17/07/672975/3144-kZT4LWMQw6Rh7Kpd.ism/manifest.mpd',
|
||||||
|
'fragment_base_url': 'https://sdn-global-streaming-cache-3qsdn.akamaized.net/stream/3144/files/17/07/672975/3144-kZT4LWMQw6Rh7Kpd.ism/dash/',
|
||||||
|
'protocol': 'http_dash_segments',
|
||||||
|
}, {
|
||||||
|
'format_id': 'video=326000',
|
||||||
|
'manifest_url': 'https://sdn-global-streaming-cache-3qsdn.akamaized.net/stream/3144/files/17/07/672975/3144-kZT4LWMQw6Rh7Kpd.ism/manifest.mpd',
|
||||||
|
'ext': 'mp4',
|
||||||
|
'width': 562,
|
||||||
|
'height': 240,
|
||||||
|
'tbr': 326,
|
||||||
|
'format_note': 'DASH video',
|
||||||
|
'container': 'mp4_dash',
|
||||||
|
'vcodec': 'avc1.4D401F',
|
||||||
|
'acodec': 'none',
|
||||||
|
'url': 'https://sdn-global-streaming-cache-3qsdn.akamaized.net/stream/3144/files/17/07/672975/3144-kZT4LWMQw6Rh7Kpd.ism/manifest.mpd',
|
||||||
|
'fragment_base_url': 'https://sdn-global-streaming-cache-3qsdn.akamaized.net/stream/3144/files/17/07/672975/3144-kZT4LWMQw6Rh7Kpd.ism/dash/',
|
||||||
|
'protocol': 'http_dash_segments',
|
||||||
|
}, {
|
||||||
|
'format_id': 'video=698000',
|
||||||
|
'manifest_url': 'https://sdn-global-streaming-cache-3qsdn.akamaized.net/stream/3144/files/17/07/672975/3144-kZT4LWMQw6Rh7Kpd.ism/manifest.mpd',
|
||||||
|
'ext': 'mp4',
|
||||||
|
'width': 844,
|
||||||
|
'height': 360,
|
||||||
|
'tbr': 698,
|
||||||
|
'format_note': 'DASH video',
|
||||||
|
'container': 'mp4_dash',
|
||||||
|
'vcodec': 'avc1.4D401F',
|
||||||
|
'acodec': 'none',
|
||||||
|
'url': 'https://sdn-global-streaming-cache-3qsdn.akamaized.net/stream/3144/files/17/07/672975/3144-kZT4LWMQw6Rh7Kpd.ism/manifest.mpd',
|
||||||
|
'fragment_base_url': 'https://sdn-global-streaming-cache-3qsdn.akamaized.net/stream/3144/files/17/07/672975/3144-kZT4LWMQw6Rh7Kpd.ism/dash/',
|
||||||
|
'protocol': 'http_dash_segments',
|
||||||
|
}, {
|
||||||
|
'format_id': 'video=1493000',
|
||||||
|
'manifest_url': 'https://sdn-global-streaming-cache-3qsdn.akamaized.net/stream/3144/files/17/07/672975/3144-kZT4LWMQw6Rh7Kpd.ism/manifest.mpd',
|
||||||
|
'ext': 'mp4',
|
||||||
|
'width': 1126,
|
||||||
|
'height': 480,
|
||||||
|
'tbr': 1493,
|
||||||
|
'format_note': 'DASH video',
|
||||||
|
'container': 'mp4_dash',
|
||||||
|
'vcodec': 'avc1.4D401F',
|
||||||
|
'acodec': 'none',
|
||||||
|
'url': 'https://sdn-global-streaming-cache-3qsdn.akamaized.net/stream/3144/files/17/07/672975/3144-kZT4LWMQw6Rh7Kpd.ism/manifest.mpd',
|
||||||
|
'fragment_base_url': 'https://sdn-global-streaming-cache-3qsdn.akamaized.net/stream/3144/files/17/07/672975/3144-kZT4LWMQw6Rh7Kpd.ism/dash/',
|
||||||
|
'protocol': 'http_dash_segments',
|
||||||
|
}, {
|
||||||
|
'format_id': 'video=4482000',
|
||||||
|
'manifest_url': 'https://sdn-global-streaming-cache-3qsdn.akamaized.net/stream/3144/files/17/07/672975/3144-kZT4LWMQw6Rh7Kpd.ism/manifest.mpd',
|
||||||
|
'ext': 'mp4',
|
||||||
|
'width': 1688,
|
||||||
|
'height': 720,
|
||||||
|
'tbr': 4482,
|
||||||
|
'format_note': 'DASH video',
|
||||||
|
'container': 'mp4_dash',
|
||||||
|
'vcodec': 'avc1.4D401F',
|
||||||
|
'acodec': 'none',
|
||||||
|
'url': 'https://sdn-global-streaming-cache-3qsdn.akamaized.net/stream/3144/files/17/07/672975/3144-kZT4LWMQw6Rh7Kpd.ism/manifest.mpd',
|
||||||
|
'fragment_base_url': 'https://sdn-global-streaming-cache-3qsdn.akamaized.net/stream/3144/files/17/07/672975/3144-kZT4LWMQw6Rh7Kpd.ism/dash/',
|
||||||
|
'protocol': 'http_dash_segments',
|
||||||
|
}],
|
||||||
|
{
|
||||||
|
'en': [
|
||||||
|
{
|
||||||
|
'ext': 'mp4',
|
||||||
|
'manifest_url': 'https://sdn-global-streaming-cache-3qsdn.akamaized.net/stream/3144/files/17/07/672975/3144-kZT4LWMQw6Rh7Kpd.ism/manifest.mpd',
|
||||||
|
'url': 'https://sdn-global-streaming-cache-3qsdn.akamaized.net/stream/3144/files/17/07/672975/3144-kZT4LWMQw6Rh7Kpd.ism/manifest.mpd',
|
||||||
|
'fragment_base_url': 'https://sdn-global-streaming-cache-3qsdn.akamaized.net/stream/3144/files/17/07/672975/3144-kZT4LWMQw6Rh7Kpd.ism/dash/',
|
||||||
|
'protocol': 'http_dash_segments',
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
)
|
)
|
||||||
]
|
]
|
||||||
|
|
||||||
for mpd_file, mpd_url, mpd_base_url, expected_formats in _TEST_CASES:
|
for mpd_file, mpd_url, mpd_base_url, expected_formats, expected_subtitles in _TEST_CASES:
|
||||||
with io.open('./test/testdata/mpd/%s.mpd' % mpd_file,
|
with open('./test/testdata/mpd/%s.mpd' % mpd_file,
|
||||||
mode='r', encoding='utf-8') as f:
|
mode='r', encoding='utf-8') as f:
|
||||||
formats = self.ie._parse_mpd_formats(
|
formats, subtitles = self.ie._parse_mpd_formats_and_subtitles(
|
||||||
compat_etree_fromstring(f.read().encode('utf-8')),
|
compat_etree_fromstring(f.read().encode('utf-8')),
|
||||||
mpd_base_url=mpd_base_url, mpd_url=mpd_url)
|
mpd_base_url=mpd_base_url, mpd_url=mpd_url)
|
||||||
self.ie._sort_formats(formats)
|
self.ie._sort_formats(formats)
|
||||||
expect_value(self, formats, expected_formats, None)
|
expect_value(self, formats, expected_formats, None)
|
||||||
|
expect_value(self, subtitles, expected_subtitles, None)
|
||||||
|
|
||||||
def test_parse_f4m_formats(self):
|
def test_parse_f4m_formats(self):
|
||||||
_TEST_CASES = [
|
_TEST_CASES = [
|
||||||
|
@ -990,8 +1326,8 @@ jwplayer("mediaplayer").setup({"abouttext":"Visit Indie DB","aboutlink":"http:\/
|
||||||
]
|
]
|
||||||
|
|
||||||
for f4m_file, f4m_url, expected_formats in _TEST_CASES:
|
for f4m_file, f4m_url, expected_formats in _TEST_CASES:
|
||||||
with io.open('./test/testdata/f4m/%s.f4m' % f4m_file,
|
with open('./test/testdata/f4m/%s.f4m' % f4m_file,
|
||||||
mode='r', encoding='utf-8') as f:
|
mode='r', encoding='utf-8') as f:
|
||||||
formats = self.ie._parse_f4m_formats(
|
formats = self.ie._parse_f4m_formats(
|
||||||
compat_etree_fromstring(f.read().encode('utf-8')),
|
compat_etree_fromstring(f.read().encode('utf-8')),
|
||||||
f4m_url, None)
|
f4m_url, None)
|
||||||
|
@ -1038,8 +1374,8 @@ jwplayer("mediaplayer").setup({"abouttext":"Visit Indie DB","aboutlink":"http:\/
|
||||||
]
|
]
|
||||||
|
|
||||||
for xspf_file, xspf_url, expected_entries in _TEST_CASES:
|
for xspf_file, xspf_url, expected_entries in _TEST_CASES:
|
||||||
with io.open('./test/testdata/xspf/%s.xspf' % xspf_file,
|
with open('./test/testdata/xspf/%s.xspf' % xspf_file,
|
||||||
mode='r', encoding='utf-8') as f:
|
mode='r', encoding='utf-8') as f:
|
||||||
entries = self.ie._parse_xspf(
|
entries = self.ie._parse_xspf(
|
||||||
compat_etree_fromstring(f.read().encode('utf-8')),
|
compat_etree_fromstring(f.read().encode('utf-8')),
|
||||||
xspf_file, xspf_url=xspf_url, xspf_base_url=xspf_url)
|
xspf_file, xspf_url=xspf_url, xspf_base_url=xspf_url)
|
||||||
|
|
|
@ -10,14 +10,31 @@ import unittest
|
||||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||||
|
|
||||||
import copy
|
import copy
|
||||||
|
import json
|
||||||
|
|
||||||
from test.helper import FakeYDL, assertRegexpMatches
|
from test.helper import (
|
||||||
|
FakeYDL,
|
||||||
|
assertRegexpMatches,
|
||||||
|
try_rm,
|
||||||
|
)
|
||||||
from youtube_dl import YoutubeDL
|
from youtube_dl import YoutubeDL
|
||||||
from youtube_dl.compat import compat_str, compat_urllib_error
|
from youtube_dl.compat import (
|
||||||
|
compat_http_cookiejar_Cookie,
|
||||||
|
compat_http_cookies_SimpleCookie,
|
||||||
|
compat_kwargs,
|
||||||
|
compat_open as open,
|
||||||
|
compat_str,
|
||||||
|
compat_urllib_error,
|
||||||
|
)
|
||||||
|
|
||||||
from youtube_dl.extractor import YoutubeIE
|
from youtube_dl.extractor import YoutubeIE
|
||||||
from youtube_dl.extractor.common import InfoExtractor
|
from youtube_dl.extractor.common import InfoExtractor
|
||||||
from youtube_dl.postprocessor.common import PostProcessor
|
from youtube_dl.postprocessor.common import PostProcessor
|
||||||
from youtube_dl.utils import ExtractorError, match_filter_func
|
from youtube_dl.utils import (
|
||||||
|
ExtractorError,
|
||||||
|
match_filter_func,
|
||||||
|
traverse_obj,
|
||||||
|
)
|
||||||
|
|
||||||
TEST_URL = 'http://localhost/sample.mp4'
|
TEST_URL = 'http://localhost/sample.mp4'
|
||||||
|
|
||||||
|
@ -29,11 +46,14 @@ class YDL(FakeYDL):
|
||||||
self.msgs = []
|
self.msgs = []
|
||||||
|
|
||||||
def process_info(self, info_dict):
|
def process_info(self, info_dict):
|
||||||
self.downloaded_info_dicts.append(info_dict)
|
self.downloaded_info_dicts.append(info_dict.copy())
|
||||||
|
|
||||||
def to_screen(self, msg):
|
def to_screen(self, msg):
|
||||||
self.msgs.append(msg)
|
self.msgs.append(msg)
|
||||||
|
|
||||||
|
def dl(self, *args, **kwargs):
|
||||||
|
assert False, 'Downloader must not be invoked for test_YoutubeDL'
|
||||||
|
|
||||||
|
|
||||||
def _make_result(formats, **kwargs):
|
def _make_result(formats, **kwargs):
|
||||||
res = {
|
res = {
|
||||||
|
@ -42,8 +62,9 @@ def _make_result(formats, **kwargs):
|
||||||
'title': 'testttitle',
|
'title': 'testttitle',
|
||||||
'extractor': 'testex',
|
'extractor': 'testex',
|
||||||
'extractor_key': 'TestEx',
|
'extractor_key': 'TestEx',
|
||||||
|
'webpage_url': 'http://example.com/watch?v=shenanigans',
|
||||||
}
|
}
|
||||||
res.update(**kwargs)
|
res.update(**compat_kwargs(kwargs))
|
||||||
return res
|
return res
|
||||||
|
|
||||||
|
|
||||||
|
@ -464,6 +485,7 @@ class TestFormatSelection(unittest.TestCase):
|
||||||
assert_syntax_error('+bestaudio')
|
assert_syntax_error('+bestaudio')
|
||||||
assert_syntax_error('bestvideo+')
|
assert_syntax_error('bestvideo+')
|
||||||
assert_syntax_error('/')
|
assert_syntax_error('/')
|
||||||
|
assert_syntax_error('bestvideo+bestvideo+bestaudio')
|
||||||
|
|
||||||
def test_format_filtering(self):
|
def test_format_filtering(self):
|
||||||
formats = [
|
formats = [
|
||||||
|
@ -632,13 +654,20 @@ class TestYoutubeDL(unittest.TestCase):
|
||||||
'title2': '%PATH%',
|
'title2': '%PATH%',
|
||||||
}
|
}
|
||||||
|
|
||||||
def fname(templ):
|
def fname(templ, na_placeholder='NA'):
|
||||||
ydl = YoutubeDL({'outtmpl': templ})
|
params = {'outtmpl': templ}
|
||||||
|
if na_placeholder != 'NA':
|
||||||
|
params['outtmpl_na_placeholder'] = na_placeholder
|
||||||
|
ydl = YoutubeDL(params)
|
||||||
return ydl.prepare_filename(info)
|
return ydl.prepare_filename(info)
|
||||||
self.assertEqual(fname('%(id)s.%(ext)s'), '1234.mp4')
|
self.assertEqual(fname('%(id)s.%(ext)s'), '1234.mp4')
|
||||||
self.assertEqual(fname('%(id)s-%(width)s.%(ext)s'), '1234-NA.mp4')
|
self.assertEqual(fname('%(id)s-%(width)s.%(ext)s'), '1234-NA.mp4')
|
||||||
# Replace missing fields with 'NA'
|
NA_TEST_OUTTMPL = '%(uploader_date)s-%(width)d-%(id)s.%(ext)s'
|
||||||
self.assertEqual(fname('%(uploader_date)s-%(id)s.%(ext)s'), 'NA-1234.mp4')
|
# Replace missing fields with 'NA' by default
|
||||||
|
self.assertEqual(fname(NA_TEST_OUTTMPL), 'NA-NA-1234.mp4')
|
||||||
|
# Or by provided placeholder
|
||||||
|
self.assertEqual(fname(NA_TEST_OUTTMPL, na_placeholder='none'), 'none-none-1234.mp4')
|
||||||
|
self.assertEqual(fname(NA_TEST_OUTTMPL, na_placeholder=''), '--1234.mp4')
|
||||||
self.assertEqual(fname('%(height)d.%(ext)s'), '1080.mp4')
|
self.assertEqual(fname('%(height)d.%(ext)s'), '1080.mp4')
|
||||||
self.assertEqual(fname('%(height)6d.%(ext)s'), ' 1080.mp4')
|
self.assertEqual(fname('%(height)6d.%(ext)s'), ' 1080.mp4')
|
||||||
self.assertEqual(fname('%(height)-6d.%(ext)s'), '1080 .mp4')
|
self.assertEqual(fname('%(height)-6d.%(ext)s'), '1080 .mp4')
|
||||||
|
@ -673,12 +702,12 @@ class TestYoutubeDL(unittest.TestCase):
|
||||||
|
|
||||||
class SimplePP(PostProcessor):
|
class SimplePP(PostProcessor):
|
||||||
def run(self, info):
|
def run(self, info):
|
||||||
with open(audiofile, 'wt') as f:
|
with open(audiofile, 'w') as f:
|
||||||
f.write('EXAMPLE')
|
f.write('EXAMPLE')
|
||||||
return [info['filepath']], info
|
return [info['filepath']], info
|
||||||
|
|
||||||
def run_pp(params, PP):
|
def run_pp(params, PP):
|
||||||
with open(filename, 'wt') as f:
|
with open(filename, 'w') as f:
|
||||||
f.write('EXAMPLE')
|
f.write('EXAMPLE')
|
||||||
ydl = YoutubeDL(params)
|
ydl = YoutubeDL(params)
|
||||||
ydl.add_post_processor(PP())
|
ydl.add_post_processor(PP())
|
||||||
|
@ -697,7 +726,7 @@ class TestYoutubeDL(unittest.TestCase):
|
||||||
|
|
||||||
class ModifierPP(PostProcessor):
|
class ModifierPP(PostProcessor):
|
||||||
def run(self, info):
|
def run(self, info):
|
||||||
with open(info['filepath'], 'wt') as f:
|
with open(info['filepath'], 'w') as f:
|
||||||
f.write('MODIFIED')
|
f.write('MODIFIED')
|
||||||
return [], info
|
return [], info
|
||||||
|
|
||||||
|
@ -919,6 +948,244 @@ class TestYoutubeDL(unittest.TestCase):
|
||||||
self.assertEqual(downloaded['extractor'], 'testex')
|
self.assertEqual(downloaded['extractor'], 'testex')
|
||||||
self.assertEqual(downloaded['extractor_key'], 'TestEx')
|
self.assertEqual(downloaded['extractor_key'], 'TestEx')
|
||||||
|
|
||||||
|
# Test case for https://github.com/ytdl-org/youtube-dl/issues/27064
|
||||||
|
def test_ignoreerrors_for_playlist_with_url_transparent_iterable_entries(self):
|
||||||
|
|
||||||
|
ydl = YDL({
|
||||||
|
'format': 'extra',
|
||||||
|
'ignoreerrors': True,
|
||||||
|
})
|
||||||
|
ydl.trouble = lambda *_, **__: None
|
||||||
|
|
||||||
|
class VideoIE(InfoExtractor):
|
||||||
|
_VALID_URL = r'video:(?P<id>\d+)'
|
||||||
|
|
||||||
|
def _real_extract(self, url):
|
||||||
|
video_id = self._match_id(url)
|
||||||
|
formats = [{
|
||||||
|
'format_id': 'default',
|
||||||
|
'url': 'url:',
|
||||||
|
}]
|
||||||
|
if video_id == '0':
|
||||||
|
raise ExtractorError('foo')
|
||||||
|
if video_id == '2':
|
||||||
|
formats.append({
|
||||||
|
'format_id': 'extra',
|
||||||
|
'url': TEST_URL,
|
||||||
|
})
|
||||||
|
return {
|
||||||
|
'id': video_id,
|
||||||
|
'title': 'Video %s' % video_id,
|
||||||
|
'formats': formats,
|
||||||
|
}
|
||||||
|
|
||||||
|
class PlaylistIE(InfoExtractor):
|
||||||
|
_VALID_URL = r'playlist:'
|
||||||
|
|
||||||
|
def _entries(self):
|
||||||
|
for n in range(3):
|
||||||
|
video_id = compat_str(n)
|
||||||
|
yield {
|
||||||
|
'_type': 'url_transparent',
|
||||||
|
'ie_key': VideoIE.ie_key(),
|
||||||
|
'id': video_id,
|
||||||
|
'url': 'video:%s' % video_id,
|
||||||
|
'title': 'Video Transparent %s' % video_id,
|
||||||
|
}
|
||||||
|
|
||||||
|
def _real_extract(self, url):
|
||||||
|
return self.playlist_result(self._entries())
|
||||||
|
|
||||||
|
ydl.add_info_extractor(VideoIE(ydl))
|
||||||
|
ydl.add_info_extractor(PlaylistIE(ydl))
|
||||||
|
info = ydl.extract_info('playlist:')
|
||||||
|
entries = info['entries']
|
||||||
|
self.assertEqual(len(entries), 3)
|
||||||
|
self.assertTrue(entries[0] is None)
|
||||||
|
self.assertTrue(entries[1] is None)
|
||||||
|
self.assertEqual(len(ydl.downloaded_info_dicts), 1)
|
||||||
|
downloaded = ydl.downloaded_info_dicts[0]
|
||||||
|
self.assertEqual(entries[2], downloaded)
|
||||||
|
self.assertEqual(downloaded['url'], TEST_URL)
|
||||||
|
self.assertEqual(downloaded['title'], 'Video Transparent 2')
|
||||||
|
self.assertEqual(downloaded['id'], '2')
|
||||||
|
self.assertEqual(downloaded['extractor'], 'Video')
|
||||||
|
self.assertEqual(downloaded['extractor_key'], 'Video')
|
||||||
|
|
||||||
|
def test_default_times(self):
|
||||||
|
"""Test addition of missing upload/release/_date from /release_/timestamp"""
|
||||||
|
info = {
|
||||||
|
'id': '1234',
|
||||||
|
'url': TEST_URL,
|
||||||
|
'title': 'Title',
|
||||||
|
'ext': 'mp4',
|
||||||
|
'timestamp': 1631352900,
|
||||||
|
'release_timestamp': 1632995931,
|
||||||
|
}
|
||||||
|
|
||||||
|
params = {'simulate': True, }
|
||||||
|
ydl = FakeYDL(params)
|
||||||
|
out_info = ydl.process_ie_result(info)
|
||||||
|
self.assertTrue(isinstance(out_info['upload_date'], compat_str))
|
||||||
|
self.assertEqual(out_info['upload_date'], '20210911')
|
||||||
|
self.assertTrue(isinstance(out_info['release_date'], compat_str))
|
||||||
|
self.assertEqual(out_info['release_date'], '20210930')
|
||||||
|
|
||||||
|
|
||||||
|
class TestYoutubeDLCookies(unittest.TestCase):
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def encode_cookie(cookie):
|
||||||
|
if not isinstance(cookie, dict):
|
||||||
|
cookie = vars(cookie)
|
||||||
|
for name, value in cookie.items():
|
||||||
|
yield name, compat_str(value)
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def comparable_cookies(cls, cookies):
|
||||||
|
# Work around cookiejar cookies not being unicode strings
|
||||||
|
return sorted(map(tuple, map(sorted, map(cls.encode_cookie, cookies))))
|
||||||
|
|
||||||
|
def assertSameCookies(self, c1, c2, msg=None):
|
||||||
|
return self.assertEqual(
|
||||||
|
*map(self.comparable_cookies, (c1, c2)),
|
||||||
|
msg=msg)
|
||||||
|
|
||||||
|
def assertSameCookieStrings(self, c1, c2, msg=None):
|
||||||
|
return self.assertSameCookies(
|
||||||
|
*map(lambda c: compat_http_cookies_SimpleCookie(c).values(), (c1, c2)),
|
||||||
|
msg=msg)
|
||||||
|
|
||||||
|
def test_header_cookies(self):
|
||||||
|
|
||||||
|
ydl = FakeYDL()
|
||||||
|
ydl.report_warning = lambda *_, **__: None
|
||||||
|
|
||||||
|
def cookie(name, value, version=None, domain='', path='', secure=False, expires=None):
|
||||||
|
return compat_http_cookiejar_Cookie(
|
||||||
|
version or 0, name, value, None, False,
|
||||||
|
domain, bool(domain), bool(domain), path, bool(path),
|
||||||
|
secure, expires, False, None, None, rest={})
|
||||||
|
|
||||||
|
test_url, test_domain = (t % ('yt.dl',) for t in ('https://%s/test', '.%s'))
|
||||||
|
|
||||||
|
def test(encoded_cookies, cookies, headers=False, round_trip=None, error_re=None):
|
||||||
|
def _test():
|
||||||
|
ydl.cookiejar.clear()
|
||||||
|
ydl._load_cookies(encoded_cookies, autoscope=headers)
|
||||||
|
if headers:
|
||||||
|
ydl._apply_header_cookies(test_url)
|
||||||
|
data = {'url': test_url}
|
||||||
|
ydl._calc_headers(data)
|
||||||
|
self.assertSameCookies(
|
||||||
|
cookies, ydl.cookiejar,
|
||||||
|
'Extracted cookiejar.Cookie is not the same')
|
||||||
|
if not headers:
|
||||||
|
self.assertSameCookieStrings(
|
||||||
|
data.get('cookies'), round_trip or encoded_cookies,
|
||||||
|
msg='Cookie is not the same as round trip')
|
||||||
|
ydl.__dict__['_YoutubeDL__header_cookies'] = []
|
||||||
|
|
||||||
|
try:
|
||||||
|
_test()
|
||||||
|
except AssertionError:
|
||||||
|
raise
|
||||||
|
except Exception as e:
|
||||||
|
if not error_re:
|
||||||
|
raise
|
||||||
|
assertRegexpMatches(self, e.args[0], error_re.join(('.*',) * 2))
|
||||||
|
|
||||||
|
test('test=value; Domain=' + test_domain, [cookie('test', 'value', domain=test_domain)])
|
||||||
|
test('test=value', [cookie('test', 'value')], error_re='Unscoped cookies are not allowed')
|
||||||
|
test('cookie1=value1; Domain={0}; Path=/test; cookie2=value2; Domain={0}; Path=/'.format(test_domain), [
|
||||||
|
cookie('cookie1', 'value1', domain=test_domain, path='/test'),
|
||||||
|
cookie('cookie2', 'value2', domain=test_domain, path='/')])
|
||||||
|
cookie_kw = compat_kwargs(
|
||||||
|
{'domain': test_domain, 'path': '/test', 'secure': True, 'expires': '9999999999', })
|
||||||
|
test('test=value; Domain={domain}; Path={path}; Secure; Expires={expires}'.format(**cookie_kw), [
|
||||||
|
cookie('test', 'value', **cookie_kw)])
|
||||||
|
test('test="value; "; path=/test; domain=' + test_domain, [
|
||||||
|
cookie('test', 'value; ', domain=test_domain, path='/test')],
|
||||||
|
round_trip='test="value\\073 "; Domain={0}; Path=/test'.format(test_domain))
|
||||||
|
test('name=; Domain=' + test_domain, [cookie('name', '', domain=test_domain)],
|
||||||
|
round_trip='name=""; Domain=' + test_domain)
|
||||||
|
test('test=value', [cookie('test', 'value', domain=test_domain)], headers=True)
|
||||||
|
test('cookie1=value; Domain={0}; cookie2=value'.format(test_domain), [],
|
||||||
|
headers=True, error_re='Invalid syntax')
|
||||||
|
ydl.report_warning = ydl.report_error
|
||||||
|
test('test=value', [], headers=True, error_re='Passing cookies as a header is a potential security risk')
|
||||||
|
|
||||||
|
def test_infojson_cookies(self):
|
||||||
|
TEST_FILE = 'test_infojson_cookies.info.json'
|
||||||
|
TEST_URL = 'https://example.com/example.mp4'
|
||||||
|
COOKIES = 'a=b; Domain=.example.com; c=d; Domain=.example.com'
|
||||||
|
COOKIE_HEADER = {'Cookie': 'a=b; c=d'}
|
||||||
|
|
||||||
|
ydl = FakeYDL()
|
||||||
|
ydl.process_info = lambda x: ydl._write_info_json('test', x, TEST_FILE)
|
||||||
|
|
||||||
|
def make_info(info_header_cookies=False, fmts_header_cookies=False, cookies_field=False):
|
||||||
|
fmt = {'url': TEST_URL}
|
||||||
|
if fmts_header_cookies:
|
||||||
|
fmt['http_headers'] = COOKIE_HEADER
|
||||||
|
if cookies_field:
|
||||||
|
fmt['cookies'] = COOKIES
|
||||||
|
return _make_result([fmt], http_headers=COOKIE_HEADER if info_header_cookies else None)
|
||||||
|
|
||||||
|
def test(initial_info, note):
|
||||||
|
|
||||||
|
def failure_msg(why):
|
||||||
|
return ' when '.join((why, note))
|
||||||
|
|
||||||
|
result = {}
|
||||||
|
result['processed'] = ydl.process_ie_result(initial_info)
|
||||||
|
self.assertTrue(ydl.cookiejar.get_cookies_for_url(TEST_URL),
|
||||||
|
msg=failure_msg('No cookies set in cookiejar after initial process'))
|
||||||
|
ydl.cookiejar.clear()
|
||||||
|
with open(TEST_FILE) as infojson:
|
||||||
|
result['loaded'] = ydl.sanitize_info(json.load(infojson), True)
|
||||||
|
result['final'] = ydl.process_ie_result(result['loaded'].copy(), download=False)
|
||||||
|
self.assertTrue(ydl.cookiejar.get_cookies_for_url(TEST_URL),
|
||||||
|
msg=failure_msg('No cookies set in cookiejar after final process'))
|
||||||
|
ydl.cookiejar.clear()
|
||||||
|
for key in ('processed', 'loaded', 'final'):
|
||||||
|
info = result[key]
|
||||||
|
self.assertIsNone(
|
||||||
|
traverse_obj(info, ((None, ('formats', 0)), 'http_headers', 'Cookie'), casesense=False, get_all=False),
|
||||||
|
msg=failure_msg('Cookie header not removed in {0} result'.format(key)))
|
||||||
|
self.assertSameCookieStrings(
|
||||||
|
traverse_obj(info, ((None, ('formats', 0)), 'cookies'), get_all=False), COOKIES,
|
||||||
|
msg=failure_msg('No cookies field found in {0} result'.format(key)))
|
||||||
|
|
||||||
|
test({'url': TEST_URL, 'http_headers': COOKIE_HEADER, 'id': '1', 'title': 'x'}, 'no formats field')
|
||||||
|
test(make_info(info_header_cookies=True), 'info_dict header cokies')
|
||||||
|
test(make_info(fmts_header_cookies=True), 'format header cookies')
|
||||||
|
test(make_info(info_header_cookies=True, fmts_header_cookies=True), 'info_dict and format header cookies')
|
||||||
|
test(make_info(info_header_cookies=True, fmts_header_cookies=True, cookies_field=True), 'all cookies fields')
|
||||||
|
test(make_info(cookies_field=True), 'cookies format field')
|
||||||
|
test({'url': TEST_URL, 'cookies': COOKIES, 'id': '1', 'title': 'x'}, 'info_dict cookies field only')
|
||||||
|
|
||||||
|
try_rm(TEST_FILE)
|
||||||
|
|
||||||
|
def test_add_headers_cookie(self):
|
||||||
|
def check_for_cookie_header(result):
|
||||||
|
return traverse_obj(result, ((None, ('formats', 0)), 'http_headers', 'Cookie'), casesense=False, get_all=False)
|
||||||
|
|
||||||
|
ydl = FakeYDL({'http_headers': {'Cookie': 'a=b'}})
|
||||||
|
ydl._apply_header_cookies(_make_result([])['webpage_url']) # Scope to input webpage URL: .example.com
|
||||||
|
|
||||||
|
fmt = {'url': 'https://example.com/video.mp4'}
|
||||||
|
result = ydl.process_ie_result(_make_result([fmt]), download=False)
|
||||||
|
self.assertIsNone(check_for_cookie_header(result), msg='http_headers cookies in result info_dict')
|
||||||
|
self.assertEqual(result.get('cookies'), 'a=b; Domain=.example.com', msg='No cookies were set in cookies field')
|
||||||
|
self.assertIn('a=b', ydl.cookiejar.get_cookie_header(fmt['url']), msg='No cookies were set in cookiejar')
|
||||||
|
|
||||||
|
fmt = {'url': 'https://wrong.com/video.mp4'}
|
||||||
|
result = ydl.process_ie_result(_make_result([fmt]), download=False)
|
||||||
|
self.assertIsNone(check_for_cookie_header(result), msg='http_headers cookies for wrong domain')
|
||||||
|
self.assertFalse(result.get('cookies'), msg='Cookies set in cookies field for wrong domain')
|
||||||
|
self.assertFalse(ydl.cookiejar.get_cookie_header(fmt['url']), msg='Cookies set in cookiejar for wrong domain')
|
||||||
|
|
||||||
|
|
||||||
if __name__ == '__main__':
|
if __name__ == '__main__':
|
||||||
unittest.main()
|
unittest.main()
|
||||||
|
|
|
@ -39,6 +39,27 @@ class TestYoutubeDLCookieJar(unittest.TestCase):
|
||||||
assert_cookie_has_value('HTTPONLY_COOKIE')
|
assert_cookie_has_value('HTTPONLY_COOKIE')
|
||||||
assert_cookie_has_value('JS_ACCESSIBLE_COOKIE')
|
assert_cookie_has_value('JS_ACCESSIBLE_COOKIE')
|
||||||
|
|
||||||
|
def test_malformed_cookies(self):
|
||||||
|
cookiejar = YoutubeDLCookieJar('./test/testdata/cookies/malformed_cookies.txt')
|
||||||
|
cookiejar.load(ignore_discard=True, ignore_expires=True)
|
||||||
|
# Cookies should be empty since all malformed cookie file entries
|
||||||
|
# will be ignored
|
||||||
|
self.assertFalse(cookiejar._cookies)
|
||||||
|
|
||||||
|
def test_get_cookie_header(self):
|
||||||
|
cookiejar = YoutubeDLCookieJar('./test/testdata/cookies/httponly_cookies.txt')
|
||||||
|
cookiejar.load(ignore_discard=True, ignore_expires=True)
|
||||||
|
header = cookiejar.get_cookie_header('https://www.foobar.foobar')
|
||||||
|
self.assertIn('HTTPONLY_COOKIE', header)
|
||||||
|
|
||||||
|
def test_get_cookies_for_url(self):
|
||||||
|
cookiejar = YoutubeDLCookieJar('./test/testdata/cookies/session_cookies.txt')
|
||||||
|
cookiejar.load(ignore_discard=True, ignore_expires=True)
|
||||||
|
cookies = cookiejar.get_cookies_for_url('https://www.foobar.foobar/')
|
||||||
|
self.assertEqual(len(cookies), 2)
|
||||||
|
cookies = cookiejar.get_cookies_for_url('https://foobar.foobar/')
|
||||||
|
self.assertFalse(cookies)
|
||||||
|
|
||||||
|
|
||||||
if __name__ == '__main__':
|
if __name__ == '__main__':
|
||||||
unittest.main()
|
unittest.main()
|
||||||
|
|
|
@ -8,7 +8,7 @@ import sys
|
||||||
import unittest
|
import unittest
|
||||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||||
|
|
||||||
from youtube_dl.aes import aes_decrypt, aes_encrypt, aes_cbc_decrypt, aes_cbc_encrypt, aes_decrypt_text
|
from youtube_dl.aes import aes_decrypt, aes_encrypt, aes_cbc_decrypt, aes_cbc_encrypt, aes_decrypt_text, aes_ecb_encrypt
|
||||||
from youtube_dl.utils import bytes_to_intlist, intlist_to_bytes
|
from youtube_dl.utils import bytes_to_intlist, intlist_to_bytes
|
||||||
import base64
|
import base64
|
||||||
|
|
||||||
|
@ -58,6 +58,13 @@ class TestAES(unittest.TestCase):
|
||||||
decrypted = (aes_decrypt_text(encrypted, password, 32))
|
decrypted = (aes_decrypt_text(encrypted, password, 32))
|
||||||
self.assertEqual(decrypted, self.secret_msg)
|
self.assertEqual(decrypted, self.secret_msg)
|
||||||
|
|
||||||
|
def test_ecb_encrypt(self):
|
||||||
|
data = bytes_to_intlist(self.secret_msg)
|
||||||
|
encrypted = intlist_to_bytes(aes_ecb_encrypt(data, self.key))
|
||||||
|
self.assertEqual(
|
||||||
|
encrypted,
|
||||||
|
b'\xaa\x86]\x81\x97>\x02\x92\x9d\x1bR[[L/u\xd3&\xd1(h\xde{\x81\x94\xba\x02\xae\xbd\xa6\xd0:')
|
||||||
|
|
||||||
|
|
||||||
if __name__ == '__main__':
|
if __name__ == '__main__':
|
||||||
unittest.main()
|
unittest.main()
|
||||||
|
|
|
@ -11,6 +11,7 @@ from test.helper import try_rm
|
||||||
|
|
||||||
|
|
||||||
from youtube_dl import YoutubeDL
|
from youtube_dl import YoutubeDL
|
||||||
|
from youtube_dl.utils import DownloadError
|
||||||
|
|
||||||
|
|
||||||
def _download_restricted(url, filename, age):
|
def _download_restricted(url, filename, age):
|
||||||
|
@ -26,7 +27,10 @@ def _download_restricted(url, filename, age):
|
||||||
ydl.add_default_info_extractors()
|
ydl.add_default_info_extractors()
|
||||||
json_filename = os.path.splitext(filename)[0] + '.info.json'
|
json_filename = os.path.splitext(filename)[0] + '.info.json'
|
||||||
try_rm(json_filename)
|
try_rm(json_filename)
|
||||||
ydl.download([url])
|
try:
|
||||||
|
ydl.download([url])
|
||||||
|
except DownloadError:
|
||||||
|
try_rm(json_filename)
|
||||||
res = os.path.exists(json_filename)
|
res = os.path.exists(json_filename)
|
||||||
try_rm(json_filename)
|
try_rm(json_filename)
|
||||||
return res
|
return res
|
||||||
|
@ -38,12 +42,12 @@ class TestAgeRestriction(unittest.TestCase):
|
||||||
self.assertFalse(_download_restricted(url, filename, age))
|
self.assertFalse(_download_restricted(url, filename, age))
|
||||||
|
|
||||||
def test_youtube(self):
|
def test_youtube(self):
|
||||||
self._assert_restricted('07FYdnEawAQ', '07FYdnEawAQ.mp4', 10)
|
self._assert_restricted('HtVdAasjOgU', 'HtVdAasjOgU.mp4', 10)
|
||||||
|
|
||||||
def test_youporn(self):
|
def test_youporn(self):
|
||||||
self._assert_restricted(
|
self._assert_restricted(
|
||||||
'http://www.youporn.com/watch/505835/sex-ed-is-it-safe-to-masturbate-daily/',
|
'https://www.youporn.com/watch/16715086/sex-ed-in-detention-18-asmr/',
|
||||||
'505835.mp4', 2, old_age=25)
|
'16715086.mp4', 2, old_age=25)
|
||||||
|
|
||||||
|
|
||||||
if __name__ == '__main__':
|
if __name__ == '__main__':
|
||||||
|
|
|
@ -31,16 +31,17 @@ class TestAllURLsMatching(unittest.TestCase):
|
||||||
|
|
||||||
def test_youtube_playlist_matching(self):
|
def test_youtube_playlist_matching(self):
|
||||||
assertPlaylist = lambda url: self.assertMatch(url, ['youtube:playlist'])
|
assertPlaylist = lambda url: self.assertMatch(url, ['youtube:playlist'])
|
||||||
|
assertTab = lambda url: self.assertMatch(url, ['youtube:tab'])
|
||||||
assertPlaylist('ECUl4u3cNGP61MdtwGTqZA0MreSaDybji8')
|
assertPlaylist('ECUl4u3cNGP61MdtwGTqZA0MreSaDybji8')
|
||||||
assertPlaylist('UUBABnxM4Ar9ten8Mdjj1j0Q') # 585
|
assertPlaylist('UUBABnxM4Ar9ten8Mdjj1j0Q') # 585
|
||||||
assertPlaylist('PL63F0C78739B09958')
|
assertPlaylist('PL63F0C78739B09958')
|
||||||
assertPlaylist('https://www.youtube.com/playlist?list=UUBABnxM4Ar9ten8Mdjj1j0Q')
|
assertTab('https://www.youtube.com/playlist?list=UUBABnxM4Ar9ten8Mdjj1j0Q')
|
||||||
assertPlaylist('https://www.youtube.com/course?list=ECUl4u3cNGP61MdtwGTqZA0MreSaDybji8')
|
assertTab('https://www.youtube.com/course?list=ECUl4u3cNGP61MdtwGTqZA0MreSaDybji8')
|
||||||
assertPlaylist('https://www.youtube.com/playlist?list=PLwP_SiAcdui0KVebT0mU9Apz359a4ubsC')
|
assertTab('https://www.youtube.com/playlist?list=PLwP_SiAcdui0KVebT0mU9Apz359a4ubsC')
|
||||||
assertPlaylist('https://www.youtube.com/watch?v=AV6J6_AeFEQ&playnext=1&list=PL4023E734DA416012') # 668
|
assertTab('https://www.youtube.com/watch?v=AV6J6_AeFEQ&playnext=1&list=PL4023E734DA416012') # 668
|
||||||
self.assertFalse('youtube:playlist' in self.matching_ies('PLtS2H6bU1M'))
|
self.assertFalse('youtube:playlist' in self.matching_ies('PLtS2H6bU1M'))
|
||||||
# Top tracks
|
# Top tracks
|
||||||
assertPlaylist('https://www.youtube.com/playlist?list=MCUS.20142101')
|
assertTab('https://www.youtube.com/playlist?list=MCUS.20142101')
|
||||||
|
|
||||||
def test_youtube_matching(self):
|
def test_youtube_matching(self):
|
||||||
self.assertTrue(YoutubeIE.suitable('PLtS2H6bU1M'))
|
self.assertTrue(YoutubeIE.suitable('PLtS2H6bU1M'))
|
||||||
|
@ -51,36 +52,24 @@ class TestAllURLsMatching(unittest.TestCase):
|
||||||
self.assertMatch('http://www.cleanvideosearch.com/media/action/yt/watch?videoId=8v_4O44sfjM', ['youtube'])
|
self.assertMatch('http://www.cleanvideosearch.com/media/action/yt/watch?videoId=8v_4O44sfjM', ['youtube'])
|
||||||
|
|
||||||
def test_youtube_channel_matching(self):
|
def test_youtube_channel_matching(self):
|
||||||
assertChannel = lambda url: self.assertMatch(url, ['youtube:channel'])
|
assertChannel = lambda url: self.assertMatch(url, ['youtube:tab'])
|
||||||
assertChannel('https://www.youtube.com/channel/HCtnHdj3df7iM')
|
assertChannel('https://www.youtube.com/channel/HCtnHdj3df7iM')
|
||||||
assertChannel('https://www.youtube.com/channel/HCtnHdj3df7iM?feature=gb_ch_rec')
|
assertChannel('https://www.youtube.com/channel/HCtnHdj3df7iM?feature=gb_ch_rec')
|
||||||
assertChannel('https://www.youtube.com/channel/HCtnHdj3df7iM/videos')
|
assertChannel('https://www.youtube.com/channel/HCtnHdj3df7iM/videos')
|
||||||
|
|
||||||
def test_youtube_user_matching(self):
|
def test_youtube_user_matching(self):
|
||||||
self.assertMatch('http://www.youtube.com/NASAgovVideo/videos', ['youtube:user'])
|
self.assertMatch('http://www.youtube.com/NASAgovVideo/videos', ['youtube:tab'])
|
||||||
|
|
||||||
def test_youtube_feeds(self):
|
def test_youtube_feeds(self):
|
||||||
self.assertMatch('https://www.youtube.com/feed/watch_later', ['youtube:watchlater'])
|
self.assertMatch('https://www.youtube.com/feed/library', ['youtube:tab'])
|
||||||
self.assertMatch('https://www.youtube.com/feed/subscriptions', ['youtube:subscriptions'])
|
self.assertMatch('https://www.youtube.com/feed/history', ['youtube:tab'])
|
||||||
self.assertMatch('https://www.youtube.com/feed/recommended', ['youtube:recommended'])
|
self.assertMatch('https://www.youtube.com/feed/watch_later', ['youtube:tab'])
|
||||||
self.assertMatch('https://www.youtube.com/my_favorites', ['youtube:favorites'])
|
self.assertMatch('https://www.youtube.com/feed/subscriptions', ['youtube:tab'])
|
||||||
|
|
||||||
def test_youtube_show_matching(self):
|
|
||||||
self.assertMatch('http://www.youtube.com/show/airdisasters', ['youtube:show'])
|
|
||||||
|
|
||||||
def test_youtube_search_matching(self):
|
def test_youtube_search_matching(self):
|
||||||
self.assertMatch('http://www.youtube.com/results?search_query=making+mustard', ['youtube:search_url'])
|
self.assertMatch('http://www.youtube.com/results?search_query=making+mustard', ['youtube:search_url'])
|
||||||
self.assertMatch('https://www.youtube.com/results?baz=bar&search_query=youtube-dl+test+video&filters=video&lclk=video', ['youtube:search_url'])
|
self.assertMatch('https://www.youtube.com/results?baz=bar&search_query=youtube-dl+test+video&filters=video&lclk=video', ['youtube:search_url'])
|
||||||
|
|
||||||
def test_youtube_extract(self):
|
|
||||||
assertExtractId = lambda url, id: self.assertEqual(YoutubeIE.extract_id(url), id)
|
|
||||||
assertExtractId('http://www.youtube.com/watch?&v=BaW_jenozKc', 'BaW_jenozKc')
|
|
||||||
assertExtractId('https://www.youtube.com/watch?&v=BaW_jenozKc', 'BaW_jenozKc')
|
|
||||||
assertExtractId('https://www.youtube.com/watch?feature=player_embedded&v=BaW_jenozKc', 'BaW_jenozKc')
|
|
||||||
assertExtractId('https://www.youtube.com/watch_popup?v=BaW_jenozKc', 'BaW_jenozKc')
|
|
||||||
assertExtractId('http://www.youtube.com/watch?v=BaW_jenozKcsharePLED17F32AD9753930', 'BaW_jenozKc')
|
|
||||||
assertExtractId('BaW_jenozKc', 'BaW_jenozKc')
|
|
||||||
|
|
||||||
def test_facebook_matching(self):
|
def test_facebook_matching(self):
|
||||||
self.assertTrue(FacebookIE.suitable('https://www.facebook.com/Shiniknoh#!/photo.php?v=10153317450565268'))
|
self.assertTrue(FacebookIE.suitable('https://www.facebook.com/Shiniknoh#!/photo.php?v=10153317450565268'))
|
||||||
self.assertTrue(FacebookIE.suitable('https://www.facebook.com/cindyweather?fref=ts#!/photo.php?v=10152183998945793'))
|
self.assertTrue(FacebookIE.suitable('https://www.facebook.com/cindyweather?fref=ts#!/photo.php?v=10152183998945793'))
|
||||||
|
|
|
@ -3,17 +3,18 @@
|
||||||
|
|
||||||
from __future__ import unicode_literals
|
from __future__ import unicode_literals
|
||||||
|
|
||||||
import shutil
|
|
||||||
|
|
||||||
# Allow direct execution
|
# Allow direct execution
|
||||||
import os
|
import os
|
||||||
import sys
|
import sys
|
||||||
import unittest
|
import unittest
|
||||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||||
|
|
||||||
|
import shutil
|
||||||
|
|
||||||
from test.helper import FakeYDL
|
from test.helper import FakeYDL
|
||||||
from youtube_dl.cache import Cache
|
from youtube_dl.cache import Cache
|
||||||
|
from youtube_dl.utils import version_tuple
|
||||||
|
from youtube_dl.version import __version__
|
||||||
|
|
||||||
|
|
||||||
def _is_empty(d):
|
def _is_empty(d):
|
||||||
|
@ -54,6 +55,17 @@ class TestCache(unittest.TestCase):
|
||||||
self.assertFalse(os.path.exists(self.test_dir))
|
self.assertFalse(os.path.exists(self.test_dir))
|
||||||
self.assertEqual(c.load('test_cache', 'k.'), None)
|
self.assertEqual(c.load('test_cache', 'k.'), None)
|
||||||
|
|
||||||
|
def test_cache_validation(self):
|
||||||
|
ydl = FakeYDL({
|
||||||
|
'cachedir': self.test_dir,
|
||||||
|
})
|
||||||
|
c = Cache(ydl)
|
||||||
|
obj = {'x': 1, 'y': ['ä', '\\a', True]}
|
||||||
|
c.store('test_cache', 'k.', obj)
|
||||||
|
self.assertEqual(c.load('test_cache', 'k.', min_ver='1970.01.01'), obj)
|
||||||
|
new_version = '.'.join(('%d' % ((v + 1) if i == 0 else v, )) for i, v in enumerate(version_tuple(__version__)))
|
||||||
|
self.assertIs(c.load('test_cache', 'k.', min_ver=new_version), None)
|
||||||
|
|
||||||
|
|
||||||
if __name__ == '__main__':
|
if __name__ == '__main__':
|
||||||
unittest.main()
|
unittest.main()
|
||||||
|
|
|
@ -11,6 +11,7 @@ sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||||
|
|
||||||
|
|
||||||
from youtube_dl.compat import (
|
from youtube_dl.compat import (
|
||||||
|
compat_casefold,
|
||||||
compat_getenv,
|
compat_getenv,
|
||||||
compat_setenv,
|
compat_setenv,
|
||||||
compat_etree_Element,
|
compat_etree_Element,
|
||||||
|
@ -22,6 +23,7 @@ from youtube_dl.compat import (
|
||||||
compat_urllib_parse_unquote,
|
compat_urllib_parse_unquote,
|
||||||
compat_urllib_parse_unquote_plus,
|
compat_urllib_parse_unquote_plus,
|
||||||
compat_urllib_parse_urlencode,
|
compat_urllib_parse_urlencode,
|
||||||
|
compat_urllib_request,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@ -47,10 +49,11 @@ class TestCompat(unittest.TestCase):
|
||||||
|
|
||||||
def test_all_present(self):
|
def test_all_present(self):
|
||||||
import youtube_dl.compat
|
import youtube_dl.compat
|
||||||
all_names = youtube_dl.compat.__all__
|
all_names = sorted(
|
||||||
present_names = set(filter(
|
youtube_dl.compat.__all__ + youtube_dl.compat.legacy)
|
||||||
|
present_names = set(map(compat_str, filter(
|
||||||
lambda c: '_' in c and not c.startswith('_'),
|
lambda c: '_' in c and not c.startswith('_'),
|
||||||
dir(youtube_dl.compat))) - set(['unicode_literals'])
|
dir(youtube_dl.compat)))) - set(['unicode_literals'])
|
||||||
self.assertEqual(all_names, sorted(present_names))
|
self.assertEqual(all_names, sorted(present_names))
|
||||||
|
|
||||||
def test_compat_urllib_parse_unquote(self):
|
def test_compat_urllib_parse_unquote(self):
|
||||||
|
@ -118,9 +121,34 @@ class TestCompat(unittest.TestCase):
|
||||||
<smil xmlns="http://www.w3.org/2001/SMIL20/Language"></smil>'''
|
<smil xmlns="http://www.w3.org/2001/SMIL20/Language"></smil>'''
|
||||||
compat_etree_fromstring(xml)
|
compat_etree_fromstring(xml)
|
||||||
|
|
||||||
def test_struct_unpack(self):
|
def test_compat_struct_unpack(self):
|
||||||
self.assertEqual(compat_struct_unpack('!B', b'\x00'), (0,))
|
self.assertEqual(compat_struct_unpack('!B', b'\x00'), (0,))
|
||||||
|
|
||||||
|
def test_compat_casefold(self):
|
||||||
|
if hasattr(compat_str, 'casefold'):
|
||||||
|
# don't bother to test str.casefold() (again)
|
||||||
|
return
|
||||||
|
# thanks https://bugs.python.org/file24232/casefolding.patch
|
||||||
|
self.assertEqual(compat_casefold('hello'), 'hello')
|
||||||
|
self.assertEqual(compat_casefold('hELlo'), 'hello')
|
||||||
|
self.assertEqual(compat_casefold('ß'), 'ss')
|
||||||
|
self.assertEqual(compat_casefold('fi'), 'fi')
|
||||||
|
self.assertEqual(compat_casefold('\u03a3'), '\u03c3')
|
||||||
|
self.assertEqual(compat_casefold('A\u0345\u03a3'), 'a\u03b9\u03c3')
|
||||||
|
|
||||||
|
def test_compat_urllib_request_Request(self):
|
||||||
|
self.assertEqual(
|
||||||
|
compat_urllib_request.Request('http://127.0.0.1', method='PUT').get_method(),
|
||||||
|
'PUT')
|
||||||
|
|
||||||
|
class PUTrequest(compat_urllib_request.Request):
|
||||||
|
def get_method(self):
|
||||||
|
return 'PUT'
|
||||||
|
|
||||||
|
self.assertEqual(
|
||||||
|
PUTrequest('http://127.0.0.1').get_method(),
|
||||||
|
'PUT')
|
||||||
|
|
||||||
|
|
||||||
if __name__ == '__main__':
|
if __name__ == '__main__':
|
||||||
unittest.main()
|
unittest.main()
|
||||||
|
|
|
@ -9,7 +9,6 @@ import unittest
|
||||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||||
|
|
||||||
from test.helper import (
|
from test.helper import (
|
||||||
assertGreaterEqual,
|
|
||||||
expect_warnings,
|
expect_warnings,
|
||||||
get_params,
|
get_params,
|
||||||
gettestcases,
|
gettestcases,
|
||||||
|
@ -20,26 +19,35 @@ from test.helper import (
|
||||||
|
|
||||||
|
|
||||||
import hashlib
|
import hashlib
|
||||||
import io
|
|
||||||
import json
|
import json
|
||||||
import socket
|
import socket
|
||||||
|
|
||||||
import youtube_dl.YoutubeDL
|
import youtube_dl.YoutubeDL
|
||||||
from youtube_dl.compat import (
|
from youtube_dl.compat import (
|
||||||
compat_http_client,
|
compat_http_client,
|
||||||
compat_urllib_error,
|
|
||||||
compat_HTTPError,
|
compat_HTTPError,
|
||||||
|
compat_open as open,
|
||||||
|
compat_urllib_error,
|
||||||
)
|
)
|
||||||
from youtube_dl.utils import (
|
from youtube_dl.utils import (
|
||||||
DownloadError,
|
DownloadError,
|
||||||
ExtractorError,
|
ExtractorError,
|
||||||
|
error_to_compat_str,
|
||||||
format_bytes,
|
format_bytes,
|
||||||
|
IDENTITY,
|
||||||
|
preferredencoding,
|
||||||
UnavailableVideoError,
|
UnavailableVideoError,
|
||||||
)
|
)
|
||||||
from youtube_dl.extractor import get_info_extractor
|
from youtube_dl.extractor import get_info_extractor
|
||||||
|
|
||||||
RETRIES = 3
|
RETRIES = 3
|
||||||
|
|
||||||
|
# Some unittest APIs require actual str
|
||||||
|
if not isinstance('TEST', str):
|
||||||
|
_encode_str = lambda s: s.encode(preferredencoding())
|
||||||
|
else:
|
||||||
|
_encode_str = IDENTITY
|
||||||
|
|
||||||
|
|
||||||
class YoutubeDL(youtube_dl.YoutubeDL):
|
class YoutubeDL(youtube_dl.YoutubeDL):
|
||||||
def __init__(self, *args, **kwargs):
|
def __init__(self, *args, **kwargs):
|
||||||
|
@ -100,27 +108,31 @@ def generator(test_case, tname):
|
||||||
|
|
||||||
def print_skipping(reason):
|
def print_skipping(reason):
|
||||||
print('Skipping %s: %s' % (test_case['name'], reason))
|
print('Skipping %s: %s' % (test_case['name'], reason))
|
||||||
|
self.skipTest(_encode_str(reason))
|
||||||
|
|
||||||
if not ie.working():
|
if not ie.working():
|
||||||
print_skipping('IE marked as not _WORKING')
|
print_skipping('IE marked as not _WORKING')
|
||||||
return
|
|
||||||
|
|
||||||
for tc in test_cases:
|
for tc in test_cases:
|
||||||
info_dict = tc.get('info_dict', {})
|
info_dict = tc.get('info_dict', {})
|
||||||
if not (info_dict.get('id') and info_dict.get('ext')):
|
if not (info_dict.get('id') and info_dict.get('ext')):
|
||||||
raise Exception('Test definition incorrect. The output file cannot be known. Are both \'id\' and \'ext\' keys present?')
|
raise Exception('Test definition (%s) requires both \'id\' and \'ext\' keys present to define the output file' % (tname, ))
|
||||||
|
|
||||||
if 'skip' in test_case:
|
if 'skip' in test_case:
|
||||||
print_skipping(test_case['skip'])
|
print_skipping(test_case['skip'])
|
||||||
return
|
|
||||||
for other_ie in other_ies:
|
for other_ie in other_ies:
|
||||||
if not other_ie.working():
|
if not other_ie.working():
|
||||||
print_skipping('test depends on %sIE, marked as not WORKING' % other_ie.ie_key())
|
print_skipping('test depends on %sIE, marked as not WORKING' % other_ie.ie_key())
|
||||||
return
|
|
||||||
|
|
||||||
params = get_params(test_case.get('params', {}))
|
params = get_params(test_case.get('params', {}))
|
||||||
params['outtmpl'] = tname + '_' + params['outtmpl']
|
params['outtmpl'] = tname + '_' + params['outtmpl']
|
||||||
if is_playlist and 'playlist' not in test_case:
|
if is_playlist and 'playlist' not in test_case:
|
||||||
params.setdefault('extract_flat', 'in_playlist')
|
params.setdefault('extract_flat', 'in_playlist')
|
||||||
|
params.setdefault('playlistend',
|
||||||
|
test_case['playlist_maxcount'] + 1
|
||||||
|
if test_case.get('playlist_maxcount')
|
||||||
|
else test_case.get('playlist_mincount'))
|
||||||
params.setdefault('skip_download', True)
|
params.setdefault('skip_download', True)
|
||||||
|
|
||||||
ydl = YoutubeDL(params, auto_init=False)
|
ydl = YoutubeDL(params, auto_init=False)
|
||||||
|
@ -146,6 +158,7 @@ def generator(test_case, tname):
|
||||||
try_rm(tc_filename)
|
try_rm(tc_filename)
|
||||||
try_rm(tc_filename + '.part')
|
try_rm(tc_filename + '.part')
|
||||||
try_rm(os.path.splitext(tc_filename)[0] + '.info.json')
|
try_rm(os.path.splitext(tc_filename)[0] + '.info.json')
|
||||||
|
|
||||||
try_rm_tcs_files()
|
try_rm_tcs_files()
|
||||||
try:
|
try:
|
||||||
try_num = 1
|
try_num = 1
|
||||||
|
@ -160,7 +173,9 @@ def generator(test_case, tname):
|
||||||
except (DownloadError, ExtractorError) as err:
|
except (DownloadError, ExtractorError) as err:
|
||||||
# Check if the exception is not a network related one
|
# Check if the exception is not a network related one
|
||||||
if not err.exc_info[0] in (compat_urllib_error.URLError, socket.timeout, UnavailableVideoError, compat_http_client.BadStatusLine) or (err.exc_info[0] == compat_HTTPError and err.exc_info[1].code == 503):
|
if not err.exc_info[0] in (compat_urllib_error.URLError, socket.timeout, UnavailableVideoError, compat_http_client.BadStatusLine) or (err.exc_info[0] == compat_HTTPError and err.exc_info[1].code == 503):
|
||||||
raise
|
msg = getattr(err, 'msg', error_to_compat_str(err))
|
||||||
|
err.msg = '%s (%s)' % (msg, tname, )
|
||||||
|
raise err
|
||||||
|
|
||||||
if try_num == RETRIES:
|
if try_num == RETRIES:
|
||||||
report_warning('%s failed due to network errors, skipping...' % tname)
|
report_warning('%s failed due to network errors, skipping...' % tname)
|
||||||
|
@ -178,13 +193,19 @@ def generator(test_case, tname):
|
||||||
expect_info_dict(self, res_dict, test_case.get('info_dict', {}))
|
expect_info_dict(self, res_dict, test_case.get('info_dict', {}))
|
||||||
|
|
||||||
if 'playlist_mincount' in test_case:
|
if 'playlist_mincount' in test_case:
|
||||||
assertGreaterEqual(
|
self.assertGreaterEqual(
|
||||||
self,
|
|
||||||
len(res_dict['entries']),
|
len(res_dict['entries']),
|
||||||
test_case['playlist_mincount'],
|
test_case['playlist_mincount'],
|
||||||
'Expected at least %d in playlist %s, but got only %d' % (
|
'Expected at least %d in playlist %s, but got only %d' % (
|
||||||
test_case['playlist_mincount'], test_case['url'],
|
test_case['playlist_mincount'], test_case['url'],
|
||||||
len(res_dict['entries'])))
|
len(res_dict['entries'])))
|
||||||
|
if 'playlist_maxcount' in test_case:
|
||||||
|
self.assertLessEqual(
|
||||||
|
len(res_dict['entries']),
|
||||||
|
test_case['playlist_maxcount'],
|
||||||
|
'Expected at most %d in playlist %s, but got %d' % (
|
||||||
|
test_case['playlist_maxcount'], test_case['url'],
|
||||||
|
len(res_dict['entries'])))
|
||||||
if 'playlist_count' in test_case:
|
if 'playlist_count' in test_case:
|
||||||
self.assertEqual(
|
self.assertEqual(
|
||||||
len(res_dict['entries']),
|
len(res_dict['entries']),
|
||||||
|
@ -209,7 +230,15 @@ def generator(test_case, tname):
|
||||||
# First, check test cases' data against extracted data alone
|
# First, check test cases' data against extracted data alone
|
||||||
expect_info_dict(self, tc_res_dict, tc.get('info_dict', {}))
|
expect_info_dict(self, tc_res_dict, tc.get('info_dict', {}))
|
||||||
# Now, check downloaded file consistency
|
# Now, check downloaded file consistency
|
||||||
|
# support test-case with volatile ID, signalled by regexp value
|
||||||
|
if tc.get('info_dict', {}).get('id', '').startswith('re:'):
|
||||||
|
test_id = tc['info_dict']['id']
|
||||||
|
tc['info_dict']['id'] = tc_res_dict['id']
|
||||||
|
else:
|
||||||
|
test_id = None
|
||||||
tc_filename = get_tc_filename(tc)
|
tc_filename = get_tc_filename(tc)
|
||||||
|
if test_id:
|
||||||
|
tc['info_dict']['id'] = test_id
|
||||||
if not test_case.get('params', {}).get('skip_download', False):
|
if not test_case.get('params', {}).get('skip_download', False):
|
||||||
self.assertTrue(os.path.exists(tc_filename), msg='Missing file ' + tc_filename)
|
self.assertTrue(os.path.exists(tc_filename), msg='Missing file ' + tc_filename)
|
||||||
self.assertTrue(tc_filename in finished_hook_called)
|
self.assertTrue(tc_filename in finished_hook_called)
|
||||||
|
@ -218,8 +247,8 @@ def generator(test_case, tname):
|
||||||
if params.get('test'):
|
if params.get('test'):
|
||||||
expected_minsize = max(expected_minsize, 10000)
|
expected_minsize = max(expected_minsize, 10000)
|
||||||
got_fsize = os.path.getsize(tc_filename)
|
got_fsize = os.path.getsize(tc_filename)
|
||||||
assertGreaterEqual(
|
self.assertGreaterEqual(
|
||||||
self, got_fsize, expected_minsize,
|
got_fsize, expected_minsize,
|
||||||
'Expected %s to be at least %s, but it\'s only %s ' %
|
'Expected %s to be at least %s, but it\'s only %s ' %
|
||||||
(tc_filename, format_bytes(expected_minsize),
|
(tc_filename, format_bytes(expected_minsize),
|
||||||
format_bytes(got_fsize)))
|
format_bytes(got_fsize)))
|
||||||
|
@ -232,7 +261,7 @@ def generator(test_case, tname):
|
||||||
self.assertTrue(
|
self.assertTrue(
|
||||||
os.path.exists(info_json_fn),
|
os.path.exists(info_json_fn),
|
||||||
'Missing info file %s' % info_json_fn)
|
'Missing info file %s' % info_json_fn)
|
||||||
with io.open(info_json_fn, encoding='utf-8') as infof:
|
with open(info_json_fn, encoding='utf-8') as infof:
|
||||||
info_dict = json.load(infof)
|
info_dict = json.load(infof)
|
||||||
expect_info_dict(self, info_dict, tc.get('info_dict', {}))
|
expect_info_dict(self, info_dict, tc.get('info_dict', {}))
|
||||||
finally:
|
finally:
|
||||||
|
|
272
test/test_downloader_external.py
Normal file
272
test/test_downloader_external.py
Normal file
|
@ -0,0 +1,272 @@
|
||||||
|
#!/usr/bin/env python
|
||||||
|
# coding: utf-8
|
||||||
|
from __future__ import unicode_literals
|
||||||
|
|
||||||
|
# Allow direct execution
|
||||||
|
import os
|
||||||
|
import re
|
||||||
|
import sys
|
||||||
|
import subprocess
|
||||||
|
import unittest
|
||||||
|
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||||
|
|
||||||
|
from test.helper import (
|
||||||
|
FakeLogger,
|
||||||
|
FakeYDL,
|
||||||
|
http_server_port,
|
||||||
|
try_rm,
|
||||||
|
)
|
||||||
|
from youtube_dl import YoutubeDL
|
||||||
|
from youtube_dl.compat import (
|
||||||
|
compat_contextlib_suppress,
|
||||||
|
compat_http_cookiejar_Cookie,
|
||||||
|
compat_http_server,
|
||||||
|
compat_kwargs,
|
||||||
|
)
|
||||||
|
from youtube_dl.utils import (
|
||||||
|
encodeFilename,
|
||||||
|
join_nonempty,
|
||||||
|
)
|
||||||
|
from youtube_dl.downloader.external import (
|
||||||
|
Aria2cFD,
|
||||||
|
Aria2pFD,
|
||||||
|
AxelFD,
|
||||||
|
CurlFD,
|
||||||
|
FFmpegFD,
|
||||||
|
HttpieFD,
|
||||||
|
WgetFD,
|
||||||
|
)
|
||||||
|
from youtube_dl.postprocessor import (
|
||||||
|
FFmpegPostProcessor,
|
||||||
|
)
|
||||||
|
import threading
|
||||||
|
|
||||||
|
TEST_SIZE = 10 * 1024
|
||||||
|
|
||||||
|
TEST_COOKIE = {
|
||||||
|
'version': 0,
|
||||||
|
'name': 'test',
|
||||||
|
'value': 'ytdlp',
|
||||||
|
'port': None,
|
||||||
|
'port_specified': False,
|
||||||
|
'domain': '.example.com',
|
||||||
|
'domain_specified': True,
|
||||||
|
'domain_initial_dot': False,
|
||||||
|
'path': '/',
|
||||||
|
'path_specified': True,
|
||||||
|
'secure': False,
|
||||||
|
'expires': None,
|
||||||
|
'discard': False,
|
||||||
|
'comment': None,
|
||||||
|
'comment_url': None,
|
||||||
|
'rest': {},
|
||||||
|
}
|
||||||
|
|
||||||
|
TEST_COOKIE_VALUE = join_nonempty('name', 'value', delim='=', from_dict=TEST_COOKIE)
|
||||||
|
|
||||||
|
TEST_INFO = {'url': 'http://www.example.com/'}
|
||||||
|
|
||||||
|
|
||||||
|
def cookiejar_Cookie(**cookie_args):
|
||||||
|
return compat_http_cookiejar_Cookie(**compat_kwargs(cookie_args))
|
||||||
|
|
||||||
|
|
||||||
|
def ifExternalFDAvailable(externalFD):
|
||||||
|
return unittest.skipUnless(externalFD.available(),
|
||||||
|
externalFD.get_basename() + ' not found')
|
||||||
|
|
||||||
|
|
||||||
|
class HTTPTestRequestHandler(compat_http_server.BaseHTTPRequestHandler):
|
||||||
|
def log_message(self, format, *args):
|
||||||
|
pass
|
||||||
|
|
||||||
|
def send_content_range(self, total=None):
|
||||||
|
range_header = self.headers.get('Range')
|
||||||
|
start = end = None
|
||||||
|
if range_header:
|
||||||
|
mobj = re.match(r'bytes=(\d+)-(\d+)', range_header)
|
||||||
|
if mobj:
|
||||||
|
start, end = (int(mobj.group(i)) for i in (1, 2))
|
||||||
|
valid_range = start is not None and end is not None
|
||||||
|
if valid_range:
|
||||||
|
content_range = 'bytes %d-%d' % (start, end)
|
||||||
|
if total:
|
||||||
|
content_range += '/%d' % total
|
||||||
|
self.send_header('Content-Range', content_range)
|
||||||
|
return (end - start + 1) if valid_range else total
|
||||||
|
|
||||||
|
def serve(self, range=True, content_length=True):
|
||||||
|
self.send_response(200)
|
||||||
|
self.send_header('Content-Type', 'video/mp4')
|
||||||
|
size = TEST_SIZE
|
||||||
|
if range:
|
||||||
|
size = self.send_content_range(TEST_SIZE)
|
||||||
|
if content_length:
|
||||||
|
self.send_header('Content-Length', size)
|
||||||
|
self.end_headers()
|
||||||
|
self.wfile.write(b'#' * size)
|
||||||
|
|
||||||
|
def do_GET(self):
|
||||||
|
if self.path == '/regular':
|
||||||
|
self.serve()
|
||||||
|
elif self.path == '/no-content-length':
|
||||||
|
self.serve(content_length=False)
|
||||||
|
elif self.path == '/no-range':
|
||||||
|
self.serve(range=False)
|
||||||
|
elif self.path == '/no-range-no-content-length':
|
||||||
|
self.serve(range=False, content_length=False)
|
||||||
|
else:
|
||||||
|
assert False, 'unrecognised server path'
|
||||||
|
|
||||||
|
|
||||||
|
@ifExternalFDAvailable(Aria2pFD)
|
||||||
|
class TestAria2pFD(unittest.TestCase):
|
||||||
|
def setUp(self):
|
||||||
|
self.httpd = compat_http_server.HTTPServer(
|
||||||
|
('127.0.0.1', 0), HTTPTestRequestHandler)
|
||||||
|
self.port = http_server_port(self.httpd)
|
||||||
|
self.server_thread = threading.Thread(target=self.httpd.serve_forever)
|
||||||
|
self.server_thread.daemon = True
|
||||||
|
self.server_thread.start()
|
||||||
|
|
||||||
|
def download(self, params, ep):
|
||||||
|
with subprocess.Popen(
|
||||||
|
['aria2c', '--enable-rpc'],
|
||||||
|
stdout=subprocess.DEVNULL,
|
||||||
|
stderr=subprocess.DEVNULL
|
||||||
|
) as process:
|
||||||
|
if not process.poll():
|
||||||
|
filename = 'testfile.mp4'
|
||||||
|
params['logger'] = FakeLogger()
|
||||||
|
params['outtmpl'] = filename
|
||||||
|
ydl = YoutubeDL(params)
|
||||||
|
try_rm(encodeFilename(filename))
|
||||||
|
self.assertEqual(ydl.download(['http://127.0.0.1:%d/%s' % (self.port, ep)]), 0)
|
||||||
|
self.assertEqual(os.path.getsize(encodeFilename(filename)), TEST_SIZE)
|
||||||
|
try_rm(encodeFilename(filename))
|
||||||
|
process.kill()
|
||||||
|
|
||||||
|
def download_all(self, params):
|
||||||
|
for ep in ('regular', 'no-content-length', 'no-range', 'no-range-no-content-length'):
|
||||||
|
self.download(params, ep)
|
||||||
|
|
||||||
|
def test_regular(self):
|
||||||
|
self.download_all({'external_downloader': 'aria2p'})
|
||||||
|
|
||||||
|
def test_chunked(self):
|
||||||
|
self.download_all({
|
||||||
|
'external_downloader': 'aria2p',
|
||||||
|
'http_chunk_size': 1000,
|
||||||
|
})
|
||||||
|
|
||||||
|
|
||||||
|
@ifExternalFDAvailable(HttpieFD)
|
||||||
|
class TestHttpieFD(unittest.TestCase):
|
||||||
|
def test_make_cmd(self):
|
||||||
|
with FakeYDL() as ydl:
|
||||||
|
downloader = HttpieFD(ydl, {})
|
||||||
|
self.assertEqual(
|
||||||
|
downloader._make_cmd('test', TEST_INFO),
|
||||||
|
['http', '--download', '--output', 'test', 'http://www.example.com/'])
|
||||||
|
|
||||||
|
# Test cookie header is added
|
||||||
|
ydl.cookiejar.set_cookie(cookiejar_Cookie(**TEST_COOKIE))
|
||||||
|
self.assertEqual(
|
||||||
|
downloader._make_cmd('test', TEST_INFO),
|
||||||
|
['http', '--download', '--output', 'test',
|
||||||
|
'http://www.example.com/', 'Cookie:' + TEST_COOKIE_VALUE])
|
||||||
|
|
||||||
|
|
||||||
|
@ifExternalFDAvailable(AxelFD)
|
||||||
|
class TestAxelFD(unittest.TestCase):
|
||||||
|
def test_make_cmd(self):
|
||||||
|
with FakeYDL() as ydl:
|
||||||
|
downloader = AxelFD(ydl, {})
|
||||||
|
self.assertEqual(
|
||||||
|
downloader._make_cmd('test', TEST_INFO),
|
||||||
|
['axel', '-o', 'test', '--', 'http://www.example.com/'])
|
||||||
|
|
||||||
|
# Test cookie header is added
|
||||||
|
ydl.cookiejar.set_cookie(cookiejar_Cookie(**TEST_COOKIE))
|
||||||
|
self.assertEqual(
|
||||||
|
downloader._make_cmd('test', TEST_INFO),
|
||||||
|
['axel', '-o', 'test', '-H', 'Cookie: ' + TEST_COOKIE_VALUE,
|
||||||
|
'--max-redirect=0', '--', 'http://www.example.com/'])
|
||||||
|
|
||||||
|
|
||||||
|
@ifExternalFDAvailable(WgetFD)
|
||||||
|
class TestWgetFD(unittest.TestCase):
|
||||||
|
def test_make_cmd(self):
|
||||||
|
with FakeYDL() as ydl:
|
||||||
|
downloader = WgetFD(ydl, {})
|
||||||
|
self.assertNotIn('--load-cookies', downloader._make_cmd('test', TEST_INFO))
|
||||||
|
# Test cookiejar tempfile arg is added
|
||||||
|
ydl.cookiejar.set_cookie(cookiejar_Cookie(**TEST_COOKIE))
|
||||||
|
self.assertIn('--load-cookies', downloader._make_cmd('test', TEST_INFO))
|
||||||
|
|
||||||
|
|
||||||
|
@ifExternalFDAvailable(CurlFD)
|
||||||
|
class TestCurlFD(unittest.TestCase):
|
||||||
|
def test_make_cmd(self):
|
||||||
|
with FakeYDL() as ydl:
|
||||||
|
downloader = CurlFD(ydl, {})
|
||||||
|
self.assertNotIn('--cookie', downloader._make_cmd('test', TEST_INFO))
|
||||||
|
# Test cookie header is added
|
||||||
|
ydl.cookiejar.set_cookie(cookiejar_Cookie(**TEST_COOKIE))
|
||||||
|
self.assertIn('--cookie', downloader._make_cmd('test', TEST_INFO))
|
||||||
|
self.assertIn(TEST_COOKIE_VALUE, downloader._make_cmd('test', TEST_INFO))
|
||||||
|
|
||||||
|
|
||||||
|
@ifExternalFDAvailable(Aria2cFD)
|
||||||
|
class TestAria2cFD(unittest.TestCase):
|
||||||
|
def test_make_cmd(self):
|
||||||
|
with FakeYDL() as ydl:
|
||||||
|
downloader = Aria2cFD(ydl, {})
|
||||||
|
downloader._make_cmd('test', TEST_INFO)
|
||||||
|
self.assertFalse(hasattr(downloader, '_cookies_tempfile'))
|
||||||
|
|
||||||
|
# Test cookiejar tempfile arg is added
|
||||||
|
ydl.cookiejar.set_cookie(cookiejar_Cookie(**TEST_COOKIE))
|
||||||
|
cmd = downloader._make_cmd('test', TEST_INFO)
|
||||||
|
self.assertIn('--load-cookies=%s' % downloader._cookies_tempfile, cmd)
|
||||||
|
|
||||||
|
|
||||||
|
# Handle delegated availability
|
||||||
|
def ifFFmpegFDAvailable(externalFD):
|
||||||
|
# raise SkipTest, or set False!
|
||||||
|
avail = ifExternalFDAvailable(externalFD) and False
|
||||||
|
with compat_contextlib_suppress(Exception):
|
||||||
|
avail = FFmpegPostProcessor(downloader=None).available
|
||||||
|
return unittest.skipUnless(
|
||||||
|
avail, externalFD.get_basename() + ' not found')
|
||||||
|
|
||||||
|
|
||||||
|
@ifFFmpegFDAvailable(FFmpegFD)
|
||||||
|
class TestFFmpegFD(unittest.TestCase):
|
||||||
|
_args = []
|
||||||
|
|
||||||
|
def _test_cmd(self, args):
|
||||||
|
self._args = args
|
||||||
|
|
||||||
|
def test_make_cmd(self):
|
||||||
|
with FakeYDL() as ydl:
|
||||||
|
downloader = FFmpegFD(ydl, {})
|
||||||
|
downloader._debug_cmd = self._test_cmd
|
||||||
|
info_dict = TEST_INFO.copy()
|
||||||
|
info_dict['ext'] = 'mp4'
|
||||||
|
|
||||||
|
downloader._call_downloader('test', info_dict)
|
||||||
|
self.assertEqual(self._args, [
|
||||||
|
'ffmpeg', '-y', '-i', 'http://www.example.com/',
|
||||||
|
'-c', 'copy', '-f', 'mp4', 'file:test'])
|
||||||
|
|
||||||
|
# Test cookies arg is added
|
||||||
|
ydl.cookiejar.set_cookie(cookiejar_Cookie(**TEST_COOKIE))
|
||||||
|
downloader._call_downloader('test', info_dict)
|
||||||
|
self.assertEqual(self._args, [
|
||||||
|
'ffmpeg', '-y', '-cookies', TEST_COOKIE_VALUE + '; path=/; domain=.example.com;\r\n',
|
||||||
|
'-i', 'http://www.example.com/', '-c', 'copy', '-f', 'mp4', 'file:test'])
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
unittest.main()
|
|
@ -9,7 +9,11 @@ import sys
|
||||||
import unittest
|
import unittest
|
||||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||||
|
|
||||||
from test.helper import http_server_port, try_rm
|
from test.helper import (
|
||||||
|
FakeLogger,
|
||||||
|
http_server_port,
|
||||||
|
try_rm,
|
||||||
|
)
|
||||||
from youtube_dl import YoutubeDL
|
from youtube_dl import YoutubeDL
|
||||||
from youtube_dl.compat import compat_http_server
|
from youtube_dl.compat import compat_http_server
|
||||||
from youtube_dl.downloader.http import HttpFD
|
from youtube_dl.downloader.http import HttpFD
|
||||||
|
@ -66,17 +70,6 @@ class HTTPTestRequestHandler(compat_http_server.BaseHTTPRequestHandler):
|
||||||
assert False
|
assert False
|
||||||
|
|
||||||
|
|
||||||
class FakeLogger(object):
|
|
||||||
def debug(self, msg):
|
|
||||||
pass
|
|
||||||
|
|
||||||
def warning(self, msg):
|
|
||||||
pass
|
|
||||||
|
|
||||||
def error(self, msg):
|
|
||||||
pass
|
|
||||||
|
|
||||||
|
|
||||||
class TestHttpFD(unittest.TestCase):
|
class TestHttpFD(unittest.TestCase):
|
||||||
def setUp(self):
|
def setUp(self):
|
||||||
self.httpd = compat_http_server.HTTPServer(
|
self.httpd = compat_http_server.HTTPServer(
|
||||||
|
@ -95,7 +88,7 @@ class TestHttpFD(unittest.TestCase):
|
||||||
self.assertTrue(downloader.real_download(filename, {
|
self.assertTrue(downloader.real_download(filename, {
|
||||||
'url': 'http://127.0.0.1:%d/%s' % (self.port, ep),
|
'url': 'http://127.0.0.1:%d/%s' % (self.port, ep),
|
||||||
}))
|
}))
|
||||||
self.assertEqual(os.path.getsize(encodeFilename(filename)), TEST_SIZE)
|
self.assertEqual(os.path.getsize(encodeFilename(filename)), TEST_SIZE, ep)
|
||||||
try_rm(encodeFilename(filename))
|
try_rm(encodeFilename(filename))
|
||||||
|
|
||||||
def download_all(self, params):
|
def download_all(self, params):
|
||||||
|
|
|
@ -8,37 +8,55 @@ import unittest
|
||||||
import sys
|
import sys
|
||||||
import os
|
import os
|
||||||
import subprocess
|
import subprocess
|
||||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
|
||||||
|
|
||||||
from youtube_dl.utils import encodeArgument
|
|
||||||
|
|
||||||
rootDir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
rootDir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||||
|
|
||||||
|
sys.path.insert(0, rootDir)
|
||||||
|
|
||||||
try:
|
from youtube_dl.compat import compat_register_utf8, compat_subprocess_get_DEVNULL
|
||||||
_DEV_NULL = subprocess.DEVNULL
|
from youtube_dl.utils import encodeArgument
|
||||||
except AttributeError:
|
|
||||||
_DEV_NULL = open(os.devnull, 'wb')
|
compat_register_utf8()
|
||||||
|
|
||||||
|
|
||||||
|
_DEV_NULL = compat_subprocess_get_DEVNULL()
|
||||||
|
|
||||||
|
|
||||||
class TestExecution(unittest.TestCase):
|
class TestExecution(unittest.TestCase):
|
||||||
|
def setUp(self):
|
||||||
|
self.module = 'youtube_dl'
|
||||||
|
if sys.version_info < (2, 7):
|
||||||
|
self.module += '.__main__'
|
||||||
|
|
||||||
def test_import(self):
|
def test_import(self):
|
||||||
subprocess.check_call([sys.executable, '-c', 'import youtube_dl'], cwd=rootDir)
|
subprocess.check_call([sys.executable, '-c', 'import youtube_dl'], cwd=rootDir)
|
||||||
|
|
||||||
def test_module_exec(self):
|
def test_module_exec(self):
|
||||||
if sys.version_info >= (2, 7): # Python 2.6 doesn't support package execution
|
subprocess.check_call([sys.executable, '-m', self.module, '--version'], cwd=rootDir, stdout=_DEV_NULL)
|
||||||
subprocess.check_call([sys.executable, '-m', 'youtube_dl', '--version'], cwd=rootDir, stdout=_DEV_NULL)
|
|
||||||
|
|
||||||
def test_main_exec(self):
|
def test_main_exec(self):
|
||||||
subprocess.check_call([sys.executable, 'youtube_dl/__main__.py', '--version'], cwd=rootDir, stdout=_DEV_NULL)
|
subprocess.check_call([sys.executable, os.path.normpath('youtube_dl/__main__.py'), '--version'], cwd=rootDir, stdout=_DEV_NULL)
|
||||||
|
|
||||||
def test_cmdline_umlauts(self):
|
def test_cmdline_umlauts(self):
|
||||||
|
os.environ['PYTHONIOENCODING'] = 'utf-8'
|
||||||
p = subprocess.Popen(
|
p = subprocess.Popen(
|
||||||
[sys.executable, 'youtube_dl/__main__.py', encodeArgument('ä'), '--version'],
|
[sys.executable, '-m', self.module, encodeArgument('ä'), '--version'],
|
||||||
cwd=rootDir, stdout=_DEV_NULL, stderr=subprocess.PIPE)
|
cwd=rootDir, stdout=_DEV_NULL, stderr=subprocess.PIPE)
|
||||||
_, stderr = p.communicate()
|
_, stderr = p.communicate()
|
||||||
self.assertFalse(stderr)
|
self.assertFalse(stderr)
|
||||||
|
|
||||||
|
def test_lazy_extractors(self):
|
||||||
|
lazy_extractors = os.path.normpath('youtube_dl/extractor/lazy_extractors.py')
|
||||||
|
try:
|
||||||
|
subprocess.check_call([sys.executable, os.path.normpath('devscripts/make_lazy_extractors.py'), lazy_extractors], cwd=rootDir, stdout=_DEV_NULL)
|
||||||
|
subprocess.check_call([sys.executable, os.path.normpath('test/test_all_urls.py')], cwd=rootDir, stdout=_DEV_NULL)
|
||||||
|
finally:
|
||||||
|
for x in ('', 'c') if sys.version_info[0] < 3 else ('',):
|
||||||
|
try:
|
||||||
|
os.remove(lazy_extractors + x)
|
||||||
|
except OSError:
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
if __name__ == '__main__':
|
if __name__ == '__main__':
|
||||||
unittest.main()
|
unittest.main()
|
||||||
|
|
|
@ -8,30 +8,163 @@ import sys
|
||||||
import unittest
|
import unittest
|
||||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||||
|
|
||||||
from test.helper import http_server_port
|
import contextlib
|
||||||
from youtube_dl import YoutubeDL
|
import gzip
|
||||||
from youtube_dl.compat import compat_http_server, compat_urllib_request
|
import io
|
||||||
import ssl
|
import ssl
|
||||||
|
import tempfile
|
||||||
import threading
|
import threading
|
||||||
|
import zlib
|
||||||
|
|
||||||
|
# avoid deprecated alias assertRaisesRegexp
|
||||||
|
if hasattr(unittest.TestCase, 'assertRaisesRegex'):
|
||||||
|
unittest.TestCase.assertRaisesRegexp = unittest.TestCase.assertRaisesRegex
|
||||||
|
|
||||||
|
try:
|
||||||
|
import brotli
|
||||||
|
except ImportError:
|
||||||
|
brotli = None
|
||||||
|
try:
|
||||||
|
from urllib.request import pathname2url
|
||||||
|
except ImportError:
|
||||||
|
from urllib import pathname2url
|
||||||
|
|
||||||
|
from youtube_dl.compat import (
|
||||||
|
compat_http_cookiejar_Cookie,
|
||||||
|
compat_http_server,
|
||||||
|
compat_str as str,
|
||||||
|
compat_urllib_error,
|
||||||
|
compat_urllib_HTTPError,
|
||||||
|
compat_urllib_parse,
|
||||||
|
compat_urllib_request,
|
||||||
|
)
|
||||||
|
|
||||||
|
from youtube_dl.utils import (
|
||||||
|
sanitized_Request,
|
||||||
|
update_Request,
|
||||||
|
urlencode_postdata,
|
||||||
|
)
|
||||||
|
|
||||||
|
from test.helper import (
|
||||||
|
expectedFailureIf,
|
||||||
|
FakeYDL,
|
||||||
|
FakeLogger,
|
||||||
|
http_server_port,
|
||||||
|
)
|
||||||
|
from youtube_dl import YoutubeDL
|
||||||
|
|
||||||
TEST_DIR = os.path.dirname(os.path.abspath(__file__))
|
TEST_DIR = os.path.dirname(os.path.abspath(__file__))
|
||||||
|
|
||||||
|
|
||||||
class HTTPTestRequestHandler(compat_http_server.BaseHTTPRequestHandler):
|
class HTTPTestRequestHandler(compat_http_server.BaseHTTPRequestHandler):
|
||||||
|
protocol_version = 'HTTP/1.1'
|
||||||
|
|
||||||
|
# work-around old/new -style class inheritance
|
||||||
|
def super(self, meth_name, *args, **kwargs):
|
||||||
|
from types import MethodType
|
||||||
|
try:
|
||||||
|
super()
|
||||||
|
fn = lambda s, m, *a, **k: getattr(super(), m)(*a, **k)
|
||||||
|
except TypeError:
|
||||||
|
fn = lambda s, m, *a, **k: getattr(compat_http_server.BaseHTTPRequestHandler, m)(s, *a, **k)
|
||||||
|
self.super = MethodType(fn, self)
|
||||||
|
return self.super(meth_name, *args, **kwargs)
|
||||||
|
|
||||||
def log_message(self, format, *args):
|
def log_message(self, format, *args):
|
||||||
pass
|
pass
|
||||||
|
|
||||||
|
def _headers(self):
|
||||||
|
payload = str(self.headers).encode('utf-8')
|
||||||
|
self.send_response(200)
|
||||||
|
self.send_header('Content-Type', 'application/json')
|
||||||
|
self.send_header('Content-Length', str(len(payload)))
|
||||||
|
self.end_headers()
|
||||||
|
self.wfile.write(payload)
|
||||||
|
|
||||||
|
def _redirect(self):
|
||||||
|
self.send_response(int(self.path[len('/redirect_'):]))
|
||||||
|
self.send_header('Location', '/method')
|
||||||
|
self.send_header('Content-Length', '0')
|
||||||
|
self.end_headers()
|
||||||
|
|
||||||
|
def _method(self, method, payload=None):
|
||||||
|
self.send_response(200)
|
||||||
|
self.send_header('Content-Length', str(len(payload or '')))
|
||||||
|
self.send_header('Method', method)
|
||||||
|
self.end_headers()
|
||||||
|
if payload:
|
||||||
|
self.wfile.write(payload)
|
||||||
|
|
||||||
|
def _status(self, status):
|
||||||
|
payload = '<html>{0} NOT FOUND</html>'.format(status).encode('utf-8')
|
||||||
|
self.send_response(int(status))
|
||||||
|
self.send_header('Content-Type', 'text/html; charset=utf-8')
|
||||||
|
self.send_header('Content-Length', str(len(payload)))
|
||||||
|
self.end_headers()
|
||||||
|
self.wfile.write(payload)
|
||||||
|
|
||||||
|
def _read_data(self):
|
||||||
|
if 'Content-Length' in self.headers:
|
||||||
|
return self.rfile.read(int(self.headers['Content-Length']))
|
||||||
|
|
||||||
|
def _test_url(self, path, host='127.0.0.1', scheme='http', port=None):
|
||||||
|
return '{0}://{1}:{2}/{3}'.format(
|
||||||
|
scheme, host,
|
||||||
|
port if port is not None
|
||||||
|
else http_server_port(self.server), path)
|
||||||
|
|
||||||
|
def do_POST(self):
|
||||||
|
data = self._read_data()
|
||||||
|
if self.path.startswith('/redirect_'):
|
||||||
|
self._redirect()
|
||||||
|
elif self.path.startswith('/method'):
|
||||||
|
self._method('POST', data)
|
||||||
|
elif self.path.startswith('/headers'):
|
||||||
|
self._headers()
|
||||||
|
else:
|
||||||
|
self._status(404)
|
||||||
|
|
||||||
|
def do_HEAD(self):
|
||||||
|
if self.path.startswith('/redirect_'):
|
||||||
|
self._redirect()
|
||||||
|
elif self.path.startswith('/method'):
|
||||||
|
self._method('HEAD')
|
||||||
|
else:
|
||||||
|
self._status(404)
|
||||||
|
|
||||||
|
def do_PUT(self):
|
||||||
|
data = self._read_data()
|
||||||
|
if self.path.startswith('/redirect_'):
|
||||||
|
self._redirect()
|
||||||
|
elif self.path.startswith('/method'):
|
||||||
|
self._method('PUT', data)
|
||||||
|
else:
|
||||||
|
self._status(404)
|
||||||
|
|
||||||
def do_GET(self):
|
def do_GET(self):
|
||||||
|
|
||||||
|
def respond(payload=b'<html><video src="/vid.mp4" /></html>',
|
||||||
|
payload_type='text/html; charset=utf-8',
|
||||||
|
payload_encoding=None,
|
||||||
|
resp_code=200):
|
||||||
|
self.send_response(resp_code)
|
||||||
|
self.send_header('Content-Type', payload_type)
|
||||||
|
if payload_encoding:
|
||||||
|
self.send_header('Content-Encoding', payload_encoding)
|
||||||
|
self.send_header('Content-Length', str(len(payload))) # required for persistent connections
|
||||||
|
self.end_headers()
|
||||||
|
self.wfile.write(payload)
|
||||||
|
|
||||||
|
def gzip_compress(p):
|
||||||
|
buf = io.BytesIO()
|
||||||
|
with contextlib.closing(gzip.GzipFile(fileobj=buf, mode='wb')) as f:
|
||||||
|
f.write(p)
|
||||||
|
return buf.getvalue()
|
||||||
|
|
||||||
if self.path == '/video.html':
|
if self.path == '/video.html':
|
||||||
self.send_response(200)
|
respond()
|
||||||
self.send_header('Content-Type', 'text/html; charset=utf-8')
|
|
||||||
self.end_headers()
|
|
||||||
self.wfile.write(b'<html><video src="/vid.mp4" /></html>')
|
|
||||||
elif self.path == '/vid.mp4':
|
elif self.path == '/vid.mp4':
|
||||||
self.send_response(200)
|
respond(b'\x00\x00\x00\x00\x20\x66\x74[video]', 'video/mp4')
|
||||||
self.send_header('Content-Type', 'video/mp4')
|
|
||||||
self.end_headers()
|
|
||||||
self.wfile.write(b'\x00\x00\x00\x00\x20\x66\x74[video]')
|
|
||||||
elif self.path == '/302':
|
elif self.path == '/302':
|
||||||
if sys.version_info[0] == 3:
|
if sys.version_info[0] == 3:
|
||||||
# XXX: Python 3 http server does not allow non-ASCII header values
|
# XXX: Python 3 http server does not allow non-ASCII header values
|
||||||
|
@ -39,71 +172,336 @@ class HTTPTestRequestHandler(compat_http_server.BaseHTTPRequestHandler):
|
||||||
self.end_headers()
|
self.end_headers()
|
||||||
return
|
return
|
||||||
|
|
||||||
new_url = 'http://127.0.0.1:%d/中文.html' % http_server_port(self.server)
|
new_url = self._test_url('中文.html')
|
||||||
self.send_response(302)
|
self.send_response(302)
|
||||||
self.send_header(b'Location', new_url.encode('utf-8'))
|
self.send_header(b'Location', new_url.encode('utf-8'))
|
||||||
self.end_headers()
|
self.end_headers()
|
||||||
elif self.path == '/%E4%B8%AD%E6%96%87.html':
|
elif self.path == '/%E4%B8%AD%E6%96%87.html':
|
||||||
self.send_response(200)
|
respond()
|
||||||
self.send_header('Content-Type', 'text/html; charset=utf-8')
|
elif self.path == '/%c7%9f':
|
||||||
|
respond()
|
||||||
|
elif self.path == '/redirect_dotsegments':
|
||||||
|
self.send_response(301)
|
||||||
|
# redirect to /headers but with dot segments before
|
||||||
|
self.send_header('Location', '/a/b/./../../headers')
|
||||||
|
self.send_header('Content-Length', '0')
|
||||||
self.end_headers()
|
self.end_headers()
|
||||||
self.wfile.write(b'<html><video src="/vid.mp4" /></html>')
|
elif self.path.startswith('/redirect_'):
|
||||||
|
self._redirect()
|
||||||
|
elif self.path.startswith('/method'):
|
||||||
|
self._method('GET')
|
||||||
|
elif self.path.startswith('/headers'):
|
||||||
|
self._headers()
|
||||||
|
elif self.path.startswith('/308-to-headers'):
|
||||||
|
self.send_response(308)
|
||||||
|
self.send_header('Location', '/headers')
|
||||||
|
self.send_header('Content-Length', '0')
|
||||||
|
self.end_headers()
|
||||||
|
elif self.path == '/trailing_garbage':
|
||||||
|
payload = b'<html><video src="/vid.mp4" /></html>'
|
||||||
|
compressed = gzip_compress(payload) + b'trailing garbage'
|
||||||
|
respond(compressed, payload_encoding='gzip')
|
||||||
|
elif self.path == '/302-non-ascii-redirect':
|
||||||
|
new_url = self._test_url('中文.html')
|
||||||
|
# actually respond with permanent redirect
|
||||||
|
self.send_response(301)
|
||||||
|
self.send_header('Location', new_url)
|
||||||
|
self.send_header('Content-Length', '0')
|
||||||
|
self.end_headers()
|
||||||
|
elif self.path == '/content-encoding':
|
||||||
|
encodings = self.headers.get('ytdl-encoding', '')
|
||||||
|
payload = b'<html><video src="/vid.mp4" /></html>'
|
||||||
|
for encoding in filter(None, (e.strip() for e in encodings.split(','))):
|
||||||
|
if encoding == 'br' and brotli:
|
||||||
|
payload = brotli.compress(payload)
|
||||||
|
elif encoding == 'gzip':
|
||||||
|
payload = gzip_compress(payload)
|
||||||
|
elif encoding == 'deflate':
|
||||||
|
payload = zlib.compress(payload)
|
||||||
|
elif encoding == 'unsupported':
|
||||||
|
payload = b'raw'
|
||||||
|
break
|
||||||
|
else:
|
||||||
|
self._status(415)
|
||||||
|
return
|
||||||
|
respond(payload, payload_encoding=encodings)
|
||||||
|
|
||||||
else:
|
else:
|
||||||
assert False
|
self._status(404)
|
||||||
|
|
||||||
|
def send_header(self, keyword, value):
|
||||||
|
"""
|
||||||
|
Forcibly allow HTTP server to send non percent-encoded non-ASCII characters in headers.
|
||||||
|
This is against what is defined in RFC 3986: but we need to test that we support this
|
||||||
|
since some sites incorrectly do this.
|
||||||
|
"""
|
||||||
|
if keyword.lower() == 'connection':
|
||||||
|
return self.super('send_header', keyword, value)
|
||||||
|
|
||||||
class FakeLogger(object):
|
if not hasattr(self, '_headers_buffer'):
|
||||||
def debug(self, msg):
|
self._headers_buffer = []
|
||||||
pass
|
|
||||||
|
|
||||||
def warning(self, msg):
|
self._headers_buffer.append('{0}: {1}\r\n'.format(keyword, value).encode('utf-8'))
|
||||||
pass
|
|
||||||
|
|
||||||
def error(self, msg):
|
def end_headers(self):
|
||||||
pass
|
if hasattr(self, '_headers_buffer'):
|
||||||
|
self.wfile.write(b''.join(self._headers_buffer))
|
||||||
|
self._headers_buffer = []
|
||||||
|
self.super('end_headers')
|
||||||
|
|
||||||
|
|
||||||
class TestHTTP(unittest.TestCase):
|
class TestHTTP(unittest.TestCase):
|
||||||
|
# when does it make sense to check the SSL certificate?
|
||||||
|
_check_cert = (
|
||||||
|
sys.version_info >= (3, 2)
|
||||||
|
or (sys.version_info[0] == 2 and sys.version_info[1:] >= (7, 19)))
|
||||||
|
|
||||||
def setUp(self):
|
def setUp(self):
|
||||||
self.httpd = compat_http_server.HTTPServer(
|
# HTTP server
|
||||||
|
self.http_httpd = compat_http_server.HTTPServer(
|
||||||
('127.0.0.1', 0), HTTPTestRequestHandler)
|
('127.0.0.1', 0), HTTPTestRequestHandler)
|
||||||
self.port = http_server_port(self.httpd)
|
self.http_port = http_server_port(self.http_httpd)
|
||||||
self.server_thread = threading.Thread(target=self.httpd.serve_forever)
|
|
||||||
self.server_thread.daemon = True
|
self.http_server_thread = threading.Thread(target=self.http_httpd.serve_forever)
|
||||||
self.server_thread.start()
|
self.http_server_thread.daemon = True
|
||||||
|
self.http_server_thread.start()
|
||||||
|
|
||||||
|
try:
|
||||||
|
from http.server import ThreadingHTTPServer
|
||||||
|
except ImportError:
|
||||||
|
try:
|
||||||
|
from socketserver import ThreadingMixIn
|
||||||
|
except ImportError:
|
||||||
|
from SocketServer import ThreadingMixIn
|
||||||
|
|
||||||
|
class ThreadingHTTPServer(ThreadingMixIn, compat_http_server.HTTPServer):
|
||||||
|
pass
|
||||||
|
|
||||||
|
# HTTPS server
|
||||||
|
certfn = os.path.join(TEST_DIR, 'testcert.pem')
|
||||||
|
self.https_httpd = ThreadingHTTPServer(
|
||||||
|
('127.0.0.1', 0), HTTPTestRequestHandler)
|
||||||
|
try:
|
||||||
|
sslctx = ssl.SSLContext(ssl.PROTOCOL_SSLv23)
|
||||||
|
sslctx.verify_mode = ssl.CERT_NONE
|
||||||
|
sslctx.check_hostname = False
|
||||||
|
sslctx.load_cert_chain(certfn, None)
|
||||||
|
self.https_httpd.socket = sslctx.wrap_socket(
|
||||||
|
self.https_httpd.socket, server_side=True)
|
||||||
|
except AttributeError:
|
||||||
|
self.https_httpd.socket = ssl.wrap_socket(
|
||||||
|
self.https_httpd.socket, certfile=certfn, server_side=True)
|
||||||
|
|
||||||
|
self.https_port = http_server_port(self.https_httpd)
|
||||||
|
self.https_server_thread = threading.Thread(target=self.https_httpd.serve_forever)
|
||||||
|
self.https_server_thread.daemon = True
|
||||||
|
self.https_server_thread.start()
|
||||||
|
|
||||||
|
def tearDown(self):
|
||||||
|
|
||||||
|
def closer(svr):
|
||||||
|
def _closer():
|
||||||
|
svr.shutdown()
|
||||||
|
svr.server_close()
|
||||||
|
return _closer
|
||||||
|
|
||||||
|
shutdown_thread = threading.Thread(target=closer(self.http_httpd))
|
||||||
|
shutdown_thread.start()
|
||||||
|
self.http_server_thread.join(2.0)
|
||||||
|
|
||||||
|
shutdown_thread = threading.Thread(target=closer(self.https_httpd))
|
||||||
|
shutdown_thread.start()
|
||||||
|
self.https_server_thread.join(2.0)
|
||||||
|
|
||||||
|
def _test_url(self, path, host='127.0.0.1', scheme='http', port=None):
|
||||||
|
return '{0}://{1}:{2}/{3}'.format(
|
||||||
|
scheme, host,
|
||||||
|
port if port is not None
|
||||||
|
else self.https_port if scheme == 'https'
|
||||||
|
else self.http_port, path)
|
||||||
|
|
||||||
|
@unittest.skipUnless(_check_cert, 'No support for certificate check in SSL')
|
||||||
|
def test_nocheckcertificate(self):
|
||||||
|
with FakeYDL({'logger': FakeLogger()}) as ydl:
|
||||||
|
with self.assertRaises(compat_urllib_error.URLError):
|
||||||
|
ydl.urlopen(sanitized_Request(self._test_url('headers', scheme='https')))
|
||||||
|
|
||||||
|
with FakeYDL({'logger': FakeLogger(), 'nocheckcertificate': True}) as ydl:
|
||||||
|
r = ydl.urlopen(sanitized_Request(self._test_url('headers', scheme='https')))
|
||||||
|
self.assertEqual(r.getcode(), 200)
|
||||||
|
r.close()
|
||||||
|
|
||||||
|
def test_percent_encode(self):
|
||||||
|
with FakeYDL() as ydl:
|
||||||
|
# Unicode characters should be encoded with uppercase percent-encoding
|
||||||
|
res = ydl.urlopen(sanitized_Request(self._test_url('中文.html')))
|
||||||
|
self.assertEqual(res.getcode(), 200)
|
||||||
|
res.close()
|
||||||
|
# don't normalize existing percent encodings
|
||||||
|
res = ydl.urlopen(sanitized_Request(self._test_url('%c7%9f')))
|
||||||
|
self.assertEqual(res.getcode(), 200)
|
||||||
|
res.close()
|
||||||
|
|
||||||
def test_unicode_path_redirection(self):
|
def test_unicode_path_redirection(self):
|
||||||
# XXX: Python 3 http server does not allow non-ASCII header values
|
with FakeYDL() as ydl:
|
||||||
if sys.version_info[0] == 3:
|
r = ydl.urlopen(sanitized_Request(self._test_url('302-non-ascii-redirect')))
|
||||||
return
|
self.assertEqual(r.url, self._test_url('%E4%B8%AD%E6%96%87.html'))
|
||||||
|
r.close()
|
||||||
|
|
||||||
ydl = YoutubeDL({'logger': FakeLogger()})
|
def test_redirect(self):
|
||||||
r = ydl.extract_info('http://127.0.0.1:%d/302' % self.port)
|
with FakeYDL() as ydl:
|
||||||
self.assertEqual(r['entries'][0]['url'], 'http://127.0.0.1:%d/vid.mp4' % self.port)
|
def do_req(redirect_status, method, check_no_content=False):
|
||||||
|
data = b'testdata' if method in ('POST', 'PUT') else None
|
||||||
|
res = ydl.urlopen(sanitized_Request(
|
||||||
|
self._test_url('redirect_{0}'.format(redirect_status)),
|
||||||
|
method=method, data=data))
|
||||||
|
if check_no_content:
|
||||||
|
self.assertNotIn('Content-Type', res.headers)
|
||||||
|
return res.read().decode('utf-8'), res.headers.get('method', '')
|
||||||
|
# A 303 must either use GET or HEAD for subsequent request
|
||||||
|
self.assertEqual(do_req(303, 'POST'), ('', 'GET'))
|
||||||
|
self.assertEqual(do_req(303, 'HEAD'), ('', 'HEAD'))
|
||||||
|
|
||||||
|
self.assertEqual(do_req(303, 'PUT'), ('', 'GET'))
|
||||||
|
|
||||||
class TestHTTPS(unittest.TestCase):
|
# 301 and 302 turn POST only into a GET, with no Content-Type
|
||||||
def setUp(self):
|
self.assertEqual(do_req(301, 'POST', True), ('', 'GET'))
|
||||||
certfn = os.path.join(TEST_DIR, 'testcert.pem')
|
self.assertEqual(do_req(301, 'HEAD'), ('', 'HEAD'))
|
||||||
self.httpd = compat_http_server.HTTPServer(
|
self.assertEqual(do_req(302, 'POST', True), ('', 'GET'))
|
||||||
('127.0.0.1', 0), HTTPTestRequestHandler)
|
self.assertEqual(do_req(302, 'HEAD'), ('', 'HEAD'))
|
||||||
self.httpd.socket = ssl.wrap_socket(
|
|
||||||
self.httpd.socket, certfile=certfn, server_side=True)
|
|
||||||
self.port = http_server_port(self.httpd)
|
|
||||||
self.server_thread = threading.Thread(target=self.httpd.serve_forever)
|
|
||||||
self.server_thread.daemon = True
|
|
||||||
self.server_thread.start()
|
|
||||||
|
|
||||||
def test_nocheckcertificate(self):
|
self.assertEqual(do_req(301, 'PUT'), ('testdata', 'PUT'))
|
||||||
if sys.version_info >= (2, 7, 9): # No certificate checking anyways
|
self.assertEqual(do_req(302, 'PUT'), ('testdata', 'PUT'))
|
||||||
ydl = YoutubeDL({'logger': FakeLogger()})
|
|
||||||
self.assertRaises(
|
|
||||||
Exception,
|
|
||||||
ydl.extract_info, 'https://127.0.0.1:%d/video.html' % self.port)
|
|
||||||
|
|
||||||
ydl = YoutubeDL({'logger': FakeLogger(), 'nocheckcertificate': True})
|
# 307 and 308 should not change method
|
||||||
r = ydl.extract_info('https://127.0.0.1:%d/video.html' % self.port)
|
for m in ('POST', 'PUT'):
|
||||||
self.assertEqual(r['entries'][0]['url'], 'https://127.0.0.1:%d/vid.mp4' % self.port)
|
self.assertEqual(do_req(307, m), ('testdata', m))
|
||||||
|
self.assertEqual(do_req(308, m), ('testdata', m))
|
||||||
|
|
||||||
|
self.assertEqual(do_req(307, 'HEAD'), ('', 'HEAD'))
|
||||||
|
self.assertEqual(do_req(308, 'HEAD'), ('', 'HEAD'))
|
||||||
|
|
||||||
|
# These should not redirect and instead raise an HTTPError
|
||||||
|
for code in (300, 304, 305, 306):
|
||||||
|
with self.assertRaises(compat_urllib_HTTPError):
|
||||||
|
do_req(code, 'GET')
|
||||||
|
|
||||||
|
# Jython 2.7.1 times out for some reason
|
||||||
|
@expectedFailureIf(sys.platform.startswith('java') and sys.version_info < (2, 7, 2))
|
||||||
|
def test_content_type(self):
|
||||||
|
# https://github.com/yt-dlp/yt-dlp/commit/379a4f161d4ad3e40932dcf5aca6e6fb9715ab28
|
||||||
|
with FakeYDL({'nocheckcertificate': True}) as ydl:
|
||||||
|
# method should be auto-detected as POST
|
||||||
|
r = sanitized_Request(self._test_url('headers', scheme='https'), data=urlencode_postdata({'test': 'test'}))
|
||||||
|
|
||||||
|
headers = ydl.urlopen(r).read().decode('utf-8')
|
||||||
|
self.assertIn('Content-Type: application/x-www-form-urlencoded', headers)
|
||||||
|
|
||||||
|
# test http
|
||||||
|
r = sanitized_Request(self._test_url('headers'), data=urlencode_postdata({'test': 'test'}))
|
||||||
|
headers = ydl.urlopen(r).read().decode('utf-8')
|
||||||
|
self.assertIn('Content-Type: application/x-www-form-urlencoded', headers)
|
||||||
|
|
||||||
|
def test_update_req(self):
|
||||||
|
req = sanitized_Request('http://example.com')
|
||||||
|
assert req.data is None
|
||||||
|
assert req.get_method() == 'GET'
|
||||||
|
assert not req.has_header('Content-Type')
|
||||||
|
# Test that zero-byte payloads will be sent
|
||||||
|
req = update_Request(req, data=b'')
|
||||||
|
assert req.data == b''
|
||||||
|
assert req.get_method() == 'POST'
|
||||||
|
# yt-dl expects data to be encoded and Content-Type to be added by sender
|
||||||
|
# assert req.get_header('Content-Type') == 'application/x-www-form-urlencoded'
|
||||||
|
|
||||||
|
def test_cookiejar(self):
|
||||||
|
with FakeYDL() as ydl:
|
||||||
|
ydl.cookiejar.set_cookie(compat_http_cookiejar_Cookie(
|
||||||
|
0, 'test', 'ytdl', None, False, '127.0.0.1', True,
|
||||||
|
False, '/headers', True, False, None, False, None, None, {}))
|
||||||
|
data = ydl.urlopen(sanitized_Request(
|
||||||
|
self._test_url('headers'))).read().decode('utf-8')
|
||||||
|
self.assertIn('Cookie: test=ytdl', data)
|
||||||
|
|
||||||
|
def test_passed_cookie_header(self):
|
||||||
|
# We should accept a Cookie header being passed as in normal headers and handle it appropriately.
|
||||||
|
with FakeYDL() as ydl:
|
||||||
|
# Specified Cookie header should be used
|
||||||
|
res = ydl.urlopen(sanitized_Request(
|
||||||
|
self._test_url('headers'), headers={'Cookie': 'test=test'})).read().decode('utf-8')
|
||||||
|
self.assertIn('Cookie: test=test', res)
|
||||||
|
|
||||||
|
# Specified Cookie header should be removed on any redirect
|
||||||
|
res = ydl.urlopen(sanitized_Request(
|
||||||
|
self._test_url('308-to-headers'), headers={'Cookie': 'test=test'})).read().decode('utf-8')
|
||||||
|
self.assertNotIn('Cookie: test=test', res)
|
||||||
|
|
||||||
|
# Specified Cookie header should override global cookiejar for that request
|
||||||
|
ydl.cookiejar.set_cookie(compat_http_cookiejar_Cookie(
|
||||||
|
0, 'test', 'ytdlp', None, False, '127.0.0.1', True,
|
||||||
|
False, '/headers', True, False, None, False, None, None, {}))
|
||||||
|
data = ydl.urlopen(sanitized_Request(
|
||||||
|
self._test_url('headers'), headers={'Cookie': 'test=test'})).read().decode('utf-8')
|
||||||
|
self.assertNotIn('Cookie: test=ytdlp', data)
|
||||||
|
self.assertIn('Cookie: test=test', data)
|
||||||
|
|
||||||
|
def test_no_compression_compat_header(self):
|
||||||
|
with FakeYDL() as ydl:
|
||||||
|
data = ydl.urlopen(
|
||||||
|
sanitized_Request(
|
||||||
|
self._test_url('headers'),
|
||||||
|
headers={'Youtubedl-no-compression': True})).read()
|
||||||
|
self.assertIn(b'Accept-Encoding: identity', data)
|
||||||
|
self.assertNotIn(b'youtubedl-no-compression', data.lower())
|
||||||
|
|
||||||
|
def test_gzip_trailing_garbage(self):
|
||||||
|
# https://github.com/ytdl-org/youtube-dl/commit/aa3e950764337ef9800c936f4de89b31c00dfcf5
|
||||||
|
# https://github.com/ytdl-org/youtube-dl/commit/6f2ec15cee79d35dba065677cad9da7491ec6e6f
|
||||||
|
with FakeYDL() as ydl:
|
||||||
|
data = ydl.urlopen(sanitized_Request(self._test_url('trailing_garbage'))).read().decode('utf-8')
|
||||||
|
self.assertEqual(data, '<html><video src="/vid.mp4" /></html>')
|
||||||
|
|
||||||
|
def __test_compression(self, encoding):
|
||||||
|
with FakeYDL() as ydl:
|
||||||
|
res = ydl.urlopen(
|
||||||
|
sanitized_Request(
|
||||||
|
self._test_url('content-encoding'),
|
||||||
|
headers={'ytdl-encoding': encoding}))
|
||||||
|
# decoded encodings are removed: only check for valid decompressed data
|
||||||
|
self.assertEqual(res.read(), b'<html><video src="/vid.mp4" /></html>')
|
||||||
|
|
||||||
|
@unittest.skipUnless(brotli, 'brotli support is not installed')
|
||||||
|
def test_brotli(self):
|
||||||
|
self.__test_compression('br')
|
||||||
|
|
||||||
|
def test_deflate(self):
|
||||||
|
self.__test_compression('deflate')
|
||||||
|
|
||||||
|
def test_gzip(self):
|
||||||
|
self.__test_compression('gzip')
|
||||||
|
|
||||||
|
def test_multiple_encodings(self):
|
||||||
|
# https://www.rfc-editor.org/rfc/rfc9110.html#section-8.4
|
||||||
|
for pair in ('gzip,deflate', 'deflate, gzip', 'gzip, gzip', 'deflate, deflate'):
|
||||||
|
self.__test_compression(pair)
|
||||||
|
|
||||||
|
def test_unsupported_encoding(self):
|
||||||
|
# it should return the raw content
|
||||||
|
with FakeYDL() as ydl:
|
||||||
|
res = ydl.urlopen(
|
||||||
|
sanitized_Request(
|
||||||
|
self._test_url('content-encoding'),
|
||||||
|
headers={'ytdl-encoding': 'unsupported'}))
|
||||||
|
self.assertEqual(res.headers.get('Content-Encoding'), 'unsupported')
|
||||||
|
self.assertEqual(res.read(), b'raw')
|
||||||
|
|
||||||
|
def test_remove_dot_segments(self):
|
||||||
|
with FakeYDL() as ydl:
|
||||||
|
res = ydl.urlopen(sanitized_Request(self._test_url('a/b/./../../headers')))
|
||||||
|
self.assertEqual(compat_urllib_parse.urlparse(res.geturl()).path, '/headers')
|
||||||
|
|
||||||
|
res = ydl.urlopen(sanitized_Request(self._test_url('redirect_dotsegments')))
|
||||||
|
self.assertEqual(compat_urllib_parse.urlparse(res.geturl()).path, '/headers')
|
||||||
|
|
||||||
|
|
||||||
def _build_proxy_handler(name):
|
def _build_proxy_handler(name):
|
||||||
|
@ -117,7 +515,7 @@ def _build_proxy_handler(name):
|
||||||
self.send_response(200)
|
self.send_response(200)
|
||||||
self.send_header('Content-Type', 'text/plain; charset=utf-8')
|
self.send_header('Content-Type', 'text/plain; charset=utf-8')
|
||||||
self.end_headers()
|
self.end_headers()
|
||||||
self.wfile.write('{self.proxy_name}: {self.path}'.format(self=self).encode('utf-8'))
|
self.wfile.write('{0}: {1}'.format(self.proxy_name, self.path).encode('utf-8'))
|
||||||
return HTTPTestRequestHandler
|
return HTTPTestRequestHandler
|
||||||
|
|
||||||
|
|
||||||
|
@ -137,10 +535,30 @@ class TestProxy(unittest.TestCase):
|
||||||
self.geo_proxy_thread.daemon = True
|
self.geo_proxy_thread.daemon = True
|
||||||
self.geo_proxy_thread.start()
|
self.geo_proxy_thread.start()
|
||||||
|
|
||||||
|
def tearDown(self):
|
||||||
|
|
||||||
|
def closer(svr):
|
||||||
|
def _closer():
|
||||||
|
svr.shutdown()
|
||||||
|
svr.server_close()
|
||||||
|
return _closer
|
||||||
|
|
||||||
|
shutdown_thread = threading.Thread(target=closer(self.proxy))
|
||||||
|
shutdown_thread.start()
|
||||||
|
self.proxy_thread.join(2.0)
|
||||||
|
|
||||||
|
shutdown_thread = threading.Thread(target=closer(self.geo_proxy))
|
||||||
|
shutdown_thread.start()
|
||||||
|
self.geo_proxy_thread.join(2.0)
|
||||||
|
|
||||||
|
def _test_proxy(self, host='127.0.0.1', port=None):
|
||||||
|
return '{0}:{1}'.format(
|
||||||
|
host, port if port is not None else self.port)
|
||||||
|
|
||||||
def test_proxy(self):
|
def test_proxy(self):
|
||||||
geo_proxy = '127.0.0.1:{0}'.format(self.geo_port)
|
geo_proxy = self._test_proxy(port=self.geo_port)
|
||||||
ydl = YoutubeDL({
|
ydl = YoutubeDL({
|
||||||
'proxy': '127.0.0.1:{0}'.format(self.port),
|
'proxy': self._test_proxy(),
|
||||||
'geo_verification_proxy': geo_proxy,
|
'geo_verification_proxy': geo_proxy,
|
||||||
})
|
})
|
||||||
url = 'http://foo.com/bar'
|
url = 'http://foo.com/bar'
|
||||||
|
@ -154,7 +572,7 @@ class TestProxy(unittest.TestCase):
|
||||||
|
|
||||||
def test_proxy_with_idn(self):
|
def test_proxy_with_idn(self):
|
||||||
ydl = YoutubeDL({
|
ydl = YoutubeDL({
|
||||||
'proxy': '127.0.0.1:{0}'.format(self.port),
|
'proxy': self._test_proxy(),
|
||||||
})
|
})
|
||||||
url = 'http://中文.tw/'
|
url = 'http://中文.tw/'
|
||||||
response = ydl.urlopen(url).read().decode('utf-8')
|
response = ydl.urlopen(url).read().decode('utf-8')
|
||||||
|
@ -162,5 +580,25 @@ class TestProxy(unittest.TestCase):
|
||||||
self.assertEqual(response, 'normal: http://xn--fiq228c.tw/')
|
self.assertEqual(response, 'normal: http://xn--fiq228c.tw/')
|
||||||
|
|
||||||
|
|
||||||
|
class TestFileURL(unittest.TestCase):
|
||||||
|
# See https://github.com/ytdl-org/youtube-dl/issues/8227
|
||||||
|
def test_file_urls(self):
|
||||||
|
tf = tempfile.NamedTemporaryFile(delete=False)
|
||||||
|
tf.write(b'foobar')
|
||||||
|
tf.close()
|
||||||
|
url = compat_urllib_parse.urljoin('file://', pathname2url(tf.name))
|
||||||
|
with FakeYDL() as ydl:
|
||||||
|
self.assertRaisesRegexp(
|
||||||
|
compat_urllib_error.URLError, 'file:// scheme is explicitly disabled in youtube-dl for security reasons', ydl.urlopen, url)
|
||||||
|
# not yet implemented
|
||||||
|
"""
|
||||||
|
with FakeYDL({'enable_file_urls': True}) as ydl:
|
||||||
|
res = ydl.urlopen(url)
|
||||||
|
self.assertEqual(res.read(), b'foobar')
|
||||||
|
res.close()
|
||||||
|
"""
|
||||||
|
os.unlink(tf.name)
|
||||||
|
|
||||||
|
|
||||||
if __name__ == '__main__':
|
if __name__ == '__main__':
|
||||||
unittest.main()
|
unittest.main()
|
||||||
|
|
|
@ -8,109 +8,450 @@ import sys
|
||||||
import unittest
|
import unittest
|
||||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||||
|
|
||||||
from youtube_dl.jsinterp import JSInterpreter
|
import math
|
||||||
|
import re
|
||||||
|
|
||||||
|
from youtube_dl.compat import compat_str
|
||||||
|
from youtube_dl.jsinterp import JS_Undefined, JSInterpreter
|
||||||
|
|
||||||
|
NaN = object()
|
||||||
|
|
||||||
|
|
||||||
class TestJSInterpreter(unittest.TestCase):
|
class TestJSInterpreter(unittest.TestCase):
|
||||||
|
def _test(self, jsi_or_code, expected, func='f', args=()):
|
||||||
|
if isinstance(jsi_or_code, compat_str):
|
||||||
|
jsi_or_code = JSInterpreter(jsi_or_code)
|
||||||
|
got = jsi_or_code.call_function(func, *args)
|
||||||
|
if expected is NaN:
|
||||||
|
self.assertTrue(math.isnan(got), '{0} is not NaN'.format(got))
|
||||||
|
else:
|
||||||
|
self.assertEqual(got, expected)
|
||||||
|
|
||||||
def test_basic(self):
|
def test_basic(self):
|
||||||
jsi = JSInterpreter('function x(){;}')
|
jsi = JSInterpreter('function f(){;}')
|
||||||
self.assertEqual(jsi.call_function('x'), None)
|
self.assertEqual(repr(jsi.extract_function('f')), 'F<f>')
|
||||||
|
self._test(jsi, None)
|
||||||
|
|
||||||
jsi = JSInterpreter('function x3(){return 42;}')
|
self._test('function f(){return 42;}', 42)
|
||||||
self.assertEqual(jsi.call_function('x3'), 42)
|
self._test('function f(){42}', None)
|
||||||
|
self._test('var f = function(){return 42;}', 42)
|
||||||
|
|
||||||
jsi = JSInterpreter('var x5 = function(){return 42;}')
|
def test_add(self):
|
||||||
self.assertEqual(jsi.call_function('x5'), 42)
|
self._test('function f(){return 42 + 7;}', 49)
|
||||||
|
self._test('function f(){return 42 + undefined;}', NaN)
|
||||||
|
self._test('function f(){return 42 + null;}', 42)
|
||||||
|
|
||||||
|
def test_sub(self):
|
||||||
|
self._test('function f(){return 42 - 7;}', 35)
|
||||||
|
self._test('function f(){return 42 - undefined;}', NaN)
|
||||||
|
self._test('function f(){return 42 - null;}', 42)
|
||||||
|
|
||||||
|
def test_mul(self):
|
||||||
|
self._test('function f(){return 42 * 7;}', 294)
|
||||||
|
self._test('function f(){return 42 * undefined;}', NaN)
|
||||||
|
self._test('function f(){return 42 * null;}', 0)
|
||||||
|
|
||||||
|
def test_div(self):
|
||||||
|
jsi = JSInterpreter('function f(a, b){return a / b;}')
|
||||||
|
self._test(jsi, NaN, args=(0, 0))
|
||||||
|
self._test(jsi, NaN, args=(JS_Undefined, 1))
|
||||||
|
self._test(jsi, float('inf'), args=(2, 0))
|
||||||
|
self._test(jsi, 0, args=(0, 3))
|
||||||
|
|
||||||
|
def test_mod(self):
|
||||||
|
self._test('function f(){return 42 % 7;}', 0)
|
||||||
|
self._test('function f(){return 42 % 0;}', NaN)
|
||||||
|
self._test('function f(){return 42 % undefined;}', NaN)
|
||||||
|
|
||||||
|
def test_exp(self):
|
||||||
|
self._test('function f(){return 42 ** 2;}', 1764)
|
||||||
|
self._test('function f(){return 42 ** undefined;}', NaN)
|
||||||
|
self._test('function f(){return 42 ** null;}', 1)
|
||||||
|
self._test('function f(){return undefined ** 42;}', NaN)
|
||||||
|
|
||||||
def test_calc(self):
|
def test_calc(self):
|
||||||
jsi = JSInterpreter('function x4(a){return 2*a+1;}')
|
self._test('function f(a){return 2*a+1;}', 7, args=[3])
|
||||||
self.assertEqual(jsi.call_function('x4', 3), 7)
|
|
||||||
|
|
||||||
def test_empty_return(self):
|
def test_empty_return(self):
|
||||||
jsi = JSInterpreter('function f(){return; y()}')
|
self._test('function f(){return; y()}', None)
|
||||||
self.assertEqual(jsi.call_function('f'), None)
|
|
||||||
|
|
||||||
def test_morespace(self):
|
def test_morespace(self):
|
||||||
jsi = JSInterpreter('function x (a) { return 2 * a + 1 ; }')
|
self._test('function f (a) { return 2 * a + 1 ; }', 7, args=[3])
|
||||||
self.assertEqual(jsi.call_function('x', 3), 7)
|
self._test('function f () { x = 2 ; return x; }', 2)
|
||||||
|
|
||||||
jsi = JSInterpreter('function f () { x = 2 ; return x; }')
|
|
||||||
self.assertEqual(jsi.call_function('f'), 2)
|
|
||||||
|
|
||||||
def test_strange_chars(self):
|
def test_strange_chars(self):
|
||||||
jsi = JSInterpreter('function $_xY1 ($_axY1) { var $_axY2 = $_axY1 + 1; return $_axY2; }')
|
self._test('function $_xY1 ($_axY1) { var $_axY2 = $_axY1 + 1; return $_axY2; }',
|
||||||
self.assertEqual(jsi.call_function('$_xY1', 20), 21)
|
21, args=[20], func='$_xY1')
|
||||||
|
|
||||||
def test_operators(self):
|
def test_operators(self):
|
||||||
jsi = JSInterpreter('function f(){return 1 << 5;}')
|
self._test('function f(){return 1 << 5;}', 32)
|
||||||
self.assertEqual(jsi.call_function('f'), 32)
|
self._test('function f(){return 2 ** 5}', 32)
|
||||||
|
self._test('function f(){return 19 & 21;}', 17)
|
||||||
jsi = JSInterpreter('function f(){return 19 & 21;}')
|
self._test('function f(){return 11 >> 2;}', 2)
|
||||||
self.assertEqual(jsi.call_function('f'), 17)
|
self._test('function f(){return []? 2+3: 4;}', 5)
|
||||||
|
self._test('function f(){return 1 == 2}', False)
|
||||||
jsi = JSInterpreter('function f(){return 11 >> 2;}')
|
self._test('function f(){return 0 && 1 || 2;}', 2)
|
||||||
self.assertEqual(jsi.call_function('f'), 2)
|
self._test('function f(){return 0 ?? 42;}', 0)
|
||||||
|
self._test('function f(){return "life, the universe and everything" < 42;}', False)
|
||||||
|
# https://github.com/ytdl-org/youtube-dl/issues/32815
|
||||||
|
self._test('function f(){return 0 - 7 * - 6;}', 42)
|
||||||
|
|
||||||
def test_array_access(self):
|
def test_array_access(self):
|
||||||
jsi = JSInterpreter('function f(){var x = [1,2,3]; x[0] = 4; x[0] = 5; x[2] = 7; return x;}')
|
self._test('function f(){var x = [1,2,3]; x[0] = 4; x[0] = 5; x[2.0] = 7; return x;}', [5, 2, 7])
|
||||||
self.assertEqual(jsi.call_function('f'), [5, 2, 7])
|
|
||||||
|
|
||||||
def test_parens(self):
|
def test_parens(self):
|
||||||
jsi = JSInterpreter('function f(){return (1) + (2) * ((( (( (((((3)))))) )) ));}')
|
self._test('function f(){return (1) + (2) * ((( (( (((((3)))))) )) ));}', 7)
|
||||||
self.assertEqual(jsi.call_function('f'), 7)
|
self._test('function f(){return (1 + 2) * 3;}', 9)
|
||||||
|
|
||||||
jsi = JSInterpreter('function f(){return (1 + 2) * 3;}')
|
def test_quotes(self):
|
||||||
self.assertEqual(jsi.call_function('f'), 9)
|
self._test(r'function f(){return "a\"\\("}', r'a"\(')
|
||||||
|
|
||||||
def test_assignments(self):
|
def test_assignments(self):
|
||||||
jsi = JSInterpreter('function f(){var x = 20; x = 30 + 1; return x;}')
|
self._test('function f(){var x = 20; x = 30 + 1; return x;}', 31)
|
||||||
self.assertEqual(jsi.call_function('f'), 31)
|
self._test('function f(){var x = 20; x += 30 + 1; return x;}', 51)
|
||||||
|
self._test('function f(){var x = 20; x -= 30 + 1; return x;}', -11)
|
||||||
jsi = JSInterpreter('function f(){var x = 20; x += 30 + 1; return x;}')
|
|
||||||
self.assertEqual(jsi.call_function('f'), 51)
|
|
||||||
|
|
||||||
jsi = JSInterpreter('function f(){var x = 20; x -= 30 + 1; return x;}')
|
|
||||||
self.assertEqual(jsi.call_function('f'), -11)
|
|
||||||
|
|
||||||
|
@unittest.skip('Not yet fully implemented')
|
||||||
def test_comments(self):
|
def test_comments(self):
|
||||||
'Skipping: Not yet fully implemented'
|
self._test('''
|
||||||
return
|
function f() {
|
||||||
jsi = JSInterpreter('''
|
var x = /* 1 + */ 2;
|
||||||
function x() {
|
var y = /* 30
|
||||||
var x = /* 1 + */ 2;
|
* 40 */ 50;
|
||||||
var y = /* 30
|
return x + y;
|
||||||
* 40 */ 50;
|
}
|
||||||
return x + y;
|
''', 52)
|
||||||
}
|
|
||||||
''')
|
|
||||||
self.assertEqual(jsi.call_function('x'), 52)
|
|
||||||
|
|
||||||
jsi = JSInterpreter('''
|
self._test('''
|
||||||
function f() {
|
function f() {
|
||||||
var x = "/*";
|
var x = "/*";
|
||||||
var y = 1 /* comment */ + 2;
|
var y = 1 /* comment */ + 2;
|
||||||
return y;
|
return y;
|
||||||
}
|
}
|
||||||
''')
|
''', 3)
|
||||||
self.assertEqual(jsi.call_function('f'), 3)
|
|
||||||
|
|
||||||
def test_precedence(self):
|
def test_precedence(self):
|
||||||
jsi = JSInterpreter('''
|
self._test('''
|
||||||
function x() {
|
function f() {
|
||||||
var a = [10, 20, 30, 40, 50];
|
var a = [10, 20, 30, 40, 50];
|
||||||
var b = 6;
|
var b = 6;
|
||||||
a[0]=a[b%a.length];
|
a[0]=a[b%a.length];
|
||||||
return a;
|
return a;
|
||||||
}''')
|
}
|
||||||
self.assertEqual(jsi.call_function('x'), [20, 20, 30, 40, 50])
|
''', [20, 20, 30, 40, 50])
|
||||||
|
|
||||||
|
def test_builtins(self):
|
||||||
|
self._test('function f() { return NaN }', NaN)
|
||||||
|
|
||||||
|
def test_Date(self):
|
||||||
|
self._test('function f() { return new Date("Wednesday 31 December 1969 18:01:26 MDT") - 0; }', 86000)
|
||||||
|
|
||||||
|
jsi = JSInterpreter('function f(dt) { return new Date(dt) - 0; }')
|
||||||
|
# date format m/d/y
|
||||||
|
self._test(jsi, 86000, args=['12/31/1969 18:01:26 MDT'])
|
||||||
|
# epoch 0
|
||||||
|
self._test(jsi, 0, args=['1 January 1970 00:00:00 UTC'])
|
||||||
|
|
||||||
def test_call(self):
|
def test_call(self):
|
||||||
jsi = JSInterpreter('''
|
jsi = JSInterpreter('''
|
||||||
function x() { return 2; }
|
function x() { return 2; }
|
||||||
function y(a) { return x() + a; }
|
function y(a) { return x() + (a?a:0); }
|
||||||
function z() { return y(3); }
|
function z() { return y(3); }
|
||||||
''')
|
''')
|
||||||
self.assertEqual(jsi.call_function('z'), 5)
|
self._test(jsi, 5, func='z')
|
||||||
|
self._test(jsi, 2, func='y')
|
||||||
|
|
||||||
|
def test_if(self):
|
||||||
|
self._test('''
|
||||||
|
function f() {
|
||||||
|
let a = 9;
|
||||||
|
if (0==0) {a++}
|
||||||
|
return a
|
||||||
|
}
|
||||||
|
''', 10)
|
||||||
|
|
||||||
|
self._test('''
|
||||||
|
function f() {
|
||||||
|
if (0==0) {return 10}
|
||||||
|
}
|
||||||
|
''', 10)
|
||||||
|
|
||||||
|
self._test('''
|
||||||
|
function f() {
|
||||||
|
if (0!=0) {return 1}
|
||||||
|
else {return 10}
|
||||||
|
}
|
||||||
|
''', 10)
|
||||||
|
|
||||||
|
def test_elseif(self):
|
||||||
|
self._test('''
|
||||||
|
function f() {
|
||||||
|
if (0!=0) {return 1}
|
||||||
|
else if (1==0) {return 2}
|
||||||
|
else {return 10}
|
||||||
|
}
|
||||||
|
''', 10)
|
||||||
|
|
||||||
|
def test_for_loop(self):
|
||||||
|
self._test('function f() { a=0; for (i=0; i-10; i++) {a++} return a }', 10)
|
||||||
|
|
||||||
|
def test_while_loop(self):
|
||||||
|
self._test('function f() { a=0; while (a<10) {a++} return a }', 10)
|
||||||
|
|
||||||
|
def test_switch(self):
|
||||||
|
jsi = JSInterpreter('''
|
||||||
|
function f(x) { switch(x){
|
||||||
|
case 1:x+=1;
|
||||||
|
case 2:x+=2;
|
||||||
|
case 3:x+=3;break;
|
||||||
|
case 4:x+=4;
|
||||||
|
default:x=0;
|
||||||
|
} return x }
|
||||||
|
''')
|
||||||
|
self._test(jsi, 7, args=[1])
|
||||||
|
self._test(jsi, 6, args=[3])
|
||||||
|
self._test(jsi, 0, args=[5])
|
||||||
|
|
||||||
|
def test_switch_default(self):
|
||||||
|
jsi = JSInterpreter('''
|
||||||
|
function f(x) { switch(x){
|
||||||
|
case 2: x+=2;
|
||||||
|
default: x-=1;
|
||||||
|
case 5:
|
||||||
|
case 6: x+=6;
|
||||||
|
case 0: break;
|
||||||
|
case 1: x+=1;
|
||||||
|
} return x }
|
||||||
|
''')
|
||||||
|
self._test(jsi, 2, args=[1])
|
||||||
|
self._test(jsi, 11, args=[5])
|
||||||
|
self._test(jsi, 14, args=[9])
|
||||||
|
|
||||||
|
def test_try(self):
|
||||||
|
self._test('function f() { try{return 10} catch(e){return 5} }', 10)
|
||||||
|
|
||||||
|
def test_catch(self):
|
||||||
|
self._test('function f() { try{throw 10} catch(e){return 5} }', 5)
|
||||||
|
|
||||||
|
def test_finally(self):
|
||||||
|
self._test('function f() { try{throw 10} finally {return 42} }', 42)
|
||||||
|
self._test('function f() { try{throw 10} catch(e){return 5} finally {return 42} }', 42)
|
||||||
|
|
||||||
|
def test_nested_try(self):
|
||||||
|
self._test('''
|
||||||
|
function f() {try {
|
||||||
|
try{throw 10} finally {throw 42}
|
||||||
|
} catch(e){return 5} }
|
||||||
|
''', 5)
|
||||||
|
|
||||||
|
def test_for_loop_continue(self):
|
||||||
|
self._test('function f() { a=0; for (i=0; i-10; i++) { continue; a++ } return a }', 0)
|
||||||
|
|
||||||
|
def test_for_loop_break(self):
|
||||||
|
self._test('function f() { a=0; for (i=0; i-10; i++) { break; a++ } return a }', 0)
|
||||||
|
|
||||||
|
def test_for_loop_try(self):
|
||||||
|
self._test('''
|
||||||
|
function f() {
|
||||||
|
for (i=0; i-10; i++) { try { if (i == 5) throw i} catch {return 10} finally {break} };
|
||||||
|
return 42 }
|
||||||
|
''', 42)
|
||||||
|
|
||||||
|
def test_literal_list(self):
|
||||||
|
self._test('function f() { return [1, 2, "asdf", [5, 6, 7]][3] }', [5, 6, 7])
|
||||||
|
|
||||||
|
def test_comma(self):
|
||||||
|
self._test('function f() { a=5; a -= 1, a+=3; return a }', 7)
|
||||||
|
self._test('function f() { a=5; return (a -= 1, a+=3, a); }', 7)
|
||||||
|
self._test('function f() { return (l=[0,1,2,3], function(a, b){return a+b})((l[1], l[2]), l[3]) }', 5)
|
||||||
|
|
||||||
|
def test_void(self):
|
||||||
|
self._test('function f() { return void 42; }', None)
|
||||||
|
|
||||||
|
def test_return_function(self):
|
||||||
|
jsi = JSInterpreter('''
|
||||||
|
function x() { return [1, function(){return 1}][1] }
|
||||||
|
''')
|
||||||
|
self.assertEqual(jsi.call_function('x')([]), 1)
|
||||||
|
|
||||||
|
def test_null(self):
|
||||||
|
self._test('function f() { return null; }', None)
|
||||||
|
self._test('function f() { return [null > 0, null < 0, null == 0, null === 0]; }',
|
||||||
|
[False, False, False, False])
|
||||||
|
self._test('function f() { return [null >= 0, null <= 0]; }', [True, True])
|
||||||
|
|
||||||
|
def test_undefined(self):
|
||||||
|
self._test('function f() { return undefined === undefined; }', True)
|
||||||
|
self._test('function f() { return undefined; }', JS_Undefined)
|
||||||
|
self._test('function f() {return undefined ?? 42; }', 42)
|
||||||
|
self._test('function f() { let v; return v; }', JS_Undefined)
|
||||||
|
self._test('function f() { let v; return v**0; }', 1)
|
||||||
|
self._test('function f() { let v; return [v>42, v<=42, v&&42, 42&&v]; }',
|
||||||
|
[False, False, JS_Undefined, JS_Undefined])
|
||||||
|
|
||||||
|
self._test('''
|
||||||
|
function f() { return [
|
||||||
|
undefined === undefined,
|
||||||
|
undefined == undefined,
|
||||||
|
undefined == null
|
||||||
|
]; }
|
||||||
|
''', [True] * 3)
|
||||||
|
self._test('''
|
||||||
|
function f() { return [
|
||||||
|
undefined < undefined,
|
||||||
|
undefined > undefined,
|
||||||
|
undefined === 0,
|
||||||
|
undefined == 0,
|
||||||
|
undefined < 0,
|
||||||
|
undefined > 0,
|
||||||
|
undefined >= 0,
|
||||||
|
undefined <= 0,
|
||||||
|
undefined > null,
|
||||||
|
undefined < null,
|
||||||
|
undefined === null
|
||||||
|
]; }
|
||||||
|
''', [False] * 11)
|
||||||
|
|
||||||
|
jsi = JSInterpreter('''
|
||||||
|
function x() { let v; return [42+v, v+42, v**42, 42**v, 0**v]; }
|
||||||
|
''')
|
||||||
|
for y in jsi.call_function('x'):
|
||||||
|
self.assertTrue(math.isnan(y))
|
||||||
|
|
||||||
|
def test_object(self):
|
||||||
|
self._test('function f() { return {}; }', {})
|
||||||
|
self._test('function f() { let a = {m1: 42, m2: 0 }; return [a["m1"], a.m2]; }', [42, 0])
|
||||||
|
self._test('function f() { let a; return a?.qq; }', JS_Undefined)
|
||||||
|
self._test('function f() { let a = {m1: 42, m2: 0 }; return a?.qq; }', JS_Undefined)
|
||||||
|
|
||||||
|
def test_regex(self):
|
||||||
|
self._test('function f() { let a=/,,[/,913,/](,)}/; }', None)
|
||||||
|
|
||||||
|
jsi = JSInterpreter('''
|
||||||
|
function x() { let a=/,,[/,913,/](,)}/; "".replace(a, ""); return a; }
|
||||||
|
''')
|
||||||
|
attrs = set(('findall', 'finditer', 'match', 'scanner', 'search',
|
||||||
|
'split', 'sub', 'subn'))
|
||||||
|
if sys.version_info >= (2, 7):
|
||||||
|
# documented for 2.6 but may not be found
|
||||||
|
attrs.update(('flags', 'groupindex', 'groups', 'pattern'))
|
||||||
|
self.assertSetEqual(set(dir(jsi.call_function('x'))) & attrs, attrs)
|
||||||
|
|
||||||
|
jsi = JSInterpreter('''
|
||||||
|
function x() { let a=/,,[/,913,/](,)}/i; return a; }
|
||||||
|
''')
|
||||||
|
self.assertEqual(jsi.call_function('x').flags & ~re.U, re.I)
|
||||||
|
|
||||||
|
jsi = JSInterpreter(r'function f() { let a=/,][}",],()}(\[)/; return a; }')
|
||||||
|
self.assertEqual(jsi.call_function('f').pattern, r',][}",],()}(\[)')
|
||||||
|
|
||||||
|
jsi = JSInterpreter(r'function f() { let a=[/[)\\]/]; return a[0]; }')
|
||||||
|
self.assertEqual(jsi.call_function('f').pattern, r'[)\\]')
|
||||||
|
|
||||||
|
def test_replace(self):
|
||||||
|
self._test('function f() { let a="data-name".replace("data-", ""); return a }',
|
||||||
|
'name')
|
||||||
|
self._test('function f() { let a="data-name".replace(new RegExp("^.+-"), ""); return a; }',
|
||||||
|
'name')
|
||||||
|
self._test('function f() { let a="data-name".replace(/^.+-/, ""); return a; }',
|
||||||
|
'name')
|
||||||
|
self._test('function f() { let a="data-name".replace(/a/g, "o"); return a; }',
|
||||||
|
'doto-nome')
|
||||||
|
self._test('function f() { let a="data-name".replaceAll("a", "o"); return a; }',
|
||||||
|
'doto-nome')
|
||||||
|
|
||||||
|
def test_char_code_at(self):
|
||||||
|
jsi = JSInterpreter('function f(i){return "test".charCodeAt(i)}')
|
||||||
|
self._test(jsi, 116, args=[0])
|
||||||
|
self._test(jsi, 101, args=[1])
|
||||||
|
self._test(jsi, 115, args=[2])
|
||||||
|
self._test(jsi, 116, args=[3])
|
||||||
|
self._test(jsi, None, args=[4])
|
||||||
|
self._test(jsi, 116, args=['not_a_number'])
|
||||||
|
|
||||||
|
def test_bitwise_operators_overflow(self):
|
||||||
|
self._test('function f(){return -524999584 << 5}', 379882496)
|
||||||
|
self._test('function f(){return 1236566549 << 5}', 915423904)
|
||||||
|
|
||||||
|
def test_bitwise_operators_typecast(self):
|
||||||
|
# madness
|
||||||
|
self._test('function f(){return null << 5}', 0)
|
||||||
|
self._test('function f(){return undefined >> 5}', 0)
|
||||||
|
self._test('function f(){return 42 << NaN}', 42)
|
||||||
|
self._test('function f(){return 42 << Infinity}', 42)
|
||||||
|
|
||||||
|
def test_negative(self):
|
||||||
|
self._test('function f(){return 2 * -2.0 ;}', -4)
|
||||||
|
self._test('function f(){return 2 - - -2 ;}', 0)
|
||||||
|
self._test('function f(){return 2 - - - -2 ;}', 4)
|
||||||
|
self._test('function f(){return 2 - + + - -2;}', 0)
|
||||||
|
self._test('function f(){return 2 + - + - -2;}', 0)
|
||||||
|
|
||||||
|
def test_32066(self):
|
||||||
|
self._test(
|
||||||
|
"function f(){return Math.pow(3, 5) + new Date('1970-01-01T08:01:42.000+08:00') / 1000 * -239 - -24205;}",
|
||||||
|
70)
|
||||||
|
|
||||||
|
@unittest.skip('Not yet working')
|
||||||
|
def test_packed(self):
|
||||||
|
self._test(
|
||||||
|
'''function f(p,a,c,k,e,d){while(c--)if(k[c])p=p.replace(new RegExp('\\b'+c.toString(a)+'\\b','g'),k[c]);return p}''',
|
||||||
|
'''h 7=g("1j");7.7h({7g:[{33:"w://7f-7e-7d-7c.v.7b/7a/79/78/77/76.74?t=73&s=2s&e=72&f=2t&71=70.0.0.1&6z=6y&6x=6w"}],6v:"w://32.v.u/6u.31",16:"r%",15:"r%",6t:"6s",6r:"",6q:"l",6p:"l",6o:"6n",6m:\'6l\',6k:"6j",9:[{33:"/2u?b=6i&n=50&6h=w://32.v.u/6g.31",6f:"6e"}],1y:{6d:1,6c:\'#6b\',6a:\'#69\',68:"67",66:30,65:r,},"64":{63:"%62 2m%m%61%5z%5y%5x.u%5w%5v%5u.2y%22 2k%m%1o%22 5t%m%1o%22 5s%m%1o%22 2j%m%5r%22 16%m%5q%22 15%m%5p%22 5o%2z%5n%5m%2z",5l:"w://v.u/d/1k/5k.2y",5j:[]},\'5i\':{"5h":"5g"},5f:"5e",5d:"w://v.u",5c:{},5b:l,1x:[0.25,0.50,0.75,1,1.25,1.5,2]});h 1m,1n,5a;h 59=0,58=0;h 7=g("1j");h 2x=0,57=0,56=0;$.55({54:{\'53-52\':\'2i-51\'}});7.j(\'4z\',6(x){c(5>0&&x.1l>=5&&1n!=1){1n=1;$(\'q.4y\').4x(\'4w\')}});7.j(\'13\',6(x){2x=x.1l});7.j(\'2g\',6(x){2w(x)});7.j(\'4v\',6(){$(\'q.2v\').4u()});6 2w(x){$(\'q.2v\').4t();c(1m)19;1m=1;17=0;c(4s.4r===l){17=1}$.4q(\'/2u?b=4p&2l=1k&4o=2t-4n-4m-2s-4l&4k=&4j=&4i=&17=\'+17,6(2r){$(\'#4h\').4g(2r)});$(\'.3-8-4f-4e:4d("4c")\').2h(6(e){2q();g().4b(0);g().4a(l)});6 2q(){h $14=$("<q />").2p({1l:"49",16:"r%",15:"r%",48:0,2n:0,2o:47,46:"45(10%, 10%, 10%, 0.4)","44-43":"42"});$("<41 />").2p({16:"60%",15:"60%",2o:40,"3z-2n":"3y"}).3x({\'2m\':\'/?b=3w&2l=1k\',\'2k\':\'0\',\'2j\':\'2i\'}).2f($14);$14.2h(6(){$(3v).3u();g().2g()});$14.2f($(\'#1j\'))}g().13(0);}6 3t(){h 9=7.1b(2e);2d.2c(9);c(9.n>1){1r(i=0;i<9.n;i++){c(9[i].1a==2e){2d.2c(\'!!=\'+i);7.1p(i)}}}}7.j(\'3s\',6(){g().1h("/2a/3r.29","3q 10 28",6(){g().13(g().27()+10)},"2b");$("q[26=2b]").23().21(\'.3-20-1z\');g().1h("/2a/3p.29","3o 10 28",6(){h 12=g().27()-10;c(12<0)12=0;g().13(12)},"24");$("q[26=24]").23().21(\'.3-20-1z\');});6 1i(){}7.j(\'3n\',6(){1i()});7.j(\'3m\',6(){1i()});7.j("k",6(y){h 9=7.1b();c(9.n<2)19;$(\'.3-8-3l-3k\').3j(6(){$(\'#3-8-a-k\').1e(\'3-8-a-z\');$(\'.3-a-k\').p(\'o-1f\',\'11\')});7.1h("/3i/3h.3g","3f 3e",6(){$(\'.3-1w\').3d(\'3-8-1v\');$(\'.3-8-1y, .3-8-1x\').p(\'o-1g\',\'11\');c($(\'.3-1w\').3c(\'3-8-1v\')){$(\'.3-a-k\').p(\'o-1g\',\'l\');$(\'.3-a-k\').p(\'o-1f\',\'l\');$(\'.3-8-a\').1e(\'3-8-a-z\');$(\'.3-8-a:1u\').3b(\'3-8-a-z\')}3a{$(\'.3-a-k\').p(\'o-1g\',\'11\');$(\'.3-a-k\').p(\'o-1f\',\'11\');$(\'.3-8-a:1u\').1e(\'3-8-a-z\')}},"39");7.j("38",6(y){1d.37(\'1c\',y.9[y.36].1a)});c(1d.1t(\'1c\')){35("1s(1d.1t(\'1c\'));",34)}});h 18;6 1s(1q){h 9=7.1b();c(9.n>1){1r(i=0;i<9.n;i++){c(9[i].1a==1q){c(i==18){19}18=i;7.1p(i)}}}}',36,270,'|||jw|||function|player|settings|tracks|submenu||if||||jwplayer|var||on|audioTracks|true|3D|length|aria|attr|div|100|||sx|filemoon|https||event|active||false|tt|seek|dd|height|width|adb|current_audio|return|name|getAudioTracks|default_audio|localStorage|removeClass|expanded|checked|addButton|callMeMaybe|vplayer|0fxcyc2ajhp1|position|vvplay|vvad|220|setCurrentAudioTrack|audio_name|for|audio_set|getItem|last|open|controls|playbackRates|captions|rewind|icon|insertAfter||detach|ff00||button|getPosition|sec|png|player8|ff11|log|console|track_name|appendTo|play|click|no|scrolling|frameborder|file_code|src|top|zIndex|css|showCCform|data|1662367683|383371|dl|video_ad|doPlay|prevt|mp4|3E||jpg|thumbs|file|300|setTimeout|currentTrack|setItem|audioTrackChanged|dualSound|else|addClass|hasClass|toggleClass|Track|Audio|svg|dualy|images|mousedown|buttons|topbar|playAttemptFailed|beforePlay|Rewind|fr|Forward|ff|ready|set_audio_track|remove|this|upload_srt|prop|50px|margin|1000001|iframe|center|align|text|rgba|background|1000000|left|absolute|pause|setCurrentCaptions|Upload|contains|item|content|html|fviews|referer|prem|embed|3e57249ef633e0d03bf76ceb8d8a4b65|216|83|hash|view|get|TokenZir|window|hide|show|complete|slow|fadeIn|video_ad_fadein|time||cache|Cache|Content|headers|ajaxSetup|v2done|tott|vastdone2|vastdone1|vvbefore|playbackRateControls|cast|aboutlink|FileMoon|abouttext|UHD|1870|qualityLabels|sites|GNOME_POWER|link|2Fiframe|3C|allowfullscreen|22360|22640|22no|marginheight|marginwidth|2FGNOME_POWER|2F0fxcyc2ajhp1|2Fe|2Ffilemoon|2F|3A||22https|3Ciframe|code|sharing|fontOpacity|backgroundOpacity|Tahoma|fontFamily|303030|backgroundColor|FFFFFF|color|userFontScale|thumbnails|kind|0fxcyc2ajhp10000|url|get_slides|start|startparam|none|preload|html5|primary|hlshtml|androidhls|duration|uniform|stretching|0fxcyc2ajhp1_xt|image|2048|sp|6871|asn|127|srv|43200|_g3XlBcu2lmD9oDexD2NLWSmah2Nu3XcDrl93m9PwXY|m3u8||master|0fxcyc2ajhp1_x|00076|01|hls2|to|s01|delivery|storage|moon|sources|setup'''.split('|'))
|
||||||
|
|
||||||
|
def test_join(self):
|
||||||
|
test_input = list('test')
|
||||||
|
tests = [
|
||||||
|
'function f(a, b){return a.join(b)}',
|
||||||
|
'function f(a, b){return Array.prototype.join.call(a, b)}',
|
||||||
|
'function f(a, b){return Array.prototype.join.apply(a, [b])}',
|
||||||
|
]
|
||||||
|
for test in tests:
|
||||||
|
jsi = JSInterpreter(test)
|
||||||
|
self._test(jsi, 'test', args=[test_input, ''])
|
||||||
|
self._test(jsi, 't-e-s-t', args=[test_input, '-'])
|
||||||
|
self._test(jsi, '', args=[[], '-'])
|
||||||
|
|
||||||
|
def test_split(self):
|
||||||
|
test_result = list('test')
|
||||||
|
tests = [
|
||||||
|
'function f(a, b){return a.split(b)}',
|
||||||
|
'function f(a, b){return String.prototype.split.call(a, b)}',
|
||||||
|
'function f(a, b){return String.prototype.split.apply(a, [b])}',
|
||||||
|
]
|
||||||
|
for test in tests:
|
||||||
|
jsi = JSInterpreter(test)
|
||||||
|
self._test(jsi, test_result, args=['test', ''])
|
||||||
|
self._test(jsi, test_result, args=['t-e-s-t', '-'])
|
||||||
|
self._test(jsi, [''], args=['', '-'])
|
||||||
|
self._test(jsi, [], args=['', ''])
|
||||||
|
|
||||||
|
def test_slice(self):
|
||||||
|
self._test('function f(){return [0, 1, 2, 3, 4, 5, 6, 7, 8].slice()}', [0, 1, 2, 3, 4, 5, 6, 7, 8])
|
||||||
|
self._test('function f(){return [0, 1, 2, 3, 4, 5, 6, 7, 8].slice(0)}', [0, 1, 2, 3, 4, 5, 6, 7, 8])
|
||||||
|
self._test('function f(){return [0, 1, 2, 3, 4, 5, 6, 7, 8].slice(5)}', [5, 6, 7, 8])
|
||||||
|
self._test('function f(){return [0, 1, 2, 3, 4, 5, 6, 7, 8].slice(99)}', [])
|
||||||
|
self._test('function f(){return [0, 1, 2, 3, 4, 5, 6, 7, 8].slice(-2)}', [7, 8])
|
||||||
|
self._test('function f(){return [0, 1, 2, 3, 4, 5, 6, 7, 8].slice(-99)}', [0, 1, 2, 3, 4, 5, 6, 7, 8])
|
||||||
|
self._test('function f(){return [0, 1, 2, 3, 4, 5, 6, 7, 8].slice(0, 0)}', [])
|
||||||
|
self._test('function f(){return [0, 1, 2, 3, 4, 5, 6, 7, 8].slice(1, 0)}', [])
|
||||||
|
self._test('function f(){return [0, 1, 2, 3, 4, 5, 6, 7, 8].slice(0, 1)}', [0])
|
||||||
|
self._test('function f(){return [0, 1, 2, 3, 4, 5, 6, 7, 8].slice(3, 6)}', [3, 4, 5])
|
||||||
|
self._test('function f(){return [0, 1, 2, 3, 4, 5, 6, 7, 8].slice(1, -1)}', [1, 2, 3, 4, 5, 6, 7])
|
||||||
|
self._test('function f(){return [0, 1, 2, 3, 4, 5, 6, 7, 8].slice(-1, 1)}', [])
|
||||||
|
self._test('function f(){return [0, 1, 2, 3, 4, 5, 6, 7, 8].slice(-3, -1)}', [6, 7])
|
||||||
|
self._test('function f(){return "012345678".slice()}', '012345678')
|
||||||
|
self._test('function f(){return "012345678".slice(0)}', '012345678')
|
||||||
|
self._test('function f(){return "012345678".slice(5)}', '5678')
|
||||||
|
self._test('function f(){return "012345678".slice(99)}', '')
|
||||||
|
self._test('function f(){return "012345678".slice(-2)}', '78')
|
||||||
|
self._test('function f(){return "012345678".slice(-99)}', '012345678')
|
||||||
|
self._test('function f(){return "012345678".slice(0, 0)}', '')
|
||||||
|
self._test('function f(){return "012345678".slice(1, 0)}', '')
|
||||||
|
self._test('function f(){return "012345678".slice(0, 1)}', '0')
|
||||||
|
self._test('function f(){return "012345678".slice(3, 6)}', '345')
|
||||||
|
self._test('function f(){return "012345678".slice(1, -1)}', '1234567')
|
||||||
|
self._test('function f(){return "012345678".slice(-1, 1)}', '')
|
||||||
|
self._test('function f(){return "012345678".slice(-3, -1)}', '67')
|
||||||
|
|
||||||
|
|
||||||
if __name__ == '__main__':
|
if __name__ == '__main__':
|
||||||
|
|
|
@ -38,6 +38,9 @@ class BaseTestSubtitles(unittest.TestCase):
|
||||||
self.DL = FakeYDL()
|
self.DL = FakeYDL()
|
||||||
self.ie = self.IE()
|
self.ie = self.IE()
|
||||||
self.DL.add_info_extractor(self.ie)
|
self.DL.add_info_extractor(self.ie)
|
||||||
|
if not self.IE.working():
|
||||||
|
print('Skipping: %s marked as not _WORKING' % self.IE.ie_key())
|
||||||
|
self.skipTest('IE marked as not _WORKING')
|
||||||
|
|
||||||
def getInfoDict(self):
|
def getInfoDict(self):
|
||||||
info_dict = self.DL.extract_info(self.url, download=False)
|
info_dict = self.DL.extract_info(self.url, download=False)
|
||||||
|
@ -56,6 +59,21 @@ class BaseTestSubtitles(unittest.TestCase):
|
||||||
|
|
||||||
|
|
||||||
class TestYoutubeSubtitles(BaseTestSubtitles):
|
class TestYoutubeSubtitles(BaseTestSubtitles):
|
||||||
|
# Available subtitles for QRS8MkLhQmM:
|
||||||
|
# Language formats
|
||||||
|
# ru vtt, ttml, srv3, srv2, srv1, json3
|
||||||
|
# fr vtt, ttml, srv3, srv2, srv1, json3
|
||||||
|
# en vtt, ttml, srv3, srv2, srv1, json3
|
||||||
|
# nl vtt, ttml, srv3, srv2, srv1, json3
|
||||||
|
# de vtt, ttml, srv3, srv2, srv1, json3
|
||||||
|
# ko vtt, ttml, srv3, srv2, srv1, json3
|
||||||
|
# it vtt, ttml, srv3, srv2, srv1, json3
|
||||||
|
# zh-Hant vtt, ttml, srv3, srv2, srv1, json3
|
||||||
|
# hi vtt, ttml, srv3, srv2, srv1, json3
|
||||||
|
# pt-BR vtt, ttml, srv3, srv2, srv1, json3
|
||||||
|
# es-MX vtt, ttml, srv3, srv2, srv1, json3
|
||||||
|
# ja vtt, ttml, srv3, srv2, srv1, json3
|
||||||
|
# pl vtt, ttml, srv3, srv2, srv1, json3
|
||||||
url = 'QRS8MkLhQmM'
|
url = 'QRS8MkLhQmM'
|
||||||
IE = YoutubeIE
|
IE = YoutubeIE
|
||||||
|
|
||||||
|
@ -64,41 +82,60 @@ class TestYoutubeSubtitles(BaseTestSubtitles):
|
||||||
self.DL.params['allsubtitles'] = True
|
self.DL.params['allsubtitles'] = True
|
||||||
subtitles = self.getSubtitles()
|
subtitles = self.getSubtitles()
|
||||||
self.assertEqual(len(subtitles.keys()), 13)
|
self.assertEqual(len(subtitles.keys()), 13)
|
||||||
self.assertEqual(md5(subtitles['en']), '3cb210999d3e021bd6c7f0ea751eab06')
|
self.assertEqual(md5(subtitles['en']), 'ae1bd34126571a77aabd4d276b28044d')
|
||||||
self.assertEqual(md5(subtitles['it']), '6d752b98c31f1cf8d597050c7a2cb4b5')
|
self.assertEqual(md5(subtitles['it']), '0e0b667ba68411d88fd1c5f4f4eab2f9')
|
||||||
for lang in ['fr', 'de']:
|
for lang in ['fr', 'de']:
|
||||||
self.assertTrue(subtitles.get(lang) is not None, 'Subtitles for \'%s\' not extracted' % lang)
|
self.assertTrue(subtitles.get(lang) is not None, 'Subtitles for \'%s\' not extracted' % lang)
|
||||||
|
|
||||||
def test_youtube_subtitles_ttml_format(self):
|
def _test_subtitles_format(self, fmt, md5_hash, lang='en'):
|
||||||
self.DL.params['writesubtitles'] = True
|
self.DL.params['writesubtitles'] = True
|
||||||
self.DL.params['subtitlesformat'] = 'ttml'
|
self.DL.params['subtitlesformat'] = fmt
|
||||||
subtitles = self.getSubtitles()
|
subtitles = self.getSubtitles()
|
||||||
self.assertEqual(md5(subtitles['en']), 'e306f8c42842f723447d9f63ad65df54')
|
self.assertEqual(md5(subtitles[lang]), md5_hash)
|
||||||
|
|
||||||
|
def test_youtube_subtitles_ttml_format(self):
|
||||||
|
self._test_subtitles_format('ttml', 'c97ddf1217390906fa9fbd34901f3da2')
|
||||||
|
|
||||||
def test_youtube_subtitles_vtt_format(self):
|
def test_youtube_subtitles_vtt_format(self):
|
||||||
self.DL.params['writesubtitles'] = True
|
self._test_subtitles_format('vtt', 'ae1bd34126571a77aabd4d276b28044d')
|
||||||
self.DL.params['subtitlesformat'] = 'vtt'
|
|
||||||
|
def test_youtube_subtitles_json3_format(self):
|
||||||
|
self._test_subtitles_format('json3', '688dd1ce0981683867e7fe6fde2a224b')
|
||||||
|
|
||||||
|
def _test_automatic_captions(self, url, lang):
|
||||||
|
self.url = url
|
||||||
|
self.DL.params['writeautomaticsub'] = True
|
||||||
|
self.DL.params['subtitleslangs'] = [lang]
|
||||||
subtitles = self.getSubtitles()
|
subtitles = self.getSubtitles()
|
||||||
self.assertEqual(md5(subtitles['en']), '3cb210999d3e021bd6c7f0ea751eab06')
|
self.assertTrue(subtitles[lang] is not None)
|
||||||
|
|
||||||
def test_youtube_automatic_captions(self):
|
def test_youtube_automatic_captions(self):
|
||||||
self.url = '8YoUxe5ncPo'
|
# Available automatic captions for 8YoUxe5ncPo:
|
||||||
self.DL.params['writeautomaticsub'] = True
|
# Language formats (all in vtt, ttml, srv3, srv2, srv1, json3)
|
||||||
self.DL.params['subtitleslangs'] = ['it']
|
# gu, zh-Hans, zh-Hant, gd, ga, gl, lb, la, lo, tt, tr,
|
||||||
subtitles = self.getSubtitles()
|
# lv, lt, tk, th, tg, te, fil, haw, yi, ceb, yo, de, da,
|
||||||
self.assertTrue(subtitles['it'] is not None)
|
# el, eo, en, eu, et, es, ru, rw, ro, bn, be, bg, uk, jv,
|
||||||
|
# bs, ja, or, xh, co, ca, cy, cs, ps, pt, pa, vi, pl, hy,
|
||||||
|
# hr, ht, hu, hmn, hi, ha, mg, uz, ml, mn, mi, mk, ur,
|
||||||
|
# mt, ms, mr, ug, ta, my, af, sw, is, am,
|
||||||
|
# *it*, iw, sv, ar,
|
||||||
|
# su, zu, az, id, ig, nl, no, ne, ny, fr, ku, fy, fa, fi,
|
||||||
|
# ka, kk, sr, sq, ko, kn, km, st, sk, si, so, sn, sm, sl,
|
||||||
|
# ky, sd
|
||||||
|
# ...
|
||||||
|
self._test_automatic_captions('8YoUxe5ncPo', 'it')
|
||||||
|
|
||||||
|
@unittest.skip('ASR subs all in all supported langs now')
|
||||||
def test_youtube_translated_subtitles(self):
|
def test_youtube_translated_subtitles(self):
|
||||||
# This video has a subtitles track, which can be translated
|
# This video has a subtitles track, which can be translated (#4555)
|
||||||
self.url = 'Ky9eprVWzlI'
|
self._test_automatic_captions('Ky9eprVWzlI', 'it')
|
||||||
self.DL.params['writeautomaticsub'] = True
|
|
||||||
self.DL.params['subtitleslangs'] = ['it']
|
|
||||||
subtitles = self.getSubtitles()
|
|
||||||
self.assertTrue(subtitles['it'] is not None)
|
|
||||||
|
|
||||||
def test_youtube_nosubtitles(self):
|
def test_youtube_nosubtitles(self):
|
||||||
self.DL.expect_warning('video doesn\'t have subtitles')
|
self.DL.expect_warning('video doesn\'t have subtitles')
|
||||||
self.url = 'n5BB19UTcdA'
|
# Available automatic captions for 8YoUxe5ncPo:
|
||||||
|
# ...
|
||||||
|
# 8YoUxe5ncPo has no subtitles
|
||||||
|
self.url = '8YoUxe5ncPo'
|
||||||
self.DL.params['writesubtitles'] = True
|
self.DL.params['writesubtitles'] = True
|
||||||
self.DL.params['allsubtitles'] = True
|
self.DL.params['allsubtitles'] = True
|
||||||
subtitles = self.getSubtitles()
|
subtitles = self.getSubtitles()
|
||||||
|
@ -128,6 +165,7 @@ class TestDailymotionSubtitles(BaseTestSubtitles):
|
||||||
self.assertFalse(subtitles)
|
self.assertFalse(subtitles)
|
||||||
|
|
||||||
|
|
||||||
|
@unittest.skip('IE broken')
|
||||||
class TestTedSubtitles(BaseTestSubtitles):
|
class TestTedSubtitles(BaseTestSubtitles):
|
||||||
url = 'http://www.ted.com/talks/dan_dennett_on_our_consciousness.html'
|
url = 'http://www.ted.com/talks/dan_dennett_on_our_consciousness.html'
|
||||||
IE = TEDIE
|
IE = TEDIE
|
||||||
|
@ -152,18 +190,19 @@ class TestVimeoSubtitles(BaseTestSubtitles):
|
||||||
self.DL.params['allsubtitles'] = True
|
self.DL.params['allsubtitles'] = True
|
||||||
subtitles = self.getSubtitles()
|
subtitles = self.getSubtitles()
|
||||||
self.assertEqual(set(subtitles.keys()), set(['de', 'en', 'es', 'fr']))
|
self.assertEqual(set(subtitles.keys()), set(['de', 'en', 'es', 'fr']))
|
||||||
self.assertEqual(md5(subtitles['en']), '8062383cf4dec168fc40a088aa6d5888')
|
self.assertEqual(md5(subtitles['en']), '386cbc9320b94e25cb364b97935e5dd1')
|
||||||
self.assertEqual(md5(subtitles['fr']), 'b6191146a6c5d3a452244d853fde6dc8')
|
self.assertEqual(md5(subtitles['fr']), 'c9b69eef35bc6641c0d4da8a04f9dfac')
|
||||||
|
|
||||||
def test_nosubtitles(self):
|
def test_nosubtitles(self):
|
||||||
self.DL.expect_warning('video doesn\'t have subtitles')
|
self.DL.expect_warning('video doesn\'t have subtitles')
|
||||||
self.url = 'http://vimeo.com/56015672'
|
self.url = 'http://vimeo.com/68093876'
|
||||||
self.DL.params['writesubtitles'] = True
|
self.DL.params['writesubtitles'] = True
|
||||||
self.DL.params['allsubtitles'] = True
|
self.DL.params['allsubtitles'] = True
|
||||||
subtitles = self.getSubtitles()
|
subtitles = self.getSubtitles()
|
||||||
self.assertFalse(subtitles)
|
self.assertFalse(subtitles)
|
||||||
|
|
||||||
|
|
||||||
|
@unittest.skip('IE broken')
|
||||||
class TestWallaSubtitles(BaseTestSubtitles):
|
class TestWallaSubtitles(BaseTestSubtitles):
|
||||||
url = 'http://vod.walla.co.il/movie/2705958/the-yes-men'
|
url = 'http://vod.walla.co.il/movie/2705958/the-yes-men'
|
||||||
IE = WallaIE
|
IE = WallaIE
|
||||||
|
@ -185,6 +224,7 @@ class TestWallaSubtitles(BaseTestSubtitles):
|
||||||
self.assertFalse(subtitles)
|
self.assertFalse(subtitles)
|
||||||
|
|
||||||
|
|
||||||
|
@unittest.skip('IE broken')
|
||||||
class TestCeskaTelevizeSubtitles(BaseTestSubtitles):
|
class TestCeskaTelevizeSubtitles(BaseTestSubtitles):
|
||||||
url = 'http://www.ceskatelevize.cz/ivysilani/10600540290-u6-uzasny-svet-techniky'
|
url = 'http://www.ceskatelevize.cz/ivysilani/10600540290-u6-uzasny-svet-techniky'
|
||||||
IE = CeskaTelevizeIE
|
IE = CeskaTelevizeIE
|
||||||
|
@ -206,6 +246,7 @@ class TestCeskaTelevizeSubtitles(BaseTestSubtitles):
|
||||||
self.assertFalse(subtitles)
|
self.assertFalse(subtitles)
|
||||||
|
|
||||||
|
|
||||||
|
@unittest.skip('IE broken')
|
||||||
class TestLyndaSubtitles(BaseTestSubtitles):
|
class TestLyndaSubtitles(BaseTestSubtitles):
|
||||||
url = 'http://www.lynda.com/Bootstrap-tutorials/Using-exercise-files/110885/114408-4.html'
|
url = 'http://www.lynda.com/Bootstrap-tutorials/Using-exercise-files/110885/114408-4.html'
|
||||||
IE = LyndaIE
|
IE = LyndaIE
|
||||||
|
@ -218,6 +259,7 @@ class TestLyndaSubtitles(BaseTestSubtitles):
|
||||||
self.assertEqual(md5(subtitles['en']), '09bbe67222259bed60deaa26997d73a7')
|
self.assertEqual(md5(subtitles['en']), '09bbe67222259bed60deaa26997d73a7')
|
||||||
|
|
||||||
|
|
||||||
|
@unittest.skip('IE broken')
|
||||||
class TestNPOSubtitles(BaseTestSubtitles):
|
class TestNPOSubtitles(BaseTestSubtitles):
|
||||||
url = 'http://www.npo.nl/nos-journaal/28-08-2014/POW_00722860'
|
url = 'http://www.npo.nl/nos-journaal/28-08-2014/POW_00722860'
|
||||||
IE = NPOIE
|
IE = NPOIE
|
||||||
|
@ -230,6 +272,7 @@ class TestNPOSubtitles(BaseTestSubtitles):
|
||||||
self.assertEqual(md5(subtitles['nl']), 'fc6435027572b63fb4ab143abd5ad3f4')
|
self.assertEqual(md5(subtitles['nl']), 'fc6435027572b63fb4ab143abd5ad3f4')
|
||||||
|
|
||||||
|
|
||||||
|
@unittest.skip('IE broken')
|
||||||
class TestMTVSubtitles(BaseTestSubtitles):
|
class TestMTVSubtitles(BaseTestSubtitles):
|
||||||
url = 'http://www.cc.com/video-clips/p63lk0/adam-devine-s-house-party-chasing-white-swans'
|
url = 'http://www.cc.com/video-clips/p63lk0/adam-devine-s-house-party-chasing-white-swans'
|
||||||
IE = ComedyCentralIE
|
IE = ComedyCentralIE
|
||||||
|
@ -252,23 +295,33 @@ class TestNRKSubtitles(BaseTestSubtitles):
|
||||||
def test_allsubtitles(self):
|
def test_allsubtitles(self):
|
||||||
self.DL.params['writesubtitles'] = True
|
self.DL.params['writesubtitles'] = True
|
||||||
self.DL.params['allsubtitles'] = True
|
self.DL.params['allsubtitles'] = True
|
||||||
|
self.DL.params['format'] = 'best/bestvideo'
|
||||||
subtitles = self.getSubtitles()
|
subtitles = self.getSubtitles()
|
||||||
self.assertEqual(set(subtitles.keys()), set(['no']))
|
self.assertEqual(set(subtitles.keys()), set(['nb-ttv']))
|
||||||
self.assertEqual(md5(subtitles['no']), '544fa917d3197fcbee64634559221cc2')
|
self.assertEqual(md5(subtitles['nb-ttv']), '67e06ff02d0deaf975e68f6cb8f6a149')
|
||||||
|
|
||||||
|
|
||||||
class TestRaiPlaySubtitles(BaseTestSubtitles):
|
class TestRaiPlaySubtitles(BaseTestSubtitles):
|
||||||
url = 'http://www.raiplay.it/video/2014/04/Report-del-07042014-cb27157f-9dd0-4aee-b788-b1f67643a391.html'
|
|
||||||
IE = RaiPlayIE
|
IE = RaiPlayIE
|
||||||
|
|
||||||
def test_allsubtitles(self):
|
def test_subtitles_key(self):
|
||||||
|
self.url = 'http://www.raiplay.it/video/2014/04/Report-del-07042014-cb27157f-9dd0-4aee-b788-b1f67643a391.html'
|
||||||
self.DL.params['writesubtitles'] = True
|
self.DL.params['writesubtitles'] = True
|
||||||
self.DL.params['allsubtitles'] = True
|
self.DL.params['allsubtitles'] = True
|
||||||
subtitles = self.getSubtitles()
|
subtitles = self.getSubtitles()
|
||||||
self.assertEqual(set(subtitles.keys()), set(['it']))
|
self.assertEqual(set(subtitles.keys()), set(['it']))
|
||||||
self.assertEqual(md5(subtitles['it']), 'b1d90a98755126b61e667567a1f6680a')
|
self.assertEqual(md5(subtitles['it']), 'b1d90a98755126b61e667567a1f6680a')
|
||||||
|
|
||||||
|
def test_subtitles_array_key(self):
|
||||||
|
self.url = 'https://www.raiplay.it/video/2020/12/Report---04-01-2021-2e90f1de-8eee-4de4-ac0e-78d21db5b600.html'
|
||||||
|
self.DL.params['writesubtitles'] = True
|
||||||
|
self.DL.params['allsubtitles'] = True
|
||||||
|
subtitles = self.getSubtitles()
|
||||||
|
self.assertEqual(set(subtitles.keys()), set(['it']))
|
||||||
|
self.assertEqual(md5(subtitles['it']), '4b3264186fbb103508abe5311cfcb9cd')
|
||||||
|
|
||||||
|
|
||||||
|
@unittest.skip('IE broken - DRM only')
|
||||||
class TestVikiSubtitles(BaseTestSubtitles):
|
class TestVikiSubtitles(BaseTestSubtitles):
|
||||||
url = 'http://www.viki.com/videos/1060846v-punch-episode-18'
|
url = 'http://www.viki.com/videos/1060846v-punch-episode-18'
|
||||||
IE = VikiIE
|
IE = VikiIE
|
||||||
|
@ -295,6 +348,7 @@ class TestThePlatformSubtitles(BaseTestSubtitles):
|
||||||
self.assertEqual(md5(subtitles['en']), '97e7670cbae3c4d26ae8bcc7fdd78d4b')
|
self.assertEqual(md5(subtitles['en']), '97e7670cbae3c4d26ae8bcc7fdd78d4b')
|
||||||
|
|
||||||
|
|
||||||
|
@unittest.skip('IE broken')
|
||||||
class TestThePlatformFeedSubtitles(BaseTestSubtitles):
|
class TestThePlatformFeedSubtitles(BaseTestSubtitles):
|
||||||
url = 'http://feed.theplatform.com/f/7wvmTC/msnbc_video-p-test?form=json&pretty=true&range=-40&byGuid=n_hardball_5biden_140207'
|
url = 'http://feed.theplatform.com/f/7wvmTC/msnbc_video-p-test?form=json&pretty=true&range=-40&byGuid=n_hardball_5biden_140207'
|
||||||
IE = ThePlatformFeedIE
|
IE = ThePlatformFeedIE
|
||||||
|
@ -330,7 +384,7 @@ class TestDemocracynowSubtitles(BaseTestSubtitles):
|
||||||
self.DL.params['allsubtitles'] = True
|
self.DL.params['allsubtitles'] = True
|
||||||
subtitles = self.getSubtitles()
|
subtitles = self.getSubtitles()
|
||||||
self.assertEqual(set(subtitles.keys()), set(['en']))
|
self.assertEqual(set(subtitles.keys()), set(['en']))
|
||||||
self.assertEqual(md5(subtitles['en']), 'acaca989e24a9e45a6719c9b3d60815c')
|
self.assertEqual(md5(subtitles['en']), 'a3cc4c0b5eadd74d9974f1c1f5101045')
|
||||||
|
|
||||||
def test_subtitles_in_page(self):
|
def test_subtitles_in_page(self):
|
||||||
self.url = 'http://www.democracynow.org/2015/7/3/this_flag_comes_down_today_bree'
|
self.url = 'http://www.democracynow.org/2015/7/3/this_flag_comes_down_today_bree'
|
||||||
|
@ -338,7 +392,7 @@ class TestDemocracynowSubtitles(BaseTestSubtitles):
|
||||||
self.DL.params['allsubtitles'] = True
|
self.DL.params['allsubtitles'] = True
|
||||||
subtitles = self.getSubtitles()
|
subtitles = self.getSubtitles()
|
||||||
self.assertEqual(set(subtitles.keys()), set(['en']))
|
self.assertEqual(set(subtitles.keys()), set(['en']))
|
||||||
self.assertEqual(md5(subtitles['en']), 'acaca989e24a9e45a6719c9b3d60815c')
|
self.assertEqual(md5(subtitles['en']), 'a3cc4c0b5eadd74d9974f1c1f5101045')
|
||||||
|
|
||||||
|
|
||||||
if __name__ == '__main__':
|
if __name__ == '__main__':
|
||||||
|
|
|
@ -5,16 +5,18 @@ from __future__ import unicode_literals
|
||||||
import os
|
import os
|
||||||
import sys
|
import sys
|
||||||
import unittest
|
import unittest
|
||||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
|
||||||
|
|
||||||
|
dirn = os.path.dirname
|
||||||
|
|
||||||
|
sys.path.insert(0, dirn(dirn(os.path.abspath(__file__))))
|
||||||
|
|
||||||
import errno
|
import errno
|
||||||
import io
|
|
||||||
import json
|
import json
|
||||||
import re
|
import re
|
||||||
import subprocess
|
import subprocess
|
||||||
|
|
||||||
from youtube_dl.swfinterp import SWFInterpreter
|
from youtube_dl.swfinterp import SWFInterpreter
|
||||||
|
from youtube_dl.compat import compat_open as open
|
||||||
|
|
||||||
|
|
||||||
TEST_DIR = os.path.join(
|
TEST_DIR = os.path.join(
|
||||||
|
@ -43,7 +45,7 @@ def _make_testfunc(testfile):
|
||||||
'-static-link-runtime-shared-libraries', as_file])
|
'-static-link-runtime-shared-libraries', as_file])
|
||||||
except OSError as ose:
|
except OSError as ose:
|
||||||
if ose.errno == errno.ENOENT:
|
if ose.errno == errno.ENOENT:
|
||||||
print('mxmlc not found! Skipping test.')
|
self.skipTest('mxmlc not found!')
|
||||||
return
|
return
|
||||||
raise
|
raise
|
||||||
|
|
||||||
|
@ -51,7 +53,7 @@ def _make_testfunc(testfile):
|
||||||
swf_content = swf_f.read()
|
swf_content = swf_f.read()
|
||||||
swfi = SWFInterpreter(swf_content)
|
swfi = SWFInterpreter(swf_content)
|
||||||
|
|
||||||
with io.open(as_file, 'r', encoding='utf-8') as as_f:
|
with open(as_file, 'r', encoding='utf-8') as as_f:
|
||||||
as_content = as_f.read()
|
as_content = as_f.read()
|
||||||
|
|
||||||
def _find_spec(key):
|
def _find_spec(key):
|
||||||
|
|
509
test/test_traversal.py
Normal file
509
test/test_traversal.py
Normal file
|
@ -0,0 +1,509 @@
|
||||||
|
#!/usr/bin/env python
|
||||||
|
# coding: utf-8
|
||||||
|
from __future__ import unicode_literals
|
||||||
|
|
||||||
|
# Allow direct execution
|
||||||
|
import os
|
||||||
|
import sys
|
||||||
|
import unittest
|
||||||
|
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||||
|
|
||||||
|
|
||||||
|
import re
|
||||||
|
|
||||||
|
from youtube_dl.traversal import (
|
||||||
|
dict_get,
|
||||||
|
get_first,
|
||||||
|
T,
|
||||||
|
traverse_obj,
|
||||||
|
)
|
||||||
|
from youtube_dl.compat import (
|
||||||
|
compat_etree_fromstring,
|
||||||
|
compat_http_cookies,
|
||||||
|
compat_str,
|
||||||
|
)
|
||||||
|
from youtube_dl.utils import (
|
||||||
|
int_or_none,
|
||||||
|
str_or_none,
|
||||||
|
)
|
||||||
|
|
||||||
|
_TEST_DATA = {
|
||||||
|
100: 100,
|
||||||
|
1.2: 1.2,
|
||||||
|
'str': 'str',
|
||||||
|
'None': None,
|
||||||
|
'...': Ellipsis,
|
||||||
|
'urls': [
|
||||||
|
{'index': 0, 'url': 'https://www.example.com/0'},
|
||||||
|
{'index': 1, 'url': 'https://www.example.com/1'},
|
||||||
|
],
|
||||||
|
'data': (
|
||||||
|
{'index': 2},
|
||||||
|
{'index': 3},
|
||||||
|
),
|
||||||
|
'dict': {},
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
if sys.version_info < (3, 0):
|
||||||
|
class _TestCase(unittest.TestCase):
|
||||||
|
|
||||||
|
def assertCountEqual(self, *args, **kwargs):
|
||||||
|
return self.assertItemsEqual(*args, **kwargs)
|
||||||
|
else:
|
||||||
|
_TestCase = unittest.TestCase
|
||||||
|
|
||||||
|
|
||||||
|
class TestTraversal(_TestCase):
|
||||||
|
def assertMaybeCountEqual(self, *args, **kwargs):
|
||||||
|
if sys.version_info < (3, 7):
|
||||||
|
# random dict order
|
||||||
|
return self.assertCountEqual(*args, **kwargs)
|
||||||
|
else:
|
||||||
|
return self.assertEqual(*args, **kwargs)
|
||||||
|
|
||||||
|
def test_traverse_obj(self):
|
||||||
|
# instant compat
|
||||||
|
str = compat_str
|
||||||
|
|
||||||
|
# define a pukka Iterable
|
||||||
|
def iter_range(stop):
|
||||||
|
for from_ in range(stop):
|
||||||
|
yield from_
|
||||||
|
|
||||||
|
# Test base functionality
|
||||||
|
self.assertEqual(traverse_obj(_TEST_DATA, ('str',)), 'str',
|
||||||
|
msg='allow tuple path')
|
||||||
|
self.assertEqual(traverse_obj(_TEST_DATA, ['str']), 'str',
|
||||||
|
msg='allow list path')
|
||||||
|
self.assertEqual(traverse_obj(_TEST_DATA, (value for value in ("str",))), 'str',
|
||||||
|
msg='allow iterable path')
|
||||||
|
self.assertEqual(traverse_obj(_TEST_DATA, 'str'), 'str',
|
||||||
|
msg='single items should be treated as a path')
|
||||||
|
self.assertEqual(traverse_obj(_TEST_DATA, None), _TEST_DATA)
|
||||||
|
self.assertEqual(traverse_obj(_TEST_DATA, 100), 100)
|
||||||
|
self.assertEqual(traverse_obj(_TEST_DATA, 1.2), 1.2)
|
||||||
|
|
||||||
|
# Test Ellipsis behavior
|
||||||
|
self.assertCountEqual(traverse_obj(_TEST_DATA, Ellipsis),
|
||||||
|
(item for item in _TEST_DATA.values() if item not in (None, {})),
|
||||||
|
msg='`...` should give all non-discarded values')
|
||||||
|
self.assertCountEqual(traverse_obj(_TEST_DATA, ('urls', 0, Ellipsis)), _TEST_DATA['urls'][0].values(),
|
||||||
|
msg='`...` selection for dicts should select all values')
|
||||||
|
self.assertEqual(traverse_obj(_TEST_DATA, (Ellipsis, Ellipsis, 'url')),
|
||||||
|
['https://www.example.com/0', 'https://www.example.com/1'],
|
||||||
|
msg='nested `...` queries should work')
|
||||||
|
self.assertCountEqual(traverse_obj(_TEST_DATA, (Ellipsis, Ellipsis, 'index')), iter_range(4),
|
||||||
|
msg='`...` query result should be flattened')
|
||||||
|
self.assertEqual(traverse_obj(iter(range(4)), Ellipsis), list(range(4)),
|
||||||
|
msg='`...` should accept iterables')
|
||||||
|
|
||||||
|
# Test function as key
|
||||||
|
self.assertEqual(traverse_obj(_TEST_DATA, lambda x, y: x == 'urls' and isinstance(y, list)),
|
||||||
|
[_TEST_DATA['urls']],
|
||||||
|
msg='function as query key should perform a filter based on (key, value)')
|
||||||
|
self.assertCountEqual(traverse_obj(_TEST_DATA, lambda _, x: isinstance(x[0], str)), set(('str',)),
|
||||||
|
msg='exceptions in the query function should be caught')
|
||||||
|
self.assertEqual(traverse_obj(iter(range(4)), lambda _, x: x % 2 == 0), [0, 2],
|
||||||
|
msg='function key should accept iterables')
|
||||||
|
if __debug__:
|
||||||
|
with self.assertRaises(Exception, msg='Wrong function signature should raise in debug'):
|
||||||
|
traverse_obj(_TEST_DATA, lambda a: Ellipsis)
|
||||||
|
with self.assertRaises(Exception, msg='Wrong function signature should raise in debug'):
|
||||||
|
traverse_obj(_TEST_DATA, lambda a, b, c: Ellipsis)
|
||||||
|
|
||||||
|
# Test set as key (transformation/type, like `expected_type`)
|
||||||
|
self.assertEqual(traverse_obj(_TEST_DATA, (Ellipsis, T(str.upper), )), ['STR'],
|
||||||
|
msg='Function in set should be a transformation')
|
||||||
|
self.assertEqual(traverse_obj(_TEST_DATA, ('fail', T(lambda _: 'const'))), 'const',
|
||||||
|
msg='Function in set should always be called')
|
||||||
|
self.assertEqual(traverse_obj(_TEST_DATA, (Ellipsis, T(str))), ['str'],
|
||||||
|
msg='Type in set should be a type filter')
|
||||||
|
self.assertMaybeCountEqual(traverse_obj(_TEST_DATA, (Ellipsis, T(str, int))), [100, 'str'],
|
||||||
|
msg='Multiple types in set should be a type filter')
|
||||||
|
self.assertEqual(traverse_obj(_TEST_DATA, T(dict)), _TEST_DATA,
|
||||||
|
msg='A single set should be wrapped into a path')
|
||||||
|
self.assertEqual(traverse_obj(_TEST_DATA, (Ellipsis, T(str.upper))), ['STR'],
|
||||||
|
msg='Transformation function should not raise')
|
||||||
|
self.assertMaybeCountEqual(traverse_obj(_TEST_DATA, (Ellipsis, T(str_or_none))),
|
||||||
|
[item for item in map(str_or_none, _TEST_DATA.values()) if item is not None],
|
||||||
|
msg='Function in set should be a transformation')
|
||||||
|
if __debug__:
|
||||||
|
with self.assertRaises(Exception, msg='Sets with length != 1 should raise in debug'):
|
||||||
|
traverse_obj(_TEST_DATA, set())
|
||||||
|
with self.assertRaises(Exception, msg='Sets with length != 1 should raise in debug'):
|
||||||
|
traverse_obj(_TEST_DATA, set((str.upper, str)))
|
||||||
|
|
||||||
|
# Test `slice` as a key
|
||||||
|
_SLICE_DATA = [0, 1, 2, 3, 4]
|
||||||
|
self.assertEqual(traverse_obj(_TEST_DATA, ('dict', slice(1))), None,
|
||||||
|
msg='slice on a dictionary should not throw')
|
||||||
|
self.assertEqual(traverse_obj(_SLICE_DATA, slice(1)), _SLICE_DATA[:1],
|
||||||
|
msg='slice key should apply slice to sequence')
|
||||||
|
self.assertEqual(traverse_obj(_SLICE_DATA, slice(1, 2)), _SLICE_DATA[1:2],
|
||||||
|
msg='slice key should apply slice to sequence')
|
||||||
|
self.assertEqual(traverse_obj(_SLICE_DATA, slice(1, 4, 2)), _SLICE_DATA[1:4:2],
|
||||||
|
msg='slice key should apply slice to sequence')
|
||||||
|
|
||||||
|
# Test alternative paths
|
||||||
|
self.assertEqual(traverse_obj(_TEST_DATA, 'fail', 'str'), 'str',
|
||||||
|
msg='multiple `paths` should be treated as alternative paths')
|
||||||
|
self.assertEqual(traverse_obj(_TEST_DATA, 'str', 100), 'str',
|
||||||
|
msg='alternatives should exit early')
|
||||||
|
self.assertEqual(traverse_obj(_TEST_DATA, 'fail', 'fail'), None,
|
||||||
|
msg='alternatives should return `default` if exhausted')
|
||||||
|
self.assertEqual(traverse_obj(_TEST_DATA, (Ellipsis, 'fail'), 100), 100,
|
||||||
|
msg='alternatives should track their own branching return')
|
||||||
|
self.assertEqual(traverse_obj(_TEST_DATA, ('dict', Ellipsis), ('data', Ellipsis)), list(_TEST_DATA['data']),
|
||||||
|
msg='alternatives on empty objects should search further')
|
||||||
|
|
||||||
|
# Test branch and path nesting
|
||||||
|
self.assertEqual(traverse_obj(_TEST_DATA, ('urls', (3, 0), 'url')), ['https://www.example.com/0'],
|
||||||
|
msg='tuple as key should be treated as branches')
|
||||||
|
self.assertEqual(traverse_obj(_TEST_DATA, ('urls', [3, 0], 'url')), ['https://www.example.com/0'],
|
||||||
|
msg='list as key should be treated as branches')
|
||||||
|
self.assertEqual(traverse_obj(_TEST_DATA, ('urls', ((1, 'fail'), (0, 'url')))), ['https://www.example.com/0'],
|
||||||
|
msg='double nesting in path should be treated as paths')
|
||||||
|
self.assertEqual(traverse_obj(['0', [1, 2]], [(0, 1), 0]), [1],
|
||||||
|
msg='do not fail early on branching')
|
||||||
|
self.assertCountEqual(traverse_obj(_TEST_DATA, ('urls', ((1, ('fail', 'url')), (0, 'url')))),
|
||||||
|
['https://www.example.com/0', 'https://www.example.com/1'],
|
||||||
|
msg='triple nesting in path should be treated as branches')
|
||||||
|
self.assertEqual(traverse_obj(_TEST_DATA, ('urls', ('fail', (Ellipsis, 'url')))),
|
||||||
|
['https://www.example.com/0', 'https://www.example.com/1'],
|
||||||
|
msg='ellipsis as branch path start gets flattened')
|
||||||
|
|
||||||
|
# Test dictionary as key
|
||||||
|
self.assertEqual(traverse_obj(_TEST_DATA, {0: 100, 1: 1.2}), {0: 100, 1: 1.2},
|
||||||
|
msg='dict key should result in a dict with the same keys')
|
||||||
|
self.assertEqual(traverse_obj(_TEST_DATA, {0: ('urls', 0, 'url')}),
|
||||||
|
{0: 'https://www.example.com/0'},
|
||||||
|
msg='dict key should allow paths')
|
||||||
|
self.assertEqual(traverse_obj(_TEST_DATA, {0: ('urls', (3, 0), 'url')}),
|
||||||
|
{0: ['https://www.example.com/0']},
|
||||||
|
msg='tuple in dict path should be treated as branches')
|
||||||
|
self.assertEqual(traverse_obj(_TEST_DATA, {0: ('urls', ((1, 'fail'), (0, 'url')))}),
|
||||||
|
{0: ['https://www.example.com/0']},
|
||||||
|
msg='double nesting in dict path should be treated as paths')
|
||||||
|
self.assertEqual(traverse_obj(_TEST_DATA, {0: ('urls', ((1, ('fail', 'url')), (0, 'url')))}),
|
||||||
|
{0: ['https://www.example.com/1', 'https://www.example.com/0']},
|
||||||
|
msg='triple nesting in dict path should be treated as branches')
|
||||||
|
self.assertEqual(traverse_obj(_TEST_DATA, {0: 'fail'}), {},
|
||||||
|
msg='remove `None` values when top level dict key fails')
|
||||||
|
self.assertEqual(traverse_obj(_TEST_DATA, {0: 'fail'}, default=Ellipsis), {0: Ellipsis},
|
||||||
|
msg='use `default` if key fails and `default`')
|
||||||
|
self.assertEqual(traverse_obj(_TEST_DATA, {0: 'dict'}), {},
|
||||||
|
msg='remove empty values when dict key')
|
||||||
|
self.assertEqual(traverse_obj(_TEST_DATA, {0: 'dict'}, default=Ellipsis), {0: Ellipsis},
|
||||||
|
msg='use `default` when dict key and a default')
|
||||||
|
self.assertEqual(traverse_obj(_TEST_DATA, {0: {0: 'fail'}}), {},
|
||||||
|
msg='remove empty values when nested dict key fails')
|
||||||
|
self.assertEqual(traverse_obj(None, {0: 'fail'}), {},
|
||||||
|
msg='default to dict if pruned')
|
||||||
|
self.assertEqual(traverse_obj(None, {0: 'fail'}, default=Ellipsis), {0: Ellipsis},
|
||||||
|
msg='default to dict if pruned and default is given')
|
||||||
|
self.assertEqual(traverse_obj(_TEST_DATA, {0: {0: 'fail'}}, default=Ellipsis), {0: {0: Ellipsis}},
|
||||||
|
msg='use nested `default` when nested dict key fails and `default`')
|
||||||
|
self.assertEqual(traverse_obj(_TEST_DATA, {0: ('dict', Ellipsis)}), {},
|
||||||
|
msg='remove key if branch in dict key not successful')
|
||||||
|
|
||||||
|
# Testing default parameter behavior
|
||||||
|
_DEFAULT_DATA = {'None': None, 'int': 0, 'list': []}
|
||||||
|
self.assertEqual(traverse_obj(_DEFAULT_DATA, 'fail'), None,
|
||||||
|
msg='default value should be `None`')
|
||||||
|
self.assertEqual(traverse_obj(_DEFAULT_DATA, 'fail', 'fail', default=Ellipsis), Ellipsis,
|
||||||
|
msg='chained fails should result in default')
|
||||||
|
self.assertEqual(traverse_obj(_DEFAULT_DATA, 'None', 'int'), 0,
|
||||||
|
msg='should not short cirquit on `None`')
|
||||||
|
self.assertEqual(traverse_obj(_DEFAULT_DATA, 'fail', default=1), 1,
|
||||||
|
msg='invalid dict key should result in `default`')
|
||||||
|
self.assertEqual(traverse_obj(_DEFAULT_DATA, 'None', default=1), 1,
|
||||||
|
msg='`None` is a deliberate sentinel and should become `default`')
|
||||||
|
self.assertEqual(traverse_obj(_DEFAULT_DATA, ('list', 10)), None,
|
||||||
|
msg='`IndexError` should result in `default`')
|
||||||
|
self.assertEqual(traverse_obj(_DEFAULT_DATA, (Ellipsis, 'fail'), default=1), 1,
|
||||||
|
msg='if branched but not successful return `default` if defined, not `[]`')
|
||||||
|
self.assertEqual(traverse_obj(_DEFAULT_DATA, (Ellipsis, 'fail'), default=None), None,
|
||||||
|
msg='if branched but not successful return `default` even if `default` is `None`')
|
||||||
|
self.assertEqual(traverse_obj(_DEFAULT_DATA, (Ellipsis, 'fail')), [],
|
||||||
|
msg='if branched but not successful return `[]`, not `default`')
|
||||||
|
self.assertEqual(traverse_obj(_DEFAULT_DATA, ('list', Ellipsis)), [],
|
||||||
|
msg='if branched but object is empty return `[]`, not `default`')
|
||||||
|
self.assertEqual(traverse_obj(None, Ellipsis), [],
|
||||||
|
msg='if branched but object is `None` return `[]`, not `default`')
|
||||||
|
self.assertEqual(traverse_obj({0: None}, (0, Ellipsis)), [],
|
||||||
|
msg='if branched but state is `None` return `[]`, not `default`')
|
||||||
|
|
||||||
|
branching_paths = [
|
||||||
|
('fail', Ellipsis),
|
||||||
|
(Ellipsis, 'fail'),
|
||||||
|
100 * ('fail',) + (Ellipsis,),
|
||||||
|
(Ellipsis,) + 100 * ('fail',),
|
||||||
|
]
|
||||||
|
for branching_path in branching_paths:
|
||||||
|
self.assertEqual(traverse_obj({}, branching_path), [],
|
||||||
|
msg='if branched but state is `None`, return `[]` (not `default`)')
|
||||||
|
self.assertEqual(traverse_obj({}, 'fail', branching_path), [],
|
||||||
|
msg='if branching in last alternative and previous did not match, return `[]` (not `default`)')
|
||||||
|
self.assertEqual(traverse_obj({0: 'x'}, 0, branching_path), 'x',
|
||||||
|
msg='if branching in last alternative and previous did match, return single value')
|
||||||
|
self.assertEqual(traverse_obj({0: 'x'}, branching_path, 0), 'x',
|
||||||
|
msg='if branching in first alternative and non-branching path does match, return single value')
|
||||||
|
self.assertEqual(traverse_obj({}, branching_path, 'fail'), None,
|
||||||
|
msg='if branching in first alternative and non-branching path does not match, return `default`')
|
||||||
|
|
||||||
|
# Testing expected_type behavior
|
||||||
|
_EXPECTED_TYPE_DATA = {'str': 'str', 'int': 0}
|
||||||
|
self.assertEqual(traverse_obj(_EXPECTED_TYPE_DATA, 'str', expected_type=str),
|
||||||
|
'str', msg='accept matching `expected_type` type')
|
||||||
|
self.assertEqual(traverse_obj(_EXPECTED_TYPE_DATA, 'str', expected_type=int),
|
||||||
|
None, msg='reject non-matching `expected_type` type')
|
||||||
|
self.assertEqual(traverse_obj(_EXPECTED_TYPE_DATA, 'int', expected_type=lambda x: str(x)),
|
||||||
|
'0', msg='transform type using type function')
|
||||||
|
self.assertEqual(traverse_obj(_EXPECTED_TYPE_DATA, 'str', expected_type=lambda _: 1 / 0),
|
||||||
|
None, msg='wrap expected_type function in try_call')
|
||||||
|
self.assertEqual(traverse_obj(_EXPECTED_TYPE_DATA, Ellipsis, expected_type=str),
|
||||||
|
['str'], msg='eliminate items that expected_type fails on')
|
||||||
|
self.assertEqual(traverse_obj(_TEST_DATA, {0: 100, 1: 1.2}, expected_type=int),
|
||||||
|
{0: 100}, msg='type as expected_type should filter dict values')
|
||||||
|
self.assertEqual(traverse_obj(_TEST_DATA, {0: 100, 1: 1.2, 2: 'None'}, expected_type=str_or_none),
|
||||||
|
{0: '100', 1: '1.2'}, msg='function as expected_type should transform dict values')
|
||||||
|
self.assertEqual(traverse_obj(_TEST_DATA, ({0: 1.2}, 0, set((int_or_none,))), expected_type=int),
|
||||||
|
1, msg='expected_type should not filter non-final dict values')
|
||||||
|
self.assertEqual(traverse_obj(_TEST_DATA, {0: {0: 100, 1: 'str'}}, expected_type=int),
|
||||||
|
{0: {0: 100}}, msg='expected_type should transform deep dict values')
|
||||||
|
self.assertEqual(traverse_obj(_TEST_DATA, [({0: '...'}, {0: '...'})], expected_type=type(Ellipsis)),
|
||||||
|
[{0: Ellipsis}, {0: Ellipsis}], msg='expected_type should transform branched dict values')
|
||||||
|
self.assertEqual(traverse_obj({1: {3: 4}}, [(1, 2), 3], expected_type=int),
|
||||||
|
[4], msg='expected_type regression for type matching in tuple branching')
|
||||||
|
self.assertEqual(traverse_obj(_TEST_DATA, ['data', Ellipsis], expected_type=int),
|
||||||
|
[], msg='expected_type regression for type matching in dict result')
|
||||||
|
|
||||||
|
# Test get_all behavior
|
||||||
|
_GET_ALL_DATA = {'key': [0, 1, 2]}
|
||||||
|
self.assertEqual(traverse_obj(_GET_ALL_DATA, ('key', Ellipsis), get_all=False), 0,
|
||||||
|
msg='if not `get_all`, return only first matching value')
|
||||||
|
self.assertEqual(traverse_obj(_GET_ALL_DATA, Ellipsis, get_all=False), [0, 1, 2],
|
||||||
|
msg='do not overflatten if not `get_all`')
|
||||||
|
|
||||||
|
# Test casesense behavior
|
||||||
|
_CASESENSE_DATA = {
|
||||||
|
'KeY': 'value0',
|
||||||
|
0: {
|
||||||
|
'KeY': 'value1',
|
||||||
|
0: {'KeY': 'value2'},
|
||||||
|
},
|
||||||
|
# FULLWIDTH LATIN CAPITAL LETTER K
|
||||||
|
'\uff2bey': 'value3',
|
||||||
|
}
|
||||||
|
self.assertEqual(traverse_obj(_CASESENSE_DATA, 'key'), None,
|
||||||
|
msg='dict keys should be case sensitive unless `casesense`')
|
||||||
|
self.assertEqual(traverse_obj(_CASESENSE_DATA, 'keY',
|
||||||
|
casesense=False), 'value0',
|
||||||
|
msg='allow non matching key case if `casesense`')
|
||||||
|
self.assertEqual(traverse_obj(_CASESENSE_DATA, '\uff4bey', # FULLWIDTH LATIN SMALL LETTER K
|
||||||
|
casesense=False), 'value3',
|
||||||
|
msg='allow non matching Unicode key case if `casesense`')
|
||||||
|
self.assertEqual(traverse_obj(_CASESENSE_DATA, (0, ('keY',)),
|
||||||
|
casesense=False), ['value1'],
|
||||||
|
msg='allow non matching key case in branch if `casesense`')
|
||||||
|
self.assertEqual(traverse_obj(_CASESENSE_DATA, (0, ((0, 'keY'),)),
|
||||||
|
casesense=False), ['value2'],
|
||||||
|
msg='allow non matching key case in branch path if `casesense`')
|
||||||
|
|
||||||
|
# Test traverse_string behavior
|
||||||
|
_TRAVERSE_STRING_DATA = {'str': 'str', 1.2: 1.2}
|
||||||
|
self.assertEqual(traverse_obj(_TRAVERSE_STRING_DATA, ('str', 0)), None,
|
||||||
|
msg='do not traverse into string if not `traverse_string`')
|
||||||
|
self.assertEqual(traverse_obj(_TRAVERSE_STRING_DATA, ('str', 0),
|
||||||
|
_traverse_string=True), 's',
|
||||||
|
msg='traverse into string if `traverse_string`')
|
||||||
|
self.assertEqual(traverse_obj(_TRAVERSE_STRING_DATA, (1.2, 1),
|
||||||
|
_traverse_string=True), '.',
|
||||||
|
msg='traverse into converted data if `traverse_string`')
|
||||||
|
self.assertEqual(traverse_obj(_TRAVERSE_STRING_DATA, ('str', Ellipsis),
|
||||||
|
_traverse_string=True), 'str',
|
||||||
|
msg='`...` should result in string (same value) if `traverse_string`')
|
||||||
|
self.assertEqual(traverse_obj(_TRAVERSE_STRING_DATA, ('str', slice(0, None, 2)),
|
||||||
|
_traverse_string=True), 'sr',
|
||||||
|
msg='`slice` should result in string if `traverse_string`')
|
||||||
|
self.assertEqual(traverse_obj(_TRAVERSE_STRING_DATA, ('str', lambda i, v: i or v == 's'),
|
||||||
|
_traverse_string=True), 'str',
|
||||||
|
msg='function should result in string if `traverse_string`')
|
||||||
|
self.assertEqual(traverse_obj(_TRAVERSE_STRING_DATA, ('str', (0, 2)),
|
||||||
|
_traverse_string=True), ['s', 'r'],
|
||||||
|
msg='branching should result in list if `traverse_string`')
|
||||||
|
self.assertEqual(traverse_obj({}, (0, Ellipsis), _traverse_string=True), [],
|
||||||
|
msg='branching should result in list if `traverse_string`')
|
||||||
|
self.assertEqual(traverse_obj({}, (0, lambda x, y: True), _traverse_string=True), [],
|
||||||
|
msg='branching should result in list if `traverse_string`')
|
||||||
|
self.assertEqual(traverse_obj({}, (0, slice(1)), _traverse_string=True), [],
|
||||||
|
msg='branching should result in list if `traverse_string`')
|
||||||
|
|
||||||
|
# Test re.Match as input obj
|
||||||
|
mobj = re.match(r'^0(12)(?P<group>3)(4)?$', '0123')
|
||||||
|
self.assertEqual(traverse_obj(mobj, Ellipsis), [x for x in mobj.groups() if x is not None],
|
||||||
|
msg='`...` on a `re.Match` should give its `groups()`')
|
||||||
|
self.assertEqual(traverse_obj(mobj, lambda k, _: k in (0, 2)), ['0123', '3'],
|
||||||
|
msg='function on a `re.Match` should give groupno, value starting at 0')
|
||||||
|
self.assertEqual(traverse_obj(mobj, 'group'), '3',
|
||||||
|
msg='str key on a `re.Match` should give group with that name')
|
||||||
|
self.assertEqual(traverse_obj(mobj, 2), '3',
|
||||||
|
msg='int key on a `re.Match` should give group with that name')
|
||||||
|
self.assertEqual(traverse_obj(mobj, 'gRoUp', casesense=False), '3',
|
||||||
|
msg='str key on a `re.Match` should respect casesense')
|
||||||
|
self.assertEqual(traverse_obj(mobj, 'fail'), None,
|
||||||
|
msg='failing str key on a `re.Match` should return `default`')
|
||||||
|
self.assertEqual(traverse_obj(mobj, 'gRoUpS', casesense=False), None,
|
||||||
|
msg='failing str key on a `re.Match` should return `default`')
|
||||||
|
self.assertEqual(traverse_obj(mobj, 8), None,
|
||||||
|
msg='failing int key on a `re.Match` should return `default`')
|
||||||
|
self.assertEqual(traverse_obj(mobj, lambda k, _: k in (0, 'group')), ['0123', '3'],
|
||||||
|
msg='function on a `re.Match` should give group name as well')
|
||||||
|
|
||||||
|
# Test xml.etree.ElementTree.Element as input obj
|
||||||
|
etree = compat_etree_fromstring('''<?xml version="1.0"?>
|
||||||
|
<data>
|
||||||
|
<country name="Liechtenstein">
|
||||||
|
<rank>1</rank>
|
||||||
|
<year>2008</year>
|
||||||
|
<gdppc>141100</gdppc>
|
||||||
|
<neighbor name="Austria" direction="E"/>
|
||||||
|
<neighbor name="Switzerland" direction="W"/>
|
||||||
|
</country>
|
||||||
|
<country name="Singapore">
|
||||||
|
<rank>4</rank>
|
||||||
|
<year>2011</year>
|
||||||
|
<gdppc>59900</gdppc>
|
||||||
|
<neighbor name="Malaysia" direction="N"/>
|
||||||
|
</country>
|
||||||
|
<country name="Panama">
|
||||||
|
<rank>68</rank>
|
||||||
|
<year>2011</year>
|
||||||
|
<gdppc>13600</gdppc>
|
||||||
|
<neighbor name="Costa Rica" direction="W"/>
|
||||||
|
<neighbor name="Colombia" direction="E"/>
|
||||||
|
</country>
|
||||||
|
</data>''')
|
||||||
|
self.assertEqual(traverse_obj(etree, ''), etree,
|
||||||
|
msg='empty str key should return the element itself')
|
||||||
|
self.assertEqual(traverse_obj(etree, 'country'), list(etree),
|
||||||
|
msg='str key should return all children with that tag name')
|
||||||
|
self.assertEqual(traverse_obj(etree, Ellipsis), list(etree),
|
||||||
|
msg='`...` as key should return all children')
|
||||||
|
self.assertEqual(traverse_obj(etree, lambda _, x: x[0].text == '4'), [etree[1]],
|
||||||
|
msg='function as key should get element as value')
|
||||||
|
self.assertEqual(traverse_obj(etree, lambda i, _: i == 1), [etree[1]],
|
||||||
|
msg='function as key should get index as key')
|
||||||
|
self.assertEqual(traverse_obj(etree, 0), etree[0],
|
||||||
|
msg='int key should return the nth child')
|
||||||
|
self.assertEqual(traverse_obj(etree, './/neighbor/@name'),
|
||||||
|
['Austria', 'Switzerland', 'Malaysia', 'Costa Rica', 'Colombia'],
|
||||||
|
msg='`@<attribute>` at end of path should give that attribute')
|
||||||
|
self.assertEqual(traverse_obj(etree, '//neighbor/@fail'), [None, None, None, None, None],
|
||||||
|
msg='`@<nonexistent>` at end of path should give `None`')
|
||||||
|
self.assertEqual(traverse_obj(etree, ('//neighbor/@', 2)), {'name': 'Malaysia', 'direction': 'N'},
|
||||||
|
msg='`@` should give the full attribute dict')
|
||||||
|
self.assertEqual(traverse_obj(etree, '//year/text()'), ['2008', '2011', '2011'],
|
||||||
|
msg='`text()` at end of path should give the inner text')
|
||||||
|
self.assertEqual(traverse_obj(etree, '//*[@direction]/@direction'), ['E', 'W', 'N', 'W', 'E'],
|
||||||
|
msg='full python xpath features should be supported')
|
||||||
|
self.assertEqual(traverse_obj(etree, (0, '@name')), 'Liechtenstein',
|
||||||
|
msg='special transformations should act on current element')
|
||||||
|
self.assertEqual(traverse_obj(etree, ('country', 0, Ellipsis, 'text()', T(int_or_none))), [1, 2008, 141100],
|
||||||
|
msg='special transformations should act on current element')
|
||||||
|
|
||||||
|
def test_traversal_unbranching(self):
|
||||||
|
self.assertEqual(traverse_obj(_TEST_DATA, [(100, 1.2), all]), [100, 1.2],
|
||||||
|
msg='`all` should give all results as list')
|
||||||
|
self.assertEqual(traverse_obj(_TEST_DATA, [(100, 1.2), any]), 100,
|
||||||
|
msg='`any` should give the first result')
|
||||||
|
self.assertEqual(traverse_obj(_TEST_DATA, [100, all]), [100],
|
||||||
|
msg='`all` should give list if non branching')
|
||||||
|
self.assertEqual(traverse_obj(_TEST_DATA, [100, any]), 100,
|
||||||
|
msg='`any` should give single item if non branching')
|
||||||
|
self.assertEqual(traverse_obj(_TEST_DATA, [('dict', 'None', 100), all]), [100],
|
||||||
|
msg='`all` should filter `None` and empty dict')
|
||||||
|
self.assertEqual(traverse_obj(_TEST_DATA, [('dict', 'None', 100), any]), 100,
|
||||||
|
msg='`any` should filter `None` and empty dict')
|
||||||
|
self.assertEqual(traverse_obj(_TEST_DATA, [{
|
||||||
|
'all': [('dict', 'None', 100, 1.2), all],
|
||||||
|
'any': [('dict', 'None', 100, 1.2), any],
|
||||||
|
}]), {'all': [100, 1.2], 'any': 100},
|
||||||
|
msg='`all`/`any` should apply to each dict path separately')
|
||||||
|
self.assertEqual(traverse_obj(_TEST_DATA, [{
|
||||||
|
'all': [('dict', 'None', 100, 1.2), all],
|
||||||
|
'any': [('dict', 'None', 100, 1.2), any],
|
||||||
|
}], get_all=False), {'all': [100, 1.2], 'any': 100},
|
||||||
|
msg='`all`/`any` should apply to dict regardless of `get_all`')
|
||||||
|
self.assertIs(traverse_obj(_TEST_DATA, [('dict', 'None', 100, 1.2), all, T(float)]), None,
|
||||||
|
msg='`all` should reset branching status')
|
||||||
|
self.assertIs(traverse_obj(_TEST_DATA, [('dict', 'None', 100, 1.2), any, T(float)]), None,
|
||||||
|
msg='`any` should reset branching status')
|
||||||
|
self.assertEqual(traverse_obj(_TEST_DATA, [('dict', 'None', 100, 1.2), all, Ellipsis, T(float)]), [1.2],
|
||||||
|
msg='`all` should allow further branching')
|
||||||
|
self.assertEqual(traverse_obj(_TEST_DATA, [('dict', 'None', 'urls', 'data'), any, Ellipsis, 'index']), [0, 1],
|
||||||
|
msg='`any` should allow further branching')
|
||||||
|
|
||||||
|
def test_traversal_morsel(self):
|
||||||
|
values = {
|
||||||
|
'expires': 'a',
|
||||||
|
'path': 'b',
|
||||||
|
'comment': 'c',
|
||||||
|
'domain': 'd',
|
||||||
|
'max-age': 'e',
|
||||||
|
'secure': 'f',
|
||||||
|
'httponly': 'g',
|
||||||
|
'version': 'h',
|
||||||
|
'samesite': 'i',
|
||||||
|
}
|
||||||
|
# SameSite added in Py3.8, breaks .update for 3.5-3.7
|
||||||
|
if sys.version_info < (3, 8):
|
||||||
|
del values['samesite']
|
||||||
|
morsel = compat_http_cookies.Morsel()
|
||||||
|
morsel.set(str('item_key'), 'item_value', 'coded_value')
|
||||||
|
morsel.update(values)
|
||||||
|
values['key'] = str('item_key')
|
||||||
|
values['value'] = 'item_value'
|
||||||
|
values = dict((str(k), v) for k, v in values.items())
|
||||||
|
# make test pass even without ordered dict
|
||||||
|
value_set = set(values.values())
|
||||||
|
|
||||||
|
for key, value in values.items():
|
||||||
|
self.assertEqual(traverse_obj(morsel, key), value,
|
||||||
|
msg='Morsel should provide access to all values')
|
||||||
|
self.assertEqual(set(traverse_obj(morsel, Ellipsis)), value_set,
|
||||||
|
msg='`...` should yield all values')
|
||||||
|
self.assertEqual(set(traverse_obj(morsel, lambda k, v: True)), value_set,
|
||||||
|
msg='function key should yield all values')
|
||||||
|
self.assertIs(traverse_obj(morsel, [(None,), any]), morsel,
|
||||||
|
msg='Morsel should not be implicitly changed to dict on usage')
|
||||||
|
|
||||||
|
def test_get_first(self):
|
||||||
|
self.assertEqual(get_first([{'a': None}, {'a': 'spam'}], 'a'), 'spam')
|
||||||
|
|
||||||
|
def test_dict_get(self):
|
||||||
|
FALSE_VALUES = {
|
||||||
|
'none': None,
|
||||||
|
'false': False,
|
||||||
|
'zero': 0,
|
||||||
|
'empty_string': '',
|
||||||
|
'empty_list': [],
|
||||||
|
}
|
||||||
|
d = FALSE_VALUES.copy()
|
||||||
|
d['a'] = 42
|
||||||
|
self.assertEqual(dict_get(d, 'a'), 42)
|
||||||
|
self.assertEqual(dict_get(d, 'b'), None)
|
||||||
|
self.assertEqual(dict_get(d, 'b', 42), 42)
|
||||||
|
self.assertEqual(dict_get(d, ('a', )), 42)
|
||||||
|
self.assertEqual(dict_get(d, ('b', 'a', )), 42)
|
||||||
|
self.assertEqual(dict_get(d, ('b', 'c', 'a', 'd', )), 42)
|
||||||
|
self.assertEqual(dict_get(d, ('b', 'c', )), None)
|
||||||
|
self.assertEqual(dict_get(d, ('b', 'c', ), 42), 42)
|
||||||
|
for key, false_value in FALSE_VALUES.items():
|
||||||
|
self.assertEqual(dict_get(d, ('b', 'c', key, )), None)
|
||||||
|
self.assertEqual(dict_get(d, ('b', 'c', key, ), skip_false_values=False), false_value)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
unittest.main()
|
|
@ -2,19 +2,21 @@ from __future__ import unicode_literals
|
||||||
|
|
||||||
# Allow direct execution
|
# Allow direct execution
|
||||||
import os
|
import os
|
||||||
|
import re
|
||||||
import sys
|
import sys
|
||||||
import unittest
|
import unittest
|
||||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
|
||||||
|
|
||||||
import io
|
dirn = os.path.dirname
|
||||||
import re
|
|
||||||
|
|
||||||
rootDir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
rootDir = dirn(dirn(os.path.abspath(__file__)))
|
||||||
|
|
||||||
|
sys.path.insert(0, rootDir)
|
||||||
|
|
||||||
IGNORED_FILES = [
|
IGNORED_FILES = [
|
||||||
'setup.py', # http://bugs.python.org/issue13943
|
'setup.py', # http://bugs.python.org/issue13943
|
||||||
'conf.py',
|
'conf.py',
|
||||||
'buildserver.py',
|
'buildserver.py',
|
||||||
|
'get-pip.py',
|
||||||
]
|
]
|
||||||
|
|
||||||
IGNORED_DIRS = [
|
IGNORED_DIRS = [
|
||||||
|
@ -23,6 +25,7 @@ IGNORED_DIRS = [
|
||||||
]
|
]
|
||||||
|
|
||||||
from test.helper import assertRegexpMatches
|
from test.helper import assertRegexpMatches
|
||||||
|
from youtube_dl.compat import compat_open as open
|
||||||
|
|
||||||
|
|
||||||
class TestUnicodeLiterals(unittest.TestCase):
|
class TestUnicodeLiterals(unittest.TestCase):
|
||||||
|
@ -40,7 +43,7 @@ class TestUnicodeLiterals(unittest.TestCase):
|
||||||
continue
|
continue
|
||||||
|
|
||||||
fn = os.path.join(dirpath, basename)
|
fn = os.path.join(dirpath, basename)
|
||||||
with io.open(fn, encoding='utf-8') as inf:
|
with open(fn, encoding='utf-8') as inf:
|
||||||
code = inf.read()
|
code = inf.read()
|
||||||
|
|
||||||
if "'" not in code and '"' not in code:
|
if "'" not in code and '"' not in code:
|
||||||
|
|
|
@ -12,24 +12,29 @@ sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||||
|
|
||||||
# Various small unit tests
|
# Various small unit tests
|
||||||
import io
|
import io
|
||||||
|
import itertools
|
||||||
import json
|
import json
|
||||||
|
import types
|
||||||
import xml.etree.ElementTree
|
import xml.etree.ElementTree
|
||||||
|
|
||||||
from youtube_dl.utils import (
|
from youtube_dl.utils import (
|
||||||
|
_UnsafeExtensionError,
|
||||||
age_restricted,
|
age_restricted,
|
||||||
args_to_str,
|
args_to_str,
|
||||||
encode_base_n,
|
base_url,
|
||||||
caesar,
|
caesar,
|
||||||
clean_html,
|
clean_html,
|
||||||
|
clean_podcast_url,
|
||||||
date_from_str,
|
date_from_str,
|
||||||
DateRange,
|
DateRange,
|
||||||
detect_exe_version,
|
detect_exe_version,
|
||||||
determine_ext,
|
determine_ext,
|
||||||
dict_get,
|
encode_base_n,
|
||||||
encode_compat_str,
|
encode_compat_str,
|
||||||
encodeFilename,
|
encodeFilename,
|
||||||
escape_rfc3986,
|
escape_rfc3986,
|
||||||
escape_url,
|
escape_url,
|
||||||
|
expand_path,
|
||||||
extract_attributes,
|
extract_attributes,
|
||||||
ExtractorError,
|
ExtractorError,
|
||||||
find_xpath_attr,
|
find_xpath_attr,
|
||||||
|
@ -43,8 +48,11 @@ from youtube_dl.utils import (
|
||||||
int_or_none,
|
int_or_none,
|
||||||
intlist_to_bytes,
|
intlist_to_bytes,
|
||||||
is_html,
|
is_html,
|
||||||
|
join_nonempty,
|
||||||
js_to_json,
|
js_to_json,
|
||||||
|
LazyList,
|
||||||
limit_length,
|
limit_length,
|
||||||
|
lowercase_escape,
|
||||||
merge_dicts,
|
merge_dicts,
|
||||||
mimetype2ext,
|
mimetype2ext,
|
||||||
month_by_name,
|
month_by_name,
|
||||||
|
@ -53,24 +61,26 @@ from youtube_dl.utils import (
|
||||||
OnDemandPagedList,
|
OnDemandPagedList,
|
||||||
orderedSet,
|
orderedSet,
|
||||||
parse_age_limit,
|
parse_age_limit,
|
||||||
|
parse_bitrate,
|
||||||
parse_duration,
|
parse_duration,
|
||||||
parse_filesize,
|
parse_filesize,
|
||||||
|
parse_codecs,
|
||||||
parse_count,
|
parse_count,
|
||||||
parse_iso8601,
|
parse_iso8601,
|
||||||
parse_resolution,
|
parse_resolution,
|
||||||
parse_bitrate,
|
parse_qs,
|
||||||
pkcs1pad,
|
pkcs1pad,
|
||||||
read_batch_urls,
|
|
||||||
sanitize_filename,
|
|
||||||
sanitize_path,
|
|
||||||
sanitize_url,
|
|
||||||
expand_path,
|
|
||||||
prepend_extension,
|
prepend_extension,
|
||||||
replace_extension,
|
read_batch_urls,
|
||||||
remove_start,
|
remove_start,
|
||||||
remove_end,
|
remove_end,
|
||||||
remove_quotes,
|
remove_quotes,
|
||||||
|
replace_extension,
|
||||||
rot47,
|
rot47,
|
||||||
|
sanitize_filename,
|
||||||
|
sanitize_path,
|
||||||
|
sanitize_url,
|
||||||
|
sanitized_Request,
|
||||||
shell_quote,
|
shell_quote,
|
||||||
smuggle_url,
|
smuggle_url,
|
||||||
str_to_int,
|
str_to_int,
|
||||||
|
@ -78,19 +88,19 @@ from youtube_dl.utils import (
|
||||||
strip_or_none,
|
strip_or_none,
|
||||||
subtitles_filename,
|
subtitles_filename,
|
||||||
timeconvert,
|
timeconvert,
|
||||||
|
try_call,
|
||||||
unescapeHTML,
|
unescapeHTML,
|
||||||
unified_strdate,
|
unified_strdate,
|
||||||
unified_timestamp,
|
unified_timestamp,
|
||||||
unsmuggle_url,
|
unsmuggle_url,
|
||||||
uppercase_escape,
|
uppercase_escape,
|
||||||
lowercase_escape,
|
|
||||||
url_basename,
|
url_basename,
|
||||||
url_or_none,
|
url_or_none,
|
||||||
base_url,
|
|
||||||
urljoin,
|
urljoin,
|
||||||
urlencode_postdata,
|
urlencode_postdata,
|
||||||
urshift,
|
urshift,
|
||||||
update_url_query,
|
update_url_query,
|
||||||
|
variadic,
|
||||||
version_tuple,
|
version_tuple,
|
||||||
xpath_with_ns,
|
xpath_with_ns,
|
||||||
xpath_element,
|
xpath_element,
|
||||||
|
@ -103,7 +113,7 @@ from youtube_dl.utils import (
|
||||||
cli_option,
|
cli_option,
|
||||||
cli_valueless_option,
|
cli_valueless_option,
|
||||||
cli_bool_option,
|
cli_bool_option,
|
||||||
parse_codecs,
|
YoutubeDLHandler,
|
||||||
)
|
)
|
||||||
from youtube_dl.compat import (
|
from youtube_dl.compat import (
|
||||||
compat_chr,
|
compat_chr,
|
||||||
|
@ -111,12 +121,13 @@ from youtube_dl.compat import (
|
||||||
compat_getenv,
|
compat_getenv,
|
||||||
compat_os_name,
|
compat_os_name,
|
||||||
compat_setenv,
|
compat_setenv,
|
||||||
|
compat_str,
|
||||||
compat_urlparse,
|
compat_urlparse,
|
||||||
compat_parse_qs,
|
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
class TestUtil(unittest.TestCase):
|
class TestUtil(unittest.TestCase):
|
||||||
|
|
||||||
def test_timeconvert(self):
|
def test_timeconvert(self):
|
||||||
self.assertTrue(timeconvert('') is None)
|
self.assertTrue(timeconvert('') is None)
|
||||||
self.assertTrue(timeconvert('bougrg') is None)
|
self.assertTrue(timeconvert('bougrg') is None)
|
||||||
|
@ -235,6 +246,19 @@ class TestUtil(unittest.TestCase):
|
||||||
self.assertEqual(sanitize_url('httpss://foo.bar'), 'https://foo.bar')
|
self.assertEqual(sanitize_url('httpss://foo.bar'), 'https://foo.bar')
|
||||||
self.assertEqual(sanitize_url('rmtps://foo.bar'), 'rtmps://foo.bar')
|
self.assertEqual(sanitize_url('rmtps://foo.bar'), 'rtmps://foo.bar')
|
||||||
self.assertEqual(sanitize_url('https://foo.bar'), 'https://foo.bar')
|
self.assertEqual(sanitize_url('https://foo.bar'), 'https://foo.bar')
|
||||||
|
self.assertEqual(sanitize_url('foo bar'), 'foo bar')
|
||||||
|
|
||||||
|
def test_sanitized_Request(self):
|
||||||
|
self.assertFalse(sanitized_Request('http://foo.bar').has_header('Authorization'))
|
||||||
|
self.assertFalse(sanitized_Request('http://:foo.bar').has_header('Authorization'))
|
||||||
|
self.assertEqual(sanitized_Request('http://@foo.bar').get_header('Authorization'),
|
||||||
|
'Basic Og==')
|
||||||
|
self.assertEqual(sanitized_Request('http://:pass@foo.bar').get_header('Authorization'),
|
||||||
|
'Basic OnBhc3M=')
|
||||||
|
self.assertEqual(sanitized_Request('http://user:@foo.bar').get_header('Authorization'),
|
||||||
|
'Basic dXNlcjo=')
|
||||||
|
self.assertEqual(sanitized_Request('http://user:pass@foo.bar').get_header('Authorization'),
|
||||||
|
'Basic dXNlcjpwYXNz')
|
||||||
|
|
||||||
def test_expand_path(self):
|
def test_expand_path(self):
|
||||||
def env(var):
|
def env(var):
|
||||||
|
@ -248,6 +272,27 @@ class TestUtil(unittest.TestCase):
|
||||||
expand_path('~/%s' % env('YOUTUBE_DL_EXPATH_PATH')),
|
expand_path('~/%s' % env('YOUTUBE_DL_EXPATH_PATH')),
|
||||||
'%s/expanded' % compat_getenv('HOME'))
|
'%s/expanded' % compat_getenv('HOME'))
|
||||||
|
|
||||||
|
_uncommon_extensions = [
|
||||||
|
('exe', 'abc.exe.ext'),
|
||||||
|
('de', 'abc.de.ext'),
|
||||||
|
('../.mp4', None),
|
||||||
|
('..\\.mp4', None),
|
||||||
|
]
|
||||||
|
|
||||||
|
def assertUnsafeExtension(self, ext=None):
|
||||||
|
assert_raises = self.assertRaises(_UnsafeExtensionError)
|
||||||
|
assert_raises.ext = ext
|
||||||
|
orig_exit = assert_raises.__exit__
|
||||||
|
|
||||||
|
def my_exit(self_, exc_type, exc_val, exc_tb):
|
||||||
|
did_raise = orig_exit(exc_type, exc_val, exc_tb)
|
||||||
|
if did_raise and assert_raises.ext is not None:
|
||||||
|
self.assertEqual(assert_raises.ext, assert_raises.exception.extension, 'Unsafe extension not as unexpected')
|
||||||
|
return did_raise
|
||||||
|
|
||||||
|
assert_raises.__exit__ = types.MethodType(my_exit, assert_raises)
|
||||||
|
return assert_raises
|
||||||
|
|
||||||
def test_prepend_extension(self):
|
def test_prepend_extension(self):
|
||||||
self.assertEqual(prepend_extension('abc.ext', 'temp'), 'abc.temp.ext')
|
self.assertEqual(prepend_extension('abc.ext', 'temp'), 'abc.temp.ext')
|
||||||
self.assertEqual(prepend_extension('abc.ext', 'temp', 'ext'), 'abc.temp.ext')
|
self.assertEqual(prepend_extension('abc.ext', 'temp', 'ext'), 'abc.temp.ext')
|
||||||
|
@ -256,6 +301,19 @@ class TestUtil(unittest.TestCase):
|
||||||
self.assertEqual(prepend_extension('.abc', 'temp'), '.abc.temp')
|
self.assertEqual(prepend_extension('.abc', 'temp'), '.abc.temp')
|
||||||
self.assertEqual(prepend_extension('.abc.ext', 'temp'), '.abc.temp.ext')
|
self.assertEqual(prepend_extension('.abc.ext', 'temp'), '.abc.temp.ext')
|
||||||
|
|
||||||
|
# Test uncommon extensions
|
||||||
|
self.assertEqual(prepend_extension('abc.ext', 'bin'), 'abc.bin.ext')
|
||||||
|
for ext, result in self._uncommon_extensions:
|
||||||
|
with self.assertUnsafeExtension(ext):
|
||||||
|
prepend_extension('abc', ext)
|
||||||
|
if result:
|
||||||
|
self.assertEqual(prepend_extension('abc.ext', ext, 'ext'), result)
|
||||||
|
else:
|
||||||
|
with self.assertUnsafeExtension(ext):
|
||||||
|
prepend_extension('abc.ext', ext, 'ext')
|
||||||
|
with self.assertUnsafeExtension(ext):
|
||||||
|
prepend_extension('abc.unexpected_ext', ext, 'ext')
|
||||||
|
|
||||||
def test_replace_extension(self):
|
def test_replace_extension(self):
|
||||||
self.assertEqual(replace_extension('abc.ext', 'temp'), 'abc.temp')
|
self.assertEqual(replace_extension('abc.ext', 'temp'), 'abc.temp')
|
||||||
self.assertEqual(replace_extension('abc.ext', 'temp', 'ext'), 'abc.temp')
|
self.assertEqual(replace_extension('abc.ext', 'temp', 'ext'), 'abc.temp')
|
||||||
|
@ -264,6 +322,16 @@ class TestUtil(unittest.TestCase):
|
||||||
self.assertEqual(replace_extension('.abc', 'temp'), '.abc.temp')
|
self.assertEqual(replace_extension('.abc', 'temp'), '.abc.temp')
|
||||||
self.assertEqual(replace_extension('.abc.ext', 'temp'), '.abc.temp')
|
self.assertEqual(replace_extension('.abc.ext', 'temp'), '.abc.temp')
|
||||||
|
|
||||||
|
# Test uncommon extensions
|
||||||
|
self.assertEqual(replace_extension('abc.ext', 'bin'), 'abc.unknown_video')
|
||||||
|
for ext, _ in self._uncommon_extensions:
|
||||||
|
with self.assertUnsafeExtension(ext):
|
||||||
|
replace_extension('abc', ext)
|
||||||
|
with self.assertUnsafeExtension(ext):
|
||||||
|
replace_extension('abc.ext', ext, 'ext')
|
||||||
|
with self.assertUnsafeExtension(ext):
|
||||||
|
replace_extension('abc.unexpected_ext', ext, 'ext')
|
||||||
|
|
||||||
def test_subtitles_filename(self):
|
def test_subtitles_filename(self):
|
||||||
self.assertEqual(subtitles_filename('abc.ext', 'en', 'vtt'), 'abc.en.vtt')
|
self.assertEqual(subtitles_filename('abc.ext', 'en', 'vtt'), 'abc.en.vtt')
|
||||||
self.assertEqual(subtitles_filename('abc.ext', 'en', 'vtt', 'ext'), 'abc.en.vtt')
|
self.assertEqual(subtitles_filename('abc.ext', 'en', 'vtt', 'ext'), 'abc.en.vtt')
|
||||||
|
@ -369,6 +437,9 @@ class TestUtil(unittest.TestCase):
|
||||||
self.assertEqual(unified_timestamp('Sep 11, 2013 | 5:49 AM'), 1378878540)
|
self.assertEqual(unified_timestamp('Sep 11, 2013 | 5:49 AM'), 1378878540)
|
||||||
self.assertEqual(unified_timestamp('December 15, 2017 at 7:49 am'), 1513324140)
|
self.assertEqual(unified_timestamp('December 15, 2017 at 7:49 am'), 1513324140)
|
||||||
self.assertEqual(unified_timestamp('2018-03-14T08:32:43.1493874+00:00'), 1521016363)
|
self.assertEqual(unified_timestamp('2018-03-14T08:32:43.1493874+00:00'), 1521016363)
|
||||||
|
self.assertEqual(unified_timestamp('December 31 1969 20:00:01 EDT'), 1)
|
||||||
|
self.assertEqual(unified_timestamp('Wednesday 31 December 1969 18:01:26 MDT'), 86)
|
||||||
|
self.assertEqual(unified_timestamp('12/31/1969 20:01:18 EDT', False), 78)
|
||||||
|
|
||||||
def test_determine_ext(self):
|
def test_determine_ext(self):
|
||||||
self.assertEqual(determine_ext('http://example.com/foo/bar.mp4/?download'), 'mp4')
|
self.assertEqual(determine_ext('http://example.com/foo/bar.mp4/?download'), 'mp4')
|
||||||
|
@ -490,11 +561,14 @@ class TestUtil(unittest.TestCase):
|
||||||
self.assertEqual(float_or_none(set()), None)
|
self.assertEqual(float_or_none(set()), None)
|
||||||
|
|
||||||
def test_int_or_none(self):
|
def test_int_or_none(self):
|
||||||
|
self.assertEqual(int_or_none(42), 42)
|
||||||
self.assertEqual(int_or_none('42'), 42)
|
self.assertEqual(int_or_none('42'), 42)
|
||||||
self.assertEqual(int_or_none(''), None)
|
self.assertEqual(int_or_none(''), None)
|
||||||
self.assertEqual(int_or_none(None), None)
|
self.assertEqual(int_or_none(None), None)
|
||||||
self.assertEqual(int_or_none([]), None)
|
self.assertEqual(int_or_none([]), None)
|
||||||
self.assertEqual(int_or_none(set()), None)
|
self.assertEqual(int_or_none(set()), None)
|
||||||
|
self.assertEqual(int_or_none('42', base=8), 34)
|
||||||
|
self.assertRaises(TypeError, int_or_none(42, base=8))
|
||||||
|
|
||||||
def test_str_to_int(self):
|
def test_str_to_int(self):
|
||||||
self.assertEqual(str_to_int('123,456'), 123456)
|
self.assertEqual(str_to_int('123,456'), 123456)
|
||||||
|
@ -554,6 +628,11 @@ class TestUtil(unittest.TestCase):
|
||||||
self.assertEqual(url_or_none('http$://foo.de'), None)
|
self.assertEqual(url_or_none('http$://foo.de'), None)
|
||||||
self.assertEqual(url_or_none('http://foo.de'), 'http://foo.de')
|
self.assertEqual(url_or_none('http://foo.de'), 'http://foo.de')
|
||||||
self.assertEqual(url_or_none('//foo.de'), '//foo.de')
|
self.assertEqual(url_or_none('//foo.de'), '//foo.de')
|
||||||
|
self.assertEqual(url_or_none('s3://foo.de'), None)
|
||||||
|
self.assertEqual(url_or_none('rtmpte://foo.de'), 'rtmpte://foo.de')
|
||||||
|
self.assertEqual(url_or_none('mms://foo.de'), 'mms://foo.de')
|
||||||
|
self.assertEqual(url_or_none('rtspu://foo.de'), 'rtspu://foo.de')
|
||||||
|
self.assertEqual(url_or_none('ftps://foo.de'), 'ftps://foo.de')
|
||||||
|
|
||||||
def test_parse_age_limit(self):
|
def test_parse_age_limit(self):
|
||||||
self.assertEqual(parse_age_limit(None), None)
|
self.assertEqual(parse_age_limit(None), None)
|
||||||
|
@ -656,38 +735,36 @@ class TestUtil(unittest.TestCase):
|
||||||
self.assertTrue(isinstance(data, bytes))
|
self.assertTrue(isinstance(data, bytes))
|
||||||
|
|
||||||
def test_update_url_query(self):
|
def test_update_url_query(self):
|
||||||
def query_dict(url):
|
self.assertEqual(parse_qs(update_url_query(
|
||||||
return compat_parse_qs(compat_urlparse.urlparse(url).query)
|
|
||||||
self.assertEqual(query_dict(update_url_query(
|
|
||||||
'http://example.com/path', {'quality': ['HD'], 'format': ['mp4']})),
|
'http://example.com/path', {'quality': ['HD'], 'format': ['mp4']})),
|
||||||
query_dict('http://example.com/path?quality=HD&format=mp4'))
|
parse_qs('http://example.com/path?quality=HD&format=mp4'))
|
||||||
self.assertEqual(query_dict(update_url_query(
|
self.assertEqual(parse_qs(update_url_query(
|
||||||
'http://example.com/path', {'system': ['LINUX', 'WINDOWS']})),
|
'http://example.com/path', {'system': ['LINUX', 'WINDOWS']})),
|
||||||
query_dict('http://example.com/path?system=LINUX&system=WINDOWS'))
|
parse_qs('http://example.com/path?system=LINUX&system=WINDOWS'))
|
||||||
self.assertEqual(query_dict(update_url_query(
|
self.assertEqual(parse_qs(update_url_query(
|
||||||
'http://example.com/path', {'fields': 'id,formats,subtitles'})),
|
'http://example.com/path', {'fields': 'id,formats,subtitles'})),
|
||||||
query_dict('http://example.com/path?fields=id,formats,subtitles'))
|
parse_qs('http://example.com/path?fields=id,formats,subtitles'))
|
||||||
self.assertEqual(query_dict(update_url_query(
|
self.assertEqual(parse_qs(update_url_query(
|
||||||
'http://example.com/path', {'fields': ('id,formats,subtitles', 'thumbnails')})),
|
'http://example.com/path', {'fields': ('id,formats,subtitles', 'thumbnails')})),
|
||||||
query_dict('http://example.com/path?fields=id,formats,subtitles&fields=thumbnails'))
|
parse_qs('http://example.com/path?fields=id,formats,subtitles&fields=thumbnails'))
|
||||||
self.assertEqual(query_dict(update_url_query(
|
self.assertEqual(parse_qs(update_url_query(
|
||||||
'http://example.com/path?manifest=f4m', {'manifest': []})),
|
'http://example.com/path?manifest=f4m', {'manifest': []})),
|
||||||
query_dict('http://example.com/path'))
|
parse_qs('http://example.com/path'))
|
||||||
self.assertEqual(query_dict(update_url_query(
|
self.assertEqual(parse_qs(update_url_query(
|
||||||
'http://example.com/path?system=LINUX&system=WINDOWS', {'system': 'LINUX'})),
|
'http://example.com/path?system=LINUX&system=WINDOWS', {'system': 'LINUX'})),
|
||||||
query_dict('http://example.com/path?system=LINUX'))
|
parse_qs('http://example.com/path?system=LINUX'))
|
||||||
self.assertEqual(query_dict(update_url_query(
|
self.assertEqual(parse_qs(update_url_query(
|
||||||
'http://example.com/path', {'fields': b'id,formats,subtitles'})),
|
'http://example.com/path', {'fields': b'id,formats,subtitles'})),
|
||||||
query_dict('http://example.com/path?fields=id,formats,subtitles'))
|
parse_qs('http://example.com/path?fields=id,formats,subtitles'))
|
||||||
self.assertEqual(query_dict(update_url_query(
|
self.assertEqual(parse_qs(update_url_query(
|
||||||
'http://example.com/path', {'width': 1080, 'height': 720})),
|
'http://example.com/path', {'width': 1080, 'height': 720})),
|
||||||
query_dict('http://example.com/path?width=1080&height=720'))
|
parse_qs('http://example.com/path?width=1080&height=720'))
|
||||||
self.assertEqual(query_dict(update_url_query(
|
self.assertEqual(parse_qs(update_url_query(
|
||||||
'http://example.com/path', {'bitrate': 5020.43})),
|
'http://example.com/path', {'bitrate': 5020.43})),
|
||||||
query_dict('http://example.com/path?bitrate=5020.43'))
|
parse_qs('http://example.com/path?bitrate=5020.43'))
|
||||||
self.assertEqual(query_dict(update_url_query(
|
self.assertEqual(parse_qs(update_url_query(
|
||||||
'http://example.com/path', {'test': '第二行тест'})),
|
'http://example.com/path', {'test': '第二行тест'})),
|
||||||
query_dict('http://example.com/path?test=%E7%AC%AC%E4%BA%8C%E8%A1%8C%D1%82%D0%B5%D1%81%D1%82'))
|
parse_qs('http://example.com/path?test=%E7%AC%AC%E4%BA%8C%E8%A1%8C%D1%82%D0%B5%D1%81%D1%82'))
|
||||||
|
|
||||||
def test_multipart_encode(self):
|
def test_multipart_encode(self):
|
||||||
self.assertEqual(
|
self.assertEqual(
|
||||||
|
@ -699,28 +776,6 @@ class TestUtil(unittest.TestCase):
|
||||||
self.assertRaises(
|
self.assertRaises(
|
||||||
ValueError, multipart_encode, {b'field': b'value'}, boundary='value')
|
ValueError, multipart_encode, {b'field': b'value'}, boundary='value')
|
||||||
|
|
||||||
def test_dict_get(self):
|
|
||||||
FALSE_VALUES = {
|
|
||||||
'none': None,
|
|
||||||
'false': False,
|
|
||||||
'zero': 0,
|
|
||||||
'empty_string': '',
|
|
||||||
'empty_list': [],
|
|
||||||
}
|
|
||||||
d = FALSE_VALUES.copy()
|
|
||||||
d['a'] = 42
|
|
||||||
self.assertEqual(dict_get(d, 'a'), 42)
|
|
||||||
self.assertEqual(dict_get(d, 'b'), None)
|
|
||||||
self.assertEqual(dict_get(d, 'b', 42), 42)
|
|
||||||
self.assertEqual(dict_get(d, ('a', )), 42)
|
|
||||||
self.assertEqual(dict_get(d, ('b', 'a', )), 42)
|
|
||||||
self.assertEqual(dict_get(d, ('b', 'c', 'a', 'd', )), 42)
|
|
||||||
self.assertEqual(dict_get(d, ('b', 'c', )), None)
|
|
||||||
self.assertEqual(dict_get(d, ('b', 'c', ), 42), 42)
|
|
||||||
for key, false_value in FALSE_VALUES.items():
|
|
||||||
self.assertEqual(dict_get(d, ('b', 'c', key, )), None)
|
|
||||||
self.assertEqual(dict_get(d, ('b', 'c', key, ), skip_false_values=False), false_value)
|
|
||||||
|
|
||||||
def test_merge_dicts(self):
|
def test_merge_dicts(self):
|
||||||
self.assertEqual(merge_dicts({'a': 1}, {'b': 2}), {'a': 1, 'b': 2})
|
self.assertEqual(merge_dicts({'a': 1}, {'b': 2}), {'a': 1, 'b': 2})
|
||||||
self.assertEqual(merge_dicts({'a': 1}, {'a': 2}), {'a': 1})
|
self.assertEqual(merge_dicts({'a': 1}, {'a': 2}), {'a': 1})
|
||||||
|
@ -803,6 +858,8 @@ class TestUtil(unittest.TestCase):
|
||||||
self.assertEqual(mimetype2ext('text/vtt'), 'vtt')
|
self.assertEqual(mimetype2ext('text/vtt'), 'vtt')
|
||||||
self.assertEqual(mimetype2ext('text/vtt;charset=utf-8'), 'vtt')
|
self.assertEqual(mimetype2ext('text/vtt;charset=utf-8'), 'vtt')
|
||||||
self.assertEqual(mimetype2ext('text/html; charset=utf-8'), 'html')
|
self.assertEqual(mimetype2ext('text/html; charset=utf-8'), 'html')
|
||||||
|
self.assertEqual(mimetype2ext('audio/x-wav'), 'wav')
|
||||||
|
self.assertEqual(mimetype2ext('audio/x-wav;codec=pcm'), 'wav')
|
||||||
|
|
||||||
def test_month_by_name(self):
|
def test_month_by_name(self):
|
||||||
self.assertEqual(month_by_name(None), None)
|
self.assertEqual(month_by_name(None), None)
|
||||||
|
@ -877,6 +934,111 @@ class TestUtil(unittest.TestCase):
|
||||||
)
|
)
|
||||||
self.assertEqual(escape_url('http://vimeo.com/56015672#at=0'), 'http://vimeo.com/56015672#at=0')
|
self.assertEqual(escape_url('http://vimeo.com/56015672#at=0'), 'http://vimeo.com/56015672#at=0')
|
||||||
|
|
||||||
|
def test_remove_dot_segments(self):
|
||||||
|
|
||||||
|
def remove_dot_segments(p):
|
||||||
|
q = '' if p.startswith('/') else '/'
|
||||||
|
p = 'http://example.com' + q + p
|
||||||
|
p = compat_urlparse.urlsplit(YoutubeDLHandler._fix_path(p)).path
|
||||||
|
return p[1:] if q else p
|
||||||
|
|
||||||
|
self.assertEqual(remove_dot_segments('/a/b/c/./../../g'), '/a/g')
|
||||||
|
self.assertEqual(remove_dot_segments('mid/content=5/../6'), 'mid/6')
|
||||||
|
self.assertEqual(remove_dot_segments('/ad/../cd'), '/cd')
|
||||||
|
self.assertEqual(remove_dot_segments('/ad/../cd/'), '/cd/')
|
||||||
|
self.assertEqual(remove_dot_segments('/..'), '/')
|
||||||
|
self.assertEqual(remove_dot_segments('/./'), '/')
|
||||||
|
self.assertEqual(remove_dot_segments('/./a'), '/a')
|
||||||
|
self.assertEqual(remove_dot_segments('/abc/./.././d/././e/.././f/./../../ghi'), '/ghi')
|
||||||
|
self.assertEqual(remove_dot_segments('/'), '/')
|
||||||
|
self.assertEqual(remove_dot_segments('/t'), '/t')
|
||||||
|
self.assertEqual(remove_dot_segments('t'), 't')
|
||||||
|
self.assertEqual(remove_dot_segments(''), '')
|
||||||
|
self.assertEqual(remove_dot_segments('/../a/b/c'), '/a/b/c')
|
||||||
|
self.assertEqual(remove_dot_segments('../a'), 'a')
|
||||||
|
self.assertEqual(remove_dot_segments('./a'), 'a')
|
||||||
|
self.assertEqual(remove_dot_segments('.'), '')
|
||||||
|
self.assertEqual(remove_dot_segments('////'), '////')
|
||||||
|
|
||||||
|
def test_js_to_json_vars_strings(self):
|
||||||
|
self.assertDictEqual(
|
||||||
|
json.loads(js_to_json(
|
||||||
|
'''{
|
||||||
|
'null': a,
|
||||||
|
'nullStr': b,
|
||||||
|
'true': c,
|
||||||
|
'trueStr': d,
|
||||||
|
'false': e,
|
||||||
|
'falseStr': f,
|
||||||
|
'unresolvedVar': g,
|
||||||
|
}''',
|
||||||
|
{
|
||||||
|
'a': 'null',
|
||||||
|
'b': '"null"',
|
||||||
|
'c': 'true',
|
||||||
|
'd': '"true"',
|
||||||
|
'e': 'false',
|
||||||
|
'f': '"false"',
|
||||||
|
'g': 'var',
|
||||||
|
}
|
||||||
|
)),
|
||||||
|
{
|
||||||
|
'null': None,
|
||||||
|
'nullStr': 'null',
|
||||||
|
'true': True,
|
||||||
|
'trueStr': 'true',
|
||||||
|
'false': False,
|
||||||
|
'falseStr': 'false',
|
||||||
|
'unresolvedVar': 'var'
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
self.assertDictEqual(
|
||||||
|
json.loads(js_to_json(
|
||||||
|
'''{
|
||||||
|
'int': a,
|
||||||
|
'intStr': b,
|
||||||
|
'float': c,
|
||||||
|
'floatStr': d,
|
||||||
|
}''',
|
||||||
|
{
|
||||||
|
'a': '123',
|
||||||
|
'b': '"123"',
|
||||||
|
'c': '1.23',
|
||||||
|
'd': '"1.23"',
|
||||||
|
}
|
||||||
|
)),
|
||||||
|
{
|
||||||
|
'int': 123,
|
||||||
|
'intStr': '123',
|
||||||
|
'float': 1.23,
|
||||||
|
'floatStr': '1.23',
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
self.assertDictEqual(
|
||||||
|
json.loads(js_to_json(
|
||||||
|
'''{
|
||||||
|
'object': a,
|
||||||
|
'objectStr': b,
|
||||||
|
'array': c,
|
||||||
|
'arrayStr': d,
|
||||||
|
}''',
|
||||||
|
{
|
||||||
|
'a': '{}',
|
||||||
|
'b': '"{}"',
|
||||||
|
'c': '[]',
|
||||||
|
'd': '"[]"',
|
||||||
|
}
|
||||||
|
)),
|
||||||
|
{
|
||||||
|
'object': {},
|
||||||
|
'objectStr': '{}',
|
||||||
|
'array': [],
|
||||||
|
'arrayStr': '[]',
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
def test_js_to_json_realworld(self):
|
def test_js_to_json_realworld(self):
|
||||||
inp = '''{
|
inp = '''{
|
||||||
'clip':{'provider':'pseudo'}
|
'clip':{'provider':'pseudo'}
|
||||||
|
@ -935,6 +1097,28 @@ class TestUtil(unittest.TestCase):
|
||||||
self.assertEqual(d['x'], 1)
|
self.assertEqual(d['x'], 1)
|
||||||
self.assertEqual(d['y'], 'a')
|
self.assertEqual(d['y'], 'a')
|
||||||
|
|
||||||
|
# Just drop ! prefix for now though this results in a wrong value
|
||||||
|
on = js_to_json('''{
|
||||||
|
a: !0,
|
||||||
|
b: !1,
|
||||||
|
c: !!0,
|
||||||
|
d: !!42.42,
|
||||||
|
e: !!![],
|
||||||
|
f: !"abc",
|
||||||
|
g: !"",
|
||||||
|
!42: 42
|
||||||
|
}''')
|
||||||
|
self.assertEqual(json.loads(on), {
|
||||||
|
'a': True,
|
||||||
|
'b': False,
|
||||||
|
'c': False,
|
||||||
|
'd': True,
|
||||||
|
'e': [],
|
||||||
|
'f': "abc",
|
||||||
|
'g': "",
|
||||||
|
'42': 42
|
||||||
|
})
|
||||||
|
|
||||||
on = js_to_json('["abc", "def",]')
|
on = js_to_json('["abc", "def",]')
|
||||||
self.assertEqual(json.loads(on), ['abc', 'def'])
|
self.assertEqual(json.loads(on), ['abc', 'def'])
|
||||||
|
|
||||||
|
@ -992,10 +1176,32 @@ class TestUtil(unittest.TestCase):
|
||||||
on = js_to_json('{42:4.2e1}')
|
on = js_to_json('{42:4.2e1}')
|
||||||
self.assertEqual(json.loads(on), {'42': 42.0})
|
self.assertEqual(json.loads(on), {'42': 42.0})
|
||||||
|
|
||||||
|
on = js_to_json('{ "0x40": "0x40" }')
|
||||||
|
self.assertEqual(json.loads(on), {'0x40': '0x40'})
|
||||||
|
|
||||||
|
on = js_to_json('{ "040": "040" }')
|
||||||
|
self.assertEqual(json.loads(on), {'040': '040'})
|
||||||
|
|
||||||
|
on = js_to_json('[1,//{},\n2]')
|
||||||
|
self.assertEqual(json.loads(on), [1, 2])
|
||||||
|
|
||||||
|
on = js_to_json(r'"\^\$\#"')
|
||||||
|
self.assertEqual(json.loads(on), R'^$#', msg='Unnecessary escapes should be stripped')
|
||||||
|
|
||||||
|
on = js_to_json('\'"\\""\'')
|
||||||
|
self.assertEqual(json.loads(on), '"""', msg='Unnecessary quote escape should be escaped')
|
||||||
|
|
||||||
def test_js_to_json_malformed(self):
|
def test_js_to_json_malformed(self):
|
||||||
self.assertEqual(js_to_json('42a1'), '42"a1"')
|
self.assertEqual(js_to_json('42a1'), '42"a1"')
|
||||||
self.assertEqual(js_to_json('42a-1'), '42"a"-1')
|
self.assertEqual(js_to_json('42a-1'), '42"a"-1')
|
||||||
|
|
||||||
|
def test_js_to_json_template_literal(self):
|
||||||
|
self.assertEqual(js_to_json('`Hello ${name}`', {'name': '"world"'}), '"Hello world"')
|
||||||
|
self.assertEqual(js_to_json('`${name}${name}`', {'name': '"X"'}), '"XX"')
|
||||||
|
self.assertEqual(js_to_json('`${name}${name}`', {'name': '5'}), '"55"')
|
||||||
|
self.assertEqual(js_to_json('`${name}"${name}"`', {'name': '5'}), '"5\\"5\\""')
|
||||||
|
self.assertEqual(js_to_json('`${name}`', {}), '"name"')
|
||||||
|
|
||||||
def test_extract_attributes(self):
|
def test_extract_attributes(self):
|
||||||
self.assertEqual(extract_attributes('<e x="y">'), {'x': 'y'})
|
self.assertEqual(extract_attributes('<e x="y">'), {'x': 'y'})
|
||||||
self.assertEqual(extract_attributes("<e x='y'>"), {'x': 'y'})
|
self.assertEqual(extract_attributes("<e x='y'>"), {'x': 'y'})
|
||||||
|
@ -1435,6 +1641,88 @@ Line 1
|
||||||
self.assertEqual(get_elements_by_attribute('class', 'foo', html), [])
|
self.assertEqual(get_elements_by_attribute('class', 'foo', html), [])
|
||||||
self.assertEqual(get_elements_by_attribute('class', 'no-such-foo', html), [])
|
self.assertEqual(get_elements_by_attribute('class', 'no-such-foo', html), [])
|
||||||
|
|
||||||
|
def test_clean_podcast_url(self):
|
||||||
|
self.assertEqual(clean_podcast_url('https://www.podtrac.com/pts/redirect.mp3/chtbl.com/track/5899E/traffic.megaphone.fm/HSW7835899191.mp3'), 'https://traffic.megaphone.fm/HSW7835899191.mp3')
|
||||||
|
self.assertEqual(clean_podcast_url('https://play.podtrac.com/npr-344098539/edge1.pod.npr.org/anon.npr-podcasts/podcast/npr/waitwait/2020/10/20201003_waitwait_wwdtmpodcast201003-015621a5-f035-4eca-a9a1-7c118d90bc3c.mp3'), 'https://edge1.pod.npr.org/anon.npr-podcasts/podcast/npr/waitwait/2020/10/20201003_waitwait_wwdtmpodcast201003-015621a5-f035-4eca-a9a1-7c118d90bc3c.mp3')
|
||||||
|
|
||||||
|
def test_LazyList(self):
|
||||||
|
it = list(range(10))
|
||||||
|
|
||||||
|
self.assertEqual(list(LazyList(it)), it)
|
||||||
|
self.assertEqual(LazyList(it).exhaust(), it)
|
||||||
|
self.assertEqual(LazyList(it)[5], it[5])
|
||||||
|
|
||||||
|
self.assertEqual(LazyList(it)[5:], it[5:])
|
||||||
|
self.assertEqual(LazyList(it)[:5], it[:5])
|
||||||
|
self.assertEqual(LazyList(it)[::2], it[::2])
|
||||||
|
self.assertEqual(LazyList(it)[1::2], it[1::2])
|
||||||
|
self.assertEqual(LazyList(it)[5::-1], it[5::-1])
|
||||||
|
self.assertEqual(LazyList(it)[6:2:-2], it[6:2:-2])
|
||||||
|
self.assertEqual(LazyList(it)[::-1], it[::-1])
|
||||||
|
|
||||||
|
self.assertTrue(LazyList(it))
|
||||||
|
self.assertFalse(LazyList(range(0)))
|
||||||
|
self.assertEqual(len(LazyList(it)), len(it))
|
||||||
|
self.assertEqual(repr(LazyList(it)), repr(it))
|
||||||
|
self.assertEqual(compat_str(LazyList(it)), compat_str(it))
|
||||||
|
|
||||||
|
self.assertEqual(list(LazyList(it, reverse=True)), it[::-1])
|
||||||
|
self.assertEqual(list(reversed(LazyList(it))[::-1]), it)
|
||||||
|
self.assertEqual(list(reversed(LazyList(it))[1:3:7]), it[::-1][1:3:7])
|
||||||
|
|
||||||
|
def test_LazyList_laziness(self):
|
||||||
|
|
||||||
|
def test(ll, idx, val, cache):
|
||||||
|
self.assertEqual(ll[idx], val)
|
||||||
|
self.assertEqual(ll._cache, list(cache))
|
||||||
|
|
||||||
|
ll = LazyList(range(10))
|
||||||
|
test(ll, 0, 0, range(1))
|
||||||
|
test(ll, 5, 5, range(6))
|
||||||
|
test(ll, -3, 7, range(10))
|
||||||
|
|
||||||
|
ll = LazyList(range(10), reverse=True)
|
||||||
|
test(ll, -1, 0, range(1))
|
||||||
|
test(ll, 3, 6, range(10))
|
||||||
|
|
||||||
|
ll = LazyList(itertools.count())
|
||||||
|
test(ll, 10, 10, range(11))
|
||||||
|
ll = reversed(ll)
|
||||||
|
test(ll, -15, 14, range(15))
|
||||||
|
|
||||||
|
def test_try_call(self):
|
||||||
|
def total(*x, **kwargs):
|
||||||
|
return sum(x) + sum(kwargs.values())
|
||||||
|
|
||||||
|
self.assertEqual(try_call(None), None,
|
||||||
|
msg='not a fn should give None')
|
||||||
|
self.assertEqual(try_call(lambda: 1), 1,
|
||||||
|
msg='int fn with no expected_type should give int')
|
||||||
|
self.assertEqual(try_call(lambda: 1, expected_type=int), 1,
|
||||||
|
msg='int fn with expected_type int should give int')
|
||||||
|
self.assertEqual(try_call(lambda: 1, expected_type=dict), None,
|
||||||
|
msg='int fn with wrong expected_type should give None')
|
||||||
|
self.assertEqual(try_call(total, args=(0, 1, 0, ), expected_type=int), 1,
|
||||||
|
msg='fn should accept arglist')
|
||||||
|
self.assertEqual(try_call(total, kwargs={'a': 0, 'b': 1, 'c': 0}, expected_type=int), 1,
|
||||||
|
msg='fn should accept kwargs')
|
||||||
|
self.assertEqual(try_call(lambda: 1, expected_type=dict), None,
|
||||||
|
msg='int fn with no expected_type should give None')
|
||||||
|
self.assertEqual(try_call(lambda x: {}, total, args=(42, ), expected_type=int), 42,
|
||||||
|
msg='expect first int result with expected_type int')
|
||||||
|
|
||||||
|
def test_variadic(self):
|
||||||
|
self.assertEqual(variadic(None), (None, ))
|
||||||
|
self.assertEqual(variadic('spam'), ('spam', ))
|
||||||
|
self.assertEqual(variadic('spam', allowed_types=dict), 'spam')
|
||||||
|
self.assertEqual(variadic('spam', allowed_types=[dict]), 'spam')
|
||||||
|
|
||||||
|
def test_join_nonempty(self):
|
||||||
|
self.assertEqual(join_nonempty('a', 'b'), 'a-b')
|
||||||
|
self.assertEqual(join_nonempty(
|
||||||
|
'a', 'b', 'c', 'd',
|
||||||
|
from_dict={'a': 'c', 'c': [], 'b': 'd', 'd': None}), 'c-d')
|
||||||
|
|
||||||
|
|
||||||
if __name__ == '__main__':
|
if __name__ == '__main__':
|
||||||
unittest.main()
|
unittest.main()
|
||||||
|
|
|
@ -11,12 +11,11 @@ sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||||
from test.helper import get_params, try_rm
|
from test.helper import get_params, try_rm
|
||||||
|
|
||||||
|
|
||||||
import io
|
|
||||||
|
|
||||||
import xml.etree.ElementTree
|
import xml.etree.ElementTree
|
||||||
|
|
||||||
import youtube_dl.YoutubeDL
|
import youtube_dl.YoutubeDL
|
||||||
import youtube_dl.extractor
|
import youtube_dl.extractor
|
||||||
|
from youtube_dl.compat import compat_open as open
|
||||||
|
|
||||||
|
|
||||||
class YoutubeDL(youtube_dl.YoutubeDL):
|
class YoutubeDL(youtube_dl.YoutubeDL):
|
||||||
|
@ -51,7 +50,7 @@ class TestAnnotations(unittest.TestCase):
|
||||||
ydl.download([TEST_ID])
|
ydl.download([TEST_ID])
|
||||||
self.assertTrue(os.path.exists(ANNOTATIONS_FILE))
|
self.assertTrue(os.path.exists(ANNOTATIONS_FILE))
|
||||||
annoxml = None
|
annoxml = None
|
||||||
with io.open(ANNOTATIONS_FILE, 'r', encoding='utf-8') as annof:
|
with open(ANNOTATIONS_FILE, 'r', encoding='utf-8') as annof:
|
||||||
annoxml = xml.etree.ElementTree.parse(annof)
|
annoxml = xml.etree.ElementTree.parse(annof)
|
||||||
self.assertTrue(annoxml is not None, 'Failed to parse annotations XML')
|
self.assertTrue(annoxml is not None, 'Failed to parse annotations XML')
|
||||||
root = annoxml.getroot()
|
root = annoxml.getroot()
|
||||||
|
|
|
@ -1,275 +0,0 @@
|
||||||
#!/usr/bin/env python
|
|
||||||
# coding: utf-8
|
|
||||||
from __future__ import unicode_literals
|
|
||||||
|
|
||||||
# Allow direct execution
|
|
||||||
import os
|
|
||||||
import sys
|
|
||||||
import unittest
|
|
||||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
|
||||||
|
|
||||||
from test.helper import expect_value
|
|
||||||
from youtube_dl.extractor import YoutubeIE
|
|
||||||
|
|
||||||
|
|
||||||
class TestYoutubeChapters(unittest.TestCase):
|
|
||||||
|
|
||||||
_TEST_CASES = [
|
|
||||||
(
|
|
||||||
# https://www.youtube.com/watch?v=A22oy8dFjqc
|
|
||||||
# pattern: 00:00 - <title>
|
|
||||||
'''This is the absolute ULTIMATE experience of Queen's set at LIVE AID, this is the best video mixed to the absolutely superior stereo radio broadcast. This vastly superior audio mix takes a huge dump on all of the official mixes. Best viewed in 1080p. ENJOY! ***MAKE SURE TO READ THE DESCRIPTION***<br /><a href="#" onclick="yt.www.watch.player.seekTo(00*60+36);return false;">00:36</a> - Bohemian Rhapsody<br /><a href="#" onclick="yt.www.watch.player.seekTo(02*60+42);return false;">02:42</a> - Radio Ga Ga<br /><a href="#" onclick="yt.www.watch.player.seekTo(06*60+53);return false;">06:53</a> - Ay Oh!<br /><a href="#" onclick="yt.www.watch.player.seekTo(07*60+34);return false;">07:34</a> - Hammer To Fall<br /><a href="#" onclick="yt.www.watch.player.seekTo(12*60+08);return false;">12:08</a> - Crazy Little Thing Called Love<br /><a href="#" onclick="yt.www.watch.player.seekTo(16*60+03);return false;">16:03</a> - We Will Rock You<br /><a href="#" onclick="yt.www.watch.player.seekTo(17*60+18);return false;">17:18</a> - We Are The Champions<br /><a href="#" onclick="yt.www.watch.player.seekTo(21*60+12);return false;">21:12</a> - Is This The World We Created...?<br /><br />Short song analysis:<br /><br />- "Bohemian Rhapsody": Although it's a short medley version, it's one of the best performances of the ballad section, with Freddie nailing the Bb4s with the correct studio phrasing (for the first time ever!).<br /><br />- "Radio Ga Ga": Although it's missing one chorus, this is one of - if not the best - the best versions ever, Freddie nails all the Bb4s and sounds very clean! Spike Edney's Roland Jupiter 8 also really shines through on this mix, compared to the DVD releases!<br /><br />- "Audience Improv": A great improv, Freddie sounds strong and confident. You gotta love when he sustains that A4 for 4 seconds!<br /><br />- "Hammer To Fall": Despite missing a verse and a chorus, it's a strong version (possibly the best ever). Freddie sings the song amazingly, and even ad-libs a C#5 and a C5! Also notice how heavy Brian's guitar sounds compared to the thin DVD mixes - it roars!<br /><br />- "Crazy Little Thing Called Love": A great version, the crowd loves the song, the jam is great as well! Only downside to this is the slight feedback issues.<br /><br />- "We Will Rock You": Although cut down to the 1st verse and chorus, Freddie sounds strong. He nails the A4, and the solo from Dr. May is brilliant!<br /><br />- "We Are the Champions": Perhaps the high-light of the performance - Freddie is very daring on this version, he sustains the pre-chorus Bb4s, nails the 1st C5, belts great A4s, but most importantly: He nails the chorus Bb4s, in all 3 choruses! This is the only time he has ever done so! It has to be said though, the last one sounds a bit rough, but that's a side effect of belting high notes for the past 18 minutes, with nodules AND laryngitis!<br /><br />- "Is This The World We Created... ?": Freddie and Brian perform a beautiful version of this, and it is one of the best versions ever. It's both sad and hilarious that a couple of BBC engineers are talking over the song, one of them being completely oblivious of the fact that he is interrupting the performance, on live television... Which was being televised to almost 2 billion homes.<br /><br /><br />All rights go to their respective owners!<br />-----Copyright Disclaimer Under Section 107 of the Copyright Act 1976, allowance is made for fair use for purposes such as criticism, comment, news reporting, teaching, scholarship, and research. Fair use is a use permitted by copyright statute that might otherwise be infringing. Non-profit, educational or personal use tips the balance in favor of fair use''',
|
|
||||||
1477,
|
|
||||||
[{
|
|
||||||
'start_time': 36,
|
|
||||||
'end_time': 162,
|
|
||||||
'title': 'Bohemian Rhapsody',
|
|
||||||
}, {
|
|
||||||
'start_time': 162,
|
|
||||||
'end_time': 413,
|
|
||||||
'title': 'Radio Ga Ga',
|
|
||||||
}, {
|
|
||||||
'start_time': 413,
|
|
||||||
'end_time': 454,
|
|
||||||
'title': 'Ay Oh!',
|
|
||||||
}, {
|
|
||||||
'start_time': 454,
|
|
||||||
'end_time': 728,
|
|
||||||
'title': 'Hammer To Fall',
|
|
||||||
}, {
|
|
||||||
'start_time': 728,
|
|
||||||
'end_time': 963,
|
|
||||||
'title': 'Crazy Little Thing Called Love',
|
|
||||||
}, {
|
|
||||||
'start_time': 963,
|
|
||||||
'end_time': 1038,
|
|
||||||
'title': 'We Will Rock You',
|
|
||||||
}, {
|
|
||||||
'start_time': 1038,
|
|
||||||
'end_time': 1272,
|
|
||||||
'title': 'We Are The Champions',
|
|
||||||
}, {
|
|
||||||
'start_time': 1272,
|
|
||||||
'end_time': 1477,
|
|
||||||
'title': 'Is This The World We Created...?',
|
|
||||||
}]
|
|
||||||
),
|
|
||||||
(
|
|
||||||
# https://www.youtube.com/watch?v=ekYlRhALiRQ
|
|
||||||
# pattern: <num>. <title> 0:00
|
|
||||||
'1. Those Beaten Paths of Confusion <a href="#" onclick="yt.www.watch.player.seekTo(0*60+00);return false;">0:00</a><br />2. Beyond the Shadows of Emptiness & Nothingness <a href="#" onclick="yt.www.watch.player.seekTo(11*60+47);return false;">11:47</a><br />3. Poison Yourself...With Thought <a href="#" onclick="yt.www.watch.player.seekTo(26*60+30);return false;">26:30</a><br />4. The Agents of Transformation <a href="#" onclick="yt.www.watch.player.seekTo(35*60+57);return false;">35:57</a><br />5. Drowning in the Pain of Consciousness <a href="#" onclick="yt.www.watch.player.seekTo(44*60+32);return false;">44:32</a><br />6. Deny the Disease of Life <a href="#" onclick="yt.www.watch.player.seekTo(53*60+07);return false;">53:07</a><br /><br />More info/Buy: http://crepusculonegro.storenvy.com/products/257645-cn-03-arizmenda-within-the-vacuum-of-infinity<br /><br />No copyright is intended. The rights to this video are assumed by the owner and its affiliates.',
|
|
||||||
4009,
|
|
||||||
[{
|
|
||||||
'start_time': 0,
|
|
||||||
'end_time': 707,
|
|
||||||
'title': '1. Those Beaten Paths of Confusion',
|
|
||||||
}, {
|
|
||||||
'start_time': 707,
|
|
||||||
'end_time': 1590,
|
|
||||||
'title': '2. Beyond the Shadows of Emptiness & Nothingness',
|
|
||||||
}, {
|
|
||||||
'start_time': 1590,
|
|
||||||
'end_time': 2157,
|
|
||||||
'title': '3. Poison Yourself...With Thought',
|
|
||||||
}, {
|
|
||||||
'start_time': 2157,
|
|
||||||
'end_time': 2672,
|
|
||||||
'title': '4. The Agents of Transformation',
|
|
||||||
}, {
|
|
||||||
'start_time': 2672,
|
|
||||||
'end_time': 3187,
|
|
||||||
'title': '5. Drowning in the Pain of Consciousness',
|
|
||||||
}, {
|
|
||||||
'start_time': 3187,
|
|
||||||
'end_time': 4009,
|
|
||||||
'title': '6. Deny the Disease of Life',
|
|
||||||
}]
|
|
||||||
),
|
|
||||||
(
|
|
||||||
# https://www.youtube.com/watch?v=WjL4pSzog9w
|
|
||||||
# pattern: 00:00 <title>
|
|
||||||
'<a href="https://arizmenda.bandcamp.com/merch/despairs-depths-descended-cd" class="yt-uix-servicelink " data-target-new-window="True" data-servicelink="CDAQ6TgYACITCNf1raqT2dMCFdRjGAod_o0CBSj4HQ" data-url="https://arizmenda.bandcamp.com/merch/despairs-depths-descended-cd" rel="nofollow noopener" target="_blank">https://arizmenda.bandcamp.com/merch/...</a><br /><br /><a href="#" onclick="yt.www.watch.player.seekTo(00*60+00);return false;">00:00</a> Christening Unborn Deformities <br /><a href="#" onclick="yt.www.watch.player.seekTo(07*60+08);return false;">07:08</a> Taste of Purity<br /><a href="#" onclick="yt.www.watch.player.seekTo(16*60+16);return false;">16:16</a> Sculpting Sins of a Universal Tongue<br /><a href="#" onclick="yt.www.watch.player.seekTo(24*60+45);return false;">24:45</a> Birth<br /><a href="#" onclick="yt.www.watch.player.seekTo(31*60+24);return false;">31:24</a> Neves<br /><a href="#" onclick="yt.www.watch.player.seekTo(37*60+55);return false;">37:55</a> Libations in Limbo',
|
|
||||||
2705,
|
|
||||||
[{
|
|
||||||
'start_time': 0,
|
|
||||||
'end_time': 428,
|
|
||||||
'title': 'Christening Unborn Deformities',
|
|
||||||
}, {
|
|
||||||
'start_time': 428,
|
|
||||||
'end_time': 976,
|
|
||||||
'title': 'Taste of Purity',
|
|
||||||
}, {
|
|
||||||
'start_time': 976,
|
|
||||||
'end_time': 1485,
|
|
||||||
'title': 'Sculpting Sins of a Universal Tongue',
|
|
||||||
}, {
|
|
||||||
'start_time': 1485,
|
|
||||||
'end_time': 1884,
|
|
||||||
'title': 'Birth',
|
|
||||||
}, {
|
|
||||||
'start_time': 1884,
|
|
||||||
'end_time': 2275,
|
|
||||||
'title': 'Neves',
|
|
||||||
}, {
|
|
||||||
'start_time': 2275,
|
|
||||||
'end_time': 2705,
|
|
||||||
'title': 'Libations in Limbo',
|
|
||||||
}]
|
|
||||||
),
|
|
||||||
(
|
|
||||||
# https://www.youtube.com/watch?v=o3r1sn-t3is
|
|
||||||
# pattern: <title> 00:00 <note>
|
|
||||||
'Download this show in MP3: <a href="http://sh.st/njZKK" class="yt-uix-servicelink " data-url="http://sh.st/njZKK" data-target-new-window="True" data-servicelink="CDAQ6TgYACITCK3j8_6o2dMCFVDCGAoduVAKKij4HQ" rel="nofollow noopener" target="_blank">http://sh.st/njZKK</a><br /><br />Setlist:<br />I-E-A-I-A-I-O <a href="#" onclick="yt.www.watch.player.seekTo(00*60+45);return false;">00:45</a><br />Suite-Pee <a href="#" onclick="yt.www.watch.player.seekTo(4*60+26);return false;">4:26</a> (Incomplete)<br />Attack <a href="#" onclick="yt.www.watch.player.seekTo(5*60+31);return false;">5:31</a> (First live performance since 2011)<br />Prison Song <a href="#" onclick="yt.www.watch.player.seekTo(8*60+42);return false;">8:42</a><br />Know <a href="#" onclick="yt.www.watch.player.seekTo(12*60+32);return false;">12:32</a> (First live performance since 2011)<br />Aerials <a href="#" onclick="yt.www.watch.player.seekTo(15*60+32);return false;">15:32</a><br />Soldier Side - Intro <a href="#" onclick="yt.www.watch.player.seekTo(19*60+13);return false;">19:13</a><br />B.Y.O.B. <a href="#" onclick="yt.www.watch.player.seekTo(20*60+09);return false;">20:09</a><br />Soil <a href="#" onclick="yt.www.watch.player.seekTo(24*60+32);return false;">24:32</a><br />Darts <a href="#" onclick="yt.www.watch.player.seekTo(27*60+48);return false;">27:48</a><br />Radio/Video <a href="#" onclick="yt.www.watch.player.seekTo(30*60+38);return false;">30:38</a><br />Hypnotize <a href="#" onclick="yt.www.watch.player.seekTo(35*60+05);return false;">35:05</a><br />Temper <a href="#" onclick="yt.www.watch.player.seekTo(38*60+08);return false;">38:08</a> (First live performance since 1999)<br />CUBErt <a href="#" onclick="yt.www.watch.player.seekTo(41*60+00);return false;">41:00</a><br />Needles <a href="#" onclick="yt.www.watch.player.seekTo(42*60+57);return false;">42:57</a><br />Deer Dance <a href="#" onclick="yt.www.watch.player.seekTo(46*60+27);return false;">46:27</a><br />Bounce <a href="#" onclick="yt.www.watch.player.seekTo(49*60+38);return false;">49:38</a><br />Suggestions <a href="#" onclick="yt.www.watch.player.seekTo(51*60+25);return false;">51:25</a><br />Psycho <a href="#" onclick="yt.www.watch.player.seekTo(53*60+52);return false;">53:52</a><br />Chop Suey! <a href="#" onclick="yt.www.watch.player.seekTo(58*60+13);return false;">58:13</a><br />Lonely Day <a href="#" onclick="yt.www.watch.player.seekTo(1*3600+01*60+15);return false;">1:01:15</a><br />Question! <a href="#" onclick="yt.www.watch.player.seekTo(1*3600+04*60+14);return false;">1:04:14</a><br />Lost in Hollywood <a href="#" onclick="yt.www.watch.player.seekTo(1*3600+08*60+10);return false;">1:08:10</a><br />Vicinity of Obscenity <a href="#" onclick="yt.www.watch.player.seekTo(1*3600+13*60+40);return false;">1:13:40</a>(First live performance since 2012)<br />Forest <a href="#" onclick="yt.www.watch.player.seekTo(1*3600+16*60+17);return false;">1:16:17</a><br />Cigaro <a href="#" onclick="yt.www.watch.player.seekTo(1*3600+20*60+02);return false;">1:20:02</a><br />Toxicity <a href="#" onclick="yt.www.watch.player.seekTo(1*3600+23*60+57);return false;">1:23:57</a>(with Chino Moreno)<br />Sugar <a href="#" onclick="yt.www.watch.player.seekTo(1*3600+27*60+53);return false;">1:27:53</a>',
|
|
||||||
5640,
|
|
||||||
[{
|
|
||||||
'start_time': 45,
|
|
||||||
'end_time': 266,
|
|
||||||
'title': 'I-E-A-I-A-I-O',
|
|
||||||
}, {
|
|
||||||
'start_time': 266,
|
|
||||||
'end_time': 331,
|
|
||||||
'title': 'Suite-Pee (Incomplete)',
|
|
||||||
}, {
|
|
||||||
'start_time': 331,
|
|
||||||
'end_time': 522,
|
|
||||||
'title': 'Attack (First live performance since 2011)',
|
|
||||||
}, {
|
|
||||||
'start_time': 522,
|
|
||||||
'end_time': 752,
|
|
||||||
'title': 'Prison Song',
|
|
||||||
}, {
|
|
||||||
'start_time': 752,
|
|
||||||
'end_time': 932,
|
|
||||||
'title': 'Know (First live performance since 2011)',
|
|
||||||
}, {
|
|
||||||
'start_time': 932,
|
|
||||||
'end_time': 1153,
|
|
||||||
'title': 'Aerials',
|
|
||||||
}, {
|
|
||||||
'start_time': 1153,
|
|
||||||
'end_time': 1209,
|
|
||||||
'title': 'Soldier Side - Intro',
|
|
||||||
}, {
|
|
||||||
'start_time': 1209,
|
|
||||||
'end_time': 1472,
|
|
||||||
'title': 'B.Y.O.B.',
|
|
||||||
}, {
|
|
||||||
'start_time': 1472,
|
|
||||||
'end_time': 1668,
|
|
||||||
'title': 'Soil',
|
|
||||||
}, {
|
|
||||||
'start_time': 1668,
|
|
||||||
'end_time': 1838,
|
|
||||||
'title': 'Darts',
|
|
||||||
}, {
|
|
||||||
'start_time': 1838,
|
|
||||||
'end_time': 2105,
|
|
||||||
'title': 'Radio/Video',
|
|
||||||
}, {
|
|
||||||
'start_time': 2105,
|
|
||||||
'end_time': 2288,
|
|
||||||
'title': 'Hypnotize',
|
|
||||||
}, {
|
|
||||||
'start_time': 2288,
|
|
||||||
'end_time': 2460,
|
|
||||||
'title': 'Temper (First live performance since 1999)',
|
|
||||||
}, {
|
|
||||||
'start_time': 2460,
|
|
||||||
'end_time': 2577,
|
|
||||||
'title': 'CUBErt',
|
|
||||||
}, {
|
|
||||||
'start_time': 2577,
|
|
||||||
'end_time': 2787,
|
|
||||||
'title': 'Needles',
|
|
||||||
}, {
|
|
||||||
'start_time': 2787,
|
|
||||||
'end_time': 2978,
|
|
||||||
'title': 'Deer Dance',
|
|
||||||
}, {
|
|
||||||
'start_time': 2978,
|
|
||||||
'end_time': 3085,
|
|
||||||
'title': 'Bounce',
|
|
||||||
}, {
|
|
||||||
'start_time': 3085,
|
|
||||||
'end_time': 3232,
|
|
||||||
'title': 'Suggestions',
|
|
||||||
}, {
|
|
||||||
'start_time': 3232,
|
|
||||||
'end_time': 3493,
|
|
||||||
'title': 'Psycho',
|
|
||||||
}, {
|
|
||||||
'start_time': 3493,
|
|
||||||
'end_time': 3675,
|
|
||||||
'title': 'Chop Suey!',
|
|
||||||
}, {
|
|
||||||
'start_time': 3675,
|
|
||||||
'end_time': 3854,
|
|
||||||
'title': 'Lonely Day',
|
|
||||||
}, {
|
|
||||||
'start_time': 3854,
|
|
||||||
'end_time': 4090,
|
|
||||||
'title': 'Question!',
|
|
||||||
}, {
|
|
||||||
'start_time': 4090,
|
|
||||||
'end_time': 4420,
|
|
||||||
'title': 'Lost in Hollywood',
|
|
||||||
}, {
|
|
||||||
'start_time': 4420,
|
|
||||||
'end_time': 4577,
|
|
||||||
'title': 'Vicinity of Obscenity (First live performance since 2012)',
|
|
||||||
}, {
|
|
||||||
'start_time': 4577,
|
|
||||||
'end_time': 4802,
|
|
||||||
'title': 'Forest',
|
|
||||||
}, {
|
|
||||||
'start_time': 4802,
|
|
||||||
'end_time': 5037,
|
|
||||||
'title': 'Cigaro',
|
|
||||||
}, {
|
|
||||||
'start_time': 5037,
|
|
||||||
'end_time': 5273,
|
|
||||||
'title': 'Toxicity (with Chino Moreno)',
|
|
||||||
}, {
|
|
||||||
'start_time': 5273,
|
|
||||||
'end_time': 5640,
|
|
||||||
'title': 'Sugar',
|
|
||||||
}]
|
|
||||||
),
|
|
||||||
(
|
|
||||||
# https://www.youtube.com/watch?v=PkYLQbsqCE8
|
|
||||||
# pattern: <num> - <title> [<latinized title>] 0:00:00
|
|
||||||
'''Затемно (Zatemno) is an Obscure Black Metal Band from Russia.<br /><br />"Во прах (Vo prakh)'' Into The Ashes", Debut mini-album released may 6, 2016, by Death Knell Productions<br />Released on 6 panel digipak CD, limited to 100 copies only<br />And digital format on Bandcamp<br /><br />Tracklist<br /><br />1 - Во прах [Vo prakh] <a href="#" onclick="yt.www.watch.player.seekTo(0*3600+00*60+00);return false;">0:00:00</a><br />2 - Искупление [Iskupleniye] <a href="#" onclick="yt.www.watch.player.seekTo(0*3600+08*60+10);return false;">0:08:10</a><br />3 - Из серпов луны...[Iz serpov luny] <a href="#" onclick="yt.www.watch.player.seekTo(0*3600+14*60+30);return false;">0:14:30</a><br /><br />Links:<br /><a href="https://deathknellprod.bandcamp.com/album/--2" class="yt-uix-servicelink " data-target-new-window="True" data-url="https://deathknellprod.bandcamp.com/album/--2" data-servicelink="CC8Q6TgYACITCNP234Kr2dMCFcNxGAodQqsIwSj4HQ" target="_blank" rel="nofollow noopener">https://deathknellprod.bandcamp.com/a...</a><br /><a href="https://www.facebook.com/DeathKnellProd/" class="yt-uix-servicelink " data-target-new-window="True" data-url="https://www.facebook.com/DeathKnellProd/" data-servicelink="CC8Q6TgYACITCNP234Kr2dMCFcNxGAodQqsIwSj4HQ" target="_blank" rel="nofollow noopener">https://www.facebook.com/DeathKnellProd/</a><br /><br /><br />I don't have any right about this artifact, my only intention is to spread the music of the band, all rights are reserved to the Затемно (Zatemno) and his producers, Death Knell Productions.<br /><br />------------------------------------------------------------------<br /><br />Subscribe for more videos like this.<br />My link: <a href="https://web.facebook.com/AttackOfTheDragons" class="yt-uix-servicelink " data-target-new-window="True" data-url="https://web.facebook.com/AttackOfTheDragons" data-servicelink="CC8Q6TgYACITCNP234Kr2dMCFcNxGAodQqsIwSj4HQ" target="_blank" rel="nofollow noopener">https://web.facebook.com/AttackOfTheD...</a>''',
|
|
||||||
1138,
|
|
||||||
[{
|
|
||||||
'start_time': 0,
|
|
||||||
'end_time': 490,
|
|
||||||
'title': '1 - Во прах [Vo prakh]',
|
|
||||||
}, {
|
|
||||||
'start_time': 490,
|
|
||||||
'end_time': 870,
|
|
||||||
'title': '2 - Искупление [Iskupleniye]',
|
|
||||||
}, {
|
|
||||||
'start_time': 870,
|
|
||||||
'end_time': 1138,
|
|
||||||
'title': '3 - Из серпов луны...[Iz serpov luny]',
|
|
||||||
}]
|
|
||||||
),
|
|
||||||
(
|
|
||||||
# https://www.youtube.com/watch?v=xZW70zEasOk
|
|
||||||
# time point more than duration
|
|
||||||
'''● LCS Spring finals: Saturday and Sunday from <a href="#" onclick="yt.www.watch.player.seekTo(13*60+30);return false;">13:30</a> outside the venue! <br />● PAX East: Fri, Sat & Sun - more info in tomorrows video on the main channel!''',
|
|
||||||
283,
|
|
||||||
[]
|
|
||||||
),
|
|
||||||
]
|
|
||||||
|
|
||||||
def test_youtube_chapters(self):
|
|
||||||
for description, duration, expected_chapters in self._TEST_CASES:
|
|
||||||
ie = YoutubeIE()
|
|
||||||
expect_value(
|
|
||||||
self, ie._extract_chapters(description, duration),
|
|
||||||
expected_chapters, None)
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == '__main__':
|
|
||||||
unittest.main()
|
|
|
@ -1,4 +1,5 @@
|
||||||
#!/usr/bin/env python
|
#!/usr/bin/env python
|
||||||
|
# -*- coding: utf-8 -*-
|
||||||
from __future__ import unicode_literals
|
from __future__ import unicode_literals
|
||||||
|
|
||||||
# Allow direct execution
|
# Allow direct execution
|
||||||
|
@ -9,10 +10,10 @@ sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||||
|
|
||||||
from test.helper import FakeYDL
|
from test.helper import FakeYDL
|
||||||
|
|
||||||
|
|
||||||
from youtube_dl.extractor import (
|
from youtube_dl.extractor import (
|
||||||
YoutubePlaylistIE,
|
|
||||||
YoutubeIE,
|
YoutubeIE,
|
||||||
|
YoutubePlaylistIE,
|
||||||
|
YoutubeTabIE,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@ -24,47 +25,40 @@ class TestYoutubeLists(unittest.TestCase):
|
||||||
def test_youtube_playlist_noplaylist(self):
|
def test_youtube_playlist_noplaylist(self):
|
||||||
dl = FakeYDL()
|
dl = FakeYDL()
|
||||||
dl.params['noplaylist'] = True
|
dl.params['noplaylist'] = True
|
||||||
|
dl.params['format'] = 'best'
|
||||||
ie = YoutubePlaylistIE(dl)
|
ie = YoutubePlaylistIE(dl)
|
||||||
result = ie.extract('https://www.youtube.com/watch?v=FXxLjLQi3Fg&list=PLwiyx1dc3P2JR9N8gQaQN_BCvlSlap7re')
|
result = ie.extract('https://www.youtube.com/watch?v=FXxLjLQi3Fg&list=PLwiyx1dc3P2JR9N8gQaQN_BCvlSlap7re')
|
||||||
self.assertEqual(result['_type'], 'url')
|
self.assertEqual(result['_type'], 'url')
|
||||||
|
result = dl.extract_info(result['url'], download=False, ie_key=result.get('ie_key'), process=False)
|
||||||
self.assertEqual(YoutubeIE().extract_id(result['url']), 'FXxLjLQi3Fg')
|
self.assertEqual(YoutubeIE().extract_id(result['url']), 'FXxLjLQi3Fg')
|
||||||
|
|
||||||
def test_youtube_course(self):
|
|
||||||
dl = FakeYDL()
|
|
||||||
ie = YoutubePlaylistIE(dl)
|
|
||||||
# TODO find a > 100 (paginating?) videos course
|
|
||||||
result = ie.extract('https://www.youtube.com/course?list=ECUl4u3cNGP61MdtwGTqZA0MreSaDybji8')
|
|
||||||
entries = list(result['entries'])
|
|
||||||
self.assertEqual(YoutubeIE().extract_id(entries[0]['url']), 'j9WZyLZCBzs')
|
|
||||||
self.assertEqual(len(entries), 25)
|
|
||||||
self.assertEqual(YoutubeIE().extract_id(entries[-1]['url']), 'rYefUsYuEp0')
|
|
||||||
|
|
||||||
def test_youtube_mix(self):
|
def test_youtube_mix(self):
|
||||||
dl = FakeYDL()
|
dl = FakeYDL()
|
||||||
ie = YoutubePlaylistIE(dl)
|
dl.params['format'] = 'best'
|
||||||
result = ie.extract('https://www.youtube.com/watch?v=W01L70IGBgE&index=2&list=RDOQpdSVF_k_w')
|
ie = YoutubeTabIE(dl)
|
||||||
entries = result['entries']
|
result = dl.extract_info('https://www.youtube.com/watch?v=tyITL_exICo&list=RDCLAK5uy_kLWIr9gv1XLlPbaDS965-Db4TrBoUTxQ8',
|
||||||
self.assertTrue(len(entries) >= 50)
|
download=False, ie_key=ie.ie_key(), process=True)
|
||||||
|
entries = (result or {}).get('entries', [{'id': 'not_found', }])
|
||||||
|
self.assertTrue(len(entries) >= 25)
|
||||||
original_video = entries[0]
|
original_video = entries[0]
|
||||||
self.assertEqual(original_video['id'], 'OQpdSVF_k_w')
|
self.assertEqual(original_video['id'], 'tyITL_exICo')
|
||||||
|
|
||||||
def test_youtube_toptracks(self):
|
def test_youtube_flat_playlist_extraction(self):
|
||||||
print('Skipping: The playlist page gives error 500')
|
|
||||||
return
|
|
||||||
dl = FakeYDL()
|
|
||||||
ie = YoutubePlaylistIE(dl)
|
|
||||||
result = ie.extract('https://www.youtube.com/playlist?list=MCUS')
|
|
||||||
entries = result['entries']
|
|
||||||
self.assertEqual(len(entries), 100)
|
|
||||||
|
|
||||||
def test_youtube_flat_playlist_titles(self):
|
|
||||||
dl = FakeYDL()
|
dl = FakeYDL()
|
||||||
dl.params['extract_flat'] = True
|
dl.params['extract_flat'] = True
|
||||||
ie = YoutubePlaylistIE(dl)
|
ie = YoutubeTabIE(dl)
|
||||||
result = ie.extract('https://www.youtube.com/playlist?list=PL-KKIb8rvtMSrAO9YFbeM6UQrAqoFTUWv')
|
result = ie.extract('https://www.youtube.com/playlist?list=PL4lCao7KL_QFVb7Iudeipvc2BCavECqzc')
|
||||||
self.assertIsPlaylist(result)
|
self.assertIsPlaylist(result)
|
||||||
for entry in result['entries']:
|
entries = list(result['entries'])
|
||||||
self.assertTrue(entry.get('title'))
|
self.assertTrue(len(entries) == 1)
|
||||||
|
video = entries[0]
|
||||||
|
self.assertEqual(video['_type'], 'url')
|
||||||
|
self.assertEqual(video['ie_key'], 'Youtube')
|
||||||
|
self.assertEqual(video['id'], 'BaW_jenozKc')
|
||||||
|
self.assertEqual(video['url'], 'BaW_jenozKc')
|
||||||
|
self.assertEqual(video['title'], 'youtube-dl test video "\'/\\ä↭𝕐')
|
||||||
|
self.assertEqual(video['duration'], 10)
|
||||||
|
self.assertEqual(video['uploader'], 'Philipp Hagemeister')
|
||||||
|
|
||||||
|
|
||||||
if __name__ == '__main__':
|
if __name__ == '__main__':
|
||||||
|
|
26
test/test_youtube_misc.py
Normal file
26
test/test_youtube_misc.py
Normal file
|
@ -0,0 +1,26 @@
|
||||||
|
#!/usr/bin/env python
|
||||||
|
from __future__ import unicode_literals
|
||||||
|
|
||||||
|
# Allow direct execution
|
||||||
|
import os
|
||||||
|
import sys
|
||||||
|
import unittest
|
||||||
|
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||||
|
|
||||||
|
|
||||||
|
from youtube_dl.extractor import YoutubeIE
|
||||||
|
|
||||||
|
|
||||||
|
class TestYoutubeMisc(unittest.TestCase):
|
||||||
|
def test_youtube_extract(self):
|
||||||
|
assertExtractId = lambda url, id: self.assertEqual(YoutubeIE.extract_id(url), id)
|
||||||
|
assertExtractId('http://www.youtube.com/watch?&v=BaW_jenozKc', 'BaW_jenozKc')
|
||||||
|
assertExtractId('https://www.youtube.com/watch?&v=BaW_jenozKc', 'BaW_jenozKc')
|
||||||
|
assertExtractId('https://www.youtube.com/watch?feature=player_embedded&v=BaW_jenozKc', 'BaW_jenozKc')
|
||||||
|
assertExtractId('https://www.youtube.com/watch_popup?v=BaW_jenozKc', 'BaW_jenozKc')
|
||||||
|
assertExtractId('http://www.youtube.com/watch?v=BaW_jenozKcsharePLED17F32AD9753930', 'BaW_jenozKc')
|
||||||
|
assertExtractId('BaW_jenozKc', 'BaW_jenozKc')
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
unittest.main()
|
|
@ -8,115 +8,264 @@ import sys
|
||||||
import unittest
|
import unittest
|
||||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||||
|
|
||||||
import io
|
|
||||||
import re
|
import re
|
||||||
import string
|
import string
|
||||||
|
|
||||||
|
from youtube_dl.compat import (
|
||||||
|
compat_open as open,
|
||||||
|
compat_str,
|
||||||
|
compat_urlretrieve,
|
||||||
|
)
|
||||||
|
|
||||||
from test.helper import FakeYDL
|
from test.helper import FakeYDL
|
||||||
from youtube_dl.extractor import YoutubeIE
|
from youtube_dl.extractor import YoutubeIE
|
||||||
from youtube_dl.compat import compat_str, compat_urlretrieve
|
from youtube_dl.jsinterp import JSInterpreter
|
||||||
|
|
||||||
_TESTS = [
|
_SIG_TESTS = [
|
||||||
(
|
(
|
||||||
'https://s.ytimg.com/yts/jsbin/html5player-vflHOr_nV.js',
|
'https://s.ytimg.com/yts/jsbin/html5player-vflHOr_nV.js',
|
||||||
'js',
|
|
||||||
86,
|
86,
|
||||||
'>=<;:/.-[+*)(\'&%$#"!ZYX0VUTSRQPONMLKJIHGFEDCBA\\yxwvutsrqponmlkjihgfedcba987654321',
|
'>=<;:/.-[+*)(\'&%$#"!ZYX0VUTSRQPONMLKJIHGFEDCBA\\yxwvutsrqponmlkjihgfedcba987654321',
|
||||||
),
|
),
|
||||||
(
|
(
|
||||||
'https://s.ytimg.com/yts/jsbin/html5player-vfldJ8xgI.js',
|
'https://s.ytimg.com/yts/jsbin/html5player-vfldJ8xgI.js',
|
||||||
'js',
|
|
||||||
85,
|
85,
|
||||||
'3456789a0cdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRS[UVWXYZ!"#$%&\'()*+,-./:;<=>?@',
|
'3456789a0cdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRS[UVWXYZ!"#$%&\'()*+,-./:;<=>?@',
|
||||||
),
|
),
|
||||||
(
|
(
|
||||||
'https://s.ytimg.com/yts/jsbin/html5player-vfle-mVwz.js',
|
'https://s.ytimg.com/yts/jsbin/html5player-vfle-mVwz.js',
|
||||||
'js',
|
|
||||||
90,
|
90,
|
||||||
']\\[@?>=<;:/.-,+*)(\'&%$#"hZYXWVUTSRQPONMLKJIHGFEDCBAzyxwvutsrqponmlkjiagfedcb39876',
|
']\\[@?>=<;:/.-,+*)(\'&%$#"hZYXWVUTSRQPONMLKJIHGFEDCBAzyxwvutsrqponmlkjiagfedcb39876',
|
||||||
),
|
),
|
||||||
(
|
(
|
||||||
'https://s.ytimg.com/yts/jsbin/html5player-en_US-vfl0Cbn9e.js',
|
'https://s.ytimg.com/yts/jsbin/html5player-en_US-vfl0Cbn9e.js',
|
||||||
'js',
|
|
||||||
84,
|
84,
|
||||||
'O1I3456789abcde0ghijklmnopqrstuvwxyzABCDEFGHfJKLMN2PQRSTUVW@YZ!"#$%&\'()*+,-./:;<=',
|
'O1I3456789abcde0ghijklmnopqrstuvwxyzABCDEFGHfJKLMN2PQRSTUVW@YZ!"#$%&\'()*+,-./:;<=',
|
||||||
),
|
),
|
||||||
(
|
(
|
||||||
'https://s.ytimg.com/yts/jsbin/html5player-en_US-vflXGBaUN.js',
|
'https://s.ytimg.com/yts/jsbin/html5player-en_US-vflXGBaUN.js',
|
||||||
'js',
|
|
||||||
'2ACFC7A61CA478CD21425E5A57EBD73DDC78E22A.2094302436B2D377D14A3BBA23022D023B8BC25AA',
|
'2ACFC7A61CA478CD21425E5A57EBD73DDC78E22A.2094302436B2D377D14A3BBA23022D023B8BC25AA',
|
||||||
'A52CB8B320D22032ABB3A41D773D2B6342034902.A22E87CDD37DBE75A5E52412DC874AC16A7CFCA2',
|
'A52CB8B320D22032ABB3A41D773D2B6342034902.A22E87CDD37DBE75A5E52412DC874AC16A7CFCA2',
|
||||||
),
|
),
|
||||||
(
|
(
|
||||||
'https://s.ytimg.com/yts/jsbin/html5player-en_US-vflBb0OQx.js',
|
'https://s.ytimg.com/yts/jsbin/html5player-en_US-vflBb0OQx.js',
|
||||||
'js',
|
|
||||||
84,
|
84,
|
||||||
'123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQ0STUVWXYZ!"#$%&\'()*+,@./:;<=>'
|
'123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQ0STUVWXYZ!"#$%&\'()*+,@./:;<=>'
|
||||||
),
|
),
|
||||||
(
|
(
|
||||||
'https://s.ytimg.com/yts/jsbin/html5player-en_US-vfl9FYC6l.js',
|
'https://s.ytimg.com/yts/jsbin/html5player-en_US-vfl9FYC6l.js',
|
||||||
'js',
|
|
||||||
83,
|
83,
|
||||||
'123456789abcdefghijklmnopqr0tuvwxyzABCDETGHIJKLMNOPQRS>UVWXYZ!"#$%&\'()*+,-./:;<=F'
|
'123456789abcdefghijklmnopqr0tuvwxyzABCDETGHIJKLMNOPQRS>UVWXYZ!"#$%&\'()*+,-./:;<=F'
|
||||||
),
|
),
|
||||||
(
|
(
|
||||||
'https://s.ytimg.com/yts/jsbin/html5player-en_US-vflCGk6yw/html5player.js',
|
'https://s.ytimg.com/yts/jsbin/html5player-en_US-vflCGk6yw/html5player.js',
|
||||||
'js',
|
|
||||||
'4646B5181C6C3020DF1D9C7FCFEA.AD80ABF70C39BD369CCCAE780AFBB98FA6B6CB42766249D9488C288',
|
'4646B5181C6C3020DF1D9C7FCFEA.AD80ABF70C39BD369CCCAE780AFBB98FA6B6CB42766249D9488C288',
|
||||||
'82C8849D94266724DC6B6AF89BBFA087EACCD963.B93C07FBA084ACAEFCF7C9D1FD0203C6C1815B6B'
|
'82C8849D94266724DC6B6AF89BBFA087EACCD963.B93C07FBA084ACAEFCF7C9D1FD0203C6C1815B6B'
|
||||||
),
|
),
|
||||||
(
|
(
|
||||||
'https://s.ytimg.com/yts/jsbin/html5player-en_US-vflKjOTVq/html5player.js',
|
'https://s.ytimg.com/yts/jsbin/html5player-en_US-vflKjOTVq/html5player.js',
|
||||||
'js',
|
|
||||||
'312AA52209E3623129A412D56A40F11CB0AF14AE.3EE09501CB14E3BCDC3B2AE808BF3F1D14E7FBF12',
|
'312AA52209E3623129A412D56A40F11CB0AF14AE.3EE09501CB14E3BCDC3B2AE808BF3F1D14E7FBF12',
|
||||||
'112AA5220913623229A412D56A40F11CB0AF14AE.3EE0950FCB14EEBCDC3B2AE808BF331D14E7FBF3',
|
'112AA5220913623229A412D56A40F11CB0AF14AE.3EE0950FCB14EEBCDC3B2AE808BF331D14E7FBF3',
|
||||||
)
|
)
|
||||||
]
|
]
|
||||||
|
|
||||||
|
_NSIG_TESTS = [
|
||||||
|
(
|
||||||
|
'https://www.youtube.com/s/player/7862ca1f/player_ias.vflset/en_US/base.js',
|
||||||
|
'X_LCxVDjAavgE5t', 'yxJ1dM6iz5ogUg',
|
||||||
|
),
|
||||||
|
(
|
||||||
|
'https://www.youtube.com/s/player/9216d1f7/player_ias.vflset/en_US/base.js',
|
||||||
|
'SLp9F5bwjAdhE9F-', 'gWnb9IK2DJ8Q1w',
|
||||||
|
),
|
||||||
|
(
|
||||||
|
'https://www.youtube.com/s/player/f8cb7a3b/player_ias.vflset/en_US/base.js',
|
||||||
|
'oBo2h5euWy6osrUt', 'ivXHpm7qJjJN',
|
||||||
|
),
|
||||||
|
(
|
||||||
|
'https://www.youtube.com/s/player/2dfe380c/player_ias.vflset/en_US/base.js',
|
||||||
|
'oBo2h5euWy6osrUt', '3DIBbn3qdQ',
|
||||||
|
),
|
||||||
|
(
|
||||||
|
'https://www.youtube.com/s/player/f1ca6900/player_ias.vflset/en_US/base.js',
|
||||||
|
'cu3wyu6LQn2hse', 'jvxetvmlI9AN9Q',
|
||||||
|
),
|
||||||
|
(
|
||||||
|
'https://www.youtube.com/s/player/8040e515/player_ias.vflset/en_US/base.js',
|
||||||
|
'wvOFaY-yjgDuIEg5', 'HkfBFDHmgw4rsw',
|
||||||
|
),
|
||||||
|
(
|
||||||
|
'https://www.youtube.com/s/player/e06dea74/player_ias.vflset/en_US/base.js',
|
||||||
|
'AiuodmaDDYw8d3y4bf', 'ankd8eza2T6Qmw',
|
||||||
|
),
|
||||||
|
(
|
||||||
|
'https://www.youtube.com/s/player/5dd88d1d/player-plasma-ias-phone-en_US.vflset/base.js',
|
||||||
|
'kSxKFLeqzv_ZyHSAt', 'n8gS8oRlHOxPFA',
|
||||||
|
),
|
||||||
|
(
|
||||||
|
'https://www.youtube.com/s/player/324f67b9/player_ias.vflset/en_US/base.js',
|
||||||
|
'xdftNy7dh9QGnhW', '22qLGxrmX8F1rA',
|
||||||
|
),
|
||||||
|
(
|
||||||
|
'https://www.youtube.com/s/player/4c3f79c5/player_ias.vflset/en_US/base.js',
|
||||||
|
'TDCstCG66tEAO5pR9o', 'dbxNtZ14c-yWyw',
|
||||||
|
),
|
||||||
|
(
|
||||||
|
'https://www.youtube.com/s/player/c81bbb4a/player_ias.vflset/en_US/base.js',
|
||||||
|
'gre3EcLurNY2vqp94', 'Z9DfGxWP115WTg',
|
||||||
|
),
|
||||||
|
(
|
||||||
|
'https://www.youtube.com/s/player/1f7d5369/player_ias.vflset/en_US/base.js',
|
||||||
|
'batNX7sYqIJdkJ', 'IhOkL_zxbkOZBw',
|
||||||
|
),
|
||||||
|
(
|
||||||
|
'https://www.youtube.com/s/player/009f1d77/player_ias.vflset/en_US/base.js',
|
||||||
|
'5dwFHw8aFWQUQtffRq', 'audescmLUzI3jw',
|
||||||
|
),
|
||||||
|
(
|
||||||
|
'https://www.youtube.com/s/player/dc0c6770/player_ias.vflset/en_US/base.js',
|
||||||
|
'5EHDMgYLV6HPGk_Mu-kk', 'n9lUJLHbxUI0GQ',
|
||||||
|
),
|
||||||
|
(
|
||||||
|
'https://www.youtube.com/s/player/c2199353/player_ias.vflset/en_US/base.js',
|
||||||
|
'5EHDMgYLV6HPGk_Mu-kk', 'AD5rgS85EkrE7',
|
||||||
|
),
|
||||||
|
(
|
||||||
|
'https://www.youtube.com/s/player/113ca41c/player_ias.vflset/en_US/base.js',
|
||||||
|
'cgYl-tlYkhjT7A', 'hI7BBr2zUgcmMg',
|
||||||
|
),
|
||||||
|
(
|
||||||
|
'https://www.youtube.com/s/player/c57c113c/player_ias.vflset/en_US/base.js',
|
||||||
|
'-Txvy6bT5R6LqgnQNx', 'dcklJCnRUHbgSg',
|
||||||
|
),
|
||||||
|
(
|
||||||
|
'https://www.youtube.com/s/player/5a3b6271/player_ias.vflset/en_US/base.js',
|
||||||
|
'B2j7f_UPT4rfje85Lu_e', 'm5DmNymaGQ5RdQ',
|
||||||
|
),
|
||||||
|
(
|
||||||
|
'https://www.youtube.com/s/player/dac945fd/player_ias.vflset/en_US/base.js',
|
||||||
|
'o8BkRxXhuYsBCWi6RplPdP', '3Lx32v_hmzTm6A',
|
||||||
|
),
|
||||||
|
(
|
||||||
|
'https://www.youtube.com/s/player/6f20102c/player_ias.vflset/en_US/base.js',
|
||||||
|
'lE8DhoDmKqnmJJ', 'pJTTX6XyJP2BYw',
|
||||||
|
),
|
||||||
|
(
|
||||||
|
'https://www.youtube.com/s/player/cfa9e7cb/player_ias.vflset/en_US/base.js',
|
||||||
|
'qO0NiMtYQ7TeJnfFG2', 'k9cuJDHNS5O7kQ',
|
||||||
|
),
|
||||||
|
(
|
||||||
|
'https://www.youtube.com/s/player/b7910ca8/player_ias.vflset/en_US/base.js',
|
||||||
|
'_hXMCwMt9qE310D', 'LoZMgkkofRMCZQ',
|
||||||
|
),
|
||||||
|
(
|
||||||
|
'https://www.youtube.com/s/player/590f65a6/player_ias.vflset/en_US/base.js',
|
||||||
|
'1tm7-g_A9zsI8_Lay_', 'xI4Vem4Put_rOg',
|
||||||
|
),
|
||||||
|
(
|
||||||
|
'https://www.youtube.com/s/player/b22ef6e7/player_ias.vflset/en_US/base.js',
|
||||||
|
'b6HcntHGkvBLk_FRf', 'kNPW6A7FyP2l8A',
|
||||||
|
),
|
||||||
|
(
|
||||||
|
'https://www.youtube.com/s/player/3400486c/player_ias.vflset/en_US/base.js',
|
||||||
|
'lL46g3XifCKUZn1Xfw', 'z767lhet6V2Skl',
|
||||||
|
),
|
||||||
|
(
|
||||||
|
'https://www.youtube.com/s/player/5604538d/player_ias.vflset/en_US/base.js',
|
||||||
|
'7X-he4jjvMx7BCX', 'sViSydX8IHtdWA',
|
||||||
|
),
|
||||||
|
(
|
||||||
|
'https://www.youtube.com/s/player/20dfca59/player_ias.vflset/en_US/base.js',
|
||||||
|
'-fLCxedkAk4LUTK2', 'O8kfRq1y1eyHGw',
|
||||||
|
),
|
||||||
|
(
|
||||||
|
'https://www.youtube.com/s/player/b12cc44b/player_ias.vflset/en_US/base.js',
|
||||||
|
'keLa5R2U00sR9SQK', 'N1OGyujjEwMnLw',
|
||||||
|
),
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
class TestPlayerInfo(unittest.TestCase):
|
||||||
|
def test_youtube_extract_player_info(self):
|
||||||
|
PLAYER_URLS = (
|
||||||
|
('https://www.youtube.com/s/player/4c3f79c5/player_ias.vflset/en_US/base.js', '4c3f79c5'),
|
||||||
|
('https://www.youtube.com/s/player/64dddad9/player_ias.vflset/en_US/base.js', '64dddad9'),
|
||||||
|
('https://www.youtube.com/s/player/64dddad9/player_ias.vflset/fr_FR/base.js', '64dddad9'),
|
||||||
|
('https://www.youtube.com/s/player/64dddad9/player-plasma-ias-phone-en_US.vflset/base.js', '64dddad9'),
|
||||||
|
('https://www.youtube.com/s/player/64dddad9/player-plasma-ias-phone-de_DE.vflset/base.js', '64dddad9'),
|
||||||
|
('https://www.youtube.com/s/player/64dddad9/player-plasma-ias-tablet-en_US.vflset/base.js', '64dddad9'),
|
||||||
|
# obsolete
|
||||||
|
('https://www.youtube.com/yts/jsbin/player_ias-vfle4-e03/en_US/base.js', 'vfle4-e03'),
|
||||||
|
('https://www.youtube.com/yts/jsbin/player_ias-vfl49f_g4/en_US/base.js', 'vfl49f_g4'),
|
||||||
|
('https://www.youtube.com/yts/jsbin/player_ias-vflCPQUIL/en_US/base.js', 'vflCPQUIL'),
|
||||||
|
('https://www.youtube.com/yts/jsbin/player-vflzQZbt7/en_US/base.js', 'vflzQZbt7'),
|
||||||
|
('https://www.youtube.com/yts/jsbin/player-en_US-vflaxXRn1/base.js', 'vflaxXRn1'),
|
||||||
|
('https://s.ytimg.com/yts/jsbin/html5player-en_US-vflXGBaUN.js', 'vflXGBaUN'),
|
||||||
|
('https://s.ytimg.com/yts/jsbin/html5player-en_US-vflKjOTVq/html5player.js', 'vflKjOTVq'),
|
||||||
|
)
|
||||||
|
for player_url, expected_player_id in PLAYER_URLS:
|
||||||
|
player_id = YoutubeIE._extract_player_info(player_url)
|
||||||
|
self.assertEqual(player_id, expected_player_id)
|
||||||
|
|
||||||
|
|
||||||
class TestSignature(unittest.TestCase):
|
class TestSignature(unittest.TestCase):
|
||||||
def setUp(self):
|
def setUp(self):
|
||||||
TEST_DIR = os.path.dirname(os.path.abspath(__file__))
|
TEST_DIR = os.path.dirname(os.path.abspath(__file__))
|
||||||
self.TESTDATA_DIR = os.path.join(TEST_DIR, 'testdata')
|
self.TESTDATA_DIR = os.path.join(TEST_DIR, 'testdata/sigs')
|
||||||
if not os.path.exists(self.TESTDATA_DIR):
|
if not os.path.exists(self.TESTDATA_DIR):
|
||||||
os.mkdir(self.TESTDATA_DIR)
|
os.mkdir(self.TESTDATA_DIR)
|
||||||
|
|
||||||
|
def tearDown(self):
|
||||||
|
try:
|
||||||
|
for f in os.listdir(self.TESTDATA_DIR):
|
||||||
|
os.remove(f)
|
||||||
|
except OSError:
|
||||||
|
pass
|
||||||
|
|
||||||
def make_tfunc(url, stype, sig_input, expected_sig):
|
|
||||||
m = re.match(r'.*-([a-zA-Z0-9_-]+)(?:/watch_as3|/html5player)?\.[a-z]+$', url)
|
|
||||||
assert m, '%r should follow URL format' % url
|
|
||||||
test_id = m.group(1)
|
|
||||||
|
|
||||||
def test_func(self):
|
def t_factory(name, sig_func, url_pattern):
|
||||||
basename = 'player-%s.%s' % (test_id, stype)
|
def make_tfunc(url, sig_input, expected_sig):
|
||||||
fn = os.path.join(self.TESTDATA_DIR, basename)
|
m = url_pattern.match(url)
|
||||||
|
assert m, '%r should follow URL format' % url
|
||||||
|
test_id = m.group('id')
|
||||||
|
|
||||||
if not os.path.exists(fn):
|
def test_func(self):
|
||||||
compat_urlretrieve(url, fn)
|
basename = 'player-{0}-{1}.js'.format(name, test_id)
|
||||||
|
fn = os.path.join(self.TESTDATA_DIR, basename)
|
||||||
|
|
||||||
ydl = FakeYDL()
|
if not os.path.exists(fn):
|
||||||
ie = YoutubeIE(ydl)
|
compat_urlretrieve(url, fn)
|
||||||
if stype == 'js':
|
with open(fn, encoding='utf-8') as testf:
|
||||||
with io.open(fn, encoding='utf-8') as testf:
|
|
||||||
jscode = testf.read()
|
jscode = testf.read()
|
||||||
func = ie._parse_sig_js(jscode)
|
self.assertEqual(sig_func(jscode, sig_input), expected_sig)
|
||||||
else:
|
|
||||||
assert stype == 'swf'
|
|
||||||
with open(fn, 'rb') as testf:
|
|
||||||
swfcode = testf.read()
|
|
||||||
func = ie._parse_sig_swf(swfcode)
|
|
||||||
src_sig = (
|
|
||||||
compat_str(string.printable[:sig_input])
|
|
||||||
if isinstance(sig_input, int) else sig_input)
|
|
||||||
got_sig = func(src_sig)
|
|
||||||
self.assertEqual(got_sig, expected_sig)
|
|
||||||
|
|
||||||
test_func.__name__ = str('test_signature_' + stype + '_' + test_id)
|
test_func.__name__ = str('test_{0}_js_{1}'.format(name, test_id))
|
||||||
setattr(TestSignature, test_func.__name__, test_func)
|
setattr(TestSignature, test_func.__name__, test_func)
|
||||||
|
return make_tfunc
|
||||||
|
|
||||||
|
|
||||||
for test_spec in _TESTS:
|
def signature(jscode, sig_input):
|
||||||
make_tfunc(*test_spec)
|
func = YoutubeIE(FakeYDL())._parse_sig_js(jscode)
|
||||||
|
src_sig = (
|
||||||
|
compat_str(string.printable[:sig_input])
|
||||||
|
if isinstance(sig_input, int) else sig_input)
|
||||||
|
return func(src_sig)
|
||||||
|
|
||||||
|
|
||||||
|
def n_sig(jscode, sig_input):
|
||||||
|
funcname = YoutubeIE(FakeYDL())._extract_n_function_name(jscode)
|
||||||
|
return JSInterpreter(jscode).call_function(funcname, sig_input)
|
||||||
|
|
||||||
|
|
||||||
|
make_sig_test = t_factory(
|
||||||
|
'signature', signature, re.compile(r'.*-(?P<id>[a-zA-Z0-9_-]+)(?:/watch_as3|/html5player)?\.[a-z]+$'))
|
||||||
|
for test_spec in _SIG_TESTS:
|
||||||
|
make_sig_test(*test_spec)
|
||||||
|
|
||||||
|
make_nsig_test = t_factory(
|
||||||
|
'nsig', n_sig, re.compile(r'.+/player/(?P<id>[a-zA-Z0-9_-]+)/.+.js$'))
|
||||||
|
for test_spec in _NSIG_TESTS:
|
||||||
|
make_nsig_test(*test_spec)
|
||||||
|
|
||||||
|
|
||||||
if __name__ == '__main__':
|
if __name__ == '__main__':
|
||||||
|
|
9
test/testdata/cookies/malformed_cookies.txt
vendored
Normal file
9
test/testdata/cookies/malformed_cookies.txt
vendored
Normal file
|
@ -0,0 +1,9 @@
|
||||||
|
# Netscape HTTP Cookie File
|
||||||
|
# http://curl.haxx.se/rfc/cookie_spec.html
|
||||||
|
# This is a generated file! Do not edit.
|
||||||
|
|
||||||
|
# Cookie file entry with invalid number of fields - 6 instead of 7
|
||||||
|
www.foobar.foobar FALSE / FALSE 0 COOKIE
|
||||||
|
|
||||||
|
# Cookie file entry with invalid expires at
|
||||||
|
www.foobar.foobar FALSE / FALSE 1.7976931348623157e+308 COOKIE VALUE
|
35
test/testdata/mpd/range_only.mpd
vendored
Normal file
35
test/testdata/mpd/range_only.mpd
vendored
Normal file
|
@ -0,0 +1,35 @@
|
||||||
|
<?xml version="1.0"?>
|
||||||
|
<!-- MPD file Generated with GPAC version 1.0.1-revrelease at 2021-11-27T20:53:11.690Z -->
|
||||||
|
<MPD xmlns="urn:mpeg:dash:schema:mpd:2011" minBufferTime="PT1.500S" type="static" mediaPresentationDuration="PT0H0M30.196S" maxSegmentDuration="PT0H0M10.027S" profiles="urn:mpeg:dash:profile:full:2011">
|
||||||
|
<ProgramInformation moreInformationURL="http://gpac.io">
|
||||||
|
<Title>manifest.mpd generated by GPAC</Title>
|
||||||
|
</ProgramInformation>
|
||||||
|
|
||||||
|
<Period duration="PT0H0M30.196S">
|
||||||
|
<AdaptationSet segmentAlignment="true" maxWidth="768" maxHeight="432" maxFrameRate="30000/1001" par="16:9" lang="und" startWithSAP="1">
|
||||||
|
<Representation id="1" mimeType="video/mp4" codecs="avc1.4D401E" width="768" height="432" frameRate="30000/1001" sar="1:1" bandwidth="526987">
|
||||||
|
<BaseURL>video_dashinit.mp4</BaseURL>
|
||||||
|
<SegmentList timescale="90000" duration="900000">
|
||||||
|
<Initialization range="0-881"/>
|
||||||
|
<SegmentURL mediaRange="882-876094" indexRange="882-925"/>
|
||||||
|
<SegmentURL mediaRange="876095-1466732" indexRange="876095-876138"/>
|
||||||
|
<SegmentURL mediaRange="1466733-1953615" indexRange="1466733-1466776"/>
|
||||||
|
<SegmentURL mediaRange="1953616-1994211" indexRange="1953616-1953659"/>
|
||||||
|
</SegmentList>
|
||||||
|
</Representation>
|
||||||
|
</AdaptationSet>
|
||||||
|
<AdaptationSet segmentAlignment="true" lang="und" startWithSAP="1">
|
||||||
|
<Representation id="2" mimeType="audio/mp4" codecs="mp4a.40.2" audioSamplingRate="48000" bandwidth="98096">
|
||||||
|
<AudioChannelConfiguration schemeIdUri="urn:mpeg:dash:23003:3:audio_channel_configuration:2011" value="2"/>
|
||||||
|
<BaseURL>audio_dashinit.mp4</BaseURL>
|
||||||
|
<SegmentList timescale="48000" duration="480000">
|
||||||
|
<Initialization range="0-752"/>
|
||||||
|
<SegmentURL mediaRange="753-124129" indexRange="753-796"/>
|
||||||
|
<SegmentURL mediaRange="124130-250544" indexRange="124130-124173"/>
|
||||||
|
<SegmentURL mediaRange="250545-374929" indexRange="250545-250588"/>
|
||||||
|
</SegmentList>
|
||||||
|
</Representation>
|
||||||
|
</AdaptationSet>
|
||||||
|
</Period>
|
||||||
|
</MPD>
|
||||||
|
|
351
test/testdata/mpd/subtitles.mpd
vendored
Normal file
351
test/testdata/mpd/subtitles.mpd
vendored
Normal file
|
@ -0,0 +1,351 @@
|
||||||
|
<?xml version="1.0" encoding="utf-8"?>
|
||||||
|
<!-- Created with Unified Streaming Platform (version=1.10.18-20255) -->
|
||||||
|
<MPD
|
||||||
|
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||||
|
xmlns="urn:mpeg:dash:schema:mpd:2011"
|
||||||
|
xsi:schemaLocation="urn:mpeg:dash:schema:mpd:2011 http://standards.iso.org/ittf/PubliclyAvailableStandards/MPEG-DASH_schema_files/DASH-MPD.xsd"
|
||||||
|
type="static"
|
||||||
|
mediaPresentationDuration="PT14M48S"
|
||||||
|
maxSegmentDuration="PT1M"
|
||||||
|
minBufferTime="PT10S"
|
||||||
|
profiles="urn:mpeg:dash:profile:isoff-live:2011">
|
||||||
|
<Period
|
||||||
|
id="1"
|
||||||
|
duration="PT14M48S">
|
||||||
|
<BaseURL>dash/</BaseURL>
|
||||||
|
<AdaptationSet
|
||||||
|
id="1"
|
||||||
|
group="1"
|
||||||
|
contentType="audio"
|
||||||
|
segmentAlignment="true"
|
||||||
|
audioSamplingRate="48000"
|
||||||
|
mimeType="audio/mp4"
|
||||||
|
codecs="mp4a.40.2"
|
||||||
|
startWithSAP="1">
|
||||||
|
<AudioChannelConfiguration
|
||||||
|
schemeIdUri="urn:mpeg:dash:23003:3:audio_channel_configuration:2011"
|
||||||
|
value="2" />
|
||||||
|
<Role schemeIdUri="urn:mpeg:dash:role:2011" value="main" />
|
||||||
|
<SegmentTemplate
|
||||||
|
timescale="48000"
|
||||||
|
initialization="3144-kZT4LWMQw6Rh7Kpd-$RepresentationID$.dash"
|
||||||
|
media="3144-kZT4LWMQw6Rh7Kpd-$RepresentationID$-$Time$.dash">
|
||||||
|
<SegmentTimeline>
|
||||||
|
<S t="0" d="96256" r="2" />
|
||||||
|
<S d="95232" />
|
||||||
|
<S d="96256" r="2" />
|
||||||
|
<S d="95232" />
|
||||||
|
<S d="96256" r="2" />
|
||||||
|
<S d="95232" />
|
||||||
|
<S d="96256" r="2" />
|
||||||
|
<S d="95232" />
|
||||||
|
<S d="96256" r="2" />
|
||||||
|
<S d="95232" />
|
||||||
|
<S d="96256" r="2" />
|
||||||
|
<S d="95232" />
|
||||||
|
<S d="96256" r="2" />
|
||||||
|
<S d="95232" />
|
||||||
|
<S d="96256" r="2" />
|
||||||
|
<S d="95232" />
|
||||||
|
<S d="96256" r="2" />
|
||||||
|
<S d="95232" />
|
||||||
|
<S d="96256" r="2" />
|
||||||
|
<S d="95232" />
|
||||||
|
<S d="96256" r="2" />
|
||||||
|
<S d="95232" />
|
||||||
|
<S d="96256" r="2" />
|
||||||
|
<S d="95232" />
|
||||||
|
<S d="96256" r="2" />
|
||||||
|
<S d="95232" />
|
||||||
|
<S d="96256" r="2" />
|
||||||
|
<S d="95232" />
|
||||||
|
<S d="96256" r="2" />
|
||||||
|
<S d="95232" />
|
||||||
|
<S d="96256" r="2" />
|
||||||
|
<S d="95232" />
|
||||||
|
<S d="96256" r="2" />
|
||||||
|
<S d="95232" />
|
||||||
|
<S d="96256" r="2" />
|
||||||
|
<S d="95232" />
|
||||||
|
<S d="96256" r="2" />
|
||||||
|
<S d="95232" />
|
||||||
|
<S d="96256" r="2" />
|
||||||
|
<S d="95232" />
|
||||||
|
<S d="96256" r="2" />
|
||||||
|
<S d="95232" />
|
||||||
|
<S d="96256" r="2" />
|
||||||
|
<S d="95232" />
|
||||||
|
<S d="96256" r="2" />
|
||||||
|
<S d="95232" />
|
||||||
|
<S d="96256" r="2" />
|
||||||
|
<S d="95232" />
|
||||||
|
<S d="96256" r="2" />
|
||||||
|
<S d="95232" />
|
||||||
|
<S d="96256" r="2" />
|
||||||
|
<S d="95232" />
|
||||||
|
<S d="96256" r="2" />
|
||||||
|
<S d="95232" />
|
||||||
|
<S d="96256" r="2" />
|
||||||
|
<S d="95232" />
|
||||||
|
<S d="96256" r="2" />
|
||||||
|
<S d="95232" />
|
||||||
|
<S d="96256" r="2" />
|
||||||
|
<S d="95232" />
|
||||||
|
<S d="96256" r="2" />
|
||||||
|
<S d="95232" />
|
||||||
|
<S d="96256" r="2" />
|
||||||
|
<S d="95232" />
|
||||||
|
<S d="96256" r="2" />
|
||||||
|
<S d="95232" />
|
||||||
|
<S d="96256" r="2" />
|
||||||
|
<S d="95232" />
|
||||||
|
<S d="96256" r="2" />
|
||||||
|
<S d="95232" />
|
||||||
|
<S d="96256" r="2" />
|
||||||
|
<S d="95232" />
|
||||||
|
<S d="96256" r="2" />
|
||||||
|
<S d="95232" />
|
||||||
|
<S d="96256" r="2" />
|
||||||
|
<S d="95232" />
|
||||||
|
<S d="96256" r="2" />
|
||||||
|
<S d="95232" />
|
||||||
|
<S d="96256" r="2" />
|
||||||
|
<S d="95232" />
|
||||||
|
<S d="96256" r="2" />
|
||||||
|
<S d="95232" />
|
||||||
|
<S d="96256" r="2" />
|
||||||
|
<S d="95232" />
|
||||||
|
<S d="96256" r="2" />
|
||||||
|
<S d="95232" />
|
||||||
|
<S d="96256" r="2" />
|
||||||
|
<S d="95232" />
|
||||||
|
<S d="96256" r="2" />
|
||||||
|
<S d="95232" />
|
||||||
|
<S d="96256" r="2" />
|
||||||
|
<S d="95232" />
|
||||||
|
<S d="96256" r="2" />
|
||||||
|
<S d="95232" />
|
||||||
|
<S d="96256" r="2" />
|
||||||
|
<S d="95232" />
|
||||||
|
<S d="96256" r="2" />
|
||||||
|
<S d="95232" />
|
||||||
|
<S d="96256" r="2" />
|
||||||
|
<S d="95232" />
|
||||||
|
<S d="96256" r="2" />
|
||||||
|
<S d="95232" />
|
||||||
|
<S d="96256" r="2" />
|
||||||
|
<S d="95232" />
|
||||||
|
<S d="96256" r="2" />
|
||||||
|
<S d="95232" />
|
||||||
|
<S d="96256" r="2" />
|
||||||
|
<S d="95232" />
|
||||||
|
<S d="96256" r="2" />
|
||||||
|
<S d="95232" />
|
||||||
|
<S d="96256" r="2" />
|
||||||
|
<S d="95232" />
|
||||||
|
<S d="96256" r="2" />
|
||||||
|
<S d="95232" />
|
||||||
|
<S d="96256" r="2" />
|
||||||
|
<S d="95232" />
|
||||||
|
<S d="96256" r="2" />
|
||||||
|
<S d="95232" />
|
||||||
|
<S d="96256" r="2" />
|
||||||
|
<S d="95232" />
|
||||||
|
<S d="96256" r="2" />
|
||||||
|
<S d="95232" />
|
||||||
|
<S d="96256" r="2" />
|
||||||
|
<S d="95232" />
|
||||||
|
<S d="96256" r="2" />
|
||||||
|
<S d="95232" />
|
||||||
|
<S d="96256" r="2" />
|
||||||
|
<S d="95232" />
|
||||||
|
<S d="96256" r="2" />
|
||||||
|
<S d="95232" />
|
||||||
|
<S d="96256" r="2" />
|
||||||
|
<S d="95232" />
|
||||||
|
<S d="96256" r="2" />
|
||||||
|
<S d="95232" />
|
||||||
|
<S d="96256" r="2" />
|
||||||
|
<S d="95232" />
|
||||||
|
<S d="96256" r="2" />
|
||||||
|
<S d="95232" />
|
||||||
|
<S d="96256" r="2" />
|
||||||
|
<S d="95232" />
|
||||||
|
<S d="96256" r="2" />
|
||||||
|
<S d="95232" />
|
||||||
|
<S d="96256" r="2" />
|
||||||
|
<S d="95232" />
|
||||||
|
<S d="96256" r="2" />
|
||||||
|
<S d="95232" />
|
||||||
|
<S d="96256" r="2" />
|
||||||
|
<S d="95232" />
|
||||||
|
<S d="96256" r="2" />
|
||||||
|
<S d="95232" />
|
||||||
|
<S d="96256" r="2" />
|
||||||
|
<S d="95232" />
|
||||||
|
<S d="96256" r="2" />
|
||||||
|
<S d="95232" />
|
||||||
|
<S d="96256" r="2" />
|
||||||
|
<S d="95232" />
|
||||||
|
<S d="96256" r="2" />
|
||||||
|
<S d="95232" />
|
||||||
|
<S d="96256" r="2" />
|
||||||
|
<S d="95232" />
|
||||||
|
<S d="96256" r="2" />
|
||||||
|
<S d="95232" />
|
||||||
|
<S d="96256" r="2" />
|
||||||
|
<S d="95232" />
|
||||||
|
<S d="96256" r="2" />
|
||||||
|
<S d="95232" />
|
||||||
|
<S d="96256" r="2" />
|
||||||
|
<S d="95232" />
|
||||||
|
<S d="96256" r="2" />
|
||||||
|
<S d="95232" />
|
||||||
|
<S d="96256" r="2" />
|
||||||
|
<S d="95232" />
|
||||||
|
<S d="96256" r="2" />
|
||||||
|
<S d="95232" />
|
||||||
|
<S d="96256" r="2" />
|
||||||
|
<S d="95232" />
|
||||||
|
<S d="96256" r="2" />
|
||||||
|
<S d="95232" />
|
||||||
|
<S d="96256" r="2" />
|
||||||
|
<S d="95232" />
|
||||||
|
<S d="96256" r="2" />
|
||||||
|
<S d="95232" />
|
||||||
|
<S d="96256" r="2" />
|
||||||
|
<S d="95232" />
|
||||||
|
<S d="96256" r="2" />
|
||||||
|
<S d="95232" />
|
||||||
|
<S d="96256" r="2" />
|
||||||
|
<S d="95232" />
|
||||||
|
<S d="96256" r="2" />
|
||||||
|
<S d="95232" />
|
||||||
|
<S d="96256" r="2" />
|
||||||
|
<S d="95232" />
|
||||||
|
<S d="96256" r="2" />
|
||||||
|
<S d="95232" />
|
||||||
|
<S d="96256" r="2" />
|
||||||
|
<S d="95232" />
|
||||||
|
<S d="96256" r="2" />
|
||||||
|
<S d="95232" />
|
||||||
|
<S d="96256" r="2" />
|
||||||
|
<S d="95232" />
|
||||||
|
<S d="96256" r="2" />
|
||||||
|
<S d="95232" />
|
||||||
|
<S d="96256" r="2" />
|
||||||
|
<S d="95232" />
|
||||||
|
<S d="96256" r="2" />
|
||||||
|
<S d="95232" />
|
||||||
|
<S d="96256" r="2" />
|
||||||
|
<S d="95232" />
|
||||||
|
<S d="96256" r="2" />
|
||||||
|
<S d="95232" />
|
||||||
|
<S d="96256" r="2" />
|
||||||
|
<S d="95232" />
|
||||||
|
<S d="96256" r="2" />
|
||||||
|
<S d="95232" />
|
||||||
|
<S d="96256" r="2" />
|
||||||
|
<S d="95232" />
|
||||||
|
<S d="96256" r="2" />
|
||||||
|
<S d="95232" />
|
||||||
|
<S d="96256" r="2" />
|
||||||
|
<S d="95232" />
|
||||||
|
<S d="96256" r="2" />
|
||||||
|
<S d="95232" />
|
||||||
|
<S d="3584" />
|
||||||
|
</SegmentTimeline>
|
||||||
|
</SegmentTemplate>
|
||||||
|
<Representation
|
||||||
|
id="audio=128001"
|
||||||
|
bandwidth="128001">
|
||||||
|
</Representation>
|
||||||
|
</AdaptationSet>
|
||||||
|
<AdaptationSet
|
||||||
|
id="2"
|
||||||
|
group="3"
|
||||||
|
contentType="text"
|
||||||
|
lang="en"
|
||||||
|
mimeType="application/mp4"
|
||||||
|
codecs="stpp"
|
||||||
|
startWithSAP="1">
|
||||||
|
<Role schemeIdUri="urn:mpeg:dash:role:2011" value="subtitle" />
|
||||||
|
<SegmentTemplate
|
||||||
|
timescale="1000"
|
||||||
|
initialization="3144-kZT4LWMQw6Rh7Kpd-$RepresentationID$.dash"
|
||||||
|
media="3144-kZT4LWMQw6Rh7Kpd-$RepresentationID$-$Time$.dash">
|
||||||
|
<SegmentTimeline>
|
||||||
|
<S t="0" d="60000" r="9" />
|
||||||
|
<S d="24000" />
|
||||||
|
</SegmentTimeline>
|
||||||
|
</SegmentTemplate>
|
||||||
|
<Representation
|
||||||
|
id="textstream_eng=1000"
|
||||||
|
bandwidth="1000">
|
||||||
|
</Representation>
|
||||||
|
</AdaptationSet>
|
||||||
|
<AdaptationSet
|
||||||
|
id="3"
|
||||||
|
group="2"
|
||||||
|
contentType="video"
|
||||||
|
par="960:409"
|
||||||
|
minBandwidth="100000"
|
||||||
|
maxBandwidth="4482000"
|
||||||
|
maxWidth="1689"
|
||||||
|
maxHeight="720"
|
||||||
|
segmentAlignment="true"
|
||||||
|
mimeType="video/mp4"
|
||||||
|
codecs="avc1.4D401F"
|
||||||
|
startWithSAP="1">
|
||||||
|
<Role schemeIdUri="urn:mpeg:dash:role:2011" value="main" />
|
||||||
|
<SegmentTemplate
|
||||||
|
timescale="12288"
|
||||||
|
initialization="3144-kZT4LWMQw6Rh7Kpd-$RepresentationID$.dash"
|
||||||
|
media="3144-kZT4LWMQw6Rh7Kpd-$RepresentationID$-$Time$.dash">
|
||||||
|
<SegmentTimeline>
|
||||||
|
<S t="0" d="24576" r="443" />
|
||||||
|
</SegmentTimeline>
|
||||||
|
</SegmentTemplate>
|
||||||
|
<Representation
|
||||||
|
id="video=100000"
|
||||||
|
bandwidth="100000"
|
||||||
|
width="336"
|
||||||
|
height="144"
|
||||||
|
sar="2880:2863"
|
||||||
|
scanType="progressive">
|
||||||
|
</Representation>
|
||||||
|
<Representation
|
||||||
|
id="video=326000"
|
||||||
|
bandwidth="326000"
|
||||||
|
width="562"
|
||||||
|
height="240"
|
||||||
|
sar="115200:114929"
|
||||||
|
scanType="progressive">
|
||||||
|
</Representation>
|
||||||
|
<Representation
|
||||||
|
id="video=698000"
|
||||||
|
bandwidth="698000"
|
||||||
|
width="844"
|
||||||
|
height="360"
|
||||||
|
sar="86400:86299"
|
||||||
|
scanType="progressive">
|
||||||
|
</Representation>
|
||||||
|
<Representation
|
||||||
|
id="video=1493000"
|
||||||
|
bandwidth="1493000"
|
||||||
|
width="1126"
|
||||||
|
height="480"
|
||||||
|
sar="230400:230267"
|
||||||
|
scanType="progressive">
|
||||||
|
</Representation>
|
||||||
|
<Representation
|
||||||
|
id="video=4482000"
|
||||||
|
bandwidth="4482000"
|
||||||
|
width="1688"
|
||||||
|
height="720"
|
||||||
|
sar="86400:86299"
|
||||||
|
scanType="progressive">
|
||||||
|
</Representation>
|
||||||
|
</AdaptationSet>
|
||||||
|
</Period>
|
||||||
|
</MPD>
|
32
test/testdata/mpd/url_and_range.mpd
vendored
Normal file
32
test/testdata/mpd/url_and_range.mpd
vendored
Normal file
|
@ -0,0 +1,32 @@
|
||||||
|
<?xml version="1.0" ?>
|
||||||
|
<MPD xmlns="urn:mpeg:dash:schema:mpd:2011" profiles="urn:mpeg:dash:profile:isoff-live:2011" minBufferTime="PT10.01S" mediaPresentationDuration="PT30.097S" type="static">
|
||||||
|
<!-- Created with Bento4 mp4-dash.py, VERSION=2.0.0-639 -->
|
||||||
|
<Period>
|
||||||
|
<!-- Video -->
|
||||||
|
<AdaptationSet mimeType="video/mp4" segmentAlignment="true" startWithSAP="1" maxWidth="768" maxHeight="432">
|
||||||
|
<Representation id="video-avc1" codecs="avc1.4D401E" width="768" height="432" scanType="progressive" frameRate="30000/1001" bandwidth="699597">
|
||||||
|
<SegmentList timescale="1000" duration="10010">
|
||||||
|
<Initialization sourceURL="video-frag.mp4" range="36-746"/>
|
||||||
|
<SegmentURL media="video-frag.mp4" mediaRange="747-876117"/>
|
||||||
|
<SegmentURL media="video-frag.mp4" mediaRange="876118-1466913"/>
|
||||||
|
<SegmentURL media="video-frag.mp4" mediaRange="1466914-1953954"/>
|
||||||
|
<SegmentURL media="video-frag.mp4" mediaRange="1953955-1994652"/>
|
||||||
|
</SegmentList>
|
||||||
|
</Representation>
|
||||||
|
</AdaptationSet>
|
||||||
|
<!-- Audio -->
|
||||||
|
<AdaptationSet mimeType="audio/mp4" startWithSAP="1" segmentAlignment="true">
|
||||||
|
<Representation id="audio-und-mp4a.40.2" codecs="mp4a.40.2" bandwidth="98808" audioSamplingRate="48000">
|
||||||
|
<AudioChannelConfiguration schemeIdUri="urn:mpeg:mpegB:cicp:ChannelConfiguration" value="2"/>
|
||||||
|
<SegmentList timescale="1000" duration="10010">
|
||||||
|
<Initialization sourceURL="audio-frag.mp4" range="32-623"/>
|
||||||
|
<SegmentURL media="audio-frag.mp4" mediaRange="624-124199"/>
|
||||||
|
<SegmentURL media="audio-frag.mp4" mediaRange="124200-250303"/>
|
||||||
|
<SegmentURL media="audio-frag.mp4" mediaRange="250304-374365"/>
|
||||||
|
<SegmentURL media="audio-frag.mp4" mediaRange="374366-374836"/>
|
||||||
|
</SegmentList>
|
||||||
|
</Representation>
|
||||||
|
</AdaptationSet>
|
||||||
|
</Period>
|
||||||
|
</MPD>
|
||||||
|
|
File diff suppressed because it is too large
Load diff
|
@ -5,7 +5,6 @@ from __future__ import unicode_literals
|
||||||
|
|
||||||
__license__ = 'Public Domain'
|
__license__ = 'Public Domain'
|
||||||
|
|
||||||
import codecs
|
|
||||||
import io
|
import io
|
||||||
import os
|
import os
|
||||||
import random
|
import random
|
||||||
|
@ -17,10 +16,12 @@ from .options import (
|
||||||
)
|
)
|
||||||
from .compat import (
|
from .compat import (
|
||||||
compat_getpass,
|
compat_getpass,
|
||||||
|
compat_register_utf8,
|
||||||
compat_shlex_split,
|
compat_shlex_split,
|
||||||
workaround_optparse_bug9161,
|
workaround_optparse_bug9161,
|
||||||
)
|
)
|
||||||
from .utils import (
|
from .utils import (
|
||||||
|
_UnsafeExtensionError,
|
||||||
DateRange,
|
DateRange,
|
||||||
decodeOption,
|
decodeOption,
|
||||||
DEFAULT_OUTTMPL,
|
DEFAULT_OUTTMPL,
|
||||||
|
@ -46,10 +47,8 @@ from .YoutubeDL import YoutubeDL
|
||||||
|
|
||||||
|
|
||||||
def _real_main(argv=None):
|
def _real_main(argv=None):
|
||||||
# Compatibility fixes for Windows
|
# Compatibility fix for Windows
|
||||||
if sys.platform == 'win32':
|
compat_register_utf8()
|
||||||
# https://github.com/ytdl-org/youtube-dl/issues/820
|
|
||||||
codecs.register(lambda name: codecs.lookup('utf-8') if name == 'cp65001' else None)
|
|
||||||
|
|
||||||
workaround_optparse_bug9161()
|
workaround_optparse_bug9161()
|
||||||
|
|
||||||
|
@ -175,6 +174,9 @@ def _real_main(argv=None):
|
||||||
if opts.ap_mso and opts.ap_mso not in MSO_INFO:
|
if opts.ap_mso and opts.ap_mso not in MSO_INFO:
|
||||||
parser.error('Unsupported TV Provider, use --ap-list-mso to get a list of supported TV Providers')
|
parser.error('Unsupported TV Provider, use --ap-list-mso to get a list of supported TV Providers')
|
||||||
|
|
||||||
|
if opts.no_check_extensions:
|
||||||
|
_UnsafeExtensionError.lenient = True
|
||||||
|
|
||||||
def parse_retries(retries):
|
def parse_retries(retries):
|
||||||
if retries in ('inf', 'infinite'):
|
if retries in ('inf', 'infinite'):
|
||||||
parsed_retries = float('inf')
|
parsed_retries = float('inf')
|
||||||
|
@ -340,6 +342,7 @@ def _real_main(argv=None):
|
||||||
'format': opts.format,
|
'format': opts.format,
|
||||||
'listformats': opts.listformats,
|
'listformats': opts.listformats,
|
||||||
'outtmpl': outtmpl,
|
'outtmpl': outtmpl,
|
||||||
|
'outtmpl_na_placeholder': opts.outtmpl_na_placeholder,
|
||||||
'autonumber_size': opts.autonumber_size,
|
'autonumber_size': opts.autonumber_size,
|
||||||
'autonumber_start': opts.autonumber_start,
|
'autonumber_start': opts.autonumber_start,
|
||||||
'restrictfilenames': opts.restrictfilenames,
|
'restrictfilenames': opts.restrictfilenames,
|
||||||
|
|
|
@ -8,6 +8,18 @@ from .utils import bytes_to_intlist, intlist_to_bytes
|
||||||
BLOCK_SIZE_BYTES = 16
|
BLOCK_SIZE_BYTES = 16
|
||||||
|
|
||||||
|
|
||||||
|
def pkcs7_padding(data):
|
||||||
|
"""
|
||||||
|
PKCS#7 padding
|
||||||
|
|
||||||
|
@param {int[]} data cleartext
|
||||||
|
@returns {int[]} padding data
|
||||||
|
"""
|
||||||
|
|
||||||
|
remaining_length = BLOCK_SIZE_BYTES - len(data) % BLOCK_SIZE_BYTES
|
||||||
|
return data + [remaining_length] * remaining_length
|
||||||
|
|
||||||
|
|
||||||
def aes_ctr_decrypt(data, key, counter):
|
def aes_ctr_decrypt(data, key, counter):
|
||||||
"""
|
"""
|
||||||
Decrypt with aes in counter mode
|
Decrypt with aes in counter mode
|
||||||
|
@ -76,8 +88,7 @@ def aes_cbc_encrypt(data, key, iv):
|
||||||
previous_cipher_block = iv
|
previous_cipher_block = iv
|
||||||
for i in range(block_count):
|
for i in range(block_count):
|
||||||
block = data[i * BLOCK_SIZE_BYTES: (i + 1) * BLOCK_SIZE_BYTES]
|
block = data[i * BLOCK_SIZE_BYTES: (i + 1) * BLOCK_SIZE_BYTES]
|
||||||
remaining_length = BLOCK_SIZE_BYTES - len(block)
|
block = pkcs7_padding(block)
|
||||||
block += [remaining_length] * remaining_length
|
|
||||||
mixed_block = xor(block, previous_cipher_block)
|
mixed_block = xor(block, previous_cipher_block)
|
||||||
|
|
||||||
encrypted_block = aes_encrypt(mixed_block, expanded_key)
|
encrypted_block = aes_encrypt(mixed_block, expanded_key)
|
||||||
|
@ -88,6 +99,28 @@ def aes_cbc_encrypt(data, key, iv):
|
||||||
return encrypted_data
|
return encrypted_data
|
||||||
|
|
||||||
|
|
||||||
|
def aes_ecb_encrypt(data, key):
|
||||||
|
"""
|
||||||
|
Encrypt with aes in ECB mode. Using PKCS#7 padding
|
||||||
|
|
||||||
|
@param {int[]} data cleartext
|
||||||
|
@param {int[]} key 16/24/32-Byte cipher key
|
||||||
|
@returns {int[]} encrypted data
|
||||||
|
"""
|
||||||
|
expanded_key = key_expansion(key)
|
||||||
|
block_count = int(ceil(float(len(data)) / BLOCK_SIZE_BYTES))
|
||||||
|
|
||||||
|
encrypted_data = []
|
||||||
|
for i in range(block_count):
|
||||||
|
block = data[i * BLOCK_SIZE_BYTES: (i + 1) * BLOCK_SIZE_BYTES]
|
||||||
|
block = pkcs7_padding(block)
|
||||||
|
|
||||||
|
encrypted_block = aes_encrypt(block, expanded_key)
|
||||||
|
encrypted_data += encrypted_block
|
||||||
|
|
||||||
|
return encrypted_data
|
||||||
|
|
||||||
|
|
||||||
def key_expansion(data):
|
def key_expansion(data):
|
||||||
"""
|
"""
|
||||||
Generate key schedule
|
Generate key schedule
|
||||||
|
@ -303,7 +336,7 @@ def xor(data1, data2):
|
||||||
|
|
||||||
|
|
||||||
def rijndael_mul(a, b):
|
def rijndael_mul(a, b):
|
||||||
if(a == 0 or b == 0):
|
if (a == 0 or b == 0):
|
||||||
return 0
|
return 0
|
||||||
return RIJNDAEL_EXP_TABLE[(RIJNDAEL_LOG_TABLE[a] + RIJNDAEL_LOG_TABLE[b]) % 0xFF]
|
return RIJNDAEL_EXP_TABLE[(RIJNDAEL_LOG_TABLE[a] + RIJNDAEL_LOG_TABLE[b]) % 0xFF]
|
||||||
|
|
||||||
|
|
|
@ -1,21 +1,32 @@
|
||||||
from __future__ import unicode_literals
|
from __future__ import unicode_literals
|
||||||
|
|
||||||
import errno
|
import errno
|
||||||
import io
|
|
||||||
import json
|
import json
|
||||||
import os
|
import os
|
||||||
import re
|
import re
|
||||||
import shutil
|
import shutil
|
||||||
import traceback
|
import traceback
|
||||||
|
|
||||||
from .compat import compat_getenv
|
from .compat import (
|
||||||
|
compat_getenv,
|
||||||
|
compat_open as open,
|
||||||
|
)
|
||||||
from .utils import (
|
from .utils import (
|
||||||
|
error_to_compat_str,
|
||||||
expand_path,
|
expand_path,
|
||||||
|
is_outdated_version,
|
||||||
|
try_get,
|
||||||
write_json_file,
|
write_json_file,
|
||||||
)
|
)
|
||||||
|
from .version import __version__
|
||||||
|
|
||||||
|
|
||||||
class Cache(object):
|
class Cache(object):
|
||||||
|
|
||||||
|
_YTDL_DIR = 'youtube-dl'
|
||||||
|
_VERSION_KEY = _YTDL_DIR + '_version'
|
||||||
|
_DEFAULT_VERSION = '2021.12.17'
|
||||||
|
|
||||||
def __init__(self, ydl):
|
def __init__(self, ydl):
|
||||||
self._ydl = ydl
|
self._ydl = ydl
|
||||||
|
|
||||||
|
@ -23,7 +34,7 @@ class Cache(object):
|
||||||
res = self._ydl.params.get('cachedir')
|
res = self._ydl.params.get('cachedir')
|
||||||
if res is None:
|
if res is None:
|
||||||
cache_root = compat_getenv('XDG_CACHE_HOME', '~/.cache')
|
cache_root = compat_getenv('XDG_CACHE_HOME', '~/.cache')
|
||||||
res = os.path.join(cache_root, 'youtube-dl')
|
res = os.path.join(cache_root, self._YTDL_DIR)
|
||||||
return expand_path(res)
|
return expand_path(res)
|
||||||
|
|
||||||
def _get_cache_fn(self, section, key, dtype):
|
def _get_cache_fn(self, section, key, dtype):
|
||||||
|
@ -50,13 +61,22 @@ class Cache(object):
|
||||||
except OSError as ose:
|
except OSError as ose:
|
||||||
if ose.errno != errno.EEXIST:
|
if ose.errno != errno.EEXIST:
|
||||||
raise
|
raise
|
||||||
write_json_file(data, fn)
|
write_json_file({self._VERSION_KEY: __version__, 'data': data}, fn)
|
||||||
except Exception:
|
except Exception:
|
||||||
tb = traceback.format_exc()
|
tb = traceback.format_exc()
|
||||||
self._ydl.report_warning(
|
self._ydl.report_warning(
|
||||||
'Writing cache to %r failed: %s' % (fn, tb))
|
'Writing cache to %r failed: %s' % (fn, tb))
|
||||||
|
|
||||||
def load(self, section, key, dtype='json', default=None):
|
def _validate(self, data, min_ver):
|
||||||
|
version = try_get(data, lambda x: x[self._VERSION_KEY])
|
||||||
|
if not version: # Backward compatibility
|
||||||
|
data, version = {'data': data}, self._DEFAULT_VERSION
|
||||||
|
if not is_outdated_version(version, min_ver or '0', assume_new=False):
|
||||||
|
return data['data']
|
||||||
|
self._ydl.to_screen(
|
||||||
|
'Discarding old cache from version {version} (needs {min_ver})'.format(**locals()))
|
||||||
|
|
||||||
|
def load(self, section, key, dtype='json', default=None, min_ver=None):
|
||||||
assert dtype in ('json',)
|
assert dtype in ('json',)
|
||||||
|
|
||||||
if not self.enabled:
|
if not self.enabled:
|
||||||
|
@ -65,13 +85,13 @@ class Cache(object):
|
||||||
cache_fn = self._get_cache_fn(section, key, dtype)
|
cache_fn = self._get_cache_fn(section, key, dtype)
|
||||||
try:
|
try:
|
||||||
try:
|
try:
|
||||||
with io.open(cache_fn, 'r', encoding='utf-8') as cachef:
|
with open(cache_fn, 'r', encoding='utf-8') as cachef:
|
||||||
return json.load(cachef)
|
return self._validate(json.load(cachef), min_ver)
|
||||||
except ValueError:
|
except ValueError:
|
||||||
try:
|
try:
|
||||||
file_size = os.path.getsize(cache_fn)
|
file_size = os.path.getsize(cache_fn)
|
||||||
except (OSError, IOError) as oe:
|
except (OSError, IOError) as oe:
|
||||||
file_size = str(oe)
|
file_size = error_to_compat_str(oe)
|
||||||
self._ydl.report_warning(
|
self._ydl.report_warning(
|
||||||
'Cache retrieval from %s failed (%s)' % (cache_fn, file_size))
|
'Cache retrieval from %s failed (%s)' % (cache_fn, file_size))
|
||||||
except IOError:
|
except IOError:
|
||||||
|
|
1667
youtube_dl/casefold.py
Normal file
1667
youtube_dl/casefold.py
Normal file
File diff suppressed because it is too large
Load diff
File diff suppressed because it is too large
Load diff
|
@ -1,22 +1,31 @@
|
||||||
from __future__ import unicode_literals
|
from __future__ import unicode_literals
|
||||||
|
|
||||||
|
from ..utils import (
|
||||||
|
determine_protocol,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def get_suitable_downloader(info_dict, params={}):
|
||||||
|
info_dict['protocol'] = determine_protocol(info_dict)
|
||||||
|
info_copy = info_dict.copy()
|
||||||
|
return _get_suitable_downloader(info_copy, params)
|
||||||
|
|
||||||
|
|
||||||
|
# Some of these require get_suitable_downloader
|
||||||
from .common import FileDownloader
|
from .common import FileDownloader
|
||||||
|
from .dash import DashSegmentsFD
|
||||||
from .f4m import F4mFD
|
from .f4m import F4mFD
|
||||||
from .hls import HlsFD
|
from .hls import HlsFD
|
||||||
from .http import HttpFD
|
from .http import HttpFD
|
||||||
from .rtmp import RtmpFD
|
from .rtmp import RtmpFD
|
||||||
from .dash import DashSegmentsFD
|
|
||||||
from .rtsp import RtspFD
|
from .rtsp import RtspFD
|
||||||
from .ism import IsmFD
|
from .ism import IsmFD
|
||||||
|
from .niconico import NiconicoDmcFD
|
||||||
from .external import (
|
from .external import (
|
||||||
get_external_downloader,
|
get_external_downloader,
|
||||||
FFmpegFD,
|
FFmpegFD,
|
||||||
)
|
)
|
||||||
|
|
||||||
from ..utils import (
|
|
||||||
determine_protocol,
|
|
||||||
)
|
|
||||||
|
|
||||||
PROTOCOL_MAP = {
|
PROTOCOL_MAP = {
|
||||||
'rtmp': RtmpFD,
|
'rtmp': RtmpFD,
|
||||||
'm3u8_native': HlsFD,
|
'm3u8_native': HlsFD,
|
||||||
|
@ -26,13 +35,12 @@ PROTOCOL_MAP = {
|
||||||
'f4m': F4mFD,
|
'f4m': F4mFD,
|
||||||
'http_dash_segments': DashSegmentsFD,
|
'http_dash_segments': DashSegmentsFD,
|
||||||
'ism': IsmFD,
|
'ism': IsmFD,
|
||||||
|
'niconico_dmc': NiconicoDmcFD,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
def get_suitable_downloader(info_dict, params={}):
|
def _get_suitable_downloader(info_dict, params={}):
|
||||||
"""Get the downloader class that can handle the info dict."""
|
"""Get the downloader class that can handle the info dict."""
|
||||||
protocol = determine_protocol(info_dict)
|
|
||||||
info_dict['protocol'] = protocol
|
|
||||||
|
|
||||||
# if (info_dict.get('start_time') or info_dict.get('end_time')) and not info_dict.get('requested_formats') and FFmpegFD.can_download(info_dict):
|
# if (info_dict.get('start_time') or info_dict.get('end_time')) and not info_dict.get('requested_formats') and FFmpegFD.can_download(info_dict):
|
||||||
# return FFmpegFD
|
# return FFmpegFD
|
||||||
|
@ -42,7 +50,11 @@ def get_suitable_downloader(info_dict, params={}):
|
||||||
ed = get_external_downloader(external_downloader)
|
ed = get_external_downloader(external_downloader)
|
||||||
if ed.can_download(info_dict):
|
if ed.can_download(info_dict):
|
||||||
return ed
|
return ed
|
||||||
|
# Avoid using unwanted args since external_downloader was rejected
|
||||||
|
if params.get('external_downloader_args'):
|
||||||
|
params['external_downloader_args'] = None
|
||||||
|
|
||||||
|
protocol = info_dict['protocol']
|
||||||
if protocol.startswith('m3u8') and info_dict.get('is_live'):
|
if protocol.startswith('m3u8') and info_dict.get('is_live'):
|
||||||
return FFmpegFD
|
return FFmpegFD
|
||||||
|
|
||||||
|
|
|
@ -88,17 +88,21 @@ class FileDownloader(object):
|
||||||
return '---.-%'
|
return '---.-%'
|
||||||
return '%6s' % ('%3.1f%%' % percent)
|
return '%6s' % ('%3.1f%%' % percent)
|
||||||
|
|
||||||
@staticmethod
|
@classmethod
|
||||||
def calc_eta(start, now, total, current):
|
def calc_eta(cls, start_or_rate, now_or_remaining, *args):
|
||||||
|
if len(args) < 2:
|
||||||
|
rate, remaining = (start_or_rate, now_or_remaining)
|
||||||
|
if None in (rate, remaining):
|
||||||
|
return None
|
||||||
|
return int(float(remaining) / rate)
|
||||||
|
start, now = (start_or_rate, now_or_remaining)
|
||||||
|
total, current = args[:2]
|
||||||
if total is None:
|
if total is None:
|
||||||
return None
|
return None
|
||||||
if now is None:
|
if now is None:
|
||||||
now = time.time()
|
now = time.time()
|
||||||
dif = now - start
|
rate = cls.calc_speed(start, now, current)
|
||||||
if current == 0 or dif < 0.001: # One millisecond
|
return rate and int((float(total) - float(current)) / rate)
|
||||||
return None
|
|
||||||
rate = float(current) / dif
|
|
||||||
return int((float(total) - float(current)) / rate)
|
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def format_eta(eta):
|
def format_eta(eta):
|
||||||
|
@ -123,6 +127,12 @@ class FileDownloader(object):
|
||||||
def format_retries(retries):
|
def format_retries(retries):
|
||||||
return 'inf' if retries == float('inf') else '%.0f' % retries
|
return 'inf' if retries == float('inf') else '%.0f' % retries
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def filesize_or_none(unencoded_filename):
|
||||||
|
fn = encodeFilename(unencoded_filename)
|
||||||
|
if os.path.isfile(fn):
|
||||||
|
return os.path.getsize(fn)
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def best_block_size(elapsed_time, bytes):
|
def best_block_size(elapsed_time, bytes):
|
||||||
new_min = max(bytes / 2.0, 1.0)
|
new_min = max(bytes / 2.0, 1.0)
|
||||||
|
@ -329,6 +339,10 @@ class FileDownloader(object):
|
||||||
def download(self, filename, info_dict):
|
def download(self, filename, info_dict):
|
||||||
"""Download to a filename using the info from info_dict
|
"""Download to a filename using the info from info_dict
|
||||||
Return True on success and False otherwise
|
Return True on success and False otherwise
|
||||||
|
|
||||||
|
This method filters the `Cookie` header from the info_dict to prevent leaks.
|
||||||
|
Downloaders have their own way of handling cookies.
|
||||||
|
See: https://github.com/yt-dlp/yt-dlp/security/advisories/GHSA-v8mc-9377-rwjj
|
||||||
"""
|
"""
|
||||||
|
|
||||||
nooverwrites_and_exists = (
|
nooverwrites_and_exists = (
|
||||||
|
|
|
@ -1,5 +1,7 @@
|
||||||
from __future__ import unicode_literals
|
from __future__ import unicode_literals
|
||||||
|
|
||||||
|
import itertools
|
||||||
|
|
||||||
from .fragment import FragmentFD
|
from .fragment import FragmentFD
|
||||||
from ..compat import compat_urllib_error
|
from ..compat import compat_urllib_error
|
||||||
from ..utils import (
|
from ..utils import (
|
||||||
|
@ -30,26 +32,28 @@ class DashSegmentsFD(FragmentFD):
|
||||||
fragment_retries = self.params.get('fragment_retries', 0)
|
fragment_retries = self.params.get('fragment_retries', 0)
|
||||||
skip_unavailable_fragments = self.params.get('skip_unavailable_fragments', True)
|
skip_unavailable_fragments = self.params.get('skip_unavailable_fragments', True)
|
||||||
|
|
||||||
frag_index = 0
|
for frag_index, fragment in enumerate(fragments, 1):
|
||||||
for i, fragment in enumerate(fragments):
|
|
||||||
frag_index += 1
|
|
||||||
if frag_index <= ctx['fragment_index']:
|
if frag_index <= ctx['fragment_index']:
|
||||||
continue
|
continue
|
||||||
|
success = False
|
||||||
# In DASH, the first segment contains necessary headers to
|
# In DASH, the first segment contains necessary headers to
|
||||||
# generate a valid MP4 file, so always abort for the first segment
|
# generate a valid MP4 file, so always abort for the first segment
|
||||||
fatal = i == 0 or not skip_unavailable_fragments
|
fatal = frag_index == 1 or not skip_unavailable_fragments
|
||||||
count = 0
|
fragment_url = fragment.get('url')
|
||||||
while count <= fragment_retries:
|
if not fragment_url:
|
||||||
|
assert fragment_base_url
|
||||||
|
fragment_url = urljoin(fragment_base_url, fragment['path'])
|
||||||
|
headers = info_dict.get('http_headers')
|
||||||
|
fragment_range = fragment.get('range')
|
||||||
|
if fragment_range:
|
||||||
|
headers = headers.copy() if headers else {}
|
||||||
|
headers['Range'] = 'bytes=%s' % (fragment_range,)
|
||||||
|
for count in itertools.count():
|
||||||
try:
|
try:
|
||||||
fragment_url = fragment.get('url')
|
success, frag_content = self._download_fragment(ctx, fragment_url, info_dict, headers)
|
||||||
if not fragment_url:
|
|
||||||
assert fragment_base_url
|
|
||||||
fragment_url = urljoin(fragment_base_url, fragment['path'])
|
|
||||||
success, frag_content = self._download_fragment(ctx, fragment_url, info_dict)
|
|
||||||
if not success:
|
if not success:
|
||||||
return False
|
return False
|
||||||
self._append_fragment(ctx, frag_content)
|
self._append_fragment(ctx, frag_content)
|
||||||
break
|
|
||||||
except compat_urllib_error.HTTPError as err:
|
except compat_urllib_error.HTTPError as err:
|
||||||
# YouTube may often return 404 HTTP error for a fragment causing the
|
# YouTube may often return 404 HTTP error for a fragment causing the
|
||||||
# whole download to fail. However if the same fragment is immediately
|
# whole download to fail. However if the same fragment is immediately
|
||||||
|
@ -57,22 +61,21 @@ class DashSegmentsFD(FragmentFD):
|
||||||
# is usually enough) thus allowing to download the whole file successfully.
|
# is usually enough) thus allowing to download the whole file successfully.
|
||||||
# To be future-proof we will retry all fragments that fail with any
|
# To be future-proof we will retry all fragments that fail with any
|
||||||
# HTTP error.
|
# HTTP error.
|
||||||
count += 1
|
if count < fragment_retries:
|
||||||
if count <= fragment_retries:
|
self.report_retry_fragment(err, frag_index, count + 1, fragment_retries)
|
||||||
self.report_retry_fragment(err, frag_index, count, fragment_retries)
|
continue
|
||||||
except DownloadError:
|
except DownloadError:
|
||||||
# Don't retry fragment if error occurred during HTTP downloading
|
# Don't retry fragment if error occurred during HTTP downloading
|
||||||
# itself since it has own retry settings
|
# itself since it has its own retry settings
|
||||||
if not fatal:
|
if fatal:
|
||||||
self.report_skip_fragment(frag_index)
|
raise
|
||||||
break
|
break
|
||||||
raise
|
|
||||||
|
|
||||||
if count > fragment_retries:
|
if not success:
|
||||||
if not fatal:
|
if not fatal:
|
||||||
self.report_skip_fragment(frag_index)
|
self.report_skip_fragment(frag_index)
|
||||||
continue
|
continue
|
||||||
self.report_error('giving up after %s fragment retries' % fragment_retries)
|
self.report_error('giving up after %s fragment retries' % count)
|
||||||
return False
|
return False
|
||||||
|
|
||||||
self._finish_frag_download(ctx)
|
self._finish_frag_download(ctx)
|
||||||
|
|
|
@ -1,17 +1,24 @@
|
||||||
from __future__ import unicode_literals
|
from __future__ import unicode_literals
|
||||||
|
|
||||||
import os.path
|
import os
|
||||||
import re
|
import re
|
||||||
import subprocess
|
import subprocess
|
||||||
import sys
|
import sys
|
||||||
|
import tempfile
|
||||||
import time
|
import time
|
||||||
|
|
||||||
from .common import FileDownloader
|
from .common import FileDownloader
|
||||||
from ..compat import (
|
from ..compat import (
|
||||||
compat_setenv,
|
compat_setenv,
|
||||||
compat_str,
|
compat_str,
|
||||||
|
compat_subprocess_Popen,
|
||||||
)
|
)
|
||||||
from ..postprocessor.ffmpeg import FFmpegPostProcessor, EXT_TO_OUT_FORMATS
|
|
||||||
|
try:
|
||||||
|
from ..postprocessor.ffmpeg import FFmpegPostProcessor, EXT_TO_OUT_FORMATS
|
||||||
|
except ImportError:
|
||||||
|
FFmpegPostProcessor = None
|
||||||
|
|
||||||
from ..utils import (
|
from ..utils import (
|
||||||
cli_option,
|
cli_option,
|
||||||
cli_valueless_option,
|
cli_valueless_option,
|
||||||
|
@ -22,6 +29,9 @@ from ..utils import (
|
||||||
handle_youtubedl_headers,
|
handle_youtubedl_headers,
|
||||||
check_executable,
|
check_executable,
|
||||||
is_outdated_version,
|
is_outdated_version,
|
||||||
|
process_communicate_or_kill,
|
||||||
|
T,
|
||||||
|
traverse_obj,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@ -29,6 +39,7 @@ class ExternalFD(FileDownloader):
|
||||||
def real_download(self, filename, info_dict):
|
def real_download(self, filename, info_dict):
|
||||||
self.report_destination(filename)
|
self.report_destination(filename)
|
||||||
tmpfilename = self.temp_name(filename)
|
tmpfilename = self.temp_name(filename)
|
||||||
|
self._cookies_tempfile = None
|
||||||
|
|
||||||
try:
|
try:
|
||||||
started = time.time()
|
started = time.time()
|
||||||
|
@ -41,6 +52,13 @@ class ExternalFD(FileDownloader):
|
||||||
# should take place
|
# should take place
|
||||||
retval = 0
|
retval = 0
|
||||||
self.to_screen('[%s] Interrupted by user' % self.get_basename())
|
self.to_screen('[%s] Interrupted by user' % self.get_basename())
|
||||||
|
finally:
|
||||||
|
if self._cookies_tempfile and os.path.isfile(self._cookies_tempfile):
|
||||||
|
try:
|
||||||
|
os.remove(self._cookies_tempfile)
|
||||||
|
except OSError:
|
||||||
|
self.report_warning(
|
||||||
|
'Unable to delete temporary cookies file "{0}"'.format(self._cookies_tempfile))
|
||||||
|
|
||||||
if retval == 0:
|
if retval == 0:
|
||||||
status = {
|
status = {
|
||||||
|
@ -96,6 +114,16 @@ class ExternalFD(FileDownloader):
|
||||||
def _configuration_args(self, default=[]):
|
def _configuration_args(self, default=[]):
|
||||||
return cli_configuration_args(self.params, 'external_downloader_args', default)
|
return cli_configuration_args(self.params, 'external_downloader_args', default)
|
||||||
|
|
||||||
|
def _write_cookies(self):
|
||||||
|
if not self.ydl.cookiejar.filename:
|
||||||
|
tmp_cookies = tempfile.NamedTemporaryFile(suffix='.cookies', delete=False)
|
||||||
|
tmp_cookies.close()
|
||||||
|
self._cookies_tempfile = tmp_cookies.name
|
||||||
|
self.to_screen('[download] Writing temporary cookies file to "{0}"'.format(self._cookies_tempfile))
|
||||||
|
# real_download resets _cookies_tempfile; if it's None, save() will write to cookiejar.filename
|
||||||
|
self.ydl.cookiejar.save(self._cookies_tempfile, ignore_discard=True, ignore_expires=True)
|
||||||
|
return self.ydl.cookiejar.filename or self._cookies_tempfile
|
||||||
|
|
||||||
def _call_downloader(self, tmpfilename, info_dict):
|
def _call_downloader(self, tmpfilename, info_dict):
|
||||||
""" Either overwrite this or implement _make_cmd """
|
""" Either overwrite this or implement _make_cmd """
|
||||||
cmd = [encodeArgument(a) for a in self._make_cmd(tmpfilename, info_dict)]
|
cmd = [encodeArgument(a) for a in self._make_cmd(tmpfilename, info_dict)]
|
||||||
|
@ -104,18 +132,26 @@ class ExternalFD(FileDownloader):
|
||||||
|
|
||||||
p = subprocess.Popen(
|
p = subprocess.Popen(
|
||||||
cmd, stderr=subprocess.PIPE)
|
cmd, stderr=subprocess.PIPE)
|
||||||
_, stderr = p.communicate()
|
_, stderr = process_communicate_or_kill(p)
|
||||||
if p.returncode != 0:
|
if p.returncode != 0:
|
||||||
self.to_stderr(stderr.decode('utf-8', 'replace'))
|
self.to_stderr(stderr.decode('utf-8', 'replace'))
|
||||||
return p.returncode
|
return p.returncode
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _header_items(info_dict):
|
||||||
|
return traverse_obj(
|
||||||
|
info_dict, ('http_headers', T(dict.items), Ellipsis))
|
||||||
|
|
||||||
|
|
||||||
class CurlFD(ExternalFD):
|
class CurlFD(ExternalFD):
|
||||||
AVAILABLE_OPT = '-V'
|
AVAILABLE_OPT = '-V'
|
||||||
|
|
||||||
def _make_cmd(self, tmpfilename, info_dict):
|
def _make_cmd(self, tmpfilename, info_dict):
|
||||||
cmd = [self.exe, '--location', '-o', tmpfilename]
|
cmd = [self.exe, '--location', '-o', tmpfilename, '--compressed']
|
||||||
for key, val in info_dict['http_headers'].items():
|
cookie_header = self.ydl.cookiejar.get_cookie_header(info_dict['url'])
|
||||||
|
if cookie_header:
|
||||||
|
cmd += ['--cookie', cookie_header]
|
||||||
|
for key, val in self._header_items(info_dict):
|
||||||
cmd += ['--header', '%s: %s' % (key, val)]
|
cmd += ['--header', '%s: %s' % (key, val)]
|
||||||
cmd += self._bool_option('--continue-at', 'continuedl', '-', '0')
|
cmd += self._bool_option('--continue-at', 'continuedl', '-', '0')
|
||||||
cmd += self._valueless_option('--silent', 'noprogress')
|
cmd += self._valueless_option('--silent', 'noprogress')
|
||||||
|
@ -141,7 +177,7 @@ class CurlFD(ExternalFD):
|
||||||
|
|
||||||
# curl writes the progress to stderr so don't capture it.
|
# curl writes the progress to stderr so don't capture it.
|
||||||
p = subprocess.Popen(cmd)
|
p = subprocess.Popen(cmd)
|
||||||
p.communicate()
|
process_communicate_or_kill(p)
|
||||||
return p.returncode
|
return p.returncode
|
||||||
|
|
||||||
|
|
||||||
|
@ -150,8 +186,11 @@ class AxelFD(ExternalFD):
|
||||||
|
|
||||||
def _make_cmd(self, tmpfilename, info_dict):
|
def _make_cmd(self, tmpfilename, info_dict):
|
||||||
cmd = [self.exe, '-o', tmpfilename]
|
cmd = [self.exe, '-o', tmpfilename]
|
||||||
for key, val in info_dict['http_headers'].items():
|
for key, val in self._header_items(info_dict):
|
||||||
cmd += ['-H', '%s: %s' % (key, val)]
|
cmd += ['-H', '%s: %s' % (key, val)]
|
||||||
|
cookie_header = self.ydl.cookiejar.get_cookie_header(info_dict['url'])
|
||||||
|
if cookie_header:
|
||||||
|
cmd += ['-H', 'Cookie: {0}'.format(cookie_header), '--max-redirect=0']
|
||||||
cmd += self._configuration_args()
|
cmd += self._configuration_args()
|
||||||
cmd += ['--', info_dict['url']]
|
cmd += ['--', info_dict['url']]
|
||||||
return cmd
|
return cmd
|
||||||
|
@ -161,8 +200,10 @@ class WgetFD(ExternalFD):
|
||||||
AVAILABLE_OPT = '--version'
|
AVAILABLE_OPT = '--version'
|
||||||
|
|
||||||
def _make_cmd(self, tmpfilename, info_dict):
|
def _make_cmd(self, tmpfilename, info_dict):
|
||||||
cmd = [self.exe, '-O', tmpfilename, '-nv', '--no-cookies']
|
cmd = [self.exe, '-O', tmpfilename, '-nv', '--compression=auto']
|
||||||
for key, val in info_dict['http_headers'].items():
|
if self.ydl.cookiejar.get_cookie_header(info_dict['url']):
|
||||||
|
cmd += ['--load-cookies', self._write_cookies()]
|
||||||
|
for key, val in self._header_items(info_dict):
|
||||||
cmd += ['--header', '%s: %s' % (key, val)]
|
cmd += ['--header', '%s: %s' % (key, val)]
|
||||||
cmd += self._option('--limit-rate', 'ratelimit')
|
cmd += self._option('--limit-rate', 'ratelimit')
|
||||||
retry = self._option('--tries', 'retries')
|
retry = self._option('--tries', 'retries')
|
||||||
|
@ -171,7 +212,10 @@ class WgetFD(ExternalFD):
|
||||||
retry[1] = '0'
|
retry[1] = '0'
|
||||||
cmd += retry
|
cmd += retry
|
||||||
cmd += self._option('--bind-address', 'source_address')
|
cmd += self._option('--bind-address', 'source_address')
|
||||||
cmd += self._option('--proxy', 'proxy')
|
proxy = self.params.get('proxy')
|
||||||
|
if proxy:
|
||||||
|
for var in ('http_proxy', 'https_proxy'):
|
||||||
|
cmd += ['--execute', '%s=%s' % (var, proxy)]
|
||||||
cmd += self._valueless_option('--no-check-certificate', 'nocheckcertificate')
|
cmd += self._valueless_option('--no-check-certificate', 'nocheckcertificate')
|
||||||
cmd += self._configuration_args()
|
cmd += self._configuration_args()
|
||||||
cmd += ['--', info_dict['url']]
|
cmd += ['--', info_dict['url']]
|
||||||
|
@ -181,24 +225,121 @@ class WgetFD(ExternalFD):
|
||||||
class Aria2cFD(ExternalFD):
|
class Aria2cFD(ExternalFD):
|
||||||
AVAILABLE_OPT = '-v'
|
AVAILABLE_OPT = '-v'
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _aria2c_filename(fn):
|
||||||
|
return fn if os.path.isabs(fn) else os.path.join('.', fn)
|
||||||
|
|
||||||
def _make_cmd(self, tmpfilename, info_dict):
|
def _make_cmd(self, tmpfilename, info_dict):
|
||||||
cmd = [self.exe, '-c']
|
cmd = [self.exe, '-c',
|
||||||
cmd += self._configuration_args([
|
'--console-log-level=warn', '--summary-interval=0', '--download-result=hide',
|
||||||
'--min-split-size', '1M', '--max-connection-per-server', '4'])
|
'--http-accept-gzip=true', '--file-allocation=none', '-x16', '-j16', '-s16']
|
||||||
dn = os.path.dirname(tmpfilename)
|
if 'fragments' in info_dict:
|
||||||
if dn:
|
cmd += ['--allow-overwrite=true', '--allow-piece-length-change=true']
|
||||||
cmd += ['--dir', dn]
|
else:
|
||||||
cmd += ['--out', os.path.basename(tmpfilename)]
|
cmd += ['--min-split-size', '1M']
|
||||||
for key, val in info_dict['http_headers'].items():
|
|
||||||
|
if self.ydl.cookiejar.get_cookie_header(info_dict['url']):
|
||||||
|
cmd += ['--load-cookies={0}'.format(self._write_cookies())]
|
||||||
|
for key, val in self._header_items(info_dict):
|
||||||
cmd += ['--header', '%s: %s' % (key, val)]
|
cmd += ['--header', '%s: %s' % (key, val)]
|
||||||
|
cmd += self._configuration_args(['--max-connection-per-server', '4'])
|
||||||
|
cmd += ['--out', os.path.basename(tmpfilename)]
|
||||||
|
cmd += self._option('--max-overall-download-limit', 'ratelimit')
|
||||||
cmd += self._option('--interface', 'source_address')
|
cmd += self._option('--interface', 'source_address')
|
||||||
cmd += self._option('--all-proxy', 'proxy')
|
cmd += self._option('--all-proxy', 'proxy')
|
||||||
cmd += self._bool_option('--check-certificate', 'nocheckcertificate', 'false', 'true', '=')
|
cmd += self._bool_option('--check-certificate', 'nocheckcertificate', 'false', 'true', '=')
|
||||||
cmd += self._bool_option('--remote-time', 'updatetime', 'true', 'false', '=')
|
cmd += self._bool_option('--remote-time', 'updatetime', 'true', 'false', '=')
|
||||||
cmd += ['--', info_dict['url']]
|
cmd += self._bool_option('--show-console-readout', 'noprogress', 'false', 'true', '=')
|
||||||
|
cmd += self._configuration_args()
|
||||||
|
|
||||||
|
# aria2c strips out spaces from the beginning/end of filenames and paths.
|
||||||
|
# We work around this issue by adding a "./" to the beginning of the
|
||||||
|
# filename and relative path, and adding a "/" at the end of the path.
|
||||||
|
# See: https://github.com/yt-dlp/yt-dlp/issues/276
|
||||||
|
# https://github.com/ytdl-org/youtube-dl/issues/20312
|
||||||
|
# https://github.com/aria2/aria2/issues/1373
|
||||||
|
dn = os.path.dirname(tmpfilename)
|
||||||
|
if dn:
|
||||||
|
cmd += ['--dir', self._aria2c_filename(dn) + os.path.sep]
|
||||||
|
if 'fragments' not in info_dict:
|
||||||
|
cmd += ['--out', self._aria2c_filename(os.path.basename(tmpfilename))]
|
||||||
|
cmd += ['--auto-file-renaming=false']
|
||||||
|
if 'fragments' in info_dict:
|
||||||
|
cmd += ['--file-allocation=none', '--uri-selector=inorder']
|
||||||
|
url_list_file = '%s.frag.urls' % (tmpfilename, )
|
||||||
|
url_list = []
|
||||||
|
for frag_index, fragment in enumerate(info_dict['fragments']):
|
||||||
|
fragment_filename = '%s-Frag%d' % (os.path.basename(tmpfilename), frag_index)
|
||||||
|
url_list.append('%s\n\tout=%s' % (fragment['url'], self._aria2c_filename(fragment_filename)))
|
||||||
|
stream, _ = self.sanitize_open(url_list_file, 'wb')
|
||||||
|
stream.write('\n'.join(url_list).encode())
|
||||||
|
stream.close()
|
||||||
|
cmd += ['-i', self._aria2c_filename(url_list_file)]
|
||||||
|
else:
|
||||||
|
cmd += ['--', info_dict['url']]
|
||||||
return cmd
|
return cmd
|
||||||
|
|
||||||
|
|
||||||
|
class Aria2pFD(ExternalFD):
|
||||||
|
''' Aria2pFD class
|
||||||
|
This class support to use aria2p as downloader.
|
||||||
|
(Aria2p, a command-line tool and Python library to interact with an aria2c daemon process
|
||||||
|
through JSON-RPC.)
|
||||||
|
It can help you to get download progress more easily.
|
||||||
|
To use aria2p as downloader, you need to install aria2c and aria2p, aria2p can download with pip.
|
||||||
|
Then run aria2c in the background and enable with the --enable-rpc option.
|
||||||
|
'''
|
||||||
|
try:
|
||||||
|
import aria2p
|
||||||
|
__avail = True
|
||||||
|
except ImportError:
|
||||||
|
__avail = False
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def available(cls):
|
||||||
|
return cls.__avail
|
||||||
|
|
||||||
|
def _call_downloader(self, tmpfilename, info_dict):
|
||||||
|
aria2 = self.aria2p.API(
|
||||||
|
self.aria2p.Client(
|
||||||
|
host='http://localhost',
|
||||||
|
port=6800,
|
||||||
|
secret=''
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
options = {
|
||||||
|
'min-split-size': '1M',
|
||||||
|
'max-connection-per-server': 4,
|
||||||
|
'auto-file-renaming': 'false',
|
||||||
|
}
|
||||||
|
options['dir'] = os.path.dirname(tmpfilename) or os.path.abspath('.')
|
||||||
|
options['out'] = os.path.basename(tmpfilename)
|
||||||
|
if self.ydl.cookiejar.get_cookie_header(info_dict['url']):
|
||||||
|
options['load-cookies'] = self._write_cookies()
|
||||||
|
options['header'] = []
|
||||||
|
for key, val in self._header_items(info_dict):
|
||||||
|
options['header'].append('{0}: {1}'.format(key, val))
|
||||||
|
download = aria2.add_uris([info_dict['url']], options)
|
||||||
|
status = {
|
||||||
|
'status': 'downloading',
|
||||||
|
'tmpfilename': tmpfilename,
|
||||||
|
}
|
||||||
|
started = time.time()
|
||||||
|
while download.status in ['active', 'waiting']:
|
||||||
|
download = aria2.get_download(download.gid)
|
||||||
|
status.update({
|
||||||
|
'downloaded_bytes': download.completed_length,
|
||||||
|
'total_bytes': download.total_length,
|
||||||
|
'elapsed': time.time() - started,
|
||||||
|
'eta': download.eta.total_seconds(),
|
||||||
|
'speed': download.download_speed,
|
||||||
|
})
|
||||||
|
self._hook_progress(status)
|
||||||
|
time.sleep(.5)
|
||||||
|
return download.status != 'complete'
|
||||||
|
|
||||||
|
|
||||||
class HttpieFD(ExternalFD):
|
class HttpieFD(ExternalFD):
|
||||||
@classmethod
|
@classmethod
|
||||||
def available(cls):
|
def available(cls):
|
||||||
|
@ -206,25 +347,34 @@ class HttpieFD(ExternalFD):
|
||||||
|
|
||||||
def _make_cmd(self, tmpfilename, info_dict):
|
def _make_cmd(self, tmpfilename, info_dict):
|
||||||
cmd = ['http', '--download', '--output', tmpfilename, info_dict['url']]
|
cmd = ['http', '--download', '--output', tmpfilename, info_dict['url']]
|
||||||
for key, val in info_dict['http_headers'].items():
|
for key, val in self._header_items(info_dict):
|
||||||
cmd += ['%s:%s' % (key, val)]
|
cmd += ['%s:%s' % (key, val)]
|
||||||
|
|
||||||
|
# httpie 3.1.0+ removes the Cookie header on redirect, so this should be safe for now. [1]
|
||||||
|
# If we ever need cookie handling for redirects, we can export the cookiejar into a session. [2]
|
||||||
|
# 1: https://github.com/httpie/httpie/security/advisories/GHSA-9w4w-cpc8-h2fq
|
||||||
|
# 2: https://httpie.io/docs/cli/sessions
|
||||||
|
cookie_header = self.ydl.cookiejar.get_cookie_header(info_dict['url'])
|
||||||
|
if cookie_header:
|
||||||
|
cmd += ['Cookie:%s' % cookie_header]
|
||||||
return cmd
|
return cmd
|
||||||
|
|
||||||
|
|
||||||
class FFmpegFD(ExternalFD):
|
class FFmpegFD(ExternalFD):
|
||||||
@classmethod
|
@classmethod
|
||||||
def supports(cls, info_dict):
|
def supports(cls, info_dict):
|
||||||
return info_dict['protocol'] in ('http', 'https', 'ftp', 'ftps', 'm3u8', 'rtsp', 'rtmp', 'mms')
|
return info_dict['protocol'] in ('http', 'https', 'ftp', 'ftps', 'm3u8', 'rtsp', 'rtmp', 'mms', 'http_dash_segments')
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def available(cls):
|
def available(cls):
|
||||||
return FFmpegPostProcessor().available
|
# actual availability can only be confirmed for an instance
|
||||||
|
return bool(FFmpegPostProcessor)
|
||||||
|
|
||||||
def _call_downloader(self, tmpfilename, info_dict):
|
def _call_downloader(self, tmpfilename, info_dict):
|
||||||
url = info_dict['url']
|
# `downloader` means the parent `YoutubeDL`
|
||||||
ffpp = FFmpegPostProcessor(downloader=self)
|
ffpp = FFmpegPostProcessor(downloader=self.ydl)
|
||||||
if not ffpp.available:
|
if not ffpp.available:
|
||||||
self.report_error('m3u8 download detected but ffmpeg or avconv could not be found. Please install one.')
|
self.report_error('ffmpeg required for download but no ffmpeg (nor avconv) executable could be found. Please install one.')
|
||||||
return False
|
return False
|
||||||
ffpp.check_version()
|
ffpp.check_version()
|
||||||
|
|
||||||
|
@ -253,7 +403,15 @@ class FFmpegFD(ExternalFD):
|
||||||
# if end_time:
|
# if end_time:
|
||||||
# args += ['-t', compat_str(end_time - start_time)]
|
# args += ['-t', compat_str(end_time - start_time)]
|
||||||
|
|
||||||
if info_dict['http_headers'] and re.match(r'^https?://', url):
|
url = info_dict['url']
|
||||||
|
cookies = self.ydl.cookiejar.get_cookies_for_url(url)
|
||||||
|
if cookies:
|
||||||
|
args.extend(['-cookies', ''.join(
|
||||||
|
'{0}={1}; path={2}; domain={3};\r\n'.format(
|
||||||
|
cookie.name, cookie.value, cookie.path, cookie.domain)
|
||||||
|
for cookie in cookies)])
|
||||||
|
|
||||||
|
if info_dict.get('http_headers') and re.match(r'^https?://', url):
|
||||||
# Trailing \r\n after each HTTP header is important to prevent warning from ffmpeg/avconv:
|
# Trailing \r\n after each HTTP header is important to prevent warning from ffmpeg/avconv:
|
||||||
# [http @ 00000000003d2fa0] No trailing CRLF found in HTTP header.
|
# [http @ 00000000003d2fa0] No trailing CRLF found in HTTP header.
|
||||||
headers = handle_youtubedl_headers(info_dict['http_headers'])
|
headers = handle_youtubedl_headers(info_dict['http_headers'])
|
||||||
|
@ -333,18 +491,25 @@ class FFmpegFD(ExternalFD):
|
||||||
|
|
||||||
self._debug_cmd(args)
|
self._debug_cmd(args)
|
||||||
|
|
||||||
proc = subprocess.Popen(args, stdin=subprocess.PIPE, env=env)
|
# From [1], a PIPE opened in Popen() should be closed, unless
|
||||||
try:
|
# .communicate() is called. Avoid leaking any PIPEs by using Popen
|
||||||
retval = proc.wait()
|
# as a context manager (newer Python 3.x and compat)
|
||||||
except KeyboardInterrupt:
|
# Fixes "Resource Warning" in test/test_downloader_external.py
|
||||||
# subprocces.run would send the SIGKILL signal to ffmpeg and the
|
# [1] https://devpress.csdn.net/python/62fde12d7e66823466192e48.html
|
||||||
# mp4 file couldn't be played, but if we ask ffmpeg to quit it
|
with compat_subprocess_Popen(args, stdin=subprocess.PIPE, env=env) as proc:
|
||||||
# produces a file that is playable (this is mostly useful for live
|
try:
|
||||||
# streams). Note that Windows is not affected and produces playable
|
retval = proc.wait()
|
||||||
# files (see https://github.com/ytdl-org/youtube-dl/issues/8300).
|
except BaseException as e:
|
||||||
if sys.platform != 'win32':
|
# subprocess.run would send the SIGKILL signal to ffmpeg and the
|
||||||
proc.communicate(b'q')
|
# mp4 file couldn't be played, but if we ask ffmpeg to quit it
|
||||||
raise
|
# produces a file that is playable (this is mostly useful for live
|
||||||
|
# streams). Note that Windows is not affected and produces playable
|
||||||
|
# files (see https://github.com/ytdl-org/youtube-dl/issues/8300).
|
||||||
|
if isinstance(e, KeyboardInterrupt) and sys.platform != 'win32':
|
||||||
|
process_communicate_or_kill(proc, b'q')
|
||||||
|
else:
|
||||||
|
proc.kill()
|
||||||
|
raise
|
||||||
return retval
|
return retval
|
||||||
|
|
||||||
|
|
||||||
|
|
|
@ -71,7 +71,7 @@ class FragmentFD(FileDownloader):
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def __do_ytdl_file(ctx):
|
def __do_ytdl_file(ctx):
|
||||||
return not ctx['live'] and not ctx['tmpfilename'] == '-'
|
return ctx['live'] is not True and ctx['tmpfilename'] != '-'
|
||||||
|
|
||||||
def _read_ytdl_file(self, ctx):
|
def _read_ytdl_file(self, ctx):
|
||||||
assert 'ytdl_corrupt' not in ctx
|
assert 'ytdl_corrupt' not in ctx
|
||||||
|
@ -97,12 +97,22 @@ class FragmentFD(FileDownloader):
|
||||||
|
|
||||||
def _download_fragment(self, ctx, frag_url, info_dict, headers=None):
|
def _download_fragment(self, ctx, frag_url, info_dict, headers=None):
|
||||||
fragment_filename = '%s-Frag%d' % (ctx['tmpfilename'], ctx['fragment_index'])
|
fragment_filename = '%s-Frag%d' % (ctx['tmpfilename'], ctx['fragment_index'])
|
||||||
success = ctx['dl'].download(fragment_filename, {
|
fragment_info_dict = {
|
||||||
'url': frag_url,
|
'url': frag_url,
|
||||||
'http_headers': headers or info_dict.get('http_headers'),
|
'http_headers': headers or info_dict.get('http_headers'),
|
||||||
})
|
}
|
||||||
|
frag_resume_len = 0
|
||||||
|
if ctx['dl'].params.get('continuedl', True):
|
||||||
|
frag_resume_len = self.filesize_or_none(
|
||||||
|
self.temp_name(fragment_filename))
|
||||||
|
fragment_info_dict['frag_resume_len'] = frag_resume_len
|
||||||
|
ctx['frag_resume_len'] = frag_resume_len or 0
|
||||||
|
|
||||||
|
success = ctx['dl'].download(fragment_filename, fragment_info_dict)
|
||||||
if not success:
|
if not success:
|
||||||
return False, None
|
return False, None
|
||||||
|
if fragment_info_dict.get('filetime'):
|
||||||
|
ctx['fragment_filetime'] = fragment_info_dict.get('filetime')
|
||||||
down, frag_sanitized = sanitize_open(fragment_filename, 'rb')
|
down, frag_sanitized = sanitize_open(fragment_filename, 'rb')
|
||||||
ctx['fragment_filename_sanitized'] = frag_sanitized
|
ctx['fragment_filename_sanitized'] = frag_sanitized
|
||||||
frag_content = down.read()
|
frag_content = down.read()
|
||||||
|
@ -121,9 +131,7 @@ class FragmentFD(FileDownloader):
|
||||||
del ctx['fragment_filename_sanitized']
|
del ctx['fragment_filename_sanitized']
|
||||||
|
|
||||||
def _prepare_frag_download(self, ctx):
|
def _prepare_frag_download(self, ctx):
|
||||||
if 'live' not in ctx:
|
if not ctx.setdefault('live', False):
|
||||||
ctx['live'] = False
|
|
||||||
if not ctx['live']:
|
|
||||||
total_frags_str = '%d' % ctx['total_frags']
|
total_frags_str = '%d' % ctx['total_frags']
|
||||||
ad_frags = ctx.get('ad_frags', 0)
|
ad_frags = ctx.get('ad_frags', 0)
|
||||||
if ad_frags:
|
if ad_frags:
|
||||||
|
@ -133,10 +141,11 @@ class FragmentFD(FileDownloader):
|
||||||
self.to_screen(
|
self.to_screen(
|
||||||
'[%s] Total fragments: %s' % (self.FD_NAME, total_frags_str))
|
'[%s] Total fragments: %s' % (self.FD_NAME, total_frags_str))
|
||||||
self.report_destination(ctx['filename'])
|
self.report_destination(ctx['filename'])
|
||||||
|
continuedl = self.params.get('continuedl', True)
|
||||||
dl = HttpQuietDownloader(
|
dl = HttpQuietDownloader(
|
||||||
self.ydl,
|
self.ydl,
|
||||||
{
|
{
|
||||||
'continuedl': True,
|
'continuedl': continuedl,
|
||||||
'quiet': True,
|
'quiet': True,
|
||||||
'noprogress': True,
|
'noprogress': True,
|
||||||
'ratelimit': self.params.get('ratelimit'),
|
'ratelimit': self.params.get('ratelimit'),
|
||||||
|
@ -147,12 +156,11 @@ class FragmentFD(FileDownloader):
|
||||||
)
|
)
|
||||||
tmpfilename = self.temp_name(ctx['filename'])
|
tmpfilename = self.temp_name(ctx['filename'])
|
||||||
open_mode = 'wb'
|
open_mode = 'wb'
|
||||||
resume_len = 0
|
|
||||||
|
|
||||||
# Establish possible resume length
|
# Establish possible resume length
|
||||||
if os.path.isfile(encodeFilename(tmpfilename)):
|
resume_len = self.filesize_or_none(tmpfilename) or 0
|
||||||
|
if resume_len > 0:
|
||||||
open_mode = 'ab'
|
open_mode = 'ab'
|
||||||
resume_len = os.path.getsize(encodeFilename(tmpfilename))
|
|
||||||
|
|
||||||
# Should be initialized before ytdl file check
|
# Should be initialized before ytdl file check
|
||||||
ctx.update({
|
ctx.update({
|
||||||
|
@ -161,7 +169,8 @@ class FragmentFD(FileDownloader):
|
||||||
})
|
})
|
||||||
|
|
||||||
if self.__do_ytdl_file(ctx):
|
if self.__do_ytdl_file(ctx):
|
||||||
if os.path.isfile(encodeFilename(self.ytdl_filename(ctx['filename']))):
|
ytdl_file_exists = os.path.isfile(encodeFilename(self.ytdl_filename(ctx['filename'])))
|
||||||
|
if continuedl and ytdl_file_exists:
|
||||||
self._read_ytdl_file(ctx)
|
self._read_ytdl_file(ctx)
|
||||||
is_corrupt = ctx.get('ytdl_corrupt') is True
|
is_corrupt = ctx.get('ytdl_corrupt') is True
|
||||||
is_inconsistent = ctx['fragment_index'] > 0 and resume_len == 0
|
is_inconsistent = ctx['fragment_index'] > 0 and resume_len == 0
|
||||||
|
@ -175,7 +184,12 @@ class FragmentFD(FileDownloader):
|
||||||
if 'ytdl_corrupt' in ctx:
|
if 'ytdl_corrupt' in ctx:
|
||||||
del ctx['ytdl_corrupt']
|
del ctx['ytdl_corrupt']
|
||||||
self._write_ytdl_file(ctx)
|
self._write_ytdl_file(ctx)
|
||||||
|
|
||||||
else:
|
else:
|
||||||
|
if not continuedl:
|
||||||
|
if ytdl_file_exists:
|
||||||
|
self._read_ytdl_file(ctx)
|
||||||
|
ctx['fragment_index'] = resume_len = 0
|
||||||
self._write_ytdl_file(ctx)
|
self._write_ytdl_file(ctx)
|
||||||
assert ctx['fragment_index'] == 0
|
assert ctx['fragment_index'] == 0
|
||||||
|
|
||||||
|
@ -206,6 +220,7 @@ class FragmentFD(FileDownloader):
|
||||||
start = time.time()
|
start = time.time()
|
||||||
ctx.update({
|
ctx.update({
|
||||||
'started': start,
|
'started': start,
|
||||||
|
'fragment_started': start,
|
||||||
# Amount of fragment's bytes downloaded by the time of the previous
|
# Amount of fragment's bytes downloaded by the time of the previous
|
||||||
# frag progress hook invocation
|
# frag progress hook invocation
|
||||||
'prev_frag_downloaded_bytes': 0,
|
'prev_frag_downloaded_bytes': 0,
|
||||||
|
@ -215,6 +230,9 @@ class FragmentFD(FileDownloader):
|
||||||
if s['status'] not in ('downloading', 'finished'):
|
if s['status'] not in ('downloading', 'finished'):
|
||||||
return
|
return
|
||||||
|
|
||||||
|
if not total_frags and ctx.get('fragment_count'):
|
||||||
|
state['fragment_count'] = ctx['fragment_count']
|
||||||
|
|
||||||
time_now = time.time()
|
time_now = time.time()
|
||||||
state['elapsed'] = time_now - start
|
state['elapsed'] = time_now - start
|
||||||
frag_total_bytes = s.get('total_bytes') or 0
|
frag_total_bytes = s.get('total_bytes') or 0
|
||||||
|
@ -229,16 +247,17 @@ class FragmentFD(FileDownloader):
|
||||||
ctx['fragment_index'] = state['fragment_index']
|
ctx['fragment_index'] = state['fragment_index']
|
||||||
state['downloaded_bytes'] += frag_total_bytes - ctx['prev_frag_downloaded_bytes']
|
state['downloaded_bytes'] += frag_total_bytes - ctx['prev_frag_downloaded_bytes']
|
||||||
ctx['complete_frags_downloaded_bytes'] = state['downloaded_bytes']
|
ctx['complete_frags_downloaded_bytes'] = state['downloaded_bytes']
|
||||||
|
ctx['speed'] = state['speed'] = self.calc_speed(
|
||||||
|
ctx['fragment_started'], time_now, frag_total_bytes)
|
||||||
|
ctx['fragment_started'] = time.time()
|
||||||
ctx['prev_frag_downloaded_bytes'] = 0
|
ctx['prev_frag_downloaded_bytes'] = 0
|
||||||
else:
|
else:
|
||||||
frag_downloaded_bytes = s['downloaded_bytes']
|
frag_downloaded_bytes = s['downloaded_bytes']
|
||||||
state['downloaded_bytes'] += frag_downloaded_bytes - ctx['prev_frag_downloaded_bytes']
|
state['downloaded_bytes'] += frag_downloaded_bytes - ctx['prev_frag_downloaded_bytes']
|
||||||
|
ctx['speed'] = state['speed'] = self.calc_speed(
|
||||||
|
ctx['fragment_started'], time_now, frag_downloaded_bytes - ctx['frag_resume_len'])
|
||||||
if not ctx['live']:
|
if not ctx['live']:
|
||||||
state['eta'] = self.calc_eta(
|
state['eta'] = self.calc_eta(state['speed'], estimated_size - state['downloaded_bytes'])
|
||||||
start, time_now, estimated_size - resume_len,
|
|
||||||
state['downloaded_bytes'] - resume_len)
|
|
||||||
state['speed'] = s.get('speed') or ctx.get('speed')
|
|
||||||
ctx['speed'] = state['speed']
|
|
||||||
ctx['prev_frag_downloaded_bytes'] = frag_downloaded_bytes
|
ctx['prev_frag_downloaded_bytes'] = frag_downloaded_bytes
|
||||||
self._hook_progress(state)
|
self._hook_progress(state)
|
||||||
|
|
||||||
|
@ -258,7 +277,14 @@ class FragmentFD(FileDownloader):
|
||||||
downloaded_bytes = ctx['complete_frags_downloaded_bytes']
|
downloaded_bytes = ctx['complete_frags_downloaded_bytes']
|
||||||
else:
|
else:
|
||||||
self.try_rename(ctx['tmpfilename'], ctx['filename'])
|
self.try_rename(ctx['tmpfilename'], ctx['filename'])
|
||||||
downloaded_bytes = os.path.getsize(encodeFilename(ctx['filename']))
|
if self.params.get('updatetime', True):
|
||||||
|
filetime = ctx.get('fragment_filetime')
|
||||||
|
if filetime:
|
||||||
|
try:
|
||||||
|
os.utime(ctx['filename'], (time.time(), filetime))
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
downloaded_bytes = self.filesize_or_none(ctx['filename']) or 0
|
||||||
|
|
||||||
self._hook_progress({
|
self._hook_progress({
|
||||||
'downloaded_bytes': downloaded_bytes,
|
'downloaded_bytes': downloaded_bytes,
|
||||||
|
|
|
@ -42,11 +42,13 @@ class HlsFD(FragmentFD):
|
||||||
# no segments will definitely be appended to the end of the playlist.
|
# no segments will definitely be appended to the end of the playlist.
|
||||||
# r'#EXT-X-PLAYLIST-TYPE:EVENT', # media segments may be appended to the end of
|
# r'#EXT-X-PLAYLIST-TYPE:EVENT', # media segments may be appended to the end of
|
||||||
# # event media playlists [4]
|
# # event media playlists [4]
|
||||||
|
r'#EXT-X-MAP:', # media initialization [5]
|
||||||
|
|
||||||
# 1. https://tools.ietf.org/html/draft-pantos-http-live-streaming-17#section-4.3.2.4
|
# 1. https://tools.ietf.org/html/draft-pantos-http-live-streaming-17#section-4.3.2.4
|
||||||
# 2. https://tools.ietf.org/html/draft-pantos-http-live-streaming-17#section-4.3.2.2
|
# 2. https://tools.ietf.org/html/draft-pantos-http-live-streaming-17#section-4.3.2.2
|
||||||
# 3. https://tools.ietf.org/html/draft-pantos-http-live-streaming-17#section-4.3.3.2
|
# 3. https://tools.ietf.org/html/draft-pantos-http-live-streaming-17#section-4.3.3.2
|
||||||
# 4. https://tools.ietf.org/html/draft-pantos-http-live-streaming-17#section-4.3.3.5
|
# 4. https://tools.ietf.org/html/draft-pantos-http-live-streaming-17#section-4.3.3.5
|
||||||
|
# 5. https://tools.ietf.org/html/draft-pantos-http-live-streaming-17#section-4.3.2.5
|
||||||
)
|
)
|
||||||
check_results = [not re.search(feature, manifest) for feature in UNSUPPORTED_FEATURES]
|
check_results = [not re.search(feature, manifest) for feature in UNSUPPORTED_FEATURES]
|
||||||
is_aes128_enc = '#EXT-X-KEY:METHOD=AES-128' in manifest
|
is_aes128_enc = '#EXT-X-KEY:METHOD=AES-128' in manifest
|
||||||
|
@ -141,7 +143,7 @@ class HlsFD(FragmentFD):
|
||||||
count = 0
|
count = 0
|
||||||
headers = info_dict.get('http_headers', {})
|
headers = info_dict.get('http_headers', {})
|
||||||
if byte_range:
|
if byte_range:
|
||||||
headers['Range'] = 'bytes=%d-%d' % (byte_range['start'], byte_range['end'])
|
headers['Range'] = 'bytes=%d-%d' % (byte_range['start'], byte_range['end'] - 1)
|
||||||
while count <= fragment_retries:
|
while count <= fragment_retries:
|
||||||
try:
|
try:
|
||||||
success, frag_content = self._download_fragment(
|
success, frag_content = self._download_fragment(
|
||||||
|
@ -170,8 +172,12 @@ class HlsFD(FragmentFD):
|
||||||
iv = decrypt_info.get('IV') or compat_struct_pack('>8xq', media_sequence)
|
iv = decrypt_info.get('IV') or compat_struct_pack('>8xq', media_sequence)
|
||||||
decrypt_info['KEY'] = decrypt_info.get('KEY') or self.ydl.urlopen(
|
decrypt_info['KEY'] = decrypt_info.get('KEY') or self.ydl.urlopen(
|
||||||
self._prepare_url(info_dict, info_dict.get('_decryption_key_url') or decrypt_info['URI'])).read()
|
self._prepare_url(info_dict, info_dict.get('_decryption_key_url') or decrypt_info['URI'])).read()
|
||||||
frag_content = AES.new(
|
# Don't decrypt the content in tests since the data is explicitly truncated and it's not to a valid block
|
||||||
decrypt_info['KEY'], AES.MODE_CBC, iv).decrypt(frag_content)
|
# size (see https://github.com/ytdl-org/youtube-dl/pull/27660). Tests only care that the correct data downloaded,
|
||||||
|
# not what it decrypts to.
|
||||||
|
if not test:
|
||||||
|
frag_content = AES.new(
|
||||||
|
decrypt_info['KEY'], AES.MODE_CBC, iv).decrypt(frag_content)
|
||||||
self._append_fragment(ctx, frag_content)
|
self._append_fragment(ctx, frag_content)
|
||||||
# We only download the first fragment during the test
|
# We only download the first fragment during the test
|
||||||
if test:
|
if test:
|
||||||
|
|
|
@ -58,9 +58,9 @@ class HttpFD(FileDownloader):
|
||||||
|
|
||||||
if self.params.get('continuedl', True):
|
if self.params.get('continuedl', True):
|
||||||
# Establish possible resume length
|
# Establish possible resume length
|
||||||
if os.path.isfile(encodeFilename(ctx.tmpfilename)):
|
ctx.resume_len = info_dict.get('frag_resume_len')
|
||||||
ctx.resume_len = os.path.getsize(
|
if ctx.resume_len is None:
|
||||||
encodeFilename(ctx.tmpfilename))
|
ctx.resume_len = self.filesize_or_none(ctx.tmpfilename) or 0
|
||||||
|
|
||||||
ctx.is_resume = ctx.resume_len > 0
|
ctx.is_resume = ctx.resume_len > 0
|
||||||
|
|
||||||
|
@ -106,11 +106,18 @@ class HttpFD(FileDownloader):
|
||||||
set_range(request, range_start, range_end)
|
set_range(request, range_start, range_end)
|
||||||
# Establish connection
|
# Establish connection
|
||||||
try:
|
try:
|
||||||
ctx.data = self.ydl.urlopen(request)
|
try:
|
||||||
|
ctx.data = self.ydl.urlopen(request)
|
||||||
|
except (compat_urllib_error.URLError, ) as err:
|
||||||
|
# reason may not be available, e.g. for urllib2.HTTPError on python 2.6
|
||||||
|
reason = getattr(err, 'reason', None)
|
||||||
|
if isinstance(reason, socket.timeout):
|
||||||
|
raise RetryDownload(err)
|
||||||
|
raise err
|
||||||
# When trying to resume, Content-Range HTTP header of response has to be checked
|
# When trying to resume, Content-Range HTTP header of response has to be checked
|
||||||
# to match the value of requested Range HTTP header. This is due to a webservers
|
# to match the value of requested Range HTTP header. This is due to webservers
|
||||||
# that don't support resuming and serve a whole file with no Content-Range
|
# that don't support resuming and serve a whole file with no Content-Range
|
||||||
# set in response despite of requested Range (see
|
# set in response despite requested Range (see
|
||||||
# https://github.com/ytdl-org/youtube-dl/issues/6057#issuecomment-126129799)
|
# https://github.com/ytdl-org/youtube-dl/issues/6057#issuecomment-126129799)
|
||||||
if has_range:
|
if has_range:
|
||||||
content_range = ctx.data.headers.get('Content-Range')
|
content_range = ctx.data.headers.get('Content-Range')
|
||||||
|
@ -134,7 +141,8 @@ class HttpFD(FileDownloader):
|
||||||
# Content-Range is either not present or invalid. Assuming remote webserver is
|
# Content-Range is either not present or invalid. Assuming remote webserver is
|
||||||
# trying to send the whole file, resume is not possible, so wiping the local file
|
# trying to send the whole file, resume is not possible, so wiping the local file
|
||||||
# and performing entire redownload
|
# and performing entire redownload
|
||||||
self.report_unable_to_resume()
|
if range_start > 0:
|
||||||
|
self.report_unable_to_resume()
|
||||||
ctx.resume_len = 0
|
ctx.resume_len = 0
|
||||||
ctx.open_mode = 'wb'
|
ctx.open_mode = 'wb'
|
||||||
ctx.data_len = int_or_none(ctx.data.info().get('Content-length', None))
|
ctx.data_len = int_or_none(ctx.data.info().get('Content-length', None))
|
||||||
|
@ -218,24 +226,27 @@ class HttpFD(FileDownloader):
|
||||||
|
|
||||||
def retry(e):
|
def retry(e):
|
||||||
to_stdout = ctx.tmpfilename == '-'
|
to_stdout = ctx.tmpfilename == '-'
|
||||||
if not to_stdout:
|
if ctx.stream is not None:
|
||||||
ctx.stream.close()
|
if not to_stdout:
|
||||||
ctx.stream = None
|
ctx.stream.close()
|
||||||
|
ctx.stream = None
|
||||||
ctx.resume_len = byte_counter if to_stdout else os.path.getsize(encodeFilename(ctx.tmpfilename))
|
ctx.resume_len = byte_counter if to_stdout else os.path.getsize(encodeFilename(ctx.tmpfilename))
|
||||||
raise RetryDownload(e)
|
raise RetryDownload(e)
|
||||||
|
|
||||||
while True:
|
while True:
|
||||||
try:
|
try:
|
||||||
# Download and write
|
# Download and write
|
||||||
data_block = ctx.data.read(block_size if not is_test else min(block_size, data_len - byte_counter))
|
data_block = ctx.data.read(block_size if data_len is None else min(block_size, data_len - byte_counter))
|
||||||
# socket.timeout is a subclass of socket.error but may not have
|
# socket.timeout is a subclass of socket.error but may not have
|
||||||
# errno set
|
# errno set
|
||||||
except socket.timeout as e:
|
except socket.timeout as e:
|
||||||
retry(e)
|
retry(e)
|
||||||
except socket.error as e:
|
except socket.error as e:
|
||||||
if e.errno not in (errno.ECONNRESET, errno.ETIMEDOUT):
|
# SSLError on python 2 (inherits socket.error) may have
|
||||||
raise
|
# no errno set but this error message
|
||||||
retry(e)
|
if e.errno in (errno.ECONNRESET, errno.ETIMEDOUT) or getattr(e, 'message', None) == 'The read operation timed out':
|
||||||
|
retry(e)
|
||||||
|
raise
|
||||||
|
|
||||||
byte_counter += len(data_block)
|
byte_counter += len(data_block)
|
||||||
|
|
||||||
|
@ -283,10 +294,7 @@ class HttpFD(FileDownloader):
|
||||||
|
|
||||||
# Progress message
|
# Progress message
|
||||||
speed = self.calc_speed(start, now, byte_counter - ctx.resume_len)
|
speed = self.calc_speed(start, now, byte_counter - ctx.resume_len)
|
||||||
if ctx.data_len is None:
|
eta = self.calc_eta(speed, ctx.data_len and (ctx.data_len - byte_counter))
|
||||||
eta = None
|
|
||||||
else:
|
|
||||||
eta = self.calc_eta(start, time.time(), ctx.data_len - ctx.resume_len, byte_counter - ctx.resume_len)
|
|
||||||
|
|
||||||
self._hook_progress({
|
self._hook_progress({
|
||||||
'status': 'downloading',
|
'status': 'downloading',
|
||||||
|
@ -299,7 +307,7 @@ class HttpFD(FileDownloader):
|
||||||
'elapsed': now - ctx.start_time,
|
'elapsed': now - ctx.start_time,
|
||||||
})
|
})
|
||||||
|
|
||||||
if is_test and byte_counter == data_len:
|
if data_len is not None and byte_counter == data_len:
|
||||||
break
|
break
|
||||||
|
|
||||||
if not is_test and ctx.chunk_size and ctx.data_len is not None and byte_counter < ctx.data_len:
|
if not is_test and ctx.chunk_size and ctx.data_len is not None and byte_counter < ctx.data_len:
|
||||||
|
|
66
youtube_dl/downloader/niconico.py
Normal file
66
youtube_dl/downloader/niconico.py
Normal file
|
@ -0,0 +1,66 @@
|
||||||
|
# coding: utf-8
|
||||||
|
from __future__ import unicode_literals
|
||||||
|
|
||||||
|
try:
|
||||||
|
import threading
|
||||||
|
except ImportError:
|
||||||
|
threading = None
|
||||||
|
|
||||||
|
from .common import FileDownloader
|
||||||
|
from ..downloader import get_suitable_downloader
|
||||||
|
from ..extractor.niconico import NiconicoIE
|
||||||
|
from ..utils import sanitized_Request
|
||||||
|
|
||||||
|
|
||||||
|
class NiconicoDmcFD(FileDownloader):
|
||||||
|
""" Downloading niconico douga from DMC with heartbeat """
|
||||||
|
|
||||||
|
FD_NAME = 'niconico_dmc'
|
||||||
|
|
||||||
|
def real_download(self, filename, info_dict):
|
||||||
|
self.to_screen('[%s] Downloading from DMC' % self.FD_NAME)
|
||||||
|
|
||||||
|
ie = NiconicoIE(self.ydl)
|
||||||
|
info_dict, heartbeat_info_dict = ie._get_heartbeat_info(info_dict)
|
||||||
|
|
||||||
|
fd = get_suitable_downloader(info_dict, params=self.params)(self.ydl, self.params)
|
||||||
|
for ph in self._progress_hooks:
|
||||||
|
fd.add_progress_hook(ph)
|
||||||
|
|
||||||
|
if not threading:
|
||||||
|
self.to_screen('[%s] Threading for Heartbeat not available' % self.FD_NAME)
|
||||||
|
return fd.real_download(filename, info_dict)
|
||||||
|
|
||||||
|
success = download_complete = False
|
||||||
|
timer = [None]
|
||||||
|
heartbeat_lock = threading.Lock()
|
||||||
|
heartbeat_url = heartbeat_info_dict['url']
|
||||||
|
heartbeat_data = heartbeat_info_dict['data'].encode()
|
||||||
|
heartbeat_interval = heartbeat_info_dict.get('interval', 30)
|
||||||
|
|
||||||
|
request = sanitized_Request(heartbeat_url, heartbeat_data)
|
||||||
|
|
||||||
|
def heartbeat():
|
||||||
|
try:
|
||||||
|
self.ydl.urlopen(request).read()
|
||||||
|
except Exception:
|
||||||
|
self.to_screen('[%s] Heartbeat failed' % self.FD_NAME)
|
||||||
|
|
||||||
|
with heartbeat_lock:
|
||||||
|
if not download_complete:
|
||||||
|
timer[0] = threading.Timer(heartbeat_interval, heartbeat)
|
||||||
|
timer[0].start()
|
||||||
|
|
||||||
|
heartbeat_info_dict['ping']()
|
||||||
|
self.to_screen('[%s] Heartbeat with %d second interval ...' % (self.FD_NAME, heartbeat_interval))
|
||||||
|
try:
|
||||||
|
heartbeat()
|
||||||
|
if type(fd).__name__ == 'HlsFD':
|
||||||
|
info_dict.update(ie._extract_m3u8_formats(info_dict['url'], info_dict['id'])[0])
|
||||||
|
success = fd.real_download(filename, info_dict)
|
||||||
|
finally:
|
||||||
|
if heartbeat_lock:
|
||||||
|
with heartbeat_lock:
|
||||||
|
timer[0].cancel()
|
||||||
|
download_complete = True
|
||||||
|
return success
|
|
@ -89,11 +89,13 @@ class RtmpFD(FileDownloader):
|
||||||
self.to_screen('')
|
self.to_screen('')
|
||||||
cursor_in_new_line = True
|
cursor_in_new_line = True
|
||||||
self.to_screen('[rtmpdump] ' + line)
|
self.to_screen('[rtmpdump] ' + line)
|
||||||
finally:
|
if not cursor_in_new_line:
|
||||||
|
self.to_screen('')
|
||||||
|
return proc.wait()
|
||||||
|
except BaseException: # Including KeyboardInterrupt
|
||||||
|
proc.kill()
|
||||||
proc.wait()
|
proc.wait()
|
||||||
if not cursor_in_new_line:
|
raise
|
||||||
self.to_screen('')
|
|
||||||
return proc.returncode
|
|
||||||
|
|
||||||
url = info_dict['url']
|
url = info_dict['url']
|
||||||
player_url = info_dict.get('player_url')
|
player_url = info_dict.get('player_url')
|
||||||
|
|
|
@ -1,14 +1,15 @@
|
||||||
# coding: utf-8
|
# coding: utf-8
|
||||||
from __future__ import unicode_literals
|
from __future__ import unicode_literals
|
||||||
|
|
||||||
import calendar
|
|
||||||
import re
|
import re
|
||||||
import time
|
|
||||||
|
|
||||||
from .amp import AMPIE
|
from .amp import AMPIE
|
||||||
from .common import InfoExtractor
|
from .common import InfoExtractor
|
||||||
from .youtube import YoutubeIE
|
from ..utils import (
|
||||||
from ..compat import compat_urlparse
|
parse_duration,
|
||||||
|
parse_iso8601,
|
||||||
|
try_get,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
class AbcNewsVideoIE(AMPIE):
|
class AbcNewsVideoIE(AMPIE):
|
||||||
|
@ -18,8 +19,8 @@ class AbcNewsVideoIE(AMPIE):
|
||||||
(?:
|
(?:
|
||||||
abcnews\.go\.com/
|
abcnews\.go\.com/
|
||||||
(?:
|
(?:
|
||||||
[^/]+/video/(?P<display_id>[0-9a-z-]+)-|
|
(?:[^/]+/)*video/(?P<display_id>[0-9a-z-]+)-|
|
||||||
video/embed\?.*?\bid=
|
video/(?:embed|itemfeed)\?.*?\bid=
|
||||||
)|
|
)|
|
||||||
fivethirtyeight\.abcnews\.go\.com/video/embed/\d+/
|
fivethirtyeight\.abcnews\.go\.com/video/embed/\d+/
|
||||||
)
|
)
|
||||||
|
@ -36,6 +37,8 @@ class AbcNewsVideoIE(AMPIE):
|
||||||
'description': 'George Stephanopoulos goes one-on-one with Iranian Foreign Minister Dr. Javad Zarif.',
|
'description': 'George Stephanopoulos goes one-on-one with Iranian Foreign Minister Dr. Javad Zarif.',
|
||||||
'duration': 180,
|
'duration': 180,
|
||||||
'thumbnail': r're:^https?://.*\.jpg$',
|
'thumbnail': r're:^https?://.*\.jpg$',
|
||||||
|
'timestamp': 1380454200,
|
||||||
|
'upload_date': '20130929',
|
||||||
},
|
},
|
||||||
'params': {
|
'params': {
|
||||||
# m3u8 download
|
# m3u8 download
|
||||||
|
@ -47,6 +50,12 @@ class AbcNewsVideoIE(AMPIE):
|
||||||
}, {
|
}, {
|
||||||
'url': 'http://abcnews.go.com/2020/video/2020-husband-stands-teacher-jail-student-affairs-26119478',
|
'url': 'http://abcnews.go.com/2020/video/2020-husband-stands-teacher-jail-student-affairs-26119478',
|
||||||
'only_matching': True,
|
'only_matching': True,
|
||||||
|
}, {
|
||||||
|
'url': 'http://abcnews.go.com/video/itemfeed?id=46979033',
|
||||||
|
'only_matching': True,
|
||||||
|
}, {
|
||||||
|
'url': 'https://abcnews.go.com/GMA/News/video/history-christmas-story-67894761',
|
||||||
|
'only_matching': True,
|
||||||
}]
|
}]
|
||||||
|
|
||||||
def _real_extract(self, url):
|
def _real_extract(self, url):
|
||||||
|
@ -67,28 +76,23 @@ class AbcNewsIE(InfoExtractor):
|
||||||
_VALID_URL = r'https?://abcnews\.go\.com/(?:[^/]+/)+(?P<display_id>[0-9a-z-]+)/story\?id=(?P<id>\d+)'
|
_VALID_URL = r'https?://abcnews\.go\.com/(?:[^/]+/)+(?P<display_id>[0-9a-z-]+)/story\?id=(?P<id>\d+)'
|
||||||
|
|
||||||
_TESTS = [{
|
_TESTS = [{
|
||||||
'url': 'http://abcnews.go.com/Blotter/News/dramatic-video-rare-death-job-america/story?id=10498713#.UIhwosWHLjY',
|
# Youtube Embeds
|
||||||
|
'url': 'https://abcnews.go.com/Entertainment/peter-billingsley-child-actor-christmas-story-hollywood-power/story?id=51286501',
|
||||||
'info_dict': {
|
'info_dict': {
|
||||||
'id': '10505354',
|
'id': '51286501',
|
||||||
'ext': 'flv',
|
'title': "Peter Billingsley: From child actor in 'A Christmas Story' to Hollywood power player",
|
||||||
'display_id': 'dramatic-video-rare-death-job-america',
|
'description': 'Billingsley went from a child actor to Hollywood power player.',
|
||||||
'title': 'Occupational Hazards',
|
|
||||||
'description': 'Nightline investigates the dangers that lurk at various jobs.',
|
|
||||||
'thumbnail': r're:^https?://.*\.jpg$',
|
|
||||||
'upload_date': '20100428',
|
|
||||||
'timestamp': 1272412800,
|
|
||||||
},
|
},
|
||||||
'add_ie': ['AbcNewsVideo'],
|
'playlist_count': 5,
|
||||||
}, {
|
}, {
|
||||||
'url': 'http://abcnews.go.com/Entertainment/justin-timberlake-performs-stop-feeling-eurovision-2016/story?id=39125818',
|
'url': 'http://abcnews.go.com/Entertainment/justin-timberlake-performs-stop-feeling-eurovision-2016/story?id=39125818',
|
||||||
'info_dict': {
|
'info_dict': {
|
||||||
'id': '38897857',
|
'id': '38897857',
|
||||||
'ext': 'mp4',
|
'ext': 'mp4',
|
||||||
'display_id': 'justin-timberlake-performs-stop-feeling-eurovision-2016',
|
|
||||||
'title': 'Justin Timberlake Drops Hints For Secret Single',
|
'title': 'Justin Timberlake Drops Hints For Secret Single',
|
||||||
'description': 'Lara Spencer reports the buzziest stories of the day in "GMA" Pop News.',
|
'description': 'Lara Spencer reports the buzziest stories of the day in "GMA" Pop News.',
|
||||||
'upload_date': '20160515',
|
'upload_date': '20160505',
|
||||||
'timestamp': 1463329500,
|
'timestamp': 1462442280,
|
||||||
},
|
},
|
||||||
'params': {
|
'params': {
|
||||||
# m3u8 download
|
# m3u8 download
|
||||||
|
@ -100,49 +104,55 @@ class AbcNewsIE(InfoExtractor):
|
||||||
}, {
|
}, {
|
||||||
'url': 'http://abcnews.go.com/Technology/exclusive-apple-ceo-tim-cook-iphone-cracking-software/story?id=37173343',
|
'url': 'http://abcnews.go.com/Technology/exclusive-apple-ceo-tim-cook-iphone-cracking-software/story?id=37173343',
|
||||||
'only_matching': True,
|
'only_matching': True,
|
||||||
|
}, {
|
||||||
|
# inline.type == 'video'
|
||||||
|
'url': 'http://abcnews.go.com/Technology/exclusive-apple-ceo-tim-cook-iphone-cracking-software/story?id=37173343',
|
||||||
|
'only_matching': True,
|
||||||
}]
|
}]
|
||||||
|
|
||||||
def _real_extract(self, url):
|
def _real_extract(self, url):
|
||||||
mobj = re.match(self._VALID_URL, url)
|
story_id = self._match_id(url)
|
||||||
display_id = mobj.group('display_id')
|
webpage = self._download_webpage(url, story_id)
|
||||||
video_id = mobj.group('id')
|
story = self._parse_json(self._search_regex(
|
||||||
|
r"window\['__abcnews__'\]\s*=\s*({.+?});",
|
||||||
|
webpage, 'data'), story_id)['page']['content']['story']['everscroll'][0]
|
||||||
|
article_contents = story.get('articleContents') or {}
|
||||||
|
|
||||||
webpage = self._download_webpage(url, video_id)
|
def entries():
|
||||||
video_url = self._search_regex(
|
featured_video = story.get('featuredVideo') or {}
|
||||||
r'window\.abcnvideo\.url\s*=\s*"([^"]+)"', webpage, 'video URL')
|
feed = try_get(featured_video, lambda x: x['video']['feed'])
|
||||||
full_video_url = compat_urlparse.urljoin(url, video_url)
|
if feed:
|
||||||
|
yield {
|
||||||
|
'_type': 'url',
|
||||||
|
'id': featured_video.get('id'),
|
||||||
|
'title': featured_video.get('name'),
|
||||||
|
'url': feed,
|
||||||
|
'thumbnail': featured_video.get('images'),
|
||||||
|
'description': featured_video.get('description'),
|
||||||
|
'timestamp': parse_iso8601(featured_video.get('uploadDate')),
|
||||||
|
'duration': parse_duration(featured_video.get('duration')),
|
||||||
|
'ie_key': AbcNewsVideoIE.ie_key(),
|
||||||
|
}
|
||||||
|
|
||||||
youtube_url = YoutubeIE._extract_url(webpage)
|
for inline in (article_contents.get('inlines') or []):
|
||||||
|
inline_type = inline.get('type')
|
||||||
|
if inline_type == 'iframe':
|
||||||
|
iframe_url = try_get(inline, lambda x: x['attrs']['src'])
|
||||||
|
if iframe_url:
|
||||||
|
yield self.url_result(iframe_url)
|
||||||
|
elif inline_type == 'video':
|
||||||
|
video_id = inline.get('id')
|
||||||
|
if video_id:
|
||||||
|
yield {
|
||||||
|
'_type': 'url',
|
||||||
|
'id': video_id,
|
||||||
|
'url': 'http://abcnews.go.com/video/embed?id=' + video_id,
|
||||||
|
'thumbnail': inline.get('imgSrc') or inline.get('imgDefault'),
|
||||||
|
'description': inline.get('description'),
|
||||||
|
'duration': parse_duration(inline.get('duration')),
|
||||||
|
'ie_key': AbcNewsVideoIE.ie_key(),
|
||||||
|
}
|
||||||
|
|
||||||
timestamp = None
|
return self.playlist_result(
|
||||||
date_str = self._html_search_regex(
|
entries(), story_id, article_contents.get('headline'),
|
||||||
r'<span[^>]+class="timestamp">([^<]+)</span>',
|
article_contents.get('subHead'))
|
||||||
webpage, 'timestamp', fatal=False)
|
|
||||||
if date_str:
|
|
||||||
tz_offset = 0
|
|
||||||
if date_str.endswith(' ET'): # Eastern Time
|
|
||||||
tz_offset = -5
|
|
||||||
date_str = date_str[:-3]
|
|
||||||
date_formats = ['%b. %d, %Y', '%b %d, %Y, %I:%M %p']
|
|
||||||
for date_format in date_formats:
|
|
||||||
try:
|
|
||||||
timestamp = calendar.timegm(time.strptime(date_str.strip(), date_format))
|
|
||||||
except ValueError:
|
|
||||||
continue
|
|
||||||
if timestamp is not None:
|
|
||||||
timestamp -= tz_offset * 3600
|
|
||||||
|
|
||||||
entry = {
|
|
||||||
'_type': 'url_transparent',
|
|
||||||
'ie_key': AbcNewsVideoIE.ie_key(),
|
|
||||||
'url': full_video_url,
|
|
||||||
'id': video_id,
|
|
||||||
'display_id': display_id,
|
|
||||||
'timestamp': timestamp,
|
|
||||||
}
|
|
||||||
|
|
||||||
if youtube_url:
|
|
||||||
entries = [entry, self.url_result(youtube_url, ie=YoutubeIE.ie_key())]
|
|
||||||
return self.playlist_result(entries)
|
|
||||||
|
|
||||||
return entry
|
|
||||||
|
|
|
@ -2,21 +2,48 @@
|
||||||
from __future__ import unicode_literals
|
from __future__ import unicode_literals
|
||||||
|
|
||||||
import re
|
import re
|
||||||
import functools
|
|
||||||
|
|
||||||
from .common import InfoExtractor
|
from .common import InfoExtractor
|
||||||
from ..compat import compat_str
|
|
||||||
from ..utils import (
|
from ..utils import (
|
||||||
clean_html,
|
clean_html,
|
||||||
float_or_none,
|
clean_podcast_url,
|
||||||
int_or_none,
|
int_or_none,
|
||||||
try_get,
|
parse_iso8601,
|
||||||
unified_timestamp,
|
|
||||||
OnDemandPagedList,
|
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
class ACastIE(InfoExtractor):
|
class ACastBaseIE(InfoExtractor):
|
||||||
|
def _extract_episode(self, episode, show_info):
|
||||||
|
title = episode['title']
|
||||||
|
info = {
|
||||||
|
'id': episode['id'],
|
||||||
|
'display_id': episode.get('episodeUrl'),
|
||||||
|
'url': clean_podcast_url(episode['url']),
|
||||||
|
'title': title,
|
||||||
|
'description': clean_html(episode.get('description') or episode.get('summary')),
|
||||||
|
'thumbnail': episode.get('image'),
|
||||||
|
'timestamp': parse_iso8601(episode.get('publishDate')),
|
||||||
|
'duration': int_or_none(episode.get('duration')),
|
||||||
|
'filesize': int_or_none(episode.get('contentLength')),
|
||||||
|
'season_number': int_or_none(episode.get('season')),
|
||||||
|
'episode': title,
|
||||||
|
'episode_number': int_or_none(episode.get('episode')),
|
||||||
|
}
|
||||||
|
info.update(show_info)
|
||||||
|
return info
|
||||||
|
|
||||||
|
def _extract_show_info(self, show):
|
||||||
|
return {
|
||||||
|
'creator': show.get('author'),
|
||||||
|
'series': show.get('title'),
|
||||||
|
}
|
||||||
|
|
||||||
|
def _call_api(self, path, video_id, query=None):
|
||||||
|
return self._download_json(
|
||||||
|
'https://feeder.acast.com/api/v1/shows/' + path, video_id, query=query)
|
||||||
|
|
||||||
|
|
||||||
|
class ACastIE(ACastBaseIE):
|
||||||
IE_NAME = 'acast'
|
IE_NAME = 'acast'
|
||||||
_VALID_URL = r'''(?x)
|
_VALID_URL = r'''(?x)
|
||||||
https?://
|
https?://
|
||||||
|
@ -28,15 +55,15 @@ class ACastIE(InfoExtractor):
|
||||||
'''
|
'''
|
||||||
_TESTS = [{
|
_TESTS = [{
|
||||||
'url': 'https://www.acast.com/sparpodcast/2.raggarmordet-rosterurdetforflutna',
|
'url': 'https://www.acast.com/sparpodcast/2.raggarmordet-rosterurdetforflutna',
|
||||||
'md5': '16d936099ec5ca2d5869e3a813ee8dc4',
|
'md5': 'f5598f3ad1e4776fed12ec1407153e4b',
|
||||||
'info_dict': {
|
'info_dict': {
|
||||||
'id': '2a92b283-1a75-4ad8-8396-499c641de0d9',
|
'id': '2a92b283-1a75-4ad8-8396-499c641de0d9',
|
||||||
'ext': 'mp3',
|
'ext': 'mp3',
|
||||||
'title': '2. Raggarmordet - Röster ur det förflutna',
|
'title': '2. Raggarmordet - Röster ur det förflutna',
|
||||||
'description': 'md5:4f81f6d8cf2e12ee21a321d8bca32db4',
|
'description': 'md5:a992ae67f4d98f1c0141598f7bebbf67',
|
||||||
'timestamp': 1477346700,
|
'timestamp': 1477346700,
|
||||||
'upload_date': '20161024',
|
'upload_date': '20161024',
|
||||||
'duration': 2766.602563,
|
'duration': 2766,
|
||||||
'creator': 'Anton Berg & Martin Johnson',
|
'creator': 'Anton Berg & Martin Johnson',
|
||||||
'series': 'Spår',
|
'series': 'Spår',
|
||||||
'episode': '2. Raggarmordet - Röster ur det förflutna',
|
'episode': '2. Raggarmordet - Röster ur det förflutna',
|
||||||
|
@ -45,7 +72,7 @@ class ACastIE(InfoExtractor):
|
||||||
'url': 'http://embed.acast.com/adambuxton/ep.12-adam-joeschristmaspodcast2015',
|
'url': 'http://embed.acast.com/adambuxton/ep.12-adam-joeschristmaspodcast2015',
|
||||||
'only_matching': True,
|
'only_matching': True,
|
||||||
}, {
|
}, {
|
||||||
'url': 'https://play.acast.com/s/rattegangspodden/s04e09-styckmordet-i-helenelund-del-22',
|
'url': 'https://play.acast.com/s/rattegangspodden/s04e09styckmordetihelenelund-del2-2',
|
||||||
'only_matching': True,
|
'only_matching': True,
|
||||||
}, {
|
}, {
|
||||||
'url': 'https://play.acast.com/s/sparpodcast/2a92b283-1a75-4ad8-8396-499c641de0d9',
|
'url': 'https://play.acast.com/s/sparpodcast/2a92b283-1a75-4ad8-8396-499c641de0d9',
|
||||||
|
@ -54,40 +81,14 @@ class ACastIE(InfoExtractor):
|
||||||
|
|
||||||
def _real_extract(self, url):
|
def _real_extract(self, url):
|
||||||
channel, display_id = re.match(self._VALID_URL, url).groups()
|
channel, display_id = re.match(self._VALID_URL, url).groups()
|
||||||
s = self._download_json(
|
episode = self._call_api(
|
||||||
'https://feeder.acast.com/api/v1/shows/%s/episodes/%s' % (channel, display_id),
|
'%s/episodes/%s' % (channel, display_id),
|
||||||
display_id)
|
display_id, {'showInfo': 'true'})
|
||||||
media_url = s['url']
|
return self._extract_episode(
|
||||||
if re.search(r'[0-9a-f]{8}-(?:[0-9a-f]{4}-){3}[0-9a-f]{12}', display_id):
|
episode, self._extract_show_info(episode.get('show') or {}))
|
||||||
episode_url = s.get('episodeUrl')
|
|
||||||
if episode_url:
|
|
||||||
display_id = episode_url
|
|
||||||
else:
|
|
||||||
channel, display_id = re.match(self._VALID_URL, s['link']).groups()
|
|
||||||
cast_data = self._download_json(
|
|
||||||
'https://play-api.acast.com/splash/%s/%s' % (channel, display_id),
|
|
||||||
display_id)['result']
|
|
||||||
e = cast_data['episode']
|
|
||||||
title = e.get('name') or s['title']
|
|
||||||
return {
|
|
||||||
'id': compat_str(e['id']),
|
|
||||||
'display_id': display_id,
|
|
||||||
'url': media_url,
|
|
||||||
'title': title,
|
|
||||||
'description': e.get('summary') or clean_html(e.get('description') or s.get('description')),
|
|
||||||
'thumbnail': e.get('image'),
|
|
||||||
'timestamp': unified_timestamp(e.get('publishingDate') or s.get('publishDate')),
|
|
||||||
'duration': float_or_none(e.get('duration') or s.get('duration')),
|
|
||||||
'filesize': int_or_none(e.get('contentLength')),
|
|
||||||
'creator': try_get(cast_data, lambda x: x['show']['author'], compat_str),
|
|
||||||
'series': try_get(cast_data, lambda x: x['show']['name'], compat_str),
|
|
||||||
'season_number': int_or_none(e.get('seasonNumber')),
|
|
||||||
'episode': title,
|
|
||||||
'episode_number': int_or_none(e.get('episodeNumber')),
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
class ACastChannelIE(InfoExtractor):
|
class ACastChannelIE(ACastBaseIE):
|
||||||
IE_NAME = 'acast:channel'
|
IE_NAME = 'acast:channel'
|
||||||
_VALID_URL = r'''(?x)
|
_VALID_URL = r'''(?x)
|
||||||
https?://
|
https?://
|
||||||
|
@ -102,34 +103,24 @@ class ACastChannelIE(InfoExtractor):
|
||||||
'info_dict': {
|
'info_dict': {
|
||||||
'id': '4efc5294-5385-4847-98bd-519799ce5786',
|
'id': '4efc5294-5385-4847-98bd-519799ce5786',
|
||||||
'title': 'Today in Focus',
|
'title': 'Today in Focus',
|
||||||
'description': 'md5:9ba5564de5ce897faeb12963f4537a64',
|
'description': 'md5:c09ce28c91002ce4ffce71d6504abaae',
|
||||||
},
|
},
|
||||||
'playlist_mincount': 35,
|
'playlist_mincount': 200,
|
||||||
}, {
|
}, {
|
||||||
'url': 'http://play.acast.com/s/ft-banking-weekly',
|
'url': 'http://play.acast.com/s/ft-banking-weekly',
|
||||||
'only_matching': True,
|
'only_matching': True,
|
||||||
}]
|
}]
|
||||||
_API_BASE_URL = 'https://play.acast.com/api/'
|
|
||||||
_PAGE_SIZE = 10
|
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def suitable(cls, url):
|
def suitable(cls, url):
|
||||||
return False if ACastIE.suitable(url) else super(ACastChannelIE, cls).suitable(url)
|
return False if ACastIE.suitable(url) else super(ACastChannelIE, cls).suitable(url)
|
||||||
|
|
||||||
def _fetch_page(self, channel_slug, page):
|
|
||||||
casts = self._download_json(
|
|
||||||
self._API_BASE_URL + 'channels/%s/acasts?page=%s' % (channel_slug, page),
|
|
||||||
channel_slug, note='Download page %d of channel data' % page)
|
|
||||||
for cast in casts:
|
|
||||||
yield self.url_result(
|
|
||||||
'https://play.acast.com/s/%s/%s' % (channel_slug, cast['url']),
|
|
||||||
'ACast', cast['id'])
|
|
||||||
|
|
||||||
def _real_extract(self, url):
|
def _real_extract(self, url):
|
||||||
channel_slug = self._match_id(url)
|
show_slug = self._match_id(url)
|
||||||
channel_data = self._download_json(
|
show = self._call_api(show_slug, show_slug)
|
||||||
self._API_BASE_URL + 'channels/%s' % channel_slug, channel_slug)
|
show_info = self._extract_show_info(show)
|
||||||
entries = OnDemandPagedList(functools.partial(
|
entries = []
|
||||||
self._fetch_page, channel_slug), self._PAGE_SIZE)
|
for episode in (show.get('episodes') or []):
|
||||||
return self.playlist_result(entries, compat_str(
|
entries.append(self._extract_episode(episode, show_info))
|
||||||
channel_data['id']), channel_data['name'], channel_data.get('description'))
|
return self.playlist_result(
|
||||||
|
entries, show.get('id'), show.get('title'), show.get('description'))
|
||||||
|
|
|
@ -10,6 +10,7 @@ import random
|
||||||
from .common import InfoExtractor
|
from .common import InfoExtractor
|
||||||
from ..aes import aes_cbc_decrypt
|
from ..aes import aes_cbc_decrypt
|
||||||
from ..compat import (
|
from ..compat import (
|
||||||
|
compat_HTTPError,
|
||||||
compat_b64decode,
|
compat_b64decode,
|
||||||
compat_ord,
|
compat_ord,
|
||||||
)
|
)
|
||||||
|
@ -18,29 +19,50 @@ from ..utils import (
|
||||||
bytes_to_long,
|
bytes_to_long,
|
||||||
ExtractorError,
|
ExtractorError,
|
||||||
float_or_none,
|
float_or_none,
|
||||||
|
int_or_none,
|
||||||
intlist_to_bytes,
|
intlist_to_bytes,
|
||||||
long_to_bytes,
|
long_to_bytes,
|
||||||
pkcs1pad,
|
pkcs1pad,
|
||||||
strip_or_none,
|
strip_or_none,
|
||||||
urljoin,
|
try_get,
|
||||||
|
unified_strdate,
|
||||||
|
urlencode_postdata,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
class ADNIE(InfoExtractor):
|
class ADNIE(InfoExtractor):
|
||||||
IE_DESC = 'Anime Digital Network'
|
IE_DESC = 'Animation Digital Network'
|
||||||
_VALID_URL = r'https?://(?:www\.)?animedigitalnetwork\.fr/video/[^/]+/(?P<id>\d+)'
|
_VALID_URL = r'https?://(?:www\.)?(?:animation|anime)digitalnetwork\.fr/video/[^/]+/(?P<id>\d+)'
|
||||||
_TEST = {
|
_TESTS = [{
|
||||||
'url': 'http://animedigitalnetwork.fr/video/blue-exorcist-kyoto-saga/7778-episode-1-debut-des-hostilites',
|
'url': 'https://animationdigitalnetwork.fr/video/fruits-basket/9841-episode-1-a-ce-soir',
|
||||||
'md5': 'e497370d847fd79d9d4c74be55575c7a',
|
'md5': '1c9ef066ceb302c86f80c2b371615261',
|
||||||
'info_dict': {
|
'info_dict': {
|
||||||
'id': '7778',
|
'id': '9841',
|
||||||
'ext': 'mp4',
|
'ext': 'mp4',
|
||||||
'title': 'Blue Exorcist - Kyôto Saga - Épisode 1',
|
'title': 'Fruits Basket - Episode 1',
|
||||||
'description': 'md5:2f7b5aa76edbc1a7a92cedcda8a528d5',
|
'description': 'md5:14be2f72c3c96809b0ca424b0097d336',
|
||||||
}
|
'series': 'Fruits Basket',
|
||||||
}
|
'duration': 1437,
|
||||||
_BASE_URL = 'http://animedigitalnetwork.fr'
|
'release_date': '20190405',
|
||||||
_RSA_KEY = (0xc35ae1e4356b65a73b551493da94b8cb443491c0aa092a357a5aee57ffc14dda85326f42d716e539a34542a0d3f363adf16c5ec222d713d5997194030ee2e4f0d1fb328c01a81cf6868c090d50de8e169c6b13d1675b9eeed1cbc51e1fffca9b38af07f37abd790924cd3bee59d0257cfda4fe5f3f0534877e21ce5821447d1b, 65537)
|
'comment_count': int,
|
||||||
|
'average_rating': float,
|
||||||
|
'season_number': 1,
|
||||||
|
'episode': 'À ce soir !',
|
||||||
|
'episode_number': 1,
|
||||||
|
},
|
||||||
|
'skip': 'Only available in region (FR, ...)',
|
||||||
|
}, {
|
||||||
|
'url': 'http://animedigitalnetwork.fr/video/blue-exorcist-kyoto-saga/7778-episode-1-debut-des-hostilites',
|
||||||
|
'only_matching': True,
|
||||||
|
}]
|
||||||
|
|
||||||
|
_NETRC_MACHINE = 'animationdigitalnetwork'
|
||||||
|
_BASE = 'animationdigitalnetwork.fr'
|
||||||
|
_API_BASE_URL = 'https://gw.api.' + _BASE + '/'
|
||||||
|
_PLAYER_BASE_URL = _API_BASE_URL + 'player/'
|
||||||
|
_HEADERS = {}
|
||||||
|
_LOGIN_ERR_MESSAGE = 'Unable to log in'
|
||||||
|
_RSA_KEY = (0x9B42B08905199A5CCE2026274399CA560ECB209EE9878A708B1C0812E1BB8CB5D1FB7441861147C1A1F2F3A0476DD63A9CAC20D3E983613346850AA6CB38F16DC7D720FD7D86FC6E5B3D5BBC72E14CD0BF9E869F2CEA2CCAD648F1DCE38F1FF916CEFB2D339B64AA0264372344BC775E265E8A852F88144AB0BD9AA06C1A4ABB, 65537)
|
||||||
_POS_ALIGN_MAP = {
|
_POS_ALIGN_MAP = {
|
||||||
'start': 1,
|
'start': 1,
|
||||||
'end': 3,
|
'end': 3,
|
||||||
|
@ -54,26 +76,24 @@ class ADNIE(InfoExtractor):
|
||||||
def _ass_subtitles_timecode(seconds):
|
def _ass_subtitles_timecode(seconds):
|
||||||
return '%01d:%02d:%02d.%02d' % (seconds / 3600, (seconds % 3600) / 60, seconds % 60, (seconds % 1) * 100)
|
return '%01d:%02d:%02d.%02d' % (seconds / 3600, (seconds % 3600) / 60, seconds % 60, (seconds % 1) * 100)
|
||||||
|
|
||||||
def _get_subtitles(self, sub_path, video_id):
|
def _get_subtitles(self, sub_url, video_id):
|
||||||
if not sub_path:
|
if not sub_url:
|
||||||
return None
|
return None
|
||||||
|
|
||||||
enc_subtitles = self._download_webpage(
|
enc_subtitles = self._download_webpage(
|
||||||
urljoin(self._BASE_URL, sub_path),
|
sub_url, video_id, 'Downloading subtitles location', fatal=False) or '{}'
|
||||||
video_id, 'Downloading subtitles location', fatal=False) or '{}'
|
|
||||||
subtitle_location = (self._parse_json(enc_subtitles, video_id, fatal=False) or {}).get('location')
|
subtitle_location = (self._parse_json(enc_subtitles, video_id, fatal=False) or {}).get('location')
|
||||||
if subtitle_location:
|
if subtitle_location:
|
||||||
enc_subtitles = self._download_webpage(
|
enc_subtitles = self._download_webpage(
|
||||||
urljoin(self._BASE_URL, subtitle_location),
|
subtitle_location, video_id, 'Downloading subtitles data',
|
||||||
video_id, 'Downloading subtitles data', fatal=False,
|
fatal=False, headers={'Origin': 'https://' + self._BASE})
|
||||||
headers={'Origin': 'https://animedigitalnetwork.fr'})
|
|
||||||
if not enc_subtitles:
|
if not enc_subtitles:
|
||||||
return None
|
return None
|
||||||
|
|
||||||
# http://animedigitalnetwork.fr/components/com_vodvideo/videojs/adn-vjs.min.js
|
# http://animationdigitalnetwork.fr/components/com_vodvideo/videojs/adn-vjs.min.js
|
||||||
dec_subtitles = intlist_to_bytes(aes_cbc_decrypt(
|
dec_subtitles = intlist_to_bytes(aes_cbc_decrypt(
|
||||||
bytes_to_intlist(compat_b64decode(enc_subtitles[24:])),
|
bytes_to_intlist(compat_b64decode(enc_subtitles[24:])),
|
||||||
bytes_to_intlist(binascii.unhexlify(self._K + '4b8ef13ec1872730')),
|
bytes_to_intlist(binascii.unhexlify(self._K + '7fac1178830cfe0c')),
|
||||||
bytes_to_intlist(compat_b64decode(enc_subtitles[:24]))
|
bytes_to_intlist(compat_b64decode(enc_subtitles[:24]))
|
||||||
))
|
))
|
||||||
subtitles_json = self._parse_json(
|
subtitles_json = self._parse_json(
|
||||||
|
@ -117,61 +137,103 @@ Format: Marked,Start,End,Style,Name,MarginL,MarginR,MarginV,Effect,Text'''
|
||||||
}])
|
}])
|
||||||
return subtitles
|
return subtitles
|
||||||
|
|
||||||
|
def _real_initialize(self):
|
||||||
|
username, password = self._get_login_info()
|
||||||
|
if not username:
|
||||||
|
return
|
||||||
|
try:
|
||||||
|
url = self._API_BASE_URL + 'authentication/login'
|
||||||
|
access_token = (self._download_json(
|
||||||
|
url, None, 'Logging in', self._LOGIN_ERR_MESSAGE, fatal=False,
|
||||||
|
data=urlencode_postdata({
|
||||||
|
'password': password,
|
||||||
|
'rememberMe': False,
|
||||||
|
'source': 'Web',
|
||||||
|
'username': username,
|
||||||
|
})) or {}).get('accessToken')
|
||||||
|
if access_token:
|
||||||
|
self._HEADERS = {'authorization': 'Bearer ' + access_token}
|
||||||
|
except ExtractorError as e:
|
||||||
|
message = None
|
||||||
|
if isinstance(e.cause, compat_HTTPError) and e.cause.code == 401:
|
||||||
|
resp = self._parse_json(
|
||||||
|
self._webpage_read_content(e.cause, url, username),
|
||||||
|
username, fatal=False) or {}
|
||||||
|
message = resp.get('message') or resp.get('code')
|
||||||
|
self.report_warning(message or self._LOGIN_ERR_MESSAGE)
|
||||||
|
|
||||||
def _real_extract(self, url):
|
def _real_extract(self, url):
|
||||||
video_id = self._match_id(url)
|
video_id = self._match_id(url)
|
||||||
webpage = self._download_webpage(url, video_id)
|
video_base_url = self._PLAYER_BASE_URL + 'video/%s/' % video_id
|
||||||
player_config = self._parse_json(self._search_regex(
|
player = self._download_json(
|
||||||
r'playerConfig\s*=\s*({.+});', webpage,
|
video_base_url + 'configuration', video_id,
|
||||||
'player config', default='{}'), video_id, fatal=False)
|
'Downloading player config JSON metadata',
|
||||||
if not player_config:
|
headers=self._HEADERS)['player']
|
||||||
config_url = urljoin(self._BASE_URL, self._search_regex(
|
options = player['options']
|
||||||
r'(?:id="player"|class="[^"]*adn-player-container[^"]*")[^>]+data-url="([^"]+)"',
|
|
||||||
webpage, 'config url'))
|
|
||||||
player_config = self._download_json(
|
|
||||||
config_url, video_id,
|
|
||||||
'Downloading player config JSON metadata')['player']
|
|
||||||
|
|
||||||
video_info = {}
|
user = options['user']
|
||||||
video_info_str = self._search_regex(
|
if not user.get('hasAccess'):
|
||||||
r'videoInfo\s*=\s*({.+});', webpage,
|
self.raise_login_required()
|
||||||
'video info', fatal=False)
|
|
||||||
if video_info_str:
|
|
||||||
video_info = self._parse_json(
|
|
||||||
video_info_str, video_id, fatal=False) or {}
|
|
||||||
|
|
||||||
options = player_config.get('options') or {}
|
token = self._download_json(
|
||||||
metas = options.get('metas') or {}
|
user.get('refreshTokenUrl') or (self._PLAYER_BASE_URL + 'refresh/token'),
|
||||||
links = player_config.get('links') or {}
|
video_id, 'Downloading access token', headers={
|
||||||
sub_path = player_config.get('subtitles')
|
'x-player-refresh-token': user['refreshToken']
|
||||||
error = None
|
}, data=b'')['token']
|
||||||
if not links:
|
|
||||||
links_url = player_config.get('linksurl') or options['videoUrl']
|
links_url = try_get(options, lambda x: x['video']['url']) or (video_base_url + 'link')
|
||||||
token = options['token']
|
self._K = ''.join([random.choice('0123456789abcdef') for _ in range(16)])
|
||||||
self._K = ''.join([random.choice('0123456789abcdef') for _ in range(16)])
|
message = bytes_to_intlist(json.dumps({
|
||||||
message = bytes_to_intlist(json.dumps({
|
'k': self._K,
|
||||||
'k': self._K,
|
't': token,
|
||||||
'e': 60,
|
}))
|
||||||
't': token,
|
|
||||||
}))
|
# Sometimes authentication fails for no good reason, retry with
|
||||||
|
# a different random padding
|
||||||
|
links_data = None
|
||||||
|
for _ in range(3):
|
||||||
padded_message = intlist_to_bytes(pkcs1pad(message, 128))
|
padded_message = intlist_to_bytes(pkcs1pad(message, 128))
|
||||||
n, e = self._RSA_KEY
|
n, e = self._RSA_KEY
|
||||||
encrypted_message = long_to_bytes(pow(bytes_to_long(padded_message), e, n))
|
encrypted_message = long_to_bytes(pow(bytes_to_long(padded_message), e, n))
|
||||||
authorization = base64.b64encode(encrypted_message).decode()
|
authorization = base64.b64encode(encrypted_message).decode()
|
||||||
links_data = self._download_json(
|
|
||||||
urljoin(self._BASE_URL, links_url), video_id,
|
try:
|
||||||
'Downloading links JSON metadata', headers={
|
links_data = self._download_json(
|
||||||
'Authorization': 'Bearer ' + authorization,
|
links_url, video_id, 'Downloading links JSON metadata', headers={
|
||||||
})
|
'X-Player-Token': authorization
|
||||||
links = links_data.get('links') or {}
|
}, query={
|
||||||
metas = metas or links_data.get('meta') or {}
|
'freeWithAds': 'true',
|
||||||
sub_path = sub_path or links_data.get('subtitles') or \
|
'adaptive': 'false',
|
||||||
'index.php?option=com_vodapi&task=subtitles.getJSON&format=json&id=' + video_id
|
'withMetadata': 'true',
|
||||||
sub_path += '&token=' + token
|
'source': 'Web'
|
||||||
error = links_data.get('error')
|
})
|
||||||
title = metas.get('title') or video_info['title']
|
break
|
||||||
|
except ExtractorError as e:
|
||||||
|
if not isinstance(e.cause, compat_HTTPError):
|
||||||
|
raise e
|
||||||
|
|
||||||
|
if e.cause.code == 401:
|
||||||
|
# This usually goes away with a different random pkcs1pad, so retry
|
||||||
|
continue
|
||||||
|
|
||||||
|
error = self._parse_json(
|
||||||
|
self._webpage_read_content(e.cause, links_url, video_id),
|
||||||
|
video_id, fatal=False) or {}
|
||||||
|
message = error.get('message')
|
||||||
|
if e.cause.code == 403 and error.get('code') == 'player-bad-geolocation-country':
|
||||||
|
self.raise_geo_restricted(msg=message)
|
||||||
|
raise ExtractorError(message)
|
||||||
|
else:
|
||||||
|
raise ExtractorError('Giving up retrying')
|
||||||
|
|
||||||
|
links = links_data.get('links') or {}
|
||||||
|
metas = links_data.get('metadata') or {}
|
||||||
|
sub_url = (links.get('subtitles') or {}).get('all')
|
||||||
|
video_info = links_data.get('video') or {}
|
||||||
|
title = metas['title']
|
||||||
|
|
||||||
formats = []
|
formats = []
|
||||||
for format_id, qualities in links.items():
|
for format_id, qualities in (links.get('streaming') or {}).items():
|
||||||
if not isinstance(qualities, dict):
|
if not isinstance(qualities, dict):
|
||||||
continue
|
continue
|
||||||
for quality, load_balancer_url in qualities.items():
|
for quality, load_balancer_url in qualities.items():
|
||||||
|
@ -189,19 +251,26 @@ Format: Marked,Start,End,Style,Name,MarginL,MarginR,MarginV,Effect,Text'''
|
||||||
for f in m3u8_formats:
|
for f in m3u8_formats:
|
||||||
f['language'] = 'fr'
|
f['language'] = 'fr'
|
||||||
formats.extend(m3u8_formats)
|
formats.extend(m3u8_formats)
|
||||||
if not error:
|
|
||||||
error = options.get('error')
|
|
||||||
if not formats and error:
|
|
||||||
raise ExtractorError('%s said: %s' % (self.IE_NAME, error), expected=True)
|
|
||||||
self._sort_formats(formats)
|
self._sort_formats(formats)
|
||||||
|
|
||||||
|
video = (self._download_json(
|
||||||
|
self._API_BASE_URL + 'video/%s' % video_id, video_id,
|
||||||
|
'Downloading additional video metadata', fatal=False) or {}).get('video') or {}
|
||||||
|
show = video.get('show') or {}
|
||||||
|
|
||||||
return {
|
return {
|
||||||
'id': video_id,
|
'id': video_id,
|
||||||
'title': title,
|
'title': title,
|
||||||
'description': strip_or_none(metas.get('summary') or video_info.get('resume')),
|
'description': strip_or_none(metas.get('summary') or video.get('summary')),
|
||||||
'thumbnail': video_info.get('image'),
|
'thumbnail': video_info.get('image') or player.get('image'),
|
||||||
'formats': formats,
|
'formats': formats,
|
||||||
'subtitles': self.extract_subtitles(sub_path, video_id),
|
'subtitles': self.extract_subtitles(sub_url, video_id),
|
||||||
'episode': metas.get('subtitle') or video_info.get('videoTitle'),
|
'episode': metas.get('subtitle') or video.get('name'),
|
||||||
'series': video_info.get('playlistTitle'),
|
'episode_number': int_or_none(video.get('shortNumber')),
|
||||||
|
'series': show.get('title'),
|
||||||
|
'season_number': int_or_none(video.get('season')),
|
||||||
|
'duration': int_or_none(video_info.get('duration') or video.get('duration')),
|
||||||
|
'release_date': unified_strdate(video.get('releaseDate')),
|
||||||
|
'average_rating': float_or_none(video.get('rating') or metas.get('rating')),
|
||||||
|
'comment_count': int_or_none(video.get('commentsCount')),
|
||||||
}
|
}
|
||||||
|
|
|
@ -5,33 +5,50 @@ import re
|
||||||
|
|
||||||
from .theplatform import ThePlatformIE
|
from .theplatform import ThePlatformIE
|
||||||
from ..utils import (
|
from ..utils import (
|
||||||
extract_attributes,
|
|
||||||
ExtractorError,
|
ExtractorError,
|
||||||
|
GeoRestrictedError,
|
||||||
int_or_none,
|
int_or_none,
|
||||||
smuggle_url,
|
remove_start,
|
||||||
|
traverse_obj,
|
||||||
update_url_query,
|
update_url_query,
|
||||||
)
|
urlencode_postdata,
|
||||||
from ..compat import (
|
|
||||||
compat_urlparse,
|
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
class AENetworksBaseIE(ThePlatformIE):
|
class AENetworksBaseIE(ThePlatformIE):
|
||||||
_THEPLATFORM_KEY = 'crazyjava'
|
_BASE_URL_REGEX = r'''(?x)https?://
|
||||||
_THEPLATFORM_SECRET = 's3cr3t'
|
(?:(?:www|play|watch)\.)?
|
||||||
|
(?P<domain>
|
||||||
|
(?:history(?:vault)?|aetv|mylifetime|lifetimemovieclub)\.com|
|
||||||
|
fyi\.tv
|
||||||
|
)/'''
|
||||||
|
_THEPLATFORM_KEY = '43jXaGRQud'
|
||||||
|
_THEPLATFORM_SECRET = 'S10BPXHMlb'
|
||||||
|
_DOMAIN_MAP = {
|
||||||
|
'history.com': ('HISTORY', 'history'),
|
||||||
|
'aetv.com': ('AETV', 'aetv'),
|
||||||
|
'mylifetime.com': ('LIFETIME', 'lifetime'),
|
||||||
|
'lifetimemovieclub.com': ('LIFETIMEMOVIECLUB', 'lmc'),
|
||||||
|
'fyi.tv': ('FYI', 'fyi'),
|
||||||
|
'historyvault.com': (None, 'historyvault'),
|
||||||
|
'biography.com': (None, 'biography'),
|
||||||
|
}
|
||||||
|
|
||||||
def _extract_aen_smil(self, smil_url, video_id, auth=None):
|
def _extract_aen_smil(self, smil_url, video_id, auth=None):
|
||||||
query = {'mbr': 'true'}
|
query = {
|
||||||
|
'mbr': 'true',
|
||||||
|
'formats': 'M3U+none,MPEG-DASH+none,MPEG4,MP3',
|
||||||
|
}
|
||||||
if auth:
|
if auth:
|
||||||
query['auth'] = auth
|
query['auth'] = auth
|
||||||
TP_SMIL_QUERY = [{
|
TP_SMIL_QUERY = [{
|
||||||
'assetTypes': 'high_video_ak',
|
'assetTypes': 'high_video_ak',
|
||||||
'switch': 'hls_high_ak'
|
'switch': 'hls_high_ak',
|
||||||
}, {
|
|
||||||
'assetTypes': 'high_video_s3'
|
|
||||||
}, {
|
}, {
|
||||||
'assetTypes': 'high_video_s3',
|
'assetTypes': 'high_video_s3',
|
||||||
'switch': 'hls_ingest_fastly'
|
}, {
|
||||||
|
'assetTypes': 'high_video_s3',
|
||||||
|
'switch': 'hls_high_fastly',
|
||||||
}]
|
}]
|
||||||
formats = []
|
formats = []
|
||||||
subtitles = {}
|
subtitles = {}
|
||||||
|
@ -44,6 +61,8 @@ class AENetworksBaseIE(ThePlatformIE):
|
||||||
tp_formats, tp_subtitles = self._extract_theplatform_smil(
|
tp_formats, tp_subtitles = self._extract_theplatform_smil(
|
||||||
m_url, video_id, 'Downloading %s SMIL data' % (q.get('switch') or q['assetTypes']))
|
m_url, video_id, 'Downloading %s SMIL data' % (q.get('switch') or q['assetTypes']))
|
||||||
except ExtractorError as e:
|
except ExtractorError as e:
|
||||||
|
if isinstance(e, GeoRestrictedError):
|
||||||
|
raise
|
||||||
last_e = e
|
last_e = e
|
||||||
continue
|
continue
|
||||||
formats.extend(tp_formats)
|
formats.extend(tp_formats)
|
||||||
|
@ -57,24 +76,52 @@ class AENetworksBaseIE(ThePlatformIE):
|
||||||
'subtitles': subtitles,
|
'subtitles': subtitles,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
def _extract_aetn_info(self, domain, filter_key, filter_value, url):
|
||||||
|
requestor_id, brand = self._DOMAIN_MAP[domain]
|
||||||
|
result = self._download_json(
|
||||||
|
'https://feeds.video.aetnd.com/api/v2/%s/videos' % brand,
|
||||||
|
filter_value, query={'filter[%s]' % filter_key: filter_value})
|
||||||
|
result = traverse_obj(
|
||||||
|
result, ('results',
|
||||||
|
lambda k, v: k == 0 and v[filter_key] == filter_value),
|
||||||
|
get_all=False)
|
||||||
|
if not result:
|
||||||
|
raise ExtractorError('Show not found in A&E feed (too new?)', expected=True,
|
||||||
|
video_id=remove_start(filter_value, '/'))
|
||||||
|
title = result['title']
|
||||||
|
video_id = result['id']
|
||||||
|
media_url = result['publicUrl']
|
||||||
|
theplatform_metadata = self._download_theplatform_metadata(self._search_regex(
|
||||||
|
r'https?://link\.theplatform\.com/s/([^?]+)', media_url, 'theplatform_path'), video_id)
|
||||||
|
info = self._parse_theplatform_metadata(theplatform_metadata)
|
||||||
|
auth = None
|
||||||
|
if theplatform_metadata.get('AETN$isBehindWall'):
|
||||||
|
resource = self._get_mvpd_resource(
|
||||||
|
requestor_id, theplatform_metadata['title'],
|
||||||
|
theplatform_metadata.get('AETN$PPL_pplProgramId') or theplatform_metadata.get('AETN$PPL_pplProgramId_OLD'),
|
||||||
|
theplatform_metadata['ratings'][0]['rating'])
|
||||||
|
auth = self._extract_mvpd_auth(
|
||||||
|
url, video_id, requestor_id, resource)
|
||||||
|
info.update(self._extract_aen_smil(media_url, video_id, auth))
|
||||||
|
info.update({
|
||||||
|
'title': title,
|
||||||
|
'series': result.get('seriesName'),
|
||||||
|
'season_number': int_or_none(result.get('tvSeasonNumber')),
|
||||||
|
'episode_number': int_or_none(result.get('tvSeasonEpisodeNumber')),
|
||||||
|
})
|
||||||
|
return info
|
||||||
|
|
||||||
|
|
||||||
class AENetworksIE(AENetworksBaseIE):
|
class AENetworksIE(AENetworksBaseIE):
|
||||||
IE_NAME = 'aenetworks'
|
IE_NAME = 'aenetworks'
|
||||||
IE_DESC = 'A+E Networks: A&E, Lifetime, History.com, FYI Network and History Vault'
|
IE_DESC = 'A+E Networks: A&E, Lifetime, History.com, FYI Network and History Vault'
|
||||||
_VALID_URL = r'''(?x)
|
_VALID_URL = AENetworksBaseIE._BASE_URL_REGEX + r'''(?P<id>
|
||||||
https?://
|
shows/[^/]+/season-\d+/episode-\d+|
|
||||||
(?:www\.)?
|
(?:
|
||||||
(?P<domain>
|
(?:movie|special)s/[^/]+|
|
||||||
(?:history(?:vault)?|aetv|mylifetime|lifetimemovieclub)\.com|
|
(?:shows/[^/]+/)?videos
|
||||||
fyi\.tv
|
)/[^/?#&]+
|
||||||
)/
|
)'''
|
||||||
(?:
|
|
||||||
shows/(?P<show_path>[^/]+(?:/[^/]+){0,2})|
|
|
||||||
movies/(?P<movie_display_id>[^/]+)(?:/full-movie)?|
|
|
||||||
specials/(?P<special_display_id>[^/]+)/(?:full-special|preview-)|
|
|
||||||
collections/[^/]+/(?P<collection_display_id>[^/]+)
|
|
||||||
)
|
|
||||||
'''
|
|
||||||
_TESTS = [{
|
_TESTS = [{
|
||||||
'url': 'http://www.history.com/shows/mountain-men/season-1/episode-1',
|
'url': 'http://www.history.com/shows/mountain-men/season-1/episode-1',
|
||||||
'info_dict': {
|
'info_dict': {
|
||||||
|
@ -91,22 +138,24 @@ class AENetworksIE(AENetworksBaseIE):
|
||||||
'skip_download': True,
|
'skip_download': True,
|
||||||
},
|
},
|
||||||
'add_ie': ['ThePlatform'],
|
'add_ie': ['ThePlatform'],
|
||||||
}, {
|
'skip': 'Geo-restricted - This content is not available in your location.'
|
||||||
'url': 'http://www.history.com/shows/ancient-aliens/season-1',
|
|
||||||
'info_dict': {
|
|
||||||
'id': '71889446852',
|
|
||||||
},
|
|
||||||
'playlist_mincount': 5,
|
|
||||||
}, {
|
|
||||||
'url': 'http://www.mylifetime.com/shows/atlanta-plastic',
|
|
||||||
'info_dict': {
|
|
||||||
'id': 'SERIES4317',
|
|
||||||
'title': 'Atlanta Plastic',
|
|
||||||
},
|
|
||||||
'playlist_mincount': 2,
|
|
||||||
}, {
|
}, {
|
||||||
'url': 'http://www.aetv.com/shows/duck-dynasty/season-9/episode-1',
|
'url': 'http://www.aetv.com/shows/duck-dynasty/season-9/episode-1',
|
||||||
'only_matching': True
|
'info_dict': {
|
||||||
|
'id': '600587331957',
|
||||||
|
'ext': 'mp4',
|
||||||
|
'title': 'Inlawful Entry',
|
||||||
|
'description': 'md5:57c12115a2b384d883fe64ca50529e08',
|
||||||
|
'timestamp': 1452634428,
|
||||||
|
'upload_date': '20160112',
|
||||||
|
'uploader': 'AENE-NEW',
|
||||||
|
},
|
||||||
|
'params': {
|
||||||
|
# m3u8 download
|
||||||
|
'skip_download': True,
|
||||||
|
},
|
||||||
|
'add_ie': ['ThePlatform'],
|
||||||
|
'skip': 'This video is only available for users of participating TV providers.',
|
||||||
}, {
|
}, {
|
||||||
'url': 'http://www.fyi.tv/shows/tiny-house-nation/season-1/episode-8',
|
'url': 'http://www.fyi.tv/shows/tiny-house-nation/season-1/episode-8',
|
||||||
'only_matching': True
|
'only_matching': True
|
||||||
|
@ -117,78 +166,125 @@ class AENetworksIE(AENetworksBaseIE):
|
||||||
'url': 'http://www.mylifetime.com/movies/center-stage-on-pointe/full-movie',
|
'url': 'http://www.mylifetime.com/movies/center-stage-on-pointe/full-movie',
|
||||||
'only_matching': True
|
'only_matching': True
|
||||||
}, {
|
}, {
|
||||||
'url': 'https://www.lifetimemovieclub.com/movies/a-killer-among-us',
|
'url': 'https://watch.lifetimemovieclub.com/movies/10-year-reunion/full-movie',
|
||||||
'only_matching': True
|
'only_matching': True
|
||||||
}, {
|
}, {
|
||||||
'url': 'http://www.history.com/specials/sniper-into-the-kill-zone/full-special',
|
'url': 'http://www.history.com/specials/sniper-into-the-kill-zone/full-special',
|
||||||
'only_matching': True
|
'only_matching': True
|
||||||
}, {
|
|
||||||
'url': 'https://www.historyvault.com/collections/america-the-story-of-us/westward',
|
|
||||||
'only_matching': True
|
|
||||||
}, {
|
}, {
|
||||||
'url': 'https://www.aetv.com/specials/hunting-jonbenets-killer-the-untold-story/preview-hunting-jonbenets-killer-the-untold-story',
|
'url': 'https://www.aetv.com/specials/hunting-jonbenets-killer-the-untold-story/preview-hunting-jonbenets-killer-the-untold-story',
|
||||||
'only_matching': True
|
'only_matching': True
|
||||||
|
}, {
|
||||||
|
'url': 'http://www.history.com/videos/history-of-valentines-day',
|
||||||
|
'only_matching': True
|
||||||
|
}, {
|
||||||
|
'url': 'https://play.aetv.com/shows/duck-dynasty/videos/best-of-duck-dynasty-getting-quack-in-shape',
|
||||||
|
'only_matching': True
|
||||||
}]
|
}]
|
||||||
_DOMAIN_TO_REQUESTOR_ID = {
|
|
||||||
'history.com': 'HISTORY',
|
|
||||||
'aetv.com': 'AETV',
|
|
||||||
'mylifetime.com': 'LIFETIME',
|
|
||||||
'lifetimemovieclub.com': 'LIFETIMEMOVIECLUB',
|
|
||||||
'fyi.tv': 'FYI',
|
|
||||||
}
|
|
||||||
|
|
||||||
def _real_extract(self, url):
|
def _real_extract(self, url):
|
||||||
domain, show_path, movie_display_id, special_display_id, collection_display_id = re.match(self._VALID_URL, url).groups()
|
domain, canonical = re.match(self._VALID_URL, url).groups()
|
||||||
display_id = show_path or movie_display_id or special_display_id or collection_display_id
|
return self._extract_aetn_info(domain, 'canonical', '/' + canonical, url)
|
||||||
webpage = self._download_webpage(url, display_id, headers=self.geo_verification_headers())
|
|
||||||
if show_path:
|
|
||||||
url_parts = show_path.split('/')
|
|
||||||
url_parts_len = len(url_parts)
|
|
||||||
if url_parts_len == 1:
|
|
||||||
entries = []
|
|
||||||
for season_url_path in re.findall(r'(?s)<li[^>]+data-href="(/shows/%s/season-\d+)"' % url_parts[0], webpage):
|
|
||||||
entries.append(self.url_result(
|
|
||||||
compat_urlparse.urljoin(url, season_url_path), 'AENetworks'))
|
|
||||||
if entries:
|
|
||||||
return self.playlist_result(
|
|
||||||
entries, self._html_search_meta('aetn:SeriesId', webpage),
|
|
||||||
self._html_search_meta('aetn:SeriesTitle', webpage))
|
|
||||||
else:
|
|
||||||
# single season
|
|
||||||
url_parts_len = 2
|
|
||||||
if url_parts_len == 2:
|
|
||||||
entries = []
|
|
||||||
for episode_item in re.findall(r'(?s)<[^>]+class="[^"]*(?:episode|program)-item[^"]*"[^>]*>', webpage):
|
|
||||||
episode_attributes = extract_attributes(episode_item)
|
|
||||||
episode_url = compat_urlparse.urljoin(
|
|
||||||
url, episode_attributes['data-canonical'])
|
|
||||||
entries.append(self.url_result(
|
|
||||||
episode_url, 'AENetworks',
|
|
||||||
episode_attributes.get('data-videoid') or episode_attributes.get('data-video-id')))
|
|
||||||
return self.playlist_result(
|
|
||||||
entries, self._html_search_meta('aetn:SeasonId', webpage))
|
|
||||||
|
|
||||||
video_id = self._html_search_meta('aetn:VideoID', webpage)
|
|
||||||
media_url = self._search_regex(
|
class AENetworksListBaseIE(AENetworksBaseIE):
|
||||||
[r"media_url\s*=\s*'(?P<url>[^']+)'",
|
def _call_api(self, resource, slug, brand, fields):
|
||||||
r'data-media-url=(?P<url>(?:https?:)?//[^\s>]+)',
|
return self._download_json(
|
||||||
r'data-media-url=(["\'])(?P<url>(?:(?!\1).)+?)\1'],
|
'https://yoga.appsvcs.aetnd.com/graphql',
|
||||||
webpage, 'video url', group='url')
|
slug, query={'brand': brand}, data=urlencode_postdata({
|
||||||
theplatform_metadata = self._download_theplatform_metadata(self._search_regex(
|
'query': '''{
|
||||||
r'https?://link\.theplatform\.com/s/([^?]+)', media_url, 'theplatform_path'), video_id)
|
%s(slug: "%s") {
|
||||||
info = self._parse_theplatform_metadata(theplatform_metadata)
|
%s
|
||||||
auth = None
|
}
|
||||||
if theplatform_metadata.get('AETN$isBehindWall'):
|
}''' % (resource, slug, fields),
|
||||||
requestor_id = self._DOMAIN_TO_REQUESTOR_ID[domain]
|
}))['data'][resource]
|
||||||
resource = self._get_mvpd_resource(
|
|
||||||
requestor_id, theplatform_metadata['title'],
|
def _real_extract(self, url):
|
||||||
theplatform_metadata.get('AETN$PPL_pplProgramId') or theplatform_metadata.get('AETN$PPL_pplProgramId_OLD'),
|
domain, slug = re.match(self._VALID_URL, url).groups()
|
||||||
theplatform_metadata['ratings'][0]['rating'])
|
_, brand = self._DOMAIN_MAP[domain]
|
||||||
auth = self._extract_mvpd_auth(
|
playlist = self._call_api(self._RESOURCE, slug, brand, self._FIELDS)
|
||||||
url, video_id, requestor_id, resource)
|
base_url = 'http://watch.%s' % domain
|
||||||
info.update(self._search_json_ld(webpage, video_id, fatal=False))
|
|
||||||
info.update(self._extract_aen_smil(media_url, video_id, auth))
|
entries = []
|
||||||
return info
|
for item in (playlist.get(self._ITEMS_KEY) or []):
|
||||||
|
doc = self._get_doc(item)
|
||||||
|
canonical = doc.get('canonical')
|
||||||
|
if not canonical:
|
||||||
|
continue
|
||||||
|
entries.append(self.url_result(
|
||||||
|
base_url + canonical, AENetworksIE.ie_key(), doc.get('id')))
|
||||||
|
|
||||||
|
description = None
|
||||||
|
if self._PLAYLIST_DESCRIPTION_KEY:
|
||||||
|
description = playlist.get(self._PLAYLIST_DESCRIPTION_KEY)
|
||||||
|
|
||||||
|
return self.playlist_result(
|
||||||
|
entries, playlist.get('id'),
|
||||||
|
playlist.get(self._PLAYLIST_TITLE_KEY), description)
|
||||||
|
|
||||||
|
|
||||||
|
class AENetworksCollectionIE(AENetworksListBaseIE):
|
||||||
|
IE_NAME = 'aenetworks:collection'
|
||||||
|
_VALID_URL = AENetworksBaseIE._BASE_URL_REGEX + r'(?:[^/]+/)*(?:list|collections)/(?P<id>[^/?#&]+)/?(?:[?#&]|$)'
|
||||||
|
_TESTS = [{
|
||||||
|
'url': 'https://watch.historyvault.com/list/america-the-story-of-us',
|
||||||
|
'info_dict': {
|
||||||
|
'id': '282',
|
||||||
|
'title': 'America The Story of Us',
|
||||||
|
},
|
||||||
|
'playlist_mincount': 12,
|
||||||
|
}, {
|
||||||
|
'url': 'https://watch.historyvault.com/shows/america-the-story-of-us-2/season-1/list/america-the-story-of-us',
|
||||||
|
'only_matching': True
|
||||||
|
}, {
|
||||||
|
'url': 'https://www.historyvault.com/collections/mysteryquest',
|
||||||
|
'only_matching': True
|
||||||
|
}]
|
||||||
|
_RESOURCE = 'list'
|
||||||
|
_ITEMS_KEY = 'items'
|
||||||
|
_PLAYLIST_TITLE_KEY = 'display_title'
|
||||||
|
_PLAYLIST_DESCRIPTION_KEY = None
|
||||||
|
_FIELDS = '''id
|
||||||
|
display_title
|
||||||
|
items {
|
||||||
|
... on ListVideoItem {
|
||||||
|
doc {
|
||||||
|
canonical
|
||||||
|
id
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}'''
|
||||||
|
|
||||||
|
def _get_doc(self, item):
|
||||||
|
return item.get('doc') or {}
|
||||||
|
|
||||||
|
|
||||||
|
class AENetworksShowIE(AENetworksListBaseIE):
|
||||||
|
IE_NAME = 'aenetworks:show'
|
||||||
|
_VALID_URL = AENetworksBaseIE._BASE_URL_REGEX + r'shows/(?P<id>[^/?#&]+)/?(?:[?#&]|$)'
|
||||||
|
_TESTS = [{
|
||||||
|
'url': 'http://www.history.com/shows/ancient-aliens',
|
||||||
|
'info_dict': {
|
||||||
|
'id': 'SERIES1574',
|
||||||
|
'title': 'Ancient Aliens',
|
||||||
|
'description': 'md5:3f6d74daf2672ff3ae29ed732e37ea7f',
|
||||||
|
},
|
||||||
|
'playlist_mincount': 150,
|
||||||
|
}]
|
||||||
|
_RESOURCE = 'series'
|
||||||
|
_ITEMS_KEY = 'episodes'
|
||||||
|
_PLAYLIST_TITLE_KEY = 'title'
|
||||||
|
_PLAYLIST_DESCRIPTION_KEY = 'description'
|
||||||
|
_FIELDS = '''description
|
||||||
|
id
|
||||||
|
title
|
||||||
|
episodes {
|
||||||
|
canonical
|
||||||
|
id
|
||||||
|
}'''
|
||||||
|
|
||||||
|
def _get_doc(self, item):
|
||||||
|
return item
|
||||||
|
|
||||||
|
|
||||||
class HistoryTopicIE(AENetworksBaseIE):
|
class HistoryTopicIE(AENetworksBaseIE):
|
||||||
|
@ -204,6 +300,7 @@ class HistoryTopicIE(AENetworksBaseIE):
|
||||||
'description': 'md5:7b57ea4829b391995b405fa60bd7b5f7',
|
'description': 'md5:7b57ea4829b391995b405fa60bd7b5f7',
|
||||||
'timestamp': 1375819729,
|
'timestamp': 1375819729,
|
||||||
'upload_date': '20130806',
|
'upload_date': '20130806',
|
||||||
|
'uploader': 'AENE-NEW',
|
||||||
},
|
},
|
||||||
'params': {
|
'params': {
|
||||||
# m3u8 download
|
# m3u8 download
|
||||||
|
@ -212,36 +309,47 @@ class HistoryTopicIE(AENetworksBaseIE):
|
||||||
'add_ie': ['ThePlatform'],
|
'add_ie': ['ThePlatform'],
|
||||||
}]
|
}]
|
||||||
|
|
||||||
def theplatform_url_result(self, theplatform_url, video_id, query):
|
def _real_extract(self, url):
|
||||||
return {
|
display_id = self._match_id(url)
|
||||||
'_type': 'url_transparent',
|
return self.url_result(
|
||||||
'id': video_id,
|
'http://www.history.com/videos/' + display_id,
|
||||||
'url': smuggle_url(
|
AENetworksIE.ie_key())
|
||||||
update_url_query(theplatform_url, query),
|
|
||||||
{
|
|
||||||
'sig': {
|
class HistoryPlayerIE(AENetworksBaseIE):
|
||||||
'key': self._THEPLATFORM_KEY,
|
IE_NAME = 'history:player'
|
||||||
'secret': self._THEPLATFORM_SECRET,
|
_VALID_URL = r'https?://(?:www\.)?(?P<domain>(?:history|biography)\.com)/player/(?P<id>\d+)'
|
||||||
},
|
_TESTS = []
|
||||||
'force_smil_url': True
|
|
||||||
}),
|
def _real_extract(self, url):
|
||||||
'ie_key': 'ThePlatform',
|
domain, video_id = re.match(self._VALID_URL, url).groups()
|
||||||
}
|
return self._extract_aetn_info(domain, 'id', video_id, url)
|
||||||
|
|
||||||
|
|
||||||
|
class BiographyIE(AENetworksBaseIE):
|
||||||
|
_VALID_URL = r'https?://(?:www\.)?biography\.com/video/(?P<id>[^/?#&]+)'
|
||||||
|
_TESTS = [{
|
||||||
|
'url': 'https://www.biography.com/video/vincent-van-gogh-full-episode-2075049808',
|
||||||
|
'info_dict': {
|
||||||
|
'id': '30322987',
|
||||||
|
'ext': 'mp4',
|
||||||
|
'title': 'Vincent Van Gogh - Full Episode',
|
||||||
|
'description': 'A full biography about the most influential 20th century painter, Vincent Van Gogh.',
|
||||||
|
'timestamp': 1311970571,
|
||||||
|
'upload_date': '20110729',
|
||||||
|
'uploader': 'AENE-NEW',
|
||||||
|
},
|
||||||
|
'params': {
|
||||||
|
# m3u8 download
|
||||||
|
'skip_download': True,
|
||||||
|
},
|
||||||
|
'add_ie': ['ThePlatform'],
|
||||||
|
}]
|
||||||
|
|
||||||
def _real_extract(self, url):
|
def _real_extract(self, url):
|
||||||
display_id = self._match_id(url)
|
display_id = self._match_id(url)
|
||||||
webpage = self._download_webpage(url, display_id)
|
webpage = self._download_webpage(url, display_id)
|
||||||
video_id = self._search_regex(
|
player_url = self._search_regex(
|
||||||
r'<phoenix-iframe[^>]+src="[^"]+\btpid=(\d+)', webpage, 'tpid')
|
r'<phoenix-iframe[^>]+src="(%s)' % HistoryPlayerIE._VALID_URL,
|
||||||
result = self._download_json(
|
webpage, 'player URL')
|
||||||
'https://feeds.video.aetnd.com/api/v2/history/videos',
|
return self.url_result(player_url, HistoryPlayerIE.ie_key())
|
||||||
video_id, query={'filter[id]': video_id})['results'][0]
|
|
||||||
title = result['title']
|
|
||||||
info = self._extract_aen_smil(result['publicUrl'], video_id)
|
|
||||||
info.update({
|
|
||||||
'title': title,
|
|
||||||
'description': result.get('description'),
|
|
||||||
'duration': int_or_none(result.get('duration')),
|
|
||||||
'timestamp': int_or_none(result.get('added'), 1000),
|
|
||||||
})
|
|
||||||
return info
|
|
||||||
|
|
|
@ -275,7 +275,7 @@ class AfreecaTVIE(InfoExtractor):
|
||||||
video_element = video_xml.findall(compat_xpath('./track/video'))[-1]
|
video_element = video_xml.findall(compat_xpath('./track/video'))[-1]
|
||||||
if video_element is None or video_element.text is None:
|
if video_element is None or video_element.text is None:
|
||||||
raise ExtractorError(
|
raise ExtractorError(
|
||||||
'Video %s video does not exist' % video_id, expected=True)
|
'Video %s does not exist' % video_id, expected=True)
|
||||||
|
|
||||||
video_url = video_element.text.strip()
|
video_url = video_element.text.strip()
|
||||||
|
|
||||||
|
|
|
@ -18,7 +18,7 @@ class AliExpressLiveIE(InfoExtractor):
|
||||||
'id': '2800002704436634',
|
'id': '2800002704436634',
|
||||||
'ext': 'mp4',
|
'ext': 'mp4',
|
||||||
'title': 'CASIMA7.22',
|
'title': 'CASIMA7.22',
|
||||||
'thumbnail': r're:http://.*\.jpg',
|
'thumbnail': r're:https?://.*\.jpg',
|
||||||
'uploader': 'CASIMA Official Store',
|
'uploader': 'CASIMA Official Store',
|
||||||
'timestamp': 1500717600,
|
'timestamp': 1500717600,
|
||||||
'upload_date': '20170722',
|
'upload_date': '20170722',
|
||||||
|
|
|
@ -1,13 +1,16 @@
|
||||||
from __future__ import unicode_literals
|
from __future__ import unicode_literals
|
||||||
|
|
||||||
|
import json
|
||||||
|
import re
|
||||||
|
|
||||||
from .common import InfoExtractor
|
from .common import InfoExtractor
|
||||||
|
|
||||||
|
|
||||||
class AlJazeeraIE(InfoExtractor):
|
class AlJazeeraIE(InfoExtractor):
|
||||||
_VALID_URL = r'https?://(?:www\.)?aljazeera\.com/(?:programmes|video)/.*?/(?P<id>[^/]+)\.html'
|
_VALID_URL = r'https?://(?:www\.)?aljazeera\.com/(?P<type>program/[^/]+|(?:feature|video)s)/\d{4}/\d{1,2}/\d{1,2}/(?P<id>[^/?&#]+)'
|
||||||
|
|
||||||
_TESTS = [{
|
_TESTS = [{
|
||||||
'url': 'http://www.aljazeera.com/programmes/the-slum/2014/08/deliverance-201482883754237240.html',
|
'url': 'https://www.aljazeera.com/program/episode/2014/9/19/deliverance',
|
||||||
'info_dict': {
|
'info_dict': {
|
||||||
'id': '3792260579001',
|
'id': '3792260579001',
|
||||||
'ext': 'mp4',
|
'ext': 'mp4',
|
||||||
|
@ -20,14 +23,34 @@ class AlJazeeraIE(InfoExtractor):
|
||||||
'add_ie': ['BrightcoveNew'],
|
'add_ie': ['BrightcoveNew'],
|
||||||
'skip': 'Not accessible from Travis CI server',
|
'skip': 'Not accessible from Travis CI server',
|
||||||
}, {
|
}, {
|
||||||
'url': 'http://www.aljazeera.com/video/news/2017/05/sierra-leone-709-carat-diamond-auctioned-170511100111930.html',
|
'url': 'https://www.aljazeera.com/videos/2017/5/11/sierra-leone-709-carat-diamond-to-be-auctioned-off',
|
||||||
|
'only_matching': True,
|
||||||
|
}, {
|
||||||
|
'url': 'https://www.aljazeera.com/features/2017/8/21/transforming-pakistans-buses-into-art',
|
||||||
'only_matching': True,
|
'only_matching': True,
|
||||||
}]
|
}]
|
||||||
BRIGHTCOVE_URL_TEMPLATE = 'http://players.brightcove.net/665003303001/default_default/index.html?videoId=%s'
|
BRIGHTCOVE_URL_TEMPLATE = 'http://players.brightcove.net/%s/%s_default/index.html?videoId=%s'
|
||||||
|
|
||||||
def _real_extract(self, url):
|
def _real_extract(self, url):
|
||||||
program_name = self._match_id(url)
|
post_type, name = re.match(self._VALID_URL, url).groups()
|
||||||
webpage = self._download_webpage(url, program_name)
|
post_type = {
|
||||||
brightcove_id = self._search_regex(
|
'features': 'post',
|
||||||
r'RenderPagesVideo\(\'(.+?)\'', webpage, 'brightcove id')
|
'program': 'episode',
|
||||||
return self.url_result(self.BRIGHTCOVE_URL_TEMPLATE % brightcove_id, 'BrightcoveNew', brightcove_id)
|
'videos': 'video',
|
||||||
|
}[post_type.split('/')[0]]
|
||||||
|
video = self._download_json(
|
||||||
|
'https://www.aljazeera.com/graphql', name, query={
|
||||||
|
'operationName': 'SingleArticleQuery',
|
||||||
|
'variables': json.dumps({
|
||||||
|
'name': name,
|
||||||
|
'postType': post_type,
|
||||||
|
}),
|
||||||
|
}, headers={
|
||||||
|
'wp-site': 'aje',
|
||||||
|
})['data']['article']['video']
|
||||||
|
video_id = video['id']
|
||||||
|
account_id = video.get('accountId') or '665003303001'
|
||||||
|
player_id = video.get('playerId') or 'BkeSH5BDb'
|
||||||
|
return self.url_result(
|
||||||
|
self.BRIGHTCOVE_URL_TEMPLATE % (account_id, player_id, video_id),
|
||||||
|
'BrightcoveNew', video_id)
|
||||||
|
|
89
youtube_dl/extractor/alsace20tv.py
Normal file
89
youtube_dl/extractor/alsace20tv.py
Normal file
|
@ -0,0 +1,89 @@
|
||||||
|
# coding: utf-8
|
||||||
|
from __future__ import unicode_literals
|
||||||
|
|
||||||
|
from .common import InfoExtractor
|
||||||
|
from ..utils import (
|
||||||
|
clean_html,
|
||||||
|
dict_get,
|
||||||
|
get_element_by_class,
|
||||||
|
int_or_none,
|
||||||
|
unified_strdate,
|
||||||
|
url_or_none,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class Alsace20TVIE(InfoExtractor):
|
||||||
|
_VALID_URL = r'https?://(?:www\.)?alsace20\.tv/(?:[\w-]+/)+[\w-]+-(?P<id>[\w]+)'
|
||||||
|
_TESTS = [{
|
||||||
|
'url': 'https://www.alsace20.tv/VOD/Actu/JT/Votre-JT-jeudi-3-fevrier-lyNHCXpYJh.html',
|
||||||
|
# 'md5': 'd91851bf9af73c0ad9b2cdf76c127fbb',
|
||||||
|
'info_dict': {
|
||||||
|
'id': 'lyNHCXpYJh',
|
||||||
|
'ext': 'mp4',
|
||||||
|
'description': 'md5:fc0bc4a0692d3d2dba4524053de4c7b7',
|
||||||
|
'title': 'Votre JT du jeudi 3 février',
|
||||||
|
'upload_date': '20220203',
|
||||||
|
'thumbnail': r're:https?://.+\.jpg',
|
||||||
|
'duration': 1073,
|
||||||
|
'view_count': int,
|
||||||
|
},
|
||||||
|
'params': {
|
||||||
|
'format': 'bestvideo',
|
||||||
|
},
|
||||||
|
}]
|
||||||
|
|
||||||
|
def _extract_video(self, video_id, url=None):
|
||||||
|
info = self._download_json(
|
||||||
|
'https://www.alsace20.tv/visionneuse/visio_v9_js.php?key=%s&habillage=0&mode=html' % (video_id, ),
|
||||||
|
video_id) or {}
|
||||||
|
title = info['titre']
|
||||||
|
|
||||||
|
formats = []
|
||||||
|
for res, fmt_url in (info.get('files') or {}).items():
|
||||||
|
formats.extend(
|
||||||
|
self._extract_smil_formats(fmt_url, video_id, fatal=False)
|
||||||
|
if '/smil:_' in fmt_url
|
||||||
|
else self._extract_mpd_formats(fmt_url, video_id, mpd_id=res, fatal=False))
|
||||||
|
self._sort_formats(formats)
|
||||||
|
|
||||||
|
webpage = (url and self._download_webpage(url, video_id, fatal=False)) or ''
|
||||||
|
thumbnail = url_or_none(dict_get(info, ('image', 'preview', )) or self._og_search_thumbnail(webpage))
|
||||||
|
upload_date = self._search_regex(r'/(\d{6})_', thumbnail, 'upload_date', default=None)
|
||||||
|
upload_date = unified_strdate('20%s-%s-%s' % (upload_date[:2], upload_date[2:4], upload_date[4:])) if upload_date else None
|
||||||
|
return {
|
||||||
|
'id': video_id,
|
||||||
|
'title': title,
|
||||||
|
'formats': formats,
|
||||||
|
'description': clean_html(get_element_by_class('wysiwyg', webpage)),
|
||||||
|
'upload_date': upload_date,
|
||||||
|
'thumbnail': thumbnail,
|
||||||
|
'duration': int_or_none(self._og_search_property('video:duration', webpage) if webpage else None),
|
||||||
|
'view_count': int_or_none(info.get('nb_vues')),
|
||||||
|
}
|
||||||
|
|
||||||
|
def _real_extract(self, url):
|
||||||
|
video_id = self._match_id(url)
|
||||||
|
return self._extract_video(video_id, url)
|
||||||
|
|
||||||
|
|
||||||
|
class Alsace20TVEmbedIE(Alsace20TVIE):
|
||||||
|
_VALID_URL = r'https?://(?:www\.)?alsace20\.tv/emb/(?P<id>[\w]+)'
|
||||||
|
_TESTS = [{
|
||||||
|
'url': 'https://www.alsace20.tv/emb/lyNHCXpYJh',
|
||||||
|
# 'md5': 'd91851bf9af73c0ad9b2cdf76c127fbb',
|
||||||
|
'info_dict': {
|
||||||
|
'id': 'lyNHCXpYJh',
|
||||||
|
'ext': 'mp4',
|
||||||
|
'title': 'Votre JT du jeudi 3 février',
|
||||||
|
'upload_date': '20220203',
|
||||||
|
'thumbnail': r're:https?://.+\.jpg',
|
||||||
|
'view_count': int,
|
||||||
|
},
|
||||||
|
'params': {
|
||||||
|
'format': 'bestvideo',
|
||||||
|
},
|
||||||
|
}]
|
||||||
|
|
||||||
|
def _real_extract(self, url):
|
||||||
|
video_id = self._match_id(url)
|
||||||
|
return self._extract_video(video_id)
|
103
youtube_dl/extractor/amara.py
Normal file
103
youtube_dl/extractor/amara.py
Normal file
|
@ -0,0 +1,103 @@
|
||||||
|
# coding: utf-8
|
||||||
|
from __future__ import unicode_literals
|
||||||
|
|
||||||
|
from .common import InfoExtractor
|
||||||
|
from .youtube import YoutubeIE
|
||||||
|
from .vimeo import VimeoIE
|
||||||
|
from ..utils import (
|
||||||
|
int_or_none,
|
||||||
|
parse_iso8601,
|
||||||
|
update_url_query,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class AmaraIE(InfoExtractor):
|
||||||
|
_VALID_URL = r'https?://(?:www\.)?amara\.org/(?:\w+/)?videos/(?P<id>\w+)'
|
||||||
|
_TESTS = [{
|
||||||
|
# Youtube
|
||||||
|
'url': 'https://amara.org/en/videos/jVx79ZKGK1ky/info/why-jury-trials-are-becoming-less-common/?tab=video',
|
||||||
|
'md5': 'ea10daf2b6154b8c1ecf9922aca5e8ae',
|
||||||
|
'info_dict': {
|
||||||
|
'id': 'h6ZuVdvYnfE',
|
||||||
|
'ext': 'mp4',
|
||||||
|
'title': 'Why jury trials are becoming less common',
|
||||||
|
'description': 'md5:a61811c319943960b6ab1c23e0cbc2c1',
|
||||||
|
'thumbnail': r're:^https?://.*\.jpg$',
|
||||||
|
'subtitles': dict,
|
||||||
|
'upload_date': '20160813',
|
||||||
|
'uploader': 'PBS NewsHour',
|
||||||
|
'uploader_id': 'PBSNewsHour',
|
||||||
|
'timestamp': 1549639570,
|
||||||
|
}
|
||||||
|
}, {
|
||||||
|
# Vimeo
|
||||||
|
'url': 'https://amara.org/en/videos/kYkK1VUTWW5I/info/vimeo-at-ces-2011',
|
||||||
|
'md5': '99392c75fa05d432a8f11df03612195e',
|
||||||
|
'info_dict': {
|
||||||
|
'id': '18622084',
|
||||||
|
'ext': 'mov',
|
||||||
|
'title': 'Vimeo at CES 2011!',
|
||||||
|
'description': 'md5:d41d8cd98f00b204e9800998ecf8427e',
|
||||||
|
'thumbnail': r're:^https?://.*\.jpg$',
|
||||||
|
'subtitles': dict,
|
||||||
|
'timestamp': 1294763658,
|
||||||
|
'upload_date': '20110111',
|
||||||
|
'uploader': 'Sam Morrill',
|
||||||
|
'uploader_id': 'sammorrill'
|
||||||
|
}
|
||||||
|
}, {
|
||||||
|
# Direct Link
|
||||||
|
'url': 'https://amara.org/en/videos/s8KL7I3jLmh6/info/the-danger-of-a-single-story/',
|
||||||
|
'md5': 'd3970f08512738ee60c5807311ff5d3f',
|
||||||
|
'info_dict': {
|
||||||
|
'id': 's8KL7I3jLmh6',
|
||||||
|
'ext': 'mp4',
|
||||||
|
'title': 'The danger of a single story',
|
||||||
|
'description': 'md5:d769b31139c3b8bb5be9177f62ea3f23',
|
||||||
|
'thumbnail': r're:^https?://.*\.jpg$',
|
||||||
|
'subtitles': dict,
|
||||||
|
'upload_date': '20091007',
|
||||||
|
'timestamp': 1254942511,
|
||||||
|
}
|
||||||
|
}]
|
||||||
|
|
||||||
|
def _real_extract(self, url):
|
||||||
|
video_id = self._match_id(url)
|
||||||
|
meta = self._download_json(
|
||||||
|
'https://amara.org/api/videos/%s/' % video_id,
|
||||||
|
video_id, query={'format': 'json'})
|
||||||
|
title = meta['title']
|
||||||
|
video_url = meta['all_urls'][0]
|
||||||
|
|
||||||
|
subtitles = {}
|
||||||
|
for language in (meta.get('languages') or []):
|
||||||
|
subtitles_uri = language.get('subtitles_uri')
|
||||||
|
if not (subtitles_uri and language.get('published')):
|
||||||
|
continue
|
||||||
|
subtitle = subtitles.setdefault(language.get('code') or 'en', [])
|
||||||
|
for f in ('json', 'srt', 'vtt'):
|
||||||
|
subtitle.append({
|
||||||
|
'ext': f,
|
||||||
|
'url': update_url_query(subtitles_uri, {'format': f}),
|
||||||
|
})
|
||||||
|
|
||||||
|
info = {
|
||||||
|
'url': video_url,
|
||||||
|
'id': video_id,
|
||||||
|
'subtitles': subtitles,
|
||||||
|
'title': title,
|
||||||
|
'description': meta.get('description'),
|
||||||
|
'thumbnail': meta.get('thumbnail'),
|
||||||
|
'duration': int_or_none(meta.get('duration')),
|
||||||
|
'timestamp': parse_iso8601(meta.get('created')),
|
||||||
|
}
|
||||||
|
|
||||||
|
for ie in (YoutubeIE, VimeoIE):
|
||||||
|
if ie.suitable(video_url):
|
||||||
|
info.update({
|
||||||
|
'_type': 'url_transparent',
|
||||||
|
'ie_key': ie.ie_key(),
|
||||||
|
})
|
||||||
|
break
|
||||||
|
|
||||||
|
return info
|
|
@ -1,6 +1,8 @@
|
||||||
# coding: utf-8
|
# coding: utf-8
|
||||||
from __future__ import unicode_literals
|
from __future__ import unicode_literals
|
||||||
|
|
||||||
|
import re
|
||||||
|
|
||||||
from .theplatform import ThePlatformIE
|
from .theplatform import ThePlatformIE
|
||||||
from ..utils import (
|
from ..utils import (
|
||||||
int_or_none,
|
int_or_none,
|
||||||
|
@ -11,25 +13,22 @@ from ..utils import (
|
||||||
|
|
||||||
|
|
||||||
class AMCNetworksIE(ThePlatformIE):
|
class AMCNetworksIE(ThePlatformIE):
|
||||||
_VALID_URL = r'https?://(?:www\.)?(?:amc|bbcamerica|ifc|(?:we|sundance)tv)\.com/(?:movies|shows(?:/[^/]+)+)/(?P<id>[^/?#]+)'
|
_VALID_URL = r'https?://(?:www\.)?(?P<site>amc|bbcamerica|ifc|(?:we|sundance)tv)\.com/(?P<id>(?:movies|shows(?:/[^/]+)+)/[^/?#&]+)'
|
||||||
_TESTS = [{
|
_TESTS = [{
|
||||||
'url': 'http://www.ifc.com/shows/maron/season-04/episode-01/step-1',
|
'url': 'https://www.bbcamerica.com/shows/the-graham-norton-show/videos/tina-feys-adorable-airline-themed-family-dinner--51631',
|
||||||
'md5': '',
|
|
||||||
'info_dict': {
|
'info_dict': {
|
||||||
'id': 's3MX01Nl4vPH',
|
'id': '4Lq1dzOnZGt0',
|
||||||
'ext': 'mp4',
|
'ext': 'mp4',
|
||||||
'title': 'Maron - Season 4 - Step 1',
|
'title': "The Graham Norton Show - Season 28 - Tina Fey's Adorable Airline-Themed Family Dinner",
|
||||||
'description': 'In denial about his current situation, Marc is reluctantly convinced by his friends to enter rehab. Starring Marc Maron and Constance Zimmer.',
|
'description': "It turns out child stewardesses are very generous with the wine! All-new episodes of 'The Graham Norton Show' premiere Fridays at 11/10c on BBC America.",
|
||||||
'age_limit': 17,
|
'upload_date': '20201120',
|
||||||
'upload_date': '20160505',
|
'timestamp': 1605904350,
|
||||||
'timestamp': 1462468831,
|
|
||||||
'uploader': 'AMCN',
|
'uploader': 'AMCN',
|
||||||
},
|
},
|
||||||
'params': {
|
'params': {
|
||||||
# m3u8 download
|
# m3u8 download
|
||||||
'skip_download': True,
|
'skip_download': True,
|
||||||
},
|
},
|
||||||
'skip': 'Requires TV provider accounts',
|
|
||||||
}, {
|
}, {
|
||||||
'url': 'http://www.bbcamerica.com/shows/the-hunt/full-episodes/season-1/episode-01-the-hardest-challenge',
|
'url': 'http://www.bbcamerica.com/shows/the-hunt/full-episodes/season-1/episode-01-the-hardest-challenge',
|
||||||
'only_matching': True,
|
'only_matching': True,
|
||||||
|
@ -55,32 +54,34 @@ class AMCNetworksIE(ThePlatformIE):
|
||||||
'url': 'https://www.sundancetv.com/shows/riviera/full-episodes/season-1/episode-01-episode-1',
|
'url': 'https://www.sundancetv.com/shows/riviera/full-episodes/season-1/episode-01-episode-1',
|
||||||
'only_matching': True,
|
'only_matching': True,
|
||||||
}]
|
}]
|
||||||
|
_REQUESTOR_ID_MAP = {
|
||||||
|
'amc': 'AMC',
|
||||||
|
'bbcamerica': 'BBCA',
|
||||||
|
'ifc': 'IFC',
|
||||||
|
'sundancetv': 'SUNDANCE',
|
||||||
|
'wetv': 'WETV',
|
||||||
|
}
|
||||||
|
|
||||||
def _real_extract(self, url):
|
def _real_extract(self, url):
|
||||||
display_id = self._match_id(url)
|
site, display_id = re.match(self._VALID_URL, url).groups()
|
||||||
webpage = self._download_webpage(url, display_id)
|
requestor_id = self._REQUESTOR_ID_MAP[site]
|
||||||
|
properties = self._download_json(
|
||||||
|
'https://content-delivery-gw.svc.ds.amcn.com/api/v2/content/amcn/%s/url/%s' % (requestor_id.lower(), display_id),
|
||||||
|
display_id)['data']['properties']
|
||||||
query = {
|
query = {
|
||||||
'mbr': 'true',
|
'mbr': 'true',
|
||||||
'manifest': 'm3u',
|
'manifest': 'm3u',
|
||||||
}
|
}
|
||||||
media_url = self._search_regex(
|
tp_path = 'M_UwQC/media/' + properties['videoPid']
|
||||||
r'window\.platformLinkURL\s*=\s*[\'"]([^\'"]+)',
|
media_url = 'https://link.theplatform.com/s/' + tp_path
|
||||||
webpage, 'media url')
|
theplatform_metadata = self._download_theplatform_metadata(tp_path, display_id)
|
||||||
theplatform_metadata = self._download_theplatform_metadata(self._search_regex(
|
|
||||||
r'link\.theplatform\.com/s/([^?]+)',
|
|
||||||
media_url, 'theplatform_path'), display_id)
|
|
||||||
info = self._parse_theplatform_metadata(theplatform_metadata)
|
info = self._parse_theplatform_metadata(theplatform_metadata)
|
||||||
video_id = theplatform_metadata['pid']
|
video_id = theplatform_metadata['pid']
|
||||||
title = theplatform_metadata['title']
|
title = theplatform_metadata['title']
|
||||||
rating = try_get(
|
rating = try_get(
|
||||||
theplatform_metadata, lambda x: x['ratings'][0]['rating'])
|
theplatform_metadata, lambda x: x['ratings'][0]['rating'])
|
||||||
auth_required = self._search_regex(
|
video_category = properties.get('videoCategory')
|
||||||
r'window\.authRequired\s*=\s*(true|false);',
|
if video_category and video_category.endswith('-Auth'):
|
||||||
webpage, 'auth required')
|
|
||||||
if auth_required == 'true':
|
|
||||||
requestor_id = self._search_regex(
|
|
||||||
r'window\.requestor_id\s*=\s*[\'"]([^\'"]+)',
|
|
||||||
webpage, 'requestor id')
|
|
||||||
resource = self._get_mvpd_resource(
|
resource = self._get_mvpd_resource(
|
||||||
requestor_id, title, video_id, rating)
|
requestor_id, title, video_id, rating)
|
||||||
query['auth'] = self._extract_mvpd_auth(
|
query['auth'] = self._extract_mvpd_auth(
|
||||||
|
|
|
@ -1,34 +1,68 @@
|
||||||
# coding: utf-8
|
# coding: utf-8
|
||||||
from __future__ import unicode_literals
|
from __future__ import unicode_literals
|
||||||
|
|
||||||
|
import json
|
||||||
|
import re
|
||||||
|
|
||||||
from .common import InfoExtractor
|
from .common import InfoExtractor
|
||||||
from ..utils import (
|
from ..utils import (
|
||||||
clean_html,
|
clean_html,
|
||||||
int_or_none,
|
int_or_none,
|
||||||
js_to_json,
|
|
||||||
try_get,
|
try_get,
|
||||||
unified_strdate,
|
unified_strdate,
|
||||||
|
unified_timestamp,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
class AmericasTestKitchenIE(InfoExtractor):
|
class AmericasTestKitchenIE(InfoExtractor):
|
||||||
_VALID_URL = r'https?://(?:www\.)?americastestkitchen\.com/(?:episode|videos)/(?P<id>\d+)'
|
_VALID_URL = r'https?://(?:www\.)?(?:americastestkitchen|cooks(?:country|illustrated))\.com/(?:cooks(?:country|illustrated)/)?(?P<resource_type>episode|videos)/(?P<id>\d+)'
|
||||||
_TESTS = [{
|
_TESTS = [{
|
||||||
'url': 'https://www.americastestkitchen.com/episode/582-weeknight-japanese-suppers',
|
'url': 'https://www.americastestkitchen.com/episode/582-weeknight-japanese-suppers',
|
||||||
'md5': 'b861c3e365ac38ad319cfd509c30577f',
|
'md5': 'b861c3e365ac38ad319cfd509c30577f',
|
||||||
'info_dict': {
|
'info_dict': {
|
||||||
'id': '5b400b9ee338f922cb06450c',
|
'id': '5b400b9ee338f922cb06450c',
|
||||||
'title': 'Weeknight Japanese Suppers',
|
'title': 'Japanese Suppers',
|
||||||
'ext': 'mp4',
|
'ext': 'mp4',
|
||||||
'description': 'md5:3d0c1a44bb3b27607ce82652db25b4a8',
|
'display_id': 'weeknight-japanese-suppers',
|
||||||
'thumbnail': r're:^https?://',
|
'description': 'md5:64e606bfee910627efc4b5f050de92b3',
|
||||||
'timestamp': 1523664000,
|
'timestamp': 1523304000,
|
||||||
'upload_date': '20180414',
|
'upload_date': '20180409',
|
||||||
'release_date': '20180414',
|
'release_date': '20180409',
|
||||||
'series': "America's Test Kitchen",
|
'series': "America's Test Kitchen",
|
||||||
|
'season': 'Season 18',
|
||||||
'season_number': 18,
|
'season_number': 18,
|
||||||
'episode': 'Weeknight Japanese Suppers',
|
'episode': 'Japanese Suppers',
|
||||||
'episode_number': 15,
|
'episode_number': 15,
|
||||||
|
'duration': 1376,
|
||||||
|
'thumbnail': r're:^https?://',
|
||||||
|
'average_rating': 0,
|
||||||
|
'view_count': int,
|
||||||
|
},
|
||||||
|
'params': {
|
||||||
|
'skip_download': True,
|
||||||
|
},
|
||||||
|
}, {
|
||||||
|
# Metadata parsing behaves differently for newer episodes (705) as opposed to older episodes (582 above)
|
||||||
|
'url': 'https://www.americastestkitchen.com/episode/705-simple-chicken-dinner',
|
||||||
|
'md5': '06451608c57651e985a498e69cec17e5',
|
||||||
|
'info_dict': {
|
||||||
|
'id': '5fbe8c61bda2010001c6763b',
|
||||||
|
'title': 'Simple Chicken Dinner',
|
||||||
|
'ext': 'mp4',
|
||||||
|
'display_id': 'atktv_2103_simple-chicken-dinner_full-episode_web-mp4',
|
||||||
|
'description': 'md5:eb68737cc2fd4c26ca7db30139d109e7',
|
||||||
|
'timestamp': 1610737200,
|
||||||
|
'upload_date': '20210115',
|
||||||
|
'release_date': '20210115',
|
||||||
|
'series': "America's Test Kitchen",
|
||||||
|
'season': 'Season 21',
|
||||||
|
'season_number': 21,
|
||||||
|
'episode': 'Simple Chicken Dinner',
|
||||||
|
'episode_number': 3,
|
||||||
|
'duration': 1397,
|
||||||
|
'thumbnail': r're:^https?://',
|
||||||
|
'view_count': int,
|
||||||
|
'average_rating': 0,
|
||||||
},
|
},
|
||||||
'params': {
|
'params': {
|
||||||
'skip_download': True,
|
'skip_download': True,
|
||||||
|
@ -36,47 +70,151 @@ class AmericasTestKitchenIE(InfoExtractor):
|
||||||
}, {
|
}, {
|
||||||
'url': 'https://www.americastestkitchen.com/videos/3420-pan-seared-salmon',
|
'url': 'https://www.americastestkitchen.com/videos/3420-pan-seared-salmon',
|
||||||
'only_matching': True,
|
'only_matching': True,
|
||||||
|
}, {
|
||||||
|
'url': 'https://www.americastestkitchen.com/cookscountry/episode/564-when-only-chocolate-will-do',
|
||||||
|
'only_matching': True,
|
||||||
|
}, {
|
||||||
|
'url': 'https://www.americastestkitchen.com/cooksillustrated/videos/4478-beef-wellington',
|
||||||
|
'only_matching': True,
|
||||||
|
}, {
|
||||||
|
'url': 'https://www.cookscountry.com/episode/564-when-only-chocolate-will-do',
|
||||||
|
'only_matching': True,
|
||||||
|
}, {
|
||||||
|
'url': 'https://www.cooksillustrated.com/videos/4478-beef-wellington',
|
||||||
|
'only_matching': True,
|
||||||
}]
|
}]
|
||||||
|
|
||||||
def _real_extract(self, url):
|
def _real_extract(self, url):
|
||||||
video_id = self._match_id(url)
|
resource_type, video_id = re.match(self._VALID_URL, url).groups()
|
||||||
|
is_episode = resource_type == 'episode'
|
||||||
|
if is_episode:
|
||||||
|
resource_type = 'episodes'
|
||||||
|
|
||||||
webpage = self._download_webpage(url, video_id)
|
resource = self._download_json(
|
||||||
|
'https://www.americastestkitchen.com/api/v6/%s/%s' % (resource_type, video_id), video_id)
|
||||||
video_data = self._parse_json(
|
video = resource['video'] if is_episode else resource
|
||||||
self._search_regex(
|
episode = resource if is_episode else resource.get('episode') or {}
|
||||||
r'window\.__INITIAL_STATE__\s*=\s*({.+?})\s*;\s*</script>',
|
|
||||||
webpage, 'initial context'),
|
|
||||||
video_id, js_to_json)
|
|
||||||
|
|
||||||
ep_data = try_get(
|
|
||||||
video_data,
|
|
||||||
(lambda x: x['episodeDetail']['content']['data'],
|
|
||||||
lambda x: x['videoDetail']['content']['data']), dict)
|
|
||||||
ep_meta = ep_data.get('full_video', {})
|
|
||||||
|
|
||||||
zype_id = ep_data.get('zype_id') or ep_meta['zype_id']
|
|
||||||
|
|
||||||
title = ep_data.get('title') or ep_meta.get('title')
|
|
||||||
description = clean_html(ep_meta.get('episode_description') or ep_data.get(
|
|
||||||
'description') or ep_meta.get('description'))
|
|
||||||
thumbnail = try_get(ep_meta, lambda x: x['photo']['image_url'])
|
|
||||||
release_date = unified_strdate(ep_data.get('aired_at'))
|
|
||||||
|
|
||||||
season_number = int_or_none(ep_meta.get('season_number'))
|
|
||||||
episode = ep_meta.get('title')
|
|
||||||
episode_number = int_or_none(ep_meta.get('episode_number'))
|
|
||||||
|
|
||||||
return {
|
return {
|
||||||
'_type': 'url_transparent',
|
'_type': 'url_transparent',
|
||||||
'url': 'https://player.zype.com/embed/%s.js?api_key=jZ9GUhRmxcPvX7M3SlfejB6Hle9jyHTdk2jVxG7wOHPLODgncEKVdPYBhuz9iWXQ' % zype_id,
|
'url': 'https://player.zype.com/embed/%s.js?api_key=jZ9GUhRmxcPvX7M3SlfejB6Hle9jyHTdk2jVxG7wOHPLODgncEKVdPYBhuz9iWXQ' % video['zypeId'],
|
||||||
'ie_key': 'Zype',
|
'ie_key': 'Zype',
|
||||||
'title': title,
|
'description': clean_html(video.get('description')),
|
||||||
'description': description,
|
'timestamp': unified_timestamp(video.get('publishDate')),
|
||||||
'thumbnail': thumbnail,
|
'release_date': unified_strdate(video.get('publishDate')),
|
||||||
'release_date': release_date,
|
'episode_number': int_or_none(episode.get('number')),
|
||||||
'series': "America's Test Kitchen",
|
'season_number': int_or_none(episode.get('season')),
|
||||||
'season_number': season_number,
|
'series': try_get(episode, lambda x: x['show']['title']),
|
||||||
'episode': episode,
|
'episode': episode.get('title'),
|
||||||
'episode_number': episode_number,
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
class AmericasTestKitchenSeasonIE(InfoExtractor):
|
||||||
|
_VALID_URL = r'https?://(?:www\.)?(?P<show>americastestkitchen|(?P<cooks>cooks(?:country|illustrated)))\.com(?:(?:/(?P<show2>cooks(?:country|illustrated)))?(?:/?$|(?<!ated)(?<!ated\.com)/episodes/browse/season_(?P<season>\d+)))'
|
||||||
|
_TESTS = [{
|
||||||
|
# ATK Season
|
||||||
|
'url': 'https://www.americastestkitchen.com/episodes/browse/season_1',
|
||||||
|
'info_dict': {
|
||||||
|
'id': 'season_1',
|
||||||
|
'title': 'Season 1',
|
||||||
|
},
|
||||||
|
'playlist_count': 13,
|
||||||
|
}, {
|
||||||
|
# Cooks Country Season
|
||||||
|
'url': 'https://www.americastestkitchen.com/cookscountry/episodes/browse/season_12',
|
||||||
|
'info_dict': {
|
||||||
|
'id': 'season_12',
|
||||||
|
'title': 'Season 12',
|
||||||
|
},
|
||||||
|
'playlist_count': 13,
|
||||||
|
}, {
|
||||||
|
# America's Test Kitchen Series
|
||||||
|
'url': 'https://www.americastestkitchen.com/',
|
||||||
|
'info_dict': {
|
||||||
|
'id': 'americastestkitchen',
|
||||||
|
'title': 'America\'s Test Kitchen',
|
||||||
|
},
|
||||||
|
'playlist_count': 558,
|
||||||
|
}, {
|
||||||
|
# Cooks Country Series
|
||||||
|
'url': 'https://www.americastestkitchen.com/cookscountry',
|
||||||
|
'info_dict': {
|
||||||
|
'id': 'cookscountry',
|
||||||
|
'title': 'Cook\'s Country',
|
||||||
|
},
|
||||||
|
'playlist_count': 199,
|
||||||
|
}, {
|
||||||
|
'url': 'https://www.americastestkitchen.com/cookscountry/',
|
||||||
|
'only_matching': True,
|
||||||
|
}, {
|
||||||
|
'url': 'https://www.cookscountry.com/episodes/browse/season_12',
|
||||||
|
'only_matching': True,
|
||||||
|
}, {
|
||||||
|
'url': 'https://www.cookscountry.com',
|
||||||
|
'only_matching': True,
|
||||||
|
}, {
|
||||||
|
'url': 'https://www.americastestkitchen.com/cooksillustrated/',
|
||||||
|
'only_matching': True,
|
||||||
|
}, {
|
||||||
|
'url': 'https://www.cooksillustrated.com',
|
||||||
|
'only_matching': True,
|
||||||
|
}]
|
||||||
|
|
||||||
|
def _real_extract(self, url):
|
||||||
|
match = re.match(self._VALID_URL, url).groupdict()
|
||||||
|
show = match.get('show2')
|
||||||
|
show_path = ('/' + show) if show else ''
|
||||||
|
show = show or match['show']
|
||||||
|
season_number = int_or_none(match.get('season'))
|
||||||
|
|
||||||
|
slug, title = {
|
||||||
|
'americastestkitchen': ('atk', 'America\'s Test Kitchen'),
|
||||||
|
'cookscountry': ('cco', 'Cook\'s Country'),
|
||||||
|
'cooksillustrated': ('cio', 'Cook\'s Illustrated'),
|
||||||
|
}[show]
|
||||||
|
|
||||||
|
facet_filters = [
|
||||||
|
'search_document_klass:episode',
|
||||||
|
'search_show_slug:' + slug,
|
||||||
|
]
|
||||||
|
|
||||||
|
if season_number:
|
||||||
|
playlist_id = 'season_%d' % season_number
|
||||||
|
playlist_title = 'Season %d' % season_number
|
||||||
|
facet_filters.append('search_season_list:' + playlist_title)
|
||||||
|
else:
|
||||||
|
playlist_id = show
|
||||||
|
playlist_title = title
|
||||||
|
|
||||||
|
season_search = self._download_json(
|
||||||
|
'https://y1fnzxui30-dsn.algolia.net/1/indexes/everest_search_%s_season_desc_production' % slug,
|
||||||
|
playlist_id, headers={
|
||||||
|
'Origin': 'https://www.americastestkitchen.com',
|
||||||
|
'X-Algolia-API-Key': '8d504d0099ed27c1b73708d22871d805',
|
||||||
|
'X-Algolia-Application-Id': 'Y1FNZXUI30',
|
||||||
|
}, query={
|
||||||
|
'facetFilters': json.dumps(facet_filters),
|
||||||
|
'attributesToRetrieve': 'description,search_%s_episode_number,search_document_date,search_url,title,search_atk_episode_season' % slug,
|
||||||
|
'attributesToHighlight': '',
|
||||||
|
'hitsPerPage': 1000,
|
||||||
|
})
|
||||||
|
|
||||||
|
def entries():
|
||||||
|
for episode in (season_search.get('hits') or []):
|
||||||
|
search_url = episode.get('search_url') # always formatted like '/episode/123-title-of-episode'
|
||||||
|
if not search_url:
|
||||||
|
continue
|
||||||
|
yield {
|
||||||
|
'_type': 'url',
|
||||||
|
'url': 'https://www.americastestkitchen.com%s%s' % (show_path, search_url),
|
||||||
|
'id': try_get(episode, lambda e: e['objectID'].rsplit('_', 1)[-1]),
|
||||||
|
'title': episode.get('title'),
|
||||||
|
'description': episode.get('description'),
|
||||||
|
'timestamp': unified_timestamp(episode.get('search_document_date')),
|
||||||
|
'season_number': season_number,
|
||||||
|
'episode_number': int_or_none(episode.get('search_%s_episode_number' % slug)),
|
||||||
|
'ie_key': AmericasTestKitchenIE.ie_key(),
|
||||||
|
}
|
||||||
|
|
||||||
|
return self.playlist_result(
|
||||||
|
entries(), playlist_id, playlist_title)
|
||||||
|
|
|
@ -8,6 +8,7 @@ from ..utils import (
|
||||||
int_or_none,
|
int_or_none,
|
||||||
mimetype2ext,
|
mimetype2ext,
|
||||||
parse_iso8601,
|
parse_iso8601,
|
||||||
|
unified_timestamp,
|
||||||
url_or_none,
|
url_or_none,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@ -88,7 +89,7 @@ class AMPIE(InfoExtractor):
|
||||||
|
|
||||||
self._sort_formats(formats)
|
self._sort_formats(formats)
|
||||||
|
|
||||||
timestamp = parse_iso8601(item.get('pubDate'), ' ') or parse_iso8601(item.get('dc-date'))
|
timestamp = unified_timestamp(item.get('pubDate'), ' ') or parse_iso8601(item.get('dc-date'))
|
||||||
|
|
||||||
return {
|
return {
|
||||||
'id': video_id,
|
'id': video_id,
|
||||||
|
|
|
@ -116,8 +116,6 @@ class AnimeOnDemandIE(InfoExtractor):
|
||||||
r'(?s)<div[^>]+itemprop="description"[^>]*>(.+?)</div>',
|
r'(?s)<div[^>]+itemprop="description"[^>]*>(.+?)</div>',
|
||||||
webpage, 'anime description', default=None)
|
webpage, 'anime description', default=None)
|
||||||
|
|
||||||
entries = []
|
|
||||||
|
|
||||||
def extract_info(html, video_id, num=None):
|
def extract_info(html, video_id, num=None):
|
||||||
title, description = [None] * 2
|
title, description = [None] * 2
|
||||||
formats = []
|
formats = []
|
||||||
|
@ -233,7 +231,7 @@ class AnimeOnDemandIE(InfoExtractor):
|
||||||
self._sort_formats(info['formats'])
|
self._sort_formats(info['formats'])
|
||||||
f = common_info.copy()
|
f = common_info.copy()
|
||||||
f.update(info)
|
f.update(info)
|
||||||
entries.append(f)
|
yield f
|
||||||
|
|
||||||
# Extract teaser/trailer only when full episode is not available
|
# Extract teaser/trailer only when full episode is not available
|
||||||
if not info['formats']:
|
if not info['formats']:
|
||||||
|
@ -247,7 +245,7 @@ class AnimeOnDemandIE(InfoExtractor):
|
||||||
'title': m.group('title'),
|
'title': m.group('title'),
|
||||||
'url': urljoin(url, m.group('href')),
|
'url': urljoin(url, m.group('href')),
|
||||||
})
|
})
|
||||||
entries.append(f)
|
yield f
|
||||||
|
|
||||||
def extract_episodes(html):
|
def extract_episodes(html):
|
||||||
for num, episode_html in enumerate(re.findall(
|
for num, episode_html in enumerate(re.findall(
|
||||||
|
@ -275,7 +273,8 @@ class AnimeOnDemandIE(InfoExtractor):
|
||||||
'episode_number': episode_number,
|
'episode_number': episode_number,
|
||||||
}
|
}
|
||||||
|
|
||||||
extract_entries(episode_html, video_id, common_info)
|
for e in extract_entries(episode_html, video_id, common_info):
|
||||||
|
yield e
|
||||||
|
|
||||||
def extract_film(html, video_id):
|
def extract_film(html, video_id):
|
||||||
common_info = {
|
common_info = {
|
||||||
|
@ -283,11 +282,18 @@ class AnimeOnDemandIE(InfoExtractor):
|
||||||
'title': anime_title,
|
'title': anime_title,
|
||||||
'description': anime_description,
|
'description': anime_description,
|
||||||
}
|
}
|
||||||
extract_entries(html, video_id, common_info)
|
for e in extract_entries(html, video_id, common_info):
|
||||||
|
yield e
|
||||||
|
|
||||||
extract_episodes(webpage)
|
def entries():
|
||||||
|
has_episodes = False
|
||||||
|
for e in extract_episodes(webpage):
|
||||||
|
has_episodes = True
|
||||||
|
yield e
|
||||||
|
|
||||||
if not entries:
|
if not has_episodes:
|
||||||
extract_film(webpage, anime_id)
|
for e in extract_film(webpage, anime_id):
|
||||||
|
yield e
|
||||||
|
|
||||||
return self.playlist_result(entries, anime_id, anime_title, anime_description)
|
return self.playlist_result(
|
||||||
|
entries(), anime_id, anime_title, anime_description)
|
||||||
|
|
|
@ -116,7 +116,76 @@ class AnvatoIE(InfoExtractor):
|
||||||
'anvato_scripps_app_ios_prod_409c41960c60b308db43c3cc1da79cab9f1c3d93': 'WPxj5GraLTkYCyj3M7RozLqIycjrXOEcDGFMIJPn',
|
'anvato_scripps_app_ios_prod_409c41960c60b308db43c3cc1da79cab9f1c3d93': 'WPxj5GraLTkYCyj3M7RozLqIycjrXOEcDGFMIJPn',
|
||||||
'EZqvRyKBJLrgpClDPDF8I7Xpdp40Vx73': '4OxGd2dEakylntVKjKF0UK9PDPYB6A9W',
|
'EZqvRyKBJLrgpClDPDF8I7Xpdp40Vx73': '4OxGd2dEakylntVKjKF0UK9PDPYB6A9W',
|
||||||
'M2v78QkpleXm9hPp9jUXI63x5vA6BogR': 'ka6K32k7ZALmpINkjJUGUo0OE42Md1BQ',
|
'M2v78QkpleXm9hPp9jUXI63x5vA6BogR': 'ka6K32k7ZALmpINkjJUGUo0OE42Md1BQ',
|
||||||
'nbcu_nbcd_desktop_web_prod_93d8ead38ce2024f8f544b78306fbd15895ae5e6_secure': 'NNemUkySjxLyPTKvZRiGntBIjEyK8uqicjMakIaQ'
|
'nbcu_nbcd_desktop_web_prod_93d8ead38ce2024f8f544b78306fbd15895ae5e6_secure': 'NNemUkySjxLyPTKvZRiGntBIjEyK8uqicjMakIaQ',
|
||||||
|
'X8POa4zPPaKVZHqmWjuEzfP31b1QM9VN': 'Dn5vOY9ooDw7VSl9qztjZI5o0g08mA0z',
|
||||||
|
'M2v78QkBMpNJlSPp9diX5F2PBmBy6Bog': 'ka6K32kyo7nDZfNkjQCGWf1lpApXMd1B',
|
||||||
|
'bvJ0dQpav07l0hG5JgfVLF2dv1vARwpP': 'BzoQW24GrJZoJfmNodiJKSPeB9B8NOxj',
|
||||||
|
'lxQMLg2XZKuEZaWgsqubBxV9INZ6bryY': 'Vm2Mx6noKds9jB71h6urazwlTG3m9x8l',
|
||||||
|
'04EnjvXeoSmkbJ9ckPs7oY0mcxv7PlyN': 'aXERQP9LMfQVlEDsgGs6eEA1SWznAQ8P',
|
||||||
|
'mQbO2ge6BFRWVPYCYpU06YvNt80XLvAX': 'E2BV1NGmasN5v7eujECVPJgwflnLPm2A',
|
||||||
|
'g43oeBzJrCml7o6fa5fRL1ErCdeD8z4K': 'RX34mZ6zVH4Nr6whbxIGLv9WSbxEKo8V',
|
||||||
|
'VQrDJoP7mtdBzkxhXbSPwGB1coeElk4x': 'j2VejQx0VFKQepAF7dI0mJLKtOVJE18z',
|
||||||
|
'WxA5NzLRjCrmq0NUgaU5pdMDuZO7RJ4w': 'lyY5ADLKaIOLEgAsGQCveEMAcqnx3rY9',
|
||||||
|
'M4lpMXB71ie0PjMCjdFzVXq0SeRVqz49': 'n2zVkOqaLIv3GbLfBjcwW51LcveWOZ2e',
|
||||||
|
'dyDZGEqN8u8nkJZcJns0oxYmtP7KbGAn': 'VXOEqQW9BtEVLajfZQSLEqxgS5B7qn2D',
|
||||||
|
'E7QNjrVY5u5mGvgu67IoDgV1CjEND8QR': 'rz8AaDmdKIkLmPNhB5ILPJnjS5PnlL8d',
|
||||||
|
'a4zrqjoKlfzg0dwHEWtP31VqcLBpjm4g': 'LY9J16gwETdGWa3hjBu5o0RzuoQDjqXQ',
|
||||||
|
'dQP5BZroMsMVLO1hbmT5r2Enu86GjxA6': '7XR3oOdbPF6x3PRFLDCq9RkgsRjAo48V',
|
||||||
|
'M4lKNBO1NFe0PjMCj1tzVXq0SeRVqzA9': 'n2zoRqGLRUv3GbLfBmTwW51LcveWOZYe',
|
||||||
|
'nAZ7MZdpGCGg1pqFEbsoJOz2C60mv143': 'dYJgdqA9aT4yojETqGi7yNgoFADxqmXP',
|
||||||
|
'3y1MERYgOuE9NzbFgwhV6Wv2F0YKvbyz': '081xpZDQgC4VadLTavhWQxrku56DAgXV',
|
||||||
|
'bmQvmEXr5HWklBMCZOcpE2Z3HBYwqGyl': 'zxXPbVNyMiMAZldhr9FkOmA0fl4aKr2v',
|
||||||
|
'wA7oDNYldfr6050Hwxi52lPZiVlB86Ap': 'ZYK16aA7ni0d3l3c34uwpxD7CbReMm8Q',
|
||||||
|
'g43MbKMWmFml7o7sJoSRkXxZiXRvJ3QK': 'RX3oBJonvs4Nr6rUWBCGn3matRGqJPXV',
|
||||||
|
'mA9VdlqpLS0raGaSDvtoqNrBTzb8XY4q': '0XN4OjBD3fnW7r7IbmtJB4AyfOmlrE2r',
|
||||||
|
'mAajOwgkGt17oGoFmEuklMP9H0GnW54d': 'lXbBLPGyzikNGeGujAuAJGjZiwLRxyXR',
|
||||||
|
'vy8vjJ9kbUwrRqRu59Cj5dWZfzYErlAb': 'K8l7gpwaGcBpnAnCLNCmPZRdin3eaQX0',
|
||||||
|
'xQMWBpR8oHEZaWaSMGUb0avOHjLVYn4Y': 'm2MrN4vEaf9jB7BFy5Srb40jTrN67AYl',
|
||||||
|
'xyKEmVO3miRr6D6UVkt7oB8jtD6aJEAv': 'g2ddDebqDfqdgKgswyUKwGjbTWwzq923',
|
||||||
|
'7Qk0wa2D9FjKapacoJF27aLvUDKkLGA0': 'b2kgBEkephJaMkMTL7s1PLe4Ua6WyP2P',
|
||||||
|
'3QLg6nqmNTJ5VvVTo7f508LPidz1xwyY': 'g2L1GgpraipmAOAUqmIbBnPxHOmw4MYa',
|
||||||
|
'3y1B7zZjXTE9NZNSzZSVNPZaTNLjo6Qz': '081b5G6wzH4VagaURmcWbN5mT4JGEe2V',
|
||||||
|
'lAqnwvkw6SG6D8DSqmUg6DRLUp0w3G4x': 'O2pbP0xPDFNJjpjIEvcdryOJtpkVM4X5',
|
||||||
|
'awA7xd1N0Hr6050Hw2c52lPZiVlB864p': 'GZYKpn4aoT0d3l3c3PiwpxD7CbReMmXQ',
|
||||||
|
'jQVqPLl9YHL1WGWtR1HDgWBGT63qRNyV': '6X03ne6vrU4oWyWUN7tQVoajikxJR3Ye',
|
||||||
|
'GQRMR8mL7uZK797t7xH3eNzPIP5dOny1': 'm2vqPWGd4U31zWzSyasDRAoMT1PKRp8o',
|
||||||
|
'zydq9RdmRhXLkNkfNoTJlMzaF0lWekQB': '3X7LnvE7vH5nkEkSqLiey793Un7dLB8e',
|
||||||
|
'VQrDzwkB2IdBzjzu9MHPbEYkSB50gR4x': 'j2VebLzoKUKQeEesmVh0gM1eIp9jKz8z',
|
||||||
|
'mAa2wMamBs17oGoFmktklMP9H0GnW54d': 'lXbgP74xZTkNGeGujVUAJGjZiwLRxy8R',
|
||||||
|
'7yjB6ZLG6sW8R6RF2xcan1KGfJ5dNoyd': 'wXQkPorvPHZ45N5t4Jf6qwg5Tp4xvw29',
|
||||||
|
'a4zPpNeWGuzg0m0iX3tPeanGSkRKWXQg': 'LY9oa3QAyHdGW9Wu3Ri5JGeEik7l1N8Q',
|
||||||
|
'k2rneA2M38k25cXDwwSknTJlxPxQLZ6M': '61lyA2aEVDzklfdwmmh31saPxQx2VRjp',
|
||||||
|
'bK9Zk4OvPnvxduLgxvi8VUeojnjA02eV': 'o5jANYjbeMb4nfBaQvcLAt1jzLzYx6ze',
|
||||||
|
'5VD6EydM3R9orHmNMGInGCJwbxbQvGRw': 'w3zjmX7g4vnxzCxElvUEOiewkokXprkZ',
|
||||||
|
'70X35QbVYVYNPUmP9YfbzI06YqYQk2R1': 'vG4Aj2BMjMjoztB7zeFOnCVPJpJ8lMOa',
|
||||||
|
'26qYwQVG9p1Bks2GgBckjfDJOXOAMgG1': 'r4ev9X0mv5zqJc0yk5IBDcQOwZw8mnwQ',
|
||||||
|
'rvVKpA56MBXWlSxMw3cobT5pdkd4Dm7q': '1J7ZkY53pZ645c93owcLZuveE7E8B3rL',
|
||||||
|
'qN1zdy1zlYL23IWZGWtDvfV6WeWQWkJo': 'qN1zdy1zlYL23IWZGWtDvfV6WeWQWkJo',
|
||||||
|
'jdKqRGF16dKsBviMDae7IGDl7oTjEbVV': 'Q09l7vhlNxPFErIOK6BVCe7KnwUW5DVV',
|
||||||
|
'3QLkogW1OUJ5VvPsrDH56DY2u7lgZWyY': 'g2LRE1V9espmAOPhE4ubj4ZdUA57yDXa',
|
||||||
|
'wyJvWbXGBSdbkEzhv0CW8meou82aqRy8': 'M2wolPvyBIpQGkbT4juedD4ruzQGdK2y',
|
||||||
|
'7QkdZrzEkFjKap6IYDU2PB0oCNZORmA0': 'b2kN1l96qhJaMkPs9dt1lpjBfwqZoA8P',
|
||||||
|
'pvA05113MHG1w3JTYxc6DVlRCjErVz4O': 'gQXeAbblBUnDJ7vujbHvbRd1cxlz3AXO',
|
||||||
|
'mA9blJDZwT0raG1cvkuoeVjLC7ZWd54q': '0XN9jRPwMHnW7rvumgfJZOD9CJgVkWYr',
|
||||||
|
'5QwRN5qKJTvGKlDTmnf7xwNZcjRmvEy9': 'R2GP6LWBJU1QlnytwGt0B9pytWwAdDYy',
|
||||||
|
'eyn5rPPbkfw2KYxH32fG1q58CbLJzM40': 'p2gyqooZnS56JWeiDgfmOy1VugOQEBXn',
|
||||||
|
'3BABn3b5RfPJGDwilbHe7l82uBoR05Am': '7OYZG7KMVhbPdKJS3xcWEN3AuDlLNmXj',
|
||||||
|
'xA5zNGXD3HrmqMlF6OS5pdMDuZO7RJ4w': 'yY5DAm6r1IOLE3BCVMFveEMAcqnx3r29',
|
||||||
|
'g43PgW3JZfml7o6fDEURL1ErCdeD8zyK': 'RX3aQn1zrS4Nr6whDgCGLv9WSbxEKo2V',
|
||||||
|
'lAqp8WbGgiG6D8LTKJcg3O72CDdre1Qx': 'O2pnm6473HNJjpKuVosd3vVeh975yrX5',
|
||||||
|
'wyJbYEDxKSdbkJ6S6RhW8meou82aqRy8': 'M2wPm7EgRSpQGlAh70CedD4ruzQGdKYy',
|
||||||
|
'M4lgW28nLCe0PVdtaXszVXq0SeRVqzA9': 'n2zmJvg4jHv3G0ETNgiwW51LcveWOZ8e',
|
||||||
|
'5Qw3OVvp9FvGKlDTmOC7xwNZcjRmvEQ9': 'R2GzDdml9F1Qlnytw9s0B9pytWwAdD8y',
|
||||||
|
'vy8a98X7zCwrRqbHrLUjYzwDiK2b70Qb': 'K8lVwzyjZiBpnAaSGeUmnAgxuGOBxmY0',
|
||||||
|
'g4eGjJLLoiqRD3Pf9oT5O03LuNbLRDQp': '6XqD59zzpfN4EwQuaGt67qNpSyRBlnYy',
|
||||||
|
'g43OPp9boIml7o6fDOIRL1ErCdeD8z4K': 'RX33alNB4s4Nr6whDPUGLv9WSbxEKoXV',
|
||||||
|
'xA2ng9OkBcGKzDbTkKsJlx7dUK8R3dA5': 'z2aPnJvzBfObkwGC3vFaPxeBhxoMqZ8K',
|
||||||
|
'xyKEgBajZuRr6DEC0Kt7XpD1cnNW9gAv': 'g2ddlEBvRsqdgKaI4jUK9PrgfMexGZ23',
|
||||||
|
'BAogww51jIMa2JnH1BcYpXM5F658RNAL': 'rYWDmm0KptlkGv4FGJFMdZmjs9RDE6XR',
|
||||||
|
'BAokpg62VtMa2JnH1mHYpXM5F658RNAL': 'rYWryDnlNslkGv4FG4HMdZmjs9RDE62R',
|
||||||
|
'a4z1Px5e2hzg0m0iMMCPeanGSkRKWXAg': 'LY9eorNQGUdGW9WuKKf5JGeEik7l1NYQ',
|
||||||
|
'kAx69R58kF9nY5YcdecJdl2pFXP53WyX': 'gXyRxELpbfPvLeLSaRil0mp6UEzbZJ8L',
|
||||||
|
'BAoY13nwViMa2J2uo2cY6BlETgmdwryL': 'rYWwKzJmNFlkGvGtNoUM9bzwIJVzB1YR',
|
||||||
}
|
}
|
||||||
|
|
||||||
_MCP_TO_ACCESS_KEY_TABLE = {
|
_MCP_TO_ACCESS_KEY_TABLE = {
|
||||||
|
@ -189,19 +258,17 @@ class AnvatoIE(InfoExtractor):
|
||||||
|
|
||||||
video_data_url += '&X-Anvato-Adst-Auth=' + base64.b64encode(auth_secret).decode('ascii')
|
video_data_url += '&X-Anvato-Adst-Auth=' + base64.b64encode(auth_secret).decode('ascii')
|
||||||
anvrid = md5_text(time.time() * 1000 * random.random())[:30]
|
anvrid = md5_text(time.time() * 1000 * random.random())[:30]
|
||||||
payload = {
|
api = {
|
||||||
'api': {
|
'anvrid': anvrid,
|
||||||
'anvrid': anvrid,
|
'anvts': server_time,
|
||||||
'anvstk': md5_text('%s|%s|%d|%s' % (
|
|
||||||
access_key, anvrid, server_time,
|
|
||||||
self._ANVACK_TABLE.get(access_key, self._API_KEY))),
|
|
||||||
'anvts': server_time,
|
|
||||||
},
|
|
||||||
}
|
}
|
||||||
|
api['anvstk'] = md5_text('%s|%s|%d|%s' % (
|
||||||
|
access_key, anvrid, server_time,
|
||||||
|
self._ANVACK_TABLE.get(access_key, self._API_KEY)))
|
||||||
|
|
||||||
return self._download_json(
|
return self._download_json(
|
||||||
video_data_url, video_id, transform_source=strip_jsonp,
|
video_data_url, video_id, transform_source=strip_jsonp,
|
||||||
data=json.dumps(payload).encode('utf-8'))
|
data=json.dumps({'api': api}).encode('utf-8'))
|
||||||
|
|
||||||
def _get_anvato_videos(self, access_key, video_id):
|
def _get_anvato_videos(self, access_key, video_id):
|
||||||
video_data = self._get_video_json(access_key, video_id)
|
video_data = self._get_video_json(access_key, video_id)
|
||||||
|
@ -259,7 +326,7 @@ class AnvatoIE(InfoExtractor):
|
||||||
'description': video_data.get('def_description'),
|
'description': video_data.get('def_description'),
|
||||||
'tags': video_data.get('def_tags', '').split(','),
|
'tags': video_data.get('def_tags', '').split(','),
|
||||||
'categories': video_data.get('categories'),
|
'categories': video_data.get('categories'),
|
||||||
'thumbnail': video_data.get('thumbnail'),
|
'thumbnail': video_data.get('src_image_url') or video_data.get('thumbnail'),
|
||||||
'timestamp': int_or_none(video_data.get(
|
'timestamp': int_or_none(video_data.get(
|
||||||
'ts_published') or video_data.get('ts_added')),
|
'ts_published') or video_data.get('ts_added')),
|
||||||
'uploader': video_data.get('mcp_id'),
|
'uploader': video_data.get('mcp_id'),
|
||||||
|
|
|
@ -3,7 +3,7 @@ from __future__ import unicode_literals
|
||||||
|
|
||||||
import re
|
import re
|
||||||
|
|
||||||
from .common import InfoExtractor
|
from .yahoo import YahooIE
|
||||||
from ..compat import (
|
from ..compat import (
|
||||||
compat_parse_qs,
|
compat_parse_qs,
|
||||||
compat_urllib_parse_urlparse,
|
compat_urllib_parse_urlparse,
|
||||||
|
@ -15,9 +15,9 @@ from ..utils import (
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
class AolIE(InfoExtractor):
|
class AolIE(YahooIE):
|
||||||
IE_NAME = 'aol.com'
|
IE_NAME = 'aol.com'
|
||||||
_VALID_URL = r'(?:aol-video:|https?://(?:www\.)?aol\.(?:com|ca|co\.uk|de|jp)/video/(?:[^/]+/)*)(?P<id>[0-9a-f]+)'
|
_VALID_URL = r'(?:aol-video:|https?://(?:www\.)?aol\.(?:com|ca|co\.uk|de|jp)/video/(?:[^/]+/)*)(?P<id>\d{9}|[0-9a-f]{24}|[0-9a-f]{8}-(?:[0-9a-f]{4}-){3}[0-9a-f]{12})'
|
||||||
|
|
||||||
_TESTS = [{
|
_TESTS = [{
|
||||||
# video with 5min ID
|
# video with 5min ID
|
||||||
|
@ -76,10 +76,16 @@ class AolIE(InfoExtractor):
|
||||||
}, {
|
}, {
|
||||||
'url': 'https://www.aol.jp/video/playlist/5a28e936a1334d000137da0c/5a28f3151e642219fde19831/',
|
'url': 'https://www.aol.jp/video/playlist/5a28e936a1334d000137da0c/5a28f3151e642219fde19831/',
|
||||||
'only_matching': True,
|
'only_matching': True,
|
||||||
|
}, {
|
||||||
|
# Yahoo video
|
||||||
|
'url': 'https://www.aol.com/video/play/991e6700-ac02-11ea-99ff-357400036f61/24bbc846-3e30-3c46-915e-fe8ccd7fcc46/',
|
||||||
|
'only_matching': True,
|
||||||
}]
|
}]
|
||||||
|
|
||||||
def _real_extract(self, url):
|
def _real_extract(self, url):
|
||||||
video_id = self._match_id(url)
|
video_id = self._match_id(url)
|
||||||
|
if '-' in video_id:
|
||||||
|
return self._extract_yahoo_video(video_id, 'us')
|
||||||
|
|
||||||
response = self._download_json(
|
response = self._download_json(
|
||||||
'https://feedapi.b2c.on.aol.com/v1.0/app/videos/aolon/%s/details' % video_id,
|
'https://feedapi.b2c.on.aol.com/v1.0/app/videos/aolon/%s/details' % video_id,
|
||||||
|
|
|
@ -6,25 +6,21 @@ import re
|
||||||
from .common import InfoExtractor
|
from .common import InfoExtractor
|
||||||
from ..utils import (
|
from ..utils import (
|
||||||
determine_ext,
|
determine_ext,
|
||||||
js_to_json,
|
int_or_none,
|
||||||
url_or_none,
|
url_or_none,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
class APAIE(InfoExtractor):
|
class APAIE(InfoExtractor):
|
||||||
_VALID_URL = r'https?://[^/]+\.apa\.at/embed/(?P<id>[\da-f]{8}-[\da-f]{4}-[\da-f]{4}-[\da-f]{4}-[\da-f]{12})'
|
_VALID_URL = r'(?P<base_url>https?://[^/]+\.apa\.at)/embed/(?P<id>[\da-f]{8}-[\da-f]{4}-[\da-f]{4}-[\da-f]{4}-[\da-f]{12})'
|
||||||
_TESTS = [{
|
_TESTS = [{
|
||||||
'url': 'http://uvp.apa.at/embed/293f6d17-692a-44e3-9fd5-7b178f3a1029',
|
'url': 'http://uvp.apa.at/embed/293f6d17-692a-44e3-9fd5-7b178f3a1029',
|
||||||
'md5': '2b12292faeb0a7d930c778c7a5b4759b',
|
'md5': '2b12292faeb0a7d930c778c7a5b4759b',
|
||||||
'info_dict': {
|
'info_dict': {
|
||||||
'id': 'jjv85FdZ',
|
'id': '293f6d17-692a-44e3-9fd5-7b178f3a1029',
|
||||||
'ext': 'mp4',
|
'ext': 'mp4',
|
||||||
'title': '"Blau ist mysteriös": Die Blue Man Group im Interview',
|
'title': '293f6d17-692a-44e3-9fd5-7b178f3a1029',
|
||||||
'description': 'md5:d41d8cd98f00b204e9800998ecf8427e',
|
|
||||||
'thumbnail': r're:^https?://.*\.jpg$',
|
'thumbnail': r're:^https?://.*\.jpg$',
|
||||||
'duration': 254,
|
|
||||||
'timestamp': 1519211149,
|
|
||||||
'upload_date': '20180221',
|
|
||||||
},
|
},
|
||||||
}, {
|
}, {
|
||||||
'url': 'https://uvp-apapublisher.sf.apa.at/embed/2f94e9e6-d945-4db2-9548-f9a41ebf7b78',
|
'url': 'https://uvp-apapublisher.sf.apa.at/embed/2f94e9e6-d945-4db2-9548-f9a41ebf7b78',
|
||||||
|
@ -46,9 +42,11 @@ class APAIE(InfoExtractor):
|
||||||
webpage)]
|
webpage)]
|
||||||
|
|
||||||
def _real_extract(self, url):
|
def _real_extract(self, url):
|
||||||
video_id = self._match_id(url)
|
mobj = re.match(self._VALID_URL, url)
|
||||||
|
video_id, base_url = mobj.group('id', 'base_url')
|
||||||
|
|
||||||
webpage = self._download_webpage(url, video_id)
|
webpage = self._download_webpage(
|
||||||
|
'%s/player/%s' % (base_url, video_id), video_id)
|
||||||
|
|
||||||
jwplatform_id = self._search_regex(
|
jwplatform_id = self._search_regex(
|
||||||
r'media[iI]d\s*:\s*["\'](?P<id>[a-zA-Z0-9]{8})', webpage,
|
r'media[iI]d\s*:\s*["\'](?P<id>[a-zA-Z0-9]{8})', webpage,
|
||||||
|
@ -59,16 +57,18 @@ class APAIE(InfoExtractor):
|
||||||
'jwplatform:' + jwplatform_id, ie='JWPlatform',
|
'jwplatform:' + jwplatform_id, ie='JWPlatform',
|
||||||
video_id=video_id)
|
video_id=video_id)
|
||||||
|
|
||||||
sources = self._parse_json(
|
def extract(field, name=None):
|
||||||
self._search_regex(
|
return self._search_regex(
|
||||||
r'sources\s*=\s*(\[.+?\])\s*;', webpage, 'sources'),
|
r'\b%s["\']\s*:\s*(["\'])(?P<value>(?:(?!\1).)+)\1' % field,
|
||||||
video_id, transform_source=js_to_json)
|
webpage, name or field, default=None, group='value')
|
||||||
|
|
||||||
|
title = extract('title') or video_id
|
||||||
|
description = extract('description')
|
||||||
|
thumbnail = extract('poster', 'thumbnail')
|
||||||
|
|
||||||
formats = []
|
formats = []
|
||||||
for source in sources:
|
for format_id in ('hls', 'progressive'):
|
||||||
if not isinstance(source, dict):
|
source_url = url_or_none(extract(format_id))
|
||||||
continue
|
|
||||||
source_url = url_or_none(source.get('file'))
|
|
||||||
if not source_url:
|
if not source_url:
|
||||||
continue
|
continue
|
||||||
ext = determine_ext(source_url)
|
ext = determine_ext(source_url)
|
||||||
|
@ -77,18 +77,19 @@ class APAIE(InfoExtractor):
|
||||||
source_url, video_id, 'mp4', entry_protocol='m3u8_native',
|
source_url, video_id, 'mp4', entry_protocol='m3u8_native',
|
||||||
m3u8_id='hls', fatal=False))
|
m3u8_id='hls', fatal=False))
|
||||||
else:
|
else:
|
||||||
|
height = int_or_none(self._search_regex(
|
||||||
|
r'(\d+)\.mp4', source_url, 'height', default=None))
|
||||||
formats.append({
|
formats.append({
|
||||||
'url': source_url,
|
'url': source_url,
|
||||||
|
'format_id': format_id,
|
||||||
|
'height': height,
|
||||||
})
|
})
|
||||||
self._sort_formats(formats)
|
self._sort_formats(formats)
|
||||||
|
|
||||||
thumbnail = self._search_regex(
|
|
||||||
r'image\s*:\s*(["\'])(?P<url>(?:(?!\1).)+)\1', webpage,
|
|
||||||
'thumbnail', fatal=False, group='url')
|
|
||||||
|
|
||||||
return {
|
return {
|
||||||
'id': video_id,
|
'id': video_id,
|
||||||
'title': video_id,
|
'title': title,
|
||||||
|
'description': description,
|
||||||
'thumbnail': thumbnail,
|
'thumbnail': thumbnail,
|
||||||
'formats': formats,
|
'formats': formats,
|
||||||
}
|
}
|
||||||
|
|
|
@ -3,6 +3,7 @@ from __future__ import unicode_literals
|
||||||
|
|
||||||
from .common import InfoExtractor
|
from .common import InfoExtractor
|
||||||
from ..utils import (
|
from ..utils import (
|
||||||
|
get_element_by_id,
|
||||||
int_or_none,
|
int_or_none,
|
||||||
merge_dicts,
|
merge_dicts,
|
||||||
mimetype2ext,
|
mimetype2ext,
|
||||||
|
@ -39,23 +40,15 @@ class AparatIE(InfoExtractor):
|
||||||
webpage = self._download_webpage(url, video_id, fatal=False)
|
webpage = self._download_webpage(url, video_id, fatal=False)
|
||||||
|
|
||||||
if not webpage:
|
if not webpage:
|
||||||
# Note: There is an easier-to-parse configuration at
|
|
||||||
# http://www.aparat.com/video/video/config/videohash/%video_id
|
|
||||||
# but the URL in there does not work
|
|
||||||
webpage = self._download_webpage(
|
webpage = self._download_webpage(
|
||||||
'http://www.aparat.com/video/video/embed/vt/frame/showvideo/yes/videohash/' + video_id,
|
'http://www.aparat.com/video/video/embed/vt/frame/showvideo/yes/videohash/' + video_id,
|
||||||
video_id)
|
video_id)
|
||||||
|
|
||||||
options = self._parse_json(
|
options = self._parse_json(self._search_regex(
|
||||||
self._search_regex(
|
r'options\s*=\s*({.+?})\s*;', webpage, 'options'), video_id)
|
||||||
r'options\s*=\s*JSON\.parse\(\s*(["\'])(?P<value>(?:(?!\1).)+)\1\s*\)',
|
|
||||||
webpage, 'options', group='value'),
|
|
||||||
video_id)
|
|
||||||
|
|
||||||
player = options['plugins']['sabaPlayerPlugin']
|
|
||||||
|
|
||||||
formats = []
|
formats = []
|
||||||
for sources in player['multiSRC']:
|
for sources in (options.get('multiSRC') or []):
|
||||||
for item in sources:
|
for item in sources:
|
||||||
if not isinstance(item, dict):
|
if not isinstance(item, dict):
|
||||||
continue
|
continue
|
||||||
|
@ -85,11 +78,12 @@ class AparatIE(InfoExtractor):
|
||||||
info = self._search_json_ld(webpage, video_id, default={})
|
info = self._search_json_ld(webpage, video_id, default={})
|
||||||
|
|
||||||
if not info.get('title'):
|
if not info.get('title'):
|
||||||
info['title'] = player['title']
|
info['title'] = get_element_by_id('videoTitle', webpage) or \
|
||||||
|
self._html_search_meta(['og:title', 'twitter:title', 'DC.Title', 'title'], webpage, fatal=True)
|
||||||
|
|
||||||
return merge_dicts(info, {
|
return merge_dicts(info, {
|
||||||
'id': video_id,
|
'id': video_id,
|
||||||
'thumbnail': url_or_none(options.get('poster')),
|
'thumbnail': url_or_none(options.get('poster')),
|
||||||
'duration': int_or_none(player.get('duration')),
|
'duration': int_or_none(options.get('duration')),
|
||||||
'formats': formats,
|
'formats': formats,
|
||||||
})
|
})
|
||||||
|
|
|
@ -9,10 +9,10 @@ from ..utils import (
|
||||||
|
|
||||||
|
|
||||||
class AppleConnectIE(InfoExtractor):
|
class AppleConnectIE(InfoExtractor):
|
||||||
_VALID_URL = r'https?://itunes\.apple\.com/\w{0,2}/?post/idsa\.(?P<id>[\w-]+)'
|
_VALID_URL = r'https?://itunes\.apple\.com/\w{0,2}/?post/(?:id)?sa\.(?P<id>[\w-]+)'
|
||||||
_TEST = {
|
_TESTS = [{
|
||||||
'url': 'https://itunes.apple.com/us/post/idsa.4ab17a39-2720-11e5-96c5-a5b38f6c42d3',
|
'url': 'https://itunes.apple.com/us/post/idsa.4ab17a39-2720-11e5-96c5-a5b38f6c42d3',
|
||||||
'md5': 'e7c38568a01ea45402570e6029206723',
|
'md5': 'c1d41f72c8bcaf222e089434619316e4',
|
||||||
'info_dict': {
|
'info_dict': {
|
||||||
'id': '4ab17a39-2720-11e5-96c5-a5b38f6c42d3',
|
'id': '4ab17a39-2720-11e5-96c5-a5b38f6c42d3',
|
||||||
'ext': 'm4v',
|
'ext': 'm4v',
|
||||||
|
@ -22,7 +22,10 @@ class AppleConnectIE(InfoExtractor):
|
||||||
'upload_date': '20150710',
|
'upload_date': '20150710',
|
||||||
'timestamp': 1436545535,
|
'timestamp': 1436545535,
|
||||||
},
|
},
|
||||||
}
|
}, {
|
||||||
|
'url': 'https://itunes.apple.com/us/post/sa.0fe0229f-2457-11e5-9f40-1bb645f2d5d9',
|
||||||
|
'only_matching': True,
|
||||||
|
}]
|
||||||
|
|
||||||
def _real_extract(self, url):
|
def _real_extract(self, url):
|
||||||
video_id = self._match_id(url)
|
video_id = self._match_id(url)
|
||||||
|
@ -36,7 +39,7 @@ class AppleConnectIE(InfoExtractor):
|
||||||
|
|
||||||
video_data = self._parse_json(video_json, video_id)
|
video_data = self._parse_json(video_json, video_id)
|
||||||
timestamp = str_to_int(self._html_search_regex(r'data-timestamp="(\d+)"', webpage, 'timestamp'))
|
timestamp = str_to_int(self._html_search_regex(r'data-timestamp="(\d+)"', webpage, 'timestamp'))
|
||||||
like_count = str_to_int(self._html_search_regex(r'(\d+) Loves', webpage, 'like count'))
|
like_count = str_to_int(self._html_search_regex(r'(\d+) Loves', webpage, 'like count', default=None))
|
||||||
|
|
||||||
return {
|
return {
|
||||||
'id': video_id,
|
'id': video_id,
|
||||||
|
|
93
youtube_dl/extractor/applepodcasts.py
Normal file
93
youtube_dl/extractor/applepodcasts.py
Normal file
|
@ -0,0 +1,93 @@
|
||||||
|
# coding: utf-8
|
||||||
|
from __future__ import unicode_literals
|
||||||
|
|
||||||
|
from .common import InfoExtractor
|
||||||
|
from ..utils import (
|
||||||
|
clean_html,
|
||||||
|
clean_podcast_url,
|
||||||
|
get_element_by_class,
|
||||||
|
int_or_none,
|
||||||
|
parse_codecs,
|
||||||
|
parse_iso8601,
|
||||||
|
try_get,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class ApplePodcastsIE(InfoExtractor):
|
||||||
|
_VALID_URL = r'https?://podcasts\.apple\.com/(?:[^/]+/)?podcast(?:/[^/]+){1,2}.*?\bi=(?P<id>\d+)'
|
||||||
|
_TESTS = [{
|
||||||
|
'url': 'https://podcasts.apple.com/us/podcast/207-whitney-webb-returns/id1135137367?i=1000482637777',
|
||||||
|
'md5': '41dc31cd650143e530d9423b6b5a344f',
|
||||||
|
'info_dict': {
|
||||||
|
'id': '1000482637777',
|
||||||
|
'ext': 'mp3',
|
||||||
|
'title': '207 - Whitney Webb Returns',
|
||||||
|
'description': 'md5:75ef4316031df7b41ced4e7b987f79c6',
|
||||||
|
'upload_date': '20200705',
|
||||||
|
'timestamp': 1593932400,
|
||||||
|
'duration': 6454,
|
||||||
|
'series': 'The Tim Dillon Show',
|
||||||
|
'thumbnail': 're:.+[.](png|jpe?g|webp)',
|
||||||
|
}
|
||||||
|
}, {
|
||||||
|
'url': 'https://podcasts.apple.com/podcast/207-whitney-webb-returns/id1135137367?i=1000482637777',
|
||||||
|
'only_matching': True,
|
||||||
|
}, {
|
||||||
|
'url': 'https://podcasts.apple.com/podcast/207-whitney-webb-returns?i=1000482637777',
|
||||||
|
'only_matching': True,
|
||||||
|
}, {
|
||||||
|
'url': 'https://podcasts.apple.com/podcast/id1135137367?i=1000482637777',
|
||||||
|
'only_matching': True,
|
||||||
|
}]
|
||||||
|
|
||||||
|
def _real_extract(self, url):
|
||||||
|
episode_id = self._match_id(url)
|
||||||
|
webpage = self._download_webpage(url, episode_id)
|
||||||
|
episode_data = {}
|
||||||
|
ember_data = {}
|
||||||
|
# new page type 2021-11
|
||||||
|
amp_data = self._parse_json(self._search_regex(
|
||||||
|
r'(?s)id="shoebox-media-api-cache-amp-podcasts"[^>]*>\s*({.+?})\s*<',
|
||||||
|
webpage, 'AMP data', default='{}'), episode_id, fatal=False) or {}
|
||||||
|
amp_data = try_get(amp_data,
|
||||||
|
lambda a: self._parse_json(
|
||||||
|
next(a[x] for x in iter(a) if episode_id in x),
|
||||||
|
episode_id),
|
||||||
|
dict) or {}
|
||||||
|
amp_data = amp_data.get('d') or []
|
||||||
|
episode_data = try_get(
|
||||||
|
amp_data,
|
||||||
|
lambda a: next(x for x in a
|
||||||
|
if x['type'] == 'podcast-episodes' and x['id'] == episode_id),
|
||||||
|
dict)
|
||||||
|
if not episode_data:
|
||||||
|
# try pre 2021-11 page type: TODO: consider deleting if no longer used
|
||||||
|
ember_data = self._parse_json(self._search_regex(
|
||||||
|
r'(?s)id="shoebox-ember-data-store"[^>]*>\s*({.+?})\s*<',
|
||||||
|
webpage, 'ember data'), episode_id) or {}
|
||||||
|
ember_data = ember_data.get(episode_id) or ember_data
|
||||||
|
episode_data = try_get(ember_data, lambda x: x['data'], dict)
|
||||||
|
episode = episode_data['attributes']
|
||||||
|
description = episode.get('description') or {}
|
||||||
|
|
||||||
|
series = None
|
||||||
|
for inc in (amp_data or ember_data.get('included') or []):
|
||||||
|
if inc.get('type') == 'media/podcast':
|
||||||
|
series = try_get(inc, lambda x: x['attributes']['name'])
|
||||||
|
series = series or clean_html(get_element_by_class('podcast-header__identity', webpage))
|
||||||
|
|
||||||
|
info = [{
|
||||||
|
'id': episode_id,
|
||||||
|
'title': episode['name'],
|
||||||
|
'url': clean_podcast_url(episode['assetUrl']),
|
||||||
|
'description': description.get('standard') or description.get('short'),
|
||||||
|
'timestamp': parse_iso8601(episode.get('releaseDateTime')),
|
||||||
|
'duration': int_or_none(episode.get('durationInMilliseconds'), 1000),
|
||||||
|
'series': series,
|
||||||
|
'thumbnail': self._og_search_thumbnail(webpage),
|
||||||
|
}]
|
||||||
|
self._sort_formats(info)
|
||||||
|
info = info[0]
|
||||||
|
codecs = parse_codecs(info.get('ext', 'mp3'))
|
||||||
|
info.update(codecs)
|
||||||
|
return info
|
|
@ -2,15 +2,17 @@ from __future__ import unicode_literals
|
||||||
|
|
||||||
from .common import InfoExtractor
|
from .common import InfoExtractor
|
||||||
from ..utils import (
|
from ..utils import (
|
||||||
unified_strdate,
|
|
||||||
clean_html,
|
clean_html,
|
||||||
|
extract_attributes,
|
||||||
|
unified_strdate,
|
||||||
|
unified_timestamp,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
class ArchiveOrgIE(InfoExtractor):
|
class ArchiveOrgIE(InfoExtractor):
|
||||||
IE_NAME = 'archive.org'
|
IE_NAME = 'archive.org'
|
||||||
IE_DESC = 'archive.org videos'
|
IE_DESC = 'archive.org videos'
|
||||||
_VALID_URL = r'https?://(?:www\.)?archive\.org/(?:details|embed)/(?P<id>[^/?#]+)(?:[?].*)?$'
|
_VALID_URL = r'https?://(?:www\.)?archive\.org/(?:details|embed)/(?P<id>[^/?#&]+)'
|
||||||
_TESTS = [{
|
_TESTS = [{
|
||||||
'url': 'http://archive.org/details/XD300-23_68HighlightsAResearchCntAugHumanIntellect',
|
'url': 'http://archive.org/details/XD300-23_68HighlightsAResearchCntAugHumanIntellect',
|
||||||
'md5': '8af1d4cf447933ed3c7f4871162602db',
|
'md5': '8af1d4cf447933ed3c7f4871162602db',
|
||||||
|
@ -19,8 +21,11 @@ class ArchiveOrgIE(InfoExtractor):
|
||||||
'ext': 'ogg',
|
'ext': 'ogg',
|
||||||
'title': '1968 Demo - FJCC Conference Presentation Reel #1',
|
'title': '1968 Demo - FJCC Conference Presentation Reel #1',
|
||||||
'description': 'md5:da45c349df039f1cc8075268eb1b5c25',
|
'description': 'md5:da45c349df039f1cc8075268eb1b5c25',
|
||||||
'upload_date': '19681210',
|
'creator': 'SRI International',
|
||||||
'uploader': 'SRI International'
|
'release_date': '19681210',
|
||||||
|
'uploader': 'SRI International',
|
||||||
|
'timestamp': 1268695290,
|
||||||
|
'upload_date': '20100315',
|
||||||
}
|
}
|
||||||
}, {
|
}, {
|
||||||
'url': 'https://archive.org/details/Cops1922',
|
'url': 'https://archive.org/details/Cops1922',
|
||||||
|
@ -29,22 +34,43 @@ class ArchiveOrgIE(InfoExtractor):
|
||||||
'id': 'Cops1922',
|
'id': 'Cops1922',
|
||||||
'ext': 'mp4',
|
'ext': 'mp4',
|
||||||
'title': 'Buster Keaton\'s "Cops" (1922)',
|
'title': 'Buster Keaton\'s "Cops" (1922)',
|
||||||
'description': 'md5:89e7c77bf5d965dd5c0372cfb49470f6',
|
'description': 'md5:43a603fd6c5b4b90d12a96b921212b9c',
|
||||||
|
'timestamp': 1387699629,
|
||||||
|
'upload_date': '20131222',
|
||||||
}
|
}
|
||||||
}, {
|
}, {
|
||||||
'url': 'http://archive.org/embed/XD300-23_68HighlightsAResearchCntAugHumanIntellect',
|
'url': 'http://archive.org/embed/XD300-23_68HighlightsAResearchCntAugHumanIntellect',
|
||||||
'only_matching': True,
|
'only_matching': True,
|
||||||
|
}, {
|
||||||
|
'url': 'https://archive.org/details/MSNBCW_20131125_040000_To_Catch_a_Predator/',
|
||||||
|
'only_matching': True,
|
||||||
}]
|
}]
|
||||||
|
|
||||||
def _real_extract(self, url):
|
def _real_extract(self, url):
|
||||||
video_id = self._match_id(url)
|
video_id = self._match_id(url)
|
||||||
webpage = self._download_webpage(
|
webpage = self._download_webpage(
|
||||||
'http://archive.org/embed/' + video_id, video_id)
|
'http://archive.org/embed/' + video_id, video_id)
|
||||||
jwplayer_playlist = self._parse_json(self._search_regex(
|
|
||||||
r"(?s)Play\('[^']+'\s*,\s*(\[.+\])\s*,\s*{.*?}\)",
|
playlist = None
|
||||||
webpage, 'jwplayer playlist'), video_id)
|
play8 = self._search_regex(
|
||||||
info = self._parse_jwplayer_data(
|
r'(<[^>]+\bclass=["\']js-play8-playlist[^>]+>)', webpage,
|
||||||
{'playlist': jwplayer_playlist}, video_id, base_url=url)
|
'playlist', default=None)
|
||||||
|
if play8:
|
||||||
|
attrs = extract_attributes(play8)
|
||||||
|
playlist = attrs.get('value')
|
||||||
|
if not playlist:
|
||||||
|
# Old jwplayer fallback
|
||||||
|
playlist = self._search_regex(
|
||||||
|
r"(?s)Play\('[^']+'\s*,\s*(\[.+\])\s*,\s*{.*?}\)",
|
||||||
|
webpage, 'jwplayer playlist', default='[]')
|
||||||
|
jwplayer_playlist = self._parse_json(playlist, video_id, fatal=False)
|
||||||
|
if jwplayer_playlist:
|
||||||
|
info = self._parse_jwplayer_data(
|
||||||
|
{'playlist': jwplayer_playlist}, video_id, base_url=url)
|
||||||
|
else:
|
||||||
|
# HTML5 media fallback
|
||||||
|
info = self._parse_html5_media_entries(url, webpage, video_id)[0]
|
||||||
|
info['id'] = video_id
|
||||||
|
|
||||||
def get_optional(metadata, field):
|
def get_optional(metadata, field):
|
||||||
return metadata.get(field, [None])[0]
|
return metadata.get(field, [None])[0]
|
||||||
|
@ -58,8 +84,12 @@ class ArchiveOrgIE(InfoExtractor):
|
||||||
'description': clean_html(get_optional(metadata, 'description')),
|
'description': clean_html(get_optional(metadata, 'description')),
|
||||||
})
|
})
|
||||||
if info.get('_type') != 'playlist':
|
if info.get('_type') != 'playlist':
|
||||||
|
creator = get_optional(metadata, 'creator')
|
||||||
info.update({
|
info.update({
|
||||||
'uploader': get_optional(metadata, 'creator'),
|
'creator': creator,
|
||||||
'upload_date': unified_strdate(get_optional(metadata, 'date')),
|
'release_date': unified_strdate(get_optional(metadata, 'date')),
|
||||||
|
'uploader': get_optional(metadata, 'publisher') or creator,
|
||||||
|
'timestamp': unified_timestamp(get_optional(metadata, 'publicdate')),
|
||||||
|
'language': get_optional(metadata, 'language'),
|
||||||
})
|
})
|
||||||
return info
|
return info
|
||||||
|
|
174
youtube_dl/extractor/arcpublishing.py
Normal file
174
youtube_dl/extractor/arcpublishing.py
Normal file
|
@ -0,0 +1,174 @@
|
||||||
|
# coding: utf-8
|
||||||
|
from __future__ import unicode_literals
|
||||||
|
|
||||||
|
import re
|
||||||
|
|
||||||
|
from .common import InfoExtractor
|
||||||
|
from ..utils import (
|
||||||
|
extract_attributes,
|
||||||
|
int_or_none,
|
||||||
|
parse_iso8601,
|
||||||
|
try_get,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class ArcPublishingIE(InfoExtractor):
|
||||||
|
_UUID_REGEX = r'[\da-f]{8}-(?:[\da-f]{4}-){3}[\da-f]{12}'
|
||||||
|
_VALID_URL = r'arcpublishing:(?P<org>[a-z]+):(?P<id>%s)' % _UUID_REGEX
|
||||||
|
_TESTS = [{
|
||||||
|
# https://www.adn.com/politics/2020/11/02/video-senate-candidates-campaign-in-anchorage-on-eve-of-election-day/
|
||||||
|
'url': 'arcpublishing:adn:8c99cb6e-b29c-4bc9-9173-7bf9979225ab',
|
||||||
|
'only_matching': True,
|
||||||
|
}, {
|
||||||
|
# https://www.bostonglobe.com/video/2020/12/30/metro/footage-released-showing-officer-talking-about-striking-protesters-with-car/
|
||||||
|
'url': 'arcpublishing:bostonglobe:232b7ae6-7d73-432d-bc0a-85dbf0119ab1',
|
||||||
|
'only_matching': True,
|
||||||
|
}, {
|
||||||
|
# https://www.actionnewsjax.com/video/live-stream/
|
||||||
|
'url': 'arcpublishing:cmg:cfb1cf1b-3ab5-4d1b-86c5-a5515d311f2a',
|
||||||
|
'only_matching': True,
|
||||||
|
}, {
|
||||||
|
# https://elcomercio.pe/videos/deportes/deporte-total-futbol-peruano-seleccion-peruana-la-valorizacion-de-los-peruanos-en-el-exterior-tras-un-2020-atipico-nnav-vr-video-noticia/
|
||||||
|
'url': 'arcpublishing:elcomercio:27a7e1f8-2ec7-4177-874f-a4feed2885b3',
|
||||||
|
'only_matching': True,
|
||||||
|
}, {
|
||||||
|
# https://www.clickondetroit.com/video/community/2020/05/15/events-surrounding-woodward-dream-cruise-being-canceled/
|
||||||
|
'url': 'arcpublishing:gmg:c8793fb2-8d44-4242-881e-2db31da2d9fe',
|
||||||
|
'only_matching': True,
|
||||||
|
}, {
|
||||||
|
# https://www.wabi.tv/video/2020/12/30/trenton-company-making-equipment-pfizer-covid-vaccine/
|
||||||
|
'url': 'arcpublishing:gray:0b0ba30e-032a-4598-8810-901d70e6033e',
|
||||||
|
'only_matching': True,
|
||||||
|
}, {
|
||||||
|
# https://www.lateja.cr/el-mundo/video-china-aprueba-con-condiciones-su-primera/dfcbfa57-527f-45ff-a69b-35fe71054143/video/
|
||||||
|
'url': 'arcpublishing:gruponacion:dfcbfa57-527f-45ff-a69b-35fe71054143',
|
||||||
|
'only_matching': True,
|
||||||
|
}, {
|
||||||
|
# https://www.fifthdomain.com/video/2018/03/09/is-america-vulnerable-to-a-cyber-attack/
|
||||||
|
'url': 'arcpublishing:mco:aa0ca6fe-1127-46d4-b32c-be0d6fdb8055',
|
||||||
|
'only_matching': True,
|
||||||
|
}, {
|
||||||
|
# https://www.vl.no/kultur/2020/12/09/en-melding-fra-en-lytter-endret-julelista-til-lewi-bergrud/
|
||||||
|
'url': 'arcpublishing:mentormedier:47a12084-650b-4011-bfd0-3699b6947b2d',
|
||||||
|
'only_matching': True,
|
||||||
|
}, {
|
||||||
|
# https://www.14news.com/2020/12/30/whiskey-theft-caught-camera-henderson-liquor-store/
|
||||||
|
'url': 'arcpublishing:raycom:b89f61f8-79fa-4c09-8255-e64237119bf7',
|
||||||
|
'only_matching': True,
|
||||||
|
}, {
|
||||||
|
# https://www.theglobeandmail.com/world/video-ethiopian-woman-who-became-symbol-of-integration-in-italy-killed-on/
|
||||||
|
'url': 'arcpublishing:tgam:411b34c1-8701-4036-9831-26964711664b',
|
||||||
|
'only_matching': True,
|
||||||
|
}, {
|
||||||
|
# https://www.pilotonline.com/460f2931-8130-4719-8ea1-ffcb2d7cb685-132.html
|
||||||
|
'url': 'arcpublishing:tronc:460f2931-8130-4719-8ea1-ffcb2d7cb685',
|
||||||
|
'only_matching': True,
|
||||||
|
}]
|
||||||
|
_POWA_DEFAULTS = [
|
||||||
|
(['cmg', 'prisa'], '%s-config-prod.api.cdn.arcpublishing.com/video'),
|
||||||
|
([
|
||||||
|
'adn', 'advancelocal', 'answers', 'bonnier', 'bostonglobe', 'demo',
|
||||||
|
'gmg', 'gruponacion', 'infobae', 'mco', 'nzme', 'pmn', 'raycom',
|
||||||
|
'spectator', 'tbt', 'tgam', 'tronc', 'wapo', 'wweek',
|
||||||
|
], 'video-api-cdn.%s.arcpublishing.com/api'),
|
||||||
|
]
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _extract_urls(webpage):
|
||||||
|
entries = []
|
||||||
|
# https://arcpublishing.atlassian.net/wiki/spaces/POWA/overview
|
||||||
|
for powa_el in re.findall(r'(<div[^>]+class="[^"]*\bpowa\b[^"]*"[^>]+data-uuid="%s"[^>]*>)' % ArcPublishingIE._UUID_REGEX, webpage):
|
||||||
|
powa = extract_attributes(powa_el) or {}
|
||||||
|
org = powa.get('data-org')
|
||||||
|
uuid = powa.get('data-uuid')
|
||||||
|
if org and uuid:
|
||||||
|
entries.append('arcpublishing:%s:%s' % (org, uuid))
|
||||||
|
return entries
|
||||||
|
|
||||||
|
def _real_extract(self, url):
|
||||||
|
org, uuid = re.match(self._VALID_URL, url).groups()
|
||||||
|
for orgs, tmpl in self._POWA_DEFAULTS:
|
||||||
|
if org in orgs:
|
||||||
|
base_api_tmpl = tmpl
|
||||||
|
break
|
||||||
|
else:
|
||||||
|
base_api_tmpl = '%s-prod-cdn.video-api.arcpublishing.com/api'
|
||||||
|
if org == 'wapo':
|
||||||
|
org = 'washpost'
|
||||||
|
video = self._download_json(
|
||||||
|
'https://%s/v1/ansvideos/findByUuid' % (base_api_tmpl % org),
|
||||||
|
uuid, query={'uuid': uuid})[0]
|
||||||
|
title = video['headlines']['basic']
|
||||||
|
is_live = video.get('status') == 'live'
|
||||||
|
|
||||||
|
urls = []
|
||||||
|
formats = []
|
||||||
|
for s in video.get('streams', []):
|
||||||
|
s_url = s.get('url')
|
||||||
|
if not s_url or s_url in urls:
|
||||||
|
continue
|
||||||
|
urls.append(s_url)
|
||||||
|
stream_type = s.get('stream_type')
|
||||||
|
if stream_type == 'smil':
|
||||||
|
smil_formats = self._extract_smil_formats(
|
||||||
|
s_url, uuid, fatal=False)
|
||||||
|
for f in smil_formats:
|
||||||
|
if f['url'].endswith('/cfx/st'):
|
||||||
|
f['app'] = 'cfx/st'
|
||||||
|
if not f['play_path'].startswith('mp4:'):
|
||||||
|
f['play_path'] = 'mp4:' + f['play_path']
|
||||||
|
if isinstance(f['tbr'], float):
|
||||||
|
f['vbr'] = f['tbr'] * 1000
|
||||||
|
del f['tbr']
|
||||||
|
f['format_id'] = 'rtmp-%d' % f['vbr']
|
||||||
|
formats.extend(smil_formats)
|
||||||
|
elif stream_type in ('ts', 'hls'):
|
||||||
|
m3u8_formats = self._extract_m3u8_formats(
|
||||||
|
s_url, uuid, 'mp4', 'm3u8' if is_live else 'm3u8_native',
|
||||||
|
m3u8_id='hls', fatal=False)
|
||||||
|
if all([f.get('acodec') == 'none' for f in m3u8_formats]):
|
||||||
|
continue
|
||||||
|
for f in m3u8_formats:
|
||||||
|
if f.get('acodec') == 'none':
|
||||||
|
f['preference'] = -40
|
||||||
|
elif f.get('vcodec') == 'none':
|
||||||
|
f['preference'] = -50
|
||||||
|
height = f.get('height')
|
||||||
|
if not height:
|
||||||
|
continue
|
||||||
|
vbr = self._search_regex(
|
||||||
|
r'[_x]%d[_-](\d+)' % height, f['url'], 'vbr', default=None)
|
||||||
|
if vbr:
|
||||||
|
f['vbr'] = int(vbr)
|
||||||
|
formats.extend(m3u8_formats)
|
||||||
|
else:
|
||||||
|
vbr = int_or_none(s.get('bitrate'))
|
||||||
|
formats.append({
|
||||||
|
'format_id': '%s-%d' % (stream_type, vbr) if vbr else stream_type,
|
||||||
|
'vbr': vbr,
|
||||||
|
'width': int_or_none(s.get('width')),
|
||||||
|
'height': int_or_none(s.get('height')),
|
||||||
|
'filesize': int_or_none(s.get('filesize')),
|
||||||
|
'url': s_url,
|
||||||
|
'preference': -1,
|
||||||
|
})
|
||||||
|
self._sort_formats(
|
||||||
|
formats, ('preference', 'width', 'height', 'vbr', 'filesize', 'tbr', 'ext', 'format_id'))
|
||||||
|
|
||||||
|
subtitles = {}
|
||||||
|
for subtitle in (try_get(video, lambda x: x['subtitles']['urls'], list) or []):
|
||||||
|
subtitle_url = subtitle.get('url')
|
||||||
|
if subtitle_url:
|
||||||
|
subtitles.setdefault('en', []).append({'url': subtitle_url})
|
||||||
|
|
||||||
|
return {
|
||||||
|
'id': uuid,
|
||||||
|
'title': self._live_title(title) if is_live else title,
|
||||||
|
'thumbnail': try_get(video, lambda x: x['promo_image']['url']),
|
||||||
|
'description': try_get(video, lambda x: x['subheadlines']['basic']),
|
||||||
|
'formats': formats,
|
||||||
|
'duration': int_or_none(video.get('duration'), 100),
|
||||||
|
'timestamp': parse_iso8601(video.get('created_date')),
|
||||||
|
'subtitles': subtitles,
|
||||||
|
'is_live': is_live,
|
||||||
|
}
|
Some files were not shown because too many files have changed in this diff Show more
Loading…
Reference in a new issue