From bf529967401dbac12e32ab852723a59e673dadab Mon Sep 17 00:00:00 2001 From: geauxlo <66712139+geauxlo@users.noreply.github.com> Date: Wed, 10 Jun 2020 04:00:01 +0000 Subject: [PATCH 1/4] Update api call to v5 spec in TwitchPlaylistBaseIE --- youtube_dl/extractor/twitch.py | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/youtube_dl/extractor/twitch.py b/youtube_dl/extractor/twitch.py index e211cd4c8..3f0f7e277 100644 --- a/youtube_dl/extractor/twitch.py +++ b/youtube_dl/extractor/twitch.py @@ -380,11 +380,13 @@ class TwitchPlaylistBaseIE(TwitchBaseIE): _PLAYLIST_PATH = 'kraken/channels/%s/videos/?offset=%d&limit=%d' _PAGE_LIMIT = 100 - def _extract_playlist(self, channel_id): + def _extract_playlist(self, channel_name): info = self._call_api( - 'kraken/channels/%s' % channel_id, - channel_id, 'Downloading channel info JSON') - channel_name = info.get('display_name') or info.get('name') + 'kraken/users?login=%s' % channel_name, + channel_name, 'Downloading channel info JSON') + info = info['users'][0] + channel_id = info['_id'] + channel_name = info.get('display_name') or info.get('name') or channel_name entries = [] offset = 0 limit = self._PAGE_LIMIT @@ -444,7 +446,7 @@ class TwitchProfileIE(TwitchPlaylistBaseIE): _TESTS = [{ 'url': 'http://www.twitch.tv/vanillatv/profile', 'info_dict': { - 'id': 'vanillatv', + 'id': '22744919', 'title': 'VanillaTV', }, 'playlist_mincount': 412, @@ -468,7 +470,7 @@ class TwitchAllVideosIE(TwitchVideosBaseIE): _TESTS = [{ 'url': 'https://www.twitch.tv/spamfish/videos/all', 'info_dict': { - 'id': 'spamfish', + 'id': '497952', 'title': 'Spamfish', }, 'playlist_mincount': 869, @@ -487,7 +489,7 @@ class TwitchUploadsIE(TwitchVideosBaseIE): _TESTS = [{ 'url': 'https://www.twitch.tv/spamfish/videos/uploads', 'info_dict': { - 'id': 'spamfish', + 'id': '497952', 'title': 'Spamfish', }, 'playlist_mincount': 0, @@ -506,7 +508,7 @@ class TwitchPastBroadcastsIE(TwitchVideosBaseIE): _TESTS = [{ 'url': 'https://www.twitch.tv/spamfish/videos/past-broadcasts', 'info_dict': { - 'id': 'spamfish', + 'id': '497952', 'title': 'Spamfish', }, 'playlist_mincount': 0, @@ -525,7 +527,7 @@ class TwitchHighlightsIE(TwitchVideosBaseIE): _TESTS = [{ 'url': 'https://www.twitch.tv/spamfish/videos/highlights', 'info_dict': { - 'id': 'spamfish', + 'id': '497952', 'title': 'Spamfish', }, 'playlist_mincount': 805, From 3951a7faae1c3916827d01b1e242bb12a9cdadec Mon Sep 17 00:00:00 2001 From: geauxlo <66712139+geauxlo@users.noreply.github.com> Date: Wed, 10 Jun 2020 06:38:32 +0000 Subject: [PATCH 2/4] Prefer API to scraping HTML when possible Also changed instances of `var is None` to `var == None`, and replaced `var.replace('http%3A', 'http:')` with a regex --- youtube_dl/extractor/screencast.py | 52 +++++++++++++++++++++++------- 1 file changed, 40 insertions(+), 12 deletions(-) diff --git a/youtube_dl/extractor/screencast.py b/youtube_dl/extractor/screencast.py index 69a0d01f3..d52d46cc3 100644 --- a/youtube_dl/extractor/screencast.py +++ b/youtube_dl/extractor/screencast.py @@ -1,6 +1,8 @@ # coding: utf-8 from __future__ import unicode_literals +import re + from .common import InfoExtractor from ..compat import ( compat_parse_qs, @@ -13,6 +15,8 @@ class ScreencastIE(InfoExtractor): _VALID_URL = r'https?://(?:www\.)?screencast\.com/t/(?P[a-zA-Z0-9]+)' + _API_URL = 'https://www.screencast.com/api/external/oembed?url=%s&format=json' + _TESTS = [{ 'url': 'http://www.screencast.com/t/3ZEjQXlT', 'md5': '917df1c13798a3e96211dd1561fded83', @@ -60,13 +64,32 @@ class ScreencastIE(InfoExtractor): def _real_extract(self, url): video_id = self._match_id(url) + + # The info JSON given by the API has a thumbnail URL, + # but it's inferior to the webpage's thumbnail. + # It also has no video description, so we + # definitely still need to get the webpage. + + info = self._download_json( + self._API_URL % url, video_id, + 'Downloading video info JSON') + + video_url = info.get('url') + if video_url != None: + video_url_raw = compat_urllib_request.quote(video_url) + video_url = re.sub(r'^(?Phttps|http)%3A', + lambda match: '%s:' % match.group('proto'), + video_url_raw) + + title = info.get('title') webpage = self._download_webpage(url, video_id) - video_url = self._html_search_regex( - r'http|https)%3A', + lambda match: '%s:' % match.group('proto'), + video_url_raw) - if video_url is None: + if video_url == None: video_meta = self._html_search_meta( 'og:video', webpage, default=None) if video_meta: @@ -90,28 +115,31 @@ def _real_extract(self, url): r'src=(.*?)(?:$|&)', video_meta, 'meta tag video URL', default=None) - if video_url is None: + if video_url == None: video_url = self._html_search_regex( r'MediaContentUrl["\']\s*:(["\'])(?P(?:(?!\1).)+)\1', webpage, 'video url', default=None, group='url') - if video_url is None: + if video_url == None: video_url = self._html_search_meta( 'og:video', webpage, default=None) - if video_url is None: + if video_url == None: raise ExtractorError('Cannot find video') - title = self._og_search_title(webpage, default=None) - if title is None: + if title == None: + title = self._og_search_title(webpage, default=None) + + if title == None: title = self._html_search_regex( [r'Title: ([^<]+)', r'class="tabSeperator">>(.+?)<', r'([^<]+)'], webpage, 'title') + thumbnail = self._og_search_thumbnail(webpage) description = self._og_search_description(webpage, default=None) - if description is None: + if description == None: description = self._html_search_meta('description', webpage) return { From 66c498e5586e0b90b0579e6e0754cfc03775a3a8 Mon Sep 17 00:00:00 2001 From: geauxlo <66712139+geauxlo@users.noreply.github.com> Date: Wed, 10 Jun 2020 06:39:35 +0000 Subject: [PATCH 3/4] Recognize more valid URLs URLs like `https://www.screencast.com/users/cindyhailes/folders/Jing/media/c9be177c-5808-4c4f-af56-eadceb3a7c82` weren't being accepted before --- youtube_dl/extractor/screencast.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/youtube_dl/extractor/screencast.py b/youtube_dl/extractor/screencast.py index d52d46cc3..d23a53706 100644 --- a/youtube_dl/extractor/screencast.py +++ b/youtube_dl/extractor/screencast.py @@ -14,7 +14,7 @@ class ScreencastIE(InfoExtractor): - _VALID_URL = r'https?://(?:www\.)?screencast\.com/t/(?P[a-zA-Z0-9]+)' + _VALID_URL = r'https?://(?:www\.)?screencast\.com/(?:t|users/[^/]+/folders/[^/]+/media)/(?P[a-zA-Z0-9\-]+)' _API_URL = 'https://www.screencast.com/api/external/oembed?url=%s&format=json' _TESTS = [{ @@ -60,12 +60,22 @@ class ScreencastIE(InfoExtractor): }, { 'url': 'http://screencast.com/t/aAB3iowa', 'only_matching': True, + }, { + 'url': 'https://www.screencast.com/users/cindyhailes/folders/Jing/media/c9be177c-5808-4c4f-af56-eadceb3a7c82', + 'md5': '589d37a28d2add53c8bf16b9126d9dc2', + 'info_dict': { + 'id': 'c9be177c-5808-4c4f-af56-eadceb3a7c82', + 'ext': 'swf', + 'title': '2020-05-31_1737', + 'description': 'Shared from Screencast.com', + 'thumbnail': r're:^https?://.*\.(?:gif|jpg)$', + } }] def _real_extract(self, url): video_id = self._match_id(url) - # The info JSON given by the API has a thumbnail URL, + # The JSON given by the API has a thumbnail URL, # but it's inferior to the webpage's thumbnail. # It also has no video description, so we # definitely still need to get the webpage. From 33afd662d96f82f7a5af4ac89513666f2a078a84 Mon Sep 17 00:00:00 2001 From: geauxlo <66712139+geauxlo@users.noreply.github.com> Date: Wed, 10 Jun 2020 06:42:44 +0000 Subject: [PATCH 4/4] UNDO --- youtube_dl/extractor/screencast.py | 64 ++++++------------------------ 1 file changed, 13 insertions(+), 51 deletions(-) diff --git a/youtube_dl/extractor/screencast.py b/youtube_dl/extractor/screencast.py index d23a53706..69a0d01f3 100644 --- a/youtube_dl/extractor/screencast.py +++ b/youtube_dl/extractor/screencast.py @@ -1,8 +1,6 @@ # coding: utf-8 from __future__ import unicode_literals -import re - from .common import InfoExtractor from ..compat import ( compat_parse_qs, @@ -14,9 +12,7 @@ class ScreencastIE(InfoExtractor): - _VALID_URL = r'https?://(?:www\.)?screencast\.com/(?:t|users/[^/]+/folders/[^/]+/media)/(?P[a-zA-Z0-9\-]+)' - _API_URL = 'https://www.screencast.com/api/external/oembed?url=%s&format=json' - + _VALID_URL = r'https?://(?:www\.)?screencast\.com/t/(?P[a-zA-Z0-9]+)' _TESTS = [{ 'url': 'http://www.screencast.com/t/3ZEjQXlT', 'md5': '917df1c13798a3e96211dd1561fded83', @@ -60,46 +56,17 @@ class ScreencastIE(InfoExtractor): }, { 'url': 'http://screencast.com/t/aAB3iowa', 'only_matching': True, - }, { - 'url': 'https://www.screencast.com/users/cindyhailes/folders/Jing/media/c9be177c-5808-4c4f-af56-eadceb3a7c82', - 'md5': '589d37a28d2add53c8bf16b9126d9dc2', - 'info_dict': { - 'id': 'c9be177c-5808-4c4f-af56-eadceb3a7c82', - 'ext': 'swf', - 'title': '2020-05-31_1737', - 'description': 'Shared from Screencast.com', - 'thumbnail': r're:^https?://.*\.(?:gif|jpg)$', - } }] def _real_extract(self, url): video_id = self._match_id(url) - - # The JSON given by the API has a thumbnail URL, - # but it's inferior to the webpage's thumbnail. - # It also has no video description, so we - # definitely still need to get the webpage. - - info = self._download_json( - self._API_URL % url, video_id, - 'Downloading video info JSON') - - video_url = info.get('url') - if video_url != None: - video_url_raw = compat_urllib_request.quote(video_url) - video_url = re.sub(r'^(?Phttps|http)%3A', - lambda match: '%s:' % match.group('proto'), - video_url_raw) - - title = info.get('title') webpage = self._download_webpage(url, video_id) - if video_url == None: - video_url = self._html_search_regex( - r'http|https)%3A', - lambda match: '%s:' % match.group('proto'), - video_url_raw) + video_url = video_url_raw.replace('http%3A', 'http:') - if video_url == None: + if video_url is None: video_meta = self._html_search_meta( 'og:video', webpage, default=None) if video_meta: @@ -125,31 +90,28 @@ def _real_extract(self, url): r'src=(.*?)(?:$|&)', video_meta, 'meta tag video URL', default=None) - if video_url == None: + if video_url is None: video_url = self._html_search_regex( r'MediaContentUrl["\']\s*:(["\'])(?P(?:(?!\1).)+)\1', webpage, 'video url', default=None, group='url') - if video_url == None: + if video_url is None: video_url = self._html_search_meta( 'og:video', webpage, default=None) - if video_url == None: + if video_url is None: raise ExtractorError('Cannot find video') - if title == None: - title = self._og_search_title(webpage, default=None) - - if title == None: + title = self._og_search_title(webpage, default=None) + if title is None: title = self._html_search_regex( [r'Title: ([^<]+)', r'class="tabSeperator">>(.+?)<', r'([^<]+)'], webpage, 'title') - thumbnail = self._og_search_thumbnail(webpage) description = self._og_search_description(webpage, default=None) - if description == None: + if description is None: description = self._html_search_meta('description', webpage) return {