From dd634acd71620877e4543cfae66c30302505605d Mon Sep 17 00:00:00 2001 From: Locke Date: Wed, 13 Jul 2022 22:18:03 +0800 Subject: [PATCH] [extractor/Ximalaya] Fix extractors (#4339) Authored by: lockmatrix --- yt_dlp/extractor/ximalaya.py | 157 ++++++++++------------------------- 1 file changed, 44 insertions(+), 113 deletions(-) diff --git a/yt_dlp/extractor/ximalaya.py b/yt_dlp/extractor/ximalaya.py index c3447fba0..b25be772e 100644 --- a/yt_dlp/extractor/ximalaya.py +++ b/yt_dlp/extractor/ximalaya.py @@ -1,7 +1,7 @@ -import itertools -import re +import math from .common import InfoExtractor +from ..utils import traverse_obj, try_call, InAdvancePagedList class XimalayaBaseIE(InfoExtractor): @@ -11,11 +11,10 @@ class XimalayaBaseIE(InfoExtractor): class XimalayaIE(XimalayaBaseIE): IE_NAME = 'ximalaya' IE_DESC = '喜马拉雅FM' - _VALID_URL = r'https?://(?:www\.|m\.)?ximalaya\.com/(?P[0-9]+)/sound/(?P[0-9]+)' - _USER_URL_FORMAT = '%s://www.ximalaya.com/zhubo/%i/' + _VALID_URL = r'https?://(?:www\.|m\.)?ximalaya\.com/(:?(?P\d+)/)?sound/(?P[0-9]+)' _TESTS = [ { - 'url': 'http://www.ximalaya.com/61425525/sound/47740352/', + 'url': 'http://www.ximalaya.com/sound/47740352/', 'info_dict': { 'id': '47740352', 'ext': 'm4a', @@ -24,19 +23,20 @@ class XimalayaIE(XimalayaBaseIE): 'uploader_url': 'http://www.ximalaya.com/zhubo/61425525/', 'title': '261.唐诗三百首.卷八.送孟浩然之广陵.李白', 'description': "contains:《送孟浩然之广陵》\n作者：李白\n故人西辞黄鹤楼，烟花三月下扬州。\n孤帆远影碧空尽，惟见长江天际流。", + 'thumbnail': r're:^https?://.*\.jpg', 'thumbnails': [ { 'name': 'cover_url', - 'url': r're:^https?://.*\.jpg$', + 'url': r're:^https?://.*\.jpg', }, { 'name': 'cover_url_142', - 'url': r're:^https?://.*\.jpg$', + 'url': r're:^https?://.*\.jpg', 'width': 180, 'height': 180 } ], - 'categories': ['renwen', '人文'], + 'categories': ['人文'], 'duration': 93, 'view_count': int, 'like_count': int, @@ -52,77 +52,42 @@ class XimalayaIE(XimalayaBaseIE): 'uploader_url': 'http://www.ximalaya.com/zhubo/61425525/', 'title': '261.唐诗三百首.卷八.送孟浩然之广陵.李白', 'description': "contains:《送孟浩然之广陵》\n作者：李白\n故人西辞黄鹤楼，烟花三月下扬州。\n孤帆远影碧空尽，惟见长江天际流。", + 'thumbnail': r're:^https?://.*\.jpg', 'thumbnails': [ { 'name': 'cover_url', - 'url': r're:^https?://.*\.jpg$', + 'url': r're:^https?://.*\.jpg', }, { 'name': 'cover_url_142', - 'url': r're:^https?://.*\.jpg$', + 'url': r're:^https?://.*\.jpg', 'width': 180, 'height': 180 } ], - 'categories': ['renwen', '人文'], + 'categories': ['人文'], 'duration': 93, 'view_count': int, 'like_count': int, } - }, - { - 'url': 'https://www.ximalaya.com/11045267/sound/15705996/', - 'info_dict': { - 'id': '15705996', - 'ext': 'm4a', - 'uploader': '李延隆老师', - 'uploader_id': 11045267, - 'uploader_url': 'https://www.ximalaya.com/zhubo/11045267/', - 'title': 'Lesson 1 Excuse me!', - 'description': "contains:Listen to the tape then answer\xa0this question. Whose handbag is it?\n" - "听录音，然后回答问题，这是谁的手袋？", - 'thumbnails': [ - { - 'name': 'cover_url', - 'url': r're:^https?://.*\.jpg$', - }, - { - 'name': 'cover_url_142', - 'url': r're:^https?://.*\.jpg$', - 'width': 180, - 'height': 180 - } - ], - 'categories': ['train', '外语'], - 'duration': 40, - 'view_count': int, - 'like_count': int, - } - }, + } ] def _real_extract(self, url): - - is_m = 'm.ximalaya' in url scheme = 'https' if url.startswith('https') else 'http' audio_id = self._match_id(url) - webpage = self._download_webpage(url, audio_id, - note='Download sound page for %s' % audio_id, - errnote='Unable to get sound page') - audio_info_file = '%s://m.ximalaya.com/tracks/%s.json' % (scheme, audio_id) audio_info = self._download_json(audio_info_file, audio_id, 'Downloading info json %s' % audio_info_file, 'Unable to download info file') - formats = [] - for bps, k in (('24k', 'play_path_32'), ('64k', 'play_path_64')): - if audio_info.get(k): - formats.append({ - 'format_id': bps, - 'url': audio_info[k], - }) + formats = [{ + 'format_id': f'{bps}k', + 'url': audio_info[k], + 'abr': bps, + 'vcodec': 'none' + } for bps, k in ((24, 'play_path_32'), (64, 'play_path_64')) if audio_info.get(k)] thumbnails = [] for k in audio_info.keys(): @@ -136,30 +101,18 @@ def _real_extract(self, url): audio_uploader_id = audio_info.get('uid') - if is_m: - audio_description = self._html_search_regex(r'(?s)]+>(.+?)', - webpage, 'audio_description', fatal=False) - else: - audio_description = self._html_search_regex(r'(?s)]*>(.+?)', - webpage, 'audio_description', fatal=False) - - if not audio_description: - audio_description_file = '%s://www.ximalaya.com/sounds/%s/rich_intro' % (scheme, audio_id) - audio_description = self._download_webpage(audio_description_file, audio_id, - note='Downloading description file %s' % audio_description_file, - errnote='Unable to download descrip file', - fatal=False) - audio_description = audio_description.strip() if audio_description else None + audio_description = try_call( + lambda: audio_info['intro'].replace('\r\n\r\n\r\n ', '\n').replace('\r\n', '\n')) return { 'id': audio_id, 'uploader': audio_info.get('nickname'), 'uploader_id': audio_uploader_id, - 'uploader_url': self._USER_URL_FORMAT % (scheme, audio_uploader_id) if audio_uploader_id else None, + 'uploader_url': f'{scheme}://www.ximalaya.com/zhubo/{audio_uploader_id}/' if audio_uploader_id else None, 'title': audio_info['title'], 'thumbnails': thumbnails, 'description': audio_description, - 'categories': list(filter(None, (audio_info.get('category_name'), audio_info.get('category_title')))), + 'categories': list(filter(None, [audio_info.get('category_name')])), 'duration': audio_info.get('duration'), 'view_count': audio_info.get('play_count'), 'like_count': audio_info.get('favorites_count'), @@ -170,60 +123,38 @@ def _real_extract(self, url): class XimalayaAlbumIE(XimalayaBaseIE): IE_NAME = 'ximalaya:album' IE_DESC = '喜马拉雅FM 专辑' - _VALID_URL = r'https?://(?:www\.|m\.)?ximalaya\.com/(?P[0-9]+)/album/(?P[0-9]+)' - _TEMPLATE_URL = '%s://www.ximalaya.com/%s/album/%s/' - _BASE_URL_TEMPL = '%s://www.ximalaya.com%s' - _LIST_VIDEO_RE = r']+?href="(?P/%s/sound/(?P\d+)/?)"[^>]+?title="(?P[^>]+)">' + _VALID_URL = r'https?://(?:www\.|m\.)?ximalaya\.com/\d+/album/(?P<id>[0-9]+)' _TESTS = [{ 'url': 'http://www.ximalaya.com/61425525/album/5534601/', 'info_dict': { 'title': '唐诗三百首（含赏析）', 'id': '5534601', }, - 'playlist_count': 312, - }, { - 'url': 'http://m.ximalaya.com/61425525/album/5534601', - 'info_dict': { - 'title': '唐诗三百首（含赏析）', - 'id': '5534601', - }, - 'playlist_count': 312, - }, - ] + 'playlist_mincount': 323, + }] def _real_extract(self, url): - self.scheme = scheme = 'https' if url.startswith('https') else 'http' + playlist_id = self._match_id(url) - mobj = self._match_valid_url(url) - uid, playlist_id = mobj.group('uid'), mobj.group('id') + first_page = self._fetch_page(playlist_id, 1) + page_count = math.ceil(first_page['trackTotalCount'] / first_page['pageSize']) - webpage = self._download_webpage(self._TEMPLATE_URL % (scheme, uid, playlist_id), playlist_id, - note='Download album page for %s' % playlist_id, - errnote='Unable to get album info') + entries = InAdvancePagedList( + lambda idx: self._get_entries(self._fetch_page(playlist_id, idx + 1) if idx else first_page), + page_count, first_page['pageSize']) - title = self._html_search_regex(r'detailContent_title[^>]*><h1(?:[^>]+)?>([^<]+)</h1>', - webpage, 'title', fatal=False) + title = traverse_obj(first_page, ('tracks', 0, 'albumTitle'), expected_type=str) - return self.playlist_result(self._entries(webpage, playlist_id, uid), playlist_id, title) + return self.playlist_result(entries, playlist_id, title) - def _entries(self, page, playlist_id, uid): - html = page - for page_num in itertools.count(1): - for entry in self._process_page(html, uid): - yield entry + def _fetch_page(self, playlist_id, page_idx): + return self._download_json( + 'https://www.ximalaya.com/revision/album/v1/getTracksList', + playlist_id, note=f'Downloading tracks list page {page_idx}', + query={'albumId': playlist_id, 'pageNum': page_idx, 'sort': 1})['data'] - next_url = self._search_regex(r'<a\s+href=(["\'])(?P<more>[\S]+)\1[^>]+rel=(["\'])next\3', - html, 'list_next_url', default=None, group='more') - if not next_url: - break - - next_full_url = self._BASE_URL_TEMPL % (self.scheme, next_url) - html = self._download_webpage(next_full_url, playlist_id) - - def _process_page(self, html, uid): - find_from = html.index('album_soundlist') - for mobj in re.finditer(self._LIST_VIDEO_RE % uid, html[find_from:]): - yield self.url_result(self._BASE_URL_TEMPL % (self.scheme, mobj.group('url')), - XimalayaIE.ie_key(), - mobj.group('id'), - mobj.group('title')) + def _get_entries(self, page_data): + for e in page_data['tracks']: + yield self.url_result( + self._proto_relative_url(f'//www.ximalaya.com{e["url"]}'), + XimalayaIE, e.get('trackId'), e.get('title'))