[extractor/tunein] Fix extractors (#6310)

Authored by: elyse0
Closes #2973

yt_dlp/extractor/_extractors.py

@@ -1967,10 +1967,9 @@
 )
 from .tumblr import TumblrIE
 from .tunein import (
-    TuneInClipIE,
     TuneInStationIE,
-    TuneInProgramIE,
+    TuneInPodcastIE,
-    TuneInTopicIE,
+    TuneInPodcastEpisodeIE,
     TuneInShortenerIE,
 )
 from .tunepk import TunePkIE

yt_dlp/extractor/tunein.py

@@ -1,149 +1,201 @@
-import re
+import urllib.parse
 
 from .common import InfoExtractor
-from ..utils import ExtractorError
+from ..utils import (
-from ..compat import compat_urlparse
+    OnDemandPagedList,
+    determine_ext,
+    parse_iso8601,
+    traverse_obj,
+)
 
 
 class TuneInBaseIE(InfoExtractor):
-    _API_BASE_URL = 'http://tunein.com/tuner/tune/'
+    _VALID_URL_BASE = r'https?://(?:www\.)?tunein\.com'
 
-    def _real_extract(self, url):
+    def _extract_metadata(self, webpage, content_id):
-        content_id = self._match_id(url)
+        return self._search_json(r'window.INITIAL_STATE=', webpage, 'hydration', content_id, fatal=False)
-
-        content_info = self._download_json(
-            self._API_BASE_URL + self._API_URL_QUERY % content_id,
-            content_id, note='Downloading JSON metadata')
-
-        title = content_info['Title']
-        thumbnail = content_info.get('Logo')
-        location = content_info.get('Location')
-        streams_url = content_info.get('StreamUrl')
-        if not streams_url:
-            raise ExtractorError('No downloadable streams found', expected=True)
-        if not streams_url.startswith('http://'):
-            streams_url = compat_urlparse.urljoin(url, streams_url)
 
+    def _extract_formats_and_subtitles(self, content_id):
         streams = self._download_json(
-            streams_url, content_id, note='Downloading stream data',
+            f'https://opml.radiotime.com/Tune.ashx?render=json&formats=mp3,aac,ogg,flash,hls&id={content_id}',
-            transform_source=lambda s: re.sub(r'^\s*\((.*)\);\s*$', r'\1', s))['Streams']
+            content_id)['body']
 
-        is_live = None
+        formats, subtitles = [], {}
-        formats = []
         for stream in streams:
-            if stream.get('Type') == 'Live':
+            if stream.get('media_type') == 'hls':
-                is_live = True
+                fmts, subs = self._extract_m3u8_formats_and_subtitles(stream['url'], content_id, fatal=False)
-            reliability = stream.get('Reliability')
+                formats.extend(fmts)
-            format_note = (
+                self._merge_subtitles(subs, target=subtitles)
-                'Reliability: %d%%' % reliability
+            elif determine_ext(stream['url']) == 'pls':
-                if reliability is not None else None)
+                playlist_content = self._download_webpage(stream['url'], content_id)
-            formats.append({
+                formats.append({
-                'preference': (
+                    'url': self._search_regex(r'File1=(.*)', playlist_content, 'url', fatal=False),
-                    0 if reliability is None or reliability > 90
+                    'abr': stream.get('bitrate'),
-                    else 1),
+                    'ext': stream.get('media_type'),
-                'abr': stream.get('Bandwidth'),
+                })
-                'ext': stream.get('MediaType').lower(),
+            else:
-                'acodec': stream.get('MediaType'),
+                formats.append({
-                'vcodec': 'none',
+                    'url': stream['url'],
-                'url': stream.get('Url'),
+                    'abr': stream.get('bitrate'),
-                'source_preference': reliability,
+                    'ext': stream.get('media_type'),
-                'format_note': format_note,
+                })
-            })
 
-        return {
+        return formats, subtitles
-            'id': content_id,
-            'title': title,
-            'formats': formats,
-            'thumbnail': thumbnail,
-            'location': location,
-            'is_live': is_live,
-        }
-
-
-class TuneInClipIE(TuneInBaseIE):
-    IE_NAME = 'tunein:clip'
-    _VALID_URL = r'https?://(?:www\.)?tunein\.com/station/.*?audioClipId\=(?P<id>\d+)'
-    _API_URL_QUERY = '?tuneType=AudioClip&audioclipId=%s'
-
-    _TESTS = [{
-        'url': 'http://tunein.com/station/?stationId=246119&audioClipId=816',
-        'md5': '99f00d772db70efc804385c6b47f4e77',
-        'info_dict': {
-            'id': '816',
-            'title': '32m',
-            'ext': 'mp3',
-        },
-    }]
 
 
 class TuneInStationIE(TuneInBaseIE):
-    IE_NAME = 'tunein:station'
+    _VALID_URL = TuneInBaseIE._VALID_URL_BASE + r'(?:/radio/[^?#]+-|/embed/player/)(?P<id>s\d+)'
-    _VALID_URL = r'https?://(?:www\.)?tunein\.com/(?:radio/.*?-s|station/.*?StationId=|embed/player/s)(?P<id>\d+)'
+    _EMBED_REGEX = [r'<iframe[^>]+src=["\'](?P<url>(?:https?://)?tunein\.com/embed/player/s\d+)']
-    _EMBED_REGEX = [r'<iframe[^>]+src=["\'](?P<url>(?:https?://)?tunein\.com/embed/player/[pst]\d+)']
-    _API_URL_QUERY = '?tuneType=Station&stationId=%s'
-
-    @classmethod
-    def suitable(cls, url):
-        return False if TuneInClipIE.suitable(url) else super(TuneInStationIE, cls).suitable(url)
 
     _TESTS = [{
-        'url': 'http://tunein.com/radio/Jazz24-885-s34682/',
+        'url': 'https://tunein.com/radio/Jazz24-885-s34682/',
         'info_dict': {
-            'id': '34682',
+            'id': 's34682',
-            'title': 'Jazz 24 on 88.5 Jazz24 - KPLU-HD2',
+            'title': 're:^Jazz24',
+            'description': 'md5:d6d0b89063fd68d529fa7058ee98619b',
+            'thumbnail': 're:^https?://[^?&]+/s34682',
+            'location': 'Seattle-Tacoma, US',
             'ext': 'mp3',
-            'location': 'Tacoma, WA',
+            'live_status': 'is_live',
         },
         'params': {
-            'skip_download': True,  # live stream
+            'skip_download': True,
         },
     }, {
-        'url': 'http://tunein.com/embed/player/s6404/',
+        'url': 'https://tunein.com/embed/player/s6404/',
         'only_matching': True,
-    }]
+    }, {
-
+        'url': 'https://tunein.com/radio/BBC-Radio-1-988-s24939/',
-
-class TuneInProgramIE(TuneInBaseIE):
-    IE_NAME = 'tunein:program'
-    _VALID_URL = r'https?://(?:www\.)?tunein\.com/(?:radio/.*?-p|program/.*?ProgramId=|embed/player/p)(?P<id>\d+)'
-    _API_URL_QUERY = '?tuneType=Program&programId=%s'
-
-    _TESTS = [{
-        'url': 'http://tunein.com/radio/Jazz-24-p2506/',
         'info_dict': {
-            'id': '2506',
+            'id': 's24939',
-            'title': 'Jazz 24 on 91.3 WUKY-HD3',
+            'title': 're:^BBC Radio 1',
+            'description': 'md5:f3f75f7423398d87119043c26e7bfb84',
+            'thumbnail': 're:^https?://[^?&]+/s24939',
+            'location': 'London, UK',
             'ext': 'mp3',
-            'location': 'Lexington, KY',
+            'live_status': 'is_live',
         },
         'params': {
-            'skip_download': True,  # live stream
+            'skip_download': True,
         },
-    }, {
-        'url': 'http://tunein.com/embed/player/p191660/',
-        'only_matching': True,
     }]
 
+    def _real_extract(self, url):
+        station_id = self._match_id(url)
 
-class TuneInTopicIE(TuneInBaseIE):
+        webpage = self._download_webpage(url, station_id)
-    IE_NAME = 'tunein:topic'
+        metadata = self._extract_metadata(webpage, station_id)
-    _VALID_URL = r'https?://(?:www\.)?tunein\.com/(?:topic/.*?TopicId=|embed/player/t)(?P<id>\d+)'
+
-    _API_URL_QUERY = '?tuneType=Topic&topicId=%s'
+        formats, subtitles = self._extract_formats_and_subtitles(station_id)
+        return {
+            'id': station_id,
+            'title': traverse_obj(metadata, ('profiles', station_id, 'title')),
+            'description': traverse_obj(metadata, ('profiles', station_id, 'description')),
+            'thumbnail': traverse_obj(metadata, ('profiles', station_id, 'image')),
+            'timestamp': parse_iso8601(
+                traverse_obj(metadata, ('profiles', station_id, 'actions', 'play', 'publishTime'))),
+            'location': traverse_obj(
+                metadata, ('profiles', station_id, 'metadata', 'properties', 'location', 'displayName'),
+                ('profiles', station_id, 'properties', 'location', 'displayName')),
+            'formats': formats,
+            'subtitles': subtitles,
+            'is_live': traverse_obj(metadata, ('profiles', station_id, 'actions', 'play', 'isLive')),
+        }
+
+
+class TuneInPodcastIE(TuneInBaseIE):
+    _VALID_URL = TuneInBaseIE._VALID_URL_BASE + r'/(?:podcasts/[^?#]+-|embed/player/)(?P<id>p\d+)/?(?:#|$)'
+    _EMBED_REGEX = [r'<iframe[^>]+src=["\'](?P<url>(?:https?://)?tunein\.com/embed/player/p\d+)']
 
     _TESTS = [{
-        'url': 'http://tunein.com/topic/?TopicId=101830576',
+        'url': 'https://tunein.com/podcasts/Technology-Podcasts/Artificial-Intelligence-p1153019',
-        'md5': 'c31a39e6f988d188252eae7af0ef09c9',
         'info_dict': {
-            'id': '101830576',
+            'id': 'p1153019',
-            'title': 'Votez pour moi du 29 octobre 2015 (29/10/15)',
+            'title': 'Lex Fridman Podcast',
-            'ext': 'mp3',
+            'description': 'md5:bedc4e5f1c94f7dec6e4317b5654b00d',
-            'location': 'Belgium',
         },
+        'playlist_mincount': 200,
     }, {
-        'url': 'http://tunein.com/embed/player/t101830576/',
+        'url': 'https://tunein.com/embed/player/p191660/',
-        'only_matching': True,
+        'only_matching': True
+    }, {
+        'url': 'https://tunein.com/podcasts/World-News/BBC-News-p14/',
+        'info_dict': {
+            'id': 'p14',
+            'title': 'BBC News',
+            'description': 'md5:1218e575eeaff75f48ed978261fa2068',
+        },
+        'playlist_mincount': 200,
     }]
 
+    _PAGE_SIZE = 30
+
+    def _real_extract(self, url):
+        podcast_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, podcast_id, fatal=False)
+        metadata = self._extract_metadata(webpage, podcast_id)
+
+        def page_func(page_num):
+            api_response = self._download_json(
+                f'https://api.tunein.com/profiles/{podcast_id}/contents', podcast_id,
+                note=f'Downloading page {page_num + 1}', query={
+                    'filter': 't:free',
+                    'offset': page_num * self._PAGE_SIZE,
+                    'limit': self._PAGE_SIZE,
+                })
+
+            return [
+                self.url_result(
+                    f'https://tunein.com/podcasts/{podcast_id}?topicId={episode["GuideId"][1:]}',
+                    TuneInPodcastEpisodeIE, title=episode.get('Title'))
+                for episode in api_response['Items']]
+
+        entries = OnDemandPagedList(page_func, self._PAGE_SIZE)
+        return self.playlist_result(
+            entries, playlist_id=podcast_id, title=traverse_obj(metadata, ('profiles', podcast_id, 'title')),
+            description=traverse_obj(metadata, ('profiles', podcast_id, 'description')))
+
+
+class TuneInPodcastEpisodeIE(TuneInBaseIE):
+    _VALID_URL = TuneInBaseIE._VALID_URL_BASE + r'/podcasts/(?:[^?&]+-)?(?P<podcast_id>p\d+)/?\?topicId=(?P<id>\w\d+)'
+
+    _TESTS = [{
+        'url': 'https://tunein.com/podcasts/Technology-Podcasts/Artificial-Intelligence-p1153019/?topicId=236404354',
+        'info_dict': {
+            'id': 't236404354',
+            'title': '#351 \u2013 MrBeast: Future of YouTube, Twitter, TikTok, and Instagram',
+            'description': 'md5:e1734db6f525e472c0c290d124a2ad77',
+            'thumbnail': 're:^https?://[^?&]+/p1153019',
+            'timestamp': 1673458571,
+            'upload_date': '20230111',
+            'series_id': 'p1153019',
+            'series': 'Lex Fridman Podcast',
+            'ext': 'mp3',
+        },
+    }]
+
+    def _real_extract(self, url):
+        podcast_id, episode_id = self._match_valid_url(url).group('podcast_id', 'id')
+        episode_id = f't{episode_id}'
+
+        webpage = self._download_webpage(url, episode_id)
+        metadata = self._extract_metadata(webpage, episode_id)
+
+        formats, subtitles = self._extract_formats_and_subtitles(episode_id)
+        return {
+            'id': episode_id,
+            'title': traverse_obj(metadata, ('profiles', episode_id, 'title')),
+            'description': traverse_obj(metadata, ('profiles', episode_id, 'description')),
+            'thumbnail': traverse_obj(metadata, ('profiles', episode_id, 'image')),
+            'timestamp': parse_iso8601(
+                traverse_obj(metadata, ('profiles', episode_id, 'actions', 'play', 'publishTime'))),
+            'series_id': podcast_id,
+            'series': traverse_obj(metadata, ('profiles', podcast_id, 'title')),
+            'formats': formats,
+            'subtitles': subtitles,
+        }
+
 
 class TuneInShortenerIE(InfoExtractor):
     IE_NAME = 'tunein:shortener'
@@ -154,10 +206,13 @@ class TuneInShortenerIE(InfoExtractor):
         # test redirection
         'url': 'http://tun.in/ser7s',
         'info_dict': {
-            'id': '34682',
+            'id': 's34682',
-            'title': 'Jazz 24 on 88.5 Jazz24 - KPLU-HD2',
+            'title': 're:^Jazz24',
+            'description': 'md5:d6d0b89063fd68d529fa7058ee98619b',
+            'thumbnail': 're:^https?://[^?&]+/s34682',
+            'location': 'Seattle-Tacoma, US',
             'ext': 'mp3',
-            'location': 'Tacoma, WA',
+            'live_status': 'is_live',
         },
         'params': {
             'skip_download': True,  # live stream
@@ -169,6 +224,11 @@ def _real_extract(self, url):
         # The server doesn't support HEAD requests
         urlh = self._request_webpage(
             url, redirect_id, note='Downloading redirect page')
+
         url = urlh.geturl()
+        url_parsed = urllib.parse.urlparse(url)
+        if url_parsed.port == 443:
+            url = url_parsed._replace(netloc=url_parsed.hostname).geturl()
+
         self.to_screen('Following redirect: %s' % url)
         return self.url_result(url)