[fox] Fix extraction (#14147)

youtube_dl/extractor/fox.py

@@ -3,56 +3,99 @@
 
 from .adobepass import AdobePassIE
 from ..utils import (
-    smuggle_url,
+    int_or_none,
-    update_url_query,
+    parse_age_limit,
+    parse_duration,
+    try_get,
+    unified_timestamp,
 )
 
 
 class FOXIE(AdobePassIE):
-    _VALID_URL = r'https?://(?:www\.)?fox\.com/watch/(?P<id>[0-9]+)'
+    _VALID_URL = r'https?://(?:www\.)?fox\.com/watch/(?P<id>[\da-fA-F]+)'
-    _TEST = {
+    _TESTS = [{
-        'url': 'http://www.fox.com/watch/255180355939/7684182528',
+        # clip
+        'url': 'https://www.fox.com/watch/4b765a60490325103ea69888fb2bd4e8/',
         'md5': 'ebd296fcc41dd4b19f8115d8461a3165',
         'info_dict': {
-            'id': '255180355939',
+            'id': '4b765a60490325103ea69888fb2bd4e8',
             'ext': 'mp4',
-            'title': 'Official Trailer: Gotham',
+            'title': 'Aftermath: Bruce Wayne Develops Into The Dark Knight',
-            'description': 'Tracing the rise of the great DC Comics Super-Villains and vigilantes, Gotham reveals an entirely new chapter that has never been told.',
+            'description': 'md5:549cd9c70d413adb32ce2a779b53b486',
-            'duration': 129,
+            'duration': 102,
-            'timestamp': 1400020798,
+            'timestamp': 1504291893,
-            'upload_date': '20140513',
+            'upload_date': '20170901',
-            'uploader': 'NEWA-FNG-FOXCOM',
+            'creator': 'FOX',
+            'series': 'Gotham',
         },
-        'add_ie': ['ThePlatform'],
+        'params': {
-    }
+            'skip_download': True,
+        },
+    }, {
+        # episode, geo-restricted
+        'url': 'https://www.fox.com/watch/087036ca7f33c8eb79b08152b4dd75c1/',
+        'only_matching': True,
+    }, {
+        # episode, geo-restricted, tv provided required
+        'url': 'https://www.fox.com/watch/30056b295fb57f7452aeeb4920bc3024/',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
-        webpage = self._download_webpage(url, video_id)
 
-        settings = self._parse_json(self._search_regex(
+        video = self._download_json(
-            r'jQuery\.extend\(Drupal\.settings\s*,\s*({.+?})\);',
+            'https://api.fox.com/fbc-content/v1_4/video/%s' % video_id,
-            webpage, 'drupal settings'), video_id)
+            video_id, headers={
-        fox_pdk_player = settings['fox_pdk_player']
+                'apikey': 'abdcbed02c124d393b39e818a4312055',
-        release_url = fox_pdk_player['release_url']
+                'Content-Type': 'application/json',
-        query = {
+                'Referer': url,
-            'mbr': 'true',
+            })
-            'switch': 'http'
-        }
-        if fox_pdk_player.get('access') == 'locked':
-            ap_p = settings['foxAdobePassProvider']
-            rating = ap_p.get('videoRating')
-            if rating == 'n/a':
-                rating = None
-            resource = self._get_mvpd_resource('fbc-fox', None, ap_p['videoGUID'], rating)
-            query['auth'] = self._extract_mvpd_auth(url, video_id, 'fbc-fox', resource)
 
-        info = self._search_json_ld(webpage, video_id, fatal=False)
+        title = video['name']
-        info.update({
+
-            '_type': 'url_transparent',
+        m3u8_url = self._download_json(
-            'ie_key': 'ThePlatform',
+            video['videoRelease']['url'], video_id)['playURL']
-            'url': smuggle_url(update_url_query(release_url, query), {'force_smil_url': True}),
+
+        formats = self._extract_m3u8_formats(
+            m3u8_url, video_id, 'mp4',
+            entry_protocol='m3u8_native', m3u8_id='hls')
+        self._sort_formats(formats)
+
+        description = video.get('description')
+        duration = int_or_none(video.get('durationInSeconds')) or int_or_none(
+            video.get('duration')) or parse_duration(video.get('duration'))
+        timestamp = unified_timestamp(video.get('datePublished'))
+        age_limit = parse_age_limit(video.get('contentRating'))
+
+        data = try_get(
+            video, lambda x: x['trackingData']['properties'], dict) or {}
+
+        creator = data.get('brand') or data.get('network') or video.get('network')
+
+        series = video.get('seriesName') or data.get(
+            'seriesName') or data.get('show')
+        season_number = int_or_none(video.get('seasonNumber'))
+        episode = video.get('name')
+        episode_number = int_or_none(video.get('episodeNumber'))
+        release_year = int_or_none(video.get('releaseYear'))
+
+        if data.get('authRequired'):
+            # TODO: AP
+            pass
+
+        return {
             'id': video_id,
-        })
+            'title': title,
-
+            'description': description,
-        return info
+            'duration': duration,
+            'timestamp': timestamp,
+            'age_limit': age_limit,
+            'creator': creator,
+            'series': series,
+            'season_number': season_number,
+            'episode': episode,
+            'episode_number': episode_number,
+            'release_year': release_year,
+            'formats': formats,
+        }