From 000c15a4cad1079fe91609a5d5a0ff8d20cca865 Mon Sep 17 00:00:00 2001
From: pukkandan <pukkandan.ytdlp@gmail.com>
Date: Sat, 31 Jul 2021 01:15:04 +0530
Subject: [PATCH] [youtube] simplify and de-duplicate client definitions (#577)

---
 yt_dlp/extractor/youtube.py | 406 ++++++++++++++----------------------
 1 file changed, 157 insertions(+), 249 deletions(-)

diff --git a/yt_dlp/extractor/youtube.py b/yt_dlp/extractor/youtube.py
index 76d2038e0..365c2b95b 100644
--- a/yt_dlp/extractor/youtube.py
+++ b/yt_dlp/extractor/youtube.py
@@ -67,6 +67,137 @@ def parse_qs(url):
     return compat_urlparse.parse_qs(compat_urlparse.urlparse(url).query)
 
 
+# any clients starting with _ cannot be explicity requested by the user
+INNERTUBE_CLIENTS = {
+    'web': {
+        'INNERTUBE_API_KEY': 'AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8',
+        'INNERTUBE_CONTEXT': {
+            'client': {
+                'clientName': 'WEB',
+                'clientVersion': '2.20210622.10.00',
+            }
+        },
+        'INNERTUBE_CONTEXT_CLIENT_NAME': 1
+    },
+    'web_embedded': {
+        'INNERTUBE_API_KEY': 'AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8',
+        'INNERTUBE_CONTEXT': {
+            'client': {
+                'clientName': 'WEB_EMBEDDED_PLAYER',
+                'clientVersion': '1.20210620.0.1',
+            },
+        },
+        'INNERTUBE_CONTEXT_CLIENT_NAME': 56
+    },
+    'web_music': {
+        'INNERTUBE_API_KEY': 'AIzaSyC9XL3ZjWddXya6X74dJoCTL-WEYFDNX30',
+        'INNERTUBE_HOST': 'music.youtube.com',
+        'INNERTUBE_CONTEXT': {
+            'client': {
+                'clientName': 'WEB_REMIX',
+                'clientVersion': '1.20210621.00.00',
+            }
+        },
+        'INNERTUBE_CONTEXT_CLIENT_NAME': 67,
+    },
+    'android': {
+        'INNERTUBE_API_KEY': 'AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8',
+        'INNERTUBE_CONTEXT': {
+            'client': {
+                'clientName': 'ANDROID',
+                'clientVersion': '16.20',
+            }
+        },
+        'INNERTUBE_CONTEXT_CLIENT_NAME': 3,
+    },
+    'android_embedded': {
+        'INNERTUBE_API_KEY': 'AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8',
+        'INNERTUBE_CONTEXT': {
+            'client': {
+                'clientName': 'ANDROID_EMBEDDED_PLAYER',
+                'clientVersion': '16.20',
+            },
+        },
+        'INNERTUBE_CONTEXT_CLIENT_NAME': 55
+    },
+    'android_music': {
+        'INNERTUBE_API_KEY': 'AIzaSyC9XL3ZjWddXya6X74dJoCTL-WEYFDNX30',
+        'INNERTUBE_HOST': 'music.youtube.com',
+        'INNERTUBE_CONTEXT': {
+            'client': {
+                'clientName': 'ANDROID_MUSIC',
+                'clientVersion': '4.32',
+            }
+        },
+        'INNERTUBE_CONTEXT_CLIENT_NAME': 21,
+    },
+    'ios': {
+        'INNERTUBE_API_KEY': 'AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8',
+        'INNERTUBE_CONTEXT': {
+            'client': {
+                'clientName': 'IOS',
+                'clientVersion': '16.20',
+            }
+        },
+        'INNERTUBE_CONTEXT_CLIENT_NAME': 5
+    },
+    'ios_embedded': {
+        'INNERTUBE_API_KEY': 'AIzaSyDCU8hByM-4DrUqRUYnGn-3llEO78bcxq8',
+        'INNERTUBE_CONTEXT': {
+            'client': {
+                'clientName': 'IOS_MESSAGES_EXTENSION',
+                'clientVersion': '16.20',
+            },
+        },
+        'INNERTUBE_CONTEXT_CLIENT_NAME': 66
+    },
+    'ios_music': {
+        'INNERTUBE_API_KEY': 'AIzaSyDK3iBpDP9nHVTk2qL73FLJICfOC3c51Og',
+        'INNERTUBE_HOST': 'music.youtube.com',
+        'INNERTUBE_CONTEXT': {
+            'client': {
+                'clientName': 'IOS_MUSIC',
+                'clientVersion': '4.32',
+            },
+        },
+        'INNERTUBE_CONTEXT_CLIENT_NAME': 26
+    },
+    'mweb': {
+        'INNERTUBE_API_KEY': 'AIzaSyDCU8hByM-4DrUqRUYnGn-3llEO78bcxq8',
+        'INNERTUBE_CONTEXT': {
+            'client': {
+                'clientName': 'MWEB',
+                'clientVersion': '2.20210721.07.00',
+            }
+        },
+        'INNERTUBE_CONTEXT_CLIENT_NAME': 2
+    },
+}
+
+
+def build_innertube_clients():
+    base_clients = ('android', 'web', 'ios', 'mweb')
+    priority = qualities(base_clients[::-1])
+
+    for client, ytcfg in tuple(INNERTUBE_CLIENTS.items()):
+        ytcfg.setdefault('INNERTUBE_API_KEY', 'AIzaSyDCU8hByM4DrUqRUYnGn3llEO78bcxq8')
+        ytcfg.setdefault('INNERTUBE_HOST', 'www.youtube.com')
+        ytcfg['INNERTUBE_CONTEXT']['client'].setdefault('hl', 'en')
+        ytcfg['priority'] = 10 * priority(client.split('_', 1)[0])
+
+        if client in base_clients:
+            INNERTUBE_CLIENTS[f'{client}_agegate'] = agegate_ytcfg = copy.deepcopy(ytcfg)
+            agegate_ytcfg['INNERTUBE_CONTEXT']['client']['clientScreen'] = 'EMBED'
+            agegate_ytcfg['priority'] -= 1
+        elif client.endswith('_embedded'):
+            ytcfg['priority'] -= 2
+        else:
+            ytcfg['priority'] -= 3
+
+
+build_innertube_clients()
+
+
 class YoutubeBaseInfoExtractor(InfoExtractor):
     """Provide base functions for Youtube extractors"""
     _LOGIN_URL = 'https://accounts.google.com/ServiceLogin'
@@ -312,250 +443,22 @@ def _real_initialize(self):
     _YT_INITIAL_PLAYER_RESPONSE_RE = r'ytInitialPlayerResponse\s*=\s*({.+?})\s*;'
     _YT_INITIAL_BOUNDARY_RE = r'(?:var\s+meta|</script|\n)'
 
-    _YT_DEFAULT_YTCFGS = {
-        'WEB': {
-            'INNERTUBE_API_VERSION': 'v1',
-            'INNERTUBE_CLIENT_NAME': 'WEB',
-            'INNERTUBE_CLIENT_VERSION': '2.20210622.10.00',
-            'INNERTUBE_API_KEY': 'AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8',
-            'INNERTUBE_CONTEXT': {
-                'client': {
-                    'clientName': 'WEB',
-                    'clientVersion': '2.20210622.10.00',
-                    'hl': 'en',
-                }
-            },
-            'INNERTUBE_CONTEXT_CLIENT_NAME': 1
-        },
-        'WEB_AGEGATE': {
-            'INNERTUBE_API_VERSION': 'v1',
-            'INNERTUBE_CLIENT_NAME': 'WEB',
-            'INNERTUBE_CLIENT_VERSION': '2.20210622.10.00',
-            'INNERTUBE_API_KEY': 'AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8',
-            'INNERTUBE_CONTEXT': {
-                'client': {
-                    'clientName': 'WEB',
-                    'clientVersion': '2.20210622.10.00',
-                    'clientScreen': 'EMBED',
-                    'hl': 'en',
-                }
-            },
-            'INNERTUBE_CONTEXT_CLIENT_NAME': 1
-        },
-        'WEB_REMIX': {
-            'INNERTUBE_API_VERSION': 'v1',
-            'INNERTUBE_CLIENT_NAME': 'WEB_REMIX',
-            'INNERTUBE_CLIENT_VERSION': '1.20210621.00.00',
-            'INNERTUBE_API_KEY': 'AIzaSyC9XL3ZjWddXya6X74dJoCTL-WEYFDNX30',
-            'INNERTUBE_CONTEXT': {
-                'client': {
-                    'clientName': 'WEB_REMIX',
-                    'clientVersion': '1.20210621.00.00',
-                    'hl': 'en',
-                }
-            },
-            'INNERTUBE_CONTEXT_CLIENT_NAME': 67
-        },
-        'WEB_EMBEDDED_PLAYER': {
-            'INNERTUBE_API_VERSION': 'v1',
-            'INNERTUBE_CLIENT_NAME': 'WEB_EMBEDDED_PLAYER',
-            'INNERTUBE_CLIENT_VERSION': '1.20210620.0.1',
-            'INNERTUBE_API_KEY': 'AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8',
-            'INNERTUBE_CONTEXT': {
-                'client': {
-                    'clientName': 'WEB_EMBEDDED_PLAYER',
-                    'clientVersion': '1.20210620.0.1',
-                    'hl': 'en',
-                }
-            },
-            'INNERTUBE_CONTEXT_CLIENT_NAME': 56
-        },
-        'ANDROID': {
-            'INNERTUBE_API_VERSION': 'v1',
-            'INNERTUBE_CLIENT_NAME': 'ANDROID',
-            'INNERTUBE_CLIENT_VERSION': '16.20',
-            'INNERTUBE_API_KEY': 'AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8',
-            'INNERTUBE_CONTEXT': {
-                'client': {
-                    'clientName': 'ANDROID',
-                    'clientVersion': '16.20',
-                    'hl': 'en',
-                }
-            },
-            'INNERTUBE_CONTEXT_CLIENT_NAME': 3
-        },
-        'ANDROID_AGEGATE': {
-            'INNERTUBE_API_VERSION': 'v1',
-            'INNERTUBE_CLIENT_NAME': 'ANDROID',
-            'INNERTUBE_CLIENT_VERSION': '16.20',
-            'INNERTUBE_API_KEY': 'AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8',
-            'INNERTUBE_CONTEXT': {
-                'client': {
-                    'clientName': 'ANDROID',
-                    'clientVersion': '16.20',
-                    'clientScreen': 'EMBED',
-                    'hl': 'en',
-                }
-            },
-            'INNERTUBE_CONTEXT_CLIENT_NAME': 3
-        },
-        'ANDROID_EMBEDDED_PLAYER': {
-            'INNERTUBE_API_VERSION': 'v1',
-            'INNERTUBE_CLIENT_NAME': 'ANDROID_EMBEDDED_PLAYER',
-            'INNERTUBE_CLIENT_VERSION': '16.20',
-            'INNERTUBE_API_KEY': 'AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8',
-            'INNERTUBE_CONTEXT': {
-                'client': {
-                    'clientName': 'ANDROID_EMBEDDED_PLAYER',
-                    'clientVersion': '16.20',
-                    'hl': 'en',
-                }
-            },
-            'INNERTUBE_CONTEXT_CLIENT_NAME': 55
-        },
-        'ANDROID_MUSIC': {
-            'INNERTUBE_API_VERSION': 'v1',
-            'INNERTUBE_CLIENT_NAME': 'ANDROID_MUSIC',
-            'INNERTUBE_CLIENT_VERSION': '4.32',
-            'INNERTUBE_API_KEY': 'AIzaSyC9XL3ZjWddXya6X74dJoCTL-WEYFDNX30',
-            'INNERTUBE_CONTEXT': {
-                'client': {
-                    'clientName': 'ANDROID_MUSIC',
-                    'clientVersion': '4.32',
-                    'hl': 'en',
-                }
-            },
-            'INNERTUBE_CONTEXT_CLIENT_NAME': 21
-        },
-        'IOS': {
-            'INNERTUBE_API_VERSION': 'v1',
-            'INNERTUBE_CLIENT_NAME': 'IOS',
-            'INNERTUBE_CLIENT_VERSION': '16.20',
-            'INNERTUBE_API_KEY': 'AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8',
-            'INNERTUBE_CONTEXT': {
-                'client': {
-                    'clientName': 'IOS',
-                    'clientVersion': '16.20',
-                    'hl': 'en',
-                }
-            },
-            'INNERTUBE_CONTEXT_CLIENT_NAME': 5
-        },
-        'IOS_AGEGATE': {
-            'INNERTUBE_API_VERSION': 'v1',
-            'INNERTUBE_CLIENT_NAME': 'IOS',
-            'INNERTUBE_CLIENT_VERSION': '16.20',
-            'INNERTUBE_API_KEY': 'AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8',
-            'INNERTUBE_CONTEXT': {
-                'client': {
-                    'clientName': 'IOS',
-                    'clientVersion': '16.20',
-                    'clientScreen': 'EMBED',
-                    'hl': 'en',
-                }
-            },
-            'INNERTUBE_CONTEXT_CLIENT_NAME': 5
-        },
-        'IOS_MUSIC': {
-            'INNERTUBE_API_VERSION': 'v1',
-            'INNERTUBE_CLIENT_NAME': 'IOS_MUSIC',
-            'INNERTUBE_CLIENT_VERSION': '4.32',
-            'INNERTUBE_API_KEY': 'AIzaSyDK3iBpDP9nHVTk2qL73FLJICfOC3c51Og',
-            'INNERTUBE_CONTEXT': {
-                'client': {
-                    'clientName': 'IOS_MUSIC',
-                    'clientVersion': '4.32',
-                    'hl': 'en',
-                }
-            },
-            'INNERTUBE_CONTEXT_CLIENT_NAME': 26
-        },
-        'IOS_MESSAGES_EXTENSION': {
-            'INNERTUBE_API_VERSION': 'v1',
-            'INNERTUBE_CLIENT_NAME': 'IOS_MESSAGES_EXTENSION',
-            'INNERTUBE_CLIENT_VERSION': '16.20',
-            'INNERTUBE_API_KEY': 'AIzaSyDCU8hByM-4DrUqRUYnGn-3llEO78bcxq8',
-            'INNERTUBE_CONTEXT': {
-                'client': {
-                    'clientName': 'IOS_MESSAGES_EXTENSION',
-                    'clientVersion': '16.20',
-                    'hl': 'en',
-                }
-            },
-            'INNERTUBE_CONTEXT_CLIENT_NAME': 66
-        },
-        'MWEB': {
-            'INNERTUBE_API_VERSION': 'v1',
-            'INNERTUBE_CLIENT_NAME': 'MWEB',
-            'INNERTUBE_CLIENT_VERSION': '2.20210721.07.00',
-            'INNERTUBE_API_KEY': 'AIzaSyDCU8hByM-4DrUqRUYnGn-3llEO78bcxq8',
-            'INNERTUBE_CONTEXT': {
-                'client': {
-                    'clientName': 'MWEB',
-                    'clientVersion': '2.20210721.07.00',
-                    'hl': 'en',
-                }
-            },
-            'INNERTUBE_CONTEXT_CLIENT_NAME': 2
-        },
-        'MWEB_AGEGATE': {
-            'INNERTUBE_API_VERSION': 'v1',
-            'INNERTUBE_CLIENT_NAME': 'MWEB',
-            'INNERTUBE_CLIENT_VERSION': '2.20210721.07.00',
-            'INNERTUBE_API_KEY': 'AIzaSyDCU8hByM-4DrUqRUYnGn-3llEO78bcxq8',
-            'INNERTUBE_CONTEXT': {
-                'client': {
-                    'clientName': 'MWEB',
-                    'clientVersion': '2.20210721.07.00',
-                    'clientScreen': 'EMBED',
-                    'hl': 'en',
-                }
-            },
-            'INNERTUBE_CONTEXT_CLIENT_NAME': 2
-        },
-    }
+    def _get_default_ytcfg(self, client='web'):
+        return copy.deepcopy(INNERTUBE_CLIENTS[client])
 
-    _YT_DEFAULT_INNERTUBE_HOSTS = {
-        'DIRECT': 'youtubei.googleapis.com',
-        'WEB': 'www.youtube.com',
-        'WEB_REMIX': 'music.youtube.com',
-        'ANDROID_MUSIC': 'music.youtube.com'
-    }
+    def _get_innertube_host(self, client='web'):
+        return INNERTUBE_CLIENTS[client]['INNERTUBE_HOST']
 
-    # clients starting with _ cannot be explicity requested by the user
-    _YT_CLIENTS = {
-        'android': 'ANDROID',
-        'android_music': 'ANDROID_MUSIC',
-        'android_embedded': 'ANDROID_EMBEDDED_PLAYER',
-        'android_agegate': 'ANDROID_AGEGATE',
-        'ios': 'IOS',
-        'ios_music': 'IOS_MUSIC',
-        'ios_embedded': 'IOS_MESSAGES_EXTENSION',
-        'ios_agegate': 'IOS_AGEGATE',
-        'web': 'WEB',
-        'web_music': 'WEB_REMIX',
-        'web_embedded': 'WEB_EMBEDDED_PLAYER',
-        'web_agegate': 'WEB_AGEGATE',
-        'mweb': 'MWEB',
-        'mweb_agegate': 'MWEB_AGEGATE',
-    }
-
-    def _get_default_ytcfg(self, client='WEB'):
-        if client in self._YT_DEFAULT_YTCFGS:
-            return copy.deepcopy(self._YT_DEFAULT_YTCFGS[client])
-        self.write_debug(f'INNERTUBE default client {client} does not exist - falling back to WEB client.')
-        return copy.deepcopy(self._YT_DEFAULT_YTCFGS['WEB'])
-
-    def _get_innertube_host(self, client='WEB'):
-        return dict_get(self._YT_DEFAULT_INNERTUBE_HOSTS, (client, 'WEB'))
-
-    def _ytcfg_get_safe(self, ytcfg, getter, expected_type=None, default_client='WEB'):
+    def _ytcfg_get_safe(self, ytcfg, getter, expected_type=None, default_client='web'):
         # try_get but with fallback to default ytcfg client values when present
         _func = lambda y: try_get(y, getter, expected_type)
         return _func(ytcfg) or _func(self._get_default_ytcfg(default_client))
 
-    def _extract_client_name(self, ytcfg, default_client='WEB'):
-        return self._ytcfg_get_safe(ytcfg, lambda x: x['INNERTUBE_CLIENT_NAME'], compat_str, default_client)
+    def _extract_client_name(self, ytcfg, default_client='web'):
+        return (
+            try_get(ytcfg, lambda x: x['INNERTUBE_CLIENT_NAME'], compat_str)
+            or self._ytcfg_get_safe(
+                ytcfg, lambda x: x['INNERTUBE_CONTEXT']['client']['clientName'], compat_str, default_client))
 
     @staticmethod
     def _extract_session_index(*data):
@@ -564,13 +467,16 @@ def _extract_session_index(*data):
             if session_index is not None:
                 return session_index
 
-    def _extract_client_version(self, ytcfg, default_client='WEB'):
-        return self._ytcfg_get_safe(ytcfg, lambda x: x['INNERTUBE_CLIENT_VERSION'], compat_str, default_client)
+    def _extract_client_version(self, ytcfg, default_client='web'):
+        return (
+            try_get(ytcfg, lambda x: x['INNERTUBE_CLIENT_VERSION'], compat_str)
+            or self._ytcfg_get_safe(
+                ytcfg, lambda x: x['INNERTUBE_CONTEXT']['client']['clientVersion'], compat_str, default_client))
 
-    def _extract_api_key(self, ytcfg=None, default_client='WEB'):
+    def _extract_api_key(self, ytcfg=None, default_client='web'):
         return self._ytcfg_get_safe(ytcfg, lambda x: x['INNERTUBE_API_KEY'], compat_str, default_client)
 
-    def _extract_context(self, ytcfg=None, default_client='WEB'):
+    def _extract_context(self, ytcfg=None, default_client='web'):
         _get_context = lambda y: try_get(y, lambda x: x['INNERTUBE_CONTEXT'], dict)
         context = _get_context(ytcfg)
         if context:
@@ -612,7 +518,7 @@ def _generate_sapisidhash_header(self, origin='https://www.youtube.com'):
 
     def _call_api(self, ep, query, video_id, fatal=True, headers=None,
                   note='Downloading API JSON', errnote='Unable to download API page',
-                  context=None, api_key=None, api_hostname=None, default_client='WEB'):
+                  context=None, api_key=None, api_hostname=None, default_client='web'):
 
         data = {'context': context} if context else {'context': self._extract_context(default_client=default_client)}
         data.update(query)
@@ -674,7 +580,7 @@ def extract_ytcfg(self, video_id, webpage):
 
     def generate_api_headers(
             self, ytcfg=None, identity_token=None, account_syncid=None,
-            visitor_data=None, api_hostname=None, default_client='WEB', session_index=None):
+            visitor_data=None, api_hostname=None, default_client='web', session_index=None):
         origin = 'https://' + (api_hostname if api_hostname else self._get_innertube_host(default_client))
         headers = {
             'X-YouTube-Client-Name': compat_str(
@@ -819,7 +725,7 @@ def _get_text(data, *path_list, max_runs=None):
 
     def _extract_response(self, item_id, query, note='Downloading API JSON', headers=None,
                           ytcfg=None, check_get_keys=None, ep='browse', fatal=True, api_hostname=None,
-                          default_client='WEB'):
+                          default_client='web'):
         response = None
         last_error = None
         count = -1
@@ -2452,20 +2358,22 @@ def _extract_player_response(self, client, video_id, master_ytcfg, player_ytcfg,
         sts = self._extract_signature_timestamp(video_id, player_url, master_ytcfg, fatal=False)
         headers = self.generate_api_headers(
             player_ytcfg, identity_token, syncid,
-            default_client=self._YT_CLIENTS[client], session_index=session_index)
+            default_client=client, session_index=session_index)
 
         yt_query = {'videoId': video_id}
         yt_query.update(self._generate_player_context(sts))
         return self._extract_response(
             item_id=video_id, ep='player', query=yt_query,
             ytcfg=player_ytcfg, headers=headers, fatal=False,
-            default_client=self._YT_CLIENTS[client],
+            default_client=client,
             note='Downloading %s player API JSON' % client.replace('_', ' ').strip()
         ) or None
 
     def _get_requested_clients(self, url, smuggled_data):
         requested_clients = []
-        allowed_clients = [client for client in self._YT_CLIENTS.keys() if client[:1] != '_']
+        allowed_clients = sorted(
+            [client for client in INNERTUBE_CLIENTS.keys() if client[:1] != '_'],
+            key=lambda client: INNERTUBE_CLIENTS[client]['priority'], reverse=True)
         for client in self._configuration_arg('player_client'):
             if client in allowed_clients:
                 requested_clients.append(client)
@@ -2516,7 +2424,7 @@ def _extract_player_responses(self, clients, video_id, webpage, master_ytcfg, pl
 
             if self._is_agegated(pr):
                 client = f'{client}_agegate'
-                if client in self._YT_CLIENTS and client not in original_clients:
+                if client in INNERTUBE_CLIENTS and client not in original_clients:
                     clients.append(client)
 
         # Android player_response does not have microFormats which are needed for