From 57cf9b7f0689ed7b643ce863427c3211e407e5bf Mon Sep 17 00:00:00 2001 From: Peter Rowlands Date: Thu, 5 May 2016 03:11:04 +0900 Subject: [PATCH 01/11] [afreecatv] Add new extractor for afreecatv.com VODs --- youtube_dl/extractor/afreecatv.py | 84 ++++++++++++++++++++++++++++++ youtube_dl/extractor/extractors.py | 1 + 2 files changed, 85 insertions(+) create mode 100644 youtube_dl/extractor/afreecatv.py diff --git a/youtube_dl/extractor/afreecatv.py b/youtube_dl/extractor/afreecatv.py new file mode 100644 index 000000000..d57546e49 --- /dev/null +++ b/youtube_dl/extractor/afreecatv.py @@ -0,0 +1,84 @@ +# coding: utf-8 +from __future__ import unicode_literals + +from .common import InfoExtractor +from ..compat import ( + compat_urllib_parse_urlparse, + compat_urlparse, +) +from ..utils import ( + ExtractorError, + int_or_none, +) + + +class AfreecaTVIE(InfoExtractor): + IE_DESC = 'afreecatv.com' + _VALID_URL = r'''(?x)^ + https?://(?:(live|afbbs|www)\.)?afreeca(?:tv)?\.com(?::\d+)? + (?: + /app/(?:index|read_ucc_bbs)\.cgi| + /player/[Pp]layer\.(?:swf|html)) + \?.*?\bnTitleNo=(?P\d+)''' + _TEST = { + 'url': 'http://live.afreecatv.com:8079/app/index.cgi?szType=read_ucc_bbs&szBjId=dailyapril&nStationNo=16711924&nBbsNo=18605867&nTitleNo=36164052&szSkin=', + 'md5': 'f72c89fe7ecc14c1b5ce506c4996046e', + 'info_dict': { + 'id': '36164052', + 'ext': 'mp4', + 'title': '데일리 에이프릴 요정들의 시상식!', + 'thumbnail': 're:^https?://videoimg.afreecatv.com/.*$', + 'uploader': 'dailyapril', + 'uploader_id': 'dailyapril', + } + } + + def _real_extract(self, url): + video_id = self._match_id(url) + parsed_url = compat_urllib_parse_urlparse(url) + info_url = compat_urlparse.urlunparse(parsed_url._replace( + netloc='afbbs.afreecatv.com:8080', + path='/api/video/get_video_info.php')) + video_xml = self._download_xml(info_url, video_id) + + track = video_xml.find('track') + if track.find('flag').text != 'SUCCEED': + raise ExtractorError('Specified AfreecaTV video does not exist', + expected=True) + title = track.find('title').text + uploader = track.find('nickname').text + uploader_id = track.find('bj_id').text + duration = int_or_none(track.find('duration').text) + thumbnail = track.find('titleImage').text + + entries = [] + for video in track.findall('video'): + for video_file in video.findall('file'): + entries.append({ + 'id': video_file.get('key'), + 'title': title, + 'duration': int_or_none(video_file.get('duration')), + 'formats': [{'url': video_file.text}] + }) + + info = { + 'id': video_id, + 'title': title, + 'uploader': uploader, + 'uploader_id': uploader_id, + 'duration': duration, + 'thumbnail': thumbnail, + } + + if len(entries) > 1: + info['_type'] = 'multi_video' + info['entries'] = entries + elif len(entries) == 1: + info['formats'] = entries[0]['formats'] + else: + raise ExtractorError( + 'No files found for the specified AfreecaTV video, either' + ' the URL is incorrect or the video has been made private.', + expected=True) + + return info diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py index ef4431364..f85d75933 100644 --- a/youtube_dl/extractor/extractors.py +++ b/youtube_dl/extractor/extractors.py @@ -16,6 +16,7 @@ AdobeTVVideoIE, ) from .adultswim import AdultSwimIE +from .afreecatv import AfreecaTVIE from .aenetworks import AENetworksIE from .aftonbladet import AftonbladetIE from .airmozilla import AirMozillaIE From 833b644fffae4d4cb0807591006e34ef963d9bc0 Mon Sep 17 00:00:00 2001 From: Peter Rowlands Date: Fri, 6 May 2016 01:24:02 +0900 Subject: [PATCH 02/11] use xpath_text --- youtube_dl/extractor/afreecatv.py | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/youtube_dl/extractor/afreecatv.py b/youtube_dl/extractor/afreecatv.py index d57546e49..9f9399edc 100644 --- a/youtube_dl/extractor/afreecatv.py +++ b/youtube_dl/extractor/afreecatv.py @@ -9,6 +9,7 @@ from ..utils import ( ExtractorError, int_or_none, + xpath_text, ) @@ -41,25 +42,24 @@ def _real_extract(self, url): path='/api/video/get_video_info.php')) video_xml = self._download_xml(info_url, video_id) - track = video_xml.find('track') - if track.find('flag').text != 'SUCCEED': + if xpath_text(video_xml, './track/flag', default='FAIL') != 'SUCCEED': raise ExtractorError('Specified AfreecaTV video does not exist', expected=True) - title = track.find('title').text - uploader = track.find('nickname').text - uploader_id = track.find('bj_id').text - duration = int_or_none(track.find('duration').text) - thumbnail = track.find('titleImage').text + title = xpath_text(video_xml, './track/title', 'title') + uploader = xpath_text(video_xml, './track/nickname', 'uploader') + uploader_id = xpath_text(video_xml, './track/bj_id', 'uploader id') + duration = int_or_none(xpath_text(video_xml, './track/duration', + 'duration')) + thumbnail = xpath_text(video_xml, './track/titleImage', 'thumbnail') entries = [] - for video in track.findall('video'): - for video_file in video.findall('file'): - entries.append({ - 'id': video_file.get('key'), - 'title': title, - 'duration': int_or_none(video_file.get('duration')), - 'formats': [{'url': video_file.text}] - }) + for video_file in video_xml.findall('./track/video/file'): + entries.append({ + 'id': video_file.get('key'), + 'title': title, + 'duration': int_or_none(video_file.get('duration')), + 'formats': [{'url': video_file.text}] + }) info = { 'id': video_id, From 22e35adefdcea7dccade852ad7dff0953060381d Mon Sep 17 00:00:00 2001 From: Peter Rowlands Date: Fri, 6 May 2016 10:41:30 +0900 Subject: [PATCH 03/11] use url instead of single formats entry --- youtube_dl/extractor/afreecatv.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/youtube_dl/extractor/afreecatv.py b/youtube_dl/extractor/afreecatv.py index 9f9399edc..c9a4b7311 100644 --- a/youtube_dl/extractor/afreecatv.py +++ b/youtube_dl/extractor/afreecatv.py @@ -58,7 +58,7 @@ def _real_extract(self, url): 'id': video_file.get('key'), 'title': title, 'duration': int_or_none(video_file.get('duration')), - 'formats': [{'url': video_file.text}] + 'url': video_file.text, }) info = { @@ -74,7 +74,7 @@ def _real_extract(self, url): info['_type'] = 'multi_video' info['entries'] = entries elif len(entries) == 1: - info['formats'] = entries[0]['formats'] + info['url'] = entries[0]['url'] else: raise ExtractorError( 'No files found for the specified AfreecaTV video, either' From 1dbfd7875497398e8c92c67b8307bafef20e8113 Mon Sep 17 00:00:00 2001 From: Peter Rowlands Date: Fri, 6 May 2016 12:07:29 +0900 Subject: [PATCH 04/11] fix multi_video part naming, add upload_date field --- youtube_dl/extractor/afreecatv.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/youtube_dl/extractor/afreecatv.py b/youtube_dl/extractor/afreecatv.py index c9a4b7311..e927e4a48 100644 --- a/youtube_dl/extractor/afreecatv.py +++ b/youtube_dl/extractor/afreecatv.py @@ -1,6 +1,8 @@ # coding: utf-8 from __future__ import unicode_literals +import re + from .common import InfoExtractor from ..compat import ( compat_urllib_parse_urlparse, @@ -34,6 +36,15 @@ class AfreecaTVIE(InfoExtractor): } } + @staticmethod + def parse_video_key(key): + video_key = {'upload_date': None, 'part': '0'} + m = re.match(r'^(?P\d{8})_\w+_(?P\d+)$', key) + if m: + video_key['upload_date'] = m.group('upload_date') + video_key['part'] = m.group('part') + return video_key + def _real_extract(self, url): video_id = self._match_id(url) parsed_url = compat_urllib_parse_urlparse(url) @@ -54,9 +65,11 @@ def _real_extract(self, url): entries = [] for video_file in video_xml.findall('./track/video/file'): + video_key = self.parse_video_key(video_file.get('key')) entries.append({ - 'id': video_file.get('key'), + 'id': '%s_%s' % (video_id, video_key['part']), 'title': title, + 'upload_date': video_key['upload_date'], 'duration': int_or_none(video_file.get('duration')), 'url': video_file.text, }) @@ -75,6 +88,7 @@ def _real_extract(self, url): info['entries'] = entries elif len(entries) == 1: info['url'] = entries[0]['url'] + info['upload_date'] = entries[0]['upload_date'] else: raise ExtractorError( 'No files found for the specified AfreecaTV video, either' From 8d93c214664e5442320a20e899123b8bfd51cd08 Mon Sep 17 00:00:00 2001 From: Peter Rowlands Date: Fri, 6 May 2016 12:08:43 +0900 Subject: [PATCH 05/11] add multi_video test case --- youtube_dl/extractor/afreecatv.py | 32 +++++++++++++++++++++++++++++-- 1 file changed, 30 insertions(+), 2 deletions(-) diff --git a/youtube_dl/extractor/afreecatv.py b/youtube_dl/extractor/afreecatv.py index e927e4a48..aa5847677 100644 --- a/youtube_dl/extractor/afreecatv.py +++ b/youtube_dl/extractor/afreecatv.py @@ -23,7 +23,7 @@ class AfreecaTVIE(InfoExtractor): /app/(?:index|read_ucc_bbs)\.cgi| /player/[Pp]layer\.(?:swf|html)) \?.*?\bnTitleNo=(?P\d+)''' - _TEST = { + _TESTS = [{ 'url': 'http://live.afreecatv.com:8079/app/index.cgi?szType=read_ucc_bbs&szBjId=dailyapril&nStationNo=16711924&nBbsNo=18605867&nTitleNo=36164052&szSkin=', 'md5': 'f72c89fe7ecc14c1b5ce506c4996046e', 'info_dict': { @@ -33,8 +33,36 @@ class AfreecaTVIE(InfoExtractor): 'thumbnail': 're:^https?://videoimg.afreecatv.com/.*$', 'uploader': 'dailyapril', 'uploader_id': 'dailyapril', + 'upload_date': '20160503', } - } + }, { + 'url': 'http://afbbs.afreecatv.com:8080/app/read_ucc_bbs.cgi?nStationNo=16711924&nTitleNo=36153164&szBjId=dailyapril&nBbsNo=18605867', + 'info_dict': { + 'id': '36153164', + 'title': "BJ유트루와 함께하는 '팅커벨 메이크업!'", + 'thumbnail': 're:^https?://videoimg.afreecatv.com/.*$', + 'uploader': 'dailyapril', + 'uploader_id': 'dailyapril', + }, + 'playlist_count': 2, + 'playlist': [{ + 'md5': 'd8b7c174568da61d774ef0203159bf97', + 'info_dict': { + 'id': '36153164_1', + 'ext': 'mp4', + 'title': "BJ유트루와 함께하는 '팅커벨 메이크업!'", + 'upload_date': '20160502', + }, + }, { + 'md5': '58f2ce7f6044e34439ab2d50612ab02b', + 'info_dict': { + 'id': '36153164_2', + 'ext': 'mp4', + 'title': "BJ유트루와 함께하는 '팅커벨 메이크업!'", + 'upload_date': '20160502', + }, + }], + }] @staticmethod def parse_video_key(key): From 0fdbe3146c2b3825cc26aca7e918df041b0f9adf Mon Sep 17 00:00:00 2001 From: Peter Rowlands Date: Sun, 8 May 2016 08:56:22 +0900 Subject: [PATCH 06/11] use dict.get in case upload_date does not exist --- youtube_dl/extractor/afreecatv.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/youtube_dl/extractor/afreecatv.py b/youtube_dl/extractor/afreecatv.py index aa5847677..4ebc61bae 100644 --- a/youtube_dl/extractor/afreecatv.py +++ b/youtube_dl/extractor/afreecatv.py @@ -66,7 +66,7 @@ class AfreecaTVIE(InfoExtractor): @staticmethod def parse_video_key(key): - video_key = {'upload_date': None, 'part': '0'} + video_key = {} m = re.match(r'^(?P\d{8})_\w+_(?P\d+)$', key) if m: video_key['upload_date'] = m.group('upload_date') @@ -92,12 +92,12 @@ def _real_extract(self, url): thumbnail = xpath_text(video_xml, './track/titleImage', 'thumbnail') entries = [] - for video_file in video_xml.findall('./track/video/file'): + for i, video_file in enumerate(video_xml.findall('./track/video/file')): video_key = self.parse_video_key(video_file.get('key')) entries.append({ - 'id': '%s_%s' % (video_id, video_key['part']), + 'id': '%s_%s' % (video_id, video_key.get('part', i + 1)), 'title': title, - 'upload_date': video_key['upload_date'], + 'upload_date': video_key.get('upload_date'), 'duration': int_or_none(video_file.get('duration')), 'url': video_file.text, }) From 81f35fee2fd2b58d909887aaa7667310a4d65759 Mon Sep 17 00:00:00 2001 From: Peter Rowlands Date: Sun, 8 May 2016 08:56:44 +0900 Subject: [PATCH 07/11] fix extractors.py import order --- youtube_dl/extractor/extractors.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py index f85d75933..1f95530a5 100644 --- a/youtube_dl/extractor/extractors.py +++ b/youtube_dl/extractor/extractors.py @@ -16,8 +16,8 @@ AdobeTVVideoIE, ) from .adultswim import AdultSwimIE -from .afreecatv import AfreecaTVIE from .aenetworks import AENetworksIE +from .afreecatv import AfreecaTVIE from .aftonbladet import AftonbladetIE from .airmozilla import AirMozillaIE from .aljazeera import AlJazeeraIE From 3452c3a27c2bfd278746314cda4247c2226a35f3 Mon Sep 17 00:00:00 2001 From: Peter Rowlands Date: Sun, 8 May 2016 10:02:19 +0900 Subject: [PATCH 08/11] update tests --- youtube_dl/extractor/afreecatv.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/youtube_dl/extractor/afreecatv.py b/youtube_dl/extractor/afreecatv.py index 4ebc61bae..b90095881 100644 --- a/youtube_dl/extractor/afreecatv.py +++ b/youtube_dl/extractor/afreecatv.py @@ -30,7 +30,7 @@ class AfreecaTVIE(InfoExtractor): 'id': '36164052', 'ext': 'mp4', 'title': '데일리 에이프릴 요정들의 시상식!', - 'thumbnail': 're:^https?://videoimg.afreecatv.com/.*$', + 'thumbnail': 're:^https?://(?:video|st)img.afreecatv.com/.*$', 'uploader': 'dailyapril', 'uploader_id': 'dailyapril', 'upload_date': '20160503', @@ -40,7 +40,7 @@ class AfreecaTVIE(InfoExtractor): 'info_dict': { 'id': '36153164', 'title': "BJ유트루와 함께하는 '팅커벨 메이크업!'", - 'thumbnail': 're:^https?://videoimg.afreecatv.com/.*$', + 'thumbnail': 're:^https?://(?:video|st)img.afreecatv.com/.*$', 'uploader': 'dailyapril', 'uploader_id': 'dailyapril', }, @@ -62,6 +62,9 @@ class AfreecaTVIE(InfoExtractor): 'upload_date': '20160502', }, }], + }, { + 'url': 'http://www.afreecatv.com/player/Player.swf?szType=szBjId=djleegoon&nStationNo=11273158&nBbsNo=13161095&nTitleNo=36327652', + 'only_matching': True, }] @staticmethod From 370d4eb8ad3d9d092fc5eb116509eaf4a3e83177 Mon Sep 17 00:00:00 2001 From: Peter Rowlands Date: Sun, 8 May 2016 10:02:48 +0900 Subject: [PATCH 09/11] use stricter file selector in case of empty in case of empty ./track/video/file entries --- youtube_dl/extractor/afreecatv.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/youtube_dl/extractor/afreecatv.py b/youtube_dl/extractor/afreecatv.py index b90095881..527386be3 100644 --- a/youtube_dl/extractor/afreecatv.py +++ b/youtube_dl/extractor/afreecatv.py @@ -95,7 +95,7 @@ def _real_extract(self, url): thumbnail = xpath_text(video_xml, './track/titleImage', 'thumbnail') entries = [] - for i, video_file in enumerate(video_xml.findall('./track/video/file')): + for i, video_file in enumerate(video_xml.findall('./track/video/file[@key]')): video_key = self.parse_video_key(video_file.get('key')) entries.append({ 'id': '%s_%s' % (video_id, video_key.get('part', i + 1)), @@ -119,7 +119,7 @@ def _real_extract(self, url): info['entries'] = entries elif len(entries) == 1: info['url'] = entries[0]['url'] - info['upload_date'] = entries[0]['upload_date'] + info['upload_date'] = entries[0].get('upload_date') else: raise ExtractorError( 'No files found for the specified AfreecaTV video, either' From 93fdb1417766015ddadcd13a709cdfae4de5e246 Mon Sep 17 00:00:00 2001 From: Peter Rowlands Date: Sun, 8 May 2016 10:33:17 +0900 Subject: [PATCH 10/11] don't use selection by attribute --- youtube_dl/extractor/afreecatv.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/youtube_dl/extractor/afreecatv.py b/youtube_dl/extractor/afreecatv.py index 527386be3..0fcbea0d1 100644 --- a/youtube_dl/extractor/afreecatv.py +++ b/youtube_dl/extractor/afreecatv.py @@ -95,8 +95,10 @@ def _real_extract(self, url): thumbnail = xpath_text(video_xml, './track/titleImage', 'thumbnail') entries = [] - for i, video_file in enumerate(video_xml.findall('./track/video/file[@key]')): - video_key = self.parse_video_key(video_file.get('key')) + for i, video_file in enumerate(video_xml.findall('./track/video/file')): + video_key = self.parse_video_key(video_file.get('key', '')) + if not video_key: + continue entries.append({ 'id': '%s_%s' % (video_id, video_key.get('part', i + 1)), 'title': title, From e7d85c4ef7d2c74058d41ded1e2a6d6aa527dc9a Mon Sep 17 00:00:00 2001 From: Peter Rowlands Date: Tue, 31 May 2016 17:28:49 +0900 Subject: [PATCH 11/11] use /track/video/file to determine if video exists --- youtube_dl/extractor/afreecatv.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/youtube_dl/extractor/afreecatv.py b/youtube_dl/extractor/afreecatv.py index 0fcbea0d1..518c61f67 100644 --- a/youtube_dl/extractor/afreecatv.py +++ b/youtube_dl/extractor/afreecatv.py @@ -11,6 +11,7 @@ from ..utils import ( ExtractorError, int_or_none, + xpath_element, xpath_text, ) @@ -84,9 +85,10 @@ def _real_extract(self, url): path='/api/video/get_video_info.php')) video_xml = self._download_xml(info_url, video_id) - if xpath_text(video_xml, './track/flag', default='FAIL') != 'SUCCEED': + if xpath_element(video_xml, './track/video/file') is None: raise ExtractorError('Specified AfreecaTV video does not exist', expected=True) + title = xpath_text(video_xml, './track/title', 'title') uploader = xpath_text(video_xml, './track/nickname', 'uploader') uploader_id = xpath_text(video_xml, './track/bj_id', 'uploader id')