From c6b657867ad68af6b930ed0aa11ec5d93ee187b7 Mon Sep 17 00:00:00 2001 From: nixxo Date: Sun, 12 Feb 2023 15:43:20 +0100 Subject: [PATCH] [extractor/rcs] Fix extractors (#5700) Authored by: nixxo, pukkandan Closes #5683 --- yt_dlp/extractor/rcs.py | 408 ++++++++++++++++++---------------------- 1 file changed, 182 insertions(+), 226 deletions(-) diff --git a/yt_dlp/extractor/rcs.py b/yt_dlp/extractor/rcs.py index b905f8d2e..2440858ca 100644 --- a/yt_dlp/extractor/rcs.py +++ b/yt_dlp/extractor/rcs.py @@ -3,9 +3,18 @@ import re from .common import InfoExtractor from ..utils import ( ExtractorError, + HEADRequest, base_url, clean_html, + extract_attributes, + get_element_html_by_class, + get_element_html_by_id, + int_or_none, js_to_json, + mimetype2ext, + sanitize_url, + traverse_obj, + try_call, url_basename, urljoin, ) @@ -15,41 +24,8 @@ class RCSBaseIE(InfoExtractor): # based on VideoPlayerLoader.prototype.getVideoSrc # and VideoPlayerLoader.prototype.transformSrc from # https://js2.corriereobjects.it/includes2013/LIBS/js/corriere_video.sjs - _ALL_REPLACE = { - 'media2vam.corriere.it.edgesuite.net': - 'media2vam-corriere-it.akamaized.net', - 'media.youreporter.it.edgesuite.net': - 'media-youreporter-it.akamaized.net', - 'corrierepmd.corriere.it.edgesuite.net': - 'corrierepmd-corriere-it.akamaized.net', - 'media2vam-corriere-it.akamaized.net/fcs.quotidiani/vr/videos/': - 'video.corriere.it/vr360/videos/', - '.net//': '.net/', - } - _MP4_REPLACE = { - 'media2vam.corbologna.corriere.it.edgesuite.net': - 'media2vam-bologna-corriere-it.akamaized.net', - 'media2vam.corfiorentino.corriere.it.edgesuite.net': - 'media2vam-fiorentino-corriere-it.akamaized.net', - 'media2vam.cormezzogiorno.corriere.it.edgesuite.net': - 'media2vam-mezzogiorno-corriere-it.akamaized.net', - 'media2vam.corveneto.corriere.it.edgesuite.net': - 'media2vam-veneto-corriere-it.akamaized.net', - 'media2.oggi.it.edgesuite.net': - 'media2-oggi-it.akamaized.net', - 'media2.quimamme.it.edgesuite.net': - 'media2-quimamme-it.akamaized.net', - 'media2.amica.it.edgesuite.net': - 'media2-amica-it.akamaized.net', - 'media2.living.corriere.it.edgesuite.net': - 'media2-living-corriere-it.akamaized.net', - 'media2.style.corriere.it.edgesuite.net': - 'media2-style-corriere-it.akamaized.net', - 'media2.iodonna.it.edgesuite.net': - 'media2-iodonna-it.akamaized.net', - 'media2.leitv.it.edgesuite.net': - 'media2-leitv-it.akamaized.net', - } + _UUID_RE = r'[\da-f]{8}-[\da-f]{4}-[\da-f]{4}-[\da-f]{4}-[\da-f]{12}' + _RCS_ID_RE = r'[\w-]+-\d{10}' _MIGRATION_MAP = { 'videoamica-vh.akamaihd': 'amica', 'media2-amica-it.akamaized': 'amica', @@ -90,183 +66,140 @@ class RCSBaseIE(InfoExtractor): 'vivimilano-vh.akamaihd': 'vivimilano', 'media2-youreporter-it.akamaized': 'youreporter' } - _MIGRATION_MEDIA = { - 'advrcs-vh.akamaihd': '', - 'corriere-f.akamaihd': '', - 'corrierepmd-corriere-it.akamaized': '', - 'corrprotetto-vh.akamaihd': '', - 'gazzetta-f.akamaihd': '', - 'gazzettapmd-gazzetta-it.akamaized': '', - 'gazzprotetto-vh.akamaihd': '', - 'periodici-f.akamaihd': '', - 'periodicisecure-vh.akamaihd': '', - 'videocoracademy-vh.akamaihd': '' - } def _get_video_src(self, video): - mediaFiles = video.get('mediaProfile').get('mediaFile') - src = {} - # audio - if video.get('mediaType') == 'AUDIO': - for aud in mediaFiles: - # todo: check - src['mp3'] = aud.get('value') - # video - else: - for vid in mediaFiles: - if vid.get('mimeType') == 'application/vnd.apple.mpegurl': - src['m3u8'] = vid.get('value') - if vid.get('mimeType') == 'video/mp4': - src['mp4'] = vid.get('value') + for source in traverse_obj(video, ( + 'mediaProfile', 'mediaFile', lambda _, v: v.get('mimeType'))): + url = source['value'] + for s, r in ( + ('media2vam.corriere.it.edgesuite.net', 'media2vam-corriere-it.akamaized.net'), + ('media.youreporter.it.edgesuite.net', 'media-youreporter-it.akamaized.net'), + ('corrierepmd.corriere.it.edgesuite.net', 'corrierepmd-corriere-it.akamaized.net'), + ('media2vam-corriere-it.akamaized.net/fcs.quotidiani/vr/videos/', 'video.corriere.it/vr360/videos/'), + ('http://', 'https://'), + ): + url = url.replace(s, r) - # replace host - for t in src: - for s, r in self._ALL_REPLACE.items(): - src[t] = src[t].replace(s, r) - for s, r in self._MP4_REPLACE.items(): - src[t] = src[t].replace(s, r) + type_ = mimetype2ext(source['mimeType']) + if type_ == 'm3u8' and '-vh.akamaihd' in url: + # still needed for some old content: see _TESTS #3 + matches = re.search(r'(?:https?:)?//(?P[\w\.\-]+)\.net/i(?P.+)$', url) + if matches: + url = f'https://vod.rcsobjects.it/hls/{self._MIGRATION_MAP[matches.group("host")]}{matches.group("path")}' + if traverse_obj(video, ('mediaProfile', 'geoblocking')) or ( + type_ == 'm3u8' and 'fcs.quotidiani_!' in url): + url = url.replace('vod.rcsobjects', 'vod-it.rcsobjects') + if type_ == 'm3u8' and 'vod' in url: + url = url.replace('.csmil', '.urlset') + if type_ == 'mp3': + url = url.replace('media2vam-corriere-it.akamaized.net', 'vod.rcsobjects.it/corriere') - # switch cdn - if 'mp4' in src and 'm3u8' in src: - if ('-lh.akamaihd' not in src.get('m3u8') - and 'akamai' in src.get('mp4')): - if 'm3u8' in src: - matches = re.search(r'(?:https*:)?\/\/(?P.*)\.net\/i(?P.*)$', src.get('m3u8')) - src['m3u8'] = 'https://vod.rcsobjects.it/hls/%s%s' % ( - self._MIGRATION_MAP[matches.group('host')], - matches.group('path').replace( - '///', '/').replace( - '//', '/').replace( - '.csmil', '.urlset' - ) - ) - if 'mp4' in src: - matches = re.search(r'(?:https*:)?\/\/(?P.*)\.net\/i(?P.*)$', src.get('mp4')) - if matches: - if matches.group('host') in self._MIGRATION_MEDIA: - vh_stream = 'https://media2.corriereobjects.it' - if src.get('mp4').find('fcs.quotidiani_!'): - vh_stream = 'https://media2-it.corriereobjects.it' - src['mp4'] = '%s%s' % ( - vh_stream, - matches.group('path').replace( - '///', '/').replace( - '//', '/').replace( - '/fcs.quotidiani/mediacenter', '').replace( - '/fcs.quotidiani_!/mediacenter', '').replace( - 'corriere/content/mediacenter/', '').replace( - 'gazzetta/content/mediacenter/', '') - ) - else: - src['mp4'] = 'https://vod.rcsobjects.it/%s%s' % ( - self._MIGRATION_MAP[matches.group('host')], - matches.group('path').replace('///', '/').replace('//', '/') - ) + yield { + 'type': type_, + 'url': url, + 'bitrate': source.get('bitrate') + } - if 'mp3' in src: - src['mp3'] = src.get('mp3').replace( - 'media2vam-corriere-it.akamaized.net', - 'vod.rcsobjects.it/corriere') - if 'mp4' in src: - if src.get('mp4').find('fcs.quotidiani_!'): - src['mp4'] = src.get('mp4').replace('vod.rcsobjects', 'vod-it.rcsobjects') - if 'm3u8' in src: - if src.get('m3u8').find('fcs.quotidiani_!'): - src['m3u8'] = src.get('m3u8').replace('vod.rcsobjects', 'vod-it.rcsobjects') + def _create_http_formats(self, m3u8_formats, video_id): + for f in m3u8_formats: + if f['vcodec'] == 'none': + continue + http_url = re.sub(r'(https?://[^/]+)/hls/([^?#]+?\.mp4).+', r'\g<1>/\g<2>', f['url']) + if http_url == f['url']: + continue - if 'geoblocking' in video.get('mediaProfile'): - if 'm3u8' in src: - src['m3u8'] = src.get('m3u8').replace('vod.rcsobjects', 'vod-it.rcsobjects') - if 'mp4' in src: - src['mp4'] = src.get('mp4').replace('vod.rcsobjects', 'vod-it.rcsobjects') - if 'm3u8' in src: - if src.get('m3u8').find('csmil') and src.get('m3u8').find('vod'): - src['m3u8'] = src.get('m3u8').replace('.csmil', '.urlset') + http_f = f.copy() + del http_f['manifest_url'] + format_id = try_call(lambda: http_f['format_id'].replace('hls-', 'https-')) + urlh = self._request_webpage(HEADRequest(http_url), video_id, fatal=False, + note=f'Check filesize for {format_id}') + if not urlh: + continue - return src - - def _create_formats(self, urls, video_id): - formats = [] - formats = self._extract_m3u8_formats( - urls.get('m3u8'), video_id, 'mp4', entry_protocol='m3u8_native', - m3u8_id='hls', fatal=False) - - if urls.get('mp4'): - formats.append({ - 'format_id': 'http-mp4', - 'url': urls['mp4'] + http_f.update({ + 'format_id': format_id, + 'url': http_url, + 'protocol': 'https', + 'filesize_approx': int_or_none(urlh.headers.get('Content-Length', None)), }) - return formats + yield http_f + + def _create_formats(self, sources, video_id): + for source in sources: + if source['type'] == 'm3u8': + m3u8_formats = self._extract_m3u8_formats( + source['url'], video_id, 'mp4', m3u8_id='hls', fatal=False) + yield from m3u8_formats + yield from self._create_http_formats(m3u8_formats, video_id) + elif source['type'] == 'mp3': + yield { + 'format_id': 'https-mp3', + 'ext': 'mp3', + 'acodec': 'mp3', + 'vcodec': 'none', + 'abr': source.get('bitrate'), + 'url': source['url'], + } def _real_extract(self, url): - mobj = self._match_valid_url(url) - video_id = mobj.group('id') + cdn, video_id = self._match_valid_url(url).group('cdn', 'id') + display_id, video_data = None, None - if 'cdn' not in mobj.groupdict(): - raise ExtractorError('CDN not found in url: %s' % url) - - # for leitv/youreporter/viaggi don't use the embed page - if ((mobj.group('cdn') not in ['leitv.it', 'youreporter.it']) - and (mobj.group('vid') == 'video')): - url = 'https://video.%s/video-embed/%s' % (mobj.group('cdn'), video_id) - - page = self._download_webpage(url, video_id) - - video_data = None - # look for json video data url - json = self._search_regex( - r'''(?x)url\s*=\s*(["']) - (?P - (?:https?:)?//video\.rcs\.it - /fragment-includes/video-includes/.+?\.json - )\1;''', - page, video_id, group='url', default=None) - if json: - if json.startswith('//'): - json = 'https:%s' % json - video_data = self._download_json(json, video_id) - - # if json url not found, look for json video data directly in the page + if re.match(self._UUID_RE, video_id) or re.match(self._RCS_ID_RE, video_id): + url = f'https://video.{cdn}/video-json/{video_id}' else: - # RCS normal pages and most of the embeds - json = self._search_regex( - r'[\s;]video\s*=\s*({[\s\S]+?})(?:;|,playlist=)', - page, video_id, default=None) - if not json and 'video-embed' in url: - page = self._download_webpage(url.replace('video-embed', 'video-json'), video_id) - json = self._search_regex( - r'##start-video##({[\s\S]+?})##end-video##', - page, video_id, default=None) - if not json: - # if no video data found try search for iframes - emb = RCSEmbedsIE._extract_url(page) + webpage = self._download_webpage(url, video_id) + data_config = get_element_html_by_id('divVideoPlayer', webpage) or get_element_html_by_class('divVideoPlayer', webpage) + + if data_config: + data_config = self._parse_json( + extract_attributes(data_config).get('data-config'), + video_id, fatal=False) or {} + if data_config.get('newspaper'): + cdn = f'{data_config["newspaper"]}.it' + display_id, video_id = video_id, data_config.get('uuid') or video_id + url = f'https://video.{cdn}/video-json/{video_id}' + else: + json_url = self._search_regex( + r'''(?x)url\s*=\s*(["']) + (?P + (?:https?:)?//video\.rcs\.it + /fragment-includes/video-includes/[^"']+?\.json + )\1;''', + webpage, video_id, group='url', default=None) + if json_url: + video_data = self._download_json(sanitize_url(json_url, scheme='https'), video_id) + display_id, video_id = video_id, video_data.get('id') or video_id + + if not video_data: + webpage = self._download_webpage(url, video_id) + + video_data = self._search_json( + '##start-video##', webpage, 'video data', video_id, default=None, + end_pattern='##end-video##', transform_source=js_to_json) + + if not video_data: + # try search for iframes + emb = RCSEmbedsIE._extract_url(webpage) if emb: return { '_type': 'url_transparent', 'url': emb, 'ie_key': RCSEmbedsIE.ie_key() } - if json: - video_data = self._parse_json( - json, video_id, transform_source=js_to_json) if not video_data: raise ExtractorError('Video data not found in the page') - formats = self._create_formats( - self._get_video_src(video_data), video_id) - - description = (video_data.get('description') - or clean_html(video_data.get('htmlDescription')) - or self._html_search_meta('description', page)) - uploader = video_data.get('provider') or mobj.group('cdn') - return { 'id': video_id, + 'display_id': display_id, 'title': video_data.get('title'), - 'description': description, - 'uploader': uploader, - 'formats': formats + 'description': (clean_html(video_data.get('description')) + or clean_html(video_data.get('htmlDescription')) + or self._html_search_meta('description', webpage)), + 'uploader': video_data.get('provider') or cdn, + 'formats': list(self._create_formats(self._get_video_src(video_data), video_id)), } @@ -296,7 +229,7 @@ class RCSEmbedsIE(RCSBaseIE): \1'''] _TESTS = [{ 'url': 'https://video.rcs.it/video-embed/iodonna-0001585037', - 'md5': '623ecc8ffe7299b2d0c1046d8331a9df', + 'md5': '0faca97df525032bb9847f690bc3720c', 'info_dict': { 'id': 'iodonna-0001585037', 'ext': 'mp4', @@ -305,38 +238,31 @@ class RCSEmbedsIE(RCSBaseIE): 'uploader': 'rcs.it', } }, { - # redownload the page changing 'video-embed' in 'video-json' 'url': 'https://video.gazzanet.gazzetta.it/video-embed/gazzanet-mo05-0000260789', - 'md5': 'a043e3fecbe4d9ed7fc5d888652a5440', - 'info_dict': { - 'id': 'gazzanet-mo05-0000260789', - 'ext': 'mp4', - 'title': 'Valentino Rossi e papà Graziano si divertono col drifting', - 'description': 'md5:a8bf90d6adafd9815f70fc74c0fc370a', - 'uploader': 'rcd', - } - }, { - 'url': 'https://video.corriere.it/video-embed/b727632a-f9d0-11ea-91b0-38d50a849abb?player', 'match_only': True }, { 'url': 'https://video.gazzetta.it/video-embed/49612410-00ca-11eb-bcd8-30d4253e0140', 'match_only': True }] + _WEBPAGE_TESTS = [{ + 'url': 'https://www.iodonna.it/video-iodonna/personaggi-video/monica-bellucci-piu-del-lavoro-oggi-per-me-sono-importanti-lamicizia-e-la-famiglia/', + 'info_dict': { + 'id': 'iodonna-0002033648', + 'ext': 'mp4', + 'title': 'Monica Bellucci: «Più del lavoro, oggi per me sono importanti l\'amicizia e la famiglia»', + 'description': 'md5:daea6d9837351e56b1ab615c06bebac1', + 'uploader': 'rcs.it', + } + }] @staticmethod - def _sanitize_urls(urls): - # add protocol if missing - for i, e in enumerate(urls): - if e.startswith('//'): - urls[i] = 'https:%s' % e - # clean iframes urls - for i, e in enumerate(urls): - urls[i] = urljoin(base_url(e), url_basename(e)) - return urls + def _sanitize_url(url): + url = sanitize_url(url, scheme='https') + return urljoin(base_url(url), url_basename(url)) @classmethod def _extract_embed_urls(cls, url, webpage): - return cls._sanitize_urls(list(super()._extract_embed_urls(url, webpage))) + return map(cls._sanitize_url, super()._extract_embed_urls(url, webpage)) class RCSIE(RCSBaseIE): @@ -349,37 +275,53 @@ class RCSIE(RCSBaseIE): |corrierefiorentino\. )?corriere\.it |(?:gazzanet\.)?gazzetta\.it) - /(?!video-embed/).+?/(?P[^/\?]+)(?=\?|/$|$)''' + /(?!video-embed/)[^?#]+?/(?P[^/\?]+)(?=\?|/$|$)''' _TESTS = [{ + # json iframe directly from id 'url': 'https://video.corriere.it/sport/formula-1/vettel-guida-ferrari-sf90-mugello-suo-fianco-c-elecrerc-bendato-video-esilarante/b727632a-f9d0-11ea-91b0-38d50a849abb', - 'md5': '0f4ededc202b0f00b6e509d831e2dcda', + 'md5': '14946840dec46ecfddf66ba4eea7d2b2', 'info_dict': { 'id': 'b727632a-f9d0-11ea-91b0-38d50a849abb', 'ext': 'mp4', 'title': 'Vettel guida la Ferrari SF90 al Mugello e al suo fianco c\'è Leclerc (bendato): il video è esilarante', - 'description': 'md5:93b51c9161ac8a64fb2f997b054d0152', + 'description': 'md5:3915ce5ebb3d2571deb69a5eb85ac9b5', 'uploader': 'Corriere Tv', } }, { - # video data inside iframe + # search for video id inside the page 'url': 'https://viaggi.corriere.it/video/norvegia-il-nuovo-ponte-spettacolare-sopra-la-cascata-di-voringsfossen/', - 'md5': 'da378e4918d2afbf7d61c35abb948d4c', + 'md5': 'f22a92d9e666e80f2fffbf2825359c81', 'info_dict': { 'id': '5b7cd134-e2c1-11ea-89b3-b56dd0df2aa2', + 'display_id': 'norvegia-il-nuovo-ponte-spettacolare-sopra-la-cascata-di-voringsfossen', 'ext': 'mp4', 'title': 'La nuova spettacolare attrazione in Norvegia: il ponte sopra Vøringsfossen', 'description': 'md5:18b35a291f6746c0c8dacd16e5f5f4f8', 'uploader': 'DOVE Viaggi', } }, { - 'url': 'https://video.gazzetta.it/video-motogp-catalogna-cadute-dovizioso-vale-rossi/49612410-00ca-11eb-bcd8-30d4253e0140?vclk=Videobar', - 'md5': 'eedc1b5defd18e67383afef51ff7bdf9', + # only audio format https://github.com/yt-dlp/yt-dlp/issues/5683 + 'url': 'https://video.corriere.it/cronaca/audio-telefonata-il-papa-becciu-santita-lettera-che-mi-ha-inviato-condanna/b94c0d20-70c2-11ed-9572-e4b947a0ebd2', + 'md5': 'aaffb08d02f2ce4292a4654694c78150', 'info_dict': { - 'id': '49612410-00ca-11eb-bcd8-30d4253e0140', + 'id': 'b94c0d20-70c2-11ed-9572-e4b947a0ebd2', + 'ext': 'mp3', + 'title': 'L\'audio della telefonata tra il Papa e Becciu: «Santità, la lettera che mi ha inviato è una condanna»', + 'description': 'md5:c0ddb61bd94a8d4e0d4bb9cda50a689b', + 'uploader': 'Corriere Tv', + 'formats': [{'format_id': 'https-mp3', 'ext': 'mp3'}], + } + }, { + # old content still needs cdn migration + 'url': 'https://viaggi.corriere.it/video/milano-varallo-sesia-sul-treno-a-vapore/', + 'md5': '2dfdce7af249654ad27eeba03fe1e08d', + 'info_dict': { + 'id': 'd8f6c8d0-f7d7-11e8-bfca-f74cf4634191', + 'display_id': 'milano-varallo-sesia-sul-treno-a-vapore', 'ext': 'mp4', - 'title': 'Dovizioso, il contatto con Zarco e la caduta. E anche Vale finisce a terra', - 'description': 'md5:8c6e905dc3b9413218beca11ebd69778', - 'uploader': 'AMorici', + 'title': 'Milano-Varallo Sesia sul treno a vapore', + 'description': 'md5:6348f47aac230397fe341a74f7678d53', + 'uploader': 'DOVE Viaggi', } }, { 'url': 'https://video.corriere.it/video-360/metro-copenaghen-tutta-italiana/a248a7f0-e2db-11e9-9830-af2de6b1f945', @@ -391,13 +333,15 @@ class RCSVariousIE(RCSBaseIE): _VALID_URL = r'''(?x)https?://www\. (?P leitv\.it| - youreporter\.it + youreporter\.it| + amica\.it )/(?:[^/]+/)?(?P[^/]+?)(?:$|\?|/)''' _TESTS = [{ - 'url': 'https://www.leitv.it/benessere/mal-di-testa-come-combatterlo-ed-evitarne-la-comparsa/', - 'md5': '92b4e63667b8f95acb0a04da25ae28a1', + 'url': 'https://www.leitv.it/benessere/mal-di-testa/', + 'md5': '3b7a683d105a7313ec7513b014443631', 'info_dict': { - 'id': 'mal-di-testa-come-combatterlo-ed-evitarne-la-comparsa', + 'id': 'leitv-0000125151', + 'display_id': 'mal-di-testa', 'ext': 'mp4', 'title': 'Cervicalgia e mal di testa, il video con i suggerimenti dell\'esperto', 'description': 'md5:ae21418f34cee0b8d02a487f55bcabb5', @@ -405,12 +349,24 @@ class RCSVariousIE(RCSBaseIE): } }, { 'url': 'https://www.youreporter.it/fiume-sesia-3-ottobre-2020/', - 'md5': '8dccd436b47a830bab5b4a88232f391a', + 'md5': '3989b6d603482611a2abd2f32b79f739', 'info_dict': { - 'id': 'fiume-sesia-3-ottobre-2020', + 'id': 'youreporter-0000332574', + 'display_id': 'fiume-sesia-3-ottobre-2020', 'ext': 'mp4', 'title': 'Fiume Sesia 3 ottobre 2020', 'description': 'md5:0070eef1cc884d13c970a4125063de55', 'uploader': 'youreporter.it', } + }, { + 'url': 'https://www.amica.it/video-post/saint-omer-al-cinema-il-film-leone-dargento-che-ribalta-gli-stereotipi/', + 'md5': '187cce524dfd0343c95646c047375fc4', + 'info_dict': { + 'id': 'amica-0001225365', + 'display_id': 'saint-omer-al-cinema-il-film-leone-dargento-che-ribalta-gli-stereotipi', + 'ext': 'mp4', + 'title': '"Saint Omer": al cinema il film Leone d\'argento che ribalta gli stereotipi', + 'description': 'md5:b1c8869c2dcfd6073a2a311ba0008aa8', + 'uploader': 'rcs.it', + } }]