diff --git a/README.md b/README.md index 4abbf1d..333e65e 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,7 @@ Currently supported content providers: |----------------------|-----------|----------|---------------------| | VRT MAX | ✔️ | ✔️ | ✔️ | | GoPlay | ✔️ | ✔️ | ✔️ | -| VTM GO | ❌ | ❌ | ❌ | +| VTM GO | ✔️ | ✔️ | ❌ | | Streamz | ❌ | ❌ | ❌ | | YouTube | ✔️ | ✔️ | ✔️ | | Plain manifest url | ✔️ | ✔️ | ✔️ | diff --git a/dl-downer/requirements.txt b/dl-downer/requirements.txt index 98ea713..f0096a7 100644 --- a/dl-downer/requirements.txt +++ b/dl-downer/requirements.txt @@ -3,6 +3,7 @@ requests protobuf>=4.25.1 pywidevine==1.8.0 playwright==1.42.0 +playwright_stealth python-dotenv psycopg2-binary yt-dlp diff --git a/dl-downer/src/downloaders/VTMGO.py b/dl-downer/src/downloaders/VTMGO.py new file mode 100644 index 0000000..0537f1f --- /dev/null +++ b/dl-downer/src/downloaders/VTMGO.py @@ -0,0 +1,177 @@ +import os +import re +import time +import requests + +from loguru import logger + +from ..models.dl_request import DLRequest +from ..models.dl_request_platform import DLRequestPlatform +from ..utils.download_video_nre import download_video_nre +from ..utils.local_cdm import Local_CDM +from ..utils.filename import parse_filename +from ..utils.files import insert_subtitle +from ..utils.browser import create_playwright_page, get_storage_state_location, user_agent + +def handle_vtmgo_consent_popup(page): + ''' + Handle consent popup if it appears + ''' + + try: + logger.debug('Accepting cookies') + page.wait_for_selector('div#pg-first-layer', timeout=2000) + except: + logger.debug(f'No consent popup found:') + return + acceptButton = page.wait_for_selector('button#pg-accept-btn') + acceptButton.click() + logger.debug('Cookies accepted') + +def get_vtmgo_data(video_page_url: str): + browser = None + playwright = None + + config = None + + try: + playwright, browser, page = create_playwright_page(DLRequestPlatform.VTMGO) + + page.goto("https://www.vtmgo.be/vtmgo", wait_until='networkidle') + handle_vtmgo_consent_popup(page) + + try: + page.wait_for_selector('li.nav__item--userdropdown', timeout=2000) + logger.debug('Already logged in') + page.context.storage_state(path=get_storage_state_location(DLRequestPlatform.VTMGO)) + except: + logger.debug('Logging in ...') + page.goto('https://www.vtmgo.be/vtmgo/aanmelden', wait_until='networkidle') + + emailInput = page.wait_for_selector('input#username') + assert os.getenv('AUTH_VTMGO_EMAIL'), 'AUTH_VTMGO_EMAIL not set' + emailInput.type(os.getenv('AUTH_VTMGO_EMAIL')) + submitButton = page.wait_for_selector('form button[type="submit"]') + submitButton.click() + + passwordInput = page.wait_for_selector('input#password') + assert os.getenv('AUTH_VTMGO_PASSWORD'), 'AUTH_VTMGO_PASSWORD not set' + passwordInput.type(os.getenv('AUTH_VTMGO_PASSWORD')) + submitButton = page.wait_for_selector('form button[type="submit"]') + submitButton.click() + + page.wait_for_selector('li.nav__item--userdropdown', timeout=200000000) + logger.debug('Logged in successfully') + page.context.storage_state(path=get_storage_state_location(DLRequestPlatform.VTMGO)) + + config_response = None + def handle_response(response): + nonlocal config_response + if 'https://videoplayer-service.dpgmedia.net/play-config/' in response.url: + config_response = response + page.on('response', handle_response) + page.goto(video_page_url, wait_until='load') + max_wait = 10 + while config_response is None: + time.sleep(2) + if max_wait == 0: + raise Exception('Failed to get config response') + max_wait -= 1 + logger.debug('Got config response') + config = config_response.json() + + finally: + if browser is not None: + browser.close() + if playwright is not None: + playwright.stop() + + return config + +def VTMGO_DL(dl_request: DLRequest): + config = get_vtmgo_data(dl_request.video_page_or_manifest_url) + + # find dash stream + streams = config['video']['streams'] + dash_stream = None + for stream in streams: + if stream['type'] == 'dash': + dash_stream = stream + break + assert dash_stream, 'No dash stream found' + + mpd_url = dash_stream['url'] + license_url = dash_stream['drm']['com.widevine.alpha']['licenseUrl'] + auth_token = dash_stream['drm']['com.widevine.alpha']['drmtoday']['authToken'] + logger.debug(f'MPD: {mpd_url}') + logger.debug(f'License: {license_url}') + logger.debug(f'Auth token: {auth_token}') + + filename = None + if dl_request.output_filename: + filename = dl_request.output_filename + else: + # use metadata to generate filename + metadata = config['video']['metadata'] + prog = metadata['program']['title'] + ep = metadata['episode']['order'] + season = metadata['episode']['season']['order'] + filename = f'{prog}.S{season:02}E{ep:02}' + filename = parse_filename(filename) + logger.debug(f'Filename: {filename}') + + # get pssh from mpd + manifest_response = requests.get(mpd_url) + manifest_response.raise_for_status() + pssh = re.findall(r']*>(.{,180})', manifest_response.text)[0] + logger.debug(f'PSSH: {pssh}') + + cdm = Local_CDM() + challenge = cdm.generate_challenge(pssh) + headers = { + 'user-agent': user_agent, + 'origin': 'https://www.vtmgo.be', + 'connection': 'keep-alive', + 'accept': '*/*', + 'accept-encoding': 'gzip, deflate, br', + 'X-Dt-Auth-Token': auth_token, + } + license_response = requests.post(license_url, data=challenge, headers=headers) + license_response.raise_for_status() + license_response_json = license_response.json() + license = license_response_json['license'] + logger.debug(f'License: {license}') + keys = cdm.parse_license(license) + cdm.close() + + downloaded_file = download_video_nre( + mpd_url, + filename, + DLRequestPlatform.VTMGO, + dl_request.preferred_quality_matcher, + keys=keys, + ) + + # find 'nl-tt' subtitle or default to first subtitle + subtitles = config['video']['subtitles'] + subtitle = None + for sub in subtitles: + if sub['language'] == 'nl-tt': + subtitle = sub + break + if subtitle is None: + subtitle = subtitles[0] + subtitle_url = subtitle['url'] + logger.debug(f'Subtitle: {subtitle_url}') + + # download the subtitle and store it next to the video + subtitle_response = requests.get(subtitle_url) + subtitle_response.raise_for_status() + subtitle_filename = f'{downloaded_file}.vtt' + with open(subtitle_filename, 'wb') as f: + f.write(subtitle_response.content) + logger.debug(f'Subtitle saved to {subtitle_filename}') + insert_subtitle(downloaded_file, subtitle_filename) + os.remove(subtitle_filename) + + return diff --git a/dl-downer/src/server.py b/dl-downer/src/server.py index 110c3b0..5fb79b5 100644 --- a/dl-downer/src/server.py +++ b/dl-downer/src/server.py @@ -55,6 +55,9 @@ def start_server(): elif dl_request.platform == DLRequestPlatform.GOPLAY.value: from .downloaders.GOPLAY import GOPLAY_DL GOPLAY_DL(dl_request) + elif dl_request.platform == DLRequestPlatform.VTMGO.value: + from .downloaders.VTMGO import VTMGO_DL + VTMGO_DL(dl_request) elif dl_request.platform == DLRequestPlatform.GENERIC_MANIFEST.value: from .downloaders.GENERIC_MANIFEST import GENERIC_MANIFEST_DL GENERIC_MANIFEST_DL(dl_request) diff --git a/dl-downer/src/utils/browser.py b/dl-downer/src/utils/browser.py index 6cf3df0..9d46b7f 100644 --- a/dl-downer/src/utils/browser.py +++ b/dl-downer/src/utils/browser.py @@ -3,6 +3,7 @@ from playwright.sync_api import sync_playwright from playwright.sync_api import Browser, Page, Playwright +from playwright_stealth import stealth_sync from ..models.dl_request_platform import DLRequestPlatform @@ -39,7 +40,7 @@ def create_playwright_page(platform: DLRequestPlatform) -> tuple[Playwright, Bro playwright = sync_playwright().start() browser = playwright.chromium.launch( headless=os.getenv('HEADLESS', 'true') == 'true', - slow_mo=50, + slow_mo=200, ) custom_context = browser.new_context( user_agent=user_agent, @@ -47,5 +48,6 @@ def create_playwright_page(platform: DLRequestPlatform) -> tuple[Playwright, Bro storage_state=get_storage_state_location(platform), ) page = custom_context.new_page() + stealth_sync(page) return (playwright, browser, page) diff --git a/dl-downer/src/utils/files.py b/dl-downer/src/utils/files.py index 83e6bee..6f7d838 100644 --- a/dl-downer/src/utils/files.py +++ b/dl-downer/src/utils/files.py @@ -93,3 +93,44 @@ def merge_files( subprocess.run(command, check=True) logger.info(f'Merged successfully!') +def insert_subtitle( + input_file: str, + subtitle_file: str, +): + did_convert = False + # convert to srt if subtitle if needed + if subtitle_file.endswith('.vtt'): + converted_subtitle_file = os.path.join( + os.path.dirname(subtitle_file), + f'{os.path.basename(subtitle_file)[:-4]}.srt', + ) + command = [ 'ffmpeg', + '-i', subtitle_file, + converted_subtitle_file, + ] + logger.info(f'Converting {subtitle_file} to srt...') + subprocess.run(command, check=True) + subtitle_file = converted_subtitle_file + did_convert = True + logger.info(f'Converted successfully!') + + temp_output_file = os.path.join( + os.path.dirname(input_file), + f'subbed_{os.path.basename(input_file)}', + ) + command = [ 'ffmpeg', + '-i', input_file, + '-i', subtitle_file, + '-c', 'copy', + '-y', + temp_output_file, + ] + + logger.info(f'Inserting subtitle {subtitle_file} into {input_file}...') + subprocess.run(command, check=True) + # Move the temp file to the original file + overwrite + shutil.move(temp_output_file, input_file) + # if subs were converted, remove the converted file + if did_convert: + os.remove(subtitle_file) + logger.info(f'Subtitle inserted successfully!') diff --git a/dl-downer/src/utils/local_cdm.py b/dl-downer/src/utils/local_cdm.py index e7e45c9..3e1d5c9 100644 --- a/dl-downer/src/utils/local_cdm.py +++ b/dl-downer/src/utils/local_cdm.py @@ -39,6 +39,8 @@ def decrypt_response(self, response): logger.debug(f'Keys: {keys}') return keys + def parse_license(self, license): + return self.decrypt_response(license) def close(self): '''