from YtManagerApp.models import Video from YtManagerApp import scheduler from YtManagerApp.appconfig import settings import os import youtube_dl import logging import re from threading import Lock log = logging.getLogger('video_downloader') log_youtube_dl = log.getChild('youtube_dl') _lock = Lock() def __get_valid_path(path): """ Normalizes string, converts to lowercase, removes non-alpha characters, and converts spaces to hyphens. """ import unicodedata value = unicodedata.normalize('NFKD', path).encode('ascii', 'ignore').decode('ascii') value = re.sub('[:"*]', '', value).strip() value = re.sub('[?<>|]', '#', value) return value def __build_youtube_dl_params(video: Video): # resolve path pattern_dict = { 'channel': video.subscription.channel_name, 'channel_id': video.subscription.channel_id, 'playlist': video.subscription.name, 'playlist_id': video.subscription.playlist_id, 'playlist_index': "{:03d}".format(1 + video.playlist_index), 'title': video.name, 'id': video.video_id, } download_path = settings.get_sub(video.subscription, 'user', 'DownloadPath') output_pattern = __get_valid_path(settings.get_sub( video.subscription, 'user', 'DownloadFilePattern', vars=pattern_dict)) output_path = os.path.join(download_path, output_pattern) output_path = os.path.normpath(output_path) youtube_dl_params = { 'logger': log_youtube_dl, 'format': settings.get_sub(video.subscription, 'user', 'DownloadFormat'), 'outtmpl': output_path, 'writethumbnail': True, 'writedescription': True, 'writesubtitles': settings.getboolean_sub(video.subscription, 'user', 'DownloadSubtitles'), 'writeautomaticsub': settings.getboolean_sub(video.subscription, 'user', 'DownloadAutogeneratedSubtitles'), 'allsubtitles': settings.getboolean_sub(video.subscription, 'user', 'DownloadSubtitlesAll'), 'postprocessors': [ { 'key': 'FFmpegMetadata' }, ] } sub_langs = settings.get_sub(video.subscription, 'user', 'DownloadSubtitlesLangs').split(',') sub_langs = [i.strip() for i in sub_langs] if len(sub_langs) > 0: youtube_dl_params['subtitleslangs'] = sub_langs sub_format = settings.get_sub(video.subscription, 'user', 'DownloadSubtitlesFormat') if len(sub_format) > 0: youtube_dl_params['subtitlesformat'] = sub_format return youtube_dl_params, output_path def download_video(video: Video, attempt: int = 1): log.info('Downloading video %d [%s %s]', video.id, video.video_id, video.name) # Issue: if multiple videos are downloaded at the same time, a race condition appears in the mkdirs() call that # youtube-dl makes, which causes it to fail with the error 'Cannot create folder - file already exists'. # For now, allow a single download instance. _lock.acquire() try: max_attempts = settings.getint_sub(video.subscription, 'user', 'DownloadMaxAttempts', fallback=3) youtube_dl_params, output_path = __build_youtube_dl_params(video) with youtube_dl.YoutubeDL(youtube_dl_params) as yt: ret = yt.download(["https://www.youtube.com/watch?v=" + video.video_id]) log.info('Download finished with code %d', ret) if ret == 0: video.downloaded_path = output_path video.save() log.info('Video %d [%s %s] downloaded successfully!', video.id, video.video_id, video.name) elif attempt <= max_attempts: log.warning('Re-enqueueing video (attempt %d/%d)', attempt, max_attempts) __schedule_download_video(video, attempt + 1) else: log.error('Multiple attempts to download video %d [%s %s] failed!', video.id, video.video_id, video.name) video.downloaded_path = '' video.save() finally: _lock.release() def __schedule_download_video(video: Video, attempt=1): job = scheduler.scheduler.add_job(download_video, args=[video, attempt]) log.info('Scheduled download video job video=(%s), attempt=%d, job=%s', video, attempt, job.id) def schedule_download_video(video: Video): """ Schedules a download video job to run immediately. :param video: :return: """ __schedule_download_video(video)