Big refactor

YtManagerApp/management/downloader.py

@@ -0,0 +1,110 @@
+from YtManagerApp.appconfig import instance as app_config
+from YtManagerApp.management.jobs.download_video import schedule_download_video
+from YtManagerApp.models import Video, Subscription
+from django.conf import settings
+import logging
+import requests
+import mimetypes
+import os
+from urllib.parse import urljoin
+
+log = logging.getLogger('downloader')
+
+
+def __get_subscription_config(sub: Subscription):
+    user_config = app_config.get_user_config(sub.user)
+
+    enabled = sub.auto_download
+    if enabled is None:
+        enabled = user_config.getboolean('user', 'AutoDownload')
+
+    global_limit = -1
+    if len(user_config.get('user', 'DownloadGlobalLimit')) > 0:
+        global_limit = user_config.getint('user', 'DownloadGlobalLimit')
+
+    limit = sub.download_limit
+    if limit is None:
+        limit = -1
+        if len(user_config.get('user', 'DownloadSubscriptionLimit')) > 0:
+            limit = user_config.getint('user', 'DownloadSubscriptionLimit')
+
+    order = sub.download_order
+    if order is None:
+        order = user_config.get('user', 'DownloadOrder')
+
+    return enabled, global_limit, limit, order
+
+
+def __process_subscription(sub: Subscription):
+    log.info('Processing subscription %d [%s %s]', sub.id, sub.playlist_id, sub.id)
+
+    enabled, global_limit, limit, order = __get_subscription_config(sub)
+    log.info('Determined settings enabled=%s global_limit=%d limit=%d order="%s"', enabled, global_limit, limit, order)
+
+    if enabled:
+        videos_to_download = Video.objects\
+            .filter(subscription=sub, downloaded_path__isnull=True, watched=False)\
+            .order_by(order)
+
+        log.info('%d download candidates.', len(videos_to_download))
+
+        if global_limit > 0:
+            global_downloaded = Video.objects.filter(subscription__user=sub.user, downloaded_path__isnull=False).count()
+            allowed_count = max(global_limit - global_downloaded, 0)
+            videos_to_download = videos_to_download[0:allowed_count]
+            log.info('Global limit is set, can only download up to %d videos.', allowed_count)
+
+        if limit > 0:
+            sub_downloaded = Video.objects.filter(subscription=sub, downloaded_path__isnull=False).count()
+            allowed_count = max(limit - sub_downloaded, 0)
+            videos_to_download = videos_to_download[0:allowed_count]
+            log.info('Limit is set, can only download up to %d videos.', allowed_count)
+
+        # enqueue download
+        for video in videos_to_download:
+            log.info('Enqueuing video %d [%s %s] index=%d', video.id, video.video_id, video.name, video.playlist_index)
+            schedule_download_video(video)
+
+    log.info('Finished processing subscription %d [%s %s]', sub.id, sub.playlist_id, sub.id)
+
+
+def downloader_process_all():
+    for subscription in Subscription.objects.all():
+        __process_subscription(subscription)
+
+
+def fetch_thumbnail(url, object_type, identifier, quality):
+
+    log.info('Fetching thumbnail url=%s object_type=%s identifier=%s quality=%s', url, object_type, identifier, quality)
+
+    # Make request to obtain mime type
+    try:
+        response = requests.get(url, stream=True)
+    except requests.exceptions.RequestException as e:
+        log.error('Failed to fetch thumbnail %s. Error: %s', url, e)
+        return url
+
+    ext = mimetypes.guess_extension(response.headers['Content-Type'])
+
+    # Build file path
+    file_name = f"{identifier}-{quality}{ext}"
+    abs_path_dir = os.path.join(settings.MEDIA_ROOT, "thumbs", object_type)
+    abs_path = os.path.join(abs_path_dir, file_name)
+
+    # Store image
+    try:
+        os.makedirs(abs_path_dir, exist_ok=True)
+        with open(abs_path, "wb") as f:
+            for chunk in response.iter_content(chunk_size=1024):
+                if chunk:
+                    f.write(chunk)
+    except requests.exceptions.RequestException as e:
+        log.error('Error while downloading stream for thumbnail %s. Error: %s', url, e)
+        return url
+    except OSError as e:
+        log.error('Error while writing to file %s for thumbnail %s. Error: %s', abs_path, url, e)
+        return url
+
+    # Return
+    media_url = urljoin(settings.MEDIA_URL, f"thumbs/{object_type}/{file_name}")
+    return media_url

YtManagerApp/management/jobs/download_video.py

@@ -0,0 +1,93 @@
+from YtManagerApp.models import Video
+from YtManagerApp.scheduler import instance as scheduler
+from YtManagerApp.appconfig import instance as app_config
+import os
+import youtube_dl
+import logging
+
+log = logging.getLogger('video_downloader')
+log_youtube_dl = log.getChild('youtube_dl')
+
+
+def __build_youtube_dl_params(video: Video, user_config):
+    # resolve path
+    format_dict = {
+        'channel': video.subscription.channel.name,
+        'channel_id': video.subscription.channel.channel_id,
+        'playlist': video.subscription.name,
+        'playlist_id': video.subscription.playlist_id,
+        'playlist_index': "{:03d}".format(1 + video.playlist_index),
+        'title': video.name,
+        'id': video.video_id,
+    }
+
+    user_config.set_additional_interpolation_options(**format_dict)
+
+    download_path = user_config.get('user', 'DownloadPath')
+    output_pattern = user_config.get('user', 'DownloadFilePattern')
+    output_path = os.path.join(download_path, output_pattern)
+    output_path = os.path.normpath(output_path)
+
+    youtube_dl_params = {
+        'logger': log_youtube_dl,
+        'format': user_config.get('user', 'DownloadFormat'),
+        'outtmpl': output_path,
+        'writethumbnail': True,
+        'writedescription': True,
+        'writesubtitles': user_config.getboolean('user', 'DownloadSubtitles'),
+        'writeautomaticsub': user_config.getboolean('user', 'DownloadAutogeneratedSubtitles'),
+        'allsubtitles': user_config.getboolean('user', 'DownloadSubtitlesAll'),
+        'postprocessors': [
+            {
+                'key': 'FFmpegMetadataPP'
+            },
+        ]
+    }
+
+    sub_langs = user_config.get('user', 'DownloadSubtitlesLangs').split(',')
+    sub_langs = [i.strip() for i in sub_langs]
+    if len(sub_langs) > 0:
+        youtube_dl_params['subtitleslangs'] = sub_langs
+
+    sub_format = user_config.get('user', 'DownloadSubtitlesFormat')
+    if len(sub_format) > 0:
+        youtube_dl_params['subtitlesformat'] = sub_format
+
+    return youtube_dl_params, output_path
+
+
+def download_video(video: Video, attempt: int = 1):
+
+    log.info('Downloading video %d [%s %s]', video.id, video.video_id, video.name)
+
+    user_config = app_config.get_user_config(video.subscription.user)
+    max_attempts = user_config.getint('user', 'DownloadMaxAttempts', fallback=3)
+
+    youtube_dl_params, output_path = __build_youtube_dl_params(video, user_config)
+    with youtube_dl.YoutubeDL(youtube_dl_params) as yt:
+        ret = yt.download(["https://www.youtube.com/watch?v=" + video.video_id])
+
+    log.info('Download finished with code %d', ret)
+
+    if ret == 0:
+        video.downloaded_path = output_path
+        video.save()
+        log.error('Video %d [%s %s] downloaded successfully!', video.id, video.video_id, video.name)
+
+    elif attempt <= max_attempts:
+        log.warning('Re-enqueueing video (attempt %d/%d)', attempt, max_attempts)
+        scheduler.add_job(download_video, args=[video, attempt + 1])
+
+    else:
+        log.error('Multiple attempts to download video %d [%s %s] failed!', video.id, video.video_id, video.name)
+        video.downloaded_path = ''
+        video.save()
+
+
+def schedule_download_video(video: Video):
+    """
+    Schedules a download video job to run immediately.
+    :param video:
+    :return:
+    """
+    scheduler.add_job(download_video, args=[video, 1])

YtManagerApp/management/jobs/synchronize.py

@@ -0,0 +1,46 @@
+from YtManagerApp.models import *
+import logging
+
+
+def synchronize():
+    logger = logging.getLogger('sync')
+
+    logger.info("Running scheduled synchronization... ")
+
+    # Sync subscribed playlists/channels
+    yt_api = YoutubeAPI.build_public()
+    for subscription in Subscription.objects.all():
+        SubscriptionManager.__synchronize(subscription, yt_api)
+
+    # Fetch thumbnails
+    logger.info("Fetching channel thumbnails... ")
+    for ch in Channel.objects.filter(icon_default__istartswith='http'):
+        ch.icon_default = SubscriptionManager.__fetch_thumbnail(ch.icon_default, 'channel', ch.channel_id, 'default')
+        ch.save()
+
+    for ch in Channel.objects.filter(icon_best__istartswith='http'):
+        ch.icon_best = SubscriptionManager.__fetch_thumbnail(ch.icon_best, 'channel', ch.channel_id, 'best')
+        ch.save()
+
+    logger.info("Fetching subscription thumbnails... ")
+    for sub in Subscription.objects.filter(icon_default__istartswith='http'):
+        sub.icon_default = SubscriptionManager.__fetch_thumbnail(sub.icon_default, 'sub', sub.playlist_id, 'default')
+        sub.save()
+
+    for sub in Subscription.objects.filter(icon_best__istartswith='http'):
+        sub.icon_best = SubscriptionManager.__fetch_thumbnail(sub.icon_best, 'sub', sub.playlist_id, 'best')
+        sub.save()
+
+    logger.info("Fetching video thumbnails... ")
+    for vid in Video.objects.filter(icon_default__istartswith='http'):
+        vid.icon_default = SubscriptionManager.__fetch_thumbnail(vid.icon_default, 'video', vid.video_id, 'default')
+        vid.save()
+
+    for vid in Video.objects.filter(icon_best__istartswith='http'):
+        vid.icon_best = SubscriptionManager.__fetch_thumbnail(vid.icon_best, 'video', vid.video_id, 'best')
+        vid.save()
+
+    print("Downloading videos...")
+    Downloader.download_all()
+
+    print("Synchronization finished.")

YtManagerApp/management/management.py

@@ -0,0 +1,278 @@
+from YtManagerApp.models import SubscriptionFolder, Subscription, Video, Channel
+from YtManagerApp.utils.youtube import YoutubeAPI, YoutubeChannelInfo, YoutubePlaylistItem
+from django.conf import settings
+from apscheduler.schedulers.background import BackgroundScheduler
+import os
+import os.path
+import requests
+from urllib.parse import urljoin
+import mimetypes
+import youtube_dl
+
+from YtManagerApp.scheduler import instance as scheduler
+from YtManagerApp.appconfig import instance as app_config
+from apscheduler.triggers.cron import CronTrigger
+
+
+class FolderManager(object):
+
+    @staticmethod
+    def create_or_edit(fid, name, parent_id):
+        # Create or edit
+        if fid == '#':
+            folder = SubscriptionFolder()
+        else:
+            folder = SubscriptionFolder.objects.get(id=int(fid))
+
+        # Set attributes
+        folder.name = name
+        if parent_id == '#':
+            folder.parent = None
+        else:
+            folder.parent = SubscriptionFolder.objects.get(id=int(parent_id))
+
+        FolderManager.__validate(folder)
+        folder.save()
+
+    @staticmethod
+    def __validate(folder: SubscriptionFolder):
+        # Make sure folder name is unique in the parent folder
+        for dbFolder in SubscriptionFolder.objects.filter(parent_id=folder.parent_id):
+            if dbFolder.id != folder.id and dbFolder.name == folder.name:
+                raise ValueError('Folder name is not unique!')
+
+        # Prevent parenting loops
+        current = folder
+        visited = []
+
+        while not (current is None):
+            if current in visited:
+                raise ValueError('Parenting cycle detected!')
+            visited.append(current)
+            current = current.parent
+
+    @staticmethod
+    def delete(fid: int):
+        folder = SubscriptionFolder.objects.get(id=fid)
+        folder.delete()
+
+    @staticmethod
+    def list_videos(fid: int):
+        folder = SubscriptionFolder.objects.get(id=fid)
+        folder_list = []
+        queue = [folder]
+        while len(queue) > 0:
+            folder = queue.pop()
+            folder_list.append(folder)
+            queue.extend(SubscriptionFolder.objects.filter(parent=folder))
+
+        return Video.objects.filter(subscription__parent_folder__in=folder_list).order_by('-publish_date')
+
+
+class SubscriptionManager(object):
+    __scheduler = BackgroundScheduler()
+
+    @staticmethod
+    def create_or_edit(sid, url, name, parent_id):
+        # Create or edit
+        if sid == '#':
+            SubscriptionManager.create(url, parent_id, YoutubeAPI.build_public())
+        else:
+            sub = Subscription.objects.get(id=int(sid))
+            sub.name = name
+
+            if parent_id == '#':
+                sub.parent_folder = None
+            else:
+                sub.parent_folder = SubscriptionFolder.objects.get(id=int(parent_id))
+
+            sub.save()
+
+    @staticmethod
+    def create(url, parent_id, yt_api: YoutubeAPI):
+        sub = Subscription()
+        # Set parent
+        if parent_id == '#':
+            sub.parent_folder = None
+        else:
+            sub.parent_folder = SubscriptionFolder.objects.get(id=int(parent_id))
+
+        # Pull information about the channel and playlist
+        url_type, url_id = yt_api.parse_channel_url(url)
+
+        if url_type == 'playlist_id':
+            info_playlist = yt_api.get_playlist_info(url_id)
+            channel = SubscriptionManager.__get_or_create_channel('channel_id', info_playlist.getChannelId(), yt_api)
+            sub.name = info_playlist.getTitle()
+            sub.playlist_id = info_playlist.getId()
+            sub.description = info_playlist.getDescription()
+            sub.channel = channel
+            sub.icon_default = info_playlist.getDefaultThumbnailUrl()
+            sub.icon_best = info_playlist.getBestThumbnailUrl()
+
+        else:
+            channel = SubscriptionManager.__get_or_create_channel(url_type, url_id, yt_api)
+            # No point in getting the 'uploads' playlist info
+            sub.name = channel.name
+            sub.playlist_id = channel.upload_playlist_id
+            sub.description = channel.description
+            sub.channel = channel
+            sub.icon_default = channel.icon_default
+            sub.icon_best = channel.icon_best
+
+        sub.save()
+
+    @staticmethod
+    def list_videos(fid: int):
+        sub = Subscription.objects.get(id=fid)
+        return Video.objects.filter(subscription=sub).order_by('playlist_index')
+
+    @staticmethod
+    def __get_or_create_channel(url_type, url_id, yt_api: YoutubeAPI):
+
+        channel: Channel = None
+        info_channel: YoutubeChannelInfo = None
+
+        if url_type == 'user':
+            channel = Channel.find_by_username(url_id)
+            if not channel:
+                info_channel = yt_api.get_channel_info_by_username(url_id)
+                channel = Channel.find_by_channel_id(info_channel.getId())
+
+        elif url_type == 'channel_id':
+            channel = Channel.find_by_channel_id(url_id)
+            if not channel:
+                info_channel = yt_api.get_channel_info(url_id)
+
+        elif url_type == 'channel_custom':
+            channel = Channel.find_by_custom_url(url_id)
+            if not channel:
+                found_channel_id = yt_api.search_channel(url_id)
+                channel = Channel.find_by_channel_id(found_channel_id)
+                if not channel:
+                    info_channel = yt_api.get_channel_info(found_channel_id)
+
+        # Store information about the channel
+        if info_channel:
+            if not channel:
+                channel = Channel()
+            if url_type == 'user':
+                channel.username = url_id
+            SubscriptionManager.__update_channel(channel, info_channel)
+
+        return channel
+
+    @staticmethod
+    def __update_channel(channel: Channel, yt_info: YoutubeChannelInfo):
+        channel.channel_id = yt_info.getId()
+        channel.custom_url = yt_info.getCustomUrl()
+        channel.name = yt_info.getTitle()
+        channel.description = yt_info.getDescription()
+        channel.icon_default = yt_info.getDefaultThumbnailUrl()
+        channel.icon_best = yt_info.getBestThumbnailUrl()
+        channel.upload_playlist_id = yt_info.getUploadsPlaylist()
+        channel.save()
+
+    @staticmethod
+    def __create_video(yt_video: YoutubePlaylistItem, subscription: Subscription):
+        video = Video()
+        video.video_id = yt_video.getVideoId()
+        video.name = yt_video.getTitle()
+        video.description = yt_video.getDescription()
+        video.watched = False
+        video.downloaded_path = None
+        video.subscription = subscription
+        video.playlist_index = yt_video.getPlaylistIndex()
+        video.publish_date = yt_video.getPublishDate()
+        video.icon_default = yt_video.getDefaultThumbnailUrl()
+        video.icon_best = yt_video.getBestThumbnailUrl()
+        video.save()
+
+    @staticmethod
+    def __synchronize(subscription: Subscription, yt_api: YoutubeAPI):
+        # Get list of videos
+        for video in yt_api.list_playlist_videos(subscription.playlist_id):
+            results = Video.objects.filter(video_id=video.getVideoId(), subscription=subscription)
+            if len(results) == 0:
+                print('New video for subscription "', subscription, '": ', video.getVideoId(), video.getTitle())
+                SubscriptionManager.__create_video(video, subscription)
+
+    @staticmethod
+    def __synchronize_all():
+        print("Running scheduled synchronization... ")
+
+        # Sync subscribed playlists/channels
+        yt_api = YoutubeAPI.build_public()
+        for subscription in Subscription.objects.all():
+            SubscriptionManager.__synchronize(subscription, yt_api)
+
+        # Fetch thumbnails
+        print("Fetching channel thumbnails... ")
+        for ch in Channel.objects.filter(icon_default__istartswith='http'):
+            ch.icon_default = SubscriptionManager.__fetch_thumbnail(ch.icon_default, 'channel', ch.channel_id, 'default')
+            ch.save()
+
+        for ch in Channel.objects.filter(icon_best__istartswith='http'):
+            ch.icon_best = SubscriptionManager.__fetch_thumbnail(ch.icon_best, 'channel', ch.channel_id, 'best')
+            ch.save()
+
+        print("Fetching subscription thumbnails... ")
+        for sub in Subscription.objects.filter(icon_default__istartswith='http'):
+            sub.icon_default = SubscriptionManager.__fetch_thumbnail(sub.icon_default, 'sub', sub.playlist_id, 'default')
+            sub.save()
+
+        for sub in Subscription.objects.filter(icon_best__istartswith='http'):
+            sub.icon_best = SubscriptionManager.__fetch_thumbnail(sub.icon_best, 'sub', sub.playlist_id, 'best')
+            sub.save()
+
+        print("Fetching video thumbnails... ")
+        for vid in Video.objects.filter(icon_default__istartswith='http'):
+            vid.icon_default = SubscriptionManager.__fetch_thumbnail(vid.icon_default, 'video', vid.video_id, 'default')
+            vid.save()
+
+        for vid in Video.objects.filter(icon_best__istartswith='http'):
+            vid.icon_best = SubscriptionManager.__fetch_thumbnail(vid.icon_best, 'video', vid.video_id, 'best')
+            vid.save()
+
+        print("Downloading videos...")
+        Downloader.download_all()
+
+        print("Synchronization finished.")
+
+    @staticmethod
+    def __fetch_thumbnail(url, object_type, identifier, quality):
+
+        # Make request to obtain mime type
+        response = requests.get(url, stream=True)
+        ext = mimetypes.guess_extension(response.headers['Content-Type'])
+
+        # Build file path
+        file_name = f"{identifier}-{quality}{ext}"
+        abs_path_dir = os.path.join(settings.MEDIA_ROOT, "thumbs", object_type)
+        abs_path = os.path.join(abs_path_dir, file_name)
+
+        # Store image
+        os.makedirs(abs_path_dir, exist_ok=True)
+        with open(abs_path, "wb") as f:
+            for chunk in response.iter_content(chunk_size=1024):
+                if chunk:
+                    f.write(chunk)
+
+        # Return
+        media_url = urljoin(settings.MEDIA_URL, f"thumbs/{object_type}/{file_name}")
+        return media_url
+
+    @staticmethod
+    def start_scheduler():
+        SubscriptionManager.__scheduler.add_job(SubscriptionManager.__synchronize_all, 'cron',
+                                                hour='*', minute=38, max_instances=1)
+        SubscriptionManager.__scheduler.start()
+
+
+def setup_synchronization_job():
+    trigger = CronTrigger.from_crontab(app_config.get('global', 'SynchronizationSchedule'))
+    scheduler.add_job(synchronize_all, trigger, max_instances=1)
+
+
+def synchronize_all():
+    pass