Added docker support

app/YtManagerApp/management/jobs/delete_video.py

@@ -0,0 +1,39 @@
+import logging
+import os
+
+from YtManagerApp import scheduler
+from YtManagerApp.models import Video
+
+log = logging.getLogger('video_downloader')
+
+
+def delete_video(video: Video):
+    log.info('Deleting video %d [%s %s]', video.id, video.video_id, video.name)
+    count = 0
+
+    try:
+        for file in video.get_files():
+            log.info("Deleting file %s", file)
+            count += 1
+            try:
+                os.unlink(file)
+            except OSError as e:
+                log.error("Failed to delete file %s: Error: %s", file, e)
+
+    except OSError as e:
+        log.error("Failed to delete video %d [%s %s]. Error: %s", video.id, video.video_id, video.name, e)
+
+    video.downloaded_path = None
+    video.save()
+
+    log.info('Deleted video %d successfully! (%d files) [%s %s]', video.id, count, video.video_id, video.name)
+
+
+def schedule_delete_video(video: Video):
+    """
+    Schedules a download video job to run immediately.
+    :param video:
+    :return:
+    """
+    job = scheduler.scheduler.add_job(delete_video, args=[video])
+    log.info('Scheduled delete video job video=(%s), job=%s', video, job.id)

app/YtManagerApp/management/jobs/download_video.py

@@ -0,0 +1,110 @@
+from YtManagerApp.models import Video
+from YtManagerApp import scheduler
+from YtManagerApp.appconfig import settings
+import os
+import youtube_dl
+import logging
+import re
+
+log = logging.getLogger('video_downloader')
+log_youtube_dl = log.getChild('youtube_dl')
+
+
+def __get_valid_path(path):
+    """
+    Normalizes string, converts to lowercase, removes non-alpha characters,
+    and converts spaces to hyphens.
+    """
+    import unicodedata
+    value = unicodedata.normalize('NFKD', path).encode('ascii', 'ignore').decode('ascii')
+    value = re.sub('[:"*]', '', value).strip()
+    value = re.sub('[?<>|]', '#', value)
+    return value
+
+
+def __build_youtube_dl_params(video: Video):
+    # resolve path
+    pattern_dict = {
+        'channel': video.subscription.channel_name,
+        'channel_id': video.subscription.channel_id,
+        'playlist': video.subscription.name,
+        'playlist_id': video.subscription.playlist_id,
+        'playlist_index': "{:03d}".format(1 + video.playlist_index),
+        'title': video.name,
+        'id': video.video_id,
+    }
+
+    download_path = settings.get_sub(video.subscription, 'user', 'DownloadPath')
+    output_pattern = __get_valid_path(settings.get_sub(
+        video.subscription, 'user', 'DownloadFilePattern', vars=pattern_dict))
+
+    output_path = os.path.join(download_path, output_pattern)
+    output_path = os.path.normpath(output_path)
+
+    youtube_dl_params = {
+        'logger': log_youtube_dl,
+        'format': settings.get_sub(video.subscription, 'user', 'DownloadFormat'),
+        'outtmpl': output_path,
+        'writethumbnail': True,
+        'writedescription': True,
+        'writesubtitles': settings.getboolean_sub(video.subscription, 'user', 'DownloadSubtitles'),
+        'writeautomaticsub': settings.getboolean_sub(video.subscription, 'user', 'DownloadAutogeneratedSubtitles'),
+        'allsubtitles': settings.getboolean_sub(video.subscription, 'user', 'DownloadSubtitlesAll'),
+        'postprocessors': [
+            {
+                'key': 'FFmpegMetadata'
+            },
+        ]
+    }
+
+    sub_langs = settings.get_sub(video.subscription, 'user', 'DownloadSubtitlesLangs').split(',')
+    sub_langs = [i.strip() for i in sub_langs]
+    if len(sub_langs) > 0:
+        youtube_dl_params['subtitleslangs'] = sub_langs
+
+    sub_format = settings.get_sub(video.subscription, 'user', 'DownloadSubtitlesFormat')
+    if len(sub_format) > 0:
+        youtube_dl_params['subtitlesformat'] = sub_format
+
+    return youtube_dl_params, output_path
+
+
+def download_video(video: Video, attempt: int = 1):
+
+    log.info('Downloading video %d [%s %s]', video.id, video.video_id, video.name)
+
+    max_attempts = settings.getint_sub(video.subscription, 'user', 'DownloadMaxAttempts', fallback=3)
+
+    youtube_dl_params, output_path = __build_youtube_dl_params(video)
+    with youtube_dl.YoutubeDL(youtube_dl_params) as yt:
+        ret = yt.download(["https://www.youtube.com/watch?v=" + video.video_id])
+
+    log.info('Download finished with code %d', ret)
+
+    if ret == 0:
+        video.downloaded_path = output_path
+        video.save()
+        log.info('Video %d [%s %s] downloaded successfully!', video.id, video.video_id, video.name)
+
+    elif attempt <= max_attempts:
+        log.warning('Re-enqueueing video (attempt %d/%d)', attempt, max_attempts)
+        __schedule_download_video(video, attempt + 1)
+
+    else:
+        log.error('Multiple attempts to download video %d [%s %s] failed!', video.id, video.video_id, video.name)
+        video.downloaded_path = ''
+        video.save()
+
+
+def __schedule_download_video(video: Video, attempt=1):
+    job = scheduler.scheduler.add_job(download_video, args=[video, attempt])
+    log.info('Scheduled download video job video=(%s), attempt=%d, job=%s', video, attempt, job.id)
+
+
+def schedule_download_video(video: Video):
+    """
+    Schedules a download video job to run immediately.
+    :param video:
+    :return:
+    """
+    __schedule_download_video(video)

app/YtManagerApp/management/jobs/synchronize.py

@@ -0,0 +1,162 @@
+import errno
+import mimetypes
+from threading import Lock
+
+from apscheduler.triggers.cron import CronTrigger
+
+from YtManagerApp import scheduler
+from YtManagerApp.appconfig import settings
+from YtManagerApp.management.downloader import fetch_thumbnail, downloader_process_all, downloader_process_subscription
+from YtManagerApp.models import *
+from YtManagerApp.utils import youtube
+
+log = logging.getLogger('sync')
+__lock = Lock()
+
+_ENABLE_UPDATE_STATS = True
+
+
+def __check_new_videos_sub(subscription: Subscription, yt_api: youtube.YoutubeAPI):
+    # Get list of videos
+    for item in yt_api.playlist_items(subscription.playlist_id):
+        results = Video.objects.filter(video_id=item.resource_video_id, subscription=subscription)
+        if len(results) == 0:
+            log.info('New video for subscription %s: %s %s"', subscription, item.resource_video_id, item.title)
+            Video.create(item, subscription)
+
+    if _ENABLE_UPDATE_STATS:
+        all_vids = Video.objects.filter(subscription=subscription)
+        all_vids_ids = [video.video_id for video in all_vids]
+        all_vids_dict = {v.video_id: v for v in all_vids}
+
+        for yt_video in yt_api.videos(all_vids_ids, part='id,statistics'):
+            video = all_vids_dict.get(yt_video.id)
+
+            if yt_video.n_likes is not None \
+                    and yt_video.n_dislikes is not None \
+                    and yt_video.n_likes + yt_video.n_dislikes > 0:
+                video.rating = yt_video.n_likes / (yt_video.n_likes + yt_video.n_dislikes)
+
+            video.views = yt_video.n_views
+            video.save()
+
+
+def __detect_deleted(subscription: Subscription):
+
+    for video in Video.objects.filter(subscription=subscription, downloaded_path__isnull=False):
+        found_video = False
+        files = []
+        try:
+            files = list(video.get_files())
+        except OSError as e:
+            if e.errno != errno.ENOENT:
+                log.error("Could not access path %s. Error: %s", video.downloaded_path, e)
+                return
+
+        # Try to find a valid video file
+        for file in files:
+            mime, _ = mimetypes.guess_type(file)
+            if mime is not None and mime.startswith("video"):
+                found_video = True
+
+        # Video not found, we can safely assume that the video was deleted.
+        if not found_video:
+            log.info("Video %d was deleted! [%s %s]", video.id, video.video_id, video.name)
+            # Clean up
+            for file in files:
+                try:
+                    os.unlink(file)
+                except OSError as e:
+                    log.error("Could not delete redundant file %s. Error: %s", file, e)
+            video.downloaded_path = None
+
+            # Mark watched?
+            if settings.getboolean_sub(subscription, 'user', 'MarkDeletedAsWatched'):
+                video.watched = True
+
+            video.save()
+
+
+def __fetch_thumbnails_obj(iterable, obj_type, id_attr):
+    for obj in iterable:
+        if obj.icon_default.startswith("http"):
+            obj.icon_default = fetch_thumbnail(obj.icon_default, obj_type, getattr(obj, id_attr), 'default')
+        if obj.icon_best.startswith("http"):
+            obj.icon_best = fetch_thumbnail(obj.icon_best, obj_type, getattr(obj, id_attr), 'best')
+        obj.save()
+
+
+def __fetch_thumbnails():
+    log.info("Fetching subscription thumbnails... ")
+    __fetch_thumbnails_obj(Subscription.objects.filter(icon_default__istartswith='http'), 'sub', 'playlist_id')
+    __fetch_thumbnails_obj(Subscription.objects.filter(icon_best__istartswith='http'), 'sub', 'playlist_id')
+
+    log.info("Fetching video thumbnails... ")
+    __fetch_thumbnails_obj(Video.objects.filter(icon_default__istartswith='http'), 'video', 'video_id')
+    __fetch_thumbnails_obj(Video.objects.filter(icon_best__istartswith='http'), 'video', 'video_id')
+
+
+def synchronize():
+    if not __lock.acquire(blocking=False):
+        # Synchronize already running in another thread
+        log.info("Synchronize already running in another thread")
+        return
+
+    try:
+        log.info("Running scheduled synchronization... ")
+
+        # Sync subscribed playlists/channels
+        log.info("Sync - checking videos")
+        yt_api = youtube.YoutubeAPI.build_public()
+        for subscription in Subscription.objects.all():
+            __check_new_videos_sub(subscription, yt_api)
+            __detect_deleted(subscription)
+
+        log.info("Sync - checking for videos to download")
+        downloader_process_all()
+
+        log.info("Sync - fetching missing thumbnails")
+        __fetch_thumbnails()
+
+        log.info("Synchronization finished.")
+
+    finally:
+        __lock.release()
+
+
+def synchronize_subscription(subscription: Subscription):
+    __lock.acquire()
+    try:
+        log.info("Running synchronization for single subscription %d [%s]", subscription.id, subscription.name)
+        yt_api = youtube.YoutubeAPI.build_public()
+
+        log.info("Sync - checking videos")
+        __check_new_videos_sub(subscription, yt_api)
+        __detect_deleted(subscription)
+
+        log.info("Sync - checking for videos to download")
+        downloader_process_subscription(subscription)
+
+        log.info("Sync - fetching missing thumbnails")
+        __fetch_thumbnails()
+
+        log.info("Synchronization finished for subscription %d [%s].", subscription.id, subscription.name)
+
+    finally:
+        __lock.release()
+
+
+def schedule_synchronize_global():
+    trigger = CronTrigger.from_crontab(settings.get('global', 'SynchronizationSchedule'))
+    job = scheduler.scheduler.add_job(synchronize, trigger, max_instances=1, coalesce=True)
+    log.info('Scheduled synchronize job job=%s', job.id)
+
+
+def schedule_synchronize_now():
+    job = scheduler.scheduler.add_job(synchronize, max_instances=1, coalesce=True)
+    log.info('Scheduled synchronize now job job=%s', job.id)
+
+
+def schedule_synchronize_now_subscription(subscription: Subscription):
+    job = scheduler.scheduler.add_job(synchronize_subscription, args=[subscription])
+    log.info('Scheduled synchronize subscription job subscription=(%s), job=%s', subscription, job.id)