Implemented video management facilities.

YtManagerApp/management/jobs/delete_video.py

@@ -0,0 +1,38 @@
+import logging
+import os
+
+from YtManagerApp import scheduler
+from YtManagerApp.models import Video
+
+log = logging.getLogger('video_downloader')
+
+
+def delete_video(video: Video):
+    log.info('Deleting video %d [%s %s]', video.id, video.video_id, video.name)
+    count = 0
+
+    try:
+        for file in video.get_files():
+            log.info("Deleting file %s", file)
+            count += 1
+            try:
+                os.unlink(file)
+            except OSError as e:
+                log.error("Failed to delete file %s: Error: %s", file, e)
+
+    except OSError as e:
+        log.error("Failed to delete video %d [%s %s]. Error: %s", video.id, video.video_id, video.name, e)
+
+    video.downloaded_path = None
+    video.save()
+
+    log.info('Deleted video %d successfully! (%d files) [%s %s]', video.id, count, video.video_id, video.name)
+
+
+def schedule_delete_video(video: Video):
+    """
+    Schedules a download video job to run immediately.
+    :param video:
+    :return:
+    """
+    scheduler.instance.add_job(delete_video, args=[video])

YtManagerApp/management/jobs/download_video.py

@@ -4,11 +4,24 @@ from YtManagerApp.appconfig import get_user_config
 import os
 import youtube_dl
 import logging
+import re
 
 log = logging.getLogger('video_downloader')
 log_youtube_dl = log.getChild('youtube_dl')
 
 
+def __get_valid_path(path):
+    """
+    Normalizes string, converts to lowercase, removes non-alpha characters,
+    and converts spaces to hyphens.
+    """
+    import unicodedata
+    value = unicodedata.normalize('NFKD', path).encode('ascii', 'ignore').decode('ascii')
+    value = re.sub('[:"*]', '', value).strip()
+    value = re.sub('[?<>|]', '#', value)
+    return value
+
+
 def __build_youtube_dl_params(video: Video, user_config):
     # resolve path
     format_dict = {
@@ -24,7 +37,7 @@ def __build_youtube_dl_params(video: Video, user_config):
     user_config.set_additional_interpolation_options(**format_dict)
 
     download_path = user_config.get('user', 'DownloadPath')
-    output_pattern = user_config.get('user', 'DownloadFilePattern')
+    output_pattern = __get_valid_path(user_config.get('user', 'DownloadFilePattern'))
     output_path = os.path.join(download_path, output_pattern)
     output_path = os.path.normpath(output_path)
 
@@ -39,7 +52,7 @@ def __build_youtube_dl_params(video: Video, user_config):
         'allsubtitles': user_config.getboolean('user', 'DownloadSubtitlesAll'),
         'postprocessors': [
             {
-                'key': 'FFmpegMetadataPP'
+                'key': 'FFmpegMetadata'
             },
         ]
     }
@@ -72,7 +85,7 @@ def download_video(video: Video, attempt: int = 1):
     if ret == 0:
         video.downloaded_path = output_path
         video.save()
-        log.error('Video %d [%s %s] downloaded successfully!', video.id, video.video_id, video.name)
+        log.info('Video %d [%s %s] downloaded successfully!', video.id, video.video_id, video.name)
 
     elif attempt <= max_attempts:
         log.warning('Re-enqueueing video (attempt %d/%d)', attempt, max_attempts)

YtManagerApp/management/jobs/synchronize.py

@@ -1,24 +1,67 @@
-import logging
-
 from apscheduler.triggers.cron import CronTrigger
+from threading import Lock
+import os
+import errno
+import mimetypes
 
-from YtManagerApp.appconfig import settings
-from YtManagerApp.management.downloader import fetch_thumbnail, downloader_process_all
+from YtManagerApp import scheduler
+from YtManagerApp.appconfig import settings, get_user_config
+from YtManagerApp.management.downloader import fetch_thumbnail, downloader_process_all, downloader_process_subscription
 from YtManagerApp.management.videos import create_video
 from YtManagerApp.models import *
-from YtManagerApp import scheduler
 from YtManagerApp.utils.youtube import YoutubeAPI
 
 log = logging.getLogger('sync')
+__lock = Lock()
 
 
-def __synchronize_sub(subscription: Subscription, yt_api: YoutubeAPI):
+def __check_new_videos_sub(subscription: Subscription, yt_api: YoutubeAPI):
     # Get list of videos
     for video in yt_api.list_playlist_videos(subscription.playlist_id):
         results = Video.objects.filter(video_id=video.getVideoId(), subscription=subscription)
         if len(results) == 0:
-            log.info('New video for subscription "', subscription, '": ', video.getVideoId(), video.getTitle())
+            log.info('New video for subscription %s: %s %s"', subscription, video.getVideoId(), video.getTitle())
             create_video(video, subscription)
+        else:
+            # TODO... update view count, rating etc
+            pass
+
+
+def __detect_deleted(subscription: Subscription):
+    user_settings = get_user_config(subscription.user)
+
+    for video in Video.objects.filter(subscription=subscription, downloaded_path__isnull=False):
+        found_video = False
+        files = []
+        try:
+            files = list(video.get_files())
+        except OSError as e:
+            if e.errno != errno.ENOENT:
+                log.error("Could not access path %s. Error: %s", video.downloaded_path, e)
+                return
+
+        # Try to find a valid video file
+        for file in files:
+            mime, _ = mimetypes.guess_type(file)
+            if mime is not None and mime.startswith("video"):
+                found_video = True
+
+        # Video not found, we can safely assume that the video was deleted.
+        if not found_video:
+            log.info("Video %d was deleted! [%s %s]", video.id, video.video_id, video.name)
+            # Clean up
+            for file in files:
+                try:
+                    os.unlink(file)
+                except OSError as e:
+                    log.error("Could not delete redundant file %s. Error: %s", file, e)
+            video.downloaded_path = None
+
+            # Mark watched?
+            if user_settings.getboolean('user', 'MarkDeletedAsWatched'):
+                video.watched = True
+
+            video.save()
 
 
 def __fetch_thumbnails_obj(iterable, obj_type, id_attr):
@@ -46,23 +89,63 @@ def __fetch_thumbnails():
 
 
 def synchronize():
-    log.info("Running scheduled synchronization... ")
+    if not __lock.acquire(blocking=False):
+        # Synchronize already running in another thread
+        log.info("Synchronize already running in another thread")
+        return
 
-    # Sync subscribed playlists/channels
-    log.info("Sync - checking for new videos")
-    yt_api = YoutubeAPI.build_public()
-    for subscription in Subscription.objects.all():
-        __synchronize_sub(subscription, yt_api)
+    try:
+        log.info("Running scheduled synchronization... ")
 
-    log.info("Sync - checking for videos to download")
-    downloader_process_all()
+        # Sync subscribed playlists/channels
+        log.info("Sync - checking videos")
+        yt_api = YoutubeAPI.build_public()
+        for subscription in Subscription.objects.all():
+            __check_new_videos_sub(subscription, yt_api)
+            __detect_deleted(subscription)
 
-    log.info("Sync - fetching missing thumbnails")
-    __fetch_thumbnails()
+        log.info("Sync - checking for videos to download")
+        downloader_process_all()
 
-    log.info("Synchronization finished.")
+        log.info("Sync - fetching missing thumbnails")
+        __fetch_thumbnails()
+
+        log.info("Synchronization finished.")
+
+    finally:
+        __lock.release()
 
 
-def schedule_synchronize():
+def synchronize_subscription(subscription: Subscription):
+    __lock.acquire()
+    try:
+        log.info("Running synchronization for single subscription %d [%s]", subscription.id, subscription.name)
+        yt_api = YoutubeAPI.build_public()
+
+        log.info("Sync - checking videos")
+        __check_new_videos_sub(subscription, yt_api)
+        __detect_deleted(subscription)
+
+        log.info("Sync - checking for videos to download")
+        downloader_process_subscription(subscription)
+
+        log.info("Sync - fetching missing thumbnails")
+        __fetch_thumbnails()
+
+        log.info("Synchronization finished for subscription %d [%s].", subscription.id, subscription.name)
+
+    finally:
+        __lock.release()
+
+
+def schedule_synchronize_global():
     trigger = CronTrigger.from_crontab(settings.get('global', 'SynchronizationSchedule'))
-    scheduler.instance.add_job(synchronize, trigger, max_instances=1)
+    scheduler.instance.add_job(synchronize, trigger, max_instances=1, coalesce=True)
+
+
+def schedule_synchronize_now():
+    scheduler.instance.add_job(synchronize, max_instances=1, coalesce=True)
+
+
+def schedule_synchronize_now_subscription(subscription: Subscription):
+    scheduler.instance.add_job(synchronize_subscription, args=[subscription])