ytsm/YtManagerApp/management/jobs/synchronize.py

from apscheduler.triggers.cron import CronTrigger
from threading import Lock
import os
import errno
import mimetypes

from YtManagerApp import scheduler
from YtManagerApp.appconfig import settings, get_user_config
from YtManagerApp.management.downloader import fetch_thumbnail, downloader_process_all, downloader_process_subscription
from YtManagerApp.management.videos import create_video
from YtManagerApp.models import *
from YtManagerApp.utils.youtube import YoutubeAPI

log = logging.getLogger('sync')
__lock = Lock()


def __check_new_videos_sub(subscription: Subscription, yt_api: YoutubeAPI):
    # Get list of videos
    for video in yt_api.list_playlist_videos(subscription.playlist_id):
        results = Video.objects.filter(video_id=video.getVideoId(), subscription=subscription)
        if len(results) == 0:
            log.info('New video for subscription %s: %s %s"', subscription, video.getVideoId(), video.getTitle())
            db_video = create_video(video, subscription)
        else:
            db_video = results.first()

        # Update video stats - rating and view count
        stats = yt_api.get_single_video_stats(db_video.video_id)
        db_video.rating = stats.get_like_count() / (stats.get_like_count() + stats.get_dislike_count())
        db_video.views = stats.get_view_count()
        db_video.save()


def __detect_deleted(subscription: Subscription):
    user_settings = get_user_config(subscription.user)

    for video in Video.objects.filter(subscription=subscription, downloaded_path__isnull=False):
        found_video = False
        files = []
        try:
            files = list(video.get_files())
        except OSError as e:
            if e.errno != errno.ENOENT:
                log.error("Could not access path %s. Error: %s", video.downloaded_path, e)
                return

        # Try to find a valid video file
        for file in files:
            mime, _ = mimetypes.guess_type(file)
            if mime is not None and mime.startswith("video"):
                found_video = True

        # Video not found, we can safely assume that the video was deleted.
        if not found_video:
            log.info("Video %d was deleted! [%s %s]", video.id, video.video_id, video.name)
            # Clean up
            for file in files:
                try:
                    os.unlink(file)
                except OSError as e:
                    log.error("Could not delete redundant file %s. Error: %s", file, e)
            video.downloaded_path = None

            # Mark watched?
            if user_settings.getboolean('user', 'MarkDeletedAsWatched'):
                video.watched = True

            video.save()


def __fetch_thumbnails_obj(iterable, obj_type, id_attr):
    for obj in iterable:
        if obj.icon_default.startswith("http"):
            obj.icon_default = fetch_thumbnail(obj.icon_default, obj_type, getattr(obj, id_attr), 'default')
        if obj.icon_best.startswith("http"):
            obj.icon_best = fetch_thumbnail(obj.icon_best, obj_type, getattr(obj, id_attr), 'best')
        obj.save()


def __fetch_thumbnails():
    # Fetch thumbnails
    log.info("Fetching channel thumbnails... ")
    __fetch_thumbnails_obj(Channel.objects.filter(icon_default__istartswith='http'), 'channel', 'channel_id')
    __fetch_thumbnails_obj(Channel.objects.filter(icon_best__istartswith='http'), 'channel', 'channel_id')

    log.info("Fetching subscription thumbnails... ")
    __fetch_thumbnails_obj(Subscription.objects.filter(icon_default__istartswith='http'), 'sub', 'playlist_id')
    __fetch_thumbnails_obj(Subscription.objects.filter(icon_best__istartswith='http'), 'sub', 'playlist_id')

    log.info("Fetching video thumbnails... ")
    __fetch_thumbnails_obj(Video.objects.filter(icon_default__istartswith='http'), 'video', 'video_id')
    __fetch_thumbnails_obj(Video.objects.filter(icon_best__istartswith='http'), 'video', 'video_id')


def synchronize():
    if not __lock.acquire(blocking=False):
        # Synchronize already running in another thread
        log.info("Synchronize already running in another thread")
        return

    try:
        log.info("Running scheduled synchronization... ")

        # Sync subscribed playlists/channels
        log.info("Sync - checking videos")
        yt_api = YoutubeAPI.build_public()
        for subscription in Subscription.objects.all():
            __check_new_videos_sub(subscription, yt_api)
            __detect_deleted(subscription)

        log.info("Sync - checking for videos to download")
        downloader_process_all()

        log.info("Sync - fetching missing thumbnails")
        __fetch_thumbnails()

        log.info("Synchronization finished.")

    finally:
        __lock.release()


def synchronize_subscription(subscription: Subscription):
    __lock.acquire()
    try:
        log.info("Running synchronization for single subscription %d [%s]", subscription.id, subscription.name)
        yt_api = YoutubeAPI.build_public()

        log.info("Sync - checking videos")
        __check_new_videos_sub(subscription, yt_api)
        __detect_deleted(subscription)

        log.info("Sync - checking for videos to download")
        downloader_process_subscription(subscription)

        log.info("Sync - fetching missing thumbnails")
        __fetch_thumbnails()

        log.info("Synchronization finished for subscription %d [%s].", subscription.id, subscription.name)

    finally:
        __lock.release()


def schedule_synchronize_global():
    trigger = CronTrigger.from_crontab(settings.get('global', 'SynchronizationSchedule'))
    scheduler.instance.add_job(synchronize, trigger, max_instances=1, coalesce=True)


def schedule_synchronize_now():
    scheduler.instance.add_job(synchronize, max_instances=1, coalesce=True)


def schedule_synchronize_now_subscription(subscription: Subscription):
    scheduler.instance.add_job(synchronize_subscription, args=[subscription])