From 1022ce353c7910cfba8656c371f173f58fae32c8 Mon Sep 17 00:00:00 2001 From: Tiberiu Chibici Date: Sat, 11 Apr 2020 00:30:24 +0300 Subject: [PATCH] Major refactor of codebase. --- app/YtManagerApp/appmain.py | 7 +- .../management/download_manager.py | 255 +++++++++++++++ app/YtManagerApp/management/downloader.py | 111 ------- .../management/subscription_manager.py | 246 ++++++++++++++ app/YtManagerApp/management/video_manager.py | 124 ++++++++ .../management/video_provider_manager.py | 102 ++++++ app/YtManagerApp/management/videos.py | 57 ---- .../migrations/0013_auto_20200410_2037.py | 42 +++ .../migrations/0014_fix_video_provider.py | 28 ++ app/YtManagerApp/models.py | 300 ------------------ app/YtManagerApp/models/__init__.py | 6 + app/YtManagerApp/models/jobs.py | 44 +++ app/YtManagerApp/models/subscription.py | 58 ++++ .../models/subscription_folder.py | 76 +++++ app/YtManagerApp/models/video.py | 50 +++ app/YtManagerApp/models/video_order.py | 17 + app/YtManagerApp/models/video_provider.py | 6 + app/YtManagerApp/providers/video_provider.py | 110 +++++++ .../providers/ytapi_video_provider.py | 156 +++++++++ app/YtManagerApp/scheduler/jobs/__init__.py | 0 .../scheduler/jobs/delete_video_job.py | 47 --- .../scheduler/jobs/download_video_job.py | 136 -------- .../scheduler/jobs/synchronize_job.py | 184 ----------- app/YtManagerApp/services.py | 13 + app/YtManagerApp/utils/algorithms.py | 19 ++ app/YtManagerApp/utils/youtube.py | 49 --- app/YtManagerApp/views/actions.py | 4 +- app/YtManagerApp/views/auth.py | 2 +- app/YtManagerApp/views/first_time.py | 25 +- app/YtManagerApp/views/forms/settings.py | 16 +- app/YtManagerApp/views/index.py | 73 ++--- app/YtManagerApp/views/settings.py | 4 +- app/YtManagerApp/views/video.py | 4 +- 33 files changed, 1408 insertions(+), 963 deletions(-) create mode 100644 app/YtManagerApp/management/download_manager.py delete mode 100644 app/YtManagerApp/management/downloader.py create mode 100644 app/YtManagerApp/management/subscription_manager.py create mode 100644 app/YtManagerApp/management/video_manager.py create mode 100644 app/YtManagerApp/management/video_provider_manager.py delete mode 100644 app/YtManagerApp/management/videos.py create mode 100644 app/YtManagerApp/migrations/0013_auto_20200410_2037.py create mode 100644 app/YtManagerApp/migrations/0014_fix_video_provider.py delete mode 100644 app/YtManagerApp/models.py create mode 100644 app/YtManagerApp/models/__init__.py create mode 100644 app/YtManagerApp/models/jobs.py create mode 100644 app/YtManagerApp/models/subscription.py create mode 100644 app/YtManagerApp/models/subscription_folder.py create mode 100644 app/YtManagerApp/models/video.py create mode 100644 app/YtManagerApp/models/video_order.py create mode 100644 app/YtManagerApp/models/video_provider.py create mode 100644 app/YtManagerApp/providers/video_provider.py create mode 100644 app/YtManagerApp/providers/ytapi_video_provider.py delete mode 100644 app/YtManagerApp/scheduler/jobs/__init__.py delete mode 100644 app/YtManagerApp/scheduler/jobs/delete_video_job.py delete mode 100644 app/YtManagerApp/scheduler/jobs/download_video_job.py delete mode 100644 app/YtManagerApp/scheduler/jobs/synchronize_job.py delete mode 100644 app/YtManagerApp/utils/youtube.py diff --git a/app/YtManagerApp/appmain.py b/app/YtManagerApp/appmain.py index 215786c..54ea6cf 100644 --- a/app/YtManagerApp/appmain.py +++ b/app/YtManagerApp/appmain.py @@ -5,7 +5,6 @@ import sys from django.conf import settings as dj_settings -from YtManagerApp.scheduler.jobs.synchronize_job import SynchronizeJob from YtManagerApp.services import Services from django.db.utils import OperationalError @@ -35,9 +34,9 @@ def main(): __initialize_logger() try: - if Services.appConfig.initialized: - Services.scheduler.initialize() - SynchronizeJob.schedule_global_job() + if Services.appConfig().initialized: + Services.scheduler().initialize() + Services.subscriptionManager().schedule_global_synchronize_job() except OperationalError: # Settings table is not created when running migrate or makemigrations; # Just don't do anything in this case. diff --git a/app/YtManagerApp/management/download_manager.py b/app/YtManagerApp/management/download_manager.py new file mode 100644 index 0000000..0e1732c --- /dev/null +++ b/app/YtManagerApp/management/download_manager.py @@ -0,0 +1,255 @@ +import logging +import mimetypes +import os +import re +from string import Template +from threading import Lock +from urllib.parse import urljoin + +import PIL.Image +import PIL.ImageOps +import requests +import youtube_dl +from django.conf import settings as srv_settings + +from YtManagerApp.models import Subscription, Video +from YtManagerApp.models import VIDEO_ORDER_MAPPING +from YtManagerApp.providers.video_provider import VideoProvider +from YtManagerApp.scheduler.job import Job +from YtManagerApp.utils import first_non_null + +log = logging.getLogger('DownloadManager') + + +class DownloadVideoJob(Job): + """ + Downloads a video to the disk + """ + name = "DownloadVideoJob" + __lock = Lock() + + def __init__(self, job_execution, video: Video, attempt: int = 1): + super().__init__(job_execution) + self._video = video + self._attempt = attempt + self._log_youtube_dl = self.log.getChild('youtube_dl') + + def get_description(self): + ret = "Downloading video " + self._video.name + if self._attempt > 1: + ret += f" (attempt {self._attempt})" + return ret + + def run(self): + from YtManagerApp.services import Services + + # Issue: if multiple videos are downloaded at the same time, a race condition appears in the mkdirs() call that + # youtube-dl makes, which causes it to fail with the error 'Cannot create folder - file already exists'. + # For now, allow a single download instance. + self.__lock.acquire() + + try: + user = self._video.subscription.user + provider: VideoProvider = Services.videoProviderManager().get(self._video) + + max_attempts = user.preferences['max_download_attempts'] + + youtube_dl_params, output_path = self.__build_youtube_dl_params(self._video) + with youtube_dl.YoutubeDL(youtube_dl_params) as yt: + ret = yt.download([provider.get_video_url(self._video)]) + + self.log.info('Download finished with code %d', ret) + + if ret == 0: + self._video.downloaded_path = output_path + self._video.save() + self.log.info('Video %d [%s %s] downloaded successfully!', self._video.id, self._video.video_id, + self._video.name) + + # update size + self._video.downloaded_size = 0 + for file in self._video.get_files(): + self._video.downloaded_size += os.stat(file).st_size + self._video.save() + + elif self._attempt <= max_attempts: + self.log.warning('Re-enqueueing video (attempt %d/%d)', self._attempt, max_attempts) + Services.videoManager().download(self._video, self._attempt + 1) + + else: + self.log.error('Multiple attempts to download video %d [%s %s] failed!', self._video.id, + self._video.video_id, self._video.name) + self._video.downloaded_path = '' + self._video.save() + + finally: + self.__lock.release() + + def __build_youtube_dl_params(self, video: Video): + + sub = video.subscription + user = sub.user + + # resolve path + download_path = user.preferences['download_path'] + + template_dict = self.__build_template_dict(video) + output_pattern = Template(user.preferences['download_file_pattern']).safe_substitute(template_dict) + + output_path = os.path.join(download_path, output_pattern) + output_path = os.path.normpath(output_path) + + youtube_dl_params = { + 'logger': self._log_youtube_dl, + 'format': user.preferences['download_format'], + 'outtmpl': output_path, + 'writethumbnail': True, + 'writedescription': True, + 'writesubtitles': user.preferences['download_subtitles'], + 'writeautomaticsub': user.preferences['download_autogenerated_subtitles'], + 'allsubtitles': user.preferences['download_subtitles_all'], + 'merge_output_format': 'mp4', + 'postprocessors': [ + { + 'key': 'FFmpegMetadata' + }, + ] + } + + sub_langs = user.preferences['download_subtitles_langs'].split(',') + sub_langs = [i.strip() for i in sub_langs] + if len(sub_langs) > 0: + youtube_dl_params['subtitleslangs'] = sub_langs + + sub_format = user.preferences['download_subtitles_format'] + if len(sub_format) > 0: + youtube_dl_params['subtitlesformat'] = sub_format + + return youtube_dl_params, output_path + + def __build_template_dict(self, video: Video): + return { + 'channel': video.subscription.channel_name, + 'channel_id': video.subscription.channel_id, + 'playlist': video.subscription.name, + 'playlist_id': video.subscription.playlist_id, + 'playlist_index': "{:03d}".format(1 + video.playlist_index), + 'title': video.name, + 'id': video.video_id, + } + + def __get_valid_path(self, path: str): + """ + Normalizes string, converts to lowercase, removes non-alpha characters, removes forbidden characters. + """ + import unicodedata + value = unicodedata.normalize('NFKD', path).encode('ascii', 'ignore').decode('ascii') + value = re.sub('[:"*]', '', value).strip() + value = re.sub('[?<>|]', '#', value) + return value + + +class DownloadManager(object): + + def __init__(self): + pass + + def download_video(self, video: Video, attempt: int = 1): + from YtManagerApp.services import Services + Services.scheduler().add_job(DownloadVideoJob, args=[video, attempt]) + + def __get_subscription_config(self, sub: Subscription): + user = sub.user + + enabled = first_non_null(sub.auto_download, user.preferences['auto_download']) + global_limit = user.preferences['download_global_limit'] + limit = first_non_null(sub.download_limit, user.preferences['download_subscription_limit']) + order = first_non_null(sub.download_order, user.preferences['download_order']) + order = VIDEO_ORDER_MAPPING[order] + + return enabled, global_limit, limit, order + + def process_subscription(self, sub: Subscription): + from YtManagerApp.services import Services + + log.info('Processing subscription %d [%s %s]', sub.id, sub.playlist_id, sub.id) + + enabled, global_limit, limit, order = self.__get_subscription_config(sub) + log.info('Determined settings enabled=%s global_limit=%d limit=%d order="%s"', enabled, global_limit, limit, order) + + if enabled: + videos_to_download = Video.objects\ + .filter(subscription=sub, downloaded_path__isnull=True, watched=False)\ + .order_by(order) + + log.info('%d download candidates.', len(videos_to_download)) + + if global_limit > 0: + global_downloaded = Video.objects.filter(subscription__user=sub.user, downloaded_path__isnull=False).count() + allowed_count = max(global_limit - global_downloaded, 0) + videos_to_download = videos_to_download[0:allowed_count] + log.info('Global limit is set, can only download up to %d videos.', allowed_count) + + if limit > 0: + sub_downloaded = Video.objects.filter(subscription=sub, downloaded_path__isnull=False).count() + allowed_count = max(limit - sub_downloaded, 0) + videos_to_download = videos_to_download[0:allowed_count] + log.info('Limit is set, can only download up to %d videos.', allowed_count) + + # enqueue download + for video in videos_to_download: + log.info('Enqueuing video %d [%s %s] index=%d', video.id, video.video_id, video.name, video.playlist_index) + Services.videoManager().download(video) + + log.info('Finished processing subscription %d [%s %s]', sub.id, sub.playlist_id, sub.id) + + def process_all_subscriptions(self): + for subscription in Subscription.objects.all(): + self.process_subscription(subscription) + + def fetch_thumbnail(self, url, object_type, identifier, thumb_size): + + log.info('Fetching thumbnail url=%s object_type=%s identifier=%s', url, object_type, identifier) + + # Make request to obtain mime type + try: + response = requests.get(url, stream=True) + except requests.exceptions.RequestException as e: + log.error('Failed to fetch thumbnail %s. Error: %s', url, e) + return url + + ext = mimetypes.guess_extension(response.headers['Content-Type']) + + # Build file path + file_name = f"{identifier}{ext}" + abs_path_dir = os.path.join(srv_settings.MEDIA_ROOT, "thumbs", object_type) + abs_path = os.path.join(abs_path_dir, file_name) + abs_path_tmp = file_name + '.tmp' + + # Store image + try: + os.makedirs(abs_path_dir, exist_ok=True) + with open(abs_path_tmp, "wb") as f: + for chunk in response.iter_content(chunk_size=1024): + if chunk: + f.write(chunk) + + # Resize and crop to thumbnail size + image = PIL.Image.open(abs_path_tmp) + image = PIL.ImageOps.fit(image, thumb_size) + image.save(abs_path) + image.close() + + # Delete temp file + os.unlink(abs_path_tmp) + + except requests.exceptions.RequestException as e: + log.error('Error while downloading stream for thumbnail %s. Error: %s', url, e) + return url + except OSError as e: + log.error('Error while writing to file %s for thumbnail %s. Error: %s', abs_path, url, e) + return url + + # Return + media_url = urljoin(srv_settings.MEDIA_URL, f"thumbs/{object_type}/{file_name}") + return media_url diff --git a/app/YtManagerApp/management/downloader.py b/app/YtManagerApp/management/downloader.py deleted file mode 100644 index 0b4ce75..0000000 --- a/app/YtManagerApp/management/downloader.py +++ /dev/null @@ -1,111 +0,0 @@ -from YtManagerApp.scheduler.jobs.download_video_job import DownloadVideoJob -from YtManagerApp.models import Video, Subscription, VIDEO_ORDER_MAPPING -from YtManagerApp.utils import first_non_null -from django.conf import settings as srv_settings -import logging -import requests -import mimetypes -import os -import PIL.Image -import PIL.ImageOps -from urllib.parse import urljoin - -log = logging.getLogger('downloader') - - -def __get_subscription_config(sub: Subscription): - user = sub.user - - enabled = first_non_null(sub.auto_download, user.preferences['auto_download']) - global_limit = user.preferences['download_global_limit'] - limit = first_non_null(sub.download_limit, user.preferences['download_subscription_limit']) - order = first_non_null(sub.download_order, user.preferences['download_order']) - order = VIDEO_ORDER_MAPPING[order] - - return enabled, global_limit, limit, order - - -def downloader_process_subscription(sub: Subscription): - log.info('Processing subscription %d [%s %s]', sub.id, sub.playlist_id, sub.id) - - enabled, global_limit, limit, order = __get_subscription_config(sub) - log.info('Determined settings enabled=%s global_limit=%d limit=%d order="%s"', enabled, global_limit, limit, order) - - if enabled: - videos_to_download = Video.objects\ - .filter(subscription=sub, downloaded_path__isnull=True, watched=False)\ - .order_by(order) - - log.info('%d download candidates.', len(videos_to_download)) - - if global_limit > 0: - global_downloaded = Video.objects.filter(subscription__user=sub.user, downloaded_path__isnull=False).count() - allowed_count = max(global_limit - global_downloaded, 0) - videos_to_download = videos_to_download[0:allowed_count] - log.info('Global limit is set, can only download up to %d videos.', allowed_count) - - if limit > 0: - sub_downloaded = Video.objects.filter(subscription=sub, downloaded_path__isnull=False).count() - allowed_count = max(limit - sub_downloaded, 0) - videos_to_download = videos_to_download[0:allowed_count] - log.info('Limit is set, can only download up to %d videos.', allowed_count) - - # enqueue download - for video in videos_to_download: - log.info('Enqueuing video %d [%s %s] index=%d', video.id, video.video_id, video.name, video.playlist_index) - DownloadVideoJob.schedule(video) - - log.info('Finished processing subscription %d [%s %s]', sub.id, sub.playlist_id, sub.id) - - -def downloader_process_all(): - for subscription in Subscription.objects.all(): - downloader_process_subscription(subscription) - - -def fetch_thumbnail(url, object_type, identifier, thumb_size): - - log.info('Fetching thumbnail url=%s object_type=%s identifier=%s', url, object_type, identifier) - - # Make request to obtain mime type - try: - response = requests.get(url, stream=True) - except requests.exceptions.RequestException as e: - log.error('Failed to fetch thumbnail %s. Error: %s', url, e) - return url - - ext = mimetypes.guess_extension(response.headers['Content-Type']) - - # Build file path - file_name = f"{identifier}{ext}" - abs_path_dir = os.path.join(srv_settings.MEDIA_ROOT, "thumbs", object_type) - abs_path = os.path.join(abs_path_dir, file_name) - abs_path_tmp = file_name + '.tmp' - - # Store image - try: - os.makedirs(abs_path_dir, exist_ok=True) - with open(abs_path_tmp, "wb") as f: - for chunk in response.iter_content(chunk_size=1024): - if chunk: - f.write(chunk) - - # Resize and crop to thumbnail size - image = PIL.Image.open(abs_path_tmp) - image = PIL.ImageOps.fit(image, thumb_size) - image.save(abs_path) - image.close() - - # Delete temp file - os.unlink(abs_path_tmp) - - except requests.exceptions.RequestException as e: - log.error('Error while downloading stream for thumbnail %s. Error: %s', url, e) - return url - except OSError as e: - log.error('Error while writing to file %s for thumbnail %s. Error: %s', abs_path, url, e) - return url - - # Return - media_url = urljoin(srv_settings.MEDIA_URL, f"thumbs/{object_type}/{file_name}") - return media_url diff --git a/app/YtManagerApp/management/subscription_manager.py b/app/YtManagerApp/management/subscription_manager.py new file mode 100644 index 0000000..841549d --- /dev/null +++ b/app/YtManagerApp/management/subscription_manager.py @@ -0,0 +1,246 @@ +import errno +import itertools +import mimetypes +import os +from threading import Lock +from typing import Optional, List, Union + +from apscheduler.triggers.cron import CronTrigger +from django.conf import settings +from django.db.models import Max + +from YtManagerApp.models import * +from YtManagerApp.providers.video_provider import VideoProvider, InvalidURLError +from YtManagerApp.scheduler.job import Job +from YtManagerApp.utils.algorithms import group_by + +_ENABLE_UPDATE_STATS = True + + +class SynchronizeJob(Job): + name = "SynchronizeJob" + __lock = Lock() + running = False + __global_sync_job = None + + def __init__(self, job_execution, subscription: Optional[Subscription] = None): + super().__init__(job_execution) + self.__subscription: Optional[Subscription] = subscription + self.__new_vids: List[Video] = [] + + def get_description(self): + if self.__subscription is not None: + return "Running synchronization for subscription " + self.__subscription.name + return "Running synchronization..." + + def get_subscription_list(self): + if self.__subscription is not None: + return [self.__subscription] + return Subscription.objects.all() + + def get_videos_list(self, subs): + return Video.objects.filter(subscription__in=subs) + + def run(self): + from YtManagerApp.services import Services + + self.__lock.acquire(blocking=True) + SynchronizeJob.running = True + try: + self.log.info(self.get_description()) + + # Build list of work items + work_subs = self.get_subscription_list() + work_vids = self.get_videos_list(work_subs) + + self.set_total_steps(len(work_subs) + len(work_vids)) + + # Remove the 'new' flag + work_vids.update(new=False) + + # Process subscriptions + for sub in work_subs: + self.progress_advance(1, "Synchronizing subscription " + sub.name) + self.check_new_videos(sub) + self.fetch_missing_thumbnails(sub) + + # Add new videos to progress calculation + self.set_total_steps(len(work_subs) + len(work_vids) + len(self.__new_vids)) + + # Group videos by provider + all_videos = itertools.chain(work_vids, self.__new_vids) + all_videos_by_provider = group_by(all_videos, lambda x: x.subscription.provider_id) + + for provider_id, videos in all_videos_by_provider.items(): + provider: VideoProvider = Services.videoProviderManager().get(provider_id) + if _ENABLE_UPDATE_STATS: + provider.update_videos(videos, update_statistics=True) + + for video in videos: + self.progress_advance(1, "Updating video " + video.name) + self.check_video_deleted(video) + self.fetch_missing_thumbnails(video) + + # Start downloading videos + for sub in work_subs: + Services.downloadManager().process_subscription(sub) + + finally: + SynchronizeJob.running = False + self.__lock.release() + + def check_new_videos(self, sub: Subscription): + from YtManagerApp.services import Services + provider: VideoProvider = Services.videoProviderManager().get(sub) + + playlist_videos = provider.fetch_videos(sub) + if sub.rewrite_playlist_indices: + playlist_videos = sorted(playlist_videos, key=lambda x: x.publish_date) + else: + playlist_videos = sorted(playlist_videos, key=lambda x: x.playlist_index) + + for item in playlist_videos: + results = Video.objects.filter(video_id=item.video_id, subscription=sub) + + if not results.exists(): + self.log.info('New video for subscription %s: %s %s"', sub, item.video_id, item.name) + + # fix playlist index if necessary + if sub.rewrite_playlist_indices or Video.objects.filter(subscription=sub, + playlist_index=item.playlist_index).exists(): + highest = Video.objects.filter(subscription=sub).aggregate(Max('playlist_index'))[ + 'playlist_index__max'] + item.playlist_index = 1 + (highest or -1) + + item.save() + self.__new_vids.append(item) + + def fetch_missing_thumbnails(self, obj: Union[Subscription, Video]): + from YtManagerApp.services import Services + if obj.thumbnail.startswith("http"): + if isinstance(obj, Subscription): + obj.thumbnail = Services.downloadManager().fetch_thumbnail(obj.thumbnail, 'sub', obj.playlist_id, + settings.THUMBNAIL_SIZE_SUBSCRIPTION) + elif isinstance(obj, Video): + obj.thumbnail = Services.downloadManager().fetch_thumbnail(obj.thumbnail, 'video', obj.video_id, + settings.THUMBNAIL_SIZE_VIDEO) + obj.save() + + def check_video_deleted(self, video: Video): + if video.downloaded_path is not None: + files = [] + try: + files = list(video.get_files()) + except OSError as e: + if e.errno != errno.ENOENT: + self.log.error("Could not access path %s. Error: %s", video.downloaded_path, e) + self.usr_err(f"Could not access path {video.downloaded_path}: {e}", suppress_notification=True) + return + + # Try to find a valid video file + found_video = False + for file in files: + mime, _ = mimetypes.guess_type(file) + if mime is not None and mime.startswith("video"): + found_video = True + + # Video not found, we can safely assume that the video was deleted. + if not found_video: + self.log.info("Video %d was deleted! [%s %s]", video.id, video.video_id, video.name) + # Clean up + for file in files: + try: + os.unlink(file) + except OSError as e: + self.log.error("Could not delete redundant file %s. Error: %s", file, e) + self.usr_err(f"Could not delete redundant file {file}: {e}", suppress_notification=True) + video.downloaded_path = None + + # Mark watched? + user = video.subscription.user + if user.preferences['mark_deleted_as_watched']: + video.watched = True + + video.save() + + def update_video_stats(self, video: Video, yt_video): + if yt_video.n_likes is not None \ + and yt_video.n_dislikes is not None \ + and yt_video.n_likes + yt_video.n_dislikes > 0: + video.rating = yt_video.n_likes / (yt_video.n_likes + yt_video.n_dislikes) + + video.views = yt_video.n_views + video.save() + + +class SubscriptionImporterJob(Job): + def __init__(self, job_execution, urls: List[str], + parent_folder: SubscriptionFolder, + auto_download: bool, + download_limit: int, + download_order: str, + automatically_delete_watched: bool): + + super().__init__(job_execution) + self._urls = urls + self._parent_folder = parent_folder + self._auto_download = auto_download + self._download_limit = download_limit + self._download_order = download_order + self._automatically_delete_watched = automatically_delete_watched + + def get_description(self): + return f"Importing {len(self._urls)} subscriptions..." + + def run(self): + from YtManagerApp.services import Services + self.set_total_steps(len(self._urls)) + for url in self._urls: + try: + self.progress_advance(progress_msg=url) + sub: Subscription = Services.videoProviderManager().fetch_subscription(url) + sub.parent_folder = self._parent_folder + sub.auto_download = self._auto_download + sub.download_limit = self._download_limit + sub.download_order = self._download_order + sub.automatically_delete_watched = self._automatically_delete_watched + sub.save() + except InvalidURLError as e: + self.log.error("Error importing URL %s: %s", url, e) + except ValueError as e: + self.log.error("Error importing URL %s: %s", url, e) + + +class SubscriptionManager(object): + def __init__(self): + self.__global_sync_job = None + + def synchronize(self, sub: Subscription): + from YtManagerApp.services import Services + Services.scheduler().add_job(SynchronizeJob, args=[sub]) + + def synchronize_all(self): + from YtManagerApp.services import Services + Services.scheduler().add_job(SynchronizeJob, max_instances=1, coalesce=True) + + def schedule_global_synchronize_job(self): + from YtManagerApp.services import Services + trigger = CronTrigger.from_crontab(Services.appConfig().sync_schedule) + + if self.__global_sync_job is None: + trigger = CronTrigger.from_crontab(Services.appConfig().sync_schedule) + SynchronizeJob.__global_sync_job = Services.scheduler().add_job(SynchronizeJob, trigger, max_instances=1, + coalesce=True) + + else: + self.__global_sync_job.reschedule(trigger, max_instances=1, coalesce=True) + + def import_multiple(self, urls: List[str], + parent_folder: SubscriptionFolder, + auto_download: bool, + download_limit: int, + download_order: str, + automatically_delete_watched: bool): + from YtManagerApp.services import Services + Services.scheduler().add_job(SubscriptionImporterJob, args=[urls, parent_folder, auto_download, download_limit, + download_order, automatically_delete_watched]) diff --git a/app/YtManagerApp/management/video_manager.py b/app/YtManagerApp/management/video_manager.py new file mode 100644 index 0000000..3e0d63f --- /dev/null +++ b/app/YtManagerApp/management/video_manager.py @@ -0,0 +1,124 @@ +import os +import re +from typing import Optional + +from django.contrib.auth.models import User +from django.db.models import Q + +from YtManagerApp.models import Subscription, Video, SubscriptionFolder +from YtManagerApp.scheduler.job import Job + + +class DeleteVideoJob(Job): + """ + Deletes a video's files. + """ + name = "DeleteVideoJob" + + def __init__(self, job_execution, video: Video): + super().__init__(job_execution) + self._video = video + + def get_description(self): + return f"Deleting video {self._video}" + + def run(self): + count = 0 + + try: + for file in self._video.get_files(): + self.log.info("Deleting file %s", file) + count += 1 + try: + os.unlink(file) + except OSError as e: + self.log.error("Failed to delete file %s: Error: %s", file, e) + + except OSError as e: + self.log.error("Failed to delete video %d [%s %s]. Error: %s", self._video.id, + self._video.video_id, self._video.name, e) + + self._video.downloaded_path = None + self._video.save() + + self.log.info('Deleted video %d successfully! (%d files) [%s %s]', self._video.id, count, + self._video.video_id, self._video.name) + + +class VideoManager(object): + def __init__(self): + pass + + def get_videos(self, + user: User, + sort_order: Optional[str], + query: Optional[str] = None, + subscription_id: Optional[int] = None, + folder_id: Optional[int] = None, + only_watched: Optional[bool] = None, + only_downloaded: Optional[bool] = None, + ): + + filter_args = [] + filter_kwargs = { + 'subscription__user': user + } + + # Process query string - basically, we break it down into words, + # and then search for the given text in the name, description, uploader name and subscription name + if query is not None: + for match in re.finditer(r'\w+', query): + word = match[0] + filter_args.append(Q(name__icontains=word) + | Q(description__icontains=word) + | Q(uploader_name__icontains=word) + | Q(subscription__name__icontains=word)) + + # Subscription id + if subscription_id is not None: + filter_kwargs['subscription_id'] = subscription_id + + # Folder id + if folder_id is not None: + # Visit function - returns only the subscription IDs + def visit(node): + if isinstance(node, Subscription): + return node.id + return None + + filter_kwargs['subscription_id__in'] = SubscriptionFolder.traverse(folder_id, user, visit) + + # Only watched + if only_watched is not None: + filter_kwargs['watched'] = only_watched + + # Only downloaded + # - not downloaded (False) -> is null (True) + # - downloaded (True) -> is not null (False) + if only_downloaded is not None: + filter_kwargs['downloaded_path__isnull'] = not only_downloaded + + return Video.objects.filter(*filter_args, **filter_kwargs).order_by(sort_order) + + def delete_files(self, video: Video): + from YtManagerApp.services import Services + Services.scheduler().add_job(DeleteVideoJob, args=[video]) + + def download(self, video: Video, attempt: int = 1): + from YtManagerApp.services import Services + Services.downloadManager().download_video(video, attempt) + + def mark_watched(self, video: Video): + from YtManagerApp.services import Services + video.watched = True + video.save() + if video.downloaded_path is not None: + if Services.appConfig().for_sub(video.subscription, 'automatically_delete_watched'): + self.delete_files(video) + Services.subscriptionManager().synchronize(video.subscription) + + def mark_unwatched(self, video: Video): + from YtManagerApp.services import Services + video.watched = False + video.save() + Services.subscriptionManager().synchronize(video.subscription) diff --git a/app/YtManagerApp/management/video_provider_manager.py b/app/YtManagerApp/management/video_provider_manager.py new file mode 100644 index 0000000..6db28ae --- /dev/null +++ b/app/YtManagerApp/management/video_provider_manager.py @@ -0,0 +1,102 @@ +import logging +from typing import List, Dict, Union + +from YtManagerApp.models import VideoProviderConfig, Video, Subscription +from YtManagerApp.providers.video_provider import VideoProvider, InvalidURLError +import json + +log = logging.getLogger("VideoProviderManager") + + +class VideoProviderManager(object): + def __init__(self, registered_providers: List[VideoProvider]): + self._registered_providers: Dict[str, VideoProvider] = {} + self._configured_providers: Dict[str, VideoProvider] = {} + self._pending_configs: Dict[str, VideoProviderConfig] = {} + for rp in registered_providers: + self.register_provider(rp) + self._load() + + def register_provider(self, provider: VideoProvider) -> None: + """ + Registers a video provider + :param provider: Video provider + """ + # avoid duplicates + if provider.name in self._registered_providers: + log.error(f"Duplicate video provider {provider.name}") + return + + # register + self._registered_providers[provider.name] = provider + log.info(f"Registered video provider {provider.name}") + + # load configuration (if any) + if provider.name in self._pending_configs: + self._configure(provider, self._pending_configs[provider.name]) + del self._pending_configs[provider.name] + + def _load(self) -> None: + # Loads configuration from database + for config in VideoProviderConfig.objects.all(): + provider = self._registered_providers.get(config.provider_id) + + # provider not yet registered, keep it in the pending list + if provider is None: + self._pending_configs[config.provider_id] = config + log.warning(f"Provider {config.provider_id} not registered!") + continue + + # configure + self._configure(provider, config) + + def _configure(self, provider, config): + settings = json.loads(config.settings) + provider.configure(settings) + log.info(f"Configured video provider {provider.name}") + self._configured_providers[provider.name] = provider + + def get(self, item: Union[str, Subscription, Video]): + """ + Gets provider for given item (subscription or video). + :param item: Provider ID, or subscription, or video + :return: Provider + """ + if isinstance(item, str): + return self._registered_providers[item] + elif isinstance(item, Video): + return self._registered_providers[item.subscription.provider_id] + elif isinstance(item, Subscription): + return self._registered_providers[item.provider_id] + return None + + def validate_subscription_url(self, url: str): + """ + Validates given URL using all registered and configured provider. + :param url: + :return: + """ + for provider in self._configured_providers.values(): + try: + provider.validate_subscription_url(url) + return + except InvalidURLError: + pass + + raise InvalidURLError("The given URL is not valid for any of the supported sites!") + + def fetch_subscription(self, url: str) -> Subscription: + """ + Validates given URL using all registered and configured provider. + :param url: + :return: + """ + for provider in self._configured_providers.values(): + try: + provider.validate_subscription_url(url) + # Found the right provider + return provider.fetch_subscription(url) + except InvalidURLError: + pass + + raise InvalidURLError("The given URL is not valid for any of the supported sites!") diff --git a/app/YtManagerApp/management/videos.py b/app/YtManagerApp/management/videos.py deleted file mode 100644 index 0e9bd84..0000000 --- a/app/YtManagerApp/management/videos.py +++ /dev/null @@ -1,57 +0,0 @@ -import re -from typing import Optional - -from django.contrib.auth.models import User -from django.db.models import Q - -from YtManagerApp.models import Subscription, Video, SubscriptionFolder - - -def get_videos(user: User, - sort_order: Optional[str], - query: Optional[str] = None, - subscription_id: Optional[int] = None, - folder_id: Optional[int] = None, - only_watched: Optional[bool] = None, - only_downloaded: Optional[bool] = None, - ): - - filter_args = [] - filter_kwargs = { - 'subscription__user': user - } - - # Process query string - basically, we break it down into words, - # and then search for the given text in the name, description, uploader name and subscription name - if query is not None: - for match in re.finditer(r'\w+', query): - word = match[0] - filter_args.append(Q(name__icontains=word) - | Q(description__icontains=word) - | Q(uploader_name__icontains=word) - | Q(subscription__name__icontains=word)) - - # Subscription id - if subscription_id is not None: - filter_kwargs['subscription_id'] = subscription_id - - # Folder id - if folder_id is not None: - # Visit function - returns only the subscription IDs - def visit(node): - if isinstance(node, Subscription): - return node.id - return None - filter_kwargs['subscription_id__in'] = SubscriptionFolder.traverse(folder_id, user, visit) - - # Only watched - if only_watched is not None: - filter_kwargs['watched'] = only_watched - - # Only downloaded - # - not downloaded (False) -> is null (True) - # - downloaded (True) -> is not null (False) - if only_downloaded is not None: - filter_kwargs['downloaded_path__isnull'] = not only_downloaded - - return Video.objects.filter(*filter_args, **filter_kwargs).order_by(sort_order) diff --git a/app/YtManagerApp/migrations/0013_auto_20200410_2037.py b/app/YtManagerApp/migrations/0013_auto_20200410_2037.py new file mode 100644 index 0000000..dfc46d1 --- /dev/null +++ b/app/YtManagerApp/migrations/0013_auto_20200410_2037.py @@ -0,0 +1,42 @@ +# Generated by Django 3.0.4 on 2020-04-10 20:37 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('YtManagerApp', '0012_auto_20190819_1615'), + ] + + operations = [ + migrations.CreateModel( + name='VideoProviderConfig', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('provider_id', models.CharField(help_text='Provider ID', max_length=128, unique=True)), + ('settings', models.TextField(help_text='Video provider settings')), + ], + ), + migrations.AddField( + model_name='subscription', + name='provider_id', + field=models.CharField(default='YtAPI', max_length=128), + preserve_default=False, + ), + migrations.AddField( + model_name='video', + name='downloaded_size', + field=models.IntegerField(blank=True, null=True), + ), + migrations.AddField( + model_name='video', + name='last_updated_date', + field=models.DateTimeField(auto_now=True), + ), + migrations.AlterField( + model_name='video', + name='name', + field=models.TextField(), + ), + ] diff --git a/app/YtManagerApp/migrations/0014_fix_video_provider.py b/app/YtManagerApp/migrations/0014_fix_video_provider.py new file mode 100644 index 0000000..da0a359 --- /dev/null +++ b/app/YtManagerApp/migrations/0014_fix_video_provider.py @@ -0,0 +1,28 @@ +# Generated by Django 3.0.4 on 2020-04-10 20:37 + +from django.db import migrations, models +import json + + +def fix_video_provider(apps, schema_editor): + globalPrefs = apps.get_model('dynamic_preferences', 'GlobalPreferenceModel') + api_key_entries = globalPrefs.objects.filter(name='youtube_api_key') + if len(api_key_entries) < 1: + return + + videoProviderConfig = apps.get_model('YtManagerApp', 'VideoProviderConfig') + ytApiProvider = videoProviderConfig(provider_id='YtAPI', settings=json.dumps({ + 'api_key': api_key_entries[0].raw_value + })) + ytApiProvider.save() + + +class Migration(migrations.Migration): + + dependencies = [ + ('YtManagerApp', '0013_auto_20200410_2037'), + ] + + operations = [ + migrations.RunPython(fix_video_provider) + ] diff --git a/app/YtManagerApp/models.py b/app/YtManagerApp/models.py deleted file mode 100644 index 227efda..0000000 --- a/app/YtManagerApp/models.py +++ /dev/null @@ -1,300 +0,0 @@ -import logging -import mimetypes -import os -from typing import Callable, Union, Any, Optional - -from django.contrib.auth.models import User -from django.db import models -from django.db.models.functions import Lower - -from YtManagerApp.utils import youtube - -# help_text = user shown text -# verbose_name = user shown name -# null = nullable, blank = user is allowed to set value to empty -VIDEO_ORDER_CHOICES = [ - ('newest', 'Newest'), - ('oldest', 'Oldest'), - ('playlist', 'Playlist order'), - ('playlist_reverse', 'Reverse playlist order'), - ('popularity', 'Popularity'), - ('rating', 'Top rated'), -] - -VIDEO_ORDER_MAPPING = { - 'newest': '-publish_date', - 'oldest': 'publish_date', - 'playlist': 'playlist_index', - 'playlist_reverse': '-playlist_index', - 'popularity': '-views', - 'rating': '-rating' -} - - -class SubscriptionFolder(models.Model): - name = models.CharField(null=False, max_length=250) - parent = models.ForeignKey('self', on_delete=models.CASCADE, null=True, blank=True) - user = models.ForeignKey(User, on_delete=models.CASCADE, null=False, blank=False) - - class Meta: - ordering = [Lower('parent__name'), Lower('name')] - - def __str__(self): - s = "" - current = self - while current is not None: - s = current.name + " > " + s - current = current.parent - return s[:-3] - - def __repr__(self): - return f'folder {self.id}, name="{self.name}"' - - def delete_folder(self, keep_subscriptions: bool): - if keep_subscriptions: - - def visit(node: Union["SubscriptionFolder", "Subscription"]): - if isinstance(node, Subscription): - node.parent_folder = None - node.save() - - SubscriptionFolder.traverse(self.id, self.user, visit) - - self.delete() - - @staticmethod - def traverse(root_folder_id: Optional[int], - user: User, - visit_func: Callable[[Union["SubscriptionFolder", "Subscription"]], Any]): - - data_collected = [] - - def collect(data): - if data is not None: - data_collected.append(data) - - # Visit root - if root_folder_id is not None: - root_folder = SubscriptionFolder.objects.get(id=root_folder_id) - collect(visit_func(root_folder)) - - queue = [root_folder_id] - visited = [] - - while len(queue) > 0: - folder_id = queue.pop() - - if folder_id in visited: - logging.error('Found folder tree cycle for folder id %d.', folder_id) - continue - visited.append(folder_id) - - for folder in SubscriptionFolder.objects.filter(parent_id=folder_id, user=user).order_by(Lower('name')): - collect(visit_func(folder)) - queue.append(folder.id) - - for subscription in Subscription.objects.filter(parent_folder_id=folder_id, user=user).order_by(Lower('name')): - collect(visit_func(subscription)) - - return data_collected - - -class Subscription(models.Model): - name = models.CharField(null=False, max_length=1024) - parent_folder = models.ForeignKey(SubscriptionFolder, on_delete=models.CASCADE, null=True, blank=True) - playlist_id = models.CharField(null=False, max_length=128) - description = models.TextField() - channel_id = models.CharField(max_length=128) - channel_name = models.CharField(max_length=1024) - thumbnail = models.CharField(max_length=1024) - user = models.ForeignKey(User, on_delete=models.CASCADE) - # youtube adds videos to the 'Uploads' playlist at the top instead of the bottom - rewrite_playlist_indices = models.BooleanField(null=False, default=False) - - # overrides - auto_download = models.BooleanField(null=True, blank=True) - download_limit = models.IntegerField(null=True, blank=True) - download_order = models.CharField( - null=True, blank=True, - max_length=128, - choices=VIDEO_ORDER_CHOICES) - automatically_delete_watched = models.BooleanField(null=True, blank=True) - - def __str__(self): - return self.name - - def __repr__(self): - return f'subscription {self.id}, name="{self.name}", playlist_id="{self.playlist_id}"' - - def fill_from_playlist(self, info_playlist: youtube.Playlist): - self.name = info_playlist.title - self.playlist_id = info_playlist.id - self.description = info_playlist.description - self.channel_id = info_playlist.channel_id - self.channel_name = info_playlist.channel_title - self.thumbnail = youtube.best_thumbnail(info_playlist).url - - def copy_from_channel(self, info_channel: youtube.Channel): - # No point in storing info about the 'uploads from X' playlist - self.name = info_channel.title - self.playlist_id = info_channel.uploads_playlist.id - self.description = info_channel.description - self.channel_id = info_channel.id - self.channel_name = info_channel.title - self.thumbnail = youtube.best_thumbnail(info_channel).url - self.rewrite_playlist_indices = True - - def fetch_from_url(self, url, yt_api: youtube.YoutubeAPI): - url_parsed = yt_api.parse_url(url) - if 'playlist' in url_parsed: - info_playlist = yt_api.playlist(url=url) - if info_playlist is None: - raise ValueError('Invalid playlist ID!') - - self.fill_from_playlist(info_playlist) - else: - info_channel = yt_api.channel(url=url) - if info_channel is None: - raise ValueError('Cannot find channel!') - - self.copy_from_channel(info_channel) - - def delete_subscription(self, keep_downloaded_videos: bool): - self.delete() - - -class Video(models.Model): - video_id = models.CharField(null=False, max_length=12) - name = models.TextField(null=False) - description = models.TextField() - watched = models.BooleanField(default=False, null=False) - new = models.BooleanField(default=True, null=False) - downloaded_path = models.TextField(null=True, blank=True) - subscription = models.ForeignKey(Subscription, on_delete=models.CASCADE) - playlist_index = models.IntegerField(null=False) - publish_date = models.DateTimeField(null=False) - thumbnail = models.TextField() - uploader_name = models.CharField(null=False, max_length=255) - views = models.IntegerField(null=False, default=0) - rating = models.FloatField(null=False, default=0.5) - - @staticmethod - def create(playlist_item: youtube.PlaylistItem, subscription: Subscription): - video = Video() - video.video_id = playlist_item.resource_video_id - video.name = playlist_item.title - video.description = playlist_item.description - video.watched = False - video.new = True - video.downloaded_path = None - video.subscription = subscription - video.playlist_index = playlist_item.position - video.publish_date = playlist_item.published_at - video.thumbnail = youtube.best_thumbnail(playlist_item).url - video.save() - return video - - def mark_watched(self): - self.watched = True - self.save() - if self.downloaded_path is not None: - from YtManagerApp.management.appconfig import appconfig - from YtManagerApp.scheduler.jobs import DeleteVideoJob - from YtManagerApp.scheduler.jobs import SynchronizeJob - - if appconfig.for_sub(self.subscription, 'automatically_delete_watched'): - DeleteVideoJob.schedule(self) - SynchronizeJob.schedule_now_for_subscription(self.subscription) - - def mark_unwatched(self): - self.watched = False - self.save() - from YtManagerApp.scheduler.jobs.synchronize_job import SynchronizeJob - SynchronizeJob.schedule_now_for_subscription(self.subscription) - - def get_files(self): - if self.downloaded_path is not None: - directory, file_pattern = os.path.split(self.downloaded_path) - for file in os.listdir(directory): - if file.startswith(file_pattern): - yield os.path.join(directory, file) - - def find_video(self): - """ - Finds the video file from the downloaded files, and - returns - :return: Tuple containing file path and mime type - """ - for file in self.get_files(): - mime, _ = mimetypes.guess_type(file) - if mime is not None and mime.startswith('video/'): - return file, mime - - return None, None - - def delete_files(self): - if self.downloaded_path is not None: - from YtManagerApp.scheduler.jobs import DeleteVideoJob - from YtManagerApp.management.appconfig import appconfig - from YtManagerApp.scheduler.jobs import SynchronizeJob - - DeleteVideoJob.schedule(self) - - # Mark watched? - if self.subscription.user.preferences['mark_deleted_as_watched']: - self.watched = True - SynchronizeJob.schedule_now_for_subscription(self.subscription) - - def download(self): - if not self.downloaded_path: - from YtManagerApp.scheduler.jobs.download_video_job import DownloadVideoJob - DownloadVideoJob.schedule(self) - - def __str__(self): - return self.name - - def __repr__(self): - return f'video {self.id}, video_id="{self.video_id}"' - - -JOB_STATES = [ - ('running', 0), - ('finished', 1), - ('failed', 2), - ('interrupted', 3), -] - -JOB_STATES_MAP = { - 'running': 0, - 'finished': 1, - 'failed': 2, - 'interrupted': 3, -} - -JOB_MESSAGE_LEVELS = [ - ('normal', 0), - ('warning', 1), - ('error', 2), -] -JOB_MESSAGE_LEVELS_MAP = { - 'normal': 0, - 'warning': 1, - 'error': 2, -} - - -class JobExecution(models.Model): - start_date = models.DateTimeField(auto_now=True, null=False) - end_date = models.DateTimeField(null=True) - user = models.ForeignKey(User, on_delete=models.CASCADE, null=True) - description = models.CharField(max_length=250, null=False, default="") - status = models.IntegerField(choices=JOB_STATES, null=False, default=0) - - -class JobMessage(models.Model): - timestamp = models.DateTimeField(auto_now=True, null=False) - job = models.ForeignKey(JobExecution, null=False, on_delete=models.CASCADE) - progress = models.FloatField(null=True) - message = models.CharField(max_length=1024, null=False, default="") - level = models.IntegerField(choices=JOB_MESSAGE_LEVELS, null=False, default=0) - suppress_notification = models.BooleanField(null=False, default=False) diff --git a/app/YtManagerApp/models/__init__.py b/app/YtManagerApp/models/__init__.py new file mode 100644 index 0000000..7f4a5a3 --- /dev/null +++ b/app/YtManagerApp/models/__init__.py @@ -0,0 +1,6 @@ +from .video_order import VIDEO_ORDER_CHOICES, VIDEO_ORDER_MAPPING +from .subscription_folder import SubscriptionFolder +from .subscription import Subscription +from .video import Video +from .jobs import JobExecution, JobMessage, JOB_STATES_MAP, JOB_MESSAGE_LEVELS_MAP +from .video_provider import VideoProviderConfig diff --git a/app/YtManagerApp/models/jobs.py b/app/YtManagerApp/models/jobs.py new file mode 100644 index 0000000..be178d3 --- /dev/null +++ b/app/YtManagerApp/models/jobs.py @@ -0,0 +1,44 @@ +from django.contrib.auth.models import User +from django.db import models + +JOB_STATES = [ + ('running', 0), + ('finished', 1), + ('failed', 2), + ('interrupted', 3), +] + +JOB_STATES_MAP = { + 'running': 0, + 'finished': 1, + 'failed': 2, + 'interrupted': 3, +} + +JOB_MESSAGE_LEVELS = [ + ('normal', 0), + ('warning', 1), + ('error', 2), +] +JOB_MESSAGE_LEVELS_MAP = { + 'normal': 0, + 'warning': 1, + 'error': 2, +} + + +class JobExecution(models.Model): + start_date = models.DateTimeField(auto_now=True, null=False) + end_date = models.DateTimeField(null=True) + user = models.ForeignKey(User, on_delete=models.CASCADE, null=True) + description = models.CharField(max_length=250, null=False, default="") + status = models.IntegerField(choices=JOB_STATES, null=False, default=0) + + +class JobMessage(models.Model): + timestamp = models.DateTimeField(auto_now=True, null=False) + job = models.ForeignKey(JobExecution, null=False, on_delete=models.CASCADE) + progress = models.FloatField(null=True) + message = models.CharField(max_length=1024, null=False, default="") + level = models.IntegerField(choices=JOB_MESSAGE_LEVELS, null=False, default=0) + suppress_notification = models.BooleanField(null=False, default=False) diff --git a/app/YtManagerApp/models/subscription.py b/app/YtManagerApp/models/subscription.py new file mode 100644 index 0000000..5ad1251 --- /dev/null +++ b/app/YtManagerApp/models/subscription.py @@ -0,0 +1,58 @@ +from django.contrib.auth.models import User +from django.db import models + +from .subscription_folder import SubscriptionFolder +from .video_order import VIDEO_ORDER_CHOICES + + +class Subscription(models.Model): + name = models.CharField(null=False, max_length=1024) + parent_folder = models.ForeignKey(SubscriptionFolder, on_delete=models.CASCADE, null=True, blank=True) + provider_id = models.CharField(max_length=128, null=False) + playlist_id = models.CharField(null=False, max_length=128) + description = models.TextField() + channel_id = models.CharField(max_length=128) + channel_name = models.CharField(max_length=1024) + thumbnail = models.CharField(max_length=1024) + user = models.ForeignKey(User, on_delete=models.CASCADE) + # youtube adds videos to the 'Uploads' playlist at the top instead of the bottom + rewrite_playlist_indices = models.BooleanField(null=False, default=False) + + # overrides + auto_download = models.BooleanField(null=True, blank=True) + download_limit = models.IntegerField(null=True, blank=True) + download_order = models.CharField( + null=True, blank=True, + max_length=128, + choices=VIDEO_ORDER_CHOICES) + automatically_delete_watched = models.BooleanField(null=True, blank=True) + + def __str__(self): + return self.name + + def __repr__(self): + return f'subscription {self.id}, name="{self.name}", playlist_id="{self.playlist_id}"' + + def delete_subscription(self, keep_downloaded_videos: bool): + self.delete() + + def copy_from(self, other: "Subscription"): + self.name = other.name + self.parent_folder = other.parent_folder + self.provider_id = other.provider_id + self.playlist_id = other.playlist_id + self.description = other.description + self.channel_id = other.channel_id + self.channel_name = other.channel_name + self.thumbnail = other.thumbnail + try: + self.user = other.user + except User.DoesNotExist: + self.user = None + + self.rewrite_playlist_indices = other.rewrite_playlist_indices + + self.auto_download = other.auto_download + self.download_limit = other.download_limit + self.download_order = other.download_order + self.automatically_delete_watched = other.automatically_delete_watched diff --git a/app/YtManagerApp/models/subscription_folder.py b/app/YtManagerApp/models/subscription_folder.py new file mode 100644 index 0000000..1616eca --- /dev/null +++ b/app/YtManagerApp/models/subscription_folder.py @@ -0,0 +1,76 @@ +import logging +from typing import Callable, Union, Any, Optional + +from django.contrib.auth.models import User +from django.db import models +from django.db.models.functions import Lower + + +class SubscriptionFolder(models.Model): + name = models.CharField(null=False, max_length=250) + parent = models.ForeignKey('self', on_delete=models.CASCADE, null=True, blank=True) + user = models.ForeignKey(User, on_delete=models.CASCADE, null=False, blank=False) + + class Meta: + ordering = [Lower('parent__name'), Lower('name')] + + def __str__(self): + s = "" + current = self + while current is not None: + s = current.name + " > " + s + current = current.parent + return s[:-3] + + def __repr__(self): + return f'folder {self.id}, name="{self.name}"' + + def delete_folder(self, keep_subscriptions: bool): + from .subscription import Subscription + if keep_subscriptions: + + def visit(node: Union["SubscriptionFolder", "Subscription"]): + if isinstance(node, Subscription): + node.parent_folder = None + node.save() + + SubscriptionFolder.traverse(self.id, self.user, visit) + + self.delete() + + @staticmethod + def traverse(root_folder_id: Optional[int], + user: User, + visit_func: Callable[[Union["SubscriptionFolder", "Subscription"]], Any]): + from .subscription import Subscription + + data_collected = [] + + def collect(data): + if data is not None: + data_collected.append( data) + + # Visit root + if root_folder_id is not None: + root_folder = SubscriptionFolder.objects.get(id=root_folder_id) + collect(visit_func(root_folder)) + + queue = [root_folder_id] + visited = [] + + while len(queue) > 0: + folder_id = queue.pop() + + if folder_id in visited: + logging.error('Found folder tree cycle for folder id %d.', folder_id) + continue + visited.append(folder_id) + + for folder in SubscriptionFolder.objects.filter(parent_id=folder_id, user=user).order_by(Lower('name')): + collect(visit_func(folder)) + queue.append(folder.id) + + for subscription in Subscription.objects.filter(parent_folder_id=folder_id, user=user).order_by(Lower('name')): + collect(visit_func(subscription)) + + return data_collected diff --git a/app/YtManagerApp/models/video.py b/app/YtManagerApp/models/video.py new file mode 100644 index 0000000..5028b4d --- /dev/null +++ b/app/YtManagerApp/models/video.py @@ -0,0 +1,50 @@ +import mimetypes +import os + +from django.db import models + +from .subscription import Subscription + + +class Video(models.Model): + video_id = models.CharField(null=False, max_length=12) + name = models.TextField(null=False) + description = models.TextField() + watched = models.BooleanField(default=False, null=False) + new = models.BooleanField(default=True, null=False) + downloaded_path = models.TextField(null=True, blank=True) + downloaded_size = models.IntegerField(null=True, blank=True) + subscription = models.ForeignKey(Subscription, on_delete=models.CASCADE) + playlist_index = models.IntegerField(null=False) + publish_date = models.DateTimeField(null=False) + last_updated_date = models.DateTimeField(null=False, auto_now=True) + thumbnail = models.TextField() + uploader_name = models.CharField(null=False, max_length=255) + views = models.IntegerField(null=False, default=0) + rating = models.FloatField(null=False, default=0.5) + + def get_files(self): + if self.downloaded_path is not None: + directory, file_pattern = os.path.split(self.downloaded_path) + for file in os.listdir(directory): + if file.startswith(file_pattern): + yield os.path.join(directory, file) + + def find_video(self): + """ + Finds the video file from the downloaded files, and + returns + :return: Tuple containing file path and mime type + """ + for file in self.get_files(): + mime, _ = mimetypes.guess_type(file) + if mime is not None and mime.startswith('video/'): + return file, mime + + return None, None + + def __str__(self): + return self.name + + def __repr__(self): + return f'video {self.id}, video_id="{self.video_id}"' diff --git a/app/YtManagerApp/models/video_order.py b/app/YtManagerApp/models/video_order.py new file mode 100644 index 0000000..d4c64cf --- /dev/null +++ b/app/YtManagerApp/models/video_order.py @@ -0,0 +1,17 @@ +VIDEO_ORDER_CHOICES = [ + ('newest', 'Newest'), + ('oldest', 'Oldest'), + ('playlist', 'Playlist order'), + ('playlist_reverse', 'Reverse playlist order'), + ('popularity', 'Popularity'), + ('rating', 'Top rated'), +] + +VIDEO_ORDER_MAPPING = { + 'newest': '-publish_date', + 'oldest': 'publish_date', + 'playlist': 'playlist_index', + 'playlist_reverse': '-playlist_index', + 'popularity': '-views', + 'rating': '-rating' +} diff --git a/app/YtManagerApp/models/video_provider.py b/app/YtManagerApp/models/video_provider.py new file mode 100644 index 0000000..78ab0e6 --- /dev/null +++ b/app/YtManagerApp/models/video_provider.py @@ -0,0 +1,6 @@ +from django.db import models + + +class VideoProviderConfig(models.Model): + provider_id = models.CharField(max_length=128, unique=True, help_text="Provider ID") + settings = models.TextField(help_text="Video provider settings") diff --git a/app/YtManagerApp/providers/video_provider.py b/app/YtManagerApp/providers/video_provider.py new file mode 100644 index 0000000..63453aa --- /dev/null +++ b/app/YtManagerApp/providers/video_provider.py @@ -0,0 +1,110 @@ +from abc import abstractmethod, ABC +from typing import Dict, Iterable, List, Any + +from django.forms import Field + +from YtManagerApp.models import Subscription, Video + + +class ConfigurationValidationError(ValueError): + """ + Exception type thrown when validating configurations. + """ + def __init__(self, field_messages: Dict[str, str], *args, **kwargs): + """ + Constructor + :param field_messages: A dictionary which maps field names to errors, which will be displayed to the user. + """ + super().__init__(*args, **kwargs) + self.field_messages = field_messages + + +class InvalidURLError(ValueError): + """ + Invalid URL exception type + """ + pass + + +class VideoProvider(ABC): + name: str = "" + settings: Dict[str, Field] = {} + + @abstractmethod + def configure(self, configuration: Dict[str, Any]) -> None: + """ + Configures the video provider + :param configuration: A dictionary containing key-value pairs based on the settings defined. + :return: None + """ + pass + + @abstractmethod + def validate_configuration(self, configuration: Dict[str, Any]) -> None: + """ + Validates the given configuration. This is executed when validating the settings form from the UI. + :param configuration: Dictionary containing key-value pairs, based on the settings defined. + :except ConfigurationValidationError Thrown if there are validation errors + """ + pass + + @abstractmethod + def get_subscription_url(self, subscription: Subscription) -> str: + """ + Builds an URL that links to the given subscription. + :param subscription: The subscription + :return: URL + """ + pass + + @abstractmethod + def validate_subscription_url(self, url: str) -> None: + """ + Validates given URL. Throws InvalidURLError if not valid. + :param url: URL to validate + :except InvalidURLError Thrown if the URL is not valid + """ + pass + + @abstractmethod + def fetch_subscription(self, url: str) -> Subscription: + """ + Fetches a subscription using given URL + :param url: Subscription URL + :return: Subscription + :except InvalidURLError Thrown if the URL is not valid + """ + pass + + @abstractmethod + def get_video_url(self, video: Video) -> str: + """ + Builds an URL that links to the given video. + :param video: The video + :return: URL + """ + pass + + @abstractmethod + def fetch_videos(self, subscription: Subscription) -> Iterable[Video]: + """ + Fetches all the subscription items from the given subscription. + The method only needs to fetch the minimum amount of details, the update_videos method + is used to obtain additional information (such as likes/dislikes and other statistics) + :param subscription: + :return: + """ + pass + + @abstractmethod + def update_videos(self, videos: List[Video], update_metadata=False, update_statistics=False) -> None: + """ + Updates the metadata for all the videos in the list. + :param videos: Videos + :param update_metadata: If true, video metadata (name, description) will be updated + :param update_statistics: If true, statistics (likes/dislikes, view count) will be updated. + :return: + """ + pass + + diff --git a/app/YtManagerApp/providers/ytapi_video_provider.py b/app/YtManagerApp/providers/ytapi_video_provider.py new file mode 100644 index 0000000..98bb752 --- /dev/null +++ b/app/YtManagerApp/providers/ytapi_video_provider.py @@ -0,0 +1,156 @@ +from typing import Dict, Optional, Any, Iterable, List + +from django import forms +from external.pytaw.pytaw import youtube as yt +from external.pytaw.pytaw.utils import iterate_chunks + +from YtManagerApp.models import Subscription, Video +from YtManagerApp.providers.video_provider import VideoProvider, InvalidURLError + + +class YouTubeApiVideoProvider(VideoProvider): + name = "YtAPI" + settings = { + "api_key": forms.CharField(label="YouTube API Key:") + } + + def __init__(self): + super().__init__() + self.__api_key: str = None + self.__api: yt.YouTube = None + + def configure(self, configuration: Dict[str, Any]) -> None: + self.__api_key = configuration['api_key'] + self.__api = yt.YouTube(key=self.__api_key) + + def validate_configuration(self, configuration: Dict[str, Any]): + # TODO: implement + pass + + def get_subscription_url(self, subscription: Subscription): + return f"https://youtube.com/playlist?list={subscription.playlist_id}" + + def validate_subscription_url(self, url: str) -> None: + try: + parsed_url = self.__api.parse_url(url) + except yt.InvalidURL: + raise InvalidURLError("The given URL is not valid!") + + is_playlist = 'playlist' in parsed_url + is_channel = parsed_url['type'] in ('channel', 'user', 'channel_custom') + + if not is_channel and not is_playlist: + raise InvalidURLError('The given URL is not a channel or a playlist!') + + def fetch_subscription(self, url: str) -> Subscription: + sub = Subscription() + sub.provider_id = self.name + + self.validate_subscription_url(url) + url_parsed = self.__api.parse_url(url) + + if 'playlist' in url_parsed: + info_playlist = self.__api.playlist(url=url) + if info_playlist is None: + raise ValueError('Invalid playlist ID!') + + sub.name = info_playlist.title + sub.playlist_id = info_playlist.id + sub.description = info_playlist.description + sub.channel_id = info_playlist.channel_id + sub.channel_name = info_playlist.channel_title + sub.thumbnail = self._best_thumbnail(info_playlist).url + + else: + info_channel = self.__api.channel(url=url) + if info_channel is None: + raise ValueError('Cannot find channel!') + + sub.name = info_channel.title + sub.playlist_id = info_channel.uploads_playlist.id + sub.description = info_channel.description + sub.channel_id = info_channel.id + sub.channel_name = info_channel.title + sub.thumbnail = self._best_thumbnail(info_channel).url + sub.rewrite_playlist_indices = True + + return sub + + def _default_thumbnail(self, resource: yt.Resource) -> Optional[yt.Thumbnail]: + """ + Gets the default thumbnail for a resource. + Searches in the list of thumbnails for one with the label 'default', or takes the first one. + :param resource: + :return: + """ + thumbs = getattr(resource, 'thumbnails', None) + + if thumbs is None or len(thumbs) <= 0: + return None + + return next( + (i for i in thumbs if i.id == 'default'), + thumbs[0] + ) + + def _best_thumbnail(self, resource: yt.Resource) -> Optional[yt.Thumbnail]: + """ + Gets the best thumbnail available for a resource. + :param resource: + :return: + """ + thumbs = getattr(resource, 'thumbnails', None) + + if thumbs is None or len(thumbs) <= 0: + return None + + return max(thumbs, key=lambda t: t.width * t.height) + + def get_video_url(self, video: Video) -> str: + return f"https://youtube.com/watch?v={video.video_id}" + + def fetch_videos(self, subscription: Subscription) -> Iterable[Video]: + playlist_items = self.__api.playlist_items(subscription.playlist_id) + for item in playlist_items: + video = Video() + video.video_id = item.resource_video_id + video.name = item.title + video.description = item.description + video.watched = False + video.new = True + video.downloaded_path = None + video.subscription = subscription + video.playlist_index = item.position + video.publish_date = item.published_at + video.thumbnail = self._best_thumbnail(item).url + yield video + + def update_videos(self, videos: List[Video], update_metadata=False, update_statistics=False) -> None: + parts = ['id'] + if update_metadata: + parts.append('snippet') + if update_statistics: + parts.append('statistics') + + # don't waste api resources + if len(parts) <= 1: + return + + video_dict = {video.video_id: video for video in videos} + id_list = video_dict.keys() + + for batch in iterate_chunks(id_list, 50): + resp_videos = self.__api.videos(batch, part=','.join(parts)) + for resp_video in resp_videos: + v = video_dict[resp_video.id] + + if update_metadata: + v.name = resp_video.title + v.description = resp_video.description + + if update_statistics: + if resp_video.n_likes is not None \ + and resp_video.n_dislikes is not None \ + and resp_video.n_likes + resp_video.n_dislikes > 0: + v.rating = resp_video.n_likes / (resp_video.n_likes + resp_video.n_dislikes) + v.views = resp_video.n_views diff --git a/app/YtManagerApp/scheduler/jobs/__init__.py b/app/YtManagerApp/scheduler/jobs/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/app/YtManagerApp/scheduler/jobs/delete_video_job.py b/app/YtManagerApp/scheduler/jobs/delete_video_job.py deleted file mode 100644 index 39141f7..0000000 --- a/app/YtManagerApp/scheduler/jobs/delete_video_job.py +++ /dev/null @@ -1,47 +0,0 @@ -import os - -from YtManagerApp.models import Video -from YtManagerApp.scheduler.job import Job - - -class DeleteVideoJob(Job): - name = "DeleteVideoJob" - - def __init__(self, job_execution, video: Video): - super().__init__(job_execution) - self._video = video - - def get_description(self): - return f"Deleting video {self._video}" - - def run(self): - count = 0 - - try: - for file in self._video.get_files(): - self.log.info("Deleting file %s", file) - count += 1 - try: - os.unlink(file) - except OSError as e: - self.log.error("Failed to delete file %s: Error: %s", file, e) - - except OSError as e: - self.log.error("Failed to delete video %d [%s %s]. Error: %s", self._video.id, - self._video.video_id, self._video.name, e) - - self._video.downloaded_path = None - self._video.save() - - self.log.info('Deleted video %d successfully! (%d files) [%s %s]', self._video.id, count, - self._video.video_id, self._video.name) - - @staticmethod - def schedule(video: Video): - """ - Schedules a delete video job to run immediately. - :param video: - :return: - """ - from YtManagerApp.services import Services - Services.scheduler.add_job(DeleteVideoJob, args=[video]) diff --git a/app/YtManagerApp/scheduler/jobs/download_video_job.py b/app/YtManagerApp/scheduler/jobs/download_video_job.py deleted file mode 100644 index 14984c6..0000000 --- a/app/YtManagerApp/scheduler/jobs/download_video_job.py +++ /dev/null @@ -1,136 +0,0 @@ -import os -import re -from string import Template -from threading import Lock - -import youtube_dl - -from YtManagerApp.models import Video -from YtManagerApp.scheduler.job import Job - - -class DownloadVideoJob(Job): - name = "DownloadVideoJob" - __lock = Lock() - - def __init__(self, job_execution, video: Video, attempt: int = 1): - super().__init__(job_execution) - self._video = video - self._attempt = attempt - self._log_youtube_dl = self.log.getChild('youtube_dl') - - def get_description(self): - ret = "Downloading video " + self._video.name - if self._attempt > 1: - ret += f" (attempt {self._attempt})" - return ret - - def run(self): - # Issue: if multiple videos are downloaded at the same time, a race condition appears in the mkdirs() call that - # youtube-dl makes, which causes it to fail with the error 'Cannot create folder - file already exists'. - # For now, allow a single download instance. - self.__lock.acquire() - - try: - user = self._video.subscription.user - max_attempts = user.preferences['max_download_attempts'] - - youtube_dl_params, output_path = self.__build_youtube_dl_params(self._video) - with youtube_dl.YoutubeDL(youtube_dl_params) as yt: - ret = yt.download(["https://www.youtube.com/watch?v=" + self._video.video_id]) - - self.log.info('Download finished with code %d', ret) - - if ret == 0: - self._video.downloaded_path = output_path - self._video.save() - self.log.info('Video %d [%s %s] downloaded successfully!', self._video.id, self._video.video_id, - self._video.name) - - elif self._attempt <= max_attempts: - self.log.warning('Re-enqueueing video (attempt %d/%d)', self._attempt, max_attempts) - DownloadVideoJob.schedule(self._video, self._attempt + 1) - - else: - self.log.error('Multiple attempts to download video %d [%s %s] failed!', self._video.id, - self._video.video_id, self._video.name) - self._video.downloaded_path = '' - self._video.save() - - finally: - self.__lock.release() - - def __build_youtube_dl_params(self, video: Video): - - sub = video.subscription - user = sub.user - - # resolve path - download_path = user.preferences['download_path'] - - template_dict = self.__build_template_dict(video) - output_pattern = Template(user.preferences['download_file_pattern']).safe_substitute(template_dict) - - output_path = os.path.join(download_path, output_pattern) - output_path = os.path.normpath(output_path) - - youtube_dl_params = { - 'logger': self._log_youtube_dl, - 'format': user.preferences['download_format'], - 'outtmpl': output_path, - 'writethumbnail': True, - 'writedescription': True, - 'writesubtitles': user.preferences['download_subtitles'], - 'writeautomaticsub': user.preferences['download_autogenerated_subtitles'], - 'allsubtitles': user.preferences['download_subtitles_all'], - 'merge_output_format': 'mp4', - 'postprocessors': [ - { - 'key': 'FFmpegMetadata' - }, - ] - } - - sub_langs = user.preferences['download_subtitles_langs'].split(',') - sub_langs = [i.strip() for i in sub_langs] - if len(sub_langs) > 0: - youtube_dl_params['subtitleslangs'] = sub_langs - - sub_format = user.preferences['download_subtitles_format'] - if len(sub_format) > 0: - youtube_dl_params['subtitlesformat'] = sub_format - - return youtube_dl_params, output_path - - def __build_template_dict(self, video: Video): - return { - 'channel': video.subscription.channel_name, - 'channel_id': video.subscription.channel_id, - 'playlist': video.subscription.name, - 'playlist_id': video.subscription.playlist_id, - 'playlist_index': "{:03d}".format(1 + video.playlist_index), - 'title': video.name, - 'id': video.video_id, - } - - def __get_valid_path(self, path): - """ - Normalizes string, converts to lowercase, removes non-alpha characters, - and converts spaces to hyphens. - """ - import unicodedata - value = unicodedata.normalize('NFKD', path).encode('ascii', 'ignore').decode('ascii') - value = re.sub('[:"*]', '', value).strip() - value = re.sub('[?<>|]', '#', value) - return value - - @staticmethod - def schedule(video: Video, attempt: int = 1): - """ - Schedules to download video immediately - :param video: - :param attempt: - :return: - """ - from YtManagerApp.services import Services - Services.scheduler.add_job(DownloadVideoJob, args=[video, attempt]) diff --git a/app/YtManagerApp/scheduler/jobs/synchronize_job.py b/app/YtManagerApp/scheduler/jobs/synchronize_job.py deleted file mode 100644 index d1935e7..0000000 --- a/app/YtManagerApp/scheduler/jobs/synchronize_job.py +++ /dev/null @@ -1,184 +0,0 @@ -import errno -import itertools -from threading import Lock - -from apscheduler.triggers.cron import CronTrigger -from django.db.models import Max -from django.conf import settings - -from YtManagerApp.management.downloader import fetch_thumbnail, downloader_process_subscription -from YtManagerApp.models import * -from YtManagerApp.scheduler.job import Job -from YtManagerApp.services import Services -from YtManagerApp.utils import youtube -from external.pytaw.pytaw.utils import iterate_chunks - -_ENABLE_UPDATE_STATS = True - - -class SynchronizeJob(Job): - name = "SynchronizeJob" - __lock = Lock() - running = False - __global_sync_job = None - - def __init__(self, job_execution, subscription: Optional[Subscription] = None): - super().__init__(job_execution) - self.__subscription = subscription - self.__api = youtube.YoutubeAPI.build_public() - self.__new_vids = [] - - def get_description(self): - if self.__subscription is not None: - return "Running synchronization for subscription " + self.__subscription.name - return "Running synchronization..." - - def get_subscription_list(self): - if self.__subscription is not None: - return [self.__subscription] - return Subscription.objects.all() - - def get_videos_list(self, subs): - return Video.objects.filter(subscription__in=subs) - - def run(self): - self.__lock.acquire(blocking=True) - SynchronizeJob.running = True - try: - self.log.info(self.get_description()) - - # Build list of work items - work_subs = self.get_subscription_list() - work_vids = self.get_videos_list(work_subs) - - self.set_total_steps(len(work_subs) + len(work_vids)) - - # Remove the 'new' flag - work_vids.update(new=False) - - # Process subscriptions - for sub in work_subs: - self.progress_advance(1, "Synchronizing subscription " + sub.name) - self.check_new_videos(sub) - self.fetch_missing_thumbnails(sub) - - # Add new videos to progress calculation - self.set_total_steps(len(work_subs) + len(work_vids) + len(self.__new_vids)) - - # Process videos - all_videos = itertools.chain(work_vids, self.__new_vids) - for batch in iterate_chunks(all_videos, 50): - video_stats = {} - - if _ENABLE_UPDATE_STATS: - batch_ids = [video.video_id for video in batch] - video_stats = {v.id: v for v in self.__api.videos(batch_ids, part='id,statistics')} - - for video in batch: - self.progress_advance(1, "Updating video " + video.name) - self.check_video_deleted(video) - self.fetch_missing_thumbnails(video) - - if video.video_id in video_stats: - self.update_video_stats(video, video_stats[video.video_id]) - - # Start downloading videos - for sub in work_subs: - downloader_process_subscription(sub) - - finally: - SynchronizeJob.running = False - self.__lock.release() - - def check_new_videos(self, sub: Subscription): - playlist_items = self.__api.playlist_items(sub.playlist_id) - if sub.rewrite_playlist_indices: - playlist_items = sorted(playlist_items, key=lambda x: x.published_at) - else: - playlist_items = sorted(playlist_items, key=lambda x: x.position) - - for item in playlist_items: - results = Video.objects.filter(video_id=item.resource_video_id, subscription=sub) - - if not results.exists(): - self.log.info('New video for subscription %s: %s %s"', sub, item.resource_video_id, item.title) - - # fix playlist index if necessary - if sub.rewrite_playlist_indices or Video.objects.filter(subscription=sub, playlist_index=item.position).exists(): - highest = Video.objects.filter(subscription=sub).aggregate(Max('playlist_index'))['playlist_index__max'] - item.position = 1 + (highest or -1) - - self.__new_vids.append(Video.create(item, sub)) - - def fetch_missing_thumbnails(self, obj: Union[Subscription, Video]): - if obj.thumbnail.startswith("http"): - if isinstance(obj, Subscription): - obj.thumbnail = fetch_thumbnail(obj.thumbnail, 'sub', obj.playlist_id, settings.THUMBNAIL_SIZE_SUBSCRIPTION) - elif isinstance(obj, Video): - obj.thumbnail = fetch_thumbnail(obj.thumbnail, 'video', obj.video_id, settings.THUMBNAIL_SIZE_VIDEO) - obj.save() - - def check_video_deleted(self, video: Video): - if video.downloaded_path is not None: - files = [] - try: - files = list(video.get_files()) - except OSError as e: - if e.errno != errno.ENOENT: - self.log.error("Could not access path %s. Error: %s", video.downloaded_path, e) - self.usr_err(f"Could not access path {video.downloaded_path}: {e}", suppress_notification=True) - return - - # Try to find a valid video file - found_video = False - for file in files: - mime, _ = mimetypes.guess_type(file) - if mime is not None and mime.startswith("video"): - found_video = True - - # Video not found, we can safely assume that the video was deleted. - if not found_video: - self.log.info("Video %d was deleted! [%s %s]", video.id, video.video_id, video.name) - # Clean up - for file in files: - try: - os.unlink(file) - except OSError as e: - self.log.error("Could not delete redundant file %s. Error: %s", file, e) - self.usr_err(f"Could not delete redundant file {file}: {e}", suppress_notification=True) - video.downloaded_path = None - - # Mark watched? - user = video.subscription.user - if user.preferences['mark_deleted_as_watched']: - video.watched = True - - video.save() - - def update_video_stats(self, video: Video, yt_video): - if yt_video.n_likes is not None \ - and yt_video.n_dislikes is not None \ - and yt_video.n_likes + yt_video.n_dislikes > 0: - video.rating = yt_video.n_likes / (yt_video.n_likes + yt_video.n_dislikes) - - video.views = yt_video.n_views - video.save() - - @staticmethod - def schedule_global_job(): - trigger = CronTrigger.from_crontab(Services.appConfig.sync_schedule) - - if SynchronizeJob.__global_sync_job is None: - trigger = CronTrigger.from_crontab(Services.appConfig.sync_schedule) - SynchronizeJob.__global_sync_job = Services.scheduler.add_job(SynchronizeJob, trigger, max_instances=1, coalesce=True) - - else: - SynchronizeJob.__global_sync_job.reschedule(trigger, max_instances=1, coalesce=True) - - @staticmethod - def schedule_now(): - Services.scheduler.add_job(SynchronizeJob, max_instances=1, coalesce=True) - - @staticmethod - def schedule_now_for_subscription(subscription): - Services.scheduler.add_job(SynchronizeJob, user=subscription.user, args=[subscription]) diff --git a/app/YtManagerApp/services.py b/app/YtManagerApp/services.py index 3057ae5..7393580 100644 --- a/app/YtManagerApp/services.py +++ b/app/YtManagerApp/services.py @@ -2,12 +2,25 @@ import dependency_injector.containers as containers import dependency_injector.providers as providers from dynamic_preferences.registries import global_preferences_registry from YtManagerApp.management.appconfig import AppConfig +from YtManagerApp.management.download_manager import DownloadManager +from YtManagerApp.management.subscription_manager import SubscriptionManager +from YtManagerApp.management.video_manager import VideoManager +from YtManagerApp.management.video_provider_manager import VideoProviderManager from YtManagerApp.management.youtube_dl_manager import YoutubeDlManager from YtManagerApp.scheduler.scheduler import YtsmScheduler +class VideoProviders(containers.DeclarativeContainer): + from YtManagerApp.providers.ytapi_video_provider import YouTubeApiVideoProvider + ytApiProvider = providers.Factory(YouTubeApiVideoProvider) + + class Services(containers.DeclarativeContainer): globalPreferencesRegistry = providers.Object(global_preferences_registry.manager()) appConfig = providers.Singleton(AppConfig, globalPreferencesRegistry) scheduler = providers.Singleton(YtsmScheduler, appConfig) youtubeDLManager = providers.Singleton(YoutubeDlManager) + videoManager = providers.Singleton(VideoManager) + videoProviderManager = providers.Singleton(VideoProviderManager, [VideoProviders.ytApiProvider()]) + subscriptionManager = providers.Singleton(SubscriptionManager) + downloadManager = providers.Singleton(DownloadManager) diff --git a/app/YtManagerApp/utils/algorithms.py b/app/YtManagerApp/utils/algorithms.py index 6453cda..e6d3626 100644 --- a/app/YtManagerApp/utils/algorithms.py +++ b/app/YtManagerApp/utils/algorithms.py @@ -55,3 +55,22 @@ def bisect_left(a, x, lo=0, hi=None, key=None): # Create aliases bisect = bisect_right + + +def group_by(data, key): + """ + Groups the given data into a dictionary matching the structure { key : [values] } + :param data: Iterable data to be grouped + :param key: Key used to group the data + :return: A dictionary containing the grouped data + """ + result = {} + for entry in data: + entry_key = key(entry) + if entry_key not in result: + result[entry_key] = [entry] + else: + result[entry_key].append(entry) + + return result + diff --git a/app/YtManagerApp/utils/youtube.py b/app/YtManagerApp/utils/youtube.py deleted file mode 100644 index d30097d..0000000 --- a/app/YtManagerApp/utils/youtube.py +++ /dev/null @@ -1,49 +0,0 @@ -from django.conf import settings -from external.pytaw.pytaw.youtube import YouTube, Channel, Playlist, PlaylistItem, Thumbnail, InvalidURL, Resource, Video -from typing import Optional - - -class YoutubeAPI(YouTube): - - @staticmethod - def build_public() -> 'YoutubeAPI': - from YtManagerApp.management.appconfig import appconfig - return YoutubeAPI(key=appconfig.youtube_api_key) - - # @staticmethod - # def build_oauth() -> 'YoutubeAPI': - # flow = - # credentials = - # service = build(API_SERVICE_NAME, API_VERSION, credentials) - - -def default_thumbnail(resource: Resource) -> Optional[Thumbnail]: - """ - Gets the default thumbnail for a resource. - Searches in the list of thumbnails for one with the label 'default', or takes the first one. - :param resource: - :return: - """ - thumbs = getattr(resource, 'thumbnails', None) - - if thumbs is None or len(thumbs) <= 0: - return None - - return next( - (i for i in thumbs if i.id == 'default'), - thumbs[0] - ) - - -def best_thumbnail(resource: Resource) -> Optional[Thumbnail]: - """ - Gets the best thumbnail available for a resource. - :param resource: - :return: - """ - thumbs = getattr(resource, 'thumbnails', None) - - if thumbs is None or len(thumbs) <= 0: - return None - - return max(thumbs, key=lambda t: t.width * t.height) diff --git a/app/YtManagerApp/views/actions.py b/app/YtManagerApp/views/actions.py index 35e0e84..d22668c 100644 --- a/app/YtManagerApp/views/actions.py +++ b/app/YtManagerApp/views/actions.py @@ -3,12 +3,12 @@ from django.http import JsonResponse from django.views.generic import View from YtManagerApp.models import Video -from YtManagerApp.scheduler.jobs.synchronize_job import SynchronizeJob +from YtManagerApp.services import Services class SyncNowView(LoginRequiredMixin, View): def post(self, *args, **kwargs): - SynchronizeJob.schedule_now() + Services.subscriptionManager().synchronize_all() return JsonResponse({ 'success': True }) diff --git a/app/YtManagerApp/views/auth.py b/app/YtManagerApp/views/auth.py index 4f07836..85deea7 100644 --- a/app/YtManagerApp/views/auth.py +++ b/app/YtManagerApp/views/auth.py @@ -36,7 +36,7 @@ class RegisterView(FormView): return context def post(self, request, *args, **kwargs): - if not Services.appConfig.allow_registrations: + if not Services.appConfig().allow_registrations: return HttpResponseForbidden("Registrations are disabled!") return super().post(request, *args, **kwargs) diff --git a/app/YtManagerApp/views/first_time.py b/app/YtManagerApp/views/first_time.py index fe3ef43..22aacab 100644 --- a/app/YtManagerApp/views/first_time.py +++ b/app/YtManagerApp/views/first_time.py @@ -8,7 +8,6 @@ from django.shortcuts import redirect from django.urls import reverse_lazy from django.views.generic import FormView -from YtManagerApp.scheduler.jobs.synchronize_job import SynchronizeJob from YtManagerApp.services import Services from YtManagerApp.views.forms.first_time import WelcomeForm, ApiKeyForm, PickAdminUserForm, ServerConfigForm, DoneForm, \ UserCreationForm, LoginForm @@ -16,7 +15,7 @@ from YtManagerApp.views.forms.first_time import WelcomeForm, ApiKeyForm, PickAdm logger = logging.getLogger("FirstTimeWizard") -class WizardStepMixin: +class WizardStepMixin(object): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) @@ -24,7 +23,7 @@ class WizardStepMixin: def get(self, request, *args, **kwargs): # Prevent access if application is already initialized - if Services.appConfig.initialized: + if Services.appConfig().initialized: logger.debug(f"Attempted to access {request.path}, but first time setup already run. Redirected to home " f"page.") return redirect('home') @@ -32,7 +31,7 @@ class WizardStepMixin: return super().get(request, *args, **kwargs) def post(self, request, *args, **kwargs): - if Services.appConfig.initialized: + if Services.appConfig().initialized: logger.debug(f"Attempted to post {request.path}, but first time setup already run.") return HttpResponseForbidden() return super().post(request, *args, **kwargs) @@ -65,14 +64,14 @@ class Step1ApiKeyView(WizardStepMixin, FormView): def get_initial(self): initial = super().get_initial() - initial['api_key'] = Services.appConfig.youtube_api_key + initial['api_key'] = Services.appConfig().youtube_api_key return initial def form_valid(self, form): key = form.cleaned_data['api_key'] # TODO: validate key if key is not None and len(key) > 0: - Services.appConfig.youtube_api_key = key + Services.appConfig().youtube_api_key = key return super().form_valid(form) @@ -149,8 +148,8 @@ class Step3ConfigureView(WizardStepMixin, FormView): def get_initial(self): initial = super().get_initial() - initial['allow_registrations'] = Services.appConfig.allow_registrations - initial['sync_schedule'] = Services.appConfig.sync_schedule + initial['allow_registrations'] = Services.appConfig().allow_registrations + initial['sync_schedule'] = Services.appConfig().sync_schedule initial['auto_download'] = self.request.user.preferences['auto_download'] initial['download_location'] = self.request.user.preferences['download_path'] return initial @@ -158,11 +157,11 @@ class Step3ConfigureView(WizardStepMixin, FormView): def form_valid(self, form): allow_registrations = form.cleaned_data['allow_registrations'] if allow_registrations is not None: - Services.appConfig.allow_registrations = allow_registrations + Services.appConfig().allow_registrations = allow_registrations sync_schedule = form.cleaned_data['sync_schedule'] if sync_schedule is not None and len(sync_schedule) > 0: - Services.appConfig.sync_schedule = sync_schedule + Services.appConfig().sync_schedule = sync_schedule auto_download = form.cleaned_data['auto_download'] if auto_download is not None: @@ -173,11 +172,11 @@ class Step3ConfigureView(WizardStepMixin, FormView): self.request.user.preferences['download_path'] = download_location # Set initialized to true - Services.appConfig.initialized = True + Services.appConfig().initialized = True # Start scheduler if not started - Services.scheduler.initialize() - SynchronizeJob.schedule_global_job() + Services.scheduler().initialize() + Services.subscriptionManager().schedule_global_synchronize_job() return super().form_valid(form) diff --git a/app/YtManagerApp/views/forms/settings.py b/app/YtManagerApp/views/forms/settings.py index f9b6f58..ff54eca 100644 --- a/app/YtManagerApp/views/forms/settings.py +++ b/app/YtManagerApp/views/forms/settings.py @@ -234,25 +234,25 @@ class AdminSettingsForm(forms.Form): @staticmethod def get_initials(): return { - 'api_key': Services.appConfig.youtube_api_key, - 'allow_registrations': Services.appConfig.allow_registrations, - 'sync_schedule': Services.appConfig.sync_schedule, - 'scheduler_concurrency': Services.appConfig.concurrency, + 'api_key': Services.appConfig().youtube_api_key, + 'allow_registrations': Services.appConfig().allow_registrations, + 'sync_schedule': Services.appConfig().sync_schedule, + 'scheduler_concurrency': Services.appConfig().concurrency, } def save(self): api_key = self.cleaned_data['api_key'] if api_key is not None and len(api_key) > 0: - Services.appConfig.youtube_api_key = api_key + Services.appConfig().youtube_api_key = api_key allow_registrations = self.cleaned_data['allow_registrations'] if allow_registrations is not None: - Services.appConfig.allow_registrations = allow_registrations + Services.appConfig().allow_registrations = allow_registrations sync_schedule = self.cleaned_data['sync_schedule'] if sync_schedule is not None and len(sync_schedule) > 0: - Services.appConfig.sync_schedule = sync_schedule + Services.appConfig().sync_schedule = sync_schedule concurrency = self.cleaned_data['scheduler_concurrency'] if concurrency is not None: - Services.appConfig.concurrency = concurrency + Services.appConfig().concurrency = concurrency diff --git a/app/YtManagerApp/views/index.py b/app/YtManagerApp/views/index.py index dd49d79..709ac7a 100644 --- a/app/YtManagerApp/views/index.py +++ b/app/YtManagerApp/views/index.py @@ -1,22 +1,21 @@ from crispy_forms.helper import FormHelper from crispy_forms.layout import Layout, Field, HTML from django import forms +from django.conf import settings from django.contrib.auth.decorators import login_required from django.contrib.auth.mixins import LoginRequiredMixin +from django.core.paginator import Paginator from django.db.models import Q from django.http import HttpRequest, HttpResponseBadRequest, JsonResponse from django.shortcuts import render, redirect from django.views.generic import CreateView, UpdateView, DeleteView, FormView from django.views.generic.edit import FormMixin -from django.conf import settings -from django.core.paginator import Paginator -from YtManagerApp.management.videos import get_videos -from YtManagerApp.models import Subscription, SubscriptionFolder, VIDEO_ORDER_CHOICES, VIDEO_ORDER_MAPPING -from YtManagerApp.services import Services -from YtManagerApp.utils import youtube, subscription_file_parser -from YtManagerApp.views.controls.modal import ModalMixin -import logging +from YtManagerApp.models import Subscription, SubscriptionFolder, VIDEO_ORDER_CHOICES, VIDEO_ORDER_MAPPING +from YtManagerApp.providers.video_provider import InvalidURLError +from YtManagerApp.services import Services +from YtManagerApp.utils import subscription_file_parser +from YtManagerApp.views.controls.modal import ModalMixin class VideoFilterForm(forms.Form): @@ -110,8 +109,7 @@ def __tree_sub_id(sub_id): def index(request: HttpRequest): - - if not Services.appConfig.initialized: + if not Services.appConfig().initialized: return redirect('first_time_0') context = { @@ -129,7 +127,6 @@ def index(request: HttpRequest): @login_required def ajax_get_tree(request: HttpRequest): - def visit(node): if isinstance(node, SubscriptionFolder): return { @@ -157,7 +154,7 @@ def ajax_get_videos(request: HttpRequest): if request.method == 'POST': form = VideoFilterForm(request.POST) if form.is_valid(): - videos = get_videos( + videos = Services.videoManager().get_videos( user=request.user, sort_order=form.cleaned_data['sort'], query=form.cleaned_data['query'], @@ -272,7 +269,6 @@ class CreateSubscriptionForm(forms.ModelForm): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) - self.yt_api = youtube.YoutubeAPI.build_public() self.helper = FormHelper() self.helper.form_tag = False self.helper.layout = Layout( @@ -289,16 +285,9 @@ class CreateSubscriptionForm(forms.ModelForm): def clean_playlist_url(self): playlist_url: str = self.cleaned_data['playlist_url'] try: - parsed_url = self.yt_api.parse_url(playlist_url) - except youtube.InvalidURL as e: + Services.videoProviderManager().validate_subscription_url(playlist_url) + except InvalidURLError as e: raise forms.ValidationError(str(e)) - - is_playlist = 'playlist' in parsed_url - is_channel = parsed_url['type'] in ('channel', 'user', 'channel_custom') - - if not is_channel and not is_playlist: - raise forms.ValidationError('The given URL must link to a channel or a playlist!') - return playlist_url @@ -308,10 +297,12 @@ class CreateSubscriptionModal(LoginRequiredMixin, ModalMixin, CreateView): def form_valid(self, form): form.instance.user = self.request.user - api = youtube.YoutubeAPI.build_public() try: - form.instance.fetch_from_url(form.cleaned_data['playlist_url'], api) - except youtube.InvalidURL as e: + subscription: Subscription = Services.videoProviderManager().fetch_subscription( + form.cleaned_data['playlist_url']) + form.instance.copy_from(subscription) + form.instance.user = self.request.user + except InvalidURLError as e: return self.modal_response(form, False, str(e)) except ValueError as e: return self.modal_response(form, False, str(e)) @@ -327,8 +318,9 @@ class CreateSubscriptionModal(LoginRequiredMixin, ModalMixin, CreateView): # except youtube.APIError as e: # return self.modal_response( # form, False, 'An error occurred while communicating with the YouTube API: ' + str(e)) - - return super().form_valid(form) + response = super().form_valid(form) + Services.subscriptionManager().synchronize(form.instance) + return response class UpdateSubscriptionForm(forms.ModelForm): @@ -407,7 +399,6 @@ class ImportSubscriptionsForm(forms.Form): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) - self.yt_api = youtube.YoutubeAPI.build_public() self.helper = FormHelper() self.helper.form_tag = False self.helper.layout = Layout( @@ -456,28 +447,18 @@ class ImportSubscriptionsModal(LoginRequiredMixin, ModalMixin, FormView): try: url_list = list(subscription_file_parser.parse(file)) except subscription_file_parser.FormatNotSupportedError: - return super().modal_response(form, success=False, + return super().modal_response(form, + success=False, error_msg="The file could not be parsed! " "Possible problems: format not supported, file is malformed.") print(form.cleaned_data) - # Create subscriptions - api = youtube.YoutubeAPI.build_public() - for url in url_list: - sub = Subscription() - sub.user = self.request.user - sub.parent_folder = form.cleaned_data['parent_folder'] - sub.auto_download = form.cleaned_data['auto_download'] - sub.download_limit = form.cleaned_data['download_limit'] - sub.download_order = form.cleaned_data['download_order'] - sub.automatically_delete_watched = form.cleaned_data["automatically_delete_watched"] - try: - sub.fetch_from_url(url, api) - except Exception as e: - logging.error("Import subscription error - error processing URL %s: %s", url, e) - continue - - sub.save() + Services.subscriptionManager().import_multiple(url_list, + form.cleaned_data['parent_folder'], + form.cleaned_data['auto_download'], + form.cleaned_data['download_limit'], + form.cleaned_data['download_order'], + form.cleaned_data["automatically_delete_watched"]) return super().form_valid(form) diff --git a/app/YtManagerApp/views/settings.py b/app/YtManagerApp/views/settings.py index 9d621ac..e2fc4c9 100644 --- a/app/YtManagerApp/views/settings.py +++ b/app/YtManagerApp/views/settings.py @@ -3,7 +3,7 @@ from django.http import HttpResponseForbidden from django.urls import reverse_lazy from django.views.generic import FormView -from YtManagerApp.scheduler.jobs.synchronize_job import SynchronizeJob +from YtManagerApp.services import Services from YtManagerApp.views.forms.settings import SettingsForm, AdminSettingsForm @@ -45,5 +45,5 @@ class AdminSettingsView(LoginRequiredMixin, FormView): def form_valid(self, form): form.save() - SynchronizeJob.schedule_global_job() + Services.subscriptionManager().schedule_global_synchronize_job() return super().form_valid(form) diff --git a/app/YtManagerApp/views/video.py b/app/YtManagerApp/views/video.py index f2e073b..4664855 100644 --- a/app/YtManagerApp/views/video.py +++ b/app/YtManagerApp/views/video.py @@ -1,8 +1,6 @@ from django.contrib.auth.decorators import login_required from django.contrib.auth.mixins import LoginRequiredMixin -from django.http import HttpRequest, StreamingHttpResponse, FileResponse -from django.urls import reverse, reverse_lazy -from django.views import View +from django.http import HttpRequest, FileResponse from django.views.generic import DetailView from YtManagerApp.models import Video