From 6b843f1fc28bbd3bca44857c4c66d81d2122e234 Mon Sep 17 00:00:00 2001 From: Tiberiu Chibici Date: Thu, 19 Dec 2019 00:27:06 +0200 Subject: [PATCH] Major refactor of many things. --- app/YtManagerApp/appmain.py | 6 +- app/YtManagerApp/models.py | 93 +++------- app/YtManagerApp/services.py | 13 -- app/YtManagerApp/services/__init__.py | 15 ++ .../{management => services}/appconfig.py | 0 .../{management => services}/downloader.py | 2 +- .../providers}/__init__.py | 0 .../services/providers/video_provider.py | 88 +++++++++ .../services/providers/youtube_provider.py | 167 ++++++++++++++++++ .../{ => services}/scheduler/__init__.py | 0 .../{ => services}/scheduler/job.py | 4 +- .../{ => services}/scheduler/jobs/__init__.py | 0 .../scheduler/jobs/delete_video_job.py | 8 +- .../scheduler/jobs/download_video_job.py | 4 +- .../scheduler/jobs/synchronize_job.py | 15 +- .../scheduler/jobs/youtubedl_update_job.py | 18 ++ .../scheduler/progress_tracker.py | 0 .../{ => services}/scheduler/scheduler.py | 10 +- .../services/video_provider_manager.py | 11 ++ .../{management => services}/videos.py | 0 .../youtube_dl_manager.py | 22 +-- app/YtManagerApp/utils/youtube.py | 49 ----- app/YtManagerApp/views/actions.py | 2 +- app/YtManagerApp/views/auth.py | 2 +- app/YtManagerApp/views/first_time.py | 20 +-- app/YtManagerApp/views/forms/settings.py | 2 +- app/YtManagerApp/views/index.py | 2 +- app/YtManagerApp/views/settings.py | 2 +- 28 files changed, 374 insertions(+), 181 deletions(-) delete mode 100644 app/YtManagerApp/services.py create mode 100644 app/YtManagerApp/services/__init__.py rename app/YtManagerApp/{management => services}/appconfig.py (100%) rename app/YtManagerApp/{management => services}/downloader.py (98%) rename app/YtManagerApp/{management => services/providers}/__init__.py (100%) create mode 100644 app/YtManagerApp/services/providers/video_provider.py create mode 100644 app/YtManagerApp/services/providers/youtube_provider.py rename app/YtManagerApp/{ => services}/scheduler/__init__.py (100%) rename app/YtManagerApp/{ => services}/scheduler/job.py (98%) rename app/YtManagerApp/{ => services}/scheduler/jobs/__init__.py (100%) rename app/YtManagerApp/{ => services}/scheduler/jobs/delete_video_job.py (82%) rename app/YtManagerApp/{ => services}/scheduler/jobs/download_video_job.py (97%) rename app/YtManagerApp/{ => services}/scheduler/jobs/synchronize_job.py (94%) create mode 100644 app/YtManagerApp/services/scheduler/jobs/youtubedl_update_job.py rename app/YtManagerApp/{ => services}/scheduler/progress_tracker.py (100%) rename app/YtManagerApp/{ => services}/scheduler/scheduler.py (86%) create mode 100644 app/YtManagerApp/services/video_provider_manager.py rename app/YtManagerApp/{management => services}/videos.py (100%) rename app/YtManagerApp/{management => services}/youtube_dl_manager.py (81%) delete mode 100644 app/YtManagerApp/utils/youtube.py diff --git a/app/YtManagerApp/appmain.py b/app/YtManagerApp/appmain.py index 215786c..091782d 100644 --- a/app/YtManagerApp/appmain.py +++ b/app/YtManagerApp/appmain.py @@ -5,7 +5,7 @@ import sys from django.conf import settings as dj_settings -from YtManagerApp.scheduler.jobs.synchronize_job import SynchronizeJob +from YtManagerApp.services.scheduler.jobs.synchronize_job import SynchronizeJob from YtManagerApp.services import Services from django.db.utils import OperationalError @@ -35,8 +35,8 @@ def main(): __initialize_logger() try: - if Services.appConfig.initialized: - Services.scheduler.initialize() + if Services.appConfig().initialized: + Services.scheduler().initialize() SynchronizeJob.schedule_global_job() except OperationalError: # Settings table is not created when running migrate or makemigrations; diff --git a/app/YtManagerApp/models.py b/app/YtManagerApp/models.py index 227efda..94e3520 100644 --- a/app/YtManagerApp/models.py +++ b/app/YtManagerApp/models.py @@ -7,8 +7,6 @@ from django.contrib.auth.models import User from django.db import models from django.db.models.functions import Lower -from YtManagerApp.utils import youtube - # help_text = user shown text # verbose_name = user shown name # null = nullable, blank = user is allowed to set value to empty @@ -101,15 +99,16 @@ class SubscriptionFolder(models.Model): class Subscription(models.Model): name = models.CharField(null=False, max_length=1024) - parent_folder = models.ForeignKey(SubscriptionFolder, on_delete=models.CASCADE, null=True, blank=True) - playlist_id = models.CharField(null=False, max_length=128) description = models.TextField() - channel_id = models.CharField(max_length=128) - channel_name = models.CharField(max_length=1024) + original_url = models.CharField(null=False, max_length=1024) thumbnail = models.CharField(max_length=1024) + + provider = models.CharField(null=False, max_length=64) + provider_id = models.CharField(null=False, max_length=64) + provider_data = models.CharField(null=True, max_length=1024) + + parent_folder = models.ForeignKey(SubscriptionFolder, on_delete=models.CASCADE, null=True, blank=True) user = models.ForeignKey(User, on_delete=models.CASCADE) - # youtube adds videos to the 'Uploads' playlist at the top instead of the bottom - rewrite_playlist_indices = models.BooleanField(null=False, default=False) # overrides auto_download = models.BooleanField(null=True, blank=True) @@ -126,81 +125,37 @@ class Subscription(models.Model): def __repr__(self): return f'subscription {self.id}, name="{self.name}", playlist_id="{self.playlist_id}"' - def fill_from_playlist(self, info_playlist: youtube.Playlist): - self.name = info_playlist.title - self.playlist_id = info_playlist.id - self.description = info_playlist.description - self.channel_id = info_playlist.channel_id - self.channel_name = info_playlist.channel_title - self.thumbnail = youtube.best_thumbnail(info_playlist).url - - def copy_from_channel(self, info_channel: youtube.Channel): - # No point in storing info about the 'uploads from X' playlist - self.name = info_channel.title - self.playlist_id = info_channel.uploads_playlist.id - self.description = info_channel.description - self.channel_id = info_channel.id - self.channel_name = info_channel.title - self.thumbnail = youtube.best_thumbnail(info_channel).url - self.rewrite_playlist_indices = True - - def fetch_from_url(self, url, yt_api: youtube.YoutubeAPI): - url_parsed = yt_api.parse_url(url) - if 'playlist' in url_parsed: - info_playlist = yt_api.playlist(url=url) - if info_playlist is None: - raise ValueError('Invalid playlist ID!') - - self.fill_from_playlist(info_playlist) - else: - info_channel = yt_api.channel(url=url) - if info_channel is None: - raise ValueError('Cannot find channel!') - - self.copy_from_channel(info_channel) - def delete_subscription(self, keep_downloaded_videos: bool): self.delete() class Video(models.Model): - video_id = models.CharField(null=False, max_length=12) name = models.TextField(null=False) description = models.TextField() - watched = models.BooleanField(default=False, null=False) - new = models.BooleanField(default=True, null=False) - downloaded_path = models.TextField(null=True, blank=True) - subscription = models.ForeignKey(Subscription, on_delete=models.CASCADE) - playlist_index = models.IntegerField(null=False) publish_date = models.DateTimeField(null=False) thumbnail = models.TextField() uploader_name = models.CharField(null=False, max_length=255) + + provider_id = models.CharField(null=False, max_length=64) + provider_data = models.CharField(null=True, max_length=1024) + + playlist_index = models.IntegerField(null=False) + downloaded_path = models.TextField(null=True, blank=True) + subscription = models.ForeignKey(Subscription, on_delete=models.CASCADE) + + watched = models.BooleanField(default=False, null=False) + new = models.BooleanField(default=True, null=False) + views = models.IntegerField(null=False, default=0) rating = models.FloatField(null=False, default=0.5) - @staticmethod - def create(playlist_item: youtube.PlaylistItem, subscription: Subscription): - video = Video() - video.video_id = playlist_item.resource_video_id - video.name = playlist_item.title - video.description = playlist_item.description - video.watched = False - video.new = True - video.downloaded_path = None - video.subscription = subscription - video.playlist_index = playlist_item.position - video.publish_date = playlist_item.published_at - video.thumbnail = youtube.best_thumbnail(playlist_item).url - video.save() - return video - def mark_watched(self): self.watched = True self.save() if self.downloaded_path is not None: from YtManagerApp.management.appconfig import appconfig - from YtManagerApp.scheduler.jobs import DeleteVideoJob - from YtManagerApp.scheduler.jobs import SynchronizeJob + from YtManagerApp.management.scheduler.jobs import DeleteVideoJob + from YtManagerApp.management.scheduler.jobs import SynchronizeJob if appconfig.for_sub(self.subscription, 'automatically_delete_watched'): DeleteVideoJob.schedule(self) @@ -209,7 +164,7 @@ class Video(models.Model): def mark_unwatched(self): self.watched = False self.save() - from YtManagerApp.scheduler.jobs.synchronize_job import SynchronizeJob + from YtManagerApp.management.scheduler.jobs.synchronize_job import SynchronizeJob SynchronizeJob.schedule_now_for_subscription(self.subscription) def get_files(self): @@ -234,9 +189,9 @@ class Video(models.Model): def delete_files(self): if self.downloaded_path is not None: - from YtManagerApp.scheduler.jobs import DeleteVideoJob + from YtManagerApp.management.scheduler.jobs import DeleteVideoJob from YtManagerApp.management.appconfig import appconfig - from YtManagerApp.scheduler.jobs import SynchronizeJob + from YtManagerApp.management.scheduler.jobs import SynchronizeJob DeleteVideoJob.schedule(self) @@ -247,7 +202,7 @@ class Video(models.Model): def download(self): if not self.downloaded_path: - from YtManagerApp.scheduler.jobs.download_video_job import DownloadVideoJob + from YtManagerApp.management.scheduler.jobs import DownloadVideoJob DownloadVideoJob.schedule(self) def __str__(self): diff --git a/app/YtManagerApp/services.py b/app/YtManagerApp/services.py deleted file mode 100644 index 3057ae5..0000000 --- a/app/YtManagerApp/services.py +++ /dev/null @@ -1,13 +0,0 @@ -import dependency_injector.containers as containers -import dependency_injector.providers as providers -from dynamic_preferences.registries import global_preferences_registry -from YtManagerApp.management.appconfig import AppConfig -from YtManagerApp.management.youtube_dl_manager import YoutubeDlManager -from YtManagerApp.scheduler.scheduler import YtsmScheduler - - -class Services(containers.DeclarativeContainer): - globalPreferencesRegistry = providers.Object(global_preferences_registry.manager()) - appConfig = providers.Singleton(AppConfig, globalPreferencesRegistry) - scheduler = providers.Singleton(YtsmScheduler, appConfig) - youtubeDLManager = providers.Singleton(YoutubeDlManager) diff --git a/app/YtManagerApp/services/__init__.py b/app/YtManagerApp/services/__init__.py new file mode 100644 index 0000000..2397d22 --- /dev/null +++ b/app/YtManagerApp/services/__init__.py @@ -0,0 +1,15 @@ +import dependency_injector.containers as di_containers +import dependency_injector.providers as di_providers +from dynamic_preferences.registries import global_preferences_registry +from YtManagerApp.services.appconfig import AppConfig +from YtManagerApp.services.youtube_dl_manager import YoutubeDlManager +from YtManagerApp.services.scheduler.scheduler import YtsmScheduler +from YtManagerApp.services.video_provider_manager import VideoProviderManager + + +class Services(di_containers.DeclarativeContainer): + globalPreferencesRegistry = di_providers.Object(global_preferences_registry.manager()) + appConfig = di_providers.Singleton(AppConfig, globalPreferencesRegistry) + scheduler = di_providers.Singleton(YtsmScheduler, appConfig) + youtubeDLManager = di_providers.Singleton(YoutubeDlManager) + providerManager = di_providers.Singleton(VideoProviderManager) diff --git a/app/YtManagerApp/management/appconfig.py b/app/YtManagerApp/services/appconfig.py similarity index 100% rename from app/YtManagerApp/management/appconfig.py rename to app/YtManagerApp/services/appconfig.py diff --git a/app/YtManagerApp/management/downloader.py b/app/YtManagerApp/services/downloader.py similarity index 98% rename from app/YtManagerApp/management/downloader.py rename to app/YtManagerApp/services/downloader.py index 0b4ce75..632aab1 100644 --- a/app/YtManagerApp/management/downloader.py +++ b/app/YtManagerApp/services/downloader.py @@ -1,4 +1,4 @@ -from YtManagerApp.scheduler.jobs.download_video_job import DownloadVideoJob +from YtManagerApp.services.scheduler.jobs.download_video_job import DownloadVideoJob from YtManagerApp.models import Video, Subscription, VIDEO_ORDER_MAPPING from YtManagerApp.utils import first_non_null from django.conf import settings as srv_settings diff --git a/app/YtManagerApp/management/__init__.py b/app/YtManagerApp/services/providers/__init__.py similarity index 100% rename from app/YtManagerApp/management/__init__.py rename to app/YtManagerApp/services/providers/__init__.py diff --git a/app/YtManagerApp/services/providers/video_provider.py b/app/YtManagerApp/services/providers/video_provider.py new file mode 100644 index 0000000..2531cc9 --- /dev/null +++ b/app/YtManagerApp/services/providers/video_provider.py @@ -0,0 +1,88 @@ +from abc import abstractmethod, ABC +from typing import Iterable, ClassVar + +from django import forms + +from YtManagerApp.models import Subscription, Video +from YtManagerApp.services.scheduler.progress_tracker import ProgressTracker + + +class VideoProvider(ABC): + """ + Represents a video hosting service that provides videos and playlists (e.g. YouTube, Vimeo). + Note: the method implementations should be thread safe, as they may be called from multiple jobs running in + parallel. + """ + + @abstractmethod + def update_configuration(self, **kwargs): + """ + Updates the configuration options of this video provider. + This method is called first when the provider is registered using the configuration stored in the + database. After that, the method will be called when the user changes any configuration options. + :param kwargs: Configuration arguments + """ + pass + + @abstractmethod + def get_display_name(self) -> str: + """ + Returns an user friendly name for this provider. + :return: + """ + pass + + @abstractmethod + def get_provider_id(self) -> str: + """ + Returns an identifier that uniquely identifies this provider. + :return: + """ + pass + + @abstractmethod + def validate_playlist_url(self, url: str) -> bool: + """ + Validates that the given playlist URL is valid for the given video provider service. + :param url: + :return: + """ + pass + + @abstractmethod + def fetch_playlist(self, url: str) -> Subscription: + """ + Gets metadata about the playlist identified by the given URL. + :param url: + :return: + """ + pass + + @abstractmethod + def fetch_videos(self, subscription: Subscription) -> Iterable[Video]: + """ + Gets metadata about the videos in the given playlist. + :param subscription: + :return: + """ + pass + + @abstractmethod + def update_videos(self, videos: Iterable[Video], progress_tracker: ProgressTracker, update_info: bool = True, update_stats: bool = False): + """ + Updates metadata about given videos. + :param update_info: If set to true, basic information such as title, description will be updated + :param update_stats: If set to true, video statistics (such as rating, view counts) will be updated + :param videos: Videos to be updated. + :param progress_tracker: Used to track the progress of the update process + :return: + """ + pass + + @abstractmethod + def get_config_form(self) -> ClassVar[forms.Form]: + """ + Gets the configuration form + :return: + """ + pass diff --git a/app/YtManagerApp/services/providers/youtube_provider.py b/app/YtManagerApp/services/providers/youtube_provider.py new file mode 100644 index 0000000..a4c73e7 --- /dev/null +++ b/app/YtManagerApp/services/providers/youtube_provider.py @@ -0,0 +1,167 @@ +import json +from typing import ClassVar, Iterable, Optional + +from django import forms +from external.pytaw.pytaw.utils import iterate_chunks +from external.pytaw.pytaw.youtube import YouTube, Thumbnail, InvalidURL, Resource, Video + +from YtManagerApp.models import Video, Subscription +from YtManagerApp.services.providers.video_provider import VideoProvider +from YtManagerApp.services.scheduler.progress_tracker import ProgressTracker + + +class YouTubeConfigForm(forms.Form): + api_key = forms.CharField(label="YouTube API Key:") + + +class YouTubeProvider(VideoProvider): + + def __init__(self): + self._apiKey: str = None + self._api: YouTube = None + + def _sanity_check(self): + if self._apiKey is None: + raise ValueError("The YouTube API key is not set!") + + @staticmethod + def _best_thumbnail(resource: Resource) -> Optional[Thumbnail]: + """ + Gets the best thumbnail available for a resource. + :param resource: + :return: + """ + thumbs = getattr(resource, 'thumbnails', None) + + if thumbs is None or len(thumbs) <= 0: + return None + + return max(thumbs, key=lambda t: t.width * t.height) + + def update_configuration(self, **kwargs): + self._apiKey = kwargs.get('apiKey') + self._sanity_check() + self._api = YouTube(key=self._apiKey) + + def get_display_name(self) -> str: + return "YouTube API Provider" + + def get_provider_id(self) -> str: + return 'youtube' + + def validate_playlist_url(self, url: str) -> bool: + try: + parsed_url = self._api.parse_url(url) + except InvalidURL: + return False + + is_playlist = 'playlist' in parsed_url + is_channel = parsed_url['type'] in ('channel', 'user', 'channel_custom') + + return is_playlist or is_channel + + def fetch_playlist(self, url: str) -> Subscription: + + if not self.validate_playlist_url(url): + raise ValueError("Invalid playlist or channel URL") + + parsed_url = self._api.parse_url(url) + sub = Subscription() + + if 'playlist' in parsed_url: + info = self._api.playlist(url=url) + if info is None: + raise ValueError('Invalid playlist ID!') + provider_data = { + 'channel_id': None, + 'rewrite_indices': False + } + sub.provider_id = info.id + + else: + info = self._api.channel(url=url) + if info is None: + raise ValueError('Cannot find channel!') + + provider_data = { + 'channel_id': info.id, + 'rewrite_indices': True + } + sub.provider_id = info.uploads_playlist.id + + sub.name = info.title + sub.description = info.description + sub.original_url = url + sub.thumbnail = YouTubeProvider._best_thumbnail(info).url + + sub.provider = self.get_provider_id() + sub.provider_data = json.dumps(provider_data) + + return sub + + def fetch_videos(self, subscription: Subscription) -> Iterable[Video]: + provider_data = json.loads(subscription.provider_data) + playlist_items = self._api.playlist_items(subscription.provider_id) + + if provider_data.get('rewrite_indices'): + playlist_items = sorted(playlist_items, key=lambda x: x.published_at) + else: + playlist_items = sorted(playlist_items, key=lambda x: x.position) + i = 1 + + for playlist_item in playlist_items: + video = Video() + video.name = playlist_item.title + video.description = playlist_item.description + video.publish_date = playlist_item.published_at + video.thumbnail = YouTubeProvider._best_thumbnail(playlist_item).url + video.uploader_name = "" + + video.provider_id = playlist_item.resource_video_id + video.provider_data = None + + if provider_data.get('rewrite_indices'): + video.playlist_index = i + i += 1 + else: + video.playlist_index = playlist_item.position + video.downloaded_path = None + video.subscription = subscription + + video.watched = False + video.new = True + + yield video + + def update_videos(self, videos: Iterable[Video], progress_tracker: ProgressTracker, update_info: bool = True, update_stats: bool = False): + videos_list = list(videos) + progress_tracker.total_steps = len(videos_list) + + parts = 'id' + if update_info: + parts += ',snippet' + if update_stats: + parts += ',statistics' + + for batch in iterate_chunks(videos_list, 50): + batch_ids = [video.video_id for video in batch] + videos_new = {v.id: v for v in self._api.videos(batch_ids, part=parts)} + + for video in batch: + progress_tracker.advance(1, "Updating video " + video.name) + video_new = videos_new.get(video.provider_id) + if video_new is None: + continue + + if update_info: + video.name = video_new.title + video.description = video_new.description + if update_stats: + if video_new.n_likes is not None \ + and video_new.n_dislikes is not None \ + and video_new.n_likes + video_new.n_dislikes > 0: + video.rating = video_new.n_likes / (video_new.n_likes + video_new.n_dislikes) + video.views = video_new.n_views + + def get_config_form(self) -> ClassVar[forms.Form]: + return YouTubeConfigForm diff --git a/app/YtManagerApp/scheduler/__init__.py b/app/YtManagerApp/services/scheduler/__init__.py similarity index 100% rename from app/YtManagerApp/scheduler/__init__.py rename to app/YtManagerApp/services/scheduler/__init__.py diff --git a/app/YtManagerApp/scheduler/job.py b/app/YtManagerApp/services/scheduler/job.py similarity index 98% rename from app/YtManagerApp/scheduler/job.py rename to app/YtManagerApp/services/scheduler/job.py index 34731be..6be4c23 100644 --- a/app/YtManagerApp/scheduler/job.py +++ b/app/YtManagerApp/services/scheduler/job.py @@ -1,12 +1,12 @@ import logging -from abc import abstractmethod +from abc import abstractmethod, ABC from typing import Optional from YtManagerApp.models import JOB_MESSAGE_LEVELS_MAP, JobMessage from .progress_tracker import ProgressTracker -class Job(object): +class Job(ABC): name = 'GenericJob' """ diff --git a/app/YtManagerApp/scheduler/jobs/__init__.py b/app/YtManagerApp/services/scheduler/jobs/__init__.py similarity index 100% rename from app/YtManagerApp/scheduler/jobs/__init__.py rename to app/YtManagerApp/services/scheduler/jobs/__init__.py diff --git a/app/YtManagerApp/scheduler/jobs/delete_video_job.py b/app/YtManagerApp/services/scheduler/jobs/delete_video_job.py similarity index 82% rename from app/YtManagerApp/scheduler/jobs/delete_video_job.py rename to app/YtManagerApp/services/scheduler/jobs/delete_video_job.py index 39141f7..b6b9aaf 100644 --- a/app/YtManagerApp/scheduler/jobs/delete_video_job.py +++ b/app/YtManagerApp/services/scheduler/jobs/delete_video_job.py @@ -1,7 +1,7 @@ import os from YtManagerApp.models import Video -from YtManagerApp.scheduler.job import Job +from YtManagerApp.services.scheduler.job import Job class DeleteVideoJob(Job): @@ -28,13 +28,13 @@ class DeleteVideoJob(Job): except OSError as e: self.log.error("Failed to delete video %d [%s %s]. Error: %s", self._video.id, - self._video.video_id, self._video.name, e) + self._video.provider_id, self._video.name, e) self._video.downloaded_path = None self._video.save() self.log.info('Deleted video %d successfully! (%d files) [%s %s]', self._video.id, count, - self._video.video_id, self._video.name) + self._video.provider_id, self._video.name) @staticmethod def schedule(video: Video): @@ -44,4 +44,4 @@ class DeleteVideoJob(Job): :return: """ from YtManagerApp.services import Services - Services.scheduler.add_job(DeleteVideoJob, args=[video]) + Services.scheduler().add_job(DeleteVideoJob, args=[video]) diff --git a/app/YtManagerApp/scheduler/jobs/download_video_job.py b/app/YtManagerApp/services/scheduler/jobs/download_video_job.py similarity index 97% rename from app/YtManagerApp/scheduler/jobs/download_video_job.py rename to app/YtManagerApp/services/scheduler/jobs/download_video_job.py index 14984c6..157ea42 100644 --- a/app/YtManagerApp/scheduler/jobs/download_video_job.py +++ b/app/YtManagerApp/services/scheduler/jobs/download_video_job.py @@ -6,7 +6,7 @@ from threading import Lock import youtube_dl from YtManagerApp.models import Video -from YtManagerApp.scheduler.job import Job +from YtManagerApp.services.scheduler.job import Job class DownloadVideoJob(Job): @@ -132,5 +132,5 @@ class DownloadVideoJob(Job): :param attempt: :return: """ - from YtManagerApp.services import Services + from YtManagerApp.management.services import Services Services.scheduler.add_job(DownloadVideoJob, args=[video, attempt]) diff --git a/app/YtManagerApp/scheduler/jobs/synchronize_job.py b/app/YtManagerApp/services/scheduler/jobs/synchronize_job.py similarity index 94% rename from app/YtManagerApp/scheduler/jobs/synchronize_job.py rename to app/YtManagerApp/services/scheduler/jobs/synchronize_job.py index d1935e7..6886801 100644 --- a/app/YtManagerApp/scheduler/jobs/synchronize_job.py +++ b/app/YtManagerApp/services/scheduler/jobs/synchronize_job.py @@ -6,11 +6,10 @@ from apscheduler.triggers.cron import CronTrigger from django.db.models import Max from django.conf import settings -from YtManagerApp.management.downloader import fetch_thumbnail, downloader_process_subscription +from YtManagerApp.services.downloader import fetch_thumbnail, downloader_process_subscription from YtManagerApp.models import * -from YtManagerApp.scheduler.job import Job +from YtManagerApp.services.scheduler.job import Job from YtManagerApp.services import Services -from YtManagerApp.utils import youtube from external.pytaw.pytaw.utils import iterate_chunks _ENABLE_UPDATE_STATS = True @@ -115,7 +114,7 @@ class SynchronizeJob(Job): if isinstance(obj, Subscription): obj.thumbnail = fetch_thumbnail(obj.thumbnail, 'sub', obj.playlist_id, settings.THUMBNAIL_SIZE_SUBSCRIPTION) elif isinstance(obj, Video): - obj.thumbnail = fetch_thumbnail(obj.thumbnail, 'video', obj.video_id, settings.THUMBNAIL_SIZE_VIDEO) + obj.thumbnail = fetch_thumbnail(obj.thumbnail, 'video', obj.provider_id, settings.THUMBNAIL_SIZE_VIDEO) obj.save() def check_video_deleted(self, video: Video): @@ -138,7 +137,7 @@ class SynchronizeJob(Job): # Video not found, we can safely assume that the video was deleted. if not found_video: - self.log.info("Video %d was deleted! [%s %s]", video.id, video.video_id, video.name) + self.log.info("Video %d was deleted! [%s %s]", video.id, video.provider_id, video.name) # Clean up for file in files: try: @@ -166,11 +165,11 @@ class SynchronizeJob(Job): @staticmethod def schedule_global_job(): - trigger = CronTrigger.from_crontab(Services.appConfig.sync_schedule) + trigger = CronTrigger.from_crontab(Services.appConfig().sync_schedule) if SynchronizeJob.__global_sync_job is None: - trigger = CronTrigger.from_crontab(Services.appConfig.sync_schedule) - SynchronizeJob.__global_sync_job = Services.scheduler.add_job(SynchronizeJob, trigger, max_instances=1, coalesce=True) + trigger = CronTrigger.from_crontab(Services.appConfig().sync_schedule) + SynchronizeJob.__global_sync_job = Services.scheduler().add_job(SynchronizeJob, trigger, max_instances=1, coalesce=True) else: SynchronizeJob.__global_sync_job.reschedule(trigger, max_instances=1, coalesce=True) diff --git a/app/YtManagerApp/services/scheduler/jobs/youtubedl_update_job.py b/app/YtManagerApp/services/scheduler/jobs/youtubedl_update_job.py new file mode 100644 index 0000000..be69eb8 --- /dev/null +++ b/app/YtManagerApp/services/scheduler/jobs/youtubedl_update_job.py @@ -0,0 +1,18 @@ +from YtManagerApp.models import Video +from YtManagerApp.services.scheduler.job import Job + + +class YouTubeDLUpdateJob(Job): + name = "YouTubeDLUpdateJob" + + def __init__(self, job_execution): + super().__init__(job_execution) + + def get_description(self): + return f"Updating youtube-dl runtime" + + def run(self): + from YtManagerApp.services import Services + self.set_total_steps(1) + Services.youtubeDLManager.install() + self.progress_advance(1) diff --git a/app/YtManagerApp/scheduler/progress_tracker.py b/app/YtManagerApp/services/scheduler/progress_tracker.py similarity index 100% rename from app/YtManagerApp/scheduler/progress_tracker.py rename to app/YtManagerApp/services/scheduler/progress_tracker.py diff --git a/app/YtManagerApp/scheduler/scheduler.py b/app/YtManagerApp/services/scheduler/scheduler.py similarity index 86% rename from app/YtManagerApp/scheduler/scheduler.py rename to app/YtManagerApp/services/scheduler/scheduler.py index 15bb40f..d095c1d 100644 --- a/app/YtManagerApp/scheduler/scheduler.py +++ b/app/YtManagerApp/services/scheduler/scheduler.py @@ -6,11 +6,13 @@ from typing import Type, Union, Optional import pytz from apscheduler.schedulers.background import BackgroundScheduler from apscheduler.triggers.base import BaseTrigger +from apscheduler.triggers.interval import IntervalTrigger from django.contrib.auth.models import User -from YtManagerApp.management.appconfig import AppConfig +from YtManagerApp.services.appconfig import AppConfig from YtManagerApp.models import JobExecution, JOB_STATES_MAP -from YtManagerApp.scheduler.job import Job +from YtManagerApp.services.scheduler.job import Job +from YtManagerApp.services.scheduler.jobs.youtubedl_update_job import YouTubeDLUpdateJob class YtsmScheduler(object): @@ -27,6 +29,10 @@ class YtsmScheduler(object): self._configure_scheduler() self._ap_scheduler.start() + self._schedule_main_jobs() + + def _schedule_main_jobs(self): + self.add_job(YouTubeDLUpdateJob, trigger=IntervalTrigger(days=1)) def _configure_scheduler(self): logger = logging.getLogger('scheduler') diff --git a/app/YtManagerApp/services/video_provider_manager.py b/app/YtManagerApp/services/video_provider_manager.py new file mode 100644 index 0000000..a22c910 --- /dev/null +++ b/app/YtManagerApp/services/video_provider_manager.py @@ -0,0 +1,11 @@ +from YtManagerApp.services.providers.video_provider import VideoProvider + + +class VideoProviderManager(object): + + def __init__(self): + self._providers: Dict[str, VideoProvider] = {} + + def register_provider(self, provider: VideoProvider): + pid = provider.get_provider_id() + self._providers[pid] = provider diff --git a/app/YtManagerApp/management/videos.py b/app/YtManagerApp/services/videos.py similarity index 100% rename from app/YtManagerApp/management/videos.py rename to app/YtManagerApp/services/videos.py diff --git a/app/YtManagerApp/management/youtube_dl_manager.py b/app/YtManagerApp/services/youtube_dl_manager.py similarity index 81% rename from app/YtManagerApp/management/youtube_dl_manager.py rename to app/YtManagerApp/services/youtube_dl_manager.py index 61eb291..49a4ff4 100644 --- a/app/YtManagerApp/management/youtube_dl_manager.py +++ b/app/YtManagerApp/services/youtube_dl_manager.py @@ -28,12 +28,10 @@ class YoutubeDlManager(object): def __init__(self): self.verbose = False self.progress = False + self._install_path = os.path.join(dj_settings.DATA_DIR, 'youtube-dl') - def _get_path(self): - return os.path.join(dj_settings.DATA_DIR, 'youtube-dl') - - def _check_installed(self, path): - return os.path.isfile(path) and os.access(path, os.X_OK) + def _check_installed(self): + return os.path.isfile(self._install_path) and os.access(self._install_path, os.X_OK) def _get_run_args(self): run_args = [] @@ -47,13 +45,12 @@ class YoutubeDlManager(object): return run_args def run(self, *args): - path = self._get_path() - if not self._check_installed(path): + if not self._check_installed(): log.error("Cannot run youtube-dl, it is not installed!") raise YoutubeDlNotInstalledException run_args = self._get_run_args() - ret = subprocess.run([sys.executable, path, *run_args, *args], stdout=subprocess.PIPE, stderr=subprocess.PIPE) + ret = subprocess.run([sys.executable, self._install_path, *run_args, *args], stdout=subprocess.PIPE, stderr=subprocess.PIPE) stdout = ret.stdout.decode('utf-8') if len(stdout) > 0: @@ -93,15 +90,14 @@ class YoutubeDlManager(object): resp = requests.get(LATEST_URL, allow_redirects=True, stream=True) resp.raise_for_status() - path = self._get_path() - with open(path + ".tmp", "wb") as f: + with open(self._install_path + ".tmp", "wb") as f: for chunk in resp.iter_content(10 * 1024): f.write(chunk) # Replace - os.unlink(path) - os.rename(path + ".tmp", path) - os.chmod(path, 555) + os.unlink(self._install_path) + os.rename(self._install_path + ".tmp", self._install_path) + os.chmod(self._install_path, 555) # Test run newver = self.get_installed_version() diff --git a/app/YtManagerApp/utils/youtube.py b/app/YtManagerApp/utils/youtube.py deleted file mode 100644 index d30097d..0000000 --- a/app/YtManagerApp/utils/youtube.py +++ /dev/null @@ -1,49 +0,0 @@ -from django.conf import settings -from external.pytaw.pytaw.youtube import YouTube, Channel, Playlist, PlaylistItem, Thumbnail, InvalidURL, Resource, Video -from typing import Optional - - -class YoutubeAPI(YouTube): - - @staticmethod - def build_public() -> 'YoutubeAPI': - from YtManagerApp.management.appconfig import appconfig - return YoutubeAPI(key=appconfig.youtube_api_key) - - # @staticmethod - # def build_oauth() -> 'YoutubeAPI': - # flow = - # credentials = - # service = build(API_SERVICE_NAME, API_VERSION, credentials) - - -def default_thumbnail(resource: Resource) -> Optional[Thumbnail]: - """ - Gets the default thumbnail for a resource. - Searches in the list of thumbnails for one with the label 'default', or takes the first one. - :param resource: - :return: - """ - thumbs = getattr(resource, 'thumbnails', None) - - if thumbs is None or len(thumbs) <= 0: - return None - - return next( - (i for i in thumbs if i.id == 'default'), - thumbs[0] - ) - - -def best_thumbnail(resource: Resource) -> Optional[Thumbnail]: - """ - Gets the best thumbnail available for a resource. - :param resource: - :return: - """ - thumbs = getattr(resource, 'thumbnails', None) - - if thumbs is None or len(thumbs) <= 0: - return None - - return max(thumbs, key=lambda t: t.width * t.height) diff --git a/app/YtManagerApp/views/actions.py b/app/YtManagerApp/views/actions.py index 35e0e84..c2dd63a 100644 --- a/app/YtManagerApp/views/actions.py +++ b/app/YtManagerApp/views/actions.py @@ -3,7 +3,7 @@ from django.http import JsonResponse from django.views.generic import View from YtManagerApp.models import Video -from YtManagerApp.scheduler.jobs.synchronize_job import SynchronizeJob +from YtManagerApp.services.scheduler.jobs.synchronize_job import SynchronizeJob class SyncNowView(LoginRequiredMixin, View): diff --git a/app/YtManagerApp/views/auth.py b/app/YtManagerApp/views/auth.py index 4f07836..85deea7 100644 --- a/app/YtManagerApp/views/auth.py +++ b/app/YtManagerApp/views/auth.py @@ -36,7 +36,7 @@ class RegisterView(FormView): return context def post(self, request, *args, **kwargs): - if not Services.appConfig.allow_registrations: + if not Services.appConfig().allow_registrations: return HttpResponseForbidden("Registrations are disabled!") return super().post(request, *args, **kwargs) diff --git a/app/YtManagerApp/views/first_time.py b/app/YtManagerApp/views/first_time.py index fe3ef43..bdb82ed 100644 --- a/app/YtManagerApp/views/first_time.py +++ b/app/YtManagerApp/views/first_time.py @@ -8,7 +8,7 @@ from django.shortcuts import redirect from django.urls import reverse_lazy from django.views.generic import FormView -from YtManagerApp.scheduler.jobs.synchronize_job import SynchronizeJob +from YtManagerApp.services.scheduler.jobs.synchronize_job import SynchronizeJob from YtManagerApp.services import Services from YtManagerApp.views.forms.first_time import WelcomeForm, ApiKeyForm, PickAdminUserForm, ServerConfigForm, DoneForm, \ UserCreationForm, LoginForm @@ -24,7 +24,7 @@ class WizardStepMixin: def get(self, request, *args, **kwargs): # Prevent access if application is already initialized - if Services.appConfig.initialized: + if Services.appConfig().initialized: logger.debug(f"Attempted to access {request.path}, but first time setup already run. Redirected to home " f"page.") return redirect('home') @@ -32,7 +32,7 @@ class WizardStepMixin: return super().get(request, *args, **kwargs) def post(self, request, *args, **kwargs): - if Services.appConfig.initialized: + if Services.appConfig().initialized: logger.debug(f"Attempted to post {request.path}, but first time setup already run.") return HttpResponseForbidden() return super().post(request, *args, **kwargs) @@ -65,14 +65,14 @@ class Step1ApiKeyView(WizardStepMixin, FormView): def get_initial(self): initial = super().get_initial() - initial['api_key'] = Services.appConfig.youtube_api_key + initial['api_key'] = Services.appConfig().youtube_api_key return initial def form_valid(self, form): key = form.cleaned_data['api_key'] # TODO: validate key if key is not None and len(key) > 0: - Services.appConfig.youtube_api_key = key + Services.appConfig().youtube_api_key = key return super().form_valid(form) @@ -149,8 +149,8 @@ class Step3ConfigureView(WizardStepMixin, FormView): def get_initial(self): initial = super().get_initial() - initial['allow_registrations'] = Services.appConfig.allow_registrations - initial['sync_schedule'] = Services.appConfig.sync_schedule + initial['allow_registrations'] = Services.appConfig().allow_registrations + initial['sync_schedule'] = Services.appConfig().sync_schedule initial['auto_download'] = self.request.user.preferences['auto_download'] initial['download_location'] = self.request.user.preferences['download_path'] return initial @@ -158,11 +158,11 @@ class Step3ConfigureView(WizardStepMixin, FormView): def form_valid(self, form): allow_registrations = form.cleaned_data['allow_registrations'] if allow_registrations is not None: - Services.appConfig.allow_registrations = allow_registrations + Services.appConfig().allow_registrations = allow_registrations sync_schedule = form.cleaned_data['sync_schedule'] if sync_schedule is not None and len(sync_schedule) > 0: - Services.appConfig.sync_schedule = sync_schedule + Services.appConfig().sync_schedule = sync_schedule auto_download = form.cleaned_data['auto_download'] if auto_download is not None: @@ -173,7 +173,7 @@ class Step3ConfigureView(WizardStepMixin, FormView): self.request.user.preferences['download_path'] = download_location # Set initialized to true - Services.appConfig.initialized = True + Services.appConfig().initialized = True # Start scheduler if not started Services.scheduler.initialize() diff --git a/app/YtManagerApp/views/forms/settings.py b/app/YtManagerApp/views/forms/settings.py index f9b6f58..e9a6857 100644 --- a/app/YtManagerApp/views/forms/settings.py +++ b/app/YtManagerApp/views/forms/settings.py @@ -7,7 +7,7 @@ from YtManagerApp.dynamic_preferences_registry import MarkDeletedAsWatched, Auto DownloadPath, DownloadFilePattern, DownloadFormat, DownloadSubtitles, DownloadAutogeneratedSubtitles, \ DownloadAllSubtitles, DownloadSubtitlesLangs, DownloadSubtitlesFormat from YtManagerApp.models import VIDEO_ORDER_CHOICES -from YtManagerApp.services import Services +from YtManagerApp.management.services import Services class SettingsForm(forms.Form): diff --git a/app/YtManagerApp/views/index.py b/app/YtManagerApp/views/index.py index dd49d79..b79eb7e 100644 --- a/app/YtManagerApp/views/index.py +++ b/app/YtManagerApp/views/index.py @@ -12,7 +12,7 @@ from django.conf import settings from django.core.paginator import Paginator from YtManagerApp.management.videos import get_videos from YtManagerApp.models import Subscription, SubscriptionFolder, VIDEO_ORDER_CHOICES, VIDEO_ORDER_MAPPING -from YtManagerApp.services import Services +from YtManagerApp.management.services import Services from YtManagerApp.utils import youtube, subscription_file_parser from YtManagerApp.views.controls.modal import ModalMixin diff --git a/app/YtManagerApp/views/settings.py b/app/YtManagerApp/views/settings.py index 9d621ac..647efeb 100644 --- a/app/YtManagerApp/views/settings.py +++ b/app/YtManagerApp/views/settings.py @@ -3,7 +3,7 @@ from django.http import HttpResponseForbidden from django.urls import reverse_lazy from django.views.generic import FormView -from YtManagerApp.scheduler.jobs.synchronize_job import SynchronizeJob +from YtManagerApp.management.scheduler.jobs.synchronize_job import SynchronizeJob from YtManagerApp.views.forms.settings import SettingsForm, AdminSettingsForm