Major refactor of codebase.

app/YtManagerApp/appmain.py

@@ -5,7 +5,6 @@ import sys
 
 from django.conf import settings as dj_settings
 
-from YtManagerApp.scheduler.jobs.synchronize_job import SynchronizeJob
 from YtManagerApp.services import Services
 from django.db.utils import OperationalError
 
@@ -35,9 +34,9 @@ def main():
     __initialize_logger()
 
     try:
-        if Services.appConfig.initialized:
+        if Services.appConfig().initialized:
-            Services.scheduler.initialize()
+            Services.scheduler().initialize()
-            SynchronizeJob.schedule_global_job()
+            Services.subscriptionManager().schedule_global_synchronize_job()
     except OperationalError:
         # Settings table is not created when running migrate or makemigrations;
         # Just don't do anything in this case.

app/YtManagerApp/management/download_manager.py

@@ -0,0 +1,255 @@
+import logging
+import mimetypes
+import os
+import re
+from string import Template
+from threading import Lock
+from urllib.parse import urljoin
+
+import PIL.Image
+import PIL.ImageOps
+import requests
+import youtube_dl
+from django.conf import settings as srv_settings
+
+from YtManagerApp.models import Subscription, Video
+from YtManagerApp.models import VIDEO_ORDER_MAPPING
+from YtManagerApp.providers.video_provider import VideoProvider
+from YtManagerApp.scheduler.job import Job
+from YtManagerApp.utils import first_non_null
+
+log = logging.getLogger('DownloadManager')
+
+
+class DownloadVideoJob(Job):
+    """
+    Downloads a video to the disk
+    """
+    name = "DownloadVideoJob"
+    __lock = Lock()
+
+    def __init__(self, job_execution, video: Video, attempt: int = 1):
+        super().__init__(job_execution)
+        self._video = video
+        self._attempt = attempt
+        self._log_youtube_dl = self.log.getChild('youtube_dl')
+
+    def get_description(self):
+        ret = "Downloading video " + self._video.name
+        if self._attempt > 1:
+            ret += f" (attempt {self._attempt})"
+        return ret
+
+    def run(self):
+        from YtManagerApp.services import Services
+
+        # Issue: if multiple videos are downloaded at the same time, a race condition appears in the mkdirs() call that
+        # youtube-dl makes, which causes it to fail with the error 'Cannot create folder - file already exists'.
+        # For now, allow a single download instance.
+        self.__lock.acquire()
+
+        try:
+            user = self._video.subscription.user
+            provider: VideoProvider = Services.videoProviderManager().get(self._video)
+
+            max_attempts = user.preferences['max_download_attempts']
+
+            youtube_dl_params, output_path = self.__build_youtube_dl_params(self._video)
+            with youtube_dl.YoutubeDL(youtube_dl_params) as yt:
+                ret = yt.download([provider.get_video_url(self._video)])
+
+            self.log.info('Download finished with code %d', ret)
+
+            if ret == 0:
+                self._video.downloaded_path = output_path
+                self._video.save()
+                self.log.info('Video %d [%s %s] downloaded successfully!', self._video.id, self._video.video_id,
+                              self._video.name)
+
+                # update size
+                self._video.downloaded_size = 0
+                for file in self._video.get_files():
+                    self._video.downloaded_size += os.stat(file).st_size
+                self._video.save()
+
+            elif self._attempt <= max_attempts:
+                self.log.warning('Re-enqueueing video (attempt %d/%d)', self._attempt, max_attempts)
+                Services.videoManager().download(self._video, self._attempt + 1)
+
+            else:
+                self.log.error('Multiple attempts to download video %d [%s %s] failed!', self._video.id,
+                               self._video.video_id, self._video.name)
+                self._video.downloaded_path = ''
+                self._video.save()
+
+        finally:
+            self.__lock.release()
+
+    def __build_youtube_dl_params(self, video: Video):
+
+        sub = video.subscription
+        user = sub.user
+
+        # resolve path
+        download_path = user.preferences['download_path']
+
+        template_dict = self.__build_template_dict(video)
+        output_pattern = Template(user.preferences['download_file_pattern']).safe_substitute(template_dict)
+
+        output_path = os.path.join(download_path, output_pattern)
+        output_path = os.path.normpath(output_path)
+
+        youtube_dl_params = {
+            'logger': self._log_youtube_dl,
+            'format': user.preferences['download_format'],
+            'outtmpl': output_path,
+            'writethumbnail': True,
+            'writedescription': True,
+            'writesubtitles': user.preferences['download_subtitles'],
+            'writeautomaticsub': user.preferences['download_autogenerated_subtitles'],
+            'allsubtitles': user.preferences['download_subtitles_all'],
+            'merge_output_format': 'mp4',
+            'postprocessors': [
+                {
+                    'key': 'FFmpegMetadata'
+                },
+            ]
+        }
+
+        sub_langs = user.preferences['download_subtitles_langs'].split(',')
+        sub_langs = [i.strip() for i in sub_langs]
+        if len(sub_langs) > 0:
+            youtube_dl_params['subtitleslangs'] = sub_langs
+
+        sub_format = user.preferences['download_subtitles_format']
+        if len(sub_format) > 0:
+            youtube_dl_params['subtitlesformat'] = sub_format
+
+        return youtube_dl_params, output_path
+
+    def __build_template_dict(self, video: Video):
+        return {
+            'channel': video.subscription.channel_name,
+            'channel_id': video.subscription.channel_id,
+            'playlist': video.subscription.name,
+            'playlist_id': video.subscription.playlist_id,
+            'playlist_index': "{:03d}".format(1 + video.playlist_index),
+            'title': video.name,
+            'id': video.video_id,
+        }
+
+    def __get_valid_path(self, path: str):
+        """
+        Normalizes string, converts to lowercase, removes non-alpha characters, removes forbidden characters.
+        """
+        import unicodedata
+        value = unicodedata.normalize('NFKD', path).encode('ascii', 'ignore').decode('ascii')
+        value = re.sub('[:"*]', '', value).strip()
+        value = re.sub('[?<>|]', '#', value)
+        return value
+
+
+class DownloadManager(object):
+
+    def __init__(self):
+        pass
+
+    def download_video(self, video: Video, attempt: int = 1):
+        from YtManagerApp.services import Services
+        Services.scheduler().add_job(DownloadVideoJob, args=[video, attempt])
+
+    def __get_subscription_config(self, sub: Subscription):
+        user = sub.user
+
+        enabled = first_non_null(sub.auto_download, user.preferences['auto_download'])
+        global_limit = user.preferences['download_global_limit']
+        limit = first_non_null(sub.download_limit, user.preferences['download_subscription_limit'])
+        order = first_non_null(sub.download_order, user.preferences['download_order'])
+        order = VIDEO_ORDER_MAPPING[order]
+
+        return enabled, global_limit, limit, order
+
+    def process_subscription(self, sub: Subscription):
+        from YtManagerApp.services import Services
+
+        log.info('Processing subscription %d [%s %s]', sub.id, sub.playlist_id, sub.id)
+
+        enabled, global_limit, limit, order = self.__get_subscription_config(sub)
+        log.info('Determined settings enabled=%s global_limit=%d limit=%d order="%s"', enabled, global_limit, limit, order)
+
+        if enabled:
+            videos_to_download = Video.objects\
+                .filter(subscription=sub, downloaded_path__isnull=True, watched=False)\
+                .order_by(order)
+
+            log.info('%d download candidates.', len(videos_to_download))
+
+            if global_limit > 0:
+                global_downloaded = Video.objects.filter(subscription__user=sub.user, downloaded_path__isnull=False).count()
+                allowed_count = max(global_limit - global_downloaded, 0)
+                videos_to_download = videos_to_download[0:allowed_count]
+                log.info('Global limit is set, can only download up to %d videos.', allowed_count)
+
+            if limit > 0:
+                sub_downloaded = Video.objects.filter(subscription=sub, downloaded_path__isnull=False).count()
+                allowed_count = max(limit - sub_downloaded, 0)
+                videos_to_download = videos_to_download[0:allowed_count]
+                log.info('Limit is set, can only download up to %d videos.', allowed_count)
+
+            # enqueue download
+            for video in videos_to_download:
+                log.info('Enqueuing video %d [%s %s] index=%d', video.id, video.video_id, video.name, video.playlist_index)
+                Services.videoManager().download(video)
+
+        log.info('Finished processing subscription %d [%s %s]', sub.id, sub.playlist_id, sub.id)
+
+    def process_all_subscriptions(self):
+        for subscription in Subscription.objects.all():
+            self.process_subscription(subscription)
+
+    def fetch_thumbnail(self, url, object_type, identifier, thumb_size):
+
+        log.info('Fetching thumbnail url=%s object_type=%s identifier=%s', url, object_type, identifier)
+
+        # Make request to obtain mime type
+        try:
+            response = requests.get(url, stream=True)
+        except requests.exceptions.RequestException as e:
+            log.error('Failed to fetch thumbnail %s. Error: %s', url, e)
+            return url
+
+        ext = mimetypes.guess_extension(response.headers['Content-Type'])
+
+        # Build file path
+        file_name = f"{identifier}{ext}"
+        abs_path_dir = os.path.join(srv_settings.MEDIA_ROOT, "thumbs", object_type)
+        abs_path = os.path.join(abs_path_dir, file_name)
+        abs_path_tmp = file_name + '.tmp'
+
+        # Store image
+        try:
+            os.makedirs(abs_path_dir, exist_ok=True)
+            with open(abs_path_tmp, "wb") as f:
+                for chunk in response.iter_content(chunk_size=1024):
+                    if chunk:
+                        f.write(chunk)
+
+            # Resize and crop to thumbnail size
+            image = PIL.Image.open(abs_path_tmp)
+            image = PIL.ImageOps.fit(image, thumb_size)
+            image.save(abs_path)
+            image.close()
+
+            # Delete temp file
+            os.unlink(abs_path_tmp)
+
+        except requests.exceptions.RequestException as e:
+            log.error('Error while downloading stream for thumbnail %s. Error: %s', url, e)
+            return url
+        except OSError as e:
+            log.error('Error while writing to file %s for thumbnail %s. Error: %s', abs_path, url, e)
+            return url
+
+        # Return
+        media_url = urljoin(srv_settings.MEDIA_URL, f"thumbs/{object_type}/{file_name}")
+        return media_url

app/YtManagerApp/management/downloader.py

@@ -1,111 +0,0 @@
-from YtManagerApp.scheduler.jobs.download_video_job import DownloadVideoJob
-from YtManagerApp.models import Video, Subscription, VIDEO_ORDER_MAPPING
-from YtManagerApp.utils import first_non_null
-from django.conf import settings as srv_settings
-import logging
-import requests
-import mimetypes
-import os
-import PIL.Image
-import PIL.ImageOps
-from urllib.parse import urljoin
-
-log = logging.getLogger('downloader')
-
-
-def __get_subscription_config(sub: Subscription):
-    user = sub.user
-
-    enabled = first_non_null(sub.auto_download, user.preferences['auto_download'])
-    global_limit = user.preferences['download_global_limit']
-    limit = first_non_null(sub.download_limit, user.preferences['download_subscription_limit'])
-    order = first_non_null(sub.download_order, user.preferences['download_order'])
-    order = VIDEO_ORDER_MAPPING[order]
-
-    return enabled, global_limit, limit, order
-
-
-def downloader_process_subscription(sub: Subscription):
-    log.info('Processing subscription %d [%s %s]', sub.id, sub.playlist_id, sub.id)
-
-    enabled, global_limit, limit, order = __get_subscription_config(sub)
-    log.info('Determined settings enabled=%s global_limit=%d limit=%d order="%s"', enabled, global_limit, limit, order)
-
-    if enabled:
-        videos_to_download = Video.objects\
-            .filter(subscription=sub, downloaded_path__isnull=True, watched=False)\
-            .order_by(order)
-
-        log.info('%d download candidates.', len(videos_to_download))
-
-        if global_limit > 0:
-            global_downloaded = Video.objects.filter(subscription__user=sub.user, downloaded_path__isnull=False).count()
-            allowed_count = max(global_limit - global_downloaded, 0)
-            videos_to_download = videos_to_download[0:allowed_count]
-            log.info('Global limit is set, can only download up to %d videos.', allowed_count)
-
-        if limit > 0:
-            sub_downloaded = Video.objects.filter(subscription=sub, downloaded_path__isnull=False).count()
-            allowed_count = max(limit - sub_downloaded, 0)
-            videos_to_download = videos_to_download[0:allowed_count]
-            log.info('Limit is set, can only download up to %d videos.', allowed_count)
-
-        # enqueue download
-        for video in videos_to_download:
-            log.info('Enqueuing video %d [%s %s] index=%d', video.id, video.video_id, video.name, video.playlist_index)
-            DownloadVideoJob.schedule(video)
-
-    log.info('Finished processing subscription %d [%s %s]', sub.id, sub.playlist_id, sub.id)
-
-
-def downloader_process_all():
-    for subscription in Subscription.objects.all():
-        downloader_process_subscription(subscription)
-
-
-def fetch_thumbnail(url, object_type, identifier, thumb_size):
-
-    log.info('Fetching thumbnail url=%s object_type=%s identifier=%s', url, object_type, identifier)
-
-    # Make request to obtain mime type
-    try:
-        response = requests.get(url, stream=True)
-    except requests.exceptions.RequestException as e:
-        log.error('Failed to fetch thumbnail %s. Error: %s', url, e)
-        return url
-
-    ext = mimetypes.guess_extension(response.headers['Content-Type'])
-
-    # Build file path
-    file_name = f"{identifier}{ext}"
-    abs_path_dir = os.path.join(srv_settings.MEDIA_ROOT, "thumbs", object_type)
-    abs_path = os.path.join(abs_path_dir, file_name)
-    abs_path_tmp = file_name + '.tmp'
-
-    # Store image
-    try:
-        os.makedirs(abs_path_dir, exist_ok=True)
-        with open(abs_path_tmp, "wb") as f:
-            for chunk in response.iter_content(chunk_size=1024):
-                if chunk:
-                    f.write(chunk)
-
-        # Resize and crop to thumbnail size
-        image = PIL.Image.open(abs_path_tmp)
-        image = PIL.ImageOps.fit(image, thumb_size)
-        image.save(abs_path)
-        image.close()
-
-        # Delete temp file
-        os.unlink(abs_path_tmp)
-
-    except requests.exceptions.RequestException as e:
-        log.error('Error while downloading stream for thumbnail %s. Error: %s', url, e)
-        return url
-    except OSError as e:
-        log.error('Error while writing to file %s for thumbnail %s. Error: %s', abs_path, url, e)
-        return url
-
-    # Return
-    media_url = urljoin(srv_settings.MEDIA_URL, f"thumbs/{object_type}/{file_name}")
-    return media_url

app/YtManagerApp/management/subscription_manager.py

@@ -0,0 +1,246 @@
+import errno
+import itertools
+import mimetypes
+import os
+from threading import Lock
+from typing import Optional, List, Union
+
+from apscheduler.triggers.cron import CronTrigger
+from django.conf import settings
+from django.db.models import Max
+
+from YtManagerApp.models import *
+from YtManagerApp.providers.video_provider import VideoProvider, InvalidURLError
+from YtManagerApp.scheduler.job import Job
+from YtManagerApp.utils.algorithms import group_by
+
+_ENABLE_UPDATE_STATS = True
+
+
+class SynchronizeJob(Job):
+    name = "SynchronizeJob"
+    __lock = Lock()
+    running = False
+    __global_sync_job = None
+
+    def __init__(self, job_execution, subscription: Optional[Subscription] = None):
+        super().__init__(job_execution)
+        self.__subscription: Optional[Subscription] = subscription
+        self.__new_vids: List[Video] = []
+
+    def get_description(self):
+        if self.__subscription is not None:
+            return "Running synchronization for subscription " + self.__subscription.name
+        return "Running synchronization..."
+
+    def get_subscription_list(self):
+        if self.__subscription is not None:
+            return [self.__subscription]
+        return Subscription.objects.all()
+
+    def get_videos_list(self, subs):
+        return Video.objects.filter(subscription__in=subs)
+
+    def run(self):
+        from YtManagerApp.services import Services
+
+        self.__lock.acquire(blocking=True)
+        SynchronizeJob.running = True
+        try:
+            self.log.info(self.get_description())
+
+            # Build list of work items
+            work_subs = self.get_subscription_list()
+            work_vids = self.get_videos_list(work_subs)
+
+            self.set_total_steps(len(work_subs) + len(work_vids))
+
+            # Remove the 'new' flag
+            work_vids.update(new=False)
+
+            # Process subscriptions
+            for sub in work_subs:
+                self.progress_advance(1, "Synchronizing subscription " + sub.name)
+                self.check_new_videos(sub)
+                self.fetch_missing_thumbnails(sub)
+
+            # Add new videos to progress calculation
+            self.set_total_steps(len(work_subs) + len(work_vids) + len(self.__new_vids))
+
+            # Group videos by provider
+            all_videos = itertools.chain(work_vids, self.__new_vids)
+            all_videos_by_provider = group_by(all_videos, lambda x: x.subscription.provider_id)
+
+            for provider_id, videos in all_videos_by_provider.items():
+                provider: VideoProvider = Services.videoProviderManager().get(provider_id)
+                if _ENABLE_UPDATE_STATS:
+                    provider.update_videos(videos, update_statistics=True)
+
+                for video in videos:
+                    self.progress_advance(1, "Updating video " + video.name)
+                    self.check_video_deleted(video)
+                    self.fetch_missing_thumbnails(video)
+
+            # Start downloading videos
+            for sub in work_subs:
+                Services.downloadManager().process_subscription(sub)
+
+        finally:
+            SynchronizeJob.running = False
+            self.__lock.release()
+
+    def check_new_videos(self, sub: Subscription):
+        from YtManagerApp.services import Services
+        provider: VideoProvider = Services.videoProviderManager().get(sub)
+
+        playlist_videos = provider.fetch_videos(sub)
+        if sub.rewrite_playlist_indices:
+            playlist_videos = sorted(playlist_videos, key=lambda x: x.publish_date)
+        else:
+            playlist_videos = sorted(playlist_videos, key=lambda x: x.playlist_index)
+
+        for item in playlist_videos:
+            results = Video.objects.filter(video_id=item.video_id, subscription=sub)
+
+            if not results.exists():
+                self.log.info('New video for subscription %s: %s %s"', sub, item.video_id, item.name)
+
+                # fix playlist index if necessary
+                if sub.rewrite_playlist_indices or Video.objects.filter(subscription=sub,
+                                                                        playlist_index=item.playlist_index).exists():
+                    highest = Video.objects.filter(subscription=sub).aggregate(Max('playlist_index'))[
+                        'playlist_index__max']
+                    item.playlist_index = 1 + (highest or -1)
+
+                item.save()
+                self.__new_vids.append(item)
+
+    def fetch_missing_thumbnails(self, obj: Union[Subscription, Video]):
+        from YtManagerApp.services import Services
+        if obj.thumbnail.startswith("http"):
+            if isinstance(obj, Subscription):
+                obj.thumbnail = Services.downloadManager().fetch_thumbnail(obj.thumbnail, 'sub', obj.playlist_id,
+                                                                           settings.THUMBNAIL_SIZE_SUBSCRIPTION)
+            elif isinstance(obj, Video):
+                obj.thumbnail = Services.downloadManager().fetch_thumbnail(obj.thumbnail, 'video', obj.video_id,
+                                                                           settings.THUMBNAIL_SIZE_VIDEO)
+            obj.save()
+
+    def check_video_deleted(self, video: Video):
+        if video.downloaded_path is not None:
+            files = []
+            try:
+                files = list(video.get_files())
+            except OSError as e:
+                if e.errno != errno.ENOENT:
+                    self.log.error("Could not access path %s. Error: %s", video.downloaded_path, e)
+                    self.usr_err(f"Could not access path {video.downloaded_path}: {e}", suppress_notification=True)
+                    return
+
+            # Try to find a valid video file
+            found_video = False
+            for file in files:
+                mime, _ = mimetypes.guess_type(file)
+                if mime is not None and mime.startswith("video"):
+                    found_video = True
+
+            # Video not found, we can safely assume that the video was deleted.
+            if not found_video:
+                self.log.info("Video %d was deleted! [%s %s]", video.id, video.video_id, video.name)
+                # Clean up
+                for file in files:
+                    try:
+                        os.unlink(file)
+                    except OSError as e:
+                        self.log.error("Could not delete redundant file %s. Error: %s", file, e)
+                        self.usr_err(f"Could not delete redundant file {file}: {e}", suppress_notification=True)
+                video.downloaded_path = None
+
+                # Mark watched?
+                user = video.subscription.user
+                if user.preferences['mark_deleted_as_watched']:
+                    video.watched = True
+
+                video.save()
+
+    def update_video_stats(self, video: Video, yt_video):
+        if yt_video.n_likes is not None \
+                and yt_video.n_dislikes is not None \
+                and yt_video.n_likes + yt_video.n_dislikes > 0:
+            video.rating = yt_video.n_likes / (yt_video.n_likes + yt_video.n_dislikes)
+
+        video.views = yt_video.n_views
+        video.save()
+
+
+class SubscriptionImporterJob(Job):
+    def __init__(self, job_execution, urls: List[str],
+                 parent_folder: SubscriptionFolder,
+                 auto_download: bool,
+                 download_limit: int,
+                 download_order: str,
+                 automatically_delete_watched: bool):
+
+        super().__init__(job_execution)
+        self._urls = urls
+        self._parent_folder = parent_folder
+        self._auto_download = auto_download
+        self._download_limit = download_limit
+        self._download_order = download_order
+        self._automatically_delete_watched = automatically_delete_watched
+
+    def get_description(self):
+        return f"Importing {len(self._urls)} subscriptions..."
+
+    def run(self):
+        from YtManagerApp.services import Services
+        self.set_total_steps(len(self._urls))
+        for url in self._urls:
+            try:
+                self.progress_advance(progress_msg=url)
+                sub: Subscription = Services.videoProviderManager().fetch_subscription(url)
+                sub.parent_folder = self._parent_folder
+                sub.auto_download = self._auto_download
+                sub.download_limit = self._download_limit
+                sub.download_order = self._download_order
+                sub.automatically_delete_watched = self._automatically_delete_watched
+                sub.save()
+            except InvalidURLError as e:
+                self.log.error("Error importing URL %s: %s", url, e)
+            except ValueError as e:
+                self.log.error("Error importing URL %s: %s", url, e)
+
+
+class SubscriptionManager(object):
+    def __init__(self):
+        self.__global_sync_job = None
+
+    def synchronize(self, sub: Subscription):
+        from YtManagerApp.services import Services
+        Services.scheduler().add_job(SynchronizeJob, args=[sub])
+
+    def synchronize_all(self):
+        from YtManagerApp.services import Services
+        Services.scheduler().add_job(SynchronizeJob, max_instances=1, coalesce=True)
+
+    def schedule_global_synchronize_job(self):
+        from YtManagerApp.services import Services
+        trigger = CronTrigger.from_crontab(Services.appConfig().sync_schedule)
+
+        if self.__global_sync_job is None:
+            trigger = CronTrigger.from_crontab(Services.appConfig().sync_schedule)
+            SynchronizeJob.__global_sync_job = Services.scheduler().add_job(SynchronizeJob, trigger, max_instances=1,
+                                                                            coalesce=True)
+
+        else:
+            self.__global_sync_job.reschedule(trigger, max_instances=1, coalesce=True)
+
+    def import_multiple(self, urls: List[str],
+                        parent_folder: SubscriptionFolder,
+                        auto_download: bool,
+                        download_limit: int,
+                        download_order: str,
+                        automatically_delete_watched: bool):
+        from YtManagerApp.services import Services
+        Services.scheduler().add_job(SubscriptionImporterJob, args=[urls, parent_folder, auto_download, download_limit,
+                                                                    download_order, automatically_delete_watched])

app/YtManagerApp/management/video_manager.py

@@ -0,0 +1,124 @@
+import os
+import re
+from typing import Optional
+
+from django.contrib.auth.models import User
+from django.db.models import Q
+
+from YtManagerApp.models import Subscription, Video, SubscriptionFolder
+from YtManagerApp.scheduler.job import Job
+
+
+class DeleteVideoJob(Job):
+    """
+    Deletes a video's files.
+    """
+    name = "DeleteVideoJob"
+
+    def __init__(self, job_execution, video: Video):
+        super().__init__(job_execution)
+        self._video = video
+
+    def get_description(self):
+        return f"Deleting video {self._video}"
+
+    def run(self):
+        count = 0
+
+        try:
+            for file in self._video.get_files():
+                self.log.info("Deleting file %s", file)
+                count += 1
+                try:
+                    os.unlink(file)
+                except OSError as e:
+                    self.log.error("Failed to delete file %s: Error: %s", file, e)
+
+        except OSError as e:
+            self.log.error("Failed to delete video %d [%s %s]. Error: %s", self._video.id,
+                           self._video.video_id, self._video.name, e)
+
+        self._video.downloaded_path = None
+        self._video.save()
+
+        self.log.info('Deleted video %d successfully! (%d files) [%s %s]', self._video.id, count,
+                      self._video.video_id, self._video.name)
+
+
+class VideoManager(object):
+    def __init__(self):
+        pass
+
+    def get_videos(self,
+                   user: User,
+                   sort_order: Optional[str],
+                   query: Optional[str] = None,
+                   subscription_id: Optional[int] = None,
+                   folder_id: Optional[int] = None,
+                   only_watched: Optional[bool] = None,
+                   only_downloaded: Optional[bool] = None,
+                   ):
+
+        filter_args = []
+        filter_kwargs = {
+            'subscription__user': user
+        }
+
+        # Process query string - basically, we break it down into words,
+        # and then search for the given text in the name, description, uploader name and subscription name
+        if query is not None:
+            for match in re.finditer(r'\w+', query):
+                word = match[0]
+                filter_args.append(Q(name__icontains=word)
+                                   | Q(description__icontains=word)
+                                   | Q(uploader_name__icontains=word)
+                                   | Q(subscription__name__icontains=word))
+
+        # Subscription id
+        if subscription_id is not None:
+            filter_kwargs['subscription_id'] = subscription_id
+
+        # Folder id
+        if folder_id is not None:
+            # Visit function - returns only the subscription IDs
+            def visit(node):
+                if isinstance(node, Subscription):
+                    return node.id
+                return None
+
+            filter_kwargs['subscription_id__in'] = SubscriptionFolder.traverse(folder_id, user, visit)
+
+        # Only watched
+        if only_watched is not None:
+            filter_kwargs['watched'] = only_watched
+
+        # Only downloaded
+        # - not downloaded (False) -> is null (True)
+        # - downloaded (True) -> is not null (False)
+        if only_downloaded is not None:
+            filter_kwargs['downloaded_path__isnull'] = not only_downloaded
+
+        return Video.objects.filter(*filter_args, **filter_kwargs).order_by(sort_order)
+
+    def delete_files(self, video: Video):
+        from YtManagerApp.services import Services
+        Services.scheduler().add_job(DeleteVideoJob, args=[video])
+
+    def download(self, video: Video, attempt: int = 1):
+        from YtManagerApp.services import Services
+        Services.downloadManager().download_video(video, attempt)
+
+    def mark_watched(self, video: Video):
+        from YtManagerApp.services import Services
+        video.watched = True
+        video.save()
+        if video.downloaded_path is not None:
+            if Services.appConfig().for_sub(video.subscription, 'automatically_delete_watched'):
+                self.delete_files(video)
+                Services.subscriptionManager().synchronize(video.subscription)
+
+    def mark_unwatched(self, video: Video):
+        from YtManagerApp.services import Services
+        video.watched = False
+        video.save()
+        Services.subscriptionManager().synchronize(video.subscription)

app/YtManagerApp/management/video_provider_manager.py

@@ -0,0 +1,102 @@
+import logging
+from typing import List, Dict, Union
+
+from YtManagerApp.models import VideoProviderConfig, Video, Subscription
+from YtManagerApp.providers.video_provider import VideoProvider, InvalidURLError
+import json
+
+log = logging.getLogger("VideoProviderManager")
+
+
+class VideoProviderManager(object):
+    def __init__(self, registered_providers: List[VideoProvider]):
+        self._registered_providers: Dict[str, VideoProvider] = {}
+        self._configured_providers: Dict[str, VideoProvider] = {}
+        self._pending_configs: Dict[str, VideoProviderConfig] = {}
+        for rp in registered_providers:
+            self.register_provider(rp)
+        self._load()
+
+    def register_provider(self, provider: VideoProvider) -> None:
+        """
+        Registers a video provider
+        :param provider: Video provider
+        """
+        # avoid duplicates
+        if provider.name in self._registered_providers:
+            log.error(f"Duplicate video provider {provider.name}")
+            return
+
+        # register
+        self._registered_providers[provider.name] = provider
+        log.info(f"Registered video provider {provider.name}")
+
+        # load configuration (if any)
+        if provider.name in self._pending_configs:
+            self._configure(provider, self._pending_configs[provider.name])
+            del self._pending_configs[provider.name]
+
+    def _load(self) -> None:
+        # Loads configuration from database
+        for config in VideoProviderConfig.objects.all():
+            provider = self._registered_providers.get(config.provider_id)
+
+            # provider not yet registered, keep it in the pending list
+            if provider is None:
+                self._pending_configs[config.provider_id] = config
+                log.warning(f"Provider {config.provider_id} not registered!")
+                continue
+
+            # configure
+            self._configure(provider, config)
+
+    def _configure(self, provider, config):
+        settings = json.loads(config.settings)
+        provider.configure(settings)
+        log.info(f"Configured video provider {provider.name}")
+        self._configured_providers[provider.name] = provider
+
+    def get(self, item: Union[str, Subscription, Video]):
+        """
+        Gets provider for given item (subscription or video).
+        :param item: Provider ID, or subscription, or video
+        :return: Provider
+        """
+        if isinstance(item, str):
+            return self._registered_providers[item]
+        elif isinstance(item, Video):
+            return self._registered_providers[item.subscription.provider_id]
+        elif isinstance(item, Subscription):
+            return self._registered_providers[item.provider_id]
+        return None
+
+    def validate_subscription_url(self, url: str):
+        """
+        Validates given URL using all registered and configured provider.
+        :param url:
+        :return:
+        """
+        for provider in self._configured_providers.values():
+            try:
+                provider.validate_subscription_url(url)
+                return
+            except InvalidURLError:
+                pass
+
+        raise InvalidURLError("The given URL is not valid for any of the supported sites!")
+
+    def fetch_subscription(self, url: str) -> Subscription:
+        """
+        Validates given URL using all registered and configured provider.
+        :param url:
+        :return:
+        """
+        for provider in self._configured_providers.values():
+            try:
+                provider.validate_subscription_url(url)
+                # Found the right provider
+                return provider.fetch_subscription(url)
+            except InvalidURLError:
+                pass
+
+        raise InvalidURLError("The given URL is not valid for any of the supported sites!")

app/YtManagerApp/management/videos.py

@@ -1,57 +0,0 @@
-import re
-from typing import Optional
-
-from django.contrib.auth.models import User
-from django.db.models import Q
-
-from YtManagerApp.models import Subscription, Video, SubscriptionFolder
-
-
-def get_videos(user: User,
-               sort_order: Optional[str],
-               query: Optional[str] = None,
-               subscription_id: Optional[int] = None,
-               folder_id: Optional[int] = None,
-               only_watched: Optional[bool] = None,
-               only_downloaded: Optional[bool] = None,
-               ):
-
-    filter_args = []
-    filter_kwargs = {
-        'subscription__user': user
-    }
-
-    # Process query string - basically, we break it down into words,
-    # and then search for the given text in the name, description, uploader name and subscription name
-    if query is not None:
-        for match in re.finditer(r'\w+', query):
-            word = match[0]
-            filter_args.append(Q(name__icontains=word)
-                               | Q(description__icontains=word)
-                               | Q(uploader_name__icontains=word)
-                               | Q(subscription__name__icontains=word))
-
-    # Subscription id
-    if subscription_id is not None:
-        filter_kwargs['subscription_id'] = subscription_id
-
-    # Folder id
-    if folder_id is not None:
-        # Visit function - returns only the subscription IDs
-        def visit(node):
-            if isinstance(node, Subscription):
-                return node.id
-            return None
-        filter_kwargs['subscription_id__in'] = SubscriptionFolder.traverse(folder_id, user, visit)
-
-    # Only watched
-    if only_watched is not None:
-        filter_kwargs['watched'] = only_watched
-
-    # Only downloaded
-    # - not downloaded (False) -> is null (True)
-    # - downloaded (True) -> is not null (False)
-    if only_downloaded is not None:
-        filter_kwargs['downloaded_path__isnull'] = not only_downloaded
-
-    return Video.objects.filter(*filter_args, **filter_kwargs).order_by(sort_order)

app/YtManagerApp/migrations/0013_auto_20200410_2037.py

@@ -0,0 +1,42 @@
+# Generated by Django 3.0.4 on 2020-04-10 20:37
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('YtManagerApp', '0012_auto_20190819_1615'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='VideoProviderConfig',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('provider_id', models.CharField(help_text='Provider ID', max_length=128, unique=True)),
+                ('settings', models.TextField(help_text='Video provider settings')),
+            ],
+        ),
+        migrations.AddField(
+            model_name='subscription',
+            name='provider_id',
+            field=models.CharField(default='YtAPI', max_length=128),
+            preserve_default=False,
+        ),
+        migrations.AddField(
+            model_name='video',
+            name='downloaded_size',
+            field=models.IntegerField(blank=True, null=True),
+        ),
+        migrations.AddField(
+            model_name='video',
+            name='last_updated_date',
+            field=models.DateTimeField(auto_now=True),
+        ),
+        migrations.AlterField(
+            model_name='video',
+            name='name',
+            field=models.TextField(),
+        ),
+    ]

app/YtManagerApp/migrations/0014_fix_video_provider.py

@@ -0,0 +1,28 @@
+# Generated by Django 3.0.4 on 2020-04-10 20:37
+
+from django.db import migrations, models
+import json
+
+
+def fix_video_provider(apps, schema_editor):
+    globalPrefs = apps.get_model('dynamic_preferences', 'GlobalPreferenceModel')
+    api_key_entries = globalPrefs.objects.filter(name='youtube_api_key')
+    if len(api_key_entries) < 1:
+        return
+
+    videoProviderConfig = apps.get_model('YtManagerApp', 'VideoProviderConfig')
+    ytApiProvider = videoProviderConfig(provider_id='YtAPI', settings=json.dumps({
+        'api_key': api_key_entries[0].raw_value
+    }))
+    ytApiProvider.save()
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('YtManagerApp', '0013_auto_20200410_2037'),
+    ]
+
+    operations = [
+        migrations.RunPython(fix_video_provider)
+    ]

app/YtManagerApp/models.py

@@ -1,300 +0,0 @@
-import logging
-import mimetypes
-import os
-from typing import Callable, Union, Any, Optional
-
-from django.contrib.auth.models import User
-from django.db import models
-from django.db.models.functions import Lower
-
-from YtManagerApp.utils import youtube
-
-# help_text = user shown text
-# verbose_name = user shown name
-# null = nullable, blank = user is allowed to set value to empty
-VIDEO_ORDER_CHOICES = [
-    ('newest', 'Newest'),
-    ('oldest', 'Oldest'),
-    ('playlist', 'Playlist order'),
-    ('playlist_reverse', 'Reverse playlist order'),
-    ('popularity', 'Popularity'),
-    ('rating', 'Top rated'),
-]
-
-VIDEO_ORDER_MAPPING = {
-    'newest': '-publish_date',
-    'oldest': 'publish_date',
-    'playlist': 'playlist_index',
-    'playlist_reverse': '-playlist_index',
-    'popularity': '-views',
-    'rating': '-rating'
-}
-
-
-class SubscriptionFolder(models.Model):
-    name = models.CharField(null=False, max_length=250)
-    parent = models.ForeignKey('self', on_delete=models.CASCADE, null=True, blank=True)
-    user = models.ForeignKey(User, on_delete=models.CASCADE, null=False, blank=False)
-
-    class Meta:
-        ordering = [Lower('parent__name'), Lower('name')]
-
-    def __str__(self):
-        s = ""
-        current = self
-        while current is not None:
-            s = current.name + " > " + s
-            current = current.parent
-        return s[:-3]
-
-    def __repr__(self):
-        return f'folder {self.id}, name="{self.name}"'
-
-    def delete_folder(self, keep_subscriptions: bool):
-        if keep_subscriptions:
-
-            def visit(node: Union["SubscriptionFolder", "Subscription"]):
-                if isinstance(node, Subscription):
-                    node.parent_folder = None
-                    node.save()
-
-            SubscriptionFolder.traverse(self.id, self.user, visit)
-
-        self.delete()
-
-    @staticmethod
-    def traverse(root_folder_id: Optional[int],
-                 user: User,
-                 visit_func: Callable[[Union["SubscriptionFolder", "Subscription"]], Any]):
-
-        data_collected = []
-
-        def collect(data):
-            if data is not None:
-                data_collected.append(data)
-
-        # Visit root
-        if root_folder_id is not None:
-            root_folder = SubscriptionFolder.objects.get(id=root_folder_id)
-            collect(visit_func(root_folder))
-
-        queue = [root_folder_id]
-        visited = []
-
-        while len(queue) > 0:
-            folder_id = queue.pop()
-
-            if folder_id in visited:
-                logging.error('Found folder tree cycle for folder id %d.', folder_id)
-                continue
-            visited.append(folder_id)
-
-            for folder in SubscriptionFolder.objects.filter(parent_id=folder_id, user=user).order_by(Lower('name')):
-                collect(visit_func(folder))
-                queue.append(folder.id)
-
-            for subscription in Subscription.objects.filter(parent_folder_id=folder_id, user=user).order_by(Lower('name')):
-                collect(visit_func(subscription))
-
-        return data_collected
-
-
-class Subscription(models.Model):
-    name = models.CharField(null=False, max_length=1024)
-    parent_folder = models.ForeignKey(SubscriptionFolder, on_delete=models.CASCADE, null=True, blank=True)
-    playlist_id = models.CharField(null=False, max_length=128)
-    description = models.TextField()
-    channel_id = models.CharField(max_length=128)
-    channel_name = models.CharField(max_length=1024)
-    thumbnail = models.CharField(max_length=1024)
-    user = models.ForeignKey(User, on_delete=models.CASCADE)
-    # youtube adds videos to the 'Uploads' playlist at the top instead of the bottom
-    rewrite_playlist_indices = models.BooleanField(null=False, default=False)
-
-    # overrides
-    auto_download = models.BooleanField(null=True, blank=True)
-    download_limit = models.IntegerField(null=True, blank=True)
-    download_order = models.CharField(
-        null=True, blank=True,
-        max_length=128,
-        choices=VIDEO_ORDER_CHOICES)
-    automatically_delete_watched = models.BooleanField(null=True, blank=True)
-
-    def __str__(self):
-        return self.name
-
-    def __repr__(self):
-        return f'subscription {self.id}, name="{self.name}", playlist_id="{self.playlist_id}"'
-
-    def fill_from_playlist(self, info_playlist: youtube.Playlist):
-        self.name = info_playlist.title
-        self.playlist_id = info_playlist.id
-        self.description = info_playlist.description
-        self.channel_id = info_playlist.channel_id
-        self.channel_name = info_playlist.channel_title
-        self.thumbnail = youtube.best_thumbnail(info_playlist).url
-
-    def copy_from_channel(self, info_channel: youtube.Channel):
-        # No point in storing info about the 'uploads from X' playlist
-        self.name = info_channel.title
-        self.playlist_id = info_channel.uploads_playlist.id
-        self.description = info_channel.description
-        self.channel_id = info_channel.id
-        self.channel_name = info_channel.title
-        self.thumbnail = youtube.best_thumbnail(info_channel).url
-        self.rewrite_playlist_indices = True
-
-    def fetch_from_url(self, url, yt_api: youtube.YoutubeAPI):
-        url_parsed = yt_api.parse_url(url)
-        if 'playlist' in url_parsed:
-            info_playlist = yt_api.playlist(url=url)
-            if info_playlist is None:
-                raise ValueError('Invalid playlist ID!')
-
-            self.fill_from_playlist(info_playlist)
-        else:
-            info_channel = yt_api.channel(url=url)
-            if info_channel is None:
-                raise ValueError('Cannot find channel!')
-
-            self.copy_from_channel(info_channel)
-
-    def delete_subscription(self, keep_downloaded_videos: bool):
-        self.delete()
-
-
-class Video(models.Model):
-    video_id = models.CharField(null=False, max_length=12)
-    name = models.TextField(null=False)
-    description = models.TextField()
-    watched = models.BooleanField(default=False, null=False)
-    new = models.BooleanField(default=True, null=False)
-    downloaded_path = models.TextField(null=True, blank=True)
-    subscription = models.ForeignKey(Subscription, on_delete=models.CASCADE)
-    playlist_index = models.IntegerField(null=False)
-    publish_date = models.DateTimeField(null=False)
-    thumbnail = models.TextField()
-    uploader_name = models.CharField(null=False, max_length=255)
-    views = models.IntegerField(null=False, default=0)
-    rating = models.FloatField(null=False, default=0.5)
-
-    @staticmethod
-    def create(playlist_item: youtube.PlaylistItem, subscription: Subscription):
-        video = Video()
-        video.video_id = playlist_item.resource_video_id
-        video.name = playlist_item.title
-        video.description = playlist_item.description
-        video.watched = False
-        video.new = True
-        video.downloaded_path = None
-        video.subscription = subscription
-        video.playlist_index = playlist_item.position
-        video.publish_date = playlist_item.published_at
-        video.thumbnail = youtube.best_thumbnail(playlist_item).url
-        video.save()
-        return video
-
-    def mark_watched(self):
-        self.watched = True
-        self.save()
-        if self.downloaded_path is not None:
-            from YtManagerApp.management.appconfig import appconfig
-            from YtManagerApp.scheduler.jobs import DeleteVideoJob
-            from YtManagerApp.scheduler.jobs import SynchronizeJob
-
-            if appconfig.for_sub(self.subscription, 'automatically_delete_watched'):
-                DeleteVideoJob.schedule(self)
-                SynchronizeJob.schedule_now_for_subscription(self.subscription)
-
-    def mark_unwatched(self):
-        self.watched = False
-        self.save()
-        from YtManagerApp.scheduler.jobs.synchronize_job import SynchronizeJob
-        SynchronizeJob.schedule_now_for_subscription(self.subscription)
-
-    def get_files(self):
-        if self.downloaded_path is not None:
-            directory, file_pattern = os.path.split(self.downloaded_path)
-            for file in os.listdir(directory):
-                if file.startswith(file_pattern):
-                    yield os.path.join(directory, file)
-
-    def find_video(self):
-        """
-        Finds the video file from the downloaded files, and
-        returns
-        :return: Tuple containing file path and mime type
-        """
-        for file in self.get_files():
-            mime, _ = mimetypes.guess_type(file)
-            if mime is not None and mime.startswith('video/'):
-                return file, mime
-
-        return None, None
-
-    def delete_files(self):
-        if self.downloaded_path is not None:
-            from YtManagerApp.scheduler.jobs import DeleteVideoJob
-            from YtManagerApp.management.appconfig import appconfig
-            from YtManagerApp.scheduler.jobs import SynchronizeJob
-
-            DeleteVideoJob.schedule(self)
-
-            # Mark watched?
-            if self.subscription.user.preferences['mark_deleted_as_watched']:
-                self.watched = True
-                SynchronizeJob.schedule_now_for_subscription(self.subscription)
-
-    def download(self):
-        if not self.downloaded_path:
-            from YtManagerApp.scheduler.jobs.download_video_job import DownloadVideoJob
-            DownloadVideoJob.schedule(self)
-
-    def __str__(self):
-        return self.name
-
-    def __repr__(self):
-        return f'video {self.id}, video_id="{self.video_id}"'
-
-
-JOB_STATES = [
-    ('running', 0),
-    ('finished', 1),
-    ('failed', 2),
-    ('interrupted', 3),
-]
-
-JOB_STATES_MAP = {
-    'running': 0,
-    'finished': 1,
-    'failed': 2,
-    'interrupted': 3,
-}
-
-JOB_MESSAGE_LEVELS = [
-    ('normal', 0),
-    ('warning', 1),
-    ('error', 2),
-]
-JOB_MESSAGE_LEVELS_MAP = {
-    'normal': 0,
-    'warning': 1,
-    'error': 2,
-}
-
-
-class JobExecution(models.Model):
-    start_date = models.DateTimeField(auto_now=True, null=False)
-    end_date = models.DateTimeField(null=True)
-    user = models.ForeignKey(User, on_delete=models.CASCADE, null=True)
-    description = models.CharField(max_length=250, null=False, default="")
-    status = models.IntegerField(choices=JOB_STATES, null=False, default=0)
-
-
-class JobMessage(models.Model):
-    timestamp = models.DateTimeField(auto_now=True, null=False)
-    job = models.ForeignKey(JobExecution, null=False, on_delete=models.CASCADE)
-    progress = models.FloatField(null=True)
-    message = models.CharField(max_length=1024, null=False, default="")
-    level = models.IntegerField(choices=JOB_MESSAGE_LEVELS, null=False, default=0)
-    suppress_notification = models.BooleanField(null=False, default=False)

app/YtManagerApp/models/__init__.py

@@ -0,0 +1,6 @@
+from .video_order import VIDEO_ORDER_CHOICES, VIDEO_ORDER_MAPPING
+from .subscription_folder import SubscriptionFolder
+from .subscription import Subscription
+from .video import Video
+from .jobs import JobExecution, JobMessage, JOB_STATES_MAP, JOB_MESSAGE_LEVELS_MAP
+from .video_provider import VideoProviderConfig

app/YtManagerApp/models/jobs.py

@@ -0,0 +1,44 @@
+from django.contrib.auth.models import User
+from django.db import models
+
+JOB_STATES = [
+    ('running', 0),
+    ('finished', 1),
+    ('failed', 2),
+    ('interrupted', 3),
+]
+
+JOB_STATES_MAP = {
+    'running': 0,
+    'finished': 1,
+    'failed': 2,
+    'interrupted': 3,
+}
+
+JOB_MESSAGE_LEVELS = [
+    ('normal', 0),
+    ('warning', 1),
+    ('error', 2),
+]
+JOB_MESSAGE_LEVELS_MAP = {
+    'normal': 0,
+    'warning': 1,
+    'error': 2,
+}
+
+
+class JobExecution(models.Model):
+    start_date = models.DateTimeField(auto_now=True, null=False)
+    end_date = models.DateTimeField(null=True)
+    user = models.ForeignKey(User, on_delete=models.CASCADE, null=True)
+    description = models.CharField(max_length=250, null=False, default="")
+    status = models.IntegerField(choices=JOB_STATES, null=False, default=0)
+
+
+class JobMessage(models.Model):
+    timestamp = models.DateTimeField(auto_now=True, null=False)
+    job = models.ForeignKey(JobExecution, null=False, on_delete=models.CASCADE)
+    progress = models.FloatField(null=True)
+    message = models.CharField(max_length=1024, null=False, default="")
+    level = models.IntegerField(choices=JOB_MESSAGE_LEVELS, null=False, default=0)
+    suppress_notification = models.BooleanField(null=False, default=False)

app/YtManagerApp/models/subscription.py

@@ -0,0 +1,58 @@
+from django.contrib.auth.models import User
+from django.db import models
+
+from .subscription_folder import SubscriptionFolder
+from .video_order import VIDEO_ORDER_CHOICES
+
+
+class Subscription(models.Model):
+    name = models.CharField(null=False, max_length=1024)
+    parent_folder = models.ForeignKey(SubscriptionFolder, on_delete=models.CASCADE, null=True, blank=True)
+    provider_id = models.CharField(max_length=128, null=False)
+    playlist_id = models.CharField(null=False, max_length=128)
+    description = models.TextField()
+    channel_id = models.CharField(max_length=128)
+    channel_name = models.CharField(max_length=1024)
+    thumbnail = models.CharField(max_length=1024)
+    user = models.ForeignKey(User, on_delete=models.CASCADE)
+    # youtube adds videos to the 'Uploads' playlist at the top instead of the bottom
+    rewrite_playlist_indices = models.BooleanField(null=False, default=False)
+
+    # overrides
+    auto_download = models.BooleanField(null=True, blank=True)
+    download_limit = models.IntegerField(null=True, blank=True)
+    download_order = models.CharField(
+        null=True, blank=True,
+        max_length=128,
+        choices=VIDEO_ORDER_CHOICES)
+    automatically_delete_watched = models.BooleanField(null=True, blank=True)
+
+    def __str__(self):
+        return self.name
+
+    def __repr__(self):
+        return f'subscription {self.id}, name="{self.name}", playlist_id="{self.playlist_id}"'
+
+    def delete_subscription(self, keep_downloaded_videos: bool):
+        self.delete()
+
+    def copy_from(self, other: "Subscription"):
+        self.name = other.name
+        self.parent_folder = other.parent_folder
+        self.provider_id = other.provider_id
+        self.playlist_id = other.playlist_id
+        self.description = other.description
+        self.channel_id = other.channel_id
+        self.channel_name = other.channel_name
+        self.thumbnail = other.thumbnail
+        try:
+            self.user = other.user
+        except User.DoesNotExist:
+            self.user = None
+
+        self.rewrite_playlist_indices = other.rewrite_playlist_indices
+
+        self.auto_download = other.auto_download
+        self.download_limit = other.download_limit
+        self.download_order = other.download_order
+        self.automatically_delete_watched = other.automatically_delete_watched

app/YtManagerApp/models/subscription_folder.py

@@ -0,0 +1,76 @@
+import logging
+from typing import Callable, Union, Any, Optional
+
+from django.contrib.auth.models import User
+from django.db import models
+from django.db.models.functions import Lower
+
+
+class SubscriptionFolder(models.Model):
+    name = models.CharField(null=False, max_length=250)
+    parent = models.ForeignKey('self', on_delete=models.CASCADE, null=True, blank=True)
+    user = models.ForeignKey(User, on_delete=models.CASCADE, null=False, blank=False)
+
+    class Meta:
+        ordering = [Lower('parent__name'), Lower('name')]
+
+    def __str__(self):
+        s = ""
+        current = self
+        while current is not None:
+            s = current.name + " > " + s
+            current = current.parent
+        return s[:-3]
+
+    def __repr__(self):
+        return f'folder {self.id}, name="{self.name}"'
+
+    def delete_folder(self, keep_subscriptions: bool):
+        from .subscription import Subscription
+        if keep_subscriptions:
+
+            def visit(node: Union["SubscriptionFolder", "Subscription"]):
+                if isinstance(node, Subscription):
+                    node.parent_folder = None
+                    node.save()
+
+            SubscriptionFolder.traverse(self.id, self.user, visit)
+
+        self.delete()
+
+    @staticmethod
+    def traverse(root_folder_id: Optional[int],
+                 user: User,
+                 visit_func: Callable[[Union["SubscriptionFolder", "Subscription"]], Any]):
+        from .subscription import Subscription
+
+        data_collected = []
+
+        def collect(data):
+            if data is not None:
+                data_collected.append( data)
+
+        # Visit root
+        if root_folder_id is not None:
+            root_folder = SubscriptionFolder.objects.get(id=root_folder_id)
+            collect(visit_func(root_folder))
+
+        queue = [root_folder_id]
+        visited = []
+
+        while len(queue) > 0:
+            folder_id = queue.pop()
+
+            if folder_id in visited:
+                logging.error('Found folder tree cycle for folder id %d.', folder_id)
+                continue
+            visited.append(folder_id)
+
+            for folder in SubscriptionFolder.objects.filter(parent_id=folder_id, user=user).order_by(Lower('name')):
+                collect(visit_func(folder))
+                queue.append(folder.id)
+
+            for subscription in Subscription.objects.filter(parent_folder_id=folder_id, user=user).order_by(Lower('name')):
+                collect(visit_func(subscription))
+
+        return data_collected

app/YtManagerApp/models/video.py

@@ -0,0 +1,50 @@
+import mimetypes
+import os
+
+from django.db import models
+
+from .subscription import Subscription
+
+
+class Video(models.Model):
+    video_id = models.CharField(null=False, max_length=12)
+    name = models.TextField(null=False)
+    description = models.TextField()
+    watched = models.BooleanField(default=False, null=False)
+    new = models.BooleanField(default=True, null=False)
+    downloaded_path = models.TextField(null=True, blank=True)
+    downloaded_size = models.IntegerField(null=True, blank=True)
+    subscription = models.ForeignKey(Subscription, on_delete=models.CASCADE)
+    playlist_index = models.IntegerField(null=False)
+    publish_date = models.DateTimeField(null=False)
+    last_updated_date = models.DateTimeField(null=False, auto_now=True)
+    thumbnail = models.TextField()
+    uploader_name = models.CharField(null=False, max_length=255)
+    views = models.IntegerField(null=False, default=0)
+    rating = models.FloatField(null=False, default=0.5)
+
+    def get_files(self):
+        if self.downloaded_path is not None:
+            directory, file_pattern = os.path.split(self.downloaded_path)
+            for file in os.listdir(directory):
+                if file.startswith(file_pattern):
+                    yield os.path.join(directory, file)
+
+    def find_video(self):
+        """
+        Finds the video file from the downloaded files, and
+        returns
+        :return: Tuple containing file path and mime type
+        """
+        for file in self.get_files():
+            mime, _ = mimetypes.guess_type(file)
+            if mime is not None and mime.startswith('video/'):
+                return file, mime
+
+        return None, None
+
+    def __str__(self):
+        return self.name
+
+    def __repr__(self):
+        return f'video {self.id}, video_id="{self.video_id}"'

app/YtManagerApp/models/video_order.py

@@ -0,0 +1,17 @@
+VIDEO_ORDER_CHOICES = [
+    ('newest', 'Newest'),
+    ('oldest', 'Oldest'),
+    ('playlist', 'Playlist order'),
+    ('playlist_reverse', 'Reverse playlist order'),
+    ('popularity', 'Popularity'),
+    ('rating', 'Top rated'),
+]
+
+VIDEO_ORDER_MAPPING = {
+    'newest': '-publish_date',
+    'oldest': 'publish_date',
+    'playlist': 'playlist_index',
+    'playlist_reverse': '-playlist_index',
+    'popularity': '-views',
+    'rating': '-rating'
+}

app/YtManagerApp/models/video_provider.py

@@ -0,0 +1,6 @@
+from django.db import models
+
+
+class VideoProviderConfig(models.Model):
+    provider_id = models.CharField(max_length=128, unique=True, help_text="Provider ID")
+    settings = models.TextField(help_text="Video provider settings")

app/YtManagerApp/providers/video_provider.py

@@ -0,0 +1,110 @@
+from abc import abstractmethod, ABC
+from typing import Dict, Iterable, List, Any
+
+from django.forms import Field
+
+from YtManagerApp.models import Subscription, Video
+
+
+class ConfigurationValidationError(ValueError):
+    """
+    Exception type thrown when validating configurations.
+    """
+    def __init__(self, field_messages: Dict[str, str], *args, **kwargs):
+        """
+        Constructor
+        :param field_messages: A dictionary which maps field names to errors, which will be displayed to the user.
+        """
+        super().__init__(*args, **kwargs)
+        self.field_messages = field_messages
+
+
+class InvalidURLError(ValueError):
+    """
+    Invalid URL exception type
+    """
+    pass
+
+
+class VideoProvider(ABC):
+    name: str = ""
+    settings: Dict[str, Field] = {}
+
+    @abstractmethod
+    def configure(self, configuration: Dict[str, Any]) -> None:
+        """
+        Configures the video provider
+        :param configuration: A dictionary containing key-value pairs based on the settings defined.
+        :return: None
+        """
+        pass
+
+    @abstractmethod
+    def validate_configuration(self, configuration: Dict[str, Any]) -> None:
+        """
+        Validates the given configuration. This is executed when validating the settings form from the UI.
+        :param configuration: Dictionary containing key-value pairs, based on the settings defined.
+        :except ConfigurationValidationError Thrown if there are validation errors
+        """
+        pass
+
+    @abstractmethod
+    def get_subscription_url(self, subscription: Subscription) -> str:
+        """
+        Builds an URL that links to the given subscription.
+        :param subscription: The subscription
+        :return: URL
+        """
+        pass
+
+    @abstractmethod
+    def validate_subscription_url(self, url: str) -> None:
+        """
+        Validates given URL. Throws InvalidURLError if not valid.
+        :param url: URL to validate
+        :except InvalidURLError Thrown if the URL is not valid
+        """
+        pass
+
+    @abstractmethod
+    def fetch_subscription(self, url: str) -> Subscription:
+        """
+        Fetches a subscription using given URL
+        :param url: Subscription URL
+        :return: Subscription
+        :except InvalidURLError Thrown if the URL is not valid
+        """
+        pass
+
+    @abstractmethod
+    def get_video_url(self, video: Video) -> str:
+        """
+        Builds an URL that links to the given video.
+        :param video: The video
+        :return: URL
+        """
+        pass
+
+    @abstractmethod
+    def fetch_videos(self, subscription: Subscription) -> Iterable[Video]:
+        """
+        Fetches all the subscription items from the given subscription.
+        The method only needs to fetch the minimum amount of details, the update_videos method
+        is used to obtain additional information (such as likes/dislikes and other statistics)
+        :param subscription:
+        :return:
+        """
+        pass
+
+    @abstractmethod
+    def update_videos(self, videos: List[Video], update_metadata=False, update_statistics=False) -> None:
+        """
+        Updates the metadata for all the videos in the list.
+        :param videos: Videos
+        :param update_metadata: If true, video metadata (name, description) will be updated
+        :param update_statistics: If true, statistics (likes/dislikes, view count) will be updated.
+        :return:
+        """
+        pass
+
+

app/YtManagerApp/providers/ytapi_video_provider.py

@@ -0,0 +1,156 @@
+from typing import Dict, Optional, Any, Iterable, List
+
+from django import forms
+from external.pytaw.pytaw import youtube as yt
+from external.pytaw.pytaw.utils import iterate_chunks
+
+from YtManagerApp.models import Subscription, Video
+from YtManagerApp.providers.video_provider import VideoProvider, InvalidURLError
+
+
+class YouTubeApiVideoProvider(VideoProvider):
+    name = "YtAPI"
+    settings = {
+        "api_key": forms.CharField(label="YouTube API Key:")
+    }
+
+    def __init__(self):
+        super().__init__()
+        self.__api_key: str = None
+        self.__api: yt.YouTube = None
+
+    def configure(self, configuration: Dict[str, Any]) -> None:
+        self.__api_key = configuration['api_key']
+        self.__api = yt.YouTube(key=self.__api_key)
+
+    def validate_configuration(self, configuration: Dict[str, Any]):
+        # TODO: implement
+        pass
+
+    def get_subscription_url(self, subscription: Subscription):
+        return f"https://youtube.com/playlist?list={subscription.playlist_id}"
+
+    def validate_subscription_url(self, url: str) -> None:
+        try:
+            parsed_url = self.__api.parse_url(url)
+        except yt.InvalidURL:
+            raise InvalidURLError("The given URL is not valid!")
+
+        is_playlist = 'playlist' in parsed_url
+        is_channel = parsed_url['type'] in ('channel', 'user', 'channel_custom')
+
+        if not is_channel and not is_playlist:
+            raise InvalidURLError('The given URL is not a channel or a playlist!')
+
+    def fetch_subscription(self, url: str) -> Subscription:
+        sub = Subscription()
+        sub.provider_id = self.name
+
+        self.validate_subscription_url(url)
+        url_parsed = self.__api.parse_url(url)
+
+        if 'playlist' in url_parsed:
+            info_playlist = self.__api.playlist(url=url)
+            if info_playlist is None:
+                raise ValueError('Invalid playlist ID!')
+
+            sub.name = info_playlist.title
+            sub.playlist_id = info_playlist.id
+            sub.description = info_playlist.description
+            sub.channel_id = info_playlist.channel_id
+            sub.channel_name = info_playlist.channel_title
+            sub.thumbnail = self._best_thumbnail(info_playlist).url
+
+        else:
+            info_channel = self.__api.channel(url=url)
+            if info_channel is None:
+                raise ValueError('Cannot find channel!')
+
+            sub.name = info_channel.title
+            sub.playlist_id = info_channel.uploads_playlist.id
+            sub.description = info_channel.description
+            sub.channel_id = info_channel.id
+            sub.channel_name = info_channel.title
+            sub.thumbnail = self._best_thumbnail(info_channel).url
+            sub.rewrite_playlist_indices = True
+
+        return sub
+
+    def _default_thumbnail(self, resource: yt.Resource) -> Optional[yt.Thumbnail]:
+        """
+        Gets the default thumbnail for a resource.
+        Searches in the list of thumbnails for one with the label 'default', or takes the first one.
+        :param resource:
+        :return:
+        """
+        thumbs = getattr(resource, 'thumbnails', None)
+
+        if thumbs is None or len(thumbs) <= 0:
+            return None
+
+        return next(
+            (i for i in thumbs if i.id == 'default'),
+            thumbs[0]
+        )
+
+    def _best_thumbnail(self, resource: yt.Resource) -> Optional[yt.Thumbnail]:
+        """
+        Gets the best thumbnail available for a resource.
+        :param resource:
+        :return:
+        """
+        thumbs = getattr(resource, 'thumbnails', None)
+
+        if thumbs is None or len(thumbs) <= 0:
+            return None
+
+        return max(thumbs, key=lambda t: t.width * t.height)
+
+    def get_video_url(self, video: Video) -> str:
+        return f"https://youtube.com/watch?v={video.video_id}"
+
+    def fetch_videos(self, subscription: Subscription) -> Iterable[Video]:
+        playlist_items = self.__api.playlist_items(subscription.playlist_id)
+        for item in playlist_items:
+            video = Video()
+            video.video_id = item.resource_video_id
+            video.name = item.title
+            video.description = item.description
+            video.watched = False
+            video.new = True
+            video.downloaded_path = None
+            video.subscription = subscription
+            video.playlist_index = item.position
+            video.publish_date = item.published_at
+            video.thumbnail = self._best_thumbnail(item).url
+            yield video
+
+    def update_videos(self, videos: List[Video], update_metadata=False, update_statistics=False) -> None:
+        parts = ['id']
+        if update_metadata:
+            parts.append('snippet')
+        if update_statistics:
+            parts.append('statistics')
+
+        # don't waste api resources
+        if len(parts) <= 1:
+            return
+
+        video_dict = {video.video_id: video for video in videos}
+        id_list = video_dict.keys()
+
+        for batch in iterate_chunks(id_list, 50):
+            resp_videos = self.__api.videos(batch, part=','.join(parts))
+            for resp_video in resp_videos:
+                v = video_dict[resp_video.id]
+
+                if update_metadata:
+                    v.name = resp_video.title
+                    v.description = resp_video.description
+
+                if update_statistics:
+                    if resp_video.n_likes is not None \
+                            and resp_video.n_dislikes is not None \
+                            and resp_video.n_likes + resp_video.n_dislikes > 0:
+                        v.rating = resp_video.n_likes / (resp_video.n_likes + resp_video.n_dislikes)
+                    v.views = resp_video.n_views

app/YtManagerApp/scheduler/jobs/delete_video_job.py

@@ -1,47 +0,0 @@
-import os
-
-from YtManagerApp.models import Video
-from YtManagerApp.scheduler.job import Job
-
-
-class DeleteVideoJob(Job):
-    name = "DeleteVideoJob"
-
-    def __init__(self, job_execution, video: Video):
-        super().__init__(job_execution)
-        self._video = video
-
-    def get_description(self):
-        return f"Deleting video {self._video}"
-
-    def run(self):
-        count = 0
-
-        try:
-            for file in self._video.get_files():
-                self.log.info("Deleting file %s", file)
-                count += 1
-                try:
-                    os.unlink(file)
-                except OSError as e:
-                    self.log.error("Failed to delete file %s: Error: %s", file, e)
-
-        except OSError as e:
-            self.log.error("Failed to delete video %d [%s %s]. Error: %s", self._video.id,
-                           self._video.video_id, self._video.name, e)
-
-        self._video.downloaded_path = None
-        self._video.save()
-
-        self.log.info('Deleted video %d successfully! (%d files) [%s %s]', self._video.id, count,
-                      self._video.video_id, self._video.name)
-
-    @staticmethod
-    def schedule(video: Video):
-        """
-        Schedules a delete video job to run immediately.
-        :param video:
-        :return:
-        """
-        from YtManagerApp.services import Services
-        Services.scheduler.add_job(DeleteVideoJob, args=[video])

app/YtManagerApp/scheduler/jobs/download_video_job.py

@@ -1,136 +0,0 @@
-import os
-import re
-from string import Template
-from threading import Lock
-
-import youtube_dl
-
-from YtManagerApp.models import Video
-from YtManagerApp.scheduler.job import Job
-
-
-class DownloadVideoJob(Job):
-    name = "DownloadVideoJob"
-    __lock = Lock()
-
-    def __init__(self, job_execution, video: Video, attempt: int = 1):
-        super().__init__(job_execution)
-        self._video = video
-        self._attempt = attempt
-        self._log_youtube_dl = self.log.getChild('youtube_dl')
-
-    def get_description(self):
-        ret = "Downloading video " + self._video.name
-        if self._attempt > 1:
-            ret += f" (attempt {self._attempt})"
-        return ret
-
-    def run(self):
-        # Issue: if multiple videos are downloaded at the same time, a race condition appears in the mkdirs() call that
-        # youtube-dl makes, which causes it to fail with the error 'Cannot create folder - file already exists'.
-        # For now, allow a single download instance.
-        self.__lock.acquire()
-
-        try:
-            user = self._video.subscription.user
-            max_attempts = user.preferences['max_download_attempts']
-
-            youtube_dl_params, output_path = self.__build_youtube_dl_params(self._video)
-            with youtube_dl.YoutubeDL(youtube_dl_params) as yt:
-                ret = yt.download(["https://www.youtube.com/watch?v=" + self._video.video_id])
-
-            self.log.info('Download finished with code %d', ret)
-
-            if ret == 0:
-                self._video.downloaded_path = output_path
-                self._video.save()
-                self.log.info('Video %d [%s %s] downloaded successfully!', self._video.id, self._video.video_id,
-                              self._video.name)
-
-            elif self._attempt <= max_attempts:
-                self.log.warning('Re-enqueueing video (attempt %d/%d)', self._attempt, max_attempts)
-                DownloadVideoJob.schedule(self._video, self._attempt + 1)
-
-            else:
-                self.log.error('Multiple attempts to download video %d [%s %s] failed!', self._video.id,
-                               self._video.video_id, self._video.name)
-                self._video.downloaded_path = ''
-                self._video.save()
-
-        finally:
-            self.__lock.release()
-
-    def __build_youtube_dl_params(self, video: Video):
-
-        sub = video.subscription
-        user = sub.user
-
-        # resolve path
-        download_path = user.preferences['download_path']
-
-        template_dict = self.__build_template_dict(video)
-        output_pattern = Template(user.preferences['download_file_pattern']).safe_substitute(template_dict)
-
-        output_path = os.path.join(download_path, output_pattern)
-        output_path = os.path.normpath(output_path)
-
-        youtube_dl_params = {
-            'logger': self._log_youtube_dl,
-            'format': user.preferences['download_format'],
-            'outtmpl': output_path,
-            'writethumbnail': True,
-            'writedescription': True,
-            'writesubtitles': user.preferences['download_subtitles'],
-            'writeautomaticsub': user.preferences['download_autogenerated_subtitles'],
-            'allsubtitles': user.preferences['download_subtitles_all'],
-            'merge_output_format': 'mp4',
-            'postprocessors': [
-                {
-                    'key': 'FFmpegMetadata'
-                },
-            ]
-        }
-
-        sub_langs = user.preferences['download_subtitles_langs'].split(',')
-        sub_langs = [i.strip() for i in sub_langs]
-        if len(sub_langs) > 0:
-            youtube_dl_params['subtitleslangs'] = sub_langs
-
-        sub_format = user.preferences['download_subtitles_format']
-        if len(sub_format) > 0:
-            youtube_dl_params['subtitlesformat'] = sub_format
-
-        return youtube_dl_params, output_path
-
-    def __build_template_dict(self, video: Video):
-        return {
-            'channel': video.subscription.channel_name,
-            'channel_id': video.subscription.channel_id,
-            'playlist': video.subscription.name,
-            'playlist_id': video.subscription.playlist_id,
-            'playlist_index': "{:03d}".format(1 + video.playlist_index),
-            'title': video.name,
-            'id': video.video_id,
-        }
-
-    def __get_valid_path(self, path):
-        """
-        Normalizes string, converts to lowercase, removes non-alpha characters,
-        and converts spaces to hyphens.
-        """
-        import unicodedata
-        value = unicodedata.normalize('NFKD', path).encode('ascii', 'ignore').decode('ascii')
-        value = re.sub('[:"*]', '', value).strip()
-        value = re.sub('[?<>|]', '#', value)
-        return value
-
-    @staticmethod
-    def schedule(video: Video, attempt: int = 1):
-        """
-        Schedules to download video immediately
-        :param video:
-        :param attempt:
-        :return:
-        """
-        from YtManagerApp.services import Services
-        Services.scheduler.add_job(DownloadVideoJob, args=[video, attempt])

app/YtManagerApp/scheduler/jobs/synchronize_job.py

@@ -1,184 +0,0 @@
-import errno
-import itertools
-from threading import Lock
-
-from apscheduler.triggers.cron import CronTrigger
-from django.db.models import Max
-from django.conf import settings
-
-from YtManagerApp.management.downloader import fetch_thumbnail, downloader_process_subscription
-from YtManagerApp.models import *
-from YtManagerApp.scheduler.job import Job
-from YtManagerApp.services import Services
-from YtManagerApp.utils import youtube
-from external.pytaw.pytaw.utils import iterate_chunks
-
-_ENABLE_UPDATE_STATS = True
-
-
-class SynchronizeJob(Job):
-    name = "SynchronizeJob"
-    __lock = Lock()
-    running = False
-    __global_sync_job = None
-
-    def __init__(self, job_execution, subscription: Optional[Subscription] = None):
-        super().__init__(job_execution)
-        self.__subscription = subscription
-        self.__api = youtube.YoutubeAPI.build_public()
-        self.__new_vids = []
-
-    def get_description(self):
-        if self.__subscription is not None:
-            return "Running synchronization for subscription " + self.__subscription.name
-        return "Running synchronization..."
-
-    def get_subscription_list(self):
-        if self.__subscription is not None:
-            return [self.__subscription]
-        return Subscription.objects.all()
-
-    def get_videos_list(self, subs):
-        return Video.objects.filter(subscription__in=subs)
-
-    def run(self):
-        self.__lock.acquire(blocking=True)
-        SynchronizeJob.running = True
-        try:
-            self.log.info(self.get_description())
-
-            # Build list of work items
-            work_subs = self.get_subscription_list()
-            work_vids = self.get_videos_list(work_subs)
-
-            self.set_total_steps(len(work_subs) + len(work_vids))
-
-            # Remove the 'new' flag
-            work_vids.update(new=False)
-
-            # Process subscriptions
-            for sub in work_subs:
-                self.progress_advance(1, "Synchronizing subscription " + sub.name)
-                self.check_new_videos(sub)
-                self.fetch_missing_thumbnails(sub)
-
-            # Add new videos to progress calculation
-            self.set_total_steps(len(work_subs) + len(work_vids) + len(self.__new_vids))
-
-            # Process videos
-            all_videos = itertools.chain(work_vids, self.__new_vids)
-            for batch in iterate_chunks(all_videos, 50):
-                video_stats = {}
-
-                if _ENABLE_UPDATE_STATS:
-                    batch_ids = [video.video_id for video in batch]
-                    video_stats = {v.id: v for v in self.__api.videos(batch_ids, part='id,statistics')}
-
-                for video in batch:
-                    self.progress_advance(1, "Updating video " + video.name)
-                    self.check_video_deleted(video)
-                    self.fetch_missing_thumbnails(video)
-
-                    if video.video_id in video_stats:
-                        self.update_video_stats(video, video_stats[video.video_id])
-
-            # Start downloading videos
-            for sub in work_subs:
-                downloader_process_subscription(sub)
-
-        finally:
-            SynchronizeJob.running = False
-            self.__lock.release()
-
-    def check_new_videos(self, sub: Subscription):
-        playlist_items = self.__api.playlist_items(sub.playlist_id)
-        if sub.rewrite_playlist_indices:
-            playlist_items = sorted(playlist_items, key=lambda x: x.published_at)
-        else:
-            playlist_items = sorted(playlist_items, key=lambda x: x.position)
-
-        for item in playlist_items:
-            results = Video.objects.filter(video_id=item.resource_video_id, subscription=sub)
-
-            if not results.exists():
-                self.log.info('New video for subscription %s: %s %s"', sub, item.resource_video_id, item.title)
-
-                # fix playlist index if necessary
-                if sub.rewrite_playlist_indices or Video.objects.filter(subscription=sub, playlist_index=item.position).exists():
-                    highest = Video.objects.filter(subscription=sub).aggregate(Max('playlist_index'))['playlist_index__max']
-                    item.position = 1 + (highest or -1)
-
-                self.__new_vids.append(Video.create(item, sub))
-
-    def fetch_missing_thumbnails(self, obj: Union[Subscription, Video]):
-        if obj.thumbnail.startswith("http"):
-            if isinstance(obj, Subscription):
-                obj.thumbnail = fetch_thumbnail(obj.thumbnail, 'sub', obj.playlist_id, settings.THUMBNAIL_SIZE_SUBSCRIPTION)
-            elif isinstance(obj, Video):
-                obj.thumbnail = fetch_thumbnail(obj.thumbnail, 'video', obj.video_id, settings.THUMBNAIL_SIZE_VIDEO)
-            obj.save()
-
-    def check_video_deleted(self, video: Video):
-        if video.downloaded_path is not None:
-            files = []
-            try:
-                files = list(video.get_files())
-            except OSError as e:
-                if e.errno != errno.ENOENT:
-                    self.log.error("Could not access path %s. Error: %s", video.downloaded_path, e)
-                    self.usr_err(f"Could not access path {video.downloaded_path}: {e}", suppress_notification=True)
-                    return
-
-            # Try to find a valid video file
-            found_video = False
-            for file in files:
-                mime, _ = mimetypes.guess_type(file)
-                if mime is not None and mime.startswith("video"):
-                    found_video = True
-
-            # Video not found, we can safely assume that the video was deleted.
-            if not found_video:
-                self.log.info("Video %d was deleted! [%s %s]", video.id, video.video_id, video.name)
-                # Clean up
-                for file in files:
-                    try:
-                        os.unlink(file)
-                    except OSError as e:
-                        self.log.error("Could not delete redundant file %s. Error: %s", file, e)
-                        self.usr_err(f"Could not delete redundant file {file}: {e}", suppress_notification=True)
-                video.downloaded_path = None
-
-                # Mark watched?
-                user = video.subscription.user
-                if user.preferences['mark_deleted_as_watched']:
-                    video.watched = True
-
-                video.save()
-
-    def update_video_stats(self, video: Video, yt_video):
-        if yt_video.n_likes is not None \
-                and yt_video.n_dislikes is not None \
-                and yt_video.n_likes + yt_video.n_dislikes > 0:
-            video.rating = yt_video.n_likes / (yt_video.n_likes + yt_video.n_dislikes)
-
-        video.views = yt_video.n_views
-        video.save()
-
-    @staticmethod
-    def schedule_global_job():
-        trigger = CronTrigger.from_crontab(Services.appConfig.sync_schedule)
-
-        if SynchronizeJob.__global_sync_job is None:
-            trigger = CronTrigger.from_crontab(Services.appConfig.sync_schedule)
-            SynchronizeJob.__global_sync_job = Services.scheduler.add_job(SynchronizeJob, trigger, max_instances=1, coalesce=True)
-
-        else:
-            SynchronizeJob.__global_sync_job.reschedule(trigger, max_instances=1, coalesce=True)
-
-    @staticmethod
-    def schedule_now():
-        Services.scheduler.add_job(SynchronizeJob, max_instances=1, coalesce=True)
-
-    @staticmethod
-    def schedule_now_for_subscription(subscription):
-        Services.scheduler.add_job(SynchronizeJob, user=subscription.user, args=[subscription])

app/YtManagerApp/services.py

@@ -2,12 +2,25 @@ import dependency_injector.containers as containers
 import dependency_injector.providers as providers
 from dynamic_preferences.registries import global_preferences_registry
 from YtManagerApp.management.appconfig import AppConfig
+from YtManagerApp.management.download_manager import DownloadManager
+from YtManagerApp.management.subscription_manager import SubscriptionManager
+from YtManagerApp.management.video_manager import VideoManager
+from YtManagerApp.management.video_provider_manager import VideoProviderManager
 from YtManagerApp.management.youtube_dl_manager import YoutubeDlManager
 from YtManagerApp.scheduler.scheduler import YtsmScheduler
 
 
+class VideoProviders(containers.DeclarativeContainer):
+    from YtManagerApp.providers.ytapi_video_provider import YouTubeApiVideoProvider
+    ytApiProvider = providers.Factory(YouTubeApiVideoProvider)
+
+
 class Services(containers.DeclarativeContainer):
     globalPreferencesRegistry = providers.Object(global_preferences_registry.manager())
     appConfig = providers.Singleton(AppConfig, globalPreferencesRegistry)
     scheduler = providers.Singleton(YtsmScheduler, appConfig)
     youtubeDLManager = providers.Singleton(YoutubeDlManager)
+    videoManager = providers.Singleton(VideoManager)
+    videoProviderManager = providers.Singleton(VideoProviderManager, [VideoProviders.ytApiProvider()])
+    subscriptionManager = providers.Singleton(SubscriptionManager)
+    downloadManager = providers.Singleton(DownloadManager)

app/YtManagerApp/utils/algorithms.py

@@ -55,3 +55,22 @@ def bisect_left(a, x, lo=0, hi=None, key=None):
 
 # Create aliases
 bisect = bisect_right
+
+
+def group_by(data, key):
+    """
+    Groups the given data into a dictionary matching the structure { key : [values] }
+    :param data: Iterable data to be grouped
+    :param key: Key used to group the data
+    :return: A dictionary containing the grouped data
+    """
+    result = {}
+    for entry in data:
+        entry_key = key(entry)
+        if entry_key not in result:
+            result[entry_key] = [entry]
+        else:
+            result[entry_key].append(entry)
+
+    return result
+

app/YtManagerApp/utils/youtube.py

@@ -1,49 +0,0 @@
-from django.conf import settings
-from external.pytaw.pytaw.youtube import YouTube, Channel, Playlist, PlaylistItem, Thumbnail, InvalidURL, Resource, Video
-from typing import Optional
-
-
-class YoutubeAPI(YouTube):
-
-    @staticmethod
-    def build_public() -> 'YoutubeAPI':
-        from YtManagerApp.management.appconfig import appconfig
-        return YoutubeAPI(key=appconfig.youtube_api_key)
-
-    # @staticmethod
-    # def build_oauth() -> 'YoutubeAPI':
-    #     flow =
-    #     credentials =
-    #     service = build(API_SERVICE_NAME, API_VERSION, credentials)
-
-
-def default_thumbnail(resource: Resource) -> Optional[Thumbnail]:
-    """
-    Gets the default thumbnail for a resource.
-    Searches in the list of thumbnails for one with the label 'default', or takes the first one.
-    :param resource:
-    :return:
-    """
-    thumbs = getattr(resource, 'thumbnails', None)
-
-    if thumbs is None or len(thumbs) <= 0:
-        return None
-
-    return next(
-        (i for i in thumbs if i.id == 'default'),
-        thumbs[0]
-    )
-
-
-def best_thumbnail(resource: Resource) -> Optional[Thumbnail]:
-    """
-    Gets the best thumbnail available for a resource.
-    :param resource:
-    :return:
-    """
-    thumbs = getattr(resource, 'thumbnails', None)
-
-    if thumbs is None or len(thumbs) <= 0:
-        return None
-
-    return max(thumbs, key=lambda t: t.width * t.height)

app/YtManagerApp/views/actions.py

@@ -3,12 +3,12 @@ from django.http import JsonResponse
 from django.views.generic import View
 
 from YtManagerApp.models import Video
-from YtManagerApp.scheduler.jobs.synchronize_job import SynchronizeJob
+from YtManagerApp.services import Services
 
 
 class SyncNowView(LoginRequiredMixin, View):
     def post(self, *args, **kwargs):
-        SynchronizeJob.schedule_now()
+        Services.subscriptionManager().synchronize_all()
         return JsonResponse({
             'success': True
         })

app/YtManagerApp/views/auth.py

@@ -36,7 +36,7 @@ class RegisterView(FormView):
         return context
 
     def post(self, request, *args, **kwargs):
-        if not Services.appConfig.allow_registrations:
+        if not Services.appConfig().allow_registrations:
             return HttpResponseForbidden("Registrations are disabled!")
 
         return super().post(request, *args, **kwargs)

app/YtManagerApp/views/first_time.py

@@ -8,7 +8,6 @@ from django.shortcuts import redirect
 from django.urls import reverse_lazy
 from django.views.generic import FormView
 
-from YtManagerApp.scheduler.jobs.synchronize_job import SynchronizeJob
 from YtManagerApp.services import Services
 from YtManagerApp.views.forms.first_time import WelcomeForm, ApiKeyForm, PickAdminUserForm, ServerConfigForm, DoneForm, \
     UserCreationForm, LoginForm
@@ -16,7 +15,7 @@ from YtManagerApp.views.forms.first_time import WelcomeForm, ApiKeyForm, PickAdm
 logger = logging.getLogger("FirstTimeWizard")
 
 
-class WizardStepMixin:
+class WizardStepMixin(object):
 
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
@@ -24,7 +23,7 @@ class WizardStepMixin:
     def get(self, request, *args, **kwargs):
 
         # Prevent access if application is already initialized
-        if Services.appConfig.initialized:
+        if Services.appConfig().initialized:
             logger.debug(f"Attempted to access {request.path}, but first time setup already run. Redirected to home "
                          f"page.")
             return redirect('home')
@@ -32,7 +31,7 @@ class WizardStepMixin:
         return super().get(request, *args, **kwargs)
 
     def post(self, request, *args, **kwargs):
-        if Services.appConfig.initialized:
+        if Services.appConfig().initialized:
             logger.debug(f"Attempted to post {request.path}, but first time setup already run.")
             return HttpResponseForbidden()
         return super().post(request, *args, **kwargs)
@@ -65,14 +64,14 @@ class Step1ApiKeyView(WizardStepMixin, FormView):
 
     def get_initial(self):
         initial = super().get_initial()
-        initial['api_key'] = Services.appConfig.youtube_api_key
+        initial['api_key'] = Services.appConfig().youtube_api_key
         return initial
 
     def form_valid(self, form):
         key = form.cleaned_data['api_key']
         # TODO: validate key
         if key is not None and len(key) > 0:
-            Services.appConfig.youtube_api_key = key
+            Services.appConfig().youtube_api_key = key
 
         return super().form_valid(form)
 
@@ -149,8 +148,8 @@ class Step3ConfigureView(WizardStepMixin, FormView):
 
     def get_initial(self):
         initial = super().get_initial()
-        initial['allow_registrations'] = Services.appConfig.allow_registrations
+        initial['allow_registrations'] = Services.appConfig().allow_registrations
-        initial['sync_schedule'] = Services.appConfig.sync_schedule
+        initial['sync_schedule'] = Services.appConfig().sync_schedule
         initial['auto_download'] = self.request.user.preferences['auto_download']
         initial['download_location'] = self.request.user.preferences['download_path']
         return initial
@@ -158,11 +157,11 @@ class Step3ConfigureView(WizardStepMixin, FormView):
     def form_valid(self, form):
         allow_registrations = form.cleaned_data['allow_registrations']
         if allow_registrations is not None:
-            Services.appConfig.allow_registrations = allow_registrations
+            Services.appConfig().allow_registrations = allow_registrations
 
         sync_schedule = form.cleaned_data['sync_schedule']
         if sync_schedule is not None and len(sync_schedule) > 0:
-            Services.appConfig.sync_schedule = sync_schedule
+            Services.appConfig().sync_schedule = sync_schedule
 
         auto_download = form.cleaned_data['auto_download']
         if auto_download is not None:
@@ -173,11 +172,11 @@ class Step3ConfigureView(WizardStepMixin, FormView):
             self.request.user.preferences['download_path'] = download_location
 
         # Set initialized to true
-        Services.appConfig.initialized = True
+        Services.appConfig().initialized = True
 
         # Start scheduler if not started
-        Services.scheduler.initialize()
+        Services.scheduler().initialize()
-        SynchronizeJob.schedule_global_job()
+        Services.subscriptionManager().schedule_global_synchronize_job()
 
         return super().form_valid(form)
 

app/YtManagerApp/views/forms/settings.py

@@ -234,25 +234,25 @@ class AdminSettingsForm(forms.Form):
     @staticmethod
     def get_initials():
         return {
-            'api_key': Services.appConfig.youtube_api_key,
+            'api_key': Services.appConfig().youtube_api_key,
-            'allow_registrations': Services.appConfig.allow_registrations,
+            'allow_registrations': Services.appConfig().allow_registrations,
-            'sync_schedule': Services.appConfig.sync_schedule,
+            'sync_schedule': Services.appConfig().sync_schedule,
-            'scheduler_concurrency': Services.appConfig.concurrency,
+            'scheduler_concurrency': Services.appConfig().concurrency,
         }
 
     def save(self):
         api_key = self.cleaned_data['api_key']
         if api_key is not None and len(api_key) > 0:
-            Services.appConfig.youtube_api_key = api_key
+            Services.appConfig().youtube_api_key = api_key
 
         allow_registrations = self.cleaned_data['allow_registrations']
         if allow_registrations is not None:
-            Services.appConfig.allow_registrations = allow_registrations
+            Services.appConfig().allow_registrations = allow_registrations
 
         sync_schedule = self.cleaned_data['sync_schedule']
         if sync_schedule is not None and len(sync_schedule) > 0:
-            Services.appConfig.sync_schedule = sync_schedule
+            Services.appConfig().sync_schedule = sync_schedule
 
         concurrency = self.cleaned_data['scheduler_concurrency']
         if concurrency is not None:
-            Services.appConfig.concurrency = concurrency
+            Services.appConfig().concurrency = concurrency

app/YtManagerApp/views/index.py

@@ -1,22 +1,21 @@
 from crispy_forms.helper import FormHelper
 from crispy_forms.layout import Layout, Field, HTML
 from django import forms
+from django.conf import settings
 from django.contrib.auth.decorators import login_required
 from django.contrib.auth.mixins import LoginRequiredMixin
+from django.core.paginator import Paginator
 from django.db.models import Q
 from django.http import HttpRequest, HttpResponseBadRequest, JsonResponse
 from django.shortcuts import render, redirect
 from django.views.generic import CreateView, UpdateView, DeleteView, FormView
 from django.views.generic.edit import FormMixin
-from django.conf import settings
-from django.core.paginator import Paginator
-from YtManagerApp.management.videos import get_videos
-from YtManagerApp.models import Subscription, SubscriptionFolder, VIDEO_ORDER_CHOICES, VIDEO_ORDER_MAPPING
-from YtManagerApp.services import Services
-from YtManagerApp.utils import youtube, subscription_file_parser
-from YtManagerApp.views.controls.modal import ModalMixin
 
-import logging
+from YtManagerApp.models import Subscription, SubscriptionFolder, VIDEO_ORDER_CHOICES, VIDEO_ORDER_MAPPING
+from YtManagerApp.providers.video_provider import InvalidURLError
+from YtManagerApp.services import Services
+from YtManagerApp.utils import subscription_file_parser
+from YtManagerApp.views.controls.modal import ModalMixin
 
 
 class VideoFilterForm(forms.Form):
@@ -110,8 +109,7 @@ def __tree_sub_id(sub_id):
 
 
 def index(request: HttpRequest):
-
+    if not Services.appConfig().initialized:
-    if not Services.appConfig.initialized:
         return redirect('first_time_0')
 
     context = {
@@ -129,7 +127,6 @@ def index(request: HttpRequest):
 
 @login_required
 def ajax_get_tree(request: HttpRequest):
-
     def visit(node):
         if isinstance(node, SubscriptionFolder):
             return {
@@ -157,7 +154,7 @@ def ajax_get_videos(request: HttpRequest):
     if request.method == 'POST':
         form = VideoFilterForm(request.POST)
         if form.is_valid():
-            videos = get_videos(
+            videos = Services.videoManager().get_videos(
                 user=request.user,
                 sort_order=form.cleaned_data['sort'],
                 query=form.cleaned_data['query'],
@@ -272,7 +269,6 @@ class CreateSubscriptionForm(forms.ModelForm):
 
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
-        self.yt_api = youtube.YoutubeAPI.build_public()
         self.helper = FormHelper()
         self.helper.form_tag = False
         self.helper.layout = Layout(
@@ -289,16 +285,9 @@ class CreateSubscriptionForm(forms.ModelForm):
     def clean_playlist_url(self):
         playlist_url: str = self.cleaned_data['playlist_url']
         try:
-            parsed_url = self.yt_api.parse_url(playlist_url)
+            Services.videoProviderManager().validate_subscription_url(playlist_url)
-        except youtube.InvalidURL as e:
+        except InvalidURLError as e:
             raise forms.ValidationError(str(e))
-
-        is_playlist = 'playlist' in parsed_url
-        is_channel = parsed_url['type'] in ('channel', 'user', 'channel_custom')
-
-        if not is_channel and not is_playlist:
-            raise forms.ValidationError('The given URL must link to a channel or a playlist!')
-
         return playlist_url
 
 
@@ -308,10 +297,12 @@ class CreateSubscriptionModal(LoginRequiredMixin, ModalMixin, CreateView):
 
     def form_valid(self, form):
         form.instance.user = self.request.user
-        api = youtube.YoutubeAPI.build_public()
         try:
-            form.instance.fetch_from_url(form.cleaned_data['playlist_url'], api)
+            subscription: Subscription = Services.videoProviderManager().fetch_subscription(
-        except youtube.InvalidURL as e:
+                form.cleaned_data['playlist_url'])
+            form.instance.copy_from(subscription)
+            form.instance.user = self.request.user
+        except InvalidURLError as e:
             return self.modal_response(form, False, str(e))
         except ValueError as e:
             return self.modal_response(form, False, str(e))
@@ -327,8 +318,9 @@ class CreateSubscriptionModal(LoginRequiredMixin, ModalMixin, CreateView):
         # except youtube.APIError as e:
         #     return self.modal_response(
         #         form, False, 'An error occurred while communicating with the YouTube API: ' + str(e))
-
+        response = super().form_valid(form)
-        return super().form_valid(form)
+        Services.subscriptionManager().synchronize(form.instance)
+        return response
 
 
 class UpdateSubscriptionForm(forms.ModelForm):
@@ -407,7 +399,6 @@ class ImportSubscriptionsForm(forms.Form):
 
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
-        self.yt_api = youtube.YoutubeAPI.build_public()
         self.helper = FormHelper()
         self.helper.form_tag = False
         self.helper.layout = Layout(
@@ -456,28 +447,18 @@ class ImportSubscriptionsModal(LoginRequiredMixin, ModalMixin, FormView):
         try:
             url_list = list(subscription_file_parser.parse(file))
         except subscription_file_parser.FormatNotSupportedError:
-            return super().modal_response(form, success=False,
+            return super().modal_response(form,
+                                          success=False,
                                           error_msg="The file could not be parsed! "
                                                     "Possible problems: format not supported, file is malformed.")
 
         print(form.cleaned_data)
 
-        # Create subscriptions
+        Services.subscriptionManager().import_multiple(url_list,
-        api = youtube.YoutubeAPI.build_public()
+                                                       form.cleaned_data['parent_folder'],
-        for url in url_list:
+                                                       form.cleaned_data['auto_download'],
-            sub = Subscription()
+                                                       form.cleaned_data['download_limit'],
-            sub.user = self.request.user
+                                                       form.cleaned_data['download_order'],
-            sub.parent_folder = form.cleaned_data['parent_folder']
+                                                       form.cleaned_data["automatically_delete_watched"])
-            sub.auto_download = form.cleaned_data['auto_download']
-            sub.download_limit = form.cleaned_data['download_limit']
-            sub.download_order = form.cleaned_data['download_order']
-            sub.automatically_delete_watched = form.cleaned_data["automatically_delete_watched"]
-            try:
-                sub.fetch_from_url(url, api)
-            except Exception as e:
-                logging.error("Import subscription error - error processing URL %s: %s", url, e)
-                continue
-
-            sub.save()
 
         return super().form_valid(form)

app/YtManagerApp/views/settings.py

@@ -3,7 +3,7 @@ from django.http import HttpResponseForbidden
 from django.urls import reverse_lazy
 from django.views.generic import FormView
 
-from YtManagerApp.scheduler.jobs.synchronize_job import SynchronizeJob
+from YtManagerApp.services import Services
 from YtManagerApp.views.forms.settings import SettingsForm, AdminSettingsForm
 
 
@@ -45,5 +45,5 @@ class AdminSettingsView(LoginRequiredMixin, FormView):
 
     def form_valid(self, form):
         form.save()
-        SynchronizeJob.schedule_global_job()
+        Services.subscriptionManager().schedule_global_synchronize_job()
         return super().form_valid(form)

app/YtManagerApp/views/video.py

@@ -1,8 +1,6 @@
 from django.contrib.auth.decorators import login_required
 from django.contrib.auth.mixins import LoginRequiredMixin
-from django.http import HttpRequest, StreamingHttpResponse, FileResponse
+from django.http import HttpRequest, FileResponse
-from django.urls import reverse, reverse_lazy
-from django.views import View
 from django.views.generic import DetailView
 
 from YtManagerApp.models import Video