chibicitiberiu/ytsm
1
0
Fork 0
mirror of https://github.com/chibicitiberiu/ytsm.git synced 2024-02-24 05:43:31 +00:00
Code Issues Releases Wiki Activity

Major refactor of codebase.

Browse Source
This commit is contained in:
Tiberiu Chibici 2020-04-11 00:30:24 +03:00
parent fd5d05232f
commit 1022ce353c
33 changed files with 1408 additions and 963 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
app/YtManagerApp/appmain.py
View File

@ -5,7 +5,6 @@ import sys
from django.conf import settings as dj_settings from django.conf import settings as dj_settings
from YtManagerApp.scheduler.jobs.synchronize_job import SynchronizeJob
from YtManagerApp.services import Services from YtManagerApp.services import Services
from django.db.utils import OperationalError from django.db.utils import OperationalError
@ -35,9 +34,9 @@ def main():
__initialize_logger() __initialize_logger()
try: try:
if Services.appConfig.initialized: if Services.appConfig().initialized:
Services.scheduler.initialize() Services.scheduler().initialize()
SynchronizeJob.schedule_global_job() Services.subscriptionManager().schedule_global_synchronize_job()
except OperationalError: except OperationalError:
# Settings table is not created when running migrate or makemigrations; # Settings table is not created when running migrate or makemigrations;
# Just don't do anything in this case. # Just don't do anything in this case.

255
app/YtManagerApp/management/download_manager.py Normal file
View File

@ -0,0 +1,255 @@
import logging
import mimetypes
import os
import re
from string import Template
from threading import Lock
from urllib.parse import urljoin
import PIL.Image
import PIL.ImageOps
import requests
import youtube_dl
from django.conf import settings as srv_settings
from YtManagerApp.models import Subscription, Video
from YtManagerApp.models import VIDEO_ORDER_MAPPING
from YtManagerApp.providers.video_provider import VideoProvider
from YtManagerApp.scheduler.job import Job
from YtManagerApp.utils import first_non_null
log = logging.getLogger('DownloadManager')
class DownloadVideoJob(Job):
"""
Downloads a video to the disk
"""
name = "DownloadVideoJob"
__lock = Lock()
def __init__(self, job_execution, video: Video, attempt: int = 1):
super().__init__(job_execution)
self._video = video
self._attempt = attempt
self._log_youtube_dl = self.log.getChild('youtube_dl')
def get_description(self):
ret = "Downloading video " + self._video.name
if self._attempt > 1:
ret += f" (attempt {self._attempt})"
return ret
def run(self):
from YtManagerApp.services import Services
# Issue: if multiple videos are downloaded at the same time, a race condition appears in the mkdirs() call that
# youtube-dl makes, which causes it to fail with the error 'Cannot create folder - file already exists'.
# For now, allow a single download instance.
self.__lock.acquire()
try:
user = self._video.subscription.user
provider: VideoProvider = Services.videoProviderManager().get(self._video)
max_attempts = user.preferences['max_download_attempts']
youtube_dl_params, output_path = self.__build_youtube_dl_params(self._video)
with youtube_dl.YoutubeDL(youtube_dl_params) as yt:
ret = yt.download([provider.get_video_url(self._video)])
self.log.info('Download finished with code %d', ret)
if ret == 0:
self._video.downloaded_path = output_path
self._video.save()
self.log.info('Video %d [%s %s] downloaded successfully!', self._video.id, self._video.video_id,
self._video.name)
# update size
self._video.downloaded_size = 0
for file in self._video.get_files():
self._video.downloaded_size += os.stat(file).st_size
self._video.save()
elif self._attempt <= max_attempts:
self.log.warning('Re-enqueueing video (attempt %d/%d)', self._attempt, max_attempts)
Services.videoManager().download(self._video, self._attempt + 1)
else:
self.log.error('Multiple attempts to download video %d [%s %s] failed!', self._video.id,
self._video.video_id, self._video.name)
self._video.downloaded_path = ''
self._video.save()
finally:
self.__lock.release()
def __build_youtube_dl_params(self, video: Video):
sub = video.subscription
user = sub.user
# resolve path
download_path = user.preferences['download_path']
template_dict = self.__build_template_dict(video)
output_pattern = Template(user.preferences['download_file_pattern']).safe_substitute(template_dict)
output_path = os.path.join(download_path, output_pattern)
output_path = os.path.normpath(output_path)
youtube_dl_params = {
'logger': self._log_youtube_dl,
'format': user.preferences['download_format'],
'outtmpl': output_path,
'writethumbnail': True,
'writedescription': True,
'writesubtitles': user.preferences['download_subtitles'],
'writeautomaticsub': user.preferences['download_autogenerated_subtitles'],
'allsubtitles': user.preferences['download_subtitles_all'],
'merge_output_format': 'mp4',
'postprocessors': [
{
'key': 'FFmpegMetadata'
},
]
}
sub_langs = user.preferences['download_subtitles_langs'].split(',')
sub_langs = [i.strip() for i in sub_langs]
if len(sub_langs) > 0:
youtube_dl_params['subtitleslangs'] = sub_langs
sub_format = user.preferences['download_subtitles_format']
if len(sub_format) > 0:
youtube_dl_params['subtitlesformat'] = sub_format
return youtube_dl_params, output_path
def __build_template_dict(self, video: Video):
return {
'channel': video.subscription.channel_name,
'channel_id': video.subscription.channel_id,
'playlist': video.subscription.name,
'playlist_id': video.subscription.playlist_id,
'playlist_index': "{:03d}".format(1 + video.playlist_index),
'title': video.name,
'id': video.video_id,
}
def __get_valid_path(self, path: str):
"""
Normalizes string, converts to lowercase, removes non-alpha characters, removes forbidden characters.
"""
import unicodedata
value = unicodedata.normalize('NFKD', path).encode('ascii', 'ignore').decode('ascii')
value = re.sub('[:"*]', '', value).strip()
value = re.sub('[?<>|]', '#', value)
return value
class DownloadManager(object):
def __init__(self):
pass
def download_video(self, video: Video, attempt: int = 1):
from YtManagerApp.services import Services
Services.scheduler().add_job(DownloadVideoJob, args=[video, attempt])
def __get_subscription_config(self, sub: Subscription):
user = sub.user
enabled = first_non_null(sub.auto_download, user.preferences['auto_download'])
global_limit = user.preferences['download_global_limit']
limit = first_non_null(sub.download_limit, user.preferences['download_subscription_limit'])
order = first_non_null(sub.download_order, user.preferences['download_order'])
order = VIDEO_ORDER_MAPPING[order]
return enabled, global_limit, limit, order
def process_subscription(self, sub: Subscription):
from YtManagerApp.services import Services
log.info('Processing subscription %d [%s %s]', sub.id, sub.playlist_id, sub.id)
enabled, global_limit, limit, order = self.__get_subscription_config(sub)
log.info('Determined settings enabled=%s global_limit=%d limit=%d order="%s"', enabled, global_limit, limit, order)
if enabled:
videos_to_download = Video.objects\
.filter(subscription=sub, downloaded_path__isnull=True, watched=False)\
.order_by(order)
log.info('%d download candidates.', len(videos_to_download))
if global_limit > 0:
global_downloaded = Video.objects.filter(subscription__user=sub.user, downloaded_path__isnull=False).count()
allowed_count = max(global_limit - global_downloaded, 0)
videos_to_download = videos_to_download[0:allowed_count]
log.info('Global limit is set, can only download up to %d videos.', allowed_count)
if limit > 0:
sub_downloaded = Video.objects.filter(subscription=sub, downloaded_path__isnull=False).count()
allowed_count = max(limit - sub_downloaded, 0)
videos_to_download = videos_to_download[0:allowed_count]
log.info('Limit is set, can only download up to %d videos.', allowed_count)
# enqueue download
for video in videos_to_download:
log.info('Enqueuing video %d [%s %s] index=%d', video.id, video.video_id, video.name, video.playlist_index)
Services.videoManager().download(video)
log.info('Finished processing subscription %d [%s %s]', sub.id, sub.playlist_id, sub.id)
def process_all_subscriptions(self):
for subscription in Subscription.objects.all():
self.process_subscription(subscription)
def fetch_thumbnail(self, url, object_type, identifier, thumb_size):
log.info('Fetching thumbnail url=%s object_type=%s identifier=%s', url, object_type, identifier)
# Make request to obtain mime type
try:
response = requests.get(url, stream=True)
except requests.exceptions.RequestException as e:
log.error('Failed to fetch thumbnail %s. Error: %s', url, e)
return url
ext = mimetypes.guess_extension(response.headers['Content-Type'])
# Build file path
file_name = f"{identifier}{ext}"
abs_path_dir = os.path.join(srv_settings.MEDIA_ROOT, "thumbs", object_type)
abs_path = os.path.join(abs_path_dir, file_name)
abs_path_tmp = file_name + '.tmp'
# Store image
try:
os.makedirs(abs_path_dir, exist_ok=True)
with open(abs_path_tmp, "wb") as f:
for chunk in response.iter_content(chunk_size=1024):
if chunk:
f.write(chunk)
# Resize and crop to thumbnail size
image = PIL.Image.open(abs_path_tmp)
image = PIL.ImageOps.fit(image, thumb_size)
image.save(abs_path)
image.close()
# Delete temp file
os.unlink(abs_path_tmp)
except requests.exceptions.RequestException as e:
log.error('Error while downloading stream for thumbnail %s. Error: %s', url, e)
return url
except OSError as e:
log.error('Error while writing to file %s for thumbnail %s. Error: %s', abs_path, url, e)
return url
# Return
media_url = urljoin(srv_settings.MEDIA_URL, f"thumbs/{object_type}/{file_name}")
return media_url

111
app/YtManagerApp/management/downloader.py
View File

@ -1,111 +0,0 @@
from YtManagerApp.scheduler.jobs.download_video_job import DownloadVideoJob
from YtManagerApp.models import Video, Subscription, VIDEO_ORDER_MAPPING
from YtManagerApp.utils import first_non_null
from django.conf import settings as srv_settings
import logging
import requests
import mimetypes
import os
import PIL.Image
import PIL.ImageOps
from urllib.parse import urljoin
log = logging.getLogger('downloader')
def __get_subscription_config(sub: Subscription):
user = sub.user
enabled = first_non_null(sub.auto_download, user.preferences['auto_download'])
global_limit = user.preferences['download_global_limit']
limit = first_non_null(sub.download_limit, user.preferences['download_subscription_limit'])
order = first_non_null(sub.download_order, user.preferences['download_order'])
order = VIDEO_ORDER_MAPPING[order]
return enabled, global_limit, limit, order
def downloader_process_subscription(sub: Subscription):
log.info('Processing subscription %d [%s %s]', sub.id, sub.playlist_id, sub.id)
enabled, global_limit, limit, order = __get_subscription_config(sub)
log.info('Determined settings enabled=%s global_limit=%d limit=%d order="%s"', enabled, global_limit, limit, order)
if enabled:
videos_to_download = Video.objects\
.filter(subscription=sub, downloaded_path__isnull=True, watched=False)\
.order_by(order)
log.info('%d download candidates.', len(videos_to_download))
if global_limit > 0:
global_downloaded = Video.objects.filter(subscription__user=sub.user, downloaded_path__isnull=False).count()
allowed_count = max(global_limit - global_downloaded, 0)
videos_to_download = videos_to_download[0:allowed_count]
log.info('Global limit is set, can only download up to %d videos.', allowed_count)
if limit > 0:
sub_downloaded = Video.objects.filter(subscription=sub, downloaded_path__isnull=False).count()
allowed_count = max(limit - sub_downloaded, 0)
videos_to_download = videos_to_download[0:allowed_count]
log.info('Limit is set, can only download up to %d videos.', allowed_count)
# enqueue download
for video in videos_to_download:
log.info('Enqueuing video %d [%s %s] index=%d', video.id, video.video_id, video.name, video.playlist_index)
DownloadVideoJob.schedule(video)
log.info('Finished processing subscription %d [%s %s]', sub.id, sub.playlist_id, sub.id)
def downloader_process_all():
for subscription in Subscription.objects.all():
downloader_process_subscription(subscription)
def fetch_thumbnail(url, object_type, identifier, thumb_size):
log.info('Fetching thumbnail url=%s object_type=%s identifier=%s', url, object_type, identifier)
# Make request to obtain mime type
try:
response = requests.get(url, stream=True)
except requests.exceptions.RequestException as e:
log.error('Failed to fetch thumbnail %s. Error: %s', url, e)
return url
ext = mimetypes.guess_extension(response.headers['Content-Type'])
# Build file path
file_name = f"{identifier}{ext}"
abs_path_dir = os.path.join(srv_settings.MEDIA_ROOT, "thumbs", object_type)
abs_path = os.path.join(abs_path_dir, file_name)
abs_path_tmp = file_name + '.tmp'
# Store image
try:
os.makedirs(abs_path_dir, exist_ok=True)
with open(abs_path_tmp, "wb") as f:
for chunk in response.iter_content(chunk_size=1024):
if chunk:
f.write(chunk)
# Resize and crop to thumbnail size
image = PIL.Image.open(abs_path_tmp)
image = PIL.ImageOps.fit(image, thumb_size)
image.save(abs_path)
image.close()
# Delete temp file
os.unlink(abs_path_tmp)
except requests.exceptions.RequestException as e:
log.error('Error while downloading stream for thumbnail %s. Error: %s', url, e)
return url
except OSError as e:
log.error('Error while writing to file %s for thumbnail %s. Error: %s', abs_path, url, e)
return url
# Return
media_url = urljoin(srv_settings.MEDIA_URL, f"thumbs/{object_type}/{file_name}")
return media_url

246
app/YtManagerApp/management/subscription_manager.py Normal file
View File

@ -0,0 +1,246 @@
import errno
import itertools
import mimetypes
import os
from threading import Lock
from typing import Optional, List, Union
from apscheduler.triggers.cron import CronTrigger
from django.conf import settings
from django.db.models import Max
from YtManagerApp.models import *
from YtManagerApp.providers.video_provider import VideoProvider, InvalidURLError
from YtManagerApp.scheduler.job import Job
from YtManagerApp.utils.algorithms import group_by
_ENABLE_UPDATE_STATS = True
class SynchronizeJob(Job):
name = "SynchronizeJob"
__lock = Lock()
running = False
__global_sync_job = None
def __init__(self, job_execution, subscription: Optional[Subscription] = None):
super().__init__(job_execution)
self.__subscription: Optional[Subscription] = subscription
self.__new_vids: List[Video] = []
def get_description(self):
if self.__subscription is not None:
return "Running synchronization for subscription " + self.__subscription.name
return "Running synchronization..."
def get_subscription_list(self):
if self.__subscription is not None:
return [self.__subscription]
return Subscription.objects.all()
def get_videos_list(self, subs):
return Video.objects.filter(subscription__in=subs)
def run(self):
from YtManagerApp.services import Services
self.__lock.acquire(blocking=True)
SynchronizeJob.running = True
try:
self.log.info(self.get_description())
# Build list of work items
work_subs = self.get_subscription_list()
work_vids = self.get_videos_list(work_subs)
self.set_total_steps(len(work_subs) + len(work_vids))
# Remove the 'new' flag
work_vids.update(new=False)
# Process subscriptions
for sub in work_subs:
self.progress_advance(1, "Synchronizing subscription " + sub.name)
self.check_new_videos(sub)
self.fetch_missing_thumbnails(sub)
# Add new videos to progress calculation
self.set_total_steps(len(work_subs) + len(work_vids) + len(self.__new_vids))
# Group videos by provider
all_videos = itertools.chain(work_vids, self.__new_vids)
all_videos_by_provider = group_by(all_videos, lambda x: x.subscription.provider_id)
for provider_id, videos in all_videos_by_provider.items():
provider: VideoProvider = Services.videoProviderManager().get(provider_id)
if _ENABLE_UPDATE_STATS:
provider.update_videos(videos, update_statistics=True)
for video in videos:
self.progress_advance(1, "Updating video " + video.name)
self.check_video_deleted(video)
self.fetch_missing_thumbnails(video)
# Start downloading videos
for sub in work_subs:
Services.downloadManager().process_subscription(sub)
finally:
SynchronizeJob.running = False
self.__lock.release()
def check_new_videos(self, sub: Subscription):
from YtManagerApp.services import Services
provider: VideoProvider = Services.videoProviderManager().get(sub)
playlist_videos = provider.fetch_videos(sub)
if sub.rewrite_playlist_indices:
playlist_videos = sorted(playlist_videos, key=lambda x: x.publish_date)
else:
playlist_videos = sorted(playlist_videos, key=lambda x: x.playlist_index)
for item in playlist_videos:
results = Video.objects.filter(video_id=item.video_id, subscription=sub)
if not results.exists():
self.log.info('New video for subscription %s: %s %s"', sub, item.video_id, item.name)
# fix playlist index if necessary
if sub.rewrite_playlist_indices or Video.objects.filter(subscription=sub,
playlist_index=item.playlist_index).exists():
highest = Video.objects.filter(subscription=sub).aggregate(Max('playlist_index'))[
'playlist_index__max']
item.playlist_index = 1 + (highest or -1)
item.save()
self.__new_vids.append(item)
def fetch_missing_thumbnails(self, obj: Union[Subscription, Video]):
from YtManagerApp.services import Services
if obj.thumbnail.startswith("http"):
if isinstance(obj, Subscription):
obj.thumbnail = Services.downloadManager().fetch_thumbnail(obj.thumbnail, 'sub', obj.playlist_id,
settings.THUMBNAIL_SIZE_SUBSCRIPTION)
elif isinstance(obj, Video):
obj.thumbnail = Services.downloadManager().fetch_thumbnail(obj.thumbnail, 'video', obj.video_id,
settings.THUMBNAIL_SIZE_VIDEO)
obj.save()
def check_video_deleted(self, video: Video):
if video.downloaded_path is not None:
files = []
try:
files = list(video.get_files())
except OSError as e:
if e.errno != errno.ENOENT:
self.log.error("Could not access path %s. Error: %s", video.downloaded_path, e)
self.usr_err(f"Could not access path {video.downloaded_path}: {e}", suppress_notification=True)
return
# Try to find a valid video file
found_video = False
for file in files:
mime, _ = mimetypes.guess_type(file)
if mime is not None and mime.startswith("video"):
found_video = True
# Video not found, we can safely assume that the video was deleted.
if not found_video:
self.log.info("Video %d was deleted! [%s %s]", video.id, video.video_id, video.name)
# Clean up
for file in files:
try:
os.unlink(file)
except OSError as e:
self.log.error("Could not delete redundant file %s. Error: %s", file, e)
self.usr_err(f"Could not delete redundant file {file}: {e}", suppress_notification=True)
video.downloaded_path = None
# Mark watched?
user = video.subscription.user
if user.preferences['mark_deleted_as_watched']:
video.watched = True
video.save()
def update_video_stats(self, video: Video, yt_video):
if yt_video.n_likes is not None \
and yt_video.n_dislikes is not None \
and yt_video.n_likes + yt_video.n_dislikes > 0:
video.rating = yt_video.n_likes / (yt_video.n_likes + yt_video.n_dislikes)
video.views = yt_video.n_views
video.save()
class SubscriptionImporterJob(Job):
def __init__(self, job_execution, urls: List[str],
parent_folder: SubscriptionFolder,
auto_download: bool,
download_limit: int,
download_order: str,
automatically_delete_watched: bool):
super().__init__(job_execution)
self._urls = urls
self._parent_folder = parent_folder
self._auto_download = auto_download
self._download_limit = download_limit
self._download_order = download_order
self._automatically_delete_watched = automatically_delete_watched
def get_description(self):
return f"Importing {len(self._urls)} subscriptions..."
def run(self):
from YtManagerApp.services import Services
self.set_total_steps(len(self._urls))
for url in self._urls:
try:
self.progress_advance(progress_msg=url)
sub: Subscription = Services.videoProviderManager().fetch_subscription(url)
sub.parent_folder = self._parent_folder
sub.auto_download = self._auto_download
sub.download_limit = self._download_limit
sub.download_order = self._download_order
sub.automatically_delete_watched = self._automatically_delete_watched
sub.save()
except InvalidURLError as e:
self.log.error("Error importing URL %s: %s", url, e)
except ValueError as e:
self.log.error("Error importing URL %s: %s", url, e)
class SubscriptionManager(object):
def __init__(self):
self.__global_sync_job = None
def synchronize(self, sub: Subscription):
from YtManagerApp.services import Services
Services.scheduler().add_job(SynchronizeJob, args=[sub])
def synchronize_all(self):
from YtManagerApp.services import Services
Services.scheduler().add_job(SynchronizeJob, max_instances=1, coalesce=True)
def schedule_global_synchronize_job(self):
from YtManagerApp.services import Services
trigger = CronTrigger.from_crontab(Services.appConfig().sync_schedule)
if self.__global_sync_job is None:
trigger = CronTrigger.from_crontab(Services.appConfig().sync_schedule)
SynchronizeJob.__global_sync_job = Services.scheduler().add_job(SynchronizeJob, trigger, max_instances=1,
coalesce=True)
else:
self.__global_sync_job.reschedule(trigger, max_instances=1, coalesce=True)
def import_multiple(self, urls: List[str],
parent_folder: SubscriptionFolder,
auto_download: bool,
download_limit: int,
download_order: str,
automatically_delete_watched: bool):
from YtManagerApp.services import Services
Services.scheduler().add_job(SubscriptionImporterJob, args=[urls, parent_folder, auto_download, download_limit,
download_order, automatically_delete_watched])

124
app/YtManagerApp/management/video_manager.py Normal file
View File

@ -0,0 +1,124 @@
import os
import re
from typing import Optional
from django.contrib.auth.models import User
from django.db.models import Q
from YtManagerApp.models import Subscription, Video, SubscriptionFolder
from YtManagerApp.scheduler.job import Job
class DeleteVideoJob(Job):
"""
Deletes a video's files.
"""
name = "DeleteVideoJob"
def __init__(self, job_execution, video: Video):
super().__init__(job_execution)
self._video = video
def get_description(self):
return f"Deleting video {self._video}"
def run(self):
count = 0
try:
for file in self._video.get_files():
self.log.info("Deleting file %s", file)
count += 1
try:
os.unlink(file)
except OSError as e:
self.log.error("Failed to delete file %s: Error: %s", file, e)
except OSError as e:
self.log.error("Failed to delete video %d [%s %s]. Error: %s", self._video.id,
self._video.video_id, self._video.name, e)
self._video.downloaded_path = None
self._video.save()
self.log.info('Deleted video %d successfully! (%d files) [%s %s]', self._video.id, count,
self._video.video_id, self._video.name)
class VideoManager(object):
def __init__(self):
pass
def get_videos(self,
user: User,
sort_order: Optional[str],
query: Optional[str] = None,
subscription_id: Optional[int] = None,
folder_id: Optional[int] = None,
only_watched: Optional[bool] = None,
only_downloaded: Optional[bool] = None,
):
filter_args = []
filter_kwargs = {
'subscription__user': user
}
# Process query string - basically, we break it down into words,
# and then search for the given text in the name, description, uploader name and subscription name
if query is not None:
for match in re.finditer(r'\w+', query):
word = match[0]
filter_args.append(Q(name__icontains=word)
| Q(description__icontains=word)
| Q(uploader_name__icontains=word)
| Q(subscription__name__icontains=word))
# Subscription id
if subscription_id is not None:
filter_kwargs['subscription_id'] = subscription_id
# Folder id
if folder_id is not None:
# Visit function - returns only the subscription IDs
def visit(node):
if isinstance(node, Subscription):
return node.id
return None
filter_kwargs['subscription_id__in'] = SubscriptionFolder.traverse(folder_id, user, visit)
# Only watched
if only_watched is not None:
filter_kwargs['watched'] = only_watched
# Only downloaded
# - not downloaded (False) -> is null (True)
# - downloaded (True) -> is not null (False)
if only_downloaded is not None:
filter_kwargs['downloaded_path__isnull'] = not only_downloaded
return Video.objects.filter(*filter_args, **filter_kwargs).order_by(sort_order)
def delete_files(self, video: Video):
from YtManagerApp.services import Services
Services.scheduler().add_job(DeleteVideoJob, args=[video])
def download(self, video: Video, attempt: int = 1):
from YtManagerApp.services import Services
Services.downloadManager().download_video(video, attempt)
def mark_watched(self, video: Video):
from YtManagerApp.services import Services
video.watched = True
video.save()
if video.downloaded_path is not None:
if Services.appConfig().for_sub(video.subscription, 'automatically_delete_watched'):
self.delete_files(video)
Services.subscriptionManager().synchronize(video.subscription)
def mark_unwatched(self, video: Video):
from YtManagerApp.services import Services
video.watched = False
video.save()
Services.subscriptionManager().synchronize(video.subscription)

102
app/YtManagerApp/management/video_provider_manager.py Normal file
View File

@ -0,0 +1,102 @@
import logging
from typing import List, Dict, Union
from YtManagerApp.models import VideoProviderConfig, Video, Subscription
from YtManagerApp.providers.video_provider import VideoProvider, InvalidURLError
import json
log = logging.getLogger("VideoProviderManager")
class VideoProviderManager(object):
def __init__(self, registered_providers: List[VideoProvider]):
self._registered_providers: Dict[str, VideoProvider] = {}
self._configured_providers: Dict[str, VideoProvider] = {}
self._pending_configs: Dict[str, VideoProviderConfig] = {}
for rp in registered_providers:
self.register_provider(rp)
self._load()
def register_provider(self, provider: VideoProvider) -> None:
"""
Registers a video provider
:param provider: Video provider
"""
# avoid duplicates
if provider.name in self._registered_providers:
log.error(f"Duplicate video provider {provider.name}")
return
# register
self._registered_providers[provider.name] = provider
log.info(f"Registered video provider {provider.name}")
# load configuration (if any)
if provider.name in self._pending_configs:
self._configure(provider, self._pending_configs[provider.name])
del self._pending_configs[provider.name]
def _load(self) -> None:
# Loads configuration from database
for config in VideoProviderConfig.objects.all():
provider = self._registered_providers.get(config.provider_id)
# provider not yet registered, keep it in the pending list
if provider is None:
self._pending_configs[config.provider_id] = config
log.warning(f"Provider {config.provider_id} not registered!")
continue
# configure
self._configure(provider, config)
def _configure(self, provider, config):
settings = json.loads(config.settings)
provider.configure(settings)
log.info(f"Configured video provider {provider.name}")
self._configured_providers[provider.name] = provider
def get(self, item: Union[str, Subscription, Video]):
"""
Gets provider for given item (subscription or video).
:param item: Provider ID, or subscription, or video
:return: Provider
"""
if isinstance(item, str):
return self._registered_providers[item]
elif isinstance(item, Video):
return self._registered_providers[item.subscription.provider_id]
elif isinstance(item, Subscription):
return self._registered_providers[item.provider_id]
return None
def validate_subscription_url(self, url: str):
"""
Validates given URL using all registered and configured provider.
:param url:
:return:
"""
for provider in self._configured_providers.values():
try:
provider.validate_subscription_url(url)
return
except InvalidURLError:
pass
raise InvalidURLError("The given URL is not valid for any of the supported sites!")
def fetch_subscription(self, url: str) -> Subscription:
"""
Validates given URL using all registered and configured provider.
:param url:
:return:
"""
for provider in self._configured_providers.values():
try:
provider.validate_subscription_url(url)
# Found the right provider
return provider.fetch_subscription(url)
except InvalidURLError:
pass
raise InvalidURLError("The given URL is not valid for any of the supported sites!")

57
app/YtManagerApp/management/videos.py
View File

@ -1,57 +0,0 @@
import re
from typing import Optional
from django.contrib.auth.models import User
from django.db.models import Q
from YtManagerApp.models import Subscription, Video, SubscriptionFolder
def get_videos(user: User,
sort_order: Optional[str],
query: Optional[str] = None,
subscription_id: Optional[int] = None,
folder_id: Optional[int] = None,
only_watched: Optional[bool] = None,
only_downloaded: Optional[bool] = None,
):
filter_args = []
filter_kwargs = {
'subscription__user': user
}
# Process query string - basically, we break it down into words,
# and then search for the given text in the name, description, uploader name and subscription name
if query is not None:
for match in re.finditer(r'\w+', query):
word = match[0]
filter_args.append(Q(name__icontains=word)
| Q(description__icontains=word)
| Q(uploader_name__icontains=word)
| Q(subscription__name__icontains=word))
# Subscription id
if subscription_id is not None:
filter_kwargs['subscription_id'] = subscription_id
# Folder id
if folder_id is not None:
# Visit function - returns only the subscription IDs
def visit(node):
if isinstance(node, Subscription):
return node.id
return None
filter_kwargs['subscription_id__in'] = SubscriptionFolder.traverse(folder_id, user, visit)
# Only watched
if only_watched is not None:
filter_kwargs['watched'] = only_watched
# Only downloaded
# - not downloaded (False) -> is null (True)
# - downloaded (True) -> is not null (False)
if only_downloaded is not None:
filter_kwargs['downloaded_path__isnull'] = not only_downloaded
return Video.objects.filter(*filter_args, **filter_kwargs).order_by(sort_order)

42
app/YtManagerApp/migrations/0013_auto_20200410_2037.py Normal file
View File

@ -0,0 +1,42 @@
# Generated by Django 3.0.4 on 2020-04-10 20:37
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('YtManagerApp', '0012_auto_20190819_1615'),
]
operations = [
migrations.CreateModel(
name='VideoProviderConfig',
fields=[
('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('provider_id', models.CharField(help_text='Provider ID', max_length=128, unique=True)),
('settings', models.TextField(help_text='Video provider settings')),
],
),
migrations.AddField(
model_name='subscription',
name='provider_id',
field=models.CharField(default='YtAPI', max_length=128),
preserve_default=False,
),
migrations.AddField(
model_name='video',
name='downloaded_size',
field=models.IntegerField(blank=True, null=True),
),
migrations.AddField(
model_name='video',
name='last_updated_date',
field=models.DateTimeField(auto_now=True),
),
migrations.AlterField(
model_name='video',
name='name',
field=models.TextField(),
),
]

28
app/YtManagerApp/migrations/0014_fix_video_provider.py Normal file
View File

@ -0,0 +1,28 @@
# Generated by Django 3.0.4 on 2020-04-10 20:37
from django.db import migrations, models
import json
def fix_video_provider(apps, schema_editor):
globalPrefs = apps.get_model('dynamic_preferences', 'GlobalPreferenceModel')
api_key_entries = globalPrefs.objects.filter(name='youtube_api_key')
if len(api_key_entries) < 1:
return
videoProviderConfig = apps.get_model('YtManagerApp', 'VideoProviderConfig')
ytApiProvider = videoProviderConfig(provider_id='YtAPI', settings=json.dumps({
'api_key': api_key_entries[0].raw_value
}))
ytApiProvider.save()
class Migration(migrations.Migration):
dependencies = [
('YtManagerApp', '0013_auto_20200410_2037'),
]
operations = [
migrations.RunPython(fix_video_provider)
]

300
app/YtManagerApp/models.py
View File

@ -1,300 +0,0 @@
import logging
import mimetypes
import os
from typing import Callable, Union, Any, Optional
from django.contrib.auth.models import User
from django.db import models
from django.db.models.functions import Lower
from YtManagerApp.utils import youtube
# help_text = user shown text
# verbose_name = user shown name
# null = nullable, blank = user is allowed to set value to empty
VIDEO_ORDER_CHOICES = [
('newest', 'Newest'),
('oldest', 'Oldest'),
('playlist', 'Playlist order'),
('playlist_reverse', 'Reverse playlist order'),
('popularity', 'Popularity'),
('rating', 'Top rated'),
]
VIDEO_ORDER_MAPPING = {
'newest': '-publish_date',
'oldest': 'publish_date',
'playlist': 'playlist_index',
'playlist_reverse': '-playlist_index',
'popularity': '-views',
'rating': '-rating'
}
class SubscriptionFolder(models.Model):
name = models.CharField(null=False, max_length=250)
parent = models.ForeignKey('self', on_delete=models.CASCADE, null=True, blank=True)
user = models.ForeignKey(User, on_delete=models.CASCADE, null=False, blank=False)
class Meta:
ordering = [Lower('parent__name'), Lower('name')]
def __str__(self):
s = ""
current = self
while current is not None:
s = current.name + " > " + s
current = current.parent
return s[:-3]
def __repr__(self):
return f'folder {self.id}, name="{self.name}"'
def delete_folder(self, keep_subscriptions: bool):
if keep_subscriptions:
def visit(node: Union["SubscriptionFolder", "Subscription"]):
if isinstance(node, Subscription):
node.parent_folder = None
node.save()
SubscriptionFolder.traverse(self.id, self.user, visit)
self.delete()
@staticmethod
def traverse(root_folder_id: Optional[int],
user: User,
visit_func: Callable[[Union["SubscriptionFolder", "Subscription"]], Any]):
data_collected = []
def collect(data):
if data is not None:
data_collected.append(data)
# Visit root
if root_folder_id is not None:
root_folder = SubscriptionFolder.objects.get(id=root_folder_id)
collect(visit_func(root_folder))
queue = [root_folder_id]
visited = []
while len(queue) > 0:
folder_id = queue.pop()
if folder_id in visited:
logging.error('Found folder tree cycle for folder id %d.', folder_id)
continue
visited.append(folder_id)
for folder in SubscriptionFolder.objects.filter(parent_id=folder_id, user=user).order_by(Lower('name')):
collect(visit_func(folder))
queue.append(folder.id)
for subscription in Subscription.objects.filter(parent_folder_id=folder_id, user=user).order_by(Lower('name')):
collect(visit_func(subscription))
return data_collected
class Subscription(models.Model):
name = models.CharField(null=False, max_length=1024)
parent_folder = models.ForeignKey(SubscriptionFolder, on_delete=models.CASCADE, null=True, blank=True)
playlist_id = models.CharField(null=False, max_length=128)
description = models.TextField()
channel_id = models.CharField(max_length=128)
channel_name = models.CharField(max_length=1024)
thumbnail = models.CharField(max_length=1024)
user = models.ForeignKey(User, on_delete=models.CASCADE)
# youtube adds videos to the 'Uploads' playlist at the top instead of the bottom
rewrite_playlist_indices = models.BooleanField(null=False, default=False)
# overrides
auto_download = models.BooleanField(null=True, blank=True)
download_limit = models.IntegerField(null=True, blank=True)
download_order = models.CharField(
null=True, blank=True,
max_length=128,
choices=VIDEO_ORDER_CHOICES)
automatically_delete_watched = models.BooleanField(null=True, blank=True)
def __str__(self):
return self.name
def __repr__(self):
return f'subscription {self.id}, name="{self.name}", playlist_id="{self.playlist_id}"'
def fill_from_playlist(self, info_playlist: youtube.Playlist):
self.name = info_playlist.title
self.playlist_id = info_playlist.id
self.description = info_playlist.description
self.channel_id = info_playlist.channel_id
self.channel_name = info_playlist.channel_title
self.thumbnail = youtube.best_thumbnail(info_playlist).url
def copy_from_channel(self, info_channel: youtube.Channel):
# No point in storing info about the 'uploads from X' playlist
self.name = info_channel.title
self.playlist_id = info_channel.uploads_playlist.id
self.description = info_channel.description
self.channel_id = info_channel.id
self.channel_name = info_channel.title
self.thumbnail = youtube.best_thumbnail(info_channel).url
self.rewrite_playlist_indices = True
def fetch_from_url(self, url, yt_api: youtube.YoutubeAPI):
url_parsed = yt_api.parse_url(url)
if 'playlist' in url_parsed:
info_playlist = yt_api.playlist(url=url)
if info_playlist is None:
raise ValueError('Invalid playlist ID!')
self.fill_from_playlist(info_playlist)
else:
info_channel = yt_api.channel(url=url)
if info_channel is None:
raise ValueError('Cannot find channel!')
self.copy_from_channel(info_channel)
def delete_subscription(self, keep_downloaded_videos: bool):
self.delete()
class Video(models.Model):
video_id = models.CharField(null=False, max_length=12)
name = models.TextField(null=False)
description = models.TextField()
watched = models.BooleanField(default=False, null=False)
new = models.BooleanField(default=True, null=False)
downloaded_path = models.TextField(null=True, blank=True)
subscription = models.ForeignKey(Subscription, on_delete=models.CASCADE)
playlist_index = models.IntegerField(null=False)
publish_date = models.DateTimeField(null=False)
thumbnail = models.TextField()
uploader_name = models.CharField(null=False, max_length=255)
views = models.IntegerField(null=False, default=0)
rating = models.FloatField(null=False, default=0.5)
@staticmethod
def create(playlist_item: youtube.PlaylistItem, subscription: Subscription):
video = Video()
video.video_id = playlist_item.resource_video_id
video.name = playlist_item.title
video.description = playlist_item.description
video.watched = False
video.new = True
video.downloaded_path = None
video.subscription = subscription
video.playlist_index = playlist_item.position
video.publish_date = playlist_item.published_at
video.thumbnail = youtube.best_thumbnail(playlist_item).url
video.save()
return video
def mark_watched(self):
self.watched = True
self.save()
if self.downloaded_path is not None:
from YtManagerApp.management.appconfig import appconfig
from YtManagerApp.scheduler.jobs import DeleteVideoJob
from YtManagerApp.scheduler.jobs import SynchronizeJob
if appconfig.for_sub(self.subscription, 'automatically_delete_watched'):
DeleteVideoJob.schedule(self)
SynchronizeJob.schedule_now_for_subscription(self.subscription)
def mark_unwatched(self):
self.watched = False
self.save()
from YtManagerApp.scheduler.jobs.synchronize_job import SynchronizeJob
SynchronizeJob.schedule_now_for_subscription(self.subscription)
def get_files(self):
if self.downloaded_path is not None:
directory, file_pattern = os.path.split(self.downloaded_path)
for file in os.listdir(directory):
if file.startswith(file_pattern):
yield os.path.join(directory, file)
def find_video(self):
"""
Finds the video file from the downloaded files, and
returns
:return: Tuple containing file path and mime type
"""
for file in self.get_files():
mime, _ = mimetypes.guess_type(file)
if mime is not None and mime.startswith('video/'):
return file, mime
return None, None
def delete_files(self):
if self.downloaded_path is not None:
from YtManagerApp.scheduler.jobs import DeleteVideoJob
from YtManagerApp.management.appconfig import appconfig
from YtManagerApp.scheduler.jobs import SynchronizeJob
DeleteVideoJob.schedule(self)
# Mark watched?
if self.subscription.user.preferences['mark_deleted_as_watched']:
self.watched = True
SynchronizeJob.schedule_now_for_subscription(self.subscription)
def download(self):
if not self.downloaded_path:
from YtManagerApp.scheduler.jobs.download_video_job import DownloadVideoJob
DownloadVideoJob.schedule(self)
def __str__(self):
return self.name
def __repr__(self):
return f'video {self.id}, video_id="{self.video_id}"'
JOB_STATES = [
('running', 0),
('finished', 1),
('failed', 2),
('interrupted', 3),
]
JOB_STATES_MAP = {
'running': 0,
'finished': 1,
'failed': 2,
'interrupted': 3,
}
JOB_MESSAGE_LEVELS = [
('normal', 0),
('warning', 1),
('error', 2),
]
JOB_MESSAGE_LEVELS_MAP = {
'normal': 0,
'warning': 1,
'error': 2,
}
class JobExecution(models.Model):
start_date = models.DateTimeField(auto_now=True, null=False)
end_date = models.DateTimeField(null=True)
user = models.ForeignKey(User, on_delete=models.CASCADE, null=True)
description = models.CharField(max_length=250, null=False, default="")
status = models.IntegerField(choices=JOB_STATES, null=False, default=0)
class JobMessage(models.Model):
timestamp = models.DateTimeField(auto_now=True, null=False)
job = models.ForeignKey(JobExecution, null=False, on_delete=models.CASCADE)
progress = models.FloatField(null=True)
message = models.CharField(max_length=1024, null=False, default="")
level = models.IntegerField(choices=JOB_MESSAGE_LEVELS, null=False, default=0)
suppress_notification = models.BooleanField(null=False, default=False)

6
app/YtManagerApp/models/__init__.py Normal file
View File

@ -0,0 +1,6 @@
from .video_order import VIDEO_ORDER_CHOICES, VIDEO_ORDER_MAPPING
from .subscription_folder import SubscriptionFolder
from .subscription import Subscription
from .video import Video
from .jobs import JobExecution, JobMessage, JOB_STATES_MAP, JOB_MESSAGE_LEVELS_MAP
from .video_provider import VideoProviderConfig

44
app/YtManagerApp/models/jobs.py Normal file
View File

@ -0,0 +1,44 @@
from django.contrib.auth.models import User
from django.db import models
JOB_STATES = [
('running', 0),
('finished', 1),
('failed', 2),
('interrupted', 3),
]
JOB_STATES_MAP = {
'running': 0,
'finished': 1,
'failed': 2,
'interrupted': 3,
}
JOB_MESSAGE_LEVELS = [
('normal', 0),
('warning', 1),
('error', 2),
]
JOB_MESSAGE_LEVELS_MAP = {
'normal': 0,
'warning': 1,
'error': 2,
}
class JobExecution(models.Model):
start_date = models.DateTimeField(auto_now=True, null=False)
end_date = models.DateTimeField(null=True)
user = models.ForeignKey(User, on_delete=models.CASCADE, null=True)
description = models.CharField(max_length=250, null=False, default="")
status = models.IntegerField(choices=JOB_STATES, null=False, default=0)
class JobMessage(models.Model):
timestamp = models.DateTimeField(auto_now=True, null=False)
job = models.ForeignKey(JobExecution, null=False, on_delete=models.CASCADE)
progress = models.FloatField(null=True)
message = models.CharField(max_length=1024, null=False, default="")
level = models.IntegerField(choices=JOB_MESSAGE_LEVELS, null=False, default=0)
suppress_notification = models.BooleanField(null=False, default=False)

58
app/YtManagerApp/models/subscription.py Normal file
View File

@ -0,0 +1,58 @@
from django.contrib.auth.models import User
from django.db import models
from .subscription_folder import SubscriptionFolder
from .video_order import VIDEO_ORDER_CHOICES
class Subscription(models.Model):
name = models.CharField(null=False, max_length=1024)
parent_folder = models.ForeignKey(SubscriptionFolder, on_delete=models.CASCADE, null=True, blank=True)
provider_id = models.CharField(max_length=128, null=False)
playlist_id = models.CharField(null=False, max_length=128)
description = models.TextField()
channel_id = models.CharField(max_length=128)
channel_name = models.CharField(max_length=1024)
thumbnail = models.CharField(max_length=1024)
user = models.ForeignKey(User, on_delete=models.CASCADE)
# youtube adds videos to the 'Uploads' playlist at the top instead of the bottom
rewrite_playlist_indices = models.BooleanField(null=False, default=False)
# overrides
auto_download = models.BooleanField(null=True, blank=True)
download_limit = models.IntegerField(null=True, blank=True)
download_order = models.CharField(
null=True, blank=True,
max_length=128,
choices=VIDEO_ORDER_CHOICES)
automatically_delete_watched = models.BooleanField(null=True, blank=True)
def __str__(self):
return self.name
def __repr__(self):
return f'subscription {self.id}, name="{self.name}", playlist_id="{self.playlist_id}"'
def delete_subscription(self, keep_downloaded_videos: bool):
self.delete()
def copy_from(self, other: "Subscription"):
self.name = other.name
self.parent_folder = other.parent_folder
self.provider_id = other.provider_id
self.playlist_id = other.playlist_id
self.description = other.description
self.channel_id = other.channel_id
self.channel_name = other.channel_name
self.thumbnail = other.thumbnail
try:
self.user = other.user
except User.DoesNotExist:
self.user = None
self.rewrite_playlist_indices = other.rewrite_playlist_indices
self.auto_download = other.auto_download
self.download_limit = other.download_limit
self.download_order = other.download_order
self.automatically_delete_watched = other.automatically_delete_watched

76
app/YtManagerApp/models/subscription_folder.py Normal file
View File

@ -0,0 +1,76 @@
import logging
from typing import Callable, Union, Any, Optional
from django.contrib.auth.models import User
from django.db import models
from django.db.models.functions import Lower
class SubscriptionFolder(models.Model):
name = models.CharField(null=False, max_length=250)
parent = models.ForeignKey('self', on_delete=models.CASCADE, null=True, blank=True)
user = models.ForeignKey(User, on_delete=models.CASCADE, null=False, blank=False)
class Meta:
ordering = [Lower('parent__name'), Lower('name')]
def __str__(self):
s = ""
current = self
while current is not None:
s = current.name + " > " + s
current = current.parent
return s[:-3]
def __repr__(self):
return f'folder {self.id}, name="{self.name}"'
def delete_folder(self, keep_subscriptions: bool):
from .subscription import Subscription
if keep_subscriptions:
def visit(node: Union["SubscriptionFolder", "Subscription"]):
if isinstance(node, Subscription):
node.parent_folder = None
node.save()
SubscriptionFolder.traverse(self.id, self.user, visit)
self.delete()
@staticmethod
def traverse(root_folder_id: Optional[int],
user: User,
visit_func: Callable[[Union["SubscriptionFolder", "Subscription"]], Any]):
from .subscription import Subscription
data_collected = []
def collect(data):
if data is not None:
data_collected.append( data)
# Visit root
if root_folder_id is not None:
root_folder = SubscriptionFolder.objects.get(id=root_folder_id)
collect(visit_func(root_folder))
queue = [root_folder_id]
visited = []
while len(queue) > 0:
folder_id = queue.pop()
if folder_id in visited:
logging.error('Found folder tree cycle for folder id %d.', folder_id)
continue
visited.append(folder_id)
for folder in SubscriptionFolder.objects.filter(parent_id=folder_id, user=user).order_by(Lower('name')):
collect(visit_func(folder))
queue.append(folder.id)
for subscription in Subscription.objects.filter(parent_folder_id=folder_id, user=user).order_by(Lower('name')):
collect(visit_func(subscription))
return data_collected

50
app/YtManagerApp/models/video.py Normal file
View File

@ -0,0 +1,50 @@
import mimetypes
import os
from django.db import models
from .subscription import Subscription
class Video(models.Model):
video_id = models.CharField(null=False, max_length=12)
name = models.TextField(null=False)
description = models.TextField()
watched = models.BooleanField(default=False, null=False)
new = models.BooleanField(default=True, null=False)
downloaded_path = models.TextField(null=True, blank=True)
downloaded_size = models.IntegerField(null=True, blank=True)
subscription = models.ForeignKey(Subscription, on_delete=models.CASCADE)
playlist_index = models.IntegerField(null=False)
publish_date = models.DateTimeField(null=False)
last_updated_date = models.DateTimeField(null=False, auto_now=True)
thumbnail = models.TextField()
uploader_name = models.CharField(null=False, max_length=255)
views = models.IntegerField(null=False, default=0)
rating = models.FloatField(null=False, default=0.5)
def get_files(self):
if self.downloaded_path is not None:
directory, file_pattern = os.path.split(self.downloaded_path)
for file in os.listdir(directory):
if file.startswith(file_pattern):
yield os.path.join(directory, file)
def find_video(self):
"""
Finds the video file from the downloaded files, and
returns
:return: Tuple containing file path and mime type
"""
for file in self.get_files():
mime, _ = mimetypes.guess_type(file)
if mime is not None and mime.startswith('video/'):
return file, mime
return None, None
def __str__(self):
return self.name
def __repr__(self):
return f'video {self.id}, video_id="{self.video_id}"'

17
app/YtManagerApp/models/video_order.py Normal file
View File

@ -0,0 +1,17 @@
VIDEO_ORDER_CHOICES = [
('newest', 'Newest'),
('oldest', 'Oldest'),
('playlist', 'Playlist order'),
('playlist_reverse', 'Reverse playlist order'),
('popularity', 'Popularity'),
('rating', 'Top rated'),
]
VIDEO_ORDER_MAPPING = {
'newest': '-publish_date',
'oldest': 'publish_date',
'playlist': 'playlist_index',
'playlist_reverse': '-playlist_index',
'popularity': '-views',
'rating': '-rating'
}

6
app/YtManagerApp/models/video_provider.py Normal file
View File

@ -0,0 +1,6 @@
from django.db import models
class VideoProviderConfig(models.Model):
provider_id = models.CharField(max_length=128, unique=True, help_text="Provider ID")
settings = models.TextField(help_text="Video provider settings")

110
app/YtManagerApp/providers/video_provider.py Normal file
View File

@ -0,0 +1,110 @@
from abc import abstractmethod, ABC
from typing import Dict, Iterable, List, Any
from django.forms import Field
from YtManagerApp.models import Subscription, Video
class ConfigurationValidationError(ValueError):
"""
Exception type thrown when validating configurations.
"""
def __init__(self, field_messages: Dict[str, str], *args, **kwargs):
"""
Constructor
:param field_messages: A dictionary which maps field names to errors, which will be displayed to the user.
"""
super().__init__(*args, **kwargs)
self.field_messages = field_messages
class InvalidURLError(ValueError):
"""
Invalid URL exception type
"""
pass
class VideoProvider(ABC):
name: str = ""
settings: Dict[str, Field] = {}
@abstractmethod
def configure(self, configuration: Dict[str, Any]) -> None:
"""
Configures the video provider
:param configuration: A dictionary containing key-value pairs based on the settings defined.
:return: None
"""
pass
@abstractmethod
def validate_configuration(self, configuration: Dict[str, Any]) -> None:
"""
Validates the given configuration. This is executed when validating the settings form from the UI.
:param configuration: Dictionary containing key-value pairs, based on the settings defined.
:except ConfigurationValidationError Thrown if there are validation errors
"""
pass
@abstractmethod
def get_subscription_url(self, subscription: Subscription) -> str:
"""
Builds an URL that links to the given subscription.
:param subscription: The subscription
:return: URL
"""
pass
@abstractmethod
def validate_subscription_url(self, url: str) -> None:
"""
Validates given URL. Throws InvalidURLError if not valid.
:param url: URL to validate
:except InvalidURLError Thrown if the URL is not valid
"""
pass
@abstractmethod
def fetch_subscription(self, url: str) -> Subscription:
"""
Fetches a subscription using given URL
:param url: Subscription URL
:return: Subscription
:except InvalidURLError Thrown if the URL is not valid
"""
pass
@abstractmethod
def get_video_url(self, video: Video) -> str:
"""
Builds an URL that links to the given video.
:param video: The video
:return: URL
"""
pass
@abstractmethod
def fetch_videos(self, subscription: Subscription) -> Iterable[Video]:
"""
Fetches all the subscription items from the given subscription.
The method only needs to fetch the minimum amount of details, the update_videos method
is used to obtain additional information (such as likes/dislikes and other statistics)
:param subscription:
:return:
"""
pass
@abstractmethod
def update_videos(self, videos: List[Video], update_metadata=False, update_statistics=False) -> None:
"""
Updates the metadata for all the videos in the list.
:param videos: Videos
:param update_metadata: If true, video metadata (name, description) will be updated
:param update_statistics: If true, statistics (likes/dislikes, view count) will be updated.
:return:
"""
pass

156
app/YtManagerApp/providers/ytapi_video_provider.py Normal file
View File

@ -0,0 +1,156 @@
from typing import Dict, Optional, Any, Iterable, List
from django import forms
from external.pytaw.pytaw import youtube as yt
from external.pytaw.pytaw.utils import iterate_chunks
from YtManagerApp.models import Subscription, Video
from YtManagerApp.providers.video_provider import VideoProvider, InvalidURLError
class YouTubeApiVideoProvider(VideoProvider):
name = "YtAPI"
settings = {
"api_key": forms.CharField(label="YouTube API Key:")
}
def __init__(self):
super().__init__()
self.__api_key: str = None
self.__api: yt.YouTube = None
def configure(self, configuration: Dict[str, Any]) -> None:
self.__api_key = configuration['api_key']
self.__api = yt.YouTube(key=self.__api_key)
def validate_configuration(self, configuration: Dict[str, Any]):
# TODO: implement
pass
def get_subscription_url(self, subscription: Subscription):
return f"https://youtube.com/playlist?list={subscription.playlist_id}"
def validate_subscription_url(self, url: str) -> None:
try:
parsed_url = self.__api.parse_url(url)
except yt.InvalidURL:
raise InvalidURLError("The given URL is not valid!")
is_playlist = 'playlist' in parsed_url
is_channel = parsed_url['type'] in ('channel', 'user', 'channel_custom')
if not is_channel and not is_playlist:
raise InvalidURLError('The given URL is not a channel or a playlist!')
def fetch_subscription(self, url: str) -> Subscription:
sub = Subscription()
sub.provider_id = self.name
self.validate_subscription_url(url)
url_parsed = self.__api.parse_url(url)
if 'playlist' in url_parsed:
info_playlist = self.__api.playlist(url=url)
if info_playlist is None:
raise ValueError('Invalid playlist ID!')
sub.name = info_playlist.title
sub.playlist_id = info_playlist.id
sub.description = info_playlist.description
sub.channel_id = info_playlist.channel_id
sub.channel_name = info_playlist.channel_title
sub.thumbnail = self._best_thumbnail(info_playlist).url
else:
info_channel = self.__api.channel(url=url)
if info_channel is None:
raise ValueError('Cannot find channel!')
sub.name = info_channel.title
sub.playlist_id = info_channel.uploads_playlist.id
sub.description = info_channel.description
sub.channel_id = info_channel.id
sub.channel_name = info_channel.title
sub.thumbnail = self._best_thumbnail(info_channel).url
sub.rewrite_playlist_indices = True
return sub
def _default_thumbnail(self, resource: yt.Resource) -> Optional[yt.Thumbnail]:
"""
Gets the default thumbnail for a resource.
Searches in the list of thumbnails for one with the label 'default', or takes the first one.
:param resource:
:return:
"""
thumbs = getattr(resource, 'thumbnails', None)
if thumbs is None or len(thumbs) <= 0:
return None
return next(
(i for i in thumbs if i.id == 'default'),
thumbs[0]
)
def _best_thumbnail(self, resource: yt.Resource) -> Optional[yt.Thumbnail]:
"""
Gets the best thumbnail available for a resource.
:param resource:
:return:
"""
thumbs = getattr(resource, 'thumbnails', None)
if thumbs is None or len(thumbs) <= 0:
return None
return max(thumbs, key=lambda t: t.width * t.height)
def get_video_url(self, video: Video) -> str:
return f"https://youtube.com/watch?v={video.video_id}"
def fetch_videos(self, subscription: Subscription) -> Iterable[Video]:
playlist_items = self.__api.playlist_items(subscription.playlist_id)
for item in playlist_items:
video = Video()
video.video_id = item.resource_video_id
video.name = item.title
video.description = item.description
video.watched = False
video.new = True
video.downloaded_path = None
video.subscription = subscription
video.playlist_index = item.position
video.publish_date = item.published_at
video.thumbnail = self._best_thumbnail(item).url
yield video
def update_videos(self, videos: List[Video], update_metadata=False, update_statistics=False) -> None:
parts = ['id']
if update_metadata:
parts.append('snippet')
if update_statistics:
parts.append('statistics')
# don't waste api resources
if len(parts) <= 1:
return
video_dict = {video.video_id: video for video in videos}
id_list = video_dict.keys()
for batch in iterate_chunks(id_list, 50):
resp_videos = self.__api.videos(batch, part=','.join(parts))
for resp_video in resp_videos:
v = video_dict[resp_video.id]
if update_metadata:
v.name = resp_video.title
v.description = resp_video.description
if update_statistics:
if resp_video.n_likes is not None \
and resp_video.n_dislikes is not None \
and resp_video.n_likes + resp_video.n_dislikes > 0:
v.rating = resp_video.n_likes / (resp_video.n_likes + resp_video.n_dislikes)
v.views = resp_video.n_views

0
app/YtManagerApp/scheduler/jobs/__init__.py
View File

47
app/YtManagerApp/scheduler/jobs/delete_video_job.py
View File

@ -1,47 +0,0 @@
import os
from YtManagerApp.models import Video
from YtManagerApp.scheduler.job import Job
class DeleteVideoJob(Job):
name = "DeleteVideoJob"
def __init__(self, job_execution, video: Video):
super().__init__(job_execution)
self._video = video
def get_description(self):
return f"Deleting video {self._video}"
def run(self):
count = 0
try:
for file in self._video.get_files():
self.log.info("Deleting file %s", file)
count += 1
try:
os.unlink(file)
except OSError as e:
self.log.error("Failed to delete file %s: Error: %s", file, e)
except OSError as e:
self.log.error("Failed to delete video %d [%s %s]. Error: %s", self._video.id,
self._video.video_id, self._video.name, e)
self._video.downloaded_path = None
self._video.save()
self.log.info('Deleted video %d successfully! (%d files) [%s %s]', self._video.id, count,
self._video.video_id, self._video.name)
@staticmethod
def schedule(video: Video):
"""
Schedules a delete video job to run immediately.
:param video:
:return:
"""
from YtManagerApp.services import Services
Services.scheduler.add_job(DeleteVideoJob, args=[video])

136
app/YtManagerApp/scheduler/jobs/download_video_job.py
View File

@ -1,136 +0,0 @@
import os
import re
from string import Template
from threading import Lock
import youtube_dl
from YtManagerApp.models import Video
from YtManagerApp.scheduler.job import Job
class DownloadVideoJob(Job):
name = "DownloadVideoJob"
__lock = Lock()
def __init__(self, job_execution, video: Video, attempt: int = 1):
super().__init__(job_execution)
self._video = video
self._attempt = attempt
self._log_youtube_dl = self.log.getChild('youtube_dl')
def get_description(self):
ret = "Downloading video " + self._video.name
if self._attempt > 1:
ret += f" (attempt {self._attempt})"
return ret
def run(self):
# Issue: if multiple videos are downloaded at the same time, a race condition appears in the mkdirs() call that
# youtube-dl makes, which causes it to fail with the error 'Cannot create folder - file already exists'.
# For now, allow a single download instance.
self.__lock.acquire()
try:
user = self._video.subscription.user
max_attempts = user.preferences['max_download_attempts']
youtube_dl_params, output_path = self.__build_youtube_dl_params(self._video)
with youtube_dl.YoutubeDL(youtube_dl_params) as yt:
ret = yt.download(["https://www.youtube.com/watch?v=" + self._video.video_id])
self.log.info('Download finished with code %d', ret)
if ret == 0:
self._video.downloaded_path = output_path
self._video.save()
self.log.info('Video %d [%s %s] downloaded successfully!', self._video.id, self._video.video_id,
self._video.name)
elif self._attempt <= max_attempts:
self.log.warning('Re-enqueueing video (attempt %d/%d)', self._attempt, max_attempts)
DownloadVideoJob.schedule(self._video, self._attempt + 1)
else:
self.log.error('Multiple attempts to download video %d [%s %s] failed!', self._video.id,
self._video.video_id, self._video.name)
self._video.downloaded_path = ''
self._video.save()
finally:
self.__lock.release()
def __build_youtube_dl_params(self, video: Video):
sub = video.subscription
user = sub.user
# resolve path
download_path = user.preferences['download_path']
template_dict = self.__build_template_dict(video)
output_pattern = Template(user.preferences['download_file_pattern']).safe_substitute(template_dict)
output_path = os.path.join(download_path, output_pattern)
output_path = os.path.normpath(output_path)
youtube_dl_params = {
'logger': self._log_youtube_dl,
'format': user.preferences['download_format'],
'outtmpl': output_path,
'writethumbnail': True,
'writedescription': True,
'writesubtitles': user.preferences['download_subtitles'],
'writeautomaticsub': user.preferences['download_autogenerated_subtitles'],
'allsubtitles': user.preferences['download_subtitles_all'],
'merge_output_format': 'mp4',
'postprocessors': [
{
'key': 'FFmpegMetadata'
},
]
}
sub_langs = user.preferences['download_subtitles_langs'].split(',')
sub_langs = [i.strip() for i in sub_langs]
if len(sub_langs) > 0:
youtube_dl_params['subtitleslangs'] = sub_langs
sub_format = user.preferences['download_subtitles_format']
if len(sub_format) > 0:
youtube_dl_params['subtitlesformat'] = sub_format
return youtube_dl_params, output_path
def __build_template_dict(self, video: Video):
return {
'channel': video.subscription.channel_name,
'channel_id': video.subscription.channel_id,
'playlist': video.subscription.name,
'playlist_id': video.subscription.playlist_id,
'playlist_index': "{:03d}".format(1 + video.playlist_index),
'title': video.name,
'id': video.video_id,
}
def __get_valid_path(self, path):
"""
Normalizes string, converts to lowercase, removes non-alpha characters,
and converts spaces to hyphens.
"""
import unicodedata
value = unicodedata.normalize('NFKD', path).encode('ascii', 'ignore').decode('ascii')
value = re.sub('[:"*]', '', value).strip()
value = re.sub('[?<>|]', '#', value)
return value
@staticmethod
def schedule(video: Video, attempt: int = 1):
"""
Schedules to download video immediately
:param video:
:param attempt:
:return:
"""
from YtManagerApp.services import Services
Services.scheduler.add_job(DownloadVideoJob, args=[video, attempt])

184
app/YtManagerApp/scheduler/jobs/synchronize_job.py
View File

@ -1,184 +0,0 @@
import errno
import itertools
from threading import Lock
from apscheduler.triggers.cron import CronTrigger
from django.db.models import Max
from django.conf import settings
from YtManagerApp.management.downloader import fetch_thumbnail, downloader_process_subscription
from YtManagerApp.models import *
from YtManagerApp.scheduler.job import Job
from YtManagerApp.services import Services
from YtManagerApp.utils import youtube
from external.pytaw.pytaw.utils import iterate_chunks
_ENABLE_UPDATE_STATS = True
class SynchronizeJob(Job):
name = "SynchronizeJob"
__lock = Lock()
running = False
__global_sync_job = None
def __init__(self, job_execution, subscription: Optional[Subscription] = None):
super().__init__(job_execution)
self.__subscription = subscription
self.__api = youtube.YoutubeAPI.build_public()
self.__new_vids = []
def get_description(self):
if self.__subscription is not None:
return "Running synchronization for subscription " + self.__subscription.name
return "Running synchronization..."
def get_subscription_list(self):
if self.__subscription is not None:
return [self.__subscription]
return Subscription.objects.all()
def get_videos_list(self, subs):
return Video.objects.filter(subscription__in=subs)
def run(self):
self.__lock.acquire(blocking=True)
SynchronizeJob.running = True
try:
self.log.info(self.get_description())
# Build list of work items
work_subs = self.get_subscription_list()
work_vids = self.get_videos_list(work_subs)
self.set_total_steps(len(work_subs) + len(work_vids))
# Remove the 'new' flag
work_vids.update(new=False)
# Process subscriptions
for sub in work_subs:
self.progress_advance(1, "Synchronizing subscription " + sub.name)
self.check_new_videos(sub)
self.fetch_missing_thumbnails(sub)
# Add new videos to progress calculation
self.set_total_steps(len(work_subs) + len(work_vids) + len(self.__new_vids))
# Process videos
all_videos = itertools.chain(work_vids, self.__new_vids)
for batch in iterate_chunks(all_videos, 50):
video_stats = {}
if _ENABLE_UPDATE_STATS:
batch_ids = [video.video_id for video in batch]
video_stats = {v.id: v for v in self.__api.videos(batch_ids, part='id,statistics')}
for video in batch:
self.progress_advance(1, "Updating video " + video.name)
self.check_video_deleted(video)
self.fetch_missing_thumbnails(video)
if video.video_id in video_stats:
self.update_video_stats(video, video_stats[video.video_id])
# Start downloading videos
for sub in work_subs:
downloader_process_subscription(sub)
finally:
SynchronizeJob.running = False
self.__lock.release()
def check_new_videos(self, sub: Subscription):
playlist_items = self.__api.playlist_items(sub.playlist_id)
if sub.rewrite_playlist_indices:
playlist_items = sorted(playlist_items, key=lambda x: x.published_at)
else:
playlist_items = sorted(playlist_items, key=lambda x: x.position)
for item in playlist_items:
results = Video.objects.filter(video_id=item.resource_video_id, subscription=sub)
if not results.exists():
self.log.info('New video for subscription %s: %s %s"', sub, item.resource_video_id, item.title)
# fix playlist index if necessary
if sub.rewrite_playlist_indices or Video.objects.filter(subscription=sub, playlist_index=item.position).exists():
highest = Video.objects.filter(subscription=sub).aggregate(Max('playlist_index'))['playlist_index__max']
item.position = 1 + (highest or -1)
self.__new_vids.append(Video.create(item, sub))
def fetch_missing_thumbnails(self, obj: Union[Subscription, Video]):
if obj.thumbnail.startswith("http"):
if isinstance(obj, Subscription):
obj.thumbnail = fetch_thumbnail(obj.thumbnail, 'sub', obj.playlist_id, settings.THUMBNAIL_SIZE_SUBSCRIPTION)
elif isinstance(obj, Video):
obj.thumbnail = fetch_thumbnail(obj.thumbnail, 'video', obj.video_id, settings.THUMBNAIL_SIZE_VIDEO)
obj.save()
def check_video_deleted(self, video: Video):
if video.downloaded_path is not None:
files = []
try:
files = list(video.get_files())
except OSError as e:
if e.errno != errno.ENOENT:
self.log.error("Could not access path %s. Error: %s", video.downloaded_path, e)
self.usr_err(f"Could not access path {video.downloaded_path}: {e}", suppress_notification=True)
return
# Try to find a valid video file
found_video = False
for file in files:
mime, _ = mimetypes.guess_type(file)
if mime is not None and mime.startswith("video"):
found_video = True
# Video not found, we can safely assume that the video was deleted.
if not found_video:
self.log.info("Video %d was deleted! [%s %s]", video.id, video.video_id, video.name)
# Clean up
for file in files:
try:
os.unlink(file)
except OSError as e:
self.log.error("Could not delete redundant file %s. Error: %s", file, e)
self.usr_err(f"Could not delete redundant file {file}: {e}", suppress_notification=True)
video.downloaded_path = None
# Mark watched?
user = video.subscription.user
if user.preferences['mark_deleted_as_watched']:
video.watched = True
video.save()
def update_video_stats(self, video: Video, yt_video):
if yt_video.n_likes is not None \
and yt_video.n_dislikes is not None \
and yt_video.n_likes + yt_video.n_dislikes > 0:
video.rating = yt_video.n_likes / (yt_video.n_likes + yt_video.n_dislikes)
video.views = yt_video.n_views
video.save()
@staticmethod
def schedule_global_job():
trigger = CronTrigger.from_crontab(Services.appConfig.sync_schedule)
if SynchronizeJob.__global_sync_job is None:
trigger = CronTrigger.from_crontab(Services.appConfig.sync_schedule)
SynchronizeJob.__global_sync_job = Services.scheduler.add_job(SynchronizeJob, trigger, max_instances=1, coalesce=True)
else:
SynchronizeJob.__global_sync_job.reschedule(trigger, max_instances=1, coalesce=True)
@staticmethod
def schedule_now():
Services.scheduler.add_job(SynchronizeJob, max_instances=1, coalesce=True)
@staticmethod
def schedule_now_for_subscription(subscription):
Services.scheduler.add_job(SynchronizeJob, user=subscription.user, args=[subscription])

13
app/YtManagerApp/services.py
View File

@ -2,12 +2,25 @@ import dependency_injector.containers as containers
import dependency_injector.providers as providers import dependency_injector.providers as providers
from dynamic_preferences.registries import global_preferences_registry from dynamic_preferences.registries import global_preferences_registry
from YtManagerApp.management.appconfig import AppConfig from YtManagerApp.management.appconfig import AppConfig
from YtManagerApp.management.download_manager import DownloadManager
from YtManagerApp.management.subscription_manager import SubscriptionManager
from YtManagerApp.management.video_manager import VideoManager
from YtManagerApp.management.video_provider_manager import VideoProviderManager
from YtManagerApp.management.youtube_dl_manager import YoutubeDlManager from YtManagerApp.management.youtube_dl_manager import YoutubeDlManager
from YtManagerApp.scheduler.scheduler import YtsmScheduler from YtManagerApp.scheduler.scheduler import YtsmScheduler
class VideoProviders(containers.DeclarativeContainer):
from YtManagerApp.providers.ytapi_video_provider import YouTubeApiVideoProvider
ytApiProvider = providers.Factory(YouTubeApiVideoProvider)
class Services(containers.DeclarativeContainer): class Services(containers.DeclarativeContainer):
globalPreferencesRegistry = providers.Object(global_preferences_registry.manager()) globalPreferencesRegistry = providers.Object(global_preferences_registry.manager())
appConfig = providers.Singleton(AppConfig, globalPreferencesRegistry) appConfig = providers.Singleton(AppConfig, globalPreferencesRegistry)
scheduler = providers.Singleton(YtsmScheduler, appConfig) scheduler = providers.Singleton(YtsmScheduler, appConfig)
youtubeDLManager = providers.Singleton(YoutubeDlManager) youtubeDLManager = providers.Singleton(YoutubeDlManager)
videoManager = providers.Singleton(VideoManager)
videoProviderManager = providers.Singleton(VideoProviderManager, [VideoProviders.ytApiProvider()])
subscriptionManager = providers.Singleton(SubscriptionManager)
downloadManager = providers.Singleton(DownloadManager)

19
app/YtManagerApp/utils/algorithms.py
View File

@ -55,3 +55,22 @@ def bisect_left(a, x, lo=0, hi=None, key=None):
# Create aliases # Create aliases
bisect = bisect_right bisect = bisect_right
def group_by(data, key):
"""
Groups the given data into a dictionary matching the structure { key : [values] }
:param data: Iterable data to be grouped
:param key: Key used to group the data
:return: A dictionary containing the grouped data
"""
result = {}
for entry in data:
entry_key = key(entry)
if entry_key not in result:
result[entry_key] = [entry]
else:
result[entry_key].append(entry)
return result

49
app/YtManagerApp/utils/youtube.py
View File

@ -1,49 +0,0 @@
from django.conf import settings
from external.pytaw.pytaw.youtube import YouTube, Channel, Playlist, PlaylistItem, Thumbnail, InvalidURL, Resource, Video
from typing import Optional
class YoutubeAPI(YouTube):
@staticmethod
def build_public() -> 'YoutubeAPI':
from YtManagerApp.management.appconfig import appconfig
return YoutubeAPI(key=appconfig.youtube_api_key)
# @staticmethod
# def build_oauth() -> 'YoutubeAPI':
# flow =
# credentials =
# service = build(API_SERVICE_NAME, API_VERSION, credentials)
def default_thumbnail(resource: Resource) -> Optional[Thumbnail]:
"""
Gets the default thumbnail for a resource.
Searches in the list of thumbnails for one with the label 'default', or takes the first one.
:param resource:
:return:
"""
thumbs = getattr(resource, 'thumbnails', None)
if thumbs is None or len(thumbs) <= 0:
return None
return next(
(i for i in thumbs if i.id == 'default'),
thumbs[0]
)
def best_thumbnail(resource: Resource) -> Optional[Thumbnail]:
"""
Gets the best thumbnail available for a resource.
:param resource:
:return:
"""
thumbs = getattr(resource, 'thumbnails', None)
if thumbs is None or len(thumbs) <= 0:
return None
return max(thumbs, key=lambda t: t.width * t.height)

4
app/YtManagerApp/views/actions.py
View File

@ -3,12 +3,12 @@ from django.http import JsonResponse
from django.views.generic import View from django.views.generic import View
from YtManagerApp.models import Video from YtManagerApp.models import Video
from YtManagerApp.scheduler.jobs.synchronize_job import SynchronizeJob from YtManagerApp.services import Services
class SyncNowView(LoginRequiredMixin, View): class SyncNowView(LoginRequiredMixin, View):
def post(self, *args, **kwargs): def post(self, *args, **kwargs):
SynchronizeJob.schedule_now() Services.subscriptionManager().synchronize_all()
return JsonResponse({ return JsonResponse({
'success': True 'success': True
}) })

2
app/YtManagerApp/views/auth.py
View File

@ -36,7 +36,7 @@ class RegisterView(FormView):
return context return context
def post(self, request, *args, **kwargs): def post(self, request, *args, **kwargs):
if not Services.appConfig.allow_registrations: if not Services.appConfig().allow_registrations:
return HttpResponseForbidden("Registrations are disabled!") return HttpResponseForbidden("Registrations are disabled!")
return super().post(request, *args, **kwargs) return super().post(request, *args, **kwargs)

25
app/YtManagerApp/views/first_time.py
View File

@ -8,7 +8,6 @@ from django.shortcuts import redirect
from django.urls import reverse_lazy from django.urls import reverse_lazy
from django.views.generic import FormView from django.views.generic import FormView
from YtManagerApp.scheduler.jobs.synchronize_job import SynchronizeJob
from YtManagerApp.services import Services from YtManagerApp.services import Services
from YtManagerApp.views.forms.first_time import WelcomeForm, ApiKeyForm, PickAdminUserForm, ServerConfigForm, DoneForm, \ from YtManagerApp.views.forms.first_time import WelcomeForm, ApiKeyForm, PickAdminUserForm, ServerConfigForm, DoneForm, \
UserCreationForm, LoginForm UserCreationForm, LoginForm
@ -16,7 +15,7 @@ from YtManagerApp.views.forms.first_time import WelcomeForm, ApiKeyForm, PickAdm
logger = logging.getLogger("FirstTimeWizard") logger = logging.getLogger("FirstTimeWizard")
class WizardStepMixin: class WizardStepMixin(object):
def __init__(self, *args, **kwargs): def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs) super().__init__(*args, **kwargs)
@ -24,7 +23,7 @@ class WizardStepMixin:
def get(self, request, *args, **kwargs): def get(self, request, *args, **kwargs):
# Prevent access if application is already initialized # Prevent access if application is already initialized
if Services.appConfig.initialized: if Services.appConfig().initialized:
logger.debug(f"Attempted to access {request.path}, but first time setup already run. Redirected to home " logger.debug(f"Attempted to access {request.path}, but first time setup already run. Redirected to home "
f"page.") f"page.")
return redirect('home') return redirect('home')
@ -32,7 +31,7 @@ class WizardStepMixin:
return super().get(request, *args, **kwargs) return super().get(request, *args, **kwargs)
def post(self, request, *args, **kwargs): def post(self, request, *args, **kwargs):
if Services.appConfig.initialized: if Services.appConfig().initialized:
logger.debug(f"Attempted to post {request.path}, but first time setup already run.") logger.debug(f"Attempted to post {request.path}, but first time setup already run.")
return HttpResponseForbidden() return HttpResponseForbidden()
return super().post(request, *args, **kwargs) return super().post(request, *args, **kwargs)
@ -65,14 +64,14 @@ class Step1ApiKeyView(WizardStepMixin, FormView):
def get_initial(self): def get_initial(self):
initial = super().get_initial() initial = super().get_initial()
initial['api_key'] = Services.appConfig.youtube_api_key initial['api_key'] = Services.appConfig().youtube_api_key
return initial return initial
def form_valid(self, form): def form_valid(self, form):
key = form.cleaned_data['api_key'] key = form.cleaned_data['api_key']
# TODO: validate key # TODO: validate key
if key is not None and len(key) > 0: if key is not None and len(key) > 0:
Services.appConfig.youtube_api_key = key Services.appConfig().youtube_api_key = key
return super().form_valid(form) return super().form_valid(form)
@ -149,8 +148,8 @@ class Step3ConfigureView(WizardStepMixin, FormView):
def get_initial(self): def get_initial(self):
initial = super().get_initial() initial = super().get_initial()
initial['allow_registrations'] = Services.appConfig.allow_registrations initial['allow_registrations'] = Services.appConfig().allow_registrations
initial['sync_schedule'] = Services.appConfig.sync_schedule initial['sync_schedule'] = Services.appConfig().sync_schedule
initial['auto_download'] = self.request.user.preferences['auto_download'] initial['auto_download'] = self.request.user.preferences['auto_download']
initial['download_location'] = self.request.user.preferences['download_path'] initial['download_location'] = self.request.user.preferences['download_path']
return initial return initial
@ -158,11 +157,11 @@ class Step3ConfigureView(WizardStepMixin, FormView):
def form_valid(self, form): def form_valid(self, form):
allow_registrations = form.cleaned_data['allow_registrations'] allow_registrations = form.cleaned_data['allow_registrations']
if allow_registrations is not None: if allow_registrations is not None:
Services.appConfig.allow_registrations = allow_registrations Services.appConfig().allow_registrations = allow_registrations
sync_schedule = form.cleaned_data['sync_schedule'] sync_schedule = form.cleaned_data['sync_schedule']
if sync_schedule is not None and len(sync_schedule) > 0: if sync_schedule is not None and len(sync_schedule) > 0:
Services.appConfig.sync_schedule = sync_schedule Services.appConfig().sync_schedule = sync_schedule
auto_download = form.cleaned_data['auto_download'] auto_download = form.cleaned_data['auto_download']
if auto_download is not None: if auto_download is not None:
@ -173,11 +172,11 @@ class Step3ConfigureView(WizardStepMixin, FormView):
self.request.user.preferences['download_path'] = download_location self.request.user.preferences['download_path'] = download_location
# Set initialized to true # Set initialized to true
Services.appConfig.initialized = True Services.appConfig().initialized = True
# Start scheduler if not started # Start scheduler if not started
Services.scheduler.initialize() Services.scheduler().initialize()
SynchronizeJob.schedule_global_job() Services.subscriptionManager().schedule_global_synchronize_job()
return super().form_valid(form) return super().form_valid(form)

16
app/YtManagerApp/views/forms/settings.py
View File

@ -234,25 +234,25 @@ class AdminSettingsForm(forms.Form):
@staticmethod @staticmethod
def get_initials(): def get_initials():
return { return {
'api_key': Services.appConfig.youtube_api_key, 'api_key': Services.appConfig().youtube_api_key,
'allow_registrations': Services.appConfig.allow_registrations, 'allow_registrations': Services.appConfig().allow_registrations,
'sync_schedule': Services.appConfig.sync_schedule, 'sync_schedule': Services.appConfig().sync_schedule,
'scheduler_concurrency': Services.appConfig.concurrency, 'scheduler_concurrency': Services.appConfig().concurrency,
} }
def save(self): def save(self):
api_key = self.cleaned_data['api_key'] api_key = self.cleaned_data['api_key']
if api_key is not None and len(api_key) > 0: if api_key is not None and len(api_key) > 0:
Services.appConfig.youtube_api_key = api_key Services.appConfig().youtube_api_key = api_key
allow_registrations = self.cleaned_data['allow_registrations'] allow_registrations = self.cleaned_data['allow_registrations']
if allow_registrations is not None: if allow_registrations is not None:
Services.appConfig.allow_registrations = allow_registrations Services.appConfig().allow_registrations = allow_registrations
sync_schedule = self.cleaned_data['sync_schedule'] sync_schedule = self.cleaned_data['sync_schedule']
if sync_schedule is not None and len(sync_schedule) > 0: if sync_schedule is not None and len(sync_schedule) > 0:
Services.appConfig.sync_schedule = sync_schedule Services.appConfig().sync_schedule = sync_schedule
concurrency = self.cleaned_data['scheduler_concurrency'] concurrency = self.cleaned_data['scheduler_concurrency']
if concurrency is not None: if concurrency is not None:
Services.appConfig.concurrency = concurrency Services.appConfig().concurrency = concurrency

73
app/YtManagerApp/views/index.py
View File

@ -1,22 +1,21 @@
from crispy_forms.helper import FormHelper from crispy_forms.helper import FormHelper
from crispy_forms.layout import Layout, Field, HTML from crispy_forms.layout import Layout, Field, HTML
from django import forms from django import forms
from django.conf import settings
from django.contrib.auth.decorators import login_required from django.contrib.auth.decorators import login_required
from django.contrib.auth.mixins import LoginRequiredMixin from django.contrib.auth.mixins import LoginRequiredMixin
from django.core.paginator import Paginator
from django.db.models import Q from django.db.models import Q
from django.http import HttpRequest, HttpResponseBadRequest, JsonResponse from django.http import HttpRequest, HttpResponseBadRequest, JsonResponse
from django.shortcuts import render, redirect from django.shortcuts import render, redirect
from django.views.generic import CreateView, UpdateView, DeleteView, FormView from django.views.generic import CreateView, UpdateView, DeleteView, FormView
from django.views.generic.edit import FormMixin from django.views.generic.edit import FormMixin
from django.conf import settings
from django.core.paginator import Paginator
from YtManagerApp.management.videos import get_videos
from YtManagerApp.models import Subscription, SubscriptionFolder, VIDEO_ORDER_CHOICES, VIDEO_ORDER_MAPPING
from YtManagerApp.services import Services
from YtManagerApp.utils import youtube, subscription_file_parser
from YtManagerApp.views.controls.modal import ModalMixin
import logging from YtManagerApp.models import Subscription, SubscriptionFolder, VIDEO_ORDER_CHOICES, VIDEO_ORDER_MAPPING
from YtManagerApp.providers.video_provider import InvalidURLError
from YtManagerApp.services import Services
from YtManagerApp.utils import subscription_file_parser
from YtManagerApp.views.controls.modal import ModalMixin
class VideoFilterForm(forms.Form): class VideoFilterForm(forms.Form):
@ -110,8 +109,7 @@ def __tree_sub_id(sub_id):
def index(request: HttpRequest): def index(request: HttpRequest):
if not Services.appConfig().initialized:
if not Services.appConfig.initialized:
return redirect('first_time_0') return redirect('first_time_0')
context = { context = {
@ -129,7 +127,6 @@ def index(request: HttpRequest):
@login_required @login_required
def ajax_get_tree(request: HttpRequest): def ajax_get_tree(request: HttpRequest):
def visit(node): def visit(node):
if isinstance(node, SubscriptionFolder): if isinstance(node, SubscriptionFolder):
return { return {
@ -157,7 +154,7 @@ def ajax_get_videos(request: HttpRequest):
if request.method == 'POST': if request.method == 'POST':
form = VideoFilterForm(request.POST) form = VideoFilterForm(request.POST)
if form.is_valid(): if form.is_valid():
videos = get_videos( videos = Services.videoManager().get_videos(
user=request.user, user=request.user,
sort_order=form.cleaned_data['sort'], sort_order=form.cleaned_data['sort'],
query=form.cleaned_data['query'], query=form.cleaned_data['query'],
@ -272,7 +269,6 @@ class CreateSubscriptionForm(forms.ModelForm):
def __init__(self, *args, **kwargs): def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs) super().__init__(*args, **kwargs)
self.yt_api = youtube.YoutubeAPI.build_public()
self.helper = FormHelper() self.helper = FormHelper()
self.helper.form_tag = False self.helper.form_tag = False
self.helper.layout = Layout( self.helper.layout = Layout(
@ -289,16 +285,9 @@ class CreateSubscriptionForm(forms.ModelForm):
def clean_playlist_url(self): def clean_playlist_url(self):
playlist_url: str = self.cleaned_data['playlist_url'] playlist_url: str = self.cleaned_data['playlist_url']
try: try:
parsed_url = self.yt_api.parse_url(playlist_url) Services.videoProviderManager().validate_subscription_url(playlist_url)
except youtube.InvalidURL as e: except InvalidURLError as e:
raise forms.ValidationError(str(e)) raise forms.ValidationError(str(e))
is_playlist = 'playlist' in parsed_url
is_channel = parsed_url['type'] in ('channel', 'user', 'channel_custom')
if not is_channel and not is_playlist:
raise forms.ValidationError('The given URL must link to a channel or a playlist!')
return playlist_url return playlist_url
@ -308,10 +297,12 @@ class CreateSubscriptionModal(LoginRequiredMixin, ModalMixin, CreateView):
def form_valid(self, form): def form_valid(self, form):
form.instance.user = self.request.user form.instance.user = self.request.user
api = youtube.YoutubeAPI.build_public()
try: try:
form.instance.fetch_from_url(form.cleaned_data['playlist_url'], api) subscription: Subscription = Services.videoProviderManager().fetch_subscription(
except youtube.InvalidURL as e: form.cleaned_data['playlist_url'])
form.instance.copy_from(subscription)
form.instance.user = self.request.user
except InvalidURLError as e:
return self.modal_response(form, False, str(e)) return self.modal_response(form, False, str(e))
except ValueError as e: except ValueError as e:
return self.modal_response(form, False, str(e)) return self.modal_response(form, False, str(e))
@ -327,8 +318,9 @@ class CreateSubscriptionModal(LoginRequiredMixin, ModalMixin, CreateView):
# except youtube.APIError as e: # except youtube.APIError as e:
# return self.modal_response( # return self.modal_response(
# form, False, 'An error occurred while communicating with the YouTube API: ' + str(e)) # form, False, 'An error occurred while communicating with the YouTube API: ' + str(e))
response = super().form_valid(form)
return super().form_valid(form) Services.subscriptionManager().synchronize(form.instance)
return response
class UpdateSubscriptionForm(forms.ModelForm): class UpdateSubscriptionForm(forms.ModelForm):
@ -407,7 +399,6 @@ class ImportSubscriptionsForm(forms.Form):
def __init__(self, *args, **kwargs): def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs) super().__init__(*args, **kwargs)
self.yt_api = youtube.YoutubeAPI.build_public()
self.helper = FormHelper() self.helper = FormHelper()
self.helper.form_tag = False self.helper.form_tag = False
self.helper.layout = Layout( self.helper.layout = Layout(
@ -456,28 +447,18 @@ class ImportSubscriptionsModal(LoginRequiredMixin, ModalMixin, FormView):
try: try:
url_list = list(subscription_file_parser.parse(file)) url_list = list(subscription_file_parser.parse(file))
except subscription_file_parser.FormatNotSupportedError: except subscription_file_parser.FormatNotSupportedError:
return super().modal_response(form, success=False, return super().modal_response(form,
success=False,
error_msg="The file could not be parsed! " error_msg="The file could not be parsed! "
"Possible problems: format not supported, file is malformed.") "Possible problems: format not supported, file is malformed.")
print(form.cleaned_data) print(form.cleaned_data)
# Create subscriptions Services.subscriptionManager().import_multiple(url_list,
api = youtube.YoutubeAPI.build_public() form.cleaned_data['parent_folder'],
for url in url_list: form.cleaned_data['auto_download'],
sub = Subscription() form.cleaned_data['download_limit'],
sub.user = self.request.user form.cleaned_data['download_order'],
sub.parent_folder = form.cleaned_data['parent_folder'] form.cleaned_data["automatically_delete_watched"])
sub.auto_download = form.cleaned_data['auto_download']
sub.download_limit = form.cleaned_data['download_limit']
sub.download_order = form.cleaned_data['download_order']
sub.automatically_delete_watched = form.cleaned_data["automatically_delete_watched"]
try:
sub.fetch_from_url(url, api)
except Exception as e:
logging.error("Import subscription error - error processing URL %s: %s", url, e)
continue
sub.save()
return super().form_valid(form) return super().form_valid(form)

4
app/YtManagerApp/views/settings.py
View File

@ -3,7 +3,7 @@ from django.http import HttpResponseForbidden
from django.urls import reverse_lazy from django.urls import reverse_lazy
from django.views.generic import FormView from django.views.generic import FormView
from YtManagerApp.scheduler.jobs.synchronize_job import SynchronizeJob from YtManagerApp.services import Services
from YtManagerApp.views.forms.settings import SettingsForm, AdminSettingsForm from YtManagerApp.views.forms.settings import SettingsForm, AdminSettingsForm
@ -45,5 +45,5 @@ class AdminSettingsView(LoginRequiredMixin, FormView):
def form_valid(self, form): def form_valid(self, form):
form.save() form.save()
SynchronizeJob.schedule_global_job() Services.subscriptionManager().schedule_global_synchronize_job()
return super().form_valid(form) return super().form_valid(form)

4
app/YtManagerApp/views/video.py
View File

@ -1,8 +1,6 @@
from django.contrib.auth.decorators import login_required from django.contrib.auth.decorators import login_required
from django.contrib.auth.mixins import LoginRequiredMixin from django.contrib.auth.mixins import LoginRequiredMixin
from django.http import HttpRequest, StreamingHttpResponse, FileResponse from django.http import HttpRequest, FileResponse
from django.urls import reverse, reverse_lazy
from django.views import View
from django.views.generic import DetailView from django.views.generic import DetailView
from YtManagerApp.models import Video from YtManagerApp.models import Video