Major refactor of many things.

This commit is contained in:
2019-12-19 00:27:06 +02:00
parent fd5d05232f
commit 6b843f1fc2
28 changed files with 374 additions and 181 deletions

View File

@ -0,0 +1,15 @@
import dependency_injector.containers as di_containers
import dependency_injector.providers as di_providers
from dynamic_preferences.registries import global_preferences_registry
from YtManagerApp.services.appconfig import AppConfig
from YtManagerApp.services.youtube_dl_manager import YoutubeDlManager
from YtManagerApp.services.scheduler.scheduler import YtsmScheduler
from YtManagerApp.services.video_provider_manager import VideoProviderManager
class Services(di_containers.DeclarativeContainer):
globalPreferencesRegistry = di_providers.Object(global_preferences_registry.manager())
appConfig = di_providers.Singleton(AppConfig, globalPreferencesRegistry)
scheduler = di_providers.Singleton(YtsmScheduler, appConfig)
youtubeDLManager = di_providers.Singleton(YoutubeDlManager)
providerManager = di_providers.Singleton(VideoProviderManager)

View File

@ -0,0 +1,36 @@
from YtManagerApp.dynamic_preferences_registry import Initialized, YouTubeAPIKey, AllowRegistrations, SyncSchedule, SchedulerConcurrency
class AppConfig(object):
# Properties
props = {
'initialized': Initialized,
'youtube_api_key': YouTubeAPIKey,
'allow_registrations': AllowRegistrations,
'sync_schedule': SyncSchedule,
'concurrency': SchedulerConcurrency
}
# Init
def __init__(self, pref_manager):
self.__pref_manager = pref_manager
def __getattr__(self, item):
prop_class = AppConfig.props[item]
prop_full_name = prop_class.section.name + "__" + prop_class.name
return self.__pref_manager[prop_full_name]
def __setattr__(self, key, value):
if key in AppConfig.props:
prop_class = AppConfig.props[key]
prop_full_name = prop_class.section.name + "__" + prop_class.name
self.__pref_manager[prop_full_name] = value
else:
super().__setattr__(key, value)
def for_sub(self, subscription, pref: str):
value = getattr(subscription, pref)
if value is None:
value = subscription.user.preferences[pref]
return value

View File

@ -0,0 +1,111 @@
from YtManagerApp.services.scheduler.jobs.download_video_job import DownloadVideoJob
from YtManagerApp.models import Video, Subscription, VIDEO_ORDER_MAPPING
from YtManagerApp.utils import first_non_null
from django.conf import settings as srv_settings
import logging
import requests
import mimetypes
import os
import PIL.Image
import PIL.ImageOps
from urllib.parse import urljoin
log = logging.getLogger('downloader')
def __get_subscription_config(sub: Subscription):
user = sub.user
enabled = first_non_null(sub.auto_download, user.preferences['auto_download'])
global_limit = user.preferences['download_global_limit']
limit = first_non_null(sub.download_limit, user.preferences['download_subscription_limit'])
order = first_non_null(sub.download_order, user.preferences['download_order'])
order = VIDEO_ORDER_MAPPING[order]
return enabled, global_limit, limit, order
def downloader_process_subscription(sub: Subscription):
log.info('Processing subscription %d [%s %s]', sub.id, sub.playlist_id, sub.id)
enabled, global_limit, limit, order = __get_subscription_config(sub)
log.info('Determined settings enabled=%s global_limit=%d limit=%d order="%s"', enabled, global_limit, limit, order)
if enabled:
videos_to_download = Video.objects\
.filter(subscription=sub, downloaded_path__isnull=True, watched=False)\
.order_by(order)
log.info('%d download candidates.', len(videos_to_download))
if global_limit > 0:
global_downloaded = Video.objects.filter(subscription__user=sub.user, downloaded_path__isnull=False).count()
allowed_count = max(global_limit - global_downloaded, 0)
videos_to_download = videos_to_download[0:allowed_count]
log.info('Global limit is set, can only download up to %d videos.', allowed_count)
if limit > 0:
sub_downloaded = Video.objects.filter(subscription=sub, downloaded_path__isnull=False).count()
allowed_count = max(limit - sub_downloaded, 0)
videos_to_download = videos_to_download[0:allowed_count]
log.info('Limit is set, can only download up to %d videos.', allowed_count)
# enqueue download
for video in videos_to_download:
log.info('Enqueuing video %d [%s %s] index=%d', video.id, video.video_id, video.name, video.playlist_index)
DownloadVideoJob.schedule(video)
log.info('Finished processing subscription %d [%s %s]', sub.id, sub.playlist_id, sub.id)
def downloader_process_all():
for subscription in Subscription.objects.all():
downloader_process_subscription(subscription)
def fetch_thumbnail(url, object_type, identifier, thumb_size):
log.info('Fetching thumbnail url=%s object_type=%s identifier=%s', url, object_type, identifier)
# Make request to obtain mime type
try:
response = requests.get(url, stream=True)
except requests.exceptions.RequestException as e:
log.error('Failed to fetch thumbnail %s. Error: %s', url, e)
return url
ext = mimetypes.guess_extension(response.headers['Content-Type'])
# Build file path
file_name = f"{identifier}{ext}"
abs_path_dir = os.path.join(srv_settings.MEDIA_ROOT, "thumbs", object_type)
abs_path = os.path.join(abs_path_dir, file_name)
abs_path_tmp = file_name + '.tmp'
# Store image
try:
os.makedirs(abs_path_dir, exist_ok=True)
with open(abs_path_tmp, "wb") as f:
for chunk in response.iter_content(chunk_size=1024):
if chunk:
f.write(chunk)
# Resize and crop to thumbnail size
image = PIL.Image.open(abs_path_tmp)
image = PIL.ImageOps.fit(image, thumb_size)
image.save(abs_path)
image.close()
# Delete temp file
os.unlink(abs_path_tmp)
except requests.exceptions.RequestException as e:
log.error('Error while downloading stream for thumbnail %s. Error: %s', url, e)
return url
except OSError as e:
log.error('Error while writing to file %s for thumbnail %s. Error: %s', abs_path, url, e)
return url
# Return
media_url = urljoin(srv_settings.MEDIA_URL, f"thumbs/{object_type}/{file_name}")
return media_url

View File

@ -0,0 +1,88 @@
from abc import abstractmethod, ABC
from typing import Iterable, ClassVar
from django import forms
from YtManagerApp.models import Subscription, Video
from YtManagerApp.services.scheduler.progress_tracker import ProgressTracker
class VideoProvider(ABC):
"""
Represents a video hosting service that provides videos and playlists (e.g. YouTube, Vimeo).
Note: the method implementations should be thread safe, as they may be called from multiple jobs running in
parallel.
"""
@abstractmethod
def update_configuration(self, **kwargs):
"""
Updates the configuration options of this video provider.
This method is called first when the provider is registered using the configuration stored in the
database. After that, the method will be called when the user changes any configuration options.
:param kwargs: Configuration arguments
"""
pass
@abstractmethod
def get_display_name(self) -> str:
"""
Returns an user friendly name for this provider.
:return:
"""
pass
@abstractmethod
def get_provider_id(self) -> str:
"""
Returns an identifier that uniquely identifies this provider.
:return:
"""
pass
@abstractmethod
def validate_playlist_url(self, url: str) -> bool:
"""
Validates that the given playlist URL is valid for the given video provider service.
:param url:
:return:
"""
pass
@abstractmethod
def fetch_playlist(self, url: str) -> Subscription:
"""
Gets metadata about the playlist identified by the given URL.
:param url:
:return:
"""
pass
@abstractmethod
def fetch_videos(self, subscription: Subscription) -> Iterable[Video]:
"""
Gets metadata about the videos in the given playlist.
:param subscription:
:return:
"""
pass
@abstractmethod
def update_videos(self, videos: Iterable[Video], progress_tracker: ProgressTracker, update_info: bool = True, update_stats: bool = False):
"""
Updates metadata about given videos.
:param update_info: If set to true, basic information such as title, description will be updated
:param update_stats: If set to true, video statistics (such as rating, view counts) will be updated
:param videos: Videos to be updated.
:param progress_tracker: Used to track the progress of the update process
:return:
"""
pass
@abstractmethod
def get_config_form(self) -> ClassVar[forms.Form]:
"""
Gets the configuration form
:return:
"""
pass

View File

@ -0,0 +1,167 @@
import json
from typing import ClassVar, Iterable, Optional
from django import forms
from external.pytaw.pytaw.utils import iterate_chunks
from external.pytaw.pytaw.youtube import YouTube, Thumbnail, InvalidURL, Resource, Video
from YtManagerApp.models import Video, Subscription
from YtManagerApp.services.providers.video_provider import VideoProvider
from YtManagerApp.services.scheduler.progress_tracker import ProgressTracker
class YouTubeConfigForm(forms.Form):
api_key = forms.CharField(label="YouTube API Key:")
class YouTubeProvider(VideoProvider):
def __init__(self):
self._apiKey: str = None
self._api: YouTube = None
def _sanity_check(self):
if self._apiKey is None:
raise ValueError("The YouTube API key is not set!")
@staticmethod
def _best_thumbnail(resource: Resource) -> Optional[Thumbnail]:
"""
Gets the best thumbnail available for a resource.
:param resource:
:return:
"""
thumbs = getattr(resource, 'thumbnails', None)
if thumbs is None or len(thumbs) <= 0:
return None
return max(thumbs, key=lambda t: t.width * t.height)
def update_configuration(self, **kwargs):
self._apiKey = kwargs.get('apiKey')
self._sanity_check()
self._api = YouTube(key=self._apiKey)
def get_display_name(self) -> str:
return "YouTube API Provider"
def get_provider_id(self) -> str:
return 'youtube'
def validate_playlist_url(self, url: str) -> bool:
try:
parsed_url = self._api.parse_url(url)
except InvalidURL:
return False
is_playlist = 'playlist' in parsed_url
is_channel = parsed_url['type'] in ('channel', 'user', 'channel_custom')
return is_playlist or is_channel
def fetch_playlist(self, url: str) -> Subscription:
if not self.validate_playlist_url(url):
raise ValueError("Invalid playlist or channel URL")
parsed_url = self._api.parse_url(url)
sub = Subscription()
if 'playlist' in parsed_url:
info = self._api.playlist(url=url)
if info is None:
raise ValueError('Invalid playlist ID!')
provider_data = {
'channel_id': None,
'rewrite_indices': False
}
sub.provider_id = info.id
else:
info = self._api.channel(url=url)
if info is None:
raise ValueError('Cannot find channel!')
provider_data = {
'channel_id': info.id,
'rewrite_indices': True
}
sub.provider_id = info.uploads_playlist.id
sub.name = info.title
sub.description = info.description
sub.original_url = url
sub.thumbnail = YouTubeProvider._best_thumbnail(info).url
sub.provider = self.get_provider_id()
sub.provider_data = json.dumps(provider_data)
return sub
def fetch_videos(self, subscription: Subscription) -> Iterable[Video]:
provider_data = json.loads(subscription.provider_data)
playlist_items = self._api.playlist_items(subscription.provider_id)
if provider_data.get('rewrite_indices'):
playlist_items = sorted(playlist_items, key=lambda x: x.published_at)
else:
playlist_items = sorted(playlist_items, key=lambda x: x.position)
i = 1
for playlist_item in playlist_items:
video = Video()
video.name = playlist_item.title
video.description = playlist_item.description
video.publish_date = playlist_item.published_at
video.thumbnail = YouTubeProvider._best_thumbnail(playlist_item).url
video.uploader_name = ""
video.provider_id = playlist_item.resource_video_id
video.provider_data = None
if provider_data.get('rewrite_indices'):
video.playlist_index = i
i += 1
else:
video.playlist_index = playlist_item.position
video.downloaded_path = None
video.subscription = subscription
video.watched = False
video.new = True
yield video
def update_videos(self, videos: Iterable[Video], progress_tracker: ProgressTracker, update_info: bool = True, update_stats: bool = False):
videos_list = list(videos)
progress_tracker.total_steps = len(videos_list)
parts = 'id'
if update_info:
parts += ',snippet'
if update_stats:
parts += ',statistics'
for batch in iterate_chunks(videos_list, 50):
batch_ids = [video.video_id for video in batch]
videos_new = {v.id: v for v in self._api.videos(batch_ids, part=parts)}
for video in batch:
progress_tracker.advance(1, "Updating video " + video.name)
video_new = videos_new.get(video.provider_id)
if video_new is None:
continue
if update_info:
video.name = video_new.title
video.description = video_new.description
if update_stats:
if video_new.n_likes is not None \
and video_new.n_dislikes is not None \
and video_new.n_likes + video_new.n_dislikes > 0:
video.rating = video_new.n_likes / (video_new.n_likes + video_new.n_dislikes)
video.views = video_new.n_views
def get_config_form(self) -> ClassVar[forms.Form]:
return YouTubeConfigForm

View File

@ -0,0 +1,118 @@
import logging
from abc import abstractmethod, ABC
from typing import Optional
from YtManagerApp.models import JOB_MESSAGE_LEVELS_MAP, JobMessage
from .progress_tracker import ProgressTracker
class Job(ABC):
name = 'GenericJob'
"""
Base class for jobs running in the scheduler.
"""
def __init__(self, job_execution, *_):
self.job_execution = job_execution
self.log = logging.getLogger(self.name)
self.__progress_tracker = ProgressTracker(listener=Job.__on_progress,
listener_args=[self])
@abstractmethod
def get_description(self) -> str:
"""
Gets a user friendly description of this job.
Should be overriden in job classes.
:return:
"""
return "Running job..."
#
# progress tracking
#
def __on_progress(self, percent: float, message: str):
self.usr_log(message, progress=percent)
def set_total_steps(self, steps: float):
"""
Sets the total number of work steps this task has. This is used for tracking progress.
Should be overriden in job classes.
:return:
"""
self.__progress_tracker.total_steps = steps
def progress_advance(self, steps: float = 1, progress_msg: str = ''):
"""
Advances a number of steps.
:param steps: Number of steps to advance
:param progress_msg: A message which will be passed to a listener
:return:
"""
self.__progress_tracker.advance(steps, progress_msg)
def create_subtask(self, steps: float = 1, subtask_total_steps: float = 100, subtask_initial_steps: float = 0):
"""
Creates a 'subtask' which has its own progress, which will be used in the calculation of the final progress.
:param steps: Number of steps the subtask is 'worth'
:param subtask_total_steps: Total number of steps for subtask
:param subtask_initial_steps: Initial steps for subtask
:return: ProgressTracker for subtask
"""
return self.__progress_tracker.subtask(steps, subtask_total_steps, subtask_initial_steps)
#
# user log messages
#
def usr_log(self, message, progress: Optional[float] = None, level: int = JOB_MESSAGE_LEVELS_MAP['normal'],
suppress_notification: bool = False):
"""
Creates a new log message which will be shown on the user interface.
Progress can also be updated using this method.
:param message: A message to be displayed to the user
:param progress: Progress percentage in [0,1] interval
:param level: Log level (normal, warning, error)
:param suppress_notification: If set to true, a notification will not displayed to the user, but it will
appear in the system logs.
:return:
"""
message = JobMessage(job=self.job_execution,
progress=progress,
message=message,
level=level,
suppress_notification=suppress_notification)
message.save()
def usr_warn(self, message, progress: Optional[float] = None, suppress_notification: bool = False):
"""
Creates a new warning message which will be shown on the user interface.
Progress can also be updated using this method.
:param message: A message to be displayed to the user
:param progress: Progress percentage in [0,1] interval
:param suppress_notification: If set to true, a notification will not displayed to the user, but it will
appear in the system logs.
:return:
"""
self.usr_log(message, progress, JOB_MESSAGE_LEVELS_MAP['warning'], suppress_notification)
def usr_err(self, message, progress: Optional[float] = None, suppress_notification: bool = False):
"""
Creates a new error message which will be shown on the user interface.
Progress can also be updated using this method.
:param message: A message to be displayed to the user
:param progress: Progress percentage in [0,1] interval
:param suppress_notification: If set to true, a notification will not displayed to the user, but it will
appear in the system logs.
:return:
"""
self.usr_log(message, progress, JOB_MESSAGE_LEVELS_MAP['error'], suppress_notification)
#
# main run method
#
@abstractmethod
def run(self):
pass

View File

@ -0,0 +1,47 @@
import os
from YtManagerApp.models import Video
from YtManagerApp.services.scheduler.job import Job
class DeleteVideoJob(Job):
name = "DeleteVideoJob"
def __init__(self, job_execution, video: Video):
super().__init__(job_execution)
self._video = video
def get_description(self):
return f"Deleting video {self._video}"
def run(self):
count = 0
try:
for file in self._video.get_files():
self.log.info("Deleting file %s", file)
count += 1
try:
os.unlink(file)
except OSError as e:
self.log.error("Failed to delete file %s: Error: %s", file, e)
except OSError as e:
self.log.error("Failed to delete video %d [%s %s]. Error: %s", self._video.id,
self._video.provider_id, self._video.name, e)
self._video.downloaded_path = None
self._video.save()
self.log.info('Deleted video %d successfully! (%d files) [%s %s]', self._video.id, count,
self._video.provider_id, self._video.name)
@staticmethod
def schedule(video: Video):
"""
Schedules a delete video job to run immediately.
:param video:
:return:
"""
from YtManagerApp.services import Services
Services.scheduler().add_job(DeleteVideoJob, args=[video])

View File

@ -0,0 +1,136 @@
import os
import re
from string import Template
from threading import Lock
import youtube_dl
from YtManagerApp.models import Video
from YtManagerApp.services.scheduler.job import Job
class DownloadVideoJob(Job):
name = "DownloadVideoJob"
__lock = Lock()
def __init__(self, job_execution, video: Video, attempt: int = 1):
super().__init__(job_execution)
self._video = video
self._attempt = attempt
self._log_youtube_dl = self.log.getChild('youtube_dl')
def get_description(self):
ret = "Downloading video " + self._video.name
if self._attempt > 1:
ret += f" (attempt {self._attempt})"
return ret
def run(self):
# Issue: if multiple videos are downloaded at the same time, a race condition appears in the mkdirs() call that
# youtube-dl makes, which causes it to fail with the error 'Cannot create folder - file already exists'.
# For now, allow a single download instance.
self.__lock.acquire()
try:
user = self._video.subscription.user
max_attempts = user.preferences['max_download_attempts']
youtube_dl_params, output_path = self.__build_youtube_dl_params(self._video)
with youtube_dl.YoutubeDL(youtube_dl_params) as yt:
ret = yt.download(["https://www.youtube.com/watch?v=" + self._video.video_id])
self.log.info('Download finished with code %d', ret)
if ret == 0:
self._video.downloaded_path = output_path
self._video.save()
self.log.info('Video %d [%s %s] downloaded successfully!', self._video.id, self._video.video_id,
self._video.name)
elif self._attempt <= max_attempts:
self.log.warning('Re-enqueueing video (attempt %d/%d)', self._attempt, max_attempts)
DownloadVideoJob.schedule(self._video, self._attempt + 1)
else:
self.log.error('Multiple attempts to download video %d [%s %s] failed!', self._video.id,
self._video.video_id, self._video.name)
self._video.downloaded_path = ''
self._video.save()
finally:
self.__lock.release()
def __build_youtube_dl_params(self, video: Video):
sub = video.subscription
user = sub.user
# resolve path
download_path = user.preferences['download_path']
template_dict = self.__build_template_dict(video)
output_pattern = Template(user.preferences['download_file_pattern']).safe_substitute(template_dict)
output_path = os.path.join(download_path, output_pattern)
output_path = os.path.normpath(output_path)
youtube_dl_params = {
'logger': self._log_youtube_dl,
'format': user.preferences['download_format'],
'outtmpl': output_path,
'writethumbnail': True,
'writedescription': True,
'writesubtitles': user.preferences['download_subtitles'],
'writeautomaticsub': user.preferences['download_autogenerated_subtitles'],
'allsubtitles': user.preferences['download_subtitles_all'],
'merge_output_format': 'mp4',
'postprocessors': [
{
'key': 'FFmpegMetadata'
},
]
}
sub_langs = user.preferences['download_subtitles_langs'].split(',')
sub_langs = [i.strip() for i in sub_langs]
if len(sub_langs) > 0:
youtube_dl_params['subtitleslangs'] = sub_langs
sub_format = user.preferences['download_subtitles_format']
if len(sub_format) > 0:
youtube_dl_params['subtitlesformat'] = sub_format
return youtube_dl_params, output_path
def __build_template_dict(self, video: Video):
return {
'channel': video.subscription.channel_name,
'channel_id': video.subscription.channel_id,
'playlist': video.subscription.name,
'playlist_id': video.subscription.playlist_id,
'playlist_index': "{:03d}".format(1 + video.playlist_index),
'title': video.name,
'id': video.video_id,
}
def __get_valid_path(self, path):
"""
Normalizes string, converts to lowercase, removes non-alpha characters,
and converts spaces to hyphens.
"""
import unicodedata
value = unicodedata.normalize('NFKD', path).encode('ascii', 'ignore').decode('ascii')
value = re.sub('[:"*]', '', value).strip()
value = re.sub('[?<>|]', '#', value)
return value
@staticmethod
def schedule(video: Video, attempt: int = 1):
"""
Schedules to download video immediately
:param video:
:param attempt:
:return:
"""
from YtManagerApp.management.services import Services
Services.scheduler.add_job(DownloadVideoJob, args=[video, attempt])

View File

@ -0,0 +1,183 @@
import errno
import itertools
from threading import Lock
from apscheduler.triggers.cron import CronTrigger
from django.db.models import Max
from django.conf import settings
from YtManagerApp.services.downloader import fetch_thumbnail, downloader_process_subscription
from YtManagerApp.models import *
from YtManagerApp.services.scheduler.job import Job
from YtManagerApp.services import Services
from external.pytaw.pytaw.utils import iterate_chunks
_ENABLE_UPDATE_STATS = True
class SynchronizeJob(Job):
name = "SynchronizeJob"
__lock = Lock()
running = False
__global_sync_job = None
def __init__(self, job_execution, subscription: Optional[Subscription] = None):
super().__init__(job_execution)
self.__subscription = subscription
self.__api = youtube.YoutubeAPI.build_public()
self.__new_vids = []
def get_description(self):
if self.__subscription is not None:
return "Running synchronization for subscription " + self.__subscription.name
return "Running synchronization..."
def get_subscription_list(self):
if self.__subscription is not None:
return [self.__subscription]
return Subscription.objects.all()
def get_videos_list(self, subs):
return Video.objects.filter(subscription__in=subs)
def run(self):
self.__lock.acquire(blocking=True)
SynchronizeJob.running = True
try:
self.log.info(self.get_description())
# Build list of work items
work_subs = self.get_subscription_list()
work_vids = self.get_videos_list(work_subs)
self.set_total_steps(len(work_subs) + len(work_vids))
# Remove the 'new' flag
work_vids.update(new=False)
# Process subscriptions
for sub in work_subs:
self.progress_advance(1, "Synchronizing subscription " + sub.name)
self.check_new_videos(sub)
self.fetch_missing_thumbnails(sub)
# Add new videos to progress calculation
self.set_total_steps(len(work_subs) + len(work_vids) + len(self.__new_vids))
# Process videos
all_videos = itertools.chain(work_vids, self.__new_vids)
for batch in iterate_chunks(all_videos, 50):
video_stats = {}
if _ENABLE_UPDATE_STATS:
batch_ids = [video.video_id for video in batch]
video_stats = {v.id: v for v in self.__api.videos(batch_ids, part='id,statistics')}
for video in batch:
self.progress_advance(1, "Updating video " + video.name)
self.check_video_deleted(video)
self.fetch_missing_thumbnails(video)
if video.video_id in video_stats:
self.update_video_stats(video, video_stats[video.video_id])
# Start downloading videos
for sub in work_subs:
downloader_process_subscription(sub)
finally:
SynchronizeJob.running = False
self.__lock.release()
def check_new_videos(self, sub: Subscription):
playlist_items = self.__api.playlist_items(sub.playlist_id)
if sub.rewrite_playlist_indices:
playlist_items = sorted(playlist_items, key=lambda x: x.published_at)
else:
playlist_items = sorted(playlist_items, key=lambda x: x.position)
for item in playlist_items:
results = Video.objects.filter(video_id=item.resource_video_id, subscription=sub)
if not results.exists():
self.log.info('New video for subscription %s: %s %s"', sub, item.resource_video_id, item.title)
# fix playlist index if necessary
if sub.rewrite_playlist_indices or Video.objects.filter(subscription=sub, playlist_index=item.position).exists():
highest = Video.objects.filter(subscription=sub).aggregate(Max('playlist_index'))['playlist_index__max']
item.position = 1 + (highest or -1)
self.__new_vids.append(Video.create(item, sub))
def fetch_missing_thumbnails(self, obj: Union[Subscription, Video]):
if obj.thumbnail.startswith("http"):
if isinstance(obj, Subscription):
obj.thumbnail = fetch_thumbnail(obj.thumbnail, 'sub', obj.playlist_id, settings.THUMBNAIL_SIZE_SUBSCRIPTION)
elif isinstance(obj, Video):
obj.thumbnail = fetch_thumbnail(obj.thumbnail, 'video', obj.provider_id, settings.THUMBNAIL_SIZE_VIDEO)
obj.save()
def check_video_deleted(self, video: Video):
if video.downloaded_path is not None:
files = []
try:
files = list(video.get_files())
except OSError as e:
if e.errno != errno.ENOENT:
self.log.error("Could not access path %s. Error: %s", video.downloaded_path, e)
self.usr_err(f"Could not access path {video.downloaded_path}: {e}", suppress_notification=True)
return
# Try to find a valid video file
found_video = False
for file in files:
mime, _ = mimetypes.guess_type(file)
if mime is not None and mime.startswith("video"):
found_video = True
# Video not found, we can safely assume that the video was deleted.
if not found_video:
self.log.info("Video %d was deleted! [%s %s]", video.id, video.provider_id, video.name)
# Clean up
for file in files:
try:
os.unlink(file)
except OSError as e:
self.log.error("Could not delete redundant file %s. Error: %s", file, e)
self.usr_err(f"Could not delete redundant file {file}: {e}", suppress_notification=True)
video.downloaded_path = None
# Mark watched?
user = video.subscription.user
if user.preferences['mark_deleted_as_watched']:
video.watched = True
video.save()
def update_video_stats(self, video: Video, yt_video):
if yt_video.n_likes is not None \
and yt_video.n_dislikes is not None \
and yt_video.n_likes + yt_video.n_dislikes > 0:
video.rating = yt_video.n_likes / (yt_video.n_likes + yt_video.n_dislikes)
video.views = yt_video.n_views
video.save()
@staticmethod
def schedule_global_job():
trigger = CronTrigger.from_crontab(Services.appConfig().sync_schedule)
if SynchronizeJob.__global_sync_job is None:
trigger = CronTrigger.from_crontab(Services.appConfig().sync_schedule)
SynchronizeJob.__global_sync_job = Services.scheduler().add_job(SynchronizeJob, trigger, max_instances=1, coalesce=True)
else:
SynchronizeJob.__global_sync_job.reschedule(trigger, max_instances=1, coalesce=True)
@staticmethod
def schedule_now():
Services.scheduler.add_job(SynchronizeJob, max_instances=1, coalesce=True)
@staticmethod
def schedule_now_for_subscription(subscription):
Services.scheduler.add_job(SynchronizeJob, user=subscription.user, args=[subscription])

View File

@ -0,0 +1,18 @@
from YtManagerApp.models import Video
from YtManagerApp.services.scheduler.job import Job
class YouTubeDLUpdateJob(Job):
name = "YouTubeDLUpdateJob"
def __init__(self, job_execution):
super().__init__(job_execution)
def get_description(self):
return f"Updating youtube-dl runtime"
def run(self):
from YtManagerApp.services import Services
self.set_total_steps(1)
Services.youtubeDLManager.install()
self.progress_advance(1)

View File

@ -0,0 +1,83 @@
from typing import Callable, List, Any, Optional
class ProgressTracker(object):
"""
Class which helps keep track of complex operation progress.
"""
def __init__(self, total_steps: float = 100, initial_steps: float = 0,
listener: Callable[[float, str], None] = None,
listener_args: List[Any] = None,
parent: Optional["ProgressTracker"] = None):
"""
Constructor
:param total_steps: Total number of steps required by this operation
:param initial_steps: Starting steps
:param parent: Parent progress tracker
:param listener: Callable which is called when any progress happens
"""
self.total_steps = total_steps
self.steps = initial_steps
self.__subtask: ProgressTracker = None
self.__subtask_steps = 0
self.__parent = parent
self.__listener = listener
self.__listener_args = listener_args or []
def __on_progress(self, progress_msg):
if self.__listener is not None:
self.__listener(*self.__listener_args, self.compute_progress(), progress_msg)
if self.__parent is not None:
self.__parent.__on_progress(progress_msg)
def advance(self, steps: float = 1, progress_msg: str = ''):
"""
Advances a number of steps.
:param steps: Number of steps to advance
:param progress_msg: A message which will be passed to a listener
:return:
"""
# We can assume previous subtask is now completed
if self.__subtask is not None:
self.steps += self.__subtask_steps
self.__subtask = None
self.steps += steps
self.__on_progress(progress_msg)
def subtask(self, steps: float = 1, subtask_total_steps: float = 100, subtask_initial_steps: float = 0):
"""
Creates a 'subtask' which has its own progress, which will be used in the calculation of the final progress.
:param steps: Number of steps the subtask is 'worth'
:param subtask_total_steps: Total number of steps for subtask
:param subtask_initial_steps: Initial steps for subtask
:return: ProgressTracker for subtask
"""
# We can assume previous subtask is now completed
if self.__subtask is not None:
self.steps += self.__subtask_steps
self.__subtask = ProgressTracker(total_steps=subtask_total_steps,
initial_steps=subtask_initial_steps,
parent=self)
self.__subtask_steps = steps
return self.__subtask
def compute_progress(self):
"""
Calculates final progress value in percent.
:return: value in [0,1] interval representing progress
"""
base = float(self.steps) / self.total_steps
if self.__subtask is not None:
base += self.__subtask.compute_progress() * self.__subtask_steps / self.total_steps
return min(base, 1.0)

View File

@ -0,0 +1,81 @@
import datetime
import logging
import traceback
from typing import Type, Union, Optional
import pytz
from apscheduler.schedulers.background import BackgroundScheduler
from apscheduler.triggers.base import BaseTrigger
from apscheduler.triggers.interval import IntervalTrigger
from django.contrib.auth.models import User
from YtManagerApp.services.appconfig import AppConfig
from YtManagerApp.models import JobExecution, JOB_STATES_MAP
from YtManagerApp.services.scheduler.job import Job
from YtManagerApp.services.scheduler.jobs.youtubedl_update_job import YouTubeDLUpdateJob
class YtsmScheduler(object):
def __init__(self, app_config: AppConfig):
self._ap_scheduler = BackgroundScheduler()
self._app_config = app_config
def initialize(self):
# set state of existing jobs as "interrupted"
JobExecution.objects\
.filter(status=JOB_STATES_MAP['running'])\
.update(status=JOB_STATES_MAP['interrupted'])
self._configure_scheduler()
self._ap_scheduler.start()
self._schedule_main_jobs()
def _schedule_main_jobs(self):
self.add_job(YouTubeDLUpdateJob, trigger=IntervalTrigger(days=1))
def _configure_scheduler(self):
logger = logging.getLogger('scheduler')
executors = {
'default': {
'type': 'threadpool',
'max_workers': self._app_config.concurrency
}
}
job_defaults = {
'misfire_grace_time': 60 * 60 * 24 * 365 # 1 year
}
self._ap_scheduler.configure(logger=logger, executors=executors, job_defaults=job_defaults)
def _run_job(self, job_class: Type[Job], user: Optional[User], args: Union[tuple, list]):
job_execution = JobExecution(user=user, status=JOB_STATES_MAP['running'])
job_execution.save()
job_instance = job_class(job_execution, *args)
# update description
job_execution.description = job_instance.get_description()
job_execution.save()
try:
job_instance.run()
job_execution.status = JOB_STATES_MAP['finished']
except Exception as ex:
job_instance.log.critical("Job failed with exception: %s", traceback.format_exc())
job_instance.usr_err(job_instance.name + " operation failed: " + str(ex))
job_execution.status = JOB_STATES_MAP['failed']
finally:
job_execution.end_date = datetime.datetime.now(tz=pytz.UTC)
job_execution.save()
def add_job(self, job_class: Type[Job], trigger: Union[str, BaseTrigger] = None,
args: Union[list, tuple] = None,
user: Optional[User] = None,
**kwargs):
if args is None:
args = []
return self._ap_scheduler.add_job(YtsmScheduler._run_job, trigger=trigger, args=[self, job_class, user, args],
**kwargs)

View File

@ -0,0 +1,11 @@
from YtManagerApp.services.providers.video_provider import VideoProvider
class VideoProviderManager(object):
def __init__(self):
self._providers: Dict[str, VideoProvider] = {}
def register_provider(self, provider: VideoProvider):
pid = provider.get_provider_id()
self._providers[pid] = provider

View File

@ -0,0 +1,57 @@
import re
from typing import Optional
from django.contrib.auth.models import User
from django.db.models import Q
from YtManagerApp.models import Subscription, Video, SubscriptionFolder
def get_videos(user: User,
sort_order: Optional[str],
query: Optional[str] = None,
subscription_id: Optional[int] = None,
folder_id: Optional[int] = None,
only_watched: Optional[bool] = None,
only_downloaded: Optional[bool] = None,
):
filter_args = []
filter_kwargs = {
'subscription__user': user
}
# Process query string - basically, we break it down into words,
# and then search for the given text in the name, description, uploader name and subscription name
if query is not None:
for match in re.finditer(r'\w+', query):
word = match[0]
filter_args.append(Q(name__icontains=word)
| Q(description__icontains=word)
| Q(uploader_name__icontains=word)
| Q(subscription__name__icontains=word))
# Subscription id
if subscription_id is not None:
filter_kwargs['subscription_id'] = subscription_id
# Folder id
if folder_id is not None:
# Visit function - returns only the subscription IDs
def visit(node):
if isinstance(node, Subscription):
return node.id
return None
filter_kwargs['subscription_id__in'] = SubscriptionFolder.traverse(folder_id, user, visit)
# Only watched
if only_watched is not None:
filter_kwargs['watched'] = only_watched
# Only downloaded
# - not downloaded (False) -> is null (True)
# - downloaded (True) -> is not null (False)
if only_downloaded is not None:
filter_kwargs['downloaded_path__isnull'] = not only_downloaded
return Video.objects.filter(*filter_args, **filter_kwargs).order_by(sort_order)

View File

@ -0,0 +1,107 @@
import logging
import os
import subprocess
import sys
import requests
from django.conf import settings as dj_settings
LATEST_URL = "https://yt-dl.org/downloads/latest/youtube-dl"
GITHUB_API_LATEST_RELEASE = "https://api.github.com/repos/ytdl-org/youtube-dl/releases/latest"
log = logging.getLogger("YoutubeDlManager")
class YoutubeDlException(Exception):
pass
class YoutubeDlNotInstalledException(YoutubeDlException):
pass
class YoutubeDlRuntimeException(YoutubeDlException):
pass
class YoutubeDlManager(object):
def __init__(self):
self.verbose = False
self.progress = False
self._install_path = os.path.join(dj_settings.DATA_DIR, 'youtube-dl')
def _check_installed(self):
return os.path.isfile(self._install_path) and os.access(self._install_path, os.X_OK)
def _get_run_args(self):
run_args = []
if self.verbose:
run_args.append('-v')
if self.progress:
run_args.append('--newline')
else:
run_args.append('--no-progress')
return run_args
def run(self, *args):
if not self._check_installed():
log.error("Cannot run youtube-dl, it is not installed!")
raise YoutubeDlNotInstalledException
run_args = self._get_run_args()
ret = subprocess.run([sys.executable, self._install_path, *run_args, *args], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
stdout = ret.stdout.decode('utf-8')
if len(stdout) > 0:
log.info("YoutubeDL: " + stdout)
stderr = ret.stderr.decode('utf-8')
if len(stderr) > 0:
log.error("YoutubeDL: " + stderr)
if ret.returncode != 0:
raise YoutubeDlRuntimeException()
return stdout
def get_installed_version(self):
return self.run('--version')
def get_latest_version(self):
resp = requests.get(GITHUB_API_LATEST_RELEASE, allow_redirects=True)
resp.raise_for_status()
info = resp.json()
return info['tag_name']
def install(self):
# Check if we are running the latest version
latest = self.get_latest_version()
try:
current = self.get_installed_version()
except YoutubeDlNotInstalledException:
current = None
if latest == current:
log.info(f"Running latest youtube-dl version ({current})!")
return
# Download latest
resp = requests.get(LATEST_URL, allow_redirects=True, stream=True)
resp.raise_for_status()
with open(self._install_path + ".tmp", "wb") as f:
for chunk in resp.iter_content(10 * 1024):
f.write(chunk)
# Replace
os.unlink(self._install_path)
os.rename(self._install_path + ".tmp", self._install_path)
os.chmod(self._install_path, 555)
# Test run
newver = self.get_installed_version()
if current is None:
log.info(f"Installed youtube-dl version {newver}.")
else:
log.info(f"Upgraded youtube-dl from version {current} to {newver}.")