mirror of
https://github.com/chibicitiberiu/ytsm.git
synced 2024-02-24 05:43:31 +00:00
Major refactor of many things.
This commit is contained in:
15
app/YtManagerApp/services/__init__.py
Normal file
15
app/YtManagerApp/services/__init__.py
Normal file
@ -0,0 +1,15 @@
|
||||
import dependency_injector.containers as di_containers
|
||||
import dependency_injector.providers as di_providers
|
||||
from dynamic_preferences.registries import global_preferences_registry
|
||||
from YtManagerApp.services.appconfig import AppConfig
|
||||
from YtManagerApp.services.youtube_dl_manager import YoutubeDlManager
|
||||
from YtManagerApp.services.scheduler.scheduler import YtsmScheduler
|
||||
from YtManagerApp.services.video_provider_manager import VideoProviderManager
|
||||
|
||||
|
||||
class Services(di_containers.DeclarativeContainer):
|
||||
globalPreferencesRegistry = di_providers.Object(global_preferences_registry.manager())
|
||||
appConfig = di_providers.Singleton(AppConfig, globalPreferencesRegistry)
|
||||
scheduler = di_providers.Singleton(YtsmScheduler, appConfig)
|
||||
youtubeDLManager = di_providers.Singleton(YoutubeDlManager)
|
||||
providerManager = di_providers.Singleton(VideoProviderManager)
|
36
app/YtManagerApp/services/appconfig.py
Normal file
36
app/YtManagerApp/services/appconfig.py
Normal file
@ -0,0 +1,36 @@
|
||||
from YtManagerApp.dynamic_preferences_registry import Initialized, YouTubeAPIKey, AllowRegistrations, SyncSchedule, SchedulerConcurrency
|
||||
|
||||
|
||||
class AppConfig(object):
|
||||
# Properties
|
||||
props = {
|
||||
'initialized': Initialized,
|
||||
'youtube_api_key': YouTubeAPIKey,
|
||||
'allow_registrations': AllowRegistrations,
|
||||
'sync_schedule': SyncSchedule,
|
||||
'concurrency': SchedulerConcurrency
|
||||
}
|
||||
|
||||
# Init
|
||||
def __init__(self, pref_manager):
|
||||
self.__pref_manager = pref_manager
|
||||
|
||||
def __getattr__(self, item):
|
||||
prop_class = AppConfig.props[item]
|
||||
prop_full_name = prop_class.section.name + "__" + prop_class.name
|
||||
return self.__pref_manager[prop_full_name]
|
||||
|
||||
def __setattr__(self, key, value):
|
||||
if key in AppConfig.props:
|
||||
prop_class = AppConfig.props[key]
|
||||
prop_full_name = prop_class.section.name + "__" + prop_class.name
|
||||
self.__pref_manager[prop_full_name] = value
|
||||
else:
|
||||
super().__setattr__(key, value)
|
||||
|
||||
def for_sub(self, subscription, pref: str):
|
||||
value = getattr(subscription, pref)
|
||||
if value is None:
|
||||
value = subscription.user.preferences[pref]
|
||||
|
||||
return value
|
111
app/YtManagerApp/services/downloader.py
Normal file
111
app/YtManagerApp/services/downloader.py
Normal file
@ -0,0 +1,111 @@
|
||||
from YtManagerApp.services.scheduler.jobs.download_video_job import DownloadVideoJob
|
||||
from YtManagerApp.models import Video, Subscription, VIDEO_ORDER_MAPPING
|
||||
from YtManagerApp.utils import first_non_null
|
||||
from django.conf import settings as srv_settings
|
||||
import logging
|
||||
import requests
|
||||
import mimetypes
|
||||
import os
|
||||
import PIL.Image
|
||||
import PIL.ImageOps
|
||||
from urllib.parse import urljoin
|
||||
|
||||
log = logging.getLogger('downloader')
|
||||
|
||||
|
||||
def __get_subscription_config(sub: Subscription):
|
||||
user = sub.user
|
||||
|
||||
enabled = first_non_null(sub.auto_download, user.preferences['auto_download'])
|
||||
global_limit = user.preferences['download_global_limit']
|
||||
limit = first_non_null(sub.download_limit, user.preferences['download_subscription_limit'])
|
||||
order = first_non_null(sub.download_order, user.preferences['download_order'])
|
||||
order = VIDEO_ORDER_MAPPING[order]
|
||||
|
||||
return enabled, global_limit, limit, order
|
||||
|
||||
|
||||
def downloader_process_subscription(sub: Subscription):
|
||||
log.info('Processing subscription %d [%s %s]', sub.id, sub.playlist_id, sub.id)
|
||||
|
||||
enabled, global_limit, limit, order = __get_subscription_config(sub)
|
||||
log.info('Determined settings enabled=%s global_limit=%d limit=%d order="%s"', enabled, global_limit, limit, order)
|
||||
|
||||
if enabled:
|
||||
videos_to_download = Video.objects\
|
||||
.filter(subscription=sub, downloaded_path__isnull=True, watched=False)\
|
||||
.order_by(order)
|
||||
|
||||
log.info('%d download candidates.', len(videos_to_download))
|
||||
|
||||
if global_limit > 0:
|
||||
global_downloaded = Video.objects.filter(subscription__user=sub.user, downloaded_path__isnull=False).count()
|
||||
allowed_count = max(global_limit - global_downloaded, 0)
|
||||
videos_to_download = videos_to_download[0:allowed_count]
|
||||
log.info('Global limit is set, can only download up to %d videos.', allowed_count)
|
||||
|
||||
if limit > 0:
|
||||
sub_downloaded = Video.objects.filter(subscription=sub, downloaded_path__isnull=False).count()
|
||||
allowed_count = max(limit - sub_downloaded, 0)
|
||||
videos_to_download = videos_to_download[0:allowed_count]
|
||||
log.info('Limit is set, can only download up to %d videos.', allowed_count)
|
||||
|
||||
# enqueue download
|
||||
for video in videos_to_download:
|
||||
log.info('Enqueuing video %d [%s %s] index=%d', video.id, video.video_id, video.name, video.playlist_index)
|
||||
DownloadVideoJob.schedule(video)
|
||||
|
||||
log.info('Finished processing subscription %d [%s %s]', sub.id, sub.playlist_id, sub.id)
|
||||
|
||||
|
||||
def downloader_process_all():
|
||||
for subscription in Subscription.objects.all():
|
||||
downloader_process_subscription(subscription)
|
||||
|
||||
|
||||
def fetch_thumbnail(url, object_type, identifier, thumb_size):
|
||||
|
||||
log.info('Fetching thumbnail url=%s object_type=%s identifier=%s', url, object_type, identifier)
|
||||
|
||||
# Make request to obtain mime type
|
||||
try:
|
||||
response = requests.get(url, stream=True)
|
||||
except requests.exceptions.RequestException as e:
|
||||
log.error('Failed to fetch thumbnail %s. Error: %s', url, e)
|
||||
return url
|
||||
|
||||
ext = mimetypes.guess_extension(response.headers['Content-Type'])
|
||||
|
||||
# Build file path
|
||||
file_name = f"{identifier}{ext}"
|
||||
abs_path_dir = os.path.join(srv_settings.MEDIA_ROOT, "thumbs", object_type)
|
||||
abs_path = os.path.join(abs_path_dir, file_name)
|
||||
abs_path_tmp = file_name + '.tmp'
|
||||
|
||||
# Store image
|
||||
try:
|
||||
os.makedirs(abs_path_dir, exist_ok=True)
|
||||
with open(abs_path_tmp, "wb") as f:
|
||||
for chunk in response.iter_content(chunk_size=1024):
|
||||
if chunk:
|
||||
f.write(chunk)
|
||||
|
||||
# Resize and crop to thumbnail size
|
||||
image = PIL.Image.open(abs_path_tmp)
|
||||
image = PIL.ImageOps.fit(image, thumb_size)
|
||||
image.save(abs_path)
|
||||
image.close()
|
||||
|
||||
# Delete temp file
|
||||
os.unlink(abs_path_tmp)
|
||||
|
||||
except requests.exceptions.RequestException as e:
|
||||
log.error('Error while downloading stream for thumbnail %s. Error: %s', url, e)
|
||||
return url
|
||||
except OSError as e:
|
||||
log.error('Error while writing to file %s for thumbnail %s. Error: %s', abs_path, url, e)
|
||||
return url
|
||||
|
||||
# Return
|
||||
media_url = urljoin(srv_settings.MEDIA_URL, f"thumbs/{object_type}/{file_name}")
|
||||
return media_url
|
0
app/YtManagerApp/services/providers/__init__.py
Normal file
0
app/YtManagerApp/services/providers/__init__.py
Normal file
88
app/YtManagerApp/services/providers/video_provider.py
Normal file
88
app/YtManagerApp/services/providers/video_provider.py
Normal file
@ -0,0 +1,88 @@
|
||||
from abc import abstractmethod, ABC
|
||||
from typing import Iterable, ClassVar
|
||||
|
||||
from django import forms
|
||||
|
||||
from YtManagerApp.models import Subscription, Video
|
||||
from YtManagerApp.services.scheduler.progress_tracker import ProgressTracker
|
||||
|
||||
|
||||
class VideoProvider(ABC):
|
||||
"""
|
||||
Represents a video hosting service that provides videos and playlists (e.g. YouTube, Vimeo).
|
||||
Note: the method implementations should be thread safe, as they may be called from multiple jobs running in
|
||||
parallel.
|
||||
"""
|
||||
|
||||
@abstractmethod
|
||||
def update_configuration(self, **kwargs):
|
||||
"""
|
||||
Updates the configuration options of this video provider.
|
||||
This method is called first when the provider is registered using the configuration stored in the
|
||||
database. After that, the method will be called when the user changes any configuration options.
|
||||
:param kwargs: Configuration arguments
|
||||
"""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def get_display_name(self) -> str:
|
||||
"""
|
||||
Returns an user friendly name for this provider.
|
||||
:return:
|
||||
"""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def get_provider_id(self) -> str:
|
||||
"""
|
||||
Returns an identifier that uniquely identifies this provider.
|
||||
:return:
|
||||
"""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def validate_playlist_url(self, url: str) -> bool:
|
||||
"""
|
||||
Validates that the given playlist URL is valid for the given video provider service.
|
||||
:param url:
|
||||
:return:
|
||||
"""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def fetch_playlist(self, url: str) -> Subscription:
|
||||
"""
|
||||
Gets metadata about the playlist identified by the given URL.
|
||||
:param url:
|
||||
:return:
|
||||
"""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def fetch_videos(self, subscription: Subscription) -> Iterable[Video]:
|
||||
"""
|
||||
Gets metadata about the videos in the given playlist.
|
||||
:param subscription:
|
||||
:return:
|
||||
"""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def update_videos(self, videos: Iterable[Video], progress_tracker: ProgressTracker, update_info: bool = True, update_stats: bool = False):
|
||||
"""
|
||||
Updates metadata about given videos.
|
||||
:param update_info: If set to true, basic information such as title, description will be updated
|
||||
:param update_stats: If set to true, video statistics (such as rating, view counts) will be updated
|
||||
:param videos: Videos to be updated.
|
||||
:param progress_tracker: Used to track the progress of the update process
|
||||
:return:
|
||||
"""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def get_config_form(self) -> ClassVar[forms.Form]:
|
||||
"""
|
||||
Gets the configuration form
|
||||
:return:
|
||||
"""
|
||||
pass
|
167
app/YtManagerApp/services/providers/youtube_provider.py
Normal file
167
app/YtManagerApp/services/providers/youtube_provider.py
Normal file
@ -0,0 +1,167 @@
|
||||
import json
|
||||
from typing import ClassVar, Iterable, Optional
|
||||
|
||||
from django import forms
|
||||
from external.pytaw.pytaw.utils import iterate_chunks
|
||||
from external.pytaw.pytaw.youtube import YouTube, Thumbnail, InvalidURL, Resource, Video
|
||||
|
||||
from YtManagerApp.models import Video, Subscription
|
||||
from YtManagerApp.services.providers.video_provider import VideoProvider
|
||||
from YtManagerApp.services.scheduler.progress_tracker import ProgressTracker
|
||||
|
||||
|
||||
class YouTubeConfigForm(forms.Form):
|
||||
api_key = forms.CharField(label="YouTube API Key:")
|
||||
|
||||
|
||||
class YouTubeProvider(VideoProvider):
|
||||
|
||||
def __init__(self):
|
||||
self._apiKey: str = None
|
||||
self._api: YouTube = None
|
||||
|
||||
def _sanity_check(self):
|
||||
if self._apiKey is None:
|
||||
raise ValueError("The YouTube API key is not set!")
|
||||
|
||||
@staticmethod
|
||||
def _best_thumbnail(resource: Resource) -> Optional[Thumbnail]:
|
||||
"""
|
||||
Gets the best thumbnail available for a resource.
|
||||
:param resource:
|
||||
:return:
|
||||
"""
|
||||
thumbs = getattr(resource, 'thumbnails', None)
|
||||
|
||||
if thumbs is None or len(thumbs) <= 0:
|
||||
return None
|
||||
|
||||
return max(thumbs, key=lambda t: t.width * t.height)
|
||||
|
||||
def update_configuration(self, **kwargs):
|
||||
self._apiKey = kwargs.get('apiKey')
|
||||
self._sanity_check()
|
||||
self._api = YouTube(key=self._apiKey)
|
||||
|
||||
def get_display_name(self) -> str:
|
||||
return "YouTube API Provider"
|
||||
|
||||
def get_provider_id(self) -> str:
|
||||
return 'youtube'
|
||||
|
||||
def validate_playlist_url(self, url: str) -> bool:
|
||||
try:
|
||||
parsed_url = self._api.parse_url(url)
|
||||
except InvalidURL:
|
||||
return False
|
||||
|
||||
is_playlist = 'playlist' in parsed_url
|
||||
is_channel = parsed_url['type'] in ('channel', 'user', 'channel_custom')
|
||||
|
||||
return is_playlist or is_channel
|
||||
|
||||
def fetch_playlist(self, url: str) -> Subscription:
|
||||
|
||||
if not self.validate_playlist_url(url):
|
||||
raise ValueError("Invalid playlist or channel URL")
|
||||
|
||||
parsed_url = self._api.parse_url(url)
|
||||
sub = Subscription()
|
||||
|
||||
if 'playlist' in parsed_url:
|
||||
info = self._api.playlist(url=url)
|
||||
if info is None:
|
||||
raise ValueError('Invalid playlist ID!')
|
||||
provider_data = {
|
||||
'channel_id': None,
|
||||
'rewrite_indices': False
|
||||
}
|
||||
sub.provider_id = info.id
|
||||
|
||||
else:
|
||||
info = self._api.channel(url=url)
|
||||
if info is None:
|
||||
raise ValueError('Cannot find channel!')
|
||||
|
||||
provider_data = {
|
||||
'channel_id': info.id,
|
||||
'rewrite_indices': True
|
||||
}
|
||||
sub.provider_id = info.uploads_playlist.id
|
||||
|
||||
sub.name = info.title
|
||||
sub.description = info.description
|
||||
sub.original_url = url
|
||||
sub.thumbnail = YouTubeProvider._best_thumbnail(info).url
|
||||
|
||||
sub.provider = self.get_provider_id()
|
||||
sub.provider_data = json.dumps(provider_data)
|
||||
|
||||
return sub
|
||||
|
||||
def fetch_videos(self, subscription: Subscription) -> Iterable[Video]:
|
||||
provider_data = json.loads(subscription.provider_data)
|
||||
playlist_items = self._api.playlist_items(subscription.provider_id)
|
||||
|
||||
if provider_data.get('rewrite_indices'):
|
||||
playlist_items = sorted(playlist_items, key=lambda x: x.published_at)
|
||||
else:
|
||||
playlist_items = sorted(playlist_items, key=lambda x: x.position)
|
||||
i = 1
|
||||
|
||||
for playlist_item in playlist_items:
|
||||
video = Video()
|
||||
video.name = playlist_item.title
|
||||
video.description = playlist_item.description
|
||||
video.publish_date = playlist_item.published_at
|
||||
video.thumbnail = YouTubeProvider._best_thumbnail(playlist_item).url
|
||||
video.uploader_name = ""
|
||||
|
||||
video.provider_id = playlist_item.resource_video_id
|
||||
video.provider_data = None
|
||||
|
||||
if provider_data.get('rewrite_indices'):
|
||||
video.playlist_index = i
|
||||
i += 1
|
||||
else:
|
||||
video.playlist_index = playlist_item.position
|
||||
video.downloaded_path = None
|
||||
video.subscription = subscription
|
||||
|
||||
video.watched = False
|
||||
video.new = True
|
||||
|
||||
yield video
|
||||
|
||||
def update_videos(self, videos: Iterable[Video], progress_tracker: ProgressTracker, update_info: bool = True, update_stats: bool = False):
|
||||
videos_list = list(videos)
|
||||
progress_tracker.total_steps = len(videos_list)
|
||||
|
||||
parts = 'id'
|
||||
if update_info:
|
||||
parts += ',snippet'
|
||||
if update_stats:
|
||||
parts += ',statistics'
|
||||
|
||||
for batch in iterate_chunks(videos_list, 50):
|
||||
batch_ids = [video.video_id for video in batch]
|
||||
videos_new = {v.id: v for v in self._api.videos(batch_ids, part=parts)}
|
||||
|
||||
for video in batch:
|
||||
progress_tracker.advance(1, "Updating video " + video.name)
|
||||
video_new = videos_new.get(video.provider_id)
|
||||
if video_new is None:
|
||||
continue
|
||||
|
||||
if update_info:
|
||||
video.name = video_new.title
|
||||
video.description = video_new.description
|
||||
if update_stats:
|
||||
if video_new.n_likes is not None \
|
||||
and video_new.n_dislikes is not None \
|
||||
and video_new.n_likes + video_new.n_dislikes > 0:
|
||||
video.rating = video_new.n_likes / (video_new.n_likes + video_new.n_dislikes)
|
||||
video.views = video_new.n_views
|
||||
|
||||
def get_config_form(self) -> ClassVar[forms.Form]:
|
||||
return YouTubeConfigForm
|
0
app/YtManagerApp/services/scheduler/__init__.py
Normal file
0
app/YtManagerApp/services/scheduler/__init__.py
Normal file
118
app/YtManagerApp/services/scheduler/job.py
Normal file
118
app/YtManagerApp/services/scheduler/job.py
Normal file
@ -0,0 +1,118 @@
|
||||
import logging
|
||||
from abc import abstractmethod, ABC
|
||||
from typing import Optional
|
||||
|
||||
from YtManagerApp.models import JOB_MESSAGE_LEVELS_MAP, JobMessage
|
||||
from .progress_tracker import ProgressTracker
|
||||
|
||||
|
||||
class Job(ABC):
|
||||
name = 'GenericJob'
|
||||
|
||||
"""
|
||||
Base class for jobs running in the scheduler.
|
||||
"""
|
||||
|
||||
def __init__(self, job_execution, *_):
|
||||
self.job_execution = job_execution
|
||||
self.log = logging.getLogger(self.name)
|
||||
self.__progress_tracker = ProgressTracker(listener=Job.__on_progress,
|
||||
listener_args=[self])
|
||||
|
||||
@abstractmethod
|
||||
def get_description(self) -> str:
|
||||
"""
|
||||
Gets a user friendly description of this job.
|
||||
Should be overriden in job classes.
|
||||
:return:
|
||||
"""
|
||||
return "Running job..."
|
||||
|
||||
#
|
||||
# progress tracking
|
||||
#
|
||||
|
||||
def __on_progress(self, percent: float, message: str):
|
||||
self.usr_log(message, progress=percent)
|
||||
|
||||
def set_total_steps(self, steps: float):
|
||||
"""
|
||||
Sets the total number of work steps this task has. This is used for tracking progress.
|
||||
Should be overriden in job classes.
|
||||
:return:
|
||||
"""
|
||||
self.__progress_tracker.total_steps = steps
|
||||
|
||||
def progress_advance(self, steps: float = 1, progress_msg: str = ''):
|
||||
"""
|
||||
Advances a number of steps.
|
||||
:param steps: Number of steps to advance
|
||||
:param progress_msg: A message which will be passed to a listener
|
||||
:return:
|
||||
"""
|
||||
self.__progress_tracker.advance(steps, progress_msg)
|
||||
|
||||
def create_subtask(self, steps: float = 1, subtask_total_steps: float = 100, subtask_initial_steps: float = 0):
|
||||
"""
|
||||
Creates a 'subtask' which has its own progress, which will be used in the calculation of the final progress.
|
||||
:param steps: Number of steps the subtask is 'worth'
|
||||
:param subtask_total_steps: Total number of steps for subtask
|
||||
:param subtask_initial_steps: Initial steps for subtask
|
||||
:return: ProgressTracker for subtask
|
||||
"""
|
||||
return self.__progress_tracker.subtask(steps, subtask_total_steps, subtask_initial_steps)
|
||||
|
||||
#
|
||||
# user log messages
|
||||
#
|
||||
|
||||
def usr_log(self, message, progress: Optional[float] = None, level: int = JOB_MESSAGE_LEVELS_MAP['normal'],
|
||||
suppress_notification: bool = False):
|
||||
"""
|
||||
Creates a new log message which will be shown on the user interface.
|
||||
Progress can also be updated using this method.
|
||||
:param message: A message to be displayed to the user
|
||||
:param progress: Progress percentage in [0,1] interval
|
||||
:param level: Log level (normal, warning, error)
|
||||
:param suppress_notification: If set to true, a notification will not displayed to the user, but it will
|
||||
appear in the system logs.
|
||||
:return:
|
||||
"""
|
||||
|
||||
message = JobMessage(job=self.job_execution,
|
||||
progress=progress,
|
||||
message=message,
|
||||
level=level,
|
||||
suppress_notification=suppress_notification)
|
||||
message.save()
|
||||
|
||||
def usr_warn(self, message, progress: Optional[float] = None, suppress_notification: bool = False):
|
||||
"""
|
||||
Creates a new warning message which will be shown on the user interface.
|
||||
Progress can also be updated using this method.
|
||||
:param message: A message to be displayed to the user
|
||||
:param progress: Progress percentage in [0,1] interval
|
||||
:param suppress_notification: If set to true, a notification will not displayed to the user, but it will
|
||||
appear in the system logs.
|
||||
:return:
|
||||
"""
|
||||
self.usr_log(message, progress, JOB_MESSAGE_LEVELS_MAP['warning'], suppress_notification)
|
||||
|
||||
def usr_err(self, message, progress: Optional[float] = None, suppress_notification: bool = False):
|
||||
"""
|
||||
Creates a new error message which will be shown on the user interface.
|
||||
Progress can also be updated using this method.
|
||||
:param message: A message to be displayed to the user
|
||||
:param progress: Progress percentage in [0,1] interval
|
||||
:param suppress_notification: If set to true, a notification will not displayed to the user, but it will
|
||||
appear in the system logs.
|
||||
:return:
|
||||
"""
|
||||
self.usr_log(message, progress, JOB_MESSAGE_LEVELS_MAP['error'], suppress_notification)
|
||||
|
||||
#
|
||||
# main run method
|
||||
#
|
||||
@abstractmethod
|
||||
def run(self):
|
||||
pass
|
47
app/YtManagerApp/services/scheduler/jobs/delete_video_job.py
Normal file
47
app/YtManagerApp/services/scheduler/jobs/delete_video_job.py
Normal file
@ -0,0 +1,47 @@
|
||||
import os
|
||||
|
||||
from YtManagerApp.models import Video
|
||||
from YtManagerApp.services.scheduler.job import Job
|
||||
|
||||
|
||||
class DeleteVideoJob(Job):
|
||||
name = "DeleteVideoJob"
|
||||
|
||||
def __init__(self, job_execution, video: Video):
|
||||
super().__init__(job_execution)
|
||||
self._video = video
|
||||
|
||||
def get_description(self):
|
||||
return f"Deleting video {self._video}"
|
||||
|
||||
def run(self):
|
||||
count = 0
|
||||
|
||||
try:
|
||||
for file in self._video.get_files():
|
||||
self.log.info("Deleting file %s", file)
|
||||
count += 1
|
||||
try:
|
||||
os.unlink(file)
|
||||
except OSError as e:
|
||||
self.log.error("Failed to delete file %s: Error: %s", file, e)
|
||||
|
||||
except OSError as e:
|
||||
self.log.error("Failed to delete video %d [%s %s]. Error: %s", self._video.id,
|
||||
self._video.provider_id, self._video.name, e)
|
||||
|
||||
self._video.downloaded_path = None
|
||||
self._video.save()
|
||||
|
||||
self.log.info('Deleted video %d successfully! (%d files) [%s %s]', self._video.id, count,
|
||||
self._video.provider_id, self._video.name)
|
||||
|
||||
@staticmethod
|
||||
def schedule(video: Video):
|
||||
"""
|
||||
Schedules a delete video job to run immediately.
|
||||
:param video:
|
||||
:return:
|
||||
"""
|
||||
from YtManagerApp.services import Services
|
||||
Services.scheduler().add_job(DeleteVideoJob, args=[video])
|
136
app/YtManagerApp/services/scheduler/jobs/download_video_job.py
Normal file
136
app/YtManagerApp/services/scheduler/jobs/download_video_job.py
Normal file
@ -0,0 +1,136 @@
|
||||
import os
|
||||
import re
|
||||
from string import Template
|
||||
from threading import Lock
|
||||
|
||||
import youtube_dl
|
||||
|
||||
from YtManagerApp.models import Video
|
||||
from YtManagerApp.services.scheduler.job import Job
|
||||
|
||||
|
||||
class DownloadVideoJob(Job):
|
||||
name = "DownloadVideoJob"
|
||||
__lock = Lock()
|
||||
|
||||
def __init__(self, job_execution, video: Video, attempt: int = 1):
|
||||
super().__init__(job_execution)
|
||||
self._video = video
|
||||
self._attempt = attempt
|
||||
self._log_youtube_dl = self.log.getChild('youtube_dl')
|
||||
|
||||
def get_description(self):
|
||||
ret = "Downloading video " + self._video.name
|
||||
if self._attempt > 1:
|
||||
ret += f" (attempt {self._attempt})"
|
||||
return ret
|
||||
|
||||
def run(self):
|
||||
# Issue: if multiple videos are downloaded at the same time, a race condition appears in the mkdirs() call that
|
||||
# youtube-dl makes, which causes it to fail with the error 'Cannot create folder - file already exists'.
|
||||
# For now, allow a single download instance.
|
||||
self.__lock.acquire()
|
||||
|
||||
try:
|
||||
user = self._video.subscription.user
|
||||
max_attempts = user.preferences['max_download_attempts']
|
||||
|
||||
youtube_dl_params, output_path = self.__build_youtube_dl_params(self._video)
|
||||
with youtube_dl.YoutubeDL(youtube_dl_params) as yt:
|
||||
ret = yt.download(["https://www.youtube.com/watch?v=" + self._video.video_id])
|
||||
|
||||
self.log.info('Download finished with code %d', ret)
|
||||
|
||||
if ret == 0:
|
||||
self._video.downloaded_path = output_path
|
||||
self._video.save()
|
||||
self.log.info('Video %d [%s %s] downloaded successfully!', self._video.id, self._video.video_id,
|
||||
self._video.name)
|
||||
|
||||
elif self._attempt <= max_attempts:
|
||||
self.log.warning('Re-enqueueing video (attempt %d/%d)', self._attempt, max_attempts)
|
||||
DownloadVideoJob.schedule(self._video, self._attempt + 1)
|
||||
|
||||
else:
|
||||
self.log.error('Multiple attempts to download video %d [%s %s] failed!', self._video.id,
|
||||
self._video.video_id, self._video.name)
|
||||
self._video.downloaded_path = ''
|
||||
self._video.save()
|
||||
|
||||
finally:
|
||||
self.__lock.release()
|
||||
|
||||
def __build_youtube_dl_params(self, video: Video):
|
||||
|
||||
sub = video.subscription
|
||||
user = sub.user
|
||||
|
||||
# resolve path
|
||||
download_path = user.preferences['download_path']
|
||||
|
||||
template_dict = self.__build_template_dict(video)
|
||||
output_pattern = Template(user.preferences['download_file_pattern']).safe_substitute(template_dict)
|
||||
|
||||
output_path = os.path.join(download_path, output_pattern)
|
||||
output_path = os.path.normpath(output_path)
|
||||
|
||||
youtube_dl_params = {
|
||||
'logger': self._log_youtube_dl,
|
||||
'format': user.preferences['download_format'],
|
||||
'outtmpl': output_path,
|
||||
'writethumbnail': True,
|
||||
'writedescription': True,
|
||||
'writesubtitles': user.preferences['download_subtitles'],
|
||||
'writeautomaticsub': user.preferences['download_autogenerated_subtitles'],
|
||||
'allsubtitles': user.preferences['download_subtitles_all'],
|
||||
'merge_output_format': 'mp4',
|
||||
'postprocessors': [
|
||||
{
|
||||
'key': 'FFmpegMetadata'
|
||||
},
|
||||
]
|
||||
}
|
||||
|
||||
sub_langs = user.preferences['download_subtitles_langs'].split(',')
|
||||
sub_langs = [i.strip() for i in sub_langs]
|
||||
if len(sub_langs) > 0:
|
||||
youtube_dl_params['subtitleslangs'] = sub_langs
|
||||
|
||||
sub_format = user.preferences['download_subtitles_format']
|
||||
if len(sub_format) > 0:
|
||||
youtube_dl_params['subtitlesformat'] = sub_format
|
||||
|
||||
return youtube_dl_params, output_path
|
||||
|
||||
def __build_template_dict(self, video: Video):
|
||||
return {
|
||||
'channel': video.subscription.channel_name,
|
||||
'channel_id': video.subscription.channel_id,
|
||||
'playlist': video.subscription.name,
|
||||
'playlist_id': video.subscription.playlist_id,
|
||||
'playlist_index': "{:03d}".format(1 + video.playlist_index),
|
||||
'title': video.name,
|
||||
'id': video.video_id,
|
||||
}
|
||||
|
||||
def __get_valid_path(self, path):
|
||||
"""
|
||||
Normalizes string, converts to lowercase, removes non-alpha characters,
|
||||
and converts spaces to hyphens.
|
||||
"""
|
||||
import unicodedata
|
||||
value = unicodedata.normalize('NFKD', path).encode('ascii', 'ignore').decode('ascii')
|
||||
value = re.sub('[:"*]', '', value).strip()
|
||||
value = re.sub('[?<>|]', '#', value)
|
||||
return value
|
||||
|
||||
@staticmethod
|
||||
def schedule(video: Video, attempt: int = 1):
|
||||
"""
|
||||
Schedules to download video immediately
|
||||
:param video:
|
||||
:param attempt:
|
||||
:return:
|
||||
"""
|
||||
from YtManagerApp.management.services import Services
|
||||
Services.scheduler.add_job(DownloadVideoJob, args=[video, attempt])
|
183
app/YtManagerApp/services/scheduler/jobs/synchronize_job.py
Normal file
183
app/YtManagerApp/services/scheduler/jobs/synchronize_job.py
Normal file
@ -0,0 +1,183 @@
|
||||
import errno
|
||||
import itertools
|
||||
from threading import Lock
|
||||
|
||||
from apscheduler.triggers.cron import CronTrigger
|
||||
from django.db.models import Max
|
||||
from django.conf import settings
|
||||
|
||||
from YtManagerApp.services.downloader import fetch_thumbnail, downloader_process_subscription
|
||||
from YtManagerApp.models import *
|
||||
from YtManagerApp.services.scheduler.job import Job
|
||||
from YtManagerApp.services import Services
|
||||
from external.pytaw.pytaw.utils import iterate_chunks
|
||||
|
||||
_ENABLE_UPDATE_STATS = True
|
||||
|
||||
|
||||
class SynchronizeJob(Job):
|
||||
name = "SynchronizeJob"
|
||||
__lock = Lock()
|
||||
running = False
|
||||
__global_sync_job = None
|
||||
|
||||
def __init__(self, job_execution, subscription: Optional[Subscription] = None):
|
||||
super().__init__(job_execution)
|
||||
self.__subscription = subscription
|
||||
self.__api = youtube.YoutubeAPI.build_public()
|
||||
self.__new_vids = []
|
||||
|
||||
def get_description(self):
|
||||
if self.__subscription is not None:
|
||||
return "Running synchronization for subscription " + self.__subscription.name
|
||||
return "Running synchronization..."
|
||||
|
||||
def get_subscription_list(self):
|
||||
if self.__subscription is not None:
|
||||
return [self.__subscription]
|
||||
return Subscription.objects.all()
|
||||
|
||||
def get_videos_list(self, subs):
|
||||
return Video.objects.filter(subscription__in=subs)
|
||||
|
||||
def run(self):
|
||||
self.__lock.acquire(blocking=True)
|
||||
SynchronizeJob.running = True
|
||||
try:
|
||||
self.log.info(self.get_description())
|
||||
|
||||
# Build list of work items
|
||||
work_subs = self.get_subscription_list()
|
||||
work_vids = self.get_videos_list(work_subs)
|
||||
|
||||
self.set_total_steps(len(work_subs) + len(work_vids))
|
||||
|
||||
# Remove the 'new' flag
|
||||
work_vids.update(new=False)
|
||||
|
||||
# Process subscriptions
|
||||
for sub in work_subs:
|
||||
self.progress_advance(1, "Synchronizing subscription " + sub.name)
|
||||
self.check_new_videos(sub)
|
||||
self.fetch_missing_thumbnails(sub)
|
||||
|
||||
# Add new videos to progress calculation
|
||||
self.set_total_steps(len(work_subs) + len(work_vids) + len(self.__new_vids))
|
||||
|
||||
# Process videos
|
||||
all_videos = itertools.chain(work_vids, self.__new_vids)
|
||||
for batch in iterate_chunks(all_videos, 50):
|
||||
video_stats = {}
|
||||
|
||||
if _ENABLE_UPDATE_STATS:
|
||||
batch_ids = [video.video_id for video in batch]
|
||||
video_stats = {v.id: v for v in self.__api.videos(batch_ids, part='id,statistics')}
|
||||
|
||||
for video in batch:
|
||||
self.progress_advance(1, "Updating video " + video.name)
|
||||
self.check_video_deleted(video)
|
||||
self.fetch_missing_thumbnails(video)
|
||||
|
||||
if video.video_id in video_stats:
|
||||
self.update_video_stats(video, video_stats[video.video_id])
|
||||
|
||||
# Start downloading videos
|
||||
for sub in work_subs:
|
||||
downloader_process_subscription(sub)
|
||||
|
||||
finally:
|
||||
SynchronizeJob.running = False
|
||||
self.__lock.release()
|
||||
|
||||
def check_new_videos(self, sub: Subscription):
|
||||
playlist_items = self.__api.playlist_items(sub.playlist_id)
|
||||
if sub.rewrite_playlist_indices:
|
||||
playlist_items = sorted(playlist_items, key=lambda x: x.published_at)
|
||||
else:
|
||||
playlist_items = sorted(playlist_items, key=lambda x: x.position)
|
||||
|
||||
for item in playlist_items:
|
||||
results = Video.objects.filter(video_id=item.resource_video_id, subscription=sub)
|
||||
|
||||
if not results.exists():
|
||||
self.log.info('New video for subscription %s: %s %s"', sub, item.resource_video_id, item.title)
|
||||
|
||||
# fix playlist index if necessary
|
||||
if sub.rewrite_playlist_indices or Video.objects.filter(subscription=sub, playlist_index=item.position).exists():
|
||||
highest = Video.objects.filter(subscription=sub).aggregate(Max('playlist_index'))['playlist_index__max']
|
||||
item.position = 1 + (highest or -1)
|
||||
|
||||
self.__new_vids.append(Video.create(item, sub))
|
||||
|
||||
def fetch_missing_thumbnails(self, obj: Union[Subscription, Video]):
|
||||
if obj.thumbnail.startswith("http"):
|
||||
if isinstance(obj, Subscription):
|
||||
obj.thumbnail = fetch_thumbnail(obj.thumbnail, 'sub', obj.playlist_id, settings.THUMBNAIL_SIZE_SUBSCRIPTION)
|
||||
elif isinstance(obj, Video):
|
||||
obj.thumbnail = fetch_thumbnail(obj.thumbnail, 'video', obj.provider_id, settings.THUMBNAIL_SIZE_VIDEO)
|
||||
obj.save()
|
||||
|
||||
def check_video_deleted(self, video: Video):
|
||||
if video.downloaded_path is not None:
|
||||
files = []
|
||||
try:
|
||||
files = list(video.get_files())
|
||||
except OSError as e:
|
||||
if e.errno != errno.ENOENT:
|
||||
self.log.error("Could not access path %s. Error: %s", video.downloaded_path, e)
|
||||
self.usr_err(f"Could not access path {video.downloaded_path}: {e}", suppress_notification=True)
|
||||
return
|
||||
|
||||
# Try to find a valid video file
|
||||
found_video = False
|
||||
for file in files:
|
||||
mime, _ = mimetypes.guess_type(file)
|
||||
if mime is not None and mime.startswith("video"):
|
||||
found_video = True
|
||||
|
||||
# Video not found, we can safely assume that the video was deleted.
|
||||
if not found_video:
|
||||
self.log.info("Video %d was deleted! [%s %s]", video.id, video.provider_id, video.name)
|
||||
# Clean up
|
||||
for file in files:
|
||||
try:
|
||||
os.unlink(file)
|
||||
except OSError as e:
|
||||
self.log.error("Could not delete redundant file %s. Error: %s", file, e)
|
||||
self.usr_err(f"Could not delete redundant file {file}: {e}", suppress_notification=True)
|
||||
video.downloaded_path = None
|
||||
|
||||
# Mark watched?
|
||||
user = video.subscription.user
|
||||
if user.preferences['mark_deleted_as_watched']:
|
||||
video.watched = True
|
||||
|
||||
video.save()
|
||||
|
||||
def update_video_stats(self, video: Video, yt_video):
|
||||
if yt_video.n_likes is not None \
|
||||
and yt_video.n_dislikes is not None \
|
||||
and yt_video.n_likes + yt_video.n_dislikes > 0:
|
||||
video.rating = yt_video.n_likes / (yt_video.n_likes + yt_video.n_dislikes)
|
||||
|
||||
video.views = yt_video.n_views
|
||||
video.save()
|
||||
|
||||
@staticmethod
|
||||
def schedule_global_job():
|
||||
trigger = CronTrigger.from_crontab(Services.appConfig().sync_schedule)
|
||||
|
||||
if SynchronizeJob.__global_sync_job is None:
|
||||
trigger = CronTrigger.from_crontab(Services.appConfig().sync_schedule)
|
||||
SynchronizeJob.__global_sync_job = Services.scheduler().add_job(SynchronizeJob, trigger, max_instances=1, coalesce=True)
|
||||
|
||||
else:
|
||||
SynchronizeJob.__global_sync_job.reschedule(trigger, max_instances=1, coalesce=True)
|
||||
|
||||
@staticmethod
|
||||
def schedule_now():
|
||||
Services.scheduler.add_job(SynchronizeJob, max_instances=1, coalesce=True)
|
||||
|
||||
@staticmethod
|
||||
def schedule_now_for_subscription(subscription):
|
||||
Services.scheduler.add_job(SynchronizeJob, user=subscription.user, args=[subscription])
|
@ -0,0 +1,18 @@
|
||||
from YtManagerApp.models import Video
|
||||
from YtManagerApp.services.scheduler.job import Job
|
||||
|
||||
|
||||
class YouTubeDLUpdateJob(Job):
|
||||
name = "YouTubeDLUpdateJob"
|
||||
|
||||
def __init__(self, job_execution):
|
||||
super().__init__(job_execution)
|
||||
|
||||
def get_description(self):
|
||||
return f"Updating youtube-dl runtime"
|
||||
|
||||
def run(self):
|
||||
from YtManagerApp.services import Services
|
||||
self.set_total_steps(1)
|
||||
Services.youtubeDLManager.install()
|
||||
self.progress_advance(1)
|
83
app/YtManagerApp/services/scheduler/progress_tracker.py
Normal file
83
app/YtManagerApp/services/scheduler/progress_tracker.py
Normal file
@ -0,0 +1,83 @@
|
||||
from typing import Callable, List, Any, Optional
|
||||
|
||||
|
||||
class ProgressTracker(object):
|
||||
"""
|
||||
Class which helps keep track of complex operation progress.
|
||||
"""
|
||||
|
||||
def __init__(self, total_steps: float = 100, initial_steps: float = 0,
|
||||
listener: Callable[[float, str], None] = None,
|
||||
listener_args: List[Any] = None,
|
||||
parent: Optional["ProgressTracker"] = None):
|
||||
"""
|
||||
Constructor
|
||||
:param total_steps: Total number of steps required by this operation
|
||||
:param initial_steps: Starting steps
|
||||
:param parent: Parent progress tracker
|
||||
:param listener: Callable which is called when any progress happens
|
||||
"""
|
||||
|
||||
self.total_steps = total_steps
|
||||
self.steps = initial_steps
|
||||
|
||||
self.__subtask: ProgressTracker = None
|
||||
self.__subtask_steps = 0
|
||||
|
||||
self.__parent = parent
|
||||
self.__listener = listener
|
||||
self.__listener_args = listener_args or []
|
||||
|
||||
def __on_progress(self, progress_msg):
|
||||
if self.__listener is not None:
|
||||
self.__listener(*self.__listener_args, self.compute_progress(), progress_msg)
|
||||
|
||||
if self.__parent is not None:
|
||||
self.__parent.__on_progress(progress_msg)
|
||||
|
||||
def advance(self, steps: float = 1, progress_msg: str = ''):
|
||||
"""
|
||||
Advances a number of steps.
|
||||
:param steps: Number of steps to advance
|
||||
:param progress_msg: A message which will be passed to a listener
|
||||
:return:
|
||||
"""
|
||||
|
||||
# We can assume previous subtask is now completed
|
||||
if self.__subtask is not None:
|
||||
self.steps += self.__subtask_steps
|
||||
self.__subtask = None
|
||||
|
||||
self.steps += steps
|
||||
self.__on_progress(progress_msg)
|
||||
|
||||
def subtask(self, steps: float = 1, subtask_total_steps: float = 100, subtask_initial_steps: float = 0):
|
||||
"""
|
||||
Creates a 'subtask' which has its own progress, which will be used in the calculation of the final progress.
|
||||
:param steps: Number of steps the subtask is 'worth'
|
||||
:param subtask_total_steps: Total number of steps for subtask
|
||||
:param subtask_initial_steps: Initial steps for subtask
|
||||
:return: ProgressTracker for subtask
|
||||
"""
|
||||
|
||||
# We can assume previous subtask is now completed
|
||||
if self.__subtask is not None:
|
||||
self.steps += self.__subtask_steps
|
||||
|
||||
self.__subtask = ProgressTracker(total_steps=subtask_total_steps,
|
||||
initial_steps=subtask_initial_steps,
|
||||
parent=self)
|
||||
self.__subtask_steps = steps
|
||||
|
||||
return self.__subtask
|
||||
|
||||
def compute_progress(self):
|
||||
"""
|
||||
Calculates final progress value in percent.
|
||||
:return: value in [0,1] interval representing progress
|
||||
"""
|
||||
base = float(self.steps) / self.total_steps
|
||||
if self.__subtask is not None:
|
||||
base += self.__subtask.compute_progress() * self.__subtask_steps / self.total_steps
|
||||
|
||||
return min(base, 1.0)
|
81
app/YtManagerApp/services/scheduler/scheduler.py
Normal file
81
app/YtManagerApp/services/scheduler/scheduler.py
Normal file
@ -0,0 +1,81 @@
|
||||
import datetime
|
||||
import logging
|
||||
import traceback
|
||||
from typing import Type, Union, Optional
|
||||
|
||||
import pytz
|
||||
from apscheduler.schedulers.background import BackgroundScheduler
|
||||
from apscheduler.triggers.base import BaseTrigger
|
||||
from apscheduler.triggers.interval import IntervalTrigger
|
||||
from django.contrib.auth.models import User
|
||||
|
||||
from YtManagerApp.services.appconfig import AppConfig
|
||||
from YtManagerApp.models import JobExecution, JOB_STATES_MAP
|
||||
from YtManagerApp.services.scheduler.job import Job
|
||||
from YtManagerApp.services.scheduler.jobs.youtubedl_update_job import YouTubeDLUpdateJob
|
||||
|
||||
|
||||
class YtsmScheduler(object):
|
||||
|
||||
def __init__(self, app_config: AppConfig):
|
||||
self._ap_scheduler = BackgroundScheduler()
|
||||
self._app_config = app_config
|
||||
|
||||
def initialize(self):
|
||||
# set state of existing jobs as "interrupted"
|
||||
JobExecution.objects\
|
||||
.filter(status=JOB_STATES_MAP['running'])\
|
||||
.update(status=JOB_STATES_MAP['interrupted'])
|
||||
|
||||
self._configure_scheduler()
|
||||
self._ap_scheduler.start()
|
||||
self._schedule_main_jobs()
|
||||
|
||||
def _schedule_main_jobs(self):
|
||||
self.add_job(YouTubeDLUpdateJob, trigger=IntervalTrigger(days=1))
|
||||
|
||||
def _configure_scheduler(self):
|
||||
logger = logging.getLogger('scheduler')
|
||||
executors = {
|
||||
'default': {
|
||||
'type': 'threadpool',
|
||||
'max_workers': self._app_config.concurrency
|
||||
}
|
||||
}
|
||||
job_defaults = {
|
||||
'misfire_grace_time': 60 * 60 * 24 * 365 # 1 year
|
||||
}
|
||||
self._ap_scheduler.configure(logger=logger, executors=executors, job_defaults=job_defaults)
|
||||
|
||||
def _run_job(self, job_class: Type[Job], user: Optional[User], args: Union[tuple, list]):
|
||||
|
||||
job_execution = JobExecution(user=user, status=JOB_STATES_MAP['running'])
|
||||
job_execution.save()
|
||||
job_instance = job_class(job_execution, *args)
|
||||
|
||||
# update description
|
||||
job_execution.description = job_instance.get_description()
|
||||
job_execution.save()
|
||||
|
||||
try:
|
||||
job_instance.run()
|
||||
job_execution.status = JOB_STATES_MAP['finished']
|
||||
|
||||
except Exception as ex:
|
||||
job_instance.log.critical("Job failed with exception: %s", traceback.format_exc())
|
||||
job_instance.usr_err(job_instance.name + " operation failed: " + str(ex))
|
||||
job_execution.status = JOB_STATES_MAP['failed']
|
||||
|
||||
finally:
|
||||
job_execution.end_date = datetime.datetime.now(tz=pytz.UTC)
|
||||
job_execution.save()
|
||||
|
||||
def add_job(self, job_class: Type[Job], trigger: Union[str, BaseTrigger] = None,
|
||||
args: Union[list, tuple] = None,
|
||||
user: Optional[User] = None,
|
||||
**kwargs):
|
||||
if args is None:
|
||||
args = []
|
||||
|
||||
return self._ap_scheduler.add_job(YtsmScheduler._run_job, trigger=trigger, args=[self, job_class, user, args],
|
||||
**kwargs)
|
11
app/YtManagerApp/services/video_provider_manager.py
Normal file
11
app/YtManagerApp/services/video_provider_manager.py
Normal file
@ -0,0 +1,11 @@
|
||||
from YtManagerApp.services.providers.video_provider import VideoProvider
|
||||
|
||||
|
||||
class VideoProviderManager(object):
|
||||
|
||||
def __init__(self):
|
||||
self._providers: Dict[str, VideoProvider] = {}
|
||||
|
||||
def register_provider(self, provider: VideoProvider):
|
||||
pid = provider.get_provider_id()
|
||||
self._providers[pid] = provider
|
57
app/YtManagerApp/services/videos.py
Normal file
57
app/YtManagerApp/services/videos.py
Normal file
@ -0,0 +1,57 @@
|
||||
import re
|
||||
from typing import Optional
|
||||
|
||||
from django.contrib.auth.models import User
|
||||
from django.db.models import Q
|
||||
|
||||
from YtManagerApp.models import Subscription, Video, SubscriptionFolder
|
||||
|
||||
|
||||
def get_videos(user: User,
|
||||
sort_order: Optional[str],
|
||||
query: Optional[str] = None,
|
||||
subscription_id: Optional[int] = None,
|
||||
folder_id: Optional[int] = None,
|
||||
only_watched: Optional[bool] = None,
|
||||
only_downloaded: Optional[bool] = None,
|
||||
):
|
||||
|
||||
filter_args = []
|
||||
filter_kwargs = {
|
||||
'subscription__user': user
|
||||
}
|
||||
|
||||
# Process query string - basically, we break it down into words,
|
||||
# and then search for the given text in the name, description, uploader name and subscription name
|
||||
if query is not None:
|
||||
for match in re.finditer(r'\w+', query):
|
||||
word = match[0]
|
||||
filter_args.append(Q(name__icontains=word)
|
||||
| Q(description__icontains=word)
|
||||
| Q(uploader_name__icontains=word)
|
||||
| Q(subscription__name__icontains=word))
|
||||
|
||||
# Subscription id
|
||||
if subscription_id is not None:
|
||||
filter_kwargs['subscription_id'] = subscription_id
|
||||
|
||||
# Folder id
|
||||
if folder_id is not None:
|
||||
# Visit function - returns only the subscription IDs
|
||||
def visit(node):
|
||||
if isinstance(node, Subscription):
|
||||
return node.id
|
||||
return None
|
||||
filter_kwargs['subscription_id__in'] = SubscriptionFolder.traverse(folder_id, user, visit)
|
||||
|
||||
# Only watched
|
||||
if only_watched is not None:
|
||||
filter_kwargs['watched'] = only_watched
|
||||
|
||||
# Only downloaded
|
||||
# - not downloaded (False) -> is null (True)
|
||||
# - downloaded (True) -> is not null (False)
|
||||
if only_downloaded is not None:
|
||||
filter_kwargs['downloaded_path__isnull'] = not only_downloaded
|
||||
|
||||
return Video.objects.filter(*filter_args, **filter_kwargs).order_by(sort_order)
|
107
app/YtManagerApp/services/youtube_dl_manager.py
Normal file
107
app/YtManagerApp/services/youtube_dl_manager.py
Normal file
@ -0,0 +1,107 @@
|
||||
import logging
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
import requests
|
||||
from django.conf import settings as dj_settings
|
||||
|
||||
LATEST_URL = "https://yt-dl.org/downloads/latest/youtube-dl"
|
||||
GITHUB_API_LATEST_RELEASE = "https://api.github.com/repos/ytdl-org/youtube-dl/releases/latest"
|
||||
log = logging.getLogger("YoutubeDlManager")
|
||||
|
||||
|
||||
class YoutubeDlException(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class YoutubeDlNotInstalledException(YoutubeDlException):
|
||||
pass
|
||||
|
||||
|
||||
class YoutubeDlRuntimeException(YoutubeDlException):
|
||||
pass
|
||||
|
||||
|
||||
class YoutubeDlManager(object):
|
||||
|
||||
def __init__(self):
|
||||
self.verbose = False
|
||||
self.progress = False
|
||||
self._install_path = os.path.join(dj_settings.DATA_DIR, 'youtube-dl')
|
||||
|
||||
def _check_installed(self):
|
||||
return os.path.isfile(self._install_path) and os.access(self._install_path, os.X_OK)
|
||||
|
||||
def _get_run_args(self):
|
||||
run_args = []
|
||||
if self.verbose:
|
||||
run_args.append('-v')
|
||||
if self.progress:
|
||||
run_args.append('--newline')
|
||||
else:
|
||||
run_args.append('--no-progress')
|
||||
|
||||
return run_args
|
||||
|
||||
def run(self, *args):
|
||||
if not self._check_installed():
|
||||
log.error("Cannot run youtube-dl, it is not installed!")
|
||||
raise YoutubeDlNotInstalledException
|
||||
|
||||
run_args = self._get_run_args()
|
||||
ret = subprocess.run([sys.executable, self._install_path, *run_args, *args], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
|
||||
|
||||
stdout = ret.stdout.decode('utf-8')
|
||||
if len(stdout) > 0:
|
||||
log.info("YoutubeDL: " + stdout)
|
||||
|
||||
stderr = ret.stderr.decode('utf-8')
|
||||
if len(stderr) > 0:
|
||||
log.error("YoutubeDL: " + stderr)
|
||||
|
||||
if ret.returncode != 0:
|
||||
raise YoutubeDlRuntimeException()
|
||||
|
||||
return stdout
|
||||
|
||||
def get_installed_version(self):
|
||||
return self.run('--version')
|
||||
|
||||
def get_latest_version(self):
|
||||
resp = requests.get(GITHUB_API_LATEST_RELEASE, allow_redirects=True)
|
||||
resp.raise_for_status()
|
||||
|
||||
info = resp.json()
|
||||
return info['tag_name']
|
||||
|
||||
def install(self):
|
||||
# Check if we are running the latest version
|
||||
latest = self.get_latest_version()
|
||||
try:
|
||||
current = self.get_installed_version()
|
||||
except YoutubeDlNotInstalledException:
|
||||
current = None
|
||||
if latest == current:
|
||||
log.info(f"Running latest youtube-dl version ({current})!")
|
||||
return
|
||||
|
||||
# Download latest
|
||||
resp = requests.get(LATEST_URL, allow_redirects=True, stream=True)
|
||||
resp.raise_for_status()
|
||||
|
||||
with open(self._install_path + ".tmp", "wb") as f:
|
||||
for chunk in resp.iter_content(10 * 1024):
|
||||
f.write(chunk)
|
||||
|
||||
# Replace
|
||||
os.unlink(self._install_path)
|
||||
os.rename(self._install_path + ".tmp", self._install_path)
|
||||
os.chmod(self._install_path, 555)
|
||||
|
||||
# Test run
|
||||
newver = self.get_installed_version()
|
||||
if current is None:
|
||||
log.info(f"Installed youtube-dl version {newver}.")
|
||||
else:
|
||||
log.info(f"Upgraded youtube-dl from version {current} to {newver}.")
|
Reference in New Issue
Block a user