mirror of
https://github.com/chibicitiberiu/ytsm.git
synced 2024-02-24 05:43:31 +00:00
Major refactor of many things.
This commit is contained in:
@ -1,36 +0,0 @@
|
||||
from YtManagerApp.dynamic_preferences_registry import Initialized, YouTubeAPIKey, AllowRegistrations, SyncSchedule, SchedulerConcurrency
|
||||
|
||||
|
||||
class AppConfig(object):
|
||||
# Properties
|
||||
props = {
|
||||
'initialized': Initialized,
|
||||
'youtube_api_key': YouTubeAPIKey,
|
||||
'allow_registrations': AllowRegistrations,
|
||||
'sync_schedule': SyncSchedule,
|
||||
'concurrency': SchedulerConcurrency
|
||||
}
|
||||
|
||||
# Init
|
||||
def __init__(self, pref_manager):
|
||||
self.__pref_manager = pref_manager
|
||||
|
||||
def __getattr__(self, item):
|
||||
prop_class = AppConfig.props[item]
|
||||
prop_full_name = prop_class.section.name + "__" + prop_class.name
|
||||
return self.__pref_manager[prop_full_name]
|
||||
|
||||
def __setattr__(self, key, value):
|
||||
if key in AppConfig.props:
|
||||
prop_class = AppConfig.props[key]
|
||||
prop_full_name = prop_class.section.name + "__" + prop_class.name
|
||||
self.__pref_manager[prop_full_name] = value
|
||||
else:
|
||||
super().__setattr__(key, value)
|
||||
|
||||
def for_sub(self, subscription, pref: str):
|
||||
value = getattr(subscription, pref)
|
||||
if value is None:
|
||||
value = subscription.user.preferences[pref]
|
||||
|
||||
return value
|
||||
@ -1,111 +0,0 @@
|
||||
from YtManagerApp.scheduler.jobs.download_video_job import DownloadVideoJob
|
||||
from YtManagerApp.models import Video, Subscription, VIDEO_ORDER_MAPPING
|
||||
from YtManagerApp.utils import first_non_null
|
||||
from django.conf import settings as srv_settings
|
||||
import logging
|
||||
import requests
|
||||
import mimetypes
|
||||
import os
|
||||
import PIL.Image
|
||||
import PIL.ImageOps
|
||||
from urllib.parse import urljoin
|
||||
|
||||
log = logging.getLogger('downloader')
|
||||
|
||||
|
||||
def __get_subscription_config(sub: Subscription):
|
||||
user = sub.user
|
||||
|
||||
enabled = first_non_null(sub.auto_download, user.preferences['auto_download'])
|
||||
global_limit = user.preferences['download_global_limit']
|
||||
limit = first_non_null(sub.download_limit, user.preferences['download_subscription_limit'])
|
||||
order = first_non_null(sub.download_order, user.preferences['download_order'])
|
||||
order = VIDEO_ORDER_MAPPING[order]
|
||||
|
||||
return enabled, global_limit, limit, order
|
||||
|
||||
|
||||
def downloader_process_subscription(sub: Subscription):
|
||||
log.info('Processing subscription %d [%s %s]', sub.id, sub.playlist_id, sub.id)
|
||||
|
||||
enabled, global_limit, limit, order = __get_subscription_config(sub)
|
||||
log.info('Determined settings enabled=%s global_limit=%d limit=%d order="%s"', enabled, global_limit, limit, order)
|
||||
|
||||
if enabled:
|
||||
videos_to_download = Video.objects\
|
||||
.filter(subscription=sub, downloaded_path__isnull=True, watched=False)\
|
||||
.order_by(order)
|
||||
|
||||
log.info('%d download candidates.', len(videos_to_download))
|
||||
|
||||
if global_limit > 0:
|
||||
global_downloaded = Video.objects.filter(subscription__user=sub.user, downloaded_path__isnull=False).count()
|
||||
allowed_count = max(global_limit - global_downloaded, 0)
|
||||
videos_to_download = videos_to_download[0:allowed_count]
|
||||
log.info('Global limit is set, can only download up to %d videos.', allowed_count)
|
||||
|
||||
if limit > 0:
|
||||
sub_downloaded = Video.objects.filter(subscription=sub, downloaded_path__isnull=False).count()
|
||||
allowed_count = max(limit - sub_downloaded, 0)
|
||||
videos_to_download = videos_to_download[0:allowed_count]
|
||||
log.info('Limit is set, can only download up to %d videos.', allowed_count)
|
||||
|
||||
# enqueue download
|
||||
for video in videos_to_download:
|
||||
log.info('Enqueuing video %d [%s %s] index=%d', video.id, video.video_id, video.name, video.playlist_index)
|
||||
DownloadVideoJob.schedule(video)
|
||||
|
||||
log.info('Finished processing subscription %d [%s %s]', sub.id, sub.playlist_id, sub.id)
|
||||
|
||||
|
||||
def downloader_process_all():
|
||||
for subscription in Subscription.objects.all():
|
||||
downloader_process_subscription(subscription)
|
||||
|
||||
|
||||
def fetch_thumbnail(url, object_type, identifier, thumb_size):
|
||||
|
||||
log.info('Fetching thumbnail url=%s object_type=%s identifier=%s', url, object_type, identifier)
|
||||
|
||||
# Make request to obtain mime type
|
||||
try:
|
||||
response = requests.get(url, stream=True)
|
||||
except requests.exceptions.RequestException as e:
|
||||
log.error('Failed to fetch thumbnail %s. Error: %s', url, e)
|
||||
return url
|
||||
|
||||
ext = mimetypes.guess_extension(response.headers['Content-Type'])
|
||||
|
||||
# Build file path
|
||||
file_name = f"{identifier}{ext}"
|
||||
abs_path_dir = os.path.join(srv_settings.MEDIA_ROOT, "thumbs", object_type)
|
||||
abs_path = os.path.join(abs_path_dir, file_name)
|
||||
abs_path_tmp = file_name + '.tmp'
|
||||
|
||||
# Store image
|
||||
try:
|
||||
os.makedirs(abs_path_dir, exist_ok=True)
|
||||
with open(abs_path_tmp, "wb") as f:
|
||||
for chunk in response.iter_content(chunk_size=1024):
|
||||
if chunk:
|
||||
f.write(chunk)
|
||||
|
||||
# Resize and crop to thumbnail size
|
||||
image = PIL.Image.open(abs_path_tmp)
|
||||
image = PIL.ImageOps.fit(image, thumb_size)
|
||||
image.save(abs_path)
|
||||
image.close()
|
||||
|
||||
# Delete temp file
|
||||
os.unlink(abs_path_tmp)
|
||||
|
||||
except requests.exceptions.RequestException as e:
|
||||
log.error('Error while downloading stream for thumbnail %s. Error: %s', url, e)
|
||||
return url
|
||||
except OSError as e:
|
||||
log.error('Error while writing to file %s for thumbnail %s. Error: %s', abs_path, url, e)
|
||||
return url
|
||||
|
||||
# Return
|
||||
media_url = urljoin(srv_settings.MEDIA_URL, f"thumbs/{object_type}/{file_name}")
|
||||
return media_url
|
||||
@ -1,57 +0,0 @@
|
||||
import re
|
||||
from typing import Optional
|
||||
|
||||
from django.contrib.auth.models import User
|
||||
from django.db.models import Q
|
||||
|
||||
from YtManagerApp.models import Subscription, Video, SubscriptionFolder
|
||||
|
||||
|
||||
def get_videos(user: User,
|
||||
sort_order: Optional[str],
|
||||
query: Optional[str] = None,
|
||||
subscription_id: Optional[int] = None,
|
||||
folder_id: Optional[int] = None,
|
||||
only_watched: Optional[bool] = None,
|
||||
only_downloaded: Optional[bool] = None,
|
||||
):
|
||||
|
||||
filter_args = []
|
||||
filter_kwargs = {
|
||||
'subscription__user': user
|
||||
}
|
||||
|
||||
# Process query string - basically, we break it down into words,
|
||||
# and then search for the given text in the name, description, uploader name and subscription name
|
||||
if query is not None:
|
||||
for match in re.finditer(r'\w+', query):
|
||||
word = match[0]
|
||||
filter_args.append(Q(name__icontains=word)
|
||||
| Q(description__icontains=word)
|
||||
| Q(uploader_name__icontains=word)
|
||||
| Q(subscription__name__icontains=word))
|
||||
|
||||
# Subscription id
|
||||
if subscription_id is not None:
|
||||
filter_kwargs['subscription_id'] = subscription_id
|
||||
|
||||
# Folder id
|
||||
if folder_id is not None:
|
||||
# Visit function - returns only the subscription IDs
|
||||
def visit(node):
|
||||
if isinstance(node, Subscription):
|
||||
return node.id
|
||||
return None
|
||||
filter_kwargs['subscription_id__in'] = SubscriptionFolder.traverse(folder_id, user, visit)
|
||||
|
||||
# Only watched
|
||||
if only_watched is not None:
|
||||
filter_kwargs['watched'] = only_watched
|
||||
|
||||
# Only downloaded
|
||||
# - not downloaded (False) -> is null (True)
|
||||
# - downloaded (True) -> is not null (False)
|
||||
if only_downloaded is not None:
|
||||
filter_kwargs['downloaded_path__isnull'] = not only_downloaded
|
||||
|
||||
return Video.objects.filter(*filter_args, **filter_kwargs).order_by(sort_order)
|
||||
@ -1,111 +0,0 @@
|
||||
import logging
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
import requests
|
||||
from django.conf import settings as dj_settings
|
||||
|
||||
LATEST_URL = "https://yt-dl.org/downloads/latest/youtube-dl"
|
||||
GITHUB_API_LATEST_RELEASE = "https://api.github.com/repos/ytdl-org/youtube-dl/releases/latest"
|
||||
log = logging.getLogger("YoutubeDlManager")
|
||||
|
||||
|
||||
class YoutubeDlException(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class YoutubeDlNotInstalledException(YoutubeDlException):
|
||||
pass
|
||||
|
||||
|
||||
class YoutubeDlRuntimeException(YoutubeDlException):
|
||||
pass
|
||||
|
||||
|
||||
class YoutubeDlManager(object):
|
||||
|
||||
def __init__(self):
|
||||
self.verbose = False
|
||||
self.progress = False
|
||||
|
||||
def _get_path(self):
|
||||
return os.path.join(dj_settings.DATA_DIR, 'youtube-dl')
|
||||
|
||||
def _check_installed(self, path):
|
||||
return os.path.isfile(path) and os.access(path, os.X_OK)
|
||||
|
||||
def _get_run_args(self):
|
||||
run_args = []
|
||||
if self.verbose:
|
||||
run_args.append('-v')
|
||||
if self.progress:
|
||||
run_args.append('--newline')
|
||||
else:
|
||||
run_args.append('--no-progress')
|
||||
|
||||
return run_args
|
||||
|
||||
def run(self, *args):
|
||||
path = self._get_path()
|
||||
if not self._check_installed(path):
|
||||
log.error("Cannot run youtube-dl, it is not installed!")
|
||||
raise YoutubeDlNotInstalledException
|
||||
|
||||
run_args = self._get_run_args()
|
||||
ret = subprocess.run([sys.executable, path, *run_args, *args], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
|
||||
|
||||
stdout = ret.stdout.decode('utf-8')
|
||||
if len(stdout) > 0:
|
||||
log.info("YoutubeDL: " + stdout)
|
||||
|
||||
stderr = ret.stderr.decode('utf-8')
|
||||
if len(stderr) > 0:
|
||||
log.error("YoutubeDL: " + stderr)
|
||||
|
||||
if ret.returncode != 0:
|
||||
raise YoutubeDlRuntimeException()
|
||||
|
||||
return stdout
|
||||
|
||||
def get_installed_version(self):
|
||||
return self.run('--version')
|
||||
|
||||
def get_latest_version(self):
|
||||
resp = requests.get(GITHUB_API_LATEST_RELEASE, allow_redirects=True)
|
||||
resp.raise_for_status()
|
||||
|
||||
info = resp.json()
|
||||
return info['tag_name']
|
||||
|
||||
def install(self):
|
||||
# Check if we are running the latest version
|
||||
latest = self.get_latest_version()
|
||||
try:
|
||||
current = self.get_installed_version()
|
||||
except YoutubeDlNotInstalledException:
|
||||
current = None
|
||||
if latest == current:
|
||||
log.info(f"Running latest youtube-dl version ({current})!")
|
||||
return
|
||||
|
||||
# Download latest
|
||||
resp = requests.get(LATEST_URL, allow_redirects=True, stream=True)
|
||||
resp.raise_for_status()
|
||||
|
||||
path = self._get_path()
|
||||
with open(path + ".tmp", "wb") as f:
|
||||
for chunk in resp.iter_content(10 * 1024):
|
||||
f.write(chunk)
|
||||
|
||||
# Replace
|
||||
os.unlink(path)
|
||||
os.rename(path + ".tmp", path)
|
||||
os.chmod(path, 555)
|
||||
|
||||
# Test run
|
||||
newver = self.get_installed_version()
|
||||
if current is None:
|
||||
log.info(f"Installed youtube-dl version {newver}.")
|
||||
else:
|
||||
log.info(f"Upgraded youtube-dl from version {current} to {newver}.")
|
||||
Reference in New Issue
Block a user