chibicitiberiu/ytsm
1
0
Fork 0
mirror of https://github.com/chibicitiberiu/ytsm.git synced 2024-02-24 05:43:31 +00:00
Code Issues Releases Wiki Activity

Integrated django-dynamic-preferences library into project. Settings are now stored in the database, the ini file only contains a minimal number of settings.

Browse Source
This commit is contained in:
Tiberiu Chibici
2018-12-10 01:15:10 +02:00
parent d3b057e14e
commit 0329645c37
16 changed files with 169 additions and 353 deletions
Download Patch File Download Diff File Expand all files Collapse all files

112
app/YtManagerApp/appconfig.py
View File

@ -1,112 +0,0 @@
import logging
import os
import os.path
from collections import ChainMap
from configparser import ConfigParser
from shutil import copyfile
from typing import Optional, Any
from django.conf import settings as dj_settings
from django.contrib.auth.models import User
from .models import UserSettings, Subscription
from .utils.extended_interpolation_with_env import ExtendedInterpolatorWithEnv
class AppSettings(ConfigParser):
_DEFAULT_INTERPOLATION = ExtendedInterpolatorWithEnv()
def __init__(self, *args, **kwargs):
super().__init__(allow_no_value=True, *args, **kwargs)
def initialize(self):
self.read([dj_settings.DEFAULTS_FILE, dj_settings.CONFIG_FILE])
def save(self):
if os.path.exists(dj_settings.CONFIG_FILE):
# Create a backup
copyfile(dj_settings.CONFIG_FILE, dj_settings.CONFIG_FILE + ".backup")
else:
# Ensure directory exists
settings_dir = os.path.dirname(dj_settings.CONFIG_FILE)
os.makedirs(settings_dir, exist_ok=True)
with open(dj_settings.CONFIG_FILE, 'w') as f:
self.write(f)
def __get_combined_dict(self, vars: Optional[Any], sub: Optional[Subscription], user: Optional[User]) -> ChainMap:
vars_dict = {}
sub_overloads_dict = {}
user_settings_dict = {}
if vars is not None:
vars_dict = vars
if sub is not None:
sub_overloads_dict = sub.get_overloads_dict()
if user is not None:
user_settings = UserSettings.find_by_user(user)
if user_settings is not None:
user_settings_dict = user_settings.to_dict()
return ChainMap(vars_dict, sub_overloads_dict, user_settings_dict)
def get_user(self, user: User, section: str, option: Any, vars=None, fallback=object()) -> str:
return super().get(section, option,
fallback=fallback,
vars=self.__get_combined_dict(vars, None, user))
def getboolean_user(self, user: User, section: str, option: Any, vars=None, fallback=object()) -> bool:
return super().getboolean(section, option,
fallback=fallback,
vars=self.__get_combined_dict(vars, None, user))
def getint_user(self, user: User, section: str, option: Any, vars=None, fallback=object()) -> int:
return super().getint(section, option,
fallback=fallback,
vars=self.__get_combined_dict(vars, None, user))
def getfloat_user(self, user: User, section: str, option: Any, vars=None, fallback=object()) -> float:
return super().getfloat(section, option,
fallback=fallback,
vars=self.__get_combined_dict(vars, None, user))
def get_sub(self, sub: Subscription, section: str, option: Any, vars=None, fallback=object()) -> str:
return super().get(section, option,
fallback=fallback,
vars=self.__get_combined_dict(vars, sub, sub.user))
def getboolean_sub(self, sub: Subscription, section: str, option: Any, vars=None, fallback=object()) -> bool:
return super().getboolean(section, option,
fallback=fallback,
vars=self.__get_combined_dict(vars, sub, sub.user))
def getint_sub(self, sub: Subscription, section: str, option: Any, vars=None, fallback=object()) -> int:
return super().getint(section, option,
fallback=fallback,
vars=self.__get_combined_dict(vars, sub, sub.user))
def getfloat_sub(self, sub: Subscription, section: str, option: Any, vars=None, fallback=object()) -> float:
return super().getfloat(section, option,
fallback=fallback,
vars=self.__get_combined_dict(vars, sub, sub.user))
settings = AppSettings()
def initialize_app_config():
settings.initialize()
__initialize_logger()
logging.info('Application started!')
def __initialize_logger():
log_dir = os.path.dirname(dj_settings.LOG_FILE)
os.makedirs(log_dir, exist_ok=True)
logging.basicConfig(
filename=dj_settings.LOG_FILE,
level=dj_settings.LOG_LEVEL,
format=dj_settings.LOG_FORMAT)

37
app/YtManagerApp/appmain.py
View File

@ -1,11 +1,36 @@
from .appconfig import initialize_app_config
from .scheduler import initialize_scheduler
from .management.jobs.synchronize import schedule_synchronize_global
import logging
import logging.handlers
import os
from django.conf import settings as dj_settings
from .management.appconfig import global_prefs
from .management.jobs.synchronize import schedule_synchronize_global
from .scheduler import initialize_scheduler
def __initialize_logger():
log_dir = os.path.join(dj_settings.DATA_DIR, 'logs')
os.makedirs(log_dir, exist_ok=True)
file_handler = logging.handlers.RotatingFileHandler(
os.path.join(log_dir, "log.log"),
maxBytes=1024 * 1024,
backupCount=5
)
logging.basicConfig(
level=dj_settings.LOG_LEVEL,
format=dj_settings.LOG_FORMAT,
handlers=[file_handler]
)
def main():
initialize_app_config()
initialize_scheduler()
schedule_synchronize_global()
__initialize_logger()
if global_prefs['hidden__initialized']:
initialize_scheduler()
schedule_synchronize_global()
logging.info('Initialization complete.')

1
app/YtManagerApp/apps.py
View File

@ -1,5 +1,4 @@
from django.apps import AppConfig
import os
class YtManagerAppConfig(AppConfig):

57
app/YtManagerApp/dynamic_preferences_registry.py
View File

@ -13,6 +13,7 @@ scheduler = Section('scheduler')
manager = Section('manager')
downloader = Section('downloader')
# Hidden settings
@global_preferences_registry.register
class Initialized(BooleanPreference):
@ -20,6 +21,7 @@ class Initialized(BooleanPreference):
name = 'initialized'
default = False
# General settings
@global_preferences_registry.register
class YouTubeAPIKey(StringPreference):
@ -28,13 +30,31 @@ class YouTubeAPIKey(StringPreference):
default = 'AIzaSyBabzE4Bup77WexdLMa9rN9z-wJidEfNX8'
required = True
@global_preferences_registry.register
class AllowCDN(BooleanPreference):
section = general
name = 'allow_cdn'
default = True
required = True
@global_preferences_registry.register
class AllowRegistrations(BooleanPreference):
section = general
name = 'allow_registrations'
default = True
required = True
@global_preferences_registry.register
class SyncSchedule(StringPreference):
section = scheduler
name = 'synchronization_schedule'
default = '5 * * * *' # hourly
default = '5 * * * *' # hourly
required = True
@global_preferences_registry.register
class SchedulerConcurrency(IntegerPreference):
section = scheduler
@ -42,8 +62,8 @@ class SchedulerConcurrency(IntegerPreference):
default = 2
required = True
# User settings
# User settings
@user_preferences_registry.register
class MarkDeletedAsWatched(BooleanPreference):
section = manager
@ -51,6 +71,7 @@ class MarkDeletedAsWatched(BooleanPreference):
default = True
required = True
@user_preferences_registry.register
class AutoDeleteWatched(BooleanPreference):
section = manager
@ -58,20 +79,23 @@ class AutoDeleteWatched(BooleanPreference):
default = True
required = True
@user_preferences_registry.register
class AutoDownloadEnabled(BooleanPreference):
section = downloader
name = 'enabled'
name = 'download_enabled'
default = True
required = True
@user_preferences_registry.register
class DownloadGlobalLimit(IntegerPreference):
section = downloader
name = 'global_limit'
name = 'download_global_limit'
default = None
required = False
@user_preferences_registry.register
class DownloadGlobalSizeLimit(IntegerPreference):
section = downloader
@ -79,49 +103,56 @@ class DownloadGlobalSizeLimit(IntegerPreference):
default = None
required = False
@user_preferences_registry.register
class DownloadSubscriptionLimit(IntegerPreference):
section = downloader
name = 'limit_per_subscription'
name = 'download_limit_per_subscription'
default = 5
required = False
@user_preferences_registry.register
class DownloadMaxAttempts(IntegerPreference):
section = downloader
name = 'max_attempts'
name = 'download_max_attempts'
default = 3
required = True
@user_preferences_registry.register
class DownloadOrder(ChoicePreference):
section = downloader
name = 'order'
name = 'download_order'
choices = VIDEO_ORDER_CHOICES
default = 'playlist'
required = True
@user_preferences_registry.register
class DownloadPath(StringPreference):
section = downloader
name = 'path'
name = 'download_path'
default = None
required = False
@user_preferences_registry.register
class DownloadFilePattern(StringPreference):
section = downloader
name = 'file_pattern'
name = 'download_file_pattern'
default = '${channel}/${playlist}/S01E${playlist_index} - ${title} [${id}]'
required = True
@user_preferences_registry.register
class DownloadFormat(StringPreference):
section = downloader
name = 'format'
name = 'download_format'
default = 'bestvideo+bestaudio'
required = True
@user_preferences_registry.register
class DownloadSubtitles(BooleanPreference):
section = downloader
@ -129,6 +160,7 @@ class DownloadSubtitles(BooleanPreference):
default = True
required = True
@user_preferences_registry.register
class DownloadAutogeneratedSubtitles(BooleanPreference):
section = downloader
@ -136,6 +168,7 @@ class DownloadAutogeneratedSubtitles(BooleanPreference):
default = False
required = True
@user_preferences_registry.register
class DownloadAllSubtitles(BooleanPreference):
section = downloader
@ -143,6 +176,7 @@ class DownloadAllSubtitles(BooleanPreference):
default = False
required = False
@user_preferences_registry.register
class DownloadSubtitlesLangs(StringPreference):
section = downloader
@ -150,9 +184,10 @@ class DownloadSubtitlesLangs(StringPreference):
default = 'en,ro'
required = False
@user_preferences_registry.register
class DownloadSubtitlesFormat(StringPreference):
section = downloader
name = 'subtitles_format'
default = False
required = False
required = False

3
app/YtManagerApp/management/appconfig.py Normal file
View File

@ -0,0 +1,3 @@
from dynamic_preferences.registries import global_preferences_registry
global_prefs = global_preferences_registry.manager()

17
app/YtManagerApp/management/downloader.py
View File

@ -1,6 +1,6 @@
from YtManagerApp.appconfig import settings
from YtManagerApp.management.jobs.download_video import schedule_download_video
from YtManagerApp.models import Video, Subscription, VIDEO_ORDER_MAPPING
from YtManagerApp.utils import first_non_null
from django.conf import settings as srv_settings
import logging
import requests
@ -12,17 +12,12 @@ log = logging.getLogger('downloader')
def __get_subscription_config(sub: Subscription):
enabled = settings.getboolean_sub(sub, 'user', 'AutoDownload')
user = sub.user
global_limit = -1
if len(settings.get_sub(sub, 'user', 'DownloadGlobalLimit')) > 0:
global_limit = settings.getint_sub(sub, 'user', 'DownloadGlobalLimit')
limit = -1
if len(settings.get_sub(sub, 'user', 'DownloadSubscriptionLimit')) > 0:
limit = settings.getint_sub(sub, 'user', 'DownloadSubscriptionLimit')
order = settings.get_sub(sub, 'user', 'DownloadOrder')
enabled = first_non_null(sub.auto_download, user.preferences['download_enabled'])
global_limit = user.preferences['download_global_limit']
limit = first_non_null(sub.download_limit, user.preferences['download_limit_per_subscription'])
order = first_non_null(sub.download_order, user.preferences['download_order'])
order = VIDEO_ORDER_MAPPING[order]
return enabled, global_limit, limit, order

47
app/YtManagerApp/management/jobs/download_video.py
View File

@ -1,12 +1,14 @@
from YtManagerApp.models import Video
from YtManagerApp import scheduler
from YtManagerApp.appconfig import settings
import os
import youtube_dl
import logging
import os
import re
from string import Template
from threading import Lock
import youtube_dl
from YtManagerApp import scheduler
from YtManagerApp.models import Video
log = logging.getLogger('video_downloader')
log_youtube_dl = log.getChild('youtube_dl')
@ -25,9 +27,8 @@ def __get_valid_path(path):
return value
def __build_youtube_dl_params(video: Video):
# resolve path
pattern_dict = {
def __build_template_dict(video: Video):
return {
'channel': video.subscription.channel_name,
'channel_id': video.subscription.channel_id,
'playlist': video.subscription.name,
@ -37,22 +38,30 @@ def __build_youtube_dl_params(video: Video):
'id': video.video_id,
}
download_path = settings.get_sub(video.subscription, 'user', 'DownloadPath')
output_pattern = __get_valid_path(settings.get_sub(
video.subscription, 'user', 'DownloadFilePattern', vars=pattern_dict))
def __build_youtube_dl_params(video: Video):
sub = video.subscription
user = sub.user
# resolve path
download_path = user.preferences['download_path']
template_dict = __build_template_dict(video)
output_pattern = Template(user.preferences['download_file_pattern']).safe_substitute(template_dict)
output_path = os.path.join(download_path, output_pattern)
output_path = os.path.normpath(output_path)
youtube_dl_params = {
'logger': log_youtube_dl,
'format': settings.get_sub(video.subscription, 'user', 'DownloadFormat'),
'format': user.preferences['download_format'],
'outtmpl': output_path,
'writethumbnail': True,
'writedescription': True,
'writesubtitles': settings.getboolean_sub(video.subscription, 'user', 'DownloadSubtitles'),
'writeautomaticsub': settings.getboolean_sub(video.subscription, 'user', 'DownloadAutogeneratedSubtitles'),
'allsubtitles': settings.getboolean_sub(video.subscription, 'user', 'DownloadSubtitlesAll'),
'writesubtitles': user.preferences['subtitles_enabled'],
'writeautomaticsub': user.preferences['autogenerated_subtitles'],
'allsubtitles': user.preferences['all_subtitles'],
'postprocessors': [
{
'key': 'FFmpegMetadata'
@ -60,12 +69,12 @@ def __build_youtube_dl_params(video: Video):
]
}
sub_langs = settings.get_sub(video.subscription, 'user', 'DownloadSubtitlesLangs').split(',')
sub_langs = user.preferences['subtitles_langs'].split(',')
sub_langs = [i.strip() for i in sub_langs]
if len(sub_langs) > 0:
youtube_dl_params['subtitleslangs'] = sub_langs
sub_format = settings.get_sub(video.subscription, 'user', 'DownloadSubtitlesFormat')
sub_format = user.preferences['subtitles_format']
if len(sub_format) > 0:
youtube_dl_params['subtitlesformat'] = sub_format
@ -74,6 +83,8 @@ def __build_youtube_dl_params(video: Video):
def download_video(video: Video, attempt: int = 1):
user = video.subscription.user
log.info('Downloading video %d [%s %s]', video.id, video.video_id, video.name)
# Issue: if multiple videos are downloaded at the same time, a race condition appears in the mkdirs() call that
@ -82,7 +93,7 @@ def download_video(video: Video, attempt: int = 1):
_lock.acquire()
try:
max_attempts = settings.getint_sub(video.subscription, 'user', 'DownloadMaxAttempts', fallback=3)
max_attempts = user.preferences['download_max_attempts']
youtube_dl_params, output_path = __build_youtube_dl_params(video)
with youtube_dl.YoutubeDL(youtube_dl_params) as yt:

8
app/YtManagerApp/management/jobs/synchronize.py
View File

@ -5,7 +5,7 @@ from threading import Lock
from apscheduler.triggers.cron import CronTrigger
from YtManagerApp import scheduler
from YtManagerApp.appconfig import settings
from YtManagerApp.management.appconfig import global_prefs
from YtManagerApp.management.downloader import fetch_thumbnail, downloader_process_all, downloader_process_subscription
from YtManagerApp.models import *
from YtManagerApp.utils import youtube
@ -43,6 +43,8 @@ def __check_new_videos_sub(subscription: Subscription, yt_api: youtube.YoutubeAP
def __detect_deleted(subscription: Subscription):
user = subscription.user
for video in Video.objects.filter(subscription=subscription, downloaded_path__isnull=False):
found_video = False
files = []
@ -71,7 +73,7 @@ def __detect_deleted(subscription: Subscription):
video.downloaded_path = None
# Mark watched?
if settings.getboolean_sub(subscription, 'user', 'MarkDeletedAsWatched'):
if user.preferences['MarkDeletedAsWatched']:
video.watched = True
video.save()
@ -147,7 +149,7 @@ def synchronize_subscription(subscription: Subscription):
def schedule_synchronize_global():
trigger = CronTrigger.from_crontab(settings.get('global', 'SynchronizationSchedule'))
trigger = CronTrigger.from_crontab(global_prefs['synchronization_schedule'])
job = scheduler.scheduler.add_job(synchronize, trigger, max_instances=1, coalesce=True)
log.info('Scheduled synchronize job job=%s', job.id)

0
app/YtManagerApp/management/subscriptions.py
View File

9
app/YtManagerApp/utils/__init__.py
View File

@ -0,0 +1,9 @@
def first_non_null(*iterable):
'''
Returns the first element from the iterable which is not None.
If all the elements are 'None', 'None' is returned.
:param iterable: Iterable containing list of elements.
:return: First non-null element, or None if all elements are 'None'.
'''
return next((item for item in iterable if item is not None), None)