mirror of
https://github.com/chibicitiberiu/ytsm.git
synced 2024-02-24 05:43:31 +00:00
Integrated django-dynamic-preferences library into project. Settings are now stored in the database, the ini file only contains a minimal number of settings.
This commit is contained in:
parent
d3b057e14e
commit
0329645c37
@ -14,6 +14,7 @@ ENV YTSM_DATA_PATH='/usr/src/ytsm/data'
|
||||
|
||||
VOLUME /usr/src/ytsm/config
|
||||
VOLUME /usr/src/ytsm/data
|
||||
VOLUME /usr/src/ytsm/download
|
||||
|
||||
COPY ./app/ ./
|
||||
COPY ./docker/init.sh ./
|
||||
|
||||
@ -23,17 +23,17 @@ SESSION_COOKIE_AGE = 3600 * 30 # one month
|
||||
# Application definition
|
||||
|
||||
INSTALLED_APPS = [
|
||||
'django.contrib.auth',
|
||||
'dynamic_preferences',
|
||||
'dynamic_preferences.users.apps.UserPreferencesConfig',
|
||||
'YtManagerApp.apps.YtManagerAppConfig',
|
||||
'crispy_forms',
|
||||
'django.contrib.admin',
|
||||
'django.contrib.auth',
|
||||
'django.contrib.contenttypes',
|
||||
'django.contrib.sessions',
|
||||
'django.contrib.messages',
|
||||
'django.contrib.staticfiles',
|
||||
'django.contrib.humanize',
|
||||
'dynamic_preferences',
|
||||
'dynamic_preferences.users.apps.UserPreferencesConfig',
|
||||
]
|
||||
|
||||
MIDDLEWARE = [
|
||||
@ -125,16 +125,14 @@ LOG_FORMAT = '%(asctime)s|%(process)d|%(thread)d|%(name)s|%(filename)s|%(lineno)
|
||||
# Build paths inside the project like this: os.path.join(BASE_DIR, ...)
|
||||
PROJECT_ROOT = up(up(os.path.dirname(__file__))) # Project root
|
||||
BASE_DIR = up(os.path.dirname(__file__)) # Base dir of the application
|
||||
CONFIG_DIR = os.path.join(PROJECT_ROOT, "config")
|
||||
DATA_DIR = os.path.join(PROJECT_ROOT, "data")
|
||||
|
||||
CONFIG_DIR = os.getenv("YTSM_CONFIG_DIR", os.path.join(PROJECT_ROOT, "config"))
|
||||
DATA_DIR = os.getenv("YTSM_DATA_DIR", os.path.join(PROJECT_ROOT, "data"))
|
||||
os.chdir(DATA_DIR)
|
||||
|
||||
STATIC_ROOT = os.path.join(PROJECT_ROOT, "static")
|
||||
MEDIA_ROOT = os.path.join(DATA_DIR, 'media')
|
||||
|
||||
_DEFAULT_CONFIG_FILE = os.path.join(CONFIG_DIR, 'config.ini')
|
||||
_DEFAULT_LOG_FILE = os.path.join(DATA_DIR, 'log.log')
|
||||
_DEFAULT_MEDIA_ROOT = os.path.join(DATA_DIR, 'media')
|
||||
|
||||
CONFIG_FILE = os.getenv('YTSM_CONFIG_FILE', _DEFAULT_CONFIG_FILE)
|
||||
DATA_CONFIG_FILE = os.path.join(DATA_DIR, 'config.ini')
|
||||
|
||||
#
|
||||
# Defaults
|
||||
@ -142,7 +140,6 @@ DATA_CONFIG_FILE = os.path.join(DATA_DIR, 'config.ini')
|
||||
_DEFAULT_DEBUG = False
|
||||
|
||||
_DEFAULT_SECRET_KEY = '^zv8@i2h!ko2lo=%ivq(9e#x=%q*i^^)6#4@(juzdx%&0c+9a0'
|
||||
_DEFAULT_YOUTUBE_API_KEY = 'AIzaSyBabzE4Bup77WexdLMa9rN9z-wJidEfNX8'
|
||||
|
||||
_DEFAULT_DATABASE = {
|
||||
'ENGINE': 'django.db.backends.sqlite3',
|
||||
@ -153,10 +150,6 @@ _DEFAULT_DATABASE = {
|
||||
'PORT': None,
|
||||
}
|
||||
|
||||
_SCHEDULER_SYNC_SCHEDULE = '5 * * * *'
|
||||
_DEFAULT_SCHEDULER_CONCURRENCY = 1
|
||||
|
||||
|
||||
CONFIG_ERRORS = []
|
||||
CONFIG_WARNINGS = []
|
||||
|
||||
@ -171,6 +164,8 @@ def get_global_opt(name, cfgparser, env_variable=None, fallback=None, boolean=Fa
|
||||
2. config parser
|
||||
3. fallback
|
||||
|
||||
:param integer:
|
||||
:param cfgparser:
|
||||
:param name:
|
||||
:param env_variable:
|
||||
:param fallback:
|
||||
@ -216,25 +211,22 @@ def load_config_ini():
|
||||
import dj_database_url
|
||||
|
||||
cfg = ConfigParser(allow_no_value=True, interpolation=ExtendedInterpolatorWithEnv())
|
||||
read_ok = cfg.read([DEFAULTS_FILE, CONFIG_FILE, DATA_CONFIG_FILE])
|
||||
|
||||
if DEFAULTS_FILE not in read_ok:
|
||||
CONFIG_ERRORS.append(f'File {DEFAULTS_FILE} could not be read! Please make sure the file is in the right place,'
|
||||
f' and it has read permissions.')
|
||||
cfg_file = os.path.join(CONFIG_DIR, "config.ini")
|
||||
read_ok = cfg.read([cfg_file])
|
||||
|
||||
if cfg_file not in read_ok:
|
||||
CONFIG_ERRORS.append(f'Configuration file {cfg_file} could not be read! Please make sure the file is in the '
|
||||
'right place, and it has read permissions.')
|
||||
|
||||
# Debug
|
||||
global DEBUG
|
||||
DEBUG = get_global_opt('Debug', cfg, env_variable='YTSM_DEBUG', fallback=_DEFAULT_DEBUG, boolean=True)
|
||||
|
||||
# Media root, which is where thumbnails are stored
|
||||
global MEDIA_ROOT
|
||||
MEDIA_ROOT = get_global_opt('MediaRoot', cfg, env_variable='YTSM_MEDIA_ROOT', fallback=_DEFAULT_MEDIA_ROOT)
|
||||
|
||||
# Keys - secret key, youtube API key
|
||||
# Secret key
|
||||
# SECURITY WARNING: keep the secret key used in production secret!
|
||||
global SECRET_KEY, YOUTUBE_API_KEY
|
||||
global SECRET_KEY
|
||||
SECRET_KEY = get_global_opt('SecretKey', cfg, env_variable='YTSM_SECRET_KEY', fallback=_DEFAULT_SECRET_KEY)
|
||||
YOUTUBE_API_KEY = get_global_opt('YoutubeApiKey', cfg, env_variable='YTSM_YTAPI_KEY', fallback=_DEFAULT_YOUTUBE_API_KEY)
|
||||
|
||||
# Database
|
||||
global DATABASES
|
||||
@ -247,16 +239,22 @@ def load_config_ini():
|
||||
|
||||
else:
|
||||
DATABASES['default'] = {
|
||||
'ENGINE': get_global_opt('DatabaseEngine', cfg, env_variable='YTSM_DB_ENGINE', fallback=_DEFAULT_DATABASE['ENGINE']),
|
||||
'NAME': get_global_opt('DatabaseName', cfg, env_variable='YTSM_DB_NAME', fallback=_DEFAULT_DATABASE['NAME']),
|
||||
'HOST': get_global_opt('DatabaseHost', cfg, env_variable='YTSM_DB_HOST', fallback=_DEFAULT_DATABASE['HOST']),
|
||||
'USER': get_global_opt('DatabaseUser', cfg, env_variable='YTSM_DB_USER', fallback=_DEFAULT_DATABASE['USER']),
|
||||
'PASSWORD': get_global_opt('DatabasePassword', cfg, env_variable='YTSM_DB_PASSWORD', fallback=_DEFAULT_DATABASE['PASSWORD']),
|
||||
'PORT': get_global_opt('DatabasePort', cfg, env_variable='YTSM_DB_PORT', fallback=_DEFAULT_DATABASE['PORT']),
|
||||
'ENGINE': get_global_opt('DatabaseEngine', cfg,
|
||||
env_variable='YTSM_DB_ENGINE', fallback=_DEFAULT_DATABASE['ENGINE']),
|
||||
'NAME': get_global_opt('DatabaseName', cfg,
|
||||
env_variable='YTSM_DB_NAME', fallback=_DEFAULT_DATABASE['NAME']),
|
||||
'HOST': get_global_opt('DatabaseHost', cfg,
|
||||
env_variable='YTSM_DB_HOST', fallback=_DEFAULT_DATABASE['HOST']),
|
||||
'USER': get_global_opt('DatabaseUser', cfg,
|
||||
env_variable='YTSM_DB_USER', fallback=_DEFAULT_DATABASE['USER']),
|
||||
'PASSWORD': get_global_opt('DatabasePassword', cfg,
|
||||
env_variable='YTSM_DB_PASSWORD', fallback=_DEFAULT_DATABASE['PASSWORD']),
|
||||
'PORT': get_global_opt('DatabasePort', cfg,
|
||||
env_variable='YTSM_DB_PORT', fallback=_DEFAULT_DATABASE['PORT']),
|
||||
}
|
||||
|
||||
# Log settings
|
||||
global LOG_LEVEL, LOG_FILE
|
||||
global LOG_LEVEL
|
||||
log_level_str = get_global_opt('LogLevel', cfg, env_variable='YTSM_LOG_LEVEL', fallback='INFO')
|
||||
|
||||
try:
|
||||
@ -267,12 +265,5 @@ def load_config_ini():
|
||||
print("Invalid log level " + LOG_LEVEL)
|
||||
LOG_LEVEL = logging.INFO
|
||||
|
||||
LOG_FILE = get_global_opt('LogFile', cfg, env_variable='YTSM_LOG_FILE', fallback=_DEFAULT_LOG_FILE)
|
||||
|
||||
# Scheduler settings
|
||||
global SCHEDULER_SYNC_SCHEDULE, SCHEDULER_CONCURRENCY
|
||||
SCHEDULER_SYNC_SCHEDULE = get_global_opt('SynchronizationSchedule', cfg, fallback=_SCHEDULER_SYNC_SCHEDULE)
|
||||
SCHEDULER_CONCURRENCY = get_global_opt('SchedulerConcurrency', cfg, fallback=_DEFAULT_SCHEDULER_CONCURRENCY, integer=True)
|
||||
|
||||
|
||||
load_config_ini()
|
||||
|
||||
@ -1,112 +0,0 @@
|
||||
import logging
|
||||
import os
|
||||
import os.path
|
||||
from collections import ChainMap
|
||||
from configparser import ConfigParser
|
||||
from shutil import copyfile
|
||||
from typing import Optional, Any
|
||||
|
||||
from django.conf import settings as dj_settings
|
||||
from django.contrib.auth.models import User
|
||||
|
||||
from .models import UserSettings, Subscription
|
||||
from .utils.extended_interpolation_with_env import ExtendedInterpolatorWithEnv
|
||||
|
||||
|
||||
class AppSettings(ConfigParser):
|
||||
_DEFAULT_INTERPOLATION = ExtendedInterpolatorWithEnv()
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
super().__init__(allow_no_value=True, *args, **kwargs)
|
||||
|
||||
def initialize(self):
|
||||
self.read([dj_settings.DEFAULTS_FILE, dj_settings.CONFIG_FILE])
|
||||
|
||||
def save(self):
|
||||
if os.path.exists(dj_settings.CONFIG_FILE):
|
||||
# Create a backup
|
||||
copyfile(dj_settings.CONFIG_FILE, dj_settings.CONFIG_FILE + ".backup")
|
||||
else:
|
||||
# Ensure directory exists
|
||||
settings_dir = os.path.dirname(dj_settings.CONFIG_FILE)
|
||||
os.makedirs(settings_dir, exist_ok=True)
|
||||
|
||||
with open(dj_settings.CONFIG_FILE, 'w') as f:
|
||||
self.write(f)
|
||||
|
||||
def __get_combined_dict(self, vars: Optional[Any], sub: Optional[Subscription], user: Optional[User]) -> ChainMap:
|
||||
vars_dict = {}
|
||||
sub_overloads_dict = {}
|
||||
user_settings_dict = {}
|
||||
|
||||
if vars is not None:
|
||||
vars_dict = vars
|
||||
|
||||
if sub is not None:
|
||||
sub_overloads_dict = sub.get_overloads_dict()
|
||||
|
||||
if user is not None:
|
||||
user_settings = UserSettings.find_by_user(user)
|
||||
if user_settings is not None:
|
||||
user_settings_dict = user_settings.to_dict()
|
||||
|
||||
return ChainMap(vars_dict, sub_overloads_dict, user_settings_dict)
|
||||
|
||||
def get_user(self, user: User, section: str, option: Any, vars=None, fallback=object()) -> str:
|
||||
return super().get(section, option,
|
||||
fallback=fallback,
|
||||
vars=self.__get_combined_dict(vars, None, user))
|
||||
|
||||
def getboolean_user(self, user: User, section: str, option: Any, vars=None, fallback=object()) -> bool:
|
||||
return super().getboolean(section, option,
|
||||
fallback=fallback,
|
||||
vars=self.__get_combined_dict(vars, None, user))
|
||||
|
||||
def getint_user(self, user: User, section: str, option: Any, vars=None, fallback=object()) -> int:
|
||||
return super().getint(section, option,
|
||||
fallback=fallback,
|
||||
vars=self.__get_combined_dict(vars, None, user))
|
||||
|
||||
def getfloat_user(self, user: User, section: str, option: Any, vars=None, fallback=object()) -> float:
|
||||
return super().getfloat(section, option,
|
||||
fallback=fallback,
|
||||
vars=self.__get_combined_dict(vars, None, user))
|
||||
|
||||
def get_sub(self, sub: Subscription, section: str, option: Any, vars=None, fallback=object()) -> str:
|
||||
return super().get(section, option,
|
||||
fallback=fallback,
|
||||
vars=self.__get_combined_dict(vars, sub, sub.user))
|
||||
|
||||
def getboolean_sub(self, sub: Subscription, section: str, option: Any, vars=None, fallback=object()) -> bool:
|
||||
return super().getboolean(section, option,
|
||||
fallback=fallback,
|
||||
vars=self.__get_combined_dict(vars, sub, sub.user))
|
||||
|
||||
def getint_sub(self, sub: Subscription, section: str, option: Any, vars=None, fallback=object()) -> int:
|
||||
return super().getint(section, option,
|
||||
fallback=fallback,
|
||||
vars=self.__get_combined_dict(vars, sub, sub.user))
|
||||
|
||||
def getfloat_sub(self, sub: Subscription, section: str, option: Any, vars=None, fallback=object()) -> float:
|
||||
return super().getfloat(section, option,
|
||||
fallback=fallback,
|
||||
vars=self.__get_combined_dict(vars, sub, sub.user))
|
||||
|
||||
|
||||
settings = AppSettings()
|
||||
|
||||
|
||||
def initialize_app_config():
|
||||
settings.initialize()
|
||||
__initialize_logger()
|
||||
logging.info('Application started!')
|
||||
|
||||
|
||||
def __initialize_logger():
|
||||
log_dir = os.path.dirname(dj_settings.LOG_FILE)
|
||||
os.makedirs(log_dir, exist_ok=True)
|
||||
|
||||
logging.basicConfig(
|
||||
filename=dj_settings.LOG_FILE,
|
||||
level=dj_settings.LOG_LEVEL,
|
||||
format=dj_settings.LOG_FORMAT)
|
||||
@ -1,11 +1,36 @@
|
||||
from .appconfig import initialize_app_config
|
||||
from .scheduler import initialize_scheduler
|
||||
from .management.jobs.synchronize import schedule_synchronize_global
|
||||
import logging
|
||||
import logging.handlers
|
||||
import os
|
||||
|
||||
from django.conf import settings as dj_settings
|
||||
|
||||
from .management.appconfig import global_prefs
|
||||
from .management.jobs.synchronize import schedule_synchronize_global
|
||||
from .scheduler import initialize_scheduler
|
||||
|
||||
|
||||
def __initialize_logger():
|
||||
log_dir = os.path.join(dj_settings.DATA_DIR, 'logs')
|
||||
os.makedirs(log_dir, exist_ok=True)
|
||||
|
||||
file_handler = logging.handlers.RotatingFileHandler(
|
||||
os.path.join(log_dir, "log.log"),
|
||||
maxBytes=1024 * 1024,
|
||||
backupCount=5
|
||||
)
|
||||
|
||||
logging.basicConfig(
|
||||
level=dj_settings.LOG_LEVEL,
|
||||
format=dj_settings.LOG_FORMAT,
|
||||
handlers=[file_handler]
|
||||
)
|
||||
|
||||
|
||||
def main():
|
||||
initialize_app_config()
|
||||
__initialize_logger()
|
||||
|
||||
if global_prefs['hidden__initialized']:
|
||||
initialize_scheduler()
|
||||
schedule_synchronize_global()
|
||||
|
||||
logging.info('Initialization complete.')
|
||||
|
||||
@ -1,5 +1,4 @@
|
||||
from django.apps import AppConfig
|
||||
import os
|
||||
|
||||
|
||||
class YtManagerAppConfig(AppConfig):
|
||||
|
||||
@ -13,6 +13,7 @@ scheduler = Section('scheduler')
|
||||
manager = Section('manager')
|
||||
downloader = Section('downloader')
|
||||
|
||||
|
||||
# Hidden settings
|
||||
@global_preferences_registry.register
|
||||
class Initialized(BooleanPreference):
|
||||
@ -20,6 +21,7 @@ class Initialized(BooleanPreference):
|
||||
name = 'initialized'
|
||||
default = False
|
||||
|
||||
|
||||
# General settings
|
||||
@global_preferences_registry.register
|
||||
class YouTubeAPIKey(StringPreference):
|
||||
@ -28,6 +30,23 @@ class YouTubeAPIKey(StringPreference):
|
||||
default = 'AIzaSyBabzE4Bup77WexdLMa9rN9z-wJidEfNX8'
|
||||
required = True
|
||||
|
||||
|
||||
@global_preferences_registry.register
|
||||
class AllowCDN(BooleanPreference):
|
||||
section = general
|
||||
name = 'allow_cdn'
|
||||
default = True
|
||||
required = True
|
||||
|
||||
|
||||
@global_preferences_registry.register
|
||||
class AllowRegistrations(BooleanPreference):
|
||||
section = general
|
||||
name = 'allow_registrations'
|
||||
default = True
|
||||
required = True
|
||||
|
||||
|
||||
@global_preferences_registry.register
|
||||
class SyncSchedule(StringPreference):
|
||||
section = scheduler
|
||||
@ -35,6 +54,7 @@ class SyncSchedule(StringPreference):
|
||||
default = '5 * * * *' # hourly
|
||||
required = True
|
||||
|
||||
|
||||
@global_preferences_registry.register
|
||||
class SchedulerConcurrency(IntegerPreference):
|
||||
section = scheduler
|
||||
@ -42,8 +62,8 @@ class SchedulerConcurrency(IntegerPreference):
|
||||
default = 2
|
||||
required = True
|
||||
|
||||
# User settings
|
||||
|
||||
# User settings
|
||||
@user_preferences_registry.register
|
||||
class MarkDeletedAsWatched(BooleanPreference):
|
||||
section = manager
|
||||
@ -51,6 +71,7 @@ class MarkDeletedAsWatched(BooleanPreference):
|
||||
default = True
|
||||
required = True
|
||||
|
||||
|
||||
@user_preferences_registry.register
|
||||
class AutoDeleteWatched(BooleanPreference):
|
||||
section = manager
|
||||
@ -58,20 +79,23 @@ class AutoDeleteWatched(BooleanPreference):
|
||||
default = True
|
||||
required = True
|
||||
|
||||
|
||||
@user_preferences_registry.register
|
||||
class AutoDownloadEnabled(BooleanPreference):
|
||||
section = downloader
|
||||
name = 'enabled'
|
||||
name = 'download_enabled'
|
||||
default = True
|
||||
required = True
|
||||
|
||||
|
||||
@user_preferences_registry.register
|
||||
class DownloadGlobalLimit(IntegerPreference):
|
||||
section = downloader
|
||||
name = 'global_limit'
|
||||
name = 'download_global_limit'
|
||||
default = None
|
||||
required = False
|
||||
|
||||
|
||||
@user_preferences_registry.register
|
||||
class DownloadGlobalSizeLimit(IntegerPreference):
|
||||
section = downloader
|
||||
@ -79,49 +103,56 @@ class DownloadGlobalSizeLimit(IntegerPreference):
|
||||
default = None
|
||||
required = False
|
||||
|
||||
|
||||
@user_preferences_registry.register
|
||||
class DownloadSubscriptionLimit(IntegerPreference):
|
||||
section = downloader
|
||||
name = 'limit_per_subscription'
|
||||
name = 'download_limit_per_subscription'
|
||||
default = 5
|
||||
required = False
|
||||
|
||||
|
||||
@user_preferences_registry.register
|
||||
class DownloadMaxAttempts(IntegerPreference):
|
||||
section = downloader
|
||||
name = 'max_attempts'
|
||||
name = 'download_max_attempts'
|
||||
default = 3
|
||||
required = True
|
||||
|
||||
|
||||
@user_preferences_registry.register
|
||||
class DownloadOrder(ChoicePreference):
|
||||
section = downloader
|
||||
name = 'order'
|
||||
name = 'download_order'
|
||||
choices = VIDEO_ORDER_CHOICES
|
||||
default = 'playlist'
|
||||
required = True
|
||||
|
||||
|
||||
@user_preferences_registry.register
|
||||
class DownloadPath(StringPreference):
|
||||
section = downloader
|
||||
name = 'path'
|
||||
name = 'download_path'
|
||||
default = None
|
||||
required = False
|
||||
|
||||
|
||||
@user_preferences_registry.register
|
||||
class DownloadFilePattern(StringPreference):
|
||||
section = downloader
|
||||
name = 'file_pattern'
|
||||
name = 'download_file_pattern'
|
||||
default = '${channel}/${playlist}/S01E${playlist_index} - ${title} [${id}]'
|
||||
required = True
|
||||
|
||||
|
||||
@user_preferences_registry.register
|
||||
class DownloadFormat(StringPreference):
|
||||
section = downloader
|
||||
name = 'format'
|
||||
name = 'download_format'
|
||||
default = 'bestvideo+bestaudio'
|
||||
required = True
|
||||
|
||||
|
||||
@user_preferences_registry.register
|
||||
class DownloadSubtitles(BooleanPreference):
|
||||
section = downloader
|
||||
@ -129,6 +160,7 @@ class DownloadSubtitles(BooleanPreference):
|
||||
default = True
|
||||
required = True
|
||||
|
||||
|
||||
@user_preferences_registry.register
|
||||
class DownloadAutogeneratedSubtitles(BooleanPreference):
|
||||
section = downloader
|
||||
@ -136,6 +168,7 @@ class DownloadAutogeneratedSubtitles(BooleanPreference):
|
||||
default = False
|
||||
required = True
|
||||
|
||||
|
||||
@user_preferences_registry.register
|
||||
class DownloadAllSubtitles(BooleanPreference):
|
||||
section = downloader
|
||||
@ -143,6 +176,7 @@ class DownloadAllSubtitles(BooleanPreference):
|
||||
default = False
|
||||
required = False
|
||||
|
||||
|
||||
@user_preferences_registry.register
|
||||
class DownloadSubtitlesLangs(StringPreference):
|
||||
section = downloader
|
||||
@ -150,6 +184,7 @@ class DownloadSubtitlesLangs(StringPreference):
|
||||
default = 'en,ro'
|
||||
required = False
|
||||
|
||||
|
||||
@user_preferences_registry.register
|
||||
class DownloadSubtitlesFormat(StringPreference):
|
||||
section = downloader
|
||||
|
||||
3
app/YtManagerApp/management/appconfig.py
Normal file
3
app/YtManagerApp/management/appconfig.py
Normal file
@ -0,0 +1,3 @@
|
||||
from dynamic_preferences.registries import global_preferences_registry
|
||||
|
||||
global_prefs = global_preferences_registry.manager()
|
||||
@ -1,6 +1,6 @@
|
||||
from YtManagerApp.appconfig import settings
|
||||
from YtManagerApp.management.jobs.download_video import schedule_download_video
|
||||
from YtManagerApp.models import Video, Subscription, VIDEO_ORDER_MAPPING
|
||||
from YtManagerApp.utils import first_non_null
|
||||
from django.conf import settings as srv_settings
|
||||
import logging
|
||||
import requests
|
||||
@ -12,17 +12,12 @@ log = logging.getLogger('downloader')
|
||||
|
||||
|
||||
def __get_subscription_config(sub: Subscription):
|
||||
enabled = settings.getboolean_sub(sub, 'user', 'AutoDownload')
|
||||
user = sub.user
|
||||
|
||||
global_limit = -1
|
||||
if len(settings.get_sub(sub, 'user', 'DownloadGlobalLimit')) > 0:
|
||||
global_limit = settings.getint_sub(sub, 'user', 'DownloadGlobalLimit')
|
||||
|
||||
limit = -1
|
||||
if len(settings.get_sub(sub, 'user', 'DownloadSubscriptionLimit')) > 0:
|
||||
limit = settings.getint_sub(sub, 'user', 'DownloadSubscriptionLimit')
|
||||
|
||||
order = settings.get_sub(sub, 'user', 'DownloadOrder')
|
||||
enabled = first_non_null(sub.auto_download, user.preferences['download_enabled'])
|
||||
global_limit = user.preferences['download_global_limit']
|
||||
limit = first_non_null(sub.download_limit, user.preferences['download_limit_per_subscription'])
|
||||
order = first_non_null(sub.download_order, user.preferences['download_order'])
|
||||
order = VIDEO_ORDER_MAPPING[order]
|
||||
|
||||
return enabled, global_limit, limit, order
|
||||
|
||||
@ -1,12 +1,14 @@
|
||||
from YtManagerApp.models import Video
|
||||
from YtManagerApp import scheduler
|
||||
from YtManagerApp.appconfig import settings
|
||||
import os
|
||||
import youtube_dl
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
from string import Template
|
||||
from threading import Lock
|
||||
|
||||
import youtube_dl
|
||||
|
||||
from YtManagerApp import scheduler
|
||||
from YtManagerApp.models import Video
|
||||
|
||||
log = logging.getLogger('video_downloader')
|
||||
log_youtube_dl = log.getChild('youtube_dl')
|
||||
|
||||
@ -25,9 +27,8 @@ def __get_valid_path(path):
|
||||
return value
|
||||
|
||||
|
||||
def __build_youtube_dl_params(video: Video):
|
||||
# resolve path
|
||||
pattern_dict = {
|
||||
def __build_template_dict(video: Video):
|
||||
return {
|
||||
'channel': video.subscription.channel_name,
|
||||
'channel_id': video.subscription.channel_id,
|
||||
'playlist': video.subscription.name,
|
||||
@ -37,22 +38,30 @@ def __build_youtube_dl_params(video: Video):
|
||||
'id': video.video_id,
|
||||
}
|
||||
|
||||
download_path = settings.get_sub(video.subscription, 'user', 'DownloadPath')
|
||||
output_pattern = __get_valid_path(settings.get_sub(
|
||||
video.subscription, 'user', 'DownloadFilePattern', vars=pattern_dict))
|
||||
|
||||
def __build_youtube_dl_params(video: Video):
|
||||
|
||||
sub = video.subscription
|
||||
user = sub.user
|
||||
|
||||
# resolve path
|
||||
download_path = user.preferences['download_path']
|
||||
|
||||
template_dict = __build_template_dict(video)
|
||||
output_pattern = Template(user.preferences['download_file_pattern']).safe_substitute(template_dict)
|
||||
|
||||
output_path = os.path.join(download_path, output_pattern)
|
||||
output_path = os.path.normpath(output_path)
|
||||
|
||||
youtube_dl_params = {
|
||||
'logger': log_youtube_dl,
|
||||
'format': settings.get_sub(video.subscription, 'user', 'DownloadFormat'),
|
||||
'format': user.preferences['download_format'],
|
||||
'outtmpl': output_path,
|
||||
'writethumbnail': True,
|
||||
'writedescription': True,
|
||||
'writesubtitles': settings.getboolean_sub(video.subscription, 'user', 'DownloadSubtitles'),
|
||||
'writeautomaticsub': settings.getboolean_sub(video.subscription, 'user', 'DownloadAutogeneratedSubtitles'),
|
||||
'allsubtitles': settings.getboolean_sub(video.subscription, 'user', 'DownloadSubtitlesAll'),
|
||||
'writesubtitles': user.preferences['subtitles_enabled'],
|
||||
'writeautomaticsub': user.preferences['autogenerated_subtitles'],
|
||||
'allsubtitles': user.preferences['all_subtitles'],
|
||||
'postprocessors': [
|
||||
{
|
||||
'key': 'FFmpegMetadata'
|
||||
@ -60,12 +69,12 @@ def __build_youtube_dl_params(video: Video):
|
||||
]
|
||||
}
|
||||
|
||||
sub_langs = settings.get_sub(video.subscription, 'user', 'DownloadSubtitlesLangs').split(',')
|
||||
sub_langs = user.preferences['subtitles_langs'].split(',')
|
||||
sub_langs = [i.strip() for i in sub_langs]
|
||||
if len(sub_langs) > 0:
|
||||
youtube_dl_params['subtitleslangs'] = sub_langs
|
||||
|
||||
sub_format = settings.get_sub(video.subscription, 'user', 'DownloadSubtitlesFormat')
|
||||
sub_format = user.preferences['subtitles_format']
|
||||
if len(sub_format) > 0:
|
||||
youtube_dl_params['subtitlesformat'] = sub_format
|
||||
|
||||
@ -74,6 +83,8 @@ def __build_youtube_dl_params(video: Video):
|
||||
|
||||
def download_video(video: Video, attempt: int = 1):
|
||||
|
||||
user = video.subscription.user
|
||||
|
||||
log.info('Downloading video %d [%s %s]', video.id, video.video_id, video.name)
|
||||
|
||||
# Issue: if multiple videos are downloaded at the same time, a race condition appears in the mkdirs() call that
|
||||
@ -82,7 +93,7 @@ def download_video(video: Video, attempt: int = 1):
|
||||
_lock.acquire()
|
||||
|
||||
try:
|
||||
max_attempts = settings.getint_sub(video.subscription, 'user', 'DownloadMaxAttempts', fallback=3)
|
||||
max_attempts = user.preferences['download_max_attempts']
|
||||
|
||||
youtube_dl_params, output_path = __build_youtube_dl_params(video)
|
||||
with youtube_dl.YoutubeDL(youtube_dl_params) as yt:
|
||||
|
||||
@ -5,7 +5,7 @@ from threading import Lock
|
||||
from apscheduler.triggers.cron import CronTrigger
|
||||
|
||||
from YtManagerApp import scheduler
|
||||
from YtManagerApp.appconfig import settings
|
||||
from YtManagerApp.management.appconfig import global_prefs
|
||||
from YtManagerApp.management.downloader import fetch_thumbnail, downloader_process_all, downloader_process_subscription
|
||||
from YtManagerApp.models import *
|
||||
from YtManagerApp.utils import youtube
|
||||
@ -43,6 +43,8 @@ def __check_new_videos_sub(subscription: Subscription, yt_api: youtube.YoutubeAP
|
||||
|
||||
def __detect_deleted(subscription: Subscription):
|
||||
|
||||
user = subscription.user
|
||||
|
||||
for video in Video.objects.filter(subscription=subscription, downloaded_path__isnull=False):
|
||||
found_video = False
|
||||
files = []
|
||||
@ -71,7 +73,7 @@ def __detect_deleted(subscription: Subscription):
|
||||
video.downloaded_path = None
|
||||
|
||||
# Mark watched?
|
||||
if settings.getboolean_sub(subscription, 'user', 'MarkDeletedAsWatched'):
|
||||
if user.preferences['MarkDeletedAsWatched']:
|
||||
video.watched = True
|
||||
|
||||
video.save()
|
||||
@ -147,7 +149,7 @@ def synchronize_subscription(subscription: Subscription):
|
||||
|
||||
|
||||
def schedule_synchronize_global():
|
||||
trigger = CronTrigger.from_crontab(settings.get('global', 'SynchronizationSchedule'))
|
||||
trigger = CronTrigger.from_crontab(global_prefs['synchronization_schedule'])
|
||||
job = scheduler.scheduler.add_job(synchronize, trigger, max_instances=1, coalesce=True)
|
||||
log.info('Scheduled synchronize job job=%s', job.id)
|
||||
|
||||
|
||||
@ -0,0 +1,9 @@
|
||||
|
||||
def first_non_null(*iterable):
|
||||
'''
|
||||
Returns the first element from the iterable which is not None.
|
||||
If all the elements are 'None', 'None' is returned.
|
||||
:param iterable: Iterable containing list of elements.
|
||||
:return: First non-null element, or None if all elements are 'None'.
|
||||
'''
|
||||
return next((item for item in iterable if item is not None), None)
|
||||
@ -1,87 +0,0 @@
|
||||
; Use ${env:environment_variable} to use the value of an environment variable.
|
||||
; The global section contains settings that apply to the entire server
|
||||
[global]
|
||||
|
||||
; Controls whether django debug mode is enabled. Should be false in production.
|
||||
Debug=False
|
||||
|
||||
; This is the folder where thumbnails will be downloaded. By default project_root/data/media is used.
|
||||
;MediaRoot=
|
||||
|
||||
; Secret key - django secret key
|
||||
SecretKey=^zv8@i2h!ko2lo=%ivq(9e#x=%q*i^^)6#4@(juzdx%&0c+9a0
|
||||
|
||||
; YouTube API key - get this from your user account
|
||||
YoutubeApiKey=AIzaSyBabzE4Bup77WexdLMa9rN9z-wJidEfNX8
|
||||
|
||||
; Database settings
|
||||
; You can use any database engine supported by Django, as long as you add the required dependencies.
|
||||
; Built-in engines: https://docs.djangoproject.com/en/2.1/ref/settings/#std:setting-DATABASE-ENGINE
|
||||
; Others databases might be supported by installing the corect pip package.
|
||||
|
||||
;DatabaseEngine=django.db.backends.sqlite3
|
||||
;DatabaseName=data/ytmanager.db
|
||||
;DatabaseHost=
|
||||
;DatabaseUser=
|
||||
;DatabasePassword=
|
||||
;DatabasePort=
|
||||
|
||||
; Database one-liner. If set, it will override any other Database* setting.
|
||||
; Documentation: https://github.com/kennethreitz/dj-database-url
|
||||
;DatabaseURL=sqlite:////full/path/to/your/database/file.sqlite
|
||||
|
||||
; Log settings, sets the log file location and the log level
|
||||
LogLevel=INFO
|
||||
; LogFile=data/log.log
|
||||
|
||||
; Specifies the synchronization schedule, in crontab format.
|
||||
; Format: <minute> <hour> <day-of-month> <month-of-year> <day of week>
|
||||
SynchronizationSchedule=5 * * * *
|
||||
|
||||
; Number of threads running the scheduler
|
||||
; Since most of the jobs scheduled are downloads, there is no advantage to having
|
||||
; a higher concurrency
|
||||
SchedulerConcurrency=3
|
||||
|
||||
|
||||
; Default user settings
|
||||
[user]
|
||||
; When a video is deleted on the system, it will be marked as 'watched'
|
||||
MarkDeletedAsWatched=True
|
||||
|
||||
; Videos marked as watched are automatically deleted
|
||||
DeleteWatched=True
|
||||
|
||||
; Enable automatic downloading
|
||||
AutoDownload=True
|
||||
|
||||
; Limit the total number of videos downloaded (-1 or empty = no limit)
|
||||
DownloadGlobalLimit=
|
||||
|
||||
; Limit the numbers of videos per subscription (-1 or empty = no limit)
|
||||
DownloadSubscriptionLimit=5
|
||||
|
||||
; Number of download attempts
|
||||
DownloadMaxAttempts=3
|
||||
|
||||
; Download order
|
||||
; Options: newest, oldest, playlist, playlist_reverse, popularity, rating
|
||||
DownloadOrder=playlist
|
||||
|
||||
; Path where downloaded videos are stored
|
||||
DownloadPath=data/videos
|
||||
|
||||
; A pattern which describes how downloaded files are organized. Extensions are automatically appended.
|
||||
; Supported fields: channel, channel_id, playlist, playlist_id, playlist_index, title, id
|
||||
; The default pattern should work pretty well with Plex
|
||||
DownloadFilePattern=${channel}/${playlist}/S01E${playlist_index} - ${title} [${id}]
|
||||
|
||||
; Download format that will be passed to youtube-dl. See the youtube-dl documentation for more details.
|
||||
DownloadFormat=bestvideo+bestaudio
|
||||
|
||||
; Subtitles - these options match the youtube-dl options
|
||||
DownloadSubtitles=True
|
||||
DownloadAutogeneratedSubtitles=False
|
||||
DownloadSubtitlesAll=False
|
||||
DownloadSubtitlesLangs=en,ro
|
||||
DownloadSubtitlesFormat=
|
||||
@ -6,22 +6,16 @@
|
||||
|
||||
Debug=True
|
||||
|
||||
; This is the folder where thumbnails will be downloaded. By default project_root/data/media is used.
|
||||
;MediaRoot=
|
||||
|
||||
; Secret key - django secret key
|
||||
;SecretKey=^zv8@i2h!ko2lo=%ivq(9e#x=%q*i^^)6#4@(juzdx%&0c+9a0
|
||||
|
||||
; YouTube API key - get this from your user account
|
||||
;YoutubeApiKey=AIzaSyAonB6T-DrKjfGxBGuHyFMg0x_d0T9nlP8
|
||||
SecretKey=^zv8@i2h!ko2lo=%ivq(9e#x=%q*i^^)6#4@(juzdx%&0c+9a0
|
||||
|
||||
; Database settings
|
||||
; You can use any database engine supported by Django, as long as you add the required dependencies.
|
||||
; Built-in engines: https://docs.djangoproject.com/en/2.1/ref/settings/#std:setting-DATABASE-ENGINE
|
||||
; Others databases might be supported by installing the corect pip package.
|
||||
|
||||
;DatabaseEngine=django.db.backends.sqlite3
|
||||
;DatabaseName=data/ytmanager.db
|
||||
DatabaseEngine=django.db.backends.sqlite3
|
||||
DatabaseName=ytmanager.db
|
||||
;DatabaseHost=
|
||||
;DatabaseUser=
|
||||
;DatabasePassword=
|
||||
@ -32,56 +26,4 @@ Debug=True
|
||||
;DatabaseURL=sqlite:////full/path/to/your/database/file.sqlite
|
||||
|
||||
; Log settings, sets the log file location and the log level
|
||||
;LogLevel=INFO
|
||||
;LogFile=data/log.log
|
||||
|
||||
; Specifies the synchronization schedule, in crontab format.
|
||||
; Format: <minute> <hour> <day-of-month> <month-of-year> <day of week>
|
||||
;SynchronizationSchedule=5 * * * *
|
||||
|
||||
; Number of threads running the scheduler
|
||||
; Since most of the jobs scheduled are downloads, there is no advantage to having
|
||||
; a higher concurrency
|
||||
;SchedulerConcurrency=3
|
||||
|
||||
; Default user settings
|
||||
[user]
|
||||
; When a video is deleted on the system, it will be marked as 'watched'
|
||||
;MarkDeletedAsWatched=True
|
||||
|
||||
; Videos marked as watched are automatically deleted
|
||||
;DeleteWatched=True
|
||||
|
||||
; Enable automatic downloading
|
||||
;AutoDownload=True
|
||||
|
||||
; Limit the total number of videos downloaded (-1 or empty = no limit)
|
||||
;DownloadGlobalLimit=10
|
||||
|
||||
; Limit the numbers of videos per subscription (-1 or empty = no limit)
|
||||
;DownloadSubscriptionLimit=5
|
||||
|
||||
; Number of download attempts
|
||||
;DownloadMaxAttempts=3
|
||||
|
||||
; Download order
|
||||
; Options: newest, oldest, playlist, playlist_reverse, popularity, rating
|
||||
;DownloadOrder=playlist
|
||||
|
||||
; Path where downloaded videos are stored
|
||||
DownloadPath=${env:YTSM_DATA_PATH}/videos
|
||||
|
||||
; A pattern which describes how downloaded files are organized. Extensions are automatically appended.
|
||||
; Supported fields: channel, channel_id, playlist, playlist_id, playlist_index, title, id
|
||||
; The default pattern should work pretty well with Plex
|
||||
;DownloadFilePattern=${channel}/${playlist}/S01E${playlist_index} - ${title} [${id}]
|
||||
|
||||
; Download format that will be passed to youtube-dl. See the youtube-dl documentation for more details.
|
||||
;DownloadFormat=bestvideo+bestaudio
|
||||
|
||||
; Subtitles - these options match the youtube-dl options
|
||||
;DownloadSubtitles=True
|
||||
;DownloadAutogeneratedSubtitles=False
|
||||
;DownloadSubtitlesAll=False
|
||||
;DownloadSubtitlesLangs=en,ro
|
||||
;DownloadSubtitlesFormat=
|
||||
LogLevel=INFO
|
||||
|
||||
@ -20,3 +20,4 @@ services:
|
||||
volumes:
|
||||
- ./config:/usr/src/ytsm/config
|
||||
- ./data:/usr/src/ytsm/data
|
||||
- ./downloads:/usr/src/ytsm/downloads
|
||||
@ -1,3 +1,4 @@
|
||||
requests
|
||||
apscheduler
|
||||
gunicorn
|
||||
django
|
||||
|
||||
Loading…
Reference in New Issue
Block a user