Integrated django-dynamic-preferences library into project. Settings are now stored in the database, the ini file only contains a minimal number of settings.

This commit is contained in:
Tiberiu Chibici 2018-12-10 01:15:10 +02:00
parent d3b057e14e
commit 0329645c37
16 changed files with 169 additions and 353 deletions

View File

@ -14,6 +14,7 @@ ENV YTSM_DATA_PATH='/usr/src/ytsm/data'
VOLUME /usr/src/ytsm/config
VOLUME /usr/src/ytsm/data
VOLUME /usr/src/ytsm/download
COPY ./app/ ./
COPY ./docker/init.sh ./

View File

@ -23,17 +23,17 @@ SESSION_COOKIE_AGE = 3600 * 30 # one month
# Application definition
INSTALLED_APPS = [
'django.contrib.auth',
'dynamic_preferences',
'dynamic_preferences.users.apps.UserPreferencesConfig',
'YtManagerApp.apps.YtManagerAppConfig',
'crispy_forms',
'django.contrib.admin',
'django.contrib.auth',
'django.contrib.contenttypes',
'django.contrib.sessions',
'django.contrib.messages',
'django.contrib.staticfiles',
'django.contrib.humanize',
'dynamic_preferences',
'dynamic_preferences.users.apps.UserPreferencesConfig',
]
MIDDLEWARE = [
@ -125,16 +125,14 @@ LOG_FORMAT = '%(asctime)s|%(process)d|%(thread)d|%(name)s|%(filename)s|%(lineno)
# Build paths inside the project like this: os.path.join(BASE_DIR, ...)
PROJECT_ROOT = up(up(os.path.dirname(__file__))) # Project root
BASE_DIR = up(os.path.dirname(__file__)) # Base dir of the application
CONFIG_DIR = os.path.join(PROJECT_ROOT, "config")
DATA_DIR = os.path.join(PROJECT_ROOT, "data")
CONFIG_DIR = os.getenv("YTSM_CONFIG_DIR", os.path.join(PROJECT_ROOT, "config"))
DATA_DIR = os.getenv("YTSM_DATA_DIR", os.path.join(PROJECT_ROOT, "data"))
os.chdir(DATA_DIR)
STATIC_ROOT = os.path.join(PROJECT_ROOT, "static")
MEDIA_ROOT = os.path.join(DATA_DIR, 'media')
_DEFAULT_CONFIG_FILE = os.path.join(CONFIG_DIR, 'config.ini')
_DEFAULT_LOG_FILE = os.path.join(DATA_DIR, 'log.log')
_DEFAULT_MEDIA_ROOT = os.path.join(DATA_DIR, 'media')
CONFIG_FILE = os.getenv('YTSM_CONFIG_FILE', _DEFAULT_CONFIG_FILE)
DATA_CONFIG_FILE = os.path.join(DATA_DIR, 'config.ini')
#
# Defaults
@ -142,7 +140,6 @@ DATA_CONFIG_FILE = os.path.join(DATA_DIR, 'config.ini')
_DEFAULT_DEBUG = False
_DEFAULT_SECRET_KEY = '^zv8@i2h!ko2lo=%ivq(9e#x=%q*i^^)6#4@(juzdx%&0c+9a0'
_DEFAULT_YOUTUBE_API_KEY = 'AIzaSyBabzE4Bup77WexdLMa9rN9z-wJidEfNX8'
_DEFAULT_DATABASE = {
'ENGINE': 'django.db.backends.sqlite3',
@ -153,10 +150,6 @@ _DEFAULT_DATABASE = {
'PORT': None,
}
_SCHEDULER_SYNC_SCHEDULE = '5 * * * *'
_DEFAULT_SCHEDULER_CONCURRENCY = 1
CONFIG_ERRORS = []
CONFIG_WARNINGS = []
@ -171,6 +164,8 @@ def get_global_opt(name, cfgparser, env_variable=None, fallback=None, boolean=Fa
2. config parser
3. fallback
:param integer:
:param cfgparser:
:param name:
:param env_variable:
:param fallback:
@ -216,25 +211,22 @@ def load_config_ini():
import dj_database_url
cfg = ConfigParser(allow_no_value=True, interpolation=ExtendedInterpolatorWithEnv())
read_ok = cfg.read([DEFAULTS_FILE, CONFIG_FILE, DATA_CONFIG_FILE])
if DEFAULTS_FILE not in read_ok:
CONFIG_ERRORS.append(f'File {DEFAULTS_FILE} could not be read! Please make sure the file is in the right place,'
f' and it has read permissions.')
cfg_file = os.path.join(CONFIG_DIR, "config.ini")
read_ok = cfg.read([cfg_file])
if cfg_file not in read_ok:
CONFIG_ERRORS.append(f'Configuration file {cfg_file} could not be read! Please make sure the file is in the '
'right place, and it has read permissions.')
# Debug
global DEBUG
DEBUG = get_global_opt('Debug', cfg, env_variable='YTSM_DEBUG', fallback=_DEFAULT_DEBUG, boolean=True)
# Media root, which is where thumbnails are stored
global MEDIA_ROOT
MEDIA_ROOT = get_global_opt('MediaRoot', cfg, env_variable='YTSM_MEDIA_ROOT', fallback=_DEFAULT_MEDIA_ROOT)
# Keys - secret key, youtube API key
# Secret key
# SECURITY WARNING: keep the secret key used in production secret!
global SECRET_KEY, YOUTUBE_API_KEY
global SECRET_KEY
SECRET_KEY = get_global_opt('SecretKey', cfg, env_variable='YTSM_SECRET_KEY', fallback=_DEFAULT_SECRET_KEY)
YOUTUBE_API_KEY = get_global_opt('YoutubeApiKey', cfg, env_variable='YTSM_YTAPI_KEY', fallback=_DEFAULT_YOUTUBE_API_KEY)
# Database
global DATABASES
@ -247,16 +239,22 @@ def load_config_ini():
else:
DATABASES['default'] = {
'ENGINE': get_global_opt('DatabaseEngine', cfg, env_variable='YTSM_DB_ENGINE', fallback=_DEFAULT_DATABASE['ENGINE']),
'NAME': get_global_opt('DatabaseName', cfg, env_variable='YTSM_DB_NAME', fallback=_DEFAULT_DATABASE['NAME']),
'HOST': get_global_opt('DatabaseHost', cfg, env_variable='YTSM_DB_HOST', fallback=_DEFAULT_DATABASE['HOST']),
'USER': get_global_opt('DatabaseUser', cfg, env_variable='YTSM_DB_USER', fallback=_DEFAULT_DATABASE['USER']),
'PASSWORD': get_global_opt('DatabasePassword', cfg, env_variable='YTSM_DB_PASSWORD', fallback=_DEFAULT_DATABASE['PASSWORD']),
'PORT': get_global_opt('DatabasePort', cfg, env_variable='YTSM_DB_PORT', fallback=_DEFAULT_DATABASE['PORT']),
'ENGINE': get_global_opt('DatabaseEngine', cfg,
env_variable='YTSM_DB_ENGINE', fallback=_DEFAULT_DATABASE['ENGINE']),
'NAME': get_global_opt('DatabaseName', cfg,
env_variable='YTSM_DB_NAME', fallback=_DEFAULT_DATABASE['NAME']),
'HOST': get_global_opt('DatabaseHost', cfg,
env_variable='YTSM_DB_HOST', fallback=_DEFAULT_DATABASE['HOST']),
'USER': get_global_opt('DatabaseUser', cfg,
env_variable='YTSM_DB_USER', fallback=_DEFAULT_DATABASE['USER']),
'PASSWORD': get_global_opt('DatabasePassword', cfg,
env_variable='YTSM_DB_PASSWORD', fallback=_DEFAULT_DATABASE['PASSWORD']),
'PORT': get_global_opt('DatabasePort', cfg,
env_variable='YTSM_DB_PORT', fallback=_DEFAULT_DATABASE['PORT']),
}
# Log settings
global LOG_LEVEL, LOG_FILE
global LOG_LEVEL
log_level_str = get_global_opt('LogLevel', cfg, env_variable='YTSM_LOG_LEVEL', fallback='INFO')
try:
@ -267,12 +265,5 @@ def load_config_ini():
print("Invalid log level " + LOG_LEVEL)
LOG_LEVEL = logging.INFO
LOG_FILE = get_global_opt('LogFile', cfg, env_variable='YTSM_LOG_FILE', fallback=_DEFAULT_LOG_FILE)
# Scheduler settings
global SCHEDULER_SYNC_SCHEDULE, SCHEDULER_CONCURRENCY
SCHEDULER_SYNC_SCHEDULE = get_global_opt('SynchronizationSchedule', cfg, fallback=_SCHEDULER_SYNC_SCHEDULE)
SCHEDULER_CONCURRENCY = get_global_opt('SchedulerConcurrency', cfg, fallback=_DEFAULT_SCHEDULER_CONCURRENCY, integer=True)
load_config_ini()

View File

@ -1,112 +0,0 @@
import logging
import os
import os.path
from collections import ChainMap
from configparser import ConfigParser
from shutil import copyfile
from typing import Optional, Any
from django.conf import settings as dj_settings
from django.contrib.auth.models import User
from .models import UserSettings, Subscription
from .utils.extended_interpolation_with_env import ExtendedInterpolatorWithEnv
class AppSettings(ConfigParser):
_DEFAULT_INTERPOLATION = ExtendedInterpolatorWithEnv()
def __init__(self, *args, **kwargs):
super().__init__(allow_no_value=True, *args, **kwargs)
def initialize(self):
self.read([dj_settings.DEFAULTS_FILE, dj_settings.CONFIG_FILE])
def save(self):
if os.path.exists(dj_settings.CONFIG_FILE):
# Create a backup
copyfile(dj_settings.CONFIG_FILE, dj_settings.CONFIG_FILE + ".backup")
else:
# Ensure directory exists
settings_dir = os.path.dirname(dj_settings.CONFIG_FILE)
os.makedirs(settings_dir, exist_ok=True)
with open(dj_settings.CONFIG_FILE, 'w') as f:
self.write(f)
def __get_combined_dict(self, vars: Optional[Any], sub: Optional[Subscription], user: Optional[User]) -> ChainMap:
vars_dict = {}
sub_overloads_dict = {}
user_settings_dict = {}
if vars is not None:
vars_dict = vars
if sub is not None:
sub_overloads_dict = sub.get_overloads_dict()
if user is not None:
user_settings = UserSettings.find_by_user(user)
if user_settings is not None:
user_settings_dict = user_settings.to_dict()
return ChainMap(vars_dict, sub_overloads_dict, user_settings_dict)
def get_user(self, user: User, section: str, option: Any, vars=None, fallback=object()) -> str:
return super().get(section, option,
fallback=fallback,
vars=self.__get_combined_dict(vars, None, user))
def getboolean_user(self, user: User, section: str, option: Any, vars=None, fallback=object()) -> bool:
return super().getboolean(section, option,
fallback=fallback,
vars=self.__get_combined_dict(vars, None, user))
def getint_user(self, user: User, section: str, option: Any, vars=None, fallback=object()) -> int:
return super().getint(section, option,
fallback=fallback,
vars=self.__get_combined_dict(vars, None, user))
def getfloat_user(self, user: User, section: str, option: Any, vars=None, fallback=object()) -> float:
return super().getfloat(section, option,
fallback=fallback,
vars=self.__get_combined_dict(vars, None, user))
def get_sub(self, sub: Subscription, section: str, option: Any, vars=None, fallback=object()) -> str:
return super().get(section, option,
fallback=fallback,
vars=self.__get_combined_dict(vars, sub, sub.user))
def getboolean_sub(self, sub: Subscription, section: str, option: Any, vars=None, fallback=object()) -> bool:
return super().getboolean(section, option,
fallback=fallback,
vars=self.__get_combined_dict(vars, sub, sub.user))
def getint_sub(self, sub: Subscription, section: str, option: Any, vars=None, fallback=object()) -> int:
return super().getint(section, option,
fallback=fallback,
vars=self.__get_combined_dict(vars, sub, sub.user))
def getfloat_sub(self, sub: Subscription, section: str, option: Any, vars=None, fallback=object()) -> float:
return super().getfloat(section, option,
fallback=fallback,
vars=self.__get_combined_dict(vars, sub, sub.user))
settings = AppSettings()
def initialize_app_config():
settings.initialize()
__initialize_logger()
logging.info('Application started!')
def __initialize_logger():
log_dir = os.path.dirname(dj_settings.LOG_FILE)
os.makedirs(log_dir, exist_ok=True)
logging.basicConfig(
filename=dj_settings.LOG_FILE,
level=dj_settings.LOG_LEVEL,
format=dj_settings.LOG_FORMAT)

View File

@ -1,11 +1,36 @@
from .appconfig import initialize_app_config
from .scheduler import initialize_scheduler
from .management.jobs.synchronize import schedule_synchronize_global
import logging
import logging.handlers
import os
from django.conf import settings as dj_settings
from .management.appconfig import global_prefs
from .management.jobs.synchronize import schedule_synchronize_global
from .scheduler import initialize_scheduler
def __initialize_logger():
log_dir = os.path.join(dj_settings.DATA_DIR, 'logs')
os.makedirs(log_dir, exist_ok=True)
file_handler = logging.handlers.RotatingFileHandler(
os.path.join(log_dir, "log.log"),
maxBytes=1024 * 1024,
backupCount=5
)
logging.basicConfig(
level=dj_settings.LOG_LEVEL,
format=dj_settings.LOG_FORMAT,
handlers=[file_handler]
)
def main():
initialize_app_config()
initialize_scheduler()
schedule_synchronize_global()
__initialize_logger()
if global_prefs['hidden__initialized']:
initialize_scheduler()
schedule_synchronize_global()
logging.info('Initialization complete.')

View File

@ -1,5 +1,4 @@
from django.apps import AppConfig
import os
class YtManagerAppConfig(AppConfig):

View File

@ -13,6 +13,7 @@ scheduler = Section('scheduler')
manager = Section('manager')
downloader = Section('downloader')
# Hidden settings
@global_preferences_registry.register
class Initialized(BooleanPreference):
@ -20,6 +21,7 @@ class Initialized(BooleanPreference):
name = 'initialized'
default = False
# General settings
@global_preferences_registry.register
class YouTubeAPIKey(StringPreference):
@ -28,13 +30,31 @@ class YouTubeAPIKey(StringPreference):
default = 'AIzaSyBabzE4Bup77WexdLMa9rN9z-wJidEfNX8'
required = True
@global_preferences_registry.register
class AllowCDN(BooleanPreference):
section = general
name = 'allow_cdn'
default = True
required = True
@global_preferences_registry.register
class AllowRegistrations(BooleanPreference):
section = general
name = 'allow_registrations'
default = True
required = True
@global_preferences_registry.register
class SyncSchedule(StringPreference):
section = scheduler
name = 'synchronization_schedule'
default = '5 * * * *' # hourly
default = '5 * * * *' # hourly
required = True
@global_preferences_registry.register
class SchedulerConcurrency(IntegerPreference):
section = scheduler
@ -42,8 +62,8 @@ class SchedulerConcurrency(IntegerPreference):
default = 2
required = True
# User settings
# User settings
@user_preferences_registry.register
class MarkDeletedAsWatched(BooleanPreference):
section = manager
@ -51,6 +71,7 @@ class MarkDeletedAsWatched(BooleanPreference):
default = True
required = True
@user_preferences_registry.register
class AutoDeleteWatched(BooleanPreference):
section = manager
@ -58,20 +79,23 @@ class AutoDeleteWatched(BooleanPreference):
default = True
required = True
@user_preferences_registry.register
class AutoDownloadEnabled(BooleanPreference):
section = downloader
name = 'enabled'
name = 'download_enabled'
default = True
required = True
@user_preferences_registry.register
class DownloadGlobalLimit(IntegerPreference):
section = downloader
name = 'global_limit'
name = 'download_global_limit'
default = None
required = False
@user_preferences_registry.register
class DownloadGlobalSizeLimit(IntegerPreference):
section = downloader
@ -79,49 +103,56 @@ class DownloadGlobalSizeLimit(IntegerPreference):
default = None
required = False
@user_preferences_registry.register
class DownloadSubscriptionLimit(IntegerPreference):
section = downloader
name = 'limit_per_subscription'
name = 'download_limit_per_subscription'
default = 5
required = False
@user_preferences_registry.register
class DownloadMaxAttempts(IntegerPreference):
section = downloader
name = 'max_attempts'
name = 'download_max_attempts'
default = 3
required = True
@user_preferences_registry.register
class DownloadOrder(ChoicePreference):
section = downloader
name = 'order'
name = 'download_order'
choices = VIDEO_ORDER_CHOICES
default = 'playlist'
required = True
@user_preferences_registry.register
class DownloadPath(StringPreference):
section = downloader
name = 'path'
name = 'download_path'
default = None
required = False
@user_preferences_registry.register
class DownloadFilePattern(StringPreference):
section = downloader
name = 'file_pattern'
name = 'download_file_pattern'
default = '${channel}/${playlist}/S01E${playlist_index} - ${title} [${id}]'
required = True
@user_preferences_registry.register
class DownloadFormat(StringPreference):
section = downloader
name = 'format'
name = 'download_format'
default = 'bestvideo+bestaudio'
required = True
@user_preferences_registry.register
class DownloadSubtitles(BooleanPreference):
section = downloader
@ -129,6 +160,7 @@ class DownloadSubtitles(BooleanPreference):
default = True
required = True
@user_preferences_registry.register
class DownloadAutogeneratedSubtitles(BooleanPreference):
section = downloader
@ -136,6 +168,7 @@ class DownloadAutogeneratedSubtitles(BooleanPreference):
default = False
required = True
@user_preferences_registry.register
class DownloadAllSubtitles(BooleanPreference):
section = downloader
@ -143,6 +176,7 @@ class DownloadAllSubtitles(BooleanPreference):
default = False
required = False
@user_preferences_registry.register
class DownloadSubtitlesLangs(StringPreference):
section = downloader
@ -150,9 +184,10 @@ class DownloadSubtitlesLangs(StringPreference):
default = 'en,ro'
required = False
@user_preferences_registry.register
class DownloadSubtitlesFormat(StringPreference):
section = downloader
name = 'subtitles_format'
default = False
required = False
required = False

View File

@ -0,0 +1,3 @@
from dynamic_preferences.registries import global_preferences_registry
global_prefs = global_preferences_registry.manager()

View File

@ -1,6 +1,6 @@
from YtManagerApp.appconfig import settings
from YtManagerApp.management.jobs.download_video import schedule_download_video
from YtManagerApp.models import Video, Subscription, VIDEO_ORDER_MAPPING
from YtManagerApp.utils import first_non_null
from django.conf import settings as srv_settings
import logging
import requests
@ -12,17 +12,12 @@ log = logging.getLogger('downloader')
def __get_subscription_config(sub: Subscription):
enabled = settings.getboolean_sub(sub, 'user', 'AutoDownload')
user = sub.user
global_limit = -1
if len(settings.get_sub(sub, 'user', 'DownloadGlobalLimit')) > 0:
global_limit = settings.getint_sub(sub, 'user', 'DownloadGlobalLimit')
limit = -1
if len(settings.get_sub(sub, 'user', 'DownloadSubscriptionLimit')) > 0:
limit = settings.getint_sub(sub, 'user', 'DownloadSubscriptionLimit')
order = settings.get_sub(sub, 'user', 'DownloadOrder')
enabled = first_non_null(sub.auto_download, user.preferences['download_enabled'])
global_limit = user.preferences['download_global_limit']
limit = first_non_null(sub.download_limit, user.preferences['download_limit_per_subscription'])
order = first_non_null(sub.download_order, user.preferences['download_order'])
order = VIDEO_ORDER_MAPPING[order]
return enabled, global_limit, limit, order

View File

@ -1,12 +1,14 @@
from YtManagerApp.models import Video
from YtManagerApp import scheduler
from YtManagerApp.appconfig import settings
import os
import youtube_dl
import logging
import os
import re
from string import Template
from threading import Lock
import youtube_dl
from YtManagerApp import scheduler
from YtManagerApp.models import Video
log = logging.getLogger('video_downloader')
log_youtube_dl = log.getChild('youtube_dl')
@ -25,9 +27,8 @@ def __get_valid_path(path):
return value
def __build_youtube_dl_params(video: Video):
# resolve path
pattern_dict = {
def __build_template_dict(video: Video):
return {
'channel': video.subscription.channel_name,
'channel_id': video.subscription.channel_id,
'playlist': video.subscription.name,
@ -37,22 +38,30 @@ def __build_youtube_dl_params(video: Video):
'id': video.video_id,
}
download_path = settings.get_sub(video.subscription, 'user', 'DownloadPath')
output_pattern = __get_valid_path(settings.get_sub(
video.subscription, 'user', 'DownloadFilePattern', vars=pattern_dict))
def __build_youtube_dl_params(video: Video):
sub = video.subscription
user = sub.user
# resolve path
download_path = user.preferences['download_path']
template_dict = __build_template_dict(video)
output_pattern = Template(user.preferences['download_file_pattern']).safe_substitute(template_dict)
output_path = os.path.join(download_path, output_pattern)
output_path = os.path.normpath(output_path)
youtube_dl_params = {
'logger': log_youtube_dl,
'format': settings.get_sub(video.subscription, 'user', 'DownloadFormat'),
'format': user.preferences['download_format'],
'outtmpl': output_path,
'writethumbnail': True,
'writedescription': True,
'writesubtitles': settings.getboolean_sub(video.subscription, 'user', 'DownloadSubtitles'),
'writeautomaticsub': settings.getboolean_sub(video.subscription, 'user', 'DownloadAutogeneratedSubtitles'),
'allsubtitles': settings.getboolean_sub(video.subscription, 'user', 'DownloadSubtitlesAll'),
'writesubtitles': user.preferences['subtitles_enabled'],
'writeautomaticsub': user.preferences['autogenerated_subtitles'],
'allsubtitles': user.preferences['all_subtitles'],
'postprocessors': [
{
'key': 'FFmpegMetadata'
@ -60,12 +69,12 @@ def __build_youtube_dl_params(video: Video):
]
}
sub_langs = settings.get_sub(video.subscription, 'user', 'DownloadSubtitlesLangs').split(',')
sub_langs = user.preferences['subtitles_langs'].split(',')
sub_langs = [i.strip() for i in sub_langs]
if len(sub_langs) > 0:
youtube_dl_params['subtitleslangs'] = sub_langs
sub_format = settings.get_sub(video.subscription, 'user', 'DownloadSubtitlesFormat')
sub_format = user.preferences['subtitles_format']
if len(sub_format) > 0:
youtube_dl_params['subtitlesformat'] = sub_format
@ -74,6 +83,8 @@ def __build_youtube_dl_params(video: Video):
def download_video(video: Video, attempt: int = 1):
user = video.subscription.user
log.info('Downloading video %d [%s %s]', video.id, video.video_id, video.name)
# Issue: if multiple videos are downloaded at the same time, a race condition appears in the mkdirs() call that
@ -82,7 +93,7 @@ def download_video(video: Video, attempt: int = 1):
_lock.acquire()
try:
max_attempts = settings.getint_sub(video.subscription, 'user', 'DownloadMaxAttempts', fallback=3)
max_attempts = user.preferences['download_max_attempts']
youtube_dl_params, output_path = __build_youtube_dl_params(video)
with youtube_dl.YoutubeDL(youtube_dl_params) as yt:

View File

@ -5,7 +5,7 @@ from threading import Lock
from apscheduler.triggers.cron import CronTrigger
from YtManagerApp import scheduler
from YtManagerApp.appconfig import settings
from YtManagerApp.management.appconfig import global_prefs
from YtManagerApp.management.downloader import fetch_thumbnail, downloader_process_all, downloader_process_subscription
from YtManagerApp.models import *
from YtManagerApp.utils import youtube
@ -43,6 +43,8 @@ def __check_new_videos_sub(subscription: Subscription, yt_api: youtube.YoutubeAP
def __detect_deleted(subscription: Subscription):
user = subscription.user
for video in Video.objects.filter(subscription=subscription, downloaded_path__isnull=False):
found_video = False
files = []
@ -71,7 +73,7 @@ def __detect_deleted(subscription: Subscription):
video.downloaded_path = None
# Mark watched?
if settings.getboolean_sub(subscription, 'user', 'MarkDeletedAsWatched'):
if user.preferences['MarkDeletedAsWatched']:
video.watched = True
video.save()
@ -147,7 +149,7 @@ def synchronize_subscription(subscription: Subscription):
def schedule_synchronize_global():
trigger = CronTrigger.from_crontab(settings.get('global', 'SynchronizationSchedule'))
trigger = CronTrigger.from_crontab(global_prefs['synchronization_schedule'])
job = scheduler.scheduler.add_job(synchronize, trigger, max_instances=1, coalesce=True)
log.info('Scheduled synchronize job job=%s', job.id)

View File

@ -0,0 +1,9 @@
def first_non_null(*iterable):
'''
Returns the first element from the iterable which is not None.
If all the elements are 'None', 'None' is returned.
:param iterable: Iterable containing list of elements.
:return: First non-null element, or None if all elements are 'None'.
'''
return next((item for item in iterable if item is not None), None)

View File

@ -1,87 +0,0 @@
; Use ${env:environment_variable} to use the value of an environment variable.
; The global section contains settings that apply to the entire server
[global]
; Controls whether django debug mode is enabled. Should be false in production.
Debug=False
; This is the folder where thumbnails will be downloaded. By default project_root/data/media is used.
;MediaRoot=
; Secret key - django secret key
SecretKey=^zv8@i2h!ko2lo=%ivq(9e#x=%q*i^^)6#4@(juzdx%&0c+9a0
; YouTube API key - get this from your user account
YoutubeApiKey=AIzaSyBabzE4Bup77WexdLMa9rN9z-wJidEfNX8
; Database settings
; You can use any database engine supported by Django, as long as you add the required dependencies.
; Built-in engines: https://docs.djangoproject.com/en/2.1/ref/settings/#std:setting-DATABASE-ENGINE
; Others databases might be supported by installing the corect pip package.
;DatabaseEngine=django.db.backends.sqlite3
;DatabaseName=data/ytmanager.db
;DatabaseHost=
;DatabaseUser=
;DatabasePassword=
;DatabasePort=
; Database one-liner. If set, it will override any other Database* setting.
; Documentation: https://github.com/kennethreitz/dj-database-url
;DatabaseURL=sqlite:////full/path/to/your/database/file.sqlite
; Log settings, sets the log file location and the log level
LogLevel=INFO
; LogFile=data/log.log
; Specifies the synchronization schedule, in crontab format.
; Format: <minute> <hour> <day-of-month> <month-of-year> <day of week>
SynchronizationSchedule=5 * * * *
; Number of threads running the scheduler
; Since most of the jobs scheduled are downloads, there is no advantage to having
; a higher concurrency
SchedulerConcurrency=3
; Default user settings
[user]
; When a video is deleted on the system, it will be marked as 'watched'
MarkDeletedAsWatched=True
; Videos marked as watched are automatically deleted
DeleteWatched=True
; Enable automatic downloading
AutoDownload=True
; Limit the total number of videos downloaded (-1 or empty = no limit)
DownloadGlobalLimit=
; Limit the numbers of videos per subscription (-1 or empty = no limit)
DownloadSubscriptionLimit=5
; Number of download attempts
DownloadMaxAttempts=3
; Download order
; Options: newest, oldest, playlist, playlist_reverse, popularity, rating
DownloadOrder=playlist
; Path where downloaded videos are stored
DownloadPath=data/videos
; A pattern which describes how downloaded files are organized. Extensions are automatically appended.
; Supported fields: channel, channel_id, playlist, playlist_id, playlist_index, title, id
; The default pattern should work pretty well with Plex
DownloadFilePattern=${channel}/${playlist}/S01E${playlist_index} - ${title} [${id}]
; Download format that will be passed to youtube-dl. See the youtube-dl documentation for more details.
DownloadFormat=bestvideo+bestaudio
; Subtitles - these options match the youtube-dl options
DownloadSubtitles=True
DownloadAutogeneratedSubtitles=False
DownloadSubtitlesAll=False
DownloadSubtitlesLangs=en,ro
DownloadSubtitlesFormat=

View File

@ -6,22 +6,16 @@
Debug=True
; This is the folder where thumbnails will be downloaded. By default project_root/data/media is used.
;MediaRoot=
; Secret key - django secret key
;SecretKey=^zv8@i2h!ko2lo=%ivq(9e#x=%q*i^^)6#4@(juzdx%&0c+9a0
; YouTube API key - get this from your user account
;YoutubeApiKey=AIzaSyAonB6T-DrKjfGxBGuHyFMg0x_d0T9nlP8
SecretKey=^zv8@i2h!ko2lo=%ivq(9e#x=%q*i^^)6#4@(juzdx%&0c+9a0
; Database settings
; You can use any database engine supported by Django, as long as you add the required dependencies.
; Built-in engines: https://docs.djangoproject.com/en/2.1/ref/settings/#std:setting-DATABASE-ENGINE
; Others databases might be supported by installing the corect pip package.
;DatabaseEngine=django.db.backends.sqlite3
;DatabaseName=data/ytmanager.db
DatabaseEngine=django.db.backends.sqlite3
DatabaseName=ytmanager.db
;DatabaseHost=
;DatabaseUser=
;DatabasePassword=
@ -32,56 +26,4 @@ Debug=True
;DatabaseURL=sqlite:////full/path/to/your/database/file.sqlite
; Log settings, sets the log file location and the log level
;LogLevel=INFO
;LogFile=data/log.log
; Specifies the synchronization schedule, in crontab format.
; Format: <minute> <hour> <day-of-month> <month-of-year> <day of week>
;SynchronizationSchedule=5 * * * *
; Number of threads running the scheduler
; Since most of the jobs scheduled are downloads, there is no advantage to having
; a higher concurrency
;SchedulerConcurrency=3
; Default user settings
[user]
; When a video is deleted on the system, it will be marked as 'watched'
;MarkDeletedAsWatched=True
; Videos marked as watched are automatically deleted
;DeleteWatched=True
; Enable automatic downloading
;AutoDownload=True
; Limit the total number of videos downloaded (-1 or empty = no limit)
;DownloadGlobalLimit=10
; Limit the numbers of videos per subscription (-1 or empty = no limit)
;DownloadSubscriptionLimit=5
; Number of download attempts
;DownloadMaxAttempts=3
; Download order
; Options: newest, oldest, playlist, playlist_reverse, popularity, rating
;DownloadOrder=playlist
; Path where downloaded videos are stored
DownloadPath=${env:YTSM_DATA_PATH}/videos
; A pattern which describes how downloaded files are organized. Extensions are automatically appended.
; Supported fields: channel, channel_id, playlist, playlist_id, playlist_index, title, id
; The default pattern should work pretty well with Plex
;DownloadFilePattern=${channel}/${playlist}/S01E${playlist_index} - ${title} [${id}]
; Download format that will be passed to youtube-dl. See the youtube-dl documentation for more details.
;DownloadFormat=bestvideo+bestaudio
; Subtitles - these options match the youtube-dl options
;DownloadSubtitles=True
;DownloadAutogeneratedSubtitles=False
;DownloadSubtitlesAll=False
;DownloadSubtitlesLangs=en,ro
;DownloadSubtitlesFormat=
LogLevel=INFO

View File

@ -20,3 +20,4 @@ services:
volumes:
- ./config:/usr/src/ytsm/config
- ./data:/usr/src/ytsm/data
- ./downloads:/usr/src/ytsm/downloads

View File

@ -1,4 +1,5 @@
apscheduler
requests
apscheduler
gunicorn
django
django-crispy-forms