Fixed multiple issues in configparser usage and implementation. Added dropdowns for sort order, and used the values everywhere.

YtManager/settings.py

@@ -129,3 +129,4 @@ MEDIA_ROOT = 'D:\\Dev\\youtube-channel-manager\\temp\\media'
 CRISPY_TEMPLATE_PACK = 'bootstrap4'
 
 LOGIN_REDIRECT_URL = '/'
+LOGIN_URL = '/login'

YtManagerApp/appconfig.py

@@ -1,46 +1,114 @@
 import logging
 import os
 import os.path
+from collections import ChainMap
+from configparser import ConfigParser
 from shutil import copyfile
+from typing import Optional, Any
 
 from django.conf import settings as dj_settings
 from django.contrib.auth.models import User
 
-from .models import UserSettings
+from .models import UserSettings, Subscription
-from .utils.customconfigparser import ConfigParserWithEnv
+from .utils.extended_interpolation_with_env import ExtendedInterpolatorWithEnv
 
+_CONFIG_DIR = os.path.join(dj_settings.BASE_DIR, 'config')
+_LOG_FILE = 'log.log'
+_LOG_PATH = os.path.join(_CONFIG_DIR, _LOG_FILE)
+_LOG_FORMAT = '%(asctime)s|%(process)d|%(thread)d|%(name)s|%(filename)s|%(lineno)d|%(levelname)s|%(message)s'
+
+
+class AppSettings(ConfigParser):
+    _DEFAULT_INTERPOLATION = ExtendedInterpolatorWithEnv()
+    __DEFAULTS_FILE = 'defaults.ini'
     __SETTINGS_FILE = 'config.ini'
-__LOG_FILE = 'log.log'
-__LOG_FORMAT = '%(asctime)s|%(process)d|%(thread)d|%(name)s|%(filename)s|%(lineno)d|%(levelname)s|%(message)s'
 
-__DEFAULT_SETTINGS = {
+    def __init__(self, *args, **kwargs):
-    'global': {
+        super().__init__(allow_no_value=True, *args, **kwargs)
-        'YouTubeApiKey': 'AIzaSyBabzE4Bup77WexdLMa9rN9z-wJidEfNX8',
+        self.__defaults_path = os.path.join(_CONFIG_DIR, AppSettings.__DEFAULTS_FILE)
-        'SynchronizationSchedule': '0 * * * *',
+        self.__settings_path = os.path.join(_CONFIG_DIR, AppSettings.__SETTINGS_FILE)
-        'SchedulerConcurrency': '2',
-    },
-    'user': {
-        'MarkDeletedAsWatched': 'True',
-        'DeleteWatched': 'True',
-        'AutoDownload': 'True',
-        'DownloadMaxAttempts': '3',
-        'DownloadGlobalLimit': '',
-        'DownloadSubscriptionLimit': '5',
-        'DownloadOrder': 'playlist_index',
-        'DownloadPath': '${env:USERPROFILE}${env:HOME}/Downloads',
-        'DownloadFilePattern': '${channel}/${playlist}/S01E${playlist_index} - ${title} [${id}]',
-        'DownloadFormat': 'bestvideo+bestaudio',
-        'DownloadSubtitles': 'True',
-        'DownloadAutogeneratedSubtitles': 'False',
-        'DownloadSubtitlesAll': 'False',
-        'DownloadSubtitlesLangs': 'en,ro',
-        'DownloadSubtitlesFormat': '',
-    }
-}
 
-log_path = os.path.join(dj_settings.BASE_DIR, 'config', __LOG_FILE)
+    def initialize(self):
-settings_path = os.path.join(dj_settings.BASE_DIR, 'config', __SETTINGS_FILE)
+        self.read([self.__defaults_path, self.__settings_path])
-settings = ConfigParserWithEnv(defaults=__DEFAULT_SETTINGS, allow_no_value=True)
+
+    def save(self):
+        if os.path.exists(self.__settings_path):
+            # Create a backup
+            copyfile(self.__settings_path, self.__settings_path + ".backup")
+        else:
+            # Ensure directory exists
+            settings_dir = os.path.dirname(self.__settings_path)
+            os.makedirs(settings_dir, exist_ok=True)
+
+        with open(self.__settings_path, 'w') as f:
+            self.write(f)
+
+    def __get_combined_dict(self, vars: Optional[Any], sub: Optional[Subscription], user: Optional[User]) -> ChainMap:
+        vars_dict = {}
+        sub_overloads_dict = {}
+        user_settings_dict = {}
+
+        if vars is not None:
+            vars_dict = vars
+
+        if sub is not None:
+            sub_overloads_dict = sub.get_overloads_dict()
+
+        if user is not None:
+            user_settings = UserSettings.find_by_user(user)
+            if user_settings is not None:
+                user_settings_dict = user_settings.to_dict()
+
+        return ChainMap(vars_dict, sub_overloads_dict, user_settings_dict)
+
+    def get_user(self, user: User, section: str, option: Any, vars=None, fallback=object()) -> str:
+        return super().get(section, option,
+                           fallback=fallback,
+                           vars=self.__get_combined_dict(vars, None, user))
+
+    def getboolean_user(self, user: User, section: str, option: Any, vars=None, fallback=object()) -> bool:
+        return super().getboolean(section, option,
+                                  fallback=fallback,
+                                  vars=self.__get_combined_dict(vars, None, user))
+
+    def getint_user(self, user: User, section: str, option: Any, vars=None, fallback=object()) -> int:
+        return super().getint(section, option,
+                              fallback=fallback,
+                              vars=self.__get_combined_dict(vars, None, user))
+
+    def getfloat_user(self, user: User, section: str, option: Any, vars=None, fallback=object()) -> float:
+        return super().getfloat(section, option,
+                                fallback=fallback,
+                                vars=self.__get_combined_dict(vars, None, user))
+
+    def get_sub(self, sub: Subscription, section: str, option: Any, vars=None, fallback=object()) -> str:
+        return super().get(section, option,
+                           fallback=fallback,
+                           vars=self.__get_combined_dict(vars, sub, sub.user))
+
+    def getboolean_sub(self, sub: Subscription, section: str, option: Any, vars=None, fallback=object()) -> bool:
+        return super().getboolean(section, option,
+                                  fallback=fallback,
+                                  vars=self.__get_combined_dict(vars, sub, sub.user))
+
+    def getint_sub(self, sub: Subscription, section: str, option: Any, vars=None, fallback=object()) -> int:
+        return super().getint(section, option,
+                              fallback=fallback,
+                              vars=self.__get_combined_dict(vars, sub, sub.user))
+
+    def getfloat_sub(self, sub: Subscription, section: str, option: Any, vars=None, fallback=object()) -> float:
+        return super().getfloat(section, option,
+                                fallback=fallback,
+                                vars=self.__get_combined_dict(vars, sub, sub.user))
+
+
+settings = AppSettings()
+
+
+def initialize_app_config():
+    settings.initialize()
+    __initialize_logger()
+    logging.info('Application started!')
 
 
 def __initialize_logger():
@@ -48,45 +116,8 @@ def __initialize_logger():
 
     try:
         log_level = getattr(logging, log_level_str)
-        logging.basicConfig(filename=log_path, level=log_level, format=__LOG_FORMAT)
+        logging.basicConfig(filename=_LOG_PATH, level=log_level, format=_LOG_FORMAT)
 
     except AttributeError:
-        logging.basicConfig(filename=log_path, level=logging.INFO, format=__LOG_FORMAT)
+        logging.basicConfig(filename=_LOG_PATH, level=logging.INFO, format=_LOG_FORMAT)
         logging.warning('Invalid log level "%s" in config file.', log_level_str)
-
-
-def initialize_app_config():
-    load_settings()
-    __initialize_logger()
-    logging.info('Application started!')
-
-
-def load_settings():
-    if os.path.exists(settings_path):
-        with open(settings_path, 'r') as f:
-            settings.read_file(f)
-
-
-def save_settings():
-    if os.path.exists(settings_path):
-        # Create a backup
-        copyfile(settings_path, settings_path + ".backup")
-    else:
-        # Ensure directory exists
-        settings_dir = os.path.dirname(settings_path)
-        os.makedirs(settings_dir, exist_ok=True)
-
-    with open(settings_path, 'w') as f:
-        settings.write(f)
-
-
-def get_user_config(user: User) -> ConfigParserWithEnv:
-    user_settings = UserSettings.find_by_user(user)
-    if user_settings is not None:
-        user_config = ConfigParserWithEnv(defaults=settings, allow_no_value=True)
-        user_config.read_dict({
-            'user': user_settings.to_dict()
-        })
-        return user_config
-
-    return settings

YtManagerApp/management/downloader.py

@@ -1,7 +1,7 @@
-from YtManagerApp import appconfig
+from YtManagerApp.appconfig import settings
 from YtManagerApp.management.jobs.download_video import schedule_download_video
-from YtManagerApp.models import Video, Subscription
+from YtManagerApp.models import Video, Subscription, VIDEO_ORDER_MAPPING
-from django.conf import settings
+from django.conf import settings as srv_settings
 import logging
 import requests
 import mimetypes
@@ -12,25 +12,18 @@ log = logging.getLogger('downloader')
 
 
 def __get_subscription_config(sub: Subscription):
-    user_config = appconfig.get_user_config(sub.user)
+    enabled = settings.getboolean_sub(sub, 'user', 'AutoDownload')
-
-    enabled = sub.auto_download
-    if enabled is None:
-        enabled = user_config.getboolean('user', 'AutoDownload')
 
     global_limit = -1
-    if len(user_config.get('user', 'DownloadGlobalLimit')) > 0:
+    if len(settings.get_sub(sub, 'user', 'DownloadGlobalLimit')) > 0:
-        global_limit = user_config.getint('user', 'DownloadGlobalLimit')
+        global_limit = settings.getint_sub(sub, 'user', 'DownloadGlobalLimit')
 
-    limit = sub.download_limit
-    if limit is None:
     limit = -1
-        if len(user_config.get('user', 'DownloadSubscriptionLimit')) > 0:
+    if len(settings.get_sub(sub, 'user', 'DownloadSubscriptionLimit')) > 0:
-            limit = user_config.getint('user', 'DownloadSubscriptionLimit')
+        limit = settings.getint_sub(sub, 'user', 'DownloadSubscriptionLimit')
 
-    order = sub.download_order
+    order = settings.get_sub(sub, 'user', 'DownloadOrder')
-    if order is None:
+    order = VIDEO_ORDER_MAPPING[order]
-        order = user_config.get('user', 'DownloadOrder')
 
     return enabled, global_limit, limit, order
 
@@ -88,7 +81,7 @@ def fetch_thumbnail(url, object_type, identifier, quality):
 
     # Build file path
     file_name = f"{identifier}-{quality}{ext}"
-    abs_path_dir = os.path.join(settings.MEDIA_ROOT, "thumbs", object_type)
+    abs_path_dir = os.path.join(srv_settings.MEDIA_ROOT, "thumbs", object_type)
     abs_path = os.path.join(abs_path_dir, file_name)
 
     # Store image
@@ -106,5 +99,5 @@ def fetch_thumbnail(url, object_type, identifier, quality):
         return url
 
     # Return
-    media_url = urljoin(settings.MEDIA_URL, f"thumbs/{object_type}/{file_name}")
+    media_url = urljoin(srv_settings.MEDIA_URL, f"thumbs/{object_type}/{file_name}")
     return media_url

YtManagerApp/management/jobs/download_video.py

@@ -1,6 +1,6 @@
 from YtManagerApp.models import Video
 from YtManagerApp import scheduler
-from YtManagerApp.appconfig import get_user_config
+from YtManagerApp.appconfig import settings
 import os
 import youtube_dl
 import logging
@@ -22,9 +22,9 @@ def __get_valid_path(path):
     return value
 
 
-def __build_youtube_dl_params(video: Video, user_config):
+def __build_youtube_dl_params(video: Video):
     # resolve path
-    format_dict = {
+    pattern_dict = {
         'channel': video.subscription.channel.name,
         'channel_id': video.subscription.channel.channel_id,
         'playlist': video.subscription.name,
@@ -34,22 +34,22 @@ def __build_youtube_dl_params(video: Video, user_config):
         'id': video.video_id,
     }
 
-    user_config.set_additional_interpolation_options(**format_dict)
+    download_path = settings.get_sub(video.subscription, 'user', 'DownloadPath')
+    output_pattern = __get_valid_path(settings.get_sub(
+        video.subscription, 'user', 'DownloadFilePattern', vars=pattern_dict))
 
-    download_path = user_config.get('user', 'DownloadPath')
-    output_pattern = __get_valid_path(user_config.get('user', 'DownloadFilePattern'))
     output_path = os.path.join(download_path, output_pattern)
     output_path = os.path.normpath(output_path)
 
     youtube_dl_params = {
         'logger': log_youtube_dl,
-        'format': user_config.get('user', 'DownloadFormat'),
+        'format': settings.get_sub(video.subscription, 'user', 'DownloadFormat'),
         'outtmpl': output_path,
         'writethumbnail': True,
         'writedescription': True,
-        'writesubtitles': user_config.getboolean('user', 'DownloadSubtitles'),
+        'writesubtitles': settings.getboolean_sub(video.subscription, 'user', 'DownloadSubtitles'),
-        'writeautomaticsub': user_config.getboolean('user', 'DownloadAutogeneratedSubtitles'),
+        'writeautomaticsub': settings.getboolean_sub(video.subscription, 'user', 'DownloadAutogeneratedSubtitles'),
-        'allsubtitles': user_config.getboolean('user', 'DownloadSubtitlesAll'),
+        'allsubtitles': settings.getboolean_sub(video.subscription, 'user', 'DownloadSubtitlesAll'),
         'postprocessors': [
             {
                 'key': 'FFmpegMetadata'
@@ -57,12 +57,12 @@ def __build_youtube_dl_params(video: Video, user_config):
         ]
     }
 
-    sub_langs = user_config.get('user', 'DownloadSubtitlesLangs').split(',')
+    sub_langs = settings.get_sub(video.subscription, 'user', 'DownloadSubtitlesLangs').split(',')
     sub_langs = [i.strip() for i in sub_langs]
     if len(sub_langs) > 0:
         youtube_dl_params['subtitleslangs'] = sub_langs
 
-    sub_format = user_config.get('user', 'DownloadSubtitlesFormat')
+    sub_format = settings.get_sub(video.subscription, 'user', 'DownloadSubtitlesFormat')
     if len(sub_format) > 0:
         youtube_dl_params['subtitlesformat'] = sub_format
 
@@ -73,10 +73,9 @@ def download_video(video: Video, attempt: int = 1):
 
     log.info('Downloading video %d [%s %s]', video.id, video.video_id, video.name)
 
-    user_config = get_user_config(video.subscription.user)
+    max_attempts = settings.getint_sub(video.subscription, 'user', 'DownloadMaxAttempts', fallback=3)
-    max_attempts = user_config.getint('user', 'DownloadMaxAttempts', fallback=3)
 
-    youtube_dl_params, output_path = __build_youtube_dl_params(video, user_config)
+    youtube_dl_params, output_path = __build_youtube_dl_params(video)
     with youtube_dl.YoutubeDL(youtube_dl_params) as yt:
         ret = yt.download(["https://www.youtube.com/watch?v=" + video.video_id])
 

YtManagerApp/management/jobs/synchronize.py

@@ -1,11 +1,11 @@
-from apscheduler.triggers.cron import CronTrigger
-from threading import Lock
-import os
 import errno
 import mimetypes
+from threading import Lock
+
+from apscheduler.triggers.cron import CronTrigger
 
 from YtManagerApp import scheduler
-from YtManagerApp.appconfig import settings, get_user_config
+from YtManagerApp.appconfig import settings
 from YtManagerApp.management.downloader import fetch_thumbnail, downloader_process_all, downloader_process_subscription
 from YtManagerApp.management.videos import create_video
 from YtManagerApp.models import *
@@ -14,6 +14,8 @@ from YtManagerApp.utils.youtube import YoutubeAPI
 log = logging.getLogger('sync')
 __lock = Lock()
 
+_ENABLE_UPDATE_STATS = False
+
 
 def __check_new_videos_sub(subscription: Subscription, yt_api: YoutubeAPI):
     # Get list of videos
@@ -23,6 +25,8 @@ def __check_new_videos_sub(subscription: Subscription, yt_api: YoutubeAPI):
             log.info('New video for subscription %s: %s %s"', subscription, video.getVideoId(), video.getTitle())
             db_video = create_video(video, subscription)
         else:
+            if not _ENABLE_UPDATE_STATS:
+                continue
             db_video = results.first()
 
         # Update video stats - rating and view count
@@ -33,7 +37,6 @@ def __check_new_videos_sub(subscription: Subscription, yt_api: YoutubeAPI):
 
 
 def __detect_deleted(subscription: Subscription):
-    user_settings = get_user_config(subscription.user)
 
     for video in Video.objects.filter(subscription=subscription, downloaded_path__isnull=False):
         found_video = False
@@ -63,7 +66,7 @@ def __detect_deleted(subscription: Subscription):
             video.downloaded_path = None
 
             # Mark watched?
-            if user_settings.getboolean('user', 'MarkDeletedAsWatched'):
+            if settings.getboolean_sub(subscription, 'user', 'MarkDeletedAsWatched'):
                 video.watched = True
 
             video.save()

YtManagerApp/migrations/0005_auto_20181026_2013.py

@@ -0,0 +1,134 @@
+# Generated by Django 2.1.2 on 2018-10-26 17:13
+
+from django.db import migrations, models
+import django.db.models.deletion
+import django.db.models.functions.text
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('YtManagerApp', '0004_auto_20181014_1702'),
+    ]
+
+    operations = [
+        migrations.AlterModelOptions(
+            name='subscriptionfolder',
+            options={'ordering': [django.db.models.functions.text.Lower('parent__name'), django.db.models.functions.text.Lower('name')]},
+        ),
+        migrations.AlterField(
+            model_name='subscription',
+            name='auto_download',
+            field=models.BooleanField(blank=True, null=True),
+        ),
+        migrations.AlterField(
+            model_name='subscription',
+            name='download_limit',
+            field=models.IntegerField(blank=True, null=True),
+        ),
+        migrations.AlterField(
+            model_name='subscription',
+            name='download_order',
+            field=models.CharField(blank=True, choices=[('newest', 'Newest'), ('oldest', 'Oldest'), ('playlist', 'Playlist order'), ('playlist_reverse', 'Reverse playlist order'), ('popularity', 'Popularity'), ('rating', 'Top rated')], max_length=128, null=True),
+        ),
+        migrations.AlterField(
+            model_name='subscription',
+            name='icon_best',
+            field=models.CharField(max_length=1024),
+        ),
+        migrations.AlterField(
+            model_name='subscription',
+            name='icon_default',
+            field=models.CharField(max_length=1024),
+        ),
+        migrations.AlterField(
+            model_name='subscription',
+            name='manager_delete_after_watched',
+            field=models.BooleanField(blank=True, null=True),
+        ),
+        migrations.AlterField(
+            model_name='subscription',
+            name='name',
+            field=models.CharField(max_length=1024),
+        ),
+        migrations.AlterField(
+            model_name='subscription',
+            name='parent_folder',
+            field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='YtManagerApp.SubscriptionFolder'),
+        ),
+        migrations.AlterField(
+            model_name='subscription',
+            name='playlist_id',
+            field=models.CharField(max_length=128),
+        ),
+        migrations.AlterField(
+            model_name='usersettings',
+            name='auto_download',
+            field=models.BooleanField(blank=True, help_text='Enables or disables automatic downloading.', null=True),
+        ),
+        migrations.AlterField(
+            model_name='usersettings',
+            name='delete_watched',
+            field=models.BooleanField(blank=True, help_text='Videos marked as watched are automatically deleted.', null=True),
+        ),
+        migrations.AlterField(
+            model_name='usersettings',
+            name='download_autogenerated_subtitles',
+            field=models.BooleanField(blank=True, help_text='Enables downloading the automatically generated subtitle. The flag is passed directly to youtube-dl. You can find more information <a href="https://github.com/rg3/youtube-dl/blob/master/README.md#subtitle-options">here</a>.', null=True),
+        ),
+        migrations.AlterField(
+            model_name='usersettings',
+            name='download_file_pattern',
+            field=models.CharField(blank=True, help_text='A pattern which describes how downloaded files are organized. Extensions are automatically appended. You can use the following fields, using the <code>${field}</code> syntax: channel, channel_id, playlist, playlist_id, playlist_index, title, id. Example: <code>${channel}/${playlist}/S01E${playlist_index} - ${title} [${id}]</code>', max_length=1024, null=True),
+        ),
+        migrations.AlterField(
+            model_name='usersettings',
+            name='download_format',
+            field=models.CharField(blank=True, help_text='Download format that will be passed to youtube-dl.  See the <a href="https://github.com/rg3/youtube-dl/blob/master/README.md#format-selection"> youtube-dl documentation</a> for more details.', max_length=256, null=True),
+        ),
+        migrations.AlterField(
+            model_name='usersettings',
+            name='download_global_limit',
+            field=models.IntegerField(blank=True, help_text='Limits the total number of videos downloaded (-1 = no limit).', null=True),
+        ),
+        migrations.AlterField(
+            model_name='usersettings',
+            name='download_order',
+            field=models.CharField(blank=True, choices=[('newest', 'Newest'), ('oldest', 'Oldest'), ('playlist', 'Playlist order'), ('playlist_reverse', 'Reverse playlist order'), ('popularity', 'Popularity'), ('rating', 'Top rated')], help_text='The order in which videos will be downloaded.', max_length=100, null=True),
+        ),
+        migrations.AlterField(
+            model_name='usersettings',
+            name='download_path',
+            field=models.CharField(blank=True, help_text='Path on the disk where downloaded videos are stored.  You can use environment variables using syntax: <code>${env:...}</code>', max_length=1024, null=True),
+        ),
+        migrations.AlterField(
+            model_name='usersettings',
+            name='download_subscription_limit',
+            field=models.IntegerField(blank=True, help_text='Limits the number of videos downloaded per subscription (-1 = no limit).  This setting can be overriden for each individual subscription in the subscription edit dialog.', null=True),
+        ),
+        migrations.AlterField(
+            model_name='usersettings',
+            name='download_subtitles',
+            field=models.BooleanField(blank=True, help_text='Enable downloading subtitles for the videos. The flag is passed directly to youtube-dl. You can find more information <a href="https://github.com/rg3/youtube-dl/blob/master/README.md#subtitle-options">here</a>.', null=True),
+        ),
+        migrations.AlterField(
+            model_name='usersettings',
+            name='download_subtitles_all',
+            field=models.BooleanField(blank=True, help_text='If enabled, all the subtitles in all the available languages will be downloaded. The flag is passed directly to youtube-dl. You can find more information <a href="https://github.com/rg3/youtube-dl/blob/master/README.md#subtitle-options">here</a>.', null=True),
+        ),
+        migrations.AlterField(
+            model_name='usersettings',
+            name='download_subtitles_format',
+            field=models.CharField(blank=True, help_text='Subtitles format preference. Examples: srt/ass/best The flag is passed directly to youtube-dl. You can find more information <a href="https://github.com/rg3/youtube-dl/blob/master/README.md#subtitle-options">here</a>.', max_length=100, null=True),
+        ),
+        migrations.AlterField(
+            model_name='usersettings',
+            name='download_subtitles_langs',
+            field=models.CharField(blank=True, help_text='Comma separated list of languages for which subtitles will be downloaded. The flag is passed directly to youtube-dl. You can find more information <a href="https://github.com/rg3/youtube-dl/blob/master/README.md#subtitle-options">here</a>.', max_length=250, null=True),
+        ),
+        migrations.AlterField(
+            model_name='usersettings',
+            name='mark_deleted_as_watched',
+            field=models.BooleanField(blank=True, help_text="When a downloaded video is deleted from the system, it will be marked as 'watched'.", null=True),
+        ),
+    ]

YtManagerApp/migrations/0006_auto_20181027_0256.py

@@ -0,0 +1,18 @@
+# Generated by Django 2.1.2 on 2018-10-26 23:56
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('YtManagerApp', '0005_auto_20181026_2013'),
+    ]
+
+    operations = [
+        migrations.RenameField(
+            model_name='subscription',
+            old_name='manager_delete_after_watched',
+            new_name='delete_after_watched',
+        ),
+    ]

YtManagerApp/models.py

@@ -11,6 +11,23 @@ from YtManagerApp.utils.youtube import YoutubeAPI, YoutubeChannelInfo, YoutubePl
 # help_text = user shown text
 # verbose_name = user shown name
 # null = nullable, blank = user is allowed to set value to empty
+VIDEO_ORDER_CHOICES = [
+    ('newest', 'Newest'),
+    ('oldest', 'Oldest'),
+    ('playlist', 'Playlist order'),
+    ('playlist_reverse', 'Reverse playlist order'),
+    ('popularity', 'Popularity'),
+    ('rating', 'Top rated'),
+]
+
+VIDEO_ORDER_MAPPING = {
+    'newest': '-publish_date',
+    'oldest': 'publish_date',
+    'playlist': 'playlist_index',
+    'playlist_reverse': '-playlist_index',
+    'popularity': '-views',
+    'rating': '-rating'
+}
 
 
 class UserSettings(models.Model):
@@ -40,6 +57,7 @@ class UserSettings(models.Model):
     download_order = models.CharField(
         null=True, blank=True,
         max_length=100,
+        choices=VIDEO_ORDER_CHOICES,
         help_text='The order in which videos will be downloaded.'
     )
 
@@ -300,8 +318,14 @@ class Subscription(models.Model):
     # overrides
     auto_download = models.BooleanField(null=True, blank=True)
     download_limit = models.IntegerField(null=True, blank=True)
-    download_order = models.CharField(null=True, blank=True, max_length=128)
+    download_order = models.CharField(
-    manager_delete_after_watched = models.BooleanField(null=True, blank=True)
+        null=True, blank=True,
+        max_length=128,
+        choices=VIDEO_ORDER_CHOICES)
+    delete_after_watched = models.BooleanField(null=True, blank=True)
+
+    def __str__(self):
+        return self.name
 
     def fill_from_playlist(self, info_playlist: YoutubePlaylistInfo):
         self.name = info_playlist.getTitle()
@@ -331,8 +355,17 @@ class Subscription(models.Model):
     def delete_subscription(self, keep_downloaded_videos: bool):
         self.delete()
 
-    def __str__(self):
+    def get_overloads_dict(self) -> dict:
-        return self.name
+        d = {}
+        if self.auto_download is not None:
+            d['AutoDownload'] = self.auto_download
+        if self.download_limit is not None:
+            d['DownloadSubscriptionLimit'] = self.download_limit
+        if self.download_order is not None:
+            d['DownloadOrder'] = self.download_order
+        if self.delete_after_watched is not None:
+            d['DeleteWatched'] = self.delete_after_watched
+        return d
 
 
 class Video(models.Model):
@@ -354,12 +387,11 @@ class Video(models.Model):
         self.watched = True
         self.save()
         if self.downloaded_path is not None:
-            from YtManagerApp.appconfig import get_user_config
+            from YtManagerApp.appconfig import settings
             from YtManagerApp.management.jobs.delete_video import schedule_delete_video
             from YtManagerApp.management.jobs.synchronize import schedule_synchronize_now_subscription
 
-            user_cfg = get_user_config(self.subscription.user)
+            if settings.getboolean_sub(self.subscription, 'user', 'DeleteWatched'):
-            if user_cfg.getboolean('user', 'DeleteWatched'):
                 schedule_delete_video(self)
                 schedule_synchronize_now_subscription(self.subscription)
 
@@ -379,14 +411,13 @@ class Video(models.Model):
     def delete_files(self):
         if self.downloaded_path is not None:
             from YtManagerApp.management.jobs.delete_video import schedule_delete_video
-            from YtManagerApp.appconfig import get_user_config
+            from YtManagerApp.appconfig import settings
             from YtManagerApp.management.jobs.synchronize import schedule_synchronize_now_subscription
 
             schedule_delete_video(self)
 
             # Mark watched?
-            user_cfg = get_user_config(self.subscription.user)
+            if settings.getboolean_sub(self, 'user', 'MarkDeletedAsWatched'):
-            if user_cfg.getboolean('user', 'MarkDeletedAsWatched'):
                 self.watched = True
                 schedule_synchronize_now_subscription(self.subscription)
 

YtManagerApp/utils/extended_interpolation_with_env.py

@@ -2,7 +2,7 @@ import os
 import os.path
 import re
 from configparser import Interpolation, NoSectionError, NoOptionError, InterpolationMissingOptionError, \
-    InterpolationDepthError, InterpolationSyntaxError, ConfigParser
+    InterpolationDepthError, InterpolationSyntaxError
 
 MAX_INTERPOLATION_DEPTH = 10
 
@@ -16,12 +16,6 @@ class ExtendedInterpolatorWithEnv(Interpolation):
 
     _KEYCRE = re.compile(r"\$\{([^}]+)\}")
 
-    def __init__(self, **kwargs):
-        self.__kwargs = kwargs
-
-    def set_additional_options(self, **kwargs):
-        self.__kwargs = kwargs
-
     def before_get(self, parser, section, option, value, defaults):
         L = []
         self._interpolate_some(parser, option, L, value, section, defaults, 1)
@@ -36,8 +30,6 @@ class ExtendedInterpolatorWithEnv(Interpolation):
         return value
 
     def _resolve_option(self, option, defaults):
-        if option in self.__kwargs:
-            return self.__kwargs[option]
         return defaults[option]
 
     def _resolve_section_option(self, section, option, parser):
@@ -98,17 +90,3 @@ class ExtendedInterpolatorWithEnv(Interpolation):
                     option, section,
                     "'$' must be followed by '$' or '{', "
                     "found: %r" % (rest,))
-
-
-class ConfigParserWithEnv(ConfigParser):
-    _DEFAULT_INTERPOLATION = ExtendedInterpolatorWithEnv()
-
-    def set_additional_interpolation_options(self, **kwargs):
-        """
-        Sets additional options to be used in interpolation.
-        Only works with ExtendedInterpolatorWithEnv
-        :param kwargs:
-        :return:
-        """
-        if isinstance(self._interpolation, ExtendedInterpolatorWithEnv):
-            self._interpolation.set_additional_options(**kwargs)

YtManagerApp/views/actions.py

@@ -1,19 +1,12 @@
-from django.http import HttpRequest, HttpResponseBadRequest, JsonResponse
+from django.contrib.auth.mixins import LoginRequiredMixin
-from django.shortcuts import render
+from django.http import JsonResponse
-from django import forms
+from django.views.generic import View
-from django.views.generic import CreateView, UpdateView, DeleteView, View
+
-from django.views.generic.edit import FormMixin
-from YtManagerApp.management.videos import get_videos
-from YtManagerApp.models import Subscription, SubscriptionFolder, Video
-from YtManagerApp.views.controls.modal import ModalMixin
-from crispy_forms.helper import FormHelper
-from crispy_forms.layout import Layout, Field, Div, HTML
-from django.db.models import Q
-from YtManagerApp.utils import youtube
 from YtManagerApp.management.jobs.synchronize import schedule_synchronize_now
+from YtManagerApp.models import Video
 
 
-class SyncNowView(View):
+class SyncNowView(LoginRequiredMixin, View):
     def post(self, *args, **kwargs):
         schedule_synchronize_now()
         return JsonResponse({
@@ -21,7 +14,7 @@ class SyncNowView(View):
         })
 
 
-class DeleteVideoFilesView(View):
+class DeleteVideoFilesView(LoginRequiredMixin, View):
     def post(self, *args, **kwargs):
         video = Video.objects.get(id=kwargs['pk'])
         video.delete_files()
@@ -30,7 +23,7 @@ class DeleteVideoFilesView(View):
         })
 
 
-class DownloadVideoFilesView(View):
+class DownloadVideoFilesView(LoginRequiredMixin, View):
     def post(self, *args, **kwargs):
         video = Video.objects.get(id=kwargs['pk'])
         video.download()
@@ -39,7 +32,7 @@ class DownloadVideoFilesView(View):
         })
 
 
-class MarkVideoWatchedView(View):
+class MarkVideoWatchedView(LoginRequiredMixin, View):
     def post(self, *args, **kwargs):
         video = Video.objects.get(id=kwargs['pk'])
         video.mark_watched()
@@ -48,7 +41,7 @@ class MarkVideoWatchedView(View):
         })
 
 
-class MarkVideoUnwatchedView(View):
+class MarkVideoUnwatchedView(LoginRequiredMixin, View):
     def post(self, *args, **kwargs):
         video = Video.objects.get(id=kwargs['pk'])
         video.mark_unwatched()

YtManagerApp/views/auth.py

@@ -3,6 +3,7 @@ from crispy_forms.layout import Submit
 from django import forms
 from django.contrib.auth import login, authenticate
 from django.contrib.auth.forms import AuthenticationForm, UserCreationForm
+from django.contrib.auth.mixins import LoginRequiredMixin
 from django.contrib.auth.models import User
 from django.contrib.auth.views import LoginView
 from django.urls import reverse_lazy
@@ -78,5 +79,5 @@ class RegisterView(FormView):
         return context
 
 
-class RegisterDoneView(TemplateView):
+class RegisterDoneView(LoginRequiredMixin, TemplateView):
     template_name = 'registration/register_done.html'

YtManagerApp/views/index.py

@@ -1,6 +1,8 @@
 from crispy_forms.helper import FormHelper
 from crispy_forms.layout import Layout, Field, HTML
 from django import forms
+from django.contrib.auth.decorators import login_required
+from django.contrib.auth.mixins import LoginRequiredMixin
 from django.db.models import Q
 from django.http import HttpRequest, HttpResponseBadRequest, JsonResponse
 from django.shortcuts import render
@@ -8,31 +10,12 @@ from django.views.generic import CreateView, UpdateView, DeleteView
 from django.views.generic.edit import FormMixin
 
 from YtManagerApp.management.videos import get_videos
-from YtManagerApp.models import Subscription, SubscriptionFolder
+from YtManagerApp.models import Subscription, SubscriptionFolder, VIDEO_ORDER_CHOICES, VIDEO_ORDER_MAPPING
 from YtManagerApp.utils import youtube
 from YtManagerApp.views.controls.modal import ModalMixin
 
 
 class VideoFilterForm(forms.Form):
-    CHOICES_SORT = (
-        ('newest', 'Newest'),
-        ('oldest', 'Oldest'),
-        ('playlist', 'Playlist order'),
-        ('playlist_reverse', 'Reverse playlist order'),
-        ('popularity', 'Popularity'),
-        ('rating', 'Top rated'),
-    )
-
-    # Map select values to actual column names
-    MAPPING_SORT = {
-        'newest': '-publish_date',
-        'oldest': 'publish_date',
-        'playlist': 'playlist_index',
-        'playlist_reverse': '-playlist_index',
-        'popularity': '-views',
-        'rating': '-rating'
-    }
-
     CHOICES_SHOW_WATCHED = (
         ('y', 'Watched'),
         ('n', 'Not watched'),
@@ -52,7 +35,7 @@ class VideoFilterForm(forms.Form):
     }
 
     query = forms.CharField(label='', required=False)
-    sort = forms.ChoiceField(label='Sort:', choices=CHOICES_SORT, initial='newest')
+    sort = forms.ChoiceField(label='Sort:', choices=VIDEO_ORDER_CHOICES, initial='newest')
     show_watched = forms.ChoiceField(label='Show only: ', choices=CHOICES_SHOW_WATCHED, initial='all')
     show_downloaded = forms.ChoiceField(label='', choices=CHOICES_SHOW_DOWNLOADED, initial='all')
     subscription_id = forms.IntegerField(
@@ -85,7 +68,7 @@ class VideoFilterForm(forms.Form):
 
     def clean_sort(self):
         data = self.cleaned_data['sort']
-        return VideoFilterForm.MAPPING_SORT[data]
+        return VIDEO_ORDER_MAPPING[data]
 
     def clean_show_downloaded(self):
         data = self.cleaned_data['show_downloaded']
@@ -118,6 +101,7 @@ def index(request: HttpRequest):
         return render(request, 'YtManagerApp/index_unauthenticated.html')
 
 
+@login_required
 def ajax_get_tree(request: HttpRequest):
 
     def visit(node):
@@ -142,6 +126,7 @@ def ajax_get_tree(request: HttpRequest):
     return JsonResponse(result, safe=False)
 
 
+@login_required
 def ajax_get_videos(request: HttpRequest):
     if request.method == 'POST':
         form = VideoFilterForm(request.POST)
@@ -206,7 +191,7 @@ class SubscriptionFolderForm(forms.ModelForm):
             current = current.parent
 
 
-class CreateFolderModal(ModalMixin, CreateView):
+class CreateFolderModal(LoginRequiredMixin, ModalMixin, CreateView):
     template_name = 'YtManagerApp/controls/folder_create_modal.html'
     form_class = SubscriptionFolderForm
 
@@ -215,7 +200,7 @@ class CreateFolderModal(ModalMixin, CreateView):
         return super().form_valid(form)
 
 
-class UpdateFolderModal(ModalMixin, UpdateView):
+class UpdateFolderModal(LoginRequiredMixin, ModalMixin, UpdateView):
     template_name = 'YtManagerApp/controls/folder_update_modal.html'
     model = SubscriptionFolder
     form_class = SubscriptionFolderForm
@@ -225,7 +210,7 @@ class DeleteFolderForm(forms.Form):
     keep_subscriptions = forms.BooleanField(required=False, initial=False, label="Keep subscriptions")
 
 
-class DeleteFolderModal(ModalMixin, FormMixin, DeleteView):
+class DeleteFolderModal(LoginRequiredMixin, ModalMixin, FormMixin, DeleteView):
     template_name = 'YtManagerApp/controls/folder_delete_modal.html'
     model = SubscriptionFolder
     form_class = DeleteFolderForm
@@ -248,7 +233,8 @@ class CreateSubscriptionForm(forms.ModelForm):
 
     class Meta:
         model = Subscription
-        fields = ['parent_folder']
+        fields = ['parent_folder', 'auto_download',
+                  'download_limit', 'download_order', 'delete_after_watched']
 
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
@@ -256,7 +242,13 @@ class CreateSubscriptionForm(forms.ModelForm):
         self.helper.form_tag = False
         self.helper.layout = Layout(
             'playlist_url',
-            'parent_folder'
+            'parent_folder',
+            HTML('<hr>'),
+            HTML('<h5>Download configuration overloads</h5>'),
+            'auto_download',
+            'download_limit',
+            'download_order',
+            'delete_after_watched'
         )
 
     def clean_playlist_url(self):
@@ -268,7 +260,7 @@ class CreateSubscriptionForm(forms.ModelForm):
         return playlist_url
 
 
-class CreateSubscriptionModal(ModalMixin, CreateView):
+class CreateSubscriptionModal(LoginRequiredMixin, ModalMixin, CreateView):
     template_name = 'YtManagerApp/controls/subscription_create_modal.html'
     form_class = CreateSubscriptionForm
 
@@ -300,7 +292,7 @@ class UpdateSubscriptionForm(forms.ModelForm):
     class Meta:
         model = Subscription
         fields = ['name', 'parent_folder', 'auto_download',
-                  'download_limit', 'download_order', 'manager_delete_after_watched']
+                  'download_limit', 'download_order', 'delete_after_watched']
 
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
@@ -314,11 +306,11 @@ class UpdateSubscriptionForm(forms.ModelForm):
             'auto_download',
             'download_limit',
             'download_order',
-            'manager_delete_after_watched'
+            'delete_after_watched'
         )
 
 
-class UpdateSubscriptionModal(ModalMixin, UpdateView):
+class UpdateSubscriptionModal(LoginRequiredMixin, ModalMixin, UpdateView):
     template_name = 'YtManagerApp/controls/subscription_update_modal.html'
     model = Subscription
     form_class = UpdateSubscriptionForm
@@ -328,7 +320,7 @@ class DeleteSubscriptionForm(forms.Form):
     keep_downloaded_videos = forms.BooleanField(required=False, initial=False, label="Keep downloaded videos")
 
 
-class DeleteSubscriptionModal(ModalMixin, FormMixin, DeleteView):
+class DeleteSubscriptionModal(LoginRequiredMixin, ModalMixin, FormMixin, DeleteView):
     template_name = 'YtManagerApp/controls/subscription_delete_modal.html'
     model = Subscription
     form_class = DeleteSubscriptionForm

YtManagerApp/views/settings.py

@@ -1,8 +1,9 @@
 from crispy_forms.helper import FormHelper
 from crispy_forms.layout import Layout, HTML, Submit
 from django import forms
-from django.views.generic import UpdateView
+from django.contrib.auth.mixins import LoginRequiredMixin
 from django.urls import reverse_lazy
+from django.views.generic import UpdateView
 
 from YtManagerApp.models import UserSettings
 
@@ -39,7 +40,7 @@ class SettingsForm(forms.ModelForm):
         )
 
 
-class SettingsView(UpdateView):
+class SettingsView(LoginRequiredMixin, UpdateView):
     form_class = SettingsForm
     model = UserSettings
     template_name = 'YtManagerApp/settings.html'

config/config.ini

@@ -2,16 +2,16 @@
 ; The global section contains settings that apply to the entire server
 [global]
 ; YouTube API key - get this from your user account
-YoutubeApiKey=AIzaSyBabzE4Bup77WexdLMa9rN9z-wJidEfNX8
+;YoutubeApiKey=AIzaSyBabzE4Bup77WexdLMa9rN9z-wJidEfNX8
 
 ; Specifies the synchronization schedule, in crontab format.
 ; Format: <minute> <hour> <day-of-month> <month-of-year> <day of week>
-SynchronizationSchedule=0 * * * *
+;SynchronizationSchedule=0 * * * *
 
 ; Number of threads running the scheduler
 ; Since most of the jobs scheduled are downloads, there is no advantage to having
 ; a higher concurrency
-SchedulerConcurrency=2
+;SchedulerConcurrency=2
 
 ; Log level
 LogLevel=DEBUG
@@ -19,43 +19,41 @@ LogLevel=DEBUG
 ; Default user settings
 [user]
 ; When a video is deleted on the system, it will be marked as 'watched'
-MarkDeletedAsWatched=True
+;MarkDeletedAsWatched=True
 
 ; Videos marked as watched are automatically deleted
-DeleteWatched=True
+;DeleteWatched=True
 
 ; Enable automatic downloading
-AutoDownload=True
+;AutoDownload=True
 
 ; Limit the total number of videos downloaded (-1 or empty = no limit)
-DownloadGlobalLimit=
+;DownloadGlobalLimit=
 
 ; Limit the numbers of videos per subscription (-1 or empty = no limit)
-DownloadSubscriptionLimit=5
+;DownloadSubscriptionLimit=5
 
 ; Number of download attempts
-DownloadMaxAttempts=3
+;DownloadMaxAttempts=3
 
 ; Download order
-; Options: playlist_index, publish_date, name.
+; Options: newest, oldest, playlist, playlist_reverse, popularity, rating
-; Use - to reverse order (e.g. -publish_date means to order by publish date descending)
+;DownloadOrder=playlist
-DownloadOrder=playlist_index
 
 ; Path where downloaded videos are stored
-;DownloadPath=${env:USERPROFILE}${env:HOME}/Downloads
 DownloadPath=D:\\Dev\\youtube-channel-manager\\temp\\download
 
 ; A pattern which describes how downloaded files are organized. Extensions are automatically appended.
 ; Supported fields: channel, channel_id, playlist, playlist_id, playlist_index, title, id
 ; The default pattern should work pretty well with Plex
-DownloadFilePattern=${channel}/${playlist}/S01E${playlist_index} - ${title} [${id}]
+;DownloadFilePattern=${channel}/${playlist}/S01E${playlist_index} - ${title} [${id}]
 
 ; Download format that will be passed to youtube-dl. See the youtube-dl documentation for more details.
-DownloadFormat=bestvideo+bestaudio
+DownloadFormat=worstvideo+bestaudio
 
 ; Subtitles - these options match the youtube-dl options
-DownloadSubtitles=True
+;DownloadSubtitles=True
-DownloadAutogeneratedSubtitles=False
+;DownloadAutogeneratedSubtitles=False
-DownloadSubtitlesAll=False
+;DownloadSubtitlesAll=False
-DownloadSubtitlesLangs=en,ro
+;DownloadSubtitlesLangs=en,ro
-DownloadSubtitlesFormat=
+;DownloadSubtitlesFormat=

config/defaults.ini

@@ -14,7 +14,7 @@ SynchronizationSchedule=0 * * * *
 SchedulerConcurrency=2
 
 ; Log level
-LogLevel=DEBUG
+LogLevel=INFO
 
 ; Default user settings
 [user]
@@ -37,13 +37,11 @@ DownloadSubscriptionLimit=5
 DownloadMaxAttempts=3
 
 ; Download order
-; Options: playlist_index, publish_date, name.
+; Options: newest, oldest, playlist, playlist_reverse, popularity, rating
-; Use - to reverse order (e.g. -publish_date means to order by publish date descending)
+DownloadOrder=playlist
-DownloadOrder=playlist_index
 
 ; Path where downloaded videos are stored
-;DownloadPath=${env:USERPROFILE}${env:HOME}/Downloads
+DownloadPath=${env:USERPROFILE}${env:HOME}/Downloads
-DownloadPath=D:\\Dev\\youtube-channel-manager\\temp\\download
 
 ; A pattern which describes how downloaded files are organized. Extensions are automatically appended.
 ; Supported fields: channel, channel_id, playlist, playlist_id, playlist_index, title, id