chibicitiberiu/ytsm
1
0
Fork 0
mirror of https://github.com/chibicitiberiu/ytsm.git synced 2024-02-24 05:43:31 +00:00
Code Issues Releases Wiki Activity

Fixed multiple issues in configparser usage and implementation. Added dropdowns for sort order, and used the values everywhere.

Browse Source
This commit is contained in:
Tiberiu Chibici 2018-10-27 03:33:45 +03:00
parent 58baf16802
commit 3da026dbe6
16 changed files with 849 additions and 685 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

917
.idea/workspace.xml generated
View File

File diff suppressed because it is too large Load Diff

1
YtManager/settings.py
View File

@ -129,3 +129,4 @@ MEDIA_ROOT = 'D:\\Dev\\youtube-channel-manager\\temp\\media'
CRISPY_TEMPLATE_PACK = 'bootstrap4'
LOGIN_REDIRECT_URL = '/'
LOGIN_URL = '/login'

173
YtManagerApp/appconfig.py
View File

@ -1,46 +1,114 @@
import logging
import os
import os.path
from collections import ChainMap
from configparser import ConfigParser
from shutil import copyfile
from typing import Optional, Any
from django.conf import settings as dj_settings
from django.contrib.auth.models import User
from .models import UserSettings
from .utils.customconfigparser import ConfigParserWithEnv
from .models import UserSettings, Subscription
from .utils.extended_interpolation_with_env import ExtendedInterpolatorWithEnv
__SETTINGS_FILE = 'config.ini'
__LOG_FILE = 'log.log'
__LOG_FORMAT = '%(asctime)s|%(process)d|%(thread)d|%(name)s|%(filename)s|%(lineno)d|%(levelname)s|%(message)s'
_CONFIG_DIR = os.path.join(dj_settings.BASE_DIR, 'config')
_LOG_FILE = 'log.log'
_LOG_PATH = os.path.join(_CONFIG_DIR, _LOG_FILE)
_LOG_FORMAT = '%(asctime)s|%(process)d|%(thread)d|%(name)s|%(filename)s|%(lineno)d|%(levelname)s|%(message)s'
__DEFAULT_SETTINGS = {
'global': {
'YouTubeApiKey': 'AIzaSyBabzE4Bup77WexdLMa9rN9z-wJidEfNX8',
'SynchronizationSchedule': '0 * * * *',
'SchedulerConcurrency': '2',
},
'user': {
'MarkDeletedAsWatched': 'True',
'DeleteWatched': 'True',
'AutoDownload': 'True',
'DownloadMaxAttempts': '3',
'DownloadGlobalLimit': '',
'DownloadSubscriptionLimit': '5',
'DownloadOrder': 'playlist_index',
'DownloadPath': '${env:USERPROFILE}${env:HOME}/Downloads',
'DownloadFilePattern': '${channel}/${playlist}/S01E${playlist_index} - ${title} [${id}]',
'DownloadFormat': 'bestvideo+bestaudio',
'DownloadSubtitles': 'True',
'DownloadAutogeneratedSubtitles': 'False',
'DownloadSubtitlesAll': 'False',
'DownloadSubtitlesLangs': 'en,ro',
'DownloadSubtitlesFormat': '',
}
}
log_path = os.path.join(dj_settings.BASE_DIR, 'config', __LOG_FILE)
settings_path = os.path.join(dj_settings.BASE_DIR, 'config', __SETTINGS_FILE)
settings = ConfigParserWithEnv(defaults=__DEFAULT_SETTINGS, allow_no_value=True)
class AppSettings(ConfigParser):
_DEFAULT_INTERPOLATION = ExtendedInterpolatorWithEnv()
__DEFAULTS_FILE = 'defaults.ini'
__SETTINGS_FILE = 'config.ini'
def __init__(self, *args, **kwargs):
super().__init__(allow_no_value=True, *args, **kwargs)
self.__defaults_path = os.path.join(_CONFIG_DIR, AppSettings.__DEFAULTS_FILE)
self.__settings_path = os.path.join(_CONFIG_DIR, AppSettings.__SETTINGS_FILE)
def initialize(self):
self.read([self.__defaults_path, self.__settings_path])
def save(self):
if os.path.exists(self.__settings_path):
# Create a backup
copyfile(self.__settings_path, self.__settings_path + ".backup")
else:
# Ensure directory exists
settings_dir = os.path.dirname(self.__settings_path)
os.makedirs(settings_dir, exist_ok=True)
with open(self.__settings_path, 'w') as f:
self.write(f)
def __get_combined_dict(self, vars: Optional[Any], sub: Optional[Subscription], user: Optional[User]) -> ChainMap:
vars_dict = {}
sub_overloads_dict = {}
user_settings_dict = {}
if vars is not None:
vars_dict = vars
if sub is not None:
sub_overloads_dict = sub.get_overloads_dict()
if user is not None:
user_settings = UserSettings.find_by_user(user)
if user_settings is not None:
user_settings_dict = user_settings.to_dict()
return ChainMap(vars_dict, sub_overloads_dict, user_settings_dict)
def get_user(self, user: User, section: str, option: Any, vars=None, fallback=object()) -> str:
return super().get(section, option,
fallback=fallback,
vars=self.__get_combined_dict(vars, None, user))
def getboolean_user(self, user: User, section: str, option: Any, vars=None, fallback=object()) -> bool:
return super().getboolean(section, option,
fallback=fallback,
vars=self.__get_combined_dict(vars, None, user))
def getint_user(self, user: User, section: str, option: Any, vars=None, fallback=object()) -> int:
return super().getint(section, option,
fallback=fallback,
vars=self.__get_combined_dict(vars, None, user))
def getfloat_user(self, user: User, section: str, option: Any, vars=None, fallback=object()) -> float:
return super().getfloat(section, option,
fallback=fallback,
vars=self.__get_combined_dict(vars, None, user))
def get_sub(self, sub: Subscription, section: str, option: Any, vars=None, fallback=object()) -> str:
return super().get(section, option,
fallback=fallback,
vars=self.__get_combined_dict(vars, sub, sub.user))
def getboolean_sub(self, sub: Subscription, section: str, option: Any, vars=None, fallback=object()) -> bool:
return super().getboolean(section, option,
fallback=fallback,
vars=self.__get_combined_dict(vars, sub, sub.user))
def getint_sub(self, sub: Subscription, section: str, option: Any, vars=None, fallback=object()) -> int:
return super().getint(section, option,
fallback=fallback,
vars=self.__get_combined_dict(vars, sub, sub.user))
def getfloat_sub(self, sub: Subscription, section: str, option: Any, vars=None, fallback=object()) -> float:
return super().getfloat(section, option,
fallback=fallback,
vars=self.__get_combined_dict(vars, sub, sub.user))
settings = AppSettings()
def initialize_app_config():
settings.initialize()
__initialize_logger()
logging.info('Application started!')
def __initialize_logger():
@ -48,45 +116,8 @@ def __initialize_logger():
try:
log_level = getattr(logging, log_level_str)
logging.basicConfig(filename=log_path, level=log_level, format=__LOG_FORMAT)
logging.basicConfig(filename=_LOG_PATH, level=log_level, format=_LOG_FORMAT)
except AttributeError:
logging.basicConfig(filename=log_path, level=logging.INFO, format=__LOG_FORMAT)
logging.basicConfig(filename=_LOG_PATH, level=logging.INFO, format=_LOG_FORMAT)
logging.warning('Invalid log level "%s" in config file.', log_level_str)
def initialize_app_config():
load_settings()
__initialize_logger()
logging.info('Application started!')
def load_settings():
if os.path.exists(settings_path):
with open(settings_path, 'r') as f:
settings.read_file(f)
def save_settings():
if os.path.exists(settings_path):
# Create a backup
copyfile(settings_path, settings_path + ".backup")
else:
# Ensure directory exists
settings_dir = os.path.dirname(settings_path)
os.makedirs(settings_dir, exist_ok=True)
with open(settings_path, 'w') as f:
settings.write(f)
def get_user_config(user: User) -> ConfigParserWithEnv:
user_settings = UserSettings.find_by_user(user)
if user_settings is not None:
user_config = ConfigParserWithEnv(defaults=settings, allow_no_value=True)
user_config.read_dict({
'user': user_settings.to_dict()
})
return user_config
return settings

31
YtManagerApp/management/downloader.py
View File

@ -1,7 +1,7 @@
from YtManagerApp import appconfig
from YtManagerApp.appconfig import settings
from YtManagerApp.management.jobs.download_video import schedule_download_video
from YtManagerApp.models import Video, Subscription
from django.conf import settings
from YtManagerApp.models import Video, Subscription, VIDEO_ORDER_MAPPING
from django.conf import settings as srv_settings
import logging
import requests
import mimetypes
@ -12,25 +12,18 @@ log = logging.getLogger('downloader')
def __get_subscription_config(sub: Subscription):
user_config = appconfig.get_user_config(sub.user)
enabled = sub.auto_download
if enabled is None:
enabled = user_config.getboolean('user', 'AutoDownload')
enabled = settings.getboolean_sub(sub, 'user', 'AutoDownload')
global_limit = -1
if len(user_config.get('user', 'DownloadGlobalLimit')) > 0:
global_limit = user_config.getint('user', 'DownloadGlobalLimit')
if len(settings.get_sub(sub, 'user', 'DownloadGlobalLimit')) > 0:
global_limit = settings.getint_sub(sub, 'user', 'DownloadGlobalLimit')
limit = sub.download_limit
if limit is None:
limit = -1
if len(user_config.get('user', 'DownloadSubscriptionLimit')) > 0:
limit = user_config.getint('user', 'DownloadSubscriptionLimit')
if len(settings.get_sub(sub, 'user', 'DownloadSubscriptionLimit')) > 0:
limit = settings.getint_sub(sub, 'user', 'DownloadSubscriptionLimit')
order = sub.download_order
if order is None:
order = user_config.get('user', 'DownloadOrder')
order = settings.get_sub(sub, 'user', 'DownloadOrder')
order = VIDEO_ORDER_MAPPING[order]
return enabled, global_limit, limit, order
@ -88,7 +81,7 @@ def fetch_thumbnail(url, object_type, identifier, quality):
# Build file path
file_name = f"{identifier}-{quality}{ext}"
abs_path_dir = os.path.join(settings.MEDIA_ROOT, "thumbs", object_type)
abs_path_dir = os.path.join(srv_settings.MEDIA_ROOT, "thumbs", object_type)
abs_path = os.path.join(abs_path_dir, file_name)
# Store image
@ -106,5 +99,5 @@ def fetch_thumbnail(url, object_type, identifier, quality):
return url
# Return
media_url = urljoin(settings.MEDIA_URL, f"thumbs/{object_type}/{file_name}")
media_url = urljoin(srv_settings.MEDIA_URL, f"thumbs/{object_type}/{file_name}")
return media_url

29
YtManagerApp/management/jobs/download_video.py
View File

@ -1,6 +1,6 @@
from YtManagerApp.models import Video
from YtManagerApp import scheduler
from YtManagerApp.appconfig import get_user_config
from YtManagerApp.appconfig import settings
import os
import youtube_dl
import logging
@ -22,9 +22,9 @@ def __get_valid_path(path):
return value
def __build_youtube_dl_params(video: Video, user_config):
def __build_youtube_dl_params(video: Video):
# resolve path
format_dict = {
pattern_dict = {
'channel': video.subscription.channel.name,
'channel_id': video.subscription.channel.channel_id,
'playlist': video.subscription.name,
@ -34,22 +34,22 @@ def __build_youtube_dl_params(video: Video, user_config):
'id': video.video_id,
}
user_config.set_additional_interpolation_options(**format_dict)
download_path = settings.get_sub(video.subscription, 'user', 'DownloadPath')
output_pattern = __get_valid_path(settings.get_sub(
video.subscription, 'user', 'DownloadFilePattern', vars=pattern_dict))
download_path = user_config.get('user', 'DownloadPath')
output_pattern = __get_valid_path(user_config.get('user', 'DownloadFilePattern'))
output_path = os.path.join(download_path, output_pattern)
output_path = os.path.normpath(output_path)
youtube_dl_params = {
'logger': log_youtube_dl,
'format': user_config.get('user', 'DownloadFormat'),
'format': settings.get_sub(video.subscription, 'user', 'DownloadFormat'),
'outtmpl': output_path,
'writethumbnail': True,
'writedescription': True,
'writesubtitles': user_config.getboolean('user', 'DownloadSubtitles'),
'writeautomaticsub': user_config.getboolean('user', 'DownloadAutogeneratedSubtitles'),
'allsubtitles': user_config.getboolean('user', 'DownloadSubtitlesAll'),
'writesubtitles': settings.getboolean_sub(video.subscription, 'user', 'DownloadSubtitles'),
'writeautomaticsub': settings.getboolean_sub(video.subscription, 'user', 'DownloadAutogeneratedSubtitles'),
'allsubtitles': settings.getboolean_sub(video.subscription, 'user', 'DownloadSubtitlesAll'),
'postprocessors': [
{
'key': 'FFmpegMetadata'
@ -57,12 +57,12 @@ def __build_youtube_dl_params(video: Video, user_config):
]
}
sub_langs = user_config.get('user', 'DownloadSubtitlesLangs').split(',')
sub_langs = settings.get_sub(video.subscription, 'user', 'DownloadSubtitlesLangs').split(',')
sub_langs = [i.strip() for i in sub_langs]
if len(sub_langs) > 0:
youtube_dl_params['subtitleslangs'] = sub_langs
sub_format = user_config.get('user', 'DownloadSubtitlesFormat')
sub_format = settings.get_sub(video.subscription, 'user', 'DownloadSubtitlesFormat')
if len(sub_format) > 0:
youtube_dl_params['subtitlesformat'] = sub_format
@ -73,10 +73,9 @@ def download_video(video: Video, attempt: int = 1):
log.info('Downloading video %d [%s %s]', video.id, video.video_id, video.name)
user_config = get_user_config(video.subscription.user)
max_attempts = user_config.getint('user', 'DownloadMaxAttempts', fallback=3)
max_attempts = settings.getint_sub(video.subscription, 'user', 'DownloadMaxAttempts', fallback=3)
youtube_dl_params, output_path = __build_youtube_dl_params(video, user_config)
youtube_dl_params, output_path = __build_youtube_dl_params(video)
with youtube_dl.YoutubeDL(youtube_dl_params) as yt:
ret = yt.download(["https://www.youtube.com/watch?v=" + video.video_id])

15
YtManagerApp/management/jobs/synchronize.py
View File

@ -1,11 +1,11 @@
from apscheduler.triggers.cron import CronTrigger
from threading import Lock
import os
import errno
import mimetypes
from threading import Lock
from apscheduler.triggers.cron import CronTrigger
from YtManagerApp import scheduler
from YtManagerApp.appconfig import settings, get_user_config
from YtManagerApp.appconfig import settings
from YtManagerApp.management.downloader import fetch_thumbnail, downloader_process_all, downloader_process_subscription
from YtManagerApp.management.videos import create_video
from YtManagerApp.models import *
@ -14,6 +14,8 @@ from YtManagerApp.utils.youtube import YoutubeAPI
log = logging.getLogger('sync')
__lock = Lock()
_ENABLE_UPDATE_STATS = False
def __check_new_videos_sub(subscription: Subscription, yt_api: YoutubeAPI):
# Get list of videos
@ -23,6 +25,8 @@ def __check_new_videos_sub(subscription: Subscription, yt_api: YoutubeAPI):
log.info('New video for subscription %s: %s %s"', subscription, video.getVideoId(), video.getTitle())
db_video = create_video(video, subscription)
else:
if not _ENABLE_UPDATE_STATS:
continue
db_video = results.first()
# Update video stats - rating and view count
@ -33,7 +37,6 @@ def __check_new_videos_sub(subscription: Subscription, yt_api: YoutubeAPI):
def __detect_deleted(subscription: Subscription):
user_settings = get_user_config(subscription.user)
for video in Video.objects.filter(subscription=subscription, downloaded_path__isnull=False):
found_video = False
@ -63,7 +66,7 @@ def __detect_deleted(subscription: Subscription):
video.downloaded_path = None
# Mark watched?
if user_settings.getboolean('user', 'MarkDeletedAsWatched'):
if settings.getboolean_sub(subscription, 'user', 'MarkDeletedAsWatched'):
video.watched = True
video.save()

134
YtManagerApp/migrations/0005_auto_20181026_2013.py Normal file
View File

@ -0,0 +1,134 @@
# Generated by Django 2.1.2 on 2018-10-26 17:13
from django.db import migrations, models
import django.db.models.deletion
import django.db.models.functions.text
class Migration(migrations.Migration):
dependencies = [
('YtManagerApp', '0004_auto_20181014_1702'),
]
operations = [
migrations.AlterModelOptions(
name='subscriptionfolder',
options={'ordering': [django.db.models.functions.text.Lower('parent__name'), django.db.models.functions.text.Lower('name')]},
),
migrations.AlterField(
model_name='subscription',
name='auto_download',
field=models.BooleanField(blank=True, null=True),
),
migrations.AlterField(
model_name='subscription',
name='download_limit',
field=models.IntegerField(blank=True, null=True),
),
migrations.AlterField(
model_name='subscription',
name='download_order',
field=models.CharField(blank=True, choices=[('newest', 'Newest'), ('oldest', 'Oldest'), ('playlist', 'Playlist order'), ('playlist_reverse', 'Reverse playlist order'), ('popularity', 'Popularity'), ('rating', 'Top rated')], max_length=128, null=True),
),
migrations.AlterField(
model_name='subscription',
name='icon_best',
field=models.CharField(max_length=1024),
),
migrations.AlterField(
model_name='subscription',
name='icon_default',
field=models.CharField(max_length=1024),
),
migrations.AlterField(
model_name='subscription',
name='manager_delete_after_watched',
field=models.BooleanField(blank=True, null=True),
),
migrations.AlterField(
model_name='subscription',
name='name',
field=models.CharField(max_length=1024),
),
migrations.AlterField(
model_name='subscription',
name='parent_folder',
field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='YtManagerApp.SubscriptionFolder'),
),
migrations.AlterField(
model_name='subscription',
name='playlist_id',
field=models.CharField(max_length=128),
),
migrations.AlterField(
model_name='usersettings',
name='auto_download',
field=models.BooleanField(blank=True, help_text='Enables or disables automatic downloading.', null=True),
),
migrations.AlterField(
model_name='usersettings',
name='delete_watched',
field=models.BooleanField(blank=True, help_text='Videos marked as watched are automatically deleted.', null=True),
),
migrations.AlterField(
model_name='usersettings',
name='download_autogenerated_subtitles',
field=models.BooleanField(blank=True, help_text='Enables downloading the automatically generated subtitle. The flag is passed directly to youtube-dl. You can find more information <a href="https://github.com/rg3/youtube-dl/blob/master/README.md#subtitle-options">here</a>.', null=True),
),
migrations.AlterField(
model_name='usersettings',
name='download_file_pattern',
field=models.CharField(blank=True, help_text='A pattern which describes how downloaded files are organized. Extensions are automatically appended. You can use the following fields, using the <code>${field}</code> syntax: channel, channel_id, playlist, playlist_id, playlist_index, title, id. Example: <code>${channel}/${playlist}/S01E${playlist_index} - ${title} [${id}]</code>', max_length=1024, null=True),
),
migrations.AlterField(
model_name='usersettings',
name='download_format',
field=models.CharField(blank=True, help_text='Download format that will be passed to youtube-dl. See the <a href="https://github.com/rg3/youtube-dl/blob/master/README.md#format-selection"> youtube-dl documentation</a> for more details.', max_length=256, null=True),
),
migrations.AlterField(
model_name='usersettings',
name='download_global_limit',
field=models.IntegerField(blank=True, help_text='Limits the total number of videos downloaded (-1 = no limit).', null=True),
),
migrations.AlterField(
model_name='usersettings',
name='download_order',
field=models.CharField(blank=True, choices=[('newest', 'Newest'), ('oldest', 'Oldest'), ('playlist', 'Playlist order'), ('playlist_reverse', 'Reverse playlist order'), ('popularity', 'Popularity'), ('rating', 'Top rated')], help_text='The order in which videos will be downloaded.', max_length=100, null=True),
),
migrations.AlterField(
model_name='usersettings',
name='download_path',
field=models.CharField(blank=True, help_text='Path on the disk where downloaded videos are stored. You can use environment variables using syntax: <code>${env:...}</code>', max_length=1024, null=True),
),
migrations.AlterField(
model_name='usersettings',
name='download_subscription_limit',
field=models.IntegerField(blank=True, help_text='Limits the number of videos downloaded per subscription (-1 = no limit). This setting can be overriden for each individual subscription in the subscription edit dialog.', null=True),
),
migrations.AlterField(
model_name='usersettings',
name='download_subtitles',
field=models.BooleanField(blank=True, help_text='Enable downloading subtitles for the videos. The flag is passed directly to youtube-dl. You can find more information <a href="https://github.com/rg3/youtube-dl/blob/master/README.md#subtitle-options">here</a>.', null=True),
),
migrations.AlterField(
model_name='usersettings',
name='download_subtitles_all',
field=models.BooleanField(blank=True, help_text='If enabled, all the subtitles in all the available languages will be downloaded. The flag is passed directly to youtube-dl. You can find more information <a href="https://github.com/rg3/youtube-dl/blob/master/README.md#subtitle-options">here</a>.', null=True),
),
migrations.AlterField(
model_name='usersettings',
name='download_subtitles_format',
field=models.CharField(blank=True, help_text='Subtitles format preference. Examples: srt/ass/best The flag is passed directly to youtube-dl. You can find more information <a href="https://github.com/rg3/youtube-dl/blob/master/README.md#subtitle-options">here</a>.', max_length=100, null=True),
),
migrations.AlterField(
model_name='usersettings',
name='download_subtitles_langs',
field=models.CharField(blank=True, help_text='Comma separated list of languages for which subtitles will be downloaded. The flag is passed directly to youtube-dl. You can find more information <a href="https://github.com/rg3/youtube-dl/blob/master/README.md#subtitle-options">here</a>.', max_length=250, null=True),
),
migrations.AlterField(
model_name='usersettings',
name='mark_deleted_as_watched',
field=models.BooleanField(blank=True, help_text="When a downloaded video is deleted from the system, it will be marked as 'watched'.", null=True),
),
]

18
YtManagerApp/migrations/0006_auto_20181027_0256.py Normal file
View File

@ -0,0 +1,18 @@
# Generated by Django 2.1.2 on 2018-10-26 23:56
from django.db import migrations
class Migration(migrations.Migration):
dependencies = [
('YtManagerApp', '0005_auto_20181026_2013'),
]
operations = [
migrations.RenameField(
model_name='subscription',
old_name='manager_delete_after_watched',
new_name='delete_after_watched',
),
]

51
YtManagerApp/models.py
View File

@ -11,6 +11,23 @@ from YtManagerApp.utils.youtube import YoutubeAPI, YoutubeChannelInfo, YoutubePl
# help_text = user shown text
# verbose_name = user shown name
# null = nullable, blank = user is allowed to set value to empty
VIDEO_ORDER_CHOICES = [
('newest', 'Newest'),
('oldest', 'Oldest'),
('playlist', 'Playlist order'),
('playlist_reverse', 'Reverse playlist order'),
('popularity', 'Popularity'),
('rating', 'Top rated'),
]
VIDEO_ORDER_MAPPING = {
'newest': '-publish_date',
'oldest': 'publish_date',
'playlist': 'playlist_index',
'playlist_reverse': '-playlist_index',
'popularity': '-views',
'rating': '-rating'
}
class UserSettings(models.Model):
@ -40,6 +57,7 @@ class UserSettings(models.Model):
download_order = models.CharField(
null=True, blank=True,
max_length=100,
choices=VIDEO_ORDER_CHOICES,
help_text='The order in which videos will be downloaded.'
)
@ -300,8 +318,14 @@ class Subscription(models.Model):
# overrides
auto_download = models.BooleanField(null=True, blank=True)
download_limit = models.IntegerField(null=True, blank=True)
download_order = models.CharField(null=True, blank=True, max_length=128)
manager_delete_after_watched = models.BooleanField(null=True, blank=True)
download_order = models.CharField(
null=True, blank=True,
max_length=128,
choices=VIDEO_ORDER_CHOICES)
delete_after_watched = models.BooleanField(null=True, blank=True)
def __str__(self):
return self.name
def fill_from_playlist(self, info_playlist: YoutubePlaylistInfo):
self.name = info_playlist.getTitle()
@ -331,8 +355,17 @@ class Subscription(models.Model):
def delete_subscription(self, keep_downloaded_videos: bool):
self.delete()
def __str__(self):
return self.name
def get_overloads_dict(self) -> dict:
d = {}
if self.auto_download is not None:
d['AutoDownload'] = self.auto_download
if self.download_limit is not None:
d['DownloadSubscriptionLimit'] = self.download_limit
if self.download_order is not None:
d['DownloadOrder'] = self.download_order
if self.delete_after_watched is not None:
d['DeleteWatched'] = self.delete_after_watched
return d
class Video(models.Model):
@ -354,12 +387,11 @@ class Video(models.Model):
self.watched = True
self.save()
if self.downloaded_path is not None:
from YtManagerApp.appconfig import get_user_config
from YtManagerApp.appconfig import settings
from YtManagerApp.management.jobs.delete_video import schedule_delete_video
from YtManagerApp.management.jobs.synchronize import schedule_synchronize_now_subscription
user_cfg = get_user_config(self.subscription.user)
if user_cfg.getboolean('user', 'DeleteWatched'):
if settings.getboolean_sub(self.subscription, 'user', 'DeleteWatched'):
schedule_delete_video(self)
schedule_synchronize_now_subscription(self.subscription)
@ -379,14 +411,13 @@ class Video(models.Model):
def delete_files(self):
if self.downloaded_path is not None:
from YtManagerApp.management.jobs.delete_video import schedule_delete_video
from YtManagerApp.appconfig import get_user_config
from YtManagerApp.appconfig import settings
from YtManagerApp.management.jobs.synchronize import schedule_synchronize_now_subscription
schedule_delete_video(self)
# Mark watched?
user_cfg = get_user_config(self.subscription.user)
if user_cfg.getboolean('user', 'MarkDeletedAsWatched'):
if settings.getboolean_sub(self, 'user', 'MarkDeletedAsWatched'):
self.watched = True
schedule_synchronize_now_subscription(self.subscription)

24
YtManagerApp/utils/customconfigparser.py → YtManagerApp/utils/extended_interpolation_with_env.py
View File

@ -2,7 +2,7 @@ import os
import os.path
import re
from configparser import Interpolation, NoSectionError, NoOptionError, InterpolationMissingOptionError, \
InterpolationDepthError, InterpolationSyntaxError, ConfigParser
InterpolationDepthError, InterpolationSyntaxError
MAX_INTERPOLATION_DEPTH = 10
@ -16,12 +16,6 @@ class ExtendedInterpolatorWithEnv(Interpolation):
_KEYCRE = re.compile(r"\$\{([^}]+)\}")
def __init__(self, **kwargs):
self.__kwargs = kwargs
def set_additional_options(self, **kwargs):
self.__kwargs = kwargs
def before_get(self, parser, section, option, value, defaults):
L = []
self._interpolate_some(parser, option, L, value, section, defaults, 1)
@ -36,8 +30,6 @@ class ExtendedInterpolatorWithEnv(Interpolation):
return value
def _resolve_option(self, option, defaults):
if option in self.__kwargs:
return self.__kwargs[option]
return defaults[option]
def _resolve_section_option(self, section, option, parser):
@ -98,17 +90,3 @@ class ExtendedInterpolatorWithEnv(Interpolation):
option, section,
"'$' must be followed by '$' or '{', "
"found: %r" % (rest,))
class ConfigParserWithEnv(ConfigParser):
_DEFAULT_INTERPOLATION = ExtendedInterpolatorWithEnv()
def set_additional_interpolation_options(self, **kwargs):
"""
Sets additional options to be used in interpolation.
Only works with ExtendedInterpolatorWithEnv
:param kwargs:
:return:
"""
if isinstance(self._interpolation, ExtendedInterpolatorWithEnv):
self._interpolation.set_additional_options(**kwargs)

27
YtManagerApp/views/actions.py
View File

@ -1,19 +1,12 @@
from django.http import HttpRequest, HttpResponseBadRequest, JsonResponse
from django.shortcuts import render
from django import forms
from django.views.generic import CreateView, UpdateView, DeleteView, View
from django.views.generic.edit import FormMixin
from YtManagerApp.management.videos import get_videos
from YtManagerApp.models import Subscription, SubscriptionFolder, Video
from YtManagerApp.views.controls.modal import ModalMixin
from crispy_forms.helper import FormHelper
from crispy_forms.layout import Layout, Field, Div, HTML
from django.db.models import Q
from YtManagerApp.utils import youtube
from django.contrib.auth.mixins import LoginRequiredMixin
from django.http import JsonResponse
from django.views.generic import View
from YtManagerApp.management.jobs.synchronize import schedule_synchronize_now
from YtManagerApp.models import Video
class SyncNowView(View):
class SyncNowView(LoginRequiredMixin, View):
def post(self, *args, **kwargs):
schedule_synchronize_now()
return JsonResponse({
@ -21,7 +14,7 @@ class SyncNowView(View):
})
class DeleteVideoFilesView(View):
class DeleteVideoFilesView(LoginRequiredMixin, View):
def post(self, *args, **kwargs):
video = Video.objects.get(id=kwargs['pk'])
video.delete_files()
@ -30,7 +23,7 @@ class DeleteVideoFilesView(View):
})
class DownloadVideoFilesView(View):
class DownloadVideoFilesView(LoginRequiredMixin, View):
def post(self, *args, **kwargs):
video = Video.objects.get(id=kwargs['pk'])
video.download()
@ -39,7 +32,7 @@ class DownloadVideoFilesView(View):
})
class MarkVideoWatchedView(View):
class MarkVideoWatchedView(LoginRequiredMixin, View):
def post(self, *args, **kwargs):
video = Video.objects.get(id=kwargs['pk'])
video.mark_watched()
@ -48,7 +41,7 @@ class MarkVideoWatchedView(View):
})
class MarkVideoUnwatchedView(View):
class MarkVideoUnwatchedView(LoginRequiredMixin, View):
def post(self, *args, **kwargs):
video = Video.objects.get(id=kwargs['pk'])
video.mark_unwatched()

3
YtManagerApp/views/auth.py
View File

@ -3,6 +3,7 @@ from crispy_forms.layout import Submit
from django import forms
from django.contrib.auth import login, authenticate
from django.contrib.auth.forms import AuthenticationForm, UserCreationForm
from django.contrib.auth.mixins import LoginRequiredMixin
from django.contrib.auth.models import User
from django.contrib.auth.views import LoginView
from django.urls import reverse_lazy
@ -78,5 +79,5 @@ class RegisterView(FormView):
return context
class RegisterDoneView(TemplateView):
class RegisterDoneView(LoginRequiredMixin, TemplateView):
template_name = 'registration/register_done.html'

56
YtManagerApp/views/index.py
View File

@ -1,6 +1,8 @@
from crispy_forms.helper import FormHelper
from crispy_forms.layout import Layout, Field, HTML
from django import forms
from django.contrib.auth.decorators import login_required
from django.contrib.auth.mixins import LoginRequiredMixin
from django.db.models import Q
from django.http import HttpRequest, HttpResponseBadRequest, JsonResponse
from django.shortcuts import render
@ -8,31 +10,12 @@ from django.views.generic import CreateView, UpdateView, DeleteView
from django.views.generic.edit import FormMixin
from YtManagerApp.management.videos import get_videos
from YtManagerApp.models import Subscription, SubscriptionFolder
from YtManagerApp.models import Subscription, SubscriptionFolder, VIDEO_ORDER_CHOICES, VIDEO_ORDER_MAPPING
from YtManagerApp.utils import youtube
from YtManagerApp.views.controls.modal import ModalMixin
class VideoFilterForm(forms.Form):
CHOICES_SORT = (
('newest', 'Newest'),
('oldest', 'Oldest'),
('playlist', 'Playlist order'),
('playlist_reverse', 'Reverse playlist order'),
('popularity', 'Popularity'),
('rating', 'Top rated'),
)
# Map select values to actual column names
MAPPING_SORT = {
'newest': '-publish_date',
'oldest': 'publish_date',
'playlist': 'playlist_index',
'playlist_reverse': '-playlist_index',
'popularity': '-views',
'rating': '-rating'
}
CHOICES_SHOW_WATCHED = (
('y', 'Watched'),
('n', 'Not watched'),
@ -52,7 +35,7 @@ class VideoFilterForm(forms.Form):
}
query = forms.CharField(label='', required=False)
sort = forms.ChoiceField(label='Sort:', choices=CHOICES_SORT, initial='newest')
sort = forms.ChoiceField(label='Sort:', choices=VIDEO_ORDER_CHOICES, initial='newest')
show_watched = forms.ChoiceField(label='Show only: ', choices=CHOICES_SHOW_WATCHED, initial='all')
show_downloaded = forms.ChoiceField(label='', choices=CHOICES_SHOW_DOWNLOADED, initial='all')
subscription_id = forms.IntegerField(
@ -85,7 +68,7 @@ class VideoFilterForm(forms.Form):
def clean_sort(self):
data = self.cleaned_data['sort']
return VideoFilterForm.MAPPING_SORT[data]
return VIDEO_ORDER_MAPPING[data]
def clean_show_downloaded(self):
data = self.cleaned_data['show_downloaded']
@ -118,6 +101,7 @@ def index(request: HttpRequest):
return render(request, 'YtManagerApp/index_unauthenticated.html')
@login_required
def ajax_get_tree(request: HttpRequest):
def visit(node):
@ -142,6 +126,7 @@ def ajax_get_tree(request: HttpRequest):
return JsonResponse(result, safe=False)
@login_required
def ajax_get_videos(request: HttpRequest):
if request.method == 'POST':
form = VideoFilterForm(request.POST)
@ -206,7 +191,7 @@ class SubscriptionFolderForm(forms.ModelForm):
current = current.parent
class CreateFolderModal(ModalMixin, CreateView):
class CreateFolderModal(LoginRequiredMixin, ModalMixin, CreateView):
template_name = 'YtManagerApp/controls/folder_create_modal.html'
form_class = SubscriptionFolderForm
@ -215,7 +200,7 @@ class CreateFolderModal(ModalMixin, CreateView):
return super().form_valid(form)
class UpdateFolderModal(ModalMixin, UpdateView):
class UpdateFolderModal(LoginRequiredMixin, ModalMixin, UpdateView):
template_name = 'YtManagerApp/controls/folder_update_modal.html'
model = SubscriptionFolder
form_class = SubscriptionFolderForm
@ -225,7 +210,7 @@ class DeleteFolderForm(forms.Form):
keep_subscriptions = forms.BooleanField(required=False, initial=False, label="Keep subscriptions")
class DeleteFolderModal(ModalMixin, FormMixin, DeleteView):
class DeleteFolderModal(LoginRequiredMixin, ModalMixin, FormMixin, DeleteView):
template_name = 'YtManagerApp/controls/folder_delete_modal.html'
model = SubscriptionFolder
form_class = DeleteFolderForm
@ -248,7 +233,8 @@ class CreateSubscriptionForm(forms.ModelForm):
class Meta:
model = Subscription
fields = ['parent_folder']
fields = ['parent_folder', 'auto_download',
'download_limit', 'download_order', 'delete_after_watched']
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
@ -256,7 +242,13 @@ class CreateSubscriptionForm(forms.ModelForm):
self.helper.form_tag = False
self.helper.layout = Layout(
'playlist_url',
'parent_folder'
'parent_folder',
HTML('<hr>'),
HTML('<h5>Download configuration overloads</h5>'),
'auto_download',
'download_limit',
'download_order',
'delete_after_watched'
)
def clean_playlist_url(self):
@ -268,7 +260,7 @@ class CreateSubscriptionForm(forms.ModelForm):
return playlist_url
class CreateSubscriptionModal(ModalMixin, CreateView):
class CreateSubscriptionModal(LoginRequiredMixin, ModalMixin, CreateView):
template_name = 'YtManagerApp/controls/subscription_create_modal.html'
form_class = CreateSubscriptionForm
@ -300,7 +292,7 @@ class UpdateSubscriptionForm(forms.ModelForm):
class Meta:
model = Subscription
fields = ['name', 'parent_folder', 'auto_download',
'download_limit', 'download_order', 'manager_delete_after_watched']
'download_limit', 'download_order', 'delete_after_watched']
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
@ -314,11 +306,11 @@ class UpdateSubscriptionForm(forms.ModelForm):
'auto_download',
'download_limit',
'download_order',
'manager_delete_after_watched'
'delete_after_watched'
)
class UpdateSubscriptionModal(ModalMixin, UpdateView):
class UpdateSubscriptionModal(LoginRequiredMixin, ModalMixin, UpdateView):
template_name = 'YtManagerApp/controls/subscription_update_modal.html'
model = Subscription
form_class = UpdateSubscriptionForm
@ -328,7 +320,7 @@ class DeleteSubscriptionForm(forms.Form):
keep_downloaded_videos = forms.BooleanField(required=False, initial=False, label="Keep downloaded videos")
class DeleteSubscriptionModal(ModalMixin, FormMixin, DeleteView):
class DeleteSubscriptionModal(LoginRequiredMixin, ModalMixin, FormMixin, DeleteView):
template_name = 'YtManagerApp/controls/subscription_delete_modal.html'
model = Subscription
form_class = DeleteSubscriptionForm

5
YtManagerApp/views/settings.py
View File

@ -1,8 +1,9 @@
from crispy_forms.helper import FormHelper
from crispy_forms.layout import Layout, HTML, Submit
from django import forms
from django.views.generic import UpdateView
from django.contrib.auth.mixins import LoginRequiredMixin
from django.urls import reverse_lazy
from django.views.generic import UpdateView
from YtManagerApp.models import UserSettings
@ -39,7 +40,7 @@ class SettingsForm(forms.ModelForm):
)
class SettingsView(UpdateView):
class SettingsView(LoginRequiredMixin, UpdateView):
form_class = SettingsForm
model = UserSettings
template_name = 'YtManagerApp/settings.html'

38
config/config.ini
View File

@ -2,16 +2,16 @@
; The global section contains settings that apply to the entire server
[global]
; YouTube API key - get this from your user account
YoutubeApiKey=AIzaSyBabzE4Bup77WexdLMa9rN9z-wJidEfNX8
;YoutubeApiKey=AIzaSyBabzE4Bup77WexdLMa9rN9z-wJidEfNX8
; Specifies the synchronization schedule, in crontab format.
; Format: <minute> <hour> <day-of-month> <month-of-year> <day of week>
SynchronizationSchedule=0 * * * *
;SynchronizationSchedule=0 * * * *
; Number of threads running the scheduler
; Since most of the jobs scheduled are downloads, there is no advantage to having
; a higher concurrency
SchedulerConcurrency=2
;SchedulerConcurrency=2
; Log level
LogLevel=DEBUG
@ -19,43 +19,41 @@ LogLevel=DEBUG
; Default user settings
[user]
; When a video is deleted on the system, it will be marked as 'watched'
MarkDeletedAsWatched=True
;MarkDeletedAsWatched=True
; Videos marked as watched are automatically deleted
DeleteWatched=True
;DeleteWatched=True
; Enable automatic downloading
AutoDownload=True
;AutoDownload=True
; Limit the total number of videos downloaded (-1 or empty = no limit)
DownloadGlobalLimit=
;DownloadGlobalLimit=
; Limit the numbers of videos per subscription (-1 or empty = no limit)
DownloadSubscriptionLimit=5
;DownloadSubscriptionLimit=5
; Number of download attempts
DownloadMaxAttempts=3
;DownloadMaxAttempts=3
; Download order
; Options: playlist_index, publish_date, name.
; Use - to reverse order (e.g. -publish_date means to order by publish date descending)
DownloadOrder=playlist_index
; Options: newest, oldest, playlist, playlist_reverse, popularity, rating
;DownloadOrder=playlist
; Path where downloaded videos are stored
;DownloadPath=${env:USERPROFILE}${env:HOME}/Downloads
DownloadPath=D:\\Dev\\youtube-channel-manager\\temp\\download
; A pattern which describes how downloaded files are organized. Extensions are automatically appended.
; Supported fields: channel, channel_id, playlist, playlist_id, playlist_index, title, id
; The default pattern should work pretty well with Plex
DownloadFilePattern=${channel}/${playlist}/S01E${playlist_index} - ${title} [${id}]
;DownloadFilePattern=${channel}/${playlist}/S01E${playlist_index} - ${title} [${id}]
; Download format that will be passed to youtube-dl. See the youtube-dl documentation for more details.
DownloadFormat=bestvideo+bestaudio
DownloadFormat=worstvideo+bestaudio
; Subtitles - these options match the youtube-dl options
DownloadSubtitles=True
DownloadAutogeneratedSubtitles=False
DownloadSubtitlesAll=False
DownloadSubtitlesLangs=en,ro
DownloadSubtitlesFormat=
;DownloadSubtitles=True
;DownloadAutogeneratedSubtitles=False
;DownloadSubtitlesAll=False
;DownloadSubtitlesLangs=en,ro
;DownloadSubtitlesFormat=

10
config/config.ini.default → config/defaults.ini
View File

@ -14,7 +14,7 @@ SynchronizationSchedule=0 * * * *
SchedulerConcurrency=2
; Log level
LogLevel=DEBUG
LogLevel=INFO
; Default user settings
[user]
@ -37,13 +37,11 @@ DownloadSubscriptionLimit=5
DownloadMaxAttempts=3
; Download order
; Options: playlist_index, publish_date, name.
; Use - to reverse order (e.g. -publish_date means to order by publish date descending)
DownloadOrder=playlist_index
; Options: newest, oldest, playlist, playlist_reverse, popularity, rating
DownloadOrder=playlist
; Path where downloaded videos are stored
;DownloadPath=${env:USERPROFILE}${env:HOME}/Downloads
DownloadPath=D:\\Dev\\youtube-channel-manager\\temp\\download
DownloadPath=${env:USERPROFILE}${env:HOME}/Downloads
; A pattern which describes how downloaded files are organized. Extensions are automatically appended.
; Supported fields: channel, channel_id, playlist, playlist_id, playlist_index, title, id