Big refactor

This commit is contained in:
Tiberiu Chibici 2018-10-11 01:43:50 +03:00
parent 291da16461
commit ae77251883
42 changed files with 4311 additions and 264 deletions

1
.gitignore vendored
View File

@ -1,5 +1,6 @@
.vs
.vscode
temp/
# Byte-compiled / optimized / DLL files
__pycache__/

View File

@ -0,0 +1,6 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="JavaScriptLibraryMappings">
<file url="file://$PROJECT_DIR$" libraries="{bootstrap, jquery-3.3.1, popper}" />
</component>
</project>

File diff suppressed because it is too large Load Diff

View File

@ -16,6 +16,9 @@
<content url="file://$MODULE_DIR$" />
<orderEntry type="inheritedJdk" />
<orderEntry type="sourceFolder" forTests="false" />
<orderEntry type="library" name="jquery-3.3.1" level="application" />
<orderEntry type="library" name="popper" level="application" />
<orderEntry type="library" name="bootstrap" level="application" />
</component>
<component name="TemplatesService">
<option name="TEMPLATE_CONFIGURATION" value="Django" />

View File

@ -15,7 +15,6 @@ import os
# Build paths inside the project like this: os.path.join(BASE_DIR, ...)
BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
# Quick-start development settings - unsuitable for production
# See https://docs.djangoproject.com/en/1.11/howto/deployment/checklist/
@ -63,6 +62,7 @@ TEMPLATES = [
'context_processors': [
'django.template.context_processors.debug',
'django.template.context_processors.request',
'django.template.context_processors.media',
'django.contrib.auth.context_processors.auth',
'django.contrib.messages.context_processors.messages',
],
@ -121,3 +121,22 @@ USE_TZ = True
# https://docs.djangoproject.com/en/1.11/howto/static-files/
STATIC_URL = '/static/'
MEDIA_URL = '/media/'
MEDIA_ROOT = 'D:\\Dev\\youtube-channel-manager\\temp\\media'
# Application settings
# These should be moved to an ini file or something
DOWNLOADER_PATH = "D:\\Dev\\youtube-channel-manager\\temp\\download"
DOWNLOADER_FILE_PATTERN = "{channel}/{playlist}/S01E{playlist_index} - {title} [{id}].{ext}"
DOWNLOADER_FORMAT = "bestvideo+bestaudio"
DOWNLOADER_SUBTITLES = True
DOWNLOADER_SUBTITLES_LANGS = None
DOWNLOADER_AUTOGENERATED_SUBTITLES = False
DOWNLOADER_DEFAULT_DOWNLOAD_ENABLED = True
DOWNLOADER_DEFAULT_DOWNLOAD_LIMIT = 5 # -1 = no limit (all)
DOWNLOADER_DEFAULT_ORDER = 'playlist_index' # publish_date, playlist_index
MANAGER_MARK_DELETED_AS_WATCHED = True
MANAGER_DEFAULT_DELETE_AFTER_WATCHED = True

View File

@ -15,6 +15,8 @@ Including another URLconf
"""
from django.urls import path
from django.contrib import admin
from django.conf.urls.static import static
from django.conf import settings
from YtManagerApp import views
@ -26,5 +28,6 @@ urlpatterns = [
path('ajax/delete_folder/<int:fid>/', views.ajax_delete_folder, name='ajax_delete_folder'),
path('ajax/edit_subscription', views.ajax_edit_subscription, name='ajax_edit_subscription'),
path('ajax/delete_subscription/<int:sid>/', views.ajax_delete_subscription, name='ajax_delete_subscription'),
path('ajax/list_videos', views.ajax_list_videos, name='ajax_list_videos'),
path(r'', views.index, name='home')
]
] + static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT)

108
YtManagerApp/appconfig.py Normal file
View File

@ -0,0 +1,108 @@
import logging
import os
import os.path
from shutil import copyfile
from django.conf import settings
from django.contrib.auth.models import User
from .models import UserSettings
from .utils.customconfigparser import ConfigParserWithEnv
class AppConfig(object):
__SETTINGS_FILE = 'config.ini'
__LOG_FILE = 'log.log'
DEFAULT_SETTINGS = {
'global': {
'YouTubeApiKey': 'AIzaSyBabzE4Bup77WexdLMa9rN9z-wJidEfNX8',
'SynchronizationSchedule': '0 * * * * *',
'SchedulerConcurrency': '2',
},
'user': {
'MarkDeletedAsWatched': 'True',
'DeleteWatched': 'True',
'AutoDownload': 'True',
'DownloadMaxAttempts': '3',
'DownloadGlobalLimit': '',
'DownloadSubscriptionLimit': '5',
'DownloadOrder': 'playlist_index',
'DownloadPath': '${env:USERPROFILE}${env:HOME}/Downloads',
'DownloadFilePattern': '${channel}/${playlist}/S01E${playlist_index} - ${title} [${id}]',
'DownloadFormat': 'bestvideo+bestaudio',
'DownloadSubtitles': 'True',
'DownloadAutogeneratedSubtitles': 'False',
'DownloadSubtitlesAll': 'False',
'DownloadSubtitlesLangs': 'en,ro',
'DownloadSubtitlesFormat': '',
}
}
def __init__(self):
self.log_path = os.path.join(settings.BASE_DIR, 'config', AppConfig.__LOG_FILE)
self.settings_path = os.path.join(settings.BASE_DIR, 'config', AppConfig.__SETTINGS_FILE)
self.settings = ConfigParserWithEnv(defaults=AppConfig.DEFAULT_SETTINGS, allow_no_value=True)
self.load_settings()
def load_settings(self):
if os.path.exists(self.settings_path):
with open(self.settings_path, 'r') as f:
self.settings.read_file(f)
def save_settings(self):
if os.path.exists(self.settings_path):
# Create a backup
copyfile(self.settings_path, self.settings_path + ".backup")
else:
# Ensure directory exists
settings_dir = os.path.dirname(self.settings_path)
os.makedirs(settings_dir, exist_ok=True)
with open(self.settings_path, 'w') as f:
self.settings.write(f)
def get_user_config(self, user: User) -> ConfigParserWithEnv:
user_settings = UserSettings.find_by_user(user)
if user_settings is not None:
user_config = ConfigParserWithEnv(defaults=self.settings, allow_no_value=True)
user_config.read_dict({
'user': user_settings.to_dict()
})
return user_config
return settings
instance: AppConfig = None
def __initialize_logger():
# Parse log level
log_level_str = instance.settings.get('global', 'LogLevel', fallback='INFO')
levels = {
'NOTSET': logging.NOTSET,
'DEBUG': logging.DEBUG,
'INFO': logging.INFO,
'WARNING': logging.WARNING,
'ERROR': logging.ERROR,
'CRITICAL': logging.CRITICAL
}
if log_level_str.upper() not in levels:
log_level_str = 'INFO'
# Init
logging.basicConfig(filename=instance.log_path, level=levels[log_level_str])
def initialize_config():
global instance
instance = AppConfig()
# Load settings
instance.load_settings()
# Initialize logger
__initialize_logger()
logging.info('Application started!')

11
YtManagerApp/appmain.py Normal file
View File

@ -0,0 +1,11 @@
from .appconfig import initialize_config
from .scheduler import initialize_scheduler
from .management import setup_synchronization_job
import logging
def main():
initialize_config()
initialize_scheduler()
setup_synchronization_job()
logging.info('Initialization complete.')

View File

@ -1,9 +1,13 @@
from django.apps import AppConfig
import os
class YtManagerAppConfig(AppConfig):
name = 'YtManagerApp'
def ready(self):
from .management import SubscriptionManager
SubscriptionManager.start_scheduler()
# There seems to be a problem related to the auto-reload functionality where ready() is called twice
# (in different processes). This seems like a good enough workaround (other than --noreload).
if not os.getenv('RUN_MAIN', False):
from .appmain import main
main()

View File

View File

@ -0,0 +1,110 @@
from YtManagerApp.appconfig import instance as app_config
from YtManagerApp.management.jobs.download_video import schedule_download_video
from YtManagerApp.models import Video, Subscription
from django.conf import settings
import logging
import requests
import mimetypes
import os
from urllib.parse import urljoin
log = logging.getLogger('downloader')
def __get_subscription_config(sub: Subscription):
user_config = app_config.get_user_config(sub.user)
enabled = sub.auto_download
if enabled is None:
enabled = user_config.getboolean('user', 'AutoDownload')
global_limit = -1
if len(user_config.get('user', 'DownloadGlobalLimit')) > 0:
global_limit = user_config.getint('user', 'DownloadGlobalLimit')
limit = sub.download_limit
if limit is None:
limit = -1
if len(user_config.get('user', 'DownloadSubscriptionLimit')) > 0:
limit = user_config.getint('user', 'DownloadSubscriptionLimit')
order = sub.download_order
if order is None:
order = user_config.get('user', 'DownloadOrder')
return enabled, global_limit, limit, order
def __process_subscription(sub: Subscription):
log.info('Processing subscription %d [%s %s]', sub.id, sub.playlist_id, sub.id)
enabled, global_limit, limit, order = __get_subscription_config(sub)
log.info('Determined settings enabled=%s global_limit=%d limit=%d order="%s"', enabled, global_limit, limit, order)
if enabled:
videos_to_download = Video.objects\
.filter(subscription=sub, downloaded_path__isnull=True, watched=False)\
.order_by(order)
log.info('%d download candidates.', len(videos_to_download))
if global_limit > 0:
global_downloaded = Video.objects.filter(subscription__user=sub.user, downloaded_path__isnull=False).count()
allowed_count = max(global_limit - global_downloaded, 0)
videos_to_download = videos_to_download[0:allowed_count]
log.info('Global limit is set, can only download up to %d videos.', allowed_count)
if limit > 0:
sub_downloaded = Video.objects.filter(subscription=sub, downloaded_path__isnull=False).count()
allowed_count = max(limit - sub_downloaded, 0)
videos_to_download = videos_to_download[0:allowed_count]
log.info('Limit is set, can only download up to %d videos.', allowed_count)
# enqueue download
for video in videos_to_download:
log.info('Enqueuing video %d [%s %s] index=%d', video.id, video.video_id, video.name, video.playlist_index)
schedule_download_video(video)
log.info('Finished processing subscription %d [%s %s]', sub.id, sub.playlist_id, sub.id)
def downloader_process_all():
for subscription in Subscription.objects.all():
__process_subscription(subscription)
def fetch_thumbnail(url, object_type, identifier, quality):
log.info('Fetching thumbnail url=%s object_type=%s identifier=%s quality=%s', url, object_type, identifier, quality)
# Make request to obtain mime type
try:
response = requests.get(url, stream=True)
except requests.exceptions.RequestException as e:
log.error('Failed to fetch thumbnail %s. Error: %s', url, e)
return url
ext = mimetypes.guess_extension(response.headers['Content-Type'])
# Build file path
file_name = f"{identifier}-{quality}{ext}"
abs_path_dir = os.path.join(settings.MEDIA_ROOT, "thumbs", object_type)
abs_path = os.path.join(abs_path_dir, file_name)
# Store image
try:
os.makedirs(abs_path_dir, exist_ok=True)
with open(abs_path, "wb") as f:
for chunk in response.iter_content(chunk_size=1024):
if chunk:
f.write(chunk)
except requests.exceptions.RequestException as e:
log.error('Error while downloading stream for thumbnail %s. Error: %s', url, e)
return url
except OSError as e:
log.error('Error while writing to file %s for thumbnail %s. Error: %s', abs_path, url, e)
return url
# Return
media_url = urljoin(settings.MEDIA_URL, f"thumbs/{object_type}/{file_name}")
return media_url

View File

View File

View File

@ -0,0 +1,93 @@
from YtManagerApp.models import Video
from YtManagerApp.scheduler import instance as scheduler
from YtManagerApp.appconfig import instance as app_config
import os
import youtube_dl
import logging
log = logging.getLogger('video_downloader')
log_youtube_dl = log.getChild('youtube_dl')
def __build_youtube_dl_params(video: Video, user_config):
# resolve path
format_dict = {
'channel': video.subscription.channel.name,
'channel_id': video.subscription.channel.channel_id,
'playlist': video.subscription.name,
'playlist_id': video.subscription.playlist_id,
'playlist_index': "{:03d}".format(1 + video.playlist_index),
'title': video.name,
'id': video.video_id,
}
user_config.set_additional_interpolation_options(**format_dict)
download_path = user_config.get('user', 'DownloadPath')
output_pattern = user_config.get('user', 'DownloadFilePattern')
output_path = os.path.join(download_path, output_pattern)
output_path = os.path.normpath(output_path)
youtube_dl_params = {
'logger': log_youtube_dl,
'format': user_config.get('user', 'DownloadFormat'),
'outtmpl': output_path,
'writethumbnail': True,
'writedescription': True,
'writesubtitles': user_config.getboolean('user', 'DownloadSubtitles'),
'writeautomaticsub': user_config.getboolean('user', 'DownloadAutogeneratedSubtitles'),
'allsubtitles': user_config.getboolean('user', 'DownloadSubtitlesAll'),
'postprocessors': [
{
'key': 'FFmpegMetadataPP'
},
]
}
sub_langs = user_config.get('user', 'DownloadSubtitlesLangs').split(',')
sub_langs = [i.strip() for i in sub_langs]
if len(sub_langs) > 0:
youtube_dl_params['subtitleslangs'] = sub_langs
sub_format = user_config.get('user', 'DownloadSubtitlesFormat')
if len(sub_format) > 0:
youtube_dl_params['subtitlesformat'] = sub_format
return youtube_dl_params, output_path
def download_video(video: Video, attempt: int = 1):
log.info('Downloading video %d [%s %s]', video.id, video.video_id, video.name)
user_config = app_config.get_user_config(video.subscription.user)
max_attempts = user_config.getint('user', 'DownloadMaxAttempts', fallback=3)
youtube_dl_params, output_path = __build_youtube_dl_params(video, user_config)
with youtube_dl.YoutubeDL(youtube_dl_params) as yt:
ret = yt.download(["https://www.youtube.com/watch?v=" + video.video_id])
log.info('Download finished with code %d', ret)
if ret == 0:
video.downloaded_path = output_path
video.save()
log.error('Video %d [%s %s] downloaded successfully!', video.id, video.video_id, video.name)
elif attempt <= max_attempts:
log.warning('Re-enqueueing video (attempt %d/%d)', attempt, max_attempts)
scheduler.add_job(download_video, args=[video, attempt + 1])
else:
log.error('Multiple attempts to download video %d [%s %s] failed!', video.id, video.video_id, video.name)
video.downloaded_path = ''
video.save()
def schedule_download_video(video: Video):
"""
Schedules a download video job to run immediately.
:param video:
:return:
"""
scheduler.add_job(download_video, args=[video, 1])

View File

@ -0,0 +1,46 @@
from YtManagerApp.models import *
import logging
def synchronize():
logger = logging.getLogger('sync')
logger.info("Running scheduled synchronization... ")
# Sync subscribed playlists/channels
yt_api = YoutubeAPI.build_public()
for subscription in Subscription.objects.all():
SubscriptionManager.__synchronize(subscription, yt_api)
# Fetch thumbnails
logger.info("Fetching channel thumbnails... ")
for ch in Channel.objects.filter(icon_default__istartswith='http'):
ch.icon_default = SubscriptionManager.__fetch_thumbnail(ch.icon_default, 'channel', ch.channel_id, 'default')
ch.save()
for ch in Channel.objects.filter(icon_best__istartswith='http'):
ch.icon_best = SubscriptionManager.__fetch_thumbnail(ch.icon_best, 'channel', ch.channel_id, 'best')
ch.save()
logger.info("Fetching subscription thumbnails... ")
for sub in Subscription.objects.filter(icon_default__istartswith='http'):
sub.icon_default = SubscriptionManager.__fetch_thumbnail(sub.icon_default, 'sub', sub.playlist_id, 'default')
sub.save()
for sub in Subscription.objects.filter(icon_best__istartswith='http'):
sub.icon_best = SubscriptionManager.__fetch_thumbnail(sub.icon_best, 'sub', sub.playlist_id, 'best')
sub.save()
logger.info("Fetching video thumbnails... ")
for vid in Video.objects.filter(icon_default__istartswith='http'):
vid.icon_default = SubscriptionManager.__fetch_thumbnail(vid.icon_default, 'video', vid.video_id, 'default')
vid.save()
for vid in Video.objects.filter(icon_best__istartswith='http'):
vid.icon_best = SubscriptionManager.__fetch_thumbnail(vid.icon_best, 'video', vid.video_id, 'best')
vid.save()
print("Downloading videos...")
Downloader.download_all()
print("Synchronization finished.")

View File

@ -1,7 +1,18 @@
from .models import SubscriptionFolder, Subscription, Video, Channel
from .youtube import YoutubeAPI, YoutubeChannelInfo, YoutubePlaylistItem
from YtManagerApp.models import SubscriptionFolder, Subscription, Video, Channel
from YtManagerApp.utils.youtube import YoutubeAPI, YoutubeChannelInfo, YoutubePlaylistItem
from django.conf import settings
from apscheduler.schedulers.background import BackgroundScheduler
import os
import os.path
import requests
from urllib.parse import urljoin
import mimetypes
import youtube_dl
from YtManagerApp.scheduler import instance as scheduler
from YtManagerApp.appconfig import instance as app_config
from apscheduler.triggers.cron import CronTrigger
class FolderManager(object):
@ -45,6 +56,18 @@ class FolderManager(object):
folder = SubscriptionFolder.objects.get(id=fid)
folder.delete()
@staticmethod
def list_videos(fid: int):
folder = SubscriptionFolder.objects.get(id=fid)
folder_list = []
queue = [folder]
while len(queue) > 0:
folder = queue.pop()
folder_list.append(folder)
queue.extend(SubscriptionFolder.objects.filter(parent=folder))
return Video.objects.filter(subscription__parent_folder__in=folder_list).order_by('-publish_date')
class SubscriptionManager(object):
__scheduler = BackgroundScheduler()
@ -53,7 +76,6 @@ class SubscriptionManager(object):
def create_or_edit(sid, url, name, parent_id):
# Create or edit
if sid == '#':
sub = Subscription()
SubscriptionManager.create(url, parent_id, YoutubeAPI.build_public())
else:
sub = Subscription.objects.get(id=int(sid))
@ -100,6 +122,11 @@ class SubscriptionManager(object):
sub.save()
@staticmethod
def list_videos(fid: int):
sub = Subscription.objects.get(id=fid)
return Video.objects.filter(subscription=sub).order_by('playlist_index')
@staticmethod
def __get_or_create_channel(url_type, url_id, yt_api: YoutubeAPI):
@ -173,12 +200,79 @@ class SubscriptionManager(object):
@staticmethod
def __synchronize_all():
print("Running scheduled synchronization... ")
# Sync subscribed playlists/channels
yt_api = YoutubeAPI.build_public()
for subscription in Subscription.objects.all():
SubscriptionManager.__synchronize(subscription, yt_api)
# Fetch thumbnails
print("Fetching channel thumbnails... ")
for ch in Channel.objects.filter(icon_default__istartswith='http'):
ch.icon_default = SubscriptionManager.__fetch_thumbnail(ch.icon_default, 'channel', ch.channel_id, 'default')
ch.save()
for ch in Channel.objects.filter(icon_best__istartswith='http'):
ch.icon_best = SubscriptionManager.__fetch_thumbnail(ch.icon_best, 'channel', ch.channel_id, 'best')
ch.save()
print("Fetching subscription thumbnails... ")
for sub in Subscription.objects.filter(icon_default__istartswith='http'):
sub.icon_default = SubscriptionManager.__fetch_thumbnail(sub.icon_default, 'sub', sub.playlist_id, 'default')
sub.save()
for sub in Subscription.objects.filter(icon_best__istartswith='http'):
sub.icon_best = SubscriptionManager.__fetch_thumbnail(sub.icon_best, 'sub', sub.playlist_id, 'best')
sub.save()
print("Fetching video thumbnails... ")
for vid in Video.objects.filter(icon_default__istartswith='http'):
vid.icon_default = SubscriptionManager.__fetch_thumbnail(vid.icon_default, 'video', vid.video_id, 'default')
vid.save()
for vid in Video.objects.filter(icon_best__istartswith='http'):
vid.icon_best = SubscriptionManager.__fetch_thumbnail(vid.icon_best, 'video', vid.video_id, 'best')
vid.save()
print("Downloading videos...")
Downloader.download_all()
print("Synchronization finished.")
@staticmethod
def __fetch_thumbnail(url, object_type, identifier, quality):
# Make request to obtain mime type
response = requests.get(url, stream=True)
ext = mimetypes.guess_extension(response.headers['Content-Type'])
# Build file path
file_name = f"{identifier}-{quality}{ext}"
abs_path_dir = os.path.join(settings.MEDIA_ROOT, "thumbs", object_type)
abs_path = os.path.join(abs_path_dir, file_name)
# Store image
os.makedirs(abs_path_dir, exist_ok=True)
with open(abs_path, "wb") as f:
for chunk in response.iter_content(chunk_size=1024):
if chunk:
f.write(chunk)
# Return
media_url = urljoin(settings.MEDIA_URL, f"thumbs/{object_type}/{file_name}")
return media_url
@staticmethod
def start_scheduler():
SubscriptionManager.__scheduler.add_job(SubscriptionManager.__synchronize_all, 'cron',
hour='*', minute=44, max_instances=1)
hour='*', minute=38, max_instances=1)
SubscriptionManager.__scheduler.start()
def setup_synchronization_job():
trigger = CronTrigger.from_crontab(app_config.get('global', 'SynchronizationSchedule'))
scheduler.add_job(synchronize_all, trigger, max_instances=1)
def synchronize_all():
pass

View File

View File

@ -0,0 +1,33 @@
# Generated by Django 2.1.2 on 2018-10-08 23:09
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('YtManagerApp', '0006_auto_20181008_0037'),
]
operations = [
migrations.AddField(
model_name='subscription',
name='downloader_enabled',
field=models.BooleanField(null=True),
),
migrations.AddField(
model_name='subscription',
name='downloader_limit',
field=models.IntegerField(null=True),
),
migrations.AddField(
model_name='subscription',
name='downloader_order',
field=models.TextField(null=True),
),
migrations.AddField(
model_name='subscription',
name='manager_delete_after_watched',
field=models.BooleanField(null=True),
),
]

View File

@ -1,4 +1,67 @@
from django.db import models
from django.contrib.auth.models import User
class UserSettings(models.Model):
user = models.OneToOneField(User, on_delete=models.CASCADE)
mark_deleted_as_watched = models.BooleanField(null=True)
delete_watched = models.BooleanField(null=True)
auto_download = models.BooleanField(null=True)
download_global_limit = models.IntegerField(null=True)
download_subscription_limit = models.IntegerField(null=True)
download_order = models.TextField(null=True)
download_path = models.TextField(null=True)
download_file_pattern = models.TextField(null=True)
download_format = models.TextField(null=True)
download_subtitles = models.BooleanField(null=True)
download_autogenerated_subtitles = models.BooleanField(null=True)
download_subtitles_all = models.BooleanField(null=True)
download_subtitles_langs = models.TextField(null=True)
download_subtitles_format = models.TextField(null=True)
@staticmethod
def find_by_user(user: User):
result = UserSettings.objects.filter(user=user)
if len(result) > 0:
return result.first()
return None
def __str__(self):
return str(self.user)
def to_dict(self):
ret = {}
if self.mark_deleted_as_watched is not None:
ret['MarkDeletedAsWatched'] = self.mark_deleted_as_watched
if self.delete_watched is not None:
ret['DeleteWatched'] = self.delete_watched
if self.auto_download is not None:
ret['AutoDownload'] = self.auto_download
if self.download_global_limit is not None:
ret['DownloadGlobalLimit'] = self.download_global_limit
if self.download_subscription_limit is not None:
ret['DownloadSubscriptionLimit'] = self.download_subscription_limit
if self.download_order is not None:
ret['DownloadOrder'] = self.download_order
if self.download_path is not None:
ret['DownloadPath'] = self.download_path
if self.download_file_pattern is not None:
ret['DownloadFilePattern'] = self.download_file_pattern
if self.download_format is not None:
ret['DownloadFormat'] = self.download_format
if self.download_subtitles is not None:
ret['DownloadSubtitles'] = self.download_subtitles
if self.download_autogenerated_subtitles is not None:
ret['DownloadAutogeneratedSubtitles'] = self.download_autogenerated_subtitles
if self.download_subtitles_all is not None:
ret['DownloadSubtitlesAll'] = self.download_subtitles_all
if self.download_subtitles_langs is not None:
ret['DownloadSubtitlesLangs'] = self.download_subtitles_langs
if self.download_subtitles_format is not None:
ret['DownloadSubtitlesFormat'] = self.download_subtitles_format
return ret
class SubscriptionFolder(models.Model):
@ -52,6 +115,13 @@ class Subscription(models.Model):
channel = models.ForeignKey(Channel, on_delete=models.CASCADE)
icon_default = models.TextField()
icon_best = models.TextField()
user = models.ForeignKey(User, on_delete=models.CASCADE)
# overrides
auto_download = models.BooleanField(null=True)
download_limit = models.IntegerField(null=True)
download_order = models.TextField(null=True)
manager_delete_after_watched = models.BooleanField(null=True)
def __str__(self):
return self.name

20
YtManagerApp/scheduler.py Normal file
View File

@ -0,0 +1,20 @@
import logging
from apscheduler.schedulers.background import BackgroundScheduler
from .appconfig import instance as app_config
instance: BackgroundScheduler = None
def initialize_scheduler():
global instance
logger = logging.getLogger('scheduler')
executors = {
'default': {
'type': 'threadpool',
'max_workers': int(app_config.get('global', 'SchedulerConcurrency'))
}
}
instance = BackgroundScheduler(logger=logger, executors=executors)
instance.start()

View File

@ -27,5 +27,28 @@
transform: rotate(0deg); }
100% {
transform: rotate(360deg); } }
.video-gallery .card-wrapper {
padding: 0.4rem;
margin-bottom: .5rem; }
.video-gallery .card .card-body {
padding: .75rem; }
.video-gallery .card .card-text {
font-size: 10pt;
margin-bottom: .5rem; }
.video-gallery .card .card-title {
font-size: 11pt;
margin-bottom: .5rem; }
.video-gallery .card .card-title .badge {
font-size: 8pt; }
.video-gallery .card .card-footer {
padding: .5rem .75rem; }
.video-gallery .card .card-more {
margin-right: -0.25rem; }
.video-gallery .card .card-more:hover {
text-decoration: none; }
.video-gallery .video-icon-yes {
color: #6c757d; }
.video-gallery .video-icon-no {
color: #cccccc; }
/*# sourceMappingURL=style.css.map */

View File

@ -1,6 +1,6 @@
{
"version": 3,
"mappings": "AAEA,gCAAgC;AAChC,wBAAyB;EACrB,OAAO,EAAE,OAAO;;AAGpB,wBAAyB;EACrB,OAAO,EAAE,OAAO;;AAGpB,uBAAuB;AACvB,kBAAmB;EACf,OAAO,EAAE,YAAY;EACrB,KAAK,EAAE,IAAI;EACX,MAAM,EAAE,IAAI;;AAGhB,wBAAyB;EACrB,OAAO,EAAE,GAAG;EACZ,OAAO,EAAE,KAAK;EACd,KAAK,EAAE,IAAI;EACX,MAAM,EAAE,IAAI;EACZ,MAAM,EAAE,GAAG;EACX,aAAa,EAAE,GAAG;EAClB,MAAM,EAAE,iBAAuB;EAC/B,YAAY,EAAE,uCAAmD;EACjE,SAAS,EAAE,sCAAsC;;AAGrD,4BAOC;EANG,EAAG;IACC,SAAS,EAAE,YAAY;EAE3B,IAAK;IACD,SAAS,EAAE,cAAc",
"mappings": "AAEA,gCAAgC;AAChC,wBAAyB;EACrB,OAAO,EAAE,OAAO;;AAGpB,wBAAyB;EACrB,OAAO,EAAE,OAAO;;AAGpB,uBAAuB;AACvB,kBAAmB;EACf,OAAO,EAAE,YAAY;EACrB,KAAK,EAAE,IAAI;EACX,MAAM,EAAE,IAAI;;AAGhB,wBAAyB;EACrB,OAAO,EAAE,GAAG;EACZ,OAAO,EAAE,KAAK;EACd,KAAK,EAAE,IAAI;EACX,MAAM,EAAE,IAAI;EACZ,MAAM,EAAE,GAAG;EACX,aAAa,EAAE,GAAG;EAClB,MAAM,EAAE,iBAAuB;EAC/B,YAAY,EAAE,uCAAmD;EACjE,SAAS,EAAE,sCAAsC;;AAGrD,4BAOC;EANG,EAAG;IACC,SAAS,EAAE,YAAY;EAE3B,IAAK;IACD,SAAS,EAAE,cAAc;AAK7B,4BAAc;EACV,OAAO,EAAE,MAAM;EACf,aAAa,EAAE,KAAK;AAGpB,+BAAW;EACP,OAAO,EAAE,MAAM;AAEnB,+BAAW;EACP,SAAS,EAAE,IAAI;EACf,aAAa,EAAE,KAAK;AAExB,gCAAY;EACR,SAAS,EAAE,IAAI;EACf,aAAa,EAAE,KAAK;EAEpB,uCAAO;IACH,SAAS,EAAE,GAAG;AAGtB,iCAAa;EACT,OAAO,EAAE,YAAY;AAGzB,+BAAW;EACP,YAAY,EAAE,QAAQ;EACtB,qCAAQ;IACJ,eAAe,EAAE,IAAI;AAKjC,8BAAgB;EACZ,KAAK,EAAE,OAAO;AAElB,6BAAe;EACX,KAAK,EAAE,OAAO",
"sources": ["style.scss"],
"names": [],
"file": "style.css"

View File

@ -35,4 +35,46 @@ $accent-color: #007bff;
100% {
transform: rotate(360deg);
}
}
.video-gallery {
.card-wrapper {
padding: 0.4rem;
margin-bottom: .5rem;
}
.card {
.card-body {
padding: .75rem;
}
.card-text {
font-size: 10pt;
margin-bottom: .5rem;
}
.card-title {
font-size: 11pt;
margin-bottom: .5rem;
.badge {
font-size: 8pt;
}
}
.card-footer {
padding: .5rem .75rem;
}
.card-more {
margin-right: -0.25rem;
&:hover {
text-decoration: none;
}
}
}
.video-icon-yes {
color: #6c757d;
}
.video-icon-no {
color: #cccccc;
}
}

View File

@ -0,0 +1,92 @@
Copyright (c) 2014, Stephen Hutchings (http://www.s-ings.com/).
This Font Software is licensed under the SIL Open Font License, Version 1.1.
This license is copied below, and is also available with a FAQ at: http://scripts.sil.org/OFL
SIL OPEN FONT LICENSE
Version 1.1 - 26 February 2007
PREAMBLE
The goals of the Open Font License (OFL) are to stimulate worldwide
development of collaborative font projects, to support the font creation
efforts of academic and linguistic communities, and to provide a free and
open framework in which fonts may be shared and improved in partnership
with others.
The OFL allows the licensed fonts to be used, studied, modified and
redistributed freely as long as they are not sold by themselves. The
fonts, including any derivative works, can be bundled, embedded,
redistributed and/or sold with any software provided that any reserved
names are not used by derivative works. The fonts and derivatives,
however, cannot be released under any other type of license. The
requirement for fonts to remain under this license does not apply
to any document created using the fonts or their derivatives.
DEFINITIONS
"Font Software" refers to the set of files released by the Copyright
Holder(s) under this license and clearly marked as such. This may
include source files, build scripts and documentation.
"Reserved Font Name" refers to any names specified as such after the
copyright statement(s).
"Original Version" refers to the collection of Font Software components as
distributed by the Copyright Holder(s).
"Modified Version" refers to any derivative made by adding to, deleting,
or substituting — in part or in whole — any of the components of the
Original Version, by changing formats or by porting the Font Software to a
new environment.
"Author" refers to any designer, engineer, programmer, technical
writer or other person who contributed to the Font Software.
PERMISSION & CONDITIONS
Permission is hereby granted, free of charge, to any person obtaining
a copy of the Font Software, to use, study, copy, merge, embed, modify,
redistribute, and sell modified and unmodified copies of the Font
Software, subject to the following conditions:
1) Neither the Font Software nor any of its individual components,
in Original or Modified Versions, may be sold by itself.
2) Original or Modified Versions of the Font Software may be bundled,
redistributed and/or sold with any software, provided that each copy
contains the above copyright notice and this license. These can be
included either as stand-alone text files, human-readable headers or
in the appropriate machine-readable metadata fields within text or
binary files as long as those fields can be easily viewed by the user.
3) No Modified Version of the Font Software may use the Reserved Font
Name(s) unless explicit written permission is granted by the corresponding
Copyright Holder. This restriction only applies to the primary font name as
presented to the users.
4) The name(s) of the Copyright Holder(s) or the Author(s) of the Font
Software shall not be used to promote, endorse or advertise any
Modified Version, except to acknowledge the contribution(s) of the
Copyright Holder(s) and the Author(s) or with their explicit written
permission.
5) The Font Software, modified or unmodified, in part or in whole,
must be distributed entirely under this license, and must not be
distributed under any other license. The requirement for fonts to
remain under this license does not apply to any document created
using the Font Software.
TERMINATION
This license becomes null and void if any of the above conditions are
not met.
DISCLAIMER
THE FONT SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO ANY WARRANTIES OF
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT
OF COPYRIGHT, PATENT, TRADEMARK, OR OTHER RIGHT. IN NO EVENT SHALL THE
COPYRIGHT HOLDER BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
INCLUDING ANY GENERAL, SPECIAL, INDIRECT, INCIDENTAL, OR CONSEQUENTIAL
DAMAGES, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
FROM, OUT OF THE USE OR INABILITY TO USE THE FONT SOFTWARE OR FROM
OTHER DEALINGS IN THE FONT SOFTWARE.

File diff suppressed because one or more lines are too long

File diff suppressed because it is too large Load Diff

File diff suppressed because one or more lines are too long

File diff suppressed because it is too large Load Diff

After

Width:  |  Height:  |  Size: 182 KiB

View File

@ -21,18 +21,18 @@
<div class="btn-toolbar" role="toolbar" aria-label="Subscriptions toolbar">
<div class="btn-group btn-group-sm mr-2" role="group">
<button id="btn_create_sub" type="button" class="btn btn-secondary" >
<i class="material-icons" aria-hidden="true">add</i>
<span class="typcn typcn-plus" aria-hidden="true"></span>
</button>
<button id="btn_create_folder" type="button" class="btn btn-secondary">
<i class="material-icons" aria-hidden="true">create_new_folder</i>
<span class="typcn typcn-folder-add" aria-hidden="true"></span>
</button>
</div>
<div class="btn-group btn-group-sm mr-2" role="group">
<button id="btn_edit_node" type="button" class="btn btn-secondary" >
<i class="material-icons" aria-hidden="true">edit</i>
<span class="typcn typcn-edit" aria-hidden="true"></span>
</button>
<button id="btn_delete_node" type="button" class="btn btn-secondary" >
<i class="material-icons" aria-hidden="true">delete</i>
<span class="typcn typcn-trash" aria-hidden="true"></span>
</button>
</div>
</div>
@ -41,4 +41,10 @@
<p>Loading...</p>
</div>
{% endblock %}
{% block detail %}
<div id="main_detail">
</div>
{% endblock %}

View File

@ -279,10 +279,10 @@ function tree_Initialize()
},
types : {
folder : {
icon : "material-icons material-folder"
icon : "typcn typcn-folder"
},
sub : {
icon : "material-icons material-person",
icon : "typcn typcn-user",
max_depth : 0
}
},
@ -314,6 +314,13 @@ function tree_ValidateChange(operation, node, parent, position, more)
function tree_OnSelectionChanged(e, data)
{
node = data.instance.get_selected(true)[0];
$.post("{% url 'ajax_list_videos' %}", {
type: node.type,
id: node.id.replace('folder', '').replace('sub', ''),
csrfmiddlewaretoken: '{{ csrf_token }}'
}).done(function (result) {
$("#main_detail").html(result);
});
}
///

View File

@ -7,7 +7,7 @@
<title>{% block title %}YouTube Subscription Manager{% endblock %}</title>
<link rel="stylesheet" href="https://stackpath.bootstrapcdn.com/bootstrap/4.1.3/css/bootstrap.min.css" integrity="sha384-MCw98/SFnGE8fJT3GXwEOngsV7Zt27NXFoaoApmYm81iuXoPkFOJwJ8ERdknLPMO" crossorigin="anonymous">
<link href="https://fonts.googleapis.com/icon?family=Material+Icons" rel="stylesheet">
<link rel="stylesheet" href="{% static 'YtManagerApp/import/typicons/typicons.min.css' %}" />
<link rel="stylesheet" href="{% static 'YtManagerApp/css/style.css' %}">
{% block stylesheets %}
{% endblock %}

View File

@ -0,0 +1,38 @@
<div class="video-gallery container-fluid">
<div class="row">
{% for video in videos %}
<div class="card-wrapper col-12 col-sm-6 col-lg-4 col-xl-3 d-flex align-items-stretch">
<div class="card mx-auto">
<img class="card-img-top" src="{{ video.icon_best }}" alt="Thumbnail">
<div class="card-body">
<h5 class="card-title">
{% if not video.watched %}
<sup class="badge badge-primary">New</sup>
{% endif %}
{{ video.name }}
</h5>
<p class="card-text">{{ video.description | truncatechars:120 }}</p>
</div>
<div class="card-footer dropdown show">
<span class="typcn typcn-eye {{ video.watched | yesno:"video-icon-yes,video-icon-no" }}"
title="{{ video.watched | yesno:"Watched,Not watched" }}"></span>
<span class="typcn typcn-download {{ video.downloaded_path | yesno:"video-icon-yes,,video-icon-no" }}"
title="{{ video.downloaded_path | yesno:"Downloaded,,Not downloaded" }}"></span>
<small class="text-muted">{{ video.publish_date }}</small>
<a class="card-more float-right text-muted"
href="#" role="button" id="dropdownMenuLink"
data-toggle="dropdown" aria-haspopup="true" aria-expanded="false"><span class="typcn typcn-cog"></span></a>
<div class="dropdown-menu" aria-labelledby="dropdownMenuLink">
<a class="dropdown-item" href="#">Mark {{ video.watched | yesno:"not watched,watched" }}</a>
{% if video.downloaded_path %}
<a class="dropdown-item" href="#">Delete downloaded</a>
{% else %}
<a class="dropdown-item" href="#">Download</a>
{% endif %}
</div>
</div>
</div>
</div>
{% endfor %}
</div>
</div>

0
YtManagerApp/urls.py Normal file
View File

View File

View File

@ -0,0 +1,114 @@
import os
import os.path
import re
from configparser import Interpolation, NoSectionError, NoOptionError, InterpolationMissingOptionError, \
InterpolationDepthError, InterpolationSyntaxError, ConfigParser
MAX_INTERPOLATION_DEPTH = 10
class ExtendedInterpolatorWithEnv(Interpolation):
"""Advanced variant of interpolation, supports the syntax used by
`zc.buildout'. Enables interpolation between sections.
This modified version also allows specifying environment variables
using ${env:...}, and allows adding additional options using 'set_additional_options'. """
_KEYCRE = re.compile(r"\$\{([^}]+)\}")
def __init__(self, **kwargs):
self.__kwargs = kwargs
def set_additional_options(self, **kwargs):
self.__kwargs = kwargs
def before_get(self, parser, section, option, value, defaults):
L = []
self._interpolate_some(parser, option, L, value, section, defaults, 1)
return ''.join(L)
def before_set(self, parser, section, option, value):
tmp_value = value.replace('$$', '') # escaped dollar signs
tmp_value = self._KEYCRE.sub('', tmp_value) # valid syntax
if '$' in tmp_value:
raise ValueError("invalid interpolation syntax in %r at "
"position %d" % (value, tmp_value.find('$')))
return value
def _resolve_option(self, option, defaults):
if option in self.__kwargs:
return self.__kwargs[option]
return defaults[option]
def _resolve_section_option(self, section, option, parser):
if section == 'env':
return os.getenv(option, '')
return parser.get(section, option, raw=True)
def _interpolate_some(self, parser, option, accum, rest, section, map,
depth):
rawval = parser.get(section, option, raw=True, fallback=rest)
if depth > MAX_INTERPOLATION_DEPTH:
raise InterpolationDepthError(option, section, rawval)
while rest:
p = rest.find("$")
if p < 0:
accum.append(rest)
return
if p > 0:
accum.append(rest[:p])
rest = rest[p:]
# p is no longer used
c = rest[1:2]
if c == "$":
accum.append("$")
rest = rest[2:]
elif c == "{":
m = self._KEYCRE.match(rest)
if m is None:
raise InterpolationSyntaxError(option, section,
"bad interpolation variable reference %r" % rest)
path = m.group(1).split(':')
rest = rest[m.end():]
sect = section
opt = option
try:
if len(path) == 1:
opt = parser.optionxform(path[0])
v = self._resolve_option(opt, map)
elif len(path) == 2:
sect = path[0]
opt = parser.optionxform(path[1])
v = self._resolve_section_option(sect, opt, parser)
else:
raise InterpolationSyntaxError(
option, section,
"More than one ':' found: %r" % (rest,))
except (KeyError, NoSectionError, NoOptionError):
raise InterpolationMissingOptionError(
option, section, rawval, ":".join(path)) from None
if "$" in v:
self._interpolate_some(parser, opt, accum, v, sect,
dict(parser.items(sect, raw=True)),
depth + 1)
else:
accum.append(v)
else:
raise InterpolationSyntaxError(
option, section,
"'$' must be followed by '$' or '{', "
"found: %r" % (rest,))
class ConfigParserWithEnv(ConfigParser):
_DEFAULT_INTERPOLATION = ExtendedInterpolatorWithEnv()
def set_additional_interpolation_options(self, **kwargs):
"""
Sets additional options to be used in interpolation.
Only works with ExtendedInterpolatorWithEnv
:param kwargs:
:return:
"""
if isinstance(super()._interpolation, ExtendedInterpolatorWithEnv):
super()._interpolation.set_additional_options(**kwargs)

View File

@ -20,7 +20,8 @@ def get_children_recurse(parent_id):
children.append({
"id": "sub" + str(sub.id),
"type": "sub",
"text": sub.name
"text": sub.name,
"icon": sub.icon_default
})
return children
@ -82,6 +83,21 @@ def ajax_delete_subscription(request: HttpRequest, sid):
return HttpResponse()
def ajax_list_videos(request: HttpRequest):
if request.method == 'POST':
type = request.POST['type']
id = request.POST['id']
context = {}
if type == 'sub':
context['videos'] = SubscriptionManager.list_videos(int(id))
else:
context['videos'] = FolderManager.list_videos(int(id))
return render(request, 'YtManagerApp/main_videos.html', context)
def index(request: HttpRequest):
context = {}
return render(request, 'YtManagerApp/index.html', context)

61
config/config.ini Normal file
View File

@ -0,0 +1,61 @@
; Use $<env:environment_variable> to use the value of an environment variable.
; The global section contains settings that apply to the entire server
[global]
; YouTube API key - get this from your user account
YoutubeApiKey=AIzaSyBabzE4Bup77WexdLMa9rN9z-wJidEfNX8
; Specifies the synchronization schedule, in crontab format.
; Format: <minute> <hour> <day-of-month> <month-of-year> <year>
SynchronizationSchedule=0 * * * * *
; Number of threads running the scheduler
; Since most of the jobs scheduled are downloads, there is no advantage to having
; a higher concurrency
SchedulerConcurrency=2
; Log level
LogLevel=DEBUG
; Default user settings
[user]
; When a video is deleted on the system, it will be marked as 'watched'
MarkDeletedAsWatched=True
; Videos marked as watched are automatically deleted
DeleteWatched=True
; Enable automatic downloading
AutoDownload=True
; Limit the total number of videos downloaded (-1 or empty = no limit)
DownloadGlobalLimit=
; Limit the numbers of videos per subscription (-1 or empty = no limit)
DownloadSubscriptionLimit=5
; Number of download attempts
DownloadMaxAttempts=3
; Download order
; Options: playlist_index, publish_date, name.
; Use - to reverse order (e.g. -publish_date means to order by publish date descending)
DownloadOrder=playlist_index
; Path where downloaded videos are stored
;DownloadPath=${env:USERPROFILE}${env:HOME}/Downloads
DownloadPath=D:\\Dev\\youtube-channel-manager\\temp\\download
; A pattern which describes how downloaded files are organized. Extensions are automatically appended.
; Supported fields: channel, channel_id, playlist, playlist_id, playlist_index, title, id
; The default pattern should work pretty well with Plex
DownloadFilePattern=${channel}/${playlist}/S01E${playlist_index} - ${title} [${id}]
; Download format that will be passed to youtube-dl. See the youtube-dl documentation for more details.
DownloadFormat=bestvideo+bestaudio
; Subtitles - these options match the youtube-dl options
DownloadSubtitles=True
DownloadAutogeneratedSubtitles=False
DownloadSubtitlesAll=False
DownloadSubtitlesLangs=en,ro
DownloadSubtitlesFormat=

61
config/config.ini.default Normal file
View File

@ -0,0 +1,61 @@
; Use $<env:environment_variable> to use the value of an environment variable.
; The global section contains settings that apply to the entire server
[global]
; YouTube API key - get this from your user account
YoutubeApiKey=AIzaSyBabzE4Bup77WexdLMa9rN9z-wJidEfNX8
; Specifies the synchronization schedule, in crontab format.
; Format: <minute> <hour> <day-of-month> <month-of-year> <year>
SynchronizationSchedule=0 * * * * *
; Number of threads running the scheduler
; Since most of the jobs scheduled are downloads, there is no advantage to having
; a higher concurrency
SchedulerConcurrency=2
; Log level
LogLevel=DEBUG
; Default user settings
[user]
; When a video is deleted on the system, it will be marked as 'watched'
MarkDeletedAsWatched=True
; Videos marked as watched are automatically deleted
DeleteWatched=True
; Enable automatic downloading
AutoDownload=True
; Limit the total number of videos downloaded (-1 or empty = no limit)
DownloadGlobalLimit=
; Limit the numbers of videos per subscription (-1 or empty = no limit)
DownloadSubscriptionLimit=5
; Number of download attempts
DownloadMaxAttempts=3
; Download order
; Options: playlist_index, publish_date, name.
; Use - to reverse order (e.g. -publish_date means to order by publish date descending)
DownloadOrder=playlist_index
; Path where downloaded videos are stored
;DownloadPath=${env:USERPROFILE}${env:HOME}/Downloads
DownloadPath=D:\\Dev\\youtube-channel-manager\\temp\\download
; A pattern which describes how downloaded files are organized. Extensions are automatically appended.
; Supported fields: channel, channel_id, playlist, playlist_id, playlist_index, title, id
; The default pattern should work pretty well with Plex
DownloadFilePattern=${channel}/${playlist}/S01E${playlist_index} - ${title} [${id}]
; Download format that will be passed to youtube-dl. See the youtube-dl documentation for more details.
DownloadFormat=bestvideo+bestaudio
; Subtitles - these options match the youtube-dl options
DownloadSubtitles=True
DownloadAutogeneratedSubtitles=False
DownloadSubtitlesAll=False
DownloadSubtitlesLangs=en,ro
DownloadSubtitlesFormat=