278 lines
10 KiB
Python
278 lines
10 KiB
Python
from YtManagerApp.models import SubscriptionFolder, Subscription, Video, Channel
|
|
from YtManagerApp.utils.youtube import YoutubeAPI, YoutubeChannelInfo, YoutubePlaylistItem
|
|
from django.conf import settings
|
|
from apscheduler.schedulers.background import BackgroundScheduler
|
|
import os
|
|
import os.path
|
|
import requests
|
|
from urllib.parse import urljoin
|
|
import mimetypes
|
|
import youtube_dl
|
|
|
|
from YtManagerApp.scheduler import instance as scheduler
|
|
from YtManagerApp.appconfig import instance as app_config
|
|
from apscheduler.triggers.cron import CronTrigger
|
|
|
|
|
|
class FolderManager(object):
|
|
|
|
@staticmethod
|
|
def create_or_edit(fid, name, parent_id):
|
|
# Create or edit
|
|
if fid == '#':
|
|
folder = SubscriptionFolder()
|
|
else:
|
|
folder = SubscriptionFolder.objects.get(id=int(fid))
|
|
|
|
# Set attributes
|
|
folder.name = name
|
|
if parent_id == '#':
|
|
folder.parent = None
|
|
else:
|
|
folder.parent = SubscriptionFolder.objects.get(id=int(parent_id))
|
|
|
|
FolderManager.__validate(folder)
|
|
folder.save()
|
|
|
|
@staticmethod
|
|
def __validate(folder: SubscriptionFolder):
|
|
# Make sure folder name is unique in the parent folder
|
|
for dbFolder in SubscriptionFolder.objects.filter(parent_id=folder.parent_id):
|
|
if dbFolder.id != folder.id and dbFolder.name == folder.name:
|
|
raise ValueError('Folder name is not unique!')
|
|
|
|
# Prevent parenting loops
|
|
current = folder
|
|
visited = []
|
|
|
|
while not (current is None):
|
|
if current in visited:
|
|
raise ValueError('Parenting cycle detected!')
|
|
visited.append(current)
|
|
current = current.parent
|
|
|
|
@staticmethod
|
|
def delete(fid: int):
|
|
folder = SubscriptionFolder.objects.get(id=fid)
|
|
folder.delete()
|
|
|
|
@staticmethod
|
|
def list_videos(fid: int):
|
|
folder = SubscriptionFolder.objects.get(id=fid)
|
|
folder_list = []
|
|
queue = [folder]
|
|
while len(queue) > 0:
|
|
folder = queue.pop()
|
|
folder_list.append(folder)
|
|
queue.extend(SubscriptionFolder.objects.filter(parent=folder))
|
|
|
|
return Video.objects.filter(subscription__parent_folder__in=folder_list).order_by('-publish_date')
|
|
|
|
|
|
class SubscriptionManager(object):
|
|
__scheduler = BackgroundScheduler()
|
|
|
|
@staticmethod
|
|
def create_or_edit(sid, url, name, parent_id):
|
|
# Create or edit
|
|
if sid == '#':
|
|
SubscriptionManager.create(url, parent_id, YoutubeAPI.build_public())
|
|
else:
|
|
sub = Subscription.objects.get(id=int(sid))
|
|
sub.name = name
|
|
|
|
if parent_id == '#':
|
|
sub.parent_folder = None
|
|
else:
|
|
sub.parent_folder = SubscriptionFolder.objects.get(id=int(parent_id))
|
|
|
|
sub.save()
|
|
|
|
@staticmethod
|
|
def create(url, parent_id, yt_api: YoutubeAPI):
|
|
sub = Subscription()
|
|
# Set parent
|
|
if parent_id == '#':
|
|
sub.parent_folder = None
|
|
else:
|
|
sub.parent_folder = SubscriptionFolder.objects.get(id=int(parent_id))
|
|
|
|
# Pull information about the channel and playlist
|
|
url_type, url_id = yt_api.parse_channel_url(url)
|
|
|
|
if url_type == 'playlist_id':
|
|
info_playlist = yt_api.get_playlist_info(url_id)
|
|
channel = SubscriptionManager.__get_or_create_channel('channel_id', info_playlist.getChannelId(), yt_api)
|
|
sub.name = info_playlist.getTitle()
|
|
sub.playlist_id = info_playlist.getId()
|
|
sub.description = info_playlist.getDescription()
|
|
sub.channel = channel
|
|
sub.icon_default = info_playlist.getDefaultThumbnailUrl()
|
|
sub.icon_best = info_playlist.getBestThumbnailUrl()
|
|
|
|
else:
|
|
channel = SubscriptionManager.__get_or_create_channel(url_type, url_id, yt_api)
|
|
# No point in getting the 'uploads' playlist info
|
|
sub.name = channel.name
|
|
sub.playlist_id = channel.upload_playlist_id
|
|
sub.description = channel.description
|
|
sub.channel = channel
|
|
sub.icon_default = channel.icon_default
|
|
sub.icon_best = channel.icon_best
|
|
|
|
sub.save()
|
|
|
|
@staticmethod
|
|
def list_videos(fid: int):
|
|
sub = Subscription.objects.get(id=fid)
|
|
return Video.objects.filter(subscription=sub).order_by('playlist_index')
|
|
|
|
@staticmethod
|
|
def __get_or_create_channel(url_type, url_id, yt_api: YoutubeAPI):
|
|
|
|
channel: Channel = None
|
|
info_channel: YoutubeChannelInfo = None
|
|
|
|
if url_type == 'user':
|
|
channel = Channel.find_by_username(url_id)
|
|
if not channel:
|
|
info_channel = yt_api.get_channel_info_by_username(url_id)
|
|
channel = Channel.find_by_channel_id(info_channel.getId())
|
|
|
|
elif url_type == 'channel_id':
|
|
channel = Channel.find_by_channel_id(url_id)
|
|
if not channel:
|
|
info_channel = yt_api.get_channel_info(url_id)
|
|
|
|
elif url_type == 'channel_custom':
|
|
channel = Channel.find_by_custom_url(url_id)
|
|
if not channel:
|
|
found_channel_id = yt_api.search_channel(url_id)
|
|
channel = Channel.find_by_channel_id(found_channel_id)
|
|
if not channel:
|
|
info_channel = yt_api.get_channel_info(found_channel_id)
|
|
|
|
# Store information about the channel
|
|
if info_channel:
|
|
if not channel:
|
|
channel = Channel()
|
|
if url_type == 'user':
|
|
channel.username = url_id
|
|
SubscriptionManager.__update_channel(channel, info_channel)
|
|
|
|
return channel
|
|
|
|
@staticmethod
|
|
def __update_channel(channel: Channel, yt_info: YoutubeChannelInfo):
|
|
channel.channel_id = yt_info.getId()
|
|
channel.custom_url = yt_info.getCustomUrl()
|
|
channel.name = yt_info.getTitle()
|
|
channel.description = yt_info.getDescription()
|
|
channel.icon_default = yt_info.getDefaultThumbnailUrl()
|
|
channel.icon_best = yt_info.getBestThumbnailUrl()
|
|
channel.upload_playlist_id = yt_info.getUploadsPlaylist()
|
|
channel.save()
|
|
|
|
@staticmethod
|
|
def __create_video(yt_video: YoutubePlaylistItem, subscription: Subscription):
|
|
video = Video()
|
|
video.video_id = yt_video.getVideoId()
|
|
video.name = yt_video.getTitle()
|
|
video.description = yt_video.getDescription()
|
|
video.watched = False
|
|
video.downloaded_path = None
|
|
video.subscription = subscription
|
|
video.playlist_index = yt_video.getPlaylistIndex()
|
|
video.publish_date = yt_video.getPublishDate()
|
|
video.icon_default = yt_video.getDefaultThumbnailUrl()
|
|
video.icon_best = yt_video.getBestThumbnailUrl()
|
|
video.save()
|
|
|
|
@staticmethod
|
|
def __synchronize(subscription: Subscription, yt_api: YoutubeAPI):
|
|
# Get list of videos
|
|
for video in yt_api.list_playlist_videos(subscription.playlist_id):
|
|
results = Video.objects.filter(video_id=video.getVideoId(), subscription=subscription)
|
|
if len(results) == 0:
|
|
print('New video for subscription "', subscription, '": ', video.getVideoId(), video.getTitle())
|
|
SubscriptionManager.__create_video(video, subscription)
|
|
|
|
@staticmethod
|
|
def __synchronize_all():
|
|
print("Running scheduled synchronization... ")
|
|
|
|
# Sync subscribed playlists/channels
|
|
yt_api = YoutubeAPI.build_public()
|
|
for subscription in Subscription.objects.all():
|
|
SubscriptionManager.__synchronize(subscription, yt_api)
|
|
|
|
# Fetch thumbnails
|
|
print("Fetching channel thumbnails... ")
|
|
for ch in Channel.objects.filter(icon_default__istartswith='http'):
|
|
ch.icon_default = SubscriptionManager.__fetch_thumbnail(ch.icon_default, 'channel', ch.channel_id, 'default')
|
|
ch.save()
|
|
|
|
for ch in Channel.objects.filter(icon_best__istartswith='http'):
|
|
ch.icon_best = SubscriptionManager.__fetch_thumbnail(ch.icon_best, 'channel', ch.channel_id, 'best')
|
|
ch.save()
|
|
|
|
print("Fetching subscription thumbnails... ")
|
|
for sub in Subscription.objects.filter(icon_default__istartswith='http'):
|
|
sub.icon_default = SubscriptionManager.__fetch_thumbnail(sub.icon_default, 'sub', sub.playlist_id, 'default')
|
|
sub.save()
|
|
|
|
for sub in Subscription.objects.filter(icon_best__istartswith='http'):
|
|
sub.icon_best = SubscriptionManager.__fetch_thumbnail(sub.icon_best, 'sub', sub.playlist_id, 'best')
|
|
sub.save()
|
|
|
|
print("Fetching video thumbnails... ")
|
|
for vid in Video.objects.filter(icon_default__istartswith='http'):
|
|
vid.icon_default = SubscriptionManager.__fetch_thumbnail(vid.icon_default, 'video', vid.video_id, 'default')
|
|
vid.save()
|
|
|
|
for vid in Video.objects.filter(icon_best__istartswith='http'):
|
|
vid.icon_best = SubscriptionManager.__fetch_thumbnail(vid.icon_best, 'video', vid.video_id, 'best')
|
|
vid.save()
|
|
|
|
print("Downloading videos...")
|
|
Downloader.download_all()
|
|
|
|
print("Synchronization finished.")
|
|
|
|
@staticmethod
|
|
def __fetch_thumbnail(url, object_type, identifier, quality):
|
|
|
|
# Make request to obtain mime type
|
|
response = requests.get(url, stream=True)
|
|
ext = mimetypes.guess_extension(response.headers['Content-Type'])
|
|
|
|
# Build file path
|
|
file_name = f"{identifier}-{quality}{ext}"
|
|
abs_path_dir = os.path.join(settings.MEDIA_ROOT, "thumbs", object_type)
|
|
abs_path = os.path.join(abs_path_dir, file_name)
|
|
|
|
# Store image
|
|
os.makedirs(abs_path_dir, exist_ok=True)
|
|
with open(abs_path, "wb") as f:
|
|
for chunk in response.iter_content(chunk_size=1024):
|
|
if chunk:
|
|
f.write(chunk)
|
|
|
|
# Return
|
|
media_url = urljoin(settings.MEDIA_URL, f"thumbs/{object_type}/{file_name}")
|
|
return media_url
|
|
|
|
@staticmethod
|
|
def start_scheduler():
|
|
SubscriptionManager.__scheduler.add_job(SubscriptionManager.__synchronize_all, 'cron',
|
|
hour='*', minute=38, max_instances=1)
|
|
SubscriptionManager.__scheduler.start()
|
|
|
|
|
|
def setup_synchronization_job():
|
|
trigger = CronTrigger.from_crontab(app_config.get('global', 'SynchronizationSchedule'))
|
|
scheduler.add_job(synchronize_all, trigger, max_instances=1)
|
|
|
|
|
|
def synchronize_all():
|
|
pass |