chibicitiberiu/ytsm
1
0
Fork 0
mirror of https://github.com/chibicitiberiu/ytsm.git synced 2024-02-24 05:43:31 +00:00
Code Issues Releases Wiki Activity

Work on big refactor

Browse Source
This commit is contained in:
Tiberiu Chibici
2018-10-13 23:01:45 +03:00
parent ae77251883
commit 1d5c7ea24b
447 changed files with 67198 additions and 1226 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
YtManagerApp/management/downloader.py
View File

@ -1,4 +1,4 @@
from YtManagerApp.appconfig import instance as app_config
from YtManagerApp import appconfig
from YtManagerApp.management.jobs.download_video import schedule_download_video
from YtManagerApp.models import Video, Subscription
from django.conf import settings
@ -12,7 +12,7 @@ log = logging.getLogger('downloader')
def __get_subscription_config(sub: Subscription):
user_config = app_config.get_user_config(sub.user)
user_config = appconfig.get_user_config(sub.user)
enabled = sub.auto_download
if enabled is None:

37
YtManagerApp/management/folders.py
View File

@ -0,0 +1,37 @@
from YtManagerApp.models import SubscriptionFolder, Subscription
from typing import Callable, Union, Any, Optional
from django.contrib.auth.models import User
import logging
def traverse_tree(root_folder_id: Optional[int], user: User, visit_func: Callable[[Union[SubscriptionFolder, Subscription]], Any]):
data_collected = []
def collect(data):
if data is not None:
data_collected.append(data)
# Visit root
if root_folder_id is not None:
root_folder = SubscriptionFolder.objects.get(id = root_folder_id)
collect(visit_func(root_folder))
queue = [root_folder_id]
visited = []
while len(queue) > 0:
folder_id = queue.pop()
if folder_id in visited:
logging.error('Found folder tree cycle for folder id %d.', folder_id)
continue
visited.append(folder_id)
for folder in SubscriptionFolder.objects.filter(parent_id=folder_id, user=user).order_by('name'):
collect(visit_func(folder))
queue.append(folder.id)
for subscription in Subscription.objects.filter(parent_folder_id=folder_id, user=user).order_by('name'):
collect(visit_func(subscription))
return data_collected

10
YtManagerApp/management/jobs/download_video.py
View File

@ -1,6 +1,6 @@
from YtManagerApp.models import Video
from YtManagerApp.scheduler import instance as scheduler
from YtManagerApp.appconfig import instance as app_config
from YtManagerApp import scheduler
from YtManagerApp.appconfig import get_user_config
import os
import youtube_dl
import logging
@ -60,7 +60,7 @@ def download_video(video: Video, attempt: int = 1):
log.info('Downloading video %d [%s %s]', video.id, video.video_id, video.name)
user_config = app_config.get_user_config(video.subscription.user)
user_config = get_user_config(video.subscription.user)
max_attempts = user_config.getint('user', 'DownloadMaxAttempts', fallback=3)
youtube_dl_params, output_path = __build_youtube_dl_params(video, user_config)
@ -76,7 +76,7 @@ def download_video(video: Video, attempt: int = 1):
elif attempt <= max_attempts:
log.warning('Re-enqueueing video (attempt %d/%d)', attempt, max_attempts)
scheduler.add_job(download_video, args=[video, attempt + 1])
scheduler.instance.add_job(download_video, args=[video, attempt + 1])
else:
log.error('Multiple attempts to download video %d [%s %s] failed!', video.id, video.video_id, video.name)
@ -90,4 +90,4 @@ def schedule_download_video(video: Video):
:param video:
:return:
"""
scheduler.add_job(download_video, args=[video, 1])
scheduler.instance.add_job(download_video, args=[video, 1])

88
YtManagerApp/management/jobs/synchronize.py
View File

@ -1,46 +1,68 @@
from YtManagerApp.models import *
import logging
from apscheduler.triggers.cron import CronTrigger
from YtManagerApp.appconfig import settings
from YtManagerApp.management.downloader import fetch_thumbnail, downloader_process_all
from YtManagerApp.management.videos import create_video
from YtManagerApp.models import *
from YtManagerApp import scheduler
from YtManagerApp.utils.youtube import YoutubeAPI
log = logging.getLogger('sync')
def __synchronize_sub(subscription: Subscription, yt_api: YoutubeAPI):
# Get list of videos
for video in yt_api.list_playlist_videos(subscription.playlist_id):
results = Video.objects.filter(video_id=video.getVideoId(), subscription=subscription)
if len(results) == 0:
log.info('New video for subscription "', subscription, '": ', video.getVideoId(), video.getTitle())
create_video(video, subscription)
def __fetch_thumbnails_obj(iterable, obj_type, id_attr):
for obj in iterable:
if obj.icon_default.startswith("http"):
obj.icon_default = fetch_thumbnail(obj.icon_default, obj_type, getattr(obj, id_attr), 'default')
if obj.icon_best.startswith("http"):
obj.icon_best = fetch_thumbnail(obj.icon_best, obj_type, getattr(obj, id_attr), 'best')
obj.save()
def __fetch_thumbnails():
# Fetch thumbnails
log.info("Fetching channel thumbnails... ")
__fetch_thumbnails_obj(Channel.objects.filter(icon_default__istartswith='http'), 'channel', 'channel_id')
__fetch_thumbnails_obj(Channel.objects.filter(icon_best__istartswith='http'), 'channel', 'channel_id')
log.info("Fetching subscription thumbnails... ")
__fetch_thumbnails_obj(Subscription.objects.filter(icon_default__istartswith='http'), 'sub', 'playlist_id')
__fetch_thumbnails_obj(Subscription.objects.filter(icon_best__istartswith='http'), 'sub', 'playlist_id')
log.info("Fetching video thumbnails... ")
__fetch_thumbnails_obj(Video.objects.filter(icon_default__istartswith='http'), 'video', 'video_id')
__fetch_thumbnails_obj(Video.objects.filter(icon_best__istartswith='http'), 'video', 'video_id')
def synchronize():
logger = logging.getLogger('sync')
logger.info("Running scheduled synchronization... ")
log.info("Running scheduled synchronization... ")
# Sync subscribed playlists/channels
log.info("Sync - checking for new videos")
yt_api = YoutubeAPI.build_public()
for subscription in Subscription.objects.all():
SubscriptionManager.__synchronize(subscription, yt_api)
__synchronize_sub(subscription, yt_api)
# Fetch thumbnails
logger.info("Fetching channel thumbnails... ")
for ch in Channel.objects.filter(icon_default__istartswith='http'):
ch.icon_default = SubscriptionManager.__fetch_thumbnail(ch.icon_default, 'channel', ch.channel_id, 'default')
ch.save()
log.info("Sync - checking for videos to download")
downloader_process_all()
for ch in Channel.objects.filter(icon_best__istartswith='http'):
ch.icon_best = SubscriptionManager.__fetch_thumbnail(ch.icon_best, 'channel', ch.channel_id, 'best')
ch.save()
log.info("Sync - fetching missing thumbnails")
__fetch_thumbnails()
logger.info("Fetching subscription thumbnails... ")
for sub in Subscription.objects.filter(icon_default__istartswith='http'):
sub.icon_default = SubscriptionManager.__fetch_thumbnail(sub.icon_default, 'sub', sub.playlist_id, 'default')
sub.save()
log.info("Synchronization finished.")
for sub in Subscription.objects.filter(icon_best__istartswith='http'):
sub.icon_best = SubscriptionManager.__fetch_thumbnail(sub.icon_best, 'sub', sub.playlist_id, 'best')
sub.save()
logger.info("Fetching video thumbnails... ")
for vid in Video.objects.filter(icon_default__istartswith='http'):
vid.icon_default = SubscriptionManager.__fetch_thumbnail(vid.icon_default, 'video', vid.video_id, 'default')
vid.save()
for vid in Video.objects.filter(icon_best__istartswith='http'):
vid.icon_best = SubscriptionManager.__fetch_thumbnail(vid.icon_best, 'video', vid.video_id, 'best')
vid.save()
print("Downloading videos...")
Downloader.download_all()
print("Synchronization finished.")
def schedule_synchronize():
trigger = CronTrigger.from_crontab(settings.get('global', 'SynchronizationSchedule'))
scheduler.instance.add_job(synchronize, trigger, max_instances=1)

118
YtManagerApp/management/management.py
View File

@ -1,17 +1,7 @@
from YtManagerApp.models import SubscriptionFolder, Subscription, Video, Channel
from YtManagerApp.utils.youtube import YoutubeAPI, YoutubeChannelInfo, YoutubePlaylistItem
from django.conf import settings
from apscheduler.schedulers.background import BackgroundScheduler
import os
import os.path
import requests
from urllib.parse import urljoin
import mimetypes
import youtube_dl
from YtManagerApp.scheduler import instance as scheduler
from YtManagerApp.appconfig import instance as app_config
from apscheduler.triggers.cron import CronTrigger
from YtManagerApp.models import SubscriptionFolder, Subscription, Video, Channel
from YtManagerApp.utils.youtube import YoutubeAPI, YoutubeChannelInfo
class FolderManager(object):
@ -172,107 +162,3 @@ class SubscriptionManager(object):
channel.icon_best = yt_info.getBestThumbnailUrl()
channel.upload_playlist_id = yt_info.getUploadsPlaylist()
channel.save()
@staticmethod
def __create_video(yt_video: YoutubePlaylistItem, subscription: Subscription):
video = Video()
video.video_id = yt_video.getVideoId()
video.name = yt_video.getTitle()
video.description = yt_video.getDescription()
video.watched = False
video.downloaded_path = None
video.subscription = subscription
video.playlist_index = yt_video.getPlaylistIndex()
video.publish_date = yt_video.getPublishDate()
video.icon_default = yt_video.getDefaultThumbnailUrl()
video.icon_best = yt_video.getBestThumbnailUrl()
video.save()
@staticmethod
def __synchronize(subscription: Subscription, yt_api: YoutubeAPI):
# Get list of videos
for video in yt_api.list_playlist_videos(subscription.playlist_id):
results = Video.objects.filter(video_id=video.getVideoId(), subscription=subscription)
if len(results) == 0:
print('New video for subscription "', subscription, '": ', video.getVideoId(), video.getTitle())
SubscriptionManager.__create_video(video, subscription)
@staticmethod
def __synchronize_all():
print("Running scheduled synchronization... ")
# Sync subscribed playlists/channels
yt_api = YoutubeAPI.build_public()
for subscription in Subscription.objects.all():
SubscriptionManager.__synchronize(subscription, yt_api)
# Fetch thumbnails
print("Fetching channel thumbnails... ")
for ch in Channel.objects.filter(icon_default__istartswith='http'):
ch.icon_default = SubscriptionManager.__fetch_thumbnail(ch.icon_default, 'channel', ch.channel_id, 'default')
ch.save()
for ch in Channel.objects.filter(icon_best__istartswith='http'):
ch.icon_best = SubscriptionManager.__fetch_thumbnail(ch.icon_best, 'channel', ch.channel_id, 'best')
ch.save()
print("Fetching subscription thumbnails... ")
for sub in Subscription.objects.filter(icon_default__istartswith='http'):
sub.icon_default = SubscriptionManager.__fetch_thumbnail(sub.icon_default, 'sub', sub.playlist_id, 'default')
sub.save()
for sub in Subscription.objects.filter(icon_best__istartswith='http'):
sub.icon_best = SubscriptionManager.__fetch_thumbnail(sub.icon_best, 'sub', sub.playlist_id, 'best')
sub.save()
print("Fetching video thumbnails... ")
for vid in Video.objects.filter(icon_default__istartswith='http'):
vid.icon_default = SubscriptionManager.__fetch_thumbnail(vid.icon_default, 'video', vid.video_id, 'default')
vid.save()
for vid in Video.objects.filter(icon_best__istartswith='http'):
vid.icon_best = SubscriptionManager.__fetch_thumbnail(vid.icon_best, 'video', vid.video_id, 'best')
vid.save()
print("Downloading videos...")
Downloader.download_all()
print("Synchronization finished.")
@staticmethod
def __fetch_thumbnail(url, object_type, identifier, quality):
# Make request to obtain mime type
response = requests.get(url, stream=True)
ext = mimetypes.guess_extension(response.headers['Content-Type'])
# Build file path
file_name = f"{identifier}-{quality}{ext}"
abs_path_dir = os.path.join(settings.MEDIA_ROOT, "thumbs", object_type)
abs_path = os.path.join(abs_path_dir, file_name)
# Store image
os.makedirs(abs_path_dir, exist_ok=True)
with open(abs_path, "wb") as f:
for chunk in response.iter_content(chunk_size=1024):
if chunk:
f.write(chunk)
# Return
media_url = urljoin(settings.MEDIA_URL, f"thumbs/{object_type}/{file_name}")
return media_url
@staticmethod
def start_scheduler():
SubscriptionManager.__scheduler.add_job(SubscriptionManager.__synchronize_all, 'cron',
hour='*', minute=38, max_instances=1)
SubscriptionManager.__scheduler.start()
def setup_synchronization_job():
trigger = CronTrigger.from_crontab(app_config.get('global', 'SynchronizationSchedule'))
scheduler.add_job(synchronize_all, trigger, max_instances=1)
def synchronize_all():
pass

72
YtManagerApp/management/videos.py Normal file
View File

@ -0,0 +1,72 @@
from YtManagerApp.models import Subscription, Video
from YtManagerApp.utils.youtube import YoutubePlaylistItem
from typing import Optional
import re
from django.db.models import Q
from YtManagerApp.management.folders import traverse_tree
from django.contrib.auth.models import User
def create_video(yt_video: YoutubePlaylistItem, subscription: Subscription):
video = Video()
video.video_id = yt_video.getVideoId()
video.name = yt_video.getTitle()
video.description = yt_video.getDescription()
video.watched = False
video.downloaded_path = None
video.subscription = subscription
video.playlist_index = yt_video.getPlaylistIndex()
video.publish_date = yt_video.getPublishDate()
video.icon_default = yt_video.getDefaultThumbnailUrl()
video.icon_best = yt_video.getBestThumbnailUrl()
video.save()
def get_videos(user: User,
sort_order: Optional[str],
query: Optional[str] = None,
subscription_id: Optional[int] = None,
folder_id: Optional[int] = None,
only_watched: Optional[bool] = None,
only_downloaded: Optional[bool] = None,
):
filter_args = []
filter_kwargs = {
'subscription__user': user
}
# Process query string - basically, we break it down into words,
# and then search for the given text in the name, description, uploader name and subscription name
if query is not None:
for match in re.finditer(r'\w+', query):
word = match[0]
filter_args.append(Q(name__icontains=word)
| Q(description__icontains=word)
| Q(uploader_name__icontains=word)
| Q(subscription__name__icontains=word))
# Subscription id
if subscription_id is not None:
filter_kwargs['subscription_id'] = subscription_id
# Folder id
if folder_id is not None:
# Visit function - returns only the subscription IDs
def visit(node):
if isinstance(node, Subscription):
return node.id
return None
filter_kwargs['subscription_id__in'] = traverse_tree(folder_id, user, visit)
# Only watched
if only_watched is not None:
filter_kwargs['watched'] = only_watched
# Only downloaded
# - not downloaded (False) -> is null (True)
# - downloaded (True) -> is not null (False)
if only_downloaded is not None:
filter_kwargs['downloaded_path__isnull'] = not only_downloaded
return Video.objects.filter(*filter_args, **filter_kwargs).order_by(sort_order)