mirror of
https://github.com/chibicitiberiu/ytsm.git
synced 2024-02-24 05:43:31 +00:00
Integrated pytaw library for youtube API.
This commit is contained in:
@ -35,4 +35,5 @@ def schedule_delete_video(video: Video):
|
||||
:param video:
|
||||
:return:
|
||||
"""
|
||||
scheduler.instance.add_job(delete_video, args=[video])
|
||||
job = scheduler.scheduler.add_job(delete_video, args=[video])
|
||||
log.info('Scheduled delete video job video=(%s), job=%s', video, job.id)
|
||||
|
||||
@ -25,8 +25,8 @@ def __get_valid_path(path):
|
||||
def __build_youtube_dl_params(video: Video):
|
||||
# resolve path
|
||||
pattern_dict = {
|
||||
'channel': video.subscription.channel.name,
|
||||
'channel_id': video.subscription.channel.channel_id,
|
||||
'channel': video.subscription.channel_name,
|
||||
'channel_id': video.subscription.channel_id,
|
||||
'playlist': video.subscription.name,
|
||||
'playlist_id': video.subscription.playlist_id,
|
||||
'playlist_index': "{:03d}".format(1 + video.playlist_index),
|
||||
@ -88,7 +88,7 @@ def download_video(video: Video, attempt: int = 1):
|
||||
|
||||
elif attempt <= max_attempts:
|
||||
log.warning('Re-enqueueing video (attempt %d/%d)', attempt, max_attempts)
|
||||
scheduler.instance.add_job(download_video, args=[video, attempt + 1])
|
||||
__schedule_download_video(video, attempt + 1)
|
||||
|
||||
else:
|
||||
log.error('Multiple attempts to download video %d [%s %s] failed!', video.id, video.video_id, video.name)
|
||||
@ -96,10 +96,15 @@ def download_video(video: Video, attempt: int = 1):
|
||||
video.save()
|
||||
|
||||
|
||||
def __schedule_download_video(video: Video, attempt=1):
|
||||
job = scheduler.scheduler.add_job(download_video, args=[video, attempt])
|
||||
log.info('Scheduled download video job video=(%s), attempt=%d, job=%s', video, attempt, job.id)
|
||||
|
||||
|
||||
def schedule_download_video(video: Video):
|
||||
"""
|
||||
Schedules a download video job to run immediately.
|
||||
:param video:
|
||||
:return:
|
||||
"""
|
||||
scheduler.instance.add_job(download_video, args=[video, 1])
|
||||
__schedule_download_video(video)
|
||||
|
||||
@ -7,9 +7,8 @@ from apscheduler.triggers.cron import CronTrigger
|
||||
from YtManagerApp import scheduler
|
||||
from YtManagerApp.appconfig import settings
|
||||
from YtManagerApp.management.downloader import fetch_thumbnail, downloader_process_all, downloader_process_subscription
|
||||
from YtManagerApp.management.videos import create_video
|
||||
from YtManagerApp.models import *
|
||||
from YtManagerApp.utils.youtube import YoutubeAPI
|
||||
from YtManagerApp.utils import youtube
|
||||
|
||||
log = logging.getLogger('sync')
|
||||
__lock = Lock()
|
||||
@ -17,23 +16,25 @@ __lock = Lock()
|
||||
_ENABLE_UPDATE_STATS = False
|
||||
|
||||
|
||||
def __check_new_videos_sub(subscription: Subscription, yt_api: YoutubeAPI):
|
||||
def __check_new_videos_sub(subscription: Subscription, yt_api: youtube.YoutubeAPI):
|
||||
# Get list of videos
|
||||
for video in yt_api.list_playlist_videos(subscription.playlist_id):
|
||||
results = Video.objects.filter(video_id=video.getVideoId(), subscription=subscription)
|
||||
for item in yt_api.playlist_items(subscription.playlist_id):
|
||||
results = Video.objects.filter(video_id=item.resource_video_id, subscription=subscription)
|
||||
if len(results) == 0:
|
||||
log.info('New video for subscription %s: %s %s"', subscription, video.getVideoId(), video.getTitle())
|
||||
db_video = create_video(video, subscription)
|
||||
else:
|
||||
if not _ENABLE_UPDATE_STATS:
|
||||
continue
|
||||
db_video = results.first()
|
||||
log.info('New video for subscription %s: %s %s"', subscription, item.resource_video_id, item.title)
|
||||
Video.create(item, subscription)
|
||||
|
||||
# Update video stats - rating and view count
|
||||
stats = yt_api.get_single_video_stats(db_video.video_id)
|
||||
db_video.rating = stats.get_like_count() / (stats.get_like_count() + stats.get_dislike_count())
|
||||
db_video.views = stats.get_view_count()
|
||||
db_video.save()
|
||||
if _ENABLE_UPDATE_STATS:
|
||||
all_vids = Video.objects.filter(subscription=subscription)
|
||||
all_vids_ids = [video.video_id for video in all_vids]
|
||||
all_vids_dict = {v.video_id: v for v in all_vids}
|
||||
|
||||
for yt_video in yt_api.videos(all_vids_ids, part='id,statistics'):
|
||||
video = all_vids_dict.get(yt_video.id)
|
||||
if yt_video.like_count is not None and yt_video.dislike_count is not None:
|
||||
video.rating = yt_video.n_likes / (yt_video.n_likes + yt_video.n_dislikes)
|
||||
video.views = yt_video.n_views
|
||||
video.save()
|
||||
|
||||
|
||||
def __detect_deleted(subscription: Subscription):
|
||||
@ -82,11 +83,6 @@ def __fetch_thumbnails_obj(iterable, obj_type, id_attr):
|
||||
|
||||
|
||||
def __fetch_thumbnails():
|
||||
# Fetch thumbnails
|
||||
log.info("Fetching channel thumbnails... ")
|
||||
__fetch_thumbnails_obj(Channel.objects.filter(icon_default__istartswith='http'), 'channel', 'channel_id')
|
||||
__fetch_thumbnails_obj(Channel.objects.filter(icon_best__istartswith='http'), 'channel', 'channel_id')
|
||||
|
||||
log.info("Fetching subscription thumbnails... ")
|
||||
__fetch_thumbnails_obj(Subscription.objects.filter(icon_default__istartswith='http'), 'sub', 'playlist_id')
|
||||
__fetch_thumbnails_obj(Subscription.objects.filter(icon_best__istartswith='http'), 'sub', 'playlist_id')
|
||||
@ -107,7 +103,7 @@ def synchronize():
|
||||
|
||||
# Sync subscribed playlists/channels
|
||||
log.info("Sync - checking videos")
|
||||
yt_api = YoutubeAPI.build_public()
|
||||
yt_api = youtube.YoutubeAPI.build_public()
|
||||
for subscription in Subscription.objects.all():
|
||||
__check_new_videos_sub(subscription, yt_api)
|
||||
__detect_deleted(subscription)
|
||||
@ -128,7 +124,7 @@ def synchronize_subscription(subscription: Subscription):
|
||||
__lock.acquire()
|
||||
try:
|
||||
log.info("Running synchronization for single subscription %d [%s]", subscription.id, subscription.name)
|
||||
yt_api = YoutubeAPI.build_public()
|
||||
yt_api = youtube.YoutubeAPI.build_public()
|
||||
|
||||
log.info("Sync - checking videos")
|
||||
__check_new_videos_sub(subscription, yt_api)
|
||||
@ -148,12 +144,15 @@ def synchronize_subscription(subscription: Subscription):
|
||||
|
||||
def schedule_synchronize_global():
|
||||
trigger = CronTrigger.from_crontab(settings.get('global', 'SynchronizationSchedule'))
|
||||
scheduler.instance.add_job(synchronize, trigger, max_instances=1, coalesce=True)
|
||||
job = scheduler.scheduler.add_job(synchronize, trigger, max_instances=1, coalesce=True)
|
||||
log.info('Scheduled synchronize job job=%s', job.id)
|
||||
|
||||
|
||||
def schedule_synchronize_now():
|
||||
scheduler.instance.add_job(synchronize, max_instances=1, coalesce=True)
|
||||
job = scheduler.scheduler.add_job(synchronize, max_instances=1, coalesce=True)
|
||||
log.info('Scheduled synchronize now job job=%s', job.id)
|
||||
|
||||
|
||||
def schedule_synchronize_now_subscription(subscription: Subscription):
|
||||
scheduler.instance.add_job(synchronize_subscription, args=[subscription])
|
||||
job = scheduler.scheduler.add_job(synchronize_subscription, args=[subscription])
|
||||
log.info('Scheduled synchronize subscription job subscription=(%s), job=%s', subscription, job.id)
|
||||
|
||||
@ -1,114 +0,0 @@
|
||||
from apscheduler.schedulers.background import BackgroundScheduler
|
||||
|
||||
from YtManagerApp.models import SubscriptionFolder, Subscription, Video, Channel
|
||||
from YtManagerApp.utils.youtube import YoutubeAPI, YoutubeChannelInfo
|
||||
|
||||
|
||||
class FolderManager(object):
|
||||
|
||||
@staticmethod
|
||||
def create_or_edit(fid, name, parent_id):
|
||||
# Create or edit
|
||||
if fid == '#':
|
||||
folder = SubscriptionFolder()
|
||||
else:
|
||||
folder = SubscriptionFolder.objects.get(id=int(fid))
|
||||
|
||||
# Set attributes
|
||||
folder.name = name
|
||||
if parent_id == '#':
|
||||
folder.parent = None
|
||||
else:
|
||||
folder.parent = SubscriptionFolder.objects.get(id=int(parent_id))
|
||||
|
||||
FolderManager.__validate(folder)
|
||||
folder.save()
|
||||
|
||||
@staticmethod
|
||||
def __validate(folder: SubscriptionFolder):
|
||||
# Make sure folder name is unique in the parent folder
|
||||
for dbFolder in SubscriptionFolder.objects.filter(parent_id=folder.parent_id):
|
||||
if dbFolder.id != folder.id and dbFolder.name == folder.name:
|
||||
raise ValueError('Folder name is not unique!')
|
||||
|
||||
# Prevent parenting loops
|
||||
current = folder
|
||||
visited = []
|
||||
|
||||
while not (current is None):
|
||||
if current in visited:
|
||||
raise ValueError('Parenting cycle detected!')
|
||||
visited.append(current)
|
||||
current = current.parent
|
||||
|
||||
@staticmethod
|
||||
def delete(fid: int):
|
||||
folder = SubscriptionFolder.objects.get(id=fid)
|
||||
folder.delete()
|
||||
|
||||
@staticmethod
|
||||
def list_videos(fid: int):
|
||||
folder = SubscriptionFolder.objects.get(id=fid)
|
||||
folder_list = []
|
||||
queue = [folder]
|
||||
while len(queue) > 0:
|
||||
folder = queue.pop()
|
||||
folder_list.append(folder)
|
||||
queue.extend(SubscriptionFolder.objects.filter(parent=folder))
|
||||
|
||||
return Video.objects.filter(subscription__parent_folder__in=folder_list).order_by('-publish_date')
|
||||
|
||||
|
||||
class SubscriptionManager(object):
|
||||
__scheduler = BackgroundScheduler()
|
||||
|
||||
@staticmethod
|
||||
def create_or_edit(sid, url, name, parent_id):
|
||||
# Create or edit
|
||||
if sid == '#':
|
||||
SubscriptionManager.create(url, parent_id, YoutubeAPI.build_public())
|
||||
else:
|
||||
sub = Subscription.objects.get(id=int(sid))
|
||||
sub.name = name
|
||||
|
||||
if parent_id == '#':
|
||||
sub.parent_folder = None
|
||||
else:
|
||||
sub.parent_folder = SubscriptionFolder.objects.get(id=int(parent_id))
|
||||
|
||||
sub.save()
|
||||
|
||||
@staticmethod
|
||||
def create(url, parent_id, yt_api: YoutubeAPI):
|
||||
sub = Subscription()
|
||||
# Set parent
|
||||
if parent_id == '#':
|
||||
sub.parent_folder = None
|
||||
else:
|
||||
sub.parent_folder = SubscriptionFolder.objects.get(id=int(parent_id))
|
||||
|
||||
# Pull information about the channel and playlist
|
||||
url_type, url_id = yt_api.parse_channel_url(url)
|
||||
|
||||
if url_type == 'playlist_id':
|
||||
info_playlist = yt_api.get_playlist_info(url_id)
|
||||
channel = SubscriptionManager.__get_or_create_channel('channel_id', info_playlist.getChannelId(), yt_api)
|
||||
sub.name = info_playlist.getTitle()
|
||||
sub.playlist_id = info_playlist.getId()
|
||||
sub.description = info_playlist.getDescription()
|
||||
sub.channel = channel
|
||||
sub.icon_default = info_playlist.getDefaultThumbnailUrl()
|
||||
sub.icon_best = info_playlist.getBestThumbnailUrl()
|
||||
|
||||
else:
|
||||
channel = SubscriptionManager.__get_or_create_channel(url_type, url_id, yt_api)
|
||||
# No point in getting the 'uploads' playlist info
|
||||
sub.name = channel.name
|
||||
sub.playlist_id = channel.upload_playlist_id
|
||||
sub.description = channel.description
|
||||
sub.channel = channel
|
||||
sub.icon_default = channel.icon_default
|
||||
sub.icon_best = channel.icon_best
|
||||
|
||||
sub.save()
|
||||
|
||||
@ -1,25 +1,10 @@
|
||||
from YtManagerApp.models import Subscription, Video, SubscriptionFolder
|
||||
from YtManagerApp.utils.youtube import YoutubePlaylistItem
|
||||
from typing import Optional
|
||||
import re
|
||||
from django.db.models import Q
|
||||
from typing import Optional
|
||||
|
||||
from django.contrib.auth.models import User
|
||||
from django.db.models import Q
|
||||
|
||||
|
||||
def create_video(yt_video: YoutubePlaylistItem, subscription: Subscription):
|
||||
video = Video()
|
||||
video.video_id = yt_video.getVideoId()
|
||||
video.name = yt_video.getTitle()
|
||||
video.description = yt_video.getDescription()
|
||||
video.watched = False
|
||||
video.downloaded_path = None
|
||||
video.subscription = subscription
|
||||
video.playlist_index = yt_video.getPlaylistIndex()
|
||||
video.publish_date = yt_video.getPublishDate()
|
||||
video.icon_default = yt_video.getDefaultThumbnailUrl()
|
||||
video.icon_best = yt_video.getBestThumbnailUrl()
|
||||
video.save()
|
||||
return video
|
||||
from YtManagerApp.models import Subscription, Video, SubscriptionFolder
|
||||
|
||||
|
||||
def get_videos(user: User,
|
||||
|
||||
Reference in New Issue
Block a user