mirror of
https://github.com/chibicitiberiu/ytsm.git
synced 2024-02-24 05:43:31 +00:00
Integrated pytaw library for youtube API.
This commit is contained in:
@ -35,4 +35,5 @@ def schedule_delete_video(video: Video):
|
||||
:param video:
|
||||
:return:
|
||||
"""
|
||||
scheduler.instance.add_job(delete_video, args=[video])
|
||||
job = scheduler.scheduler.add_job(delete_video, args=[video])
|
||||
log.info('Scheduled delete video job video=(%s), job=%s', video, job.id)
|
||||
|
||||
@ -25,8 +25,8 @@ def __get_valid_path(path):
|
||||
def __build_youtube_dl_params(video: Video):
|
||||
# resolve path
|
||||
pattern_dict = {
|
||||
'channel': video.subscription.channel.name,
|
||||
'channel_id': video.subscription.channel.channel_id,
|
||||
'channel': video.subscription.channel_name,
|
||||
'channel_id': video.subscription.channel_id,
|
||||
'playlist': video.subscription.name,
|
||||
'playlist_id': video.subscription.playlist_id,
|
||||
'playlist_index': "{:03d}".format(1 + video.playlist_index),
|
||||
@ -88,7 +88,7 @@ def download_video(video: Video, attempt: int = 1):
|
||||
|
||||
elif attempt <= max_attempts:
|
||||
log.warning('Re-enqueueing video (attempt %d/%d)', attempt, max_attempts)
|
||||
scheduler.instance.add_job(download_video, args=[video, attempt + 1])
|
||||
__schedule_download_video(video, attempt + 1)
|
||||
|
||||
else:
|
||||
log.error('Multiple attempts to download video %d [%s %s] failed!', video.id, video.video_id, video.name)
|
||||
@ -96,10 +96,15 @@ def download_video(video: Video, attempt: int = 1):
|
||||
video.save()
|
||||
|
||||
|
||||
def __schedule_download_video(video: Video, attempt=1):
|
||||
job = scheduler.scheduler.add_job(download_video, args=[video, attempt])
|
||||
log.info('Scheduled download video job video=(%s), attempt=%d, job=%s', video, attempt, job.id)
|
||||
|
||||
|
||||
def schedule_download_video(video: Video):
|
||||
"""
|
||||
Schedules a download video job to run immediately.
|
||||
:param video:
|
||||
:return:
|
||||
"""
|
||||
scheduler.instance.add_job(download_video, args=[video, 1])
|
||||
__schedule_download_video(video)
|
||||
|
||||
@ -7,9 +7,8 @@ from apscheduler.triggers.cron import CronTrigger
|
||||
from YtManagerApp import scheduler
|
||||
from YtManagerApp.appconfig import settings
|
||||
from YtManagerApp.management.downloader import fetch_thumbnail, downloader_process_all, downloader_process_subscription
|
||||
from YtManagerApp.management.videos import create_video
|
||||
from YtManagerApp.models import *
|
||||
from YtManagerApp.utils.youtube import YoutubeAPI
|
||||
from YtManagerApp.utils import youtube
|
||||
|
||||
log = logging.getLogger('sync')
|
||||
__lock = Lock()
|
||||
@ -17,23 +16,25 @@ __lock = Lock()
|
||||
_ENABLE_UPDATE_STATS = False
|
||||
|
||||
|
||||
def __check_new_videos_sub(subscription: Subscription, yt_api: YoutubeAPI):
|
||||
def __check_new_videos_sub(subscription: Subscription, yt_api: youtube.YoutubeAPI):
|
||||
# Get list of videos
|
||||
for video in yt_api.list_playlist_videos(subscription.playlist_id):
|
||||
results = Video.objects.filter(video_id=video.getVideoId(), subscription=subscription)
|
||||
for item in yt_api.playlist_items(subscription.playlist_id):
|
||||
results = Video.objects.filter(video_id=item.resource_video_id, subscription=subscription)
|
||||
if len(results) == 0:
|
||||
log.info('New video for subscription %s: %s %s"', subscription, video.getVideoId(), video.getTitle())
|
||||
db_video = create_video(video, subscription)
|
||||
else:
|
||||
if not _ENABLE_UPDATE_STATS:
|
||||
continue
|
||||
db_video = results.first()
|
||||
log.info('New video for subscription %s: %s %s"', subscription, item.resource_video_id, item.title)
|
||||
Video.create(item, subscription)
|
||||
|
||||
# Update video stats - rating and view count
|
||||
stats = yt_api.get_single_video_stats(db_video.video_id)
|
||||
db_video.rating = stats.get_like_count() / (stats.get_like_count() + stats.get_dislike_count())
|
||||
db_video.views = stats.get_view_count()
|
||||
db_video.save()
|
||||
if _ENABLE_UPDATE_STATS:
|
||||
all_vids = Video.objects.filter(subscription=subscription)
|
||||
all_vids_ids = [video.video_id for video in all_vids]
|
||||
all_vids_dict = {v.video_id: v for v in all_vids}
|
||||
|
||||
for yt_video in yt_api.videos(all_vids_ids, part='id,statistics'):
|
||||
video = all_vids_dict.get(yt_video.id)
|
||||
if yt_video.like_count is not None and yt_video.dislike_count is not None:
|
||||
video.rating = yt_video.n_likes / (yt_video.n_likes + yt_video.n_dislikes)
|
||||
video.views = yt_video.n_views
|
||||
video.save()
|
||||
|
||||
|
||||
def __detect_deleted(subscription: Subscription):
|
||||
@ -82,11 +83,6 @@ def __fetch_thumbnails_obj(iterable, obj_type, id_attr):
|
||||
|
||||
|
||||
def __fetch_thumbnails():
|
||||
# Fetch thumbnails
|
||||
log.info("Fetching channel thumbnails... ")
|
||||
__fetch_thumbnails_obj(Channel.objects.filter(icon_default__istartswith='http'), 'channel', 'channel_id')
|
||||
__fetch_thumbnails_obj(Channel.objects.filter(icon_best__istartswith='http'), 'channel', 'channel_id')
|
||||
|
||||
log.info("Fetching subscription thumbnails... ")
|
||||
__fetch_thumbnails_obj(Subscription.objects.filter(icon_default__istartswith='http'), 'sub', 'playlist_id')
|
||||
__fetch_thumbnails_obj(Subscription.objects.filter(icon_best__istartswith='http'), 'sub', 'playlist_id')
|
||||
@ -107,7 +103,7 @@ def synchronize():
|
||||
|
||||
# Sync subscribed playlists/channels
|
||||
log.info("Sync - checking videos")
|
||||
yt_api = YoutubeAPI.build_public()
|
||||
yt_api = youtube.YoutubeAPI.build_public()
|
||||
for subscription in Subscription.objects.all():
|
||||
__check_new_videos_sub(subscription, yt_api)
|
||||
__detect_deleted(subscription)
|
||||
@ -128,7 +124,7 @@ def synchronize_subscription(subscription: Subscription):
|
||||
__lock.acquire()
|
||||
try:
|
||||
log.info("Running synchronization for single subscription %d [%s]", subscription.id, subscription.name)
|
||||
yt_api = YoutubeAPI.build_public()
|
||||
yt_api = youtube.YoutubeAPI.build_public()
|
||||
|
||||
log.info("Sync - checking videos")
|
||||
__check_new_videos_sub(subscription, yt_api)
|
||||
@ -148,12 +144,15 @@ def synchronize_subscription(subscription: Subscription):
|
||||
|
||||
def schedule_synchronize_global():
|
||||
trigger = CronTrigger.from_crontab(settings.get('global', 'SynchronizationSchedule'))
|
||||
scheduler.instance.add_job(synchronize, trigger, max_instances=1, coalesce=True)
|
||||
job = scheduler.scheduler.add_job(synchronize, trigger, max_instances=1, coalesce=True)
|
||||
log.info('Scheduled synchronize job job=%s', job.id)
|
||||
|
||||
|
||||
def schedule_synchronize_now():
|
||||
scheduler.instance.add_job(synchronize, max_instances=1, coalesce=True)
|
||||
job = scheduler.scheduler.add_job(synchronize, max_instances=1, coalesce=True)
|
||||
log.info('Scheduled synchronize now job job=%s', job.id)
|
||||
|
||||
|
||||
def schedule_synchronize_now_subscription(subscription: Subscription):
|
||||
scheduler.instance.add_job(synchronize_subscription, args=[subscription])
|
||||
job = scheduler.scheduler.add_job(synchronize_subscription, args=[subscription])
|
||||
log.info('Scheduled synchronize subscription job subscription=(%s), job=%s', subscription, job.id)
|
||||
|
||||
@ -1,114 +0,0 @@
|
||||
from apscheduler.schedulers.background import BackgroundScheduler
|
||||
|
||||
from YtManagerApp.models import SubscriptionFolder, Subscription, Video, Channel
|
||||
from YtManagerApp.utils.youtube import YoutubeAPI, YoutubeChannelInfo
|
||||
|
||||
|
||||
class FolderManager(object):
|
||||
|
||||
@staticmethod
|
||||
def create_or_edit(fid, name, parent_id):
|
||||
# Create or edit
|
||||
if fid == '#':
|
||||
folder = SubscriptionFolder()
|
||||
else:
|
||||
folder = SubscriptionFolder.objects.get(id=int(fid))
|
||||
|
||||
# Set attributes
|
||||
folder.name = name
|
||||
if parent_id == '#':
|
||||
folder.parent = None
|
||||
else:
|
||||
folder.parent = SubscriptionFolder.objects.get(id=int(parent_id))
|
||||
|
||||
FolderManager.__validate(folder)
|
||||
folder.save()
|
||||
|
||||
@staticmethod
|
||||
def __validate(folder: SubscriptionFolder):
|
||||
# Make sure folder name is unique in the parent folder
|
||||
for dbFolder in SubscriptionFolder.objects.filter(parent_id=folder.parent_id):
|
||||
if dbFolder.id != folder.id and dbFolder.name == folder.name:
|
||||
raise ValueError('Folder name is not unique!')
|
||||
|
||||
# Prevent parenting loops
|
||||
current = folder
|
||||
visited = []
|
||||
|
||||
while not (current is None):
|
||||
if current in visited:
|
||||
raise ValueError('Parenting cycle detected!')
|
||||
visited.append(current)
|
||||
current = current.parent
|
||||
|
||||
@staticmethod
|
||||
def delete(fid: int):
|
||||
folder = SubscriptionFolder.objects.get(id=fid)
|
||||
folder.delete()
|
||||
|
||||
@staticmethod
|
||||
def list_videos(fid: int):
|
||||
folder = SubscriptionFolder.objects.get(id=fid)
|
||||
folder_list = []
|
||||
queue = [folder]
|
||||
while len(queue) > 0:
|
||||
folder = queue.pop()
|
||||
folder_list.append(folder)
|
||||
queue.extend(SubscriptionFolder.objects.filter(parent=folder))
|
||||
|
||||
return Video.objects.filter(subscription__parent_folder__in=folder_list).order_by('-publish_date')
|
||||
|
||||
|
||||
class SubscriptionManager(object):
|
||||
__scheduler = BackgroundScheduler()
|
||||
|
||||
@staticmethod
|
||||
def create_or_edit(sid, url, name, parent_id):
|
||||
# Create or edit
|
||||
if sid == '#':
|
||||
SubscriptionManager.create(url, parent_id, YoutubeAPI.build_public())
|
||||
else:
|
||||
sub = Subscription.objects.get(id=int(sid))
|
||||
sub.name = name
|
||||
|
||||
if parent_id == '#':
|
||||
sub.parent_folder = None
|
||||
else:
|
||||
sub.parent_folder = SubscriptionFolder.objects.get(id=int(parent_id))
|
||||
|
||||
sub.save()
|
||||
|
||||
@staticmethod
|
||||
def create(url, parent_id, yt_api: YoutubeAPI):
|
||||
sub = Subscription()
|
||||
# Set parent
|
||||
if parent_id == '#':
|
||||
sub.parent_folder = None
|
||||
else:
|
||||
sub.parent_folder = SubscriptionFolder.objects.get(id=int(parent_id))
|
||||
|
||||
# Pull information about the channel and playlist
|
||||
url_type, url_id = yt_api.parse_channel_url(url)
|
||||
|
||||
if url_type == 'playlist_id':
|
||||
info_playlist = yt_api.get_playlist_info(url_id)
|
||||
channel = SubscriptionManager.__get_or_create_channel('channel_id', info_playlist.getChannelId(), yt_api)
|
||||
sub.name = info_playlist.getTitle()
|
||||
sub.playlist_id = info_playlist.getId()
|
||||
sub.description = info_playlist.getDescription()
|
||||
sub.channel = channel
|
||||
sub.icon_default = info_playlist.getDefaultThumbnailUrl()
|
||||
sub.icon_best = info_playlist.getBestThumbnailUrl()
|
||||
|
||||
else:
|
||||
channel = SubscriptionManager.__get_or_create_channel(url_type, url_id, yt_api)
|
||||
# No point in getting the 'uploads' playlist info
|
||||
sub.name = channel.name
|
||||
sub.playlist_id = channel.upload_playlist_id
|
||||
sub.description = channel.description
|
||||
sub.channel = channel
|
||||
sub.icon_default = channel.icon_default
|
||||
sub.icon_best = channel.icon_best
|
||||
|
||||
sub.save()
|
||||
|
||||
@ -1,25 +1,10 @@
|
||||
from YtManagerApp.models import Subscription, Video, SubscriptionFolder
|
||||
from YtManagerApp.utils.youtube import YoutubePlaylistItem
|
||||
from typing import Optional
|
||||
import re
|
||||
from django.db.models import Q
|
||||
from typing import Optional
|
||||
|
||||
from django.contrib.auth.models import User
|
||||
from django.db.models import Q
|
||||
|
||||
|
||||
def create_video(yt_video: YoutubePlaylistItem, subscription: Subscription):
|
||||
video = Video()
|
||||
video.video_id = yt_video.getVideoId()
|
||||
video.name = yt_video.getTitle()
|
||||
video.description = yt_video.getDescription()
|
||||
video.watched = False
|
||||
video.downloaded_path = None
|
||||
video.subscription = subscription
|
||||
video.playlist_index = yt_video.getPlaylistIndex()
|
||||
video.publish_date = yt_video.getPublishDate()
|
||||
video.icon_default = yt_video.getDefaultThumbnailUrl()
|
||||
video.icon_best = yt_video.getBestThumbnailUrl()
|
||||
video.save()
|
||||
return video
|
||||
from YtManagerApp.models import Subscription, Video, SubscriptionFolder
|
||||
|
||||
|
||||
def get_videos(user: User,
|
||||
|
||||
32
YtManagerApp/migrations/0007_auto_20181029_1638.py
Normal file
32
YtManagerApp/migrations/0007_auto_20181029_1638.py
Normal file
@ -0,0 +1,32 @@
|
||||
# Generated by Django 2.1.2 on 2018-10-29 16:38
|
||||
|
||||
from django.db import migrations, models
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
dependencies = [
|
||||
('YtManagerApp', '0006_auto_20181027_0256'),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.RemoveField(
|
||||
model_name='subscription',
|
||||
name='channel',
|
||||
),
|
||||
migrations.AddField(
|
||||
model_name='subscription',
|
||||
name='channel_id',
|
||||
field=models.CharField(default='test', max_length=128),
|
||||
preserve_default=False,
|
||||
),
|
||||
migrations.AddField(
|
||||
model_name='subscription',
|
||||
name='channel_name',
|
||||
field=models.CharField(default='Unknown', max_length=1024),
|
||||
preserve_default=False,
|
||||
),
|
||||
migrations.DeleteModel(
|
||||
name='Channel',
|
||||
),
|
||||
]
|
||||
@ -6,7 +6,7 @@ from django.contrib.auth.models import User
|
||||
from django.contrib.auth.models import User
|
||||
from django.db import models
|
||||
from django.db.models.functions import Lower
|
||||
from YtManagerApp.utils.youtube import YoutubeAPI, YoutubeChannelInfo, YoutubePlaylistInfo
|
||||
from YtManagerApp.utils import youtube
|
||||
|
||||
# help_text = user shown text
|
||||
# verbose_name = user shown name
|
||||
@ -176,6 +176,9 @@ class SubscriptionFolder(models.Model):
|
||||
current = current.parent
|
||||
return s[:-3]
|
||||
|
||||
def __repr__(self):
|
||||
return f'folder {self.id}, name="{self.name}"'
|
||||
|
||||
def delete_folder(self, keep_subscriptions: bool):
|
||||
if keep_subscriptions:
|
||||
|
||||
@ -225,92 +228,13 @@ class SubscriptionFolder(models.Model):
|
||||
return data_collected
|
||||
|
||||
|
||||
class Channel(models.Model):
|
||||
channel_id = models.TextField(null=False, unique=True)
|
||||
username = models.TextField(null=True, unique=True)
|
||||
custom_url = models.TextField(null=True, unique=True)
|
||||
name = models.TextField()
|
||||
description = models.TextField()
|
||||
icon_default = models.TextField()
|
||||
icon_best = models.TextField()
|
||||
upload_playlist_id = models.TextField()
|
||||
|
||||
def __str__(self):
|
||||
return self.name
|
||||
|
||||
@staticmethod
|
||||
def find_by_channel_id(channel_id):
|
||||
result = Channel.objects.filter(channel_id=channel_id)
|
||||
if len(result) > 0:
|
||||
return result.first()
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def find_by_username(username):
|
||||
result = Channel.objects.filter(username=username)
|
||||
if len(result) > 0:
|
||||
return result.first()
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def find_by_custom_url(custom_url):
|
||||
result = Channel.objects.filter(custom_url=custom_url)
|
||||
if len(result) > 0:
|
||||
return result.first()
|
||||
return None
|
||||
|
||||
def fill(self, yt_channel_info: YoutubeChannelInfo):
|
||||
self.channel_id = yt_channel_info.getId()
|
||||
self.custom_url = yt_channel_info.getCustomUrl()
|
||||
self.name = yt_channel_info.getTitle()
|
||||
self.description = yt_channel_info.getDescription()
|
||||
self.icon_default = yt_channel_info.getDefaultThumbnailUrl()
|
||||
self.icon_best = yt_channel_info.getBestThumbnailUrl()
|
||||
self.upload_playlist_id = yt_channel_info.getUploadsPlaylist()
|
||||
self.save()
|
||||
|
||||
@staticmethod
|
||||
def get_or_create(url_type: str, url_id: str, yt_api: YoutubeAPI):
|
||||
channel: Channel = None
|
||||
info_channel: YoutubeChannelInfo = None
|
||||
|
||||
if url_type == 'user':
|
||||
channel = Channel.find_by_username(url_id)
|
||||
if not channel:
|
||||
info_channel = yt_api.get_channel_info_by_username(url_id)
|
||||
channel = Channel.find_by_channel_id(info_channel.getId())
|
||||
|
||||
elif url_type == 'channel_id':
|
||||
channel = Channel.find_by_channel_id(url_id)
|
||||
if not channel:
|
||||
info_channel = yt_api.get_channel_info(url_id)
|
||||
|
||||
elif url_type == 'channel_custom':
|
||||
channel = Channel.find_by_custom_url(url_id)
|
||||
if not channel:
|
||||
found_channel_id = yt_api.search_channel(url_id)
|
||||
channel = Channel.find_by_channel_id(found_channel_id)
|
||||
if not channel:
|
||||
info_channel = yt_api.get_channel_info(found_channel_id)
|
||||
|
||||
# If we downloaded information about the channel, store information
|
||||
# about the channel here.
|
||||
if info_channel:
|
||||
if not channel:
|
||||
channel = Channel()
|
||||
if url_type == 'user':
|
||||
channel.username = url_id
|
||||
channel.fill(info_channel)
|
||||
|
||||
return channel
|
||||
|
||||
|
||||
class Subscription(models.Model):
|
||||
name = models.CharField(null=False, max_length=1024)
|
||||
parent_folder = models.ForeignKey(SubscriptionFolder, on_delete=models.CASCADE, null=True, blank=True)
|
||||
playlist_id = models.CharField(null=False, max_length=128)
|
||||
description = models.TextField()
|
||||
channel = models.ForeignKey(Channel, on_delete=models.CASCADE)
|
||||
channel_id = models.CharField(max_length=128)
|
||||
channel_name = models.CharField(max_length=1024)
|
||||
icon_default = models.CharField(max_length=1024)
|
||||
icon_best = models.CharField(max_length=1024)
|
||||
user = models.ForeignKey(User, on_delete=models.CASCADE)
|
||||
@ -327,30 +251,42 @@ class Subscription(models.Model):
|
||||
def __str__(self):
|
||||
return self.name
|
||||
|
||||
def fill_from_playlist(self, info_playlist: YoutubePlaylistInfo):
|
||||
self.name = info_playlist.getTitle()
|
||||
self.playlist_id = info_playlist.getId()
|
||||
self.description = info_playlist.getDescription()
|
||||
self.icon_default = info_playlist.getDefaultThumbnailUrl()
|
||||
self.icon_best = info_playlist.getBestThumbnailUrl()
|
||||
def __repr__(self):
|
||||
return f'subscription {self.id}, name="{self.name}", playlist_id="{self.playlist_id}"'
|
||||
|
||||
def copy_from_channel(self):
|
||||
def fill_from_playlist(self, info_playlist: youtube.Playlist):
|
||||
self.name = info_playlist.title
|
||||
self.playlist_id = info_playlist.id
|
||||
self.description = info_playlist.description
|
||||
self.channel_id = info_playlist.channel_id
|
||||
self.channel_name = info_playlist.channel_title
|
||||
self.icon_default = youtube.default_thumbnail(info_playlist).url
|
||||
self.icon_best = youtube.best_thumbnail(info_playlist).url
|
||||
|
||||
def copy_from_channel(self, info_channel: youtube.Channel):
|
||||
# No point in storing info about the 'uploads from X' playlist
|
||||
self.name = self.channel.name
|
||||
self.playlist_id = self.channel.upload_playlist_id
|
||||
self.description = self.channel.description
|
||||
self.icon_default = self.channel.icon_default
|
||||
self.icon_best = self.channel.icon_best
|
||||
self.name = info_channel.title
|
||||
self.playlist_id = info_channel.uploads_playlist.id
|
||||
self.description = info_channel.description
|
||||
self.channel_id = info_channel.id
|
||||
self.channel_name = info_channel.title
|
||||
self.icon_default = youtube.default_thumbnail(info_channel).url
|
||||
self.icon_best = youtube.best_thumbnail(info_channel).url
|
||||
|
||||
def fetch_from_url(self, url, yt_api: youtube.YoutubeAPI):
|
||||
url_parsed = yt_api.parse_url(url)
|
||||
if 'playlist' in url_parsed:
|
||||
info_playlist = yt_api.playlist(url=url)
|
||||
if info_playlist is None:
|
||||
raise ValueError('Invalid playlist ID!')
|
||||
|
||||
def fetch_from_url(self, url, yt_api: YoutubeAPI):
|
||||
url_type, url_id = yt_api.parse_channel_url(url)
|
||||
if url_type == 'playlist_id':
|
||||
info_playlist = yt_api.get_playlist_info(url_id)
|
||||
self.channel = Channel.get_or_create('channel_id', info_playlist.getChannelId(), yt_api)
|
||||
self.fill_from_playlist(info_playlist)
|
||||
else:
|
||||
self.channel = Channel.get_or_create(url_type, url_id, yt_api)
|
||||
self.copy_from_channel()
|
||||
info_channel = yt_api.channel(url=url)
|
||||
if info_channel is None:
|
||||
raise ValueError('Cannot find channel!')
|
||||
|
||||
self.copy_from_channel(info_channel)
|
||||
|
||||
def delete_subscription(self, keep_downloaded_videos: bool):
|
||||
self.delete()
|
||||
@ -383,6 +319,22 @@ class Video(models.Model):
|
||||
views = models.IntegerField(null=False, default=0)
|
||||
rating = models.FloatField(null=False, default=0.5)
|
||||
|
||||
@staticmethod
|
||||
def create(playlist_item: youtube.PlaylistItem, subscription: Subscription):
|
||||
video = Video()
|
||||
video.video_id = playlist_item.resource_video_id
|
||||
video.name = playlist_item.title
|
||||
video.description = playlist_item.description
|
||||
video.watched = False
|
||||
video.downloaded_path = None
|
||||
video.subscription = subscription
|
||||
video.playlist_index = playlist_item.position
|
||||
video.publish_date = playlist_item.published_at
|
||||
video.icon_default = youtube.default_thumbnail(playlist_item).url
|
||||
video.icon_best = youtube.best_thumbnail(playlist_item).url
|
||||
video.save()
|
||||
return video
|
||||
|
||||
def mark_watched(self):
|
||||
self.watched = True
|
||||
self.save()
|
||||
@ -428,3 +380,6 @@ class Video(models.Model):
|
||||
|
||||
def __str__(self):
|
||||
return self.name
|
||||
|
||||
def __repr__(self):
|
||||
return f'video {self.id}, video_id="{self.video_id}"'
|
||||
|
||||
@ -2,12 +2,12 @@ import logging
|
||||
import sys
|
||||
from apscheduler.schedulers.background import BackgroundScheduler
|
||||
|
||||
instance: BackgroundScheduler = None
|
||||
scheduler: BackgroundScheduler = None
|
||||
|
||||
|
||||
def initialize_scheduler():
|
||||
from .appconfig import settings
|
||||
global instance
|
||||
global scheduler
|
||||
|
||||
logger = logging.getLogger('scheduler')
|
||||
executors = {
|
||||
@ -17,8 +17,8 @@ def initialize_scheduler():
|
||||
}
|
||||
}
|
||||
job_defaults = {
|
||||
'misfire_grace_time': sys.maxsize
|
||||
'misfire_grace_time': 60 * 60 * 24 * 365 # 1 year
|
||||
}
|
||||
|
||||
instance = BackgroundScheduler(logger=logger, executors=executors, job_defaults=job_defaults)
|
||||
instance.start()
|
||||
scheduler = BackgroundScheduler(logger=logger, executors=executors, job_defaults=job_defaults)
|
||||
scheduler.start()
|
||||
|
||||
@ -1,32 +0,0 @@
|
||||
import itertools
|
||||
from typing import Iterable
|
||||
|
||||
|
||||
def first_true(*args, default=False, pred=None):
|
||||
"""Returns the first true value in the iterable.
|
||||
|
||||
If no true value is found, returns *default*
|
||||
|
||||
If *pred* is not None, returns the first item
|
||||
for which pred(item) is true.
|
||||
|
||||
"""
|
||||
# first_true([a,b,c], x) --> a or b or c or x
|
||||
# first_true([a,b], x, f) --> a if f(a) else b if f(b) else x
|
||||
return next(filter(pred, args), default)
|
||||
|
||||
|
||||
def as_chunks(iterable: Iterable, chunk_size: int):
|
||||
"""
|
||||
Iterates an iterable in chunks of chunk_size elements.
|
||||
:param iterable: An iterable containing items to iterate.
|
||||
:param chunk_size: Chunk size
|
||||
:return: Returns a generator which will yield chunks of size chunk_size
|
||||
"""
|
||||
|
||||
it = iter(iterable)
|
||||
while True:
|
||||
chunk = tuple(itertools.islice(it, chunk_size))
|
||||
if not chunk:
|
||||
return
|
||||
yield chunk
|
||||
@ -1,285 +1,48 @@
|
||||
from googleapiclient.discovery import build
|
||||
from googleapiclient.errors import Error as APIError
|
||||
from google_auth_oauthlib.flow import InstalledAppFlow
|
||||
from django.conf import settings
|
||||
import re
|
||||
from YtManagerApp.utils.iterutils import as_chunks
|
||||
|
||||
API_SERVICE_NAME = 'youtube'
|
||||
API_VERSION = 'v3'
|
||||
|
||||
YOUTUBE_LIST_LIMIT = 50
|
||||
from external.pytaw.pytaw.youtube import YouTube, Channel, Playlist, PlaylistItem, Thumbnail, InvalidURL, Resource, Video
|
||||
from typing import Optional
|
||||
|
||||
|
||||
class YoutubeException(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class YoutubeInvalidURLException(YoutubeException):
|
||||
pass
|
||||
|
||||
|
||||
class YoutubeChannelNotFoundException(YoutubeException):
|
||||
pass
|
||||
|
||||
|
||||
class YoutubeUserNotFoundException(YoutubeException):
|
||||
pass
|
||||
|
||||
|
||||
class YoutubePlaylistNotFoundException(YoutubeException):
|
||||
pass
|
||||
|
||||
|
||||
class YoutubeVideoNotFoundException(YoutubeException):
|
||||
pass
|
||||
|
||||
|
||||
class YoutubeChannelInfo(object):
|
||||
def __init__(self, result_dict):
|
||||
self.__id = result_dict['id']
|
||||
self.__snippet = result_dict['snippet']
|
||||
self.__contentDetails = result_dict['contentDetails']
|
||||
|
||||
def getId(self):
|
||||
return self.__id
|
||||
|
||||
def getTitle(self):
|
||||
return self.__snippet['title']
|
||||
|
||||
def getDescription(self):
|
||||
return self.__snippet['description']
|
||||
|
||||
def getCustomUrl(self):
|
||||
try:
|
||||
return self.__snippet['customUrl']
|
||||
except KeyError:
|
||||
return None
|
||||
|
||||
def getDefaultThumbnailUrl(self):
|
||||
return self.__snippet['thumbnails']['default']['url']
|
||||
|
||||
def getBestThumbnailUrl(self):
|
||||
best_url = None
|
||||
best_res = 0
|
||||
for _, thumb in self.__snippet['thumbnails'].items():
|
||||
res = thumb['width'] * thumb['height']
|
||||
if res > best_res:
|
||||
best_res = res
|
||||
best_url = thumb['url']
|
||||
return best_url
|
||||
|
||||
def getUploadsPlaylist(self):
|
||||
return self.__contentDetails['relatedPlaylists']['uploads']
|
||||
|
||||
|
||||
class YoutubePlaylistInfo(object):
|
||||
def __init__(self, result_dict):
|
||||
self.__id = result_dict['id']
|
||||
self.__snippet = result_dict['snippet']
|
||||
|
||||
def getId(self):
|
||||
return self.__id
|
||||
|
||||
def getChannelId(self):
|
||||
return self.__snippet['channelId']
|
||||
|
||||
def getTitle(self):
|
||||
return self.__snippet['title']
|
||||
|
||||
def getDescription(self):
|
||||
return self.__snippet['description']
|
||||
|
||||
def getDefaultThumbnailUrl(self):
|
||||
return self.__snippet['thumbnails']['default']['url']
|
||||
|
||||
def getBestThumbnailUrl(self):
|
||||
best_url = None
|
||||
best_res = 0
|
||||
for _, thumb in self.__snippet['thumbnails'].items():
|
||||
res = thumb['width'] * thumb['height']
|
||||
if res > best_res:
|
||||
best_res = res
|
||||
best_url = thumb['url']
|
||||
return best_url
|
||||
|
||||
|
||||
class YoutubePlaylistItem(object):
|
||||
def __init__(self, result_dict):
|
||||
self.__snippet = result_dict['snippet']
|
||||
|
||||
def getVideoId(self):
|
||||
return self.__snippet['resourceId']['videoId']
|
||||
|
||||
def getPublishDate(self):
|
||||
return self.__snippet['publishedAt']
|
||||
|
||||
def getTitle(self):
|
||||
return self.__snippet['title']
|
||||
|
||||
def getDescription(self):
|
||||
return self.__snippet['description']
|
||||
|
||||
def getDefaultThumbnailUrl(self):
|
||||
return self.__snippet['thumbnails']['default']['url']
|
||||
|
||||
def getBestThumbnailUrl(self):
|
||||
best_url = None
|
||||
best_res = 0
|
||||
for _, thumb in self.__snippet['thumbnails'].items():
|
||||
res = thumb['width'] * thumb['height']
|
||||
if res > best_res:
|
||||
best_res = res
|
||||
best_url = thumb['url']
|
||||
return best_url
|
||||
|
||||
def getPlaylistIndex(self):
|
||||
return self.__snippet['position']
|
||||
|
||||
|
||||
class YoutubeVideoStatistics(object):
|
||||
def __init__(self, result_dict):
|
||||
self.id = result_dict['id']
|
||||
self.stats = result_dict['statistics']
|
||||
|
||||
def get_view_count(self):
|
||||
return int(self.stats['viewCount'])
|
||||
|
||||
def get_like_count(self):
|
||||
return int(self.stats['likeCount'])
|
||||
|
||||
def get_dislike_count(self):
|
||||
return int(self.stats['dislikeCount'])
|
||||
|
||||
def get_favorite_count(self):
|
||||
return int(self.stats['favoriteCount'])
|
||||
|
||||
def get_comment_count(self):
|
||||
return int(self.stats['commentCount'])
|
||||
|
||||
|
||||
class YoutubeAPI(object):
|
||||
def __init__(self, service):
|
||||
self.service = service
|
||||
class YoutubeAPI(YouTube):
|
||||
|
||||
@staticmethod
|
||||
def build_public() -> 'YoutubeAPI':
|
||||
service = build(API_SERVICE_NAME, API_VERSION, developerKey=settings.YOUTUBE_API_KEY)
|
||||
return YoutubeAPI(service)
|
||||
|
||||
@staticmethod
|
||||
def parse_channel_url(url):
|
||||
"""
|
||||
Parses given channel url, returns a tuple of the form (type, value), where type can be one of:
|
||||
* channel_id
|
||||
* channel_custom
|
||||
* user
|
||||
* playlist_id
|
||||
:param url: URL to parse
|
||||
:return: (type, value) tuple
|
||||
"""
|
||||
match = re.search(r'youtube\.com/.*[&?]list=([^?&/]+)', url)
|
||||
if match:
|
||||
return 'playlist_id', match.group(1)
|
||||
|
||||
match = re.search(r'youtube\.com/user/([^?&/]+)', url)
|
||||
if match:
|
||||
return 'user', match.group(1)
|
||||
|
||||
match = re.search(r'youtube\.com/channel/([^?&/]+)', url)
|
||||
if match:
|
||||
return 'channel_id', match.group(1)
|
||||
|
||||
match = re.search(r'youtube\.com/(?:c/)?([^?&/]+)', url)
|
||||
if match:
|
||||
return 'channel_custom', match.group(1)
|
||||
|
||||
raise YoutubeInvalidURLException('Unrecognized URL format!')
|
||||
|
||||
def get_playlist_info(self, list_id) -> YoutubePlaylistInfo:
|
||||
result = self.service.playlists()\
|
||||
.list(part='snippet', id=list_id)\
|
||||
.execute()
|
||||
|
||||
if len(result['items']) <= 0:
|
||||
raise YoutubePlaylistNotFoundException("Invalid playlist ID.")
|
||||
|
||||
return YoutubePlaylistInfo(result['items'][0])
|
||||
|
||||
def get_channel_info_by_username(self, user) -> YoutubeChannelInfo:
|
||||
result = self.service.channels()\
|
||||
.list(part='snippet,contentDetails', forUsername=user)\
|
||||
.execute()
|
||||
|
||||
if len(result['items']) <= 0:
|
||||
raise YoutubeUserNotFoundException('Invalid user.')
|
||||
|
||||
return YoutubeChannelInfo(result['items'][0])
|
||||
|
||||
def get_channel_info(self, channel_id) -> YoutubeChannelInfo:
|
||||
result = self.service.channels()\
|
||||
.list(part='snippet,contentDetails', id=channel_id)\
|
||||
.execute()
|
||||
|
||||
if len(result['items']) <= 0:
|
||||
raise YoutubeChannelNotFoundException('Invalid channel ID.')
|
||||
|
||||
return YoutubeChannelInfo(result['items'][0])
|
||||
|
||||
def search_channel(self, custom) -> str:
|
||||
result = self.service.search()\
|
||||
.list(part='id', q=custom, type='channel')\
|
||||
.execute()
|
||||
|
||||
if len(result['items']) <= 0:
|
||||
raise YoutubeChannelNotFoundException('Could not find channel!')
|
||||
|
||||
channel_result = result['items'][0]
|
||||
return channel_result['id']['channelId']
|
||||
|
||||
def list_playlist_videos(self, playlist_id):
|
||||
kwargs = {
|
||||
"part": "snippet",
|
||||
"maxResults": 50,
|
||||
"playlistId": playlist_id
|
||||
}
|
||||
last_page = False
|
||||
|
||||
while not last_page:
|
||||
result = self.service.playlistItems()\
|
||||
.list(**kwargs)\
|
||||
.execute()
|
||||
|
||||
for item in result['items']:
|
||||
yield YoutubePlaylistItem(item)
|
||||
|
||||
if 'nextPageToken' in result:
|
||||
kwargs['pageToken'] = result['nextPageToken']
|
||||
else:
|
||||
last_page = True
|
||||
|
||||
def get_single_video_stats(self, video_id) -> YoutubeVideoStatistics:
|
||||
result = list(self.get_video_stats([video_id]))
|
||||
if len(result) < 1:
|
||||
raise YoutubeVideoNotFoundException('Could not find video with id ' + video_id + '!')
|
||||
return result[0]
|
||||
|
||||
def get_video_stats(self, video_id_list):
|
||||
for chunk in as_chunks(video_id_list, YOUTUBE_LIST_LIMIT):
|
||||
kwargs = {
|
||||
"part": "statistics",
|
||||
"maxResults": YOUTUBE_LIST_LIMIT,
|
||||
"id": ','.join(chunk)
|
||||
}
|
||||
result = self.service.videos()\
|
||||
.list(**kwargs)\
|
||||
.execute()
|
||||
|
||||
for item in result['items']:
|
||||
yield YoutubeVideoStatistics(item)
|
||||
return YoutubeAPI(key=settings.YOUTUBE_API_KEY)
|
||||
|
||||
# @staticmethod
|
||||
# def build_oauth() -> 'YoutubeAPI':
|
||||
# flow =
|
||||
# credentials =
|
||||
# service = build(API_SERVICE_NAME, API_VERSION, credentials)
|
||||
|
||||
|
||||
def default_thumbnail(resource: Resource) -> Optional[Thumbnail]:
|
||||
"""
|
||||
Gets the default thumbnail for a resource.
|
||||
Searches in the list of thumbnails for one with the label 'default', or takes the first one.
|
||||
:param resource:
|
||||
:return:
|
||||
"""
|
||||
thumbs = getattr(resource, 'thumbnails', None)
|
||||
|
||||
if thumbs is None or len(thumbs) <= 0:
|
||||
return None
|
||||
|
||||
return next(
|
||||
(i for i in thumbs if i.id == 'default'),
|
||||
thumbs[0]
|
||||
)
|
||||
|
||||
|
||||
def best_thumbnail(resource: Resource) -> Optional[Thumbnail]:
|
||||
"""
|
||||
Gets the best thumbnail available for a resource.
|
||||
:param resource:
|
||||
:return:
|
||||
"""
|
||||
thumbs = getattr(resource, 'thumbnails', None)
|
||||
|
||||
if thumbs is None or len(thumbs) <= 0:
|
||||
return None
|
||||
|
||||
return max(thumbs, key=lambda t: t.width * t.height)
|
||||
@ -1,4 +1,4 @@
|
||||
from crispy_forms.helper import FormHelperpython3
|
||||
from crispy_forms.helper import FormHelper
|
||||
from crispy_forms.layout import Layout, Field, HTML
|
||||
from django import forms
|
||||
from django.contrib.auth.decorators import login_required
|
||||
@ -175,7 +175,8 @@ class SubscriptionFolderForm(forms.ModelForm):
|
||||
args_id.append(~Q(id=self.instance.id))
|
||||
|
||||
if SubscriptionFolder.objects.filter(parent=parent, name__iexact=name, *args_id).count() > 0:
|
||||
raise forms.ValidationError('A folder with the same name already exists in the given parent directory!', code='already_exists')
|
||||
raise forms.ValidationError(
|
||||
'A folder with the same name already exists in the given parent directory!', code='already_exists')
|
||||
|
||||
# Check for cycles
|
||||
if self.instance is not None:
|
||||
@ -238,6 +239,7 @@ class CreateSubscriptionForm(forms.ModelForm):
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
super().__init__(*args, **kwargs)
|
||||
self.yt_api = youtube.YoutubeAPI.build_public()
|
||||
self.helper = FormHelper()
|
||||
self.helper.form_tag = False
|
||||
self.helper.layout = Layout(
|
||||
@ -252,11 +254,18 @@ class CreateSubscriptionForm(forms.ModelForm):
|
||||
)
|
||||
|
||||
def clean_playlist_url(self):
|
||||
playlist_url = self.cleaned_data['playlist_url']
|
||||
playlist_url: str = self.cleaned_data['playlist_url']
|
||||
try:
|
||||
youtube.YoutubeAPI.parse_channel_url(playlist_url)
|
||||
except youtube.YoutubeInvalidURLException:
|
||||
raise forms.ValidationError('Invalid playlist/channel URL, or not in a recognized format.')
|
||||
parsed_url = self.yt_api.parse_url(playlist_url)
|
||||
except youtube.InvalidURL as e:
|
||||
raise forms.ValidationError(str(e))
|
||||
|
||||
is_playlist = 'playlist' in parsed_url
|
||||
is_channel = parsed_url['type'] in ('channel', 'user', 'channel_custom')
|
||||
|
||||
if not is_channel and not is_playlist:
|
||||
raise forms.ValidationError('The given URL must link to a channel or a playlist!')
|
||||
|
||||
return playlist_url
|
||||
|
||||
|
||||
@ -269,21 +278,22 @@ class CreateSubscriptionModal(LoginRequiredMixin, ModalMixin, CreateView):
|
||||
api = youtube.YoutubeAPI.build_public()
|
||||
try:
|
||||
form.instance.fetch_from_url(form.cleaned_data['playlist_url'], api)
|
||||
except youtube.YoutubeChannelNotFoundException:
|
||||
return self.modal_response(
|
||||
form, False, 'Could not find a channel based on the given URL. Please verify that the URL is correct.')
|
||||
except youtube.YoutubeUserNotFoundException:
|
||||
return self.modal_response(
|
||||
form, False, 'Could not find an user based on the given URL. Please verify that the URL is correct.')
|
||||
except youtube.YoutubePlaylistNotFoundException:
|
||||
return self.modal_response(
|
||||
form, False, 'Could not find a playlist based on the given URL. Please verify that the URL is correct.')
|
||||
except youtube.YoutubeException as e:
|
||||
return self.modal_response(
|
||||
form, False, str(e))
|
||||
except youtube.APIError as e:
|
||||
return self.modal_response(
|
||||
form, False, 'An error occurred while communicating with the YouTube API: ' + str(e))
|
||||
except youtube.InvalidURL as e:
|
||||
return self.modal_response(form, False, str(e))
|
||||
except ValueError as e:
|
||||
return self.modal_response(form, False, str(e))
|
||||
# except youtube.YoutubeUserNotFoundException:
|
||||
# return self.modal_response(
|
||||
# form, False, 'Could not find an user based on the given URL. Please verify that the URL is correct.')
|
||||
# except youtube.YoutubePlaylistNotFoundException:
|
||||
# return self.modal_response(
|
||||
# form, False, 'Could not find a playlist based on the given URL. Please verify that the URL is correct.')
|
||||
# except youtube.YoutubeException as e:
|
||||
# return self.modal_response(
|
||||
# form, False, str(e))
|
||||
# except youtube.APIError as e:
|
||||
# return self.modal_response(
|
||||
# form, False, 'An error occurred while communicating with the YouTube API: ' + str(e))
|
||||
|
||||
return super().form_valid(form)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user