From 209e75fa1efdfa5202ed161a0424c56557f14e0b Mon Sep 17 00:00:00 2001 From: Tiberiu Chibici Date: Mon, 19 Aug 2019 16:42:29 +0300 Subject: [PATCH] Fixed handling of channel subscriptions, where new videos are added to the beginning of the playlist instead of the end. Added 'new' field to videos, so not all the unwatched videos have 'new' badge. --- .../management/jobs/synchronize.py | 19 +++++++++++++-- .../migrations/0010_auto_20190819_1317.py | 23 +++++++++++++++++++ app/YtManagerApp/models.py | 20 +++++++++------- app/YtManagerApp/scheduler.py | 3 ++- .../templates/YtManagerApp/index_videos.html | 2 +- 5 files changed, 55 insertions(+), 12 deletions(-) create mode 100644 app/YtManagerApp/migrations/0010_auto_20190819_1317.py diff --git a/app/YtManagerApp/management/jobs/synchronize.py b/app/YtManagerApp/management/jobs/synchronize.py index a550c6c..9f126af 100644 --- a/app/YtManagerApp/management/jobs/synchronize.py +++ b/app/YtManagerApp/management/jobs/synchronize.py @@ -3,6 +3,7 @@ import itertools from threading import Lock from apscheduler.triggers.cron import CronTrigger +from django.db.models import Max from YtManagerApp.management.appconfig import appconfig from YtManagerApp.management.downloader import fetch_thumbnail, downloader_process_subscription @@ -51,6 +52,9 @@ class SynchronizeJob(Job): self.set_total_steps(len(work_subs) + len(work_vids)) + # Remove the 'new' flag + work_vids.update(new=False) + # Process subscriptions for sub in work_subs: self.progress_advance(1, "Synchronizing subscription " + sub.name) @@ -69,7 +73,7 @@ class SynchronizeJob(Job): batch_ids = [video.video_id for video in batch] video_stats = {v.id: v for v in self.__api.videos(batch_ids, part='id,statistics')} - for video in itertools.chain(work_vids, self.__new_vids): + for video in batch: self.progress_advance(1, "Updating video " + video.name) self.check_video_deleted(video) self.fetch_missing_thumbnails(video) @@ -77,6 +81,7 @@ class SynchronizeJob(Job): if video.video_id in video_stats: self.update_video_stats(video, video_stats[video.video_id]) + # Start downloading videos for sub in work_subs: downloader_process_subscription(sub) @@ -87,12 +92,22 @@ class SynchronizeJob(Job): def check_new_videos(self, sub: Subscription): playlist_items = self.__api.playlist_items(sub.playlist_id) + if sub.rewrite_playlist_indices: + playlist_items = sorted(playlist_items, key=lambda x: x.published_at) + else: + playlist_items = sorted(playlist_items, key=lambda x: x.position) for item in playlist_items: results = Video.objects.filter(video_id=item.resource_video_id, subscription=sub) - if len(results) == 0: + if not results.exists(): self.log.info('New video for subscription %s: %s %s"', sub, item.resource_video_id, item.title) + + # fix playlist index if necessary + if sub.rewrite_playlist_indices or Video.objects.filter(subscription=sub, playlist_index=item.position).exists(): + highest = Video.objects.filter(subscription=sub).aggregate(Max('playlist_index'))['playlist_index__max'] + item.position = 1 + (highest or 0) + self.__new_vids.append(Video.create(item, sub)) def fetch_missing_thumbnails(self, object: Union[Subscription, Video]): diff --git a/app/YtManagerApp/migrations/0010_auto_20190819_1317.py b/app/YtManagerApp/migrations/0010_auto_20190819_1317.py new file mode 100644 index 0000000..475971c --- /dev/null +++ b/app/YtManagerApp/migrations/0010_auto_20190819_1317.py @@ -0,0 +1,23 @@ +# Generated by Django 2.2.4 on 2019-08-19 13:17 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('YtManagerApp', '0009_jobexecution_jobmessage'), + ] + + operations = [ + migrations.AddField( + model_name='subscription', + name='rewrite_playlist_indices', + field=models.BooleanField(default=False), + ), + migrations.AddField( + model_name='video', + name='new', + field=models.BooleanField(default=True), + ), + ] diff --git a/app/YtManagerApp/models.py b/app/YtManagerApp/models.py index 2d0e771..bc04f6c 100644 --- a/app/YtManagerApp/models.py +++ b/app/YtManagerApp/models.py @@ -109,6 +109,8 @@ class Subscription(models.Model): icon_default = models.CharField(max_length=1024) icon_best = models.CharField(max_length=1024) user = models.ForeignKey(User, on_delete=models.CASCADE) + # youtube adds videos to the 'Uploads' playlist at the top instead of the bottom + rewrite_playlist_indices = models.BooleanField(null=False, default=False) # overrides auto_download = models.BooleanField(null=True, blank=True) @@ -143,6 +145,7 @@ class Subscription(models.Model): self.channel_name = info_channel.title self.icon_default = youtube.default_thumbnail(info_channel).url self.icon_best = youtube.best_thumbnail(info_channel).url + self.rewrite_playlist_indices = True def fetch_from_url(self, url, yt_api: youtube.YoutubeAPI): url_parsed = yt_api.parse_url(url) @@ -168,6 +171,7 @@ class Video(models.Model): name = models.TextField(null=False) description = models.TextField() watched = models.BooleanField(default=False, null=False) + new = models.BooleanField(default=True, null=False) downloaded_path = models.TextField(null=True, blank=True) subscription = models.ForeignKey(Subscription, on_delete=models.CASCADE) playlist_index = models.IntegerField(null=False) @@ -185,6 +189,7 @@ class Video(models.Model): video.name = playlist_item.title video.description = playlist_item.description video.watched = False + video.new = True video.downloaded_path = None video.subscription = subscription video.playlist_index = playlist_item.position @@ -228,27 +233,27 @@ class Video(models.Model): for file in self.get_files(): mime, _ = mimetypes.guess_type(file) if mime is not None and mime.startswith('video/'): - return (file, mime) + return file, mime return None, None def delete_files(self): if self.downloaded_path is not None: - from YtManagerApp.management.jobs.delete_video import schedule_delete_video + from YtManagerApp.management.jobs.delete_video import DeleteVideoJob from YtManagerApp.management.appconfig import appconfig - from YtManagerApp.management.jobs.synchronize import schedule_synchronize_now_subscription + from YtManagerApp.management.jobs.synchronize import SynchronizeJob - schedule_delete_video(self) + DeleteVideoJob.schedule(self) # Mark watched? if self.subscription.user.preferences['mark_deleted_as_watched']: self.watched = True - schedule_synchronize_now_subscription(self.subscription) + SynchronizeJob.schedule_now_for_subscription(self.subscription) def download(self): if not self.downloaded_path: - from YtManagerApp.management.jobs.download_video import schedule_download_video - schedule_download_video(self) + from YtManagerApp.management.jobs.download_video import DownloadVideoJob + DownloadVideoJob.schedule(self) def __str__(self): return self.name @@ -298,4 +303,3 @@ class JobMessage(models.Model): message = models.CharField(max_length=1024, null=False, default="") level = models.IntegerField(choices=JOB_MESSAGE_LEVELS, null=False, default=0) suppress_notification = models.BooleanField(null=False, default=False) - diff --git a/app/YtManagerApp/scheduler.py b/app/YtManagerApp/scheduler.py index 83e7d33..5f3e252 100644 --- a/app/YtManagerApp/scheduler.py +++ b/app/YtManagerApp/scheduler.py @@ -3,6 +3,7 @@ import logging import traceback from typing import Type, Union, Optional, Callable, List, Any +import pytz from apscheduler.schedulers.background import BackgroundScheduler from apscheduler.triggers.base import BaseTrigger from django.contrib.auth.models import User @@ -250,7 +251,7 @@ class YtsmScheduler(object): job_execution.status = JOB_STATES_MAP['failed'] finally: - job_execution.end_date = datetime.datetime.now() + job_execution.end_date = datetime.datetime.now(tz=pytz.UTC) job_execution.save() def add_job(self, job_class: Type[Job], trigger: Union[str, BaseTrigger] = None, diff --git a/app/YtManagerApp/templates/YtManagerApp/index_videos.html b/app/YtManagerApp/templates/YtManagerApp/index_videos.html index 8094de0..c63ea1d 100644 --- a/app/YtManagerApp/templates/YtManagerApp/index_videos.html +++ b/app/YtManagerApp/templates/YtManagerApp/index_videos.html @@ -11,7 +11,7 @@
- {% if not video.watched %} + {% if video.new and not video.watched %} New {% endif %}