From 71e389c14a1b70885100da9739683c488853dc1b Mon Sep 17 00:00:00 2001 From: Tiberiu Chibici Date: Mon, 19 Aug 2019 16:40:40 +0300 Subject: [PATCH 1/2] Forgot to modify 'delete_after_watched' to 'automatically_delete_watched' in index views, which was causing a backend error when editing a subscription. --- app/YtManagerApp/views/index.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/app/YtManagerApp/views/index.py b/app/YtManagerApp/views/index.py index acac5c2..1c93f8d 100644 --- a/app/YtManagerApp/views/index.py +++ b/app/YtManagerApp/views/index.py @@ -283,7 +283,7 @@ class CreateSubscriptionForm(forms.ModelForm): 'auto_download', 'download_limit', 'download_order', - 'delete_after_watched' + 'automatically_delete_watched' ) def clean_playlist_url(self): @@ -349,7 +349,7 @@ class UpdateSubscriptionForm(forms.ModelForm): 'auto_download', 'download_limit', 'download_order', - 'delete_after_watched' + 'automatically_delete_watched' ) @@ -403,7 +403,7 @@ class ImportSubscriptionsForm(forms.Form): auto_download = forms.ChoiceField(choices=TRUE_FALSE_CHOICES, required=False) download_limit = forms.IntegerField(required=False) download_order = forms.ChoiceField(choices=VIDEO_ORDER_CHOICES_WITH_EMPTY, required=False) - delete_after_watched = forms.ChoiceField(choices=TRUE_FALSE_CHOICES, required=False) + automatically_delete_watched = forms.ChoiceField(choices=TRUE_FALSE_CHOICES, required=False) def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) @@ -418,7 +418,7 @@ class ImportSubscriptionsForm(forms.Form): 'auto_download', 'download_limit', 'download_order', - 'delete_after_watched' + 'automatically_delete_watched' ) def __clean_empty_none(self, name: str): @@ -438,8 +438,8 @@ class ImportSubscriptionsForm(forms.Form): def clean_auto_download(self): return self.__clean_boolean('auto_download') - def clean_delete_after_watched(self): - return self.__clean_boolean('delete_after_watched') + def clean_automatically_delete_watched(self): + return self.__clean_boolean('automatically_delete_watched') def clean_download_order(self): return self.__clean_empty_none('download_order') From 209e75fa1efdfa5202ed161a0424c56557f14e0b Mon Sep 17 00:00:00 2001 From: Tiberiu Chibici Date: Mon, 19 Aug 2019 16:42:29 +0300 Subject: [PATCH 2/2] Fixed handling of channel subscriptions, where new videos are added to the beginning of the playlist instead of the end. Added 'new' field to videos, so not all the unwatched videos have 'new' badge. --- .../management/jobs/synchronize.py | 19 +++++++++++++-- .../migrations/0010_auto_20190819_1317.py | 23 +++++++++++++++++++ app/YtManagerApp/models.py | 20 +++++++++------- app/YtManagerApp/scheduler.py | 3 ++- .../templates/YtManagerApp/index_videos.html | 2 +- 5 files changed, 55 insertions(+), 12 deletions(-) create mode 100644 app/YtManagerApp/migrations/0010_auto_20190819_1317.py diff --git a/app/YtManagerApp/management/jobs/synchronize.py b/app/YtManagerApp/management/jobs/synchronize.py index a550c6c..9f126af 100644 --- a/app/YtManagerApp/management/jobs/synchronize.py +++ b/app/YtManagerApp/management/jobs/synchronize.py @@ -3,6 +3,7 @@ import itertools from threading import Lock from apscheduler.triggers.cron import CronTrigger +from django.db.models import Max from YtManagerApp.management.appconfig import appconfig from YtManagerApp.management.downloader import fetch_thumbnail, downloader_process_subscription @@ -51,6 +52,9 @@ class SynchronizeJob(Job): self.set_total_steps(len(work_subs) + len(work_vids)) + # Remove the 'new' flag + work_vids.update(new=False) + # Process subscriptions for sub in work_subs: self.progress_advance(1, "Synchronizing subscription " + sub.name) @@ -69,7 +73,7 @@ class SynchronizeJob(Job): batch_ids = [video.video_id for video in batch] video_stats = {v.id: v for v in self.__api.videos(batch_ids, part='id,statistics')} - for video in itertools.chain(work_vids, self.__new_vids): + for video in batch: self.progress_advance(1, "Updating video " + video.name) self.check_video_deleted(video) self.fetch_missing_thumbnails(video) @@ -77,6 +81,7 @@ class SynchronizeJob(Job): if video.video_id in video_stats: self.update_video_stats(video, video_stats[video.video_id]) + # Start downloading videos for sub in work_subs: downloader_process_subscription(sub) @@ -87,12 +92,22 @@ class SynchronizeJob(Job): def check_new_videos(self, sub: Subscription): playlist_items = self.__api.playlist_items(sub.playlist_id) + if sub.rewrite_playlist_indices: + playlist_items = sorted(playlist_items, key=lambda x: x.published_at) + else: + playlist_items = sorted(playlist_items, key=lambda x: x.position) for item in playlist_items: results = Video.objects.filter(video_id=item.resource_video_id, subscription=sub) - if len(results) == 0: + if not results.exists(): self.log.info('New video for subscription %s: %s %s"', sub, item.resource_video_id, item.title) + + # fix playlist index if necessary + if sub.rewrite_playlist_indices or Video.objects.filter(subscription=sub, playlist_index=item.position).exists(): + highest = Video.objects.filter(subscription=sub).aggregate(Max('playlist_index'))['playlist_index__max'] + item.position = 1 + (highest or 0) + self.__new_vids.append(Video.create(item, sub)) def fetch_missing_thumbnails(self, object: Union[Subscription, Video]): diff --git a/app/YtManagerApp/migrations/0010_auto_20190819_1317.py b/app/YtManagerApp/migrations/0010_auto_20190819_1317.py new file mode 100644 index 0000000..475971c --- /dev/null +++ b/app/YtManagerApp/migrations/0010_auto_20190819_1317.py @@ -0,0 +1,23 @@ +# Generated by Django 2.2.4 on 2019-08-19 13:17 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('YtManagerApp', '0009_jobexecution_jobmessage'), + ] + + operations = [ + migrations.AddField( + model_name='subscription', + name='rewrite_playlist_indices', + field=models.BooleanField(default=False), + ), + migrations.AddField( + model_name='video', + name='new', + field=models.BooleanField(default=True), + ), + ] diff --git a/app/YtManagerApp/models.py b/app/YtManagerApp/models.py index 2d0e771..bc04f6c 100644 --- a/app/YtManagerApp/models.py +++ b/app/YtManagerApp/models.py @@ -109,6 +109,8 @@ class Subscription(models.Model): icon_default = models.CharField(max_length=1024) icon_best = models.CharField(max_length=1024) user = models.ForeignKey(User, on_delete=models.CASCADE) + # youtube adds videos to the 'Uploads' playlist at the top instead of the bottom + rewrite_playlist_indices = models.BooleanField(null=False, default=False) # overrides auto_download = models.BooleanField(null=True, blank=True) @@ -143,6 +145,7 @@ class Subscription(models.Model): self.channel_name = info_channel.title self.icon_default = youtube.default_thumbnail(info_channel).url self.icon_best = youtube.best_thumbnail(info_channel).url + self.rewrite_playlist_indices = True def fetch_from_url(self, url, yt_api: youtube.YoutubeAPI): url_parsed = yt_api.parse_url(url) @@ -168,6 +171,7 @@ class Video(models.Model): name = models.TextField(null=False) description = models.TextField() watched = models.BooleanField(default=False, null=False) + new = models.BooleanField(default=True, null=False) downloaded_path = models.TextField(null=True, blank=True) subscription = models.ForeignKey(Subscription, on_delete=models.CASCADE) playlist_index = models.IntegerField(null=False) @@ -185,6 +189,7 @@ class Video(models.Model): video.name = playlist_item.title video.description = playlist_item.description video.watched = False + video.new = True video.downloaded_path = None video.subscription = subscription video.playlist_index = playlist_item.position @@ -228,27 +233,27 @@ class Video(models.Model): for file in self.get_files(): mime, _ = mimetypes.guess_type(file) if mime is not None and mime.startswith('video/'): - return (file, mime) + return file, mime return None, None def delete_files(self): if self.downloaded_path is not None: - from YtManagerApp.management.jobs.delete_video import schedule_delete_video + from YtManagerApp.management.jobs.delete_video import DeleteVideoJob from YtManagerApp.management.appconfig import appconfig - from YtManagerApp.management.jobs.synchronize import schedule_synchronize_now_subscription + from YtManagerApp.management.jobs.synchronize import SynchronizeJob - schedule_delete_video(self) + DeleteVideoJob.schedule(self) # Mark watched? if self.subscription.user.preferences['mark_deleted_as_watched']: self.watched = True - schedule_synchronize_now_subscription(self.subscription) + SynchronizeJob.schedule_now_for_subscription(self.subscription) def download(self): if not self.downloaded_path: - from YtManagerApp.management.jobs.download_video import schedule_download_video - schedule_download_video(self) + from YtManagerApp.management.jobs.download_video import DownloadVideoJob + DownloadVideoJob.schedule(self) def __str__(self): return self.name @@ -298,4 +303,3 @@ class JobMessage(models.Model): message = models.CharField(max_length=1024, null=False, default="") level = models.IntegerField(choices=JOB_MESSAGE_LEVELS, null=False, default=0) suppress_notification = models.BooleanField(null=False, default=False) - diff --git a/app/YtManagerApp/scheduler.py b/app/YtManagerApp/scheduler.py index 83e7d33..5f3e252 100644 --- a/app/YtManagerApp/scheduler.py +++ b/app/YtManagerApp/scheduler.py @@ -3,6 +3,7 @@ import logging import traceback from typing import Type, Union, Optional, Callable, List, Any +import pytz from apscheduler.schedulers.background import BackgroundScheduler from apscheduler.triggers.base import BaseTrigger from django.contrib.auth.models import User @@ -250,7 +251,7 @@ class YtsmScheduler(object): job_execution.status = JOB_STATES_MAP['failed'] finally: - job_execution.end_date = datetime.datetime.now() + job_execution.end_date = datetime.datetime.now(tz=pytz.UTC) job_execution.save() def add_job(self, job_class: Type[Job], trigger: Union[str, BaseTrigger] = None, diff --git a/app/YtManagerApp/templates/YtManagerApp/index_videos.html b/app/YtManagerApp/templates/YtManagerApp/index_videos.html index 8094de0..c63ea1d 100644 --- a/app/YtManagerApp/templates/YtManagerApp/index_videos.html +++ b/app/YtManagerApp/templates/YtManagerApp/index_videos.html @@ -11,7 +11,7 @@
- {% if not video.watched %} + {% if video.new and not video.watched %} New {% endif %}