diff --git a/app/YtManagerApp/management/jobs/synchronize.py b/app/YtManagerApp/management/jobs/synchronize.py index eae3f80..9f126af 100644 --- a/app/YtManagerApp/management/jobs/synchronize.py +++ b/app/YtManagerApp/management/jobs/synchronize.py @@ -3,6 +3,7 @@ import itertools from threading import Lock from apscheduler.triggers.cron import CronTrigger +from django.db.models import Max from YtManagerApp.management.appconfig import appconfig from YtManagerApp.management.downloader import fetch_thumbnail, downloader_process_subscription @@ -51,6 +52,9 @@ class SynchronizeJob(Job): self.set_total_steps(len(work_subs) + len(work_vids)) + # Remove the 'new' flag + work_vids.update(new=False) + # Process subscriptions for sub in work_subs: self.progress_advance(1, "Synchronizing subscription " + sub.name) @@ -77,6 +81,7 @@ class SynchronizeJob(Job): if video.video_id in video_stats: self.update_video_stats(video, video_stats[video.video_id]) + # Start downloading videos for sub in work_subs: downloader_process_subscription(sub) @@ -87,12 +92,22 @@ class SynchronizeJob(Job): def check_new_videos(self, sub: Subscription): playlist_items = self.__api.playlist_items(sub.playlist_id) + if sub.rewrite_playlist_indices: + playlist_items = sorted(playlist_items, key=lambda x: x.published_at) + else: + playlist_items = sorted(playlist_items, key=lambda x: x.position) for item in playlist_items: results = Video.objects.filter(video_id=item.resource_video_id, subscription=sub) - if len(results) == 0: + if not results.exists(): self.log.info('New video for subscription %s: %s %s"', sub, item.resource_video_id, item.title) + + # fix playlist index if necessary + if sub.rewrite_playlist_indices or Video.objects.filter(subscription=sub, playlist_index=item.position).exists(): + highest = Video.objects.filter(subscription=sub).aggregate(Max('playlist_index'))['playlist_index__max'] + item.position = 1 + (highest or 0) + self.__new_vids.append(Video.create(item, sub)) def fetch_missing_thumbnails(self, object: Union[Subscription, Video]): diff --git a/app/YtManagerApp/migrations/0010_auto_20190819_1317.py b/app/YtManagerApp/migrations/0010_auto_20190819_1317.py new file mode 100644 index 0000000..475971c --- /dev/null +++ b/app/YtManagerApp/migrations/0010_auto_20190819_1317.py @@ -0,0 +1,23 @@ +# Generated by Django 2.2.4 on 2019-08-19 13:17 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('YtManagerApp', '0009_jobexecution_jobmessage'), + ] + + operations = [ + migrations.AddField( + model_name='subscription', + name='rewrite_playlist_indices', + field=models.BooleanField(default=False), + ), + migrations.AddField( + model_name='video', + name='new', + field=models.BooleanField(default=True), + ), + ] diff --git a/app/YtManagerApp/models.py b/app/YtManagerApp/models.py index 7fe8265..bc04f6c 100644 --- a/app/YtManagerApp/models.py +++ b/app/YtManagerApp/models.py @@ -109,6 +109,8 @@ class Subscription(models.Model): icon_default = models.CharField(max_length=1024) icon_best = models.CharField(max_length=1024) user = models.ForeignKey(User, on_delete=models.CASCADE) + # youtube adds videos to the 'Uploads' playlist at the top instead of the bottom + rewrite_playlist_indices = models.BooleanField(null=False, default=False) # overrides auto_download = models.BooleanField(null=True, blank=True) @@ -143,6 +145,7 @@ class Subscription(models.Model): self.channel_name = info_channel.title self.icon_default = youtube.default_thumbnail(info_channel).url self.icon_best = youtube.best_thumbnail(info_channel).url + self.rewrite_playlist_indices = True def fetch_from_url(self, url, yt_api: youtube.YoutubeAPI): url_parsed = yt_api.parse_url(url) @@ -168,6 +171,7 @@ class Video(models.Model): name = models.TextField(null=False) description = models.TextField() watched = models.BooleanField(default=False, null=False) + new = models.BooleanField(default=True, null=False) downloaded_path = models.TextField(null=True, blank=True) subscription = models.ForeignKey(Subscription, on_delete=models.CASCADE) playlist_index = models.IntegerField(null=False) @@ -185,6 +189,7 @@ class Video(models.Model): video.name = playlist_item.title video.description = playlist_item.description video.watched = False + video.new = True video.downloaded_path = None video.subscription = subscription video.playlist_index = playlist_item.position diff --git a/app/YtManagerApp/scheduler.py b/app/YtManagerApp/scheduler.py index 83e7d33..5f3e252 100644 --- a/app/YtManagerApp/scheduler.py +++ b/app/YtManagerApp/scheduler.py @@ -3,6 +3,7 @@ import logging import traceback from typing import Type, Union, Optional, Callable, List, Any +import pytz from apscheduler.schedulers.background import BackgroundScheduler from apscheduler.triggers.base import BaseTrigger from django.contrib.auth.models import User @@ -250,7 +251,7 @@ class YtsmScheduler(object): job_execution.status = JOB_STATES_MAP['failed'] finally: - job_execution.end_date = datetime.datetime.now() + job_execution.end_date = datetime.datetime.now(tz=pytz.UTC) job_execution.save() def add_job(self, job_class: Type[Job], trigger: Union[str, BaseTrigger] = None, diff --git a/app/YtManagerApp/templates/YtManagerApp/index_videos.html b/app/YtManagerApp/templates/YtManagerApp/index_videos.html index 8094de0..c63ea1d 100644 --- a/app/YtManagerApp/templates/YtManagerApp/index_videos.html +++ b/app/YtManagerApp/templates/YtManagerApp/index_videos.html @@ -11,7 +11,7 @@
- {% if not video.watched %} + {% if video.new and not video.watched %} New {% endif %} diff --git a/app/YtManagerApp/views/index.py b/app/YtManagerApp/views/index.py index acac5c2..1c93f8d 100644 --- a/app/YtManagerApp/views/index.py +++ b/app/YtManagerApp/views/index.py @@ -283,7 +283,7 @@ class CreateSubscriptionForm(forms.ModelForm): 'auto_download', 'download_limit', 'download_order', - 'delete_after_watched' + 'automatically_delete_watched' ) def clean_playlist_url(self): @@ -349,7 +349,7 @@ class UpdateSubscriptionForm(forms.ModelForm): 'auto_download', 'download_limit', 'download_order', - 'delete_after_watched' + 'automatically_delete_watched' ) @@ -403,7 +403,7 @@ class ImportSubscriptionsForm(forms.Form): auto_download = forms.ChoiceField(choices=TRUE_FALSE_CHOICES, required=False) download_limit = forms.IntegerField(required=False) download_order = forms.ChoiceField(choices=VIDEO_ORDER_CHOICES_WITH_EMPTY, required=False) - delete_after_watched = forms.ChoiceField(choices=TRUE_FALSE_CHOICES, required=False) + automatically_delete_watched = forms.ChoiceField(choices=TRUE_FALSE_CHOICES, required=False) def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) @@ -418,7 +418,7 @@ class ImportSubscriptionsForm(forms.Form): 'auto_download', 'download_limit', 'download_order', - 'delete_after_watched' + 'automatically_delete_watched' ) def __clean_empty_none(self, name: str): @@ -438,8 +438,8 @@ class ImportSubscriptionsForm(forms.Form): def clean_auto_download(self): return self.__clean_boolean('auto_download') - def clean_delete_after_watched(self): - return self.__clean_boolean('delete_after_watched') + def clean_automatically_delete_watched(self): + return self.__clean_boolean('automatically_delete_watched') def clean_download_order(self): return self.__clean_empty_none('download_order')