Fixed handling of channel subscriptions, where new videos are added to the beginning of the playlist instead of the end.

Added 'new' field to videos, so not all the unwatched videos have 'new' badge.

app/YtManagerApp/management/jobs/synchronize.py

@@ -3,6 +3,7 @@ import itertools
 from threading import Lock
 
 from apscheduler.triggers.cron import CronTrigger
+from django.db.models import Max
 
 from YtManagerApp.management.appconfig import appconfig
 from YtManagerApp.management.downloader import fetch_thumbnail, downloader_process_subscription
@@ -51,6 +52,9 @@ class SynchronizeJob(Job):
 
             self.set_total_steps(len(work_subs) + len(work_vids))
 
+            # Remove the 'new' flag
+            work_vids.update(new=False)
+
             # Process subscriptions
             for sub in work_subs:
                 self.progress_advance(1, "Synchronizing subscription " + sub.name)
@@ -69,7 +73,7 @@ class SynchronizeJob(Job):
                     batch_ids = [video.video_id for video in batch]
                     video_stats = {v.id: v for v in self.__api.videos(batch_ids, part='id,statistics')}
 
-                for video in itertools.chain(work_vids, self.__new_vids):
+                for video in batch:
                     self.progress_advance(1, "Updating video " + video.name)
                     self.check_video_deleted(video)
                     self.fetch_missing_thumbnails(video)
@@ -77,6 +81,7 @@ class SynchronizeJob(Job):
                     if video.video_id in video_stats:
                         self.update_video_stats(video, video_stats[video.video_id])
 
+
             # Start downloading videos
             for sub in work_subs:
                 downloader_process_subscription(sub)
@@ -87,12 +92,22 @@ class SynchronizeJob(Job):
 
     def check_new_videos(self, sub: Subscription):
         playlist_items = self.__api.playlist_items(sub.playlist_id)
+        if sub.rewrite_playlist_indices:
+            playlist_items = sorted(playlist_items, key=lambda x: x.published_at)
+        else:
+            playlist_items = sorted(playlist_items, key=lambda x: x.position)
 
         for item in playlist_items:
             results = Video.objects.filter(video_id=item.resource_video_id, subscription=sub)
 
-            if len(results) == 0:
+            if not results.exists():
                 self.log.info('New video for subscription %s: %s %s"', sub, item.resource_video_id, item.title)
+
+                # fix playlist index if necessary
+                if sub.rewrite_playlist_indices or Video.objects.filter(subscription=sub, playlist_index=item.position).exists():
+                    highest = Video.objects.filter(subscription=sub).aggregate(Max('playlist_index'))['playlist_index__max']
+                    item.position = 1 + (highest or 0)
+
                 self.__new_vids.append(Video.create(item, sub))
 
     def fetch_missing_thumbnails(self, object: Union[Subscription, Video]):

app/YtManagerApp/migrations/0010_auto_20190819_1317.py

@@ -0,0 +1,23 @@
+# Generated by Django 2.2.4 on 2019-08-19 13:17
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('YtManagerApp', '0009_jobexecution_jobmessage'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='subscription',
+            name='rewrite_playlist_indices',
+            field=models.BooleanField(default=False),
+        ),
+        migrations.AddField(
+            model_name='video',
+            name='new',
+            field=models.BooleanField(default=True),
+        ),
+    ]

app/YtManagerApp/models.py

@@ -109,6 +109,8 @@ class Subscription(models.Model):
     icon_default = models.CharField(max_length=1024)
     icon_best = models.CharField(max_length=1024)
     user = models.ForeignKey(User, on_delete=models.CASCADE)
+    # youtube adds videos to the 'Uploads' playlist at the top instead of the bottom
+    rewrite_playlist_indices = models.BooleanField(null=False, default=False)
 
     # overrides
     auto_download = models.BooleanField(null=True, blank=True)
@@ -143,6 +145,7 @@ class Subscription(models.Model):
         self.channel_name = info_channel.title
         self.icon_default = youtube.default_thumbnail(info_channel).url
         self.icon_best = youtube.best_thumbnail(info_channel).url
+        self.rewrite_playlist_indices = True
 
     def fetch_from_url(self, url, yt_api: youtube.YoutubeAPI):
         url_parsed = yt_api.parse_url(url)
@@ -168,6 +171,7 @@ class Video(models.Model):
     name = models.TextField(null=False)
     description = models.TextField()
     watched = models.BooleanField(default=False, null=False)
+    new = models.BooleanField(default=True, null=False)
     downloaded_path = models.TextField(null=True, blank=True)
     subscription = models.ForeignKey(Subscription, on_delete=models.CASCADE)
     playlist_index = models.IntegerField(null=False)
@@ -185,6 +189,7 @@ class Video(models.Model):
         video.name = playlist_item.title
         video.description = playlist_item.description
         video.watched = False
+        video.new = True
         video.downloaded_path = None
         video.subscription = subscription
         video.playlist_index = playlist_item.position
@@ -228,27 +233,27 @@ class Video(models.Model):
         for file in self.get_files():
             mime, _ = mimetypes.guess_type(file)
             if mime is not None and mime.startswith('video/'):
-                return (file, mime)
+                return file, mime
 
         return None, None
 
     def delete_files(self):
         if self.downloaded_path is not None:
-            from YtManagerApp.management.jobs.delete_video import schedule_delete_video
+            from YtManagerApp.management.jobs.delete_video import DeleteVideoJob
             from YtManagerApp.management.appconfig import appconfig
-            from YtManagerApp.management.jobs.synchronize import schedule_synchronize_now_subscription
+            from YtManagerApp.management.jobs.synchronize import SynchronizeJob
 
-            schedule_delete_video(self)
+            DeleteVideoJob.schedule(self)
 
             # Mark watched?
             if self.subscription.user.preferences['mark_deleted_as_watched']:
                 self.watched = True
-                schedule_synchronize_now_subscription(self.subscription)
+                SynchronizeJob.schedule_now_for_subscription(self.subscription)
 
     def download(self):
         if not self.downloaded_path:
-            from YtManagerApp.management.jobs.download_video import schedule_download_video
-            schedule_download_video(self)
+            from YtManagerApp.management.jobs.download_video import DownloadVideoJob
+            DownloadVideoJob.schedule(self)
 
     def __str__(self):
         return self.name
@@ -298,4 +303,3 @@ class JobMessage(models.Model):
     message = models.CharField(max_length=1024, null=False, default="")
     level = models.IntegerField(choices=JOB_MESSAGE_LEVELS, null=False, default=0)
     suppress_notification = models.BooleanField(null=False, default=False)
-

app/YtManagerApp/scheduler.py

@@ -3,6 +3,7 @@ import logging
 import traceback
 from typing import Type, Union, Optional, Callable, List, Any
 
+import pytz
 from apscheduler.schedulers.background import BackgroundScheduler
 from apscheduler.triggers.base import BaseTrigger
 from django.contrib.auth.models import User
@@ -250,7 +251,7 @@ class YtsmScheduler(object):
             job_execution.status = JOB_STATES_MAP['failed']
 
         finally:
-            job_execution.end_date = datetime.datetime.now()
+            job_execution.end_date = datetime.datetime.now(tz=pytz.UTC)
             job_execution.save()
 
     def add_job(self, job_class: Type[Job], trigger: Union[str, BaseTrigger] = None,

app/YtManagerApp/templates/YtManagerApp/index_videos.html

@@ -11,7 +11,7 @@
                     </a>
                     <div class="card-body">
                         <h5 class="card-title">
-                            {% if not video.watched %}
+                            {% if video.new and not video.watched %}
                             <sup class="badge badge-primary">New</sup>
                             {% endif %}
                             <a href="{% url 'video' video.id %}" target="_blank">