Merge pull request #72 from chibicitiberiu/development

Fixed several issues and added 'new' field for videos

app/YtManagerApp/management/jobs/synchronize.py

@@ -3,6 +3,7 @@ import itertools
 from threading import Lock
 
 from apscheduler.triggers.cron import CronTrigger
+from django.db.models import Max
 
 from YtManagerApp.management.appconfig import appconfig
 from YtManagerApp.management.downloader import fetch_thumbnail, downloader_process_subscription
@@ -51,6 +52,9 @@ class SynchronizeJob(Job):
 
             self.set_total_steps(len(work_subs) + len(work_vids))
 
+            # Remove the 'new' flag
+            work_vids.update(new=False)
+
             # Process subscriptions
             for sub in work_subs:
                 self.progress_advance(1, "Synchronizing subscription " + sub.name)
@@ -77,6 +81,7 @@ class SynchronizeJob(Job):
                     if video.video_id in video_stats:
                         self.update_video_stats(video, video_stats[video.video_id])
 
+
             # Start downloading videos
             for sub in work_subs:
                 downloader_process_subscription(sub)
@@ -87,12 +92,22 @@ class SynchronizeJob(Job):
 
     def check_new_videos(self, sub: Subscription):
         playlist_items = self.__api.playlist_items(sub.playlist_id)
+        if sub.rewrite_playlist_indices:
+            playlist_items = sorted(playlist_items, key=lambda x: x.published_at)
+        else:
+            playlist_items = sorted(playlist_items, key=lambda x: x.position)
 
         for item in playlist_items:
             results = Video.objects.filter(video_id=item.resource_video_id, subscription=sub)
 
-            if len(results) == 0:
+            if not results.exists():
                 self.log.info('New video for subscription %s: %s %s"', sub, item.resource_video_id, item.title)
+
+                # fix playlist index if necessary
+                if sub.rewrite_playlist_indices or Video.objects.filter(subscription=sub, playlist_index=item.position).exists():
+                    highest = Video.objects.filter(subscription=sub).aggregate(Max('playlist_index'))['playlist_index__max']
+                    item.position = 1 + (highest or 0)
+
                 self.__new_vids.append(Video.create(item, sub))
 
     def fetch_missing_thumbnails(self, object: Union[Subscription, Video]):

app/YtManagerApp/migrations/0010_auto_20190819_1317.py

@@ -0,0 +1,23 @@
+# Generated by Django 2.2.4 on 2019-08-19 13:17
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('YtManagerApp', '0009_jobexecution_jobmessage'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='subscription',
+            name='rewrite_playlist_indices',
+            field=models.BooleanField(default=False),
+        ),
+        migrations.AddField(
+            model_name='video',
+            name='new',
+            field=models.BooleanField(default=True),
+        ),
+    ]

app/YtManagerApp/models.py

@@ -109,6 +109,8 @@ class Subscription(models.Model):
     icon_default = models.CharField(max_length=1024)
     icon_best = models.CharField(max_length=1024)
     user = models.ForeignKey(User, on_delete=models.CASCADE)
+    # youtube adds videos to the 'Uploads' playlist at the top instead of the bottom
+    rewrite_playlist_indices = models.BooleanField(null=False, default=False)
 
     # overrides
     auto_download = models.BooleanField(null=True, blank=True)
@@ -143,6 +145,7 @@ class Subscription(models.Model):
         self.channel_name = info_channel.title
         self.icon_default = youtube.default_thumbnail(info_channel).url
         self.icon_best = youtube.best_thumbnail(info_channel).url
+        self.rewrite_playlist_indices = True
 
     def fetch_from_url(self, url, yt_api: youtube.YoutubeAPI):
         url_parsed = yt_api.parse_url(url)
@@ -168,6 +171,7 @@ class Video(models.Model):
     name = models.TextField(null=False)
     description = models.TextField()
     watched = models.BooleanField(default=False, null=False)
+    new = models.BooleanField(default=True, null=False)
     downloaded_path = models.TextField(null=True, blank=True)
     subscription = models.ForeignKey(Subscription, on_delete=models.CASCADE)
     playlist_index = models.IntegerField(null=False)
@@ -185,6 +189,7 @@ class Video(models.Model):
         video.name = playlist_item.title
         video.description = playlist_item.description
         video.watched = False
+        video.new = True
         video.downloaded_path = None
         video.subscription = subscription
         video.playlist_index = playlist_item.position

app/YtManagerApp/scheduler.py

@@ -3,6 +3,7 @@ import logging
 import traceback
 from typing import Type, Union, Optional, Callable, List, Any
 
+import pytz
 from apscheduler.schedulers.background import BackgroundScheduler
 from apscheduler.triggers.base import BaseTrigger
 from django.contrib.auth.models import User
@@ -250,7 +251,7 @@ class YtsmScheduler(object):
             job_execution.status = JOB_STATES_MAP['failed']
 
         finally:
-            job_execution.end_date = datetime.datetime.now()
+            job_execution.end_date = datetime.datetime.now(tz=pytz.UTC)
             job_execution.save()
 
     def add_job(self, job_class: Type[Job], trigger: Union[str, BaseTrigger] = None,

app/YtManagerApp/templates/YtManagerApp/index_videos.html

@@ -11,7 +11,7 @@
                     </a>
                     <div class="card-body">
                         <h5 class="card-title">
-                            {% if not video.watched %}
+                            {% if video.new and not video.watched %}
                             <sup class="badge badge-primary">New</sup>
                             {% endif %}
                             <a href="{% url 'video' video.id %}" target="_blank">

app/YtManagerApp/views/index.py

@@ -283,7 +283,7 @@ class CreateSubscriptionForm(forms.ModelForm):
             'auto_download',
             'download_limit',
             'download_order',
-            'delete_after_watched'
+            'automatically_delete_watched'
         )
 
     def clean_playlist_url(self):
@@ -349,7 +349,7 @@ class UpdateSubscriptionForm(forms.ModelForm):
             'auto_download',
             'download_limit',
             'download_order',
-            'delete_after_watched'
+            'automatically_delete_watched'
         )
 
 
@@ -403,7 +403,7 @@ class ImportSubscriptionsForm(forms.Form):
     auto_download = forms.ChoiceField(choices=TRUE_FALSE_CHOICES, required=False)
     download_limit = forms.IntegerField(required=False)
     download_order = forms.ChoiceField(choices=VIDEO_ORDER_CHOICES_WITH_EMPTY, required=False)
-    delete_after_watched = forms.ChoiceField(choices=TRUE_FALSE_CHOICES, required=False)
+    automatically_delete_watched = forms.ChoiceField(choices=TRUE_FALSE_CHOICES, required=False)
 
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
@@ -418,7 +418,7 @@ class ImportSubscriptionsForm(forms.Form):
             'auto_download',
             'download_limit',
             'download_order',
-            'delete_after_watched'
+            'automatically_delete_watched'
         )
 
     def __clean_empty_none(self, name: str):
@@ -438,8 +438,8 @@ class ImportSubscriptionsForm(forms.Form):
     def clean_auto_download(self):
         return self.__clean_boolean('auto_download')
 
-    def clean_delete_after_watched(self):
+    def clean_automatically_delete_watched(self):
-        return self.__clean_boolean('delete_after_watched')
+        return self.__clean_boolean('automatically_delete_watched')
 
     def clean_download_order(self):
         return self.__clean_empty_none('download_order')