From 57c2265f71add0cf29334ef9a5d7b3059c7f6b34 Mon Sep 17 00:00:00 2001
From: Tiberiu Chibici <chibicitiberiu@gmail.com>
Date: Sun, 4 Nov 2018 23:32:18 +0200
Subject: [PATCH] Added a basic notification system. Long processes will call
 the notification API, and the notification events are registered. For now,
 notifications are pushed to the client by polling (client polls every second
 for new events). A basic status message is now displayed when the sync
 process starts and ends.

---
 .../management/jobs/synchronize.py            |  7 ++
 .../management/notification_manager.py        | 93 +++++++++++++++++++
 .../templates/YtManagerApp/index.html         |  2 +
 .../templates/YtManagerApp/js/index.js        | 27 ++++++
 .../YtManagerApp/master_default.html          |  2 +-
 app/YtManagerApp/urls.py                      |  3 +
 app/YtManagerApp/utils/algorithms.py          | 57 ++++++++++++
 app/YtManagerApp/views/index.py               |  4 +-
 app/YtManagerApp/views/notifications.py       | 12 +++
 9 files changed, 205 insertions(+), 2 deletions(-)
 create mode 100644 app/YtManagerApp/management/notification_manager.py
 create mode 100644 app/YtManagerApp/utils/algorithms.py
 create mode 100644 app/YtManagerApp/views/notifications.py

diff --git a/app/YtManagerApp/management/jobs/synchronize.py b/app/YtManagerApp/management/jobs/synchronize.py
index f6db0d8..5c6940a 100644
--- a/app/YtManagerApp/management/jobs/synchronize.py
+++ b/app/YtManagerApp/management/jobs/synchronize.py
@@ -10,6 +10,8 @@ from YtManagerApp.management.downloader import fetch_thumbnail, downloader_proce
 from YtManagerApp.models import *
 from YtManagerApp.utils import youtube
 
+from YtManagerApp.management import notification_manager
+
 log = logging.getLogger('sync')
 __lock = Lock()
 
@@ -104,6 +106,7 @@ def synchronize():
 
     try:
         log.info("Running scheduled synchronization... ")
+        notification_manager.notify_status_update(f'Synchronization started for all subscriptions.')
 
         # Sync subscribed playlists/channels
         log.info("Sync - checking videos")
@@ -119,6 +122,7 @@ def synchronize():
         __fetch_thumbnails()
 
         log.info("Synchronization finished.")
+        notification_manager.notify_status_update(f'Synchronization finished for all subscriptions.')
 
     finally:
         __lock.release()
@@ -128,6 +132,8 @@ def synchronize_subscription(subscription: Subscription):
     __lock.acquire()
     try:
         log.info("Running synchronization for single subscription %d [%s]", subscription.id, subscription.name)
+        notification_manager.notify_status_update(f'Synchronization started for subscription <strong>{subscription.name}</strong>.')
+
         yt_api = youtube.YoutubeAPI.build_public()
 
         log.info("Sync - checking videos")
@@ -141,6 +147,7 @@ def synchronize_subscription(subscription: Subscription):
         __fetch_thumbnails()
 
         log.info("Synchronization finished for subscription %d [%s].", subscription.id, subscription.name)
+        notification_manager.notify_status_update(f'Synchronization finished for subscription <strong>{subscription.name}</strong>.')
 
     finally:
         __lock.release()
diff --git a/app/YtManagerApp/management/notification_manager.py b/app/YtManagerApp/management/notification_manager.py
new file mode 100644
index 0000000..c115d14
--- /dev/null
+++ b/app/YtManagerApp/management/notification_manager.py
@@ -0,0 +1,93 @@
+from django.contrib.auth.models import User
+from typing import Dict, Deque, Any, Optional
+from collections import deque
+from datetime import datetime, timedelta
+from YtManagerApp.utils.algorithms import bisect_left
+from threading import Lock
+
+# Clients will request updates at most every few seconds, so a retention period of 60 seconds should be more than
+# enough. I gave it 15 minutes so that if for some reason the connection fails (internet drops) and then comes back a
+# few minutes later, the client will still get the updates
+__RETENTION_PERIOD = 15 * 60
+__NOTIFICATIONS: Deque[Dict] = deque()
+__NEXT_ID = 0
+__LOCK = Lock()
+
+
+# Messages enum
+class Messages:
+    STATUS_UPDATE = 'st-up'
+    STATUS_OPERATION_PROGRESS = 'st-op-prog'
+    STATUS_OPERATION_END = 'st-op-end'
+
+
+def __add_notification(message, user: User=None, **kwargs):
+    global __NEXT_ID
+
+    __LOCK.acquire()
+
+    try:
+        # add notification
+        notification = {
+            'time': datetime.now(),
+            'msg': message,
+            'id': __NEXT_ID,
+            'uid': user and user.id,
+        }
+        notification.update(kwargs)
+        __NOTIFICATIONS.append(notification)
+        __NEXT_ID += 1
+
+        # trim old notifications
+        oldest = __NOTIFICATIONS[0]
+        while len(__NOTIFICATIONS) > 0 and oldest['time'] + timedelta(seconds=__RETENTION_PERIOD) < datetime.now():
+            __NOTIFICATIONS.popleft()
+            oldest = __NOTIFICATIONS[0]
+
+    finally:
+        __LOCK.release()
+
+
+def get_notifications(user: User, last_received_id: Optional[int]):
+
+    __LOCK.acquire()
+
+    try:
+        first_index = 0
+        if last_received_id is not None:
+            first_index = bisect_left(__NOTIFICATIONS,
+                                 {'id': last_received_id},
+                                 key=lambda item: item['id'])
+
+        for i in range(first_index, len(__NOTIFICATIONS)):
+            item = __NOTIFICATIONS[i]
+            if item['uid'] is None or item['uid'] == user.id:
+                yield item
+
+    finally:
+        __LOCK.release()
+
+
+def get_current_notification_id():
+    return __NEXT_ID
+
+
+def notify_status_update(status_message: str, user: User=None):
+    __add_notification(Messages.STATUS_UPDATE,
+                       user=user,
+                       status=status_message)
+
+
+def notify_status_operation_progress(op_id: Any, status_message: str, progress_percent: float, user: User=None):
+    __add_notification(Messages.STATUS_OPERATION_PROGRESS,
+                       user=user,
+                       operation=op_id,
+                       status=status_message,
+                       progress=progress_percent)
+
+
+def notify_status_operation_ended(op_id: Any, status_message: str, user: User=None):
+    __add_notification(Messages.STATUS_OPERATION_END,
+                       user=user,
+                       operation=op_id,
+                       status=status_message)
diff --git a/app/YtManagerApp/templates/YtManagerApp/index.html b/app/YtManagerApp/templates/YtManagerApp/index.html
index cf158ee..0cf473d 100644
--- a/app/YtManagerApp/templates/YtManagerApp/index.html
+++ b/app/YtManagerApp/templates/YtManagerApp/index.html
@@ -9,6 +9,8 @@
 {% block scripts %}
     <script src="https://cdnjs.cloudflare.com/ajax/libs/jstree/3.2.1/jstree.min.js"></script>
     <script>
+        let LAST_NOTIFICATION_ID = {{ current_notification_id }};
+
         {% include 'YtManagerApp/js/index.js' %}
     </script>
 {% endblock %}
diff --git a/app/YtManagerApp/templates/YtManagerApp/js/index.js b/app/YtManagerApp/templates/YtManagerApp/js/index.js
index 6fe9cfd..88c9ab3 100644
--- a/app/YtManagerApp/templates/YtManagerApp/js/index.js
+++ b/app/YtManagerApp/templates/YtManagerApp/js/index.js
@@ -157,6 +157,30 @@ function videos_Submit(e)
         e.preventDefault();
 }
 
+///
+/// Notifications
+///
+const NOTIFICATION_INTERVAL = 1000;
+
+function get_and_process_notifications()
+{
+    $.get("{% url 'ajax_get_notifications' 12345 %}".replace("12345", LAST_NOTIFICATION_ID))
+        .done(function(data) {
+            for (let entry of data)
+            {
+                LAST_NOTIFICATION_ID = entry.id;
+                let dt = new Date(entry.time);
+
+                // Status update
+                if (entry.msg === 'st-up') {
+                    let txt = `<span class="status-timestamp">${dt.getHours()}:${dt.getMinutes()}</span>${entry.status}`;
+                    $('#status-message').html(txt);
+                }
+
+            }
+        });
+}
+
 ///
 /// Initialization
 ///
@@ -186,4 +210,7 @@ $(document).ready(function ()
     filters_form.find('select[name=show_watched]').on('change', videos_ReloadWithTimer);
     filters_form.find('select[name=show_downloaded]').on('change', videos_ReloadWithTimer);
     videos_Reload();
+
+    // Notification manager
+    setInterval(get_and_process_notifications, NOTIFICATION_INTERVAL);
 });
diff --git a/app/YtManagerApp/templates/YtManagerApp/master_default.html b/app/YtManagerApp/templates/YtManagerApp/master_default.html
index 594902c..264876b 100644
--- a/app/YtManagerApp/templates/YtManagerApp/master_default.html
+++ b/app/YtManagerApp/templates/YtManagerApp/master_default.html
@@ -67,7 +67,7 @@
         </div>
 
         <footer id="main_footer" class="footer bg-light">
-            <span class="ml-auto text-muted">Last synchronized: just now</span>
+            <span id="status-message" class="ml-auto text-muted">Last synchronized: just now</span>
             <button id="btn_sync_now" class="btn btn-sm btn-light" title="Synchronize now!">
                 <span class="typcn typcn-arrow-sync" aria-hidden="true"></span>
             </button>
diff --git a/app/YtManagerApp/urls.py b/app/YtManagerApp/urls.py
index aca2486..5010585 100644
--- a/app/YtManagerApp/urls.py
+++ b/app/YtManagerApp/urls.py
@@ -24,6 +24,7 @@ from .views.auth import ExtendedLoginView, RegisterView, RegisterDoneView
 from .views.index import index, ajax_get_tree, ajax_get_videos, CreateFolderModal, UpdateFolderModal, DeleteFolderModal, \
     CreateSubscriptionModal, UpdateSubscriptionModal, DeleteSubscriptionModal
 from .views.settings import SettingsView
+from .views.notifications import ajax_get_notifications
 
 urlpatterns = [
     # Authentication URLs
@@ -42,6 +43,8 @@ urlpatterns = [
     path('ajax/get_tree/', ajax_get_tree, name='ajax_get_tree'),
     path('ajax/get_videos/', ajax_get_videos, name='ajax_get_videos'),
 
+    path('ajax/get_notifications/<int:last_id>', ajax_get_notifications, name='ajax_get_notifications'),
+
     # Modals
     path('modal/create_folder/', CreateFolderModal.as_view(), name='modal_create_folder'),
     path('modal/create_folder/<int:parent_id>/', CreateFolderModal.as_view(), name='modal_create_folder'),
diff --git a/app/YtManagerApp/utils/algorithms.py b/app/YtManagerApp/utils/algorithms.py
new file mode 100644
index 0000000..6453cda
--- /dev/null
+++ b/app/YtManagerApp/utils/algorithms.py
@@ -0,0 +1,57 @@
+"""Bisection algorithms.
+
+These algorithms are taken from Python's standard library, and modified so they take a 'key' parameter (similar to how
+`sorted` works).
+"""
+
+
+def bisect_right(a, x, lo=0, hi=None, key=None):
+    """Return the index where to insert item x in list a, assuming a is sorted.
+
+    The return value i is such that all e in a[:i] have e <= x, and all e in
+    a[i:] have e > x.  So if x already appears in the list, a.insert(x) will
+    insert just after the rightmost x already there.
+
+    Optional args lo (default 0) and hi (default len(a)) bound the
+    slice of a to be searched.
+    """
+    if key is None:
+        key = lambda x: x
+
+    if lo < 0:
+        raise ValueError('lo must be non-negative')
+    if hi is None:
+        hi = len(a)
+    while lo < hi:
+        mid = (lo+hi)//2
+        if key(x) < key(a[mid]): hi = mid
+        else: lo = mid+1
+    return lo
+
+
+def bisect_left(a, x, lo=0, hi=None, key=None):
+    """Return the index where to insert item x in list a, assuming a is sorted.
+
+    The return value i is such that all e in a[:i] have e < x, and all e in
+    a[i:] have e >= x.  So if x already appears in the list, a.insert(x) will
+    insert just before the leftmost x already there.
+
+    Optional args lo (default 0) and hi (default len(a)) bound the
+    slice of a to be searched.
+    """
+    if key is None:
+        key = lambda x: x
+
+    if lo < 0:
+        raise ValueError('lo must be non-negative')
+    if hi is None:
+        hi = len(a)
+    while lo < hi:
+        mid = (lo+hi)//2
+        if key(a[mid]) < key(x): lo = mid+1
+        else: hi = mid
+    return lo
+
+
+# Create aliases
+bisect = bisect_right
diff --git a/app/YtManagerApp/views/index.py b/app/YtManagerApp/views/index.py
index 646bdf4..5377066 100644
--- a/app/YtManagerApp/views/index.py
+++ b/app/YtManagerApp/views/index.py
@@ -13,6 +13,7 @@ from YtManagerApp.management.videos import get_videos
 from YtManagerApp.models import Subscription, SubscriptionFolder, VIDEO_ORDER_CHOICES, VIDEO_ORDER_MAPPING
 from YtManagerApp.utils import youtube
 from YtManagerApp.views.controls.modal import ModalMixin
+from YtManagerApp.management.notification_manager import get_current_notification_id
 
 
 class VideoFilterForm(forms.Form):
@@ -94,7 +95,8 @@ def __tree_sub_id(sub_id):
 def index(request: HttpRequest):
     if request.user.is_authenticated:
         context = {
-            'filter_form': VideoFilterForm()
+            'filter_form': VideoFilterForm(),
+            'current_notification_id': get_current_notification_id(),
         }
         return render(request, 'YtManagerApp/index.html', context)
     else:
diff --git a/app/YtManagerApp/views/notifications.py b/app/YtManagerApp/views/notifications.py
new file mode 100644
index 0000000..e5435b5
--- /dev/null
+++ b/app/YtManagerApp/views/notifications.py
@@ -0,0 +1,12 @@
+from django.contrib.auth.decorators import login_required
+from django.http import HttpRequest, JsonResponse
+
+from YtManagerApp.management.notification_manager import get_notifications
+
+
+@login_required
+def ajax_get_notifications(request: HttpRequest, last_id: int):
+    user = request.user
+    notifications = get_notifications(user, last_id)
+    notifications = list(notifications)
+    return JsonResponse(notifications, safe=False)