From 57c2265f71add0cf29334ef9a5d7b3059c7f6b34 Mon Sep 17 00:00:00 2001 From: Tiberiu Chibici Date: Sun, 4 Nov 2018 23:32:18 +0200 Subject: [PATCH 1/3] Added a basic notification system. Long processes will call the notification API, and the notification events are registered. For now, notifications are pushed to the client by polling (client polls every second for new events). A basic status message is now displayed when the sync process starts and ends. --- .../management/jobs/synchronize.py | 7 ++ .../management/notification_manager.py | 93 +++++++++++++++++++ .../templates/YtManagerApp/index.html | 2 + .../templates/YtManagerApp/js/index.js | 27 ++++++ .../YtManagerApp/master_default.html | 2 +- app/YtManagerApp/urls.py | 3 + app/YtManagerApp/utils/algorithms.py | 57 ++++++++++++ app/YtManagerApp/views/index.py | 4 +- app/YtManagerApp/views/notifications.py | 12 +++ 9 files changed, 205 insertions(+), 2 deletions(-) create mode 100644 app/YtManagerApp/management/notification_manager.py create mode 100644 app/YtManagerApp/utils/algorithms.py create mode 100644 app/YtManagerApp/views/notifications.py diff --git a/app/YtManagerApp/management/jobs/synchronize.py b/app/YtManagerApp/management/jobs/synchronize.py index f6db0d8..5c6940a 100644 --- a/app/YtManagerApp/management/jobs/synchronize.py +++ b/app/YtManagerApp/management/jobs/synchronize.py @@ -10,6 +10,8 @@ from YtManagerApp.management.downloader import fetch_thumbnail, downloader_proce from YtManagerApp.models import * from YtManagerApp.utils import youtube +from YtManagerApp.management import notification_manager + log = logging.getLogger('sync') __lock = Lock() @@ -104,6 +106,7 @@ def synchronize(): try: log.info("Running scheduled synchronization... ") + notification_manager.notify_status_update(f'Synchronization started for all subscriptions.') # Sync subscribed playlists/channels log.info("Sync - checking videos") @@ -119,6 +122,7 @@ def synchronize(): __fetch_thumbnails() log.info("Synchronization finished.") + notification_manager.notify_status_update(f'Synchronization finished for all subscriptions.') finally: __lock.release() @@ -128,6 +132,8 @@ def synchronize_subscription(subscription: Subscription): __lock.acquire() try: log.info("Running synchronization for single subscription %d [%s]", subscription.id, subscription.name) + notification_manager.notify_status_update(f'Synchronization started for subscription {subscription.name}.') + yt_api = youtube.YoutubeAPI.build_public() log.info("Sync - checking videos") @@ -141,6 +147,7 @@ def synchronize_subscription(subscription: Subscription): __fetch_thumbnails() log.info("Synchronization finished for subscription %d [%s].", subscription.id, subscription.name) + notification_manager.notify_status_update(f'Synchronization finished for subscription {subscription.name}.') finally: __lock.release() diff --git a/app/YtManagerApp/management/notification_manager.py b/app/YtManagerApp/management/notification_manager.py new file mode 100644 index 0000000..c115d14 --- /dev/null +++ b/app/YtManagerApp/management/notification_manager.py @@ -0,0 +1,93 @@ +from django.contrib.auth.models import User +from typing import Dict, Deque, Any, Optional +from collections import deque +from datetime import datetime, timedelta +from YtManagerApp.utils.algorithms import bisect_left +from threading import Lock + +# Clients will request updates at most every few seconds, so a retention period of 60 seconds should be more than +# enough. I gave it 15 minutes so that if for some reason the connection fails (internet drops) and then comes back a +# few minutes later, the client will still get the updates +__RETENTION_PERIOD = 15 * 60 +__NOTIFICATIONS: Deque[Dict] = deque() +__NEXT_ID = 0 +__LOCK = Lock() + + +# Messages enum +class Messages: + STATUS_UPDATE = 'st-up' + STATUS_OPERATION_PROGRESS = 'st-op-prog' + STATUS_OPERATION_END = 'st-op-end' + + +def __add_notification(message, user: User=None, **kwargs): + global __NEXT_ID + + __LOCK.acquire() + + try: + # add notification + notification = { + 'time': datetime.now(), + 'msg': message, + 'id': __NEXT_ID, + 'uid': user and user.id, + } + notification.update(kwargs) + __NOTIFICATIONS.append(notification) + __NEXT_ID += 1 + + # trim old notifications + oldest = __NOTIFICATIONS[0] + while len(__NOTIFICATIONS) > 0 and oldest['time'] + timedelta(seconds=__RETENTION_PERIOD) < datetime.now(): + __NOTIFICATIONS.popleft() + oldest = __NOTIFICATIONS[0] + + finally: + __LOCK.release() + + +def get_notifications(user: User, last_received_id: Optional[int]): + + __LOCK.acquire() + + try: + first_index = 0 + if last_received_id is not None: + first_index = bisect_left(__NOTIFICATIONS, + {'id': last_received_id}, + key=lambda item: item['id']) + + for i in range(first_index, len(__NOTIFICATIONS)): + item = __NOTIFICATIONS[i] + if item['uid'] is None or item['uid'] == user.id: + yield item + + finally: + __LOCK.release() + + +def get_current_notification_id(): + return __NEXT_ID + + +def notify_status_update(status_message: str, user: User=None): + __add_notification(Messages.STATUS_UPDATE, + user=user, + status=status_message) + + +def notify_status_operation_progress(op_id: Any, status_message: str, progress_percent: float, user: User=None): + __add_notification(Messages.STATUS_OPERATION_PROGRESS, + user=user, + operation=op_id, + status=status_message, + progress=progress_percent) + + +def notify_status_operation_ended(op_id: Any, status_message: str, user: User=None): + __add_notification(Messages.STATUS_OPERATION_END, + user=user, + operation=op_id, + status=status_message) diff --git a/app/YtManagerApp/templates/YtManagerApp/index.html b/app/YtManagerApp/templates/YtManagerApp/index.html index cf158ee..0cf473d 100644 --- a/app/YtManagerApp/templates/YtManagerApp/index.html +++ b/app/YtManagerApp/templates/YtManagerApp/index.html @@ -9,6 +9,8 @@ {% block scripts %} {% endblock %} diff --git a/app/YtManagerApp/templates/YtManagerApp/js/index.js b/app/YtManagerApp/templates/YtManagerApp/js/index.js index 6fe9cfd..88c9ab3 100644 --- a/app/YtManagerApp/templates/YtManagerApp/js/index.js +++ b/app/YtManagerApp/templates/YtManagerApp/js/index.js @@ -157,6 +157,30 @@ function videos_Submit(e) e.preventDefault(); } +/// +/// Notifications +/// +const NOTIFICATION_INTERVAL = 1000; + +function get_and_process_notifications() +{ + $.get("{% url 'ajax_get_notifications' 12345 %}".replace("12345", LAST_NOTIFICATION_ID)) + .done(function(data) { + for (let entry of data) + { + LAST_NOTIFICATION_ID = entry.id; + let dt = new Date(entry.time); + + // Status update + if (entry.msg === 'st-up') { + let txt = `${dt.getHours()}:${dt.getMinutes()}${entry.status}`; + $('#status-message').html(txt); + } + + } + }); +} + /// /// Initialization /// @@ -186,4 +210,7 @@ $(document).ready(function () filters_form.find('select[name=show_watched]').on('change', videos_ReloadWithTimer); filters_form.find('select[name=show_downloaded]').on('change', videos_ReloadWithTimer); videos_Reload(); + + // Notification manager + setInterval(get_and_process_notifications, NOTIFICATION_INTERVAL); }); diff --git a/app/YtManagerApp/templates/YtManagerApp/master_default.html b/app/YtManagerApp/templates/YtManagerApp/master_default.html index 594902c..264876b 100644 --- a/app/YtManagerApp/templates/YtManagerApp/master_default.html +++ b/app/YtManagerApp/templates/YtManagerApp/master_default.html @@ -67,7 +67,7 @@