diff --git a/app/YtManagerApp/management/jobs/synchronize.py b/app/YtManagerApp/management/jobs/synchronize.py index 5d2be4d..303601e 100644 --- a/app/YtManagerApp/management/jobs/synchronize.py +++ b/app/YtManagerApp/management/jobs/synchronize.py @@ -4,19 +4,22 @@ from threading import Lock from apscheduler.triggers.cron import CronTrigger +from YtManagerApp.management.notification_manager import OPERATION_ID_SYNCHRONIZE from YtManagerApp.scheduler import scheduler from YtManagerApp.management.appconfig import appconfig from YtManagerApp.management.downloader import fetch_thumbnail, downloader_process_all, downloader_process_subscription from YtManagerApp.models import * from YtManagerApp.utils import youtube +from YtManagerApp.management import notification_manager + log = logging.getLogger('sync') __lock = Lock() _ENABLE_UPDATE_STATS = True -def __check_new_videos_sub(subscription: Subscription, yt_api: youtube.YoutubeAPI): +def __check_new_videos_sub(subscription: Subscription, yt_api: youtube.YoutubeAPI, progress_callback=None): # Get list of videos for item in yt_api.playlist_items(subscription.playlist_id): results = Video.objects.filter(video_id=item.resource_video_id, subscription=subscription) @@ -98,6 +101,10 @@ def __fetch_thumbnails(): __fetch_thumbnails_obj(Video.objects.filter(icon_best__istartswith='http'), 'video', 'video_id') +def __compute_progress(stage, stage_count, items, total_items): + stage_percent = float(stage) / stage_count + + def synchronize(): if not __lock.acquire(blocking=False): # Synchronize already running in another thread @@ -106,6 +113,12 @@ def synchronize(): try: log.info("Running scheduled synchronization... ") + notification_manager.notify_status_operation_progress( + OPERATION_ID_SYNCHRONIZE, + 'Running scheduled synchronization: checking for new videos...', + 0.1, + None + ) # Sync subscribed playlists/channels log.info("Sync - checking videos") @@ -114,13 +127,32 @@ def synchronize(): __check_new_videos_sub(subscription, yt_api) __detect_deleted(subscription) + notification_manager.notify_status_operation_progress( + OPERATION_ID_SYNCHRONIZE, + 'Running scheduled synchronization: enqueueing videos to download...', + 0.5, + None + ) + log.info("Sync - checking for videos to download") downloader_process_all() + notification_manager.notify_status_operation_progress( + OPERATION_ID_SYNCHRONIZE, + 'Running scheduled synchronization: fetching thumbnails...', + 0.7, + None + ) + log.info("Sync - fetching missing thumbnails") __fetch_thumbnails() log.info("Synchronization finished.") + notification_manager.notify_status_operation_ended( + OPERATION_ID_SYNCHRONIZE, + 'Synchronization finished.', + None + ) finally: __lock.release() @@ -130,6 +162,8 @@ def synchronize_subscription(subscription: Subscription): __lock.acquire() try: log.info("Running synchronization for single subscription %d [%s]", subscription.id, subscription.name) + notification_manager.notify_status_update(f'Synchronization started for subscription {subscription.name}.') + yt_api = youtube.YoutubeAPI.build_public() log.info("Sync - checking videos") @@ -143,6 +177,7 @@ def synchronize_subscription(subscription: Subscription): __fetch_thumbnails() log.info("Synchronization finished for subscription %d [%s].", subscription.id, subscription.name) + notification_manager.notify_status_update(f'Synchronization finished for subscription {subscription.name}.') finally: __lock.release() diff --git a/app/YtManagerApp/management/notification_manager.py b/app/YtManagerApp/management/notification_manager.py new file mode 100644 index 0000000..c95409c --- /dev/null +++ b/app/YtManagerApp/management/notification_manager.py @@ -0,0 +1,95 @@ +from django.contrib.auth.models import User +from typing import Dict, Deque, Any, Optional +from collections import deque +from datetime import datetime, timedelta +from YtManagerApp.utils.algorithms import bisect_left +from threading import Lock + +# Clients will request updates at most every few seconds, so a retention period of 60 seconds should be more than +# enough. I gave it 15 minutes so that if for some reason the connection fails (internet drops) and then comes back a +# few minutes later, the client will still get the updates +__RETENTION_PERIOD = 15 * 60 +__NOTIFICATIONS: Deque[Dict] = deque() +__NEXT_ID = 0 +__LOCK = Lock() + + +OPERATION_ID_SYNCHRONIZE = 1 + +# Messages enum +class Messages: + STATUS_UPDATE = 'st-up' + STATUS_OPERATION_PROGRESS = 'st-op-prog' + STATUS_OPERATION_END = 'st-op-end' + + +def __add_notification(message, user: User=None, **kwargs): + global __NEXT_ID + + __LOCK.acquire() + + try: + # add notification + notification = { + 'time': datetime.now(), + 'msg': message, + 'id': __NEXT_ID, + 'uid': user and user.id, + } + notification.update(kwargs) + __NOTIFICATIONS.append(notification) + __NEXT_ID += 1 + + # trim old notifications + oldest = __NOTIFICATIONS[0] + while len(__NOTIFICATIONS) > 0 and oldest['time'] + timedelta(seconds=__RETENTION_PERIOD) < datetime.now(): + __NOTIFICATIONS.popleft() + oldest = __NOTIFICATIONS[0] + + finally: + __LOCK.release() + + +def get_notifications(user: User, last_received_id: Optional[int]): + + __LOCK.acquire() + + try: + first_index = 0 + if last_received_id is not None: + first_index = bisect_left(__NOTIFICATIONS, + {'id': last_received_id}, + key=lambda item: item['id']) + + for i in range(first_index, len(__NOTIFICATIONS)): + item = __NOTIFICATIONS[i] + if item['uid'] is None or item['uid'] == user.id: + yield item + + finally: + __LOCK.release() + + +def get_current_notification_id(): + return __NEXT_ID + + +def notify_status_update(status_message: str, user: User=None): + __add_notification(Messages.STATUS_UPDATE, + user=user, + status=status_message) + + +def notify_status_operation_progress(op_id: Any, status_message: str, progress_percent: float, user: User=None): + __add_notification(Messages.STATUS_OPERATION_PROGRESS, + user=user, + operation=op_id, + status=status_message, + progress=progress_percent) + + +def notify_status_operation_ended(op_id: Any, status_message: str, user: User=None): + __add_notification(Messages.STATUS_OPERATION_END, + user=user, + operation=op_id, + status=status_message) diff --git a/app/YtManagerApp/static/YtManagerApp/css/style.css b/app/YtManagerApp/static/YtManagerApp/css/style.css index b18229c..0799bd1 100644 --- a/app/YtManagerApp/static/YtManagerApp/css/style.css +++ b/app/YtManagerApp/static/YtManagerApp/css/style.css @@ -126,4 +126,7 @@ padding: 0.15rem 0.4rem; font-size: 14pt; } +.status-timestamp { + margin-right: 0.25rem; } + /*# sourceMappingURL=style.css.map */ diff --git a/app/YtManagerApp/static/YtManagerApp/css/style.css.map b/app/YtManagerApp/static/YtManagerApp/css/style.css.map index 57051a8..74b7743 100644 --- a/app/YtManagerApp/static/YtManagerApp/css/style.css.map +++ b/app/YtManagerApp/static/YtManagerApp/css/style.css.map @@ -1,6 +1,6 @@ { "version": 3, -"mappings": "AAEA,UAAW;EACP,aAAa,EAAE,IAAI;EACnB,UAAU,EAAE,CAAC;;AAGjB,YAAa;EACT,QAAQ,EAAE,KAAK;EACf,IAAI,EAAE,CAAC;EACP,KAAK,EAAE,CAAC;EACR,MAAM,EAAE,CAAC;EACT,MAAM,EAAE,IAAI;EACZ,WAAW,EAAE,IAAI;EACjB,OAAO,EAAE,MAAM;EACf,OAAO,EAAE,IAAI;EACb,aAAa,EAAE,MAAM;EACrB,SAAS,EAAE,IAAI;;AAqBnB,uBAAuB;AACvB,kBAAmB;EAlBf,OAAO,EAAE,YAAY;EACrB,KAAK,EAAE,IAAa;EACpB,MAAM,EAAE,IAAa;EAErB,wBAAQ;IACJ,OAAO,EAAE,GAAG;IACZ,OAAO,EAAE,KAAK;IACd,KAAK,EAAE,IAAa;IACpB,MAAM,EAAE,IAAa;IACrB,MAAM,EAAE,GAAG;IACX,aAAa,EAAE,GAAG;IAClB,MAAM,EAAE,iBAAkC;IAC1C,YAAY,EAAE,uCAAmD;IACjE,SAAS,EAAE,sCAAsC;;AASzD,wBAAyB;EAtBrB,OAAO,EAAE,YAAY;EACrB,KAAK,EAAE,IAAa;EACpB,MAAM,EAAE,IAAa;EAErB,8BAAQ;IACJ,OAAO,EAAE,GAAG;IACZ,OAAO,EAAE,KAAK;IACd,KAAK,EAAE,IAAa;IACpB,MAAM,EAAE,IAAa;IACrB,MAAM,EAAE,GAAG;IACX,aAAa,EAAE,GAAG;IAClB,MAAM,EAAE,mBAAkC;IAC1C,YAAY,EAAE,uCAAmD;IACjE,SAAS,EAAE,sCAAsC;;AAazD,4BAOC;EANG,EAAG;IACC,SAAS,EAAE,YAAY;EAE3B,IAAK;IACD,SAAS,EAAE,cAAc;AAIjC,gCAAiC;EAC7B,QAAQ,EAAE,KAAK;EACf,GAAG,EAAE,GAAG;EACR,IAAI,EAAE,GAAG;EACT,UAAU,EAAE,KAAK;EACjB,WAAW,EAAE,KAAK;;AAGtB,cAAe;EACX,QAAQ,EAAE,KAAK;EAAE,oCAAoC;EACrD,OAAO,EAAE,IAAI;EAAE,uBAAuB;EACtC,KAAK,EAAE,IAAI;EAAE,uCAAuC;EACpD,MAAM,EAAE,IAAI;EAAE,wCAAwC;EACtD,GAAG,EAAE,CAAC;EACN,IAAI,EAAE,CAAC;EACP,KAAK,EAAE,CAAC;EACR,MAAM,EAAE,CAAC;EACT,gBAAgB,EAAE,kBAAe;EAAE,mCAAmC;EACtE,OAAO,EAAE,CAAC;EAAE,qFAAqF;EACjG,MAAM,EAAE,OAAO;EAAE,4BAA4B;;AAI7C,4BAAc;EACV,OAAO,EAAE,MAAM;EACf,aAAa,EAAE,KAAK;AAGpB,+BAAW;EACP,OAAO,EAAE,MAAM;AAEnB,+BAAW;EACP,SAAS,EAAE,IAAI;EACf,aAAa,EAAE,KAAK;AAExB,gCAAY;EACR,SAAS,EAAE,IAAI;EACf,aAAa,EAAE,KAAK;EAEpB,uCAAO;IACH,SAAS,EAAE,GAAG;AAGtB,iCAAa;EACT,OAAO,EAAE,YAAY;AAGzB,+BAAW;EACP,YAAY,EAAE,QAAQ;EACtB,qCAAQ;IACJ,eAAe,EAAE,IAAI;AAO7B,8BAAU;EACN,KAAK,EAAE,KAAK;AAKpB,8BAAgB;EACZ,KAAK,EAxHE,OAAO;AA0HlB,6BAAe;EACX,KAAK,EAAE,OAAO;;AAItB,WAAY;EACR,SAAS,EAAE,KAAK;EAChB,MAAM,EAAE,MAAM;;AAId,2BAAe;EACX,OAAO,EAAE,IAAI;;AAIrB,kBAAmB;EACf,MAAM,EAAE,QAAQ;EAChB,OAAO,EAAE,QAAQ;EAEjB,qBAAG;IACC,MAAM,EAAE,CAAC;;AAIjB,YAAa;EACT,OAAO,EAAE,YAAY;EACrB,aAAa,EAAE,MAAM;;AAGzB,YAAa;EACT,MAAM,EAAE,OAAO;EACf,iBAAK;IACD,OAAO,EAAE,cAAc;IACvB,SAAS,EAAE,IAAI", +"mappings": "AAEA,UAAW;EACP,aAAa,EAAE,IAAI;EACnB,UAAU,EAAE,CAAC;;AAGjB,YAAa;EACT,QAAQ,EAAE,KAAK;EACf,IAAI,EAAE,CAAC;EACP,KAAK,EAAE,CAAC;EACR,MAAM,EAAE,CAAC;EACT,MAAM,EAAE,IAAI;EACZ,WAAW,EAAE,IAAI;EACjB,OAAO,EAAE,MAAM;EACf,OAAO,EAAE,IAAI;EACb,aAAa,EAAE,MAAM;EACrB,SAAS,EAAE,IAAI;;AAqBnB,uBAAuB;AACvB,kBAAmB;EAlBf,OAAO,EAAE,YAAY;EACrB,KAAK,EAAE,IAAa;EACpB,MAAM,EAAE,IAAa;EAErB,wBAAQ;IACJ,OAAO,EAAE,GAAG;IACZ,OAAO,EAAE,KAAK;IACd,KAAK,EAAE,IAAa;IACpB,MAAM,EAAE,IAAa;IACrB,MAAM,EAAE,GAAG;IACX,aAAa,EAAE,GAAG;IAClB,MAAM,EAAE,iBAAkC;IAC1C,YAAY,EAAE,uCAAmD;IACjE,SAAS,EAAE,sCAAsC;;AASzD,wBAAyB;EAtBrB,OAAO,EAAE,YAAY;EACrB,KAAK,EAAE,IAAa;EACpB,MAAM,EAAE,IAAa;EAErB,8BAAQ;IACJ,OAAO,EAAE,GAAG;IACZ,OAAO,EAAE,KAAK;IACd,KAAK,EAAE,IAAa;IACpB,MAAM,EAAE,IAAa;IACrB,MAAM,EAAE,GAAG;IACX,aAAa,EAAE,GAAG;IAClB,MAAM,EAAE,mBAAkC;IAC1C,YAAY,EAAE,uCAAmD;IACjE,SAAS,EAAE,sCAAsC;;AAazD,4BAOC;EANG,EAAG;IACC,SAAS,EAAE,YAAY;EAE3B,IAAK;IACD,SAAS,EAAE,cAAc;AAIjC,gCAAiC;EAC7B,QAAQ,EAAE,KAAK;EACf,GAAG,EAAE,GAAG;EACR,IAAI,EAAE,GAAG;EACT,UAAU,EAAE,KAAK;EACjB,WAAW,EAAE,KAAK;;AAGtB,cAAe;EACX,QAAQ,EAAE,KAAK;EAAE,oCAAoC;EACrD,OAAO,EAAE,IAAI;EAAE,uBAAuB;EACtC,KAAK,EAAE,IAAI;EAAE,uCAAuC;EACpD,MAAM,EAAE,IAAI;EAAE,wCAAwC;EACtD,GAAG,EAAE,CAAC;EACN,IAAI,EAAE,CAAC;EACP,KAAK,EAAE,CAAC;EACR,MAAM,EAAE,CAAC;EACT,gBAAgB,EAAE,kBAAe;EAAE,mCAAmC;EACtE,OAAO,EAAE,CAAC;EAAE,qFAAqF;EACjG,MAAM,EAAE,OAAO;EAAE,4BAA4B;;AAI7C,4BAAc;EACV,OAAO,EAAE,MAAM;EACf,aAAa,EAAE,KAAK;AAGpB,+BAAW;EACP,OAAO,EAAE,MAAM;AAEnB,+BAAW;EACP,SAAS,EAAE,IAAI;EACf,aAAa,EAAE,KAAK;AAExB,gCAAY;EACR,SAAS,EAAE,IAAI;EACf,aAAa,EAAE,KAAK;EAEpB,uCAAO;IACH,SAAS,EAAE,GAAG;AAGtB,iCAAa;EACT,OAAO,EAAE,YAAY;AAGzB,+BAAW;EACP,YAAY,EAAE,QAAQ;EACtB,qCAAQ;IACJ,eAAe,EAAE,IAAI;AAO7B,8BAAU;EACN,KAAK,EAAE,KAAK;AAKpB,8BAAgB;EACZ,KAAK,EAxHE,OAAO;AA0HlB,6BAAe;EACX,KAAK,EAAE,OAAO;;AAItB,WAAY;EACR,SAAS,EAAE,KAAK;EAChB,MAAM,EAAE,MAAM;;AAId,2BAAe;EACX,OAAO,EAAE,IAAI;;AAIrB,kBAAmB;EACf,MAAM,EAAE,QAAQ;EAChB,OAAO,EAAE,QAAQ;EAEjB,qBAAG;IACC,MAAM,EAAE,CAAC;;AAIjB,YAAa;EACT,OAAO,EAAE,YAAY;EACrB,aAAa,EAAE,MAAM;;AAGzB,YAAa;EACT,MAAM,EAAE,OAAO;EACf,iBAAK;IACD,OAAO,EAAE,cAAc;IACvB,SAAS,EAAE,IAAI;;AAIvB,iBAAkB;EACd,YAAY,EAAE,OAAO", "sources": ["style.scss"], "names": [], "file": "style.css" diff --git a/app/YtManagerApp/static/YtManagerApp/css/style.scss b/app/YtManagerApp/static/YtManagerApp/css/style.scss index 83b0400..9c30bce 100644 --- a/app/YtManagerApp/static/YtManagerApp/css/style.scss +++ b/app/YtManagerApp/static/YtManagerApp/css/style.scss @@ -156,4 +156,8 @@ $accent-color: #007bff; padding: 0.15rem 0.4rem; font-size: 14pt; } +} + +.status-timestamp { + margin-right: 0.25rem; } \ No newline at end of file diff --git a/app/YtManagerApp/templates/YtManagerApp/index.html b/app/YtManagerApp/templates/YtManagerApp/index.html index 46ef6d9..08c2c1e 100644 --- a/app/YtManagerApp/templates/YtManagerApp/index.html +++ b/app/YtManagerApp/templates/YtManagerApp/index.html @@ -9,6 +9,8 @@ {% block scripts %} {% endblock %} diff --git a/app/YtManagerApp/templates/YtManagerApp/index_videos.html b/app/YtManagerApp/templates/YtManagerApp/index_videos.html index 16e1468..d885711 100644 --- a/app/YtManagerApp/templates/YtManagerApp/index_videos.html +++ b/app/YtManagerApp/templates/YtManagerApp/index_videos.html @@ -4,7 +4,7 @@
{% for video in videos %} -
+
Thumbnail diff --git a/app/YtManagerApp/templates/YtManagerApp/js/common.js b/app/YtManagerApp/templates/YtManagerApp/js/common.js index f737178..cfadd3d 100644 --- a/app/YtManagerApp/templates/YtManagerApp/js/common.js +++ b/app/YtManagerApp/templates/YtManagerApp/js/common.js @@ -1,3 +1,11 @@ +function zeroFill(number, width) { + width -= number.toString().length; + if ( width > 0 ) { + return new Array( width + (/\./.test( number ) ? 2 : 1) ).join( '0' ) + number; + } + return number + ""; // always return a string +} + class AjaxModal { constructor(url) diff --git a/app/YtManagerApp/templates/YtManagerApp/js/index.js b/app/YtManagerApp/templates/YtManagerApp/js/index.js index 1fb7438..ee0a5be 100644 --- a/app/YtManagerApp/templates/YtManagerApp/js/index.js +++ b/app/YtManagerApp/templates/YtManagerApp/js/index.js @@ -175,6 +175,70 @@ function videos_Submit(e) e.preventDefault(); } +/// +/// Notifications +/// +const NOTIFICATION_INTERVAL = 1000; +const STATUS_UPDATE = 'st-up'; +const STATUS_OPERATION_PROGRESS = 'st-op-prog'; +const STATUS_OPERATION_END = 'st-op-end'; +const OPERATION_LIST = {}; + +function notifications_update_progress_bar() { + + var count = 0; + var percent = 0; + + for(op in OPERATION_LIST) { + count++; + percent += OPERATION_LIST[op]; + } + + let progress = $('#status-progress'); + if (count > 0) { + progress.removeClass('invisible'); + let bar = progress.find('.progress-bar'); + bar.width(percent * 100 + '%'); + bar.text(count + ' operations in progress'); + } + else { + progress.addClass('invisible'); + } +} + +function get_and_process_notifications() +{ + $.get("{% url 'ajax_get_notifications' 12345 %}".replace("12345", LAST_NOTIFICATION_ID)) + .done(function(data) { + for (let entry of data) + { + LAST_NOTIFICATION_ID = entry.id; + let dt = new Date(entry.time); + + // Status update + if (entry.msg === STATUS_UPDATE) { + let txt = `${dt.getHours()}:${zeroFill(dt.getMinutes(), 2)}${entry.status}`; + $('#status-message').html(txt); + } + else if (entry.msg === STATUS_OPERATION_PROGRESS) { + let txt = `${dt.getHours()}:${zeroFill(dt.getMinutes(), 2)}${entry.status}`; + $('#status-message').html(txt); + + OPERATION_LIST[entry.operation] = entry.progress; + notifications_update_progress_bar(); + } + else if (entry.msg === STATUS_OPERATION_END) { + let txt = `${dt.getHours()}:${dt.getMinutes()}${entry.status}`; + $('#status-message').html(txt); + + delete OPERATION_LIST[entry.operation]; + notifications_update_progress_bar(); + } + + } + }); +} + /// /// Initialization /// @@ -214,4 +278,7 @@ $(document).ready(function () filters_form.find('select[name=results_per_page]').on('change', videos_ResetPageAndReloadWithTimer); videos_Reload(); + + // Notification manager + setInterval(get_and_process_notifications, NOTIFICATION_INTERVAL); }); diff --git a/app/YtManagerApp/templates/YtManagerApp/master_default.html b/app/YtManagerApp/templates/YtManagerApp/master_default.html index a36dceb..2a50390 100644 --- a/app/YtManagerApp/templates/YtManagerApp/master_default.html +++ b/app/YtManagerApp/templates/YtManagerApp/master_default.html @@ -65,11 +65,15 @@ {% endblock %}
-