From 59a766b0fe9a41df6540c94fc93a0344f8da6b4f Mon Sep 17 00:00:00 2001
From: Tiberiu Chibici <chibicitiberiu@gmail.com>
Date: Sat, 3 Nov 2018 14:43:23 +0200
Subject: [PATCH] Implemented import from file functionality.

---
 .../controls/subscriptions_import_modal.html  |  22 ++++
 .../templates/YtManagerApp/index.html         |  20 +++-
 .../templates/YtManagerApp/js/index.js        |   8 ++
 app/YtManagerApp/urls.py                      |   4 +-
 .../utils/subscription_file_parser.py         | 104 ++++++++++++++++++
 app/YtManagerApp/views/index.py               | 104 +++++++++++++++++-
 6 files changed, 253 insertions(+), 9 deletions(-)
 create mode 100644 app/YtManagerApp/templates/YtManagerApp/controls/subscriptions_import_modal.html
 create mode 100644 app/YtManagerApp/utils/subscription_file_parser.py

diff --git a/app/YtManagerApp/templates/YtManagerApp/controls/subscriptions_import_modal.html b/app/YtManagerApp/templates/YtManagerApp/controls/subscriptions_import_modal.html
new file mode 100644
index 0000000..6707e47
--- /dev/null
+++ b/app/YtManagerApp/templates/YtManagerApp/controls/subscriptions_import_modal.html
@@ -0,0 +1,22 @@
+{% extends 'YtManagerApp/controls/modal.html' %}
+{% load crispy_forms_tags %}
+
+{% block modal_title %}
+    Import subscriptions
+{% endblock modal_title %}
+
+{% block modal_content %}
+    <form action="{% url 'modal_import_subscriptions' %}" method="post"
+          enctype="multipart/form-data">
+        {{ block.super }}
+    </form>
+{% endblock %}
+
+{% block modal_body %}
+    {% crispy form %}
+{% endblock modal_body %}
+
+{% block modal_footer %}
+    <input class="btn btn-primary" type="submit" value="Import">
+    <input class="btn btn-secondary" type="button" value="Cancel" data-dismiss="modal" aria-label="Cancel">
+{% endblock modal_footer %}
\ No newline at end of file
diff --git a/app/YtManagerApp/templates/YtManagerApp/index.html b/app/YtManagerApp/templates/YtManagerApp/index.html
index cf158ee..6ec5490 100644
--- a/app/YtManagerApp/templates/YtManagerApp/index.html
+++ b/app/YtManagerApp/templates/YtManagerApp/index.html
@@ -29,19 +29,27 @@
         <div class="col-3">
             {# Tree toolbar #}
             <div class="btn-toolbar" role="toolbar" aria-label="Subscriptions toolbar">
-                <div class="btn-group btn-group-sm mr-2" role="group">
-                    <button id="btn_create_sub" type="button" class="btn btn-secondary" >
+                <div class="btn-group mr-2" role="group">
+                    <button id="btn_create_sub" type="button" class="btn btn-light"
+                            data-toggle="tooltip" title="Add subscription...">
                         <span class="typcn typcn-plus" aria-hidden="true"></span>
                     </button>
-                    <button id="btn_create_folder" type="button" class="btn btn-secondary">
+                    <button id="btn_create_folder" type="button" class="btn btn-light"
+                            data-toggle="tooltip" title="Add folder...">
                         <span class="typcn typcn-folder-add" aria-hidden="true"></span>
                     </button>
+                    <button id="btn_import" type="button" class="btn btn-light"
+                            data-toggle="tooltip" title="Import from file...">
+                        <span class="typcn typcn-document-add" aria-hidden="true"></span>
+                    </button>
                 </div>
-                <div class="btn-group btn-group-sm mr-2" role="group">
-                    <button id="btn_edit_node" type="button" class="btn btn-secondary" >
+                <div class="btn-group mr-2" role="group">
+                    <button id="btn_edit_node" type="button" class="btn btn-light"
+                            data-toggle="tooltip" title="Edit selection">
                         <span class="typcn typcn-edit" aria-hidden="true"></span>
                     </button>
-                    <button id="btn_delete_node" type="button" class="btn btn-secondary" >
+                    <button id="btn_delete_node" type="button" class="btn btn-light"
+                            data-toggle="tooltip" title="Delete selection">
                         <span class="typcn typcn-trash" aria-hidden="true"></span>
                     </button>
                 </div>
diff --git a/app/YtManagerApp/templates/YtManagerApp/js/index.js b/app/YtManagerApp/templates/YtManagerApp/js/index.js
index 6fe9cfd..a4e4bad 100644
--- a/app/YtManagerApp/templates/YtManagerApp/js/index.js
+++ b/app/YtManagerApp/templates/YtManagerApp/js/index.js
@@ -162,6 +162,9 @@ function videos_Submit(e)
 ///
 $(document).ready(function ()
 {
+    // Initialize tooltips
+    $('[data-toggle="tooltip"]').tooltip();
+
     tree_Initialize();
 
     // Subscription toolbar
@@ -175,6 +178,11 @@ $(document).ready(function ()
         modal.setSubmitCallback(tree_Refresh);
         modal.loadAndShow();
     });
+    $("#btn_import").on("click", function () {
+        let modal = new AjaxModal("{% url 'modal_import_subscriptions' %}");
+        modal.setSubmitCallback(tree_Refresh);
+        modal.loadAndShow();
+    });
     $("#btn_edit_node").on("click", treeNode_Edit);
     $("#btn_delete_node").on("click", treeNode_Delete);
 
diff --git a/app/YtManagerApp/urls.py b/app/YtManagerApp/urls.py
index aca2486..a1c0aa3 100644
--- a/app/YtManagerApp/urls.py
+++ b/app/YtManagerApp/urls.py
@@ -22,7 +22,7 @@ from .views.actions import SyncNowView, DeleteVideoFilesView, DownloadVideoFiles
     MarkVideoUnwatchedView
 from .views.auth import ExtendedLoginView, RegisterView, RegisterDoneView
 from .views.index import index, ajax_get_tree, ajax_get_videos, CreateFolderModal, UpdateFolderModal, DeleteFolderModal, \
-    CreateSubscriptionModal, UpdateSubscriptionModal, DeleteSubscriptionModal
+    CreateSubscriptionModal, UpdateSubscriptionModal, DeleteSubscriptionModal, ImportSubscriptionsModal
 from .views.settings import SettingsView
 
 urlpatterns = [
@@ -50,6 +50,8 @@ urlpatterns = [
 
     path('modal/create_subscription/', CreateSubscriptionModal.as_view(), name='modal_create_subscription'),
     path('modal/create_subscription/<int:parent_folder_id>/', CreateSubscriptionModal.as_view(), name='modal_create_subscription'),
+    path('modal/import_subscriptions/', ImportSubscriptionsModal.as_view(), name='modal_import_subscriptions'),
+    path('modal/import_subscriptions/<int:parent_folder_id>/', ImportSubscriptionsModal.as_view(), name='modal_import_subscriptions'),
     path('modal/update_subscription/<int:pk>/', UpdateSubscriptionModal.as_view(), name='modal_update_subscription'),
     path('modal/delete_subscription/<int:pk>/', DeleteSubscriptionModal.as_view(), name='modal_delete_subscription'),
 
diff --git a/app/YtManagerApp/utils/subscription_file_parser.py b/app/YtManagerApp/utils/subscription_file_parser.py
new file mode 100644
index 0000000..6860b9e
--- /dev/null
+++ b/app/YtManagerApp/utils/subscription_file_parser.py
@@ -0,0 +1,104 @@
+from typing import Iterable
+from xml.etree import ElementTree
+import re
+
+
+class FormatNotSupportedError(Exception):
+    pass
+
+
+class SubFileParser(object):
+
+    def probe(self, file_handle) -> bool:
+        """
+        Tests if file matches file format.
+        :param file: File path
+        :return: True if file matches, false otherwise
+        """
+        return False
+
+    def parse(self, file_handle) -> Iterable[str]:
+        """
+        Parses file and returns a list of subscription URLs.
+        :param file:
+        :return:
+        """
+        return []
+
+
+class SubscriptionListFileParser(SubFileParser):
+    """
+    A subscription list file is file which contains just a bunch of URLs.
+    Comments are supported using # character.
+    """
+
+    def __is_url(self, text: str) -> bool:
+        return text.startswith('http://') or text.startswith('https://')
+
+    def probe(self, file_handle):
+        file_handle.seek(0)
+        for line in file_handle:
+            # Trim comments and spaces
+            line = re.sub('#.*', '', line).strip()
+            if len(line) > 0:
+                return self.__is_url(line)
+        return False
+
+    def parse(self, file_handle):
+        file_handle.seek(0)
+        for line in file_handle:
+            # Trim comments and spaces
+            line = re.sub('#.*', '', line).strip()
+            if len(line) > 0:
+                yield line
+
+
+class OPMLParser(SubFileParser):
+    """
+    Parses OPML files (emitted by YouTube)
+    """
+    def __init__(self):
+        self.__cached_file = None
+        self.__cached_tree: ElementTree.ElementTree = None
+
+    def __parse(self, file_handle):
+        if file_handle == self.__cached_file:
+            return self.__cached_tree
+
+        file_handle.seek(0)
+        tree = ElementTree.parse(file_handle)
+
+        self.__cached_file = file_handle
+        self.__cached_tree = tree
+        return self.__cached_tree
+
+    def probe(self, file_handle):
+        try:
+            tree = self.__parse(file_handle)
+        except ElementTree.ParseError:
+            # Malformed XML
+            return False
+
+        return tree.getroot().tag.lower() == 'opml'
+
+    def parse(self, file_handle):
+        tree = self.__parse(file_handle)
+        root = tree.getroot()
+
+        for node in root.iter('outline'):
+            if 'xmlUrl' in node.keys():
+                yield node.get('xmlUrl')
+
+
+PARSERS = (
+    OPMLParser(),
+    SubscriptionListFileParser()
+)
+
+
+def parse(file_handle) -> Iterable[str]:
+    for parser in PARSERS:
+        if parser.probe(file_handle):
+            return parser.parse(file_handle)
+
+    raise FormatNotSupportedError('This file cannot be parsed!')
diff --git a/app/YtManagerApp/views/index.py b/app/YtManagerApp/views/index.py
index 646bdf4..d18fb59 100644
--- a/app/YtManagerApp/views/index.py
+++ b/app/YtManagerApp/views/index.py
@@ -6,14 +6,16 @@ from django.contrib.auth.mixins import LoginRequiredMixin
 from django.db.models import Q
 from django.http import HttpRequest, HttpResponseBadRequest, JsonResponse
 from django.shortcuts import render
-from django.views.generic import CreateView, UpdateView, DeleteView
+from django.views.generic import CreateView, UpdateView, DeleteView, FormView
 from django.views.generic.edit import FormMixin
 
 from YtManagerApp.management.videos import get_videos
 from YtManagerApp.models import Subscription, SubscriptionFolder, VIDEO_ORDER_CHOICES, VIDEO_ORDER_MAPPING
-from YtManagerApp.utils import youtube
+from YtManagerApp.utils import youtube, subscription_file_parser
 from YtManagerApp.views.controls.modal import ModalMixin
 
+import logging
+
 
 class VideoFilterForm(forms.Form):
     CHOICES_SHOW_WATCHED = (
@@ -346,3 +348,101 @@ class DeleteSubscriptionModal(LoginRequiredMixin, ModalMixin, FormMixin, DeleteV
     def form_valid(self, form):
         self.object.delete_subscription(keep_downloaded_videos=form.cleaned_data['keep_downloaded_videos'])
         return super().form_valid(form)
+
+
+class ImportSubscriptionsForm(forms.Form):
+    TRUE_FALSE_CHOICES = (
+        (None, '(default)'),
+        (True, 'Yes'),
+        (False, 'No')
+    )
+
+    VIDEO_ORDER_CHOICES_WITH_EMPTY = (
+        ('', '(default)'),
+        *VIDEO_ORDER_CHOICES,
+    )
+
+    file = forms.FileField(label='File to import',
+                           help_text='Supported file types: OPML, subscription list')
+    parent_folder = forms.ModelChoiceField(SubscriptionFolder.objects, required=False)
+    auto_download = forms.ChoiceField(choices=TRUE_FALSE_CHOICES, required=False)
+    download_limit = forms.IntegerField(required=False)
+    download_order = forms.ChoiceField(choices=VIDEO_ORDER_CHOICES_WITH_EMPTY, required=False)
+    delete_after_watched = forms.ChoiceField(choices=TRUE_FALSE_CHOICES, required=False)
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.yt_api = youtube.YoutubeAPI.build_public()
+        self.helper = FormHelper()
+        self.helper.form_tag = False
+        self.helper.layout = Layout(
+            'file',
+            'parent_folder',
+            HTML('<hr>'),
+            HTML('<h5>Download configuration overloads</h5>'),
+            'auto_download',
+            'download_limit',
+            'download_order',
+            'delete_after_watched'
+        )
+
+    def __clean_empty_none(self, name: str):
+        data = self.cleaned_data[name]
+        if isinstance(data, str) and len(data) == 0:
+            return None
+        return data
+
+    def __clean_boolean(self, name: str):
+        data = self.cleaned_data[name]
+        if isinstance(data, str) and len(data) == 0:
+            return None
+        if isinstance(data, str):
+            return data == 'True'
+        return data
+
+    def clean_auto_download(self):
+        return self.__clean_boolean('auto_download')
+
+    def clean_delete_after_watched(self):
+        return self.__clean_boolean('delete_after_watched')
+
+    def clean_download_order(self):
+        return self.__clean_empty_none('download_order')
+
+
+class ImportSubscriptionsModal(LoginRequiredMixin, ModalMixin, FormView):
+    template_name = 'YtManagerApp/controls/subscriptions_import_modal.html'
+    form_class = ImportSubscriptionsForm
+
+    def form_valid(self, form):
+        file = form.cleaned_data['file']
+
+        # Parse file
+        try:
+            url_list = list(subscription_file_parser.parse(file))
+        except subscription_file_parser.FormatNotSupportedError:
+            return super().modal_response(form, success=False,
+                                          error_msg="The file could not be parsed! "
+                                                    "Possible problems: format not supported, file is malformed.")
+
+        print(form.cleaned_data)
+
+        # Create subscriptions
+        api = youtube.YoutubeAPI.build_public()
+        for url in url_list:
+            sub = Subscription()
+            sub.user = self.request.user
+            sub.parent_folder = form.cleaned_data['parent_folder']
+            sub.auto_download = form.cleaned_data['auto_download']
+            sub.download_limit = form.cleaned_data['download_limit']
+            sub.download_order = form.cleaned_data['download_order']
+            sub.delete_after_watched = form.cleaned_data['delete_after_watched']
+            try:
+                sub.fetch_from_url(url, api)
+            except Exception as e:
+                logging.error("Import subscription error - error processing URL %s: %s", url, e)
+                continue
+
+            sub.save()
+
+        return super().form_valid(form)