Merge pull request #24 from chibicitiberiu/development

Import subscriptions from file
This commit is contained in:
chibicitiberiu 2018-11-03 23:35:08 +02:00 committed by GitHub
commit 0becefbacf
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
12 changed files with 316 additions and 11 deletions

View File

@ -119,4 +119,10 @@
display: inline-block;
margin-bottom: 0.5rem; }
.btn-toolbar {
margin: .5rem 0; }
.btn-toolbar .btn {
padding: 0.15rem 0.4rem;
font-size: 14pt; }
/*# sourceMappingURL=style.css.map */

View File

@ -1,6 +1,6 @@
{
"version": 3,
"mappings": "AAEA,UAAW;EACP,aAAa,EAAE,IAAI;;AAGvB,YAAa;EACT,QAAQ,EAAE,KAAK;EACf,IAAI,EAAE,CAAC;EACP,KAAK,EAAE,CAAC;EACR,MAAM,EAAE,CAAC;EACT,MAAM,EAAE,IAAI;EACZ,WAAW,EAAE,IAAI;EACjB,OAAO,EAAE,SAAS;EAClB,OAAO,EAAE,IAAI;EACb,aAAa,EAAE,MAAM;EACrB,SAAS,EAAE,IAAI;;AAqBnB,uBAAuB;AACvB,kBAAmB;EAlBf,OAAO,EAAE,YAAY;EACrB,KAAK,EAAE,IAAa;EACpB,MAAM,EAAE,IAAa;EAErB,wBAAQ;IACJ,OAAO,EAAE,GAAG;IACZ,OAAO,EAAE,KAAK;IACd,KAAK,EAAE,IAAa;IACpB,MAAM,EAAE,IAAa;IACrB,MAAM,EAAE,GAAG;IACX,aAAa,EAAE,GAAG;IAClB,MAAM,EAAE,iBAAkC;IAC1C,YAAY,EAAE,uCAAmD;IACjE,SAAS,EAAE,sCAAsC;;AASzD,wBAAyB;EAtBrB,OAAO,EAAE,YAAY;EACrB,KAAK,EAAE,IAAa;EACpB,MAAM,EAAE,IAAa;EAErB,8BAAQ;IACJ,OAAO,EAAE,GAAG;IACZ,OAAO,EAAE,KAAK;IACd,KAAK,EAAE,IAAa;IACpB,MAAM,EAAE,IAAa;IACrB,MAAM,EAAE,GAAG;IACX,aAAa,EAAE,GAAG;IAClB,MAAM,EAAE,mBAAkC;IAC1C,YAAY,EAAE,uCAAmD;IACjE,SAAS,EAAE,sCAAsC;;AAazD,4BAOC;EANG,EAAG;IACC,SAAS,EAAE,YAAY;EAE3B,IAAK;IACD,SAAS,EAAE,cAAc;AAIjC,gCAAiC;EAC7B,QAAQ,EAAE,KAAK;EACf,GAAG,EAAE,GAAG;EACR,IAAI,EAAE,GAAG;EACT,UAAU,EAAE,KAAK;EACjB,WAAW,EAAE,KAAK;;AAGtB,cAAe;EACX,QAAQ,EAAE,KAAK;EAAE,oCAAoC;EACrD,OAAO,EAAE,IAAI;EAAE,uBAAuB;EACtC,KAAK,EAAE,IAAI;EAAE,uCAAuC;EACpD,MAAM,EAAE,IAAI;EAAE,wCAAwC;EACtD,GAAG,EAAE,CAAC;EACN,IAAI,EAAE,CAAC;EACP,KAAK,EAAE,CAAC;EACR,MAAM,EAAE,CAAC;EACT,gBAAgB,EAAE,kBAAe;EAAE,mCAAmC;EACtE,OAAO,EAAE,CAAC;EAAE,qFAAqF;EACjG,MAAM,EAAE,OAAO;EAAE,4BAA4B;;AAI7C,4BAAc;EACV,OAAO,EAAE,MAAM;EACf,aAAa,EAAE,KAAK;AAGpB,+BAAW;EACP,OAAO,EAAE,MAAM;AAEnB,+BAAW;EACP,SAAS,EAAE,IAAI;EACf,aAAa,EAAE,KAAK;AAExB,gCAAY;EACR,SAAS,EAAE,IAAI;EACf,aAAa,EAAE,KAAK;EAEpB,uCAAO;IACH,SAAS,EAAE,GAAG;AAGtB,iCAAa;EACT,OAAO,EAAE,YAAY;AAGzB,+BAAW;EACP,YAAY,EAAE,QAAQ;EACtB,qCAAQ;IACJ,eAAe,EAAE,IAAI;AAO7B,8BAAU;EACN,KAAK,EAAE,KAAK;AAKpB,8BAAgB;EACZ,KAAK,EAvHE,OAAO;AAyHlB,6BAAe;EACX,KAAK,EAAE,OAAO;;AAItB,WAAY;EACR,SAAS,EAAE,KAAK;EAChB,MAAM,EAAE,MAAM;;AAId,2BAAe;EACX,OAAO,EAAE,IAAI;;AAIrB,kBAAmB;EACf,MAAM,EAAE,QAAQ;EAChB,OAAO,EAAE,QAAQ;EAEjB,qBAAG;IACC,MAAM,EAAE,CAAC;;AAIjB,YAAa;EACT,OAAO,EAAE,YAAY;EACrB,aAAa,EAAE,MAAM",
"mappings": "AAEA,UAAW;EACP,aAAa,EAAE,IAAI;;AAGvB,YAAa;EACT,QAAQ,EAAE,KAAK;EACf,IAAI,EAAE,CAAC;EACP,KAAK,EAAE,CAAC;EACR,MAAM,EAAE,CAAC;EACT,MAAM,EAAE,IAAI;EACZ,WAAW,EAAE,IAAI;EACjB,OAAO,EAAE,SAAS;EAClB,OAAO,EAAE,IAAI;EACb,aAAa,EAAE,MAAM;EACrB,SAAS,EAAE,IAAI;;AAqBnB,uBAAuB;AACvB,kBAAmB;EAlBf,OAAO,EAAE,YAAY;EACrB,KAAK,EAAE,IAAa;EACpB,MAAM,EAAE,IAAa;EAErB,wBAAQ;IACJ,OAAO,EAAE,GAAG;IACZ,OAAO,EAAE,KAAK;IACd,KAAK,EAAE,IAAa;IACpB,MAAM,EAAE,IAAa;IACrB,MAAM,EAAE,GAAG;IACX,aAAa,EAAE,GAAG;IAClB,MAAM,EAAE,iBAAkC;IAC1C,YAAY,EAAE,uCAAmD;IACjE,SAAS,EAAE,sCAAsC;;AASzD,wBAAyB;EAtBrB,OAAO,EAAE,YAAY;EACrB,KAAK,EAAE,IAAa;EACpB,MAAM,EAAE,IAAa;EAErB,8BAAQ;IACJ,OAAO,EAAE,GAAG;IACZ,OAAO,EAAE,KAAK;IACd,KAAK,EAAE,IAAa;IACpB,MAAM,EAAE,IAAa;IACrB,MAAM,EAAE,GAAG;IACX,aAAa,EAAE,GAAG;IAClB,MAAM,EAAE,mBAAkC;IAC1C,YAAY,EAAE,uCAAmD;IACjE,SAAS,EAAE,sCAAsC;;AAazD,4BAOC;EANG,EAAG;IACC,SAAS,EAAE,YAAY;EAE3B,IAAK;IACD,SAAS,EAAE,cAAc;AAIjC,gCAAiC;EAC7B,QAAQ,EAAE,KAAK;EACf,GAAG,EAAE,GAAG;EACR,IAAI,EAAE,GAAG;EACT,UAAU,EAAE,KAAK;EACjB,WAAW,EAAE,KAAK;;AAGtB,cAAe;EACX,QAAQ,EAAE,KAAK;EAAE,oCAAoC;EACrD,OAAO,EAAE,IAAI;EAAE,uBAAuB;EACtC,KAAK,EAAE,IAAI;EAAE,uCAAuC;EACpD,MAAM,EAAE,IAAI;EAAE,wCAAwC;EACtD,GAAG,EAAE,CAAC;EACN,IAAI,EAAE,CAAC;EACP,KAAK,EAAE,CAAC;EACR,MAAM,EAAE,CAAC;EACT,gBAAgB,EAAE,kBAAe;EAAE,mCAAmC;EACtE,OAAO,EAAE,CAAC;EAAE,qFAAqF;EACjG,MAAM,EAAE,OAAO;EAAE,4BAA4B;;AAI7C,4BAAc;EACV,OAAO,EAAE,MAAM;EACf,aAAa,EAAE,KAAK;AAGpB,+BAAW;EACP,OAAO,EAAE,MAAM;AAEnB,+BAAW;EACP,SAAS,EAAE,IAAI;EACf,aAAa,EAAE,KAAK;AAExB,gCAAY;EACR,SAAS,EAAE,IAAI;EACf,aAAa,EAAE,KAAK;EAEpB,uCAAO;IACH,SAAS,EAAE,GAAG;AAGtB,iCAAa;EACT,OAAO,EAAE,YAAY;AAGzB,+BAAW;EACP,YAAY,EAAE,QAAQ;EACtB,qCAAQ;IACJ,eAAe,EAAE,IAAI;AAO7B,8BAAU;EACN,KAAK,EAAE,KAAK;AAKpB,8BAAgB;EACZ,KAAK,EAvHE,OAAO;AAyHlB,6BAAe;EACX,KAAK,EAAE,OAAO;;AAItB,WAAY;EACR,SAAS,EAAE,KAAK;EAChB,MAAM,EAAE,MAAM;;AAId,2BAAe;EACX,OAAO,EAAE,IAAI;;AAIrB,kBAAmB;EACf,MAAM,EAAE,QAAQ;EAChB,OAAO,EAAE,QAAQ;EAEjB,qBAAG;IACC,MAAM,EAAE,CAAC;;AAIjB,YAAa;EACT,OAAO,EAAE,YAAY;EACrB,aAAa,EAAE,MAAM;;AAGzB,YAAa;EACT,MAAM,EAAE,OAAO;EACf,iBAAK;IACD,OAAO,EAAE,cAAc;IACvB,SAAS,EAAE,IAAI",
"sources": ["style.scss"],
"names": [],
"file": "style.css"

View File

@ -148,3 +148,11 @@ $accent-color: #007bff;
display: inline-block;
margin-bottom: 0.5rem;
}
.btn-toolbar {
margin: .5rem 0;
.btn {
padding: 0.15rem 0.4rem;
font-size: 14pt;
}
}

View File

@ -0,0 +1,22 @@
{% extends 'YtManagerApp/controls/modal.html' %}
{% load crispy_forms_tags %}
{% block modal_title %}
Import subscriptions
{% endblock modal_title %}
{% block modal_content %}
<form action="{% url 'modal_import_subscriptions' %}" method="post"
enctype="multipart/form-data">
{{ block.super }}
</form>
{% endblock %}
{% block modal_body %}
{% crispy form %}
{% endblock modal_body %}
{% block modal_footer %}
<input class="btn btn-primary" type="submit" value="Import">
<input class="btn btn-secondary" type="button" value="Cancel" data-dismiss="modal" aria-label="Cancel">
{% endblock modal_footer %}

View File

@ -29,19 +29,27 @@
<div class="col-3">
{# Tree toolbar #}
<div class="btn-toolbar" role="toolbar" aria-label="Subscriptions toolbar">
<div class="btn-group btn-group-sm mr-2" role="group">
<button id="btn_create_sub" type="button" class="btn btn-secondary" >
<div class="btn-group mr-2" role="group">
<button id="btn_create_sub" type="button" class="btn btn-light"
data-toggle="tooltip" title="Add subscription...">
<span class="typcn typcn-plus" aria-hidden="true"></span>
</button>
<button id="btn_create_folder" type="button" class="btn btn-secondary">
<button id="btn_create_folder" type="button" class="btn btn-light"
data-toggle="tooltip" title="Add folder...">
<span class="typcn typcn-folder-add" aria-hidden="true"></span>
</button>
<button id="btn_import" type="button" class="btn btn-light"
data-toggle="tooltip" title="Import from file...">
<span class="typcn typcn-document-add" aria-hidden="true"></span>
</button>
</div>
<div class="btn-group btn-group-sm mr-2" role="group">
<button id="btn_edit_node" type="button" class="btn btn-secondary" >
<div class="btn-group mr-2" role="group">
<button id="btn_edit_node" type="button" class="btn btn-light"
data-toggle="tooltip" title="Edit selection">
<span class="typcn typcn-edit" aria-hidden="true"></span>
</button>
<button id="btn_delete_node" type="button" class="btn btn-secondary" >
<button id="btn_delete_node" type="button" class="btn btn-light"
data-toggle="tooltip" title="Delete selection">
<span class="typcn typcn-trash" aria-hidden="true"></span>
</button>
</div>

View File

@ -53,7 +53,21 @@ class AjaxModal
_submit(e) {
let pThis = this;
let url = this.form.attr('action');
$.post(url, this.form.serialize())
let ajax_settings = {
url: url,
};
if (this.form.attr('enctype') === 'multipart/form-data') {
ajax_settings.data = new FormData(this.form[0]);
ajax_settings.contentType = false;
ajax_settings.processData = false;
ajax_settings.cache = false;
}
else {
ajax_settings.data = this.form.serialize();
}
$.post(ajax_settings)
.done(function(result) {
pThis._submitDone(result);
})

View File

@ -162,6 +162,9 @@ function videos_Submit(e)
///
$(document).ready(function ()
{
// Initialize tooltips
$('[data-toggle="tooltip"]').tooltip();
tree_Initialize();
// Subscription toolbar
@ -175,6 +178,11 @@ $(document).ready(function ()
modal.setSubmitCallback(tree_Refresh);
modal.loadAndShow();
});
$("#btn_import").on("click", function () {
let modal = new AjaxModal("{% url 'modal_import_subscriptions' %}");
modal.setSubmitCallback(tree_Refresh);
modal.loadAndShow();
});
$("#btn_edit_node").on("click", treeNode_Edit);
$("#btn_delete_node").on("click", treeNode_Delete);

View File

@ -22,7 +22,7 @@ from .views.actions import SyncNowView, DeleteVideoFilesView, DownloadVideoFiles
MarkVideoUnwatchedView
from .views.auth import ExtendedLoginView, RegisterView, RegisterDoneView
from .views.index import index, ajax_get_tree, ajax_get_videos, CreateFolderModal, UpdateFolderModal, DeleteFolderModal, \
CreateSubscriptionModal, UpdateSubscriptionModal, DeleteSubscriptionModal
CreateSubscriptionModal, UpdateSubscriptionModal, DeleteSubscriptionModal, ImportSubscriptionsModal
from .views.settings import SettingsView
urlpatterns = [
@ -50,6 +50,8 @@ urlpatterns = [
path('modal/create_subscription/', CreateSubscriptionModal.as_view(), name='modal_create_subscription'),
path('modal/create_subscription/<int:parent_folder_id>/', CreateSubscriptionModal.as_view(), name='modal_create_subscription'),
path('modal/import_subscriptions/', ImportSubscriptionsModal.as_view(), name='modal_import_subscriptions'),
path('modal/import_subscriptions/<int:parent_folder_id>/', ImportSubscriptionsModal.as_view(), name='modal_import_subscriptions'),
path('modal/update_subscription/<int:pk>/', UpdateSubscriptionModal.as_view(), name='modal_update_subscription'),
path('modal/delete_subscription/<int:pk>/', DeleteSubscriptionModal.as_view(), name='modal_delete_subscription'),

View File

@ -0,0 +1,108 @@
from typing import Iterable
from xml.etree import ElementTree
import re
class FormatNotSupportedError(Exception):
pass
class SubFileParser(object):
def probe(self, file_handle) -> bool:
"""
Tests if file matches file format.
:param file: File path
:return: True if file matches, false otherwise
"""
return False
def parse(self, file_handle) -> Iterable[str]:
"""
Parses file and returns a list of subscription URLs.
:param file:
:return:
"""
return []
class SubscriptionListFileParser(SubFileParser):
"""
A subscription list file is file which contains just a bunch of URLs.
Comments are supported using # character.
"""
def __is_url(self, text: str) -> bool:
return text.startswith('http://') or text.startswith('https://')
def probe(self, file_handle):
file_handle.seek(0)
for line in file_handle:
if isinstance(line, bytes) or isinstance(line, bytearray):
line = line.decode()
# Trim comments and spaces
line = re.sub('(^|\s)#.*', '', line).strip()
if len(line) > 0:
return self.__is_url(line)
return False
def parse(self, file_handle):
file_handle.seek(0)
for line in file_handle:
if isinstance(line, bytes) or isinstance(line, bytearray):
line = line.decode()
# Trim comments and spaces
line = re.sub('(^|\s)#.*', '', line).strip()
if len(line) > 0:
yield line
class OPMLParser(SubFileParser):
"""
Parses OPML files (emitted by YouTube)
"""
def __init__(self):
self.__cached_file = None
self.__cached_tree: ElementTree.ElementTree = None
def __parse(self, file_handle):
if file_handle == self.__cached_file:
return self.__cached_tree
file_handle.seek(0)
tree = ElementTree.parse(file_handle)
self.__cached_file = file_handle
self.__cached_tree = tree
return self.__cached_tree
def probe(self, file_handle):
try:
tree = self.__parse(file_handle)
except ElementTree.ParseError:
# Malformed XML
return False
return tree.getroot().tag.lower() == 'opml'
def parse(self, file_handle):
tree = self.__parse(file_handle)
root = tree.getroot()
for node in root.iter('outline'):
if 'xmlUrl' in node.keys():
yield node.get('xmlUrl')
PARSERS = (
OPMLParser(),
SubscriptionListFileParser()
)
def parse(file_handle) -> Iterable[str]:
for parser in PARSERS:
if parser.probe(file_handle):
return parser.parse(file_handle)
raise FormatNotSupportedError('This file cannot be parsed!')

View File

@ -6,14 +6,16 @@ from django.contrib.auth.mixins import LoginRequiredMixin
from django.db.models import Q
from django.http import HttpRequest, HttpResponseBadRequest, JsonResponse
from django.shortcuts import render
from django.views.generic import CreateView, UpdateView, DeleteView
from django.views.generic import CreateView, UpdateView, DeleteView, FormView
from django.views.generic.edit import FormMixin
from YtManagerApp.management.videos import get_videos
from YtManagerApp.models import Subscription, SubscriptionFolder, VIDEO_ORDER_CHOICES, VIDEO_ORDER_MAPPING
from YtManagerApp.utils import youtube
from YtManagerApp.utils import youtube, subscription_file_parser
from YtManagerApp.views.controls.modal import ModalMixin
import logging
class VideoFilterForm(forms.Form):
CHOICES_SHOW_WATCHED = (
@ -346,3 +348,101 @@ class DeleteSubscriptionModal(LoginRequiredMixin, ModalMixin, FormMixin, DeleteV
def form_valid(self, form):
self.object.delete_subscription(keep_downloaded_videos=form.cleaned_data['keep_downloaded_videos'])
return super().form_valid(form)
class ImportSubscriptionsForm(forms.Form):
TRUE_FALSE_CHOICES = (
(None, '(default)'),
(True, 'Yes'),
(False, 'No')
)
VIDEO_ORDER_CHOICES_WITH_EMPTY = (
('', '(default)'),
*VIDEO_ORDER_CHOICES,
)
file = forms.FileField(label='File to import',
help_text='Supported file types: OPML, subscription list')
parent_folder = forms.ModelChoiceField(SubscriptionFolder.objects, required=False)
auto_download = forms.ChoiceField(choices=TRUE_FALSE_CHOICES, required=False)
download_limit = forms.IntegerField(required=False)
download_order = forms.ChoiceField(choices=VIDEO_ORDER_CHOICES_WITH_EMPTY, required=False)
delete_after_watched = forms.ChoiceField(choices=TRUE_FALSE_CHOICES, required=False)
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.yt_api = youtube.YoutubeAPI.build_public()
self.helper = FormHelper()
self.helper.form_tag = False
self.helper.layout = Layout(
'file',
'parent_folder',
HTML('<hr>'),
HTML('<h5>Download configuration overloads</h5>'),
'auto_download',
'download_limit',
'download_order',
'delete_after_watched'
)
def __clean_empty_none(self, name: str):
data = self.cleaned_data[name]
if isinstance(data, str) and len(data) == 0:
return None
return data
def __clean_boolean(self, name: str):
data = self.cleaned_data[name]
if isinstance(data, str) and len(data) == 0:
return None
if isinstance(data, str):
return data == 'True'
return data
def clean_auto_download(self):
return self.__clean_boolean('auto_download')
def clean_delete_after_watched(self):
return self.__clean_boolean('delete_after_watched')
def clean_download_order(self):
return self.__clean_empty_none('download_order')
class ImportSubscriptionsModal(LoginRequiredMixin, ModalMixin, FormView):
template_name = 'YtManagerApp/controls/subscriptions_import_modal.html'
form_class = ImportSubscriptionsForm
def form_valid(self, form):
file = form.cleaned_data['file']
# Parse file
try:
url_list = list(subscription_file_parser.parse(file))
except subscription_file_parser.FormatNotSupportedError:
return super().modal_response(form, success=False,
error_msg="The file could not be parsed! "
"Possible problems: format not supported, file is malformed.")
print(form.cleaned_data)
# Create subscriptions
api = youtube.YoutubeAPI.build_public()
for url in url_list:
sub = Subscription()
sub.user = self.request.user
sub.parent_folder = form.cleaned_data['parent_folder']
sub.auto_download = form.cleaned_data['auto_download']
sub.download_limit = form.cleaned_data['download_limit']
sub.download_order = form.cleaned_data['download_order']
sub.delete_after_watched = form.cleaned_data['delete_after_watched']
try:
sub.fetch_from_url(url, api)
except Exception as e:
logging.error("Import subscription error - error processing URL %s: %s", url, e)
continue
sub.save()
return super().form_valid(form)

View File

@ -0,0 +1,19 @@
<?xml version="1.0" encoding="UTF-8"?>
<opml version="1.1">
<body>
<outline text="YouTube Subscriptions" title="YouTube Subscriptions">
<outline text="Doctor Mike" title="Doctor Mike" type="rss"
xmlUrl="https://www.youtube.com/feeds/videos.xml?channel_id=UC0QHWhjbe5fGJEPz3sVb6nw"/>
<outline text="Internet Historian" title="Internet Historian" type="rss"
xmlUrl="https://www.youtube.com/feeds/videos.xml?channel_id=UCR1D15p_vdP3HkrH8wgjQRw"/>
<outline text="GDC" title="GDC" type="rss"
xmlUrl="https://www.youtube.com/feeds/videos.xml?channel_id=UC0JB7TSe49lg56u6qH8y_MQ"/>
<outline text="Smarter Every Day 2" title="Smarter Every Day 2" type="rss"
xmlUrl="https://www.youtube.com/feeds/videos.xml?channel_id=UC8VkNBOwvsTlFjoSnNSMmxw"/>
<outline text="TotalBiscuit" title="TotalBiscuit" type="rss"
xmlUrl="https://www.youtube.com/feeds/videos.xml?channel_id=UCy1Ms_5qBTawC-k7PVjHXKQ"/>
<outline text="LastWeekTonight" title="LastWeekTonight" type="rss"
xmlUrl="https://www.youtube.com/feeds/videos.xml?channel_id=UC3XTzVzaHQEd30rQbuvCtTQ"/>
</outline>
</body>
</opml>

View File

@ -0,0 +1,10 @@
# This is a comment, it shold be ignored
# ##Blank lines as well
https://www.youtube.com/channel/UCMtFAi84ehTSYSE9XoHefig
https://www.youtube.com/user/adric22
https://www.youtube.com/watch?v=IuLxX07isNg&list=PLfABUWdDse7antKQRPnYLNJ6tv_hMYwIs #Comment after URL