diff --git a/.gitignore b/.gitignore index 6fb4095..3fe5d5e 100644 --- a/.gitignore +++ b/.gitignore @@ -115,10 +115,16 @@ venv.bak/ .dmypy.json dmypy.json -media/ +data/ .vscode/* !.vscode/settings.json !.vscode/tasks.json !.vscode/launch.json !.vscode/extensions.json + +.idea + +# Dolphin generated file +.directory + diff --git a/.idea/codeStyles/codeStyleConfig.xml b/.idea/codeStyles/codeStyleConfig.xml deleted file mode 100644 index a55e7a1..0000000 --- a/.idea/codeStyles/codeStyleConfig.xml +++ /dev/null @@ -1,5 +0,0 @@ - - - - \ No newline at end of file diff --git a/.idea/dataSources.local.xml b/.idea/dataSources.local.xml deleted file mode 100644 index 9148e42..0000000 --- a/.idea/dataSources.local.xml +++ /dev/null @@ -1,21 +0,0 @@ - - - - - - " - - - false - *:@ - - - - " - - - false - *:@ - - - \ No newline at end of file diff --git a/.idea/dataSources.xml b/.idea/dataSources.xml deleted file mode 100644 index 3b5db5f..0000000 --- a/.idea/dataSources.xml +++ /dev/null @@ -1,33 +0,0 @@ - - - - - sqlite.xerial - true - true - $PROJECT_DIR$/YtManager/settings.py - org.sqlite.JDBC - jdbc:sqlite:D:\Dev\youtube-channel-manager\db.sqlite3 - - - - - - sqlite.xerial - true - org.sqlite.JDBC - jdbc:sqlite:$PROJECT_DIR$/db.sqlite3 - - - - - - file://$APPLICATION_CONFIG_DIR$/jdbc-drivers/Xerial SQLiteJDBC/3.25.1/sqlite-jdbc-3.25.1.jar - - - file://$APPLICATION_CONFIG_DIR$/jdbc-drivers/Xerial SQLiteJDBC/3.25.1/license.txt - - - - - \ No newline at end of file diff --git a/.idea/dataSources/2dac2136-d902-4d27-8789-9371934602fd.xml b/.idea/dataSources/2dac2136-d902-4d27-8789-9371934602fd.xml deleted file mode 100644 index fa380df..0000000 --- a/.idea/dataSources/2dac2136-d902-4d27-8789-9371934602fd.xml +++ /dev/null @@ -1,38 +0,0 @@ - - - - - 3.25.1 - - - 1 - 1 - - - - - - 1 -
- - 1 - text|0s - - - 2 - text|0s - - - 3 - text|0s - - - 4 - int|0s - - - 5 - text|0s - - - \ No newline at end of file diff --git a/.idea/dataSources/2dac2136-d902-4d27-8789-9371934602fd/storage_v2/_src_/schema/main.uQUzAA.meta b/.idea/dataSources/2dac2136-d902-4d27-8789-9371934602fd/storage_v2/_src_/schema/main.uQUzAA.meta deleted file mode 100644 index 8dab49c..0000000 --- a/.idea/dataSources/2dac2136-d902-4d27-8789-9371934602fd/storage_v2/_src_/schema/main.uQUzAA.meta +++ /dev/null @@ -1,2 +0,0 @@ -#n:main -! [0, 0, null, null, -2147483648, -2147483648] diff --git a/.idea/dataSources/77df9da5-0b97-445e-a895-744ef8257a74.xml b/.idea/dataSources/77df9da5-0b97-445e-a895-744ef8257a74.xml deleted file mode 100644 index 2a30194..0000000 --- a/.idea/dataSources/77df9da5-0b97-445e-a895-744ef8257a74.xml +++ /dev/null @@ -1,883 +0,0 @@ - - - - - 3.25.1 - - - 1 - 1 - - - - - -
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- 1 -
- - 1 -
- - 1 - integer|0s - 1 - 1 - - - 2 - text|0s - 1 - - - 3 - text|0s - - - 4 - text|0s - - - 5 - text|0s - 1 - - - 6 - text|0s - 1 - - - 7 - text|0s - 1 - - - 8 - text|0s - 1 - - - 9 - text|0s - 1 - - - 1 - channel_id - - 1 - - - 1 - username - - 1 - - - 1 - custom_url - - 1 - - - id - 1 - - - channel_id - sqlite_autoindex_YtManagerApp_channel_1 - - - username - sqlite_autoindex_YtManagerApp_channel_2 - - - custom_url - sqlite_autoindex_YtManagerApp_channel_3 - - - 1 - integer|0s - 1 - 1 - - - 2 - varchar(1024)|0s - 1 - - - 3 - varchar(128)|0s - 1 - - - 4 - text|0s - 1 - - - 5 - varchar(1024)|0s - 1 - - - 6 - varchar(1024)|0s - 1 - - - 7 - bool|0s - - - 8 - integer|0s - - - 9 - varchar(128)|0s - - - 10 - integer|0s - 1 - - - 11 - integer|0s - - - 12 - integer|0s - 1 - - - 13 - bool|0s - - - channel_id - - - - parent_folder_id - - - - user_id - - - - id - 1 - - - channel_id - YtManagerApp_channel - id - 1 - 1 - - - parent_folder_id - YtManagerApp_subscriptionfolder - id - 1 - 1 - - - user_id - auth_user - id - 1 - 1 - - - 1 - integer|0s - 1 - 1 - - - 2 - integer|0s - 1 - - - 3 - varchar(250)|0s - 1 - - - 4 - integer|0s - - - user_id - - - - parent_id - - - - id - 1 - - - user_id - auth_user - id - 1 - 1 - - - parent_id - YtManagerApp_subscriptionfolder - id - 1 - 1 - - - 1 - integer|0s - 1 - 1 - - - 2 - bool|0s - - - 3 - bool|0s - - - 4 - integer|0s - - - 5 - integer|0s - - - 6 - varchar(100)|0s - - - 7 - varchar(1024)|0s - - - 8 - varchar(1024)|0s - - - 9 - varchar(256)|0s - - - 10 - bool|0s - - - 11 - bool|0s - - - 12 - bool|0s - - - 13 - varchar(250)|0s - - - 14 - varchar(100)|0s - - - 15 - integer|0s - 1 - - - 16 - bool|0s - - - 1 - user_id - - 1 - - - id - 1 - - - user_id - sqlite_autoindex_YtManagerApp_usersettings_1 - - - user_id - auth_user - id - 1 - 1 - - - 1 - integer|0s - 1 - 1 - - - 2 - text|0s - 1 - - - 3 - text|0s - 1 - - - 4 - text|0s - 1 - - - 5 - bool|0s - 1 - - - 6 - text|0s - - - 7 - integer|0s - 1 - - - 8 - datetime|0s - 1 - - - 9 - text|0s - 1 - - - 10 - text|0s - 1 - - - 11 - integer|0s - 1 - - - 12 - real|0s - 1 - - - 13 - text|0s - 1 - - - 14 - integer|0s - 1 - - - subscription_id - - - - id - 1 - - - subscription_id - YtManagerApp_subscription - id - 1 - 1 - - - 1 - integer|0s - 1 - 1 - - - 2 - varchar(80)|0s - 1 - - - 1 - name - - 1 - - - id - 1 - - - name - sqlite_autoindex_auth_group_1 - - - 1 - integer|0s - 1 - 1 - - - 2 - integer|0s - 1 - - - 3 - integer|0s - 1 - - - group_id -permission_id - - 1 - - - group_id - - - - permission_id - - - - id - 1 - - - group_id - auth_group - id - 1 - 1 - - - permission_id - auth_permission - id - 1 - 1 - - - 1 - integer|0s - 1 - 1 - - - 2 - integer|0s - 1 - - - 3 - varchar(100)|0s - 1 - - - 4 - varchar(255)|0s - 1 - - - content_type_id -codename - - 1 - - - content_type_id - - - - id - 1 - - - content_type_id - django_content_type - id - 1 - 1 - - - 1 - integer|0s - 1 - 1 - - - 2 - varchar(128)|0s - 1 - - - 3 - datetime|0s - - - 4 - bool|0s - 1 - - - 5 - varchar(150)|0s - 1 - - - 6 - varchar(30)|0s - 1 - - - 7 - varchar(254)|0s - 1 - - - 8 - bool|0s - 1 - - - 9 - bool|0s - 1 - - - 10 - datetime|0s - 1 - - - 11 - varchar(150)|0s - 1 - - - 1 - username - - 1 - - - id - 1 - - - username - sqlite_autoindex_auth_user_1 - - - 1 - integer|0s - 1 - 1 - - - 2 - integer|0s - 1 - - - 3 - integer|0s - 1 - - - user_id -group_id - - 1 - - - user_id - - - - group_id - - - - id - 1 - - - user_id - auth_user - id - 1 - 1 - - - group_id - auth_group - id - 1 - 1 - - - 1 - integer|0s - 1 - 1 - - - 2 - integer|0s - 1 - - - 3 - integer|0s - 1 - - - user_id -permission_id - - 1 - - - user_id - - - - permission_id - - - - id - 1 - - - user_id - auth_user - id - 1 - 1 - - - permission_id - auth_permission - id - 1 - 1 - - - 1 - integer|0s - 1 - 1 - - - 2 - datetime|0s - 1 - - - 3 - text|0s - - - 4 - varchar(200)|0s - 1 - - - 5 - text|0s - 1 - - - 6 - integer|0s - - - 7 - integer|0s - 1 - - - 8 - smallint unsigned|0s - 1 - - - content_type_id - - - - user_id - - - - id - 1 - - - content_type_id - django_content_type - id - 1 - 1 - - - user_id - auth_user - id - 1 - 1 - - - 1 - integer|0s - 1 - 1 - - - 2 - varchar(100)|0s - 1 - - - 3 - varchar(100)|0s - 1 - - - app_label -model - - 1 - - - id - 1 - - - 1 - integer|0s - 1 - 1 - - - 2 - varchar(255)|0s - 1 - - - 3 - varchar(255)|0s - 1 - - - 4 - datetime|0s - 1 - - - id - 1 - - - 1 - varchar(40)|0s - 1 - - - 2 - text|0s - 1 - - - 3 - datetime|0s - 1 - - - 1 - session_key - - 1 - - - expire_date - - - - session_key - 1 - sqlite_autoindex_django_session_1 - - - 1 - text|0s - - - 2 - text|0s - - - 3 - text|0s - - - 4 - int|0s - - - 5 - text|0s - - - 1 - - - 2 - - - \ No newline at end of file diff --git a/.idea/dataSources/77df9da5-0b97-445e-a895-744ef8257a74/storage_v2/_src_/schema/main.uQUzAA.meta b/.idea/dataSources/77df9da5-0b97-445e-a895-744ef8257a74/storage_v2/_src_/schema/main.uQUzAA.meta deleted file mode 100644 index 8dab49c..0000000 --- a/.idea/dataSources/77df9da5-0b97-445e-a895-744ef8257a74/storage_v2/_src_/schema/main.uQUzAA.meta +++ /dev/null @@ -1,2 +0,0 @@ -#n:main -! [0, 0, null, null, -2147483648, -2147483648] diff --git a/.idea/jsLibraryMappings.xml b/.idea/jsLibraryMappings.xml deleted file mode 100644 index 47c40de..0000000 --- a/.idea/jsLibraryMappings.xml +++ /dev/null @@ -1,6 +0,0 @@ - - - - - - \ No newline at end of file diff --git a/.idea/misc.xml b/.idea/misc.xml deleted file mode 100644 index 3999087..0000000 --- a/.idea/misc.xml +++ /dev/null @@ -1,7 +0,0 @@ - - - - - - \ No newline at end of file diff --git a/.idea/modules.xml b/.idea/modules.xml deleted file mode 100644 index b4d776c..0000000 --- a/.idea/modules.xml +++ /dev/null @@ -1,8 +0,0 @@ - - - - - - - - \ No newline at end of file diff --git a/.idea/vcs.xml b/.idea/vcs.xml deleted file mode 100644 index 94a25f7..0000000 --- a/.idea/vcs.xml +++ /dev/null @@ -1,6 +0,0 @@ - - - - - - \ No newline at end of file diff --git a/.idea/watcherTasks.xml b/.idea/watcherTasks.xml deleted file mode 100644 index 1f0a12e..0000000 --- a/.idea/watcherTasks.xml +++ /dev/null @@ -1,51 +0,0 @@ - - - - - - - - - - - \ No newline at end of file diff --git a/.idea/workspace.xml b/.idea/workspace.xml deleted file mode 100644 index 806b09b..0000000 --- a/.idea/workspace.xml +++ /dev/null @@ -1,1033 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - 1538745535248 - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/.idea/youtube-channel-manager.iml b/.idea/youtube-channel-manager.iml deleted file mode 100644 index 6831e78..0000000 --- a/.idea/youtube-channel-manager.iml +++ /dev/null @@ -1,38 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 4f10986..e4630bd 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,27 +1,23 @@ FROM python:3 -WORKDIR /usr/src/app +WORKDIR /usr/src/ytsm/app +# ffmpeg is needed for youtube-dl RUN apt-get update RUN apt-get install ffmpeg -y -COPY ./app/requirements.txt ./ +COPY ./requirements.txt ./ RUN pip install --no-cache-dir -r requirements.txt -ENV YTSM_DATABASE_ENGINE='django.db.backends.sqlite3' -ENV YTSM_DATABASE_NAME='/usr/src/app/data/db/ytmanager.db' -ENV YTSM_DATABASE_HOST='' -ENV YTSM_DATABASE_USERNAME='' -ENV YTSM_DATABASE_PASSWORD='' -ENV YTSM_DATABASE_PORT='' -ENV YTSM_YOUTUBE_API_KEY='AIzaSyBabzE4Bup77WexdLMa9rN9z-wJidEfNX8' +ENV YTSM_DEBUG='False' +ENV YTSM_DATA_PATH='/usr/src/ytsm/data' -VOLUME /usr/src/app/data/media -VOLUME /usr/src/app/data/db +VOLUME /usr/src/ytsm/config +VOLUME /usr/src/ytsm/data -COPY ./app/ . -COPY ./config/ ./config/ +COPY ./app/ ./ +COPY ./docker/init.sh ./ EXPOSE 8000 -CMD ["/bin/bash", "init.sh"] \ No newline at end of file +CMD ["/bin/bash", "init.sh"] diff --git a/README.md b/README.md index 028173a..b1469db 100644 --- a/README.md +++ b/README.md @@ -24,8 +24,10 @@ Of course, there are a lot of things that still need to be done. The web interfa * python3: `$ apt install python3` * pip: `$ apt install python3-pip` +* ffmpeg: `$ apt install ffmpeg` * django: `$ pip3 install django` * crispy_forms: `$ pip3 install django-crispy-forms` +* dj-config-url: `$ pip3 install dj-config-url` * youtube-dl: `$ pip3 install youtube-dl` * google-api-python-client: `$ pip3 install google-api-python-client` * google_auth_oauthlib: `$ pip3 install google_auth_oauthlib` @@ -34,65 +36,104 @@ Of course, there are a lot of things that still need to be done. The web interfa ## Installation +There are 2 ways you can install this server. Using docker is the quickest and easiest method. + ### Normal installation for development/testing -1. Install all the dependencies listed above. - - ```bash - sudo apt install python3 python3-pip - sudo pip3 install apscheduler django django-crispy-forms youtube-dl google-api-python-client google_auth_oauthlib oauth2client - ``` - -2. Clone this repository: +1. Clone this repository: ```bash git clone https://github.com/chibicitiberiu/ytsm.git cd ytsm ``` -3. Set up the database: `python3 manage.py migrate` - - By default, a SQLite database is used, which is located in the project's folder. - You can customize that in `YtManager/settings.py`, by modifying the `DATABASES` variable (search Django documentation for details). - -4. Set up the `MEDIA_ROOT` variable in `YtManager/settings.py`. This is where the thumbnails will be downloaded. -(note: this will be moved to `config.ini` in the future). +2. Install all the dependencies listed above. -5. Obtain an YouTube API developer key from [https://console.developers.google.com/apis/dashboard](https://console.developers.google.com/apis/dashboard). + ```bash + sudo apt install python3 python3-pip ffmpeg + sudo pip3 install --no-cache-dir -r requirements.txt + ``` + +3. Modify `config/config.ini` to your liking. All the settings should be documented through comments. +All these settings apply server-wide. The settings in the `user` section can be overriden from the web page for each +individual user. + +4. Obtain an YouTube API developer key from [https://console.developers.google.com/apis/dashboard](https://console.developers.google.com/apis/dashboard). You can find a detailed guide on [this page](https://www.slickremix.com/docs/get-api-key-for-youtube/). The `defaults.ini` file already has an API key, but if the quotas are reached, you won't be able to use this program any more. Also, I might decide to delete that key, which will break your installation. + + After obtaining the key, set it in `config.ini`. -6. Modify `config/config.ini` to your liking. All the settings should be documented through comments. -All these settings apply server-wide. The settings in the `user` section can be overriden from the web page for each -individual user. +5. Set up the database: - The most important settings are: - - * `[Global] YoutubeApiKey` - put your YouTube API key here - * `[User] DownloadPath` - sets the folder where videos will be downloaded - -7. Start the server: `python3 manage.py runserver [port] --noreload` + ```bash + cd app + python3 manage.py migrate + ``` + + By default, a SQLite database is used, which is located in the project's folder. The database can be configured + in `settings.ini`. + +6. Start the server: `python3 manage.py runserver [port] --noreload --insecure` The `port` parameter is optional. The `--noreload` option is necessary, otherwise the scheduler will run on 2 separate processes at the same time, - which is not ideal. + which is not ideal. + + The `--insecure` option is required only if `Debug=False` in `config.ini`, Without this option, the static resources + (CSS, javascript) won't work. -8. Open the server's page in your browser, by entering `http://localhost:port` in your address bar. +7. Open the server's page in your browser, by entering `http://localhost:port` in your address bar. -9. Create an admin user by going to the *register* page, and creating an user account. +8. Create an admin user by going to the *register* page, and creating an user account. -10. Add some subscriptions, and enjoy! +9. Add some subscriptions, and enjoy! ### Docker -A much easier way to install is to use Docker. +1. Clone this repository: -To run with docker, edit the config file (config/config.ini) and then run `docker-compose up -d`, it will bind to port 80. + ```bash + git clone https://github.com/chibicitiberiu/ytsm.git + cd ytsm + ``` -You can edit the default download locations in the docker-compose.yml file. +2. Install docker (if not installed) + +3. Modify `config/config.ini` to your liking. All the settings should be documented through comments. +All these settings apply server-wide. The settings in the `user` section can be overriden from the web page for each +individual user. + + **Attention**: you cannot modify the download location from `settings.ini` when using docker. + To do so, you will need to modify the volume mapping in `docker-compose.yml`. + +4. Obtain an YouTube API developer key from [https://console.developers.google.com/apis/dashboard](https://console.developers.google.com/apis/dashboard). +You can find a detailed guide on [this page](https://www.slickremix.com/docs/get-api-key-for-youtube/). + + The `defaults.ini` file already has an API key, but if the quotas are reached, you won't be able to use this program + any more. Also, I might decide to delete that key, which will break your installation. + + After obtaining the key, set it in `config.ini`. + +5. Build and run docker compose image: + + ```bash + docker-compose up -d + ``` + +6. Open the server's page in your browser, by entering `http://localhost` in your address bar. + +7. Create an admin user by going to the *register* page, and creating an user account. + +8. Add some subscriptions, and enjoy! + +The docker image uses a sqlite database, and stores the data in a folder `data/` located in the project directory. +You can edit the default download locations in the `docker-compose.yml` file. + +For more information about using Docker, check [this page](Docker_README.md). ### Deploying for production diff --git a/app/YtManager/settings.py b/app/YtManager/settings.py index d04936d..637a233 100644 --- a/app/YtManager/settings.py +++ b/app/YtManager/settings.py @@ -11,23 +11,13 @@ https://docs.djangoproject.com/en/1.11/ref/settings/ """ import os +import logging +from os.path import dirname as up -# Build paths inside the project like this: os.path.join(BASE_DIR, ...) -BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) - -# Quick-start development settings - unsuitable for production -# See https://docs.djangoproject.com/en/1.11/howto/deployment/checklist/ - -# SECURITY WARNING: keep the secret key used in production secret! -SECRET_KEY = '^zv8@i2h!ko2lo=%ivq(9e#x=%q*i^^)6#4@(juzdx%&0c+9a0' - -YOUTUBE_API_KEY = os.getenv('YTSM_YOUTUBE_API_KEY', 'AIzaSyBabzE4Bup77WexdLMa9rN9z-wJidEfNX8') - -# SECURITY WARNING: don't run with debug turned on in production! -DEBUG = True - +# +# Basic Django stuff +# ALLOWED_HOSTS = ['*'] - SESSION_COOKIE_AGE = 3600 * 30 # one month # Application definition @@ -76,24 +66,6 @@ TEMPLATES = [ WSGI_APPLICATION = 'YtManager.wsgi.application' -# Database -# https://docs.djangoproject.com/en/1.11/ref/settings/#databases - -DATABASES = { - 'default': { - 'ENGINE': os.getenv('YTSM_DATABASE_ENGINE', 'django.db.backends.sqlite3'), - 'NAME': os.getenv('YTSM_DATABASE_NAME', os.path.join(BASE_DIR, 'ytmanager.db')), - 'HOST': os.getenv('YTSM_DATABASE_HOST', None), - 'USER': os.getenv('YTSM_DATABASE_USERNAME', None), - 'PASSWORD': os.getenv('YTSM_DATABASE_PASSWORD', None), - 'PORT': os.getenv('YTSM_DATABASE_PORT', None) - } -} - -if os.getenv('YTSM_DATABASE_URL', None): - import dj_database_url - DATABASES['default'] = dj_database_url.parse(os.environ['YTSM_DATABASE_URL'], conn_max_age=600) - # Password validation # https://docs.djangoproject.com/en/1.11/ref/settings/#auth-password-validators @@ -112,6 +84,9 @@ AUTH_PASSWORD_VALIDATORS = [ }, ] +LOGIN_REDIRECT_URL = '/' +LOGIN_URL = '/login' + # Internationalization # https://docs.djangoproject.com/en/1.11/topics/i18n/ @@ -126,14 +101,126 @@ USE_L10N = True USE_TZ = True + # Static files (CSS, JavaScript, Images) # https://docs.djangoproject.com/en/1.11/howto/static-files/ STATIC_URL = '/static/' MEDIA_URL = '/media/' -MEDIA_ROOT = 'data/media' + + +# Misc Django stuff CRISPY_TEMPLATE_PACK = 'bootstrap4' -LOGIN_REDIRECT_URL = '/' -LOGIN_URL = '/login' +LOG_FORMAT = '%(asctime)s|%(process)d|%(thread)d|%(name)s|%(filename)s|%(lineno)d|%(levelname)s|%(message)s' + +# +# Directories +# + +# Build paths inside the project like this: os.path.join(BASE_DIR, ...) +PROJECT_ROOT = up(up(os.path.dirname(__file__))) # Project root +BASE_DIR = os.path.join(PROJECT_ROOT, "app") # Base dir of the application +CONFIG_DIR = os.path.join(PROJECT_ROOT, "config") +DATA_DIR = os.path.join(PROJECT_ROOT, "data") +STATIC_ROOT = os.path.join(PROJECT_ROOT, "static") + +_DEFAULT_CONFIG_FILE = os.path.join(CONFIG_DIR, 'config.ini') +_DEFAULT_LOG_FILE = os.path.join(DATA_DIR, 'log.log') +_DEFAULT_MEDIA_ROOT = os.path.join(DATA_DIR, 'media') + +DEFAULTS_FILE = os.path.join(CONFIG_DIR, 'defaults.ini') +CONFIG_FILE = os.getenv('YTSM_CONFIG_FILE', _DEFAULT_CONFIG_FILE) + +# +# Defaults +# +_DEFAULT_DEBUG = False + +_DEFAULT_SECRET_KEY = '^zv8@i2h!ko2lo=%ivq(9e#x=%q*i^^)6#4@(juzdx%&0c+9a0' +_DEFAULT_YOUTUBE_API_KEY = 'AIzaSyBabzE4Bup77WexdLMa9rN9z-wJidEfNX8' + +_DEFAULT_DATABASE = { + 'ENGINE': 'django.db.backends.sqlite3', + 'NAME': os.path.join(DATA_DIR, 'ytmanager.db'), + 'HOST': None, + 'USER': None, + 'PASSWORD': None, + 'PORT': None, + } + +_SCHEDULER_SYNC_SCHEDULE = '5 * * * *' +_DEFAULT_SCHEDULER_CONCURRENCY = 1 + + +# +# Load globals from config.ini +# +def load_config_ini(): + from configparser import ConfigParser + from YtManagerApp.utils.extended_interpolation_with_env import ExtendedInterpolatorWithEnv + import dj_database_url + + cfg = ConfigParser(allow_no_value=True, interpolation=ExtendedInterpolatorWithEnv()) + read_ok = cfg.read([DEFAULTS_FILE, CONFIG_FILE]) + + if DEFAULTS_FILE not in read_ok: + print('Failed to read file ' + DEFAULTS_FILE) + raise Exception('Cannot read file ' + DEFAULTS_FILE) + if CONFIG_FILE not in read_ok: + print('Failed to read file ' + CONFIG_FILE) + raise Exception('Cannot read file ' + CONFIG_FILE) + + # Debug + global DEBUG + DEBUG = cfg.getboolean('global', 'Debug', fallback=_DEFAULT_DEBUG) + + # Media root, which is where thumbnails are stored + global MEDIA_ROOT + MEDIA_ROOT = cfg.get('global', 'MediaRoot', fallback=_DEFAULT_MEDIA_ROOT) + + # Keys - secret key, youtube API key + # SECURITY WARNING: keep the secret key used in production secret! + global SECRET_KEY, YOUTUBE_API_KEY + SECRET_KEY = cfg.get('global', 'SecretKey', fallback=_DEFAULT_SECRET_KEY) + YOUTUBE_API_KEY = cfg.get('global', 'YoutubeApiKey', fallback=_DEFAULT_YOUTUBE_API_KEY) + + # Database + global DATABASES + DATABASES = { + 'default': _DEFAULT_DATABASE + } + + if cfg.has_option('global', 'DatabaseURL'): + DATABASES['default'] = dj_database_url.parse(cfg.get('global', 'DatabaseURL'), conn_max_age=600) + + else: + DATABASES['default'] = { + 'ENGINE': cfg.get('global', 'DatabaseEngine', fallback=_DEFAULT_DATABASE['ENGINE']), + 'NAME': cfg.get('global', 'DatabaseName', fallback=_DEFAULT_DATABASE['NAME']), + 'HOST': cfg.get('global', 'DatabaseHost', fallback=_DEFAULT_DATABASE['HOST']), + 'USER': cfg.get('global', 'DatabaseUser', fallback=_DEFAULT_DATABASE['USER']), + 'PASSWORD': cfg.get('global', 'DatabasePassword', fallback=_DEFAULT_DATABASE['PASSWORD']), + 'PORT': cfg.get('global', 'DatabasePort', fallback=_DEFAULT_DATABASE['PORT']), + } + + # Log settings + global LOG_LEVEL, LOG_FILE + log_level_str = cfg.get('global', 'LogLevel', fallback='INFO') + + try: + LOG_LEVEL = getattr(logging, log_level_str) + except AttributeError: + print("Invalid log level " + LOG_LEVEL) + LOG_LEVEL = logging.INFO + + LOG_FILE = cfg.get('global', 'LogFile', fallback=_DEFAULT_LOG_FILE) + + # Scheduler settings + global SCHEDULER_SYNC_SCHEDULE, SCHEDULER_CONCURRENCY + SCHEDULER_SYNC_SCHEDULE = cfg.get('global', 'SynchronizationSchedule', fallback=_SCHEDULER_SYNC_SCHEDULE) + SCHEDULER_CONCURRENCY = cfg.getint('global', 'SchedulerConcurrency', fallback=_DEFAULT_SCHEDULER_CONCURRENCY) + + +load_config_ini() diff --git a/app/YtManagerApp/appconfig.py b/app/YtManagerApp/appconfig.py index beb6b01..9905558 100644 --- a/app/YtManagerApp/appconfig.py +++ b/app/YtManagerApp/appconfig.py @@ -12,35 +12,26 @@ from django.contrib.auth.models import User from .models import UserSettings, Subscription from .utils.extended_interpolation_with_env import ExtendedInterpolatorWithEnv -_CONFIG_DIR = os.path.join(dj_settings.BASE_DIR, 'config') -_LOG_FILE = 'log.log' -_LOG_PATH = os.path.join(_CONFIG_DIR, _LOG_FILE) -_LOG_FORMAT = '%(asctime)s|%(process)d|%(thread)d|%(name)s|%(filename)s|%(lineno)d|%(levelname)s|%(message)s' - class AppSettings(ConfigParser): _DEFAULT_INTERPOLATION = ExtendedInterpolatorWithEnv() - __DEFAULTS_FILE = 'defaults.ini' - __SETTINGS_FILE = 'config.ini' def __init__(self, *args, **kwargs): super().__init__(allow_no_value=True, *args, **kwargs) - self.__defaults_path = os.path.join(_CONFIG_DIR, AppSettings.__DEFAULTS_FILE) - self.__settings_path = os.path.join(_CONFIG_DIR, AppSettings.__SETTINGS_FILE) def initialize(self): - self.read([self.__defaults_path, self.__settings_path]) + self.read([dj_settings.DEFAULTS_FILE, dj_settings.CONFIG_FILE]) def save(self): - if os.path.exists(self.__settings_path): + if os.path.exists(dj_settings.CONFIG_FILE): # Create a backup - copyfile(self.__settings_path, self.__settings_path + ".backup") + copyfile(dj_settings.CONFIG_FILE, dj_settings.CONFIG_FILE + ".backup") else: # Ensure directory exists - settings_dir = os.path.dirname(self.__settings_path) + settings_dir = os.path.dirname(dj_settings.CONFIG_FILE) os.makedirs(settings_dir, exist_ok=True) - with open(self.__settings_path, 'w') as f: + with open(dj_settings.CONFIG_FILE, 'w') as f: self.write(f) def __get_combined_dict(self, vars: Optional[Any], sub: Optional[Subscription], user: Optional[User]) -> ChainMap: @@ -112,12 +103,10 @@ def initialize_app_config(): def __initialize_logger(): - log_level_str = settings.get('global', 'LogLevel', fallback='INFO') + log_dir = os.path.dirname(dj_settings.LOG_FILE) + os.makedirs(log_dir, exist_ok=True) - try: - log_level = getattr(logging, log_level_str) - logging.basicConfig(filename=_LOG_PATH, level=log_level, format=_LOG_FORMAT) - - except AttributeError: - logging.basicConfig(filename=_LOG_PATH, level=logging.INFO, format=_LOG_FORMAT) - logging.warning('Invalid log level "%s" in config file.', log_level_str) + logging.basicConfig( + filename=dj_settings.LOG_FILE, + level=dj_settings.LOG_LEVEL, + format=dj_settings.LOG_FORMAT) diff --git a/app/YtManagerApp/management/jobs/download_video.py b/app/YtManagerApp/management/jobs/download_video.py index 7920877..76854e7 100644 --- a/app/YtManagerApp/management/jobs/download_video.py +++ b/app/YtManagerApp/management/jobs/download_video.py @@ -5,10 +5,13 @@ import os import youtube_dl import logging import re +from threading import Lock log = logging.getLogger('video_downloader') log_youtube_dl = log.getChild('youtube_dl') +_lock = Lock() + def __get_valid_path(path): """ @@ -73,27 +76,36 @@ def download_video(video: Video, attempt: int = 1): log.info('Downloading video %d [%s %s]', video.id, video.video_id, video.name) - max_attempts = settings.getint_sub(video.subscription, 'user', 'DownloadMaxAttempts', fallback=3) + # Issue: if multiple videos are downloaded at the same time, a race condition appears in the mkdirs() call that + # youtube-dl makes, which causes it to fail with the error 'Cannot create folder - file already exists'. + # For now, allow a single download instance. + _lock.acquire() - youtube_dl_params, output_path = __build_youtube_dl_params(video) - with youtube_dl.YoutubeDL(youtube_dl_params) as yt: - ret = yt.download(["https://www.youtube.com/watch?v=" + video.video_id]) + try: + max_attempts = settings.getint_sub(video.subscription, 'user', 'DownloadMaxAttempts', fallback=3) - log.info('Download finished with code %d', ret) + youtube_dl_params, output_path = __build_youtube_dl_params(video) + with youtube_dl.YoutubeDL(youtube_dl_params) as yt: + ret = yt.download(["https://www.youtube.com/watch?v=" + video.video_id]) - if ret == 0: - video.downloaded_path = output_path - video.save() - log.info('Video %d [%s %s] downloaded successfully!', video.id, video.video_id, video.name) + log.info('Download finished with code %d', ret) - elif attempt <= max_attempts: - log.warning('Re-enqueueing video (attempt %d/%d)', attempt, max_attempts) - __schedule_download_video(video, attempt + 1) + if ret == 0: + video.downloaded_path = output_path + video.save() + log.info('Video %d [%s %s] downloaded successfully!', video.id, video.video_id, video.name) - else: - log.error('Multiple attempts to download video %d [%s %s] failed!', video.id, video.video_id, video.name) - video.downloaded_path = '' - video.save() + elif attempt <= max_attempts: + log.warning('Re-enqueueing video (attempt %d/%d)', attempt, max_attempts) + __schedule_download_video(video, attempt + 1) + + else: + log.error('Multiple attempts to download video %d [%s %s] failed!', video.id, video.video_id, video.name) + video.downloaded_path = '' + video.save() + + finally: + _lock.release() def __schedule_download_video(video: Video, attempt=1): diff --git a/app/YtManagerApp/utils/extended_interpolation_with_env.py b/app/YtManagerApp/utils/extended_interpolation_with_env.py index 2c66022..cef09d4 100644 --- a/app/YtManagerApp/utils/extended_interpolation_with_env.py +++ b/app/YtManagerApp/utils/extended_interpolation_with_env.py @@ -35,7 +35,7 @@ class ExtendedInterpolatorWithEnv(Interpolation): def _resolve_section_option(self, section, option, parser): if section == 'env': return os.getenv(option, '') - return parser.get(section, option, raw=True) + return parser.get(section, parser.optionxform(option), raw=True) def _interpolate_some(self, parser, option, accum, rest, section, map, depth): @@ -70,7 +70,7 @@ class ExtendedInterpolatorWithEnv(Interpolation): v = self._resolve_option(opt, map) elif len(path) == 2: sect = path[0] - opt = parser.optionxform(path[1]) + opt = path[1] v = self._resolve_section_option(sect, opt, parser) else: raise InterpolationSyntaxError( diff --git a/app/config/config.ini b/app/config/config.ini deleted file mode 100644 index f3c10a6..0000000 --- a/app/config/config.ini +++ /dev/null @@ -1,59 +0,0 @@ -; Use $ to use the value of an environment variable. -; The global section contains settings that apply to the entire server -[global] -; YouTube API key - get this from your user account -;YoutubeApiKey= - -; Specifies the synchronization schedule, in crontab format. -; Format: -SynchronizationSchedule=5 * * * * - -; Number of threads running the scheduler -; Since most of the jobs scheduled are downloads, there is no advantage to having -; a higher concurrency -SchedulerConcurrency=1 - -; Log level -LogLevel=DEBUG - -; Default user settings -[user] -; When a video is deleted on the system, it will be marked as 'watched' -MarkDeletedAsWatched=True - -; Videos marked as watched are automatically deleted -DeleteWatched=True - -; Enable automatic downloading -AutoDownload=True - -; Limit the total number of videos downloaded (-1 or empty = no limit) -DownloadGlobalLimit=10 - -; Limit the numbers of videos per subscription (-1 or empty = no limit) -DownloadSubscriptionLimit=5 - -; Number of download attempts -DownloadMaxAttempts=3 - -; Download order -; Options: newest, oldest, playlist, playlist_reverse, popularity, rating -DownloadOrder=playlist - -; Path where downloaded videos are stored -DownloadPath=data/media/videos - -; A pattern which describes how downloaded files are organized. Extensions are automatically appended. -; Supported fields: channel, channel_id, playlist, playlist_id, playlist_index, title, id -; The default pattern should work pretty well with Plex -;DownloadFilePattern=${channel}/${playlist}/S01E${playlist_index} - ${title} [${id}] - -; Download format that will be passed to youtube-dl. See the youtube-dl documentation for more details. -DownloadFormat=bestvideo+bestaudio - -; Subtitles - these options match the youtube-dl options -;DownloadSubtitles=True -;DownloadAutogeneratedSubtitles=False -;DownloadSubtitlesAll=False -;DownloadSubtitlesLangs=en,ro -;DownloadSubtitlesFormat= diff --git a/app/config/defaults.ini b/app/config/defaults.ini deleted file mode 100644 index a4a0b30..0000000 --- a/app/config/defaults.ini +++ /dev/null @@ -1,59 +0,0 @@ -; Use $ to use the value of an environment variable. -; The global section contains settings that apply to the entire server -[global] -; YouTube API key - get this from your user account -YoutubeApiKey=AIzaSyBabzE4Bup77WexdLMa9rN9z-wJidEfNX8 - -; Specifies the synchronization schedule, in crontab format. -; Format: -SynchronizationSchedule=0 * * * * - -; Number of threads running the scheduler -; Since most of the jobs scheduled are downloads, there is no advantage to having -; a higher concurrency -SchedulerConcurrency=2 - -; Log level -LogLevel=INFO - -; Default user settings -[user] -; When a video is deleted on the system, it will be marked as 'watched' -MarkDeletedAsWatched=True - -; Videos marked as watched are automatically deleted -DeleteWatched=True - -; Enable automatic downloading -AutoDownload=True - -; Limit the total number of videos downloaded (-1 or empty = no limit) -DownloadGlobalLimit= - -; Limit the numbers of videos per subscription (-1 or empty = no limit) -DownloadSubscriptionLimit=5 - -; Number of download attempts -DownloadMaxAttempts=3 - -; Download order -; Options: newest, oldest, playlist, playlist_reverse, popularity, rating -DownloadOrder=playlist - -; Path where downloaded videos are stored -DownloadPath=${env:USERPROFILE}${env:HOME}/Downloads - -; A pattern which describes how downloaded files are organized. Extensions are automatically appended. -; Supported fields: channel, channel_id, playlist, playlist_id, playlist_index, title, id -; The default pattern should work pretty well with Plex -DownloadFilePattern=${channel}/${playlist}/S01E${playlist_index} - ${title} [${id}] - -; Download format that will be passed to youtube-dl. See the youtube-dl documentation for more details. -DownloadFormat=bestvideo+bestaudio - -; Subtitles - these options match the youtube-dl options -DownloadSubtitles=True -DownloadAutogeneratedSubtitles=False -DownloadSubtitlesAll=False -DownloadSubtitlesLangs=en,ro -DownloadSubtitlesFormat= diff --git a/config/config.ini b/config/config.ini index c3d0f48..91352d4 100644 --- a/config/config.ini +++ b/config/config.ini @@ -1,47 +1,75 @@ -; Use $ to use the value of an environment variable. +; Use ${env:environment_variable} to use the value of an environment variable. +; If a variable is not set here, it will be loaded from defaults.ini. + ; The global section contains settings that apply to the entire server [global] + +Debug=${env:YTSM_DEBUG} + +; This is the folder where thumbnails will be downloaded. By default project_root/data/media is used. +;MediaRoot= + +; Secret key - django secret key +;SecretKey=^zv8@i2h!ko2lo=%ivq(9e#x=%q*i^^)6#4@(juzdx%&0c+9a0 + ; YouTube API key - get this from your user account -YoutubeApiKey=AIzaSyAonB6T-DrKjfGxBGuHyFMg0x_d0T9nlP8 +;YoutubeApiKey=AIzaSyAonB6T-DrKjfGxBGuHyFMg0x_d0T9nlP8 + +; Database settings +; You can use any database engine supported by Django, as long as you add the required dependencies. +; Built-in engines: https://docs.djangoproject.com/en/2.1/ref/settings/#std:setting-DATABASE-ENGINE +; Others databases might be supported by installing the corect pip package. + +;DatabaseEngine=django.db.backends.sqlite3 +;DatabaseName=data/ytmanager.db +;DatabaseHost= +;DatabaseUser= +;DatabasePassword= +;DatabasePort= + +; Database one-liner. If set, it will override any other Database* setting. +; Documentation: https://github.com/kennethreitz/dj-database-url +;DatabaseURL=sqlite:////full/path/to/your/database/file.sqlite + +; Log settings, sets the log file location and the log level +;LogLevel=INFO +;LogFile=data/log.log ; Specifies the synchronization schedule, in crontab format. ; Format: -SynchronizationSchedule=5 * * * * +;SynchronizationSchedule=5 * * * * ; Number of threads running the scheduler ; Since most of the jobs scheduled are downloads, there is no advantage to having ; a higher concurrency -SchedulerConcurrency=1 - -; Log level -LogLevel=DEBUG +;SchedulerConcurrency=1 ; Default user settings [user] ; When a video is deleted on the system, it will be marked as 'watched' -MarkDeletedAsWatched=True +;MarkDeletedAsWatched=True ; Videos marked as watched are automatically deleted -DeleteWatched=True +;DeleteWatched=True ; Enable automatic downloading -AutoDownload=True +;AutoDownload=True ; Limit the total number of videos downloaded (-1 or empty = no limit) -DownloadGlobalLimit=10 +;DownloadGlobalLimit=10 ; Limit the numbers of videos per subscription (-1 or empty = no limit) -DownloadSubscriptionLimit=5 +;DownloadSubscriptionLimit=5 ; Number of download attempts -DownloadMaxAttempts=3 +;DownloadMaxAttempts=3 ; Download order ; Options: newest, oldest, playlist, playlist_reverse, popularity, rating -DownloadOrder=playlist +;DownloadOrder=playlist ; Path where downloaded videos are stored -DownloadPath=data/media/videos +DownloadPath=${env:YTSM_DATA_PATH}/videos ; A pattern which describes how downloaded files are organized. Extensions are automatically appended. ; Supported fields: channel, channel_id, playlist, playlist_id, playlist_index, title, id @@ -49,7 +77,7 @@ DownloadPath=data/media/videos ;DownloadFilePattern=${channel}/${playlist}/S01E${playlist_index} - ${title} [${id}] ; Download format that will be passed to youtube-dl. See the youtube-dl documentation for more details. -DownloadFormat=bestvideo+bestaudio +;DownloadFormat=bestvideo+bestaudio ; Subtitles - these options match the youtube-dl options ;DownloadSubtitles=True diff --git a/config/defaults.ini b/config/defaults.ini index a4a0b30..2a6f045 100644 --- a/config/defaults.ini +++ b/config/defaults.ini @@ -1,20 +1,48 @@ -; Use $ to use the value of an environment variable. +; Use ${env:environment_variable} to use the value of an environment variable. ; The global section contains settings that apply to the entire server [global] + +; Controls whether django debug mode is enabled. Should be false in production. +Debug=False + +; This is the folder where thumbnails will be downloaded. By default project_root/data/media is used. +;MediaRoot= + +; Secret key - django secret key +SecretKey=^zv8@i2h!ko2lo=%ivq(9e#x=%q*i^^)6#4@(juzdx%&0c+9a0 + ; YouTube API key - get this from your user account YoutubeApiKey=AIzaSyBabzE4Bup77WexdLMa9rN9z-wJidEfNX8 +; Database settings +; You can use any database engine supported by Django, as long as you add the required dependencies. +; Built-in engines: https://docs.djangoproject.com/en/2.1/ref/settings/#std:setting-DATABASE-ENGINE +; Others databases might be supported by installing the corect pip package. + +;DatabaseEngine=django.db.backends.sqlite3 +;DatabaseName=data/ytmanager.db +;DatabaseHost= +;DatabaseUser= +;DatabasePassword= +;DatabasePort= + +; Database one-liner. If set, it will override any other Database* setting. +; Documentation: https://github.com/kennethreitz/dj-database-url +;DatabaseURL=sqlite:////full/path/to/your/database/file.sqlite + +; Log settings, sets the log file location and the log level +LogLevel=INFO +; LogFile=data/log.log + ; Specifies the synchronization schedule, in crontab format. ; Format: -SynchronizationSchedule=0 * * * * +SynchronizationSchedule=5 * * * * ; Number of threads running the scheduler ; Since most of the jobs scheduled are downloads, there is no advantage to having ; a higher concurrency -SchedulerConcurrency=2 +SchedulerConcurrency=3 -; Log level -LogLevel=INFO ; Default user settings [user] @@ -41,7 +69,7 @@ DownloadMaxAttempts=3 DownloadOrder=playlist ; Path where downloaded videos are stored -DownloadPath=${env:USERPROFILE}${env:HOME}/Downloads +DownloadPath=data/videos ; A pattern which describes how downloaded files are organized. Extensions are automatically appended. ; Supported fields: channel, channel_id, playlist, playlist_id, playlist_index, title, id diff --git a/docker-compose.yml b/docker-compose.yml index c666333..d3069ab 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -4,8 +4,9 @@ services: nginx: image: nginx:latest volumes: - - ./nginx:/etc/nginx/conf.d/ + - ./docker/nginx:/etc/nginx/conf.d/ - ./app/YtManagerApp/static:/www/static + - ./data/media:/www/media ports: - "80:80" depends_on: @@ -13,11 +14,9 @@ services: web: build: . - env_file: - - sqlite3.env.env tty: true ports: - "8000:8000" volumes: - - ./media:/usr/src/app/data/media - - ./db:/usr/src/app/data/db + - ./config:/usr/src/ytsm/config + - ./data:/usr/src/ytsm/data diff --git a/app/init.sh b/docker/init.sh similarity index 62% rename from app/init.sh rename to docker/init.sh index e0ede49..8608bf6 100755 --- a/app/init.sh +++ b/docker/init.sh @@ -1,5 +1,4 @@ #!/bin/bash -#./manage.py runserver 0.0.0.0:8000 --noreload ./manage.py migrate gunicorn -b 0.0.0.0:8000 -w 4 YtManager.wsgi diff --git a/nginx/nginx.conf b/docker/nginx/nginx.conf similarity index 91% rename from nginx/nginx.conf rename to docker/nginx/nginx.conf index 1b60a84..2efc9d4 100644 --- a/nginx/nginx.conf +++ b/docker/nginx/nginx.conf @@ -15,6 +15,10 @@ server { alias /www/static; expires 30d; } + location /media { + alias /www/media; + expires 30d; + } location / { try_files $uri @proxy_to_app; diff --git a/app/requirements.txt b/requirements.txt similarity index 100% rename from app/requirements.txt rename to requirements.txt diff --git a/sqlite3.env.env b/sqlite3.env.env deleted file mode 100644 index adfe8b4..0000000 --- a/sqlite3.env.env +++ /dev/null @@ -1,3 +0,0 @@ -YTSM_DATABASE_ENGINE=django.db.backends.sqlite3 -YTSM_DATABASE_NAME=/usr/src/app/data/db/ytmanager.db -YTSM_YOUTUBE_API_KEY=AIzaSyBabzE4Bup77WexdLMa9rN9z-wJidEfNX8