diff --git a/root/app/calibre-web/cps/admin.py b/root/app/calibre-web/cps/admin.py index a04d555..41f853f 100644 --- a/root/app/calibre-web/cps/admin.py +++ b/root/app/calibre-web/cps/admin.py @@ -62,6 +62,7 @@ feature_support = { 'ldap': bool(services.ldap), 'goodreads': bool(services.goodreads_support), 'kobo': bool(services.kobo), + 'hardcover' : bool(services.hardcover), 'updater': constants.UPDATER_AVAILABLE, 'gmail': bool(services.gmail), 'scheduler': schedule.use_APScheduler, @@ -1831,6 +1832,7 @@ def _configuration_update_helper(): reboot_required |= _config_checkbox_int(to_save, "config_kobo_sync") _config_int(to_save, "config_external_port") _config_checkbox_int(to_save, "config_kobo_proxy") + _config_checkbox_int(to_save, "config_hardcover_sync") if "config_upload_formats" in to_save: to_save["config_upload_formats"] = ','.join( diff --git a/root/app/calibre-web/cps/config_sql.py b/root/app/calibre-web/cps/config_sql.py new file mode 100644 index 0000000..d0164a4 --- /dev/null +++ b/root/app/calibre-web/cps/config_sql.py @@ -0,0 +1,588 @@ +# -*- coding: utf-8 -*- + +# This file is part of the Calibre-Web (https://github.com/janeczku/calibre-web) +# Copyright (C) 2019 OzzieIsaacs, pwr +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import os +import sys +import json + +from sqlalchemy import Column, String, Integer, SmallInteger, Boolean, BLOB, JSON +from sqlalchemy.exc import OperationalError +from sqlalchemy.sql.expression import text +from sqlalchemy import exists +from cryptography.fernet import Fernet +import cryptography.exceptions +from base64 import urlsafe_b64decode +try: + # Compatibility with sqlalchemy 2.0 + from sqlalchemy.orm import declarative_base +except ImportError: + from sqlalchemy.ext.declarative import declarative_base + +from . import constants, logger +from .subproc_wrapper import process_wait +from .string_helper import strip_whitespaces + +log = logger.create() +_Base = declarative_base() + + +class _Flask_Settings(_Base): + __tablename__ = 'flask_settings' + + id = Column(Integer, primary_key=True) + flask_session_key = Column(BLOB, default=b"") + + def __init__(self, key): + super().__init__() + self.flask_session_key = key + + +# Baseclass for representing settings in app.db with email server settings and Calibre database settings +# (application settings) +class _Settings(_Base): + __tablename__ = 'settings' + + id = Column(Integer, primary_key=True) + mail_server = Column(String, default=constants.DEFAULT_MAIL_SERVER) + mail_port = Column(Integer, default=25) + mail_use_ssl = Column(SmallInteger, default=0) + mail_login = Column(String, default='mail@example.com') + mail_password_e = Column(String) + mail_password = Column(String) + mail_from = Column(String, default='automailer ') + mail_size = Column(Integer, default=25*1024*1024) + mail_server_type = Column(SmallInteger, default=0) + mail_gmail_token = Column(JSON, default={}) + + config_calibre_dir = Column(String) + config_calibre_uuid = Column(String) + config_calibre_split = Column(Boolean, default=False) + config_calibre_split_dir = Column(String) + config_port = Column(Integer, default=constants.DEFAULT_PORT) + config_external_port = Column(Integer, default=constants.DEFAULT_PORT) + config_certfile = Column(String) + config_keyfile = Column(String) + config_trustedhosts = Column(String, default='') + config_calibre_web_title = Column(String, default='Calibre-Web') + config_books_per_page = Column(Integer, default=60) + config_random_books = Column(Integer, default=4) + config_authors_max = Column(Integer, default=0) + config_read_column = Column(Integer, default=0) + config_title_regex = Column(String, + default=r'^(A|The|An|Der|Die|Das|Den|Ein|Eine' + r'|Einen|Dem|Des|Einem|Eines|Le|La|Les|L\'|Un|Une)\s+') + config_theme = Column(Integer, default=0) + + config_log_level = Column(SmallInteger, default=logger.DEFAULT_LOG_LEVEL) + config_logfile = Column(String, default=logger.DEFAULT_LOG_FILE) + config_access_log = Column(SmallInteger, default=0) + config_access_logfile = Column(String, default=logger.DEFAULT_ACCESS_LOG) + + config_uploading = Column(SmallInteger, default=0) + config_anonbrowse = Column(SmallInteger, default=0) + config_public_reg = Column(SmallInteger, default=0) + config_remote_login = Column(Boolean, default=False) + config_kobo_sync = Column(Boolean, default=False) + config_hardcover_sync = Column(Boolean, default=False) + + config_default_role = Column(SmallInteger, default=0) + config_default_show = Column(SmallInteger, default=constants.ADMIN_USER_SIDEBAR) + config_default_language = Column(String(3), default="all") + config_default_locale = Column(String(2), default="en") + config_columns_to_ignore = Column(String) + + config_denied_tags = Column(String, default="") + config_allowed_tags = Column(String, default="") + config_restricted_column = Column(SmallInteger, default=0) + config_denied_column_value = Column(String, default="") + config_allowed_column_value = Column(String, default="") + + config_use_google_drive = Column(Boolean, default=False) + config_google_drive_folder = Column(String) + config_google_drive_watch_changes_response = Column(JSON, default={}) + + config_use_goodreads = Column(Boolean, default=False) + config_goodreads_api_key = Column(String) + config_register_email = Column(Boolean, default=False) + config_login_type = Column(Integer, default=0) + + config_kobo_proxy = Column(Boolean, default=False) + + config_ldap_provider_url = Column(String, default='example.org') + config_ldap_port = Column(SmallInteger, default=389) + config_ldap_authentication = Column(SmallInteger, default=constants.LDAP_AUTH_SIMPLE) + config_ldap_serv_username = Column(String, default='cn=admin,dc=example,dc=org') + config_ldap_serv_password_e = Column(String) + config_ldap_serv_password = Column(String) + config_ldap_encryption = Column(SmallInteger, default=0) + config_ldap_cacert_path = Column(String, default="") + config_ldap_cert_path = Column(String, default="") + config_ldap_key_path = Column(String, default="") + config_ldap_dn = Column(String, default='dc=example,dc=org') + config_ldap_user_object = Column(String, default='uid=%s') + config_ldap_member_user_object = Column(String, default='') + config_ldap_openldap = Column(Boolean, default=True) + config_ldap_group_object_filter = Column(String, default='(&(objectclass=posixGroup)(cn=%s))') + config_ldap_group_members_field = Column(String, default='memberUid') + config_ldap_group_name = Column(String, default='calibreweb') + + config_kepubifypath = Column(String, default=None) + config_converterpath = Column(String, default=None) + config_binariesdir = Column(String, default=None) + config_calibre = Column(String) + config_rarfile_location = Column(String, default=None) + config_upload_formats = Column(String, default=','.join(constants.EXTENSIONS_UPLOAD)) + config_unicode_filename = Column(Boolean, default=False) + config_embed_metadata = Column(Boolean, default=True) + + config_updatechannel = Column(Integer, default=constants.UPDATE_STABLE) + + config_reverse_proxy_login_header_name = Column(String) + config_allow_reverse_proxy_header_login = Column(Boolean, default=False) + + schedule_start_time = Column(Integer, default=4) + schedule_duration = Column(Integer, default=10) + schedule_generate_book_covers = Column(Boolean, default=False) + schedule_generate_series_covers = Column(Boolean, default=False) + schedule_reconnect = Column(Boolean, default=False) + schedule_metadata_backup = Column(Boolean, default=False) + + config_password_policy = Column(Boolean, default=True) + config_password_min_length = Column(Integer, default=8) + config_password_number = Column(Boolean, default=True) + config_password_lower = Column(Boolean, default=True) + config_password_upper = Column(Boolean, default=True) + config_password_character = Column(Boolean, default=True) + config_password_special = Column(Boolean, default=True) + config_session = Column(Integer, default=1) + config_ratelimiter = Column(Boolean, default=True) + config_limiter_uri = Column(String, default="") + config_limiter_options = Column(String, default="") + config_check_extensions = Column(Boolean, default=True) + + def __repr__(self): + return self.__class__.__name__ + + +# Class holds all application specific settings in calibre-web +class ConfigSQL(object): + # pylint: disable=no-member + def __init__(self): + self.__dict__["dirty"] = list() + + def init_config(self, session, secret_key, cli): + self._session = session + self._settings = None + self.db_configured = None + self.config_calibre_dir = None + self._fernet = Fernet(secret_key) + self.cli = cli + self.load() + + change = False + + if self.config_binariesdir is None: + change = True + self.config_binariesdir = autodetect_calibre_binaries() + self.config_converterpath = autodetect_converter_binary(self.config_binariesdir) + + if self.config_kepubifypath is None: + change = True + self.config_kepubifypath = autodetect_kepubify_binary() + + if self.config_rarfile_location is None: + change = True + self.config_rarfile_location = autodetect_unrar_binary() + if change: + self.save() + + def _read_from_storage(self): + if self._settings is None: + log.debug("_ConfigSQL._read_from_storage") + self._settings = self._session.query(_Settings).first() + return self._settings + + def get_config_certfile(self): + if self.cli.certfilepath: + return self.cli.certfilepath + if self.cli.certfilepath == "": + return None + return self.config_certfile + + def get_config_keyfile(self): + if self.cli.keyfilepath: + return self.cli.keyfilepath + if self.cli.certfilepath == "": + return None + return self.config_keyfile + + def get_config_ipaddress(self): + return self.cli.ip_address or "" + + def _has_role(self, role_flag): + return constants.has_flag(self.config_default_role, role_flag) + + def role_admin(self): + return self._has_role(constants.ROLE_ADMIN) + + def role_download(self): + return self._has_role(constants.ROLE_DOWNLOAD) + + def role_viewer(self): + return self._has_role(constants.ROLE_VIEWER) + + def role_upload(self): + return self._has_role(constants.ROLE_UPLOAD) + + def role_edit(self): + return self._has_role(constants.ROLE_EDIT) + + def role_passwd(self): + return self._has_role(constants.ROLE_PASSWD) + + def role_edit_shelfs(self): + return self._has_role(constants.ROLE_EDIT_SHELFS) + + def role_delete_books(self): + return self._has_role(constants.ROLE_DELETE_BOOKS) + + def show_element_new_user(self, value): + return constants.has_flag(self.config_default_show, value) + + def show_detail_random(self): + return self.show_element_new_user(constants.DETAIL_RANDOM) + + def list_denied_tags(self): + mct = self.config_denied_tags or "" + return [strip_whitespaces(t) for t in mct.split(",")] + + def list_allowed_tags(self): + mct = self.config_allowed_tags or "" + return [strip_whitespaces(t) for t in mct.split(",")] + + def list_denied_column_values(self): + mct = self.config_denied_column_value or "" + return [strip_whitespaces(t) for t in mct.split(",")] + + def list_allowed_column_values(self): + mct = self.config_allowed_column_value or "" + return [strip_whitespaces(t) for t in mct.split(",")] + + def get_log_level(self): + return logger.get_level_name(self.config_log_level) + + def get_mail_settings(self): + return {k: v for k, v in self.__dict__.items() if k.startswith('mail_')} + + def get_mail_server_configured(self): + return bool((self.mail_server != constants.DEFAULT_MAIL_SERVER and self.mail_server_type == 0) + or (self.mail_gmail_token != {} and self.mail_server_type == 1)) + + def get_scheduled_task_settings(self): + return {k: v for k, v in self.__dict__.items() if k.startswith('schedule_')} + + def set_from_dictionary(self, dictionary, field, convertor=None, default=None, encode=None): + """Possibly updates a field of this object. + The new value, if present, is grabbed from the given dictionary, and optionally passed through a convertor. + + :returns: `True` if the field has changed value + """ + new_value = dictionary.get(field, default) + if new_value is None: + return False + + if field not in self.__dict__: + log.warning("_ConfigSQL trying to set unknown field '%s' = %r", field, new_value) + return False + + if convertor is not None: + if encode: + new_value = convertor(new_value.encode(encode)) + else: + new_value = convertor(new_value) + + current_value = self.__dict__.get(field) + if current_value == new_value: + return False + + setattr(self, field, new_value) + return True + + def to_dict(self): + storage = {} + for k, v in self.__dict__.items(): + if k[0] != '_' and not k.endswith("_e") and not k == "cli": + storage[k] = v + return storage + + def load(self): + """Load all configuration values from the underlying storage.""" + s = self._read_from_storage() # type: _Settings + for k, v in s.__dict__.items(): + if k[0] != '_': + if v is None: + # if the storage column has no value, apply the (possible) default + column = s.__class__.__dict__.get(k) + if column.default is not None: + v = column.default.arg + if k.endswith("_e") and v is not None: + try: + setattr(self, k, self._fernet.decrypt(v).decode()) + except cryptography.fernet.InvalidToken: + setattr(self, k, "") + else: + setattr(self, k, v) + + have_metadata_db = bool(self.config_calibre_dir) + if have_metadata_db: + db_file = os.path.join(self.config_calibre_dir, 'metadata.db') + have_metadata_db = os.path.isfile(db_file) + self.db_configured = have_metadata_db + + from . import cli_param + if os.environ.get('FLASK_DEBUG'): + logfile = logger.setup(logger.LOG_TO_STDOUT, logger.logging.DEBUG) + else: + # pylint: disable=access-member-before-definition + logfile = logger.setup(cli_param.logpath or self.config_logfile, self.config_log_level) + if logfile != os.path.abspath(self.config_logfile): + if logfile != os.path.abspath(cli_param.logpath): + log.warning("Log path %s not valid, falling back to default", self.config_logfile) + self.config_logfile = logfile + s.config_logfile = logfile + self._session.merge(s) + try: + self._session.commit() + except OperationalError as e: + log.error('Database error: %s', e) + self._session.rollback() + self.__dict__["dirty"] = list() + + def save(self): + """Apply all configuration values to the underlying storage.""" + s = self._read_from_storage() # type: _Settings + + for k in self.dirty: + if k[0] == '_': + continue + if hasattr(s, k): + if k.endswith("_e"): + setattr(s, k, self._fernet.encrypt(self.__dict__[k].encode())) + else: + setattr(s, k, self.__dict__[k]) + + log.debug("_ConfigSQL updating storage") + self._session.merge(s) + try: + self._session.commit() + except OperationalError as e: + log.error('Database error: %s', e) + self._session.rollback() + self.load() + + def invalidate(self, error=None): + if error: + log.error(error) + log.warning("invalidating configuration") + self.db_configured = False + self.save() + + def get_book_path(self): + return self.config_calibre_split_dir if self.config_calibre_split_dir else self.config_calibre_dir + + def store_calibre_uuid(self, calibre_db, Library_table): + from . import app + try: + with app.app_context(): + calibre_uuid = calibre_db.session.query(Library_table).one_or_none() + if self.config_calibre_uuid != calibre_uuid.uuid: + self.config_calibre_uuid = calibre_uuid.uuid + self.save() + except AttributeError: + pass + + def __setattr__(self, attr_name, attr_value): + super().__setattr__(attr_name, attr_value) + self.__dict__["dirty"].append(attr_name) + + +def _encrypt_fields(session, secret_key): + try: + session.query(exists().where(_Settings.mail_password_e)).scalar() + except OperationalError: + with session.bind.connect() as conn: + conn.execute(text("ALTER TABLE settings ADD column 'mail_password_e' String")) + conn.execute(text("ALTER TABLE settings ADD column 'config_ldap_serv_password_e' String")) + session.commit() + crypter = Fernet(secret_key) + settings = session.query(_Settings.mail_password, _Settings.config_ldap_serv_password).first() + if settings.mail_password: + session.query(_Settings).update( + {_Settings.mail_password_e: crypter.encrypt(settings.mail_password.encode())}) + if settings.config_ldap_serv_password: + session.query(_Settings).update( + {_Settings.config_ldap_serv_password_e: crypter.encrypt(settings.config_ldap_serv_password.encode())}) + session.commit() + + +def _migrate_table(session, orm_class, secret_key=None): + if secret_key: + _encrypt_fields(session, secret_key) + changed = False + + for column_name, column in orm_class.__dict__.items(): + if column_name[0] != '_': + try: + session.query(column).first() + except OperationalError as err: + log.debug("%s: %s", column_name, err.args[0]) + if column.default is None: + column_default = "" + else: + if isinstance(column.default.arg, bool): + column_default = "DEFAULT {}".format(int(column.default.arg)) + else: + column_default = "DEFAULT `{}`".format(column.default.arg) + if isinstance(column.type, JSON): + column_type = "JSON" + else: + column_type = column.type + alter_table = text("ALTER TABLE %s ADD COLUMN `%s` %s %s" % (orm_class.__tablename__, + column_name, + column_type, + column_default)) + log.debug(alter_table) + session.execute(alter_table) + changed = True + except json.decoder.JSONDecodeError as e: + log.error("Database corrupt column: {}".format(column_name)) + log.debug(e) + + if changed: + try: + session.commit() + except OperationalError: + session.rollback() + + +def autodetect_calibre_binaries(): + if sys.platform == "win32": + calibre_path = ["C:\\program files\\calibre\\", + "C:\\program files(x86)\\calibre\\", + "C:\\program files(x86)\\calibre2\\", + "C:\\program files\\calibre2\\"] + elif sys.platform.startswith("freebsd"): + calibre_path = ["/usr/local/bin/"] + else: + calibre_path = ["/opt/calibre/"] + for element in calibre_path: + supported_binary_paths = [os.path.join(element, binary) + for binary in constants.SUPPORTED_CALIBRE_BINARIES.values()] + if all(os.path.isfile(binary_path) and os.access(binary_path, os.X_OK) + for binary_path in supported_binary_paths): + values = [process_wait([binary_path, "--version"], + pattern=r'\(calibre (.*)\)') for binary_path in supported_binary_paths] + if all(values): + version = values[0].group(1) + log.debug("calibre version %s", version) + return element + return "" + + +def autodetect_converter_binary(calibre_path): + if sys.platform == "win32": + converter_path = os.path.join(calibre_path, "ebook-convert.exe") + else: + converter_path = os.path.join(calibre_path, "ebook-convert") + if calibre_path and os.path.isfile(converter_path) and os.access(converter_path, os.X_OK): + return converter_path + return "" + + +def autodetect_unrar_binary(): + if sys.platform == "win32": + calibre_path = ["C:\\program files\\WinRar\\unRAR.exe", + "C:\\program files(x86)\\WinRar\\unRAR.exe"] + elif sys.platform.startswith("freebsd"): + calibre_path = ["/usr/local/bin/unrar"] + else: + calibre_path = ["/usr/bin/unrar"] + for element in calibre_path: + if os.path.isfile(element) and os.access(element, os.X_OK): + return element + return "" + + +def autodetect_kepubify_binary(): + if sys.platform == "win32": + calibre_path = ["C:\\program files\\kepubify\\kepubify-windows-64Bit.exe", + "C:\\program files(x86)\\kepubify\\kepubify-windows-64Bit.exe"] + elif sys.platform.startswith("freebsd"): + calibre_path = ["/usr/local/bin/kepubify"] + else: + calibre_path = ["/opt/kepubify/kepubify-linux-64bit", "/opt/kepubify/kepubify-linux-32bit"] + for element in calibre_path: + if os.path.isfile(element) and os.access(element, os.X_OK): + return element + return "" + + +def _migrate_database(session, secret_key): + # make sure the table is created, if it does not exist + _Base.metadata.create_all(session.bind) + _migrate_table(session, _Settings, secret_key) + _migrate_table(session, _Flask_Settings) + + +def load_configuration(session, secret_key): + _migrate_database(session, secret_key) + if not session.query(_Settings).count(): + session.add(_Settings()) + session.commit() + + +def get_flask_session_key(_session): + flask_settings = _session.query(_Flask_Settings).one_or_none() + if flask_settings is None: + flask_settings = _Flask_Settings(os.urandom(32)) + _session.add(flask_settings) + _session.commit() + return flask_settings.flask_session_key + + +def get_encryption_key(key_path): + key_file = os.path.join(key_path, ".key") + generate = True + error = "" + key = None + if os.path.exists(key_file) and os.path.getsize(key_file) > 32: + with open(key_file, "rb") as f: + key = f.read() + try: + urlsafe_b64decode(key) + generate = False + except ValueError: + pass + if generate: + key = Fernet.generate_key() + try: + with open(key_file, "wb") as f: + f.write(key) + except PermissionError as e: + error = e + return key, error diff --git a/root/app/calibre-web/cps/db.py b/root/app/calibre-web/cps/db.py new file mode 100644 index 0000000..f379ceb --- /dev/null +++ b/root/app/calibre-web/cps/db.py @@ -0,0 +1,1101 @@ +# -*- coding: utf-8 -*- + +# This file is part of the Calibre-Web (https://github.com/janeczku/calibre-web) +# Copyright (C) 2012-2019 mutschler, cervinko, ok11, jkrehm, nanu-c, Wineliva, +# pjeby, elelay, idalin, Ozzieisaacs +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import os +import re +import json +from datetime import datetime, timezone +from urllib.parse import quote +import unidecode +# from weakref import WeakSet +from uuid import uuid4 + +from sqlite3 import OperationalError as sqliteOperationalError +from sqlalchemy import create_engine +from sqlalchemy import Table, Column, ForeignKey, CheckConstraint +from sqlalchemy import String, Integer, Boolean, TIMESTAMP, Float +from sqlalchemy.orm import relationship, sessionmaker, scoped_session +from sqlalchemy.orm.collections import InstrumentedList +from sqlalchemy.ext.declarative import DeclarativeMeta +from sqlalchemy.exc import OperationalError +try: + # Compatibility with sqlalchemy 2.0 + from sqlalchemy.orm import declarative_base +except ImportError: + from sqlalchemy.ext.declarative import declarative_base +from sqlalchemy.pool import StaticPool +from sqlalchemy.sql.expression import and_, true, false, text, func, or_ +from sqlalchemy.ext.associationproxy import association_proxy +from .cw_login import current_user +from flask_babel import gettext as _ +from flask_babel import get_locale +from flask import flash, g, Flask + +from . import logger, ub, isoLanguages +from .pagination import Pagination +from .string_helper import strip_whitespaces + +log = logger.create() + +cc_exceptions = ['composite', 'series'] +cc_classes = {} + +Base = declarative_base() + +books_authors_link = Table('books_authors_link', Base.metadata, + Column('book', Integer, ForeignKey('books.id'), primary_key=True), + Column('author', Integer, ForeignKey('authors.id'), primary_key=True) + ) + +books_tags_link = Table('books_tags_link', Base.metadata, + Column('book', Integer, ForeignKey('books.id'), primary_key=True), + Column('tag', Integer, ForeignKey('tags.id'), primary_key=True) + ) + +books_series_link = Table('books_series_link', Base.metadata, + Column('book', Integer, ForeignKey('books.id'), primary_key=True), + Column('series', Integer, ForeignKey('series.id'), primary_key=True) + ) + +books_ratings_link = Table('books_ratings_link', Base.metadata, + Column('book', Integer, ForeignKey('books.id'), primary_key=True), + Column('rating', Integer, ForeignKey('ratings.id'), primary_key=True) + ) + +books_languages_link = Table('books_languages_link', Base.metadata, + Column('book', Integer, ForeignKey('books.id'), primary_key=True), + Column('lang_code', Integer, ForeignKey('languages.id'), primary_key=True) + ) + +books_publishers_link = Table('books_publishers_link', Base.metadata, + Column('book', Integer, ForeignKey('books.id'), primary_key=True), + Column('publisher', Integer, ForeignKey('publishers.id'), primary_key=True) + ) + + +class Library_Id(Base): + __tablename__ = 'library_id' + id = Column(Integer, primary_key=True) + uuid = Column(String, nullable=False) + + +class Identifiers(Base): + __tablename__ = 'identifiers' + + id = Column(Integer, primary_key=True) + type = Column(String(collation='NOCASE'), nullable=False, default="isbn") + val = Column(String(collation='NOCASE'), nullable=False) + book = Column(Integer, ForeignKey('books.id'), nullable=False) + amazon = { + "jp": "co.jp", + "uk": "co.uk", + "us": "com", + "au": "com.au", + "be": "com.be", + "br": "com.br", + "tr": "com.tr", + "mx": "com.mx", + } + + def __init__(self, val, id_type, book): + super().__init__() + self.val = val + self.type = id_type + self.book = book + + def format_type(self): + format_type = self.type.lower() + if format_type == 'amazon': + return "Amazon" + elif format_type.startswith("amazon_"): + label_amazon = "Amazon.{0}" + country_code = format_type[7:].lower() + if country_code not in self.amazon: + return label_amazon.format(country_code) + return label_amazon.format(self.amazon[country_code]) + elif format_type == "isbn": + return "ISBN" + elif format_type == "doi": + return "DOI" + elif format_type == "douban": + return "Douban" + elif format_type == "goodreads": + return "Goodreads" + elif format_type == "babelio": + return "Babelio" + elif format_type == "google": + return "Google Books" + elif format_type == "kobo": + return "Kobo" + elif format_type == "barnesnoble": + return "Barnes & Noble" + elif format_type == "litres": + return "ЛитРес" + elif format_type == "issn": + return "ISSN" + elif format_type == "isfdb": + return "ISFDB" + if format_type == "lubimyczytac": + return "Lubimyczytac" + if format_type == "databazeknih": + return "Databáze knih" + if format_type == "hardcover-slug": + return "Hardcover" + else: + return self.type + + def __repr__(self): + format_type = self.type.lower() + if format_type == "amazon" or format_type == "asin": + return "https://amazon.com/dp/{0}".format(self.val) + elif format_type.startswith('amazon_'): + link_amazon = "https://amazon.{0}/dp/{1}" + country_code = format_type[7:].lower() + if country_code not in self.amazon: + return link_amazon.format(country_code, self.val) + return link_amazon.format(self.amazon[country_code], self.val) + elif format_type == "isbn": + return "https://www.worldcat.org/isbn/{0}".format(self.val) + elif format_type == "doi": + return "https://dx.doi.org/{0}".format(self.val) + elif format_type == "goodreads": + return "https://www.goodreads.com/book/show/{0}".format(self.val) + elif format_type == "babelio": + return "https://www.babelio.com/livres/titre/{0}".format(self.val) + elif format_type == "douban": + return "https://book.douban.com/subject/{0}".format(self.val) + elif format_type == "google": + return "https://books.google.com/books?id={0}".format(self.val) + elif format_type == "kobo": + return "https://www.kobo.com/ebook/{0}".format(self.val) + elif format_type == "barnesnoble": + return "https://www.barnesandnoble.com/w/{0}".format(self.val) + elif format_type == "lubimyczytac": + return "https://lubimyczytac.pl/ksiazka/{0}/ksiazka".format(self.val) + elif format_type == "litres": + return "https://www.litres.ru/{0}".format(self.val) + elif format_type == "issn": + return "https://portal.issn.org/resource/ISSN/{0}".format(self.val) + elif format_type == "isfdb": + return "http://www.isfdb.org/cgi-bin/pl.cgi?{0}".format(self.val) + elif format_type == "databazeknih": + return "https://www.databazeknih.cz/knihy/{0}".format(self.val) + elif format_type == "hardcover-slug": + return "https://hardcover.app/books/{0}".format(self.val) + elif self.val.lower().startswith("javascript:"): + return quote(self.val) + elif self.val.lower().startswith("data:"): + link, __, __ = str.partition(self.val, ",") + return link + else: + return "{0}".format(self.val) + + +class Comments(Base): + __tablename__ = 'comments' + + id = Column(Integer, primary_key=True) + book = Column(Integer, ForeignKey('books.id'), nullable=False, unique=True) + text = Column(String(collation='NOCASE'), nullable=False) + + def __init__(self, comment, book): + super().__init__() + self.text = comment + self.book = book + + def get(self): + return self.text + + def __repr__(self): + return "".format(self.text) + + +class Tags(Base): + __tablename__ = 'tags' + + id = Column(Integer, primary_key=True, autoincrement=True) + name = Column(String(collation='NOCASE'), unique=True, nullable=False) + + def __init__(self, name): + super().__init__() + self.name = name + + def get(self): + return self.name + + def __eq__(self, other): + return self.name == other + + def __repr__(self): + return "".format(self.name) + + +class Authors(Base): + __tablename__ = 'authors' + + id = Column(Integer, primary_key=True) + name = Column(String(collation='NOCASE'), unique=True, nullable=False) + sort = Column(String(collation='NOCASE')) + link = Column(String, nullable=False, default="") + + def __init__(self, name, sort, link=""): + super().__init__() + self.name = name + self.sort = sort + self.link = link + + def get(self): + return self.name + + def __eq__(self, other): + return self.name == other + + def __repr__(self): + return "".format(self.name, self.sort, self.link) + + +class Series(Base): + __tablename__ = 'series' + + id = Column(Integer, primary_key=True) + name = Column(String(collation='NOCASE'), unique=True, nullable=False) + sort = Column(String(collation='NOCASE')) + + def __init__(self, name, sort): + super().__init__() + self.name = name + self.sort = sort + + def get(self): + return self.name + + def __eq__(self, other): + return self.name == other + + def __repr__(self): + return "".format(self.name, self.sort) + + +class Ratings(Base): + __tablename__ = 'ratings' + + id = Column(Integer, primary_key=True) + rating = Column(Integer, CheckConstraint('rating>-1 AND rating<11'), unique=True) + + def __init__(self, rating): + super().__init__() + self.rating = rating + + def get(self): + return self.rating + + def __eq__(self, other): + return self.rating == other + + def __repr__(self): + return "".format(self.rating) + + +class Languages(Base): + __tablename__ = 'languages' + + id = Column(Integer, primary_key=True) + lang_code = Column(String(collation='NOCASE'), nullable=False, unique=True) + + def __init__(self, lang_code): + super().__init__() + self.lang_code = lang_code + + def get(self): + if hasattr(self, "language_name"): + return self.language_name + else: + return self.lang_code + + def __eq__(self, other): + return self.lang_code == other + + def __repr__(self): + return "".format(self.lang_code) + + +class Publishers(Base): + __tablename__ = 'publishers' + + id = Column(Integer, primary_key=True) + name = Column(String(collation='NOCASE'), nullable=False, unique=True) + sort = Column(String(collation='NOCASE')) + + def __init__(self, name, sort): + super().__init__() + self.name = name + self.sort = sort + + def get(self): + return self.name + + def __eq__(self, other): + return self.name == other + + def __repr__(self): + return "".format(self.name, self.sort) + + +class Data(Base): + __tablename__ = 'data' + __table_args__ = {'schema': 'calibre'} + + id = Column(Integer, primary_key=True) + book = Column(Integer, ForeignKey('books.id'), nullable=False) + format = Column(String(collation='NOCASE'), nullable=False) + uncompressed_size = Column(Integer, nullable=False) + name = Column(String, nullable=False) + + def __init__(self, book, book_format, uncompressed_size, name): + super().__init__() + self.book = book + self.format = book_format + self.uncompressed_size = uncompressed_size + self.name = name + + # ToDo: Check + def get(self): + return self.name + + def __repr__(self): + return "".format(self.book, self.format, self.uncompressed_size, self.name) + + +class Metadata_Dirtied(Base): + __tablename__ = 'metadata_dirtied' + id = Column(Integer, primary_key=True, autoincrement=True) + book = Column(Integer, ForeignKey('books.id'), nullable=False, unique=True) + + def __init__(self, book): + super().__init__() + self.book = book + + +class Books(Base): + __tablename__ = 'books' + + DEFAULT_PUBDATE = datetime(101, 1, 1, 0, 0, 0, 0) # ("0101-01-01 00:00:00+00:00") + + id = Column(Integer, primary_key=True, autoincrement=True) + title = Column(String(collation='NOCASE'), nullable=False, default='Unknown') + sort = Column(String(collation='NOCASE')) + author_sort = Column(String(collation='NOCASE')) + timestamp = Column(TIMESTAMP, default=lambda: datetime.now(timezone.utc)) + pubdate = Column(TIMESTAMP, default=DEFAULT_PUBDATE) + series_index = Column(String, nullable=False, default="1.0") + last_modified = Column(TIMESTAMP, default=lambda: datetime.now(timezone.utc)) + path = Column(String, default="", nullable=False) + has_cover = Column(Integer, default=0) + uuid = Column(String) + isbn = Column(String(collation='NOCASE'), default="") + flags = Column(Integer, nullable=False, default=1) + + authors = relationship(Authors, secondary=books_authors_link, backref='books') + tags = relationship(Tags, secondary=books_tags_link, backref='books', order_by="Tags.name") + comments = relationship(Comments, backref='books') + data = relationship(Data, backref='books') + series = relationship(Series, secondary=books_series_link, backref='books') + ratings = relationship(Ratings, secondary=books_ratings_link, backref='books') + languages = relationship(Languages, secondary=books_languages_link, backref='books') + publishers = relationship(Publishers, secondary=books_publishers_link, backref='books') + identifiers = relationship(Identifiers, backref='books') + + def __init__(self, title, sort, author_sort, timestamp, pubdate, series_index, last_modified, path, has_cover, + authors, tags, languages=None): + super().__init__() + self.title = title + self.sort = sort + self.author_sort = author_sort + self.timestamp = timestamp + self.pubdate = pubdate + self.series_index = series_index + self.last_modified = last_modified + self.path = path + self.has_cover = (has_cover is not None) + + def __repr__(self): + return "".format(self.title, self.sort, self.author_sort, + self.timestamp, self.pubdate, self.series_index, + self.last_modified, self.path, self.has_cover) + + @property + def atom_timestamp(self): + return self.timestamp.strftime('%Y-%m-%dT%H:%M:%S+00:00') or '' + + +class CustomColumns(Base): + __tablename__ = 'custom_columns' + + id = Column(Integer, primary_key=True) + label = Column(String) + name = Column(String) + datatype = Column(String) + mark_for_delete = Column(Boolean) + editable = Column(Boolean) + display = Column(String) + is_multiple = Column(Boolean) + normalized = Column(Boolean) + + def get_display_dict(self): + display_dict = json.loads(self.display) + return display_dict + + def to_json(self, value, extra, sequence): + content = dict() + content['table'] = "custom_column_" + str(self.id) + content['column'] = "value" + content['datatype'] = self.datatype + content['is_multiple'] = None if not self.is_multiple else "|" + content['kind'] = "field" + content['name'] = self.name + content['search_terms'] = ['#' + self.label] + content['label'] = self.label + content['colnum'] = self.id + content['display'] = self.get_display_dict() + content['is_custom'] = True + content['is_category'] = self.datatype in ['text', 'rating', 'enumeration', 'series'] + content['link_column'] = "value" + content['category_sort'] = "value" + content['is_csp'] = False + content['is_editable'] = self.editable + content['rec_index'] = sequence + 22 # toDo why ?? + if isinstance(value, datetime): + content['#value#'] = {"__class__": "datetime.datetime", + "__value__": value.strftime("%Y-%m-%dT%H:%M:%S+00:00")} + else: + content['#value#'] = value + content['#extra#'] = extra + content['is_multiple2'] = {} if not self.is_multiple else {"cache_to_list": "|", "ui_to_list": ",", + "list_to_ui": ", "} + return json.dumps(content, ensure_ascii=False) + + +class AlchemyEncoder(json.JSONEncoder): + + def default(self, o): + if isinstance(o.__class__, DeclarativeMeta): + # an SQLAlchemy class + fields = {} + for field in [x for x in dir(o) if not x.startswith('_') and x != 'metadata' and x != "password"]: + if field == 'books': + continue + data = o.__getattribute__(field) + try: + if isinstance(data, str): + data = data.replace("'", "\'") + elif isinstance(data, InstrumentedList): + el = list() + # ele = None + for ele in data: + if hasattr(ele, 'value'): # converter for custom_column values + el.append(str(ele.value)) + elif ele.get: + el.append(ele.get()) + else: + el.append(json.dumps(ele, cls=AlchemyEncoder)) + if field == 'authors': + data = " & ".join(el) + else: + data = ",".join(el) + if data == '[]': + data = "" + else: + json.dumps(data) + fields[field] = data + except Exception: + fields[field] = "" + # a json-encodable dict + return fields + + return json.JSONEncoder.default(self, o) + + +class CalibreDB: + config = None + config_calibre_dir = None + app_db_path = None + + def __init__(self, _app: Flask=None): # , expire_on_commit=True, init=False): + """ Initialize a new CalibreDB session + """ + self.Session = None + #if init: + # self.init_db(expire_on_commit) + if _app is not None and not _app._got_first_request: + self.init_app(_app) + + def init_app(self, _app): + _app.teardown_appcontext(self.teardown) + + @classmethod + def setup_db_cc_classes(cls, cc): + global cc_classes + cc_ids = [] + books_custom_column_links = {} + for row in cc: + if row.datatype not in cc_exceptions: + if row.datatype == 'series': + dicttable = {'__tablename__': 'books_custom_column_' + str(row.id) + '_link', + 'id': Column(Integer, primary_key=True), + 'book': Column(Integer, ForeignKey('books.id'), + primary_key=True), + 'map_value': Column('value', Integer, + ForeignKey('custom_column_' + + str(row.id) + '.id'), + primary_key=True), + 'extra': Column(Float), + 'asoc': relationship('custom_column_' + str(row.id), uselist=False), + 'value': association_proxy('asoc', 'value') + } + books_custom_column_links[row.id] = type(str('books_custom_column_' + str(row.id) + '_link'), + (Base,), dicttable) + if row.datatype in ['rating', 'text', 'enumeration']: + books_custom_column_links[row.id] = Table('books_custom_column_' + str(row.id) + '_link', + Base.metadata, + Column('book', Integer, ForeignKey('books.id'), + primary_key=True), + Column('value', Integer, + ForeignKey('custom_column_' + + str(row.id) + '.id'), + primary_key=True) + ) + cc_ids.append([row.id, row.datatype]) + + ccdict = {'__tablename__': 'custom_column_' + str(row.id), + 'id': Column(Integer, primary_key=True)} + if row.datatype == 'float': + ccdict['value'] = Column(Float) + elif row.datatype == 'int': + ccdict['value'] = Column(Integer) + elif row.datatype == 'datetime': + ccdict['value'] = Column(TIMESTAMP) + elif row.datatype == 'bool': + ccdict['value'] = Column(Boolean) + else: + ccdict['value'] = Column(String) + if row.datatype in ['float', 'int', 'bool', 'datetime', 'comments']: + ccdict['book'] = Column(Integer, ForeignKey('books.id')) + cc_classes[row.id] = type(str('custom_column_' + str(row.id)), (Base,), ccdict) + + for cc_id in cc_ids: + if cc_id[1] in ['bool', 'int', 'float', 'datetime', 'comments']: + setattr(Books, + 'custom_column_' + str(cc_id[0]), + relationship(cc_classes[cc_id[0]], + primaryjoin=( + Books.id == cc_classes[cc_id[0]].book), + backref='books')) + elif cc_id[1] == 'series': + setattr(Books, + 'custom_column_' + str(cc_id[0]), + relationship(books_custom_column_links[cc_id[0]], + backref='books')) + else: + setattr(Books, + 'custom_column_' + str(cc_id[0]), + relationship(cc_classes[cc_id[0]], + secondary=books_custom_column_links[cc_id[0]], + backref='books')) + + @classmethod + def check_valid_db(cls, config_calibre_dir, app_db_path, config_calibre_uuid): + if not config_calibre_dir: + return False, False + dbpath = os.path.join(config_calibre_dir, "metadata.db") + if not os.path.exists(dbpath): + return False, False + try: + check_engine = create_engine('sqlite://', + echo=False, + isolation_level="SERIALIZABLE", + connect_args={'check_same_thread': False}, + poolclass=StaticPool) + with check_engine.begin() as connection: + connection.execute(text("attach database '{}' as calibre;".format(dbpath))) + connection.execute(text("attach database '{}' as app_settings;".format(app_db_path))) + local_session = scoped_session(sessionmaker()) + local_session.configure(bind=connection) + database_uuid = local_session().query(Library_Id).one_or_none() + + check_engine.connect() + db_change = config_calibre_uuid != database_uuid.uuid + except Exception: + return False, False + return True, db_change + + def teardown(self, exception): + ctx = g.get("lib_sql") + if ctx: + ctx.close() + + @property + def session(self): + # connect or get active connection + if not g.get("lib_sql"): + g.lib_sql = self.connect() + return g.lib_sql + + @classmethod + def update_config(cls, config, config_calibre_dir, app_db_path): + cls.config = config + cls.config_calibre_dir = config_calibre_dir + cls.app_db_path = app_db_path + + + def connect(self): + return self.setup_db(self.config_calibre_dir, self.app_db_path) + + @classmethod + def setup_db(cls, config_calibre_dir, app_db_path): + + if not config_calibre_dir: + cls.config.invalidate() + return None + + dbpath = os.path.join(config_calibre_dir, "metadata.db") + if not os.path.exists(dbpath): + cls.config.invalidate() + return None + + try: + engine = create_engine('sqlite://', + echo=False, + isolation_level="SERIALIZABLE", + connect_args={'check_same_thread': False}, + poolclass=StaticPool) + with engine.begin() as connection: + connection.execute(text('PRAGMA cache_size = 10000;')) + connection.execute(text("attach database '{}' as calibre;".format(dbpath))) + connection.execute(text("attach database '{}' as app_settings;".format(app_db_path))) + + conn = engine.connect() + # conn.text_factory = lambda b: b.decode(errors = 'ignore') possible fix for #1302 + except Exception as ex: + cls.config.invalidate(ex) + return None + + cls.config.db_configured = True + + if not cc_classes: + try: + cc = conn.execute(text("SELECT id, datatype FROM custom_columns")) + cls.setup_db_cc_classes(cc) + except OperationalError as e: + log.error_or_exception(e) + return None + + return scoped_session(sessionmaker(autocommit=False, + autoflush=False, + bind=engine, future=True)) + + + def get_book(self, book_id): + return self.session.query(Books).filter(Books.id == book_id).first() + + def get_filtered_book(self, book_id, allow_show_archived=False): + return self.session.query(Books).filter(Books.id == book_id). \ + filter(self.common_filters(allow_show_archived)).first() + + def get_book_read_archived(self, book_id, read_column, allow_show_archived=False): + if not read_column: + bd = (self.session.query(Books, ub.ReadBook.read_status, ub.ArchivedBook.is_archived).select_from(Books) + .join(ub.ReadBook, and_(ub.ReadBook.user_id == int(current_user.id), ub.ReadBook.book_id == book_id), + isouter=True)) + else: + try: + read_column = cc_classes[read_column] + bd = (self.session.query(Books, read_column.value, ub.ArchivedBook.is_archived).select_from(Books) + .join(read_column, read_column.book == book_id, + isouter=True)) + except (KeyError, AttributeError, IndexError): + log.error("Custom Column No.{} does not exist in calibre database".format(read_column)) + # Skip linking read column and return None instead of read status + bd = self.session.query(Books, None, ub.ArchivedBook.is_archived) + return (bd.filter(Books.id == book_id) + .join(ub.ArchivedBook, and_(Books.id == ub.ArchivedBook.book_id, + int(current_user.id) == ub.ArchivedBook.user_id), isouter=True) + .filter(self.common_filters(allow_show_archived)).first()) + + def get_book_by_uuid(self, book_uuid): + return self.session.query(Books).filter(Books.uuid == book_uuid).first() + + def get_book_format(self, book_id, file_format): + return self.session.query(Data).filter(Data.book == book_id).filter(Data.format == file_format).first() + + def set_metadata_dirty(self, book_id): + if not self.session.query(Metadata_Dirtied).filter(Metadata_Dirtied.book == book_id).one_or_none(): + self.session.add(Metadata_Dirtied(book_id)) + + def delete_dirty_metadata(self, book_id): + try: + self.session.query(Metadata_Dirtied).filter(Metadata_Dirtied.book == book_id).delete() + self.session.commit() + except (OperationalError) as e: + self.session.rollback() + log.error("Database error: {}".format(e)) + + # Language and content filters for displaying in the UI + def common_filters(self, allow_show_archived=False, return_all_languages=False): + if not allow_show_archived: + archived_books = (ub.session.query(ub.ArchivedBook) + .filter(ub.ArchivedBook.user_id==int(current_user.id)) + .filter(ub.ArchivedBook.is_archived==True) + .all()) + archived_book_ids = [archived_book.book_id for archived_book in archived_books] + archived_filter = Books.id.notin_(archived_book_ids) + else: + archived_filter = true() + + if current_user.filter_language() == "all" or return_all_languages: + lang_filter = true() + else: + lang_filter = Books.languages.any(Languages.lang_code == current_user.filter_language()) + negtags_list = current_user.list_denied_tags() + postags_list = current_user.list_allowed_tags() + neg_content_tags_filter = false() if negtags_list == [''] else Books.tags.any(Tags.name.in_(negtags_list)) + pos_content_tags_filter = true() if postags_list == [''] else Books.tags.any(Tags.name.in_(postags_list)) + if self.config.config_restricted_column: + try: + pos_cc_list = current_user.allowed_column_value.split(',') + pos_content_cc_filter = true() if pos_cc_list == [''] else \ + getattr(Books, 'custom_column_' + str(self.config.config_restricted_column)). \ + any(cc_classes[self.config.config_restricted_column].value.in_(pos_cc_list)) + neg_cc_list = current_user.denied_column_value.split(',') + neg_content_cc_filter = false() if neg_cc_list == [''] else \ + getattr(Books, 'custom_column_' + str(self.config.config_restricted_column)). \ + any(cc_classes[self.config.config_restricted_column].value.in_(neg_cc_list)) + except (KeyError, AttributeError, IndexError): + pos_content_cc_filter = false() + neg_content_cc_filter = true() + log.error("Custom Column No.{} does not exist in calibre database".format( + self.config.config_restricted_column)) + flash(_("Custom Column No.%(column)d does not exist in calibre database", + column=self.config.config_restricted_column), + category="error") + + else: + pos_content_cc_filter = true() + neg_content_cc_filter = false() + return and_(lang_filter, pos_content_tags_filter, ~neg_content_tags_filter, + pos_content_cc_filter, ~neg_content_cc_filter, archived_filter) + + def generate_linked_query(self, config_read_column, database): + if not config_read_column: + query = (self.session.query(database, ub.ArchivedBook.is_archived, ub.ReadBook.read_status) + .select_from(Books) + .outerjoin(ub.ReadBook, + and_(ub.ReadBook.user_id == int(current_user.id), ub.ReadBook.book_id == Books.id))) + else: + try: + read_column = cc_classes[config_read_column] + query = (self.session.query(database, ub.ArchivedBook.is_archived, read_column.value) + .select_from(Books) + .outerjoin(read_column, read_column.book == Books.id)) + except (KeyError, AttributeError, IndexError): + log.error("Custom Column No.{} does not exist in calibre database".format(config_read_column)) + # Skip linking read column and return None instead of read status + query = self.session.query(database, None, ub.ArchivedBook.is_archived) + return query.outerjoin(ub.ArchivedBook, and_(Books.id == ub.ArchivedBook.book_id, + int(current_user.id) == ub.ArchivedBook.user_id)) + + @staticmethod + def get_checkbox_sorted(inputlist, state, offset, limit, order, combo=False): + outcome = list() + if combo: + elementlist = {ele[0].id: ele for ele in inputlist} + else: + elementlist = {ele.id: ele for ele in inputlist} + for entry in state: + try: + outcome.append(elementlist[entry]) + except KeyError: + pass + del elementlist[entry] + for entry in elementlist: + outcome.append(elementlist[entry]) + if order == "asc": + outcome.reverse() + return outcome[offset:offset + limit] + + # Fill indexpage with all requested data from database + def fill_indexpage(self, page, pagesize, database, db_filter, order, + join_archive_read=False, config_read_column=0, *join): + return self.fill_indexpage_with_archived_books(page, database, pagesize, db_filter, order, False, + join_archive_read, config_read_column, *join) + + def fill_indexpage_with_archived_books(self, page, database, pagesize, db_filter, order, allow_show_archived, + join_archive_read, config_read_column, *join): + pagesize = pagesize or self.config.config_books_per_page + if current_user.show_detail_random(): + random_query = self.generate_linked_query(config_read_column, database) + randm = (random_query.filter(self.common_filters(allow_show_archived)) + .order_by(func.random()) + .limit(self.config.config_random_books).all()) + else: + randm = false() + if join_archive_read: + query = self.generate_linked_query(config_read_column, database) + else: + query = self.session.query(database) + off = int(int(pagesize) * (page - 1)) + + indx = len(join) + element = 0 + while indx: + if indx >= 3: + query = query.outerjoin(join[element], join[element+1]).outerjoin(join[element+2]) + indx -= 3 + element += 3 + elif indx == 2: + query = query.outerjoin(join[element], join[element+1]) + indx -= 2 + element += 2 + elif indx == 1: + query = query.outerjoin(join[element]) + indx -= 1 + element += 1 + query = query.filter(db_filter)\ + .filter(self.common_filters(allow_show_archived)) + entries = list() + pagination = list() + try: + pagination = Pagination(page, pagesize, query.count()) + entries = query.order_by(*order).offset(off).limit(pagesize).all() + except Exception as ex: + log.error_or_exception(ex) + # display authors in right order + entries = self.order_authors(entries, True, join_archive_read) + return entries, randm, pagination + + # Orders all Authors in the list according to authors sort + def order_authors(self, entries, list_return=False, combined=False): + for entry in entries: + if combined: + sort_authors = entry.Books.author_sort.split('&') + ids = [a.id for a in entry.Books.authors] + + else: + sort_authors = entry.author_sort.split('&') + ids = [a.id for a in entry.authors] + authors_ordered = list() + # error = False + for auth in sort_authors: + auth = strip_whitespaces(auth) + results = self.session.query(Authors).filter(Authors.sort == auth).all() + # ToDo: How to handle not found author name + if not len(results): + log.error("Author {} not found to display name in right order".format(auth)) + # error = True + break + for r in results: + if r.id in ids: + authors_ordered.append(r) + ids.remove(r.id) + for author_id in ids: + result = self.session.query(Authors).filter(Authors.id == author_id).first() + authors_ordered.append(result) + + if list_return: + if combined: + entry.Books.authors = authors_ordered + else: + entry.ordered_authors = authors_ordered + else: + return authors_ordered + return entries + + def get_typeahead(self, database, query, replace=('', ''), tag_filter=true()): + query = query or '' + self.create_functions() + # self.session.connection().connection.connection.create_function("lower", 1, lcase) + entries = self.session.query(database).filter(tag_filter). \ + filter(func.lower(database.name).ilike("%" + query + "%")).all() + # json_dumps = json.dumps([dict(name=escape(r.name.replace(*replace))) for r in entries]) + json_dumps = json.dumps([dict(name=r.name.replace(*replace)) for r in entries]) + return json_dumps + + def check_exists_book(self, authr, title): + self.create_functions() + # self.session.connection().connection.connection.create_function("lower", 1, lcase) + q = list() + author_terms = re.split(r'\s*&\s*', authr) + for author_term in author_terms: + q.append(Books.authors.any(func.lower(Authors.name).ilike("%" + author_term + "%"))) + + return self.session.query(Books) \ + .filter(and_(Books.authors.any(and_(*q)), func.lower(Books.title).ilike("%" + title + "%"))).first() + + def search_query(self, term, config, *join): + strip_whitespaces(term).lower() + self.create_functions() + # self.session.connection().connection.connection.create_function("lower", 1, lcase) + q = list() + author_terms = re.split("[, ]+", term) + for author_term in author_terms: + q.append(Books.authors.any(func.lower(Authors.name).ilike("%" + author_term + "%"))) + query = self.generate_linked_query(config.config_read_column, Books) + if len(join) == 6: + query = query.outerjoin(join[0], join[1]).outerjoin(join[2]).outerjoin(join[3], join[4]).outerjoin(join[5]) + if len(join) == 3: + query = query.outerjoin(join[0], join[1]).outerjoin(join[2]) + elif len(join) == 2: + query = query.outerjoin(join[0], join[1]) + elif len(join) == 1: + query = query.outerjoin(join[0]) + + cc = self.get_cc_columns(config, filter_config_custom_read=True) + filter_expression = [Books.tags.any(func.lower(Tags.name).ilike("%" + term + "%")), + Books.series.any(func.lower(Series.name).ilike("%" + term + "%")), + Books.authors.any(and_(*q)), + Books.publishers.any(func.lower(Publishers.name).ilike("%" + term + "%")), + func.lower(Books.title).ilike("%" + term + "%")] + for c in cc: + if c.datatype not in ["datetime", "rating", "bool", "int", "float"]: + filter_expression.append( + getattr(Books, + 'custom_column_' + str(c.id)).any( + func.lower(cc_classes[c.id].value).ilike("%" + term + "%"))) + return query.filter(self.common_filters(True)).filter(or_(*filter_expression)) + + def get_cc_columns(self, config, filter_config_custom_read=False): + tmp_cc = self.session.query(CustomColumns).filter(CustomColumns.datatype.notin_(cc_exceptions)).all() + cc = [] + r = None + if config.config_columns_to_ignore: + r = re.compile(config.config_columns_to_ignore) + + for col in tmp_cc: + if filter_config_custom_read and config.config_read_column and config.config_read_column == col.id: + continue + if r and r.match(col.name): + continue + cc.append(col) + + return cc + + # read search results from calibre-database and return it (function is used for feed and simple search + def get_search_results(self, term, config, offset=None, order=None, limit=None, *join): + order = order[0] if order else [Books.sort] + pagination = None + result = self.search_query(term, config, *join).order_by(*order).all() + result_count = len(result) + if offset is not None and limit is not None: + offset = int(offset) + limit_all = offset + int(limit) + pagination = Pagination((offset / (int(limit)) + 1), limit, result_count) + else: + offset = 0 + limit_all = result_count + + ub.store_combo_ids(result) + entries = self.order_authors(result[offset:limit_all], list_return=True, combined=True) + + return entries, result_count, pagination + + # Creates for all stored languages a translated speaking name in the array for the UI + def speaking_language(self, languages=None, return_all_languages=False, with_count=False, reverse_order=False): + + if with_count: + if not languages: + languages = self.session.query(Languages, func.count('books_languages_link.book'))\ + .join(books_languages_link).join(Books)\ + .filter(self.common_filters(return_all_languages=return_all_languages)) \ + .group_by(text('books_languages_link.lang_code')).all() + tags = list() + for lang in languages: + tag = Category(isoLanguages.get_language_name(get_locale(), lang[0].lang_code), lang[0].lang_code) + tags.append([tag, lang[1]]) + # Append all books without language to list + if not return_all_languages: + no_lang_count = (self.session.query(Books) + .outerjoin(books_languages_link).outerjoin(Languages) + .filter(Languages.lang_code==None) + .filter(self.common_filters()) + .count()) + if no_lang_count: + tags.append([Category(_("None"), "none"), no_lang_count]) + return sorted(tags, key=lambda x: x[0].name.lower(), reverse=reverse_order) + else: + if not languages: + languages = self.session.query(Languages) \ + .join(books_languages_link) \ + .join(Books) \ + .filter(self.common_filters(return_all_languages=return_all_languages)) \ + .group_by(text('books_languages_link.lang_code')).all() + for lang in languages: + lang.name = isoLanguages.get_language_name(get_locale(), lang.lang_code) + return sorted(languages, key=lambda x: x.name, reverse=reverse_order) + + def create_functions(self, config=None): + # user defined sort function for calibre databases (Series, etc.) + def _title_sort(title): + # calibre sort stuff + title_pat = re.compile(config.config_title_regex, re.IGNORECASE) + match = title_pat.search(title) + if match: + prep = match.group(1) + title = title[len(prep):] + ', ' + prep + return strip_whitespaces(title) + + try: + # sqlalchemy <1.4.24 and sqlalchemy 2.0 + conn = self.session.connection().connection.driver_connection + except AttributeError: + # sqlalchemy >1.4.24 + conn = self.session.connection().connection.connection + try: + if config: + conn.create_function("title_sort", 1, _title_sort) + conn.create_function('uuid4', 0, lambda: str(uuid4())) + conn.create_function("lower", 1, lcase) + except sqliteOperationalError: + pass + + def reconnect_db(self, config, app_db_path): + # self.dispose() + # self.engine.dispose() + self.setup_db(config.config_calibre_dir, app_db_path) + self.update_config(config, config.config_calibre_dir, app_db_path) + + +def lcase(s): + try: + return unidecode.unidecode(s.lower()) + except Exception as ex: + _log = logger.create() + _log.error_or_exception(ex) + return s.lower() + + +class Category: + name = None + id = None + count = None + rating = None + + def __init__(self, name, cat_id, rating=None): + self.name = name + self.id = cat_id + self.rating = rating + self.count = 1 diff --git a/root/app/calibre-web/cps/kobo.py b/root/app/calibre-web/cps/kobo.py new file mode 100644 index 0000000..ff188ad --- /dev/null +++ b/root/app/calibre-web/cps/kobo.py @@ -0,0 +1,1286 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +# This file is part of the Calibre-Web (https://github.com/janeczku/calibre-web) +# Copyright (C) 2018-2019 shavitmichael, OzzieIsaacs +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import base64 +from datetime import datetime, timezone +import os +import uuid +import zipfile +from time import gmtime, strftime +import json +from urllib.parse import unquote + +from flask import ( + Blueprint, + request, + make_response, + jsonify, + current_app, + url_for, + redirect, + abort +) +from .cw_login import current_user +from werkzeug.datastructures import Headers +from sqlalchemy import func +from sqlalchemy.sql.expression import and_, or_ +from sqlalchemy.exc import StatementError +from sqlalchemy.sql import select +import requests + +from . import config, logger, kobo_auth, db, calibre_db, helper, shelf as shelf_lib, ub, csrf, kobo_sync_status +from . import isoLanguages +from .epub import get_epub_layout +from .constants import COVER_THUMBNAIL_SMALL, COVER_THUMBNAIL_MEDIUM, COVER_THUMBNAIL_LARGE +from .helper import get_download_link +from .services import SyncToken as SyncToken, hardcover +from .web import download_required +from .kobo_auth import requires_kobo_auth, get_auth_token + +KOBO_FORMATS = {"KEPUB": ["KEPUB"], "EPUB": ["EPUB3", "EPUB"]} +KOBO_STOREAPI_URL = "https://storeapi.kobo.com" +KOBO_IMAGEHOST_URL = "https://cdn.kobo.com/book-images" + +SYNC_ITEM_LIMIT = 100 + +kobo = Blueprint("kobo", __name__, url_prefix="/kobo/") +kobo_auth.disable_failed_auth_redirect_for_blueprint(kobo) +kobo_auth.register_url_value_preprocessor(kobo) + +log = logger.create() + + +def get_store_url_for_current_request(): + # Programmatically modify the current url to point to the official Kobo store + __, __, request_path_with_auth_token = request.full_path.rpartition("/kobo/") + __, __, request_path = request_path_with_auth_token.rstrip("?").partition( + "/" + ) + return KOBO_STOREAPI_URL + "/" + request_path + + +CONNECTION_SPECIFIC_HEADERS = [ + "connection", + "content-encoding", + "content-length", + "transfer-encoding", +] + + +def get_kobo_activated(): + return config.config_kobo_sync + + +def make_request_to_kobo_store(sync_token=None): + outgoing_headers = Headers(request.headers) + outgoing_headers.remove("Host") + if sync_token: + sync_token.set_kobo_store_header(outgoing_headers) + + store_response = requests.request( + method=request.method, + url=get_store_url_for_current_request(), + headers=outgoing_headers, + data=request.get_data(), + allow_redirects=False, + timeout=(2, 10) + ) + log.debug("Content: " + str(store_response.content)) + log.debug("StatusCode: " + str(store_response.status_code)) + return store_response + + +def redirect_or_proxy_request(): + if config.config_kobo_proxy: + if request.method == "GET": + return redirect(get_store_url_for_current_request(), 307) + else: + # The Kobo device turns other request types into GET requests on redirects, + # so we instead proxy to the Kobo store ourselves. + store_response = make_request_to_kobo_store() + + response_headers = store_response.headers + for header_key in CONNECTION_SPECIFIC_HEADERS: + response_headers.pop(header_key, default=None) + + return make_response( + store_response.content, store_response.status_code, response_headers.items() + ) + else: + return make_response(jsonify({})) + + +def convert_to_kobo_timestamp_string(timestamp): + try: + return timestamp.strftime("%Y-%m-%dT%H:%M:%SZ") + except AttributeError as exc: + log.debug("Timestamp not valid: {}".format(exc)) + return datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + + +@kobo.route("/v1/library/sync") +@requires_kobo_auth +# @download_required +def HandleSyncRequest(): + if not current_user.role_download(): + log.info("Users need download permissions for syncing library to Kobo reader") + return abort(403) + sync_token = SyncToken.SyncToken.from_headers(request.headers) + log.info("Kobo library sync request received") + log.debug("SyncToken: {}".format(sync_token)) + log.debug("Download link format {}".format(get_download_url_for_book('[bookid]', '[bookformat]'))) + if not current_app.wsgi_app.is_proxied: + log.debug('Kobo: Received unproxied request, changed request port to external server port') + + # if no books synced don't respect sync_token + if not ub.session.query(ub.KoboSyncedBooks).filter(ub.KoboSyncedBooks.user_id == current_user.id).count(): + sync_token.books_last_modified = datetime.min + sync_token.books_last_created = datetime.min + sync_token.reading_state_last_modified = datetime.min + + new_books_last_modified = sync_token.books_last_modified # needed for sync selected shelfs only + new_books_last_created = sync_token.books_last_created # needed to distinguish between new and changed entitlement + new_reading_state_last_modified = sync_token.reading_state_last_modified + + new_archived_last_modified = datetime.min + sync_results = [] + + # We reload the book database so that the user gets a fresh view of the library + # in case of external changes (e.g: adding a book through Calibre). + calibre_db.reconnect_db(config, ub.app_DB_path) + + only_kobo_shelves = current_user.kobo_only_shelves_sync + + if only_kobo_shelves: + changed_entries = calibre_db.session.query(db.Books, + ub.ArchivedBook.last_modified, + ub.BookShelf.date_added, + ub.ArchivedBook.is_archived) + changed_entries = (changed_entries + .join(db.Data).outerjoin(ub.ArchivedBook, and_(db.Books.id == ub.ArchivedBook.book_id, + ub.ArchivedBook.user_id == current_user.id)) + .filter(db.Books.id.notin_(calibre_db.session.query(ub.KoboSyncedBooks.book_id) + .filter(ub.KoboSyncedBooks.user_id == current_user.id))) + .filter(ub.BookShelf.date_added > sync_token.books_last_modified) + .filter(db.Data.format.in_(KOBO_FORMATS)) + .filter(calibre_db.common_filters(allow_show_archived=True)) + .order_by(db.Books.id) + .order_by(ub.ArchivedBook.last_modified) + .join(ub.BookShelf, db.Books.id == ub.BookShelf.book_id) + .join(ub.Shelf) + .filter(ub.Shelf.user_id == current_user.id) + .filter(ub.Shelf.kobo_sync) + .distinct()) + else: + changed_entries = calibre_db.session.query(db.Books, + ub.ArchivedBook.last_modified, + ub.ArchivedBook.is_archived) + changed_entries = (changed_entries + .join(db.Data).outerjoin(ub.ArchivedBook, and_(db.Books.id == ub.ArchivedBook.book_id, + ub.ArchivedBook.user_id == current_user.id)) + .filter(db.Books.id.notin_(calibre_db.session.query(ub.KoboSyncedBooks.book_id) + .filter(ub.KoboSyncedBooks.user_id == current_user.id))) + .filter(calibre_db.common_filters(allow_show_archived=True)) + .filter(db.Data.format.in_(KOBO_FORMATS)) + .order_by(db.Books.last_modified) + .order_by(db.Books.id)) + + reading_states_in_new_entitlements = [] + books = changed_entries.limit(SYNC_ITEM_LIMIT) + log.debug("Books to Sync: {}".format(len(books.all()))) + for book in books: + formats = [data.format for data in book.Books.data] + if 'KEPUB' not in formats and config.config_kepubifypath and 'EPUB' in formats: + helper.convert_book_format(book.Books.id, config.get_book_path(), 'EPUB', 'KEPUB', current_user.name) + + kobo_reading_state = get_or_create_reading_state(book.Books.id) + entitlement = { + "BookEntitlement": create_book_entitlement(book.Books, archived=(book.is_archived==True)), + "BookMetadata": get_metadata(book.Books), + } + + if kobo_reading_state.last_modified > sync_token.reading_state_last_modified: + entitlement["ReadingState"] = get_kobo_reading_state_response(book.Books, kobo_reading_state) + new_reading_state_last_modified = max(new_reading_state_last_modified, kobo_reading_state.last_modified) + reading_states_in_new_entitlements.append(book.Books.id) + + ts_created = book.Books.timestamp.replace(tzinfo=None) + + try: + ts_created = max(ts_created, book.date_added) + except AttributeError: + pass + + if ts_created > sync_token.books_last_created: + sync_results.append({"NewEntitlement": entitlement}) + else: + sync_results.append({"ChangedEntitlement": entitlement}) + + new_books_last_modified = max( + book.Books.last_modified.replace(tzinfo=None), new_books_last_modified + ) + try: + new_books_last_modified = max( + new_books_last_modified, book.date_added + ) + except AttributeError: + pass + + new_books_last_created = max(ts_created, new_books_last_created) + kobo_sync_status.add_synced_books(book.Books.id) + + max_change = changed_entries.filter(ub.ArchivedBook.is_archived)\ + .filter(ub.ArchivedBook.user_id == current_user.id) \ + .order_by(func.datetime(ub.ArchivedBook.last_modified).desc()).first() + + max_change = max_change.last_modified if max_change else new_archived_last_modified + + new_archived_last_modified = max(new_archived_last_modified, max_change) + + # no. of books returned + book_count = changed_entries.count() + # last entry: + cont_sync = bool(book_count) + log.debug("Remaining books to Sync: {}".format(book_count)) + # generate reading state data + changed_reading_states = ub.session.query(ub.KoboReadingState) + + if only_kobo_shelves: + changed_reading_states = changed_reading_states.join(ub.BookShelf, + ub.KoboReadingState.book_id == ub.BookShelf.book_id)\ + .join(ub.Shelf)\ + .filter(current_user.id == ub.Shelf.user_id)\ + .filter(ub.Shelf.kobo_sync, + or_( + ub.KoboReadingState.last_modified > sync_token.reading_state_last_modified, + func.datetime(ub.BookShelf.date_added) > sync_token.books_last_modified + )).distinct() + else: + changed_reading_states = changed_reading_states.filter( + ub.KoboReadingState.last_modified > sync_token.reading_state_last_modified) + + changed_reading_states = changed_reading_states.filter( + and_(ub.KoboReadingState.user_id == current_user.id, + ub.KoboReadingState.book_id.notin_(reading_states_in_new_entitlements)))\ + .order_by(ub.KoboReadingState.last_modified) + cont_sync |= bool(changed_reading_states.count() > SYNC_ITEM_LIMIT) + for kobo_reading_state in changed_reading_states.limit(SYNC_ITEM_LIMIT).all(): + book = calibre_db.session.query(db.Books).filter(db.Books.id == kobo_reading_state.book_id).one_or_none() + if book: + sync_results.append({ + "ChangedReadingState": { + "ReadingState": get_kobo_reading_state_response(book, kobo_reading_state) + } + }) + new_reading_state_last_modified = max(new_reading_state_last_modified, kobo_reading_state.last_modified) + + sync_shelves(sync_token, sync_results, only_kobo_shelves) + + # update last created timestamp to distinguish between new and changed entitlements + if not cont_sync: + sync_token.books_last_created = new_books_last_created + sync_token.books_last_modified = new_books_last_modified + sync_token.archive_last_modified = new_archived_last_modified + sync_token.reading_state_last_modified = new_reading_state_last_modified + + return generate_sync_response(sync_token, sync_results, cont_sync) + + +def generate_sync_response(sync_token, sync_results, set_cont=False): + extra_headers = {} + if config.config_kobo_proxy and not set_cont: + # Merge in sync results from the official Kobo store. + try: + store_response = make_request_to_kobo_store(sync_token) + + store_sync_results = store_response.json() + sync_results += store_sync_results + sync_token.merge_from_store_response(store_response) + extra_headers["x-kobo-sync"] = store_response.headers.get("x-kobo-sync") + extra_headers["x-kobo-sync-mode"] = store_response.headers.get("x-kobo-sync-mode") + extra_headers["x-kobo-recent-reads"] = store_response.headers.get("x-kobo-recent-reads") + + except Exception as ex: + log.error_or_exception("Failed to receive or parse response from Kobo's sync endpoint: {}".format(ex)) + if set_cont: + extra_headers["x-kobo-sync"] = "continue" + sync_token.to_headers(extra_headers) + + # log.debug("Kobo Sync Content: {}".format(sync_results)) + # jsonify decodes the unicode string different to what kobo expects + response = make_response(json.dumps(sync_results), extra_headers) + response.headers["Content-Type"] = "application/json; charset=utf-8" + return response + + +@kobo.route("/v1/library//metadata") +@requires_kobo_auth +@download_required +def HandleMetadataRequest(book_uuid): + if not current_app.wsgi_app.is_proxied: + log.debug('Kobo: Received unproxied request, changed request port to external server port') + log.info("Kobo library metadata request received for book %s" % book_uuid) + book = calibre_db.get_book_by_uuid(book_uuid) + if not book or not book.data: + log.info("Book %s not found in database", book_uuid) + return redirect_or_proxy_request() + + metadata = get_metadata(book) + response = make_response(json.dumps([metadata], ensure_ascii=False)) + response.headers["Content-Type"] = "application/json; charset=utf-8" + return response + + +def get_download_url_for_book(book_id, book_format): + if not current_app.wsgi_app.is_proxied: + if ':' in request.host and not request.host.endswith(']'): + host = "".join(request.host.split(':')[:-1]) + else: + host = request.host + + return "{url_scheme}://{url_base}:{url_port}/kobo/{auth_token}/download/{book_id}/{book_format}".format( + url_scheme=request.scheme, + url_base=host, + url_port=config.config_external_port, + auth_token=get_auth_token(), + book_id=book_id, + book_format=book_format.lower() + ) + return url_for( + "kobo.download_book", + auth_token=kobo_auth.get_auth_token(), + book_id=book_id, + book_format=book_format.lower(), + _external=True, + ) + + +def create_book_entitlement(book, archived): + book_uuid = str(book.uuid) + return { + "Accessibility": "Full", + "ActivePeriod": {"From": convert_to_kobo_timestamp_string(datetime.now(timezone.utc))}, + "Created": convert_to_kobo_timestamp_string(book.timestamp), + "CrossRevisionId": book_uuid, + "Id": book_uuid, + "IsRemoved": archived, + "IsHiddenFromArchive": False, + "IsLocked": False, + "LastModified": convert_to_kobo_timestamp_string(book.last_modified), + "OriginCategory": "Imported", + "RevisionId": book_uuid, + "Status": "Active", + } + + +def current_time(): + return strftime("%Y-%m-%dT%H:%M:%SZ", gmtime()) + + +def get_description(book): + if not book.comments: + return None + return book.comments[0].text + + +def get_author(book): + if not book.authors: + return {"Contributors": None} + author_list = [] + autor_roles = [] + for author in book.authors: + autor_roles.append({"Name": author.name}) + author_list.append(author.name) + return {"ContributorRoles": autor_roles, "Contributors": author_list} + + +def get_publisher(book): + if not book.publishers: + return None + return book.publishers[0].name + + +def get_series(book): + if not book.series: + return None + return book.series[0].name + + +def get_seriesindex(book): + return book.series_index if isinstance(book.series_index, float) else 1 + + +def get_language(book): + if not book.languages: + return 'en' + return isoLanguages.get(part3=book.languages[0].lang_code).part1 + + +def get_metadata(book): + download_urls = [] + kepub = [data for data in book.data if data.format == 'KEPUB'] + + for book_data in kepub if len(kepub) > 0 else book.data: + if book_data.format not in KOBO_FORMATS: + continue + for kobo_format in KOBO_FORMATS[book_data.format]: + # log.debug('Id: %s, Format: %s' % (book.id, kobo_format)) + try: + if get_epub_layout(book, book_data) == 'pre-paginated': + kobo_format = 'EPUB3FL' + download_urls.append( + { + "Format": kobo_format, + "Size": book_data.uncompressed_size, + "Url": get_download_url_for_book(book.id, book_data.format), + # The Kobo forma accepts platforms: (Generic, Android) + "Platform": "Generic", + # "DrmType": "None", # Not required + } + ) + except (zipfile.BadZipfile, FileNotFoundError) as e: + log.error(e) + + book_uuid = book.uuid + metadata = { + "Categories": ["00000000-0000-0000-0000-000000000001", ], + # "Contributors": get_author(book), + "CoverImageId": book_uuid, + "CrossRevisionId": book_uuid, + "CurrentDisplayPrice": {"CurrencyCode": "USD", "TotalAmount": 0}, + "CurrentLoveDisplayPrice": {"TotalAmount": 0}, + "Description": get_description(book), + "DownloadUrls": download_urls, + "EntitlementId": book_uuid, + "ExternalIds": [], + "Genre": "00000000-0000-0000-0000-000000000001", + "IsEligibleForKoboLove": False, + "IsInternetArchive": False, + "IsPreOrder": False, + "IsSocialEnabled": True, + "Language": get_language(book), + "PhoneticPronunciations": {}, + "PublicationDate": convert_to_kobo_timestamp_string(book.pubdate), + "Publisher": {"Imprint": "", "Name": get_publisher(book), }, + "RevisionId": book_uuid, + "Title": book.title, + "WorkId": book_uuid, + } + metadata.update(get_author(book)) + + if get_series(book): + name = get_series(book) + try: + metadata["Series"] = { + "Name": get_series(book), + "Number": get_seriesindex(book), # ToDo Check int() ? + "NumberFloat": float(get_seriesindex(book)), + # Get a deterministic id based on the series name. + "Id": str(uuid.uuid3(uuid.NAMESPACE_DNS, name)), + } + except Exception as e: + print(e) + return metadata + + +@csrf.exempt +@kobo.route("/v1/library/tags", methods=["POST", "DELETE"]) +@requires_kobo_auth +# Creates a Shelf with the given items, and returns the shelf's uuid. +def HandleTagCreate(): + # catch delete requests, otherwise they are handled in the book delete handler + if request.method == "DELETE": + abort(405) + name, items = None, None + try: + shelf_request = request.json + name = shelf_request["Name"] + items = shelf_request["Items"] + if not name: + raise TypeError + except (KeyError, TypeError): + log.debug("Received malformed v1/library/tags request.") + abort(400, description="Malformed tags POST request. Data has empty 'Name', missing 'Name' or 'Items' field") + + shelf = ub.session.query(ub.Shelf).filter(ub.Shelf.name == name, ub.Shelf.user_id == + current_user.id).one_or_none() + if shelf and not shelf_lib.check_shelf_edit_permissions(shelf): + abort(401, description="User is unauthaurized to create shelf.") + + if not shelf: + shelf = ub.Shelf(user_id=current_user.id, name=name, uuid=str(uuid.uuid4())) + ub.session.add(shelf) + + items_unknown_to_calibre = add_items_to_shelf(items, shelf) + if items_unknown_to_calibre: + log.debug("Received request to add unknown books to a collection. Silently ignoring items.") + ub.session_commit() + return make_response(jsonify(str(shelf.uuid)), 201) + + +@csrf.exempt +@kobo.route("/v1/library/tags/", methods=["DELETE", "PUT"]) +@requires_kobo_auth +def HandleTagUpdate(tag_id): + shelf = ub.session.query(ub.Shelf).filter(ub.Shelf.uuid == tag_id, + ub.Shelf.user_id == current_user.id).one_or_none() + if not shelf: + log.debug("Received Kobo tag update request on a collection unknown to CalibreWeb") + if config.config_kobo_proxy: + return redirect_or_proxy_request() + else: + abort(404, description="Collection isn't known to CalibreWeb") + + if request.method == "DELETE": + if not shelf_lib.delete_shelf_helper(shelf): + abort(401, description="Error deleting Shelf") + else: + name = None + try: + shelf_request = request.json + name = shelf_request["Name"] + except (KeyError, TypeError): + log.debug("Received malformed v1/library/tags rename request.") + abort(400, description="Malformed tags POST request. Data is missing 'Name' field") + + shelf.name = name + ub.session.merge(shelf) + ub.session_commit() + return make_response(' ', 200) + + +# Adds items to the given shelf. +def add_items_to_shelf(items, shelf): + book_ids_already_in_shelf = set([book_shelf.book_id for book_shelf in shelf.books]) + items_unknown_to_calibre = [] + for item in items: + try: + if item["Type"] != "ProductRevisionTagItem": + items_unknown_to_calibre.append(item) + continue + + book = calibre_db.get_book_by_uuid(item["RevisionId"]) + if not book: + items_unknown_to_calibre.append(item) + continue + + book_id = book.id + if book_id not in book_ids_already_in_shelf: + shelf.books.append(ub.BookShelf(book_id=book_id)) + except KeyError: + items_unknown_to_calibre.append(item) + return items_unknown_to_calibre + + +@csrf.exempt +@kobo.route("/v1/library/tags//items", methods=["POST"]) +@requires_kobo_auth +def HandleTagAddItem(tag_id): + items = None + try: + tag_request = request.json + items = tag_request["Items"] + except (KeyError, TypeError): + log.debug("Received malformed v1/library/tags//items/delete request.") + abort(400, description="Malformed tags POST request. Data is missing 'Items' field") + + shelf = ub.session.query(ub.Shelf).filter(ub.Shelf.uuid == tag_id, + ub.Shelf.user_id == current_user.id).one_or_none() + if not shelf: + log.debug("Received Kobo request on a collection unknown to CalibreWeb") + abort(404, description="Collection isn't known to CalibreWeb") + + if not shelf_lib.check_shelf_edit_permissions(shelf): + abort(401, description="User is unauthaurized to edit shelf.") + + items_unknown_to_calibre = add_items_to_shelf(items, shelf) + if items_unknown_to_calibre: + log.debug("Received request to add an unknown book to a collection. Silently ignoring item.") + + ub.session.merge(shelf) + ub.session_commit() + return make_response('', 201) + + +@csrf.exempt +@kobo.route("/v1/library/tags//items/delete", methods=["POST"]) +@requires_kobo_auth +def HandleTagRemoveItem(tag_id): + items = None + try: + tag_request = request.json + items = tag_request["Items"] + except (KeyError, TypeError): + log.debug("Received malformed v1/library/tags//items/delete request.") + abort(400, description="Malformed tags POST request. Data is missing 'Items' field") + + shelf = ub.session.query(ub.Shelf).filter(ub.Shelf.uuid == tag_id, + ub.Shelf.user_id == current_user.id).one_or_none() + if not shelf: + log.debug( + "Received a request to remove an item from a Collection unknown to CalibreWeb.") + abort(404, description="Collection isn't known to CalibreWeb") + + if not shelf_lib.check_shelf_edit_permissions(shelf): + abort(401, description="User is unauthaurized to edit shelf.") + + items_unknown_to_calibre = [] + for item in items: + try: + if item["Type"] != "ProductRevisionTagItem": + items_unknown_to_calibre.append(item) + continue + + book = calibre_db.get_book_by_uuid(item["RevisionId"]) + if not book: + items_unknown_to_calibre.append(item) + continue + + shelf.books.filter(ub.BookShelf.book_id == book.id).delete() + except KeyError: + items_unknown_to_calibre.append(item) + ub.session_commit() + + if items_unknown_to_calibre: + log.debug("Received request to remove an unknown book to a collecition. Silently ignoring item.") + + return make_response('', 200) + + +# Add new, changed, or deleted shelves to the sync_results. +# Note: Public shelves that aren't owned by the user aren't supported. +def sync_shelves(sync_token, sync_results, only_kobo_shelves=False): + new_tags_last_modified = sync_token.tags_last_modified + # transmit all archived shelfs independent of last sync (why should this matter?) + for shelf in ub.session.query(ub.ShelfArchive).filter(ub.ShelfArchive.user_id == current_user.id): + new_tags_last_modified = max(shelf.last_modified, new_tags_last_modified) + sync_results.append({ + "DeletedTag": { + "Tag": { + "Id": shelf.uuid, + "LastModified": convert_to_kobo_timestamp_string(shelf.last_modified) + } + } + }) + ub.session.delete(shelf) + ub.session_commit() + + extra_filters = [] + if only_kobo_shelves: + for shelf in ub.session.query(ub.Shelf).filter( + func.datetime(ub.Shelf.last_modified) > sync_token.tags_last_modified, + ub.Shelf.user_id == current_user.id, + not ub.Shelf.kobo_sync + ): + sync_results.append({ + "DeletedTag": { + "Tag": { + "Id": shelf.uuid, + "LastModified": convert_to_kobo_timestamp_string(shelf.last_modified) + } + } + }) + extra_filters.append(ub.Shelf.kobo_sync) + + shelflist = ub.session.query(ub.Shelf).outerjoin(ub.BookShelf).filter( + or_(func.datetime(ub.Shelf.last_modified) > sync_token.tags_last_modified, + func.datetime(ub.BookShelf.date_added) > sync_token.tags_last_modified), + ub.Shelf.user_id == current_user.id, + *extra_filters + ).distinct().order_by(func.datetime(ub.Shelf.last_modified).asc()) + + for shelf in shelflist: + if not shelf_lib.check_shelf_view_permissions(shelf): + continue + + new_tags_last_modified = max(shelf.last_modified, new_tags_last_modified) + + tag = create_kobo_tag(shelf) + if not tag: + continue + + if shelf.created > sync_token.tags_last_modified: + sync_results.append({ + "NewTag": tag + }) + else: + sync_results.append({ + "ChangedTag": tag + }) + sync_token.tags_last_modified = new_tags_last_modified + ub.session_commit() + + +# Creates a Kobo "Tag" object from a ub.Shelf object +def create_kobo_tag(shelf): + tag = { + "Created": convert_to_kobo_timestamp_string(shelf.created), + "Id": shelf.uuid, + "Items": [], + "LastModified": convert_to_kobo_timestamp_string(shelf.last_modified), + "Name": shelf.name, + "Type": "UserTag" + } + for book_shelf in shelf.books: + book = calibre_db.get_book(book_shelf.book_id) + if not book: + log.info("Book (id: %s) in BookShelf (id: %s) not found in book database", book_shelf.book_id, shelf.id) + continue + tag["Items"].append( + { + "RevisionId": book.uuid, + "Type": "ProductRevisionTagItem" + } + ) + return {"Tag": tag} + + +@csrf.exempt +@kobo.route("/v1/library//state", methods=["GET", "PUT"]) +@requires_kobo_auth +def HandleStateRequest(book_uuid): + book = calibre_db.get_book_by_uuid(book_uuid) + if not book or not book.data: + log.info("Book %s not found in database", book_uuid) + return redirect_or_proxy_request() + + kobo_reading_state = get_or_create_reading_state(book.id) + + if request.method == "GET": + return jsonify([get_kobo_reading_state_response(book, kobo_reading_state)]) + else: + update_results_response = {"EntitlementId": book_uuid} + + try: + request_data = request.json + request_reading_state = request_data["ReadingStates"][0] + + request_bookmark = request_reading_state["CurrentBookmark"] + if request_bookmark: + current_bookmark = kobo_reading_state.current_bookmark + current_bookmark.progress_percent = request_bookmark["ProgressPercent"] + current_bookmark.content_source_progress_percent = request_bookmark["ContentSourceProgressPercent"] + location = request_bookmark["Location"] + if location: + current_bookmark.location_value = location["Value"] + current_bookmark.location_type = location["Type"] + current_bookmark.location_source = location["Source"] + update_results_response["CurrentBookmarkResult"] = {"Result": "Success"} + + request_statistics = request_reading_state["Statistics"] + if request_statistics: + statistics = kobo_reading_state.statistics + statistics.spent_reading_minutes = int(request_statistics["SpentReadingMinutes"]) + statistics.remaining_time_minutes = int(request_statistics["RemainingTimeMinutes"]) + update_results_response["StatisticsResult"] = {"Result": "Success"} + + request_status_info = request_reading_state["StatusInfo"] + if request_status_info: + book_read = kobo_reading_state.book_read_link + new_book_read_status = get_ub_read_status(request_status_info["Status"]) + if new_book_read_status == ub.ReadBook.STATUS_IN_PROGRESS \ + and new_book_read_status != book_read.read_status: + book_read.times_started_reading += 1 + book_read.last_time_started_reading = datetime.now(timezone.utc) + book_read.read_status = new_book_read_status + update_results_response["StatusInfoResult"] = {"Result": "Success"} + except (KeyError, TypeError, ValueError, StatementError): + log.debug("Received malformed v1/library//state request.") + ub.session.rollback() + abort(400, description="Malformed request data is missing 'ReadingStates' key") + + if config.config_hardcover_sync and bool(hardcover): + hardcoverClient = hardcover.HardcoverClient(current_user.hardcover_token) + hardcoverClient.update_reading_progress(book.identifiers, request_bookmark["ProgressPercent"]) + + ub.session.merge(kobo_reading_state) + ub.session_commit() + return jsonify({ + "RequestResult": "Success", + "UpdateResults": [update_results_response], + }) + + +def get_read_status_for_kobo(ub_book_read): + enum_to_string_map = { + None: "ReadyToRead", + ub.ReadBook.STATUS_UNREAD: "ReadyToRead", + ub.ReadBook.STATUS_FINISHED: "Finished", + ub.ReadBook.STATUS_IN_PROGRESS: "Reading", + } + return enum_to_string_map[ub_book_read.read_status] + + +def get_ub_read_status(kobo_read_status): + string_to_enum_map = { + None: None, + "ReadyToRead": ub.ReadBook.STATUS_UNREAD, + "Finished": ub.ReadBook.STATUS_FINISHED, + "Reading": ub.ReadBook.STATUS_IN_PROGRESS, + } + return string_to_enum_map[kobo_read_status] + + +def get_or_create_reading_state(book_id): + book_read = ub.session.query(ub.ReadBook).filter(ub.ReadBook.book_id == book_id, + ub.ReadBook.user_id == int(current_user.id)).one_or_none() + if not book_read: + book_read = ub.ReadBook(user_id=current_user.id, book_id=book_id) + if not book_read.kobo_reading_state: + kobo_reading_state = ub.KoboReadingState(user_id=book_read.user_id, book_id=book_id) + kobo_reading_state.current_bookmark = ub.KoboBookmark() + kobo_reading_state.statistics = ub.KoboStatistics() + book_read.kobo_reading_state = kobo_reading_state + ub.session.add(book_read) + ub.session_commit() + return book_read.kobo_reading_state + + +def get_kobo_reading_state_response(book, kobo_reading_state): + return { + "EntitlementId": book.uuid, + "Created": convert_to_kobo_timestamp_string(book.timestamp), + "LastModified": convert_to_kobo_timestamp_string(kobo_reading_state.last_modified), + # AFAICT PriorityTimestamp is always equal to LastModified. + "PriorityTimestamp": convert_to_kobo_timestamp_string(kobo_reading_state.priority_timestamp), + "StatusInfo": get_status_info_response(kobo_reading_state.book_read_link), + "Statistics": get_statistics_response(kobo_reading_state.statistics), + "CurrentBookmark": get_current_bookmark_response(kobo_reading_state.current_bookmark), + } + + +def get_status_info_response(book_read): + resp = { + "LastModified": convert_to_kobo_timestamp_string(book_read.last_modified), + "Status": get_read_status_for_kobo(book_read), + "TimesStartedReading": book_read.times_started_reading, + } + if book_read.last_time_started_reading: + resp["LastTimeStartedReading"] = convert_to_kobo_timestamp_string(book_read.last_time_started_reading) + return resp + + +def get_statistics_response(statistics): + resp = { + "LastModified": convert_to_kobo_timestamp_string(statistics.last_modified), + } + if statistics.spent_reading_minutes: + resp["SpentReadingMinutes"] = statistics.spent_reading_minutes + if statistics.remaining_time_minutes: + resp["RemainingTimeMinutes"] = statistics.remaining_time_minutes + return resp + + +def get_current_bookmark_response(current_bookmark): + resp = { + "LastModified": convert_to_kobo_timestamp_string(current_bookmark.last_modified), + } + if current_bookmark.progress_percent: + resp["ProgressPercent"] = current_bookmark.progress_percent + if current_bookmark.content_source_progress_percent: + resp["ContentSourceProgressPercent"] = current_bookmark.content_source_progress_percent + if current_bookmark.location_value: + resp["Location"] = { + "Value": current_bookmark.location_value, + "Type": current_bookmark.location_type, + "Source": current_bookmark.location_source, + } + return resp + + +@kobo.route("/////image.jpg", defaults={'Quality': ""}) +@kobo.route("//////image.jpg") +@requires_kobo_auth +def HandleCoverImageRequest(book_uuid, width, height, Quality, isGreyscale): + try: + if int(height) > 1000: + resolution = COVER_THUMBNAIL_LARGE + elif int(height) > 500: + resolution = COVER_THUMBNAIL_MEDIUM + else: + resolution = COVER_THUMBNAIL_SMALL + except ValueError: + log.error("Requested height %s of book %s is invalid" % (book_uuid, height)) + resolution = COVER_THUMBNAIL_SMALL + book_cover = helper.get_book_cover_with_uuid(book_uuid, resolution=resolution) + if book_cover: + log.debug("Serving local cover image of book %s" % book_uuid) + return book_cover + + if not config.config_kobo_proxy: + log.debug("Returning 404 for cover image of unknown book %s" % book_uuid) + # additional proxy request make no sense, -> direct return + return abort(404) + + log.debug("Redirecting request for cover image of unknown book %s to Kobo" % book_uuid) + return redirect(KOBO_IMAGEHOST_URL + + "/{book_uuid}/{width}/{height}/false/image.jpg".format(book_uuid=book_uuid, + width=width, + height=height), 307) + + +@kobo.route("") +def TopLevelEndpoint(): + return make_response(jsonify({})) + + +@csrf.exempt +@kobo.route("/v1/library/", methods=["DELETE"]) +@requires_kobo_auth +def HandleBookDeletionRequest(book_uuid): + log.info("Kobo book delete request received for book %s" % book_uuid) + book = calibre_db.get_book_by_uuid(book_uuid) + if not book: + log.info("Book %s not found in database", book_uuid) + return redirect_or_proxy_request() + + book_id = book.id + is_archived = kobo_sync_status.change_archived_books(book_id, True) + if is_archived: + kobo_sync_status.remove_synced_book(book_id) + return "", 204 + + +# TODO: Implement the following routes +@csrf.exempt +@kobo.route("/v1/library/", methods=["DELETE", "GET"]) +def HandleUnimplementedRequest(dummy=None): + log.debug("Unimplemented Library Request received: %s (request is forwarded to kobo if configured)", + request.base_url) + return redirect_or_proxy_request() + + +# TODO: Implement the following routes +@csrf.exempt +@kobo.route("/v1/user/loyalty/", methods=["GET", "POST"]) +@kobo.route("/v1/user/profile", methods=["GET", "POST"]) +@kobo.route("/v1/user/wishlist", methods=["GET", "POST"]) +@kobo.route("/v1/user/recommendations", methods=["GET", "POST"]) +@kobo.route("/v1/analytics/", methods=["GET", "POST"]) +@kobo.route("/v1/assets", methods=["GET"]) +def HandleUserRequest(dummy=None): + log.debug("Unimplemented User Request received: %s (request is forwarded to kobo if configured)", request.base_url) + return redirect_or_proxy_request() + + +@csrf.exempt +@kobo.route("/v1/user/loyalty/benefits", methods=["GET"]) +def handle_benefits(): + if config.config_kobo_proxy: + return redirect_or_proxy_request() + else: + return make_response(jsonify({"Benefits": {}})) + + +@csrf.exempt +@kobo.route("/v1/analytics/gettests", methods=["GET", "POST"]) +def handle_getests(): + if config.config_kobo_proxy: + return redirect_or_proxy_request() + else: + testkey = request.headers.get("X-Kobo-userkey", "") + return make_response(jsonify({"Result": "Success", "TestKey": testkey, "Tests": {}})) + + +@csrf.exempt +@kobo.route("/v1/products//prices", methods=["GET", "POST"]) +@kobo.route("/v1/products//recommendations", methods=["GET", "POST"]) +@kobo.route("/v1/products//nextread", methods=["GET", "POST"]) +@kobo.route("/v1/products//reviews", methods=["GET", "POST"]) +@kobo.route("/v1/products/featured/", methods=["GET", "POST"]) +@kobo.route("/v1/products/featured/", methods=["GET", "POST"]) +@kobo.route("/v1/products/books/external/", methods=["GET", "POST"]) +@kobo.route("/v1/products/books/series/", methods=["GET", "POST"]) +@kobo.route("/v1/products/books/", methods=["GET", "POST"]) +@kobo.route("/v1/products/books//", methods=["GET", "POST"]) +@kobo.route("/v1/products/dailydeal", methods=["GET", "POST"]) +@kobo.route("/v1/products/deals", methods=["GET", "POST"]) +@kobo.route("/v1/products", methods=["GET", "POST"]) +@kobo.route("/v1/affiliate", methods=["GET", "POST"]) +@kobo.route("/v1/deals", methods=["GET", "POST"]) +def HandleProductsRequest(dummy=None): + log.debug("Unimplemented Products Request received: %s (request is forwarded to kobo if configured)", + request.base_url) + return redirect_or_proxy_request() + + +def make_calibre_web_auth_response(): + # As described in kobo_auth.py, CalibreWeb doesn't make use practical use of this auth/device API call for + # authentation (nor for authorization). We return a dummy response just to keep the device happy. + content = request.get_json() + AccessToken = base64.b64encode(os.urandom(24)).decode('utf-8') + RefreshToken = base64.b64encode(os.urandom(24)).decode('utf-8') + return make_response( + jsonify( + { + "AccessToken": AccessToken, + "RefreshToken": RefreshToken, + "TokenType": "Bearer", + "TrackingId": str(uuid.uuid4()), + "UserKey": content.get('UserKey', ""), + } + ) + ) + + +@csrf.exempt +@kobo.route("/v1/auth/device", methods=["POST"]) +@requires_kobo_auth +def HandleAuthRequest(): + log.debug('Kobo Auth request') + if config.config_kobo_proxy: + try: + return redirect_or_proxy_request() + except Exception: + log.error("Failed to receive or parse response from Kobo's auth endpoint. Falling back to un-proxied mode.") + return make_calibre_web_auth_response() + + +@kobo.route("/v1/initialization") +@requires_kobo_auth +def HandleInitRequest(): + log.info('Init') + + kobo_resources = None + if config.config_kobo_proxy: + try: + store_response = make_request_to_kobo_store() + store_response_json = store_response.json() + if "Resources" in store_response_json: + kobo_resources = store_response_json["Resources"] + except Exception: + log.error("Failed to receive or parse response from Kobo's init endpoint. Falling back to un-proxied mode.") + if not kobo_resources: + kobo_resources = NATIVE_KOBO_RESOURCES() + + if not current_app.wsgi_app.is_proxied: + log.debug('Kobo: Received unproxied request, changed request port to external server port') + if ':' in request.host and not request.host.endswith(']'): + host = "".join(request.host.split(':')[:-1]) + else: + host = request.host + calibre_web_url = "{url_scheme}://{url_base}:{url_port}".format( + url_scheme=request.scheme, + url_base=host, + url_port=config.config_external_port + ) + log.debug('Kobo: Received unproxied request, changed request url to %s', calibre_web_url) + kobo_resources["image_host"] = calibre_web_url + kobo_resources["image_url_quality_template"] = unquote(calibre_web_url + + url_for("kobo.HandleCoverImageRequest", + auth_token=kobo_auth.get_auth_token(), + book_uuid="{ImageId}", + width="{width}", + height="{height}", + Quality='{Quality}', + isGreyscale='isGreyscale')) + kobo_resources["image_url_template"] = unquote(calibre_web_url + + url_for("kobo.HandleCoverImageRequest", + auth_token=kobo_auth.get_auth_token(), + book_uuid="{ImageId}", + width="{width}", + height="{height}", + isGreyscale='false')) + else: + kobo_resources["image_host"] = url_for("web.index", _external=True).strip("/") + kobo_resources["image_url_quality_template"] = unquote(url_for("kobo.HandleCoverImageRequest", + auth_token=kobo_auth.get_auth_token(), + book_uuid="{ImageId}", + width="{width}", + height="{height}", + Quality='{Quality}', + isGreyscale='isGreyscale', + _external=True)) + kobo_resources["image_url_template"] = unquote(url_for("kobo.HandleCoverImageRequest", + auth_token=kobo_auth.get_auth_token(), + book_uuid="{ImageId}", + width="{width}", + height="{height}", + isGreyscale='false', + _external=True)) + + response = make_response(jsonify({"Resources": kobo_resources})) + response.headers["x-kobo-apitoken"] = "e30=" + + return response + + +@kobo.route("/download//") +@requires_kobo_auth +@download_required +def download_book(book_id, book_format): + return get_download_link(book_id, book_format, "kobo") + + +def NATIVE_KOBO_RESOURCES(): + return { + "account_page": "https://www.kobo.com/account/settings", + "account_page_rakuten": "https://my.rakuten.co.jp/", + "add_device": "https://storeapi.kobo.com/v1/user/add-device", + "add_entitlement": "https://storeapi.kobo.com/v1/library/{RevisionIds}", + "affiliaterequest": "https://storeapi.kobo.com/v1/affiliate", + "assets": "https://storeapi.kobo.com/v1/assets", + "audiobook": "https://storeapi.kobo.com/v1/products/audiobooks/{ProductId}", + "audiobook_detail_page": "https://www.kobo.com/{region}/{language}/audiobook/{slug}", + "audiobook_landing_page": "https://www.kobo.com/{region}/{language}/audiobooks", + "audiobook_preview": "https://storeapi.kobo.com/v1/products/audiobooks/{Id}/preview", + "audiobook_purchase_withcredit": "https://storeapi.kobo.com/v1/store/audiobook/{Id}", + "audiobook_subscription_orange_deal_inclusion_url": "https://authorize.kobo.com/inclusion", + "authorproduct_recommendations": "https://storeapi.kobo.com/v1/products/books/authors/recommendations", + "autocomplete": "https://storeapi.kobo.com/v1/products/autocomplete", + "blackstone_header": { + "key": "x-amz-request-payer", + "value": "requester" + }, + "book": "https://storeapi.kobo.com/v1/products/books/{ProductId}", + "book_detail_page": "https://www.kobo.com/{region}/{language}/ebook/{slug}", + "book_detail_page_rakuten": "http://books.rakuten.co.jp/rk/{crossrevisionid}", + "book_landing_page": "https://www.kobo.com/ebooks", + "book_subscription": "https://storeapi.kobo.com/v1/products/books/subscriptions", + "browse_history": "https://storeapi.kobo.com/v1/user/browsehistory", + "categories": "https://storeapi.kobo.com/v1/categories", + "categories_page": "https://www.kobo.com/ebooks/categories", + "category": "https://storeapi.kobo.com/v1/categories/{CategoryId}", + "category_featured_lists": "https://storeapi.kobo.com/v1/categories/{CategoryId}/featured", + "category_products": "https://storeapi.kobo.com/v1/categories/{CategoryId}/products", + "checkout_borrowed_book": "https://storeapi.kobo.com/v1/library/borrow", + "client_authd_referral": "https://authorize.kobo.com/api/AuthenticatedReferral/client/v1/getLink", + "configuration_data": "https://storeapi.kobo.com/v1/configuration", + "content_access_book": "https://storeapi.kobo.com/v1/products/books/{ProductId}/access", + "customer_care_live_chat": "https://v2.zopim.com/widget/livechat.html?key=Y6gwUmnu4OATxN3Tli4Av9bYN319BTdO", + "daily_deal": "https://storeapi.kobo.com/v1/products/dailydeal", + "deals": "https://storeapi.kobo.com/v1/deals", + "delete_entitlement": "https://storeapi.kobo.com/v1/library/{Ids}", + "delete_tag": "https://storeapi.kobo.com/v1/library/tags/{TagId}", + "delete_tag_items": "https://storeapi.kobo.com/v1/library/tags/{TagId}/items/delete", + "device_auth": "https://storeapi.kobo.com/v1/auth/device", + "device_refresh": "https://storeapi.kobo.com/v1/auth/refresh", + "dictionary_host": "https://ereaderfiles.kobo.com", + "discovery_host": "https://discovery.kobobooks.com", + "ereaderdevices": "https://storeapi.kobo.com/v2/products/EReaderDeviceFeeds", + "eula_page": "https://www.kobo.com/termsofuse?style=onestore", + "exchange_auth": "https://storeapi.kobo.com/v1/auth/exchange", + "external_book": "https://storeapi.kobo.com/v1/products/books/external/{Ids}", + "facebook_sso_page": "https://authorize.kobo.com/signin/provider/Facebook/login?returnUrl=http://kobo.com/", + "featured_list": "https://storeapi.kobo.com/v1/products/featured/{FeaturedListId}", + "featured_lists": "https://storeapi.kobo.com/v1/products/featured", + "free_books_page": { + "EN": "https://www.kobo.com/{region}/{language}/p/free-ebooks", + "FR": "https://www.kobo.com/{region}/{language}/p/livres-gratuits", + "IT": "https://www.kobo.com/{region}/{language}/p/libri-gratuiti", + "NL": "https://www.kobo.com/{region}/{language}/List/bekijk-het-overzicht-van-gratis-ebooks/QpkkVWnUw8sxmgjSlCbJRg", + "PT": "https://www.kobo.com/{region}/{language}/p/livros-gratis" + }, + "fte_feedback": "https://storeapi.kobo.com/v1/products/ftefeedback", + "funnel_metrics": "https://storeapi.kobo.com/v1/funnelmetrics", + "get_download_keys": "https://storeapi.kobo.com/v1/library/downloadkeys", + "get_download_link": "https://storeapi.kobo.com/v1/library/downloadlink", + "get_tests_request": "https://storeapi.kobo.com/v1/analytics/gettests", + "giftcard_epd_redeem_url": "https://www.kobo.com/{storefront}/{language}/redeem-ereader", + "giftcard_redeem_url": "https://www.kobo.com/{storefront}/{language}/redeem", + "gpb_flow_enabled": "False", + "help_page": "http://www.kobo.com/help", + "image_host": "//cdn.kobo.com/book-images/", + "image_url_quality_template": "https://cdn.kobo.com/book-images/{ImageId}/{Width}/{Height}/{Quality}/{IsGreyscale}/image.jpg", + "image_url_template": "https://cdn.kobo.com/book-images/{ImageId}/{Width}/{Height}/false/image.jpg", + "kobo_audiobooks_credit_redemption": "False", + "kobo_audiobooks_enabled": "True", + "kobo_audiobooks_orange_deal_enabled": "False", + "kobo_audiobooks_subscriptions_enabled": "False", + "kobo_display_price": "True", + "kobo_dropbox_link_account_enabled": "False", + "kobo_google_tax": "False", + "kobo_googledrive_link_account_enabled": "False", + "kobo_nativeborrow_enabled": "False", + "kobo_onedrive_link_account_enabled": "False", + "kobo_onestorelibrary_enabled": "False", + "kobo_privacyCentre_url": "https://www.kobo.com/privacy", + "kobo_redeem_enabled": "True", + "kobo_shelfie_enabled": "False", + "kobo_subscriptions_enabled": "True", + "kobo_superpoints_enabled": "True", + "kobo_wishlist_enabled": "True", + "library_book": "https://storeapi.kobo.com/v1/user/library/books/{LibraryItemId}", + "library_items": "https://storeapi.kobo.com/v1/user/library", + "library_metadata": "https://storeapi.kobo.com/v1/library/{Ids}/metadata", + "library_prices": "https://storeapi.kobo.com/v1/user/library/previews/prices", + "library_search": "https://storeapi.kobo.com/v1/library/search", + "library_sync": "https://storeapi.kobo.com/v1/library/sync", + "love_dashboard_page": "https://www.kobo.com/{region}/{language}/kobosuperpoints", + "love_points_redemption_page": "https://www.kobo.com/{region}/{language}/KoboSuperPointsRedemption?productId={ProductId}", + "magazine_landing_page": "https://www.kobo.com/emagazines", + "more_sign_in_options": "https://authorize.kobo.com/signin?returnUrl=http://kobo.com/#allProviders", + "notebooks": "https://storeapi.kobo.com/api/internal/notebooks", + "notifications_registration_issue": "https://storeapi.kobo.com/v1/notifications/registration", + "oauth_host": "https://oauth.kobo.com", + "password_retrieval_page": "https://www.kobo.com/passwordretrieval.html", + "personalizedrecommendations": "https://storeapi.kobo.com/v2/users/personalizedrecommendations", + "pocket_link_account_start": "https://authorize.kobo.com/{region}/{language}/linkpocket", + "post_analytics_event": "https://storeapi.kobo.com/v1/analytics/event", + "ppx_purchasing_url": "https://purchasing.kobo.com", + "privacy_page": "https://www.kobo.com/privacypolicy?style=onestore", + "product_nextread": "https://storeapi.kobo.com/v1/products/{ProductIds}/nextread", + "product_prices": "https://storeapi.kobo.com/v1/products/{ProductIds}/prices", + "product_recommendations": "https://storeapi.kobo.com/v1/products/{ProductId}/recommendations", + "product_reviews": "https://storeapi.kobo.com/v1/products/{ProductIds}/reviews", + "products": "https://storeapi.kobo.com/v1/products", + "productsv2": "https://storeapi.kobo.com/v2/products", + "provider_external_sign_in_page": "https://authorize.kobo.com/ExternalSignIn/{providerName}?returnUrl=http://kobo.com/", + "quickbuy_checkout": "https://storeapi.kobo.com/v1/store/quickbuy/{PurchaseId}/checkout", + "quickbuy_create": "https://storeapi.kobo.com/v1/store/quickbuy/purchase", + "rakuten_token_exchange": "https://storeapi.kobo.com/v1/auth/rakuten_token_exchange", + "rating": "https://storeapi.kobo.com/v1/products/{ProductId}/rating/{Rating}", + "reading_services_host": "https://readingservices.kobo.com", + "reading_state": "https://storeapi.kobo.com/v1/library/{Ids}/state", + "redeem_interstitial_page": "https://www.kobo.com", + "registration_page": "https://authorize.kobo.com/signup?returnUrl=http://kobo.com/", + "related_items": "https://storeapi.kobo.com/v1/products/{Id}/related", + "remaining_book_series": "https://storeapi.kobo.com/v1/products/books/series/{SeriesId}", + "rename_tag": "https://storeapi.kobo.com/v1/library/tags/{TagId}", + "review": "https://storeapi.kobo.com/v1/products/reviews/{ReviewId}", + "review_sentiment": "https://storeapi.kobo.com/v1/products/reviews/{ReviewId}/sentiment/{Sentiment}", + "shelfie_recommendations": "https://storeapi.kobo.com/v1/user/recommendations/shelfie", + "sign_in_page": "https://authorize.kobo.com/signin?returnUrl=http://kobo.com/", + "social_authorization_host": "https://social.kobobooks.com:8443", + "social_host": "https://social.kobobooks.com", + "store_home": "www.kobo.com/{region}/{language}", + "store_host": "www.kobo.com", + "store_newreleases": "https://www.kobo.com/{region}/{language}/List/new-releases/961XUjtsU0qxkFItWOutGA", + "store_search": "https://www.kobo.com/{region}/{language}/Search?Query={query}", + "store_top50": "https://www.kobo.com/{region}/{language}/ebooks/Top", + "subs_landing_page": "https://www.kobo.com/{region}/{language}/plus", + "subs_management_page": "https://www.kobo.com/{region}/{language}/account/subscriptions", + "subs_plans_page": "https://www.kobo.com/{region}/{language}/plus/plans", + "subs_purchase_buy_templated": "https://www.kobo.com/{region}/{language}/Checkoutoption/{ProductId}/{TierId}", + "tag_items": "https://storeapi.kobo.com/v1/library/tags/{TagId}/Items", + "tags": "https://storeapi.kobo.com/v1/library/tags", + "taste_profile": "https://storeapi.kobo.com/v1/products/tasteprofile", + "terms_of_sale_page": "https://authorize.kobo.com/{region}/{language}/terms/termsofsale", + "update_accessibility_to_preview": "https://storeapi.kobo.com/v1/library/{EntitlementIds}/preview", + "use_one_store": "True", + "user_loyalty_benefits": "https://storeapi.kobo.com/v1/user/loyalty/benefits", + "user_platform": "https://storeapi.kobo.com/v1/user/platform", + "user_profile": "https://storeapi.kobo.com/v1/user/profile", + "user_ratings": "https://storeapi.kobo.com/v1/user/ratings", + "user_recommendations": "https://storeapi.kobo.com/v1/user/recommendations", + "user_reviews": "https://storeapi.kobo.com/v1/user/reviews", + "user_wishlist": "https://storeapi.kobo.com/v1/user/wishlist", + "userguide_host": "https://ereaderfiles.kobo.com", + "wishlist_page": "https://www.kobo.com/{region}/{language}/account/wishlist" + } diff --git a/root/app/calibre-web/cps/metadata_provider/amazon.py b/root/app/calibre-web/cps/metadata_provider/amazon.py new file mode 100644 index 0000000..09ef1eb --- /dev/null +++ b/root/app/calibre-web/cps/metadata_provider/amazon.py @@ -0,0 +1,140 @@ +# -*- coding: utf-8 -*- + +# This file is part of the Calibre-Web (https://github.com/janeczku/calibre-web) +# Copyright (C) 2022 quarz12 +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import concurrent.futures +import requests +from bs4 import BeautifulSoup as BS # requirement +from typing import List, Optional + +try: + import cchardet #optional for better speed +except ImportError: + pass + +from cps.services.Metadata import MetaRecord, MetaSourceInfo, Metadata +import cps.logger as logger + +#from time import time +from operator import itemgetter +log = logger.create() + + +class Amazon(Metadata): + __name__ = "Amazon" + __id__ = "amazon" + headers = {'upgrade-insecure-requests': '1', + 'user-agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:130.0) Gecko/20100101 Firefox/130.0', + 'accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/png,image/svg+xml,*/*;q=0.8', + 'Sec-Fetch-Site': 'same-origin', + 'Sec-Fetch-Mode': 'navigate', + 'Sec-Fetch-User': '?1', + 'Sec-Fetch-Dest': 'document', + 'Upgrade-Insecure-Requests': '1', + 'Alt-Used' : 'www.amazon.com', + 'Priority' : 'u=0, i', + 'accept-encoding': 'gzip, deflate, br, zstd', + 'accept-language': 'en-US,en;q=0.9'} + session = requests.Session() + session.headers=headers + + def search( + self, query: str, generic_cover: str = "", locale: str = "en",**kwargs + ) -> Optional[List[MetaRecord]]: + def inner(link, index) -> [dict, int]: + with self.session as session: + try: + r = session.get(f"https://www.amazon.com/{link}") + r.raise_for_status() + except Exception as ex: + log.warning(ex) + return [] + long_soup = BS(r.text, "lxml") #~4sec :/ + soup2 = long_soup.find("div", attrs={"cel_widget_id": "dpx-ppd_csm_instrumentation_wrapper"}) + if soup2 is None: + return [] + try: + match = MetaRecord( + title = "", + authors = "", + source=MetaSourceInfo( + id=self.__id__, + description="Amazon Books", + link="https://amazon.com/" + ), + url = f"https://www.amazon.com{link}", + #the more searches the slower, these are too hard to find in reasonable time or might not even exist + publisher= "", # very unreliable + publishedDate= "", # very unreliable + id = None, # ? + tags = [] # dont exist on amazon + ) + + try: + match.description = "\n".join( + soup2.find("div", attrs={"data-feature-name": "bookDescription"}).stripped_strings)\ + .replace("\xa0"," ")[:-9].strip().strip("\n") + except (AttributeError, TypeError): + return [] # if there is no description it is not a book and therefore should be ignored + try: + match.title = soup2.find("span", attrs={"id": "productTitle"}).text + except (AttributeError, TypeError): + match.title = "" + try: + match.authors = [next( + filter(lambda i: i != " " and i != "\n" and not i.startswith("{"), + x.findAll(string=True))).strip() + for x in soup2.findAll("span", attrs={"class": "author"})] + except (AttributeError, TypeError, StopIteration): + match.authors = "" + try: + match.rating = int( + soup2.find("span", class_="a-icon-alt").text.split(" ")[0].split(".")[ + 0]) # first number in string + except (AttributeError, ValueError): + match.rating = 0 + try: + match.cover = soup2.find("img", attrs={"class": "a-dynamic-image"})["src"] + except (AttributeError, TypeError): + match.cover = "" + return match, index + except Exception as e: + log.error_or_exception(e) + return [] + + val = list() + if self.active: + try: + results = self.session.get( + f"https://www.amazon.com/s?k={query.replace(' ', '+')}&i=digital-text&sprefix={query.replace(' ', '+')}" + f"%2Cdigital-text&ref=nb_sb_noss", + headers=self.headers) + results.raise_for_status() + except requests.exceptions.HTTPError as e: + log.error_or_exception(e) + return [] + except Exception as e: + log.warning(e) + return [] + soup = BS(results.text, 'html.parser') + links_list = [next(filter(lambda i: "digital-text" in i["href"], x.findAll("a")))["href"] for x in + soup.findAll("div", attrs={"data-component-type": "s-search-result"})] + with concurrent.futures.ThreadPoolExecutor(max_workers=5) as executor: + fut = {executor.submit(inner, link, index) for index, link in enumerate(links_list[:3])} + val = list(map(lambda x : x.result(), concurrent.futures.as_completed(fut))) + result = list(filter(lambda x: x, val)) + return [x[0] for x in sorted(result, key=itemgetter(1))] #sort by amazons listing order for best relevance diff --git a/root/app/calibre-web/cps/metadata_provider/comicvine.py b/root/app/calibre-web/cps/metadata_provider/comicvine.py new file mode 100644 index 0000000..b51c1c9 --- /dev/null +++ b/root/app/calibre-web/cps/metadata_provider/comicvine.py @@ -0,0 +1,92 @@ +# -*- coding: utf-8 -*- + +# This file is part of the Calibre-Web (https://github.com/janeczku/calibre-web) +# Copyright (C) 2021 OzzieIsaacs +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +# ComicVine api document: https://comicvine.gamespot.com/api/documentation +from typing import Dict, List, Optional +from urllib.parse import quote + +import requests +from cps import logger +from cps.services.Metadata import MetaRecord, MetaSourceInfo, Metadata + +log = logger.create() + + +class ComicVine(Metadata): + __name__ = "ComicVine" + __id__ = "comicvine" + DESCRIPTION = "ComicVine Books" + META_URL = "https://comicvine.gamespot.com/" + API_KEY = "57558043c53943d5d1e96a9ad425b0eb85532ee6" + BASE_URL = ( + f"https://comicvine.gamespot.com/api/search?api_key={API_KEY}" + f"&resources=issue&query=" + ) + QUERY_PARAMS = "&sort=name:desc&format=json" + HEADERS = {"User-Agent": "Not Evil Browser"} + + def search( + self, query: str, generic_cover: str = "", locale: str = "en",**kwargs + ) -> Optional[List[MetaRecord]]: + val = list() + if self.active: + title_tokens = list(self.get_title_tokens(query, strip_joiners=False)) + if title_tokens: + tokens = [quote(t.encode("utf-8")) for t in title_tokens] + query = "%20".join(tokens) + try: + result = requests.get( + f"{ComicVine.BASE_URL}{query}{ComicVine.QUERY_PARAMS}", + headers=ComicVine.HEADERS, + ) + result.raise_for_status() + except Exception as e: + log.warning(e) + return None + for result in result.json()["results"]: + match = self._parse_search_result( + result=result, generic_cover=generic_cover, locale=locale + ) + val.append(match) + return val + + def _parse_search_result( + self, result: Dict, generic_cover: str, locale: str + ) -> MetaRecord: + series = result["volume"].get("name", "") + series_index = result.get("issue_number", 0) + issue_name = result.get("name", "") + match = MetaRecord( + id=result["id"], + title=f"{series}#{series_index} - {issue_name}", + authors=result.get("authors", []), + url=result.get("site_detail_url", ""), + source=MetaSourceInfo( + id=self.__id__, + description=ComicVine.DESCRIPTION, + link=ComicVine.META_URL, + ), + series=series, + ) + match.cover = result["image"].get("original_url", generic_cover) + match.description = result.get("description", "") + match.publishedDate = result.get("store_date", result.get("date_added")) + match.series_index = series_index + match.tags = ["Comics", series] + match.identifiers = {"comicvine": match.id} + return match diff --git a/root/app/calibre-web/cps/metadata_provider/douban.py b/root/app/calibre-web/cps/metadata_provider/douban.py new file mode 100644 index 0000000..316fbea --- /dev/null +++ b/root/app/calibre-web/cps/metadata_provider/douban.py @@ -0,0 +1,260 @@ +# -*- coding: utf-8 -*- + +# This file is part of the Calibre-Web (https://github.com/janeczku/calibre-web) +# Copyright (C) 2022 xlivevil +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +import re +from concurrent import futures +from typing import List, Optional + +import requests +from html2text import HTML2Text +from lxml import etree + +from cps import logger +from cps.services.Metadata import Metadata, MetaRecord, MetaSourceInfo + +log = logger.create() + + +def html2text(html: str) -> str: + + h2t = HTML2Text() + h2t.body_width = 0 + h2t.single_line_break = True + h2t.emphasis_mark = "*" + return h2t.handle(html) + + +class Douban(Metadata): + __name__ = "豆瓣" + __id__ = "douban" + DESCRIPTION = "豆瓣" + META_URL = "https://book.douban.com/" + SEARCH_JSON_URL = "https://www.douban.com/j/search" + SEARCH_URL = "https://www.douban.com/search" + + ID_PATTERN = re.compile(r"sid: (?P\d+),") + AUTHORS_PATTERN = re.compile(r"作者|译者") + PUBLISHER_PATTERN = re.compile(r"出版社") + SUBTITLE_PATTERN = re.compile(r"副标题") + PUBLISHED_DATE_PATTERN = re.compile(r"出版年") + SERIES_PATTERN = re.compile(r"丛书") + IDENTIFIERS_PATTERN = re.compile(r"ISBN|统一书号") + CRITERIA_PATTERN = re.compile("criteria = '(.+)'") + + TITTLE_XPATH = "//span[@property='v:itemreviewed']" + COVER_XPATH = "//a[@class='nbg']" + INFO_XPATH = "//*[@id='info']//span[@class='pl']" + TAGS_XPATH = "//a[contains(@class, 'tag')]" + DESCRIPTION_XPATH = "//div[@id='link-report']//div[@class='intro']" + RATING_XPATH = "//div[@class='rating_self clearfix']/strong" + + session = requests.Session() + session.headers = { + 'user-agent': + 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/98.0.4758.102 Safari/537.36 Edg/98.0.1108.56', + } + + def search(self, + query: str, + generic_cover: str = "", + locale: str = "en",**kwargs) -> List[MetaRecord]: + val = [] + if self.active: + log.debug(f"start searching {query} on douban") + if title_tokens := list( + self.get_title_tokens(query, strip_joiners=False)): + query = "+".join(title_tokens) + + book_id_list = self._get_book_id_list_from_html(query) + + if not book_id_list: + log.debug("No search results in Douban") + return [] + + with futures.ThreadPoolExecutor( + max_workers=5, thread_name_prefix='douban') as executor: + + fut = [ + executor.submit(self._parse_single_book, book_id, + generic_cover) for book_id in book_id_list + ] + + val = [ + future.result() for future in futures.as_completed(fut) + if future.result() + ] + + return val + + def _get_book_id_list_from_html(self, query: str) -> List[str]: + try: + r = self.session.get(self.SEARCH_URL, + params={ + "cat": 1001, + "q": query + }) + r.raise_for_status() + + except Exception as e: + log.warning(e) + return [] + + html = etree.HTML(r.content.decode("utf8")) + result_list = html.xpath(self.COVER_XPATH) + + return [ + self.ID_PATTERN.search(item.get("onclick")).group("id") + for item in result_list[:10] + if self.ID_PATTERN.search(item.get("onclick")) + ] + + def _get_book_id_list_from_json(self, query: str) -> List[str]: + try: + r = self.session.get(self.SEARCH_JSON_URL, + params={ + "cat": 1001, + "q": query + }) + r.raise_for_status() + + except Exception as e: + log.warning(e) + return [] + + results = r.json() + if results["total"] == 0: + return [] + + return [ + self.ID_PATTERN.search(item).group("id") + for item in results["items"][:10] if self.ID_PATTERN.search(item) + ] + + def _parse_single_book(self, + id: str, + generic_cover: str = "") -> Optional[MetaRecord]: + url = f"https://book.douban.com/subject/{id}/" + log.debug(f"start parsing {url}") + + try: + r = self.session.get(url) + r.raise_for_status() + except Exception as e: + log.warning(e) + return None + + match = MetaRecord( + id=id, + title="", + authors=[], + url=url, + source=MetaSourceInfo( + id=self.__id__, + description=self.DESCRIPTION, + link=self.META_URL, + ), + ) + + decode_content = r.content.decode("utf8") + html = etree.HTML(decode_content) + + match.title = html.xpath(self.TITTLE_XPATH)[0].text + match.cover = html.xpath( + self.COVER_XPATH)[0].attrib["href"] or generic_cover + try: + rating_num = float(html.xpath(self.RATING_XPATH)[0].text.strip()) + except Exception: + rating_num = 0 + match.rating = int(-1 * rating_num // 2 * -1) if rating_num else 0 + + tag_elements = html.xpath(self.TAGS_XPATH) + if len(tag_elements): + match.tags = [tag_element.text for tag_element in tag_elements] + else: + match.tags = self._get_tags(decode_content) + + description_element = html.xpath(self.DESCRIPTION_XPATH) + if len(description_element): + match.description = html2text( + etree.tostring(description_element[-1]).decode("utf8")) + + info = html.xpath(self.INFO_XPATH) + + for element in info: + text = element.text + if self.AUTHORS_PATTERN.search(text): + next_element = element.getnext() + while next_element is not None and next_element.tag != "br": + match.authors.append(next_element.text) + next_element = next_element.getnext() + elif self.PUBLISHER_PATTERN.search(text): + if publisher := element.tail.strip(): + match.publisher = publisher + else: + match.publisher = element.getnext().text + elif self.SUBTITLE_PATTERN.search(text): + match.title = f'{match.title}:{element.tail.strip()}' + elif self.PUBLISHED_DATE_PATTERN.search(text): + match.publishedDate = self._clean_date(element.tail.strip()) + elif self.SERIES_PATTERN.search(text): + match.series = element.getnext().text + elif i_type := self.IDENTIFIERS_PATTERN.search(text): + match.identifiers[i_type.group()] = element.tail.strip() + + return match + + @staticmethod + def _clean_date(date: str) -> str: + """ + Clean up the date string to be in the format YYYY-MM-DD + + Examples of possible patterns: + '2014-7-16', '1988年4月', '1995-04', '2021-8', '2020-12-1', '1996年', + '1972', '2004/11/01', '1959年3月北京第1版第1印' + """ + year = date[:4] + moon = "01" + day = "01" + + if len(date) > 5: + digit = [] + ls = [] + for i in range(5, len(date)): + if date[i].isdigit(): + digit.append(date[i]) + elif digit: + ls.append("".join(digit) if len(digit) == + 2 else f"0{digit[0]}") + digit = [] + if digit: + ls.append("".join(digit) if len(digit) == + 2 else f"0{digit[0]}") + + moon = ls[0] + if len(ls) > 1: + day = ls[1] + + return f"{year}-{moon}-{day}" + + def _get_tags(self, text: str) -> List[str]: + tags = [] + if criteria := self.CRITERIA_PATTERN.search(text): + tags.extend( + item.replace('7:', '') for item in criteria.group().split('|') + if item.startswith('7:')) + + return tags diff --git a/root/app/calibre-web/cps/metadata_provider/google.py b/root/app/calibre-web/cps/metadata_provider/google.py new file mode 100644 index 0000000..150dab5 --- /dev/null +++ b/root/app/calibre-web/cps/metadata_provider/google.py @@ -0,0 +1,129 @@ +# -*- coding: utf-8 -*- + +# This file is part of the Calibre-Web (https://github.com/janeczku/calibre-web) +# Copyright (C) 2021 OzzieIsaacs +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +# Google Books api document: https://developers.google.com/books/docs/v1/using +from typing import Dict, List, Optional +from urllib.parse import quote +from datetime import datetime + +import requests + +from cps import logger +from cps.isoLanguages import get_lang3, get_language_name +from cps.services.Metadata import MetaRecord, MetaSourceInfo, Metadata + +log = logger.create() + + +class Google(Metadata): + __name__ = "Google" + __id__ = "google" + DESCRIPTION = "Google Books" + META_URL = "https://books.google.com/" + BOOK_URL = "https://books.google.com/books?id=" + SEARCH_URL = "https://www.googleapis.com/books/v1/volumes?q=" + ISBN_TYPE = "ISBN_13" + + def search( + self, query: str, generic_cover: str = "", locale: str = "en",**kwargs + ) -> Optional[List[MetaRecord]]: + val = list() + if self.active: + + title_tokens = list(self.get_title_tokens(query, strip_joiners=False)) + if title_tokens: + tokens = [quote(t.encode("utf-8")) for t in title_tokens] + query = "+".join(tokens) + try: + results = requests.get(Google.SEARCH_URL + query) + results.raise_for_status() + except Exception as e: + log.warning(e) + return [] + for result in results.json().get("items", []): + val.append( + self._parse_search_result( + result=result, generic_cover=generic_cover, locale=locale + ) + ) + return val + + def _parse_search_result( + self, result: Dict, generic_cover: str, locale: str + ) -> MetaRecord: + match = MetaRecord( + id=result["id"], + title=result["volumeInfo"]["title"], + authors=result["volumeInfo"].get("authors", []), + url=Google.BOOK_URL + result["id"], + source=MetaSourceInfo( + id=self.__id__, + description=Google.DESCRIPTION, + link=Google.META_URL, + ), + ) + + match.cover = self._parse_cover(result=result, generic_cover=generic_cover) + match.description = result["volumeInfo"].get("description", "") + match.languages = self._parse_languages(result=result, locale=locale) + match.publisher = result["volumeInfo"].get("publisher", "") + try: + datetime.strptime(result["volumeInfo"].get("publishedDate", ""), "%Y-%m-%d") + match.publishedDate = result["volumeInfo"].get("publishedDate", "") + except ValueError: + match.publishedDate = "" + match.rating = result["volumeInfo"].get("averageRating", 0) + match.series, match.series_index = "", 1 + match.tags = result["volumeInfo"].get("categories", []) + + match.identifiers = {"google": match.id} + match = self._parse_isbn(result=result, match=match) + return match + + @staticmethod + def _parse_isbn(result: Dict, match: MetaRecord) -> MetaRecord: + identifiers = result["volumeInfo"].get("industryIdentifiers", []) + for identifier in identifiers: + if identifier.get("type") == Google.ISBN_TYPE: + match.identifiers["isbn"] = identifier.get("identifier") + break + return match + + @staticmethod + def _parse_cover(result: Dict, generic_cover: str) -> str: + if result["volumeInfo"].get("imageLinks"): + cover_url = result["volumeInfo"]["imageLinks"]["thumbnail"] + + # strip curl in cover + cover_url = cover_url.replace("&edge=curl", "") + + # request 800x900 cover image (higher resolution) + cover_url += "&fife=w800-h900" + + return cover_url.replace("http://", "https://") + return generic_cover + + @staticmethod + def _parse_languages(result: Dict, locale: str) -> List[str]: + language_iso2 = result["volumeInfo"].get("language", "") + languages = ( + [get_language_name(locale, get_lang3(language_iso2))] + if language_iso2 + else [] + ) + return languages diff --git a/root/app/calibre-web/cps/metadata_provider/hardcover.py b/root/app/calibre-web/cps/metadata_provider/hardcover.py new file mode 100644 index 0000000..9e36f0c --- /dev/null +++ b/root/app/calibre-web/cps/metadata_provider/hardcover.py @@ -0,0 +1,255 @@ +# -*- coding: utf-8 -*- + +# This file is part of the Calibre-Web (https://github.com/janeczku/calibre-web) +# Copyright (C) 2021 OzzieIsaacs +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +# Hardcover api document: https://Hardcover.gamespot.com/api/documentation +from typing import Dict, List, Optional +from urllib.parse import quote + +import requests +from cps import logger +from cps.services.Metadata import MetaRecord, MetaSourceInfo, Metadata +from importlib import reload + +from flask import g + +log = logger.create() + + +class Hardcover(Metadata): + __name__ = "Hardcover" + __id__ = "hardcover" + DESCRIPTION = "Hardcover Books" + META_URL = "https://hardcover.app" + BASE_URL = "https://api.hardcover.app/v1/graphql" + # SEARCH_QUERY = """{ + # books( + # where: {title: {_eq: "%s"}} + # limit: 10 + # order_by: {users_read_count: desc} + # ) { + # title + # book_series { + # series { + # name + # } + # position + # } + # cached_contributors + # id + # cached_image + # slug + # description + # release_date + # cached_tags + # } + # }""" + SEARCH_QUERY = """query Search($query: String!) { + search(query: $query, query_type: "Book", per_page: 50) { + results + } + } + """ + EDITION_QUERY = """query getEditions($query: Int!) { + books( + where: { id: { _eq: $query } } + order_by: { users_read_count: desc_nulls_last } + ) { + title + slug + id + + book_series { + series { + name + } + position + } + rating + editions( + where: { + _or: [{ reading_format_id: { _neq: 2 } }, { edition_format: { _is_null: true } }] + } + order_by: [{ reading_format_id: desc_nulls_last },{users_count: desc_nulls_last }] + ) { + id + isbn_13 + isbn_10 + title + reading_format_id + contributions { + author { + name + } + } + image { + url + } + language { + code3 + } + publisher { + name + } + release_date + + } + description + cached_tags(path: "Genre") + } + } + """ + HEADERS = { + "Content-Type": "application/json", + } + FORMATS = ["","Physical Book","","","E-Book"] # Map reading_format_id to text equivelant. + + def search( + self, query: str, generic_cover: str = "", locale: str = "en", **kwargs + ) -> Optional[List[MetaRecord]]: + token = kwargs.get("token") + if not token: + log.warning("Hardcover token not set for user") + return None + val = list() + if self.active: + try: + if (token == ""): + raise Exception("Current user does not have Hardcover API token") + else: + edition_seach = query.split(":")[0] == "hardcover-id" + Hardcover.HEADERS["Authorization"] = "Bearer %s" % token + result = requests.post( + Hardcover.BASE_URL, + json={ + "query":Hardcover.SEARCH_QUERY if not edition_seach else Hardcover.EDITION_QUERY, + "variables":{"query":query if not edition_seach else query.split(":")[1]} + }, + headers=Hardcover.HEADERS, + ) + result.raise_for_status() + except Exception as e: + log.warning(e) + return None + if edition_seach: + result = result.json()["data"]["books"][0] + log.debug(result) + val = self._parse_edition_results(result=result, generic_cover=generic_cover, locale=locale) + else: + for result in result.json()["data"]["search"]["results"]["hits"]: + match = self._parse_title_result( + result=result, generic_cover=generic_cover, locale=locale + ) + val.append(match) + return val + + def _parse_title_result( + self, result: Dict, generic_cover: str, locale: str + ) -> MetaRecord: + series = result["document"].get("featured_series",{}).get("series_name", "") + series_index = result["document"].get("featured_series",{}).get("position", "") + match = MetaRecord( + id=result["document"].get("id",""), + title=result["document"].get("title",""), + authors=result["document"].get("author_names", []), + url=self._parse_title_url(result, ""), + source=MetaSourceInfo( + id=self.__id__, + description=Hardcover.DESCRIPTION, + link=Hardcover.META_URL, + ), + series=series, + ) + # TODO Add parse cover function to get better size + match.cover = result["document"]["image"].get("url", generic_cover) + + match.description = result["document"].get("description","") + match.publishedDate = result["document"].get( + "release_date", "") + match.series_index = series_index + match.tags = result["document"].get("genres",[]) + match.identifiers = { + "hardcover-id": match.id, + "hardcover-slug": result["document"].get("slug", "") + } + return match + + def _parse_edition_results( + self, result: Dict, generic_cover: str, locale: str + ) -> MetaRecord: + editions = list() + id = result.get("id","") + for edition in result["editions"]: + match = MetaRecord( + id=id, + title=edition.get("title",""), + authors=self._parse_edition_authors(edition,[]), + url=self._parse_edition_url(edition, ""), + source=MetaSourceInfo( + id=self.__id__, + description=Hardcover.DESCRIPTION, + link=Hardcover.META_URL, + ), + series=result.get("book_series",[{}])[0].get("series",{}).get("name", ""), + ) + # TODO Add parse cover function to get better size + match.cover = (edition.get("image") or {}).get("url", generic_cover) + match.description = result.get("description","") + match.publisher = (edition.get("publisher") or {}).get("name","") + match.publishedDate = edition.get("release_date", "") + match.series_index = result.get("book_series",[{}])[0].get("position", "") + match.tags = self._parse_tags(result,[]) + match.languages = (edition.get("language") or {}).get("code3","") + match.identifiers = { + "hardcover-id": id, + "hardcover-slug": result.get("slug", ""), + "hardcover-edition": edition.get("id",""), + "isbn": (edition.get("isbn_13",edition.get("isbn_10")) or "") + } + match.format = Hardcover.FORMATS[edition.get("reading_format_id",0)] + editions.append(match) + return editions + + @staticmethod + def _parse_title_url(result: Dict, url: str) -> str: + hardcover_slug = result["document"].get("slug", "") + if hardcover_slug: + return f"https://hardcover.app/books/{hardcover_slug}" + return url + + @staticmethod + def _parse_edition_url(edition: Dict, url: str) -> str: + hardcover_edition = edition.get("id", "") + if hardcover_edition: + return f"https://hardcover.app/books/jurassic-park/editions/{hardcover_edition}" + return url + + @staticmethod + def _parse_edition_authors(edition: Dict, authors: List[str]) -> List[str]: + try: + return [author["author"]["name"] for author in edition.get("contributions",[]) if "author" in author and "name" in author["author"]] + except Exception as e: + log.warning(e) + return authors + + @staticmethod + def _parse_tags(result: Dict, tags: List[str]) -> List[str]: + try: + return [item["tag"] for item in result["cached_tags"] if "tag" in item] + except Exception as e: + log.warning(e) + return tags diff --git a/root/app/calibre-web/cps/metadata_provider/lubimyczytac.py b/root/app/calibre-web/cps/metadata_provider/lubimyczytac.py new file mode 100644 index 0000000..003a732 --- /dev/null +++ b/root/app/calibre-web/cps/metadata_provider/lubimyczytac.py @@ -0,0 +1,359 @@ +# -*- coding: utf-8 -*- +# This file is part of the Calibre-Web (https://github.com/janeczku/calibre-web) +# Copyright (C) 2021 OzzieIsaacs +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +import datetime +import json +import re +from multiprocessing.pool import ThreadPool +from typing import List, Optional, Tuple, Union +from urllib.parse import quote + +import requests +from dateutil import parser +from html2text import HTML2Text +from lxml.html import HtmlElement, fromstring, tostring +from markdown2 import Markdown + +from cps import logger +from cps.isoLanguages import get_language_name +from cps.services.Metadata import MetaRecord, MetaSourceInfo, Metadata + +log = logger.create() + +SYMBOLS_TO_TRANSLATE = ( + "öÖüÜóÓőŐúÚéÉáÁűŰíÍąĄćĆęĘłŁńŃóÓśŚźŹżŻ", + "oOuUoOoOuUeEaAuUiIaAcCeElLnNoOsSzZzZ", +) +SYMBOL_TRANSLATION_MAP = dict( + [(ord(a), ord(b)) for (a, b) in zip(*SYMBOLS_TO_TRANSLATE)] +) + + +def get_int_or_float(value: str) -> Union[int, float]: + number_as_float = float(value) + number_as_int = int(number_as_float) + return number_as_int if number_as_float == number_as_int else number_as_float + + +def strip_accents(s: Optional[str]) -> Optional[str]: + return s.translate(SYMBOL_TRANSLATION_MAP) if s is not None else s + + +def sanitize_comments_html(html: str) -> str: + text = html2text(html) + md = Markdown() + html = md.convert(text) + return html + + +def html2text(html: str) -> str: + # replace tags with as becomes emphasis in html2text + if isinstance(html, bytes): + html = html.decode("utf-8") + html = re.sub( + r"<\s*(?P/?)\s*[uU]\b(?P[^>]*)>", + r"<\gspan\g>", + html, + ) + h2t = HTML2Text() + h2t.body_width = 0 + h2t.single_line_break = True + h2t.emphasis_mark = "*" + return h2t.handle(html) + + +class LubimyCzytac(Metadata): + __name__ = "LubimyCzytac.pl" + __id__ = "lubimyczytac" + + BASE_URL = "https://lubimyczytac.pl" + + BOOK_SEARCH_RESULT_XPATH = ( + "*//div[@class='listSearch']//div[@class='authorAllBooks__single']" + ) + SINGLE_BOOK_RESULT_XPATH = ".//div[contains(@class,'authorAllBooks__singleText')]" + TITLE_PATH = "/div/a[contains(@class,'authorAllBooks__singleTextTitle')]" + TITLE_TEXT_PATH = f"{TITLE_PATH}//text()" + URL_PATH = f"{TITLE_PATH}/@href" + AUTHORS_PATH = "/div/a[contains(@href,'autor')]//text()" + + SIBLINGS = "/following-sibling::dd" + + CONTAINER = "//section[@class='container book']" + PUBLISHER = f"{CONTAINER}//dt[contains(text(),'Wydawnictwo:')]{SIBLINGS}/a/text()" + LANGUAGES = f"{CONTAINER}//dt[contains(text(),'Język:')]{SIBLINGS}/text()" + DESCRIPTION = f"{CONTAINER}//div[@class='collapse-content']" + SERIES = f"{CONTAINER}//span/a[contains(@href,'/cykl/')]/text()" + TRANSLATOR = f"{CONTAINER}//dt[contains(text(),'Tłumacz:')]{SIBLINGS}/a/text()" + + DETAILS = "//div[@id='book-details']" + PUBLISH_DATE = "//dt[contains(@title,'Data pierwszego wydania" + FIRST_PUBLISH_DATE = f"{DETAILS}{PUBLISH_DATE} oryginalnego')]{SIBLINGS}[1]/text()" + FIRST_PUBLISH_DATE_PL = f"{DETAILS}{PUBLISH_DATE} polskiego')]{SIBLINGS}[1]/text()" + TAGS = "//a[contains(@href,'/ksiazki/t/')]/text()" # "//nav[@aria-label='breadcrumbs']//a[contains(@href,'/ksiazki/k/')]/span/text()" + + + RATING = "//meta[@property='books:rating:value']/@content" + COVER = "//meta[@property='og:image']/@content" + ISBN = "//meta[@property='books:isbn']/@content" + META_TITLE = "//meta[@property='og:description']/@content" + + SUMMARY = "//script[@type='application/ld+json']//text()" + + def search( + self, query: str, generic_cover: str = "", locale: str = "en",**kwargs + ) -> Optional[List[MetaRecord]]: + if self.active: + try: + result = requests.get(self._prepare_query(title=query)) + result.raise_for_status() + except Exception as e: + log.warning(e) + return None + root = fromstring(result.text) + lc_parser = LubimyCzytacParser(root=root, metadata=self) + matches = lc_parser.parse_search_results() + if matches: + with ThreadPool(processes=10) as pool: + final_matches = pool.starmap( + lc_parser.parse_single_book, + [(match, generic_cover, locale) for match in matches], + ) + return final_matches + return matches + + def _prepare_query(self, title: str) -> str: + query = "" + characters_to_remove = r"\?()\/" + pattern = "[" + characters_to_remove + "]" + title = re.sub(pattern, "", title) + title = title.replace("_", " ") + if '"' in title or ",," in title: + title = title.split('"')[0].split(",,")[0] + + if "/" in title: + title_tokens = [ + token for token in title.lower().split(" ") if len(token) > 1 + ] + else: + title_tokens = list(self.get_title_tokens(title, strip_joiners=False)) + if title_tokens: + tokens = [quote(t.encode("utf-8")) for t in title_tokens] + query = query + "%20".join(tokens) + if not query: + return "" + return f"{LubimyCzytac.BASE_URL}/szukaj/ksiazki?phrase={query}" + + +class LubimyCzytacParser: + PAGES_TEMPLATE = "

Książka ma {0} stron(y).

" + TRANSLATOR_TEMPLATE = "

Tłumacz: {0}

" + PUBLISH_DATE_TEMPLATE = "

Data pierwszego wydania: {0}

" + PUBLISH_DATE_PL_TEMPLATE = ( + "

Data pierwszego wydania w Polsce: {0}

" + ) + + def __init__(self, root: HtmlElement, metadata: Metadata) -> None: + self.root = root + self.metadata = metadata + + def parse_search_results(self) -> List[MetaRecord]: + matches = [] + results = self.root.xpath(LubimyCzytac.BOOK_SEARCH_RESULT_XPATH) + for result in results: + title = self._parse_xpath_node( + root=result, + xpath=f"{LubimyCzytac.SINGLE_BOOK_RESULT_XPATH}" + f"{LubimyCzytac.TITLE_TEXT_PATH}", + ) + + book_url = self._parse_xpath_node( + root=result, + xpath=f"{LubimyCzytac.SINGLE_BOOK_RESULT_XPATH}" + f"{LubimyCzytac.URL_PATH}", + ) + authors = self._parse_xpath_node( + root=result, + xpath=f"{LubimyCzytac.SINGLE_BOOK_RESULT_XPATH}" + f"{LubimyCzytac.AUTHORS_PATH}", + take_first=False, + ) + if not all([title, book_url, authors]): + continue + matches.append( + MetaRecord( + id=book_url.replace(f"/ksiazka/", "").split("/")[0], + title=title, + authors=[strip_accents(author) for author in authors], + url=LubimyCzytac.BASE_URL + book_url, + source=MetaSourceInfo( + id=self.metadata.__id__, + description=self.metadata.__name__, + link=LubimyCzytac.BASE_URL, + ), + ) + ) + return matches + + def parse_single_book( + self, match: MetaRecord, generic_cover: str, locale: str + ) -> MetaRecord: + try: + response = requests.get(match.url) + response.raise_for_status() + except Exception as e: + log.warning(e) + return None + self.root = fromstring(response.text) + match.cover = self._parse_cover(generic_cover=generic_cover) + match.description = self._parse_description() + match.languages = self._parse_languages(locale=locale) + match.publisher = self._parse_publisher() + match.publishedDate = self._parse_from_summary(attribute_name="datePublished") + match.rating = self._parse_rating() + match.series, match.series_index = self._parse_series() + match.tags = self._parse_tags() + match.identifiers = { + "isbn": self._parse_isbn(), + "lubimyczytac": match.id, + } + return match + + def _parse_xpath_node( + self, + xpath: str, + root: HtmlElement = None, + take_first: bool = True, + strip_element: bool = True, + ) -> Optional[Union[str, List[str]]]: + root = root if root is not None else self.root + node = root.xpath(xpath) + if not node: + return None + return ( + (node[0].strip() if strip_element else node[0]) + if take_first + else [x.strip() for x in node] + ) + + def _parse_cover(self, generic_cover) -> Optional[str]: + return ( + self._parse_xpath_node(xpath=LubimyCzytac.COVER, take_first=True) + or generic_cover + ) + + def _parse_publisher(self) -> Optional[str]: + return self._parse_xpath_node(xpath=LubimyCzytac.PUBLISHER, take_first=True) + + def _parse_languages(self, locale: str) -> List[str]: + languages = list() + lang = self._parse_xpath_node(xpath=LubimyCzytac.LANGUAGES, take_first=True) + if lang: + if "polski" in lang: + languages.append("pol") + if "angielski" in lang: + languages.append("eng") + return [get_language_name(locale, language) for language in languages] + + def _parse_series(self) -> Tuple[Optional[str], Optional[Union[float, int]]]: + series_index = 0 + series = self._parse_xpath_node(xpath=LubimyCzytac.SERIES, take_first=True) + if series: + if "tom " in series: + series_name, series_info = series.split(" (tom ", 1) + series_info = series_info.replace(" ", "").replace(")", "") + # Check if book is not a bundle, i.e. chapter 1-3 + if "-" in series_info: + series_info = series_info.split("-", 1)[0] + if series_info.replace(".", "").isdigit() is True: + series_index = get_int_or_float(series_info) + return series_name, series_index + return None, None + + def _parse_tags(self) -> List[str]: + tags = self._parse_xpath_node(xpath=LubimyCzytac.TAGS, take_first=False) + if tags: + return [ + strip_accents(w.replace(", itd.", " itd.")) + for w in tags + if isinstance(w, str) + ] + return None + + def _parse_from_summary(self, attribute_name: str) -> Optional[str]: + value = None + summary_text = self._parse_xpath_node(xpath=LubimyCzytac.SUMMARY) + if summary_text: + data = json.loads(summary_text) + value = data.get(attribute_name) + return value.strip() if value is not None else value + + def _parse_rating(self) -> Optional[str]: + rating = self._parse_xpath_node(xpath=LubimyCzytac.RATING) + return round(float(rating.replace(",", ".")) / 2) if rating else rating + + def _parse_date(self, xpath="first_publish") -> Optional[datetime.datetime]: + options = { + "first_publish": LubimyCzytac.FIRST_PUBLISH_DATE, + "first_publish_pl": LubimyCzytac.FIRST_PUBLISH_DATE_PL, + } + date = self._parse_xpath_node(xpath=options.get(xpath)) + return parser.parse(date) if date else None + + def _parse_isbn(self) -> Optional[str]: + return self._parse_xpath_node(xpath=LubimyCzytac.ISBN) + + def _parse_description(self) -> str: + description = "" + description_node = self._parse_xpath_node( + xpath=LubimyCzytac.DESCRIPTION, strip_element=False + ) + if description_node is not None: + for source in self.root.xpath('//p[@class="source"]'): + source.getparent().remove(source) + description = tostring(description_node, method="html") + description = sanitize_comments_html(description) + + else: + description_node = self._parse_xpath_node(xpath=LubimyCzytac.META_TITLE) + if description_node is not None: + description = description_node + description = sanitize_comments_html(description) + description = self._add_extra_info_to_description(description=description) + return description + + def _add_extra_info_to_description(self, description: str) -> str: + pages = self._parse_from_summary(attribute_name="numberOfPages") + if pages: + description += LubimyCzytacParser.PAGES_TEMPLATE.format(pages) + + first_publish_date = self._parse_date() + if first_publish_date: + description += LubimyCzytacParser.PUBLISH_DATE_TEMPLATE.format( + first_publish_date.strftime("%d.%m.%Y") + ) + + first_publish_date_pl = self._parse_date(xpath="first_publish_pl") + if first_publish_date_pl: + description += LubimyCzytacParser.PUBLISH_DATE_PL_TEMPLATE.format( + first_publish_date_pl.strftime("%d.%m.%Y") + ) + translator = self._parse_xpath_node(xpath=LubimyCzytac.TRANSLATOR) + if translator: + description += LubimyCzytacParser.TRANSLATOR_TEMPLATE.format(translator) + + + return description diff --git a/root/app/calibre-web/cps/metadata_provider/scholar.py b/root/app/calibre-web/cps/metadata_provider/scholar.py new file mode 100644 index 0000000..b3bfbb7 --- /dev/null +++ b/root/app/calibre-web/cps/metadata_provider/scholar.py @@ -0,0 +1,83 @@ +# -*- coding: utf-8 -*- + +# This file is part of the Calibre-Web (https://github.com/janeczku/calibre-web) +# Copyright (C) 2021 OzzieIsaacs +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +import itertools +from typing import Dict, List, Optional +from urllib.parse import quote, unquote + +try: + from fake_useragent.errors import FakeUserAgentError +except (ImportError): + FakeUserAgentError = BaseException +try: + from scholarly import scholarly +except FakeUserAgentError: + raise ImportError("No module named 'scholarly'") + +from cps import logger +from cps.services.Metadata import MetaRecord, MetaSourceInfo, Metadata + +log = logger.create() + + +class scholar(Metadata): + __name__ = "Google Scholar" + __id__ = "googlescholar" + META_URL = "https://scholar.google.com/" + + def search( + self, query: str, generic_cover: str = "", locale: str = "en",**kwargs + ) -> Optional[List[MetaRecord]]: + val = list() + if self.active: + title_tokens = list(self.get_title_tokens(query, strip_joiners=False)) + if title_tokens: + tokens = [quote(t.encode("utf-8")) for t in title_tokens] + query = " ".join(tokens) + try: + scholarly.set_timeout(20) + scholarly.set_retries(2) + scholar_gen = itertools.islice(scholarly.search_pubs(query), 10) + except Exception as e: + log.warning(e) + return list() + for result in scholar_gen: + match = self._parse_search_result( + result=result, generic_cover="", locale=locale + ) + val.append(match) + return val + + def _parse_search_result( + self, result: Dict, generic_cover: str, locale: str + ) -> MetaRecord: + match = MetaRecord( + id=result.get("pub_url", result.get("eprint_url", "")), + title=result["bib"].get("title"), + authors=result["bib"].get("author", []), + url=result.get("pub_url", result.get("eprint_url", "")), + source=MetaSourceInfo( + id=self.__id__, description=self.__name__, link=scholar.META_URL + ), + ) + + match.cover = result.get("image", {}).get("original_url", generic_cover) + match.description = unquote(result["bib"].get("abstract", "")) + match.publisher = result["bib"].get("venue", "") + match.publishedDate = result["bib"].get("pub_year") + "-01-01" + match.identifiers = {"scholar": match.id} + return match diff --git a/root/app/calibre-web/cps/search_metadata.py b/root/app/calibre-web/cps/search_metadata.py new file mode 100644 index 0000000..9700c9c --- /dev/null +++ b/root/app/calibre-web/cps/search_metadata.py @@ -0,0 +1,139 @@ +# -*- coding: utf-8 -*- + +# This file is part of the Calibre-Web (https://github.com/janeczku/calibre-web) +# Copyright (C) 2021 OzzieIsaacs +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import concurrent.futures +import importlib +import inspect +import json +import os +import sys + +from flask import Blueprint, request, url_for, make_response, jsonify +from .cw_login import current_user +from flask_babel import get_locale +from sqlalchemy.exc import InvalidRequestError, OperationalError +from sqlalchemy.orm.attributes import flag_modified + +from cps.services.Metadata import Metadata +from . import constants, logger, ub, web_server +from .usermanagement import user_login_required + + +meta = Blueprint("metadata", __name__) + +log = logger.create() + +try: + from dataclasses import asdict +except ImportError: + log.info('*** "dataclasses" is needed for calibre-web to run. Please install it using pip: "pip install dataclasses" ***') + print('*** "dataclasses" is needed for calibre-web to run. Please install it using pip: "pip install dataclasses" ***') + web_server.stop(True) + sys.exit(6) + +new_list = list() +meta_dir = os.path.join(constants.BASE_DIR, "cps", "metadata_provider") +modules = os.listdir(os.path.join(constants.BASE_DIR, "cps", "metadata_provider")) +for f in modules: + if os.path.isfile(os.path.join(meta_dir, f)) and not f.endswith("__init__.py"): + a = os.path.basename(f)[:-3] + try: + importlib.import_module("cps.metadata_provider." + a) + new_list.append(a) + except (IndentationError, SyntaxError) as e: + log.error("Syntax error for metadata source: {} - {}".format(a, e)) + except ImportError as e: + log.debug("Import error for metadata source: {} - {}".format(a, e)) + + +def list_classes(provider_list): + classes = list() + for element in provider_list: + for name, obj in inspect.getmembers( + sys.modules["cps.metadata_provider." + element] + ): + if ( + inspect.isclass(obj) + and name != "Metadata" + and issubclass(obj, Metadata) + ): + classes.append(obj()) + return classes + + +cl = list_classes(new_list) + + +@meta.route("/metadata/provider") +@user_login_required +def metadata_provider(): + active = current_user.view_settings.get("metadata", {}) + provider = list() + for c in cl: + ac = active.get(c.__id__, True) + provider.append( + {"name": c.__name__, "active": ac, "initial": ac, "id": c.__id__} + ) + return make_response(jsonify(provider)) + + +@meta.route("/metadata/provider", methods=["POST"]) +@meta.route("/metadata/provider/", methods=["POST"]) +@user_login_required +def metadata_change_active_provider(prov_name): + new_state = request.get_json() + active = current_user.view_settings.get("metadata", {}) + active[new_state["id"]] = new_state["value"] + current_user.view_settings["metadata"] = active + try: + try: + flag_modified(current_user, "view_settings") + except AttributeError: + pass + ub.session.commit() + except (InvalidRequestError, OperationalError): + log.error("Invalid request received: {}".format(request)) + return "Invalid request", 400 + if "initial" in new_state and prov_name: + data = [] + provider = next((c for c in cl if c.__id__ == prov_name), None) + if provider is not None: + data = provider.search(new_state.get("query", "")) + return make_response(jsonify([asdict(x) for x in data])) + return "" + + +@meta.route("/metadata/search", methods=["POST"]) +@user_login_required +def metadata_search(): + query = request.form.to_dict().get("query") + data = list() + active = current_user.view_settings.get("metadata", {}) + locale = get_locale() + if query: + static_cover = url_for("static", filename="generic_cover.jpg") + # ret = cl[0].search(query, static_cover, locale) + with concurrent.futures.ThreadPoolExecutor(max_workers=5) as executor: + meta = { + executor.submit(c.search, query, static_cover, locale, token=getattr(current_user,f'{c.__id__}_token',None)): c + for c in cl + if active.get(c.__id__, True) + } + for future in concurrent.futures.as_completed(meta): + data.extend([asdict(x) for x in future.result() if x]) + return make_response(jsonify(data)) diff --git a/root/app/calibre-web/cps/services/Metadata.py b/root/app/calibre-web/cps/services/Metadata.py new file mode 100644 index 0000000..d975092 --- /dev/null +++ b/root/app/calibre-web/cps/services/Metadata.py @@ -0,0 +1,112 @@ +# -*- coding: utf-8 -*- + +# This file is part of the Calibre-Web (https://github.com/janeczku/calibre-web) +# Copyright (C) 2021 OzzieIsaacs +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +import abc +import dataclasses +import os +import re +from typing import Dict, Generator, List, Optional, Union + +from cps import constants + + +@dataclasses.dataclass +class MetaSourceInfo: + id: str + description: str + link: str + + +@dataclasses.dataclass +class MetaRecord: + id: Union[str, int] + title: str + authors: List[str] + url: str + source: MetaSourceInfo + cover: str = os.path.join(constants.STATIC_DIR, 'generic_cover.jpg') + description: Optional[str] = "" + series: Optional[str] = None + series_index: Optional[Union[int, float]] = 0 + identifiers: Dict[str, Union[str, int]] = dataclasses.field(default_factory=dict) + publisher: Optional[str] = None + publishedDate: Optional[str] = None + rating: Optional[int] = 0 + languages: Optional[List[str]] = dataclasses.field(default_factory=list) + tags: Optional[List[str]] = dataclasses.field(default_factory=list) + format: Optional[str] = None + + +class Metadata: + __name__ = "Generic" + __id__ = "generic" + + def __init__(self): + self.active = True + + def set_status(self, state): + self.active = state + + @abc.abstractmethod + def search( + self, query: str, generic_cover: str = "", locale: str = "en" + ) -> Optional[List[MetaRecord]]: + pass + + @staticmethod + def get_title_tokens( + title: str, strip_joiners: bool = True + ) -> Generator[str, None, None]: + """ + Taken from calibre source code + It's a simplified (cut out what is unnecessary) version of + https://github.com/kovidgoyal/calibre/blob/99d85b97918625d172227c8ffb7e0c71794966c0/ + src/calibre/ebooks/metadata/sources/base.py#L363-L367 + (src/calibre/ebooks/metadata/sources/base.py - lines 363-398) + """ + title_patterns = [ + (re.compile(pat, re.IGNORECASE), repl) + for pat, repl in [ + # Remove things like: (2010) (Omnibus) etc. + ( + r"(?i)[({\[](\d{4}|omnibus|anthology|hardcover|" + r"audiobook|audio\scd|paperback|turtleback|" + r"mass\s*market|edition|ed\.)[\])}]", + "", + ), + # Remove any strings that contain the substring edition inside + # parentheses + (r"(?i)[({\[].*?(edition|ed.).*?[\]})]", ""), + # Remove commas used a separators in numbers + (r"(\d+),(\d+)", r"\1\2"), + # Remove hyphens only if they have whitespace before them + (r"(\s-)", " "), + # Replace other special chars with a space + (r"""[:,;!@$%^&*(){}.`~"\s\[\]/]《》「」“”""", " "), + ] + ] + + for pat, repl in title_patterns: + title = pat.sub(repl, title) + + tokens = title.split() + for token in tokens: + token = token.strip().strip('"').strip("'") + if token and ( + not strip_joiners or token.lower() not in ("a", "and", "the", "&") + ): + yield token diff --git a/root/app/calibre-web/cps/services/__init__.py b/root/app/calibre-web/cps/services/__init__.py new file mode 100644 index 0000000..000a6f9 --- /dev/null +++ b/root/app/calibre-web/cps/services/__init__.py @@ -0,0 +1,56 @@ +# -*- coding: utf-8 -*- + +# This file is part of the Calibre-Web (https://github.com/janeczku/calibre-web) +# Copyright (C) 2019 pwr +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +from .. import logger + +log = logger.create() + +try: + from . import goodreads_support +except ImportError as err: + log.debug("Cannot import goodreads, showing authors-metadata will not work: %s", err) + goodreads_support = None + + +try: + from . import simpleldap as ldap + from .simpleldap import ldapVersion +except ImportError as err: + log.debug("Cannot import simpleldap, logging in with ldap will not work: %s", err) + ldap = None + ldapVersion = None + +try: + from . import SyncToken as SyncToken + kobo = True +except ImportError as err: + log.debug("Cannot import SyncToken, syncing books with Kobo Devices will not work: %s", err) + kobo = None + SyncToken = None + +try: + from . import gmail +except ImportError as err: + log.debug("Cannot import gmail, sending books via Gmail Oauth2 Verification will not work: %s", err) + gmail = None + +try: + from . import hardcover +except ImportError as err: + log.debug("Cannot import hardcover, syncing Kobo read progress to Hardcover will not work: %s", err) + hardcover = None \ No newline at end of file diff --git a/root/app/calibre-web/cps/services/hardcover.py b/root/app/calibre-web/cps/services/hardcover.py new file mode 100644 index 0000000..040289f --- /dev/null +++ b/root/app/calibre-web/cps/services/hardcover.py @@ -0,0 +1,232 @@ +# -*- coding: utf-8 -*- + +# This file is part of the Calibre-Web (https://github.com/janeczku/calibre-web) +# Copyright (C) 2018-2019 OzzieIsaacs, pwr +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +from datetime import datetime +import requests + +from .. import logger + +log = logger.create() + +GRAPHQL_ENDPOINT = "https://api.hardcover.app/v1/graphql" + +USER_BOOK_FRAGMENT = """ + fragment userBookFragment on user_books { + id + status_id + book_id + book { + slug + title + } + edition { + id + pages + } + user_book_reads(order_by: {started_at: desc}, limit: 1) { + id + started_at + finished_at + edition_id + progress_pages + } + }""" + +class HardcoverClient: + def __init__(self, token): + self.endpoint = GRAPHQL_ENDPOINT + self.headers = { + "Content-Type": "application/json", + "Authorization" : f"Bearer {token}" + } + self.privacy = self.get_privacy() + + def get_privacy(self): + query = """ + { + me { + account_privacy_setting_id + } + }""" + response = self.execute(query) + return (response.get("me")[0] or [{}]).get("account_privacy_setting_id",1) + + def get_user_book(self, ids): + query = "" + variables={} + if "hardcover-edition" in ids: + query = """ + query ($query: Int) { + me { + user_books(where: {edition_id: {_eq: $query}}) { + ...userBookFragment + } + } + }""" + variables["query"] = ids["hardcover-edition"] + elif "hardcover-id" in ids: + query = """ + query ($query: Int) { + me { + user_books(where: {book: {id: {_eq: $query}}}) { + ...userBookFragment + } + } + }""" + variables["query"] = ids["hardcover-id"] + elif "hardcover-slug" in ids: + query = """ + query ($slug: String!) { + me { + user_books(where: {book: {slug: {_eq: $query}}}) { + ...userBookFragment + } + } + }""" + variables["query"] = ids["hardcover-slug"] + query += USER_BOOK_FRAGMENT + response = self.execute(query,variables) + return next(iter(response.get("me")[0].get("user_books")),None) + + + # TODO Add option for autocreate if missing books instead of forcing it. + def update_reading_progress(self, identifiers, progress_percent): + ids = self.parse_identifiers(identifiers) + book = self.get_user_book(ids) + if not book: + book = self.add_book(ids, status=2) + if book.get("status_id") is not 2: + book = self.change_book_status(book, 2) + pages = round(book.get("edition",{}).get("pages",0)) + if pages: + pages_read = pages * (progress_percent / 100) + read = next(iter(book.get("user_book_reads")),None) + if not read: + read = self.add_read(book, pages_read) + mutation = """ + mutation ($readId: Int!, $pages: Int, $editionId: Int, $startedAt: date, $finishedAt: date) { + update_user_book_read(id: $readId, object: { + progress_pages: $pages, + edition_id: $editionId, + started_at: $startedAt, + finished_at: $finishedAt + }) { + id + } + }""" + variables = { + "readId": int(read.get("id")), + "pages": pages_read, + "editionId": int(book.get("edition").get("id")), + "startedAt":read.get("started_at",datetime.now().strftime("%Y-%m-%d")), + "finishedAt": datetime.now().strftime("%Y-%m-%d") if progress_percent is 100 else None + } + if progress_percent is 100: + self.change_book_status(book, 3) + return self.execute(query=mutation, variables=variables) + return + + def change_book_status(self, book, status): + mutation = """ + mutation ($id:Int!, $status_id: Int!) { + update_user_book(id: $id, object: {status_id: $status_id}) { + error + user_book { + ...userBookFragment + } + } + }""" + USER_BOOK_FRAGMENT + variables = { + "id":book.get("id"), + "status_id":status + } + response = self.execute(query=mutation, variables=variables) + return response.get("update_user_book",{}).get("user_book",{}) + + def add_book(self, identifiers, status=1): + ids = self.parse_identifiers(identifiers) + mutation = """ + mutation ($object: UserBookCreateInput!) { + insert_user_book(object: $object) { + error + user_book { + ...userBookFragment + } + } + }""" + USER_BOOK_FRAGMENT + variables = { + "object": { + "book_id":int(ids.get("hardcover-id")), + "edition_id":int(ids.get("hardcover-edition")) if ids.get("hardcover-edition") else None, + "status_id": status, + "privacy_setting_id": self.privacy + } + } + response = self.execute(query=mutation, variables=variables) + return response.get("insert_user_book",{}).get("user_book",{}) + + def add_read(self, book, pages=0): + mutation = """ + mutation ($id: Int!, $pages: Int, $editionId: Int, $startedAt: date) { + insert_user_book_read(user_book_id: $id, user_book_read: { + progress_pages: $pages, + edition_id: $editionId, + started_at: $startedAt, + }) { + error + user_book_read { + id + started_at + finished_at + edition_id + progress_pages + } + } + }""" + variables = { + "id":int(book.get("id")), + "editionId":int(book.get("edition").get("id")) if book.get("edition").get("id") else None, + "pages": pages, + "startedAt": datetime.now().strftime("%Y-%m-%d") + } + response = self.execute(query=mutation, variables=variables) + return response.get("insert_user_book_read").get("user_book_read") + + def parse_identifiers(self, identifiers): + if type(identifiers) != dict: + return {id.type:id.val for id in identifiers if "hardcover" in id.type} + return identifiers + + def execute(self, query, variables=None): + payload = { + "query": query, + "variables": variables or {} + } + + + response = requests.post(self.endpoint, json=payload, headers=self.headers) + try: + response.raise_for_status() + except requests.exceptions.HTTPError as e: + raise Exception(f"HTTP error occurred: {e}") + + result = response.json() + if "errors" in result: + raise Exception(f"GraphQL error: {result['errors']}") + + return result.get("data", {}) diff --git a/root/app/calibre-web/cps/shelf.py b/root/app/calibre-web/cps/shelf.py new file mode 100644 index 0000000..94deaa7 --- /dev/null +++ b/root/app/calibre-web/cps/shelf.py @@ -0,0 +1,492 @@ +# -*- coding: utf-8 -*- + +# This file is part of the Calibre-Web (https://github.com/janeczku/calibre-web) +# Copyright (C) 2018-2019 OzzieIsaacs, cervinko, jkrehm, bodybybuddha, ok11, +# andy29485, idalin, Kyosfonica, wuqi, Kennyl, lemmsh, +# falgh1, grunjol, csitko, ytils, xybydy, trasba, vrabe, +# ruben-herold, marblepebble, JackED42, SiphonSquirrel, +# apetresc, nanu-c, mutschler +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import sys +from datetime import datetime, timezone + +from flask import Blueprint, flash, redirect, request, url_for, abort +from flask_babel import gettext as _ +from .cw_login import current_user +from sqlalchemy.exc import InvalidRequestError, OperationalError +from sqlalchemy.sql.expression import func, true + +from . import calibre_db, config, db, logger, ub +from .render_template import render_title_template +from .usermanagement import login_required_if_no_ano, user_login_required +from .services import hardcover +log = logger.create() + +shelf = Blueprint('shelf', __name__) + + +@shelf.route("/shelf/add//", methods=["POST"]) +@user_login_required +def add_to_shelf(shelf_id, book_id): + xhr = request.headers.get('X-Requested-With') == 'XMLHttpRequest' + shelf = ub.session.query(ub.Shelf).filter(ub.Shelf.id == shelf_id).first() + if shelf is None: + log.error("Invalid shelf specified: %s", shelf_id) + if not xhr: + flash(_("Invalid shelf specified"), category="error") + return redirect(url_for('web.index')) + return "Invalid shelf specified", 400 + + if not check_shelf_edit_permissions(shelf): + if not xhr: + flash(_("Sorry you are not allowed to add a book to that shelf"), category="error") + return redirect(url_for('web.index')) + return "Sorry you are not allowed to add a book to the that shelf", 403 + + book_in_shelf = ub.session.query(ub.BookShelf).filter(ub.BookShelf.shelf == shelf_id, + ub.BookShelf.book_id == book_id).first() + if book_in_shelf: + log.error("Book %s is already part of %s", book_id, shelf) + if not xhr: + flash(_("Book is already part of the shelf: %(shelfname)s", shelfname=shelf.name), category="error") + return redirect(url_for('web.index')) + return "Book is already part of the shelf: %s" % shelf.name, 400 + + maxOrder = ub.session.query(func.max(ub.BookShelf.order)).filter(ub.BookShelf.shelf == shelf_id).first() + if maxOrder[0] is None: + maxOrder = 0 + else: + maxOrder = maxOrder[0] + + book = calibre_db.session.query(db.Books).filter(db.Books.id == book_id).one_or_none() + if not book: + log.error("Invalid Book Id: %s. Could not be added to shelf %s", book_id, shelf.name) + if not xhr: + flash(_("%(book_id)s is a invalid Book Id. Could not be added to Shelf", book_id=book_id), + category="error") + return redirect(url_for('web.index')) + return "%s is a invalid Book Id. Could not be added to Shelf" % book_id, 400 + + shelf.books.append(ub.BookShelf(shelf=shelf.id, book_id=book_id, order=maxOrder + 1)) + shelf.last_modified = datetime.now(timezone.utc) + try: + ub.session.merge(shelf) + ub.session.commit() + except (OperationalError, InvalidRequestError) as e: + ub.session.rollback() + log.error_or_exception("Settings Database error: {}".format(e)) + flash(_("Oops! Database Error: %(error)s.", error=e.orig), category="error") + if "HTTP_REFERER" in request.environ: + return redirect(request.environ["HTTP_REFERER"]) + else: + return redirect(url_for('web.index')) + if not xhr: + log.debug("Book has been added to shelf: {}".format(shelf.name)) + flash(_("Book has been added to shelf: %(sname)s", sname=shelf.name), category="success") + if "HTTP_REFERER" in request.environ: + return redirect(request.environ["HTTP_REFERER"]) + else: + return redirect(url_for('web.index')) + if shelf.kobo_sync and config.config_hardcover_sync and bool(hardcover): + hardcoverClient = hardcover.HardcoverClient(current_user.hardcover_token) + hardcoverClient.add_book(book.identifiers) + return "", 204 + + +@shelf.route("/shelf/massadd/", methods=["POST"]) +@user_login_required +def search_to_shelf(shelf_id): + shelf = ub.session.query(ub.Shelf).filter(ub.Shelf.id == shelf_id).first() + if shelf is None: + log.error("Invalid shelf specified: {}".format(shelf_id)) + flash(_("Invalid shelf specified"), category="error") + return redirect(url_for('web.index')) + + if not check_shelf_edit_permissions(shelf): + log.warning("You are not allowed to add a book to the shelf".format(shelf.name)) + flash(_("You are not allowed to add a book to the shelf"), category="error") + return redirect(url_for('web.index')) + + if current_user.id in ub.searched_ids and ub.searched_ids[current_user.id]: + books_for_shelf = list() + books_in_shelf = ub.session.query(ub.BookShelf).filter(ub.BookShelf.shelf == shelf_id).all() + if books_in_shelf: + book_ids = list() + for book_id in books_in_shelf: + book_ids.append(book_id.book_id) + for searchid in ub.searched_ids[current_user.id]: + if searchid not in book_ids: + books_for_shelf.append(searchid) + else: + books_for_shelf = ub.searched_ids[current_user.id] + + if not books_for_shelf: + log.error("Books are already part of {}".format(shelf.name)) + flash(_("Books are already part of the shelf: %(name)s", name=shelf.name), category="error") + return redirect(url_for('web.index')) + + maxOrder = ub.session.query(func.max(ub.BookShelf.order)).filter(ub.BookShelf.shelf == shelf_id).first()[0] or 0 + + for book in books_for_shelf: + maxOrder += 1 + shelf.books.append(ub.BookShelf(shelf=shelf.id, book_id=book, order=maxOrder)) + shelf.last_modified = datetime.now(timezone.utc) + try: + ub.session.merge(shelf) + ub.session.commit() + flash(_("Books have been added to shelf: %(sname)s", sname=shelf.name), category="success") + except (OperationalError, InvalidRequestError) as e: + ub.session.rollback() + log.error_or_exception("Settings Database error: {}".format(e)) + flash(_("Oops! Database Error: %(error)s.", error=e.orig), category="error") + else: + log.error("Could not add books to shelf: {}".format(shelf.name)) + flash(_("Could not add books to shelf: %(sname)s", sname=shelf.name), category="error") + return redirect(url_for('web.index')) + + +@shelf.route("/shelf/remove//", methods=["POST"]) +@user_login_required +def remove_from_shelf(shelf_id, book_id): + xhr = request.headers.get('X-Requested-With') == 'XMLHttpRequest' + shelf = ub.session.query(ub.Shelf).filter(ub.Shelf.id == shelf_id).first() + if shelf is None: + log.error("Invalid shelf specified: {}".format(shelf_id)) + if not xhr: + return redirect(url_for('web.index')) + return "Invalid shelf specified", 400 + + # if shelf is public and use is allowed to edit shelfs, or if shelf is private and user is owner + # allow editing shelfs + # result shelf public user allowed user owner + # false 1 0 x + # true 1 1 x + # true 0 x 1 + # false 0 x 0 + + if check_shelf_edit_permissions(shelf): + book_shelf = ub.session.query(ub.BookShelf).filter(ub.BookShelf.shelf == shelf_id, + ub.BookShelf.book_id == book_id).first() + + if book_shelf is None: + log.error("Book %s already removed from %s", book_id, shelf) + if not xhr: + return redirect(url_for('web.index')) + return "Book already removed from shelf", 410 + + try: + ub.session.delete(book_shelf) + shelf.last_modified = datetime.now(timezone.utc) + ub.session.commit() + except (OperationalError, InvalidRequestError) as e: + ub.session.rollback() + log.error_or_exception("Settings Database error: {}".format(e)) + flash(_("Oops! Database Error: %(error)s.", error=e.orig), category="error") + if "HTTP_REFERER" in request.environ: + return redirect(request.environ["HTTP_REFERER"]) + else: + return redirect(url_for('web.index')) + if not xhr: + flash(_("Book has been removed from shelf: %(sname)s", sname=shelf.name), category="success") + if "HTTP_REFERER" in request.environ: + return redirect(request.environ["HTTP_REFERER"]) + else: + return redirect(url_for('web.index')) + return "", 204 + else: + if not xhr: + log.warning("You are not allowed to remove a book from shelf: {}".format(shelf.name)) + flash(_("Sorry you are not allowed to remove a book from this shelf"), + category="error") + return redirect(url_for('web.index')) + return "Sorry you are not allowed to remove a book from this shelf", 403 + + +@shelf.route("/shelf/create", methods=["GET", "POST"]) +@user_login_required +def create_shelf(): + shelf = ub.Shelf() + return create_edit_shelf(shelf, page_title=_("Create a Shelf"), page="shelfcreate") + + +@shelf.route("/shelf/edit/", methods=["GET", "POST"]) +@user_login_required +def edit_shelf(shelf_id): + shelf = ub.session.query(ub.Shelf).filter(ub.Shelf.id == shelf_id).first() + if not check_shelf_edit_permissions(shelf): + flash(_("Sorry you are not allowed to edit this shelf"), category="error") + return redirect(url_for('web.index')) + return create_edit_shelf(shelf, page_title=_("Edit a shelf"), page="shelfedit", shelf_id=shelf_id) + + +@shelf.route("/shelf/delete/", methods=["POST"]) +@user_login_required +def delete_shelf(shelf_id): + cur_shelf = ub.session.query(ub.Shelf).filter(ub.Shelf.id == shelf_id).first() + try: + if not delete_shelf_helper(cur_shelf): + flash(_("Error deleting Shelf"), category="error") + else: + flash(_("Shelf successfully deleted"), category="success") + except InvalidRequestError as e: + ub.session.rollback() + log.error_or_exception("Settings Database error: {}".format(e)) + flash(_("Oops! Database Error: %(error)s.", error=e.orig), category="error") + return redirect(url_for('web.index')) + + +@shelf.route("/simpleshelf/") +@login_required_if_no_ano +def show_simpleshelf(shelf_id): + return render_show_shelf(2, shelf_id, 1, None) + + +@shelf.route("/shelf/", defaults={"sort_param": "stored", 'page': 1}) +@shelf.route("/shelf//", defaults={'page': 1}) +@shelf.route("/shelf///") +@login_required_if_no_ano +def show_shelf(shelf_id, sort_param, page): + return render_show_shelf(1, shelf_id, page, sort_param) + + +@shelf.route("/shelf/order/", methods=["GET", "POST"]) +@user_login_required +def order_shelf(shelf_id): + shelf = ub.session.query(ub.Shelf).filter(ub.Shelf.id == shelf_id).first() + if shelf and check_shelf_view_permissions(shelf): + if request.method == "POST": + to_save = request.form.to_dict() + books_in_shelf = ub.session.query(ub.BookShelf).filter(ub.BookShelf.shelf == shelf_id).order_by( + ub.BookShelf.order.asc()).all() + counter = 0 + for book in books_in_shelf: + setattr(book, 'order', to_save[str(book.book_id)]) + counter += 1 + # if order different from before -> shelf.last_modified = datetime.now(timezone.utc) + try: + ub.session.commit() + except (OperationalError, InvalidRequestError) as e: + ub.session.rollback() + log.error_or_exception("Settings Database error: {}".format(e)) + flash(_("Oops! Database Error: %(error)s.", error=e.orig), category="error") + + result = list() + if shelf: + result = calibre_db.session.query(db.Books) \ + .join(ub.BookShelf, ub.BookShelf.book_id == db.Books.id, isouter=True) \ + .add_columns(calibre_db.common_filters().label("visible")) \ + .filter(ub.BookShelf.shelf == shelf_id).order_by(ub.BookShelf.order.asc()).all() + return render_title_template('shelf_order.html', entries=result, + title=_("Change order of Shelf: '%(name)s'", name=shelf.name), + shelf=shelf, page="shelforder") + else: + abort(404) + + +def check_shelf_edit_permissions(cur_shelf): + if not cur_shelf.is_public and not cur_shelf.user_id == int(current_user.id): + log.error("User {} not allowed to edit shelf: {}".format(current_user.id, cur_shelf.name)) + return False + if cur_shelf.is_public and not current_user.role_edit_shelfs(): + log.info("User {} not allowed to edit public shelves".format(current_user.id)) + return False + return True + + +def check_shelf_view_permissions(cur_shelf): + try: + if cur_shelf.is_public: + return True + if current_user.is_anonymous or cur_shelf.user_id != current_user.id: + log.error("User is unauthorized to view non-public shelf: {}".format(cur_shelf.name)) + return False + except Exception as e: + log.error(e) + return True + + +# if shelf ID is set, we are editing a shelf +def create_edit_shelf(shelf, page_title, page, shelf_id=False): + sync_only_selected_shelves = current_user.kobo_only_shelves_sync + # calibre_db.session.query(ub.Shelf).filter(ub.Shelf.user_id == current_user.id).filter(ub.Shelf.kobo_sync).count() + if request.method == "POST": + to_save = request.form.to_dict() + if not current_user.role_edit_shelfs() and to_save.get("is_public") == "on": + flash(_("Sorry you are not allowed to create a public shelf"), category="error") + return redirect(url_for('web.index')) + is_public = 1 if to_save.get("is_public") == "on" else 0 + if config.config_kobo_sync: + shelf.kobo_sync = True if to_save.get("kobo_sync") else False + if shelf.kobo_sync: + ub.session.query(ub.ShelfArchive).filter(ub.ShelfArchive.user_id == current_user.id).filter( + ub.ShelfArchive.uuid == shelf.uuid).delete() + ub.session_commit() + shelf_title = to_save.get("title", "") + if check_shelf_is_unique(shelf_title, is_public, shelf_id): + shelf.name = shelf_title + shelf.is_public = is_public + if not shelf_id: + shelf.user_id = int(current_user.id) + ub.session.add(shelf) + shelf_action = "created" + flash_text = _("Shelf %(title)s created", title=shelf_title) + else: + shelf_action = "changed" + flash_text = _("Shelf %(title)s changed", title=shelf_title) + try: + ub.session.commit() + log.info("Shelf {} {}".format(shelf_title, shelf_action)) + flash(flash_text, category="success") + return redirect(url_for('shelf.show_shelf', shelf_id=shelf.id)) + except (OperationalError, InvalidRequestError) as ex: + ub.session.rollback() + log.error_or_exception(ex) + log.error_or_exception("Settings Database error: {}".format(ex)) + flash(_("Oops! Database Error: %(error)s.", error=ex.orig), category="error") + except Exception as ex: + ub.session.rollback() + log.error_or_exception(ex) + flash(_("There was an error"), category="error") + return render_title_template('shelf_edit.html', + shelf=shelf, + title=page_title, + page=page, + kobo_sync_enabled=config.config_kobo_sync, + sync_only_selected_shelves=sync_only_selected_shelves) + + +def check_shelf_is_unique(title, is_public, shelf_id=False): + if shelf_id: + ident = ub.Shelf.id != shelf_id + else: + ident = true() + if is_public == 1: + is_shelf_name_unique = ub.session.query(ub.Shelf) \ + .filter((ub.Shelf.name == title) & (ub.Shelf.is_public == 1)) \ + .filter(ident) \ + .first() is None + + if not is_shelf_name_unique: + log.error("A public shelf with the name '{}' already exists.".format(title)) + flash(_("A public shelf with the name '%(title)s' already exists.", title=title), + category="error") + else: + is_shelf_name_unique = ub.session.query(ub.Shelf) \ + .filter((ub.Shelf.name == title) & (ub.Shelf.is_public == 0) & + (ub.Shelf.user_id == int(current_user.id))) \ + .filter(ident) \ + .first() is None + + if not is_shelf_name_unique: + log.error("A private shelf with the name '{}' already exists.".format(title)) + flash(_("A private shelf with the name '%(title)s' already exists.", title=title), + category="error") + return is_shelf_name_unique + + +def delete_shelf_helper(cur_shelf): + if not cur_shelf or not check_shelf_edit_permissions(cur_shelf): + return False + shelf_id = cur_shelf.id + ub.session.delete(cur_shelf) + ub.session.query(ub.BookShelf).filter(ub.BookShelf.shelf == shelf_id).delete() + ub.session.add(ub.ShelfArchive(uuid=cur_shelf.uuid, user_id=cur_shelf.user_id)) + ub.session_commit("successfully deleted Shelf {}".format(cur_shelf.name)) + return True + + +def change_shelf_order(shelf_id, order): + result = calibre_db.session.query(db.Books).outerjoin(db.books_series_link, + db.Books.id == db.books_series_link.c.book)\ + .outerjoin(db.Series).join(ub.BookShelf, ub.BookShelf.book_id == db.Books.id) \ + .filter(ub.BookShelf.shelf == shelf_id).order_by(*order).all() + for index, entry in enumerate(result): + book = ub.session.query(ub.BookShelf).filter(ub.BookShelf.shelf == shelf_id) \ + .filter(ub.BookShelf.book_id == entry.id).first() + book.order = index + ub.session_commit("Shelf-id:{} - Order changed".format(shelf_id)) + + +def render_show_shelf(shelf_type, shelf_id, page_no, sort_param): + shelf = ub.session.query(ub.Shelf).filter(ub.Shelf.id == shelf_id).first() + status = current_user.get_view_property("shelf", 'man') + # check user is allowed to access shelf + if shelf and check_shelf_view_permissions(shelf): + if shelf_type == 1: + if status != 'on': + if sort_param == 'stored': + sort_param = current_user.get_view_property("shelf", 'stored') + else: + current_user.set_view_property("shelf", 'stored', sort_param) + if sort_param == 'pubnew': + change_shelf_order(shelf_id, [db.Books.pubdate.desc()]) + if sort_param == 'pubold': + change_shelf_order(shelf_id, [db.Books.pubdate]) + if sort_param == 'shelfnew': + change_shelf_order(shelf_id, [ub.BookShelf.date_added.desc()]) + if sort_param == 'shelfold': + change_shelf_order(shelf_id, [ub.BookShelf.date_added]) + if sort_param == 'abc': + change_shelf_order(shelf_id, [db.Books.sort]) + if sort_param == 'zyx': + change_shelf_order(shelf_id, [db.Books.sort.desc()]) + if sort_param == 'new': + change_shelf_order(shelf_id, [db.Books.timestamp.desc()]) + if sort_param == 'old': + change_shelf_order(shelf_id, [db.Books.timestamp]) + if sort_param == 'authaz': + change_shelf_order(shelf_id, [db.Books.author_sort.asc(), db.Series.name, db.Books.series_index]) + if sort_param == 'authza': + change_shelf_order(shelf_id, [db.Books.author_sort.desc(), + db.Series.name.desc(), + db.Books.series_index.desc()]) + page = "shelf.html" + pagesize = 0 + else: + pagesize = sys.maxsize + page = 'shelfdown.html' + + result, __, pagination = calibre_db.fill_indexpage(page_no, pagesize, + db.Books, + ub.BookShelf.shelf == shelf_id, + [ub.BookShelf.order.asc()], + True, config.config_read_column, + ub.BookShelf, ub.BookShelf.book_id == db.Books.id) + # delete shelf entries where book is not existent anymore, can happen if book is deleted outside calibre-web + wrong_entries = calibre_db.session.query(ub.BookShelf) \ + .join(db.Books, ub.BookShelf.book_id == db.Books.id, isouter=True) \ + .filter(db.Books.id == None).all() + for entry in wrong_entries: + log.info('Not existing book {} in {} deleted'.format(entry.book_id, shelf)) + try: + ub.session.query(ub.BookShelf).filter(ub.BookShelf.book_id == entry.book_id).delete() + ub.session.commit() + except (OperationalError, InvalidRequestError) as e: + ub.session.rollback() + log.error_or_exception("Settings Database error: {}".format(e)) + flash(_("Oops! Database Error: %(error)s.", error=e.orig), category="error") + + return render_title_template(page, + entries=result, + pagination=pagination, + title=_("Shelf: '%(name)s'", name=shelf.name), + shelf=shelf, + page="shelf", + status=status, + order=sort_param) + else: + flash(_("Error opening shelf. Shelf does not exist or is not accessible"), category="error") + return redirect(url_for("web.index")) diff --git a/root/app/calibre-web/cps/static/css/style.css b/root/app/calibre-web/cps/static/css/style.css index 8390b39..e40589b 100644 --- a/root/app/calibre-web/cps/static/css/style.css +++ b/root/app/calibre-web/cps/static/css/style.css @@ -313,7 +313,7 @@ span.glyphicon.glyphicon-tags { position: relative; top: -20px; /*left: auto; - right: 2px;*/ + right: 2px;*/ width: 17px; height: 17px; display: inline-block; diff --git a/root/app/calibre-web/cps/templates/config_edit.html b/root/app/calibre-web/cps/templates/config_edit.html new file mode 100644 index 0000000..5d93f17 --- /dev/null +++ b/root/app/calibre-web/cps/templates/config_edit.html @@ -0,0 +1,438 @@ +{% extends "layout.html" %} +{% block flash %} + +{% endblock %} +{% block body %} +
+

{{title}}

+
+ +
+
+ +
+
+
+ + +
+ +
+ + + + +
+ +
+ + + + +
+
+ + +
+
+ + +
+
+
+
+
+ +
+
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+
+
+
+ +
+
+
+ + +
+
+ + +
+
+ + +
+
+
+ + +
+
+
+ + +
+
+ + +
+
+
+ + +
+
+
+ + +
+ {% if feature_support['kobo'] %} +
+ + +
+
+
+ + +
+
+ + +
+
+ + +
+
+ {% endif %} + {% if feature_support['goodreads'] %} +
+ + +
+
+
+ + +
+
+ {% endif %} +
+ + +
+
+
+ + +
+
+ {% if not config.config_is_initial %} + {% if feature_support['ldap'] or feature_support['oauth'] %} +
+ + +
+ {% if feature_support['ldap'] %} +
+
+ + +
+
+ + +
+
+ + +
+
+ +
+ + + + +
+ +
+ + + + +
+ +
+ + + + +
+
+
+ + +
+
+
+ + +
+
+
+
+ + +
+
+
+ + +
+
+ + +
+
+ + +
+

{{_('Following Settings are Needed For User Import')}}

+
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+
+ + +
+
+ +
+ {% endif %} + {% if feature_support['oauth'] %} +
+ {% for prov in provider %} + +
+ + +
+
+ + +
+ {% endfor %} +
+ {% endif %} + {% endif %} + {% endif %} +
+
+
+
+ +
+
+ +
+ + + + +
+
+ + +
+ +
+ + + + +
+ {% if feature_support['rar'] %} + +
+ + + + +
+ {% endif %} +
+
+
+
+ +
+
+
+ + +
+
+
+ + +
+
+ + +
+
+
+ + +
+
+ + +
+
+ + +
+
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+
+
+
+
+
+ + {{_('Cancel')}} +
+
+
+{% endblock %} +{% block modal %} +{{ filechooser_modal() }} +{% endblock %} diff --git a/root/app/calibre-web/cps/templates/detail.html b/root/app/calibre-web/cps/templates/detail.html new file mode 100644 index 0000000..bc3ab48 --- /dev/null +++ b/root/app/calibre-web/cps/templates/detail.html @@ -0,0 +1,377 @@ +{% extends is_xhr|yesno("fragment.html", "layout.html") %} +{% block header %} + + + {% if entry.comments|length > 0 and entry.comments[0].text|length > 0 %} + + + {% endif %} +{% endblock %} +{% block body %} +
+
+
+
+ + +
+
+
+ +

{{ entry.title }}

+

+ {% for author in entry.ordered_authors %} + {{ author.name.replace('|',',') }} + {% if not loop.last %} + & + {% endif %} + {% endfor %} +

+ {% if entry.ratings.__len__() > 0 %} +
+

+ {% for number in range((entry.ratings[0].rating/2)|int(2)) %} + + {% if loop.last and loop.index < 5 %} + {% for numer in range(5 - loop.index) %} + + {% endfor %} + {% endif %} + {% endfor %} +

+
+ {% endif %} + {% if entry.series|length > 0 %} +

{{ _("Book %(index)s of %(range)s", index=entry.series_index|formatfloat(2), range=(url_for('web.books_list', data='series', sort_param='stored', book_id=entry.series[0].id)|escapedlink(entry.series[0].name))|safe) }}

+ + {% endif %} + + {% if entry.languages|length > 0 %} +
+

+ {{_('Language')}}: {% for language in entry.languages %}{{language.language_name}}{% if not loop.last %}, {% endif %}{% endfor %} +

+
+ {% endif %} + + {% if entry.identifiers|length > 0 %} +
+

+ + {% for identifier in entry.identifiers if identifier.__repr__() != identifier.val %} + {{ identifier.format_type() }} + {% endfor %} +

+
+ {% endif %} + + {% if entry.tags|length > 0 %} + +
+

+ + + {% for tag in entry.tags %} + {{ tag.name }} + {% endfor %} +

+ +
+ {% endif %} + + {% if entry.publishers|length > 0 %} +
+

+ {{ _('Publisher') }}: + {{ entry.publishers[0].name }} + +

+
+ {% endif %} + + {% if (entry.pubdate|string)[:10] != '0101-01-01' %} +
+

{{ _('Published') }}: {{ entry.pubdate|formatdate }}

+
+ {% endif %} + {% if cc|length > 0 %} + + + {% for c in cc %} + {% if entry['custom_column_' ~ c.id]|length > 0 %} +
+ {{ c.name }}: + {% for column in entry['custom_column_' ~ c.id] %} + {% if c.datatype == 'rating' %} + {{ (column.value / 2)|formatfloat }} + {% else %} + {% if c.datatype == 'bool' %} + {% if column.value == true %} + + {% else %} + + {% endif %} + {% else %} + {% if c.datatype == 'float' %} + {{ column.value|formatfloat(2) }} + {% elif c.datatype == 'datetime' %} + {{ column.value|formatdate }} + {% elif c.datatype == 'comments' %} + {{ column.value|safe }} + {% elif c.datatype == 'series' %} + {{ '%s [%s]' % (column.value, column.extra|formatfloat(2)) }} + {% elif c.datatype == 'text' %} + {{ column.value.strip() }}{% if not loop.last %}, {% endif %} + {% else %} + {{ column.value }} + {% endif %} + {% endif %} + {% endif %} + {% endfor %} + +
+ {% endif %} + {% endfor %} + {% endif %} + {% if not current_user.is_anonymous %} + +
+

+

+ + +
+

+ {% if current_user.check_visibility(32768) %} +

+

+ + +
+

+ {% endif %} +
+ {% endif %} + + + {% if entry.comments|length > 0 and entry.comments[0].text|length > 0 %} +
+

{{ _('Description:') }}

+ {{ entry.comments[0].text|safe }} +
+ {% endif %} + + +
+ + {% if current_user.is_authenticated %} + {% if current_user.shelf.all() or g.shelves_access %} + + {% endif %} + + {% endif %} + {% if current_user.role_edit() %} +
+ +
{{_('Cancel')}}
+
+ {% endif %} +
+
+
+{% endblock %} + +{% block js %} + + + + + + +{% endblock %} diff --git a/root/app/calibre-web/cps/templates/user_edit.html b/root/app/calibre-web/cps/templates/user_edit.html new file mode 100644 index 0000000..ed3b7b1 --- /dev/null +++ b/root/app/calibre-web/cps/templates/user_edit.html @@ -0,0 +1,189 @@ +{% extends "layout.html" %} +{% block body %} +
+

{{title}}

+
+ +
+ {% if new_user or ( current_user and content.name != "Guest" and current_user.role_admin() ) %} +
+ + +
+ {% endif %} +
+ + +
+ {% if ( current_user and current_user.role_passwd() or current_user.role_admin() ) and not content.role_anonymous() %} + {% if current_user and current_user.role_admin() and not new_user and not profile and ( mail_configured and content.email if content.email != None ) %} + {{_('Reset user Password')}} + {% endif %} +
+ + +
+ {% endif %} +
+ + +
+ {% if not content.role_anonymous() %} +
+ + +
+ {% endif %} + +
+ + +
+ {% if registered_oauth.keys()| length > 0 and not new_user and profile %} + {% for id, name in registered_oauth.items() %} +
+ + {% if id not in oauth_status %} + {{_('Link')}} + {% else %} + {{_('Unlink')}} + {% endif %} + {% endfor %} +
+ {% endif %} + {% if kobo_support and hardcover_support and not new_user %} +
+ + +
+ {% endif %} + {% if kobo_support and not new_user %} + +
+ {{_('Create/View')}} + +
+
+ +
+ {% endif %} +
+ {% for element in sidebar %} + {% if element['config_show'] %} +
+ + +
+ {% endif %} + {% endfor %} +
+ + +
+ {% if ( current_user and current_user.role_admin() and not new_user ) and not simple %} + {{_('Add Allowed/Denied Tags')}} + {{_('Add allowed/Denied Custom Column Values')}} + {% endif %} +
+
+ {% if current_user and current_user.role_admin() and not profile %} + {% if not content.role_anonymous() %} +
+ + +
+ {% endif %} +
+ + +
+
+ + +
+ {% if config.config_uploading %} +
+ + +
+ {% endif %} +
+ + +
+
+
+ + +
+
+ {% if not content.role_anonymous() %} +
+ + +
+
+ + +
+ {% endif %} + {% endif %} + {% if kobo_support and not content.role_anonymous() and not simple%} +
+ + +
+ {% endif %} +
+
+
{{_('Save')}}
+ {% if not profile %} +
{{_('Cancel')}}
+ {% endif %} + {% if current_user and current_user.role_admin() and not profile and not new_user and not content.role_anonymous() %} +
{{_('Delete User')}}
+ {% endif %} +
+
+
+
+ + + +{% endblock %} +{% block modal %} +{{ restrict_modal() }} +{{ delete_confirm_modal() }} +{% endblock %} +{% block js %} + + + + + + + + +{% endblock %} diff --git a/root/app/calibre-web/cps/ub.py b/root/app/calibre-web/cps/ub.py new file mode 100644 index 0000000..f0ba832 --- /dev/null +++ b/root/app/calibre-web/cps/ub.py @@ -0,0 +1,778 @@ +# -*- coding: utf-8 -*- + +# This file is part of the Calibre-Web (https://github.com/janeczku/calibre-web) +# Copyright (C) 2012-2019 mutschler, jkrehm, cervinko, janeczku, OzzieIsaacs, csitko +# ok11, issmirnov, idalin +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import atexit +import os +import sys +from datetime import datetime, timezone, timedelta +import itertools +import uuid +from flask import session as flask_session +from binascii import hexlify + +from .cw_login import AnonymousUserMixin, current_user +from .cw_login import user_logged_in + +try: + from flask_dance.consumer.backend.sqla import OAuthConsumerMixin + oauth_support = True +except ImportError as e: + # fails on flask-dance >1.3, due to renaming + try: + from flask_dance.consumer.storage.sqla import OAuthConsumerMixin + oauth_support = True + except ImportError as e: + OAuthConsumerMixin = BaseException + oauth_support = False +from sqlalchemy import create_engine, exc, exists, event, text +from sqlalchemy import Column, ForeignKey +from sqlalchemy import String, Integer, SmallInteger, Boolean, DateTime, Float, JSON +from sqlalchemy.orm.attributes import flag_modified +from sqlalchemy.sql.expression import func +try: + # Compatibility with sqlalchemy 2.0 + from sqlalchemy.orm import declarative_base +except ImportError: + from sqlalchemy.ext.declarative import declarative_base +from sqlalchemy.orm import backref, relationship, sessionmaker, Session, scoped_session +from werkzeug.security import generate_password_hash + +from . import constants, logger +from .string_helper import strip_whitespaces + +log = logger.create() + +session = None +app_DB_path = None +Base = declarative_base() +searched_ids = {} + +logged_in = dict() + + +def signal_store_user_session(object, user): + store_user_session() + + +def store_user_session(): + _user = flask_session.get('_user_id', "") + _id = flask_session.get('_id', "") + _random = flask_session.get('_random', "") + if flask_session.get('_user_id', ""): + try: + if not check_user_session(_user, _id, _random): + expiry = int((datetime.now() + timedelta(days=31)).timestamp()) + user_session = User_Sessions(_user, _id, _random, expiry) + session.add(user_session) + session.commit() + log.debug("Login and store session : " + _id) + else: + log.debug("Found stored session: " + _id) + except (exc.OperationalError, exc.InvalidRequestError) as e: + session.rollback() + log.exception(e) + else: + log.error("No user id in session") + + +def delete_user_session(user_id, session_key): + try: + log.debug("Deleted session_key: " + session_key) + session.query(User_Sessions).filter(User_Sessions.user_id == user_id, + User_Sessions.session_key == session_key).delete() + session.commit() + except (exc.OperationalError, exc.InvalidRequestError) as ex: + session.rollback() + log.exception(ex) + + +def check_user_session(user_id, session_key, random): + try: + found = session.query(User_Sessions).filter(User_Sessions.user_id==user_id, + User_Sessions.session_key==session_key, + User_Sessions.random == random, + ).one_or_none() + if found is not None: + new_expiry = int((datetime.now() + timedelta(days=31)).timestamp()) + if new_expiry - found.expiry > 86400: + found.expiry = new_expiry + session.merge(found) + session.commit() + return bool(found) + except (exc.OperationalError, exc.InvalidRequestError) as e: + session.rollback() + log.exception(e) + return False + + +user_logged_in.connect(signal_store_user_session) + +def store_ids(result): + ids = list() + for element in result: + ids.append(element.id) + searched_ids[current_user.id] = ids + +def store_combo_ids(result): + ids = list() + for element in result: + ids.append(element[0].id) + searched_ids[current_user.id] = ids + + +class UserBase: + + @property + def is_authenticated(self): + return self.is_active + + def _has_role(self, role_flag): + return constants.has_flag(self.role, role_flag) + + def role_admin(self): + return self._has_role(constants.ROLE_ADMIN) + + def role_download(self): + return self._has_role(constants.ROLE_DOWNLOAD) + + def role_upload(self): + return self._has_role(constants.ROLE_UPLOAD) + + def role_edit(self): + return self._has_role(constants.ROLE_EDIT) + + def role_passwd(self): + return self._has_role(constants.ROLE_PASSWD) + + def role_anonymous(self): + return self._has_role(constants.ROLE_ANONYMOUS) + + def role_edit_shelfs(self): + return self._has_role(constants.ROLE_EDIT_SHELFS) + + def role_delete_books(self): + return self._has_role(constants.ROLE_DELETE_BOOKS) + + def role_viewer(self): + return self._has_role(constants.ROLE_VIEWER) + + @property + def is_active(self): + return True + + @property + def is_anonymous(self): + return self.role_anonymous() + + def get_id(self): + return str(self.id) + + def filter_language(self): + return self.default_language + + def check_visibility(self, value): + if value == constants.SIDEBAR_RECENT: + return True + return constants.has_flag(self.sidebar_view, value) + + def show_detail_random(self): + return self.check_visibility(constants.DETAIL_RANDOM) + + def list_denied_tags(self): + mct = self.denied_tags or "" + return [strip_whitespaces(t) for t in mct.split(",")] + + def list_allowed_tags(self): + mct = self.allowed_tags or "" + return [strip_whitespaces(t) for t in mct.split(",")] + + def list_denied_column_values(self): + mct = self.denied_column_value or "" + return [strip_whitespaces(t) for t in mct.split(",")] + + def list_allowed_column_values(self): + mct = self.allowed_column_value or "" + return [strip_whitespaces(t) for t in mct.split(",")] + + def get_view_property(self, page, prop): + if not self.view_settings.get(page): + return None + return self.view_settings[page].get(prop) + + def set_view_property(self, page, prop, value): + if not self.view_settings.get(page): + self.view_settings[page] = dict() + self.view_settings[page][prop] = value + try: + flag_modified(self, "view_settings") + except AttributeError: + pass + try: + session.commit() + except (exc.OperationalError, exc.InvalidRequestError) as e: + session.rollback() + log.error_or_exception(e) + + def __repr__(self): + return '' % self.name + + +# Baseclass for Users in Calibre-Web, settings which are depending on certain users are stored here. It is derived from +# User Base (all access methods are declared there) +class User(UserBase, Base): + __tablename__ = 'user' + __table_args__ = {'sqlite_autoincrement': True} + + id = Column(Integer, primary_key=True) + name = Column(String(64), unique=True) + email = Column(String(120), unique=True, default="") + role = Column(SmallInteger, default=constants.ROLE_USER) + password = Column(String) + kindle_mail = Column(String(120), default="") + shelf = relationship('Shelf', backref='user', lazy='dynamic', order_by='Shelf.name') + downloads = relationship('Downloads', backref='user', lazy='dynamic') + locale = Column(String(2), default="en") + sidebar_view = Column(Integer, default=1) + default_language = Column(String(3), default="all") + denied_tags = Column(String, default="") + allowed_tags = Column(String, default="") + denied_column_value = Column(String, default="") + allowed_column_value = Column(String, default="") + remote_auth_token = relationship('RemoteAuthToken', backref='user', lazy='dynamic') + view_settings = Column(JSON, default={}) + kobo_only_shelves_sync = Column(Integer, default=0) + hardcover_token = Column(String, unique=True, default="") + + +if oauth_support: + class OAuth(OAuthConsumerMixin, Base): + provider_user_id = Column(String(256)) + user_id = Column(Integer, ForeignKey(User.id)) + user = relationship(User) + + +class OAuthProvider(Base): + __tablename__ = 'oauthProvider' + + id = Column(Integer, primary_key=True) + provider_name = Column(String) + oauth_client_id = Column(String) + oauth_client_secret = Column(String) + active = Column(Boolean) + + +# Class for anonymous user is derived from User base and completely overrides methods and properties for the +# anonymous user +class Anonymous(AnonymousUserMixin, UserBase): + def __init__(self): + self.kobo_only_shelves_sync = None + self.view_settings = None + self.allowed_column_value = None + self.allowed_tags = None + self.denied_tags = None + self.kindle_mail = None + self.locale = None + self.default_language = None + self.sidebar_view = None + self.id = None + self.role = None + self.name = None + self.loadSettings() + + def loadSettings(self): + data = session.query(User).filter(User.role.op('&')(constants.ROLE_ANONYMOUS) == constants.ROLE_ANONYMOUS)\ + .first() # type: User + self.name = data.name + self.role = data.role + self.id=data.id + self.sidebar_view = data.sidebar_view + self.default_language = data.default_language + self.locale = data.locale + self.kindle_mail = data.kindle_mail + self.denied_tags = data.denied_tags + self.allowed_tags = data.allowed_tags + self.denied_column_value = data.denied_column_value + self.allowed_column_value = data.allowed_column_value + self.view_settings = data.view_settings + self.kobo_only_shelves_sync = data.kobo_only_shelves_sync + + def role_admin(self): + return False + + @property + def is_active(self): + return False + + @property + def is_anonymous(self): + return True + + @property + def is_authenticated(self): + return False + + def get_view_property(self, page, prop): + if 'view' in flask_session: + if not flask_session['view'].get(page): + return None + return flask_session['view'][page].get(prop) + return None + + def set_view_property(self, page, prop, value): + if not 'view' in flask_session: + flask_session['view'] = dict() + if not flask_session['view'].get(page): + flask_session['view'][page] = dict() + flask_session['view'][page][prop] = value + +class User_Sessions(Base): + __tablename__ = 'user_session' + + id = Column(Integer, primary_key=True) + user_id = Column(Integer, ForeignKey('user.id')) + session_key = Column(String, default="") + random = Column(String, default="") + expiry = Column(Integer) + + + def __init__(self, user_id, session_key, random, expiry): + super().__init__() + self.user_id = user_id + self.session_key = session_key + self.random = random + self.expiry = expiry + + +# Baseclass representing Shelfs in calibre-web in app.db +class Shelf(Base): + __tablename__ = 'shelf' + + id = Column(Integer, primary_key=True) + uuid = Column(String, default=lambda: str(uuid.uuid4())) + name = Column(String) + is_public = Column(Integer, default=0) + user_id = Column(Integer, ForeignKey('user.id')) + kobo_sync = Column(Boolean, default=False) + books = relationship("BookShelf", backref="ub_shelf", cascade="all, delete-orphan", lazy="dynamic") + created = Column(DateTime, default=lambda: datetime.now(timezone.utc)) + last_modified = Column(DateTime, default=lambda: datetime.now(timezone.utc), onupdate=lambda: datetime.now(timezone.utc)) + + def __repr__(self): + return '' % (self.id, self.name) + + +# Baseclass representing Relationship between books and Shelfs in Calibre-Web in app.db (N:M) +class BookShelf(Base): + __tablename__ = 'book_shelf_link' + + id = Column(Integer, primary_key=True) + book_id = Column(Integer) + order = Column(Integer) + shelf = Column(Integer, ForeignKey('shelf.id')) + date_added = Column(DateTime, default=lambda: datetime.now(timezone.utc)) + + def __repr__(self): + return '' % self.id + + +# This table keeps track of deleted Shelves so that deletes can be propagated to any paired Kobo device. +class ShelfArchive(Base): + __tablename__ = 'shelf_archive' + + id = Column(Integer, primary_key=True) + uuid = Column(String) + user_id = Column(Integer, ForeignKey('user.id')) + last_modified = Column(DateTime, default=lambda: datetime.now(timezone.utc)) + + +class ReadBook(Base): + __tablename__ = 'book_read_link' + + STATUS_UNREAD = 0 + STATUS_FINISHED = 1 + STATUS_IN_PROGRESS = 2 + + id = Column(Integer, primary_key=True) + book_id = Column(Integer, unique=False) + user_id = Column(Integer, ForeignKey('user.id'), unique=False) + read_status = Column(Integer, unique=False, default=STATUS_UNREAD, nullable=False) + kobo_reading_state = relationship("KoboReadingState", uselist=False, + primaryjoin="and_(ReadBook.user_id == foreign(KoboReadingState.user_id), " + "ReadBook.book_id == foreign(KoboReadingState.book_id))", + cascade="all", + backref=backref("book_read_link", + uselist=False)) + last_modified = Column(DateTime, default=lambda: datetime.now(timezone.utc), onupdate=lambda: datetime.now(timezone.utc)) + last_time_started_reading = Column(DateTime, nullable=True) + times_started_reading = Column(Integer, default=0, nullable=False) + + +class Bookmark(Base): + __tablename__ = 'bookmark' + + id = Column(Integer, primary_key=True) + user_id = Column(Integer, ForeignKey('user.id')) + book_id = Column(Integer) + format = Column(String(collation='NOCASE')) + bookmark_key = Column(String) + + +# Baseclass representing books that are archived on the user's Kobo device. +class ArchivedBook(Base): + __tablename__ = 'archived_book' + + id = Column(Integer, primary_key=True) + user_id = Column(Integer, ForeignKey('user.id')) + book_id = Column(Integer) + is_archived = Column(Boolean, unique=False) + last_modified = Column(DateTime, default=lambda: datetime.now(timezone.utc)) + + +class KoboSyncedBooks(Base): + __tablename__ = 'kobo_synced_books' + id = Column(Integer, primary_key=True, autoincrement=True) + user_id = Column(Integer, ForeignKey('user.id')) + book_id = Column(Integer) + +# The Kobo ReadingState API keeps track of 4 timestamped entities: +# ReadingState, StatusInfo, Statistics, CurrentBookmark +# Which we map to the following 4 tables: +# KoboReadingState, ReadBook, KoboStatistics and KoboBookmark +class KoboReadingState(Base): + __tablename__ = 'kobo_reading_state' + + id = Column(Integer, primary_key=True, autoincrement=True) + user_id = Column(Integer, ForeignKey('user.id')) + book_id = Column(Integer) + last_modified = Column(DateTime, default=lambda: datetime.now(timezone.utc), onupdate=lambda: datetime.now(timezone.utc)) + priority_timestamp = Column(DateTime, default=lambda: datetime.now(timezone.utc), onupdate=lambda: datetime.now(timezone.utc)) + current_bookmark = relationship("KoboBookmark", uselist=False, backref="kobo_reading_state", cascade="all, delete") + statistics = relationship("KoboStatistics", uselist=False, backref="kobo_reading_state", cascade="all, delete") + + +class KoboBookmark(Base): + __tablename__ = 'kobo_bookmark' + + id = Column(Integer, primary_key=True) + kobo_reading_state_id = Column(Integer, ForeignKey('kobo_reading_state.id')) + last_modified = Column(DateTime, default=lambda: datetime.now(timezone.utc), onupdate=lambda: datetime.now(timezone.utc)) + location_source = Column(String) + location_type = Column(String) + location_value = Column(String) + progress_percent = Column(Float) + content_source_progress_percent = Column(Float) + + +class KoboStatistics(Base): + __tablename__ = 'kobo_statistics' + + id = Column(Integer, primary_key=True) + kobo_reading_state_id = Column(Integer, ForeignKey('kobo_reading_state.id')) + last_modified = Column(DateTime, default=lambda: datetime.now(timezone.utc), onupdate=lambda: datetime.now(timezone.utc)) + remaining_time_minutes = Column(Integer) + spent_reading_minutes = Column(Integer) + + +# Updates the last_modified timestamp in the KoboReadingState table if any of its children tables are modified. +@event.listens_for(Session, 'before_flush') +def receive_before_flush(session, flush_context, instances): + for change in itertools.chain(session.new, session.dirty): + if isinstance(change, (ReadBook, KoboStatistics, KoboBookmark)): + if change.kobo_reading_state: + change.kobo_reading_state.last_modified = datetime.now(timezone.utc) + # Maintain the last_modified_bit for the Shelf table. + for change in itertools.chain(session.new, session.deleted): + if isinstance(change, BookShelf): + change.ub_shelf.last_modified = datetime.now(timezone.utc) + + +# Baseclass representing Downloads from calibre-web in app.db +class Downloads(Base): + __tablename__ = 'downloads' + + id = Column(Integer, primary_key=True) + book_id = Column(Integer) + user_id = Column(Integer, ForeignKey('user.id')) + + def __repr__(self): + return '".format(self.domain) + + +class RemoteAuthToken(Base): + __tablename__ = 'remote_auth_token' + + id = Column(Integer, primary_key=True) + auth_token = Column(String, unique=True) + user_id = Column(Integer, ForeignKey('user.id')) + verified = Column(Boolean, default=False) + expiration = Column(DateTime) + token_type = Column(Integer, default=0) + + def __init__(self): + super().__init__() + self.auth_token = (hexlify(os.urandom(4))).decode('utf-8') + self.expiration = datetime.now() + timedelta(minutes=10) # 10 min from now + + def __repr__(self): + return '' % self.id + + +def filename(context): + file_format = context.get_current_parameters()['format'] + if file_format == 'jpeg': + return context.get_current_parameters()['uuid'] + '.jpg' + else: + return context.get_current_parameters()['uuid'] + '.' + file_format + + +class Thumbnail(Base): + __tablename__ = 'thumbnail' + + id = Column(Integer, primary_key=True) + entity_id = Column(Integer) + uuid = Column(String, default=lambda: str(uuid.uuid4()), unique=True) + format = Column(String, default='jpeg') + type = Column(SmallInteger, default=constants.THUMBNAIL_TYPE_COVER) + resolution = Column(SmallInteger, default=constants.COVER_THUMBNAIL_SMALL) + filename = Column(String, default=filename) + generated_at = Column(DateTime, default=lambda: datetime.now(timezone.utc)) + expiration = Column(DateTime, nullable=True) + + +# Add missing tables during migration of database +def add_missing_tables(engine, _session): + if not engine.dialect.has_table(engine.connect(), "archived_book"): + ArchivedBook.__table__.create(bind=engine) + if not engine.dialect.has_table(engine.connect(), "thumbnail"): + Thumbnail.__table__.create(bind=engine) + + +# migrate all settings missing in registration table +def migrate_registration_table(engine, _session): + try: + # Handle table exists, but no content + cnt = _session.query(Registration).count() + if not cnt: + with engine.connect() as conn: + trans = conn.begin() + conn.execute(text("insert into registration (domain, allow) values('%.%',1)")) + trans.commit() + except exc.OperationalError: # Database is not writeable + print('Settings database is not writeable. Exiting...') + sys.exit(2) + + +def migrate_user_session_table(engine, _session): + try: + _session.query(exists().where(User_Sessions.random)).scalar() + _session.commit() + except exc.OperationalError: # Database is not compatible, some columns are missing + with engine.connect() as conn: + trans = conn.begin() + conn.execute(text("ALTER TABLE user_session ADD column 'random' String")) + conn.execute(text("ALTER TABLE user_session ADD column 'expiry' Integer")) + trans.commit() + +def migrate_user_table(engine, _session): + try: + _session.query(exists().where(User.hardcover_token)).scalar() + _session.commit() + except exc.OperationalError: # Database is not compatible, some columns are missing + with engine.connect() as conn: + trans = conn.begin() + conn.execute(text("ALTER TABLE user ADD column 'hardcover_token' String")) + trans.commit() + +# Migrate database to current version, has to be updated after every database change. Currently migration from +# maybe 4/5 versions back to current should work. +# Migration is done by checking if relevant columns are existing, and then adding rows with SQL commands +def migrate_Database(_session): + engine = _session.bind + add_missing_tables(engine, _session) + migrate_registration_table(engine, _session) + migrate_user_session_table(engine, _session) + + +def clean_database(_session): + # Remove expired remote login tokens + now = datetime.now() + try: + _session.query(RemoteAuthToken).filter(now > RemoteAuthToken.expiration).\ + filter(RemoteAuthToken.token_type != 1).delete() + _session.commit() + except exc.OperationalError: # Database is not writeable + print('Settings database is not writeable. Exiting...') + sys.exit(2) + + +# Save downloaded books per user in calibre-web's own database +def update_download(book_id, user_id): + check = session.query(Downloads).filter(Downloads.user_id == user_id).filter(Downloads.book_id == book_id).first() + + if not check: + new_download = Downloads(user_id=user_id, book_id=book_id) + session.add(new_download) + try: + session.commit() + except exc.OperationalError: + session.rollback() + + +# Delete non existing downloaded books in calibre-web's own database +def delete_download(book_id): + session.query(Downloads).filter(book_id == Downloads.book_id).delete() + try: + session.commit() + except exc.OperationalError: + session.rollback() + +# Generate user Guest (translated text), as anonymous user, no rights +def create_anonymous_user(_session): + user = User() + user.name = "Guest" + user.email = 'no@email' + user.role = constants.ROLE_ANONYMOUS + user.password = '' + + _session.add(user) + try: + _session.commit() + except Exception: + _session.rollback() + + +# Generate User admin with admin123 password, and access to everything +def create_admin_user(_session): + user = User() + user.name = "admin" + user.email = "admin@example.org" + user.role = constants.ADMIN_USER_ROLES + user.sidebar_view = constants.ADMIN_USER_SIDEBAR + + user.password = generate_password_hash(constants.DEFAULT_PASSWORD) + + _session.add(user) + try: + _session.commit() + except Exception: + _session.rollback() + +def init_db_thread(): + global app_DB_path + engine = create_engine('sqlite:///{0}'.format(app_DB_path), echo=False) + + Session = scoped_session(sessionmaker()) + Session.configure(bind=engine) + return Session() + + +def init_db(app_db_path): + # Open session for database connection + global session + global app_DB_path + + app_DB_path = app_db_path + engine = create_engine('sqlite:///{0}'.format(app_db_path), echo=False) + + Session = scoped_session(sessionmaker()) + Session.configure(bind=engine) + session = Session() + + if os.path.exists(app_db_path): + Base.metadata.create_all(engine) + migrate_Database(session) + clean_database(session) + else: + Base.metadata.create_all(engine) + create_admin_user(session) + create_anonymous_user(session) + +def password_change(user_credentials=None): + if user_credentials: + username, password = user_credentials.split(':', 1) + user = session.query(User).filter(func.lower(User.name) == username.lower()).first() + if user: + if not password: + print("Empty password is not allowed") + sys.exit(4) + try: + from .helper import valid_password + user.password = generate_password_hash(valid_password(password)) + except Exception: + print("Password doesn't comply with password validation rules") + sys.exit(4) + if session_commit() == "": + print("Password for user '{}' changed".format(username)) + sys.exit(0) + else: + print("Failed changing password") + sys.exit(3) + else: + print("Username '{}' not valid, can't change password".format(username)) + sys.exit(3) + + +def get_new_session_instance(): + new_engine = create_engine('sqlite:///{0}'.format(app_DB_path), echo=False) + new_session = scoped_session(sessionmaker()) + new_session.configure(bind=new_engine) + + atexit.register(lambda: new_session.remove() if new_session else True) + + return new_session + + +def dispose(): + global session + + old_session = session + session = None + if old_session: + try: + old_session.close() + except Exception: + pass + if old_session.bind: + try: + old_session.bind.dispose() + except Exception: + pass + +def session_commit(success=None, _session=None): + s = _session if _session else session + try: + s.commit() + if success: + log.info(success) + except (exc.OperationalError, exc.InvalidRequestError) as e: + s.rollback() + log.error_or_exception(e) + return "" diff --git a/root/app/calibre-web/cps/web.py b/root/app/calibre-web/cps/web.py index 558d768..1c16518 100644 --- a/root/app/calibre-web/cps/web.py +++ b/root/app/calibre-web/cps/web.py @@ -70,7 +70,8 @@ from .string_helper import strip_whitespaces feature_support = { 'ldap': bool(services.ldap), 'goodreads': bool(services.goodreads_support), - 'kobo': bool(services.kobo) + 'kobo': bool(services.kobo), + 'hardcover' : bool(services.hardcover) } try: @@ -1232,7 +1233,7 @@ def serve_book(book_id, book_format, anyname): try: headers = Headers() headers["Content-Type"] = mimetypes.types_map.get('.' + book_format, "application/octet-stream") - if not range_header: + if not range_header: log.info('Serving book: %s', data.name) headers['Accept-Ranges'] = 'bytes' df = getFileFromEbooksFolder(book.path, data.name + "." + book_format) @@ -1497,7 +1498,7 @@ def logout(): # ################################### Users own configuration ######################################################### -def change_profile(kobo_support, local_oauth_check, oauth_status, translations, languages): +def change_profile(kobo_support, hardcover_support, local_oauth_check, oauth_status, translations, languages): to_save = request.form.to_dict() current_user.random_books = 0 try: @@ -1525,6 +1526,7 @@ def change_profile(kobo_support, local_oauth_check, oauth_status, translations, current_user.kobo_only_shelves_sync = int(to_save.get("kobo_only_shelves_sync") == "on") or 0 if old_state == 0 and current_user.kobo_only_shelves_sync == 1: kobo_sync_status.update_on_sync_shelfs(current_user.id) + current_user.hardcover_token = to_save.get("hardcover_token","").replace("Bearer ","") except Exception as ex: flash(str(ex), category="error") @@ -1537,6 +1539,7 @@ def change_profile(kobo_support, local_oauth_check, oauth_status, translations, title=_("%(name)s's Profile", name=current_user.name), page="me", kobo_support=kobo_support, + hardcover_support=hardcover_support, registered_oauth=local_oauth_check, oauth_status=oauth_status) @@ -1568,6 +1571,7 @@ def profile(): languages = calibre_db.speaking_language() translations = get_available_locale() kobo_support = feature_support['kobo'] and config.config_kobo_sync + hardcover_support = feature_support['hardcover'] and config.config_hardcover_sync if feature_support['oauth'] and config.config_login_type == 2: oauth_status = get_oauth_status() local_oauth_check = oauth_check @@ -1576,7 +1580,7 @@ def profile(): local_oauth_check = {} if request.method == "POST": - change_profile(kobo_support, local_oauth_check, oauth_status, translations, languages) + change_profile(kobo_support, hardcover_support, local_oauth_check, oauth_status, translations, languages) return render_title_template("user_edit.html", translations=translations, profile=1, @@ -1584,6 +1588,7 @@ def profile(): content=current_user, config=config, kobo_support=kobo_support, + hardcover_support=hardcover_support, title=_("%(name)s's Profile", name=current_user.name), page="me", registered_oauth=local_oauth_check,