Merge main into auto-hardcover-id - sync with latest changes and resolve conflicts

CONTRIBUTORS

@@ -11,8 +11,8 @@ Copyright (C) 2018-2026 Calibre-Web contributors
 Copyright (C) 2024-2026 Calibre-Web Automated contributors
 # Upstream Contributors (janeczku/calibre-web)
 
-- OzzieIsaacs (anon) (2778 commits)
-- Ozzie Isaacs (anon) (266 commits)
+- OzzieIsaacs (anon) (2788 commits)
+- Ozzie Isaacs (anon) (267 commits)
 - cbartondock (96 commits)
 - idalin (69 commits)
 - cervinko (68 commits)
@@ -41,6 +41,7 @@ Copyright (C) 2024-2026 Calibre-Web Automated contributors
 - mapi68 (15 commits)
 - ok11 (13 commits)
 - pwr (13 commits)
+- webysther (13 commits)
 - Kyosfonica (11 commits)
 - otapi (11 commits)
 - andy29485 (10 commits)
@@ -84,7 +85,6 @@ Copyright (C) 2024-2026 Calibre-Web Automated contributors
 - Petipopotam (4 commits)
 - stfrbrntyu (4 commits)
 - trasba (4 commits)
-- webysther (4 commits)
 - wildthyme (4 commits)
 - acciobugs (3 commits)
 - aliceout (3 commits)
@@ -196,6 +196,7 @@ Copyright (C) 2024-2026 Calibre-Web Automated contributors
 - dotlambda (1 commits)
 - ducbachvan (1 commits)
 - Dunrar (1 commits)
+- EdouardRouch (1 commits)
 - Efreak (1 commits)
 - elelay (1 commits)
 - elfcan (1 commits)
@@ -288,6 +289,7 @@ Copyright (C) 2024-2026 Calibre-Web Automated contributors
 - szepix (1 commits)
 - theopsall (1 commits)
 - Thijxx (1 commits)
+- tomchiverton (1 commits)
 - tomjmul (1 commits)
 - vagra (1 commits)
 - viljasenville (1 commits)
@@ -312,7 +314,7 @@ Copyright (C) 2024-2026 Calibre-Web Automated contributors
 - zhiyue (1 commits)
 # Fork Contributors (crocodilestick/calibre-web-automated)
 
-- crocodilestick (900 commits)
+- crocodilestick (941 commits)
 - jmarmstrong1207 (73 commits)
 - demitrix (30 commits)
 - sirwolfgang (29 commits)
@@ -326,6 +328,7 @@ Copyright (C) 2024-2026 Calibre-Web Automated contributors
 - alva-seal (6 commits)
 - smevawala (5 commits)
 - Aymendje (4 commits)
+- ManuelDrescher (4 commits)
 - opswhisperer (4 commits)
 - alexantao (3 commits)
 - Calychas (3 commits)
@@ -334,12 +337,14 @@ Copyright (C) 2024-2026 Calibre-Web Automated contributors
 - PulsarFTW (3 commits)
 - stewie83 (3 commits)
 - zikasak (3 commits)
+- aaronpowell (2 commits)
+- BrachiumX (2 commits)
 - chad3814 (2 commits)
 - coissac (2 commits)
 - FennyFatal (2 commits)
 - itsmarcy (2 commits)
 - JamesRy96 (2 commits)
-- ManuelDrescher (2 commits)
+- Olen (2 commits)
 - sethvoltz (2 commits)
 - Strubbl (2 commits)
 - tecosaur (2 commits)
@@ -364,6 +369,7 @@ Copyright (C) 2024-2026 Calibre-Web Automated contributors
 - emmanuel-ferdman (1 commits)
 - FilipMakowski (1 commits)
 - Flying-Tom (1 commits)
+- galeksandrp (1 commits)
 - google-labs-jules[bot] (1 commits)
 - gx1400 (1 commits)
 - halkeye (1 commits)
@@ -378,6 +384,7 @@ Copyright (C) 2024-2026 Calibre-Web Automated contributors
 - jeffonthemove (1 commits)
 - jspiers (1 commits)
 - kevpam (1 commits)
+- KucharczykL (1 commits)
 - lazyusername (1 commits)
 - Lee Nave (anon) (1 commits)
 - marauder37 (1 commits)
@@ -388,15 +395,16 @@ Copyright (C) 2024-2026 Calibre-Web Automated contributors
 - n00b42 (1 commits)
 - Nementon (1 commits)
 - NotaInutilis (1 commits)
-- Olen (1 commits)
 - r0b2g1t (1 commits)
 - rjaakke (1 commits)
 - robinwo (1 commits)
 - Rustymage (1 commits)
+- Seth Milliken (anon) (1 commits)
 - smathev (1 commits)
 - spezzino (1 commits)
 - stadler-pascal (1 commits)
 - stefanop1 (1 commits)
+- thetorminal (1 commits)
 - tmacphail (1 commits)
 - tomried (1 commits)
 - Turmaxx (1 commits)

README.md

@@ -46,11 +46,14 @@ Calibre-Web Automated aims to be an all-in-one solution, combining the modern li
 
 ## _Affiliated Projects_ 👬
 
-### Calibre-Web Automated Book Downloader
+### Shelfmark: Book Downloader
 
 - An intuitive web interface for searching and requesting book downloads, designed to work seamlessly with Calibre-Web-Automated. This project streamlines the process of downloading books and preparing them for integration into your Calibre library
 
-[<img src="https://raw.githubusercontent.com/vadret/android/master/assets/get-github.png" alt="Get it on GitHub" height="80">](https://github.com/calibrain/calibre-web-automated-book-downloader)
+> [!IMPORTANT]  
+> CWA does not approve of or support piracy of copyrighted materials and is not responsible for user behaviour
+
+[<img src="https://raw.githubusercontent.com/vadret/android/master/assets/get-github.png" alt="Get it on GitHub" height="80">](https://github.com/calibrain/shelfmark)
 
 ___
 
@@ -142,7 +145,7 @@ This tells CWA to avoid enabling WAL on the Calibre `metadata.db` and the `app.d
   - Using the information provided in the Calibre eBook-converter documentation on which formats convert best into epubs, CWA is able to determine from downloads containing multiple eBook formats, which format will convert most optimally, ignoring the other formats to ensure the **best possible quality** and no **duplicate imports** -->
 
 #### **Automatic Conversion Service** 🔃
-- On by default though can be toggled of in the CWA Settings page, with EPUB as the default target format
+- On by default though can be toggled off in the CWA Settings page, with EPUB as the default target format
   - _Available target formats include:_ **EPUB**, **MOBI**, **AZW3**, **KEPUB** & **PDF**
 - Upon detecting new files in the Ingest Directory, if any of the files are in formats the user has configured CWA to auto-convert to the current target format,
 - The following **28 file types are currently supported:**

cps/__init__.py

@@ -244,35 +244,125 @@ def create_app():
     # Ensure a valid calibre_db session exists before handling each request
     @app.before_request
     def _cwa_ensure_db_session():
+        from flask import g, request
         from .cw_login import current_user
+        from sqlalchemy import or_
         import time
 
+        if config.config_allow_reverse_proxy_header_login:
+            """
+            Load user from reverse proxy authentication header if configured.
+            Sets g.flask_httpauth_user early so that current_user proxy resolves correctly
+            for user-specific settings like theme preferences.
+
+            This must run before any blueprint before_request handlers that access current_user.
+            """
+
+            from . import usermanagement
+            user = usermanagement.load_user_from_reverse_proxy_header(request)
+            if user:
+                g.flask_httpauth_user = user
+            else:
+                # Explicitly set to None to indicate we checked but found nothing
+                g.flask_httpauth_user = None
+
         if current_user.is_authenticated:
-            g.magic_shelves_access = ub.session.query(ub.MagicShelf).filter(ub.MagicShelf.user_id == current_user.id).all()
-            
-            # Magic Shelf Count Caching
-            if 'magic_shelf_counts' not in session:
-                session['magic_shelf_counts'] = {}
-            
-            counts = session['magic_shelf_counts']
-            cache_updated = False
-            now = time.time()
-            CACHE_DURATION = 300  # 5 minutes
-            
-            for shelf in g.magic_shelves_access:
-                shelf_id_str = str(shelf.id)
-                cached_data = counts.get(shelf_id_str)
+            try:
+                # Verify required tables exist before querying
+                from sqlalchemy import inspect
+                inspector = inspect(ub.session.bind)
+                required_tables = ['magic_shelf', 'hidden_magic_shelf_templates']
+                existing_tables = inspector.get_table_names()
                 
-                if cached_data and (now - cached_data.get('timestamp', 0) < CACHE_DURATION):
-                    shelf.book_count = cached_data['count']
-                else:
-                    count = magic_shelf.get_book_count_for_magic_shelf(shelf.id)
-                    counts[shelf_id_str] = {'count': count, 'timestamp': now}
-                    shelf.book_count = count
-                    cache_updated = True
-            
-            if cache_updated:
-                session.modified = True
+                missing_tables = [t for t in required_tables if t not in existing_tables]
+                if missing_tables:
+                    log.error(f"Magic shelf tables missing from database: {missing_tables}. Run migration to create them.")
+                    g.magic_shelves_access = []
+                    return
+                
+                # Get hidden items for this user (both system templates and custom shelves)
+                hidden_items = ub.session.query(
+                    ub.HiddenMagicShelfTemplate.template_key,
+                    ub.HiddenMagicShelfTemplate.shelf_id
+                ).filter(
+                    ub.HiddenMagicShelfTemplate.user_id == current_user.id
+                ).all()
+                
+                hidden_template_keys = {item.template_key for item in hidden_items if item.template_key}
+                hidden_shelf_ids = {item.shelf_id for item in hidden_items if item.shelf_id}
+                
+                # Get user's own shelves + public shelves (will filter hidden ones below)
+                g.magic_shelves_access = ub.session.query(ub.MagicShelf).filter(
+                    or_(
+                        ub.MagicShelf.is_public == 1,
+                        ub.MagicShelf.user_id == current_user.id
+                    )
+                ).all()
+                
+                log.debug(f"Found {len(g.magic_shelves_access)} total magic shelves for user {current_user.id} before filtering")
+                
+                # Filter out hidden items
+                from . import magic_shelf
+                filtered_shelves = []
+                for shelf in g.magic_shelves_access:
+                    # Skip hidden system templates
+                    if shelf.is_system and shelf.user_id == current_user.id:
+                        # Find template key for this system shelf
+                        template_key = None
+                        for key, template in magic_shelf.SYSTEM_SHELF_TEMPLATES.items():
+                            if template['name'] == shelf.name:
+                                template_key = key
+                                break
+                        
+                        # If template_key not found, this is an orphaned/deprecated system shelf
+                        if template_key is None:
+                            log.warning(f"System shelf '{shelf.name}' (ID: {shelf.id}) doesn't match any current template - may need migration")
+                            # Show it anyway - migration should clean it up on next restart
+                            filtered_shelves.append(shelf)
+                            continue
+                        
+                        # Skip if hidden
+                        if template_key in hidden_template_keys:
+                            log.debug(f"Hiding system shelf template '{template_key}' for user {current_user.id}")
+                            continue
+                    
+                    # Skip hidden custom public shelves (not owned by user)
+                    if shelf.is_public == 1 and shelf.user_id != current_user.id:
+                        if shelf.id in hidden_shelf_ids:
+                            log.debug(f"Hiding public shelf '{shelf.name}' (ID: {shelf.id}) for user {current_user.id}")
+                            continue
+                    
+                    filtered_shelves.append(shelf)
+                
+                g.magic_shelves_access = filtered_shelves
+                log.debug(f"Filtered to {len(filtered_shelves)} visible magic shelves for user {current_user.id}")
+                
+                # Magic Shelf Count Caching
+                if 'magic_shelf_counts' not in session:
+                    session['magic_shelf_counts'] = {}
+                
+                counts = session['magic_shelf_counts']
+                cache_updated = False
+                now = time.time()
+                CACHE_DURATION = 300  # 5 minutes
+                
+                for shelf in g.magic_shelves_access:
+                    shelf_id_str = str(shelf.id)
+                    cached_data = counts.get(shelf_id_str)
+                    
+                    if cached_data and (now - cached_data.get('timestamp', 0) < CACHE_DURATION):
+                        shelf.book_count = cached_data['count']
+                    else:
+                        count = magic_shelf.get_book_count_for_magic_shelf(shelf.id)
+                        counts[shelf_id_str] = {'count': count, 'timestamp': now}
+                        shelf.book_count = count
+                        cache_updated = True
+                
+                if cache_updated:
+                    session.modified = True
+            except Exception as e:
+                log.error(f"Error populating magic shelves for user {current_user.id}: {str(e)}", exc_info=True)
+                g.magic_shelves_access = []
         else:
             g.magic_shelves_access = []
         try:
@@ -286,28 +376,6 @@ def create_app():
         if calibre_db.session_factory:
             calibre_db.session_factory.remove()
 
-    # Load user from reverse proxy header early in request lifecycle
-    # This ensures current_user resolves correctly before any code accesses user settings
-    @app.before_request
-    def _load_reverse_proxy_user():
-        """
-        Load user from reverse proxy authentication header if configured.
-        Sets g.flask_httpauth_user early so that current_user proxy resolves correctly
-        for user-specific settings like theme preferences.
-
-        This must run before any blueprint before_request handlers that access current_user.
-        """
-        from flask import g, request
-
-        if config.config_allow_reverse_proxy_header_login:
-            from . import usermanagement
-            user = usermanagement.load_user_from_reverse_proxy_header(request)
-            if user:
-                g.flask_httpauth_user = user
-            else:
-                # Explicitly set to None to indicate we checked but found nothing
-                g.flask_httpauth_user = None
-
     from .schedule import register_scheduled_tasks, register_startup_tasks
     register_scheduled_tasks(config.schedule_reconnect)
     register_startup_tasks()

cps/admin.py

@@ -849,7 +849,7 @@ def update_view_configuration():
     config.config_default_role = constants.selected_roles(to_save)
     config.config_default_role &= ~constants.ROLE_ANONYMOUS
 
-    config.config_default_show = sum(int(k[5:]) for k in to_save if k.startswith('show_'))
+    config.config_default_show = sum(int(k[5:]) for k in to_save if k.startswith('show_') and not k.startswith('show_magic_shelf_') and not k.startswith('show_custom_shelf_'))
     if "Show_detail_random" in to_save:
         config.config_default_show |= constants.DETAIL_RANDOM
 
@@ -1772,6 +1772,30 @@ def edit_user(user_id):
     languages = calibre_db.speaking_language(return_all_languages=True)
     translations = get_available_locale()
     kobo_support = feature_support['kobo'] and config.config_kobo_sync
+    
+    # Get system shelf templates and hidden templates for this user
+    from . import magic_shelf
+    system_shelf_templates = magic_shelf.SYSTEM_SHELF_TEMPLATES
+    hidden_items = ub.session.query(
+        ub.HiddenMagicShelfTemplate.template_key,
+        ub.HiddenMagicShelfTemplate.shelf_id
+    ).filter(
+        ub.HiddenMagicShelfTemplate.user_id == content.id
+    ).all()
+    hidden_shelf_templates = {item.template_key for item in hidden_items if item.template_key}
+    hidden_custom_shelf_ids = {item.shelf_id for item in hidden_items if item.shelf_id}
+    
+    # Get ALL public custom shelves that user doesn't own (both hidden and visible)
+    all_public_shelves = ub.session.query(ub.MagicShelf).filter(
+        ub.MagicShelf.is_public == 1,
+        ub.MagicShelf.user_id != content.id,
+        ub.MagicShelf.is_system == False
+    ).all()
+    
+    # Separate into hidden and visible
+    hidden_custom_shelves = [s for s in all_public_shelves if s.id in hidden_custom_shelf_ids]
+    visible_public_shelves = [s for s in all_public_shelves if s.id not in hidden_custom_shelf_ids]
+    
     if request.method == "POST":
         to_save = request.form.to_dict()
         resp = _handle_edit_user(to_save, content, languages, translations, kobo_support)
@@ -1786,6 +1810,11 @@ def edit_user(user_id):
                                  registered_oauth=oauth_bb.oauth_check,
                                  mail_configured=config.get_mail_server_configured(),
                                  kobo_support=kobo_support,
+                                 system_shelf_templates=system_shelf_templates,
+                                 hidden_shelf_templates=hidden_shelf_templates,
+                                 hidden_custom_shelf_ids=hidden_custom_shelf_ids,
+                                 hidden_custom_shelves=hidden_custom_shelves,
+                                 visible_public_shelves=visible_public_shelves,
                                  title=_("Edit User %(nick)s", nick=content.name),
                                  page="edituser")
 
@@ -2443,7 +2472,7 @@ def _handle_edit_user(to_save, content, languages, translations, kobo_support):
         flash(_("No admin user remaining, can't remove admin role"), category="error")
         return redirect(url_for('admin.admin'))
 
-    val = [int(k[5:]) for k in to_save if k.startswith('show_')]
+    val = [int(k[5:]) for k in to_save if k.startswith('show_') and not k.startswith('show_magic_shelf_') and not k.startswith('show_custom_shelf_')]
     sidebar, __ = get_sidebar_config()
     for element in sidebar:
         value = element['visibility']
@@ -2467,6 +2496,65 @@ def _handle_edit_user(to_save, content, languages, translations, kobo_support):
     # Auto-send and metadata fetch settings
     content.auto_send_enabled = to_save.get("auto_send_enabled") == "on"
     content.auto_metadata_fetch = to_save.get("auto_metadata_fetch") == "on"
+    
+    # Handle hidden magic shelf templates and custom shelves
+    from . import magic_shelf
+    if not content.is_anonymous:
+        # Get all system template keys
+        all_template_keys = set(magic_shelf.SYSTEM_SHELF_TEMPLATES.keys())
+        # Get currently hidden items for this user
+        current_hidden = ub.session.query(ub.HiddenMagicShelfTemplate).filter(
+            ub.HiddenMagicShelfTemplate.user_id == content.id
+        ).all()
+        current_hidden_template_keys = {h.template_key for h in current_hidden if h.template_key}
+        current_hidden_shelf_ids = {h.shelf_id for h in current_hidden if h.shelf_id}
+        
+        # Handle system templates
+        visible_template_keys = {key for key in all_template_keys if to_save.get(f"show_magic_shelf_{key}") == "on"}
+        should_be_hidden_templates = all_template_keys - visible_template_keys
+        
+        # Add newly hidden templates
+        for key in should_be_hidden_templates:
+            if key not in current_hidden_template_keys:
+                new_hidden = ub.HiddenMagicShelfTemplate(
+                    user_id=content.id,
+                    template_key=key
+                )
+                ub.session.add(new_hidden)
+                log.info(f"User {content.id} hid system shelf template '{key}'")
+        
+        # Remove templates that should no longer be hidden
+        for hidden in current_hidden:
+            if hidden.template_key and hidden.template_key in visible_template_keys:
+                ub.session.delete(hidden)
+                log.info(f"User {content.id} unhid system shelf template '{hidden.template_key}'")
+        
+        # Handle custom public shelves - get all available ones
+        all_public_shelves = ub.session.query(ub.MagicShelf).filter(
+            ub.MagicShelf.is_public == 1,
+            ub.MagicShelf.user_id != content.id,
+            ub.MagicShelf.is_system == False
+        ).all()
+        
+        # Check which ones should be visible (checked)
+        visible_shelf_ids = {s.id for s in all_public_shelves if to_save.get(f"show_custom_shelf_{s.id}") == "on"}
+        
+        # Hide shelves that are unchecked but not currently hidden
+        for shelf in all_public_shelves:
+            if shelf.id not in visible_shelf_ids and shelf.id not in current_hidden_shelf_ids:
+                new_hidden = ub.HiddenMagicShelfTemplate(
+                    user_id=content.id,
+                    shelf_id=shelf.id
+                )
+                ub.session.add(new_hidden)
+                log.info(f"User {content.id} hid custom shelf {shelf.id}")
+        
+        # Unhide shelves that are checked but currently hidden
+        for hidden in current_hidden:
+            if hidden.shelf_id and hidden.shelf_id in visible_shelf_ids:
+                ub.session.delete(hidden)
+                log.info(f"User {content.id} unhid custom shelf {hidden.shelf_id}")
+    
     if to_save.get("default_language"):
         content.default_language = to_save["default_language"]
     if to_save.get("locale"):

cps/cwa_functions.py

@@ -19,7 +19,7 @@ from pathlib import Path
 from time import sleep
 
 import json
-from threading import Thread
+from threading import Thread, Lock, Timer
 import queue
 import os
 import tempfile
@@ -59,6 +59,10 @@ log = logger.create()
 LOG_ARCHIVE = "/config/log_archive"
 DIRS_JSON = "/app/calibre-web-automated/dirs.json"
 
+# Debounced duplicate scan timer (web process)
+_duplicate_scan_timer = None
+_duplicate_scan_lock = Lock()
+
 ##———————————————————————————END OF GLOBAL VARIABLES——————————————————————————##
 
 ##————————————————————————————————————————————————————————————————————————————##
@@ -346,6 +350,57 @@ def cwa_internal_schedule_auto_send():
         log.error(f"Internal auto-send schedule failed: {e}")
         return jsonify({"error": str(e)}), 400
 
+
+@csrf.exempt
+@cwa_internal.route('/cwa-internal/queue-duplicate-scan', methods=["POST"])
+def cwa_internal_queue_duplicate_scan():
+    """Debounce and queue an incremental duplicate scan in the web process.
+
+    Security: Limited to localhost callers (within container/host).
+    Payload JSON: {delay_seconds:int}
+    """
+    try:
+        remote = request.headers.get('X-Forwarded-For', request.remote_addr)
+        if remote not in (None, '127.0.0.1', '::1'):
+            abort(403)
+
+        db = CWA_DB()
+        enabled = bool(db.cwa_settings.get('duplicate_scan_enabled', 0))
+        frequency = db.cwa_settings.get('duplicate_scan_frequency', 'manual')
+
+        data = request.get_json(force=True, silent=True) or {}
+        default_delay = db.cwa_settings.get('duplicate_scan_debounce_seconds', 30)
+        delay_seconds = int(data.get('delay_seconds', default_delay))
+        delay_seconds = max(5, min(600, delay_seconds))
+
+        if not enabled or frequency != 'after_import':
+            return jsonify({"success": True, "skipped": True, "reason": "disabled_or_manual"}), 200
+
+        global _duplicate_scan_timer
+        with _duplicate_scan_lock:
+            if _duplicate_scan_timer is not None:
+                try:
+                    _duplicate_scan_timer.cancel()
+                except Exception:
+                    pass
+
+            def _enqueue_scan():
+                try:
+                    from .tasks.duplicate_scan import TaskDuplicateScan
+                    WorkerThread.add('System', TaskDuplicateScan(full_scan=False, trigger_type='after_import'), hidden=False)
+                    log.info("[cwa-duplicates] Debounced duplicate scan queued (after_import)")
+                except Exception as e:
+                    log.error("[cwa-duplicates] Failed to queue debounced duplicate scan: %s", str(e))
+
+            _duplicate_scan_timer = Timer(delay_seconds, _enqueue_scan)
+            _duplicate_scan_timer.daemon = True
+            _duplicate_scan_timer.start()
+
+        return jsonify({"success": True, "queued": True, "delay_seconds": delay_seconds}), 200
+    except Exception as e:
+        log.error("[cwa-duplicates] Failed to schedule debounced duplicate scan: %s", str(e))
+        return jsonify({"success": False, "error": str(e)}), 500
+
 @csrf.exempt
 @cwa_internal.route('/cwa-internal/schedule-convert-library', methods=["POST"])
 def cwa_internal_schedule_convert_library():
@@ -554,9 +609,9 @@ def set_cwa_settings():
     boolean_settings = []
     string_settings = []
     list_settings = []
-    integer_settings = ['ingest_timeout_minutes', 'auto_send_delay_minutes', 'hardcover_auto_fetch_batch_size', 'hardcover_auto_fetch_schedule_hour']  # Special handling for integer settings
+    integer_settings = ['ingest_timeout_minutes', 'auto_send_delay_minutes', 'hardcover_auto_fetch_batch_size', 'hardcover_auto_fetch_schedule_hour', 'duplicate_scan_hour', 'duplicate_scan_chunk_size', 'duplicate_scan_debounce_seconds']  # Special handling for integer settings
     float_settings = ['hardcover_auto_fetch_min_confidence', 'hardcover_auto_fetch_rate_limit']  # Special handling for float settings
-    json_settings = ['metadata_provider_hierarchy', 'metadata_providers_enabled']  # Special handling for JSON settings
+    json_settings = ['metadata_provider_hierarchy', 'metadata_providers_enabled', 'duplicate_format_priority']  # Special handling for JSON settings
     
     for setting in cwa_default_settings:
         if setting in integer_settings or setting in float_settings or setting in json_settings:
@@ -568,6 +623,10 @@ def set_cwa_settings():
         else:
             list_settings.append(setting)
 
+    # Ensure cron expression is treated as a string even if default is empty
+    if 'duplicate_scan_cron' not in string_settings:
+        string_settings.append('duplicate_scan_cron')
+
     for format in ignorable_formats:
         string_settings.append(f"ignore_ingest_{format}")
         string_settings.append(f"ignore_convert_{format}")
@@ -602,7 +661,6 @@ def set_cwa_settings():
                 elif setting == "auto_convert_target_format":
                     if value is None:
                         value = cwa_db.cwa_settings['auto_convert_target_format']
-                    value = cwa_db.cwa_settings['auto_convert_target_format']
 
                 result |= {setting:value}
             
@@ -636,6 +694,12 @@ def set_cwa_settings():
                             int_value = max(10, min(200, int_value))  # Clamp between 10 and 200
                         elif setting == 'hardcover_auto_fetch_schedule_hour':
                             int_value = max(0, min(23, int_value))  # Clamp between 0 and 23 hours
+                        elif setting == 'duplicate_scan_hour':
+                            int_value = max(0, min(23, int_value))
+                        elif setting == 'duplicate_scan_chunk_size':
+                            int_value = max(500, min(50000, int_value))
+                        elif setting == 'duplicate_scan_debounce_seconds':
+                            int_value = max(5, min(600, int_value))
                         result[setting] = int_value
                     except (ValueError, TypeError):
                         # Use current value if conversion fails
@@ -647,6 +711,8 @@ def set_cwa_settings():
                             result[setting] = cwa_db.cwa_settings.get(setting, 50)  # Default to 50
                         elif setting == 'hardcover_auto_fetch_schedule_hour':
                             result[setting] = cwa_db.cwa_settings.get(setting, 2)  # Default to 2 AM
+                        elif setting == 'duplicate_scan_debounce_seconds':
+                            result[setting] = cwa_db.cwa_settings.get(setting, 30)
                 else:
                     if setting == 'ingest_timeout_minutes':
                         result[setting] = cwa_db.cwa_settings.get(setting, 15)  # Default to 15 minutes
@@ -656,6 +722,8 @@ def set_cwa_settings():
                         result[setting] = cwa_db.cwa_settings.get(setting, 50)  # Default to 50
                     elif setting == 'hardcover_auto_fetch_schedule_hour':
                         result[setting] = cwa_db.cwa_settings.get(setting, 2)  # Default to 2 AM
+                    elif setting == 'duplicate_scan_debounce_seconds':
+                        result[setting] = cwa_db.cwa_settings.get(setting, 30)
 
             # Handle float settings
             for setting in float_settings:
@@ -681,6 +749,7 @@ def set_cwa_settings():
                     elif setting == 'hardcover_auto_fetch_rate_limit':
                         result[setting] = cwa_db.cwa_settings.get(setting, 5.0)  # Default to 5.0 seconds
 
+
             # Handle JSON settings
             for setting in json_settings:
                 value = request.form.get(setting)
@@ -726,6 +795,17 @@ def set_cwa_settings():
                     else:
                         result[setting] = cwa_db.cwa_settings.get(setting, '[]')
 
+            # Validate cron expression if provided
+            cron_expr = result.get('duplicate_scan_cron', '')
+            if cron_expr:
+                try:
+                    from apscheduler.triggers.cron import CronTrigger
+                    CronTrigger.from_crontab(cron_expr)
+                except Exception:
+                    # Revert to previous value and notify user
+                    result['duplicate_scan_cron'] = cwa_db.cwa_settings.get('duplicate_scan_cron', '')
+                    flash(_("Invalid cron expression for duplicate scans. Changes were not saved."), category="error")
+
             # DEBUGGING
             # with open("/config/post_request" ,"w") as f:
             #     for key in result.keys():
@@ -757,10 +837,35 @@ def set_cwa_settings():
         getenv("HARDCOVER_TOKEN")
     )
 
+    next_scan_run = get_next_duplicate_scan_run(cwa_settings)
+
     return render_title_template("cwa_settings.html", title=_("Calibre-Web Automated User Settings"), page="cwa-settings",
                                     cwa_settings=cwa_settings, ignorable_formats=ignorable_formats, target_formats=target_formats,
                                     automerge_options=automerge_options, autoingest_options=autoingest_options,
-                                    hardcover_token_available=hardcover_token_available)
+                                    hardcover_token_available=hardcover_token_available,
+                                    next_duplicate_scan_run=next_scan_run, config=config)
+
+
+def get_next_duplicate_scan_run(settings):
+    """Compute next scheduled duplicate scan run time based on settings."""
+    try:
+        enabled = bool(settings.get('duplicate_scan_enabled', 0))
+        cron_expr = (settings.get('duplicate_scan_cron') or '').strip()
+
+        if not enabled:
+            return None
+
+        if not cron_expr:
+            return None
+
+        from apscheduler.triggers.cron import CronTrigger
+        now = datetime.now().astimezone()
+        trigger = CronTrigger.from_crontab(cron_expr, timezone=now.tzinfo)
+        next_run = trigger.get_next_fire_time(None, now)
+        return next_run.isoformat() if next_run else None
+    except Exception:
+        return None
+
 
 ##————————————————————————————————————————————————————————————————————————————##
 ##                                                                            ##

cps/db.py

@@ -8,6 +8,8 @@
 import os
 import re
 import json
+import time
+import threading
 from datetime import datetime, timezone
 from urllib.parse import quote
 import unidecode
@@ -531,6 +533,7 @@ class CalibreDB:
     # This is a WeakSet so that references here don't keep other CalibreDB
     # instances alive once they reach the end of their respective scopes
     instances = WeakSet()
+    _reconnect_lock = threading.RLock()  # Reentrant lock to prevent concurrent reconnect operations
 
     def __init__(self, expire_on_commit=True, init=False):
         """ Initialize a new CalibreDB session
@@ -546,6 +549,9 @@ class CalibreDB:
         self.instances.add(self)
 
     def init_session(self, expire_on_commit=True):
+        if self.session_factory is None:
+            log.error("Cannot init session: session_factory is None")
+            return
         self.session = self.session_factory()
         self.session.expire_on_commit = expire_on_commit
         self.create_functions(self.config)
@@ -553,23 +559,42 @@ class CalibreDB:
     def ensure_session(self, expire_on_commit=True):
         """Ensure a valid SQLAlchemy session exists.
         This protects against brief windows where dispose() nulled the session during a reconnect.
+        Holds lock during entire recreation to prevent race conditions.
         """
-        try:
-            if self.session is None:
-                # Recreate a session from the factory if available
-                if self.session_factory is not None:
+        if self.session is not None:
+            return  # Fast path - session already exists
+        
+        # Session is None - need to recreate it
+        # Acquire lock to ensure atomic recreation (no interruption by dispose)
+        with self._reconnect_lock:
+            # Double-check after acquiring lock (another thread may have recreated it)
+            if self.session is not None:
+                return
+            
+            # Try to recreate session from factory
+            if self.session_factory is not None:
+                try:
                     self.init_session(expire_on_commit)
-                else:
-                    # As a last resort, try to rebuild the setup if config is present
-                    if self.config and getattr(self.config, 'config_calibre_dir', None):
-                        try:
-                            self.setup_db(self.config.config_calibre_dir, ub.app_DB_path)
-                            self.init_session(expire_on_commit)
-                        except Exception as ex:
-                            log.error_or_exception(ex)
-        except Exception:
-            # Never let session recovery raise in callers; they will fail later with proper logging
-            pass
+                    return  # Success
+                except Exception as ex:
+                    log.error(f"Failed to init session from factory: {ex}")
+            
+            # Factory is None or init failed - try to rebuild entire database setup
+            if self.config and getattr(self.config, 'config_calibre_dir', None):
+                try:
+                    log.warning("Session factory unavailable, attempting to rebuild database setup")
+                    # Note: setup_db will call dispose() which is safe because we hold _reconnect_lock (RLock is reentrant)
+                    self.setup_db(self.config.config_calibre_dir, ub.app_DB_path)
+                    # After setup_db, session_factory should exist, try to init session
+                    if self.session is None and self.session_factory is not None:
+                        self.init_session(expire_on_commit)
+                except Exception as ex:
+                    log.error(f"Failed to rebuild database setup in ensure_session: {ex}")
+            
+            # If we still don't have a session, log warning
+            # Don't raise exception - let caller handle AttributeError if they try to use None session
+            if self.session is None:
+                log.error("ensure_session: Unable to create session - session factory and config unavailable")
 
     @classmethod
     def setup_db_cc_classes(cls, cc):
@@ -684,67 +709,75 @@ class CalibreDB:
 
     @classmethod
     def setup_db(cls, config_calibre_dir, app_db_path):
-        cls.dispose()
+        # Wrap entire method in lock to ensure atomic setup operation
+        # RLock is reentrant, so nested calls (e.g., from reconnect_db) are safe
+        with cls._reconnect_lock:
+            # Always call dispose to clean up old sessions/connections
+            cls.dispose()
 
-        if not config_calibre_dir:
-            if cls.config:
-                cls.config.invalidate()
-            return None
-
-        dbpath = os.path.join(config_calibre_dir, "metadata.db")
-        if not os.path.exists(dbpath):
-            if cls.config:
-                cls.config.invalidate()
-            return None
-
-        try:
-            cls.engine = create_engine('sqlite://',
-                                       echo=False,
-                                       isolation_level="SERIALIZABLE",
-                                       connect_args={'check_same_thread': False, 'timeout': 30},
-                                       poolclass=StaticPool)
-            with cls.engine.begin() as connection:
-                connection.execute(text("attach database '{}' as calibre;".format(dbpath)))
-                connection.execute(text("attach database '{}' as app_settings;".format(app_db_path)))
-                # Try enabling WAL to improve concurrency unless running on a network share
-                # Controlled by env var NETWORK_SHARE_MODE (default False)
-                try:
-                    nsm = os.getenv('NETWORK_SHARE_MODE', 'False').lower() in ('1', 'true', 'yes', 'on')
-                    if not nsm:
-                        connection.execute(text("PRAGMA calibre.journal_mode=WAL"))
-                        connection.execute(text("PRAGMA app_settings.journal_mode=WAL"))
-                except Exception:
-                    pass
-
-            conn = cls.engine.connect()
-            # conn.text_factory = lambda b: b.decode(errors = 'ignore') possible fix for #1302
-        except Exception as ex:
-            if cls.config:
-                cls.config.invalidate(ex)
-            return None
-
-        if cls.config:
-            cls.config.db_configured = True
-
-        if not cc_classes:
-            try:
-                cc = conn.execute(text("SELECT id, datatype FROM custom_columns"))
-                cls.setup_db_cc_classes(cc)
-            except OperationalError as e:
-                log.error_or_exception(e)
+            if not config_calibre_dir:
+                log.error("setup_db failed: config_calibre_dir is None or empty")
+                if cls.config:
+                    cls.config.invalidate()
                 return None
 
-        cls.session_factory = scoped_session(sessionmaker(autocommit=False,
-                                                          autoflush=True,
-                                                          bind=cls.engine, future=True))
-        for inst in cls.instances:
-            inst.init_session()
+            dbpath = os.path.join(config_calibre_dir, "metadata.db")
+            if not os.path.exists(dbpath):
+                log.error(f"setup_db failed: metadata.db not found at {dbpath}")
+                if cls.config:
+                    cls.config.invalidate()
+                return None
 
-        # Ensure progress syncing tables exist in metadata.db (book checksums)
-        from .progress_syncing.models import ensure_calibre_db_tables
-        ensure_calibre_db_tables(conn)
+            try:
+                cls.engine = create_engine('sqlite://',
+                                           echo=False,
+                                           isolation_level="SERIALIZABLE",
+                                           connect_args={'check_same_thread': False, 'timeout': 30},
+                                           poolclass=StaticPool)
+                with cls.engine.begin() as connection:
+                    connection.execute(text("attach database '{}' as calibre;".format(dbpath)))
+                    connection.execute(text("attach database '{}' as app_settings;".format(app_db_path)))
+                    # Try enabling WAL to improve concurrency unless running on a network share
+                    # Controlled by env var NETWORK_SHARE_MODE (default False)
+                    try:
+                        nsm = os.getenv('NETWORK_SHARE_MODE', 'False').lower() in ('1', 'true', 'yes', 'on')
+                        if not nsm:
+                            connection.execute(text("PRAGMA calibre.journal_mode=WAL"))
+                            connection.execute(text("PRAGMA app_settings.journal_mode=WAL"))
+                    except Exception:
+                        pass
 
-        cls._init = True
+                conn = cls.engine.connect()
+                # conn.text_factory = lambda b: b.decode(errors = 'ignore') possible fix for #1302
+            except Exception as ex:
+                log.error(f"setup_db failed during engine creation: {ex}")
+                if cls.config:
+                    cls.config.invalidate(ex)
+                return None
+
+            if cls.config:
+                cls.config.db_configured = True
+
+            if not cc_classes:
+                try:
+                    cc = conn.execute(text("SELECT id, datatype FROM custom_columns"))
+                    cls.setup_db_cc_classes(cc)
+                except OperationalError as e:
+                    log.error_or_exception(e)
+                    return None
+
+            cls.session_factory = scoped_session(sessionmaker(autocommit=False,
+                                                              autoflush=True,
+                                                              bind=cls.engine, future=True))
+            for inst in cls.instances:
+                inst.init_session()
+
+            # Ensure progress syncing tables exist in metadata.db (book checksums)
+            from .progress_syncing.models import ensure_calibre_db_tables
+            ensure_calibre_db_tables(conn)
+
+            cls._init = True
+        # End of with cls._reconnect_lock
 
     def get_book(self, book_id):
         self.ensure_session()
@@ -1146,6 +1179,10 @@ class CalibreDB:
 
     def create_functions(self, config=None):
         self.ensure_session()
+        if self.session is None:
+            log.error("create_functions: Cannot create functions because session is None")
+            return
+        
         # user defined sort function for calibre databases (Series, etc.)
         if config:
             def _title_sort(title):
@@ -1174,28 +1211,29 @@ class CalibreDB:
     @classmethod
     def dispose(cls):
         # global session
-
-        for inst in cls.instances:
-            old_session = inst.session
-            inst.session = None
-            if old_session:
-                try:
-                    old_session.close()
-                except Exception:
-                    pass
-                if old_session.bind:
+        # Use lock to prevent concurrent dispose/reconnect operations
+        with cls._reconnect_lock:
+            for inst in cls.instances:
+                old_session = inst.session
+                inst.session = None
+                if old_session:
                     try:
-                        old_session.bind.dispose()
+                        old_session.close()
                     except Exception:
                         pass
+                    if old_session.bind:
+                        try:
+                            old_session.bind.dispose()
+                        except Exception:
+                            pass
 
-        for attr in list(Books.__dict__.keys()):
-            if attr.startswith("custom_column_"):
-                setattr(Books, attr, None)
+            for attr in list(Books.__dict__.keys()):
+                if attr.startswith("custom_column_"):
+                    setattr(Books, attr, None)
 
-        for db_class in cc_classes.values():
-            Base.metadata.remove(db_class.__table__)
-        cc_classes.clear()
+            for db_class in cc_classes.values():
+                Base.metadata.remove(db_class.__table__)
+            cc_classes.clear()
 
         for table in reversed(Base.metadata.sorted_tables):
             name = table.key
@@ -1204,24 +1242,26 @@ class CalibreDB:
                     Base.metadata.remove(table)
 
     def reconnect_db(self, config, app_db_path):
-        # Be resilient if database wasn't initialized yet
-        try:
-            self.dispose()
-        except Exception:
-            # Ignore dispose errors during reconnect
-            pass
+        # Use lock to ensure atomic reconnect operation
+        with self._reconnect_lock:
+            # Be resilient if database wasn't initialized yet
+            try:
+                self.dispose()
+            except Exception:
+                # Ignore dispose errors during reconnect
+                pass
 
-        # engine is a class-level attribute that may be None before first setup
-        try:
-            if getattr(self, 'engine', None) is not None:
-                self.engine.dispose()
-        except Exception:
-            # Ignore engine dispose errors; we'll rebuild below
-            pass
+            # engine is a class-level attribute that may be None before first setup
+            try:
+                if getattr(self, 'engine', None) is not None:
+                    self.engine.dispose()
+            except Exception:
+                # Ignore engine dispose errors; we'll rebuild below
+                pass
 
-        # Rebuild engine/session factory and update config
-        self.setup_db(config.config_calibre_dir, app_db_path)
-        self.update_config(config)
+            # Rebuild engine/session factory and update config
+            self.setup_db(config.config_calibre_dir, app_db_path)
+            self.update_config(config)
 
 
 def lcase(s):

cps/editbooks.py

@@ -6,6 +6,7 @@
 # See CONTRIBUTORS for full list of authors.
 
 import os
+import sys
 from datetime import datetime, timezone
 import json
 from shutil import copyfile
@@ -544,6 +545,7 @@ def delete_selected_books():
     if vals:
         for book_id in vals:
             delete_book_from_table(book_id, "", True)
+        _queue_duplicate_scan_after_change()
         return json.dumps({'success': True})
     return ""
 
@@ -602,11 +604,36 @@ def merge_list_book():
                                                         element.format,
                                                         element.uncompressed_size,
                                                         to_name))
+                            to_file.append(element.format)
                     delete_book_from_table(from_book.id, "", True)
-                    return json.dumps({'success': True})
+            calibre_db.session.commit()
+            _queue_duplicate_scan_after_change()
+            return json.dumps({'success': True})
     return ""
 
 
+def _queue_duplicate_scan_after_change():
+    """Queue a debounced duplicate scan after manual changes."""
+    try:
+        import requests
+        sys.path.insert(1, '/app/calibre-web-automated/scripts/')
+        from cwa_db import CWA_DB
+
+        cwa_db = CWA_DB()
+        delay_seconds = int(cwa_db.cwa_settings.get('duplicate_scan_debounce_seconds', 30))
+        delay_seconds = max(5, min(600, delay_seconds))
+        url = helper.get_internal_api_url("/cwa-internal/queue-duplicate-scan")
+        requests.post(
+            url,
+            json={"delay_seconds": delay_seconds},
+            headers={"X-Forwarded-For": "127.0.0.1"},
+            timeout=5,
+            verify=False,
+        )
+    except Exception as e:
+        log.error("Failed to queue duplicate scan after change: %s", str(e))
+
+
 @editbook.route("/ajax/xchange", methods=['POST'])
 @user_login_required
 @edit_required
@@ -1171,6 +1198,18 @@ def delete_book_from_table(book_id, book_format, json_response, location=""):
                     if book_format.upper() in ['KEPUB', 'EPUB', 'EPUB3']:
                         kobo_sync_status.remove_synced_book(book.id, True)
                 calibre_db.session.commit()
+                
+                # Invalidate duplicate cache after book deletion
+                try:
+                    import sys
+                    sys.path.insert(1, '/app/calibre-web-automated/scripts/')
+                    from cwa_db import CWA_DB
+                    cwa_db = CWA_DB()
+                    cwa_db.invalidate_duplicate_cache()
+                    cwa_db.close()
+                except Exception as e:
+                    log.error("Failed to invalidate duplicate cache after deletion: %s", str(e))
+                
             except Exception as ex:
                 log.error_or_exception(ex)
                 calibre_db.session.rollback()

cps/helper.py

@@ -430,8 +430,51 @@ def rename_all_files_on_change(one_book, new_path, old_path, all_new_name, gdriv
         if not gdrive:
             if not os.path.exists(new_path):
                 os.makedirs(new_path)
-            shutil.move(os.path.join(old_path, file_format.name + '.' + file_format.format.lower()),
-                    os.path.join(new_path, all_new_name + '.' + file_format.format.lower()))
+            
+            old_file = os.path.join(old_path, file_format.name + '.' + file_format.format.lower())
+            new_file = os.path.join(new_path, all_new_name + '.' + file_format.format.lower())
+            
+            # Skip if source and destination are the same
+            if old_file == new_file:
+                log.debug("Skipping file rename - source and destination are identical: %s", old_file)
+                continue
+            
+            # Check if source file exists
+            if not os.path.exists(old_file):
+                log.warning("Source file not found for rename: %s", old_file)
+                # Check if the file already has the new name (perhaps from a previous partial operation)
+                if os.path.exists(new_file):
+                    log.info("File already exists at destination: %s", new_file)
+                    file_format.name = all_new_name
+                    continue
+                else:
+                    log.error("Neither old nor new file exists - cannot rename %s to %s", old_file, new_file)
+                    continue
+            
+            # Check if destination already exists
+            if os.path.exists(new_file) and old_file != new_file:
+                log.warning("Destination file already exists, will overwrite: %s", new_file)
+                try:
+                    os.remove(new_file)
+                except OSError as ex:
+                    log.error("Could not remove existing destination file %s: %s", new_file, ex)
+            
+            # Attempt to rename the file
+            try:
+                shutil.move(old_file, new_file)
+                log.debug("Successfully renamed %s to %s", old_file, new_file)
+            except OSError as ex:
+                log.error("Failed to rename file from %s to %s: %s", old_file, new_file, ex)
+                # Try copy+delete as fallback for permission issues (e.g., network shares)
+                try:
+                    log.info("Attempting copy+delete fallback for %s", old_file)
+                    shutil.copy2(old_file, new_file)
+                    os.remove(old_file)
+                    log.info("Successfully copied and removed old file: %s", old_file)
+                except (OSError, IOError) as fallback_ex:
+                    log.error("Copy+delete fallback also failed for %s: %s", old_file, fallback_ex)
+                    # Don't update the database name if we failed to rename the file
+                    continue
         else:
             g_file = gd.getFileFromEbooksFolder(old_path,
                                                 file_format.name + '.' + file_format.format.lower())
@@ -561,28 +604,64 @@ def move_files_on_change(calibre_path, new_author_dir, new_titledir, localbook,
                 os.makedirs(new_path)
             try:
                 shutil.move(original_filepath, os.path.join(new_path, db_filename))
-            except OSError:
-                log.error("Rename title from {} to {} failed with error, trying to "
-                          "move without metadata".format(path, new_path))
-                shutil.move(original_filepath, os.path.join(new_path, db_filename), copy_function=shutil.copy)
+            except OSError as ex:
+                log.error("Rename title from %s to %s failed with error: %s, trying copy+delete fallback",
+                         path, new_path, ex)
+                try:
+                    shutil.copy2(original_filepath, os.path.join(new_path, db_filename))
+                    os.remove(original_filepath)
+                except (OSError, IOError) as fallback_ex:
+                    log.error("Copy+delete fallback also failed: %s", fallback_ex)
+                    raise
             log.debug("Moving title: %s to %s", original_filepath, new_path)
         else:
             # Check new path is not valid path
             if not os.path.exists(new_path):
                 # move original path to new path
                 log.debug("Moving title: %s to %s", path, new_path)
-                shutil.move(path, new_path)
+                try:
+                    shutil.move(path, new_path)
+                except OSError as ex:
+                    log.error("Failed to move directory %s to %s: %s, trying copy tree approach", path, new_path, ex)
+                    # Fallback: try to copy tree and then remove original
+                    try:
+                        shutil.copytree(path, new_path, dirs_exist_ok=True)
+                        shutil.rmtree(path)
+                    except (OSError, IOError) as fallback_ex:
+                        log.error("Copy tree fallback also failed: %s", fallback_ex)
+                        raise
             else:  # path is valid copy only files to new location (merge)
                 log.info("Moving title: %s into existing: %s", path, new_path)
                 # Take all files and subfolder from old path (strange command)
                 for dir_name, __, file_list in os.walk(path):
                     for file in file_list:
-                        shutil.move(os.path.join(dir_name, file), os.path.join(new_path + dir_name[len(path):], file))
-            if not os.listdir(os.path.split(path)[0]):
+                        src_file = os.path.join(dir_name, file)
+                        dest_dir = new_path + dir_name[len(path):]
+                        dest_file = os.path.join(dest_dir, file)
+                        
+                        # Create destination directory if it doesn't exist
+                        if not os.path.exists(dest_dir):
+                            os.makedirs(dest_dir)
+                        
+                        try:
+                            shutil.move(src_file, dest_file)
+                        except OSError as ex:
+                            log.error("Failed to move file %s to %s: %s, trying copy+delete", src_file, dest_file, ex)
+                            try:
+                                shutil.copy2(src_file, dest_file)
+                                os.remove(src_file)
+                            except (OSError, IOError) as fallback_ex:
+                                log.error("Copy+delete fallback failed for %s: %s", src_file, fallback_ex)
+                                # Continue with other files even if one fails
+                                continue
+            
+            # Try to remove old author directory if empty
+            if os.path.exists(os.path.split(path)[0]) and not os.listdir(os.path.split(path)[0]):
                 try:
                     shutil.rmtree(os.path.split(path)[0])
                 except (IOError, OSError) as ex:
                     log.error("Deleting authorpath for book %s failed: %s", localbook.id, ex)
+        
         # change location in database to new author/title path
         localbook.path = os.path.join(new_author_dir, new_titledir).replace('\\', '/')
     except OSError as ex:

cps/kobo.py

@@ -328,25 +328,49 @@ def HandleSyncRequest():
 
     # Add magic shelves as collections
     if config.config_kobo_sync_magic_shelves:
+
+        for shelf in ub.session.query(ub.MagicShelf)\
+            .filter_by(user_id=current_user.id, kobo_sync=False)\
+            .all():
+
+            sync_results.append({
+                "DeletedTag": {
+                    "Tag": {
+                        "Id": shelf.uuid,
+                        "LastModified": convert_to_kobo_timestamp_string(shelf.last_modified)
+                    }
+                }
+            })
+
         magic_shelves = ub.session.query(ub.MagicShelf)\
             .filter_by(user_id=current_user.id, kobo_sync=True)\
             .all()
+
+        new_tags_last_modified = sync_token.tags_last_modified
             
         for shelf in magic_shelves:
             books, _ = magic_shelf.get_books_for_magic_shelf(
-                shelf.id, current_user.id, page=1, page_size=1000
+                shelf.id, page=1, page_size=1000
             )
-            
-            collection = {
-                "Id": f"magic-shelf-{shelf.id}",
-                "Name": shelf.name,
-                "Type": "Collection",
-                "Items": [
-                    {"Type": "Book", "Id": str(book.uuid)}
-                    for book in books
-                ]
-            }
-            sync_results.append({"Collection": collection})
+
+            new_tags_last_modified = max(shelf.last_modified, new_tags_last_modified)
+
+            tag = create_kobo_tag_magic(shelf, books)
+            if not tag:
+                continue
+
+            if shelf.created > sync_token.tags_last_modified:
+                log.debug("Syncing new magic shelf %s to Kobo device", shelf.name)
+                sync_results.append({
+                    "NewTag": tag
+                })
+            else:
+                log.debug("Syncing changed magic shelf %s to Kobo device", shelf.name)
+                sync_results.append({
+                    "ChangedTag": tag
+                })
+
+        sync_token.tags_last_modified = new_tags_last_modified
 
     # update last created timestamp to distinguish between new and changed entitlements
     if not cont_sync:
@@ -823,6 +847,25 @@ def create_kobo_tag(shelf):
         )
     return {"Tag": tag}
 
+# Creates a Kobo "Tag" object from a ub.MagicShelf object
+def create_kobo_tag_magic(shelf, books):
+    tag = {
+        "Created": convert_to_kobo_timestamp_string(shelf.created),
+        "Id": shelf.uuid,
+        "Items": [],
+        "LastModified": convert_to_kobo_timestamp_string(shelf.last_modified),
+        "Name": shelf.name,
+        "Type": "UserTag"
+    }
+    for book in books:
+        tag["Items"].append(
+            {
+                "RevisionId": book.uuid,
+                "Type": "ProductRevisionTagItem"
+            }
+        )
+    return {"Tag": tag}
+
 
 @csrf.exempt
 @kobo.route("/v1/library/<book_uuid>/state", methods=["GET", "PUT"])

cps/magic_shelf.py

@@ -139,7 +139,7 @@ FIELD_MAP = {
     'timestamp': (db.Books, 'timestamp'),
     'has_cover': (db.Books, 'has_cover'),
     'series_index': (db.Books, 'series_index'),
-    'comments': (db.Books, 'comments'),
+    'comments': (db.Comments, 'text'),  # Fixed: Points to actual text column, not relationship
     'read_status': ('custom_column', 'read_status'),  # Special handling - uses config.config_read_column
     'hardcover_id': ('identifier', 'hardcover-id'),  # Special handling - checks Identifiers table
 }
@@ -160,13 +160,13 @@ OPERATOR_MAP = {
     # 'greater_than_equal_to': lambda col, val: col >= val,  # Not used by QueryBuilder
     'between': lambda col, val: col.between(*val) if isinstance(val, (list, tuple)) and len(val) == 2 else None,
     'not_between': lambda col, val: ~col.between(*val) if isinstance(val, (list, tuple)) and len(val) == 2 else None,
-    'contains': lambda col, val: col.ilike(f'%{val}%'),
-    'not_contains': lambda col, val: ~col.ilike(f'%{val}%'),
-    'begins_with': lambda col, val: col.ilike(f'{val}%'),
-    'not_begins_with': lambda col, val: ~col.ilike(f'{val}%'),
-    'starts_with': lambda col, val: col.ilike(f'{val}%'),  # QueryBuilder emits 'begins_with', but keep for legacy
-    'ends_with': lambda col, val: col.ilike(f'%{val}'),
-    'not_ends_with': lambda col, val: ~col.ilike(f'%{val}'),
+    'contains': lambda col, val: col.ilike(f'%{val}%') if val is not None else None,
+    'not_contains': lambda col, val: ~col.ilike(f'%{val}%') if val is not None else None,
+    'begins_with': lambda col, val: col.ilike(f'{val}%') if val is not None else None,
+    'not_begins_with': lambda col, val: ~col.ilike(f'{val}%') if val is not None else None,
+    'starts_with': lambda col, val: col.ilike(f'{val}%') if val is not None else None,  # QueryBuilder emits 'begins_with', but keep for legacy
+    'ends_with': lambda col, val: col.ilike(f'%{val}') if val is not None else None,
+    'not_ends_with': lambda col, val: ~col.ilike(f'%{val}') if val is not None else None,
     'is_empty': lambda col, val: col == None,
     'is_not_empty': lambda col, val: col != None,
     'is_null': lambda col, val: col == None,
@@ -182,6 +182,7 @@ RELATIONSHIP_MAP = {
     'publisher': 'publishers',
     'rating': 'ratings',
     'language': 'languages',
+    'comments': 'comments',  # For description field - requires join to Comments table
 }
 
 def build_filter_from_rule(rule, user_id=None):
@@ -306,10 +307,25 @@ def build_filter_from_rule(rule, user_id=None):
 
     # Handle relationships using .any()
     relationship_name = RELATIONSHIP_MAP.get(field_name)
-    if relationship_name:
-        return getattr(db.Books, relationship_name).any(operator(column, value))
-    else:
-        return operator(column, value)
+    try:
+        if relationship_name:
+            # Special handling for is_empty/is_null on relationships:
+            # These check for absence of relationships, not null values in related records
+            if operator_name in ['is_empty', 'is_null']:
+                return ~getattr(db.Books, relationship_name).any()
+            elif operator_name in ['is_not_empty', 'is_not_null']:
+                return getattr(db.Books, relationship_name).any()
+            else:
+                filter_expr = operator(column, value)
+                if filter_expr is None:
+                    return None
+                return getattr(db.Books, relationship_name).any(filter_expr)
+        else:
+            filter_expr = operator(column, value)
+            return filter_expr
+    except Exception as e:
+        log.error(f"Error building filter for field '{field_name}', operator '{operator_name}', value '{value}': {str(e)}", exc_info=True)
+        return None
 
 
 def build_query_from_rules(rules_json, user_id=None):

cps/schedule.py

@@ -62,7 +62,8 @@ def register_scheduled_tasks(reconnect=True):
         # Register scheduled tasks
         timezone_info = datetime.datetime.now(datetime.timezone.utc).astimezone().tzinfo
         scheduler.schedule_tasks(tasks=get_scheduled_tasks(reconnect), trigger=CronTrigger(hour=start,
-                                                                                           timezone=timezone_info))
+                                                   timezone=timezone_info))
+        _schedule_duplicate_scan(scheduler, timezone_info)
         end_time = calclulate_end_time(start, duration)
         scheduler.schedule(func=end_scheduled_tasks, trigger=CronTrigger(hour=end_time.hour, minute=end_time.minute,
                                                                          timezone=timezone_info),
@@ -311,3 +312,33 @@ def should_task_be_running(start, duration):
 def calclulate_end_time(start, duration):
     start_time = datetime.datetime.now().replace(hour=start, minute=0)
     return start_time + datetime.timedelta(hours=duration // 60, minutes=duration % 60)
+
+
+def _schedule_duplicate_scan(scheduler, timezone_info):
+    """Schedule background duplicate scan based on CWA settings."""
+    try:
+        import sys as _sys
+        if '/app/calibre-web-automated/scripts/' not in _sys.path:
+            _sys.path.insert(1, '/app/calibre-web-automated/scripts/')
+        from cwa_db import CWA_DB
+        from .tasks.duplicate_scan import TaskDuplicateScan
+        from apscheduler.triggers.cron import CronTrigger
+
+        db = CWA_DB()
+        enabled = bool(db.cwa_settings.get('duplicate_scan_enabled', 0))
+        cron_expr = (db.cwa_settings.get('duplicate_scan_cron') or '').strip()
+
+        if not enabled:
+            return
+
+        if cron_expr:
+            trigger = CronTrigger.from_crontab(cron_expr, timezone=timezone_info)
+        else:
+            # manual/after_import handled elsewhere
+            return
+
+        scheduler.schedule_task(lambda: TaskDuplicateScan(full_scan=True, trigger_type='scheduled'),
+                                user='System', trigger=trigger, name='duplicate scan', hidden=False)
+    except Exception:
+        # Scheduling is best-effort; never block startup
+        pass

cps/services/background_scheduler.py

@@ -6,6 +6,7 @@
 # See CONTRIBUTORS for full list of authors.
 
 import atexit
+import threading
 
 from .. import logger
 from .worker import WorkerThread
@@ -35,6 +36,7 @@ class BackgroundScheduler:
             logger.logging.getLogger('tzlocal').setLevel(logger.logging.WARNING)
             cls.scheduler = BScheduler()
             cls.scheduler.start()
+            cls._schedule_lock = threading.Lock()  # Prevent concurrent task scheduling
 
         return cls._instance
 
@@ -75,8 +77,10 @@ class BackgroundScheduler:
     # Expects a list of lambda expressions for the tasks
     def schedule_tasks_immediately(self, tasks, user=None):
         if use_APScheduler:
-            for task in tasks:
-                self.schedule_task_immediately(task[0], user, name="immediately " + task[1], hidden=task[2])
+            # Use lock to prevent "Set changed size during iteration" when tasks are scheduled simultaneously
+            with self._schedule_lock:
+                for task in tasks:
+                    self.schedule_task_immediately(task[0], user, name="immediately " + task[1], hidden=task[2])
 
     # Remove all jobs
     def remove_all_jobs(self):

cps/static/css/caliBlur.css

@@ -2313,7 +2313,8 @@ body.advsearch > div.container-fluid > div > div.col-sm-10 > div.col-md-10.col-l
 }
 
 body.me > div.container-fluid > div > div.col-sm-10 > div.discover:before {
-    content: "My Profile"
+    content: "My Profile";
+    padding-left: 0px;
 }
 
 .well {

cps/static/css/duplicates-notifications.css

@@ -0,0 +1,356 @@
+/* Calibre-Web Automated – Modern Duplicates Notification System
+ * Copyright (C) 2024-2025 Calibre-Web Automated contributors
+ * SPDX-License-Identifier: GPL-3.0-or-later
+ */
+
+/* Backdrop Overlay */
+.duplicate-notification-backdrop {
+    position: fixed;
+    top: 0;
+    left: 0;
+    width: 100%;
+    height: 100%;
+    background: rgba(0, 0, 0, 0.6);
+    backdrop-filter: blur(8px);
+    -webkit-backdrop-filter: blur(8px);
+    z-index: 9998;
+    opacity: 0;
+    visibility: hidden;
+    transition: opacity 0.3s ease, visibility 0.3s ease;
+}
+
+.duplicate-notification-backdrop.active {
+    opacity: 1;
+    visibility: visible;
+}
+
+/* Modal Container */
+.duplicate-notification-modal {
+    position: fixed;
+    top: 50%;
+    left: 50%;
+    transform: translate(-50%, -50%) scale(0.9);
+    width: 90%;
+    max-width: 550px;
+    background: #1c28328a;
+    border-radius: 16px;
+    box-shadow: 
+        0 20px 60px rgba(0, 0, 0, 0.3),
+        0 0 0 1px rgba(255, 255, 255, 0.1);
+    z-index: 9999;
+    opacity: 0;
+    visibility: hidden;
+    transition: all 0.3s cubic-bezier(0.34, 1.56, 0.64, 1);
+    overflow: hidden;
+}
+
+.duplicate-notification-modal.active {
+    opacity: 1;
+    visibility: visible;
+    transform: translate(-50%, -50%) scale(1);
+}
+
+/* Dark theme adjustments */
+body.caliblur-theme .duplicate-notification-modal {
+    background: rgba(30, 30, 35, 0.98);
+    box-shadow: 
+        0 20px 60px rgba(0, 0, 0, 0.6),
+        0 0 0 1px rgba(255, 255, 255, 0.05);
+}
+
+/* Modal Header */
+.duplicate-notification-header {
+    background: linear-gradient(135deg, #ff9800 0%, #ff6f00 100%);
+    color: white;
+    padding: 28px 32px;
+    position: relative;
+}
+
+.duplicate-notification-header h2 {
+    margin: 0 0 8px 0;
+    font-size: 24px;
+    font-weight: 600;
+    line-height: 1.3;
+}
+
+.duplicate-notification-header p {
+    margin: 0;
+    opacity: 0.95;
+    font-size: 14px;
+    line-height: 1.5;
+}
+
+/* Close Button */
+.duplicate-notification-close {
+    position: absolute;
+    top: 20px;
+    right: 20px;
+    background: rgba(255, 255, 255, 0.2);
+    border: none;
+    color: white;
+    width: 32px;
+    height: 32px;
+    border-radius: 50%;
+    cursor: pointer;
+    font-size: 20px;
+    line-height: 1;
+    transition: all 0.2s ease;
+    display: flex;
+    align-items: center;
+    justify-content: center;
+}
+
+.duplicate-notification-close:hover {
+    background: rgba(255, 255, 255, 0.3);
+    transform: rotate(90deg);
+}
+
+/* Modal Body */
+.duplicate-notification-body {
+    padding: 32px;
+}
+
+.duplicate-count-section {
+    display: flex;
+    align-items: center;
+    justify-content: center;
+    gap: 24px;
+    margin-bottom: 24px;
+}
+
+.duplicate-count-badge {
+    display: flex;
+    align-items: center;
+    justify-content: center;
+    background: linear-gradient(135deg, #ff9800 0%, #ff6f00 100%);
+    color: white;
+    font-size: 32px;
+    font-weight: 700;
+    width: 80px;
+    height: 80px;
+    border-radius: 50%;
+    flex-shrink: 0;
+    box-shadow: 0 8px 24px rgba(255, 152, 0, 0.4);
+}
+
+.duplicate-notification-body h3 {
+    margin: 0;
+    font-size: 18px;
+    font-weight: 600;
+    color: white;
+    text-align: left;
+}
+
+body.caliblur-theme .duplicate-notification-body h3 {
+    color: #e2e8f0;
+}
+
+.duplicate-preview-list {
+    list-style: none;
+    padding: 0;
+    margin: 0 0 24px 0;
+}
+
+.duplicate-preview-item {
+    padding: 12px 16px;
+    margin-bottom: 8px;
+    background: #34455185;
+    border-radius: 8px;
+    border-left: 3px solid #ff9800;
+    transition: all 0.2s ease;
+}
+
+body.caliblur-theme .duplicate-preview-item {
+    background: rgba(255, 255, 255, 0.05);
+}
+
+.duplicate-preview-item:hover {
+    background: #edf2f7;
+    transform: translateX(4px);
+}
+
+body.caliblur-theme .duplicate-preview-item:hover {
+    background: rgba(255, 255, 255, 0.08);
+}
+
+.duplicate-preview-item strong {
+    display: block;
+    color: white;
+    font-size: 15px;
+    margin-bottom: 4px;
+}
+
+body.caliblur-theme .duplicate-preview-item strong {
+    color: #e2e8f0;
+}
+
+.duplicate-preview-item small {
+    color: #718096;
+    font-size: 13px;
+}
+
+body.caliblur-theme .duplicate-preview-item small {
+    color: #a0aec0;
+}
+
+/* Modal Footer */
+.duplicate-notification-footer {
+    padding: 0 32px 32px 32px;
+    display: flex;
+    gap: 12px;
+    justify-content: stretch;
+}
+
+.duplicate-notification-btn {
+    flex: 1;
+    padding: 14px 24px;
+    border: none;
+    border-radius: 8px;
+    font-size: 15px;
+    font-weight: 600;
+    cursor: pointer;
+    transition: all 0.2s ease;
+    text-align: center;
+    text-decoration: none;
+    display: inline-block;
+}
+
+.duplicate-notification-btn-primary {
+    background: linear-gradient(135deg, #ff9800 0%, #ff6f00 100%);
+    color: white;
+    box-shadow: 0 4px 12px rgba(255, 152, 0, 0.3);
+}
+
+.duplicate-notification-btn-primary:hover {
+    box-shadow: 0 6px 20px rgba(255, 152, 0, 0.4);
+    transform: translateY(-2px);
+    color: white;
+}
+
+.duplicate-notification-btn-secondary {
+    background: #34455185;
+    color: white;
+}
+
+body.caliblur-theme .duplicate-notification-btn-secondary {
+    background: rgba(255, 255, 255, 0.1);
+    color: #cbd5e0;
+}
+
+.duplicate-notification-btn-secondary:hover {
+    background: #cbd5e0;
+    color: #2d3748;
+}
+
+body.caliblur-theme .duplicate-notification-btn-secondary:hover {
+    background: rgba(255, 255, 255, 0.15);
+    color: #e2e8f0;
+}
+
+/* Sidebar Badge */
+.duplicate-badge {
+    position: absolute;
+    top: 8px;
+    right: 8px;
+    background: linear-gradient(135deg, #ff9800 0%, #ff6f00 100%);
+    color: white;
+    font-size: 11px;
+    font-weight: 700;
+    padding: 4px 7px;
+    border-radius: 12px;
+    line-height: 1;
+    box-shadow: 2px 2px 2px rgb(0 0 0 / 18%);
+    animation: pulse 2s ease-in-out infinite;
+}
+
+@keyframes pulse {
+    0%, 100% {
+        opacity: 1;
+        transform: scale(1);
+    }
+    50% {
+        opacity: 0.9;
+        transform: scale(1.05);
+    }
+}
+
+/* Responsive Design */
+@media (max-width: 600px) {
+    .duplicate-notification-modal {
+        width: 95%;
+        max-width: none;
+    }
+    
+    .duplicate-notification-header {
+        padding: 24px 20px;
+    }
+    
+    .duplicate-notification-body {
+        padding: 24px 20px;
+    }
+    
+    .duplicate-notification-footer {
+        padding: 0 20px 24px 20px;
+        flex-direction: column;
+    }
+    
+    .duplicate-notification-header h2 {
+        font-size: 20px;
+    }
+    
+    .duplicate-count-badge {
+        width: 64px;
+        height: 64px;
+        font-size: 26px;
+    }
+}
+
+/* Animations */
+@keyframes slideIn {
+    from {
+        opacity: 0;
+        transform: translate(-50%, -48%) scale(0.95);
+    }
+    to {
+        opacity: 1;
+        transform: translate(-50%, -50%) scale(1);
+    }
+}
+
+@keyframes fadeIn {
+    from {
+        opacity: 0;
+    }
+    to {
+        opacity: 1;
+    }
+}
+
+/* Accessibility */
+.duplicate-notification-modal:focus {
+    outline: 2px solid #ff9800;
+    outline-offset: 2px;
+}
+
+.duplicate-notification-btn:focus {
+    outline: 2px solid #ff9800;
+    outline-offset: 2px;
+}
+
+/* Loading State */
+.duplicate-notification-loading {
+    text-align: center;
+    padding: 40px;
+    color: #718096;
+}
+
+.duplicate-notification-loading::after {
+    content: "...";
+    animation: ellipsis 1.5s infinite;
+}
+
+@keyframes ellipsis {
+    0%, 20% { content: "."; }
+    40% { content: ".."; }
+    60%, 100% { content: "..."; }
+}

cps/static/css/query_builder.css

@@ -191,7 +191,7 @@ a.btn.btn-secondary:hover {
     top: 0;
     bottom: 0;
     width: 3px;
-    background: linear-gradient(to bottom, #cc7b19, transparent);
+    background: #cc7b19;
 }
 
 /* Better Focus States */

cps/static/js/caliBlur.js

@@ -1060,6 +1060,9 @@ $(function() {
         $.ajax({
             url: window.scriptRoot + '/ajax/toggleread/' + bookId,
             type: 'POST',
+            data: {
+                csrf_token: $("input[name='csrf_token']").val()
+            },
             headers: {
                 'X-Requested-With': 'XMLHttpRequest'
             },

cps/static/js/duplicate-notifier.js

@@ -0,0 +1,206 @@
+/* Calibre-Web Automated – Modern Duplicates Notification System
+ * Copyright (C) 2024-2025 Calibre-Web Automated contributors
+ * SPDX-License-Identifier: GPL-3.0-or-later
+ */
+
+(function() {
+    'use strict';
+    
+    const STORAGE_KEY = 'cwa_duplicates_notification_shown';
+    // Removed automatic polling - badge updates only on cache invalidation events
+    
+    let currentDuplicateCount = 0;
+    
+    /**
+     * Check if notification was already shown in this session
+     */
+    function wasNotificationShown() {
+        return sessionStorage.getItem(STORAGE_KEY) === 'true';
+    }
+    
+    /**
+     * Mark notification as shown for this session
+     */
+    function markNotificationShown() {
+        sessionStorage.setItem(STORAGE_KEY, 'true');
+    }
+    
+    /**
+     * Update the duplicate count badge in sidebar
+     */
+    function updateBadge(count) {
+        currentDuplicateCount = count;
+        const badge = document.getElementById('duplicate-count-badge');
+        
+        if (badge) {
+            if (count > 0) {
+                badge.textContent = count > 99 ? '99+' : count;
+                badge.style.display = 'inline-block';
+            } else {
+                badge.style.display = 'none';
+            }
+        }
+    }
+    
+    /**
+     * Fetch duplicate status from API
+     */
+    function fetchDuplicateStatus() {
+        const basePath = (typeof getPath === 'function') ? getPath() : '';
+        const statusUrl = basePath + '/duplicates/status';
+        return fetch(statusUrl, {
+            method: 'GET',
+            headers: {
+                'Content-Type': 'application/json'
+            },
+            credentials: 'same-origin'
+        })
+        .then(response => response.json())
+        .catch(error => {
+            console.error('[CWA Duplicates] Error fetching status:', error);
+            return { success: false, count: 0, preview: [], enabled: false };
+        });
+    }
+    
+    /**
+     * Show the notification modal
+     */
+    function showNotificationModal(data) {
+        const { count, preview } = data;
+        
+        // Don't show if already shown this session
+        if (wasNotificationShown()) {
+            return;
+        }
+        
+        // Update count in modal
+        const countBadge = document.getElementById('duplicate-notification-count');
+        if (countBadge) {
+            countBadge.textContent = count;
+        }
+        
+        // Update preview list
+        const previewList = document.getElementById('duplicate-notification-preview');
+        if (previewList && preview && preview.length > 0) {
+            previewList.innerHTML = preview.map(item => `
+                <li class="duplicate-preview-item">
+                    <strong>${escapeHtml(item.title)}</strong>
+                    <small>${escapeHtml(item.author)} - ${item.count} copies</small>
+                </li>
+            `).join('');
+        }
+        
+        // Show modal and backdrop
+        const modal = document.getElementById('duplicate-notification-modal');
+        const backdrop = document.getElementById('duplicate-notification-backdrop');
+        
+        if (modal && backdrop) {
+            // Small delay for smooth animation
+            setTimeout(() => {
+                backdrop.classList.add('active');
+                modal.classList.add('active');
+                
+                // Focus trap
+                modal.focus();
+                
+                // Mark as shown
+                markNotificationShown();
+            }, 500);
+        }
+    }
+    
+    /**
+     * Hide the notification modal
+     */
+    function hideNotificationModal() {
+        const modal = document.getElementById('duplicate-notification-modal');
+        const backdrop = document.getElementById('duplicate-notification-backdrop');
+        
+        if (modal && backdrop) {
+            modal.classList.remove('active');
+            backdrop.classList.remove('active');
+        }
+    }
+    
+    /**
+     * Escape HTML to prevent XSS
+     */
+    function escapeHtml(text) {
+        const div = document.createElement('div');
+        div.textContent = text;
+        return div.innerHTML;
+    }
+    
+    /**
+     * Initialize event listeners
+     */
+    function initializeEventListeners() {
+        // Close button
+        const closeBtn = document.getElementById('duplicate-notification-close');
+        if (closeBtn) {
+            closeBtn.addEventListener('click', hideNotificationModal);
+        }
+        
+        // Remind me later button
+        const remindBtn = document.getElementById('duplicate-notification-remind');
+        if (remindBtn) {
+            remindBtn.addEventListener('click', hideNotificationModal);
+        }
+        
+        // Click outside to close
+        const backdrop = document.getElementById('duplicate-notification-backdrop');
+        if (backdrop) {
+            backdrop.addEventListener('click', hideNotificationModal);
+        }
+        
+        // Escape key to close
+        document.addEventListener('keydown', function(e) {
+            if (e.key === 'Escape') {
+                hideNotificationModal();
+            }
+        });
+    }
+    
+    /**
+     * Main initialization function
+     */
+    function init() {
+        // Check if user has permission (admin or edit)
+        const userHasPermission = document.getElementById('duplicate-notification-modal');
+        if (!userHasPermission) {
+            return; // Modal not rendered, user doesn't have permission
+        }
+        
+        // Initialize event listeners
+        initializeEventListeners();
+        
+        // Fetch initial status once on page load
+        // No periodic updates - badge refreshes after ingest operations only
+        fetchDuplicateStatus().then(data => {
+            if (data.success) {
+                // Update badge
+                updateBadge(data.count);
+                
+                // Show notification modal if there are duplicates and notifications are enabled
+                if (data.count > 0 && data.enabled) {
+                    showNotificationModal(data);
+                }
+            }
+        });
+    }
+    
+    // Expose functions globally for use by other scripts
+    window.CWADuplicates = {
+        updateBadge: updateBadge,
+        fetchStatus: fetchDuplicateStatus,
+        hideModal: hideNotificationModal
+    };
+    
+    // Initialize when DOM is ready
+    if (document.readyState === 'loading') {
+        document.addEventListener('DOMContentLoaded', init);
+    } else {
+        init();
+    }
+    
+})();

cps/static/js/duplicates.js

@@ -26,11 +26,17 @@ $(document).ready(function() {
     function updateSelectionCount() {
         var count = selectedBooks.length;
         if (count === 0) {
-            $('#selection_count').text('');
+            $('#selection_count').text('0 BOOKS SELECTED');
             $('#delete_selected').addClass('disabled').attr('aria-disabled', true);
-        } else {
-            $('#selection_count').text(count + ' book' + (count > 1 ? 's' : '') + ' selected');
+            $('.merge-selected-btn').addClass('disabled').attr('aria-disabled', true);
+        } else if (count === 1) {
+            $('#selection_count').text('1 BOOK SELECTED');
             $('#delete_selected').removeClass('disabled').attr('aria-disabled', false);
+            $('.merge-selected-btn').addClass('disabled').attr('aria-disabled', true);
+        } else {
+            $('#selection_count').text(count + ' BOOKS SELECTED');
+            $('#delete_selected').removeClass('disabled').attr('aria-disabled', false);
+            $('.merge-selected-btn').removeClass('disabled').attr('aria-disabled', false);
         }
     }
     
@@ -95,6 +101,63 @@ $(document).ready(function() {
         updateBookItemVisuals();
     });
     
+    // Merge Selected button (delegated for multiple buttons)
+    $(document).on('click', '.merge-selected-btn', function(event) {
+        if ($(this).hasClass('disabled')) {
+            event.stopPropagation();
+        } else {
+            // Check if at least 2 books are selected
+            if (selectedBooks.length < 2) {
+                $('#error_modal_message').text('Please select at least 2 books to merge.');
+                $('#error_modal').modal('show');
+                return;
+            }
+
+            // Use relative URL like table.js to respect base paths
+            var relativeUrl = window.location.pathname + "/../ajax/displayselectedbooks";
+
+            $('#merge_selected_modal').modal('show');
+
+            // Convert book IDs to integers (same as table.js)
+            var bookIds = selectedBooks.map(function(id) { return parseInt(id, 10); });
+
+            // Show list of books to be merged (no CSRF - match table.js exactly)
+            var ajaxData = {"selections": bookIds};
+
+            $.ajax({
+                method: 'post',
+                contentType: 'application/json; charset=utf-8',
+                dataType: 'json',
+                url: relativeUrl,
+                data: JSON.stringify(ajaxData),
+                beforeSend: function(xhr) {
+                    // Add CSRF token as header (like table.js)
+                    if (csrfToken) {
+                        xhr.setRequestHeader('X-CSRFToken', csrfToken);
+                    }
+                },
+                success: function(response) {
+                    $('#display-merge-target-book').empty();
+                    $('#display-merge-source-books').empty();
+
+                    // First book is the target (kept)
+                    if (response.books && response.books.length > 0) {
+                        $("<span>✓ " + response.books[0] + "</span>").appendTo('#display-merge-target-book');
+
+                        // Rest are source books (merged and deleted)
+                        for (var i = 1; i < response.books.length; i++) {
+                            $("<span>- " + response.books[i] + "</span><p></p>").appendTo('#display-merge-source-books');
+                        }
+                    }
+                },
+                error: function(xhr, status, error) {
+                    $('#error_modal_message').text('Error loading book list for merge confirmation. Status: ' + xhr.status + '. Check browser console for details.');
+                    $('#error_modal').modal('show');
+                }
+            });
+        }
+    });
+
     // Delete Selected button
     $('#delete_selected').click(function(event) {
         if ($(this).hasClass('disabled')) {
@@ -107,8 +170,8 @@ $(document).ready(function() {
                 return;
             }
             
-            // Use absolute URL like working table.js
-            var relativeUrl = "/ajax/displayselectedbooks";
+            // Use relative URL like table.js to respect base paths
+            var relativeUrl = window.location.pathname + "/../ajax/displayselectedbooks";
             
             $('#delete_selected_modal').modal('show');
             
@@ -130,23 +193,73 @@ $(document).ready(function() {
                         xhr.setRequestHeader('X-CSRFToken', csrfToken);
                     }
                 },
-                            success: function(response) {
-                $('#display-delete-selected-books').empty();
-                $.each(response.books, function(i, item) {
-                    $("<span>- " + item + "</span><p></p>").appendTo("#display-delete-selected-books");
-                });
-            },
-            error: function(xhr, status, error) {
-                $('#error_modal_message').text("Error loading book list for confirmation. Status: " + xhr.status + ". Check browser console for details.");
-                $('#error_modal').modal('show');
-            }
+                success: function(response) {
+                    $('#display-delete-selected-books').empty();
+                    $.each(response.books, function(i, item) {
+                        $("<span>- " + item + "</span><p></p>").appendTo("#display-delete-selected-books");
+                    });
+                },
+                error: function(xhr, status, error) {
+                    $('#error_modal_message').text("Error loading book list for confirmation. Status: " + xhr.status + ". Check browser console for details.");
+                    $('#error_modal').modal('show');
+                }
             });
         }
     });
+
+    // Confirm merge
+    $('#merge_selected_confirm').click(function() {
+        var mergeUrl = window.location.pathname + "/../ajax/mergebooks";
+
+        // Convert book IDs to integers (same as table.js)
+        var bookIds = selectedBooks.map(function(id) { return parseInt(id, 10); });
+
+        // First book in array is target, rest are merged into it
+        var mergeData = {"Merge_books": bookIds};
+
+        $.ajax({
+            method: 'post',
+            contentType: 'application/json; charset=utf-8',
+            dataType: 'json',
+            url: mergeUrl,
+            data: JSON.stringify(mergeData),
+            beforeSend: function(xhr) {
+                // Add CSRF token as header (like table.js)
+                if (csrfToken) {
+                    xhr.setRequestHeader('X-CSRFToken', csrfToken);
+                }
+            },
+            success: function(response) {
+                if (response.success) {
+                    // Close the merge confirmation modal
+                    $('#merge_selected_modal').modal('hide');
+
+                    // Show success modal
+                    $('#success_modal_message').text('Selected books have been merged successfully!');
+                    $('#success_modal').modal('show');
+                } else {
+                    // Close the merge confirmation modal
+                    $('#merge_selected_modal').modal('hide');
+
+                    // Show error modal
+                    $('#error_modal_message').text('Error: ' + (response.error || 'Unknown error occurred during merge'));
+                    $('#error_modal').modal('show');
+                }
+            },
+            error: function(xhr, status, error) {
+                // Close the merge confirmation modal
+                $('#merge_selected_modal').modal('hide');
+
+                // Show error modal
+                $('#error_modal_message').text('An error occurred while merging books. Check browser console for details.');
+                $('#error_modal').modal('show');
+            }
+        });
+    });
     
     // Confirm delete
     $('#delete_selected_confirm').click(function() {
-        var deleteUrl = "/ajax/deleteselectedbooks";
+        var deleteUrl = window.location.pathname + "/../ajax/deleteselectedbooks";
         
         // Convert book IDs to integers (same as table.js)
         var bookIds = selectedBooks.map(function(id) { return parseInt(id, 10); });
@@ -199,6 +312,340 @@ $(document).ready(function() {
         window.location.reload();
     });
     
+    // Dismiss/Undismiss duplicate group handlers
+    $(document).on('click', '.dismiss-duplicate-btn', function(e) {
+        e.preventDefault();
+        var btn = $(this);
+        var groupHash = btn.data('group-hash');
+        var isDismissed = btn.data('dismissed');
+        var groupContainer = btn.closest('.duplicate-group');
+        
+        // Determine action
+        var action = isDismissed ? 'undismiss' : 'dismiss';
+        var endpoint = '/duplicates/' + action + '/' + groupHash;
+        
+        // Disable button during request
+        btn.prop('disabled', true);
+        
+        // Make AJAX request
+        $.ajax({
+            url: endpoint,
+            type: 'POST',
+            headers: {
+                'X-CSRFToken': csrfToken
+            },
+            dataType: 'json',
+            success: function(response) {
+                if (response.success) {
+                    // Update button state
+                    if (action === 'dismiss') {
+                        btn.data('dismissed', true);
+                        btn.html('<span class="glyphicon glyphicon-eye-open"></span> Show');
+                        btn.attr('title', 'Show this duplicate group');
+                        btn.css('background', 'rgba(46, 204, 113, 0.2)');
+                        btn.css('border-color', 'rgba(46, 204, 113, 0.4)');
+                        
+                        // Fade out the group
+                        groupContainer.fadeOut(300);
+                    } else {
+                        btn.data('dismissed', false);
+                        btn.html('<span class="glyphicon glyphicon-eye-close"></span> Dismiss');
+                        btn.attr('title', 'Dismiss this duplicate group');
+                        btn.css('background', 'rgba(255,255,255,0.15)');
+                        btn.css('border-color', 'rgba(255,255,255,0.3)');
+                    }
+                    
+                    // Update badge count in real-time
+                    if (window.CWADuplicates && window.CWADuplicates.updateBadge) {
+                        window.CWADuplicates.updateBadge(response.count);
+                    }
+                    
+                    // Show success message (optional)
+                    console.log('[CWA Duplicates] ' + response.message);
+                } else {
+                    // Show error
+                    alert('Error: ' + response.error);
+                }
+                
+                // Re-enable button
+                btn.prop('disabled', false);
+            },
+            error: function(xhr, status, error) {
+                console.error('[CWA Duplicates] Error ' + action + 'ing duplicate group:', error);
+                alert('Error: Failed to update duplicate group');
+                
+                // Re-enable button
+                btn.prop('disabled', false);
+            }
+        });
+    });
+    
+    let activeScanTaskId = null;
+
+    function updateProgressUI(progress, message) {
+        const pct = Math.round((progress || 0) * 100);
+        $('#scan_progress_container').show();
+        $('#scan_progress_bar').css('width', pct + '%');
+        $('#scan_progress_label').text(pct + '%');
+        $('#scan_progress_message').text(message || '');
+        $('#cancel_scan').show();
+    }
+
+    function resetProgressUI() {
+        $('#scan_progress_container').hide();
+        $('#scan_progress_bar').css('width', '0%');
+        $('#scan_progress_label').text('0%');
+        $('#scan_progress_message').text('');
+        $('#cancel_scan').hide();
+        activeScanTaskId = null;
+    }
+
+    function pollScanProgress(taskId, btn) {
+        activeScanTaskId = taskId;
+        const pollInterval = 2000;
+        function pollOnce() {
+            $.ajax({
+                url: '/duplicates/scan-progress/' + taskId,
+                type: 'GET',
+                dataType: 'json',
+                success: function(response) {
+                    if (!response.success) {
+                        clearInterval(poller);
+                        btn.prop('disabled', false);
+                        btn.html('<span class="glyphicon glyphicon-refresh"></span> Scan for Duplicates Now');
+                        resetProgressUI();
+                        alert('Scan failed: ' + (response.error || 'Unknown error'));
+                        return;
+                    }
+
+                    updateProgressUI(response.progress, response.message);
+
+                    if (response.status === 'completed') {
+                        clearInterval(poller);
+                        btn.prop('disabled', false);
+                        btn.html('<span class="glyphicon glyphicon-refresh"></span> Scan for Duplicates Now');
+                        const resultCount = (response.result_count !== null && response.result_count !== undefined)
+                            ? response.result_count
+                            : null;
+                        const doneMessage = resultCount !== null
+                            ? 'Scan completed. Found ' + resultCount + ' duplicate groups.'
+                            : 'Scan completed.';
+                        updateProgressUI(1, doneMessage + ' Refreshing...');
+                        alert(doneMessage);
+                        setTimeout(function() {
+                            resetProgressUI();
+                            location.reload();
+                        }, 800);
+                    } else if (response.status === 'failed') {
+                        clearInterval(poller);
+                        btn.prop('disabled', false);
+                        btn.html('<span class="glyphicon glyphicon-refresh"></span> Scan for Duplicates Now');
+                        resetProgressUI();
+                        alert('Scan failed: ' + (response.message || 'Unknown error'));
+                    } else if (response.status === 'cancelled') {
+                        clearInterval(poller);
+                        btn.prop('disabled', false);
+                        btn.html('<span class="glyphicon glyphicon-refresh"></span> Scan for Duplicates Now');
+                        resetProgressUI();
+                        alert('Scan cancelled');
+                    }
+                },
+                error: function(xhr, status, error) {
+                    console.error('[CWA Duplicates] Error polling scan progress:', error);
+                }
+            });
+        }
+
+        const poller = setInterval(pollOnce, pollInterval);
+        pollOnce();
+    }
+
+    // Manual scan trigger
+    $('#trigger_scan').on('click', function() {
+        var btn = $(this);
+        btn.prop('disabled', true);
+        btn.html('<span class="glyphicon glyphicon-refresh glyphicon-spin"></span> Scanning...');
+        
+        $.ajax({
+            url: '/duplicates/trigger-scan',
+            type: 'POST',
+            headers: {
+                'X-CSRFToken': csrfToken
+            },
+            dataType: 'json',
+            success: function(response) {
+                if (response.success && response.task_id) {
+                    updateProgressUI(0, 'Queued...');
+                    pollScanProgress(response.task_id, btn);
+                } else if (response.success) {
+                    if (response.queued === false && response.fallback_reason) {
+                        console.warn('[CWA Duplicates] Background queue failed, fallback used:', response.fallback_reason);
+                    }
+                    const count = (response.count !== undefined && response.count !== null)
+                        ? response.count
+                        : null;
+                    const message = count !== null
+                        ? 'Scan completed. Found ' + count + ' duplicate groups.'
+                        : 'Scan completed.';
+                    alert(message);
+                    location.reload();
+                } else {
+                    alert('Scan failed: ' + response.error);
+                }
+            },
+            error: function(xhr, status, error) {
+                console.error('[CWA Duplicates] Error triggering scan:', error);
+                alert('Error: Failed to trigger duplicate scan');
+            },
+            complete: function() {
+                // Button state will be restored by poller
+            }
+        });
+    });
+
+    // Cancel scan
+    $('#cancel_scan').on('click', function() {
+        if (!activeScanTaskId) {
+            return;
+        }
+        $.ajax({
+            url: '/duplicates/cancel-scan/' + activeScanTaskId,
+            type: 'POST',
+            headers: {
+                'X-CSRFToken': csrfToken
+            },
+            dataType: 'json',
+            success: function(response) {
+                if (!response.success) {
+                    alert('Cancel failed: ' + response.error);
+                }
+            },
+            error: function(xhr, status, error) {
+                console.error('[CWA Duplicates] Error cancelling scan:', error);
+                alert('Error: Failed to cancel duplicate scan');
+            }
+        });
+    });
+    
+    // Auto-resolution preview
+    $('#preview_resolution').on('click', function() {
+        var strategy = $('#resolution_strategy').val();
+        var btn = $(this);
+        btn.prop('disabled', true);
+        btn.html('<span class="glyphicon glyphicon-refresh glyphicon-spin"></span> Loading...');
+        
+        $.ajax({
+            url: '/duplicates/preview-resolution',
+            method: 'POST',
+            contentType: 'application/json',
+            headers: {
+                'X-CSRFToken': csrfToken
+            },
+            data: JSON.stringify({ strategy: strategy }),
+            success: function(data) {
+                if (data.success && data.preview) {
+                    showResolutionPreview(data);
+                    $('#execute_resolution').removeClass('disabled').prop('aria-disabled', false);
+                } else {
+                    alert('Error generating preview: ' + (data.errors || []).join(', '));
+                }
+            },
+            error: function() {
+                alert('Failed to generate preview');
+            },
+            complete: function() {
+                btn.prop('disabled', false);
+                btn.html('<span class="glyphicon glyphicon-eye-open"></span> Preview');
+            }
+        });
+    });
+    
+    // Execute resolution
+    $('#execute_resolution').on('click', function() {
+        if ($(this).hasClass('disabled')) return;
+        
+        if (!confirm('Are you sure you want to execute auto-resolution? This will permanently delete duplicate books.')) {
+            return;
+        }
+        
+        var strategy = $('#resolution_strategy').val();
+        var btn = $(this);
+        btn.addClass('disabled').html('<span class="glyphicon glyphicon-refresh glyphicon-spin"></span> Executing...');
+        
+        $.ajax({
+            url: '/duplicates/execute-resolution',
+            method: 'POST',
+            contentType: 'application/json',
+            headers: {
+                'X-CSRFToken': csrfToken
+            },
+            data: JSON.stringify({ strategy: strategy }),
+            success: function(data) {
+                if (data.success) {
+                    alert('Resolution complete!\n\nResolved: ' + data.resolved_count + ' groups\nKept: ' + data.kept_count + ' books\nDeleted: ' + data.deleted_count + ' books');
+                    location.reload();
+                } else {
+                    alert('Errors occurred:\n' + (data.errors || []).join('\n'));
+                    btn.removeClass('disabled').html('<span class="glyphicon glyphicon-flash"></span> Execute Resolution');
+                }
+            },
+            error: function() {
+                alert('Failed to execute resolution');
+                btn.removeClass('disabled').html('<span class="glyphicon glyphicon-flash"></span> Execute Resolution');
+            }
+        });
+    });
+    
+    function showResolutionPreview(data) {
+        var html = '<div style="margin-bottom: 20px; padding: 15px; background: #6d6d6d66; border-left: 4px solid #4caf50; border-radius: 4px; color: white">' +
+            '<h4 style="color: #98f99c; margin-top: 0;">Summary</h4>' +
+            '<p><strong>Groups to resolve:</strong> ' + data.resolved_count + '</p>' +
+            '<p><strong>Books to keep:</strong> ' + data.kept_count + '</p>' +
+            '<p><strong>Books to delete:</strong> ' + data.deleted_count + '</p>' +
+            '</div>';
+        
+        if (data.preview && data.preview.length > 0) {
+            html += '<div style="margin-top: 20px;"><h4>Preview:</h4>';
+            
+            data.preview.forEach(function(group) {
+                html += '<div style="padding: 15px; border: none; border-radius: 6px; background: #6d6d6d66;">' +
+                    '<h5 style="color: white; margin-top: 0;">' + escapeHtml(group.title) + ' <small>by ' + escapeHtml(group.author) + '</small></h5>' +
+                    '<div style="margin-bottom: 15px; padding: 10px; background: #d4edda; border-left: 3px solid #28a745; border-radius: 4px; color: #1c2832">' +
+                    '<strong style="color: #155724;">✓ KEEP:</strong> Book ID ' + group.kept_book_id + ' ' +
+                    '<small style="color: #666;">(Added: ' + group.kept_book_timestamp + ', Formats: ' + group.kept_book_formats.join(', ') + ')</small>' +
+                    '</div>' +
+                    '<div style="padding: 10px; background: #f8d7da; border-left: 3px solid #dc3545; border-radius: 4px;">' +
+                    '<strong style="color: #721c24;">✗ DELETE:</strong>' +
+                    '<ul style="margin: 5px 0 0 20px; padding: 0; color: #1c2832">';
+                
+                group.deleted_books_info.forEach(function(book) {
+                    html += '<li style="margin: 5px 0;">' +
+                        'Book ID ' + book.id + ' ' +
+                        '<small style="color: #666;">(Added: ' + book.timestamp + ', Formats: ' + book.formats.join(', ') + ')</small>' +
+                        '</li>';
+                });
+                
+                html += '</ul></div></div>';
+            });
+            
+            html += '</div>';
+        }
+        
+        $('#resolution_preview_body').html(html);
+        $('#resolution_preview_modal').modal('show');
+    }
+    
+    function escapeHtml(text) {
+        var map = {
+            '&': '&amp;',
+            '<': '&lt;',
+            '>': '&gt;',
+            '"': '&quot;',
+            "'": '&#039;'
+        };
+        return text.replace(/[&<>"']/g, function(m) { return map[m]; });
+    }
+    
     // Initialize
     updateSelectionCount();
     updateBookItemVisuals();

cps/tasks/duplicate_scan.py

@@ -0,0 +1,197 @@
+# -*- coding: utf-8 -*-
+# Calibre-Web Automated – fork of Calibre-Web
+# Copyright (C) 2018-2025 Calibre-Web contributors
+# Copyright (C) 2024-2025 Calibre-Web Automated contributors
+# SPDX-License-Identifier: GPL-3.0-or-later
+# See CONTRIBUTORS for full list of authors.
+
+import sys
+from datetime import datetime
+from sqlalchemy import func
+from flask_babel import lazy_gettext as N_
+
+from cps import calibre_db, db, logger
+from cps.duplicates import find_duplicate_books_python, find_duplicate_candidate_ids_sql
+from cps.services.worker import CalibreTask, STAT_CANCELLED, STAT_ENDED
+from cps.ub import init_db_thread
+
+# Access CWA DB (scripts path)
+if '/app/calibre-web-automated/scripts/' not in sys.path:
+    sys.path.insert(1, '/app/calibre-web-automated/scripts/')
+from cwa_db import CWA_DB
+
+log = logger.create()
+
+
+class TaskDuplicateScan(CalibreTask):
+    def __init__(self, full_scan=True, task_message=None, trigger_type='manual', user_id=None):
+        super(TaskDuplicateScan, self).__init__(task_message or N_('Duplicate scan'))
+        self.full_scan = full_scan
+        self.trigger_type = trigger_type
+        self.result_count = 0
+        self.user_id = user_id
+
+    @property
+    def name(self):
+        return str(N_('Duplicate scan'))
+
+    @property
+    def is_cancellable(self):
+        return True
+
+    def run(self, worker_thread):
+        try:
+            init_db_thread()
+        except Exception:
+            # Non-fatal; continue
+            pass
+
+        try:
+            # Ensure calibre DB session is ready in this thread
+            calibre_db.ensure_session()
+
+            if self.stat in (STAT_CANCELLED, STAT_ENDED):
+                return
+
+            cwa_db = CWA_DB()
+            cache_data = cwa_db.get_duplicate_cache() or {}
+
+            if not self.full_scan:
+                last_scanned_book_id = int(cache_data.get('last_scanned_book_id') or 0)
+                if last_scanned_book_id == 0:
+                    # No baseline yet; fall back to full scan
+                    self.full_scan = True
+
+            self.progress = 0.1
+            self.message = N_('Scanning for duplicates')
+
+            # Import here to avoid circular dependency during module import
+            from cps.duplicates import find_duplicate_books
+
+            if self.stat in (STAT_CANCELLED, STAT_ENDED):
+                return
+
+            if self.full_scan:
+                duplicate_groups = find_duplicate_books(include_dismissed=False, user_id=self.user_id)
+                self.result_count = len(duplicate_groups)
+
+                if self.stat in (STAT_CANCELLED, STAT_ENDED):
+                    return
+
+                # Update cache with full results (including dismissed groups)
+                all_groups = find_duplicate_books(include_dismissed=True, user_id=self.user_id)
+
+                max_book_id = 0
+                try:
+                    max_id_result = calibre_db.session.query(func.max(db.Books.id)).scalar()
+                    max_book_id = max_id_result if max_id_result is not None else 0
+                except Exception as ex:
+                    log.warning("[cwa-duplicates] Could not get max book ID in TaskDuplicateScan: %s", str(ex))
+
+                cwa_db.update_duplicate_cache(all_groups, len(all_groups), max_book_id)
+                log.info("[cwa-duplicates] Duplicate cache updated (full scan): groups=%s max_book_id=%s",
+                         len(all_groups), max_book_id)
+            else:
+                # Incremental scan: only groups impacted by newly added books
+                settings = cwa_db.cwa_settings
+                use_title = settings.get('duplicate_detection_title', 1)
+                use_author = settings.get('duplicate_detection_author', 1)
+                use_language = settings.get('duplicate_detection_language', 1)
+                use_series = settings.get('duplicate_detection_series', 0)
+                use_publisher = settings.get('duplicate_detection_publisher', 0)
+                use_format = settings.get('duplicate_detection_format', 0)
+
+                last_scanned_book_id = int(cache_data.get('last_scanned_book_id') or 0)
+                candidate_ids = find_duplicate_candidate_ids_sql(use_title, use_author, user_id=self.user_id,
+                                                                min_book_id=last_scanned_book_id)
+
+                max_book_id = 0
+                try:
+                    max_id_result = calibre_db.session.query(func.max(db.Books.id)).scalar()
+                    max_book_id = max_id_result if max_id_result is not None else 0
+                except Exception as ex:
+                    log.warning("[cwa-duplicates] Could not get max book ID in TaskDuplicateScan: %s", str(ex))
+
+                if not candidate_ids:
+                    # No impacted groups; just bump last_scanned_book_id
+                    try:
+                        cwa_db.cur.execute("""
+                            UPDATE cwa_duplicate_cache
+                            SET last_scanned_book_id = ?, scan_pending = 0, scan_timestamp = ?
+                            WHERE id = 1
+                        """, (max_book_id, datetime.now().isoformat()))
+                        cwa_db.con.commit()
+                        log.info("[cwa-duplicates] Incremental scan: no candidates; cache timestamp updated (last_scanned_book_id=%s)",
+                                 max_book_id)
+                    except Exception:
+                        pass
+                    self.result_count = 0
+                else:
+                    # Identify affected groups in cache
+                    cached_groups = cache_data.get('duplicate_groups', [])
+                    candidate_set = set(candidate_ids)
+                    affected_hashes = {
+                        group.get('group_hash')
+                        for group in cached_groups
+                        if group.get('book_ids') and candidate_set.intersection(set(group.get('book_ids')))
+                    }
+
+                    # Recompute groups for candidate IDs (include dismissed for cache)
+                    recomputed_groups = find_duplicate_books_python(
+                        use_title, use_author, use_language, use_series, use_publisher, use_format,
+                        include_dismissed=True, user_id=self.user_id, candidate_ids=candidate_ids
+                    )
+
+                    # Serialize recomputed groups
+                    serialized_groups = []
+                    for group in recomputed_groups:
+                        serialized_groups.append({
+                            'title': group.get('title', ''),
+                            'author': group.get('author', ''),
+                            'count': group.get('count', 0),
+                            'group_hash': group.get('group_hash', ''),
+                            'book_ids': [book.id for book in group.get('books', [])]
+                        })
+
+                    # Merge cache: remove affected, then append recomputed
+                    kept_groups = [g for g in cached_groups if g.get('group_hash') not in affected_hashes]
+                    merged_groups = kept_groups + serialized_groups
+
+                    try:
+                        import json
+                        cwa_db.cur.execute("""
+                            UPDATE cwa_duplicate_cache
+                            SET scan_timestamp = ?,
+                                duplicate_groups_json = ?,
+                                total_count = ?,
+                                scan_pending = 0,
+                                last_scanned_book_id = ?
+                            WHERE id = 1
+                        """, (datetime.now().isoformat(), json.dumps(merged_groups), len(merged_groups), max_book_id))
+                        cwa_db.con.commit()
+                        log.info("[cwa-duplicates] Duplicate cache updated (incremental): merged_groups=%s max_book_id=%s",
+                                 len(merged_groups), max_book_id)
+                    except Exception as ex:
+                        log.warning("[cwa-duplicates] Failed to update incremental cache: %s", str(ex))
+
+                    # Result count is unresolved duplicates for this run (exclude dismissed)
+                    self.result_count = len(find_duplicate_books_python(
+                        use_title, use_author, use_language, use_series, use_publisher, use_format,
+                        include_dismissed=False, user_id=self.user_id, candidate_ids=candidate_ids
+                    ))
+
+            self.progress = 1
+            if self.full_scan:
+                self.message = N_('Duplicate scan completed: %(count)s groups', count=self.result_count)
+            else:
+                self.message = N_('Duplicate scan completed: %(count)s new groups', count=self.result_count)
+            self._handleSuccess()
+        except Exception as ex:
+            log.error("[cwa-duplicates] Duplicate scan task failed: %s", str(ex))
+            self._handleError(str(ex))
+        finally:
+            try:
+                if calibre_db.session is not None:
+                    calibre_db.session.close()
+            except Exception:
+                pass

cps/tasks_status.py

@@ -44,7 +44,8 @@ def render_task_status(tasklist):
         if user == current_user.name or current_user.role_admin():
             ret = {}
             if task.start_time:
-                ret['starttime'] = format_datetime(task.start_time, format='short')
+                # Use ISO-like date format for consistency across locales
+                ret['starttime'] = format_datetime(task.start_time, format="yyyy-MM-dd HH:mm")
                 ret['runtime'] = format_runtime(task.runtime)
 
             # localize the task status

cps/templates/author.html

@@ -1,5 +1,6 @@
 {% extends "layout.html" %}
 {% block body %}
+<input type="hidden" name="csrf_token" value="{{ csrf_token() }}">
 <h2>{{title}}</h2>
 
 {% if author is not none %}

cps/templates/config_view_edit.html

@@ -2,178 +2,218 @@
 {% block header %}
 <link href="{{ url_for('static', filename='css/libs/bootstrap-table.min.css') }}" rel="stylesheet">
 <link href="{{ url_for('static', filename='css/libs/bootstrap-editable.css') }}" rel="stylesheet">
+<style>
+  /* Mobile responsive styles for config_view_edit template - triggers below 768px */
+  @media (max-width: 768px) {
+    .settings-container {
+      padding-left: 1.5rem !important;
+      padding-right: 1.5rem !important;
+      margin-left: 0 !important;
+      margin-right: 0 !important;
+    }
+    
+    .form-control {
+      font-size: 16px; /* Prevents zoom on iOS */
+    }
+    
+    /* Stack buttons vertically on mobile */
+    .btn {
+      display: block;
+      width: 100%;
+      margin-bottom: 10px;
+    }
+  }
+</style>
 {% endblock %}
 {% block body %}
 <div class="discover">
   <h2>{{title}}</h2>
-<form role="form" method="POST" autocomplete="off" >
-<input type="hidden" name="csrf_token" value="{{ csrf_token() }}">
-<div class="panel-group" class="col-md-10 col-lg-6">
-  <div class="panel panel-default">
-    <div class="panel-heading">
-      <h4 class="panel-title">
-        <a class="accordion-toggle" data-toggle="collapse" href="#collapsefour">
-          <span class="glyphicon glyphicon-plus"></span>
-          {{_('View Configuration')}}
-        </a>
-      </h4>
-    </div>
-    <div id="collapsefour" class="panel-collapse collapse">
-      <div class="panel-body">
-                <div class="form-group">
-          <label for="config_calibre_web_title">{{_('Title')}}</label>
-          <input type="text" class="form-control" name="config_calibre_web_title" id="config_calibre_web_title" value="{% if conf.config_calibre_web_title != None %}{{ conf.config_calibre_web_title }}{% endif %}" autocomplete="off" required>
-        </div>
-        <div class="form-group">
-          <label for="config_books_per_page">{{_('Books per Page')}}</label>
-          <input type="number" min="1" max="200" class="form-control" name="config_books_per_page" id="config_books_per_page" value="{% if conf.config_books_per_page != None %}{{ conf.config_books_per_page }}{% endif %}" autocomplete="off">
-        </div>
-        <div class="form-group">
-          <label for="config_random_books">{{_('No. of Random Books to Display')}}</label>
-          <input type="number" min="1" max="30" class="form-control" name="config_random_books" id="config_random_books" value="{% if conf.config_random_books != None %}{{ conf.config_random_books }}{% endif %}" autocomplete="off">
-        </div>
-        <div class="form-group">
-          <label for="config_authors_max">{{_('No. of Authors to Display Before Hiding (0=Disable Hiding)')}}</label>
-          <input type="number" min="0" max="999" class="form-control" name="config_authors_max" id="config_authors_max" value="{% if conf.config_authors_max != None %}{{ conf.config_authors_max }}{% endif %}" autocomplete="off">
-        </div>
-        <div class="form-group">
-        <label for="config_theme">{{_('Default Theme (users can select their own preferred theme however new users will receive this theme by default)')}}</label>
-            <select name="config_theme" id="config_theme" class="form-control">
-                <option value="0" {% if conf.config_theme == 0 %}selected{% endif %}>{{ _("Standard Theme") }}</option>
-                <option value="1" {% if conf.config_theme == 1 %}selected{% endif %}>{{ _("caliBlur! Dark Theme") }}</option>
-                <option disabled>── {{ _('Per-user theme switching enabled; this global default only affects new users & anonymous') }} ──</option>
-            </select>
-        </div>
-        <div class="form-group">
-          <label for="config_columns_to_ignore">{{_('Regular Expression for Ignoring Columns')}}</label>
-          <input type="text" class="form-control" name="config_columns_to_ignore" id="config_columns_to_ignore" value="{% if conf.config_columns_to_ignore != None %}{{ conf.config_columns_to_ignore }}{% endif %}" autocomplete="off">
-        </div>
-        <div class="form-group">
-          <label for="config_read_column">{{_('Link Read/Unread Status to Calibre Column')}}</label>
-              <select name="config_read_column" id="config_read_column" class="form-control">
-                <option value="0" {% if conf.config_read_column == 0 %}selected{% endif %}></option>
-                {%  for readColumn in readColumns %}
-                <option value="{{ readColumn.id }}" {% if readColumn.id == conf.config_read_column %}selected{% endif %}>{{ readColumn.name }}</option>
-                {% endfor %}
-              </select>
-        </div>
-        <div class="form-group">
-          <label for="config_restricted_column">{{_('View Restrictions based on Calibre column')}}</label>
-              <select name="config_restricted_column" id="config_restricted_column" class="form-control">
-                <option value="0" {% if conf.config_restricted_column == 0 %}selected{% endif %}>{{ _('None') }}</option>
-                {%  for restrictColumn in restrictColumns %}
-                <option value="{{ restrictColumn.id }}" {% if restrictColumn.id == conf.config_restricted_column %}selected{% endif %}>{{ restrictColumn.name }}</option>
-                {% endfor %}
-              </select>
-        </div>
-        <div class="form-group">
-          <label for="config_title_regex">{{_('Regular Expression for Title Sorting')}}</label>
-          <input type="text" class="form-control" name="config_title_regex" id="config_title_regex" value="{% if conf.config_title_regex != None %}{{ conf.config_title_regex }}{% endif %}" autocomplete="off">
-        </div>
-        </div>
+  <form role="form" method="POST" autocomplete="off" style="margin-top: 3rem;">
+    <input type="hidden" name="csrf_token" value="{{ csrf_token() }}">
+    
+    <!-- View Configuration Card -->
+    <div class="settings-container">
+      <h4 class="settings-section-header">{{_('View Configuration')}}</h4>
+      
+      <div class="form-group">
+        <label for="config_calibre_web_title">{{_('Title')}}</label>
+        <input type="text" class="form-control" name="config_calibre_web_title" id="config_calibre_web_title" value="{% if conf.config_calibre_web_title != None %}{{ conf.config_calibre_web_title }}{% endif %}" autocomplete="off" required>
+        <p class="settings-explanation">{{_('The title displayed in the browser tab and navbar.')}}</p>
+      </div>
+      
+      <div class="form-group">
+        <label for="config_books_per_page">{{_('Books per Page')}}</label>
+        <input type="number" min="1" max="200" class="form-control" name="config_books_per_page" id="config_books_per_page" value="{% if conf.config_books_per_page != None %}{{ conf.config_books_per_page }}{% endif %}" autocomplete="off">
+        <p class="settings-explanation">{{_('Number of books to display per page in list views (1-200).')}}</p>
+      </div>
+      
+      <div class="form-group">
+        <label for="config_random_books">{{_('No. of Random Books to Display')}}</label>
+        <input type="number" min="1" max="30" class="form-control" name="config_random_books" id="config_random_books" value="{% if conf.config_random_books != None %}{{ conf.config_random_books }}{% endif %}" autocomplete="off">
+        <p class="settings-explanation">{{_('Number of random books shown on the home page (1-30).')}}</p>
+      </div>
+      
+      <div class="form-group">
+        <label for="config_authors_max">{{_('No. of Authors to Display Before Hiding (0=Disable Hiding)')}}</label>
+        <input type="number" min="0" max="999" class="form-control" name="config_authors_max" id="config_authors_max" value="{% if conf.config_authors_max != None %}{{ conf.config_authors_max }}{% endif %}" autocomplete="off">
+        <p class="settings-explanation">{{_('Maximum number of authors to show in book details before truncating. Set to 0 to disable.')}}</p>
+      </div>
+      
+      <div class="form-group">
+        <label for="config_theme">{{_('Default Theme')}}</label>
+        <select name="config_theme" id="config_theme" class="form-control">
+          <option value="0" {% if conf.config_theme == 0 %}selected{% endif %}>{{ _("Standard Theme") }}</option>
+          <option value="1" {% if conf.config_theme == 1 %}selected{% endif %}>{{ _("caliBlur! Dark Theme") }}</option>
+        </select>
+        <p class="settings-explanation">{{_('Default theme for new users and anonymous browsing. Users can select their own preferred theme in their profile.')}}</p>
+      </div>
+      
+      <div class="form-group">
+        <label for="config_columns_to_ignore">{{_('Regular Expression for Ignoring Columns')}}</label>
+        <input type="text" class="form-control" name="config_columns_to_ignore" id="config_columns_to_ignore" value="{% if conf.config_columns_to_ignore != None %}{{ conf.config_columns_to_ignore }}{% endif %}" autocomplete="off">
+        <p class="settings-explanation">{{_('Regex pattern to hide custom columns from being displayed in the UI.')}}</p>
+      </div>
+      
+      <div class="form-group">
+        <label for="config_read_column">{{_('Link Read/Unread Status to Calibre Column')}}</label>
+        <select name="config_read_column" id="config_read_column" class="form-control">
+          <option value="0" {% if conf.config_read_column == 0 %}selected{% endif %}>{{ _('None') }}</option>
+          {%  for readColumn in readColumns %}
+          <option value="{{ readColumn.id }}" {% if readColumn.id == conf.config_read_column %}selected{% endif %}>{{ readColumn.name }}</option>
+          {% endfor %}
+        </select>
+        <p class="settings-explanation">{{_('Sync read/unread status with a Calibre boolean custom column.')}}</p>
+      </div>
+      
+      <div class="form-group">
+        <label for="config_restricted_column">{{_('View Restrictions based on Calibre column')}}</label>
+        <select name="config_restricted_column" id="config_restricted_column" class="form-control">
+          <option value="0" {% if conf.config_restricted_column == 0 %}selected{% endif %}>{{ _('None') }}</option>
+          {%  for restrictColumn in restrictColumns %}
+          <option value="{{ restrictColumn.id }}" {% if restrictColumn.id == conf.config_restricted_column %}selected{% endif %}>{{ restrictColumn.name }}</option>
+          {% endfor %}
+        </select>
+        <p class="settings-explanation">{{_('Restrict book visibility based on values in a Calibre text custom column.')}}</p>
+      </div>
+      
+      <div class="form-group">
+        <label for="config_title_regex">{{_('Regular Expression for Title Sorting')}}</label>
+        <input type="text" class="form-control" name="config_title_regex" id="config_title_regex" value="{% if conf.config_title_regex != None %}{{ conf.config_title_regex }}{% endif %}" autocomplete="off">
+        <p class="settings-explanation">{{_('Regex to strip prefixes from titles for proper sorting (e.g., "The", "A", "An").')}}</p>
       </div>
     </div>
-  <div class="panel panel-default">
-    <div class="panel-heading">
-      <h4 class="panel-title">
-        <a class="accordion-toggle" data-toggle="collapse" href="#collapsesix">
-          <span class="glyphicon glyphicon-plus"></span>
-          {{_('Default Settings for New Users')}}
-        </a>
-      </h4>
-    </div>
-    <div id="collapsesix" class="panel-collapse collapse">
-      <div class="panel-body">
-        <div class="form-group">
-          <input type="checkbox" name="admin_role" id="admin_role" {% if conf.role_admin() %}checked{% endif %}>
+    
+    <!-- Default Settings for New Users Card -->
+    <div class="settings-container">
+      <h4 class="settings-section-header">{{_('Default Settings for New Users')}}</h4>
+      <p class="settings-explanation">{{_('These settings will be applied to all newly created user accounts.')}}</p>
+      
+      <h5 style="color: #ffc200; margin-top: 25px; margin-bottom: 15px;">{{_('Permissions')}}</h5>
+      <div style="display: grid; grid-template-columns: repeat(auto-fit, minmax(250px, 1fr)); gap: 12px; background: #202c34a3; padding: 2rem; border-radius: 4px;">
+        <div>
+          <input type="checkbox" name="admin_role" id="admin_role" {% if conf.role_admin() %}checked{% endif %} style="accent-color: var(--color-secondary);">
           <label for="admin_role">{{_('Admin User')}}</label>
         </div>
-        <div class="form-group">
-          <input type="checkbox" name="download_role" id="download_role" {% if conf.role_download() %}checked{% endif %}>
+        
+        <div>
+          <input type="checkbox" name="download_role" id="download_role" {% if conf.role_download() %}checked{% endif %} style="accent-color: var(--color-secondary);">
           <label for="download_role">{{_('Allow Downloads')}}</label>
         </div>
-        <div class="form-group">
-          <input type="checkbox" name="viewer_role" id="viewer_role" {% if conf.role_viewer() %}checked{% endif %}>
+        
+        <div>
+          <input type="checkbox" name="viewer_role" id="viewer_role" {% if conf.role_viewer() %}checked{% endif %} style="accent-color: var(--color-secondary);">
           <label for="viewer_role">{{_('Allow eBook Viewer')}}</label>
         </div>
-      {% if conf.config_uploading %}
-        <div class="form-group">
-          <input type="checkbox" name="upload_role" id="upload_role" {% if conf.role_upload() %}checked{% endif %}>
+        
+        {% if conf.config_uploading %}
+        <div>
+          <input type="checkbox" name="upload_role" id="upload_role" {% if conf.role_upload() %}checked{% endif %} style="accent-color: var(--color-secondary);">
           <label for="upload_role">{{_('Allow Uploads')}}</label>
         </div>
-      {% endif %}
-        <div class="form-group">
-          <input type="checkbox" name="edit_role" data-control="edit_settings" id="edit_role" {% if conf.role_edit() %}checked{% endif %}>
+        {% endif %}
+        
+        <div>
+          <input type="checkbox" name="edit_role" data-control="edit_settings" id="edit_role" {% if conf.role_edit() %}checked{% endif %} style="accent-color: var(--color-secondary);">
           <label for="edit_role">{{_('Allow Edit')}}</label>
         </div>
+        
         <div data-related="edit_settings">
-          <div class="form-group">
-            <input type="checkbox" name="delete_role" id="delete_role" {% if conf.role_delete_books() %}checked{% endif %}>
-            <label for="delete_role">{{_('Allow Delete Books')}}</label>
-          </div>
+          <input type="checkbox" name="delete_role" id="delete_role" {% if conf.role_delete_books() %}checked{% endif %} style="accent-color: var(--color-secondary);">
+          <label for="delete_role">{{_('Allow Delete Books')}}</label>
         </div>
-        <div class="form-group">
-          <input type="checkbox" name="passwd_role" id="passwd_role" {% if conf.role_passwd() %}checked{% endif %}>
+        
+        <div>
+          <input type="checkbox" name="passwd_role" id="passwd_role" {% if conf.role_passwd() %}checked{% endif %} style="accent-color: var(--color-secondary);">
           <label for="passwd_role">{{_('Allow Changing Password')}}</label>
         </div>
-        <div class="form-group">
-          <input type="checkbox" name="edit_shelf_role" id="edit_shelf_role" {% if conf.role_edit_shelfs() %}checked{% endif %}>
+        
+        <div>
+          <input type="checkbox" name="edit_shelf_role" id="edit_shelf_role" {% if conf.role_edit_shelfs() %}checked{% endif %} style="accent-color: var(--color-secondary);">
           <label for="edit_shelf_role">{{_('Allow Editing Public Shelves')}}</label>
         </div>
-        <div class="form-group">
+      </div>
+      
+      <h5 style="color: #ffc200; margin-top: 25px; margin-bottom: 15px;">{{_('Language & Display')}}</h5>
+      <div class="form-group">
         <label for="config_default_locale">{{_('Default Language')}}</label>
-            <select name="config_default_locale" id="config_default_locale" class="form-control">
-                {%  for translation in translations %}
-                    <option value="{{translation}}" {% if translation|string == conf.config_default_locale %}selected{% endif %}>{{ translation.display_name|capitalize }}</option>
-                {% endfor %}
-            </select>
-        </div>
-        <div class="form-group">
-          <label for="config_default_language">{{_('Default Visible Language of Books')}}</label>
-            <select name="config_default_language" id="config_default_language" class="form-control">
-                <option value="all" {% if conf.config_default_language == "all" %}selected{% endif %}>{{ _('Show All') }}</option>
-                {%  for language in languages %}
-                <option value="{{ language.lang_code }}" {% if conf.config_default_language == language.lang_code %}selected{% endif %}>{{ language.name }}</option>
-                {% endfor %}
-            </select>
-        </div>
+        <select name="config_default_locale" id="config_default_locale" class="form-control">
+          {%  for translation in translations %}
+            <option value="{{translation}}" {% if translation|string == conf.config_default_locale %}selected{% endif %}>{{ translation.display_name|capitalize }}</option>
+          {% endfor %}
+        </select>
+        <p class="settings-explanation">{{_('Default interface language for new users.')}}</p>
+      </div>
+      
+      <div class="form-group">
+        <label for="config_default_language">{{_('Default Visible Language of Books')}}</label>
+        <select name="config_default_language" id="config_default_language" class="form-control">
+          <option value="all" {% if conf.config_default_language == "all" %}selected{% endif %}>{{ _('Show All') }}</option>
+          {%  for language in languages %}
+          <option value="{{ language.lang_code }}" {% if conf.config_default_language == language.lang_code %}selected{% endif %}>{{ language.name }}</option>
+          {% endfor %}
+        </select>
+        <p class="settings-explanation">{{_('Default book language filter for new users.')}}</p>
       </div>
     </div>
-  </div>
-  <div class="panel panel-default">
-    <div class="panel-heading">
-      <h4 class="panel-title">
-        <a class="accordion-toggle" data-toggle="collapse" href="#collapseseven">
-          <span class="glyphicon glyphicon-plus"></span>
-          {{_('Default Visibilities for New Users')}}
-        </a>
-      </h4>
-    </div>
-    <div id="collapseseven" class="panel-collapse collapse">
-      <div class="panel-body">
+    
+    <!-- Default Visibilities for New Users Card -->
+    <div class="settings-container">
+      <h4 class="settings-section-header">{{_('Default Visibilities for New Users')}}</h4>
+      <p class="settings-explanation">{{_('Choose which sidebar sections are visible by default for new users.')}}</p>
+      
+      <div style="display: grid; grid-template-columns: repeat(auto-fit, minmax(250px, 1fr)); gap: 12px; margin-top: 20px; background: #202c34a3; padding: 2rem; border-radius: 4px;">
         {% for element in sidebar %}
           {% if element['config_show'] %}
-            <div class="form-group">
-              <input type="checkbox" name="show_{{element['visibility']}}" id="show_{{element['visibility']}}" {% if conf.show_element_new_user(element['visibility']) %}checked{% endif %}>
-              <label for="show_{{element['visibility']}}">{{element['show_text']}}</label>
-            </div>
+          <div>
+            <input type="checkbox" name="show_{{element['visibility']}}" id="show_{{element['visibility']}}" {% if conf.show_element_new_user(element['visibility']) %}checked{% endif %} style="accent-color: var(--color-secondary);">
+            <label for="show_{{element['visibility']}}">{{element['show_text']}}</label>
+          </div>
           {% endif %}
         {% endfor %}
-        <div class="form-group">
-            <input type="checkbox" name="Show_detail_random" id="Show_detail_random" {% if conf.show_detail_random() %}checked{% endif %}>
-            <label for="Show_detail_random">{{_('Show Random Books in Detail View')}}</label>
+        <div>
+          <input type="checkbox" name="Show_detail_random" id="Show_detail_random" {% if conf.show_detail_random() %}checked{% endif %} style="accent-color: var(--color-secondary);">
+          <label for="Show_detail_random">{{_('Show Random Books in Detail View')}}</label>
         </div>
-      {% if not simple %}
-        <a href="#" id="get_tags" data-id="0" class="btn btn-default" data-toggle="modal" data-target="#restrictModal">{{_('Add Allowed/Denied Tags')}}</a>
-        <a href="#" id="get_column_values" data-id="0" class="btn btn-default" data-toggle="modal" data-target="#restrictModal">{{_('Add Allowed/Denied custom column values')}}</a>
-      {% endif %}
       </div>
+      
+      {% if not simple %}
+      <div style="margin-top: 25px; padding-top: 20px; border-top: 1px solid #5c6b74;">
+        <a href="#" id="get_tags" data-id="0" class="btn btn-default" data-toggle="modal" data-target="#restrictModal">{{_('Add Allowed/Denied Tags')}}</a>
+        <a href="#" id="get_column_values" data-id="0" class="btn btn-default" data-toggle="modal" data-target="#restrictModal" style="margin-left: 10px;">{{_('Add Allowed/Denied custom column values')}}</a>
+      </div>
+      {% endif %}
     </div>
-  </div>
-</div>
-    <div class="col-sm-12">
-    <button type="submit" name="submit" class="btn btn-default">{{_('Save')}}</button>
-    <a href="{{ url_for('admin.admin') }}" class="btn btn-default">{{_('Cancel')}}</a>
+    
+    <!-- Action Buttons -->
+    <div style="max-width: 900px;
+                margin-left: 2rem;
+                margin-right: 2rem;
+                margin-bottom: 4rem;
+                justify-self: center;
+                width: -webkit-fill-available;
+                display: flex;">
+      <button type="submit" name="submit" class="btn btn-default" style="width: -webkit-fill-available;">{{_('Save')}}</button>
+      <a href="{{ url_for('admin.admin') }}" class="btn btn-default" style="width: -webkit-fill-available;">{{_('Cancel')}}</a>
     </div>
   </form>
 </div>

cps/templates/cwa_epub_fixer.html

@@ -14,7 +14,7 @@
       <div style="display: flex;
                   justify-content: space-between;
                   gap: 20px;">
-        <p style="font-size: xx-small;max-width: 70%;">
+        <p style="font-size: 11px; max-width: 70%;">
           Upon loading this page, if you have previously started a run of the CWA Send-to_kindle EPUB Fixer service either here in the Web UI or through the CLI, you will see the output of the most recent previous run below. 
           Once you start a run, you are free to leave the page and return whenever you want to check on the run's progress. If you wish to cancel a run that is still in progress, simply press the Cancel button above 
           and the run will terminate ASAP. This tool is based on the <a href="https://kindle-epub-fix.netlify.app/">Amazon Kindle EPUB Fix</a> tool by <a href="https://github.com/innocenat/kindle-epub-fix">innocenat</a>
@@ -23,7 +23,7 @@
           <a class="btn btn-default" href="{{ url_for('epub_fixer.cancel_epub_fixer') }}" style="vertical-align: top; float: right; width: 100px; margin-left: 10px;">{{_('Cancel')}}</a>
           <a class="btn btn-default" href="{{ url_for('epub_fixer.start_epub_fixer') }}" style="vertical-align: top; float: right; width: 100px;">{{_('Start')}}</a>
           {% if current_user.role_admin() %}
-          <div class="btn-group" role="group" aria-label="schedule" style="float: right; margin-right: 10px;">
+          <div class="btn-group" role="group" aria-label="schedule" style="float: right; margin-top: 4px;">
             <button class="btn btn-default" onclick="scheduleEpubFixer(5)">{{ _('Schedule 5m') }}</button>
             <button class="btn btn-default" onclick="scheduleEpubFixer(15)">{{ _('Schedule 15m') }}</button>
           </div>

cps/templates/cwa_settings.html

@@ -574,7 +574,7 @@
         {{_('When disabled, you will no longer receive notifications in the Web UI when using a language other than English if the translations for your chosen language are not complete.')}}
       </p>
 
-      {% if config.config_kobo_sync_magic_shelves %}
+      {% if config['config_kobo_sync_magic_shelves'] %}
       <input type="checkbox" id="config_kobo_sync_magic_shelves" name="config_kobo_sync_magic_shelves" value="True" checked style="accent-color: var(--color-secondary);" data-toggle="tooltip" data-placement="right" title="Requires Kobo Sync to be enabled in Basic Configuration">
       {% else %}
       <input type="checkbox" id="config_kobo_sync_magic_shelves" name="config_kobo_sync_magic_shelves" value="True" style="accent-color: var(--color-secondary);" data-toggle="tooltip" data-placement="right" title="Requires Kobo Sync to be enabled in Basic Configuration">
@@ -783,6 +783,99 @@
       </tbody></table>
     </div>
 
+    <div class="settings-container">
+      <h4 class="settings-section-header">{{_('CWA Duplicate Detection System')}}</h4>
+
+      <p class="cwa-settings-explanation settings-explanation">
+        {{_('Enable or disable the duplicate detection system. When disabled, duplicate scanning will not run and no notifications will be shown.')}}
+      </p>
+
+      <div class="checkbox-wrapper" style="margin-bottom: 1.5rem;">
+        <input type="checkbox" id="duplicate_detection_enabled" name="duplicate_detection_enabled" value="1" {% if cwa_settings['duplicate_detection_enabled'] %} checked {% endif %}>
+        <label for="duplicate_detection_enabled">{{_('Enable Duplicate Detection')}}</label>
+        <small class="settings-explanation">{{_('When enabled, CWA will scan for duplicate books after each import')}}</small>
+      </div>
+
+      <!-- Detection Method (Hybrid default) -->
+      {% set scan_method = cwa_settings.get('duplicate_scan_method')|default('hybrid', true) %}
+      <div class="form-group" style="max-width: 520px;">
+        <label for="duplicate_scan_method" class="settings-section-header" style="margin-top: 8px;">{{_('Duplicate Detection Method')}}</label>
+        <select class="cwa-settings-select" name="duplicate_scan_method" id="duplicate_scan_method" style="width: fit-content;">
+          <option value="hybrid" {% if scan_method == 'hybrid' %}selected{% endif %}>
+            {{_('Hybrid (SQL prefilter + Python validation)')}}
+          </option>
+          <option value="python" {% if scan_method == 'python' %}selected{% endif %}>
+            {{_('Python only (slowest, most robust)')}}
+          </option>
+          <option value="sql" {% if scan_method == 'sql' %}selected{% endif %}>
+            {{_('Legacy SQL only (experimental)')}}
+          </option>
+        </select>
+        <p class="cwa-settings-explanation settings-explanation" style="margin-top: 10px;">
+          {{_('Hybrid mode uses a fast SQL prefilter to narrow candidates, then applies the robust Python logic for final results.') }}
+        </p>
+      </div>
+
+      <!-- Keep SQL prefilter enabled by default for hybrid/auto modes -->
+      <input type="hidden" id="duplicate_detection_use_sql" name="duplicate_detection_use_sql" value="1">
+    </div>
+
+    <div class="settings-container">
+      <h4 class="settings-section-header">{{_('Automatic Duplicate Scanning')}}</h4>
+
+      <p class="cwa-settings-explanation settings-explanation">
+        {{_('Configure background duplicate scans. These run automatically without blocking the UI.') }}
+      </p>
+
+      <div class="checkbox-wrapper" style="margin-bottom: 1rem;">
+        <input type="checkbox" id="duplicate_scan_enabled" name="duplicate_scan_enabled" value="1" {% if cwa_settings['duplicate_scan_enabled'] %} checked {% endif %}>
+        <label for="duplicate_scan_enabled">{{_('Enable automatic duplicate scans')}}</label>
+      </div>
+
+      <div class="form-group" style="max-width: 520px;">
+        <label for="duplicate_scan_frequency" class="settings-section-header">{{_('After import scans')}}</label>
+        <select class="cwa-settings-select" name="duplicate_scan_frequency" id="duplicate_scan_frequency" style="width: fit-content;">
+          <option value="after_import" {% if cwa_settings.get('duplicate_scan_frequency', 'after_import') == 'after_import' %}selected{% endif %}>
+            {{_('Run after each import (debounced)')}}
+          </option>
+          <option value="manual" {% if cwa_settings.get('duplicate_scan_frequency') == 'manual' %}selected{% endif %}>
+            {{_('Manual only')}}
+          </option>
+        </select>
+      </div>
+
+      <div class="form-group" style="max-width: 520px; margin-top: 0.75rem;">
+        <label for="duplicate_scan_debounce_seconds" class="settings-section-header">{{_('After import debounce (seconds)')}}</label>
+        <input type="number" min="5" max="600" step="1" class="cwa-settings-input"
+               name="duplicate_scan_debounce_seconds" id="duplicate_scan_debounce_seconds"
+               value="{{ cwa_settings.get('duplicate_scan_debounce_seconds', 30) }}"
+               style="width: 120px; background: #151e2680; border: none; padding: 1rem; border-radius: 4px; margin-left: 1rem;">
+        <p class="cwa-settings-explanation settings-explanation" style="margin-top: 8px;">
+          {{_('Wait this many seconds after the last import before starting a background scan.') }}
+        </p>
+      </div>
+
+      <div class="form-group" style="max-width: 520px; margin-top: 1rem;">
+        <label for="duplicate_scan_cron" class="settings-section-header">{{_('Scheduled scans (cron expression)')}}</label>
+        <input type="text" class="cwa-settings-input" name="duplicate_scan_cron" id="duplicate_scan_cron" placeholder="0 3 * * *"
+               value="{{ cwa_settings.get('duplicate_scan_cron', '') }}" style="width: 220px; background: #151e2680; border: none; padding: 1rem; border-radius: 4px; margin-left: 1rem;">
+        <p class="cwa-settings-explanation settings-explanation" style="margin-top: 8px;">
+          {{_('Leave blank to disable scheduled scans. Example:')}} <code>0 3 * * *</code>
+        </p>
+        <p class="cwa-settings-explanation settings-explanation" style="margin-top: 6px;">
+          {{_('If enabled, after-import scans and cron scans can both run. Use “Manual only” to disable after-import scans while keeping cron schedules.')}}
+        </p>
+      </div>
+
+      {% if next_duplicate_scan_run %}
+        <div class="cwa-settings-tip" style="margin-top: 12px;">
+          <small class="settings-explanation">
+            <strong>{{_('Next scheduled scan:')}}</strong> {{ next_duplicate_scan_run }}
+          </small>
+        </div>
+      {% endif %}
+    </div>
+
     <div class="settings-container">
       <h4 class="settings-section-header">{{_('CWA Duplicate Detection Criteria')}}</h4>
 
@@ -835,6 +928,99 @@
       </div>
     </div>
 
+    <div class="settings-container">
+      <h4 class="settings-section-header">{{_('Duplicate Format Priority Ranking')}}</h4>
+
+      <p class="cwa-settings-explanation settings-explanation">
+        {{_('Configure which file formats are preferred when using the "Highest Quality Format" auto-resolution strategy. Drag and drop to reorder formats by priority (higher = better).')}}
+      </p>
+
+      <input type="hidden" id="duplicate_format_priority" name="duplicate_format_priority" value='{{ cwa_settings.get("duplicate_format_priority", "{\"EPUB\":100,\"KEPUB\":95,\"AZW3\":90,\"MOBI\":80,\"AZW\":75,\"PDF\":60,\"TXT\":40,\"CBZ\":35,\"CBR\":35,\"FB2\":30,\"DJVU\":25,\"HTML\":20,\"RTF\":15,\"DOC\":10,\"DOCX\":10}") }}'>
+      
+      <div id="format_priority_container" style="margin-top: 1rem;">
+        <ul id="format_priority_list" style="list-style: none; padding: 0; margin: 0;">
+          <!-- Will be populated by JavaScript -->
+        </ul>
+      </div>
+
+      <div class="cwa-settings-tip">
+        <small class="settings-explanation">
+          <strong>{{_('Tip:')}}</strong> {{_('When resolving duplicates with "Highest Quality Format" strategy, books with higher-ranked formats will be kept. Lower priority formats will be deleted.')}}
+        </small>
+      </div>
+    </div>
+
+    <br>
+
+<!--
+    <div class="settings-container">
+      <h4 class="settings-section-header">{{_('Duplicate Notifications & Auto-Resolution')}}</h4>
+
+      <p class="cwa-settings-explanation settings-explanation">
+        {{_('Configure how you are notified about duplicate books and optionally enable automatic resolution.')}}
+      </p>
+
+      <div class="checkbox-wrapper">
+        <input type="checkbox" id="duplicate_notifications_enabled" name="duplicate_notifications_enabled" value="1" {% if cwa_settings['duplicate_notifications_enabled'] %} checked {% endif %}>
+        <label for="duplicate_notifications_enabled">{{_('Enable Duplicate Notifications')}}</label>
+      </div>
+        <p class="cwa-settings-tip" style="font-size: small;">{{_('Show popup notifications when unresolved duplicates are detected. Admins and users with edit rights will see a badge on the Duplicates sidebar button and a notification popup when they login.')}}</small>
+
+      <div class="checkbox-wrapper">
+        <input type="checkbox" id="duplicate_auto_resolve_enabled" name="duplicate_auto_resolve_enabled" value="1" {% if cwa_settings['duplicate_auto_resolve_enabled'] %} checked {% endif %}>
+        <label for="duplicate_auto_resolve_enabled">{{_('Enable Automatic Duplicate Resolution')}} ⚠️</label>
+      </div>
+      <p class="cwa-settings-tip" style="background: #e74c3c57; border-color: #e74c3c; font-size: small;">
+        <strong>{{_('Warning:')}}</strong> {{_('This will automatically delete books based on the strategy below. Original files will be backed up to')}} <code>/config/processed_books/</code>
+      </p>
+
+      <div class="form-group" style="padding: 2rem; background: #151e2680;">
+        <h4 for="duplicate_auto_resolve_strategy" class="settings-section-header">{{_('Auto-Resolution Strategy')}}</h4>
+        <select class="form-control" id="duplicate_auto_resolve_strategy" name="duplicate_auto_resolve_strategy">
+          <option value="newest" {% if cwa_settings.get('duplicate_auto_resolve_strategy', 'newest') == 'newest' %} selected {% endif %}>
+            {{_('Keep Newest')}} - {{_('Delete older copies, keep the most recently added book')}}
+          </option>
+          <option value="highest_quality_format" {% if cwa_settings.get('duplicate_auto_resolve_strategy') == 'highest_quality_format' %} selected {% endif %}>
+            {{_('Keep Highest Quality Format')}} - {{_('Keep EPUB over MOBI, AZW3 over AZW, etc.')}}
+          </option>
+          <option value="most_metadata" {% if cwa_settings.get('duplicate_auto_resolve_strategy') == 'most_metadata' %} selected {% endif %}>
+            {{_('Keep Most Complete Metadata')}} - {{_('Keep the book with most tags, series info, etc.')}}
+          </option>
+          <option value="largest_file_size" {% if cwa_settings.get('duplicate_auto_resolve_strategy') == 'largest_file_size' %} selected {% endif %}>
+            {{_('Keep Largest File Size')}} - {{_('Keep the book with the largest file size')}}
+          </option>
+        </select>
+        <p class="cwa-settings-explanation settings-explanation" style="margin-top: 2rem !important;">{{_('Choose which book to keep when duplicates are automatically resolved.')}}</p>
+      </div>
+
+      <div class="cwa-settings-tip">
+        <small class="settings-explanation">
+          <strong>{{_('Note:')}}</strong> {{_('Auto-resolution runs when the duplicates page is manually triggered. Dismissed duplicate groups are never auto-resolved.')}}
+        </small>
+      </div>
+    </div>
+-->
+
+    <!-- Duplicate Notifications -->
+    <div class="settings-container">
+      <h4 class="settings-section-header">{{_('Duplicate Notifications')}}</h4>
+
+      <p class="cwa-settings-explanation settings-explanation">
+        {{_('Configure how you are notified about duplicate books.')}}
+      </p>
+
+      <!-- Notifications Toggle -->
+      <div class="checkbox-wrapper">
+        {% if cwa_settings['duplicate_notifications_enabled'] %}
+        <input type="checkbox" id="duplicate_notifications_enabled" name="duplicate_notifications_enabled" value="True" checked style="accent-color: var(--color-secondary);">
+        {% else %}
+        <input type="checkbox" id="duplicate_notifications_enabled" name="duplicate_notifications_enabled" value="True" style="accent-color: var(--color-secondary);">
+        {% endif %}
+        <label for="duplicate_notifications_enabled">{{_('Enable Duplicate Notifications')}}</label>
+      </div>
+      <p class="cwa-settings-tip" style="font-size: small;">{{_('Show popup notifications when unresolved duplicates are detected. Admins and users with edit rights will see a badge on the Duplicates sidebar button and a notification popup when they login.')}}</p>
+    </div>
+
     <br>
 
     <br>
@@ -1064,6 +1250,147 @@ document.addEventListener('DOMContentLoaded', function() {
   
   // Initialize the hidden input value
   updateHiddenInput();
+  
+  // Format Priority Ranking System
+  function initializeFormatPriorityList() {
+    const hiddenInput = document.getElementById('duplicate_format_priority');
+    const container = document.getElementById('format_priority_list');
+    
+    if (!hiddenInput || !container) return;
+    
+    try {
+      // Decode HTML entities if present
+      const rawValue = hiddenInput.value || '{}';
+      const decodedValue = rawValue.replace(/&quot;/g, '"').replace(/&#34;/g, '"');
+      const formatPriority = JSON.parse(decodedValue);
+      
+      // Convert to array and sort by priority (highest first)
+      const formatArray = Object.entries(formatPriority).map(([format, priority]) => ({
+        format: format,
+        priority: priority
+      }));
+      formatArray.sort((a, b) => b.priority - a.priority);
+      
+      // Check if we actually have formats to display
+      if (formatArray.length === 0) {
+        console.error('No format priority data found');
+        container.innerHTML = '<li style="color: #e74c3c; padding: 12px;">No format data available. Please save settings to initialize.</li>';
+        return;
+      }
+      
+      // Populate list
+      container.innerHTML = '';
+      formatArray.forEach((item, index) => {
+        const li = document.createElement('li');
+        li.className = 'format-priority-item';
+        li.dataset.format = item.format;
+        li.dataset.priority = item.priority;
+        li.draggable = true;
+        li.innerHTML = `
+          <div style="display: flex; align-items: center; padding: 6px; margin-bottom: 8px; background: #2a3441; border: 1px solid #444; border-radius: 6px; cursor: move; transition: all 0.2s; padding-left: 2rem;">
+            <span style="color: #ffc200; margin-right: 15px; font-size: 18px;">☰</span>
+            <span style="flex: 1; color: white; font-weight: 500;">${item.format}</span>
+            <span style="color: #4CAF50; font-size: 13px; padding: 4px 8px; background: rgba(76, 160, 80, 0.1); border-radius: 4px;">Priority: ${item.priority}</span>
+          </div>
+        `;
+        container.appendChild(li);
+      });
+      
+      setupFormatPriorityDragDrop();
+    } catch (e) {
+      console.error('Error initializing format priority list:', e);
+      console.error('Hidden input value:', hiddenInput ? hiddenInput.value : 'not found');
+      container.innerHTML = '<li style="color: #e74c3c; padding: 12px;">Error loading format priority: ' + e.message + '</li>';
+    }
+  }
+  
+  function setupFormatPriorityDragDrop() {
+    let draggedFormatElement = null;
+    let isDraggingFormat = false;
+    
+    const items = document.querySelectorAll('.format-priority-item');
+    
+    items.forEach(item => {
+      item.addEventListener('mousedown', function(e) {
+        isDraggingFormat = true;
+        draggedFormatElement = this;
+        this.style.opacity = '0.5';
+        document.body.style.userSelect = 'none';
+        e.preventDefault();
+      });
+    });
+    
+    document.addEventListener('mousemove', function(e) {
+      if (!isDraggingFormat || !draggedFormatElement) return;
+      
+      const container = document.getElementById('format_priority_list');
+      const afterElement = getFormatAfterElement(container, e.clientY);
+      
+      if (afterElement == null) {
+        container.appendChild(draggedFormatElement);
+      } else {
+        container.insertBefore(draggedFormatElement, afterElement);
+      }
+    });
+    
+    document.addEventListener('mouseup', function(e) {
+      if (!isDraggingFormat) return;
+      
+      isDraggingFormat = false;
+      if (draggedFormatElement) {
+        draggedFormatElement.style.opacity = '1';
+        draggedFormatElement = null;
+      }
+      document.body.style.userSelect = '';
+      updateFormatPriorityHiddenInput();
+    });
+  }
+  
+  function getFormatAfterElement(container, y) {
+    const draggableElements = [...container.querySelectorAll('.format-priority-item')].filter(el => el.style.opacity !== '0.5');
+    
+    return draggableElements.reduce((closest, child) => {
+      const box = child.getBoundingClientRect();
+      const offset = y - box.top - box.height / 2;
+      
+      if (offset < 0 && offset > closest.offset) {
+        return { offset: offset, element: child };
+      } else {
+        return closest;
+      }
+    }, { offset: Number.NEGATIVE_INFINITY }).element;
+  }
+  
+  function updateFormatPriorityHiddenInput() {
+    const container = document.getElementById('format_priority_list');
+    const hiddenInput = document.getElementById('duplicate_format_priority');
+    const items = container.querySelectorAll('.format-priority-item');
+    
+    // Recalculate priorities based on order (top = highest)
+    const formatPriority = {};
+    const totalItems = items.length;
+    
+    items.forEach((item, index) => {
+      const format = item.dataset.format;
+      // Higher priority for items at the top
+      const priority = 100 - (index * Math.floor(100 / totalItems));
+      formatPriority[format] = priority;
+    });
+    
+    hiddenInput.value = JSON.stringify(formatPriority);
+    
+    // Update priority badges
+    items.forEach((item, index) => {
+      const format = item.dataset.format;
+      const priorityBadge = item.querySelector('span:last-child');
+      if (priorityBadge) {
+        priorityBadge.textContent = `Priority: ${formatPriority[format]}`;
+      }
+    });
+  }
+  
+  // Initialize format priority list on page load
+  initializeFormatPriorityList();
 });
 </script>
 {% endblock %}

cps/templates/cwa_user_activity.html

@@ -452,13 +452,7 @@
         <div id="heatmap-chart" style="width: 100%; height: 350px;"></div>
     </div>
     
-    <!-- Reading Velocity Trend -->
-    <div class="chart-container">
-        <div class="chart-title">📚 Reading Velocity (Books per Week)</div>
-        <div id="velocity-chart" style="width: 100%; height: 350px;"></div>
-    </div>
-    
-    <!-- Second Row: Event Breakdown & Format Distribution -->
+    <!-- Event Breakdown & Format Distribution -->
     <div class="row cwa-stats-row">
         <div class="col-md-6">
             <div class="chart-container">
@@ -474,17 +468,7 @@
         </div>
     </div>
     
-    <!-- Third Row: Download Formats Pie Chart -->
-    <div class="row cwa-stats-row">
-        <div class="col-md-12">
-            <div class="chart-container">
-                <div class="chart-title">📦 Download Formats Distribution</div>
-                <div id="formats-chart" style="width: 100%; height: 350px;"></div>
-            </div>
-        </div>
-    </div>
-    
-    <!-- Fourth Row: Discovery Sources, Device Breakdown, Security -->
+    <!-- Discovery Sources, Device Breakdown, Security -->
     <div class="row cwa-stats-row">
         <div class="col-md-4">
             <div class="chart-container">
@@ -511,7 +495,7 @@
         </div>
     </div>
     
-    <!-- Fifth Row: Top Lists -->
+    <!-- Top Lists -->
     <div class="row cwa-stats-row">
         <div class="col-md-4">
             <div class="chart-container">
@@ -581,6 +565,22 @@
         <p style="color: #bbb; font-size: 0.9em;">Downloads, reading sessions, and searches will be tracked here.</p>
     </div>
 
+    <!-- Download Formats Pie Chart -->
+    <div class="row cwa-stats-row">
+        <div class="col-md-12">
+            <div class="chart-container">
+                <div class="chart-title">📦 Download Formats Distribution</div>
+                <div id="formats-chart" style="width: 100%; height: 350px;"></div>
+            </div>
+        </div>
+    </div>
+
+    <!-- Reading Velocity Trend -->
+    <div class="chart-container">
+        <div class="chart-title">📚 Reading Velocity (Books per Week)</div>
+        <div id="velocity-chart" style="width: 100%; height: 350px;"></div>
+    </div>
+
 <script>
 // Data from backend
 const realData = {

cps/templates/duplicates.html

@@ -4,10 +4,10 @@
 <style>
   .duplicate-group {
       margin-bottom: 35px;
-      border: 1px solid #e0e6ed;
+      border: none;
       border-radius: 12px;
       padding: 25px;
-      background: linear-gradient(145deg, #ffffff 0%, #f8fafc 100%);
+      background: #1c28328a;
       box-shadow: 0 4px 6px rgba(0, 0, 0, 0.07), 0 1px 3px rgba(0, 0, 0, 0.08);
       transition: all 0.3s ease;
   }
@@ -19,33 +19,33 @@
       margin-bottom: 25px;
       padding-bottom: 15px;
       border-bottom: 3px solid #e74c3c;
-      background: linear-gradient(90deg, rgba(231, 76, 60, 0.05) 0%, rgba(231, 76, 60, 0.02) 100%);
+      background: linear-gradient(90deg, rgba(231, 76, 60, 0.2) 0%, rgba(231, 76, 60, 0.04) 100%);
       border-radius: 8px;
-      padding: 15px;
+      padding: 3rem;
       margin: -10px -10px 25px -10px;
   }
   .duplicate-title {
       font-size: 22px;
       font-weight: 600;
       margin-bottom: 8px;
-      color: #2c3e50;
+      color: white;
       text-shadow: 0 1px 3px rgba(44, 62, 80, 0.1);
   }
   .duplicate-author {
       font-size: 16px;
-      color: #34495e;
+      color: var(--color-primary);
       margin-bottom: 8px;
       font-weight: 500;
   }
   .duplicate-count {
-      color: #e74c3c;
+      color: white;
       font-weight: 700;
-      font-style: italic;
-      background: linear-gradient(135deg, rgba(231, 76, 60, 0.1) 0%, rgba(231, 76, 60, 0.05) 100%);
+      background: linear-gradient(135deg, rgb(231 76 60 / 51%) 0%, rgb(231 76 60 / 42%) 100%);
       padding: 6px 12px;
       border-radius: 6px;
       display: inline-block;
       border: 1px solid rgba(231, 76, 60, 0.2);
+      text-transform: capitalize;
   }
   .books-grid {
       display: flex;
@@ -57,9 +57,9 @@
       display: flex;
       max-width: 450px;
       padding: 20px;
-      border: 1px solid #e1e8ed;
+      border: none;
       border-radius: 10px;
-      background: linear-gradient(145deg, #ffffff 0%, #fafbfc 100%);
+      background: #34455185;
       box-shadow: 0 3px 6px rgba(0,0,0,0.08), 0 1px 3px rgba(0,0,0,0.06);
       transition: all 0.3s cubic-bezier(0.4, 0, 0.2, 1);
       position: relative;
@@ -86,7 +86,7 @@
   }
   .book-item.selected {
       border-color: #e74c3c;
-      background: linear-gradient(145deg, #fff5f5 0%, #fef2f2 100%);
+      background: #64859d78;
       box-shadow: 0 4px 8px rgba(231, 76, 60, 0.15), 0 2px 4px rgba(231, 76, 60, 0.1);
   }
   .book-item.selected::before {
@@ -125,28 +125,28 @@
       accent-color: #e74c3c;
       transform: scale(1.2);
   }
-  .book-title {
-      font-weight: 600;
-      margin-bottom: 12px;
-      line-height: 1.3;
-      color: #2c3e50;
-      font-size: 15px;
-      text-shadow: 0 1px 2px rgba(44, 62, 80, 0.1);
-  }
+    .book-title {
+        font-weight: 600;
+        margin-bottom: 12px;
+        line-height: 1.3;
+        color: white;
+        font-size: 15px;
+        text-shadow: 0 1px 2px rgba(44, 62, 80, 0.1);
+    }
   .book-meta {
-      color: #34495e;
+      color: white;
       font-size: 13px;
       line-height: 1.5;
       font-weight: 400;
   }
   .book-meta div {
       margin-bottom: 6px;
-      color: #7d7d7d !important;
-  }
-  .book-meta strong {
-      color: #2c3e50;
-      font-weight: 600;
+      color: white !important;
   }
+.book-meta strong {
+    color: var(--color-primary);
+    font-weight: 600;
+}
 .no-duplicates {
     text-align: center;
     padding: 60px 20px;
@@ -216,9 +216,9 @@
   .bulk-actions {
       margin-bottom: 30px;
       padding: 20px;
-      background: linear-gradient(135deg, #f8f9fa 0%, #e9ecef 100%);
+      background: #1c2832;
       border-radius: 10px;
-      border: 1px solid #dee2e6;
+      border: none;
       box-shadow: 0 2px 4px rgba(0,0,0,0.05);
   }
   .btn {
@@ -259,11 +259,50 @@
       opacity: 0.65;
       cursor: not-allowed;
   }
+    .btn-primary {
+      color: #fff;
+      background-color: #3498db;
+      border-color: #3498db;
+    }
+    .btn-primary:hover:not(.disabled) {
+      background-color: #2980b9;
+      border-color: #2471a3;
+    }
+    .btn-primary.disabled {
+      background-color: #6c757d;
+      opacity: 0.65;
+      cursor: not-allowed;
+    }
   .selection-info {
-      margin-left: 15px;
       font-weight: 600;
-      color: #28a745;
+      color: white;
       text-shadow: 0 1px 2px rgba(40, 167, 69, 0.1);
+      background: #cc7b1969;
+      padding: 1rem;
+      max-height: 37px;
+      border-radius: 4px;
+      text-transform: capitalize;
+      line-height: normal;
+      padding-inline: 1.5rem;
+  }
+  div.bulk-actions > div.form-inline {
+      display: flex;
+      flex-direction: row;
+      flex-wrap: wrap;
+      justify-content: flex-start;
+      gap: 1rem;
+  }
+  .glyphicon.glyphicon-refresh {
+      font-size: smaller !important;
+      margin-right: 6px;
+  }
+  .container-fluid img {
+      display: block;
+      width: max(10vw,120px) !important;
+      height: auto;
+      max-width: 26rem;
+      margin: auto;
+      box-shadow: 0 0 12px rgba(0, 0, 0, 0.6);
   }
   @media (max-width: 768px) {
     .books-grid {
@@ -284,23 +323,102 @@
       <input type="hidden" name="csrf_token" value="{{ csrf_token() }}">
       <p style="color: whitesmoke; margin-bottom: 2rem;" class="text-muted">{{_('Books with matching titles, authors, and languages. Books in different languages are not considered duplicates.')}}</p>
       
-      {% if duplicate_groups %}
-        <div class="bulk-actions">
-          <div class="form-inline">
+      <!-- Action bar - always show for manual scan trigger -->
+      <div class="bulk-actions" style="margin-bottom: 20px;">
+        <div class="form-inline">
+          <button type="button" class="btn btn-sm btn-default" id="trigger_scan">
+            <span class="glyphicon glyphicon-refresh"></span> {{_('Scan for Duplicates Now')}}
+          </button>
+          <button type="button" class="btn btn-sm btn-danger" id="cancel_scan" style="display: none;">
+            <span class="glyphicon glyphicon-ban-circle"></span> {{_('Cancel Scan')}}
+          </button>
+          {% if duplicate_groups %}
             <button type="button" class="btn btn-sm btn-default" id="select_all">{{_('Select Duplicates')}}</button>
             <button type="button" class="btn btn-sm btn-default" id="select_none">{{_('Select None')}}</button>
             <button type="button" class="btn btn-sm btn-danger disabled" id="delete_selected" aria-disabled="true">{{_('Delete Selected')}}</button>
             <span id="selection_count" class="selection-info"></span>
+          {% endif %}
+        </div>
+
+        <div id="scan_progress_container" style="margin-top: 12px; display: none;">
+          <div style="display: flex; align-items: center; gap: 12px;">
+            <div style="flex: 1; background: #22303a; border-radius: 6px; overflow: hidden; height: 10px;">
+              <div id="scan_progress_bar" style="height: 10px; width: 0%; background: linear-gradient(90deg, #2ecc71, #3498db);"></div>
+            </div>
+            <span id="scan_progress_label" style="color: #ddd; font-size: 12px;">0%</span>
+          </div>
+          <div id="scan_progress_message" style="margin-top: 6px; color: #bbb; font-size: 12px;"></div>
+        </div>
+
+        {% if next_scan_run %}
+          <div style="margin-top: 10px; color: #bbb; font-size: 12px;">
+            {{_('Next scheduled scan:')}} <span id="next_scan_run">{{ next_scan_run }}</span>
+          </div>
+        {% endif %}
+      </div>
+      
+      {% if duplicate_groups %}
+
+        <!-- Auto-Resolution Section -->
+        <div class="settings-container" style="margin-bottom: 30px; padding: 25px; background: #1c28328a; border-radius: 10px; margin-inline: 0rem; max-width: none;">
+          <h4 style="color: white; margin-bottom: 15px; display: flex; align-items: center;">
+            <span class="glyphicon glyphicon-cog" style="margin-right: 10px;"></span>
+            {{_('Auto-Resolve Duplicates')}}
+          </h4>
+          
+          <p style="color: #bbb; margin-bottom: 20px;">
+            {{_('Automatically resolve all duplicate groups by keeping one book and deleting others based on a strategy.')}}
+          </p>
+          
+          <div style="display: flex; align-items: center; gap: 15px; flex-wrap: wrap;">
+            <label for="resolution_strategy" style="color: white; font-weight: 600;">{{_('Strategy:')}}</label>
+            <select id="resolution_strategy" class="form-control" style="background: #1c2832; color: white; border: 1px solid #444;">
+              <option value="newest">{{_('Keep Newest - Delete older copies, keep the most recently added')}}</option>
+              <option value="oldest">{{_('Keep Oldest - Delete newer copies, keep the earliest added')}}</option>
+              <option value="merge">{{_('Merge Formats - Merge formats into the newest book, delete the rest')}}</option>
+              <option value="highest_quality_format">{{_('Keep Highest Quality Format - Prefer EPUB > KEPUB > AZW3 > MOBI > PDF')}}</option>
+              <option value="most_metadata">{{_('Keep Most Metadata - Keep book with most complete information')}}</option>
+              <option value="largest_file_size">{{_('Keep Largest File Size - Keep book with largest total file size')}}</option>
+            </select>
+            
+            <button type="button" class="btn btn-default" id="preview_resolution">
+              <span class="glyphicon glyphicon-eye-open"></span> {{_('Preview')}}
+            </button>
+            
+            <button type="button" class="btn btn-danger disabled" id="execute_resolution" aria-disabled="true">
+              <span class="glyphicon glyphicon-flash"></span> {{_('Execute Resolution')}}
+            </button>
+          </div>
+          
+          <div class="cwa-settings-tip" style="margin-top: 15px; background: rgba(231, 76, 60, 0.15); border-left: 3px solid #e74c3c; padding: 12px; border-radius: 4px;">
+            <strong style="color: #e74c3c;">⚠️ {{_('Warning:')}}</strong>
+            <span style="color: #ddd;">{{_('This will permanently delete books. Original files will be backed up to')}} <code style="background: #2a3441; padding: 2px 6px; border-radius: 3px; color: #f39c12; line-break: anywhere;">/config/processed_books/duplicate_resolutions/</code></span>
           </div>
         </div>
 
         <div class="duplicates-container">
           {% for group in duplicate_groups %}
-          <div class="duplicate-group">
-            <div class="duplicate-header">
-              <div class="duplicate-title">{{ group.title }}</div>
-              <div class="duplicate-author">by {{ group.author }}</div>
-              <div class="duplicate-count">{{ group.count }} duplicate{{ 's' if group.count > 1 else '' }} found</div>
+          <div class="duplicate-group" data-group-hash="{{ group.group_hash }}">
+            <div class="duplicate-header" style="display: flex; justify-content: space-between; align-items: center;">
+              <div style="flex: 1;">
+                <div class="duplicate-title">{{ group.title }}</div>
+                <div class="duplicate-author">by {{ group.author }}</div>
+                <div class="duplicate-count">{{ group.count }} duplicate{{ 's' if group.count > 1 else '' }} found</div>
+              </div>
+              <div style="display: flex; gap: 10px; align-items: center;">
+                <button type="button"
+                        class="merge-selected-btn disabled"
+                        aria-disabled="true"
+                        style="background: rgba(255,255,255,0.15); border: 1px solid rgba(255,255,255,0.3); color: white; padding: 8px 16px; border-radius: 6px; cursor: pointer; font-size: 14px; font-weight: 500; transition: all 0.2s ease; white-space: nowrap; margin-left: 10px;">
+                  <span class="glyphicon glyphicon-link"></span> {{_('Merge Selected')}}</button>
+                <button class="dismiss-duplicate-btn" 
+                        data-group-hash="{{ group.group_hash }}" 
+                        data-dismissed="false"
+                        title="{{_('Dismiss this duplicate group')}}"
+                        style="background: rgba(255,255,255,0.15); border: 1px solid rgba(255,255,255,0.3); color: white; padding: 8px 16px; border-radius: 6px; cursor: pointer; font-size: 14px; font-weight: 500; transition: all 0.2s ease; white-space: nowrap; margin-left: 10px;">
+                  <span class="glyphicon glyphicon-eye-close"></span> {{_('Dismiss')}}
+                </button>
+              </div>
             </div>
             
             <div class="books-grid">
@@ -370,6 +488,53 @@
   </div>
 </div>
 
+<!-- Merge Confirmation Modal -->
+<div class="modal fade" id="merge_selected_modal" role="dialog" aria-labelledby="mergeSelectedLabel">
+  <div class="modal-dialog">
+    <div class="modal-content">
+      <div class="modal-header bg-primary text-center">
+        <span>{{_('Merge Books')}}</span>
+      </div>
+      <div class="modal-body">
+        <p></p>
+        <div class="text-left">{{_('The following book will be kept:')}}</div>
+        <p></p>
+        <div class="text-left" id="display-merge-target-book" style="font-weight: bold; color: #28a745;"></div>
+        <p></p>
+        <div class="text-left">{{_('The following books will be merged into it (and then deleted):')}}</div>
+        <p></p>
+        <div class="text-left" id="display-merge-source-books"></div>
+        <p></p>
+        <div class="text-left"><strong>{{_('Note: File formats from source books will be added to the target book.')}}</strong></div>
+      </div>
+      <div class="modal-footer">
+        <input id="merge_selected_confirm" type="button" class="btn btn-primary" value="{{_('Merge')}}" name="merge_selected_confirm" data-dismiss="modal">
+        <button id="merge_selected_abort" type="button" class="btn btn-default" data-dismiss="modal">{{_('Cancel')}}</button>
+      </div>
+    </div>
+  </div>
+</div>
+
+<!-- Resolution Preview Modal -->
+<div class="modal fade" id="resolution_preview_modal" role="dialog">
+  <div class="modal-dialog modal-lg" style="width: 50%; max-width: 1200px;">
+    <div class="modal-content">
+      <div class="modal-header" style="background: #e59029; color: white;">
+        <button type="button" class="close" data-dismiss="modal" style="color: white;">&times;</button>
+        <h4 class="modal-title">
+          <span class="glyphicon glyphicon-eye-open"></span> {{_('Resolution Preview')}}
+        </h4>
+      </div>
+      <div class="modal-body" id="resolution_preview_body" style="max-height: 600px; overflow-y: auto; padding: 3rem;">
+        <!-- Preview content will be injected here -->
+      </div>
+      <div class="modal-footer">
+        <button type="button" class="btn btn-default" data-dismiss="modal">{{_('Close')}}</button>
+      </div>
+    </div>
+  </div>
+</div>
+
 <!-- Success Modal -->
 <div class="modal fade" id="success_modal" role="dialog" aria-labelledby="successModalLabel">
   <div class="modal-dialog">

cps/templates/index.html

@@ -42,6 +42,7 @@
 </style>
 {% endblock %}
 {% block body %}
+<input type="hidden" name="csrf_token" value="{{ csrf_token() }}">
 {% if current_user.show_detail_random() and page != "discover" %}
 <div class="discover random-books">
   <h2 class="random-books">{{_('Discover (Random Books)')}}</h2>
@@ -132,6 +133,7 @@
            title="{{_('Refresh shelf to update with latest changes')}}">
           <span style="padding: 5px;" class="glyphicon glyphicon-refresh"></span> {{_('Refresh')}}
         </a>
+        {% if shelf.user_id == current_user.id or current_user.role_admin() %}
         <a href="{{url_for('web.edit_magic_shelf', shelf_id=id)}}" 
            class="btn btn-default"
            style="display: flex;
@@ -142,6 +144,22 @@
            title="{{_('Edit shelf rules')}}">
           <span style="padding: 5px;" class="glyphicon glyphicon-edit"></span> {{_('Edit')}}
         </a>
+        {% endif %}
+        {% if shelf.user_id != current_user.id %}
+        <button id="toggle-hide-shelf" 
+                class="btn btn-{% if is_hidden_shelf %}success{% else %}warning{% endif %}"
+                data-shelf-id="{{ id }}"
+                data-is-hidden="{{ 'true' if is_hidden_shelf else 'false' }}"
+                style="display: flex;
+                       align-items: center;
+                       flex-direction: row;
+                       gap: 0.5rem;" 
+                data-toggle="tooltip" 
+                title="{{_('Hide this shelf from your sidebar') if not is_hidden_shelf else _('Show this shelf in your sidebar')}}">
+          <span style="padding: 5px;" class="glyphicon glyphicon-{% if is_hidden_shelf %}eye-open{% else %}eye-close{% endif %}"></span> 
+          <span class="hide-text">{% if is_hidden_shelf %}{{_('Show Shelf')}}{% else %}{{_('Hide Shelf')}}{% endif %}</span>
+        </button>
+        {% endif %}
     </div>
     {% endif %}
 
@@ -239,3 +257,43 @@
   </div>
 </div>
 {% endblock %}
+{% block js %}
+{{ super() }}
+{% if page == 'magicshelf' %}
+<script>
+$(document).ready(function() {
+    // Handle hide/unhide shelf button
+    $('#toggle-hide-shelf').on('click', function() {
+        var $btn = $(this);
+        var shelfId = $btn.data('shelf-id');
+        var isHidden = $btn.data('is-hidden') === 'true';
+        var action = isHidden ? 'unhide' : 'hide';
+        
+        $btn.prop('disabled', true);
+        
+        $.ajax({
+            url: '/magicshelf/' + shelfId + '/' + action,
+            method: 'POST',
+            success: function(response) {
+                if (response.success) {
+                    // Reload page to update sidebar and button state
+                    location.reload();
+                } else {
+                    alert('Error: ' + (response.message || 'Unknown error'));
+                    $btn.prop('disabled', false);
+                }
+            },
+            error: function(xhr) {
+                var message = 'An unexpected error occurred.';
+                if (xhr.responseJSON && xhr.responseJSON.message) {
+                    message = xhr.responseJSON.message;
+                }
+                alert('Error: ' + message);
+                $btn.prop('disabled', false);
+            }
+        });
+    });
+});
+</script>
+{% endif %}
+{% endblock %}

cps/templates/layout.html

@@ -4,7 +4,7 @@
 <!DOCTYPE html>
 <html lang="{{ current_user.locale }}">
   <head>
-    <title>{{instance}} | {{title}}</title>
+    <title>{{instance}} | {{title|safe}}</title>
     <meta charset="utf-8">
     <meta name="viewport" content="width=device-width, initial-scale=1.0">
     <meta http-equiv="X-UA-Compatible" content="IE=edge,chrome=1">
@@ -24,6 +24,9 @@
       <link href="{{ url_for('static', filename='css/caliBlur_override.css') }}" rel="stylesheet" media="screen">
     {% endif %}
     <link href="{{ url_for('static', filename='css/cwa.css') }}" rel="stylesheet" media="screen">
+    {% if current_user.is_authenticated and (current_user.role_admin() or current_user.role_edit()) %}
+    <link href="{{ url_for('static', filename='css/duplicates-notifications.css') }}" rel="stylesheet" media="screen">
+    {% endif %}
     <script>
       let checkMessagesInterval = null; // Store interval ID
 
@@ -278,7 +281,14 @@
               <li class="nav-head hidden-xs">{{_('Browse')}}</li>
               {% for element in sidebar %}
                 {% if current_user.check_visibility(element['visibility']) and element['public'] %}
-                    <li id="nav_{{element['id']}}" {% if page == element['page'] %}class="active"{% endif %}><a href="{{url_for(element['link'], data=element['page'], sort_param='stored')}}"><span class="glyphicon {{element['glyph']}}"></span> {{_(element['text'])}}</a></li>
+                    <li id="nav_{{element['id']}}" {% if page == element['page'] %}class="active"{% endif %} style="{% if element['id'] == 'duplicates' %}position: relative;{% endif %}">
+                      <a href="{{url_for(element['link'], data=element['page'], sort_param='stored')}}">
+                        <span class="glyphicon {{element['glyph']}}"></span> {{_(element['text'])}}
+                        {% if element['id'] == 'duplicates' %}
+                          <span id="duplicate-count-badge" class="duplicate-badge" style="display: none;">0</span>
+                        {% endif %}
+                      </a>
+                    </li>
                 {% endif %}
               {% endfor %}
               {% if current_user.is_authenticated or g.allow_anonymous %}
@@ -294,7 +304,7 @@
               {% if current_user.is_authenticated %}
                 <li class="nav-head hidden-xs" style="margin-top: 1rem;">{{_('Magic Shelves ✨')}}</li>
                   {% for shelf in g.magic_shelves_access %}
-                    <li><a href="{{url_for('web.render_magic_shelf', shelf_id=shelf.id)}}" title="{{shelf.name}} ({{shelf.book_count}})"><span style="font-size: 14px;">{{shelf.icon}}</span> {{shelf.name|shortentitle(40)}} <span style="font-size: 80%; color: #888;">({{shelf.book_count}})</span></a></li>
+                    <li><a href="{{url_for('web.render_magic_shelf', shelf_id=shelf.id)}}" title="{{shelf.name}} ({{shelf.book_count}})"><span style="font-size: 14px;">{{shelf.icon}}</span> {{shelf.name|shortentitle(40)}}{% if shelf.is_public == 1 %} {{_('(Public)')}}{% endif %} <span style="font-size: 80%; color: #888;">({{shelf.book_count}})</span></a></li>
                   {% endfor %}
                 <li id="nav_createmagicshelf" class="create-shelf"><a href="{{url_for('web.create_magic_shelf')}}" style="display: flex; align-items: center;">{{_('Create Magic Shelf')}}</a></li>
               {% endif %}
@@ -402,5 +412,34 @@
       window.addEventListener('DOMContentLoaded', updateFilterHeaderFixed);
       window.addEventListener('resize', updateFilterHeaderFixed);
       </script>
+    {% if current_user.is_authenticated and (current_user.role_admin() or current_user.role_edit()) %}
+    <!-- Duplicate Notification Modal -->
+    <div id="duplicate-notification-backdrop" class="duplicate-notification-backdrop"></div>
+    <div id="duplicate-notification-modal" class="duplicate-notification-modal" tabindex="-1" role="dialog" aria-labelledby="duplicate-notification-title">
+      <div class="duplicate-notification-header">
+        <button id="duplicate-notification-close" class="duplicate-notification-close" aria-label="{{_('Close')}}">&times;</button>
+        <h2 id="duplicate-notification-title">{{_('Duplicate Books Detected')}}</h2>
+        <p>{{_('You have unresolved duplicate books in your library')}}</p>
+      </div>
+      <div class="duplicate-notification-body">
+        <div class="duplicate-count-section">
+          <div class="duplicate-count-badge" id="duplicate-notification-count">0</div>
+          <h3>{{_('Duplicate Groups Found')}}</h3>
+        </div>
+        <ul id="duplicate-notification-preview" class="duplicate-preview-list">
+          <!-- Preview items will be inserted here by JavaScript -->
+        </ul>
+      </div>
+      <div class="duplicate-notification-footer">
+        <button id="duplicate-notification-remind" class="duplicate-notification-btn duplicate-notification-btn-secondary">
+          {{_('Remind Me Later')}}
+        </button>
+        <a href="{{ url_for('duplicates.show_duplicates') }}" class="duplicate-notification-btn duplicate-notification-btn-primary">
+          {{_('View & Resolve Duplicates')}}
+        </a>
+      </div>
+    </div>
+    <script src="{{ url_for('static', filename='js/duplicate-notifier.js') }}"></script>
+    {% endif %}
   </body>
 </html>

cps/templates/magic_shelf_edit.html

@@ -63,6 +63,23 @@
     border: 1px solid #5a2d2d;
     color: #ff9090;
 }
+
+/* Date Format Help Tooltips */
+.date-format-help {
+    margin-top: 5px;
+    padding: 8px 12px;
+    background: #21252b;
+    border: 1px solid #3e444c;
+    border-radius: 4px;
+    color: #90b8e0;
+    font-size: 12px;
+    display: none;
+}
+.date-format-help.warning {
+    background: #3a2e1e;
+    border-color: #5a4d2d;
+    color: #ffcc66;
+}
 .preview-book-list {
     list-style: none;
     padding-left: 0;
@@ -231,6 +248,31 @@ div.help-panel > .panel-body {
     background: #292d32;
     padding: 0.5rem;
     padding-inline: 1rem;
+}
+
+/* Date format hint styling - matches preview results box */
+.date-format-hint {
+    display: none;
+    margin-top: 1rem;
+    padding: 1rem 1.5rem;
+    background: #1a1d21;
+    border: 1px solid #3a3d41;
+    border-radius: 4px;
+    border-left: 4px solid #5bc0de;
+}
+
+.date-format-hint .hint-icon {
+    color: #5bc0de;
+    margin-right: 0.5rem;
+}
+
+.date-format-hint .hint-format {
+    color: #cc7b19;
+    font-weight: bold;
+}
+
+.date-format-hint.show {
+    display: block;
     border-radius: 4px;
     border: 1px solid #3e444c;
 }
@@ -349,6 +391,19 @@ div.help-panel > .panel-body {
                         <small class="form-text text-muted">Only works if Kobo integration is enabled</small>
                     </div>
 
+                    <!-- Public Shelf Checkbox (only for custom shelves with permission) -->
+                    {% if current_user.role_edit_shelfs() and not (shelf and shelf.is_system) %}
+                    <div class="form-group" style="background: #21252b; padding: 2rem; padding-inline: 4rem; border-radius: 4px;">
+                        <label>
+                            <input style="margin-right: 2rem !important;" type="checkbox" id="shelf-is-public" {{ 'checked' if shelf and shelf.is_public == 1 else '' }}>
+                            {{_('Share with Everyone (Public)')}}
+                        </label>
+                        <small class="form-text text-muted">
+                            {{_('Other users can view this shelf. Users with "Edit Public Shelves" permission can edit/delete it.')}}
+                        </small>
+                    </div>
+                    {% endif %}
+
                     <hr>
 
                     <!-- Rules Builder -->
@@ -358,6 +413,12 @@ div.help-panel > .panel-body {
                     </h4>
                     <div id="builder"></div>
                     
+                    <!-- Date Format Hint (shown when date fields are used) -->
+                    <div id="date-format-hint" class="date-format-hint">
+                        <span class="glyphicon glyphicon-info-sign hint-icon"></span>
+                        <strong>Date Format:</strong> Use <span class="hint-format">YYYY-MM-DD</span> format for date fields (e.g., 2024-01-15)
+                    </div>
+                    
                     <!-- Preview Results -->
                     <div id="preview-results" class="preview-results">
                         <h5><span class="glyphicon space glyphicon-eye-open"></span> Preview Results</h5>
@@ -378,9 +439,20 @@ div.help-panel > .panel-body {
                             <span class="glyphicon space glyphicon-remove"></span> Cancel
                         </a>
                         {% if shelf %}
-                        <button type="button" class="btn btn-danger pull-right" id="delete-shelf-btn">
-                            <span class="glyphicon space glyphicon-trash"></span> Delete Shelf
-                        </button>
+                            {% if shelf.is_system %}
+                            <button type="button" 
+                                    class="btn btn-danger pull-right" 
+                                    id="delete-shelf-btn" 
+                                    disabled
+                                    data-toggle="tooltip"
+                                    title="{{_('System shelves cannot be deleted. You can hide them in your user profile settings.')}}">
+                                <span class="glyphicon space glyphicon-trash"></span> Delete Shelf
+                            </button>
+                            {% else %}
+                            <button type="button" class="btn btn-danger pull-right" id="delete-shelf-btn">
+                                <span class="glyphicon space glyphicon-trash"></span> Delete Shelf
+                            </button>
+                            {% endif %}
                         {% endif %}
                     </div>
                 </form>
@@ -570,12 +642,84 @@ $(function() {
 
     var rules = {{ rules_json|safe }};
 
+    // Date validation helper
+    function isValidDate(dateString) {
+        if (!dateString) return true; // Empty is valid (will be handled by is_empty operator)
+        var regex = /^\d{4}-\d{2}-\d{2}$/;
+        if (!regex.test(dateString)) return false;
+        var date = new Date(dateString);
+        var timestamp = date.getTime();
+        if (typeof timestamp !== 'number' || Number.isNaN(timestamp)) return false;
+        return date.toISOString().startsWith(dateString);
+    }
+
+    // Show/hide date format hint based on rules
+    function checkForDateFields() {
+        var hasDateField = false;
+        
+        // Check all filter dropdowns for date fields
+        $('#builder .rule-filter-container select').each(function() {
+            var filterId = $(this).val();
+            if (filterId === 'pubdate' || filterId === 'timestamp') {
+                hasDateField = true;
+                return false; // break out of each loop
+            }
+        });
+        
+        if (hasDateField) {
+            $('#date-format-hint').addClass('show');
+        } else {
+            $('#date-format-hint').removeClass('show');
+        }
+    }
+
     $('#builder').queryBuilder({
         plugins: ['bt-tooltip-errors'],
         filters: fields,
         operators: operators,
         rules: rules
     });
+    
+    // Check for date fields on initialization and after changes
+    $('#builder').on('afterCreateRuleInput.queryBuilder afterUpdateRuleFilter.queryBuilder afterUpdateRuleValue.queryBuilder afterDeleteRule.queryBuilder afterUpdateRuleOperator.queryBuilder', function(e) {
+        setTimeout(function() {
+            checkForDateFields();
+        }, 100);
+        
+        // Validate date inputs
+        $(this).find('input[type="text"]').each(function() {
+            var $input = $(this);
+            var $rule = $input.closest('.rule-container');
+            var filterId = $rule.find('.rule-filter-container select').val();
+            
+            if (filterId === 'pubdate' || filterId === 'timestamp') {
+                $input.off('blur.dateValidation').on('blur.dateValidation', function() {
+                    var value = $(this).val();
+                    if (value && !isValidDate(value)) {
+                        $(this).css('border-color', '#d9534f');
+                        if (!$(this).next('.date-error').length) {
+                            $(this).after('<div class="date-error" style="color: #d9534f; font-size: 12px; margin-top: 4px;">Invalid format. Use YYYY-MM-DD (e.g., 2024-01-15)</div>');
+                        }
+                    } else {
+                        $(this).css('border-color', '');
+                        $(this).next('.date-error').remove();
+                    }
+                });
+            }
+        });
+    });
+    
+    // Also listen for direct changes on filter dropdowns
+    $('#builder').on('change', '.rule-filter-container select', function() {
+        setTimeout(function() {
+            checkForDateFields();
+        }, 100);
+    });
+    
+    // Initial check
+    setTimeout(function() {
+        checkForDateFields();
+    }, 100);
 
     // Preview Rules Functionality
     $('#preview-btn').on('click', function() {
@@ -651,7 +795,8 @@ $(function() {
             name: shelfName,
             icon: $('#shelf-icon').val(),
             rules: rules,
-            kobo_sync: $('#shelf-kobo-sync').is(':checked')
+            kobo_sync: $('#shelf-kobo-sync').is(':checked'),
+            is_public: $('#shelf-is-public').is(':checked')
         };
 
         console.log('Submitting shelf data:', shelfData);

cps/templates/tasks.html

@@ -3,8 +3,9 @@
 <link href="{{ url_for('static', filename='css/libs/bootstrap-table.min.css') }}" rel="stylesheet">
 {% endblock %}
 {% block body %}
-<div class="discover">
+<div class="discover" style="padding-inline: 2rem !important;">
     <h2>{{_('Tasks')}}</h2>
+    <h3>{{ _('Background Tasks Overview') }}</h3>
     <table class="table table-no-bordered" id="tasktable" data-url="{{  url_for('tasks.get_email_status_json') }}"  data-sort-name="starttime" data-sort-order="asc" data-locale="{{ current_user.locale }}" data-classes="table table-no-bordered table-hover">
       <thead>
         <tr>

cps/templates/user_edit.html

@@ -1,179 +1,453 @@
 {% extends "layout.html" %}
+{% block header %}
+<style>
+  /* Hide the original :before pseudo-elements for user_edit template */
+  body.me > div.container-fluid > div.row-fluid > div.col-sm-10:before {
+    display: none !important;
+  }
+  
+  body.me > div.container-fluid > div > div.col-sm-10 > div.discover[name="profile-page"]:before {
+    display: none !important;
+  }
+  
+  /* Main container positioning */
+  body.me > div.container-fluid > div > div.col-sm-10 > div.discover[name="profile-page"] {
+      margin: 4rem !important;
+      width: 80% !important;
+      padding: 0px !important;
+      border: none;
+      justify-self: center;
+  }
+  
+  /* Profile header flexbox container */
+  .profile-header-container {
+    display: flex;
+    align-items: center;
+    gap: 3rem;
+    margin-bottom: 30px;
+    padding: 20px;
+    background: #1f2c3554;
+    border-radius: 8px;
+  }
+  
+  .profile-picture {
+    width: 80px;
+    height: 80px;
+    flex-shrink: 0;
+    background-image: url(data:image/svg+xml;base64,PHN2ZyB3aWR0aD0iNTYwIiBoZWlnaHQ9IjU2MCIgeG1sbnM9Imh0dHA6Ly93d3cudzMub3JnLzIwMDAvc3ZnIj4KICAgIDxnIGZpbGw9IiNGRkYiIGZpbGwtcnVsZT0ibm9uemVybyI+CiAgICAgICAgPHBhdGggZD0iTTE0NC4yOTEgNDkyLjMyNUMxNjYuNjI0IDQ3Ni4yNjUgMTkzLjE5NCA0NjYuMTU2IDIyNCA0NjJjMTIuNDQ0IDkuMzMzIDMxLjExMSAxNCA1NiAxNHM0My41NTYtNC42NjcgNTYtMTRjMzAuODA2IDQuMTU2IDU3LjM3NiAxNC4yNjQgNzkuNzA5IDMwLjMyNUMzNzYuNTI3IDUxNy40MzUgMzI5Ljk1MSA1MzIgMjgwIDUzMmMtNDkuOTUxIDAtOTYuNTI3LTE0LjU2NS0xMzUuNzA5LTM5LjY3NXoiIGZpbGwtb3BhY2l0eT0iLjYiLz4KICAgICAgICA8cGF0aCBkPSJNMjI0IDQ2MmwxMi44OC00MC4yNTFDMTk2LjQyOSAzOTcuNDYyIDE2OCAzNDAuMDM1IDE2OCAyNzMuMDU5YzAtMzUuMTkzIDcuODQ5LTY3Ljc1IDIxLjE2OC05NC4yNDggMTYuMTczIDQuOTc3IDM1LjMxNSA3Ljg1NiA1NS44MzIgNy44NTYgNTEuMTA0IDAgOTMuNjgtMTcuODYgMTAzLjA3Mi00MS41MTZDMzc0Ljc3OSAxNzQuNTg4IDM5MiAyMjAuOTMxIDM5MiAyNzMuMDU5YzAgNjYuOTc2LTI4LjQyOSAxMjQuNDAzLTY4Ljg4IDE0OC42OUwzMzYgNDYyYy0xMi40NDQgOS4zMzMtMzEuMTExIDE0LTU2IDE0LTI0LjY5NCAwLTQzLjI2My00LjU5NC01NS43MDctMTMuNzgyTDIyNCA0NjJ6IiBmaWxsLW9wYWNpdHk9Ii43NSIvPgogICAgICAgIDxwYXRoIGQ9Ik0xODAuMDY0IDM0NS44NDlDMTU1LjI4MiAzMTguMDY3IDE0MCAyNzkuOTk3IDE0MCAyMzhjMC04NS4wNTIgNjIuNjgtMTU0IDE0MC0xNTRzMTQwIDY4Ljk0OCAxNDAgMTU0YzAgNDEuOTk3LTE1LjI4MiA4MC4wNjctNDAuMDY0IDEwNy44NDkgNy43MTYtMjEuODYyIDEyLjA2NC00Ni41OTYgMTIuMDY0LTcyLjc5IDAtNTIuMTI4LTE3LjIyMS05OC40NzEtNDMuOTI4LTEyNy45MDgtOS4zOTIgMjMuNjU2LTUxLjk2OCA0MS41MTYtMTAzLjA3MiA0MS41MTYtMjAuNTE3IDAtMzkuNjU5LTIuODc5LTU1LjgzMi03Ljg1NkMxNzUuODQ5IDIwNS4zMDkgMTY4IDIzNy44NjYgMTY4IDI3My4wNTljMCAyNi4xOTQgNC4zNDggNTAuOTI4IDEyLjA2NCA3Mi43OXoiIGZpbGwtb3BhY2l0eT0iLjQ1Ii8+CiAgICAgICAgPHBhdGggZD0iTTI4MCA1MzJjMTM5LjE3NiAwIDI1Mi0xMTIuODI0IDI1Mi0yNTJTNDE5LjE3NiAyOCAyODAgMjggMjggMTQwLjgyNCAyOCAyODBzMTEyLjgyNCAyNTIgMjUyIDI1MnptMCAyOEMxMjUuMzYgNTYwIDAgNDM0LjY0IDAgMjgwUzEyNS4zNiAwIDI4MCAwczI4MCAxMjUuMzYgMjgwIDI4MC0xMjUuMzYgMjgwLTI4MCAyODB6IiBmaWxsLW9wYWNpdHk9Ii43NSIvPgogICAgPC9nPgo8L3N2Zz4K);
+    background-size: cover;
+    background-position: center;
+    background-repeat: no-repeat;
+    border-radius: 50%;
+    position: relative;
+    display: flex;
+    align-items: center;
+    justify-content: center;
+  }
+  
+  .profile-picture .fallback-icon {
+    display: none;
+    font-size: 4rem;
+    line-height: 1;
+  }
+  
+  .profile-picture.no-custom-picture {
+    background-image: url(data:image/svg+xml;base64,PHN2ZyB3aWR0aD0iNTYwIiBoZWlnaHQ9IjU2MCIgeG1sbnM9Imh0dHA6Ly93d3cudzMub3JnLzIwMDAvc3ZnIj4KICAgIDxnIGZpbGw9IiNGRkYiIGZpbGwtcnVsZT0ibm9uemVybyI+CiAgICAgICAgPHBhdGggZD0iTTE0NC4yOTEgNDkyLjMyNUMxNjYuNjI0IDQ3Ni4yNjUgMTkzLjE5NCA0NjYuMTU2IDIyNCA0NjJjMTIuNDQ0IDkuMzMzIDMxLjExMSAxNCA1NiAxNHM0My41NTYtNC42NjcgNTYtMTRjMzAuODA2IDQuMTU2IDU3LjM3NiAxNC4yNjQgNzkuNzA5IDMwLjMyNUMzNzYuNTI3IDUxNy40MzUgMzI5Ljk1MSA1MzIgMjgwIDUzMmMtNDkuOTUxIDAtOTYuNTI3LTE0LjU2NS0xMzUuNzA5LTM5LjY3NXoiIGZpbGwtb3BhY2l0eT0iLjYiLz4KICAgICAgICA8cGF0aCBkPSJNMjI0IDQ2MmwxMi44OC00MC4yNTFDMTk2LjQyOSAzOTcuNDYyIDE2OCAzNDAuMDM1IDE2OCAyNzMuMDU5YzAtMzUuMTkzIDcuODQ5LTY3Ljc1IDIxLjE2OC05NC4yNDggMTYuMTczIDQuOTc3IDM1LjMxNSA3Ljg1NiA1NS44MzIgNy44NTYgNTEuMTA0IDAgOTMuNjgtMTcuODYgMTAzLjA3Mi00MS41MTZDMzc0Ljc3OSAxNzQuNTg4IDM5MiAyMjAuOTMxIDM5MiAyNzMuMDU5YzAgNjYuOTc2LTI4LjQyOSAxMjQuNDAzLTY4Ljg4IDE0OC42OUwzMzYgNDYyYy0xMi40NDQgOS4zMzMtMzEuMTExIDE0LTU2IDE0LTI0LjY5NCAwLTQzLjI2My00LjU5NC01NS43MDctMTMuNzgyTDIyNCA0NjJ6IiBmaWxsLW9wYWNpdHk9Ii43NSIvPgogICAgICAgIDxwYXRoIGQ9Ik0xODAuMDY0IDM0NS44NDlDMTU1LjI4MiAzMTguMDY3IDE0MCAyNzkuOTk3IDE0MCAyMzhjMC04NS4wNTIgNjIuNjgtMTU0IDE0MC0xNTRzMTQwIDY4Ljk0OCAxNDAgMTU0YzAgNDEuOTk3LTE1LjI4MiA4MC4wNjctNDAuMDY0IDEwNy44NDkgNy43MTYtMjEuODYyIDEyLjA2NC00Ni41OTYgMTIuMDY0LTcyLjc5IDAtNTIuMTI4LTE3LjIyMS05OC40NzEtNDMuOTI4LTEyNy45MDgtOS4zOTIgMjMuNjU2LTUxLjk2OCA0MS41MTYtMTAzLjA3MiA0MS41MTYtMjAuNTE3IDAtMzkuNjU5LTIuODc5LTU1LjgzMi03Ljg1NkMxNzUuODQ5IDIwNS4zMDkgMTY4IDIzNy44NjYgMTY4IDI3My4wNTljMCAyNi4xOTQgNC4zNDggNTAuOTI4IDEyLjA2NCA3Mi43OXoiIGZpbGwtb3BhY2l0eT0iLjQ1Ii8+CiAgICAgICAgPHBhdGggZD0iTTI4MCA1MzJjMTM5LjE3NiAwIDI1Mi0xMTIuODI0IDI1Mi0yNTJTNDE5LjE3NiAyOCAyODAgMjggMjggMTQwLjgyNCAyOCAyODBzMTEyLjgyNCAyNTIgMjUyIDI1MnptMCAyOEMxMjUuMzYgNTYwIDAgNDM0LjY0IDAgMjgwUzEyNS4zNiAwIDI4MCAwczI4MCAxMjUuMzYgMjgwIDI4MC0xMjUuMzYgMjgwLTI4MCAyODB6IiBmaWxsLW9wYWNpdHk9Ii43NSIvPgogICAgPC9nPgo8L3N2Zz4K);
+  }
+  
+  .profile-title-section {
+    display: flex;
+    flex-direction: column;
+    gap: 5px;
+  }
+  
+  .profile-title-section h2 {
+    margin: 0;
+    color: #ffc200;
+  }
+  
+  .profile-subtitle {
+    font-size: 1.2rem;
+    color: #bbbbbb;
+    margin: 0;
+  }
+  
+  /* Mobile responsive styles for user_edit template - triggers below 768px */
+  @media (max-width: 768px) {
+    body.me > div.container-fluid > div > div.col-sm-10 > div.discover[name="profile-page"] {
+      margin-inline: 3rem !important;
+      width: 90% !important;
+      justify-self: center !important;
+      padding-top: 0px !important;
+    }
+    
+    .profile-header-container {
+      flex-direction: column;
+      text-align: center;
+      padding: 15px;
+      gap: 15px;
+    }
+    
+    .profile-picture {
+      width: 100px;
+      height: 100px;
+    }
+    
+    .profile-picture .fallback-icon {
+      font-size: 3rem;
+    }
+    
+    .profile-title-section {
+      align-items: center;
+    }
+    
+    .profile-title-section h2 {
+      font-size: 1.5rem;
+    }
+    
+    .profile-subtitle {
+      font-size: 1rem;
+    }
+  }
+</style>
+{% endblock %}
 {% block body %}
-<div class="discover">
-  <h1>{{title}}</h1>
-  <form role="form" method="POST" autocomplete="off">
+<div name="profile-page" class="discover" style="padding-top: 0px !important;">
+  <h2>{{title}}</h2>
+  <!-- Profile Header with Picture and Title -->
+  <div class="profile-header-container">
+    <div class="profile-picture" id="user-profile-picture">
+      <span class="fallback-icon">👤</span>
+    </div>
+    <div class="profile-title-section">
+      <h2>{{title}}</h2>
+      <p class="profile-subtitle">{% if profile %}{{_('Your Profile')}}{% else %}{{_('User Settings')}}{% endif %}</p>
+    </div>
+  </div>
+  <form role="form" method="POST" autocomplete="off" style="margin-top: 3rem;">
     <input type="hidden" name="csrf_token" value="{{ csrf_token() }}">
-    <div class="col-md-10 col-lg-8">
-    {% if new_user or ( current_user and content.name != "Guest" and current_user.role_admin() ) %}
-    <div class="form-group required">
-      <label for="name">{{_('Username')}}</label>
-      <input type="text" class="form-control" name="name" id="name" value="{{ content.name if content.name != None }}" autocomplete="off">
-    </div>
-    {% endif %}
-    <div class="form-group">
-      <label for="email">{{_('Email')}}</label>
-      <input type="email" class="form-control" name="email" id="email" value="{{ content.email if content.email != None }}" autocomplete="off">
-    </div>
-    {% if ( current_user and current_user.role_passwd() or current_user.role_admin() ) and not content.role_anonymous() %}
-      {% if current_user and current_user.role_admin() and not new_user and not profile and ( mail_configured and content.email if content.email != None ) %}
-        <a class="btn btn-default postAction" id="resend_password" role="button" data-action="{{url_for('admin.reset_user_password', user_id = content.id) }}">{{_('Reset user Password')}}</a>
+    
+    <!-- User Account Information Card -->
+    <div class="settings-container" style="max-width: none; margin-inline: 0rem;">
+      <h4 class="settings-section-header">{{_('Account Information')}}</h4>
+      
+      {% if new_user or ( current_user and content.name != "Guest" and current_user.role_admin() ) %}
+      <div class="form-group required">
+        <label for="name">{{_('Username')}}</label>
+        <input type="text" class="form-control" name="name" id="name" value="{{ content.name if content.name != None }}" autocomplete="off">
+      </div>
       {% endif %}
+
+      <p class="cwa-settings-tip">{{_('If you need to change your email address or password, you can do so here.')}}</p>
+      
+      <div class="form-group">
+        <label for="email">{{_('Email')}}</label>
+        <input type="email" class="form-control" name="email" id="email" value="{{ content.email if content.email != None }}" autocomplete="off">
+      </div>
+      
+      {% if ( current_user and current_user.role_passwd() or current_user.role_admin() ) and not content.role_anonymous() %}
+        {% if current_user and current_user.role_admin() and not new_user and not profile and ( mail_configured and content.email if content.email != None ) %}
+        <div style="margin-bottom: 15px;">
+          <a class="btn btn-default postAction" id="resend_password" role="button" data-action="{{url_for('admin.reset_user_password', user_id = content.id) }}">{{_('Reset user Password')}}</a>
+        </div>
+        {% endif %}
         <div class="form-group">
           <label for="password">{{_('Password')}}</label>
           <input type="password" class="form-control" name="password" id="password" data-lang="{{ current_user.locale }}" data-verify="{{ config.config_password_policy }}" {% if config.config_password_policy %} data-min={{ config.config_password_min_length }} data-word={{ config.config_password_character }} data-special={{ config.config_password_special }} data-upper={{ config.config_password_upper }} data-lower={{ config.config_password_lower }} data-number={{ config.config_password_number }}{% endif %} value="" autocomplete="off">
         </div>
-    {% endif %}
-    <div class="form-group">
-      <label for="kindle_mail">{{_('Send to eReader Email Address. Use comma to separate emails for multiple eReaders')}}</label>
-      <input type="email" class="form-control" name="kindle_mail" id="kindle_mail" value="{{ content.kindle_mail if content.kindle_mail != None }}">
+      {% endif %}
     </div>
-    <div class="form-group">
-      <label for="kindle_mail_subject">{{_('Send to eReader Subject')}}</label>
-      <input type="text" class="form-control" name="kindle_mail_subject" id="kindle_mail_subject" value="{{ content.kindle_mail_subject if content.kindle_mail_subject != None }}">
+    
+    <!-- eReader Configuration Card -->
+    <div class="settings-container" style="max-width: none; margin-inline: 0rem;">
+      <h4 class="settings-section-header">{{_('eReader Configuration')}}</h4>
+      
+      <div class="form-group">
+        <label for="kindle_mail">{{_('Send to eReader Email Address')}}</label>
+        <input type="email" class="form-control" name="kindle_mail" id="kindle_mail" value="{{ content.kindle_mail if content.kindle_mail != None }}" placeholder="{{_('Use comma to separate multiple addresses')}}">
+        <p class="cwa-settings-tip">{{_('Email address(es) for your eReader devices. Separate multiple addresses with commas.')}}</p>
+      </div>
+      
+      <div class="form-group">
+        <label for="kindle_mail_subject">{{_('Send to eReader Subject')}}</label>
+        <input type="text" class="form-control" name="kindle_mail_subject" id="kindle_mail_subject" value="{{ content.kindle_mail_subject if content.kindle_mail_subject != None }}">
+      </div>
+      
+      {% if not content.role_anonymous() %}
+      <div style="margin-top: 20px;">
+        <input type="checkbox" id="auto_send_enabled" name="auto_send_enabled" {% if content.auto_send_enabled %}checked{% endif %} style="accent-color: var(--color-secondary);">
+        <label for="auto_send_enabled">{{_('Automatically send new books to my eReader(s)')}}</label>
+        <p class="cwa-settings-tip">{{_('When enabled, newly ingested books will be automatically sent to all configured eReader email addresses above.')}}</p>
+      </div>
+      {% endif %}
     </div>
+    
+    <!-- Language & Theme Preferences Card -->
     {% if not content.role_anonymous() %}
-    <div class="form-group">
-      <input type="checkbox" id="auto_send_enabled" name="auto_send_enabled" {% if content.auto_send_enabled %}checked{% endif %}>
-      <label for="auto_send_enabled">{{_('Automatically send new books to my eReader(s)')}}</label>
-      <p class="help-block">{{_('When enabled, newly ingested books will be automatically sent to all configured eReader email addresses above.')}}</p>
-    </div>
-    {% endif %}
-    {% if not content.role_anonymous() %}
-    <div class="form-group">
-    <label for="locale">{{_('Language')}}</label>
+    <div class="settings-container" style="max-width: none; margin-inline: 0rem;">
+      <h4 class="settings-section-header">{{_('Language & Theme Preferences')}}</h4>
+      
+      <div class="form-group">
+        <label for="locale">{{_('Interface Language')}}</label>
         <select name="locale" id="locale" class="form-control">
-            {%  for translation in translations %}
-                <option value="{{translation}}" {% if translation|string == content.locale %}selected{% endif %}>{{ translation.display_name|capitalize }}</option>
-            {% endfor %}
+          {%  for translation in translations %}
+            <option value="{{translation}}" {% if translation|string == content.locale %}selected{% endif %}>{{ translation.display_name|capitalize }}</option>
+          {% endfor %}
         </select>
-    </div>
-    {% endif %}
-
-    <div class="form-group">
-      <label for="default_language">{{_('Language of Books')}}</label>
+      </div>
+      
+      <div class="form-group">
+        <label for="default_language">{{_('Language of Books')}}</label>
         <select name="default_language" id="default_language" class="form-control">
-            <option value="all" {% if content.default_language == "all" %}selected{% endif %}>{{ _('Show All') }}</option>
-            {%  for language in languages %}
-            <option value="{{ language.lang_code }}" {% if content.default_language == language.lang_code %}selected{% endif %}>{{ language.name }}</option>
-            {% endfor %}
+          <option value="all" {% if content.default_language == "all" %}selected{% endif %}>{{ _('Show All') }}</option>
+          {%  for language in languages %}
+          <option value="{{ language.lang_code }}" {% if content.default_language == language.lang_code %}selected{% endif %}>{{ language.name }}</option>
+          {% endfor %}
         </select>
+        <p class="cwa-settings-tip">{{_('Filter the library to only show books in a specific language.')}}</p>
+      </div>
+      
+      <div class="form-group">
+        <label for="theme">{{ _('Theme') }}</label>
+        <select name="theme" id="theme" class="form-control">
+          <option value="0" {% if content.theme == 0 %}selected{% endif %}>Standard</option>
+          <option value="1" {% if content.theme == 1 %}selected{% endif %}>caliBlur (Dark)</option>
+        </select>
+      </div>
     </div>
-    {% if not content.role_anonymous() %}
-    <div class="form-group">
-      <label for="theme">{{ _('Theme') }}</label>
-      <select name="theme" id="theme" class="form-control">
-        <option value="0" {% if content.theme == 0 %}selected{% endif %}>Standard</option>
-        <option value="1" {% if content.theme == 1 %}selected{% endif %}>caliBlur (Dark)</option>
-      </select>
+    {% else %}
+    <div class="settings-container" style="max-width: none; margin-inline: 0rem;">
+      <h4 class="settings-section-header">{{_('Book Language Filter')}}</h4>
+      
+      <div class="form-group">
+        <label for="default_language">{{_('Language of Books')}}</label>
+        <select name="default_language" id="default_language" class="form-control">
+          <option value="all" {% if content.default_language == "all" %}selected{% endif %}>{{ _('Show All') }}</option>
+          {%  for language in languages %}
+          <option value="{{ language.lang_code }}" {% if content.default_language == language.lang_code %}selected{% endif %}>{{ language.name }}</option>
+          {% endfor %}
+        </select>
+        <p class="settings-explanation">{{_('Filter the library to only show books in a specific language.')}}</p>
+      </div>
     </div>
     {% endif %}
-    {% if registered_oauth.keys()| length > 0 and not new_user and profile %}
-      {% for id, name in registered_oauth.items() %}
-    <div class="form-group">
-      <label>{{ name }} {{_('OAuth Settings')}}</label>
-        {% if id not in oauth_status %}
-        <a href="{{ url_for('oauth.'+ name +'_login') }}" id="config_{{ id }}_oauth" class="btn btn-primary">{{_('Link')}}</a>
-        {% else %}
-        <a href="{{ url_for('oauth.'+ name +'_login_unlink') }}" id="config_{{ id }}_oauth" class="btn btn-primary">{{_('Unlink')}}</a>
-        {% endif %}
-      {% endfor %}
-    </div>
-    {% endif %}
-    {% if hardcover_support and not new_user %}
-    <div class="form-group">
-      <label for="hardcover_token">{{_('Hardcover API token')}}</label>
-      <input type="text" class="form-control" name="hardcover_token" id="hardcover_token" value="{{ content.hardcover_token if content.hardcover_token != None }}">
-    </div>
-    {% endif %}
-    {% if kobo_support and not new_user %}
-    <label>{{ _('Kobo Sync Token')}}</label>
-    <div class="form-group col">
-      <a class="btn btn-default" id="config_create_kobo_token" data-toggle="modal" data-target="#modal_kobo_token" data-remote="false" href="{{ url_for('kobo_auth.generate_auth_token', user_id=content.id) }}">{{_('Create/View')}}</a>
-      <div class="btn btn-danger" id="config_delete_kobo_token" data-value="{{ content.id }}" data-remote="false" {% if not content.remote_auth_token.first() %} style="display: none;" {% endif %}>{{_('Delete')}}</div>
-    </div>
-    <div class="form-group col">
-      <div class="btn btn-default" id="kobo_full_sync" data-value="{% if current_user.role_admin() %}{{ content.id }}{% else %}0{% endif %}" {% if not content.remote_auth_token.first() %} style="display: none;" {% endif %}>{{_('Force full kobo sync')}}</div>
-    </div>
-    {% endif %}
-    <div class="col-sm-6">
-    {% for element in sidebar %}
-      {% if element['config_show'] %}
-        <div class="form-group">
-          <input type="checkbox" name="show_{{element['visibility']}}" id="show_{{element['visibility']}}" {% if content.check_visibility(element['visibility']) %}checked{% endif %}>
-          <label for="show_{{element['visibility']}}">{{element['show_text']}}</label>
+    
+    <!-- OAuth & API Integrations Card -->
+    {% if (registered_oauth.keys()| length > 0 and not new_user and profile) or (hardcover_support and not new_user) or (kobo_support and not new_user) %}
+    <div class="settings-container" style="max-width: none; margin-inline: 0rem;">
+      <h4 class="settings-section-header">{{_('OAuth & API Integrations')}}</h4>
+      
+      {% if registered_oauth.keys()| length > 0 and not new_user and profile %}
+        {% for id, name in registered_oauth.items() %}
+        <div class="form-group" style="margin-bottom: 20px; display: flex; flex-direction: column; gap: 0.5rem;">
+          <label style="display: block; margin-bottom: 8px;">{{ name|capitalize }} {{_('OAuth Settings')}}</label>
+          {% if id not in oauth_status %}
+          <a href="{{ url_for('oauth.'+ name +'_login') }}" id="config_{{ id }}_oauth" class="btn btn-primary">{{_('Link')}} {{ name|capitalize }} {{_('Account')}}</a>
+          {% else %}
+          <a href="{{ url_for('oauth.'+ name +'_login_unlink') }}" id="config_{{ id }}_oauth" class="btn btn-primary">{{_('Unlink')}} {{ name|capitalize }} {{_('Account')}}</a>
+          {% endif %}
         </div>
+        {% endfor %}
       {% endif %}
-    {% endfor %}
+      
+      {% if hardcover_support and not new_user %}
       <div class="form-group">
-          <input type="checkbox" name="Show_detail_random" id="Show_detail_random" {% if content.show_detail_random() %}checked{% endif %}>
+        <label for="hardcover_token">{{_('Hardcover API Token')}}</label>
+        <input type="text" class="form-control" name="hardcover_token" id="hardcover_token" value="{{ content.hardcover_token if content.hardcover_token != None }}">
+        <p class="cwa-settings-tip">{{_('API token for Hardcover metadata provider integration.')}}</p>
+      </div>
+      {% endif %}
+      
+      {% if kobo_support and not new_user %}
+      <div style="margin-top: 25px;">
+        <label style="display: block; margin-bottom: 12px; font-weight: bold;">{{ _('Kobo Sync Token')}}</label>
+        <div style="margin-bottom: 10px;">
+          <a class="btn btn-default" id="config_create_kobo_token" data-toggle="modal" data-target="#modal_kobo_token" data-remote="false" href="{{ url_for('kobo_auth.generate_auth_token', user_id=content.id) }}">{{_('Create/View')}}</a>
+          <div class="btn btn-danger" id="config_delete_kobo_token" data-value="{{ content.id }}" data-remote="false" {% if not content.remote_auth_token.first() %} style="display: none;" {% endif %}>{{_('Delete')}}</div>
+        </div>
+        <div>
+          <div class="btn btn-default" id="kobo_full_sync" data-value="{% if current_user.role_admin() %}{{ content.id }}{% else %}0{% endif %}" {% if not content.remote_auth_token.first() %} style="display: none;" {% endif %}>{{_('Force full kobo sync')}}</div>
+        </div>
+        {% if kobo_support and not content.role_anonymous() and not simple %}
+        <div style="margin-top: 20px;
+                    background: #202c34a3;
+                    padding: 2rem;">
+          <input type="checkbox" name="kobo_only_shelves_sync" id="kobo_only_shelves_sync" {% if content.kobo_only_shelves_sync %}checked{% endif %} style="accent-color: var(--color-secondary);">
+          <label for="kobo_only_shelves_sync">{{_('Sync only books in selected shelves with Kobo')}}</label>
+        </div>
+        {% endif %}
+      </div>
+      {% endif %}
+    </div>
+    {% endif %}
+    
+    <!-- Sidebar Display Settings Card -->
+    <div class="settings-container" style="max-width: none; margin-inline: 0rem;">
+      <h4 class="settings-section-header">{{_('Sidebar Display Settings')}}</h4>
+      <p class="settings-explanation">{{_('Choose which sections to display in your sidebar navigation.')}}</p>
+      
+      <div style="display: grid;
+                  grid-template-columns: repeat(auto-fit, minmax(250px, 1fr));
+                  gap: 12px;
+                  margin-top: 20px;
+                  background: #202c34a3;
+                  padding: 2rem;">
+        {% for element in sidebar %}
+          {% if element['config_show'] %}
+          <div>
+            <input type="checkbox" name="show_{{element['visibility']}}" id="show_{{element['visibility']}}" {% if content.check_visibility(element['visibility']) %}checked{% endif %} style="accent-color: var(--color-secondary);">
+            <label for="show_{{element['visibility']}}">{{element['show_text']}}</label>
+          </div>
+          {% endif %}
+        {% endfor %}
+        <div>
+          <input type="checkbox" name="Show_detail_random" id="Show_detail_random" {% if content.show_detail_random() %}checked{% endif %} style="accent-color: var(--color-secondary);">
           <label for="Show_detail_random">{{_('Show Random Books in Detail View')}}</label>
+        </div>
       </div>
+      
       {% if ( current_user and current_user.role_admin() and not new_user ) and not simple %}
-      <a href="#" id="get_user_tags" class="btn btn-default" data-id="{{content.id}}" data-toggle="modal" data-target="#restrictModal">{{_('Add Allowed/Denied Tags')}}</a>
-      <a href="#" id="get_user_column_values" data-id="{{content.id}}" class="btn btn-default" data-toggle="modal" data-target="#restrictModal">{{_('Add allowed/Denied Custom Column Values')}}</a>
+      <div style="margin-top: 25px; padding-top: 20px; border-top: 1px solid #5c6b74;">
+        <a href="#" id="get_user_tags" class="btn btn-default" data-id="{{content.id}}" data-toggle="modal" data-target="#restrictModal">{{_('Add Allowed/Denied Tags')}}</a>
+        <a href="#" id="get_user_column_values" data-id="{{content.id}}" class="btn btn-default" data-toggle="modal" data-target="#restrictModal" style="margin-left: 10px;">{{_('Add allowed/Denied Custom Column Values')}}</a>
+      </div>
       {% endif %}
     </div>
-      <div class="col-sm-6">
+    
+    <!-- User Permissions Card (Admin Only) -->
     {% if current_user and current_user.role_admin() and not profile %}
-    {% if not content.role_anonymous() %}
-    <div class="form-group">
-      <input type="checkbox" name="admin_role" id="admin_role" {% if content.role_admin() %}checked{% endif %}>
-      <label for="admin_role">{{_('Admin User')}}</label>
-    </div>
-    {% endif %}
-    <div class="form-group">
-      <input type="checkbox" name="download_role" id="download_role" {% if content.role_download() %}checked{% endif %}>
-      <label for="download_role">{{_('Allow Downloads')}}</label>
-    </div>
-    <div class="form-group">
-      <input type="checkbox" name="viewer_role" id="viewer_role" {% if content.role_viewer() %}checked{% endif %}>
-      <label for="viewer_role">{{_('Allow eBook Viewer')}}</label>
-    </div>
-    {% if config.config_uploading %}
-    <div class="form-group">
-      <input type="checkbox" name="upload_role" id="upload_role" {% if content.role_upload() %}checked{% endif %}>
-      <label for="upload_role">{{_('Allow Uploads')}}</label>
-    </div>
-    {% endif %}
-    <div class="form-group">
-      <input type="checkbox" name="edit_role" data-control="edit_settings" id="edit_role" {% if content.role_edit() %}checked{% endif %}>
-      <label for="edit_role">{{_('Allow Edit')}}</label>
-    </div>
-    <div data-related="edit_settings">
-      <div class="form-group">
-        <input type="checkbox" name="delete_role" id="delete_role" {% if content.role_delete_books() %}checked{% endif %}>
-        <label for="delete_role">{{_('Allow Delete Books')}}</label>
+    <div class="settings-container" style="max-width: none; margin-inline: 0rem;">
+      <h4 class="settings-section-header">{{_('User Permissions')}}</h4>
+      <p class="settings-explanation">{{_('Configure what actions this user is allowed to perform.')}}</p>
+      
+      <div style="display: grid; grid-template-columns: repeat(auto-fit, minmax(250px, 1fr)); gap: 12px; margin-top: 20px;">
+        {% if not content.role_anonymous() %}
+        <div>
+          <input type="checkbox" name="admin_role" id="admin_role" {% if content.role_admin() %}checked{% endif %} style="accent-color: var(--color-secondary);">
+          <label for="admin_role">{{_('Admin User')}}</label>
+        </div>
+        {% endif %}
+        
+        <div>
+          <input type="checkbox" name="download_role" id="download_role" {% if content.role_download() %}checked{% endif %} style="accent-color: var(--color-secondary);">
+          <label for="download_role">{{_('Allow Downloads')}}</label>
+        </div>
+        
+        <div>
+          <input type="checkbox" name="viewer_role" id="viewer_role" {% if content.role_viewer() %}checked{% endif %} style="accent-color: var(--color-secondary);">
+          <label for="viewer_role">{{_('Allow eBook Viewer')}}</label>
+        </div>
+        
+        {% if config.config_uploading %}
+        <div>
+          <input type="checkbox" name="upload_role" id="upload_role" {% if content.role_upload() %}checked{% endif %} style="accent-color: var(--color-secondary);">
+          <label for="upload_role">{{_('Allow Uploads')}}</label>
+        </div>
+        {% endif %}
+        
+        <div>
+          <input type="checkbox" name="edit_role" data-control="edit_settings" id="edit_role" {% if content.role_edit() %}checked{% endif %} style="accent-color: var(--color-secondary);">
+          <label for="edit_role">{{_('Allow Edit')}}</label>
+        </div>
+        
+        <div data-related="edit_settings">
+          <input type="checkbox" name="delete_role" id="delete_role" {% if content.role_delete_books() %}checked{% endif %} style="accent-color: var(--color-secondary);">
+          <label for="delete_role">{{_('Allow Delete Books')}}</label>
+        </div>
+        
+        {% if not content.role_anonymous() %}
+        <div>
+          <input type="checkbox" name="passwd_role" id="passwd_role" {% if content.role_passwd() %}checked{% endif %} style="accent-color: var(--color-secondary);">
+          <label for="passwd_role">{{_('Allow Changing Password')}}</label>
+        </div>
+        
+        <div>
+          <input type="checkbox" name="edit_shelf_role" id="edit_shelf_role" {% if content.role_edit_shelfs() %}checked{% endif %} style="accent-color: var(--color-secondary);">
+          <label for="edit_shelf_role">{{_('Allow Editing Public Shelves')}}</label>
+        </div>
+        {% endif %}
       </div>
     </div>
-    {% if not content.role_anonymous() %}
-      <div class="form-group">
-        <input type="checkbox" name="passwd_role" id="passwd_role" {% if content.role_passwd() %}checked{% endif %}>
-        <label for="passwd_role">{{_('Allow Changing Password')}}</label>
-      </div>
-      <div class="form-group">
-        <input type="checkbox" name="edit_shelf_role" id="edit_shelf_role" {% if content.role_edit_shelfs() %}checked{% endif %}>
-        <label for="edit_shelf_role">{{_('Allow Editing Public Shelves')}}</label>
-      </div>
     {% endif %}
-  {% endif %}
-    {% if kobo_support and not content.role_anonymous() and not simple%}
-    <div class="form-group">
-      <input type="checkbox" name="kobo_only_shelves_sync" id="kobo_only_shelves_sync" {% if content.kobo_only_shelves_sync %}checked{% endif %}>
-      <label for="kobo_only_shelves_sync">{{_('Sync only books in selected shelves with Kobo')}}</label>
+    
+    <!-- Magic Shelves Visibility Card -->
+    {% if not content.role_anonymous() and not new_user %}
+    <div class="settings-container" style="max-width: none; margin-inline: 0rem;">
+      <h4 class="settings-section-header">{{_('Magic Shelves Visibility')}}</h4>
+      <p class="settings-explanation">{{_('Uncheck shelves you want to hide from your sidebar. System shelves cannot be deleted, only hidden. Public shelves from other users can also be hidden.')}}</p>
+      
+      {% if system_shelf_templates|length > 0 %}
+      <div style="margin-top: 20px; margin-bottom: 20px;">
+        <strong style="color: #ffc200;">{{_('Default System Shelves:')}}</strong>
+      </div>
+      <div style="display: grid;
+                  grid-template-columns: repeat(auto-fit, minmax(250px, 1fr));
+                  gap: 8px;
+                  margin-left: 10px;
+                  background: #202c34a3;
+                  padding: 2rem;">
+        {% for key, template in system_shelf_templates.items() %}
+        <div>
+          <input type="checkbox" 
+                name="show_magic_shelf_{{ key }}" 
+                id="show_magic_shelf_{{ key }}" 
+                {% if key not in hidden_shelf_templates %}checked{% endif %}
+                style="accent-color: var(--color-secondary);">
+          <label for="show_magic_shelf_{{ key }}">{{ template['icon'] }} {{ template['name'] }}</label>
+        </div>
+        {% endfor %}
+      </div>
+      {% endif %}
+      
+      {% set all_public_shelves = visible_public_shelves + hidden_custom_shelves %}
+      {% if all_public_shelves|length > 0 %}
+      <div style="margin-top: 30px; margin-bottom: 20px;">
+        <strong style="color: #ffc200;">{{_('Public Shelves:')}}</strong>
+      </div>
+      <div style="display: grid;
+                  grid-template-columns: repeat(auto-fit, minmax(250px, 1fr));
+                  gap: 8px;
+                  margin-left: 10px;
+                  background: #202c34a3;
+                  padding: 2rem;
+                  margin-bottom: 2rem;">
+        {% for shelf in all_public_shelves %}
+        <div>
+          <input type="checkbox" 
+                 name="show_custom_shelf_{{ shelf.id }}" 
+                 id="show_custom_shelf_{{ shelf.id }}"
+                 {% if shelf.id not in hidden_custom_shelf_ids %}checked{% endif %}
+                 style="accent-color: var(--color-secondary);">
+          <label for="show_custom_shelf_{{ shelf.id }}">{{ shelf.icon }} {{ shelf.name }} <small style="color: #888;">(by {{ shelf.user.name }})</small></label>
+        </div>
+        {% endfor %}
+      </div>
+      {% endif %}
+      
+      {% if system_shelf_templates|length > 0 or all_public_shelves|length > 0 %}
+      <div class="settings-disclaimer" style="margin-top: 25px !important; padding-top: 20px; border-top: 1px solid #5c6b74;">
+        {% if system_shelf_templates|length > 0 %}
+        {{_('%(visible)s of %(total)s default shelves visible', visible=(system_shelf_templates|length - hidden_shelf_templates|length), total=system_shelf_templates|length)}}
+        {% endif %}
+        {% if all_public_shelves|length > 0 %}
+        {% set visible_count = all_public_shelves|length - hidden_custom_shelf_ids|length %}
+        {% if system_shelf_templates|length > 0 %} • {% endif %}{{_('%(visible)s of %(total)s public shelves visible', visible=visible_count, total=all_public_shelves|length)}}
+        {% endif %}
+      </div>
+      {% endif %}
     </div>
     {% endif %}
-    </div>
-    <div class="col-sm-12">
-      <div id="user_submit" class="btn btn-default">{{_('Save')}}</div>
+    
+    <!-- Action Buttons -->
+    <div style="margin-bottom: 4rem; display: flex; flex-direction: column; gap: 0.5rem;">
+      <div id="user_submit" class="btn btn-default" style="width: -webkit-fill-available;">{{_('Save')}}</div>
       {% if not profile %}
-        <div class="btn btn-default" data-back="{{ url_for('admin.admin') }}" id="back">{{_('Cancel')}}</div>
+        <div class="btn btn-default" data-back="{{ url_for('admin.admin') }}" id="back" style="width: -webkit-fill-available;">{{_('Cancel')}}</div>
       {% endif %}
       {% if current_user and current_user.role_admin() and not profile and not new_user and not content.role_anonymous() %}
-        <div class="btn btn-danger" id="btndeluser" data-value="{{ content.id }}" data-remote="false" >{{_('Delete User')}}</div>
+        <div class="btn btn-danger" id="btndeluser" data-value="{{ content.id }}" data-remote="false" style="width: -webkit-fill-available;">{{_('Delete User')}}</div>
       {% endif %}
     </div>
-    </div>
   </form>
 </div>
 
@@ -198,6 +472,41 @@
 {{ delete_confirm_modal() }}
 {% endblock %}
 {% block js %}
+<script>
+  // Load custom profile picture if available
+  (function() {
+    const username = "{{ content.name }}";
+    const profilePictureElement = document.getElementById('user-profile-picture');
+    
+    if (username && profilePictureElement) {
+      // Fetch user profiles from the JSON endpoint
+      fetch('{{ url_for("profile_pictures.user_profiles_json") }}')
+        .then(response => {
+          if (!response.ok) {
+            throw new Error('Failed to load profile pictures');
+          }
+          return response.json();
+        })
+        .then(userProfiles => {
+          console.log('User profiles loaded:', userProfiles);
+          console.log('Looking for username:', username);
+          if (userProfiles[username]) {
+            // User has a custom profile picture
+            console.log('Setting custom profile picture:', userProfiles[username]);
+            profilePictureElement.style.backgroundImage = `url(${userProfiles[username]})`;
+          } else {
+            // No custom profile picture - keep default SVG
+            console.log('No custom profile picture found, using default');
+            profilePictureElement.classList.add('no-custom-picture');
+          }
+        })
+        .catch(error => {
+          console.error('Could not load user profile picture:', error);
+          profilePictureElement.classList.add('no-custom-picture');
+        });
+    }
+  })();
+</script>
 <script src="{{ url_for('static', filename='js/libs/bootstrap-table/bootstrap-table.min.js') }}"></script>
 <script src="{{ url_for('static', filename='js/libs/bootstrap-table/bootstrap-table-editable.min.js') }}"></script>
 <script src="{{ url_for('static', filename='js/libs/bootstrap-table/bootstrap-editable.min.js') }}"></script>

cps/translations/de/LC_MESSAGES/messages.po

@@ -1623,7 +1623,7 @@ msgstr ""
 
 #: cps/shelf.py:207 cps/templates/layout.html:270
 msgid "Create a Shelf"
-msgstr "Bücherregal anlegen"
+msgstr "Neues Bücherregal"
 
 #: cps/shelf.py:215
 msgid "Sorry you are not allowed to edit this shelf"

cps/ub.py

@@ -29,7 +29,7 @@ except ImportError as e:
         OAuthConsumerMixin = BaseException
         oauth_support = False
 from sqlalchemy import create_engine, exc, exists, event, text
-from sqlalchemy import Column, ForeignKey, Index
+from sqlalchemy import Column, ForeignKey, Index, UniqueConstraint
 from sqlalchemy import String, Integer, SmallInteger, Boolean, DateTime, Float, JSON
 from sqlalchemy.orm.attributes import flag_modified
 from sqlalchemy.sql.expression import func
@@ -402,6 +402,10 @@ class MagicShelf(Base):
     created = Column(DateTime, default=lambda: datetime.now(timezone.utc))
     last_modified = Column(DateTime, default=lambda: datetime.now(timezone.utc), onupdate=lambda: datetime.now(timezone.utc))
 
+    __table_args__ = (
+        UniqueConstraint('user_id', 'name', 'is_system', name='unique_user_system_shelf_name'),
+    )
+
     def __repr__(self):
         return '<MagicShelf %d:%r>' % (self.id, self.name)
 
@@ -423,6 +427,46 @@ class MagicShelfCache(Base):
     )
 
 
+class HiddenMagicShelfTemplate(Base):
+    __tablename__ = 'hidden_magic_shelf_templates'
+
+    id = Column(Integer, primary_key=True)
+    user_id = Column(Integer, ForeignKey('user.id'), nullable=False)
+    template_key = Column(String, nullable=True)  # For system templates: 'recently_added', 'highly_rated', etc.
+    shelf_id = Column(Integer, ForeignKey('magic_shelf.id'), nullable=True)  # For custom public shelves
+    hidden_at = Column(DateTime, default=lambda: datetime.now(timezone.utc))
+
+    __table_args__ = (
+        # Either template_key OR shelf_id must be set, but not both
+        # User can only hide the same template/shelf once
+        UniqueConstraint('user_id', 'template_key', name='unique_user_template_hidden'),
+        UniqueConstraint('user_id', 'shelf_id', name='unique_user_shelf_hidden'),
+    )
+
+    def __repr__(self):
+        if self.template_key:
+            return '<HiddenMagicShelfTemplate %d: user=%d template=%s>' % (self.id, self.user_id, self.template_key)
+        else:
+            return '<HiddenMagicShelfTemplate %d: user=%d shelf_id=%d>' % (self.id, self.user_id, self.shelf_id)
+
+
+class DismissedDuplicateGroup(Base):
+    __tablename__ = 'dismissed_duplicate_groups'
+
+    id = Column(Integer, primary_key=True)
+    user_id = Column(Integer, ForeignKey('user.id'), nullable=False)
+    group_hash = Column(String(32), nullable=False)  # MD5 hash of title+author combo
+    dismissed_at = Column(DateTime, default=lambda: datetime.now(timezone.utc))
+
+    __table_args__ = (
+        # User can only dismiss the same duplicate group once
+        UniqueConstraint('user_id', 'group_hash', name='unique_user_duplicate_dismissed'),
+    )
+
+    def __repr__(self):
+        return '<DismissedDuplicateGroup %d: user=%d hash=%s>' % (self.id, self.user_id, self.group_hash)
+
+
 # Baseclass representing Relationship between books and Shelfs in Calibre-Web in app.db (N:M)
 class BookShelf(Base):
     __tablename__ = 'book_shelf_link'
@@ -706,13 +750,17 @@ class Thumbnail(Base):
 # Add missing tables during migration of database
 def add_missing_tables(engine, _session):
     if not engine.dialect.has_table(engine.connect(), "archived_book"):
-        ArchivedBook.__table__.create(bind=engine)
+        ArchivedBook.__table__.create(bind=engine, checkfirst=True)
     if not engine.dialect.has_table(engine.connect(), "thumbnail"):
-        Thumbnail.__table__.create(bind=engine)
+        Thumbnail.__table__.create(bind=engine, checkfirst=True)
     if not engine.dialect.has_table(engine.connect(), "kosync_progress"):
-        KOSyncProgress.__table__.create(bind=engine)
+        KOSyncProgress.__table__.create(bind=engine, checkfirst=True)
     if not engine.dialect.has_table(engine.connect(), "magic_shelf"):
-        MagicShelf.__table__.create(bind=engine)
+        MagicShelf.__table__.create(bind=engine, checkfirst=True)
+    if not engine.dialect.has_table(engine.connect(), "magic_shelf_cache"):
+        MagicShelfCache.__table__.create(bind=engine, checkfirst=True)
+    if not engine.dialect.has_table(engine.connect(), "hidden_magic_shelf_templates"):
+        HiddenMagicShelfTemplate.__table__.create(bind=engine, checkfirst=True)
 
 
 # migrate all settings missing in registration table
@@ -931,27 +979,70 @@ def migrate_Database(_session):
     from .progress_syncing.models import ensure_app_db_tables
     ensure_app_db_tables(engine.raw_connection())
     
-    # Backfill system magic shelves for existing users (one-time migration)
+    # Migrate system magic shelves for existing users
     try:
         from . import magic_shelf
+        
+        # Get all valid current template names
+        current_template_names = {template['name'] for template in magic_shelf.SYSTEM_SHELF_TEMPLATES.values()}
+        
+        log.info("Migrating system magic shelves...")
         users = _session.query(User).filter(User.role != constants.ROLE_ANONYMOUS).all()
-        backfilled = 0
+        total_deleted = 0
+        total_created = 0
+        
         for user in users:
-            # Check if user has any system shelves
-            has_system_shelves = _session.query(MagicShelf).filter(
+            # Get all system shelves for this user
+            user_system_shelves = _session.query(MagicShelf).filter(
                 MagicShelf.user_id == user.id,
                 MagicShelf.is_system == True
-            ).first()
-
-            if not has_system_shelves:
-                created = magic_shelf.create_system_magic_shelves(user.id)
-                if created > 0:
-                    backfilled += 1
-
-        if backfilled > 0:
-            log.info(f"Backfilled system magic shelves for {backfilled} existing users")
+            ).all()
+            
+            # Delete system shelves that don't match current templates
+            for shelf in user_system_shelves:
+                if shelf.name not in current_template_names:
+                    # This is an old/deprecated system shelf - delete it
+                    _session.query(MagicShelfCache).filter_by(shelf_id=shelf.id).delete()
+                    _session.query(HiddenMagicShelfTemplate).filter_by(shelf_id=shelf.id).delete()
+                    _session.delete(shelf)
+                    total_deleted += 1
+                    log.debug(f"Deleted deprecated system shelf '{shelf.name}' (ID: {shelf.id}) for user {user.id}")
+            
+            # Get user's template-based hide preferences (not shelf-specific)
+            hidden_templates = _session.query(HiddenMagicShelfTemplate.template_key).filter(
+                HiddenMagicShelfTemplate.user_id == user.id,
+                HiddenMagicShelfTemplate.template_key.isnot(None)
+            ).all()
+            hidden_keys = {ht.template_key for ht in hidden_templates}
+            
+            # Create missing current templates
+            templates_to_create = []
+            for template_key, template_data in magic_shelf.SYSTEM_SHELF_TEMPLATES.items():
+                # Skip if user has hidden this template type
+                if template_key in hidden_keys:
+                    continue
+                
+                # Check if user already has this current template
+                has_template = _session.query(MagicShelf).filter(
+                    MagicShelf.user_id == user.id,
+                    MagicShelf.name == template_data['name'],
+                    MagicShelf.is_system == True
+                ).first()
+                
+                if not has_template:
+                    templates_to_create.append(template_key)
+            
+            # Create missing templates
+            if templates_to_create:
+                created = magic_shelf.create_system_magic_shelves(user.id, templates_to_create)
+                total_created += created
+        
+        if total_deleted > 0 or total_created > 0:
+            _session.commit()
+            log.info(f"System shelf migration complete: {total_deleted} old shelves removed, {total_created} new shelves created")
     except Exception as e:
-        log.error(f"Error backfilling system magic shelves: {e}")
+        log.error(f"Error during system shelf migration: {e}")
+        _session.rollback()
 
 
 def clean_database(_session):

cps/web.py

@@ -884,9 +884,9 @@ def render_magic_shelf(shelf_id, sort_param, page):
         log.warning(f"Magic shelf {shelf_id} not found")
         abort(404)
     
-    # Check ownership - users can only view their own shelves (for now)
-    if shelf.user_id != current_user.id:
-        log.warning(f"User {current_user.id} attempted to access magic shelf {shelf_id} owned by {shelf.user_id}")
+    # Check access - users can view their own shelves OR public shelves
+    if shelf.user_id != current_user.id and shelf.is_public != 1:
+        log.warning(f"User {current_user.id} attempted to access private magic shelf {shelf_id} owned by {shelf.user_id}")
         abort(403)
     
     # Get sort order using the same function as other book lists
@@ -944,11 +944,21 @@ def render_magic_shelf(shelf_id, sort_param, page):
     
     entries = [Entry(book) for book in books]
     
+    # Check if this shelf is hidden by current user (for public shelves)
+    is_hidden = False
+    if shelf.user_id != current_user.id:
+        is_hidden = ub.session.query(ub.HiddenMagicShelfTemplate).filter(
+            ub.HiddenMagicShelfTemplate.user_id == current_user.id,
+            ub.HiddenMagicShelfTemplate.shelf_id == shelf_id
+        ).first() is not None
+    
     return render_title_template('index.html', 
                                  entries=entries, 
                                  pagination=pagination,
                                  title=_("Magic Shelf&nbsp&nbsp&nbsp—&nbsp&nbsp&nbsp%(icon)s %(name)s", icon=shelf.icon, name=shelf.name), 
-                                 page="magicshelf", 
+                                 page="magicshelf",
+                                 shelf=shelf,
+                                 is_hidden_shelf=is_hidden,
                                  id=shelf_id, 
                                  order=order[1])
 
@@ -1045,7 +1055,7 @@ def preview_magic_shelf():
             })
             
         except Exception as e:
-            log.error(f"Error previewing magic shelf rules: {e}")
+            log.error(f"Error previewing magic shelf rules: {str(e)}", exc_info=True)
             return jsonify({"success": False, "message": _("Error processing rules")}), 500
             
     except Exception as e:
@@ -1086,6 +1096,11 @@ def create_magic_shelf():
         rules = data.get('rules')
         icon = data.get('icon', '🪄')
         kobo_sync = data.get('kobo_sync', False)
+        is_public = data.get('is_public', False)
+        
+        # Only allow public if user has permission
+        if is_public and not current_user.role_edit_shelfs():
+            is_public = False
         
         # Validate inputs
         if not name or not rules:
@@ -1104,7 +1119,8 @@ def create_magic_shelf():
                 user_id=current_user.id,
                 rules=rules,
                 icon=icon,
-                kobo_sync=kobo_sync
+                kobo_sync=kobo_sync,
+                is_public=1 if is_public else 0
             )
             ub.session.add(new_shelf)
             ub.session_commit()
@@ -1165,8 +1181,9 @@ def edit_magic_shelf(shelf_id):
         log.warning(f"Magic shelf {shelf_id} not found")
         abort(404)
     
-    if shelf.user_id != current_user.id:
-        log.warning(f"User {current_user.id} attempted to edit magic shelf {shelf_id} owned by {shelf.user_id}")
+    # Check if user can edit this shelf (owner or admin only)
+    if shelf.user_id != current_user.id and not current_user.role_admin():
+        log.warning(f"User {current_user.id} attempted to edit magic shelf {shelf_id} without permission")
         abort(403)
 
     if request.method == "POST":
@@ -1175,6 +1192,12 @@ def edit_magic_shelf(shelf_id):
         rules = data.get('rules', shelf.rules)
         icon = data.get('icon', shelf.icon)
         kobo_sync = data.get('kobo_sync', shelf.kobo_sync)
+        is_public = data.get('is_public', shelf.is_public == 1)
+        
+        # Only allow changing public status if user has permission
+        if is_public != (shelf.is_public == 1):
+            if not current_user.role_edit_shelfs():
+                return jsonify({"success": False, "message": _("Permission denied to change public status")}), 403
         
         # Validate inputs
         if not name:
@@ -1192,6 +1215,7 @@ def edit_magic_shelf(shelf_id):
             shelf.rules = rules
             shelf.icon = icon
             shelf.kobo_sync = kobo_sync
+            shelf.is_public = 1 if is_public else 0
             flag_modified(shelf, "rules")
             
             # Invalidate Complex Query Cache
@@ -1292,12 +1316,32 @@ def delete_magic_shelf(shelf_id):
         log.warning(f"Magic shelf {shelf_id} not found for deletion")
         abort(404)
     
-    if shelf.user_id != current_user.id:
-        log.warning(f"User {current_user.id} attempted to delete magic shelf {shelf_id} owned by {shelf.user_id}")
+    # Check if user can delete this shelf
+    can_delete = False
+    if shelf.user_id == current_user.id:
+        can_delete = True
+    elif shelf.is_public == 1 and current_user.role_edit_shelfs():
+        can_delete = True
+    
+    if not can_delete:
+        log.warning(f"User {current_user.id} attempted to delete magic shelf {shelf_id} without permission")
         abort(403)
     
+    # Prevent deletion of system shelves
+    if shelf.is_system:
+        log.warning(f"User {current_user.id} attempted to delete system shelf {shelf_id}")
+        return jsonify({
+            "success": False, 
+            "message": _("System shelves cannot be deleted. You can hide them in your user profile settings.")
+        }), 400
+    
     try:
         shelf_name = shelf.name
+        # Delete cache entries first
+        ub.session.query(ub.MagicShelfCache).filter_by(shelf_id=shelf_id).delete()
+        # Delete any hide records for this shelf
+        ub.session.query(ub.HiddenMagicShelfTemplate).filter_by(shelf_id=shelf_id).delete()
+        # Delete the shelf
         ub.session.delete(shelf)
         ub.session_commit()
         log.info(f"User {current_user.id} deleted magic shelf {shelf_id} ('{shelf_name}')")
@@ -1308,6 +1352,69 @@ def delete_magic_shelf(shelf_id):
         return jsonify({"success": False, "message": _("Error deleting shelf")}), 500
 
 
+@web.route("/magicshelf/<int:shelf_id>/hide", methods=["POST"])
+@user_login_required
+def hide_magic_shelf(shelf_id):
+    """Hide a public magic shelf (user doesn't want to see it in their sidebar)."""
+    shelf = ub.session.query(ub.MagicShelf).get(shelf_id)
+    if not shelf:
+        log.warning(f"Magic shelf {shelf_id} not found")
+        abort(404)
+    
+    # Can only hide shelves you don't own (public or system)
+    if shelf.user_id == current_user.id:
+        return jsonify({
+            "success": False, 
+            "message": _("You cannot hide your own shelves. Delete them instead if you don't want them.")
+        }), 400
+    
+    # Check if already hidden
+    existing = ub.session.query(ub.HiddenMagicShelfTemplate).filter(
+        ub.HiddenMagicShelfTemplate.user_id == current_user.id,
+        ub.HiddenMagicShelfTemplate.shelf_id == shelf_id
+    ).first()
+    
+    if existing:
+        return jsonify({"success": True, "message": _("Shelf already hidden")})
+    
+    try:
+        hidden = ub.HiddenMagicShelfTemplate(
+            user_id=current_user.id,
+            shelf_id=shelf_id
+        )
+        ub.session.add(hidden)
+        ub.session_commit()
+        log.info(f"User {current_user.id} hid magic shelf {shelf_id} ('{shelf.name}')")
+        return jsonify({"success": True})
+    except Exception as e:
+        log.error(f"Error hiding magic shelf {shelf_id}: {e}")
+        ub.session.rollback()
+        return jsonify({"success": False, "message": _("Error hiding shelf")}), 500
+
+
+@web.route("/magicshelf/<int:shelf_id>/unhide", methods=["POST"])
+@user_login_required
+def unhide_magic_shelf(shelf_id):
+    """Unhide a previously hidden magic shelf."""
+    try:
+        hidden = ub.session.query(ub.HiddenMagicShelfTemplate).filter(
+            ub.HiddenMagicShelfTemplate.user_id == current_user.id,
+            ub.HiddenMagicShelfTemplate.shelf_id == shelf_id
+        ).first()
+        
+        if not hidden:
+            return jsonify({"success": True, "message": _("Shelf was not hidden")})
+        
+        ub.session.delete(hidden)
+        ub.session_commit()
+        log.info(f"User {current_user.id} unhid magic shelf {shelf_id}")
+        return jsonify({"success": True})
+    except Exception as e:
+        log.error(f"Error unhiding magic shelf {shelf_id}: {e}")
+        ub.session.rollback()
+        return jsonify({"success": False, "message": _("Error unhiding shelf")}), 500
+
+
 @web.route("/table")
 @user_login_required
 def books_table():
@@ -2215,6 +2322,65 @@ def change_profile(kobo_support, hardcover_support, local_oauth_check, oauth_sta
         # Auto-send and metadata fetch settings
         current_user.auto_send_enabled = to_save.get("auto_send_enabled") == "on"
         current_user.auto_metadata_fetch = to_save.get("auto_metadata_fetch") == "on"
+        
+        # Handle hidden magic shelf templates and custom shelves
+        from . import magic_shelf
+        if not current_user.is_anonymous:
+            # Get all system template keys
+            all_template_keys = set(magic_shelf.SYSTEM_SHELF_TEMPLATES.keys())
+            # Get currently hidden items for this user
+            current_hidden = ub.session.query(ub.HiddenMagicShelfTemplate).filter(
+                ub.HiddenMagicShelfTemplate.user_id == current_user.id
+            ).all()
+            current_hidden_template_keys = {h.template_key for h in current_hidden if h.template_key}
+            current_hidden_shelf_ids = {h.shelf_id for h in current_hidden if h.shelf_id}
+            
+            # Handle system templates
+            visible_template_keys = {key for key in all_template_keys if to_save.get(f"show_magic_shelf_{key}") == "on"}
+            should_be_hidden_templates = all_template_keys - visible_template_keys
+            
+            # Add newly hidden templates
+            for key in should_be_hidden_templates:
+                if key not in current_hidden_template_keys:
+                    new_hidden = ub.HiddenMagicShelfTemplate(
+                        user_id=current_user.id,
+                        template_key=key
+                    )
+                    ub.session.add(new_hidden)
+                    log.info(f"User {current_user.id} hid system shelf template '{key}'")
+            
+            # Remove templates that should no longer be hidden
+            for hidden in current_hidden:
+                if hidden.template_key and hidden.template_key in visible_template_keys:
+                    ub.session.delete(hidden)
+                    log.info(f"User {current_user.id} unhid system shelf template '{hidden.template_key}'")
+            
+            # Handle custom public shelves - get all available ones
+            all_public_shelves = ub.session.query(ub.MagicShelf).filter(
+                ub.MagicShelf.is_public == 1,
+                ub.MagicShelf.user_id != current_user.id,
+                ub.MagicShelf.is_system == False
+            ).all()
+            
+            # Check which ones should be visible (checked)
+            visible_shelf_ids = {s.id for s in all_public_shelves if to_save.get(f"show_custom_shelf_{s.id}") == "on"}
+            
+            # Hide shelves that are unchecked but not currently hidden
+            for shelf in all_public_shelves:
+                if shelf.id not in visible_shelf_ids and shelf.id not in current_hidden_shelf_ids:
+                    new_hidden = ub.HiddenMagicShelfTemplate(
+                        user_id=current_user.id,
+                        shelf_id=shelf.id
+                    )
+                    ub.session.add(new_hidden)
+                    log.info(f"User {current_user.id} hid custom shelf {shelf.id}")
+            
+            # Unhide shelves that are checked but currently hidden
+            for hidden in current_hidden:
+                if hidden.shelf_id and hidden.shelf_id in visible_shelf_ids:
+                    ub.session.delete(hidden)
+                    log.info(f"User {current_user.id} unhid custom shelf {hidden.shelf_id}")
+        
         # Theme change
         if 'theme' in to_save:
             try:
@@ -2232,7 +2398,7 @@ def change_profile(kobo_support, hardcover_support, local_oauth_check, oauth_sta
                                      translations=translations,
                                      profile=1,
                                      languages=languages,
-                                     title=_("%(name)s's Profile", name=current_user.name),
+                                     title=_(f"{current_user.name.capitalize()}'s Profile", name=current_user.name),
                                      page="me",
                                      kobo_support=kobo_support,
                                      hardcover_support=hardcover_support,
@@ -2241,8 +2407,12 @@ def change_profile(kobo_support, hardcover_support, local_oauth_check, oauth_sta
 
     val = 0
     for key, __ in to_save.items():
-        if key.startswith('show'):
-            val += int(key[5:])
+        if key.startswith('show') and not key.startswith('show_magic_shelf_') and not key.startswith('show_custom_shelf_'):
+            try:
+                val += int(key[5:])
+            except (ValueError, IndexError) as e:
+                log.warning(f"Skipping invalid sidebar checkbox key: {key}")
+                continue
     current_user.sidebar_view = val
     if to_save.get("Show_detail_random"):
         current_user.sidebar_view += constants.DETAIL_RANDOM
@@ -2251,6 +2421,8 @@ def change_profile(kobo_support, hardcover_support, local_oauth_check, oauth_sta
         ub.session.commit()
         flash(_("Success! Profile Updated"), category="success")
         log.debug("Profile updated")
+        # Redirect to refresh sidebar with updated shelf visibility
+        return redirect(url_for('web.profile'))
     except IntegrityError:
         ub.session.rollback()
         flash(_("Oops! An account already exists for this Email."), category="error")
@@ -2274,9 +2446,33 @@ def profile():
     else:
         oauth_status = None
         local_oauth_check = {}
-
+    
     if request.method == "POST":
-        change_profile(kobo_support, hardcover_support, local_oauth_check, oauth_status, translations, languages)
+        return change_profile(kobo_support, hardcover_support, local_oauth_check, oauth_status, translations, languages)
+    
+    # Query magic shelf data after POST to get updated values
+    from . import magic_shelf
+    system_shelf_templates = magic_shelf.SYSTEM_SHELF_TEMPLATES
+    hidden_items = ub.session.query(
+        ub.HiddenMagicShelfTemplate.template_key,
+        ub.HiddenMagicShelfTemplate.shelf_id
+    ).filter(
+        ub.HiddenMagicShelfTemplate.user_id == current_user.id
+    ).all()
+    hidden_shelf_templates = {item.template_key for item in hidden_items if item.template_key}
+    hidden_custom_shelf_ids = {item.shelf_id for item in hidden_items if item.shelf_id}
+    
+    # Get ALL public custom shelves that user doesn't own (both hidden and visible)
+    all_public_shelves = ub.session.query(ub.MagicShelf).filter(
+        ub.MagicShelf.is_public == 1,
+        ub.MagicShelf.user_id != current_user.id,
+        ub.MagicShelf.is_system == False
+    ).all()
+    
+    # Separate into hidden and visible
+    hidden_custom_shelves = [s for s in all_public_shelves if s.id in hidden_custom_shelf_ids]
+    visible_public_shelves = [s for s in all_public_shelves if s.id not in hidden_custom_shelf_ids]
+
     return render_title_template("user_edit.html",
                                  translations=translations,
                                  profile=1,
@@ -2285,7 +2481,12 @@ def profile():
                                  config=config,
                                  kobo_support=kobo_support,
                                  hardcover_support=hardcover_support,
-                                 title=_("%(name)s's Profile", name=current_user.name),
+                                 system_shelf_templates=system_shelf_templates,
+                                 hidden_shelf_templates=hidden_shelf_templates,
+                                 hidden_custom_shelf_ids=hidden_custom_shelf_ids,
+                                 hidden_custom_shelves=hidden_custom_shelves,
+                                 visible_public_shelves=visible_public_shelves,
+                                 title=_(f"{current_user.name.capitalize()}'s Profile", name=current_user.name),
                                  page="me",
                                  registered_oauth=local_oauth_check,
                                  oauth_status=oauth_status)

root/etc/s6-overlay/s6-rc.d/cwa-init/run

@@ -90,6 +90,7 @@ install -d -o abc -g abc /config/processed_books/converted
 install -d -o abc -g abc /config/processed_books/imported
 install -d -o abc -g abc /config/processed_books/failed
 install -d -o abc -g abc /config/processed_books/fixed_originals
+install -d -o abc -g abc /config/processed_books/duplicate_resolutions
 install -d -o abc -g abc /config/log_archive
 install -d -o abc -g abc /config/.cwa_conversion_tmp
 

scripts/cover_enforcer.py

@@ -292,30 +292,69 @@ class Enforcer:
 
 
     def read_log(self, auto=True, log_path: str = "None") -> dict:
-        """Reads pertinent information from the given log file, adds the book_id from the log name and returns the info as a dict"""
+        """Reads pertinent information from the given log file, adds the book_id from the log name and returns the info as a dict.
+        Returns None if the file doesn't exist after retries (handles race conditions)."""
         if auto:
+            file_path = f'{change_logs_dir}/{self.args.log}'
             book_id = (self.args.log.split('-')[1]).split('.')[0]
             timestamp_raw = self.args.log.split('-')[0]
-            timestamp = datetime.strptime(timestamp_raw, '%Y%m%d%H%M%S')
-
-            log_info = {}
-            with open(f'{change_logs_dir}/{self.args.log}', 'r', encoding='utf-8') as f:
-                log_info = json.load(f)
-            log_info['book_id'] = book_id
-            log_info['timestamp'] = timestamp.strftime('%Y-%m-%d %H:%M:%S')
         else:
+            file_path = log_path
             log_name = os.path.basename(log_path)
             book_id = (log_name.split('-')[1]).split('.')[0]
             timestamp_raw = log_name.split('-')[0]
+        
+        try:
             timestamp = datetime.strptime(timestamp_raw, '%Y%m%d%H%M%S')
+        except ValueError as e:
+            print(f"[cover-metadata-enforcer] ERROR: Invalid timestamp format in log filename: {e}", flush=True)
+            return None
 
-            log_info = {}
-            with open(log_path, 'r', encoding='utf-8') as f:
-                log_info = json.load(f)
-            log_info['book_id'] = book_id
-            log_info['timestamp'] = timestamp.strftime('%Y-%m-%d %H:%M:%S')
-
-        return log_info
+        # Retry logic to handle race conditions where file is detected but not yet fully written
+        max_retries = 3
+        retry_delay = 0.5  # seconds
+        
+        for attempt in range(max_retries):
+            try:
+                # Check if file exists first
+                if not os.path.exists(file_path):
+                    if attempt < max_retries - 1:
+                        time.sleep(retry_delay)
+                        continue
+                    else:
+                        print(f"[cover-metadata-enforcer] WARNING: Log file '{os.path.basename(file_path)}' not found after {max_retries} attempts. "
+                              f"This may be due to a race condition or the file was already processed and deleted.", flush=True)
+                        return None
+                
+                # Try to read the file
+                with open(file_path, 'r', encoding='utf-8') as f:
+                    log_info = json.load(f)
+                
+                log_info['book_id'] = book_id
+                log_info['timestamp'] = timestamp.strftime('%Y-%m-%d %H:%M:%S')
+                return log_info
+                
+            except FileNotFoundError:
+                if attempt < max_retries - 1:
+                    time.sleep(retry_delay)
+                    continue
+                else:
+                    print(f"[cover-metadata-enforcer] WARNING: Log file '{os.path.basename(file_path)}' not found after {max_retries} attempts. "
+                          f"This may be due to a race condition or the file was already processed and deleted.", flush=True)
+                    return None
+            except json.JSONDecodeError as e:
+                if attempt < max_retries - 1:
+                    # File might still be being written
+                    time.sleep(retry_delay)
+                    continue
+                else:
+                    print(f"[cover-metadata-enforcer] ERROR: Failed to parse log file '{os.path.basename(file_path)}': {e}", flush=True)
+                    return None
+            except Exception as e:
+                print(f"[cover-metadata-enforcer] ERROR: Unexpected error reading log file '{os.path.basename(file_path)}': {e}", flush=True)
+                return None
+        
+        return None
 
 
     def get_book_dir_from_log(self, log_info: dict) -> str:
@@ -554,6 +593,9 @@ class Enforcer:
             for log in log_files:
                 if log.endswith('.json'):
                     log_info = self.read_log(auto=False, log_path=log)
+                    # Skip if log_info is None (file was deleted or invalid)
+                    if log_info is None:
+                        continue
                     book_dir = self.get_book_dir_from_log(log_info)
                     book_objects = self.enforce_cover(book_dir)
                     if book_objects:
@@ -626,6 +668,12 @@ def main():
     elif args.log is not None and args.dir is None and args.all is False and args.list is False and args.history is False:
         ### log passed: (args.log), no dir
         log_info = enforcer.read_log()
+        
+        # Handle case where log file doesn't exist (race condition)
+        if log_info is None:
+            print(f"[cover-metadata-enforcer] Skipping processing due to missing or invalid log file. This is normal if the file was already processed.")
+            sys.exit(0)
+        
         book_dir = enforcer.get_book_dir_from_log(log_info)
         if enforcer.enforcer_on:
             book_objects = enforcer.enforce_cover(book_dir)

scripts/cwa_db.py

@@ -25,7 +25,7 @@ class CWA_DB:
         # Support both Docker and CI environments for schema path
         script_dir = os.path.dirname(os.path.abspath(__file__))
         self.schema_path = os.path.join(script_dir, "cwa_schema.sql")
-        self.stats_tables = ["cwa_enforcement", "cwa_import", "cwa_conversions", "epub_fixes", "cwa_user_activity"]
+        self.stats_tables = ["cwa_enforcement", "cwa_import", "cwa_conversions", "epub_fixes", "cwa_user_activity", "cwa_duplicate_cache", "cwa_duplicate_resolutions"]
         self.tables, self.schema = self.make_tables()
 
         self.cwa_default_settings = self.get_cwa_default_settings()
@@ -79,8 +79,10 @@ class CWA_DB:
 
         settings_lines = []
         for line in settings_table.split('\n'):
-            if line[:4] == "    ":
-                settings_lines.append(line.strip())
+            stripped = line.strip()
+            # Skip comment lines and empty lines
+            if line[:4] == "    " and not stripped.startswith('--') and stripped:
+                settings_lines.append(stripped)
 
         default_settings = {}
         for line in settings_lines:
@@ -104,11 +106,18 @@ class CWA_DB:
         # print(f"[cwa-db] DEBUG: Current available cwa_settings: {cwa_setting_names}")
 
         # Add any settings present in the schema file but not in the db
+        newly_added_settings = []
         for setting in self.cwa_default_settings.keys():
             if setting not in cwa_setting_names:
                 success = self.add_missing_setting(setting)
                 if success:
                     print(f"[cwa-db] Setting '{setting}' successfully added to cwa.db!")
+                    newly_added_settings.append(setting)
+        
+        # Sync newly added settings with schema defaults
+        # This handles cases where schema default was updated after column was added
+        if newly_added_settings:
+            self.sync_new_settings_with_defaults(newly_added_settings)
         
         # Delete any settings in the db but not in the schema file
         for setting in cwa_setting_names:
@@ -120,6 +129,38 @@ class CWA_DB:
                     print(f"[cwa-db] Deprecated setting '{setting}' successfully removed from cwa.db!")
                 except Exception as e:
                     print(f"[cwa-db] The following error occurred when trying to remove {setting} from cwa.db:\n{e}")
+    
+    
+    def sync_new_settings_with_defaults(self, newly_added_settings) -> None:
+        """Sync newly added settings to match schema defaults
+        
+        This ensures that if a column was added with one default value, then the schema 
+        was updated with a different default, existing databases get the new default.
+        """
+        try:
+            # Get current values
+            self.cur.execute("SELECT * FROM cwa_settings")
+            headers = [header[0] for header in self.cur.description]
+            current_values = dict(zip(headers, self.cur.fetchone()))
+            
+            # Update any newly added settings that don't match schema defaults
+            updates_made = []
+            for setting in newly_added_settings:
+                current_val = current_values.get(setting)
+                expected_val = self.cwa_default_settings.get(setting)
+                
+                # Compare with type handling (int vs string)
+                if str(current_val) != str(expected_val):
+                    self.cur.execute(f"UPDATE cwa_settings SET {setting}=?", (expected_val,))
+                    updates_made.append(f"{setting}: {current_val} -> {expected_val}")
+            
+            if updates_made:
+                self.con.commit()
+                print(f"[cwa-db] Synced {len(updates_made)} new setting(s) with schema defaults:")
+                for update in updates_made:
+                    print(f"[cwa-db]   - {update}")
+        except Exception as e:
+            print(f"[cwa-db] Warning: Failed to sync new settings with defaults: {e}")
 
 
     def add_missing_setting(self, setting) -> bool:
@@ -128,6 +169,9 @@ class CWA_DB:
             if match:
                 try:
                     command = line.replace('\n', '').strip()
+                    # Skip SQL comments
+                    if command.startswith('--') or not command:
+                        continue
                     command = command.replace(',', ';')
                     with open('/config/.cwa_db_debug', 'a') as f:
                         f.write(command)
@@ -158,19 +202,27 @@ class CWA_DB:
         column_names_in_schema = {}
         for table in self.tables:
             column_names = []
+            table_name = None  # Reset for each table
             table = table.split('\n')
             for line in table:
                 if line[:27] == "CREATE TABLE IF NOT EXISTS ":
                     table_name = line[27:].replace('(', '').strip()
                 elif line[:4] == "    ":
                     column_names.append(line.strip().split(' ')[0])
-            if 'table_name' in locals():
-                column_names_in_schema |= {table_name:column_names} # type: ignore
+            if table_name is not None:  # Only add if table_name was actually found
+                column_names_in_schema[table_name] = column_names
 
         for table in self.stats_tables:
             # Skip if table wasn't found in current DB (it was just created empty)
             if not current_column_names[table]:
                 continue
+            
+            # Skip if table not found in schema (shouldn't happen but safety check)
+            if table not in column_names_in_schema:
+                print(f"[cwa-db] Warning: Table '{table}' in stats_tables but not found in schema")
+                continue
+            
+            columns_added = False  # Track if we added any columns this iteration
                 
             if len(current_column_names[table]) < len(column_names_in_schema[table]): # Adds new columns not yet in existing db
                 num_new_columns = len(column_names_in_schema[table]) - len(current_column_names[table])
@@ -179,11 +231,29 @@ class CWA_DB:
                         for line in self.schema:
                             matches = re.findall(column_names_in_schema[table][-x], line)
                             if matches:
-                                new_column = line.strip().replace(',', '')
-                                self.cur.execute(f"ALTER TABLE {table} ADD {new_column};")
+                                # Extract column definition, remove trailing comma and SQL comments
+                                new_column = line.strip()
+                                if '--' in new_column:
+                                    new_column = new_column[:new_column.index('--')].strip()
+                                new_column = new_column.rstrip(',')
+                                self.cur.execute(f"ALTER TABLE {table} ADD COLUMN {new_column}")
                                 self.con.commit()
                                 print(f'[cwa-db] Missing Column detected in cwa.db. Added new column "{column_names_in_schema[table][-x]}" to table "{table}" in cwa.db')
-            else: # Number of columns in table matches the schema, now checks whether the names are the same
+                                columns_added = True
+                                break  # Found and added the column, move to next missing column
+            
+            # Only check for column renames if we didn't just add columns
+            # (newly added columns are correct, don't try to rename them)
+            if not columns_added and len(current_column_names[table]) == len(column_names_in_schema[table]):
+                # Check if all columns exist but just in different order (SQLite ADD COLUMN always appends)
+                current_set = set(current_column_names[table])
+                schema_set = set(column_names_in_schema[table])
+                
+                if current_set == schema_set:
+                    # All columns exist, just in different order - this is fine, SQLite can't reorder
+                    continue
+                
+                # Columns differ, check for actual renames needed
                 for x in range(len(column_names_in_schema[table])):
                     if current_column_names[table][x] != column_names_in_schema[table][x]:
                         self.cur.execute(f"ALTER TABLE {table} RENAME COLUMN {current_column_names[table][x]} TO {column_names_in_schema[table][x]}")
@@ -258,16 +328,16 @@ class CWA_DB:
                 cwa_settings[key] = default_value
 
         # Define which settings should remain as integers (not converted to boolean)
-        integer_settings = ['ingest_timeout_minutes', 'auto_send_delay_minutes', 'hardcover_auto_fetch_batch_size', 'hardcover_auto_fetch_schedule_hour']
+        integer_settings = ['ingest_timeout_minutes', 'auto_send_delay_minutes', 'hardcover_auto_fetch_batch_size', 'hardcover_auto_fetch_schedule_hour', 'duplicate_scan_hour', 'duplicate_scan_chunk_size', 'duplicate_scan_debounce_seconds']
         
         # Define which settings should remain as floats (not converted to boolean)
         float_settings = ['hardcover_auto_fetch_min_confidence', 'hardcover_auto_fetch_rate_limit']
         
         # Define which settings should remain as JSON strings (not split by comma)
-        json_settings = ['metadata_provider_hierarchy', 'metadata_providers_enabled']
+        json_settings = ['metadata_provider_hierarchy', 'metadata_providers_enabled', 'duplicate_format_priority']
 
         for header in headers:
-            if isinstance(cwa_settings[header], int) and header not in integer_settings:
+            if isinstance(cwa_settings[header], int) and header not in integer_settings and header not in float_settings:
                 cwa_settings[header] = bool(cwa_settings[header])
             elif isinstance(cwa_settings[header], str) and ',' in cwa_settings[header] and header not in json_settings:
                 cwa_settings[header] = cwa_settings[header].split(',')
@@ -281,9 +351,18 @@ class CWA_DB:
             if setting == "auto_convert_ignored_formats" or setting == "auto_ingest_ignored_formats" or setting == "auto_convert_retained_formats":
                 result[setting] = ','.join(result[setting])
 
-            # Use parameterized queries to safely handle non-English characters and quotes
-            self.cur.execute(f"UPDATE cwa_settings SET {setting}=?;", (result[setting],))
-            self.con.commit()
+            # Skip updates for unset values to avoid NOT NULL constraint failures
+            if result[setting] is None:
+                continue
+
+            try:
+                # Use parameterized queries to safely handle non-English characters and quotes
+                self.cur.execute(f"UPDATE cwa_settings SET {setting}=?;", (result[setting],))
+                self.con.commit()
+            except Exception as e:
+                print(f"[CWA_DB] Error updating setting '{setting}' with value '{result[setting]}': {e}")
+                # Continue to next setting instead of failing completely
+                continue
         self.set_default_settings()
 
 
@@ -2105,6 +2184,145 @@ class CWA_DB:
                 }
             }
 
+    def invalidate_duplicate_cache(self):
+        """Mark duplicate cache as needing refresh"""
+        try:
+            self.cur.execute("""
+                UPDATE cwa_duplicate_cache 
+                SET scan_pending = 1 
+                WHERE id = 1
+            """)
+            self.con.commit()
+            return True
+        except Exception as e:
+            print(f"[cwa-db] Error invalidating duplicate cache: {e}")
+            return False
+
+    def get_duplicate_cache(self):
+        """Get cached duplicate scan results"""
+        import json
+        try:
+            self.cur.execute("""
+                SELECT scan_timestamp, duplicate_groups_json, total_count, scan_pending, last_scanned_book_id
+                FROM cwa_duplicate_cache 
+                WHERE id = 1
+            """)
+            row = self.cur.fetchone()
+            if row and row[1]:  # Has cached data
+                return {
+                    'scan_timestamp': row[0],
+                    'duplicate_groups': json.loads(row[1]),
+                    'total_count': row[2],
+                    'scan_pending': bool(row[3]),
+                    'last_scanned_book_id': row[4]
+                }
+            return None
+        except Exception as e:
+            print(f"[cwa-db] Error getting duplicate cache: {e}")
+            return None
+
+    def update_duplicate_cache(self, duplicate_groups, total_count, max_book_id=None):
+        """Update duplicate cache with fresh scan results
+        
+        Args:
+            duplicate_groups: List of duplicate group dictionaries
+            total_count: Total number of duplicate groups found
+            max_book_id: Maximum book ID in metadata.db (optional, for incremental scanning)
+        """
+        import json
+        from datetime import datetime
+        try:
+            # Serialize duplicate groups to JSON (extract only serializable data)
+            serializable_groups = []
+            for group in duplicate_groups:
+                serializable_group = {
+                    'title': group.get('title', ''),
+                    'author': group.get('author', ''),
+                    'count': group.get('count', 0),
+                    'group_hash': group.get('group_hash', ''),
+                    'book_ids': [book.id for book in group.get('books', [])]
+                }
+                serializable_groups.append(serializable_group)
+            
+            groups_json = json.dumps(serializable_groups)
+            
+            # Update cache with optional max_book_id for incremental scanning
+            if max_book_id is not None:
+                self.cur.execute("""
+                    UPDATE cwa_duplicate_cache 
+                    SET scan_timestamp = ?, 
+                        duplicate_groups_json = ?, 
+                        total_count = ?, 
+                        scan_pending = 0,
+                        last_scanned_book_id = ?
+                    WHERE id = 1
+                """, (datetime.now().isoformat(), groups_json, total_count, max_book_id))
+            else:
+                self.cur.execute("""
+                    UPDATE cwa_duplicate_cache 
+                    SET scan_timestamp = ?, 
+                        duplicate_groups_json = ?, 
+                        total_count = ?, 
+                        scan_pending = 0
+                    WHERE id = 1
+                """, (datetime.now().isoformat(), groups_json, total_count))
+            self.con.commit()
+            return True
+        except Exception as e:
+            print(f"[cwa-db] Error updating duplicate cache: {e}")
+            return False
+
+    def log_duplicate_resolution(self, group_hash, group_title, group_author, kept_book_id, 
+                                 deleted_book_ids, strategy, trigger_type, user_id=None, notes=None):
+        """Log a duplicate resolution to audit table"""
+        import json
+        try:
+            self.cur.execute("""
+                INSERT INTO cwa_duplicate_resolutions 
+                (group_hash, group_title, group_author, kept_book_id, deleted_book_ids, 
+                 strategy, trigger_type, user_id, notes)
+                VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
+            """, (group_hash, group_title, group_author, kept_book_id, 
+                  json.dumps(deleted_book_ids), strategy, trigger_type, user_id, notes))
+            self.con.commit()
+            return True
+        except Exception as e:
+            print(f"[cwa-db] Error logging duplicate resolution: {e}")
+            return False
+
+    def get_resolution_history(self, limit=100):
+        """Get recent resolution history"""
+        import json
+        try:
+            self.cur.execute("""
+                SELECT id, timestamp, group_hash, group_title, group_author, 
+                       kept_book_id, deleted_book_ids, strategy, trigger_type, user_id, notes
+                FROM cwa_duplicate_resolutions 
+                ORDER BY timestamp DESC 
+                LIMIT ?
+            """, (limit,))
+            
+            results = []
+            for row in self.cur.fetchall():
+                results.append({
+                    'id': row[0],
+                    'timestamp': row[1],
+                    'group_hash': row[2],
+                    'group_title': row[3],
+                    'group_author': row[4],
+                    'kept_book_id': row[5],
+                    'deleted_book_ids': json.loads(row[6]),
+                    'strategy': row[7],
+                    'trigger_type': row[8],
+                    'user_id': row[9],
+                    'notes': row[10]
+                })
+            return results
+        except Exception as e:
+            print(f"[cwa-db] Error getting resolution history: {e}")
+            return []
+
+
 def main():
     db = CWA_DB()
 

scripts/cwa_schema.sql

@@ -76,7 +76,23 @@ CREATE TABLE IF NOT EXISTS cwa_settings(
     hardcover_auto_fetch_schedule_hour INTEGER DEFAULT 2 NOT NULL,
     hardcover_auto_fetch_min_confidence REAL DEFAULT 0.85 NOT NULL,
     hardcover_auto_fetch_batch_size INTEGER DEFAULT 50 NOT NULL,
-    hardcover_auto_fetch_rate_limit REAL DEFAULT 5.0 NOT NULL
+    hardcover_auto_fetch_rate_limit REAL DEFAULT 5.0 NOT NULL,
+    -- Duplicate notification and auto-resolution settings
+    duplicate_detection_enabled SMALLINT DEFAULT 1 NOT NULL,
+    duplicate_notifications_enabled SMALLINT DEFAULT 1 NOT NULL,
+    duplicate_auto_resolve_enabled SMALLINT DEFAULT 0 NOT NULL,
+    duplicate_auto_resolve_strategy TEXT DEFAULT 'newest' NOT NULL,
+    duplicate_format_priority TEXT DEFAULT '{"EPUB":100,"KEPUB":95,"AZW3":90,"MOBI":80,"AZW":75,"PDF":60,"TXT":40,"CBZ":35,"CBR":35,"FB2":30,"DJVU":25,"HTML":20,"RTF":15,"DOC":10,"DOCX":10}' NOT NULL,
+    -- Duplicate scanning performance settings
+    duplicate_detection_use_sql SMALLINT DEFAULT 1 NOT NULL, -- Enable SQL prefilter for hybrid by default
+    duplicate_scan_method TEXT DEFAULT 'hybrid' NOT NULL, -- Use hybrid prefilter by default
+    duplicate_scan_enabled SMALLINT DEFAULT 1 NOT NULL,
+    duplicate_scan_frequency TEXT DEFAULT 'after_import' NOT NULL,
+    duplicate_scan_cron TEXT DEFAULT '' NOT NULL,
+    duplicate_scan_hour INTEGER DEFAULT 3 NOT NULL,
+    duplicate_scan_chunk_size INTEGER DEFAULT 5000 NOT NULL,
+    duplicate_scan_debounce_seconds INTEGER DEFAULT 30 NOT NULL
+>>>>>>> origin/main
 );
 
 -- Persisted scheduled jobs (initial focus: auto-send). Rows remain until dispatched or manually cleared.
@@ -138,3 +154,37 @@ CREATE TABLE IF NOT EXISTS hardcover_auto_fetch_stats(
     errors INTEGER DEFAULT 0 NOT NULL,
     avg_confidence REAL DEFAULT 0.0 NOT NULL
 );
+
+-- Duplicate detection cache table
+CREATE TABLE IF NOT EXISTS cwa_duplicate_cache (
+    id INTEGER PRIMARY KEY CHECK (id = 1),  -- Singleton table, only one row
+    scan_timestamp DATETIME DEFAULT CURRENT_TIMESTAMP,
+    duplicate_groups_json TEXT,  -- JSON serialized duplicate groups
+    total_count INTEGER DEFAULT 0,
+    scan_pending INTEGER DEFAULT 1,  -- 1=needs scan, 0=cache valid
+    last_scanned_book_id INTEGER DEFAULT 0,  -- Track last scanned book for incremental updates
+    scan_duration_seconds REAL DEFAULT 0,  -- Performance tracking
+    scan_method_used TEXT DEFAULT 'python'  -- Track which method was used: 'sql', 'python', 'hybrid'
+);
+
+-- Insert default row for cache table
+INSERT OR IGNORE INTO cwa_duplicate_cache (id, scan_pending) VALUES (1, 1);
+
+-- Auto-resolution audit log
+CREATE TABLE IF NOT EXISTS cwa_duplicate_resolutions (
+    id INTEGER PRIMARY KEY AUTOINCREMENT,
+    timestamp DATETIME DEFAULT CURRENT_TIMESTAMP,
+    group_hash TEXT NOT NULL,
+    group_title TEXT,
+    group_author TEXT,
+    kept_book_id INTEGER NOT NULL,
+    deleted_book_ids TEXT NOT NULL,  -- JSON array of deleted IDs
+    strategy TEXT NOT NULL,  -- 'newest', 'highest_quality_format', 'most_metadata', 'largest_file_size'
+    trigger_type TEXT NOT NULL,  -- 'manual', 'scheduled', 'automatic'
+    backed_up INTEGER DEFAULT 1,  -- 1=yes, 0=no
+    user_id INTEGER,  -- NULL for automatic, admin user ID for manual
+    notes TEXT
+);
+
+CREATE INDEX IF NOT EXISTS idx_duplicate_resolutions_timestamp ON cwa_duplicate_resolutions(timestamp);
+CREATE INDEX IF NOT EXISTS idx_duplicate_resolutions_group_hash ON cwa_duplicate_resolutions(group_hash);

scripts/generate_book_checksums.py

@@ -15,10 +15,11 @@ on every boot (via cwa-checksum-backfill service) to backfill any missing checks
 for newly added books.
 
 Usage:
-    python generate_book_checksums.py [--library-path /path/to/calibre/library] [--force]
+    python generate_book_checksums.py [--library-path /path/to/calibre/library] [--books-path /path/to/books] [--force]
 
 Options:
     --library-path  Path to Calibre library directory (defaults to /calibre-library)
+    --books-path    Path to books directory (defaults to config_calibre_split_dir setting with --library-path fallback)
     --force         Regenerate checksums even if they already exist
     --batch-size    Number of books to process before committing (default: 100)
 """
@@ -32,11 +33,12 @@ from pathlib import Path
 # Import the centralized partial MD5 calculation function
 sys.path.insert(0, os.path.dirname(os.path.dirname(__file__)))
 from cps.progress_syncing.checksums import calculate_koreader_partial_md5, store_checksum, CHECKSUM_VERSION
-def generate_checksums(library_path: str, force: bool = False, batch_size: int = 100):
+def generate_checksums(library_path: str, books_path: str = None, force: bool = False, batch_size: int = 100):
     """Generate checksums for all books in the library
 
     Args:
-        library_path: Path to Calibre library directory
+        library_path: Path to Calibre library directory (contains metadata.db)
+        books_path: Path to books directory (if different from library_path in split mode)
         force: If True, regenerate checksums even if they exist
         batch_size: Number of books to process before committing
     """
@@ -46,7 +48,14 @@ def generate_checksums(library_path: str, force: bool = False, batch_size: int =
         print(f"ERROR: Calibre database not found at {metadata_db}")
         sys.exit(1)
 
+    # Use books_path if provided and valid, otherwise fall back to library_path
+    base_path = books_path if (books_path and os.path.exists(books_path)) else library_path
+
     print(f"Connecting to Calibre library at: {library_path}")
+    if base_path != library_path:
+        print(f"Books path (split library mode): {base_path}")
+    else:
+        print(f"Books path: {base_path}")
     print(f"Force regenerate: {force}")
     print(f"Batch size: {batch_size}")
     print(f"Checksum version: {CHECKSUM_VERSION}")
@@ -98,7 +107,7 @@ def generate_checksums(library_path: str, force: bool = False, batch_size: int =
             processed += 1
 
             # Construct full file path
-            file_path = os.path.join(library_path, book_path, f"{format_name}.{format_ext.lower()}")
+            file_path = os.path.join(base_path, book_path, f"{format_name}.{format_ext.lower()}")
 
             if not os.path.exists(file_path):
                 print(f"[{processed}/{total}] SKIP: File not found - {title} ({format_ext})")
@@ -152,6 +161,42 @@ def generate_checksums(library_path: str, force: bool = False, batch_size: int =
         conn.close()
 
 
+def get_books_path():
+    """
+    Get the split library books path from app.db if split mode is enabled.
+    
+    Returns:
+        The books path from config_calibre_split_dir if it exists and is valid,
+        otherwise None to indicate the library path should be used.
+    """
+    try:
+        conn = sqlite3.connect("/config/app.db", timeout=30)
+        cur = conn.cursor()
+
+        # Check if split mode is enabled and get split path
+        result = cur.execute('SELECT config_calibre_split, config_calibre_split_dir FROM settings LIMIT 1;').fetchone()
+        
+        if not result:
+            return None
+            
+        split_enabled, split_path = result
+        
+        # Only return split path if split mode is enabled, path is not NULL, and path exists
+        if split_enabled and split_path and os.path.exists(split_path):
+            return split_path
+            
+        return None
+
+    except sqlite3.Error as e:
+        # Log warning but don't crash - fall back to library path
+        print(f"WARNING: Could not read split library setting from app.db: {e}")
+        print(f"WARNING: Falling back to --library-path for books location")
+        return None
+    finally:
+        if 'conn' in locals():
+            conn.close()
+
+
 def main():
     parser = argparse.ArgumentParser(
         description='Generate KOReader sync checksums for books in Calibre library',
@@ -164,6 +209,12 @@ def main():
         help='Path to Calibre library directory (default: /calibre-library)'
     )
 
+    parser.add_argument(
+        '--books-path',
+        default=get_books_path(),
+        help='Path to books directory (default: config_calibre_split_dir setting or --library-path)'
+    )
+
     parser.add_argument(
         '--force',
         action='store_true',
@@ -185,7 +236,7 @@ def main():
         sys.exit(1)
 
     try:
-        generate_checksums(args.library_path, args.force, args.batch_size)
+        generate_checksums(args.library_path, args.books_path, args.force, args.batch_size)
     except KeyboardInterrupt:
         print("\n\nInterrupted by user. Exiting...")
         sys.exit(130)

scripts/ingest_processor.py

@@ -14,6 +14,7 @@ import time
 import shutil
 import sqlite3
 import fcntl
+import threading
 from pathlib import Path
 
 from cwa_db import CWA_DB
@@ -31,6 +32,10 @@ TaskAutoSend = None
 WorkerThread = None
 _ub = None
 
+# Debounced duplicate scan timer
+_duplicate_scan_timer = None
+_duplicate_scan_lock = threading.Lock()
+
 class ProcessLock:
     """Robust process lock using both file locking and PID tracking"""
 
@@ -760,6 +765,12 @@ class NewBookProcessor:
             # This solves the issue where multiple books don't appear until container restart
             self.refresh_cwa_session()
 
+            # Invalidate duplicate cache since a new book was added
+            self.invalidate_duplicate_cache()
+
+            # Debounced duplicate scan (after import)
+            self.schedule_debounced_duplicate_scan()
+
             # Generate KOReader sync checksums for the imported book
             if self.last_added_book_id is not None:
                 self.generate_book_checksums(staged_path.stem, book_id=self.last_added_book_id)
@@ -1102,6 +1113,72 @@ class NewBookProcessor:
             print("[ingest-processor] Continuing despite session refresh failure - books may require manual refresh", flush=True)
 
 
+    def invalidate_duplicate_cache(self) -> None:
+        """Invalidate the duplicate detection cache after adding a new book
+
+        This marks the cache as stale (scan_pending=1) but does NOT trigger an automatic scan.
+        Users must manually trigger a scan from the /duplicates page, or wait for scheduled scans.
+        """
+        try:
+            url = get_internal_api_url("/duplicates/invalidate-cache")
+            resp = requests.post(
+                url,
+                headers=get_internal_api_headers(),
+                timeout=5,
+                verify=False,
+            )
+            if resp.status_code == 200:
+                print("[ingest-processor] Duplicate cache invalidated", flush=True)
+            else:
+                print(f"[ingest-processor] WARN: Duplicate cache invalidation returned {resp.status_code}", flush=True)
+        except Exception as e:
+            # Don't fail the import if cache invalidation fails
+            print(f"[ingest-processor] WARN: Failed to invalidate duplicate cache: {e}", flush=True)
+
+
+    def schedule_debounced_duplicate_scan(self) -> None:
+        """Schedule a debounced background duplicate scan if enabled.
+
+        Uses a 60-second timer that resets on each new import to avoid repeated scans
+        during batch ingest.
+        """
+        try:
+            enabled = bool(self.cwa_settings.get('duplicate_scan_enabled', 0))
+            frequency = self.cwa_settings.get('duplicate_scan_frequency', 'manual')
+
+            if not enabled or frequency != 'after_import':
+                return
+            # Schedule in the long-lived web process so the debounce survives
+            # the short-lived ingest process exiting.
+            try:
+                delay_seconds = int(self.cwa_settings.get('duplicate_scan_debounce_seconds', 30))
+                delay_seconds = max(5, min(600, delay_seconds))
+                url = get_internal_api_url("/cwa-internal/queue-duplicate-scan")
+                payload = {"delay_seconds": delay_seconds}
+                resp = requests.post(
+                    url,
+                    json=payload,
+                    headers=get_internal_api_headers(),
+                    timeout=5,
+                    verify=False,
+                )
+                if resp.status_code == 200:
+                    try:
+                        body = resp.json()
+                        if body.get('queued'):
+                            print("[ingest-processor] Debounced duplicate scan scheduled via web process", flush=True)
+                        elif body.get('skipped'):
+                            print("[ingest-processor] Duplicate scan scheduling skipped (disabled/manual)", flush=True)
+                    except Exception:
+                        print("[ingest-processor] Debounced duplicate scan scheduled via web process", flush=True)
+                else:
+                    print(f"[ingest-processor] WARN: Duplicate scan scheduling returned {resp.status_code}", flush=True)
+            except Exception as e:
+                print(f"[ingest-processor] WARN: Failed to schedule duplicate scan via web API: {e}", flush=True)
+        except Exception as e:
+            print(f"[ingest-processor] WARN: Failed to schedule debounced duplicate scan: {e}", flush=True)
+
+
     def set_library_permissions(self):
         try:
             nsm = os.getenv("NETWORK_SHARE_MODE", "false").strip().lower() in ("1", "true", "yes", "on")

tests/unit/test_generate_checksums.py

@@ -316,6 +316,183 @@ class TestChecksumGenerationScript:
         assert result.returncode == 0
         assert "SKIP" in result.stdout or "not found" in result.stdout.lower()
 
+    def test_split_library_with_separate_paths(self, tmp_path):
+        """Test checksum generation with split library (separate metadata and books)."""
+        # Create separate directories for metadata and books
+        metadata_dir = tmp_path / "metadata"
+        books_dir = tmp_path / "books"
+        
+        metadata_dir.mkdir()
+        books_dir.mkdir()
+        
+        # Create metadata.db in metadata_dir
+        create_minimal_calibre_library(metadata_dir)
+        
+        # Add book to metadata.db
+        db_path = metadata_dir / "metadata.db"
+        conn = sqlite3.connect(db_path)
+        cur = conn.cursor()
+        
+        cur.execute("""
+            INSERT INTO books (title, sort, path, has_cover)
+            VALUES ('Split Library Book', 'Split Library Book', 'split_book', 0)
+        """)
+        book_id = cur.lastrowid
+        
+        cur.execute("""
+            INSERT INTO data (book, format, name)
+            VALUES (?, 'EPUB', 'split_book')
+        """, (book_id,))
+        
+        conn.commit()
+        conn.close()
+        
+        # Create actual book file in books_dir (NOT metadata_dir)
+        book_folder = books_dir / "split_book"
+        book_folder.mkdir()
+        (book_folder / "split_book.epub").write_bytes(b"Test EPUB content for split library testing")
+        
+        # Run script with separate books-path
+        script_path = scripts_dir / "generate_book_checksums.py"
+        result = subprocess.run(
+            [sys.executable, str(script_path),
+             "--library-path", str(metadata_dir),
+             "--books-path", str(books_dir)],
+            capture_output=True,
+            text=True,
+            timeout=30
+        )
+        
+        assert result.returncode == 0
+        assert "Split Library Book" in result.stdout
+        assert "✓" in result.stdout
+        assert "split library mode" in result.stdout.lower()
+        
+        # Verify checksum was stored in metadata.db
+        conn = sqlite3.connect(db_path)
+        cur = conn.cursor()
+        checksum = cur.execute("""
+            SELECT checksum FROM book_format_checksums
+            WHERE book = ? AND format = 'EPUB'
+        """, (book_id,)).fetchone()
+        conn.close()
+        
+        assert checksum is not None
+        assert len(checksum[0]) == 32  # Valid MD5
+
+    def test_books_path_falls_back_to_library_path(self, tmp_path):
+        """Test that invalid books-path falls back to library-path."""
+        library_path = tmp_path / "library"
+        create_minimal_calibre_library(library_path)
+        add_book_to_library(library_path, "Fallback Test Book", ["EPUB"])
+        
+        # Pass nonexistent books-path
+        script_path = scripts_dir / "generate_book_checksums.py"
+        result = subprocess.run(
+            [sys.executable, str(script_path),
+             "--library-path", str(library_path),
+             "--books-path", "/nonexistent/path"],
+            capture_output=True,
+            text=True,
+            timeout=30
+        )
+        
+        # Should succeed by falling back to library_path
+        assert result.returncode == 0
+        assert "Fallback Test Book" in result.stdout
+        assert "✓" in result.stdout
+
+    def test_books_path_with_none_value(self, tmp_path):
+        """Test that None books-path uses library-path."""
+        library_path = tmp_path / "library"
+        create_minimal_calibre_library(library_path)
+        add_book_to_library(library_path, "Normal Mode Book", ["EPUB"])
+        
+        # Run without --books-path argument (default behavior)
+        script_path = scripts_dir / "generate_book_checksums.py"
+        result = subprocess.run(
+            [sys.executable, str(script_path),
+             "--library-path", str(library_path)],
+            capture_output=True,
+            text=True,
+            timeout=30
+        )
+        
+        # Should succeed using library_path for books
+        assert result.returncode == 0
+        assert "Normal Mode Book" in result.stdout
+        assert "✓" in result.stdout
+        # Should NOT show split library mode message
+        assert "Books path: " in result.stdout
+
+    def test_split_library_with_multiple_formats(self, tmp_path):
+        """Test split library with book having multiple formats."""
+        metadata_dir = tmp_path / "metadata"
+        books_dir = tmp_path / "books"
+        
+        metadata_dir.mkdir()
+        books_dir.mkdir()
+        
+        create_minimal_calibre_library(metadata_dir)
+        
+        # Add book with multiple formats
+        db_path = metadata_dir / "metadata.db"
+        conn = sqlite3.connect(db_path)
+        cur = conn.cursor()
+        
+        cur.execute("""
+            INSERT INTO books (title, sort, path, has_cover)
+            VALUES ('Multi Format Book', 'Multi Format Book', 'multi_format', 0)
+        """)
+        book_id = cur.lastrowid
+        
+        # Add multiple formats
+        for fmt in ['EPUB', 'MOBI', 'PDF']:
+            cur.execute("""
+                INSERT INTO data (book, format, name)
+                VALUES (?, ?, 'multi_format')
+            """, (book_id, fmt))
+        
+        conn.commit()
+        conn.close()
+        
+        # Create actual book files in books_dir
+        book_folder = books_dir / "multi_format"
+        book_folder.mkdir()
+        (book_folder / "multi_format.epub").write_bytes(b"EPUB content")
+        (book_folder / "multi_format.mobi").write_bytes(b"MOBI content")
+        (book_folder / "multi_format.pdf").write_bytes(b"PDF content")
+        
+        # Run script
+        script_path = scripts_dir / "generate_book_checksums.py"
+        result = subprocess.run(
+            [sys.executable, str(script_path),
+             "--library-path", str(metadata_dir),
+             "--books-path", str(books_dir)],
+            capture_output=True,
+            text=True,
+            timeout=30
+        )
+        
+        assert result.returncode == 0
+        
+        # Verify all three formats got checksums
+        conn = sqlite3.connect(db_path)
+        cur = conn.cursor()
+        checksums = cur.execute("""
+            SELECT format, checksum FROM book_format_checksums
+            WHERE book = ?
+            ORDER BY format
+        """, (book_id,)).fetchall()
+        conn.close()
+        
+        assert len(checksums) == 3
+        assert all(len(checksum[1]) == 32 for checksum in checksums)
+        formats = [c[0] for c in checksums]
+        assert 'EPUB' in formats
+        assert 'MOBI' in formats
+        assert 'PDF' in formats
+
     def test_batch_size_parameter(self, tmp_path):
         """Test that batch-size parameter is respected."""
         library_path = tmp_path / "test_library"