#!/usr/bin/env python
# License: GPLv3 Copyright: 2010, Kovid Goyal <kovid@kovidgoyal.net>

import functools
import numbers
import os
import re
import sys
import time
import traceback
from collections import defaultdict, namedtuple
from collections.abc import Callable
from itertools import groupby
from typing import cast

from qt.core import (
    QAbstractTableModel,
    QApplication,
    QColor,
    QDateTime,
    QFont,
    QFontMetrics,
    QIcon,
    QImage,
    QLocale,
    QModelIndex,
    QPainter,
    QPixmap,
    Qt,
    QTimer,
    pyqtSignal,
)

from calibre import fit_image, human_readable, prepare_string_for_xml, strftime
from calibre.constants import DEBUG, config_dir, dark_link_color, filesystem_encoding
from calibre.db.search import CONTAINS_MATCH, EQUALS_MATCH, REGEXP_MATCH, _match
from calibre.db.utils import force_to_bool
from calibre.devices.interface import BookList
from calibre.ebooks.metadata import authors_to_string, fmt_sidx, string_to_authors, title_sort
from calibre.ebooks.metadata.book.formatter import SafeFormat
from calibre.gui2 import error_dialog, is_dark_theme, qapplication_or_fail, simple_excepthook
from calibre.gui2.library import DEFAULT_SORT
from calibre.library.coloring import color_row_key
from calibre.library.save_to_disk import find_plugboard
from calibre.ptempfile import PersistentTemporaryFile
from calibre.utils.config import device_prefs, prefs, tweaks
from calibre.utils.date import UNDEFINED_DATE, dt_factory, is_date_undefined, qt_from_dt, qt_to_dt
from calibre.utils.icu import sort_key
from calibre.utils.localization import _, calibre_langcode_to_name, ngettext
from calibre.utils.resources import get_path as P
from calibre.utils.search_query_parser import ParseException, SearchQueryParser
from calibre_extensions.imageops import load_from_data_without_gil

Counts = namedtuple('Counts', 'library_total total current')

TIME_FMT = '%d %b %Y'

ALIGNMENT_MAP = {
    'left': Qt.AlignmentFlag.AlignLeft,
    'right': Qt.AlignmentFlag.AlignRight,
    'center': Qt.AlignmentFlag.AlignHCenter,
}


def render_pin(color=None, save_to=None):
    app = qapplication_or_fail()
    if color is None:
        color = '#00b000' if app.is_dark_theme else 'green'
    svg = P('pin-template.svg', data=True).replace(b'fill:#f39509', ('fill:' + color).encode('utf-8'))
    pm = QPixmap()
    dpr = app.devicePixelRatio()
    pm.setDevicePixelRatio(dpr)
    pm.loadFromData(svg, 'svg')
    if save_to:
        pm.save(save_to)
    return pm


def group_numbers(numbers):
    for k, g in groupby(enumerate(sorted(numbers)), lambda i_x: i_x[0] - i_x[1]):
        first = None
        for last in g:
            if first is None:
                first = last[1]
        yield first, last[1]


class ColumnColor:  # {{{
    def __init__(self, formatter):
        self.mi = None
        self.formatter = formatter

    def __call__(self, id_, key, fmt, db, color_cache, template_cache):
        key += str(hash(fmt))
        if id_ in color_cache and key in color_cache[id_]:
            self.mi = None
            color = color_cache[id_][key]
            if color.isValid():
                return color
            return None
        try:
            if self.mi is None:
                self.mi = db.new_api.get_proxy_metadata(id_)
            color = QColor(self.formatter.safe_format(fmt, self.mi, '', self.mi, column_name=key, template_cache=template_cache))
            color_cache[id_][key] = color
            if color.isValid():
                self.mi = None
                return color
        except Exception:
            pass


# }}}


def themed_icon_name(icon_dir, icon_name):
    root, ext = os.path.splitext(icon_name)
    # Remove any theme from the icon name
    root = root.removesuffix('-for-dark-theme').removesuffix('-for-light-theme')
    # Check if the correct themed icon exists.
    theme_suffix = '-for-dark-theme' if is_dark_theme() else '-for-light-theme'
    d = os.path.join(icon_dir, root + theme_suffix + ext)
    if os.path.exists(d):
        return d
    # No themed icon exists. Try the original name
    d = os.path.join(icon_dir, icon_name)
    return d if os.path.exists(d) else None


class ColumnIcon:  # {{{
    def __init__(self, formatter, model):
        self.mi = None
        self.formatter = formatter
        self.model = model
        self.dpr = qapplication_or_fail().devicePixelRatio()

    def __call__(self, id_, fmts, cache_index, db, icon_cache, icon_bitmap_cache, template_cache):
        if id_ in icon_cache and cache_index in icon_cache[id_]:
            self.mi = None
            return icon_cache[id_][cache_index]
        try:
            if self.mi is None:
                self.mi = db.new_api.get_proxy_metadata(id_)
            icons = []
            for dex, (kind, fmt) in enumerate(fmts):
                rule_icons = self.formatter.safe_format(fmt, self.mi, '', self.mi, column_name=cache_index + str(dex), template_cache=template_cache)
                if not rule_icons:
                    continue
                icon_list = [ic.strip() for ic in rule_icons.split(':') if ic.strip()]
                icons.extend(icon_list)
                if icon_list and not kind.endswith('_composed'):
                    break

            if icons:
                icon_string = ':'.join(icons)
                if icon_string in icon_bitmap_cache:
                    icon_bitmap = icon_bitmap_cache[icon_string]
                    icon_cache[id_][cache_index] = icon_bitmap
                    return icon_bitmap

                icon_bitmaps = []
                total_width = 0
                rh = max(2, self.model.row_height - 4)
                dim = int(self.dpr * rh)
                icon_dir = os.path.join(config_dir, 'cc_icons')
                for icon in icons:
                    d = themed_icon_name(icon_dir, icon)
                    if d is not None:
                        bm = QPixmap(d)
                        scaled, nw, nh = fit_image(bm.width(), bm.height(), bm.width(), dim)
                        bm = bm.scaled(
                            int(nw),
                            int(nh),
                            aspectRatioMode=Qt.AspectRatioMode.IgnoreAspectRatio,
                            transformMode=Qt.TransformationMode.SmoothTransformation,
                        )
                        bm.setDevicePixelRatio(self.dpr)
                        icon_bitmaps.append(bm)
                        total_width += bm.width()
                if len(icon_bitmaps) > 1:
                    i = len(icon_bitmaps)
                    result = QPixmap(total_width + ((i - 1) * 2), dim)
                    result.setDevicePixelRatio(self.dpr)
                    result.fill(Qt.GlobalColor.transparent)
                    painter = QPainter(result)
                    x = 0
                    for bm in icon_bitmaps:
                        painter.drawPixmap(x, 0, bm)
                        x += int(bm.width() / self.dpr) + 2
                    painter.end()
                else:
                    result = icon_bitmaps[0]

                icon_cache[id_][cache_index] = result
                icon_bitmap_cache[icon_string] = result
                self.mi = None
                return result
        except Exception:
            pass


# }}}


class BooksModel(QAbstractTableModel):  # {{{
    about_to_be_sorted = pyqtSignal(object, name='aboutToBeSorted')
    sorting_done = pyqtSignal(object, name='sortingDone')
    database_changed = pyqtSignal(object, name='databaseChanged')
    new_bookdisplay_data = pyqtSignal(object)
    count_changed_signal = pyqtSignal(int)
    searched = pyqtSignal(object)
    search_done = pyqtSignal()

    def __init__(self, parent=None, buffer=40):
        QAbstractTableModel.__init__(self, parent)
        base_font = parent.font() if parent else qapplication_or_fail().font()
        self.zero_page_cache = {}
        self.update_page_count_timer = t = QTimer(self)
        t.setSingleShot(True), t.setInterval(1000), t.timeout.connect(self.update_zero_page_values)
        self.bold_font = QFont(base_font)
        self.bold_font.setBold(True)
        self.italic_font = QFont(base_font)
        self.italic_font.setItalic(True)
        self.bi_font = QFont(self.bold_font)
        self.bi_font.setItalic(True)
        self.styled_columns = {}
        self.orig_headers = {
            'title': _('Title'),
            'ondevice': _('On Device'),
            'authors': _('Author(s)'),
            'size': _('Size (MB)'),
            'timestamp': _('Date'),
            'pubdate': _('Published'),
            'rating': _('Rating'),
            'publisher': _('Publisher'),
            'tags': _('Tags'),
            'series': ngettext('Series', 'Series', 1),
            'last_modified': _('Modified'),
            'languages': _('Languages'),
            'formats': _('Formats'),
            'id': _('Id'),
            'path': _('Path'),
            'pages': _('Pages'),
        }
        self.db = None

        self.formatter = SafeFormat()
        self._clear_caches()
        self.column_color = ColumnColor(self.formatter)
        self.column_icon = ColumnIcon(self.formatter, self)

        self.book_on_device = None
        self.editable_cols = [
            'title',
            'authors',
            'rating',
            'publisher',
            'tags',
            'series',
            'timestamp',
            'pubdate',
            'languages',
        ]
        self.sorted_on = DEFAULT_SORT
        self.sort_history = [self.sorted_on]
        self.last_search = ''  # The last search performed on this model
        self.column_map = []
        self.headers = {}
        self.alignment_map = {}
        self.buffer_size = buffer
        self.metadata_backup = None
        icon_height = (parent.fontMetrics() if hasattr(parent, 'fontMetrics') else QFontMetrics(QApplication.font())).lineSpacing()
        self.bool_yes_icon = QIcon.cached_icon('ok.png').pixmap(icon_height)
        self.bool_no_icon = QIcon.cached_icon('list_remove.png').pixmap(icon_height)
        self.bool_blank_icon = QIcon.cached_icon('blank.png').pixmap(icon_height)
        # Qt auto-scales marked icon correctly, so we don't need to do it (and
        # remember that the cover grid view needs a larger version of the icon,
        # anyway)
        self.marked_icon = QIcon.cached_icon('marked.png')
        self.bool_blank_icon_as_icon = QIcon.cached_icon('blank.png')
        self.row_decoration = None
        self.device_connected = False
        self.ids_to_highlight = []
        self.ids_to_highlight_set = set()
        self.current_highlighted_idx = None
        self.highlight_only = False
        self.row_height = 0
        self.marked_text_icons = {}
        self.db_prefs = {'column_color_rules': (), 'column_icon_rules': ()}
        self.read_config()

    def marked_text_icon_for(self, label):
        import random

        ans = self.marked_text_icons.get(label)
        if ans is not None:
            return ans[1]
        db = self.db
        assert db is not None
        used_labels = db.data.all_marked_labels()
        for qlabel in tuple(self.marked_text_icons):
            if qlabel not in used_labels:
                del self.marked_text_icons[qlabel]
        used_colors = {x[0] for x in self.marked_text_icons.values()}
        if qapplication_or_fail().is_dark_theme:
            all_colors = {dark_link_color, 'lightgreen', 'red', 'maroon', 'cyan', 'pink'}
        else:
            all_colors = {'blue', 'green', 'red', 'maroon', 'cyan', 'pink'}
        for c in all_colors - used_colors:
            color = c
            break
        else:
            color = random.choice(sorted(all_colors))
        pm = render_pin(color)
        ans = QIcon(pm)
        self.marked_text_icons[label] = color, ans
        return ans

    @property
    def default_image(self):
        from calibre.gui2.ui import get_gui

        gui = get_gui()
        dpr = gui.devicePixelRatio() if gui else 0
        return qapplication_or_fail().cached_qimage('default_cover.png', device_pixel_ratio=dpr)

    def _clear_caches(self):
        self.color_cache = defaultdict(dict)
        self.icon_cache = defaultdict(dict)
        self.icon_bitmap_cache = {}
        self.cover_grid_emblem_cache = defaultdict(dict)
        self.bookshelf_emblem_cache = defaultdict(dict)
        self.cover_grid_bitmap_cache = {}
        self.bookshelf_bitmap_cache = {}
        self.color_row_fmt_cache = None
        self.color_template_cache = {}
        self.icon_template_cache = {}
        self.cover_grid_template_cache = {}
        self.bookshelf_template_cache = {}

    def set_row_height(self, height):
        self.row_height = height

    def set_row_decoration(self, current_marked):
        self.row_decoration = self.bool_blank_icon_as_icon if current_marked else None

    def change_alignment(self, colname, alignment):
        if colname in self.column_map and alignment in ('left', 'right', 'center'):
            old = self.alignment_map.get(colname, 'left')
            if old == alignment:
                return
            self.alignment_map.pop(colname, None)
            if alignment != 'left':
                self.alignment_map[colname] = alignment
            col = self.column_map.index(colname)
            for row in range(self.rowCount(QModelIndex())):
                self.dataChanged.emit(self.index(row, col), self.index(row, col))

    def change_column_font(self, colname, font_type):
        if colname in self.column_map and font_type in ('normal', 'bold', 'italic', 'bi'):
            sdb = self.db
            assert sdb is not None
            db = sdb.new_api
            old = db.pref('styled_columns', {})
            old.pop(colname, None)
            self.styled_columns.pop(colname, None)
            if font_type != 'normal':
                self.styled_columns[colname] = getattr(self, f'{font_type}_font')
                old[colname] = font_type
            db.set_pref('styled_columns', old)
            col = self.column_map.index(colname)
            for row in range(self.rowCount(QModelIndex())):
                self.dataChanged.emit(self.index(row, col), self.index(row, col))

    def is_custom_column(self, cc_label):
        try:
            return cc_label in self.custom_columns
        except AttributeError:
            return False

    def read_config(self):
        pass

    def set_device_connected(self, is_connected):
        self.device_connected = is_connected

    def refresh_ondevice(self):
        db = self.db
        assert db is not None
        db.refresh_ondevice()
        self.resort()
        self.research()

    def set_book_on_device_func(self, func):
        self.book_on_device = func

    def set_database(self, db):
        self.ids_to_highlight = []
        self.zero_page_cache = {}
        if db:
            style_map = {'bold': self.bold_font, 'bi': self.bi_font, 'italic': self.italic_font}
            self.styled_columns = {k: style_map.get(v, None) for k, v in db.new_api.pref('styled_columns', {}).items()}
        self.alignment_map = {}
        self.ids_to_highlight_set = set()
        self.current_highlighted_idx = None
        self.db = db
        self.update_db_prefs_cache()
        self.custom_columns = self.db.field_metadata.custom_field_metadata()
        self.column_map = list(self.orig_headers.keys()) + list(self.custom_columns)

        assert db is not None

        def col_idx(name):
            if name == 'ondevice':
                return -1
            if name not in db.field_metadata:
                return 100000
            return db.field_metadata[name]['rec_index']

        self.column_map.sort(key=col_idx)
        for col in self.column_map:
            if col in self.orig_headers:
                self.headers[col] = self.orig_headers[col]
            elif col in self.custom_columns:
                self.headers[col] = self.custom_columns[col]['name']

        self.build_data_convertors()
        self.beginResetModel(), self.endResetModel()
        self.database_changed.emit(db)
        self.stop_metadata_backup()
        self.start_metadata_backup()

    def get_pages_description_for_zero_page(self, book_id: int) -> str:
        if (ans := self.zero_page_cache.get(book_id)) is None:
            db = self.db
            assert db is not None
            self.zero_page_cache[book_id] = ans = bool(db.new_api.pages_needs_scan((book_id,)))
        if ans:
            self.update_page_count_timer.start()
            return _('calculating')
        return '0'

    def update_zero_page_values(self) -> None:
        db = self.db
        assert db is not None
        needs_scan = tuple(bid for bid, needs_scan in self.zero_page_cache.items() if needs_scan)
        new_needs_scan = db.new_api.pages_needs_scan(needs_scan)
        changed = frozenset(needs_scan) - new_needs_scan
        if changed:
            cc = 0
            for idx, name in enumerate(self.column_map):
                if name == 'pages':
                    cc = idx
                    break
            for book_id in changed:
                del self.zero_page_cache[book_id]
                try:
                    row = db.data.id_to_index(book_id)
                except Exception:
                    pass
                else:
                    self.dataChanged.emit(self.index(row, cc), self.index(row, cc))

    def update_db_prefs_cache(self):
        db = self.db
        assert db is not None
        self.db_prefs = {
            'column_icon_rules': tuple(db.new_api.pref('column_icon_rules', ())),
            'column_color_rules': tuple(db.new_api.pref('column_color_rules', ())),
        }

    def start_metadata_backup(self):
        from calibre.db.backup import MetadataBackup

        self.metadata_backup = MetadataBackup(self.db)
        self.metadata_backup.start()

    def stop_metadata_backup(self):
        metadata_backup = getattr(self, 'metadata_backup', None)
        if metadata_backup is not None:
            metadata_backup.stop()
            # Would like to a join here, but the thread might be waiting to
            # do something on the GUI thread. Deadlock.

    def refresh_ids(self, ids, current_row=-1):
        self._clear_caches()
        db = self.db
        assert db is not None
        rows = db.refresh_ids(ids)
        if rows:
            self.refresh_rows(rows, current_row=current_row)

    def refresh_rows(self, rows, current_row=-1):
        self._clear_caches()
        cc = self.columnCount(QModelIndex()) - 1
        db = self.db
        assert db is not None
        for r in rows:
            db.new_api.clear_extra_files_cache(db.id(r))
        for first_row, last_row in group_numbers(rows):
            self.dataChanged.emit(self.index(first_row, 0), self.index(last_row, cc))
            if current_row >= 0 and first_row <= current_row <= last_row:
                self.new_bookdisplay_data.emit(self.get_book_display_info(current_row))

    def close(self):
        self.update_page_count_timer.stop()
        if self.db is not None:
            self.db.close()
            self.db = None
            self.beginResetModel(), self.endResetModel()

    def add_books(self, paths, formats, metadata, add_duplicates=False, return_ids=False):
        db = self.db
        assert db is not None
        ret = db.add_books(paths, formats, metadata, add_duplicates=add_duplicates, return_ids=return_ids)
        self.count_changed()
        return ret

    def add_news(self, path, arg):
        db = self.db
        assert db is not None
        ret = db.add_news(path, arg)
        self.count_changed()
        return ret

    def add_catalog(self, path, title):
        db = self.db
        assert db is not None
        ret = db.add_catalog(path, title)
        self.count_changed()
        return ret

    def count_changed(self, *args):
        self._clear_caches()
        db = self.db
        assert db is not None
        self.count_changed_signal.emit(db.count())

    def counts(self):
        db = self.db
        assert db is not None
        library_total = total = db.count()
        if db.data.search_restriction_applied():
            total = db.data.get_search_restriction_book_count()
        return Counts(library_total, total, self.count())

    def row_indices(self, index):
        """Return list indices of all cells in index.row()"""
        return [self.index(index.row(), c) for c in range(self.columnCount())]

    @property
    def by_author(self):
        return self.sorted_on[0] == 'authors'

    def books_deleted(self):
        self.count_changed()
        self.beginResetModel(), self.endResetModel()

    def delete_books(self, indices, permanent=False):
        ids = list(map(self.id, indices))
        self.delete_books_by_id(ids, permanent=permanent)
        return ids

    def delete_books_by_id(self, ids, permanent=False):
        db = self.db
        assert db is not None
        db.new_api.remove_books(ids, permanent=permanent)
        self.ids_deleted(ids)

    def ids_deleted(self, ids):
        db = self.db
        assert db is not None
        db.data.books_deleted(tuple(ids))
        db.notify('delete', list(ids))
        self.books_deleted()

    def books_added(self, num):
        if num > 0:
            self.beginInsertRows(QModelIndex(), 0, num - 1)
            self.endInsertRows()
            self.count_changed()

    def set_highlight_only(self, toWhat):
        self.highlight_only = toWhat

    def get_current_highlighted_id(self):
        if len(self.ids_to_highlight) == 0 or self.current_highlighted_idx is None:
            return None
        try:
            return self.ids_to_highlight[self.current_highlighted_idx]
        except Exception:
            return None

    def get_next_highlighted_id(self, current_row, forward):
        if len(self.ids_to_highlight) == 0 or self.current_highlighted_idx is None:
            return None
        if current_row is None:
            row_ = self.current_highlighted_idx
        else:
            row_ = current_row
        while True:
            row_ += 1 if forward else -1
            if row_ < 0:
                row_ = self.count() - 1
            elif row_ >= self.count():
                row_ = 0
            if self.id(row_) in self.ids_to_highlight_set:
                break
        try:
            self.current_highlighted_idx = self.ids_to_highlight.index(self.id(row_))
        except Exception:
            # This shouldn't happen ...
            return None
        return self.get_current_highlighted_id()

    def highlight_ids(self, ids_to_highlight):
        self.ids_to_highlight = ids_to_highlight
        self.ids_to_highlight_set = set(self.ids_to_highlight)
        if self.ids_to_highlight:
            self.current_highlighted_idx = 0
        else:
            self.current_highlighted_idx = None
        self.beginResetModel(), self.endResetModel()

    def search(self, text, reset=True):
        db = self.db
        assert db is not None
        try:
            if self.highlight_only:
                db.search('')
                if not text:
                    self.ids_to_highlight = []
                    self.ids_to_highlight_set = set()
                    self.current_highlighted_idx = None
                else:
                    self.ids_to_highlight = db.search(text, return_matches=True)
                    self.ids_to_highlight_set = set(self.ids_to_highlight)
                    if self.ids_to_highlight:
                        self.current_highlighted_idx = 0
                    else:
                        self.current_highlighted_idx = None
            else:
                self.ids_to_highlight = []
                self.ids_to_highlight_set = set()
                self.current_highlighted_idx = None
                db.search(text)
        except ParseException as e:
            self.searched.emit(e.msg)
            return
        self.last_search = text
        if reset:
            self.beginResetModel(), self.endResetModel()
        if self.last_search:
            # Do not issue search done for the null search. It is used to clear
            # the search and count records for restrictions
            self.searched.emit(True)
        self.search_done.emit()

    def sort(self, column, order=Qt.SortOrder.AscendingOrder, reset=True):
        if not self.db:
            return
        if not isinstance(order, bool):
            order = order == Qt.SortOrder.AscendingOrder
        label = self.column_map[column]
        self._sort(label, order, reset)

    def sort_by_named_field(self, field, order, reset=True):
        db = self.db
        assert db is not None
        if field in list(db.field_metadata.keys()):
            self._sort(field, order, reset)

    def _sort(self, label, order, reset):
        db = self.db
        assert db is not None
        self.about_to_be_sorted.emit(db.id)
        db.data.incremental_sort([(label, order)])
        if reset:
            self.beginResetModel(), self.endResetModel()
        self.sorted_on = (label, order)
        self.sort_history.insert(0, self.sorted_on)
        self.sorting_done.emit(db.index)

    def refresh(self, reset=True):
        db = self.db
        assert db is not None
        db.refresh(field=None)
        self.resort(reset=reset)

    def beginResetModel(self):
        self._clear_caches()
        QAbstractTableModel.beginResetModel(self)

    def reset(self):
        self.beginResetModel(), self.endResetModel()

    def resort(self, reset=True):
        if not self.db:
            return
        self.db.multisort(self.sort_history[: tweaks['maximum_resort_levels']])
        if self.sort_history:
            # There are ways to get here that don't set sorted_on, e.g., the
            # sort_by action. I don't think sort_history can be empty, but it
            # doesn't hurt to check
            self.sorted_on = self.sort_history[0]
        if reset:
            self.beginResetModel(), self.endResetModel()

    def research(self, reset=True):
        self.search(self.last_search, reset=reset)

    def columnCount(self, parent: QModelIndex = QModelIndex()):
        if parent is not None and parent.isValid():
            return 0
        return len(self.column_map)

    def rowCount(self, parent: QModelIndex = QModelIndex()):
        if parent is not None and parent.isValid():
            return 0
        return len(self.db.data) if self.db else 0

    def all_current_book_ids(self):
        db = self.db
        assert db is not None
        return db.data._map_filtered

    def count(self):
        return self.rowCount()

    def get_book_display_info(self, idx):
        db = self.db
        assert db is not None
        mi = db.get_metadata(idx)
        mi.size = mi._proxy_metadata.book_size
        mi.book_display_info_is_from_device = False
        mi.cover_data = ('jpg', self.cover(idx))
        mi.id = db.id(idx)
        mi.field_metadata = db.field_metadata
        mi.path = db.abspath(idx, create_dirs=False)
        mi.format_files = db.new_api.format_files(db.data.index_to_id(idx))
        mi.row_number = idx
        try:
            mi.marked = db.data.get_marked(idx, index_is_id=False)
        except Exception:
            mi.marked = None
        return mi

    def current_changed(self, current, previous, emit_signal=True):
        if current.isValid():
            idx = current.row()
            db = self.db
            assert db is not None
            try:
                db.id(idx)
            except Exception:
                # can happen if an out of band search is done causing the index
                # to no longer be valid since this function is now called after
                # an event loop tick.
                return
            try:
                data = self.get_book_display_info(idx)
            except Exception as e:
                if sys.excepthook is simple_excepthook or sys.excepthook is sys.__excepthook__:
                    return  # ignore failures during startup/shutdown
                setattr(e, 'locking_violation_msg', _('Failed to read cover file for this book from the calibre library.'))
                raise
            else:
                if emit_signal:
                    self.new_bookdisplay_data.emit(data)
                else:
                    return data

    def get_book_info(self, index):
        if isinstance(index, numbers.Integral):
            index = self.index(index, 0)
        # If index is not valid returns None
        data = self.current_changed(index, None, False)
        return data

    def metadata_for(self, ids, get_cover=True):
        """
        WARNING: if get_cover=True temp files are created for mi.cover.
        Remember to delete them once you are done with them.
        """
        db = self.db
        assert db is not None
        ans = []
        for id in ids:
            mi = db.get_metadata(id, index_is_id=True, get_cover=get_cover)
            ans.append(mi)
        return ans

    def get_metadata(self, rows, rows_are_ids=False, full_metadata=False):
        db = self.db
        assert db is not None
        metadata, _full_metadata = [], []
        if not rows_are_ids:
            rows = [db.id(row.row()) for row in rows]
        for id in rows:
            mi = db.get_metadata(id, index_is_id=True)
            _full_metadata.append(mi)
            au = authors_to_string(mi.authors or [_('Unknown')])
            tags = mi.tags or []
            if mi.series is not None:
                tags.append(mi.series)
            info = {
                'title': mi.title,
                'authors': au,
                'author_sort': mi.author_sort,
                'cover': db.cover(id, index_is_id=True),
                'tags': tags,
                'comments': mi.comments,
            }
            if mi.series is not None:
                info['tag order'] = {mi.series: db.books_in_series_of(id, index_is_id=True)}

            metadata.append(info)
        if full_metadata:
            return metadata, _full_metadata
        else:
            return metadata

    def get_preferred_formats_from_ids(
        self,
        ids,
        formats,
        set_metadata=False,
        specific_format=None,
        exclude_auto=False,
        mode='r+b',
        use_plugboard=None,
        plugboard_formats=None,
        modified_metadata=None,
    ):
        from calibre.ebooks.metadata.meta import set_metadata as _set_metadata

        db = self.db
        assert db is not None
        ans = []
        need_auto = []
        modified_metadata = [] if modified_metadata is None else modified_metadata
        if specific_format is not None:
            formats = [specific_format.lower()]
        for id in ids:
            format = None
            fmts = db.formats(id, index_is_id=True)
            if not fmts:
                fmts = ''
            db_formats = set(fmts.lower().split(','))
            available_formats = {f.lower() for f in formats}
            u = available_formats.intersection(db_formats)
            for f in formats:
                if f.lower() in u:
                    format = f
                    break
            if format is not None:
                pt = PersistentTemporaryFile(suffix='caltmpfmt.' + format)
                db.copy_format_to(id, format, pt, index_is_id=True)
                pt.seek(0)
                newmi = None
                if set_metadata:
                    try:
                        mi = db.get_metadata(id, get_cover=True, index_is_id=True, cover_as_data=True)
                        if use_plugboard and plugboard_formats is not None and format.lower() in plugboard_formats:
                            plugboards = db.new_api.pref('plugboards', {})
                            cpb = find_plugboard(use_plugboard, format.lower(), plugboards)
                            if cpb:
                                newmi = mi.deepcopy_metadata()
                                newmi.template_to_attribute(mi, cpb)
                        if newmi is not None:
                            _set_metadata(pt, newmi, format)
                        else:
                            _set_metadata(pt, mi, format)
                    except Exception:
                        traceback.print_exc()
                pt.close()

                def to_uni(x):
                    if isinstance(x, bytes):
                        x = x.decode(filesystem_encoding)
                    return x

                ans.append(to_uni(os.path.abspath(pt.name)))
                modified_metadata.append(newmi)
            else:
                need_auto.append(id)
                if not exclude_auto:
                    ans.append(None)
                    modified_metadata.append(None)
        return ans, need_auto

    def get_preferred_formats(self, rows, formats, paths=False, set_metadata=False, specific_format=None, exclude_auto=False):
        from calibre.ebooks.metadata.meta import set_metadata as _set_metadata

        db = self.db
        assert db is not None
        ans = []
        need_auto = []
        if specific_format is not None:
            formats = [specific_format.lower()]
        for row in (row.row() for row in rows):
            format = None
            fmts = db.formats(row)
            if not fmts:
                fmts = ''
            db_formats = set(fmts.lower().split(','))
            available_formats = {f.lower() for f in formats}
            u = available_formats.intersection(db_formats)
            for f in formats:
                if f.lower() in u:
                    format = f
                    break
            if format is not None:
                pt = PersistentTemporaryFile(suffix='.' + format)
                db.copy_format_to(id, format, pt, index_is_id=True)
                pt.seek(0)
                if set_metadata:
                    _set_metadata(pt, db.get_metadata(row, get_cover=True, cover_as_data=True), format)
                pt.close() if paths else pt.seek(0)
                ans.append(pt)
            else:
                need_auto.append(row)
                if not exclude_auto:
                    ans.append(None)
        return ans, need_auto

    def id(self, row):
        db = self.db
        assert db is not None
        return db.id(getattr(row, 'row', lambda: row)())

    def authors(self, row_number):
        db = self.db
        assert db is not None
        return db.authors(row_number)

    def title(self, row_number):
        db = self.db
        assert db is not None
        return db.title(row_number)

    def rating(self, row_number):
        db = self.db
        assert db is not None
        ans = db.rating(row_number)
        ans = ans / 2 if ans else 0
        return int(ans)

    def cover(self, row_number):
        db = self.db
        assert db is not None
        data = None
        try:
            data = db.cover(row_number)
        except IndexError:  # Happens if database has not yet been refreshed
            pass
        except MemoryError:
            raise ValueError(_('The cover for the book %s is too large, cannot load it. Resize or delete it.') % db.title(row_number))

        if not data:
            return self.default_image
        img = QImage()
        load_from_data_without_gil(img, data)
        return self.default_image if img.isNull() else img

    def build_data_convertors(self):
        rating_fields = {}
        bool_fields = set()

        def renderer(field, decorator=False):
            db = self.db
            assert db is not None
            idfunc = db.id
            if field == 'id':

                def func(idx):
                    return idfunc(idx)

                return func
            fffunc = db.new_api.fast_field_for
            field_obj = db.new_api.fields[field]
            m = field_obj.metadata.copy()
            if 'display' not in m:
                m['display'] = {}
            dt = m['datatype']

            if decorator == 'bool':
                bool_fields.add(field)
                bt = db.new_api.pref('bools_are_tristate')
                bn = self.bool_no_icon
                by = self.bool_yes_icon

                if dt != 'bool':

                    def func(idx):
                        val = fffunc(field_obj, idfunc(idx))
                        if val is None:
                            return None
                        val = force_to_bool(val)
                        if val is None:
                            return None
                        return by if val else bn

                elif m['display'].get('bools_show_icons', True):

                    def func(idx):
                        val = force_to_bool(fffunc(field_obj, idfunc(idx)))
                        if val is None:
                            return None if bt else bn
                        return by if val else bn

                else:

                    def func(idx):
                        return None

            elif field == 'size':
                sz_mult = 1 / (1024**2)

                def func(idx):
                    val = fffunc(field_obj, idfunc(idx), default_value=0) or 0
                    if val == 0:
                        return None
                    ans = f'{val * sz_mult:.1f}'
                    return '<0.1' if ans == '0.0' else ans

            elif field == 'languages':

                def func(idx):
                    return ', '.join(calibre_langcode_to_name(x) for x in fffunc(field_obj, idfunc(idx)))

            elif field == 'pages':

                def func(idx):
                    book_id = idfunc(idx)
                    ans = fffunc(field_obj, book_id, 0)
                    match ans:
                        case 0:
                            return self.get_pages_description_for_zero_page(book_id)
                        case -1:
                            return _('None')
                        case -2:
                            return _('Error')
                        case -3:
                            return _('DRM')
                        case _:
                            return str(ans)

            elif field == 'ondevice' and decorator:
                by = self.bool_yes_icon
                bb = self.bool_blank_icon

                def func(idx):
                    return by if fffunc(field_obj, idfunc(idx)) else bb

            elif dt in {'text', 'comments', 'composite', 'enumeration'}:
                if m['is_multiple']:
                    jv = m['is_multiple']['list_to_ui']
                    do_sort = '&' not in jv
                    if field_obj.is_composite:
                        if do_sort:
                            sv = m['is_multiple']['cache_to_list']

                            def func(idx):
                                val = fffunc(field_obj, idfunc(idx), default_value='') or ''
                                return jv.join(sorted((x.strip() for x in val.split(sv)), key=sort_key))

                        else:

                            def func(idx):
                                return fffunc(field_obj, idfunc(idx), default_value='')

                    elif do_sort:

                        def func(idx):
                            return jv.join(sorted(fffunc(field_obj, idfunc(idx), default_value=()), key=sort_key))

                    else:

                        def func(idx):
                            return jv.join(fffunc(field_obj, idfunc(idx), default_value=()))

                elif dt in {'text', 'composite', 'enumeration'} and m['display'].get('use_decorations', False):

                    def func(idx):
                        text = fffunc(field_obj, idfunc(idx))
                        return (text) if force_to_bool(text) is None else None

                else:

                    def func(idx):
                        return fffunc(field_obj, idfunc(idx), default_value='')

            elif dt == 'datetime':

                def func(idx):
                    val = fffunc(field_obj, idfunc(idx), default_value=UNDEFINED_DATE)
                    return None if is_date_undefined(val) else qt_from_dt(val)

            elif dt == 'rating':
                rating_fields[field] = m['display'].get('allow_half_stars', False)

                def func(idx):
                    return int(fffunc(field_obj, idfunc(idx), default_value=0))

            elif dt == 'series':
                sidx_field = db.new_api.fields[field + '_index']

                def func(idx):
                    book_id = idfunc(idx)
                    series = fffunc(field_obj, book_id, default_value=False)
                    if series:
                        return f'{series} [{fmt_sidx(fffunc(sidx_field, book_id, default_value=1.0))}]'
                    return None

            elif dt in {'int', 'float'}:
                fmt = m['display'].get('number_format', None)

                def func(idx):
                    val = fffunc(field_obj, idfunc(idx))
                    if val is None:
                        return None
                    if fmt:
                        try:
                            return fmt.format(val)
                        except TypeError, ValueError, AttributeError, IndexError, KeyError:
                            pass
                    return val

            elif dt == 'bool':
                if m['display'].get('bools_show_text', False):

                    def func(idx):
                        v = fffunc(field_obj, idfunc(idx))
                        return None if v is None else (_('Yes') if v else _('No'))

                else:

                    def func(idx):
                        return None

            else:

                def func(idx):
                    return None

            return func

        self.dc = {f: renderer(f) for f in self.orig_headers.keys()}
        self.dc_decorator = {f: renderer(f, True) for f in ('ondevice',)}

        for col in self.custom_columns:
            self.dc[col] = renderer(col)
            m = self.custom_columns[col]
            dt = m['datatype']
            mult = m['is_multiple']
            if dt in {'text', 'composite', 'enumeration'} and not mult and m['display'].get('use_decorations', False):
                self.dc_decorator[col] = renderer(col, 'bool')
            elif dt == 'bool':
                self.dc_decorator[col] = renderer(col, 'bool')

        tc = self.dc.copy()

        def stars_tooltip(func, allow_half=True):
            def f(idx):
                val = int(func(idx))
                ans = str(val // 2)
                if allow_half and val % 2:
                    ans += '.5'
                return _('%s stars') % ans

            return f

        def bool_tooltip(key):
            def f(idx):
                db = self.db
                assert db is not None
                return db.new_api.fast_field_for(db.new_api.fields[key], db.id(idx))

            return f

        def template_tooltip(key, orig_tt_func):
            def f(idx):
                try:
                    db = self.db
                    assert db is not None
                    template = db.new_api.pref('column_tooltip_templates', {}).get(key, '')
                    # Simulate what Qt does to get the text for non-string data
                    # types so that idx.data(Qt.ItemDataRole.ToolTipRole)
                    # always returns a string.
                    orig_obj = orig_tt_func(idx)
                    match orig_obj:
                        case None:
                            v = ''
                        case str():
                            v = orig_obj
                        case bool():
                            v = str(orig_obj)
                        case int() | float() | QDateTime():
                            v = QLocale().toString(orig_obj)
                        case _:
                            v = f'unsupported type {type(orig_obj)}'
                    if template:
                        global_vars = {'column_lookup_name': key, 'original_text': v}
                        mi = db.new_api.get_proxy_metadata(db.data.index_to_id(idx))
                        return self.formatter.safe_format(template, {}, _('tooltip template error'), mi, global_vars=global_vars)
                    return v
                except Exception as e:
                    return str(e)

            return f

        for f, allow_half in rating_fields.items():
            tc[f] = stars_tooltip(self.dc[f], allow_half)
        for f in bool_fields:
            tc[f] = bool_tooltip(f)
        # Make all columns use the template tooltip renderer. This avoids the
        # "first time" problem where a template tooltip is added but not
        # used until calibre is restarted
        for f in self.column_map:
            tc[f] = template_tooltip(f, tc[f])
        # build a index column to data converter map, to remove the string lookup in the data loop
        self.column_to_dc_map = [self.dc[col] for col in self.column_map]
        self.column_to_tc_map = [tc[col] for col in self.column_map]
        self.column_to_dc_decorator_map = [self.dc_decorator.get(col, None) for col in self.column_map]

    def data(self, index, role=...):
        db = self.db
        assert db is not None
        if db.new_api.is_doing_rebuild_or_vacuum:
            return None
        col = index.column()
        # in obscure cases where custom columns are both edited and added, for a time
        # the column map does not accurately represent the screen. In these cases,
        # we will get asked to display columns we don't know about. Must test for this.
        if col >= len(self.column_to_dc_map) or col < 0:
            return None
        try:
            return self.actual_data(index, role, col)
        except Exception:
            import traceback

            traceback.print_exc()

    def actual_data(self, index, role, col):
        if role == Qt.ItemDataRole.DisplayRole:
            rules = self.db_prefs['column_icon_rules']
            if rules:
                key = self.column_map[col]
                id_ = None
                fmts = []
                for kind, k, fmt in rules:
                    if k == key and kind in {'icon_only', 'icon_only_composed'}:
                        if id_ is None:
                            id_ = self.id(index)
                            self.column_icon.mi = None
                        fmts.append((kind, fmt))

                if fmts:
                    cache_index = key + ':DisplayRole'
                    ccicon = self.column_icon(
                        id_,
                        fmts,
                        cache_index,
                        self.db,
                        self.icon_cache,
                        self.icon_bitmap_cache,
                        self.icon_template_cache,
                    )
                    if ccicon is not None:
                        return None
                    self.icon_cache[id_][cache_index] = None
            return self.column_to_dc_map[col](index.row())
        elif role == Qt.ItemDataRole.ToolTipRole:
            return self.column_to_tc_map[col](index.row())
        elif role == Qt.ItemDataRole.EditRole:
            return self.column_to_dc_map[col](index.row())
        elif role == Qt.ItemDataRole.BackgroundRole:
            if self.id(index) in self.ids_to_highlight_set:
                return QColor('#027524') if qapplication_or_fail().is_dark_theme else QColor('#b4ecb4')
        elif role == Qt.ItemDataRole.ForegroundRole:
            key = self.column_map[col]
            id_ = self.id(index)
            self.column_color.mi = None

            for k, fmt in self.db_prefs['column_color_rules']:
                if k == key:
                    ccol = self.column_color(id_, key, fmt, self.db, self.color_cache, self.color_template_cache)
                    if ccol is not None:
                        return ccol

            if self.is_custom_column(key) and self.custom_columns[key]['datatype'] == 'enumeration':
                cc = self.custom_columns[self.column_map[col]]['display']
                colors = cc.get('enum_colors', [])
                values = cc.get('enum_values', [])
                txt = str(index.data(Qt.ItemDataRole.DisplayRole) or '')
                if len(colors) > 0 and txt in values:
                    try:
                        color = QColor(colors[values.index(txt)])
                        if color.isValid():
                            self.column_color.mi = None
                            return color
                    except Exception:
                        pass

            if self.color_row_fmt_cache is None:
                self.color_row_fmt_cache = tuple(fmt for key, fmt in self.db_prefs['column_color_rules'] if key == color_row_key)
            for fmt in self.color_row_fmt_cache:
                ccol = self.column_color(id_, color_row_key, fmt, self.db, self.color_cache, self.color_template_cache)
                if ccol is not None:
                    return ccol

            self.column_color.mi = None
            return None
        elif role == Qt.ItemDataRole.DecorationRole:
            default_icon = None
            if self.column_to_dc_decorator_map[col] is not None:
                default_icon = self.column_to_dc_decorator_map[index.column()](index.row())
            rules = self.db_prefs['column_icon_rules']
            if rules:
                key = self.column_map[col]
                id_ = None
                need_icon_with_text = False
                fmts = []
                for kind, k, fmt in rules:
                    if k == key and kind.startswith('icon'):
                        if id_ is None:
                            id_ = self.id(index)
                            self.column_icon.mi = None
                        fmts.append((kind, fmt))
                        if kind in ('icon', 'icon_composed'):
                            need_icon_with_text = True
                if fmts:
                    cache_index = key + ':DecorationRole'
                    ccicon = self.column_icon(
                        id_,
                        fmts,
                        cache_index,
                        self.db,
                        self.icon_cache,
                        self.icon_bitmap_cache,
                        self.icon_template_cache,
                    )
                    if ccicon is not None:
                        return ccicon
                    if need_icon_with_text and default_icon is None:
                        self.icon_cache[id_][cache_index] = self.bool_blank_icon
                        return self.bool_blank_icon
                    self.icon_cache[id_][cache_index] = None
            return default_icon
        elif role == Qt.ItemDataRole.TextAlignmentRole:
            cname = self.column_map[index.column()]
            ans = Qt.AlignmentFlag.AlignVCenter | ALIGNMENT_MAP[self.alignment_map.get(cname, 'left')]
            return int(ans)  # https://bugreports.qt.io/browse/PYSIDE-1974
        elif role == Qt.ItemDataRole.FontRole and self.styled_columns:
            cname = self.column_map[index.column()]
            return self.styled_columns.get(cname)
        elif role == Qt.ItemDataRole.UserRole:
            return self.id(index)
        # elif role == Qt.ItemDataRole.ToolTipRole and index.isValid():
        #     if self.column_map[index.column()] in self.editable_cols:
        #         return (_("Double click to <b>edit</b> me<br><br>"))
        return None

    def headerData(self, section, orientation, role=...):
        db = self.db
        if db is None:
            return None
        if orientation == Qt.Orientation.Horizontal:
            if section >= len(self.column_map):  # same problem as in data, the column_map can be wrong
                return None
            if role == Qt.ItemDataRole.ToolTipRole:
                ht = self.column_map[section]
                title = self.headers[ht]
                fm = db.field_metadata[self.column_map[section]]
                if ht == 'timestamp':  # change help text because users know this field as 'date'
                    ht = 'date'
                if fm['is_category']:
                    is_cat = '<br><br>' + prepare_string_for_xml(_('Click in this column and press Q to Quickview books with the same "%s"') % ht)
                else:
                    is_cat = ''
                cust_desc = ''
                if fm['is_custom']:
                    cust_desc = fm['display'].get('description', '')
                    if cust_desc:
                        cust_desc = (
                            '<br><b>{}</b>'.format(_('Description:')) + '<span style="white-space:pre-wrap"> ' + prepare_string_for_xml(cust_desc) + '</span>'
                        )
                return '<b>{}</b>: {}'.format(
                    prepare_string_for_xml(title),
                    _('The lookup/search name is <i>{0}</i>').format(ht) + cust_desc + is_cat,
                )
            if role == Qt.ItemDataRole.DisplayRole:
                return self.headers[self.column_map[section]]
            return None
        if role == Qt.ItemDataRole.ToolTipRole and orientation == Qt.Orientation.Vertical:
            col = db.field_metadata['marked']['rec_index']
            marked = db.data[section][col]
            if marked is not None:
                s = _('Marked with text "{0}"').format(marked) if marked != 'true' else _('Marked without text')
            else:
                s = ''
            if DEBUG:
                col = db.field_metadata['uuid']['rec_index']
                s += ('\n' if s else '') + _('This book\'s UUID is "{0}"').format(db.data[section][col])
            return s

        if role == Qt.ItemDataRole.DisplayRole:  # orientation is vertical
            return section + 1
        if role == Qt.ItemDataRole.DecorationRole:
            try:
                m = db.data.get_marked(db.data.index_to_id(section))
                if m:
                    i = self.marked_icon if m == 'true' else self.marked_text_icon_for(m)
                else:
                    i = self.row_decoration
                return i
            except ValueError, IndexError:
                pass
        return None

    def flags(self, index):
        flags = QAbstractTableModel.flags(self, index)
        if index.isValid():
            colhead = self.column_map[index.column()]
            if colhead in self.editable_cols:
                flags |= Qt.ItemFlag.ItemIsEditable
            elif self.is_custom_column(colhead):
                if self.custom_columns[colhead]['is_editable']:
                    flags |= Qt.ItemFlag.ItemIsEditable
        return flags

    def set_custom_column_data(self, row, colhead, value):
        db = self.db
        assert db is not None
        cc = self.custom_columns[colhead]
        typ = cc['datatype']
        label = db.field_metadata.key_to_label(colhead)
        s_index = None
        if typ in ('text', 'comments'):
            val = str(value or '').strip()
            val = val or None
        elif typ == 'enumeration':
            val = str(value or '').strip()
            if not val:
                val = None
        elif typ == 'bool':
            val = value if value is None else bool(value)
        elif typ == 'rating':
            val = max(0, min(int(value or 0), 10))
        elif typ in ('int', 'float'):
            if value == 0:
                val = '0'
            else:
                val = str(value or '').strip()
            if not val:
                val = None
        elif typ == 'datetime':
            val = value
            if val is None:
                val = None
            else:
                if not val.isValid():
                    return False
                val = qt_to_dt(val, as_utc=False)
        elif typ == 'series':
            val = str(value or '').strip()
            if val:
                pat = re.compile(r'\[([.0-9]+)\]')
                match = pat.search(val)
                if match is not None:
                    s_index = float(match.group(1))
                    val = pat.sub('', val).strip()
                elif val:
                    # it is OK to leave s_index == None when using 'no_change'
                    if tweaks['series_index_auto_increment'] != 'const' and tweaks['series_index_auto_increment'] != 'no_change':
                        s_index = db.get_next_cc_series_num_for(val, label=label, num=None)
        elif typ == 'composite':
            tmpl = str(value or '').strip()
            disp = cc['display']
            disp['composite_template'] = tmpl
            db.set_custom_column_metadata(cc['colnum'], display=disp, update_last_modified=True)
            self.refresh(reset=False)
            self.research(reset=True)
            return True

        id = db.id(row)
        books_to_refresh = {id}
        books_to_refresh |= db.set_custom(id, val, extra=s_index, label=label, num=None, append=False, notify=True, allow_case_change=True)
        self.refresh_ids(list(books_to_refresh), current_row=row)
        return True

    def setData(self, index, value, role=...):
        from calibre.gui2.ui import get_gui

        gui = get_gui()
        if gui is not None and gui.shutting_down:
            return False
        if role == Qt.ItemDataRole.EditRole:
            from calibre.gui2.ui import get_gui

            try:
                return self._set_data(index, value)
            except OSError as err:
                import traceback

                traceback.print_exc()
                det_msg = traceback.format_exc()
                gui = get_gui()
                assert gui is not None
                if gui.show_possible_sharing_violation(err, det_msg):
                    return False
                error_dialog(
                    get_gui(),
                    _('Failed to set data'),
                    _('Could not set data, click "Show details" to see why.'),
                    det_msg=traceback.format_exc(),
                    show=True,
                )
            except Exception:
                import traceback

                traceback.print_exc()
                error_dialog(
                    get_gui(),
                    _('Failed to set data'),
                    _('Could not set data, click "Show details" to see why.'),
                    det_msg=traceback.format_exc(),
                    show=True,
                )
        return False

    def _set_data(self, index, value):
        db = self.db
        assert db is not None
        row, col = index.row(), index.column()
        column = self.column_map[col]
        if self.is_custom_column(column):
            if not self.set_custom_column_data(row, column, value):
                return False
        else:
            if column not in self.editable_cols:
                return False
            val = int(value) if column == 'rating' else value if column in ('timestamp', 'pubdate') else re.sub(r'\s', ' ', str(value or '').strip())
            id = db.id(row)
            books_to_refresh = {id}
            if column == 'rating':
                val = max(0, min(int(val or 0), 10))
                db.set_rating(id, val)
            elif column == 'series':
                val = str(val).strip()
                if not val:
                    books_to_refresh |= db.set_series(id, val, allow_case_change=True)
                    db.set_series_index(id, 1.0)
                else:
                    pat = re.compile(r'\[([.0-9]+)\]')
                    match = pat.search(val)
                    if match is not None:
                        db.set_series_index(id, float(match.group(1)))
                        val = pat.sub('', val).strip()
                    elif val:
                        if tweaks['series_index_auto_increment'] != 'const' and tweaks['series_index_auto_increment'] != 'no_change':
                            ni = db.get_next_series_num_for(val)
                            if ni != 1:
                                db.set_series_index(id, ni)
                    if val:
                        books_to_refresh |= db.set_series(id, val, allow_case_change=True)
            elif column == 'timestamp':
                if not isinstance(val, QDateTime) or not val.isValid():
                    return False
                db.set_timestamp(id, qt_to_dt(val, as_utc=False))
            elif column == 'pubdate':
                if not isinstance(val, QDateTime) or not val.isValid():
                    return False
                db.set_pubdate(id, qt_to_dt(val, as_utc=False))
            elif column == 'languages':
                val = str(val).split(',')
                db.set_languages(id, val)
            else:
                if column == 'authors' and val:
                    val = authors_to_string(string_to_authors(val))
                books_to_refresh |= db.set(row, column, val, allow_case_change=True)
                if column == 'title':
                    if (lang := db.languages(row)) is not None:
                        lang = lang.partition(',')[0]
                    db.set(row, 'sort', title_sort(val, lang=lang), allow_case_change=True)
            self.refresh_ids(list(books_to_refresh), row)
        self.dataChanged.emit(index, index)
        return True


# }}}


class OnDeviceSearch(SearchQueryParser):  # {{{
    USABLE_LOCATIONS = [
        'all',
        'author',
        'authors',
        'collections',
        'format',
        'formats',
        'title',
        'inlibrary',
        'tags',
        'search',
    ]

    def __init__(self, model):
        SearchQueryParser.__init__(self, locations=self.USABLE_LOCATIONS)
        self.model = model

    def universal_set(self):
        return set(range(len(self.model.db)))

    def get_matches(self, location, query, candidates=None):
        location = location.lower().strip()
        if location == 'authors':
            location = 'author'

        matchkind = CONTAINS_MATCH
        if len(query) > 1:
            if query.startswith('\\'):
                query = query[1:]
            elif query.startswith('='):
                matchkind = EQUALS_MATCH
                query = query[1:]
            elif query.startswith('~'):
                matchkind = REGEXP_MATCH
                query = query[1:]
        if matchkind != REGEXP_MATCH:  # leave case in regexps because it can be significant e.g. \S \W \D
            query = query.lower()

        if location not in self.USABLE_LOCATIONS:
            return set()
        matches = set()
        all_locs = set(self.USABLE_LOCATIONS) - {'all', 'tags', 'search'}
        locations = all_locs if location == 'all' else [location]
        q = {
            'title': lambda x: getattr(x, 'title').lower(),
            'author': lambda x: ' & '.join(getattr(x, 'authors')).lower(),
            'collections': lambda x: ','.join(getattr(x, 'device_collections')).lower(),
            'format': lambda x: os.path.splitext(x.path)[1].lower(),
            'inlibrary': lambda x: getattr(x, 'in_library'),
            'tags': lambda x: getattr(x, 'tags', []),
        }
        for x in ('author', 'format'):
            q[x + 's'] = q[x]
        upf = prefs['use_primary_find_in_search']
        for index, row in enumerate(self.model.db):
            for locvalue in locations:
                accessor = q[locvalue]
                if query == 'true':
                    if accessor(row):
                        matches.add(index)
                    continue
                if query == 'false':
                    if not accessor(row):
                        matches.add(index)
                    continue
                if locvalue == 'inlibrary':
                    continue  # this is bool, so can't match below
                try:
                    # Can't separate authors because comma is used for name sep and author sep
                    # Exact match might not get what you want. For that reason, turn author
                    # exactmatch searches into contains searches.
                    if locvalue == 'author' and matchkind == EQUALS_MATCH:
                        m = CONTAINS_MATCH
                    else:
                        m = matchkind

                    vals = accessor(row)
                    if vals is None:
                        vals = ''
                    if isinstance(vals, (str, bytes)):
                        vals = vals.split(',') if locvalue == 'collections' else [vals]
                    if _match(query, vals, m, use_primary_find_in_search=upf):
                        matches.add(index)
                        break
                except ValueError:  # Unicode errors
                    traceback.print_exc()
        return matches


# }}}


class DeviceDBSortKeyGen:  # {{{
    def __init__(self, attr, keyfunc, db):
        self.attr = attr
        self.db = db
        self.keyfunc = keyfunc

    def __call__(self, x) -> bytes | str:
        try:
            ans = self.keyfunc(getattr(self.db[x], self.attr))
        except Exception:
            ans = ''
        return ans


# }}}


class DeviceBooksModel(BooksModel):  # {{{
    booklist_dirtied = pyqtSignal()
    upload_collections = pyqtSignal(object)
    resize_rows = pyqtSignal()
    db: BookList
    map: list[int]
    sorted_map: list[int]

    def __init__(self, parent):
        BooksModel.__init__(self, parent)
        self.db = BookList()
        self.map = []
        self.sorted_map = []
        self.sorted_on = DEFAULT_SORT
        self.sort_history = [self.sorted_on]
        self.unknown = _('Unknown')
        self.column_map = ['inlibrary', 'title', 'authors', 'timestamp', 'size', 'collections']
        self.headers = {
            'inlibrary': _('In Library'),
            'title': _('Title'),
            'authors': _('Author(s)'),
            'timestamp': _('Date'),
            'size': _('Size'),
            'collections': _('Collections'),
        }
        self.marked_for_deletion = {}
        self.search_engine = OnDeviceSearch(self)
        self.editable = ['title', 'authors', 'collections']
        self.book_in_library = None
        self.sync_icon = QIcon.cached_icon('sync.png')

    def counts(self):
        return Counts(len(self.db), len(self.db), len(self.map))

    def count_changed(self, *args):
        self.count_changed_signal.emit(len(self.db))

    def mark_for_deletion(self, job, rows, rows_are_ids=False):
        db_indices = rows if rows_are_ids else self.indices(rows)
        db_items = [self.db[i] for i in db_indices if -1 < i < len(self.db)]
        self.marked_for_deletion[job] = db_items
        if rows_are_ids:
            self.beginResetModel(), self.endResetModel()
        else:
            for row in rows:
                indices = self.row_indices(row)
                self.dataChanged.emit(indices[0], indices[-1])

    def find_item_in_db(self, item):
        idx = None
        db = self.db
        assert db is not None
        try:
            idx = db.index(item)
        except Exception:
            path = getattr(item, 'path', None)
            if path:
                for i, x in enumerate(db):
                    if getattr(x, 'path', None) == path:
                        idx = i
                        break
        return idx

    def deletion_done(self, job, succeeded=True):
        db_items = self.marked_for_deletion.pop(job, [])
        rows = []
        for item in db_items:
            idx = self.find_item_in_db(item)
            if idx is not None:
                try:
                    rows.append(self.map.index(idx))
                except ValueError:
                    pass

        for row in rows:
            if not succeeded:
                indices = self.row_indices(self.index(row, 0))
                self.dataChanged.emit(indices[0], indices[-1])
        self.count_changed()

    def paths_deleted(self, paths):
        self.map = list(range(len(self.db)))
        self.resort(False)
        self.research(True)
        self.count_changed()

    def is_row_marked_for_deletion(self, row):
        try:
            item = self.db[self.map[row]]
        except IndexError:
            return False

        path = getattr(item, 'path', None)
        for items in self.marked_for_deletion.values():
            for x in items:
                if x is item or (path and path == getattr(x, 'path', None)):
                    return True
        return False

    def clear_ondevice(self, db_ids, to_what=None):
        for data in self.db:
            if data is None:
                continue
            app_id = getattr(data, 'application_id', None)
            if app_id is not None and app_id in db_ids:
                data.in_library = to_what
            self.beginResetModel(), self.endResetModel()

    def flags(self, index):
        if self.is_row_marked_for_deletion(index.row()):
            return Qt.ItemFlag.NoItemFlags
        flags = QAbstractTableModel.flags(self, index)
        if index.isValid():
            cname = self.column_map[index.column()]
            if cname in self.editable and (
                cname != 'collections'
                or (
                    callable(getattr(self.db, 'supports_collections', None))
                    and self.db.supports_collections()
                    and device_prefs['manage_device_metadata'] == 'manual'
                )
            ):
                flags |= Qt.ItemFlag.ItemIsEditable
        return flags

    def search(self, text, reset=True):
        # This should not be here, but since the DeviceBooksModel does not
        # implement count_changed and I am too lazy to fix that, this kludge
        # will have to do
        self.resize_rows.emit()

        if not text or not text.strip():
            self.map = list(range(len(self.db)))
        else:
            try:
                matches = self.search_engine.parse(text)
            except ParseException:
                self.searched.emit(False)
                return

            self.map = []
            for i in range(len(self.db)):
                if i in matches:
                    self.map.append(i)
        self.resort(reset=False)
        if reset:
            self.beginResetModel(), self.endResetModel()
        self.last_search = text
        if self.last_search:
            self.searched.emit(True)
        self.count_changed()

    def research(self, reset=True):
        self.search(self.last_search, reset)

    def sort(self, column, order=..., reset=True):
        if not isinstance(order, Qt.SortOrder):
            order = Qt.SortOrder.AscendingOrder if order else Qt.SortOrder.DescendingOrder
        descending = order != Qt.SortOrder.AscendingOrder
        cname = self.column_map[column]

        def author_key(x):
            try:
                ax = self.db[x].author_sort
                if not ax:
                    raise Exception('')
            except Exception:
                try:
                    ax = authors_to_string(self.db[x].authors)
                except Exception:
                    ax = ''
            try:
                return sort_key(ax)
            except Exception:
                return ax

        keygen = {
            'title': ('title_sorter', lambda x: sort_key(x) if x else b''),
            'authors': author_key,
            'size': ('size', int),
            'timestamp': ('datetime', functools.partial(dt_factory, assume_utc=True)),
            'collections': ('device_collections', lambda x: sorted(x, key=sort_key)),
            'inlibrary': ('in_library', lambda x: x or ''),
        }[cname]
        keygen = keygen if callable(keygen) else DeviceDBSortKeyGen(keygen[0], keygen[1], self.db)
        keygen_fn = cast(Callable[[int], bytes | str], keygen)
        self.map.sort(key=keygen_fn, reverse=descending)
        if len(self.map) == len(self.db):
            self.sorted_map = list(self.map)
        else:
            self.sorted_map = list(range(len(self.db)))
            self.sorted_map.sort(key=keygen_fn, reverse=descending)
        self.sorted_on = (self.column_map[column], not descending)
        self.sort_history.insert(0, self.sorted_on)
        if hasattr(keygen, 'db'):
            setattr(keygen, 'db', None)
        if reset:
            self.beginResetModel(), self.endResetModel()

    def resort(self, reset=True):
        if self.sorted_on:
            self.sort(self.column_map.index(self.sorted_on[0]), self.sorted_on[1], reset=False)
        if reset:
            self.beginResetModel(), self.endResetModel()

    def columnCount(self, parent: QModelIndex = QModelIndex()):
        if parent is not None and parent.isValid():
            return 0
        return len(self.column_map)

    def rowCount(self, parent: QModelIndex = QModelIndex()):
        if parent is not None and parent.isValid():
            return 0
        return len(self.map)

    def set_database(self, db):
        self.custom_columns = {}
        self.zero_page_cache = {}
        self.db = db
        self.map = list(range(len(db)))
        self.research(reset=False)
        self.resort()
        self.count_changed()

    def cover(self, row_number):
        item = self.db[self.map[row_number]]
        cdata = item.thumbnail
        img = QImage()
        if cdata is not None:
            if hasattr(cdata, 'image_path'):
                img.load(cdata.image_path)
            elif cdata:
                if isinstance(cdata, (tuple, list)):
                    load_from_data_without_gil(img, cdata[-1])
                else:
                    load_from_data_without_gil(img, cdata)
        if img.isNull():
            img = self.default_image
        return img

    def get_book_display_info(self, idx):
        from calibre.ebooks.metadata.book.base import Metadata

        item = self.db[self.map[idx]]
        cover = self.cover(idx)
        if cover is self.default_image:
            cover = None
        title = item.title
        if not title:
            title = _('Unknown')
        au = item.authors
        if not au:
            au = [_('Unknown')]
        mi = Metadata(title, au)
        mi.cover_data = ('jpg', cover)
        fmt = _('Unknown')
        ext = os.path.splitext(item.path)[1]
        if ext:
            fmt = ext[1:].lower()
        mi.formats = [fmt]
        mi.path = item.path or None
        dt = dt_factory(item.datetime, assume_utc=True)
        mi.timestamp = dt
        mi.device_collections = list(item.device_collections)
        mi.tags = list(getattr(item, 'tags', []))
        mi.comments = getattr(item, 'comments', None)
        mi.book_display_info_is_from_device = True
        series = getattr(item, 'series', None)
        if series:
            sidx = getattr(item, 'series_index', 0)
            mi.series = series
            mi.series_index = sidx
        return mi

    def current_changed(self, current, previous, emit_signal=True):
        if current.isValid():
            idx = current.row()
            try:
                self.db[self.map[idx]]
            except Exception:
                return  # can happen if the device is ejected
            try:
                data = self.get_book_display_info(idx)
            except Exception:
                import traceback

                error_dialog(
                    None,
                    _('Unhandled error'),
                    _('Failed to read book data from calibre library. Click "Show details" for more information'),
                    det_msg=traceback.format_exc(),
                    show=True,
                )
            else:
                if emit_signal:
                    self.new_bookdisplay_data.emit(data)
                else:
                    return data

    def paths(self, rows):
        return [self.db[self.map[r.row()]].path for r in rows]

    def id(self, row):
        row = getattr(row, 'row', lambda: row)()
        try:
            return self.db[self.map[row]].path
        except Exception:
            return None

    def paths_for_db_ids(self, db_ids, as_map=False):
        res = defaultdict(list) if as_map else []
        for r, b in enumerate(self.db):
            if b.application_id in db_ids:
                if as_map:
                    res[b.application_id].append(b)
                else:
                    assert isinstance(res, list)
                    res.append((r, b))
        return res

    def get_collections_with_ids(self):
        collections = set()
        for book in self.db:
            if book.device_collections is not None:
                collections.update(set(book.device_collections))
        self.collections = []
        result = []
        for i, collection in enumerate(collections):
            result.append((i, collection))
            self.collections.append(collection)
        return result

    def rename_collection(self, old_id, new_name):
        old_name = self.collections[old_id]
        for book in self.db:
            if book.device_collections is None:
                continue
            if old_name in book.device_collections:
                book.device_collections.remove(old_name)
                if new_name not in book.device_collections:
                    book.device_collections.append(new_name)

    def delete_collection_using_id(self, old_id):
        old_name = self.collections[old_id]
        for book in self.db:
            if book.device_collections is None:
                continue
            if old_name in book.device_collections:
                book.device_collections.remove(old_name)

    def indices(self, rows):
        """
        Return indices into underlying database from rows
        """
        return [self.map[r.row()] for r in rows]

    def data(self, index, role=...):
        row, column = index.row(), index.column()
        cname = self.column_map[column]
        if role in (Qt.ItemDataRole.DisplayRole, Qt.ItemDataRole.EditRole):
            if cname == 'title':
                text = self.db[self.map[row]].title
                if not text:
                    text = self.unknown
                return text
            elif cname == 'authors':
                au = self.db[self.map[row]].authors
                if not au:
                    au = [_('Unknown')]
                return authors_to_string(au)
            elif cname == 'size':
                size = self.db[self.map[row]].size
                if not isinstance(size, numbers.Number):
                    size = 0
                return human_readable(size)
            elif cname == 'timestamp':
                dt = self.db[self.map[row]].datetime
                try:
                    dt = dt_factory(dt, assume_utc=True, as_utc=False)
                except OverflowError:
                    dt = dt_factory(time.gmtime(), assume_utc=True, as_utc=False)
                return strftime(TIME_FMT, dt.timetuple())
            elif cname == 'collections':
                tags = self.db[self.map[row]].device_collections
                if tags:
                    tags.sort(key=sort_key)
                    return ', '.join(tags)
            elif DEBUG and cname == 'inlibrary':
                return self.db[self.map[row]].in_library
        elif role == Qt.ItemDataRole.ToolTipRole and index.isValid():
            if column == 0 and hasattr(self.db[self.map[row]], 'in_library_waiting'):
                return _('Waiting for metadata to be updated')
            if self.is_row_marked_for_deletion(row):
                return _('Marked for deletion')
            if cname in ['title', 'authors'] or (
                cname == 'collections' and (callable(getattr(self.db, 'supports_collections', None)) and self.db.supports_collections())
            ):
                return _('Double click to <b>edit</b> me<br><br>')
        elif role == Qt.ItemDataRole.DecorationRole and cname == 'inlibrary':
            if hasattr(self.db[self.map[row]], 'in_library_waiting'):
                return self.sync_icon
            elif self.db[self.map[row]].in_library:
                return self.bool_yes_icon
            elif self.db[self.map[row]].in_library is not None:
                return self.bool_no_icon
        elif role == Qt.ItemDataRole.TextAlignmentRole:
            cname = self.column_map[index.column()]
            ans = Qt.AlignmentFlag.AlignVCenter | ALIGNMENT_MAP[self.alignment_map.get(cname, 'left')]
            return int(ans)  # https://bugreports.qt.io/browse/PYSIDE-1974
        return None

    def headerData(self, section, orientation, role=...):
        if role == Qt.ItemDataRole.ToolTipRole and orientation == Qt.Orientation.Horizontal:
            cname = self.column_map[section]
            text = self.headers[cname]
            return '<b>{}</b>: {}'.format(
                prepare_string_for_xml(text),
                prepare_string_for_xml(_('The lookup/search name is')) + f' <i>{self.column_map[section]}</i>',
            )
        if DEBUG and role == Qt.ItemDataRole.ToolTipRole and orientation == Qt.Orientation.Vertical:
            return _('This book\'s UUID is "{0}"').format(self.db[self.map[section]].uuid)
        if role != Qt.ItemDataRole.DisplayRole:
            return None
        if orientation == Qt.Orientation.Horizontal:
            cname = self.column_map[section]
            text = self.headers[cname]
            return text
        else:
            return section + 1

    def setData(self, index, value, role=...):
        from calibre.gui2.ui import get_gui

        gui = get_gui()
        if gui is not None and gui.shutting_down:
            return False
        done = False
        if role == Qt.ItemDataRole.EditRole:
            row, column = index.row(), index.column()
            cname = self.column_map[column]
            if cname in ('size', 'timestamp', 'inlibrary'):
                return False
            val = str(value or '').strip()
            idx = self.map[row]
            if cname == 'collections':
                tags = [i.strip() for i in val.split(',')]
                tags = [t for t in tags if t]
                self.db[idx].device_collections = tags
                self.dataChanged.emit(index, index)
                self.upload_collections.emit(self.db)
                return True

            if cname == 'title':
                self.db[idx].title = val
            elif cname == 'authors':
                self.db[idx].authors = string_to_authors(val)
            self.dataChanged.emit(index, index)
            self.booklist_dirtied.emit()
            done = True
        return done

    def set_editable(self, editable):
        # Cannot edit if metadata is sent on connect. Reason: changes will
        # revert to what is in the library on next connect.
        if isinstance(editable, list):
            self.editable = editable
        elif editable:
            self.editable = ['title', 'authors', 'collections']
        else:
            self.editable = []
        if device_prefs['manage_device_metadata'] == 'on_connect':
            self.editable = []


# }}}
