"""DuckDB-backed ingest daemon — the single storage + orchestration unit.

WS traffic is delegated to ``relay_bus.RelayBus`` (one async loop, one WS
per relay URL, multiplexed sub_ids). This module supplies the storage hooks
and the higher-level workflows (backfill, mention-sweep, auto-tracker,
tracked-pubkey lifecycle) that sit on top of the bus.

Concurrency: the daemon owns one DuckDB connection guarded by a single
write lock. Mention-sweep / auto-tracker threads + FastMCP worker threads
(``update_graph_*``) + live-sub callbacks (run on the bus loop thread) all
serialize on it. Read-only consumers (`run_sql` in mcp_admin) open their
own connection on the same file — DuckDB shares one underlying database
instance per process, so multi-connection access is safe.

Schema bootstrap is idempotent (`CREATE TABLE IF NOT EXISTS`), so the
DuckDB file does not need to be pre-seeded — fresh runs build empty
tables and ingest from there.
"""

import collections
import hashlib
import json
import logging
import threading
import time
from concurrent.futures import ThreadPoolExecutor, as_completed
from datetime import datetime, timezone
from typing import Any, Callable

import duckdb
from coincurve import PublicKeyXOnly

from ingest import (
    BACKFILL_TOTAL_TIMEOUT_S,
    _PROFILE_FIELDS,
    _normalize_relay_url,
    _p_target,
    _prepare_tags,
    apply_duckdb_runtime_config,
)
from live_subscription import LiveSubscriptionPool
from relay_bus import (
    RelayBus,
    RelayBusError,
    RelayConnectError,
    RelayTimeout,
)
from stats import StatsMixin
from tracked_pubkeys_model import (
    DISCOVERY_REJECT_DISABLE_THRESHOLD,
    TrackedPubkeysMixin,
)

# REQ fan-out workers for parallel relay queries. Each worker submits one
# coroutine to the bus loop; concurrency is bounded so a single tool call
# (mention-sweep, update_graph_*) doesn't queue 1000+ pending REQs at once.
REQ_FANOUT_WORKERS = 8

log = logging.getLogger("ingest_sql")

# Auto-tracker parameters. Worker scans `tracked_pubkeys` for rows with
# `last_scan_at IS NULL` (freshly inserted secondaries from p-tag harvesting,
# plus newly promoted primaries that haven't run discovery yet) and runs
# `track_pubkey.run` on each.
AUTO_TRACK_INTERVAL_S = 30
AUTO_TRACK_BATCH = 5  # max pubkeys per scan tick

# Reverse frontier (someone p-tagged a primary → that someone becomes
# secondary) skips events that are declarative LISTS rather than
# interactions. A popular primary may have 10k followers periodically
# republishing their own kind-3 with the primary in `p`-tags — without
# this filter all 10k would land in the discovery queue from passive
# metadata. Blacklist (rather than whitelist of "interaction kinds") so
# new kinds we haven't seen yet still expand the frontier by default;
# we only carve out kinds where a `p`-tag is known to mean "this person
# is on my list" rather than "I just interacted with them".
#
# Forward frontier (primary's own events expanding into secondary) is
# untouched — primary's follow list IS a deliberate "who matters to me"
# signal from the user, so kind-3 stays in forward.
REVERSE_FRONTIER_SKIP_KINDS = frozenset({
    3,      # NIP-02 follow list
    10000,  # NIP-51 mute list
    30000,  # NIP-51 categorized people list
})

# MentionSweep: periodic combined `{"#p":[all_primaries]}` REQ across every
# ok relay, to catch incoming mentions that landed on relays which aren't
# in any primary's productive_relays (so the primary live-sub `#p` filter
# doesn't see them). Authors of these events become secondary candidates
# via the reverse-frontier branch in `_ingest_event`. Since cursor uses
# wall-clock minus a generous overlap so a missed tick doesn't lose data —
# duplicates are killed by the recent-ids cache and idempotent SQL upserts.
MENTION_SWEEP_INTERVAL_S = 300       # 5 min
MENTION_SWEEP_OVERLAP_S = 60         # window overlap between successive sweeps
MENTION_SWEEP_LIMIT = 100            # per-relay event cap on the combined REQ

# Cooldown for repeatedly-failing relays. After `RELAY_COOLDOWN_THRESHOLD`
# consecutive errors a relay enters cooldown — `active` stays true (we don't
# permanently ban a relay just because it had a bad afternoon), but the
# reconcile loop only retries it once every `RELAY_COOLDOWN_INTERVAL_S`.
# Single 'ok' reconcile resets the counter and pulls it back into normal
# rotation. The threshold deliberately doesn't deactivate the row — UI can
# still surface it with `last_error`, and a transient outage that resolves
# itself within an hour gets recovered automatically without manual SQL.
RELAY_COOLDOWN_THRESHOLD = 5
RELAY_COOLDOWN_INTERVAL_S = 60 * 60  # 1h

# Live-coverage watermark tuning. The flush periodically persists "caught up to
# now" for every live sub past its end-of-stored-events signal; on restart we
# read that watermark and rewind by the margin before re-subscribing, so a
# crash loses at most one flush interval of coverage (re-pulled, deduped).
# Currently 10 min for observability while bedding in — will be raised to ~1h.
COVERAGE_FLUSH_INTERVAL_S = 10 * 60
COVERAGE_SINCE_MARGIN_S = 10 * 60

# Invalid-event disable: a relay that delivers ANY event failing
# `_verify_event` (forged signature, mismatched id, malformed fields) is
# auto-disabled on the spot. Different intent than the cooldown counter:
# a single bad-signature event is a hard signal of a misconfigured or
# hostile relay — there's no transient-failure interpretation. One strike.
INVALID_EVENT_DISABLE_THRESHOLD = 1


# ---- event verification (NIP-01) -----------------------------------------------
# Two-stage check: (1) `id` matches sha256 of the canonical NIP-01 serialization,
# (2) `sig` is a valid BIP-340 schnorr signature over `id` with `pubkey`.
# A bad relay can otherwise impersonate any author or forge content. Failed
# events are NOT ingested — they go to `fake_events` and bump
# `relays.invalid_events_count`.

def _canonical_event_serialize(evt: dict) -> bytes:
    """NIP-01 canonical serialization: a JSON array
    `[0, pubkey, created_at, kind, tags, content]` with no extra whitespace
    and no escapes beyond the JSON minimum. UTF-8 bytes."""
    serialized = json.dumps(
        [
            0,
            evt.get("pubkey", "") or "",
            int(evt.get("created_at", 0) or 0),
            int(evt.get("kind", 0) or 0),
            evt.get("tags", []) or [],
            evt.get("content", "") or "",
        ],
        separators=(",", ":"),
        ensure_ascii=False,
    )
    return serialized.encode("utf-8")


def _verify_event(evt: dict) -> tuple[bool, str]:
    """Verify a Nostr event. Returns (True, "") if valid, (False, reason)
    otherwise. `reason` is one of: 'malformed', 'id_mismatch', 'bad_sig'."""
    eid = (evt.get("id") or "").lower()
    pubkey_hex = (evt.get("pubkey") or "").lower()
    sig_hex = (evt.get("sig") or "").lower()
    if len(eid) != 64 or len(pubkey_hex) != 64 or len(sig_hex) != 128:
        return False, "malformed"
    try:
        eid_bytes = bytes.fromhex(eid)
        pubkey_bytes = bytes.fromhex(pubkey_hex)
        sig_bytes = bytes.fromhex(sig_hex)
    except ValueError:
        return False, "malformed"

    computed_id = hashlib.sha256(_canonical_event_serialize(evt)).hexdigest()
    if computed_id != eid:
        return False, "id_mismatch"

    try:
        ok = PublicKeyXOnly(pubkey_bytes).verify(sig_bytes, eid_bytes)
    except Exception:
        return False, "bad_sig"
    return (True, "") if ok else (False, "bad_sig")

# Hot in-memory cache of recently-ingested event ids. We get the same event
# from many relays in burst (each WS reader thread delivers it), and the
# expensive part of `_upsert_event_sql` (DELETE+REINSERT of all tag rows) is
# pure waste for duplicates. Cache-hit short-circuits BEFORE acquiring
# `_sql_lock` so 70+ threads don't fight over it for nothing.
RECENT_EVENT_IDS_CAPACITY = 10000


# ---- schema bootstrap ----------------------------------------------------------
# Schema DDL (CREATE TABLE + CREATE INDEX) and the static legacy-column
# migrations live in `ingest_schema.py` so this file isn't dominated by SQL
# text. `_ensure_schema` below runs both lists, plus the conditional backfills
# that need Python-side branching.

from ingest_schema import (
    ADDRESSABLES_VIEW_DDL,
    BOOTSTRAP_DDL,
    LEGACY_COLUMN_MIGRATIONS,
)


class IngestDaemonSQL(TrackedPubkeysMixin, StatsMixin):
    def __init__(self, db_path: str, bus: RelayBus | None = None):
        self._db_path = db_path
        self._conn = duckdb.connect(db_path)
        apply_duckdb_runtime_config(self._conn, db_path)
        # Serializes every storage call. DuckDB connections aren't thread-safe;
        # background-task threads (mention-sweep, auto-tracker) + FastMCP
        # worker threads (update_graph_*) + live-sub callbacks (run on the bus
        # loop thread) all write through self._conn, so a single mutex keeps
        # it simple.
        self._sql_lock = threading.Lock()
        self._ensure_schema()
        # Single async client for all WS traffic. Owned by the daemon — start()
        # boots its event-loop thread, stop() tears it down. Callers that
        # already have a bus (tests, parallel daemons) can pass one in.
        self.bus = bus or RelayBus()
        # Lock + connection-status mirror kept for compatibility with the
        # live_subscription / track_pubkey API that used to inspect
        # ``daemon.conns``. The bus is the source of truth for the WS layer;
        # this dict is updated by `_backfill_one` after each REQ.
        self.lock = threading.RLock()
        self.conns: dict[str, dict] = {}
        self.live_subs = LiveSubscriptionPool(self)
        # Optional callback fired when a tracked-pubkey resource ought to be
        # re-fetched by subscribers (ingest of an event involving a tracked
        # pubkey, role change, secondary registration, …). Hot-pluggable so
        # tests/headless boots can leave it None. Called from arbitrary
        # threads (LiveSub workers, FastMCP request threads, daemon thread)
        # — implementation must be thread-safe and non-blocking.
        self.notify_callback: Callable[[str], None] | None = None
        # Auto-tracker: discovers + activates pubkeys inserted as secondaries
        # by `_ingest_event` (and any primary row whose discovery hasn't run
        # yet). Sequential — one discovery may take seconds, so a queue is
        # the simplest fit. Worker started lazily by `start_auto_tracker()`
        # so unit tests that don't need it can avoid the thread.
        self._auto_track_stop = threading.Event()
        self._auto_track_thread: threading.Thread | None = None
        # Tick interval for the auto-tracker loop. Defaults to the module-level
        # constant for production; tests override to a small value so they
        # don't have to wait 30s for the loop to pick up a freshly inserted
        # pubkey. Read on every wait, so changes during the test run take
        # effect on the next tick.
        self.auto_track_interval_s: float = AUTO_TRACK_INTERVAL_S
        # MentionSweep: periodic combined `#p` REQ across all ok relays so
        # mentions of any primary that landed outside their productive_relays
        # still get captured. Same lifecycle pattern as auto-tracker.
        self._mention_sweep_stop = threading.Event()
        self._mention_sweep_thread: threading.Thread | None = None
        # Coverage flush: periodically persists the live-sub coverage watermark
        # (see COVERAGE_FLUSH_INTERVAL_S). Same lifecycle pattern as the others.
        self._coverage_flush_stop = threading.Event()
        self._coverage_flush_thread: threading.Thread | None = None
        # Runtime scan toggle: 'full-scan' (default) lets the auto-tracker,
        # mention-sweep and `track_pubkey.run` issue REQ probes; 'no-scan'
        # suspends those active scans so only events arriving on already-open
        # live-sub WSes are ingested. Always boots in 'full-scan' — the flag
        # is in-memory only, every MCP restart resets it.
        self._scan_mode = "full-scan"
        # Recent-event-ids cache (FIFO with set for O(1) membership). Mark
        # AFTER a successful commit, never before — if two threads race on
        # the same id, the second one falls through to SQL where
        # `_upsert_event_sql` short-circuits via the in-DB existence check.
        # That keeps correctness even if one thread crashes mid-ingest.
        # `maxlen=None` because we manage eviction manually — `deque.maxlen`
        # silently drops elements without telling us, so the parallel `set`
        # would leak entries for evicted ids and report false positives.
        self._recent_ids: collections.deque[str] = collections.deque()
        self._recent_ids_set: set[str] = set()
        self._recent_ids_lock = threading.Lock()

        # Rolling 60s window of fresh-ingest timestamps. `_ingest_event` pushes
        # `time.time()` after a successful commit (i.e., only when the event
        # was actually new — duplicates short-circuit before reaching this).
        # `events_per_minute()` prunes entries older than 60s and returns the
        # remaining count, so the metric is "unique events accepted in the
        # last minute" rather than raw inbound WS message rate.
        self._ingest_times: collections.deque[float] = collections.deque()
        self._ingest_times_lock = threading.Lock()

    # ---- lifecycle ----------------------------------------------------------

    def start(self) -> None:
        """Boot the bus event loop. Mention-sweep / auto-tracker stay
        suspended until the caller invokes their own ``start_*`` methods —
        unit tests that exercise storage hooks directly skip those."""
        self.bus.start()

    def stop(self) -> None:
        """Tear down the bus and any running background threads. Idempotent."""
        self.stop_mention_sweep()
        self.stop_auto_tracker()
        self.stop_coverage_flush()
        # Final coverage flush BEFORE stop_all — the flush reads the live-sub
        # pool, which stop_all clears. A clean shutdown thus persists coverage
        # right up to now; a hard crash falls back to the last periodic flush.
        try:
            self._coverage_flush(reason="shutdown")
        except Exception:
            log.exception("coverage-flush on shutdown failed")
        if self.live_subs is not None:
            self.live_subs.stop_all()
        self.bus.stop()

    # ---- WS-using orchestration --------------------------------------------

    def _backfill_one(
        self, url: str, flt: dict, unique_ids: set[str],
        *, timeout: float = BACKFILL_TOTAL_TIMEOUT_S,
    ) -> tuple[int, str | None]:
        """Send a single REQ via the bus and ingest every returned event.

        Mirrors the legacy ``_RelayDaemonBase._backfill_one`` signature
        (``url`` substituted for the old ``RelayConn`` since the bus owns
        connection state). On failure returns ``(count_so_far, error_str)``.

        Any per-relay transport failure (bus error, DNS/IDNA failure on a
        malformed URL, kernel-level OSError) is contained here — fan-out
        callers like ``req_and_ingest`` shouldn't lose every other relay
        because of one bad row in ``relays``.
        """
        try:
            events = self.bus.req_sync(url, flt, timeout=timeout)
        except (
            RelayConnectError, RelayTimeout, RelayBusError,
            OSError, UnicodeError, ValueError,
        ) as e:
            err = str(e)[:200] or type(e).__name__
            with self.lock:
                self.conns[url] = {"status": "error", "last_error": err}
            return 0, err
        count = 0
        for evt in events:
            if not isinstance(evt, dict) or "id" not in evt:
                continue
            try:
                self._ingest_event(evt, url)
                unique_ids.add(evt["id"])
                count += 1
            except Exception:
                log.exception("ingest failed for event %s", evt.get("id"))
        with self.lock:
            self.conns[url] = {"status": "ok", "last_error": None}
        return count, None

    def _reconcile_after_backfill(self) -> None:
        """Mirror the bus's per-relay status into the `relays` registry.

        Bus connections come and go on demand; this hook gives the rest of
        the system (UI, observability) a snapshot of the most recent outcome
        for each URL we touched.
        """
        with self.lock:
            snapshot = [
                {"url": url, "status": st["status"], "last_error": st["last_error"]}
                for url, st in self.conns.items()
            ]
        self._persist_status(snapshot)

    def req_and_ingest(self, flt: dict) -> dict:
        """Send the same REQ to every active relay in parallel; ingest every
        returned event; return per-relay summary + unique event count."""
        urls = sorted(self._read_active_relay_urls())
        unique_ids: set[str] = set()
        unique_lock = threading.Lock()

        def _one(url: str) -> dict:
            local: set[str] = set()
            count, err = self._backfill_one(url, flt, local)
            if local:
                with unique_lock:
                    unique_ids.update(local)
            return {
                "url": url,
                "status": "ok" if err is None else "error",
                "events_received": count,
                "error": err,
            }

        per_relay: list[dict] = []
        if urls:
            workers = min(REQ_FANOUT_WORKERS, len(urls))
            with ThreadPoolExecutor(
                max_workers=workers, thread_name_prefix="req-fanout",
            ) as pool:
                per_relay = list(pool.map(_one, urls))
        self._reconcile_after_backfill()
        return {
            "relays": per_relay,
            "unique_events": len(unique_ids),
            "event_ids": list(unique_ids),
        }

    def update_graph_with_events_authored_by(
        self, pubkey_hex: str, limit: int = 100, full: bool = False,
    ) -> dict:
        urls = sorted(self._read_active_relay_urls())
        per_relay: list[dict] = []
        unique_ids: set[str] = set()
        for url in urls:
            last_seen = None if full else self._last_seen_for(pubkey_hex, url)
            since = last_seen + 1 if last_seen is not None else None
            flt: dict[str, Any] = {"authors": [pubkey_hex], "limit": limit}
            if since is not None:
                flt["since"] = since
            count, err = self._backfill_one(url, flt, unique_ids)
            per_relay.append({
                "url": url,
                "status": "ok" if err is None else "error",
                "events_received": count,
                "error": err,
                "since": since,
            })
        self._reconcile_after_backfill()
        return {
            "pubkey": pubkey_hex,
            "limit_per_relay": limit,
            "full": full,
            "relays": per_relay,
            "unique_events": len(unique_ids),
        }

    def update_graph_with_events_authored_by_each(
        self, pubkeys_hex: list[str], limit: int = 500, full: bool = False,
    ) -> dict:
        """Relay-outer, follow-inner backfill. Each (pubkey, relay) gets its
        own ``since`` so well-synced authors aren't bottlenecked by laggards.
        """
        if not pubkeys_hex:
            return {
                "pubkeys": [], "limit_per_relay": limit, "full": full,
                "relays": [], "unique_events": 0,
            }
        urls = sorted(self._read_active_relay_urls())
        per_relay: list[dict] = []
        unique_ids: set[str] = set()
        for url in urls:
            per_pk: list[dict] = []
            relay_total = 0
            relay_err: str | None = None
            for pk in pubkeys_hex:
                if relay_err is not None:
                    per_pk.append({
                        "pubkey": pk, "events_received": 0,
                        "since": None, "skipped": True,
                    })
                    continue
                last_seen = None if full else self._last_seen_for(pk, url)
                since = last_seen + 1 if last_seen is not None else None
                flt: dict[str, Any] = {"authors": [pk], "limit": limit}
                if since is not None:
                    flt["since"] = since
                count, err = self._backfill_one(url, flt, unique_ids)
                per_pk.append({
                    "pubkey": pk, "events_received": count,
                    "since": since, "skipped": False,
                })
                relay_total += count
                if err is not None:
                    relay_err = err
            per_relay.append({
                "url": url,
                "status": "ok" if relay_err is None else "error",
                "events_received": relay_total,
                "error": relay_err,
                "per_pk": per_pk,
            })
        self._reconcile_after_backfill()
        return {
            "pubkeys": pubkeys_hex,
            "limit_per_relay": limit,
            "full": full,
            "relays": per_relay,
            "unique_events": len(unique_ids),
        }

    def update_graph_with_events_authored_by_outbox(
        self, pubkey_hex: str, kinds: list[int] | None = None,
        limit: int = 500, full: bool = False,
    ) -> dict:
        """NIP-65 outbox-aware: REQ goes only to the author's advertised
        write relays. If their kind-10002 is not yet stored, one broadcast
        REQ is issued first to discover it."""
        write_urls = self._author_write_urls(pubkey_hex)
        bootstrapped = False
        if not write_urls:
            self.req_and_ingest({"authors": [pubkey_hex], "kinds": [10002], "limit": 1})
            write_urls = self._author_write_urls(pubkey_hex)
            bootstrapped = True
        all_urls = self._read_active_relay_urls()
        targeted = sorted(all_urls & write_urls)
        if targeted:
            urls = targeted
            outbox_used = True
        else:
            urls = sorted(all_urls)
            outbox_used = False
        per_relay: list[dict] = []
        unique_ids: set[str] = set()
        for url in urls:
            last_seen = None if full else self._last_seen_for(pubkey_hex, url)
            since = last_seen + 1 if last_seen is not None else None
            flt: dict[str, Any] = {"authors": [pubkey_hex], "limit": limit}
            if kinds:
                flt["kinds"] = kinds
            if since is not None:
                flt["since"] = since
            count, err = self._backfill_one(url, flt, unique_ids)
            per_relay.append({
                "url": url, "status": "ok" if err is None else "error",
                "events_received": count, "error": err, "since": since,
            })
        self._reconcile_after_backfill()
        return {
            "pubkey": pubkey_hex,
            "kinds": kinds,
            "limit_per_relay": limit,
            "full": full,
            "outbox_used": outbox_used,
            "outbox_bootstrapped": bootstrapped,
            "outbox_relays_known": len(write_urls),
            "outbox_relays_queried": len(urls) if outbox_used else 0,
            "relays": per_relay,
            "unique_events": len(unique_ids),
        }

    # ---- scan-mode (run-time toggle) ----------------------------------------

    def get_scan_mode(self) -> str:
        return self._scan_mode

    def set_scan_mode(self, mode: str) -> str:
        """Toggle the active-scan suspension flag. Threads keep running but
        each tick short-circuits when mode is 'no-scan' — live subs and the
        reconcile loop are untouched, so events from already-open relays keep
        flowing into storage. Returns the new mode."""
        if mode not in ("full-scan", "no-scan"):
            raise ValueError(
                f"scan_mode must be 'full-scan' or 'no-scan', got {mode!r}"
            )
        if self._scan_mode == mode:
            log.info("scan_mode unchanged: %s", mode)
            return mode
        self._scan_mode = mode
        log.info("scan_mode set to %s", mode)
        self._notify("status://overview")
        return mode

    def start_auto_tracker(self) -> None:
        if self._auto_track_thread is not None:
            return
        self._auto_track_stop.clear()
        self._auto_track_thread = threading.Thread(
            target=self._auto_track_loop, daemon=True, name="auto-track",
        )
        self._auto_track_thread.start()
        log.info("auto-tracker started (interval=%ss, batch=%d)",
                 self.auto_track_interval_s, AUTO_TRACK_BATCH)

    def stop_auto_tracker(self) -> None:
        if self._auto_track_thread is None:
            return
        self._auto_track_stop.set()
        self._auto_track_thread.join(timeout=5.0)
        self._auto_track_thread = None

    def start_mention_sweep(self) -> None:
        if self._mention_sweep_thread is not None:
            return
        self._mention_sweep_stop.clear()
        self._mention_sweep_thread = threading.Thread(
            target=self._mention_sweep_loop, daemon=True, name="mention-sweep",
        )
        self._mention_sweep_thread.start()
        log.info(
            "mention-sweep started (interval=%ds, overlap=%ds, limit=%d)",
            MENTION_SWEEP_INTERVAL_S, MENTION_SWEEP_OVERLAP_S, MENTION_SWEEP_LIMIT,
        )

    def stop_mention_sweep(self) -> None:
        if self._mention_sweep_thread is None:
            return
        self._mention_sweep_stop.set()
        self._mention_sweep_thread.join(timeout=5.0)
        self._mention_sweep_thread = None

    def start_coverage_flush(self) -> None:
        if self._coverage_flush_thread is not None:
            return
        self._coverage_flush_stop.clear()
        self._coverage_flush_thread = threading.Thread(
            target=self._coverage_flush_loop, daemon=True, name="coverage-flush",
        )
        self._coverage_flush_thread.start()
        log.info(
            "coverage-flush started (interval=%ds, margin=%ds)",
            COVERAGE_FLUSH_INTERVAL_S, COVERAGE_SINCE_MARGIN_S,
        )

    def stop_coverage_flush(self) -> None:
        if self._coverage_flush_thread is None:
            return
        self._coverage_flush_stop.set()
        self._coverage_flush_thread.join(timeout=5.0)
        self._coverage_flush_thread = None

    def _coverage_flush_loop(self) -> None:
        """Persist the live-sub coverage watermark every interval. Wall-clock
        timer; first write lands one interval after start (subs need a moment
        to reach their end-of-stored-events signal anyway)."""
        while not self._coverage_flush_stop.is_set():
            if self._coverage_flush_stop.wait(COVERAGE_FLUSH_INTERVAL_S):
                break
            try:
                self._coverage_flush(reason="periodic")
            except Exception:
                log.exception("coverage-flush tick failed")

    def _mention_sweep_loop(self) -> None:
        """Periodic combined `{"#p":[primaries]}` REQ to every ok relay.
        Catches mentions of any primary that landed on a relay we don't
        otherwise live-sub for that primary. The reverse-frontier branch
        in `_ingest_event` registers each event's author as secondary.

        `since` cursor is wall-clock minus interval-plus-overlap so a
        missed tick (e.g. shutdown / startup gap) re-pulls the recent
        window. Duplicates are absorbed by `_recent_id_seen` and
        idempotent upserts; cost is bounded.
        """
        while not self._mention_sweep_stop.is_set():
            if self._scan_mode == "no-scan":
                if self._mention_sweep_stop.wait(MENTION_SWEEP_INTERVAL_S):
                    return
                continue
            try:
                with self._sql_lock:
                    rows = self._conn.execute(
                        "SELECT hex FROM tracked_pubkeys WHERE role = 'primary'"
                    ).fetchall()
                primaries = [r[0] for r in rows]
                if primaries:
                    since = int(time.time()) - MENTION_SWEEP_INTERVAL_S - MENTION_SWEEP_OVERLAP_S
                    flt = {
                        "#p": primaries,
                        "since": since,
                        "limit": MENTION_SWEEP_LIMIT,
                    }
                    log.info(
                        "mention-sweep tick: primaries=%d since=%d",
                        len(primaries), since,
                    )
                    summary = self.req_and_ingest(flt)
                    log.info(
                        "mention-sweep done: %d unique events across %d relays",
                        summary.get("unique_events", 0),
                        len(summary.get("relays", []) or []),
                    )
            except Exception:
                log.exception("mention-sweep: tick failed")
            if self._mention_sweep_stop.wait(MENTION_SWEEP_INTERVAL_S):
                return

    def _auto_track_loop(self) -> None:
        # Lazy import to dodge the circular: track_pubkey imports `ingest`
        # (for _RelayDaemonBase), and that module is imported at module top
        # of ingest_sql. Importing track_pubkey here keeps things tidy.
        from track_pubkey import run as _track  # noqa: PLC0415

        while not self._auto_track_stop.is_set():
            if self._scan_mode == "no-scan":
                if self._auto_track_stop.wait(self.auto_track_interval_s):
                    return
                continue
            try:
                rows = self._claim_pending_secondaries(AUTO_TRACK_BATCH)
            except Exception:
                log.exception("auto-tracker: claim failed")
                rows = []
            if rows:
                log.info(
                    "auto-tracker tick: claimed %d row(s): %s",
                    len(rows),
                    ", ".join(f"{r['hex'][:8]}({r['role']})" for r in rows),
                )
                # Batched NIP-65 bootstrap: one REQ per relay across the
                # whole tick instead of one per pubkey. Pubkeys whose
                # outbox we already know are excluded — they don't need
                # the lookup, and adding them would just inflate the
                # `authors` set on every relay. `track_pubkey.run` is
                # then told `outbox_bootstrap=False` so it doesn't
                # repeat this fan-out per pubkey.
                needs_bootstrap = [
                    r["hex"] for r in rows
                    if r["role"] == "secondary" and not self._author_write_urls(r["hex"])
                ]
                if needs_bootstrap:
                    try:
                        self.req_and_ingest({
                            "kinds": [10002],
                            "authors": needs_bootstrap,
                            "limit": len(needs_bootstrap),
                        })
                    except Exception:
                        log.exception(
                            "auto-tracker: batched outbox bootstrap failed",
                        )
                    log.info(
                        "auto-tracker: batched outbox bootstrap for %d pubkey(s)",
                        len(needs_bootstrap),
                    )
            # Parallelize discovery within the batch. Sequential burned
            # ~5-8 min on a primary's full-relay probe before the next
            # secondary even started — combined with frontier expansion
            # outpacing the drain, the queue grew faster than we worked it
            # off. Each worker's internal `PROBE_WORKERS=8` still stands;
            # with `AUTO_TRACK_BATCH=5` we cap concurrency at ~40 in-flight
            # connect/REQ ops, which is the same order as the live-sub
            # pool (~70+) so the box already handles the load.
            #
            # Role passthrough and `outbox_bootstrap=False` rationale match
            # the old sequential loop — a secondary must stay secondary,
            # and the kind-10002 bootstrap was already fanned out above.
            def _track_one(row: dict) -> None:
                if self._auto_track_stop.is_set():
                    return
                hex_key = row["hex"]
                limit = row["limit"]
                role = row["role"]
                log.info(
                    "auto-tracker pick %s: role=%s limit=%d",
                    hex_key[:8], role, limit,
                )
                try:
                    _track(self, hex_key, limit, role=role, outbox_bootstrap=False)
                except Exception:
                    log.exception("auto-tracker: discovery failed for %s", hex_key[:6])

            if rows:
                workers = min(AUTO_TRACK_BATCH, len(rows))
                exec_pool = ThreadPoolExecutor(
                    max_workers=workers, thread_name_prefix="auto-track-disc",
                )
                try:
                    futures = [exec_pool.submit(_track_one, r) for r in rows]
                    for _ in as_completed(futures):
                        if self._auto_track_stop.is_set():
                            break
                finally:
                    # cancel_futures=True drops queued workers instantly so
                    # shutdown isn't held by 5 in-flight discoveries each
                    # blocking on their own 8-min worst-case probe pass.
                    exec_pool.shutdown(wait=False, cancel_futures=True)
            if self._auto_track_stop.wait(self.auto_track_interval_s):
                return

    def _ensure_schema(self) -> None:
        with self._sql_lock:
            for stmt in BOOTSTRAP_DDL:
                self._conn.execute(stmt)
            # Snapshot which discovery columns exist BEFORE the ALTERs —
            # if they didn't, this is a legacy DB being migrated and we
            # backfill from the current `productive_relays` state below.
            # Fresh DBs land here with the columns already created by
            # `BOOTSTRAP_DDL`, so the backfill is skipped (DEFAULT 0 stays).
            existing_cols = {
                r[0] for r in self._conn.execute(
                    "SELECT column_name FROM information_schema.columns "
                    "WHERE table_name='relays'"
                ).fetchall()
            }
            discovery_cols_existed = (
                "discovery_picks" in existing_cols
                and "discovery_rejections" in existing_cols
            )
            for stmt in LEGACY_COLUMN_MIGRATIONS:
                self._conn.execute(stmt)
            if not discovery_cols_existed:
                # Picks: count actual occurrences in productive_relays. Each
                # pubkey has a URL at most once in its productive_relays, so
                # this is "how many tracked_pubkeys picked this relay".
                self._conn.execute(
                    "UPDATE relays r SET discovery_picks = p.n "
                    "FROM (SELECT u.url, count(*) AS n "
                    "      FROM tracked_pubkeys t, "
                    "           unnest(t.productive_relays) AS u(url) "
                    "      WHERE t.productive_relays IS NOT NULL "
                    "      GROUP BY 1) p "
                    "WHERE r.url = p.url"
                )
                # Rejections: relays never picked land at the disable
                # threshold, so the next discovery rejection tips them over.
                # Rejection history isn't reconstructable from the schema, so
                # we don't try to estimate it for picked relays.
                self._conn.execute(
                    "UPDATE relays SET discovery_rejections = ? "
                    "WHERE discovery_picks = 0",
                    [DISCOVERY_REJECT_DISABLE_THRESHOLD],
                )

            # ---- tag-schema migration: polymorphic tags+tag_elements → flat
            # event_tags. One-time reshape on legacy DB files; a no-op once the
            # old tables are gone. `tags` existing is the migration sentinel.
            table_types = {
                r[0]: r[1] for r in self._conn.execute(
                    "SELECT table_name, table_type FROM information_schema.tables "
                    "WHERE table_schema = 'main'"
                ).fetchall()
            }
            if "tags" in table_types:
                # position_in_event → tag_idx, tag_elements.position → elem_idx,
                # tag_elements.text → value. ref_type/ref_value are dropped:
                # derivable from name + the elem_idx=1 primary value. ON CONFLICT
                # makes a crash-interrupted migration safe to re-run.
                self._conn.execute(
                    "INSERT INTO event_tags "
                    "(event_id, tag_idx, elem_idx, name, value) "
                    "SELECT t.event_id, t.position_in_event, te.position, "
                    "       t.name, te.text "
                    "FROM tags t JOIN tag_elements te ON te.tag_id = t.tag_id "
                    "ON CONFLICT DO NOTHING"
                )
                self._conn.execute("DROP TABLE tag_elements")
                self._conn.execute("DROP TABLE tags")
            # ---- relay-table merge: pubkey_publishes_to + pubkey_reads_from →
            # flat pubkey_relays with read/write flags. One-time reshape on
            # legacy DBs; a no-op once the old tables are gone. `pubkey_relays`
            # is created by BOOTSTRAP_DDL above; here we fold the two legacy
            # tables into it. A relay present in both becomes a single row with
            # read=write=TRUE. ON CONFLICT makes a crash-interrupted merge safe
            # to re-run; `bool_or` collapses the two sources per (pubkey, relay).
            if "pubkey_publishes_to" in table_types:
                self._conn.execute(
                    "INSERT INTO pubkey_relays (pubkey, relay_url, read, write) "
                    "SELECT pubkey, relay_url, "
                    "       bool_or(src='r') AS read, bool_or(src='w') AS write "
                    "FROM ("
                    "  SELECT pubkey, relay_url, 'w' AS src FROM pubkey_publishes_to "
                    "  UNION ALL "
                    "  SELECT pubkey, relay_url, 'r' AS src FROM pubkey_reads_from"
                    ") GROUP BY pubkey, relay_url "
                    "ON CONFLICT DO NOTHING"
                )
                self._conn.execute("DROP TABLE pubkey_publishes_to")
                self._conn.execute("DROP TABLE pubkey_reads_from")

            # ---- timestamp normalization: relays.last_checked_at,
            # fake_events.received_at and tracked_pubkeys.added_at/last_scan_at
            # used to be stored in milliseconds while every other timestamp in
            # the DB is unix seconds. Fold them down to seconds. Idempotent by
            # magnitude: a millisecond timestamp from this century is > 1e12, a
            # seconds timestamp is ~1.7e9, so the guard converts each value at
            # most once no matter how often we boot. NULLs and the -1
            # `_SCAN_IN_PROGRESS_SENTINEL` on last_scan_at fall below the guard
            # and are left untouched.
            self._conn.execute(
                "UPDATE relays SET last_checked_at = last_checked_at // 1000 "
                "WHERE last_checked_at > 1000000000000"
            )
            self._conn.execute(
                "UPDATE fake_events SET received_at = received_at // 1000 "
                "WHERE received_at > 1000000000000"
            )
            self._conn.execute(
                "UPDATE tracked_pubkeys SET added_at = added_at // 1000 "
                "WHERE added_at > 1000000000000"
            )
            self._conn.execute(
                "UPDATE tracked_pubkeys SET last_scan_at = last_scan_at // 1000 "
                "WHERE last_scan_at > 1000000000000"
            )

            # `addressables` was a base table in the old schema; it's now a view
            # over event_tags. Drop the table form before (re)creating the view —
            # a view cannot shadow a same-named table.
            if table_types.get("addressables") == "BASE TABLE":
                self._conn.execute("DROP TABLE addressables")
            self._conn.execute(ADDRESSABLES_VIEW_DDL)

    # ---- storage hooks ----------------------------------------------------------

    def _read_active_relay_urls(self) -> set[str]:
        with self._sql_lock:
            rows = self._conn.execute(
                "SELECT url FROM relays WHERE COALESCE(active, true)"
            ).fetchall()
        # Drop rows whose URL no longer passes the validator. Some legacy
        # rows entered the table from raw NIP-65 `r` tags before the stricter
        # validation landed (free-form text starting with "wss://", trailing
        # dot, etc.) — handing those to ``getaddrinfo`` raises UnicodeError
        # and used to take down the mention-sweep fan-out. Skipping them
        # here keeps the registry usable without forcing a manual DELETE.
        return {r[0] for r in rows if _normalize_relay_url(r[0]) == r[0]}

    def _read_active_relays_due(self) -> set[str]:
        """Active relays whose connect attempt is allowed in this reconcile
        pass. Relays past `RELAY_COOLDOWN_THRESHOLD` consecutive failures are
        skipped until `RELAY_COOLDOWN_INTERVAL_S` has elapsed since their
        last check. They stay in `daemon.conns` (so dict membership matches
        `_read_active_relay_urls`) but reconcile won't burn DNS/TLS on them
        until the cooldown window closes."""
        cutoff = int(time.time()) - RELAY_COOLDOWN_INTERVAL_S
        with self._sql_lock:
            rows = self._conn.execute(
                "SELECT url FROM relays "
                "WHERE COALESCE(active, true) "
                "  AND ( "
                "    COALESCE(consecutive_failures, 0) < ? "
                "    OR last_checked_at IS NULL "
                "    OR last_checked_at < ? "
                "  )",
                [RELAY_COOLDOWN_THRESHOLD, cutoff],
            ).fetchall()
        return {r[0] for r in rows}

    def _persist_status(self, items: list[dict]) -> None:
        if not items:
            return
        # UPDATE only — we don't create relay rows here; rows enter via the
        # `relays` registry (managed by the user) or via tag-target inserts
        # during ingest.
        #
        # Failure tracking: 'ok' resets `consecutive_failures` to 0 and pulls
        # the relay back into normal rotation. 'error' increments. The
        # cooldown gate (skip retries until 1h elapsed) is enforced in
        # `_read_active_relays_due()` — here we just maintain the counter so
        # the gate has data to work with.
        now_s = int(time.time())
        cooled_down: list[str] = []
        with self._sql_lock:
            for it in items:
                if it["status"] == "ok":
                    self._conn.execute(
                        "UPDATE relays "
                        "SET status = ?, last_checked_at = ?, last_error = ?, "
                        "    consecutive_failures = 0 "
                        "WHERE url = ?",
                        [it["status"], now_s, it["last_error"], it["url"]],
                    )
                else:
                    row = self._conn.execute(
                        "UPDATE relays "
                        "SET status = ?, last_checked_at = ?, last_error = ?, "
                        "    consecutive_failures = COALESCE(consecutive_failures, 0) + 1 "
                        "WHERE url = ? "
                        "RETURNING consecutive_failures",
                        [it["status"], now_s, it["last_error"], it["url"]],
                    ).fetchone()
                    if row is None:
                        continue
                    fails = int(row[0])
                    if fails == RELAY_COOLDOWN_THRESHOLD:
                        cooled_down.append(it["url"])
        for url in cooled_down:
            log.info(
                "relay entered cooldown after %d consecutive failures, "
                "next retry in %dh: %s",
                RELAY_COOLDOWN_THRESHOLD,
                RELAY_COOLDOWN_INTERVAL_S // 3600,
                url,
            )
        self._notify("relays://status")

    def _recent_id_seen(self, eid: str) -> bool:
        """O(1) membership check on the recent-ids cache. Held under a tiny
        per-cache lock — does NOT acquire `_sql_lock`."""
        with self._recent_ids_lock:
            return eid in self._recent_ids_set

    def _recent_id_mark(self, eid: str) -> None:
        """Add `eid` to the recent-ids cache, evicting the oldest entry if
        at capacity. Idempotent: re-marking an existing id is a no-op."""
        with self._recent_ids_lock:
            if eid in self._recent_ids_set:
                return
            self._recent_ids.append(eid)
            self._recent_ids_set.add(eid)
            if len(self._recent_ids) > RECENT_EVENT_IDS_CAPACITY:
                old = self._recent_ids.popleft()
                self._recent_ids_set.discard(old)

    def _ingest_event(self, evt: dict, relay_url: str) -> None:
        eid = (evt.get("id") or "").lower()
        # Hot-path short-circuit: this event id was ingested very recently
        # (most often: another relay just delivered the same event seconds
        # ago). Skip BEFORE acquiring `_sql_lock` so 70+ reader threads
        # don't queue up on duplicates. Trade-off: we lose `event_seen_on`
        # registration for the second/third relay that delivered the same
        # event within the cache window — accepted, the live-sub `since`
        # cursor is keyed on author/p-tag activity, not on per-relay
        # delivery accuracy.
        if eid and self._recent_id_seen(eid):
            return

        # NIP-01 verification (id sha256 + schnorr sig). A bad relay can
        # otherwise impersonate any pubkey or forge content. Ordering after
        # the cache check is safe: bypassing the cache requires producing a
        # fake event whose id collides with an already-seen real id —
        # computationally infeasible.
        ok, reason = _verify_event(evt)
        if not ok:
            self._record_fake_event(evt, relay_url, reason)
            return

        canonical = _normalize_relay_url(relay_url) or relay_url
        tag_data = _prepare_tags(evt.get("tags") or [])
        author = (evt.get("pubkey") or "").lower()
        notify_pubkeys: set[str] = set()
        list_dirty = False
        with self._sql_lock:
            self._conn.execute("BEGIN")
            try:
                self._upsert_profile_sql(evt)
                self._upsert_relay_list_sql(evt, tag_data)
                self._upsert_event_sql(evt, canonical, tag_data)
                self._conn.execute("COMMIT")
            except Exception:
                self._conn.execute("ROLLBACK")
                raise
            # Mark cache only AFTER successful commit — if a concurrent
            # thread races us on the same id and we crash before COMMIT,
            # they can still complete the ingest via the in-DB existence
            # check inside `_upsert_event_sql`.
            if eid:
                self._recent_id_mark(eid)
            # Note tracked pubkeys touched by this event so we can ping
            # their resource subscribers after we drop the lock. Author +
            # every `p`-tag value, only the ones that are actually tracked.
            if author and self._is_tracked_unlocked(author):
                notify_pubkeys.add(author)
            for t in tag_data:
                if t["name"] != "p":
                    continue
                target = _p_target(t)
                if not target or not self._is_tracked_unlocked(target):
                    continue
                notify_pubkeys.add(target)
            # Frontier expansion: every `p`-tag of an event authored by a
            # primary becomes a secondary candidate. Secondaries themselves
            # don't expand further — that's the only thing distinguishing
            # the two roles. Insert is ON CONFLICT DO NOTHING so an existing
            # primary won't be demoted, and a secondary already there stays.
            if author and self._is_primary_unlocked(author):
                for t in tag_data:
                    if t["name"] != "p":
                        continue
                    target = _p_target(t)
                    if not target or target == author:
                        continue
                    inserted = self._register_secondary_unlocked(target)
                    if inserted:
                        list_dirty = True
                        notify_pubkeys.add(target)
                        log.info(
                            "frontier: primary %s → secondary %s (NEW)",
                            author[:8], target[:8],
                        )
            # Reverse frontier: an event whose AUTHOR isn't primary but
            # whose `p`-tags include a primary — i.e. someone replied to,
            # mentioned, or reacted to a primary — registers the author as
            # secondary. The forward frontier (above) catches outgoing
            # references from primary; this catches incoming. Source events
            # come from the primary live-sub `{"#p":[hex]}` filter on
            # productive_relays AND from `MentionSweep` (periodic
            # `{"#p":[primaries]}` REQ across all ok relays). Secondary
            # cannot generate further secondary because the gate here is
            # `_is_primary_unlocked(target)` — an existing secondary in the
            # p-tag wouldn't trigger it. Declarative-list kinds
            # (`REVERSE_FRONTIER_SKIP_KINDS`) are excluded so a 10k-follower
            # primary doesn't drown the queue in passive metadata.
            elif author and int(evt.get("kind") or 0) not in REVERSE_FRONTIER_SKIP_KINDS:
                primary_match = None
                for t in tag_data:
                    if t["name"] != "p":
                        continue
                    target = _p_target(t)
                    if not target or target == author:
                        continue
                    if self._is_primary_unlocked(target):
                        primary_match = target
                        break
                if primary_match is not None:
                    inserted = self._register_secondary_unlocked(author)
                    if inserted:
                        list_dirty = True
                        notify_pubkeys.add(author)
                        log.info(
                            "reverse frontier: %s mentioned primary %s → secondary (NEW)",
                            author[:8], primary_match[:8],
                        )
        with self._ingest_times_lock:
            self._ingest_times.append(time.time())
        # Notifications outside the lock (callback may take its own lock,
        # we don't want to nest).
        for pk in notify_pubkeys:
            self._notify(f"tracked://pubkey/{pk}")
        if list_dirty:
            self._notify("tracked://list")

    def _last_seen_for(self, pubkey: str, relay_url: str) -> int | None:
        with self._sql_lock:
            row = self._conn.execute(
                "SELECT max(e.created_at) "
                "FROM events e "
                "JOIN event_seen_on s ON s.event_id = e.id "
                "WHERE e.pubkey = ? AND s.relay_url = ?",
                [pubkey, relay_url],
            ).fetchone()
        return row[0] if row and row[0] is not None else None

    def _last_seen_either(self, pubkey: str, relay_url: str) -> int | None:
        """max(created_at) of events seen on `relay_url` that involve `pubkey`
        in either direction: authored by it (`pubkey`) or referencing it
        via a `p`-tag. Used as `since` on the live multi-filter REQ so that
        a reconnect doesn't re-walk history we already ingested through
        either direction.
        """
        with self._sql_lock:
            row = self._conn.execute(
                "SELECT max(e.created_at) "
                "FROM events e "
                "JOIN event_seen_on s ON s.event_id = e.id "
                "WHERE s.relay_url = ? "
                "  AND ("
                "    e.pubkey = ? "
                "    OR EXISTS ("
                "      SELECT 1 FROM event_tags t "
                "      WHERE t.event_id = e.id "
                "        AND t.name = 'p' AND t.elem_idx = 1 "
                "        AND t.value = ?"
                "    )"
                "  )",
                [relay_url, pubkey, pubkey],
            ).fetchone()
        return row[0] if row and row[0] is not None else None

    # ---- live-coverage watermark -------------------------------------------

    def _coverage_read(
        self, pubkey: str, relay_url: str, direction: str,
    ) -> int | None:
        with self._sql_lock:
            row = self._conn.execute(
                "SELECT covered_through FROM live_coverage "
                "WHERE pubkey = ? AND relay_url = ? AND direction = ?",
                [pubkey, relay_url, direction],
            ).fetchone()
        return row[0] if row and row[0] is not None else None

    def _coverage_since(
        self, pubkey: str, relay_url: str, direction: str, role: str,
    ) -> int | None:
        """`since` for a (re)subscribe. Prefer the coverage watermark (how far
        we KNOW we're caught up) minus the safety margin. Before we've ever been
        caught up there's no watermark — fall back to the last event seen (old
        behavior) so we don't re-walk all of history on the very first sub."""
        cov = self._coverage_read(pubkey, relay_url, direction)
        if cov is not None:
            return max(0, cov - COVERAGE_SINCE_MARGIN_S)
        if role == "primary":
            last = self._last_seen_either(pubkey, relay_url)
        else:
            last = self._last_seen_for(pubkey, relay_url)
        return last + 1 if last is not None else None

    def _coverage_flush(self, reason: str = "periodic") -> int:
        """Persist `covered_through = now` for every live sub past its end-of-
        stored-events signal, in one batched write. Prints a stdout marker on
        every actual write. No-op (no write, no print) when nothing is caught
        up. Returns the number of (pubkey, relay, direction) rows stamped."""
        pool = self.live_subs
        if pool is None:
            return 0
        triples = pool.caught_up_triples()
        if not triples:
            return 0
        now = int(time.time())
        with self._sql_lock:
            self._conn.executemany(
                "INSERT INTO live_coverage "
                "(pubkey, relay_url, direction, covered_through) "
                "VALUES (?, ?, ?, ?) "
                "ON CONFLICT (pubkey, relay_url, direction) "
                "DO UPDATE SET covered_through = excluded.covered_through",
                [[pk, url, d, now] for (pk, url, d) in triples],
            )
        stamp = datetime.fromtimestamp(now, tz=timezone.utc).strftime(
            "%Y-%m-%d %H:%M:%S UTC"
        )
        print(
            f"---------- ZAPIS DO BAZY, DO KIEDY JESTEŚMY ZSYNCHRONIZOWANI: {stamp}",
            flush=True,
        )
        log.info("coverage flush (%s): %d sub(s) stamped @ %s",
                 reason, len(triples), stamp)
        return len(triples)

    def _p_tag_pubkeys_for_author(self, pubkey: str) -> list[str]:
        """Distinct pubkeys appearing as `p`-tag values in any event authored
        by `pubkey`. Used at promotion time to retroactively register
        secondaries from a freshly promoted primary's existing events.
        """
        with self._sql_lock:
            rows = self._conn.execute(
                "SELECT DISTINCT t.value "
                "FROM event_tags t "
                "JOIN events e ON e.id = t.event_id "
                "WHERE e.pubkey = ? "
                "  AND t.name = 'p' AND t.elem_idx = 1 "
                "  AND t.value != ?",
                [pubkey, pubkey],
            ).fetchall()
        return [r[0] for r in rows]

    def _notify(self, uri: str) -> None:
        """Best-effort: hand `uri` to the registered callback (resource-
        update bus). Swallows exceptions — a broken sink must never break
        ingest. Called outside the SQL lock to keep the lock window short.

        Roster-changing notifications (`tracked://list`) and relay status
        reconciles also fan out a `status://overview` ping so the aggregate
        counters stay live without callers needing to know about it.
        """
        cb = self.notify_callback
        if cb is None:
            return
        try:
            cb(uri)
            if uri in ("tracked://list", "relays://status"):
                cb("status://overview")
        except Exception:
            log.exception("notify_callback failed for %s", uri)

    def _author_write_urls(self, pubkey_hex: str) -> set[str]:
        with self._sql_lock:
            rows = self._conn.execute(
                "SELECT relay_url FROM pubkey_relays WHERE pubkey = ? AND write",
                [pubkey_hex],
            ).fetchall()
        return {r[0] for r in rows}

    def follows_of(self, user_hex: str) -> list[str]:
        with self._sql_lock:
            rows = self._conn.execute(
                """
                WITH latest AS (
                    SELECT id
                    FROM events
                    WHERE pubkey = ? AND kind = 3
                    ORDER BY created_at DESC
                    LIMIT 1
                )
                SELECT DISTINCT t.value AS hex
                FROM event_tags t
                WHERE t.event_id = (SELECT id FROM latest)
                  AND t.name = 'p' AND t.elem_idx = 1
                """,
                [user_hex],
            ).fetchall()
        return [r[0] for r in rows]

    def events_count_per_pubkey(self, pubkeys: list[str]) -> dict[str, int]:
        if not pubkeys:
            return {}
        # DuckDB binds Python list as a LIST argument — safe and avoids
        # building IN-list with placeholders manually.
        with self._sql_lock:
            rows = self._conn.execute(
                "SELECT pubkey, count(*) "
                "FROM events "
                "WHERE pubkey IN (SELECT unnest(?)) "
                "GROUP BY pubkey",
                [pubkeys],
            ).fetchall()
        counts = {pk: 0 for pk in pubkeys}
        for pk, c in rows:
            counts[pk] = int(c)
        return counts

    def _record_fake_event(
        self, evt: dict, relay_url: str, reason: str,
    ) -> None:
        """Persist a verification failure to `fake_events` and bump the
        relay's `invalid_events_count`. Caller already established this
        event will not be ingested into `events`. Failures here are logged
        but never raised — we don't want a forensics-table issue to take
        down the WS reader thread."""
        canonical = _normalize_relay_url(relay_url) or relay_url
        claimed_id = (evt.get("id") or "")
        claimed_pubkey = (evt.get("pubkey") or "")
        try:
            raw = json.dumps(evt, ensure_ascii=False)
        except (TypeError, ValueError):
            raw = None
        now_s = int(time.time())
        try:
            with self._sql_lock:
                self._conn.execute(
                    "INSERT INTO fake_events "
                    "(received_at, relay_url, claimed_id, claimed_pubkey, "
                    " reason, raw) "
                    "VALUES (?, ?, ?, ?, ?, ?)",
                    [now_s, canonical, claimed_id, claimed_pubkey, reason, raw],
                )
                self._conn.execute(
                    "UPDATE relays SET invalid_events_count = "
                    "COALESCE(invalid_events_count, 0) + 1 "
                    "WHERE url = ?",
                    [canonical],
                )
                disabled = self._conn.execute(
                    "UPDATE relays SET active = false "
                    "WHERE url = ? "
                    "  AND COALESCE(active, true) "
                    "  AND COALESCE(invalid_events_count, 0) >= ? "
                    "RETURNING url",
                    [canonical, INVALID_EVENT_DISABLE_THRESHOLD],
                ).fetchall()
        except Exception:
            log.exception(
                "fake-event recording failed: relay=%s reason=%s",
                canonical, reason,
            )
            return
        if disabled:
            log.warning(
                "auto-disabled %s — sent %d invalid event(s) (threshold %d)",
                canonical,
                INVALID_EVENT_DISABLE_THRESHOLD,
                INVALID_EVENT_DISABLE_THRESHOLD,
            )
        log.warning(
            "fake event from %s reason=%s claimed_id=%s pubkey=%s",
            canonical, reason,
            claimed_id[:8] if claimed_id else "(none)",
            claimed_pubkey[:8] if claimed_pubkey else "(none)",
        )

    def backfill_relay_lists(self) -> dict:
        """Re-parse every stored kind-10002 event into PUBLISHES_TO / READS_FROM
        edges. Idempotent; respects replaceable-event `created_at` order."""
        with self._sql_lock:
            rows = self._conn.execute(
                "SELECT pubkey, raw, created_at "
                "FROM events "
                "WHERE kind = 10002 AND pubkey IS NOT NULL "
                "ORDER BY created_at ASC"
            ).fetchall()
        applied = 0
        skipped = 0
        for pubkey, raw, created_at in rows:
            try:
                evt = json.loads(raw) if raw else {}
            except (TypeError, ValueError):
                skipped += 1
                continue
            evt["kind"] = 10002
            evt["pubkey"] = pubkey
            evt["created_at"] = created_at or 0
            tag_data = _prepare_tags(evt.get("tags") or [])
            with self._sql_lock:
                self._conn.execute("BEGIN")
                try:
                    ok = self._upsert_relay_list_sql(evt, tag_data)
                    self._conn.execute("COMMIT")
                except Exception:
                    self._conn.execute("ROLLBACK")
                    raise
            if ok:
                applied += 1
            else:
                skipped += 1
        return {
            "kind10002_events_scanned": len(rows),
            "upserts_applied": applied,
            "skipped_invalid": skipped,
        }

    def backfill_profiles(self) -> dict:
        """Re-parse every stored kind-0 event into a profile row. Idempotent;
        respects the replaceable-event rule (newer `created_at` wins)."""
        with self._sql_lock:
            rows = self._conn.execute(
                "SELECT pubkey, content, created_at "
                "FROM events "
                "WHERE kind = 0 AND pubkey IS NOT NULL"
            ).fetchall()
        seen_pubkeys: set[str] = set()
        upserted = 0
        skipped = 0
        for pubkey, content, created_at in rows:
            evt = {
                "kind": 0,
                "pubkey": pubkey,
                "content": content or "",
                "created_at": created_at or 0,
            }
            with self._sql_lock:
                self._conn.execute("BEGIN")
                try:
                    ok = self._upsert_profile_sql(evt)
                    self._conn.execute("COMMIT")
                except Exception:
                    self._conn.execute("ROLLBACK")
                    raise
            if ok:
                upserted += 1
                seen_pubkeys.add(pubkey)
            else:
                skipped += 1
        return {
            "kind0_events_scanned": len(rows),
            "profiles_touched": len(seen_pubkeys),
            "upserts_applied": upserted,
            "skipped_invalid": skipped,
        }

    # ---- SQL transaction bodies (callers hold self._sql_lock + active txn) -----

    def _upsert_profile_sql(self, evt: dict) -> bool:
        if int(evt.get("kind", -1)) != 0:
            return False
        try:
            meta = json.loads(evt.get("content") or "{}")
        except (TypeError, ValueError):
            return False
        if not isinstance(meta, dict):
            return False
        pubkey = evt.get("pubkey") or ""
        if not pubkey:
            return False

        def _s(v: Any) -> str | None:
            return v if isinstance(v, str) and v else None

        props: dict[str, str | None] = {f: _s(meta.get(f)) for f in _PROFILE_FIELDS}
        # NIP-01 also accepts `displayName` camelCase as alias for display_name.
        if props["display_name"] is None:
            props["display_name"] = _s(meta.get("displayName"))

        created_at = int(evt.get("created_at", 0))
        raw = evt.get("content") or ""

        self._conn.execute(
            "INSERT INTO pubkeys (hex) VALUES (?) "
            "ON CONFLICT (hex) DO NOTHING",
            [pubkey],
        )

        cols = ("pubkey", "created_at", "raw", *_PROFILE_FIELDS)
        placeholders = ", ".join(["?"] * len(cols))
        update_set = ", ".join(
            f"{c} = excluded.{c}" for c in cols if c != "pubkey"
        )
        values = [pubkey, created_at, raw, *(props[f] for f in _PROFILE_FIELDS)]
        self._conn.execute(
            f"INSERT INTO profiles ({', '.join(cols)}) "
            f"VALUES ({placeholders}) "
            f"ON CONFLICT (pubkey) DO UPDATE SET {update_set} "
            f"WHERE profiles.created_at IS NULL "
            f"   OR profiles.created_at < excluded.created_at",
            values,
        )
        return True

    def _upsert_relay_list_sql(self, evt: dict, tag_data: list[dict]) -> bool:
        if int(evt.get("kind", -1)) != 10002:
            return False
        pubkey = evt.get("pubkey") or ""
        if not pubkey:
            return False

        new_ts = int(evt.get("created_at", 0))
        prev = self._conn.execute(
            "SELECT relay_list_ts FROM pubkeys WHERE hex = ?",
            [pubkey],
        ).fetchone()
        prev_ts = prev[0] if prev else None
        if prev_ts is not None and prev_ts >= new_ts:
            # Stale or duplicate — replaceable-event semantics: skip.
            return False

        writes: list[str] = []
        reads: list[str] = []
        for t in tag_data:
            if t["name"] != "r" or len(t["elements"]) < 2:
                continue
            # NIP-65 `r` tags carry a relay URL; canonicalize and drop
            # free-form garbage here (no trailing slash, valid host). This is
            # the only place that treats an `r` value as a relay — storage
            # keeps it as a plain tag value.
            url = _normalize_relay_url(t["elements"][1])
            if not url:
                continue
            marker = ""
            if len(t["elements"]) >= 3 and isinstance(t["elements"][2], str):
                marker = t["elements"][2].strip().lower()
            if marker in ("", "write"):
                writes.append(url)
            if marker in ("", "read"):
                reads.append(url)

        self._conn.execute(
            "INSERT INTO pubkeys (hex, relay_list_ts) VALUES (?, ?) "
            "ON CONFLICT (hex) DO UPDATE SET relay_list_ts = excluded.relay_list_ts",
            [pubkey, new_ts],
        )
        # NIP-65 explicitly overrides the previous list in full — wipe and rebuild.
        self._conn.execute(
            "DELETE FROM pubkey_relays WHERE pubkey = ?", [pubkey],
        )
        # Fold the write/read URL lists into one row per relay with both flags.
        # Insertion order (writes first, then reads) is preserved by dict so the
        # rebuilt rows mirror the tag order the author declared.
        flags: dict[str, list[bool]] = {}
        for url in writes:
            flags.setdefault(url, [False, False])[1] = True   # [read, write]
        for url in reads:
            flags.setdefault(url, [False, False])[0] = True
        for url, (is_read, is_write) in flags.items():
            self._conn.execute(
                "INSERT INTO relays (url) VALUES (?) ON CONFLICT (url) DO NOTHING",
                [url],
            )
            self._conn.execute(
                "INSERT INTO pubkey_relays (pubkey, relay_url, read, write) "
                "VALUES (?, ?, ?, ?) ON CONFLICT DO NOTHING",
                [pubkey, url, is_read, is_write],
            )
        return True

    def _upsert_event_sql(
        self, evt: dict, relay_url: str, tag_data: list[dict],
    ) -> None:
        eid = evt["id"]
        pubkey = evt.get("pubkey", "") or ""

        # Already FULLY ingested in DB? Tags are content-addressed by event id,
        # so the rebuild below would be a no-op churn (DELETE every tag row +
        # REINSERT same values). Just register seen-on for the new relay and
        # bail. This is the second layer of defense after `_recent_id_seen` —
        # handles cache misses (eviction, post-restart with old DB) where the
        # event still exists in storage.
        # `pubkey IS NOT NULL` distinguishes a real event from a STUB row
        # created by another event's `["e", id]` tag (line below — only `id`
        # is set, everything else NULL). A stub MUST go through the full upsert
        # so its real columns get filled in.
        existing = self._conn.execute(
            "SELECT 1 FROM events WHERE id = ? AND pubkey IS NOT NULL",
            [eid],
        ).fetchone()
        if existing is not None:
            self._conn.execute(
                "INSERT INTO relays (url) VALUES (?) ON CONFLICT (url) DO NOTHING",
                [relay_url],
            )
            self._conn.execute(
                "INSERT INTO event_seen_on (event_id, relay_url) VALUES (?, ?) "
                "ON CONFLICT DO NOTHING",
                [eid, relay_url],
            )
            return

        if pubkey:
            self._conn.execute(
                "INSERT INTO pubkeys (hex) VALUES (?) ON CONFLICT (hex) DO NOTHING",
                [pubkey],
            )
        # The relay must already exist as a registry row for status updates to
        # land; ensure it here so SEEN_ON has a target even if a tag pulls in
        # a relay we never registered manually.
        self._conn.execute(
            "INSERT INTO relays (url) VALUES (?) ON CONFLICT (url) DO NOTHING",
            [relay_url],
        )

        # COALESCE preserves first non-null: stubs fill in on first real
        # ingest, repeated deliveries from other relays don't churn the row.
        self._conn.execute(
            "INSERT INTO events (id, kind, created_at, content, sig, raw, pubkey) "
            "VALUES (?, ?, ?, ?, ?, ?, ?) "
            "ON CONFLICT (id) DO UPDATE SET "
            "  kind        = COALESCE(events.kind, excluded.kind),"
            "  created_at  = COALESCE(events.created_at, excluded.created_at),"
            "  content     = COALESCE(events.content, excluded.content),"
            "  sig         = COALESCE(events.sig, excluded.sig),"
            "  raw         = COALESCE(events.raw, excluded.raw),"
            "  pubkey  = COALESCE(events.pubkey, excluded.pubkey)",
            [
                eid,
                int(evt.get("kind", 0)),
                int(evt.get("created_at", 0)),
                evt.get("content", "") or "",
                evt.get("sig", "") or "",
                json.dumps(evt),
                pubkey or None,
            ],
        )

        self._conn.execute(
            "INSERT INTO event_seen_on (event_id, relay_url) VALUES (?, ?) "
            "ON CONFLICT DO NOTHING",
            [eid, relay_url],
        )

        # Rebuild tags wholesale: event_tags rows are content-addressed via
        # (event_id, tag_idx, elem_idx), so DELETE+INSERT is the simple,
        # correct path. One row per NIP-01 element; no reference typing.
        self._conn.execute("DELETE FROM event_tags WHERE event_id = ?", [eid])

        if not tag_data:
            return

        for t in tag_data:
            name = t["name"]
            elements = t["elements"]
            for ei, text in enumerate(elements):
                self._conn.execute(
                    "INSERT INTO event_tags "
                    "(event_id, tag_idx, elem_idx, name, value) "
                    "VALUES (?, ?, ?, ?, ?)",
                    [eid, t["idx"], ei, name, text],
                )

            # Catalog population from reference tags, read uniformly as the
            # primary value (element 1): `e` registers a stub event, `p` a
            # pubkey. Relays come from the NIP-65 handler and addressables are
            # a view over event_tags — neither needs an insert here.
            if len(elements) < 2:
                continue
            ref = elements[1]
            if name == "e":
                self._conn.execute(
                    "INSERT INTO events (id) VALUES (?) "
                    "ON CONFLICT (id) DO NOTHING",
                    [ref],
                )
            elif name == "p":
                self._conn.execute(
                    "INSERT INTO pubkeys (hex) VALUES (?) "
                    "ON CONFLICT (hex) DO NOTHING",
                    [ref],
                )
