Coverage for astrocyte/pipeline/wiki_incremental.py: 89%
90 statements
« prev ^ index » next coverage.py v7.15.0, created at 2026-10-29 22:10 +0000
1"""M14.2: Karpathy update-affected-wikis.
3When a new document is retained, identify existing wiki pages whose
4provenance entities overlap with the new document's entities, then run
5an LLM update pass per affected wiki with the new content folded in.
6Models Karpathy's LLM-wiki spec ("update-affected-wikis" op) and
7Hindsight's observation-consolidation freshness pattern — both share
8the shape: detect what changed, ripple to dependent compiled artefacts.
10Why this is the next M14 lever after M14.6 revert:
12The post-M14.7-revert decision was: stop auto-injecting preferences
13into the answerer's context (Hindsight design); double down on the
14consolidation pipeline so retrieved candidates ARE fresh. M14.2
15attacks that directly:
171. **knowledge-update** category on LME (current 5/5 at top_20 — already
18 strong, but only because wikis happen to recompile from scratch each
19 bench run). Real users add documents incrementally; stale wikis
20 would otherwise hurt knowledge-update over time.
212. **multi-session** on LME — questions that span sessions need
22 cross-document wiki updates to surface coherent state. Currently
23 wikis are document-local; M14.2 enables document-spanning updates.
243. **open-domain** + **temporal** on LoCoMo — both benefit from
25 freshness signal in the wiki content (e.g. "user now lives in
26 Berlin (previously Paris)" inline in the relevant wiki).
28Algorithm:
30::
32 on_document_retained(doc):
33 affected = find_affected_wikis(doc.entities) # entity overlap
34 for wiki in affected:
35 ctx = (wiki.content, new_doc.relevant_sections)
36 out = llm.update_wiki(ctx) # one call per wiki
37 if out.verdict == "UPDATE":
38 save_revision(wiki, out.new_content)
40The "find_affected_wikis" step is a cheap SQL join via
41``astrocyte_pi_wiki_provenance`` × ``astrocyte_pi_section_entities``
42(both already populated by M9 + M12.5).
44Cost: ~$0.05 per source (one LLM call per affected wiki, typically
452-5 wikis per source). At LoCoMo 200 docs that's ~$1-2 per bench run.
46Marginal vs the answerer's per-question cost.
48Design references:
49- ``docs/_design/llm-wiki-compile.md`` (Karpathy spec)
50- ``docs/_design/m13-m14-roadmap.md`` §4 (M14.2 row)
51- Hindsight: ``hindsight_api/engine/consolidation/consolidator.py``
52 (similar shape: identify novel facts → update or create observations)
54Status: SKELETON — interface + entity-overlap query + LLM prompt
55designed; ``update_affected_wikis_for_document`` is the entry point.
56The orchestration wiring into retain (FSM state ``UPDATE_AFFECTED_WIKIS``
57in the M14.0 scaffold, or direct call from ``bench_pageindex_locomo``)
58is the next implementation step.
59"""
61from __future__ import annotations
63import json
64import logging
65from dataclasses import dataclass, field
66from datetime import datetime, timezone
67from typing import TYPE_CHECKING
69from astrocyte.pipeline._json_tolerant import tolerant_json_loads_or_raise
70from astrocyte.types import Message
72if TYPE_CHECKING:
73 from astrocyte.provider import LLMProvider, PageIndexStore
74 from astrocyte.types import WikiPage
76_logger = logging.getLogger("astrocyte.pipeline.wiki_incremental")
79# ---------------------------------------------------------------------------
80# Tunables
81# ---------------------------------------------------------------------------
83#: Minimum entities shared between new doc and wiki provenance before we
84#: consider the wiki "affected". 1 is the loosest threshold; raising to 2
85#: cuts spurious cross-doc updates but misses true single-entity-overlap
86#: updates. Calibrated against LoCoMo's entity density (~5-15 unique
87#: entities per session) — 1 is right for the bench.
88MIN_ENTITY_OVERLAP: int = 1
90#: Cap on wikis updated per retain. If a single source touches >N wikis
91#: we update only the top-N by overlap count — keeps cost bounded.
92MAX_AFFECTED_WIKIS_PER_SOURCE: int = 8
94#: max_tokens for the update LLM call. Sized for a 1-3 paragraph wiki
95#: with revision notes; ample margin for context preservation.
96UPDATE_LLM_MAX_TOKENS: int = 1500
99# ---------------------------------------------------------------------------
100# Result types
101# ---------------------------------------------------------------------------
104@dataclass
105class WikiUpdateResult:
106 """One wiki's update outcome from an incremental update pass."""
108 page_id: str
109 verdict: str # 'UPDATED' | 'NO_CHANGE' | 'FAILED'
110 new_revision: int | None = None
111 detail: str = ""
114@dataclass
115class IncrementalUpdateReport:
116 """Per-document aggregate of wiki update outcomes."""
118 document_id: str
119 affected_count: int # how many wikis the entity-overlap query found
120 updated: list[WikiUpdateResult] = field(default_factory=list)
121 skipped: list[WikiUpdateResult] = field(default_factory=list)
122 failed: list[WikiUpdateResult] = field(default_factory=list)
125# ---------------------------------------------------------------------------
126# LLM prompt
127# ---------------------------------------------------------------------------
130_UPDATE_PROMPT = """\
131You are maintaining a knowledge wiki. A new conversation has been retained \
132that overlaps with this existing wiki entry. Decide whether the new \
133content REQUIRES the wiki to be updated, or whether the existing wiki \
134already captures the new information (no change needed).
136Existing wiki (title: "{title}", revision {revision}):
137=== BEGIN WIKI ===
138{wiki_content}
139=== END WIKI ===
141New content from the retained conversation (chronologically AFTER the \
142wiki's existing provenance):
143=== BEGIN NEW CONTENT ===
144{new_content}
145=== END NEW CONTENT ===
147Rules:
148- The wiki tracks LATEST state. If the new content updates a fact \
149 the wiki currently states (e.g. user moved cities, changed jobs, \
150 finished a project they were planning), emit UPDATE with the \
151 revised content reflecting the new state. Preserve the previous \
152 state inline as parenthetical when historically relevant (e.g. \
153 "(previously lived in Paris)").
154- If the new content adds NEW information not covered by the wiki, \
155 emit UPDATE with the addition integrated naturally.
156- If the new content merely re-states what the wiki already says, \
157 emit NO_CHANGE.
158- If the new content is unrelated to the wiki's topic despite the \
159 entity overlap (e.g. shared person name in different contexts), \
160 emit NO_CHANGE.
161- Preserve the wiki's voice and structure. Output the FULL revised \
162 content, not a diff.
164Output a JSON object:
165{{
166 "verdict": "UPDATE" or "NO_CHANGE",
167 "revised_content": "<full revised wiki content; only when UPDATE>",
168 "revised_title": "<optional new title; only when UPDATE and title needs change>"
169}}
171OUTPUT MUST BE VALID JSON. No prose around it.
172"""
175# ---------------------------------------------------------------------------
176# Public entry point
177# ---------------------------------------------------------------------------
180async def update_affected_wikis_for_document( # noqa: PLR0913
181 *,
182 page_index_store: PageIndexStore,
183 provider: LLMProvider,
184 bank_id: str,
185 document_id: str,
186 new_entities: list[str],
187 new_content_excerpts: dict[str, str],
188 model: str | None = None,
189 min_overlap: int = MIN_ENTITY_OVERLAP,
190 max_updates: int = MAX_AFFECTED_WIKIS_PER_SOURCE,
191) -> IncrementalUpdateReport:
192 """Run Karpathy update-affected-wikis for one retained source.
194 Args:
195 page_index_store: source of the entity-overlap query AND wiki
196 persistence (``save_wiki_page`` for revised wikis).
197 provider: LLM provider for the update calls.
198 bank_id: tenant scope.
199 document_id: the newly-retained document whose entities trigger
200 the update sweep.
201 new_entities: distinct entity names extracted from the new
202 document (already deduped by caller).
203 new_content_excerpts: ``entity_name → relevant excerpt`` from
204 the new document; passed to the update LLM as evidence.
205 model: LLM model name (defaults to provider's default).
206 min_overlap: minimum shared entities to flag a wiki as affected.
207 max_updates: cap on update calls (top-N by overlap count).
209 Returns:
210 :class:`IncrementalUpdateReport` with per-wiki outcomes.
212 Idempotent across re-runs: the second invocation finds the wiki's
213 revision already reflects the new state and emits NO_CHANGE.
214 """
215 report = IncrementalUpdateReport(
216 document_id=document_id,
217 affected_count=0,
218 )
220 if not new_entities:
221 _logger.debug(
222 "wiki_incremental: doc=%s has no entities — skip",
223 document_id,
224 )
225 return report
227 # ── Find affected wikis via entity overlap (SPI returns full WikiPage rows) ──
228 try:
229 affected = await page_index_store.list_wikis_affected_by_entities(
230 bank_id,
231 list(new_entities),
232 min_overlap=min_overlap,
233 limit=max_updates,
234 )
235 except Exception as exc: # noqa: BLE001
236 _logger.warning(
237 "wiki_incremental.spi: list_wikis_affected_by_entities failed doc=%s (%s)",
238 document_id,
239 exc,
240 )
241 return report
243 report.affected_count = len(affected)
244 if not affected:
245 _logger.debug(
246 "wiki_incremental: doc=%s no wikis affected by entities=%s",
247 document_id,
248 new_entities[:5],
249 )
250 return report
252 # ── Per-wiki update calls ────────────────────────────────────────
253 for wiki, _overlap_count, shared_entities in affected:
254 result = await _update_one_wiki(
255 wiki=wiki,
256 page_index_store=page_index_store,
257 provider=provider,
258 shared_entities=shared_entities,
259 new_content_excerpts=new_content_excerpts,
260 model=model,
261 )
262 if result.verdict == "UPDATED":
263 report.updated.append(result)
264 elif result.verdict == "NO_CHANGE":
265 report.skipped.append(result)
266 else:
267 report.failed.append(result)
269 _logger.info(
270 "wiki_incremental: doc=%s affected=%d updated=%d skipped=%d failed=%d",
271 document_id,
272 report.affected_count,
273 len(report.updated),
274 len(report.skipped),
275 len(report.failed),
276 )
277 return report
280# ---------------------------------------------------------------------------
281# Internals
282# ---------------------------------------------------------------------------
285async def _update_one_wiki( # noqa: PLR0913
286 *,
287 wiki: WikiPage,
288 page_index_store: PageIndexStore,
289 provider: LLMProvider,
290 shared_entities: list[str],
291 new_content_excerpts: dict[str, str],
292 model: str | None,
293) -> WikiUpdateResult:
294 """Run one update prompt and persist the result.
296 Composes ``new_content`` as the union of excerpts keyed by
297 entities the wiki and new doc share, so the LLM only sees evidence
298 relevant to this wiki's topic.
299 """
300 new_content_parts = [
301 f"[{e}] {new_content_excerpts.get(e, '').strip()}"
302 for e in shared_entities
303 if new_content_excerpts.get(e, "").strip()
304 ]
305 if not new_content_parts:
306 return WikiUpdateResult(
307 page_id=wiki.page_id,
308 verdict="NO_CHANGE",
309 detail="no excerpt content for shared entities",
310 )
311 new_content = "\n\n".join(new_content_parts)
313 msg = _UPDATE_PROMPT.format(
314 title=wiki.title,
315 revision=wiki.revision,
316 wiki_content=wiki.content,
317 new_content=new_content,
318 )
319 try:
320 completion = await provider.complete(
321 [Message(role="user", content=msg)],
322 model=model,
323 max_tokens=UPDATE_LLM_MAX_TOKENS,
324 temperature=0.0,
325 response_format={"type": "json_object"},
326 )
327 except Exception as exc: # noqa: BLE001
328 _logger.warning(
329 "wiki_incremental.llm: call failed for page=%s (%s)",
330 wiki.page_id,
331 exc,
332 )
333 return WikiUpdateResult(
334 page_id=wiki.page_id,
335 verdict="FAILED",
336 detail=str(exc),
337 )
339 try:
340 data = tolerant_json_loads_or_raise(completion.text)
341 except (json.JSONDecodeError, AttributeError) as exc:
342 _logger.warning(
343 "wiki_incremental.parse: bad JSON for page=%s (%s)",
344 wiki.page_id,
345 exc,
346 )
347 return WikiUpdateResult(
348 page_id=wiki.page_id,
349 verdict="FAILED",
350 detail=f"bad JSON: {exc}",
351 )
353 verdict = str(data.get("verdict", "")).upper()
354 if verdict == "NO_CHANGE":
355 return WikiUpdateResult(page_id=wiki.page_id, verdict="NO_CHANGE")
357 if verdict != "UPDATE":
358 return WikiUpdateResult(
359 page_id=wiki.page_id,
360 verdict="FAILED",
361 detail=f"unrecognised verdict: {verdict!r}",
362 )
364 revised_content = (data.get("revised_content") or "").strip()
365 if not revised_content:
366 return WikiUpdateResult(
367 page_id=wiki.page_id,
368 verdict="FAILED",
369 detail="UPDATE verdict with empty revised_content",
370 )
371 revised_title = (data.get("revised_title") or wiki.title).strip()
373 # Save as a new revision via ``PageIndexStore.save_wiki_page`` — its
374 # upsert semantics auto-bump revision and archive the prior version.
375 # The wiki's existing provenance is preserved: we re-parse it from
376 # ``source_ids`` (M12.6 format: ``"<doc_id>:<line_num>"``) if present;
377 # otherwise we pass an empty list (provenance row already exists in
378 # ``astrocyte_pi_wiki_provenance`` from the initial compile and is
379 # untouched by content-only revisions).
380 from dataclasses import replace # noqa: PLC0415
382 new_wiki = replace(
383 wiki,
384 title=revised_title,
385 content=revised_content,
386 revision=wiki.revision + 1,
387 revised_at=datetime.now(tz=timezone.utc),
388 )
389 provenance: list[tuple[str, int]] = []
390 for sid in wiki.source_ids or []:
391 if ":" in sid:
392 doc_id, _, line_str = sid.rpartition(":")
393 try:
394 provenance.append((doc_id, int(line_str)))
395 except ValueError:
396 continue
397 try:
398 await page_index_store.save_wiki_page(
399 page=new_wiki,
400 embedding=None,
401 provenance=provenance,
402 )
403 except Exception as exc: # noqa: BLE001
404 _logger.warning(
405 "wiki_incremental.save: page=%s revision=%d failed (%s)",
406 wiki.page_id,
407 new_wiki.revision,
408 exc,
409 )
410 return WikiUpdateResult(
411 page_id=wiki.page_id,
412 verdict="FAILED",
413 detail=f"save failed: {exc}",
414 )
416 return WikiUpdateResult(
417 page_id=wiki.page_id,
418 verdict="UPDATED",
419 new_revision=new_wiki.revision,
420 )