Coverage for astrocyte/config.py: 92%
774 statements
« prev ^ index » next coverage.py v7.15.0, created at 2026-07-04 05:24 +0000
1"""Astrocyte configuration — YAML loading, profile resolution, env var substitution."""
3from __future__ import annotations
5import fnmatch
6import os
7import re
8from dataclasses import dataclass, field, fields
9from pathlib import Path
10from typing import Any, Literal
12import yaml
14from astrocyte.errors import ConfigError
15from astrocyte.types import AccessGrant
17_ENV_VAR_PATTERN = re.compile(r"\$\{([A-Za-z_][A-Za-z0-9_]*)\}")
19_BASE_SENSITIVE_FIELD_KEYS = (
20 "api_key",
21 "password",
22 "token",
23 "secret",
24)
26# ---------------------------------------------------------------------------
27# Profile directory (shipped inside the package)
28# ---------------------------------------------------------------------------
29_PROFILES_DIR = Path(__file__).parent / "profiles"
32# ---------------------------------------------------------------------------
33# Configuration dataclasses
34# ---------------------------------------------------------------------------
37@dataclass
38class RateLimitConfig:
39 retain_per_minute: int | None = None
40 recall_per_minute: int | None = None
41 reflect_per_minute: int | None = None
42 global_per_minute: int | None = None
45@dataclass
46class QuotaConfig:
47 retain_per_day: int | None = None
48 reflect_per_day: int | None = None
51@dataclass
52class HomeostasisConfig:
53 recall_max_tokens: int | None = None
54 reflect_max_tokens: int | None = None
55 retain_max_content_bytes: int | None = None
56 rate_limits: RateLimitConfig = field(default_factory=RateLimitConfig)
57 quotas: QuotaConfig = field(default_factory=QuotaConfig)
60@dataclass
61class PiiConfig:
62 mode: str = "regex" # "regex" | "ner" | "llm" | "rules_then_llm" | "disabled"
63 action: str = "redact" # "redact" | "reject" | "warn"
64 patterns: list[dict[str, str]] | None = None
65 countries: list[str] | None = None # ["SG", "IN", "GB", "US", "DE", "FR", "IT", "ES", "AU", "CA", "JP", "CN"]
66 type_overrides: dict[str, dict[str, str]] | None = None # {"credit_card": {"action": "reject"}}
69@dataclass
70class ValidationConfig:
71 max_content_length: int = 50000
72 reject_empty_content: bool = True
73 reject_binary_content: bool = True
74 allowed_content_types: list[str] | None = None
77@dataclass
78class MetadataSanitizationConfig:
79 blocked_keys: list[str] = field(default_factory=lambda: list(_BASE_SENSITIVE_FIELD_KEYS))
80 max_metadata_size_bytes: int = 4096
83@dataclass
84class BarrierConfig:
85 pii: PiiConfig = field(default_factory=PiiConfig)
86 validation: ValidationConfig = field(default_factory=ValidationConfig)
87 metadata: MetadataSanitizationConfig = field(default_factory=MetadataSanitizationConfig)
90@dataclass
91class DedupConfig:
92 enabled: bool = True
93 similarity_threshold: float = 0.95
94 action: str = "skip" # "skip" | "warn" | "update"
97@dataclass
98class NoisyBankConfig:
99 enabled: bool = True
100 retain_spike_multiplier: float = 5.0
101 min_avg_content_length: int = 20
102 max_dedup_rate: float = 0.8
103 action: str = "warn" # "warn" | "throttle" | "reject"
106@dataclass
107class RecallCacheConfig:
108 enabled: bool = False
109 similarity_threshold: float = 0.95
110 max_entries: int = 256
111 ttl_seconds: float = 300.0
114@dataclass
115class Bm25IdfConfig:
116 """BM25-with-IDF full-text recall (M9).
118 When enabled, the recall pipeline routes its keyword strategy through
119 :meth:`PostgresStore.search_fulltext_bm25` instead of the classic
120 ``ts_rank_cd`` path. Requires migration 013 (the materialized views) to
121 be applied AND :meth:`PostgresStore.refresh_bm25_views` to have run at
122 least once after ingest — otherwise the views are empty and every
123 keyword recall returns no hits.
125 Off by default — opt-in only after the deployment has wired up a
126 refresh schedule. See :doc:`/_plugins/bm25-idf` for the operator guide.
127 """
129 enabled: bool = False
132@dataclass
133class SourceAwareRetrievalConfig:
134 """Source-aware retain + recall (M10).
136 Two switches with independent risk profiles:
138 - ``retain_provenance`` is cheap and almost-free: when a SourceStore is
139 configured AND this flag is on, retain creates one SourceDocument
140 and N SourceChunk rows per ingestion call and stamps each
141 ``VectorItem.chunk_id`` with the matching chunk's id. Costs one
142 extra DB roundtrip per retain. Required for any provenance feature
143 downstream (citations, chunk expansion) but not sufficient — the
144 SourceStore also has to be wired in YAML.
145 - ``chunk_expansion`` is the recall-side bet: when on, the recall
146 pipeline fetches sibling chunks of each top-K vector hit from the
147 SourceStore and merges them into the candidate set with a
148 configurable score multiplier. Helps multi-hop / split-evidence
149 questions where the answer key is in chunk N±1 of a chunk that
150 hit. Adds one DB roundtrip per top-K hit. **Off by default** —
151 enable in benchmark configs and measure before flipping in prod.
153 ``expansion_score_multiplier`` controls how much weight expanded
154 chunks get relative to the seed hit. ``1.0`` keeps them at parity
155 (often too aggressive); ``0.5`` is a conservative starting point.
156 ``expansion_max_per_hit`` caps fan-out so a long document doesn't
157 flood the candidate pool.
158 """
160 retain_provenance: bool = False
161 chunk_expansion: bool = False
162 expansion_score_multiplier: float = 0.5
163 expansion_max_per_hit: int = 4
166@dataclass
167class BenchmarkBudgetConfig:
168 """Named benchmark budget for Hindsight-parity preset routing."""
170 candidate_limit: int = 30
171 graph_expansion_limit: int = 30
172 rerank_top_k: int = 30
173 max_tokens: int = 8192
174 observation_weight: float = 0.0
175 agentic_reflect_allowed: bool = False
178@dataclass
179class BenchmarkPresetConfig:
180 """Versioned benchmark preset metadata.
182 Runtime operators can ignore this section; benchmark scripts and regression
183 tests use it to keep preset semantics explicit across branches.
184 """
186 name: str = "custom"
187 version: int = 1
188 budget: str = "mid"
189 low: BenchmarkBudgetConfig = field(
190 default_factory=lambda: BenchmarkBudgetConfig(
191 candidate_limit=12,
192 graph_expansion_limit=12,
193 rerank_top_k=12,
194 max_tokens=2048,
195 agentic_reflect_allowed=False,
196 )
197 )
198 mid: BenchmarkBudgetConfig = field(default_factory=BenchmarkBudgetConfig)
199 high: BenchmarkBudgetConfig = field(
200 default_factory=lambda: BenchmarkBudgetConfig(
201 candidate_limit=60,
202 graph_expansion_limit=40,
203 rerank_top_k=40,
204 max_tokens=16384,
205 observation_weight=0.25,
206 agentic_reflect_allowed=True,
207 )
208 )
211@dataclass
212class AdversarialDefenseConfig:
213 """Adversarial-question defense layer.
215 Targets the LoCoMo adversarial category (negative-existence,
216 false-premise, time-shift, cross-entity-confusion) where the LLM
217 left to its own devices invents an answer from weak retrieval hits.
219 Three layered guards (each independently configurable):
221 1. ``abstention_enabled`` + ``abstention_floor`` — score-floor
222 short-circuit. When all top-K recall hits score below the floor,
223 skip the LLM and return "insufficient evidence." Default floor
224 ``0.2`` is conservative — only fires when retrieval is genuinely
225 disconnected from the question.
227 **DEPRECATED:** ``abstention_enabled`` (the bool) is now a
228 fallback for callers that don't pass per-call
229 :class:`~astrocyte.types.Dispositions`. The orchestrator
230 derives the effective abstention floor from
231 ``dispositions.skepticism`` (1=trust→never abstain,
232 5=skeptical→aggressive). Migration path: set the disposition
233 on the bank or per-request instead of toggling this bool. The
234 legacy mapping is ``abstention_enabled=True`` ⇔ skepticism=3,
235 ``abstention_enabled=False`` ⇔ skepticism=1. ``abstention_floor``
236 (the float) remains the BASE that gets scaled by skepticism —
237 see :func:`astrocyte.pipeline.orchestrator._abstention_floor_for_skepticism`.
239 2. ``premise_verification_enabled`` — pre-loop premise extraction +
240 per-claim verification. The question is decomposed into atomic
241 claims; each is verified against memory before answering. Adds
242 1 LLM call per question; targets false-premise failures.
244 3. ``adversarial_prompt_enabled`` — tightens the agentic-reflect
245 system prompt with explicit "insufficient evidence is always a
246 valid answer" + premise-check rules. Free, defense-in-depth.
247 """
249 abstention_enabled: bool = False
250 abstention_floor: float = 0.2
251 premise_verification_enabled: bool = False
252 premise_verification_min_confidence: float = 0.6
253 adversarial_prompt_enabled: bool = False
256@dataclass
257class AgenticReflectConfig:
258 """Agentic reflect loop (Hindsight parity).
260 When enabled, ``reflect()`` runs an LLM-driven loop that selects
261 between two tools — ``recall`` (refine and re-retrieve) and ``done``
262 (commit the final answer with citations) — for up to
263 ``max_iterations`` turns. Targets multi-hop and open-domain queries
264 where a single retrieval often misses the bridge memory.
266 Cost: each turn is one LLM call. Default cap of 3 means worst-case
267 3× the LLM cost of single-shot reflect; typical case is 1-2 turns
268 when initial evidence is sufficient.
270 See ``astrocyte/pipeline/agentic_reflect.py`` for protocol details.
271 """
273 enabled: bool = False
274 max_iterations: int = 3
275 recall_step_max_results: int = 10
276 max_evidence_pool_size: int = 30
279@dataclass
280class SemanticLinkGraphConfig:
281 """Precomputed semantic-kNN graph at retain time (Hindsight parity, C3a).
283 Each new memory is linked to its top-``k`` most-similar existing
284 memories with cosine similarity ≥ ``similarity_threshold``. The
285 edges feed the link-expansion retrieval CTE as a parallel signal
286 alongside entity-overlap and causal links.
288 Costs one extra ``search_similar`` per chunk during retain (cheap
289 against the HNSW index). Disabled by default; opt-in for
290 benchmarks / production where multi-hop synthesis matters.
291 """
293 enabled: bool = False
294 top_k: int = 5
295 similarity_threshold: float = 0.7
298@dataclass
299class QueryAnalyzerConfig:
300 """Query-level temporal constraint extraction.
302 When ``enabled``, recall runs the regex pre-pass to extract
303 temporal expressions ("last week", "in March 2024", "yesterday")
304 into a time_range filter. Free of LLM cost.
306 ``allow_llm_fallback`` opts in to a structured-JSON LLM extraction
307 for queries that contain a temporal marker but no regex match
308 (e.g. "last spring", "around the launch"). Adds 1 LLM call per
309 such query; gated to keep cost predictable.
311 A caller-supplied ``RecallRequest.time_range`` always wins over
312 the analyzer's extraction.
313 """
315 enabled: bool = False
316 allow_llm_fallback: bool = True
317 #: M18a-1 — extended relative-time patterns plus Hindsight-parity
318 #: dateparser Pass B. The chain runs:
319 #: 1. ``_try_iso_date`` / ``_try_relative`` — narrow exact regex
320 #: 2. ``_try_temporal_expansion`` (Pass A) — fuzzy "a few weeks ago"
321 #: 3. ``_try_month_year`` / ``_try_year`` — calendar references
322 #: 4. ``_try_dateparser`` (Pass B) — wide-net for named dates,
323 #: weekdays, ordinals, ISO refs (requires ``dateparser`` pkg —
324 #: see ``pyproject.toml`` ``[bench]`` extras; no-ops if missing)
325 #: The extracted date range is fed to ``fact_recall`` which RRF-
326 #: fuses semantic + episodic + temporal as parallel siblings
327 #: (Hindsight architecture), so false-positive dateparser hits are
328 #: damped by cross-strategy agreement scoring.
329 #:
330 #: **Default True as of M18b ship** (2026-05-17): the B1-dp+RRF
331 #: 2-run mean was 83.75% LoCoMo (+3.25pp vs B0 baseline 80.5%,
332 #: +2.17σ above M17 mean — cleared M17+1σ ship gate by 1.75pp).
333 #: Bench env override ``ASTROCYTE_M18_ENABLE_TEMPORAL_EXPANSION=0``
334 #: forces off for ablation replication.
335 enable_temporal_expansion: bool = True
338@dataclass
339class StructuredFactExtractionConfig:
340 """Single-pass structured fact extraction at retain time.
342 When enabled, replaces the legacy chunk + entity-extraction +
343 fact-causal-extraction three-pass pipeline with a single LLM call
344 that produces structured metadata (entities, causal_relations,
345 temporal range, where/who/why annotations) per memory.
347 Two extraction modes:
349 - ``"verbatim"`` (recommended for benchmarks): pre-chunks the
350 text, then asks the LLM to produce per-chunk metadata. The
351 stored memory text is the ORIGINAL chunk text — preserves
352 vocabulary for embedding-match against questions. Adds rich
353 metadata without losing surface terms.
355 - ``"concise"``: the LLM generates atomic fact statements that
356 replace the chunks. The stored text is the LLM's structured
357 paraphrase (``"what | Involving: who | why"``). Higher-density
358 knowledge representation, BUT loses surface vocabulary which
359 can hurt recall_hit_rate when questions share words with the
360 original text. Documented regression on LoCoMo 2026-05-02.
362 Each fact becomes ONE memory in both modes. The structured fields
363 populate ``metadata['_fact_*']`` either way.
365 Cost approximately equal to the legacy two-pass (one LLM call
366 replaces two); output substantially richer, especially for
367 multi-hop and temporal questions.
369 Net: opt-in, defaults conservative.
370 """
372 enabled: bool = False
373 #: Only ``"verbatim"`` is supported; the ``"concise"`` mode was removed
374 #: in M9 because it caused severe recall_hit_rate degradation (the
375 #: 2026-05-02 finding — LLM-paraphrased ``what`` fields lost the
376 #: surface vocabulary that question embeddings share). The field is
377 #: kept so existing configs parse, but ``validate_astrocyte_config``
378 #: raises ``ConfigError`` if any other value is set.
379 extraction_mode: str = "verbatim"
380 max_facts_per_call: int = 30
381 #: Chunking strategy used by verbatim mode to pre-chunk the
382 #: source text before LLM enrichment. SFE has different
383 #: granularity needs than legacy retrieval — measured wins on
384 #: LoCoMo with ``"paragraph"`` (large context per chunk → LLM
385 #: extracts richer metadata) vs ``"dialogue"`` (legacy default
386 #: for conversations, which fragments context and lost 2.5 pts
387 #: overall, 8.4 pts on open-domain). Override per workload.
388 #: Available: ``"paragraph"``, ``"dialogue"``, ``"sentence"``.
389 chunk_strategy: str = "paragraph"
390 #: Per-chunk character budget for verbatim SFE pre-chunking. When
391 #: None, falls through to the orchestrator's default chunk size
392 #: (512 chars for Astrocyte legacy compat). Bumping this reduces
393 #: chunk count per session, which is the dominant retain throughput
394 #: lever — the LLM produces one ``facts[]`` entry per chunk and
395 #: gpt-4o-mini latency is roughly linear in output tokens. 2048 is
396 #: a measured sweet spot for LongMemEval-shaped traffic
397 #: (avg ~12.7 KB per session → ~6 chunks instead of ~25).
398 #:
399 #: This overrides any chunk-size set by extraction profile or MIP
400 #: rules for the SFE path specifically; legacy chunking for non-SFE
401 #: retain is unaffected.
402 chunk_max_size: int | None = None
403 #: Per-chunk parallel verbatim extraction (Phase 3 of cost-control
404 #: port). When ``True``, sends one LLM call per chunk in parallel
405 #: (asyncio.gather with a semaphore), instead of one batched call
406 #: that asks for all N chunks' metadata at once. Drops cross-chunk
407 #: causal_relations — set ``causal_links.enabled: false`` to be
408 #: explicit. Default off so existing deployments are unaffected.
409 parallel_chunks: bool = False
410 #: Max in-flight LLM calls per session when ``parallel_chunks`` is
411 #: True. Should be ≤ avg chunks/session for the workload. 6 is the
412 #: sweet spot for LongMemEval-shaped traffic at chunk_max_size=2048
413 #: (~6 chunks per session).
414 parallel_chunks_max_concurrency: int = 6
417@dataclass
418class EntityCooccurrenceConfig:
419 """Co-occurrence link creation between entities in the same memory.
421 When retain extracts N entities from a single memory, the
422 orchestrator creates ``co_occurs`` ``EntityLink`` rows between
423 them so spreading activation at recall time can hop across these
424 edges (Hindsight parity).
426 The all-pairs Cartesian product is **O(N²) per retain**. The
427 2026-05-06 LME profile measured this stage at 34% of total retain
428 wall time, with per-call cost growing as the entity-link table
429 grew (52s tail latency on a single retain at session 100). Capping
430 the number of entities used to form co-occurrence pairs bounds the
431 work to O(K²) per retain regardless of corpus size.
433 ``max_entities_per_memory=5`` (default) produces ``C(5,2)=10``
434 links per retain — a 43× reduction vs the unbounded 30-entity
435 sessions LME emits. Top-K entities are selected by their order in
436 the extraction result (typically tracks prominence in the source
437 text). When N ≤ K the cap is a no-op.
439 Disable entirely (``enabled=False``) to skip co-occurrence link
440 creation altogether — useful for retain-heavy bench runs where
441 spreading activation isn't the critical recall signal.
442 """
444 enabled: bool = True
445 max_entities_per_memory: int = 5
448@dataclass
449class CausalLinksConfig:
450 """Cause→effect link extraction at retain time (Hindsight parity).
452 When enabled, retain runs an additional LLM pass that identifies
453 causal relationships between extracted entities and persists them
454 as ``EntityLink(link_type="causes", ...)`` rows. These edges feed
455 temporal spreading activation's "trace reasoning chains" path.
457 Costs one extra LLM call per record. Disabled by default — opt in
458 for benchmarks / production where causal walks add value.
459 """
461 enabled: bool = False
462 max_pairs_per_memory: int = 4
463 min_confidence: float = 0.7
466@dataclass
467class SpreadingActivationConfig:
468 """Spreading activation through entity links (Hindsight parity).
470 When enabled, recall expands the seed RRF result set by walking
471 entity-link edges (default: ``co_occurs``) out to ``max_hops``,
472 decaying activation per hop. Spread hits join the candidate pool
473 before final cross-encoder rerank, with metadata tags
474 (``_spread_hop``, ``_spread_via_entity``) so synthesis can tell
475 them apart from direct evidence.
477 Defaults are conservative — leave disabled until the workload is
478 verified to benefit (multi-hop / open-domain QA categories) and
479 monitor single-hop precision for regressions caused by spread noise.
480 """
482 enabled: bool = False
483 max_hops: int = 2
484 decay_per_hop: float = 0.6
485 expansion_limit: int = 30
486 activation_threshold: float = 0.2
487 link_types: list[str] = field(default_factory=lambda: ["co_occurs", "causes"])
488 #: Hindsight blog 2026-03-12 — temporal proximity bonus on top of
489 #: the entity-link spread. ``0.0`` disables (entity-link only).
490 temporal_proximity_weight: float = 0.3
491 temporal_half_life_days: float = 7.0
494@dataclass
495class CrossEncoderRerankConfig:
496 """Final-stage cross-encoder reranker (Hindsight parity).
498 When ``enabled=True``, :meth:`PipelineOrchestrator._rank_reflect_context`
499 uses a real cross-encoder (sentence-transformers backend) to rerank the
500 top-``top_k`` recall hits before synthesis. When disabled (default),
501 the existing ``cross_encoder_like_rerank`` heuristic runs instead.
503 The default model — ``cross-encoder/ms-marco-MiniLM-L-6-v2`` — matches
504 Hindsight's default and is the standard MS MARCO baseline.
505 """
507 enabled: bool = False
508 model_name: str = "cross-encoder/ms-marco-MiniLM-L-6-v2"
509 top_k: int = 30
510 force_cpu: bool = False
513@dataclass
514class TieredRetrievalConfig:
515 enabled: bool = False
516 min_results: int = 3
517 min_score: float = 0.3
518 max_tier: int = 3 # 0-4
519 #: Which recall path runs at tier 3+ (and tier 4 after reformulation). ``pipeline`` = built-in
520 #: pipeline only (default). ``hybrid`` = :class:`~astrocyte.hybrid.HybridEngineProvider` merge
521 #: (engine + pipeline); requires a hybrid engine provider and ``tiered_retrieval.enabled``.
522 full_recall: Literal["pipeline", "hybrid"] = "pipeline"
525@dataclass
526class RecallAuthorityTierConfig:
527 """One precedence band for :class:`RecallAuthorityConfig` (matches ``metadata[\"authority_tier\"]``)."""
529 id: str = ""
530 priority: int = 1
531 label: str = ""
534@dataclass
535class RecallAuthorityConfig:
536 """Structured recall authority — labels fused hits for synthesis (M7)."""
538 enabled: bool = False
539 rules_inline: str | None = None
540 rules_path: str | None = None
541 tiers: list[RecallAuthorityTierConfig] = field(default_factory=list)
542 #: When True, :meth:`Astrocyte.reflect` / pipeline reflect inject ``authority_context`` into the synthesis prompt.
543 apply_to_reflect: bool = True
544 #: Default ``metadata[\"authority_tier\"]`` for vectors in a bank (profile ``authority_tier`` overrides).
545 tier_by_bank: dict[str, str] = field(default_factory=dict)
548@dataclass
549class CuratedRetainConfig:
550 enabled: bool = False
551 model: str | None = None
552 context_recall_limit: int = 5
555@dataclass
556class CuratedRecallConfig:
557 enabled: bool = False
558 freshness_weight: float = 0.3
559 reliability_weight: float = 0.2
560 salience_weight: float = 0.2
561 original_score_weight: float = 0.3
562 freshness_half_life_days: float = 30.0
563 min_score: float | None = None
566@dataclass
567class SignalQualityConfig:
568 dedup: DedupConfig = field(default_factory=DedupConfig)
569 noisy_bank: NoisyBankConfig = field(default_factory=NoisyBankConfig)
572@dataclass
573class CircuitBreakerConfig:
574 failure_threshold: int = 5
575 recovery_timeout_seconds: float = 30.0
576 half_open_max_calls: int = 2
579@dataclass
580class EscalationConfig:
581 circuit_breaker: CircuitBreakerConfig = field(default_factory=CircuitBreakerConfig)
582 degraded_mode: str = "empty_recall" # "empty_recall" | "error" | "cache"
585@dataclass
586class ObservabilityConfig:
587 otel_enabled: bool = False
588 prometheus_enabled: bool = False
589 log_level: str = "info"
592@dataclass
593class AccessControlConfig:
594 enabled: bool = False
595 default_policy: str = "owner_only" # "owner_only" | "open" | "deny"
598@dataclass
599class JwtMiddlewareConfig:
600 """JWT identity middleware (identity spec §3 Gap 1 wiring).
602 When ``enabled``, the MCP server extracts the ``Authorization: Bearer``
603 token from each inbound request, validates it against the configured
604 JWKS, classifies the claims via :mod:`astrocyte.identity_jwt`, and
605 populates :attr:`AstrocyteContext.actor` with a resolved
606 :class:`ActorIdentity` for the call. See
607 ``docs/_plugins/jwt-identity-middleware.md`` for the operator guide.
609 When ``enabled=False`` (default), the MCP server preserves pre-middleware
610 behavior: a single static ``AstrocyteContext`` is used for all calls.
611 """
613 enabled: bool = False
614 #: JWKS endpoint for signature key retrieval. Required when enabled.
615 jwks_uri: str | None = None
616 #: Expected token ``aud`` claim. Required when enabled — unset audience
617 #: is a common misconfiguration that can result in cross-tenant accepts.
618 token_audience: str | None = None
619 #: Expected token ``iss`` claim. Validated when set; left unchecked when
620 #: None (some IdPs rotate issuers).
621 token_issuer: str | None = None
622 #: Signing algorithms accepted. Defaults to asymmetric only so HS* keys
623 #: stolen from misconfigured deployments can't forge tokens.
624 algorithms: list[str] = field(default_factory=lambda: ["RS256", "ES256"])
625 #: When True, a missing or malformed Authorization header raises.
626 #: When False (with ``allow_anonymous=True``), falls through to anonymous.
627 fail_closed: bool = True
628 #: Permit calls with no Authorization header. Ignored when fail_closed=True.
629 allow_anonymous: bool = False
630 #: JWKS cache refresh interval. Most JWKS endpoints rotate every 24h.
631 jwks_refresh_interval_hours: int = 24
634@dataclass
635class IdentityConfig:
636 """Identity-driven bank resolution and ACL helpers (M1–M2 / v0.5.0)."""
638 auto_resolve_banks: bool = False
639 user_bank_prefix: str = "user-"
640 agent_bank_prefix: str = "agent-"
641 service_bank_prefix: str = "service-"
642 resolver: Literal["convention", "config", "custom"] | None = None
643 obo_enabled: bool = False
644 #: JWT identity middleware wiring (identity spec §3 Gap 1).
645 jwt_middleware: JwtMiddlewareConfig = field(default_factory=JwtMiddlewareConfig)
648# ---------------------------------------------------------------------------
649# M2 — Config schema evolution (ADR-003, v0.5.0 with M1)
650# ---------------------------------------------------------------------------
653@dataclass
654class SourceConfig:
655 """External data source definition (``astrocyte.ingest``).
657 * **webhook** — HTTP push; gateway / ASGI binds the route.
658 * **stream** — long-running consumer — ``driver: redis`` (Redis Streams) or ``kafka`` (Kafka).
659 Requires ``url``, ``topic``, ``consumer_group``, ``target_bank`` / ``target_bank_template``.
660 For Redis, ``url`` is a Redis URL; for Kafka, ``url`` is bootstrap servers (e.g. ``localhost:9092``).
661 Optional ``path``: Redis consumer name or Kafka ``client_id``.
662 * **poll** / **api_poll** — scheduled HTTP pull — ``driver: github`` (``astrocyte-ingestion-github``).
663 Requires ``interval_seconds``, ``path`` as ``owner/repo``, ``target_bank`` (or template), and
664 ``auth.token`` (or env-substituted) for the GitHub API. Optional ``url`` overrides the API base
665 (default ``https://api.github.com``; use GitHub Enterprise ``.../api/v3`` when needed).
666 """
668 type: str = ""
669 extraction_profile: str | None = None
670 target_bank: str | None = None
671 target_bank_template: str | None = None
672 principal: str | None = None
673 auth: dict[str, str | int | float | bool | None] | None = None
674 path: str | None = None
675 driver: str | None = None
676 topic: str | None = None
677 consumer_group: str | None = None
678 url: str | None = None
679 interval_seconds: int | None = None
680 # M4.1 proxy recall: GET (default) or POST JSON to ``url``
681 recall_method: str | None = None # "GET" | "POST"
682 recall_body: Any | None = None # POST JSON: dict/str with placeholders (see ``astrocyte.recall.proxy``)
685@dataclass
686class AgentRegistrationConfig:
687 """Registered agent with bank access and optional rate hints (ADR-003 / v0.5.0)."""
689 principal: str | None = None
690 banks: list[str] | None = None
691 allowed_banks: list[str] | None = None # roadmap alias for banks; glob patterns allowed
692 default_bank: str | None = None
693 permissions: list[str] | None = None
694 max_retain_per_minute: int | None = None
695 max_recall_per_minute: int | None = None
698@dataclass
699class TlsConfig:
700 cert_path: str | None = None
701 key_path: str | None = None
704@dataclass
705class DeploymentConfig:
706 """Standalone gateway settings; ignored in library mode."""
708 mode: Literal["library", "standalone", "plugin"] = "library"
709 host: str | None = None
710 port: int | None = None
711 workers: int | None = None
712 cors_origins: list[str] | None = None
713 tls: TlsConfig | None = None
716@dataclass
717class ExtractionProfileConfig:
718 """Reusable extraction defaults for sources (pipeline implementation in M3)."""
720 content_type: str | None = None
721 chunking_strategy: str | None = None
722 entity_extraction: bool | str | None = None
723 metadata_mapping: dict[str, str] | None = None
724 tag_rules: list[dict[str, str | list[str]]] | None = None
725 chunk_size: int | None = None
726 fact_type: str | None = None # default "world"; e.g. "experience", "observation"
727 #: Optional recall-authority band id (overrides ``recall_authority.tier_by_bank`` for this profile).
728 authority_tier: str | None = None
731@dataclass
732class McpConfig:
733 default_bank_id: str | None = None
734 expose_reflect: bool = True
735 expose_forget: bool = False
736 expose_admin: bool = False
737 max_results_limit: int = 50
738 principal: str | None = None
741@dataclass
742class DefaultsConfig:
743 """Per-profile default settings."""
745 skepticism: int = 3
746 literalism: int = 3
747 empathy: int = 3
748 preferred_fact_types: list[str] | None = None
749 tags: list[str] | None = None
752@dataclass
753class DlpConfig:
754 """Data Loss Prevention — output scanning for PII in recall/reflect results."""
756 scan_recall_output: bool = False
757 scan_reflect_output: bool = False
758 output_pii_action: str = "warn" # "redact" | "reject" | "warn"
761@dataclass
762class LifecycleTtlConfig:
763 archive_after_days: int = 90 # Days since last recall before archiving
764 delete_after_days: int = 365 # Days since creation before deletion
765 exempt_tags: list[str] | None = None # Tags that exempt from TTL
766 fact_type_overrides: dict[str, int | None] | None = None # Override archive_after_days by fact_type
769@dataclass
770class LifecycleConfig:
771 enabled: bool = False
772 ttl: LifecycleTtlConfig = field(default_factory=LifecycleTtlConfig)
775@dataclass
776class WikiCompileConfig:
777 """Configuration for M8 wiki compile and its background trigger."""
779 enabled: bool = False
780 auto_start: bool = False
781 size_threshold: int = 50
782 staleness_days: float = 7.0
783 staleness_min_memories: int = 10
784 max_queue_size: int = 100
787@dataclass
788class PreferenceCompileConfig:
789 """M18a-4 follow-up — preference consolidation gate (Hindsight parity).
791 Per-document LLM pass that distills ``fact_type='preference'`` facts
792 into ``MentalModel(kind='preference')`` rows for advisory recall.
793 Default ON (matches pre-M18 bench behavior; was always run inline).
795 Disable to ablate the preference tier (useful for measuring its
796 isolated contribution vs the directive tier).
797 """
799 enabled: bool = True
802@dataclass
803class EpisodicExtractConfig:
804 """M18a-4 — episodic event index (tag + retrieve).
806 Two integration points (both bench-gated today; core orchestrator
807 hook deferred to a separate cycle):
809 1. **Retain side**: after fact extraction for a document,
810 ``tag_episodic_facts(all_facts)`` walks the fact list and tags
811 any matching an episodic verb pattern ("attended/visited/met X
812 at/during Y") by appending ``EPISODIC_MARKER`` ("episodic:event")
813 to the fact's ``entities`` array. Uses the existing fact_type
814 schema (migration 020) — no new SQL value, just a namespaced
815 marker entity. See ``astrocyte/pipeline/episodic_extract.py``.
817 2. **Recall side**: when ``question_has_episodic_cue(query)`` matches
818 (question contains "where did I", "when did I meet", etc.),
819 the recall path explicitly queries
820 ``store.search_facts_by_entity(bank_id, EPISODIC_MARKER, ...)``
821 to surface the episodic-tagged facts alongside generic
822 semantic hits.
824 Both call sites are bench-gated by ``ASTROCYTE_M18_ENABLE_EPISODIC_EXTRACTION=1``
825 today. When the orchestrator gains a "document post-process" hook,
826 the retain-side tagger moves there; the recall-side search becomes
827 a new strategy in ``section_recall``.
828 """
830 enabled: bool = False
833# NOTE: SpreadingActivationConfig is defined ~earlier in this file (line ~467).
834# The earlier definition is the live one — it carries the Hindsight-parity
835# link-walk fields (``max_hops``, ``decay_per_hop``, ``link_types``,
836# ``temporal_proximity_weight``) used by ``link_expansion.py`` and read in
837# ``_astrocyte.py:379``. The duplicate definition removed here was an
838# M18a-3 stub with ``seed_count``/``top_k`` fields that were never wired
839# (the actual M18a-3 entity-co-occurrence implementation takes those as
840# function parameters in ``section_recall.py``, not config fields).
843@dataclass
844class DirectiveCompileConfig:
845 """M18a-2 — directive-style preference compile (Hindsight parity).
847 Hindsight's ``mental_models.subtype='directive'`` rows are user-
848 curated hard rules treated as authoritative by the answerer. We
849 auto-distill preference facts into a tight directive tier (≤5 rows
850 per document) via ``astrocyte.pipeline.directive_compile``. The
851 advisory preference tier (consolidation pass) stays in the store
852 on-demand-recallable; only the directive tier surfaces verbatim.
854 Currently called from the bench harness pipeline only — the core
855 orchestrator does not yet have a "document post-process" hook for
856 document-level passes (preference_compile, directive_compile,
857 episodic_extract). Adding that hook is a separate cycle; until
858 then this flag controls the bench-side call only.
860 Bench-time env-var override: ``ASTROCYTE_M18_ENABLE_DIRECTIVE_COMPILE=1``.
862 **DEPRECATED (M19, 2026-05-18):** the auto-compile architecture is
863 misaligned with Hindsight's user-authored ``create_directive`` MCP
864 tool. Bench evidence (M18b, B2 × 2 runs) showed a replicated −30pp
865 SSP regression because the compressed directives override the
866 answerer's access to the original preference facts' nuance. Keep
867 ``enabled=False``; the in-tree code stays as reference for the
868 future user-authored MCP-tool path (M20+). Setting ``enabled=True``
869 will emit a runtime warning when ``run_document_postprocess`` runs.
870 """
872 enabled: bool = False
875@dataclass
876class EntityResolutionConfig:
877 """Configuration for M11 retain-time entity resolution."""
879 enabled: bool = False
880 defer_to_async: bool = False
881 similarity_threshold: float = 0.8
882 confirmation_threshold: float = 0.75
883 max_candidates_per_entity: int = 3
884 enable_llm_disambiguation: bool = True
885 canonical_resolution: bool = False
888@dataclass
889class AsyncTasksConfig:
890 """Configuration for durable background memory tasks."""
892 enabled: bool = False
893 backend: str = "pgqueuer" # "pgqueuer" | "pgqueuer_in_memory"
894 dsn: str | None = None
895 install_on_start: bool = False
896 auto_start_worker: bool = False
897 batch_size: int = 10
900@dataclass
901class BankConfig:
902 """Per-bank override settings."""
904 profile: str | None = None
905 access: list[dict[str, str | list[str]]] | None = None
906 homeostasis: HomeostasisConfig | None = None
907 barriers: BarrierConfig | None = None
908 signal_quality: SignalQualityConfig | None = None
911@dataclass
912class AstrocyteConfig:
913 """Top-level Astrocyte configuration."""
915 # Provider tier
916 provider_tier: Literal["storage", "engine"] = "engine"
918 # Profile
919 profile: str | None = None
921 # Engine provider
922 provider: str | None = None
923 provider_config: dict[str, str | int | float | bool | None] | None = None
925 # Tier 1: Storage
926 vector_store: str | None = None
927 vector_store_config: dict[str, str | int | float | bool | None] | None = None
928 graph_store: str | None = None
929 graph_store_config: dict[str, str | int | float | bool | None] | None = None
930 document_store: str | None = None
931 document_store_config: dict[str, str | int | float | bool | None] | None = None
932 wiki_store: str | None = None
933 wiki_store_config: dict[str, str | int | float | bool | None] | None = None
934 # M9 first-class mental-model storage (replaces wiki-piggyback path).
935 # Optional — when unset, /v1/mental-models endpoints raise 501.
936 mental_model_store: str | None = None
937 mental_model_store_config: dict[str, str | int | float | bool | None] | None = None
938 # M10 source-document / chunk store (provenance hierarchy). Optional —
939 # when unset, vectors remain anonymous flat rows (backward compatible).
940 source_store: str | None = None
941 source_store_config: dict[str, str | int | float | bool | None] | None = None
942 # M9 section recall — PageIndex tree + section graph. Optional.
943 # When unset, section recall is disabled (the SPI is gated on this
944 # store being configured). See docs/_design/recall.md.
945 pageindex_store: str | None = None
946 pageindex_store_config: dict[str, str | int | float | bool | None] | None = None
948 # LLM
949 llm_provider: str | None = None
950 llm_provider_config: dict[str, str | int | float | bool | None] | None = None
951 embedding_provider: str | None = None
952 embedding_provider_config: dict[str, str | int | float | bool | None] | None = None
954 # Fallback
955 fallback_strategy: str = "error" # "local_llm" | "error" | "degrade"
957 # Policy
958 homeostasis: HomeostasisConfig = field(default_factory=HomeostasisConfig)
959 barriers: BarrierConfig = field(default_factory=BarrierConfig)
960 signal_quality: SignalQualityConfig = field(default_factory=SignalQualityConfig)
961 escalation: EscalationConfig = field(default_factory=EscalationConfig)
962 observability: ObservabilityConfig = field(default_factory=ObservabilityConfig)
963 access_control: AccessControlConfig = field(default_factory=AccessControlConfig)
964 identity: IdentityConfig = field(default_factory=IdentityConfig)
965 defaults: DefaultsConfig = field(default_factory=DefaultsConfig)
967 # MCP
968 mcp: McpConfig = field(default_factory=McpConfig)
970 # Phase 2 innovations
971 recall_cache: RecallCacheConfig = field(default_factory=RecallCacheConfig)
972 bm25_idf: Bm25IdfConfig = field(default_factory=Bm25IdfConfig)
973 source_aware_retrieval: SourceAwareRetrievalConfig = field(
974 default_factory=SourceAwareRetrievalConfig,
975 )
976 tiered_retrieval: TieredRetrievalConfig = field(default_factory=TieredRetrievalConfig)
977 cross_encoder_rerank: CrossEncoderRerankConfig = field(default_factory=CrossEncoderRerankConfig)
978 spreading_activation: SpreadingActivationConfig = field(default_factory=SpreadingActivationConfig)
979 causal_links: CausalLinksConfig = field(default_factory=CausalLinksConfig)
980 entity_cooccurrence: EntityCooccurrenceConfig = field(default_factory=EntityCooccurrenceConfig)
981 structured_fact_extraction: StructuredFactExtractionConfig = field(default_factory=StructuredFactExtractionConfig)
982 query_analyzer: QueryAnalyzerConfig = field(default_factory=QueryAnalyzerConfig)
983 semantic_link_graph: SemanticLinkGraphConfig = field(default_factory=SemanticLinkGraphConfig)
984 agentic_reflect: AgenticReflectConfig = field(default_factory=AgenticReflectConfig)
985 adversarial_defense: AdversarialDefenseConfig = field(default_factory=AdversarialDefenseConfig)
986 recall_authority: RecallAuthorityConfig = field(default_factory=RecallAuthorityConfig)
987 curated_retain: CuratedRetainConfig = field(default_factory=CuratedRetainConfig)
988 curated_recall: CuratedRecallConfig = field(default_factory=CuratedRecallConfig)
990 # Compliance profile
991 compliance_profile: str | None = None # "gdpr" | "hipaa" | "pdpa" | None
993 # DLP
994 dlp: DlpConfig = field(default_factory=DlpConfig)
996 # Lifecycle
997 lifecycle: LifecycleConfig = field(default_factory=LifecycleConfig)
999 # M8 / M11 intelligence features
1000 wiki_compile: WikiCompileConfig = field(default_factory=WikiCompileConfig)
1001 entity_resolution: EntityResolutionConfig = field(default_factory=EntityResolutionConfig)
1002 async_tasks: AsyncTasksConfig = field(default_factory=AsyncTasksConfig)
1003 benchmark_preset: BenchmarkPresetConfig = field(default_factory=BenchmarkPresetConfig)
1004 # M18a-2 — directive-style preference compile (off by default; flip via
1005 # this field per-bank or env var ASTROCYTE_M18_ENABLE_DIRECTIVE_COMPILE=1).
1006 directive_compile: DirectiveCompileConfig = field(default_factory=DirectiveCompileConfig)
1007 # M18a-3 spreading_activation field already declared above (line ~969,
1008 # alongside cross_encoder_rerank, causal_links, etc.) — duplicate
1009 # declaration removed when the duplicate dataclass was deleted.
1010 # M18a-4 — episodic event index (retain-side tag + recall-side search).
1011 # Both call paths integrated via document_postprocess + fact_recall in
1012 # core. Off by default; flip via this field per-bank or env var
1013 # ASTROCYTE_M18_ENABLE_EPISODIC_EXTRACTION=1.
1014 episodic_extract: EpisodicExtractConfig = field(
1015 default_factory=EpisodicExtractConfig,
1016 )
1017 # M18a-4 follow-up — preference compile gate (was always-on in bench
1018 # before M18). Default True for back-compat; off only when ablating.
1019 preference_compile: PreferenceCompileConfig = field(
1020 default_factory=PreferenceCompileConfig,
1021 )
1023 # MIP (Memory Intent Protocol)
1024 mip_config_path: str | None = None # Path to mip.yaml
1026 # Per-bank overrides
1027 banks: dict[str, BankConfig] | None = None
1029 # Top-level access grants (merged with banks.*.access by access_grants_for_astrocyte)
1030 access_grants: list[AccessGrant] | None = None
1032 # ADR-003 (v0.5.0 with M1)
1033 sources: dict[str, SourceConfig] | None = None
1034 agents: dict[str, AgentRegistrationConfig] | None = None
1035 deployment: DeploymentConfig | None = None
1036 extraction_profiles: dict[str, ExtractionProfileConfig] | None = None
1039# ---------------------------------------------------------------------------
1040# Loading
1041# ---------------------------------------------------------------------------
1044def _substitute_env_vars(value: str) -> str:
1045 """Replace ${VAR_NAME} with environment variable values."""
1047 def _replace(match: re.Match[str]) -> str:
1048 var_name = match.group(1)
1049 env_value = os.environ.get(var_name)
1050 if env_value is None:
1051 return match.group(0) # Leave unresolved
1052 return env_value
1054 return _ENV_VAR_PATTERN.sub(_replace, value)
1057def _find_unresolved_env_vars(
1058 data: dict | list | str | int | float | bool | None,
1059 path: str = "",
1060) -> list[str]:
1061 """Find all unresolved ${VAR_NAME} references after env var substitution.
1063 Returns list of strings like "vector_store_config.dsn: ${DATABASE_URL}".
1064 """
1065 unresolved: list[str] = []
1066 if isinstance(data, str):
1067 for match in _ENV_VAR_PATTERN.finditer(data):
1068 unresolved.append(f"{path}: ${{{match.group(1)}}}")
1069 elif isinstance(data, dict):
1070 for k, v in data.items():
1071 child_path = f"{path}.{k}" if path else str(k)
1072 unresolved.extend(_find_unresolved_env_vars(v, child_path))
1073 elif isinstance(data, list):
1074 for i, v in enumerate(data):
1075 unresolved.extend(_find_unresolved_env_vars(v, f"{path}[{i}]"))
1076 return unresolved
1079def _substitute_env_recursive(
1080 data: dict | list | str | int | float | bool | None,
1081) -> dict | list | str | int | float | bool | None:
1082 """Recursively substitute env vars in a parsed YAML structure."""
1083 if isinstance(data, str):
1084 return _substitute_env_vars(data)
1085 if isinstance(data, dict):
1086 return {k: _substitute_env_recursive(v) for k, v in data.items()}
1087 if isinstance(data, list):
1088 return [_substitute_env_recursive(item) for item in data]
1089 return data
1092def _load_profile(profile_name: str) -> dict:
1093 """Load a profile YAML from the profiles directory or a file path."""
1094 if profile_name.startswith("./") or profile_name.startswith("/"):
1095 profile_path = Path(profile_name)
1096 else:
1097 profile_path = _PROFILES_DIR / f"{profile_name}.yaml"
1099 if not profile_path.exists():
1100 raise ConfigError(f"Profile not found: {profile_path}")
1102 try:
1103 with open(profile_path) as f:
1104 return yaml.safe_load(f) or {}
1105 except yaml.YAMLError as exc:
1106 raise ConfigError(f"Invalid YAML in {profile_path}: {exc}") from exc
1109_COMPLIANCE_PROFILES_DIR = _PROFILES_DIR / "compliance"
1112def _load_compliance_profile(name: str) -> dict:
1113 """Load a compliance profile YAML (gdpr, hipaa, pdpa)."""
1114 if name.startswith("./") or name.startswith("/"):
1115 profile_path = Path(name)
1116 else:
1117 profile_path = _COMPLIANCE_PROFILES_DIR / f"{name}.yaml"
1119 if not profile_path.exists():
1120 raise ConfigError(f"Compliance profile not found: {profile_path}")
1122 try:
1123 with open(profile_path) as f:
1124 return yaml.safe_load(f) or {}
1125 except yaml.YAMLError as exc:
1126 raise ConfigError(f"Invalid YAML in {profile_path}: {exc}") from exc
1129def _deep_merge(base: dict, override: dict) -> dict:
1130 """Deep merge override into base. Override values win."""
1131 result = base.copy()
1132 for key, value in override.items():
1133 if key in result and isinstance(result[key], dict) and isinstance(value, dict):
1134 result[key] = _deep_merge(result[key], value)
1135 else:
1136 result[key] = value
1137 return result
1140def _filter_dataclass_fields(cls: type, data: dict, *, drop_none: bool = False) -> dict:
1141 """Filter dict to valid dataclass fields; optionally drop ``None`` values."""
1142 valid = {f.name for f in fields(cls)}
1143 return {k: v for k, v in data.items() if k in valid and (not drop_none or v is not None)}
1146_SENSITIVE_FIELD_PATTERNS = frozenset(
1147 _BASE_SENSITIVE_FIELD_KEYS
1148 + (
1149 "dsn",
1150 "connection_string",
1151 "credentials",
1152 "auth",
1153 "jwks_url",
1154 "issuer",
1155 "audience",
1156 )
1157)
1160def _is_sensitive_field(ref: str) -> bool:
1161 """Return True if an unresolved env-var reference is in a security-sensitive field."""
1162 # ref format: "path.to.field: ${VAR}" — extract field name before the colon
1163 field_path = ref.split(":")[0].strip().lower()
1164 return any(pat in field_path for pat in _SENSITIVE_FIELD_PATTERNS)
1167def _safe_sub_dict(data: dict, key: str) -> dict:
1168 """Safely extract a nested dict from *data*, defaulting to ``{}``."""
1169 val = data.get(key)
1170 return val if isinstance(val, dict) else {}
1173def _parse_homeostasis(data: dict) -> HomeostasisConfig:
1174 """Parse a homeostasis config block (used at top-level and per-bank)."""
1175 rl = _safe_sub_dict(data, "rate_limits")
1176 q = _safe_sub_dict(data, "quotas")
1177 return HomeostasisConfig(
1178 recall_max_tokens=data.get("recall_max_tokens"),
1179 reflect_max_tokens=data.get("reflect_max_tokens"),
1180 retain_max_content_bytes=data.get("retain_max_content_bytes"),
1181 rate_limits=RateLimitConfig(**_filter_dataclass_fields(RateLimitConfig, rl, drop_none=True)),
1182 quotas=QuotaConfig(**_filter_dataclass_fields(QuotaConfig, q, drop_none=True)),
1183 )
1186def _parse_benchmark_preset(data: dict) -> BenchmarkPresetConfig:
1187 """Parse versioned benchmark preset metadata with nested budget blocks."""
1188 preset = BenchmarkPresetConfig(**_filter_dataclass_fields(BenchmarkPresetConfig, data))
1189 for budget_name in ("low", "mid", "high"):
1190 budget_data = data.get(budget_name)
1191 if isinstance(budget_data, dict):
1192 setattr(
1193 preset,
1194 budget_name,
1195 BenchmarkBudgetConfig(**_filter_dataclass_fields(BenchmarkBudgetConfig, budget_data)),
1196 )
1197 return preset
1200def _parse_barriers(data: dict) -> BarrierConfig:
1201 """Parse a barriers config block (used at top-level and per-bank)."""
1202 return BarrierConfig(
1203 pii=PiiConfig(**_filter_dataclass_fields(PiiConfig, _safe_sub_dict(data, "pii"))),
1204 validation=ValidationConfig(**_filter_dataclass_fields(ValidationConfig, _safe_sub_dict(data, "validation"))),
1205 metadata=MetadataSanitizationConfig(
1206 **_filter_dataclass_fields(MetadataSanitizationConfig, _safe_sub_dict(data, "metadata"))
1207 ),
1208 )
1211def _parse_signal_quality(data: dict) -> SignalQualityConfig:
1212 """Parse a signal_quality config block (used at top-level and per-bank)."""
1213 return SignalQualityConfig(
1214 dedup=DedupConfig(**_filter_dataclass_fields(DedupConfig, _safe_sub_dict(data, "dedup"))),
1215 noisy_bank=NoisyBankConfig(**_filter_dataclass_fields(NoisyBankConfig, _safe_sub_dict(data, "noisy_bank"))),
1216 )
1219def _parse_escalation(data: dict) -> EscalationConfig:
1220 """Parse an ``escalation:`` config block."""
1221 cb = data.get("circuit_breaker", {})
1222 return EscalationConfig(
1223 circuit_breaker=CircuitBreakerConfig(**_filter_dataclass_fields(CircuitBreakerConfig, cb)),
1224 degraded_mode=data.get("degraded_mode", "empty_recall"),
1225 )
1228def _parse_recall_authority(data: dict) -> RecallAuthorityConfig:
1229 """Parse a ``recall_authority:`` config block."""
1230 tiers_raw = data.get("tiers") or []
1231 tiers: list[RecallAuthorityTierConfig] = []
1232 if isinstance(tiers_raw, list):
1233 for row in tiers_raw:
1234 if isinstance(row, dict):
1235 tiers.append(RecallAuthorityTierConfig(**_filter_dataclass_fields(RecallAuthorityTierConfig, row)))
1236 tb = data.get("tier_by_bank")
1237 tier_by_bank: dict[str, str] = {}
1238 if isinstance(tb, dict):
1239 tier_by_bank = {str(k): str(v) for k, v in tb.items()}
1240 return RecallAuthorityConfig(
1241 enabled=bool(data.get("enabled", False)),
1242 rules_inline=data.get("rules_inline"),
1243 rules_path=data.get("rules_path"),
1244 apply_to_reflect=bool(data.get("apply_to_reflect", True)),
1245 tier_by_bank=tier_by_bank,
1246 tiers=tiers,
1247 )
1250def _parse_access_grants(data: list) -> list[AccessGrant]:
1251 """Parse an ``access_grants:`` list, validating required fields."""
1252 grants: list[AccessGrant] = []
1253 for idx, row in enumerate(data):
1254 if not isinstance(row, dict):
1255 continue
1256 required_keys = ("bank_id", "principal", "permissions")
1257 missing = [k for k in required_keys if k not in row]
1258 if missing:
1259 raise ConfigError(
1260 f"Invalid access_grants entry at index {idx}: missing required field(s): {', '.join(missing)}"
1261 )
1262 if not isinstance(row["permissions"], list):
1263 raise ConfigError(f"Invalid access_grants entry at index {idx}: 'permissions' must be a list.")
1264 grants.append(
1265 AccessGrant(
1266 bank_id=str(row["bank_id"]),
1267 principal=str(row["principal"]),
1268 permissions=[str(p) for p in row["permissions"]],
1269 )
1270 )
1271 return grants
1274def _parse_lifecycle(data: dict) -> LifecycleConfig:
1275 """Parse a ``lifecycle:`` config block."""
1276 ttl_data = data.get("ttl", {})
1277 return LifecycleConfig(
1278 enabled=data.get("enabled", False),
1279 ttl=LifecycleTtlConfig(**_filter_dataclass_fields(LifecycleTtlConfig, ttl_data)),
1280 )
1283def _parse_banks(data: dict) -> dict[str, BankConfig]:
1284 """Parse a ``banks:`` config block with per-bank overrides."""
1285 banks: dict[str, BankConfig] = {}
1286 for bid, bdata in data.items():
1287 if not isinstance(bdata, dict):
1288 continue
1289 bc = BankConfig(
1290 profile=bdata.get("profile"),
1291 access=bdata.get("access"),
1292 )
1293 if "homeostasis" in bdata and isinstance(bdata["homeostasis"], dict):
1294 bc.homeostasis = _parse_homeostasis(bdata["homeostasis"])
1295 if "barriers" in bdata and isinstance(bdata["barriers"], dict):
1296 bc.barriers = _parse_barriers(bdata["barriers"])
1297 if "signal_quality" in bdata and isinstance(bdata["signal_quality"], dict):
1298 bc.signal_quality = _parse_signal_quality(bdata["signal_quality"])
1299 banks[str(bid)] = bc
1300 return banks
1303def _parse_agents(data: dict) -> dict[str, AgentRegistrationConfig]:
1304 """Parse an ``agents:`` config block."""
1305 agents: dict[str, AgentRegistrationConfig] = {}
1306 for aid, adata in data.items():
1307 if not isinstance(adata, dict):
1308 continue
1309 row = dict(adata)
1310 if row.get("banks") is None and row.get("allowed_banks") is not None:
1311 row["banks"] = list(row["allowed_banks"])
1312 agents[str(aid)] = AgentRegistrationConfig(**_filter_dataclass_fields(AgentRegistrationConfig, row))
1313 return agents
1316def _parse_deployment(data: dict) -> DeploymentConfig:
1317 """Parse a ``deployment:`` config block."""
1318 tls: TlsConfig | None = None
1319 if isinstance(data.get("tls"), dict):
1320 tls = TlsConfig(**_filter_dataclass_fields(TlsConfig, data["tls"]))
1321 dep_no_tls = {k: v for k, v in data.items() if k != "tls"}
1322 return DeploymentConfig(
1323 **_filter_dataclass_fields(DeploymentConfig, dep_no_tls),
1324 tls=tls,
1325 )
1328# Fields copied verbatim from the YAML dict onto AstrocyteConfig.
1329_SCALAR_CONFIG_FIELDS = (
1330 "provider_tier",
1331 "profile",
1332 "provider",
1333 "provider_config",
1334 "vector_store",
1335 "vector_store_config",
1336 "graph_store",
1337 "graph_store_config",
1338 "document_store",
1339 "document_store_config",
1340 "wiki_store",
1341 "wiki_store_config",
1342 "mental_model_store",
1343 "mental_model_store_config",
1344 "source_store",
1345 "source_store_config",
1346 "pageindex_store",
1347 "pageindex_store_config",
1348 "llm_provider",
1349 "llm_provider_config",
1350 "embedding_provider",
1351 "embedding_provider_config",
1352 "fallback_strategy",
1353)
1355# Sections whose value is passed through ``_filter_dataclass_fields`` directly.
1356_SIMPLE_SECTION_MAP: dict[str, type] = {
1357 "observability": ObservabilityConfig,
1358 "access_control": AccessControlConfig,
1359 "identity": IdentityConfig,
1360 "defaults": DefaultsConfig,
1361 "mcp": McpConfig,
1362 "recall_cache": RecallCacheConfig,
1363 "bm25_idf": Bm25IdfConfig,
1364 "source_aware_retrieval": SourceAwareRetrievalConfig,
1365 "tiered_retrieval": TieredRetrievalConfig,
1366 "cross_encoder_rerank": CrossEncoderRerankConfig,
1367 "spreading_activation": SpreadingActivationConfig,
1368 "causal_links": CausalLinksConfig,
1369 "entity_cooccurrence": EntityCooccurrenceConfig,
1370 "structured_fact_extraction": StructuredFactExtractionConfig,
1371 "query_analyzer": QueryAnalyzerConfig,
1372 "semantic_link_graph": SemanticLinkGraphConfig,
1373 "agentic_reflect": AgenticReflectConfig,
1374 "adversarial_defense": AdversarialDefenseConfig,
1375 "curated_retain": CuratedRetainConfig,
1376 "curated_recall": CuratedRecallConfig,
1377 "wiki_compile": WikiCompileConfig,
1378 "entity_resolution": EntityResolutionConfig,
1379 "async_tasks": AsyncTasksConfig,
1380 "dlp": DlpConfig,
1381 "directive_compile": DirectiveCompileConfig,
1382 # spreading_activation already in the map above (line ~1358) — duplicate
1383 # key removed when the M18a-3 duplicate dataclass was consolidated.
1384 "episodic_extract": EpisodicExtractConfig,
1385 "preference_compile": PreferenceCompileConfig,
1386}
1389def _dict_to_config(data: dict) -> AstrocyteConfig:
1390 """Convert a flat/nested dict to AstrocyteConfig with nested dataclasses."""
1391 config = AstrocyteConfig()
1393 # ── Scalar fields ──
1394 for field_name in _SCALAR_CONFIG_FIELDS:
1395 if field_name in data:
1396 setattr(config, field_name, data[field_name])
1398 # ── Simple nested sections (filter + construct) ──
1399 for section, cls in _SIMPLE_SECTION_MAP.items():
1400 if section in data:
1401 setattr(config, section, cls(**_filter_dataclass_fields(cls, data[section])))
1403 # ── Complex nested sections (dedicated parsers) ──
1404 if "homeostasis" in data:
1405 config.homeostasis = _parse_homeostasis(data["homeostasis"])
1407 if "barriers" in data:
1408 config.barriers = _parse_barriers(data["barriers"])
1410 if "escalation" in data:
1411 config.escalation = _parse_escalation(data["escalation"])
1413 if "signal_quality" in data:
1414 config.signal_quality = _parse_signal_quality(data["signal_quality"])
1416 if "benchmark_preset" in data and isinstance(data["benchmark_preset"], dict):
1417 config.benchmark_preset = _parse_benchmark_preset(data["benchmark_preset"])
1419 if "recall_authority" in data and isinstance(data["recall_authority"], dict):
1420 config.recall_authority = _parse_recall_authority(data["recall_authority"])
1422 if "access_grants" in data and data["access_grants"]:
1423 config.access_grants = _parse_access_grants(data["access_grants"])
1425 if "lifecycle" in data:
1426 config.lifecycle = _parse_lifecycle(data["lifecycle"])
1428 if "banks" in data and data["banks"]:
1429 config.banks = _parse_banks(data["banks"])
1431 if "extraction_profiles" in data and isinstance(data["extraction_profiles"], dict):
1432 profiles: dict[str, ExtractionProfileConfig] = {}
1433 for pname, pdata in data["extraction_profiles"].items():
1434 if isinstance(pdata, dict):
1435 profiles[str(pname)] = ExtractionProfileConfig(
1436 **_filter_dataclass_fields(ExtractionProfileConfig, pdata)
1437 )
1438 config.extraction_profiles = profiles
1440 if "sources" in data and isinstance(data["sources"], dict):
1441 sources: dict[str, SourceConfig] = {}
1442 for sid, sdata in data["sources"].items():
1443 if isinstance(sdata, dict):
1444 sources[str(sid)] = SourceConfig(**_filter_dataclass_fields(SourceConfig, sdata))
1445 config.sources = sources
1447 if "agents" in data and isinstance(data["agents"], dict):
1448 config.agents = _parse_agents(data["agents"])
1450 if "deployment" in data and isinstance(data["deployment"], dict):
1451 config.deployment = _parse_deployment(data["deployment"])
1453 # ── Scalar fallbacks ──
1454 if "compliance_profile" in data:
1455 config.compliance_profile = data["compliance_profile"]
1457 if "mip_config_path" in data:
1458 config.mip_config_path = data["mip_config_path"]
1459 elif "mip" in data and isinstance(data["mip"], str):
1460 config.mip_config_path = data["mip"]
1462 return config
1465def _agent_bank_list(ar: AgentRegistrationConfig) -> list[str]:
1466 if ar.banks:
1467 return list(ar.banks)
1468 if ar.allowed_banks:
1469 return list(ar.allowed_banks)
1470 return []
1473def _resolve_agent_bank_ids(
1474 patterns: list[str],
1475 declared: set[str] | None,
1476 *,
1477 label: str,
1478) -> list[str]:
1479 """Expand glob patterns against declared bank ids; validate literals when banks: is present."""
1480 if not patterns:
1481 return []
1482 out: list[str] = []
1483 for p in patterns:
1484 has_glob = any(c in p for c in "*?[")
1485 if has_glob:
1486 if not declared:
1487 raise ConfigError(f"{label}: bank pattern {p!r} uses wildcards but no banks: section is declared.")
1488 matches = sorted(bid for bid in declared if fnmatch.fnmatch(bid, p))
1489 if not matches:
1490 raise ConfigError(f"{label}: bank pattern {p!r} matches no declared banks.")
1491 out.extend(matches)
1492 elif declared is not None and p not in declared:
1493 raise ConfigError(f"{label}: bank id {p!r} is not listed under banks:.")
1494 else:
1495 out.append(p)
1496 return out
1499def validate_astrocyte_config(config: AstrocyteConfig) -> None:
1500 """Cross-field checks for ADR-003 sections (v0.5.0 with M1)."""
1501 # M9 / ADR-008: Apache AGE was removed entirely. Existing configs that
1502 # still set ``graph_store: age`` must be migrated. There is no migration
1503 # tool — operators rebuild banks from raw memory_units. See
1504 # docs/_design/adr/adr-008-section-graph-replaces-age.md.
1505 if (config.graph_store or "").strip().lower() == "age":
1506 raise ConfigError(
1507 "graph_store: age was removed in M9. Apache AGE is no longer "
1508 "supported; the recall path now uses flat tables + SQL CTEs "
1509 "(Hindsight pattern). To upgrade: remove the `graph_store: age` "
1510 "line from your config, then rebuild any AGE-backed banks from "
1511 "raw memory_units. See docs/_design/adr/adr-008-section-graph-"
1512 "replaces-age.md for rationale and the rebuild command."
1513 )
1515 # M9: ``extraction_mode: concise`` was removed. The legacy concise path
1516 # paraphrased chunk text, which lost the surface vocabulary question
1517 # embeddings rely on; recall_hit_rate dropped sharply (2026-05-02
1518 # finding). Verbatim is now the only supported mode.
1519 sfe_mode = (config.structured_fact_extraction.extraction_mode or "verbatim").strip().lower()
1520 if sfe_mode != "verbatim":
1521 raise ConfigError(
1522 f"structured_fact_extraction.extraction_mode={sfe_mode!r} is not "
1523 "supported. The 'concise' path was removed in M9 (it caused severe "
1524 "recall_hit_rate degradation — see the 2026-05-02 finding). Drop "
1525 "the extraction_mode line entirely (verbatim is the default), or "
1526 "set extraction_mode: verbatim explicitly."
1527 )
1529 if config.sources:
1530 from astrocyte.pipeline.extraction import merged_extraction_profiles
1532 profiles = merged_extraction_profiles(config)
1533 for name, src in config.sources.items():
1534 if not (src.type or "").strip():
1535 raise ConfigError(f"sources.{name}: type is required")
1536 st = (src.type or "").strip().lower()
1537 if st == "proxy":
1538 if not (src.url or "").strip():
1539 raise ConfigError(f"sources.{name}: type proxy requires url")
1540 if not (src.target_bank or "").strip():
1541 raise ConfigError(f"sources.{name}: type proxy requires target_bank")
1542 if st in ("poll", "api_poll"):
1543 driver = (src.driver or "").strip().lower()
1544 if driver != "github":
1545 raise ConfigError(
1546 f"sources.{name}: poll driver {driver!r} is not supported (use github; "
1547 "install astrocyte-ingestion-github)"
1548 )
1549 if not src.interval_seconds or int(src.interval_seconds) < 60:
1550 raise ConfigError(
1551 f"sources.{name}: type poll requires interval_seconds >= 60 (GitHub API rate limits)"
1552 )
1553 pr = (src.path or "").strip()
1554 if not pr or "/" not in pr or pr.count("/") != 1:
1555 raise ConfigError(
1556 f"sources.{name}: type poll with driver github requires path: owner/repo (got {pr!r})"
1557 )
1558 if not (src.target_bank or "").strip() and not (src.target_bank_template or "").strip():
1559 raise ConfigError(f"sources.{name}: type poll requires target_bank or target_bank_template")
1560 tok = (src.auth or {}).get("token") if src.auth else None
1561 if not (str(tok).strip() if tok is not None else ""):
1562 raise ConfigError(
1563 f"sources.{name}: type poll with driver github requires auth.token "
1564 "(GitHub personal access token or fine-grained token)"
1565 )
1566 if st == "stream":
1567 driver = (src.driver or "redis").strip().lower()
1568 if driver not in ("redis", "kafka"):
1569 raise ConfigError(f"sources.{name}: stream driver {driver!r} is not supported (use redis or kafka)")
1570 if not (src.url or "").strip():
1571 raise ConfigError(
1572 f"sources.{name}: type stream requires url (Redis URL or Kafka bootstrap servers)"
1573 )
1574 if not (src.topic or "").strip():
1575 tlabel = "Redis stream key / Kafka topic"
1576 raise ConfigError(f"sources.{name}: type stream requires topic ({tlabel})")
1577 if not (src.consumer_group or "").strip():
1578 raise ConfigError(f"sources.{name}: type stream requires consumer_group")
1579 if not (src.target_bank or "").strip() and not (src.target_bank_template or "").strip():
1580 raise ConfigError(f"sources.{name}: type stream requires target_bank or target_bank_template")
1581 if src.extraction_profile:
1582 if src.extraction_profile not in profiles:
1583 raise ConfigError(
1584 f"sources.{name}: extraction_profile {src.extraction_profile!r} not found under extraction_profiles"
1585 )
1587 if config.agents:
1588 declared = set(config.banks.keys()) if config.banks else None
1589 for agent_id, ar in config.agents.items():
1590 label = f"agents.{agent_id}"
1591 _resolve_agent_bank_ids(_agent_bank_list(ar), declared, label=label)
1593 ident = config.identity
1594 if ident.resolver is not None and ident.resolver not in ("convention", "config", "custom"):
1595 raise ConfigError(f"identity.resolver must be 'convention', 'config', or 'custom', got {ident.resolver!r}")
1597 if config.recall_authority.enabled and config.recall_authority.tiers:
1598 ra = config.recall_authority
1599 ids: list[str] = []
1600 for t in ra.tiers:
1601 tid = (t.id or "").strip()
1602 if not tid:
1603 raise ConfigError("recall_authority.tiers: each tier must have a non-empty id")
1604 ids.append(tid)
1605 if len(ids) != len(set(ids)):
1606 raise ConfigError("recall_authority.tiers: duplicate id")
1609def _grants_from_agents(config: AstrocyteConfig) -> list[AccessGrant]:
1610 if not config.agents:
1611 return []
1612 declared = set(config.banks.keys()) if config.banks else None
1613 out: list[AccessGrant] = []
1614 for agent_id, ar in config.agents.items():
1615 principal = ar.principal or f"agent:{agent_id}"
1616 perms = ar.permissions or ["read", "write"]
1617 label = f"agents.{agent_id}"
1618 for bid in _resolve_agent_bank_ids(_agent_bank_list(ar), declared, label=label):
1619 out.append(AccessGrant(bank_id=bid, principal=principal, permissions=list(perms)))
1620 return out
1623def _dedupe_grants(grants: list[AccessGrant]) -> list[AccessGrant]:
1624 seen: set[tuple[str, str, tuple[str, ...]]] = set()
1625 out: list[AccessGrant] = []
1626 for g in grants:
1627 key = (g.bank_id, g.principal, tuple(sorted(g.permissions)))
1628 if key in seen:
1629 continue
1630 seen.add(key)
1631 out.append(g)
1632 return out
1635def access_grants_for_astrocyte(config: AstrocyteConfig) -> list[AccessGrant]:
1636 """Flatten ``access_grants``, ``banks.*.access``, and ``agents:``-derived grants into one list for ``Astrocyte.set_access_grants``."""
1637 out: list[AccessGrant] = []
1638 if config.access_grants:
1639 out.extend(config.access_grants)
1640 if config.banks:
1641 for bank_id, bc in config.banks.items():
1642 if not bc.access:
1643 continue
1644 for idx, row in enumerate(bc.access):
1645 if not isinstance(row, dict):
1646 continue
1647 label = f"banks.{bank_id}.access[{idx}]"
1648 if "principal" not in row:
1649 raise ConfigError(f"{label} missing required key: principal")
1650 if "permissions" not in row or not isinstance(row["permissions"], list):
1651 raise ConfigError(f"{label} missing or invalid 'permissions' (must be a list)")
1652 out.append(
1653 AccessGrant(
1654 bank_id=bank_id,
1655 principal=str(row["principal"]),
1656 permissions=[str(p) for p in row["permissions"]],
1657 )
1658 )
1659 out.extend(_grants_from_agents(config))
1660 return _dedupe_grants(out)
1663def load_config(path: str | Path) -> AstrocyteConfig:
1664 """Load Astrocyte configuration from a YAML file.
1666 Resolution order: compliance profile → behavior profile → user config → per-bank overrides.
1667 Environment variables are substituted (${VAR_NAME}).
1668 """
1669 config_path = Path(path)
1670 if not config_path.exists():
1671 raise ConfigError(f"Config file not found: {config_path}")
1673 try:
1674 with open(config_path) as f:
1675 raw = yaml.safe_load(f) or {}
1676 except yaml.YAMLError as exc:
1677 raise ConfigError(f"Invalid YAML in {config_path}: {exc}") from exc
1679 # Substitute environment variables
1680 raw = _substitute_env_recursive(raw)
1682 # Check for unresolved env vars — fail on sensitive fields, warn on others
1683 unresolved = _find_unresolved_env_vars(raw)
1684 if unresolved:
1685 import logging
1687 _cfg_logger = logging.getLogger("astrocyte.config")
1688 sensitive = [r for r in unresolved if _is_sensitive_field(r)]
1689 if sensitive:
1690 raise ConfigError("Unresolved environment variables in sensitive config fields: " + "; ".join(sensitive))
1691 for ref in unresolved:
1692 _cfg_logger.warning("Unresolved environment variable in config: %s", ref)
1694 # Merge order: compliance (lowest) → behavior profile → user config (highest).
1695 # _deep_merge(base, override) → override wins.
1696 # Build base from lowest priority, then let higher priority layers override.
1697 base: dict = {}
1699 compliance_name = raw.get("compliance_profile")
1700 if compliance_name:
1701 compliance_data = _load_compliance_profile(compliance_name)
1702 base = _deep_merge(base, compliance_data)
1704 profile_name = raw.get("profile")
1705 if profile_name:
1706 profile_data = _load_profile(profile_name)
1707 base = _deep_merge(base, profile_data)
1709 # User config wins over everything
1710 merged = _deep_merge(base, raw)
1712 cfg = _dict_to_config(merged)
1713 if cfg.mip_config_path:
1714 mip = Path(cfg.mip_config_path)
1715 if not mip.is_absolute():
1716 cfg.mip_config_path = str((config_path.parent / mip).resolve())
1717 validate_astrocyte_config(cfg)
1718 return cfg