Coverage for astrocyte/config.py: 93%
810 statements
« prev ^ index » next coverage.py v7.15.0, created at 2026-10-29 22:10 +0000
1"""Astrocyte configuration — YAML loading, profile resolution, env var substitution."""
3from __future__ import annotations
5import fnmatch
6import os
7import re
8from collections.abc import Callable
9from dataclasses import dataclass, field, fields
10from pathlib import Path
11from typing import Any, Literal
13import yaml
15from astrocyte._text_files import read_user_text
16from astrocyte.errors import ConfigError
17from astrocyte.types import AccessGrant
19_ENV_VAR_PATTERN = re.compile(r"\$\{([A-Za-z_][A-Za-z0-9_]*)\}")
21_BASE_SENSITIVE_FIELD_KEYS = (
22 "api_key",
23 "password",
24 "token",
25 "secret",
26)
28# ---------------------------------------------------------------------------
29# Profile directory (shipped inside the package)
30# ---------------------------------------------------------------------------
31_PROFILES_DIR = Path(__file__).parent / "profiles"
34# ---------------------------------------------------------------------------
35# Configuration dataclasses
36# ---------------------------------------------------------------------------
39@dataclass
40class RateLimitConfig:
41 retain_per_minute: int | None = None
42 recall_per_minute: int | None = None
43 reflect_per_minute: int | None = None
44 global_per_minute: int | None = None
47@dataclass
48class QuotaConfig:
49 retain_per_day: int | None = None
50 reflect_per_day: int | None = None
53@dataclass
54class HomeostasisConfig:
55 recall_max_tokens: int | None = None
56 reflect_max_tokens: int | None = None
57 retain_max_content_bytes: int | None = None
58 rate_limits: RateLimitConfig = field(default_factory=RateLimitConfig)
59 quotas: QuotaConfig = field(default_factory=QuotaConfig)
62@dataclass
63class PiiConfig:
64 mode: str = "regex" # "regex" | "ner" | "llm" | "rules_then_llm" | "disabled"
65 action: str = "redact" # "redact" | "reject" | "warn"
66 patterns: list[dict[str, str]] | None = None
67 countries: list[str] | None = None # ["SG", "IN", "GB", "US", "DE", "FR", "IT", "ES", "AU", "CA", "JP", "CN"]
68 type_overrides: dict[str, dict[str, str]] | None = None # {"credit_card": {"action": "reject"}}
71@dataclass
72class ValidationConfig:
73 max_content_length: int = 50000
74 reject_empty_content: bool = True
75 reject_binary_content: bool = True
76 allowed_content_types: list[str] | None = None
79@dataclass
80class MetadataSanitizationConfig:
81 blocked_keys: list[str] = field(default_factory=lambda: list(_BASE_SENSITIVE_FIELD_KEYS))
82 max_metadata_size_bytes: int = 4096
85@dataclass
86class BarrierConfig:
87 pii: PiiConfig = field(default_factory=PiiConfig)
88 validation: ValidationConfig = field(default_factory=ValidationConfig)
89 metadata: MetadataSanitizationConfig = field(default_factory=MetadataSanitizationConfig)
92#: ``signal_quality.dedup.action`` values, mapped to the retain pipeline's
93#: (MIP ``dedup.action``) vocabulary. Config ``skip`` keeps its documented
94#: meaning — duplicates are dropped — which is the pipeline's ``skip_chunk``
95#: (drop duplicate chunks, store the rest); ``skip_chunk`` is accepted as an
96#: alias. ``warn`` stores everything; ``update`` is not implemented and behaves
97#: as ``skip_chunk``. Rejecting a whole retain when any chunk is a duplicate is
98#: MIP's ``skip``, available per rule only.
99DEDUP_ACTION_TO_PIPELINE = {"skip": "skip_chunk", "skip_chunk": "skip_chunk", "warn": "warn", "update": "update"}
100DEDUP_ACTIONS = tuple(DEDUP_ACTION_TO_PIPELINE)
101NOISY_BANK_ACTIONS = ("warn", "throttle", "reject")
104@dataclass
105class DedupConfig:
106 enabled: bool = True
107 similarity_threshold: float = 0.95
108 action: str = "skip" # "skip" | "skip_chunk" | "warn" | "update" — see DEDUP_ACTION_TO_PIPELINE
110 @property
111 def pipeline_action(self) -> str:
112 """``action`` in the retain pipeline's vocabulary (``skip`` -> ``skip_chunk``)."""
113 return DEDUP_ACTION_TO_PIPELINE.get(self.action, "skip_chunk")
115 #: Retain-time dedup also checks each chunk against its nearest stored
116 #: neighbours, so it holds across processes (a restarted daemon, a CLI
117 #: run, a gateway restart) and not only within one. One extra
118 #: ``search_similar`` per chunk: cheap on an ANN index, a full bank scan on
119 #: SqliteStore (~5 ms at 1k memories, ~25 ms at 10k, ~110 ms at 50k).
120 consult_store: bool = True
123@dataclass
124class NoisyBankConfig:
125 enabled: bool = True
126 retain_spike_multiplier: float = 5.0
127 min_avg_content_length: int = 20
128 max_dedup_rate: float = 0.8
129 action: str = "warn" # see NOISY_BANK_ACTIONS
132@dataclass
133class RecallCacheConfig:
134 enabled: bool = False
135 similarity_threshold: float = 0.95
136 max_entries: int = 256
137 ttl_seconds: float = 300.0
140@dataclass
141class Bm25IdfConfig:
142 """BM25-with-IDF full-text recall (M9).
144 When enabled, the recall pipeline routes its keyword strategy through
145 :meth:`PostgresStore.search_fulltext_bm25` instead of the classic
146 ``ts_rank_cd`` path. Requires migration 013 (the materialized views) to
147 be applied AND :meth:`PostgresStore.refresh_bm25_views` to have run at
148 least once after ingest — otherwise the views are empty and every
149 keyword recall returns no hits.
151 Off by default — opt-in only after the deployment has wired up a
152 refresh schedule. See :doc:`/_plugins/bm25-idf` for the operator guide.
153 """
155 enabled: bool = False
158@dataclass
159class SourceAwareRetrievalConfig:
160 """Source-aware retain + recall (M10).
162 Two switches with independent risk profiles:
164 - ``retain_provenance`` is cheap and almost-free: when a SourceStore is
165 configured AND this flag is on, retain creates one SourceDocument
166 and N SourceChunk rows per ingestion call and stamps each
167 ``VectorItem.chunk_id`` with the matching chunk's id. Costs one
168 extra DB roundtrip per retain. Required for any provenance feature
169 downstream (citations, chunk expansion) but not sufficient — the
170 SourceStore also has to be wired in YAML.
171 - ``chunk_expansion`` is the recall-side bet: when on, the recall
172 pipeline fetches sibling chunks of each top-K vector hit from the
173 SourceStore and merges them into the candidate set with a
174 configurable score multiplier. Helps multi-hop / split-evidence
175 questions where the answer key is in chunk N±1 of a chunk that
176 hit. Adds one DB roundtrip per top-K hit. **Off by default** —
177 enable in benchmark configs and measure before flipping in prod.
179 ``expansion_score_multiplier`` controls how much weight expanded
180 chunks get relative to the seed hit. ``1.0`` keeps them at parity
181 (often too aggressive); ``0.5`` is a conservative starting point.
182 ``expansion_max_per_hit`` caps fan-out so a long document doesn't
183 flood the candidate pool.
184 """
186 retain_provenance: bool = False
187 chunk_expansion: bool = False
188 expansion_score_multiplier: float = 0.5
189 expansion_max_per_hit: int = 4
192@dataclass
193class BenchmarkBudgetConfig:
194 """Named benchmark budget for Hindsight-parity preset routing."""
196 candidate_limit: int = 30
197 graph_expansion_limit: int = 30
198 rerank_top_k: int = 30
199 max_tokens: int = 8192
200 observation_weight: float = 0.0
201 agentic_reflect_allowed: bool = False
204@dataclass
205class BenchmarkPresetConfig:
206 """Versioned benchmark preset metadata.
208 Runtime operators can ignore this section; benchmark scripts and regression
209 tests use it to keep preset semantics explicit across branches.
210 """
212 name: str = "custom"
213 version: int = 1
214 budget: str = "mid"
215 low: BenchmarkBudgetConfig = field(
216 default_factory=lambda: BenchmarkBudgetConfig(
217 candidate_limit=12,
218 graph_expansion_limit=12,
219 rerank_top_k=12,
220 max_tokens=2048,
221 agentic_reflect_allowed=False,
222 )
223 )
224 mid: BenchmarkBudgetConfig = field(default_factory=BenchmarkBudgetConfig)
225 high: BenchmarkBudgetConfig = field(
226 default_factory=lambda: BenchmarkBudgetConfig(
227 candidate_limit=60,
228 graph_expansion_limit=40,
229 rerank_top_k=40,
230 max_tokens=16384,
231 observation_weight=0.25,
232 agentic_reflect_allowed=True,
233 )
234 )
237@dataclass
238class AdversarialDefenseConfig:
239 """Adversarial-question defense layer.
241 Targets the LoCoMo adversarial category (negative-existence,
242 false-premise, time-shift, cross-entity-confusion) where the LLM
243 left to its own devices invents an answer from weak retrieval hits.
245 Three layered guards (each independently configurable):
247 1. ``abstention_enabled`` + ``abstention_floor`` — score-floor
248 short-circuit. When all top-K recall hits score below the floor,
249 skip the LLM and return "insufficient evidence." Default floor
250 ``0.2`` is conservative — only fires when retrieval is genuinely
251 disconnected from the question.
253 **DEPRECATED:** ``abstention_enabled`` (the bool) is now a
254 fallback for callers that don't pass per-call
255 :class:`~astrocyte.types.Dispositions`. The orchestrator
256 derives the effective abstention floor from
257 ``dispositions.skepticism`` (1=trust→never abstain,
258 5=skeptical→aggressive). Migration path: set the disposition
259 on the bank or per-request instead of toggling this bool. The
260 legacy mapping is ``abstention_enabled=True`` ⇔ skepticism=3,
261 ``abstention_enabled=False`` ⇔ skepticism=1. ``abstention_floor``
262 (the float) remains the BASE that gets scaled by skepticism —
263 see :func:`astrocyte.pipeline.orchestrator._abstention_floor_for_skepticism`.
265 2. ``premise_verification_enabled`` — pre-loop premise extraction +
266 per-claim verification. The question is decomposed into atomic
267 claims; each is verified against memory before answering. Adds
268 1 LLM call per question; targets false-premise failures.
270 3. ``adversarial_prompt_enabled`` — tightens the agentic-reflect
271 system prompt with explicit "insufficient evidence is always a
272 valid answer" + premise-check rules. Free, defense-in-depth.
273 """
275 abstention_enabled: bool = False
276 abstention_floor: float = 0.2
277 premise_verification_enabled: bool = False
278 premise_verification_min_confidence: float = 0.6
279 adversarial_prompt_enabled: bool = False
282@dataclass
283class AgenticReflectConfig:
284 """Agentic reflect loop (Hindsight parity).
286 When enabled, ``reflect()`` runs an LLM-driven loop that selects
287 between two tools — ``recall`` (refine and re-retrieve) and ``done``
288 (commit the final answer with citations) — for up to
289 ``max_iterations`` turns. Targets multi-hop and open-domain queries
290 where a single retrieval often misses the bridge memory.
292 Cost: each turn is one LLM call. Default cap of 3 means worst-case
293 3× the LLM cost of single-shot reflect; typical case is 1-2 turns
294 when initial evidence is sufficient.
296 See ``astrocyte/pipeline/agentic_reflect.py`` for protocol details.
297 """
299 enabled: bool = False
300 max_iterations: int = 3
301 recall_step_max_results: int = 10
302 max_evidence_pool_size: int = 30
305@dataclass
306class SemanticLinkGraphConfig:
307 """Precomputed semantic-kNN graph at retain time (Hindsight parity, C3a).
309 Each new memory is linked to its top-``k`` most-similar existing
310 memories with cosine similarity ≥ ``similarity_threshold``. The
311 edges feed the link-expansion retrieval CTE as a parallel signal
312 alongside entity-overlap and causal links.
314 Costs one extra ``search_similar`` per chunk during retain (cheap
315 against the HNSW index). Disabled by default; opt-in for
316 benchmarks / production where multi-hop synthesis matters.
317 """
319 enabled: bool = False
320 top_k: int = 5
321 similarity_threshold: float = 0.7
324@dataclass
325class QueryAnalyzerConfig:
326 """Query-level temporal constraint extraction.
328 When ``enabled``, recall runs the regex pre-pass to extract
329 temporal expressions ("last week", "in March 2024", "yesterday")
330 into a time_range filter. Free of LLM cost.
332 ``allow_llm_fallback`` opts in to a structured-JSON LLM extraction
333 for queries that contain a temporal marker but no regex match
334 (e.g. "last spring", "around the launch"). Adds 1 LLM call per
335 such query; gated to keep cost predictable.
337 A caller-supplied ``RecallRequest.time_range`` always wins over
338 the analyzer's extraction.
339 """
341 enabled: bool = False
342 allow_llm_fallback: bool = True
343 #: M18a-1 — extended relative-time patterns plus Hindsight-parity
344 #: dateparser Pass B. The chain runs:
345 #: 1. ``_try_iso_date`` / ``_try_relative`` — narrow exact regex
346 #: 2. ``_try_temporal_expansion`` (Pass A) — fuzzy "a few weeks ago"
347 #: 3. ``_try_month_year`` / ``_try_year`` — calendar references
348 #: 4. ``_try_dateparser`` (Pass B) — wide-net for named dates,
349 #: weekdays, ordinals, ISO refs (requires ``dateparser`` pkg —
350 #: see ``pyproject.toml`` ``[bench]`` extras; no-ops if missing)
351 #: The extracted date range is fed to ``fact_recall`` which RRF-
352 #: fuses semantic + episodic + temporal as parallel siblings
353 #: (Hindsight architecture), so false-positive dateparser hits are
354 #: damped by cross-strategy agreement scoring.
355 #:
356 #: **Default True as of M18b ship** (2026-05-17): the B1-dp+RRF
357 #: 2-run mean was 83.75% LoCoMo (+3.25pp vs B0 baseline 80.5%,
358 #: +2.17σ above M17 mean — cleared M17+1σ ship gate by 1.75pp).
359 #: Bench env override ``ASTROCYTE_M18_ENABLE_TEMPORAL_EXPANSION=0``
360 #: forces off for ablation replication.
361 enable_temporal_expansion: bool = True
364@dataclass
365class StructuredFactExtractionConfig:
366 """Single-pass structured fact extraction at retain time.
368 When enabled, replaces the legacy chunk + entity-extraction +
369 fact-causal-extraction three-pass pipeline with a single LLM call
370 that produces structured metadata (entities, causal_relations,
371 temporal range, where/who/why annotations) per memory.
373 Two extraction modes:
375 - ``"verbatim"`` (recommended for benchmarks): pre-chunks the
376 text, then asks the LLM to produce per-chunk metadata. The
377 stored memory text is the ORIGINAL chunk text — preserves
378 vocabulary for embedding-match against questions. Adds rich
379 metadata without losing surface terms.
381 - ``"concise"``: the LLM generates atomic fact statements that
382 replace the chunks. The stored text is the LLM's structured
383 paraphrase (``"what | Involving: who | why"``). Higher-density
384 knowledge representation, BUT loses surface vocabulary which
385 can hurt recall_hit_rate when questions share words with the
386 original text. Documented regression on LoCoMo 2026-05-02.
388 Each fact becomes ONE memory in both modes. The structured fields
389 populate ``metadata['_fact_*']`` either way.
391 Cost approximately equal to the legacy two-pass (one LLM call
392 replaces two); output substantially richer, especially for
393 multi-hop and temporal questions.
395 Net: opt-in, defaults conservative.
396 """
398 enabled: bool = False
399 #: Only ``"verbatim"`` is supported; the ``"concise"`` mode was removed
400 #: in M9 because it caused severe recall_hit_rate degradation (the
401 #: 2026-05-02 finding — LLM-paraphrased ``what`` fields lost the
402 #: surface vocabulary that question embeddings share). The field is
403 #: kept so existing configs parse, but ``validate_astrocyte_config``
404 #: raises ``ConfigError`` if any other value is set.
405 extraction_mode: str = "verbatim"
406 max_facts_per_call: int = 30
407 #: Chunking strategy used by verbatim mode to pre-chunk the
408 #: source text before LLM enrichment. SFE has different
409 #: granularity needs than legacy retrieval — measured wins on
410 #: LoCoMo with ``"paragraph"`` (large context per chunk → LLM
411 #: extracts richer metadata) vs ``"dialogue"`` (legacy default
412 #: for conversations, which fragments context and lost 2.5 pts
413 #: overall, 8.4 pts on open-domain). Override per workload.
414 #: Available: ``"paragraph"``, ``"dialogue"``, ``"sentence"``.
415 chunk_strategy: str = "paragraph"
416 #: Per-chunk character budget for verbatim SFE pre-chunking. When
417 #: None, falls through to the orchestrator's default chunk size
418 #: (512 chars for Astrocyte legacy compat). Bumping this reduces
419 #: chunk count per session, which is the dominant retain throughput
420 #: lever — the LLM produces one ``facts[]`` entry per chunk and
421 #: gpt-4o-mini latency is roughly linear in output tokens. 2048 is
422 #: a measured sweet spot for LongMemEval-shaped traffic
423 #: (avg ~12.7 KB per session → ~6 chunks instead of ~25).
424 #:
425 #: This overrides any chunk-size set by extraction profile or MIP
426 #: rules for the SFE path specifically; legacy chunking for non-SFE
427 #: retain is unaffected.
428 chunk_max_size: int | None = None
429 #: Per-chunk parallel verbatim extraction (Phase 3 of cost-control
430 #: port). When ``True``, sends one LLM call per chunk in parallel
431 #: (asyncio.gather with a semaphore), instead of one batched call
432 #: that asks for all N chunks' metadata at once. Drops cross-chunk
433 #: causal_relations — set ``causal_links.enabled: false`` to be
434 #: explicit. Default off so existing deployments are unaffected.
435 parallel_chunks: bool = False
436 #: Max in-flight LLM calls per session when ``parallel_chunks`` is
437 #: True. Should be ≤ avg chunks/session for the workload. 6 is the
438 #: sweet spot for LongMemEval-shaped traffic at chunk_max_size=2048
439 #: (~6 chunks per session).
440 parallel_chunks_max_concurrency: int = 6
443@dataclass
444class EntityCooccurrenceConfig:
445 """Co-occurrence link creation between entities in the same memory.
447 When retain extracts N entities from a single memory, the
448 orchestrator creates ``co_occurs`` ``EntityLink`` rows between
449 them so spreading activation at recall time can hop across these
450 edges (Hindsight parity).
452 The all-pairs Cartesian product is **O(N²) per retain**. The
453 2026-05-06 LME profile measured this stage at 34% of total retain
454 wall time, with per-call cost growing as the entity-link table
455 grew (52s tail latency on a single retain at session 100). Capping
456 the number of entities used to form co-occurrence pairs bounds the
457 work to O(K²) per retain regardless of corpus size.
459 ``max_entities_per_memory=5`` (default) produces ``C(5,2)=10``
460 links per retain — a 43× reduction vs the unbounded 30-entity
461 sessions LME emits. Top-K entities are selected by their order in
462 the extraction result (typically tracks prominence in the source
463 text). When N ≤ K the cap is a no-op.
465 Disable entirely (``enabled=False``) to skip co-occurrence link
466 creation altogether — useful for retain-heavy bench runs where
467 spreading activation isn't the critical recall signal.
468 """
470 enabled: bool = True
471 max_entities_per_memory: int = 5
474@dataclass
475class CausalLinksConfig:
476 """Cause→effect link extraction at retain time (Hindsight parity).
478 When enabled, retain runs an additional LLM pass that identifies
479 causal relationships between extracted entities and persists them
480 as ``EntityLink(link_type="causes", ...)`` rows. These edges feed
481 temporal spreading activation's "trace reasoning chains" path.
483 Costs one extra LLM call per record. Disabled by default — opt in
484 for benchmarks / production where causal walks add value.
485 """
487 enabled: bool = False
488 max_pairs_per_memory: int = 4
489 min_confidence: float = 0.7
492@dataclass
493class SpreadingActivationConfig:
494 """Spreading activation through entity links (Hindsight parity).
496 When enabled, recall expands the seed RRF result set by walking
497 entity-link edges (default: ``co_occurs``) out to ``max_hops``,
498 decaying activation per hop. Spread hits join the candidate pool
499 before final cross-encoder rerank, with metadata tags
500 (``_spread_hop``, ``_spread_via_entity``) so synthesis can tell
501 them apart from direct evidence.
503 Defaults are conservative — leave disabled until the workload is
504 verified to benefit (multi-hop / open-domain QA categories) and
505 monitor single-hop precision for regressions caused by spread noise.
506 """
508 enabled: bool = False
509 max_hops: int = 2
510 decay_per_hop: float = 0.6
511 expansion_limit: int = 30
512 activation_threshold: float = 0.2
513 link_types: list[str] = field(default_factory=lambda: ["co_occurs", "causes"])
514 #: Hindsight blog 2026-03-12 — temporal proximity bonus on top of
515 #: the entity-link spread. ``0.0`` disables (entity-link only).
516 temporal_proximity_weight: float = 0.3
517 temporal_half_life_days: float = 7.0
520@dataclass
521class CrossEncoderRerankConfig:
522 """Final-stage cross-encoder reranker (Hindsight parity).
524 When ``enabled=True``, :meth:`PipelineOrchestrator._rank_reflect_context`
525 uses a real cross-encoder (sentence-transformers backend) to rerank the
526 top-``top_k`` recall hits before synthesis. When disabled (default),
527 the existing ``cross_encoder_like_rerank`` heuristic runs instead.
529 The default model — ``cross-encoder/ms-marco-MiniLM-L-6-v2`` — matches
530 Hindsight's default and is the standard MS MARCO baseline.
531 """
533 enabled: bool = False
534 model_name: str = "cross-encoder/ms-marco-MiniLM-L-6-v2"
535 top_k: int = 30
536 force_cpu: bool = False
539@dataclass
540class TieredRetrievalConfig:
541 enabled: bool = False
542 min_results: int = 3
543 min_score: float = 0.3
544 max_tier: int = 3 # 0-4
545 #: Which recall path runs at tier 3+ (and tier 4 after reformulation). ``pipeline`` = built-in
546 #: pipeline only (default). ``hybrid`` = :class:`~astrocyte.hybrid.HybridEngineProvider` merge
547 #: (engine + pipeline); requires a hybrid engine provider and ``tiered_retrieval.enabled``.
548 full_recall: Literal["pipeline", "hybrid"] = "pipeline"
551@dataclass
552class RecallAuthorityTierConfig:
553 """One precedence band for :class:`RecallAuthorityConfig` (matches ``metadata[\"authority_tier\"]``)."""
555 id: str = ""
556 priority: int = 1
557 label: str = ""
560@dataclass
561class RecallAuthorityConfig:
562 """Structured recall authority — labels fused hits for synthesis (M7)."""
564 enabled: bool = False
565 rules_inline: str | None = None
566 rules_path: str | None = None
567 tiers: list[RecallAuthorityTierConfig] = field(default_factory=list)
568 #: When True, :meth:`Astrocyte.reflect` / pipeline reflect inject ``authority_context`` into the synthesis prompt.
569 apply_to_reflect: bool = True
570 #: Default ``metadata[\"authority_tier\"]`` for vectors in a bank (profile ``authority_tier`` overrides).
571 tier_by_bank: dict[str, str] = field(default_factory=dict)
574@dataclass
575class CuratedRetainConfig:
576 enabled: bool = False
577 model: str | None = None
578 context_recall_limit: int = 5
581@dataclass
582class CuratedRecallConfig:
583 enabled: bool = False
584 freshness_weight: float = 0.3
585 reliability_weight: float = 0.2
586 salience_weight: float = 0.2
587 original_score_weight: float = 0.3
588 freshness_half_life_days: float = 30.0
589 min_score: float | None = None
592@dataclass
593class SignalQualityConfig:
594 dedup: DedupConfig = field(default_factory=DedupConfig)
595 noisy_bank: NoisyBankConfig = field(default_factory=NoisyBankConfig)
598@dataclass
599class CircuitBreakerConfig:
600 failure_threshold: int = 5
601 recovery_timeout_seconds: float = 30.0
602 half_open_max_calls: int = 2
605@dataclass
606class EscalationConfig:
607 circuit_breaker: CircuitBreakerConfig = field(default_factory=CircuitBreakerConfig)
608 degraded_mode: str = "empty_recall" # "empty_recall" | "error" | "cache"
611@dataclass
612class ObservabilityConfig:
613 otel_enabled: bool = False
614 prometheus_enabled: bool = False
615 log_level: str = "info"
618@dataclass
619class AccessControlConfig:
620 enabled: bool = False
621 default_policy: str = "owner_only" # "owner_only" | "open" | "deny"
624@dataclass
625class JwtMiddlewareConfig:
626 """JWT identity middleware (identity spec §3 Gap 1 wiring).
628 When ``enabled``, the MCP server extracts the ``Authorization: Bearer``
629 token from each inbound request, validates it against the configured
630 JWKS, classifies the claims via :mod:`astrocyte.identity_jwt`, and
631 populates :attr:`AstrocyteContext.actor` with a resolved
632 :class:`ActorIdentity` for the call. See
633 ``docs/_plugins/jwt-identity-middleware.md`` for the operator guide.
635 When ``enabled=False`` (default), the MCP server preserves pre-middleware
636 behavior: a single static ``AstrocyteContext`` is used for all calls.
637 """
639 enabled: bool = False
640 #: JWKS endpoint for signature key retrieval. Required when enabled.
641 jwks_uri: str | None = None
642 #: Expected token ``aud`` claim. Required when enabled — unset audience
643 #: is a common misconfiguration that can result in cross-tenant accepts.
644 token_audience: str | None = None
645 #: Expected token ``iss`` claim. Validated when set; left unchecked when
646 #: None (some IdPs rotate issuers).
647 token_issuer: str | None = None
648 #: Signing algorithms accepted. Defaults to asymmetric only so HS* keys
649 #: stolen from misconfigured deployments can't forge tokens.
650 algorithms: list[str] = field(default_factory=lambda: ["RS256", "ES256"])
651 #: When True, a missing or malformed Authorization header raises.
652 #: When False (with ``allow_anonymous=True``), falls through to anonymous.
653 fail_closed: bool = True
654 #: Permit calls with no Authorization header. Ignored when fail_closed=True.
655 allow_anonymous: bool = False
656 #: JWKS cache refresh interval. Most JWKS endpoints rotate every 24h.
657 jwks_refresh_interval_hours: int = 24
660@dataclass
661class IdentityConfig:
662 """Identity-driven bank resolution and ACL helpers (M1–M2 / v0.5.0)."""
664 auto_resolve_banks: bool = False
665 user_bank_prefix: str = "user-"
666 agent_bank_prefix: str = "agent-"
667 service_bank_prefix: str = "service-"
668 resolver: Literal["convention", "config", "custom"] | None = None
669 obo_enabled: bool = False
670 #: JWT identity middleware wiring (identity spec §3 Gap 1).
671 jwt_middleware: JwtMiddlewareConfig = field(default_factory=JwtMiddlewareConfig)
674# ---------------------------------------------------------------------------
675# M2 — Config schema evolution (ADR-003, v0.5.0 with M1)
676# ---------------------------------------------------------------------------
679@dataclass
680class SourceConfig:
681 """External data source definition (``astrocyte.ingest``).
683 * **webhook** — HTTP push; gateway / ASGI binds the route.
684 * **stream** — long-running consumer — ``driver: redis`` (Redis Streams) or ``kafka`` (Kafka).
685 Requires ``url``, ``topic``, ``consumer_group``, ``target_bank`` / ``target_bank_template``.
686 For Redis, ``url`` is a Redis URL; for Kafka, ``url`` is bootstrap servers (e.g. ``localhost:9092``).
687 Optional ``path``: Redis consumer name or Kafka ``client_id``.
688 * **poll** / **api_poll** — scheduled HTTP pull — ``driver: github`` (``astrocyte-ingestion-github``).
689 Requires ``interval_seconds``, ``path`` as ``owner/repo``, ``target_bank`` (or template), and
690 ``auth.token`` (or env-substituted) for the GitHub API. Optional ``url`` overrides the API base
691 (default ``https://api.github.com``; use GitHub Enterprise ``.../api/v3`` when needed).
692 """
694 type: str = ""
695 extraction_profile: str | None = None
696 target_bank: str | None = None
697 target_bank_template: str | None = None
698 principal: str | None = None
699 auth: dict[str, str | int | float | bool | None] | None = None
700 path: str | None = None
701 driver: str | None = None
702 topic: str | None = None
703 consumer_group: str | None = None
704 url: str | None = None
705 interval_seconds: int | None = None
706 # M4.1 proxy recall: GET (default) or POST JSON to ``url``
707 recall_method: str | None = None # "GET" | "POST"
708 recall_body: Any | None = None # POST JSON: dict/str with placeholders (see ``astrocyte.recall.proxy``)
709 # Per-source cap on one proxy recall request, in seconds. None = the shared
710 # deadline (``ASTROCYTE_PROXY_RECALL_DEADLINE_SECONDS``, default 0.8 s);
711 # a value above the deadline is capped by it. See federated-sources §4.
712 recall_timeout_seconds: float | None = None
715@dataclass
716class AgentRegistrationConfig:
717 """Registered agent with bank access and optional rate hints (ADR-003 / v0.5.0)."""
719 principal: str | None = None
720 banks: list[str] | None = None
721 allowed_banks: list[str] | None = None # roadmap alias for banks; glob patterns allowed
722 default_bank: str | None = None
723 permissions: list[str] | None = None
724 max_retain_per_minute: int | None = None
725 max_recall_per_minute: int | None = None
728@dataclass
729class TlsConfig:
730 cert_path: str | None = None
731 key_path: str | None = None
734@dataclass
735class DeploymentConfig:
736 """Standalone gateway settings; ignored in library mode."""
738 mode: Literal["library", "standalone", "plugin"] = "library"
739 host: str | None = None
740 port: int | None = None
741 workers: int | None = None
742 cors_origins: list[str] | None = None
743 tls: TlsConfig | None = None
746@dataclass
747class ExtractionProfileConfig:
748 """Reusable extraction defaults for sources (pipeline implementation in M3)."""
750 content_type: str | None = None
751 chunking_strategy: str | None = None
752 entity_extraction: bool | str | None = None
753 metadata_mapping: dict[str, str] | None = None
754 tag_rules: list[dict[str, str | list[str]]] | None = None
755 chunk_size: int | None = None
756 fact_type: str | None = None # default "world"; e.g. "experience", "observation"
757 #: Optional recall-authority band id (overrides ``recall_authority.tier_by_bank`` for this profile).
758 authority_tier: str | None = None
761@dataclass
762class McpConfig:
763 default_bank_id: str | None = None
764 expose_reflect: bool = True
765 expose_forget: bool = False
766 expose_admin: bool = False
767 max_results_limit: int = 50
768 principal: str | None = None
771@dataclass
772class DefaultsConfig:
773 """Per-profile default settings."""
775 skepticism: int = 3
776 literalism: int = 3
777 empathy: int = 3
778 preferred_fact_types: list[str] | None = None
779 tags: list[str] | None = None
782@dataclass
783class DlpConfig:
784 """Data Loss Prevention — output scanning for PII in recall/reflect results."""
786 scan_recall_output: bool = False
787 scan_reflect_output: bool = False
788 output_pii_action: str = "warn" # "redact" | "reject" | "warn"
791@dataclass
792class LifecycleTtlConfig:
793 archive_after_days: int = 90 # Days since last recall before archiving
794 delete_after_days: int = 365 # Days since creation before deletion
795 exempt_tags: list[str] | None = None # Tags that exempt from TTL
796 fact_type_overrides: dict[str, int | None] | None = None # Override archive_after_days by fact_type
799@dataclass
800class LifecycleConfig:
801 enabled: bool = False
802 ttl: LifecycleTtlConfig = field(default_factory=LifecycleTtlConfig)
805@dataclass
806class WikiCompileConfig:
807 """Configuration for M8 wiki compile and its background trigger."""
809 enabled: bool = False
810 auto_start: bool = False
811 size_threshold: int = 50
812 staleness_days: float = 7.0
813 staleness_min_memories: int = 10
814 max_queue_size: int = 100
817@dataclass
818class PreferenceCompileConfig:
819 """M18a-4 follow-up — preference consolidation gate (Hindsight parity).
821 Per-document LLM pass that distills ``fact_type='preference'`` facts
822 into ``MentalModel(kind='preference')`` rows for advisory recall.
823 Default ON (matches pre-M18 bench behavior; was always run inline).
825 Disable to ablate the preference tier (useful for measuring its
826 isolated contribution vs the directive tier).
827 """
829 enabled: bool = True
832@dataclass
833class EpisodicExtractConfig:
834 """M18a-4 — episodic event index (tag + retrieve).
836 Two integration points (both bench-gated today; core orchestrator
837 hook deferred to a separate cycle):
839 1. **Retain side**: after fact extraction for a document,
840 ``tag_episodic_facts(all_facts)`` walks the fact list and tags
841 any matching an episodic verb pattern ("attended/visited/met X
842 at/during Y") by appending ``EPISODIC_MARKER`` ("episodic:event")
843 to the fact's ``entities`` array. Uses the existing fact_type
844 schema (migration 020) — no new SQL value, just a namespaced
845 marker entity. See ``astrocyte/pipeline/episodic_extract.py``.
847 2. **Recall side**: when ``question_has_episodic_cue(query)`` matches
848 (question contains "where did I", "when did I meet", etc.),
849 the recall path explicitly queries
850 ``store.search_facts_by_entity(bank_id, EPISODIC_MARKER, ...)``
851 to surface the episodic-tagged facts alongside generic
852 semantic hits.
854 Both call sites are bench-gated by ``ASTROCYTE_M18_ENABLE_EPISODIC_EXTRACTION=1``
855 today. When the orchestrator gains a "document post-process" hook,
856 the retain-side tagger moves there; the recall-side search becomes
857 a new strategy in ``section_recall``.
858 """
860 enabled: bool = False
863# NOTE: SpreadingActivationConfig is defined ~earlier in this file (line ~467).
864# The earlier definition is the live one — it carries the Hindsight-parity
865# link-walk fields (``max_hops``, ``decay_per_hop``, ``link_types``,
866# ``temporal_proximity_weight``) used by ``link_expansion.py`` and read in
867# ``_astrocyte.py:379``. The duplicate definition removed here was an
868# M18a-3 stub with ``seed_count``/``top_k`` fields that were never wired
869# (the actual M18a-3 entity-co-occurrence implementation takes those as
870# function parameters in ``section_recall.py``, not config fields).
873@dataclass
874class DirectiveCompileConfig:
875 """M18a-2 — directive-style preference compile (Hindsight parity).
877 Hindsight's ``mental_models.subtype='directive'`` rows are user-
878 curated hard rules treated as authoritative by the answerer. We
879 auto-distill preference facts into a tight directive tier (≤5 rows
880 per document) via ``astrocyte.pipeline.directive_compile``. The
881 advisory preference tier (consolidation pass) stays in the store
882 on-demand-recallable; only the directive tier surfaces verbatim.
884 Currently called from the bench harness pipeline only — the core
885 orchestrator does not yet have a "document post-process" hook for
886 document-level passes (preference_compile, directive_compile,
887 episodic_extract). Adding that hook is a separate cycle; until
888 then this flag controls the bench-side call only.
890 Bench-time env-var override: ``ASTROCYTE_M18_ENABLE_DIRECTIVE_COMPILE=1``.
892 **DEPRECATED (M19, 2026-05-18):** the auto-compile architecture is
893 misaligned with Hindsight's user-authored ``create_directive`` MCP
894 tool. Bench evidence (M18b, B2 × 2 runs) showed a replicated −30pp
895 SSP regression because the compressed directives override the
896 answerer's access to the original preference facts' nuance. Keep
897 ``enabled=False``; the in-tree code stays as reference for the
898 future user-authored MCP-tool path (M20+). Setting ``enabled=True``
899 will emit a runtime warning when ``run_document_postprocess`` runs.
900 """
902 enabled: bool = False
905@dataclass
906class EntityResolutionConfig:
907 """Configuration for M11 retain-time entity resolution."""
909 enabled: bool = False
910 defer_to_async: bool = False
911 similarity_threshold: float = 0.8
912 confirmation_threshold: float = 0.75
913 max_candidates_per_entity: int = 3
914 enable_llm_disambiguation: bool = True
915 canonical_resolution: bool = False
918@dataclass
919class AsyncTasksConfig:
920 """Configuration for durable background memory tasks."""
922 enabled: bool = False
923 backend: str = "pgqueuer" # "pgqueuer" | "pgqueuer_in_memory"
924 dsn: str | None = None
925 install_on_start: bool = False
926 auto_start_worker: bool = False
927 batch_size: int = 10
930@dataclass
931class BankConfig:
932 """Per-bank override settings.
934 ``homeostasis``, ``barriers`` and ``signal_quality`` are the bank's
935 *resolved* sections when loaded from YAML: the top-level section, then the
936 bank's ``profile``, then the bank's own block (see ``_parse_banks``).
937 ``None`` means the bank uses the top-level section. Read them through
938 ``AstrocyteConfig.bank_homeostasis`` / ``bank_barriers`` /
939 ``bank_signal_quality``.
940 """
942 profile: str | None = None
943 access: list[dict[str, str | list[str]]] | None = None
944 homeostasis: HomeostasisConfig | None = None
945 barriers: BarrierConfig | None = None
946 signal_quality: SignalQualityConfig | None = None
949@dataclass
950class AstrocyteConfig:
951 """Top-level Astrocyte configuration."""
953 # Provider tier
954 provider_tier: Literal["storage", "engine"] = "engine"
956 # Profile
957 profile: str | None = None
959 # Engine provider
960 provider: str | None = None
961 provider_config: dict[str, str | int | float | bool | None] | None = None
963 # Tier 1: Storage
964 vector_store: str | None = None
965 vector_store_config: dict[str, str | int | float | bool | None] | None = None
966 graph_store: str | None = None
967 graph_store_config: dict[str, str | int | float | bool | None] | None = None
968 document_store: str | None = None
969 document_store_config: dict[str, str | int | float | bool | None] | None = None
970 wiki_store: str | None = None
971 wiki_store_config: dict[str, str | int | float | bool | None] | None = None
972 # M9 first-class mental-model storage (replaces wiki-piggyback path).
973 # Optional — when unset, /v1/mental-models endpoints raise 501.
974 mental_model_store: str | None = None
975 mental_model_store_config: dict[str, str | int | float | bool | None] | None = None
976 # M10 source-document / chunk store (provenance hierarchy). Optional —
977 # when unset, vectors remain anonymous flat rows (backward compatible).
978 source_store: str | None = None
979 source_store_config: dict[str, str | int | float | bool | None] | None = None
980 # M9 section recall — PageIndex tree + section graph. Optional.
981 # When unset, section recall is disabled (the SPI is gated on this
982 # store being configured). See docs/_design/recall.md.
983 pageindex_store: str | None = None
984 pageindex_store_config: dict[str, str | int | float | bool | None] | None = None
986 # LLM
987 llm_provider: str | None = None
988 llm_provider_config: dict[str, str | int | float | bool | None] | None = None
989 embedding_provider: str | None = None
990 embedding_provider_config: dict[str, str | int | float | bool | None] | None = None
992 # Fallback
993 fallback_strategy: str = "error" # "local_llm" | "error" | "degrade"
995 # Policy
996 homeostasis: HomeostasisConfig = field(default_factory=HomeostasisConfig)
997 barriers: BarrierConfig = field(default_factory=BarrierConfig)
998 signal_quality: SignalQualityConfig = field(default_factory=SignalQualityConfig)
999 escalation: EscalationConfig = field(default_factory=EscalationConfig)
1000 observability: ObservabilityConfig = field(default_factory=ObservabilityConfig)
1001 access_control: AccessControlConfig = field(default_factory=AccessControlConfig)
1002 identity: IdentityConfig = field(default_factory=IdentityConfig)
1003 defaults: DefaultsConfig = field(default_factory=DefaultsConfig)
1005 # MCP
1006 mcp: McpConfig = field(default_factory=McpConfig)
1008 # Phase 2 innovations
1009 recall_cache: RecallCacheConfig = field(default_factory=RecallCacheConfig)
1010 bm25_idf: Bm25IdfConfig = field(default_factory=Bm25IdfConfig)
1011 source_aware_retrieval: SourceAwareRetrievalConfig = field(
1012 default_factory=SourceAwareRetrievalConfig,
1013 )
1014 tiered_retrieval: TieredRetrievalConfig = field(default_factory=TieredRetrievalConfig)
1015 cross_encoder_rerank: CrossEncoderRerankConfig = field(default_factory=CrossEncoderRerankConfig)
1016 spreading_activation: SpreadingActivationConfig = field(default_factory=SpreadingActivationConfig)
1017 causal_links: CausalLinksConfig = field(default_factory=CausalLinksConfig)
1018 entity_cooccurrence: EntityCooccurrenceConfig = field(default_factory=EntityCooccurrenceConfig)
1019 structured_fact_extraction: StructuredFactExtractionConfig = field(default_factory=StructuredFactExtractionConfig)
1020 query_analyzer: QueryAnalyzerConfig = field(default_factory=QueryAnalyzerConfig)
1021 semantic_link_graph: SemanticLinkGraphConfig = field(default_factory=SemanticLinkGraphConfig)
1022 agentic_reflect: AgenticReflectConfig = field(default_factory=AgenticReflectConfig)
1023 adversarial_defense: AdversarialDefenseConfig = field(default_factory=AdversarialDefenseConfig)
1024 recall_authority: RecallAuthorityConfig = field(default_factory=RecallAuthorityConfig)
1025 curated_retain: CuratedRetainConfig = field(default_factory=CuratedRetainConfig)
1026 curated_recall: CuratedRecallConfig = field(default_factory=CuratedRecallConfig)
1028 # Compliance profile
1029 compliance_profile: str | None = None # "gdpr" | "hipaa" | "pdpa" | None
1031 # DLP
1032 dlp: DlpConfig = field(default_factory=DlpConfig)
1034 # Lifecycle
1035 lifecycle: LifecycleConfig = field(default_factory=LifecycleConfig)
1037 # M8 / M11 intelligence features
1038 wiki_compile: WikiCompileConfig = field(default_factory=WikiCompileConfig)
1039 entity_resolution: EntityResolutionConfig = field(default_factory=EntityResolutionConfig)
1040 async_tasks: AsyncTasksConfig = field(default_factory=AsyncTasksConfig)
1041 benchmark_preset: BenchmarkPresetConfig = field(default_factory=BenchmarkPresetConfig)
1042 # M18a-2 — directive-style preference compile (off by default; flip via
1043 # this field per-bank or env var ASTROCYTE_M18_ENABLE_DIRECTIVE_COMPILE=1).
1044 directive_compile: DirectiveCompileConfig = field(default_factory=DirectiveCompileConfig)
1045 # M18a-3 spreading_activation field already declared above (line ~969,
1046 # alongside cross_encoder_rerank, causal_links, etc.) — duplicate
1047 # declaration removed when the duplicate dataclass was deleted.
1048 # M18a-4 — episodic event index (retain-side tag + recall-side search).
1049 # Both call paths integrated via document_postprocess + fact_recall in
1050 # core. Off by default; flip via this field per-bank or env var
1051 # ASTROCYTE_M18_ENABLE_EPISODIC_EXTRACTION=1.
1052 episodic_extract: EpisodicExtractConfig = field(
1053 default_factory=EpisodicExtractConfig,
1054 )
1055 # M18a-4 follow-up — preference compile gate (was always-on in bench
1056 # before M18). Default True for back-compat; off only when ablating.
1057 preference_compile: PreferenceCompileConfig = field(
1058 default_factory=PreferenceCompileConfig,
1059 )
1061 # MIP (Memory Intent Protocol)
1062 mip_config_path: str | None = None # Path to mip.yaml
1064 # Per-bank overrides
1065 banks: dict[str, BankConfig] | None = None
1067 # Top-level access grants (merged with banks.*.access by access_grants_for_astrocyte)
1068 access_grants: list[AccessGrant] | None = None
1070 # ADR-003 (v0.5.0 with M1)
1071 sources: dict[str, SourceConfig] | None = None
1072 agents: dict[str, AgentRegistrationConfig] | None = None
1073 deployment: DeploymentConfig | None = None
1074 extraction_profiles: dict[str, ExtractionProfileConfig] | None = None
1076 def _bank(self, bank_id: str | None) -> BankConfig | None:
1077 return (self.banks or {}).get(bank_id) if bank_id else None
1079 def bank_homeostasis(self, bank_id: str | None) -> HomeostasisConfig:
1080 """``homeostasis`` in effect for ``bank_id`` (its override, else top-level)."""
1081 bank = self._bank(bank_id)
1082 return bank.homeostasis if bank and bank.homeostasis else self.homeostasis
1084 def bank_barriers(self, bank_id: str | None) -> BarrierConfig:
1085 """``barriers`` in effect for ``bank_id`` (its override, else top-level)."""
1086 bank = self._bank(bank_id)
1087 return bank.barriers if bank and bank.barriers else self.barriers
1089 def bank_signal_quality(self, bank_id: str | None) -> SignalQualityConfig:
1090 """``signal_quality`` in effect for ``bank_id`` (its override, else top-level)."""
1091 bank = self._bank(bank_id)
1092 return bank.signal_quality if bank and bank.signal_quality else self.signal_quality
1095# ---------------------------------------------------------------------------
1096# Loading
1097# ---------------------------------------------------------------------------
1100def _substitute_env_vars(value: str) -> str:
1101 """Replace ${VAR_NAME} with environment variable values."""
1103 def _replace(match: re.Match[str]) -> str:
1104 var_name = match.group(1)
1105 env_value = os.environ.get(var_name)
1106 if env_value is None:
1107 return match.group(0) # Leave unresolved
1108 return env_value
1110 return _ENV_VAR_PATTERN.sub(_replace, value)
1113def _find_unresolved_env_vars(
1114 data: dict | list | str | int | float | bool | None,
1115 path: str = "",
1116) -> list[str]:
1117 """Find all unresolved ${VAR_NAME} references after env var substitution.
1119 Returns list of strings like "vector_store_config.dsn: ${DATABASE_URL}".
1120 """
1121 unresolved: list[str] = []
1122 if isinstance(data, str):
1123 for match in _ENV_VAR_PATTERN.finditer(data):
1124 unresolved.append(f"{path}: ${{{match.group(1)}}}")
1125 elif isinstance(data, dict):
1126 for k, v in data.items():
1127 child_path = f"{path}.{k}" if path else str(k)
1128 unresolved.extend(_find_unresolved_env_vars(v, child_path))
1129 elif isinstance(data, list):
1130 for i, v in enumerate(data):
1131 unresolved.extend(_find_unresolved_env_vars(v, f"{path}[{i}]"))
1132 return unresolved
1135def _substitute_env_recursive(
1136 data: dict | list | str | int | float | bool | None,
1137) -> dict | list | str | int | float | bool | None:
1138 """Recursively substitute env vars in a parsed YAML structure."""
1139 if isinstance(data, str):
1140 return _substitute_env_vars(data)
1141 if isinstance(data, dict):
1142 return {k: _substitute_env_recursive(v) for k, v in data.items()}
1143 if isinstance(data, list):
1144 return [_substitute_env_recursive(item) for item in data]
1145 return data
1148def _load_profile(profile_name: str) -> dict:
1149 """Load a profile YAML from the profiles directory or a file path."""
1150 if profile_name.startswith("./") or profile_name.startswith("/"):
1151 profile_path = Path(profile_name)
1152 else:
1153 profile_path = _PROFILES_DIR / f"{profile_name}.yaml"
1155 if not profile_path.exists():
1156 raise ConfigError(f"Profile not found: {profile_path}")
1158 try:
1159 with open(profile_path, encoding="utf-8") as f:
1160 return yaml.safe_load(f) or {}
1161 except yaml.YAMLError as exc:
1162 raise ConfigError(f"Invalid YAML in {profile_path}: {exc}") from exc
1165_COMPLIANCE_PROFILES_DIR = _PROFILES_DIR / "compliance"
1168def _load_compliance_profile(name: str) -> dict:
1169 """Load a compliance profile YAML (gdpr, hipaa, pdpa)."""
1170 if name.startswith("./") or name.startswith("/"):
1171 profile_path = Path(name)
1172 else:
1173 profile_path = _COMPLIANCE_PROFILES_DIR / f"{name}.yaml"
1175 if not profile_path.exists():
1176 raise ConfigError(f"Compliance profile not found: {profile_path}")
1178 try:
1179 with open(profile_path, encoding="utf-8") as f:
1180 return yaml.safe_load(f) or {}
1181 except yaml.YAMLError as exc:
1182 raise ConfigError(f"Invalid YAML in {profile_path}: {exc}") from exc
1185def _deep_merge(base: dict, override: dict) -> dict:
1186 """Deep merge override into base. Override values win."""
1187 result = base.copy()
1188 for key, value in override.items():
1189 if key in result and isinstance(result[key], dict) and isinstance(value, dict):
1190 result[key] = _deep_merge(result[key], value)
1191 else:
1192 result[key] = value
1193 return result
1196def _filter_dataclass_fields(cls: type, data: dict, *, drop_none: bool = False) -> dict:
1197 """Filter dict to valid dataclass fields; optionally drop ``None`` values."""
1198 valid = {f.name for f in fields(cls)}
1199 return {k: v for k, v in data.items() if k in valid and (not drop_none or v is not None)}
1202_SENSITIVE_FIELD_PATTERNS = frozenset(
1203 _BASE_SENSITIVE_FIELD_KEYS
1204 + (
1205 "dsn",
1206 "connection_string",
1207 "credentials",
1208 "auth",
1209 "jwks_url",
1210 "issuer",
1211 "audience",
1212 )
1213)
1216def _is_sensitive_field(ref: str) -> bool:
1217 """Return True if an unresolved env-var reference is in a security-sensitive field."""
1218 # ref format: "path.to.field: ${VAR}" — extract field name before the colon
1219 field_path = ref.split(":")[0].strip().lower()
1220 return any(pat in field_path for pat in _SENSITIVE_FIELD_PATTERNS)
1223def _safe_sub_dict(data: dict, key: str) -> dict:
1224 """Safely extract a nested dict from *data*, defaulting to ``{}``."""
1225 val = data.get(key)
1226 return val if isinstance(val, dict) else {}
1229def _parse_homeostasis(data: dict) -> HomeostasisConfig:
1230 """Parse a homeostasis config block (used at top-level and per-bank)."""
1231 rl = _safe_sub_dict(data, "rate_limits")
1232 q = _safe_sub_dict(data, "quotas")
1233 return HomeostasisConfig(
1234 recall_max_tokens=data.get("recall_max_tokens"),
1235 reflect_max_tokens=data.get("reflect_max_tokens"),
1236 retain_max_content_bytes=data.get("retain_max_content_bytes"),
1237 rate_limits=RateLimitConfig(**_filter_dataclass_fields(RateLimitConfig, rl, drop_none=True)),
1238 quotas=QuotaConfig(**_filter_dataclass_fields(QuotaConfig, q, drop_none=True)),
1239 )
1242def _parse_benchmark_preset(data: dict) -> BenchmarkPresetConfig:
1243 """Parse versioned benchmark preset metadata with nested budget blocks."""
1244 preset = BenchmarkPresetConfig(**_filter_dataclass_fields(BenchmarkPresetConfig, data))
1245 for budget_name in ("low", "mid", "high"):
1246 budget_data = data.get(budget_name)
1247 if isinstance(budget_data, dict):
1248 setattr(
1249 preset,
1250 budget_name,
1251 BenchmarkBudgetConfig(**_filter_dataclass_fields(BenchmarkBudgetConfig, budget_data)),
1252 )
1253 return preset
1256def _parse_barriers(data: dict) -> BarrierConfig:
1257 """Parse a barriers config block (used at top-level and per-bank)."""
1258 return BarrierConfig(
1259 pii=PiiConfig(**_filter_dataclass_fields(PiiConfig, _safe_sub_dict(data, "pii"))),
1260 validation=ValidationConfig(**_filter_dataclass_fields(ValidationConfig, _safe_sub_dict(data, "validation"))),
1261 metadata=MetadataSanitizationConfig(
1262 **_filter_dataclass_fields(MetadataSanitizationConfig, _safe_sub_dict(data, "metadata"))
1263 ),
1264 )
1267def _parse_signal_quality(data: dict) -> SignalQualityConfig:
1268 """Parse a signal_quality config block (used at top-level and per-bank)."""
1269 return SignalQualityConfig(
1270 dedup=DedupConfig(**_filter_dataclass_fields(DedupConfig, _safe_sub_dict(data, "dedup"))),
1271 noisy_bank=NoisyBankConfig(**_filter_dataclass_fields(NoisyBankConfig, _safe_sub_dict(data, "noisy_bank"))),
1272 )
1275def _parse_escalation(data: dict) -> EscalationConfig:
1276 """Parse an ``escalation:`` config block."""
1277 cb = data.get("circuit_breaker", {})
1278 return EscalationConfig(
1279 circuit_breaker=CircuitBreakerConfig(**_filter_dataclass_fields(CircuitBreakerConfig, cb)),
1280 degraded_mode=data.get("degraded_mode", "empty_recall"),
1281 )
1284def _parse_recall_authority(data: dict) -> RecallAuthorityConfig:
1285 """Parse a ``recall_authority:`` config block."""
1286 tiers_raw = data.get("tiers") or []
1287 tiers: list[RecallAuthorityTierConfig] = []
1288 if isinstance(tiers_raw, list):
1289 for row in tiers_raw:
1290 if isinstance(row, dict):
1291 tiers.append(RecallAuthorityTierConfig(**_filter_dataclass_fields(RecallAuthorityTierConfig, row)))
1292 tb = data.get("tier_by_bank")
1293 tier_by_bank: dict[str, str] = {}
1294 if isinstance(tb, dict):
1295 tier_by_bank = {str(k): str(v) for k, v in tb.items()}
1296 return RecallAuthorityConfig(
1297 enabled=bool(data.get("enabled", False)),
1298 rules_inline=data.get("rules_inline"),
1299 rules_path=data.get("rules_path"),
1300 apply_to_reflect=bool(data.get("apply_to_reflect", True)),
1301 tier_by_bank=tier_by_bank,
1302 tiers=tiers,
1303 )
1306def _parse_access_grants(data: list) -> list[AccessGrant]:
1307 """Parse an ``access_grants:`` list, validating required fields."""
1308 grants: list[AccessGrant] = []
1309 for idx, row in enumerate(data):
1310 if not isinstance(row, dict):
1311 continue
1312 required_keys = ("bank_id", "principal", "permissions")
1313 missing = [k for k in required_keys if k not in row]
1314 if missing:
1315 raise ConfigError(
1316 f"Invalid access_grants entry at index {idx}: missing required field(s): {', '.join(missing)}"
1317 )
1318 if not isinstance(row["permissions"], list):
1319 raise ConfigError(f"Invalid access_grants entry at index {idx}: 'permissions' must be a list.")
1320 grants.append(
1321 AccessGrant(
1322 bank_id=str(row["bank_id"]),
1323 principal=str(row["principal"]),
1324 permissions=[str(p) for p in row["permissions"]],
1325 )
1326 )
1327 return grants
1330def _parse_lifecycle(data: dict) -> LifecycleConfig:
1331 """Parse a ``lifecycle:`` config block."""
1332 ttl_data = data.get("ttl", {})
1333 return LifecycleConfig(
1334 enabled=data.get("enabled", False),
1335 ttl=LifecycleTtlConfig(**_filter_dataclass_fields(LifecycleTtlConfig, ttl_data)),
1336 )
1339#: Sections a bank can override, with their parsers.
1340_BANK_SECTION_PARSERS: dict[str, Callable[[dict], Any]] = {
1341 "homeostasis": _parse_homeostasis,
1342 "barriers": _parse_barriers,
1343 "signal_quality": _parse_signal_quality,
1344}
1347def _parse_banks(data: dict, top_level: dict | None = None) -> dict[str, BankConfig]:
1348 """Parse a ``banks:`` config block with per-bank overrides.
1350 Each overridable section resolves as top-level section (``top_level``,
1351 already merged with the top-level profile) → the bank's ``profile`` →
1352 the bank's own block, later layers winning key by key. A bank therefore
1353 states only what differs: one that sets ``dedup.similarity_threshold``
1354 keeps the top-level ``enabled`` rather than resetting it to the dataclass
1355 default. A section neither the bank nor its profile mentions stays ``None``
1356 (the bank uses the top-level section). Only these three sections are
1357 per-bank; a bank profile's other sections (``defaults``, ``escalation``,
1358 ...) do not apply.
1359 """
1360 top_level = top_level or {}
1361 banks: dict[str, BankConfig] = {}
1362 for bid, bdata in data.items():
1363 if not isinstance(bdata, dict):
1364 continue
1365 bc = BankConfig(
1366 profile=bdata.get("profile"),
1367 access=bdata.get("access"),
1368 )
1369 profile_data = _load_profile(bc.profile) if bc.profile else {}
1370 for section, parse in _BANK_SECTION_PARSERS.items():
1371 layers = [layer.get(section) for layer in (profile_data, bdata)]
1372 layers = [layer for layer in layers if isinstance(layer, dict)]
1373 if not layers:
1374 continue
1375 base = top_level.get(section)
1376 resolved = base if isinstance(base, dict) else {}
1377 for layer in layers:
1378 resolved = _deep_merge(resolved, layer)
1379 setattr(bc, section, parse(resolved))
1380 banks[str(bid)] = bc
1381 return banks
1384def _parse_agents(data: dict) -> dict[str, AgentRegistrationConfig]:
1385 """Parse an ``agents:`` config block."""
1386 agents: dict[str, AgentRegistrationConfig] = {}
1387 for aid, adata in data.items():
1388 if not isinstance(adata, dict):
1389 continue
1390 row = dict(adata)
1391 if row.get("banks") is None and row.get("allowed_banks") is not None:
1392 row["banks"] = list(row["allowed_banks"])
1393 agents[str(aid)] = AgentRegistrationConfig(**_filter_dataclass_fields(AgentRegistrationConfig, row))
1394 return agents
1397def _parse_deployment(data: dict) -> DeploymentConfig:
1398 """Parse a ``deployment:`` config block."""
1399 tls: TlsConfig | None = None
1400 if isinstance(data.get("tls"), dict):
1401 tls = TlsConfig(**_filter_dataclass_fields(TlsConfig, data["tls"]))
1402 dep_no_tls = {k: v for k, v in data.items() if k != "tls"}
1403 return DeploymentConfig(
1404 **_filter_dataclass_fields(DeploymentConfig, dep_no_tls),
1405 tls=tls,
1406 )
1409# Fields copied verbatim from the YAML dict onto AstrocyteConfig.
1410_SCALAR_CONFIG_FIELDS = (
1411 "provider_tier",
1412 "profile",
1413 "provider",
1414 "provider_config",
1415 "vector_store",
1416 "vector_store_config",
1417 "graph_store",
1418 "graph_store_config",
1419 "document_store",
1420 "document_store_config",
1421 "wiki_store",
1422 "wiki_store_config",
1423 "mental_model_store",
1424 "mental_model_store_config",
1425 "source_store",
1426 "source_store_config",
1427 "pageindex_store",
1428 "pageindex_store_config",
1429 "llm_provider",
1430 "llm_provider_config",
1431 "embedding_provider",
1432 "embedding_provider_config",
1433 "fallback_strategy",
1434)
1436# Sections whose value is passed through ``_filter_dataclass_fields`` directly.
1437_SIMPLE_SECTION_MAP: dict[str, type] = {
1438 "observability": ObservabilityConfig,
1439 "access_control": AccessControlConfig,
1440 "identity": IdentityConfig,
1441 "defaults": DefaultsConfig,
1442 "mcp": McpConfig,
1443 "recall_cache": RecallCacheConfig,
1444 "bm25_idf": Bm25IdfConfig,
1445 "source_aware_retrieval": SourceAwareRetrievalConfig,
1446 "tiered_retrieval": TieredRetrievalConfig,
1447 "cross_encoder_rerank": CrossEncoderRerankConfig,
1448 "spreading_activation": SpreadingActivationConfig,
1449 "causal_links": CausalLinksConfig,
1450 "entity_cooccurrence": EntityCooccurrenceConfig,
1451 "structured_fact_extraction": StructuredFactExtractionConfig,
1452 "query_analyzer": QueryAnalyzerConfig,
1453 "semantic_link_graph": SemanticLinkGraphConfig,
1454 "agentic_reflect": AgenticReflectConfig,
1455 "adversarial_defense": AdversarialDefenseConfig,
1456 "curated_retain": CuratedRetainConfig,
1457 "curated_recall": CuratedRecallConfig,
1458 "wiki_compile": WikiCompileConfig,
1459 "entity_resolution": EntityResolutionConfig,
1460 "async_tasks": AsyncTasksConfig,
1461 "dlp": DlpConfig,
1462 "directive_compile": DirectiveCompileConfig,
1463 # spreading_activation already in the map above (line ~1358) — duplicate
1464 # key removed when the M18a-3 duplicate dataclass was consolidated.
1465 "episodic_extract": EpisodicExtractConfig,
1466 "preference_compile": PreferenceCompileConfig,
1467}
1470def _dict_to_config(data: dict) -> AstrocyteConfig:
1471 """Convert a flat/nested dict to AstrocyteConfig with nested dataclasses."""
1472 config = AstrocyteConfig()
1474 # ── Scalar fields ──
1475 for field_name in _SCALAR_CONFIG_FIELDS:
1476 if field_name in data:
1477 setattr(config, field_name, data[field_name])
1479 # ── Simple nested sections (filter + construct) ──
1480 for section, cls in _SIMPLE_SECTION_MAP.items():
1481 if section in data:
1482 setattr(config, section, cls(**_filter_dataclass_fields(cls, data[section])))
1484 # ── Complex nested sections (dedicated parsers) ──
1485 if "homeostasis" in data:
1486 config.homeostasis = _parse_homeostasis(data["homeostasis"])
1488 if "barriers" in data:
1489 config.barriers = _parse_barriers(data["barriers"])
1491 if "escalation" in data:
1492 config.escalation = _parse_escalation(data["escalation"])
1494 if "signal_quality" in data:
1495 config.signal_quality = _parse_signal_quality(data["signal_quality"])
1497 if "benchmark_preset" in data and isinstance(data["benchmark_preset"], dict):
1498 config.benchmark_preset = _parse_benchmark_preset(data["benchmark_preset"])
1500 if "recall_authority" in data and isinstance(data["recall_authority"], dict):
1501 config.recall_authority = _parse_recall_authority(data["recall_authority"])
1503 if "access_grants" in data and data["access_grants"]:
1504 config.access_grants = _parse_access_grants(data["access_grants"])
1506 if "lifecycle" in data:
1507 config.lifecycle = _parse_lifecycle(data["lifecycle"])
1509 if "banks" in data and data["banks"]:
1510 config.banks = _parse_banks(data["banks"], data)
1512 if "extraction_profiles" in data and isinstance(data["extraction_profiles"], dict):
1513 profiles: dict[str, ExtractionProfileConfig] = {}
1514 for pname, pdata in data["extraction_profiles"].items():
1515 if isinstance(pdata, dict):
1516 profiles[str(pname)] = ExtractionProfileConfig(
1517 **_filter_dataclass_fields(ExtractionProfileConfig, pdata)
1518 )
1519 config.extraction_profiles = profiles
1521 if "sources" in data and isinstance(data["sources"], dict):
1522 sources: dict[str, SourceConfig] = {}
1523 for sid, sdata in data["sources"].items():
1524 if isinstance(sdata, dict):
1525 sources[str(sid)] = SourceConfig(**_filter_dataclass_fields(SourceConfig, sdata))
1526 config.sources = sources
1528 if "agents" in data and isinstance(data["agents"], dict):
1529 config.agents = _parse_agents(data["agents"])
1531 if "deployment" in data and isinstance(data["deployment"], dict):
1532 config.deployment = _parse_deployment(data["deployment"])
1534 # ── Scalar fallbacks ──
1535 if "compliance_profile" in data:
1536 config.compliance_profile = data["compliance_profile"]
1538 if "mip_config_path" in data:
1539 config.mip_config_path = data["mip_config_path"]
1540 elif "mip" in data and isinstance(data["mip"], str):
1541 config.mip_config_path = data["mip"]
1543 return config
1546def _agent_bank_list(ar: AgentRegistrationConfig) -> list[str]:
1547 if ar.banks:
1548 return list(ar.banks)
1549 if ar.allowed_banks:
1550 return list(ar.allowed_banks)
1551 return []
1554def _resolve_agent_bank_ids(
1555 patterns: list[str],
1556 declared: set[str] | None,
1557 *,
1558 label: str,
1559) -> list[str]:
1560 """Expand glob patterns against declared bank ids; validate literals when banks: is present."""
1561 if not patterns:
1562 return []
1563 out: list[str] = []
1564 for p in patterns:
1565 has_glob = any(c in p for c in "*?[")
1566 if has_glob:
1567 if not declared:
1568 raise ConfigError(f"{label}: bank pattern {p!r} uses wildcards but no banks: section is declared.")
1569 matches = sorted(bid for bid in declared if fnmatch.fnmatch(bid, p))
1570 if not matches:
1571 raise ConfigError(f"{label}: bank pattern {p!r} matches no declared banks.")
1572 out.extend(matches)
1573 elif declared is not None and p not in declared:
1574 raise ConfigError(f"{label}: bank id {p!r} is not listed under banks:.")
1575 else:
1576 out.append(p)
1577 return out
1580def _validate_signal_quality(label: str, sq: SignalQualityConfig) -> None:
1581 if sq.dedup.action not in DEDUP_ACTIONS:
1582 raise ConfigError(f"{label}.dedup.action={sq.dedup.action!r} must be one of {', '.join(DEDUP_ACTIONS)}")
1583 if sq.noisy_bank.action not in NOISY_BANK_ACTIONS:
1584 raise ConfigError(
1585 f"{label}.noisy_bank.action={sq.noisy_bank.action!r} must be one of {', '.join(NOISY_BANK_ACTIONS)}"
1586 )
1589def validate_astrocyte_config(config: AstrocyteConfig) -> None:
1590 """Cross-field checks for ADR-003 sections (v0.5.0 with M1)."""
1591 _validate_signal_quality("signal_quality", config.signal_quality)
1592 for bank_id, bank in (config.banks or {}).items():
1593 if bank.signal_quality is not None:
1594 _validate_signal_quality(f"banks.{bank_id}.signal_quality", bank.signal_quality)
1596 # M9 / ADR-008: Apache AGE was removed entirely. Existing configs that
1597 # still set ``graph_store: age`` must be migrated. There is no migration
1598 # tool — operators rebuild banks from raw memory_units. See
1599 # docs/_design/adr/adr-008-section-graph-replaces-age.md.
1600 if (config.graph_store or "").strip().lower() == "age":
1601 raise ConfigError(
1602 "graph_store: age was removed in M9. Apache AGE is no longer "
1603 "supported; the recall path now uses flat tables + SQL CTEs "
1604 "(Hindsight pattern). To upgrade: remove the `graph_store: age` "
1605 "line from your config, then rebuild any AGE-backed banks from "
1606 "raw memory_units. See docs/_design/adr/adr-008-section-graph-"
1607 "replaces-age.md for rationale and the rebuild command."
1608 )
1610 # M9: ``extraction_mode: concise`` was removed. The legacy concise path
1611 # paraphrased chunk text, which lost the surface vocabulary question
1612 # embeddings rely on; recall_hit_rate dropped sharply (2026-05-02
1613 # finding). Verbatim is now the only supported mode.
1614 sfe_mode = (config.structured_fact_extraction.extraction_mode or "verbatim").strip().lower()
1615 if sfe_mode != "verbatim":
1616 raise ConfigError(
1617 f"structured_fact_extraction.extraction_mode={sfe_mode!r} is not "
1618 "supported. The 'concise' path was removed in M9 (it caused severe "
1619 "recall_hit_rate degradation — see the 2026-05-02 finding). Drop "
1620 "the extraction_mode line entirely (verbatim is the default), or "
1621 "set extraction_mode: verbatim explicitly."
1622 )
1624 if config.sources:
1625 from astrocyte.pipeline.extraction import merged_extraction_profiles
1627 profiles = merged_extraction_profiles(config)
1628 for name, src in config.sources.items():
1629 if not (src.type or "").strip():
1630 raise ConfigError(f"sources.{name}: type is required")
1631 st = (src.type or "").strip().lower()
1632 if st == "proxy":
1633 if not (src.url or "").strip():
1634 raise ConfigError(f"sources.{name}: type proxy requires url")
1635 if not (src.target_bank or "").strip():
1636 raise ConfigError(f"sources.{name}: type proxy requires target_bank")
1637 if st in ("poll", "api_poll"):
1638 driver = (src.driver or "").strip().lower()
1639 if driver != "github":
1640 raise ConfigError(
1641 f"sources.{name}: poll driver {driver!r} is not supported (use github; "
1642 "install astrocyte-ingestion-github)"
1643 )
1644 if not src.interval_seconds or int(src.interval_seconds) < 60:
1645 raise ConfigError(
1646 f"sources.{name}: type poll requires interval_seconds >= 60 (GitHub API rate limits)"
1647 )
1648 pr = (src.path or "").strip()
1649 if not pr or "/" not in pr or pr.count("/") != 1:
1650 raise ConfigError(
1651 f"sources.{name}: type poll with driver github requires path: owner/repo (got {pr!r})"
1652 )
1653 if not (src.target_bank or "").strip() and not (src.target_bank_template or "").strip():
1654 raise ConfigError(f"sources.{name}: type poll requires target_bank or target_bank_template")
1655 tok = (src.auth or {}).get("token") if src.auth else None
1656 if not (str(tok).strip() if tok is not None else ""):
1657 raise ConfigError(
1658 f"sources.{name}: type poll with driver github requires auth.token "
1659 "(GitHub personal access token or fine-grained token)"
1660 )
1661 if st == "stream":
1662 driver = (src.driver or "redis").strip().lower()
1663 if driver not in ("redis", "kafka"):
1664 raise ConfigError(f"sources.{name}: stream driver {driver!r} is not supported (use redis or kafka)")
1665 if not (src.url or "").strip():
1666 raise ConfigError(
1667 f"sources.{name}: type stream requires url (Redis URL or Kafka bootstrap servers)"
1668 )
1669 if not (src.topic or "").strip():
1670 tlabel = "Redis stream key / Kafka topic"
1671 raise ConfigError(f"sources.{name}: type stream requires topic ({tlabel})")
1672 if not (src.consumer_group or "").strip():
1673 raise ConfigError(f"sources.{name}: type stream requires consumer_group")
1674 if not (src.target_bank or "").strip() and not (src.target_bank_template or "").strip():
1675 raise ConfigError(f"sources.{name}: type stream requires target_bank or target_bank_template")
1676 if src.extraction_profile:
1677 if src.extraction_profile not in profiles:
1678 raise ConfigError(
1679 f"sources.{name}: extraction_profile {src.extraction_profile!r} not found under extraction_profiles"
1680 )
1682 if config.agents:
1683 declared = set(config.banks.keys()) if config.banks else None
1684 for agent_id, ar in config.agents.items():
1685 label = f"agents.{agent_id}"
1686 _resolve_agent_bank_ids(_agent_bank_list(ar), declared, label=label)
1688 ident = config.identity
1689 if ident.resolver is not None and ident.resolver not in ("convention", "config", "custom"):
1690 raise ConfigError(f"identity.resolver must be 'convention', 'config', or 'custom', got {ident.resolver!r}")
1692 if config.recall_authority.enabled and config.recall_authority.tiers:
1693 ra = config.recall_authority
1694 ids: list[str] = []
1695 for t in ra.tiers:
1696 tid = (t.id or "").strip()
1697 if not tid:
1698 raise ConfigError("recall_authority.tiers: each tier must have a non-empty id")
1699 ids.append(tid)
1700 if len(ids) != len(set(ids)):
1701 raise ConfigError("recall_authority.tiers: duplicate id")
1704def _grants_from_agents(config: AstrocyteConfig) -> list[AccessGrant]:
1705 if not config.agents:
1706 return []
1707 declared = set(config.banks.keys()) if config.banks else None
1708 out: list[AccessGrant] = []
1709 for agent_id, ar in config.agents.items():
1710 principal = ar.principal or f"agent:{agent_id}"
1711 perms = ar.permissions or ["read", "write"]
1712 label = f"agents.{agent_id}"
1713 for bid in _resolve_agent_bank_ids(_agent_bank_list(ar), declared, label=label):
1714 out.append(AccessGrant(bank_id=bid, principal=principal, permissions=list(perms)))
1715 return out
1718def _dedupe_grants(grants: list[AccessGrant]) -> list[AccessGrant]:
1719 seen: set[tuple[str, str, tuple[str, ...]]] = set()
1720 out: list[AccessGrant] = []
1721 for g in grants:
1722 key = (g.bank_id, g.principal, tuple(sorted(g.permissions)))
1723 if key in seen:
1724 continue
1725 seen.add(key)
1726 out.append(g)
1727 return out
1730def access_grants_for_astrocyte(config: AstrocyteConfig) -> list[AccessGrant]:
1731 """Flatten ``access_grants``, ``banks.*.access``, and ``agents:``-derived grants into one list for ``Astrocyte.set_access_grants``."""
1732 out: list[AccessGrant] = []
1733 if config.access_grants:
1734 out.extend(config.access_grants)
1735 if config.banks:
1736 for bank_id, bc in config.banks.items():
1737 if not bc.access:
1738 continue
1739 for idx, row in enumerate(bc.access):
1740 if not isinstance(row, dict):
1741 continue
1742 label = f"banks.{bank_id}.access[{idx}]"
1743 if "principal" not in row:
1744 raise ConfigError(f"{label} missing required key: principal")
1745 if "permissions" not in row or not isinstance(row["permissions"], list):
1746 raise ConfigError(f"{label} missing or invalid 'permissions' (must be a list)")
1747 out.append(
1748 AccessGrant(
1749 bank_id=bank_id,
1750 principal=str(row["principal"]),
1751 permissions=[str(p) for p in row["permissions"]],
1752 )
1753 )
1754 out.extend(_grants_from_agents(config))
1755 return _dedupe_grants(out)
1758def load_config(path: str | Path) -> AstrocyteConfig:
1759 """Load Astrocyte configuration from a YAML file.
1761 Resolution order: compliance profile → behavior profile → user config → per-bank overrides.
1762 Environment variables are substituted (${VAR_NAME}).
1763 """
1764 config_path = Path(path)
1765 if not config_path.exists():
1766 raise ConfigError(f"Config file not found: {config_path}")
1768 try:
1769 raw = yaml.safe_load(read_user_text(config_path)) or {}
1770 except yaml.YAMLError as exc:
1771 raise ConfigError(f"Invalid YAML in {config_path}: {exc}") from exc
1773 # Substitute environment variables
1774 raw = _substitute_env_recursive(raw)
1776 # Check for unresolved env vars — fail on sensitive fields, warn on others
1777 unresolved = _find_unresolved_env_vars(raw)
1778 if unresolved:
1779 import logging
1781 _cfg_logger = logging.getLogger("astrocyte.config")
1782 sensitive = [r for r in unresolved if _is_sensitive_field(r)]
1783 if sensitive:
1784 raise ConfigError("Unresolved environment variables in sensitive config fields: " + "; ".join(sensitive))
1785 for ref in unresolved:
1786 _cfg_logger.warning("Unresolved environment variable in config: %s", ref)
1788 # Merge order: compliance (lowest) → behavior profile → user config (highest).
1789 # _deep_merge(base, override) → override wins.
1790 # Build base from lowest priority, then let higher priority layers override.
1791 base: dict = {}
1793 compliance_name = raw.get("compliance_profile")
1794 if compliance_name:
1795 compliance_data = _load_compliance_profile(compliance_name)
1796 base = _deep_merge(base, compliance_data)
1798 profile_name = raw.get("profile")
1799 if profile_name:
1800 profile_data = _load_profile(profile_name)
1801 base = _deep_merge(base, profile_data)
1803 # User config wins over everything
1804 merged = _deep_merge(base, raw)
1806 cfg = _dict_to_config(merged)
1807 if cfg.mip_config_path:
1808 mip = Path(cfg.mip_config_path)
1809 if not mip.is_absolute():
1810 cfg.mip_config_path = str((config_path.parent / mip).resolve())
1811 validate_astrocyte_config(cfg)
1812 return cfg