Coverage for astrocyte/eval/_terminal_error.py: 24%

1"""Provider-error classification — terminal vs transient. 

2 

3Used by bench scripts to decide whether to abort a run on an exception 

4vs. retry/fall-through. Extracted from the v0.x ``locomo.py`` bench 

5runner because the same logic is needed by the PageIndex bench scripts 

6(``scripts/bench_pageindex_locomo.py``) and the v0.x runner is being 

7removed. 

8 

9Public API: :func:`is_terminal_error`. Tracks LLM-provider conventions: 

10 

11Transient (return False — bench continues, often after a retry): 

12- HTTP 429 *rate-limit* (per-minute throttle, recovers in seconds) 

13- HTTP 5xx (server-side blip) 

14- timeout / connection reset 

15 

16Terminal (return True — abort the run, every call will fail the same way): 

17- 429 ``insufficient_quota`` (out of credits) 

18- 401 ``invalid_api_key`` (bad credential) 

19- account deactivated / hard billing limit 

20""" 

21 

22from __future__ import annotations 

23 

24_TERMINAL_ERROR_CODES: frozenset[str] = frozenset( 

25 { 

26 "insufficient_quota", 

27 "invalid_api_key", 

28 "account_deactivated", 

29 "billing_hard_limit_reached", 

30 } 

31) 

32 

33# Fallback substring markers for providers that don't expose structured 

34# error codes (older OpenAI SDKs, third-party adapters). Compared against 

35# ``str(exc).lower()``. Kept narrow: the discriminator strings are 

36# specific enough that a question echoing them in a traceback is unlikely. 

37_TERMINAL_ERROR_MARKERS: tuple[str, ...] = ( 

38 "insufficient_quota", 

39 "invalid_api_key", 

40 "incorrect api key", 

41 "account_deactivated", 

42 "account is deactivated", 

43 "billing_hard_limit_reached", 

44 "you exceeded your current quota", 

45) 

46 

47 

48def _structured_error_code(exc: BaseException) -> str | None: 

49 """Pull a stable error code out of common SDK exception shapes. 

50 

51 OpenAI SDK ≥ 1.x exposes both ``exc.code`` (string) and 

52 ``exc.body == {"error": {"code": ...}}``. Anthropic and other SDKs 

53 follow similar shapes. Returns the lowercased code string when found, 

54 otherwise None — caller falls through to the substring fallback. 

55 """ 

56 code = getattr(exc, "code", None) 

57 if isinstance(code, str) and code: 

58 return code.lower() 

59 body = getattr(exc, "body", None) 

60 if isinstance(body, dict): 

61 err = body.get("error") 

62 if isinstance(err, dict): 

63 body_code = err.get("code") 

64 if isinstance(body_code, str) and body_code: 

65 return body_code.lower() 

66 return None 

67 

68 

69def is_terminal_error(exc: BaseException) -> bool: 

70 """Return True if ``exc`` is unrecoverable (vs. a transient hiccup 

71 that retry / fallback can handle). 

72 

73 Resolution order: 

74 1. Structured ``.code`` / ``.body['error']['code']`` (preferred — 

75 stable identifiers, no false positives on prose). 

76 2. Substring match on ``str(exc).lower()`` for older SDKs / adapters. 

77 """ 

78 code = _structured_error_code(exc) 

79 if code is not None: 

80 return code in _TERMINAL_ERROR_CODES 

81 msg = str(exc).lower() 

82 return any(marker in msg for marker in _TERMINAL_ERROR_MARKERS)