-
Notifications
You must be signed in to change notification settings - Fork 1
Expand file tree
/
Copy pathwiring.py
More file actions
5047 lines (4571 loc) · 279 KB
/
Copy pathwiring.py
File metadata and controls
5047 lines (4571 loc) · 279 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
# SPDX-License-Identifier: AGPL-3.0-or-later
# Copyright (C) 2026 MessageFoundry Organization and contributors
"""Code-first wiring: declare **Connections** and decorate **Router**/**Handler** functions.
A config module (loaded from a directory via :func:`load_config`) declares named inbound/outbound
**Connections** and registers Router/Handler scripts — wired by name, with no enclosing "channel"
object::
from messagefoundry import inbound, outbound, router, handler, Send, MLLP, File
inbound("IB_Test_ADT", MLLP(port=2575), router="adt_router")
outbound("FILE-OUT_Test_ADT", File(directory="./out/adt"))
@router("adt_router")
def route(msg):
return ["archive"] if msg["MSH-9.1"] == "ADT" else [] # [] -> logged UNROUTED
@handler("archive")
def handle(msg):
if msg["MSH-9.2"] not in ("A01", "A04", "A08"):
return None # None -> logged FILTERED
msg["MSH-3"] = "FOUNDRY"
return Send("FILE-OUT_Test_ADT", msg)
This module only **declares** the graph (the registry); running it (inbound → router → handlers →
outbox → ACK) is the engine's job. Routers/Handlers are pure: they return where a message goes,
they never do network I/O (the outbox worker delivers, preserving at-least-once).
"""
from __future__ import annotations
import hashlib
import importlib.util
import inspect
import ipaddress
import logging
import os
import re
import sys
import threading
from collections.abc import Callable, Iterable, Iterator, Mapping, Sequence
from contextlib import contextmanager, suppress
from dataclasses import dataclass, field
from pathlib import Path
from typing import Any, Literal
from messagefoundry.config.code_sets import (
CODESETS_DIR_NAME,
CodeSet,
CodeSetError,
load_code_sets,
)
from messagefoundry.config.code_sets import (
activated as _code_sets_activated,
)
from messagefoundry.config.code_sets import (
code_set as _resolve_code_set,
)
from messagefoundry.config.models import (
AckAfter,
AckMode,
BatchConfig,
BuildupThreshold,
ConnectorType,
ContentType,
InternalErrorPolicy,
OrderingMode,
Priority,
RetryPolicy,
Schedule,
StallThreshold,
Validation,
_check_cleartext_acceptance,
)
from messagefoundry.config.send_snapshot import snapshot_on_send_active
from messagefoundry.parsing.message import Message, RawMessage, snapshot_payload
__all__ = [
"ConnectionSpec",
"MLLP",
"Tcp",
"X12",
"Http",
"File",
"Timer",
"Loopback",
"PassThrough",
"Rest",
"Direct",
"FHIR",
"DICOM",
"DICOMweb",
"Database",
"DatabasePoll",
"Soap",
"Sftp",
"Ftp",
"Send",
"SetState",
"SetMeta",
"EnvRef",
"env",
"CodeSet",
"code_set",
"Reference",
"FileRef",
"DatabaseRef",
"ReferenceSpec",
"ReferenceSourceSpec",
"resolve_env_settings",
"referenced_env_keys",
"connector_secret_env_values",
"display_settings",
"redacted_settings",
"InboundConnection",
"OutboundConnection",
"Registry",
"WiringError",
"PortConflictError",
"API_LISTENER_LABEL",
"inbound_binding_conflicts",
"resolve_listener_binding",
"bindings_overlap",
"Diagnostic",
"inbound",
"outbound",
"build_inbound_connection",
"build_outbound_connection",
"parse_env_setting",
"router",
"handler",
"HandlerAccepts",
"message_type_of",
"MessageTypeError",
"load_config",
"validate_config",
"accepted_cleartext_hops",
"expiry_relaxed_hops",
"unverified_generic_db_hops",
]
_logger = logging.getLogger(__name__)
class WiringError(ValueError):
"""A connection/router/handler was declared wrong, or references something missing."""
class PortConflictError(WiringError):
"""Two inbound listeners — or a listener and a reserved service binding (the API listener) — want
the same ``(host, port)``.
A subclass of :class:`WiringError`, so every existing handler keeps working: the API still maps it
to 422, ``messagefoundry check`` still reports it, and the runner's ADR 0031 per-connection
isolation still records the offending inbound as failed (the engine comes up DEGRADED rather than
aborting). Callers that care can still catch the conflict specifically."""
@dataclass(frozen=True)
class Diagnostic:
"""One config problem, for tools (e.g. the IDE Problems panel) that want all errors at once."""
message: str
file: str | None = None
severity: str = "error"
@dataclass(frozen=True)
class ConnectionSpec:
"""The transport bits of a Connection (type + settings); the logic lives in Router/Handler."""
type: ConnectorType
settings: dict[str, Any]
# --- environment-specific values (DEV/PROD) ----------------------------------
#: Sentinel for "no default" so ``env("k", default=None)`` (a deliberate None) is distinguishable
#: from "unset" (which makes a missing value a hard load error).
_UNSET: Any = object()
@dataclass(frozen=True)
class EnvRef:
"""A reference to an environment-specific value (e.g. a downstream host that differs DEV vs PROD).
The graph carries the *reference*; the engine resolves it against the running instance's
environment values when it builds the connector. One graph therefore runs in every environment,
and a referenced-but-undefined value fails **loud** at load/promote rather than silently
becoming a blank host (the classic Mirth ``${key}`` footgun). Authored via :func:`env`."""
key: str
default: Any = _UNSET
cast: Callable[[Any], Any] | None = None
def env(key: str, *, default: Any = _UNSET, cast: Callable[[Any], Any] | None = None) -> EnvRef:
"""Reference an environment-specific value, resolved per running instance (DEV/PROD).
Use it inside a connection spec for anything that differs by environment — a downstream peer,
a path, a credential::
outbound("OB_EPIC_ADT", MLLP(host=env("epic_host"), port=env("epic_port", cast=int)))
Values come from the instance's environment: ``environments/<env>.toml`` (non-secrets, versioned)
overlaid with ``MEFOR_VALUE_<KEY>`` env vars (secrets). A referenced key with no value and no
``default`` makes the engine refuse to load/promote that graph — never a silent blank.
The key is matched case-insensitively (lower-cased here, as it is on the value side), so
``env("EPIC_HOST")``, the file key ``epic_host``, and ``MEFOR_VALUE_EPIC_HOST`` all line up."""
return EnvRef(key=key.lower(), default=default, cast=cast)
#: Named casts a ``connections.toml`` env-ref may request (ADR 0007). A data file/GUI can't author an
#: arbitrary Python callable the way :func:`env` can, so the file form is restricted to these — and
#: ``int`` is the only cast used across the migration estate today.
_NAMED_CASTS: dict[str, Callable[[Any], Any]] = {
"int": int,
"float": float,
"bool": bool,
"str": str,
}
#: The only keys an env-ref inline table may carry (the inverse of :func:`display_settings`).
_ENVREF_KEYS = frozenset({"env", "default", "cast"})
def parse_env_setting(value: Any) -> Any:
"""Decode one ``connections.toml`` settings value into a literal or an :class:`EnvRef` (ADR 0007).
An inline table carrying the reserved key ``env`` (and only ``env``/``default``/``cast``) becomes an
:class:`EnvRef` — the inverse of :func:`display_settings`'s ``{"env": key[, "default"]}`` encoding;
``cast`` is a **named** cast (``"int"``/``"float"``/``"bool"``/``"str"``) since a file can't carry a
Python callable. Any other value (a scalar, list, or a plain dict like a REST ``headers`` map) is
returned verbatim. Raises :class:`WiringError` on a malformed env marker or an unknown cast name."""
if not (isinstance(value, dict) and "env" in value and set(value) <= _ENVREF_KEYS):
return value
key = value["env"]
if not isinstance(key, str) or not key:
raise WiringError(f"env reference must name a non-empty string key, got {key!r}")
cast_name = value.get("cast")
if cast_name is not None and cast_name not in _NAMED_CASTS:
raise WiringError(
f"env reference {key!r}: unknown cast {cast_name!r} "
f"(use one of {', '.join(sorted(_NAMED_CASTS))})"
)
cast = _NAMED_CASTS[cast_name] if cast_name is not None else None
default = value["default"] if "default" in value else _UNSET # noqa: SIM401
return EnvRef(key=key.lower(), default=default, cast=cast)
# --- code sets (reference lookup tables) -------------------------------------
def code_set(name: str) -> CodeSet:
"""Reference a managed reference table from ``codesets/<name>.{csv,toml}`` (next to ``--config``).
The code-first alternative to a hand-maintained dict: capture it once at a module's top level
(``DIET = code_set("epic_diets")``) or look it up at call time inside a handler
(``code_set("epic_diets").get(x)``) — both resolve against the active set the loader/runner has
published. Returns a frozen, read-only :class:`CodeSet` (a mapping: ``cs[k]`` / ``cs.get(k, d)`` /
``k in cs`` / ``len(cs)`` / iteration); it is shared across transforms, so it must not be mutated.
A missing or malformed code set fails **loud** as a :class:`WiringError`, surfaced by ``validate`` /
``check`` / reload exactly like a missing ``env()`` value — never a silent empty table. The
reference data is read-only, so the lookup stays pure (re-run-safe); see
:mod:`messagefoundry.config.code_sets` for the one reload-vs-re-run caveat."""
try:
return _resolve_code_set(name)
except CodeSetError as exc:
raise WiringError(str(exc)) from exc
# --- reference sets (external-data enrichment, ADR 0006 Tier 1) ---------------
# A reference set is declared in a wiring module with Reference(name, source=…); the engine's
# ReferenceSyncRunner materializes the source OFF the message path into a versioned, encrypted store
# snapshot, and a Handler reads it PURELY at run time via reference("name").get(key) (the read accessor
# lives in messagefoundry.config.reference). The DECLARATION here is the source + cadence only.
@dataclass(frozen=True)
class ReferenceSourceSpec:
"""Where a reference set's data is materialized from (the analog of :class:`ConnectionSpec`).
``kind`` selects the source connector (``"file"`` today; ``"database"`` is ADR-0006 increment 2);
``settings`` carries its options (may hold :class:`EnvRef` values, resolved per environment)."""
kind: str
settings: dict[str, Any]
def FileRef(
*,
path: str | EnvRef,
encoding: str = "utf-8",
) -> ReferenceSourceSpec:
"""A reference **source** backed by a local CSV/TOML file (ADR 0006 Tier 1).
The file has the same shape as a code set (``code_set`` format: header row, first column the key;
one value column → scalar, several → ``{header: cell}``; or a flat/nested TOML). It is the path for
an externally-produced export (e.g. a nightly job dumps a provider directory to a share): the engine
re-reads it on the set's refresh cadence and materializes it into a versioned, encrypted snapshot,
so an updated export is picked up without a config reload. ``path`` may be an :func:`env` ref."""
return ReferenceSourceSpec("file", {"path": path, "encoding": encoding})
def DatabaseRef(
*,
server: str | EnvRef,
database: str | EnvRef,
statement: str,
key_column: str,
value_column: str | None = None,
auth: str = "sql",
username: str | EnvRef | None = None,
password: str | EnvRef | None = None,
port: int | EnvRef = 1433,
encrypt: bool = True,
trust_server_certificate: bool = False,
connect_timeout: int = 15,
app_name: str = "messagefoundry",
odbc_driver: str = "ODBC Driver 18 for SQL Server",
pool_max: int = 5,
acquire_timeout: float = 30.0, # cap this source's pooled-connection borrow (s) — BACKLOG #1052
) -> ReferenceSourceSpec:
"""A reference **source** backed by a SQL query (ADR 0006 increment 2; SQL Server via the
``[sqlserver]`` extra + ODBC Driver 18 — **production / supported**, like the DATABASE connector).
The engine runs ``statement`` (a read-only ``SELECT``/proc) on the set's refresh cadence and builds
the snapshot from the rows: ``key_column`` is the lookup key; ``value_column`` (if given) is that
column's value, else the value is a dict of the remaining columns (the multi-column ``code_set``
shape). Put secrets (``password``) in :func:`env`. TLS is on by default; weakening it needs
``MEFOR_ALLOW_INSECURE_TLS``. The dial-out is gated by the **fail-closed** ``[egress].allowed_db``
allowlist, exactly like a DATABASE poll source — point the engine only at allowed hosts.
``acquire_timeout`` bounds the borrow from this source's throwaway pool (default 30 s, matching
the DATABASE connector and ``[store].acquire_timeout``). On expiry the set's sync fails, the
last-good snapshot stays active and the AlertSink fires — the runner syncs sets sequentially, so
the bound is what stops one unresponsive server from stalling every other set's refresh."""
return ReferenceSourceSpec(
"database",
{
"server": server,
"database": database,
"statement": statement,
"key_column": key_column,
"value_column": value_column,
"auth": auth,
"username": username,
"password": password,
"port": port,
"encrypt": encrypt,
"trust_server_certificate": trust_server_certificate,
"connect_timeout": connect_timeout,
"app_name": app_name,
"odbc_driver": odbc_driver,
"pool_max": pool_max,
"acquire_timeout": acquire_timeout,
},
)
@dataclass(frozen=True)
class ReferenceSpec:
"""A declared reference set: ``name`` + its :class:`ReferenceSourceSpec` + sync cadence.
Held in :class:`Registry` and consumed by the engine's ``ReferenceSyncRunner``; the data lives in
the store, read via ``reference(name)``. ``refresh_seconds`` is the materialization cadence (the
runner also syncs once on startup); ``max_staleness_seconds`` (0 = off) is a reserved freshness
knob for a follow-up."""
name: str
source: ReferenceSourceSpec
refresh_seconds: float = 3600.0
max_staleness_seconds: float = 0.0
def Reference(
name: str,
*,
source: ReferenceSourceSpec,
refresh_seconds: float = 3600.0,
max_staleness_seconds: float = 0.0,
) -> None:
"""Declare a reference set into the graph being loaded (side-effecting, like :func:`inbound`).
The engine materializes ``source`` into a versioned snapshot every ``refresh_seconds`` (and once at
startup); a Handler reads it purely with ``reference(name).get(key)``. Example::
Reference("provider_npi", source=FileRef(path=env("provider_npi_csv")), refresh_seconds=3600)
"""
if refresh_seconds < 0:
raise WiringError(f"Reference({name!r}): refresh_seconds must be >= 0")
_active_registry().add_reference(
ReferenceSpec(
name=name,
source=source,
refresh_seconds=refresh_seconds,
max_staleness_seconds=max_staleness_seconds,
)
)
# --- live lookup connections (handler-callable db_lookup, ADR 0010) -----------
# A DatabaseLookup declares a NAMED, read-only database connection a Handler queries LIVE at run time via
# db_lookup(name, statement, params) (the read accessor lives in messagefoundry.config.db_lookup). Unlike
# a reference set (a synced snapshot read purely), there is no statement or cadence here — only the
# connection; each call supplies its own statement. The engine builds one pooled executor from these.
@dataclass(frozen=True)
class DatabaseLookupSpec:
"""A declared live-lookup database connection: ``name`` + connection ``settings`` (no statement — the
statement is supplied per :func:`~messagefoundry.config.db_lookup.db_lookup` call). ``settings`` may
hold :class:`EnvRef` values (put secrets like ``password`` in :func:`env`)."""
name: str
settings: dict[str, Any]
def DatabaseLookup(
name: str,
*,
server: str | EnvRef,
database: str | EnvRef,
auth: str = "sql",
username: str | EnvRef | None = None,
password: str | EnvRef | None = None,
port: int | EnvRef = 1433,
encrypt: bool = True,
trust_server_certificate: bool = False,
connect_timeout: int = 15,
app_name: str = "messagefoundry",
odbc_driver: str = "ODBC Driver 18 for SQL Server",
pool_max: int = 5,
acquire_timeout: float = 30.0, # cap a pooled-connection borrow (s) — fail transiently, not forever
) -> None:
"""Declare a named live-lookup database connection (SQL Server via the ``[sqlserver]`` extra + ODBC
Driver 18 — **production / supported**, like the DATABASE connector). A Handler queries it at run time with
``db_lookup(name, statement, params)`` (a read-only ``SELECT``/proc); the rows come back as
``{column: value}`` dicts. Side-effecting, like :func:`Reference`/:func:`inbound`.
Put secrets (``password``) in :func:`env`. TLS is on by default; weakening it needs
``MEFOR_ALLOW_INSECURE_TLS``. The dial-out is gated by the **fail-closed** ``[egress].allowed_db``
allowlist, like a DATABASE source — point the engine only at allowed hosts. Example::
DatabaseLookup("clarity", server=env("clarity_host"), database="Clarity",
username=env("clarity_user"), password=env("clarity_pw"))
"""
_active_registry().add_lookup(
DatabaseLookupSpec(
name,
{
"server": server,
"database": database,
"auth": auth,
"username": username,
"password": password,
"port": port,
"encrypt": encrypt,
"trust_server_certificate": trust_server_certificate,
"connect_timeout": connect_timeout,
"app_name": app_name,
"odbc_driver": odbc_driver,
"pool_max": pool_max,
"acquire_timeout": acquire_timeout,
},
)
)
# A FhirLookup declares a NAMED, read-only FHIR connection a Handler reads LIVE at run time via
# fhir_lookup(name, query) (the read accessor lives in messagefoundry.config.fhir_lookup, ADR 0043). It
# is the FHIR mirror of DatabaseLookup: only the connection (the FHIR service base URL + the SMART auth
# seam the FHIR outbound uses); each call supplies its own read-by-id / search query. Unlike DatabaseLookup
# it returns the spec so with_smart_backend(FhirLookup(...)) can compose SMART auth onto it (the registry
# holds the same object), AND it self-registers — so the flat FhirLookup("epic", ...) form also lands in
# the graph. The engine builds one read executor from these.
@dataclass(frozen=True)
class FhirLookupSpec:
"""A declared live FHIR-lookup connection: ``name`` + connection ``settings`` (no query — the query is
supplied per :func:`~messagefoundry.config.fhir_lookup.fhir_lookup` call). ``settings`` may hold
:class:`EnvRef` values (put secrets like ``bearer_token`` / ``smart_private_key`` in :func:`env`).
Mutable ``settings`` dict so :func:`~messagefoundry.transports.smart.with_smart_backend` can compose
SMART auth onto it (the dataclass stays frozen — only the dict is mutated)."""
name: str
settings: dict[str, Any]
def FhirLookup(
name: str,
*,
url: str | EnvRef, # the FHIR service BASE url, e.g. https://host/fhir (may be env())
fhir_version: str = "R4B", # "R4B" (default) | "R5" | "STU3" — explicit, no autodetect
headers: dict[str, str] | None = None, # static extra headers (no secrets — not env()-resolved)
bearer_token: str
| EnvRef
| None = None, # Authorization: Bearer … (static; or compose with_smart_backend)
basic_user: str
| EnvRef
| None = None, # HTTP Basic (with basic_password); use env() for secrets
basic_password: str | EnvRef | None = None,
timeout_seconds: float = 30.0,
verify_tls: bool = True, # False (dev only) needs MEFOR_ALLOW_INSECURE_TLS
encoding: str = "utf-8",
# ADR 0153 decision 2 — the same per-connection cleartext declaration an outbound carries. It must
# be authorable HERE: the read executor honours the pair, so leaving it to a hand-mutated
# `spec.settings` would be an escape with no load validation and nothing for the loosening registry
# to name — a deviation the registry cannot see is a second posture by the back door.
cleartext_accepted: bool = False,
cleartext_reason: str | None = None,
) -> FhirLookupSpec:
"""Declare a named live-lookup FHIR connection (ADR 0043). A Handler reads it at run time with
``fhir_lookup(name, query, params)`` — a **read-only** read-by-id (``fhir_lookup(name,
"Patient/123")``) or a search whose path is ``query`` and whose fields are the structured ``params``
mapping (``fhir_lookup(name, "Patient", {"identifier": "MRN|123"})``). ``params`` is the **only**
search form — each value is percent-encoded by the engine, so a value cannot inject an extra search
parameter, and a ``?``-query inside ``query`` is refused (BACKLOG #1243). The parsed resource /
searchset ``Bundle`` comes back as a dict.
Side-effecting (it self-registers), like :func:`Reference` / :func:`inbound`, **and** returns the spec
so SMART auth can be composed onto it::
FhirLookup("epic", url=env("epic_fhir_base")) # static / no auth
with_smart_backend( # SMART Backend Services bearer
FhirLookup("epic", url=env("epic_fhir_base")),
token_url=env("epic_token_url"), client_id=env("epic_client_id"),
private_key=env("epic_smart_key"), scope="system/*.rs",
)
The read is **GET-only** (structurally read-only — a Handler cannot mutate the FHIR server through it;
FHIR writes stay on the :func:`FHIR` outbound). The dial-out is gated by the **fail-closed**
``[egress].allowed_http`` allowlist (the same arm the FHIR outbound + SMART token endpoint use) — point
the engine only at allowed hosts. Put secrets (``bearer_token`` / ``basic_*`` / SMART keys) in
:func:`env`. TLS is on by default; weakening it needs ``MEFOR_ALLOW_INSECURE_TLS``. The pure
``parsing/fhir/`` codec parses the reply, so a ``FhirLookup``-declaring graph needs the optional
``messagefoundry[fhir]`` extra.
``cleartext_accepted`` / ``cleartext_reason`` (ADR 0153) declare that this lookup's read hop is
cleartext, is not secure, and the operator accepts that — a mandatory written reason, a loud WARN
plus an audit record at every construction, and an entry in ``security_loosenings()`` /
``GET /security/posture`` naming this connection. Same flag/reason coherence rules as an
``outbound()``: the flag without a reason, a blank reason, or a reason without the flag all fail
loud at load."""
# ADR 0153: coherence-checked at the ONE authoring surface, exactly as build_outbound_connection
# does for an outbound, so the declaration cannot reach the read executor unvalidated.
try:
_check_cleartext_acceptance(cleartext_accepted, cleartext_reason)
except ValueError as exc:
raise WiringError(f"fhir lookup {name!r}: {exc}") from exc
settings: dict[str, Any] = {
"url": url, # stored under "url" (NOT base_url) so the egress gate reads the same key as FHIR()
"fhir_version": fhir_version,
"headers": headers or {},
"bearer_token": bearer_token,
"basic_user": basic_user,
"basic_password": basic_password,
"timeout_seconds": timeout_seconds,
"verify_tls": verify_tls,
"encoding": encoding,
}
if cleartext_accepted:
# Written only when declared, so an undeclared lookup's settings are byte-identical (and the
# redacted settings view, which several surfaces render, gains no empty governance keys).
settings["cleartext_accepted"] = True
settings["cleartext_reason"] = cleartext_reason
settings["cleartext_connection"] = name
spec = FhirLookupSpec(name, settings)
_active_registry().add_fhir_lookup(spec)
return spec
def resolve_env_settings(settings: Mapping[str, Any], values: Mapping[str, Any]) -> dict[str, Any]:
"""Return a copy of ``settings`` with every :class:`EnvRef` resolved against ``values``.
Resolution order per ref: the environment value (cast if a ``cast`` was given), else its
``default``, else it's *missing*. Raises a single :class:`WiringError` listing **all** problems
at once — both missing keys and values that fail their ``cast`` (naming setting/key/value) — so
the failure is loud and actionable, not a raw ``ValueError`` traceback that names nothing and
aborts on the first bad value (fail loud, never blank; review M-22)."""
resolved: dict[str, Any] = {}
missing: list[str] = []
bad: list[str] = []
for name, value in settings.items():
if isinstance(value, EnvRef):
if value.key in values:
raw = values[value.key]
if value.cast is None:
resolved[name] = raw
else:
try:
resolved[name] = value.cast(raw)
except (ValueError, TypeError) as exc:
bad.append(f"setting {name!r} (env {value.key!r}={raw!r}): {exc}")
elif value.default is not _UNSET:
resolved[name] = value.default
else:
missing.append(value.key)
else:
resolved[name] = value
problems: list[str] = []
if missing:
problems.append("missing: " + ", ".join(sorted(set(missing))))
if bad:
problems.append("uncastable: " + "; ".join(bad))
if problems:
raise WiringError(
"environment value(s) unusable — "
+ "; ".join(problems)
+ " — set/fix them in this environment's values (environments/<env>.toml or MEFOR_VALUE_*)"
)
return resolved
def referenced_env_keys(settings: Mapping[str, Any]) -> list[str]:
"""The environment keys a settings dict references (sorted, de-duplicated) — for tooling."""
return sorted({v.key for v in settings.values() if isinstance(v, EnvRef)})
#: Settings keys whose values are credentials — redacted in the API metadata view. Secrets are
#: required to be ``env()`` refs (so they already render as ``{"env": ...}``); this is defence in
#: depth against an inline value, and it suppresses an ``env()`` *default* for a secret field. Covers
#: every credential-bearing connector setting (HTTP auth, DB user/password, SFTP key + passphrase).
_SECRET_SETTING_KEYS = frozenset(
{
"password",
"username",
"bearer_token",
"basic_password",
"basic_user",
"key_password",
"tls_key_password", # MLLP-over-TLS encrypted-key passphrase (WP-13b)
"private_key",
"api_key",
"token",
# ADR 0024 — SMART Backend Services signing-key material (the minted access token / assertion
# are runtime-only and never persisted, so only the key inputs need redacting in /metadata).
"smart_private_key",
"smart_private_key_password",
# BACKLOG #65 — generic outbound HTTP auth secrets (OAuth2 client-credentials symmetric secret;
# HTTP Digest / NTLM password). The minted bearer / digest response are runtime-only.
"oauth2_client_secret",
"http_auth_password",
# BACKLOG #1106 follow-up — the HTTP Digest USERNAME, redacted defence-in-depth alongside
# `basic_user`/`proxy_user`/`ws_username`/`credential_username`/`username` on the ground stated
# there: a username names a principal and can leak directory structure. It was the sixth member
# of a five-member class and the only one unclassified, because `with_http_digest` renames
# parameter `user` into setting `http_auth_user` — the SAME parameter-to-setting boundary
# `with_signing` crosses (`private_key` -> `sign_private_key`), which is the whole of #1106.
# Measured before the fix: served verbatim by /metadata and printed by `graph --json`, with its
# env() FALLBACK DEFAULT intact, beside a `proxy_user` that masked on the same object.
"http_auth_user",
# ADR 0126 (#127) — the forward/egress web-proxy credential. The Basic Proxy-Authorization header /
# Digest response are runtime-only; the password + username inputs are redacted in /metadata (the
# username alongside `basic_user`/`ws_username`, defence-in-depth).
"proxy_password",
"proxy_user",
# ADR 0015 — WS-* SOAP outbound: the WS-Security UsernameToken credentials and the mTLS
# client-key passphrase. ``ws_password`` back-fills from ``basic_password`` in the connector,
# so omitting these disclosed under one name the very credential the other name masks.
"ws_username",
"ws_password",
"client_key_password",
# ADR 0085 — Direct S/MIME-over-SMTP: the signing-key passphrase. (``signing_key`` itself is a
# *path* to the key file, like ``tls_key_file``/``client_key_file``, so it is not listed here.)
"signing_key_password",
# ADR 0132 (#111) — File-endpoint alternate Windows/UNC-share credential. The password is the
# secret (env() only, enforced by the File() factory); the username is redacted defence-in-depth
# alongside ``basic_user``/``ws_username``. ``credential_domain`` is non-secret (an AD domain
# name), so it is intentionally not listed.
"credential_username",
"credential_password",
# ADR 0154 (D6) — the inbound HTTP listener's intake-auth peer credential and its rotation
# partner. Both are env()-only (enforced by the Http() factory) and rotatable, so they are
# deliberately NOT in _NON_ROTATABLE_SECRET_SETTING_KEYS: that enrols them in the ASVS 13.3.4
# fingerprinter and the 13.1.4 registration gate, which is the point. ``intake_api_key_header``
# is a header NAME, not a credential, and is classified non-secret in tests/test_connection_api.
"intake_api_key",
"intake_api_key_next",
}
)
#: Header names whose value is a credential — redacted inside a REST/SOAP ``headers`` table (the
#: project requires secrets via ``env()`` bearer/basic settings, not inline headers; this is defence
#: in depth for an operator who hard-codes one anyway). Compared case-insensitively.
_SECRET_HEADER_NAMES = frozenset(
{"authorization", "proxy-authorization", "x-api-key", "api-key", "cookie"}
)
#: Substrings that make a header name credential-bearing. BACKLOG #1201.
#:
#: The five names above were the WHOLE test, by exact membership. That is the same defect as #1106 and
#: strictly worse, because header names are OPERATOR-AUTHORED FREE TEXT -- there is no factory, no
#: signature and no registry to enumerate, so an exhaustive list cannot exist even in principle.
#: Measured 2026-08-09 against the shipped list: ``X-Auth-Token``, ``X-Amz-Security-Token`` (an AWS
#: SigV4 session credential) and ``Private-Token`` (GitLab's standard auth header) were all returned
#: VERBATIM by ``/metadata`` and printed by ``graph --json``.
#:
#: So the test is by SHAPE, with the explicit set kept as a floor rather than deleted -- ``cookie``
#: matches no substring rule and must stay named.
_SECRET_HEADER_SUBSTRINGS = (
"auth",
"token",
"secret",
"credential",
"password",
"passphrase",
"key",
)
#: Header names that CONTAIN a secret-ish substring and are not credentials. Each is here because
#: redacting it would destroy operator-visible routing or tracing information that is public by nature.
#: Suffix-matched, because the convention is consistent: an ``-id`` names something, it is not the thing.
_NOT_SECRET_HEADER_SUFFIXES = (
"-id",
"-url",
"-uri",
"-name",
"-type",
"-version",
"-agent",
"-for",
)
#: Exact non-credential headers whose name defeats the suffix rule. ``Idempotency-Key`` is the live one:
#: it carries "key" and is a client-generated REQUEST identifier, published in the API docs of every
#: service that uses it.
_NOT_SECRET_HEADERS = frozenset({"idempotency-key", "x-idempotency-key"})
#: Value shapes that are credentials whatever the header is called. The NAME rule below is a heuristic
#: over free text and therefore has a permanent blind spot -- a vendor picks ``X-Shared-Signature`` or an
#: opaque internal name and no substring matches. This is the second arm, and it closes that blind spot
#: from the other side: it does not matter what the header is called if the VALUE is recognisably a
#: credential. Deliberately narrow, because a false positive here masks a value an operator may need:
#: - an RFC 7235 auth scheme prefix (``Bearer``/``Basic``/``Digest``/``Negotiate``/``AWS4-HMAC-...``)
#: - a JWT, which is unmistakable and is what most opaque bearer headers actually carry
_CREDENTIAL_VALUE_PREFIXES = ("bearer ", "basic ", "digest ", "negotiate ", "aws4-hmac")
_JWT_SHAPE = re.compile(r"^eyJ[A-Za-z0-9_-]{6,}\.[A-Za-z0-9_-]{6,}\.[A-Za-z0-9_-]+$")
def _looks_like_a_credential_value(value: object) -> bool:
"""Is this VALUE a credential regardless of what the header is called?"""
if not isinstance(value, str):
return False
v = value.strip()
return v.lower().startswith(_CREDENTIAL_VALUE_PREFIXES) or bool(_JWT_SHAPE.match(v))
#: Credential-bearing ODBC/libpq driver keywords, by SHAPE and case-insensitively (BACKLOG #1206).
#:
#: A THIRD predicate rather than reuse, and the first attempt at this fix proves why. I reached for
#: :func:`_is_secret_setting` -- and it returned False for every one of ``PWD``, ``Password`` and
#: ``sslpassword``, because it matches a fixed frozenset of MessageFoundry SETTINGS names and these are
#: ODBC DRIVER keywords with different spellings and different case. A fix that shipped on that
#: predicate would have masked nothing while reading as a fix, in the change closing a defect whose
#: whole shape is a control whose domain is narrower than its surface.
#:
#: ``pwd`` is listed explicitly because it is an ABBREVIATION and matches no substring rule -- it is
#: also the single most common spelling in a SQL Server DSN.
_SECRET_ODBC_SUBSTRINGS = (
"pwd",
"password",
"passwd",
"secret",
"token",
"credential",
"passphrase",
)
#: ODBC keywords that carry a credential-ish substring and are PATHS, not material. Masking a path
#: hides configuration an operator needs to see and protects nothing: the file's contents never enter
#: settings. ``sslkey`` and ``sslcert`` are libpq file paths.
_NOT_SECRET_ODBC_KEYS = frozenset({"sslkey", "sslcert", "sslrootcert", "sslcrl"})
def _is_secret_odbc_key(name: str) -> bool:
"""Would printing this ODBC keyword's VALUE disclose a credential?"""
low = str(name).strip().lower()
if low in _NOT_SECRET_ODBC_KEYS or low.endswith(("_file", "_path")):
return False
return any(tok in low for tok in _SECRET_ODBC_SUBSTRINGS)
def _is_secret_header(name: str, value: object = None) -> bool:
"""Would printing this header's VALUE disclose a credential?
TWO ARMS, because either alone has a gap. The NAME arm (see :data:`_SECRET_HEADER_SUBSTRINGS`) is a
heuristic over operator-authored free text, so an opaque vendor header name defeats it. The VALUE
arm catches those, and cannot be defeated by naming, but only recognises shapes it knows. Together
they cover a name that looks like a credential OR a value that is one; neither is a proof.
A false positive costs an operator one redacted value in a diagnostic view and one line in
:data:`_NOT_SECRET_HEADERS`; a false negative serves a bearer credential to anyone holding
``MONITORING_READ``. The asymmetry is not close, so this errs toward redacting -- but the VALUE arm
is kept narrow (auth-scheme prefixes and JWTs only) rather than "long and high-entropy", because
masking every long header value would quietly destroy the view rather than protect it.
"""
if _looks_like_a_credential_value(value):
return True
low = str(name).strip().lower()
if low in _SECRET_HEADER_NAMES:
return True
if low in _NOT_SECRET_HEADERS or low.endswith(_NOT_SECRET_HEADER_SUFFIXES):
return False
return any(tok in low for tok in _SECRET_HEADER_SUBSTRINGS)
def _is_secret_setting(name: str) -> bool:
"""Is ``name`` a credential-bearing settings key?
The single source of truth for **both** settings serializers — ``redacted_settings`` (the API
``/metadata`` view) and ``display_settings`` (``graph --json`` → stdout, CI logs, the IDE graph
view). They must never disagree: a key masked on one surface and printed on the other is a
disclosure wearing a false sense of cover, which is exactly how ``ws_password`` was served in
plaintext while ``basic_password`` — the same credential — was masked.
The ``body_secret_value_*`` prefix covers the SOAP body-secret values (ADR 0015 amendment /
BACKLOG #236): the factory already forbids an inline literal and an ``env()`` default on them, so
each renders as a bare ``{"env": key}`` regardless — but the prefix is belt-and-suspenders in case
a value ever reaches a serializer resolved. The paired ``body_secret_tokens`` are **not** secret:
a placeholder is public by nature (it sits in the committed Handler source).
``sign_private_key`` / ``sign_private_key_password`` are named EXPLICITLY (BACKLOG #1106), and the
reason they were missing is the point. ``with_signing`` takes parameters ``private_key`` and
``private_key_password`` — both of which this function already classified — and RENAMES them on the
way into the settings map (``transports/signing.py``). The parameter was covered and the setting it
became was not, so both were served verbatim by ``/metadata`` behind ``MONITORING_READ`` alone and
printed by ``graph --json``. Measured 2026-08-09; the leak predated the cell that scored it, so no
change-detector was ever in play.
NOT a ``sign_`` prefix rule: ``sign_key_id`` is an identifier, ``sign_algorithm`` and ``sign_header``
are configuration, and a prefix would redact all three while reading as more thorough. The domain is
guarded instead by ``tests/test_connection_factory_redaction_domain.py``, which calls every
spec-returning factory and asserts nothing credential-shaped survives this function — at the level
of EMITTED settings rather than parameters, which is the boundary the rename crosses."""
return (
name in _SECRET_SETTING_KEYS
or name.startswith("body_secret_value_")
or name in ("sign_private_key", "sign_private_key_password")
)
#: Connector secret-setting keys that are IDENTIFIERS (usernames), not rotatable credentials — a
#: username names a principal, it is not itself cycled on a cadence (you rotate its paired *password*).
#: They live in :data:`_SECRET_SETTING_KEYS` only so ``/metadata`` redacts them defence-in-depth (a
#: username can leak directory structure). The **single source of truth** for "which secret settings are
#: rotatable": imported by ``tests/test_secret_rotation_inventory.py`` (the ASVS-13.1.4 registration gate)
#: and read by :func:`connector_secret_env_values` (the ASVS-13.3.4 rotation fingerprinter). That the
#: redaction list, the doc registration gate, and the runtime fingerprint set agree about which members
#: are credentials you rotate is **enforced by two gates, not assumed**: the forward gate
#: (``test_secret_setting_keys_are_registered``) proves every rotatable key is registered, and the
#: reverse gate (``test_registered_connector_secrets_are_reachable_by_the_fingerprinter``, BACKLOG #1009)
#: proves every registered connector secret is reachable by the fingerprinter — the direction a
#: hand-added registry entry (the SOAP ``body_secret_value`` class) had slipped through.
_NON_ROTATABLE_SECRET_SETTING_KEYS: frozenset[str] = frozenset(
{
"username",
"basic_user",
"proxy_user",
"ws_username",
"credential_username",
"http_auth_user", # BACKLOG #1106 follow-up — the HTTP Digest principal; you rotate its password
}
)
def connector_secret_env_values(
registry: Registry, env_values: Mapping[str, Any]
) -> dict[str, str]:
"""The per-Connection ``env()``-sourced credential VALUES the wired graph references right now, keyed
by their environment-value key (a NON-SECRET identifier) — the input to the ASVS-13.3.4 rotation
watcher's ``extra_values`` (``pipeline/secret_rotation.reconcile_rotation_meta``), which fingerprints
each with the DEK-derived keyed MAC so a per-Connection connector credential is monitored for rotation
exactly like the fixed ``MEFOR_*`` classes.
A setting is included when its key is a **rotatable** credential — recognised by
:func:`_is_secret_setting` (so the SOAP ``body_secret_value_<i>`` prefix class is covered, not only
the fixed :data:`_SECRET_SETTING_KEYS` names) and not a non-rotatable identifier in
:data:`_NON_ROTATABLE_SECRET_SETTING_KEYS` — AND it is an ``env()``
ref whose key resolves to a **non-empty string** in ``env_values``. Values are returned **transiently**
to be MAC'd — never persisted or logged; the map key is the operator-chosen env name, never the value.
Connections sharing an env key collapse to one entry (one secret → one rotation clock). Inline (non-
``env()``) credentials are skipped: they carry no stable per-environment identity to fingerprint, and
the factories already forbid an inline value for a secret setting."""
out: dict[str, str] = {}
# Both connection kinds carry a ``.spec`` (ConnectionSpec); collect specs so the loop is typed to
# ConnectionSpec rather than the join of the two connection types.
specs: list[ConnectionSpec] = [c.spec for c in registry.inbound.values()]
specs += [c.spec for c in registry.outbound.values()]
for spec in specs:
for name, value in spec.settings.items():
if name in _NON_ROTATABLE_SECRET_SETTING_KEYS or not _is_secret_setting(name):
continue
if isinstance(value, EnvRef):
resolved = env_values.get(value.key)
if isinstance(resolved, str) and resolved:
out[value.key] = resolved
return out
#: Settings whose value is a URL that may carry `user:password@` userinfo. `proxy` has no `_url`
#: suffix, which is why this is a NAME set plus a suffix rule rather than a suffix rule alone.
_URL_SETTING_SUFFIXES = ("url", "_url", "_uri", "endpoint", "_endpoint")
def _mask_url_userinfo(value: object) -> object:
"""Replace the PASSWORD half of a URL's userinfo with ``***``, keeping everything else readable.
BACKLOG #1207. ``url="https://user:SECRET@host/path"`` was returned verbatim by both serializers
while ``proxy_password`` on the SAME object masked -- the credential was safe in the typed field
and disclosed in the URL beside it.
The user half and the host and path are PRESERVED deliberately: an operator diagnosing a
connection needs to see which account and which host, and masking the whole URL would destroy the
view rather than protect it. Only the secret is removed.
"""
if not isinstance(value, str) or "@" not in value or "//" not in value:
return value
scheme, _, rest = value.partition("//")
userinfo, at, hostpart = rest.rpartition("@")
if not at or ":" not in userinfo:
return value # no userinfo, or a user with no password -- nothing secret to remove
user, _, _pw = userinfo.partition(":")
return f"{scheme}//{user}:***@{hostpart}"
def _redact_header_value(name: str, value: object) -> object:
"""One header's value, scrubbed. Handles the ``EnvRef`` case the headers branch used to miss.
BACKLOG #1207. The headers branch had no ``EnvRef`` arm, so an ``env()`` ref inside a headers
table came back as the RAW OBJECT -- not JSON-safe, and carrying its ``default`` intact. The same
``env()`` on a top-level credential correctly emits ``{"env": key}`` with the default dropped, so
the hole was INSIDE the one container this control claims to handle.
THE DEFAULT IS DROPPED FOR EVERY HEADER, not only credential-shaped ones. A header value sourced
from ``env()`` is a credential by intent -- nobody env-refs a ``Content-Type`` -- so the name
heuristic is the wrong gate here, and it is exactly the gate that failed: the measured instance
used ``X-Vendor-Thing``, which matches no substring rule.
"""
if isinstance(value, EnvRef):
return {"env": value.key}
return "***" if _is_secret_header(name, value) else value
def redacted_settings(settings: Mapping[str, Any]) -> dict[str, Any]:
"""A JSON-safe, secret-scrubbed view of a connection's settings for the API ``/metadata`` endpoint:
each EnvRef becomes ``{"env": key}`` (the value is never resolved — only the key is shown), a
credential field rendered inline is replaced with ``"***"`` (an ``env()`` *default* is dropped for
a credential field so a fallback secret can't leak), and a credential header inside a ``headers``
table is redacted too."""
out: dict[str, Any] = {}
for name, value in settings.items():
is_secret = _is_secret_setting(name)
if isinstance(value, EnvRef):
ref: dict[str, Any] = {"env": value.key}
if value.default is not _UNSET and not is_secret:
ref["default"] = value.default
out[name] = ref
elif is_secret:
out[name] = "***"
elif isinstance(value, str) and name.lower().endswith(_URL_SETTING_SUFFIXES):
# BACKLOG #1207 -- a credential in URL userinfo, masked without destroying the view.
out[name] = _mask_url_userinfo(value)
elif name == "headers" and isinstance(value, dict):
out[name] = {k: _redact_header_value(k, v) for k, v in value.items()}
elif name == "odbc_params" and isinstance(value, dict):
# BACKLOG #1206. This bag is documented as carrying "only static driver keywords", and the
# redactor honoured that by not descending -- so a credential inside it was served verbatim
# by /metadata behind MONITORING_READ and printed by graph --json, on the SAME object whose
# top-level `password` masked correctly.
#
# It is not merely operator misuse, which is why this masks rather than warns. The typed
# fields carry exactly ONE credential (`username`/`password`, key names configurable via
# `odbc_user_key`/`odbc_password_key`), and `_reject_envref_odbc_params` refuses `env()`
# here. So a connection needing a SECOND driver credential -- libpq `sslpassword` beside
# `PWD` -- has no typed home and no env() form, and the inline literal is the only
# expressible shape. A refusal that removes the SAFE expression while leaving the UNSAFE
# one is not a mitigation.
#
# Keys only, by the same predicate the rest of this function uses: real static driver
# keywords (`Encrypt`, `TrustServerCertificate`, `ApplicationIntent`) are not
# credential-shaped, and the ones that are -- `PWD`, `Password`, `sslpassword` -- are
# credentials. Values are NOT shape-tested here: a driver keyword's value is opaque and
# masking on its content would hide ordinary configuration with nothing to say so.
#
# THIS IS A DISPLAY FIX, NOT A STORAGE FIX. The credential remains an inline literal in
# the config file. Keeping it out of the file needs `env()` to work here, which needs
# nested settings to be env-resolved -- filed as #1206's route-onward, deliberately not
# folded in, because it changes the resolution path and what the refusal above means.
out[name] = {k: ("***" if _is_secret_odbc_key(k) else v) for k, v in value.items()}
else:
out[name] = value
return out
def display_settings(settings: Mapping[str, Any]) -> dict[str, Any]: