Skip to content

Commit 386710a

Browse files
committed
Merge branch 'dev' into feat/audit-seam
2 parents 82b92c3 + 035012f commit 386710a

48 files changed

Lines changed: 3897 additions & 60 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

.github/workflows/ci.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -108,7 +108,7 @@ jobs:
108108
- name: Checkout the cpex Python framework
109109
uses: actions/checkout@v7
110110
with:
111-
ref: "feat/python_plugin_compat_0.1.x"
111+
ref: "0.1.x"
112112
path: cpex-python
113113
fetch-depth: 1
114114
- name: Install Python

Cargo.lock

Lines changed: 1 addition & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

Cargo.toml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -142,6 +142,7 @@ cpex-pdp-cedar-direct = { path = "builtins/pdps/cedar-direct", v
142142
cpex-pdp-cel = { path = "builtins/pdps/cel", version = "0.2.2" }
143143
cpex-pdp-opa = { path = "builtins/pdps/opa", version = "0.2.2" }
144144
cpex-session-valkey = { path = "builtins/session/valkey", version = "0.2.2" }
145+
cpex-hosts-python = { path = "crates/cpex-hosts-python", version = "0.2.2", default-features = false }
145146

146147
# Size-first release profile. The FFI artifact (libcpex_ffi.a) is linked
147148
# statically into host binaries, so its compiled size flows straight into

Makefile

Lines changed: 25 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -331,10 +331,16 @@ examples-run: examples-build tutorial-check-local
331331
# modules that need no infrastructure; `tutorial-check` additionally brings
332332
# up the tutorial Keycloak (docker compose) and runs the IdP-backed modules,
333333
# tearing the stack down afterward. CI runs `tutorial-check`.
334+
#
335+
# Module 16 needs the SPIRE overlay and a one-time Keycloak setup on top of
336+
# the base stack, so it has its own opt-in target (`tutorial-check-spire`).
334337

335338
TUTORIAL_IDP_COMPOSE = examples/tutorial/idp/docker-compose.yml
339+
TUTORIAL_SPIRE_COMPOSE = examples/tutorial/idp/docker-compose.spire.yml
336340
TUTORIAL_NOIDP_MODULES = m01_hello m03_shaping m04_effects m09_custom_plugin m10_testing
337-
TUTORIAL_IDP_MODULES = m02_identity m05_pdp m06_delegation m07_tainting m08_elicitation capstone
341+
TUTORIAL_IDP_MODULES = m02_identity m05_pdp m06_delegation m07_tainting m08_elicitation \
342+
m11_groups m12_subjects m13_client m14_passthrough m15_dual_principal \
343+
m17_federation m18_attributes capstone
338344

339345
.PHONY: tutorial-check-local
340346
tutorial-check-local:
@@ -359,6 +365,24 @@ tutorial-check: tutorial-check-local
359365
@docker compose -f $(TUTORIAL_IDP_COMPOSE) down
360366
@echo "✅ Tutorial checks passed (incl. IdP-backed modules)"
361367

368+
# Module 16 only. Brings up the SPIRE overlay, trusts SPIRE in Keycloak, and
369+
# runs the workload-identity module. Not part of the CI gate: it needs two
370+
# extra containers and a Keycloak that speaks SPIFFE.
371+
.PHONY: tutorial-check-spire
372+
tutorial-check-spire:
373+
@echo "→ starting tutorial IdP + SPIRE"
374+
@docker compose -f $(TUTORIAL_IDP_COMPOSE) -f $(TUTORIAL_SPIRE_COMPOSE) up -d
375+
@echo "→ waiting for Keycloak realm to be ready"
376+
@$(CARGO) run -q -p cpex-tutorial --example wait_for_idp || { \
377+
docker compose -f $(TUTORIAL_IDP_COMPOSE) -f $(TUTORIAL_SPIRE_COMPOSE) down; exit 1; }
378+
@examples/tutorial/idp/spire/setup-spiffe.sh || { \
379+
docker compose -f $(TUTORIAL_IDP_COMPOSE) -f $(TUTORIAL_SPIRE_COMPOSE) down; exit 1; }
380+
@echo "→ tutorial m16_workload --check"
381+
@$(CARGO) run -q -p cpex-tutorial --example m16_workload -- --check || { \
382+
docker compose -f $(TUTORIAL_IDP_COMPOSE) -f $(TUTORIAL_SPIRE_COMPOSE) down; exit 1; }
383+
@docker compose -f $(TUTORIAL_IDP_COMPOSE) -f $(TUTORIAL_SPIRE_COMPOSE) down
384+
@echo "✅ Tutorial SPIRE check passed (module 16)"
385+
362386
.PHONY: tutorial-recordings
363387
tutorial-recordings:
364388
@examples/tutorial/recordings/record.sh

crates/cpex/Cargo.toml

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -61,13 +61,22 @@ cedar = ["cpex-builtins/cedar-direct"]
6161
cel = ["cpex-builtins/cel"]
6262
valkey = ["cpex-builtins/valkey"]
6363

64+
# Out-of-process host for existing Python CPEX plugins. Not part of
65+
# `builtins` / `full`: those are the in-process bundled extension set, while
66+
# this pulls a host that spawns a `worker.py` subprocess per plugin.
67+
python-host = ["dep:cpex-hosts-python"]
68+
6469
[dependencies]
6570
# Host runtime — always present, this is the point of the facade.
6671
cpex-core = { workspace = true }
6772
apl-core = { workspace = true }
6873
apl-cmf = { workspace = true }
6974
apl-cpex = { workspace = true }
7075

76+
# Python plugin host — runs each Python plugin out-of-process, in its own
77+
# virtualenv, from the Rust PluginManager. Present only with `python-host`.
78+
cpex-hosts-python = { workspace = true, optional = true, default-features = false }
79+
7180
# Bundled extension set — present only when a builtins feature is enabled.
7281
# `default-features = false` so the granular plugin features compose; an
7382
# enabling feature turns on exactly the cpex-builtins features it names.

crates/cpex/src/lib.rs

Lines changed: 70 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,8 @@
2626
//!
2727
//! By default it is the **engine only**: no builtin plugins are compiled in.
2828
//! The bundled extension set lives in [`cpex-builtins`](cpex_builtins) and is
29-
//! pulled in only when a builtins feature is enabled.
29+
//! pulled in only when a builtins feature is enabled. The out-of-process
30+
//! Python plugin host is separate again, behind `python-host`.
3031
//!
3132
//! # Usage
3233
//!
@@ -61,6 +62,27 @@
6162
//! `audit`, `cedar`, `cel`, `valkey`). When any builtins feature is on, the
6263
//! registration helpers and the concrete factory types are re-exported here
6364
//! from [`cpex-builtins`](cpex_builtins).
65+
//!
66+
//! `python-host` is orthogonal to all of those, and is in neither `builtins`
67+
//! nor `full`. It pulls
68+
//! [`cpex-hosts-python`](cpex_hosts_python), which runs existing Python CPEX
69+
//! plugins out-of-process — one cached virtualenv and one `worker.py`
70+
//! subprocess per plugin — and re-exports [`IsolatedVenvFactory`] and
71+
//! [`ISOLATED_VENV_KIND`] here. Unlike the builtins there is no
72+
//! `install_*` helper: register the factory yourself, because the `kind` is
73+
//! one host serving arbitrarily many Python plugins.
74+
//!
75+
//! ```ignore
76+
//! use cpex::{IsolatedVenvFactory, PluginManager, ISOLATED_VENV_KIND};
77+
//! use cpex::cpex_core::factory::PluginFactoryRegistry;
78+
//!
79+
//! let mut factories = PluginFactoryRegistry::new();
80+
//! factories.register(ISOLATED_VENV_KIND, Box::new(IsolatedVenvFactory));
81+
//! let mgr = PluginManager::from_config(config, &factories)?;
82+
//! // `initialize()` builds each venv and launches its worker — a cold pip
83+
//! // install is measured in minutes, so do it at startup, not on demand.
84+
//! mgr.initialize().await?;
85+
//! ```
6486
6587
// Whole-crate re-exports for advanced use (types not surfaced below).
6688
pub use {apl_cmf, apl_core, apl_cpex, cpex_core};
@@ -74,6 +96,9 @@ pub use cpex_core::manager::PluginManager;
7496
// The whole aggregator, for advanced use.
7597
#[cfg(feature = "cpex-builtins")]
7698
pub use cpex_builtins;
99+
// The whole Python host crate, for advanced use (venv and worker internals).
100+
#[cfg(feature = "python-host")]
101+
pub use cpex_hosts_python;
77102

78103
// Registration helpers — delegated to cpex-builtins, keeping the facade's
79104
// historical names (`register_builtin_plugins`, `builtin_pdp_factories`).
@@ -99,6 +124,11 @@ pub use cpex_builtins::{OAuthDelegatorFactory, OAUTH_KIND};
99124
pub use cpex_builtins::{PiiScannerFactory, PII_KIND};
100125
#[cfg(feature = "valkey")]
101126
pub use cpex_builtins::{ValkeyConfig, ValkeySessionStoreFactory, VALKEY_KIND};
127+
// The Python host's `KIND` is renamed on re-export: bare `KIND` at the facade
128+
// root says nothing about which plugin kind it is, and the builtins above all
129+
// use a prefixed const.
130+
#[cfg(feature = "python-host")]
131+
pub use cpex_hosts_python::{IsolatedVenvFactory, KIND as ISOLATED_VENV_KIND};
102132

103133
#[cfg(all(test, feature = "cpex-builtins"))]
104134
mod tests {
@@ -111,3 +141,42 @@ mod tests {
111141
install_builtins(&mgr);
112142
}
113143
}
144+
#[cfg(all(test, feature = "python-host"))]
145+
mod python_host_tests {
146+
use super::*;
147+
use cpex_core::config::parse_config;
148+
use cpex_core::factory::PluginFactoryRegistry;
149+
150+
/// A config with one `isolated_venv` plugin, mirroring the YAML shape an
151+
/// operator writes. No `plugin_dirs`: the host always resolves
152+
/// `<project root>/plugins` — see `plugin::DEFAULT_PLUGIN_DIR`.
153+
fn minimal_config_yaml() -> &'static str {
154+
r#"
155+
plugins:
156+
- name: pii-filter
157+
kind: isolated_venv
158+
hooks: [tool_pre_invoke]
159+
config:
160+
class_name: my_pkg.filters.PiiFilter
161+
"#
162+
}
163+
164+
/// The facade's re-exported `IsolatedVenvFactory` and kind const are
165+
/// wired up well enough to instantiate a plugin from config. This stops
166+
/// at `from_config` deliberately — `initialize()` is what builds the venv
167+
/// and spawns `worker.py`, which needs a real interpreter and a real
168+
/// package, so it belongs in cpex-hosts-python's integration tests.
169+
#[test]
170+
fn from_config_instantiates_the_python_host() {
171+
let config = parse_config(minimal_config_yaml()).expect("valid YAML");
172+
173+
let mut factories = PluginFactoryRegistry::new();
174+
factories.register(ISOLATED_VENV_KIND, Box::new(IsolatedVenvFactory));
175+
176+
let mgr = PluginManager::from_config(config, &factories)
177+
.expect("isolated_venv factory is registered, so instantiation succeeds");
178+
179+
assert_eq!(mgr.plugin_count(), 1);
180+
assert!(mgr.has_hooks_for("tool_pre_invoke"));
181+
}
182+
}

docs/content/docs/tutorial/06-delegation.md

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,8 @@ weight: 7
66
# Module 6: Scoped credentials (Delegation)
77

88
> You are in the [CPEX tutorial]({{< relref "_index" >}}). This module needs the IdP.
9+
>
10+
> **Cookbook recipe:** [Recipe 1: User acting through an agent (on-behalf-of)]({{< relref "/docs/identity-delegation#recipe-1-user-acting-through-an-agent-on-behalf-of" >}}).
911
1012
**Goal:** mint a narrow, downstream-scoped credential for a call with a real OAuth 2.0 token exchange (RFC 8693), instead of forwarding the caller's full token.
1113

@@ -86,6 +88,7 @@ Keycloak mints the scoped token during the exchange, constrained by the requeste
8688
## Go deeper
8789
8890
- [Delegation]({{< relref "/docs/apl/delegation" >}}) for token exchange, capability reduction, and downstream verification.
91+
- [Module 12: Delegation subjects]({{< relref "12-subjects" >}}) for the other principals a minted token can speak for.
8992
9093
## Next
9194

docs/content/docs/tutorial/10-testing.md

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -82,4 +82,4 @@ The `Outcome` from `mediate()`: allowed or denied, and for denials the reason co
8282

8383
## Next
8484

85-
[Capstone: the three-backend agent]({{< relref "capstone" >}}): assemble every control you have built into the full Overview scenario.
85+
[Module 11: Organizing policy]({{< relref "11-groups" >}}): factor the setup your routes share into reusable groups.
Lines changed: 105 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,105 @@
1+
---
2+
title: "Organizing policy (Groups)"
3+
weight: 12
4+
---
5+
6+
# Module 11: Organizing policy (Groups)
7+
8+
> You are in the [CPEX tutorial]({{< relref "_index" >}}). This module needs the IdP.
9+
10+
**Goal:** factor the setup every route shares, here the identity resolver, into one reusable group that routes join instead of repeating.
11+
12+
## The problem
13+
14+
By now you have written `authentication: [keycloak]` on route after route. The resolver is the same everywhere; only the per-route authorization differs. That repetition is a maintenance hazard: add a second issuer or a claim mapper later, and you have to change every route and hope you caught them all.
15+
16+
A group is a named, reusable bundle of policy (authentication steps, authorization steps, plugins) that routes opt into. Put the shared part in a group once; each route joins it and adds only what is specific to it.
17+
18+
## Build it
19+
20+
Define a top-level `groups:` section and join it from each route with `groups:`. From [`policies/m11.yaml`](https://github.com/contextforge-org/cpex/tree/main/examples/tutorial/policies/m11.yaml):
21+
22+
```yaml
23+
plugins:
24+
- name: keycloak
25+
kind: identity/jwt
26+
hooks: [identity.resolve]
27+
config: { ... as in module 2 ... }
28+
29+
# One reusable bundle: the resolver every route needs, written once.
30+
groups:
31+
identified:
32+
authentication:
33+
- keycloak
34+
35+
routes:
36+
- tool: get_compensation
37+
groups: identified # inherits keycloak; no authentication: line
38+
authorization:
39+
pre_invocation:
40+
- "require(role.hr)"
41+
- tool: search_repos
42+
groups: identified
43+
authorization:
44+
pre_invocation:
45+
- "require(role.engineer)"
46+
- tool: send_email
47+
groups: identified # still resolves the token; no role required
48+
authorization:
49+
pre_invocation:
50+
- "require(authenticated)"
51+
```
52+
53+
The `identified` group carries the one thing all three routes share. Each route joins it and adds only its own authorization, so identity is resolved the same way everywhere while policy still decides each outcome per caller.
54+
55+
- **`groups:` is sugar over tags.** `groups: identified` is exactly `meta: { tags: [identified] }`. The field just makes that membership a first-class, discoverable spelling, and a runtime tag a host injects joins the group the same way.
56+
- **An unknown group is a load error.** Join a group that isn't defined, a typo like `groups: identifed`, and the config is rejected at load, so a mistake can't silently leave a route unauthenticated.
57+
58+
## Run it
59+
60+
```bash
61+
cargo run -p cpex-tutorial --example m11_groups
62+
```
63+
64+
```
65+
▸ alice (hr) → get_compensation (group resolves her token, require(role.hr) passes)
66+
✓ ALLOWED { ... }
67+
68+
▸ evan (engineer) → get_compensation (resolved by the same group, denied at require(role.hr))
69+
✗ DENIED [...] access denied
70+
71+
▸ evan (engineer) → search_repos (same group, require(role.engineer) passes)
72+
✓ ALLOWED { ... }
73+
74+
▸ alice (hr) → search_repos (denied at require(role.engineer))
75+
✗ DENIED [...] access denied
76+
77+
▸ alice (hr) → send_email (group resolves her token; require(authenticated) passes)
78+
✓ ALLOWED { ... }
79+
```
80+
81+
Every route resolved the caller's token through the same group, yet each outcome is decided by that route's own authorization. The resolver appears once, not three times.
82+
83+
## Try it
84+
85+
1. **Break a join.** Change one route's `groups:` to a name that doesn't exist (`groups: identifed`) and re-run. Expect: the config is rejected at load with an unknown-group error, so the typo fails loudly instead of silently dropping authentication.
86+
2. **Change the resolver once.** Add `leeway_seconds: 5` (or a second entry under `trusted_issuers:`) to the `keycloak` plugin. Every route that joins `identified` picks it up: you edited one place, not three.
87+
3. **Tags are the same thing.** Replace `groups: identified` on a route with `meta: { tags: [identified] }` and re-run. Same result, because a group and a matching tag are the same membership.
88+
89+
## Checkpoint
90+
91+
{{< details "Does the route repeat the group's authentication?" >}}
92+
No. The route has no `authentication:` block, so it inherits the group's. Identity resolution stacks broad → narrow (global → group → route); a route joining `identified` runs the group's `keycloak` resolver without naming it again.
93+
{{< /details >}}
94+
95+
{{< details "Group or the reserved `all` group?" >}}
96+
Groups are opt-in: a route joins one by name. The reserved `all` group is the other case, applied to every request unconditionally. Reach for a named group when a *subset* of routes shares setup, as here.
97+
{{< /details >}}
98+
99+
## Go deeper
100+
101+
- [Configuration]({{< relref "/docs/configuration" >}}#groups) for the full `groups:` schema, defaults, and how membership resolves.
102+
103+
## Next
104+
105+
[Module 12: Delegation subjects]({{< relref "12-subjects" >}}): choose whether a minted token speaks for the caller or the gateway itself.

0 commit comments

Comments
 (0)