Merge branch 'dev' into feat/plugin-wasm-e2e

Author: monshri

.github/workflows/ci.yml

@@ -110,7 +110,7 @@ jobs:
       - name: Checkout the cpex Python framework
         uses: actions/checkout@v7
         with:
-          ref: "feat/python_plugin_compat_0.1.x"
+          ref: "0.1.x"
           path: cpex-python
           fetch-depth: 1
       - name: Install Python

Cargo.lock

@@ -147,6 +147,7 @@ cpex-pdp-cedar-direct         = { path = "builtins/pdps/cedar-direct",         v
 cpex-pdp-cel                  = { path = "builtins/pdps/cel",                   version = "0.2.2" }
 cpex-pdp-opa                  = { path = "builtins/pdps/opa",                   version = "0.2.2" }
 cpex-session-valkey           = { path = "builtins/session/valkey",            version = "0.2.2" }
+cpex-hosts-python             = { path = "crates/cpex-hosts-python",           version = "0.2.2", default-features = false }
 
 # Size-first release profile. The FFI artifact (libcpex_ffi.a) is linked
 # statically into host binaries, so its compiled size flows straight into

Cargo.toml

@@ -62,13 +62,22 @@ cedar = ["cpex-builtins/cedar-direct"]
 cel = ["cpex-builtins/cel"]
 valkey = ["cpex-builtins/valkey"]
 
+# Out-of-process host for existing Python CPEX plugins. Not part of
+# `builtins` / `full`: those are the in-process bundled extension set, while
+# this pulls a host that spawns a `worker.py` subprocess per plugin.
+python-host = ["dep:cpex-hosts-python"]
+
 [dependencies]
 # Host runtime — always present, this is the point of the facade.
 cpex-core = { workspace = true }
 apl-core = { workspace = true }
 apl-cmf = { workspace = true }
 apl-cpex = { workspace = true }
 
+# Python plugin host — runs each Python plugin out-of-process, in its own
+# virtualenv, from the Rust PluginManager. Present only with `python-host`.
+cpex-hosts-python = { workspace = true, optional = true, default-features = false }
+
 # Bundled extension set — present only when a builtins feature is enabled.
 # `default-features = false` so the granular plugin features compose; an
 # enabling feature turns on exactly the cpex-builtins features it names.

crates/cpex/Cargo.toml

@@ -26,7 +26,8 @@
 //!
 //! By default it is the **engine only**: no builtin plugins are compiled in.
 //! The bundled extension set lives in [`cpex-builtins`](cpex_builtins) and is
-//! pulled in only when a builtins feature is enabled.
+//! pulled in only when a builtins feature is enabled. The out-of-process
+//! Python plugin host is separate again, behind `python-host`.
 //!
 //! # Usage
 //!
@@ -61,6 +62,27 @@
 //! `audit`, `cedar`, `cel`, `valkey`). When any builtins feature is on, the
 //! registration helpers and the concrete factory types are re-exported here
 //! from [`cpex-builtins`](cpex_builtins).
+//!
+//! `python-host` is orthogonal to all of those, and is in neither `builtins`
+//! nor `full`. It pulls
+//! [`cpex-hosts-python`](cpex_hosts_python), which runs existing Python CPEX
+//! plugins out-of-process — one cached virtualenv and one `worker.py`
+//! subprocess per plugin — and re-exports [`IsolatedVenvFactory`] and
+//! [`ISOLATED_VENV_KIND`] here. Unlike the builtins there is no
+//! `install_*` helper: register the factory yourself, because the `kind` is
+//! one host serving arbitrarily many Python plugins.
+//!
+//! ```ignore
+//! use cpex::{IsolatedVenvFactory, PluginManager, ISOLATED_VENV_KIND};
+//! use cpex::cpex_core::factory::PluginFactoryRegistry;
+//!
+//! let mut factories = PluginFactoryRegistry::new();
+//! factories.register(ISOLATED_VENV_KIND, Box::new(IsolatedVenvFactory));
+//! let mgr = PluginManager::from_config(config, &factories)?;
+//! // `initialize()` builds each venv and launches its worker — a cold pip
+//! // install is measured in minutes, so do it at startup, not on demand.
+//! mgr.initialize().await?;
+//! ```
 
 // Whole-crate re-exports for advanced use (types not surfaced below).
 pub use {apl_cmf, apl_core, apl_cpex, cpex_core};
@@ -78,6 +100,9 @@ pub use cpex_wasm_host;
 // The whole aggregator, for advanced use.
 #[cfg(feature = "cpex-builtins")]
 pub use cpex_builtins;
+// The whole Python host crate, for advanced use (venv and worker internals).
+#[cfg(feature = "python-host")]
+pub use cpex_hosts_python;
 
 // Registration helpers — delegated to cpex-builtins, keeping the facade's
 // historical names (`register_builtin_plugins`, `builtin_pdp_factories`).
@@ -103,6 +128,11 @@ pub use cpex_builtins::{OAuthDelegatorFactory, OAUTH_KIND};
 pub use cpex_builtins::{PiiScannerFactory, PII_KIND};
 #[cfg(feature = "valkey")]
 pub use cpex_builtins::{ValkeyConfig, ValkeySessionStoreFactory, VALKEY_KIND};
+// The Python host's `KIND` is renamed on re-export: bare `KIND` at the facade
+// root says nothing about which plugin kind it is, and the builtins above all
+// use a prefixed const.
+#[cfg(feature = "python-host")]
+pub use cpex_hosts_python::{IsolatedVenvFactory, KIND as ISOLATED_VENV_KIND};
 
 #[cfg(all(test, feature = "cpex-builtins"))]
 mod tests {
@@ -115,3 +145,42 @@ mod tests {
         install_builtins(&mgr);
     }
 }
+#[cfg(all(test, feature = "python-host"))]
+mod python_host_tests {
+    use super::*;
+    use cpex_core::config::parse_config;
+    use cpex_core::factory::PluginFactoryRegistry;
+
+    /// A config with one `isolated_venv` plugin, mirroring the YAML shape an
+    /// operator writes. No `plugin_dirs`: the host always resolves
+    /// `<project root>/plugins` — see `plugin::DEFAULT_PLUGIN_DIR`.
+    fn minimal_config_yaml() -> &'static str {
+        r#"
+plugins:
+  - name: pii-filter
+    kind: isolated_venv
+    hooks: [tool_pre_invoke]
+    config:
+      class_name: my_pkg.filters.PiiFilter
+"#
+    }
+
+    /// The facade's re-exported `IsolatedVenvFactory` and kind const are
+    /// wired up well enough to instantiate a plugin from config. This stops
+    /// at `from_config` deliberately — `initialize()` is what builds the venv
+    /// and spawns `worker.py`, which needs a real interpreter and a real
+    /// package, so it belongs in cpex-hosts-python's integration tests.
+    #[test]
+    fn from_config_instantiates_the_python_host() {
+        let config = parse_config(minimal_config_yaml()).expect("valid YAML");
+
+        let mut factories = PluginFactoryRegistry::new();
+        factories.register(ISOLATED_VENV_KIND, Box::new(IsolatedVenvFactory));
+
+        let mgr = PluginManager::from_config(config, &factories)
+            .expect("isolated_venv factory is registered, so instantiation succeeds");
+
+        assert_eq!(mgr.plugin_count(), 1);
+        assert!(mgr.has_hooks_for("tool_pre_invoke"));
+    }
+}