From b9964ed66b202a25501fec7ca64c35ce1bd3fd30 Mon Sep 17 00:00:00 2001 From: solaawojobi00-bit Date: Sat, 26 Sep 2026 14:19:43 +0100 Subject: [PATCH] feat(deploy): run post-deploy smoke tests from project manifest (#753) Projects can declare [[smoke_tests]] in starforge-project.toml. After `deploy --execute` is confirmed (contract ID returned), each test runs against the new contract via `stellar contract invoke` or a shell command, with exit/output assertions and a per-test timeout. - Declarations are validated at manifest load, before deploying - Smoke failures exit with new code 9 (SMOKE_TEST_FAILURE), classified by error type and distinct from deploy failures - --skip-smoke opts out; dry runs and failed deploys never run them - Sample project in examples/smoke-tests, schema in docs/SMOKE_TESTS.md --- docs/COMMAND_REFERENCE.md | 7 +- docs/CONFIGURATION.md | 4 + docs/SMOKE_TESTS.md | 107 +++ docs/SUMMARY.md | 1 + examples/smoke-tests/README.md | 46 ++ .../failing/starforge-project.toml | 16 + examples/smoke-tests/starforge-project.toml | 27 + src/commands/deploy.rs | 91 ++- src/main.rs | 12 +- src/utils/exit_codes.rs | 14 + src/utils/mod.rs | 1 + src/utils/project_config.rs | 72 +- src/utils/smoke_tests.rs | 673 ++++++++++++++++++ tests/cli_exit_codes.rs | 25 + 14 files changed, 1092 insertions(+), 4 deletions(-) create mode 100644 docs/SMOKE_TESTS.md create mode 100644 examples/smoke-tests/README.md create mode 100644 examples/smoke-tests/failing/starforge-project.toml create mode 100644 examples/smoke-tests/starforge-project.toml create mode 100644 src/utils/smoke_tests.rs diff --git a/docs/COMMAND_REFERENCE.md b/docs/COMMAND_REFERENCE.md index e98b088e..5a79ef57 100644 --- a/docs/COMMAND_REFERENCE.md +++ b/docs/COMMAND_REFERENCE.md @@ -101,7 +101,12 @@ starforge multisig notify proposal.json --message "Please sign the treasury paym | `inspect storage` | Deep storage inspection | | `deploy --wasm ` | Prepare Soroban deployment | -**`deploy` flags:** `--network`, `--wallet`, `--optimize`, `--simulate`, `--yes`, `--execute`, `--policy`, `--checklist` +**`deploy` flags:** `--network`, `--wallet`, `--optimize`, `--simulate`, `--yes`, `--execute`, `--policy`, `--checklist`, `--skip-smoke` + +After a successful `--execute`, `deploy` runs the `[[smoke_tests]]` declared in +`starforge-project.toml` against the new contract. A smoke failure exits with +code **9** (`SMOKE_TEST_FAILURE`), which is distinct from a failed deploy. +`--skip-smoke` opts out. See [SMOKE_TESTS.md](SMOKE_TESTS.md). Deploy policy files (`starforge-deploy-policy.toml`) gate allowed networks, required reviewers, and checklist items. Validate in CI with diff --git a/docs/CONFIGURATION.md b/docs/CONFIGURATION.md index f4640916..624eea02 100644 --- a/docs/CONFIGURATION.md +++ b/docs/CONFIGURATION.md @@ -93,6 +93,10 @@ Security model: - `starforge config show` tells you which lockfile (if any) is participating, so an override is never mistaken for a personal setting. +The lockfile can also declare `[[smoke_tests]]`, which `starforge deploy +--execute` runs after a successful deploy. These entries are not config +overrides. See [SMOKE_TESTS.md](SMOKE_TESTS.md) for the schema. + Commands that only read configuration use the effective config. Commands that write configuration still operate on the user config only — project overrides are an input, never something persisted back. diff --git a/docs/SMOKE_TESTS.md b/docs/SMOKE_TESTS.md new file mode 100644 index 00000000..808032bc --- /dev/null +++ b/docs/SMOKE_TESTS.md @@ -0,0 +1,107 @@ +# Post-deploy smoke tests + +A project can declare smoke tests in its `starforge-project.toml`. When +`starforge deploy --execute` succeeds, StarForge runs them against the freshly +deployed contract. A broken initialisation or configuration then shows up in +the deploy itself instead of in the first user report. + +A complete sample project lives in +[`examples/smoke-tests/`](../examples/smoke-tests/). + +## When smoke tests run + +| Situation | Smoke tests | +|-----------|-------------| +| `deploy --execute` succeeds and the Stellar CLI returns a contract ID | **Run**, in declaration order | +| `deploy --execute` fails | Not run (the deploy error is reported as before) | +| `deploy --execute` succeeds but no contract ID appears in the CLI output | Not run; a warning is printed | +| `deploy` without `--execute`, `--dry-run`, or `--json` | Not run | +| `--skip-smoke` is passed | Not run, and the manifest's `smoke_tests` are not loaded | +| No `smoke_tests` in the manifest, or no manifest | Nothing changes: deploy behaves exactly as before | + +Every test runs, even after one fails, so a single report shows everything +that is broken. + +## Manifest schema + +Add one `[[smoke_tests]]` table per test: + +```toml +[[smoke_tests]] +name = "hello-returns-greeting" # required, unique +invoke = { function = "hello", args = ["--to", "world"] } +expect_contains = "world" +timeout_secs = 60 + +[[smoke_tests]] +name = "health-script" +command = "./scripts/health.sh" +expect_exit = 0 +``` + +| Key | Type | Required | Meaning | +|-----|------|----------|---------| +| `name` | string | yes | Unique, non-empty name shown in the report | +| `invoke` | table | one of `invoke` / `command` | Contract call on the deployed contract, run as `stellar contract invoke --id --source --network -- ` | +| `invoke.function` | string | yes, with `invoke` | Contract function name | +| `invoke.args` | array of strings | no | Arguments after the function name, passed as-is with no shell involved | +| `command` | string | one of `invoke` / `command` | Shell command, run with `sh -c` (`cmd /C` on Windows) from the directory that contains `starforge-project.toml` | +| `expect_exit` | integer | no (default `0`) | Exit code the check must return | +| `expect_output` | string | no | Stdout must equal this exactly, with surrounding whitespace trimmed | +| `expect_contains` | string | no | Stdout must contain this substring | +| `timeout_secs` | integer, 1–3600 | no (default `60`) | Longer runs are killed and count as a failure | + +When several assertions are set, all of them must hold. The exit code is +checked first. + +`command` checks receive the deployment through environment variables: + +| Variable | Value | +|----------|-------| +| `STARFORGE_CONTRACT_ID` | Contract ID returned by the deploy | +| `STARFORGE_NETWORK` | Network deployed to (`testnet` / `mainnet`) | +| `STARFORGE_SOURCE` | Source account the deploy used | + +### Validation + +Smoke test declarations are validated when the manifest loads, which happens +**before** the deploy starts. A mistake therefore never surfaces after a +contract is already live. The deploy is refused with a configuration error +(exit code 3) when: + +- a test is missing `name`, or two tests share a name +- a test sets both `command` and `invoke`, or neither of them +- `command` or `invoke.function` is empty +- `timeout_secs` is outside 1–3600 +- a key is unknown (for example the typo `expect_contain`) + +## Results and exit codes + +Each test prints one labelled line: + +```text + SMOKE PASS hello-returns-greeting (412 ms) + SMOKE FAIL counter-starts-at-zero (388 ms): expected output "0", got "7" + SMOKE FAIL health-script (60001 ms): timed out after 60s +``` + +A smoke failure is kept separate from a deploy failure: + +| Code | Name | Meaning | +|------|------|---------| +| 0 | SUCCESS | Deploy succeeded and every smoke test passed | +| 6 | EXECUTION_ERROR | The deploy itself failed; smoke tests did not run | +| 3 | CONFIG_ERROR | Invalid `smoke_tests` declaration; nothing was deployed | +| **9** | **SMOKE_TEST_FAILURE** | The deploy succeeded, but at least one smoke test failed or timed out | + +When a smoke test fails, the deploy is still recorded as successful in +`starforge deployments`, because the contract is on-chain. The error message +says so: + +```text +Post-deploy smoke tests failed (1 of 3). The deployment itself succeeded: +contract CABC…XYZ is live on testnet. +``` + +In CI, a check like `if [ $? -eq 9 ]` separates "the deployed contract +misbehaves" from "the deploy never happened". diff --git a/docs/SUMMARY.md b/docs/SUMMARY.md index 74fce911..edb2632b 100644 --- a/docs/SUMMARY.md +++ b/docs/SUMMARY.md @@ -23,6 +23,7 @@ # Deploying and operating - [Deploy policy as code](DEPLOY_POLICY.md) +- [Post-deploy smoke tests](SMOKE_TESTS.md) - [Deployment checkpoints](DEPLOYMENT_CHECKPOINTS.md) - [Deployment scaling](DEPLOYMENT_SCALING.md) - [Simulation resource fees](SIMULATION_RESOURCES.md) diff --git a/examples/smoke-tests/README.md b/examples/smoke-tests/README.md new file mode 100644 index 00000000..701d0f0e --- /dev/null +++ b/examples/smoke-tests/README.md @@ -0,0 +1,46 @@ +# Sample: post-deploy smoke tests + +This sample shows the `[[smoke_tests]]` section of `starforge-project.toml`. +The tests target the `simple-counter` template. + +| File | What it shows | +|------|---------------| +| [`starforge-project.toml`](starforge-project.toml) | Passing tests: two contract invocations and one shell check | +| [`failing/starforge-project.toml`](failing/starforge-project.toml) | One deliberately failing assertion | + +## Try it + +1. Build a counter contract from `templates/examples/simple-counter`. It + exposes `get_count`, `increment` and `reset`. +2. Copy `starforge-project.toml` from this directory to the root of that + project. +3. Deploy from inside the project. The smoke tests run once the deploy is + confirmed: + +```bash norun +starforge deploy --wasm path/to/counter.wasm \ + --wallet deployer --network testnet --yes --execute +``` + +Expected output after the deploy report: + +```text + SMOKE PASS counter-starts-at-zero (402 ms) + SMOKE PASS increment-returns-one (5130 ms) + SMOKE PASS contract-id-is-a-strkey (6 ms) + +✓ 3 smoke test(s) passed +``` + +With `failing/starforge-project.toml` in place instead, the deploy still +succeeds, but the command exits with code **9** (`SMOKE_TEST_FAILURE`): + +```text + SMOKE PASS counter-starts-at-zero (398 ms) + SMOKE FAIL counter-is-42 (401 ms): expected output "42", got "0" + +✗ 1 of 2 smoke test(s) failed. The deployment succeeded and the contract is live. +``` + +Pass `--skip-smoke` to deploy without running the tests. The full schema is +in [docs/SMOKE_TESTS.md](../../docs/SMOKE_TESTS.md). diff --git a/examples/smoke-tests/failing/starforge-project.toml b/examples/smoke-tests/failing/starforge-project.toml new file mode 100644 index 00000000..f08c2de0 --- /dev/null +++ b/examples/smoke-tests/failing/starforge-project.toml @@ -0,0 +1,16 @@ +# Deliberately failing smoke test, for the docs and for trying out the +# failure path. The deploy succeeds, then `counter-is-42` fails, so +# `starforge deploy --execute` exits with code 9 (SMOKE_TEST_FAILURE), +# not with a deploy error code. + +network = "testnet" + +[[smoke_tests]] +name = "counter-starts-at-zero" +invoke = { function = "get_count" } +expect_output = "0" + +[[smoke_tests]] +name = "counter-is-42" +invoke = { function = "get_count" } +expect_output = "42" diff --git a/examples/smoke-tests/starforge-project.toml b/examples/smoke-tests/starforge-project.toml new file mode 100644 index 00000000..03ce51af --- /dev/null +++ b/examples/smoke-tests/starforge-project.toml @@ -0,0 +1,27 @@ +# Sample project manifest with post-deploy smoke tests (#753). +# +# These tests target the `simple-counter` template +# (templates/examples/simple-counter). `starforge deploy --execute` runs them +# in order after the deploy is confirmed. See docs/SMOKE_TESTS.md. + +network = "testnet" + +# A freshly deployed counter must start at zero. +[[smoke_tests]] +name = "counter-starts-at-zero" +invoke = { function = "get_count" } +expect_output = "0" + +# The write path works and returns the new value. +[[smoke_tests]] +name = "increment-returns-one" +invoke = { function = "increment" } +expect_output = "1" +timeout_secs = 90 + +# Shell checks receive STARFORGE_CONTRACT_ID / STARFORGE_NETWORK / +# STARFORGE_SOURCE. This one is POSIX sh; use `cmd` syntax on Windows. +[[smoke_tests]] +name = "contract-id-is-a-strkey" +command = 'test "${#STARFORGE_CONTRACT_ID}" -eq 56 && test "${STARFORGE_CONTRACT_ID#C}" != "$STARFORGE_CONTRACT_ID"' +timeout_secs = 5 diff --git a/src/commands/deploy.rs b/src/commands/deploy.rs index 76dd44c8..ca3f0b34 100644 --- a/src/commands/deploy.rs +++ b/src/commands/deploy.rs @@ -6,7 +6,9 @@ use crate::utils::{ DeployRecord, DeployStatus, }, deploy_policy, deployment_monitor, horizon, notifications, optimizer, output, print as p, - simulation_resources, soroban, wallet_signer, + project_config, simulation_resources, + smoke_tests::{self, SmokeContext, SmokeTest}, + soroban, wallet_signer, wasm_hash::{compute_wasm_hash, BuildEnvironment}, wasm_preflight, }; @@ -78,6 +80,65 @@ pub struct DeployArgs { /// Comma-separated checklist item ids satisfied for this deploy (see deploy policy) #[arg(long, value_delimiter = ',')] pub checklist: Option>, + /// Do not run the `[[smoke_tests]]` declared in `starforge-project.toml` + /// after a successful `--execute` deploy + #[arg(long)] + pub skip_smoke: bool, +} + +/// Smoke tests declared in the discovered project manifest, plus the +/// directory they run from. Loading validates them, so a bad declaration +/// stops the command before anything is deployed. +fn load_smoke_tests(skip: bool) -> Result, PathBuf)>> { + if skip { + return Ok(None); + } + let start = std::env::current_dir().unwrap_or_else(|_| PathBuf::from(".")); + Ok(project_config::find_and_load_project_lockfile(&start)? + .filter(|(_, lockfile)| !lockfile.smoke_tests.is_empty()) + .map(|(path, lockfile)| { + ( + lockfile.smoke_tests, + smoke_tests::workdir_for_manifest(&path), + ) + })) +} + +/// Run post-deploy smoke tests against a confirmed deployment and report +/// each result. Returns a [`smoke_tests::SmokeTestFailure`] error if any failed. +fn run_post_deploy_smoke_tests( + tests: &[SmokeTest], + workdir: PathBuf, + contract_id: &str, + network: &str, + source: &str, +) -> Result<()> { + p::header("Post-deploy Smoke Tests"); + let ctx = SmokeContext { + contract_id: contract_id.to_string(), + network: network.to_string(), + source: source.to_string(), + workdir, + }; + let report = smoke_tests::run_all(tests, &ctx); + for (line, result) in report.lines().iter().zip(&report.results) { + if result.outcome.is_pass() { + println!(" {}", line.green()); + } else { + println!(" {}", line.red()); + } + } + println!(); + if report.all_passed() { + p::success(&format!("{} smoke test(s) passed", report.passed())); + } else { + p::error(&format!( + "{} of {} smoke test(s) failed. The deployment succeeded and the contract is live.", + report.failed(), + report.results.len() + )); + } + smoke_tests::into_result(&report, &ctx) } /// Extract a Soroban contract id (56-char `C…` strkey) from CLI stdout/stderr. @@ -455,6 +516,10 @@ pub async fn handle(args: DeployArgs) -> Result<()> { ); } + // Validate declared smoke tests up front: a broken declaration must fail + // here, never after a contract is already live. + let smoke = load_smoke_tests(args.skip_smoke)?; + let mut wasm_path = args.wasm.clone(); let mut wasm_bytes = fs::read(&wasm_path)?; let mut wasm_size_kb = wasm_bytes.len() as f64 / 1024.0; @@ -940,7 +1005,31 @@ pub async fn handle(args: DeployArgs) -> Result<()> { p::success("Deployment executed successfully!"); p::kv("Recorded deployment", &record_id[..8.min(record_id.len())]); println!("{}", stdout); + + // Smoke tests run only against a confirmed deployment, which means + // the Stellar CLI returned a contract ID. + if let Some((tests, workdir)) = smoke { + match parsed_contract_id.as_deref() { + Some(contract_id) => run_post_deploy_smoke_tests( + &tests, + workdir, + contract_id, + &args.network, + &wallet.public_key, + )?, + None => p::warn(&format!( + "Skipped {} smoke test(s): no contract ID in Stellar CLI output to test against.", + tests.len() + )), + } + } } else { + if let Some((tests, _)) = &smoke { + p::info(&format!( + "{} smoke test(s) declared; they run only after an executed deploy.", + tests.len() + )); + } p::info("Dry-run complete. Use --execute to deploy for real."); } diff --git a/src/main.rs b/src/main.rs index 9903acfe..720c2826 100644 --- a/src/main.rs +++ b/src/main.rs @@ -814,7 +814,17 @@ fn recovery_hints(command: &str, err: &anyhow::Error) -> Vec { } } "deploy" => { - if msg.contains("wasm") || msg.contains("not found") || msg.contains("no such file") { + if msg.contains("smoke tests failed") { + hints.push("The contract is deployed; only the post-deploy checks failed.".into()); + hints.push( + "Fix the contract or the [[smoke_tests]] entry in starforge-project.toml." + .into(), + ); + hints.push("Deploy without running smoke tests: --skip-smoke".into()); + } else if msg.contains("wasm") + || msg.contains("not found") + || msg.contains("no such file") + { hints.push("Build your contract first: stellar contract build".into()); hints.push("Make sure you pass the correct --wasm path to deploy.".into()); } else if msg.contains("account") || msg.contains("not found on") { diff --git a/src/utils/exit_codes.rs b/src/utils/exit_codes.rs index 5458eaf4..f156fd54 100644 --- a/src/utils/exit_codes.rs +++ b/src/utils/exit_codes.rs @@ -29,6 +29,9 @@ pub enum ExitCode { /// diff tool when the new WASM removes or changes public ABI surface without /// an explicit acknowledgment (`--acknowledge`). BreakingChange = 8, + /// Post-deploy smoke test failure (9) — the deploy was confirmed on-chain, + /// but a smoke test declared in the project manifest failed (#753). + SmokeTestFailure = 9, } impl ExitCode { @@ -49,6 +52,7 @@ impl ExitCode { Self::Execution => "EXECUTION_ERROR", Self::Environment => "ENVIRONMENT_ERROR", Self::BreakingChange => "BREAKING_INTERFACE_CHANGE", + Self::SmokeTestFailure => "SMOKE_TEST_FAILURE", } } @@ -68,6 +72,7 @@ impl ExitCode { Self::BreakingChange => { "Contract interface diff found a breaking change in the new WASM" } + Self::SmokeTestFailure => "Deployment succeeded but a post-deploy smoke test failed", } } @@ -94,6 +99,15 @@ pub fn determine_exit_code(err: &anyhow::Error) -> ExitCode { .collect::>() .join(" "); + // Smoke test failure (Code 9) — matched by type, before any message + // heuristics, because its text mentions contracts and networks. + if err + .chain() + .any(|c| c.is::()) + { + return ExitCode::SmokeTestFailure; + } + // 0. Breaking interface change (Code 8) — checked before Execution so the // interface-diff failure is never misclassified as a generic wasm error. if msg.contains("breaking interface change") { diff --git a/src/utils/mod.rs b/src/utils/mod.rs index 496f6ccb..03043d57 100644 --- a/src/utils/mod.rs +++ b/src/utils/mod.rs @@ -148,6 +148,7 @@ pub mod security_scanner; pub mod sep10; pub mod shamir; pub mod simulation_resources; +pub mod smoke_tests; pub mod social; pub mod soroban; pub mod state_diff; diff --git a/src/utils/project_config.rs b/src/utils/project_config.rs index 475cb706..f067a99f 100644 --- a/src/utils/project_config.rs +++ b/src/utils/project_config.rs @@ -22,6 +22,7 @@ use crate::utils::config::{ merge_configs, AiTelemetryConfig, Config, ConfigOverlay, FeatureFlagsConfig, NetworkConfig, PluginTrustConfig, }; +use crate::utils::smoke_tests::{self, SmokeTest}; use anyhow::{Context, Result}; use serde::{Deserialize, Serialize}; use std::collections::HashMap; @@ -61,6 +62,10 @@ pub struct ProjectLockfile { /// Networks to add, or to replace by name. #[serde(default, skip_serializing_if = "HashMap::is_empty")] pub networks: HashMap, + /// Post-deploy smoke tests run by `starforge deploy --execute` (#753). + /// Not a config override: it never reaches the merged [`Config`]. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub smoke_tests: Vec, } impl ProjectLockfile { @@ -93,7 +98,10 @@ impl ProjectLockfile { /// (`feautre_flags`) or a secret-bearing section (`wallets`) fails here /// instead of being ignored at load time. pub fn parse_project_lockfile_str(contents: &str) -> Result { - toml::from_str(contents).context("Invalid project lockfile TOML") + let lockfile: ProjectLockfile = + toml::from_str(contents).context("Invalid project lockfile TOML")?; + smoke_tests::validate(&lockfile.smoke_tests).context("Invalid project lockfile smoke_tests")?; + Ok(lockfile) } /// Load a [`ProjectLockfile`] from a TOML file. @@ -342,4 +350,66 @@ soroban_rpc_url = "https://rpc.example.com" assert_eq!(merged.ai_telemetry.enabled, false); assert_eq!(merged.plugin_trust.trusted_publishers.len(), 1); } + + // ── Smoke tests (#753) ──────────────────────────────────────────────────── + + #[test] + fn smoke_tests_parse_from_manifest() { + let lock = parse_project_lockfile_str( + r#" +[[smoke_tests]] +name = "hello" +invoke = { function = "hello", args = ["--to", "world"] } +expect_contains = "world" + +[[smoke_tests]] +name = "health" +command = "echo ok" +expect_output = "ok" +timeout_secs = 10 +"#, + ) + .unwrap(); + assert_eq!(lock.smoke_tests.len(), 2); + assert_eq!( + lock.smoke_tests[0].invoke.as_ref().unwrap().function, + "hello" + ); + assert_eq!(lock.smoke_tests[1].timeout_secs, Some(10)); + // Smoke tests are not a config override. + let merged = apply_project_overrides(Config::default(), &lock).unwrap(); + assert_eq!(merged, Config::default()); + } + + #[test] + fn sample_project_manifests_are_valid() { + let passing = parse_project_lockfile_str(include_str!( + "../../examples/smoke-tests/starforge-project.toml" + )) + .unwrap(); + assert_eq!(passing.smoke_tests.len(), 3); + + let failing = parse_project_lockfile_str(include_str!( + "../../examples/smoke-tests/failing/starforge-project.toml" + )) + .unwrap(); + assert_eq!(failing.smoke_tests.len(), 2); + } + + #[test] + fn manifest_without_smoke_tests_has_none() { + let lock = parse_project_lockfile_str("network = \"testnet\"").unwrap(); + assert!(lock.smoke_tests.is_empty()); + } + + #[test] + fn invalid_smoke_tests_fail_manifest_validation() { + let err = parse_project_lockfile_str("[[smoke_tests]]\nname = \"empty\"\n").unwrap_err(); + assert!(format!("{err:#}").contains("smoke_tests"), "{err:#}"); + + let err = parse_project_lockfile_str( + "[[smoke_tests]]\nname = \"typo\"\ncommand = \"echo\"\nexpect_contain = \"x\"\n", + ); + assert!(err.is_err(), "unknown smoke test keys must be rejected"); + } } diff --git a/src/utils/smoke_tests.rs b/src/utils/smoke_tests.rs new file mode 100644 index 00000000..23cb942e --- /dev/null +++ b/src/utils/smoke_tests.rs @@ -0,0 +1,673 @@ +//! Post-deploy smoke tests declared in the project manifest (#753). +//! +//! A project can list `[[smoke_tests]]` entries in `starforge-project.toml`. +//! `starforge deploy --execute` runs them automatically, and only, after the +//! Stellar CLI has confirmed the deployment and returned a contract ID. This +//! closes the loop between deploy and verification: a contract whose +//! initialisation or configuration is broken is caught immediately instead of +//! by the first user. +//! +//! Classification guarantees: +//! - Invalid smoke test declarations fail manifest validation *before* the +//! deploy starts, never after a contract is already live. +//! - A smoke failure is reported as [`SmokeTestFailure`], which maps to its +//! own exit code ([`crate::utils::exit_codes::ExitCode::SmokeTestFailure`]) +//! so CI can tell "the deploy broke" apart from "the deploy worked but the +//! deployed contract misbehaves". The deployment record stays `Success`: +//! the contract is on-chain either way. + +use anyhow::{bail, Context, Result}; +use serde::{Deserialize, Serialize}; +use std::collections::HashSet; +use std::fmt; +use std::io::Read; +use std::path::Path; +use std::process::{Command, Stdio}; +use std::time::{Duration, Instant}; + +/// Per-test timeout used when an entry does not set `timeout_secs`. +pub const DEFAULT_TIMEOUT_SECS: u64 = 60; + +/// Upper bound on `timeout_secs`, so a typo cannot hang a pipeline for days. +pub const MAX_TIMEOUT_SECS: u64 = 3600; + +/// One `[[smoke_tests]]` entry from the project manifest. +/// +/// Exactly one of `command` or `invoke` must be set. Assertions are optional; +/// with none set the test passes when the check exits with code 0. +#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub struct SmokeTest { + /// Human-readable name, unique within the manifest. + pub name: String, + /// Shell command, run through `sh -c` (or `cmd /C` on Windows) from the + /// directory containing the manifest. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub command: Option, + /// Contract function to invoke on the freshly deployed contract through + /// `stellar contract invoke`. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub invoke: Option, + /// Expected exit code (default `0`). + #[serde(default, skip_serializing_if = "Option::is_none")] + pub expect_exit: Option, + /// Expected stdout, compared after trimming surrounding whitespace. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub expect_output: Option, + /// Substring that stdout must contain. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub expect_contains: Option, + /// Timeout in seconds (default [`DEFAULT_TIMEOUT_SECS`]). + #[serde(default, skip_serializing_if = "Option::is_none")] + pub timeout_secs: Option, +} + +/// A contract function call used as a smoke check. +#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub struct InvokeCheck { + /// Contract function name. + pub function: String, + /// Arguments passed after the function name, e.g. `["--to", "world"]`. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub args: Vec, +} + +impl SmokeTest { + /// Effective timeout for this test. + pub fn timeout(&self) -> Duration { + Duration::from_secs(self.timeout_secs.unwrap_or(DEFAULT_TIMEOUT_SECS)) + } + + /// Effective expected exit code for this test. + pub fn expected_exit(&self) -> i32 { + self.expect_exit.unwrap_or(0) + } +} + +/// Validate smoke test declarations. +/// +/// Runs as part of manifest parsing, so every rule here is enforced before +/// a deploy begins. +pub fn validate(tests: &[SmokeTest]) -> Result<()> { + let mut seen = HashSet::new(); + for (index, test) in tests.iter().enumerate() { + let label = if test.name.trim().is_empty() { + format!("smoke_tests[{}]", index) + } else { + format!("smoke test '{}'", test.name) + }; + + if test.name.trim().is_empty() { + bail!("{}: `name` must not be empty", label); + } + if !seen.insert(test.name.as_str()) { + bail!("{}: duplicate smoke test name", label); + } + match (&test.command, &test.invoke) { + (Some(_), Some(_)) => bail!("{}: set only one of `command` or `invoke`", label), + (None, None) => bail!("{}: one of `command` or `invoke` is required", label), + (Some(command), None) if command.trim().is_empty() => { + bail!("{}: `command` must not be empty", label) + } + (None, Some(invoke)) if invoke.function.trim().is_empty() => { + bail!("{}: `invoke.function` must not be empty", label) + } + _ => {} + } + if let Some(timeout) = test.timeout_secs { + if timeout == 0 || timeout > MAX_TIMEOUT_SECS { + bail!( + "{}: `timeout_secs` must be between 1 and {}", + label, + MAX_TIMEOUT_SECS + ); + } + } + } + Ok(()) +} + +/// What the smoke tests run against: the confirmed deployment. +#[derive(Debug, Clone)] +pub struct SmokeContext { + /// Contract ID returned by the confirmed deploy. + pub contract_id: String, + /// Network the contract was deployed to. + pub network: String, + /// Stellar CLI identity or public key used as the invoke source. + pub source: String, + /// Working directory for `command` checks (the manifest's directory). + pub workdir: std::path::PathBuf, +} + +/// Outcome of a single smoke test. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum SmokeOutcome { + Passed, + /// An assertion did not hold; the string says which one and why. + Failed(String), + /// The check did not finish within its timeout. + TimedOut(Duration), + /// The check could not be started at all (e.g. `stellar` not installed). + Error(String), +} + +impl SmokeOutcome { + pub fn is_pass(&self) -> bool { + matches!(self, Self::Passed) + } + + /// One-line reason for a non-passing outcome. + pub fn reason(&self) -> Option { + match self { + Self::Passed => None, + Self::Failed(reason) => Some(reason.clone()), + Self::TimedOut(timeout) => Some(format!("timed out after {}s", timeout.as_secs())), + Self::Error(reason) => Some(format!("could not run: {}", reason)), + } + } +} + +/// Result of one smoke test, for reporting. +#[derive(Debug, Clone)] +pub struct SmokeResult { + pub name: String, + pub outcome: SmokeOutcome, + pub duration: Duration, +} + +/// Results of a full smoke run. +#[derive(Debug, Clone, Default)] +pub struct SmokeReport { + pub results: Vec, +} + +impl SmokeReport { + pub fn passed(&self) -> usize { + self.results.iter().filter(|r| r.outcome.is_pass()).count() + } + + pub fn failed(&self) -> usize { + self.results.len() - self.passed() + } + + pub fn all_passed(&self) -> bool { + self.failed() == 0 + } + + /// Plain-text report lines, each labelled `SMOKE PASS` or `SMOKE FAIL`. + pub fn lines(&self) -> Vec { + self.results + .iter() + .map(|r| { + let ms = r.duration.as_millis(); + match r.outcome.reason() { + None => format!("SMOKE PASS {} ({} ms)", r.name, ms), + Some(reason) => format!("SMOKE FAIL {} ({} ms): {}", r.name, ms, reason), + } + }) + .collect() + } +} + +/// Error returned when at least one smoke test failed after a confirmed deploy. +/// +/// Kept as a distinct type so exit-code classification can recognise it by +/// downcasting instead of guessing from message text. +#[derive(Debug, Clone)] +pub struct SmokeTestFailure { + pub failed: usize, + pub total: usize, + pub contract_id: String, + pub network: String, +} + +impl fmt::Display for SmokeTestFailure { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!( + f, + "Post-deploy smoke tests failed ({} of {}). The deployment itself succeeded: \ + contract {} is live on {}.", + self.failed, self.total, self.contract_id, self.network + ) + } +} + +impl std::error::Error for SmokeTestFailure {} + +/// Run every smoke test in order and collect the results. +/// +/// All tests run even after a failure, so one report shows everything that is +/// broken. +pub fn run_all(tests: &[SmokeTest], ctx: &SmokeContext) -> SmokeReport { + let results = tests + .iter() + .map(|test| { + let started = Instant::now(); + let outcome = run_one(test, ctx); + SmokeResult { + name: test.name.clone(), + outcome, + duration: started.elapsed(), + } + }) + .collect(); + SmokeReport { results } +} + +/// Turn a report into the command result: `Ok` when everything passed, +/// otherwise a [`SmokeTestFailure`]. +pub fn into_result(report: &SmokeReport, ctx: &SmokeContext) -> Result<()> { + if report.all_passed() { + return Ok(()); + } + Err(SmokeTestFailure { + failed: report.failed(), + total: report.results.len(), + contract_id: ctx.contract_id.clone(), + network: ctx.network.clone(), + } + .into()) +} + +fn run_one(test: &SmokeTest, ctx: &SmokeContext) -> SmokeOutcome { + let mut cmd = build_command(test, ctx); + match run_with_timeout(&mut cmd, test.timeout()) { + Ok(Some(output)) => check_assertions(test, &output), + Ok(None) => SmokeOutcome::TimedOut(test.timeout()), + Err(e) => SmokeOutcome::Error(e.to_string()), + } +} + +/// Arguments for `stellar contract invoke` against the deployed contract. +/// +/// Built as an argument vector (never a shell string) so manifest values +/// cannot inject extra shell commands. +pub fn build_invoke_args(invoke: &InvokeCheck, ctx: &SmokeContext) -> Vec { + let mut args = vec![ + "contract".to_string(), + "invoke".to_string(), + "--id".to_string(), + ctx.contract_id.clone(), + "--source".to_string(), + ctx.source.clone(), + "--network".to_string(), + ctx.network.clone(), + "--".to_string(), + invoke.function.clone(), + ]; + args.extend(invoke.args.iter().cloned()); + args +} + +fn build_command(test: &SmokeTest, ctx: &SmokeContext) -> Command { + let mut cmd = match (&test.command, &test.invoke) { + (Some(command), _) => shell(command), + (None, Some(invoke)) => { + let mut c = Command::new("stellar"); + c.args(build_invoke_args(invoke, ctx)); + c + } + // Unreachable after `validate`; fall back to a check that fails loudly. + (None, None) => shell("exit 127"), + }; + cmd.current_dir(&ctx.workdir) + .env("STARFORGE_CONTRACT_ID", &ctx.contract_id) + .env("STARFORGE_NETWORK", &ctx.network) + .env("STARFORGE_SOURCE", &ctx.source); + cmd +} + +fn shell(command: &str) -> Command { + if cfg!(windows) { + let mut c = Command::new("cmd"); + c.arg("/C").arg(command); + c + } else { + let mut c = Command::new("sh"); + c.arg("-c").arg(command); + c + } +} + +/// Captured output of a finished check. +#[derive(Debug, Clone)] +struct CheckOutput { + code: Option, + stdout: String, + stderr: String, +} + +/// Run `cmd`, returning `Ok(None)` if it exceeded `timeout`. +fn run_with_timeout(cmd: &mut Command, timeout: Duration) -> Result> { + cmd.stdin(Stdio::null()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()); + let mut child = cmd.spawn().context("failed to start smoke test process")?; + + // Drain both pipes on background threads so a chatty check cannot fill + // the OS pipe buffer and block while we poll for exit. + let mut child_stdout = child.stdout.take(); + let mut child_stderr = child.stderr.take(); + let stdout_reader = std::thread::spawn(move || { + let mut buf = Vec::new(); + if let Some(s) = child_stdout.as_mut() { + let _ = s.read_to_end(&mut buf); + } + buf + }); + let stderr_reader = std::thread::spawn(move || { + let mut buf = Vec::new(); + if let Some(s) = child_stderr.as_mut() { + let _ = s.read_to_end(&mut buf); + } + buf + }); + + let deadline = Instant::now() + timeout; + let status = loop { + match child.try_wait() { + Ok(Some(status)) => break status, + Ok(None) => { + if Instant::now() >= deadline { + let _ = child.kill(); + let _ = child.wait(); + // Grandchildren may keep the pipes open; detach the + // readers rather than block on them. + drop(stdout_reader); + drop(stderr_reader); + return Ok(None); + } + std::thread::sleep(Duration::from_millis(20)); + } + Err(e) => return Err(anyhow::anyhow!("failed waiting on smoke test: {}", e)), + } + }; + + let stdout = stdout_reader.join().unwrap_or_default(); + let stderr = stderr_reader.join().unwrap_or_default(); + Ok(Some(CheckOutput { + code: status.code(), + stdout: String::from_utf8_lossy(&stdout).into_owned(), + stderr: String::from_utf8_lossy(&stderr).into_owned(), + })) +} + +fn check_assertions(test: &SmokeTest, output: &CheckOutput) -> SmokeOutcome { + let expected = test.expected_exit(); + if output.code != Some(expected) { + let got = output + .code + .map(|c| c.to_string()) + .unwrap_or_else(|| "terminated by signal".to_string()); + let mut reason = format!("expected exit code {}, got {}", expected, got); + let stderr = output.stderr.trim(); + if !stderr.is_empty() { + reason.push_str(&format!(" (stderr: {})", truncate(stderr, 200))); + } + return SmokeOutcome::Failed(reason); + } + + let stdout = output.stdout.trim(); + if let Some(want) = &test.expect_output { + if stdout != want.trim() { + return SmokeOutcome::Failed(format!( + "expected output {:?}, got {:?}", + want.trim(), + truncate(stdout, 200) + )); + } + } + if let Some(needle) = &test.expect_contains { + if !stdout.contains(needle.as_str()) { + return SmokeOutcome::Failed(format!( + "expected output to contain {:?}, got {:?}", + needle, + truncate(stdout, 200) + )); + } + } + SmokeOutcome::Passed +} + +fn truncate(s: &str, max_chars: usize) -> String { + if s.chars().count() <= max_chars { + s.to_string() + } else { + let head: String = s.chars().take(max_chars).collect(); + format!("{}…", head) + } +} + +/// Directory that `command` checks run from: the manifest's own directory. +pub fn workdir_for_manifest(manifest_path: &Path) -> std::path::PathBuf { + manifest_path + .parent() + .filter(|p| !p.as_os_str().is_empty()) + .map(Path::to_path_buf) + .unwrap_or_else(|| std::path::PathBuf::from(".")) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn ctx() -> SmokeContext { + SmokeContext { + contract_id: format!("C{}", "A".repeat(55)), + network: "testnet".to_string(), + source: "deployer".to_string(), + workdir: std::env::temp_dir(), + } + } + + fn cmd_test(name: &str, command: &str) -> SmokeTest { + SmokeTest { + name: name.to_string(), + command: Some(command.to_string()), + ..Default::default() + } + } + + /// A command that sleeps for a few seconds on either platform. + fn sleep_command() -> &'static str { + if cfg!(windows) { + "ping -n 6 127.0.0.1 >NUL" + } else { + "sleep 5" + } + } + + // ── Schema validation ───────────────────────────────────────────────────── + + #[test] + fn valid_declarations_pass_validation() { + let tests = vec![ + cmd_test("health", "echo ok"), + SmokeTest { + name: "hello".to_string(), + invoke: Some(InvokeCheck { + function: "hello".to_string(), + args: vec!["--to".to_string(), "world".to_string()], + }), + expect_contains: Some("world".to_string()), + timeout_secs: Some(30), + ..Default::default() + }, + ]; + validate(&tests).unwrap(); + } + + #[test] + fn missing_check_is_rejected() { + let tests = vec![SmokeTest { + name: "nothing".to_string(), + ..Default::default() + }]; + let err = validate(&tests).unwrap_err().to_string(); + assert!( + err.contains("one of `command` or `invoke` is required"), + "{err}" + ); + } + + #[test] + fn both_command_and_invoke_is_rejected() { + let mut test = cmd_test("both", "echo ok"); + test.invoke = Some(InvokeCheck { + function: "f".to_string(), + args: vec![], + }); + let err = validate(&[test]).unwrap_err().to_string(); + assert!(err.contains("only one of"), "{err}"); + } + + #[test] + fn duplicate_and_empty_names_are_rejected() { + let err = validate(&[cmd_test("a", "echo"), cmd_test("a", "echo")]) + .unwrap_err() + .to_string(); + assert!(err.contains("duplicate"), "{err}"); + + let err = validate(&[cmd_test(" ", "echo")]).unwrap_err().to_string(); + assert!(err.contains("`name` must not be empty"), "{err}"); + } + + #[test] + fn out_of_range_timeout_is_rejected() { + for bad in [0, MAX_TIMEOUT_SECS + 1] { + let mut test = cmd_test("t", "echo"); + test.timeout_secs = Some(bad); + assert!(validate(&[test]).is_err(), "timeout {bad} must be rejected"); + } + } + + #[test] + fn unknown_fields_are_rejected_by_serde() { + let result: std::result::Result = + toml::from_str("name = \"x\"\ncommand = \"echo\"\nexpect_exitcode = 0\n"); + assert!( + result.is_err(), + "typos in assertion keys must not be ignored" + ); + } + + // ── Invoke argument construction ────────────────────────────────────────── + + #[test] + fn invoke_args_target_the_deployed_contract() { + let invoke = InvokeCheck { + function: "hello".to_string(), + args: vec!["--to".to_string(), "world; rm -rf /".to_string()], + }; + let c = ctx(); + let args = build_invoke_args(&invoke, &c); + let id = args.iter().position(|a| a == "--id").unwrap(); + assert_eq!(args[id + 1], c.contract_id); + let sep = args.iter().position(|a| a == "--").unwrap(); + assert_eq!(args[sep + 1], "hello"); + // Arguments are passed verbatim as argv entries, never via a shell. + assert_eq!(args.last().unwrap(), "world; rm -rf /"); + } + + // ── Execution and classification ────────────────────────────────────────── + + #[test] + fn passing_command_passes() { + let mut test = cmd_test("echo", "echo hello-smoke"); + test.expect_contains = Some("hello-smoke".to_string()); + let report = run_all(&[test], &ctx()); + assert!(report.all_passed(), "{:?}", report.lines()); + assert!(report.lines()[0].starts_with("SMOKE PASS")); + into_result(&report, &ctx()).unwrap(); + } + + #[test] + fn contract_id_is_exported_to_commands() { + let command = if cfg!(windows) { + "echo %STARFORGE_CONTRACT_ID%" + } else { + "echo $STARFORGE_CONTRACT_ID" + }; + let mut test = cmd_test("env", command); + test.expect_output = Some(ctx().contract_id); + assert!(run_all(&[test], &ctx()).all_passed()); + } + + #[test] + fn wrong_exit_code_fails_with_reason() { + let report = run_all(&[cmd_test("exit", "exit 3")], &ctx()); + assert_eq!(report.failed(), 1); + let line = &report.lines()[0]; + assert!(line.starts_with("SMOKE FAIL"), "{line}"); + assert!(line.contains("expected exit code 0, got 3"), "{line}"); + } + + #[test] + fn expected_nonzero_exit_can_pass() { + let mut test = cmd_test("exit", "exit 3"); + test.expect_exit = Some(3); + assert!(run_all(&[test], &ctx()).all_passed()); + } + + #[test] + fn output_mismatch_fails() { + let mut test = cmd_test("out", "echo actual"); + test.expect_output = Some("expected".to_string()); + let report = run_all(&[test], &ctx()); + assert!(report.lines()[0].contains("expected output \"expected\"")); + } + + #[test] + fn timeout_is_a_smoke_failure() { + let mut test = cmd_test("slow", sleep_command()); + test.timeout_secs = Some(1); + let started = Instant::now(); + let report = run_all(&[test], &ctx()); + assert!( + started.elapsed() < Duration::from_secs(4), + "timeout not enforced" + ); + assert_eq!( + report.results[0].outcome, + SmokeOutcome::TimedOut(Duration::from_secs(1)) + ); + assert!(report.lines()[0].contains("timed out after 1s")); + } + + #[test] + fn all_tests_run_after_a_failure() { + let report = run_all( + &[cmd_test("bad", "exit 1"), cmd_test("good", "echo ok")], + &ctx(), + ); + assert_eq!(report.results.len(), 2); + assert_eq!((report.passed(), report.failed()), (1, 1)); + } + + #[test] + fn failure_error_is_typed_and_says_deploy_succeeded() { + let report = run_all(&[cmd_test("bad", "exit 1")], &ctx()); + let err = into_result(&report, &ctx()).unwrap_err(); + let failure = err + .downcast_ref::() + .expect("smoke failures must be a typed error"); + assert_eq!((failure.failed, failure.total), (1, 1)); + let msg = err.to_string(); + assert!(msg.contains("smoke tests failed"), "{msg}"); + assert!(msg.contains("deployment itself succeeded"), "{msg}"); + } + + #[test] + fn manifest_workdir_is_its_parent() { + assert_eq!( + workdir_for_manifest(Path::new("proj/starforge-project.toml")), + std::path::PathBuf::from("proj") + ); + assert_eq!( + workdir_for_manifest(Path::new("starforge-project.toml")), + std::path::PathBuf::from(".") + ); + } +} diff --git a/tests/cli_exit_codes.rs b/tests/cli_exit_codes.rs index acad7fc4..e8419eaa 100644 --- a/tests/cli_exit_codes.rs +++ b/tests/cli_exit_codes.rs @@ -20,6 +20,7 @@ fn test_exit_code_numeric_values() { assert_eq!(ExitCode::Execution.code(), 6); assert_eq!(ExitCode::Environment.code(), 7); assert_eq!(ExitCode::BreakingChange.code(), 8); + assert_eq!(ExitCode::SmokeTestFailure.code(), 9); } #[test] @@ -134,6 +135,30 @@ fn test_classify_breaking_interface_change() { assert_eq!(determine_exit_code(&err), ExitCode::BreakingChange); } +#[test] +fn test_classify_smoke_test_failure_distinct_from_deploy_failure() { + use starforge::utils::smoke_tests::SmokeTestFailure; + + let smoke: anyhow::Error = SmokeTestFailure { + failed: 1, + total: 2, + contract_id: format!("C{}", "A".repeat(55)), + network: "testnet".to_string(), + } + .into(); + // The message mentions a contract and a network, but the type wins. + assert_eq!(determine_exit_code(&smoke), ExitCode::SmokeTestFailure); + assert_eq!(ExitCode::SmokeTestFailure.name(), "SMOKE_TEST_FAILURE"); + + // Still classified when wrapped in extra context. + let wrapped = smoke.context("deploy post-checks"); + assert_eq!(determine_exit_code(&wrapped), ExitCode::SmokeTestFailure); + + // A real deploy failure keeps its existing classification. + let deploy = anyhow!("Stellar CLI deployment failed: tx failed"); + assert_eq!(determine_exit_code(&deploy), ExitCode::Execution); +} + // ── 3. Chained error context classification ─────────────────────────────────── #[test]