diff --git a/study-power-feasibility-assistant/README.md b/study-power-feasibility-assistant/README.md new file mode 100644 index 00000000..870ad65d --- /dev/null +++ b/study-power-feasibility-assistant/README.md @@ -0,0 +1,39 @@ +# Study Power Feasibility Assistant + +This module adds a focused statistical power review slice for SCIBASE issue +[#16](https://github.com/SCIBASE-AI/SCIBASE.AI/issues/16). It helps the AI +Research Assistant Suite detect manuscript claims that overstate what the study +design can support. + +The contribution is intentionally narrow. It does not duplicate prompt-safety, +supplement readiness, rebuttal generation, uncertainty calibration, grant-fit, +citation-context, benchmark leakage, figure-claim, or analysis-variable +assistants already submitted for the same broad issue. + +## What It Checks + +- Required sample size per group for continuous and binary outcomes. +- Actual sample size coverage against target alpha and power. +- Expected effect size versus the minimum detectable effect. +- Multiple-comparison load when no analysis plan is declared. +- Missing preregistration, missing outcome declaration, or missing analysis plan. +- Reviewer actions for revise/hold decisions. + +## Local Usage + +```bash +cd study-power-feasibility-assistant +npm run check +npm test +npm run demo +``` + +`npm run demo` writes reviewer artifacts under `reports/`: + +- `study-power-review-packet.json` +- `study-power-review-report.md` +- `summary.svg` +- `demo.mp4` + +All examples use synthetic study metadata. No external services, API keys, +accounts, or private research records are required. diff --git a/study-power-feasibility-assistant/acceptance-notes.md b/study-power-feasibility-assistant/acceptance-notes.md new file mode 100644 index 00000000..d45386c0 --- /dev/null +++ b/study-power-feasibility-assistant/acceptance-notes.md @@ -0,0 +1,26 @@ +# Acceptance Notes + +## Reviewer Checklist + +- Self-contained under `study-power-feasibility-assistant/`. +- Dependency-free Node.js implementation. +- Synthetic study metadata only. +- Tests cover pass, revise, and hold decisions. +- Demo artifacts are generated locally and include an MP4 proof artifact. + +## Commands Run + +```bash +npm run check +npm test +npm run demo +ffprobe -v error -show_entries format=duration,size -show_entries stream=codec_name,width,height -of default=noprint_wrappers=1 reports/demo.mp4 +git diff --check +``` + +## Limitations + +- The sample-size formulas are deterministic screening heuristics for review + triage, not a substitute for a full statistical analysis plan. +- Production integration should replace the sample policy constants with + SCIBASE domain-specific statistical review templates. diff --git a/study-power-feasibility-assistant/demo.js b/study-power-feasibility-assistant/demo.js new file mode 100644 index 00000000..8667b55b --- /dev/null +++ b/study-power-feasibility-assistant/demo.js @@ -0,0 +1,81 @@ +const fs = require("node:fs") +const path = require("node:path") +const { spawnSync } = require("node:child_process") +const { evaluateStudyPortfolio } = require("./index") +const { reviewPolicy, studyClaims } = require("./sample-data") + +const reportsDir = path.join(__dirname, "reports") +fs.mkdirSync(reportsDir, { recursive: true }) + +const packet = evaluateStudyPortfolio({ studyClaims, reviewPolicy }) +const { summary } = packet + +fs.writeFileSync( + path.join(reportsDir, "study-power-review-packet.json"), + `${JSON.stringify(packet, null, 2)}\n`, +) + +const markdown = [ + "# Study Power Feasibility Assistant Report", + "", + `Generated claims: ${summary.totalClaims}`, + `Passed: ${summary.passed}`, + `Needs revision: ${summary.revise}`, + `Held: ${summary.held}`, + `Reviewer actions: ${summary.reviewerActions}`, + `Audit digest: \`${packet.audit.digest}\``, + "", + "## Claim Decisions", + ...packet.decisions.flatMap((decision) => [ + "", + `### ${decision.id}: ${decision.title}`, + `- Status: ${decision.status}`, + `- Domain: ${decision.domain}`, + `- Required n/group: ${decision.powerAssessment.requiredPerGroup}`, + `- Actual n/group: ${decision.powerAssessment.sampleSizePerGroup}`, + `- Coverage ratio: ${decision.powerAssessment.coverageRatio}`, + `- Findings: ${decision.findings.map((finding) => finding.code).join(", ") || "none"}`, + `- First action: ${decision.reviewerActions[0]?.message || "none"}`, + ]), + "", +] + +fs.writeFileSync(path.join(reportsDir, "study-power-review-report.md"), markdown.join("\n")) + +const svg = ` + + Study Power Feasibility Assistant + Synthetic statistical review packet for SCIBASE issue #16 + + ${summary.passed} + passed + + ${summary.revise} + revise + + ${summary.held} + held + Checks: n per group, effect size, variance, alpha, target power, multiple testing, preregistration evidence. + Digest ${packet.audit.digest.slice(0, 24)}... + +` +fs.writeFileSync(path.join(reportsDir, "summary.svg"), svg) + +const ffmpeg = spawnSync("ffmpeg", [ + "-y", + "-f", + "lavfi", + "-i", + "color=c=0x111827:s=960x540:d=6:r=15", + "-vf", + "drawbox=x=48:y=170:w=250:h=150:color=0x0f766e@1:t=fill,drawbox=x=355:y=170:w=250:h=150:color=0xb45309@1:t=fill,drawbox=x=662:y=170:w=250:h=150:color=0x991b1b@1:t=fill,drawbox=x=48:y=370:w=864:h=18:color=0x22d3ee@1:t=fill", + "-pix_fmt", + "yuv420p", + path.join(reportsDir, "demo.mp4"), +], { stdio: "ignore" }) + +if (ffmpeg.status !== 0) { + console.warn("ffmpeg video generation failed; summary.svg and JSON/Markdown reports were still generated.") +} + +console.log(`Wrote study power artifacts to ${reportsDir}`) diff --git a/study-power-feasibility-assistant/index.js b/study-power-feasibility-assistant/index.js new file mode 100644 index 00000000..771c473e --- /dev/null +++ b/study-power-feasibility-assistant/index.js @@ -0,0 +1,204 @@ +const crypto = require("node:crypto") + +function stableJson(value) { + if (Array.isArray(value)) { + return `[${value.map(stableJson).join(",")}]` + } + if (value && typeof value === "object") { + return `{${Object.keys(value).sort().map((key) => `${JSON.stringify(key)}:${stableJson(value[key])}`).join(",")}}` + } + return JSON.stringify(value) +} + +function digestFor(value) { + return crypto.createHash("sha256").update(stableJson(value)).digest("hex") +} + +function finding(code, severity, message, detail = {}) { + return { code, severity, message, detail } +} + +function getZAlpha(alpha, policy) { + return policy.alphaByFamily[String(alpha)] ?? 1.96 +} + +function getZPower(targetPower, policy) { + return policy.zPowerByTarget[String(targetPower)] ?? policy.zPowerByTarget[String(policy.targetPower)] ?? 0.84 +} + +function requiredContinuousN(claim, policy) { + const zAlpha = getZAlpha(claim.alpha, policy) + const zPower = getZPower(claim.targetPower, policy) + const variance = claim.observedStandardDeviation ** 2 + const effect = Math.abs(claim.expectedEffectSize) + + if (effect <= 0) { + return Number.POSITIVE_INFINITY + } + + return Math.ceil((2 * (zAlpha + zPower) ** 2 * variance) / (effect ** 2)) +} + +function requiredBinaryN(claim, policy) { + const zAlpha = getZAlpha(claim.alpha, policy) + const zPower = getZPower(claim.targetPower, policy) + const p1 = claim.baselineRate + const p2 = claim.expectedRate + const delta = Math.abs(p1 - p2) + + if (!Number.isFinite(delta) || delta <= 0) { + return Number.POSITIVE_INFINITY + } + + const pooled = (p1 + p2) / 2 + const pooledVariance = 2 * pooled * (1 - pooled) + const armVariance = p1 * (1 - p1) + p2 * (1 - p2) + return Math.ceil(((zAlpha * Math.sqrt(pooledVariance) + zPower * Math.sqrt(armVariance)) ** 2) / (delta ** 2)) +} + +function requiredPerGroup(claim, policy) { + if (claim.outcomeType === "binary") { + return requiredBinaryN(claim, policy) + } + return requiredContinuousN(claim, policy) +} + +function estimateCoverageRatio(claim, requiredN) { + if (!Number.isFinite(requiredN) || requiredN <= 0) { + return 0 + } + return Number((claim.sampleSizePerGroup / requiredN).toFixed(3)) +} + +function minimumDetectableContinuousEffect(claim, policy) { + const zAlpha = getZAlpha(claim.alpha, policy) + const zPower = getZPower(claim.targetPower, policy) + const variance = claim.observedStandardDeviation ** 2 + return Math.sqrt((2 * (zAlpha + zPower) ** 2 * variance) / claim.sampleSizePerGroup) +} + +function evaluatePreregistration(claim) { + const findings = [] + if (!claim.preregistration.present) { + findings.push(finding("PREREGISTRATION_MISSING", "blocker", "Power-sensitive confirmatory claim lacks preregistration evidence.")) + return findings + } + if (!claim.preregistration.outcomeDeclared) { + findings.push(finding("OUTCOME_NOT_PREREGISTERED", "warning", "Primary outcome is not declared in the preregistration metadata.")) + } + if (!claim.preregistration.analysisPlanDeclared) { + findings.push(finding("ANALYSIS_PLAN_NOT_DECLARED", "warning", "Analysis plan is missing from preregistration metadata.")) + } + return findings +} + +function evaluateClaim(claim, policy) { + const requiredN = requiredPerGroup(claim, policy) + const coverageRatio = estimateCoverageRatio(claim, requiredN) + const findings = [] + + if (coverageRatio < policy.minPowerCoverageRatio) { + findings.push(finding("UNDERPOWERED_CLAIM", coverageRatio < 0.45 ? "blocker" : "warning", "Claimed effect is not supported by the available sample size.", { + sampleSizePerGroup: claim.sampleSizePerGroup, + requiredPerGroup: requiredN, + coverageRatio, + })) + } + + if (claim.outcomeType === "continuous") { + const minimumDetectableEffect = minimumDetectableContinuousEffect(claim, policy) + const effectCoverage = Number((Math.abs(claim.expectedEffectSize) / minimumDetectableEffect).toFixed(3)) + if (effectCoverage < policy.minDetectableEffectCoverage) { + findings.push(finding("EFFECT_SIZE_TOO_SMALL_FOR_DESIGN", "warning", "Expected effect is below the design's detectable effect threshold.", { + expectedEffectSize: claim.expectedEffectSize, + minimumDetectableEffect: Number(minimumDetectableEffect.toFixed(3)), + effectCoverage, + })) + } + } + + if (claim.plannedComparisons > policy.maxUndisclosedTests && !claim.preregistration.analysisPlanDeclared) { + findings.push(finding("MULTIPLE_TESTING_PLAN_MISSING", "warning", "Large comparison set needs an explicit multiplicity correction plan.", { + plannedComparisons: claim.plannedComparisons, + })) + } + + findings.push(...evaluatePreregistration(claim)) + + const blockers = findings.filter((item) => item.severity === "blocker") + const warnings = findings.filter((item) => item.severity === "warning") + const status = blockers.length > 0 ? "hold" : (warnings.length > 0 ? "revise" : "pass") + + const decision = { + id: claim.id, + title: claim.title, + domain: claim.domain, + status, + manuscriptClaim: claim.manuscriptClaim, + powerAssessment: { + outcomeType: claim.outcomeType, + sampleSizePerGroup: claim.sampleSizePerGroup, + requiredPerGroup: requiredN, + coverageRatio, + targetPower: claim.targetPower, + alpha: claim.alpha, + plannedComparisons: claim.plannedComparisons, + }, + findings, + reviewerActions: buildReviewerActions(status, findings), + } + + return { + ...decision, + auditDigest: digestFor(decision), + } +} + +function buildReviewerActions(status, findings) { + if (status === "pass") { + return [{ code: "KEEP_POWER_CLAIM", owner: "author", message: "Keep the current power claim with the existing methods disclosure." }] + } + + const actions = findings.map((item) => { + if (item.code === "UNDERPOWERED_CLAIM") { + return { code: "SOFTEN_OR_REPOWER_CLAIM", owner: "author", message: "Revise the claim, increase sample size, or report it as exploratory." } + } + if (item.code === "MULTIPLE_TESTING_PLAN_MISSING") { + return { code: "ADD_MULTIPLICITY_PLAN", owner: "statistics-reviewer", message: "Add correction strategy or clearly label exploratory endpoints." } + } + if (item.code === "PREREGISTRATION_MISSING") { + return { code: "ADD_PREREGISTRATION_LIMITATION", owner: "author", message: "Add preregistration status and limitation text before submission." } + } + return { code: `ADDRESS_${item.code}`, owner: "author", message: item.message } + }) + + return [...new Map(actions.map((item) => [item.code, item])).values()] +} + +function evaluateStudyPortfolio({ studyClaims, reviewPolicy }) { + const decisions = studyClaims.map((claim) => evaluateClaim(claim, reviewPolicy)) + const summary = { + totalClaims: decisions.length, + passed: decisions.filter((decision) => decision.status === "pass").length, + revise: decisions.filter((decision) => decision.status === "revise").length, + held: decisions.filter((decision) => decision.status === "hold").length, + reviewerActions: decisions.reduce((sum, decision) => sum + decision.reviewerActions.length, 0), + } + + return { + generatedAt: "2026-05-21T14:05:00.000Z", + policy: reviewPolicy, + summary, + decisions, + audit: { + source: "synthetic-study-power-feasibility-review", + digest: digestFor({ summary, decisions }), + }, + } +} + +module.exports = { + evaluateClaim, + evaluateStudyPortfolio, + requiredPerGroup, +} diff --git a/study-power-feasibility-assistant/package.json b/study-power-feasibility-assistant/package.json new file mode 100644 index 00000000..6c8d9ce1 --- /dev/null +++ b/study-power-feasibility-assistant/package.json @@ -0,0 +1,12 @@ +{ + "name": "study-power-feasibility-assistant", + "version": "1.0.0", + "description": "Statistical power and effect-size feasibility assistant for SCIBASE issue #16", + "private": true, + "scripts": { + "check": "node --check index.js && node --check sample-data.js && node --check test.js && node --check demo.js", + "test": "node test.js", + "demo": "node demo.js" + }, + "license": "MIT" +} diff --git a/study-power-feasibility-assistant/reports/demo.mp4 b/study-power-feasibility-assistant/reports/demo.mp4 new file mode 100644 index 00000000..d25efbd3 Binary files /dev/null and b/study-power-feasibility-assistant/reports/demo.mp4 differ diff --git a/study-power-feasibility-assistant/reports/study-power-review-packet.json b/study-power-feasibility-assistant/reports/study-power-review-packet.json new file mode 100644 index 00000000..f8129532 --- /dev/null +++ b/study-power-feasibility-assistant/reports/study-power-review-packet.json @@ -0,0 +1,248 @@ +{ + "generatedAt": "2026-05-21T14:05:00.000Z", + "policy": { + "targetPower": 0.8, + "minPowerCoverageRatio": 0.9, + "minDetectableEffectCoverage": 0.75, + "maxUndisclosedTests": 8, + "alphaByFamily": { + "0.05": 1.96, + "0.01": 2.58 + }, + "zPowerByTarget": { + "0.8": 0.84, + "0.9": 1.28 + } + }, + "summary": { + "totalClaims": 4, + "passed": 1, + "revise": 2, + "held": 1, + "reviewerActions": 11 + }, + "decisions": [ + { + "id": "SP-OK-001", + "title": "Expected cytokine reduction in randomized pilot cohort", + "domain": "molecular-biology", + "status": "pass", + "manuscriptClaim": "The study is sufficiently powered to detect a moderate cytokine response.", + "powerAssessment": { + "outcomeType": "continuous", + "sampleSizePerGroup": 96, + "requiredPerGroup": 41, + "coverageRatio": 2.341, + "targetPower": 0.8, + "alpha": 0.05, + "plannedComparisons": 4 + }, + "findings": [], + "reviewerActions": [ + { + "code": "KEEP_POWER_CLAIM", + "owner": "author", + "message": "Keep the current power claim with the existing methods disclosure." + } + ], + "auditDigest": "6165acfe2dc281edc3e80032b838c0a27dc29387f53372636c0a6e01df5c9c3b" + }, + { + "id": "SP-REV-002", + "title": "Behavioral intervention claim with thin recruitment", + "domain": "clinical-trials", + "status": "revise", + "manuscriptClaim": "The intervention is powered for clinically meaningful effects across all behavioral endpoints.", + "powerAssessment": { + "outcomeType": "continuous", + "sampleSizePerGroup": 80, + "requiredPerGroup": 128, + "coverageRatio": 0.625, + "targetPower": 0.8, + "alpha": 0.05, + "plannedComparisons": 12 + }, + "findings": [ + { + "code": "UNDERPOWERED_CLAIM", + "severity": "warning", + "message": "Claimed effect is not supported by the available sample size.", + "detail": { + "sampleSizePerGroup": 80, + "requiredPerGroup": 128, + "coverageRatio": 0.625 + } + }, + { + "code": "MULTIPLE_TESTING_PLAN_MISSING", + "severity": "warning", + "message": "Large comparison set needs an explicit multiplicity correction plan.", + "detail": { + "plannedComparisons": 12 + } + }, + { + "code": "OUTCOME_NOT_PREREGISTERED", + "severity": "warning", + "message": "Primary outcome is not declared in the preregistration metadata.", + "detail": {} + }, + { + "code": "ANALYSIS_PLAN_NOT_DECLARED", + "severity": "warning", + "message": "Analysis plan is missing from preregistration metadata.", + "detail": {} + } + ], + "reviewerActions": [ + { + "code": "SOFTEN_OR_REPOWER_CLAIM", + "owner": "author", + "message": "Revise the claim, increase sample size, or report it as exploratory." + }, + { + "code": "ADD_MULTIPLICITY_PLAN", + "owner": "statistics-reviewer", + "message": "Add correction strategy or clearly label exploratory endpoints." + }, + { + "code": "ADDRESS_OUTCOME_NOT_PREREGISTERED", + "owner": "author", + "message": "Primary outcome is not declared in the preregistration metadata." + }, + { + "code": "ADDRESS_ANALYSIS_PLAN_NOT_DECLARED", + "owner": "author", + "message": "Analysis plan is missing from preregistration metadata." + } + ], + "auditDigest": "8c1ca640efe6f00af92b66996fd8c76464e02882e07b19767a7fd3e25409e2f8" + }, + { + "id": "SP-HOLD-003", + "title": "Rare adverse event safety claim", + "domain": "clinical-safety", + "status": "hold", + "manuscriptClaim": "The study rules out rare adverse-event differences between arms.", + "powerAssessment": { + "outcomeType": "binary", + "sampleSizePerGroup": 60, + "requiredPerGroup": 3414, + "coverageRatio": 0.018, + "targetPower": 0.9, + "alpha": 0.05, + "plannedComparisons": 3 + }, + "findings": [ + { + "code": "UNDERPOWERED_CLAIM", + "severity": "blocker", + "message": "Claimed effect is not supported by the available sample size.", + "detail": { + "sampleSizePerGroup": 60, + "requiredPerGroup": 3414, + "coverageRatio": 0.018 + } + }, + { + "code": "PREREGISTRATION_MISSING", + "severity": "blocker", + "message": "Power-sensitive confirmatory claim lacks preregistration evidence.", + "detail": {} + } + ], + "reviewerActions": [ + { + "code": "SOFTEN_OR_REPOWER_CLAIM", + "owner": "author", + "message": "Revise the claim, increase sample size, or report it as exploratory." + }, + { + "code": "ADD_PREREGISTRATION_LIMITATION", + "owner": "author", + "message": "Add preregistration status and limitation text before submission." + } + ], + "auditDigest": "6e717ea90218666bd51cd517810d75ab036a309c0d210f7e00105c0265bdae90" + }, + { + "id": "SP-REV-004", + "title": "Exploratory single-cell pathway screen", + "domain": "single-cell-rna-seq", + "status": "revise", + "manuscriptClaim": "The pathway screen identifies robust differential signals across all declared modules.", + "powerAssessment": { + "outcomeType": "continuous", + "sampleSizePerGroup": 90, + "requiredPerGroup": 170, + "coverageRatio": 0.529, + "targetPower": 0.8, + "alpha": 0.01, + "plannedComparisons": 84 + }, + "findings": [ + { + "code": "UNDERPOWERED_CLAIM", + "severity": "warning", + "message": "Claimed effect is not supported by the available sample size.", + "detail": { + "sampleSizePerGroup": 90, + "requiredPerGroup": 170, + "coverageRatio": 0.529 + } + }, + { + "code": "EFFECT_SIZE_TOO_SMALL_FOR_DESIGN", + "severity": "warning", + "message": "Expected effect is below the design's detectable effect threshold.", + "detail": { + "expectedEffectSize": 0.52, + "minimumDetectableEffect": 0.714, + "effectCoverage": 0.729 + } + }, + { + "code": "MULTIPLE_TESTING_PLAN_MISSING", + "severity": "warning", + "message": "Large comparison set needs an explicit multiplicity correction plan.", + "detail": { + "plannedComparisons": 84 + } + }, + { + "code": "ANALYSIS_PLAN_NOT_DECLARED", + "severity": "warning", + "message": "Analysis plan is missing from preregistration metadata.", + "detail": {} + } + ], + "reviewerActions": [ + { + "code": "SOFTEN_OR_REPOWER_CLAIM", + "owner": "author", + "message": "Revise the claim, increase sample size, or report it as exploratory." + }, + { + "code": "ADDRESS_EFFECT_SIZE_TOO_SMALL_FOR_DESIGN", + "owner": "author", + "message": "Expected effect is below the design's detectable effect threshold." + }, + { + "code": "ADD_MULTIPLICITY_PLAN", + "owner": "statistics-reviewer", + "message": "Add correction strategy or clearly label exploratory endpoints." + }, + { + "code": "ADDRESS_ANALYSIS_PLAN_NOT_DECLARED", + "owner": "author", + "message": "Analysis plan is missing from preregistration metadata." + } + ], + "auditDigest": "24dfd0a5f0a81235e6afa28ee6c7062e4a4542a0e6d5c04bd5fcd62595c080ff" + } + ], + "audit": { + "source": "synthetic-study-power-feasibility-review", + "digest": "7f58b7da5b188969bb899994be5950262cd7672cccf806638b9f2241c070889a" + } +} diff --git a/study-power-feasibility-assistant/reports/study-power-review-report.md b/study-power-feasibility-assistant/reports/study-power-review-report.md new file mode 100644 index 00000000..0547baf3 --- /dev/null +++ b/study-power-feasibility-assistant/reports/study-power-review-report.md @@ -0,0 +1,46 @@ +# Study Power Feasibility Assistant Report + +Generated claims: 4 +Passed: 1 +Needs revision: 2 +Held: 1 +Reviewer actions: 11 +Audit digest: `7f58b7da5b188969bb899994be5950262cd7672cccf806638b9f2241c070889a` + +## Claim Decisions + +### SP-OK-001: Expected cytokine reduction in randomized pilot cohort +- Status: pass +- Domain: molecular-biology +- Required n/group: 41 +- Actual n/group: 96 +- Coverage ratio: 2.341 +- Findings: none +- First action: Keep the current power claim with the existing methods disclosure. + +### SP-REV-002: Behavioral intervention claim with thin recruitment +- Status: revise +- Domain: clinical-trials +- Required n/group: 128 +- Actual n/group: 80 +- Coverage ratio: 0.625 +- Findings: UNDERPOWERED_CLAIM, MULTIPLE_TESTING_PLAN_MISSING, OUTCOME_NOT_PREREGISTERED, ANALYSIS_PLAN_NOT_DECLARED +- First action: Revise the claim, increase sample size, or report it as exploratory. + +### SP-HOLD-003: Rare adverse event safety claim +- Status: hold +- Domain: clinical-safety +- Required n/group: 3414 +- Actual n/group: 60 +- Coverage ratio: 0.018 +- Findings: UNDERPOWERED_CLAIM, PREREGISTRATION_MISSING +- First action: Revise the claim, increase sample size, or report it as exploratory. + +### SP-REV-004: Exploratory single-cell pathway screen +- Status: revise +- Domain: single-cell-rna-seq +- Required n/group: 170 +- Actual n/group: 90 +- Coverage ratio: 0.529 +- Findings: UNDERPOWERED_CLAIM, EFFECT_SIZE_TOO_SMALL_FOR_DESIGN, MULTIPLE_TESTING_PLAN_MISSING, ANALYSIS_PLAN_NOT_DECLARED +- First action: Revise the claim, increase sample size, or report it as exploratory. diff --git a/study-power-feasibility-assistant/reports/summary.svg b/study-power-feasibility-assistant/reports/summary.svg new file mode 100644 index 00000000..6f7f5f9c --- /dev/null +++ b/study-power-feasibility-assistant/reports/summary.svg @@ -0,0 +1,16 @@ + + + Study Power Feasibility Assistant + Synthetic statistical review packet for SCIBASE issue #16 + + 1 + passed + + 2 + revise + + 1 + held + Checks: n per group, effect size, variance, alpha, target power, multiple testing, preregistration evidence. + Digest 7f58b7da5b188969bb899994... + diff --git a/study-power-feasibility-assistant/requirements-map.md b/study-power-feasibility-assistant/requirements-map.md new file mode 100644 index 00000000..b63457e6 --- /dev/null +++ b/study-power-feasibility-assistant/requirements-map.md @@ -0,0 +1,18 @@ +# Requirements Map + +| Issue #16 capability | Coverage in this module | +| --- | --- | +| Auto peer review reports | Produces structured statistical-review findings and reviewer actions. | +| Statistical or methodological red flags | Flags underpowered claims, weak detectable-effect coverage, and missing multiplicity plans. | +| Claims vs. evidence alignment | Compares manuscript claims against sample size, effect-size, variance, alpha, and power evidence. | +| Adaptive templates per domain | Includes domain metadata in each decision packet for downstream template selection. | +| Reproducibility confidence support | Requires preregistration and analysis-plan evidence before confirmatory power claims pass. | +| Immediate research assistant value | Runs locally with deterministic sample data, JSON/Markdown/SVG/MP4 artifacts, and focused tests. | + +## Non-Overlap Notes + +This is a study-design feasibility assistant. It avoids duplicating existing +SCIBASE #16 slices around prompt injection, supplementary material readiness, +rebuttal response packs, uncertainty calibration, limitations disclosure, +grant-fit alignment, citation context, benchmark leakage, figure consistency, +and analysis-variable provenance. diff --git a/study-power-feasibility-assistant/sample-data.js b/study-power-feasibility-assistant/sample-data.js new file mode 100644 index 00000000..e4b79f5f --- /dev/null +++ b/study-power-feasibility-assistant/sample-data.js @@ -0,0 +1,98 @@ +const reviewPolicy = { + targetPower: 0.8, + minPowerCoverageRatio: 0.9, + minDetectableEffectCoverage: 0.75, + maxUndisclosedTests: 8, + alphaByFamily: { + "0.05": 1.96, + "0.01": 2.58, + }, + zPowerByTarget: { + "0.8": 0.84, + "0.9": 1.28, + }, +} + +const studyClaims = [ + { + id: "SP-OK-001", + title: "Expected cytokine reduction in randomized pilot cohort", + domain: "molecular-biology", + outcomeType: "continuous", + groups: 2, + sampleSizePerGroup: 96, + expectedEffectSize: 0.62, + observedStandardDeviation: 1.0, + alpha: 0.05, + targetPower: 0.8, + plannedComparisons: 4, + preregistration: { + present: true, + outcomeDeclared: true, + analysisPlanDeclared: true, + }, + manuscriptClaim: "The study is sufficiently powered to detect a moderate cytokine response.", + }, + { + id: "SP-REV-002", + title: "Behavioral intervention claim with thin recruitment", + domain: "clinical-trials", + outcomeType: "continuous", + groups: 2, + sampleSizePerGroup: 80, + expectedEffectSize: 0.35, + observedStandardDeviation: 1.0, + alpha: 0.05, + targetPower: 0.8, + plannedComparisons: 12, + preregistration: { + present: true, + outcomeDeclared: false, + analysisPlanDeclared: false, + }, + manuscriptClaim: "The intervention is powered for clinically meaningful effects across all behavioral endpoints.", + }, + { + id: "SP-HOLD-003", + title: "Rare adverse event safety claim", + domain: "clinical-safety", + outcomeType: "binary", + groups: 2, + sampleSizePerGroup: 60, + baselineRate: 0.03, + expectedRate: 0.018, + alpha: 0.05, + targetPower: 0.9, + plannedComparisons: 3, + preregistration: { + present: false, + outcomeDeclared: false, + analysisPlanDeclared: false, + }, + manuscriptClaim: "The study rules out rare adverse-event differences between arms.", + }, + { + id: "SP-REV-004", + title: "Exploratory single-cell pathway screen", + domain: "single-cell-rna-seq", + outcomeType: "continuous", + groups: 2, + sampleSizePerGroup: 90, + expectedEffectSize: 0.52, + observedStandardDeviation: 1.4, + alpha: 0.01, + targetPower: 0.8, + plannedComparisons: 84, + preregistration: { + present: true, + outcomeDeclared: true, + analysisPlanDeclared: false, + }, + manuscriptClaim: "The pathway screen identifies robust differential signals across all declared modules.", + }, +] + +module.exports = { + reviewPolicy, + studyClaims, +} diff --git a/study-power-feasibility-assistant/test.js b/study-power-feasibility-assistant/test.js new file mode 100644 index 00000000..7d39df4e --- /dev/null +++ b/study-power-feasibility-assistant/test.js @@ -0,0 +1,37 @@ +const assert = require("node:assert/strict") +const { evaluateStudyPortfolio, requiredPerGroup } = require("./index") +const { reviewPolicy, studyClaims } = require("./sample-data") + +const packet = evaluateStudyPortfolio({ studyClaims, reviewPolicy }) + +assert.equal(packet.summary.totalClaims, 4) +assert.equal(packet.summary.passed, 1) +assert.equal(packet.summary.revise, 2) +assert.equal(packet.summary.held, 1) +assert.match(packet.audit.digest, /^[a-f0-9]{64}$/) + +const supported = packet.decisions.find((decision) => decision.id === "SP-OK-001") +assert.equal(supported.status, "pass") +assert.equal(supported.findings.length, 0) +assert.ok(supported.powerAssessment.coverageRatio >= 1) +assert.equal(supported.reviewerActions[0].code, "KEEP_POWER_CLAIM") + +const underpowered = packet.decisions.find((decision) => decision.id === "SP-REV-002") +assert.equal(underpowered.status, "revise") +assert.ok(underpowered.findings.some((finding) => finding.code === "UNDERPOWERED_CLAIM")) +assert.ok(underpowered.findings.some((finding) => finding.code === "MULTIPLE_TESTING_PLAN_MISSING")) +assert.ok(underpowered.reviewerActions.some((action) => action.code === "SOFTEN_OR_REPOWER_CLAIM")) + +const rareEvent = packet.decisions.find((decision) => decision.id === "SP-HOLD-003") +assert.equal(rareEvent.status, "hold") +assert.ok(rareEvent.findings.some((finding) => finding.code === "PREREGISTRATION_MISSING")) +assert.ok(rareEvent.findings.some((finding) => finding.code === "UNDERPOWERED_CLAIM")) +assert.ok(rareEvent.powerAssessment.requiredPerGroup > 1000) + +const singleCell = packet.decisions.find((decision) => decision.id === "SP-REV-004") +assert.equal(singleCell.status, "revise") +assert.ok(singleCell.findings.some((finding) => finding.code === "ANALYSIS_PLAN_NOT_DECLARED")) + +assert.equal(requiredPerGroup(studyClaims[0], reviewPolicy), supported.powerAssessment.requiredPerGroup) + +console.log("study-power-feasibility-assistant tests passed")