diff --git a/review-civility-evidence-gate/README.md b/review-civility-evidence-gate/README.md new file mode 100644 index 0000000..1b07d72 --- /dev/null +++ b/review-civility-evidence-gate/README.md @@ -0,0 +1,36 @@ +# Peer Review Civility And Evidence Gate + +This module is a focused slice for issue #15, Community & User Reputation System. It protects the review and comment layer before content is published to project timelines or allowed to affect researcher reputation. + +It is intentionally not another broad reputation ledger. The gate evaluates structured peer reviews and inline comments for: + +- harassment or personal attacks +- protected or personal identity disclosures +- anonymous, blind, or double-blind reviewer identity leaks +- low structured scores that lack evidence anchors +- missing inline artifact anchors +- repeated comments that should be collapsed before reputation impact + +The output is a deterministic moderation packet with blockers, warnings, redactions, withheld item IDs, and a release recommendation. + +## Run + +```bash +node review-civility-evidence-gate/test.js +node review-civility-evidence-gate/demo.js +``` + +The demo writes: + +- `reports/civility-evidence-packet.json` +- `reports/civility-evidence-report.md` +- `reports/summary.svg` +- `reports/summary.png` +- `reports/demo.mp4` + +## Design Notes + +- Dependency-free Node.js implementation for the core logic. +- Synthetic data only. No credentials or external services. +- Evidence anchors support DOI, URLs, figure/table references, dataset IDs, commits, notebook cells, lines, protocols, sections, and artifacts. +- Reputation impact is withheld while blockers are present. diff --git a/review-civility-evidence-gate/acceptance-notes.md b/review-civility-evidence-gate/acceptance-notes.md new file mode 100644 index 0000000..1a5b4d1 --- /dev/null +++ b/review-civility-evidence-gate/acceptance-notes.md @@ -0,0 +1,25 @@ +# Acceptance Notes + +## What To Review + +- `index.js` implements the peer-review civility and evidence gate. +- `sample-data.js` includes synthetic unsafe and clean packets. +- `test.js` validates blocker, redaction, anonymous leak, clean-pass, and report rendering behavior. +- `demo.js` generates the reviewer packet and visual demo artifact. + +## Verification + +```bash +node review-civility-evidence-gate/test.js +node review-civility-evidence-gate/demo.js +node --check review-civility-evidence-gate/index.js +node --check review-civility-evidence-gate/sample-data.js +node --check review-civility-evidence-gate/test.js +node --check review-civility-evidence-gate/demo.js +git diff --check +ffprobe -v error -select_streams v:0 -show_entries stream=codec_name,width,height,duration -of default=noprint_wrappers=1 review-civility-evidence-gate/reports/demo.mp4 +``` + +## Expected Demo Result + +The sample packet should return `hold_review_publication` with blockers for harassment, unsupported low-score claims, anonymous reviewer identity leakage, and protected identity disclosure. The clean packet should return `community_ready`. diff --git a/review-civility-evidence-gate/demo.js b/review-civility-evidence-gate/demo.js new file mode 100644 index 0000000..45a2c43 --- /dev/null +++ b/review-civility-evidence-gate/demo.js @@ -0,0 +1,52 @@ +const { spawnSync } = require("child_process"); +const fs = require("fs"); +const path = require("path"); +const { evaluateReviewCivility, writeReportBundle } = require("./index"); +const { samplePacket } = require("./sample-data"); + +const moduleDir = __dirname; +const reportsDir = path.join(moduleDir, "reports"); +const result = evaluateReviewCivility(samplePacket); +const paths = writeReportBundle(result, reportsDir); +const pngPath = path.join(reportsDir, "summary.png"); +const mp4Path = path.join(reportsDir, "demo.mp4"); + +function runCommand(command, args) { + const child = spawnSync(command, args, { encoding: "utf8" }); + if (child.status !== 0) { + throw new Error(`${command} failed: ${child.stderr || child.stdout}`); + } + return child; +} + +if (fs.existsSync(paths.svgPath)) { + runCommand("rsvg-convert", ["-w", "1280", "-h", "720", paths.svgPath, "-o", pngPath]); + runCommand("ffmpeg", [ + "-y", + "-loop", + "1", + "-i", + pngPath, + "-t", + "12", + "-vf", + "format=yuv420p", + "-c:v", + "libx264", + "-movflags", + "+faststart", + mp4Path + ]); +} + +console.log( + [ + `status=${result.status}`, + `blockers=${result.summary.blockers}`, + `warnings=${result.summary.warnings}`, + `redactions=${result.summary.redactions}`, + `report=${paths.markdownPath}`, + `packet=${paths.jsonPath}`, + `demo=${mp4Path}` + ].join("\n") +); diff --git a/review-civility-evidence-gate/index.js b/review-civility-evidence-gate/index.js new file mode 100644 index 0000000..0962363 --- /dev/null +++ b/review-civility-evidence-gate/index.js @@ -0,0 +1,382 @@ +const fs = require("fs"); +const path = require("path"); + +const DEFAULT_POLICY = { + severityHoldThreshold: 2, + evidenceRequiredForScoresAtOrBelow: 2, + minimumActionableCharacters: 48, + repeatedCommentLimit: 2, + toxicTerms: [ + "fraud", + "fake scientist", + "fabricator", + "liar", + "stupid", + "idiot", + "incompetent", + "should be fired", + "career is over", + "worthless", + "trash work" + ], + identityDisclosureTerms: [ + "home address", + "phone number", + "personal email", + "real name is", + "legal name", + "visa status", + "medical condition", + "religion", + "caste", + "salary", + "family member" + ], + evidencePattern: + "(doi:|https?://|commit\\s+[0-9a-f]{7,40}|line\\s*[0-9]+|fig\\.\\s*[0-9a-z.-]+|figure\\s*[0-9]+[0-9a-z.-]*|table\\s*[0-9]+[0-9a-z.-]*|dataset\\s*(?:#|:)\\s*[0-9a-z][0-9a-z._-]*|notebook\\s*(?:#|:)\\s*[0-9a-z][0-9a-z._-]*|cell\\s*[a-z]?[0-9]+|protocol\\s*(?:#|:)\\s*[0-9a-z][0-9a-z._-]*|section\\s*[0-9]+[0-9a-z._-]*|artifact\\s*(?:#|:)\\s*[0-9a-z][0-9a-z._-]*)" +}; + +function normalizeText(value) { + return String(value || "") + .replace(/\s+/g, " ") + .trim(); +} + +function lowerText(value) { + return normalizeText(value).toLowerCase(); +} + +function createIssue(kind, severity, itemId, message, evidence = {}) { + return { + kind, + severity, + itemId, + message, + evidence + }; +} + +function containsAny(text, terms) { + const normalized = lowerText(text); + return terms + .filter((term) => normalized.includes(lowerText(term))) + .map((term) => term.toLowerCase()); +} + +function hasEvidenceAnchor(text, policy) { + const pattern = new RegExp(policy.evidencePattern, "i"); + return pattern.test(text || ""); +} + +function reviewerIdentityHints(review) { + const text = lowerText([review.title, review.summary, review.body, review.recommendation].join(" ")); + const hints = []; + const reviewer = review.reviewer || {}; + const candidates = [ + reviewer.displayName, + reviewer.handle, + reviewer.email, + reviewer.institution, + reviewer.orcid + ].filter(Boolean); + + for (const candidate of candidates) { + const token = lowerText(candidate); + if (token.length >= 4 && text.includes(token)) { + hints.push(candidate); + } + } + + if (/\b(i am|this is|as)\s+(dr\.|prof\.|professor|reviewer)\b/i.test(text)) { + hints.push("self-identifying phrase"); + } + + return [...new Set(hints)]; +} + +function scoreEntries(review) { + return Object.entries(review.scores || {}) + .filter(([, value]) => Number.isFinite(Number(value))) + .map(([name, value]) => ({ name, value: Number(value) })); +} + +function evaluateReview(review, policy) { + const id = review.id || "review"; + const text = normalizeText([review.title, review.summary, review.body, review.recommendation].join(" ")); + const issues = []; + const warnings = []; + const redactions = []; + const evidenceAnchored = hasEvidenceAnchor(text, policy); + + const toxicHits = containsAny(text, policy.toxicTerms); + if (toxicHits.length) { + issues.push( + createIssue("harassment_or_personal_attack", "blocker", id, "Review contains language that should be held before publication.", { + terms: toxicHits + }) + ); + } + + const identityHits = containsAny(text, policy.identityDisclosureTerms); + if (identityHits.length) { + issues.push( + createIssue("protected_identity_disclosure", "blocker", id, "Review appears to disclose personal or protected identity information.", { + terms: identityHits + }) + ); + redactions.push({ itemId: id, terms: identityHits }); + } + + if (["anonymous", "blind", "double_blind"].includes(lowerText(review.mode))) { + const hints = reviewerIdentityHints(review); + if (hints.length) { + issues.push( + createIssue("anonymous_reviewer_identity_leak", "blocker", id, "Anonymous review text appears to reveal reviewer identity.", { + hints + }) + ); + redactions.push({ itemId: id, terms: hints }); + } + } + + const severeScores = scoreEntries(review).filter((score) => score.value <= policy.evidenceRequiredForScoresAtOrBelow); + if (severeScores.length && !evidenceAnchored) { + issues.push( + createIssue("unsupported_reputation_damaging_score", "blocker", id, "Low structured scores must cite evidence before affecting public reputation.", { + scores: severeScores + }) + ); + } + + if (text.length < policy.minimumActionableCharacters) { + warnings.push( + createIssue("low_actionability", "warning", id, "Review is too short to carry reputation weight without moderator confirmation.", { + characters: text.length + }) + ); + } + + return { + id, + type: "review", + mode: review.mode || "public", + evidenceAnchored, + issues, + warnings, + redactions, + allowProfileImpact: issues.length === 0 + }; +} + +function evaluateComment(comment, policy, seenTexts) { + const id = comment.id || "comment"; + const text = normalizeText(comment.body); + const issues = []; + const warnings = []; + const redactions = []; + + const toxicHits = containsAny(text, policy.toxicTerms); + if (toxicHits.length) { + issues.push( + createIssue("harassment_or_personal_attack", "blocker", id, "Inline comment contains language that should be held before publication.", { + terms: toxicHits + }) + ); + } + + const identityHits = containsAny(text, policy.identityDisclosureTerms); + if (identityHits.length) { + issues.push( + createIssue("protected_identity_disclosure", "blocker", id, "Inline comment may expose personal or protected identity information.", { + terms: identityHits + }) + ); + redactions.push({ itemId: id, terms: identityHits }); + } + + if (!comment.anchor || !comment.anchor.artifactId) { + warnings.push(createIssue("missing_inline_anchor", "warning", id, "Inline comments need an artifact anchor for project timelines.", {})); + } + + const repeatedCount = (seenTexts.get(lowerText(text)) || 0) + 1; + seenTexts.set(lowerText(text), repeatedCount); + if (repeatedCount > policy.repeatedCommentLimit) { + warnings.push( + createIssue("repeated_comment", "warning", id, "Repeated comments should be collapsed before they influence reputation.", { + repeatedCount + }) + ); + } + + return { + id, + type: "comment", + anchor: comment.anchor || null, + issues, + warnings, + redactions, + allowTimelinePublication: issues.length === 0 + }; +} + +function evaluateReviewCivility(packet, policyOverrides = {}) { + const policy = { ...DEFAULT_POLICY, ...(packet.policy || {}), ...policyOverrides }; + const reviews = Array.isArray(packet.reviews) ? packet.reviews : []; + const comments = Array.isArray(packet.comments) ? packet.comments : []; + const seenCommentTexts = new Map(); + const decisions = [ + ...reviews.map((review) => evaluateReview(review, policy)), + ...comments.map((comment) => evaluateComment(comment, policy, seenCommentTexts)) + ]; + + const blockers = decisions.flatMap((decision) => decision.issues); + const warnings = decisions.flatMap((decision) => decision.warnings); + const redactions = decisions.flatMap((decision) => decision.redactions); + const withheldItems = decisions + .filter((decision) => decision.issues.length) + .map((decision) => decision.id); + + return { + projectId: packet.projectId || "unknown-project", + generatedAt: packet.generatedAt || new Date().toISOString(), + status: blockers.length ? "hold_review_publication" : warnings.length ? "moderation_review_needed" : "community_ready", + summary: { + reviewsChecked: reviews.length, + commentsChecked: comments.length, + blockers: blockers.length, + warnings: warnings.length, + redactions: redactions.length, + withheldItems: withheldItems.length + }, + moderationPacket: { + blockers, + warnings, + redactions, + withheldItems, + recommendation: blockers.length + ? "Hold publication and prevent reputation impact until moderator review." + : warnings.length + ? "Publish only after reviewer confirms anchors and actionability." + : "Safe to publish and apply profile/timeline credit." + }, + decisions + }; +} + +function escapeXml(value) { + return String(value) + .replace(/&/g, "&") + .replace(//g, ">") + .replace(/"/g, """); +} + +function buildMarkdownReport(result) { + const lines = [ + "# Peer Review Civility And Evidence Gate Report", + "", + `Project: ${result.projectId}`, + `Status: ${result.status}`, + `Generated: ${result.generatedAt}`, + "", + "## Summary", + "", + `- Reviews checked: ${result.summary.reviewsChecked}`, + `- Comments checked: ${result.summary.commentsChecked}`, + `- Blockers: ${result.summary.blockers}`, + `- Warnings: ${result.summary.warnings}`, + `- Redactions: ${result.summary.redactions}`, + `- Withheld items: ${result.summary.withheldItems}`, + "", + "## Recommendation", + "", + result.moderationPacket.recommendation, + "", + "## Blockers", + "" + ]; + + if (!result.moderationPacket.blockers.length) { + lines.push("- None"); + } else { + for (const issue of result.moderationPacket.blockers) { + lines.push(`- ${issue.itemId}: ${issue.kind} - ${issue.message}`); + } + } + + lines.push("", "## Warnings", ""); + if (!result.moderationPacket.warnings.length) { + lines.push("- None"); + } else { + for (const issue of result.moderationPacket.warnings) { + lines.push(`- ${issue.itemId}: ${issue.kind} - ${issue.message}`); + } + } + + lines.push("", "## Decision Trace", ""); + for (const decision of result.decisions) { + lines.push( + `- ${decision.id} (${decision.type}): ${decision.issues.length} blocker(s), ${decision.warnings.length} warning(s)` + ); + } + + return `${lines.join("\n")}\n`; +} + +function buildSvgSummary(result) { + const statusColor = result.status === "community_ready" ? "#1f8f5f" : result.status === "moderation_review_needed" ? "#b26b00" : "#b42318"; + const bars = [ + ["Reviews", result.summary.reviewsChecked, "#2563eb"], + ["Comments", result.summary.commentsChecked, "#0f766e"], + ["Blockers", result.summary.blockers, "#b42318"], + ["Warnings", result.summary.warnings, "#b26b00"], + ["Redactions", result.summary.redactions, "#7c3aed"] + ]; + const maxValue = Math.max(1, ...bars.map((bar) => bar[1])); + const barRows = bars + .map(([label, value, color], index) => { + const y = 180 + index * 54; + const width = Math.max(8, Math.round((value / maxValue) * 520)); + return ` + ${escapeXml(label)} + + + ${value} + `; + }) + .join(""); + + return ` + + + + Peer Review Civility And Evidence Gate + Project ${escapeXml(result.projectId)} pre-publication moderation packet + + ${escapeXml(result.status)} + ${barRows} + Release decision + ${escapeXml(result.moderationPacket.recommendation)} + Synthetic data only. No credentials. Deterministic local demo. + +`; +} + +function writeReportBundle(result, outputDir) { + fs.mkdirSync(outputDir, { recursive: true }); + const jsonPath = path.join(outputDir, "civility-evidence-packet.json"); + const markdownPath = path.join(outputDir, "civility-evidence-report.md"); + const svgPath = path.join(outputDir, "summary.svg"); + fs.writeFileSync(jsonPath, `${JSON.stringify(result, null, 2)}\n`); + fs.writeFileSync(markdownPath, buildMarkdownReport(result)); + fs.writeFileSync(svgPath, buildSvgSummary(result)); + return { jsonPath, markdownPath, svgPath }; +} + +module.exports = { + DEFAULT_POLICY, + evaluateReviewCivility, + buildMarkdownReport, + buildSvgSummary, + writeReportBundle +}; diff --git a/review-civility-evidence-gate/reports/civility-evidence-packet.json b/review-civility-evidence-gate/reports/civility-evidence-packet.json new file mode 100644 index 0000000..08577ea --- /dev/null +++ b/review-civility-evidence-gate/reports/civility-evidence-packet.json @@ -0,0 +1,250 @@ +{ + "projectId": "scibase-neuro-open-review-42", + "generatedAt": "2026-05-21T09:00:00.000Z", + "status": "hold_review_publication", + "summary": { + "reviewsChecked": 3, + "commentsChecked": 3, + "blockers": 4, + "warnings": 1, + "redactions": 2, + "withheldItems": 3 + }, + "moderationPacket": { + "blockers": [ + { + "kind": "harassment_or_personal_attack", + "severity": "blocker", + "itemId": "review-toxic-001", + "message": "Review contains language that should be held before publication.", + "evidence": { + "terms": [ + "fraud", + "fake scientist", + "career is over" + ] + } + }, + { + "kind": "unsupported_reputation_damaging_score", + "severity": "blocker", + "itemId": "review-toxic-001", + "message": "Low structured scores must cite evidence before affecting public reputation.", + "evidence": { + "scores": [ + { + "name": "rigor", + "value": 1 + }, + { + "name": "reproducibility", + "value": 1 + } + ] + } + }, + { + "kind": "anonymous_reviewer_identity_leak", + "severity": "blocker", + "itemId": "review-blind-leak-002", + "message": "Anonymous review text appears to reveal reviewer identity.", + "evidence": { + "hints": [ + "Prof. Elena Moor", + "Lakeview Institute" + ] + } + }, + { + "kind": "protected_identity_disclosure", + "severity": "blocker", + "itemId": "comment-private-001", + "message": "Inline comment may expose personal or protected identity information.", + "evidence": { + "terms": [ + "personal email", + "visa status" + ] + } + } + ], + "warnings": [ + { + "kind": "missing_inline_anchor", + "severity": "warning", + "itemId": "comment-anchor-002", + "message": "Inline comments need an artifact anchor for project timelines.", + "evidence": {} + } + ], + "redactions": [ + { + "itemId": "review-blind-leak-002", + "terms": [ + "Prof. Elena Moor", + "Lakeview Institute" + ] + }, + { + "itemId": "comment-private-001", + "terms": [ + "personal email", + "visa status" + ] + } + ], + "withheldItems": [ + "review-toxic-001", + "review-blind-leak-002", + "comment-private-001" + ], + "recommendation": "Hold publication and prevent reputation impact until moderator review." + }, + "decisions": [ + { + "id": "review-toxic-001", + "type": "review", + "mode": "public", + "evidenceAnchored": false, + "issues": [ + { + "kind": "harassment_or_personal_attack", + "severity": "blocker", + "itemId": "review-toxic-001", + "message": "Review contains language that should be held before publication.", + "evidence": { + "terms": [ + "fraud", + "fake scientist", + "career is over" + ] + } + }, + { + "kind": "unsupported_reputation_damaging_score", + "severity": "blocker", + "itemId": "review-toxic-001", + "message": "Low structured scores must cite evidence before affecting public reputation.", + "evidence": { + "scores": [ + { + "name": "rigor", + "value": 1 + }, + { + "name": "reproducibility", + "value": 1 + } + ] + } + } + ], + "warnings": [], + "redactions": [], + "allowProfileImpact": false + }, + { + "id": "review-blind-leak-002", + "type": "review", + "mode": "double_blind", + "evidenceAnchored": true, + "issues": [ + { + "kind": "anonymous_reviewer_identity_leak", + "severity": "blocker", + "itemId": "review-blind-leak-002", + "message": "Anonymous review text appears to reveal reviewer identity.", + "evidence": { + "hints": [ + "Prof. Elena Moor", + "Lakeview Institute" + ] + } + } + ], + "warnings": [], + "redactions": [ + { + "itemId": "review-blind-leak-002", + "terms": [ + "Prof. Elena Moor", + "Lakeview Institute" + ] + } + ], + "allowProfileImpact": false + }, + { + "id": "review-clean-003", + "type": "review", + "mode": "semi_private", + "evidenceAnchored": true, + "issues": [], + "warnings": [], + "redactions": [], + "allowProfileImpact": true + }, + { + "id": "comment-private-001", + "type": "comment", + "anchor": { + "artifactId": "dataset-cleaning-log", + "line": 88 + }, + "issues": [ + { + "kind": "protected_identity_disclosure", + "severity": "blocker", + "itemId": "comment-private-001", + "message": "Inline comment may expose personal or protected identity information.", + "evidence": { + "terms": [ + "personal email", + "visa status" + ] + } + } + ], + "warnings": [], + "redactions": [ + { + "itemId": "comment-private-001", + "terms": [ + "personal email", + "visa status" + ] + } + ], + "allowTimelinePublication": false + }, + { + "id": "comment-anchor-002", + "type": "comment", + "anchor": null, + "issues": [], + "warnings": [ + { + "kind": "missing_inline_anchor", + "severity": "warning", + "itemId": "comment-anchor-002", + "message": "Inline comments need an artifact anchor for project timelines.", + "evidence": {} + } + ], + "redactions": [], + "allowTimelinePublication": true + }, + { + "id": "comment-clean-003", + "type": "comment", + "anchor": { + "artifactId": "analysis-notebook", + "cell": "C12" + }, + "issues": [], + "warnings": [], + "redactions": [], + "allowTimelinePublication": true + } + ] +} diff --git a/review-civility-evidence-gate/reports/civility-evidence-report.md b/review-civility-evidence-gate/reports/civility-evidence-report.md new file mode 100644 index 0000000..055994f --- /dev/null +++ b/review-civility-evidence-gate/reports/civility-evidence-report.md @@ -0,0 +1,38 @@ +# Peer Review Civility And Evidence Gate Report + +Project: scibase-neuro-open-review-42 +Status: hold_review_publication +Generated: 2026-05-21T09:00:00.000Z + +## Summary + +- Reviews checked: 3 +- Comments checked: 3 +- Blockers: 4 +- Warnings: 1 +- Redactions: 2 +- Withheld items: 3 + +## Recommendation + +Hold publication and prevent reputation impact until moderator review. + +## Blockers + +- review-toxic-001: harassment_or_personal_attack - Review contains language that should be held before publication. +- review-toxic-001: unsupported_reputation_damaging_score - Low structured scores must cite evidence before affecting public reputation. +- review-blind-leak-002: anonymous_reviewer_identity_leak - Anonymous review text appears to reveal reviewer identity. +- comment-private-001: protected_identity_disclosure - Inline comment may expose personal or protected identity information. + +## Warnings + +- comment-anchor-002: missing_inline_anchor - Inline comments need an artifact anchor for project timelines. + +## Decision Trace + +- review-toxic-001 (review): 2 blocker(s), 0 warning(s) +- review-blind-leak-002 (review): 1 blocker(s), 0 warning(s) +- review-clean-003 (review): 0 blocker(s), 0 warning(s) +- comment-private-001 (comment): 1 blocker(s), 0 warning(s) +- comment-anchor-002 (comment): 0 blocker(s), 1 warning(s) +- comment-clean-003 (comment): 0 blocker(s), 0 warning(s) diff --git a/review-civility-evidence-gate/reports/demo.mp4 b/review-civility-evidence-gate/reports/demo.mp4 new file mode 100644 index 0000000..30fc5ff Binary files /dev/null and b/review-civility-evidence-gate/reports/demo.mp4 differ diff --git a/review-civility-evidence-gate/reports/summary.png b/review-civility-evidence-gate/reports/summary.png new file mode 100644 index 0000000..d443c6c Binary files /dev/null and b/review-civility-evidence-gate/reports/summary.png differ diff --git a/review-civility-evidence-gate/reports/summary.svg b/review-civility-evidence-gate/reports/summary.svg new file mode 100644 index 0000000..209260a --- /dev/null +++ b/review-civility-evidence-gate/reports/summary.svg @@ -0,0 +1,38 @@ + + + + + Peer Review Civility And Evidence Gate + Project scibase-neuro-open-review-42 pre-publication moderation packet + + hold_review_publication + + Reviews + + + 3 + + Comments + + + 3 + + Blockers + + + 4 + + Warnings + + + 1 + + Redactions + + + 2 + + Release decision + Hold publication and prevent reputation impact until moderator review. + Synthetic data only. No credentials. Deterministic local demo. + diff --git a/review-civility-evidence-gate/requirements-map.md b/review-civility-evidence-gate/requirements-map.md new file mode 100644 index 0000000..dd4a2bd --- /dev/null +++ b/review-civility-evidence-gate/requirements-map.md @@ -0,0 +1,21 @@ +# Requirements Map + +Issue #15 requires a community and reputation layer with peer reviews, comments, contributor credit, reputation scoring, leaderboards, badges, and incentives. This submission covers a distinct safety gate inside the peer-review/comment path so reputation signals are not polluted by unsafe or unsupported content. + +| Requirement | Coverage | +| --- | --- | +| Structured peer reviews | Evaluates review modes, rubric scores, review bodies, and evidence anchors before profile impact. | +| Optional scoring on clarity, rigor, novelty, reproducibility | Blocks low scores when they lack evidence references. | +| Inline commenting | Checks artifact anchors and timeline publication readiness for comments. | +| Public, semi-private, anonymous, blind, double-blind modes | Detects reviewer identity leakage in anonymous/blind/double-blind modes. | +| Review history on profiles/project timelines | Withholds unsafe review/comment IDs from profile and timeline impact. | +| Reputation scoring | Prevents reputation-damaging scores from applying without evidence. | +| Transparent trust decisions | Writes JSON and Markdown moderation packets with blocker and warning reasons. | +| Community safety | Holds harassment, personal attacks, and protected identity disclosures before publication. | + +## Non-Goals + +- This is not a full reputation ledger. +- This is not a badge renewal or leaderboard eligibility module. +- This is not a broad abuse appeals system. +- This is not an endorsement-ring or Sybil detector. diff --git a/review-civility-evidence-gate/sample-data.js b/review-civility-evidence-gate/sample-data.js new file mode 100644 index 0000000..2675dd2 --- /dev/null +++ b/review-civility-evidence-gate/sample-data.js @@ -0,0 +1,128 @@ +const samplePacket = { + projectId: "scibase-neuro-open-review-42", + generatedAt: "2026-05-21T09:00:00.000Z", + reviews: [ + { + id: "review-toxic-001", + mode: "public", + reviewer: { + displayName: "Dr. Asha Raman", + handle: "araman", + institution: "Northbridge Lab" + }, + title: "Major reproducibility concern", + summary: + "The analysis labels the authors as fraud and says their career is over without citing a specific artifact reference.", + body: + "The reproducibility score is 1 because this is fake scientist behavior. This should not affect reputation until a moderator requires evidence.", + scores: { + clarity: 3, + rigor: 1, + novelty: 4, + reproducibility: 1 + } + }, + { + id: "review-blind-leak-002", + mode: "double_blind", + reviewer: { + displayName: "Prof. Elena Moor", + handle: "emoor", + institution: "Lakeview Institute" + }, + title: "Blind review with identity leak", + summary: + "As Prof. Elena Moor from Lakeview Institute, I can confirm the protocol is underpowered.", + body: + "The critique references Figure 2 and notebook cell A14, but the self-identification breaks double-blind review mode.", + scores: { + clarity: 4, + rigor: 2, + novelty: 3, + reproducibility: 2 + } + }, + { + id: "review-clean-003", + mode: "semi_private", + reviewer: { + displayName: "Reviewer 12", + handle: "reviewer12", + institution: "Hidden" + }, + title: "Actionable replication note", + summary: + "Rigor score is lowered because Figure 4 and notebook cell B7 disagree on the normalization constant.", + body: + "The review is specific, evidence-anchored, and avoids personal claims. It should be safe for author-visible publication.", + scores: { + clarity: 4, + rigor: 2, + novelty: 4, + reproducibility: 3 + } + } + ], + comments: [ + { + id: "comment-private-001", + body: + "Please redact the personal email and visa status note before this appears on the project timeline.", + anchor: { + artifactId: "dataset-cleaning-log", + line: 88 + } + }, + { + id: "comment-anchor-002", + body: "Needs more detail on the preprocessing step.", + anchor: null + }, + { + id: "comment-clean-003", + body: "Notebook cell C12 clarifies the missing unit conversion.", + anchor: { + artifactId: "analysis-notebook", + cell: "C12" + } + } + ] +}; + +const cleanPacket = { + projectId: "scibase-clean-review-7", + generatedAt: "2026-05-21T09:05:00.000Z", + reviews: [ + { + id: "review-clean-pass-001", + mode: "public", + reviewer: { + displayName: "Reviewer 8", + handle: "reviewer8" + }, + title: "Specific reproducibility improvement", + summary: + "Table 2 and commit 9f2ab41 show a preprocessing mismatch; scores should be author-visible with no reputation block.", + body: + "The note is evidence-based and avoids personal or protected identity claims.", + scores: { + clarity: 4, + rigor: 3, + novelty: 4, + reproducibility: 3 + } + } + ], + comments: [ + { + id: "comment-clean-pass-001", + body: "Line 43 should cite the calibration script output.", + anchor: { + artifactId: "manuscript", + line: 43 + } + } + ] +}; + +module.exports = { samplePacket, cleanPacket }; diff --git a/review-civility-evidence-gate/test.js b/review-civility-evidence-gate/test.js new file mode 100644 index 0000000..eeb0200 --- /dev/null +++ b/review-civility-evidence-gate/test.js @@ -0,0 +1,71 @@ +const assert = require("assert"); +const fs = require("fs"); +const os = require("os"); +const path = require("path"); +const { + evaluateReviewCivility, + buildMarkdownReport, + buildSvgSummary, + writeReportBundle +} = require("./index"); +const { samplePacket, cleanPacket } = require("./sample-data"); + +function testHoldsHarassmentAndUnsupportedScore() { + const result = evaluateReviewCivility(samplePacket); + assert.strictEqual(result.status, "hold_review_publication"); + assert(result.moderationPacket.blockers.some((issue) => issue.kind === "harassment_or_personal_attack")); + assert(result.moderationPacket.blockers.some((issue) => issue.kind === "unsupported_reputation_damaging_score")); + assert(result.moderationPacket.withheldItems.includes("review-toxic-001")); +} + +function testBlocksAnonymousReviewerLeak() { + const result = evaluateReviewCivility(samplePacket); + const leak = result.moderationPacket.blockers.find( + (issue) => issue.kind === "anonymous_reviewer_identity_leak" && issue.itemId === "review-blind-leak-002" + ); + assert(leak, "expected double-blind identity leak blocker"); + assert(leak.evidence.hints.some((hint) => String(hint).includes("Elena"))); +} + +function testRedactsProtectedIdentityInComments() { + const result = evaluateReviewCivility(samplePacket); + const redaction = result.moderationPacket.redactions.find((entry) => entry.itemId === "comment-private-001"); + assert(redaction, "expected comment redaction"); + assert(redaction.terms.includes("personal email")); + assert(redaction.terms.includes("visa status")); +} + +function testCleanPacketPublishes() { + const result = evaluateReviewCivility(cleanPacket); + assert.strictEqual(result.status, "community_ready"); + assert.strictEqual(result.summary.blockers, 0); + assert.strictEqual(result.summary.warnings, 0); + assert.strictEqual(result.moderationPacket.recommendation, "Safe to publish and apply profile/timeline credit."); +} + +function testReportsRender() { + const result = evaluateReviewCivility(samplePacket); + const markdown = buildMarkdownReport(result); + const svg = buildSvgSummary(result); + assert(markdown.includes("Peer Review Civility And Evidence Gate Report")); + assert(markdown.includes("hold_review_publication")); + assert(svg.includes("