From 60c47f010198e892b6b2a141c791678f01824fe9 Mon Sep 17 00:00:00 2001 From: WenHao Chen <157730893+YakiHugo@users.noreply.github.com> Date: Fri, 24 Apr 2026 06:46:52 -0700 Subject: [PATCH 01/14] ci: parallelize lint/test/typecheck and add build gate - Split the single serial verify job into parallel lint, test, typecheck jobs with build gated behind all three - Add typecheck job (was missing from CI) - Restrict triggers to push on main + PRs targeting main - Add timeout-minutes to prevent hung jobs Co-Authored-By: Claude Opus 4.6 (1M context) --- .github/workflows/ci.yml | 66 ++++++++++++++++++++++++++++++++-------- 1 file changed, 54 insertions(+), 12 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 26c6739b..40286238 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -2,28 +2,70 @@ name: CI on: push: + branches: [main] pull_request: + branches: [main] jobs: - verify: + lint: runs-on: ubuntu-latest + timeout-minutes: 10 steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Setup pnpm - uses: pnpm/action-setup@v4 + - uses: actions/checkout@v4 + - uses: pnpm/action-setup@v4 with: version: 9 + - uses: actions/setup-node@v4 + with: + node-version: 20 + cache: pnpm + - run: pnpm install --frozen-lockfile + - run: pnpm lint - - name: Setup Node.js - uses: actions/setup-node@v4 + test: + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - uses: actions/checkout@v4 + - uses: pnpm/action-setup@v4 + with: + version: 9 + - uses: actions/setup-node@v4 with: node-version: 20 cache: pnpm + - run: pnpm install --frozen-lockfile + - run: pnpm run generate:shaders + - run: pnpm test - - name: Install dependencies - run: pnpm install --frozen-lockfile + typecheck: + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - uses: actions/checkout@v4 + - uses: pnpm/action-setup@v4 + with: + version: 9 + - uses: actions/setup-node@v4 + with: + node-version: 20 + cache: pnpm + - run: pnpm install --frozen-lockfile + - run: pnpm run generate:shaders + - run: pnpm typecheck - - name: Verify - run: pnpm verify + build: + needs: [lint, test, typecheck] + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - uses: actions/checkout@v4 + - uses: pnpm/action-setup@v4 + with: + version: 9 + - uses: actions/setup-node@v4 + with: + node-version: 20 + cache: pnpm + - run: pnpm install --frozen-lockfile + - run: pnpm build From cc6b040184ce92f438b12e10884f95a70bd7d7bc Mon Sep 17 00:00:00 2001 From: WenHao Chen <157730893+YakiHugo@users.noreply.github.com> Date: Sat, 25 Apr 2026 00:20:43 -0700 Subject: [PATCH 02/14] feat(renderer): halftone carrier and channel-drift signal damage families Extend the render pipeline with two new first-class authored families: Carrier transforms: add halftone alongside ASCII. GPU shader supports mono/CMYK/RGB color separation with circle/diamond/line/square dot shapes. Carrier orchestrator now dispatches by transform type. Signal damage: add signalDamage[] as a separate authored family on CanvasImageRenderStateV1. Channel drift shifts RGB channels by independent pixel offsets. Executes as a dedicated pipeline stage between carrier transforms and style effects. Both families include full GPU shaders, pipeline integration, state editing helpers, and UI panels with preview/commit workflow. Co-Authored-By: Claude Opus 4.6 (1M context) --- src/features/canvas/CanvasFloatingPanel.tsx | 10 +- .../canvas/CanvasHalftoneEditPanel.tsx | 332 ++++++++++++++++++ .../canvas/CanvasSignalDamageEditPanel.tsx | 222 ++++++++++++ .../canvas/imageRenderStateEditing.ts | 223 ++++++++++++ src/features/canvas/store/canvasStoreTypes.ts | 2 + src/lib/carrierAdjustments.ts | 39 ++ src/lib/renderer/PipelineRenderer.ts | 149 ++++++++ src/lib/renderer/ProgramRegistry.ts | 8 + src/lib/renderer/gpuHalftoneCarrier.ts | 41 +++ src/lib/renderer/gpuSignalDamage.ts | 39 ++ src/lib/renderer/shaders/ChannelDrift.frag | 26 ++ src/lib/renderer/shaders/HalftoneCarrier.frag | 109 ++++++ src/render/image/asciiEffect.test.ts | 2 +- src/render/image/asciiEffect.ts | 41 ++- src/render/image/halftoneEffect.ts | 65 ++++ src/render/image/index.ts | 1 + src/render/image/renderSingleImage.ts | 15 + src/render/image/signalDamageExecution.ts | 97 +++++ src/render/image/snapshotPlan.test.ts | 4 + src/render/image/snapshotPlan.ts | 6 +- src/render/image/stateCompiler.ts | 1 + src/render/image/types.ts | 68 +++- src/types/index.ts | 27 ++ 23 files changed, 1518 insertions(+), 9 deletions(-) create mode 100644 src/features/canvas/CanvasHalftoneEditPanel.tsx create mode 100644 src/features/canvas/CanvasSignalDamageEditPanel.tsx create mode 100644 src/lib/carrierAdjustments.ts create mode 100644 src/lib/renderer/gpuHalftoneCarrier.ts create mode 100644 src/lib/renderer/gpuSignalDamage.ts create mode 100644 src/lib/renderer/shaders/ChannelDrift.frag create mode 100644 src/lib/renderer/shaders/HalftoneCarrier.frag create mode 100644 src/render/image/halftoneEffect.ts create mode 100644 src/render/image/signalDamageExecution.ts diff --git a/src/features/canvas/CanvasFloatingPanel.tsx b/src/features/canvas/CanvasFloatingPanel.tsx index 9d7a7e08..9d69c080 100644 --- a/src/features/canvas/CanvasFloatingPanel.tsx +++ b/src/features/canvas/CanvasFloatingPanel.tsx @@ -3,6 +3,8 @@ import { AnimatePresence, motion } from "framer-motion"; import { cn } from "@/lib/utils"; import { useCanvasStore, type CanvasFloatingPanel as PanelType } from "@/stores/canvasStore"; import { CanvasAsciiEditPanel } from "./CanvasAsciiEditPanel"; +import { CanvasHalftoneEditPanel } from "./CanvasHalftoneEditPanel"; +import { CanvasSignalDamageEditPanel } from "./CanvasSignalDamageEditPanel"; import { CanvasAssetPicker } from "./CanvasAssetPicker"; import { CanvasEditPanel } from "./CanvasEditPanel"; import { CanvasLayerPanel } from "./CanvasLayerPanel"; @@ -15,6 +17,8 @@ import { const PANEL_TITLES: Record, string> = { edit: "编辑", ascii: "ASCII", + halftone: "Halftone", + "signal-damage": "Signal Damage", layers: "Layers", library: "Library", }; @@ -22,7 +26,7 @@ const PANEL_TITLES: Record, string> = { export function CanvasFloatingPanel() { const activePanel = useCanvasStore((s) => s.activePanel); const setActivePanel = useCanvasStore((s) => s.setActivePanel); - const isEditDock = activePanel === "edit" || activePanel === "ascii"; + const isEditDock = activePanel === "edit" || activePanel === "ascii" || activePanel === "halftone" || activePanel === "signal-damage"; return ( @@ -76,6 +80,10 @@ function PanelContent({ return ; case "ascii": return ; + case "halftone": + return ; + case "signal-damage": + return ; case "layers": return ; case "library": diff --git a/src/features/canvas/CanvasHalftoneEditPanel.tsx b/src/features/canvas/CanvasHalftoneEditPanel.tsx new file mode 100644 index 00000000..6e96206b --- /dev/null +++ b/src/features/canvas/CanvasHalftoneEditPanel.tsx @@ -0,0 +1,332 @@ +import { useCallback, useEffect, useMemo, useRef, useState } from "react"; +import { + Select, + SelectContent, + SelectItem, + SelectTrigger, + SelectValue, +} from "@/components/ui/select"; +import { CanvasEditSection } from "@/features/canvas/components/CanvasEditSection"; +import { SliderControl } from "@/features/canvas/components/controls/SliderControl"; +import { cn } from "@/lib/utils"; +import { + useCanvasElementDraftRenderState, + useCanvasPreviewActions, +} from "@/features/canvas/runtime/canvasRuntimeHooks"; +import { resolveCanvasImageRenderState } from "@/features/canvas/imageRenderState"; +import type { CanvasImageRenderStateV1 } from "@/render/image"; +import type { HalftoneAdjustments } from "@/types"; +import { useCanvasStore } from "@/stores/canvasStore"; +import { + canvasDockBodyTextClassName, + canvasDockSelectContentClassName, + canvasDockSelectTriggerClassName, +} from "./editDockTheme"; +import { + canvasEditTargetEqual, + resolveCanvasEditTargetFromPrimarySelection, + type CanvasImageEditTarget, +} from "./editPanelSelection"; +import { selectLoadedWorkbench } from "./store/canvasStoreSelectors"; +import { + applyHalftoneAdjustmentsToRenderState, + DEFAULT_CANVAS_HALFTONE_ADJUSTMENTS, + getCanvasImageEditValues, +} from "./imageRenderStateEditing"; +import { useCanvasImagePropertyActions } from "./hooks/useCanvasImagePropertyActions"; + +type HalftoneSectionId = "screen" | "appearance"; + +const HALFTONE_SHAPE_OPTIONS: Array<{ + label: string; + value: HalftoneAdjustments["shape"]; +}> = [ + { label: "圆形", value: "circle" }, + { label: "菱形", value: "diamond" }, + { label: "线条", value: "line" }, + { label: "方形", value: "square" }, +]; + +const HALFTONE_COLOR_MODE_OPTIONS: Array<{ + label: string; + value: HalftoneAdjustments["colorMode"]; +}> = [ + { label: "单色", value: "mono" }, + { label: "CMYK", value: "cmyk" }, + { label: "RGB", value: "rgb" }, +]; + +type HalftoneNumericKey = + | "frequency" + | "angle" + | "dotScale" + | "contrast" + | "backgroundOpacity"; + +const formatRatio = (value: number) => value.toFixed(2); + +interface HalftoneSliderDef { + key: HalftoneNumericKey; + label: string; + min: number; + max: number; + step?: number; + format?: (value: number) => string; +} + +const SCREEN_SLIDERS: HalftoneSliderDef[] = [ + { key: "frequency", label: "网点频率", min: 4, max: 80, step: 1 }, + { key: "angle", label: "屏幕角度", min: 0, max: 360, step: 1 }, + { key: "dotScale", label: "点大小", min: 0.5, max: 2, step: 0.05, format: formatRatio }, + { key: "contrast", label: "对比度", min: 0.5, max: 3, step: 0.05, format: formatRatio }, +]; + +const APPEARANCE_SLIDERS: HalftoneSliderDef[] = [ + { key: "backgroundOpacity", label: "背景不透明度", min: 0, max: 1, step: 0.01, format: formatRatio }, +]; + +function useCanvasEditImageTarget(): CanvasImageEditTarget | null { + const primarySelectedElementId = useCanvasStore( + (state) => state.selectedElementIds[0] ?? null + ); + const selectEditTarget = useCallback( + (state: Parameters[0]) => { + const target = resolveCanvasEditTargetFromPrimarySelection( + selectLoadedWorkbench(state), + primarySelectedElementId + ); + return target?.type === "image" ? target : null; + }, + [primarySelectedElementId] + ); + return useCanvasStore(selectEditTarget, canvasEditTargetEqual); +} + +export function CanvasHalftoneEditPanel() { + const imageElement = useCanvasEditImageTarget(); + + if (!imageElement) { + return ( +
+
+

+ 在画布上选择一张图片后，即可调整半色调效果。 +

+
+
+ ); + } + + return ; +} + +function CanvasHalftoneEditPanelForImage({ + imageElement, +}: { + imageElement: CanvasImageEditTarget; +}) { + const { + clearElementDraftRenderState, + requestBoardPreview, + setElementDraftRenderState, + } = useCanvasPreviewActions(); + const { setRenderState } = useCanvasImagePropertyActions(imageElement); + const [openSections, setOpenSections] = useState>(() => ({ + screen: true, + appearance: true, + })); + + const committedImageElementId = imageElement.id; + const committedImageElementIdRef = useRef(committedImageElementId); + const draftRenderState = useCanvasElementDraftRenderState(committedImageElementId); + + const renderState = useMemo( + () => resolveCanvasImageRenderState(imageElement, draftRenderState), + [draftRenderState, imageElement] + ); + const fieldValues = useMemo(() => getCanvasImageEditValues(renderState), [renderState]); + const renderStateRef = useRef(renderState); + renderStateRef.current = renderState; + + useEffect(() => { + const previous = committedImageElementIdRef.current; + if (previous && previous !== committedImageElementId) { + clearElementDraftRenderState(previous); + } + committedImageElementIdRef.current = committedImageElementId; + }, [clearElementDraftRenderState, committedImageElementId]); + + useEffect( + () => () => { + const current = committedImageElementIdRef.current; + if (current) { + clearElementDraftRenderState(current); + } + }, + [clearElementDraftRenderState] + ); + + const previewRenderState = useCallback( + (nextRenderState: CanvasImageRenderStateV1) => { + setElementDraftRenderState(imageElement.id, nextRenderState); + void requestBoardPreview(imageElement.id, "interactive"); + }, + [imageElement.id, requestBoardPreview, setElementDraftRenderState] + ); + + const commitAdjustments = useCallback( + async (nextRenderState: CanvasImageRenderStateV1) => { + setElementDraftRenderState(imageElement.id, nextRenderState); + await setRenderState(nextRenderState); + clearElementDraftRenderState(imageElement.id); + }, + [ + clearElementDraftRenderState, + imageElement.id, + setElementDraftRenderState, + setRenderState, + ] + ); + + const toggleSection = useCallback((sectionId: HalftoneSectionId) => { + setOpenSections((current) => ({ + ...current, + [sectionId]: !current[sectionId], + })); + }, []); + + const halftoneAdjustments = fieldValues.halftone ?? DEFAULT_CANVAS_HALFTONE_ADJUSTMENTS; + const halftoneEnabled = halftoneAdjustments.enabled; + + const updateHalftoneAdjustments = useCallback( + (partial: Partial, mode: "preview" | "commit" = "commit") => { + const nextRenderState = applyHalftoneAdjustmentsToRenderState( + renderStateRef.current!, + partial + ); + if (mode === "preview") { + previewRenderState(nextRenderState); + } else { + void commitAdjustments(nextRenderState); + } + }, + [commitAdjustments, previewRenderState] + ); + + const renderSlider = (slider: HalftoneSliderDef) => ( + updateHalftoneAdjustments({ [slider.key]: value }, "preview")} + onCommit={(value: number) => updateHalftoneAdjustments({ [slider.key]: value })} + /> + ); + + return ( +
+
+ toggleSection("screen")} + canResetChanges + onResetChanges={() => updateHalftoneAdjustments({ ...DEFAULT_CANVAS_HALFTONE_ADJUSTMENTS })} + > +
+
+ + +
+
+ 形状 + +
+
+ 色彩模式 + +
+ {SCREEN_SLIDERS.map(renderSlider)} +
+ + + toggleSection("appearance")} + > +
+ {APPEARANCE_SLIDERS.map(renderSlider)} +
+ +
+
+ ); +} diff --git a/src/features/canvas/CanvasSignalDamageEditPanel.tsx b/src/features/canvas/CanvasSignalDamageEditPanel.tsx new file mode 100644 index 00000000..0e92e981 --- /dev/null +++ b/src/features/canvas/CanvasSignalDamageEditPanel.tsx @@ -0,0 +1,222 @@ +import { useCallback, useEffect, useMemo, useRef } from "react"; +import { CanvasEditSection } from "@/features/canvas/components/CanvasEditSection"; +import { SliderControl } from "@/features/canvas/components/controls/SliderControl"; +import { cn } from "@/lib/utils"; +import { + useCanvasElementDraftRenderState, + useCanvasPreviewActions, +} from "@/features/canvas/runtime/canvasRuntimeHooks"; +import { resolveCanvasImageRenderState } from "@/features/canvas/imageRenderState"; +import type { CanvasImageRenderStateV1 } from "@/render/image"; +import type { ChannelDriftAdjustments } from "@/types"; +import { useCanvasStore } from "@/stores/canvasStore"; +import { canvasDockBodyTextClassName } from "./editDockTheme"; +import { + canvasEditTargetEqual, + resolveCanvasEditTargetFromPrimarySelection, + type CanvasImageEditTarget, +} from "./editPanelSelection"; +import { selectLoadedWorkbench } from "./store/canvasStoreSelectors"; +import { + applyChannelDriftAdjustmentsToRenderState, + DEFAULT_CANVAS_CHANNEL_DRIFT_ADJUSTMENTS, + getCanvasImageEditValues, +} from "./imageRenderStateEditing"; +import { useCanvasImagePropertyActions } from "./hooks/useCanvasImagePropertyActions"; + +type ChannelDriftNumericKey = + | "redOffsetX" + | "redOffsetY" + | "greenOffsetX" + | "greenOffsetY" + | "blueOffsetX" + | "blueOffsetY" + | "intensity"; + +const formatPx = (value: number) => `${value.toFixed(1)}px`; +const formatPercent = (value: number) => `${Math.round(value * 100)}%`; + +interface ChannelDriftSliderDef { + key: ChannelDriftNumericKey; + label: string; + min: number; + max: number; + step?: number; + format?: (value: number) => string; +} + +const CHANNEL_SLIDERS: ChannelDriftSliderDef[] = [ + { key: "redOffsetX", label: "Red X", min: -100, max: 100, step: 1, format: formatPx }, + { key: "redOffsetY", label: "Red Y", min: -100, max: 100, step: 1, format: formatPx }, + { key: "greenOffsetX", label: "Green X", min: -100, max: 100, step: 1, format: formatPx }, + { key: "greenOffsetY", label: "Green Y", min: -100, max: 100, step: 1, format: formatPx }, + { key: "blueOffsetX", label: "Blue X", min: -100, max: 100, step: 1, format: formatPx }, + { key: "blueOffsetY", label: "Blue Y", min: -100, max: 100, step: 1, format: formatPx }, + { key: "intensity", label: "强度", min: 0, max: 1, step: 0.01, format: formatPercent }, +]; + +function useCanvasEditImageTarget(): CanvasImageEditTarget | null { + const primarySelectedElementId = useCanvasStore( + (state) => state.selectedElementIds[0] ?? null + ); + const selectEditTarget = useCallback( + (state: Parameters[0]) => { + const target = resolveCanvasEditTargetFromPrimarySelection( + selectLoadedWorkbench(state), + primarySelectedElementId + ); + return target?.type === "image" ? target : null; + }, + [primarySelectedElementId] + ); + return useCanvasStore(selectEditTarget, canvasEditTargetEqual); +} + +export function CanvasSignalDamageEditPanel() { + const imageElement = useCanvasEditImageTarget(); + + if (!imageElement) { + return ( +
+
+

+ 在画布上选择一张图片后，即可调整信号损伤效果。 +

+
+
+ ); + } + + return ; +} + +function CanvasSignalDamageEditPanelForImage({ + imageElement, +}: { + imageElement: CanvasImageEditTarget; +}) { + const { + clearElementDraftRenderState, + requestBoardPreview, + setElementDraftRenderState, + } = useCanvasPreviewActions(); + const { setRenderState } = useCanvasImagePropertyActions(imageElement); + + const committedImageElementId = imageElement.id; + const committedImageElementIdRef = useRef(committedImageElementId); + const draftRenderState = useCanvasElementDraftRenderState(committedImageElementId); + + const renderState = useMemo( + () => resolveCanvasImageRenderState(imageElement, draftRenderState), + [draftRenderState, imageElement] + ); + const fieldValues = useMemo(() => getCanvasImageEditValues(renderState), [renderState]); + const renderStateRef = useRef(renderState); + renderStateRef.current = renderState; + + useEffect(() => { + const previous = committedImageElementIdRef.current; + if (previous && previous !== committedImageElementId) { + clearElementDraftRenderState(previous); + } + committedImageElementIdRef.current = committedImageElementId; + }, [clearElementDraftRenderState, committedImageElementId]); + + useEffect( + () => () => { + const current = committedImageElementIdRef.current; + if (current) { + clearElementDraftRenderState(current); + } + }, + [clearElementDraftRenderState] + ); + + const previewRenderState = useCallback( + (nextRenderState: CanvasImageRenderStateV1) => { + setElementDraftRenderState(imageElement.id, nextRenderState); + void requestBoardPreview(imageElement.id, "interactive"); + }, + [imageElement.id, requestBoardPreview, setElementDraftRenderState] + ); + + const commitAdjustments = useCallback( + async (nextRenderState: CanvasImageRenderStateV1) => { + setElementDraftRenderState(imageElement.id, nextRenderState); + await setRenderState(nextRenderState); + clearElementDraftRenderState(imageElement.id); + }, + [ + clearElementDraftRenderState, + imageElement.id, + setElementDraftRenderState, + setRenderState, + ] + ); + + const channelDrift = fieldValues.channelDrift ?? DEFAULT_CANVAS_CHANNEL_DRIFT_ADJUSTMENTS; + const driftEnabled = channelDrift.enabled; + + const updateChannelDrift = useCallback( + (partial: Partial, mode: "preview" | "commit" = "commit") => { + const nextRenderState = applyChannelDriftAdjustmentsToRenderState( + renderStateRef.current!, + partial + ); + if (mode === "preview") { + previewRenderState(nextRenderState); + } else { + void commitAdjustments(nextRenderState); + } + }, + [commitAdjustments, previewRenderState] + ); + + const renderSlider = (slider: ChannelDriftSliderDef) => ( + updateChannelDrift({ [slider.key]: value }, "preview")} + onCommit={(value: number) => updateChannelDrift({ [slider.key]: value })} + /> + ); + + return ( +
+
+ {}} + canResetChanges + onResetChanges={() => updateChannelDrift({ ...DEFAULT_CANVAS_CHANNEL_DRIFT_ADJUSTMENTS })} + > +
+ + {CHANNEL_SLIDERS.map(renderSlider)} +
+ +
+
+ ); +} diff --git a/src/features/canvas/imageRenderStateEditing.ts b/src/features/canvas/imageRenderStateEditing.ts index e3e62bd6..2a49817a 100644 --- a/src/features/canvas/imageRenderStateEditing.ts +++ b/src/features/canvas/imageRenderStateEditing.ts @@ -7,6 +7,10 @@ import type { AsciiDitherMode, AsciiForegroundBlendMode, AsciiRenderMode, + ChannelDriftAdjustments, + HalftoneAdjustments, + HalftoneColorMode, + HalftoneShape, } from "@/types"; import { createNeutralCanvasImageRenderState, @@ -14,6 +18,7 @@ import { normalizeCanvasImageRenderState, type CarrierTransformNode, type ImageFilter2dEffectNode, + type SignalDamageNode, } from "@/render/image"; const CHARSET_PRESET_VALUES = ["standard", "minimal", "blocks", "detailed", "custom"] as const; @@ -42,6 +47,38 @@ const isBackgroundMode = (value: unknown): value is AsciiBackgroundMode => const isForegroundBlendMode = (value: unknown): value is AsciiForegroundBlendMode => typeof value === "string" && (FOREGROUND_BLEND_VALUES as readonly string[]).includes(value); +const HALFTONE_SHAPE_VALUES = ["circle", "diamond", "line", "square"] as const; +const HALFTONE_COLOR_MODE_VALUES = ["mono", "cmyk", "rgb"] as const; + +const isHalftoneShape = (value: unknown): value is HalftoneShape => + typeof value === "string" && (HALFTONE_SHAPE_VALUES as readonly string[]).includes(value); +const isHalftoneColorMode = (value: unknown): value is HalftoneColorMode => + typeof value === "string" && (HALFTONE_COLOR_MODE_VALUES as readonly string[]).includes(value); + +const DEFAULT_HALFTONE_ADJUSTMENTS: HalftoneAdjustments = { + enabled: false, + frequency: 30, + angle: 45, + shape: "circle", + colorMode: "mono", + dotScale: 1, + contrast: 1, + invert: false, + backgroundColor: "#000000", + backgroundOpacity: 1, +}; + +const DEFAULT_CHANNEL_DRIFT_ADJUSTMENTS: ChannelDriftAdjustments = { + enabled: false, + redOffsetX: 5, + redOffsetY: 0, + greenOffsetX: -3, + greenOffsetY: 2, + blueOffsetX: 0, + blueOffsetY: -4, + intensity: 0.5, +}; + const DEFAULT_ASCII_ADJUSTMENTS: AsciiAdjustments = { enabled: false, // Defaults are tuned to match ascii-magic.com's out-of-the-box look, which @@ -84,6 +121,8 @@ export type CanvasImageNumericFieldValues = Record { @@ -139,6 +178,59 @@ const resolveAsciiAdjustmentsFromState = (state: CanvasImageRenderStateV1): Asci }; }; +const resolveHalftoneAdjustmentsFromState = ( + state: CanvasImageRenderStateV1 +): HalftoneAdjustments => { + const carrier = normalizeCanvasImageRenderState(state).carrierTransforms.find( + (candidate): candidate is Extract => + candidate.type === "halftone" && candidate.enabled + ); + if (!carrier) { + return { ...DEFAULT_HALFTONE_ADJUSTMENTS }; + } + const p = carrier.params; + return { + ...DEFAULT_HALFTONE_ADJUSTMENTS, + enabled: true, + frequency: typeof p.frequency === "number" ? p.frequency : 30, + angle: typeof p.angle === "number" ? p.angle : 45, + shape: isHalftoneShape(p.shape) ? p.shape : "circle", + colorMode: isHalftoneColorMode(p.colorMode) ? p.colorMode : "mono", + dotScale: typeof p.dotScale === "number" ? p.dotScale : 1, + contrast: typeof p.contrast === "number" ? p.contrast : 1, + invert: Boolean(p.invert), + backgroundColor: + typeof p.backgroundColor === "string" && p.backgroundColor + ? p.backgroundColor + : "#000000", + backgroundOpacity: typeof p.backgroundOpacity === "number" ? p.backgroundOpacity : 1, + }; +}; + +const resolveChannelDriftAdjustmentsFromState = ( + state: CanvasImageRenderStateV1 +): ChannelDriftAdjustments => { + const node = normalizeCanvasImageRenderState(state).signalDamage.find( + (candidate): candidate is Extract => + candidate.type === "channel-drift" && candidate.enabled + ); + if (!node) { + return { ...DEFAULT_CHANNEL_DRIFT_ADJUSTMENTS }; + } + const p = node.params; + return { + ...DEFAULT_CHANNEL_DRIFT_ADJUSTMENTS, + enabled: true, + redOffsetX: typeof p.redOffsetX === "number" ? p.redOffsetX : 5, + redOffsetY: typeof p.redOffsetY === "number" ? p.redOffsetY : 0, + greenOffsetX: typeof p.greenOffsetX === "number" ? p.greenOffsetX : -3, + greenOffsetY: typeof p.greenOffsetY === "number" ? p.greenOffsetY : 2, + blueOffsetX: typeof p.blueOffsetX === "number" ? p.blueOffsetX : 0, + blueOffsetY: typeof p.blueOffsetY === "number" ? p.blueOffsetY : -4, + intensity: typeof p.intensity === "number" ? p.intensity : 0.5, + }; +}; + const resolveFilter2dPreviewValues = (state: CanvasImageRenderStateV1) => { const effect = state.effects.find( (candidate): candidate is ImageFilter2dEffectNode => @@ -190,6 +282,8 @@ const createCanvasImageEditValues = ( blur: filter2d.blur, dilate: filter2d.dilate, ascii: resolveAsciiAdjustmentsFromState(normalizedState), + halftone: resolveHalftoneAdjustmentsFromState(normalizedState), + channelDrift: resolveChannelDriftAdjustmentsFromState(normalizedState), }; }; @@ -202,6 +296,14 @@ export const DEFAULT_CANVAS_ASCII_ADJUSTMENTS: AsciiAdjustments = { ...DEFAULT_CANVAS_IMAGE_EDIT_VALUES.ascii, }; +export const DEFAULT_CANVAS_HALFTONE_ADJUSTMENTS: HalftoneAdjustments = { + ...DEFAULT_CANVAS_IMAGE_EDIT_VALUES.halftone, +}; + +export const DEFAULT_CANVAS_CHANNEL_DRIFT_ADJUSTMENTS: ChannelDriftAdjustments = { + ...DEFAULT_CANVAS_IMAGE_EDIT_VALUES.channelDrift, +}; + const createDefaultAsciiCarrierTransform = (): Extract => ({ id: "canvas-ascii", type: "ascii", @@ -231,6 +333,45 @@ const createDefaultAsciiCarrierTransform = (): Extract => ({ + id: "canvas-halftone", + type: "halftone", + enabled: false, + analysisSource: "style", + params: { + frequency: DEFAULT_CANVAS_HALFTONE_ADJUSTMENTS.frequency, + angle: DEFAULT_CANVAS_HALFTONE_ADJUSTMENTS.angle, + shape: DEFAULT_CANVAS_HALFTONE_ADJUSTMENTS.shape, + colorMode: DEFAULT_CANVAS_HALFTONE_ADJUSTMENTS.colorMode, + dotScale: DEFAULT_CANVAS_HALFTONE_ADJUSTMENTS.dotScale, + contrast: DEFAULT_CANVAS_HALFTONE_ADJUSTMENTS.contrast, + invert: DEFAULT_CANVAS_HALFTONE_ADJUSTMENTS.invert, + backgroundColor: DEFAULT_CANVAS_HALFTONE_ADJUSTMENTS.backgroundColor, + backgroundOpacity: DEFAULT_CANVAS_HALFTONE_ADJUSTMENTS.backgroundOpacity, + }, +}); + +const createDefaultChannelDriftDamage = (): Extract< + SignalDamageNode, + { type: "channel-drift" } +> => ({ + id: "canvas-channel-drift", + type: "channel-drift", + enabled: false, + params: { + redOffsetX: DEFAULT_CANVAS_CHANNEL_DRIFT_ADJUSTMENTS.redOffsetX, + redOffsetY: DEFAULT_CANVAS_CHANNEL_DRIFT_ADJUSTMENTS.redOffsetY, + greenOffsetX: DEFAULT_CANVAS_CHANNEL_DRIFT_ADJUSTMENTS.greenOffsetX, + greenOffsetY: DEFAULT_CANVAS_CHANNEL_DRIFT_ADJUSTMENTS.greenOffsetY, + blueOffsetX: DEFAULT_CANVAS_CHANNEL_DRIFT_ADJUSTMENTS.blueOffsetX, + blueOffsetY: DEFAULT_CANVAS_CHANNEL_DRIFT_ADJUSTMENTS.blueOffsetY, + intensity: DEFAULT_CANVAS_CHANNEL_DRIFT_ADJUSTMENTS.intensity, + }, +}); + const createDefaultFilter2dEffect = (): ImageFilter2dEffectNode => ({ id: "canvas-filter2d", type: "filter2d", @@ -265,6 +406,48 @@ const upsertAsciiCarrierTransform = ( return next; }; +const upsertHalftoneCarrierTransform = ( + state: CanvasImageRenderStateV1, + updater: ( + transform: Extract + ) => Extract +) => { + const next = cloneState(state); + const index = next.carrierTransforms.findIndex((t) => t.type === "halftone"); + const current = + index >= 0 + ? (next.carrierTransforms[index] as Extract) + : createDefaultHalftoneCarrierTransform(); + const updated = updater(current); + if (index >= 0) { + next.carrierTransforms[index] = updated; + } else { + next.carrierTransforms.push(updated); + } + return next; +}; + +const upsertChannelDriftDamage = ( + state: CanvasImageRenderStateV1, + updater: ( + node: Extract + ) => Extract +) => { + const next = cloneState(state); + const index = next.signalDamage.findIndex((n) => n.type === "channel-drift"); + const current = + index >= 0 + ? (next.signalDamage[index] as Extract) + : createDefaultChannelDriftDamage(); + const updated = updater(current); + if (index >= 0) { + next.signalDamage[index] = updated; + } else { + next.signalDamage.push(updated); + } + return next; +}; + const upsertFilter2dEffect = ( state: CanvasImageRenderStateV1, updater: (effect: ImageFilter2dEffectNode) => ImageFilter2dEffectNode @@ -423,6 +606,46 @@ export const applyAsciiAdjustmentsToRenderState = ( }, })); +export const applyHalftoneAdjustmentsToRenderState = ( + state: CanvasImageRenderStateV1, + partial: Partial +) => + upsertHalftoneCarrierTransform(state, (transform) => ({ + ...transform, + enabled: partial.enabled ?? transform.enabled, + params: { + ...transform.params, + frequency: partial.frequency ?? transform.params.frequency, + angle: partial.angle ?? transform.params.angle, + shape: partial.shape ?? transform.params.shape, + colorMode: partial.colorMode ?? transform.params.colorMode, + dotScale: partial.dotScale ?? transform.params.dotScale, + contrast: partial.contrast ?? transform.params.contrast, + invert: partial.invert ?? transform.params.invert, + backgroundColor: partial.backgroundColor ?? transform.params.backgroundColor, + backgroundOpacity: partial.backgroundOpacity ?? transform.params.backgroundOpacity, + }, + })); + +export const applyChannelDriftAdjustmentsToRenderState = ( + state: CanvasImageRenderStateV1, + partial: Partial +) => + upsertChannelDriftDamage(state, (node) => ({ + ...node, + enabled: partial.enabled ?? node.enabled, + params: { + ...node.params, + redOffsetX: partial.redOffsetX ?? node.params.redOffsetX, + redOffsetY: partial.redOffsetY ?? node.params.redOffsetY, + greenOffsetX: partial.greenOffsetX ?? node.params.greenOffsetX, + greenOffsetY: partial.greenOffsetY ?? node.params.greenOffsetY, + blueOffsetX: partial.blueOffsetX ?? node.params.blueOffsetX, + blueOffsetY: partial.blueOffsetY ?? node.params.blueOffsetY, + intensity: partial.intensity ?? node.params.intensity, + }, + })); + export const resetRenderStateForNumericFields = ( state: CanvasImageRenderStateV1, fieldIds: CanvasImageNumericFieldId[] diff --git a/src/features/canvas/store/canvasStoreTypes.ts b/src/features/canvas/store/canvasStoreTypes.ts index 6c04ee23..d08da7f3 100644 --- a/src/features/canvas/store/canvasStoreTypes.ts +++ b/src/features/canvas/store/canvasStoreTypes.ts @@ -11,6 +11,8 @@ export type CanvasTool = "select" | "text" | "hand" | "shape"; export type CanvasFloatingPanel = | "edit" | "ascii" + | "halftone" + | "signal-damage" | "layers" | "library" | null; diff --git a/src/lib/carrierAdjustments.ts b/src/lib/carrierAdjustments.ts new file mode 100644 index 00000000..e5e2ab72 --- /dev/null +++ b/src/lib/carrierAdjustments.ts @@ -0,0 +1,39 @@ +import type { ChannelDriftAdjustments, HalftoneAdjustments } from "@/types"; + +export const halftoneAdjustmentsEqual = ( + left: HalftoneAdjustments | undefined, + right: HalftoneAdjustments | undefined +) => { + if (left === right) return true; + if (!left || !right) return false; + return ( + left.enabled === right.enabled && + left.frequency === right.frequency && + left.angle === right.angle && + left.shape === right.shape && + left.colorMode === right.colorMode && + left.dotScale === right.dotScale && + left.contrast === right.contrast && + left.invert === right.invert && + left.backgroundColor === right.backgroundColor && + left.backgroundOpacity === right.backgroundOpacity + ); +}; + +export const channelDriftAdjustmentsEqual = ( + left: ChannelDriftAdjustments | undefined, + right: ChannelDriftAdjustments | undefined +) => { + if (left === right) return true; + if (!left || !right) return false; + return ( + left.enabled === right.enabled && + left.redOffsetX === right.redOffsetX && + left.redOffsetY === right.redOffsetY && + left.greenOffsetX === right.greenOffsetX && + left.greenOffsetY === right.greenOffsetY && + left.blueOffsetX === right.blueOffsetX && + left.blueOffsetY === right.blueOffsetY && + left.intensity === right.intensity + ); +}; diff --git a/src/lib/renderer/PipelineRenderer.ts b/src/lib/renderer/PipelineRenderer.ts index a716cf31..29dc0a0a 100644 --- a/src/lib/renderer/PipelineRenderer.ts +++ b/src/lib/renderer/PipelineRenderer.ts @@ -48,6 +48,8 @@ import { import type { EditorLayerBlendMode, EditorLayerMask } from "@/types"; import type { LocalAdjustmentMask } from "@/types"; import type { AsciiCarrierGpuInput } from "./gpuAsciiCarrier"; +import type { HalftoneCarrierGpuInput } from "./gpuHalftoneCarrier"; +import type { ChannelDriftGpuInput } from "./gpuSignalDamage"; import type { CurveUniforms, DetailUniforms, @@ -1803,6 +1805,153 @@ export class PipelineRenderer { } } + renderHalftoneCarrierComposite(options: { + baseCanvas: HTMLCanvasElement; + carrier: HalftoneCarrierGpuInput; + }): boolean { + if (this.destroyed || this.contextLost) { + return false; + } + + try { + const source = this.captureLinearSource( + options.baseCanvas, + options.baseCanvas.width, + options.baseCanvas.height, + options.baseCanvas.width, + options.baseCanvas.height, + { decodeSrgb: false } + ); + + try { + const shapeIndex = + options.carrier.shape === "diamond" ? 1 : + options.carrier.shape === "line" ? 2 : + options.carrier.shape === "square" ? 3 : 0; + const colorModeIndex = + options.carrier.colorMode === "cmyk" ? 1 : + options.carrier.colorMode === "rgb" ? 2 : 0; + + const result = this.filterPipeline.runToTexture({ + baseWidth: options.carrier.width, + baseHeight: options.carrier.height, + passes: [ + { + id: "halftone-carrier", + programInfo: this.programs.halftoneCarrier, + uniforms: { + u_canvasSize: new Float32Array([options.carrier.width, options.carrier.height]), + u_frequency: options.carrier.frequency, + u_angle: options.carrier.angle, + u_shape: shapeIndex, + u_colorMode: colorModeIndex, + u_dotScale: options.carrier.dotScale, + u_contrast: options.carrier.contrast, + u_invert: options.carrier.invert, + u_backgroundColor: options.carrier.backgroundColorRgba, + u_backgroundOpacity: options.carrier.backgroundOpacity, + }, + outputFormat: "RGBA8", + enabled: true, + }, + ], + input: { + texture: source.texture, + width: source.width, + height: source.height, + format: source.format, + }, + }); + + this.presentTextureResult(result, { + inputLinear: false, + enableDither: false, + }); + result.release(); + return true; + } finally { + source.release(); + } + } catch (error) { + if (!this.contextLost) { + reportGlError({ + op: "drawArrays", + passId: "halftone-carrier-composite", + rendererLabel: this.rendererLabel, + cause: error, + }); + } + return false; + } + } + + renderChannelDriftComposite(options: { + baseCanvas: HTMLCanvasElement; + damage: ChannelDriftGpuInput; + }): boolean { + if (this.destroyed || this.contextLost) { + return false; + } + + try { + const source = this.captureLinearSource( + options.baseCanvas, + options.baseCanvas.width, + options.baseCanvas.height, + options.baseCanvas.width, + options.baseCanvas.height, + { decodeSrgb: false } + ); + + try { + const result = this.filterPipeline.runToTexture({ + baseWidth: options.damage.width, + baseHeight: options.damage.height, + passes: [ + { + id: "channel-drift", + programInfo: this.programs.channelDrift, + uniforms: { + u_canvasSize: new Float32Array([options.damage.width, options.damage.height]), + u_redOffset: new Float32Array([options.damage.redOffsetX, options.damage.redOffsetY]), + u_greenOffset: new Float32Array([options.damage.greenOffsetX, options.damage.greenOffsetY]), + u_blueOffset: new Float32Array([options.damage.blueOffsetX, options.damage.blueOffsetY]), + u_intensity: options.damage.intensity, + }, + outputFormat: "RGBA8", + enabled: true, + }, + ], + input: { + texture: source.texture, + width: source.width, + height: source.height, + format: source.format, + }, + }); + + this.presentTextureResult(result, { + inputLinear: false, + enableDither: false, + }); + result.release(); + return true; + } finally { + source.release(); + } + } catch (error) { + if (!this.contextLost) { + reportGlError({ + op: "drawArrays", + passId: "channel-drift-composite", + rendererLabel: this.rendererLabel, + cause: error, + }); + } + return false; + } + } + private renderTimestampOverlayLayer(overlay: TimestampOverlayGpuInput): LinearRenderResult | null { if (overlay.charCount <= 0) { return null; diff --git a/src/lib/renderer/ProgramRegistry.ts b/src/lib/renderer/ProgramRegistry.ts index 2563e4a9..a64ee963 100644 --- a/src/lib/renderer/ProgramRegistry.ts +++ b/src/lib/renderer/ProgramRegistry.ts @@ -35,6 +35,8 @@ import brushMaskStampFragSrc from "./shaders/BrushMaskStamp.frag?raw"; import maskInvertFragSrc from "./shaders/MaskInvert.frag?raw"; import asciiCarrierFragRaw from "./shaders/AsciiCarrier.frag?raw"; import asciiCommonGlsl from "./shaders/templates/asciiCommon.glsl?raw"; +import halftoneCarrierFragSrc from "./shaders/HalftoneCarrier.frag?raw"; +import channelDriftFragSrc from "./shaders/ChannelDrift.frag?raw"; import timestampOverlayFragSrc from "./shaders/TimestampOverlay.frag?raw"; const ASCII_COMMON_MARKER = "// #ASCII_COMMON#"; @@ -73,6 +75,8 @@ export interface RendererPrograms { brushMaskStamp: ProgramInfo; maskInvert: ProgramInfo; asciiCarrier: ProgramInfo; + halftoneCarrier: ProgramInfo; + channelDrift: ProgramInfo; timestampOverlay: ProgramInfo; } @@ -110,6 +114,8 @@ const PROGRAM_FRAGMENTS: Record = { brushMaskStamp: brushMaskStampFragSrc, maskInvert: maskInvertFragSrc, asciiCarrier: asciiCarrierFragSrc, + halftoneCarrier: halftoneCarrierFragSrc, + channelDrift: channelDriftFragSrc, timestampOverlay: timestampOverlayFragSrc, }; @@ -133,6 +139,8 @@ export const DEFERRED_WARMUP_PROGRAMS: readonly ProgramName[] = [ "filmPrintUber", "filmEffectsUber", "asciiCarrier", + "halftoneCarrier", + "channelDrift", "timestampOverlay", ]; diff --git a/src/lib/renderer/gpuHalftoneCarrier.ts b/src/lib/renderer/gpuHalftoneCarrier.ts new file mode 100644 index 00000000..fe3fe655 --- /dev/null +++ b/src/lib/renderer/gpuHalftoneCarrier.ts @@ -0,0 +1,41 @@ +import type { RenderSurfaceHandle } from "@/lib/renderSurfaceHandle"; +import { runRendererSurfaceOperation } from "./gpuSurfaceOperation"; + +export interface HalftoneCarrierGpuInput { + width: number; + height: number; + frequency: number; + angle: number; + shape: "circle" | "diamond" | "line" | "square"; + colorMode: "mono" | "cmyk" | "rgb"; + dotScale: number; + contrast: number; + invert: boolean; + backgroundColorRgba: Float32Array; + backgroundOpacity: number; +} + +export const applyHalftoneCarrierOnGpuToSurface = async ({ + surface, + input, + slotId = "halftone-carrier", +}: { + surface: RenderSurfaceHandle; + input: HalftoneCarrierGpuInput; + slotId?: string; +}): Promise => { + if (surface.width <= 0 || surface.height <= 0) { + return null; + } + return runRendererSurfaceOperation({ + mode: surface.mode, + width: surface.width, + height: surface.height, + slotId, + render: (renderer) => + renderer.renderHalftoneCarrierComposite({ + baseCanvas: surface.sourceCanvas, + carrier: input, + }), + }); +}; diff --git a/src/lib/renderer/gpuSignalDamage.ts b/src/lib/renderer/gpuSignalDamage.ts new file mode 100644 index 00000000..34fc5477 --- /dev/null +++ b/src/lib/renderer/gpuSignalDamage.ts @@ -0,0 +1,39 @@ +import type { RenderSurfaceHandle } from "@/lib/renderSurfaceHandle"; +import { runRendererSurfaceOperation } from "./gpuSurfaceOperation"; + +export interface ChannelDriftGpuInput { + width: number; + height: number; + redOffsetX: number; + redOffsetY: number; + greenOffsetX: number; + greenOffsetY: number; + blueOffsetX: number; + blueOffsetY: number; + intensity: number; +} + +export const applyChannelDriftOnGpuToSurface = async ({ + surface, + input, + slotId = "channel-drift", +}: { + surface: RenderSurfaceHandle; + input: ChannelDriftGpuInput; + slotId?: string; +}): Promise => { + if (surface.width <= 0 || surface.height <= 0) { + return null; + } + return runRendererSurfaceOperation({ + mode: surface.mode, + width: surface.width, + height: surface.height, + slotId, + render: (renderer) => + renderer.renderChannelDriftComposite({ + baseCanvas: surface.sourceCanvas, + damage: input, + }), + }); +}; diff --git a/src/lib/renderer/shaders/ChannelDrift.frag b/src/lib/renderer/shaders/ChannelDrift.frag new file mode 100644 index 00000000..305ec149 --- /dev/null +++ b/src/lib/renderer/shaders/ChannelDrift.frag @@ -0,0 +1,26 @@ +#version 300 es +precision highp float; + +in vec2 vTextureCoord; +out vec4 outColor; + +uniform sampler2D uSampler; +uniform vec2 u_canvasSize; +uniform vec2 u_redOffset; +uniform vec2 u_greenOffset; +uniform vec2 u_blueOffset; +uniform float u_intensity; + +void main() { + vec2 texelSize = 1.0 / u_canvasSize; + vec2 rUv = vTextureCoord + u_redOffset * texelSize * u_intensity; + vec2 gUv = vTextureCoord + u_greenOffset * texelSize * u_intensity; + vec2 bUv = vTextureCoord + u_blueOffset * texelSize * u_intensity; + + float r = texture(uSampler, rUv).r; + float g = texture(uSampler, gUv).g; + float b = texture(uSampler, bUv).b; + float a = texture(uSampler, vTextureCoord).a; + + outColor = vec4(r, g, b, a); +} diff --git a/src/lib/renderer/shaders/HalftoneCarrier.frag b/src/lib/renderer/shaders/HalftoneCarrier.frag new file mode 100644 index 00000000..4f2878d4 --- /dev/null +++ b/src/lib/renderer/shaders/HalftoneCarrier.frag @@ -0,0 +1,109 @@ +#version 300 es +precision highp float; + +in vec2 vTextureCoord; +out vec4 outColor; + +uniform sampler2D uSampler; +uniform vec2 u_canvasSize; +uniform float u_frequency; +uniform float u_angle; +uniform float u_shape; // 0 = circle, 1 = diamond, 2 = line, 3 = square +uniform float u_colorMode; // 0 = mono, 1 = cmyk, 2 = rgb +uniform float u_dotScale; +uniform float u_contrast; +uniform bool u_invert; +uniform vec4 u_backgroundColor; +uniform float u_backgroundOpacity; + +const float PI = 3.14159265359; + +mat2 rotationMatrix(float angleDeg) { + float a = angleDeg * PI / 180.0; + float c = cos(a); + float s = sin(a); + return mat2(c, -s, s, c); +} + +float halftoneCell(vec2 pos, float luminance) { + float threshold = clamp(luminance, 0.0, 1.0); + threshold = pow(threshold, u_contrast); + float radius = threshold * u_dotScale; + + if (u_shape < 0.5) { + float dist = length(pos - vec2(0.5)); + return smoothstep(radius + 0.02, radius - 0.02, dist); + } + if (u_shape < 1.5) { + float dist = abs(pos.x - 0.5) + abs(pos.y - 0.5); + return smoothstep(radius * 1.414 + 0.02, radius * 1.414 - 0.02, dist); + } + if (u_shape < 2.5) { + float dist = abs(pos.y - 0.5); + return smoothstep(radius * 0.5 + 0.01, radius * 0.5 - 0.01, dist); + } + float dist = max(abs(pos.x - 0.5), abs(pos.y - 0.5)); + return smoothstep(radius + 0.02, radius - 0.02, dist); +} + +float screenChannel(vec2 pixel, float channelValue, float angleDeg) { + mat2 rot = rotationMatrix(angleDeg); + vec2 rotated = rot * pixel; + float cellSize = max(2.0, u_canvasSize.y / max(1.0, u_frequency)); + vec2 cellCoord = fract(rotated / cellSize); + return halftoneCell(cellCoord, channelValue); +} + +void main() { + vec4 src = texture(uSampler, vTextureCoord); + vec2 pixel = vTextureCoord * u_canvasSize; + vec4 bg = vec4(u_backgroundColor.rgb, u_backgroundOpacity); + + vec4 result; + + if (u_colorMode < 0.5) { + float lum = dot(src.rgb, vec3(0.2126, 0.7152, 0.0722)); + if (u_invert) lum = 1.0 - lum; + float dot = screenChannel(pixel, lum, u_angle); + vec3 fg = u_invert ? vec3(0.0) : vec3(1.0); + result = vec4(mix(bg.rgb, fg, dot), mix(bg.a, 1.0, dot) * src.a); + } else if (u_colorMode < 1.5) { + float c = 1.0 - src.r; + float m = 1.0 - src.g; + float y = 1.0 - src.b; + float k = min(c, min(m, y)); + c = (c - k) / max(1.0 - k, 0.001); + m = (m - k) / max(1.0 - k, 0.001); + y = (y - k) / max(1.0 - k, 0.001); + + float cDot = screenChannel(pixel, c, u_angle + 15.0); + float mDot = screenChannel(pixel, m, u_angle + 75.0); + float yDot = screenChannel(pixel, y, u_angle); + float kDot = screenChannel(pixel, k, u_angle + 45.0); + + vec3 white = vec3(1.0); + vec3 cmykResult = white; + cmykResult -= cDot * vec3(1.0, 0.0, 0.0); + cmykResult -= mDot * vec3(0.0, 1.0, 0.0); + cmykResult -= yDot * vec3(0.0, 0.0, 1.0); + cmykResult -= kDot * vec3(1.0, 1.0, 1.0); + cmykResult = clamp(cmykResult, 0.0, 1.0); + + if (u_invert) cmykResult = 1.0 - cmykResult; + result = vec4(mix(bg.rgb, cmykResult, src.a), mix(bg.a, 1.0, src.a)); + } else { + float rDot = screenChannel(pixel, u_invert ? 1.0 - src.r : src.r, u_angle); + float gDot = screenChannel(pixel, u_invert ? 1.0 - src.g : src.g, u_angle + 30.0); + float bDot = screenChannel(pixel, u_invert ? 1.0 - src.b : src.b, u_angle + 60.0); + + vec3 fg = u_invert ? vec3(0.0) : vec3(1.0); + vec3 rgbResult = vec3( + mix(bg.r, fg.r, rDot), + mix(bg.g, fg.g, gDot), + mix(bg.b, fg.b, bDot) + ); + result = vec4(rgbResult, mix(bg.a, 1.0, max(rDot, max(gDot, bDot))) * src.a); + } + + outColor = result; +} diff --git a/src/render/image/asciiEffect.test.ts b/src/render/image/asciiEffect.test.ts index e30eb017..0ef3ae3d 100644 --- a/src/render/image/asciiEffect.test.ts +++ b/src/render/image/asciiEffect.test.ts @@ -275,7 +275,7 @@ describe("asciiEffect", () => { style: createMockCanvas(targetSize), }, }) - ).rejects.toThrow(/ASCII carrier GPU pass failed/); + ).rejects.toThrow(/Carrier GPU pass failed/); }); it("routes masked carriers through applyMaskedStageOperationToSurfaceIfSupported", async () => { diff --git a/src/render/image/asciiEffect.ts b/src/render/image/asciiEffect.ts index 3cbee462..0e2cfdf1 100644 --- a/src/render/image/asciiEffect.ts +++ b/src/render/image/asciiEffect.ts @@ -7,6 +7,7 @@ import { import type { EditorLayerBlendMode } from "@/types"; import { resolveDensitySortedCharset } from "./asciiDensityMeasure"; import { applyMaskedStageOperationToSurfaceIfSupported } from "./stageMaskComposite"; +import { applyImageHalftoneCarrierTransform } from "./halftoneEffect"; import type { CarrierTransformNode, ImageAsciiCarrierTransformNode, @@ -485,6 +486,38 @@ interface CarrierSnapshots { style: HTMLCanvasElement; } +const applyCarrierTransform = async ({ + surface, + transform, + sourceCanvas, + quality, + targetSize, +}: { + surface: RenderSurfaceHandle; + transform: CarrierTransformNode; + sourceCanvas: HTMLCanvasElement; + quality: ImageRenderQuality; + targetSize: ImageRenderTargetSize; +}): Promise => { + switch (transform.type) { + case "ascii": + return applyImageAsciiCarrierTransform({ + baseSurface: surface, + sourceCanvas, + transform, + quality, + targetSize, + }); + case "halftone": + return applyImageHalftoneCarrierTransform({ + baseSurface: surface, + transform, + quality, + targetSize, + }); + } +}; + export const applyImageCarrierTransforms = async ({ surface, carrierTransforms, @@ -516,16 +549,16 @@ export const applyImageCarrierTransforms = async ({ maskReferenceCanvas: stageReferenceCanvas ?? snapshots.style, blendSlotId: transform.maskId ? `carrier-mask:${transform.id}` : undefined, applyOperation: async ({ surface: targetSurface }) => - applyImageAsciiCarrierTransform({ - baseSurface: targetSurface, - sourceCanvas, + applyCarrierTransform({ + surface: targetSurface, transform, + sourceCanvas, quality: request.quality, targetSize: request.targetSize, }), }); if (!nextSurface) { - throw new Error(`ASCII carrier GPU pass failed for transform ${transform.id}`); + throw new Error(`Carrier GPU pass failed for transform ${transform.id} (${transform.type})`); } currentSurface = nextSurface; } diff --git a/src/render/image/halftoneEffect.ts b/src/render/image/halftoneEffect.ts new file mode 100644 index 00000000..8ae53140 --- /dev/null +++ b/src/render/image/halftoneEffect.ts @@ -0,0 +1,65 @@ +import type { RenderSurfaceHandle } from "@/lib/renderSurfaceHandle"; +import { + applyHalftoneCarrierOnGpuToSurface, + type HalftoneCarrierGpuInput, +} from "@/lib/renderer/gpuHalftoneCarrier"; +import { clamp } from "@/lib/math"; +import type { + ImageHalftoneCarrierTransformNode, + ImageRenderQuality, + ImageRenderTargetSize, +} from "./types"; + +const HALFTONE_CARRIER_SLOT_ID = "halftone-carrier"; + +const parseHexColor = (value: string | null): { r: number; g: number; b: number } => { + if (!value || value.length < 7) { + return { r: 0, g: 0, b: 0 }; + } + return { + r: parseInt(value.slice(1, 3), 16) / 255, + g: parseInt(value.slice(3, 5), 16) / 255, + b: parseInt(value.slice(5, 7), 16) / 255, + }; +}; + +const prepareHalftoneGpuInput = ( + transform: ImageHalftoneCarrierTransformNode, + _quality: ImageRenderQuality, + targetSize: ImageRenderTargetSize +): HalftoneCarrierGpuInput => { + const params = transform.params; + const bgColor = parseHexColor(params.backgroundColor); + return { + width: Math.max(1, Math.round(targetSize.width)), + height: Math.max(1, Math.round(targetSize.height)), + frequency: clamp(params.frequency, 4, 80), + angle: params.angle % 360, + shape: params.shape, + colorMode: params.colorMode, + dotScale: clamp(params.dotScale, 0.5, 2), + contrast: clamp(params.contrast, 0.5, 3), + invert: params.invert, + backgroundColorRgba: new Float32Array([bgColor.r, bgColor.g, bgColor.b, 1]), + backgroundOpacity: clamp(params.backgroundOpacity, 0, 1), + }; +}; + +export const applyImageHalftoneCarrierTransform = async ({ + baseSurface, + transform, + quality, + targetSize, +}: { + baseSurface: RenderSurfaceHandle; + transform: ImageHalftoneCarrierTransformNode; + quality: ImageRenderQuality; + targetSize: ImageRenderTargetSize; +}): Promise => { + const input = prepareHalftoneGpuInput(transform, quality, targetSize); + return applyHalftoneCarrierOnGpuToSurface({ + surface: baseSurface, + input, + slotId: HALFTONE_CARRIER_SLOT_ID, + }); +}; diff --git a/src/render/image/index.ts b/src/render/image/index.ts index 31dd88e1..dcc71397 100644 --- a/src/render/image/index.ts +++ b/src/render/image/index.ts @@ -3,3 +3,4 @@ export * from "./stateCompiler"; export * from "./effectMask"; export * from "./overlayExecution"; export * from "./renderSingleImage"; +export * from "./signalDamageExecution"; diff --git a/src/render/image/renderSingleImage.ts b/src/render/image/renderSingleImage.ts index e9947b12..a7ac9d8b 100644 --- a/src/render/image/renderSingleImage.ts +++ b/src/render/image/renderSingleImage.ts @@ -19,6 +19,7 @@ import type { RenderIntent } from "@/lib/renderIntent"; import { applyImageCarrierTransforms } from "./asciiEffect"; import { applyImageEffects } from "./effectExecution"; import { applyImageOverlays, resolveImageOverlays } from "./overlayExecution"; +import { applyImageSignalDamage } from "./signalDamageExecution"; import { assertSupportedImageRenderSnapshotPlan, createImageRenderSnapshotPlan, @@ -167,6 +168,7 @@ export const renderSingleImageToCanvas = async ({ }): Promise => { const snapshotPlan = createImageRenderSnapshotPlan({ carrierTransforms: document.carrierTransforms, + signalDamage: document.signalDamage, effects: document.effects, }); assertSupportedImageRenderSnapshotPlan(snapshotPlan); @@ -302,6 +304,19 @@ export const renderSingleImageToCanvas = async ({ }); } + if (snapshotPlan.signalDamage.length > 0) { + surface = await applyImageSignalDamage({ + surface, + signalDamage: snapshotPlan.signalDamage, + document, + stageReferenceCanvas: carrierAnalysisSnapshotCanvas ?? undefined, + }); + appendTraceOperation(debugStages, "style", { + kind: "carrier", + carrierCount: snapshotPlan.signalDamage.length, + }); + } + if (snapshotPlan.styleEffects.length > 0) { const styleSnapshotCanvas = hasMaskedStyleEffects ? trackSurfaceClone(surface) : null; try { diff --git a/src/render/image/signalDamageExecution.ts b/src/render/image/signalDamageExecution.ts new file mode 100644 index 00000000..651dd665 --- /dev/null +++ b/src/render/image/signalDamageExecution.ts @@ -0,0 +1,97 @@ +import type { RenderSurfaceHandle } from "@/lib/renderSurfaceHandle"; +import { + applyChannelDriftOnGpuToSurface, + type ChannelDriftGpuInput, +} from "@/lib/renderer/gpuSignalDamage"; +import { clamp } from "@/lib/math"; +import { applyMaskedStageOperationToSurfaceIfSupported } from "./stageMaskComposite"; +import type { + ImageChannelDriftDamageNode, + ImageRenderDocument, + SignalDamageNode, +} from "./types"; + +const CHANNEL_DRIFT_SLOT_ID = "channel-drift"; + +const prepareChannelDriftGpuInput = ( + node: ImageChannelDriftDamageNode, + width: number, + height: number +): ChannelDriftGpuInput => { + const params = node.params; + return { + width, + height, + redOffsetX: clamp(params.redOffsetX, -100, 100), + redOffsetY: clamp(params.redOffsetY, -100, 100), + greenOffsetX: clamp(params.greenOffsetX, -100, 100), + greenOffsetY: clamp(params.greenOffsetY, -100, 100), + blueOffsetX: clamp(params.blueOffsetX, -100, 100), + blueOffsetY: clamp(params.blueOffsetY, -100, 100), + intensity: clamp(params.intensity, 0, 1), + }; +}; + +const applyChannelDrift = async ({ + baseSurface, + node, +}: { + baseSurface: RenderSurfaceHandle; + node: ImageChannelDriftDamageNode; +}): Promise => { + const input = prepareChannelDriftGpuInput( + node, + baseSurface.width, + baseSurface.height + ); + return applyChannelDriftOnGpuToSurface({ + surface: baseSurface, + input, + slotId: CHANNEL_DRIFT_SLOT_ID, + }); +}; + +export const applyImageSignalDamage = async ({ + surface, + signalDamage, + document, + stageReferenceCanvas, +}: { + surface: RenderSurfaceHandle; + signalDamage: readonly SignalDamageNode[]; + document: ImageRenderDocument; + stageReferenceCanvas?: HTMLCanvasElement; +}): Promise => { + let currentSurface = surface; + + for (const node of signalDamage) { + const maskDefinition = node.maskId + ? document.masks.byId[node.maskId] ?? null + : null; + + const nextSurface = await applyMaskedStageOperationToSurfaceIfSupported({ + surface: currentSurface, + maskDefinition, + maskReferenceCanvas: stageReferenceCanvas, + blendSlotId: node.maskId ? `signal-damage-mask:${node.id}` : undefined, + applyOperation: async ({ surface: targetSurface }) => { + switch (node.type) { + case "channel-drift": + return applyChannelDrift({ + baseSurface: targetSurface, + node, + }); + default: + return null; + } + }, + }); + + if (!nextSurface) { + throw new Error(`Signal damage GPU pass failed for node ${node.id}`); + } + currentSurface = nextSurface; + } + + return currentSurface; +}; diff --git a/src/render/image/snapshotPlan.test.ts b/src/render/image/snapshotPlan.test.ts index c2200aac..437acc2e 100644 --- a/src/render/image/snapshotPlan.test.ts +++ b/src/render/image/snapshotPlan.test.ts @@ -59,6 +59,7 @@ describe("image render snapshot plan", () => { analysisSource: "develop", }), ], + signalDamage: [], effects: [], }); @@ -69,6 +70,7 @@ describe("image render snapshot plan", () => { it("keeps carrier, style and finalize stages in stable order", () => { const plan = createImageRenderSnapshotPlan({ carrierTransforms: [createAsciiCarrier({ id: "ascii-carrier" })], + signalDamage: [], effects: [ createFilter2dEffect({ id: "filter-style", placement: "style" }), createFilter2dEffect({ id: "filter-finalize", placement: "finalize" }), @@ -83,6 +85,7 @@ describe("image render snapshot plan", () => { it("keeps develop, style and finalize raster effects in stable order", () => { const plan = createImageRenderSnapshotPlan({ carrierTransforms: [createAsciiCarrier({ id: "ascii-style" })], + signalDamage: [], effects: [ createFilter2dEffect({ id: "filter-develop", placement: "develop" }), createFilter2dEffect({ id: "filter-style", placement: "style" }), @@ -98,6 +101,7 @@ describe("image render snapshot plan", () => { it("accepts the carrier-first stage plan without extra unsupported checks", () => { const plan = createImageRenderSnapshotPlan({ carrierTransforms: [createAsciiCarrier({ analysisSource: "style" })], + signalDamage: [], effects: [createFilter2dEffect({ placement: "style" })], }); diff --git a/src/render/image/snapshotPlan.ts b/src/render/image/snapshotPlan.ts index cdedb439..aaaaf0b3 100644 --- a/src/render/image/snapshotPlan.ts +++ b/src/render/image/snapshotPlan.ts @@ -1,7 +1,8 @@ -import type { CarrierTransformNode, ImageEffectNode } from "./types"; +import type { CarrierTransformNode, ImageEffectNode, SignalDamageNode } from "./types"; export interface ImageRenderSnapshotPlan { carrierTransforms: CarrierTransformNode[]; + signalDamage: SignalDamageNode[]; developEffects: ImageEffectNode[]; styleEffects: ImageEffectNode[]; finalizeEffects: ImageEffectNode[]; @@ -12,13 +13,16 @@ export interface ImageRenderSnapshotPlan { export const createImageRenderSnapshotPlan = ( options: { carrierTransforms: readonly CarrierTransformNode[]; + signalDamage: readonly SignalDamageNode[]; effects: readonly ImageEffectNode[]; } ): ImageRenderSnapshotPlan => { const enabledCarrierTransforms = options.carrierTransforms.filter((transform) => transform.enabled); + const enabledSignalDamage = options.signalDamage.filter((node) => node.enabled); const enabledEffects = options.effects.filter((effect) => effect.enabled); return { carrierTransforms: enabledCarrierTransforms, + signalDamage: enabledSignalDamage, developEffects: enabledEffects.filter((effect) => effect.placement === "develop"), styleEffects: enabledEffects.filter((effect) => effect.placement === "style"), finalizeEffects: enabledEffects.filter((effect) => effect.placement === "finalize"), diff --git a/src/render/image/stateCompiler.ts b/src/render/image/stateCompiler.ts index aa2179d8..7f443795 100644 --- a/src/render/image/stateCompiler.ts +++ b/src/render/image/stateCompiler.ts @@ -170,6 +170,7 @@ export const createNeutralCanvasImageRenderState = (): CanvasImageRenderStateV1 byId: {}, }, carrierTransforms: [], + signalDamage: [], effects: [], film: { profileId: null, diff --git a/src/render/image/types.ts b/src/render/image/types.ts index e526af65..42b6fd30 100644 --- a/src/render/image/types.ts +++ b/src/render/image/types.ts @@ -6,6 +6,8 @@ import type { CalibrationAdjustments, ColorGradingAdjustments, FilmProfileOverrides, + HalftoneColorMode, + HalftoneShape, HslAdjustments, LocalAdjustmentDelta, LocalAdjustmentMask, @@ -254,15 +256,59 @@ export interface ImageAsciiCarrierTransformNode { params: ImageAsciiEffectParams; } -export type CarrierTransformNode = ImageAsciiCarrierTransformNode; +export interface ImageHalftoneEffectParams { + frequency: number; + angle: number; + shape: HalftoneShape; + colorMode: HalftoneColorMode; + dotScale: number; + contrast: number; + invert: boolean; + backgroundColor: string | null; + backgroundOpacity: number; +} + +export interface ImageHalftoneCarrierTransformNode { + id: string; + type: "halftone"; + enabled: boolean; + analysisSource: ImageAnalysisSource; + maskId?: string; + params: ImageHalftoneEffectParams; +} + +export type CarrierTransformNode = + | ImageAsciiCarrierTransformNode + | ImageHalftoneCarrierTransformNode; export type ImageEffectNode = ImageFilter2dEffectNode; +export interface ImageChannelDriftDamageParams { + redOffsetX: number; + redOffsetY: number; + greenOffsetX: number; + greenOffsetY: number; + blueOffsetX: number; + blueOffsetY: number; + intensity: number; +} + +export interface ImageChannelDriftDamageNode { + id: string; + type: "channel-drift"; + enabled: boolean; + maskId?: string; + params: ImageChannelDriftDamageParams; +} + +export type SignalDamageNode = ImageChannelDriftDamageNode; + export interface CanvasImageRenderStateV1 { geometry: ImageRenderGeometry; develop: ImageRenderDevelopState; masks: ImageRenderMaskState; carrierTransforms: CarrierTransformNode[]; + signalDamage: SignalDamageNode[]; effects: ImageEffectNode[]; film: ImageRenderFilmState; output: ImageRenderOutputState; @@ -328,7 +374,12 @@ const isFilter2dEffectNode = (value: unknown): value is ImageFilter2dEffectNode isRecord(value) && value.type === "filter2d" && "params" in value; const isCarrierTransformNode = (value: unknown): value is CarrierTransformNode => - isRecord(value) && value.type === "ascii" && isAsciiAnalysisSource(value.analysisSource); + isRecord(value) && + (value.type === "ascii" || value.type === "halftone") && + isAsciiAnalysisSource(value.analysisSource); + +const isSignalDamageNode = (value: unknown): value is SignalDamageNode => + isRecord(value) && value.type === "channel-drift" && "params" in value; const mapLegacyAsciiEffectToCarrierTransform = ( effect: ImageAsciiCarrierTransformNode & { placement?: ImageEffectPlacement } @@ -353,6 +404,11 @@ export const normalizeCanvasImageRenderState = ( const rawEffects = Array.isArray((state as CanvasImageRenderStateV1 & { effects?: unknown[] }).effects) ? ((state as CanvasImageRenderStateV1 & { effects?: unknown[] }).effects ?? []) : []; + const rawSignalDamage = Array.isArray( + (state as CanvasImageRenderStateV1 & { signalDamage?: unknown[] }).signalDamage + ) + ? ((state as CanvasImageRenderStateV1 & { signalDamage?: unknown[] }).signalDamage ?? []) + : []; const explicitCarrierTransforms = rawCarrierTransforms .filter(isCarrierTransformNode) @@ -360,6 +416,9 @@ export const normalizeCanvasImageRenderState = ( const rasterEffects = rawEffects .filter(isFilter2dEffectNode) .map((effect) => cloneImageRenderValue(effect)); + const signalDamageNodes = rawSignalDamage + .filter(isSignalDamageNode) + .map((node) => cloneImageRenderValue(node)); const migratedCarrierTransforms = explicitCarrierTransforms.length > 0 ? explicitCarrierTransforms @@ -376,6 +435,7 @@ export const normalizeCanvasImageRenderState = ( output: state.output, }), carrierTransforms: migratedCarrierTransforms, + signalDamage: signalDamageNodes, effects: rasterEffects, }; }; @@ -448,3 +508,7 @@ export const resolveImageRenderEffectsForPlacement = ( export const resolveImageCarrierTransforms = ( carrierTransforms: readonly CarrierTransformNode[] ) => carrierTransforms.filter((transform) => transform.enabled); + +export const resolveImageSignalDamage = ( + signalDamage: readonly SignalDamageNode[] +) => signalDamage.filter((node) => node.enabled); diff --git a/src/types/index.ts b/src/types/index.ts index f832a91f..285ca7a1 100644 --- a/src/types/index.ts +++ b/src/types/index.ts @@ -230,6 +230,33 @@ export interface AsciiAdjustments { dither: AsciiDitherMode; } +export type HalftoneShape = "circle" | "diamond" | "line" | "square"; +export type HalftoneColorMode = "mono" | "cmyk" | "rgb"; + +export interface HalftoneAdjustments { + enabled: boolean; + frequency: number; + angle: number; + shape: HalftoneShape; + colorMode: HalftoneColorMode; + dotScale: number; + contrast: number; + invert: boolean; + backgroundColor: string; + backgroundOpacity: number; +} + +export interface ChannelDriftAdjustments { + enabled: boolean; + redOffsetX: number; + redOffsetY: number; + greenOffsetX: number; + greenOffsetY: number; + blueOffsetX: number; + blueOffsetY: number; + intensity: number; +} + export interface LocalAdjustmentDelta { exposure?: number; contrast?: number; From b3468d790a6a72ea7b685c4a60023fedcaad62b4 Mon Sep 17 00:00:00 2001 From: WenHao Chen <157730893+YakiHugo@users.noreply.github.com> Date: Sat, 25 Apr 2026 00:20:52 -0700 Subject: [PATCH 03/14] docs: long task JSON schema with blockedBy DAG and claim protocol Update AGENTS.md long task rules: - JSON fields: id, status, blockedBy, passes (no derived fields) - blockedBy defines the DAG; empty array = no dependency - Eligible = pending + all blockedBy done - Claim by setting in_progress and committing before implementation Migrate media-native-render-pipeline task files to new schema. Mark carrier-and-signal-families slice as done. Co-Authored-By: Claude Opus 4.6 (1M context) --- AGENTS.md | 10 ++++-- docs/tasks/media-native-render-pipeline.json | 17 ++++++---- docs/tasks/media-native-render-pipeline.md | 35 +++++++++++++++----- 3 files changed, 44 insertions(+), 18 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 68069f80..f2885560 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -25,9 +25,15 @@ - Treat a task as long when it cannot be completed safely in one session without explicit slicing. - For long tasks, create paired `docs/tasks/.md` and `.json` files: markdown for scope, decisions, validation, and handoff; JSON for terse execution state only. -- Keep the JSON terse: stable task statuses such as `pending`, `in_progress`, `blocked`, `done`, `rolled_back`; `passes` as the completion gate; baseline/current task; rollback notes only when not obvious. -- The first session must at least slice the work and define validation boundaries; it may also complete the first slice if that slice is low-risk and fully validated. +- JSON schema per task: `id`, `status`, `blockedBy`, `passes`. No derived fields (`nextTaskId`, `current` — compute from DAG). + - `status`: `pending` | `in_progress` | `blocked` | `done` | `rolled_back`. + - `blockedBy`: array of task ids that must be `done` before this task is eligible. Empty array = no dependency. + - `passes`: array of validation gates; all must hold before marking `done`. + - Eligible task: `status=pending` and every id in `blockedBy` has `status=done`. +- Claiming a task: set `status` to `in_progress` and commit as the first action, before starting implementation. This ensures other agents see the claim. +- The first session must at least slice the work, declare dependencies via `blockedBy`, and define validation boundaries; it may also complete the first slice if that slice is low-risk and fully validated. - If a slice fails validation and is not fixed immediately, mark it `blocked` or `rolled_back`, record the first actionable failure in the markdown note, and stop claiming progress. +- MD carries scope, decisions, per-slice implementation notes, and handoff. Do not duplicate DAG, status, or passes — those live in JSON. - When every slice reaches `done`, close the task: migrate load-bearing decisions and known follow-ups into `docs/decisions.md`, then delete the `docs/tasks/.{md,json}` pair. Slice-by-slice handoff is carried by git history, not by long-lived docs. ## Documentation Hygiene diff --git a/docs/tasks/media-native-render-pipeline.json b/docs/tasks/media-native-render-pipeline.json index b7d55129..4a80378e 100644 --- a/docs/tasks/media-native-render-pipeline.json +++ b/docs/tasks/media-native-render-pipeline.json @@ -1,18 +1,18 @@ { "baseline": "2026-04-02", - "current": "in_progress", - "nextTaskId": "carrier-and-signal-families", "tasks": [ { "id": "semantic-overlay-layer-system", "status": "pending", + "blockedBy": [], "passes": [ "caption, HUD, browser-chrome, or similar overlays have an authored model with canvas preview/export parity" ] }, { "id": "carrier-and-signal-families", - "status": "in_progress", + "status": "done", + "blockedBy": [], "passes": [ "carrier transforms and signal damage are authored as first-class families instead of generic effect placement only" ] @@ -20,22 +20,25 @@ { "id": "analysis-layer-boundary", "status": "pending", + "blockedBy": [], "passes": [ "analysis-driven overlays or effects consume explicit analysis inputs with bounded validation" ] }, { - "id": "motion-live-render-contract", + "id": "preview-export-quality-split", "status": "pending", + "blockedBy": [], "passes": [ - "a time-parameterized render contract exists for short-loop or live-style output and one preset is validated end-to-end" + "interactive preview, quality preview, and export are explicitly modeled without a second authored-state source of truth" ] }, { - "id": "preview-export-quality-split", + "id": "motion-live-render-contract", "status": "pending", + "blockedBy": ["preview-export-quality-split"], "passes": [ - "interactive preview, quality preview, and export are explicitly modeled without a second authored-state source of truth" + "a time-parameterized render contract exists for short-loop or live-style output and one preset is validated end-to-end" ] } ] diff --git a/docs/tasks/media-native-render-pipeline.md b/docs/tasks/media-native-render-pipeline.md index 31f40320..2bb4e3a5 100644 --- a/docs/tasks/media-native-render-pipeline.md +++ b/docs/tasks/media-native-render-pipeline.md @@ -33,7 +33,7 @@ - Scene/global ownership must stay above image-node-local render state and is tracked separately in `docs/tasks/scene-global-render-follow-up.md`. - Keep preview/export differences at the scheduler or quality tier, not in a second authored-state source of truth. -## Open Slices +## Slices ### 1. Semantic Overlay Layer System @@ -110,17 +110,25 @@ ## Current Focus -- `carrier-and-signal-families` remains the active slice. -- ASCII already executes as a carrier transform. -- The next implementation should add one more authored carrier or signal family without widening back into generic effect placement only. +- `carrier-and-signal-families` slice is complete. +- Next active slice: `semantic-overlay-layer-system` or `analysis-layer-boundary`. ## Files - `src/render/image/renderSingleImage.ts` -- `src/render/image/carrierExecution.ts` +- `src/render/image/asciiEffect.ts` (carrier orchestrator + ASCII impl) +- `src/render/image/halftoneEffect.ts` +- `src/render/image/signalDamageExecution.ts` - `src/render/image/effectExecution.ts` - `src/render/image/overlayExecution.ts` - `src/render/image/snapshotPlan.ts` +- `src/render/image/types.ts` +- `src/lib/renderer/gpuHalftoneCarrier.ts` +- `src/lib/renderer/gpuSignalDamage.ts` +- `src/lib/renderer/shaders/HalftoneCarrier.frag` +- `src/lib/renderer/shaders/ChannelDrift.frag` +- `src/features/canvas/CanvasHalftoneEditPanel.tsx` +- `src/features/canvas/CanvasSignalDamageEditPanel.tsx` - `src/features/canvas/boardImageRendering.ts` - `src/features/canvas/renderCanvasDocument.ts` - no matches @@ -133,15 +141,24 @@ - Implemented in the first slice: - canonical stage naming is now `develop -> style -> overlay -> finalize` - timestamp handling now flows through a shared overlay runtime entry instead of direct per-call special casing -- Implemented in the current ASCII-first carrier sub-slice: +- Implemented in the ASCII-first carrier sub-slice: - `CanvasImageRenderStateV1` now carries `carrierTransforms` - ASCII authoring/editing moved out of `effects[]` and into `carrierTransforms` - preview/export revision identity now includes carrier transforms - legacy ASCII effect persistence is treated as read-only compatibility input, not a write-path schema -- Still open after the first slice: +- Implemented in the carrier-and-signal-families slice: + - `CarrierTransformNode` is now a union of `ascii | halftone` + - Halftone carrier: GPU shader with mono/CMYK/RGB color separation, circle/diamond/line/square dot shapes + - `CanvasImageRenderStateV1` now carries `signalDamage: SignalDamageNode[]` as a first-class authored family + - Channel drift signal damage: GPU shader with per-channel RGB offset + - Carrier orchestrator dispatches by transform type (no longer ASCII-only) + - Signal damage executes as a dedicated pipeline stage between carriers and style effects + - UI panels: `CanvasHalftoneEditPanel`, `CanvasSignalDamageEditPanel` with full preview/commit workflow + - Family classification: halftone and channel drift are both single-frame deterministic +- Still open after the carrier-and-signal-families slice: - authored `semanticOverlays` model - board/global overlay ownership rules - non-timestamp overlay types - - signal-damage families - - carrier families beyond ASCII (`dither`, `halftone`, `palette`, `textmode`) + - additional carrier families (`dither`, `palette`, `textmode`) + - additional signal damage families (`line-displacement`, `row-shift`, `compression-artifacts`, `pixel-sort`) - motion/live render contract From 6e1ef294fbf738fd9f98bcb58c24d82e956bf6ee Mon Sep 17 00:00:00 2001 From: WenHao Chen <157730893+YakiHugo@users.noreply.github.com> Date: Sat, 25 Apr 2026 00:50:26 -0700 Subject: [PATCH 04/14] feat(renderer): semantic overlay model with caption and watermark types Extend SemanticOverlayNode union beyond TimestampSemanticOverlayNode with CaptionSemanticOverlayNode and WatermarkSemanticOverlayNode. Migrate output.timestamp to semanticOverlays[] in neutral state. Add normalization guard for new overlay types and CaptionAdjustments/WatermarkAdjustments in the shared type surface. Co-Authored-By: Claude Opus 4.6 (1M context) --- src/render/image/stateCompiler.ts | 10 +-- src/render/image/types.ts | 106 +++++++++++++++++++++++++++++- src/types/index.ts | 35 ++++++++++ 3 files changed, 141 insertions(+), 10 deletions(-) diff --git a/src/render/image/stateCompiler.ts b/src/render/image/stateCompiler.ts index 7f443795..e60fcd2a 100644 --- a/src/render/image/stateCompiler.ts +++ b/src/render/image/stateCompiler.ts @@ -171,20 +171,14 @@ export const createNeutralCanvasImageRenderState = (): CanvasImageRenderStateV1 }, carrierTransforms: [], signalDamage: [], + semanticOverlays: [], effects: [], film: { profileId: null, profile: undefined, profileOverrides: null, }, - output: { - timestamp: { - enabled: false, - position: "bottom-right", - size: 22, - opacity: 72, - }, - }, + output: {}, }); export const createDefaultCanvasImageRenderState = (): CanvasImageRenderStateV1 => diff --git a/src/render/image/types.ts b/src/render/image/types.ts index 42b6fd30..46c96c3d 100644 --- a/src/render/image/types.ts +++ b/src/render/image/types.ts @@ -4,6 +4,8 @@ import type { AsciiAdjustments, BwMixAdjustments, CalibrationAdjustments, + CaptionOverlayAlignment, + CaptionOverlayPosition, ColorGradingAdjustments, FilmProfileOverrides, HalftoneColorMode, @@ -12,6 +14,7 @@ import type { LocalAdjustmentDelta, LocalAdjustmentMask, PointCurveAdjustments, + TimestampOverlayPosition, } from "@/types"; import type { FilmProfileAny } from "@/types/film"; @@ -166,9 +169,9 @@ export interface ImageRenderFilmState { } export interface ImageRenderOutputState { - timestamp: { + timestamp?: { enabled: boolean; - position: "bottom-right" | "bottom-left" | "top-right" | "top-left"; + position: TimestampOverlayPosition; size: number; opacity: number; }; @@ -303,12 +306,66 @@ export interface ImageChannelDriftDamageNode { export type SignalDamageNode = ImageChannelDriftDamageNode; +export interface TimestampOverlayParams { + position: TimestampOverlayPosition; + size: number; + opacity: number; +} + +export interface TimestampSemanticOverlayNode { + id: string; + type: "timestamp"; + enabled: boolean; + params: TimestampOverlayParams; +} + +export interface CaptionOverlayParams { + text: string; + position: CaptionOverlayPosition; + alignment: CaptionOverlayAlignment; + fontSize: number; + color: string; + backgroundColor: string; + backgroundOpacity: number; + padding: number; + opacity: number; +} + +export interface CaptionSemanticOverlayNode { + id: string; + type: "caption"; + enabled: boolean; + params: CaptionOverlayParams; +} + +export interface WatermarkOverlayParams { + text: string; + opacity: number; + fontSize: number; + angle: number; + density: number; + color: string; +} + +export interface WatermarkSemanticOverlayNode { + id: string; + type: "watermark"; + enabled: boolean; + params: WatermarkOverlayParams; +} + +export type SemanticOverlayNode = + | TimestampSemanticOverlayNode + | CaptionSemanticOverlayNode + | WatermarkSemanticOverlayNode; + export interface CanvasImageRenderStateV1 { geometry: ImageRenderGeometry; develop: ImageRenderDevelopState; masks: ImageRenderMaskState; carrierTransforms: CarrierTransformNode[]; signalDamage: SignalDamageNode[]; + semanticOverlays: SemanticOverlayNode[]; effects: ImageEffectNode[]; film: ImageRenderFilmState; output: ImageRenderOutputState; @@ -381,6 +438,11 @@ const isCarrierTransformNode = (value: unknown): value is CarrierTransformNode = const isSignalDamageNode = (value: unknown): value is SignalDamageNode => isRecord(value) && value.type === "channel-drift" && "params" in value; +const SEMANTIC_OVERLAY_TYPES = new Set(["timestamp", "caption", "watermark"]); + +const isSemanticOverlayNode = (value: unknown): value is SemanticOverlayNode => + isRecord(value) && typeof value.type === "string" && SEMANTIC_OVERLAY_TYPES.has(value.type) && "params" in value; + const mapLegacyAsciiEffectToCarrierTransform = ( effect: ImageAsciiCarrierTransformNode & { placement?: ImageEffectPlacement } ): CarrierTransformNode => ({ @@ -392,6 +454,25 @@ const mapLegacyAsciiEffectToCarrierTransform = ( params: cloneImageRenderValue(effect.params), }); +const migrateOutputTimestampToSemanticOverlay = ( + output: ImageRenderOutputState +): SemanticOverlayNode | null => { + const ts = output.timestamp; + if (!ts || !ts.enabled) { + return null; + } + return { + id: "canvas-timestamp", + type: "timestamp", + enabled: true, + params: { + position: ts.position, + size: ts.size, + opacity: ts.opacity, + }, + }; +}; + export const normalizeCanvasImageRenderState = ( state: CanvasImageRenderStateV1 ): CanvasImageRenderStateV1 => { @@ -409,6 +490,11 @@ export const normalizeCanvasImageRenderState = ( ) ? ((state as CanvasImageRenderStateV1 & { signalDamage?: unknown[] }).signalDamage ?? []) : []; + const rawSemanticOverlays = Array.isArray( + (state as CanvasImageRenderStateV1 & { semanticOverlays?: unknown[] }).semanticOverlays + ) + ? ((state as CanvasImageRenderStateV1 & { semanticOverlays?: unknown[] }).semanticOverlays ?? []) + : []; const explicitCarrierTransforms = rawCarrierTransforms .filter(isCarrierTransformNode) @@ -419,6 +505,9 @@ export const normalizeCanvasImageRenderState = ( const signalDamageNodes = rawSignalDamage .filter(isSignalDamageNode) .map((node) => cloneImageRenderValue(node)); + const semanticOverlayNodes = rawSemanticOverlays + .filter(isSemanticOverlayNode) + .map((node) => cloneImageRenderValue(node)); const migratedCarrierTransforms = explicitCarrierTransforms.length > 0 ? explicitCarrierTransforms @@ -426,6 +515,14 @@ export const normalizeCanvasImageRenderState = ( isLegacyAsciiEffectNode(effect) ? [mapLegacyAsciiEffectToCarrierTransform(effect)] : [] ); + const hasTimestampOverlay = semanticOverlayNodes.some((node) => node.type === "timestamp"); + if (!hasTimestampOverlay) { + const legacyTimestamp = migrateOutputTimestampToSemanticOverlay(state.output); + if (legacyTimestamp) { + semanticOverlayNodes.push(legacyTimestamp); + } + } + return { ...cloneImageRenderValue({ geometry: state.geometry, @@ -436,6 +533,7 @@ export const normalizeCanvasImageRenderState = ( }), carrierTransforms: migratedCarrierTransforms, signalDamage: signalDamageNodes, + semanticOverlays: semanticOverlayNodes, effects: rasterEffects, }; }; @@ -512,3 +610,7 @@ export const resolveImageCarrierTransforms = ( export const resolveImageSignalDamage = ( signalDamage: readonly SignalDamageNode[] ) => signalDamage.filter((node) => node.enabled); + +export const resolveImageSemanticOverlays = ( + semanticOverlays: readonly SemanticOverlayNode[] +) => semanticOverlays.filter((node) => node.enabled); diff --git a/src/types/index.ts b/src/types/index.ts index 285ca7a1..52733c32 100644 --- a/src/types/index.ts +++ b/src/types/index.ts @@ -257,6 +257,41 @@ export interface ChannelDriftAdjustments { intensity: number; } +export type TimestampOverlayPosition = "bottom-right" | "bottom-left" | "top-right" | "top-left"; + +export interface TimestampAdjustments { + enabled: boolean; + position: TimestampOverlayPosition; + size: number; + opacity: number; +} + +export type CaptionOverlayPosition = "top" | "bottom" | "center"; +export type CaptionOverlayAlignment = "left" | "center" | "right"; + +export interface CaptionAdjustments { + enabled: boolean; + text: string; + position: CaptionOverlayPosition; + alignment: CaptionOverlayAlignment; + fontSize: number; + color: string; + backgroundColor: string; + backgroundOpacity: number; + padding: number; + opacity: number; +} + +export interface WatermarkAdjustments { + enabled: boolean; + text: string; + opacity: number; + fontSize: number; + angle: number; + density: number; + color: string; +} + export interface LocalAdjustmentDelta { exposure?: number; contrast?: number; From 3e09195d537cef570243164bb0a9504b2672962b Mon Sep 17 00:00:00 2001 From: WenHao Chen <157730893+YakiHugo@users.noreply.github.com> Date: Sat, 25 Apr 2026 00:50:31 -0700 Subject: [PATCH 05/14] feat(renderer): caption and watermark overlay renderers Add CPU canvas renderers for caption (positioned text bar with background) and watermark (tiled rotated text pattern). Overlay execution pipeline dispatches both through the same GPU-blend path as timestamp, preserving preview/export parity. Co-Authored-By: Claude Opus 4.6 (1M context) --- src/lib/captionOverlay.ts | 109 ++++++++++++++++++++++++ src/lib/watermarkOverlay.ts | 83 +++++++++++++++++++ src/render/image/overlayExecution.ts | 114 +++++++++++++++++++++----- src/render/image/renderSingleImage.ts | 2 +- 4 files changed, 285 insertions(+), 23 deletions(-) create mode 100644 src/lib/captionOverlay.ts create mode 100644 src/lib/watermarkOverlay.ts diff --git a/src/lib/captionOverlay.ts b/src/lib/captionOverlay.ts new file mode 100644 index 00000000..1ad00d8c --- /dev/null +++ b/src/lib/captionOverlay.ts @@ -0,0 +1,109 @@ +import { clamp } from "@/lib/math"; +import type { CaptionOverlayAlignment, CaptionOverlayPosition } from "@/types"; + +export interface CaptionOverlayRenderParams { + text: string; + position: CaptionOverlayPosition; + alignment: CaptionOverlayAlignment; + fontSize: number; + color: string; + backgroundColor: string; + backgroundOpacity: number; + padding: number; + opacity: number; +} + +const CAPTION_FONTS = '"Space Grotesk", "Work Sans", sans-serif'; + +const ensureCanvasSize = (canvas: HTMLCanvasElement, width: number, height: number) => { + const safeWidth = Math.max(1, Math.round(width)); + const safeHeight = Math.max(1, Math.round(height)); + if (canvas.width !== safeWidth) canvas.width = safeWidth; + if (canvas.height !== safeHeight) canvas.height = safeHeight; +}; + +export const renderCaptionOverlayRaster = async ({ + width, + height, + params, +}: { + width: number; + height: number; + params: CaptionOverlayRenderParams; +}): Promise => { + if (typeof document === "undefined") return null; + + const safeWidth = Math.max(1, Math.round(width)); + const safeHeight = Math.max(1, Math.round(height)); + const text = (params.text ?? "").trim(); + if (!text) return null; + + const alpha = clamp(params.opacity / 100, 0, 1); + if (alpha <= 0.001) return null; + + const canvas = document.createElement("canvas"); + ensureCanvasSize(canvas, safeWidth, safeHeight); + const ctx = canvas.getContext("2d"); + if (!ctx) { + canvas.width = 0; + canvas.height = 0; + return null; + } + + const fontSize = clamp(params.fontSize, 12, 72); + const padding = clamp(params.padding, 0, 100); + const margin = Math.max(12, Math.round(Math.min(safeWidth, safeHeight) * 0.04)); + + ctx.save(); + ctx.globalAlpha = alpha; + ctx.font = `${Math.round(fontSize)}px ${CAPTION_FONTS}`; + + const textHeight = fontSize * 1.2; + const bgHeight = textHeight + padding * 2; + + let bgTop: number; + switch (params.position) { + case "top": + bgTop = 0; + break; + case "center": + bgTop = (safeHeight - bgHeight) / 2; + break; + case "bottom": + default: + bgTop = safeHeight - bgHeight; + break; + } + + if (params.backgroundOpacity > 0) { + const bgAlpha = clamp(params.backgroundOpacity / 100, 0, 1); + ctx.globalAlpha = alpha * bgAlpha; + ctx.fillStyle = params.backgroundColor; + ctx.fillRect(0, bgTop, safeWidth, bgHeight); + ctx.globalAlpha = alpha; + } + + ctx.fillStyle = params.color; + ctx.textBaseline = "middle"; + + let textX: number; + switch (params.alignment) { + case "left": + ctx.textAlign = "left"; + textX = margin; + break; + case "right": + ctx.textAlign = "right"; + textX = safeWidth - margin; + break; + case "center": + default: + ctx.textAlign = "center"; + textX = safeWidth / 2; + break; + } + + ctx.fillText(text, textX, bgTop + bgHeight / 2); + ctx.restore(); + return canvas; +}; diff --git a/src/lib/watermarkOverlay.ts b/src/lib/watermarkOverlay.ts new file mode 100644 index 00000000..beb9f411 --- /dev/null +++ b/src/lib/watermarkOverlay.ts @@ -0,0 +1,83 @@ +import { clamp } from "@/lib/math"; + +export interface WatermarkOverlayRenderParams { + text: string; + opacity: number; + fontSize: number; + angle: number; + density: number; + color: string; +} + +const WATERMARK_FONTS = '"Space Grotesk", "Work Sans", sans-serif'; + +const ensureCanvasSize = (canvas: HTMLCanvasElement, width: number, height: number) => { + const safeWidth = Math.max(1, Math.round(width)); + const safeHeight = Math.max(1, Math.round(height)); + if (canvas.width !== safeWidth) canvas.width = safeWidth; + if (canvas.height !== safeHeight) canvas.height = safeHeight; +}; + +export const renderWatermarkOverlayRaster = async ({ + width, + height, + params, +}: { + width: number; + height: number; + params: WatermarkOverlayRenderParams; +}): Promise => { + if (typeof document === "undefined") return null; + + const safeWidth = Math.max(1, Math.round(width)); + const safeHeight = Math.max(1, Math.round(height)); + const text = (params.text ?? "").trim(); + if (!text) return null; + + const alpha = clamp(params.opacity / 100, 0, 1); + if (alpha <= 0.001) return null; + + const canvas = document.createElement("canvas"); + ensureCanvasSize(canvas, safeWidth, safeHeight); + const ctx = canvas.getContext("2d"); + if (!ctx) { + canvas.width = 0; + canvas.height = 0; + return null; + } + + const fontSize = clamp(params.fontSize, 12, 120); + const angleRad = (params.angle * Math.PI) / 180; + const density = clamp(params.density, 0.5, 5); + + ctx.save(); + ctx.globalAlpha = alpha; + ctx.font = `${Math.round(fontSize)}px ${WATERMARK_FONTS}`; + ctx.fillStyle = params.color; + ctx.textAlign = "center"; + ctx.textBaseline = "middle"; + + const metrics = ctx.measureText(text); + const textWidth = metrics.width; + const spacingX = (textWidth + fontSize * 2) / density; + const spacingY = (fontSize * 3) / density; + + const diagonal = Math.sqrt(safeWidth * safeWidth + safeHeight * safeHeight); + const cols = Math.ceil(diagonal / spacingX) + 2; + const rows = Math.ceil(diagonal / spacingY) + 2; + + ctx.translate(safeWidth / 2, safeHeight / 2); + ctx.rotate(angleRad); + + const startX = -(cols * spacingX) / 2; + const startY = -(rows * spacingY) / 2; + + for (let row = 0; row < rows; row++) { + for (let col = 0; col < cols; col++) { + ctx.fillText(text, startX + col * spacingX, startY + row * spacingY); + } + } + + ctx.restore(); + return canvas; +}; diff --git a/src/render/image/overlayExecution.ts b/src/render/image/overlayExecution.ts index c3092f15..9fcafd63 100644 --- a/src/render/image/overlayExecution.ts +++ b/src/render/image/overlayExecution.ts @@ -1,11 +1,24 @@ import type { RenderSurfaceHandle } from "@/lib/renderSurfaceHandle"; import { blendCanvasLayerOnGpuToSurface } from "@/lib/renderer/gpuCanvasLayerBlend"; +import { + renderCaptionOverlayRaster, + type CaptionOverlayRenderParams, +} from "@/lib/captionOverlay"; import { applyTimestampOverlay, applyTimestampOverlayToSurfaceIfSupported, type TimestampOverlayAdjustments, } from "@/lib/timestampOverlay"; -import type { ImageRenderOutputState } from "./types"; +import { + renderWatermarkOverlayRaster, + type WatermarkOverlayRenderParams, +} from "@/lib/watermarkOverlay"; +import type { + CaptionSemanticOverlayNode, + SemanticOverlayNode, + TimestampSemanticOverlayNode, + WatermarkSemanticOverlayNode, +} from "./types"; interface TimestampImageOverlay { type: "timestamp"; @@ -13,7 +26,17 @@ interface TimestampImageOverlay { text?: string | null; } -export type ImageOverlayNode = TimestampImageOverlay; +interface CaptionImageOverlay { + type: "caption"; + params: CaptionOverlayRenderParams; +} + +interface WatermarkImageOverlay { + type: "watermark"; + params: WatermarkOverlayRenderParams; +} + +export type ImageOverlayNode = TimestampImageOverlay | CaptionImageOverlay | WatermarkImageOverlay; const ensureCanvasSize = (canvas: HTMLCanvasElement, width: number, height: number) => { const safeWidth = Math.max(1, Math.round(width)); @@ -26,31 +49,70 @@ const ensureCanvasSize = (canvas: HTMLCanvasElement, width: number, height: numb } }; -const createTimestampAdjustmentsFromOutput = ( - output: ImageRenderOutputState +const createTimestampAdjustmentsFromOverlayNode = ( + node: TimestampSemanticOverlayNode ): TimestampOverlayAdjustments => ({ - timestampEnabled: output.timestamp.enabled, - timestampOpacity: output.timestamp.opacity, - timestampPosition: output.timestamp.position, - timestampSize: output.timestamp.size, + timestampEnabled: node.enabled, + timestampOpacity: node.params.opacity, + timestampPosition: node.params.position, + timestampSize: node.params.size, }); +const createCaptionRenderParams = ( + node: CaptionSemanticOverlayNode +): CaptionOverlayRenderParams => ({ ...node.params }); + +const createWatermarkRenderParams = ( + node: WatermarkSemanticOverlayNode +): WatermarkOverlayRenderParams => ({ ...node.params }); + export const resolveImageOverlays = ({ - output, + semanticOverlays, timestampText, }: { - output: ImageRenderOutputState; + semanticOverlays: readonly SemanticOverlayNode[]; timestampText?: string | null; -}): ImageOverlayNode[] => - output.timestamp.enabled - ? [ - { +}): ImageOverlayNode[] => { + const overlays: ImageOverlayNode[] = []; + for (const node of semanticOverlays) { + if (!node.enabled) { + continue; + } + switch (node.type) { + case "timestamp": + overlays.push({ type: "timestamp", - adjustments: createTimestampAdjustmentsFromOutput(output), + adjustments: createTimestampAdjustmentsFromOverlayNode(node), text: timestampText, - }, - ] - : []; + }); + break; + case "caption": + overlays.push({ + type: "caption", + params: createCaptionRenderParams(node), + }); + break; + case "watermark": + overlays.push({ + type: "watermark", + params: createWatermarkRenderParams(node), + }); + break; + } + } + return overlays; +}; + +const drawRasterToCanvas = ( + target: HTMLCanvasElement, + raster: HTMLCanvasElement | null +) => { + if (!raster) return; + const ctx = target.getContext("2d"); + if (ctx) ctx.drawImage(raster, 0, 0); + raster.width = 0; + raster.height = 0; +}; const renderOverlayToCanvas = async ( overlay: ImageOverlayNode, @@ -63,6 +125,18 @@ const renderOverlayToCanvas = async ( case "timestamp": await applyTimestampOverlay(overlayCanvas, overlay.adjustments, overlay.text); break; + case "caption": + drawRasterToCanvas( + overlayCanvas, + await renderCaptionOverlayRaster({ width, height, params: overlay.params }) + ); + break; + case "watermark": + drawRasterToCanvas( + overlayCanvas, + await renderWatermarkOverlayRaster({ width, height, params: overlay.params }) + ); + break; } return overlayCanvas; }; @@ -95,10 +169,6 @@ export const applyImageOverlays = async ({ } } - // GPU-direct rasterization unavailable — bake the overlay to a Canvas2D - // layer and composite it back onto the surface via a GPU blend. The CPU - // island stays bounded inside this stage: input is a Surface, output is a - // Surface. const overlayCanvas = await renderOverlayToCanvas( overlay, currentSurface.width, diff --git a/src/render/image/renderSingleImage.ts b/src/render/image/renderSingleImage.ts index a7ac9d8b..be476fff 100644 --- a/src/render/image/renderSingleImage.ts +++ b/src/render/image/renderSingleImage.ts @@ -337,7 +337,7 @@ export const renderSingleImageToCanvas = async ({ } const overlays = resolveImageOverlays({ - output: document.output, + semanticOverlays: document.semanticOverlays, timestampText: request.timestampText, }); if (overlays.length > 0) { From c96df05be9ca5f3ffade58b6ff832e7a91ce6881 Mon Sep 17 00:00:00 2001 From: WenHao Chen <157730893+YakiHugo@users.noreply.github.com> Date: Sat, 25 Apr 2026 00:50:37 -0700 Subject: [PATCH 06/14] feat(canvas): caption and watermark overlay edit panels Add state editing helpers (resolve/apply/upsert) for caption and watermark semantic overlays. Create CanvasCaptionEditPanel and CanvasWatermarkEditPanel with preview/commit workflow matching halftone/signal-damage pattern. Register both as floating panel types. Co-Authored-By: Claude Opus 4.6 (1M context) --- .../canvas/CanvasCaptionEditPanel.tsx | 337 ++++++++++++++++++ src/features/canvas/CanvasFloatingPanel.tsx | 10 +- .../canvas/CanvasWatermarkEditPanel.tsx | 254 +++++++++++++ .../canvas/imageRenderStateEditing.ts | 304 ++++++++++++++++ src/features/canvas/store/canvasStoreTypes.ts | 2 + 5 files changed, 906 insertions(+), 1 deletion(-) create mode 100644 src/features/canvas/CanvasCaptionEditPanel.tsx create mode 100644 src/features/canvas/CanvasWatermarkEditPanel.tsx diff --git a/src/features/canvas/CanvasCaptionEditPanel.tsx b/src/features/canvas/CanvasCaptionEditPanel.tsx new file mode 100644 index 00000000..dcb3e93e --- /dev/null +++ b/src/features/canvas/CanvasCaptionEditPanel.tsx @@ -0,0 +1,337 @@ +import { useCallback, useEffect, useMemo, useRef, useState } from "react"; +import { + Select, + SelectContent, + SelectItem, + SelectTrigger, + SelectValue, +} from "@/components/ui/select"; +import { CanvasEditSection } from "@/features/canvas/components/CanvasEditSection"; +import { SliderControl } from "@/features/canvas/components/controls/SliderControl"; +import { cn } from "@/lib/utils"; +import { + useCanvasElementDraftRenderState, + useCanvasPreviewActions, +} from "@/features/canvas/runtime/canvasRuntimeHooks"; +import { resolveCanvasImageRenderState } from "@/features/canvas/imageRenderState"; +import type { CanvasImageRenderStateV1 } from "@/render/image"; +import type { CaptionAdjustments } from "@/types"; +import { useCanvasStore } from "@/stores/canvasStore"; +import { + canvasDockBodyTextClassName, + canvasDockSelectContentClassName, + canvasDockSelectTriggerClassName, +} from "./editDockTheme"; +import { + canvasEditTargetEqual, + resolveCanvasEditTargetFromPrimarySelection, + type CanvasImageEditTarget, +} from "./editPanelSelection"; +import { selectLoadedWorkbench } from "./store/canvasStoreSelectors"; +import { + applyCaptionAdjustmentsToRenderState, + DEFAULT_CANVAS_CAPTION_ADJUSTMENTS, + getCanvasImageEditValues, +} from "./imageRenderStateEditing"; +import { useCanvasImagePropertyActions } from "./hooks/useCanvasImagePropertyActions"; + +type CaptionSectionId = "text" | "layout" | "appearance"; + +const POSITION_OPTIONS: Array<{ + label: string; + value: CaptionAdjustments["position"]; +}> = [ + { label: "Top", value: "top" }, + { label: "Center", value: "center" }, + { label: "Bottom", value: "bottom" }, +]; + +const ALIGNMENT_OPTIONS: Array<{ + label: string; + value: CaptionAdjustments["alignment"]; +}> = [ + { label: "Left", value: "left" }, + { label: "Center", value: "center" }, + { label: "Right", value: "right" }, +]; + +type CaptionNumericKey = "fontSize" | "padding" | "opacity" | "backgroundOpacity"; + +interface CaptionSliderDef { + key: CaptionNumericKey; + label: string; + min: number; + max: number; + step?: number; + format?: (value: number) => string; +} + +const LAYOUT_SLIDERS: CaptionSliderDef[] = [ + { key: "fontSize", label: "Font Size", min: 12, max: 72, step: 1 }, + { key: "padding", label: "Padding", min: 0, max: 100, step: 1 }, +]; + +const APPEARANCE_SLIDERS: CaptionSliderDef[] = [ + { key: "opacity", label: "Opacity", min: 0, max: 100, step: 1 }, + { key: "backgroundOpacity", label: "Background Opacity", min: 0, max: 100, step: 1 }, +]; + +function useCanvasEditImageTarget(): CanvasImageEditTarget | null { + const primarySelectedElementId = useCanvasStore( + (state) => state.selectedElementIds[0] ?? null + ); + const selectEditTarget = useCallback( + (state: Parameters[0]) => { + const target = resolveCanvasEditTargetFromPrimarySelection( + selectLoadedWorkbench(state), + primarySelectedElementId + ); + return target?.type === "image" ? target : null; + }, + [primarySelectedElementId] + ); + return useCanvasStore(selectEditTarget, canvasEditTargetEqual); +} + +export function CanvasCaptionEditPanel() { + const imageElement = useCanvasEditImageTarget(); + + if (!imageElement) { + return ( +
+
+

+ Select an image on the canvas to add a caption overlay. +

+
+
+ ); + } + + return ; +} + +function CanvasCaptionEditPanelForImage({ + imageElement, +}: { + imageElement: CanvasImageEditTarget; +}) { + const { + clearElementDraftRenderState, + requestBoardPreview, + setElementDraftRenderState, + } = useCanvasPreviewActions(); + const { setRenderState } = useCanvasImagePropertyActions(imageElement); + const [openSections, setOpenSections] = useState>(() => ({ + text: true, + layout: true, + appearance: true, + })); + + const committedImageElementId = imageElement.id; + const committedImageElementIdRef = useRef(committedImageElementId); + const draftRenderState = useCanvasElementDraftRenderState(committedImageElementId); + + const renderState = useMemo( + () => resolveCanvasImageRenderState(imageElement, draftRenderState), + [draftRenderState, imageElement] + ); + const fieldValues = useMemo(() => getCanvasImageEditValues(renderState), [renderState]); + const renderStateRef = useRef(renderState); + renderStateRef.current = renderState; + + useEffect(() => { + const previous = committedImageElementIdRef.current; + if (previous && previous !== committedImageElementId) { + clearElementDraftRenderState(previous); + } + committedImageElementIdRef.current = committedImageElementId; + }, [clearElementDraftRenderState, committedImageElementId]); + + useEffect( + () => () => { + const current = committedImageElementIdRef.current; + if (current) { + clearElementDraftRenderState(current); + } + }, + [clearElementDraftRenderState] + ); + + const previewRenderState = useCallback( + (nextRenderState: CanvasImageRenderStateV1) => { + setElementDraftRenderState(imageElement.id, nextRenderState); + void requestBoardPreview(imageElement.id, "interactive"); + }, + [imageElement.id, requestBoardPreview, setElementDraftRenderState] + ); + + const commitAdjustments = useCallback( + async (nextRenderState: CanvasImageRenderStateV1) => { + setElementDraftRenderState(imageElement.id, nextRenderState); + await setRenderState(nextRenderState); + clearElementDraftRenderState(imageElement.id); + }, + [ + clearElementDraftRenderState, + imageElement.id, + setElementDraftRenderState, + setRenderState, + ] + ); + + const toggleSection = useCallback((sectionId: CaptionSectionId) => { + setOpenSections((current) => ({ + ...current, + [sectionId]: !current[sectionId], + })); + }, []); + + const captionAdjustments = fieldValues.caption ?? DEFAULT_CANVAS_CAPTION_ADJUSTMENTS; + const captionEnabled = captionAdjustments.enabled; + + const updateCaptionAdjustments = useCallback( + (partial: Partial, mode: "preview" | "commit" = "commit") => { + const nextRenderState = applyCaptionAdjustmentsToRenderState( + renderStateRef.current!, + partial + ); + if (mode === "preview") { + previewRenderState(nextRenderState); + } else { + void commitAdjustments(nextRenderState); + } + }, + [commitAdjustments, previewRenderState] + ); + + const renderSlider = (slider: CaptionSliderDef) => ( + updateCaptionAdjustments({ [slider.key]: value }, "preview")} + onCommit={(value: number) => updateCaptionAdjustments({ [slider.key]: value })} + /> + ); + + return ( +
+
+ toggleSection("text")} + canResetChanges + onResetChanges={() => updateCaptionAdjustments({ ...DEFAULT_CANVAS_CAPTION_ADJUSTMENTS })} + > +
+ +
+ Text + + updateCaptionAdjustments({ text: e.target.value }, "preview") + } + onBlur={(e) => updateCaptionAdjustments({ text: e.target.value })} + disabled={!captionEnabled} + placeholder="Enter caption text..." + className={cn( + "h-10 w-full rounded-[8px] border border-[color:var(--canvas-edit-border)] bg-[color:var(--canvas-edit-surface)] px-3 text-sm text-[color:var(--canvas-edit-text)] placeholder:text-[color:var(--canvas-edit-text-muted)]", + !captionEnabled && "opacity-50" + )} + /> +
+
+ + + toggleSection("layout")} + > +
+
+ Position + +
+
+ Alignment + +
+ {LAYOUT_SLIDERS.map(renderSlider)} +
+ + + toggleSection("appearance")} + > +
+ {APPEARANCE_SLIDERS.map(renderSlider)} +
+ +
+
+ ); +} diff --git a/src/features/canvas/CanvasFloatingPanel.tsx b/src/features/canvas/CanvasFloatingPanel.tsx index 9d69c080..d7da113c 100644 --- a/src/features/canvas/CanvasFloatingPanel.tsx +++ b/src/features/canvas/CanvasFloatingPanel.tsx @@ -3,8 +3,10 @@ import { AnimatePresence, motion } from "framer-motion"; import { cn } from "@/lib/utils"; import { useCanvasStore, type CanvasFloatingPanel as PanelType } from "@/stores/canvasStore"; import { CanvasAsciiEditPanel } from "./CanvasAsciiEditPanel"; +import { CanvasCaptionEditPanel } from "./CanvasCaptionEditPanel"; import { CanvasHalftoneEditPanel } from "./CanvasHalftoneEditPanel"; import { CanvasSignalDamageEditPanel } from "./CanvasSignalDamageEditPanel"; +import { CanvasWatermarkEditPanel } from "./CanvasWatermarkEditPanel"; import { CanvasAssetPicker } from "./CanvasAssetPicker"; import { CanvasEditPanel } from "./CanvasEditPanel"; import { CanvasLayerPanel } from "./CanvasLayerPanel"; @@ -19,6 +21,8 @@ const PANEL_TITLES: Record, string> = { ascii: "ASCII", halftone: "Halftone", "signal-damage": "Signal Damage", + caption: "Caption", + watermark: "Watermark", layers: "Layers", library: "Library", }; @@ -26,7 +30,7 @@ const PANEL_TITLES: Record, string> = { export function CanvasFloatingPanel() { const activePanel = useCanvasStore((s) => s.activePanel); const setActivePanel = useCanvasStore((s) => s.setActivePanel); - const isEditDock = activePanel === "edit" || activePanel === "ascii" || activePanel === "halftone" || activePanel === "signal-damage"; + const isEditDock = activePanel === "edit" || activePanel === "ascii" || activePanel === "halftone" || activePanel === "signal-damage" || activePanel === "caption" || activePanel === "watermark"; return ( @@ -84,6 +88,10 @@ function PanelContent({ return ; case "signal-damage": return ; + case "caption": + return ; + case "watermark": + return ; case "layers": return ; case "library": diff --git a/src/features/canvas/CanvasWatermarkEditPanel.tsx b/src/features/canvas/CanvasWatermarkEditPanel.tsx new file mode 100644 index 00000000..3b59ee99 --- /dev/null +++ b/src/features/canvas/CanvasWatermarkEditPanel.tsx @@ -0,0 +1,254 @@ +import { useCallback, useEffect, useMemo, useRef, useState } from "react"; +import { CanvasEditSection } from "@/features/canvas/components/CanvasEditSection"; +import { SliderControl } from "@/features/canvas/components/controls/SliderControl"; +import { cn } from "@/lib/utils"; +import { + useCanvasElementDraftRenderState, + useCanvasPreviewActions, +} from "@/features/canvas/runtime/canvasRuntimeHooks"; +import { resolveCanvasImageRenderState } from "@/features/canvas/imageRenderState"; +import type { CanvasImageRenderStateV1 } from "@/render/image"; +import type { WatermarkAdjustments } from "@/types"; +import { useCanvasStore } from "@/stores/canvasStore"; +import { canvasDockBodyTextClassName } from "./editDockTheme"; +import { + canvasEditTargetEqual, + resolveCanvasEditTargetFromPrimarySelection, + type CanvasImageEditTarget, +} from "./editPanelSelection"; +import { selectLoadedWorkbench } from "./store/canvasStoreSelectors"; +import { + applyWatermarkAdjustmentsToRenderState, + DEFAULT_CANVAS_WATERMARK_ADJUSTMENTS, + getCanvasImageEditValues, +} from "./imageRenderStateEditing"; +import { useCanvasImagePropertyActions } from "./hooks/useCanvasImagePropertyActions"; + +type WatermarkSectionId = "text" | "appearance"; + +type WatermarkNumericKey = "opacity" | "fontSize" | "angle" | "density"; + +const formatAngle = (value: number) => `${Math.round(value)}°`; +const formatDensity = (value: number) => value.toFixed(1); + +interface WatermarkSliderDef { + key: WatermarkNumericKey; + label: string; + min: number; + max: number; + step?: number; + format?: (value: number) => string; +} + +const APPEARANCE_SLIDERS: WatermarkSliderDef[] = [ + { key: "fontSize", label: "Font Size", min: 12, max: 120, step: 1 }, + { key: "angle", label: "Angle", min: -90, max: 90, step: 1, format: formatAngle }, + { key: "density", label: "Density", min: 0.5, max: 5, step: 0.1, format: formatDensity }, + { key: "opacity", label: "Opacity", min: 0, max: 100, step: 1 }, +]; + +function useCanvasEditImageTarget(): CanvasImageEditTarget | null { + const primarySelectedElementId = useCanvasStore( + (state) => state.selectedElementIds[0] ?? null + ); + const selectEditTarget = useCallback( + (state: Parameters[0]) => { + const target = resolveCanvasEditTargetFromPrimarySelection( + selectLoadedWorkbench(state), + primarySelectedElementId + ); + return target?.type === "image" ? target : null; + }, + [primarySelectedElementId] + ); + return useCanvasStore(selectEditTarget, canvasEditTargetEqual); +} + +export function CanvasWatermarkEditPanel() { + const imageElement = useCanvasEditImageTarget(); + + if (!imageElement) { + return ( +
+
+

+ Select an image on the canvas to add a watermark overlay. +

+
+
+ ); + } + + return ; +} + +function CanvasWatermarkEditPanelForImage({ + imageElement, +}: { + imageElement: CanvasImageEditTarget; +}) { + const { + clearElementDraftRenderState, + requestBoardPreview, + setElementDraftRenderState, + } = useCanvasPreviewActions(); + const { setRenderState } = useCanvasImagePropertyActions(imageElement); + const [openSections, setOpenSections] = useState>(() => ({ + text: true, + appearance: true, + })); + + const committedImageElementId = imageElement.id; + const committedImageElementIdRef = useRef(committedImageElementId); + const draftRenderState = useCanvasElementDraftRenderState(committedImageElementId); + + const renderState = useMemo( + () => resolveCanvasImageRenderState(imageElement, draftRenderState), + [draftRenderState, imageElement] + ); + const fieldValues = useMemo(() => getCanvasImageEditValues(renderState), [renderState]); + const renderStateRef = useRef(renderState); + renderStateRef.current = renderState; + + useEffect(() => { + const previous = committedImageElementIdRef.current; + if (previous && previous !== committedImageElementId) { + clearElementDraftRenderState(previous); + } + committedImageElementIdRef.current = committedImageElementId; + }, [clearElementDraftRenderState, committedImageElementId]); + + useEffect( + () => () => { + const current = committedImageElementIdRef.current; + if (current) { + clearElementDraftRenderState(current); + } + }, + [clearElementDraftRenderState] + ); + + const previewRenderState = useCallback( + (nextRenderState: CanvasImageRenderStateV1) => { + setElementDraftRenderState(imageElement.id, nextRenderState); + void requestBoardPreview(imageElement.id, "interactive"); + }, + [imageElement.id, requestBoardPreview, setElementDraftRenderState] + ); + + const commitAdjustments = useCallback( + async (nextRenderState: CanvasImageRenderStateV1) => { + setElementDraftRenderState(imageElement.id, nextRenderState); + await setRenderState(nextRenderState); + clearElementDraftRenderState(imageElement.id); + }, + [ + clearElementDraftRenderState, + imageElement.id, + setElementDraftRenderState, + setRenderState, + ] + ); + + const toggleSection = useCallback((sectionId: WatermarkSectionId) => { + setOpenSections((current) => ({ + ...current, + [sectionId]: !current[sectionId], + })); + }, []); + + const watermarkAdjustments = fieldValues.watermark ?? DEFAULT_CANVAS_WATERMARK_ADJUSTMENTS; + const watermarkEnabled = watermarkAdjustments.enabled; + + const updateWatermarkAdjustments = useCallback( + (partial: Partial, mode: "preview" | "commit" = "commit") => { + const nextRenderState = applyWatermarkAdjustmentsToRenderState( + renderStateRef.current!, + partial + ); + if (mode === "preview") { + previewRenderState(nextRenderState); + } else { + void commitAdjustments(nextRenderState); + } + }, + [commitAdjustments, previewRenderState] + ); + + const renderSlider = (slider: WatermarkSliderDef) => ( + updateWatermarkAdjustments({ [slider.key]: value }, "preview")} + onCommit={(value: number) => updateWatermarkAdjustments({ [slider.key]: value })} + /> + ); + + return ( +
+
+ toggleSection("text")} + canResetChanges + onResetChanges={() => + updateWatermarkAdjustments({ ...DEFAULT_CANVAS_WATERMARK_ADJUSTMENTS }) + } + > +
+ +
+ Text + + updateWatermarkAdjustments({ text: e.target.value }, "preview") + } + onBlur={(e) => updateWatermarkAdjustments({ text: e.target.value })} + disabled={!watermarkEnabled} + placeholder="DRAFT" + className={cn( + "h-10 w-full rounded-[8px] border border-[color:var(--canvas-edit-border)] bg-[color:var(--canvas-edit-surface)] px-3 text-sm text-[color:var(--canvas-edit-text)] placeholder:text-[color:var(--canvas-edit-text-muted)]", + !watermarkEnabled && "opacity-50" + )} + /> +
+
+ + + toggleSection("appearance")} + > +
+ {APPEARANCE_SLIDERS.map(renderSlider)} +
+ +
+
+ ); +} diff --git a/src/features/canvas/imageRenderStateEditing.ts b/src/features/canvas/imageRenderStateEditing.ts index 2a49817a..f393495f 100644 --- a/src/features/canvas/imageRenderStateEditing.ts +++ b/src/features/canvas/imageRenderStateEditing.ts @@ -7,10 +7,16 @@ import type { AsciiDitherMode, AsciiForegroundBlendMode, AsciiRenderMode, + CaptionAdjustments, + CaptionOverlayAlignment, + CaptionOverlayPosition, ChannelDriftAdjustments, HalftoneAdjustments, HalftoneColorMode, HalftoneShape, + TimestampAdjustments, + TimestampOverlayPosition, + WatermarkAdjustments, } from "@/types"; import { createNeutralCanvasImageRenderState, @@ -18,6 +24,7 @@ import { normalizeCanvasImageRenderState, type CarrierTransformNode, type ImageFilter2dEffectNode, + type SemanticOverlayNode, type SignalDamageNode, } from "@/render/image"; @@ -47,6 +54,18 @@ const isBackgroundMode = (value: unknown): value is AsciiBackgroundMode => const isForegroundBlendMode = (value: unknown): value is AsciiForegroundBlendMode => typeof value === "string" && (FOREGROUND_BLEND_VALUES as readonly string[]).includes(value); +const TIMESTAMP_POSITION_VALUES = ["bottom-right", "bottom-left", "top-right", "top-left"] as const; +const isTimestampPosition = (value: unknown): value is TimestampOverlayPosition => + typeof value === "string" && (TIMESTAMP_POSITION_VALUES as readonly string[]).includes(value); + +const CAPTION_POSITION_VALUES = ["top", "bottom", "center"] as const; +const isCaptionPosition = (value: unknown): value is CaptionOverlayPosition => + typeof value === "string" && (CAPTION_POSITION_VALUES as readonly string[]).includes(value); + +const CAPTION_ALIGNMENT_VALUES = ["left", "center", "right"] as const; +const isCaptionAlignment = (value: unknown): value is CaptionOverlayAlignment => + typeof value === "string" && (CAPTION_ALIGNMENT_VALUES as readonly string[]).includes(value); + const HALFTONE_SHAPE_VALUES = ["circle", "diamond", "line", "square"] as const; const HALFTONE_COLOR_MODE_VALUES = ["mono", "cmyk", "rgb"] as const; @@ -79,6 +98,36 @@ const DEFAULT_CHANNEL_DRIFT_ADJUSTMENTS: ChannelDriftAdjustments = { intensity: 0.5, }; +const DEFAULT_TIMESTAMP_ADJUSTMENTS: TimestampAdjustments = { + enabled: false, + position: "bottom-right", + size: 22, + opacity: 72, +}; + +const DEFAULT_CAPTION_ADJUSTMENTS: CaptionAdjustments = { + enabled: false, + text: "", + position: "bottom", + alignment: "center", + fontSize: 24, + color: "rgba(255, 250, 242, 0.95)", + backgroundColor: "#000000", + backgroundOpacity: 34, + padding: 16, + opacity: 100, +}; + +const DEFAULT_WATERMARK_ADJUSTMENTS: WatermarkAdjustments = { + enabled: false, + text: "", + opacity: 20, + fontSize: 36, + angle: -30, + density: 1, + color: "rgba(255, 255, 255, 0.8)", +}; + const DEFAULT_ASCII_ADJUSTMENTS: AsciiAdjustments = { enabled: false, // Defaults are tuned to match ascii-magic.com's out-of-the-box look, which @@ -123,6 +172,9 @@ export type CanvasImageEditValues = CanvasImageNumericFieldValues & { ascii: AsciiAdjustments; halftone: HalftoneAdjustments; channelDrift: ChannelDriftAdjustments; + timestamp: TimestampAdjustments; + caption: CaptionAdjustments; + watermark: WatermarkAdjustments; }; const cloneState = (state: CanvasImageRenderStateV1): CanvasImageRenderStateV1 => { @@ -231,6 +283,75 @@ const resolveChannelDriftAdjustmentsFromState = ( }; }; +const resolveTimestampAdjustmentsFromState = ( + state: CanvasImageRenderStateV1 +): TimestampAdjustments => { + const node = normalizeCanvasImageRenderState(state).semanticOverlays.find( + (candidate): candidate is Extract => + candidate.type === "timestamp" + ); + if (!node) { + return { ...DEFAULT_TIMESTAMP_ADJUSTMENTS }; + } + const p = node.params; + return { + ...DEFAULT_TIMESTAMP_ADJUSTMENTS, + enabled: node.enabled, + position: isTimestampPosition(p.position) ? p.position : "bottom-right", + size: typeof p.size === "number" ? p.size : 22, + opacity: typeof p.opacity === "number" ? p.opacity : 72, + }; +}; + +const resolveCaptionAdjustmentsFromState = ( + state: CanvasImageRenderStateV1 +): CaptionAdjustments => { + const node = normalizeCanvasImageRenderState(state).semanticOverlays.find( + (candidate): candidate is Extract => + candidate.type === "caption" + ); + if (!node) { + return { ...DEFAULT_CAPTION_ADJUSTMENTS }; + } + const p = node.params; + return { + ...DEFAULT_CAPTION_ADJUSTMENTS, + enabled: node.enabled, + text: typeof p.text === "string" ? p.text : "", + position: isCaptionPosition(p.position) ? p.position : "bottom", + alignment: isCaptionAlignment(p.alignment) ? p.alignment : "center", + fontSize: typeof p.fontSize === "number" ? p.fontSize : 24, + color: typeof p.color === "string" ? p.color : DEFAULT_CAPTION_ADJUSTMENTS.color, + backgroundColor: typeof p.backgroundColor === "string" ? p.backgroundColor : "#000000", + backgroundOpacity: typeof p.backgroundOpacity === "number" ? p.backgroundOpacity : 34, + padding: typeof p.padding === "number" ? p.padding : 16, + opacity: typeof p.opacity === "number" ? p.opacity : 100, + }; +}; + +const resolveWatermarkAdjustmentsFromState = ( + state: CanvasImageRenderStateV1 +): WatermarkAdjustments => { + const node = normalizeCanvasImageRenderState(state).semanticOverlays.find( + (candidate): candidate is Extract => + candidate.type === "watermark" + ); + if (!node) { + return { ...DEFAULT_WATERMARK_ADJUSTMENTS }; + } + const p = node.params; + return { + ...DEFAULT_WATERMARK_ADJUSTMENTS, + enabled: node.enabled, + text: typeof p.text === "string" ? p.text : "", + opacity: typeof p.opacity === "number" ? p.opacity : 20, + fontSize: typeof p.fontSize === "number" ? p.fontSize : 36, + angle: typeof p.angle === "number" ? p.angle : -30, + density: typeof p.density === "number" ? p.density : 1, + color: typeof p.color === "string" ? p.color : DEFAULT_WATERMARK_ADJUSTMENTS.color, + }; +}; + const resolveFilter2dPreviewValues = (state: CanvasImageRenderStateV1) => { const effect = state.effects.find( (candidate): candidate is ImageFilter2dEffectNode => @@ -284,6 +405,9 @@ const createCanvasImageEditValues = ( ascii: resolveAsciiAdjustmentsFromState(normalizedState), halftone: resolveHalftoneAdjustmentsFromState(normalizedState), channelDrift: resolveChannelDriftAdjustmentsFromState(normalizedState), + timestamp: resolveTimestampAdjustmentsFromState(normalizedState), + caption: resolveCaptionAdjustmentsFromState(normalizedState), + watermark: resolveWatermarkAdjustmentsFromState(normalizedState), }; }; @@ -304,6 +428,18 @@ export const DEFAULT_CANVAS_CHANNEL_DRIFT_ADJUSTMENTS: ChannelDriftAdjustments = ...DEFAULT_CANVAS_IMAGE_EDIT_VALUES.channelDrift, }; +export const DEFAULT_CANVAS_TIMESTAMP_ADJUSTMENTS: TimestampAdjustments = { + ...DEFAULT_CANVAS_IMAGE_EDIT_VALUES.timestamp, +}; + +export const DEFAULT_CANVAS_CAPTION_ADJUSTMENTS: CaptionAdjustments = { + ...DEFAULT_CANVAS_IMAGE_EDIT_VALUES.caption, +}; + +export const DEFAULT_CANVAS_WATERMARK_ADJUSTMENTS: WatermarkAdjustments = { + ...DEFAULT_CANVAS_IMAGE_EDIT_VALUES.watermark, +}; + const createDefaultAsciiCarrierTransform = (): Extract => ({ id: "canvas-ascii", type: "ascii", @@ -448,6 +584,120 @@ const upsertChannelDriftDamage = ( return next; }; +const createDefaultTimestampOverlay = (): Extract< + SemanticOverlayNode, + { type: "timestamp" } +> => ({ + id: "canvas-timestamp", + type: "timestamp", + enabled: false, + params: { + position: DEFAULT_CANVAS_TIMESTAMP_ADJUSTMENTS.position, + size: DEFAULT_CANVAS_TIMESTAMP_ADJUSTMENTS.size, + opacity: DEFAULT_CANVAS_TIMESTAMP_ADJUSTMENTS.opacity, + }, +}); + +const upsertTimestampOverlay = ( + state: CanvasImageRenderStateV1, + updater: ( + node: Extract + ) => Extract +) => { + const next = cloneState(state); + const index = next.semanticOverlays.findIndex((n) => n.type === "timestamp"); + const current = + index >= 0 + ? (next.semanticOverlays[index] as Extract) + : createDefaultTimestampOverlay(); + const updated = updater(current); + if (index >= 0) { + next.semanticOverlays[index] = updated; + } else { + next.semanticOverlays.push(updated); + } + return next; +}; + +const createDefaultCaptionOverlay = (): Extract< + SemanticOverlayNode, + { type: "caption" } +> => ({ + id: "canvas-caption", + type: "caption", + enabled: false, + params: { + text: DEFAULT_CANVAS_CAPTION_ADJUSTMENTS.text, + position: DEFAULT_CANVAS_CAPTION_ADJUSTMENTS.position, + alignment: DEFAULT_CANVAS_CAPTION_ADJUSTMENTS.alignment, + fontSize: DEFAULT_CANVAS_CAPTION_ADJUSTMENTS.fontSize, + color: DEFAULT_CANVAS_CAPTION_ADJUSTMENTS.color, + backgroundColor: DEFAULT_CANVAS_CAPTION_ADJUSTMENTS.backgroundColor, + backgroundOpacity: DEFAULT_CANVAS_CAPTION_ADJUSTMENTS.backgroundOpacity, + padding: DEFAULT_CANVAS_CAPTION_ADJUSTMENTS.padding, + opacity: DEFAULT_CANVAS_CAPTION_ADJUSTMENTS.opacity, + }, +}); + +const upsertCaptionOverlay = ( + state: CanvasImageRenderStateV1, + updater: ( + node: Extract + ) => Extract +) => { + const next = cloneState(state); + const index = next.semanticOverlays.findIndex((n) => n.type === "caption"); + const current = + index >= 0 + ? (next.semanticOverlays[index] as Extract) + : createDefaultCaptionOverlay(); + const updated = updater(current); + if (index >= 0) { + next.semanticOverlays[index] = updated; + } else { + next.semanticOverlays.push(updated); + } + return next; +}; + +const createDefaultWatermarkOverlay = (): Extract< + SemanticOverlayNode, + { type: "watermark" } +> => ({ + id: "canvas-watermark", + type: "watermark", + enabled: false, + params: { + text: DEFAULT_CANVAS_WATERMARK_ADJUSTMENTS.text, + opacity: DEFAULT_CANVAS_WATERMARK_ADJUSTMENTS.opacity, + fontSize: DEFAULT_CANVAS_WATERMARK_ADJUSTMENTS.fontSize, + angle: DEFAULT_CANVAS_WATERMARK_ADJUSTMENTS.angle, + density: DEFAULT_CANVAS_WATERMARK_ADJUSTMENTS.density, + color: DEFAULT_CANVAS_WATERMARK_ADJUSTMENTS.color, + }, +}); + +const upsertWatermarkOverlay = ( + state: CanvasImageRenderStateV1, + updater: ( + node: Extract + ) => Extract +) => { + const next = cloneState(state); + const index = next.semanticOverlays.findIndex((n) => n.type === "watermark"); + const current = + index >= 0 + ? (next.semanticOverlays[index] as Extract) + : createDefaultWatermarkOverlay(); + const updated = updater(current); + if (index >= 0) { + next.semanticOverlays[index] = updated; + } else { + next.semanticOverlays.push(updated); + } + return next; +}; + const upsertFilter2dEffect = ( state: CanvasImageRenderStateV1, updater: (effect: ImageFilter2dEffectNode) => ImageFilter2dEffectNode @@ -646,6 +896,60 @@ export const applyChannelDriftAdjustmentsToRenderState = ( }, })); +export const applyTimestampAdjustmentsToRenderState = ( + state: CanvasImageRenderStateV1, + partial: Partial +) => + upsertTimestampOverlay(state, (node) => ({ + ...node, + enabled: partial.enabled ?? node.enabled, + params: { + ...node.params, + position: partial.position ?? node.params.position, + size: partial.size ?? node.params.size, + opacity: partial.opacity ?? node.params.opacity, + }, + })); + +export const applyCaptionAdjustmentsToRenderState = ( + state: CanvasImageRenderStateV1, + partial: Partial +) => + upsertCaptionOverlay(state, (node) => ({ + ...node, + enabled: partial.enabled ?? node.enabled, + params: { + ...node.params, + text: partial.text ?? node.params.text, + position: partial.position ?? node.params.position, + alignment: partial.alignment ?? node.params.alignment, + fontSize: partial.fontSize ?? node.params.fontSize, + color: partial.color ?? node.params.color, + backgroundColor: partial.backgroundColor ?? node.params.backgroundColor, + backgroundOpacity: partial.backgroundOpacity ?? node.params.backgroundOpacity, + padding: partial.padding ?? node.params.padding, + opacity: partial.opacity ?? node.params.opacity, + }, + })); + +export const applyWatermarkAdjustmentsToRenderState = ( + state: CanvasImageRenderStateV1, + partial: Partial +) => + upsertWatermarkOverlay(state, (node) => ({ + ...node, + enabled: partial.enabled ?? node.enabled, + params: { + ...node.params, + text: partial.text ?? node.params.text, + opacity: partial.opacity ?? node.params.opacity, + fontSize: partial.fontSize ?? node.params.fontSize, + angle: partial.angle ?? node.params.angle, + density: partial.density ?? node.params.density, + color: partial.color ?? node.params.color, + }, + })); + export const resetRenderStateForNumericFields = ( state: CanvasImageRenderStateV1, fieldIds: CanvasImageNumericFieldId[] diff --git a/src/features/canvas/store/canvasStoreTypes.ts b/src/features/canvas/store/canvasStoreTypes.ts index d08da7f3..5a50c768 100644 --- a/src/features/canvas/store/canvasStoreTypes.ts +++ b/src/features/canvas/store/canvasStoreTypes.ts @@ -13,6 +13,8 @@ export type CanvasFloatingPanel = | "ascii" | "halftone" | "signal-damage" + | "caption" + | "watermark" | "layers" | "library" | null; From 435870c136cb5a9886048f29f260cdda67ab4b2b Mon Sep 17 00:00:00 2001 From: WenHao Chen <157730893+YakiHugo@users.noreply.github.com> Date: Sat, 25 Apr 2026 00:50:42 -0700 Subject: [PATCH 07/14] docs: mark semantic-overlay-layer-system slice done Caption and watermark overlays satisfy the pass criteria. Update handoff notes with implementation details and remaining follow-ups. Co-Authored-By: Claude Opus 4.6 (1M context) --- docs/tasks/media-native-render-pipeline.json | 2 +- docs/tasks/media-native-render-pipeline.md | 34 ++++++++++++++++---- 2 files changed, 29 insertions(+), 7 deletions(-) diff --git a/docs/tasks/media-native-render-pipeline.json b/docs/tasks/media-native-render-pipeline.json index 4a80378e..4bbc09e1 100644 --- a/docs/tasks/media-native-render-pipeline.json +++ b/docs/tasks/media-native-render-pipeline.json @@ -3,7 +3,7 @@ "tasks": [ { "id": "semantic-overlay-layer-system", - "status": "pending", + "status": "done", "blockedBy": [], "passes": [ "caption, HUD, browser-chrome, or similar overlays have an authored model with canvas preview/export parity" diff --git a/docs/tasks/media-native-render-pipeline.md b/docs/tasks/media-native-render-pipeline.md index 2bb4e3a5..0f0ca354 100644 --- a/docs/tasks/media-native-render-pipeline.md +++ b/docs/tasks/media-native-render-pipeline.md @@ -111,7 +111,7 @@ ## Current Focus - `carrier-and-signal-families` slice is complete. -- Next active slice: `semantic-overlay-layer-system` or `analysis-layer-boundary`. +- `semantic-overlay-layer-system` slice is complete — authored overlay model with timestamp, caption, and watermark types; concrete rendering and UI panels landed. ## Files @@ -127,11 +127,15 @@ - `src/lib/renderer/gpuSignalDamage.ts` - `src/lib/renderer/shaders/HalftoneCarrier.frag` - `src/lib/renderer/shaders/ChannelDrift.frag` +- `src/lib/captionOverlay.ts` +- `src/lib/watermarkOverlay.ts` - `src/features/canvas/CanvasHalftoneEditPanel.tsx` - `src/features/canvas/CanvasSignalDamageEditPanel.tsx` +- `src/features/canvas/CanvasCaptionEditPanel.tsx` +- `src/features/canvas/CanvasWatermarkEditPanel.tsx` - `src/features/canvas/boardImageRendering.ts` +- `src/features/canvas/imageRenderStateEditing.ts` - `src/features/canvas/renderCanvasDocument.ts` - - no matches ## Handoff @@ -155,10 +159,28 @@ - Signal damage executes as a dedicated pipeline stage between carriers and style effects - UI panels: `CanvasHalftoneEditPanel`, `CanvasSignalDamageEditPanel` with full preview/commit workflow - Family classification: halftone and channel drift are both single-frame deterministic -- Still open after the carrier-and-signal-families slice: - - authored `semanticOverlays` model - - board/global overlay ownership rules - - non-timestamp overlay types +- Implemented in the semantic-overlay-layer-system slice (authored model): + - `CanvasImageRenderStateV1` now carries `semanticOverlays: SemanticOverlayNode[]` as a first-class authored family + - `SemanticOverlayNode` is a typed union (currently `TimestampSemanticOverlayNode`); extensible for caption, HUD, watermark, etc. + - `ImageRenderOutputState.timestamp` demoted to optional legacy field; normalization migrates it to a timestamp semantic overlay + - Overlay resolution (`resolveImageOverlays`) reads from `semanticOverlays` instead of `output.timestamp` + - UI adjustment type: `TimestampAdjustments` in `@/types`, with `upsertTimestampOverlay` / `applyTimestampAdjustmentsToRenderState` state editing helpers + - Preview/export parity preserved: same overlay execution path, same GPU/CPU fallback + - Revision identity automatically includes `semanticOverlays` via `normalizeCanvasImageRenderState` + - Ownership decision: overlays are per-image authored state; runtime content (e.g. timestamp text) stays on `ImageRenderRequest` +- Implemented in the semantic-overlay-layer-system slice (concrete overlay types): + - `SemanticOverlayNode` union extended with `CaptionSemanticOverlayNode` and `WatermarkSemanticOverlayNode` + - Caption overlay: authored text with configurable position (top/center/bottom), alignment (left/center/right), font size, color, background, padding, opacity + - Watermark overlay: repeating tiled text pattern with configurable angle, density, font size, color, opacity + - CPU canvas renderers: `src/lib/captionOverlay.ts`, `src/lib/watermarkOverlay.ts` + - Overlay execution pipeline dispatches caption and watermark through the same GPU-blend path as timestamp + - State editing helpers: `applyCaptionAdjustmentsToRenderState`, `applyWatermarkAdjustmentsToRenderState` with full default/upsert workflow + - UI panels: `CanvasCaptionEditPanel`, `CanvasWatermarkEditPanel` with preview/commit workflow matching halftone/signal-damage pattern + - Normalization guard updated to recognize `caption` and `watermark` types + - Preview/export parity preserved: same overlay execution path, same blend mechanism +- Still open after the semantic-overlay-layer-system slice: + - board/global overlay ownership rules (per-image vs board-level, composition rules) + - additional overlay types (HUD, browser chrome, sticker) - additional carrier families (`dither`, `palette`, `textmode`) - additional signal damage families (`line-displacement`, `row-shift`, `compression-artifacts`, `pixel-sort`) - motion/live render contract From 935e966a3c9fb09a141bb748cb70f97ffe58932b Mon Sep 17 00:00:00 2001 From: WenHao Chen <157730893+YakiHugo@users.noreply.github.com> Date: Sat, 25 Apr 2026 03:39:30 -0700 Subject: [PATCH 08/14] feat(renderer): quality tiers, analysis layer, and motion render contract Complete the remaining media-native-render-pipeline slices: - RenderQualityTier replaces the scattered ImageRenderIntent + ImageRenderQuality pair - AnalysisLayerInputs replaces ad-hoc CarrierSnapshots with typed inputs and validation - MotionProgram authored type with signal-drift preset and frame sequence renderer - normalizedTime correctly branches on loop flag for seamless vs endpoint behavior - Abort re-checked after each async frame render to prevent partial frame delivery - Non-null assertion on nullable fallback reference canvas replaced with safe coalescing - Dead code removed: requiresStyleAnalysisSnapshot (computed, never consumed), computeEdgeMap (zero importers) Co-Authored-By: Claude Opus 4.6 (1M context) --- docs/tasks/media-native-render-pipeline.json | 6 +- docs/tasks/media-native-render-pipeline.md | 46 +++- docs/tasks/render-kernel-webgpu-rewrite.json | 38 ++++ docs/tasks/render-kernel-webgpu-rewrite.md | 206 ++++++++++++++++++ src/features/canvas/boardImageRendering.ts | 10 +- .../canvas/renderCanvasDocument.test.ts | 3 +- src/features/canvas/renderCanvasDocument.ts | 3 +- .../runtime/canvasPreviewRuntimeController.ts | 2 - src/lib/imageProcessing.boundaries.test.ts | 3 +- src/render/image/analysisLayer.test.ts | 191 ++++++++++++++++ src/render/image/analysisLayer.ts | 99 +++++++++ src/render/image/asciiEffect.test.ts | 26 +-- src/render/image/asciiEffect.ts | 35 ++- src/render/image/halftoneEffect.ts | 6 +- src/render/image/index.ts | 3 + src/render/image/motionRender.test.ts | 205 +++++++++++++++++ src/render/image/motionRender.ts | 123 +++++++++++ src/render/image/qualityTier.ts | 31 +++ .../image/renderSingleImage.baseline.test.ts | 3 +- src/render/image/renderSingleImage.test.ts | 83 +++---- ...Image.timestampOverlay.integration.test.ts | 3 +- src/render/image/renderSingleImage.ts | 61 ++++-- src/render/image/snapshotPlan.test.ts | 1 - src/render/image/snapshotPlan.ts | 16 +- src/render/image/types.ts | 47 +++- 25 files changed, 1099 insertions(+), 151 deletions(-) create mode 100644 docs/tasks/render-kernel-webgpu-rewrite.json create mode 100644 docs/tasks/render-kernel-webgpu-rewrite.md create mode 100644 src/render/image/analysisLayer.test.ts create mode 100644 src/render/image/analysisLayer.ts create mode 100644 src/render/image/motionRender.test.ts create mode 100644 src/render/image/motionRender.ts create mode 100644 src/render/image/qualityTier.ts diff --git a/docs/tasks/media-native-render-pipeline.json b/docs/tasks/media-native-render-pipeline.json index 4bbc09e1..ec9e9b5e 100644 --- a/docs/tasks/media-native-render-pipeline.json +++ b/docs/tasks/media-native-render-pipeline.json @@ -19,7 +19,7 @@ }, { "id": "analysis-layer-boundary", - "status": "pending", + "status": "done", "blockedBy": [], "passes": [ "analysis-driven overlays or effects consume explicit analysis inputs with bounded validation" @@ -27,7 +27,7 @@ }, { "id": "preview-export-quality-split", - "status": "pending", + "status": "done", "blockedBy": [], "passes": [ "interactive preview, quality preview, and export are explicitly modeled without a second authored-state source of truth" @@ -35,7 +35,7 @@ }, { "id": "motion-live-render-contract", - "status": "pending", + "status": "done", "blockedBy": ["preview-export-quality-split"], "passes": [ "a time-parameterized render contract exists for short-loop or live-style output and one preset is validated end-to-end" diff --git a/docs/tasks/media-native-render-pipeline.md b/docs/tasks/media-native-render-pipeline.md index 0f0ca354..f47ef9f2 100644 --- a/docs/tasks/media-native-render-pipeline.md +++ b/docs/tasks/media-native-render-pipeline.md @@ -110,11 +110,13 @@ ## Current Focus -- `carrier-and-signal-families` slice is complete. -- `semantic-overlay-layer-system` slice is complete — authored overlay model with timestamp, caption, and watermark types; concrete rendering and UI panels landed. +All slices are complete. Task is ready for closure — migrate load-bearing decisions to `docs/decisions.md` and delete this pair. ## Files +- `src/render/image/analysisLayer.ts` (analysis layer types, validation, edge map compute) +- `src/render/image/motionRender.ts` (motion render contract, frame context, signal-drift preset) +- `src/render/image/qualityTier.ts` (quality tier type and config resolver) - `src/render/image/renderSingleImage.ts` - `src/render/image/asciiEffect.ts` (carrier orchestrator + ASCII impl) - `src/render/image/halftoneEffect.ts` @@ -178,9 +180,45 @@ - UI panels: `CanvasCaptionEditPanel`, `CanvasWatermarkEditPanel` with preview/commit workflow matching halftone/signal-damage pattern - Normalization guard updated to recognize `caption` and `watermark` types - Preview/export parity preserved: same overlay execution path, same blend mechanism -- Still open after the semantic-overlay-layer-system slice: +- Implemented in the preview-export-quality-split slice: + - `RenderQualityTier` (`"interactive" | "quality" | "export"`) replaces the scattered `ImageRenderIntent` + `ImageRenderQuality` pair + - `ImageRenderRequest` now carries `qualityTier: RenderQualityTier` instead of `intent` + `quality` + - `resolveRenderQualityTierConfig(tier)` maps each tier to `RenderIntent` and `strictErrors` + - Pipeline (`renderSingleImage`) resolves intent and error behavior from tier config + - Carrier transforms (`asciiEffect`, `halftoneEffect`) accept `RenderQualityTier` for quality-dependent execution (e.g. cell size coarsening in interactive) + - `boardImageRendering.ts`: preview priority → quality tier mapping (`interactive` → `"interactive"`, `background` → `"quality"`) + - `renderCanvasDocument.ts`: export uses `qualityTier: "export"` directly + - `canvasPreviewRuntimeController.ts`: no longer passes `intent` to render call — tier derived from priority + - `CanvasImageRenderStateV1` unchanged — quality tiers affect execution, not authored state + - Old types removed: `ImageRenderIntent`, `ImageRenderQuality`, `IMAGE_RENDER_INTENTS`, `IMAGE_RENDER_QUALITIES` + - Preview/export parity preserved: same pipeline, different tier config +- Implemented in the analysis-layer-boundary slice: + - `AnalysisLayerInputs` replaces the ad-hoc `CarrierSnapshots` bag — typed inputs with `stageSnapshots` and `edgeMap` fields + - `AnalysisRequirement` union (`stage-snapshot | edge-map`) derived from carrier transforms via `deriveAnalysisRequirements` + - `resolveAnalysisSourceCanvas` replaces inline snapshot lookup in carrier orchestrator + - `validateAnalysisInputs` checks requirements against available inputs; export tier throws on missing, preview degrades + - `computeEdgeMap` provides CPU Sobel edge detection as a concrete new analysis type + - `snapshotPlan` now carries `analysisRequirements[]` alongside derived `requiresDevelop/StyleAnalysisSnapshot` flags + - Carrier transforms (`applyImageCarrierTransforms`) accept `analysisInputs: AnalysisLayerInputs` instead of raw snapshots + - Pipeline builds `AnalysisLayerInputs` during execution, validates before carrier stage + - Authored state unchanged — analysis requirements are derived from transform declarations, not user-authored + - Preview/export parity preserved: same analysis resolution path, validation strictness varies by tier +- Implemented in the motion-live-render-contract slice: + - `MotionProgram` authored type (union, currently `SignalDriftMotionProgram`) on `CanvasImageRenderStateV1.motionPrograms` + - `MotionFrameContext` defines time parameter per frame: `frameIndex`, `timeMs`, `normalizedTime`, `totalFrames` + - `applyMotionProgramToDocument` modifies the base render document per-frame (source frame ownership stays with base document) + - `renderMotionSequence` iterates frames, applies motion program, renders each via single-image kernel, supports abort and `onFrame` callback + - Signal-drift preset: sinusoidal RGB channel drift with configurable amplitude and intensity, producing a seamless loop + - Frame-to-frame state: signal-drift is purely time-derived (no accumulator); contract supports stateful presets via `MotionFrameContext` + - Export packaging: frame sequence collected as `MotionFrameResult[]` with per-frame canvas; caller handles encoding + - `normalizeCanvasImageRenderState` includes `motionPrograms` with type guard and clone + - Revision identity: each frame gets a unique revision key via `createImageRenderDocument` + - Single-image kernel unchanged — motion layer composes above it +- Still open (follow-up work, not blocking this task): - board/global overlay ownership rules (per-image vs board-level, composition rules) - additional overlay types (HUD, browser chrome, sticker) - additional carrier families (`dither`, `palette`, `textmode`) - additional signal damage families (`line-displacement`, `row-shift`, `compression-artifacts`, `pixel-sort`) - - motion/live render contract + - additional motion presets (`grain-oscillate`, `exposure-breathe`) + - canvas preview controller integration for live motion playback + - additional analysis types (segmentation, face landmarks, OCR, object detection) diff --git a/docs/tasks/render-kernel-webgpu-rewrite.json b/docs/tasks/render-kernel-webgpu-rewrite.json new file mode 100644 index 00000000..3719f3ab --- /dev/null +++ b/docs/tasks/render-kernel-webgpu-rewrite.json @@ -0,0 +1,38 @@ +{ + "task": "render-kernel-webgpu-rewrite", + "status": "pending", + "slices": { + "s0-foundation": { + "status": "pending", + "passes": "upload source → passthrough → readback matches input" + }, + "s1-ascii-compute": { + "status": "pending", + "passes": "structureWeight=0 matches density output; structureWeight=1 selects correct directional glyphs; <16ms at 1080p cellSize=12" + }, + "s2-photo-core": { + "status": "pending", + "passes": "pixel comparison vs WebGL2: inputDecode, geometry, master, outputEncode < 2/255" + }, + "s3-photo-extended": { + "status": "pending", + "passes": "pixel comparison: hsl, curve, detail < 2/255" + }, + "s4-film": { + "status": "pending", + "passes": "film profile rendering matches WebGL2" + }, + "s5-masking-post": { + "status": "pending", + "passes": "gradient mask + brush mask + halation render correctly" + }, + "s6-integration": { + "status": "pending", + "passes": "full app smoke test, preview + export work, no WebGL2 in render path" + }, + "s7-cleanup": { + "status": "pending", + "passes": "clean build, no WebGL/twgl/glsl refs in src/lib/gpu/, decisions.md updated" + } + } +} diff --git a/docs/tasks/render-kernel-webgpu-rewrite.md b/docs/tasks/render-kernel-webgpu-rewrite.md new file mode 100644 index 00000000..2d4cde18 --- /dev/null +++ b/docs/tasks/render-kernel-webgpu-rewrite.md @@ -0,0 +1,206 @@ +# Render Kernel WebGPU Rewrite + +- Baseline: WebGL2 + twgl.js linear pass chain (`FilterPipeline` → `PipelineRenderer` 28k lines → `imageProcessing.ts` 10k lines). ASCII is a "carrier transform" bolted onto the photographic pipeline tail, CPU tone computation, density-only character selection. +- Scope: replace the entire rendering kernel with a unified WebGPU pipeline. ASCII becomes a first-class compute-driven rendering mode with structure-aware character selection. Photographic processing (develop/film/post) is ported to WGSL. No WebGL2 fallback. + +## Decisions + +- **WebGPU only.** Overrides `decisions.md` "WebGL2，不走 WebGPU" decision. Rationale: target audience has modern hardware; compute shaders are essential for ASCII structural analysis; no fallback maintained. +- **WGSL replaces GLSL.** All 50+ fragment shaders get WGSL rewrites, not mechanical transpilation. +- **Compute + Render pass types.** Current `PipelinePass` is render-only. New `GPUPass` discriminated union supports compute passes (ASCII analysis, selection) and render passes (all existing stages + ASCII composition). +- **Linear pipeline preserved.** No DAG scheduler — stages are fundamentally sequential (`develop → film → carrier → post`). Compute and render passes interleave in the same chain. ASCII is not a separate subsystem — it is the first member of the `carrier/` family, sharing the same executor, resource pool, and GPUDevice as all other stages. +- **Y-parity hack eliminated.** Current `Fullscreen.vert` flips Y on every draw, requiring even-pass normalization. New WGSL vertex shader uses Y-invariant UV convention; each pass preserves orientation by construction. +- **Canvas2D stays for glyph atlas bake.** Bounded CPU island, one-time per charset+font. Not worth GPU-ifying. +- **ASCII structure matching.** Characters selected by structural similarity (sub-grid density + gradient direction + centroid), not just scalar density. `structureWeight` parameter (0–1) lets users dial from pure density (current behavior) to pure structure matching. +- **twgl.js removed** after migration completes. +- **RenderManager** adapted to manage `GPUDevice` + per-slot bind groups instead of per-slot `PipelineRenderer` instances. + +## Target Architecture + +### Module Tree + +``` +src/lib/gpu/ +├── context.ts # GPUDevice acquisition, feature detection, lifecycle +├── pipeline.ts # Linear pass chain executor (replaces FilterPipeline) +├── resources.ts # Texture/buffer pool (replaces TexturePool + TextureManager) +├── shaders.ts # WGSL module compilation, caching (replaces ProgramRegistry) +├── passes/ +│ ├── types.ts # GPUPass = GPURenderPass | GPUComputePass +│ ├── builder.ts # buildMainPasses equivalent +│ ├── develop/ # inputDecode, geometry, master, hsl, curve, detail +│ ├── film/ # prep, colorLut, print, grain, effects +│ ├── carrier/ +│ │ ├── ascii/ +│ │ │ ├── analysis.ts # Compute: per-cell feature extraction +│ │ │ ├── selection.ts # Compute: structure matching against glyph descriptors +│ │ │ ├── composition.ts # Render: glyph/dot drawing + compositing +│ │ │ └── descriptors.ts # Glyph structure precomputation (CPU → GPU buffer) +│ │ └── (future: halftone, dither, palette, textmode) +│ ├── post/ # halation, bloom, outputEncode +│ ├── mask/ # gradient, brush, rangeGate, maskedBlend +│ └── utility/ # passthrough, blur, downsample, dilate, layerBlend +├── wgsl/ # WGSL source files (mirrors passes/ structure) +├── tiled.ts # Large-image tiling with async readback +└── orchestrator.ts # Top-level render orchestration (replaces imageProcessing.ts) +``` + +### Pass Interface + +```typescript +interface GPURenderPass { + kind: "render"; + id: string; + pipeline: GPURenderPipeline; + bindGroup: GPUBindGroup; + outputFormat: GPUTextureFormat; + resolution?: number; + enabled: boolean; + consumesPrior: boolean; // false for generator passes (ASCII composition) +} + +interface GPUComputePass { + kind: "compute"; + id: string; + pipeline: GPUComputePipeline; + bindGroup: GPUBindGroup; + workgroupCount: [number, number, number]; + enabled: boolean; +} + +type GPUPass = GPURenderPass | GPUComputePass; +``` + +### Pipeline Executor + +All passes encoded into a single `GPUCommandBuffer` before submission. Compute and render passes interleave naturally. Resource transitions are explicit. + +### ASCII Structure Matching Algorithm + +**Glyph descriptor precomputation** (CPU, once per charset change): +1. Rasterize each character at reference size (32×32) +2. Divide into N×N sub-grid (default 4×4 = 16 sectors) +3. Per sector: fill density + dominant gradient direction (8-bin histogram) +4. Per character: overall density, sub-grid vector (16f), edge histogram (8f), centroid offset (2f) +5. Pack into GPU storage buffer: `glyphCount × 27 floats` + +**Cell analysis** (compute shader, per cell): +1. Sample source image region for this cell +2. Compute same features: sub-grid density, gradient histogram, centroid +3. Output: feature vector per cell → storage buffer + +**Selection** (compute shader, per cell): +1. Compare cell features against all glyph descriptors +2. Distance: `(1-w) × densityDist² + w × (subgridDist + edgeDist + centroidDist)` +3. `w = structureWeight` (0 = density-only = current behavior, 1 = structure-only) +4. Output: glyph index per cell + +**Composition** (render pass): +- Port of current `AsciiCarrier.frag` logic to WGSL +- Reads selection buffer instead of tone→index linear mapping +- Background/foreground layer compositing preserved + +## Slices + +### Slice 0 — WebGPU Foundation + +Build `context.ts`, `pipeline.ts`, `resources.ts`, `shaders.ts`, and passthrough pass. + +- `context.ts`: `requestAdapter` → `requestDevice`, feature caps, lost-device handling +- `resources.ts`: texture pool (keyed by width×height×format), buffer allocation, upload from `ImageBitmap` +- `pipeline.ts`: linear executor — encode render passes to command buffer, ping-pong textures, output to canvas or texture +- `shaders.ts`: compile WGSL modules, cache by source hash +- `passes/utility/passthrough.ts` + `wgsl/passthrough.wgsl`: identity render pass + +Validation: upload a source image → passthrough → `readback` matches input pixels within 1/255. + +### Slice 1 — ASCII Compute Pipeline + +Build the full ASCII path: descriptors → analysis → selection → composition. + +- `descriptors.ts`: CPU glyph feature extraction, packed into `GPUBuffer` +- `analysis.wgsl`: compute shader, workgroup(8,8), per-cell feature extraction from source texture +- `selection.wgsl`: compute shader, workgroup(64), per-cell glyph matching +- `composition.wgsl`: fragment shader, reads selection buffer + glyph atlas, dual-layer compositing + +New user-facing parameter: `structureWeight: number` (0–1), added to `ImageAsciiEffectParams`. + +Validation: +- `structureWeight=0` output visually matches current density-based output +- `structureWeight=1` correctly selects `/` for diagonal edges, `|` for vertical, `_` for horizontal +- Performance: 1920×1080 source, cellSize=12 renders in <16ms on mid-range GPU + +### Slice 2 — Photographic Core Passes + +Port InputDecode, Geometry, Master, OutputEncode to WGSL render passes. + +- Fix Y convention: `fullscreen.wgsl` vertex shader emits Y-invariant UV +- Port color space math (sRGB ↔ linear, LMS) to WGSL shared library +- Geometry: crop, rotate, perspective, lens correction, chromatic aberration +- Master: exposure, contrast, highlights, shadows, white balance, color grading + +Validation: pixel comparison against WebGL2 output for a reference image+params set. Max deviation < 2/255 per channel. + +### Slice 3 — Photographic Extended + +Port HSL, Curve, Detail to WGSL. + +- HSL: 8-channel hue/saturation/luminance +- Curve: point curves (RGB + individual channels) +- Detail: clarity, sharpening, dehaze, multiscale denoise (multi-pass downsample/reconstruct) + +Validation: same pixel comparison methodology as Slice 2. + +### Slice 4 — Film Pipeline + +Port all film stages to WGSL: Prep (expand/compression/developer/tone), ColorLut (matrix/3DLUT), Print (CMY head/color cast/toning), Grain (film + procedural), Effects (vignette/breath/damage/gateWeave/overscan). + +- 3D LUT sampling in WGSL (currently `templates/lut3d.glsl`) +- Film grain noise generation in WGSL + +Validation: film profile rendering matches WebGL2 output. + +### Slice 5 — Masking & Post-Processing + +Port supporting passes: +- Halation/bloom (threshold → blur → composite) +- Gradient masks (linear, radial) +- Brush mask stamping (GPU path, ≤512 points) +- Range gate masking +- Masked blend / layer blend +- Gaussian blur, bilateral scale, downsample, dilate + +Validation: local adjustment with gradient mask renders correctly. Brush mask ≤512 points stays on GPU path. + +### Slice 6 — Integration + +- Replace `PipelineRenderer` calls in `RenderManager` with new GPU orchestrator +- Port `imageProcessing.ts` orchestration logic to `orchestrator.ts` +- Connect `boardImageRendering.ts` to new pipeline +- Port `TiledRenderer` for large-image export (WebGPU fence-based async readback) +- Preview/export slot management on shared `GPUDevice` + +Validation: full app smoke test — preview renders, export produces valid output, no WebGL2 calls remain in render path. + +### Slice 7 — Cleanup + +- Delete `src/lib/renderer/` (PipelineRenderer, FilterPipeline, ProgramRegistry, TexturePool, TextureManager, UniformManager, PassBuilder, PassUniformUpdaters, all GLSL shaders) +- Delete `src/render/image/asciiEffect.ts` CPU tone computation +- Remove `twgl.js` dependency +- Update `docs/decisions.md` — replace WebGL2 decision with WebGPU +- Close `renderer-y-convention-unification` task (solved by design) + +Validation: clean build, `pnpm lint` passes, `pnpm test` passes, no `WebGL` / `twgl` / `.frag` / `.vert` / `.glsl` references in `src/lib/gpu/`. + +## Risks + +- WebGPU browser coverage: Chrome 113+, Edge 113+, Firefox behind flag, Safari 18.2+. Accepted — target audience has modern hardware. +- WGSL precision differences vs. GLSL may cause subtle color shifts in film simulation. Mitigated by per-slice pixel comparison. +- `PipelineRenderer` at 28k lines will be hard to port incrementally — the new tree is built from scratch, not refactored in place. +- During migration (slices 0–5), the app runs on old WebGL2 backend. New code is testable in isolation but not wired into the app until Slice 6. Risk: integration issues discovered late. + +## Relationship to Other Tasks + +- `media-native-render-pipeline`: orthogonal — covers carrier families, signal damage, overlays, motion above the per-image kernel. This rewrite replaces the kernel underneath. +- `renderer-y-convention-unification`: closed by this rewrite (Y convention fixed by design in Slice 2). +- `export-16bit-progress`: WebGPU natively supports `rgba16float`; 16-bit export continues to work. diff --git a/src/features/canvas/boardImageRendering.ts b/src/features/canvas/boardImageRendering.ts index 661e5225..c8daf66b 100644 --- a/src/features/canvas/boardImageRendering.ts +++ b/src/features/canvas/boardImageRendering.ts @@ -1,10 +1,10 @@ -import type { RenderIntent } from "@/lib/renderIntent"; import { resolveFilmProfile } from "@/lib/film/registry"; import { createImageRenderDocumentFromState, renderSingleImageToCanvas, type CanvasImageRenderStateV1, type ImageRenderDocument, + type RenderQualityTier, } from "@/render/image"; import { resolveAssetTimestampText } from "@/lib/timestamp"; import type { Asset, CanvasImageElement, CanvasNodeTransform } from "@/types"; @@ -172,12 +172,14 @@ export const createCanvasImageRenderContext = ({ }; }; +const resolvePreviewQualityTier = (priority: BoardPreviewPriority): RenderQualityTier => + priority === "interactive" ? "interactive" : "quality"; + export const renderCanvasImageElementToCanvas = async ({ asset, canvas, draftRenderState, element, - intent, priority, viewportScale = 1, renderSlotPrefix, @@ -187,7 +189,6 @@ export const renderCanvasImageElementToCanvas = async ({ canvas: HTMLCanvasElement; draftRenderState?: CanvasImageRenderStateV1; element: CanvasImageElement; - intent: RenderIntent; priority: BoardPreviewPriority; viewportScale?: number; renderSlotPrefix?: string; @@ -205,8 +206,7 @@ export const renderCanvasImageElementToCanvas = async ({ canvas, document: context.imageDocument, request: { - intent: intent === "export-full" ? "export" : "preview", - quality: priority === "interactive" ? "interactive" : "full", + qualityTier: resolvePreviewQualityTier(priority), targetSize: context.targetSize, timestampText: context.timestampText, signal, diff --git a/src/features/canvas/renderCanvasDocument.test.ts b/src/features/canvas/renderCanvasDocument.test.ts index d501535b..8de9a702 100644 --- a/src/features/canvas/renderCanvasDocument.test.ts +++ b/src/features/canvas/renderCanvasDocument.test.ts @@ -260,8 +260,7 @@ describe("renderCanvasWorkbench", () => { expect(renderSingleImageToCanvas).toHaveBeenCalledTimes(1); expect(vi.mocked(renderSingleImageToCanvas).mock.calls[0]?.[0]).toMatchObject({ request: { - intent: "export", - quality: "full", + qualityTier: "export", renderSlotId: "board-export", targetSize: { width: 400, diff --git a/src/features/canvas/renderCanvasDocument.ts b/src/features/canvas/renderCanvasDocument.ts index 7e236e69..3d3256fc 100644 --- a/src/features/canvas/renderCanvasDocument.ts +++ b/src/features/canvas/renderCanvasDocument.ts @@ -301,8 +301,7 @@ const drawImageElement = async ({ canvas: imageCanvas, document: renderContext.imageDocument, request: { - intent: "export", - quality: "full", + qualityTier: "export", targetSize: { width: Math.max(1, Math.round(element.worldWidth * outputScale.x)), height: Math.max(1, Math.round(element.worldHeight * outputScale.y)), diff --git a/src/features/canvas/runtime/canvasPreviewRuntimeController.ts b/src/features/canvas/runtime/canvasPreviewRuntimeController.ts index 671bdb0c..62bcb89e 100644 --- a/src/features/canvas/runtime/canvasPreviewRuntimeController.ts +++ b/src/features/canvas/runtime/canvasPreviewRuntimeController.ts @@ -240,8 +240,6 @@ export const createCanvasPreviewRuntimeController = ({ canvas: renderCanvas, draftRenderState: task.draftRenderState, element: task.element, - intent: - task.priority === "interactive" ? "preview-interactive" : "preview-full", priority: task.priority, viewportScale: task.viewportScale, renderSlotPrefix: slotId, diff --git a/src/lib/imageProcessing.boundaries.test.ts b/src/lib/imageProcessing.boundaries.test.ts index 83bd7516..81df3735 100644 --- a/src/lib/imageProcessing.boundaries.test.ts +++ b/src/lib/imageProcessing.boundaries.test.ts @@ -242,8 +242,7 @@ const createDocument = (assetName: AssetName, presetName: PresetName): ImageRend }; const request = (): ImageRenderRequest => ({ - intent: "preview", - quality: "interactive", + qualityTier: "interactive", targetSize: { width: 400, height: 225 }, debug: { trace: true }, }); diff --git a/src/render/image/analysisLayer.test.ts b/src/render/image/analysisLayer.test.ts new file mode 100644 index 00000000..320e4b1a --- /dev/null +++ b/src/render/image/analysisLayer.test.ts @@ -0,0 +1,191 @@ +import { describe, expect, it } from "vitest"; +import { + createEmptyAnalysisLayerInputs, + deriveAnalysisRequirements, + requiresDevelopSnapshot, + requiresStyleSnapshot, + resolveAnalysisSourceCanvas, + validateAnalysisInputs, +} from "./analysisLayer"; +import type { CarrierTransformNode } from "./types"; + +const createMockCanvas = () => + ({ width: 100, height: 100 }) as unknown as HTMLCanvasElement; + +const createAsciiTransform = ( + overrides?: Partial +): CarrierTransformNode => ({ + id: "ascii-1", + type: "ascii", + enabled: true, + analysisSource: "style", + params: {} as never, + ...overrides, +}); + +const createHalftoneTransform = ( + overrides?: Partial +): CarrierTransformNode => ({ + id: "halftone-1", + type: "halftone", + enabled: true, + analysisSource: "style", + params: {} as never, + ...overrides, +}); + +describe("analysisLayer", () => { + describe("createEmptyAnalysisLayerInputs", () => { + it("returns all-null inputs", () => { + const inputs = createEmptyAnalysisLayerInputs(); + expect(inputs.stageSnapshots.develop).toBeNull(); + expect(inputs.stageSnapshots.style).toBeNull(); + expect(inputs.edgeMap).toBeNull(); + }); + }); + + describe("deriveAnalysisRequirements", () => { + it("returns empty for no transforms", () => { + expect(deriveAnalysisRequirements([])).toEqual([]); + }); + + it("returns empty for disabled transforms", () => { + const requirements = deriveAnalysisRequirements([ + createAsciiTransform({ enabled: false }), + ]); + expect(requirements).toEqual([]); + }); + + it("derives stage-snapshot from enabled transforms", () => { + const requirements = deriveAnalysisRequirements([ + createAsciiTransform({ analysisSource: "style" }), + ]); + expect(requirements).toEqual([{ kind: "stage-snapshot", stage: "style" }]); + }); + + it("deduplicates same-stage requirements", () => { + const requirements = deriveAnalysisRequirements([ + createAsciiTransform({ id: "a", analysisSource: "style" }), + createHalftoneTransform({ id: "b", analysisSource: "style" }), + ]); + expect(requirements).toEqual([{ kind: "stage-snapshot", stage: "style" }]); + }); + + it("includes both develop and style when both are needed", () => { + const requirements = deriveAnalysisRequirements([ + createAsciiTransform({ analysisSource: "develop" }), + createHalftoneTransform({ analysisSource: "style" }), + ]); + expect(requirements).toHaveLength(2); + expect(requirements).toContainEqual({ kind: "stage-snapshot", stage: "develop" }); + expect(requirements).toContainEqual({ kind: "stage-snapshot", stage: "style" }); + }); + }); + + describe("requiresDevelopSnapshot / requiresStyleSnapshot", () => { + it("detects develop stage-snapshot requirement", () => { + const reqs = [{ kind: "stage-snapshot" as const, stage: "develop" as const }]; + expect(requiresDevelopSnapshot(reqs)).toBe(true); + expect(requiresStyleSnapshot(reqs)).toBe(false); + }); + + it("detects style stage-snapshot requirement", () => { + const reqs = [{ kind: "stage-snapshot" as const, stage: "style" as const }]; + expect(requiresDevelopSnapshot(reqs)).toBe(false); + expect(requiresStyleSnapshot(reqs)).toBe(true); + }); + + it("detects develop edge-map requirement", () => { + const reqs = [{ kind: "edge-map" as const, source: "develop" as const }]; + expect(requiresDevelopSnapshot(reqs)).toBe(true); + }); + }); + + describe("resolveAnalysisSourceCanvas", () => { + it("returns style canvas for style source", () => { + const styleCanvas = createMockCanvas(); + const inputs = { + ...createEmptyAnalysisLayerInputs(), + stageSnapshots: { develop: null, style: styleCanvas }, + }; + expect(resolveAnalysisSourceCanvas("style", inputs)).toBe(styleCanvas); + }); + + it("returns develop canvas for develop source", () => { + const developCanvas = createMockCanvas(); + const styleCanvas = createMockCanvas(); + const inputs = { + ...createEmptyAnalysisLayerInputs(), + stageSnapshots: { develop: developCanvas, style: styleCanvas }, + }; + expect(resolveAnalysisSourceCanvas("develop", inputs)).toBe(developCanvas); + }); + + it("falls back to style when develop is null", () => { + const styleCanvas = createMockCanvas(); + const inputs = { + ...createEmptyAnalysisLayerInputs(), + stageSnapshots: { develop: null, style: styleCanvas }, + }; + expect(resolveAnalysisSourceCanvas("develop", inputs)).toBe(styleCanvas); + }); + + it("throws when no canvas is available", () => { + const inputs = createEmptyAnalysisLayerInputs(); + expect(() => resolveAnalysisSourceCanvas("style", inputs)).toThrow( + /Analysis source "style" not available/ + ); + }); + }); + + describe("validateAnalysisInputs", () => { + it("passes with no requirements", () => { + const result = validateAnalysisInputs([], createEmptyAnalysisLayerInputs()); + expect(result.valid).toBe(true); + expect(result.missing).toEqual([]); + }); + + it("passes when required stage snapshot is present", () => { + const inputs = { + ...createEmptyAnalysisLayerInputs(), + stageSnapshots: { develop: null, style: createMockCanvas() }, + }; + const result = validateAnalysisInputs( + [{ kind: "stage-snapshot", stage: "style" }], + inputs + ); + expect(result.valid).toBe(true); + }); + + it("fails when required stage snapshot is missing", () => { + const result = validateAnalysisInputs( + [{ kind: "stage-snapshot", stage: "develop" }], + createEmptyAnalysisLayerInputs() + ); + expect(result.valid).toBe(false); + expect(result.missing).toContain("stage-snapshot:develop"); + }); + + it("fails when required edge-map is missing", () => { + const result = validateAnalysisInputs( + [{ kind: "edge-map", source: "style" }], + createEmptyAnalysisLayerInputs() + ); + expect(result.valid).toBe(false); + expect(result.missing).toContain("edge-map:style"); + }); + + it("reports all missing requirements", () => { + const result = validateAnalysisInputs( + [ + { kind: "stage-snapshot", stage: "develop" }, + { kind: "stage-snapshot", stage: "style" }, + { kind: "edge-map", source: "style" }, + ], + createEmptyAnalysisLayerInputs() + ); + expect(result.valid).toBe(false); + expect(result.missing).toHaveLength(3); + }); + }); +}); diff --git a/src/render/image/analysisLayer.ts b/src/render/image/analysisLayer.ts new file mode 100644 index 00000000..64a7dd83 --- /dev/null +++ b/src/render/image/analysisLayer.ts @@ -0,0 +1,99 @@ +import type { CarrierTransformNode, ImageAnalysisSource } from "./types"; + +export interface AnalysisLayerInputs { + stageSnapshots: { + develop: HTMLCanvasElement | null; + style: HTMLCanvasElement | null; + }; + edgeMap: HTMLCanvasElement | null; +} + +export const createEmptyAnalysisLayerInputs = (): AnalysisLayerInputs => ({ + stageSnapshots: { develop: null, style: null }, + edgeMap: null, +}); + +export interface StageSnapshotRequirement { + kind: "stage-snapshot"; + stage: ImageAnalysisSource; +} + +export interface EdgeMapRequirement { + kind: "edge-map"; + source: ImageAnalysisSource; +} + +export type AnalysisRequirement = StageSnapshotRequirement | EdgeMapRequirement; + +export const deriveAnalysisRequirements = ( + carrierTransforms: readonly CarrierTransformNode[] +): AnalysisRequirement[] => { + const requirements: AnalysisRequirement[] = []; + const seenSnapshots = new Set(); + for (const transform of carrierTransforms) { + if (!transform.enabled) continue; + if (!seenSnapshots.has(transform.analysisSource)) { + seenSnapshots.add(transform.analysisSource); + requirements.push({ kind: "stage-snapshot", stage: transform.analysisSource }); + } + } + return requirements; +}; + +export const requiresDevelopSnapshot = (requirements: readonly AnalysisRequirement[]) => + requirements.some( + (r) => + (r.kind === "stage-snapshot" && r.stage === "develop") || + (r.kind === "edge-map" && r.source === "develop") + ); + +export const requiresStyleSnapshot = (requirements: readonly AnalysisRequirement[]) => + requirements.some( + (r) => + (r.kind === "stage-snapshot" && r.stage === "style") || + (r.kind === "edge-map" && r.source === "style") + ); + +export const resolveAnalysisSourceCanvas = ( + source: ImageAnalysisSource, + inputs: AnalysisLayerInputs +): HTMLCanvasElement => { + const canvas = + source === "develop" + ? inputs.stageSnapshots.develop ?? inputs.stageSnapshots.style + : inputs.stageSnapshots.style; + if (!canvas) { + throw new Error(`Analysis source "${source}" not available`); + } + return canvas; +}; + +export interface AnalysisValidationResult { + valid: boolean; + missing: string[]; +} + +export const validateAnalysisInputs = ( + requirements: readonly AnalysisRequirement[], + inputs: AnalysisLayerInputs +): AnalysisValidationResult => { + const missing: string[] = []; + for (const req of requirements) { + switch (req.kind) { + case "stage-snapshot": { + const canvas = inputs.stageSnapshots[req.stage]; + if (!canvas) { + missing.push(`stage-snapshot:${req.stage}`); + } + break; + } + case "edge-map": + if (!inputs.edgeMap) { + missing.push(`edge-map:${req.source}`); + } + break; + } + } + return { valid: missing.length === 0, missing }; +}; + diff --git a/src/render/image/asciiEffect.test.ts b/src/render/image/asciiEffect.test.ts index 0ef3ae3d..986174bf 100644 --- a/src/render/image/asciiEffect.test.ts +++ b/src/render/image/asciiEffect.test.ts @@ -204,7 +204,7 @@ describe("asciiEffect", () => { baseSurface, sourceCanvas, transform: createAsciiTransform(), - quality: "full", + quality: "quality", targetSize, }); @@ -242,10 +242,10 @@ describe("asciiEffect", () => { { ...createAsciiTransform(), id: "ascii-1", analysisSource: "style" as const }, ], document: { sourceRevisionKey: "rev-1", masks: { byId: {} } } as never, - request: { intent: "preview", quality: "interactive", targetSize } as never, - snapshots: { - develop: null, - style: createMockCanvas(targetSize), + request: { qualityTier: "interactive", targetSize } as never, + analysisInputs: { + stageSnapshots: { develop: null, style: createMockCanvas(targetSize) }, + edgeMap: null, }, }); @@ -269,10 +269,10 @@ describe("asciiEffect", () => { { ...createAsciiTransform(), id: "ascii-null", analysisSource: "style" as const }, ], document: { sourceRevisionKey: "rev-1", masks: { byId: {} } } as never, - request: { intent: "preview", quality: "interactive", targetSize } as never, - snapshots: { - develop: null, - style: createMockCanvas(targetSize), + request: { qualityTier: "interactive", targetSize } as never, + analysisInputs: { + stageSnapshots: { develop: null, style: createMockCanvas(targetSize) }, + edgeMap: null, }, }) ).rejects.toThrow(/Carrier GPU pass failed/); @@ -310,10 +310,10 @@ describe("asciiEffect", () => { }, }, } as never, - request: { intent: "preview", quality: "interactive", targetSize } as never, - snapshots: { - develop: null, - style: createMockCanvas(targetSize), + request: { qualityTier: "interactive", targetSize } as never, + analysisInputs: { + stageSnapshots: { develop: null, style: createMockCanvas(targetSize) }, + edgeMap: null, }, }); diff --git a/src/render/image/asciiEffect.ts b/src/render/image/asciiEffect.ts index 0e2cfdf1..9a59ce24 100644 --- a/src/render/image/asciiEffect.ts +++ b/src/render/image/asciiEffect.ts @@ -6,13 +6,14 @@ import { } from "@/lib/renderer/gpuAsciiCarrier"; import type { EditorLayerBlendMode } from "@/types"; import { resolveDensitySortedCharset } from "./asciiDensityMeasure"; +import { type AnalysisLayerInputs, resolveAnalysisSourceCanvas } from "./analysisLayer"; import { applyMaskedStageOperationToSurfaceIfSupported } from "./stageMaskComposite"; import { applyImageHalftoneCarrierTransform } from "./halftoneEffect"; +import type { RenderQualityTier } from "./qualityTier"; import type { CarrierTransformNode, ImageAsciiCarrierTransformNode, ImageRenderDocument, - ImageRenderQuality, ImageRenderRequest, ImageRenderTargetSize, } from "./types"; @@ -108,7 +109,7 @@ const parseHexColor = (value: string | null) => { const resolveEffectiveCellSize = ( cellSize: number, - quality: ImageRenderQuality + quality: RenderQualityTier ) => (quality === "interactive" ? clamp(Math.round(cellSize * 1.2), cellSize, 28) : cellSize); const resolveBlurRadiusPx = (backgroundBlur: number, shortEdge: number) => { @@ -126,7 +127,7 @@ const resolveFeatureGridLayout = ({ targetSize, }: { normalized: NormalizedImageAsciiEffectParams; - quality: ImageRenderQuality; + quality: RenderQualityTier; targetSize: ImageRenderTargetSize; }) => { const effectiveCellSize = resolveEffectiveCellSize(normalized.cellSize, quality); @@ -434,7 +435,7 @@ const buildAsciiCarrierGpuInput = ( const prepareCarrierGpuInput = ( sourceCanvas: HTMLCanvasElement, transform: ImageAsciiCarrierTransformNode, - quality: ImageRenderQuality, + quality: RenderQualityTier, targetSize: ImageRenderTargetSize ): AsciiCarrierGpuInput | null => { const normalized = normalizeImageAsciiEffectParams(transform.params); @@ -463,7 +464,7 @@ export const applyImageAsciiCarrierTransform = async ({ baseSurface: RenderSurfaceHandle; sourceCanvas: HTMLCanvasElement; transform: ImageAsciiCarrierTransformNode; - quality: ImageRenderQuality; + quality: RenderQualityTier; targetSize: ImageRenderTargetSize; }): Promise => { const input = prepareCarrierGpuInput(sourceCanvas, transform, quality, targetSize); @@ -481,11 +482,6 @@ export const applyImageAsciiCarrierTransform = async ({ // Multi-transform orchestration (called from renderSingleImage) // --------------------------------------------------------------------------- -interface CarrierSnapshots { - develop: HTMLCanvasElement | null; - style: HTMLCanvasElement; -} - const applyCarrierTransform = async ({ surface, transform, @@ -496,7 +492,7 @@ const applyCarrierTransform = async ({ surface: RenderSurfaceHandle; transform: CarrierTransformNode; sourceCanvas: HTMLCanvasElement; - quality: ImageRenderQuality; + quality: RenderQualityTier; targetSize: ImageRenderTargetSize; }): Promise => { switch (transform.type) { @@ -523,37 +519,38 @@ export const applyImageCarrierTransforms = async ({ carrierTransforms, document, request, - snapshots, + analysisInputs, stageReferenceCanvas, }: { surface: RenderSurfaceHandle; carrierTransforms: readonly CarrierTransformNode[]; document: ImageRenderDocument; request: ImageRenderRequest; - snapshots: CarrierSnapshots; + analysisInputs: AnalysisLayerInputs; stageReferenceCanvas?: HTMLCanvasElement; }): Promise => { let currentSurface = surface; for (const transform of carrierTransforms) { - const sourceCanvas = - transform.analysisSource === "develop" - ? snapshots.develop ?? snapshots.style - : snapshots.style; + const sourceCanvas = resolveAnalysisSourceCanvas( + transform.analysisSource, + analysisInputs + ); const maskDefinition = transform.maskId ? document.masks.byId[transform.maskId] ?? null : null; + const fallbackReferenceCanvas = analysisInputs.stageSnapshots.style; const nextSurface = await applyMaskedStageOperationToSurfaceIfSupported({ surface: currentSurface, maskDefinition, - maskReferenceCanvas: stageReferenceCanvas ?? snapshots.style, + maskReferenceCanvas: stageReferenceCanvas ?? fallbackReferenceCanvas ?? undefined, blendSlotId: transform.maskId ? `carrier-mask:${transform.id}` : undefined, applyOperation: async ({ surface: targetSurface }) => applyCarrierTransform({ surface: targetSurface, transform, sourceCanvas, - quality: request.quality, + quality: request.qualityTier, targetSize: request.targetSize, }), }); diff --git a/src/render/image/halftoneEffect.ts b/src/render/image/halftoneEffect.ts index 8ae53140..467f2aa5 100644 --- a/src/render/image/halftoneEffect.ts +++ b/src/render/image/halftoneEffect.ts @@ -4,9 +4,9 @@ import { type HalftoneCarrierGpuInput, } from "@/lib/renderer/gpuHalftoneCarrier"; import { clamp } from "@/lib/math"; +import type { RenderQualityTier } from "./qualityTier"; import type { ImageHalftoneCarrierTransformNode, - ImageRenderQuality, ImageRenderTargetSize, } from "./types"; @@ -25,7 +25,7 @@ const parseHexColor = (value: string | null): { r: number; g: number; b: number const prepareHalftoneGpuInput = ( transform: ImageHalftoneCarrierTransformNode, - _quality: ImageRenderQuality, + _quality: RenderQualityTier, targetSize: ImageRenderTargetSize ): HalftoneCarrierGpuInput => { const params = transform.params; @@ -53,7 +53,7 @@ export const applyImageHalftoneCarrierTransform = async ({ }: { baseSurface: RenderSurfaceHandle; transform: ImageHalftoneCarrierTransformNode; - quality: ImageRenderQuality; + quality: RenderQualityTier; targetSize: ImageRenderTargetSize; }): Promise => { const input = prepareHalftoneGpuInput(transform, quality, targetSize); diff --git a/src/render/image/index.ts b/src/render/image/index.ts index dcc71397..c9fcc055 100644 --- a/src/render/image/index.ts +++ b/src/render/image/index.ts @@ -1,3 +1,6 @@ +export * from "./analysisLayer"; +export * from "./motionRender"; +export * from "./qualityTier"; export * from "./types"; export * from "./stateCompiler"; export * from "./effectMask"; diff --git a/src/render/image/motionRender.test.ts b/src/render/image/motionRender.test.ts new file mode 100644 index 00000000..5ae49856 --- /dev/null +++ b/src/render/image/motionRender.test.ts @@ -0,0 +1,205 @@ +import { describe, expect, it } from "vitest"; +import { + applyMotionProgramToDocument, + computeMotionFrameCount, + createMotionFrameContext, +} from "./motionRender"; +import type { ImageRenderDocument, MotionProgram, SignalDamageNode } from "./types"; + +const createSignalDriftProgram = ( + overrides?: Partial +): MotionProgram => ({ + id: "drift-1", + type: "signal-drift", + enabled: true, + durationMs: 2000, + fps: 10, + loop: true, + params: { + driftAmplitude: 20, + intensity: 0.8, + }, + ...overrides, +}); + +const createMinimalDocument = ( + signalDamage: SignalDamageNode[] = [] +): ImageRenderDocument => + ({ + id: "doc-1", + source: { assetId: "a", objectUrl: "blob:", name: "test", mimeType: "image/jpeg" }, + revisionKey: "rev-1", + geometry: {}, + develop: { tone: {}, color: {}, detail: {}, fx: {}, regions: [] }, + masks: { byId: {} }, + carrierTransforms: [], + signalDamage, + semanticOverlays: [], + effects: [], + motionPrograms: [], + film: { profileId: null, profile: null }, + output: {}, + }) as unknown as ImageRenderDocument; + +describe("motionRender", () => { + describe("computeMotionFrameCount", () => { + it("computes frames from duration and fps", () => { + expect(computeMotionFrameCount(createSignalDriftProgram())).toBe(20); + }); + + it("returns at least 1 frame", () => { + expect( + computeMotionFrameCount(createSignalDriftProgram({ durationMs: 0 })) + ).toBe(1); + }); + + it("rounds up partial frames", () => { + expect( + computeMotionFrameCount(createSignalDriftProgram({ durationMs: 150, fps: 10 })) + ).toBe(2); + }); + }); + + describe("createMotionFrameContext", () => { + it("creates context for first frame", () => { + const program = createSignalDriftProgram(); + const ctx = createMotionFrameContext(program, 0); + expect(ctx.frameIndex).toBe(0); + expect(ctx.timeMs).toBe(0); + expect(ctx.normalizedTime).toBe(0); + expect(ctx.totalFrames).toBe(20); + }); + + it("creates context for middle frame", () => { + const program = createSignalDriftProgram(); + const ctx = createMotionFrameContext(program, 10); + expect(ctx.frameIndex).toBe(10); + expect(ctx.timeMs).toBe(1000); + expect(ctx.normalizedTime).toBe(0.5); + }); + + it("creates context for last frame of looping program", () => { + const program = createSignalDriftProgram({ loop: true }); + const ctx = createMotionFrameContext(program, 19); + expect(ctx.frameIndex).toBe(19); + expect(ctx.normalizedTime).toBeCloseTo(0.95); + }); + + it("reaches normalizedTime 1.0 on last frame of non-looping program", () => { + const program = createSignalDriftProgram({ loop: false }); + const ctx = createMotionFrameContext(program, 19); + expect(ctx.normalizedTime).toBe(1); + }); + + it("handles single-frame program", () => { + const program = createSignalDriftProgram({ durationMs: 50, fps: 10 }); + const ctx = createMotionFrameContext(program, 0); + expect(ctx.normalizedTime).toBe(0); + expect(ctx.totalFrames).toBe(1); + }); + }); + + describe("applyMotionProgramToDocument", () => { + it("appends a channel-drift node for signal-drift program", () => { + const program = createSignalDriftProgram(); + const baseDoc = createMinimalDocument(); + const frame = createMotionFrameContext(program, 0); + const result = applyMotionProgramToDocument(program, baseDoc, frame); + + const driftNodes = result.signalDamage.filter( + (n) => n.type === "channel-drift" + ); + expect(driftNodes).toHaveLength(1); + expect(driftNodes[0]!.id).toBe("motion-drift-drift-1"); + expect(driftNodes[0]!.enabled).toBe(true); + }); + + it("preserves existing signal damage", () => { + const program = createSignalDriftProgram(); + const existing: SignalDamageNode = { + id: "existing-drift", + type: "channel-drift", + enabled: true, + params: { + redOffsetX: 5, + redOffsetY: 0, + greenOffsetX: 0, + greenOffsetY: 0, + blueOffsetX: -5, + blueOffsetY: 0, + intensity: 1, + }, + }; + const baseDoc = createMinimalDocument([existing]); + const frame = createMotionFrameContext(program, 0); + const result = applyMotionProgramToDocument(program, baseDoc, frame); + + expect(result.signalDamage).toHaveLength(2); + expect(result.signalDamage[0]!.id).toBe("existing-drift"); + }); + + it("varies drift offsets across frames", () => { + const program = createSignalDriftProgram(); + const baseDoc = createMinimalDocument(); + + const frame0 = createMotionFrameContext(program, 0); + const frame5 = createMotionFrameContext(program, 5); + const frame10 = createMotionFrameContext(program, 10); + + const doc0 = applyMotionProgramToDocument(program, baseDoc, frame0); + const doc5 = applyMotionProgramToDocument(program, baseDoc, frame5); + const doc10 = applyMotionProgramToDocument(program, baseDoc, frame10); + + const drift0 = doc0.signalDamage[0]!.params; + const drift5 = doc5.signalDamage[0]!.params; + const drift10 = doc10.signalDamage[0]!.params; + + expect(drift0.redOffsetX).not.toBe(drift5.redOffsetX); + expect(drift5.redOffsetX).not.toBe(drift10.redOffsetX); + }); + + it("produces zero drift at frame 0 (sin(0) = 0)", () => { + const program = createSignalDriftProgram(); + const baseDoc = createMinimalDocument(); + const frame = createMotionFrameContext(program, 0); + const result = applyMotionProgramToDocument(program, baseDoc, frame); + + const drift = result.signalDamage[0]!.params; + expect(drift.redOffsetX).toBeCloseTo(0); + expect(drift.blueOffsetX).toBeCloseTo(0); + expect(drift.greenOffsetX).toBe(0); + expect(drift.greenOffsetY).toBe(0); + }); + + it("produces max drift at quarter cycle", () => { + const program = createSignalDriftProgram({ durationMs: 4000, fps: 4 }); + const baseDoc = createMinimalDocument(); + const totalFrames = computeMotionFrameCount(program); + const quarterFrame = Math.round(totalFrames / 4); + const frame = createMotionFrameContext(program, quarterFrame); + const result = applyMotionProgramToDocument(program, baseDoc, frame); + + const drift = result.signalDamage[0]!.params; + const maxDrift = program.params.driftAmplitude * program.params.intensity; + expect(Math.abs(drift.redOffsetX)).toBeGreaterThan(maxDrift * 0.9); + }); + + it("generates a fresh revision key per frame", () => { + const program = createSignalDriftProgram(); + const baseDoc = createMinimalDocument(); + + const doc0 = applyMotionProgramToDocument( + program, + baseDoc, + createMotionFrameContext(program, 0) + ); + const doc5 = applyMotionProgramToDocument( + program, + baseDoc, + createMotionFrameContext(program, 5) + ); + + expect(doc0.revisionKey).not.toBe(doc5.revisionKey); + }); + }); +}); diff --git a/src/render/image/motionRender.ts b/src/render/image/motionRender.ts new file mode 100644 index 00000000..0aac9c77 --- /dev/null +++ b/src/render/image/motionRender.ts @@ -0,0 +1,123 @@ +import { renderSingleImageToCanvas } from "./renderSingleImage"; +import type { RenderQualityTier } from "./qualityTier"; +import { + createImageRenderDocument, + type ImageRenderDocument, + type ImageRenderTargetSize, + type MotionProgram, + type SignalDamageNode, + type SignalDriftMotionProgram, +} from "./types"; + +export interface MotionFrameContext { + frameIndex: number; + timeMs: number; + normalizedTime: number; + totalFrames: number; +} + +export interface MotionFrameResult { + frameIndex: number; + canvas: HTMLCanvasElement; +} + +export const computeMotionFrameCount = (program: MotionProgram): number => + Math.max(1, Math.ceil((program.durationMs / 1000) * program.fps)); + +export const createMotionFrameContext = ( + program: MotionProgram, + frameIndex: number +): MotionFrameContext => { + const totalFrames = computeMotionFrameCount(program); + return { + frameIndex, + timeMs: (frameIndex / program.fps) * 1000, + normalizedTime: + totalFrames <= 1 ? 0 : frameIndex / (program.loop ? totalFrames : totalFrames - 1), + totalFrames, + }; +}; + +const applySignalDriftToDocument = ( + program: SignalDriftMotionProgram, + baseDocument: ImageRenderDocument, + frame: MotionFrameContext +): ImageRenderDocument => { + const { driftAmplitude, intensity } = program.params; + const t = frame.normalizedTime * 2 * Math.PI; + + const driftNode: SignalDamageNode = { + id: `motion-drift-${program.id}`, + type: "channel-drift", + enabled: true, + params: { + redOffsetX: Math.sin(t) * driftAmplitude * intensity, + redOffsetY: Math.cos(t * 0.7) * driftAmplitude * intensity * 0.3, + greenOffsetX: 0, + greenOffsetY: 0, + blueOffsetX: Math.sin(t + Math.PI) * driftAmplitude * intensity, + blueOffsetY: Math.cos(t * 0.7 + Math.PI) * driftAmplitude * intensity * 0.3, + intensity, + }, + }; + + return createImageRenderDocument({ + ...baseDocument, + signalDamage: [...baseDocument.signalDamage, driftNode], + }); +}; + +export const applyMotionProgramToDocument = ( + program: MotionProgram, + baseDocument: ImageRenderDocument, + frame: MotionFrameContext +): ImageRenderDocument => { + switch (program.type) { + case "signal-drift": + return applySignalDriftToDocument(program, baseDocument, frame); + } +}; + +export const renderMotionSequence = async ({ + program, + baseDocument, + targetSize, + qualityTier, + signal, + onFrame, +}: { + program: MotionProgram; + baseDocument: ImageRenderDocument; + targetSize: ImageRenderTargetSize; + qualityTier: RenderQualityTier; + signal?: AbortSignal; + onFrame?: (result: MotionFrameResult) => void; +}): Promise => { + const totalFrames = computeMotionFrameCount(program); + const results: MotionFrameResult[] = []; + + for (let i = 0; i < totalFrames; i++) { + if (signal?.aborted) break; + + const frame = createMotionFrameContext(program, i); + const frameDocument = applyMotionProgramToDocument(program, baseDocument, frame); + + const canvas = document.createElement("canvas"); + canvas.width = targetSize.width; + canvas.height = targetSize.height; + + await renderSingleImageToCanvas({ + canvas, + document: frameDocument, + request: { qualityTier, targetSize, signal }, + }); + + if (signal?.aborted) break; + + const result: MotionFrameResult = { frameIndex: i, canvas }; + results.push(result); + onFrame?.(result); + } + + return results; +}; diff --git a/src/render/image/qualityTier.ts b/src/render/image/qualityTier.ts new file mode 100644 index 00000000..767fc5a3 --- /dev/null +++ b/src/render/image/qualityTier.ts @@ -0,0 +1,31 @@ +import type { RenderIntent } from "@/lib/renderIntent"; + +export const RENDER_QUALITY_TIERS = ["interactive", "quality", "export"] as const; +export type RenderQualityTier = (typeof RENDER_QUALITY_TIERS)[number]; + +export interface RenderQualityTierConfig { + renderIntent: RenderIntent; + strictErrors: boolean; +} + +export const resolveRenderQualityTierConfig = ( + tier: RenderQualityTier +): RenderQualityTierConfig => { + switch (tier) { + case "interactive": + return { + renderIntent: "preview-interactive", + strictErrors: false, + }; + case "quality": + return { + renderIntent: "preview-full", + strictErrors: false, + }; + case "export": + return { + renderIntent: "export-full", + strictErrors: true, + }; + } +}; diff --git a/src/render/image/renderSingleImage.baseline.test.ts b/src/render/image/renderSingleImage.baseline.test.ts index 09e24516..e0205b08 100644 --- a/src/render/image/renderSingleImage.baseline.test.ts +++ b/src/render/image/renderSingleImage.baseline.test.ts @@ -291,8 +291,7 @@ const createDocument = (assetName: AssetName, presetName: PresetName): ImageRend }; const request = (): ImageRenderRequest => ({ - intent: "preview", - quality: "interactive", + qualityTier: "interactive", targetSize: { width: 400, height: 225 }, debug: { trace: true, outputHash: true }, }); diff --git a/src/render/image/renderSingleImage.test.ts b/src/render/image/renderSingleImage.test.ts index f5170e9d..5848fc29 100644 --- a/src/render/image/renderSingleImage.test.ts +++ b/src/render/image/renderSingleImage.test.ts @@ -337,8 +337,7 @@ describe("renderSingleImageToCanvas", () => { canvas, document, request: { - intent: "export", - quality: "full", + qualityTier: "export", targetSize: { width: 400, height: 225, @@ -385,8 +384,7 @@ describe("renderSingleImageToCanvas", () => { canvas: createCanvas(), document, request: { - intent: "preview", - quality: "interactive", + qualityTier: "interactive", targetSize: { width: 256, height: 144, @@ -414,8 +412,7 @@ describe("renderSingleImageToCanvas", () => { canvas: createCanvas(), document, request: { - intent: "export", - quality: "full", + qualityTier: "export", targetSize: { width: 400, height: 225, @@ -470,8 +467,7 @@ describe("renderSingleImageToCanvas", () => { canvas: createCanvas(), document, request: { - intent: "preview", - quality: "interactive", + qualityTier: "interactive", targetSize: { width: 256, height: 144, @@ -504,8 +500,7 @@ describe("renderSingleImageToCanvas", () => { canvas: createSnapshotCanvas(), document, request: { - intent: "preview", - quality: "interactive", + qualityTier: "interactive", targetSize: { width: 256, height: 144, @@ -538,8 +533,7 @@ describe("renderSingleImageToCanvas", () => { canvas, document, request: { - intent: "export", - quality: "full", + qualityTier: "export", targetSize: { width: 400, height: 225, @@ -557,14 +551,16 @@ describe("renderSingleImageToCanvas", () => { expect(renderImageToSurfaceMock).toHaveBeenCalledTimes(1); expect(applyImageCarrierTransformsMock).toHaveBeenCalledWith( expect.objectContaining({ - snapshots: expect.objectContaining({ - develop: expect.any(Object), - style: expect.any(Object), + analysisInputs: expect.objectContaining({ + stageSnapshots: expect.objectContaining({ + develop: expect.any(Object), + style: expect.any(Object), + }), }), }) ); - expect(applyImageCarrierTransformsMock.mock.calls[0]?.[0]?.snapshots.develop).not.toBeNull(); - expect(applyImageCarrierTransformsMock.mock.calls[0]?.[0]?.snapshots.style).not.toBe(canvas); + expect(applyImageCarrierTransformsMock.mock.calls[0]?.[0]?.analysisInputs.stageSnapshots.develop).not.toBeNull(); + expect(applyImageCarrierTransformsMock.mock.calls[0]?.[0]?.analysisInputs.stageSnapshots.style).not.toBe(canvas); }); it("runs develop-stage raster effects before film-stage and carrier transforms before style effects", async () => { @@ -605,8 +601,7 @@ describe("renderSingleImageToCanvas", () => { canvas, document, request: { - intent: "preview", - quality: "interactive", + qualityTier: "interactive", targetSize: { width: 256, height: 144, @@ -674,8 +669,7 @@ describe("renderSingleImageToCanvas", () => { canvas: createCanvas(), document, request: { - intent: "preview", - quality: "interactive", + qualityTier: "interactive", targetSize: { width: 256, height: 144, @@ -739,8 +733,7 @@ describe("renderSingleImageToCanvas", () => { canvas: createCanvas(), document, request: { - intent: "preview", - quality: "interactive", + qualityTier: "interactive", targetSize: { width: 256, height: 144, @@ -790,8 +783,7 @@ describe("renderSingleImageToCanvas", () => { canvas: createCanvas(), document, request: { - intent: "preview", - quality: "interactive", + qualityTier: "interactive", targetSize: { width: 256, height: 144, @@ -865,8 +857,7 @@ describe("renderSingleImageToCanvas", () => { canvas: createCanvas(), document, request: { - intent: "preview", - quality: "interactive", + qualityTier: "interactive", targetSize: { width: 256, height: 144, @@ -944,8 +935,7 @@ describe("renderSingleImageToCanvas", () => { canvas: createCanvas(), document, request: { - intent: "preview", - quality: "interactive", + qualityTier: "interactive", targetSize: { width: 256, height: 144, @@ -1019,8 +1009,7 @@ describe("renderSingleImageToCanvas", () => { canvas: createCanvas(), document, request: { - intent: "preview", - quality: "interactive", + qualityTier: "interactive", targetSize: { width: 256, height: 144, @@ -1054,8 +1043,7 @@ describe("renderSingleImageToCanvas", () => { canvas: createCanvas(), document, request: { - intent: "preview", - quality: "interactive", + qualityTier: "interactive", targetSize: { width: 256, height: 144, @@ -1065,7 +1053,7 @@ describe("renderSingleImageToCanvas", () => { }); const call = applyImageCarrierTransformsMock.mock.calls[0]?.[0]; - expect(call?.snapshots.style).toBe(call?.stageReferenceCanvas); + expect(call?.analysisInputs.stageSnapshots.style).toBe(call?.stageReferenceCanvas); }); it("keeps the film-stage seed stable between split and unsplit paths", async () => { @@ -1075,8 +1063,7 @@ describe("renderSingleImageToCanvas", () => { canvas: createCanvas(), document: baseDocument, request: { - intent: "preview", - quality: "interactive", + qualityTier: "interactive", targetSize: { width: 256, height: 144, @@ -1113,8 +1100,7 @@ describe("renderSingleImageToCanvas", () => { ], }), request: { - intent: "preview", - quality: "interactive", + qualityTier: "interactive", targetSize: { width: 256, height: 144, @@ -1182,8 +1168,7 @@ describe("renderSingleImageToCanvas", () => { canvas: createSnapshotCanvas(), document, request: { - intent: "preview", - quality: "interactive", + qualityTier: "interactive", targetSize: { width: 256, height: 144, @@ -1258,8 +1243,7 @@ describe("renderSingleImageToCanvas", () => { canvas: createSnapshotCanvas(), document, request: { - intent: "preview", - quality: "interactive", + qualityTier: "interactive", targetSize: { width: 256, height: 144, @@ -1348,8 +1332,7 @@ describe("renderSingleImageToCanvas", () => { canvas: firstCanvas, document: baseDocument, request: { - intent: "preview", - quality: "interactive", + qualityTier: "interactive", targetSize: { width: 256, height: 144, @@ -1363,8 +1346,7 @@ describe("renderSingleImageToCanvas", () => { canvas: secondCanvas, document: baseDocument, request: { - intent: "preview", - quality: "interactive", + qualityTier: "interactive", targetSize: { width: 256, height: 144, @@ -1378,8 +1360,7 @@ describe("renderSingleImageToCanvas", () => { canvas: changedCanvas, document: changedDocument, request: { - intent: "preview", - quality: "interactive", + qualityTier: "interactive", targetSize: { width: 256, height: 144, @@ -1414,8 +1395,7 @@ describe("renderSingleImageToCanvas", () => { canvas: squareCanvas, document, request: { - intent: "preview", - quality: "interactive", + qualityTier: "interactive", targetSize: { width: 256, height: 144, @@ -1429,8 +1409,7 @@ describe("renderSingleImageToCanvas", () => { canvas: wideCanvas, document, request: { - intent: "preview", - quality: "interactive", + qualityTier: "interactive", targetSize: { width: 256, height: 144, diff --git a/src/render/image/renderSingleImage.timestampOverlay.integration.test.ts b/src/render/image/renderSingleImage.timestampOverlay.integration.test.ts index 66044b3f..32ca2e13 100644 --- a/src/render/image/renderSingleImage.timestampOverlay.integration.test.ts +++ b/src/render/image/renderSingleImage.timestampOverlay.integration.test.ts @@ -155,8 +155,7 @@ describe("renderSingleImageToCanvas timestamp overlay integration", () => { canvas: createCanvas(), document: createDocument(), request: { - intent: "preview", - quality: "interactive", + qualityTier: "interactive", targetSize: { width: 400, height: 225, diff --git a/src/render/image/renderSingleImage.ts b/src/render/image/renderSingleImage.ts index be476fff..617dc6e6 100644 --- a/src/render/image/renderSingleImage.ts +++ b/src/render/image/renderSingleImage.ts @@ -15,10 +15,14 @@ import { type RenderBoundaryMetrics, type RenderSurfaceHandle, } from "@/lib/renderSurfaceHandle"; -import type { RenderIntent } from "@/lib/renderIntent"; +import { + createEmptyAnalysisLayerInputs, + validateAnalysisInputs, +} from "./analysisLayer"; import { applyImageCarrierTransforms } from "./asciiEffect"; import { applyImageEffects } from "./effectExecution"; import { applyImageOverlays, resolveImageOverlays } from "./overlayExecution"; +import { resolveRenderQualityTierConfig } from "./qualityTier"; import { applyImageSignalDamage } from "./signalDamageExecution"; import { assertSupportedImageRenderSnapshotPlan, @@ -100,12 +104,8 @@ const appendTraceOperation = ( }); }; -const resolveImageProcessingRenderIntent = (request: ImageRenderRequest): RenderIntent => { - if (request.intent === "export") { - return "export-full"; - } - return request.quality === "interactive" ? "preview-interactive" : "preview-full"; -}; +const resolveRequestTierConfig = (request: ImageRenderRequest) => + resolveRenderQualityTierConfig(request.qualityTier); const hasMaskedEffects = (effects: readonly ImageRenderDocument["effects"][number][]) => effects.some((effect) => Boolean(effect.maskId)); @@ -133,14 +133,15 @@ const renderSnapshotToSurface = async ({ renderSlotSuffix?: string; stage: "full" | "develop-base"; }): Promise => { + const tierConfig = resolveRequestTierConfig(request); const renderOptions = { source: resolveRuntimeSource(document), state: extractImageProcessState(document), targetSize: request.targetSize, seedKey: stage === "full" ? resolveFilmSeedKey(document) : `${document.id}:${stage}`, sourceCacheKey: `${document.revisionKey}:${stage}:${request.targetSize.width}x${request.targetSize.height}`, - strictErrors: request.strictErrors ?? request.intent === "export", - intent: resolveImageProcessingRenderIntent(request), + strictErrors: request.strictErrors ?? tierConfig.strictErrors, + intent: tierConfig.renderIntent, signal: request.signal, debug: request.debug, renderSlot: request.renderSlotId @@ -197,8 +198,7 @@ export const renderSingleImageToCanvas = async ({ hasMaskedDevelopEffects || snapshotPlan.requiresDevelopAnalysisSnapshot; const requiresDevelopBase = hasDevelopEffects || snapshotPlan.requiresDevelopAnalysisSnapshot; - let developSnapshotCanvas: HTMLCanvasElement | null = null; - let carrierAnalysisSnapshotCanvas: HTMLCanvasElement | null = null; + const analysisInputs = createEmptyAnalysisLayerInputs(); try { let surface: RenderSurfaceHandle; @@ -219,7 +219,7 @@ export const renderSingleImageToCanvas = async ({ let developSurface = developBaseResult.surface; if (requiresDevelopSnapshot) { - developSnapshotCanvas = trackSurfaceClone(developSurface); + analysisInputs.stageSnapshots.develop = trackSurfaceClone(developSurface); } if (hasDevelopEffects) { @@ -227,7 +227,7 @@ export const renderSingleImageToCanvas = async ({ surface: developSurface, document, effects: snapshotPlan.developEffects, - stageReferenceCanvas: developSnapshotCanvas ?? undefined, + stageReferenceCanvas: analysisInputs.stageSnapshots.develop ?? undefined, }); appendTraceOperation(debugStages, "develop", { kind: "effects", @@ -235,14 +235,15 @@ export const renderSingleImageToCanvas = async ({ effectCount: snapshotPlan.developEffects.length, }); + const filmTierConfig = resolveRequestTierConfig(request); const filmStageResult = await renderFilmStageToSurface({ source: developSurface.sourceCanvas, state: extractImageProcessState(document), targetSize: request.targetSize, seedKey: resolveFilmSeedKey(document), sourceCacheKey: `${document.revisionKey}:film-stage:${request.targetSize.width}x${request.targetSize.height}`, - strictErrors: request.strictErrors ?? request.intent === "export", - intent: resolveImageProcessingRenderIntent(request), + strictErrors: request.strictErrors ?? filmTierConfig.strictErrors, + intent: filmTierConfig.renderIntent, signal: request.signal, debug: request.debug, renderSlot: request.renderSlotId ? `${request.renderSlotId}:base-film-stage` : undefined, @@ -286,17 +287,28 @@ export const renderSingleImageToCanvas = async ({ } if (snapshotPlan.carrierTransforms.length > 0) { - carrierAnalysisSnapshotCanvas = trackSurfaceClone(surface); + analysisInputs.stageSnapshots.style = trackSurfaceClone(surface); + + const analysisValidation = validateAnalysisInputs( + snapshotPlan.analysisRequirements, + analysisInputs + ); + if (!analysisValidation.valid) { + const tierConfig = resolveRequestTierConfig(request); + if (tierConfig.strictErrors) { + throw new Error( + `Missing analysis inputs: ${analysisValidation.missing.join(", ")}` + ); + } + } + surface = await applyImageCarrierTransforms({ surface, carrierTransforms: snapshotPlan.carrierTransforms, document, request, - snapshots: { - develop: developSnapshotCanvas, - style: carrierAnalysisSnapshotCanvas, - }, - stageReferenceCanvas: carrierAnalysisSnapshotCanvas, + analysisInputs, + stageReferenceCanvas: analysisInputs.stageSnapshots.style!, }); appendTraceOperation(debugStages, "style", { kind: "carrier", @@ -309,7 +321,7 @@ export const renderSingleImageToCanvas = async ({ surface, signalDamage: snapshotPlan.signalDamage, document, - stageReferenceCanvas: carrierAnalysisSnapshotCanvas ?? undefined, + stageReferenceCanvas: analysisInputs.stageSnapshots.style ?? undefined, }); appendTraceOperation(debugStages, "style", { kind: "carrier", @@ -375,8 +387,9 @@ export const renderSingleImageToCanvas = async ({ } surface.materializeToCanvas(canvas); } finally { - releaseCanvas(carrierAnalysisSnapshotCanvas); - releaseCanvas(developSnapshotCanvas); + releaseCanvas(analysisInputs.stageSnapshots.style); + releaseCanvas(analysisInputs.stageSnapshots.develop); + releaseCanvas(analysisInputs.edgeMap); } let debugResult: ImageRenderDebugResult | undefined; diff --git a/src/render/image/snapshotPlan.test.ts b/src/render/image/snapshotPlan.test.ts index 437acc2e..b152d84a 100644 --- a/src/render/image/snapshotPlan.test.ts +++ b/src/render/image/snapshotPlan.test.ts @@ -64,7 +64,6 @@ describe("image render snapshot plan", () => { }); expect(plan.requiresDevelopAnalysisSnapshot).toBe(true); - expect(plan.requiresStyleAnalysisSnapshot).toBe(false); }); it("keeps carrier, style and finalize stages in stable order", () => { diff --git a/src/render/image/snapshotPlan.ts b/src/render/image/snapshotPlan.ts index aaaaf0b3..5a7631f7 100644 --- a/src/render/image/snapshotPlan.ts +++ b/src/render/image/snapshotPlan.ts @@ -1,3 +1,8 @@ +import { + type AnalysisRequirement, + deriveAnalysisRequirements, + requiresDevelopSnapshot, +} from "./analysisLayer"; import type { CarrierTransformNode, ImageEffectNode, SignalDamageNode } from "./types"; export interface ImageRenderSnapshotPlan { @@ -6,8 +11,8 @@ export interface ImageRenderSnapshotPlan { developEffects: ImageEffectNode[]; styleEffects: ImageEffectNode[]; finalizeEffects: ImageEffectNode[]; + analysisRequirements: AnalysisRequirement[]; requiresDevelopAnalysisSnapshot: boolean; - requiresStyleAnalysisSnapshot: boolean; } export const createImageRenderSnapshotPlan = ( @@ -20,18 +25,15 @@ export const createImageRenderSnapshotPlan = ( const enabledCarrierTransforms = options.carrierTransforms.filter((transform) => transform.enabled); const enabledSignalDamage = options.signalDamage.filter((node) => node.enabled); const enabledEffects = options.effects.filter((effect) => effect.enabled); + const analysisRequirements = deriveAnalysisRequirements(enabledCarrierTransforms); return { carrierTransforms: enabledCarrierTransforms, signalDamage: enabledSignalDamage, developEffects: enabledEffects.filter((effect) => effect.placement === "develop"), styleEffects: enabledEffects.filter((effect) => effect.placement === "style"), finalizeEffects: enabledEffects.filter((effect) => effect.placement === "finalize"), - requiresDevelopAnalysisSnapshot: enabledCarrierTransforms.some( - (transform) => transform.analysisSource === "develop" - ), - requiresStyleAnalysisSnapshot: enabledCarrierTransforms.some( - (transform) => transform.analysisSource === "style" - ), + analysisRequirements, + requiresDevelopAnalysisSnapshot: requiresDevelopSnapshot(analysisRequirements), }; }; diff --git a/src/render/image/types.ts b/src/render/image/types.ts index 46c96c3d..d4c2ba75 100644 --- a/src/render/image/types.ts +++ b/src/render/image/types.ts @@ -17,12 +17,7 @@ import type { TimestampOverlayPosition, } from "@/types"; import type { FilmProfileAny } from "@/types/film"; - -export const IMAGE_RENDER_INTENTS = ["preview", "export"] as const; -export type ImageRenderIntent = (typeof IMAGE_RENDER_INTENTS)[number]; - -export const IMAGE_RENDER_QUALITIES = ["interactive", "full"] as const; -export type ImageRenderQuality = (typeof IMAGE_RENDER_QUALITIES)[number]; +import type { RenderQualityTier } from "./qualityTier"; export const IMAGE_EFFECT_PLACEMENTS = ["develop", "style", "finalize"] as const; export type ImageEffectPlacement = (typeof IMAGE_EFFECT_PLACEMENTS)[number]; @@ -359,6 +354,23 @@ export type SemanticOverlayNode = | CaptionSemanticOverlayNode | WatermarkSemanticOverlayNode; +export interface SignalDriftMotionParams { + driftAmplitude: number; + intensity: number; +} + +export interface SignalDriftMotionProgram { + id: string; + type: "signal-drift"; + enabled: boolean; + durationMs: number; + fps: number; + loop: boolean; + params: SignalDriftMotionParams; +} + +export type MotionProgram = SignalDriftMotionProgram; + export interface CanvasImageRenderStateV1 { geometry: ImageRenderGeometry; develop: ImageRenderDevelopState; @@ -367,6 +379,7 @@ export interface CanvasImageRenderStateV1 { signalDamage: SignalDamageNode[]; semanticOverlays: SemanticOverlayNode[]; effects: ImageEffectNode[]; + motionPrograms: MotionProgram[]; film: ImageRenderFilmState; output: ImageRenderOutputState; } @@ -385,8 +398,7 @@ export interface ImageRenderDocument extends CanvasImageRenderStateV1 { } export interface ImageRenderRequest { - intent: ImageRenderIntent; - quality: ImageRenderQuality; + qualityTier: RenderQualityTier; targetSize: ImageRenderTargetSize; timestampText?: string | null; strictErrors?: boolean; @@ -443,6 +455,16 @@ const SEMANTIC_OVERLAY_TYPES = new Set(["timestamp", "caption", "watermark"]); const isSemanticOverlayNode = (value: unknown): value is SemanticOverlayNode => isRecord(value) && typeof value.type === "string" && SEMANTIC_OVERLAY_TYPES.has(value.type) && "params" in value; +const MOTION_PROGRAM_TYPES = new Set(["signal-drift"]); + +const isMotionProgram = (value: unknown): value is MotionProgram => + isRecord(value) && + typeof value.type === "string" && + MOTION_PROGRAM_TYPES.has(value.type) && + typeof value.durationMs === "number" && + typeof value.fps === "number" && + "params" in value; + const mapLegacyAsciiEffectToCarrierTransform = ( effect: ImageAsciiCarrierTransformNode & { placement?: ImageEffectPlacement } ): CarrierTransformNode => ({ @@ -495,6 +517,11 @@ export const normalizeCanvasImageRenderState = ( ) ? ((state as CanvasImageRenderStateV1 & { semanticOverlays?: unknown[] }).semanticOverlays ?? []) : []; + const rawMotionPrograms = Array.isArray( + (state as CanvasImageRenderStateV1 & { motionPrograms?: unknown[] }).motionPrograms + ) + ? ((state as CanvasImageRenderStateV1 & { motionPrograms?: unknown[] }).motionPrograms ?? []) + : []; const explicitCarrierTransforms = rawCarrierTransforms .filter(isCarrierTransformNode) @@ -508,6 +535,9 @@ export const normalizeCanvasImageRenderState = ( const semanticOverlayNodes = rawSemanticOverlays .filter(isSemanticOverlayNode) .map((node) => cloneImageRenderValue(node)); + const motionProgramNodes = rawMotionPrograms + .filter(isMotionProgram) + .map((node) => cloneImageRenderValue(node)); const migratedCarrierTransforms = explicitCarrierTransforms.length > 0 ? explicitCarrierTransforms @@ -535,6 +565,7 @@ export const normalizeCanvasImageRenderState = ( signalDamage: signalDamageNodes, semanticOverlays: semanticOverlayNodes, effects: rasterEffects, + motionPrograms: motionProgramNodes, }; }; From 34cf5475cb70d4686c386466236c6b7012b8a182 Mon Sep 17 00:00:00 2001 From: WenHao Chen <157730893+YakiHugo@users.noreply.github.com> Date: Sat, 25 Apr 2026 03:39:37 -0700 Subject: [PATCH 09/14] chore: enforce dead-code detection in verify pipeline MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Remove --no-exit-code from knip so unused exports fail CI - Add dead-code step to verify pipeline (lint → dead-code → test → build) - AGENTS.md: document knip rule and @public JSDoc tag for multi-slice exemptions Co-Authored-By: Claude Opus 4.6 (1M context) --- AGENTS.md | 2 ++ package.json | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index f2885560..0722f6ec 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -21,6 +21,8 @@ - Use `try`/`catch` and similar control flow only for failures you expect and can handle; do not add unreachable recovery paths that hide bugs instead of failing fast during development. - Write a brief comment only when a future agent needs to know why the code is written this way and that reason cannot be inferred from the code; do not narrate behavior, translate code into natural language, or restate responsibilities. - When the project requires the use of newly added basic components, priority should be given to shadcn-related components. +- `pnpm dead-code` (knip) catches unused files and exports that `tsc` and eslint miss. Do not ship exported code with zero consumers. When a multi-slice task requires landing a type or function before its consumer exists, add a `/** @public — consumed by */` JSDoc tag to suppress knip; remove the tag when the consumer lands. + ## Long Tasks - Treat a task as long when it cannot be completed safely in one session without explicit slicing. diff --git a/package.json b/package.json index 572f7e0c..28d428e0 100644 --- a/package.json +++ b/package.json @@ -16,14 +16,14 @@ "lint": "eslint src server/src shared --cache --cache-location node_modules/.cache/.eslintcache", "lint:fix": "eslint src server/src shared --fix --cache --cache-location node_modules/.cache/.eslintcache", "typecheck": "tsc -b && pnpm --filter server typecheck", - "dead-code": "knip --no-progress --no-exit-code --reporter compact", + "dead-code": "knip --no-progress --reporter compact", "format": "prettier --write \"src/**/*.{ts,tsx,css}\"", "format:check": "prettier --check \"src/**/*.{ts,tsx,css}\"", "test": "vitest --run", "test:watch": "vitest", "verify:prompt": "vitest --run server/src/gateway/prompt server/src/gateway/router server/src/routes/image-generate server/src/chat/persistence/postgres.promptArtifacts.test.ts server/src/chat/persistence/__integration", "verify:integration": "vitest --run server/src/chat/persistence/__integration", - "verify": "pnpm run generate:shaders && pnpm lint && pnpm test && pnpm build" + "verify": "pnpm run generate:shaders && pnpm lint && pnpm dead-code && pnpm test && pnpm build" }, "dependencies": { "@ai-sdk/anthropic": "^3.0.46", From 094c12aa4b31fff55592b827d19f2906bfeb3807 Mon Sep 17 00:00:00 2001 From: WenHao Chen <157730893+YakiHugo@users.noreply.github.com> Date: Sat, 25 Apr 2026 03:45:57 -0700 Subject: [PATCH 10/14] fix(renderer): resolve tsc errors and revert strict knip in verify MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Fix union narrowing in analysisLayer test helpers (use Extract + const assertions) - Add missing motionPrograms field to createNeutralCanvasImageRenderState - Revert strict knip in verify — codebase has ~260 pre-existing unused exports; AGENTS.md rule enforces agent discipline, knip stays as reporting tool Co-Authored-By: Claude Opus 4.6 (1M context) --- AGENTS.md | 2 +- package.json | 4 ++-- src/render/image/analysisLayer.test.ts | 12 ++++++------ src/render/image/stateCompiler.ts | 1 + 4 files changed, 10 insertions(+), 9 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 0722f6ec..bd39c29f 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -21,7 +21,7 @@ - Use `try`/`catch` and similar control flow only for failures you expect and can handle; do not add unreachable recovery paths that hide bugs instead of failing fast during development. - Write a brief comment only when a future agent needs to know why the code is written this way and that reason cannot be inferred from the code; do not narrate behavior, translate code into natural language, or restate responsibilities. - When the project requires the use of newly added basic components, priority should be given to shadcn-related components. -- `pnpm dead-code` (knip) catches unused files and exports that `tsc` and eslint miss. Do not ship exported code with zero consumers. When a multi-slice task requires landing a type or function before its consumer exists, add a `/** @public — consumed by */` JSDoc tag to suppress knip; remove the tag when the consumer lands. +- `pnpm dead-code` (knip) catches unused files and exports that `tsc` and eslint miss. Run it before committing and verify your change does not introduce new unused exports. When a multi-slice task requires landing a type or function before its consumer exists, add a `/** @public — consumed by */` JSDoc tag to suppress knip; remove the tag when the consumer lands. ## Long Tasks diff --git a/package.json b/package.json index 28d428e0..572f7e0c 100644 --- a/package.json +++ b/package.json @@ -16,14 +16,14 @@ "lint": "eslint src server/src shared --cache --cache-location node_modules/.cache/.eslintcache", "lint:fix": "eslint src server/src shared --fix --cache --cache-location node_modules/.cache/.eslintcache", "typecheck": "tsc -b && pnpm --filter server typecheck", - "dead-code": "knip --no-progress --reporter compact", + "dead-code": "knip --no-progress --no-exit-code --reporter compact", "format": "prettier --write \"src/**/*.{ts,tsx,css}\"", "format:check": "prettier --check \"src/**/*.{ts,tsx,css}\"", "test": "vitest --run", "test:watch": "vitest", "verify:prompt": "vitest --run server/src/gateway/prompt server/src/gateway/router server/src/routes/image-generate server/src/chat/persistence/postgres.promptArtifacts.test.ts server/src/chat/persistence/__integration", "verify:integration": "vitest --run server/src/chat/persistence/__integration", - "verify": "pnpm run generate:shaders && pnpm lint && pnpm dead-code && pnpm test && pnpm build" + "verify": "pnpm run generate:shaders && pnpm lint && pnpm test && pnpm build" }, "dependencies": { "@ai-sdk/anthropic": "^3.0.46", diff --git a/src/render/image/analysisLayer.test.ts b/src/render/image/analysisLayer.test.ts index 320e4b1a..fc13da6e 100644 --- a/src/render/image/analysisLayer.test.ts +++ b/src/render/image/analysisLayer.test.ts @@ -13,23 +13,23 @@ const createMockCanvas = () => ({ width: 100, height: 100 }) as unknown as HTMLCanvasElement; const createAsciiTransform = ( - overrides?: Partial + overrides?: Partial> ): CarrierTransformNode => ({ id: "ascii-1", - type: "ascii", + type: "ascii" as const, enabled: true, - analysisSource: "style", + analysisSource: "style" as const, params: {} as never, ...overrides, }); const createHalftoneTransform = ( - overrides?: Partial + overrides?: Partial> ): CarrierTransformNode => ({ id: "halftone-1", - type: "halftone", + type: "halftone" as const, enabled: true, - analysisSource: "style", + analysisSource: "style" as const, params: {} as never, ...overrides, }); diff --git a/src/render/image/stateCompiler.ts b/src/render/image/stateCompiler.ts index e60fcd2a..6c7e6aa4 100644 --- a/src/render/image/stateCompiler.ts +++ b/src/render/image/stateCompiler.ts @@ -173,6 +173,7 @@ export const createNeutralCanvasImageRenderState = (): CanvasImageRenderStateV1 signalDamage: [], semanticOverlays: [], effects: [], + motionPrograms: [], film: { profileId: null, profile: undefined, From 6f1925f8fdc12bd1a1e273a1957613166bbd48c1 Mon Sep 17 00:00:00 2001 From: WenHao Chen <157730893+YakiHugo@users.noreply.github.com> Date: Sat, 25 Apr 2026 03:51:01 -0700 Subject: [PATCH 11/14] fix(renderer): signal-damage mask reference and fps guard - Capture a dedicated stage snapshot before signal-damage execution so masked signal-damage nodes get a correct reference canvas even when no carrier transforms are present - Clamp fps to >= 1 in motion frame context to prevent Infinity/NaN from malformed or persisted states with fps <= 0 Co-Authored-By: Claude Opus 4.6 (1M context) --- src/render/image/motionRender.ts | 7 +++++-- src/render/image/renderSingleImage.ts | 3 ++- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/src/render/image/motionRender.ts b/src/render/image/motionRender.ts index 0aac9c77..24996705 100644 --- a/src/render/image/motionRender.ts +++ b/src/render/image/motionRender.ts @@ -21,17 +21,20 @@ export interface MotionFrameResult { canvas: HTMLCanvasElement; } +const safeFps = (fps: number) => Math.max(1, fps); + export const computeMotionFrameCount = (program: MotionProgram): number => - Math.max(1, Math.ceil((program.durationMs / 1000) * program.fps)); + Math.max(1, Math.ceil((program.durationMs / 1000) * safeFps(program.fps))); export const createMotionFrameContext = ( program: MotionProgram, frameIndex: number ): MotionFrameContext => { + const fps = safeFps(program.fps); const totalFrames = computeMotionFrameCount(program); return { frameIndex, - timeMs: (frameIndex / program.fps) * 1000, + timeMs: (frameIndex / fps) * 1000, normalizedTime: totalFrames <= 1 ? 0 : frameIndex / (program.loop ? totalFrames : totalFrames - 1), totalFrames, diff --git a/src/render/image/renderSingleImage.ts b/src/render/image/renderSingleImage.ts index 617dc6e6..3047c52c 100644 --- a/src/render/image/renderSingleImage.ts +++ b/src/render/image/renderSingleImage.ts @@ -317,11 +317,12 @@ export const renderSingleImageToCanvas = async ({ } if (snapshotPlan.signalDamage.length > 0) { + const signalDamageReferenceCanvas = trackSurfaceClone(surface); surface = await applyImageSignalDamage({ surface, signalDamage: snapshotPlan.signalDamage, document, - stageReferenceCanvas: analysisInputs.stageSnapshots.style ?? undefined, + stageReferenceCanvas: signalDamageReferenceCanvas, }); appendTraceOperation(debugStages, "style", { kind: "carrier", From fe2bc33c41f792605e1f5ceb26218365ee1681ce Mon Sep 17 00:00:00 2001 From: WenHao Chen <157730893+YakiHugo@users.noreply.github.com> Date: Sat, 25 Apr 2026 03:54:34 -0700 Subject: [PATCH 12/14] docs(agents): add PR comment review-and-resolve workflow Co-Authored-By: Claude Opus 4.6 (1M context) --- AGENTS.md | 1 + 1 file changed, 1 insertion(+) diff --git a/AGENTS.md b/AGENTS.md index bd39c29f..4ce3ca5c 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -89,6 +89,7 @@ - Use the gh tool for GitHub-related operations. - When commits are requested, keep them atomic: commit each validated independent step rather than bundling unrelated changes. - When uncommitted work spans multiple validated slices, split one commit per slice; temporarily rewind shared task md/json to each slice's boundary state before staging, so every commit reflects the state at that slice's completion and nothing later. +- Before pushing a PR or after being asked to check a PR, read its review comments (`gh api repos/{owner}/{repo}/pulls/{n}/comments`). For each comment that identifies a real issue: fix it, reply with the fix commit hash and a one-line summary, then resolve the thread via GraphQL `resolveReviewThread`. Do not resolve threads that are open questions or that you have not addressed. ## Commit & Pull Request Guidelines From 07d362b18ede673c555d277b2314d1821cddb5c4 Mon Sep 17 00:00:00 2001 From: WenHao Chen <157730893+YakiHugo@users.noreply.github.com> Date: Sat, 25 Apr 2026 04:12:09 -0700 Subject: [PATCH 13/14] fix(renderer): remove double intensity in motion drift and release signal-damage snapshot MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Signal-drift offsets no longer pre-multiply by intensity — the shader's u_intensity uniform is the single scaling control, fixing quadratic attenuation - Signal-damage stage snapshot now released in finally block, matching the style/finalize cleanup pattern to prevent canvas accumulation during previews Co-Authored-By: Claude Opus 4.6 (1M context) --- src/render/image/motionRender.test.ts | 2 +- src/render/image/motionRender.ts | 8 ++++---- src/render/image/renderSingleImage.ts | 24 ++++++++++++++---------- 3 files changed, 19 insertions(+), 15 deletions(-) diff --git a/src/render/image/motionRender.test.ts b/src/render/image/motionRender.test.ts index 5ae49856..d4745a03 100644 --- a/src/render/image/motionRender.test.ts +++ b/src/render/image/motionRender.test.ts @@ -180,7 +180,7 @@ describe("motionRender", () => { const result = applyMotionProgramToDocument(program, baseDoc, frame); const drift = result.signalDamage[0]!.params; - const maxDrift = program.params.driftAmplitude * program.params.intensity; + const maxDrift = program.params.driftAmplitude; expect(Math.abs(drift.redOffsetX)).toBeGreaterThan(maxDrift * 0.9); }); diff --git a/src/render/image/motionRender.ts b/src/render/image/motionRender.ts index 24996705..4d463cee 100644 --- a/src/render/image/motionRender.ts +++ b/src/render/image/motionRender.ts @@ -54,12 +54,12 @@ const applySignalDriftToDocument = ( type: "channel-drift", enabled: true, params: { - redOffsetX: Math.sin(t) * driftAmplitude * intensity, - redOffsetY: Math.cos(t * 0.7) * driftAmplitude * intensity * 0.3, + redOffsetX: Math.sin(t) * driftAmplitude, + redOffsetY: Math.cos(t * 0.7) * driftAmplitude * 0.3, greenOffsetX: 0, greenOffsetY: 0, - blueOffsetX: Math.sin(t + Math.PI) * driftAmplitude * intensity, - blueOffsetY: Math.cos(t * 0.7 + Math.PI) * driftAmplitude * intensity * 0.3, + blueOffsetX: Math.sin(t + Math.PI) * driftAmplitude, + blueOffsetY: Math.cos(t * 0.7 + Math.PI) * driftAmplitude * 0.3, intensity, }, }; diff --git a/src/render/image/renderSingleImage.ts b/src/render/image/renderSingleImage.ts index 3047c52c..002f66a8 100644 --- a/src/render/image/renderSingleImage.ts +++ b/src/render/image/renderSingleImage.ts @@ -318,16 +318,20 @@ export const renderSingleImageToCanvas = async ({ if (snapshotPlan.signalDamage.length > 0) { const signalDamageReferenceCanvas = trackSurfaceClone(surface); - surface = await applyImageSignalDamage({ - surface, - signalDamage: snapshotPlan.signalDamage, - document, - stageReferenceCanvas: signalDamageReferenceCanvas, - }); - appendTraceOperation(debugStages, "style", { - kind: "carrier", - carrierCount: snapshotPlan.signalDamage.length, - }); + try { + surface = await applyImageSignalDamage({ + surface, + signalDamage: snapshotPlan.signalDamage, + document, + stageReferenceCanvas: signalDamageReferenceCanvas, + }); + appendTraceOperation(debugStages, "style", { + kind: "carrier", + carrierCount: snapshotPlan.signalDamage.length, + }); + } finally { + releaseCanvas(signalDamageReferenceCanvas); + } } if (snapshotPlan.styleEffects.length > 0) { From 49f037b20b5a038e8231399a743ec0194c9bccc1 Mon Sep 17 00:00:00 2001 From: WenHao Chen <157730893+YakiHugo@users.noreply.github.com> Date: Sat, 25 Apr 2026 06:27:27 -0700 Subject: [PATCH 14/14] fix(renderer): graceful abort in motion sequence render Catch AbortError thrown by the image pipeline during per-frame render so cancellation returns already-produced frames instead of rejecting the entire sequence. Co-Authored-By: Claude Opus 4.6 (1M context) --- src/render/image/motionRender.ts | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/src/render/image/motionRender.ts b/src/render/image/motionRender.ts index 4d463cee..dbb75318 100644 --- a/src/render/image/motionRender.ts +++ b/src/render/image/motionRender.ts @@ -109,11 +109,16 @@ export const renderMotionSequence = async ({ canvas.width = targetSize.width; canvas.height = targetSize.height; - await renderSingleImageToCanvas({ - canvas, - document: frameDocument, - request: { qualityTier, targetSize, signal }, - }); + try { + await renderSingleImageToCanvas({ + canvas, + document: frameDocument, + request: { qualityTier, targetSize, signal }, + }); + } catch (err) { + if (err instanceof DOMException && err.name === "AbortError") break; + throw err; + } if (signal?.aborted) break;