feat: add detail surface protocol workflow
Some checks failed
Lint / build (20) (push) Has been cancelled
Playwright Tests / test (push) Has been cancelled

This commit is contained in:
Your Name 2026-03-21 20:55:04 +08:00
parent f082c20141
commit 10e1096c83
13 changed files with 2439 additions and 62 deletions

View File

@ -30,6 +30,13 @@ type DetailArchiveSlot = {
contentType?: string;
} | null;
type DetailArchiveSurfaceReview = {
screenIndex?: number;
action?: string;
passed?: boolean;
issues?: string[];
} | null;
type DetailGenerationMode = "classic" | "agent_layout";
function sanitizeGenerationMode(value: unknown): DetailGenerationMode {
@ -75,6 +82,25 @@ function sanitizeSlots(input: unknown): DetailArchiveSlot[] {
});
}
function sanitizeUiSurfaceSnapshot(input: unknown) {
if (!input || typeof input !== "object" || Array.isArray(input)) return null;
return input as Record<string, unknown>;
}
function sanitizeUiSurfaceLatestReview(input: unknown): DetailArchiveSurfaceReview {
if (!input || typeof input !== "object" || Array.isArray(input)) return null;
const raw = input as Record<string, unknown>;
const issues = Array.isArray(raw.issues)
? raw.issues.map((item) => String(item || "").trim()).filter(Boolean).slice(0, 10)
: [];
return {
screenIndex: Number.isFinite(Number(raw.screenIndex)) ? Math.max(0, Number(raw.screenIndex)) : undefined,
action: String(raw.action || "").trim() || undefined,
passed: typeof raw.passed === "boolean" ? raw.passed : undefined,
issues,
};
}
async function ensureProjectOwnership(projectId: string, userId: string) {
const [project] = await db
.select({ id: canvasProject.id })
@ -146,6 +172,8 @@ export async function POST(
const brief = sanitizeBrief(archiveRaw.brief);
const slots = sanitizeSlots(archiveRaw.slots);
const uiSurfaceSnapshot = sanitizeUiSurfaceSnapshot(archiveRaw.uiSurfaceSnapshot);
const uiSurfaceLatestReview = sanitizeUiSurfaceLatestReview(archiveRaw.uiSurfaceLatestReview);
const [saved] = await db
.insert(canvasDetailArchive)
@ -160,6 +188,8 @@ export async function POST(
summary: summary || null,
imageCount: Number.isFinite(imageCount) ? Math.max(0, Math.round(imageCount)) : null,
coverUrl: coverUrl || null,
uiSurfaceSnapshot,
uiSurfaceLatestReview,
})
.returning();
@ -201,6 +231,8 @@ export async function PATCH(
const generationMode = sanitizeGenerationMode(archiveRaw.generationMode);
const brief = sanitizeBrief(archiveRaw.brief);
const slots = sanitizeSlots(archiveRaw.slots);
const uiSurfaceSnapshot = sanitizeUiSurfaceSnapshot(archiveRaw.uiSurfaceSnapshot);
const uiSurfaceLatestReview = sanitizeUiSurfaceLatestReview(archiveRaw.uiSurfaceLatestReview);
const [updated] = await db
.update(canvasDetailArchive)
@ -213,6 +245,8 @@ export async function PATCH(
summary: summary || null,
imageCount: Number.isFinite(imageCount) ? Math.max(0, Math.round(imageCount)) : null,
coverUrl: coverUrl || null,
uiSurfaceSnapshot,
uiSurfaceLatestReview,
updatedAt: new Date(),
})
.where(

View File

@ -16,7 +16,9 @@ import {
buildDetailUiProtocol,
type DetailUiProtocol,
} from "@/lib/detail-ui-protocol";
import { buildDetailUiSurfaceSnapshot } from "@/lib/detail-ui-surface-adapter";
import { validateDetailUiProtocol } from "@/lib/detail-ui-validator";
import type { UiSurfaceSnapshot } from "@/lib/ui-surface-protocol";
export const runtime = "nodejs";
export const maxDuration = 180;
@ -236,6 +238,7 @@ type DetailPlanResponse = {
};
screens: DetailPlanScreen[];
uiProtocol?: DetailUiProtocol;
uiSurface?: UiSurfaceSnapshot;
model?: {
id: string;
displayName: string;
@ -1237,6 +1240,27 @@ function buildFallbackPlan(params: {
},
screens,
uiProtocol,
uiSurface:
params.generationMode === "agent_layout"
? buildDetailUiSurfaceSnapshot({
generationMode: params.generationMode,
screenSize: {
width: 790,
height: 1300,
},
summary:
language === "en"
? `Generated ${count} storyboard frames with fixed size 790x1300.`
: `已生成 ${count} 段详情分镜,固定尺寸 790x1300。`,
globalStyle,
screens,
uiProtocol,
uiValidation: {
passed: uiValidation.passed,
issues: uiValidation.issues.map((item) => item.message),
},
})
: undefined,
};
}
@ -2160,6 +2184,24 @@ export async function POST(request: Request) {
},
screens,
uiProtocol,
uiSurface:
generationMode === "agent_layout"
? buildDetailUiSurfaceSnapshot({
generationMode,
screenSize: {
width: 790,
height: 1300,
},
summary,
globalStyle,
screens,
uiProtocol,
uiValidation: {
passed: uiValidation.passed,
issues: uiValidation.issues.map((item) => item.message),
},
})
: undefined,
model: {
id: plannerConfig.model.id,
displayName: plannerConfig.model.displayName,

View File

@ -17,7 +17,9 @@ import {
type DetailGenerationMode,
type DetailUiScreenReviewPatch,
} from "@/lib/detail-ui-protocol";
import { buildDetailUiSurfacePatchFromReview } from "@/lib/detail-ui-surface-adapter";
import { validateDetailUiProtocol } from "@/lib/detail-ui-validator";
import type { UiSurfacePatch } from "@/lib/ui-surface-protocol";
export const runtime = "nodejs";
export const maxDuration = 180;
@ -25,6 +27,7 @@ export const maxDuration = 180;
type DetailReviewRequest = {
outputLanguage?: string;
generationMode?: string;
surfaceId?: string;
imageUrl?: string;
referenceImages?: string[];
screen?: {
@ -249,6 +252,7 @@ export async function POST(request: Request) {
layoutFixes: [],
revisedPrompt: fallbackPrompt,
uiPatch: undefined,
uiSurfacePatch: undefined,
model: null,
},
});
@ -272,6 +276,7 @@ export async function POST(request: Request) {
layoutFixes: [],
revisedPrompt: fallbackPrompt,
uiPatch: undefined,
uiSurfacePatch: undefined,
model: null,
},
});
@ -373,6 +378,7 @@ export async function POST(request: Request) {
layoutFixes: [],
revisedPrompt: fallbackPrompt,
uiPatch: undefined,
uiSurfacePatch: undefined,
model: {
id: reviewerConfig.model.id,
displayName: reviewerConfig.model.displayName,
@ -482,6 +488,35 @@ export async function POST(request: Request) {
(uiPatch.notes.length > 0 || !uiValidation.passed)
? "protocol_patch"
: "regenerate_image";
const uiSurfacePatch: UiSurfacePatch | null =
generationMode === "agent_layout" &&
reviewAction === "protocol_patch" &&
uiPatch &&
body.surfaceId
? buildDetailUiSurfacePatchFromReview({
surfaceId: body.surfaceId,
screenIndex: Number(body.screen?.index || 1),
screen: {
index: Number(body.screen?.index || 1),
title: String(body.screen?.title || "").trim(),
subtitle: String(body.screen?.subtitle || "").trim(),
copy: String(body.screen?.copy || "").trim(),
screenType:
body.screen?.screenType === "A" ||
body.screen?.screenType === "B" ||
body.screen?.screenType === "C"
? body.screen.screenType
: undefined,
objective: undefined,
layoutNote: String(body.screen?.layoutNote || "").trim(),
},
uiPatch,
uiValidation: {
passed: uiValidation.passed,
issues: uiValidation.issues.map((item) => item.message),
},
})
: null;
return NextResponse.json({
success: true,
@ -495,6 +530,7 @@ export async function POST(request: Request) {
layoutFixes: [...layoutFixes, ...typographyFixes, ...conversionFixes].slice(0, 10),
revisedPrompt,
uiPatch,
uiSurfacePatch: uiSurfacePatch || undefined,
uiValidation: {
passed: uiValidation.passed,
issues: uiValidation.issues.map((item) => item.message),

View File

@ -8,10 +8,16 @@ import { createPortal } from "react-dom";
import type { DetailUiProtocol } from "@/lib/detail-ui-protocol";
import type { DetailUiScreenReviewPatch } from "@/lib/detail-ui-protocol";
import {
buildDetailTextLayerNodesFromProtocol,
buildDetailTextLayerNodes,
buildDetailTextLayerNodesFromReviewPatch,
buildLegacyDetailTextLayerNodes,
buildDetailTextLayerNodesFromSurfaceScreen,
collectManagedDetailTextLayerNodeIds,
} from "@/lib/detail-ui-renderer";
import {
applyUiSurfacePatch,
type UiSurfacePatch,
type UiSurfaceSnapshot,
} from "@/lib/ui-surface-protocol";
import useSWR, { useSWRConfig } from "swr";
import { Send, Loader2, ImagePlus, Paperclip, X, Plus, History, LayoutGrid, ArrowRightToLine, RefreshCw, Globe, Box, ArrowUp, Video, Sparkles, Check, ChevronDown, ChevronUp, AlertCircle, CheckCircle2, Brain, ThumbsUp, ThumbsDown, Eye, Trash2, Download, MessageCircle } from "lucide-react";
import { Button } from "@/components/ui/button";
@ -76,6 +82,8 @@ import { ImageParamsBar, type ImageGenerationParams } from "@/components/image-p
import { VideoParamsBar, type VideoGenerationParams } from "@/components/video-params-bar";
import { ImageGenerationLoading } from "@/components/image-generation-loading";
import type { VideoToolCapabilityContract } from "@/types/video-tool-capability";
import { DetailUiValidationPanel } from "@/components/canvas/DetailUiValidationPanel";
import { DetailSurfaceInspectorPanel } from "@/components/canvas/DetailSurfaceInspectorPanel";
// 标签悬停预览信息
interface TagPreviewInfo {
@ -1177,6 +1185,13 @@ type DetailArchive = {
summary?: string;
imageCount?: number;
coverUrl?: string;
uiSurfaceSnapshot?: UiSurfaceSnapshot | null;
uiSurfaceLatestReview?: {
screenIndex?: number;
action?: "pass" | "protocol_patch" | "regenerate_image";
passed?: boolean;
issues?: string[];
} | null;
};
type DetailPlanResponse = {
@ -1231,6 +1246,10 @@ type DetailPlanResponse = {
passed?: boolean;
issues?: string[];
};
uiValidation?: {
passed?: boolean;
issues?: string[];
};
preflight?: {
passed?: boolean;
designDirectorScore?: number;
@ -1244,6 +1263,7 @@ type DetailPlanResponse = {
providerType: string;
};
uiProtocol?: DetailUiProtocol;
uiSurface?: UiSurfaceSnapshot;
};
type DetailPlanScreenSpec = NonNullable<DetailPlanResponse["screens"]>[number];
@ -1370,6 +1390,11 @@ type DetailScreenReviewResult = {
layoutFixes?: string[];
revisedPrompt?: string;
uiPatch?: DetailUiScreenReviewPatch;
uiSurfacePatch?: UiSurfacePatch;
uiValidation?: {
passed?: boolean;
issues?: string[];
};
model?: {
id?: string;
displayName?: string;
@ -1377,6 +1402,18 @@ type DetailScreenReviewResult = {
} | null;
};
type DetailUiValidationSnapshot = {
passed?: boolean;
issues?: string[];
};
type DetailUiReviewSnapshot = {
screenIndex: number;
action?: "pass" | "protocol_patch" | "regenerate_image";
passed?: boolean;
issues?: string[];
};
function normalizeDetailGenerationMode(value: unknown): DetailGenerationMode {
return value === "agent_layout" ? "agent_layout" : "classic";
}
@ -1549,6 +1586,10 @@ function buildDetailResultMarkdown(params: {
passed?: boolean;
issues?: string[];
};
uiValidation?: {
passed?: boolean;
issues?: string[];
};
globalStyle?: {
designLanguage?: string;
typography?: {
@ -1610,6 +1651,14 @@ function buildDetailResultMarkdown(params: {
params.qualityChecks.issues.forEach((issue) => lines.push(`- ${issue}`));
}
}
if (params.uiValidation) {
if (params.uiValidation.passed) {
lines.push("协议排版校验:通过");
} else if (params.uiValidation.issues?.length) {
lines.push("协议排版校验:存在待优化项");
params.uiValidation.issues.slice(0, 3).forEach((issue) => lines.push(`- ${issue}`));
}
}
lines.push(`共生成 ${params.images.length} 张详情图。`);
if (params.globalStyle) {
const typography = params.globalStyle.typography;
@ -1746,6 +1795,16 @@ export function CanvasChatPanel({
current: 0,
total: 0,
});
const [detailUiValidation, setDetailUiValidation] = useState<{
plan: DetailUiValidationSnapshot | null;
latestScreen: DetailUiReviewSnapshot | null;
}>({
plan: null,
latestScreen: null,
});
const [detailSurfaceSnapshot, setDetailSurfaceSnapshot] = useState<UiSurfaceSnapshot | null>(
null,
);
const [detailSlots, setDetailSlots] = useState<Array<PickedObjectAttachment | null>>(
() => DETAIL_SLOT_CONFIGS.map(() => null)
);
@ -3796,6 +3855,36 @@ export function CanvasChatPanel({
summary: item.summary ? String(item.summary) : undefined,
imageCount: Number(item.imageCount || 0) || undefined,
coverUrl: item.coverUrl ? String(item.coverUrl) : undefined,
uiSurfaceSnapshot:
item.uiSurfaceSnapshot &&
typeof item.uiSurfaceSnapshot === "object" &&
!Array.isArray(item.uiSurfaceSnapshot)
? (item.uiSurfaceSnapshot as UiSurfaceSnapshot)
: null,
uiSurfaceLatestReview:
item.uiSurfaceLatestReview &&
typeof item.uiSurfaceLatestReview === "object" &&
!Array.isArray(item.uiSurfaceLatestReview)
? {
screenIndex: Number((item.uiSurfaceLatestReview as any).screenIndex || 0) || undefined,
action:
(item.uiSurfaceLatestReview as any).action === "protocol_patch" ||
(item.uiSurfaceLatestReview as any).action === "regenerate_image" ||
(item.uiSurfaceLatestReview as any).action === "pass"
? (item.uiSurfaceLatestReview as any).action
: undefined,
passed:
typeof (item.uiSurfaceLatestReview as any).passed === "boolean"
? (item.uiSurfaceLatestReview as any).passed
: undefined,
issues: Array.isArray((item.uiSurfaceLatestReview as any).issues)
? (item.uiSurfaceLatestReview as any).issues
.map((issue: unknown) => String(issue || "").trim())
.filter(Boolean)
.slice(0, 10)
: [],
}
: null,
}))
.sort(
(a, b) =>
@ -5575,6 +5664,8 @@ export function CanvasChatPanel({
generationMode: DetailGenerationMode;
brief: DetailBriefForm;
slots: Array<PickedObjectAttachment | null>;
uiSurfaceSnapshot?: UiSurfaceSnapshot | null;
uiSurfaceLatestReview?: DetailArchive["uiSurfaceLatestReview"];
}): Promise<DetailArchive | null> => {
const archiveId = generateUUID();
const promptText = params.prompt.replace(/\s+/g, " ").trim();
@ -5611,6 +5702,8 @@ export function CanvasChatPanel({
summary: params.summary,
imageCount: params.imageCount,
coverUrl: params.coverUrl,
uiSurfaceSnapshot: params.uiSurfaceSnapshot || null,
uiSurfaceLatestReview: params.uiSurfaceLatestReview || null,
};
setDetailArchives((prev) => [archive, ...prev].slice(0, 30));
@ -5631,6 +5724,24 @@ export function CanvasChatPanel({
setIsDetailGenerationMode(true);
setDetailGenerationMode(normalizeDetailGenerationMode(archive.generationMode));
setDetailBrief({ ...archive.brief });
setDetailSurfaceSnapshot(archive.uiSurfaceSnapshot || null);
setDetailUiValidation({
plan: archive.uiSurfaceSnapshot?.validation
? {
passed: archive.uiSurfaceSnapshot.validation.passed,
issues: archive.uiSurfaceSnapshot.validation.issues || [],
}
: null,
latestScreen:
archive.uiSurfaceLatestReview?.screenIndex
? {
screenIndex: archive.uiSurfaceLatestReview.screenIndex,
action: archive.uiSurfaceLatestReview.action,
passed: archive.uiSurfaceLatestReview.passed,
issues: archive.uiSurfaceLatestReview.issues || [],
}
: null,
});
setDetailSlots(
DETAIL_SLOT_CONFIGS.map((_, index) => {
const slot = archive.slots[index];
@ -5805,6 +5916,8 @@ export function CanvasChatPanel({
let completedDetailImageCount = 0;
const generatedImages: Array<{ originalUrl: string; thumbnailUrl: string }> = [];
let detailPlaceholderNodeIds: string[] = [];
let currentDetailSurfaceSnapshot: UiSurfaceSnapshot | null = null;
let latestSurfaceReviewState: DetailArchive["uiSurfaceLatestReview"] = null;
const detailPlaceholderSpacing = 24;
const detailPlaceholderItemWidth = DETAIL_TARGET_WIDTH;
const detailPlaceholderItemHeight = DETAIL_TARGET_HEIGHT;
@ -5905,6 +6018,11 @@ export function CanvasChatPanel({
setIsDetailWorkbenchCollapsed(true);
setIsDetailGenerating(true);
setDetailUiValidation({
plan: null,
latestScreen: null,
});
setDetailSurfaceSnapshot(null);
setDetailProgress({
status: "running",
stage: "Agent 正在解析需求并规划详情结构...",
@ -5921,6 +6039,8 @@ export function CanvasChatPanel({
generationMode: archiveSnapshot.generationMode,
brief: archiveSnapshot.brief,
slots: archiveSnapshot.slots,
uiSurfaceSnapshot: null,
uiSurfaceLatestReview: null,
});
const detailReferenceAttachments = DETAIL_SLOT_CONFIGS.map((config, index) => {
@ -5993,6 +6113,10 @@ export function CanvasChatPanel({
const activeDetailGenerationMode = normalizeDetailGenerationMode(
planData.generationMode || detailGenerationMode
);
currentDetailSurfaceSnapshot = planData.uiSurface || null;
if (activeDetailGenerationMode !== detailGenerationMode) {
setDetailGenerationMode(activeDetailGenerationMode);
}
const perScreenSteps = activeDetailGenerationMode === "agent_layout" ? 1 : 2;
const prompts = Array.isArray(planData.prompts)
? planData.prompts.map((item) => String(item || "").trim()).filter(Boolean)
@ -6014,14 +6138,40 @@ export function CanvasChatPanel({
const preflightIssues = Array.isArray(planData.preflight?.issues)
? planData.preflight?.issues.filter(Boolean)
: [];
const planUiIssues = Array.isArray(planData.uiValidation?.issues)
? planData.uiValidation.issues.filter(Boolean)
: [];
const preflightDesignScore = Number(planData.preflight?.designDirectorScore || 0);
const preflightMarketingScore = Number(planData.preflight?.marketingDirectorScore || 0);
const preflightPassed = planData.preflight?.passed !== false;
setDetailUiValidation({
plan: {
passed: planData.uiValidation?.passed !== false,
issues: planUiIssues,
},
latestScreen: null,
});
setDetailSurfaceSnapshot(planData.uiSurface || null);
if (workingArchive?.id && planData.uiSurface) {
const plannedArchive: DetailArchive = {
...workingArchive,
uiSurfaceSnapshot: planData.uiSurface,
uiSurfaceLatestReview: null,
};
workingArchive = plannedArchive;
updateLocalDetailArchive(plannedArchive.id, {
uiSurfaceSnapshot: plannedArchive.uiSurfaceSnapshot,
uiSurfaceLatestReview: plannedArchive.uiSurfaceLatestReview,
});
await patchDetailArchiveInDatabase(plannedArchive.id, plannedArchive);
}
setDetailProgress({
status: "running",
stage:
planData.qualityChecks?.passed === false
? `分镜预审未通过:${qualityIssues[0] || "已自动切换为稳定方案"}`
: planData.uiValidation?.passed === false
? `协议排版校验提示:${planUiIssues[0] || "已记录待优化项,继续逐屏执行"}`
: preflightPassed
? `双总监预审通过(设计${preflightDesignScore || "--"} / 营销${preflightMarketingScore || "--"}),开始逐屏生成`
: `双总监预审提示:${preflightIssues[0] || "已自动重写分镜后放行"}`,
@ -6073,14 +6223,12 @@ export function CanvasChatPanel({
drafts: Array<Omit<CanvasNode, "id" | "createdAt" | "updatedAt">>;
}) => {
const storeState = useCanvasStore.getState();
const childTextNodes = storeState.nodes.filter(
(node) =>
node.parentId === params.nodeId &&
node.type === "text" &&
Boolean((node.data as any)?.metadata?.detailUiBlockId),
const childTextNodeIds = collectManagedDetailTextLayerNodeIds(
storeState.nodes,
params.nodeId,
);
childTextNodes.forEach((node) => deleteNode(node.id));
childTextNodeIds.forEach((nodeId) => deleteNode(nodeId));
params.drafts.forEach((draft) => addNode(draft));
};
const createAgentLayoutTextLayers = (params: {
@ -6088,6 +6236,7 @@ export function CanvasChatPanel({
screenIndex: number;
screen?: DetailPlanScreenSpec;
screenProtocol?: DetailUiProtocol["screens"][number];
surfaceSnapshot?: UiSurfaceSnapshot | null;
}) => {
if (!isAgentLayoutMode) return;
const storeState = useCanvasStore.getState();
@ -6097,10 +6246,20 @@ export function CanvasChatPanel({
const y = Number(imageNode.position?.y || 0);
const width = Number(imageNode.size?.width || DETAIL_TARGET_WIDTH);
const height = Number(imageNode.size?.height || DETAIL_TARGET_HEIGHT);
const screen = params.screen;
const drafts =
params.screenProtocol
? buildDetailTextLayerNodesFromProtocol({
params.surfaceSnapshot
? buildDetailTextLayerNodesFromSurfaceScreen({
parentNodeId: params.nodeId,
parentPosition: { x, y },
parentSize: { width, height },
surface: params.surfaceSnapshot,
screenIndex: params.screenIndex + 1,
})
: [];
const finalDrafts =
drafts.length > 0
? drafts
: buildDetailTextLayerNodes({
parentNodeId: params.nodeId,
parentPosition: { x, y },
parentSize: { width, height },
@ -6108,27 +6267,22 @@ export function CanvasChatPanel({
width: DETAIL_TARGET_WIDTH,
height: DETAIL_TARGET_HEIGHT,
},
screenProtocol: params.screenProtocol,
})
: buildLegacyDetailTextLayerNodes({
parentNodeId: params.nodeId,
parentPosition: { x, y },
parentWidth: width,
screenIndex: params.screenIndex,
title: screen?.title,
subtitle: screen?.subtitle,
body: screen?.copy,
titleFontSize,
titleFontWeight,
subtitleFontSize,
subtitleFontWeight,
bodyFontSize,
bodyFontWeight,
screen: params.screen,
screenProtocol: params.screenProtocol,
typography: {
titleFontSize,
titleFontWeight,
subtitleFontSize,
subtitleFontWeight,
bodyFontSize,
bodyFontWeight,
},
});
replaceDetailTextLayers({
nodeId: params.nodeId,
drafts,
drafts: finalDrafts,
});
};
for (let index = 0; index < prompts.length; index += 1) {
@ -6345,6 +6499,7 @@ export function CanvasChatPanel({
body: JSON.stringify({
outputLanguage,
generationMode: activeDetailGenerationMode,
surfaceId: currentDetailSurfaceSnapshot?.surfaceId,
imageUrl: firstPassImage.originalUrl || firstPassImage.thumbnailUrl,
referenceImages: generationReferenceImagesBase.slice(0, 6),
brief,
@ -6368,6 +6523,48 @@ export function CanvasChatPanel({
console.error("[CanvasChat] Detail screen review failed:", reviewError);
}
if (screenReviewResult) {
const latestScreenIssues = Array.isArray(screenReviewResult.uiValidation?.issues)
? screenReviewResult.uiValidation.issues.filter(Boolean)
: Array.isArray(screenReviewResult.issues)
? screenReviewResult.issues.filter(Boolean)
: [];
latestSurfaceReviewState = {
screenIndex: index + 1,
action: screenReviewResult?.reviewAction,
passed:
typeof screenReviewResult.uiValidation?.passed === "boolean"
? screenReviewResult.uiValidation.passed
: screenReviewResult.pass,
issues: latestScreenIssues,
};
if (screenReviewResult.uiSurfacePatch) {
currentDetailSurfaceSnapshot = applyUiSurfacePatch(
currentDetailSurfaceSnapshot,
screenReviewResult.uiSurfacePatch,
);
setDetailSurfaceSnapshot(currentDetailSurfaceSnapshot);
}
const nextLatestSurfaceReview = latestSurfaceReviewState || {
screenIndex: index + 1,
action: screenReviewResult?.reviewAction,
passed:
typeof screenReviewResult.uiValidation?.passed === "boolean"
? screenReviewResult.uiValidation.passed
: screenReviewResult.pass,
issues: latestScreenIssues,
};
setDetailUiValidation((prev) => ({
...prev,
latestScreen: {
screenIndex: nextLatestSurfaceReview.screenIndex || index + 1,
action: nextLatestSurfaceReview.action,
passed: nextLatestSurfaceReview.passed,
issues: nextLatestSurfaceReview.issues || [],
},
}));
}
const shouldApplyProtocolPatch = Boolean(
screenReviewResult?.uiPatch &&
screenReviewResult?.reviewAction === "protocol_patch",
@ -6406,27 +6603,37 @@ export function CanvasChatPanel({
? useCanvasStore.getState().getNode(targetNodeId)
: null;
if (currentNode && targetNodeId) {
const replacementDrafts = buildDetailTextLayerNodesFromReviewPatch({
parentNodeId: targetNodeId,
parentPosition: {
x: Number(currentNode.position?.x || 0),
y: Number(currentNode.position?.y || 0),
},
parentSize: {
width: Number(currentNode.size?.width || DETAIL_TARGET_WIDTH),
height: Number(currentNode.size?.height || DETAIL_TARGET_HEIGHT),
},
screenSize: planData.uiProtocol?.screenSize || {
width: DETAIL_TARGET_WIDTH,
height: DETAIL_TARGET_HEIGHT,
},
uiPatch: screenReviewResult?.uiPatch,
});
if (replacementDrafts.length > 0) {
replaceDetailTextLayers({
if (currentDetailSurfaceSnapshot) {
createAgentLayoutTextLayers({
nodeId: targetNodeId,
drafts: replacementDrafts,
screenIndex: index,
screen,
screenProtocol: planData.uiProtocol?.screens?.[index],
surfaceSnapshot: currentDetailSurfaceSnapshot,
});
} else {
const replacementDrafts = buildDetailTextLayerNodesFromReviewPatch({
parentNodeId: targetNodeId,
parentPosition: {
x: Number(currentNode.position?.x || 0),
y: Number(currentNode.position?.y || 0),
},
parentSize: {
width: Number(currentNode.size?.width || DETAIL_TARGET_WIDTH),
height: Number(currentNode.size?.height || DETAIL_TARGET_HEIGHT),
},
screenSize: planData.uiProtocol?.screenSize || {
width: DETAIL_TARGET_WIDTH,
height: DETAIL_TARGET_HEIGHT,
},
uiPatch: screenReviewResult?.uiPatch,
});
if (replacementDrafts.length > 0) {
replaceDetailTextLayers({
nodeId: targetNodeId,
drafts: replacementDrafts,
});
}
}
}
} else if (shouldRegenerate) {
@ -6488,12 +6695,16 @@ export function CanvasChatPanel({
summary: progressSummary,
imageCount: completedDetailImageCount,
coverUrl: progressCoverUrl,
uiSurfaceSnapshot: currentDetailSurfaceSnapshot,
uiSurfaceLatestReview: latestSurfaceReviewState,
};
workingArchive = nextArchive;
updateLocalDetailArchive(nextArchive.id, {
summary: progressSummary,
imageCount: completedDetailImageCount,
coverUrl: progressCoverUrl,
uiSurfaceSnapshot: currentDetailSurfaceSnapshot,
uiSurfaceLatestReview: latestSurfaceReviewState,
});
await patchDetailArchiveInDatabase(nextArchive.id, nextArchive);
}
@ -6536,6 +6747,8 @@ export function CanvasChatPanel({
detailGlobalStyle: planData.globalStyle || null,
detailGenerationMode: activeDetailGenerationMode,
detailScreenReview: screenReviewResult || null,
detailUiSurfaceId: currentDetailSurfaceSnapshot?.surfaceId || null,
detailUiSurfaceType: currentDetailSurfaceSnapshot?.surfaceType || null,
detailGeneratingPlaceholder: false,
detailProgressLabel: `该屏完成 ${index + 1}/${prompts.length}`,
},
@ -6546,6 +6759,7 @@ export function CanvasChatPanel({
screenIndex: index,
screen,
screenProtocol: planData.uiProtocol?.screens?.[index],
surfaceSnapshot: currentDetailSurfaceSnapshot,
});
}
}
@ -6678,6 +6892,7 @@ export function CanvasChatPanel({
prompts,
skillFlow: planData.skillFlow,
qualityChecks: planData.qualityChecks,
uiValidation: planData.uiValidation,
globalStyle: planData.globalStyle,
screens: screenSpecs,
reviewGroups,
@ -6717,12 +6932,16 @@ export function CanvasChatPanel({
summary: finalSummary,
imageCount: generatedImages.length,
coverUrl: generatedImages[0]?.thumbnailUrl || generatedImages[0]?.originalUrl,
uiSurfaceSnapshot: currentDetailSurfaceSnapshot,
uiSurfaceLatestReview: latestSurfaceReviewState,
};
workingArchive = finalArchive;
updateLocalDetailArchive(finalArchive.id, {
summary: finalArchive.summary,
imageCount: finalArchive.imageCount,
coverUrl: finalArchive.coverUrl,
uiSurfaceSnapshot: finalArchive.uiSurfaceSnapshot,
uiSurfaceLatestReview: finalArchive.uiSurfaceLatestReview,
});
await patchDetailArchiveInDatabase(finalArchive.id, finalArchive);
} else {
@ -6734,6 +6953,8 @@ export function CanvasChatPanel({
generationMode: archiveSnapshot.generationMode,
brief: archiveSnapshot.brief,
slots: archiveSnapshot.slots,
uiSurfaceSnapshot: currentDetailSurfaceSnapshot,
uiSurfaceLatestReview: latestSurfaceReviewState,
});
}
@ -6777,11 +6998,15 @@ export function CanvasChatPanel({
summary: failedSummary,
imageCount: completedDetailImageCount,
coverUrl: generatedImages[0]?.thumbnailUrl || generatedImages[0]?.originalUrl,
uiSurfaceSnapshot: currentDetailSurfaceSnapshot,
uiSurfaceLatestReview: latestSurfaceReviewState,
};
updateLocalDetailArchive(failedArchive.id, {
summary: failedSummary,
imageCount: completedDetailImageCount,
coverUrl: failedArchive.coverUrl,
uiSurfaceSnapshot: failedArchive.uiSurfaceSnapshot,
uiSurfaceLatestReview: failedArchive.uiSurfaceLatestReview,
});
void patchDetailArchiveInDatabase(failedArchive.id, failedArchive);
}
@ -9073,6 +9298,18 @@ export function CanvasChatPanel({
)}
</div>
<DetailUiValidationPanel
generationMode={detailGenerationMode}
planValidation={detailUiValidation.plan}
latestScreenValidation={detailUiValidation.latestScreen}
surfaceId={detailSurfaceSnapshot?.surfaceId}
/>
<DetailSurfaceInspectorPanel
surface={detailSurfaceSnapshot}
latestScreenValidation={detailUiValidation.latestScreen}
/>
<div className="mt-3 rounded-xl border border-border/50 bg-zinc-50/40 p-3 dark:bg-zinc-900/40">
<div className="mb-2 flex items-center justify-between">
<p className="text-[11px] font-medium text-foreground">生成路线</p>

View File

@ -0,0 +1,172 @@
"use client";
import { Blocks, CheckCircle2, LayoutTemplate, Layers3 } from "lucide-react";
import {
countUiSurfaceComponents,
getUiSurfaceChildren,
getUiSurfaceScreenComponents,
resolveUiSurfaceTextBinding,
type UiSurfaceSnapshot,
} from "@/lib/ui-surface-protocol";
import { cn } from "@/lib/utils";
type DetailSurfaceInspectorReviewState = {
screenIndex: number;
action?: "pass" | "protocol_patch" | "regenerate_image";
passed?: boolean;
};
function getActionLabel(action?: DetailSurfaceInspectorReviewState["action"]) {
if (action === "protocol_patch") return "协议修正";
if (action === "regenerate_image") return "重生图";
return "通过";
}
function getActionClassName(action?: DetailSurfaceInspectorReviewState["action"]) {
if (action === "protocol_patch") {
return "bg-blue-50 text-blue-700 ring-1 ring-inset ring-blue-500/20 dark:bg-blue-500/10 dark:text-blue-300";
}
if (action === "regenerate_image") {
return "bg-red-50 text-red-700 ring-1 ring-inset ring-red-500/20 dark:bg-red-500/10 dark:text-red-300";
}
return "bg-emerald-50 text-emerald-700 ring-1 ring-inset ring-emerald-500/20 dark:bg-emerald-500/10 dark:text-emerald-300";
}
export function DetailSurfaceInspectorPanel(params: {
surface: UiSurfaceSnapshot | null;
latestScreenValidation?: DetailSurfaceInspectorReviewState | null;
}) {
if (!params.surface) return null;
const screenComponents = getUiSurfaceScreenComponents(params.surface);
const totalComponents = countUiSurfaceComponents(params.surface);
const totalTextBlocks = countUiSurfaceComponents(params.surface, "detail_text_block");
const totalScreens = screenComponents.length;
return (
<div className="mt-3 rounded-xl border border-border/50 bg-zinc-50/40 p-3 dark:bg-zinc-900/40">
<div className="mb-3 flex items-center justify-between gap-3">
<div className="flex items-center gap-2">
<LayoutTemplate className="h-4 w-4 text-violet-500" />
<div>
<p className="text-[11px] font-medium text-foreground">Detail Surface Inspector</p>
<p className="mt-0.5 text-[10px] text-muted-foreground">
<span className="font-mono">{params.surface.surfaceId}</span>
</p>
</div>
</div>
<span className="rounded-full bg-violet-50 px-2 py-0.5 text-[10px] font-medium text-violet-700 ring-1 ring-inset ring-violet-500/20 dark:bg-violet-500/10 dark:text-violet-300">
{params.surface.generationMode === "agent_layout" ? "AGENT Surface" : "Classic Surface"}
</span>
</div>
<div className="grid grid-cols-3 gap-2">
<div className="rounded-lg border border-border/40 bg-background/80 px-3 py-2 dark:bg-zinc-950/40">
<div className="flex items-center gap-1.5 text-[10px] text-muted-foreground">
<Layers3 className="h-3.5 w-3.5" />
Screens
</div>
<div className="mt-1 text-sm font-semibold text-foreground">{totalScreens}</div>
</div>
<div className="rounded-lg border border-border/40 bg-background/80 px-3 py-2 dark:bg-zinc-950/40">
<div className="flex items-center gap-1.5 text-[10px] text-muted-foreground">
<Blocks className="h-3.5 w-3.5" />
Components
</div>
<div className="mt-1 text-sm font-semibold text-foreground">{totalComponents}</div>
</div>
<div className="rounded-lg border border-border/40 bg-background/80 px-3 py-2 dark:bg-zinc-950/40">
<div className="flex items-center gap-1.5 text-[10px] text-muted-foreground">
<CheckCircle2 className="h-3.5 w-3.5" />
Text Blocks
</div>
<div className="mt-1 text-sm font-semibold text-foreground">{totalTextBlocks}</div>
</div>
</div>
<div className="mt-2 flex items-center justify-between rounded-lg border border-border/40 bg-background/80 px-3 py-2 dark:bg-zinc-950/40">
<div>
<div className="text-[10px] text-muted-foreground">Surface Validation</div>
<div className="mt-0.5 text-[11px] font-medium text-foreground">
{params.surface.validation?.passed === false
? params.surface.validation.issues[0] || "存在待优化项"
: "当前 surface 已通过基础校验"}
</div>
</div>
<span
className={cn(
"rounded-full px-2 py-0.5 text-[10px] font-medium",
params.surface.validation?.passed === false
? "bg-amber-50 text-amber-700 ring-1 ring-inset ring-amber-500/20 dark:bg-amber-500/10 dark:text-amber-300"
: "bg-emerald-50 text-emerald-700 ring-1 ring-inset ring-emerald-500/20 dark:bg-emerald-500/10 dark:text-emerald-300",
)}
>
{params.surface.validation?.passed === false ? "待优化" : "通过"}
</span>
</div>
<div className="mt-3 space-y-2">
{screenComponents.slice(0, 6).map((screenComponent) => {
const screenIndex = Number(screenComponent.props?.screenIndex || 0);
const textStack = getUiSurfaceChildren(params.surface, screenComponent.id).find(
(component) => component.type === "detail_text_stack",
);
const textBlocks = textStack
? getUiSurfaceChildren(params.surface, textStack.id).filter(
(component) => component.type === "detail_text_block",
)
: [];
const titleBlock =
textBlocks.find((component) => component.props?.blockType === "title") || null;
const subtitleBlock =
textBlocks.find((component) => component.props?.blockType === "subtitle") || null;
const isLatestScreen = params.latestScreenValidation?.screenIndex === screenIndex;
return (
<div
key={screenComponent.id}
className={cn(
"rounded-lg border border-border/40 bg-background/80 px-3 py-2 dark:bg-zinc-950/40",
isLatestScreen && "border-blue-300/60 ring-1 ring-inset ring-blue-500/20",
)}
>
<div className="flex items-start justify-between gap-3">
<div className="min-w-0">
<div className="text-[11px] font-medium text-foreground">
第 {screenIndex} 屏
{screenComponent.props?.screenType
? ` · ${String(screenComponent.props?.screenType)}`
: ""}
</div>
<div className="mt-0.5 truncate text-[11px] text-foreground/85">
{resolveUiSurfaceTextBinding(params.surface, titleBlock, "content") || "未定义标题"}
</div>
{subtitleBlock ? (
<div className="mt-0.5 truncate text-[10px] text-muted-foreground">
{resolveUiSurfaceTextBinding(params.surface, subtitleBlock, "content")}
</div>
) : null}
</div>
<div className="flex shrink-0 items-center gap-2">
<span className="rounded-full bg-zinc-100 px-2 py-0.5 text-[10px] font-medium text-zinc-700 dark:bg-zinc-800 dark:text-zinc-300">
{textBlocks.length} 块
</span>
{isLatestScreen ? (
<span
className={cn(
"rounded-full px-2 py-0.5 text-[10px] font-medium",
getActionClassName(params.latestScreenValidation?.action),
)}
>
{getActionLabel(params.latestScreenValidation?.action)}
</span>
) : null}
</div>
</div>
</div>
);
})}
</div>
</div>
);
}

View File

@ -0,0 +1,144 @@
"use client";
import { AlertCircle, CheckCircle2 } from "lucide-react";
import { cn } from "@/lib/utils";
type DetailUiValidationSnapshot = {
passed?: boolean;
issues?: string[];
};
type DetailUiReviewSnapshot = {
screenIndex: number;
action?: "pass" | "protocol_patch" | "regenerate_image";
passed?: boolean;
issues?: string[];
};
export function DetailUiValidationPanel(params: {
generationMode: "classic" | "agent_layout";
planValidation?: DetailUiValidationSnapshot | null;
latestScreenValidation?: DetailUiReviewSnapshot | null;
surfaceId?: string | null;
}) {
const planIssues = Array.isArray(params.planValidation?.issues)
? params.planValidation?.issues.filter(Boolean)
: [];
const latestIssues = Array.isArray(params.latestScreenValidation?.issues)
? params.latestScreenValidation.issues.filter(Boolean)
: [];
if (!params.planValidation && !params.latestScreenValidation) {
return null;
}
const title =
params.generationMode === "agent_layout" ? "协议排版质检" : "排版与复审状态";
const hasPlanValidation = Boolean(params.planValidation);
const latestActionLabel =
params.latestScreenValidation?.action === "protocol_patch"
? "已走协议修正"
: params.latestScreenValidation?.action === "regenerate_image"
? "已要求重生图"
: "通过";
return (
<div className="mt-3 rounded-xl border border-border/50 bg-zinc-50/40 p-3 dark:bg-zinc-900/40">
<div className="mb-2 flex items-center justify-between gap-3">
<div className="flex items-center gap-2">
{params.planValidation?.passed === false || latestIssues.length > 0 ? (
<AlertCircle className="h-4 w-4 text-amber-500" />
) : (
<CheckCircle2 className="h-4 w-4 text-emerald-500" />
)}
<p className="text-[11px] font-medium text-foreground">{title}</p>
</div>
<span className="text-[10px] text-muted-foreground">
{params.surfaceId
? params.generationMode === "agent_layout"
? "结构化 surface 已建立"
: "当前结果可继续复审"
: params.generationMode === "agent_layout"
? "文字层可直接修正"
: "问题会触发二次修图"}
</span>
</div>
<div className="space-y-2">
<div className="flex items-start justify-between gap-3 rounded-lg border border-border/40 bg-background/80 px-3 py-2 dark:bg-zinc-950/40">
<div>
<div className="text-[11px] font-medium text-foreground">规划阶段</div>
<div className="mt-0.5 text-[10px] text-muted-foreground">
{params.planValidation?.passed === false
? `发现 ${planIssues.length} 条协议/排版提示`
: hasPlanValidation
? "协议结构已通过基础校验"
: "等待本轮规划结果"}
</div>
</div>
<span
className={cn(
"rounded-full px-2 py-0.5 text-[10px] font-medium",
!hasPlanValidation
? "bg-zinc-100 text-zinc-600 ring-1 ring-inset ring-zinc-500/10 dark:bg-zinc-800 dark:text-zinc-300"
: params.planValidation?.passed === false
? "bg-amber-50 text-amber-700 ring-1 ring-inset ring-amber-500/20 dark:bg-amber-500/10 dark:text-amber-300"
: "bg-emerald-50 text-emerald-700 ring-1 ring-inset ring-emerald-500/20 dark:bg-emerald-500/10 dark:text-emerald-300",
)}
>
{!hasPlanValidation
? "待开始"
: params.planValidation?.passed === false
? "待优化"
: "通过"}
</span>
</div>
{params.latestScreenValidation ? (
<div className="flex items-start justify-between gap-3 rounded-lg border border-border/40 bg-background/80 px-3 py-2 dark:bg-zinc-950/40">
<div>
<div className="text-[11px] font-medium text-foreground">
最近一屏:第 {params.latestScreenValidation.screenIndex} 屏
</div>
<div className="mt-0.5 text-[10px] text-muted-foreground">
{params.latestScreenValidation.passed === false
? latestIssues[0] || "存在可优化项"
: "最近一屏已通过当前质检"}
</div>
</div>
<span
className={cn(
"rounded-full px-2 py-0.5 text-[10px] font-medium",
params.latestScreenValidation.action === "regenerate_image"
? "bg-red-50 text-red-700 ring-1 ring-inset ring-red-500/20 dark:bg-red-500/10 dark:text-red-300"
: params.latestScreenValidation.action === "protocol_patch"
? "bg-blue-50 text-blue-700 ring-1 ring-inset ring-blue-500/20 dark:bg-blue-500/10 dark:text-blue-300"
: "bg-emerald-50 text-emerald-700 ring-1 ring-inset ring-emerald-500/20 dark:bg-emerald-500/10 dark:text-emerald-300",
)}
>
{latestActionLabel}
</span>
</div>
) : null}
{(planIssues.length > 0 || latestIssues.length > 0) && (
<div className="rounded-lg border border-dashed border-border/40 bg-background/60 px-3 py-2 dark:bg-zinc-950/30">
<div className="text-[10px] font-medium uppercase tracking-wide text-muted-foreground">
当前关注点
</div>
<div className="mt-1 space-y-1">
{[...planIssues, ...latestIssues]
.filter((item, index, array) => array.indexOf(item) === index)
.slice(0, 3)
.map((issue) => (
<div key={issue} className="text-[11px] leading-5 text-foreground/85">
· {issue}
</div>
))}
</div>
</div>
)}
</div>
</div>
);
}

View File

@ -0,0 +1,873 @@
# Canvas UI Surface Protocol 设计方案(基于 Google Stitch / A2UI 调研)
> 日期:2026-03-21
> 目标:把我们当前“会聊天、会生图、会在画布上落结果”的 Agent,升级成“能稳定产出结构化界面、支持交互、支持持续迭代、支持原型播放”的系统。
---
## 1. 先说结论
### 1.1 Google Stitch 对我们的帮助是什么
从官方资料看,**Stitch 的核心路线不是单纯生一张 UI 图,而是把“想法 → 界面设计 → 代码/原型”缩短为一条连续链路**。
目前我能确认到的官方能力有:
- 2025-05-20:支持 **自然语言/图片/线框 → UI 设计 + 前端代码**,支持 **交互式迭代**、**多版本探索**、**Paste to Figma**、**导出前端代码**。
- 2025-11-18 之后:随着 Gemini 3 接入,官方明确提到 Stitch 的 **UI 生成质量提升**。
- 2025-12-23:官方确认 **Nano Banana Pro** 已可在 Stitch 中使用,说明 Stitch 已接入更强图像生成/编辑能力。
- 2026-02(官方可检索到的最新明确更新):Google 官方已给 Stitch 加入 **Prototypes**,可以把多个 screen 串成 **可交互原型流**。
### 1.2 A2UI 对我们的帮助是什么
A2UI 最值得我们学的不是视觉风格,而是它的**运行时协议思想**:
- Agent 不直接输出任意 HTML/JS,而是输出 **声明式 UI 协议**
- 客户端只渲染 **白名单 catalog 中的可信组件**
- **结构(components)** 与 **数据(data model)** 分离
- 支持 **增量更新**
- 支持 **用户动作回传给 Agent**
这套思想非常适合我们,因为我们现在最缺的不是“多一个好看的卡片”,而是**Agent 产出 UI 的统一运行时**。
### 1.3 最终判断
我们最优路线不是“抄 Stitch UI”,也不是“整套照搬 A2UI”,而是:
1. **借鉴 Stitch 的产品路线**:从结果图升级到“设计迭代 + 原型 + 代码/结构化输出”
2. **借鉴 A2UI 的系统路线**:把 Agent 输出收进统一的 UI Surface Protocol
3. **结合我们现有 conversation-state-protocol**:形成“对话理解协议 + UI surface 协议”双协议架构
---
## 2. 外部调研纠偏
### 2.1 关于 Stitch,需要纠正的点
我这次在线检索后,**没有找到 Google 官方可验证来源**来证明以下功能已经正式公开:
- “DESIGN.md”
- “Agent Manager”
- “无限画布”
- “语音设计批改”
这些概念在社区讨论和第三方文章里有出现,但**本方案不把它们当作已被官方确认的能力**。
本方案只基于已确认的官方信息来提炼技术路线:
- 多模态输入
- UI 生成
- 多版本探索
- Figma / 前端代码导出
- 原型串联
- 更强图像模型接入
### 2.2 关于 A2UI,需要纠正的点
A2UI 非常强,但它**不是“可直接拿来替代我们前端”的现成方案**。
官方 README 里明确写了:
- 目前仍是 **Early stage public preview**
- v0.8 可用,但协议仍在演进
- 适合学习其 **协议设计、渲染器架构、catalog 白名单机制**
- 不适合让我们当前系统重度依赖其未稳定版本
所以正确姿势是:**借鉴架构,不强绑定实现**。
---
## 3. 我们当前系统已经具备的基础
### 3.1 已有的对话/工作流协议基础
我们已经有一条很重要的底座:
- `lib/ai/conversation-state-protocol.ts`
- `lib/ai/conversation-state-protocol-events.ts`
这意味着我们已经不是“全靠 route 里写很多 if/else”的状态了,而是已经有:
- 工作状态读取
- 事件化写回
- 图片/视频/分镜等流程记忆
- 用户意图与上下文边界管理
这部分非常重要,它可以继续作为**上层 Agent 理解协议**保留。
### 3.2 已有的详情模式 UI 协议雏形
我们已经做了 detail mode 的第一代结构化协议:
- `lib/detail-ui-protocol.ts`
- `lib/detail-ui-renderer.ts`
- `lib/detail-ui-validator.ts`
- `app/api/canvas/detail-plan/route.ts`
- `app/api/canvas/detail-review/route.ts`
这说明我们已经迈出了最难的一步:
- 不再只是一张整图
- 开始区分背景图层与文字层
- 开始让 review 直接 patch 协议
但当前它仍然是**“详情模式专用协议”**,不是通用 UI Surface 协议。
---
## 4. 当前距离“最优体验”还缺什么
如果以 Stitch 的产品方向 + A2UI 的系统方向为参照,我们还缺 8 个关键层级。
### 4.1 缺统一 UI Surface 协议
当前 `detail-ui-protocol` 只覆盖:
- title
- subtitle
- body
- safe area
这还不是一个通用 UI 表达层,无法稳定表示:
- 商品参数区
- 卖点列表
- 价格区
- 对比表
- 买家评价卡
- 按钮区
- 标签/角标/徽章
- 多区域联动布局
### 4.2 缺“结构”和“数据”的彻底分离
A2UI 的一个核心点是:
- 结构描述组件树
- 数据模型存值
- 组件通过 path 绑定数据
我们现在 detail 协议里的文本内容还是直接写死在 block 里,导致:
- 改一段文案需要改整个 block
- 很难做局部 patch
- 很难做模板复用
### 4.3 缺用户动作回传协议
现在的 Agent 还是更像“对话生成 + 画布落图”,还不是真正的“UI agent runtime”。
我们缺:
- 点击某个卡片后告诉 Agent 用户想放大这一块
- 编辑某个标题后告诉 Agent 用户要保留结构仅改文案
- 拖拽组件后告诉 Agent 用户偏好布局发生变化
- 点击“生成变体”后让 Agent 只重做某个 section
### 4.4 缺流式 patch 更新
现在很多地方仍然是:
- plan 完整返回
- review 完整返回
- 再由前端一次性处理
缺少:
- `surface.created`
- `components.appended`
- `data.updated`
- `review.patch_applied`
- `prototype.link_added`
这种**统一流式 surface update**。
### 4.5 缺受控组件 catalog
我们目前的详情协议本质上还是自由度比较高的“文字块+图片”。
真正稳定后,Agent 不应该直接自由拼任何 UI,而应该只能调用受控 catalog,例如:
- `HeroBlock`
- `SellingPointGrid`
- `FeatureSpecTable`
- `ComparisonCard`
- `TrustBadgeRow`
- `ReviewCard`
- `CTASection`
- `StickyBottomBar`
这样才能保证:
- 风格一致
- 质量稳定
- patch 容易
- renderer 可预测
### 4.6 缺原型图与 screen flow 层
Stitch 已经明确加了 Prototypes。
而我们现在详情模式仍偏“长图生产”,没有统一表达:
- screen 之间跳转关系
- 点击区域
- hotspot
- 状态切换
- 原型播放
如果未来我们想让“Agent详情模式”真正高级,必须支持:
- 从静态 screen 升级为可交互 flow
- 在画布里直接预览原型行为
### 4.7 缺设计系统记忆层
这是我们现在最容易被忽视,但长期最重要的能力。
我们需要一个项目级设计系统记忆,不一定叫 `DESIGN.md`,但本质要存在:
- 颜色 token
- 字体等级
- 间距规则
- 卡片圆角
- 阴影
- 图片风格
- 品牌语气
- 常用 section 模板
否则 Agent 每次都像第一次做设计,结果不可能稳定。
### 4.8 缺多 Agent 分工
现在很多能力压在一个 route 或一轮模型输出里。
理想形态应该拆成:
- Design Orchestrator
- Surface Planner
- Copy Agent
- Visual Critic
- Prototype Linker
- Action Interpreter
这样才能做到:
- 规划与执行分离
- 文案与布局分离
- 质检与生成分离
- 交互与渲染分离
---
## 5. 新目标:Canvas UI Surface Protocol(CUSP)
建议正式定义一套内部协议:
**Canvas UI Surface Protocol(简称 CUSP)**
它不是替代 conversation-state-protocol,而是和它并列:
- **Conversation State Protocol**:负责“用户在说什么、当前任务做到哪、记忆怎么延续”
- **Canvas UI Surface Protocol**:负责“当前 UI 长什么样、有哪些组件、数据是什么、用户点了什么、下一步怎么 patch”
---
## 6. CUSP 的核心对象
### 6.1 Surface
一个可被 Agent 管理、可被前端渲染、可被 patch 的界面单元。
建议字段:
- `surfaceId`
- `projectId`
- `chatId`
- `surfaceType`:`detail_page` / `agent_panel` / `storyboard` / `prototype_screen`
- `mode`:`draft` / `review` / `published`
- `designSystemId`
- `rootComponentId`
- `status`
### 6.2 Components
受控 catalog 中的组件树,建议采用 **扁平邻接表**,类似 A2UI 的思路。
建议字段:
- `id`
- `type`
- `parentId`
- `slot`
- `props`
- `bindings`
- `styleTokens`
- `layout`
- `visible`
### 6.3 Data Model
每个 surface 都应该有自己的数据模型。
示例:
```json
{
"product": {
"name": "由点笔迹 离线日记",
"tagline": "离线可用,私密记录"
},
"hero": {
"title": "Capture your day, anytime.",
"subtitle": "离线优先的私人记录空间"
},
"sections": {
"features": [
{ "title": "离线优先", "desc": "无网络也能访问内容" },
{ "title": "隐私保护", "desc": "支持密码与 Face ID" }
]
}
}
```
### 6.4 Bindings
组件不直接存最终文案,而是尽量绑定到 data model:
- `text: { path: "/hero/title" }`
- `items: { path: "/sections/features" }`
这可以带来:
- 改文案不改结构
- 改数据不重做整屏
- 可做局部翻译/局部重写
### 6.5 Actions
前端产生的交互动作,回传给 Agent。
建议动作类型:
- `surface.regenerate_section`
- `surface.edit_copy`
- `surface.move_component`
- `surface.replace_asset`
- `surface.generate_variant`
- `surface.link_prototype`
- `surface.publish_snapshot`
### 6.6 Patch
Agent 不应该每次都重新返回整棵 UI 树,而应该支持 patch:
- `append_components`
- `replace_component`
- `remove_component`
- `update_data`
- `update_binding`
- `update_style_tokens`
- `reorder_children`
---
## 7. CUSP 与我们现有协议的关系
### 7.1 不推翻 conversation-state-protocol
现有:
- `lib/ai/conversation-state-protocol.ts`
- `lib/ai/conversation-state-protocol-events.ts`
继续保留,负责:
- 意图理解
- 工作流阶段
- 图片/视频/分镜记忆
- 目标资产解析
### 7.2 detail-ui-protocol 升级为 CUSP adapter
现有:
- `lib/detail-ui-protocol.ts`
建议不要直接废弃,而是把它变成:
- `detail-page` 类型 surface 的兼容适配器
也就是说:
- 旧 detail 协议仍可用
- 新系统统一转成 CUSP surface
- 渐进迁移,不一次性推倒
### 7.3 Canvas node 变为 artifact / render target
当前画布节点更像“结果容器”。
未来建议分层:
- `surface`:结构化 UI
- `artifact`:图片、视频、导出图、切图结果
- `canvas node`:在无限画布上的可视承载对象
这样就能同时支持:
- 一个 surface 多个版本
- 一个版本多个 artifact
- 一个 artifact 多个展示位置
---
## 8. 数据库设计建议
### 8.1 `UiSurfaceSession`
用于描述一个 surface 的主记录。
建议字段:
- `id`
- `projectId`
- `chatId`
- `canvasId`
- `surfaceType`
- `mode`
- `status`
- `designSystemId`
- `currentBranchId`
- `latestSnapshotId`
- `createdBy`
- `createdAt`
- `updatedAt`
### 8.2 `UiSurfaceBranch`
用于支持多方案探索。
建议字段:
- `id`
- `surfaceSessionId`
- `name`
- `sourceSnapshotId`
- `status`
- `summary`
- `createdAt`
### 8.3 `UiSurfaceSnapshot`
用于版本化保存协议结果。
建议字段:
- `id`
- `surfaceSessionId`
- `branchId`
- `version`
- `protocolVersion`
- `surfaceJson`
- `dataModelJson`
- `validationJson`
- `reviewJson`
- `createdAt`
### 8.4 `UiSurfaceAction`
用于保存用户交互与 Agent 响应事件。
建议字段:
- `id`
- `surfaceSessionId`
- `snapshotId`
- `actionType`
- `actionPayload`
- `source`:`user` / `agent` / `system`
- `messageId`
- `createdAt`
### 8.5 `DesignSystemProfile`
项目级设计记忆层。
建议字段:
- `id`
- `projectId`
- `name`
- `brandVoice`
- `colorTokensJson`
- `typographyTokensJson`
- `spacingTokensJson`
- `componentRulesJson`
- `imageStyleRulesJson`
- `updatedAt`
---
## 9. API 设计建议
### 9.1 规划接口
`POST /api/canvas/ui-surface/plan`
输入:
- 用户需求
- 当前项目上下文
- 参考图/参考资产
- design system profile
- surfaceType
输出:
- `surfaceSession`
- `initialSnapshot`
- `validation`
- `executionPlan`
### 9.2 流式更新接口
`POST /api/canvas/ui-surface/stream`
输出事件建议:
- `surface.created`
- `branch.created`
- `components.appended`
- `data.updated`
- `validation.updated`
- `review.patch_ready`
- `artifact.generated`
- `surface.completed`
### 9.3 动作回传接口
`POST /api/canvas/ui-surface/action`
前端把动作结构化发回:
- 点击按钮
- 编辑文案
- 拖拽卡片
- 请求生成变体
- 接受/拒绝 Agent patch
### 9.4 原型接口
`POST /api/canvas/ui-surface/prototype/link`
用于表达:
- 当前 screen 到下一 screen 的跳转
- hotspot
- 过渡信息
- 行为上下文
### 9.5 导出接口
`POST /api/canvas/ui-surface/export`
导出方向:
- JSON snapshot
- 前端组件 DSL
- Figma intermediary schema
- 静态预览图
---
## 10. 前端渲染层建议
### 10.1 新增 `SurfaceProvider`
作用类似 A2UI Provider,但不强依赖其实现。
职责:
- 接收 surface snapshot / patch
- 管理本地 data model
- 管理交互 action
- 维护 renderer catalog
### 10.2 新增 `SurfaceRenderer`
统一把 surface 渲染到:
- 详情模式面板
- Agent 详情模式
- 原型预览器
- 画布节点内部 preview
### 10.3 新增 `SurfaceCatalog`
统一注册可信组件:
- `HeroBlock`
- `FeatureGrid`
- `SpecTable`
- `ReviewCard`
- `BadgeRow`
- `CTASection`
- `ComparisonTable`
### 10.4 新增 `PrototypePlayer`
用于:
- screen 间跳转
- 交互热点高亮
- 模拟点击流程
- 从静态详情升级到可交互原型
### 10.5 新增 `SurfaceInspector`
用于:
- 查看当前组件树
- 查看 data binding
- 查看 validation issue
- 人工修正 style token
---
## 11. Agent 侧需要新增的能力
### 11.1 Design Orchestrator
负责:
- 理解用户目标
- 选择 surfaceType
- 分配子 Agent
- 决定是改数据、改结构还是改资产
### 11.2 Surface Planner
负责:
- 生成初始 surface 结构
- 选择 catalog 组件
- 输出 bindings 与 layout
### 11.3 Copy Agent
负责:
- 生成标题/副标题/卖点文案
- 根据品牌语气做变体
- 局部重写
### 11.4 Visual Critic
负责:
- 层级校验
- 安全区校验
- 文案长度校验
- 信息顺序校验
- 风格一致性校验
### 11.5 Prototype Linker
负责:
- 定义按钮跳转关系
- 定义 screen flow
- 构建 demo 路径
### 11.6 Action Interpreter
负责:
- 把用户在 UI 上的点击/编辑/拖拽解释成结构化意图
- 决定下一步是 `update_data`、`replace_component` 还是 `generate_variant`
---
## 12. Prompt / System 规则要怎么升级
我们现在的 Agent prompt 偏“直接回答用户”和“调用技能”。
未来增加一层 UI surface 之后,prompt 里要明确告诉模型:
### 12.1 什么时候输出 surface,而不是聊天文本
例如:
- 用户要求“给我一个详情页结构”
- 用户要求“改这一块 UI”
- 用户要求“把这个区域做三个方案”
- 用户要求“把几屏串成演示流程”
### 12.2 什么时候只 patch 数据,不重建结构
例如:
- “把标题再短一点”
- “这一屏换成更像苹果风”
- “第二块卖点文案改成更偏功能”
### 12.3 什么时候只 patch 资产
例如:
- “把主图换成更干净的白底”
- “把图标统一成线性风格”
### 12.4 什么时候向用户确认
例如:
- 改动会影响整体结构
- 需要重做多屏
- 需要丢弃已有 branch
- 需要进入 prototype flow
---
## 13. 推荐开发顺序(务实版)
### Phase 1:把详情模式正式抽象成 surface
目标:
- 先不要碰所有 Agent UI
- 先把 `AI详情生成` 升级成第一个正式的 `surfaceType=detail_page`
产出:
- `UiSurfaceSession`
- `UiSurfaceSnapshot`
- `SurfaceRenderer`
- `detail-ui-protocol -> CUSP adapter`
### Phase 2:支持 data model + binding
目标:
- title/subtitle/body 不再直接写死在 block 上
- 让内容更新优先走 `data update`
产出:
- surface data model
- path binding
- copy patch
### Phase 3:支持 action 回传
目标:
- 用户在 UI 上直接操作
- Agent 理解这些操作,而不是都回退成聊天文本
产出:
- action schema
- action -> protocol patch pipeline
- 前端交互埋点
### Phase 4:支持 branch / variant
目标:
- 用户一键出 2~4 套方案
- 方案可以并行存在、比较、切换
### Phase 5:支持 prototype flow
目标:
- screen 间连线
- 点击预览
- 演示模式
---
## 14. 这件事做成后,我们的体验会怎么变
现在的模式更像:
- Agent 理解用户
- 生成结果
- 落到画布
升级后会变成:
- Agent 理解用户
- 决定输出 **聊天 / 工作流 / UI surface / 原型**
- UI 不是一次性结果,而是一个可持续演化的对象
- 用户在画布里的行为也成为 Agent 输入
这时我们的 Agent 就不是“很多智能 patch”,而是真正的:
**对话协议 + UI 协议 + 工作流协议** 三层统一系统。
---
## 15. 下一步的最小可落地任务
建议立刻开始的第一批任务:
1. 新建 `lib/ui-surface-protocol.ts`
2. 定义 `Surface / Component / DataModel / Action / Patch` 基础 schema
3. 新建 `lib/ui-surface-renderer.ts`
4. 把 `detail-ui-protocol.ts` 适配到新 schema
5. 新建 `UiSurfaceSession / UiSurfaceSnapshot` 两张表
6. 增加 `/api/canvas/ui-surface/plan`
7. 增加 `/api/canvas/ui-surface/action`
8. 在 `CanvasChatPanel` 中插入 `SurfaceRenderer`
9. 先支持 5 个基础组件:`HeroBlock / FeatureGrid / ReviewCard / CTASection / SpecTable`
10. 再把 detail mode 升级为第一条正式 surface workflow
---
## 16. 参考资料(本次在线调研)
### Google 官方:Stitch
- Stitch 首发(2025-05-20):
https://developers.googleblog.com/stitch-a-new-way-to-design-uis/
- Stitch 接入 Gemini 3 与 Prototypes:
https://blog.google/innovation-and-ai/models-and-research/google-labs/stitch-gemini-3/
- Nano Banana Pro 已可在 Stitch 中使用:
https://blog.google/products-and-platforms/products/gemini/where-to-use-nano-banana-pro/
### Google 官方:A2UI
- A2UI README:
https://github.com/google/A2UI
- A2UI 数据绑定:
https://github.com/google/A2UI/blob/main/docs/concepts/data-binding.md
- A2UI Client-to-Server Actions:
https://github.com/google/A2UI/blob/main/docs/concepts/client_to_server_actions.md
- A2UI Renderers:
https://github.com/google/A2UI/blob/main/docs/reference/renderers.md
---
## 17. 最终判断
**Stitch 告诉我们产品应该长成什么样。**
**A2UI 告诉我们底层协议应该怎么搭。**
**而我们现有的 conversation-state-protocol,决定了我们有机会把“聪明 Agent”真正做成统一系统。**
接下来不该再继续堆零散 patch。
接下来应该正式进入:
**Conversation State Protocol + Canvas UI Surface Protocol + Workflow Events**
这三层统一化。

View File

@ -0,0 +1,5 @@
ALTER TABLE "CanvasDetailArchive"
ADD COLUMN IF NOT EXISTS "uiSurfaceSnapshot" jsonb;
ALTER TABLE "CanvasDetailArchive"
ADD COLUMN IF NOT EXISTS "uiSurfaceLatestReview" jsonb;

View File

@ -696,6 +696,8 @@ export const canvasDetailArchive = pgTable("CanvasDetailArchive", {
summary: text("summary"),
imageCount: integer("imageCount"),
coverUrl: text("coverUrl"),
uiSurfaceSnapshot: jsonb("uiSurfaceSnapshot").$type<Record<string, any> | null>(),
uiSurfaceLatestReview: jsonb("uiSurfaceLatestReview").$type<Record<string, any> | null>(),
createdAt: timestamp("createdAt").notNull().defaultNow(),
updatedAt: timestamp("updatedAt").notNull().defaultNow(),
});

View File

@ -271,6 +271,93 @@ export function buildDetailUiProtocol(params: {
};
}
function applyReviewHeuristicsToScreen(params: {
screen: DetailUiScreenProtocol;
screenSize: { width: number; height: number };
notes: DetailUiPatchNote[];
}) {
const noteText = params.notes.map((note) => note.message).join(" | ").toLowerCase();
if (!noteText.trim()) {
return params.screen;
}
const emphasizeTitle =
/标题层级|headline|title hierarchy|主标题|标题不够突出|主副标题/.test(noteText);
const addBreathingRoom =
/留白|spacing|rhythm|拥挤|挤|crowd|breathing|间距/.test(noteText);
const protectHeadlineZone =
/顶部|headline zone|safe area|安全区|避免重叠|不要遮挡|hero/.test(noteText);
const softenBodyDensity =
/正文|文案过密|copy too dense|copy density|太多文字|可读性/.test(noteText);
const safeArea = {
...params.screen.safeArea,
top: protectHeadlineZone
? Math.min(params.screen.safeArea.top + 12, params.screen.safeArea.bottom - 120)
: params.screen.safeArea.top,
};
const gapTitleToSubtitle = addBreathingRoom ? 24 : 16;
const gapSubtitleToBody = addBreathingRoom ? 20 : 14;
let cursorY = safeArea.top;
const blocks = params.screen.blocks.map((block) => {
const nextBlock = {
...block,
frame: { ...block.frame },
style: { ...block.style },
};
if (nextBlock.type === "title" && emphasizeTitle) {
nextBlock.style.fontSize = Math.min(84, nextBlock.style.fontSize + 4);
nextBlock.style.lineHeight = 1.12;
}
if (nextBlock.type === "subtitle" && emphasizeTitle) {
nextBlock.style.fontSize = Math.max(22, nextBlock.style.fontSize - 1);
}
if (nextBlock.type === "body" && softenBodyDensity) {
nextBlock.style.fontSize = Math.max(18, nextBlock.style.fontSize - 2);
nextBlock.style.lineHeight = 1.42;
}
const width = Math.max(
220,
Math.round(params.screenSize.width - safeArea.left - safeArea.right),
);
nextBlock.frame.x = safeArea.left;
nextBlock.frame.width = width;
nextBlock.frame.y = cursorY;
nextBlock.frame.height = estimateBlockHeight(
nextBlock.content,
nextBlock.style,
nextBlock.frame.width,
nextBlock.type === "title" ? 2 : nextBlock.type === "subtitle" ? 3 : 5,
);
if (nextBlock.type === "body") {
const maxBodyBottom = Math.max(cursorY + 80, safeArea.bottom);
nextBlock.frame.height = Math.min(nextBlock.frame.height, maxBodyBottom - cursorY);
}
cursorY +=
nextBlock.frame.height +
(nextBlock.type === "title"
? gapTitleToSubtitle
: nextBlock.type === "subtitle"
? gapSubtitleToBody
: 0);
return nextBlock;
});
return {
...params.screen,
safeArea,
blocks,
} satisfies DetailUiScreenProtocol;
}
export function buildDetailUiReviewPatch(params: {
generationMode: DetailGenerationMode;
screenSize: { width: number; height: number };
@ -280,6 +367,11 @@ export function buildDetailUiReviewPatch(params: {
typographyFixes?: string[];
conversionFixes?: string[];
}): DetailUiScreenReviewPatch {
const notes = [
...(params.layoutFixes || []).map((message) => ({ type: "layout" as const, message })),
...(params.typographyFixes || []).map((message) => ({ type: "typography" as const, message })),
...(params.conversionFixes || []).map((message) => ({ type: "conversion" as const, message })),
];
const protocol = buildDetailUiProtocol({
generationMode: params.generationMode,
screenSize: params.screenSize,
@ -291,11 +383,11 @@ export function buildDetailUiReviewPatch(params: {
version: "detail-ui-review-patch/v1",
mode: params.generationMode,
screenIndex: params.screen.index,
recommendedScreen: protocol.screens[0]!,
notes: [
...(params.layoutFixes || []).map((message) => ({ type: "layout" as const, message })),
...(params.typographyFixes || []).map((message) => ({ type: "typography" as const, message })),
...(params.conversionFixes || []).map((message) => ({ type: "conversion" as const, message })),
],
recommendedScreen: applyReviewHeuristicsToScreen({
screen: protocol.screens[0]!,
screenSize: params.screenSize,
notes,
}),
notes,
};
}

View File

@ -1,10 +1,25 @@
import type {
DetailUiProtocol,
DetailUiScreenInput,
DetailUiScreenReviewPatch,
} from "@/lib/detail-ui-protocol";
import {
getUiSurfaceChildren,
resolveUiSurfaceTextBinding,
type UiSurfaceSnapshot,
} from "@/lib/ui-surface-protocol";
import type { CanvasNode, TextNodeData } from "@/types/canvas";
type CanvasTextNodeDraft = Omit<CanvasNode, "id" | "createdAt" | "updatedAt">;
export type DetailTextLayerDraft = Omit<CanvasNode, "id" | "createdAt" | "updatedAt">;
export type DetailTypographyTokens = {
titleFontSize: number;
titleFontWeight: string;
subtitleFontSize: number;
subtitleFontWeight: string;
bodyFontSize: number;
bodyFontWeight: string;
};
function createTextNodeDraft(params: {
parentId: string;
@ -19,7 +34,7 @@ function createTextNodeDraft(params: {
lineHeight: number;
textAlign: "left" | "center" | "right";
metadata?: Record<string, unknown>;
}): CanvasTextNodeDraft {
}): DetailTextLayerDraft {
return {
type: "text",
position: { x: params.x, y: params.y },
@ -39,6 +54,19 @@ function createTextNodeDraft(params: {
};
}
function createManagedDetailMetadata(params: {
screenIndex?: number;
blockId: string;
blockType: "title" | "subtitle" | "body";
}) {
return {
detailUiManaged: true,
detailUiBlockId: params.blockId,
detailUiBlockType: params.blockType,
detailUiScreenIndex: params.screenIndex,
};
}
export function buildDetailTextLayerNodesFromProtocol(params: {
parentNodeId: string;
parentPosition: { x: number; y: number };
@ -47,7 +75,7 @@ export function buildDetailTextLayerNodesFromProtocol(params: {
screenProtocol?: DetailUiProtocol["screens"][number];
}) {
const protocolBlocks = params.screenProtocol?.blocks || [];
if (protocolBlocks.length === 0) return [] as CanvasTextNodeDraft[];
if (protocolBlocks.length === 0) return [] as DetailTextLayerDraft[];
const scaleX = params.parentSize.width / Math.max(1, params.screenSize.width);
const scaleY = params.parentSize.height / Math.max(1, params.screenSize.height);
@ -65,11 +93,11 @@ export function buildDetailTextLayerNodesFromProtocol(params: {
color: block.style.color,
lineHeight: block.style.lineHeight,
textAlign: block.style.textAlign,
metadata: {
detailUiBlockId: block.id,
detailUiBlockType: block.type,
detailUiScreenIndex: params.screenProtocol?.index,
},
metadata: createManagedDetailMetadata({
blockId: block.id,
blockType: block.type,
screenIndex: params.screenProtocol?.index,
}),
}),
);
}
@ -81,7 +109,7 @@ export function buildDetailTextLayerNodesFromReviewPatch(params: {
screenSize: { width: number; height: number };
uiPatch?: DetailUiScreenReviewPatch;
}) {
if (!params.uiPatch) return [] as CanvasTextNodeDraft[];
if (!params.uiPatch) return [] as DetailTextLayerDraft[];
return buildDetailTextLayerNodesFromProtocol({
parentNodeId: params.parentNodeId,
parentPosition: params.parentPosition,
@ -91,6 +119,70 @@ export function buildDetailTextLayerNodesFromReviewPatch(params: {
});
}
export function buildDetailTextLayerNodesFromSurfaceScreen(params: {
parentNodeId: string;
parentPosition: { x: number; y: number };
parentSize: { width: number; height: number };
surface: UiSurfaceSnapshot | null | undefined;
screenIndex: number;
}) {
if (!params.surface) return [] as DetailTextLayerDraft[];
const screenComponent =
params.surface.components.find(
(component) =>
component.type === "detail_screen" &&
Number(component.props?.screenIndex || 0) === params.screenIndex,
) || null;
if (!screenComponent) return [] as DetailTextLayerDraft[];
const textStack = getUiSurfaceChildren(params.surface, screenComponent.id).find(
(component) => component.type === "detail_text_stack",
);
if (!textStack) return [] as DetailTextLayerDraft[];
const scaleX = params.parentSize.width / Math.max(1, params.surface.screenSize.width);
const scaleY = params.parentSize.height / Math.max(1, params.surface.screenSize.height);
return getUiSurfaceChildren(params.surface, textStack.id)
.filter((component) => component.type === "detail_text_block")
.map((component) => {
const frame = (component.layout?.frame || {}) as {
x?: number;
y?: number;
width?: number;
height?: number;
};
const style = (component.style || {}) as Record<string, unknown>;
return createTextNodeDraft({
parentId: params.parentNodeId,
x: params.parentPosition.x + Math.round(Number(frame.x || 0) * scaleX),
y: params.parentPosition.y + Math.round(Number(frame.y || 0) * scaleY),
width: Math.max(120, Math.round(Number(frame.width || 240) * scaleX)),
height: Math.max(48, Math.round(Number(frame.height || 60) * scaleY)),
content: resolveUiSurfaceTextBinding(params.surface, component, "content"),
fontSize: Math.max(
12,
Math.round(Number(style.fontSize || 24) * Math.min(scaleX, scaleY)),
),
fontWeight: String(style.fontWeight || "500"),
color: String(style.color || "#111827"),
lineHeight: Number(style.lineHeight || 1.3),
textAlign:
style.textAlign === "center" || style.textAlign === "right"
? style.textAlign
: "left",
metadata: {
detailUiManaged: true,
detailUiBlockId: String(component.metadata?.blockId || component.id),
detailUiBlockType: String(component.metadata?.blockType || ""),
detailUiScreenIndex: params.screenIndex,
detailSurfaceComponentId: component.id,
},
});
});
}
export function buildLegacyDetailTextLayerNodes(params: {
parentNodeId: string;
parentPosition: { x: number; y: number };
@ -108,7 +200,7 @@ export function buildLegacyDetailTextLayerNodes(params: {
}) {
const safeLeft = Math.round(params.parentWidth * 0.07);
const textWidth = Math.max(240, Math.round(params.parentWidth - safeLeft * 2));
const drafts: CanvasTextNodeDraft[] = [];
const drafts: DetailTextLayerDraft[] = [];
const titleText = String(params.title || "").trim();
const subtitleText = String(params.subtitle || "").trim();
@ -131,6 +223,11 @@ export function buildLegacyDetailTextLayerNodes(params: {
color: "#111827",
lineHeight: 1.15,
textAlign: "left",
metadata: createManagedDetailMetadata({
blockId: `legacy-screen-${params.screenIndex + 1}-title`,
blockType: "title",
screenIndex: params.screenIndex + 1,
}),
}),
);
}
@ -149,6 +246,11 @@ export function buildLegacyDetailTextLayerNodes(params: {
color: "#374151",
lineHeight: 1.22,
textAlign: "left",
metadata: createManagedDetailMetadata({
blockId: `legacy-screen-${params.screenIndex + 1}-subtitle`,
blockType: "subtitle",
screenIndex: params.screenIndex + 1,
}),
}),
);
}
@ -167,9 +269,65 @@ export function buildLegacyDetailTextLayerNodes(params: {
color: "#4b5563",
lineHeight: 1.38,
textAlign: "left",
metadata: createManagedDetailMetadata({
blockId: `legacy-screen-${params.screenIndex + 1}-body`,
blockType: "body",
screenIndex: params.screenIndex + 1,
}),
}),
);
}
return drafts;
}
export function buildDetailTextLayerNodes(params: {
parentNodeId: string;
parentPosition: { x: number; y: number };
parentSize: { width: number; height: number };
screenSize: { width: number; height: number };
screenIndex: number;
screen?: Pick<DetailUiScreenInput, "title" | "subtitle" | "copy">;
screenProtocol?: DetailUiProtocol["screens"][number];
typography: DetailTypographyTokens;
}) {
if (params.screenProtocol) {
return buildDetailTextLayerNodesFromProtocol({
parentNodeId: params.parentNodeId,
parentPosition: params.parentPosition,
parentSize: params.parentSize,
screenSize: params.screenSize,
screenProtocol: params.screenProtocol,
});
}
return buildLegacyDetailTextLayerNodes({
parentNodeId: params.parentNodeId,
parentPosition: params.parentPosition,
parentWidth: params.parentSize.width,
screenIndex: params.screenIndex,
title: params.screen?.title,
subtitle: params.screen?.subtitle,
body: params.screen?.copy,
titleFontSize: params.typography.titleFontSize,
titleFontWeight: params.typography.titleFontWeight,
subtitleFontSize: params.typography.subtitleFontSize,
subtitleFontWeight: params.typography.subtitleFontWeight,
bodyFontSize: params.typography.bodyFontSize,
bodyFontWeight: params.typography.bodyFontWeight,
});
}
export function collectManagedDetailTextLayerNodeIds(
nodes: CanvasNode[],
parentNodeId: string,
) {
return nodes
.filter(
(node) =>
node.parentId === parentNodeId &&
node.type === "text" &&
Boolean((node.data as any)?.metadata?.detailUiManaged),
)
.map((node) => node.id);
}

View File

@ -0,0 +1,297 @@
import type {
DetailGenerationMode,
DetailUiGlobalStyle,
DetailUiProtocol,
DetailUiScreenInput,
DetailUiScreenProtocol,
DetailUiScreenReviewPatch,
} from "@/lib/detail-ui-protocol";
import type {
UiSurfaceComponent,
UiSurfacePatch,
UiSurfaceSnapshot,
} from "@/lib/ui-surface-protocol";
function mapBlockTypeToBindingKey(blockType: "title" | "subtitle" | "body") {
if (blockType === "title") return "title";
if (blockType === "subtitle") return "subtitle";
return "copy";
}
function createDetailSurfaceId(params: {
generationMode: DetailGenerationMode;
screenCount: number;
}) {
return `detail-surface-${params.generationMode}-${params.screenCount}-790x1300`;
}
function createRootComponent(): UiSurfaceComponent {
return {
id: "detail-page-root",
type: "detail_page_root",
parentId: null,
props: {
role: "detail_page",
},
};
}
function createScreenSubtree(params: {
screenIndex: number;
screen: Pick<
DetailUiScreenInput,
"title" | "subtitle" | "copy" | "screenType" | "objective" | "layoutNote"
>;
screenProtocol?: DetailUiScreenProtocol;
rootId?: string;
}) {
const screenId = params.rootId || `detail-screen-${params.screenIndex}`;
const backgroundId = `${screenId}-background`;
const textStackId = `${screenId}-text-stack`;
const components: UiSurfaceComponent[] = [
{
id: screenId,
type: "detail_screen",
parentId: "detail-page-root",
props: {
screenIndex: params.screenIndex,
screenType: params.screen.screenType || "",
objective: params.screen.objective || "",
layoutNote: params.screen.layoutNote || "",
},
metadata: {
screenIndex: params.screenIndex,
},
},
{
id: backgroundId,
type: "detail_background_image",
parentId: screenId,
slot: "background",
props: {
screenIndex: params.screenIndex,
},
metadata: {
screenIndex: params.screenIndex,
},
},
{
id: textStackId,
type: "detail_text_stack",
parentId: screenId,
slot: "content",
props: {
screenIndex: params.screenIndex,
safeArea: params.screenProtocol?.safeArea || null,
},
metadata: {
screenIndex: params.screenIndex,
},
},
];
const blocks = params.screenProtocol?.blocks || [];
blocks.forEach((block, blockIndex) => {
const bindingKey = mapBlockTypeToBindingKey(block.type);
components.push({
id: `${screenId}-text-${blockIndex + 1}`,
type: "detail_text_block",
parentId: textStackId,
props: {
blockType: block.type,
frame: block.frame,
textKey: bindingKey,
},
bindings: {
content: {
kind: "text",
path: `/screens/${params.screenIndex - 1}/${bindingKey}`,
},
},
style: {
fontFamily: block.style.fontFamily || "",
fontSize: block.style.fontSize,
fontWeight: block.style.fontWeight,
lineHeight: block.style.lineHeight,
color: block.style.color,
textAlign: block.style.textAlign,
},
layout: {
frame: block.frame,
},
metadata: {
screenIndex: params.screenIndex,
blockType: block.type,
blockId: block.id,
},
});
});
return components;
}
function buildDetailSurfaceDataModel(params: {
summary?: string;
screens: DetailUiScreenInput[];
globalStyle?: DetailUiGlobalStyle;
}) {
return {
summary: params.summary || "",
globalStyle: params.globalStyle || {},
screens: params.screens.map((screen) => ({
index: screen.index,
title: String(screen.title || "").trim(),
subtitle: String(screen.subtitle || "").trim(),
copy: String(screen.copy || "").trim(),
screenType: screen.screenType || "",
objective: screen.objective || "",
layoutNote: screen.layoutNote || "",
})),
};
}
export function buildDetailUiSurfaceSnapshot(params: {
surfaceId?: string;
generationMode: DetailGenerationMode;
screenSize: { width: number; height: number };
summary?: string;
globalStyle?: DetailUiGlobalStyle;
screens: DetailUiScreenInput[];
uiProtocol?: DetailUiProtocol;
uiValidation?: {
passed: boolean;
issues: string[];
};
}) {
const surfaceId =
params.surfaceId ||
createDetailSurfaceId({
generationMode: params.generationMode,
screenCount: params.screens.length,
});
const components: UiSurfaceComponent[] = [createRootComponent()];
params.screens.forEach((screen) => {
const screenProtocol = params.uiProtocol?.screens.find(
(item) => item.index === screen.index,
);
components.push(
...createScreenSubtree({
screenIndex: screen.index,
screen,
screenProtocol,
}),
);
});
return {
version: "ui-surface/v1",
surfaceId,
surfaceType: "detail_page",
generationMode: params.generationMode,
rootComponentId: "detail-page-root",
screenSize: params.screenSize,
components,
dataModel: buildDetailSurfaceDataModel({
summary: params.summary,
screens: params.screens,
globalStyle: params.globalStyle,
}),
validation: params.uiValidation
? {
passed: params.uiValidation.passed,
issues: params.uiValidation.issues,
source: "plan",
}
: undefined,
metadata: {
detailScreenCount: params.screens.length,
},
} satisfies UiSurfaceSnapshot;
}
export function buildDetailUiSurfacePatchFromReview(params: {
surfaceId: string;
screenIndex: number;
screen: Pick<
DetailUiScreenInput,
"index" | "title" | "subtitle" | "copy" | "screenType" | "objective" | "layoutNote"
>;
uiPatch?: DetailUiScreenReviewPatch;
uiValidation?: {
passed: boolean;
issues: string[];
};
}) {
if (!params.uiPatch) return null;
const screenId = `detail-screen-${params.screenIndex}`;
const textStackId = `${screenId}-text-stack`;
const replacementComponents = createScreenSubtree({
screenIndex: params.screenIndex,
screen: params.screen,
screenProtocol: params.uiPatch.recommendedScreen,
rootId: screenId,
}).filter((component) => component.id === textStackId || component.parentId === textStackId);
return {
version: "ui-surface-patch/v1",
surfaceId: params.surfaceId,
surfaceType: "detail_page",
operations: [
{
type: "replace_component_subtree",
componentId: textStackId,
components: replacementComponents,
},
{
type: "set_data",
updates: [
{
path: `/screens/${params.screenIndex - 1}/title`,
value: String(params.screen.title || "").trim(),
},
{
path: `/screens/${params.screenIndex - 1}/subtitle`,
value: String(params.screen.subtitle || "").trim(),
},
{
path: `/screens/${params.screenIndex - 1}/copy`,
value: String(params.screen.copy || "").trim(),
},
{
path: `/screens/${params.screenIndex - 1}/layoutNote`,
value: String(params.screen.layoutNote || "").trim(),
},
],
},
...(params.uiValidation
? [
{
type: "set_validation" as const,
validation: {
passed: params.uiValidation.passed,
issues: params.uiValidation.issues,
source: "review" as const,
},
},
]
: []),
],
} satisfies UiSurfacePatch;
}
export function findDetailSurfaceScreenComponent(
snapshot: UiSurfaceSnapshot | null | undefined,
screenIndex: number,
) {
if (!snapshot) return null;
return (
snapshot.components.find(
(component) =>
component.type === "detail_screen" &&
Number(component.props?.screenIndex || 0) === screenIndex,
) || null
);
}

285
lib/ui-surface-protocol.ts Normal file
View File

@ -0,0 +1,285 @@
export type UiSurfaceType = "detail_page";
export type UiSurfaceGenerationMode = "classic" | "agent_layout";
export type UiSurfaceComponentType =
| "detail_page_root"
| "detail_screen"
| "detail_background_image"
| "detail_text_stack"
| "detail_text_block";
export type UiSurfaceTextBindingKey = "title" | "subtitle" | "copy";
export interface UiSurfaceFrame {
x: number;
y: number;
width: number;
height: number;
}
export interface UiSurfaceBinding {
kind: "text";
path: string;
}
export interface UiSurfaceComponent {
id: string;
type: UiSurfaceComponentType;
parentId: string | null;
slot?: string;
props?: Record<string, unknown>;
bindings?: Record<string, UiSurfaceBinding | undefined>;
layout?: {
frame?: UiSurfaceFrame;
zIndex?: number;
};
style?: Record<string, unknown>;
metadata?: Record<string, unknown>;
}
export interface UiSurfaceValidation {
passed: boolean;
issues: string[];
source?: "plan" | "review";
}
export interface UiSurfaceSnapshot {
version: "ui-surface/v1";
surfaceId: string;
surfaceType: UiSurfaceType;
generationMode: UiSurfaceGenerationMode;
rootComponentId: string;
screenSize: {
width: number;
height: number;
};
components: UiSurfaceComponent[];
dataModel: Record<string, unknown>;
validation?: UiSurfaceValidation;
metadata?: Record<string, unknown>;
}
export interface UiSurfaceDataUpdate {
path: string;
value: unknown;
}
export type UiSurfacePatchOperation =
| {
type: "replace_component_subtree";
componentId: string;
components: UiSurfaceComponent[];
}
| {
type: "set_data";
updates: UiSurfaceDataUpdate[];
}
| {
type: "set_validation";
validation: UiSurfaceValidation;
};
export interface UiSurfacePatch {
version: "ui-surface-patch/v1";
surfaceId: string;
surfaceType: UiSurfaceType;
operations: UiSurfacePatchOperation[];
}
function normalizePathSegments(path: string) {
return String(path || "")
.split("/")
.map((segment) => segment.trim())
.filter(Boolean);
}
function cloneValue<T>(value: T): T {
return JSON.parse(JSON.stringify(value)) as T;
}
function setValueAtPath(target: Record<string, unknown>, path: string, value: unknown) {
const segments = normalizePathSegments(path);
if (segments.length === 0) return target;
let pointer: unknown = target;
for (let index = 0; index < segments.length; index += 1) {
const segment = segments[index]!;
const isLast = index === segments.length - 1;
const nextSegment = segments[index + 1];
const nextIsIndex = nextSegment ? /^\d+$/.test(nextSegment) : false;
if (Array.isArray(pointer)) {
const arrayIndex = Number(segment);
if (!Number.isFinite(arrayIndex) || arrayIndex < 0) return target;
if (isLast) {
pointer[arrayIndex] = value;
return target;
}
if (
pointer[arrayIndex] === undefined ||
pointer[arrayIndex] === null ||
(typeof pointer[arrayIndex] !== "object" && !Array.isArray(pointer[arrayIndex]))
) {
pointer[arrayIndex] = nextIsIndex ? [] : {};
}
pointer = pointer[arrayIndex] as unknown;
continue;
}
if (!pointer || typeof pointer !== "object") return target;
const objectPointer = pointer as Record<string, unknown>;
if (isLast) {
objectPointer[segment] = value;
return target;
}
const currentValue = objectPointer[segment];
if (
currentValue === undefined ||
currentValue === null ||
(typeof currentValue !== "object" && !Array.isArray(currentValue))
) {
objectPointer[segment] = nextIsIndex ? [] : {};
}
pointer = objectPointer[segment] as unknown;
}
return target;
}
export function getUiSurfaceComponent(
snapshot: UiSurfaceSnapshot | null | undefined,
componentId: string,
) {
if (!snapshot) return null;
return snapshot.components.find((component) => component.id === componentId) || null;
}
export function getUiSurfaceChildren(
snapshot: UiSurfaceSnapshot | null | undefined,
parentId: string,
) {
if (!snapshot) return [];
return snapshot.components.filter((component) => component.parentId === parentId);
}
export function collectUiSurfaceDescendantIds(
snapshot: UiSurfaceSnapshot | null | undefined,
parentId: string,
) {
if (!snapshot) return [];
const collected = new Set<string>();
const queue = [parentId];
while (queue.length > 0) {
const current = queue.shift();
if (!current) continue;
const children = snapshot.components.filter((component) => component.parentId === current);
children.forEach((child) => {
if (collected.has(child.id)) return;
collected.add(child.id);
queue.push(child.id);
});
}
return [...collected];
}
export function resolveUiSurfaceTextBinding(
snapshot: UiSurfaceSnapshot | null | undefined,
component: UiSurfaceComponent | null | undefined,
bindingKey: string,
) {
if (!snapshot || !component) return "";
const binding = component.bindings?.[bindingKey];
if (!binding || binding.kind !== "text") {
return String(component.props?.[bindingKey] || "");
}
const segments = normalizePathSegments(binding.path);
let pointer: unknown = snapshot.dataModel;
for (const segment of segments) {
if (Array.isArray(pointer)) {
const arrayIndex = Number(segment);
if (!Number.isFinite(arrayIndex)) return "";
pointer = pointer[arrayIndex];
continue;
}
if (!pointer || typeof pointer !== "object") return "";
pointer = (pointer as Record<string, unknown>)[segment];
}
return typeof pointer === "string" ? pointer : String(pointer || "");
}
export function applyUiSurfacePatch(
snapshot: UiSurfaceSnapshot | null | undefined,
patch: UiSurfacePatch | null | undefined,
) {
if (!snapshot || !patch) return snapshot || null;
if (
patch.version !== "ui-surface-patch/v1" ||
patch.surfaceType !== snapshot.surfaceType ||
patch.surfaceId !== snapshot.surfaceId
) {
return snapshot;
}
const nextSnapshot = cloneValue(snapshot);
patch.operations.forEach((operation) => {
if (operation.type === "replace_component_subtree") {
const descendantIds = collectUiSurfaceDescendantIds(nextSnapshot, operation.componentId);
const removeIdSet = new Set(descendantIds);
nextSnapshot.components = nextSnapshot.components.filter(
(component) => !removeIdSet.has(component.id),
);
const replacementComponentIds = new Set(operation.components.map((component) => component.id));
nextSnapshot.components = nextSnapshot.components.filter(
(component) => !replacementComponentIds.has(component.id),
);
nextSnapshot.components.push(...cloneValue(operation.components));
return;
}
if (operation.type === "set_data") {
operation.updates.forEach((update) => {
setValueAtPath(nextSnapshot.dataModel, update.path, update.value);
});
return;
}
if (operation.type === "set_validation") {
nextSnapshot.validation = cloneValue(operation.validation);
}
});
return nextSnapshot;
}
export function countUiSurfaceComponents(
snapshot: UiSurfaceSnapshot | null | undefined,
type?: UiSurfaceComponentType,
) {
if (!snapshot) return 0;
return type
? snapshot.components.filter((component) => component.type === type).length
: snapshot.components.length;
}
export function getUiSurfaceScreenComponents(
snapshot: UiSurfaceSnapshot | null | undefined,
) {
if (!snapshot) return [] as UiSurfaceComponent[];
return snapshot.components
.filter((component) => component.type === "detail_screen")
.sort(
(left, right) =>
Number(left.props?.screenIndex || 0) - Number(right.props?.screenIndex || 0),
);
}