From c2ef33753a94c523c725e27cf5d771da36958dfa Mon Sep 17 00:00:00 2001 From: 134 Date: Mon, 22 Jun 2026 17:56:58 +0800 Subject: [PATCH] fix: improve detail replacement generation controls --- .../api/chat/_lib/detail-replacement.ts | 98 ++++- app/(chat)/api/chat/route.ts | 73 +++- app/(chat)/api/chat/schema.ts | 19 +- app/api/canvas/detail-plan/route.ts | 131 +++++- app/api/canvas/local-edit/route.ts | 63 ++- .../agents/AgentGenerationWorkspaceDarkV2.tsx | 294 +++++++++++-- components/canvas/CanvasChatPanel.tsx | 234 ++++++++++- components/canvas/nodes/ImageNode.tsx | 7 +- lib/ai/context-memory/types.ts | 4 +- lib/canvas/agent-context.ts | 392 ++++++++++++++++++ services/canvas/index.ts | 2 +- tests/routes/canvas-chat-helpers.test.ts | 76 ++++ types/canvas/agent-protocol.ts | 161 +++++++ 13 files changed, 1455 insertions(+), 99 deletions(-) create mode 100644 lib/canvas/agent-context.ts create mode 100644 types/canvas/agent-protocol.ts diff --git a/app/(chat)/api/chat/_lib/detail-replacement.ts b/app/(chat)/api/chat/_lib/detail-replacement.ts index 25ebed7c7..c17b68e19 100644 --- a/app/(chat)/api/chat/_lib/detail-replacement.ts +++ b/app/(chat)/api/chat/_lib/detail-replacement.ts @@ -1286,6 +1286,28 @@ function applyDetailReplacementScenePlanGuards( next.routeReason = appendUniqueInstruction(next.routeReason, params.visualProfile.reason, 220); } + if ( + params.hasModelImage && + !isProductOnly && + !next.useModelImage && + (next.peoplePolicy !== "none" || + next.generationRoute === "people_relation_locked" || + next.pageKind === "people_scene" || + next.pageKind === "action_poster") + ) { + next = { + ...next, + useModelImage: true, + modelImageReason: + next.modelImageReason || + "本屏已判定为人物/动作详情页，必须使用用户提供的模特图锁定人物身份。", + generationPrompt: appendUniqueInstruction( + next.generationPrompt, + "本屏出现人物时必须使用用户提供的同一模特：脸型五官、发型发色、体型肤色和穿搭风格保持一致；不得换成其他人物。", + ), + }; + } + return next; } @@ -1723,19 +1745,31 @@ export function normalizeAgentModeReferences( ): AgentReplacementModePayload["references"] { if (!payload?.references?.length) return []; return payload.references - .map((item) => ({ - id: String(item.id || "").trim(), - label: String(item.label || "").trim() || "参考图", - imageUrl: String(item.imageUrl || "").trim(), - candidateId: String(item.candidateId || "").trim() || null, - sortOrder: Number.isFinite(Number(item.sortOrder)) - ? Math.max(0, Math.min(999, Math.round(Number(item.sortOrder)))) - : 0, - passthrough: item.passthrough === true, - assignedProductIndex: Number.isFinite(Number(item.assignedProductIndex)) - ? Math.max(0, Math.min(9, Math.round(Number(item.assignedProductIndex)))) - : null, - })) + .map((item) => { + const rerunReferenceImages = Array.isArray(item.rerunReferenceImages) + ? item.rerunReferenceImages + .map((image) => normalizeAgentModeSourceImage(image)) + .filter( + (image): image is NonNullable> => + Boolean(image) + ) + .slice(0, 6) + : []; + return { + id: String(item.id || "").trim(), + label: String(item.label || "").trim() || "参考图", + imageUrl: String(item.imageUrl || "").trim(), + candidateId: String(item.candidateId || "").trim() || null, + sortOrder: Number.isFinite(Number(item.sortOrder)) + ? Math.max(0, Math.min(999, Math.round(Number(item.sortOrder)))) + : 0, + passthrough: item.passthrough === true, + assignedProductIndex: Number.isFinite(Number(item.assignedProductIndex)) + ? Math.max(0, Math.min(9, Math.round(Number(item.assignedProductIndex)))) + : null, + ...(rerunReferenceImages.length > 0 ? { rerunReferenceImages } : {}), + }; + }) .filter((item) => item.id && /^https?:\/\//i.test(item.imageUrl)) .sort((left, right) => left.sortOrder - right.sortOrder); } @@ -2210,7 +2244,7 @@ export async function analyzeReplacementExecutionPlan(params: { isDetailReplacement ? "用户上传图可能是产品、材质、板材、原材料或细节参考，不要因为它不是完整商品主体就排除；后续应根据上传图外观与参考图里的材质描述共同判断处理方向。" : "", - "这张图片是唯一的详情参考图底板，它决定当前这一屏的版式、文字排版、UI图标、按钮、机位、构图、留白和信息层级。", + "这张图片是唯一的详情参考图底板，它决定当前这一屏的版式、文字排版、UI图标、按钮、机位、构图、留白和信息层级；如果画面里有产品主体/展示区，它也决定产品的拍摄角度、朝向、透视、可见面数量、占比和标注指向。", isProductOnlyDetailReplacement ? "这张参考图要求最大程度原样保留。后续生图只允许替换其中的产品主体/产品展示区域，背景、文案、按钮、图标、分割线、品牌标识、留白、装饰和光影氛围都必须保留。" : isDetailReplacement @@ -2248,12 +2282,12 @@ export async function analyzeReplacementExecutionPlan(params: { ? "6. 判断 peoplePolicy：有人物/多人/自拍/聚会/围坐/聚餐/模特互动/拿手机/用餐/试坐/承重动作时必须 preserve_relation；如果文字明确写了“多人围坐、聚餐、合照、用餐、单侧承重、横向受力测试、拿手机”等动作，generationPrompt 必须把这个动作写成硬约束；如果只提供 1 张模特图但参考图多人，仍要保留参考图人数和关系，额外人物按参考图生成相近人群。" : "", isDetailReplacement - ? "7. 提炼 sceneSummary、cameraSummary、placementSummary、preserveSummary、styleSummary、avoidSummary。" + ? "7. 提炼 sceneSummary、cameraSummary、placementSummary、preserveSummary、styleSummary、avoidSummary；cameraSummary 必须写出产品主体的视角特征，例如正侧视/三分之四/俯视/平视、倾斜方向、可见面数量、主体朝向和透视消失方向。" : "", isProductOnlyDetailReplacement ? "4. 输出 generationPrompt：这是一段给后续生图模型的本屏生成指令，要求只替换产品主体，保留其余背景、文案、排版、装饰和光影完全不变。" : isDetailReplacement - ? "8. 输出 generationPrompt：这是一段给后续生图模型的本屏生成指令。若 promptPackPolicy=forbid，必须明确“不使用前置词包，参考图语义优先”；若是动作海报，必须写清人物数量、围坐/试坐/承重等动作、产品形状和版式必须照参考图，同时写清“前置词包必须完整重建原照片内容区里的背景/空间/软装/灯光/产品场景，不能原图复刻”；若是产品尺寸页，必须写清白底抠图、尺寸线、单位和数字标注，不允许室内场景；若 promptPackPolicy=allow/soft 且 peoplePolicy=preserve_relation，明确“保留人物数量/动作/关系与文字层级，同时使用前置词包重建背景/产品场景”。" + ? "8. 输出 generationPrompt：这是一段给后续生图模型的本屏生成指令。若 promptPackPolicy=forbid，必须明确“不使用前置词包，参考图语义优先”；若是动作海报，必须写清人物数量、围坐/试坐/承重等动作、产品形状和版式必须照参考图，同时写清“前置词包必须完整重建原照片内容区里的背景/空间/软装/灯光/产品场景，不能原图复刻”；若是产品尺寸页，必须写清白底抠图、尺寸线、单位和数字标注，不允许室内场景；若 promptPackPolicy=allow/soft 且 peoplePolicy=preserve_relation，明确“保留人物数量/动作/关系与文字层级，同时使用前置词包重建背景/产品场景”。只要参考图有产品主体，generationPrompt 必须明确产品拍摄角度、朝向、透视和标注指向以参考图为准。" : "4. 输出 generationPrompt：这是一段给后续生图模型的本屏生成指令，描述这一屏需要的场景、机位、主体位置、周边环境重建方向以及必须保留的内容。", isProductOnlyDetailReplacement ? "5. sceneSummary、styleSummary 和 generationPrompt 必须强调‘只替换产品，不动其他内容’，不要引导重建场景。" @@ -2421,6 +2455,7 @@ export function buildAgentReplacementPrompt(params: { hasModelImage: boolean; productImageCount: number; originalProductImageCount: number; + rerunReferenceCount?: number; templateReferenceCount: number; pairedProductIndex?: number; scenePlan?: ReplacementScenePlan | null; @@ -2455,14 +2490,14 @@ export function buildAgentReplacementPrompt(params: { ? `图片${nextImageIndex}` : `图片${nextImageIndex}-${nextImageIndex + params.productImageCount - 1}`; imageOrderLines.push( - `${productRangeLabel}：用户上传的产品/材质/板材/原材料或细节参考图，必须综合理解外观、材质、颜色、纹理和结构，不可只看第一张，也不要因为它不是完整商品主体就忽略。` + `${productRangeLabel}：用户上传的产品/材质/板材/原材料或细节参考图，必须综合理解外观、材质、颜色、纹理和结构，不可只看第一张，也不要因为它不是完整商品主体就忽略；如结果中需要生成商品主体，商品拍摄角度、朝向、透视和画面占比必须以最后一张参考图为准，不得直接沿用这些上传图的拍摄角度。` ); nextImageIndex += params.productImageCount; } if (params.hasModelImage) { imageOrderLines.push( - `图片${nextImageIndex}：模特图，仅在当前参考图本身包含人物展示时参考；如果参考图是多人，只把这张模特图作为主人物身份/气质参考，其余人物按参考图的人数、关系和风格补齐。` + `图片${nextImageIndex}：用户提供的固定模特图，是人物身份锁定图；当前屏需要人物时，必须复用这张图里的同一人物（脸型五官、发型发色、体型肤色、穿搭风格保持一致）。如果参考图是多人，只把这张模特图作为主人物身份参考，其余人物按参考图的人数、关系和风格补齐。` ); nextImageIndex += 1; } @@ -2478,8 +2513,20 @@ export function buildAgentReplacementPrompt(params: { nextImageIndex += params.templateReferenceCount; } + const rerunReferenceCount = Math.max(0, Math.round(Number(params.rerunReferenceCount || 0))); + if (rerunReferenceCount > 0) { + const rerunRangeLabel = + rerunReferenceCount === 1 + ? `图片${nextImageIndex}` + : `图片${nextImageIndex}-${nextImageIndex + rerunReferenceCount - 1}`; + imageOrderLines.push( + `${rerunRangeLabel}：本次重新生图的补充参考图，只用于理解用户新增的风格、构图、细节或局部调整方向；不得覆盖最终参考图底板的版式、文字层级和页面结构。` + ); + nextImageIndex += rerunReferenceCount; + } + imageOrderLines.push( - `图片${nextImageIndex}：当前参考图底板（唯一版式基准图），必须保留它的文字排版、信息区、分割线、留白、机位和整体结构。` + `图片${nextImageIndex}：当前参考图底板（唯一版式基准图），必须保留它的文字排版、信息区、分割线、留白、机位和整体结构；如果它包含产品主体/展示区，它同时决定产品姿态、朝向、透视、可见面数量、占比和标注指向。` ); const marketplaceBrandRemovalLines = params.modeType === "detail_replacement" && params.marketplaceProductDetailReference @@ -2489,6 +2536,12 @@ export function buildAgentReplacementPrompt(params: { "如果参考图标题、标签或画面中出现可识别的外部品牌字样，不要保留该品牌名，不要换成相似字，不要生成伪 Logo；需要文字时只保留非品牌卖点语义或改成无品牌通用描述。", ] : []; + const productAngleLockLines = [ + "【产品角度锁定规则】最终商品/主体的拍摄角度以最后一张参考图为准，而不是以前面的产品输入图为准；前面的产品图只提供款式、材质、颜色、结构和细节。", + "必须逐项对齐最后一张参考图中的产品姿态：朝向、正侧视/三分之四/俯视/平视关系、桌面或面板倾斜方向、可见侧面数量、支架/腿部/把手/转轴位置、主体外轮廓、占比、重心和透视消失方向。", + "如果最后一张参考图的产品是侧视或低视角，不允许改成俯视三分之四；如果是俯视三分之四，不允许改成正侧视；不允许镜像、旋转、翻转或把产品重画成另一个角度。", + "若参考图含功能标签、箭头、尺寸线、编号或引导线，它们指向的实际结构部位必须保持一致，不得因产品转角改变而让标签指向错误位置。", + ]; const normalizedBasePrompt = String(params.basePrompt || "").trim(); const promptPackInstructionLines = params.modeType === "detail_replacement" && @@ -2555,7 +2608,7 @@ export function buildAgentReplacementPrompt(params: { params.scenePlan.useModelImage ? params.scenePlan.modelImageReason ? `本屏允许使用模特图：${params.scenePlan.modelImageReason}` - : "本屏允许使用模特图，但仅在参考图当前这一屏确实存在人物展示需求时使用。" + : "本屏允许使用模特图；只要最终画面出现人物，就必须使用用户提供的同一模特身份。" : params.hasModelImage ? params.scenePlan.modelImageReason ? `本屏禁止使用模特图：${params.scenePlan.modelImageReason}` @@ -2588,6 +2641,7 @@ export function buildAgentReplacementPrompt(params: { ...promptPackInstructionLines, ...scenePlanLines, ...marketplaceBrandRemovalLines, + ...productAngleLockLines, "你现在做的是‘仅替换产品’，不是重建场景，也不是重新设计整张详情图。", "忽略用户补充提示词里的场景风格、氛围、空间改造要求；这一次只服务于产品主体替换。", "最后一张参考图是唯一底板，除产品主体/产品展示区外，其余所有内容都必须保持不变。", @@ -2616,6 +2670,7 @@ export function buildAgentReplacementPrompt(params: { ...promptPackInstructionLines, ...scenePlanLines, ...marketplaceBrandRemovalLines, + ...productAngleLockLines, "你现在做的是“详情参考图背景/场景图替换”，不是重新设计整张详情图。", "【版式锁层规则】把最后一张参考图当成 PSD 模板：文字、标题、编号、卖点标签、按钮、图标、白底/浅色底、留白、分割线、原有圆角照片区域和原有裁切范围都属于锁定版式层，不允许移动、缩放、重排或重新设计；不要把这些版式边界画成新的白色线框、描边或卡片。", isReferencePosterActionPlan @@ -2730,6 +2785,7 @@ export function buildAgentReplacementPrompt(params: { "【图片顺序规则】", ...imageOrderLines, ...scenePlanLines, + ...productAngleLockLines, params.pairedProductIndex !== undefined ? `请优先使用第 ${params.pairedProductIndex + 1} 张输入商品图作为本张参考图的一对一替换主体。` : "请使用输入商品图作为本张参考图的一对一替换主体。", @@ -2754,7 +2810,7 @@ export function buildAgentReplacementPrompt(params: { if (params.hasModelImage) { instructionLines.push( - "如有模特图，请在需要人物展示时保持模特体态、穿搭和出镜关系一致；多人参考图要保留参考图人数和互动关系，只把上传模特作为主人物参考，其余人物按参考图风格补齐。" + "固定模特身份硬约束：如果本屏最终画面出现人物，必须使用上传模特图中的同一人物；脸型五官、发型发色、体型肤色和穿搭风格必须保持一致，不得换脸、换人、换发型或改成无关人物。多人参考图要保留参考图人数和互动关系，只把上传模特作为主人物参考，其余人物按参考图风格补齐。" ); } diff --git a/app/(chat)/api/chat/route.ts b/app/(chat)/api/chat/route.ts index 5bd0282ab..b571480c3 100644 --- a/app/(chat)/api/chat/route.ts +++ b/app/(chat)/api/chat/route.ts @@ -45,6 +45,7 @@ import { import { buildAgentSceneImageRolePrompt } from "@/lib/agents/scene-image-role-prompt"; import { optimizeAgentSceneWorkflowPrompt } from "@/lib/agents/scene-workflow-prompt-optimizer"; import { getEnabledTools } from "@/lib/ai/tools-registry"; +import { normalizeCanvasAgentContext } from "@/lib/canvas/agent-context"; import { isProductionEnvironment } from "@/lib/constants"; import { createStreamId, @@ -343,6 +344,8 @@ const chatApiLogger = createLogger("chat.route"); const INTERNAL_IMAGE_GENERATION_FAILURE_PATTERN = /生成图片失败|图片生成失败|未找到图片数据|未解析到图片|模型未返回生成图片|图片数据不可用|未返回可用图片/i; +const DETAIL_REPLACEMENT_MODEL_TEXT_HINT_PATTERN = + /自拍|合照|聚会|聚餐|围坐|用餐|试坐|承重|受力|拿手机|手持|人物场景|人物关系|真人演示|人物展示|人像展示|固定模特|用户模特|same\s+model|fixed\s+model|person|people|human/i; function assertGeneratedImageMarkdownHasImage(markdown: string, modelLabel: string) { const normalizedMarkdown = String(markdown || "").trim(); @@ -365,6 +368,35 @@ function getPublicImageGenerationErrorText(error: unknown) { return message; } +function replacementScenePlanShouldIncludeModelImage(scenePlan: any) { + if (!scenePlan) return false; + if (scenePlan.useModelImage === true) return true; + if ( + scenePlan.peoplePolicy && + scenePlan.peoplePolicy !== "none" + ) { + return true; + } + if (scenePlan.generationRoute === "people_relation_locked") return true; + if (scenePlan.pageKind === "people_scene" || scenePlan.pageKind === "action_poster") { + return true; + } + + const text = [ + scenePlan.modelImageReason, + scenePlan.routeReason, + scenePlan.sceneSummary, + scenePlan.cameraSummary, + scenePlan.placementSummary, + scenePlan.preserveSummary, + scenePlan.styleSummary, + scenePlan.avoidSummary, + scenePlan.generationPrompt, + ] + .filter(Boolean) + .join(" "); + return DETAIL_REPLACEMENT_MODEL_TEXT_HINT_PATTERN.test(text); +} const AGENT_REPLACEMENT_PARALLEL_WORKER_COUNT = Math.max( 1, @@ -721,6 +753,20 @@ export async function POST(request: Request) { .join(" ") .trim(); const autoContextText = typeof autoContext === "string" ? autoContext.trim() : ""; + const requestCanvasAgentContext = normalizeCanvasAgentContext( + contextPayload || undefined + ); + if (requestCanvasAgentContext) { + chatApiLogger.info("[CanvasAgentContext] Normalized request context:", { + source: requestCanvasAgentContext.source, + projectId: requestCanvasAgentContext.projectId || null, + selectedNodeCount: requestCanvasAgentContext.selectedNodes.length, + candidateNodeCount: requestCanvasAgentContext.candidateNodes.length, + slotCount: requestCanvasAgentContext.slots.length, + annotationCount: requestCanvasAgentContext.annotations.length, + focusPolicy: requestCanvasAgentContext.focusPolicy, + }); + } const now = Date.now(); for (const [key, startedAt] of inFlightChatRequests.entries()) { @@ -3465,10 +3511,15 @@ export async function POST(request: Request) { sourceProductImages: productInputImageUrls.length, sourceModelImage: Boolean(modelInputImageUrl), }); + const rerunReferenceImageUrls = (reference.rerunReferenceImages || []) + .map((image) => String(image.url || "").trim()) + .filter((url) => /^https?:\/\//i.test(url)) + .slice(0, 6); const referenceSourceImageUrls = Array.from( new Set([ ...productInputImageUrls, ...(modelInputImageUrl ? [modelInputImageUrl] : []), + ...rerunReferenceImageUrls, reference.imageUrl, ]), ); @@ -3494,7 +3545,9 @@ export async function POST(request: Request) { const shouldIncludeModelImage = detailReplacementStrategy !== "product_only" && Boolean(modelInputImageUrl) && - (replacementExecutionPlan.scenePlan.useModelImage || + (replacementScenePlanShouldIncludeModelImage( + replacementExecutionPlan.scenePlan, + ) || selectedProductContextImageUrls.length === 0); const sourceImageUrls = Array.from( new Set([ @@ -3502,6 +3555,7 @@ export async function POST(request: Request) { ...(shouldIncludeModelImage && modelInputImageUrl ? [modelInputImageUrl] : []), + ...rerunReferenceImageUrls, reference.imageUrl, ]), ); @@ -3515,6 +3569,7 @@ export async function POST(request: Request) { selectedProductImages: selectedProductContextImageUrls.length, sourceProductImages: productInputImageUrls.length, includeModelImage: shouldIncludeModelImage, + rerunReferenceImages: rerunReferenceImageUrls.length, sourceImageCount: sourceImageUrls.length, whiteLayout: replacementExecutionPlan.scenePlan.whiteBackgroundLikely, copyCoupled: replacementExecutionPlan.scenePlan.copyImageCoupled, @@ -3532,6 +3587,7 @@ export async function POST(request: Request) { hasModelImage: shouldIncludeModelImage, productImageCount: selectedProductContextImageUrls.length, originalProductImageCount: productInputImageUrls.length, + rerunReferenceCount: rerunReferenceImageUrls.length, templateReferenceCount: 0, scenePlan: replacementExecutionPlan.scenePlan, referenceGeometry, @@ -4246,10 +4302,15 @@ export async function POST(request: Request) { promptPackPolicy: replacementExecutionPlan.scenePlan.promptPackPolicy, routeReason: replacementExecutionPlan.scenePlan.routeReason, }); + const rerunReferenceImageUrls = (reference.rerunReferenceImages || []) + .map((image) => String(image.url || "").trim()) + .filter((url) => /^https?:\/\//i.test(url)) + .slice(0, 6); const referenceSourceImageUrls = Array.from( new Set([ ...sharedUploadedImageUrls, ...replacementTemplateReferenceImageUrls, + ...rerunReferenceImageUrls, reference.imageUrl, ]), ); @@ -4309,7 +4370,9 @@ export async function POST(request: Request) { !(isDetailReplacementPreset && detailReplacementStrategy === "product_only") && Boolean(modelInputImageUrl) && - (replacementExecutionPlan.scenePlan.useModelImage || + (replacementScenePlanShouldIncludeModelImage( + replacementExecutionPlan.scenePlan, + ) || (isDetailReplacementPreset && selectedProductContextImageUrls.length === 0)); const sourceImageUrls = Array.from( @@ -4320,6 +4383,7 @@ export async function POST(request: Request) { ? [modelInputImageUrl] : []), ...replacementTemplateReferenceImageUrls, + ...rerunReferenceImageUrls, reference.imageUrl, ].filter(Boolean), ), @@ -4343,6 +4407,7 @@ export async function POST(request: Request) { hasModelImage: shouldIncludeModelImage, productImageCount: selectedProductContextImageUrls.length, originalProductImageCount: productInputImageUrls.length, + rerunReferenceCount: rerunReferenceImageUrls.length, templateReferenceCount: replacementTemplateReferenceImageUrls.length, pairedProductIndex, scenePlan: replacementExecutionPlan.scenePlan, @@ -4368,6 +4433,7 @@ export async function POST(request: Request) { originalProductImages: productInputImageUrls.length, hasModelImage: shouldIncludeModelImage, templateReferenceImages: replacementTemplateReferenceImageUrls.length, + rerunReferenceImages: rerunReferenceImageUrls.length, finalReferenceImageIndex: sourceImageUrls.length, }, }); @@ -4475,6 +4541,9 @@ export async function POST(request: Request) { new Set([ ...sharedUploadedImageUrls, ...replacementTemplateReferenceImageUrls, + ...replacementReferences.flatMap((item) => + (item.rerunReferenceImages || []).map((image) => image.url) + ), ...replacementReferences.map((item) => item.imageUrl), ]), ), diff --git a/app/(chat)/api/chat/schema.ts b/app/(chat)/api/chat/schema.ts index 9775e07dd..87226885c 100644 --- a/app/(chat)/api/chat/schema.ts +++ b/app/(chat)/api/chat/schema.ts @@ -1,4 +1,5 @@ import { z } from "zod"; +import type { CanvasAgentContext } from "@/types/canvas/agent-protocol"; const textPartSchema = z.object({ type: z.enum(["text"]), @@ -62,7 +63,8 @@ const contextPayloadSchema = z projectId: z.string().uuid().optional(), selectedNodeIds: z.array(z.string().min(1).max(128)).max(20).optional(), selectedAssetIds: z.array(z.string().uuid()).max(20).optional(), - focusPolicy: z.enum(["auto", "selected", "latest_result", "original"]).optional(), + focusPolicy: z.enum(["auto", "selected", "latest_result", "original", "slot"]).optional(), + canvasAgentContext: z.custom().optional(), canvasImageNodes: z .array( z.object({ @@ -83,6 +85,13 @@ const contextPayloadSchema = z }) .optional(); +const agentModeSourceImageSchema = z.object({ + url: z.string().url(), + label: z.string().max(255).optional(), + name: z.string().max(255).optional(), + assetId: z.string().min(1).max(128).nullable().optional(), +}); + const agentModeReferenceSchema = z.object({ id: z.string().min(1).max(128), label: z.string().min(1).max(255), @@ -91,13 +100,7 @@ const agentModeReferenceSchema = z.object({ sortOrder: z.number().min(0).max(999).optional(), assignedProductIndex: z.number().min(0).max(9).nullable().optional(), passthrough: z.boolean().nullable().optional(), -}); - -const agentModeSourceImageSchema = z.object({ - url: z.string().url(), - label: z.string().max(255).optional(), - name: z.string().max(255).optional(), - assetId: z.string().min(1).max(128).nullable().optional(), + rerunReferenceImages: z.array(agentModeSourceImageSchema).max(6).optional(), }); const agentModePayloadSchema = z diff --git a/app/api/canvas/detail-plan/route.ts b/app/api/canvas/detail-plan/route.ts index e1fe91534..4a3375b8a 100644 --- a/app/api/canvas/detail-plan/route.ts +++ b/app/api/canvas/detail-plan/route.ts @@ -49,6 +49,10 @@ const DETAIL_PLAN_INLINE_RETRIES = 1; const DETAIL_PLAN_MAIN_TIMEOUT_MS = 45000; const DETAIL_PLAN_PREFLIGHT_TIMEOUT_MS = 10000; const logger = createLogger("canvas-detail-plan"); +const DETAIL_MODEL_USAGE_HINT_PATTERN = + /模特|人物|人像|真人|人体|露脸|人脸|面部|发型|体型|肤色|穿搭|服装|试坐|试用|手持|拿着|自拍|合照|真人演示|人物场景|场景演示|使用者|顾客.{0,8}使用|model|person|people|human|face|hair|body|outfit|wearing|holding|selfie|lifestyle\s+demo/i; +const DETAIL_NO_MODEL_HINT_PATTERN = + /不要出现人物|不出现人物|不要有人物|无人|无人物|只展示产品|产品信息展示|不使用人物|不使用模特|禁止出现人物|no\s+(human|person|people)|without\s+(human|person|people)|product[-\s]?only/i; function isAbortError(error: unknown): boolean { const target = error as { @@ -271,6 +275,77 @@ type DetailPlanResponse = { }; }; +function buildFixedModelIdentityInstruction(language: "zh" | "en") { + return language === "en" + ? "Fixed model identity lock: if this frame includes a person, use only the same person from the uploaded model image. Keep face, hairstyle, body type, skin tone, and outfit style consistent; do not replace with another person. Pose and camera angle may adapt to the composition." + : "固定模特身份硬约束：如果本屏出现人物，只允许使用上传模特图中的同一人物；脸型五官、发型发色、体型肤色和穿搭风格必须保持一致，不得换成其他人。姿势、视线和机位可按构图调整。"; +} + +function screenSuggestsModelUsage(screen: DetailPlanScreen) { + const text = [ + screen.objective, + screen.title, + screen.subtitle, + screen.copy, + screen.layoutNote, + screen.modelReason, + screen.visualPrompt, + screen.fabe?.b, + screen.fabe?.f, + screen.fabe?.a, + screen.fabe?.e, + ] + .filter(Boolean) + .join(" "); + return ( + DETAIL_MODEL_USAGE_HINT_PATTERN.test(text) && + !DETAIL_NO_MODEL_HINT_PATTERN.test(text) + ); +} + +function normalizeDetailModelUsage(params: { + screens: DetailPlanScreen[]; + hasModelImage: boolean; + language: "zh" | "en"; +}) { + const { screens, hasModelImage, language } = params; + const identityInstruction = buildFixedModelIdentityInstruction(language); + return screens.map((screen) => { + if (!hasModelImage) { + return { + ...screen, + useModel: false, + modelReason: + language === "en" + ? "No model image was provided, so this frame does not use a fixed model." + : "未提供模特图，当前分镜不使用人物。", + }; + } + + const shouldUseModel = screen.useModel || screenSuggestsModelUsage(screen); + if (!shouldUseModel) { + return screen; + } + + const modelReason = + screen.modelReason && !DETAIL_NO_MODEL_HINT_PATTERN.test(screen.modelReason) + ? screen.modelReason + : language === "en" + ? "This frame contains a human/model scenario, so the uploaded fixed model must be used." + : "本屏包含人物/模特展示语义，必须使用上传的固定模特图。"; + const visualPrompt = screen.visualPrompt.includes(identityInstruction) + ? screen.visualPrompt + : `${screen.visualPrompt} ${identityInstruction}`; + + return { + ...screen, + useModel: true, + modelReason, + visualPrompt, + }; + }); +} + type DetailPlanPreflight = { passed: boolean; designDirectorScore: number; @@ -1230,9 +1305,13 @@ function buildFallbackPlan(params: { ...params, globalStyle, }); - const screens = enforceCoreFocusOnScreens({ - screens: fallbackScreens, - coreFocus: params.coreFocus, + const screens = normalizeDetailModelUsage({ + screens: enforceCoreFocusOnScreens({ + screens: fallbackScreens, + coreFocus: params.coreFocus, + language, + }), + hasModelImage: params.hasModelImage, language, }); const qualityChecks = evaluatePlanQuality({ @@ -2181,27 +2260,28 @@ export async function POST(request: Request) { dslV2Screens?: unknown; }>(text || ""); const globalStyleInitial = sanitizeGlobalStyle(parsed?.globalStyle, fallback.globalStyle); - const rawScreens = sanitizePlannedScreens(parsed?.screens, fallback.screens, language).map((item) => - hasModelImage - ? item - : { - ...item, - useModel: false, - modelReason: "未提供模特图，当前分镜不使用人物。", - } - ); - const normalizedScreens = rawScreens.map((item) => + const rawScreens = sanitizePlannedScreens(parsed?.screens, fallback.screens, language); + const normalizedScreensWithoutBundles = rawScreens.map((item) => hasBundleImages ? item : { ...item, bundleShowcase: false, - bundleReason: "未提供搭配产品图，当前分镜非搭配展示页。", + bundleReason: "未提供搭配产品图，当前分镜非搭配展示页。", } ); - const coreFocusedScreens = enforceCoreFocusOnScreens({ - screens: normalizedScreens, - coreFocus, + const normalizedScreens = normalizeDetailModelUsage({ + screens: normalizedScreensWithoutBundles, + hasModelImage, + language, + }); + const coreFocusedScreens = normalizeDetailModelUsage({ + screens: enforceCoreFocusOnScreens({ + screens: normalizedScreens, + coreFocus, + language, + }), + hasModelImage, language, }); const qualityChecksInitial = evaluatePlanQuality({ @@ -2225,7 +2305,7 @@ export async function POST(request: Request) { hasBundleImages, }); - const preflightScreensRaw = preflight.candidateScreens.map((item) => + const preflightScreensWithoutBundles = preflight.candidateScreens.map((item) => hasBundleImages ? item : { @@ -2234,9 +2314,18 @@ export async function POST(request: Request) { bundleReason: "未提供搭配产品图，当前分镜非搭配展示页。", } ); - const preflightScreens = enforceCoreFocusOnScreens({ - screens: preflightScreensRaw, - coreFocus, + const preflightScreensRaw = normalizeDetailModelUsage({ + screens: preflightScreensWithoutBundles, + hasModelImage, + language, + }); + const preflightScreens = normalizeDetailModelUsage({ + screens: enforceCoreFocusOnScreens({ + screens: preflightScreensRaw, + coreFocus, + language, + }), + hasModelImage, language, }); const qualityChecksAfterPreflight = evaluatePlanQuality({ diff --git a/app/api/canvas/local-edit/route.ts b/app/api/canvas/local-edit/route.ts index 6bd1b7d5e..52b9da060 100644 --- a/app/api/canvas/local-edit/route.ts +++ b/app/api/canvas/local-edit/route.ts @@ -87,7 +87,7 @@ const SCALE_ADJUST_CROP_MAX_EDGE = 1536; const MIN_REGION_EDGE_PX = 24; const SCALE_ADJUST_OPERATION = "scale-adjust"; type LocalEditOperation = "local-edit" | typeof SCALE_ADJUST_OPERATION; -const SCALE_ADJUST_PERCENT_OPTIONS = [-30, -20, -10, 10, 20, 30] as const; +const SCALE_ADJUST_PERCENT_OPTIONS = [-30, -20, -10, 10, 20, 30, 50, 100] as const; type ScaleAdjustPercent = (typeof SCALE_ADJUST_PERCENT_OPTIONS)[number]; type ScaleAdjustCropReference = { regionIndex: number; @@ -385,14 +385,59 @@ function scaleRectAroundCenter( }; } +function expandRectWithMargins( + rect: NormalizedLocalEditRegion["rect"], + marginX: number, + marginY: number +) { + const left = clamp(rect.x - marginX, 0, 1); + const top = clamp(rect.y - marginY, 0, 1); + const right = clamp(rect.x + rect.width + marginX, left + 0.001, 1); + const bottom = clamp(rect.y + rect.height + marginY, top + 0.001, 1); + + return { + x: left, + y: top, + width: Math.max(0.001, right - left), + height: Math.max(0.001, bottom - top), + }; +} + +function getScaleAdjustWorkspaceRect( + sourceRect: NormalizedLocalEditRegion["rect"], + targetRect: NormalizedLocalEditRegion["rect"], + scalePercent: ScaleAdjustPercent +) { + if (scalePercent <= 0) return targetRect; + + const scaleRatio = Math.min(1, Math.max(0, scalePercent / 100)); + const marginX = Math.min( + 0.12, + Math.max(0.018, sourceRect.width * scaleRatio * 0.22) + ); + const marginY = Math.min( + 0.12, + Math.max(0.018, sourceRect.height * scaleRatio * 0.22) + ); + + return expandRectWithMargins(targetRect, marginX, marginY); +} + function buildScaleAdjustCompositeRegions( regions: NormalizedLocalEditRegion[], scalePercent: ScaleAdjustPercent ) { - return regions.flatMap((region) => [ - region.rect, - scaleRectAroundCenter(region.rect, scalePercent), - ]); + return regions.flatMap((region) => { + const targetRect = scaleRectAroundCenter(region.rect, scalePercent); + const workspaceRect = getScaleAdjustWorkspaceRect( + region.rect, + targetRect, + scalePercent + ); + return scalePercent > 0 + ? [region.rect, targetRect, workspaceRect] + : [region.rect, targetRect]; + }); } function expandPixelRectForCrop(params: { @@ -704,7 +749,7 @@ function buildScaleAdjustPrompt( cropIndex >= 0 ? `；输入图片${cropIndex + 3}是该区域的干净局部裁切参考图，只用于识别目标主体的身份、轮廓与原始比例` : ""; - return `区域 ${region.index}${cropNote}：请在输入图片1中找到输入图片2红色实线框所标记范围内最主要、可独立缩放的商品/物体主体。红色实线框只是“寻找主体”的范围，不是要缩放或贴回的矩形补丁；目标虚线框只是表达目标尺寸范围，不是要求主体填满矩形或变成矩形比例。请以主体自身轮廓、原始宽高比和视觉中心为准，执行整体等比${actionLabel}，宽度与高度必须使用同一个缩放倍率，目标主体视觉尺寸约为原主体的 ${targetLabel}。严禁宽度变窄但高度变高，严禁横向挤压、纵向拉伸或改变主体透视形体。如果框中只覆盖到主体的一部分，请根据连接关系推断该主体的完整可见结构，只调整该主体，不调整矩形框内背景。`; + return `区域 ${region.index}${cropNote}：请在输入图片1中找到输入图片2红色实线框所标记范围内最主要、可独立缩放的商品/物体主体。红色实线框只是“寻找主体”的范围，不是放大后的硬边界，也不是要缩放或贴回的矩形补丁；放大时主体可以自然超出红色实线框，并由周边背景补全衔接。目标虚线框只是表达目标尺寸范围，不是要求主体填满矩形或变成矩形比例。请以主体自身轮廓、原始宽高比和视觉中心为准，执行整体等比${actionLabel}，宽度与高度必须使用同一个缩放倍率，目标主体视觉尺寸约为原主体的 ${targetLabel}。严禁宽度变窄但高度变高，严禁横向挤压、纵向拉伸或改变主体透视形体。如果框中只覆盖到主体的一部分，请根据连接关系推断该主体的完整可见结构，只调整该主体，不调整矩形框内背景。`; }) .join("\n"); const cropLines = cropReferences @@ -722,9 +767,9 @@ function buildScaleAdjustPrompt( cropLines ? `输入图片3及之后是框选区域的干净局部裁切参考图，作用是帮助你确认“要缩放的是哪个主体”。这些裁切图不是输出底图，不得被整体放大、贴回或覆盖到原图上。\n${cropLines}` : "", - "输入图片2中的红色实线框表示输入图片1里目标主体当前大致位置；青绿色或绿色虚线框只表示比例调整后的安全尺寸范围。最终画面不得出现这些框线，主体也不需要填满虚线矩形；必须保持主体自身原始宽高比和透视形体。", + "输入图片2中的红色实线框表示输入图片1里目标主体当前大致位置；它只用于定位主体，不限制放大后的空间。青绿色或绿色虚线框只表示比例调整后的安全尺寸范围。最终画面不得出现这些框线，主体也不需要填满虚线矩形；必须保持主体自身原始宽高比和透视形体。", "如果系统或模型只能读取一张参考图，必须优先以输入图片1为输出基底，不得清理、改写或删除输入图片1中框选区域外的任何文字和图形。", - "系统已根据用户框选区域计算出下方每个区域的原图坐标；框选区域只是定位目标主体的搜索范围，不是最终画面的裁切块、贴片、蒙版边界，也不是要整体缩放的矩形图层。", + "系统已根据用户框选区域计算出下方每个区域的原图坐标；框选区域只是定位目标主体的搜索范围，不是最终画面的裁切块、贴片、蒙版边界，也不是放大后的空间上限，更不是要整体缩放的矩形图层。", "最终结果必须保持原始干净图的画幅比例、构图边界、镜头透视、光线、色温、材质质感和清晰度；严禁整张图整体缩放、裁切、平移、重新排版或重新生成。", "Highest priority rule: do NOT zoom, crop, resize, reframe, pan, or regenerate the full canvas. Treat image 1 as a locked background plate and only inpaint the selected subject footprint.", "请不要生成任何彩色框、矩形边框、编号圆点、提示词气泡、标注文字、UI 控件或半透明色块。", @@ -737,7 +782,7 @@ function buildScaleAdjustPrompt( "2. 严禁把整个矩形框选区域、输入图片1整张图或任意裁切图当成图片块整体缩放、平移、复制、裁切或贴回；最终画面不得出现矩形拼接边缘、色差硬边、补丁感、裁切线或局部贴片痕迹。", "2.1 严禁把输入图片1整张图整体放大、缩小、裁切或重新构图；如果画面中的标题、Logo、页面文字、边框、背景家具或非目标主体发生整体位移/缩放/消失，必须视为失败并重新生成。", `3. 缩小时，目标主体应围绕原位置视觉中心自然变小到约 ${targetLabel}；主体缩小后，原来被遮挡的背景、桌面、墙面、阴影、纹理和透视必须被重新生成并无缝补全，选区内应出现明显新增留白或背景。`, - `4. 放大时，目标主体应围绕原位置视觉中心自然变大到约 ${targetLabel}；宽度和高度都要同步变大，不能出现“高度变高但宽度变窄”的非等比变形；主体边缘与遮挡关系要合理，不能产生突兀裁切。`, + `4. 放大时，目标主体应围绕原位置视觉中心自然变大到约 ${targetLabel}；允许主体自然超出用户手画的原始红框，使用周围真实背景完成边缘、阴影和遮挡衔接。宽度和高度都要同步变大，不能出现“高度变高但宽度变窄”的非等比变形；主体边缘与遮挡关系要合理，不能产生突兀裁切。`, "5. 如果主体靠近画布边界导致目标尺寸空间不足，请用能完整放入画面的最大统一倍率整体缩放；绝不能为了塞进画面而只压缩宽度或只拉伸高度。", "6. 框选区域内非目标主体的背景、墙面、桌面、植物、文字和其它物体应保持原场景逻辑，不要跟着主体一起缩放。", "6.1 比例调整后的主体边缘必须和周围环境自然融合，选区边界不能被肉眼看出；不得出现一整块矩形区域清晰度、颜色、光照、纹理或透视突然变化。", diff --git a/components/agents/AgentGenerationWorkspaceDarkV2.tsx b/components/agents/AgentGenerationWorkspaceDarkV2.tsx index af9fe2203..351cb3700 100644 --- a/components/agents/AgentGenerationWorkspaceDarkV2.tsx +++ b/components/agents/AgentGenerationWorkspaceDarkV2.tsx @@ -247,7 +247,7 @@ type DetailResultLocalEditTarget = { }; type DetailResultLocalEditMode = "local_edit" | "scale_adjust"; -type DetailResultScalePercent = -30 | -20 | -10 | 10 | 20 | 30; +type DetailResultScalePercent = -30 | -20 | -10 | 10 | 20 | 30 | 50 | 100; type DetailResultLocalEditRect = { x: number; @@ -290,6 +290,7 @@ const MAX_CHAT_ATTACHMENT_NAME_LENGTH = 100; const MAX_AGENT_MODE_QUERY_LENGTH = 64; const MAX_AGENT_MODE_REFERENCE_COUNT = 40; const DETAIL_LOCAL_EDIT_REFERENCE_MAX_UPLOAD_BYTES = 30 * 1024 * 1024; +const REPLACEMENT_RERUN_MAX_REFERENCE_IMAGES = 4; const DETAIL_SCALE_ADJUST_OPTIONS: DetailResultScalePercent[] = [ -30, -20, @@ -297,6 +298,8 @@ const DETAIL_SCALE_ADJUST_OPTIONS: DetailResultScalePercent[] = [ 10, 20, 30, + 50, + 100, ]; const DETAIL_LOCAL_EDIT_MIN_RECT_RATIO = 0.025; const DIRECT_MODEL_UPLOAD_SLOT_COUNT = 4; @@ -860,6 +863,7 @@ function getDetailScaleAdjustRegionPrompt(scalePercent: DetailResultScalePercent return [ `对框选范围内的主物体执行等比${actionLabel}，调整后主体视觉尺寸必须约为原来的 ${targetLabel}，需要肉眼明显看出已经${scaleVerb}。`, "必须锁定主体自身原始宽高比和透视形体，宽度与高度用同一个倍率同步变化；严禁宽度变窄但高度变高、严禁横向挤压或纵向拉伸。", + "框选区域只用于识别要调整的主体，不是放大后的边界；放大时主体可以自然超出原始框选范围，并由周边背景补全衔接。", "只改变目标主体的尺寸，不改变主体外观、颜色、材质、图案和文字内容；不要把整个框选矩形当成图片块缩放。", scalePercent < 0 ? "主体缩小后露出的背景、桌面、墙面、阴影和遮挡关系必须自然补全。" @@ -1475,7 +1479,7 @@ function DetailResultLocalEditDialog({
Scale
-
+
{DETAIL_SCALE_ADJUST_OPTIONS.map((option) => { const active = selectedScalePercent === option; const Icon = option > 0 ? Maximize2 : Minimize2; @@ -2035,6 +2039,23 @@ interface ReplacementRerunTarget { title: string; } +type ReplacementRerunReferenceImage = UploadedDetailReferenceImage & { + id: string; +}; + +type ReplacementRerunRequestReferenceImage = { + assetId?: string | null; + label?: string; + name?: string; + url: string; +}; + +type ReplacementRerunRequestSnapshot = { + cardId: string; + images: ReplacementRerunRequestReferenceImage[]; + modeType: ReplacementModeType; +} | null; + type ReplacementResultState = Record< ReplacementModeType, Record @@ -2729,9 +2750,11 @@ export function AgentGenerationWorkspaceDarkV2({ const generationRequestSnapshotRef = useRef<{ hiddenPrompt: string | null; replacementExecutionCardIds: string[] | null; + replacementRerunReferenceImages: ReplacementRerunRequestSnapshot; }>({ hiddenPrompt: null, replacementExecutionCardIds: null, + replacementRerunReferenceImages: null, }); const [localAgentModeState, setLocalAgentModeState] = useState( () => createEmptyAgentModeWorkspaceState() @@ -2858,6 +2881,10 @@ export function AgentGenerationWorkspaceDarkV2({ const [replacementRerunTarget, setReplacementRerunTarget] = useState(null); const [replacementRerunPrompt, setReplacementRerunPrompt] = useState(""); + const [replacementRerunReferenceImages, setReplacementRerunReferenceImages] = + useState([]); + const [replacementRerunReferenceUploading, setReplacementRerunReferenceUploading] = + useState(false); const [detailWatermarkRemoveCardIds, setDetailWatermarkRemoveCardIds] = useState>(() => new Set()); const processedResultHintTimerRef = useRef>({}); @@ -3684,6 +3711,8 @@ export function AgentGenerationWorkspaceDarkV2({ .map((item) => String(item || "").trim()) .filter(Boolean) ); + const replacementRerunReferenceSnapshot = + generationRequestSnapshotRef.current.replacementRerunReferenceImages; const resolvedAgentPresetId = requiresPresetSelection ? transportState.selectedAgentPresetId || selectedAgentPresetId || undefined @@ -3726,23 +3755,42 @@ export function AgentGenerationWorkspaceDarkV2({ requestedReplacementCardIdSet.has(String(item.id || "").trim()) ) .filter((item) => Boolean(String(item.imageUrl || "").trim())) - .map((item) => ({ - id: String(item.id || "").slice(0, 128), - label: String(item.label || "").slice(0, 255), - imageUrl: String(item.imageUrl || "").trim(), - candidateId: item.candidateId - ? String(item.candidateId).slice(0, 128) - : null, - sortOrder: item.sortOrder ?? 0, - passthrough: item.passthrough === true, - assignedProductIndex: - transportState.selectedProductAttachmentCount > 0 - ? Math.min( - item.assignedProductIndex ?? 0, - Math.max(transportState.selectedProductAttachmentCount - 1, 0) - ) + .map((item) => { + const itemId = String(item.id || "").trim(); + const rerunReferenceImages = + replacementRerunReferenceSnapshot && + replacementRerunReferenceSnapshot.modeType === + (transportState.isDetailReplacementPreset + ? "detail_replacement" + : "sku_replacement") && + replacementRerunReferenceSnapshot.cardId === itemId + ? replacementRerunReferenceSnapshot.images + : []; + return { + id: itemId.slice(0, 128), + label: String(item.label || "").slice(0, 255), + imageUrl: String(item.imageUrl || "").trim(), + candidateId: item.candidateId + ? String(item.candidateId).slice(0, 128) : null, - })), + sortOrder: item.sortOrder ?? 0, + passthrough: item.passthrough === true, + assignedProductIndex: + transportState.selectedProductAttachmentCount > 0 + ? Math.min( + item.assignedProductIndex ?? 0, + Math.max(transportState.selectedProductAttachmentCount - 1, 0) + ) + : null, + rerunReferenceImages: + rerunReferenceImages.length > 0 + ? rerunReferenceImages.slice( + 0, + REPLACEMENT_RERUN_MAX_REFERENCE_IMAGES + ) + : undefined, + }; + }), sourceImages: transportState.isDetailReplacementPreset ? { productImages: transportState.detailReplacementProductSourceImages, @@ -5472,6 +5520,7 @@ export function AgentGenerationWorkspaceDarkV2({ replacementModeType?: ReplacementModeType; replacementCardIds?: string[] | null; promptText?: string | null; + replacementRerunReferenceImages?: ReplacementRerunRequestSnapshot; }) => { if (isSelectedBusinessModeUnavailable || activeBusinessModeKey === "buyer_show") { toast.error("当前生图模式暂未开放"); @@ -5634,6 +5683,8 @@ export function AgentGenerationWorkspaceDarkV2({ replacementModeType && replacementExecutionCards.length > 0 ? replacementExecutionCards.map((card) => card.id) : null, + replacementRerunReferenceImages: + options?.replacementRerunReferenceImages || null, }; setReplacementExecutionContext( replacementModeType && replacementExecutionCards.length > 0 @@ -5675,6 +5726,7 @@ export function AgentGenerationWorkspaceDarkV2({ generationRequestSnapshotRef.current = { hiddenPrompt: null, replacementExecutionCardIds: null, + replacementRerunReferenceImages: null, }; setPendingGeneration(null); toast.error(error instanceof Error ? error.message : "发送生成请求失败"); @@ -5722,6 +5774,7 @@ export function AgentGenerationWorkspaceDarkV2({ generationRequestSnapshotRef.current = { hiddenPrompt: hiddenPromptRef.current, replacementExecutionCardIds: null, + replacementRerunReferenceImages: null, }; setPendingGeneration(null); setMessages((current) => current); @@ -5753,19 +5806,124 @@ export function AgentGenerationWorkspaceDarkV2({ : current ); }, [handleUpdateReplacementResultRecord, replacementTextEditTarget]); + const handleReplacementRerunReferenceUpload = useCallback( + async (event: React.ChangeEvent) => { + const files = Array.from(event.target.files || []); + event.currentTarget.value = ""; + if (files.length === 0) return; + + const remainingSlots = Math.max( + 0, + REPLACEMENT_RERUN_MAX_REFERENCE_IMAGES - + replacementRerunReferenceImages.length + ); + if (remainingSlots <= 0) { + toast.error( + `重新生图最多添加 ${REPLACEMENT_RERUN_MAX_REFERENCE_IMAGES} 张参考图` + ); + return; + } + + const targetFiles = files.slice(0, remainingSlots); + if (files.length > targetFiles.length) { + toast.info( + `已自动保留前 ${REPLACEMENT_RERUN_MAX_REFERENCE_IMAGES} 张参考图` + ); + } + + setReplacementRerunReferenceUploading(true); + try { + const uploaded = await Promise.all( + targetFiles.map(async (file) => { + const upload = await uploadDetailReferenceImageFile(file); + return { + ...upload, + id: `replacement-rerun-reference-${Date.now()}-${Math.random() + .toString(36) + .slice(2, 8)}`, + }; + }) + ); + setReplacementRerunReferenceImages((current) => [ + ...current, + ...uploaded, + ]); + toast.success( + uploaded.length > 1 + ? `已添加 ${uploaded.length} 张重生参考图` + : "已添加重生参考图" + ); + } catch (error) { + toast.error(error instanceof Error ? error.message : "参考图上传失败"); + } finally { + setReplacementRerunReferenceUploading(false); + } + }, + [replacementRerunReferenceImages.length] + ); + + const handleRemoveReplacementRerunReferenceImage = useCallback((id: string) => { + setReplacementRerunReferenceImages((current) => + current.filter((image) => image.id !== id) + ); + }, []); + const handleConfirmReplacementRerun = useCallback(() => { if (!replacementRerunTarget) { return; } + const userPrompt = replacementRerunPrompt.trim(); + if (!userPrompt) { + toast.error("请先填写本次重新生图的调整要求"); + return; + } + if (replacementRerunReferenceUploading) { + toast.info("参考图上传中，请稍候"); + return; + } + + const rerunReferenceImages = replacementRerunReferenceImages + .map((image, index) => ({ + assetId: image.assetId || null, + label: `重新生图补充参考图 ${index + 1}`, + name: image.name || `重新生图参考图 ${index + 1}`, + url: image.url, + })) + .filter((image) => /^https?:\/\//i.test(image.url)) + .slice(0, REPLACEMENT_RERUN_MAX_REFERENCE_IMAGES); + const promptText = [ + "【单张重新生图】请重新生成当前这张结果图，必须根据本次补充要求产生可见变化，避免返回与上一版几乎相同的图片。", + `本次补充要求：${userPrompt}`, + rerunReferenceImages.length > 0 + ? `本次额外提供 ${rerunReferenceImages.length} 张补充参考图；它们只用于理解风格、细节、构图或局部调整方向，不得覆盖当前目标参考图的版式边界和文字层级。` + : "", + ] + .filter(Boolean) + .join("\n"); handleSend({ replacementModeType: replacementRerunTarget.modeType, replacementCardIds: [replacementRerunTarget.cardId], - promptText: replacementRerunPrompt, + promptText, + replacementRerunReferenceImages: + rerunReferenceImages.length > 0 + ? { + cardId: replacementRerunTarget.cardId, + images: rerunReferenceImages, + modeType: replacementRerunTarget.modeType, + } + : null, }); setReplacementRerunTarget(null); setReplacementRerunPrompt(""); - }, [handleSend, replacementRerunPrompt, replacementRerunTarget]); + setReplacementRerunReferenceImages([]); + }, [ + handleSend, + replacementRerunPrompt, + replacementRerunReferenceImages, + replacementRerunReferenceUploading, + replacementRerunTarget, + ]); const handleDownloadOriginalImage = useCallback(async (imageUrl: string) => { const raw = String(imageUrl || "").trim(); @@ -6952,7 +7110,7 @@ export function AgentGenerationWorkspaceDarkV2({ ? "首屏封面规则：必须做成海报主视觉，单一主体强焦点，产品主体占画面55%-70%，顶部标题区清晰留白，禁止拼贴式说明图。" : "", useModelForScene - ? "人物规则：本屏涉及人物场景，固定使用提供的模特图同一人物（面部、发型、体型、服装风格保持一致）。" + ? "人物规则：本屏涉及人物场景，必须固定使用提供的模特图同一人物；脸型五官、发型发色、体型肤色、穿搭风格保持一致，不得换脸、换人、换发型或改成无关人物。" : "人物规则：本屏不要出现人物或人体部位，只展示产品与场景元素。", screen?.modelReason ? `人物融入依据：${screen.modelReason}` : "", bundleShowcase @@ -9426,6 +9584,7 @@ export function AgentGenerationWorkspaceDarkV2({ title: `${detailReferenceSourceLabel}结果图 ${card.index + 1}`, }); setReplacementRerunPrompt(""); + setReplacementRerunReferenceImages([]); }} className={cn(IMAGE_OVERLAY_ACTION_CLASS, "disabled:cursor-not-allowed disabled:opacity-45")} > @@ -9971,6 +10130,7 @@ export function AgentGenerationWorkspaceDarkV2({ title: `${skuReferenceSourceLabel}结果图 ${card.index + 1}`, }); setReplacementRerunPrompt(""); + setReplacementRerunReferenceImages([]); }} className={cn(IMAGE_OVERLAY_ACTION_CLASS, "disabled:cursor-not-allowed disabled:opacity-45")} > @@ -10544,6 +10704,7 @@ export function AgentGenerationWorkspaceDarkV2({ if (!open) { setReplacementRerunTarget(null); setReplacementRerunPrompt(""); + setReplacementRerunReferenceImages([]); } }} > @@ -10551,7 +10712,7 @@ export function AgentGenerationWorkspaceDarkV2({ 重新生图 - 当前会只重做这一张结果图，不会把已经完成的其它结果重新跑一遍。 + 当前会只重做这一张结果图。请填写本次调整要求，也可以添加补充参考图帮助模型拉开差异。
@@ -10566,9 +10727,86 @@ export function AgentGenerationWorkspaceDarkV2({ value={replacementRerunPrompt} onChange={(event) => setReplacementRerunPrompt(event.target.value)} className="min-h-[140px] resize-y" - placeholder="例如：标题更有冲击力、背景更明亮、保留版式但优化卖点文案层级..." + placeholder="例如：背景更明亮，产品占比更大，标题更有冲击力，保留版式但优化卖点文案层级..." />
+
+
+
+ 补充参考图 +
+ + {replacementRerunReferenceImages.length}/ + {REPLACEMENT_RERUN_MAX_REFERENCE_IMAGES} + +
+ { + void handleReplacementRerunReferenceUpload(event); + }} + type="file" + /> + + {replacementRerunReferenceImages.length > 0 ? ( +
+ {replacementRerunReferenceImages.map((image) => ( +
+ {image.name + +
+ ))} +
+ ) : ( +
+ 可选。参考图只作为本次重生的风格、细节或构图补充，不会加入长期参考图列表。 +
+ )} +
- diff --git a/components/canvas/CanvasChatPanel.tsx b/components/canvas/CanvasChatPanel.tsx index 97813d63d..4fd4c0b59 100644 --- a/components/canvas/CanvasChatPanel.tsx +++ b/components/canvas/CanvasChatPanel.tsx @@ -78,6 +78,13 @@ import type { AgentWorkflowComposerContext, } from "@/types/agent"; import type { CanvasNode } from "@/types/canvas"; +import type { + CanvasAgentAnnotationRef, + CanvasAgentContext, + CanvasAgentNodeRef, + CanvasAgentSlotRef, + CanvasAgentNodeType, +} from "@/types/canvas/agent-protocol"; import { type CanvasLaunchPayload, getCanvasLaunchStorageKey } from "@/lib/canvas-launch"; import { DEFAULT_CANVAS_SKILL_TAGS_TITLE, @@ -1619,6 +1626,7 @@ export function CanvasChatPanel({ const nodes = useCanvasStore((state) => state.nodes); const projectTitle = useCanvasStore((state) => state.projectTitle); const selectedNodeIds = useCanvasStore((state) => state.selectedNodeIds); + const viewport = useCanvasStore((state) => state.viewport); const selectNodes = useCanvasStore((state) => state.selectNodes); const setViewport = useCanvasStore((state) => state.setViewport); const clearSelection = useCanvasStore((state) => state.clearSelection); @@ -5325,17 +5333,13 @@ export function CanvasChatPanel({ }, [nodes, selectedImageNodes]); const buildContextPayload = useCallback(() => { + const selectedNodeIdSet = new Set(selectedNodeIds); const imageNodes = nodes.filter( (node) => node.type === "image" && isValidChatHttpUrl(String((node.data as any)?.src || "").trim()), ); - if (imageNodes.length === 0) { - generationAnchorNodesRef.current = []; - return undefined; - } - const effectiveComposerMode = activeComposerModeRef.current || activeComposerMode; const shouldIsolateDirectModeContext = effectiveComposerMode !== "agent"; const selectedImageNodesForContext = selectedImageNodes.filter((node) => @@ -5348,6 +5352,10 @@ export function CanvasChatPanel({ } const MAX_CONTEXT_IMAGES = 12; + const MAX_AGENT_CANDIDATE_NODES = 30; + const selectedCanvasNodes = nodes + .filter((node) => selectedNodeIdSet.has(node.id)) + .slice(0, 20); const selectedContextNodes = selectedImageNodesForContext.slice(0, 20); generationAnchorNodesRef.current = selectedContextNodes.slice(0, 4); const selectedIds = new Set(selectedContextNodes.map((node) => node.id)); @@ -5361,6 +5369,13 @@ export function CanvasChatPanel({ ] : imageNodes; const contextNodes = prioritizedNodes.slice(0, MAX_CONTEXT_IMAGES); + const candidateSourceNodes = [ + ...selectedCanvasNodes, + ...contextNodes.filter((node) => !selectedNodeIdSet.has(node.id)), + ]; + const uniqueCandidateSourceNodes = Array.from( + new Map(candidateSourceNodes.map((node) => [node.id, node])).values() + ).slice(0, MAX_AGENT_CANDIDATE_NODES); const selectedAssetIds = Array.from( new Set( selectedContextNodes @@ -5372,12 +5387,210 @@ export function CanvasChatPanel({ ), ); + const projectId = isValidChatUuid(canvasId) ? canvasId : undefined; + const toAgentNodeRef = (node: CanvasNode): CanvasAgentNodeRef | null => { + const data = (node.data || {}) as Record; + const type = String(node.type) as CanvasAgentNodeType; + const metadata = + data.metadata && typeof data.metadata === "object" && !Array.isArray(data.metadata) + ? data.metadata + : undefined; + const srcCandidate = + typeof data.src === "string" + ? data.src + : typeof data.previewImage === "string" + ? data.previewImage + : typeof data.imageUrl === "string" + ? data.imageUrl + : typeof data.worldUrl === "string" + ? data.worldUrl + : typeof data.renderArtifacts?.screenshotUrl === "string" + ? data.renderArtifacts.screenshotUrl + : ""; + const url = isValidChatHttpUrl(String(srcCandidate || "").trim()) + ? String(srcCandidate).trim() + : undefined; + const titleCandidate = + typeof data.title === "string" + ? data.title + : typeof data.content === "string" + ? data.content + : typeof data.prompt === "string" + ? data.prompt + : undefined; + + return { + nodeId: node.id, + type, + assetId: isValidChatUuid(data.assetId) ? String(data.assetId).trim() : undefined, + url, + title: + typeof titleCandidate === "string" + ? String(titleCandidate).trim().slice(0, 256) + : undefined, + prompt: + typeof data.prompt === "string" + ? String(data.prompt).trim().slice(0, 2000) + : undefined, + x: Number(node.position.x || 0), + y: Number(node.position.y || 0), + width: Number(node.size?.width || 0) || undefined, + height: Number(node.size?.height || 0) || undefined, + isSelected: selectedNodeIdSet.has(node.id), + metadata, + }; + }; + const toAgentSlotRef = (node: CanvasNode): CanvasAgentSlotRef | null => { + if (String(node.type) !== "agent-slot") return null; + const data = (node.data || {}) as Record; + const slotType = + data.slotType === "video" || data.slotType === "ui-screen" ? data.slotType : "image"; + const fillPolicy = + data.fillPolicy === "append-version" || data.fillPolicy === "replace-existing" + ? data.fillPolicy + : "replace-empty"; + return { + nodeId: node.id, + slotType, + label: + typeof data.label === "string" ? String(data.label).slice(0, 128) : undefined, + x: Number(node.position.x || 0), + y: Number(node.position.y || 0), + width: Number(node.size?.width || 0) || 1, + height: Number(node.size?.height || 0) || 1, + aspectRatio: + typeof data.aspectRatio === "string" ? String(data.aspectRatio).slice(0, 32) : "1:1", + isSelected: selectedNodeIdSet.has(node.id), + fillPolicy, + acceptedSources: Array.isArray(data.acceptedSources) + ? data.acceptedSources + .filter( + (value: unknown): value is "text" | "image" | "selection" | "annotation" => + value === "text" || + value === "image" || + value === "selection" || + value === "annotation" + ) + .slice(0, 8) + : undefined, + }; + }; + const toAgentAnnotations = (node: CanvasNode): CanvasAgentAnnotationRef[] => { + const metadata = ((node.data as any)?.metadata || {}) as Record; + const annotations = Array.isArray(metadata.annotations) ? metadata.annotations : []; + return annotations + .map((annotation, index): CanvasAgentAnnotationRef | null => { + if (!annotation || typeof annotation !== "object") return null; + const type = String(annotation.type || "text-note"); + if ( + type !== "text-note" && + type !== "arrow-note" && + type !== "box" && + type !== "freehand" && + type !== "crop-box" + ) { + return null; + } + const bounds = + annotation.bounds && typeof annotation.bounds === "object" + ? { + x: Number(annotation.bounds.x || 0), + y: Number(annotation.bounds.y || 0), + width: Number(annotation.bounds.width || 0), + height: Number(annotation.bounds.height || 0), + } + : undefined; + const targetPoint = + annotation.targetPoint && typeof annotation.targetPoint === "object" + ? { + x: Number(annotation.targetPoint.x || 0), + y: Number(annotation.targetPoint.y || 0), + } + : undefined; + return { + id: String(annotation.id || `${node.id}-annotation-${index}`).slice(0, 128), + sourceNodeId: + typeof annotation.sourceNodeId === "string" + ? String(annotation.sourceNodeId).slice(0, 128) + : node.id, + type: type as CanvasAgentAnnotationRef["type"], + text: + typeof annotation.text === "string" + ? String(annotation.text).slice(0, 1000) + : undefined, + bounds, + targetPoint, + color: + typeof annotation.color === "string" + ? String(annotation.color).slice(0, 64) + : undefined, + createdAt: + typeof annotation.createdAt === "string" + ? String(annotation.createdAt).slice(0, 64) + : undefined, + }; + }) + .filter((annotation): annotation is CanvasAgentAnnotationRef => Boolean(annotation)) + .slice(0, 20); + }; + const selectedAgentNodes = selectedCanvasNodes + .map(toAgentNodeRef) + .filter((node): node is CanvasAgentNodeRef => Boolean(node)); + const candidateAgentNodes = uniqueCandidateSourceNodes + .map(toAgentNodeRef) + .filter((node): node is CanvasAgentNodeRef => Boolean(node)); + const agentSlots = nodes + .map(toAgentSlotRef) + .filter((slot): slot is CanvasAgentSlotRef => Boolean(slot)) + .slice(0, 20); + const agentAnnotations = uniqueCandidateSourceNodes + .flatMap(toAgentAnnotations) + .slice(0, 20); + const selectedAgentSlotCount = agentSlots.filter( + (slot) => slot.isSelected || selectedNodeIdSet.has(slot.nodeId) + ).length; + const agentFocusPolicy = + selectedAgentSlotCount > 0 + ? ("slot" as const) + : selectedCanvasNodes.length > 0 + ? ("selected" as const) + : ("auto" as const); + const canvasAgentContext: CanvasAgentContext = { + version: 1, + source: "canvas", + projectId, + viewport: { + x: Number(viewport.x || 0), + y: Number(viewport.y || 0), + zoom: Number(viewport.zoom || 1), + width: typeof window !== "undefined" ? window.innerWidth : undefined, + height: typeof window !== "undefined" ? window.innerHeight : undefined, + }, + selectedNodeIds: selectedCanvasNodes.map((node) => node.id).slice(0, 20), + selectedNodes: selectedAgentNodes, + candidateNodes: candidateAgentNodes, + slots: agentSlots, + annotations: agentAnnotations, + focusPolicy: agentFocusPolicy, + }; + const hasAgentContext = + canvasAgentContext.selectedNodes.length > 0 || + canvasAgentContext.candidateNodes.length > 0 || + canvasAgentContext.slots.length > 0 || + canvasAgentContext.annotations.length > 0 || + Boolean(canvasAgentContext.viewport); + + if (contextNodes.length === 0 && !hasAgentContext) { + generationAnchorNodesRef.current = []; + return undefined; + } + return { source: "canvas" as const, - projectId: isValidChatUuid(canvasId) ? canvasId : undefined, - selectedNodeIds: selectedContextNodes.map((node) => node.id).slice(0, 20), + projectId, + selectedNodeIds: selectedCanvasNodes.map((node) => node.id).slice(0, 20), selectedAssetIds, - focusPolicy: selectedContextNodes.length > 0 ? ("selected" as const) : ("auto" as const), + focusPolicy: agentFocusPolicy, canvasImageNodes: contextNodes.map((node) => ({ nodeId: node.id, assetId: isValidChatUuid((node.data as any)?.assetId) @@ -5398,8 +5611,9 @@ export function CanvasChatPanel({ height: Number(node.size?.height || 0) || undefined, isSelected: selectedIds.has(node.id), })), + canvasAgentContext, }; - }, [activeComposerMode, canvasId, nodes, selectedImageNodes]); + }, [activeComposerMode, canvasId, nodes, selectedImageNodes, selectedNodeIds, viewport]); const updateAttachmentAtVideoSlot = useCallback( (slotIndex: number, attachment: PickedObjectAttachment | null) => { @@ -6557,7 +6771,7 @@ export function CanvasChatPanel({ ? "首屏封面规则：必须做成海报主视觉，单一主体强焦点，产品主体占画面55%-70%，顶部标题区清晰留白，禁止拼贴式说明图。" : "", useModelForScene - ? `人物规则：本屏涉及人物场景，固定使用提供的模特图同一人物（面部、发型、体型、服装风格保持一致）。` + ? `人物规则：本屏涉及人物场景，必须固定使用提供的模特图同一人物；脸型五官、发型发色、体型肤色、穿搭风格保持一致，不得换脸、换人、换发型或改成无关人物。` : "人物规则：本屏不要出现人物或人体部位，只展示产品与场景元素。", screen?.modelReason ? `人物融入依据：${screen.modelReason}` : "", bundleShowcase diff --git a/components/canvas/nodes/ImageNode.tsx b/components/canvas/nodes/ImageNode.tsx index df8671ca9..8859f0d20 100644 --- a/components/canvas/nodes/ImageNode.tsx +++ b/components/canvas/nodes/ImageNode.tsx @@ -395,7 +395,7 @@ type LocalEditRegionInteractionState = { }; type CanvasLocalEditOperationMode = "local_edit" | "scale_adjust"; -type CanvasScaleAdjustPercent = -30 | -20 | -10 | 10 | 20 | 30; +type CanvasScaleAdjustPercent = -30 | -20 | -10 | 10 | 20 | 30 | 50 | 100; const CANVAS_SCALE_ADJUST_OPTIONS: CanvasScaleAdjustPercent[] = [ -30, @@ -404,6 +404,8 @@ const CANVAS_SCALE_ADJUST_OPTIONS: CanvasScaleAdjustPercent[] = [ 10, 20, 30, + 50, + 100, ]; function getCanvasScaleAdjustLabel(scalePercent: CanvasScaleAdjustPercent) { @@ -421,6 +423,7 @@ function getCanvasScaleAdjustRegionPrompt(scalePercent: CanvasScaleAdjustPercent const scaleVerb = scalePercent > 0 ? "放大" : "缩小"; return [ `对框选范围内的主物体执行等比${actionLabel}，调整后主体视觉尺寸必须约为原来的 ${targetLabel}，需要肉眼明显看出已经${scaleVerb}。`, + "框选区域只用于识别要调整的主体，不是放大后的边界；放大时主体可以自然超出原始框选范围，并由周边背景补全衔接。", "只改变目标主体的尺寸，不改变主体外观、颜色、材质、图案和文字内容；不要把整个框选矩形当成图片块缩放。", scalePercent < 0 ? "主体缩小后露出的背景、桌面、墙面、阴影和遮挡关系必须自然补全。" @@ -8671,7 +8674,7 @@ export function ImageNode({ )} {/* 处理中遮罩 */} - {isProcessing && ( + {isProcessing && !isSubmittingLocalEdit && ( ([ + "message", + "image", + "video", + "text", + "ui-screen", + "workflow", + "scene-fusion", + "model3d", + "world3d", + "agent-slot", +]); + +const CANVAS_AGENT_FOCUS_POLICIES = new Set([ + "auto", + "selected", + "latest_result", + "original", + "slot", +]); + +function normalizeString(value: unknown, maxLength = 512) { + if (typeof value !== "string") return undefined; + const normalized = value.trim(); + return normalized ? normalized.slice(0, maxLength) : undefined; +} + +function normalizeStringArray(value: unknown, maxItems: number) { + if (!Array.isArray(value)) return []; + return value + .map((item) => normalizeString(item, 128)) + .filter((item): item is string => Boolean(item)) + .slice(0, maxItems); +} + +function normalizeNumber(value: unknown) { + const parsed = typeof value === "number" ? value : Number(value); + return Number.isFinite(parsed) ? parsed : undefined; +} + +function normalizeBoolean(value: unknown) { + return typeof value === "boolean" ? value : undefined; +} + +function normalizeSource(value: unknown): CanvasAgentSource { + return value === "realtime-canvas" ? "realtime-canvas" : "canvas"; +} + +function normalizeFocusPolicy(value: unknown): CanvasAgentFocusPolicy { + return CANVAS_AGENT_FOCUS_POLICIES.has(String(value)) + ? (value as CanvasAgentFocusPolicy) + : "auto"; +} + +function normalizeNodeType(value: unknown): CanvasAgentNodeType | undefined { + const normalized = normalizeString(value, 64); + if (!normalized || !CANVAS_AGENT_NODE_TYPES.has(normalized)) return undefined; + return normalized as CanvasAgentNodeType; +} + +function normalizeViewport(value: unknown): CanvasViewportContext | undefined { + if (!value || typeof value !== "object" || Array.isArray(value)) return undefined; + const raw = value as Record; + const x = normalizeNumber(raw.x); + const y = normalizeNumber(raw.y); + const zoom = normalizeNumber(raw.zoom); + if (x === undefined || y === undefined || zoom === undefined) return undefined; + return { + x, + y, + zoom, + width: normalizeNumber(raw.width), + height: normalizeNumber(raw.height), + }; +} + +function normalizeNodeRef(value: unknown): CanvasAgentNodeRef | null { + if (!value || typeof value !== "object" || Array.isArray(value)) return null; + const raw = value as Record; + const nodeId = normalizeString(raw.nodeId, 128); + const type = normalizeNodeType(raw.type); + const x = normalizeNumber(raw.x); + const y = normalizeNumber(raw.y); + if (!nodeId || !type || x === undefined || y === undefined) return null; + return { + nodeId, + type, + assetId: normalizeString(raw.assetId, 128), + url: normalizeString(raw.url, 2048), + title: normalizeString(raw.title, 256), + prompt: normalizeString(raw.prompt, 2000), + x, + y, + width: normalizeNumber(raw.width), + height: normalizeNumber(raw.height), + isSelected: normalizeBoolean(raw.isSelected), + metadata: + raw.metadata && typeof raw.metadata === "object" && !Array.isArray(raw.metadata) + ? (raw.metadata as Record) + : undefined, + }; +} + +function normalizeSlotRef(value: unknown): CanvasAgentSlotRef | null { + if (!value || typeof value !== "object" || Array.isArray(value)) return null; + const raw = value as Record; + const nodeId = normalizeString(raw.nodeId, 128); + const slotType = normalizeString(raw.slotType, 32); + const x = normalizeNumber(raw.x); + const y = normalizeNumber(raw.y); + const width = normalizeNumber(raw.width); + const height = normalizeNumber(raw.height); + if ( + !nodeId || + (slotType !== "image" && slotType !== "video" && slotType !== "ui-screen") || + x === undefined || + y === undefined || + width === undefined || + height === undefined + ) { + return null; + } + const fillPolicy = normalizeString(raw.fillPolicy, 32); + return { + nodeId, + slotType, + label: normalizeString(raw.label, 128), + x, + y, + width, + height, + aspectRatio: normalizeString(raw.aspectRatio, 32) || "1:1", + isSelected: normalizeBoolean(raw.isSelected), + fillPolicy: + fillPolicy === "append-version" || fillPolicy === "replace-existing" + ? fillPolicy + : "replace-empty", + acceptedSources: Array.isArray(raw.acceptedSources) + ? raw.acceptedSources + .map((item) => normalizeString(item, 32)) + .filter( + (item): item is "text" | "image" | "selection" | "annotation" => + item === "text" || + item === "image" || + item === "selection" || + item === "annotation" + ) + .slice(0, 8) + : undefined, + }; +} + +function normalizeAnnotationRef(value: unknown): CanvasAgentAnnotationRef | null { + if (!value || typeof value !== "object" || Array.isArray(value)) return null; + const raw = value as Record; + const id = normalizeString(raw.id, 128); + const sourceNodeId = normalizeString(raw.sourceNodeId, 128); + const type = normalizeString(raw.type, 32); + if ( + !id || + !sourceNodeId || + (type !== "text-note" && + type !== "arrow-note" && + type !== "box" && + type !== "freehand" && + type !== "crop-box") + ) { + return null; + } + const rawBounds = + raw.bounds && typeof raw.bounds === "object" && !Array.isArray(raw.bounds) + ? (raw.bounds as Record) + : null; + const rawTargetPoint = + raw.targetPoint && typeof raw.targetPoint === "object" && !Array.isArray(raw.targetPoint) + ? (raw.targetPoint as Record) + : null; + return { + id, + sourceNodeId, + type, + text: normalizeString(raw.text, 1000), + bounds: rawBounds + ? { + x: normalizeNumber(rawBounds.x) || 0, + y: normalizeNumber(rawBounds.y) || 0, + width: normalizeNumber(rawBounds.width) || 0, + height: normalizeNumber(rawBounds.height) || 0, + } + : undefined, + targetPoint: rawTargetPoint + ? { + x: normalizeNumber(rawTargetPoint.x) || 0, + y: normalizeNumber(rawTargetPoint.y) || 0, + } + : undefined, + color: normalizeString(raw.color, 64), + createdAt: normalizeString(raw.createdAt, 64), + }; +} + +function normalizeNodeRefs(value: unknown, maxItems: number) { + if (!Array.isArray(value)) return []; + return value + .map(normalizeNodeRef) + .filter((node): node is CanvasAgentNodeRef => Boolean(node)) + .slice(0, maxItems); +} + +function normalizeSlotRefs(value: unknown, maxItems: number) { + if (!Array.isArray(value)) return []; + return value + .map(normalizeSlotRef) + .filter((slot): slot is CanvasAgentSlotRef => Boolean(slot)) + .slice(0, maxItems); +} + +function normalizeAnnotationRefs(value: unknown, maxItems: number) { + if (!Array.isArray(value)) return []; + return value + .map(normalizeAnnotationRef) + .filter((annotation): annotation is CanvasAgentAnnotationRef => Boolean(annotation)) + .slice(0, maxItems); +} + +function legacyImageToNodeRef(value: LegacyCanvasImageNode): CanvasAgentNodeRef | null { + const nodeId = normalizeString(value.nodeId, 128); + const url = normalizeString(value.url, 2048); + const x = normalizeNumber(value.x); + const y = normalizeNumber(value.y); + if (!nodeId || !url || x === undefined || y === undefined) return null; + return { + nodeId, + type: "image", + assetId: normalizeString(value.assetId, 128), + url, + title: normalizeString(value.title, 256), + prompt: normalizeString(value.prompt, 2000), + x, + y, + width: normalizeNumber(value.width), + height: normalizeNumber(value.height), + isSelected: normalizeBoolean(value.isSelected), + }; +} + +function normalizeLegacyImageNodes(value: unknown, maxItems: number) { + if (!Array.isArray(value)) return []; + return value + .map((item) => (item && typeof item === "object" ? legacyImageToNodeRef(item) : null)) + .filter((node): node is CanvasAgentNodeRef => Boolean(node)) + .slice(0, maxItems); +} + +function hasAgentContextValue(context: CanvasAgentContext) { + return ( + context.selectedNodes.length > 0 || + context.candidateNodes.length > 0 || + context.slots.length > 0 || + context.annotations.length > 0 || + Boolean(context.viewport) + ); +} + +function normalizeExplicitCanvasAgentContext( + value: unknown, + fallback: CanvasAgentContextPayloadLike +): CanvasAgentContext | null { + if (!value || typeof value !== "object" || Array.isArray(value)) return null; + const raw = value as Record; + const selectedNodeIds = normalizeStringArray(raw.selectedNodeIds, 20); + const selectedNodes = normalizeNodeRefs(raw.selectedNodes, 20); + const candidateNodes = normalizeNodeRefs(raw.candidateNodes, 30); + const slots = normalizeSlotRefs(raw.slots, 20); + const annotations = normalizeAnnotationRefs(raw.annotations, 20); + const context: CanvasAgentContext = { + version: 1, + source: normalizeSource(raw.source || fallback.source), + projectId: normalizeString(raw.projectId || fallback.projectId, 128), + canvasRevision: normalizeString(raw.canvasRevision, 128), + canvasUpdatedAt: normalizeString(raw.canvasUpdatedAt, 128), + viewport: normalizeViewport(raw.viewport), + selectedNodeIds: + selectedNodeIds.length > 0 + ? selectedNodeIds + : selectedNodes.map((node) => node.nodeId).slice(0, 20), + selectedNodes, + candidateNodes, + slots, + annotations, + focusPolicy: normalizeFocusPolicy(raw.focusPolicy || fallback.focusPolicy), + }; + return hasAgentContextValue(context) ? context : null; +} + +export function normalizeCanvasAgentContext( + payload?: CanvasAgentContextPayloadLike | null +): CanvasAgentContext | null { + if (!payload) return null; + + const explicit = normalizeExplicitCanvasAgentContext( + payload.canvasAgentContext, + payload + ); + if (explicit) return explicit; + + const candidateNodes = normalizeLegacyImageNodes(payload.canvasImageNodes, 20); + const selectedNodeIdSet = new Set(normalizeStringArray(payload.selectedNodeIds, 20)); + const selectedAssetIdSet = new Set(normalizeStringArray(payload.selectedAssetIds, 20)); + const selectedNodes = candidateNodes.filter((node) => { + if (node.isSelected) return true; + if (selectedNodeIdSet.has(node.nodeId)) return true; + if (node.assetId && selectedAssetIdSet.has(node.assetId)) return true; + return false; + }); + + const context: CanvasAgentContext = { + version: 1, + source: normalizeSource(payload.source), + projectId: normalizeString(payload.projectId, 128), + selectedNodeIds: + selectedNodes.length > 0 + ? selectedNodes.map((node) => node.nodeId).slice(0, 20) + : Array.from(selectedNodeIdSet).slice(0, 20), + selectedNodes, + candidateNodes, + slots: [], + annotations: [], + focusPolicy: normalizeFocusPolicy(payload.focusPolicy), + }; + + return hasAgentContextValue(context) ? context : null; +} + +export function getSelectedSlots(context?: CanvasAgentContext | null) { + return (context?.slots || []).filter( + (slot) => slot.isSelected || context?.selectedNodeIds.includes(slot.nodeId) + ); +} + +export function getSelectedMediaNodes(context?: CanvasAgentContext | null) { + return (context?.selectedNodes || []).filter( + (node) => node.type === "image" || node.type === "video" + ); +} + +export function getBestAnchorNode(context?: CanvasAgentContext | null) { + return ( + getSelectedMediaNodes(context)[0] || + context?.selectedNodes[0] || + context?.candidateNodes[0] || + null + ); +} + +export function getCanvasAgentFocusPolicy(context?: CanvasAgentContext | null) { + return context?.focusPolicy || "auto"; +} diff --git a/services/canvas/index.ts b/services/canvas/index.ts index ffda15169..f5e264627 100644 --- a/services/canvas/index.ts +++ b/services/canvas/index.ts @@ -340,7 +340,7 @@ export const imageGenerationService = { imageSize?: string; aspectRatio?: string; operation?: "scale-adjust"; - scalePercent?: -30 | -20 | -10 | 10 | 20 | 30; + scalePercent?: -30 | -20 | -10 | 10 | 20 | 30 | 50 | 100; }): Promise< ApiResponse<{ annotatedReferenceUrl: string; diff --git a/tests/routes/canvas-chat-helpers.test.ts b/tests/routes/canvas-chat-helpers.test.ts index 04a9f5266..97c4e76e1 100644 --- a/tests/routes/canvas-chat-helpers.test.ts +++ b/tests/routes/canvas-chat-helpers.test.ts @@ -6,6 +6,7 @@ import { normalizeMediaUrl, stripMediaMarkup, } from "@/components/canvas/chat-panel/canvas-chat-media"; +import { normalizeCanvasAgentContext } from "@/lib/canvas/agent-context"; import { buildDetailResultMarkdown, buildDetailReviewAppendMarkdown, @@ -49,6 +50,81 @@ import type { PickedObjectAttachment } from "@/components/canvas/chat-panel/canv import type { ChatMessage } from "@/lib/types"; test.describe("canvas chat pure helpers", () => { + test("normalizes slot-only canvas agent context without image nodes", () => { + const context = normalizeCanvasAgentContext({ + source: "canvas", + projectId: "11111111-1111-4111-8111-111111111111", + selectedNodeIds: ["slot-1"], + canvasImageNodes: [], + canvasAgentContext: { + version: 1, + source: "canvas", + projectId: "11111111-1111-4111-8111-111111111111", + selectedNodeIds: ["slot-1"], + selectedNodes: [ + { + nodeId: "slot-1", + type: "agent-slot", + x: 120, + y: 80, + width: 320, + height: 240, + isSelected: true, + }, + ], + candidateNodes: [], + slots: [ + { + nodeId: "slot-1", + slotType: "image", + x: 120, + y: 80, + width: 320, + height: 240, + aspectRatio: "4:3", + isSelected: true, + fillPolicy: "replace-empty", + }, + ], + annotations: [], + viewport: { x: 0, y: 0, zoom: 1 }, + focusPolicy: "slot", + }, + }); + + expect(context?.selectedNodeIds).toEqual(["slot-1"]); + expect(context?.slots).toHaveLength(1); + expect(context?.candidateNodes).toEqual([]); + expect(context?.focusPolicy).toBe("slot"); + }); + + test("builds canvas agent context from legacy canvas image nodes", () => { + const context = normalizeCanvasAgentContext({ + source: "canvas", + selectedNodeIds: ["image-1"], + canvasImageNodes: [ + { + nodeId: "image-1", + url: "https://cdn.example.com/image.png", + title: "主图", + x: 10, + y: 20, + width: 300, + height: 300, + isSelected: true, + }, + ], + }); + + expect(context?.selectedNodes).toHaveLength(1); + expect(context?.candidateNodes[0]).toMatchObject({ + nodeId: "image-1", + type: "image", + url: "https://cdn.example.com/image.png", + isSelected: true, + }); + }); + test("normalizes media urls without query strings or trailing slashes", () => { expect(normalizeMediaUrl(" HTTPS://EXAMPLE.COM/assets/image.png?sig=secret#preview ")).toBe( "https://example.com/assets/image.png" diff --git a/types/canvas/agent-protocol.ts b/types/canvas/agent-protocol.ts new file mode 100644 index 000000000..149dac628 --- /dev/null +++ b/types/canvas/agent-protocol.ts @@ -0,0 +1,161 @@ +import type { NodeType } from "@/types/canvas"; + +export type CanvasAgentSource = "canvas" | "realtime-canvas"; + +export type CanvasAgentFocusPolicy = + | "auto" + | "selected" + | "latest_result" + | "original" + | "slot"; + +export interface CanvasViewportContext { + x: number; + y: number; + zoom: number; + width?: number; + height?: number; +} + +export type CanvasAgentNodeType = NodeType | "agent-slot"; + +export interface CanvasAgentNodeRef { + nodeId: string; + type: CanvasAgentNodeType; + assetId?: string; + url?: string; + title?: string; + prompt?: string; + x: number; + y: number; + width?: number; + height?: number; + isSelected?: boolean; + metadata?: { + sourceNodeId?: string; + revisionOf?: string; + generatedByAgent?: boolean; + model?: string; + taskId?: string; + role?: string; + [key: string]: unknown; + }; +} + +export interface CanvasAgentSlotRef { + nodeId: string; + slotType: "image" | "video" | "ui-screen"; + label?: string; + x: number; + y: number; + width: number; + height: number; + aspectRatio: string; + isSelected?: boolean; + fillPolicy: "replace-empty" | "append-version" | "replace-existing"; + acceptedSources?: Array<"text" | "image" | "selection" | "annotation">; +} + +export interface CanvasAgentAnnotationRef { + id: string; + sourceNodeId: string; + type: "text-note" | "arrow-note" | "box" | "freehand" | "crop-box"; + text?: string; + bounds?: { + x: number; + y: number; + width: number; + height: number; + }; + targetPoint?: { + x: number; + y: number; + }; + color?: string; + createdAt?: string; +} + +export interface CanvasAgentContext { + version: 1; + source: CanvasAgentSource; + projectId?: string; + canvasRevision?: string; + canvasUpdatedAt?: string; + viewport?: CanvasViewportContext; + selectedNodeIds: string[]; + selectedNodes: CanvasAgentNodeRef[]; + candidateNodes: CanvasAgentNodeRef[]; + slots: CanvasAgentSlotRef[]; + annotations: CanvasAgentAnnotationRef[]; + focusPolicy: CanvasAgentFocusPolicy; +} + +export type CanvasOperationKind = + | "fill_slot" + | "insert_beside" + | "append_revision" + | "replace_node_media" + | "add_annotation" + | "link_nodes" + | "move_nodes" + | "update_text"; + +export interface CanvasMediaItem { + mediaItemId: string; + type: "image" | "video"; + url: string; + assetId?: string; + width?: number; + height?: number; + title?: string; + prompt?: string; + sourceNodeIds?: string[]; + metadata?: Record; +} + +export interface CanvasOperationPlan { + version: 1; + operationId: string; + operation: CanvasOperationKind; + projectId: string; + baseCanvasRevision?: string; + baseCanvasUpdatedAt?: string; + sourceNodeIds?: string[]; + targetSlotId?: string; + anchorNodeId?: string; + placement?: "right" | "left" | "below" | "above" | "viewport-center"; + placementStrategy?: "single" | "row" | "grid" | "overlay" | "beside_each_source"; + preserveOriginal?: boolean; + media?: Omit; + mediaItems?: CanvasMediaItem[]; + metadata?: Record; +} + +export type CanvasViewOperationKind = + | "focus_node" + | "focus_nodes" + | "focus_selection" + | "focus_latest_result" + | "fit_all" + | "zoom_in" + | "zoom_out" + | "zoom_to" + | "pan_to"; + +export interface CanvasViewOperationPlan { + version: 1; + viewOperationId: string; + operation: CanvasViewOperationKind; + projectId?: string; + nodeIds?: string[]; + anchorNodeId?: string; + zoom?: number; + padding?: number; + durationMs?: number; + persist?: false; + center?: { + x: number; + y: number; + }; + reason?: string; +}