From 4fb829a41788c44a6df5e365874a40bbdfb678c2 Mon Sep 17 00:00:00 2001 From: WuYongJie <549727955@qq.com> Date: Fri, 20 Nov 2026 11:44:18 +0800 Subject: [PATCH] Limit shared-reference edits to the intended region --- app/api/canvas/batch-instruction/route.ts | 1 + lib/canvas/batch-instruction-reference.ts | 96 +++++++++++++++++++++ lib/canvas/generate-image.ts | 1 + lib/canvas/image-task-queue.ts | 5 +- tests/routes/batch-instruction-edit.test.ts | 22 +++++ 5 files changed, 124 insertions(+), 1 deletion(-) create mode 100644 lib/canvas/batch-instruction-reference.ts diff --git a/app/api/canvas/batch-instruction/route.ts b/app/api/canvas/batch-instruction/route.ts index 2cd554ab6..d5d80866c 100644 --- a/app/api/canvas/batch-instruction/route.ts +++ b/app/api/canvas/batch-instruction/route.ts @@ -54,6 +54,7 @@ export async function POST(request: Request) { agentImageMode: "auto", promptExtend: false, padBatchInstructionToSquare: square || undefined, + batchInstructionWithReference: Boolean(reference) || undefined, debugLabel: "batch-instruction-edit", }; const task = await createCanvasImageTask({ diff --git a/lib/canvas/batch-instruction-reference.ts b/lib/canvas/batch-instruction-reference.ts new file mode 100644 index 000000000..e5fb543e9 --- /dev/null +++ b/lib/canvas/batch-instruction-reference.ts @@ -0,0 +1,96 @@ +import "server-only"; + +import sharp from "sharp"; +import { callQwen3Vl } from "@/lib/ai/qwen3-vl"; +import { fetchImageBufferFromUrl } from "@/lib/ai/rmbg/service"; +import type { CanvasGenerateImageRequest, CanvasLocalEditCompositeRegion } from "./generate-image"; + +export function parseBatchInstructionRegion(value: unknown): CanvasLocalEditCompositeRegion | null { + if (!Array.isArray(value) || value.length !== 4 || !value.every( + point => typeof point === "number" && Number.isFinite(point) && point >= 0 && point <= 1000, + )) return null; + + const [left, top, right, bottom] = value; + if (right - left < 10 || bottom - top < 10) return null; + return { + x: left / 1000, y: top / 1000, + width: (right - left) / 1000, height: (bottom - top) / 1000, + }; +} + +async function cropReference(url: string, region: CanvasLocalEditCompositeRegion) { + const bytes = await fetchImageBufferFromUrl(url); + const oriented = await sharp(bytes).autoOrient().png().toBuffer({ resolveWithObject: true }); + const { width, height } = oriented.info; + const left = Math.floor(region.x * width); + const top = Math.floor(region.y * height); + return sharp(oriented.data).extract({ + left, top, + width: Math.min(width - left, Math.ceil(region.width * width)), + height: Math.min(height - top, Math.ceil(region.height * height)), + }).resize({ width: 1280, height: 1280, fit: "inside", withoutEnlargement: true }).png().toBuffer(); +} + +export async function prepareBatchInstructionWithReference(request: CanvasGenerateImageRequest) { + const [base, reference] = request.referenceImages || []; + if (!base || !reference) throw new Error("局部指令编辑缺少原图或共用参考图"); + + let text: string; + try { + ({ text } = await callQwen3Vl({ + imageUrls: [base, reference], + temperature: 0, + maxTokens: 280, + timeoutMs: 25_000, + retries: 0, + prompt: [ + "定位图片编辑目标。图1是要编辑的原图,图2只提供参照特征。", + "只输出 JSON,格式为 {\"scope\":\"local\",\"base_box\":[x1,y1,x2,y2],\"reference_box\":[x1,y1,x2,y2]}。", + "两个框分别使用各自图片的 0 至 999 归一化坐标。", + "换脸、五官或面部时,框住原人脸内部的眉毛、眼、鼻、嘴和下巴上方,不选头发、耳朵、颈部、衣服或背景。图2也只框五官。", + "替换商品时,图1框住指定旧商品的完整外轮廓,包括扶手和腿。图2框住替换商品,勿框入其他人或物。", + "只有指令明确要求修改整张图的整体风格、色调或场景时,返回 {\"scope\":\"full\"}。", + "找不到明确目标,或多张脸、多件商品无法确定哪一个时,返回 {\"scope\":\"unresolved\"}。", + "用户指令:" + request.displayPrompt, + ].join("\n"), + })); + } catch { + throw new Error("暂时无法识别要编辑的区域,生图任务尚未提交,请稍后重试"); + } + + let result: { scope?: string; base_box?: unknown; reference_box?: unknown } = {}; + try { + const match = text.match(/\{[\s\S]*\}/); + if (match) result = JSON.parse(match[0]) || {}; + } catch { /* Invalid or ambiguous localization is reported below. */ } + + const requiresLocalTarget = /换脸|五官|脸部|面部|换商品|换产品|(?:商品|产品|主体).{0,8}替换|替换.{0,10}(?:商品|产品|主体|模特)|face swap|replace (?:the )?(?:face|product)/i.test(request.displayPrompt || ""); + if (result.scope === "full" && !requiresLocalTarget) return request; + const baseRegion = parseBatchInstructionRegion(result.base_box); + const referenceRegion = parseBatchInstructionRegion(result.reference_box); + if (result.scope !== "local" || !baseRegion || !referenceRegion || baseRegion.width * baseRegion.height >= 0.85) { + throw new Error("未能定位唯一替换目标。请在指令里描述位置或特征,再重试这张图片"); + } + + // Alibaba can use the cropped bytes directly without writing a new reference object. + const cropped = await cropReference(reference, referenceRegion).catch(() => null); + return { + ...request, + prompt: [ + "这是局部编辑。以第一张图为唯一底图。图2只提供目标特征。", + "把修改放回原目标的位置和大小范围内。保持原目标的中心、落地位置或五官定位,保留原图其他内容。", + "禁止添加图1原本没有的人物、物件、文字或标记。只输出一张与图1原构图对齐的图片。", + "指令:" + request.displayPrompt, + ].join("\n"), + referenceImageBuffers: cropped ? new Map([[reference, cropped]]) : undefined, + localEditComposite: { + sourceImageUrl: base, + regions: [baseRegion], + expandRegions: false, + maskMode: "region", + featherMinPx: 2, + featherMaxPx: 7, + featherRatio: 0.004, + }, + } satisfies CanvasGenerateImageRequest; +} diff --git a/lib/canvas/generate-image.ts b/lib/canvas/generate-image.ts index 65c118e88..abb7bf2d7 100644 --- a/lib/canvas/generate-image.ts +++ b/lib/canvas/generate-image.ts @@ -186,6 +186,7 @@ export interface CanvasGenerateImageRequest { billingFeature?: "multi-view" | "repaint"; requiredReferenceImageCount?: number; padBatchInstructionToSquare?: boolean; + batchInstructionWithReference?: boolean; ownerKey?: string; debugLabel?: string; debugMeta?: Record; diff --git a/lib/canvas/image-task-queue.ts b/lib/canvas/image-task-queue.ts index 75501995b..0c06d0d60 100644 --- a/lib/canvas/image-task-queue.ts +++ b/lib/canvas/image-task-queue.ts @@ -403,8 +403,11 @@ async function runSingleTask(entry: PendingTaskEntry) { : (async () => { const imageRequest = request as CanvasGenerateImageRequest; const ownerKey = String(task.userId || entry.userId); + const prepared = imageRequest.batchInstructionWithReference + ? await (await import("./batch-instruction-reference")).prepareBatchInstructionWithReference(imageRequest) + : imageRequest; const initialResult = await generateCanvasImage({ - ...imageRequest, + ...prepared, ownerKey, }); if (imageRequest.padBatchInstructionToSquare) { diff --git a/tests/routes/batch-instruction-edit.test.ts b/tests/routes/batch-instruction-edit.test.ts index becf635a8..54eb4ef6b 100644 --- a/tests/routes/batch-instruction-edit.test.ts +++ b/tests/routes/batch-instruction-edit.test.ts @@ -58,3 +58,25 @@ test("instruction requests preserve base and shared reference roles and task ide expect(request?.prompt).toContain("第二张图只提供指令指定"); expect(request?.prompt).toContain("只换五官"); }); + +test("shared face edits are confined to the source region and crop the reference", async () => { + const reference = "https://media.example/face.png"; + const base = "https://media.example/person.png"; + const photo = await sharp({ create: { width: 500, height: 500, channels: 3, background: "white" } }).png().toBuffer(); + const { prepareBatchInstructionWithReference } = serverLoader({ + "@/lib/ai/qwen3-vl": { callQwen3Vl: async () => ({ + text: '{"scope":"local","base_box":[300,180,660,430],"reference_box":[350,260,650,620]}', + }) }, + "@/lib/ai/rmbg/service": { fetchImageBufferFromUrl: async (url: string) => { + expect(url).toBe(reference); + return photo; + } }, + })("lib/canvas/batch-instruction-reference.ts"); + + const prepared = await prepareBatchInstructionWithReference({ displayPrompt: "只换五官", referenceImages: [base, reference] }); + expect(prepared.localEditComposite).toMatchObject({ + sourceImageUrl: base, expandRegions: false, regions: [{ x: 0.3, y: 0.18, width: 0.36, height: 0.25 }], + }); + const cropped = await sharp(prepared.referenceImageBuffers.get(reference)).metadata(); + expect([cropped.width, cropped.height]).toEqual([150, 180]); +});