Limit shared-reference edits to the intended region

This commit is contained in:
WuYongJie 2026-11-20 11:44:18 +08:00
parent dfe39a8d20
commit 4fb829a417
5 changed files with 124 additions and 1 deletions

View File

@ -54,6 +54,7 @@ export async function POST(request: Request) {
agentImageMode: "auto", agentImageMode: "auto",
promptExtend: false, promptExtend: false,
padBatchInstructionToSquare: square || undefined, padBatchInstructionToSquare: square || undefined,
batchInstructionWithReference: Boolean(reference) || undefined,
debugLabel: "batch-instruction-edit", debugLabel: "batch-instruction-edit",
}; };
const task = await createCanvasImageTask({ const task = await createCanvasImageTask({

View File

@ -0,0 +1,96 @@
import "server-only";
import sharp from "sharp";
import { callQwen3Vl } from "@/lib/ai/qwen3-vl";
import { fetchImageBufferFromUrl } from "@/lib/ai/rmbg/service";
import type { CanvasGenerateImageRequest, CanvasLocalEditCompositeRegion } from "./generate-image";
export function parseBatchInstructionRegion(value: unknown): CanvasLocalEditCompositeRegion | null {
if (!Array.isArray(value) || value.length !== 4 || !value.every(
point => typeof point === "number" && Number.isFinite(point) && point >= 0 && point <= 1000,
)) return null;
const [left, top, right, bottom] = value;
if (right - left < 10 || bottom - top < 10) return null;
return {
x: left / 1000, y: top / 1000,
width: (right - left) / 1000, height: (bottom - top) / 1000,
};
}
async function cropReference(url: string, region: CanvasLocalEditCompositeRegion) {
const bytes = await fetchImageBufferFromUrl(url);
const oriented = await sharp(bytes).autoOrient().png().toBuffer({ resolveWithObject: true });
const { width, height } = oriented.info;
const left = Math.floor(region.x * width);
const top = Math.floor(region.y * height);
return sharp(oriented.data).extract({
left, top,
width: Math.min(width - left, Math.ceil(region.width * width)),
height: Math.min(height - top, Math.ceil(region.height * height)),
}).resize({ width: 1280, height: 1280, fit: "inside", withoutEnlargement: true }).png().toBuffer();
}
export async function prepareBatchInstructionWithReference(request: CanvasGenerateImageRequest) {
const [base, reference] = request.referenceImages || [];
if (!base || !reference) throw new Error("局部指令编辑缺少原图或共用参考图");
let text: string;
try {
({ text } = await callQwen3Vl({
imageUrls: [base, reference],
temperature: 0,
maxTokens: 280,
timeoutMs: 25_000,
retries: 0,
prompt: [
"定位图片编辑目标。图1是要编辑的原图,图2只提供参照特征。",
"只输出 JSON,格式为 {\"scope\":\"local\",\"base_box\":[x1,y1,x2,y2],\"reference_box\":[x1,y1,x2,y2]}。",
"两个框分别使用各自图片的 0 至 999 归一化坐标。",
"换脸、五官或面部时,框住原人脸内部的眉毛、眼、鼻、嘴和下巴上方,不选头发、耳朵、颈部、衣服或背景。图2也只框五官。",
"替换商品时,图1框住指定旧商品的完整外轮廓,包括扶手和腿。图2框住替换商品,勿框入其他人或物。",
"只有指令明确要求修改整张图的整体风格、色调或场景时,返回 {\"scope\":\"full\"}。",
"找不到明确目标,或多张脸、多件商品无法确定哪一个时,返回 {\"scope\":\"unresolved\"}。",
"用户指令:" + request.displayPrompt,
].join("\n"),
}));
} catch {
throw new Error("暂时无法识别要编辑的区域,生图任务尚未提交,请稍后重试");
}
let result: { scope?: string; base_box?: unknown; reference_box?: unknown } = {};
try {
const match = text.match(/\{[\s\S]*\}/);
if (match) result = JSON.parse(match[0]) || {};
} catch { /* Invalid or ambiguous localization is reported below. */ }
const requiresLocalTarget = /换脸|五官|脸部|面部|换商品|换产品|(?:商品|产品|主体).{0,8}替换|替换.{0,10}(?:商品|产品|主体|模特)|face swap|replace (?:the )?(?:face|product)/i.test(request.displayPrompt || "");
if (result.scope === "full" && !requiresLocalTarget) return request;
const baseRegion = parseBatchInstructionRegion(result.base_box);
const referenceRegion = parseBatchInstructionRegion(result.reference_box);
if (result.scope !== "local" || !baseRegion || !referenceRegion || baseRegion.width * baseRegion.height >= 0.85) {
throw new Error("未能定位唯一替换目标。请在指令里描述位置或特征,再重试这张图片");
}
// Alibaba can use the cropped bytes directly without writing a new reference object.
const cropped = await cropReference(reference, referenceRegion).catch(() => null);
return {
...request,
prompt: [
"这是局部编辑。以第一张图为唯一底图。图2只提供目标特征。",
"把修改放回原目标的位置和大小范围内。保持原目标的中心、落地位置或五官定位,保留原图其他内容。",
"禁止添加图1原本没有的人物、物件、文字或标记。只输出一张与图1原构图对齐的图片。",
"指令:" + request.displayPrompt,
].join("\n"),
referenceImageBuffers: cropped ? new Map([[reference, cropped]]) : undefined,
localEditComposite: {
sourceImageUrl: base,
regions: [baseRegion],
expandRegions: false,
maskMode: "region",
featherMinPx: 2,
featherMaxPx: 7,
featherRatio: 0.004,
},
} satisfies CanvasGenerateImageRequest;
}

View File

@ -186,6 +186,7 @@ export interface CanvasGenerateImageRequest {
billingFeature?: "multi-view" | "repaint"; billingFeature?: "multi-view" | "repaint";
requiredReferenceImageCount?: number; requiredReferenceImageCount?: number;
padBatchInstructionToSquare?: boolean; padBatchInstructionToSquare?: boolean;
batchInstructionWithReference?: boolean;
ownerKey?: string; ownerKey?: string;
debugLabel?: string; debugLabel?: string;
debugMeta?: Record<string, unknown>; debugMeta?: Record<string, unknown>;

View File

@ -403,8 +403,11 @@ async function runSingleTask(entry: PendingTaskEntry) {
: (async () => { : (async () => {
const imageRequest = request as CanvasGenerateImageRequest; const imageRequest = request as CanvasGenerateImageRequest;
const ownerKey = String(task.userId || entry.userId); const ownerKey = String(task.userId || entry.userId);
const prepared = imageRequest.batchInstructionWithReference
? await (await import("./batch-instruction-reference")).prepareBatchInstructionWithReference(imageRequest)
: imageRequest;
const initialResult = await generateCanvasImage({ const initialResult = await generateCanvasImage({
...imageRequest, ...prepared,
ownerKey, ownerKey,
}); });
if (imageRequest.padBatchInstructionToSquare) { if (imageRequest.padBatchInstructionToSquare) {

View File

@ -58,3 +58,25 @@ test("instruction requests preserve base and shared reference roles and task ide
expect(request?.prompt).toContain("第二张图只提供指令指定"); expect(request?.prompt).toContain("第二张图只提供指令指定");
expect(request?.prompt).toContain("只换五官"); expect(request?.prompt).toContain("只换五官");
}); });
test("shared face edits are confined to the source region and crop the reference", async () => {
const reference = "https://media.example/face.png";
const base = "https://media.example/person.png";
const photo = await sharp({ create: { width: 500, height: 500, channels: 3, background: "white" } }).png().toBuffer();
const { prepareBatchInstructionWithReference } = serverLoader({
"@/lib/ai/qwen3-vl": { callQwen3Vl: async () => ({
text: '{"scope":"local","base_box":[300,180,660,430],"reference_box":[350,260,650,620]}',
}) },
"@/lib/ai/rmbg/service": { fetchImageBufferFromUrl: async (url: string) => {
expect(url).toBe(reference);
return photo;
} },
})("lib/canvas/batch-instruction-reference.ts");
const prepared = await prepareBatchInstructionWithReference({ displayPrompt: "只换五官", referenceImages: [base, reference] });
expect(prepared.localEditComposite).toMatchObject({
sourceImageUrl: base, expandRegions: false, regions: [{ x: 0.3, y: 0.18, width: 0.36, height: 0.25 }],
});
const cropped = await sharp(prepared.referenceImageBuffers.get(reference)).metadata();
expect([cropped.width, cropped.height]).toEqual([150, 180]);
});