Merge pull request 'Blend only the requested facial features' (#113) from codex/batch-instruction-face-blending into main

This commit is contained in:
WuYongJie 2026-11-20 12:07:29 +08:00
commit d79b0b8e49
3 changed files with 63 additions and 17 deletions

View File

@ -18,6 +18,38 @@ export function parseBatchInstructionRegion(value: unknown): CanvasLocalEditComp
}; };
} }
function faceFeaturesRegion(points: unknown) {
if (!Array.isArray(points) || points.length !== 6 || !points.every(
point => typeof point === "number" && Number.isFinite(point) && point >= 0 && point <= 1000,
)) return null;
const [leftX, leftY, rightX, rightY, mouthX, mouthY] = points;
const span = Math.abs(rightX - leftX);
const eyeY = (leftY + rightY) / 2;
const eyeX = (leftX + rightX) / 2;
const height = mouthY - eyeY;
if (span < 10 || height < 10 || Math.abs(mouthX - eyeX) > span) return null;
const center = (2 * eyeX + mouthX) / 3;
return parseBatchInstructionRegion([
Math.max(0, center - span * 0.86),
Math.max(0, Math.min(leftY, rightY) - height * 0.53),
Math.min(1000, center + span * 0.86),
Math.min(1000, mouthY + height * 0.34),
]);
}
function resizeEditRegion(region: CanvasLocalEditCompositeRegion, face: boolean) {
const xMargin = region.width * (face ? -0.08 : 0.11);
const yMargin = region.height * (face ? -0.11 : 0.11);
const x = Math.max(0, region.x - xMargin);
const y = Math.max(0, region.y - yMargin);
return {
x, y,
width: Math.min(1, region.x + region.width + xMargin) - x,
height: Math.min(1, region.y + region.height + yMargin) - y,
};
}
async function cropReference(url: string, region: CanvasLocalEditCompositeRegion) { async function cropReference(url: string, region: CanvasLocalEditCompositeRegion) {
const bytes = await fetchImageBufferFromUrl(url); const bytes = await fetchImageBufferFromUrl(url);
const oriented = await sharp(bytes).autoOrient().png().toBuffer({ resolveWithObject: true }); const oriented = await sharp(bytes).autoOrient().png().toBuffer({ resolveWithObject: true });
@ -47,7 +79,8 @@ export async function prepareBatchInstructionWithReference(request: CanvasGenera
"定位图片编辑目标。图1是要编辑的原图,图2只提供参照特征。", "定位图片编辑目标。图1是要编辑的原图,图2只提供参照特征。",
"只输出 JSON,格式为 {\"scope\":\"local\",\"base_box\":[x1,y1,x2,y2],\"reference_box\":[x1,y1,x2,y2]}。", "只输出 JSON,格式为 {\"scope\":\"local\",\"base_box\":[x1,y1,x2,y2],\"reference_box\":[x1,y1,x2,y2]}。",
"两个框分别使用各自图片的 0 至 999 归一化坐标。", "两个框分别使用各自图片的 0 至 999 归一化坐标。",
"换脸、五官或面部时,框住原人脸内部的眉毛、眼、鼻、嘴和下巴上方,不选头发、耳朵、颈部、衣服或背景。图2也只框五官。", "换脸或换五官时,将 scope 设为 face。两个框只框人脸内部的眉、眼、鼻、嘴,不包含头发、颈部、衣服和背景。",
"换脸时再返回 base_face 和 reference_face;每个数组分别为 [左眼中心x,左眼中心y,右眼中心x,右眼中心y,嘴中心x,嘴中心y]。",
"替换商品时,图1框住指定旧商品的完整外轮廓,包括扶手和腿。图2框住替换商品,勿框入其他人或物。", "替换商品时,图1框住指定旧商品的完整外轮廓,包括扶手和腿。图2框住替换商品,勿框入其他人或物。",
"只有指令明确要求修改整张图的整体风格、色调或场景时,返回 {\"scope\":\"full\"}。", "只有指令明确要求修改整张图的整体风格、色调或场景时,返回 {\"scope\":\"full\"}。",
"找不到明确目标,或多张脸、多件商品无法确定哪一个时,返回 {\"scope\":\"unresolved\"}。", "找不到明确目标,或多张脸、多件商品无法确定哪一个时,返回 {\"scope\":\"unresolved\"}。",
@ -58,7 +91,7 @@ export async function prepareBatchInstructionWithReference(request: CanvasGenera
throw new Error("暂时无法识别要编辑的区域,生图任务尚未提交,请稍后重试"); throw new Error("暂时无法识别要编辑的区域,生图任务尚未提交,请稍后重试");
} }
let result: { scope?: string; base_box?: unknown; reference_box?: unknown } = {}; let result: { scope?: string; base_box?: unknown; reference_box?: unknown; base_face?: unknown; reference_face?: unknown } = {};
try { try {
const match = text.match(/\{[\s\S]*\}/); const match = text.match(/\{[\s\S]*\}/);
if (match) result = JSON.parse(match[0]) || {}; if (match) result = JSON.parse(match[0]) || {};
@ -68,29 +101,36 @@ export async function prepareBatchInstructionWithReference(request: CanvasGenera
if (result.scope === "full" && !requiresLocalTarget) return request; if (result.scope === "full" && !requiresLocalTarget) return request;
const baseRegion = parseBatchInstructionRegion(result.base_box); const baseRegion = parseBatchInstructionRegion(result.base_box);
const referenceRegion = parseBatchInstructionRegion(result.reference_box); const referenceRegion = parseBatchInstructionRegion(result.reference_box);
if (result.scope !== "local" || !baseRegion || !referenceRegion || baseRegion.width * baseRegion.height >= 0.85) { if ((result.scope !== "local" && result.scope !== "face") || !baseRegion || !referenceRegion) {
throw new Error("未能定位唯一替换目标。请在指令里描述位置或特征,再重试这张图片"); throw new Error("未能定位唯一替换目标。请在指令里描述位置或特征,再重试这张图片");
} }
// Alibaba can use the cropped bytes directly without writing a new reference object. // Alibaba can use the cropped bytes directly without writing a new reference object.
const cropped = await cropReference(reference, referenceRegion).catch(() => null); const face = result.scope === "face" || /换脸|五官|face swap/i.test(request.displayPrompt || "");
const target = (face && faceFeaturesRegion(result.base_face)) || resizeEditRegion(baseRegion, face);
const referenceCrop = face
? faceFeaturesRegion(result.reference_face) || resizeEditRegion(referenceRegion, true)
: referenceRegion;
const cropped = await cropReference(reference, referenceCrop).catch(() => null);
return { return {
...request, ...request,
prompt: [ prompt: [
"这是局部编辑。以第一张图为唯一底图。图2只提供目标特征。", "这是局部编辑。以第一张图为唯一底图。图2只提供目标特征。",
"把修改放回原目标的位置和大小范围内。保持原目标的中心、落地位置或五官定位,保留原图其他内容。", "把修改放回原目标的位置和大小范围内。保持原目标的中心、落地位置或五官定位,保留原图其他内容。",
face ? "仅对齐图1的眉毛、眼睛、鼻子和嘴。保持两眼及嘴的位置,不改变头发、发际线、肤色和脸的位置。" : "完整移除原目标的轮廓、腿和残影,保持原物件的落地位置。",
"禁止添加图1原本没有的人物、物件、文字或标记。只输出一张与图1原构图对齐的图片。", "禁止添加图1原本没有的人物、物件、文字或标记。只输出一张与图1原构图对齐的图片。",
"指令:" + request.displayPrompt, "指令:" + request.displayPrompt,
].join("\n"), ].join("\n"),
referenceImageBuffers: cropped ? new Map([[reference, cropped]]) : undefined, referenceImageBuffers: cropped ? new Map([[reference, cropped]]) : undefined,
localEditComposite: { localEditComposite: {
sourceImageUrl: base, sourceImageUrl: base,
regions: [baseRegion], regions: [target],
expandRegions: false, expandRegions: false,
maskMode: "region", maskMode: "region",
featherMinPx: 2, maskShape: face ? "ellipse" : undefined,
featherMaxPx: 7, featherMinPx: face ? 3 : 7,
featherRatio: 0.004, featherMaxPx: face ? 10 : 20,
featherRatio: face ? 0.007 : 0.012,
}, },
} satisfies CanvasGenerateImageRequest; } satisfies CanvasGenerateImageRequest;
} }

View File

@ -60,6 +60,7 @@ export interface CanvasLocalEditCompositeConfig {
regions: CanvasLocalEditCompositeRegion[]; regions: CanvasLocalEditCompositeRegion[];
expandRegions?: boolean; expandRegions?: boolean;
maskMode?: "region" | "diff"; maskMode?: "region" | "diff";
maskShape?: "ellipse";
featherMinPx?: number; featherMinPx?: number;
featherMaxPx?: number; featherMaxPx?: number;
featherRatio?: number; featherRatio?: number;
@ -545,6 +546,7 @@ function normalizeLocalEditCompositeConfig(
regions, regions,
expandRegions, expandRegions,
maskMode, maskMode,
maskShape: value.maskShape === "ellipse" ? value.maskShape : undefined,
diffHighThreshold: normalizeOptionalNumber(value.diffHighThreshold, { diffHighThreshold: normalizeOptionalNumber(value.diffHighThreshold, {
max: 128, max: 128,
min: 1, min: 1,
@ -2895,6 +2897,7 @@ function buildLocalEditCompositeMaskSvg(params: {
height: number; height: number;
regions: CanvasLocalEditCompositeRegion[]; regions: CanvasLocalEditCompositeRegion[];
featherPx?: number; featherPx?: number;
maskShape?: "ellipse";
}) { }) {
const { width, height, regions } = params; const { width, height, regions } = params;
const blur = const blur =
@ -2922,12 +2925,14 @@ function buildLocalEditCompositeMaskSvg(params: {
return { left, top, rectHeight, rectWidth }; return { left, top, rectHeight, rectWidth };
}) })
.filter(({ rectHeight, rectWidth }) => rectWidth > 0 && rectHeight > 0); .filter(({ rectHeight, rectWidth }) => rectWidth > 0 && rectHeight > 0);
const shape = (left: number, top: number, w: number, h: number, radius: number) =>
params.maskShape === "ellipse"
? `<ellipse cx="${left + w / 2}" cy="${top + h / 2}" rx="${w / 2}" ry="${h / 2}" fill="#ffffff" />`
: `<rect x="${left}" y="${top}" width="${w}" height="${h}" rx="${radius}" fill="#ffffff" />`;
const softRects = rectEntries const softRects = rectEntries
.map( .map(
({ left, top, rectHeight, rectWidth }) => ({ left, top, rectHeight, rectWidth }) =>
`<rect x="${left}" y="${top}" width="${rectWidth}" height="${rectHeight}" rx="${Math.round( shape(left, top, rectWidth, rectHeight, Math.round(blur * 0.9))
blur * 0.9
)}" fill="#ffffff" />`
) )
.join(""); .join("");
const coreRects = rectEntries const coreRects = rectEntries
@ -2943,9 +2948,7 @@ function buildLocalEditCompositeMaskSvg(params: {
const coreWidth = rectWidth - inset * 2; const coreWidth = rectWidth - inset * 2;
const coreHeight = rectHeight - inset * 2; const coreHeight = rectHeight - inset * 2;
if (coreWidth <= 0 || coreHeight <= 0) return ""; if (coreWidth <= 0 || coreHeight <= 0) return "";
return `<rect x="${left + inset}" y="${top + inset}" width="${coreWidth}" height="${coreHeight}" rx="${Math.round( return shape(left + inset, top + inset, coreWidth, coreHeight, Math.round(blur * 0.45));
blur * 0.45
)}" fill="#ffffff" />`;
}) })
.join(""); .join("");
@ -3426,6 +3429,7 @@ async function applyLocalEditCompositeToImages(params: {
input: buildLocalEditCompositeMaskSvg({ input: buildLocalEditCompositeMaskSvg({
featherPx, featherPx,
height: sourceHeight, height: sourceHeight,
maskShape: composite.maskShape,
regions: composite.regions, regions: composite.regions,
width: sourceWidth, width: sourceWidth,
}), }),

View File

@ -65,7 +65,7 @@ test("shared face edits are confined to the source region and crop the reference
const photo = await sharp({ create: { width: 500, height: 500, channels: 3, background: "white" } }).png().toBuffer(); const photo = await sharp({ create: { width: 500, height: 500, channels: 3, background: "white" } }).png().toBuffer();
const { prepareBatchInstructionWithReference } = serverLoader({ const { prepareBatchInstructionWithReference } = serverLoader({
"@/lib/ai/qwen3-vl": { callQwen3Vl: async () => ({ "@/lib/ai/qwen3-vl": { callQwen3Vl: async () => ({
text: '{"scope":"local","base_box":[300,180,660,430],"reference_box":[350,260,650,620]}', text: '{"scope":"face","base_box":[300,110,690,490],"reference_box":[350,260,650,620],"base_face":[420,220,580,220,500,340],"reference_face":[430,350,570,350,500,500]}',
}) }, }) },
"@/lib/ai/rmbg/service": { fetchImageBufferFromUrl: async (url: string) => { "@/lib/ai/rmbg/service": { fetchImageBufferFromUrl: async (url: string) => {
expect(url).toBe(reference); expect(url).toBe(reference);
@ -75,8 +75,10 @@ test("shared face edits are confined to the source region and crop the reference
const prepared = await prepareBatchInstructionWithReference({ displayPrompt: "只换五官", referenceImages: [base, reference] }); const prepared = await prepareBatchInstructionWithReference({ displayPrompt: "只换五官", referenceImages: [base, reference] });
expect(prepared.localEditComposite).toMatchObject({ expect(prepared.localEditComposite).toMatchObject({
sourceImageUrl: base, expandRegions: false, regions: [{ x: 0.3, y: 0.18, width: 0.36, height: 0.25 }], sourceImageUrl: base, expandRegions: false, maskShape: "ellipse",
}); });
const cropped = await sharp(prepared.referenceImageBuffers.get(reference)).metadata(); const cropped = await sharp(prepared.referenceImageBuffers.get(reference)).metadata();
expect([cropped.width, cropped.height]).toEqual([150, 180]); expect([cropped.width, cropped.height]).toEqual([121, 141]);
expect(prepared.localEditComposite.regions[0].x).toBeGreaterThan(0.3);
expect(prepared.localEditComposite.regions[0].y).toBeGreaterThan(0.11);
}); });