Compare commits
5 Commits
codex/batc
...
main
| Author | SHA1 | Date | |
|---|---|---|---|
| d79b0b8e49 | |||
| 171e5d6a49 | |||
| c0054cfa7c | |||
| 4fb829a417 | |||
| 6ff8a0c43d |
@ -54,6 +54,7 @@ export async function POST(request: Request) {
|
||||
agentImageMode: "auto",
|
||||
promptExtend: false,
|
||||
padBatchInstructionToSquare: square || undefined,
|
||||
batchInstructionWithReference: Boolean(reference) || undefined,
|
||||
debugLabel: "batch-instruction-edit",
|
||||
};
|
||||
const task = await createCanvasImageTask({
|
||||
|
||||
136
lib/canvas/batch-instruction-reference.ts
Normal file
136
lib/canvas/batch-instruction-reference.ts
Normal file
@ -0,0 +1,136 @@
|
||||
import "server-only";
|
||||
|
||||
import sharp from "sharp";
|
||||
import { callQwen3Vl } from "@/lib/ai/qwen3-vl";
|
||||
import { fetchImageBufferFromUrl } from "@/lib/ai/rmbg/service";
|
||||
import type { CanvasGenerateImageRequest, CanvasLocalEditCompositeRegion } from "./generate-image";
|
||||
|
||||
export function parseBatchInstructionRegion(value: unknown): CanvasLocalEditCompositeRegion | null {
|
||||
if (!Array.isArray(value) || value.length !== 4 || !value.every(
|
||||
point => typeof point === "number" && Number.isFinite(point) && point >= 0 && point <= 1000,
|
||||
)) return null;
|
||||
|
||||
const [left, top, right, bottom] = value;
|
||||
if (right - left < 10 || bottom - top < 10) return null;
|
||||
return {
|
||||
x: left / 1000, y: top / 1000,
|
||||
width: (right - left) / 1000, height: (bottom - top) / 1000,
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
function faceFeaturesRegion(points: unknown) {
|
||||
if (!Array.isArray(points) || points.length !== 6 || !points.every(
|
||||
point => typeof point === "number" && Number.isFinite(point) && point >= 0 && point <= 1000,
|
||||
)) return null;
|
||||
const [leftX, leftY, rightX, rightY, mouthX, mouthY] = points;
|
||||
const span = Math.abs(rightX - leftX);
|
||||
const eyeY = (leftY + rightY) / 2;
|
||||
const eyeX = (leftX + rightX) / 2;
|
||||
const height = mouthY - eyeY;
|
||||
if (span < 10 || height < 10 || Math.abs(mouthX - eyeX) > span) return null;
|
||||
const center = (2 * eyeX + mouthX) / 3;
|
||||
return parseBatchInstructionRegion([
|
||||
Math.max(0, center - span * 0.86),
|
||||
Math.max(0, Math.min(leftY, rightY) - height * 0.53),
|
||||
Math.min(1000, center + span * 0.86),
|
||||
Math.min(1000, mouthY + height * 0.34),
|
||||
]);
|
||||
}
|
||||
|
||||
function resizeEditRegion(region: CanvasLocalEditCompositeRegion, face: boolean) {
|
||||
const xMargin = region.width * (face ? -0.08 : 0.11);
|
||||
const yMargin = region.height * (face ? -0.11 : 0.11);
|
||||
const x = Math.max(0, region.x - xMargin);
|
||||
const y = Math.max(0, region.y - yMargin);
|
||||
return {
|
||||
x, y,
|
||||
width: Math.min(1, region.x + region.width + xMargin) - x,
|
||||
height: Math.min(1, region.y + region.height + yMargin) - y,
|
||||
};
|
||||
}
|
||||
|
||||
async function cropReference(url: string, region: CanvasLocalEditCompositeRegion) {
|
||||
const bytes = await fetchImageBufferFromUrl(url);
|
||||
const oriented = await sharp(bytes).autoOrient().png().toBuffer({ resolveWithObject: true });
|
||||
const { width, height } = oriented.info;
|
||||
const left = Math.floor(region.x * width);
|
||||
const top = Math.floor(region.y * height);
|
||||
return sharp(oriented.data).extract({
|
||||
left, top,
|
||||
width: Math.min(width - left, Math.ceil(region.width * width)),
|
||||
height: Math.min(height - top, Math.ceil(region.height * height)),
|
||||
}).resize({ width: 1280, height: 1280, fit: "inside", withoutEnlargement: true }).png().toBuffer();
|
||||
}
|
||||
|
||||
export async function prepareBatchInstructionWithReference(request: CanvasGenerateImageRequest) {
|
||||
const [base, reference] = request.referenceImages || [];
|
||||
if (!base || !reference) throw new Error("局部指令编辑缺少原图或共用参考图");
|
||||
|
||||
let text: string;
|
||||
try {
|
||||
({ text } = await callQwen3Vl({
|
||||
imageUrls: [base, reference],
|
||||
temperature: 0,
|
||||
maxTokens: 280,
|
||||
timeoutMs: 25_000,
|
||||
retries: 0,
|
||||
prompt: [
|
||||
"定位图片编辑目标。图1是要编辑的原图,图2只提供参照特征。",
|
||||
"只输出 JSON,格式为 {\"scope\":\"local\",\"base_box\":[x1,y1,x2,y2],\"reference_box\":[x1,y1,x2,y2]}。",
|
||||
"两个框分别使用各自图片的 0 至 999 归一化坐标。",
|
||||
"换脸或换五官时,将 scope 设为 face。两个框只框人脸内部的眉、眼、鼻、嘴,不包含头发、颈部、衣服和背景。",
|
||||
"换脸时再返回 base_face 和 reference_face;每个数组分别为 [左眼中心x,左眼中心y,右眼中心x,右眼中心y,嘴中心x,嘴中心y]。",
|
||||
"替换商品时,图1框住指定旧商品的完整外轮廓,包括扶手和腿。图2框住替换商品,勿框入其他人或物。",
|
||||
"只有指令明确要求修改整张图的整体风格、色调或场景时,返回 {\"scope\":\"full\"}。",
|
||||
"找不到明确目标,或多张脸、多件商品无法确定哪一个时,返回 {\"scope\":\"unresolved\"}。",
|
||||
"用户指令:" + request.displayPrompt,
|
||||
].join("\n"),
|
||||
}));
|
||||
} catch {
|
||||
throw new Error("暂时无法识别要编辑的区域,生图任务尚未提交,请稍后重试");
|
||||
}
|
||||
|
||||
let result: { scope?: string; base_box?: unknown; reference_box?: unknown; base_face?: unknown; reference_face?: unknown } = {};
|
||||
try {
|
||||
const match = text.match(/\{[\s\S]*\}/);
|
||||
if (match) result = JSON.parse(match[0]) || {};
|
||||
} catch { /* Invalid or ambiguous localization is reported below. */ }
|
||||
|
||||
const requiresLocalTarget = /换脸|五官|脸部|面部|换商品|换产品|(?:商品|产品|主体).{0,8}替换|替换.{0,10}(?:商品|产品|主体|模特)|face swap|replace (?:the )?(?:face|product)/i.test(request.displayPrompt || "");
|
||||
if (result.scope === "full" && !requiresLocalTarget) return request;
|
||||
const baseRegion = parseBatchInstructionRegion(result.base_box);
|
||||
const referenceRegion = parseBatchInstructionRegion(result.reference_box);
|
||||
if ((result.scope !== "local" && result.scope !== "face") || !baseRegion || !referenceRegion) {
|
||||
throw new Error("未能定位唯一替换目标。请在指令里描述位置或特征,再重试这张图片");
|
||||
}
|
||||
|
||||
// Alibaba can use the cropped bytes directly without writing a new reference object.
|
||||
const face = result.scope === "face" || /换脸|五官|face swap/i.test(request.displayPrompt || "");
|
||||
const target = (face && faceFeaturesRegion(result.base_face)) || resizeEditRegion(baseRegion, face);
|
||||
const referenceCrop = face
|
||||
? faceFeaturesRegion(result.reference_face) || resizeEditRegion(referenceRegion, true)
|
||||
: referenceRegion;
|
||||
const cropped = await cropReference(reference, referenceCrop).catch(() => null);
|
||||
return {
|
||||
...request,
|
||||
prompt: [
|
||||
"这是局部编辑。以第一张图为唯一底图。图2只提供目标特征。",
|
||||
"把修改放回原目标的位置和大小范围内。保持原目标的中心、落地位置或五官定位,保留原图其他内容。",
|
||||
face ? "仅对齐图1的眉毛、眼睛、鼻子和嘴。保持两眼及嘴的位置,不改变头发、发际线、肤色和脸的位置。" : "完整移除原目标的轮廓、腿和残影,保持原物件的落地位置。",
|
||||
"禁止添加图1原本没有的人物、物件、文字或标记。只输出一张与图1原构图对齐的图片。",
|
||||
"指令:" + request.displayPrompt,
|
||||
].join("\n"),
|
||||
referenceImageBuffers: cropped ? new Map([[reference, cropped]]) : undefined,
|
||||
localEditComposite: {
|
||||
sourceImageUrl: base,
|
||||
regions: [target],
|
||||
expandRegions: false,
|
||||
maskMode: "region",
|
||||
maskShape: face ? "ellipse" : undefined,
|
||||
featherMinPx: face ? 3 : 7,
|
||||
featherMaxPx: face ? 10 : 20,
|
||||
featherRatio: face ? 0.007 : 0.012,
|
||||
},
|
||||
} satisfies CanvasGenerateImageRequest;
|
||||
}
|
||||
@ -60,6 +60,7 @@ export interface CanvasLocalEditCompositeConfig {
|
||||
regions: CanvasLocalEditCompositeRegion[];
|
||||
expandRegions?: boolean;
|
||||
maskMode?: "region" | "diff";
|
||||
maskShape?: "ellipse";
|
||||
featherMinPx?: number;
|
||||
featherMaxPx?: number;
|
||||
featherRatio?: number;
|
||||
@ -186,6 +187,7 @@ export interface CanvasGenerateImageRequest {
|
||||
billingFeature?: "multi-view" | "repaint";
|
||||
requiredReferenceImageCount?: number;
|
||||
padBatchInstructionToSquare?: boolean;
|
||||
batchInstructionWithReference?: boolean;
|
||||
ownerKey?: string;
|
||||
debugLabel?: string;
|
||||
debugMeta?: Record<string, unknown>;
|
||||
@ -544,6 +546,7 @@ function normalizeLocalEditCompositeConfig(
|
||||
regions,
|
||||
expandRegions,
|
||||
maskMode,
|
||||
maskShape: value.maskShape === "ellipse" ? value.maskShape : undefined,
|
||||
diffHighThreshold: normalizeOptionalNumber(value.diffHighThreshold, {
|
||||
max: 128,
|
||||
min: 1,
|
||||
@ -2894,6 +2897,7 @@ function buildLocalEditCompositeMaskSvg(params: {
|
||||
height: number;
|
||||
regions: CanvasLocalEditCompositeRegion[];
|
||||
featherPx?: number;
|
||||
maskShape?: "ellipse";
|
||||
}) {
|
||||
const { width, height, regions } = params;
|
||||
const blur =
|
||||
@ -2921,12 +2925,14 @@ function buildLocalEditCompositeMaskSvg(params: {
|
||||
return { left, top, rectHeight, rectWidth };
|
||||
})
|
||||
.filter(({ rectHeight, rectWidth }) => rectWidth > 0 && rectHeight > 0);
|
||||
const shape = (left: number, top: number, w: number, h: number, radius: number) =>
|
||||
params.maskShape === "ellipse"
|
||||
? `<ellipse cx="${left + w / 2}" cy="${top + h / 2}" rx="${w / 2}" ry="${h / 2}" fill="#ffffff" />`
|
||||
: `<rect x="${left}" y="${top}" width="${w}" height="${h}" rx="${radius}" fill="#ffffff" />`;
|
||||
const softRects = rectEntries
|
||||
.map(
|
||||
({ left, top, rectHeight, rectWidth }) =>
|
||||
`<rect x="${left}" y="${top}" width="${rectWidth}" height="${rectHeight}" rx="${Math.round(
|
||||
blur * 0.9
|
||||
)}" fill="#ffffff" />`
|
||||
shape(left, top, rectWidth, rectHeight, Math.round(blur * 0.9))
|
||||
)
|
||||
.join("");
|
||||
const coreRects = rectEntries
|
||||
@ -2942,9 +2948,7 @@ function buildLocalEditCompositeMaskSvg(params: {
|
||||
const coreWidth = rectWidth - inset * 2;
|
||||
const coreHeight = rectHeight - inset * 2;
|
||||
if (coreWidth <= 0 || coreHeight <= 0) return "";
|
||||
return `<rect x="${left + inset}" y="${top + inset}" width="${coreWidth}" height="${coreHeight}" rx="${Math.round(
|
||||
blur * 0.45
|
||||
)}" fill="#ffffff" />`;
|
||||
return shape(left + inset, top + inset, coreWidth, coreHeight, Math.round(blur * 0.45));
|
||||
})
|
||||
.join("");
|
||||
|
||||
@ -3425,6 +3429,7 @@ async function applyLocalEditCompositeToImages(params: {
|
||||
input: buildLocalEditCompositeMaskSvg({
|
||||
featherPx,
|
||||
height: sourceHeight,
|
||||
maskShape: composite.maskShape,
|
||||
regions: composite.regions,
|
||||
width: sourceWidth,
|
||||
}),
|
||||
|
||||
@ -403,8 +403,11 @@ async function runSingleTask(entry: PendingTaskEntry) {
|
||||
: (async () => {
|
||||
const imageRequest = request as CanvasGenerateImageRequest;
|
||||
const ownerKey = String(task.userId || entry.userId);
|
||||
const prepared = imageRequest.batchInstructionWithReference
|
||||
? await (await import("./batch-instruction-reference")).prepareBatchInstructionWithReference(imageRequest)
|
||||
: imageRequest;
|
||||
const initialResult = await generateCanvasImage({
|
||||
...imageRequest,
|
||||
...prepared,
|
||||
ownerKey,
|
||||
});
|
||||
if (imageRequest.padBatchInstructionToSquare) {
|
||||
|
||||
@ -58,3 +58,27 @@ test("instruction requests preserve base and shared reference roles and task ide
|
||||
expect(request?.prompt).toContain("第二张图只提供指令指定");
|
||||
expect(request?.prompt).toContain("只换五官");
|
||||
});
|
||||
|
||||
test("shared face edits are confined to the source region and crop the reference", async () => {
|
||||
const reference = "https://media.example/face.png";
|
||||
const base = "https://media.example/person.png";
|
||||
const photo = await sharp({ create: { width: 500, height: 500, channels: 3, background: "white" } }).png().toBuffer();
|
||||
const { prepareBatchInstructionWithReference } = serverLoader({
|
||||
"@/lib/ai/qwen3-vl": { callQwen3Vl: async () => ({
|
||||
text: '{"scope":"face","base_box":[300,110,690,490],"reference_box":[350,260,650,620],"base_face":[420,220,580,220,500,340],"reference_face":[430,350,570,350,500,500]}',
|
||||
}) },
|
||||
"@/lib/ai/rmbg/service": { fetchImageBufferFromUrl: async (url: string) => {
|
||||
expect(url).toBe(reference);
|
||||
return photo;
|
||||
} },
|
||||
})("lib/canvas/batch-instruction-reference.ts");
|
||||
|
||||
const prepared = await prepareBatchInstructionWithReference({ displayPrompt: "只换五官", referenceImages: [base, reference] });
|
||||
expect(prepared.localEditComposite).toMatchObject({
|
||||
sourceImageUrl: base, expandRegions: false, maskShape: "ellipse",
|
||||
});
|
||||
const cropped = await sharp(prepared.referenceImageBuffers.get(reference)).metadata();
|
||||
expect([cropped.width, cropped.height]).toEqual([121, 141]);
|
||||
expect(prepared.localEditComposite.regions[0].x).toBeGreaterThan(0.3);
|
||||
expect(prepared.localEditComposite.regions[0].y).toBeGreaterThan(0.11);
|
||||
});
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user