ai-chatbot/lib/canvas/batch-instruction-reference.ts

137 lines
6.8 KiB
TypeScript
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

import "server-only";
import sharp from "sharp";
import { callQwen3Vl } from "@/lib/ai/qwen3-vl";
import { fetchImageBufferFromUrl } from "@/lib/ai/rmbg/service";
import type { CanvasGenerateImageRequest, CanvasLocalEditCompositeRegion } from "./generate-image";
export function parseBatchInstructionRegion(value: unknown): CanvasLocalEditCompositeRegion | null {
if (!Array.isArray(value) || value.length !== 4 || !value.every(
point => typeof point === "number" && Number.isFinite(point) && point >= 0 && point <= 1000,
)) return null;
const [left, top, right, bottom] = value;
if (right - left < 10 || bottom - top < 10) return null;
return {
x: left / 1000, y: top / 1000,
width: (right - left) / 1000, height: (bottom - top) / 1000,
};
}
function faceFeaturesRegion(points: unknown) {
if (!Array.isArray(points) || points.length !== 6 || !points.every(
point => typeof point === "number" && Number.isFinite(point) && point >= 0 && point <= 1000,
)) return null;
const [leftX, leftY, rightX, rightY, mouthX, mouthY] = points;
const span = Math.abs(rightX - leftX);
const eyeY = (leftY + rightY) / 2;
const eyeX = (leftX + rightX) / 2;
const height = mouthY - eyeY;
if (span < 10 || height < 10 || Math.abs(mouthX - eyeX) > span) return null;
const center = (2 * eyeX + mouthX) / 3;
return parseBatchInstructionRegion([
Math.max(0, center - span * 0.86),
Math.max(0, Math.min(leftY, rightY) - height * 0.53),
Math.min(1000, center + span * 0.86),
Math.min(1000, mouthY + height * 0.34),
]);
}
function resizeEditRegion(region: CanvasLocalEditCompositeRegion, face: boolean) {
const xMargin = region.width * (face ? -0.08 : 0.11);
const yMargin = region.height * (face ? -0.11 : 0.11);
const x = Math.max(0, region.x - xMargin);
const y = Math.max(0, region.y - yMargin);
return {
x, y,
width: Math.min(1, region.x + region.width + xMargin) - x,
height: Math.min(1, region.y + region.height + yMargin) - y,
};
}
async function cropReference(url: string, region: CanvasLocalEditCompositeRegion) {
const bytes = await fetchImageBufferFromUrl(url);
const oriented = await sharp(bytes).autoOrient().png().toBuffer({ resolveWithObject: true });
const { width, height } = oriented.info;
const left = Math.floor(region.x * width);
const top = Math.floor(region.y * height);
return sharp(oriented.data).extract({
left, top,
width: Math.min(width - left, Math.ceil(region.width * width)),
height: Math.min(height - top, Math.ceil(region.height * height)),
}).resize({ width: 1280, height: 1280, fit: "inside", withoutEnlargement: true }).png().toBuffer();
}
export async function prepareBatchInstructionWithReference(request: CanvasGenerateImageRequest) {
const [base, reference] = request.referenceImages || [];
if (!base || !reference) throw new Error("局部指令编辑缺少原图或共用参考图");
let text: string;
try {
({ text } = await callQwen3Vl({
imageUrls: [base, reference],
temperature: 0,
maxTokens: 280,
timeoutMs: 25_000,
retries: 0,
prompt: [
"定位图片编辑目标。图1是要编辑的原图,图2只提供参照特征。",
"只输出 JSON,格式为 {\"scope\":\"local\",\"base_box\":[x1,y1,x2,y2],\"reference_box\":[x1,y1,x2,y2]}。",
"两个框分别使用各自图片的 0 至 999 归一化坐标。",
"换脸或换五官时,将 scope 设为 face。两个框只框人脸内部的眉、眼、鼻、嘴,不包含头发、颈部、衣服和背景。",
"换脸时再返回 base_face 和 reference_face;每个数组分别为 [左眼中心x,左眼中心y,右眼中心x,右眼中心y,嘴中心x,嘴中心y]。",
"替换商品时,图1框住指定旧商品的完整外轮廓,包括扶手和腿。图2框住替换商品,勿框入其他人或物。",
"只有指令明确要求修改整张图的整体风格、色调或场景时,返回 {\"scope\":\"full\"}。",
"找不到明确目标,或多张脸、多件商品无法确定哪一个时,返回 {\"scope\":\"unresolved\"}。",
"用户指令:" + request.displayPrompt,
].join("\n"),
}));
} catch {
throw new Error("暂时无法识别要编辑的区域,生图任务尚未提交,请稍后重试");
}
let result: { scope?: string; base_box?: unknown; reference_box?: unknown; base_face?: unknown; reference_face?: unknown } = {};
try {
const match = text.match(/\{[\s\S]*\}/);
if (match) result = JSON.parse(match[0]) || {};
} catch { /* Invalid or ambiguous localization is reported below. */ }
const requiresLocalTarget = /换脸|五官|脸部|面部|换商品|换产品|(?:商品|产品|主体).{0,8}替换|替换.{0,10}(?:商品|产品|主体|模特)|face swap|replace (?:the )?(?:face|product)/i.test(request.displayPrompt || "");
if (result.scope === "full" && !requiresLocalTarget) return request;
const baseRegion = parseBatchInstructionRegion(result.base_box);
const referenceRegion = parseBatchInstructionRegion(result.reference_box);
if ((result.scope !== "local" && result.scope !== "face") || !baseRegion || !referenceRegion) {
throw new Error("未能定位唯一替换目标。请在指令里描述位置或特征,再重试这张图片");
}
// Alibaba can use the cropped bytes directly without writing a new reference object.
const face = result.scope === "face" || /换脸|五官|face swap/i.test(request.displayPrompt || "");
const target = (face && faceFeaturesRegion(result.base_face)) || resizeEditRegion(baseRegion, face);
const referenceCrop = face
? faceFeaturesRegion(result.reference_face) || resizeEditRegion(referenceRegion, true)
: referenceRegion;
const cropped = await cropReference(reference, referenceCrop).catch(() => null);
return {
...request,
prompt: [
"这是局部编辑。以第一张图为唯一底图。图2只提供目标特征。",
"把修改放回原目标的位置和大小范围内。保持原目标的中心、落地位置或五官定位,保留原图其他内容。",
face ? "仅对齐图1的眉毛、眼睛、鼻子和嘴。保持两眼及嘴的位置,不改变头发、发际线、肤色和脸的位置。" : "完整移除原目标的轮廓、腿和残影,保持原物件的落地位置。",
"禁止添加图1原本没有的人物、物件、文字或标记。只输出一张与图1原构图对齐的图片。",
"指令:" + request.displayPrompt,
].join("\n"),
referenceImageBuffers: cropped ? new Map([[reference, cropped]]) : undefined,
localEditComposite: {
sourceImageUrl: base,
regions: [target],
expandRegions: false,
maskMode: "region",
maskShape: face ? "ellipse" : undefined,
featherMinPx: face ? 3 : 7,
featherMaxPx: face ? 10 : 20,
featherRatio: face ? 0.007 : 0.012,
},
} satisfies CanvasGenerateImageRequest;
}