ai-chatbot/lib/canvas/batch-instruction-reference.ts

97 lines
4.8 KiB
TypeScript
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

import "server-only";
import sharp from "sharp";
import { callQwen3Vl } from "@/lib/ai/qwen3-vl";
import { fetchImageBufferFromUrl } from "@/lib/ai/rmbg/service";
import type { CanvasGenerateImageRequest, CanvasLocalEditCompositeRegion } from "./generate-image";
export function parseBatchInstructionRegion(value: unknown): CanvasLocalEditCompositeRegion | null {
if (!Array.isArray(value) || value.length !== 4 || !value.every(
point => typeof point === "number" && Number.isFinite(point) && point >= 0 && point <= 1000,
)) return null;
const [left, top, right, bottom] = value;
if (right - left < 10 || bottom - top < 10) return null;
return {
x: left / 1000, y: top / 1000,
width: (right - left) / 1000, height: (bottom - top) / 1000,
};
}
async function cropReference(url: string, region: CanvasLocalEditCompositeRegion) {
const bytes = await fetchImageBufferFromUrl(url);
const oriented = await sharp(bytes).autoOrient().png().toBuffer({ resolveWithObject: true });
const { width, height } = oriented.info;
const left = Math.floor(region.x * width);
const top = Math.floor(region.y * height);
return sharp(oriented.data).extract({
left, top,
width: Math.min(width - left, Math.ceil(region.width * width)),
height: Math.min(height - top, Math.ceil(region.height * height)),
}).resize({ width: 1280, height: 1280, fit: "inside", withoutEnlargement: true }).png().toBuffer();
}
export async function prepareBatchInstructionWithReference(request: CanvasGenerateImageRequest) {
const [base, reference] = request.referenceImages || [];
if (!base || !reference) throw new Error("局部指令编辑缺少原图或共用参考图");
let text: string;
try {
({ text } = await callQwen3Vl({
imageUrls: [base, reference],
temperature: 0,
maxTokens: 280,
timeoutMs: 25_000,
retries: 0,
prompt: [
"定位图片编辑目标。图1是要编辑的原图,图2只提供参照特征。",
"只输出 JSON,格式为 {\"scope\":\"local\",\"base_box\":[x1,y1,x2,y2],\"reference_box\":[x1,y1,x2,y2]}。",
"两个框分别使用各自图片的 0 至 999 归一化坐标。",
"换脸、五官或面部时,框住原人脸内部的眉毛、眼、鼻、嘴和下巴上方,不选头发、耳朵、颈部、衣服或背景。图2也只框五官。",
"替换商品时,图1框住指定旧商品的完整外轮廓,包括扶手和腿。图2框住替换商品,勿框入其他人或物。",
"只有指令明确要求修改整张图的整体风格、色调或场景时,返回 {\"scope\":\"full\"}。",
"找不到明确目标,或多张脸、多件商品无法确定哪一个时,返回 {\"scope\":\"unresolved\"}。",
"用户指令:" + request.displayPrompt,
].join("\n"),
}));
} catch {
throw new Error("暂时无法识别要编辑的区域,生图任务尚未提交,请稍后重试");
}
let result: { scope?: string; base_box?: unknown; reference_box?: unknown } = {};
try {
const match = text.match(/\{[\s\S]*\}/);
if (match) result = JSON.parse(match[0]) || {};
} catch { /* Invalid or ambiguous localization is reported below. */ }
const requiresLocalTarget = /换脸|五官|脸部|面部|换商品|换产品|(?:商品|产品|主体).{0,8}替换|替换.{0,10}(?:商品|产品|主体|模特)|face swap|replace (?:the )?(?:face|product)/i.test(request.displayPrompt || "");
if (result.scope === "full" && !requiresLocalTarget) return request;
const baseRegion = parseBatchInstructionRegion(result.base_box);
const referenceRegion = parseBatchInstructionRegion(result.reference_box);
if (result.scope !== "local" || !baseRegion || !referenceRegion || baseRegion.width * baseRegion.height >= 0.85) {
throw new Error("未能定位唯一替换目标。请在指令里描述位置或特征,再重试这张图片");
}
// Alibaba can use the cropped bytes directly without writing a new reference object.
const cropped = await cropReference(reference, referenceRegion).catch(() => null);
return {
...request,
prompt: [
"这是局部编辑。以第一张图为唯一底图。图2只提供目标特征。",
"把修改放回原目标的位置和大小范围内。保持原目标的中心、落地位置或五官定位,保留原图其他内容。",
"禁止添加图1原本没有的人物、物件、文字或标记。只输出一张与图1原构图对齐的图片。",
"指令:" + request.displayPrompt,
].join("\n"),
referenceImageBuffers: cropped ? new Map([[reference, cropped]]) : undefined,
localEditComposite: {
sourceImageUrl: base,
regions: [baseRegion],
expandRegions: false,
maskMode: "region",
featherMinPx: 2,
featherMaxPx: 7,
featherRatio: 0.004,
},
} satisfies CanvasGenerateImageRequest;
}