import "server-only"; import sharp from "sharp"; import { callQwen3Vl } from "@/lib/ai/qwen3-vl"; import { fetchImageBufferFromUrl } from "@/lib/ai/rmbg/service"; import type { CanvasGenerateImageRequest, CanvasLocalEditCompositeRegion } from "./generate-image"; export function parseBatchInstructionRegion(value: unknown): CanvasLocalEditCompositeRegion | null { if (!Array.isArray(value) || value.length !== 4 || !value.every( point => typeof point === "number" && Number.isFinite(point) && point >= 0 && point <= 1000, )) return null; const [left, top, right, bottom] = value; if (right - left < 10 || bottom - top < 10) return null; return { x: left / 1000, y: top / 1000, width: (right - left) / 1000, height: (bottom - top) / 1000, }; } const faceSubject = /人脸|脸部|脸型|五官|面容|发型|face|facial features|hairstyle/i; const faceAction = /替换|换成|换为|置换|改成|改为|swap|replace|transfer|change/i; function requestsFaceEdit(instruction: string) { return instruction.split(/[,,。;;!!??\n]+/).some((clause) => { const editable = clause.split(/保持|保留|维持|不要|别|不得|禁止|避免|do not|don't|keep|preserve/i, 1)[0]; return /换(?:人?脸|五官|发型)|face[- ]swap/i.test(editable) || (faceSubject.test(editable) && faceAction.test(editable)); }); } function directFaceEdit(request: CanvasGenerateImageRequest): CanvasGenerateImageRequest { return { ...request, batchInstructionFaceEdit: true, requiredReferenceImageCount: 2, prompt: [ "第一张是要编辑的完整底图。第二张只提供指令指定的面部或头发特征。", "只局部编辑底图人物头部。仅修改指令明确要求的五官、脸型、发型或发色,保留其余头部特征。匹配底图的头部位置、转头或低头角度、视线、遮挡、肤色及场景光线,脸部和颈部自然衔接。", "保留原始画幅、姿势、手、道具、服饰颜色与印花、背景物件、标牌、文字和非头部区域的原有纹理。避免裁切、全图重绘、锐化或改色。输出一张完整成品图。", "用户指令:" + request.displayPrompt, ].join("\n"), }; } function expandProductRegion(region: CanvasLocalEditCompositeRegion) { const xMargin = region.width * 0.11; const yMargin = region.height * 0.11; const x = Math.max(0, region.x - xMargin); const y = Math.max(0, region.y - yMargin); return { x, y, width: Math.min(1, region.x + region.width + xMargin) - x, height: Math.min(1, region.y + region.height + yMargin) - y, }; } async function cropReference(url: string, region: CanvasLocalEditCompositeRegion) { const bytes = await fetchImageBufferFromUrl(url); const oriented = await sharp(bytes).autoOrient().png().toBuffer({ resolveWithObject: true }); const { width, height } = oriented.info; const left = Math.floor(region.x * width); const top = Math.floor(region.y * height); return sharp(oriented.data).extract({ left, top, width: Math.min(width - left, Math.ceil(region.width * width)), height: Math.min(height - top, Math.ceil(region.height * height)), }).resize({ width: 1280, height: 1280, fit: "inside", withoutEnlargement: true }).png().toBuffer(); } export async function prepareBatchInstructionWithReference(request: CanvasGenerateImageRequest) { const [base, reference] = request.referenceImages || []; if (!base || !reference) throw new Error("局部指令编辑缺少原图或共用参考图"); if (requestsFaceEdit(request.displayPrompt || "")) return directFaceEdit(request); let text: string; try { ({ text } = await callQwen3Vl({ imageUrls: [base, reference], temperature: 0, maxTokens: 280, timeoutMs: 25_000, retries: 0, prompt: [ "定位图片编辑目标。图1是要编辑的原图,图2只提供参照特征。", "只输出 JSON,格式为 {\"scope\":\"local\",\"base_box\":[x1,y1,x2,y2],\"reference_box\":[x1,y1,x2,y2]}。", "两个框分别使用各自图片的 0 至 999 归一化坐标。", "指令明确要求换脸、换五官、脸型或发型时,直接返回 {\"scope\":\"face\"}。只要求保留这些特征时,按实际编辑目标判断。", "替换商品时,图1框住指定旧商品的完整外轮廓,包括扶手和腿。图2框住替换商品,勿框入其他人或物。", "只有指令明确要求修改整张图的整体风格、色调或场景时,返回 {\"scope\":\"full\"}。", "找不到明确目标,或多张脸、多件商品无法确定哪一个时,返回 {\"scope\":\"unresolved\"}。", "用户指令:" + request.displayPrompt, ].join("\n"), })); } catch { throw new Error("暂时无法识别要编辑的区域,生图任务尚未提交,请稍后重试"); } let result: { scope?: string; base_box?: unknown; reference_box?: unknown } = {}; try { const match = text.match(/\{[\s\S]*\}/); if (match) result = JSON.parse(match[0]) || {}; } catch { /* Invalid or ambiguous localization is reported below. */ } if (result.scope === "face") return directFaceEdit(request); const requiresLocalTarget = /换商品|换产品|(?:商品|产品|主体).{0,8}替换|替换.{0,10}(?:商品|产品|主体|模特)|face swap|replace (?:the )?(?:face|product)/i.test(request.displayPrompt || ""); if (result.scope === "full" && !requiresLocalTarget) return request; const baseRegion = parseBatchInstructionRegion(result.base_box); const referenceRegion = parseBatchInstructionRegion(result.reference_box); if (result.scope !== "local" || !baseRegion || !referenceRegion) { throw new Error("未能定位唯一替换目标。请在指令里描述位置或特征,再重试这张图片"); } // Alibaba can use the cropped bytes directly without writing a new reference object. const target = expandProductRegion(baseRegion); const cropped = await cropReference(reference, referenceRegion).catch(() => null); return { ...request, prompt: [ "这是局部编辑。以第一张图为唯一底图。图2只提供目标特征。", "把修改放回原目标的位置和大小范围内。保持原目标的中心、落地位置或五官定位,保留原图其他内容。", "完整移除原目标的轮廓、腿和残影,保持原物件的落地位置。", "禁止添加图1原本没有的人物、物件、文字或标记。只输出一张与图1原构图对齐的图片。", "指令:" + request.displayPrompt, ].join("\n"), referenceImageBuffers: cropped ? new Map([[reference, cropped]]) : undefined, localEditComposite: { sourceImageUrl: base, regions: [target], expandRegions: false, maskMode: "region", featherMinPx: 7, featherMaxPx: 20, featherRatio: 0.012, }, } satisfies CanvasGenerateImageRequest; }