import "server-only"; import sharp from "sharp"; import { callQwen3Vl } from "@/lib/ai/qwen3-vl"; import { fetchImageBufferFromUrl } from "@/lib/ai/rmbg/service"; import type { CanvasGenerateImageRequest, CanvasLocalEditCompositeRegion } from "./generate-image"; export function parseBatchInstructionRegion(value: unknown): CanvasLocalEditCompositeRegion | null { if (!Array.isArray(value) || value.length !== 4 || !value.every( point => typeof point === "number" && Number.isFinite(point) && point >= 0 && point <= 1000, )) return null; const [left, top, right, bottom] = value; if (right - left < 10 || bottom - top < 10) return null; return { x: left / 1000, y: top / 1000, width: (right - left) / 1000, height: (bottom - top) / 1000, }; } function faceFeaturesRegion(points: unknown) { if (!Array.isArray(points) || points.length !== 6 || !points.every( point => typeof point === "number" && Number.isFinite(point) && point >= 0 && point <= 1000, )) return null; const [leftX, leftY, rightX, rightY, mouthX, mouthY] = points; const span = Math.abs(rightX - leftX); const eyeY = (leftY + rightY) / 2; const eyeX = (leftX + rightX) / 2; const height = mouthY - eyeY; if (span < 10 || height < 10 || Math.abs(mouthX - eyeX) > span) return null; const center = (2 * eyeX + mouthX) / 3; return parseBatchInstructionRegion([ Math.max(0, center - span * 0.86), Math.max(0, Math.min(leftY, rightY) - height * 0.53), Math.min(1000, center + span * 0.86), Math.min(1000, mouthY + height * 0.34), ]); } function resizeEditRegion(region: CanvasLocalEditCompositeRegion, face: boolean) { const xMargin = region.width * (face ? -0.08 : 0.11); const yMargin = region.height * (face ? -0.11 : 0.11); const x = Math.max(0, region.x - xMargin); const y = Math.max(0, region.y - yMargin); return { x, y, width: Math.min(1, region.x + region.width + xMargin) - x, height: Math.min(1, region.y + region.height + yMargin) - y, }; } async function cropReference(url: string, region: CanvasLocalEditCompositeRegion) { const bytes = await fetchImageBufferFromUrl(url); const oriented = await sharp(bytes).autoOrient().png().toBuffer({ resolveWithObject: true }); const { width, height } = oriented.info; const left = Math.floor(region.x * width); const top = Math.floor(region.y * height); return sharp(oriented.data).extract({ left, top, width: Math.min(width - left, Math.ceil(region.width * width)), height: Math.min(height - top, Math.ceil(region.height * height)), }).resize({ width: 1280, height: 1280, fit: "inside", withoutEnlargement: true }).png().toBuffer(); } export async function prepareBatchInstructionWithReference(request: CanvasGenerateImageRequest) { const [base, reference] = request.referenceImages || []; if (!base || !reference) throw new Error("局部指令编辑缺少原图或共用参考图"); let text: string; try { ({ text } = await callQwen3Vl({ imageUrls: [base, reference], temperature: 0, maxTokens: 280, timeoutMs: 25_000, retries: 0, prompt: [ "定位图片编辑目标。图1是要编辑的原图,图2只提供参照特征。", "只输出 JSON,格式为 {\"scope\":\"local\",\"base_box\":[x1,y1,x2,y2],\"reference_box\":[x1,y1,x2,y2]}。", "两个框分别使用各自图片的 0 至 999 归一化坐标。", "换脸或换五官时,将 scope 设为 face。两个框只框人脸内部的眉、眼、鼻、嘴,不包含头发、颈部、衣服和背景。", "换脸时再返回 base_face 和 reference_face;每个数组分别为 [左眼中心x,左眼中心y,右眼中心x,右眼中心y,嘴中心x,嘴中心y]。", "替换商品时,图1框住指定旧商品的完整外轮廓,包括扶手和腿。图2框住替换商品,勿框入其他人或物。", "只有指令明确要求修改整张图的整体风格、色调或场景时,返回 {\"scope\":\"full\"}。", "找不到明确目标,或多张脸、多件商品无法确定哪一个时,返回 {\"scope\":\"unresolved\"}。", "用户指令:" + request.displayPrompt, ].join("\n"), })); } catch { throw new Error("暂时无法识别要编辑的区域,生图任务尚未提交,请稍后重试"); } let result: { scope?: string; base_box?: unknown; reference_box?: unknown; base_face?: unknown; reference_face?: unknown } = {}; try { const match = text.match(/\{[\s\S]*\}/); if (match) result = JSON.parse(match[0]) || {}; } catch { /* Invalid or ambiguous localization is reported below. */ } const requiresLocalTarget = /换脸|五官|脸部|面部|换商品|换产品|(?:商品|产品|主体).{0,8}替换|替换.{0,10}(?:商品|产品|主体|模特)|face swap|replace (?:the )?(?:face|product)/i.test(request.displayPrompt || ""); if (result.scope === "full" && !requiresLocalTarget) return request; const baseRegion = parseBatchInstructionRegion(result.base_box); const referenceRegion = parseBatchInstructionRegion(result.reference_box); if ((result.scope !== "local" && result.scope !== "face") || !baseRegion || !referenceRegion) { throw new Error("未能定位唯一替换目标。请在指令里描述位置或特征,再重试这张图片"); } // Alibaba can use the cropped bytes directly without writing a new reference object. const face = result.scope === "face" || /换脸|五官|face swap/i.test(request.displayPrompt || ""); const target = (face && faceFeaturesRegion(result.base_face)) || resizeEditRegion(baseRegion, face); const referenceCrop = face ? faceFeaturesRegion(result.reference_face) || resizeEditRegion(referenceRegion, true) : referenceRegion; const cropped = await cropReference(reference, referenceCrop).catch(() => null); return { ...request, prompt: [ "这是局部编辑。以第一张图为唯一底图。图2只提供目标特征。", "把修改放回原目标的位置和大小范围内。保持原目标的中心、落地位置或五官定位,保留原图其他内容。", face ? "仅对齐图1的眉毛、眼睛、鼻子和嘴。保持两眼及嘴的位置,不改变头发、发际线、肤色和脸的位置。" : "完整移除原目标的轮廓、腿和残影,保持原物件的落地位置。", "禁止添加图1原本没有的人物、物件、文字或标记。只输出一张与图1原构图对齐的图片。", "指令:" + request.displayPrompt, ].join("\n"), referenceImageBuffers: cropped ? new Map([[reference, cropped]]) : undefined, localEditComposite: { sourceImageUrl: base, regions: [target], expandRegions: false, maskMode: "region", maskShape: face ? "ellipse" : undefined, featherMinPx: face ? 3 : 7, featherMaxPx: face ? 10 : 20, featherRatio: face ? 0.007 : 0.012, }, } satisfies CanvasGenerateImageRequest; }