import "server-only"; import sharp from "sharp"; import { generateText } from "ai"; import { getLanguageModelById } from "@/lib/ai/dynamic-provider"; import { getMultimodalAiModelsWithProvider } from "@/lib/db/queries"; import { fetchImageBufferFromUrl } from "@/lib/ai/rmbg/service"; import { LAYOUT_REPLACE_QUALITY_PROMPT, parseLayoutReplaceQuality } from "./layout-replace-quality"; import { LAYOUT_REPLACE_PRODUCT_PROMPT, parseLayoutReplaceProductAnalysis } from "./layout-replace-product-crop"; async function visionInput(buffer: Buffer) { const resized = await sharp(buffer, { failOn: "error" }).rotate() .resize(1400, 1400, { fit: "inside", withoutEnlargement: true }) .flatten({ background: "#ffffff" }).jpeg({ quality: 90 }).toBuffer(); return new URL(`data:image/jpeg;base64,${resized.toString("base64")}`); } async function inspectLayoutImages(buffers: Buffer[], prompt: string, parse: (text: string) => T): Promise { const candidates = (await getMultimodalAiModelsWithProvider()).filter(({ model }) => model.supportsVision && !model.supportsImageGen && !model.isSubModel ).sort((a, b) => Number(b.model.isDefault) - Number(a.model.isDefault) || Number(b.provider.type === "vip98k") - Number(a.provider.type === "vip98k")); if (!candidates.length) throw new Error("未配置可用的多模态质量检查模型,未放行未经检查的替换结果"); const images = await Promise.all(buffers.map(visionInput)); let lastError: unknown; for (const candidate of candidates.slice(0, 2)) { try { const model = await getLanguageModelById(candidate.model.id); if (!model || typeof model === "string") continue; const { text } = await generateText({ model: model as any, maxRetries: 0, maxOutputTokens: 1200, abortSignal: AbortSignal.timeout(60_000), messages: [{ role: "user", content: [ { type: "text", text: prompt }, ...images.map(image => ({ type: "image" as const, image })), ] }], }); return parse(text); } catch (error) { lastError = error; // The fallback is another read-only check, never another image generation. console.warn("[Canvas Layout Replace] quality verifier unavailable", { model: candidate.model.modelId, errorType: error instanceof Error ? error.name : "unknown", }); } } throw new Error("替换结果质量检查暂不可用,未将未经检查的图片标记为成功", { cause: lastError }); } export async function locateLayoutReplaceProduct(sourceBuffer: Buffer, templateBuffer: Buffer) { return inspectLayoutImages([templateBuffer, sourceBuffer], LAYOUT_REPLACE_PRODUCT_PROMPT, parseLayoutReplaceProductAnalysis); } export async function verifyCanvasLayoutReplacement(params: { sourceBuffer: Buffer; templateBuffer: Buffer; resultUrl: string; resultBuffer?: Buffer; sourceIdentity?: string; }) { const resultBuffer = params.resultBuffer || await fetchImageBufferFromUrl(params.resultUrl); const prompt = [LAYOUT_REPLACE_QUALITY_PROMPT, params.sourceIdentity ? `生成前识别的新商品结构(仅作核对线索,若与图2矛盾以图2实际图像为准):${params.sourceIdentity.slice(0, 600)}` : ""].filter(Boolean).join("\n"); return inspectLayoutImages([params.templateBuffer, params.sourceBuffer, resultBuffer], prompt, parseLayoutReplaceQuality); }