import "server-only"; import { createHash } from "node:crypto"; import sharp from "sharp"; import { generateText } from "ai"; import { getLanguageModelById } from "@/lib/ai/dynamic-provider"; import { getMultimodalAiModelsWithProvider } from "@/lib/db/queries"; import { fetchImageBufferFromUrl } from "@/lib/ai/rmbg/service"; import { LAYOUT_REPLACE_QUALITY_PROMPT, parseLayoutReplaceQuality } from "./layout-replace-quality"; import { LAYOUT_REPLACE_PRODUCT_PROMPT, parseLayoutReplaceProductAnalysis } from "./layout-replace-product-crop"; import { LAYOUT_REPLACE_TEXT_PROMPT, parseLayoutReplaceTextBoxes, LAYOUT_REPLACE_TEXT_CHECK_PROMPT, parseLayoutReplaceTextCheck } from "./layout-replace-text-lock"; import type { NormalizedReplacementBox } from "@/lib/ai/replacement-layout-lock"; async function visionInput(buffer: Buffer) { const resized = await sharp(buffer, { failOn: "error" }).rotate() .resize(1400, 1400, { fit: "inside", withoutEnlargement: true }) .flatten({ background: "#ffffff" }).jpeg({ quality: 90 }).toBuffer(); return new URL(`data:image/jpeg;base64,${resized.toString("base64")}`); } const unavailableUntil = new Map(); async function inspectLayoutImages(buffers: Buffer[], prompt: string, parse: (text: string) => T): Promise { const candidates = (await getMultimodalAiModelsWithProvider()).filter(({ model }) => model.supportsVision && !model.supportsImageGen && !model.isSubModel ).sort((a, b) => Number(b.model.isDefault) - Number(a.model.isDefault) || Number(b.provider.type === "vip98k") - Number(a.provider.type === "vip98k")); const available = candidates.filter(c => (unavailableUntil.get(c.model.id) || 0) <= Date.now()); if (!available.length) throw new Error("多模态检查服务暂不可用,未放行未经检查的替换结果"); const primary = available[0]; // Two model names on the same account do not provide an independent fallback // when that provider resets connections or exhausts its quota. const independent = available.find(c => c.provider.id !== primary.provider.id && c.model.healthStatus === "healthy"); const fallback = independent || available[1]; const selected = fallback ? [primary, fallback] : [primary]; const images = await Promise.all(buffers.map(visionInput)); let lastError: unknown; for (const candidate of selected) { try { const model = await getLanguageModelById(candidate.model.id); if (!model || typeof model === "string") continue; const { text } = await generateText({ model: model as any, maxRetries: 0, maxOutputTokens: 1200, abortSignal: AbortSignal.timeout(60_000), messages: [{ role: "user", content: [ { type: "text", text: prompt }, ...images.map(image => ({ type: "image" as const, image })), ] }], }); const parsed = parse(text); console.info("[Canvas Layout Replace] vision check completed", { model: candidate.model.modelId, imageCount: buffers.length }); return parsed; } catch (error) { lastError = error; if (unavailableUntil.size >= 64) unavailableUntil.delete(unavailableUntil.keys().next().value!); unavailableUntil.set(candidate.model.id, Date.now() + 60_000); // The fallback is another read-only check, never another image generation. console.warn("[Canvas Layout Replace] quality verifier unavailable", { model: candidate.model.modelId, errorType: error instanceof Error ? error.name : "unknown", }); } } throw new Error("替换结果质量检查暂不可用,未将未经检查的图片标记为成功", { cause: lastError }); } export async function locateLayoutReplaceProduct(sourceBuffer: Buffer, templateBuffer: Buffer) { return inspectLayoutImages([templateBuffer, sourceBuffer], LAYOUT_REPLACE_PRODUCT_PROMPT, parseLayoutReplaceProductAnalysis); } const textRegions = new Map }>(); function inspectTemplateText(buffer: Buffer) { return inspectLayoutImages([buffer], LAYOUT_REPLACE_TEXT_PROMPT, parseLayoutReplaceTextBoxes); } export async function locateLayoutReplaceTemplateText(buffer: Buffer) { const key = createHash("sha256").update(buffer).digest("hex"); const cached = textRegions.get(key); if (cached && cached.expires > Date.now()) return cached.result; if (textRegions.size >= 32) textRegions.delete(textRegions.keys().next().value!); const result = inspectTemplateText(buffer); const entry = { expires: Date.now() + 30 * 60_000, result }; textRegions.set(key, entry); result.catch(() => { if (textRegions.get(key) === entry) textRegions.delete(key); }); return result; } export async function verifyCanvasLayoutReplacement(params: { sourceBuffer: Buffer; templateBuffer: Buffer; resultUrl: string; resultBuffer?: Buffer; sourceIdentity?: string; productBuffer?: Buffer; textBoundingBoxes?: NormalizedReplacementBox[]; }) { const resultBuffer = params.resultBuffer || await fetchImageBufferFromUrl(params.resultUrl); const prompt = [LAYOUT_REPLACE_QUALITY_PROMPT, params.productBuffer ? "图4是图2新商品的放大裁剪,仅用于看清结构细节。逐项核对图4与图3中的开放格、分隔板、门板和下缘支撑,增减任何部件都不合格;图2原图仍为最终依据。" : "", params.sourceIdentity ? `生成前识别的新商品结构(仅作核对线索,若与图2矛盾以图2实际图像为准):${params.sourceIdentity.slice(0, 600)}` : ""].filter(Boolean).join("\n"); const quality = await inspectLayoutImages([params.templateBuffer, params.sourceBuffer, resultBuffer, ...(params.productBuffer ? [params.productBuffer] : [])], prompt, parseLayoutReplaceQuality); if (!quality.passed || !params.textBoundingBoxes?.length) return quality; const text = await verifyLayoutReplaceText(params.templateBuffer, resultBuffer, params.textBoundingBoxes); return { ...quality, passed: text.passed, templateTextPreserved: text.passed, confidence: Math.min(quality.confidence, text.confidence), reasons: text.passed ? quality.reasons : [...quality.reasons, "原模版文案校对未通过", ...text.differences] }; } export async function verifyLayoutReplaceText(template: Buffer, result: Buffer, boxes: NormalizedReplacementBox[]) { const boards = await makeTextComparisonBoards(template, result, boxes); return inspectLayoutImages(boards, LAYOUT_REPLACE_TEXT_CHECK_PROMPT, parseLayoutReplaceTextCheck); } async function makeTextComparisonBoards(template: Buffer, result: Buffer, boxes: NormalizedReplacementBox[]) { const metadata = await sharp(template).metadata(); const width = metadata.width!; const height = metadata.height!; const generated = await sharp(result).resize(width, height, { fit: "fill" }).png().toBuffer(); const boards: Buffer[] = []; // At most four legible boards per check; keep all regions for unusually dense // templates by increasing the number of rows, never silently dropping text. const rows = Math.max(6, Math.ceil(boxes.length / 4)); for (let start = 0; start < boxes.length; start += rows) { const overlays: { input: Buffer; left: number; top: number }[] = []; const group = boxes.slice(start, start + rows); for (const [index, box] of group.entries()) { const left = Math.max(0, Math.floor(box[0] / 1000 * width) - 24); const top = Math.max(0, Math.floor(box[1] / 1000 * height) - 24); const right = Math.min(width, Math.ceil(box[2] / 1000 * width) + 24); const bottom = Math.min(height, Math.ceil(box[3] / 1000 * height) + 24); for (const [column, buffer] of [template, generated].entries()) { const input = await sharp(buffer).extract({ left, top, width: right-left, height: bottom-top }) .resize(680, 150, { fit: "contain", background: "#ffffff" }).png().toBuffer(); overlays.push({ input, left: 10 + column * 690, top: 10 + index * 160 }); } } boards.push(await sharp({ create: { width: 1390, height: 10 + group.length * 160, channels: 3, background: "#d9d9d9" } }).composite(overlays).png().toBuffer()); } return boards; }