import "server-only"; import { createHash } from "node:crypto"; import sharp from "sharp"; import { generateText } from "ai"; import { getLanguageModelById } from "@/lib/ai/dynamic-provider"; import { getMultimodalAiModelsWithProvider } from "@/lib/db/queries"; import { fetchImageBufferFromUrl } from "@/lib/ai/rmbg/service"; import { LAYOUT_REPLACE_QUALITY_PROMPT, parseLayoutReplaceQuality } from "./layout-replace-quality"; import { LAYOUT_REPLACE_PRODUCT_PROMPT, parseLayoutReplaceProductAnalysis } from "./layout-replace-product-crop"; import { LAYOUT_REPLACE_TEXT_PROMPT, parseLayoutReplaceTextBoxes, LAYOUT_REPLACE_TEXT_CHECK_PROMPT, parseLayoutReplaceTextCheck } from "./layout-replace-text-lock"; import type { NormalizedReplacementBox } from "@/lib/ai/replacement-layout-lock"; import { makeLayoutReplaceTextBoards } from "./layout-replace-text-board"; import { makeLayoutReplaceQualityBoard } from "./layout-replace-quality-board"; import { LAYOUT_REPLACE_SUPPORT_PROMPT, parseLayoutReplaceSupport, compareLayoutReplaceSupport } from "./layout-replace-product-support"; async function visionInput(buffer: Buffer) { const resized = await sharp(buffer, { failOn: "error" }).rotate() .resize(1400, 1400, { fit: "inside", withoutEnlargement: true }) .flatten({ background: "#ffffff" }).jpeg({ quality: 90 }).toBuffer(); return new URL(`data:image/jpeg;base64,${resized.toString("base64")}`); } const unavailableUntil = new Map(); async function inspectLayoutImages(buffers: Buffer[], prompt: string, parse: (text: string) => T, labels?: string[]): Promise { const candidates = (await getMultimodalAiModelsWithProvider()).filter(({ model }) => model.supportsVision && !model.supportsImageGen && !model.isSubModel ).sort((a, b) => Number(b.model.isDefault) - Number(a.model.isDefault) || Number(b.provider.type === "vip98k") - Number(a.provider.type === "vip98k")); const available = candidates.filter(c => (unavailableUntil.get(c.model.id) || 0) <= Date.now()); if (!available.length) throw new Error("多模态检查服务暂不可用,未放行未经检查的替换结果"); const primary = available[0]; // Two model names on the same account do not provide an independent fallback // when that provider resets connections or exhausts its quota. const independent = available.find(c => c.provider.id !== primary.provider.id && c.model.healthStatus === "healthy"); const fallback = independent || available[1]; const selected = fallback ? [primary, fallback] : [primary]; const images = await Promise.all(buffers.map(visionInput)); let lastError: unknown; for (const candidate of selected) { try { const model = await getLanguageModelById(candidate.model.id); if (!model || typeof model === "string") continue; const { text } = await generateText({ model: model as any, maxRetries: 0, maxOutputTokens: 1200, abortSignal: AbortSignal.timeout(60_000), messages: [{ role: "user", content: [ { type: "text", text: prompt }, ...images.flatMap((image, index) => [ ...(labels?.[index] ? [{ type: "text" as const, text: labels[index] }] : []), { type: "image" as const, image }, ]), ] }], }); const parsed = parse(text); console.info("[Canvas Layout Replace] vision check completed", { model: candidate.model.modelId, imageCount: buffers.length }); return parsed; } catch (error) { lastError = error; if (unavailableUntil.size >= 64) unavailableUntil.delete(unavailableUntil.keys().next().value!); unavailableUntil.set(candidate.model.id, Date.now() + 3 * 60_000); // The fallback is another read-only check, never another image generation. console.warn("[Canvas Layout Replace] quality verifier unavailable", { model: candidate.model.modelId, errorType: error instanceof Error ? error.name : "unknown", }); } } throw new Error("替换结果质量检查暂不可用,未将未经检查的图片标记为成功", { cause: lastError }); } export async function locateLayoutReplaceProduct(sourceBuffer: Buffer, templateBuffer: Buffer) { return inspectLayoutImages([templateBuffer, sourceBuffer], LAYOUT_REPLACE_PRODUCT_PROMPT, parseLayoutReplaceProductAnalysis, ["图1:目标模版,含旧商品与原场景", "图2:新商品完整原图;商品身份与定位坐标均针对这张图"]); } const textRegions = new Map }>(); function inspectTemplateText(buffer: Buffer) { return inspectLayoutImages([buffer], LAYOUT_REPLACE_TEXT_PROMPT, parseLayoutReplaceTextBoxes); } export async function locateLayoutReplaceTemplateText(buffer: Buffer) { const key = createHash("sha256").update(buffer).digest("hex"); const cached = textRegions.get(key); if (cached && cached.expires > Date.now()) return cached.result; if (textRegions.size >= 32) textRegions.delete(textRegions.keys().next().value!); const result = inspectTemplateText(buffer); const entry = { expires: Date.now() + 30 * 60_000, result }; textRegions.set(key, entry); result.catch(() => { if (textRegions.get(key) === entry) textRegions.delete(key); }); return result; } const supportChecks = new Map }>(); function inspectProductSupport(buffer: Buffer) { return inspectLayoutImages([buffer], LAYOUT_REPLACE_SUPPORT_PROMPT, parseLayoutReplaceSupport); } export async function inspectLayoutReplaceSupport(buffer: Buffer) { const key = createHash("sha256").update(buffer).digest("hex"); const cached = supportChecks.get(key); if (cached && cached.expires > Date.now()) return cached.result; if (supportChecks.size >= 64) supportChecks.delete(supportChecks.keys().next().value!); const result = inspectProductSupport(buffer); const entry = { expires: Date.now() + 30 * 60_000, result }; supportChecks.set(key, entry); result.catch(() => { if (supportChecks.get(key) === entry) supportChecks.delete(key); }); return result; } export async function verifyCanvasLayoutReplacement(params: { sourceBuffer: Buffer; templateBuffer: Buffer; resultUrl: string; resultBuffer?: Buffer; sourceIdentity?: string; productBuffer?: Buffer; textBoundingBoxes?: NormalizedReplacementBox[]; detailedChecks?: boolean; }) { const resultBuffer = params.resultBuffer || await fetchImageBufferFromUrl(params.resultUrl); const prompt = [LAYOUT_REPLACE_QUALITY_PROMPT, params.sourceIdentity ? `新商品结构辅助线索(与B原图矛盾时以B为准):${params.sourceIdentity.slice(0, 600)}` : ""].filter(Boolean).join("\n"); const board = await makeLayoutReplaceQualityBoard({ template: params.templateBuffer, source: params.sourceBuffer, product: params.productBuffer || params.sourceBuffer, result: resultBuffer }); const quality = await inspectLayoutImages([board], prompt, parseLayoutReplaceQuality); // Fine-grained OCR/support classification is experimental: real fixtures // showed provider-dependent false rejections. Keep it opt-in until stable. if (params.detailedChecks === false) return quality; if (quality.sourceProductPreserved) { // Read the actual result product at the same scale as the source crop. // Poster copy and floor shadows must not obscure its support structure. const { box } = await locateLayoutReplaceProduct(resultBuffer, params.templateBuffer); const metadata = await sharp(resultBuffer).metadata(); const { layoutReplaceProductCrop } = await import("./layout-replace-product-crop"); const resultProduct = await sharp(resultBuffer) .extract(layoutReplaceProductCrop(box, metadata.width!, metadata.height!)).png().toBuffer(); const [sourceSupport, resultSupport] = await Promise.all([ inspectLayoutReplaceSupport(params.productBuffer || params.sourceBuffer), inspectLayoutReplaceSupport(resultProduct), ]); const support = compareLayoutReplaceSupport(sourceSupport, resultSupport); if (!support.passed) return { ...quality, passed: false, sourceProductPreserved: false, confidence: Math.min(quality.confidence, sourceSupport.confidence, resultSupport.confidence), reasons: [...quality.reasons, support.reason] }; } if (!quality.passed || !params.textBoundingBoxes?.length) return quality; const text = await verifyLayoutReplaceText(params.templateBuffer, resultBuffer, params.textBoundingBoxes); return { ...quality, passed: text.passed, templateTextPreserved: text.passed, confidence: Math.min(quality.confidence, text.confidence), reasons: text.passed ? quality.reasons : [...quality.reasons, "原模版文案校对未通过", ...text.differences] }; } export async function verifyLayoutReplaceText(template: Buffer, result: Buffer, boxes: NormalizedReplacementBox[]) { const boards = await makeLayoutReplaceTextBoards(template, result, boxes); const checks = await Promise.all(boards.map(board => inspectLayoutImages([board], LAYOUT_REPLACE_TEXT_CHECK_PROMPT, parseLayoutReplaceTextCheck))); return { passed: checks.every(check => check.passed), confidence: Math.min(...checks.map(check => check.confidence)), differences: [...new Set(checks.flatMap(check => check.differences))].slice(0, 8) }; }