156 lines
9.1 KiB
TypeScript
156 lines
9.1 KiB
TypeScript
import "server-only";
|
||
import { createHash } from "node:crypto";
|
||
import sharp from "sharp";
|
||
import { generateText } from "ai";
|
||
import { getLanguageModelById } from "@/lib/ai/dynamic-provider";
|
||
import { getMultimodalAiModelsWithProvider } from "@/lib/db/queries";
|
||
import { fetchImageBufferFromUrl } from "@/lib/ai/rmbg/service";
|
||
import { LAYOUT_REPLACE_QUALITY_PROMPT, parseLayoutReplaceQuality } from "./layout-replace-quality";
|
||
import { LAYOUT_REPLACE_PRODUCT_PROMPT, parseLayoutReplaceProductAnalysis } from "./layout-replace-product-crop";
|
||
import { LAYOUT_REPLACE_TEXT_PROMPT, parseLayoutReplaceTextBoxes, LAYOUT_REPLACE_TEXT_CHECK_PROMPT, parseLayoutReplaceTextCheck } from "./layout-replace-text-lock";
|
||
import type { NormalizedReplacementBox } from "@/lib/ai/replacement-layout-lock";
|
||
import { makeLayoutReplaceTextBoards } from "./layout-replace-text-board";
|
||
import { makeLayoutReplaceQualityBoard } from "./layout-replace-quality-board";
|
||
import { LAYOUT_REPLACE_SUPPORT_PROMPT, parseLayoutReplaceSupport, compareLayoutReplaceSupport } from "./layout-replace-product-support";
|
||
|
||
async function visionInput(buffer: Buffer) {
|
||
const resized = await sharp(buffer, { failOn: "error" }).rotate()
|
||
.resize(1400, 1400, { fit: "inside", withoutEnlargement: true })
|
||
.flatten({ background: "#ffffff" }).jpeg({ quality: 90 }).toBuffer();
|
||
return new URL(`data:image/jpeg;base64,${resized.toString("base64")}`);
|
||
}
|
||
|
||
const unavailableUntil = new Map<string, number>();
|
||
async function inspectLayoutImages<T>(buffers: Buffer[], prompt: string, parse: (text: string) => T, labels?: string[]): Promise<T> {
|
||
const candidates = (await getMultimodalAiModelsWithProvider()).filter(({ model }) =>
|
||
model.supportsVision && !model.supportsImageGen && !model.isSubModel
|
||
).sort((a, b) => Number(b.model.isDefault) - Number(a.model.isDefault) ||
|
||
Number(b.provider.type === "vip98k") - Number(a.provider.type === "vip98k"));
|
||
const available = candidates.filter(c => (unavailableUntil.get(c.model.id) || 0) <= Date.now());
|
||
if (!available.length) throw new Error("多模态检查服务暂不可用，未放行未经检查的替换结果");
|
||
const primary = available[0];
|
||
// Two model names on the same account do not provide an independent fallback
|
||
// when that provider resets connections or exhausts its quota.
|
||
const independent = available.find(c => c.provider.id !== primary.provider.id && c.model.healthStatus === "healthy");
|
||
const fallback = independent || available[1];
|
||
const selected = fallback ? [primary, fallback] : [primary];
|
||
const images = await Promise.all(buffers.map(visionInput));
|
||
let lastError: unknown;
|
||
for (const candidate of selected) {
|
||
try {
|
||
const model = await getLanguageModelById(candidate.model.id);
|
||
if (!model || typeof model === "string") continue;
|
||
const { text } = await generateText({
|
||
model: model as any, maxRetries: 0, maxOutputTokens: 1200,
|
||
abortSignal: AbortSignal.timeout(60_000),
|
||
messages: [{ role: "user", content: [
|
||
{ type: "text", text: prompt },
|
||
...images.flatMap((image, index) => [
|
||
...(labels?.[index] ? [{ type: "text" as const, text: labels[index] }] : []),
|
||
{ type: "image" as const, image },
|
||
]),
|
||
] }],
|
||
});
|
||
const parsed = parse(text);
|
||
console.info("[Canvas Layout Replace] vision check completed", { model: candidate.model.modelId, imageCount: buffers.length });
|
||
return parsed;
|
||
} catch (error) {
|
||
lastError = error;
|
||
if (unavailableUntil.size >= 64) unavailableUntil.delete(unavailableUntil.keys().next().value!);
|
||
unavailableUntil.set(candidate.model.id, Date.now() + 3 * 60_000);
|
||
// The fallback is another read-only check, never another image generation.
|
||
console.warn("[Canvas Layout Replace] quality verifier unavailable", {
|
||
model: candidate.model.modelId, errorType: error instanceof Error ? error.name : "unknown",
|
||
});
|
||
}
|
||
}
|
||
throw new Error("替换结果质量检查暂不可用，未将未经检查的图片标记为成功", { cause: lastError });
|
||
}
|
||
|
||
export async function locateLayoutReplaceProduct(sourceBuffer: Buffer, templateBuffer: Buffer) {
|
||
return inspectLayoutImages([templateBuffer, sourceBuffer], LAYOUT_REPLACE_PRODUCT_PROMPT, parseLayoutReplaceProductAnalysis,
|
||
["图1：目标模版，含旧商品与原场景", "图2：新商品完整原图；商品身份与定位坐标均针对这张图"]);
|
||
}
|
||
|
||
const textRegions = new Map<string, { expires: number; result: ReturnType<typeof inspectTemplateText> }>();
|
||
function inspectTemplateText(buffer: Buffer) {
|
||
return inspectLayoutImages([buffer], LAYOUT_REPLACE_TEXT_PROMPT, parseLayoutReplaceTextBoxes);
|
||
}
|
||
export async function locateLayoutReplaceTemplateText(buffer: Buffer) {
|
||
const key = createHash("sha256").update(buffer).digest("hex");
|
||
const cached = textRegions.get(key);
|
||
if (cached && cached.expires > Date.now()) return cached.result;
|
||
if (textRegions.size >= 32) textRegions.delete(textRegions.keys().next().value!);
|
||
const result = inspectTemplateText(buffer);
|
||
const entry = { expires: Date.now() + 30 * 60_000, result };
|
||
textRegions.set(key, entry);
|
||
result.catch(() => { if (textRegions.get(key) === entry) textRegions.delete(key); });
|
||
return result;
|
||
}
|
||
|
||
const supportChecks = new Map<string, { expires: number; result: ReturnType<typeof inspectProductSupport> }>();
|
||
function inspectProductSupport(buffer: Buffer) {
|
||
return inspectLayoutImages([buffer], LAYOUT_REPLACE_SUPPORT_PROMPT, parseLayoutReplaceSupport);
|
||
}
|
||
export async function inspectLayoutReplaceSupport(buffer: Buffer) {
|
||
const key = createHash("sha256").update(buffer).digest("hex");
|
||
const cached = supportChecks.get(key);
|
||
if (cached && cached.expires > Date.now()) return cached.result;
|
||
if (supportChecks.size >= 64) supportChecks.delete(supportChecks.keys().next().value!);
|
||
const result = inspectProductSupport(buffer);
|
||
const entry = { expires: Date.now() + 30 * 60_000, result };
|
||
supportChecks.set(key, entry);
|
||
result.catch(() => { if (supportChecks.get(key) === entry) supportChecks.delete(key); });
|
||
return result;
|
||
}
|
||
|
||
export async function verifyCanvasLayoutReplacement(params: {
|
||
sourceBuffer: Buffer;
|
||
templateBuffer: Buffer;
|
||
resultUrl: string;
|
||
resultBuffer?: Buffer;
|
||
sourceIdentity?: string;
|
||
productBuffer?: Buffer;
|
||
textBoundingBoxes?: NormalizedReplacementBox[];
|
||
detailedChecks?: boolean;
|
||
}) {
|
||
const resultBuffer = params.resultBuffer || await fetchImageBufferFromUrl(params.resultUrl);
|
||
const prompt = [LAYOUT_REPLACE_QUALITY_PROMPT, params.sourceIdentity
|
||
? `新商品结构辅助线索（与B原图矛盾时以B为准）：${params.sourceIdentity.slice(0, 600)}` : ""].filter(Boolean).join("\n");
|
||
const board = await makeLayoutReplaceQualityBoard({ template: params.templateBuffer, source: params.sourceBuffer,
|
||
product: params.productBuffer || params.sourceBuffer, result: resultBuffer });
|
||
const quality = await inspectLayoutImages([board], prompt, parseLayoutReplaceQuality);
|
||
// Fine-grained OCR/support classification is experimental: real fixtures
|
||
// showed provider-dependent false rejections. Keep it opt-in until stable.
|
||
if (params.detailedChecks === false) return quality;
|
||
if (quality.sourceProductPreserved) {
|
||
// Read the actual result product at the same scale as the source crop.
|
||
// Poster copy and floor shadows must not obscure its support structure.
|
||
const { box } = await locateLayoutReplaceProduct(resultBuffer, params.templateBuffer);
|
||
const metadata = await sharp(resultBuffer).metadata();
|
||
const { layoutReplaceProductCrop } = await import("./layout-replace-product-crop");
|
||
const resultProduct = await sharp(resultBuffer)
|
||
.extract(layoutReplaceProductCrop(box, metadata.width!, metadata.height!)).png().toBuffer();
|
||
const [sourceSupport, resultSupport] = await Promise.all([
|
||
inspectLayoutReplaceSupport(params.productBuffer || params.sourceBuffer), inspectLayoutReplaceSupport(resultProduct),
|
||
]);
|
||
const support = compareLayoutReplaceSupport(sourceSupport, resultSupport);
|
||
if (!support.passed) return { ...quality, passed: false, sourceProductPreserved: false,
|
||
confidence: Math.min(quality.confidence, sourceSupport.confidence, resultSupport.confidence),
|
||
reasons: [...quality.reasons, support.reason] };
|
||
}
|
||
if (!quality.passed || !params.textBoundingBoxes?.length) return quality;
|
||
const text = await verifyLayoutReplaceText(params.templateBuffer, resultBuffer, params.textBoundingBoxes);
|
||
return { ...quality, passed: text.passed, templateTextPreserved: text.passed,
|
||
confidence: Math.min(quality.confidence, text.confidence),
|
||
reasons: text.passed ? quality.reasons : [...quality.reasons, "原模版文案校对未通过", ...text.differences] };
|
||
}
|
||
|
||
export async function verifyLayoutReplaceText(template: Buffer, result: Buffer, boxes: NormalizedReplacementBox[]) {
|
||
const boards = await makeLayoutReplaceTextBoards(template, result, boxes);
|
||
const checks = await Promise.all(boards.map(board => inspectLayoutImages([board], LAYOUT_REPLACE_TEXT_CHECK_PROMPT, parseLayoutReplaceTextCheck)));
|
||
return { passed: checks.every(check => check.passed), confidence: Math.min(...checks.map(check => check.confidence)),
|
||
differences: [...new Set(checks.flatMap(check => check.differences))].slice(0, 8) };
|
||
}
|
||
|