138 lines
8.0 KiB
TypeScript
138 lines
8.0 KiB
TypeScript
import "server-only";
|
||
import { createHash } from "node:crypto";
|
||
import sharp from "sharp";
|
||
import { generateText } from "ai";
|
||
import { getLanguageModelById } from "@/lib/ai/dynamic-provider";
|
||
import { getMultimodalAiModelsWithProvider } from "@/lib/db/queries";
|
||
import { fetchImageBufferFromUrl } from "@/lib/ai/rmbg/service";
|
||
import { LAYOUT_REPLACE_QUALITY_PROMPT, parseLayoutReplaceQuality } from "./layout-replace-quality";
|
||
import { LAYOUT_REPLACE_PRODUCT_PROMPT, parseLayoutReplaceProductAnalysis } from "./layout-replace-product-crop";
|
||
import { LAYOUT_REPLACE_TEXT_PROMPT, parseLayoutReplaceTextBoxes, LAYOUT_REPLACE_TEXT_CHECK_PROMPT, parseLayoutReplaceTextCheck } from "./layout-replace-text-lock";
|
||
import type { NormalizedReplacementBox } from "@/lib/ai/replacement-layout-lock";
|
||
|
||
async function visionInput(buffer: Buffer) {
|
||
const resized = await sharp(buffer, { failOn: "error" }).rotate()
|
||
.resize(1400, 1400, { fit: "inside", withoutEnlargement: true })
|
||
.flatten({ background: "#ffffff" }).jpeg({ quality: 90 }).toBuffer();
|
||
return new URL(`data:image/jpeg;base64,${resized.toString("base64")}`);
|
||
}
|
||
|
||
const unavailableUntil = new Map<string, number>();
|
||
async function inspectLayoutImages<T>(buffers: Buffer[], prompt: string, parse: (text: string) => T): Promise<T> {
|
||
const candidates = (await getMultimodalAiModelsWithProvider()).filter(({ model }) =>
|
||
model.supportsVision && !model.supportsImageGen && !model.isSubModel
|
||
).sort((a, b) => Number(b.model.isDefault) - Number(a.model.isDefault) ||
|
||
Number(b.provider.type === "vip98k") - Number(a.provider.type === "vip98k"));
|
||
const available = candidates.filter(c => (unavailableUntil.get(c.model.id) || 0) <= Date.now());
|
||
if (!available.length) throw new Error("多模态检查服务暂不可用，未放行未经检查的替换结果");
|
||
const primary = available[0];
|
||
// Two model names on the same account do not provide an independent fallback
|
||
// when that provider resets connections or exhausts its quota.
|
||
const independent = available.find(c => c.provider.id !== primary.provider.id && c.model.healthStatus === "healthy");
|
||
const fallback = independent || available[1];
|
||
const selected = fallback ? [primary, fallback] : [primary];
|
||
const images = await Promise.all(buffers.map(visionInput));
|
||
let lastError: unknown;
|
||
for (const candidate of selected) {
|
||
try {
|
||
const model = await getLanguageModelById(candidate.model.id);
|
||
if (!model || typeof model === "string") continue;
|
||
const { text } = await generateText({
|
||
model: model as any, maxRetries: 0, maxOutputTokens: 1200,
|
||
abortSignal: AbortSignal.timeout(60_000),
|
||
messages: [{ role: "user", content: [
|
||
{ type: "text", text: prompt },
|
||
...images.map(image => ({ type: "image" as const, image })),
|
||
] }],
|
||
});
|
||
const parsed = parse(text);
|
||
console.info("[Canvas Layout Replace] vision check completed", { model: candidate.model.modelId, imageCount: buffers.length });
|
||
return parsed;
|
||
} catch (error) {
|
||
lastError = error;
|
||
if (unavailableUntil.size >= 64) unavailableUntil.delete(unavailableUntil.keys().next().value!);
|
||
unavailableUntil.set(candidate.model.id, Date.now() + 60_000);
|
||
// The fallback is another read-only check, never another image generation.
|
||
console.warn("[Canvas Layout Replace] quality verifier unavailable", {
|
||
model: candidate.model.modelId, errorType: error instanceof Error ? error.name : "unknown",
|
||
});
|
||
}
|
||
}
|
||
throw new Error("替换结果质量检查暂不可用，未将未经检查的图片标记为成功", { cause: lastError });
|
||
}
|
||
|
||
export async function locateLayoutReplaceProduct(sourceBuffer: Buffer, templateBuffer: Buffer) {
|
||
return inspectLayoutImages([templateBuffer, sourceBuffer], LAYOUT_REPLACE_PRODUCT_PROMPT, parseLayoutReplaceProductAnalysis);
|
||
}
|
||
|
||
const textRegions = new Map<string, { expires: number; result: ReturnType<typeof inspectTemplateText> }>();
|
||
function inspectTemplateText(buffer: Buffer) {
|
||
return inspectLayoutImages([buffer], LAYOUT_REPLACE_TEXT_PROMPT, parseLayoutReplaceTextBoxes);
|
||
}
|
||
export async function locateLayoutReplaceTemplateText(buffer: Buffer) {
|
||
const key = createHash("sha256").update(buffer).digest("hex");
|
||
const cached = textRegions.get(key);
|
||
if (cached && cached.expires > Date.now()) return cached.result;
|
||
if (textRegions.size >= 32) textRegions.delete(textRegions.keys().next().value!);
|
||
const result = inspectTemplateText(buffer);
|
||
const entry = { expires: Date.now() + 30 * 60_000, result };
|
||
textRegions.set(key, entry);
|
||
result.catch(() => { if (textRegions.get(key) === entry) textRegions.delete(key); });
|
||
return result;
|
||
}
|
||
|
||
export async function verifyCanvasLayoutReplacement(params: {
|
||
sourceBuffer: Buffer;
|
||
templateBuffer: Buffer;
|
||
resultUrl: string;
|
||
resultBuffer?: Buffer;
|
||
sourceIdentity?: string;
|
||
productBuffer?: Buffer;
|
||
textBoundingBoxes?: NormalizedReplacementBox[];
|
||
}) {
|
||
const resultBuffer = params.resultBuffer || await fetchImageBufferFromUrl(params.resultUrl);
|
||
const prompt = [LAYOUT_REPLACE_QUALITY_PROMPT, params.productBuffer
|
||
? "图4是图2新商品的放大裁剪，仅用于看清结构细节。逐项核对图4与图3中的开放格、分隔板、门板和下缘支撑，增减任何部件都不合格；图2原图仍为最终依据。" : "", params.sourceIdentity
|
||
? `生成前识别的新商品结构（仅作核对线索，若与图2矛盾以图2实际图像为准）：${params.sourceIdentity.slice(0, 600)}` : ""].filter(Boolean).join("\n");
|
||
const quality = await inspectLayoutImages([params.templateBuffer, params.sourceBuffer, resultBuffer,
|
||
...(params.productBuffer ? [params.productBuffer] : [])], prompt, parseLayoutReplaceQuality);
|
||
if (!quality.passed || !params.textBoundingBoxes?.length) return quality;
|
||
const text = await verifyLayoutReplaceText(params.templateBuffer, resultBuffer, params.textBoundingBoxes);
|
||
return { ...quality, passed: text.passed, templateTextPreserved: text.passed,
|
||
confidence: Math.min(quality.confidence, text.confidence),
|
||
reasons: text.passed ? quality.reasons : [...quality.reasons, "原模版文案校对未通过", ...text.differences] };
|
||
}
|
||
|
||
export async function verifyLayoutReplaceText(template: Buffer, result: Buffer, boxes: NormalizedReplacementBox[]) {
|
||
const boards = await makeTextComparisonBoards(template, result, boxes);
|
||
return inspectLayoutImages(boards, LAYOUT_REPLACE_TEXT_CHECK_PROMPT, parseLayoutReplaceTextCheck);
|
||
}
|
||
|
||
async function makeTextComparisonBoards(template: Buffer, result: Buffer, boxes: NormalizedReplacementBox[]) {
|
||
const metadata = await sharp(template).metadata();
|
||
const width = metadata.width!; const height = metadata.height!;
|
||
const generated = await sharp(result).resize(width, height, { fit: "fill" }).png().toBuffer();
|
||
const boards: Buffer[] = [];
|
||
// At most four legible boards per check; keep all regions for unusually dense
|
||
// templates by increasing the number of rows, never silently dropping text.
|
||
const rows = Math.max(6, Math.ceil(boxes.length / 4));
|
||
for (let start = 0; start < boxes.length; start += rows) {
|
||
const overlays: { input: Buffer; left: number; top: number }[] = [];
|
||
const group = boxes.slice(start, start + rows);
|
||
for (const [index, box] of group.entries()) {
|
||
const left = Math.max(0, Math.floor(box[0] / 1000 * width) - 24);
|
||
const top = Math.max(0, Math.floor(box[1] / 1000 * height) - 24);
|
||
const right = Math.min(width, Math.ceil(box[2] / 1000 * width) + 24);
|
||
const bottom = Math.min(height, Math.ceil(box[3] / 1000 * height) + 24);
|
||
for (const [column, buffer] of [template, generated].entries()) {
|
||
const input = await sharp(buffer).extract({ left, top, width: right-left, height: bottom-top })
|
||
.resize(680, 150, { fit: "contain", background: "#ffffff" }).png().toBuffer();
|
||
overlays.push({ input, left: 10 + column * 690, top: 10 + index * 160 });
|
||
}
|
||
}
|
||
boards.push(await sharp({ create: { width: 1390, height: 10 + group.length * 160,
|
||
channels: 3, background: "#d9d9d9" } }).composite(overlays).png().toBuffer());
|
||
}
|
||
return boards;
|
||
}
|