fix(canvas): independently verify product support structure before accepting replacements
This commit is contained in:
parent
3614828bce
commit
7a07742729
27
lib/canvas/layout-replace-product-support.ts
Normal file
27
lib/canvas/layout-replace-product-support.ts
Normal file
@ -0,0 +1,27 @@
|
||||
export const LAYOUT_REPLACE_SUPPORT_PROMPT = [
|
||||
"只观察这张图中主体商品本身的下缘与支撑方式,不比较其他图片,不根据商品类别或广告文案猜测。",
|
||||
"supportType只能选:flush=连续底座直接贴地;legs=可见独立柜腿、支撑脚或脚轮使柜体离地;suspended=明确悬挂;not_applicable=该类别不适用支撑结构检查(如服装、食品);unclear=无法看清。",
|
||||
"贴地底座的接触阴影不是腿。只有实际看到独立支撑才选legs,不要凭空想象隐藏的脚。关注实际商品,忽略海报图标、数字及其他独立摆件。",
|
||||
'只返回JSON,supportType填写上述英文枚举之一:{"supportType":"从枚举中选择","confidence":0.95}。',
|
||||
].join("\n");
|
||||
|
||||
export type LayoutReplaceSupport = "flush" | "legs" | "suspended" | "not_applicable" | "unclear";
|
||||
|
||||
export function parseLayoutReplaceSupport(text: string) {
|
||||
let value: { supportType?: unknown; confidence?: unknown };
|
||||
try { value = JSON.parse(text.trim().replace(/^```(?:json)?\s*/i, "").replace(/\s*```$/, "")); }
|
||||
catch { throw new Error("商品支撑结构检查未返回有效结论"); }
|
||||
if (!value || !["flush", "legs", "suspended", "not_applicable", "unclear"].includes(String(value.supportType)) ||
|
||||
typeof value.confidence !== "number" || value.confidence < 0 || value.confidence > 1) throw new Error("商品支撑结构检查不完整");
|
||||
return { supportType: value.supportType as LayoutReplaceSupport, confidence: value.confidence };
|
||||
}
|
||||
|
||||
export function compareLayoutReplaceSupport(source: ReturnType<typeof parseLayoutReplaceSupport>, result: ReturnType<typeof parseLayoutReplaceSupport>) {
|
||||
if (source.supportType === "not_applicable") return { passed: true, reason: "" };
|
||||
if (source.supportType === "unclear" || result.supportType === "unclear" || source.confidence < .8 || result.confidence < .8) {
|
||||
return { passed: false, reason: "无法可靠确认商品下缘的支撑结构" };
|
||||
}
|
||||
const names = { flush: "贴地底座", legs: "独立腿脚或脚轮", suspended: "悬挂", not_applicable: "类别不适用", unclear: "不清晰" };
|
||||
return source.supportType === result.supportType ? { passed: true, reason: "" }
|
||||
: { passed: false, reason: `原商品为${names[source.supportType]},结果却为${names[result.supportType]},支撑结构已改变` };
|
||||
}
|
||||
@ -10,6 +10,7 @@ import { LAYOUT_REPLACE_PRODUCT_PROMPT, parseLayoutReplaceProductAnalysis } from
|
||||
import { LAYOUT_REPLACE_TEXT_PROMPT, parseLayoutReplaceTextBoxes, LAYOUT_REPLACE_TEXT_CHECK_PROMPT, parseLayoutReplaceTextCheck } from "./layout-replace-text-lock";
|
||||
import type { NormalizedReplacementBox } from "@/lib/ai/replacement-layout-lock";
|
||||
import { makeLayoutReplaceQualityBoard } from "./layout-replace-quality-board";
|
||||
import { LAYOUT_REPLACE_SUPPORT_PROMPT, parseLayoutReplaceSupport, compareLayoutReplaceSupport } from "./layout-replace-product-support";
|
||||
|
||||
async function visionInput(buffer: Buffer) {
|
||||
const resized = await sharp(buffer, { failOn: "error" }).rotate()
|
||||
@ -86,6 +87,22 @@ export async function locateLayoutReplaceTemplateText(buffer: Buffer) {
|
||||
return result;
|
||||
}
|
||||
|
||||
const supportChecks = new Map<string, { expires: number; result: ReturnType<typeof inspectProductSupport> }>();
|
||||
function inspectProductSupport(buffer: Buffer) {
|
||||
return inspectLayoutImages([buffer], LAYOUT_REPLACE_SUPPORT_PROMPT, parseLayoutReplaceSupport);
|
||||
}
|
||||
export async function inspectLayoutReplaceSupport(buffer: Buffer) {
|
||||
const key = createHash("sha256").update(buffer).digest("hex");
|
||||
const cached = supportChecks.get(key);
|
||||
if (cached && cached.expires > Date.now()) return cached.result;
|
||||
if (supportChecks.size >= 64) supportChecks.delete(supportChecks.keys().next().value!);
|
||||
const result = inspectProductSupport(buffer);
|
||||
const entry = { expires: Date.now() + 30 * 60_000, result };
|
||||
supportChecks.set(key, entry);
|
||||
result.catch(() => { if (supportChecks.get(key) === entry) supportChecks.delete(key); });
|
||||
return result;
|
||||
}
|
||||
|
||||
export async function verifyCanvasLayoutReplacement(params: {
|
||||
sourceBuffer: Buffer;
|
||||
templateBuffer: Buffer;
|
||||
@ -101,6 +118,15 @@ export async function verifyCanvasLayoutReplacement(params: {
|
||||
const board = await makeLayoutReplaceQualityBoard({ template: params.templateBuffer, source: params.sourceBuffer,
|
||||
product: params.productBuffer || params.sourceBuffer, result: resultBuffer });
|
||||
const quality = await inspectLayoutImages([board], prompt, parseLayoutReplaceQuality);
|
||||
if (quality.sourceProductPreserved) {
|
||||
const [sourceSupport, resultSupport] = await Promise.all([
|
||||
inspectLayoutReplaceSupport(params.productBuffer || params.sourceBuffer), inspectLayoutReplaceSupport(resultBuffer),
|
||||
]);
|
||||
const support = compareLayoutReplaceSupport(sourceSupport, resultSupport);
|
||||
if (!support.passed) return { ...quality, passed: false, sourceProductPreserved: false,
|
||||
confidence: Math.min(quality.confidence, sourceSupport.confidence, resultSupport.confidence),
|
||||
reasons: [...quality.reasons, support.reason] };
|
||||
}
|
||||
if (!quality.passed || !params.textBoundingBoxes?.length) return quality;
|
||||
const text = await verifyLayoutReplaceText(params.templateBuffer, resultBuffer, params.textBoundingBoxes);
|
||||
return { ...quality, passed: text.passed, templateTextPreserved: text.passed,
|
||||
|
||||
@ -6,6 +6,7 @@ import { runValidatedLayoutReplacement } from "@/lib/canvas/layout-replace-valid
|
||||
import { buildCanvasLayoutReplacePrompt, rankCanvasLayoutReplaceModels } from "@/lib/canvas/layout-replace-model-selection";
|
||||
import { parseLayoutReplaceProductBox, parseLayoutReplaceProductAnalysis, layoutReplaceProductCrop, layoutReplaceProductAspectRatio } from "@/lib/canvas/layout-replace-product-crop";
|
||||
import { makeLayoutReplaceQualityBoard } from "@/lib/canvas/layout-replace-quality-board";
|
||||
import { parseLayoutReplaceSupport, compareLayoutReplaceSupport } from "@/lib/canvas/layout-replace-product-support";
|
||||
import { parseLayoutReplaceTextBoxes, parseLayoutReplaceTextCheck } from "@/lib/canvas/layout-replace-text-lock";
|
||||
|
||||
const verdict = { sourceProductPreserved: true, templateLayoutPreserved: true,
|
||||
@ -117,6 +118,18 @@ test("structured OCR differences preserve numeric changes and discard unchanged
|
||||
assert.equal(equivalent.passed, true);
|
||||
});
|
||||
|
||||
test("independently observed support types reject added legs without confusing template legs with results", () => {
|
||||
const observed = (supportType: string, confidence = .95) => parseLayoutReplaceSupport(JSON.stringify({ supportType, confidence }));
|
||||
assert.equal(compareLayoutReplaceSupport(observed("flush"), observed("flush")).passed, true);
|
||||
assert.equal(compareLayoutReplaceSupport(observed("flush"), observed("legs")).passed, false);
|
||||
assert.equal(compareLayoutReplaceSupport(observed("legs"), observed("flush")).passed, false);
|
||||
assert.equal(compareLayoutReplaceSupport(observed("suspended"), observed("legs")).passed, false);
|
||||
assert.equal(compareLayoutReplaceSupport(observed("not_applicable"), observed("unclear")).passed, true);
|
||||
assert.equal(compareLayoutReplaceSupport(observed("flush"), observed("unclear")).passed, false);
|
||||
assert.equal(compareLayoutReplaceSupport(observed("flush"), observed("flush", .3)).passed, false);
|
||||
assert.throws(() => parseLayoutReplaceSupport('{"supportType":"maybe","confidence":0.95}'));
|
||||
});
|
||||
|
||||
test("quality rejection covers identity, background direction, copy loss, collage and low confidence", () => {
|
||||
assert.equal(quality().passed, true);
|
||||
for (const changes of [{ sourceProductPreserved: false }, { templateLayoutPreserved: false },
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user