fix(bom-costing): 修正多模态事实结果契约

This commit is contained in:
WuYongJie 2026-08-01 19:59:57 +08:00
parent d9ebf7d9cb
commit 70a2d8eb55
3 changed files with 439 additions and 89 deletions

View File

@ -24,8 +24,23 @@ export const bomCostingImageFactKindSchema = z.enum([
])
const nullToUndefined = (value: unknown) => value === null ? undefined : value
function parseFiniteJsonNumber(value: unknown) {
if (typeof value === 'number') return value
if (typeof value !== 'string') return value
const normalized = value.trim().replace(/,/g, '')
if (!normalized) return undefined
if (!/^[+-]?(?:\d+(?:\.\d+)?|\.\d+)$/.test(normalized)) {
return undefined
}
const parsed = Number(normalized)
return Number.isFinite(parsed) ? parsed : undefined
}
const optionalFiniteNumberSchema = z.preprocess(
nullToUndefined,
(value) => parseFiniteJsonNumber(nullToUndefined(value)),
z.number().finite().optional(),
)
const optionalUnitSchema = z.preprocess(
@ -46,7 +61,10 @@ export const bomCostingExtractedFactSchema = z.object({
),
basis: z.enum(['IMAGE', 'SOURCE_TEXT']),
note: z.string().trim().min(1).max(1_000),
confidence: z.number().finite().min(0).max(1),
confidence: z.preprocess(
parseFiniteJsonNumber,
z.number().finite().min(0).max(1),
),
})
export const bomCostingImageFactRecordSchema = z.object({

View File

@ -221,6 +221,144 @@ describe('BOM 逐图事实提取', () => {
})
})
it('把模型返回的数值字符串规范为数字,非单值文本保留原文但不拒绝整批', async () => {
const singleImageSnapshot: MarketDetailSnapshot = {
...snapshot,
mainImages: ['https://img.example.com/numeric-fact.jpg'],
skuImages: [],
detailImages: [],
skus: snapshot.skus.map((sku) => ({ ...sku, imageUrl: null })),
}
let request: ReturnType<typeof readBatchRequest> | null = null
jest.spyOn(global, 'fetch').mockImplementation(async (_input, init) => {
request = readBatchRequest(init)
return createStreamingResponse(JSON.stringify({
images: [{
imageId: request.imageIds[0],
observedTexts: ['内空34.4cm', '整机74*80*62cm'],
facts: [{
factId: 'inner-width',
kind: 'DIMENSION',
subject: '侧面内空',
value: '内空34.4cm',
numericValue: '344',
unit: 'mm',
relatedSkuIds: [],
basis: 'SOURCE_TEXT',
note: '图片尺寸标注',
confidence: '0.96',
}, {
factId: 'overall-size-text',
kind: 'DIMENSION',
subject: '整机尺寸原文',
value: '74*80*62cm',
numericValue: '74*80*62',
unit: null,
relatedSkuIds: [],
basis: 'SOURCE_TEXT',
note: '这是多维尺寸,不应压成单一数值',
confidence: 0.9,
}],
}],
}))
})
const output = await extractBomCostingImageFacts({
config: providerConfig,
snapshot: singleImageSnapshot,
})
expect(request?.prompt.requiredOutputShape.images[0].facts[0]).toMatchObject({
numericValue: 344,
confidence: 0.85,
})
expect(request?.prompt.outputFieldRules.numericValue).toContain('JSON number')
expect(output.snapshot.imageFacts[0]?.facts).toEqual([
expect.objectContaining({
factId: 'inner-width',
numericValue: 344,
confidence: 0.96,
}),
expect.objectContaining({
factId: 'overall-size-text',
value: '74*80*62cm',
numericValue: undefined,
}),
])
})
it('同批部分图片格式异常时先保存有效图片,续跑只重新分析未完成图片', async () => {
process.env.BOM_COSTING_AI_FACT_BATCH_SIZE = '3'
const threeImageSnapshot: MarketDetailSnapshot = {
...snapshot,
mainImages: [
'https://img.example.com/partial-1.jpg',
'https://img.example.com/partial-2.jpg',
'https://img.example.com/partial-3.jpg',
],
skuImages: [],
detailImages: [],
skus: snapshot.skus.map((sku) => ({ ...sku, imageUrl: null })),
}
const sentBatches: string[][] = []
let partialSnapshot: BomCostingSourceFactSnapshot | null = null
jest.spyOn(console, 'warn').mockImplementation(() => undefined)
jest.spyOn(global, 'fetch').mockImplementation(async (_input, init) => {
const batch = readBatchRequest(init)
sentBatches.push(batch.imageUrls)
if (sentBatches.length === 1) {
return createStreamingResponse(JSON.stringify({
images: batch.imageIds.map((imageId, index) => ({
imageId,
observedTexts: [],
facts: index < 2
? []
: [{
factId: 'invalid-optional-number',
kind: 'DIMENSION',
subject: '尺寸',
value: '尺寸信息',
numericValue: { raw: '34.4cm' },
unit: 'mm',
relatedSkuIds: [],
basis: 'SOURCE_TEXT',
note: '图片标注',
confidence: 0.8,
}],
})),
}))
}
return createStreamingResponse(JSON.stringify({
images: batch.imageIds.map((imageId) => ({
imageId,
observedTexts: [],
facts: [],
})),
}))
})
await expect(extractBomCostingImageFacts({
config: providerConfig,
snapshot: threeImageSnapshot,
onBatchCompleted: async (progress) => {
partialSnapshot = progress.snapshot
},
})).rejects.toThrow('其余 2 张已保存,继续处理时只分析未完成图片')
expect(partialSnapshot?.imageFacts).toHaveLength(2)
const output = await extractBomCostingImageFacts({
config: providerConfig,
snapshot: threeImageSnapshot,
existingSnapshot: partialSnapshot ?? undefined,
})
expect(sentBatches).toEqual([
threeImageSnapshot.mainImages,
[threeImageSnapshot.mainImages[2]],
])
expect(output.snapshot.imageFacts).toHaveLength(3)
})
it('断点续跑只发送尚未保存事实的图片', async () => {
const sentUrls: string[] = []
const fetchSpy = jest.spyOn(global, 'fetch').mockImplementation(

View File

@ -50,8 +50,12 @@ const aiImageFactResultSchema = z.object({
),
})
const aiFactBatchPayloadSchema = z.object({
images: z.array(aiImageFactResultSchema).min(1).max(8),
const aiFactBatchEnvelopeSchema = z.object({
images: z.array(z.unknown()).min(1).max(8),
})
const aiFactBatchImageIdentitySchema = z.object({
imageId: z.string().trim().min(1).max(80),
})
export interface BomCostingMultimodalInputImage {
@ -270,22 +274,32 @@ function buildFactBatchUserText(
: undefined,
requiredOutputShape: {
images: [{
imageId: '必须来自本批输入',
observedTexts: ['图片中与产品和成本有关的原文'],
imageId: 'IMG_0001',
observedTexts: ['内空34.4cm'],
facts: [{
factId: '当前图片内唯一',
kind: 'DIMENSION | MATERIAL | PART | STRUCTURE | MECHANISM | HARDWARE | PACKAGING | PROCESS | WEIGHT | SKU_DIFFERENCE | TEXT',
subject: '事实主体',
value: '事实内容',
numericValue: '明确数值 optional',
unit: 'mm | kg | 个 | 件等 optional',
relatedSkuIds: ['只能来自 SKU 真相'],
basis: 'IMAGE | SOURCE_TEXT',
note: '证据位置、可见程度或待确认说明',
confidence: '0 to 1',
factId: 'inner_width_1',
kind: 'DIMENSION',
subject: '侧面内空',
value: '图片标注内空34.4cm',
numericValue: 344,
unit: 'mm',
relatedSkuIds: [],
basis: 'SOURCE_TEXT',
note: '尺寸标注位于图片右侧',
confidence: 0.85,
}],
}],
},
outputFieldRules: {
imageId: '必须逐一使用本批 batchImages 中的真实 imageId,不得照抄示例',
factId: '同一图片内唯一,使用简短稳定的英文或数字标识',
kind: '只能使用系统提示中列出的事实类型',
numericValue:
'必须是 JSON number,不得加引号;没有单一明确数值时省略或返回 null,并把原文保留在 value',
confidence: '必须是 0 到 1 的 JSON number,不得加引号',
relatedSkuIds: '只能使用 skuTruth 中存在的真实 externalSkuId;无法绑定时返回空数组',
optionalFields: '可选字段没有证据时省略或返回 null,不得填写 optional 等说明文字',
},
productTruth: {
itemId: snapshot.itemId,
channel: snapshot.channel,
@ -305,36 +319,115 @@ function buildFactBatchUserText(
})
}
function assertExactImageCoverage(
expectedImages: BomCostingMultimodalInputImage[],
actualImages: z.infer<typeof aiImageFactResultSchema>[],
) {
const expectedIds = new Set(expectedImages.map((image) => image.imageId))
const actualIds = new Set(actualImages.map((image) => image.imageId))
if (actualIds.size !== actualImages.length) {
throw new Error('模型返回了重复图片事实')
}
const missing = Array.from(expectedIds).filter((id) => !actualIds.has(id))
const unexpected = Array.from(actualIds).filter((id) => !expectedIds.has(id))
if (missing.length > 0 || unexpected.length > 0) {
throw new Error(
`图片事实覆盖不一致,缺少 ${missing.length} 张,多出 ${unexpected.length} 张`,
)
}
interface RejectedFactImage {
imageId: string
reason:
| 'MISSING'
| 'DUPLICATE'
| 'INVALID_SHAPE'
| 'INVALID_SKU_REFERENCE'
| 'DUPLICATE_FACT_ID'
issuePaths: string[]
issueCodes: string[]
}
function assertFactSkuReferences(
function validateFactBatchImages(
snapshot: MarketDetailSnapshot,
images: z.infer<typeof aiImageFactResultSchema>[],
expectedImages: BomCostingMultimodalInputImage[],
rawImages: unknown[],
) {
const skuIds = new Set(snapshot.skus.map((sku) => sku.externalSkuId))
for (const image of images) {
for (const fact of image.facts) {
const unexpected = fact.relatedSkuIds.find((id) => !skuIds.has(id))
if (unexpected) {
throw new Error(`图片事实引用了不存在的 SKU ${unexpected}`)
}
const expectedIds = new Set(expectedImages.map((image) => image.imageId))
const rawByImageId = new Map<string, unknown[]>()
const unexpectedImageIds: string[] = []
let unreadableImageCount = 0
for (const rawImage of rawImages) {
const identity = aiFactBatchImageIdentitySchema.safeParse(rawImage)
if (!identity.success) {
unreadableImageCount += 1
continue
}
if (!expectedIds.has(identity.data.imageId)) {
unexpectedImageIds.push(identity.data.imageId)
continue
}
const entries = rawByImageId.get(identity.data.imageId) ?? []
entries.push(rawImage)
rawByImageId.set(identity.data.imageId, entries)
}
const skuIds = new Set(snapshot.skus.map((sku) => sku.externalSkuId))
const acceptedImages: z.infer<typeof aiImageFactResultSchema>[] = []
const rejectedImages: RejectedFactImage[] = []
for (const expectedImage of expectedImages) {
const entries = rawByImageId.get(expectedImage.imageId) ?? []
if (entries.length === 0) {
rejectedImages.push({
imageId: expectedImage.imageId,
reason: 'MISSING',
issuePaths: [],
issueCodes: [],
})
continue
}
if (entries.length > 1) {
rejectedImages.push({
imageId: expectedImage.imageId,
reason: 'DUPLICATE',
issuePaths: [],
issueCodes: [],
})
continue
}
const parsed = aiImageFactResultSchema.safeParse(entries[0])
if (!parsed.success) {
rejectedImages.push({
imageId: expectedImage.imageId,
reason: 'INVALID_SHAPE',
issuePaths: parsed.error.issues
.slice(0, 8)
.map((issue) => issue.path.join('.')),
issueCodes: parsed.error.issues
.slice(0, 8)
.map((issue) => issue.code),
})
continue
}
const hasInvalidSkuReference = parsed.data.facts.some((fact) =>
fact.relatedSkuIds.some((id) => !skuIds.has(id)),
)
if (hasInvalidSkuReference) {
rejectedImages.push({
imageId: expectedImage.imageId,
reason: 'INVALID_SKU_REFERENCE',
issuePaths: [],
issueCodes: [],
})
continue
}
const factIds = parsed.data.facts.map((fact) => fact.factId)
if (new Set(factIds).size !== factIds.length) {
rejectedImages.push({
imageId: expectedImage.imageId,
reason: 'DUPLICATE_FACT_ID',
issuePaths: [],
issueCodes: [],
})
continue
}
acceptedImages.push(parsed.data)
}
return {
acceptedImages,
rejectedImages,
unexpectedImageIds,
unreadableImageCount,
}
}
@ -367,6 +460,92 @@ function createTrustedImageFacts(
})
}
function createFactSnapshot(
images: BomCostingMultimodalInputImage[],
completedByIdentity: Map<
string,
BomCostingSourceFactSnapshot['imageFacts'][number]
>,
extractedAt = new Date().toISOString(),
) {
return bomCostingSourceFactSnapshotSchema.parse({
schemaVersion: BOM_COSTING_FACT_SCHEMA_VERSION,
imageFacts: images
.map((image) =>
completedByIdentity.get(buildBomCostingImageIdentity(image.url)),
)
.filter(Boolean),
extractedAt,
})
}
function describeRejectedFactBatch(
batchIndex: number,
acceptedImageCount: number,
rejectedImageCount: number,
) {
if (acceptedImageCount === 0) {
return `第 ${batchIndex} 批有 ${rejectedImageCount} 张图片结果格式不完整,没有产生可保存的图片事实`
}
return (
`第 ${batchIndex} 批有 ${rejectedImageCount} 张图片结果格式不完整;` +
`其余 ${acceptedImageCount} 张已保存,继续处理时只分析未完成图片`
)
}
function logRejectedFactBatch(input: {
batchIndex: number
totalBatches: number
batchImageCount: number
model: string
callId: string | null
finishReason: string | null
rejectedImages: RejectedFactImage[]
unexpectedImageIds: string[]
unreadableImageCount: number
}) {
console.warn('[BOM costing] image fact contract partially rejected', {
batchIndex: input.batchIndex,
totalBatches: input.totalBatches,
imageCount: input.batchImageCount,
model: input.model,
callId: input.callId,
finishReason: input.finishReason,
rejectedImages: input.rejectedImages.slice(0, 8),
unexpectedImageIds: input.unexpectedImageIds.slice(0, 8),
unreadableImageCount: input.unreadableImageCount,
})
}
function logUnreadableFactEnvelope(input: {
batchIndex: number
totalBatches: number
batchImageCount: number
model: string
callId: string | null
finishReason: string | null
error: unknown
}) {
console.warn('[BOM costing] image fact envelope rejected', {
batchIndex: input.batchIndex,
totalBatches: input.totalBatches,
imageCount: input.batchImageCount,
model: input.model,
callId: input.callId,
finishReason: input.finishReason,
issueCount: input.error instanceof z.ZodError ? input.error.issues.length : null,
issuePaths:
input.error instanceof z.ZodError
? input.error.issues.slice(0, 8).map((issue) => issue.path.join('.'))
: [],
issueCodes:
input.error instanceof z.ZodError
? input.error.issues.slice(0, 8).map((issue) => issue.code)
: [],
errorName: input.error instanceof Error ? input.error.name : 'UnknownError',
})
}
function chunk<T>(items: T[], size: number): T[][] {
const result: T[][] = []
for (let index = 0; index < items.length; index += size) {
@ -495,82 +674,97 @@ export async function extractBomCostingImageFacts(input: {
maxTokens,
timeoutMs,
})
let parsed: z.infer<typeof aiFactBatchPayloadSchema>
let rawImages: unknown[]
try {
parsed = aiFactBatchPayloadSchema.parse(
rawImages = aiFactBatchEnvelopeSchema.parse(
parseBomCostingJsonObject(call.content),
)
assertExactImageCoverage(batchImages, parsed.images)
assertFactSkuReferences(input.snapshot, parsed.images)
).images
} catch (error) {
console.warn('[BOM costing] image fact contract rejected', {
logUnreadableFactEnvelope({
batchIndex,
totalBatches: batches.length,
imageCount: batchImages.length,
batchImageCount: batchImages.length,
model: call.model,
callId: call.id,
finishReason: call.finishReason,
issueCount: error instanceof z.ZodError ? error.issues.length : null,
issuePaths:
error instanceof z.ZodError
? error.issues.slice(0, 8).map((issue) => issue.path.join('.'))
: [],
issueCodes:
error instanceof z.ZodError
? error.issues.slice(0, 8).map((issue) => issue.code)
: [],
errorName: error instanceof Error ? error.name : 'UnknownError',
error,
})
throw new AppError(
ErrorCode.BOM_COSTING_INVALID_PROVIDER,
`第 ${batchIndex} 批图片事实返回不完整,已完成批次不会重复分析`,
`第 ${batchIndex} 批图片事实无法读取,本次没有产生可保存的图片结果`,
)
}
const trustedFacts = createTrustedImageFacts(batchImages, parsed.images)
const validation = validateFactBatchImages(
input.snapshot,
batchImages,
rawImages,
)
const acceptedImageIds = new Set(
validation.acceptedImages.map((image) => image.imageId),
)
const acceptedBatchImages = batchImages.filter((image) =>
acceptedImageIds.has(image.imageId),
)
const trustedFacts = createTrustedImageFacts(
acceptedBatchImages,
validation.acceptedImages,
)
trustedFacts.forEach((image) => {
completedByIdentity.set(
buildBomCostingImageIdentity(image.sourceImageUrl),
image,
)
})
analyzedImageCount += batchImages.length
analyzedImageCount += trustedFacts.length
if (call.id) callIds.push(call.id)
promptTokens = addNullableNumber(promptTokens, call.promptTokens)
completionTokens = addNullableNumber(
completionTokens,
call.completionTokens,
)
const snapshot = bomCostingSourceFactSnapshotSchema.parse({
schemaVersion: BOM_COSTING_FACT_SCHEMA_VERSION,
imageFacts: images
.map((image) =>
completedByIdentity.get(buildBomCostingImageIdentity(image.url)),
)
.filter(Boolean),
extractedAt: new Date().toISOString(),
})
await input.onBatchCompleted?.({
batchIndex,
totalBatches: batches.length,
batchImages,
completedImageCount: completedByIdentity.size,
totalImageCount: images.length,
snapshot,
call,
})
if (trustedFacts.length > 0) {
await input.onBatchCompleted?.({
batchIndex,
totalBatches: batches.length,
batchImages: acceptedBatchImages,
completedImageCount: completedByIdentity.size,
totalImageCount: images.length,
snapshot: createFactSnapshot(images, completedByIdentity),
call,
})
}
if (validation.rejectedImages.length > 0) {
logRejectedFactBatch({
batchIndex,
totalBatches: batches.length,
batchImageCount: batchImages.length,
model: call.model,
callId: call.id,
finishReason: call.finishReason,
rejectedImages: validation.rejectedImages,
unexpectedImageIds: validation.unexpectedImageIds,
unreadableImageCount: validation.unreadableImageCount,
})
throw new AppError(
ErrorCode.BOM_COSTING_INVALID_PROVIDER,
describeRejectedFactBatch(
batchIndex,
trustedFacts.length,
validation.rejectedImages.length,
),
)
}
}
const snapshot = bomCostingSourceFactSnapshotSchema.parse({
schemaVersion: BOM_COSTING_FACT_SCHEMA_VERSION,
imageFacts: images.map((image) =>
completedByIdentity.get(buildBomCostingImageIdentity(image.url)),
),
extractedAt:
batches.length > 0
? new Date().toISOString()
: input.existingSnapshot?.extractedAt ?? new Date().toISOString(),
})
const snapshot = createFactSnapshot(
images,
completedByIdentity,
batches.length > 0
? new Date().toISOString()
: input.existingSnapshot?.extractedAt ?? new Date().toISOString(),
)
return {
snapshot,
trace: {