WuYongJie/furniture-3d-system-Wu
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(bom-costing): 接入供应商异步智能抠图模型

Browse Source
This commit is contained in:
WuYongJie 2026-09-03 18:44:51 +08:00
parent 966a8340e4
commit a5cca36d69
9 changed files with 290 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
furniture-3d-system/src/app/admin/ai-render/AiRenderConfigClient.test.tsx
View File

@ -120,6 +120,7 @@ describe('AI 服务配置模型选择', () => {
fireEvent.change(screen.getByRole('combobox', { name: '接口协议' }), {
target: { value: 'openai-compatible-image-edit' },
})
expect(screen.getByRole('option', { name: '异步媒体任务' })).toBeInTheDocument()
fireEvent.click(screen.getByRole('button', { name: '测试并获取模型' }))
const modelSelect = screen.getByRole('combobox', { name: '模型' })

11
furniture-3d-system/src/app/admin/ai-render/AiRenderConfigClient.tsx
View File

@ -46,7 +46,10 @@ type ActiveTab = 'ai-render' | 'prompt-optimizer' | 'image-background-removal'
interface TestResult { status: 'idle' | 'testing' | 'success' | 'error'; message: string }
interface ModelOption { value: string; label: string }
type BackgroundRemovalApiMode = 'external-image-gateway' | 'openai-compatible-image-edit'
type BackgroundRemovalApiMode =
| 'external-image-gateway'
| 'openai-compatible-image-edit'
| 'async-media-task'
const BACKGROUND_REMOVAL_API_MODE_META: Record<BackgroundRemovalApiMode, { label: string; desc: string }> = {
'openai-compatible-image-edit': {
@ -57,12 +60,16 @@ const BACKGROUND_REMOVAL_API_MODE_META: Record<BackgroundRemovalApiMode, { label
label: '图片生成网关',
desc: '使用 /api/external/image/generate，兼容当前图片网关。',
},
'async-media-task': {
label: '异步媒体任务',
desc: '服务地址填写站点根地址；从 /api/v2/catalog/models 读取模型，并通过 /extensions/v1/tasks 提交和查询结果。',
},
}
function readBackgroundRemovalApiMode(config?: ProviderConfig | null): BackgroundRemovalApiMode {
if (!config) return 'openai-compatible-image-edit'
const value = config?.extra?.apiMode
return value === 'openai-compatible-image-edit' || value === 'external-image-gateway'
return value === 'openai-compatible-image-edit' || value === 'external-image-gateway' || value === 'async-media-task'
? value
: 'external-image-gateway'
}

2
furniture-3d-system/src/app/api/admin/test-connection/route.ts
View File

@ -7,7 +7,7 @@ import { providerModelCatalogService } from '@/server/services'
const schema = z.object({
baseUrl: z.string().url('地址格式不正确'),
serviceType: z.enum(['ai-render', 'prompt-optimizer', 'image-background-removal']),
apiMode: z.enum(['external-image-gateway', 'openai-compatible-image-edit']).optional(),
apiMode: z.enum(['external-image-gateway', 'openai-compatible-image-edit', 'async-media-task']).optional(),
})
export async function POST(request: NextRequest) {

46
furniture-3d-system/src/server/services/__tests__/provider-model-catalog.service.test.ts
View File

@ -188,6 +188,52 @@ describe('ProviderModelCatalogService', () => {
)
})
it('异步媒体任务目录会返回智能抠图等图片处理模型', async () => {
const fetchMock = jest.fn().mockResolvedValue(createResponse(200, {
data: {
sections: [{
id: 'task',
models: [
{
id: 'remove-background',
display_name: '智能抠图',
available: true,
input_modes: ['file', 'url'],
output_modes: ['base64', 'image'],
invocation: { path: '/extensions/v1/tasks' },
},
{
id: 'chat-only',
display_name: '对话模型',
available: true,
input_modes: ['text'],
output_modes: ['text'],
invocation: { path: '/v1/chat/completions' },
},
],
}],
},
}))
global.fetch = fetchMock as typeof fetch
const result = await service.fetchCatalog({
serviceType: 'image-background-removal',
apiMode: 'async-media-task',
baseUrl: 'https://api.vip98k.com',
apiKey: 'secret',
})
expect(result).toMatchObject({
ok: true,
endpointPath: '/api/v2/catalog/models',
models: [{ value: 'remove-background', label: '智能抠图（remove-background）' }],
})
expect(fetchMock).toHaveBeenCalledWith(
'https://api.vip98k.com/api/v2/catalog/models',
expect.any(Object),
)
})
it('密钥无权限时返回明确错误且不继续尝试其他地址', async () => {
const fetchMock = jest.fn().mockResolvedValue(createResponse(401, {
error: { message: 'unauthorized' },

1
furniture-3d-system/src/server/services/bom-costing/bom-costing-1688-image-search.test.ts
View File

@ -268,6 +268,7 @@ describe('BomCosting1688ImageSearchService', () => {
expect(prepareImageForSearch).toHaveBeenCalledWith({
buffer: Buffer.from('poster-image'),
contentType: 'image/jpeg',
sourceUrl: 'https://images.example.com/sku-poster.jpg',
})
const requestBody = JSON.parse(String(fetch.mock.calls[1]?.[1]?.body))
expect(requestBody.imgBase64).toBe(

7
furniture-3d-system/src/server/services/bom-costing/bom-costing-1688-image-search.ts
View File

@ -75,6 +75,7 @@ export interface BomCosting1688ImageSearchGateway {
prepareImageForSearch?: (input: {
buffer: Buffer
contentType: string
sourceUrl?: string
}) => Promise<{ buffer: Buffer; contentType: string } | null>
}
@ -547,7 +548,11 @@ export class BomCosting1688ImageSearchService {
let searchImage: Uint8Array = image
if (this.gateway.prepareImageForSearch) {
try {
const prepared = await this.gateway.prepareImageForSearch({ buffer: image, contentType })
const prepared = await this.gateway.prepareImageForSearch({
buffer: image,
contentType,
sourceUrl: imageUrl.toString(),
})
if (prepared?.buffer.length) searchImage = prepared.buffer
} catch {
console.warn('[BOM costing] product cutout unavailable; using original search image')

65
furniture-3d-system/src/server/services/bom-costing/bom-costing-image-background-removal.service.test.ts
View File

@ -143,4 +143,69 @@ describe('BomCostingImageBackgroundRemovalService', () => {
format: 'jpeg',
})
})
it('submits and polls an async media task such as remove-background', async () => {
const generatedPng = await sharp({
create: {
width: 40,
height: 30,
channels: 4,
background: { r: 30, g: 80, b: 110, alpha: 1 },
},
}).png().toBuffer()
const jsonResponse = (payload: unknown) => ({
ok: true,
status: 200,
json: async () => payload,
text: async () => JSON.stringify(payload),
} as unknown as Response)
const fetch = jest.fn()
.mockResolvedValueOnce(jsonResponse({ data: { task_id: 'task-123' } }))
.mockResolvedValueOnce(jsonResponse({ data: { status: 'processing' } }))
.mockResolvedValueOnce(jsonResponse({
data: { status: 'succeeded', result: { base64: generatedPng.toString('base64') } },
}))
const sleep = jest.fn().mockResolvedValue(undefined)
const generate = jest.fn()
const service = new BomCostingImageBackgroundRemovalService({
getActiveConfig: jest.fn().mockResolvedValue({
...activeConfig(),
baseUrl: 'https://api.vip98k.com',
model: 'remove-background',
extra: { apiMode: 'async-media-task' },
}),
generate,
fetch,
sleep,
})
const result = await service.prepare({
buffer: Buffer.from('source-buffer'),
contentType: 'image/jpeg',
sourceUrl: 'https://images.example.com/product.jpg',
})
expect(generate).not.toHaveBeenCalled()
expect(fetch).toHaveBeenNthCalledWith(
1,
'https://api.vip98k.com/extensions/v1/tasks',
expect.objectContaining({ method: 'POST' }),
)
expect(JSON.parse(String(fetch.mock.calls[0]?.[1]?.body))).toEqual({
model: 'remove-background',
file_urls: ['https://images.example.com/product.jpg'],
})
expect(fetch).toHaveBeenNthCalledWith(
2,
'https://api.vip98k.com/extensions/v1/tasks/task-123/result',
expect.objectContaining({ method: 'GET' }),
)
expect(fetch).toHaveBeenNthCalledWith(
3,
'https://api.vip98k.com/extensions/v1/tasks/task-123/result',
expect.objectContaining({ method: 'GET' }),
)
expect(sleep).toHaveBeenCalledTimes(1)
expect(result).toMatchObject({ model: 'remove-background', contentType: 'image/jpeg' })
})
})

110
furniture-3d-system/src/server/services/bom-costing/bom-costing-image-background-removal.service.ts
View File

@ -15,6 +15,7 @@ const BACKGROUND_REMOVAL_CACHE_LIMIT = 24
type BackgroundRemovalApiMode =
| 'external-image-gateway'
| 'openai-compatible-image-edit'
| 'async-media-task'
export const BOM_COSTING_BACKGROUND_REMOVAL_PROMPT = [
'Create a faithful product cutout from the reference image.',
@ -39,12 +40,14 @@ interface BomCostingBackgroundRemovalDependencies {
getActiveConfig(): Promise<ActiveBackgroundRemovalConfig>
generate: typeof callExternalImageGenerateApi
fetch: typeof fetch
sleep?(milliseconds: number): Promise<void>
}
const defaultDependencies: BomCostingBackgroundRemovalDependencies = {
getActiveConfig: () => providerConfigService.getActive(BACKGROUND_REMOVAL_SERVICE_TYPE),
generate: callExternalImageGenerateApi,
fetch: (input, init) => fetch(input, init),
sleep: (milliseconds) => new Promise((resolve) => setTimeout(resolve, milliseconds)),
}
function sourceFileName(contentType: string) {
@ -83,9 +86,9 @@ function buildCacheKey(
}
function getApiMode(config: NonNullable<ActiveBackgroundRemovalConfig>): BackgroundRemovalApiMode {
return config.extra?.apiMode === 'openai-compatible-image-edit'
? 'openai-compatible-image-edit'
: 'external-image-gateway'
if (config.extra?.apiMode === 'openai-compatible-image-edit') return 'openai-compatible-image-edit'
if (config.extra?.apiMode === 'async-media-task') return 'async-media-task'
return 'external-image-gateway'
}
function decodeBase64Image(value: string): Buffer {
@ -100,6 +103,42 @@ function decodeBase64Image(value: string): Buffer {
return buffer
}
function collectPayloadRecords(value: unknown, depth = 0): Record<string, unknown>[] {
if (depth > 8 || !value || typeof value !== 'object') return []
if (Array.isArray(value)) return value.flatMap((item) => collectPayloadRecords(item, depth + 1))
const record = value as Record<string, unknown>
return [record, ...Object.values(record).flatMap((item) => collectPayloadRecords(item, depth + 1))]
}
function findPayloadString(payload: unknown, fields: string[]): string | null {
for (const record of collectPayloadRecords(payload)) {
for (const field of fields) {
const value = record[field]
if (typeof value === 'string' && value.trim()) return value.trim()
}
}
return null
}
function findAsyncTaskId(payload: unknown): string | null {
return findPayloadString(payload, ['task_id', 'taskId'])
}
function findAsyncTaskStatus(payload: unknown): string {
return (findPayloadString(payload, ['status', 'state', 'task_status', 'taskStatus']) ?? '').toLowerCase()
}
function findAsyncTaskImage(payload: unknown): { base64?: string; url?: string } | null {
const base64 = findPayloadString(payload, [
'b64_json', 'base64', 'image_base64', 'imageBase64', 'output_base64', 'outputBase64',
])
if (base64) return { base64 }
const url = findPayloadString(payload, [
'image_url', 'imageUrl', 'output_url', 'outputUrl', 'file_url', 'fileUrl', 'url',
])
return url ? { url } : null
}
export class BomCostingImageBackgroundRemovalService {
private readonly cache = new Map<string, Buffer>()
@ -158,9 +197,67 @@ export class BomCostingImageBackgroundRemovalService {
throw new Error('商品抠图服务没有返回可用图片')
}
private async editWithAsyncMediaTask(
config: NonNullable<ActiveBackgroundRemovalConfig>,
input: { sourceUrl?: string },
): Promise<Buffer> {
if (!input.sourceUrl) throw new Error('异步抠图任务缺少可公开读取的商品图片地址')
const sourceUrl = parseSafeCompetitorUrl(input.sourceUrl).toString()
const baseUrl = config.baseUrl.replace(/\/+$/, '')
const createResponse = await this.dependencies.fetch(`${baseUrl}/extensions/v1/tasks`, {
method: 'POST',
headers: {
Authorization: `Bearer ${config.apiKey}`,
'Content-Type': 'application/json',
Accept: 'application/json',
},
body: JSON.stringify({ model: config.model, file_urls: [sourceUrl] }),
cache: 'no-store',
signal: AbortSignal.timeout(BACKGROUND_REMOVAL_TIMEOUT_MS),
})
if (!createResponse.ok) {
const body = await createResponse.text().catch(() => '')
throw new Error(`异步抠图任务创建失败 ${createResponse.status}: ${body.slice(0, 200)}`)
}
const createdPayload = await createResponse.json() as unknown
const immediateImage = findAsyncTaskImage(createdPayload)
if (immediateImage?.base64) return decodeBase64Image(immediateImage.base64)
if (immediateImage?.url) return this.downloadGeneratedImage(immediateImage.url)
const taskId = findAsyncTaskId(createdPayload)
if (!taskId) throw new Error('异步抠图任务没有返回 task_id')
const deadline = Date.now() + BACKGROUND_REMOVAL_TIMEOUT_MS
while (Date.now() < deadline) {
const resultResponse = await this.dependencies.fetch(
`${baseUrl}/extensions/v1/tasks/${encodeURIComponent(taskId)}/result`,
{
method: 'GET',
headers: { Authorization: `Bearer ${config.apiKey}`, Accept: 'application/json' },
cache: 'no-store',
signal: AbortSignal.timeout(BACKGROUND_REMOVAL_RESULT_TIMEOUT_MS),
},
)
if (!resultResponse.ok) {
const body = await resultResponse.text().catch(() => '')
throw new Error(`异步抠图结果读取失败 ${resultResponse.status}: ${body.slice(0, 200)}`)
}
const resultPayload = await resultResponse.json() as unknown
const image = findAsyncTaskImage(resultPayload)
if (image?.base64) return decodeBase64Image(image.base64)
if (image?.url) return this.downloadGeneratedImage(image.url)
const status = findAsyncTaskStatus(resultPayload)
if (['failed', 'error', 'cancelled', 'canceled'].includes(status)) {
throw new Error(`异步抠图任务失败${status ? `（${status}）` : ''}`)
}
await (this.dependencies.sleep?.(1_200) ?? Promise.resolve())
}
throw new Error('异步抠图任务等待超时')
}
async prepare(input: {
buffer: Buffer
contentType: string
sourceUrl?: string
}): Promise<BomCostingBackgroundRemovalResult | null> {
const config = await this.dependencies.getActiveConfig()
if (!config?.baseUrl || !config.apiKey || !config.model) return null
@ -178,9 +275,12 @@ export class BomCostingImageBackgroundRemovalService {
}
}
const generatedBuffer = getApiMode(config) === 'openai-compatible-image-edit'
const apiMode = getApiMode(config)
const generatedBuffer = apiMode === 'openai-compatible-image-edit'
? await this.editWithOpenAiCompatibleApi(config, input)
: await (async () => {
: apiMode === 'async-media-task'
? await this.editWithAsyncMediaTask(config, input)
: await (async () => {
const generated = await this.dependencies.generate({
serviceType: BACKGROUND_REMOVAL_SERVICE_TYPE,
prompt: BOM_COSTING_BACKGROUND_REMOVAL_PROMPT,

61
furniture-3d-system/src/server/services/provider-model-catalog.service.ts
View File

@ -8,6 +8,7 @@ export type ProviderModelServiceType =
export type ImageBackgroundRemovalApiMode =
| 'external-image-gateway'
| 'openai-compatible-image-edit'
| 'async-media-task'
export interface ProviderModelOption {
value: string
@ -126,6 +127,46 @@ export function extractProviderModelOptions(
return Array.from(deduplicated.values())
}
export function extractAsyncMediaTaskModelOptions(payload: unknown): ProviderModelOption[] {
if (!payload || typeof payload !== 'object' || Array.isArray(payload)) return []
const root = payload as Record<string, unknown>
const data = root.data && typeof root.data === 'object' && !Array.isArray(root.data)
? root.data as Record<string, unknown>
: root
const sections = Array.isArray(data.sections) ? data.sections : []
const options = new Map<string, ProviderModelOption>()
for (const section of sections) {
if (!section || typeof section !== 'object' || Array.isArray(section)) continue
const models = Array.isArray((section as Record<string, unknown>).models)
? (section as Record<string, unknown>).models as unknown[]
: []
for (const item of models) {
if (!item || typeof item !== 'object' || Array.isArray(item)) continue
const model = item as Record<string, unknown>
const invocation = model.invocation && typeof model.invocation === 'object' && !Array.isArray(model.invocation)
? model.invocation as Record<string, unknown>
: null
const inputModes = Array.isArray(model.input_modes) ? model.input_modes.map(String) : []
const outputModes = Array.isArray(model.output_modes) ? model.output_modes.map(String) : []
if (
model.available === false ||
invocation?.path !== '/extensions/v1/tasks' ||
!inputModes.some((mode) => ['file', 'url'].includes(mode)) ||
!outputModes.some((mode) => ['image', 'mask'].includes(mode))
) continue
const id = typeof model.id === 'string' ? model.id.trim() : ''
if (!id || options.has(id)) continue
const displayName = typeof model.display_name === 'string' ? model.display_name.trim() : ''
options.set(id, {
value: id,
label: displayName && displayName !== id ? `${displayName}（${id}）` : id,
})
}
}
return Array.from(options.values())
}
function joinUrl(baseUrl: string, path: string): string {
return `${baseUrl.replace(/\/+$/, '')}/${path.replace(/^\/+/, '')}`
}
@ -135,6 +176,13 @@ function buildEndpointCandidates(
serviceType: ProviderModelServiceType,
apiMode?: ImageBackgroundRemovalApiMode,
): Array<{ url: string; path: string }> {
if (serviceType === 'image-background-removal' && apiMode === 'async-media-task') {
return [{
url: joinUrl(baseUrl, '/api/v2/catalog/models'),
path: '/api/v2/catalog/models',
}]
}
if (
serviceType === 'ai-render' ||
(serviceType === 'image-background-removal' && apiMode !== 'openai-compatible-image-edit')
@ -175,6 +223,7 @@ async function requestCatalog(
endpoint: { url: string; path: string },
apiKey: string,
serviceType: ProviderModelServiceType,
apiMode?: ImageBackgroundRemovalApiMode,
): Promise<CatalogAttempt> {
const startedAt = Date.now()
try {
@ -199,10 +248,12 @@ async function requestCatalog(
}
const models = response.ok
? extractProviderModelOptions(payload, {
preferExternalImageModelId:
serviceType === 'ai-render' || serviceType === 'image-background-removal',
})
? apiMode === 'async-media-task'
? extractAsyncMediaTaskModelOptions(payload)
: extractProviderModelOptions(payload, {
preferExternalImageModelId:
serviceType === 'ai-render' || serviceType === 'image-background-removal',
})
: []
return {
status: response.status,
@ -236,7 +287,7 @@ export class ProviderModelCatalogService {
let reachedProvider = false
for (const candidate of candidates) {
const attempt = await requestCatalog(candidate, apiKey, input.serviceType)
const attempt = await requestCatalog(candidate, apiKey, input.serviceType, input.apiMode)
lastAttempt = attempt
reachedProvider ||= attempt.status >= 200 && attempt.status < 500