diff --git a/furniture-3d-system/src/app/admin/ai-render/AiRenderConfigClient.test.tsx b/furniture-3d-system/src/app/admin/ai-render/AiRenderConfigClient.test.tsx index d25b619a..86a40708 100644 --- a/furniture-3d-system/src/app/admin/ai-render/AiRenderConfigClient.test.tsx +++ b/furniture-3d-system/src/app/admin/ai-render/AiRenderConfigClient.test.tsx @@ -120,6 +120,7 @@ describe('AI 服务配置模型选择', () => { fireEvent.change(screen.getByRole('combobox', { name: '接口协议' }), { target: { value: 'openai-compatible-image-edit' }, }) + expect(screen.getByRole('option', { name: '异步媒体任务' })).toBeInTheDocument() fireEvent.click(screen.getByRole('button', { name: '测试并获取模型' })) const modelSelect = screen.getByRole('combobox', { name: '模型' }) diff --git a/furniture-3d-system/src/app/admin/ai-render/AiRenderConfigClient.tsx b/furniture-3d-system/src/app/admin/ai-render/AiRenderConfigClient.tsx index 7d1a5687..8b1204a7 100644 --- a/furniture-3d-system/src/app/admin/ai-render/AiRenderConfigClient.tsx +++ b/furniture-3d-system/src/app/admin/ai-render/AiRenderConfigClient.tsx @@ -46,7 +46,10 @@ type ActiveTab = 'ai-render' | 'prompt-optimizer' | 'image-background-removal' interface TestResult { status: 'idle' | 'testing' | 'success' | 'error'; message: string } interface ModelOption { value: string; label: string } -type BackgroundRemovalApiMode = 'external-image-gateway' | 'openai-compatible-image-edit' +type BackgroundRemovalApiMode = + | 'external-image-gateway' + | 'openai-compatible-image-edit' + | 'async-media-task' const BACKGROUND_REMOVAL_API_MODE_META: Record = { 'openai-compatible-image-edit': { @@ -57,12 +60,16 @@ const BACKGROUND_REMOVAL_API_MODE_META: Record { ) }) + it('异步媒体任务目录会返回智能抠图等图片处理模型', async () => { + const fetchMock = jest.fn().mockResolvedValue(createResponse(200, { + data: { + sections: [{ + id: 'task', + models: [ + { + id: 'remove-background', + display_name: '智能抠图', + available: true, + input_modes: ['file', 'url'], + output_modes: ['base64', 'image'], + invocation: { path: '/extensions/v1/tasks' }, + }, + { + id: 'chat-only', + display_name: '对话模型', + available: true, + input_modes: ['text'], + output_modes: ['text'], + invocation: { path: '/v1/chat/completions' }, + }, + ], + }], + }, + })) + global.fetch = fetchMock as typeof fetch + + const result = await service.fetchCatalog({ + serviceType: 'image-background-removal', + apiMode: 'async-media-task', + baseUrl: 'https://api.vip98k.com', + apiKey: 'secret', + }) + + expect(result).toMatchObject({ + ok: true, + endpointPath: '/api/v2/catalog/models', + models: [{ value: 'remove-background', label: '智能抠图(remove-background)' }], + }) + expect(fetchMock).toHaveBeenCalledWith( + 'https://api.vip98k.com/api/v2/catalog/models', + expect.any(Object), + ) + }) + it('密钥无权限时返回明确错误且不继续尝试其他地址', async () => { const fetchMock = jest.fn().mockResolvedValue(createResponse(401, { error: { message: 'unauthorized' }, diff --git a/furniture-3d-system/src/server/services/bom-costing/bom-costing-1688-image-search.test.ts b/furniture-3d-system/src/server/services/bom-costing/bom-costing-1688-image-search.test.ts index 13171c82..3e470107 100644 --- a/furniture-3d-system/src/server/services/bom-costing/bom-costing-1688-image-search.test.ts +++ b/furniture-3d-system/src/server/services/bom-costing/bom-costing-1688-image-search.test.ts @@ -268,6 +268,7 @@ describe('BomCosting1688ImageSearchService', () => { expect(prepareImageForSearch).toHaveBeenCalledWith({ buffer: Buffer.from('poster-image'), contentType: 'image/jpeg', + sourceUrl: 'https://images.example.com/sku-poster.jpg', }) const requestBody = JSON.parse(String(fetch.mock.calls[1]?.[1]?.body)) expect(requestBody.imgBase64).toBe( diff --git a/furniture-3d-system/src/server/services/bom-costing/bom-costing-1688-image-search.ts b/furniture-3d-system/src/server/services/bom-costing/bom-costing-1688-image-search.ts index b1b037dc..9b183b33 100644 --- a/furniture-3d-system/src/server/services/bom-costing/bom-costing-1688-image-search.ts +++ b/furniture-3d-system/src/server/services/bom-costing/bom-costing-1688-image-search.ts @@ -75,6 +75,7 @@ export interface BomCosting1688ImageSearchGateway { prepareImageForSearch?: (input: { buffer: Buffer contentType: string + sourceUrl?: string }) => Promise<{ buffer: Buffer; contentType: string } | null> } @@ -547,7 +548,11 @@ export class BomCosting1688ImageSearchService { let searchImage: Uint8Array = image if (this.gateway.prepareImageForSearch) { try { - const prepared = await this.gateway.prepareImageForSearch({ buffer: image, contentType }) + const prepared = await this.gateway.prepareImageForSearch({ + buffer: image, + contentType, + sourceUrl: imageUrl.toString(), + }) if (prepared?.buffer.length) searchImage = prepared.buffer } catch { console.warn('[BOM costing] product cutout unavailable; using original search image') diff --git a/furniture-3d-system/src/server/services/bom-costing/bom-costing-image-background-removal.service.test.ts b/furniture-3d-system/src/server/services/bom-costing/bom-costing-image-background-removal.service.test.ts index 2e6b6837..40e781bc 100644 --- a/furniture-3d-system/src/server/services/bom-costing/bom-costing-image-background-removal.service.test.ts +++ b/furniture-3d-system/src/server/services/bom-costing/bom-costing-image-background-removal.service.test.ts @@ -143,4 +143,69 @@ describe('BomCostingImageBackgroundRemovalService', () => { format: 'jpeg', }) }) + + it('submits and polls an async media task such as remove-background', async () => { + const generatedPng = await sharp({ + create: { + width: 40, + height: 30, + channels: 4, + background: { r: 30, g: 80, b: 110, alpha: 1 }, + }, + }).png().toBuffer() + const jsonResponse = (payload: unknown) => ({ + ok: true, + status: 200, + json: async () => payload, + text: async () => JSON.stringify(payload), + } as unknown as Response) + const fetch = jest.fn() + .mockResolvedValueOnce(jsonResponse({ data: { task_id: 'task-123' } })) + .mockResolvedValueOnce(jsonResponse({ data: { status: 'processing' } })) + .mockResolvedValueOnce(jsonResponse({ + data: { status: 'succeeded', result: { base64: generatedPng.toString('base64') } }, + })) + const sleep = jest.fn().mockResolvedValue(undefined) + const generate = jest.fn() + const service = new BomCostingImageBackgroundRemovalService({ + getActiveConfig: jest.fn().mockResolvedValue({ + ...activeConfig(), + baseUrl: 'https://api.vip98k.com', + model: 'remove-background', + extra: { apiMode: 'async-media-task' }, + }), + generate, + fetch, + sleep, + }) + + const result = await service.prepare({ + buffer: Buffer.from('source-buffer'), + contentType: 'image/jpeg', + sourceUrl: 'https://images.example.com/product.jpg', + }) + + expect(generate).not.toHaveBeenCalled() + expect(fetch).toHaveBeenNthCalledWith( + 1, + 'https://api.vip98k.com/extensions/v1/tasks', + expect.objectContaining({ method: 'POST' }), + ) + expect(JSON.parse(String(fetch.mock.calls[0]?.[1]?.body))).toEqual({ + model: 'remove-background', + file_urls: ['https://images.example.com/product.jpg'], + }) + expect(fetch).toHaveBeenNthCalledWith( + 2, + 'https://api.vip98k.com/extensions/v1/tasks/task-123/result', + expect.objectContaining({ method: 'GET' }), + ) + expect(fetch).toHaveBeenNthCalledWith( + 3, + 'https://api.vip98k.com/extensions/v1/tasks/task-123/result', + expect.objectContaining({ method: 'GET' }), + ) + expect(sleep).toHaveBeenCalledTimes(1) + expect(result).toMatchObject({ model: 'remove-background', contentType: 'image/jpeg' }) + }) }) diff --git a/furniture-3d-system/src/server/services/bom-costing/bom-costing-image-background-removal.service.ts b/furniture-3d-system/src/server/services/bom-costing/bom-costing-image-background-removal.service.ts index 9804f8df..7b13cfac 100644 --- a/furniture-3d-system/src/server/services/bom-costing/bom-costing-image-background-removal.service.ts +++ b/furniture-3d-system/src/server/services/bom-costing/bom-costing-image-background-removal.service.ts @@ -15,6 +15,7 @@ const BACKGROUND_REMOVAL_CACHE_LIMIT = 24 type BackgroundRemovalApiMode = | 'external-image-gateway' | 'openai-compatible-image-edit' + | 'async-media-task' export const BOM_COSTING_BACKGROUND_REMOVAL_PROMPT = [ 'Create a faithful product cutout from the reference image.', @@ -39,12 +40,14 @@ interface BomCostingBackgroundRemovalDependencies { getActiveConfig(): Promise generate: typeof callExternalImageGenerateApi fetch: typeof fetch + sleep?(milliseconds: number): Promise } const defaultDependencies: BomCostingBackgroundRemovalDependencies = { getActiveConfig: () => providerConfigService.getActive(BACKGROUND_REMOVAL_SERVICE_TYPE), generate: callExternalImageGenerateApi, fetch: (input, init) => fetch(input, init), + sleep: (milliseconds) => new Promise((resolve) => setTimeout(resolve, milliseconds)), } function sourceFileName(contentType: string) { @@ -83,9 +86,9 @@ function buildCacheKey( } function getApiMode(config: NonNullable): BackgroundRemovalApiMode { - return config.extra?.apiMode === 'openai-compatible-image-edit' - ? 'openai-compatible-image-edit' - : 'external-image-gateway' + if (config.extra?.apiMode === 'openai-compatible-image-edit') return 'openai-compatible-image-edit' + if (config.extra?.apiMode === 'async-media-task') return 'async-media-task' + return 'external-image-gateway' } function decodeBase64Image(value: string): Buffer { @@ -100,6 +103,42 @@ function decodeBase64Image(value: string): Buffer { return buffer } +function collectPayloadRecords(value: unknown, depth = 0): Record[] { + if (depth > 8 || !value || typeof value !== 'object') return [] + if (Array.isArray(value)) return value.flatMap((item) => collectPayloadRecords(item, depth + 1)) + const record = value as Record + return [record, ...Object.values(record).flatMap((item) => collectPayloadRecords(item, depth + 1))] +} + +function findPayloadString(payload: unknown, fields: string[]): string | null { + for (const record of collectPayloadRecords(payload)) { + for (const field of fields) { + const value = record[field] + if (typeof value === 'string' && value.trim()) return value.trim() + } + } + return null +} + +function findAsyncTaskId(payload: unknown): string | null { + return findPayloadString(payload, ['task_id', 'taskId']) +} + +function findAsyncTaskStatus(payload: unknown): string { + return (findPayloadString(payload, ['status', 'state', 'task_status', 'taskStatus']) ?? '').toLowerCase() +} + +function findAsyncTaskImage(payload: unknown): { base64?: string; url?: string } | null { + const base64 = findPayloadString(payload, [ + 'b64_json', 'base64', 'image_base64', 'imageBase64', 'output_base64', 'outputBase64', + ]) + if (base64) return { base64 } + const url = findPayloadString(payload, [ + 'image_url', 'imageUrl', 'output_url', 'outputUrl', 'file_url', 'fileUrl', 'url', + ]) + return url ? { url } : null +} + export class BomCostingImageBackgroundRemovalService { private readonly cache = new Map() @@ -158,9 +197,67 @@ export class BomCostingImageBackgroundRemovalService { throw new Error('商品抠图服务没有返回可用图片') } + private async editWithAsyncMediaTask( + config: NonNullable, + input: { sourceUrl?: string }, + ): Promise { + if (!input.sourceUrl) throw new Error('异步抠图任务缺少可公开读取的商品图片地址') + const sourceUrl = parseSafeCompetitorUrl(input.sourceUrl).toString() + const baseUrl = config.baseUrl.replace(/\/+$/, '') + const createResponse = await this.dependencies.fetch(`${baseUrl}/extensions/v1/tasks`, { + method: 'POST', + headers: { + Authorization: `Bearer ${config.apiKey}`, + 'Content-Type': 'application/json', + Accept: 'application/json', + }, + body: JSON.stringify({ model: config.model, file_urls: [sourceUrl] }), + cache: 'no-store', + signal: AbortSignal.timeout(BACKGROUND_REMOVAL_TIMEOUT_MS), + }) + if (!createResponse.ok) { + const body = await createResponse.text().catch(() => '') + throw new Error(`异步抠图任务创建失败 ${createResponse.status}: ${body.slice(0, 200)}`) + } + const createdPayload = await createResponse.json() as unknown + const immediateImage = findAsyncTaskImage(createdPayload) + if (immediateImage?.base64) return decodeBase64Image(immediateImage.base64) + if (immediateImage?.url) return this.downloadGeneratedImage(immediateImage.url) + const taskId = findAsyncTaskId(createdPayload) + if (!taskId) throw new Error('异步抠图任务没有返回 task_id') + + const deadline = Date.now() + BACKGROUND_REMOVAL_TIMEOUT_MS + while (Date.now() < deadline) { + const resultResponse = await this.dependencies.fetch( + `${baseUrl}/extensions/v1/tasks/${encodeURIComponent(taskId)}/result`, + { + method: 'GET', + headers: { Authorization: `Bearer ${config.apiKey}`, Accept: 'application/json' }, + cache: 'no-store', + signal: AbortSignal.timeout(BACKGROUND_REMOVAL_RESULT_TIMEOUT_MS), + }, + ) + if (!resultResponse.ok) { + const body = await resultResponse.text().catch(() => '') + throw new Error(`异步抠图结果读取失败 ${resultResponse.status}: ${body.slice(0, 200)}`) + } + const resultPayload = await resultResponse.json() as unknown + const image = findAsyncTaskImage(resultPayload) + if (image?.base64) return decodeBase64Image(image.base64) + if (image?.url) return this.downloadGeneratedImage(image.url) + const status = findAsyncTaskStatus(resultPayload) + if (['failed', 'error', 'cancelled', 'canceled'].includes(status)) { + throw new Error(`异步抠图任务失败${status ? `(${status})` : ''}`) + } + await (this.dependencies.sleep?.(1_200) ?? Promise.resolve()) + } + throw new Error('异步抠图任务等待超时') + } + async prepare(input: { buffer: Buffer contentType: string + sourceUrl?: string }): Promise { const config = await this.dependencies.getActiveConfig() if (!config?.baseUrl || !config.apiKey || !config.model) return null @@ -178,9 +275,12 @@ export class BomCostingImageBackgroundRemovalService { } } - const generatedBuffer = getApiMode(config) === 'openai-compatible-image-edit' + const apiMode = getApiMode(config) + const generatedBuffer = apiMode === 'openai-compatible-image-edit' ? await this.editWithOpenAiCompatibleApi(config, input) - : await (async () => { + : apiMode === 'async-media-task' + ? await this.editWithAsyncMediaTask(config, input) + : await (async () => { const generated = await this.dependencies.generate({ serviceType: BACKGROUND_REMOVAL_SERVICE_TYPE, prompt: BOM_COSTING_BACKGROUND_REMOVAL_PROMPT, diff --git a/furniture-3d-system/src/server/services/provider-model-catalog.service.ts b/furniture-3d-system/src/server/services/provider-model-catalog.service.ts index dc568923..0546f09a 100644 --- a/furniture-3d-system/src/server/services/provider-model-catalog.service.ts +++ b/furniture-3d-system/src/server/services/provider-model-catalog.service.ts @@ -8,6 +8,7 @@ export type ProviderModelServiceType = export type ImageBackgroundRemovalApiMode = | 'external-image-gateway' | 'openai-compatible-image-edit' + | 'async-media-task' export interface ProviderModelOption { value: string @@ -126,6 +127,46 @@ export function extractProviderModelOptions( return Array.from(deduplicated.values()) } +export function extractAsyncMediaTaskModelOptions(payload: unknown): ProviderModelOption[] { + if (!payload || typeof payload !== 'object' || Array.isArray(payload)) return [] + const root = payload as Record + const data = root.data && typeof root.data === 'object' && !Array.isArray(root.data) + ? root.data as Record + : root + const sections = Array.isArray(data.sections) ? data.sections : [] + const options = new Map() + + for (const section of sections) { + if (!section || typeof section !== 'object' || Array.isArray(section)) continue + const models = Array.isArray((section as Record).models) + ? (section as Record).models as unknown[] + : [] + for (const item of models) { + if (!item || typeof item !== 'object' || Array.isArray(item)) continue + const model = item as Record + const invocation = model.invocation && typeof model.invocation === 'object' && !Array.isArray(model.invocation) + ? model.invocation as Record + : null + const inputModes = Array.isArray(model.input_modes) ? model.input_modes.map(String) : [] + const outputModes = Array.isArray(model.output_modes) ? model.output_modes.map(String) : [] + if ( + model.available === false || + invocation?.path !== '/extensions/v1/tasks' || + !inputModes.some((mode) => ['file', 'url'].includes(mode)) || + !outputModes.some((mode) => ['image', 'mask'].includes(mode)) + ) continue + const id = typeof model.id === 'string' ? model.id.trim() : '' + if (!id || options.has(id)) continue + const displayName = typeof model.display_name === 'string' ? model.display_name.trim() : '' + options.set(id, { + value: id, + label: displayName && displayName !== id ? `${displayName}(${id})` : id, + }) + } + } + return Array.from(options.values()) +} + function joinUrl(baseUrl: string, path: string): string { return `${baseUrl.replace(/\/+$/, '')}/${path.replace(/^\/+/, '')}` } @@ -135,6 +176,13 @@ function buildEndpointCandidates( serviceType: ProviderModelServiceType, apiMode?: ImageBackgroundRemovalApiMode, ): Array<{ url: string; path: string }> { + if (serviceType === 'image-background-removal' && apiMode === 'async-media-task') { + return [{ + url: joinUrl(baseUrl, '/api/v2/catalog/models'), + path: '/api/v2/catalog/models', + }] + } + if ( serviceType === 'ai-render' || (serviceType === 'image-background-removal' && apiMode !== 'openai-compatible-image-edit') @@ -175,6 +223,7 @@ async function requestCatalog( endpoint: { url: string; path: string }, apiKey: string, serviceType: ProviderModelServiceType, + apiMode?: ImageBackgroundRemovalApiMode, ): Promise { const startedAt = Date.now() try { @@ -199,10 +248,12 @@ async function requestCatalog( } const models = response.ok - ? extractProviderModelOptions(payload, { - preferExternalImageModelId: - serviceType === 'ai-render' || serviceType === 'image-background-removal', - }) + ? apiMode === 'async-media-task' + ? extractAsyncMediaTaskModelOptions(payload) + : extractProviderModelOptions(payload, { + preferExternalImageModelId: + serviceType === 'ai-render' || serviceType === 'image-background-removal', + }) : [] return { status: response.status, @@ -236,7 +287,7 @@ export class ProviderModelCatalogService { let reachedProvider = false for (const candidate of candidates) { - const attempt = await requestCatalog(candidate, apiKey, input.serviceType) + const attempt = await requestCatalog(candidate, apiKey, input.serviceType, input.apiMode) lastAttempt = attempt reachedProvider ||= attempt.status >= 200 && attempt.status < 500