diff --git a/furniture-3d-system/src/server/services/assisted-assembly-ai-render.service.ts b/furniture-3d-system/src/server/services/assisted-assembly-ai-render.service.ts index 5d9d0403..1bee06a4 100644 --- a/furniture-3d-system/src/server/services/assisted-assembly-ai-render.service.ts +++ b/furniture-3d-system/src/server/services/assisted-assembly-ai-render.service.ts @@ -15,6 +15,78 @@ const MAX_IMAGE_COUNT = 8 const MAX_PROMPT_LENGTH = 5000 const MAX_NEGATIVE_PROMPT_LENGTH = 800 const DATA_URL_IMAGE_PATTERN = /^data:(image\/(?:png|jpeg|webp));base64,(.+)$/i +const MAX_MATERIAL_TEXTURES = 6 + +interface MaterialTextureEntry { + materialId: string + materialName: string + textureUrl: string + nodeNames: string[] +} + +function extractMaterialTexturesFromSnapshot(assetSnapshot: unknown): MaterialTextureEntry[] { + const snapshot = toRecord(assetSnapshot) + const nodes = Array.isArray(snapshot.nodes) ? snapshot.nodes : [] + + // 按 materialConfig.id 聚合,记录每个材质被哪些节点使用 + const materialMap = new Map() + + for (const rawNode of nodes) { + const node = toRecord(rawNode) + const mc = toRecord(node.materialConfig) + const materialId = typeof mc.id === 'string' ? mc.id : null + const textureUrl = typeof mc.textureUrl === 'string' ? mc.textureUrl : null + const materialName = typeof mc.name === 'string' ? mc.name : null + const nodeName = typeof node.name === 'string' ? node.name : null + + if (!materialId || !textureUrl) continue + + const existing = materialMap.get(materialId) + if (existing) { + if (nodeName) existing.nodeNames.push(nodeName) + } else { + materialMap.set(materialId, { + materialId, + materialName: materialName ?? materialId, + textureUrl, + nodeNames: nodeName ? [nodeName] : [], + }) + } + } + + return Array.from(materialMap.values()).slice(0, MAX_MATERIAL_TEXTURES) +} + +function buildMaterialMappingPromptSuffix(entries: MaterialTextureEntry[]): string { + if (entries.length === 0) return '' + + const lines = entries.map((entry, index) => { + const partDesc = entry.nodeNames.length > 0 + ? entry.nodeNames.map(n => `"${n}"`).join(', ') + : `part #${index + 1}` + return `- [Reference #${index + 2}] ${entry.materialName} texture → apply to: ${partDesc}` + }) + + return `\n\nMaterial mapping for this product (each attached texture reference is labeled):\n${lines.join('\n')}\nApply each material texture to its corresponding part as shown in the reference image.` +} + +async function downloadTextureBuffer(url: string): Promise<{ buffer: Buffer; contentType: string } | null> { + try { + const response = await fetch(url, { + signal: AbortSignal.timeout(10_000), + }) + if (!response.ok) return null + + const contentType = response.headers.get('content-type') ?? 'image/jpeg' + const arrayBuffer = await response.arrayBuffer() + const buffer = Buffer.from(arrayBuffer) + if (buffer.length === 0 || buffer.length > 8 * 1024 * 1024) return null + + return { buffer, contentType: contentType.split(';')[0]!.trim() } + } catch { + return null + } +} export interface AssistedAssemblyAiRenderDTO { id: string @@ -335,8 +407,32 @@ export class AssistedAssemblyAiRenderService { } try { + // 提取材质贴图信息 + const materialEntries = extractMaterialTexturesFromSnapshot(input.assetSnapshot) + const materialSuffix = buildMaterialMappingPromptSuffix(materialEntries) + const enhancedPrompt = materialSuffix ? `${prompt}${materialSuffix}` : prompt + + // 下载材质贴图 + const additionalReferenceImages: Array<{ buffer: Buffer; contentType: string; fileName: string }> = [] + if (materialEntries.length > 0) { + const downloadResults = await Promise.allSettled( + materialEntries.map((entry) => downloadTextureBuffer(entry.textureUrl)), + ) + for (let i = 0; i < downloadResults.length; i++) { + const result = downloadResults[i]! + if (result.status === 'fulfilled' && result.value) { + const ext = result.value.contentType.includes('png') ? 'png' : 'jpg' + additionalReferenceImages.push({ + buffer: result.value.buffer, + contentType: result.value.contentType, + fileName: `material-${materialEntries[i]!.materialName.replace(/[^a-zA-Z0-9\u4e00-\u9fff]+/g, '-')}.${ext}`, + }) + } + } + } + const result = await callExternalImageGenerateApi({ - prompt, + prompt: enhancedPrompt, negativePrompt, imageCount, aspectRatio, @@ -345,6 +441,7 @@ export class AssistedAssemblyAiRenderService { ...referenceImage, fileName: `assembly-reference.${referenceImage.extension}`, }, + additionalReferenceImages: additionalReferenceImages.length > 0 ? additionalReferenceImages : undefined, disabledMessage: '辅助装配 AI 生图服务未启用或未配置', }) diff --git a/furniture-3d-system/src/server/services/external-image-generate.service.ts b/furniture-3d-system/src/server/services/external-image-generate.service.ts index 58cf4a89..0899ad9a 100644 --- a/furniture-3d-system/src/server/services/external-image-generate.service.ts +++ b/furniture-3d-system/src/server/services/external-image-generate.service.ts @@ -40,7 +40,11 @@ export type CallExternalImageGenerateApiInput = { imageCount: number aspectRatio: string imageSize: string + /** 主参考图(文件上传),保持向后兼容 */ referenceImage?: ExternalImageGenerateReferenceImage + /** 额外参考图(文件上传),用于材质贴图等多图场景 */ + additionalReferenceImages?: ExternalImageGenerateReferenceImage[] + /** 参考图 URL 数组 */ referenceImageUrls?: string[] disabledMessage?: string } @@ -235,7 +239,11 @@ async function requestExternalImageGenerateViaCurl(params: { `\n${CURL_HTTP_STATUS_MARKER}%{http_code}`, ] - if (params.normalizedReferenceImageUrls.length > 0) { + const additionalImages = params.input.additionalReferenceImages ?? [] + const hasFileUploads = Boolean(params.input.referenceImage) || additionalImages.length > 0 + + // 纯 URL 模式 → JSON body + if (!hasFileUploads && params.normalizedReferenceImageUrls.length > 0) { const bodyPath = path.join(tempDirectory, 'request.json') await writeFile( bodyPath, @@ -254,20 +262,15 @@ async function requestExternalImageGenerateViaCurl(params: { args.push('-H', 'Content-Type: application/json', '--data-binary', `@${bodyPath}`) } else { - const referenceImage = params.input.referenceImage - if (!referenceImage) { + // 有文件上传 → multipart/form-data + const primaryImage = params.input.referenceImage + if (!primaryImage && additionalImages.length === 0) { throw new AppError( ErrorCode.VALIDATION_FAILED, 'AI 生图至少需要 1 张参考图', ) } - const referenceImagePath = path.join( - tempDirectory, - referenceImage.fileName?.trim() || 'external-image-reference.png', - ) - await writeFile(referenceImagePath, referenceImage.buffer) - args.push( '-F', `prompt=${params.input.prompt}`, @@ -282,9 +285,39 @@ async function requestExternalImageGenerateViaCurl(params: { `aspectRatio=${params.input.aspectRatio}`, '-F', `imageSize=${params.input.imageSize}`, - '-F', - `images=@${referenceImagePath};type=${referenceImage.contentType};filename=${path.basename(referenceImagePath)}`, ) + + // 主参考图 + if (primaryImage) { + const referenceImagePath = path.join( + tempDirectory, + primaryImage.fileName?.trim() || 'external-image-reference.png', + ) + await writeFile(referenceImagePath, primaryImage.buffer) + args.push( + '-F', + `images=@${referenceImagePath};type=${primaryImage.contentType};filename=${path.basename(referenceImagePath)}`, + ) + } + + // 额外参考图(材质贴图等) + for (let i = 0; i < additionalImages.length; i++) { + const extra = additionalImages[i]! + const extraPath = path.join( + tempDirectory, + extra.fileName?.trim() || `external-image-extra-${i}.png`, + ) + await writeFile(extraPath, extra.buffer) + args.push( + '-F', + `images=@${extraPath};type=${extra.contentType};filename=${path.basename(extraPath)}`, + ) + } + + // URL 参考图(混合模式) + for (const url of params.normalizedReferenceImageUrls) { + args.push('-F', `imageUrls=${url}`) + } } const { stdout, stderr, exitCode } = await runCurl(args) @@ -369,8 +402,13 @@ export async function callExternalImageGenerateApi( ) } - const requestInit: RequestInit = normalizedReferenceImageUrls.length > 0 - ? { + const additionalImages = input.additionalReferenceImages ?? [] + const hasFileUploads = Boolean(input.referenceImage) || additionalImages.length > 0 + + const requestInit: RequestInit = (() => { + // 纯 URL 模式(无文件上传)→ JSON + if (!hasFileUploads && normalizedReferenceImageUrls.length > 0) { + return { method: 'POST', headers: { Authorization: `Bearer ${config.apiKey}`, @@ -386,42 +424,67 @@ export async function callExternalImageGenerateApi( imageSize: input.imageSize, referenceImages: normalizedReferenceImageUrls, }), - } - : (() => { - const referenceImage = input.referenceImage - if (!referenceImage) { - throw new AppError( - ErrorCode.VALIDATION_FAILED, - 'AI 生图至少需要 1 张参考图', - ) - } + } satisfies RequestInit + } - const formData = new FormData() - const file = new Blob([new Uint8Array(referenceImage.buffer)], { - type: referenceImage.contentType, - }) + // 有文件上传 → FormData(支持混合:files + URLs) + const primaryImage = input.referenceImage + if (!primaryImage && additionalImages.length === 0) { + throw new AppError( + ErrorCode.VALIDATION_FAILED, + 'AI 生图至少需要 1 张参考图', + ) + } - formData.append('prompt', input.prompt) - if (trimToNull(input.negativePrompt)) { - formData.append('negativePrompt', trimToNull(input.negativePrompt)!) - } - formData.append('imageCount', String(input.imageCount)) - formData.append('aspectRatio', input.aspectRatio) - formData.append('imageSize', input.imageSize) - formData.append( - 'images', - file, - referenceImage.fileName?.trim() || 'external-image-reference.png', - ) + const formData = new FormData() - return { - method: 'POST', - headers: { - Authorization: `Bearer ${config.apiKey}`, - }, - body: formData, - } satisfies RequestInit - })() + formData.append('prompt', input.prompt) + if (trimToNull(input.negativePrompt)) { + formData.append('negativePrompt', trimToNull(input.negativePrompt)!) + } + formData.append('imageCount', String(input.imageCount)) + formData.append('aspectRatio', input.aspectRatio) + formData.append('imageSize', input.imageSize) + + // 主参考图(3D 截图) + if (primaryImage) { + const file = new Blob([new Uint8Array(primaryImage.buffer)], { + type: primaryImage.contentType, + }) + formData.append( + 'images', + file, + primaryImage.fileName?.trim() || 'external-image-reference.png', + ) + } + + // 额外参考图(材质贴图等) + for (const extra of additionalImages) { + const file = new Blob([new Uint8Array(extra.buffer)], { + type: extra.contentType, + }) + formData.append( + 'images', + file, + extra.fileName?.trim() || 'external-image-extra.png', + ) + } + + // URL 参考图(混合模式) + for (const url of normalizedReferenceImageUrls) { + formData.append('imageUrls', url) + } + + const headers: Record = { + Authorization: `Bearer ${config.apiKey}`, + } + + return { + method: 'POST', + headers, + body: formData, + } satisfies RequestInit + })() let lastError: unknown = null