feat(ai-render): 支持多材质贴图传入外部生图API
- external-image-generate: 新增 additionalReferenceImages 字段支持多文件上传 - assisted-assembly-ai-render: 自动提取 assetSnapshot 中的材质贴图 - 按 materialConfig.id 去重聚合，最多 6 种材质 - 构建材质映射描述追加到 prompt，告诉 AI 每张贴图对应哪个部位 - 支持 FormData 混合模式（文件 + URL）和 curl 降级
This commit is contained in:
WuYongJie
parent
a2e9335fc5
commit
05d90fe57e
@ -15,6 +15,78 @@ const MAX_IMAGE_COUNT = 8
|
||||
const MAX_PROMPT_LENGTH = 5000
|
||||
const MAX_NEGATIVE_PROMPT_LENGTH = 800
|
||||
const DATA_URL_IMAGE_PATTERN = /^data:(image\/(?:png|jpeg|webp));base64,(.+)$/i
|
||||
const MAX_MATERIAL_TEXTURES = 6
|
||||
|
||||
interface MaterialTextureEntry {
|
||||
materialId: string
|
||||
materialName: string
|
||||
textureUrl: string
|
||||
nodeNames: string[]
|
||||
}
|
||||
|
||||
function extractMaterialTexturesFromSnapshot(assetSnapshot: unknown): MaterialTextureEntry[] {
|
||||
const snapshot = toRecord(assetSnapshot)
|
||||
const nodes = Array.isArray(snapshot.nodes) ? snapshot.nodes : []
|
||||
|
||||
// 按 materialConfig.id 聚合，记录每个材质被哪些节点使用
|
||||
const materialMap = new Map<string, MaterialTextureEntry>()
|
||||
|
||||
for (const rawNode of nodes) {
|
||||
const node = toRecord(rawNode)
|
||||
const mc = toRecord(node.materialConfig)
|
||||
const materialId = typeof mc.id === 'string' ? mc.id : null
|
||||
const textureUrl = typeof mc.textureUrl === 'string' ? mc.textureUrl : null
|
||||
const materialName = typeof mc.name === 'string' ? mc.name : null
|
||||
const nodeName = typeof node.name === 'string' ? node.name : null
|
||||
|
||||
if (!materialId || !textureUrl) continue
|
||||
|
||||
const existing = materialMap.get(materialId)
|
||||
if (existing) {
|
||||
if (nodeName) existing.nodeNames.push(nodeName)
|
||||
} else {
|
||||
materialMap.set(materialId, {
|
||||
materialId,
|
||||
materialName: materialName ?? materialId,
|
||||
textureUrl,
|
||||
nodeNames: nodeName ? [nodeName] : [],
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
return Array.from(materialMap.values()).slice(0, MAX_MATERIAL_TEXTURES)
|
||||
}
|
||||
|
||||
function buildMaterialMappingPromptSuffix(entries: MaterialTextureEntry[]): string {
|
||||
if (entries.length === 0) return ''
|
||||
|
||||
const lines = entries.map((entry, index) => {
|
||||
const partDesc = entry.nodeNames.length > 0
|
||||
? entry.nodeNames.map(n => `"${n}"`).join(', ')
|
||||
: `part #${index + 1}`
|
||||
return `- [Reference #${index + 2}] ${entry.materialName} texture → apply to: ${partDesc}`
|
||||
})
|
||||
|
||||
return `\n\nMaterial mapping for this product (each attached texture reference is labeled):\n${lines.join('\n')}\nApply each material texture to its corresponding part as shown in the reference image.`
|
||||
}
|
||||
|
||||
async function downloadTextureBuffer(url: string): Promise<{ buffer: Buffer; contentType: string } | null> {
|
||||
try {
|
||||
const response = await fetch(url, {
|
||||
signal: AbortSignal.timeout(10_000),
|
||||
})
|
||||
if (!response.ok) return null
|
||||
|
||||
const contentType = response.headers.get('content-type') ?? 'image/jpeg'
|
||||
const arrayBuffer = await response.arrayBuffer()
|
||||
const buffer = Buffer.from(arrayBuffer)
|
||||
if (buffer.length === 0 || buffer.length > 8 * 1024 * 1024) return null
|
||||
|
||||
return { buffer, contentType: contentType.split(';')[0]!.trim() }
|
||||
} catch {
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
export interface AssistedAssemblyAiRenderDTO {
|
||||
id: string
|
||||
@ -335,8 +407,32 @@ export class AssistedAssemblyAiRenderService {
|
||||
}
|
||||
|
||||
try {
|
||||
// 提取材质贴图信息
|
||||
const materialEntries = extractMaterialTexturesFromSnapshot(input.assetSnapshot)
|
||||
const materialSuffix = buildMaterialMappingPromptSuffix(materialEntries)
|
||||
const enhancedPrompt = materialSuffix ? `${prompt}${materialSuffix}` : prompt
|
||||
|
||||
// 下载材质贴图
|
||||
const additionalReferenceImages: Array<{ buffer: Buffer; contentType: string; fileName: string }> = []
|
||||
if (materialEntries.length > 0) {
|
||||
const downloadResults = await Promise.allSettled(
|
||||
materialEntries.map((entry) => downloadTextureBuffer(entry.textureUrl)),
|
||||
)
|
||||
for (let i = 0; i < downloadResults.length; i++) {
|
||||
const result = downloadResults[i]!
|
||||
if (result.status === 'fulfilled' && result.value) {
|
||||
const ext = result.value.contentType.includes('png') ? 'png' : 'jpg'
|
||||
additionalReferenceImages.push({
|
||||
buffer: result.value.buffer,
|
||||
contentType: result.value.contentType,
|
||||
fileName: `material-${materialEntries[i]!.materialName.replace(/[^a-zA-Z0-9\u4e00-\u9fff]+/g, '-')}.${ext}`,
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const result = await callExternalImageGenerateApi({
|
||||
prompt,
|
||||
prompt: enhancedPrompt,
|
||||
negativePrompt,
|
||||
imageCount,
|
||||
aspectRatio,
|
||||
@ -345,6 +441,7 @@ export class AssistedAssemblyAiRenderService {
|
||||
...referenceImage,
|
||||
fileName: `assembly-reference.${referenceImage.extension}`,
|
||||
},
|
||||
additionalReferenceImages: additionalReferenceImages.length > 0 ? additionalReferenceImages : undefined,
|
||||
disabledMessage: '辅助装配 AI 生图服务未启用或未配置',
|
||||
})
|
||||
|
||||
|
||||
@ -40,7 +40,11 @@ export type CallExternalImageGenerateApiInput = {
|
||||
imageCount: number
|
||||
aspectRatio: string
|
||||
imageSize: string
|
||||
/** 主参考图（文件上传），保持向后兼容 */
|
||||
referenceImage?: ExternalImageGenerateReferenceImage
|
||||
/** 额外参考图（文件上传），用于材质贴图等多图场景 */
|
||||
additionalReferenceImages?: ExternalImageGenerateReferenceImage[]
|
||||
/** 参考图 URL 数组 */
|
||||
referenceImageUrls?: string[]
|
||||
disabledMessage?: string
|
||||
}
|
||||
@ -235,7 +239,11 @@ async function requestExternalImageGenerateViaCurl(params: {
|
||||
`\n${CURL_HTTP_STATUS_MARKER}%{http_code}`,
|
||||
]
|
||||
|
||||
if (params.normalizedReferenceImageUrls.length > 0) {
|
||||
const additionalImages = params.input.additionalReferenceImages ?? []
|
||||
const hasFileUploads = Boolean(params.input.referenceImage) || additionalImages.length > 0
|
||||
|
||||
// 纯 URL 模式 → JSON body
|
||||
if (!hasFileUploads && params.normalizedReferenceImageUrls.length > 0) {
|
||||
const bodyPath = path.join(tempDirectory, 'request.json')
|
||||
await writeFile(
|
||||
bodyPath,
|
||||
@ -254,20 +262,15 @@ async function requestExternalImageGenerateViaCurl(params: {
|
||||
|
||||
args.push('-H', 'Content-Type: application/json', '--data-binary', `@${bodyPath}`)
|
||||
} else {
|
||||
const referenceImage = params.input.referenceImage
|
||||
if (!referenceImage) {
|
||||
// 有文件上传 → multipart/form-data
|
||||
const primaryImage = params.input.referenceImage
|
||||
if (!primaryImage && additionalImages.length === 0) {
|
||||
throw new AppError(
|
||||
ErrorCode.VALIDATION_FAILED,
|
||||
'AI 生图至少需要 1 张参考图',
|
||||
)
|
||||
}
|
||||
|
||||
const referenceImagePath = path.join(
|
||||
tempDirectory,
|
||||
referenceImage.fileName?.trim() || 'external-image-reference.png',
|
||||
)
|
||||
await writeFile(referenceImagePath, referenceImage.buffer)
|
||||
|
||||
args.push(
|
||||
'-F',
|
||||
`prompt=${params.input.prompt}`,
|
||||
@ -282,9 +285,39 @@ async function requestExternalImageGenerateViaCurl(params: {
|
||||
`aspectRatio=${params.input.aspectRatio}`,
|
||||
'-F',
|
||||
`imageSize=${params.input.imageSize}`,
|
||||
'-F',
|
||||
`images=@${referenceImagePath};type=${referenceImage.contentType};filename=${path.basename(referenceImagePath)}`,
|
||||
)
|
||||
|
||||
// 主参考图
|
||||
if (primaryImage) {
|
||||
const referenceImagePath = path.join(
|
||||
tempDirectory,
|
||||
primaryImage.fileName?.trim() || 'external-image-reference.png',
|
||||
)
|
||||
await writeFile(referenceImagePath, primaryImage.buffer)
|
||||
args.push(
|
||||
'-F',
|
||||
`images=@${referenceImagePath};type=${primaryImage.contentType};filename=${path.basename(referenceImagePath)}`,
|
||||
)
|
||||
}
|
||||
|
||||
// 额外参考图（材质贴图等）
|
||||
for (let i = 0; i < additionalImages.length; i++) {
|
||||
const extra = additionalImages[i]!
|
||||
const extraPath = path.join(
|
||||
tempDirectory,
|
||||
extra.fileName?.trim() || `external-image-extra-${i}.png`,
|
||||
)
|
||||
await writeFile(extraPath, extra.buffer)
|
||||
args.push(
|
||||
'-F',
|
||||
`images=@${extraPath};type=${extra.contentType};filename=${path.basename(extraPath)}`,
|
||||
)
|
||||
}
|
||||
|
||||
// URL 参考图（混合模式）
|
||||
for (const url of params.normalizedReferenceImageUrls) {
|
||||
args.push('-F', `imageUrls=${url}`)
|
||||
}
|
||||
}
|
||||
|
||||
const { stdout, stderr, exitCode } = await runCurl(args)
|
||||
@ -369,8 +402,13 @@ export async function callExternalImageGenerateApi(
|
||||
)
|
||||
}
|
||||
|
||||
const requestInit: RequestInit = normalizedReferenceImageUrls.length > 0
|
||||
? {
|
||||
const additionalImages = input.additionalReferenceImages ?? []
|
||||
const hasFileUploads = Boolean(input.referenceImage) || additionalImages.length > 0
|
||||
|
||||
const requestInit: RequestInit = (() => {
|
||||
// 纯 URL 模式（无文件上传）→ JSON
|
||||
if (!hasFileUploads && normalizedReferenceImageUrls.length > 0) {
|
||||
return {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
Authorization: `Bearer ${config.apiKey}`,
|
||||
@ -386,42 +424,67 @@ export async function callExternalImageGenerateApi(
|
||||
imageSize: input.imageSize,
|
||||
referenceImages: normalizedReferenceImageUrls,
|
||||
}),
|
||||
}
|
||||
: (() => {
|
||||
const referenceImage = input.referenceImage
|
||||
if (!referenceImage) {
|
||||
throw new AppError(
|
||||
ErrorCode.VALIDATION_FAILED,
|
||||
'AI 生图至少需要 1 张参考图',
|
||||
)
|
||||
}
|
||||
} satisfies RequestInit
|
||||
}
|
||||
|
||||
const formData = new FormData()
|
||||
const file = new Blob([new Uint8Array(referenceImage.buffer)], {
|
||||
type: referenceImage.contentType,
|
||||
})
|
||||
// 有文件上传 → FormData（支持混合：files + URLs）
|
||||
const primaryImage = input.referenceImage
|
||||
if (!primaryImage && additionalImages.length === 0) {
|
||||
throw new AppError(
|
||||
ErrorCode.VALIDATION_FAILED,
|
||||
'AI 生图至少需要 1 张参考图',
|
||||
)
|
||||
}
|
||||
|
||||
formData.append('prompt', input.prompt)
|
||||
if (trimToNull(input.negativePrompt)) {
|
||||
formData.append('negativePrompt', trimToNull(input.negativePrompt)!)
|
||||
}
|
||||
formData.append('imageCount', String(input.imageCount))
|
||||
formData.append('aspectRatio', input.aspectRatio)
|
||||
formData.append('imageSize', input.imageSize)
|
||||
formData.append(
|
||||
'images',
|
||||
file,
|
||||
referenceImage.fileName?.trim() || 'external-image-reference.png',
|
||||
)
|
||||
const formData = new FormData()
|
||||
|
||||
return {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
Authorization: `Bearer ${config.apiKey}`,
|
||||
},
|
||||
body: formData,
|
||||
} satisfies RequestInit
|
||||
})()
|
||||
formData.append('prompt', input.prompt)
|
||||
if (trimToNull(input.negativePrompt)) {
|
||||
formData.append('negativePrompt', trimToNull(input.negativePrompt)!)
|
||||
}
|
||||
formData.append('imageCount', String(input.imageCount))
|
||||
formData.append('aspectRatio', input.aspectRatio)
|
||||
formData.append('imageSize', input.imageSize)
|
||||
|
||||
// 主参考图（3D 截图）
|
||||
if (primaryImage) {
|
||||
const file = new Blob([new Uint8Array(primaryImage.buffer)], {
|
||||
type: primaryImage.contentType,
|
||||
})
|
||||
formData.append(
|
||||
'images',
|
||||
file,
|
||||
primaryImage.fileName?.trim() || 'external-image-reference.png',
|
||||
)
|
||||
}
|
||||
|
||||
// 额外参考图（材质贴图等）
|
||||
for (const extra of additionalImages) {
|
||||
const file = new Blob([new Uint8Array(extra.buffer)], {
|
||||
type: extra.contentType,
|
||||
})
|
||||
formData.append(
|
||||
'images',
|
||||
file,
|
||||
extra.fileName?.trim() || 'external-image-extra.png',
|
||||
)
|
||||
}
|
||||
|
||||
// URL 参考图（混合模式）
|
||||
for (const url of normalizedReferenceImageUrls) {
|
||||
formData.append('imageUrls', url)
|
||||
}
|
||||
|
||||
const headers: Record<string, string> = {
|
||||
Authorization: `Bearer ${config.apiKey}`,
|
||||
}
|
||||
|
||||
return {
|
||||
method: 'POST',
|
||||
headers,
|
||||
body: formData,
|
||||
} satisfies RequestInit
|
||||
})()
|
||||
|
||||
let lastError: unknown = null
|
||||
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user