238 lines
20 KiB
TypeScript
238 lines
20 KiB
TypeScript
import sharp from 'sharp'
|
||
import { requestAiModelOperation, requestAiModelOutput } from './ai-model-http'
|
||
|
||
export const AI_MODEL_ADAPTER_VERSION = 2
|
||
export interface AiModelProtocol {
|
||
id: 'openai-chat' | 'anthropic-messages' | 'openai-image-edit' | 'openai-image-generate' | 'async-image-task'
|
||
path: string
|
||
headers: Record<string, string>
|
||
inputField?: string
|
||
supportsImageInput: boolean
|
||
scopeId?: string
|
||
routeId?: string
|
||
operationId?: string
|
||
maxImages?: number
|
||
maxPromptLength?: number
|
||
}
|
||
const paths: Record<AiModelProtocol['id'], string> = { 'openai-chat': '/v1/chat/completions', 'anthropic-messages': '/v1/messages', 'openai-image-edit': '/v1/images/edits', 'openai-image-generate': '/v1/images/generations', 'async-image-task': '/extensions/v1/tasks' }
|
||
function record(value: unknown): Record<string, any> | null { return value && typeof value === 'object' && !Array.isArray(value) ? value as Record<string, any> : null }
|
||
function list(value: unknown): string[] { return Array.isArray(value) && value.every(item => typeof item === 'string') ? value : [] }
|
||
function safeHeaders(value: unknown): Record<string, string> | null {
|
||
const raw = record(value)
|
||
if (!raw) return null
|
||
const result: Record<string, string> = {}
|
||
for (const [name, item] of Object.entries(raw).sort(([left], [right]) => left.localeCompare(right))) {
|
||
if (!['X-Relay-Route-Option', 'Anthropic-Version'].includes(name) || typeof item !== 'string' || !/^[A-Za-z0-9._-]{1,160}$/.test(item)) return null
|
||
result[name] = item
|
||
}
|
||
return result
|
||
}
|
||
function identifier(value: unknown): value is string { return typeof value === 'string' && /^[A-Za-z0-9._-]{1,160}$/.test(value) }
|
||
|
||
/** Parse only a known operation, keeping route evidence separate from a model's combined capabilities. */
|
||
function parseInvocation(model: Record<string, any>, offer: Record<string, any>, route: Record<string, any>): AiModelProtocol | null {
|
||
const invocation = record(route.invocation ?? offer.invocation ?? model.invocation)
|
||
if (!invocation || invocation.method !== 'POST') return null
|
||
const id = (Object.keys(paths) as AiModelProtocol['id'][]).find(key => paths[key] === invocation.path)
|
||
const headers = safeHeaders(invocation.headers)
|
||
if (!id || !headers) return null
|
||
if (invocation.content_type !== (id === 'openai-image-edit' ? 'multipart/form-data' : 'application/json')) return null
|
||
const scopeId = record(offer.service_scope)?.id, routeId = route.id
|
||
if (scopeId !== undefined && !identifier(scopeId) || routeId !== undefined && !identifier(routeId)) return null
|
||
if (routeId && headers['X-Relay-Route-Option'] !== routeId) return null
|
||
const schema = record(invocation.input_schema ?? route.input_schema)
|
||
const properties = record(schema?.properties) ?? {}
|
||
const modes = list(route.input_modes ?? offer.input_modes ?? model.input_modes)
|
||
const capabilities = list(offer.capabilities ?? model.capabilities)
|
||
const isTextModel = ['text', 'multimodal'].includes(model.modality) && model.type === 'model'
|
||
const isImageModel = model.modality === 'image' && [...capabilities, ...list(model.capabilities)].some(value => ['text_to_image', 'image_to_image', 'openai.images.generations', 'openai.images.edits'].includes(value))
|
||
let inputField: string | undefined, supportsImageInput = false
|
||
if (id === 'openai-chat' || id === 'anthropic-messages') {
|
||
if (!isTextModel) return null
|
||
supportsImageInput = modes.some(mode => ['image', 'image_url', 'image_base64'].includes(mode)) || (!modes.length && model.modality === 'multimodal')
|
||
} else {
|
||
if (!isImageModel) return null
|
||
if (id === 'openai-image-edit') {
|
||
inputField = properties['image[]'] !== false && properties['image[]'] ? 'image[]' : properties.image !== false && properties.image ? 'image' : undefined
|
||
supportsImageInput = Boolean(inputField)
|
||
} else {
|
||
// A declared image-URL array accepts a data URI unless its schema excludes it.
|
||
// Keep private image bytes inline; never expose a product file through public storage.
|
||
inputField = ['file_urls', 'images', 'image'].find(field => {
|
||
const property = record(properties[field]), items = record(property?.items)
|
||
const pattern = items?.pattern
|
||
if (items?.type !== 'string' || !(property?.type === 'array' || Array.isArray(property?.type) && property.type.includes('array'))) return false
|
||
if (items.format !== undefined && !['uri', 'uri-reference'].includes(items.format)) return false
|
||
const declaredImageInput = modes.some(mode => ['image', 'image_url', 'image_base64'].includes(mode))
|
||
if (modes.length && !declaredImageInput) return false
|
||
if (pattern === undefined) return declaredImageInput && [...capabilities, ...list(model.capabilities)].includes('image_to_image')
|
||
if (typeof pattern !== 'string' || pattern.length > 512 || !pattern.includes('data:image/')) return false
|
||
try { return new RegExp(pattern).test('data:image/png;base64,AA==') } catch { return false }
|
||
})
|
||
supportsImageInput = Boolean(inputField)
|
||
}
|
||
if (id === 'async-image-task' && !supportsImageInput) return null
|
||
}
|
||
const maxImages = inputField && record(properties[inputField])?.maxItems
|
||
const maxPromptLength = record(properties.prompt)?.maxLength
|
||
return { id, path: paths[id], headers, supportsImageInput, ...(inputField ? { inputField } : {}),
|
||
...(scopeId ? { scopeId } : {}), ...(routeId ? { routeId } : {}),
|
||
...(identifier(offer.public_operation_id) ? { operationId: offer.public_operation_id } : {}),
|
||
...(Number.isInteger(maxImages) && maxImages > 0 ? { maxImages: Math.min(8, maxImages) } : {}),
|
||
...(Number.isInteger(maxPromptLength) && maxPromptLength > 0 ? { maxPromptLength } : {}) }
|
||
}
|
||
|
||
/** At setup time pick a concrete compatible route in the default scope; no runtime routing fallback. */
|
||
export function parseAiModelProtocols(value: unknown): AiModelProtocol[] {
|
||
const model = record(value)
|
||
if (!model || model.available === false || model.status && model.status !== 'available') return []
|
||
const offers = Array.isArray(model.service_offers) ? model.service_offers.map(record).filter(Boolean) as Record<string, any>[] : []
|
||
if (!offers.length) { const protocol = parseInvocation(model, {}, {}); return protocol ? [protocol] : [] }
|
||
if (!identifier(model.default_service_scope_id)) return []
|
||
const found: AiModelProtocol[] = []
|
||
for (const offer of offers) {
|
||
if (record(offer.service_scope)?.id !== model.default_service_scope_id || offer.available === false) continue
|
||
const routes = Array.isArray(offer.route_options) ? offer.route_options.map(record).filter(Boolean) as Record<string, any>[] : []
|
||
const eligible = routes.filter(route => route.available !== false).sort((a, b) => Number(b.id === model.default_route_option_id) - Number(a.id === model.default_route_option_id))
|
||
const protocols = eligible.map(route => parseInvocation(model, offer, route)).filter(Boolean) as AiModelProtocol[]
|
||
const chosen = protocols.find(protocol => protocol.supportsImageInput) ?? protocols[0]
|
||
if (chosen && !found.some(protocol => JSON.stringify(protocol) === JSON.stringify(chosen))) found.push(chosen)
|
||
}
|
||
return found.sort((a, b) => Number(b.supportsImageInput) - Number(a.supportsImageInput))
|
||
}
|
||
|
||
export function validateStoredAiModelProtocols(value: unknown): AiModelProtocol[] {
|
||
if (!Array.isArray(value) || value.length > 8) return []
|
||
const result: AiModelProtocol[] = []
|
||
for (const entry of value) {
|
||
const raw = record(entry), headers = safeHeaders(raw?.headers)
|
||
if (!raw || !Object.prototype.hasOwnProperty.call(paths, raw.id) || raw.path !== paths[raw.id as AiModelProtocol['id']] || !headers || typeof raw.supportsImageInput !== 'boolean') return []
|
||
if (['scopeId', 'routeId', 'operationId'].some(field => raw[field] !== undefined && !identifier(raw[field]))) return []
|
||
if (raw.routeId && headers['X-Relay-Route-Option'] !== raw.routeId) return []
|
||
if (raw.inputField !== undefined && !['image', 'image[]', 'images', 'file_urls'].includes(raw.inputField)) return []
|
||
if (['maxImages', 'maxPromptLength'].some(field => raw[field] !== undefined && (!Number.isInteger(raw[field]) || raw[field] < 1 || raw[field] > (field === 'maxImages' ? 8 : 1000000)))) return []
|
||
if (raw.id === 'async-image-task' && (!raw.supportsImageInput || raw.inputField !== 'file_urls') || raw.id === 'openai-image-generate' && raw.supportsImageInput && !raw.inputField || raw.id === 'openai-image-edit' && (!raw.supportsImageInput || !['image', 'image[]'].includes(raw.inputField))) return []
|
||
result.push({ id: raw.id, path: raw.path, headers, supportsImageInput: raw.supportsImageInput,
|
||
...Object.fromEntries(['inputField', 'scopeId', 'routeId', 'operationId', 'maxImages', 'maxPromptLength'].filter(key => raw[key] !== undefined).map(key => [key, raw[key]])) })
|
||
}
|
||
return result
|
||
}
|
||
|
||
export class AiModelProtocolError extends Error {
|
||
readonly taskPending: boolean
|
||
constructor(message: string, readonly outcomeUnknown: boolean, readonly requestId: string | null = null, readonly taskId: string | null = null, taskPending = outcomeUnknown && Boolean(taskId)) { super(message); this.name = 'AiModelProtocolError'; this.taskPending = taskPending }
|
||
}
|
||
export interface AiModelProtocolInput {
|
||
baseUrl: string; apiKey: string; modelId: string; protocol: AiModelProtocol; prompt: string
|
||
images?: Array<{ buffer: Buffer; mimeType: string }>
|
||
idempotencyKey: string; beforeRequest?: () => Promise<void>
|
||
onTaskSubmitted?: (taskId: string) => Promise<void>; resumeTaskId?: string
|
||
maxOutputTokens?: number
|
||
}
|
||
export interface AiModelProtocolResult { text?: string; image?: Buffer; requestId: string | null; summary: Record<string, unknown> }
|
||
|
||
function safeRequestId(response: Response, data: any, apiKey: string): string | null {
|
||
const id = response.headers.get('x-request-id') || data?.request_id || data?.error?.request_id
|
||
return typeof id === 'string' && /^[\w-]{1,120}$/.test(id) && !id.includes(apiKey) ? id : null
|
||
}
|
||
async function readJson(response: Response, apiKey: string, taskId: string | null = null): Promise<{ data: any; requestId: string | null }> {
|
||
const text = await response.text()
|
||
let data: any
|
||
try { data = JSON.parse(text) } catch { throw new AiModelProtocolError(`上游返回异常响应（${response.status}），请核查请求结果；勿重复提交`, true, safeRequestId(response, null, apiKey), taskId) }
|
||
const requestId = safeRequestId(response, data, apiKey)
|
||
const code = data?.error?.code ?? data?.code
|
||
if (!response.ok || data?.status === 'pending_review') {
|
||
const terminalTaskFailure = response.status === 409 && ['task_failed', 'task_cancelled', 'task_expired', 'task_timeout', 'content_policy_violation'].includes(code)
|
||
const review = ['billing_reconciliation_required', 'submission_pending_reconciliation', 'image_size_mismatch', 'task_state_unknown'].includes(code) || data?.status === 'pending_review'
|
||
// Losing access to an accepted task does not prove that generation failed.
|
||
const definiteRejection = ['service_rejected', 'model_temporarily_unavailable'].includes(code)
|
||
const unknown = Boolean(taskId) ? !terminalTaskFailure : review || code === 'request_outcome_unknown' || !definiteRejection && response.status >= 500
|
||
const pending = Boolean(taskId) && unknown && !review && (response.status >= 500 || response.status === 429)
|
||
throw new AiModelProtocolError(unknown ? '模型请求结果待核查，已暂停；请勿重复提交' : code === 'model_temporarily_unavailable' ? '上游模型暂不可用，请稍后重新测试' : `上游拒绝模型请求（HTTP ${response.status}），测试未通过`, unknown, requestId, taskId, pending)
|
||
}
|
||
if (text.includes(apiKey)) throw new AiModelProtocolError('模型响应含敏感内容，已暂停', true, requestId, taskId)
|
||
return { data, requestId }
|
||
}
|
||
async function normalizeImages(images: AiModelProtocolInput['images'], forAnalysis: boolean): Promise<Buffer[]> {
|
||
return Promise.all((images ?? []).map(image => {
|
||
const pipeline = sharp(image.buffer, { limitInputPixels: 40_000_000 }).rotate().resize(forAnalysis ? 1024 : 1536, forAnalysis ? 1024 : 1536, { fit: 'inside', withoutEnlargement: true })
|
||
return forAnalysis ? pipeline.flatten({ background: '#ffffff' }).jpeg({ quality: 85 }).toBuffer() : pipeline.png().toBuffer()
|
||
}))
|
||
}
|
||
async function decodeImage(data: any, input: AiModelProtocolInput, taskId: string | null): Promise<Buffer> {
|
||
let bytes: Buffer
|
||
if (typeof data?.b64_json === 'string' && /^[A-Za-z0-9+/]+={0,2}$/.test(data.b64_json) && data.b64_json.length <= 28 * 1024 * 1024) bytes = Buffer.from(data.b64_json, 'base64')
|
||
else if (typeof data?.url === 'string') {
|
||
const response = await requestAiModelOutput(input.baseUrl, input.apiKey, data.url, input.beforeRequest)
|
||
if (!response.ok) throw new AiModelProtocolError('模型图片暂时无法读取，已保留任务', true, null, taskId)
|
||
bytes = Buffer.from(await response.arrayBuffer())
|
||
} else throw new AiModelProtocolError('模型没有返回可保存的图片', false, null, taskId)
|
||
try {
|
||
const output = await sharp(bytes, { limitInputPixels: 40_000_000 }).rotate().resize(1440, 1440, { fit: 'inside', withoutEnlargement: true }).png().toBuffer()
|
||
if (output.length > 20 * 1024 * 1024) throw new Error('size')
|
||
return output
|
||
} catch { throw new AiModelProtocolError('模型图片无法解码或超出限制', false, null, taskId) }
|
||
}
|
||
|
||
export async function callAiModelProtocol(input: AiModelProtocolInput): Promise<AiModelProtocolResult> {
|
||
const protocol = validateStoredAiModelProtocols([input.protocol])[0]
|
||
if (!protocol) throw new AiModelProtocolError('模型协议无效，请重新同步', false)
|
||
const imageCount = input.images?.length ?? 0
|
||
if (imageCount > (protocol.maxImages ?? 8) || imageCount && !protocol.supportsImageInput) throw new AiModelProtocolError('该模型接口不支持当前参考图片', false)
|
||
if (protocol.maxPromptLength && input.prompt.length > protocol.maxPromptLength) throw new AiModelProtocolError(`该模型提示词最多 ${protocol.maxPromptLength} 字，请精简内容`, false)
|
||
if (protocol.id === 'async-image-task' && !input.onTaskSubmitted) throw new AiModelProtocolError('异步任务缺少持久化检查点', false)
|
||
if (input.resumeTaskId && (!/^[A-Za-z0-9_-]{1,160}$/.test(input.resumeTaskId) || protocol.id !== 'async-image-task')) throw new AiModelProtocolError('模型任务编号无效', false)
|
||
let taskId = input.resumeTaskId ?? null, requestId: string | null = null, dispatched = Boolean(input.resumeTaskId)
|
||
const summary = { adapterVersion: AI_MODEL_ADAPTER_VERSION, protocol: protocol.id, routeId: protocol.routeId ?? null }
|
||
try {
|
||
if (!taskId) {
|
||
const forAnalysis = protocol.id === 'openai-chat' || protocol.id === 'anthropic-messages'
|
||
const images = await normalizeImages(input.images, forAnalysis)
|
||
const imageMimeType = forAnalysis ? 'image/jpeg' : 'image/png'
|
||
if (images.some(image => image.length > 10 * 1024 * 1024) || images.reduce((total, image) => total + image.length, 0) > 24 * 1024 * 1024) throw new AiModelProtocolError('参考图片总量超出模型限制，请减少图片', false)
|
||
const dataUris = images.map(image => `data:${imageMimeType};base64,${image.toString('base64')}`)
|
||
let body: string | FormData
|
||
const maxTokens = input.maxOutputTokens && Number.isInteger(input.maxOutputTokens) ? Math.max(32, Math.min(8000, input.maxOutputTokens)) : 8000
|
||
if (protocol.id === 'openai-chat') body = JSON.stringify({ model: input.modelId, messages: [{ role: 'user', content: images.length ? [{ type: 'text', text: input.prompt }, ...dataUris.map(url => ({ type: 'image_url', image_url: { url } }))] : input.prompt }], max_tokens: maxTokens, stream: false })
|
||
else if (protocol.id === 'anthropic-messages') body = JSON.stringify({ model: input.modelId, messages: [{ role: 'user', content: [{ type: 'text', text: input.prompt }, ...images.map(image => ({ type: 'image', source: { type: 'base64', media_type: imageMimeType, data: image.toString('base64') } }))] }], max_tokens: maxTokens, stream: false })
|
||
else if (protocol.id === 'openai-image-edit') {
|
||
body = new FormData(); body.set('model', input.modelId); body.set('prompt', input.prompt); body.set('n', '1'); body.set('response_format', 'b64_json')
|
||
images.forEach((image, index) => (body as FormData).append(protocol.inputField!, new Blob([new Uint8Array(image)], { type: 'image/png' }), `reference-${index + 1}.png`))
|
||
} else body = JSON.stringify({ model: input.modelId, prompt: input.prompt, ...(images.length ? { [protocol.inputField!]: dataUris } : {}), ...(protocol.id === 'openai-image-generate' ? { n: 1, response_format: 'b64_json' } : {}) })
|
||
dispatched = true
|
||
const response = await requestAiModelOperation(input.baseUrl, input.apiKey, protocol.path, body, input.idempotencyKey, protocol.headers, input.beforeRequest)
|
||
const parsed = await readJson(response, input.apiKey); requestId = parsed.requestId
|
||
if (protocol.id === 'async-image-task') {
|
||
if (typeof parsed.data?.id !== 'string' || !/^[A-Za-z0-9_-]{1,160}$/.test(parsed.data.id)) throw new AiModelProtocolError('模型未返回可核查的任务编号，请勿重发', true, requestId)
|
||
taskId = parsed.data.id
|
||
try { await input.onTaskSubmitted!(taskId!) } catch { throw new AiModelProtocolError('模型任务已受理，但检查点未保存；已暂停后续请求', true, requestId, taskId) }
|
||
} else if (protocol.id === 'openai-chat' || protocol.id === 'anthropic-messages') {
|
||
const content = protocol.id === 'openai-chat' ? parsed.data?.choices?.[0]?.message?.content : parsed.data?.content
|
||
const text = typeof content === 'string' ? content : Array.isArray(content) ? content.filter(item => item?.type === 'text' && typeof item.text === 'string').map(item => item.text).join('\n') : ''
|
||
if (!text.trim()) throw new AiModelProtocolError('模型未返回可用文字', false, requestId)
|
||
return { text, requestId, summary }
|
||
} else return { image: await decodeImage(parsed.data?.data?.[0], input, null), requestId, summary }
|
||
}
|
||
const deadline = Date.now() + 40_000
|
||
for (let attempt = 0; attempt < 20 && Date.now() < deadline; attempt++) {
|
||
const response = await requestAiModelOperation(input.baseUrl, input.apiKey, `/extensions/v1/tasks/${taskId}/result`, undefined, undefined, {}, input.beforeRequest)
|
||
const parsed = await readJson(response, input.apiKey, taskId); requestId = parsed.requestId ?? requestId
|
||
if (parsed.data?.ready === true) {
|
||
if (!['succeeded', 'partially_succeeded'].includes(parsed.data.status)) throw new AiModelProtocolError('模型任务未成功完成', false, requestId, taskId)
|
||
const outputs = parsed.data?.result?.outputs
|
||
const image = Array.isArray(outputs) ? outputs.find(item => item?.type === 'image' && (typeof item.url === 'string' || typeof item.b64_json === 'string')) : null
|
||
if (!image) throw new AiModelProtocolError('模型任务没有图片产物', false, requestId, taskId)
|
||
return { image: await decodeImage(image, input, taskId), requestId, summary: { ...summary, taskId } }
|
||
}
|
||
if (response.status !== 202 || parsed.data?.ready !== false) throw new AiModelProtocolError('模型任务状态无法确认，已保留任务', true, requestId, taskId)
|
||
await new Promise(resolve => setTimeout(resolve, 2000))
|
||
}
|
||
throw new AiModelProtocolError('模型仍在处理中，已保留任务，可继续查询', true, requestId, taskId)
|
||
} catch (error) {
|
||
if (error instanceof AiModelProtocolError) throw error
|
||
// Authentication/revision guards fail before the transport opens a socket.
|
||
if (error && typeof error === 'object' && 'status' in error && [400, 401, 403, 409].includes(Number(error.status))) throw error
|
||
throw new AiModelProtocolError(dispatched ? '模型请求结果无法确认，已暂停；请勿重复提交' : '参考图片无法读取，请更换图片', dispatched, requestId, taskId)
|
||
}
|
||
}
|