import sharp from 'sharp' import { requestAiModelOperation, requestAiModelOutput } from './ai-model-http' export const AI_MODEL_ADAPTER_VERSION = 2 export interface AiModelProtocol { id: 'openai-chat' | 'anthropic-messages' | 'openai-image-edit' | 'openai-image-generate' | 'async-image-task' path: string headers: Record inputField?: string supportsImageInput: boolean scopeId?: string routeId?: string operationId?: string maxImages?: number maxPromptLength?: number } const paths: Record = { 'openai-chat': '/v1/chat/completions', 'anthropic-messages': '/v1/messages', 'openai-image-edit': '/v1/images/edits', 'openai-image-generate': '/v1/images/generations', 'async-image-task': '/extensions/v1/tasks' } function record(value: unknown): Record | null { return value && typeof value === 'object' && !Array.isArray(value) ? value as Record : null } function list(value: unknown): string[] { return Array.isArray(value) && value.every(item => typeof item === 'string') ? value : [] } function safeHeaders(value: unknown): Record | null { const raw = record(value) if (!raw) return null const result: Record = {} for (const [name, item] of Object.entries(raw).sort(([left], [right]) => left.localeCompare(right))) { if (!['X-Relay-Route-Option', 'Anthropic-Version'].includes(name) || typeof item !== 'string' || !/^[A-Za-z0-9._-]{1,160}$/.test(item)) return null result[name] = item } return result } function identifier(value: unknown): value is string { return typeof value === 'string' && /^[A-Za-z0-9._-]{1,160}$/.test(value) } /** Parse only a known operation, keeping route evidence separate from a model's combined capabilities. */ function parseInvocation(model: Record, offer: Record, route: Record): AiModelProtocol | null { const invocation = record(route.invocation ?? offer.invocation ?? model.invocation) if (!invocation || invocation.method !== 'POST') return null const id = (Object.keys(paths) as AiModelProtocol['id'][]).find(key => paths[key] === invocation.path) const headers = safeHeaders(invocation.headers) if (!id || !headers) return null if (invocation.content_type !== (id === 'openai-image-edit' ? 'multipart/form-data' : 'application/json')) return null const scopeId = record(offer.service_scope)?.id, routeId = route.id if (scopeId !== undefined && !identifier(scopeId) || routeId !== undefined && !identifier(routeId)) return null if (routeId && headers['X-Relay-Route-Option'] !== routeId) return null const schema = record(invocation.input_schema ?? route.input_schema) const properties = record(schema?.properties) ?? {} const modes = list(route.input_modes ?? offer.input_modes ?? model.input_modes) const capabilities = list(offer.capabilities ?? model.capabilities) const isTextModel = ['text', 'multimodal'].includes(model.modality) && model.type === 'model' const isImageModel = model.modality === 'image' && [...capabilities, ...list(model.capabilities)].some(value => ['text_to_image', 'image_to_image', 'openai.images.generations', 'openai.images.edits'].includes(value)) let inputField: string | undefined, supportsImageInput = false if (id === 'openai-chat' || id === 'anthropic-messages') { if (!isTextModel) return null supportsImageInput = modes.some(mode => ['image', 'image_url', 'image_base64'].includes(mode)) || (!modes.length && model.modality === 'multimodal') } else { if (!isImageModel) return null if (id === 'openai-image-edit') { inputField = properties['image[]'] !== false && properties['image[]'] ? 'image[]' : properties.image !== false && properties.image ? 'image' : undefined supportsImageInput = Boolean(inputField) } else { // JSON image requests are safe for private files only when the route explicitly permits inline data. inputField = ['file_urls', 'images', 'image'].find(field => { const property = record(properties[field]), items = record(property?.items) const pattern = items?.pattern if (typeof pattern !== 'string' || pattern.length > 512 || !pattern.includes('data:image/') || items?.type !== 'string' || !(property?.type === 'array' || Array.isArray(property?.type) && property.type.includes('array'))) return false try { return new RegExp(pattern).test('data:image/png;base64,AA==') } catch { return false } }) supportsImageInput = Boolean(inputField) } if (id === 'async-image-task' && !supportsImageInput) return null } const maxImages = inputField && record(properties[inputField])?.maxItems const maxPromptLength = record(properties.prompt)?.maxLength return { id, path: paths[id], headers, supportsImageInput, ...(inputField ? { inputField } : {}), ...(scopeId ? { scopeId } : {}), ...(routeId ? { routeId } : {}), ...(identifier(offer.public_operation_id) ? { operationId: offer.public_operation_id } : {}), ...(Number.isInteger(maxImages) && maxImages > 0 ? { maxImages: Math.min(8, maxImages) } : {}), ...(Number.isInteger(maxPromptLength) && maxPromptLength > 0 ? { maxPromptLength } : {}) } } /** At setup time pick a concrete compatible route in the default scope; no runtime routing fallback. */ export function parseAiModelProtocols(value: unknown): AiModelProtocol[] { const model = record(value) if (!model || model.available === false || model.status && model.status !== 'available') return [] const offers = Array.isArray(model.service_offers) ? model.service_offers.map(record).filter(Boolean) as Record[] : [] if (!offers.length) { const protocol = parseInvocation(model, {}, {}); return protocol ? [protocol] : [] } if (!identifier(model.default_service_scope_id)) return [] const found: AiModelProtocol[] = [] for (const offer of offers) { if (record(offer.service_scope)?.id !== model.default_service_scope_id || offer.available === false) continue const routes = Array.isArray(offer.route_options) ? offer.route_options.map(record).filter(Boolean) as Record[] : [] const eligible = routes.filter(route => route.available !== false).sort((a, b) => Number(b.id === model.default_route_option_id) - Number(a.id === model.default_route_option_id)) const protocols = eligible.map(route => parseInvocation(model, offer, route)).filter(Boolean) as AiModelProtocol[] const chosen = protocols.find(protocol => protocol.supportsImageInput) ?? protocols[0] if (chosen && !found.some(protocol => JSON.stringify(protocol) === JSON.stringify(chosen))) found.push(chosen) } return found.sort((a, b) => Number(b.supportsImageInput) - Number(a.supportsImageInput)) } export function validateStoredAiModelProtocols(value: unknown): AiModelProtocol[] { if (!Array.isArray(value) || value.length > 8) return [] const result: AiModelProtocol[] = [] for (const entry of value) { const raw = record(entry), headers = safeHeaders(raw?.headers) if (!raw || !Object.prototype.hasOwnProperty.call(paths, raw.id) || raw.path !== paths[raw.id as AiModelProtocol['id']] || !headers || typeof raw.supportsImageInput !== 'boolean') return [] if (['scopeId', 'routeId', 'operationId'].some(field => raw[field] !== undefined && !identifier(raw[field]))) return [] if (raw.routeId && headers['X-Relay-Route-Option'] !== raw.routeId) return [] if (raw.inputField !== undefined && !['image', 'image[]', 'images', 'file_urls'].includes(raw.inputField)) return [] if (['maxImages', 'maxPromptLength'].some(field => raw[field] !== undefined && (!Number.isInteger(raw[field]) || raw[field] < 1 || raw[field] > (field === 'maxImages' ? 8 : 1000000)))) return [] if (raw.id === 'async-image-task' && (!raw.supportsImageInput || raw.inputField !== 'file_urls') || raw.id === 'openai-image-generate' && raw.supportsImageInput && !raw.inputField || raw.id === 'openai-image-edit' && (!raw.supportsImageInput || !['image', 'image[]'].includes(raw.inputField))) return [] result.push({ id: raw.id, path: raw.path, headers, supportsImageInput: raw.supportsImageInput, ...Object.fromEntries(['inputField', 'scopeId', 'routeId', 'operationId', 'maxImages', 'maxPromptLength'].filter(key => raw[key] !== undefined).map(key => [key, raw[key]])) }) } return result } export class AiModelProtocolError extends Error { readonly taskPending: boolean constructor(message: string, readonly outcomeUnknown: boolean, readonly requestId: string | null = null, readonly taskId: string | null = null, taskPending = outcomeUnknown && Boolean(taskId)) { super(message); this.name = 'AiModelProtocolError'; this.taskPending = taskPending } } export interface AiModelProtocolInput { baseUrl: string; apiKey: string; modelId: string; protocol: AiModelProtocol; prompt: string images?: Array<{ buffer: Buffer; mimeType: string }> idempotencyKey: string; beforeRequest?: () => Promise onTaskSubmitted?: (taskId: string) => Promise; resumeTaskId?: string maxOutputTokens?: number } export interface AiModelProtocolResult { text?: string; image?: Buffer; requestId: string | null; summary: Record } function safeRequestId(response: Response, data: any, apiKey: string): string | null { const id = response.headers.get('x-request-id') || data?.request_id || data?.error?.request_id return typeof id === 'string' && /^[\w-]{1,120}$/.test(id) && !id.includes(apiKey) ? id : null } async function readJson(response: Response, apiKey: string, taskId: string | null = null): Promise<{ data: any; requestId: string | null }> { const text = await response.text() let data: any try { data = JSON.parse(text) } catch { throw new AiModelProtocolError('模型响应无法确认,已暂停;请勿重复提交', true, null, taskId) } const requestId = safeRequestId(response, data, apiKey) const code = data?.error?.code ?? data?.code if (!response.ok || data?.status === 'pending_review') { const terminalTaskFailure = response.status === 409 && ['task_failed', 'task_cancelled', 'task_expired', 'task_timeout', 'content_policy_violation'].includes(code) const review = ['billing_reconciliation_required', 'submission_pending_reconciliation', 'image_size_mismatch', 'task_state_unknown'].includes(code) || data?.status === 'pending_review' // Losing access to an accepted task does not prove that generation failed. const unknown = Boolean(taskId) ? !terminalTaskFailure : response.status >= 500 || review || code === 'request_outcome_unknown' const pending = Boolean(taskId) && unknown && !review && (response.status >= 500 || response.status === 429) throw new AiModelProtocolError(unknown ? '模型请求结果待核查,已暂停;请勿重复提交' : `模型服务拒绝请求(${response.status})`, unknown, requestId, taskId, pending) } if (text.includes(apiKey)) throw new AiModelProtocolError('模型响应含敏感内容,已暂停', true, requestId, taskId) return { data, requestId } } async function normalizeImages(images: AiModelProtocolInput['images'], forAnalysis: boolean): Promise { return Promise.all((images ?? []).map(image => { const pipeline = sharp(image.buffer, { limitInputPixels: 40_000_000 }).rotate().resize(forAnalysis ? 1024 : 1536, forAnalysis ? 1024 : 1536, { fit: 'inside', withoutEnlargement: true }) return forAnalysis ? pipeline.flatten({ background: '#ffffff' }).jpeg({ quality: 85 }).toBuffer() : pipeline.png().toBuffer() })) } async function decodeImage(data: any, input: AiModelProtocolInput, taskId: string | null): Promise { let bytes: Buffer if (typeof data?.b64_json === 'string' && /^[A-Za-z0-9+/]+={0,2}$/.test(data.b64_json) && data.b64_json.length <= 28 * 1024 * 1024) bytes = Buffer.from(data.b64_json, 'base64') else if (typeof data?.url === 'string') { const response = await requestAiModelOutput(input.baseUrl, input.apiKey, data.url, input.beforeRequest) if (!response.ok) throw new AiModelProtocolError('模型图片暂时无法读取,已保留任务', true, null, taskId) bytes = Buffer.from(await response.arrayBuffer()) } else throw new AiModelProtocolError('模型没有返回可保存的图片', false, null, taskId) try { const output = await sharp(bytes, { limitInputPixels: 40_000_000 }).rotate().resize(1440, 1440, { fit: 'inside', withoutEnlargement: true }).png().toBuffer() if (output.length > 20 * 1024 * 1024) throw new Error('size') return output } catch { throw new AiModelProtocolError('模型图片无法解码或超出限制', false, null, taskId) } } export async function callAiModelProtocol(input: AiModelProtocolInput): Promise { const protocol = validateStoredAiModelProtocols([input.protocol])[0] if (!protocol) throw new AiModelProtocolError('模型协议无效,请重新同步', false) const imageCount = input.images?.length ?? 0 if (imageCount > (protocol.maxImages ?? 8) || imageCount && !protocol.supportsImageInput) throw new AiModelProtocolError('该模型接口不支持当前参考图片', false) if (protocol.maxPromptLength && input.prompt.length > protocol.maxPromptLength) throw new AiModelProtocolError(`该模型提示词最多 ${protocol.maxPromptLength} 字,请精简内容`, false) if (protocol.id === 'async-image-task' && !input.onTaskSubmitted) throw new AiModelProtocolError('异步任务缺少持久化检查点', false) if (input.resumeTaskId && (!/^[A-Za-z0-9_-]{1,160}$/.test(input.resumeTaskId) || protocol.id !== 'async-image-task')) throw new AiModelProtocolError('模型任务编号无效', false) let taskId = input.resumeTaskId ?? null, requestId: string | null = null, dispatched = Boolean(input.resumeTaskId) const summary = { adapterVersion: AI_MODEL_ADAPTER_VERSION, protocol: protocol.id, routeId: protocol.routeId ?? null } try { if (!taskId) { const forAnalysis = protocol.id === 'openai-chat' || protocol.id === 'anthropic-messages' const images = await normalizeImages(input.images, forAnalysis) const imageMimeType = forAnalysis ? 'image/jpeg' : 'image/png' if (images.some(image => image.length > 10 * 1024 * 1024) || images.reduce((total, image) => total + image.length, 0) > 24 * 1024 * 1024) throw new AiModelProtocolError('参考图片总量超出模型限制,请减少图片', false) const dataUris = images.map(image => `data:${imageMimeType};base64,${image.toString('base64')}`) let body: string | FormData const maxTokens = input.maxOutputTokens && Number.isInteger(input.maxOutputTokens) ? Math.max(32, Math.min(8000, input.maxOutputTokens)) : 8000 if (protocol.id === 'openai-chat') body = JSON.stringify({ model: input.modelId, messages: [{ role: 'user', content: images.length ? [{ type: 'text', text: input.prompt }, ...dataUris.map(url => ({ type: 'image_url', image_url: { url } }))] : input.prompt }], max_tokens: maxTokens, stream: false }) else if (protocol.id === 'anthropic-messages') body = JSON.stringify({ model: input.modelId, messages: [{ role: 'user', content: [{ type: 'text', text: input.prompt }, ...images.map(image => ({ type: 'image', source: { type: 'base64', media_type: imageMimeType, data: image.toString('base64') } }))] }], max_tokens: maxTokens, stream: false }) else if (protocol.id === 'openai-image-edit') { body = new FormData(); body.set('model', input.modelId); body.set('prompt', input.prompt); body.set('n', '1'); body.set('response_format', 'b64_json') images.forEach((image, index) => (body as FormData).append(protocol.inputField!, new Blob([new Uint8Array(image)], { type: 'image/png' }), `reference-${index + 1}.png`)) } else body = JSON.stringify({ model: input.modelId, prompt: input.prompt, ...(images.length ? { [protocol.inputField!]: dataUris } : {}), ...(protocol.id === 'openai-image-generate' ? { n: 1, response_format: 'b64_json' } : {}) }) dispatched = true const response = await requestAiModelOperation(input.baseUrl, input.apiKey, protocol.path, body, input.idempotencyKey, protocol.headers, input.beforeRequest) const parsed = await readJson(response, input.apiKey); requestId = parsed.requestId if (protocol.id === 'async-image-task') { if (typeof parsed.data?.id !== 'string' || !/^[A-Za-z0-9_-]{1,160}$/.test(parsed.data.id)) throw new AiModelProtocolError('模型未返回可核查的任务编号,请勿重发', true, requestId) taskId = parsed.data.id try { await input.onTaskSubmitted!(taskId!) } catch { throw new AiModelProtocolError('模型任务已受理,但检查点未保存;已暂停后续请求', true, requestId, taskId) } } else if (protocol.id === 'openai-chat' || protocol.id === 'anthropic-messages') { const content = protocol.id === 'openai-chat' ? parsed.data?.choices?.[0]?.message?.content : parsed.data?.content const text = typeof content === 'string' ? content : Array.isArray(content) ? content.filter(item => item?.type === 'text' && typeof item.text === 'string').map(item => item.text).join('\n') : '' if (!text.trim()) throw new AiModelProtocolError('模型未返回可用文字', false, requestId) return { text, requestId, summary } } else return { image: await decodeImage(parsed.data?.data?.[0], input, null), requestId, summary } } const deadline = Date.now() + 40_000 for (let attempt = 0; attempt < 20 && Date.now() < deadline; attempt++) { const response = await requestAiModelOperation(input.baseUrl, input.apiKey, `/extensions/v1/tasks/${taskId}/result`, undefined, undefined, {}, input.beforeRequest) const parsed = await readJson(response, input.apiKey, taskId); requestId = parsed.requestId ?? requestId if (parsed.data?.ready === true) { if (!['succeeded', 'partially_succeeded'].includes(parsed.data.status)) throw new AiModelProtocolError('模型任务未成功完成', false, requestId, taskId) const outputs = parsed.data?.result?.outputs const image = Array.isArray(outputs) ? outputs.find(item => item?.type === 'image' && (typeof item.url === 'string' || typeof item.b64_json === 'string')) : null if (!image) throw new AiModelProtocolError('模型任务没有图片产物', false, requestId, taskId) return { image: await decodeImage(image, input, taskId), requestId, summary: { ...summary, taskId } } } if (response.status !== 202 || parsed.data?.ready !== false) throw new AiModelProtocolError('模型任务状态无法确认,已保留任务', true, requestId, taskId) await new Promise(resolve => setTimeout(resolve, 2000)) } throw new AiModelProtocolError('模型仍在处理中,已保留任务,可继续查询', true, requestId, taskId) } catch (error) { if (error instanceof AiModelProtocolError) throw error // Authentication/revision guards fail before the transport opens a socket. if (error && typeof error === 'object' && 'status' in error && [400, 401, 403, 409].includes(Number(error.status))) throw error throw new AiModelProtocolError(dispatched ? '模型请求结果无法确认,已暂停;请勿重复提交' : '参考图片无法读取,请更换图片', dispatched, requestId, taskId) } }