462 lines
41 KiB
TypeScript
462 lines
41 KiB
TypeScript
import sharp from 'sharp'
|
||
import { requestAiModelOperation, requestAiModelOutput } from './ai-model-http'
|
||
|
||
export const AI_MODEL_ADAPTER_VERSION = 2
|
||
export interface AiModelProtocol {
|
||
id: 'openai-chat' | 'anthropic-messages' | 'openai-image-edit' | 'openai-image-generate' | 'async-image-task'
|
||
path: string
|
||
headers: Record<string, string>
|
||
inputField?: string
|
||
supportsImageInput: boolean
|
||
scopeId?: string
|
||
routeId?: string
|
||
operationId?: string
|
||
maxImages?: number
|
||
maxPromptLength?: number
|
||
imageQuality?: 'medium'
|
||
referenceTransport?: 'json-inline' | 'multipart'
|
||
referenceContract?: 'probe-required'
|
||
requestParameters?: Record<string, string | number | boolean>
|
||
}
|
||
const paths: Record<AiModelProtocol['id'], string> = { 'openai-chat': '/v1/chat/completions', 'anthropic-messages': '/v1/messages', 'openai-image-edit': '/v1/images/edits', 'openai-image-generate': '/v1/images/generations', 'async-image-task': '/extensions/v1/tasks' }
|
||
function record(value: unknown): Record<string, any> | null { return value && typeof value === 'object' && !Array.isArray(value) ? value as Record<string, any> : null }
|
||
function list(value: unknown): string[] { return Array.isArray(value) && value.every(item => typeof item === 'string') ? value : [] }
|
||
function safeHeaders(value: unknown): Record<string, string> | null {
|
||
const raw = record(value)
|
||
if (!raw) return null
|
||
const result: Record<string, string> = {}
|
||
for (const [name, item] of Object.entries(raw).sort(([left], [right]) => left.localeCompare(right))) {
|
||
if (!['X-Relay-Route-Option', 'Anthropic-Version'].includes(name) || typeof item !== 'string' || !/^[A-Za-z0-9._-]{1,160}$/.test(item)) return null
|
||
result[name] = item
|
||
}
|
||
return result
|
||
}
|
||
function identifier(value: unknown): value is string { return typeof value === 'string' && /^[A-Za-z0-9._-]{1,160}$/.test(value) }
|
||
|
||
const imageParameterChecks: Record<string, (value: unknown) => boolean> = {
|
||
n: value => value === 1,
|
||
stream: value => value === false,
|
||
response_format: value => typeof value === 'string' && ['url', 'b64_json'].includes(value),
|
||
size: value => typeof value === 'string' && (value === 'auto' || /^\d{3,4}x\d{3,4}$/.test(value) && value.split('x').every(part => Number(part) >= 256 && Number(part) <= 4096)),
|
||
quality: value => typeof value === 'string' && ['auto', 'low', 'medium', 'high', 'standard', 'hd', 'xhigh', 'max'].includes(value),
|
||
background: value => typeof value === 'string' && ['transparent', 'opaque', 'auto'].includes(value),
|
||
output_format: value => typeof value === 'string' && ['png', 'jpeg', 'webp'].includes(value),
|
||
aspect_ratio: value => typeof value === 'string' && (value === 'auto' || /^\d{1,2}:\d{1,2}$/.test(value)),
|
||
resolution: value => typeof value === 'string' && ['0.5K', '1K', '2K', '4K'].includes(value),
|
||
version: value => typeof value === 'string' && /^[A-Za-z0-9][A-Za-z0-9._-]{0,63}$/.test(value),
|
||
thinkingLevel: value => typeof value === 'string' && ['minimal', 'low', 'medium', 'high'].includes(value),
|
||
}
|
||
function safeImageParameters(value: unknown): AiModelProtocol['requestParameters'] | null {
|
||
const raw = record(value)
|
||
if (!raw || Object.keys(raw).some(key => !Object.prototype.hasOwnProperty.call(imageParameterChecks, key) || !imageParameterChecks[key](raw[key]))) return null
|
||
return Object.fromEntries(Object.entries(raw).sort(([a], [b]) => a.localeCompare(b)))
|
||
}
|
||
function schemaParts(value: unknown, depth = 0): Record<string, any>[] {
|
||
const raw = record(value)
|
||
if (!raw || depth > 5) return []
|
||
return [raw, ...(Array.isArray(raw.allOf) ? raw.allOf.flatMap((item: unknown) => schemaParts(item, depth + 1)) : [])]
|
||
}
|
||
function inspectableSchema(value: unknown, depth = 0): boolean {
|
||
if (depth > 5) return false
|
||
if (value === undefined || value === true || value === false) return true
|
||
const raw = record(value)
|
||
if (!raw || ['if', 'then', 'else', '$ref', '$dynamicRef'].some(key => raw[key] !== undefined)) return false
|
||
for (const key of ['allOf', 'anyOf', 'oneOf']) {
|
||
if (raw[key] !== undefined && (!Array.isArray(raw[key]) || !raw[key].length || raw[key].length > 32 || !raw[key].every((item: unknown) => inspectableSchema(item, depth + 1)))) return false
|
||
}
|
||
return raw.not === undefined || inspectableSchema(raw.not, depth + 1)
|
||
}
|
||
function acceptsSchemaValue(value: unknown, schema: unknown, depth = 0): boolean {
|
||
if (schema === false || depth > 5 || !inspectableSchema(schema)) return false
|
||
const raw = record(schema)
|
||
if (!raw) return true
|
||
if (raw.const !== undefined && raw.const !== value || Array.isArray(raw.enum) && !raw.enum.includes(value)) return false
|
||
const valueType = Array.isArray(value) ? 'array' : value === null ? 'null' : typeof value
|
||
if (raw.type !== undefined && ![raw.type].flat().some(type => type === valueType || type === 'integer' && Number.isInteger(value))) return false
|
||
if (typeof value === 'number' && (typeof raw.minimum === 'number' && value < raw.minimum || typeof raw.maximum === 'number' && value > raw.maximum)) return false
|
||
if (typeof value === 'number' && (typeof raw.exclusiveMinimum === 'number' && value <= raw.exclusiveMinimum || typeof raw.exclusiveMaximum === 'number' && value >= raw.exclusiveMaximum || typeof raw.multipleOf === 'number' && (raw.multipleOf <= 0 || value % raw.multipleOf !== 0))) return false
|
||
if (typeof value === 'string' && (typeof raw.minLength === 'number' && value.length < raw.minLength || typeof raw.maxLength === 'number' && value.length > raw.maxLength)) return false
|
||
if (Array.isArray(value)) {
|
||
if (typeof raw.minItems === 'number' && value.length < raw.minItems || typeof raw.maxItems === 'number' && value.length > raw.maxItems || raw.prefixItems !== undefined || raw.contains !== undefined) return false
|
||
if (raw.items !== undefined && !value.every(item => acceptsSchemaValue(item, raw.items, depth + 1))) return false
|
||
}
|
||
if (typeof value === 'string' && typeof raw.pattern === 'string') {
|
||
if (raw.pattern.length > 512) return false
|
||
try { if (!new RegExp(raw.pattern).test(value)) return false } catch { return false }
|
||
}
|
||
if (Array.isArray(raw.allOf) && !raw.allOf.every(item => acceptsSchemaValue(value, item, depth + 1))) return false
|
||
if (Array.isArray(raw.anyOf) && !raw.anyOf.some(item => acceptsSchemaValue(value, item, depth + 1))) return false
|
||
if (Array.isArray(raw.oneOf) && raw.oneOf.filter(item => acceptsSchemaValue(value, item, depth + 1)).length !== 1) return false
|
||
if (raw.not !== undefined && acceptsSchemaValue(value, raw.not, depth + 1)) return false
|
||
return true
|
||
}
|
||
/** Keep only bounded public request fields; never carry arbitrary metadata or callback URLs. */
|
||
function imageRequestParameters(schema: Record<string, any> | null, example: unknown, standardImages = true): AiModelProtocol['requestParameters'] | null {
|
||
if (!inspectableSchema(schema)) return null
|
||
const parts = schemaParts(schema), sample = record(example) ?? {}, parameters: Record<string, string | number | boolean> = {}
|
||
const implicit = standardImages ? ['n', 'response_format'] : []
|
||
const names = new Set(parts.flatMap(part => Object.keys(record(part.properties) ?? {})))
|
||
for (const name of Object.keys(imageParameterChecks)) {
|
||
const constraints = parts.flatMap(part => Object.prototype.hasOwnProperty.call(record(part.properties) ?? {}, name) ? [part.properties[name]] : [])
|
||
if (constraints.some(item => item === false)) {
|
||
if (implicit.includes(name)) return null
|
||
continue
|
||
}
|
||
if (!constraints.length && !implicit.includes(name)) continue
|
||
if (!constraints.length && parts.some(part => part.additionalProperties === false && !part.properties?.[name])) return null
|
||
const required = parts.some(part => list(part.required).includes(name)) || implicit.includes(name)
|
||
const declared = constraints.flatMap(item => schemaParts(item))
|
||
// A concrete size is the canonical resolution. Do not also send an optional
|
||
// resolution alias from an example, which may contradict that size.
|
||
if (name === 'resolution' && typeof parameters.size === 'string' && parameters.size !== 'auto' && !required && !declared.some(item => item.const !== undefined)) continue
|
||
const choices = declared.flatMap(item => Array.isArray(item.enum) ? item.enum : [])
|
||
if (name === 'size') choices.sort((a, b) => {
|
||
const area = (value: unknown) => typeof value === 'string' && /^\d+x\d+$/.test(value) ? value.split('x').map(Number).reduce((x, y) => x * y) : Number.MAX_SAFE_INTEGER
|
||
return area(a) - area(b)
|
||
})
|
||
const candidates = name === 'n' ? [1] : name === 'response_format' ? ['b64_json', 'url'] : [
|
||
...declared.filter(item => item.const !== undefined).map(item => item.const),
|
||
...declared.filter(item => item.default !== undefined).map(item => item.default),
|
||
...(name === 'size' ? choices : []), sample[name], ...choices,
|
||
]
|
||
const chosen = candidates.find(value => imageParameterChecks[name](value) && constraints.every(item => acceptsSchemaValue(value, item)))
|
||
if (chosen !== undefined) parameters[name] = chosen
|
||
else if (required || declared.some(item => item.const !== undefined)) return null
|
||
}
|
||
const required = parts.flatMap(part => list(part.required))
|
||
if (required.some(name => !['model', 'prompt', 'image', 'images', 'file_urls'].includes(name) && parameters[name] === undefined)) return null
|
||
if (typeof parameters.size === 'string' && parameters.size !== 'auto' && typeof parameters.resolution === 'string') {
|
||
const pixels = { '0.5K': 512, '1K': 1024, '2K': 2048, '4K': 4096 }[parameters.resolution]
|
||
if (Math.max(...parameters.size.split('x').map(Number)) !== pixels) return null
|
||
}
|
||
// These aliases describe the same resolution. Use the canonical size from the published route.
|
||
if (names.has('size') && parameters.size === undefined && parts.some(part => part.properties?.size?.enum || part.properties?.size?.const)) return null
|
||
return safeImageParameters(parameters)
|
||
}
|
||
|
||
/** An open JSON schema allows a bounded standard-field probe, never an override of a declared restriction. */
|
||
function canProbeImageReference(schema: Record<string, any> | null, supportedFields: unknown): boolean {
|
||
if (schema?.additionalProperties !== true || !inspectableSchema(schema)) return false
|
||
const referenceFields = ['image', 'image[]', 'images', 'file_urls', 'input_image', 'input_images']
|
||
for (const part of schemaParts(schema)) {
|
||
if (part.additionalProperties !== undefined && part.additionalProperties !== true || part.unevaluatedProperties !== undefined && part.unevaluatedProperties !== true) return false
|
||
if (['not', 'oneOf', 'anyOf', 'if', 'then', 'else', 'propertyNames'].some(name => part[name] !== undefined)) return false
|
||
if (Object.keys(record(part.patternProperties) ?? {}).length) return false
|
||
if (referenceFields.some(name => Object.prototype.hasOwnProperty.call(record(part.properties) ?? {}, name))) return false
|
||
}
|
||
return !Array.isArray(supportedFields) || !supportedFields.some(item => referenceFields.includes(record(item)?.name))
|
||
}
|
||
|
||
/** Parse only a known operation, keeping route evidence separate from a model's combined capabilities. */
|
||
function parseInvocation(model: Record<string, any>, offer: Record<string, any>, route: Record<string, any>): AiModelProtocol | null {
|
||
const invocation = record(route.invocation ?? offer.invocation ?? model.invocation)
|
||
if (!invocation || invocation.method !== 'POST') return null
|
||
const id = (Object.keys(paths) as AiModelProtocol['id'][]).find(key => paths[key] === invocation.path)
|
||
const headers = safeHeaders(invocation.headers)
|
||
if (!id || !headers) return null
|
||
if (invocation.content_type !== (id === 'openai-image-edit' ? 'multipart/form-data' : 'application/json') && !(id === 'openai-image-generate' && invocation.content_type === 'multipart/form-data')) return null
|
||
const scopeId = record(offer.service_scope)?.id, routeId = route.id
|
||
if (scopeId !== undefined && !identifier(scopeId) || routeId !== undefined && !identifier(routeId)) return null
|
||
if (routeId && headers['X-Relay-Route-Option'] !== routeId) return null
|
||
const schema = record(invocation.input_schema ?? route.input_schema)
|
||
const properties = record(schema?.properties) ?? {}
|
||
const schemaNodes = schemaParts(schema)
|
||
// Whole-request alternatives/negations are not implemented by the field mapper.
|
||
// Refuse such contracts rather than checking individual fields and bypassing them.
|
||
if (['openai-image-generate', 'async-image-task'].includes(id) && schema && (!inspectableSchema(schema) || schemaNodes.some(part => ['not', 'oneOf', 'anyOf'].some(name => part[name] !== undefined)))) return null
|
||
const referenceSchemas = (field: string) => schemaNodes.flatMap(part => Object.prototype.hasOwnProperty.call(record(part.properties) ?? {}, field) ? [part.properties[field]] : [])
|
||
const modes = list(route.input_modes ?? offer.input_modes ?? model.input_modes)
|
||
const capabilities = list(offer.capabilities ?? model.capabilities)
|
||
const isTextModel = ['text', 'multimodal'].includes(model.modality) && model.type === 'model'
|
||
const isImageModel = model.modality === 'image' && [...capabilities, ...list(model.capabilities)].some(value => ['text_to_image', 'image_to_image', 'openai.images.generations', 'openai.images.edits'].includes(value))
|
||
let inputField: string | undefined, supportsImageInput = false
|
||
let referenceTransport: AiModelProtocol['referenceTransport'], referenceContract: AiModelProtocol['referenceContract'], requestParameters: AiModelProtocol['requestParameters']
|
||
if (id === 'openai-chat' || id === 'anthropic-messages') {
|
||
if (!isTextModel) return null
|
||
supportsImageInput = modes.some(mode => ['image', 'image_url', 'image_base64'].includes(mode)) || (!modes.length && model.modality === 'multimodal')
|
||
} else {
|
||
if (!isImageModel) return null
|
||
if (id === 'openai-image-edit') {
|
||
inputField = properties['image[]'] !== false && properties['image[]'] ? 'image[]' : properties.image !== false && properties.image ? 'image' : undefined
|
||
supportsImageInput = Boolean(inputField)
|
||
} else if (id === 'openai-image-generate' && invocation.content_type === 'multipart/form-data' && modes.includes('file') && !referenceSchemas('image').includes(false) && ['image', 'images', 'file_urls'].some(field => referenceSchemas(field).some(record) && !referenceSchemas(field).includes(false))) {
|
||
// Use multipart only when the operation explicitly publishes that content type.
|
||
// A file input mode alone does not change an application/json contract.
|
||
inputField = 'image'
|
||
supportsImageInput = true
|
||
referenceTransport = 'multipart'
|
||
} else {
|
||
if (id === 'openai-image-generate' && invocation.content_type !== 'application/json') return null
|
||
// A declared image-URL array accepts a data URI unless its schema excludes it.
|
||
// Keep private image bytes inline; never expose a product file through public storage.
|
||
inputField = ['file_urls', 'images', 'image'].find(field => {
|
||
const property = record(properties[field]), items = record(property?.items)
|
||
const pattern = items?.pattern
|
||
const declared = referenceSchemas(field)
|
||
if (declared.includes(false)) return false
|
||
if (items?.type !== 'string' || !(property?.type === 'array' || Array.isArray(property?.type) && property.type.includes('array'))) return false
|
||
if (items.format !== undefined && !['uri', 'uri-reference'].includes(items.format)) return false
|
||
const declaredImageInput = modes.some(mode => ['image', 'image_url', 'image_base64'].includes(mode))
|
||
if ((id === 'openai-image-generate' || modes.length) && !declaredImageInput) return false
|
||
if (!declared.every(item => acceptsSchemaValue(['data:image/png;base64,AA=='], item))) return false
|
||
const itemSchemas = declared.flatMap(item => schemaParts(item)).flatMap(item => item.items !== undefined ? [item.items] : [])
|
||
if (!itemSchemas.every(item => acceptsSchemaValue('data:image/png;base64,AA==', item) && schemaParts(item).every(part => part.format === undefined || ['uri', 'uri-reference'].includes(part.format)))) return false
|
||
if (pattern === undefined) return declaredImageInput
|
||
if (typeof pattern !== 'string' || pattern.length > 512 || !pattern.includes('data:image/')) return false
|
||
try { return new RegExp(pattern).test('data:image/png;base64,AA==') } catch { return false }
|
||
})
|
||
supportsImageInput = Boolean(inputField)
|
||
if (id === 'openai-image-generate' && supportsImageInput) referenceTransport = 'json-inline'
|
||
}
|
||
if (id === 'openai-image-generate' && !supportsImageInput && invocation.content_type === 'application/json' && identifier(routeId) && routeId === model.default_route_option_id && canProbeImageReference(schema, invocation.supported_fields)) {
|
||
inputField = 'image'
|
||
supportsImageInput = true
|
||
referenceTransport = 'json-inline'
|
||
referenceContract = 'probe-required'
|
||
}
|
||
if (referenceTransport || id === 'async-image-task') {
|
||
const parameters = imageRequestParameters(schema, invocation.request_example, id !== 'async-image-task')
|
||
if (!parameters) return null
|
||
if (referenceTransport || Object.keys(parameters).length) requestParameters = parameters
|
||
}
|
||
if (id === 'async-image-task' && !supportsImageInput) return null
|
||
}
|
||
const imageConstraints = inputField && (referenceTransport === 'multipart' ? ['image', 'images', 'file_urls'].flatMap(field => referenceSchemas(field).flatMap(value => schemaParts(value))) : referenceSchemas(inputField).flatMap(value => schemaParts(value)))
|
||
if (imageConstraints && imageConstraints.some(part => part.maxItems !== undefined && (!Number.isInteger(part.maxItems) || part.maxItems < 1))) return null
|
||
const declaredLimits = imageConstraints ? imageConstraints.flatMap(part => [part.maxItems, ...(Array.isArray(part.anyOf) ? part.anyOf.map((entry: any) => record(entry)?.maxItems) : [])]).filter(value => Number.isInteger(value) && value > 0) : []
|
||
// This is the platform's test ceiling, not a claimed upstream limit. Runtime
|
||
// bindings are reduced to the number of independent references actually verified.
|
||
const maxImages = referenceContract === 'probe-required' ? 8 : declaredLimits.length ? Math.min(...declaredLimits) : undefined
|
||
const maxPromptLength = record(properties.prompt)?.maxLength
|
||
// The editor has no quality control: choose an explicit supported working
|
||
// quality instead of relying on an undocumented upstream default.
|
||
const quality = record(properties.quality)
|
||
const imageQuality = id === 'openai-image-generate' && referenceTransport !== 'multipart' && list(quality?.enum).includes('medium') &&
|
||
(quality?.const === undefined || quality.const === 'medium') &&
|
||
(quality?.default === undefined || quality.default === 'medium') && referenceSchemas('quality').every(item => acceptsSchemaValue('medium', item)) ? 'medium' as const : undefined
|
||
if (requestParameters && imageQuality) requestParameters = safeImageParameters({ ...requestParameters, quality: imageQuality }) ?? undefined
|
||
return { id, path: paths[id], headers, supportsImageInput, ...(inputField ? { inputField } : {}),
|
||
...(scopeId ? { scopeId } : {}), ...(routeId ? { routeId } : {}),
|
||
...(identifier(offer.public_operation_id) ? { operationId: offer.public_operation_id } : {}),
|
||
...(typeof maxImages === 'number' && Number.isInteger(maxImages) && maxImages > 0 ? { maxImages: Math.min(8, maxImages) } : {}),
|
||
...(Number.isInteger(maxPromptLength) && maxPromptLength > 0 ? { maxPromptLength } : {}),
|
||
...(imageQuality ? { imageQuality } : {}),
|
||
...(referenceContract ? { referenceContract } : {}),
|
||
...(referenceTransport ? { referenceTransport } : {}),
|
||
...(requestParameters ? { requestParameters } : {}) }
|
||
}
|
||
|
||
/** At setup time pick a concrete compatible route in the default scope; no runtime routing fallback. */
|
||
export function parseAiModelProtocols(value: unknown): AiModelProtocol[] {
|
||
const model = record(value)
|
||
if (!model || model.available === false || model.status && model.status !== 'available') return []
|
||
const offers = Array.isArray(model.service_offers) ? model.service_offers.map(record).filter(Boolean) as Record<string, any>[] : []
|
||
if (!offers.length) { const protocol = parseInvocation(model, {}, {}); return protocol ? [protocol] : [] }
|
||
if (!identifier(model.default_service_scope_id)) return []
|
||
const found: AiModelProtocol[] = []
|
||
for (const offer of offers) {
|
||
if (record(offer.service_scope)?.id !== model.default_service_scope_id || offer.available === false) continue
|
||
const routes = Array.isArray(offer.route_options) ? offer.route_options.map(record).filter(Boolean) as Record<string, any>[] : []
|
||
const eligible = routes.filter(route => route.available !== false).sort((a, b) => Number(b.id === model.default_route_option_id) - Number(a.id === model.default_route_option_id))
|
||
const protocols = eligible.map(route => parseInvocation(model, offer, route)).filter(Boolean) as AiModelProtocol[]
|
||
const chosen = protocols.find(protocol => protocol.id === 'async-image-task' && protocol.supportsImageInput) ?? protocols.find(protocol => !protocol.referenceContract && protocol.referenceTransport === 'json-inline') ?? protocols.find(protocol => !protocol.referenceContract && protocol.supportsImageInput) ?? protocols.find(protocol => protocol.supportsImageInput) ?? protocols[0]
|
||
if (chosen && !found.some(protocol => JSON.stringify(protocol) === JSON.stringify(chosen))) found.push(chosen)
|
||
}
|
||
// An explicitly published task operation can persist its ID before a long
|
||
// generation finishes. Prefer that recoverable contract at sync time.
|
||
return found.sort((a, b) => Number(b.supportsImageInput) - Number(a.supportsImageInput) || Number(Boolean(a.referenceContract)) - Number(Boolean(b.referenceContract)) || Number(b.id === 'async-image-task') - Number(a.id === 'async-image-task') || Number(b.referenceTransport === 'json-inline') - Number(a.referenceTransport === 'json-inline'))
|
||
}
|
||
|
||
export function validateStoredAiModelProtocols(value: unknown): AiModelProtocol[] {
|
||
if (!Array.isArray(value) || value.length > 8) return []
|
||
const result: AiModelProtocol[] = []
|
||
for (const entry of value) {
|
||
const raw = record(entry), headers = safeHeaders(raw?.headers)
|
||
if (!raw || !Object.prototype.hasOwnProperty.call(paths, raw.id) || raw.path !== paths[raw.id as AiModelProtocol['id']] || !headers || typeof raw.supportsImageInput !== 'boolean') return []
|
||
if (['scopeId', 'routeId', 'operationId'].some(field => raw[field] !== undefined && !identifier(raw[field]))) return []
|
||
if (raw.routeId && headers['X-Relay-Route-Option'] !== raw.routeId) return []
|
||
if (raw.inputField !== undefined && !['image', 'image[]', 'images', 'file_urls'].includes(raw.inputField)) return []
|
||
if (['maxImages', 'maxPromptLength'].some(field => raw[field] !== undefined && (!Number.isInteger(raw[field]) || raw[field] < 1 || raw[field] > (field === 'maxImages' ? 8 : 1000000)))) return []
|
||
if (raw.imageQuality !== undefined && (raw.id !== 'openai-image-generate' || raw.imageQuality !== 'medium')) return []
|
||
if (raw.referenceTransport !== undefined && (!['json-inline', 'multipart'].includes(raw.referenceTransport) || raw.id !== 'openai-image-generate' || !raw.supportsImageInput || !['image', 'images', 'file_urls'].includes(raw.inputField) || raw.referenceTransport === 'multipart' && raw.inputField !== 'image')) return []
|
||
if (raw.referenceContract !== undefined && (raw.referenceContract !== 'probe-required' || raw.id !== 'openai-image-generate' || raw.referenceTransport !== 'json-inline' || raw.inputField !== 'image' || !Number.isInteger(raw.maxImages) || raw.maxImages < 1 || raw.maxImages > 8 || !raw.routeId || !raw.scopeId)) return []
|
||
const requestParameters = raw.requestParameters !== undefined ? safeImageParameters(raw.requestParameters) : undefined
|
||
if (raw.referenceTransport !== undefined && (!requestParameters || requestParameters.n !== 1 || !requestParameters.response_format) || raw.requestParameters !== undefined && (raw.referenceTransport === undefined && raw.id !== 'async-image-task' || !requestParameters)) return []
|
||
if (raw.id === 'async-image-task' && (!raw.supportsImageInput || raw.inputField !== 'file_urls') || raw.id === 'openai-image-generate' && raw.supportsImageInput && !raw.inputField || raw.id === 'openai-image-edit' && (!raw.supportsImageInput || !['image', 'image[]'].includes(raw.inputField))) return []
|
||
result.push({ id: raw.id, path: raw.path, headers, supportsImageInput: raw.supportsImageInput,
|
||
...Object.fromEntries(['inputField', 'scopeId', 'routeId', 'operationId', 'maxImages', 'maxPromptLength', 'imageQuality', 'referenceTransport', 'referenceContract'].filter(key => raw[key] !== undefined).map(key => [key, raw[key]])),
|
||
...(requestParameters ? { requestParameters } : {}) })
|
||
}
|
||
return result
|
||
}
|
||
|
||
/** The actual bounded image settings, shared by request construction and admin diagnostics. */
|
||
export function aiModelImageRequestParameters(protocol: AiModelProtocol): Record<string, string | number | boolean> {
|
||
if (protocol.id === 'async-image-task') return { ...protocol.requestParameters }
|
||
if (!['openai-image-edit', 'openai-image-generate'].includes(protocol.id)) return {}
|
||
return { n: 1, response_format: 'b64_json', ...(protocol.imageQuality ? { quality: protocol.imageQuality } : {}), ...protocol.requestParameters }
|
||
}
|
||
|
||
export class AiModelProtocolError extends Error {
|
||
readonly taskPending: boolean
|
||
constructor(message: string, readonly outcomeUnknown: boolean, readonly requestId: string | null = null, readonly taskId: string | null = null, taskPending = outcomeUnknown && Boolean(taskId)) { super(message); this.name = 'AiModelProtocolError'; this.taskPending = taskPending }
|
||
}
|
||
export interface AiModelProtocolInput {
|
||
baseUrl: string; apiKey: string; modelId: string; protocol: AiModelProtocol; prompt: string
|
||
images?: Array<{ buffer: Buffer; mimeType: string }>
|
||
idempotencyKey: string; beforeRequest?: () => Promise<void>
|
||
onTaskSubmitted?: (taskId: string) => Promise<void>; resumeTaskId?: string; resumeRequestId?: string
|
||
maxOutputTokens?: number
|
||
}
|
||
export interface AiModelProtocolResult { text?: string; image?: Buffer; requestId: string | null; summary: Record<string, unknown> }
|
||
|
||
function safeRequestId(response: Response, data: any, apiKey: string): string | null {
|
||
const id = response.headers.get('x-request-id') || data?.request_id || data?.error?.request_id
|
||
return typeof id === 'string' && /^[\w-]{1,120}$/.test(id) && !id.includes(apiKey) ? id : null
|
||
}
|
||
async function readJson(response: Response, apiKey: string, taskId: string | null = null): Promise<{ data: any; requestId: string | null }> {
|
||
const text = await response.text()
|
||
let data: any
|
||
try { data = JSON.parse(text) } catch { throw new AiModelProtocolError(`上游返回异常响应（${response.status}），请核查请求结果；勿重复提交`, true, safeRequestId(response, null, apiKey), taskId) }
|
||
const requestId = safeRequestId(response, data, apiKey)
|
||
const code = data?.error?.code ?? data?.code
|
||
if (!response.ok || data?.status === 'pending_review') {
|
||
const terminalTaskFailure = response.status === 409 && ['task_failed', 'task_cancelled', 'task_expired', 'task_timeout', 'content_policy_violation'].includes(code)
|
||
const review = ['billing_reconciliation_required', 'submission_pending_reconciliation', 'image_size_mismatch', 'task_state_unknown'].includes(code) || data?.status === 'pending_review'
|
||
// Losing access to an accepted task does not prove that generation failed.
|
||
const definiteRejection = ['service_rejected', 'model_temporarily_unavailable'].includes(code)
|
||
const overloaded = typeof data?.error?.message === 'string' && /model load is too high|model is overloaded|capacity exceeded/i.test(data.error.message.slice(0, 1000))
|
||
const unknown = Boolean(taskId) ? !terminalTaskFailure : review || code === 'request_outcome_unknown' || !definiteRejection && response.status >= 500
|
||
const pending = Boolean(taskId) && unknown && !review && (response.status >= 500 || response.status === 429)
|
||
const parameterCodes = ['unsupported_parameter', 'unsupported_value', 'invalid_parameter', 'invalid_value', 'missing_required_parameter']
|
||
const publicParameters = ['model', 'messages', 'max_tokens', 'max_completion_tokens', 'stream', 'temperature', 'response_format', 'reasoning_effort', 'thinking', 'image', 'image[]', 'images', 'file_urls', 'size', 'quality', 'n']
|
||
const parameter = publicParameters.includes(data?.error?.param) ? data.error.param : null
|
||
const parameterReason = parameterCodes.includes(code) ? `上游拒绝${parameter ? `参数 ${parameter}` : '调用参数'}（HTTP ${response.status}）；请核对该模型的参数合同` : code === 'context_length_exceeded' ? '产品资料超过该模型的上下文限制，本次调用已结束' : null
|
||
throw new AiModelProtocolError(unknown ? '模型请求结果待核查，已暂停；请勿重复提交' : overloaded ? '上游模型负载过高，本次请求已被拒绝，请稍后重新测试' : code === 'model_temporarily_unavailable' ? '上游模型暂不可用，请稍后重新测试' : parameterReason || `上游拒绝模型请求（HTTP ${response.status}），测试未通过`, unknown, requestId, taskId, pending)
|
||
}
|
||
if (text.includes(apiKey)) throw new AiModelProtocolError('模型响应含敏感内容，已暂停', true, requestId, taskId)
|
||
return { data, requestId }
|
||
}
|
||
async function normalizeImages(images: AiModelProtocolInput['images'], forAnalysis: boolean): Promise<Buffer[]> {
|
||
return Promise.all((images ?? []).map(image => {
|
||
const pipeline = sharp(image.buffer, { limitInputPixels: 40_000_000 }).rotate().resize(forAnalysis ? 1024 : 1536, forAnalysis ? 1024 : 1536, { fit: 'inside', withoutEnlargement: true })
|
||
return forAnalysis ? pipeline.flatten({ background: '#ffffff' }).jpeg({ quality: 85 }).toBuffer() : pipeline.png().toBuffer()
|
||
}))
|
||
}
|
||
|
||
function decodeTextResult(data: any, protocol: AiModelProtocol, requestId: string | null): string {
|
||
const finishReason = protocol.id === 'openai-chat' ? data?.choices?.[0]?.finish_reason : data?.stop_reason
|
||
if (finishReason === 'length' || finishReason === 'max_tokens') throw new AiModelProtocolError('模型输出达到当前预算，结果不完整；本次调用已结束', false, requestId)
|
||
if (finishReason === 'content_filter' || finishReason === 'refusal') throw new AiModelProtocolError('模型拒绝输出当前内容，本次调用已结束', false, requestId)
|
||
if (['tool_calls', 'function_call', 'tool_use'].includes(finishReason)) throw new AiModelProtocolError('模型要求执行工具，当前详情页未启用工具执行；本次调用已结束', false, requestId)
|
||
const content = protocol.id === 'openai-chat' ? data?.choices?.[0]?.message?.content : data?.content
|
||
const text = typeof content === 'string' ? content : Array.isArray(content) ? content.filter(item => item?.type === 'text' && typeof item.text === 'string').map(item => item.text).join('\n') : ''
|
||
if (!text.trim()) throw new AiModelProtocolError('模型未返回可用文字，本次调用已结束', false, requestId)
|
||
return text
|
||
}
|
||
|
||
async function decodeImage(data: any, input: AiModelProtocolInput, taskId: string | null, summary: Record<string, unknown>): Promise<Buffer> {
|
||
let bytes: Buffer
|
||
if (typeof data?.b64_json === 'string' && /^[A-Za-z0-9+/]+={0,2}$/.test(data.b64_json) && data.b64_json.length <= 28 * 1024 * 1024) bytes = Buffer.from(data.b64_json, 'base64')
|
||
else if (typeof data?.url === 'string') {
|
||
const response = await requestAiModelOutput(input.baseUrl, input.apiKey, data.url, input.beforeRequest)
|
||
if (!response.ok) throw new AiModelProtocolError('模型图片暂时无法读取，已保留任务', true, null, taskId)
|
||
bytes = Buffer.from(await response.arrayBuffer())
|
||
} else throw new AiModelProtocolError('模型没有返回可保存的图片', false, null, taskId)
|
||
try {
|
||
const source = await sharp(bytes, { limitInputPixels: 40_000_000 }).metadata()
|
||
const output = await sharp(bytes, { limitInputPixels: 40_000_000 }).rotate().resize(1440, 1440, { fit: 'inside', withoutEnlargement: true }).png().toBuffer()
|
||
if (output.length > 20 * 1024 * 1024) throw new Error('size')
|
||
const stored = await sharp(output).metadata()
|
||
Object.assign(summary, { generatedWidth: source.width, generatedHeight: source.height, storedWidth: stored.width, storedHeight: stored.height })
|
||
return output
|
||
} catch { throw new AiModelProtocolError('模型图片无法解码或超出限制', false, null, taskId) }
|
||
}
|
||
|
||
export async function callAiModelProtocol(input: AiModelProtocolInput): Promise<AiModelProtocolResult> {
|
||
const protocol = validateStoredAiModelProtocols([input.protocol])[0]
|
||
if (!protocol) throw new AiModelProtocolError('模型协议无效，请重新同步', Boolean(input.resumeRequestId), input.resumeRequestId || null)
|
||
const retainedRead = Boolean(input.resumeRequestId)
|
||
const imageCount = input.images?.length ?? 0
|
||
if (!retainedRead && imageCount && !protocol.supportsImageInput) throw new AiModelProtocolError('参考图传入方式尚未适配，请重新同步模型目录', false)
|
||
if (!retainedRead && imageCount > (protocol.maxImages ?? 8)) throw new AiModelProtocolError(`当前模型最多可接收 ${protocol.maxImages ?? 8} 张参考图，请减少图片`, false)
|
||
if (!retainedRead && protocol.maxPromptLength && input.prompt.length > protocol.maxPromptLength) throw new AiModelProtocolError(`该模型提示词最多 ${protocol.maxPromptLength} 字，请精简内容`, false)
|
||
if (!retainedRead && protocol.id === 'async-image-task' && !input.onTaskSubmitted) throw new AiModelProtocolError('异步任务缺少持久化检查点', false)
|
||
if (input.resumeTaskId && (!/^[A-Za-z0-9_-]{1,160}$/.test(input.resumeTaskId) || protocol.id !== 'async-image-task')) throw new AiModelProtocolError('模型任务编号无效', false)
|
||
if (input.resumeRequestId && (input.resumeTaskId || !/^[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}$/i.test(input.resumeRequestId) || !['openai-chat', 'anthropic-messages', 'openai-image-generate', 'openai-image-edit'].includes(protocol.id))) throw new AiModelProtocolError('原请求编号无效或当前协议不能取回原响应', true, input.resumeRequestId)
|
||
let taskId = input.resumeTaskId ?? null, requestId: string | null = null, dispatched = Boolean(input.resumeTaskId)
|
||
const summary: Record<string, unknown> = { adapterVersion: AI_MODEL_ADAPTER_VERSION, protocol: protocol.id, routeId: protocol.routeId ?? null,
|
||
...(['openai-image-edit', 'openai-image-generate', 'async-image-task'].includes(protocol.id) ? { requestParameters: aiModelImageRequestParameters(protocol) } : {}),
|
||
...(protocol.referenceTransport ? { referenceTransport: protocol.referenceTransport } : {}),
|
||
...(protocol.referenceContract ? { referenceContract: protocol.referenceContract } : {}) }
|
||
try {
|
||
if (input.resumeRequestId) {
|
||
// This path is GET-only, with no body or idempotency header. Missing retention does not prove failure.
|
||
requestId = input.resumeRequestId
|
||
let original: any
|
||
try {
|
||
const response = await requestAiModelOperation(input.baseUrl, input.apiKey, `/api/external/requests/${requestId}/result`, undefined, undefined, {}, input.beforeRequest)
|
||
const parsed = await readJson(response, input.apiKey)
|
||
if (response.status !== 200) throw new Error('Original response unavailable')
|
||
original = parsed.data
|
||
} catch (error) {
|
||
if (error && typeof error === 'object' && 'status' in error && [400, 401, 403, 409].includes(Number(error.status))) throw error
|
||
throw new AiModelProtocolError('原请求结果尚未取回，请核查编号、访问权限或留存状态；未留存不代表执行失败，勿重复提交', true, requestId)
|
||
}
|
||
if (protocol.id === 'openai-chat' || protocol.id === 'anthropic-messages') {
|
||
// A supplied request ID must not import another model's output into this execution.
|
||
if (original?.model !== input.modelId || protocol.id === 'openai-chat' && !Array.isArray(original?.choices) || protocol.id === 'anthropic-messages' && !Array.isArray(original?.content)) throw new AiModelProtocolError('原响应的模型或格式与当前任务不一致，不能恢复；请核查原请求', true, requestId)
|
||
return { text: decodeTextResult(original, protocol, requestId), requestId, summary: { ...summary, retainedResponse: true } }
|
||
}
|
||
if (!Array.isArray(original?.data) || original.data.length !== 1) throw new AiModelProtocolError('原图片响应格式尚未确认，请核查原请求；勿重复提交', true, requestId)
|
||
return { image: await decodeImage(original.data[0], input, null, summary), requestId, summary: { ...summary, retainedResponse: true } }
|
||
}
|
||
if (!taskId) {
|
||
const forAnalysis = protocol.id === 'openai-chat' || protocol.id === 'anthropic-messages'
|
||
const images = await normalizeImages(input.images, forAnalysis)
|
||
const imageMimeType = forAnalysis ? 'image/jpeg' : 'image/png'
|
||
if (images.some(image => image.length > 10 * 1024 * 1024) || images.reduce((total, image) => total + image.length, 0) > 24 * 1024 * 1024) throw new AiModelProtocolError('参考图片总量超出模型限制，请减少图片', false)
|
||
const dataUris = images.map(image => `data:${imageMimeType};base64,${image.toString('base64')}`)
|
||
let body: string | FormData
|
||
const maxTokens = input.maxOutputTokens && Number.isInteger(input.maxOutputTokens) ? Math.max(32, Math.min(8000, input.maxOutputTokens)) : 8000
|
||
if (protocol.id === 'openai-chat') body = JSON.stringify({ model: input.modelId, messages: [{ role: 'user', content: images.length ? [{ type: 'text', text: input.prompt }, ...dataUris.map(url => ({ type: 'image_url', image_url: { url } }))] : input.prompt }], max_tokens: maxTokens, stream: false })
|
||
else if (protocol.id === 'anthropic-messages') body = JSON.stringify({ model: input.modelId, messages: [{ role: 'user', content: [{ type: 'text', text: input.prompt }, ...images.map(image => ({ type: 'image', source: { type: 'base64', media_type: imageMimeType, data: image.toString('base64') } }))] }], max_tokens: maxTokens, stream: false })
|
||
else if (protocol.id === 'openai-image-edit' || protocol.referenceTransport === 'multipart') {
|
||
body = new FormData(); body.set('model', input.modelId); body.set('prompt', input.prompt)
|
||
for (const [field, value] of Object.entries(aiModelImageRequestParameters(protocol))) body.set(field, String(value))
|
||
images.forEach((image, index) => (body as FormData).append(protocol.inputField!, new Blob([new Uint8Array(image)], { type: 'image/png' }), `reference-${index + 1}.png`))
|
||
} else body = JSON.stringify({ model: input.modelId, prompt: input.prompt, ...(images.length ? { [protocol.inputField!]: dataUris } : {}), ...aiModelImageRequestParameters(protocol) })
|
||
dispatched = true
|
||
const response = await requestAiModelOperation(input.baseUrl, input.apiKey, protocol.path, body, input.idempotencyKey, protocol.headers, input.beforeRequest)
|
||
const parsed = await readJson(response, input.apiKey); requestId = parsed.requestId
|
||
if (protocol.id === 'async-image-task') {
|
||
if (typeof parsed.data?.id !== 'string' || !/^[A-Za-z0-9_-]{1,160}$/.test(parsed.data.id)) throw new AiModelProtocolError('模型未返回可核查的任务编号，请勿重发', true, requestId)
|
||
taskId = parsed.data.id
|
||
try { await input.onTaskSubmitted!(taskId!) } catch { throw new AiModelProtocolError('模型任务已受理，但检查点未保存；已暂停后续请求', true, requestId, taskId) }
|
||
} else if (protocol.id === 'openai-chat' || protocol.id === 'anthropic-messages') {
|
||
return { text: decodeTextResult(parsed.data, protocol, requestId), requestId, summary }
|
||
} else return { image: await decodeImage(parsed.data?.data?.[0], input, null, summary), requestId, summary }
|
||
}
|
||
const deadline = Date.now() + 40_000
|
||
for (let attempt = 0; attempt < 20 && Date.now() < deadline; attempt++) {
|
||
const response = await requestAiModelOperation(input.baseUrl, input.apiKey, `/extensions/v1/tasks/${taskId}/result`, undefined, undefined, {}, input.beforeRequest)
|
||
const parsed = await readJson(response, input.apiKey, taskId); requestId = parsed.requestId ?? requestId
|
||
if (parsed.data?.ready === true) {
|
||
if (!['succeeded', 'partially_succeeded'].includes(parsed.data.status)) throw new AiModelProtocolError('模型任务未成功完成', false, requestId, taskId)
|
||
const outputs = parsed.data?.result?.outputs
|
||
const image = Array.isArray(outputs) ? outputs.find(item => item?.type === 'image' && (typeof item.url === 'string' || typeof item.b64_json === 'string')) : null
|
||
if (!image) throw new AiModelProtocolError('模型任务没有图片产物', false, requestId, taskId)
|
||
return { image: await decodeImage(image, input, taskId, summary), requestId, summary: { ...summary, taskId } }
|
||
}
|
||
if (response.status !== 202 || parsed.data?.ready !== false) throw new AiModelProtocolError('模型任务状态无法确认，已保留任务', true, requestId, taskId)
|
||
await new Promise(resolve => setTimeout(resolve, 2000))
|
||
}
|
||
throw new AiModelProtocolError('模型仍在处理中，已保留任务，可继续查询', true, requestId, taskId)
|
||
} catch (error) {
|
||
if (error instanceof AiModelProtocolError) throw error
|
||
// Authentication/revision guards fail before the transport opens a socket.
|
||
if (error && typeof error === 'object' && 'status' in error && [400, 401, 403, 409].includes(Number(error.status))) throw error
|
||
throw new AiModelProtocolError(dispatched ? '模型请求结果无法确认，已暂停；请勿重复提交' : '参考图片无法读取，请更换图片', dispatched, requestId, taskId)
|
||
}
|
||
}
|