diff --git a/components/admin/AiModelManagementPanel.test.tsx b/components/admin/AiModelManagementPanel.test.tsx index 9e36361..b3a6d53 100644 --- a/components/admin/AiModelManagementPanel.test.tsx +++ b/components/admin/AiModelManagementPanel.test.tsx @@ -1,6 +1,6 @@ import { act, fireEvent, render, screen, waitFor, within } from '@testing-library/react' import AiModelManagementPanel from './AiModelManagementPanel' -import type { AiModelAdminState, AiModelVerificationSummary } from '@/lib/ai-model-types' +import type { AiModelAdminState, AiModelCatalogEntry, AiModelVerificationSummary } from '@/lib/ai-model-types' function response(body: unknown, status = 200) { return { ok: status < 400, status, json: async () => body } as Response } const initial = (): AiModelAdminState => ({ providers: [], revision: 0, defaults: { analysis: null, copy: null, image: null }, encryptionConfigured: true, legacyAvailable: true }) @@ -25,14 +25,17 @@ function jointConfiguration() { return next } -test.each(['failed', 'unknown'] as const)('生图 %s 的具体原因直接可见，未知结果不能重试', async status => { +test.each(['failed', 'unknown'] as const)('生图 %s 显示明确状态，具体原因可展开，未知结果不能重试', async status => { const next = configured(), model = next.providers[0].models[1] model.enabled = false; model.stages = [] model.verification.image = { status, id: 'verification-image', checkedAt: null, requestId: 'request-image', message: '上游返回异常响应（502），请核查请求结果；勿重复提交', canReconcile: status === 'unknown' } global.fetch = jest.fn(async () => response(next)) as typeof fetch render() const row = await screen.findByLabelText('nano-banana-2 · 现有上游', { selector: 'div' }) - expect(within(row).getAllByText(model.verification.image.message!, { selector: 'p' }).find(element => !element.closest('[hidden]'))).toBeVisible() + expect(within(row).getByRole('status')).toHaveTextContent(status === 'failed' ? '测试失败' : '待核查') + expect(within(row).getByText(model.verification.image.message!, { selector: 'p' })).not.toBeVisible() + fireEvent.click(within(row).getByRole('button', { name: 'nano-banana-2 · 现有上游 图片生成测试详情' })) + expect(within(row).getByText(model.verification.image.message!, { selector: 'p' })).toBeVisible() expect(within(row).getByRole('checkbox', { name: 'nano-banana-2 · 现有上游 用于图片生成' })).toBeDisabled() if (status === 'unknown') expect(within(row).getByRole('button', { name: '核查 nano-banana-2 · 现有上游 的图片生成结果' })).toBeEnabled() expect((global.fetch as jest.Mock).mock.calls.every(([, init]) => !init?.method)).toBe(true) @@ -278,8 +281,8 @@ test('按能力筛选未开放模型，纯对话不能分配生图或图片分 expect(screen.queryByText('new-vision-model')).not.toBeInTheDocument() const image = screen.getByRole('checkbox', { name: 'text-chat-only · 现有上游 用于图片生成' }) expect(image).toBeDisabled() - expect(image).toHaveAccessibleDescription('此模型不支持图片生成') - expect(image.closest('label')).toHaveAttribute('title', '此模型不支持图片生成') + expect(image).toHaveAccessibleDescription('当前未接入图片生成用途') + expect(image.closest('label')).toHaveAttribute('title', '当前未接入图片生成用途') expect(screen.getByRole('checkbox', { name: 'text-chat-only · 现有上游 用于资料分析' })).toBeDisabled() expect(screen.getByRole('checkbox', { name: 'text-chat-only · 现有上游 用于卖点文案' })).toBeEnabled() expect(image).toBeDisabled() @@ -307,21 +310,90 @@ test('能力待识别模型没有人工兼容开关，不能解锁用途或实 expect(screen.queryByText('gemini-2.5-flash')).not.toBeInTheDocument() }) -test('生图接口未接入时就近显示服务端具体原因', async () => { +test('生图接口未接入时显示待适配，详情解释原因而不冒充测试结果', async () => { const next = configured() - next.providers[0].models[2] = { ...next.providers[0].models[2], id: 'image-other-protocol', capabilities: { kind: 'image', allowedStages: [], source: 'upstream', reason: '具备图片输出能力，当前参考图协议待适配' } } + next.providers[0].models[2] = { ...next.providers[0].models[2], id: 'image-other-protocol', capabilities: { kind: 'image', allowedStages: [], source: 'upstream', reason: '参考图待适配：目录已声明图片输入，当前传入方式尚未接入' } } global.fetch = jest.fn(async () => response(next)) as typeof fetch render() const image = await screen.findByRole('checkbox', { name: 'image-other-protocol · 现有上游 用于图片生成' }) expect(image).toBeDisabled() - expect(image.closest('label')).toHaveAttribute('title', '具备图片输出能力，当前参考图协议待适配') - expect(image).toHaveAccessibleDescription('具备图片输出能力，当前参考图协议待适配') - expect(screen.getAllByRole('status').find(element => element.textContent === '具备图片输出能力，当前参考图协议待适配')).toBeVisible() + expect(image.closest('label')).toHaveAttribute('title', '参考图接入待适配，完成后可测试') + expect(image).toHaveAccessibleDescription('参考图接入待适配，完成后可测试') + const row = screen.getByLabelText('image-other-protocol · 现有上游', { selector: 'div' }) + expect(within(row).getByRole('status')).toHaveTextContent('待适配') + expect(within(row).getByText('参考图待适配：目录已声明图片输入，当前传入方式尚未接入')).not.toBeVisible() + fireEvent.click(within(row).getByRole('button', { name: 'image-other-protocol · 现有上游 参考图接入详情' })) + expect(within(row).getByText('参考图待适配：目录已声明图片输入，当前传入方式尚未接入')).toBeVisible() + expect(within(row).queryByRole('button', { name: /^测试 / })).not.toBeInTheDocument() fireEvent.change(screen.getByLabelText('按能力筛选模型'), { target: { value: 'image' } }) expect(screen.getByText('image-other-protocol')).toBeInTheDocument() expect(screen.queryByText('gemini-2.5-flash')).not.toBeInTheDocument() }) +test('状态筛选区分待适配、未测试、真实失败、待核查和人工核查历史，筛选不触发调用', async () => { + const next = configured(), template = next.providers[0].models[1] + const entries = [ + { key: 'adapting', status: 'untested' }, { key: 'untested', status: 'untested' }, + { key: 'failed', status: 'failed' }, { key: 'unknown', status: 'unknown' }, + { key: 'reconciled', status: 'failed' }, { key: 'stale', status: 'stale' }, + ] as const + next.providers[0].models = entries.map(({ key, status }): AiModelCatalogEntry => ({ + ...template, key, id: `image-${key}`, enabled: false, stages: [], + capabilities: { ...template.capabilities, allowedStages: key === 'adapting' ? [] : ['image'], reason: key === 'adapting' ? '参考图待适配：当前目录尚未确认可用的参考图传入方式' : template.capabilities.reason }, + verification: { ...verification(), image: { status, id: status === 'untested' ? null : `verification-${key}`, checkedAt: null, requestId: null, message: key === 'reconciled' ? '人工核查确认历史请求已结束' : null, reconciled: key === 'reconciled' } }, + })) + next.defaults = { analysis: null, copy: null, image: null } + global.fetch = jest.fn(async () => response(next)) as typeof fetch + render() + const filter = await screen.findByLabelText('按状态筛选模型') + for (const { key } of entries) { + fireEvent.change(filter, { target: { value: key } }) + expect(screen.getByText(`image-${key}`)).toBeInTheDocument() + for (const other of entries.filter(entry => entry.key !== key)) expect(screen.queryByText(`image-${other.key}`)).not.toBeInTheDocument() + } + fireEvent.change(filter, { target: { value: 'reconciled' } }) + const row = screen.getByLabelText('image-reconciled · 现有上游', { selector: 'div' }) + expect(within(row).getByRole('status')).toHaveTextContent('已核查，可重测') + expect(within(row).queryByText('测试失败')).not.toBeInTheDocument() + expect(within(row).getByRole('checkbox', { name: 'image-reconciled · 现有上游 用于图片生成' })).toBeDisabled() + fireEvent.click(within(row).getByRole('button', { name: 'image-reconciled · 现有上游 图片生成测试详情' })) + expect(within(row).getByText('历史请求经人工核查结束，尚需重新测试。')).toBeVisible() + expect(within(row).getByText('人工核查确认历史请求已结束')).toBeVisible() + expect(global.fetch).toHaveBeenCalledTimes(1) +}) + +test('已适配但上游不可用的模型显示真实原因，不进入待适配或未测试筛选', async () => { + const next = configured(), model = next.providers[0].models[1] + model.enabled = false; model.stages = [] + model.verification = verification() + model.capabilities = { kind: 'image', allowedStages: [], source: 'upstream', reason: '上游当前不可用，暂不能测试或开放' } + global.fetch = jest.fn(async () => response(next)) as typeof fetch + render() + const row = await screen.findByLabelText('nano-banana-2 · 现有上游', { selector: 'div' }) + expect(within(row).getByRole('status')).toHaveTextContent('上游当前不可用，暂不能测试或开放') + expect(within(row).queryByText('待适配')).not.toBeInTheDocument() + expect(within(row).queryByRole('button', { name: /^测试 / })).not.toBeInTheDocument() + expect(within(row).getByRole('checkbox', { name: '开放 nano-banana-2 · 现有上游 给用户' })).toBeDisabled() + expect(within(row).getByRole('checkbox', { name: 'nano-banana-2 · 现有上游 用于图片生成' })).toBeDisabled() + for (const value of ['adapting', 'untested']) { + fireEvent.change(screen.getByLabelText('按状态筛选模型'), { target: { value } }) + expect(screen.queryByText('nano-banana-2')).not.toBeInTheDocument() + } + expect(global.fetch).toHaveBeenCalledTimes(1) +}) + +test('上游停用只阻止操作，不掩盖模型的真实失败状态', async () => { + const next = configured(), provider = next.providers[0], model = provider.models[1] + provider.enabled = false; model.enabled = false; model.stages = [] + model.verification.image.status = 'failed' + global.fetch = jest.fn(async () => response(next)) as typeof fetch + render() + const row = await screen.findByLabelText('nano-banana-2 · 现有上游', { selector: 'div' }) + expect(within(row).getByRole('status')).toHaveTextContent('测试失败') + expect(within(row).getByText('请先启用上游连接')).toBeVisible() + expect(within(row).getByRole('button', { name: '测试 nano-banana-2 · 现有上游 的图片生成' })).toBeDisabled() +}) + test('取消最后用途同步关闭开放并清空默认值，保存不会携带失效默认模型', async () => { const next = configured() global.fetch = jest.fn(async () => response(next)) as typeof fetch @@ -451,7 +523,9 @@ test.each(['disabled', 'credential'] as const)('测试无法执行时就近显 const button = await screen.findByRole('button', { name: '测试 new-vision-model · 现有上游 的资料分析' }) expect(button).toBeDisabled() expect(button).toHaveAccessibleDescription(reason === 'disabled' ? '请先启用上游连接' : '请先配置上游密钥') - expect(within(screen.getByLabelText('new-vision-model · 现有上游', { selector: 'div' })).getByRole('status')).toHaveTextContent(reason === 'disabled' ? '请先启用上游连接' : '请先配置上游密钥') + const row = screen.getByLabelText('new-vision-model · 现有上游', { selector: 'div' }) + expect(within(row).getByRole('status')).toHaveTextContent('未测试') + expect(within(row).getByText(reason === 'disabled' ? '请先启用上游连接' : '请先配置上游密钥')).toBeVisible() }) test('有开放设置草稿时查询原异步任务不重新测试且保留草稿', async () => { @@ -500,7 +574,7 @@ test('测试处理中禁止重复请求；未知结果暂停整个模型并可 expect(screen.getByRole('button', { name: '同步 现有上游 的模型目录' })).toBeDisabled() expect((global.fetch as jest.Mock).mock.calls.filter(([url]) => url.endsWith('/verify'))).toHaveLength(1) finish(response(unknown)) - await screen.findByText('结果未知，暂停核查') + await screen.findByText('待核查', { selector: 'p' }) expect(testButton).toBeDisabled() fireEvent.change(screen.getByLabelText('new-vision-model · 现有上游 测试用途'), { target: { value: 'copy' } }) expect(screen.getByRole('button', { name: '测试 new-vision-model · 现有上游 的资料分析' })).toBeDisabled() @@ -552,7 +626,7 @@ test('已有上游任务仅查询结果，无费用弹窗，不发送新验证 completed.providers[0].models[2].verification.analysis = { status: 'passed', id: 'existing-verification', checkedAt: '2026-10-13T01:00:00Z', requestId: 'existing-task', message: '测试通过' } global.fetch = jest.fn(async (url: string) => response(url.endsWith('/refresh') ? completed : next)) as typeof fetch render() - expect(await screen.findByText('等待上游结果，可查询')).toBeInTheDocument() + expect(await screen.findByText('待核查', { selector: 'p' })).toBeInTheDocument() expect(screen.queryByRole('button', { name: '测试 new-vision-model · 现有上游 的资料分析' })).not.toBeInTheDocument() fireEvent.click(screen.getByRole('button', { name: '查询 new-vision-model · 现有上游 的资料分析结果' })) await waitFor(() => expect(screen.getByRole('checkbox', { name: 'new-vision-model · 现有上游 用于资料分析' })).toBeEnabled()) diff --git a/components/admin/AiModelManagementPanel.tsx b/components/admin/AiModelManagementPanel.tsx index 4002dd6..9943ffd 100644 --- a/components/admin/AiModelManagementPanel.tsx +++ b/components/admin/AiModelManagementPanel.tsx @@ -4,7 +4,7 @@ import { useEffect, useMemo, useRef, useState } from 'react' import { Info, Plus, RefreshCw, Save, Search } from 'lucide-react' import { useConfirmActionDialog } from '@/components/ConfirmActionDialog' import { createBrowserUuid } from '@/lib/browser-uuid' -import { AI_MODEL_TEST_PURPOSES, aiModelTestStages, type AiModelAdminState, type AiModelCatalogEntry, type AiModelProvider, type AiModelStage, type AiModelTestPurpose } from '@/lib/ai-model-types' +import { AI_MODEL_TEST_PURPOSES, aiModelTestStages, type AiModelAdminState, type AiModelCatalogEntry, type AiModelProvider, type AiModelStage, type AiModelTestPurpose, type AiModelVerificationSummary } from '@/lib/ai-model-types' const stages: Array<{ id: AiModelStage; label: string; shortLabel: string }> = [ { id: 'analysis', label: '资料分析', shortLabel: '分析' }, { id: 'copy', label: '卖点文案', shortLabel: '文案' }, { id: 'image', label: '图片生成', shortLabel: '生图' }, @@ -20,7 +20,16 @@ const buttonClass = 'inline-flex min-h-11 shrink-0 items-center justify-center g const primaryClass = `${buttonClass} !border-[#0050cb] !bg-[#0050cb] !text-white hover:!bg-[#003da3]` const checkboxClass = 'h-4 w-4 shrink-0 accent-[#0050cb] focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-[#1677ff] focus-visible:ring-offset-2 disabled:cursor-not-allowed' const capabilityLabels = { text: '对话', vision: '视觉理解', image: '生图', tts: '语音合成', asr: '语音识别', music: '音乐', video: '视频', embedding: '向量', 'image-tool': '图像工具', 'three-d': '3D', other: '其他', unknown: '待识别' } as const -const verificationLabels = { untested: '待测试', running: '测试中', passed: '已通过', failed: '未通过', unknown: '暂停核查', stale: '需重新测试' } as const +const verificationLabels = { untested: '未测试', running: '测试中', passed: '已通过', failed: '测试失败', unknown: '待核查', stale: '需重新测试' } as const +function verificationLabel(result: AiModelVerificationSummary) { + return result.reconciled && result.status === 'failed' ? '已核查，可重测' : verificationLabels[result.status] +} +function verificationColor(result: AiModelVerificationSummary) { + return result.status === 'passed' ? 'text-[#087052]' : result.status === 'unknown' ? 'text-[#854a0e]' : result.status === 'failed' && !result.reconciled ? 'text-[#b42318]' : 'text-[#66758b]' +} +function needsReferenceAdaptation(model: AiModelCatalogEntry) { + return model.capabilities.kind === 'image' && !model.capabilities.allowedStages.includes('image') && model.capabilities.reason.startsWith('参考图待适配：') +} interface ProviderForm { id: string | null; revision: number; name: string; baseUrl: string; apiKey: string } interface RecoveryForm { providerId: string; verificationId: string; modelId: string; outcome: 'not_executed' | 'failed' | 'completed'; reference: string; note: string; confirmed: boolean } @@ -215,8 +224,10 @@ export default function AiModelManagementPanel({ onStateChange }: { onStateChang if (statusFilter === 'enabled' && !model.enabled) return false if (statusFilter === 'disabled' && model.enabled) return false if (statusFilter === 'missing' && model.present) return false - if (statusFilter === 'untested' && !model.capabilities.allowedStages.some((stage) => ['untested', 'stale'].includes(model.verification[stage].status))) return false - if (statusFilter === 'attention' && !Object.values(model.verification).some((result) => ['failed', 'unknown', 'running'].includes(result.status))) return false + if (statusFilter === 'adapting' && !needsReferenceAdaptation(model)) return false + if (['untested', 'stale', 'passed'].includes(statusFilter) && !model.capabilities.allowedStages.some((stage) => model.verification[stage].status === statusFilter)) return false + if (['failed', 'unknown', 'running'].includes(statusFilter) && !Object.values(model.verification).some((result) => result.status === statusFilter && !(statusFilter === 'failed' && result.reconciled))) return false + if (statusFilter === 'reconciled' && !Object.values(model.verification).some((result) => result.reconciled && result.status === 'failed')) return false return true }) const providerBlocked = !!busy || dirty || !!form || !!recovery @@ -279,12 +290,14 @@ export default function AiModelManagementPanel({ onStateChange }: { onStateChang })} {!!rows.length && <> -
+

多模态模型一次测试分析与文案，通过的用途可勾选；开放后保存配置。

{!visibleRows.length &&

没有匹配模型

} {pageRows.map(({ provider, model }) => { + const referenceAdaptation = needsReferenceAdaptation(model) + const capabilityReason = referenceAdaptation ? '参考图接入待适配，完成后可测试' : model.capabilities.reason const testableStages = testPurposes(model) const unresolvedStage = stages.find(stage => ['unknown', 'running'].includes(model.verification[stage.id].status)) const unresolvedPurpose = unresolvedStage && (model.verification[unresolvedStage.id].purpose || unresolvedStage.id) @@ -296,7 +309,7 @@ export default function AiModelManagementPanel({ onStateChange }: { onStateChang const protocol = resultStage ? model.capabilities.protocols?.[resultStage] : null const detailsId = `test-details-${model.key}-${selectedStage?.id}` const canInspect = !!protocol || testedStages.some(stage => { const value = model.verification[stage]; return !!(value.message || value.checkedAt || value.requestId) }) - const retest = testedStages.some(stage => { const value = model.verification[stage]; return value.status === 'passed' || value.status === 'stale' || value.reconciled }) + const retest = testedStages.some(stage => { const value = model.verification[stage]; return value.status === 'passed' || value.status === 'failed' || value.status === 'stale' || value.reconciled }) const unsettled = Object.values(model.verification).some((entry) => entry.status === 'running' || entry.status === 'unknown') const testing = busy.startsWith(`verify:${model.key}:`) const attempt = uncertainAttempts.current.get(model.key) @@ -307,16 +320,18 @@ export default function AiModelManagementPanel({ onStateChange }: { onStateChang : !provider.hasApiKey ? '请先配置上游密钥' : !model.present ? '模型已离开上游目录，请同步目录' : unsettled ? '已有测试进行中或结果未知，请先查询状态' - : !selectedStage ? model.capabilities.reason : '' + : !selectedStage ? capabilityReason : '' const testStatusId = `test-status-${model.key}` + const blockedReasonId = `test-blocked-${model.key}` + const showBlockedReason = !!blockedReason && !testing && !attempt && !unsettled return
-

{model.id}

{capabilityLabels[model.capabilities.kind]} · {provider.name}{!provider.enabled ? ' · 上游已停用' : ''}{!model.present ? ' · 已离开上游目录' : ''}

{model.capabilities.reason}
+

{model.id}

{capabilityLabels[model.capabilities.kind]} · {provider.name}{!provider.enabled ? ' · 上游已停用' : ''}{!model.present ? ' · 已离开上游目录' : ''}

{capabilityReason}
{model.id} 的用户可选用途{stages.map((stage) => { const adapted = model.capabilities.allowedStages.includes(stage.id) const supported = adapted && model.verification[stage.id].status === 'passed' const checked = model.stages.includes(stage.id) - const reason = supported ? `${stage.label}测试已通过` : adapted ? `${stage.label}：${verificationLabels[model.verification[stage.id].status]}，通过后可开放` : model.capabilities.allowedStages.length ? `此模型不支持${stage.label}` : model.capabilities.reason + const reason = supported ? `${stage.label}测试已通过` : adapted ? `${stage.label}：${verificationLabel(model.verification[stage.id])}，通过后可开放` : referenceAdaptation && stage.id === 'image' ? capabilityReason : model.capabilities.allowedStages.length || referenceAdaptation ? `当前未接入${stage.label}用途` : capabilityReason return })}
@@ -328,20 +343,22 @@ export default function AiModelManagementPanel({ onStateChange }: { onStateChang {attempt ? : result.canResume && result.id ? : result.canReconcile && result.id ? : } + }}>{busy === `result:${model.key}` ? '查询中…' : '查询结果'} : result.canReconcile && result.id ? : } {canInspect && }
-

{testing ? '正在调用模型，请稍候' : attempt ? '提交结果未确认，请核查' : selectedStage.id === 'analysis-copy' ? testedStages.map(stage => { +

{testing ? '正在调用模型，请稍候' : attempt ? '提交结果未确认，请核查' : selectedStage.id === 'analysis-copy' ? testedStages.map(stage => { const value = model.verification[stage] - return {stages.find(entry => entry.id === stage)!.shortLabel}：{value.reconciled ? '已核查，可重测' : verificationLabels[value.status]} - }) : result.status === 'unknown' ? result.canResume ? '等待上游结果，可查询' : '结果未知，暂停核查' : result.status === 'running' ? '测试进行中，自动更新状态' : result.reconciled ? '已核查，可重测' : blockedReason || verificationLabels[result.status]}

- {selectedStage.id === 'analysis-copy' && blockedReason && !testing && !attempt && !unsettled &&

{blockedReason}

} + return {stages.find(entry => entry.id === stage)!.shortLabel}：{verificationLabel(value)} + }) : result.status === 'running' ? '测试进行中，自动更新状态' : verificationLabel(result)}

+ {showBlockedReason &&

{blockedReason}

} {canInspect && } - {!testing && !attempt && !expandedDetails[detailsId] && ['failed', 'stale', 'unknown'].includes(result.status) && result.message &&

{result.message}

} + : referenceAdaptation ? <> +

待适配

+ :

{blockedReason}

}
})} diff --git a/components/admin/AiModelRecovery.test.tsx b/components/admin/AiModelRecovery.test.tsx index 12ec747..970984c 100644 --- a/components/admin/AiModelRecovery.test.tsx +++ b/components/admin/AiModelRecovery.test.tsx @@ -27,7 +27,7 @@ test('reload restores lost intake, server-confirmed same submission clears it wi const known = snapshot(); known.providers[0].models[0].verification.analysis = { status: 'passed', id: 'verification', checkedAt: null, requestId: 'upstream', message: null, submissionKey: key } global.fetch = jest.fn(async () => response(known)) as typeof fetch render() - await screen.findByText('已通过') + await screen.findByText('已通过', { selector: 'p' }) expect(screen.queryByRole('button', { name: intakeName })).not.toBeInTheDocument() expect(screen.getByRole('checkbox', { name: 'vision-model · 平台模型服务 用于资料分析' })).toBeEnabled() expect((global.fetch as jest.Mock).mock.calls.every(([, init]) => !init?.method)).toBe(true) diff --git a/lib/__tests__/ai-model-capabilities.test.ts b/lib/__tests__/ai-model-capabilities.test.ts index 59e7846..9592bc3 100644 --- a/lib/__tests__/ai-model-capabilities.test.ts +++ b/lib/__tests__/ai-model-capabilities.test.ts @@ -22,16 +22,69 @@ test('image understanding and image output require separate interfaces', () => { expect(getAiModelCapabilities(generic, 'editor', metadata(['text', 'image'], ['image'], ['/images/edits']))).toMatchObject({ kind: 'image', allowedStages: ['image'] }) }) -test('image models explain missing reference input and unavailable upstream separately', () => { - expect(getAiModelCapabilities(generic, 'text-image', metadata(['text'], ['image'], ['/images/generations']))).toMatchObject({ allowedStages: [], reason: '仅支持文生图，不能接收产品参考图' }) - expect(getAiModelCapabilities(generic, 'reference-image', metadata(['text', 'image'], ['image'], ['/images/generations']))).toMatchObject({ allowedStages: [], reason: '上游尚未提供已适配的参考图接口' }) +test('image models explain pending adaptation without treating missing metadata as proof of no reference support', () => { + expect(getAiModelCapabilities(generic, 'text-image', metadata(['text'], ['image'], ['/images/generations']))).toMatchObject({ allowedStages: [], reason: '参考图待适配：当前目录尚未确认可用的参考图传入方式' }) + expect(getAiModelCapabilities(generic, 'reference-image', metadata(['text', 'image'], ['image'], ['/images/generations']))).toMatchObject({ allowedStages: [], reason: '参考图待适配：目录已声明图片输入，当前传入方式尚未接入' }) const undeclaredReference = metadata(['text'], ['image'], ['/images/generations']) undeclaredReference.inputDeclared = false undeclaredReference.protocols = [{ id: 'openai-image-generate', path: '/v1/images/generations', headers: {}, supportsImageInput: false }] - expect(getAiModelCapabilities(generic, 'undeclared-reference', undeclaredReference)).toMatchObject({ allowedStages: [], reason: '上游未提供可用的参考图参数，暂不能接收产品参考图' }) + expect(getAiModelCapabilities(generic, 'undeclared-reference', undeclaredReference)).toMatchObject({ allowedStages: [], reason: '参考图待适配：当前目录尚未确认可用的参考图传入方式' }) const unavailable = { ...metadata(['text', 'image'], ['image'], ['/images/edits']), unavailable: true } expect(getAiModelCapabilities(generic, 'unavailable', unavailable)).toMatchObject({ allowedStages: [], reason: '上游当前不可用，暂不能测试或开放' }) }) + +test('selected reference route declarations override the default offer summary without claiming a real test passed', () => { + const referenceRoute = { id: 'reference-route', input_modes: ['text', 'image_url'], output_modes: ['image'], invocation: { + ...invocation('/v1/images/edits', { 'X-Relay-Route-Option': 'reference-route' }), + input_schema: { properties: { 'image[]': { type: 'array' } } }, + } } + const entry = model({ type: 'image', modality: 'image', capabilities: ['text_to_image', 'image_to_image'], input_modes: ['text'], output_modes: ['text'], service_offers: [ + { is_default: true, service_scope: { id: 'scope-default' }, input_modes: ['text'], output_modes: ['text'], route_options: [ + { id: 'default-route', invocation: { ...invocation('/v1/images/generations', { 'X-Relay-Route-Option': 'default-route' }), content_type: 'application/json' } }, + referenceRoute, + ] }, + ] }) + const evidence = publicMetadata(entry, 'tool') + expect(evidence).toMatchObject({ inputModalities: ['image', 'text'], outputModalities: ['image'], inputDeclared: true, outputDeclared: true }) + expect(getAiModelCapabilities(base, entry.id, evidence)).toMatchObject({ kind: 'image', allowedStages: ['image'], reason: '接口已适配；通过对应用途的实际测试后可开放', protocols: { image: { routeId: 'reference-route' } } }) +}) + +test('a reference adapter remains testable when the directory summary only lists text input', () => { + const entry = model({ type: 'image', modality: 'image', capabilities: ['image_to_image'], input_modes: ['text'], invocation: { + ...invocation('/v1/images/edits'), input_schema: { properties: { image: { type: 'string', format: 'binary' } } }, + } }) + const evidence = publicMetadata(entry, 'tool') + expect(evidence).toMatchObject({ inputModalities: ['image', 'text'], outputModalities: ['image'] }) + expect(getAiModelCapabilities(base, entry.id, evidence)).toMatchObject({ kind: 'image', allowedStages: ['image'] }) +}) + +test('generic URL input does not become image input until a concrete reference adapter is available', () => { + const entry = model({ type: 'image', modality: 'image', capabilities: ['text_to_image', 'image_to_image'], input_modes: ['text', 'url'], invocation: { + ...invocation('/v1/images/generations'), content_type: 'application/json', + } }) + const evidence = publicMetadata(entry, 'tool') + expect(evidence.inputModalities).toEqual(['text', 'url']) + expect(getAiModelCapabilities(base, entry.id, evidence)).toMatchObject({ allowedStages: [], reason: '参考图待适配：当前目录尚未确认可用的参考图传入方式' }) +}) + +test('an explicit non-image output declaration keeps an image route closed with a precise adaptation reason', () => { + const evidence = { ...metadata(['text', 'image'], ['text'], ['/images/edits']), kindHint: 'image' as const } + expect(getAiModelCapabilities(base, 'any-image-model', evidence)).toMatchObject({ allowedStages: [], reason: '参考图待适配：当前路线的输出声明与生图要求不一致' }) +}) +test('a standard reference probe is testable without claiming a declared or already verified reference contract', () => { + const evidence = { ...metadata(['text'], ['image'], ['/images/generations']), protocols: [{ + id: 'openai-image-generate', path: '/v1/images/generations', headers: { 'X-Relay-Route-Option': 'candidate-route' }, supportsImageInput: true, + inputField: 'image', referenceTransport: 'json-inline', referenceContract: 'probe-required', + requestParameters: { n: 1, response_format: 'b64_json' }, maxImages: 1, scopeId: 'candidate-scope', routeId: 'candidate-route', + }] } + expect(getAiModelCapabilities(base, 'standard-reference-candidate', evidence)).toMatchObject({ + kind: 'image', allowedStages: ['image'], reason: '参考图采用标准图片参数；通过实际测试后可开放', + }) + expect(getAiModelCapabilities(base, 'unavailable-candidate', { ...evidence, unavailable: true })).toMatchObject({ + allowedStages: [], reason: '上游当前不可用，暂不能测试或开放', + }) + expect(aiModelCapabilityFingerprint(evidence)).not.toBe(aiModelCapabilityFingerprint({ ...evidence, protocols: evidence.protocols.map(({ referenceContract: _candidate, ...protocol }) => protocol) })) +}) test('audio transcription is not a copy-writing model just because it returns text over chat', () => { expect(getAiModelCapabilities(generic, 'asr', metadata(['audio'], ['text'], ['/chat/completions']))).toMatchObject({ kind: 'asr', allowedStages: [] }) }) diff --git a/lib/__tests__/ai-model-protocol.test.ts b/lib/__tests__/ai-model-protocol.test.ts index ea077cd..9535316 100644 --- a/lib/__tests__/ai-model-protocol.test.ts +++ b/lib/__tests__/ai-model-protocol.test.ts @@ -71,6 +71,230 @@ test('working quality survives storage and reaches the selected image route with expect(validateStoredAiModelProtocols([{ ...chat, imageQuality: 'medium' }])).toEqual([]) }) +const fileRouteModel = (schema: any = {}, example: any = {}) => ({ + type: 'image', modality: 'image', capabilities: ['text_to_image', 'openai.images.generations'], + default_service_scope_id: 'scope-one', default_route_option_id: 'text-default', + service_offers: [{ service_scope: { id: 'scope-one' }, public_operation_id: 'openai.images.generations.create', route_options: [ + { id: 'text-default', input_modes: ['text'], invocation: invocation('/v1/images/generations', 'text-default') }, + { id: 'file-route', input_modes: ['file', 'image_url', 'text'], invocation: { + ...invocation('/v1/images/generations', 'file-route', { ...schema, properties: { + image: { type: 'array', items: { type: 'string', pattern: '^https://' }, anyOf: [{ type: 'string' }, { type: 'array', maxItems: 4 }] }, + n: { const: 1 }, response_format: { enum: ['url', 'b64_json'] }, ...schema.properties, + } }), content_type: 'multipart/form-data', request_example: example, + } }, + ] }], +}) + +test('chooses explicit file reference transport in the default scope and preserves a fixed 4K request', () => { + const model = fileRouteModel({ properties: { size: { enum: ['4096x4096'] }, stream: { enum: [false] } } }, { size: '4096x4096', stream: false }) + model.service_offers.unshift({ service_scope: { id: 'other-scope' }, public_operation_id: 'openai.images.generations.create', route_options: [ + { id: 'wrong-scope', input_modes: ['file', 'image_url'], invocation: invocation('/v1/images/generations', 'wrong-scope', { properties: { image: { type: 'array' } } }) } as any, + ] }) + const parsed = parseAiModelProtocols(model) + expect(parsed).toEqual([{ + id: 'openai-image-generate', path: '/v1/images/generations', headers: { 'X-Relay-Route-Option': 'file-route' }, + supportsImageInput: true, inputField: 'image', maxImages: 4, scopeId: 'scope-one', routeId: 'file-route', operationId: 'openai.images.generations.create', + referenceTransport: 'multipart', requestParameters: { n: 1, response_format: 'b64_json', size: '4096x4096', stream: false }, + }]) + expect(validateStoredAiModelProtocols(JSON.parse(JSON.stringify(parsed)))).toEqual(parsed) +}) + +test('published JSON reference route is preferred without changing service scope', () => { + const model = fileRouteModel() + Object.assign(model.service_offers[0].route_options[0], { input_modes: ['image_url', 'text'], invocation: invocation('/v1/images/generations', 'text-default', { properties: { image: { type: 'array', items: { type: 'string' } } } }) }) + model.capabilities.push('image_to_image') + expect(parseAiModelProtocols(model)[0]).toMatchObject({ routeId: 'text-default', referenceTransport: 'json-inline' }) +}) + +test('file mode does not override the JSON invocation contract or require a combined image_to_image tag', async () => { + const model = fileRouteModel({ properties: { + image: { type: 'array', items: { type: 'string', format: 'uri' }, maxItems: 2 }, + size: { const: '4096x4096' }, version: { const: 'flare' }, + } }) + model.service_offers[0].route_options[1].invocation.content_type = 'application/json' + const protocol = parseAiModelProtocols(model)[0] + expect(protocol).toMatchObject({ referenceTransport: 'json-inline', inputField: 'image', maxImages: 2, requestParameters: { size: '4096x4096', version: 'flare' }, routeId: 'file-route', scopeId: 'scope-one' }) + expect(validateStoredAiModelProtocols(JSON.parse(JSON.stringify([protocol])))).toEqual([protocol]) + send.mockResolvedValue(Response.json({ data: [{ b64_json: png.toString('base64') }] })) + await callAiModelProtocol({ ...base, protocol, images: [{ buffer: png, mimeType: 'image/png' }] }) + expect(send).toHaveBeenCalledTimes(1) + expect(typeof send.mock.calls[0][3]).toBe('string') + expect(JSON.parse(send.mock.calls[0][3])).toMatchObject({ image: [expect.stringMatching(/^data:image\/png;base64,/)], size: '4096x4096', version: 'flare', n: 1, response_format: 'b64_json' }) + expect(send.mock.calls[0][5]).toEqual({ 'X-Relay-Route-Option': 'file-route' }) +}) + +test('JSON-only routes reject HTTPS-only or conflicting reference item constraints before any invocation', () => { + for (const schema of [ + { properties: { image: { type: 'array', items: { type: 'string', pattern: '^https://' } } } }, + { properties: { image: { type: 'array', items: { type: 'string' } } }, allOf: [{ properties: { image: { type: 'array', items: { type: 'string', pattern: '^https://' } } } }] }, + { properties: { image: { type: 'array', items: { type: 'string', format: 'url' } } } }, + ]) { + const model = fileRouteModel(schema) + model.service_offers[0].route_options[1].invocation.content_type = 'application/json' + expect(parseAiModelProtocols(model)).toMatchObject([{ routeId: 'text-default', supportsImageInput: false }]) + } +}) + +test('JSON file_urls without a declared image input mode does not become a reference adapter', () => { + const model = fileRouteModel({ properties: { image: false, file_urls: { type: 'array', items: { type: 'string', pattern: '^(https://|data:image/)' } } } }) + const route = model.service_offers[0].route_options[1] + route.invocation.content_type = 'application/json' + route.input_modes = ['file', 'text'] + expect(parseAiModelProtocols(model)).toMatchObject([{ routeId: 'text-default', supportsImageInput: false }]) +}) + +test('a JSON reference rejection does not retry with multipart or another route', async () => { + const model = fileRouteModel({ properties: { image: { type: 'array', items: { type: 'string' } } } }) + model.service_offers[0].route_options[1].invocation.content_type = 'application/json' + send.mockResolvedValue(Response.json({ error: { code: 'invalid_request' } }, { status: 400, headers: { 'x-request-id': 'json-rejected' } })) + await expect(callAiModelProtocol({ ...base, protocol: parseAiModelProtocols(model)[0], images: [{ buffer: png, mimeType: 'image/png' }] })).rejects.toMatchObject({ outcomeUnknown: false, requestId: 'json-rejected' }) + expect(send).toHaveBeenCalledTimes(1) + expect(typeof send.mock.calls[0][3]).toBe('string') + expect(send.mock.calls[0][5]).toEqual({ 'X-Relay-Route-Option': 'file-route' }) +}) + +test('multipart generations sends binary images and the exact declared size, not private data URIs', async () => { + const protocol = parseAiModelProtocols(fileRouteModel({ properties: { size: { enum: ['4096x4096'] }, version: { const: 'sunburst' }, response_format: { const: 'url' }, background: { enum: ['opaque'] } } }, { background: 'opaque' }))[0] + const generated = await sharp({ create: { width: 4096, height: 4096, channels: 3, background: '#00ff00' } }).png().toBuffer() + send.mockResolvedValue(Response.json({ data: [{ b64_json: generated.toString('base64') }] })) + const result = await callAiModelProtocol({ ...base, protocol, images: [{ buffer: png, mimeType: 'image/png' }, { buffer: png, mimeType: 'image/png' }] }) + expect(send).toHaveBeenCalledTimes(1) + const body = send.mock.calls[0][3] as FormData + expect(body).toBeInstanceOf(FormData) + expect(body.get('size')).toBe('4096x4096') + expect(body.get('version')).toBe('sunburst') + expect(body.get('response_format')).toBe('url') + expect(body.get('background')).toBe('opaque') + expect(body.get('n')).toBe('1') + expect(body.has('file_urls')).toBe(false) + expect(body.getAll('image')).toHaveLength(2) + for (const file of body.getAll('image')) { + expect(file).toBeInstanceOf(Blob) + expect((file as File).type).toBe('image/png') + expect((await sharp(Buffer.from(await (file as File).arrayBuffer())).metadata()).width).toBe(32) + } + expect(send.mock.calls[0][5]).toEqual({ 'X-Relay-Route-Option': 'file-route' }) + expect(result.summary).toMatchObject({ referenceTransport: 'multipart', requestParameters: { size: '4096x4096' }, generatedWidth: 4096, generatedHeight: 4096, storedWidth: 1440, storedHeight: 1440 }) +}) + +test('selects the smallest declared generation size and honors allOf fixed values', () => { + const protocol = parseAiModelProtocols(fileRouteModel({ + allOf: [{ properties: { version: { const: 'flare' }, quality: { enum: ['auto'] } } }], + properties: { size: { enum: ['4096x4096', '2048x2048', '1024x1024'] }, version: { type: 'string' }, quality: { enum: ['auto', 'high'] } }, + }, { size: '4096x4096', version: 'sunburst', quality: 'high' }))[0] + expect(protocol.requestParameters).toMatchObject({ size: '1024x1024', version: 'flare', quality: 'auto' }) +}) + +test.each([ + { properties: { n: { const: 2 } } }, + { properties: { n: false } }, + { properties: { n: { type: 'integer', not: { const: 1 } } } }, + { properties: { n: { oneOf: [{ const: 1 }, { type: 'integer' }] } } }, + { not: { required: ['image'] } }, + { oneOf: [{ required: ['image'] }, { required: ['model'] }] }, + { properties: { size: { enum: ['8192x8192'] } } }, + { required: ['callback_url'], properties: { callback_url: { type: 'string' } } }, + { allOf: [{ properties: { version: { const: 'a' } } }], properties: { version: { const: 'b' } } }, +])('does not offer a reference route whose mandatory request constraints cannot be satisfied: %j', schema => { + expect(parseAiModelProtocols(fileRouteModel(schema))).toMatchObject([{ routeId: 'text-default', supportsImageInput: false }]) +}) + +test.each([ + { type: 'array', items: { type: 'string', not: { pattern: '^data:' } } }, + { type: 'array', maxItems: 0, items: { type: 'string' } }, + { type: 'array', items: { type: 'string' }, not: { minItems: 1 } }, +])('explicit negative array and item constraints never admit inline references: %j', image => { + const model = fileRouteModel({ properties: { image } }) + model.service_offers[0].route_options[1].invocation.content_type = 'application/json' + expect(parseAiModelProtocols(model)).toMatchObject([{ routeId: 'text-default', supportsImageInput: false }]) +}) + +test('file mode alone and undeclared reference fields remain awaiting adaptation', () => { + const model = fileRouteModel({ properties: { image: false } }) + expect(parseAiModelProtocols(model)).toMatchObject([{ routeId: 'text-default', supportsImageInput: false }]) + model.service_offers[0].route_options[1].input_modes = ['image_url', 'text'] + expect(parseAiModelProtocols(model)).toMatchObject([{ supportsImageInput: false }]) +}) + +test('stored multipart request validation rejects arbitrary fields, type changes and impossible transport combinations', () => { + const protocol = parseAiModelProtocols(fileRouteModel())[0] + for (const requestParameters of [{ ...protocol.requestParameters, callback_url: 'https://example.com' }, { ...protocol.requestParameters, n: 2 }, { ...protocol.requestParameters, size: '8192x8192' }, { ...protocol.requestParameters, response_format: true }]) { + expect(validateStoredAiModelProtocols([{ ...protocol, requestParameters }])).toEqual([]) + } + expect(validateStoredAiModelProtocols([{ ...protocol, referenceTransport: 'json' }])).toEqual([]) + expect(validateStoredAiModelProtocols([{ ...protocol, inputField: 'file_urls' }])).toEqual([]) + expect(validateStoredAiModelProtocols([{ ...chat, referenceTransport: 'multipart', requestParameters: protocol.requestParameters }])).toEqual([]) + expect(validateStoredAiModelProtocols([{ ...protocol, requestParameters: undefined }])).toEqual([]) +}) + +const probeModel = (schema: any = { additionalProperties: true, properties: {} }, supported_fields: any[] = []) => ({ + type: 'image', modality: 'image', capabilities: ['openai.images.generations', 'text_to_image'], + default_service_scope_id: 'scope-probe', default_route_option_id: 'route-probe', + service_offers: [{ service_scope: { id: 'scope-probe' }, public_operation_id: 'openai.images.generations.create', route_options: [{ + id: 'route-probe', input_modes: ['text'], invocation: { ...invocation('/v1/images/generations', 'route-probe', schema), supported_fields }, + }] }], +}) + +test('an explicitly open standard generations schema permits one bounded reference test with a persistent candidate marker', async () => { + const protocol = parseAiModelProtocols(probeModel())[0] + expect(protocol).toMatchObject({ id: 'openai-image-generate', supportsImageInput: true, inputField: 'image', referenceTransport: 'json-inline', referenceContract: 'probe-required', maxImages: 1, routeId: 'route-probe', scopeId: 'scope-probe', requestParameters: { n: 1, response_format: 'b64_json' } }) + expect(validateStoredAiModelProtocols(JSON.parse(JSON.stringify([protocol])))).toEqual([protocol]) + send.mockResolvedValue(Response.json({ data: [{ b64_json: png.toString('base64') }] })) + const result = await callAiModelProtocol({ ...base, protocol, images: [{ buffer: png, mimeType: 'image/png' }] }) + expect(send).toHaveBeenCalledTimes(1) + expect(JSON.parse(send.mock.calls[0][3])).toMatchObject({ image: [expect.stringMatching(/^data:image\/png;base64,/)] }) + expect(result.summary).toMatchObject({ referenceContract: 'probe-required', referenceTransport: 'json-inline', routeId: 'route-probe' }) +}) + +test.each([ + {}, + { additionalProperties: false }, + { additionalProperties: true, properties: { image: false } }, + { additionalProperties: true, properties: { image: { type: 'array', items: { type: 'string', pattern: '^https://' } } } }, + { additionalProperties: true, properties: { file_urls: { type: 'array', items: { type: 'string' } } } }, + { additionalProperties: true, allOf: [{ additionalProperties: false }] }, + { additionalProperties: true, allOf: [{ properties: { image: false } }] }, + { additionalProperties: true, patternProperties: { '^image': false } }, + { additionalProperties: true, propertyNames: { enum: ['model', 'prompt'] } }, +])('a candidate never bypasses an unknown or negative reference constraint: %j', schema => { + expect(parseAiModelProtocols(probeModel(schema))).toMatchObject([{ supportsImageInput: false }]) +}) + +test('whole-request negations are rejected before declared or candidate reference mapping', () => { + expect(parseAiModelProtocols(probeModel({ additionalProperties: true, not: { required: ['image'] } }))).toEqual([]) +}) + +test('an explicitly disabled reference field and a nondefault route do not become probe candidates', () => { + expect(parseAiModelProtocols(probeModel(undefined, [{ name: 'image', state: 'offer_disabled' }]))).toMatchObject([{ supportsImageInput: false }]) + const model = probeModel() + model.default_route_option_id = 'not-the-offered-route' + expect(parseAiModelProtocols(model)).toMatchObject([{ supportsImageInput: false }]) + model.default_service_scope_id = 'different-scope' + expect(parseAiModelProtocols(model)).toEqual([]) +}) + +test('deeply nested unknown constraints fail closed rather than disappearing from probe evidence', () => { + let schema: any = { properties: { image: false } } + for (let index = 0; index < 7; index++) schema = { allOf: [schema] } + expect(parseAiModelProtocols(probeModel({ additionalProperties: true, allOf: [schema] }))).toEqual([]) +}) + +test('a declared reference route takes precedence over an open-schema default candidate', () => { + const model = fileRouteModel() + model.service_offers[0].route_options[0].invocation.input_schema = { additionalProperties: true, properties: {} } + expect(parseAiModelProtocols(model)).toMatchObject([{ routeId: 'file-route', referenceTransport: 'multipart' }]) + expect(parseAiModelProtocols(model)[0].referenceContract).toBeUndefined() +}) + +test('candidate storage cannot switch transport, endpoint, count or identity, and multi-reference probes never reach the API', async () => { + const protocol = parseAiModelProtocols(probeModel())[0] + for (const mutation of [{ referenceContract: 'verified' }, { referenceTransport: 'multipart' }, { maxImages: 8 }, { scopeId: undefined }, { routeId: undefined }, { inputField: 'file_urls' }, { id: 'openai-chat', path: '/v1/chat/completions' }]) { + expect(validateStoredAiModelProtocols([{ ...protocol, ...mutation }])).toEqual([]) + } + await expect(callAiModelProtocol({ ...base, protocol, images: [{ buffer: png, mimeType: 'image/png' }, { buffer: png, mimeType: 'image/png' }] })).rejects.toMatchObject({ outcomeUnknown: false, message: '当前模型最多可接收 1 张参考图，请减少图片' }) + expect(send).not.toHaveBeenCalled() +}) + test.each(['audio', 'video'])('does not turn %s models into chat or detail generation adapters', modality => { expect(parseAiModelProtocols({ type: modality, modality, invocation: { method: 'POST', path: '/v1/chat/completions', content_type: 'application/json', headers: {} } })).toEqual([]) }) @@ -200,7 +424,7 @@ test('unreadable local image is a definite preflight failure with no paid reques test('rejects async submission without checkpoint and preflights image and prompt limits', async () => { await expect(callAiModelProtocol({ ...base, protocol: asyncImage })).rejects.toMatchObject({ outcomeUnknown: false }) - await expect(callAiModelProtocol({ ...base, protocol: { ...chat, supportsImageInput: false }, images: [{ buffer: png, mimeType: 'image/png' }] })).rejects.toMatchObject({ outcomeUnknown: false }) + await expect(callAiModelProtocol({ ...base, protocol: { ...chat, supportsImageInput: false }, images: [{ buffer: png, mimeType: 'image/png' }] })).rejects.toMatchObject({ outcomeUnknown: false, message: '参考图传入方式尚未适配，请重新同步模型目录' }) await expect(callAiModelProtocol({ ...base, protocol: { ...chat, maxPromptLength: 1 } })).rejects.toMatchObject({ outcomeUnknown: false }) expect(send).not.toHaveBeenCalled() }) diff --git a/lib/__tests__/ai-model-service.test.ts b/lib/__tests__/ai-model-service.test.ts index 4fdad3a..fb86be9 100644 --- a/lib/__tests__/ai-model-service.test.ts +++ b/lib/__tests__/ai-model-service.test.ts @@ -215,6 +215,73 @@ test('stale enabled rows are filtered from menus, defaults and old project execu await expect(configureAiModels('admin', { revision: 1, providers: [], defaults: { analysis: null, copy: null, image: legacyModelKey } })).rejects.toMatchObject({ code: 'default_unavailable' }) expect(requestAiModel).not.toHaveBeenCalled() }) +function referenceCandidateMetadata() { + return { ...parseAiModelCapabilityMetadata({ input_modalities: ['text'], output_modalities: ['image'], endpoints: ['/images/generations'] })!, + protocols: [{ id: 'openai-image-generate', path: '/v1/images/generations', headers: { 'X-Relay-Route-Option': 'candidate-route' }, + supportsImageInput: true, inputField: 'image', referenceTransport: 'json-inline', referenceContract: 'probe-required', + maxImages: 1, scopeId: 'candidate-scope', routeId: 'candidate-route', requestParameters: { n: 1, response_format: 'b64_json' } }], + } +} + +test.each(['untested', 'old-fingerprint'] as const)('a standard reference candidate stays unavailable and cannot be enabled with %s evidence', async state => { + const previous = await listAiModelVerifications() + const metadata = referenceCandidateMetadata() + models[0].capabilityMetadata = metadata; models[0].stages = ['image'] + const priorMetadata = { ...metadata, protocols: metadata.protocols.map(({ referenceContract: _probe, ...protocol }) => protocol) } + const oldContext = aiModelVerificationContext(providers[0].baseUrl, models[0].id, priorMetadata, 'synthetic-legacy-secret', getAiModelStageProtocol(priorMetadata, 'image')!) + const currentContext = aiModelVerificationContext(providers[0].baseUrl, models[0].id, metadata, 'synthetic-legacy-secret', getAiModelStageProtocol(metadata, 'image')!) + expect(oldContext).not.toBe(currentContext) + const oldPassed = { ...previous.find(value => value.modelKey === legacyModelKey && value.stage === 'image')!, contextHash: oldContext } + ;(listAiModelVerifications as jest.Mock).mockResolvedValue([ + ...previous.filter(value => value.modelKey !== legacyModelKey), ...(state === 'old-fingerprint' ? [oldPassed] : []), + ]) + const admin = (await getAiModelAdminState()).providers[0].models[0] + expect(admin.capabilities.allowedStages).toEqual(['image']) // Testing admission is separate from runtime permission. + expect(admin.verification.image.status).toBe(state === 'untested' ? 'untested' : 'stale') + const runtime = await getAiModelRuntimeState() + expect(runtime.models.image).not.toContain(legacyModelKey) + expect(runtime.modelOptions.image.some(option => option.value === legacyModelKey)).toBe(false) + expect(runtime.defaults.image).toBe('') + await expect(resolveAiModel('image', legacyModelKey)).rejects.toMatchObject({ code: 'model_unavailable' }) + await expect(resolveAiModel('image', models[0].id)).rejects.toMatchObject({ code: 'model_unavailable' }) + models[0].enabled = false; models[0].stages = [] + await expect(configureAiModels('admin', { revision: 1, defaults: { analysis: null, copy: null, image: null }, + providers: [{ id: legacyId, revision: 1, models: [{ key: legacyModelKey, enabled: true, stages: ['image'] }] }], + })).rejects.toMatchObject({ code: 'model_verification_required' }) + expect(mockDb.$executeRawUnsafe).not.toHaveBeenCalled() + expect(saveAiModelDefaults).not.toHaveBeenCalled() + expect(requestAiModel).not.toHaveBeenCalled() +}) + +test('only current passed reference-candidate evidence permits explicit enablement and image selection', async () => { + const previous = await listAiModelVerifications() + const metadata = referenceCandidateMetadata() + models[0].capabilityMetadata = metadata; models[0].enabled = false; models[0].stages = [] + const contextHash = aiModelVerificationContext(providers[0].baseUrl, models[0].id, metadata, 'synthetic-legacy-secret', getAiModelStageProtocol(metadata, 'image')!) + const passed = { ...previous.find(value => value.modelKey === legacyModelKey && value.stage === 'image')!, id: 'candidate-current-pass', contextHash, + evidence: { ruleVersion: 2, referenceImageAccepted: true, referenceLayoutMatched: true, imageDecoded: true }, + } + ;(listAiModelVerifications as jest.Mock).mockResolvedValue([...previous.filter(value => value.modelKey !== legacyModelKey), passed]) + expect((await getAiModelAdminState()).providers[0].models[0].verification.image.status).toBe('passed') + expect((await getAiModelRuntimeState()).models.image).not.toContain(legacyModelKey) + await expect(resolveAiModel('image', legacyModelKey)).rejects.toMatchObject({ code: 'model_unavailable' }) + await configureAiModels('admin', { revision: 1, defaults: { analysis: null, copy: null, image: null }, + providers: [{ id: legacyId, revision: 1, models: [{ key: legacyModelKey, enabled: true, stages: ['image'] }] }], + }) + expect(mockDb.$executeRawUnsafe).toHaveBeenCalledWith(expect.stringContaining('UPDATE ai_model_catalog SET enabled=$2'), legacyModelKey, true, ['image']) + // Reflect the asserted database update in this repository mock for the subsequent read. + models[0].enabled = true; models[0].stages = ['image'] + const runtime = await getAiModelRuntimeState() + expect(runtime.models.image).toContain(legacyModelKey) + expect(runtime.modelOptions.image).toContainEqual(expect.objectContaining({ value: legacyModelKey })) + expect(runtime.defaults.image).toBe(legacyModelKey) + await expect(resolveAiModel('image', legacyModelKey)).resolves.toMatchObject({ + protocol: { referenceContract: 'probe-required', referenceTransport: 'json-inline' }, + binding: { modelKey: legacyModelKey, verificationId: passed.id }, + }) + expect(requestAiModel).not.toHaveBeenCalled() +}) + test('admin read removes invalid old stages without writes or raw capability metadata exposure', async () => { models[0].capabilityMetadata = null const state = await getAiModelAdminState() diff --git a/lib/__tests__/ai-model-verification.test.ts b/lib/__tests__/ai-model-verification.test.ts index b958a39..86b0c21 100644 --- a/lib/__tests__/ai-model-verification.test.ts +++ b/lib/__tests__/ai-model-verification.test.ts @@ -2,7 +2,7 @@ import { randomUUID } from 'node:crypto' import { AiModelError, assertAiModelAdmin } from '@/lib/ai-model-access' import { verifyAiModel, refreshAiModelVerification, validateAiModelVerificationOutput } from '@/lib/ai-model-verification' -import { validateAiModelCombinedVerificationOutput } from '@/lib/ai-model-verification-output' +import { validateAiModelCombinedVerificationOutput, verificationReferenceImage } from '@/lib/ai-model-verification-output' import { callAiModelProtocol, AiModelProtocolError } from '@/lib/ai-model-protocol' import { resolveAiModelVerification } from '@/lib/ai-model-service' import { prisma } from '@/lib/prisma' @@ -51,6 +51,83 @@ test('a real image-understanding response persists passed evidence without enabl expect(mockDb.$executeRawUnsafe.mock.calls.some(([sql]) => /UPDATE ai_model_catalog|ai_model_configuration/.test(sql))).toBe(false) expect(detailTransactionLock).toHaveBeenCalledWith(mockDb, 'detail-generator-execution') }) +test('reference-image acceptance requests an unchanged flat redraw without leaking the random color answers', async () => { + ;(callAiModelProtocol as jest.Mock).mockImplementation(async request => { + await request.beforeRequest() + return { image: await verificationReferenceImage(row.evidence.challenge), requestId: 'flat-reference-check', summary: {} } + }) + await verifyAiModel('admin', providerId, modelKey, { ...input(), stage: 'image' }) + const request = (callAiModelProtocol as jest.Mock).mock.calls[0][0] + expect(request.prompt).toBe('将参考图重新绘制为干净的平面图，严格保留两个方块各自的原始颜色、形状和相对位置，纯白背景；不要添加文字、标志、阴影、纹理、渐变、光照或其它内容。') + expect(request.prompt).not.toMatch(/red|blue|green|yellow|红色|蓝色|绿色|黄色/i) + expect(request.prompt).not.toContain(row.evidence.challenge.topLeft) + expect(request.prompt).not.toContain(row.evidence.challenge.bottomRight) + expect(request.images).toHaveLength(1) + expect(request.images[0].buffer).toEqual(await verificationReferenceImage(row.evidence.challenge)) + expect(callAiModelProtocol).toHaveBeenCalledTimes(1) + expect(mockDb.$executeRawUnsafe.mock.calls.some(([sql]) => sql.includes("status='passed'"))).toBe(true) +}) +test.each(['multipart', 'json-inline'])('image acceptance preserves the bounded %s request and original versus stored dimensions', async referenceTransport => { + const parameters = { n: 1, response_format: 'b64_json', size: '4096x4096', quality: 'high' } + const context = await (resolveAiModelVerification as jest.Mock)() + ;(resolveAiModelVerification as jest.Mock).mockResolvedValue({ ...context, protocol: { id: 'openai-image-generate', path: '/v1/images/generations', headers: {}, supportsImageInput: true, inputField: 'image', referenceTransport, requestParameters: parameters } }) + ;(callAiModelProtocol as jest.Mock).mockImplementation(async request => { + await request.beforeRequest() + const image = await sharp(await verificationReferenceImage(row.evidence.challenge)).resize(1440, 1440).png().toBuffer() + return { image, requestId: 'actual-4k-request', summary: { referenceTransport, requestParameters: parameters, + generatedWidth: 4096, generatedHeight: 4096, storedWidth: 1440, storedHeight: 1440, + referenceImageAccepted: false, referenceLayoutMatched: false, imageDecoded: false, width: 9999, + ruleVersion: -1, adapterVersion: -1, prompt: 'must not persist', secret: 'synthetic-test-secret' } } + }) + await verifyAiModel('admin', providerId, modelKey, { ...input(), stage: 'image' }) + const passed = mockDb.$executeRawUnsafe.mock.calls.find(([sql]) => sql.includes("status='passed'"))! + expect(passed[2]).toBe('actual-4k-request') + const evidence = JSON.parse(passed[3]) + expect(evidence).toMatchObject({ referenceTransport, requestParameters: parameters, + generatedWidth: 4096, generatedHeight: 4096, storedWidth: 1440, storedHeight: 1440, width: 1440, height: 1440, + referenceImageAccepted: true, referenceLayoutMatched: true, imageDecoded: true }) + expect(evidence.ruleVersion).toBeGreaterThan(0); expect(evidence.adapterVersion).toBeGreaterThan(0) + expect(evidence).not.toHaveProperty('prompt'); expect(evidence).not.toHaveProperty('secret') + expect(passed[3]).not.toContain('synthetic-test-secret') + expect((aiModelAudit as jest.Mock).mock.calls.find(([, , action]) => action === 'model.verification-passed')?.[4].evidence).toMatchObject({ generatedWidth: 4096, storedWidth: 1440 }) +}) +test('an unexpected reference transport cannot replace the locked native route evidence', async () => { + const context = await (resolveAiModelVerification as jest.Mock)() + const parameters = { n: 1, response_format: 'b64_json' } + ;(resolveAiModelVerification as jest.Mock).mockResolvedValue({ ...context, protocol: { id: 'openai-image-generate', path: '/v1/images/generations', headers: {}, supportsImageInput: true, inputField: 'image', referenceTransport: 'json-inline', requestParameters: parameters } }) + ;(callAiModelProtocol as jest.Mock).mockImplementation(async request => { + await request.beforeRequest() + return { image: await verificationReferenceImage(row.evidence.challenge), requestId: 'safe-result', summary: { referenceTransport: 'multipart', requestParameters: parameters } } + }) + await verifyAiModel('admin', providerId, modelKey, { ...input(), stage: 'image' }) + const evidence = JSON.parse(mockDb.$executeRawUnsafe.mock.calls.find(([sql]) => sql.includes("status='passed'"))![3]) + expect(evidence.referenceLayoutMatched).toBe(true) + expect(evidence).not.toHaveProperty('referenceTransport') + expect(evidence).not.toHaveProperty('requestParameters') +}) +test.each([ + { n: 1, response_format: 'b64_json', callback_url: 'https://example.com/private' }, + { n: 2, response_format: 'b64_json', size: '4096x4096' }, + { n: 1, response_format: 'b64_json', size: '8192x8192' }, + { n: 1, response_format: 'b64_json', size: '2048x2048' }, +])('adapter evidence rejects unknown, unbounded or changed request parameters: %j', async requestParameters => { + const context = await (resolveAiModelVerification as jest.Mock)() + ;(resolveAiModelVerification as jest.Mock).mockResolvedValue({ ...context, protocol: { id: 'openai-image-generate', path: '/v1/images/generations', headers: {}, supportsImageInput: true, inputField: 'image', referenceTransport: 'multipart', requestParameters: { n: 1, response_format: 'b64_json', size: '4096x4096' } } }) + ;(callAiModelProtocol as jest.Mock).mockImplementation(async request => { + await request.beforeRequest() + return { image: await verificationReferenceImage(row.evidence.challenge), requestId: 'safe-result', summary: { + referenceTransport: 'multipart', requestParameters, generatedWidth: Infinity, generatedHeight: 1, + storedWidth: 512, storedHeight: -1, referenceLayoutMatched: false, + } } + }) + await verifyAiModel('admin', providerId, modelKey, { ...input(), stage: 'image' }) + const evidence = JSON.parse(mockDb.$executeRawUnsafe.mock.calls.find(([sql]) => sql.includes("status='passed'"))![3]) + expect(evidence.referenceLayoutMatched).toBe(true) + expect(evidence.referenceTransport).toBe('multipart') + expect(evidence).not.toHaveProperty('requestParameters') + expect(evidence).not.toHaveProperty('generatedWidth'); expect(evidence).not.toHaveProperty('generatedHeight') + expect(evidence).not.toHaveProperty('storedWidth'); expect(evidence).not.toHaveProperty('storedHeight') +}) test('repeated idempotency keys and unresolved requests cannot submit another model call', async () => { duplicate = true await verifyAiModel('admin', providerId, modelKey, input()) @@ -165,11 +242,19 @@ test('a closed verification stops dispatch and a late response cannot append a f expect((aiModelAudit as jest.Mock).mock.calls.some(([, , action]) => action === 'model.verification-passed')).toBe(false) }) test('a terminal decoded image that fails reference validation is failed, not an endless unknown task', async () => { - const image = await sharp({ create: { width: 128, height: 128, channels: 3, background: '#ffffff' } }).png().toBuffer() - ;(callAiModelProtocol as jest.Mock).mockImplementation(async request => { await request.beforeRequest(); await request.onTaskSubmitted('task-ended'); return { image, requestId: 'known-result', summary: {} } }) + ;(callAiModelProtocol as jest.Mock).mockImplementation(async request => { + await request.beforeRequest(); await request.onTaskSubmitted('task-ended') + const challenge = row.evidence.challenge + const image = await verificationReferenceImage({ topLeft: challenge.bottomRight, bottomRight: challenge.topLeft }) + return { image, requestId: 'known-result', summary: {} } + }) row.stage = 'image' await verifyAiModel('admin', providerId, modelKey, { ...input(), stage: 'image' }) - expect(mockDb.$executeRawUnsafe.mock.calls.find(([sql]) => sql.includes('status=$2'))?.[2]).toBe('failed') + const failed = mockDb.$executeRawUnsafe.mock.calls.find(([sql]) => sql.includes('status=$2'))! + expect(failed[2]).toBe('failed') + expect(failed[3]).toBe('known-result') + expect(failed[4]).toBe('task-ended') + expect(failed[5]).toContain('颜色与相对位置') expect(callAiModelProtocol).toHaveBeenCalledTimes(1) }) diff --git a/lib/ai-model-capabilities.ts b/lib/ai-model-capabilities.ts index 2925225..38db0f1 100644 --- a/lib/ai-model-capabilities.ts +++ b/lib/ai-model-capabilities.ts @@ -71,17 +71,28 @@ export function parseAiModelPublicCapabilityCatalog(value: unknown): Map 200 || result.has(model.id)) throw new AiModelError('上游能力目录模型编号无效或重复', 502, 'capability_catalog_invalid') const offers = Array.isArray(model.service_offers) ? model.service_offers.map(object).filter((offer): offer is Record => Boolean(offer)) : [] // Do not combine non-default service offers or route-specific protocols into the default request path. - const selected = offers.find(offer => offer.is_default === true && object(offer.service_scope)?.id === model.default_service_scope_id) - const rawInputModes = selected?.input_modes ?? model.input_modes, rawOutputModes = selected?.output_modes ?? model.output_modes + const protocols = parseAiModelProtocols(model) + const selectedProtocol = protocols[0] + const selected = offers.find(offer => object(offer.service_scope)?.id === selectedProtocol?.scopeId && + Array.isArray(offer.route_options) && offer.route_options.some(route => object(route)?.id === selectedProtocol?.routeId)) || + offers.find(offer => offer.is_default === true && object(offer.service_scope)?.id === model.default_service_scope_id) + const selectedRoute = selectedProtocol?.routeId && Array.isArray(selected?.route_options) + ? selected.route_options.map(object).find(route => route?.id === selectedProtocol.routeId) : null + // The adapter selects a concrete route. Its declarations take precedence + // over the model/offer summary, which can describe a different route. + const rawInputModes = selectedRoute?.input_modes ?? selected?.input_modes ?? model.input_modes + const rawOutputModes = selectedRoute?.output_modes ?? selected?.output_modes ?? model.output_modes const inputModes = strings(rawInputModes), outputModes = strings(rawOutputModes) if (rawInputModes !== undefined && inputModes === null || rawOutputModes !== undefined && outputModes === null) throw new AiModelError('上游能力目录模态字段无效', 502, 'capability_catalog_invalid') const textModel = model.type === 'model' && ['text', 'multimodal'].includes(String(model.modality)) && section.id === 'chat' const declaredCapabilities = strings(model.capabilities) || [] - const imageModel = (model.type === 'image' || model.modality === 'image') && declaredCapabilities.some(capability => ['text_to_image', 'openai.images.generations', 'openai.images.edits'].includes(capability)) + const imageModel = (model.type === 'image' || model.modality === 'image') && declaredCapabilities.some(capability => ['text_to_image', 'image_to_image', 'openai.images.generations', 'openai.images.edits'].includes(capability)) const normalizeModes = (modes: string[]) => Array.from(new Set(modes.map(mode => ['image_url', 'image_base64'].includes(mode) ? 'image' : mode === 'prompt' ? 'text' : mode))).sort() - const inputModalities = normalizeModes(inputModes || (textModel ? ['text'] : [])) + // A generic URL mode alone proves nothing about file type. The parsed + // image adapter has already validated the concrete reference parameter. + const inputModalities = normalizeModes([...(inputModes || (textModel ? ['text'] : [])), + ...(imageModel && selectedProtocol?.supportsImageInput ? ['image'] : [])]) const outputModalities = normalizeModes(outputModes || (textModel ? ['text'] : imageModel ? ['image'] : [])) - const protocols = parseAiModelProtocols(model) const endpoints = protocols.map(protocol => protocol.path) const kindHint = publicKind(model, section, inputModes || [], outputModes || []) result.set(model.id, { version: 1, inputModalities, outputModalities, endpoints, protocols, sourceUrl: AI_MODEL_PUBLIC_CATALOG_URL, @@ -134,16 +145,18 @@ export function getAiModelCapabilities(baseUrl: string, modelId: string, metadat // the analysis protocol; its explicit input flag may differ on legacy catalogs. const transport = (protocol: AiModelProtocol | null) => protocol && JSON.stringify({ ...protocol, supportsImageInput: undefined }) const combinedTextTest = allowedStages.includes('analysis') && allowedStages.includes('copy') && !!analysisProtocol?.supportsImageInput && transport(analysisProtocol) === transport(copyProtocol) + const referenceProbe = allowedStages.includes('image') && getAiModelStageProtocol(metadata, 'image')?.referenceContract === 'probe-required' const reason = evidence.unavailable ? '上游当前不可用，暂不能测试或开放' + : referenceProbe ? '参考图采用标准图片参数；通过实际测试后可开放' : allowedStages.length ? '接口已适配；通过对应用途的实际测试后可开放' - : kind === 'image' ? evidence.inputDeclared && !imageInput && !imageEdit ? '仅支持文生图，不能接收产品参考图' - : protocols.some(protocol => protocol.id === 'openai-image-generate') ? '上游未提供可用的参考图参数，暂不能接收产品参考图' - : '上游尚未提供已适配的参考图接口' + : kind === 'image' ? imageEdit ? '参考图待适配：当前路线的输出声明与生图要求不一致' + : imageInput ? '参考图待适配：目录已声明图片输入，当前传入方式尚未接入' + : '参考图待适配：当前目录尚未确认可用的参考图传入方式' : ['text', 'vision'].includes(kind) ? '输入输出或接口不符合详情页分析、文案要求' : '该类型不适用于详情页的分析、文案或生图' return { kind, allowedStages, combinedTextTest, source: 'upstream', protocols: Object.fromEntries(allowedStages.map(stage => { const protocol = getAiModelStageProtocol(metadata, stage)! - return [stage, { id: protocol.id, path: protocol.path, scopeId: protocol.scopeId, routeId: protocol.routeId, operationId: protocol.operationId }] + return [stage, { id: protocol.id, path: protocol.path, scopeId: protocol.scopeId, routeId: protocol.routeId, operationId: protocol.operationId, ...(protocol.maxImages ? { maxImages: protocol.maxImages } : {}) }] })), reason } } diff --git a/lib/ai-model-protocol.ts b/lib/ai-model-protocol.ts index 7fff362..99d8d8f 100644 --- a/lib/ai-model-protocol.ts +++ b/lib/ai-model-protocol.ts @@ -14,6 +14,9 @@ export interface AiModelProtocol { maxImages?: number maxPromptLength?: number imageQuality?: 'medium' + referenceTransport?: 'json-inline' | 'multipart' + referenceContract?: 'probe-required' + requestParameters?: Record } const paths: Record = { 'openai-chat': '/v1/chat/completions', 'anthropic-messages': '/v1/messages', 'openai-image-edit': '/v1/images/edits', 'openai-image-generate': '/v1/images/generations', 'async-image-task': '/extensions/v1/tasks' } function record(value: unknown): Record | null { return value && typeof value === 'object' && !Array.isArray(value) ? value as Record : null } @@ -30,6 +33,111 @@ function safeHeaders(value: unknown): Record | null { } function identifier(value: unknown): value is string { return typeof value === 'string' && /^[A-Za-z0-9._-]{1,160}$/.test(value) } +const imageParameterChecks: Record boolean> = { + n: value => value === 1, + stream: value => value === false, + response_format: value => typeof value === 'string' && ['url', 'b64_json'].includes(value), + size: value => typeof value === 'string' && (value === 'auto' || /^\d{3,4}x\d{3,4}$/.test(value) && value.split('x').every(part => Number(part) >= 256 && Number(part) <= 4096)), + quality: value => typeof value === 'string' && ['auto', 'low', 'medium', 'high', 'standard', 'hd', 'xhigh', 'max'].includes(value), + background: value => typeof value === 'string' && ['transparent', 'opaque', 'auto'].includes(value), + output_format: value => typeof value === 'string' && ['png', 'jpeg', 'webp'].includes(value), + aspect_ratio: value => typeof value === 'string' && (value === 'auto' || /^\d{1,2}:\d{1,2}$/.test(value)), + version: value => typeof value === 'string' && /^[A-Za-z0-9][A-Za-z0-9._-]{0,63}$/.test(value), + thinkingLevel: value => typeof value === 'string' && ['minimal', 'low', 'medium', 'high'].includes(value), +} +function safeImageParameters(value: unknown): AiModelProtocol['requestParameters'] | null { + const raw = record(value) + if (!raw || Object.keys(raw).some(key => !Object.prototype.hasOwnProperty.call(imageParameterChecks, key) || !imageParameterChecks[key](raw[key]))) return null + return Object.fromEntries(Object.entries(raw).sort(([a], [b]) => a.localeCompare(b))) +} +function schemaParts(value: unknown, depth = 0): Record[] { + const raw = record(value) + if (!raw || depth > 5) return [] + return [raw, ...(Array.isArray(raw.allOf) ? raw.allOf.flatMap((item: unknown) => schemaParts(item, depth + 1)) : [])] +} +function inspectableSchema(value: unknown, depth = 0): boolean { + if (depth > 5) return false + if (value === undefined || value === true || value === false) return true + const raw = record(value) + if (!raw || ['if', 'then', 'else', '$ref', '$dynamicRef'].some(key => raw[key] !== undefined)) return false + for (const key of ['allOf', 'anyOf', 'oneOf']) { + if (raw[key] !== undefined && (!Array.isArray(raw[key]) || !raw[key].length || raw[key].length > 32 || !raw[key].every((item: unknown) => inspectableSchema(item, depth + 1)))) return false + } + return raw.not === undefined || inspectableSchema(raw.not, depth + 1) +} +function acceptsSchemaValue(value: unknown, schema: unknown, depth = 0): boolean { + if (schema === false || depth > 5 || !inspectableSchema(schema)) return false + const raw = record(schema) + if (!raw) return true + if (raw.const !== undefined && raw.const !== value || Array.isArray(raw.enum) && !raw.enum.includes(value)) return false + const valueType = Array.isArray(value) ? 'array' : value === null ? 'null' : typeof value + if (raw.type !== undefined && ![raw.type].flat().some(type => type === valueType || type === 'integer' && Number.isInteger(value))) return false + if (typeof value === 'number' && (typeof raw.minimum === 'number' && value < raw.minimum || typeof raw.maximum === 'number' && value > raw.maximum)) return false + if (typeof value === 'number' && (typeof raw.exclusiveMinimum === 'number' && value <= raw.exclusiveMinimum || typeof raw.exclusiveMaximum === 'number' && value >= raw.exclusiveMaximum || typeof raw.multipleOf === 'number' && (raw.multipleOf <= 0 || value % raw.multipleOf !== 0))) return false + if (typeof value === 'string' && (typeof raw.minLength === 'number' && value.length < raw.minLength || typeof raw.maxLength === 'number' && value.length > raw.maxLength)) return false + if (Array.isArray(value)) { + if (typeof raw.minItems === 'number' && value.length < raw.minItems || typeof raw.maxItems === 'number' && value.length > raw.maxItems || raw.prefixItems !== undefined || raw.contains !== undefined) return false + if (raw.items !== undefined && !value.every(item => acceptsSchemaValue(item, raw.items, depth + 1))) return false + } + if (typeof value === 'string' && typeof raw.pattern === 'string') { + if (raw.pattern.length > 512) return false + try { if (!new RegExp(raw.pattern).test(value)) return false } catch { return false } + } + if (Array.isArray(raw.allOf) && !raw.allOf.every(item => acceptsSchemaValue(value, item, depth + 1))) return false + if (Array.isArray(raw.anyOf) && !raw.anyOf.some(item => acceptsSchemaValue(value, item, depth + 1))) return false + if (Array.isArray(raw.oneOf) && raw.oneOf.filter(item => acceptsSchemaValue(value, item, depth + 1)).length !== 1) return false + if (raw.not !== undefined && acceptsSchemaValue(value, raw.not, depth + 1)) return false + return true +} +/** Keep only bounded public request fields; never carry arbitrary metadata or callback URLs. */ +function imageRequestParameters(schema: Record | null, example: unknown): AiModelProtocol['requestParameters'] | null { + if (!inspectableSchema(schema)) return null + const parts = schemaParts(schema), sample = record(example) ?? {}, parameters: Record = {} + const names = new Set(parts.flatMap(part => Object.keys(record(part.properties) ?? {}))) + for (const name of Object.keys(imageParameterChecks)) { + const constraints = parts.flatMap(part => Object.prototype.hasOwnProperty.call(record(part.properties) ?? {}, name) ? [part.properties[name]] : []) + if (constraints.some(item => item === false)) { + if (['n', 'response_format'].includes(name)) return null + continue + } + if (!constraints.length && !['n', 'response_format'].includes(name)) continue + if (!constraints.length && parts.some(part => part.additionalProperties === false && !part.properties?.[name])) return null + const required = parts.some(part => list(part.required).includes(name)) || ['n', 'response_format'].includes(name) + const declared = constraints.flatMap(item => schemaParts(item)) + const choices = declared.flatMap(item => Array.isArray(item.enum) ? item.enum : []) + if (name === 'size') choices.sort((a, b) => { + const area = (value: unknown) => typeof value === 'string' && /^\d+x\d+$/.test(value) ? value.split('x').map(Number).reduce((x, y) => x * y) : Number.MAX_SAFE_INTEGER + return area(a) - area(b) + }) + const candidates = name === 'n' ? [1] : name === 'response_format' ? ['b64_json', 'url'] : [ + ...declared.filter(item => item.const !== undefined).map(item => item.const), + ...declared.filter(item => item.default !== undefined).map(item => item.default), + ...(name === 'size' ? choices : []), sample[name], ...choices, + ] + const chosen = candidates.find(value => imageParameterChecks[name](value) && constraints.every(item => acceptsSchemaValue(value, item))) + if (chosen !== undefined) parameters[name] = chosen + else if (required || declared.some(item => item.const !== undefined)) return null + } + const required = parts.flatMap(part => list(part.required)) + if (required.some(name => !['model', 'prompt', 'image', 'images', 'file_urls'].includes(name) && parameters[name] === undefined)) return null + // These aliases describe the same resolution. Use the canonical size from the published route. + if (names.has('size') && parameters.size === undefined && parts.some(part => part.properties?.size?.enum || part.properties?.size?.const)) return null + return safeImageParameters(parameters) +} + +/** An open JSON schema allows a bounded standard-field probe, never an override of a declared restriction. */ +function canProbeImageReference(schema: Record | null, supportedFields: unknown): boolean { + if (schema?.additionalProperties !== true || !inspectableSchema(schema)) return false + const referenceFields = ['image', 'image[]', 'images', 'file_urls', 'input_image', 'input_images'] + for (const part of schemaParts(schema)) { + if (part.additionalProperties !== undefined && part.additionalProperties !== true || part.unevaluatedProperties !== undefined && part.unevaluatedProperties !== true) return false + if (['not', 'oneOf', 'anyOf', 'if', 'then', 'else', 'propertyNames'].some(name => part[name] !== undefined)) return false + if (Object.keys(record(part.patternProperties) ?? {}).length) return false + if (referenceFields.some(name => Object.prototype.hasOwnProperty.call(record(part.properties) ?? {}, name))) return false + } + return !Array.isArray(supportedFields) || !supportedFields.some(item => referenceFields.includes(record(item)?.name)) +} + /** Parse only a known operation, keeping route evidence separate from a model's combined capabilities. */ function parseInvocation(model: Record, offer: Record, route: Record): AiModelProtocol | null { const invocation = record(route.invocation ?? offer.invocation ?? model.invocation) @@ -37,17 +145,23 @@ function parseInvocation(model: Record, offer: Record, const id = (Object.keys(paths) as AiModelProtocol['id'][]).find(key => paths[key] === invocation.path) const headers = safeHeaders(invocation.headers) if (!id || !headers) return null - if (invocation.content_type !== (id === 'openai-image-edit' ? 'multipart/form-data' : 'application/json')) return null + if (invocation.content_type !== (id === 'openai-image-edit' ? 'multipart/form-data' : 'application/json') && !(id === 'openai-image-generate' && invocation.content_type === 'multipart/form-data')) return null const scopeId = record(offer.service_scope)?.id, routeId = route.id if (scopeId !== undefined && !identifier(scopeId) || routeId !== undefined && !identifier(routeId)) return null if (routeId && headers['X-Relay-Route-Option'] !== routeId) return null const schema = record(invocation.input_schema ?? route.input_schema) const properties = record(schema?.properties) ?? {} + const schemaNodes = schemaParts(schema) + // Whole-request alternatives/negations are not implemented by the field mapper. + // Refuse such contracts rather than checking individual fields and bypassing them. + if (id === 'openai-image-generate' && schema && (!inspectableSchema(schema) || schemaNodes.some(part => ['not', 'oneOf', 'anyOf'].some(name => part[name] !== undefined)))) return null + const referenceSchemas = (field: string) => schemaNodes.flatMap(part => Object.prototype.hasOwnProperty.call(record(part.properties) ?? {}, field) ? [part.properties[field]] : []) const modes = list(route.input_modes ?? offer.input_modes ?? model.input_modes) const capabilities = list(offer.capabilities ?? model.capabilities) const isTextModel = ['text', 'multimodal'].includes(model.modality) && model.type === 'model' const isImageModel = model.modality === 'image' && [...capabilities, ...list(model.capabilities)].some(value => ['text_to_image', 'image_to_image', 'openai.images.generations', 'openai.images.edits'].includes(value)) let inputField: string | undefined, supportsImageInput = false + let referenceTransport: AiModelProtocol['referenceTransport'], referenceContract: AiModelProtocol['referenceContract'], requestParameters: AiModelProtocol['requestParameters'] if (id === 'openai-chat' || id === 'anthropic-messages') { if (!isTextModel) return null supportsImageInput = modes.some(mode => ['image', 'image_url', 'image_base64'].includes(mode)) || (!modes.length && model.modality === 'multimodal') @@ -56,38 +170,68 @@ function parseInvocation(model: Record, offer: Record, if (id === 'openai-image-edit') { inputField = properties['image[]'] !== false && properties['image[]'] ? 'image[]' : properties.image !== false && properties.image ? 'image' : undefined supportsImageInput = Boolean(inputField) + } else if (id === 'openai-image-generate' && invocation.content_type === 'multipart/form-data' && modes.includes('file') && !referenceSchemas('image').includes(false) && ['image', 'images', 'file_urls'].some(field => referenceSchemas(field).some(record) && !referenceSchemas(field).includes(false))) { + // Use multipart only when the operation explicitly publishes that content type. + // A file input mode alone does not change an application/json contract. + inputField = 'image' + supportsImageInput = true + referenceTransport = 'multipart' } else { + if (id === 'openai-image-generate' && invocation.content_type !== 'application/json') return null // A declared image-URL array accepts a data URI unless its schema excludes it. // Keep private image bytes inline; never expose a product file through public storage. inputField = ['file_urls', 'images', 'image'].find(field => { const property = record(properties[field]), items = record(property?.items) const pattern = items?.pattern + const declared = referenceSchemas(field) + if (declared.includes(false)) return false if (items?.type !== 'string' || !(property?.type === 'array' || Array.isArray(property?.type) && property.type.includes('array'))) return false if (items.format !== undefined && !['uri', 'uri-reference'].includes(items.format)) return false const declaredImageInput = modes.some(mode => ['image', 'image_url', 'image_base64'].includes(mode)) - if (modes.length && !declaredImageInput) return false - if (pattern === undefined) return declaredImageInput && [...capabilities, ...list(model.capabilities)].includes('image_to_image') + if ((id === 'openai-image-generate' || modes.length) && !declaredImageInput) return false + if (!declared.every(item => acceptsSchemaValue(['data:image/png;base64,AA=='], item))) return false + const itemSchemas = declared.flatMap(item => schemaParts(item)).flatMap(item => item.items !== undefined ? [item.items] : []) + if (!itemSchemas.every(item => acceptsSchemaValue('data:image/png;base64,AA==', item) && schemaParts(item).every(part => part.format === undefined || ['uri', 'uri-reference'].includes(part.format)))) return false + if (pattern === undefined) return declaredImageInput if (typeof pattern !== 'string' || pattern.length > 512 || !pattern.includes('data:image/')) return false try { return new RegExp(pattern).test('data:image/png;base64,AA==') } catch { return false } }) supportsImageInput = Boolean(inputField) + if (id === 'openai-image-generate' && supportsImageInput) referenceTransport = 'json-inline' + } + if (id === 'openai-image-generate' && !supportsImageInput && invocation.content_type === 'application/json' && identifier(routeId) && routeId === model.default_route_option_id && canProbeImageReference(schema, invocation.supported_fields)) { + inputField = 'image' + supportsImageInput = true + referenceTransport = 'json-inline' + referenceContract = 'probe-required' + } + if (referenceTransport) { + requestParameters = imageRequestParameters(schema, invocation.request_example) ?? undefined + if (!requestParameters) return null } if (id === 'async-image-task' && !supportsImageInput) return null } - const maxImages = inputField && record(properties[inputField])?.maxItems + const imageConstraints = inputField && (referenceTransport === 'multipart' ? ['image', 'images', 'file_urls'].flatMap(field => referenceSchemas(field).flatMap(value => schemaParts(value))) : referenceSchemas(inputField).flatMap(value => schemaParts(value))) + if (imageConstraints && imageConstraints.some(part => part.maxItems !== undefined && (!Number.isInteger(part.maxItems) || part.maxItems < 1))) return null + const declaredLimits = imageConstraints ? imageConstraints.flatMap(part => [part.maxItems, ...(Array.isArray(part.anyOf) ? part.anyOf.map((entry: any) => record(entry)?.maxItems) : [])]).filter(value => Number.isInteger(value) && value > 0) : [] + // Without a published image-array contract, only the single-image challenge is verified. + const maxImages = referenceContract === 'probe-required' ? 1 : declaredLimits.length ? Math.min(...declaredLimits) : undefined const maxPromptLength = record(properties.prompt)?.maxLength // The editor has no quality control: choose an explicit supported working // quality instead of relying on an undocumented upstream default. const quality = record(properties.quality) - const imageQuality = id === 'openai-image-generate' && list(quality?.enum).includes('medium') && + const imageQuality = id === 'openai-image-generate' && referenceTransport !== 'multipart' && list(quality?.enum).includes('medium') && (quality?.const === undefined || quality.const === 'medium') && - (quality?.default === undefined || quality.default === 'medium') ? 'medium' as const : undefined + (quality?.default === undefined || quality.default === 'medium') && referenceSchemas('quality').every(item => acceptsSchemaValue('medium', item)) ? 'medium' as const : undefined + if (requestParameters && imageQuality) requestParameters = safeImageParameters({ ...requestParameters, quality: imageQuality }) ?? undefined return { id, path: paths[id], headers, supportsImageInput, ...(inputField ? { inputField } : {}), ...(scopeId ? { scopeId } : {}), ...(routeId ? { routeId } : {}), ...(identifier(offer.public_operation_id) ? { operationId: offer.public_operation_id } : {}), - ...(Number.isInteger(maxImages) && maxImages > 0 ? { maxImages: Math.min(8, maxImages) } : {}), + ...(typeof maxImages === 'number' && Number.isInteger(maxImages) && maxImages > 0 ? { maxImages: Math.min(8, maxImages) } : {}), ...(Number.isInteger(maxPromptLength) && maxPromptLength > 0 ? { maxPromptLength } : {}), - ...(imageQuality ? { imageQuality } : {}) } + ...(imageQuality ? { imageQuality } : {}), + ...(referenceContract ? { referenceContract } : {}), + ...(referenceTransport ? { referenceTransport, requestParameters } : {}) } } /** At setup time pick a concrete compatible route in the default scope; no runtime routing fallback. */ @@ -103,10 +247,10 @@ export function parseAiModelProtocols(value: unknown): AiModelProtocol[] { const routes = Array.isArray(offer.route_options) ? offer.route_options.map(record).filter(Boolean) as Record[] : [] const eligible = routes.filter(route => route.available !== false).sort((a, b) => Number(b.id === model.default_route_option_id) - Number(a.id === model.default_route_option_id)) const protocols = eligible.map(route => parseInvocation(model, offer, route)).filter(Boolean) as AiModelProtocol[] - const chosen = protocols.find(protocol => protocol.supportsImageInput) ?? protocols[0] + const chosen = protocols.find(protocol => !protocol.referenceContract && protocol.referenceTransport === 'json-inline') ?? protocols.find(protocol => !protocol.referenceContract && protocol.supportsImageInput) ?? protocols.find(protocol => protocol.supportsImageInput) ?? protocols[0] if (chosen && !found.some(protocol => JSON.stringify(protocol) === JSON.stringify(chosen))) found.push(chosen) } - return found.sort((a, b) => Number(b.supportsImageInput) - Number(a.supportsImageInput)) + return found.sort((a, b) => Number(b.supportsImageInput) - Number(a.supportsImageInput) || Number(Boolean(a.referenceContract)) - Number(Boolean(b.referenceContract)) || Number(b.referenceTransport === 'json-inline') - Number(a.referenceTransport === 'json-inline')) } export function validateStoredAiModelProtocols(value: unknown): AiModelProtocol[] { @@ -120,9 +264,14 @@ export function validateStoredAiModelProtocols(value: unknown): AiModelProtocol[ if (raw.inputField !== undefined && !['image', 'image[]', 'images', 'file_urls'].includes(raw.inputField)) return [] if (['maxImages', 'maxPromptLength'].some(field => raw[field] !== undefined && (!Number.isInteger(raw[field]) || raw[field] < 1 || raw[field] > (field === 'maxImages' ? 8 : 1000000)))) return [] if (raw.imageQuality !== undefined && (raw.id !== 'openai-image-generate' || raw.imageQuality !== 'medium')) return [] + if (raw.referenceTransport !== undefined && (!['json-inline', 'multipart'].includes(raw.referenceTransport) || raw.id !== 'openai-image-generate' || !raw.supportsImageInput || !['image', 'images', 'file_urls'].includes(raw.inputField) || raw.referenceTransport === 'multipart' && raw.inputField !== 'image')) return [] + if (raw.referenceContract !== undefined && (raw.referenceContract !== 'probe-required' || raw.id !== 'openai-image-generate' || raw.referenceTransport !== 'json-inline' || raw.inputField !== 'image' || raw.maxImages !== 1 || !raw.routeId || !raw.scopeId)) return [] + const requestParameters = raw.requestParameters !== undefined ? safeImageParameters(raw.requestParameters) : undefined + if (raw.referenceTransport !== undefined && (!requestParameters || requestParameters.n !== 1 || !requestParameters.response_format) || raw.requestParameters !== undefined && (raw.referenceTransport === undefined || !requestParameters)) return [] if (raw.id === 'async-image-task' && (!raw.supportsImageInput || raw.inputField !== 'file_urls') || raw.id === 'openai-image-generate' && raw.supportsImageInput && !raw.inputField || raw.id === 'openai-image-edit' && (!raw.supportsImageInput || !['image', 'image[]'].includes(raw.inputField))) return [] result.push({ id: raw.id, path: raw.path, headers, supportsImageInput: raw.supportsImageInput, - ...Object.fromEntries(['inputField', 'scopeId', 'routeId', 'operationId', 'maxImages', 'maxPromptLength', 'imageQuality'].filter(key => raw[key] !== undefined).map(key => [key, raw[key]])) }) + ...Object.fromEntries(['inputField', 'scopeId', 'routeId', 'operationId', 'maxImages', 'maxPromptLength', 'imageQuality', 'referenceTransport', 'referenceContract'].filter(key => raw[key] !== undefined).map(key => [key, raw[key]])), + ...(requestParameters ? { requestParameters } : {}) }) } return result } @@ -168,7 +317,7 @@ async function normalizeImages(images: AiModelProtocolInput['images'], forAnalys return forAnalysis ? pipeline.flatten({ background: '#ffffff' }).jpeg({ quality: 85 }).toBuffer() : pipeline.png().toBuffer() })) } -async function decodeImage(data: any, input: AiModelProtocolInput, taskId: string | null): Promise { +async function decodeImage(data: any, input: AiModelProtocolInput, taskId: string | null, summary: Record): Promise { let bytes: Buffer if (typeof data?.b64_json === 'string' && /^[A-Za-z0-9+/]+={0,2}$/.test(data.b64_json) && data.b64_json.length <= 28 * 1024 * 1024) bytes = Buffer.from(data.b64_json, 'base64') else if (typeof data?.url === 'string') { @@ -177,8 +326,11 @@ async function decodeImage(data: any, input: AiModelProtocolInput, taskId: strin bytes = Buffer.from(await response.arrayBuffer()) } else throw new AiModelProtocolError('模型没有返回可保存的图片', false, null, taskId) try { + const source = await sharp(bytes, { limitInputPixels: 40_000_000 }).metadata() const output = await sharp(bytes, { limitInputPixels: 40_000_000 }).rotate().resize(1440, 1440, { fit: 'inside', withoutEnlargement: true }).png().toBuffer() if (output.length > 20 * 1024 * 1024) throw new Error('size') + const stored = await sharp(output).metadata() + Object.assign(summary, { generatedWidth: source.width, generatedHeight: source.height, storedWidth: stored.width, storedHeight: stored.height }) return output } catch { throw new AiModelProtocolError('模型图片无法解码或超出限制', false, null, taskId) } } @@ -187,12 +339,15 @@ export async function callAiModelProtocol(input: AiModelProtocolInput): Promise< const protocol = validateStoredAiModelProtocols([input.protocol])[0] if (!protocol) throw new AiModelProtocolError('模型协议无效，请重新同步', false) const imageCount = input.images?.length ?? 0 - if (imageCount > (protocol.maxImages ?? 8) || imageCount && !protocol.supportsImageInput) throw new AiModelProtocolError('该模型接口不支持当前参考图片', false) + if (imageCount && !protocol.supportsImageInput) throw new AiModelProtocolError('参考图传入方式尚未适配，请重新同步模型目录', false) + if (imageCount > (protocol.maxImages ?? 8)) throw new AiModelProtocolError(`当前模型最多可接收 ${protocol.maxImages ?? 8} 张参考图，请减少图片`, false) if (protocol.maxPromptLength && input.prompt.length > protocol.maxPromptLength) throw new AiModelProtocolError(`该模型提示词最多 ${protocol.maxPromptLength} 字，请精简内容`, false) if (protocol.id === 'async-image-task' && !input.onTaskSubmitted) throw new AiModelProtocolError('异步任务缺少持久化检查点', false) if (input.resumeTaskId && (!/^[A-Za-z0-9_-]{1,160}$/.test(input.resumeTaskId) || protocol.id !== 'async-image-task')) throw new AiModelProtocolError('模型任务编号无效', false) let taskId = input.resumeTaskId ?? null, requestId: string | null = null, dispatched = Boolean(input.resumeTaskId) - const summary = { adapterVersion: AI_MODEL_ADAPTER_VERSION, protocol: protocol.id, routeId: protocol.routeId ?? null } + const summary: Record = { adapterVersion: AI_MODEL_ADAPTER_VERSION, protocol: protocol.id, routeId: protocol.routeId ?? null, + ...(protocol.referenceTransport ? { referenceTransport: protocol.referenceTransport, requestParameters: protocol.requestParameters } : {}), + ...(protocol.referenceContract ? { referenceContract: protocol.referenceContract } : {}) } try { if (!taskId) { const forAnalysis = protocol.id === 'openai-chat' || protocol.id === 'anthropic-messages' @@ -204,10 +359,11 @@ export async function callAiModelProtocol(input: AiModelProtocolInput): Promise< const maxTokens = input.maxOutputTokens && Number.isInteger(input.maxOutputTokens) ? Math.max(32, Math.min(8000, input.maxOutputTokens)) : 8000 if (protocol.id === 'openai-chat') body = JSON.stringify({ model: input.modelId, messages: [{ role: 'user', content: images.length ? [{ type: 'text', text: input.prompt }, ...dataUris.map(url => ({ type: 'image_url', image_url: { url } }))] : input.prompt }], max_tokens: maxTokens, stream: false }) else if (protocol.id === 'anthropic-messages') body = JSON.stringify({ model: input.modelId, messages: [{ role: 'user', content: [{ type: 'text', text: input.prompt }, ...images.map(image => ({ type: 'image', source: { type: 'base64', media_type: imageMimeType, data: image.toString('base64') } }))] }], max_tokens: maxTokens, stream: false }) - else if (protocol.id === 'openai-image-edit') { - body = new FormData(); body.set('model', input.modelId); body.set('prompt', input.prompt); body.set('n', '1'); body.set('response_format', 'b64_json') + else if (protocol.id === 'openai-image-edit' || protocol.referenceTransport === 'multipart') { + body = new FormData(); body.set('model', input.modelId); body.set('prompt', input.prompt) + for (const [field, value] of Object.entries(protocol.requestParameters ?? { n: 1, response_format: 'b64_json' })) body.set(field, String(value)) images.forEach((image, index) => (body as FormData).append(protocol.inputField!, new Blob([new Uint8Array(image)], { type: 'image/png' }), `reference-${index + 1}.png`)) - } else body = JSON.stringify({ model: input.modelId, prompt: input.prompt, ...(images.length ? { [protocol.inputField!]: dataUris } : {}), ...(protocol.id === 'openai-image-generate' ? { n: 1, response_format: 'b64_json', ...(protocol.imageQuality ? { quality: protocol.imageQuality } : {}) } : {}) }) + } else body = JSON.stringify({ model: input.modelId, prompt: input.prompt, ...(images.length ? { [protocol.inputField!]: dataUris } : {}), ...(protocol.id === 'openai-image-generate' ? { n: 1, response_format: 'b64_json', ...(protocol.imageQuality ? { quality: protocol.imageQuality } : {}), ...protocol.requestParameters } : {}) }) dispatched = true const response = await requestAiModelOperation(input.baseUrl, input.apiKey, protocol.path, body, input.idempotencyKey, protocol.headers, input.beforeRequest) const parsed = await readJson(response, input.apiKey); requestId = parsed.requestId @@ -220,7 +376,7 @@ export async function callAiModelProtocol(input: AiModelProtocolInput): Promise< const text = typeof content === 'string' ? content : Array.isArray(content) ? content.filter(item => item?.type === 'text' && typeof item.text === 'string').map(item => item.text).join('\n') : '' if (!text.trim()) throw new AiModelProtocolError('模型未返回可用文字', false, requestId) return { text, requestId, summary } - } else return { image: await decodeImage(parsed.data?.data?.[0], input, null), requestId, summary } + } else return { image: await decodeImage(parsed.data?.data?.[0], input, null, summary), requestId, summary } } const deadline = Date.now() + 40_000 for (let attempt = 0; attempt < 20 && Date.now() < deadline; attempt++) { @@ -231,7 +387,7 @@ export async function callAiModelProtocol(input: AiModelProtocolInput): Promise< const outputs = parsed.data?.result?.outputs const image = Array.isArray(outputs) ? outputs.find(item => item?.type === 'image' && (typeof item.url === 'string' || typeof item.b64_json === 'string')) : null if (!image) throw new AiModelProtocolError('模型任务没有图片产物', false, requestId, taskId) - return { image: await decodeImage(image, input, taskId), requestId, summary: { ...summary, taskId } } + return { image: await decodeImage(image, input, taskId, summary), requestId, summary: { ...summary, taskId } } } if (response.status !== 202 || parsed.data?.ready !== false) throw new AiModelProtocolError('模型任务状态无法确认，已保留任务', true, requestId, taskId) await new Promise(resolve => setTimeout(resolve, 2000)) diff --git a/lib/ai-model-types.ts b/lib/ai-model-types.ts index fd35e65..4785c0d 100644 --- a/lib/ai-model-types.ts +++ b/lib/ai-model-types.ts @@ -24,7 +24,7 @@ export interface AiModelCapabilities { allowedStages: AiModelStage[] source: 'upstream' | 'verified' | 'unknown' reason: string - protocols?: Partial> + protocols?: Partial> combinedTextTest?: boolean } export interface AiModelCatalogEntry { diff --git a/lib/ai-model-verification.ts b/lib/ai-model-verification.ts index 39a4246..56e4dcb 100644 --- a/lib/ai-model-verification.ts +++ b/lib/ai-model-verification.ts @@ -3,7 +3,7 @@ import { z } from 'zod' import { AiModelError, assertAiModelAdmin, assertAiModelAdminForWrite } from './ai-model-access' import { aiModelAudit, aiModelTransaction } from './ai-model-repository' import { getAiModelAdminState, resolveAiModelVerification } from './ai-model-service' -import { AI_MODEL_ADAPTER_VERSION, AiModelProtocolError, callAiModelProtocol } from './ai-model-protocol' +import { AI_MODEL_ADAPTER_VERSION, AiModelProtocolError, callAiModelProtocol, validateStoredAiModelProtocols, type AiModelProtocol } from './ai-model-protocol' import { AI_MODEL_TEST_PURPOSES, aiModelTestStages } from './ai-model-types' import { AI_MODEL_VERIFICATION_COLUMNS, lockAiModelVerificationExecution, verificationExecutionId, type AiModelVerificationRow } from './ai-model-verification-state' import { prisma } from './prisma' @@ -14,6 +14,28 @@ import type { AiModelDatabase } from './ai-model-repository' const requestSchema = z.object({ stage: z.enum(AI_MODEL_TEST_PURPOSES), revision: z.number().int().positive(), idempotencyKey: z.string().uuid() }).strict() function safeId(value: unknown, secret: string): string | null { return typeof value === 'string' && /^[\w.-]{1,160}$/.test(value) && !value.includes(secret) ? value : null } +/** Persist only bounded adapter facts; model output cannot supply or override validation evidence. */ +function imageTransportEvidence(value: unknown, protocol: AiModelProtocol, secret: string): Record { + if (!value || typeof value !== 'object' || Array.isArray(value)) return {} + const summary = value as Record, evidence: Record = {} + if ((summary.referenceTransport === 'multipart' || summary.referenceTransport === 'json-inline') && summary.referenceTransport === protocol.referenceTransport) { + evidence.referenceTransport = summary.referenceTransport + const actual = validateStoredAiModelProtocols([{ ...protocol, requestParameters: summary.requestParameters }])[0] + const expected = validateStoredAiModelProtocols([protocol])[0] + // Use the adapter's existing parameter allowlist, and require the actual + // values to match the route whose context was locked for this execution. + if (actual?.requestParameters && expected?.requestParameters && JSON.stringify(actual.requestParameters) === JSON.stringify(expected.requestParameters) && + (!secret || !JSON.stringify(actual.requestParameters).includes(secret))) evidence.requestParameters = actual.requestParameters + } + for (const [widthKey, heightKey, maximum] of [['generatedWidth', 'generatedHeight', 40_000_000], ['storedWidth', 'storedHeight', 1440]] as const) { + const width = summary[widthKey], height = summary[heightKey] + if (typeof width === 'number' && typeof height === 'number' && Number.isInteger(width) && Number.isInteger(height) && + width > 0 && height > 0 && width <= maximum && height <= maximum && width * height <= 40_000_000) { + evidence[widthKey] = width; evidence[heightKey] = height + } + } + return evidence +} async function assertExecutionSlot(db: AiModelDatabase, actorId: string, resumeId: string | null = null): Promise { await detailTransactionLock(db, 'detail-generator-execution') const rows = await db.$queryRawUnsafe>(`SELECT COUNT(*)::int AS count,COUNT(*) FILTER (WHERE actor_id=$1)::int AS own FROM ( @@ -30,7 +52,7 @@ function assertCombinedContexts(contexts: VerificationContext[]): void { } async function executeVerification(rows: AiModelVerificationRow[], userId: string, providerId: string, resume: boolean): Promise { const row = rows[0], combined = rows.length === 2 - let sent = false, resultReceived = false, apiKey = '', knownTaskId = row.taskId + let sent = false, resultReceived = false, apiKey = '', knownTaskId = row.taskId, knownRequestId: string | null = null const beforeRequest = async () => { await assertAiModelAdmin(userId) const current = await Promise.all(rows.map(async item => { @@ -52,12 +74,13 @@ async function executeVerification(rows: AiModelVerificationRow[], userId: strin if (combined) assertCombinedContexts(contexts) const context = contexts[0] apiKey = context.apiKey + knownRequestId = safeId(row.requestId, apiKey) if (context.contextHash !== row.contextHash) throw new AiModelError('验证对应的连接或路由已改变，不能继续此任务', 409, 'verification_context_changed') const challenge = readVerificationChallenge(row.evidence?.challenge) const prompt = combined ? '一次完成两项独立任务。analysis：必须读取参考图，识别左上与右下方块的颜色，以英文小写填写。copy：真实产品资料为型号TEST、颜色蓝色，图片只是读图测试图，不是该产品外观；只依据这两条真实资料写一屏中文详情页，不编造材质、尺寸、性能或认证。严格只输出一个JSON对象：{"analysis":{"topLeft":"英文小写颜色","bottomRight":"英文小写颜色"},"copy":{"sellingPoints":["卖点"],"warnings":[],"screens":[{"title":"标题","body":"正文","imagePrompt":"无文字的场景描述"}]}}。' : row.stage === 'analysis' ? '观察图片：左上方方块与右下方方块分别是什么颜色？只输出 JSON：{"topLeft":"英文小写颜色","bottomRight":"英文小写颜色"}。不要根据文字猜测，必须读取图片。' : row.stage === 'copy' ? '真实产品资料：型号TEST，颜色蓝色。不要编造其他规格。写一屏中文详情页，严格只输出 JSON：{"sellingPoints":["卖点"],"warnings":[],"screens":[{"title":"标题","body":"正文","imagePrompt":"无文字的场景描述"}]}。' - : '依据参考图片生成一张简洁的产品展示图，保留两个方块的颜色与相对位置，白色背景，不要加入文字或标志。' + : '将参考图重新绘制为干净的平面图，严格保留两个方块各自的原始颜色、形状和相对位置，纯白背景；不要添加文字、标志、阴影、纹理、渐变、光照或其它内容。' const result = await callAiModelProtocol({ baseUrl: context.provider.baseUrl, apiKey: context.apiKey, modelId: context.model.id, protocol: context.protocol, prompt, ...(row.stage !== 'copy' ? { images: [{ buffer: await verificationReferenceImage(challenge), mimeType: 'image/png' }] } : {}), maxOutputTokens: combined ? 1800 : row.stage === 'copy' ? 1200 : 1024, idempotencyKey: row.idempotencyKey, beforeRequest, ...(resume ? { resumeTaskId: row.taskId! } : {}), @@ -72,8 +95,12 @@ async function executeVerification(rows: AiModelVerificationRow[], userId: strin }) } }) resultReceived = true // The native protocol has returned a terminal, decoded result. + knownRequestId = safeId(result.requestId, context.apiKey) || knownRequestId const checks: Partial> = combined ? await validateAiModelCombinedVerificationOutput(result, challenge) - : { [row.stage]: { status: 'passed', message: '实际接口测试通过', evidence: await validateAiModelVerificationOutput(row.stage, result, challenge) } } + : { [row.stage]: { status: 'passed', message: '实际接口测试通过', evidence: { + ...(row.stage === 'image' ? imageTransportEvidence(result.summary, context.protocol, context.apiKey) : {}), + ...await validateAiModelVerificationOutput(row.stage, result, challenge), + } } } await assertAiModelAdmin(userId) await aiModelTransaction(async db => { await assertAiModelAdminForWrite(userId, db) @@ -101,7 +128,7 @@ async function executeVerification(rows: AiModelVerificationRow[], userId: strin const message = safeProtocolMessage || (unknown ? '请求结果尚未确认，请核查上游记录；禁止重复提交' : error instanceof AiModelError ? error.message : '模型测试未通过') await aiModelTransaction(async db => { for (const item of rows) { - const updated = await db.$executeRawUnsafe(`UPDATE ai_model_verifications SET status=$2,request_id=$3,task_id=COALESCE(task_id,$4),message=$5,finished_at=NOW() WHERE id=$1 AND status='running'`, item.id, unknown ? 'unknown' : 'failed', safeId(outcome.requestId, apiKey), safeId(outcome.taskId || knownTaskId, apiKey), message) + const updated = await db.$executeRawUnsafe(`UPDATE ai_model_verifications SET status=$2,request_id=$3,task_id=COALESCE(task_id,$4),message=$5,finished_at=NOW() WHERE id=$1 AND status='running'`, item.id, unknown ? 'unknown' : 'failed', safeId(outcome.requestId, apiKey) || knownRequestId, safeId(outcome.taskId || knownTaskId, apiKey), message) if (updated === 1) await aiModelAudit(db, userId, unknown ? 'model.verification-unknown' : 'model.verification-failed', providerId, { verificationId: item.id, modelKey: item.modelKey, stage: item.stage, ...(combined ? { executionGroupId: verificationExecutionId(item) } : {}) }) } })