feat(ai): verify analysis and copy in one model request
This commit is contained in:
parent
efb8e1af3d
commit
21b6eb767c
@ -19,6 +19,61 @@ const configured = (): AiModelAdminState => ({
|
||||
afterEach(() => jest.restoreAllMocks())
|
||||
beforeEach(() => sessionStorage.clear())
|
||||
|
||||
function jointConfiguration() {
|
||||
const next = configured()
|
||||
next.providers[0].models[2].capabilities.combinedTextTest = true
|
||||
return next
|
||||
}
|
||||
test('多模态模型一次确认、一次请求，通过后两项可选但不会自动开放', async () => {
|
||||
const next = jointConfiguration(), passed = jointConfiguration()
|
||||
for (const stage of ['analysis', 'copy'] as const) passed.providers[0].models[2].verification[stage] = { status: 'passed', id: `joint-${stage}`, checkedAt: null, requestId: 'one-upstream-request', message: '该用途已通过', purpose: 'analysis-copy' }
|
||||
global.fetch = jest.fn(async (url: string) => response(url.endsWith('/verify') ? passed : next)) as typeof fetch
|
||||
render(<AiModelManagementPanel />)
|
||||
const select = await screen.findByLabelText('new-vision-model · 现有上游 测试用途')
|
||||
expect(select).toHaveValue('analysis-copy')
|
||||
expect(within(select).getAllByRole('option')).toHaveLength(1)
|
||||
fireEvent.click(screen.getByRole('button', { name: '测试 new-vision-model · 现有上游 的分析与文案' }))
|
||||
const dialog = await screen.findByRole('dialog', { name: '测试分析与文案' })
|
||||
expect(dialog).toHaveTextContent('一次调用，分别验证读图和文案能力。')
|
||||
fireEvent.click(within(dialog).getByRole('button', { name: '开始测试' }))
|
||||
await screen.findByText('分析：已通过')
|
||||
expect(screen.getByText('文案：已通过')).toBeInTheDocument()
|
||||
expect(screen.getByRole('checkbox', { name: 'new-vision-model · 现有上游 用于资料分析' })).toBeEnabled()
|
||||
expect(screen.getByRole('checkbox', { name: 'new-vision-model · 现有上游 用于卖点文案' })).toBeEnabled()
|
||||
expect(screen.getByRole('checkbox', { name: '开放 new-vision-model · 现有上游 给用户' })).not.toBeChecked()
|
||||
const calls = (global.fetch as jest.Mock).mock.calls.filter(([url]) => url.endsWith('/verify'))
|
||||
expect(calls).toHaveLength(1)
|
||||
expect(JSON.parse(calls[0][1].body)).toMatchObject({ stage: 'analysis-copy', revision: 3 })
|
||||
})
|
||||
test.each(['analysis', 'copy'] as const)('联合测试只有 %s 通过时，可单独勾选并保存该用途', async passedStage => {
|
||||
const next = jointConfiguration(), partial = jointConfiguration()
|
||||
for (const stage of ['analysis', 'copy'] as const) partial.providers[0].models[2].verification[stage] = { status: stage === passedStage ? 'passed' : 'failed', id: `joint-${stage}`, checkedAt: null, requestId: 'partial-request', message: '独立判定', purpose: 'analysis-copy' }
|
||||
global.fetch = jest.fn(async (url: string) => response(url.endsWith('/verify') || url.endsWith('/configuration') ? partial : next)) as typeof fetch
|
||||
render(<AiModelManagementPanel />)
|
||||
fireEvent.click(await screen.findByRole('button', { name: '测试 new-vision-model · 现有上游 的分析与文案' }))
|
||||
fireEvent.click(within(await screen.findByRole('dialog')).getByRole('button', { name: '开始测试' }))
|
||||
const passedName = passedStage === 'analysis' ? '资料分析' : '卖点文案', failedName = passedStage === 'analysis' ? '卖点文案' : '资料分析'
|
||||
const checkbox = await screen.findByRole('checkbox', { name: `new-vision-model · 现有上游 用于${passedName}` })
|
||||
await waitFor(() => expect(checkbox).toBeEnabled())
|
||||
expect(screen.getByRole('checkbox', { name: `new-vision-model · 现有上游 用于${failedName}` })).toBeDisabled()
|
||||
fireEvent.click(checkbox)
|
||||
fireEvent.click(screen.getByRole('checkbox', { name: '开放 new-vision-model · 现有上游 给用户' }))
|
||||
fireEvent.click(screen.getByRole('button', { name: '保存配置' }))
|
||||
await screen.findByText('模型配置已保存')
|
||||
const saved = (global.fetch as jest.Mock).mock.calls.find(([url]) => url.endsWith('/configuration'))
|
||||
expect(JSON.parse(saved[1].body).providers[0].models.find((value: any) => value.key === 'model-new')).toEqual({ key: 'model-new', enabled: true, stages: [passedStage] })
|
||||
})
|
||||
test('刷新恢复联合测试提交，两项匹配同一请求编号后才解除暂停', async () => {
|
||||
const key = '11111111-1111-4111-8111-111111111111', next = jointConfiguration()
|
||||
sessionStorage.setItem('ai-model-pending-submissions-v1', JSON.stringify([['model-new', { stage: 'analysis-copy', requestId: key, previousId: null }]]))
|
||||
next.providers[0].models[2].verification.analysis.submissionKey = key
|
||||
global.fetch = jest.fn(async () => response(next)) as typeof fetch
|
||||
render(<AiModelManagementPanel />)
|
||||
expect(await screen.findByRole('button', { name: '核查 new-vision-model · 现有上游 的测试提交' })).toBeEnabled()
|
||||
expect(screen.getByRole('checkbox', { name: 'new-vision-model · 现有上游 用于卖点文案' })).toBeDisabled()
|
||||
expect((global.fetch as jest.Mock).mock.calls.every(([, init]) => !init?.method)).toBe(true)
|
||||
})
|
||||
|
||||
test('开放范围有未保存修改时仍可测试尚未开放的模型，结果保留原草稿', async () => {
|
||||
const next = configured()
|
||||
const passed = configured()
|
||||
|
||||
@ -3,11 +3,17 @@
|
||||
import { useEffect, useMemo, useRef, useState } from 'react'
|
||||
import { Info, Plus, RefreshCw, Save, Search } from 'lucide-react'
|
||||
import { useConfirmActionDialog } from '@/components/ConfirmActionDialog'
|
||||
import type { AiModelAdminState, AiModelCatalogEntry, AiModelProvider, AiModelStage } from '@/lib/ai-model-types'
|
||||
import { AI_MODEL_TEST_PURPOSES, aiModelTestStages, type AiModelAdminState, type AiModelCatalogEntry, type AiModelProvider, type AiModelStage, type AiModelTestPurpose } from '@/lib/ai-model-types'
|
||||
|
||||
const stages: Array<{ id: AiModelStage; label: string; shortLabel: string }> = [
|
||||
{ id: 'analysis', label: '资料分析', shortLabel: '分析' }, { id: 'copy', label: '卖点文案', shortLabel: '文案' }, { id: 'image', label: '图片生成', shortLabel: '生图' },
|
||||
]
|
||||
const combinedPurpose = { id: 'analysis-copy' as const, label: '分析与文案' }
|
||||
function testPurposes(model: AiModelCatalogEntry): Array<{ id: AiModelTestPurpose; label: string }> {
|
||||
return model.capabilities.combinedTextTest ? [combinedPurpose, ...stages.filter(stage => stage.id === 'image' && model.capabilities.allowedStages.includes(stage.id))]
|
||||
: stages.filter(stage => model.capabilities.allowedStages.includes(stage.id))
|
||||
}
|
||||
function purposeLabel(purpose: AiModelTestPurpose) { return purpose === 'analysis-copy' ? combinedPurpose.label : stages.find(stage => stage.id === purpose)!.label }
|
||||
const fieldClass = 'min-h-11 w-full min-w-0 rounded-lg border border-[#cfd8e3] bg-white px-3 py-2 text-base text-[#101828] placeholder:text-[#66758b] focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-[#1677ff] disabled:bg-[#f8fafc] disabled:text-[#66758b] sm:text-sm'
|
||||
const buttonClass = 'inline-flex min-h-11 shrink-0 items-center justify-center gap-2 rounded-lg border border-[#cfd8e3] bg-white px-3 text-sm font-medium text-[#344054] transition-colors hover:bg-[#f8fafc] focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-[#1677ff] focus-visible:ring-offset-2 disabled:cursor-not-allowed disabled:opacity-50'
|
||||
const primaryClass = `${buttonClass} !border-[#0050cb] !bg-[#0050cb] !text-white hover:!bg-[#003da3]`
|
||||
@ -46,10 +52,10 @@ export default function AiModelManagementPanel({ onStateChange }: { onStateChang
|
||||
const [capabilityFilter, setCapabilityFilter] = useState('all')
|
||||
const [statusFilter, setStatusFilter] = useState('all')
|
||||
const [page, setPage] = useState(1)
|
||||
const [testStages, setTestStages] = useState<Record<string, AiModelStage>>({})
|
||||
const [testStages, setTestStages] = useState<Record<string, AiModelTestPurpose>>({})
|
||||
const [expandedDetails, setExpandedDetails] = useState<Record<string, boolean>>({})
|
||||
const busyRef = useRef(false)
|
||||
const uncertainAttempts = useRef(new Map<string, { stage: AiModelStage; previousId: string | null; requestId: string }>())
|
||||
const uncertainAttempts = useRef(new Map<string, { stage: AiModelTestPurpose; previousId: string | null; requestId: string }>())
|
||||
const { confirmAction, confirmActionDialog } = useConfirmActionDialog()
|
||||
const dirty = !!state && !!draft && JSON.stringify(state) !== JSON.stringify(draft)
|
||||
function rememberAttempts() {
|
||||
@ -62,9 +68,9 @@ export default function AiModelManagementPanel({ onStateChange }: { onStateChang
|
||||
const resolved = { ...snapshot, providers: next.providers.map((provider) => ({ ...provider, models: provider.models.map((model) => {
|
||||
const attempt = uncertainAttempts.current.get(model.key)
|
||||
if (!attempt) return model
|
||||
const result = model.verification[attempt.stage]
|
||||
if (result.submissionKey === attempt.requestId) { uncertainAttempts.current.delete(model.key); return model }
|
||||
return { ...model, verification: { ...model.verification, [attempt.stage]: { ...result, status: 'unknown' as const, requestId: attempt.requestId, canResume: false, canReconcile: false, message: '尚未获取本次测试记录，请核查提交后再操作' } } }
|
||||
const attemptedStages = aiModelTestStages(attempt.stage)
|
||||
if (attemptedStages.every(stage => model.verification[stage].submissionKey === attempt.requestId)) { uncertainAttempts.current.delete(model.key); return model }
|
||||
return { ...model, verification: { ...model.verification, ...Object.fromEntries(attemptedStages.map(stage => [stage, { ...model.verification[stage], purpose: attempt.stage, status: 'unknown' as const, requestId: attempt.requestId, canResume: false, canReconcile: false, message: '尚未获取本次测试记录，请核查提交后再操作' }])) } }
|
||||
}) })) }
|
||||
rememberAttempts()
|
||||
setState(resolved)
|
||||
@ -89,7 +95,7 @@ export default function AiModelManagementPanel({ onStateChange }: { onStateChang
|
||||
let cancelled = false
|
||||
try {
|
||||
const saved = JSON.parse(sessionStorage.getItem(pendingStorageKey) || '[]')
|
||||
if (Array.isArray(saved)) for (const entry of saved.slice(0, 20)) if (Array.isArray(entry) && typeof entry[0] === 'string' && stages.some(stage => stage.id === entry[1]?.stage) && /^[a-f0-9-]{36}$/i.test(entry[1]?.requestId)) uncertainAttempts.current.set(entry[0], entry[1])
|
||||
if (Array.isArray(saved)) for (const entry of saved.slice(0, 20)) if (Array.isArray(entry) && typeof entry[0] === 'string' && AI_MODEL_TEST_PURPOSES.includes(entry[1]?.stage) && /^[a-f0-9-]{36}$/i.test(entry[1]?.requestId)) uncertainAttempts.current.set(entry[0], entry[1])
|
||||
} catch { /* Ignore malformed browser state; no model calls are made here. */ }
|
||||
request().then((next) => { if (!cancelled) acceptRef.current(next) }).catch((cause) => { if (!cancelled) setError(cause.message) }).finally(() => { if (!cancelled) setLoading(false) })
|
||||
return () => { cancelled = true }
|
||||
@ -164,32 +170,32 @@ export default function AiModelManagementPanel({ onStateChange }: { onStateChang
|
||||
void run('provider-save', () => request(form.id ? `/providers/${encodeURIComponent(form.id)}` : '/providers', { method: form.id ? 'PATCH' : 'POST', body: JSON.stringify(body) }), form.id ? '上游已保存' : '上游已添加，请同步模型目录', { closeForm: true })
|
||||
}
|
||||
|
||||
async function testModel(provider: AiModelProvider, model: AiModelCatalogEntry, stage: AiModelStage) {
|
||||
async function testModel(provider: AiModelProvider, model: AiModelCatalogEntry, stage: AiModelTestPurpose) {
|
||||
const savedProvider = state?.providers.find((entry) => entry.id === provider.id)
|
||||
const savedModel = savedProvider?.models.find((entry) => entry.key === model.key)
|
||||
if (busyRef.current || form || !savedProvider?.enabled || !savedProvider.hasApiKey || !savedModel?.present || !savedModel.capabilities.allowedStages.includes(stage) || Object.values(savedModel.verification).some((result) => result.status === 'running' || result.status === 'unknown')) return
|
||||
if (busyRef.current || form || !savedProvider?.enabled || !savedProvider.hasApiKey || !savedModel?.present || !testPurposes(savedModel).some(purpose => purpose.id === stage) || Object.values(savedModel.verification).some((result) => result.status === 'running' || result.status === 'unknown')) return
|
||||
busyRef.current = true
|
||||
setBusy(`confirm:${model.key}`)
|
||||
setTestStages((current) => ({ ...current, [model.key]: stage }))
|
||||
const label = stages.find((item) => item.id === stage)!.label
|
||||
const confirmation = await confirmAction({ title: `测试${label}`, description: <><span className="break-all">{model.id}</span><br />将实际调用模型，可能产生上游费用。</>, confirmLabel: '开始测试', tone: 'primary' })
|
||||
const label = purposeLabel(stage), attemptedStages = aiModelTestStages(stage)
|
||||
const confirmation = await confirmAction({ title: `测试${label}`, description: <><span className="break-all">{model.id}</span><br />{stage === 'analysis-copy' && <>一次调用，分别验证读图和文案能力。<br /></>}将实际调用模型，可能产生上游费用。</>, confirmLabel: '开始测试', tone: 'primary' })
|
||||
if (!confirmation.confirmed) { busyRef.current = false; setBusy(''); return }
|
||||
setBusy(`verify:${model.key}:${stage}`); setError(''); setNotice('')
|
||||
let idempotencyKey = ''
|
||||
try {
|
||||
idempotencyKey = crypto.randomUUID()
|
||||
uncertainAttempts.current.set(model.key, { stage, previousId: savedModel.verification[stage].id, requestId: idempotencyKey }); rememberAttempts()
|
||||
uncertainAttempts.current.set(model.key, { stage, previousId: savedModel.verification[attemptedStages[0]].id, requestId: idempotencyKey }); rememberAttempts()
|
||||
const next = await request(`/providers/${encodeURIComponent(provider.id)}/models/${encodeURIComponent(model.key)}/verify`, { method: 'POST', body: JSON.stringify({ stage, revision: savedProvider.revision, idempotencyKey }) })
|
||||
uncertainAttempts.current.delete(model.key); rememberAttempts()
|
||||
accept(next, true)
|
||||
} catch (cause) {
|
||||
// Refresh state after a lost response; never repeat a potentially billable request.
|
||||
const message = cause instanceof Error ? cause.message : '测试请求未完成'
|
||||
if (idempotencyKey && (!(cause instanceof ModelRequestError) || cause.status >= 500)) uncertainAttempts.current.set(model.key, { stage, previousId: savedModel.verification[stage].id, requestId: idempotencyKey })
|
||||
if (idempotencyKey && (!(cause instanceof ModelRequestError) || cause.status >= 500)) uncertainAttempts.current.set(model.key, { stage, previousId: savedModel.verification[attemptedStages[0]].id, requestId: idempotencyKey })
|
||||
else uncertainAttempts.current.delete(model.key)
|
||||
rememberAttempts()
|
||||
try { accept(await request(), true) } catch {
|
||||
const markUnknown = (current: AiModelAdminState | null) => current && ({ ...current, providers: current.providers.map((entry) => entry.id !== provider.id ? entry : { ...entry, models: entry.models.map((item) => item.key !== model.key ? item : { ...item, verification: { ...item.verification, [stage]: { ...item.verification[stage], status: 'unknown' as const, message: '未能获取测试结果，请刷新状态核查' } } }) }) })
|
||||
const markUnknown = (current: AiModelAdminState | null) => current && ({ ...current, providers: current.providers.map((entry) => entry.id !== provider.id ? entry : { ...entry, models: entry.models.map((item) => item.key !== model.key ? item : { ...item, verification: { ...item.verification, ...Object.fromEntries(attemptedStages.map(purpose => [purpose, { ...item.verification[purpose], purpose: stage, status: 'unknown' as const, message: '未能获取测试结果，请刷新状态核查' }])) } }) }) })
|
||||
setState(markUnknown); setDraft(markUnknown)
|
||||
}
|
||||
setError(`${message}；已停止提交，请核查测试状态。`)
|
||||
@ -269,17 +275,23 @@ export default function AiModelManagementPanel({ onStateChange }: { onStateChang
|
||||
</fieldset>
|
||||
{!!rows.length && <>
|
||||
<div className="grid min-w-0 grid-cols-2 gap-2 sm:flex sm:flex-wrap"><label className="relative col-span-2 min-w-0 sm:flex-1 sm:basis-48"><span className="sr-only">搜索模型或上游</span><Search className="pointer-events-none absolute left-3 top-3.5 h-4 w-4 text-[#66758b]" aria-hidden="true" /><input className={`${fieldClass} !pl-9`} value={search} onChange={(event) => { setSearch(event.target.value); setPage(1) }} placeholder="搜索模型或上游" /></label><select aria-label="按能力筛选模型" className={`${fieldClass} sm:!w-auto sm:flex-none`} value={capabilityFilter} onChange={(event) => { setCapabilityFilter(event.target.value); setPage(1) }}><option value="all">全部能力</option>{Object.entries(capabilityLabels).map(([kind, label]) => <option key={kind} value={kind}>{label}</option>)}</select><select aria-label="按状态筛选模型" className={`${fieldClass} sm:!w-auto sm:flex-none`} value={statusFilter} onChange={(event) => { setStatusFilter(event.target.value); setPage(1) }}><option value="all">全部状态</option><option value="enabled">已开放</option><option value="disabled">未开放</option><option value="untested">待测试</option><option value="attention">待处理</option><option value="missing">已离开目录</option></select></div>
|
||||
<p className="text-xs text-[#66758b]">模型无需先开放即可测试。通过后勾选用途与开放，再保存配置。</p>
|
||||
<p className="text-xs text-[#66758b]">多模态模型一次测试分析与文案，通过的用途可勾选；开放后保存配置。</p>
|
||||
<div className="hidden grid-cols-[minmax(0,1fr)_88px_200px_248px] gap-3 border-b border-[#e2e8f0] bg-[#f8fafc] px-3 py-2 text-xs font-semibold text-[#4f6077] xl:grid" aria-hidden="true"><span>模型 / 上游</span><span>开放给用户</span><span>用户可选用途</span><span>实际测试</span></div>
|
||||
<div className="divide-y divide-[#e2e8f0] border-b border-[#e2e8f0]" aria-label="模型目录">
|
||||
{!visibleRows.length && <p role="status" className="px-3 py-5 text-sm text-[#66758b]">没有匹配模型</p>}
|
||||
{pageRows.map(({ provider, model }) => {
|
||||
const testableStages = stages.filter((stage) => model.capabilities.allowedStages.includes(stage.id))
|
||||
const selectedStage = stages.find((stage) => ['unknown', 'running'].includes(model.verification[stage.id].status)) || testableStages.find((stage) => stage.id === testStages[model.key]) || testableStages.find((stage) => model.verification[stage.id].status !== 'passed') || testableStages[0]
|
||||
const result = selectedStage ? model.verification[selectedStage.id] : null
|
||||
const protocol = selectedStage ? model.capabilities.protocols?.[selectedStage.id] : null
|
||||
const testableStages = testPurposes(model)
|
||||
const unresolvedStage = stages.find(stage => ['unknown', 'running'].includes(model.verification[stage.id].status))
|
||||
const unresolvedPurpose = unresolvedStage && (model.verification[unresolvedStage.id].purpose || unresolvedStage.id)
|
||||
const selectedStage = unresolvedPurpose ? { id: unresolvedPurpose, label: purposeLabel(unresolvedPurpose) }
|
||||
: testableStages.find(stage => stage.id === testStages[model.key]) || testableStages.find(stage => aiModelTestStages(stage.id).some(purpose => model.verification[purpose].status !== 'passed')) || testableStages[0]
|
||||
const testedStages = selectedStage ? aiModelTestStages(selectedStage.id) : []
|
||||
const resultStage = testedStages.find(stage => ['unknown', 'running'].includes(model.verification[stage].status)) || testedStages[0]
|
||||
const result = resultStage ? model.verification[resultStage] : null
|
||||
const protocol = resultStage ? model.capabilities.protocols?.[resultStage] : null
|
||||
const detailsId = `test-details-${model.key}-${selectedStage?.id}`
|
||||
const canInspect = !!protocol || !!result && !!(result.message || result.checkedAt || result.requestId)
|
||||
const canInspect = !!protocol || testedStages.some(stage => { const value = model.verification[stage]; return !!(value.message || value.checkedAt || value.requestId) })
|
||||
const retest = testedStages.some(stage => { const value = model.verification[stage]; return value.status === 'passed' || value.status === 'stale' || value.reconciled })
|
||||
const unsettled = Object.values(model.verification).some((entry) => entry.status === 'running' || entry.status === 'unknown')
|
||||
const testing = busy.startsWith(`verify:${model.key}:`)
|
||||
const attempt = uncertainAttempts.current.get(model.key)
|
||||
@ -305,19 +317,23 @@ export default function AiModelManagementPanel({ onStateChange }: { onStateChang
|
||||
<div className="min-w-0 space-y-1.5">
|
||||
{selectedStage && result ? <>
|
||||
<div className="flex min-w-0 items-center gap-2">
|
||||
<select aria-label={`${model.id} · ${provider.name} 测试用途`} className={`${fieldClass} !px-2`} disabled={!!busy || unsettled} value={selectedStage.id} onChange={(event) => setTestStages((current) => ({ ...current, [model.key]: event.target.value as AiModelStage }))}>
|
||||
<select aria-label={`${model.id} · ${provider.name} 测试用途`} className={`${fieldClass} !px-2`} disabled={!!busy || unsettled} value={selectedStage.id} onChange={(event) => setTestStages((current) => ({ ...current, [model.key]: event.target.value as AiModelTestPurpose }))}>
|
||||
{(!testableStages.some(stage => stage.id === selectedStage.id) ? [selectedStage] : testableStages).map((stage) => <option key={stage.id} value={stage.id}>{stage.label}</option>)}
|
||||
</select>
|
||||
{attempt ? <button type="button" className={buttonClass} disabled={statusBlocked} aria-label={`核查 ${model.id} · ${provider.name} 的测试提交`} onClick={() => void run(`submission:${model.key}`, () => request(`/providers/${encodeURIComponent(provider.id)}/models/${encodeURIComponent(model.key)}/submission`, { method: 'POST', body: JSON.stringify({ stage: attempt.stage, idempotencyKey: attempt.requestId }) }), '', { preserveDraft: true })}>核查提交</button> : result.canResume && result.id ? <button type="button" className={buttonClass} disabled={statusBlocked || !provider.enabled || !provider.hasApiKey} aria-label={`查询 ${model.id} · ${provider.name} 的${selectedStage.label}结果`} onClick={() => {
|
||||
setTestStages((current) => ({ ...current, [model.key]: selectedStage.id }))
|
||||
void run(`result:${model.key}`, () => request(`/providers/${encodeURIComponent(provider.id)}/verifications/${encodeURIComponent(result.id!)}/refresh`, { method: 'POST' }), '', { preserveDraft: true })
|
||||
}}>{busy === `result:${model.key}` ? '查询中…' : '查询结果'}</button> : result.canReconcile && result.id ? <button type="button" className={buttonClass} disabled={statusBlocked} aria-label={`核查 ${model.id} · ${provider.name} 的${selectedStage.label}结果`} onClick={() => { setError(''); setNotice(''); setRecovery({ providerId: provider.id, verificationId: result.id!, modelId: model.id, outcome: 'not_executed', reference: '', note: '', confirmed: false }) }}>核查结果</button> : <button type="button" className={buttonClass} disabled={!!blockedReason} aria-describedby={testStatusId} aria-label={`测试 ${model.id} · ${provider.name} 的${selectedStage.label}`} onClick={() => void testModel(provider, model, selectedStage.id)}>{testing || result.status === 'running' ? '测试中…' : result.status === 'passed' || result.status === 'stale' || result.reconciled ? '重测' : '测试'}</button>}
|
||||
}}>{busy === `result:${model.key}` ? '查询中…' : '查询结果'}</button> : result.canReconcile && result.id ? <button type="button" className={buttonClass} disabled={statusBlocked} aria-label={`核查 ${model.id} · ${provider.name} 的${selectedStage.label}结果`} onClick={() => { setError(''); setNotice(''); setRecovery({ providerId: provider.id, verificationId: result.id!, modelId: model.id, outcome: 'not_executed', reference: '', note: '', confirmed: false }) }}>核查结果</button> : <button type="button" className={buttonClass} disabled={!!blockedReason} aria-describedby={testStatusId} aria-label={`测试 ${model.id} · ${provider.name} 的${selectedStage.label}`} onClick={() => void testModel(provider, model, selectedStage.id)}>{testing || result.status === 'running' ? '测试中…' : retest ? '重测' : '测试'}</button>}
|
||||
{canInspect && <button type="button" className={`${buttonClass} !w-11 !px-0`} aria-label={`${model.id} · ${provider.name} ${selectedStage.label}测试详情`} aria-expanded={!!expandedDetails[detailsId]} aria-controls={detailsId} title="测试详情" onClick={() => setExpandedDetails((current) => ({ ...current, [detailsId]: !current[detailsId] }))}><Info className="h-4 w-4" aria-hidden="true" /></button>}
|
||||
</div>
|
||||
<p id={testStatusId} role="status" className={`text-xs ${blockedReason ? 'text-[#66758b]' : result.status === 'passed' ? 'text-[#087052]' : result.status === 'failed' || result.status === 'unknown' ? 'text-[#b42318]' : 'text-[#66758b]'}`}>{testing ? '正在调用模型，请稍候' : attempt ? '提交结果未确认，请核查' : result.status === 'unknown' ? result.canResume ? '等待上游结果，可查询' : '结果未知，暂停核查' : result.status === 'running' ? '测试进行中，自动更新状态' : result.reconciled ? '已核查，可重测' : blockedReason || verificationLabels[result.status]}</p>
|
||||
<p id={testStatusId} role="status" className={`flex flex-wrap gap-x-3 gap-y-1 text-xs ${blockedReason ? 'text-[#66758b]' : result.status === 'passed' ? 'text-[#087052]' : result.status === 'failed' || result.status === 'unknown' ? 'text-[#b42318]' : 'text-[#66758b]'}`}>{testing ? '正在调用模型，请稍候' : attempt ? '提交结果未确认，请核查' : selectedStage.id === 'analysis-copy' ? testedStages.map(stage => {
|
||||
const value = model.verification[stage]
|
||||
return <span key={stage} className={value.status === 'passed' ? 'text-[#087052]' : ['failed', 'unknown'].includes(value.status) ? 'text-[#b42318]' : 'text-[#66758b]'}>{stages.find(entry => entry.id === stage)!.shortLabel}：{value.reconciled ? '已核查，可重测' : verificationLabels[value.status]}</span>
|
||||
}) : result.status === 'unknown' ? result.canResume ? '等待上游结果，可查询' : '结果未知，暂停核查' : result.status === 'running' ? '测试进行中，自动更新状态' : result.reconciled ? '已核查，可重测' : blockedReason || verificationLabels[result.status]}</p>
|
||||
{selectedStage.id === 'analysis-copy' && blockedReason && !testing && !attempt && !unsettled && <p className="text-xs text-[#66758b]">{blockedReason}</p>}
|
||||
{canInspect && <div id={detailsId} hidden={!expandedDetails[detailsId]} className="space-y-1 break-words pb-2 text-xs text-[#66758b]">
|
||||
{protocol && <><p className="break-all">协议：{protocol.id}</p><p className="break-all">接口：{protocol.path}</p>{(protocol.scopeId || protocol.routeId || protocol.operationId) && <p className="break-all">路由：{[protocol.scopeId, protocol.routeId, protocol.operationId].filter(Boolean).join(' / ')}</p>}</>}
|
||||
{result.message && <p>{result.message}</p>}{result.checkedAt && <p>{new Date(result.checkedAt).toLocaleString('zh-CN')}</p>}{result.requestId && <p className="break-all">请求编号：{result.requestId}</p>}
|
||||
{testedStages.map(stage => { const value = model.verification[stage]; return <div key={stage} className="space-y-1">{selectedStage.id === 'analysis-copy' && <p className="font-medium text-[#344054]">{stages.find(entry => entry.id === stage)!.label}</p>}{value.message && <p>{value.message}</p>}{value.checkedAt && <p>{new Date(value.checkedAt).toLocaleString('zh-CN')}</p>}{value.requestId && <p className="break-all">请求编号：{value.requestId}</p>}</div> })}
|
||||
{result.taskId && <p className="break-all">上游任务：{result.taskId}</p>}
|
||||
</div>}
|
||||
</> : <p className="text-xs text-[#66758b]">{model.capabilities.kind === 'unknown' ? '能力待识别，暂不可测试' : '暂未接入详情页用途'}</p>}
|
||||
|
||||
25
docs/ai-model-combined-acceptance-2026-10-14.md
Normal file
25
docs/ai-model-combined-acceptance-2026-10-14.md
Normal file
@ -0,0 +1,25 @@
|
||||
# 分析与文案联合测试验收
|
||||
|
||||
范围：同一路由的多模态模型，一次测试分析与文案；两项独立记录、通过的用途可独立开放。纯文本及生图用途保留各自测试。
|
||||
|
||||
## 真实模型
|
||||
|
||||
- 在隔离数据库、原生管理页面点击“分析与文案”并确认测试，Gemini 3 Flash 返回实际读图结果及一屏中文文案大纲；两项均通过。
|
||||
- 请求编号为 `f1c5d300-a809-4389-8d4c-59595d4dd14d`，分析和文案分别持久化，两条记录共享这个上游请求编号、提交编号及执行编号。
|
||||
- 受理审计只增加一次；使用原提交编号重复请求没有新增模型调用。
|
||||
|
||||
## 原生接口与异常恢复
|
||||
|
||||
`scripts/verify-ai-model-combined.ts` 固定限定到本机 `127.0.0.1:55441/ai_model_test`，拒绝生产数据库。恢复及部分失败用例使用显式标记的 QA 记录，不伪称实际模型输出。
|
||||
|
||||
- 联合测试两条进行中或未知记录只占一个执行名额，保留每人 1 个、全局 2 个的限制。
|
||||
- 响应丢失后核查同一提交；缺少受理记录时关闭编号，迟到提交被拒绝。
|
||||
- 核查一条联合记录会在同一事务关闭两条未知记录；不将核查当作测试通过。
|
||||
- 分析通过、文案失败时，保存文案开放被拒绝；仅开放分析成功。QA 记录在验收后移除，已有未知真实请求保留。
|
||||
- 回归测试覆盖反向部分失败、格式错误、不可读图、路由不一致、重复提交、撤权、已关闭记录和旧单项恢复。
|
||||
|
||||
## 页面
|
||||
|
||||
Impeccable 按 Operate 模式检查桌面 1280 / 1440px 和移动端 390px。两项结果同时显示；通过的用途可勾选，未通过的用途禁用。移动端按钮高度至少 44px，输入文字 16px，无页面横向溢出。检测器对改动面板报告 0 项问题。
|
||||
|
||||
证据目录：`output/ai-combined-20261014/`。本项验收不扩大模型开放范围，也不改变详情页正式验收前仅超级管理员可用的限制。
|
||||
@ -8,7 +8,7 @@
|
||||
2. 新增上游填写名称、HTTPS API 地址和 API Key。新增或替换凭据需已有专用 `API_KEY_ENCRYPTION_KEY`，最少 32 字节。凭据使用 AES-256-GCM 加密，接口只返回配置状态；编辑时密钥留空保持现状。
|
||||
3. “同步目录”请求上游 `/v1/models`。当前接入的 `api.vip98k.com` 还读取其公开的 `/api/v2/catalog/models` 结构化能力目录，只保留当前 Key 有权使用的模型。模型数量取决于上游目录、Key 权限和同步结果，不固定为某个数量。新发现模型默认未开放；同步不会调用生成接口，也不会自动批量测试。
|
||||
4. 搜索模型或上游，按对话、视觉理解、生图、语音合成、语音识别、音乐、视频、向量、图像工具、3D、其他或待识别筛选。分类由服务端读取能力及协议证据，不按模型名字猜测。类型已识别不代表当前详情页已接入该类型。
|
||||
5. 在模型行选择**测试用途**，点击“测试”。确认“将实际调用模型，可能产生上游费用”后，只测试当前模型的当前用途。资料分析、卖点文案、图片生成分别记录结果；一个用途通过，不会替另一个用途背书。模型无需先开放即可测试；用户开放范围与默认值的未保存修改不阻止测试，结果刷新后保留草稿及原版本。上游连接必须已启用且配置密钥；编辑连接期间须先保存或取消，测试按钮旁显示阻止原因。
|
||||
5. 支持读图和文案、且两项使用同一原生路由的多模态模型，测试用途显示**分析与文案**。确认费用后只发送一次模型请求，返回读图结果与文案大纲；服务端独立校验两部分、分别保存分析与文案记录，界面同时显示两个结果。只有其中一项通过时也可单独勾选该用途，不要求另一项通过。纯文本模型只测试文案，生图仍单独测试；不能因文案通过就认定模型能读图。模型无需先开放即可测试；用户开放范围与默认值的未保存修改不阻止测试，结果刷新后保留草稿及原版本。上游连接必须已启用且配置密钥；编辑连接期间须先保存或取消，测试按钮旁显示阻止原因。
|
||||
6. 用途实际测试通过后，勾选允许用户使用的用途，再勾选“开放”。测试成功不会自动开放模型。为三个阶段选择默认模型，点击“保存配置”后生效；没有有效测试结果、能力不符或尚未适配的用途不能开放。
|
||||
7. 用户可在详情页“模型选择”中自行选择管理员已开放的模型，默认值只是新项目的初始选择。不同上游的同名模型以独立编号区分，实际请求仍使用上游原生模型 ID。详情页正式验收前仍只向超级管理员开放。
|
||||
|
||||
@ -30,6 +30,8 @@
|
||||
|
||||
浏览器在提交前将请求编号保存在当前浏览器会话，丢失响应或重新加载后先恢复核查，不自动重发付费请求。“核查提交”与服务端受理共用事务锁：找到原记录就读取结果；没有受理记录就关闭该请求编号，拒绝其之后迟到的原提交。该操作不新建用途验证记录，也不会遮住其他未结束的测试。再次测试必须重新确认费用并使用新编号。真实测试与详情页生成共用执行额度：每人同时 1 个，全局同时 2 个；所有结果未知的测试在核查结束前保留占用。
|
||||
|
||||
联合测试的两个用途记录共享请求编号、执行编号和参考图检查点，只占一个执行名额；分析和文案各自保留验证上下文及通过证据。结果未知时暂停整次请求，核查提交须找到完整的两条记录，核查结束时在同一事务关闭两条记录；原请求仍可查询时只查询该请求。重新测试不会自动开放用途，旧单项测试和旧上游任务仍可按原方式核查。
|
||||
|
||||
2026-10-14 修复测试入口及异常恢复流程。测试、查询和刷新与未保存的开放设置独立，读取结果保留草稿及原版本；上层管理菜单也保护未保存修改与正在执行的操作。用途选择器与状态分开显示。真实隔离验收中，Gemini 3 Flash 的资料分析与文案、Nano Banana 2 的参考图生图通过；恢复查询沿用同一上游任务和测试编号，无新增生成受理。补充的 Claude Sonnet 4.6 请求结果未知，保持未开放，未重发。证据保存在 `output/ai-workflow-20261014/`；这不表示所有目录模型均已通过实测。
|
||||
|
||||
模型行的详情按钮默认折叠。展开后可查看当前用途实际绑定的协议、接口路径，以及目录提供的服务范围、路由和操作标识；已有结果还显示测试时间、消息和请求编号。这里显示服务端绑定，不允许前端随意修改路由。
|
||||
@ -40,6 +42,7 @@
|
||||
| --- | --- | --- |
|
||||
| 资料分析 | 支持图片输入的 OpenAI Chat Completions 或 Anthropic Messages | 能读取测试图片，并正确返回指定位置的颜色 JSON。 |
|
||||
| 卖点文案 | OpenAI Chat Completions 或 Anthropic Messages | 返回符合详情页大纲校验的一屏 JSON。 |
|
||||
| 分析与文案联合测试 | 同一路由、支持图片输入的 OpenAI Chat Completions 或 Anthropic Messages | 一次请求返回 `analysis`、`copy`；两部分独立校验，分别保存通过或失败结果。 |
|
||||
| 图片生成 | OpenAI Images Edits；明确支持私有参考图内联输入的 Images Generations 或异步图片任务 | 请求可接受参考图，结果可解码、尺寸有效且非空白。 |
|
||||
|
||||
具体接口以每个模型的服务端协议绑定为准。同步时在目录明确的服务范围内选择可执行路由，并保留对应标识；执行时不会静默换路由。专用协议或异步接口只有符合已实现适配及私有参考图输入要求时，才可进入相应用途测试；纯文生图不因此获得产品参考图能力。
|
||||
|
||||
@ -33,6 +33,13 @@ test('public catalog classifies models dynamically without guessing names', () =
|
||||
expect(getAiModelCapabilities(base, 'catalog-model', publicMetadata(model()))).toMatchObject({ kind: 'text', allowedStages: ['analysis', 'copy'], source: 'upstream' })
|
||||
expect(getAiModelCapabilities(base, 'catalog-model', publicMetadata(model({ modality: 'multimodal' })))).toMatchObject({ kind: 'vision', allowedStages: ['analysis', 'copy'] })
|
||||
})
|
||||
|
||||
test('same-route image-and-text models offer a combined test while explicitly text-only models do not', () => {
|
||||
const multimodal = publicMetadata(model({ modality: 'multimodal', input_modes: ['text', 'image'], output_modes: ['text'] }))
|
||||
expect(getAiModelCapabilities(base, 'catalog-model', multimodal)).toMatchObject({ allowedStages: ['analysis', 'copy'], combinedTextTest: true })
|
||||
const textOnly = publicMetadata(model({ input_modes: ['text'], output_modes: ['text'] }))
|
||||
expect(getAiModelCapabilities(base, 'catalog-model', textOnly)).toMatchObject({ allowedStages: ['copy'], combinedTextTest: false })
|
||||
})
|
||||
test('public image processing tools do not appear as generative image models', () => {
|
||||
const entry = model({ type: 'image', modality: 'image', capabilities: ['media.tasks'], input_modes: ['file'], output_modes: ['image'] })
|
||||
expect(getAiModelCapabilities(base, entry.id, publicMetadata(entry, 'task'))).toMatchObject({ kind: 'image-tool', allowedStages: [] })
|
||||
|
||||
@ -16,15 +16,15 @@ const mockDb = { $queryRawUnsafe: jest.fn(), $executeRawUnsafe: jest.fn() }
|
||||
jest.mock('@/lib/ai-model-repository', () => ({ aiModelTransaction: jest.fn(async action => action(mockDb)), aiModelAudit: jest.fn() }))
|
||||
const providerId = randomUUID(), modelKey = randomUUID(), request = { stage: 'analysis', idempotencyKey: randomUUID() }
|
||||
const proof = { outcome: 'failed', reference: 'upstream-log-123', note: '核查上游日志，原请求已失败并停止执行。', confirmed: true }
|
||||
let row: any, exists: boolean, closed: boolean, found: boolean
|
||||
let row: any, group: any[] | null, exists: boolean, closed: boolean, found: boolean
|
||||
beforeEach(() => {
|
||||
jest.clearAllMocks(); exists = true; closed = false; found = true
|
||||
jest.clearAllMocks(); exists = true; closed = false; found = true; group = null
|
||||
row = { id: randomUUID(), modelKey, stage: 'analysis', idempotencyKey: request.idempotencyKey, actorId: 'admin', status: 'unknown', taskId: null, evidence: {}, contextHash: 'a'.repeat(64), deadlineAt: new Date(Date.now() - 1), startedAt: new Date(), finishedAt: new Date(), requestId: null }
|
||||
;(assertAiModelAdmin as jest.Mock).mockResolvedValue('admin')
|
||||
;(assertAiModelAdminForWrite as jest.Mock).mockResolvedValue(undefined)
|
||||
;(assertAiModelRecoveryTextSafe as jest.Mock).mockResolvedValue(undefined)
|
||||
;(resolveAiModelVerification as jest.Mock).mockResolvedValue({ contextHash: row.contextHash })
|
||||
mockDb.$queryRawUnsafe.mockImplementation(async sql => sql.includes('FROM ai_model_catalog WHERE id=') ? found ? [{ id: modelKey }] : [] : sql.includes('FROM ai_model_audit') ? closed ? [{ id: 'audit' }] : [] : exists ? [row] : [])
|
||||
mockDb.$queryRawUnsafe.mockImplementation(async sql => sql.includes("evidence->>'executionGroupId'=$3") ? group || [] : sql.includes('FROM ai_model_catalog WHERE id=') ? found ? [{ id: modelKey }] : [] : sql.includes('FROM ai_model_audit') ? closed ? [{ id: 'audit' }] : [] : exists ? [row] : [])
|
||||
mockDb.$executeRawUnsafe.mockResolvedValue(1)
|
||||
})
|
||||
test('absent intake is closed by audit and never creates a verification or calls a model', async () => {
|
||||
@ -39,6 +39,34 @@ test('existing intake is read without cancellation or repeat model calls', async
|
||||
expect(aiModelAudit).not.toHaveBeenCalled()
|
||||
expect(resolveAiModelVerification).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
function jointRows() {
|
||||
const evidence = { purpose: 'analysis-copy', executionGroupId: randomUUID() }
|
||||
group = [{ ...row, stage: 'analysis', evidence }, { ...row, id: randomUUID(), stage: 'copy', evidence }]
|
||||
row = group[1]
|
||||
}
|
||||
test('one joint intake resolves both records and one recovery closes both without approving them', async () => {
|
||||
jointRows()
|
||||
await expect(checkAiModelSubmission('admin', providerId, modelKey, { ...request, stage: 'analysis-copy' })).resolves.toMatchObject({ submissionCheck: { received: true, stage: 'analysis-copy' } })
|
||||
await reconcileAiModelVerification('admin', providerId, row.id, proof)
|
||||
expect(mockDb.$executeRawUnsafe.mock.calls.map(([, id]) => id)).toEqual(group!.map(value => value.id))
|
||||
expect(mockDb.$executeRawUnsafe.mock.calls.every(([sql]) => sql.includes("status='failed'"))).toBe(true)
|
||||
expect(aiModelAudit).toHaveBeenCalledWith(mockDb, 'admin', 'model.verification-reconciled', providerId, expect.objectContaining({ stage: 'analysis-copy', verificationIds: group!.map(value => value.id) }))
|
||||
})
|
||||
test('incomplete or still-running joint records cannot be manually closed', async () => {
|
||||
jointRows(); group![0].status = 'running'; group![0].deadlineAt = new Date(Date.now() + 600000)
|
||||
await expect(reconcileAiModelVerification('admin', providerId, row.id, proof)).rejects.toMatchObject({ code: 'verification_not_unknown' })
|
||||
group!.shift()
|
||||
await expect(reconcileAiModelVerification('admin', providerId, row.id, proof)).rejects.toMatchObject({ code: 'verification_checkpoint_invalid' })
|
||||
expect(mockDb.$executeRawUnsafe).not.toHaveBeenCalled()
|
||||
})
|
||||
test('a joint original task must be queried unless at least one of its contexts changed', async () => {
|
||||
jointRows(); group!.forEach(value => { value.taskId = 'task-joint' })
|
||||
await expect(reconcileAiModelVerification('admin', providerId, row.id, proof)).rejects.toMatchObject({ code: 'verification_query_required' })
|
||||
;(resolveAiModelVerification as jest.Mock).mockImplementation(async (_provider, _model, stage) => ({ contextHash: stage === 'analysis' ? 'changed' : row.contextHash }))
|
||||
await reconcileAiModelVerification('admin', providerId, row.id, proof)
|
||||
expect(mockDb.$executeRawUnsafe).toHaveBeenCalledTimes(2)
|
||||
})
|
||||
test('closing the same absent intake twice does not duplicate audit', async () => {
|
||||
exists = false; closed = true
|
||||
await checkAiModelSubmission('admin', providerId, modelKey, request)
|
||||
|
||||
@ -2,6 +2,7 @@
|
||||
import { randomUUID } from 'node:crypto'
|
||||
import { AiModelError, assertAiModelAdmin } from '@/lib/ai-model-access'
|
||||
import { verifyAiModel, refreshAiModelVerification, validateAiModelVerificationOutput } from '@/lib/ai-model-verification'
|
||||
import { validateAiModelCombinedVerificationOutput } from '@/lib/ai-model-verification-output'
|
||||
import { callAiModelProtocol, AiModelProtocolError } from '@/lib/ai-model-protocol'
|
||||
import { resolveAiModelVerification } from '@/lib/ai-model-service'
|
||||
import { prisma } from '@/lib/prisma'
|
||||
@ -21,15 +22,17 @@ const mockDb = { $queryRawUnsafe: jest.fn(), $executeRawUnsafe: jest.fn() }
|
||||
jest.mock('@/lib/ai-model-repository', () => ({ aiModelTransaction: jest.fn(async callback => callback(mockDb)), aiModelAudit: jest.fn() }))
|
||||
const providerId = '11111111-1111-4111-8111-111111111111', modelKey = '22222222-2222-4222-8222-222222222222'
|
||||
const input = () => ({ stage: 'analysis' as const, revision: 1, idempotencyKey: randomUUID() })
|
||||
let row: any, duplicate = false, unresolved = 0, active = 0
|
||||
let row: any, executions: any[], duplicate = false, unresolved = 0, active = 0
|
||||
const validCopy = { sellingPoints: ['蓝色外观'], warnings: [], screens: [{ title: '蓝色产品', body: '型号TEST，蓝色外观', imagePrompt: '无文字的蓝色产品展示场景' }] }
|
||||
beforeEach(() => {
|
||||
jest.clearAllMocks(); duplicate = false; unresolved = 0; active = 0
|
||||
jest.clearAllMocks(); duplicate = false; unresolved = 0; active = 0; executions = []
|
||||
row = { id: randomUUID(), modelKey, stage: 'analysis', contextHash: 'a'.repeat(64), idempotencyKey: randomUUID(), status: 'running', taskId: null, deadlineAt: new Date(Date.now() - 1) }
|
||||
;(assertAiModelAdmin as jest.Mock).mockResolvedValue('admin')
|
||||
;(resolveAiModelVerification as jest.Mock).mockResolvedValue({ provider: { baseUrl: 'https://models.example.com/v1' }, model: { id: 'model' }, protocol: { id: 'openai-chat', path: '/v1/chat/completions', headers: {}, supportsImageInput: true }, apiKey: 'synthetic-test-secret', contextHash: row.contextHash })
|
||||
mockDb.$queryRawUnsafe.mockImplementation(async (sql: string, ...args: any[]) => {
|
||||
if (sql.startsWith('INSERT')) { row.idempotencyKey = args[4]; row.evidence = JSON.parse(args[6]); return [row] }
|
||||
if (sql.includes('idempotency_key=$3')) return duplicate ? [row] : []
|
||||
if (sql.startsWith('INSERT')) { row = { ...row, id: args[0], stage: args[2], contextHash: args[3], idempotencyKey: args[4], evidence: JSON.parse(args[6]) }; executions.push(row); return [row] }
|
||||
if (sql.includes("evidence->>'executionGroupId'=$3")) return [...executions].sort((left, right) => left.stage.localeCompare(right.stage))
|
||||
if (sql.includes('idempotency_key=$2')) return duplicate ? [row] : []
|
||||
if (sql.includes('FOR UPDATE')) return [row]
|
||||
if (sql.includes('active_executions')) return [{ count: active, own: 0 }]
|
||||
if (sql.includes('COUNT(*)')) return [{ count: unresolved }]
|
||||
@ -38,7 +41,7 @@ beforeEach(() => {
|
||||
mockDb.$executeRawUnsafe.mockResolvedValue(1)
|
||||
;(prisma.$executeRawUnsafe as jest.Mock).mockResolvedValue(1)
|
||||
;(prisma.$queryRawUnsafe as jest.Mock).mockResolvedValue([{ id: row.id }])
|
||||
;(callAiModelProtocol as jest.Mock).mockImplementation(async request => { await request.beforeRequest(); return { text: JSON.stringify(row.evidence?.challenge || { topLeft: 'red', bottomRight: 'blue' }), requestId: 'request-1', summary: {} } })
|
||||
;(callAiModelProtocol as jest.Mock).mockImplementation(async request => { await request.beforeRequest(); const analysis = row.evidence?.challenge || { topLeft: 'red', bottomRight: 'blue' }; return { text: JSON.stringify(row.evidence?.purpose === 'analysis-copy' ? { analysis, copy: validCopy } : analysis), requestId: 'request-1', summary: {} } })
|
||||
})
|
||||
test('a real image-understanding response persists passed evidence without enabling the model', async () => {
|
||||
await verifyAiModel('admin', providerId, modelKey, input())
|
||||
@ -159,3 +162,64 @@ test('a terminal decoded image that fails reference validation is failed, not an
|
||||
expect(mockDb.$executeRawUnsafe.mock.calls.find(([sql]) => sql.includes('status=$2'))?.[2]).toBe('failed')
|
||||
expect(callAiModelProtocol).toHaveBeenCalledTimes(1)
|
||||
})
|
||||
|
||||
test('one multimodal request creates independent passed records and consumes one execution identity', async () => {
|
||||
const submission = { ...input(), stage: 'analysis-copy' }
|
||||
await verifyAiModel('admin', providerId, modelKey, submission)
|
||||
expect(callAiModelProtocol).toHaveBeenCalledTimes(1)
|
||||
expect(callAiModelProtocol).toHaveBeenCalledWith(expect.objectContaining({ idempotencyKey: submission.idempotencyKey, maxOutputTokens: 1800 }))
|
||||
expect(executions.map(value => value.stage)).toEqual(['analysis', 'copy'])
|
||||
expect(new Set(executions.map(value => value.evidence.executionGroupId)).size).toBe(1)
|
||||
expect(executions.every(value => value.evidence.purpose === 'analysis-copy' && value.idempotencyKey === submission.idempotencyKey)).toBe(true)
|
||||
expect(mockDb.$executeRawUnsafe.mock.calls.filter(([sql]) => sql.includes("status='passed'")).map(([, id]) => id)).toEqual(executions.map(value => value.id))
|
||||
expect((aiModelAudit as jest.Mock).mock.calls.filter(([, , action]) => action === 'model.verification-start')).toHaveLength(1)
|
||||
expect(mockDb.$queryRawUnsafe.mock.calls.find(([sql]) => sql.includes('active_executions'))?.[0]).toContain("SELECT DISTINCT COALESCE(evidence->>'executionGroupId',id)")
|
||||
duplicate = true
|
||||
await verifyAiModel('admin', providerId, modelKey, submission)
|
||||
expect(callAiModelProtocol).toHaveBeenCalledTimes(1)
|
||||
})
|
||||
|
||||
test.each(['analysis', 'copy'] as const)('a failed %s section leaves the other purpose independently passed', async failedStage => {
|
||||
;(callAiModelProtocol as jest.Mock).mockImplementation(async request => {
|
||||
await request.beforeRequest()
|
||||
const content = { analysis: row.evidence.challenge, copy: validCopy, [failedStage]: failedStage === 'analysis' ? { topLeft: 'pink', bottomRight: 'pink' } : { screens: [] } }
|
||||
return { text: JSON.stringify(content), requestId: 'partial-result', summary: {} }
|
||||
})
|
||||
await verifyAiModel('admin', providerId, modelKey, { ...input(), stage: 'analysis-copy' })
|
||||
const failed = executions.find(value => value.stage === failedStage), passed = executions.find(value => value.stage !== failedStage)
|
||||
expect(mockDb.$executeRawUnsafe.mock.calls.some(([sql, id]) => sql.includes("status='failed'") && id === failed.id)).toBe(true)
|
||||
expect(mockDb.$executeRawUnsafe.mock.calls.some(([sql, id]) => sql.includes("status='passed'") && id === passed.id)).toBe(true)
|
||||
expect(callAiModelProtocol).toHaveBeenCalledTimes(1)
|
||||
})
|
||||
|
||||
test('unsupported image input or a different copy route cannot dispatch a joint test', async () => {
|
||||
const original = await (resolveAiModelVerification as jest.Mock)()
|
||||
;(resolveAiModelVerification as jest.Mock).mockResolvedValue({ ...original, protocol: { ...original.protocol, supportsImageInput: false } })
|
||||
await expect(verifyAiModel('admin', providerId, modelKey, { ...input(), stage: 'analysis-copy' })).rejects.toMatchObject({ code: 'verification_combined_unsupported' })
|
||||
;(resolveAiModelVerification as jest.Mock).mockImplementation(async (_provider, _model, stage) => ({ ...original, protocol: { ...original.protocol, path: stage === 'copy' ? '/v1/messages' : original.protocol.path } }))
|
||||
await expect(verifyAiModel('admin', providerId, modelKey, { ...input(), stage: 'analysis-copy' })).rejects.toMatchObject({ code: 'verification_combined_unsupported' })
|
||||
expect(callAiModelProtocol).not.toHaveBeenCalled()
|
||||
expect(executions).toHaveLength(0)
|
||||
})
|
||||
|
||||
test('unknown joint outcomes pause both purposes and query only the same original task', async () => {
|
||||
;(callAiModelProtocol as jest.Mock).mockImplementation(async request => {
|
||||
await request.beforeRequest(); await request.onTaskSubmitted('task-joint')
|
||||
throw new AiModelProtocolError('interrupted', true, 'request-joint', 'task-joint')
|
||||
})
|
||||
const submission = { ...input(), stage: 'analysis-copy' }
|
||||
await verifyAiModel('admin', providerId, modelKey, submission)
|
||||
expect(mockDb.$executeRawUnsafe.mock.calls.filter(([sql, , status]) => sql.includes('status=$2') && status === 'unknown')).toHaveLength(2)
|
||||
executions.forEach(value => { value.status = 'unknown'; value.taskId = 'task-joint' })
|
||||
row = executions[1]
|
||||
;(callAiModelProtocol as jest.Mock).mockImplementation(async request => { await request.beforeRequest(); return { text: JSON.stringify({ analysis: row.evidence.challenge, copy: validCopy }), requestId: 'request-joint', summary: {} } })
|
||||
await refreshAiModelVerification('admin', providerId, row.id)
|
||||
expect(callAiModelProtocol).toHaveBeenLastCalledWith(expect.objectContaining({ resumeTaskId: 'task-joint', idempotencyKey: submission.idempotencyKey }))
|
||||
expect(executions).toHaveLength(2)
|
||||
expect(mockDb.$queryRawUnsafe.mock.calls.filter(([sql]) => sql.includes('active_executions')).at(-1)?.[2]).toBe(row.evidence.executionGroupId)
|
||||
})
|
||||
|
||||
test('malformed joint JSON fails both checks and null sections cannot pass', async () => {
|
||||
await expect(validateAiModelCombinedVerificationOutput({ text: 'not JSON' }, { topLeft: 'red', bottomRight: 'blue' })).resolves.toMatchObject({ analysis: { status: 'failed' }, copy: { status: 'failed' } })
|
||||
await expect(validateAiModelCombinedVerificationOutput({ text: JSON.stringify({ analysis: null, copy: validCopy }) }, { topLeft: 'red', bottomRight: 'blue' })).resolves.toMatchObject({ analysis: { status: 'failed' }, copy: { status: 'passed' } })
|
||||
})
|
||||
|
||||
@ -129,7 +129,12 @@ export function getAiModelCapabilities(baseUrl: string, modelId: string, metadat
|
||||
const kind = evidence.kindHint || (evidence.outputModalities.includes('audio') && textInput ? 'tts' : evidence.inputModalities.some(value => value.startsWith('audio')) && textOutput ? 'asr' : evidence.outputModalities.includes('video') ? 'video' : evidence.outputModalities.includes('embedding') ? 'embedding' : imageOutput ? 'image' : textOutput && textInput ? imageInput ? 'vision' : 'text' : evidence.outputModalities.length ? 'other' : 'unknown')
|
||||
if (!evidence.unavailable && ['text', 'vision', 'image'].includes(kind) && chat && (!evidence.inputDeclared || textInput) && (!evidence.outputDeclared || textOutput)) { if (!evidence.inputDeclared || imageInput) allowedStages.push('analysis'); allowedStages.push('copy') }
|
||||
if (!evidence.unavailable && kind === 'image' && imageEdit && (!evidence.outputDeclared || imageOutput)) allowedStages.push('image')
|
||||
return { kind, allowedStages, source: 'upstream', protocols: Object.fromEntries(allowedStages.map(stage => {
|
||||
const analysisProtocol = getAiModelStageProtocol(metadata, 'analysis'), copyProtocol = getAiModelStageProtocol(metadata, 'copy')
|
||||
// Both checks must use the same native route. Image-input admission is proven by
|
||||
// the analysis protocol; its explicit input flag may differ on legacy catalogs.
|
||||
const transport = (protocol: AiModelProtocol | null) => protocol && JSON.stringify({ ...protocol, supportsImageInput: undefined })
|
||||
const combinedTextTest = allowedStages.includes('analysis') && allowedStages.includes('copy') && !!analysisProtocol?.supportsImageInput && transport(analysisProtocol) === transport(copyProtocol)
|
||||
return { kind, allowedStages, combinedTextTest, source: 'upstream', protocols: Object.fromEntries(allowedStages.map(stage => {
|
||||
const protocol = getAiModelStageProtocol(metadata, stage)!
|
||||
return [stage, { id: protocol.id, path: protocol.path, scopeId: protocol.scopeId, routeId: protocol.routeId, operationId: protocol.operationId }]
|
||||
})), reason: evidence.unavailable ? '上游已停用该模型' : allowedStages.length ? '接口已适配；通过对应用途的实际测试后可开放' : kind === 'image' ? '当前路由尚不支持私有产品参考图' : ['text', 'vision'].includes(kind) ? '该接口或明确输入能力尚不匹配详情页' : '该类型不适用于详情页的三个用途' }
|
||||
|
||||
@ -2,10 +2,10 @@ import { z } from 'zod'
|
||||
import { AiModelError, assertAiModelAdmin, assertAiModelAdminForWrite } from './ai-model-access'
|
||||
import { aiModelAudit, aiModelTransaction } from './ai-model-repository'
|
||||
import { assertAiModelRecoveryTextSafe, getAiModelAdminState, resolveAiModelVerification } from './ai-model-service'
|
||||
import { AI_MODEL_STAGES } from './ai-model-types'
|
||||
import { AI_MODEL_VERIFICATION_COLUMNS, verificationEffectiveStatus, type AiModelVerificationRow } from './ai-model-verification-state'
|
||||
import { AI_MODEL_TEST_PURPOSES } from './ai-model-types'
|
||||
import { AI_MODEL_VERIFICATION_COLUMNS, lockAiModelVerificationExecution, verificationEffectiveStatus, verificationExecutionId, type AiModelVerificationRow } from './ai-model-verification-state'
|
||||
|
||||
const submissionSchema = z.object({ stage: z.enum(AI_MODEL_STAGES), idempotencyKey: z.string().uuid() }).strict()
|
||||
const submissionSchema = z.object({ stage: z.enum(AI_MODEL_TEST_PURPOSES), idempotencyKey: z.string().uuid() }).strict()
|
||||
const reconciliationSchema = z.object({ outcome: z.enum(['not_executed', 'failed', 'completed']), reference: z.string().trim().min(3).max(160), note: z.string().trim().min(10).max(500), confirmed: z.literal(true) }).strict()
|
||||
|
||||
/** Serialize intake checking against admission. Closing an absent submission prevents a late HTTP request from being admitted. */
|
||||
@ -18,11 +18,13 @@ export async function checkAiModelSubmission(userId: string, providerId: string,
|
||||
await assertAiModelAdminForWrite(userId, db)
|
||||
const models = await db.$queryRawUnsafe<Array<{ id: string }>>('SELECT id FROM ai_model_catalog WHERE id=$1 AND provider_id=$2', modelKey, providerId)
|
||||
if (!models[0]) throw new AiModelError('模型不存在', 404)
|
||||
const rows = await db.$queryRawUnsafe<AiModelVerificationRow[]>(`SELECT ${AI_MODEL_VERIFICATION_COLUMNS} FROM ai_model_verifications WHERE model_key=$1 AND stage=$2 AND idempotency_key=$3 FOR UPDATE`, modelKey, stage, idempotencyKey)
|
||||
const rows = await db.$queryRawUnsafe<AiModelVerificationRow[]>(`SELECT ${AI_MODEL_VERIFICATION_COLUMNS} FROM ai_model_verifications WHERE model_key=$1 AND idempotency_key=$2 ORDER BY stage FOR UPDATE`, modelKey, idempotencyKey)
|
||||
if (rows[0]) {
|
||||
if ((rows[0].evidence?.purpose || rows[0].stage) !== stage) throw new AiModelError('该请求编号对应其他测试用途，请刷新状态', 409, 'verification_idempotency_conflict')
|
||||
if (stage === 'analysis-copy') await lockAiModelVerificationExecution(db, rows[0])
|
||||
return true
|
||||
}
|
||||
const closed = await db.$queryRawUnsafe<Array<{ id: string }>>(`SELECT id FROM ai_model_audit WHERE action='model.verification-intake-cancelled' AND details->>'modelKey'=$1 AND details->>'stage'=$2 AND details->>'idempotencyKey'=$3`, modelKey, stage, idempotencyKey)
|
||||
const closed = await db.$queryRawUnsafe<Array<{ id: string }>>(`SELECT id FROM ai_model_audit WHERE action='model.verification-intake-cancelled' AND details->>'modelKey'=$1 AND details->>'idempotencyKey'=$2`, modelKey, idempotencyKey)
|
||||
if (!closed[0]) await aiModelAudit(db, userId, 'model.verification-intake-cancelled', providerId, { modelKey, stage, idempotencyKey })
|
||||
return false
|
||||
})
|
||||
@ -40,18 +42,22 @@ export async function reconcileAiModelVerification(userId: string, providerId: s
|
||||
const rows = await db.$queryRawUnsafe<AiModelVerificationRow[]>(`SELECT ${AI_MODEL_VERIFICATION_COLUMNS} FROM ai_model_verifications WHERE id=$1 AND model_key IN (SELECT id FROM ai_model_catalog WHERE provider_id=$2) FOR UPDATE`, verificationId, providerId)
|
||||
const row = rows[0]
|
||||
if (!row) throw new AiModelError('测试记录不存在', 404)
|
||||
if (row.evidence?.reconciliation) return // Repeated close cannot rewrite the original evidence.
|
||||
if (verificationEffectiveStatus(row) !== 'unknown') throw new AiModelError('仅结果未知的测试可核查；进行中的测试请等待', 409, 'verification_not_unknown')
|
||||
const execution = await lockAiModelVerificationExecution(db, row)
|
||||
if (execution.every(item => item.evidence?.reconciliation)) return // Repeated close cannot rewrite the original evidence.
|
||||
if (execution.some(item => verificationEffectiveStatus(item) !== 'unknown')) throw new AiModelError('仅结果未知的测试可核查；进行中的测试请等待', 409, 'verification_not_unknown')
|
||||
if (row.taskId) {
|
||||
let contextHash: string | null = null
|
||||
try { contextHash = (await resolveAiModelVerification(providerId, row.modelKey, row.stage)).contextHash } catch (error) {
|
||||
if (!(error instanceof AiModelError) || error.status >= 500) throw error
|
||||
let queryAvailable = true
|
||||
for (const item of execution) {
|
||||
try { if ((await resolveAiModelVerification(providerId, item.modelKey, item.stage)).contextHash !== item.contextHash) queryAvailable = false } catch (error) {
|
||||
if (!(error instanceof AiModelError) || error.status >= 500) throw error
|
||||
queryAvailable = false
|
||||
}
|
||||
}
|
||||
if (contextHash === row.contextHash) throw new AiModelError('原任务仍可查询，请使用查询结果；不能通过人工核查跳过', 409, 'verification_query_required')
|
||||
if (queryAvailable) throw new AiModelError('原任务仍可查询，请使用查询结果；不能通过人工核查跳过', 409, 'verification_query_required')
|
||||
}
|
||||
const reconciliation = { ...parsed.data, actorId: userId, checkedAt: new Date().toISOString() }
|
||||
await db.$executeRawUnsafe(`UPDATE ai_model_verifications SET status='failed',message='原请求核查已结束，可重新测试；核查不代表接口测试通过',evidence=evidence || $2::jsonb,finished_at=NOW() WHERE id=$1`, row.id, JSON.stringify({ reconciliation }))
|
||||
await aiModelAudit(db, userId, 'model.verification-reconciled', providerId, { verificationId: row.id, modelKey: row.modelKey, stage: row.stage, reconciliation })
|
||||
for (const item of execution) await db.$executeRawUnsafe(`UPDATE ai_model_verifications SET status='failed',message='原请求核查已结束，可重新测试；核查不代表接口测试通过',evidence=evidence || $2::jsonb,finished_at=NOW() WHERE id=$1`, item.id, JSON.stringify({ reconciliation }))
|
||||
await aiModelAudit(db, userId, 'model.verification-reconciled', providerId, { verificationId: row.id, verificationIds: execution.map(item => item.id), modelKey: row.modelKey, stage: row.evidence?.purpose || row.stage, executionGroupId: verificationExecutionId(row), reconciliation })
|
||||
})
|
||||
return getAiModelAdminState()
|
||||
}
|
||||
|
||||
@ -49,7 +49,13 @@ export async function getAiModelAdminState(): Promise<AiModelAdminState> {
|
||||
return { providers: providers.map(provider => ({ id: provider.id, name: provider.name, baseUrl: provider.baseUrl, enabled: provider.enabled, credentialSource: provider.credentialSource, hasApiKey: hasCredential(provider), revision: provider.revision, syncedAt: provider.syncedAt?.toISOString() || null, models: models.filter(model => model.providerId === provider.id).map(({ providerId: _providerId, capabilityMetadata, ...model }) => {
|
||||
const storedCapabilities = getAiModelCapabilities(provider.baseUrl, model.id, capabilityMetadata)
|
||||
const capabilities = model.present ? storedCapabilities : { ...storedCapabilities, allowedStages: [], protocols: {}, reason: '已从当前连接的模型目录下架，暂不可用' }
|
||||
const verification = Object.fromEntries(AI_MODEL_STAGES.map(stage => [stage, aiModelVerificationSummary(verifications, model.key, stage, modelContext({ ...model, providerId: provider.id, capabilityMetadata }, provider, stage), provider.enabled && model.present)])) as import('./ai-model-types').AiModelCatalogEntry['verification']
|
||||
const fullModel = { ...model, providerId: provider.id, capabilityMetadata }
|
||||
const verification = Object.fromEntries(AI_MODEL_STAGES.map(stage => {
|
||||
const result = verifications.find(value => value.modelKey === model.key && value.stage === stage)
|
||||
const group = result?.evidence?.purpose === 'analysis-copy' ? verifications.filter(value => value.modelKey === model.key && value.evidence?.executionGroupId === result.evidence.executionGroupId) : []
|
||||
const queryAvailable = provider.enabled && model.present && (result?.evidence?.purpose !== 'analysis-copy' || group.length === 2 && group.every(value => storedCapabilities.allowedStages.includes(value.stage) && value.contextHash === modelContext(fullModel, provider, value.stage)))
|
||||
return [stage, aiModelVerificationSummary(verifications, model.key, stage, modelContext(fullModel, provider, stage), queryAvailable)]
|
||||
})) as import('./ai-model-types').AiModelCatalogEntry['verification']
|
||||
return { ...model, capabilities, verification }
|
||||
}) })), ...configuration, encryptionConfigured: isAiModelEncryptionConfigured(), legacyAvailable: Boolean(process.env.DETAIL_GENERATOR_API_KEY?.trim()) && !providers.some(provider => provider.legacyBinding) }
|
||||
}
|
||||
|
||||
@ -1,5 +1,8 @@
|
||||
export const AI_MODEL_STAGES = ['analysis', 'copy', 'image'] as const
|
||||
export type AiModelStage = typeof AI_MODEL_STAGES[number]
|
||||
export const AI_MODEL_TEST_PURPOSES = ['analysis-copy', ...AI_MODEL_STAGES] as const
|
||||
export type AiModelTestPurpose = typeof AI_MODEL_TEST_PURPOSES[number]
|
||||
export function aiModelTestStages(purpose: AiModelTestPurpose): AiModelStage[] { return purpose === 'analysis-copy' ? ['analysis', 'copy'] : [purpose] }
|
||||
export type AiModelDefaults = Record<AiModelStage, string | null>
|
||||
export const AI_MODEL_KINDS = ['text', 'vision', 'image', 'tts', 'asr', 'music', 'video', 'embedding', 'image-tool', 'three-d', 'other', 'unknown'] as const
|
||||
export type AiModelKind = typeof AI_MODEL_KINDS[number]
|
||||
@ -14,6 +17,7 @@ export interface AiModelVerificationSummary {
|
||||
reconciled?: boolean
|
||||
taskId?: string | null
|
||||
submissionKey?: string
|
||||
purpose?: AiModelTestPurpose
|
||||
}
|
||||
export interface AiModelCapabilities {
|
||||
kind: AiModelKind
|
||||
@ -21,6 +25,7 @@ export interface AiModelCapabilities {
|
||||
source: 'upstream' | 'verified' | 'unknown'
|
||||
reason: string
|
||||
protocols?: Partial<Record<AiModelStage, { id: string; path: string; scopeId?: string; routeId?: string; operationId?: string }>>
|
||||
combinedTextTest?: boolean
|
||||
}
|
||||
export interface AiModelCatalogEntry {
|
||||
key: string
|
||||
@ -48,7 +53,7 @@ export interface AiModelAdminState {
|
||||
revision: number
|
||||
encryptionConfigured: boolean
|
||||
legacyAvailable: boolean
|
||||
submissionCheck?: { idempotencyKey: string; modelKey: string; stage: AiModelStage; received: boolean }
|
||||
submissionCheck?: { idempotencyKey: string; modelKey: string; stage: AiModelTestPurpose; received: boolean }
|
||||
}
|
||||
export interface AiModelConfigurationInput {
|
||||
revision: number
|
||||
|
||||
@ -53,7 +53,7 @@ async function referenceLayoutMatches(bytes: Buffer, expected: VerificationChall
|
||||
export async function validateAiModelVerificationOutput(stage: AiModelStage, result: { text?: string; image?: Buffer }, challenge: VerificationChallenge = { topLeft: 'red', bottomRight: 'blue' }): Promise<Record<string, unknown>> {
|
||||
if (stage === 'analysis') {
|
||||
const value = parseJson(result.text)
|
||||
if (value.topLeft !== challenge.topLeft || value.bottomRight !== challenge.bottomRight) throw new AiModelError('图片内容识别未通过，不能开放资料分析', 422, 'verification_output_invalid')
|
||||
if (value?.topLeft !== challenge.topLeft || value?.bottomRight !== challenge.bottomRight) throw new AiModelError('图片内容识别未通过，不能开放资料分析', 422, 'verification_output_invalid')
|
||||
return { imageInputVerified: true, colorPositionsMatched: true, textCharacters: result.text!.length }
|
||||
}
|
||||
if (stage === 'copy') {
|
||||
@ -68,3 +68,24 @@ export async function validateAiModelVerificationOutput(stage: AiModelStage, res
|
||||
if (!(await referenceLayoutMatches(result.image, challenge))) throw new AiModelError('图片未保留参考图的颜色与相对位置，不能开放参考图生图', 422, 'verification_output_invalid')
|
||||
return { referenceImageAccepted: true, referenceLayoutMatched: true, imageDecoded: true, width: metadata.width, height: metadata.height, bytes: result.image.length }
|
||||
}
|
||||
|
||||
export type AiModelTextCheck = { status: 'passed' | 'failed'; message: string; evidence: Record<string, unknown> }
|
||||
/** A malformed or failed section cannot approve or erase the other purpose. */
|
||||
export async function validateAiModelCombinedVerificationOutput(result: { text?: string }, challenge: VerificationChallenge): Promise<Record<'analysis' | 'copy', AiModelTextCheck>> {
|
||||
let value: any
|
||||
try { value = parseJson(result.text) } catch (error) {
|
||||
if (!(error instanceof AiModelError)) throw error
|
||||
const failed: AiModelTextCheck = { status: 'failed', message: error.message, evidence: {} }
|
||||
return { analysis: { ...failed }, copy: { ...failed } }
|
||||
}
|
||||
const check = async (stage: 'analysis' | 'copy'): Promise<AiModelTextCheck> => {
|
||||
try {
|
||||
const evidence = await validateAiModelVerificationOutput(stage, { text: JSON.stringify(value?.[stage] ?? null) }, challenge)
|
||||
return { status: 'passed', message: '联合测试：该用途已通过', evidence }
|
||||
} catch (error) {
|
||||
if (!(error instanceof AiModelError)) throw error
|
||||
return { status: 'failed', message: error.message, evidence: {} }
|
||||
}
|
||||
}
|
||||
return { analysis: await check('analysis'), copy: await check('copy') }
|
||||
}
|
||||
|
||||
@ -5,6 +5,7 @@ import type { AiModelStage, AiModelVerificationSummary } from './ai-model-types'
|
||||
import { aiModelCapabilityFingerprint } from './ai-model-capabilities'
|
||||
import { AI_MODEL_ADAPTER_VERSION, type AiModelProtocol } from './ai-model-protocol'
|
||||
import { AI_MODEL_VERIFICATION_RULE_VERSION } from './ai-model-verification-output'
|
||||
import { AiModelError } from './ai-model-access'
|
||||
|
||||
export interface AiModelVerificationRow {
|
||||
id: string; modelKey: string; stage: AiModelStage; contextHash: string; idempotencyKey: string; actorId: string;
|
||||
@ -13,6 +14,20 @@ export interface AiModelVerificationRow {
|
||||
}
|
||||
export const AI_MODEL_VERIFICATION_COLUMNS = `id,model_key AS "modelKey",stage,context_hash AS "contextHash",idempotency_key AS "idempotencyKey",actor_id AS "actorId",status,request_id AS "requestId",task_id AS "taskId",message,evidence,started_at AS "startedAt",deadline_at AS "deadlineAt",finished_at AS "finishedAt"`
|
||||
|
||||
export function verificationExecutionId(row: AiModelVerificationRow): string {
|
||||
return row.evidence?.purpose === 'analysis-copy' && typeof row.evidence.executionGroupId === 'string' ? row.evidence.executionGroupId : row.id
|
||||
}
|
||||
|
||||
/** Lock both purpose records together; never recover only half of one upstream request. */
|
||||
export async function lockAiModelVerificationExecution(db: AiModelDatabase, row: AiModelVerificationRow): Promise<AiModelVerificationRow[]> {
|
||||
if (row.evidence?.purpose !== 'analysis-copy') return [row]
|
||||
const executionId = verificationExecutionId(row)
|
||||
if (!/^[a-f0-9-]{36}$/i.test(executionId) || executionId === row.id) throw new AiModelError('联合测试检查点无效，请核查记录', 409, 'verification_checkpoint_invalid')
|
||||
const rows = await db.$queryRawUnsafe<AiModelVerificationRow[]>(`SELECT ${AI_MODEL_VERIFICATION_COLUMNS} FROM ai_model_verifications WHERE model_key=$1 AND idempotency_key=$2 AND evidence->>'executionGroupId'=$3 ORDER BY stage FOR UPDATE`, row.modelKey, row.idempotencyKey, executionId)
|
||||
if (rows.length !== 2 || !rows.some(value => value.id === row.id) || rows[0].stage !== 'analysis' || rows[1].stage !== 'copy' || rows.some(value => value.evidence?.purpose !== 'analysis-copy' || value.taskId !== row.taskId)) throw new AiModelError('联合测试记录不完整，请核查原请求', 409, 'verification_checkpoint_invalid')
|
||||
return rows
|
||||
}
|
||||
|
||||
export async function listAiModelVerifications(db: AiModelDatabase = prisma): Promise<AiModelVerificationRow[]> {
|
||||
return db.$queryRawUnsafe<AiModelVerificationRow[]>(`SELECT DISTINCT ON (model_key,stage) ${AI_MODEL_VERIFICATION_COLUMNS} FROM ai_model_verifications ORDER BY model_key,stage,started_at DESC,id DESC`)
|
||||
}
|
||||
@ -34,6 +49,6 @@ export function aiModelVerificationSummary(rows: AiModelVerificationRow[], model
|
||||
const status = effective === 'unknown' ? 'unknown' : row.contextHash !== contextHash || effective === 'passed' && row.evidence?.ruleVersion !== AI_MODEL_VERIFICATION_RULE_VERSION ? 'stale' : effective
|
||||
const canResume = Boolean(queryAvailable && row.taskId && row.contextHash === contextHash && effective === 'unknown')
|
||||
return { status, id: row.id, checkedAt: row.finishedAt?.toISOString() || row.startedAt.toISOString(), requestId: row.requestId,
|
||||
canResume, canReconcile: effective === 'unknown' && !canResume, reconciled: Boolean(row.evidence?.reconciliation), taskId: row.taskId, submissionKey: row.idempotencyKey,
|
||||
canResume, canReconcile: effective === 'unknown' && !canResume, reconciled: Boolean(row.evidence?.reconciliation), taskId: row.taskId, submissionKey: row.idempotencyKey, purpose: row.evidence?.purpose === 'analysis-copy' ? 'analysis-copy' : stage,
|
||||
message: status === 'stale' ? '连接、路由或测试规则已改变，请重新测试' : status === 'unknown' ? '请求结果尚未确认，已暂停重复测试；请核查上游请求记录' : row.message }
|
||||
}
|
||||
|
||||
@ -4,69 +4,105 @@ import { AiModelError, assertAiModelAdmin, assertAiModelAdminForWrite } from './
|
||||
import { aiModelAudit, aiModelTransaction } from './ai-model-repository'
|
||||
import { getAiModelAdminState, resolveAiModelVerification } from './ai-model-service'
|
||||
import { AI_MODEL_ADAPTER_VERSION, callAiModelProtocol } from './ai-model-protocol'
|
||||
import { AI_MODEL_STAGES, type AiModelStage } from './ai-model-types'
|
||||
import { AI_MODEL_VERIFICATION_COLUMNS, type AiModelVerificationRow } from './ai-model-verification-state'
|
||||
import { AI_MODEL_TEST_PURPOSES, aiModelTestStages } from './ai-model-types'
|
||||
import { AI_MODEL_VERIFICATION_COLUMNS, lockAiModelVerificationExecution, verificationExecutionId, type AiModelVerificationRow } from './ai-model-verification-state'
|
||||
import { prisma } from './prisma'
|
||||
import { AI_MODEL_VERIFICATION_RULE_VERSION, createVerificationChallenge, readVerificationChallenge, validateAiModelVerificationOutput, verificationReferenceImage } from './ai-model-verification-output'
|
||||
import { AI_MODEL_VERIFICATION_RULE_VERSION, createVerificationChallenge, readVerificationChallenge, validateAiModelCombinedVerificationOutput, validateAiModelVerificationOutput, verificationReferenceImage, type AiModelTextCheck } from './ai-model-verification-output'
|
||||
export { validateAiModelVerificationOutput } from './ai-model-verification-output'
|
||||
import { detailTransactionLock } from './detail-generator/repository'
|
||||
import type { AiModelDatabase } from './ai-model-repository'
|
||||
|
||||
const requestSchema = z.object({ stage: z.enum(AI_MODEL_STAGES), revision: z.number().int().positive(), idempotencyKey: z.string().uuid() }).strict()
|
||||
const requestSchema = z.object({ stage: z.enum(AI_MODEL_TEST_PURPOSES), revision: z.number().int().positive(), idempotencyKey: z.string().uuid() }).strict()
|
||||
function safeId(value: unknown, secret: string): string | null { return typeof value === 'string' && /^[\w.-]{1,160}$/.test(value) && !value.includes(secret) ? value : null }
|
||||
async function assertExecutionSlot(db: AiModelDatabase, actorId: string, resumeId: string | null = null): Promise<void> {
|
||||
await detailTransactionLock(db, 'detail-generator-execution')
|
||||
const rows = await db.$queryRawUnsafe<Array<{ count: number; own: number }>>(`SELECT COUNT(*)::int AS count,COUNT(*) FILTER (WHERE actor_id=$1)::int AS own FROM (
|
||||
SELECT actor_id FROM ai_model_verifications WHERE ($2::text IS NULL OR id<>$2) AND status IN ('running','unknown')
|
||||
UNION ALL SELECT owner_id AS actor_id FROM detail_generator_jobs WHERE status='running' AND lease_expires_at>NOW()
|
||||
SELECT DISTINCT COALESCE(evidence->>'executionGroupId',id) AS execution_id,actor_id FROM ai_model_verifications WHERE ($2::text IS NULL OR COALESCE(evidence->>'executionGroupId',id)<>$2) AND status IN ('running','unknown')
|
||||
UNION ALL SELECT id AS execution_id,owner_id AS actor_id FROM detail_generator_jobs WHERE status='running' AND lease_expires_at>NOW()
|
||||
OR status='queued' AND checkpoint->>'dispatchPending'='true' AND COALESCE(checkpoint #>> '{upstreamTask,id}', '') ~ '^[a-zA-Z0-9][a-zA-Z0-9_.:-]{0,199}$'
|
||||
) AS active_executions`, actorId, resumeId)
|
||||
if (rows[0]?.count >= 2 || rows[0]?.own >= 1) throw new AiModelError('已有生成或模型测试正在执行，请等待完成', 409, 'verification_busy')
|
||||
}
|
||||
async function executeVerification(row: AiModelVerificationRow, userId: string, providerId: string, resume: boolean): Promise<void> {
|
||||
type VerificationContext = Awaited<ReturnType<typeof resolveAiModelVerification>>
|
||||
function assertCombinedContexts(contexts: VerificationContext[]): void {
|
||||
const transport = (context: VerificationContext) => JSON.stringify({ ...context.protocol, supportsImageInput: undefined })
|
||||
if (contexts.length !== 2 || !contexts[0].protocol.supportsImageInput || transport(contexts[0]) !== transport(contexts[1])) throw new AiModelError('当前模型不能在同一路由联合验证分析和文案', 400, 'verification_combined_unsupported')
|
||||
}
|
||||
async function executeVerification(rows: AiModelVerificationRow[], userId: string, providerId: string, resume: boolean): Promise<void> {
|
||||
const row = rows[0], combined = rows.length === 2
|
||||
let sent = false, resultReceived = false, apiKey = '', knownTaskId = row.taskId
|
||||
const beforeRequest = async () => {
|
||||
await assertAiModelAdmin(userId)
|
||||
const current = await resolveAiModelVerification(providerId, row.modelKey, row.stage)
|
||||
if (current.contextHash !== row.contextHash) throw new AiModelError('连接或路由已改变，测试已停止', 409, 'verification_context_changed')
|
||||
const pending = await prisma.$queryRawUnsafe<Array<{ id: string }>>("SELECT id FROM ai_model_verifications WHERE id=$1 AND status='running' AND deadline_at>NOW()", row.id)
|
||||
if (!pending[0]) throw new AiModelError('原请求已暂停或核查结束，不能继续调用', 409, 'verification_closed')
|
||||
const current = await Promise.all(rows.map(async item => {
|
||||
const context = await resolveAiModelVerification(providerId, item.modelKey, item.stage)
|
||||
if (context.contextHash !== item.contextHash) throw new AiModelError('连接或路由已改变，测试已停止', 409, 'verification_context_changed')
|
||||
const pending = await prisma.$queryRawUnsafe<Array<{ id: string }>>("SELECT id FROM ai_model_verifications WHERE id=$1 AND status='running' AND deadline_at>NOW()", item.id)
|
||||
if (!pending[0]) throw new AiModelError('原请求已暂停或核查结束，不能继续调用', 409, 'verification_closed')
|
||||
return context
|
||||
}))
|
||||
if (combined) assertCombinedContexts(current)
|
||||
sent = true
|
||||
}
|
||||
try {
|
||||
const context = await resolveAiModelVerification(providerId, row.modelKey, row.stage)
|
||||
const contexts = await Promise.all(rows.map(async item => {
|
||||
const context = await resolveAiModelVerification(providerId, item.modelKey, item.stage)
|
||||
if (context.contextHash !== item.contextHash) throw new AiModelError('验证对应的连接或路由已改变，不能继续此任务', 409, 'verification_context_changed')
|
||||
return context
|
||||
}))
|
||||
if (combined) assertCombinedContexts(contexts)
|
||||
const context = contexts[0]
|
||||
apiKey = context.apiKey
|
||||
if (context.contextHash !== row.contextHash) throw new AiModelError('验证对应的连接或路由已改变，不能继续此任务', 409, 'verification_context_changed')
|
||||
const challenge = readVerificationChallenge(row.evidence?.challenge)
|
||||
const prompt = row.stage === 'analysis' ? '观察图片：左上方方块与右下方方块分别是什么颜色？只输出 JSON：{"topLeft":"英文小写颜色","bottomRight":"英文小写颜色"}。不要根据文字猜测，必须读取图片。'
|
||||
const prompt = combined ? '一次完成两项独立任务。analysis：必须读取参考图，识别左上与右下方块的颜色，以英文小写填写。copy：真实产品资料为型号TEST、颜色蓝色，图片只是读图测试图，不是该产品外观；只依据这两条真实资料写一屏中文详情页，不编造材质、尺寸、性能或认证。严格只输出一个JSON对象：{"analysis":{"topLeft":"英文小写颜色","bottomRight":"英文小写颜色"},"copy":{"sellingPoints":["卖点"],"warnings":[],"screens":[{"title":"标题","body":"正文","imagePrompt":"无文字的场景描述"}]}}。'
|
||||
: row.stage === 'analysis' ? '观察图片：左上方方块与右下方方块分别是什么颜色？只输出 JSON：{"topLeft":"英文小写颜色","bottomRight":"英文小写颜色"}。不要根据文字猜测，必须读取图片。'
|
||||
: row.stage === 'copy' ? '真实产品资料：型号TEST，颜色蓝色。不要编造其他规格。写一屏中文详情页，严格只输出 JSON：{"sellingPoints":["卖点"],"warnings":[],"screens":[{"title":"标题","body":"正文","imagePrompt":"无文字的场景描述"}]}。'
|
||||
: '依据参考图片生成一张简洁的产品展示图，保留两个方块的颜色与相对位置，白色背景，不要加入文字或标志。'
|
||||
const result = await callAiModelProtocol({ baseUrl: context.provider.baseUrl, apiKey: context.apiKey, modelId: context.model.id, protocol: context.protocol,
|
||||
prompt, ...(row.stage !== 'copy' ? { images: [{ buffer: await verificationReferenceImage(challenge), mimeType: 'image/png' }] } : {}), maxOutputTokens: row.stage === 'copy' ? 1200 : 1024,
|
||||
prompt, ...(row.stage !== 'copy' ? { images: [{ buffer: await verificationReferenceImage(challenge), mimeType: 'image/png' }] } : {}), maxOutputTokens: combined ? 1800 : row.stage === 'copy' ? 1200 : 1024,
|
||||
idempotencyKey: row.idempotencyKey, beforeRequest, ...(resume ? { resumeTaskId: row.taskId! } : {}),
|
||||
onTaskSubmitted: async taskId => {
|
||||
if (!safeId(taskId, context.apiKey)) throw new AiModelError('上游任务编号无效', 502, 'verification_task_invalid')
|
||||
knownTaskId = taskId
|
||||
const updated = await prisma.$executeRawUnsafe(`UPDATE ai_model_verifications SET task_id=$2 WHERE id=$1 AND status='running' AND (task_id IS NULL OR task_id=$2)`, row.id, taskId)
|
||||
if (updated !== 1) throw new AiModelError('任务检查点未保存，已暂停后续请求', 409, 'verification_checkpoint_failed')
|
||||
await aiModelTransaction(async db => {
|
||||
for (const item of rows) {
|
||||
const updated = await db.$executeRawUnsafe(`UPDATE ai_model_verifications SET task_id=$2 WHERE id=$1 AND status='running' AND (task_id IS NULL OR task_id=$2)`, item.id, taskId)
|
||||
if (updated !== 1) throw new AiModelError('任务检查点未保存，已暂停后续请求', 409, 'verification_checkpoint_failed')
|
||||
}
|
||||
})
|
||||
} })
|
||||
resultReceived = true // The native protocol has returned a terminal, decoded result.
|
||||
const evidence = await validateAiModelVerificationOutput(row.stage, result, challenge)
|
||||
const checks: Partial<Record<AiModelVerificationRow['stage'], AiModelTextCheck>> = combined ? await validateAiModelCombinedVerificationOutput(result, challenge)
|
||||
: { [row.stage]: { status: 'passed', message: '实际接口测试通过', evidence: await validateAiModelVerificationOutput(row.stage, result, challenge) } }
|
||||
await assertAiModelAdmin(userId)
|
||||
await aiModelTransaction(async db => {
|
||||
await assertAiModelAdminForWrite(userId, db)
|
||||
const current = await resolveAiModelVerification(providerId, row.modelKey, row.stage)
|
||||
if (current.contextHash !== row.contextHash) throw new AiModelError('验证期间连接或路由已改变', 409, 'verification_context_changed')
|
||||
const updated = await db.$executeRawUnsafe(`UPDATE ai_model_verifications SET status='passed',request_id=$2,message='实际接口测试通过',evidence=evidence || $3::jsonb,finished_at=NOW() WHERE id=$1 AND status='running' AND deadline_at>NOW()`, row.id, safeId(result.requestId, context.apiKey), JSON.stringify({ ...evidence, ruleVersion: AI_MODEL_VERIFICATION_RULE_VERSION, adapterVersion: AI_MODEL_ADAPTER_VERSION, protocol: context.protocol.id, scopeId: context.protocol.scopeId, routeId: context.protocol.routeId, operationId: context.protocol.operationId }))
|
||||
if (updated === 1) await aiModelAudit(db, userId, 'model.verification-passed', providerId, { verificationId: row.id, modelKey: row.modelKey, stage: row.stage, evidence })
|
||||
for (const item of rows) {
|
||||
const current = await resolveAiModelVerification(providerId, item.modelKey, item.stage)
|
||||
if (current.contextHash !== item.contextHash) throw new AiModelError('验证期间连接或路由已改变', 409, 'verification_context_changed')
|
||||
}
|
||||
const changes: number[] = []
|
||||
for (let index = 0; index < rows.length; index++) {
|
||||
const item = rows[index]
|
||||
const check = checks[item.stage]!, protocol = contexts[index].protocol
|
||||
changes.push(await db.$executeRawUnsafe(`UPDATE ai_model_verifications SET status='${check.status}',request_id=$2,message=$4,evidence=evidence || $3::jsonb,finished_at=NOW() WHERE id=$1 AND status='running' AND deadline_at>NOW()`, item.id, safeId(result.requestId, context.apiKey), JSON.stringify({ ...check.evidence, ruleVersion: AI_MODEL_VERIFICATION_RULE_VERSION, adapterVersion: AI_MODEL_ADAPTER_VERSION, protocol: protocol.id, scopeId: protocol.scopeId, routeId: protocol.routeId, operationId: protocol.operationId }), check.message))
|
||||
}
|
||||
if (combined && changes.some(value => value !== changes[0])) throw new AiModelError('联合测试记录已改变，已停止写入', 409, 'verification_checkpoint_failed')
|
||||
for (let index = 0; index < rows.length; index++) if (changes[index] === 1) {
|
||||
const item = rows[index]
|
||||
const check = checks[item.stage]!
|
||||
await aiModelAudit(db, userId, `model.verification-${check.status}`, providerId, { verificationId: item.id, modelKey: item.modelKey, stage: item.stage, evidence: check.evidence, ...(combined ? { executionGroupId: verificationExecutionId(item) } : {}) })
|
||||
}
|
||||
})
|
||||
} catch (error) {
|
||||
const outcome = error as { outcomeUnknown?: boolean; taskId?: string; requestId?: string }
|
||||
const unknown = !resultReceived && (typeof outcome.outcomeUnknown === 'boolean' ? (sent || Boolean(knownTaskId)) && outcome.outcomeUnknown : Boolean(knownTaskId) || sent && !(error instanceof AiModelError))
|
||||
const message = unknown ? '请求结果尚未确认，请核查上游记录；禁止重复提交' : error instanceof AiModelError ? error.message : '模型测试未通过'
|
||||
await aiModelTransaction(async db => {
|
||||
const updated = await db.$executeRawUnsafe(`UPDATE ai_model_verifications SET status=$2,request_id=$3,task_id=COALESCE(task_id,$4),message=$5,finished_at=NOW() WHERE id=$1 AND status='running'`, row.id, unknown ? 'unknown' : 'failed', safeId(outcome.requestId, apiKey), safeId(outcome.taskId || knownTaskId, apiKey), message)
|
||||
if (updated === 1) await aiModelAudit(db, userId, unknown ? 'model.verification-unknown' : 'model.verification-failed', providerId, { verificationId: row.id, modelKey: row.modelKey, stage: row.stage })
|
||||
for (const item of rows) {
|
||||
const updated = await db.$executeRawUnsafe(`UPDATE ai_model_verifications SET status=$2,request_id=$3,task_id=COALESCE(task_id,$4),message=$5,finished_at=NOW() WHERE id=$1 AND status='running'`, item.id, unknown ? 'unknown' : 'failed', safeId(outcome.requestId, apiKey), safeId(outcome.taskId || knownTaskId, apiKey), message)
|
||||
if (updated === 1) await aiModelAudit(db, userId, unknown ? 'model.verification-unknown' : 'model.verification-failed', providerId, { verificationId: item.id, modelKey: item.modelKey, stage: item.stage, ...(combined ? { executionGroupId: verificationExecutionId(item) } : {}) })
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@ -75,20 +111,31 @@ export async function verifyAiModel(userId: string, providerId: string, modelKey
|
||||
const parsed = requestSchema.safeParse(body)
|
||||
if (!parsed.success) throw new AiModelError('测试参数无效')
|
||||
await assertAiModelAdmin(userId)
|
||||
const input = parsed.data, context = await resolveAiModelVerification(providerId, modelKey, input.stage, input.revision)
|
||||
const input = parsed.data, stages = aiModelTestStages(input.stage)
|
||||
const contexts = await Promise.all(stages.map(stage => resolveAiModelVerification(providerId, modelKey, stage, input.revision)))
|
||||
if (input.stage === 'analysis-copy') assertCombinedContexts(contexts)
|
||||
const created = await aiModelTransaction(async db => {
|
||||
await assertAiModelAdminForWrite(userId, db)
|
||||
const prior = await db.$queryRawUnsafe<AiModelVerificationRow[]>(`SELECT ${AI_MODEL_VERIFICATION_COLUMNS} FROM ai_model_verifications WHERE model_key=$1 AND stage=$2 AND idempotency_key=$3`, modelKey, input.stage, input.idempotencyKey)
|
||||
if (prior[0]) return null // A repeated HTTP request never repeats a model call.
|
||||
const closed = await db.$queryRawUnsafe<Array<{ id: string }>>(`SELECT id FROM ai_model_audit WHERE action='model.verification-intake-cancelled' AND details->>'modelKey'=$1 AND details->>'stage'=$2 AND details->>'idempotencyKey'=$3`, modelKey, input.stage, input.idempotencyKey)
|
||||
const prior = await db.$queryRawUnsafe<AiModelVerificationRow[]>(`SELECT ${AI_MODEL_VERIFICATION_COLUMNS} FROM ai_model_verifications WHERE model_key=$1 AND idempotency_key=$2`, modelKey, input.idempotencyKey)
|
||||
if (prior[0]) {
|
||||
if ((prior[0].evidence?.purpose || prior[0].stage) !== input.stage) throw new AiModelError('该请求编号已用于其他测试，请创建新请求', 409, 'verification_idempotency_conflict')
|
||||
return null // A repeated HTTP request never repeats a model call.
|
||||
}
|
||||
const closed = await db.$queryRawUnsafe<Array<{ id: string }>>(`SELECT id FROM ai_model_audit WHERE action='model.verification-intake-cancelled' AND details->>'modelKey'=$1 AND details->>'idempotencyKey'=$2`, modelKey, input.idempotencyKey)
|
||||
if (closed[0]) throw new AiModelError('原提交已关闭，请重新确认测试并创建新请求', 409, 'verification_submission_closed')
|
||||
await db.$executeRawUnsafe(`UPDATE ai_model_verifications SET status='unknown',message='进程或请求中断，结果需核查',finished_at=NOW() WHERE status='running' AND deadline_at<=NOW()`)
|
||||
const active = await db.$queryRawUnsafe<Array<{ count: number }>>(`SELECT COUNT(*)::int AS count FROM ai_model_verifications WHERE model_key=$1 AND status IN ('running','unknown')`, modelKey)
|
||||
if (active[0]?.count) throw new AiModelError('该模型存在进行中或结果未知的请求，不能重复测试；请先核查原请求', 409, 'verification_unresolved')
|
||||
await assertExecutionSlot(db, userId)
|
||||
const rows = await db.$queryRawUnsafe<AiModelVerificationRow[]>(`INSERT INTO ai_model_verifications (id,model_key,stage,context_hash,idempotency_key,actor_id,status,evidence) VALUES ($1,$2,$3,$4,$5,$6,'running',$7::jsonb) RETURNING ${AI_MODEL_VERIFICATION_COLUMNS}`, randomUUID(), modelKey, input.stage, context.contextHash, input.idempotencyKey, userId, JSON.stringify({ challenge: createVerificationChallenge() }))
|
||||
await aiModelAudit(db, userId, 'model.verification-start', providerId, { verificationId: rows[0].id, modelKey, stage: input.stage })
|
||||
return rows[0]
|
||||
const evidence = { challenge: createVerificationChallenge(), purpose: input.stage, ...(input.stage === 'analysis-copy' ? { executionGroupId: randomUUID() } : {}) }
|
||||
const rows: AiModelVerificationRow[] = []
|
||||
for (let index = 0; index < stages.length; index++) {
|
||||
const stage = stages[index]
|
||||
const inserted = await db.$queryRawUnsafe<AiModelVerificationRow[]>(`INSERT INTO ai_model_verifications (id,model_key,stage,context_hash,idempotency_key,actor_id,status,evidence) VALUES ($1,$2,$3,$4,$5,$6,'running',$7::jsonb) RETURNING ${AI_MODEL_VERIFICATION_COLUMNS}`, randomUUID(), modelKey, stage, contexts[index].contextHash, input.idempotencyKey, userId, JSON.stringify(evidence))
|
||||
rows.push(inserted[0])
|
||||
}
|
||||
await aiModelAudit(db, userId, 'model.verification-start', providerId, { verificationId: rows[0].id, modelKey, stage: input.stage, verificationIds: rows.map(item => item.id), ...(input.stage === 'analysis-copy' ? { executionGroupId: verificationExecutionId(rows[0]) } : {}) })
|
||||
return rows
|
||||
})
|
||||
if (created) await executeVerification(created, userId, providerId, false)
|
||||
return getAiModelAdminState()
|
||||
@ -104,12 +151,18 @@ export async function refreshAiModelVerification(userId: string, providerId: str
|
||||
const current = rows[0]
|
||||
if (!current?.taskId || !['unknown','running'].includes(current.status)) throw new AiModelError('没有可以查询的上游任务', 409, 'verification_not_resumable')
|
||||
if (current.status === 'running' && current.deadlineAt.getTime() > Date.now()) throw new AiModelError('测试仍在执行，请等待', 409, 'verification_busy')
|
||||
const context = await resolveAiModelVerification(providerId, current.modelKey, current.stage)
|
||||
if (context.contextHash !== current.contextHash) throw new AiModelError('原连接或路由已改变，请在上游核查原任务', 409, 'verification_context_changed')
|
||||
await assertExecutionSlot(db, userId, current.id)
|
||||
await db.$executeRawUnsafe(`UPDATE ai_model_verifications SET status='running',actor_id=$2,deadline_at=NOW()+INTERVAL '10 minutes',finished_at=NULL WHERE id=$1`, current.id, userId)
|
||||
const execution = await lockAiModelVerificationExecution(db, current)
|
||||
const contexts = await Promise.all(execution.map(async item => {
|
||||
if (!['unknown', 'running'].includes(item.status) || item.status === 'running' && item.deadlineAt.getTime() > Date.now()) throw new AiModelError('联合测试仍在执行或已结束，请刷新状态', 409, 'verification_busy')
|
||||
const context = await resolveAiModelVerification(providerId, item.modelKey, item.stage)
|
||||
if (context.contextHash !== item.contextHash) throw new AiModelError('原连接或路由已改变，请在上游核查原任务', 409, 'verification_context_changed')
|
||||
return context
|
||||
}))
|
||||
if (execution.length === 2) assertCombinedContexts(contexts)
|
||||
await assertExecutionSlot(db, userId, verificationExecutionId(current))
|
||||
for (const item of execution) await db.$executeRawUnsafe(`UPDATE ai_model_verifications SET status='running',actor_id=$2,deadline_at=NOW()+INTERVAL '10 minutes',finished_at=NULL WHERE id=$1`, item.id, userId)
|
||||
await aiModelAudit(db, userId, 'model.verification-query', providerId, { verificationId: current.id, modelKey: current.modelKey, stage: current.stage })
|
||||
return current
|
||||
return execution
|
||||
})
|
||||
await executeVerification(row, userId, providerId, true)
|
||||
return getAiModelAdminState()
|
||||
|
||||
@ -162,7 +162,7 @@ export async function claimDetailJob(): Promise<StoredDetailJob | null> {
|
||||
await tx.$executeRawUnsafe(`UPDATE detail_generator_jobs SET status = CASE WHEN ${unresolved} THEN 'unknown' ELSE 'queued' END, error = CASE WHEN ${unresolved} THEN '执行中断，模型请求结果未知；请核查服务商记录，勿重复提交' ELSE NULL END, lease_token = NULL, lease_expires_at = NULL, updated_at = NOW() WHERE status = 'running' AND lease_expires_at < NOW()`)
|
||||
const pendingTask = `checkpoint->>'dispatchPending' = 'true' AND COALESCE(checkpoint #>> '{upstreamTask,id}', '') ~ '^[a-zA-Z0-9][a-zA-Z0-9_.:-]{0,199}$'`
|
||||
const activeVerification = `(status IN ('running','unknown'))`
|
||||
const count = await tx.$queryRawUnsafe<Row[]>(`SELECT ((SELECT COUNT(*) FROM detail_generator_jobs WHERE (status='running' AND lease_expires_at>=NOW()) OR (status='queued' AND ${pendingTask})) + (SELECT COUNT(*) FROM ai_model_verifications WHERE ${activeVerification}))::int AS count`)
|
||||
const count = await tx.$queryRawUnsafe<Row[]>(`SELECT ((SELECT COUNT(*) FROM detail_generator_jobs WHERE (status='running' AND lease_expires_at>=NOW()) OR (status='queued' AND ${pendingTask})) + (SELECT COUNT(DISTINCT COALESCE(evidence->>'executionGroupId',id)) FROM ai_model_verifications WHERE ${activeVerification}))::int AS count`)
|
||||
// A queued known upstream task already owns its slot. Let it poll even when both slots are occupied.
|
||||
const candidates = await tx.$queryRawUnsafe<Row[]>(`SELECT ${jobColumns} FROM detail_generator_jobs j WHERE status = 'queued' AND ($1::boolean OR (${pendingTask})) AND (CASE WHEN jsonb_typeof(checkpoint->'pollNotBefore') = 'number' THEN (checkpoint->>'pollNotBefore')::numeric ELSE 0 END) <= EXTRACT(EPOCH FROM NOW()) * 1000 AND EXISTS (SELECT 1 FROM detail_generator_projects p WHERE p.id = j.project_id AND p.deleted_at IS NULL) AND NOT EXISTS (SELECT 1 FROM detail_generator_jobs other WHERE other.owner_id = j.owner_id AND other.status = 'running' AND other.lease_expires_at >= NOW()) AND NOT EXISTS (SELECT 1 FROM ai_model_verifications v WHERE v.actor_id=j.owner_id AND ${activeVerification}) ORDER BY created_at LIMIT 1 FOR UPDATE SKIP LOCKED`, Number(count[0]?.count) < 2)
|
||||
if (!candidates[0]) return null
|
||||
|
||||
104
scripts/verify-ai-model-combined.ts
Normal file
104
scripts/verify-ai-model-combined.ts
Normal file
@ -0,0 +1,104 @@
|
||||
/** Native HTTP acceptance; all synthetic fixtures are confined to the named local QA database. */
|
||||
import assert from 'node:assert/strict'
|
||||
import { randomUUID } from 'node:crypto'
|
||||
import { mkdir, readFile, writeFile } from 'node:fs/promises'
|
||||
import { PrismaClient } from '@prisma/client'
|
||||
import { getAiModelStageProtocol } from '../lib/ai-model-capabilities'
|
||||
import { aiModelVerificationContext } from '../lib/ai-model-verification-state'
|
||||
import type { AiModelAdminState, AiModelStage } from '../lib/ai-model-types'
|
||||
const database = new URL(process.env.DATABASE_URL || '')
|
||||
assert.equal(database.hostname, '127.0.0.1'); assert.equal(database.port, '55441'); assert.equal(database.pathname, '/ai_model_test')
|
||||
const base = 'http://localhost:3125', output = 'D:/AI Code/gongxiangp2/output/ai-combined-20261014', phase = process.argv[2] || 'plan'
|
||||
const prisma = new PrismaClient({ log: [] })
|
||||
async function save(name: string, data: unknown) { await mkdir(output, { recursive: true }); await writeFile(`${output}/${name}.json`, JSON.stringify(data, null, 2)) }
|
||||
async function session() {
|
||||
const jar = new Map<string, string>()
|
||||
const store = (response: Response) => { for (const cookie of response.headers.getSetCookie()) { const value = cookie.split(';')[0], at = value.indexOf('='); jar.set(value.slice(0, at), value.slice(at + 1)) } }
|
||||
const cookie = () => Array.from(jar).map(([key, value]) => `${key}=${value}`).join('; ')
|
||||
const csrf = await fetch(base + '/api/auth/csrf'); store(csrf)
|
||||
const { csrfToken } = await csrf.json()
|
||||
const signed = await fetch(base + '/api/auth/callback/credentials', { method: 'POST', redirect: 'manual', headers: { 'Content-Type': 'application/x-www-form-urlencoded', Origin: base, Cookie: cookie() }, body: new URLSearchParams({ csrfToken, email: 'ai-admin-1@example.invalid', password: 'AiAdminLocalTest-20261012', callbackUrl: base + '/admin', json: 'true' }) })
|
||||
store(signed); assert.equal(signed.status, 200); return cookie()
|
||||
}
|
||||
async function main() {
|
||||
const cookie = await session(), actor = await prisma.user.findUniqueOrThrow({ where: { email: 'ai-admin-1@example.invalid' } })
|
||||
assert.ok(actor.isSuperAdmin && actor.isActive)
|
||||
async function request(path = '', method = 'GET', body?: unknown) {
|
||||
const response = await fetch(base + '/api/admin/ai-models' + path, { method, headers: { Cookie: cookie, 'Content-Type': 'application/json' }, ...(body === undefined ? {} : { body: JSON.stringify(body) }), signal: AbortSignal.timeout(120000) })
|
||||
const text = await response.text(); assert.ok(!process.env.DETAIL_GENERATOR_API_KEY || !text.includes(process.env.DETAIL_GENERATOR_API_KEY), 'Credential leaked')
|
||||
return { status: response.status, data: JSON.parse(text) }
|
||||
}
|
||||
const state = (await request()).data as AiModelAdminState
|
||||
const provider = state.providers.find(value => value.credentialSource === 'environment')!; assert.ok(provider)
|
||||
const model = provider.models.find(value => value.id === 'gemini-3-flash')!; assert.ok(model?.capabilities.combinedTextTest)
|
||||
const startCount = async () => (await prisma.$queryRawUnsafe<Array<{ count: number }>>(`SELECT COUNT(*)::int AS count FROM ai_model_audit WHERE action='model.verification-start' AND details->>'modelKey'=$1`, model.key))[0].count
|
||||
if (phase === 'plan') {
|
||||
const plan = { providerId: provider.id, modelKey: model.key, actorId: actor.id, revision: provider.revision, model: model.id, starts: await startCount(), createdAt: new Date().toISOString() }
|
||||
await save('real-plan', plan); console.log(JSON.stringify(plan)); return
|
||||
}
|
||||
if (phase === 'receipt') {
|
||||
const plan = JSON.parse(await readFile(`${output}/real-plan.json`, 'utf8'))
|
||||
assert.equal(model.key, plan.modelKey)
|
||||
const rows = await prisma.$queryRawUnsafe<Array<{ id: string; stage: AiModelStage; status: string; idempotency_key: string; request_id: string; evidence: Record<string, any> }>>(`SELECT id,stage,status,idempotency_key,request_id,evidence FROM ai_model_verifications WHERE model_key=$1 AND actor_id=$2 AND started_at>=$3::timestamptz ORDER BY stage`, model.key, actor.id, plan.createdAt)
|
||||
assert.equal(rows.length, 2); assert.deepEqual(rows.map(value => value.stage), ['analysis', 'copy'])
|
||||
assert.ok(rows.every(value => value.status === 'passed' && value.evidence.purpose === 'analysis-copy' && value.evidence.ruleVersion === 2))
|
||||
assert.equal(rows[0].idempotency_key, rows[1].idempotency_key); assert.equal(rows[0].request_id, rows[1].request_id)
|
||||
assert.equal(rows[0].evidence.executionGroupId, rows[1].evidence.executionGroupId)
|
||||
assert.equal(await startCount() - plan.starts, 1)
|
||||
const repeated = await request(`/providers/${provider.id}/models/${model.key}/verify`, 'POST', { stage: 'analysis-copy', revision: provider.revision, idempotencyKey: rows[0].idempotency_key })
|
||||
assert.equal(repeated.status, 200); assert.equal(await startCount() - plan.starts, 1)
|
||||
const evidence = { nativeHttp: true, productionTouched: false, model: model.id, upstreamCallsAdmitted: 1, passedStages: rows.map(value => value.stage), sameUpstreamRequestId: rows[0].request_id, sameExecutionGroupId: rows[0].evidence.executionGroupId, verificationIds: rows.map(value => value.id), repeatedSubmissionAddedCalls: 0, records: rows }
|
||||
await save('real-acceptance', evidence); console.log(JSON.stringify({ ...evidence, records: undefined })); return
|
||||
}
|
||||
if (phase === 'recovery') {
|
||||
const source = await prisma.$queryRawUnsafe<Array<{ capability_metadata: unknown }>>('SELECT capability_metadata FROM ai_model_catalog WHERE id=$1', model.key)
|
||||
const metadata = source[0].capability_metadata, modelKey = randomUUID(), modelId = `qa-joint-${randomUUID()}`, key = randomUUID(), groupId = randomUUID(), ids = [randomUUID(), randomUUID()]
|
||||
await prisma.$transaction(async db => {
|
||||
await db.$executeRawUnsafe(`INSERT INTO ai_model_catalog (id,provider_id,model_id,enabled,stages,present,capability_metadata) VALUES ($1,$2,$3,FALSE,ARRAY[]::text[],TRUE,$4::jsonb)`, modelKey, provider.id, modelId, JSON.stringify(metadata))
|
||||
for (const [index, stage] of (['analysis', 'copy'] as const).entries()) {
|
||||
const protocol = getAiModelStageProtocol(metadata, stage)!; assert.ok(protocol)
|
||||
const contextHash = aiModelVerificationContext(provider.baseUrl, modelId, metadata, process.env.DETAIL_GENERATOR_API_KEY!, protocol)
|
||||
await db.$executeRawUnsafe(`INSERT INTO ai_model_verifications (id,model_key,stage,context_hash,idempotency_key,actor_id,status,evidence) VALUES ($1,$2,$3,$4,$5,$6,'unknown',$7::jsonb)`, ids[index], modelKey, stage, contextHash, key, actor.id, JSON.stringify({ purpose: 'analysis-copy', executionGroupId: groupId, challenge: { topLeft: 'red', bottomRight: 'blue' }, fixture: true }))
|
||||
}
|
||||
})
|
||||
await save('fixture', { modelKey, modelId, ids, providerId: provider.id, actorId: actor.id })
|
||||
const before = await startCount()
|
||||
const count = (await prisma.$queryRawUnsafe<Array<{ physical: number; executions: number }>>(`SELECT COUNT(*)::int AS physical,COUNT(DISTINCT COALESCE(evidence->>'executionGroupId',id))::int AS executions FROM ai_model_verifications WHERE model_key=$1 AND status IN ('running','unknown')`, modelKey))[0]
|
||||
assert.deepEqual(count, { physical: 2, executions: 1 })
|
||||
const busy = await request(`/providers/${provider.id}/models/${model.key}/verify`, 'POST', { stage: 'analysis-copy', revision: provider.revision, idempotencyKey: randomUUID() }); assert.equal(busy.data.code, 'verification_busy')
|
||||
const checked = await request(`/providers/${provider.id}/models/${modelKey}/submission`, 'POST', { stage: 'analysis-copy', idempotencyKey: key }); assert.equal(checked.status, 200); assert.equal(checked.data.submissionCheck.received, true)
|
||||
const proof = { outcome: 'not_executed', reference: 'qa-joint-fixture-no-upstream-call', note: '隔离测试注入的记录，没有调用上游；核查后关闭同一次请求的两条用途记录。', confirmed: true }
|
||||
const invalid = await request(`/providers/${provider.id}/verifications/${ids[1]}/reconcile`, 'POST', { ...proof, confirmed: false }); assert.equal(invalid.status, 400)
|
||||
const ended = await request(`/providers/${provider.id}/verifications/${ids[1]}/reconcile`, 'POST', proof); assert.equal(ended.status, 200)
|
||||
const records = await prisma.$queryRawUnsafe<Array<{ status: string; evidence: Record<string, any> }>>('SELECT status,evidence FROM ai_model_verifications WHERE model_key=$1', modelKey)
|
||||
assert.equal(records.length, 2); assert.ok(records.every(value => value.status === 'failed' && value.evidence.reconciliation.reference === proof.reference))
|
||||
const absent = { stage: 'analysis-copy', idempotencyKey: randomUUID() }
|
||||
const closed = await request(`/providers/${provider.id}/models/${modelKey}/submission`, 'POST', absent); assert.equal(closed.data.submissionCheck.received, false)
|
||||
const late = await request(`/providers/${provider.id}/models/${modelKey}/verify`, 'POST', { ...absent, revision: provider.revision }); assert.equal(late.data.code, 'verification_submission_closed')
|
||||
assert.equal(await startCount(), before)
|
||||
const analysisId = randomUUID(), copyId = randomUUID(), partialKey = randomUUID(), partialGroup = randomUUID()
|
||||
for (const [stage, id] of [['analysis', analysisId], ['copy', copyId]] as const) {
|
||||
const protocol = getAiModelStageProtocol(metadata, stage)!, contextHash = aiModelVerificationContext(provider.baseUrl, modelId, metadata, process.env.DETAIL_GENERATOR_API_KEY!, protocol)
|
||||
await prisma.$executeRawUnsafe(`INSERT INTO ai_model_verifications (id,model_key,stage,context_hash,idempotency_key,actor_id,status,evidence,message,finished_at) VALUES ($1,$2,$3,$4,$5,$6,$7,$8::jsonb,$9,NOW())`, id, modelKey, stage, contextHash, partialKey, actor.id, stage === 'analysis' ? 'passed' : 'failed', JSON.stringify({ purpose: 'analysis-copy', executionGroupId: partialGroup, ruleVersion: 2, fixture: true, ...(stage === 'analysis' ? { imageInputVerified: true } : {}) }), '隔离测试：分析通过，文案结构未通过')
|
||||
}
|
||||
async function configure(enabled: boolean, stages: AiModelStage[]) {
|
||||
const fresh = (await request()).data as AiModelAdminState, connection = fresh.providers.find(value => value.id === provider.id)!
|
||||
return request('/configuration', 'PUT', { revision: fresh.revision, defaults: fresh.defaults, providers: [{ id: provider.id, revision: connection.revision, models: [{ key: modelKey, enabled, stages }] }] })
|
||||
}
|
||||
const denied = await configure(true, ['copy']); assert.equal(denied.data.code, 'model_verification_required')
|
||||
assert.equal((await configure(true, ['analysis'])).status, 200)
|
||||
assert.equal((await configure(false, [])).status, 200)
|
||||
const evidence = { nativeHttp: true, productionTouched: false, paidCalls: 0, groupRecords: 2, executionSlots: 1, jointSubmissionReceived: true, ownerBusy: 409, invalidRecovery: 400, bothUnknownRecordsClosed: true, lateSubmission: 409, failedPurposeCannotEnable: 400, onlyPassedPurposeCanEnable: true, uiPartialFixtureModel: modelId }
|
||||
await save('native-acceptance', evidence); console.log(JSON.stringify(evidence)); return
|
||||
}
|
||||
assert.equal(phase, 'cleanup')
|
||||
const receipt = JSON.parse(await readFile(`${output}/fixture.json`, 'utf8'))
|
||||
const fixture = await prisma.$queryRawUnsafe<Array<{ model_id: string; enabled: boolean }>>('SELECT model_id,enabled FROM ai_model_catalog WHERE id=$1 AND provider_id=$2', receipt.modelKey, receipt.providerId)
|
||||
assert.equal(fixture[0]?.model_id, receipt.modelId); assert.ok(receipt.modelId.startsWith('qa-joint-')); assert.equal(fixture[0].enabled, false)
|
||||
await prisma.$transaction(async db => {
|
||||
await db.$executeRawUnsafe('DELETE FROM ai_model_verifications WHERE model_key=$1 AND actor_id=$2', receipt.modelKey, receipt.actorId)
|
||||
await db.$executeRawUnsafe('DELETE FROM ai_model_catalog WHERE id=$1 AND model_id=$2 AND enabled=FALSE', receipt.modelKey, receipt.modelId)
|
||||
})
|
||||
await save('fixture-cleanup', { modelKey: receipt.modelKey, productionTouched: false }); console.log('Named QA fixture removed; original unknown upstream request retained')
|
||||
}
|
||||
main().catch(error => { console.error(error instanceof Error ? error.message : 'Acceptance failed'); process.exitCode = 1 }).finally(() => prisma.$disconnect())
|
||||
Loading…
x
Reference in New Issue
Block a user