105 lines
12 KiB
TypeScript
105 lines
12 KiB
TypeScript
/** Native HTTP acceptance; all synthetic fixtures are confined to the named local QA database. */
|
|
import assert from 'node:assert/strict'
|
|
import { randomUUID } from 'node:crypto'
|
|
import { mkdir, readFile, writeFile } from 'node:fs/promises'
|
|
import { PrismaClient } from '@prisma/client'
|
|
import { getAiModelStageProtocol } from '../lib/ai-model-capabilities'
|
|
import { aiModelVerificationContext } from '../lib/ai-model-verification-state'
|
|
import type { AiModelAdminState, AiModelStage } from '../lib/ai-model-types'
|
|
const database = new URL(process.env.DATABASE_URL || '')
|
|
assert.equal(database.hostname, '127.0.0.1'); assert.equal(database.port, '55441'); assert.equal(database.pathname, '/ai_model_test')
|
|
const base = 'http://localhost:3125', output = 'D:/AI Code/gongxiangp2/output/ai-combined-20261014', phase = process.argv[2] || 'plan'
|
|
const prisma = new PrismaClient({ log: [] })
|
|
async function save(name: string, data: unknown) { await mkdir(output, { recursive: true }); await writeFile(`${output}/${name}.json`, JSON.stringify(data, null, 2)) }
|
|
async function session() {
|
|
const jar = new Map<string, string>()
|
|
const store = (response: Response) => { for (const cookie of response.headers.getSetCookie()) { const value = cookie.split(';')[0], at = value.indexOf('='); jar.set(value.slice(0, at), value.slice(at + 1)) } }
|
|
const cookie = () => Array.from(jar).map(([key, value]) => `${key}=${value}`).join('; ')
|
|
const csrf = await fetch(base + '/api/auth/csrf'); store(csrf)
|
|
const { csrfToken } = await csrf.json()
|
|
const signed = await fetch(base + '/api/auth/callback/credentials', { method: 'POST', redirect: 'manual', headers: { 'Content-Type': 'application/x-www-form-urlencoded', Origin: base, Cookie: cookie() }, body: new URLSearchParams({ csrfToken, email: 'ai-admin-1@example.invalid', password: 'AiAdminLocalTest-20261012', callbackUrl: base + '/admin', json: 'true' }) })
|
|
store(signed); assert.equal(signed.status, 200); return cookie()
|
|
}
|
|
async function main() {
|
|
const cookie = await session(), actor = await prisma.user.findUniqueOrThrow({ where: { email: 'ai-admin-1@example.invalid' } })
|
|
assert.ok(actor.isSuperAdmin && actor.isActive)
|
|
async function request(path = '', method = 'GET', body?: unknown) {
|
|
const response = await fetch(base + '/api/admin/ai-models' + path, { method, headers: { Cookie: cookie, 'Content-Type': 'application/json' }, ...(body === undefined ? {} : { body: JSON.stringify(body) }), signal: AbortSignal.timeout(120000) })
|
|
const text = await response.text(); assert.ok(!process.env.DETAIL_GENERATOR_API_KEY || !text.includes(process.env.DETAIL_GENERATOR_API_KEY), 'Credential leaked')
|
|
return { status: response.status, data: JSON.parse(text) }
|
|
}
|
|
const state = (await request()).data as AiModelAdminState
|
|
const provider = state.providers.find(value => value.credentialSource === 'environment')!; assert.ok(provider)
|
|
const model = provider.models.find(value => value.id === 'gemini-3-flash')!; assert.ok(model?.capabilities.combinedTextTest)
|
|
const startCount = async () => (await prisma.$queryRawUnsafe<Array<{ count: number }>>(`SELECT COUNT(*)::int AS count FROM ai_model_audit WHERE action='model.verification-start' AND details->>'modelKey'=$1`, model.key))[0].count
|
|
if (phase === 'plan') {
|
|
const plan = { providerId: provider.id, modelKey: model.key, actorId: actor.id, revision: provider.revision, model: model.id, starts: await startCount(), createdAt: new Date().toISOString() }
|
|
await save('real-plan', plan); console.log(JSON.stringify(plan)); return
|
|
}
|
|
if (phase === 'receipt') {
|
|
const plan = JSON.parse(await readFile(`${output}/real-plan.json`, 'utf8'))
|
|
assert.equal(model.key, plan.modelKey)
|
|
const rows = await prisma.$queryRawUnsafe<Array<{ id: string; stage: AiModelStage; status: string; idempotency_key: string; request_id: string; evidence: Record<string, any> }>>(`SELECT id,stage,status,idempotency_key,request_id,evidence FROM ai_model_verifications WHERE model_key=$1 AND actor_id=$2 AND started_at>=$3::timestamptz ORDER BY stage`, model.key, actor.id, plan.createdAt)
|
|
assert.equal(rows.length, 2); assert.deepEqual(rows.map(value => value.stage), ['analysis', 'copy'])
|
|
assert.ok(rows.every(value => value.status === 'passed' && value.evidence.purpose === 'analysis-copy' && value.evidence.ruleVersion === 2))
|
|
assert.equal(rows[0].idempotency_key, rows[1].idempotency_key); assert.equal(rows[0].request_id, rows[1].request_id)
|
|
assert.equal(rows[0].evidence.executionGroupId, rows[1].evidence.executionGroupId)
|
|
assert.equal(await startCount() - plan.starts, 1)
|
|
const repeated = await request(`/providers/${provider.id}/models/${model.key}/verify`, 'POST', { stage: 'analysis-copy', revision: provider.revision, idempotencyKey: rows[0].idempotency_key })
|
|
assert.equal(repeated.status, 200); assert.equal(await startCount() - plan.starts, 1)
|
|
const evidence = { nativeHttp: true, productionTouched: false, model: model.id, upstreamCallsAdmitted: 1, passedStages: rows.map(value => value.stage), sameUpstreamRequestId: rows[0].request_id, sameExecutionGroupId: rows[0].evidence.executionGroupId, verificationIds: rows.map(value => value.id), repeatedSubmissionAddedCalls: 0, records: rows }
|
|
await save('real-acceptance', evidence); console.log(JSON.stringify({ ...evidence, records: undefined })); return
|
|
}
|
|
if (phase === 'recovery') {
|
|
const source = await prisma.$queryRawUnsafe<Array<{ capability_metadata: unknown }>>('SELECT capability_metadata FROM ai_model_catalog WHERE id=$1', model.key)
|
|
const metadata = source[0].capability_metadata, modelKey = randomUUID(), modelId = `qa-joint-${randomUUID()}`, key = randomUUID(), groupId = randomUUID(), ids = [randomUUID(), randomUUID()]
|
|
await prisma.$transaction(async db => {
|
|
await db.$executeRawUnsafe(`INSERT INTO ai_model_catalog (id,provider_id,model_id,enabled,stages,present,capability_metadata) VALUES ($1,$2,$3,FALSE,ARRAY[]::text[],TRUE,$4::jsonb)`, modelKey, provider.id, modelId, JSON.stringify(metadata))
|
|
for (const [index, stage] of (['analysis', 'copy'] as const).entries()) {
|
|
const protocol = getAiModelStageProtocol(metadata, stage)!; assert.ok(protocol)
|
|
const contextHash = aiModelVerificationContext(provider.baseUrl, modelId, metadata, process.env.DETAIL_GENERATOR_API_KEY!, protocol)
|
|
await db.$executeRawUnsafe(`INSERT INTO ai_model_verifications (id,model_key,stage,context_hash,idempotency_key,actor_id,status,evidence) VALUES ($1,$2,$3,$4,$5,$6,'unknown',$7::jsonb)`, ids[index], modelKey, stage, contextHash, key, actor.id, JSON.stringify({ purpose: 'analysis-copy', executionGroupId: groupId, challenge: { topLeft: 'red', bottomRight: 'blue' }, fixture: true }))
|
|
}
|
|
})
|
|
await save('fixture', { modelKey, modelId, ids, providerId: provider.id, actorId: actor.id })
|
|
const before = await startCount()
|
|
const count = (await prisma.$queryRawUnsafe<Array<{ physical: number; executions: number }>>(`SELECT COUNT(*)::int AS physical,COUNT(DISTINCT COALESCE(evidence->>'executionGroupId',id))::int AS executions FROM ai_model_verifications WHERE model_key=$1 AND status IN ('running','unknown')`, modelKey))[0]
|
|
assert.deepEqual(count, { physical: 2, executions: 1 })
|
|
const busy = await request(`/providers/${provider.id}/models/${model.key}/verify`, 'POST', { stage: 'analysis-copy', revision: provider.revision, idempotencyKey: randomUUID() }); assert.equal(busy.data.code, 'verification_busy')
|
|
const checked = await request(`/providers/${provider.id}/models/${modelKey}/submission`, 'POST', { stage: 'analysis-copy', idempotencyKey: key }); assert.equal(checked.status, 200); assert.equal(checked.data.submissionCheck.received, true)
|
|
const proof = { outcome: 'not_executed', reference: 'qa-joint-fixture-no-upstream-call', note: '隔离测试注入的记录，没有调用上游；核查后关闭同一次请求的两条用途记录。', confirmed: true }
|
|
const invalid = await request(`/providers/${provider.id}/verifications/${ids[1]}/reconcile`, 'POST', { ...proof, confirmed: false }); assert.equal(invalid.status, 400)
|
|
const ended = await request(`/providers/${provider.id}/verifications/${ids[1]}/reconcile`, 'POST', proof); assert.equal(ended.status, 200)
|
|
const records = await prisma.$queryRawUnsafe<Array<{ status: string; evidence: Record<string, any> }>>('SELECT status,evidence FROM ai_model_verifications WHERE model_key=$1', modelKey)
|
|
assert.equal(records.length, 2); assert.ok(records.every(value => value.status === 'failed' && value.evidence.reconciliation.reference === proof.reference))
|
|
const absent = { stage: 'analysis-copy', idempotencyKey: randomUUID() }
|
|
const closed = await request(`/providers/${provider.id}/models/${modelKey}/submission`, 'POST', absent); assert.equal(closed.data.submissionCheck.received, false)
|
|
const late = await request(`/providers/${provider.id}/models/${modelKey}/verify`, 'POST', { ...absent, revision: provider.revision }); assert.equal(late.data.code, 'verification_submission_closed')
|
|
assert.equal(await startCount(), before)
|
|
const analysisId = randomUUID(), copyId = randomUUID(), partialKey = randomUUID(), partialGroup = randomUUID()
|
|
for (const [stage, id] of [['analysis', analysisId], ['copy', copyId]] as const) {
|
|
const protocol = getAiModelStageProtocol(metadata, stage)!, contextHash = aiModelVerificationContext(provider.baseUrl, modelId, metadata, process.env.DETAIL_GENERATOR_API_KEY!, protocol)
|
|
await prisma.$executeRawUnsafe(`INSERT INTO ai_model_verifications (id,model_key,stage,context_hash,idempotency_key,actor_id,status,evidence,message,finished_at) VALUES ($1,$2,$3,$4,$5,$6,$7,$8::jsonb,$9,NOW())`, id, modelKey, stage, contextHash, partialKey, actor.id, stage === 'analysis' ? 'passed' : 'failed', JSON.stringify({ purpose: 'analysis-copy', executionGroupId: partialGroup, ruleVersion: 2, fixture: true, ...(stage === 'analysis' ? { imageInputVerified: true } : {}) }), '隔离测试：分析通过，文案结构未通过')
|
|
}
|
|
async function configure(enabled: boolean, stages: AiModelStage[]) {
|
|
const fresh = (await request()).data as AiModelAdminState, connection = fresh.providers.find(value => value.id === provider.id)!
|
|
return request('/configuration', 'PUT', { revision: fresh.revision, defaults: fresh.defaults, providers: [{ id: provider.id, revision: connection.revision, models: [{ key: modelKey, enabled, stages }] }] })
|
|
}
|
|
const denied = await configure(true, ['copy']); assert.equal(denied.data.code, 'model_verification_required')
|
|
assert.equal((await configure(true, ['analysis'])).status, 200)
|
|
assert.equal((await configure(false, [])).status, 200)
|
|
const evidence = { nativeHttp: true, productionTouched: false, paidCalls: 0, groupRecords: 2, executionSlots: 1, jointSubmissionReceived: true, ownerBusy: 409, invalidRecovery: 400, bothUnknownRecordsClosed: true, lateSubmission: 409, failedPurposeCannotEnable: 400, onlyPassedPurposeCanEnable: true, uiPartialFixtureModel: modelId }
|
|
await save('native-acceptance', evidence); console.log(JSON.stringify(evidence)); return
|
|
}
|
|
assert.equal(phase, 'cleanup')
|
|
const receipt = JSON.parse(await readFile(`${output}/fixture.json`, 'utf8'))
|
|
const fixture = await prisma.$queryRawUnsafe<Array<{ model_id: string; enabled: boolean }>>('SELECT model_id,enabled FROM ai_model_catalog WHERE id=$1 AND provider_id=$2', receipt.modelKey, receipt.providerId)
|
|
assert.equal(fixture[0]?.model_id, receipt.modelId); assert.ok(receipt.modelId.startsWith('qa-joint-')); assert.equal(fixture[0].enabled, false)
|
|
await prisma.$transaction(async db => {
|
|
await db.$executeRawUnsafe('DELETE FROM ai_model_verifications WHERE model_key=$1 AND actor_id=$2', receipt.modelKey, receipt.actorId)
|
|
await db.$executeRawUnsafe('DELETE FROM ai_model_catalog WHERE id=$1 AND model_id=$2 AND enabled=FALSE', receipt.modelKey, receipt.modelId)
|
|
})
|
|
await save('fixture-cleanup', { modelKey: receipt.modelKey, productionTouched: false }); console.log('Named QA fixture removed; original unknown upstream request retained')
|
|
}
|
|
main().catch(error => { console.error(error instanceof Error ? error.message : 'Acceptance failed'); process.exitCode = 1 }).finally(() => prisma.$disconnect())
|