85 lines
8.5 KiB
TypeScript
85 lines
8.5 KiB
TypeScript
/** Isolated HTTP acceptance. Paid tests are explicit, one model + purpose per invocation. */
|
|
import assert from 'node:assert/strict'
|
|
import { randomUUID } from 'node:crypto'
|
|
import { mkdir, readFile, writeFile } from 'node:fs/promises'
|
|
import { PrismaClient } from '@prisma/client'
|
|
import type { AiModelAdminState, AiModelStage } from '../lib/ai-model-types'
|
|
|
|
const db = new URL(process.env.DATABASE_URL || '')
|
|
assert.equal(db.hostname, '127.0.0.1'); assert.equal(db.port, '55441'); assert.equal(db.pathname, '/ai_model_test')
|
|
const base = 'http://localhost:3121', output = 'output/ai-model-integration-v2'
|
|
const prisma = new PrismaClient({ log: [] })
|
|
const phase = process.argv[2] || 'inspect'
|
|
async function save(name: string, value: unknown) { await mkdir(output, { recursive: true }); await writeFile(`${output}/${name}.json`, JSON.stringify(value, null, 2)) }
|
|
async function session(index: number) {
|
|
const jar = new Map<string, string>()
|
|
const store = (r: Response) => { for (const c of r.headers.getSetCookie()) { const first = c.split(';')[0], split = first.indexOf('='); jar.set(first.slice(0, split), first.slice(split + 1)) } }
|
|
const cookie = () => [...jar].map(([k,v]) => `${k}=${v}`).join('; ')
|
|
const csrf = await fetch(base + '/api/auth/csrf'); store(csrf)
|
|
const { csrfToken } = await csrf.json()
|
|
const response = await fetch(base + '/api/auth/callback/credentials', { method:'POST', redirect:'manual', headers:{'Content-Type':'application/x-www-form-urlencoded', Origin:base, Cookie:cookie()}, body:new URLSearchParams({csrfToken,email:`ai-admin-${index}@example.invalid`,password:'AiAdminLocalTest-20261012',callbackUrl:base+'/admin',json:'true'}) })
|
|
store(response); assert.equal(response.status,200); return cookie()
|
|
}
|
|
async function request(cookie: string, path = '', method = 'GET', body?: unknown) {
|
|
const response = await fetch(base+'/api/admin/ai-models'+path,{method,redirect:'manual',headers:{Cookie:cookie,'Content-Type':'application/json'},...(body===undefined?{}:{body:JSON.stringify(body)})})
|
|
const raw = await response.text()
|
|
if(process.env.DETAIL_GENERATOR_API_KEY)assert.ok(!raw.includes(process.env.DETAIL_GENERATOR_API_KEY),'Secret leaked in response')
|
|
return {status:response.status,data:raw?JSON.parse(raw):null}
|
|
}
|
|
function ok(result: Awaited<ReturnType<typeof request>>): AiModelAdminState { assert.equal(result.status,200,JSON.stringify(result.data));return result.data }
|
|
async function main() {
|
|
const admin = await session(0)
|
|
let state = ok(await request(admin))
|
|
if(phase==='bootstrap') {
|
|
if(state.legacyAvailable)state=ok(await request(admin,'/import-environment','POST'))
|
|
let provider=state.providers.find(p=>p.credentialSource==='environment'); assert.ok(provider)
|
|
if(!provider.enabled)state=ok(await request(admin,`/providers/${provider.id}`,'PATCH',{revision:provider.revision,enabled:true}))
|
|
provider=state.providers.find(p=>p.id===provider!.id)!
|
|
state=ok(await request(admin,`/providers/${provider.id}/sync`,'POST',{revision:provider.revision}))
|
|
await save('catalog-state',state)
|
|
} else if(phase==='test') {
|
|
const stage=process.argv[3] as AiModelStage; assert.ok(['analysis','copy','image'].includes(stage))
|
|
const modelId=stage==='image'?'nano-banana-2':'gemini-3-flash'
|
|
const provider=state.providers.find(p=>p.credentialSource==='environment'); assert.ok(provider)
|
|
const model=provider.models.find(m=>m.id===modelId); assert.ok(model)
|
|
assert.ok(model.capabilities.allowedStages.includes(stage),`${modelId} has no ${stage} adapter`)
|
|
let receipt:{idempotencyKey:string;revision:number;providerId:string;modelKey:string;stage:AiModelStage}
|
|
try {receipt=JSON.parse(await readFile(`${output}/live-${stage}-request.json`,'utf8'))}
|
|
catch {receipt={idempotencyKey:randomUUID(),revision:provider.revision,providerId:provider.id,modelKey:model.key,stage};await save(`live-${stage}-request`,receipt)}
|
|
assert.equal(receipt.providerId,provider.id);assert.equal(receipt.modelKey,model.key)
|
|
state=ok(await request(admin,`/providers/${provider.id}/models/${model.key}/verify`,'POST',{stage,revision:receipt.revision,idempotencyKey:receipt.idempotencyKey}))
|
|
const result=state.providers.find(p=>p.id===provider.id)!.models.find(m=>m.key===model.key)!.verification[stage]
|
|
await save(`live-${stage}-result`,result);console.log(JSON.stringify({model:modelId,stage,...result}));return
|
|
} else if(phase==='query') {
|
|
const stage=process.argv[3] as AiModelStage;assert.ok(['analysis','copy','image'].includes(stage))
|
|
const provider=state.providers.find(p=>p.credentialSource==='environment')!, model=provider.models.find(m=>m.id===(stage==='image'?'nano-banana-2':'gemini-3-flash'))!
|
|
const result=model.verification[stage];assert.ok(result.id&&result.canResume,'No known task to query')
|
|
state=ok(await request(admin,`/providers/${provider.id}/verifications/${result.id}/refresh`,'POST'))
|
|
const checked=state.providers.find(p=>p.id===provider.id)!.models.find(m=>m.key===model.key)!.verification[stage]
|
|
await save(`live-${stage}-result`,checked);console.log(JSON.stringify({stage,...checked}));return
|
|
} else if(phase==='configure') {
|
|
const provider=state.providers.find(p=>p.credentialSource==='environment')!, gemini=provider.models.find(m=>m.id==='gemini-3-flash')!, image=provider.models.find(m=>m.id==='nano-banana-2')!
|
|
for(const [model,stage] of [[gemini,'analysis'],[gemini,'copy'],[image,'image']] as const)assert.equal(model.verification[stage].status,'passed')
|
|
state=ok(await request(admin,'/configuration','PUT',{revision:state.revision,defaults:{analysis:gemini.key,copy:gemini.key,image:image.key},providers:state.providers.map(p=>({id:p.id,revision:p.revision,models:p.models.map(m=>({key:m.key,enabled:p.id===provider.id&&[gemini.key,image.key].includes(m.key),stages:p.id===provider.id&&m.key===gemini.key?['analysis','copy']:p.id===provider.id&&m.key===image.key?['image']:[]}))}))}))
|
|
await save('configured-state',state)
|
|
} else if(phase==='access') {
|
|
const counts=await prisma.$queryRawUnsafe<Array<{count:number}>>('SELECT COUNT(*)::int AS count FROM ai_model_verifications')
|
|
assert.equal((await request('')).status,401)
|
|
const provider=state.providers.find(p=>p.credentialSource==='environment')!, model=provider.models.find(m=>m.id==='gemini-3-flash')!
|
|
for(const index of [2,3]) {
|
|
const cookie=await session(index)
|
|
for(const [url,method,body] of [['','GET',undefined],['/configuration','PUT',{}],[`/providers/${provider.id}/sync`,'POST',{revision:provider.revision}],[`/providers/${provider.id}/models/${model.key}/verify`,'POST',{stage:'copy',revision:provider.revision,idempotencyKey:randomUUID()}]] as const)assert.equal((await request(cookie,url,method,body)).status,403)
|
|
}
|
|
const revoked=await session(1), user=await prisma.user.findUniqueOrThrow({where:{email:'ai-admin-1@example.invalid'}})
|
|
try{await prisma.user.update({where:{id:user.id},data:{isSuperAdmin:false}});assert.equal((await request(revoked)).status,403);assert.equal((await request(revoked,`/providers/${provider.id}/models/${model.key}/verify`,'POST',{stage:'copy',revision:provider.revision,idempotencyKey:randomUUID()})).status,403)}finally{await prisma.user.update({where:{id:user.id},data:{isSuperAdmin:true}})}
|
|
assert.equal((await request(admin,'/configuration','PUT',{revision:state.revision,defaults:state.defaults,providers:[{id:provider.id,revision:provider.revision,models:[{key:model.key,enabled:true,stages:['copy'],compatibility:'confirmed'}]}]})).status,400)
|
|
assert.equal((await request(admin,`/providers/${provider.id}/models/${model.key}/verify`,'POST',{stage:'image',revision:provider.revision,idempotencyKey:randomUUID()})).status,400)
|
|
const after=await prisma.$queryRawUnsafe<Array<{count:number}>>('SELECT COUNT(*)::int AS count FROM ai_model_verifications')
|
|
assert.equal(after[0].count,counts[0].count)
|
|
const evidence={anonymous:401,ordinaryAndRegularAdmin:403,oldSessionRevocation:403,manualCompatibilitySpoof:400,incompatibleImageTest:400,newVerificationRecords:0,productionTouched:false}
|
|
await save('http-access-evidence',evidence);console.log(JSON.stringify(evidence));return
|
|
} else assert.equal(phase,'inspect')
|
|
console.log(JSON.stringify({phase,providers:state.providers.map(p=>({name:p.name,enabled:p.enabled,models:p.models.length,defaults:p.models.filter(m=>['gemini-3-flash','nano-banana-2'].includes(m.id)).map(m=>({id:m.id,capabilities:m.capabilities,verification:m.verification}))}))}))
|
|
}
|
|
main().catch(error=>{console.error(error instanceof Error?error.message:'Acceptance failed');process.exitCode=1}).finally(()=>prisma.$disconnect())
|