WuYongJie
a890660853
Preserve known legacy failures by reviewed identity, probe material delivery before and after releases, and retain originals and downloads.
620 lines
36 KiB
JavaScript
620 lines
36 KiB
JavaScript
import assert from 'node:assert/strict'
|
|
import { existsSync, readFileSync, mkdtempSync, writeFileSync, rmSync } from 'node:fs'
|
|
import { spawnSync } from 'node:child_process'
|
|
import test from 'node:test'
|
|
import { runInNewContext } from 'node:vm'
|
|
import { createHash } from 'node:crypto'
|
|
import os from 'node:os'
|
|
import path from 'node:path'
|
|
import { aiModelRegistryEvidence } from './fixtures/ai-model-registry-evidence.mjs'
|
|
|
|
const bash = process.platform === 'win32' && existsSync('C:/Program Files/Git/bin/bash.exe') ? 'C:/Program Files/Git/bin/bash.exe' : 'bash'
|
|
|
|
import {
|
|
extractRemoteTemplate,
|
|
parseArguments,
|
|
renderRemoteTemplate,
|
|
} from './git-incremental-deploy-macos.mjs'
|
|
|
|
test('generated remote deployment payload is valid bash', () => {
|
|
const source = readFileSync(new URL('./git-incremental-deploy.ps1', import.meta.url), 'utf8')
|
|
const payload = extractRemoteTemplate(source).replace(/__[A-Z0-9_]+__/g, "''")
|
|
const result = spawnSync(bash, ['-n'], {
|
|
input: payload,
|
|
encoding: 'utf8',
|
|
})
|
|
|
|
assert.equal(result.status, 0, result.stderr || result.stdout)
|
|
})
|
|
|
|
test('Mac entry renders every canonical remote deployment token', () => {
|
|
const source = readFileSync(new URL('./git-incremental-deploy.ps1', import.meta.url), 'utf8')
|
|
const template = extractRemoteTemplate(source)
|
|
const tokens = [...new Set(template.match(/__[A-Z0-9_]+__/g) ?? [])]
|
|
const values = Object.fromEntries(tokens.map((token, index) => [token, token === '__COMMIT__'
|
|
? 'a'.repeat(40)
|
|
: `value ${index} with quote ' safely escaped`]))
|
|
const payload = renderRemoteTemplate(template, values)
|
|
assert.doesNotMatch(payload, /__[A-Z0-9_]+__/)
|
|
|
|
const result = spawnSync(bash, ['-n'], { input: payload, encoding: 'utf8' })
|
|
assert.equal(result.status, 0, result.stderr || result.stdout)
|
|
})
|
|
|
|
test('Mac entry requires an exact commit outside DryRun', () => {
|
|
assert.throws(() => parseArguments([]), /正式部署必须/)
|
|
assert.throws(() => parseArguments(['--commit', 'abc']), /40 位小写/)
|
|
assert.equal(parseArguments(['--dry-run']).dryRun, true)
|
|
assert.equal(parseArguments(['--commit', 'b'.repeat(40)]).commit, 'b'.repeat(40))
|
|
assert.equal(parseArguments(['--commit', 'b'.repeat(40), '--skip-deploy']).skipDeploy, true)
|
|
assert.throws(
|
|
() => parseArguments(['--commit', 'b'.repeat(40), '--reuse-verified-image', '--skip-deploy']),
|
|
/不能与 --skip-deploy/,
|
|
)
|
|
})
|
|
|
|
test('deployment SSH hops keep long builds and cutovers alive', () => {
|
|
const macSource = readFileSync(new URL('./git-incremental-deploy-macos.mjs', import.meta.url), 'utf8')
|
|
const powerShellSource = readFileSync(new URL('./git-incremental-deploy.ps1', import.meta.url), 'utf8')
|
|
const payload = extractRemoteTemplate(powerShellSource)
|
|
|
|
for (const source of [macSource, payload]) {
|
|
assert.match(source, /ServerAliveInterval=3/)
|
|
assert.match(source, /ServerAliveCountMax=10/)
|
|
assert.match(source, /TCPKeepAlive=yes/)
|
|
}
|
|
})
|
|
|
|
test('migration deployment gate recognizes the latest approved schema migration', () => {
|
|
const source = readFileSync(new URL('./git-incremental-deploy.ps1', import.meta.url), 'utf8')
|
|
const gateStart = source.indexOf('const expectedMigrationNames = new Set([')
|
|
const gateEnd = source.indexOf(']);', gateStart)
|
|
|
|
assert.ok(gateStart >= 0 && gateEnd > gateStart, 'migration gate list is missing')
|
|
const gate = source.slice(gateStart, gateEnd)
|
|
assert.match(gate, /20261008_permission_group_department_identity/)
|
|
assert.match(gate, /20261008_team_function_configs/)
|
|
assert.match(gate, /20260831_add_private_task_detail/)
|
|
assert.match(gate, /20260908_add_brand_knowledge/)
|
|
assert.match(gate, /20260914_add_brand_knowledge_engagement/)
|
|
assert.match(gate, /20260914_expand_brand_knowledge_download_count/)
|
|
assert.match(gate, /20260914_expand_brand_knowledge_preview_slices/)
|
|
assert.match(gate, /20260915_add_brand_knowledge_soft_delete/)
|
|
assert.match(gate, /20260918_add_performance_conversion_sync/)
|
|
assert.match(gate, /20261012_detail_generator/)
|
|
assert.match(gate, /20261012_ai_model_registry/)
|
|
assert.match(gate, /20261013_ai_model_capabilities/)
|
|
assert.match(gate, /20261013_ai_model_verifications/)
|
|
assert.match(gate, /20261020090000_desktop_client_workspace/)
|
|
})
|
|
|
|
test('direct replacement runs database and dependency gates before switching traffic', () => {
|
|
const source = readFileSync(new URL('./git-incremental-deploy.ps1', import.meta.url), 'utf8')
|
|
.replace(/\r\n/g, '\n')
|
|
const functionStart = source.indexOf('direct_replace_active_container() {')
|
|
const functionEnd = source.indexOf('\n}\n\ncontainer_upstream_connection_count()', functionStart)
|
|
assert.ok(functionStart >= 0 && functionEnd > functionStart, 'direct replacement function is missing')
|
|
const direct = source.slice(functionStart, functionEnd)
|
|
const switchTraffic = direct.indexOf('write_proxy_config "${replacement_ip}:3000"')
|
|
assert.ok(switchTraffic > 0, 'direct traffic switch is missing')
|
|
|
|
for (const gate of [
|
|
'verify_transcode_lease_schema "$name"',
|
|
'verify_task_detail_upload_lease_schema "$name"',
|
|
'verify_brand_logo_defaults "$name"',
|
|
'verify_material_upload_idempotency_schema "$name"',
|
|
'verify_candidate_database_identity "$name"',
|
|
'verify_candidate_brand_and_storage_dependencies "$name" "$replacement_ip"',
|
|
'verify_video_transcode_capability "$name"',
|
|
'verify_task_detail_upload_capability "$name"',
|
|
'verify_detail_generator_schema "$name"',
|
|
'verify_detail_generator_dependencies "$name"',
|
|
]) {
|
|
const position = direct.indexOf(gate)
|
|
assert.ok(position > 0 && position < switchTraffic, `${gate} must run before the direct traffic switch`)
|
|
}
|
|
assert.match(direct, /fail_direct_replace_with_restore "New direct container database identity differs/)
|
|
assert.match(direct, /fail_direct_replace_with_restore "New direct container cannot authenticate to S3/)
|
|
assert.ok(direct.indexOf('prepare_detail_generator_release') < direct.indexOf('docker stop --time 10'), 'configuration and private volume must be validated before stopping the old slot')
|
|
})
|
|
|
|
function deploySource() { return readFileSync(new URL('./git-incremental-deploy.ps1', import.meta.url), 'utf8').replace(/\r\n/g, '\n') }
|
|
function shellFunction(name) {
|
|
const source = deploySource()
|
|
const start = source.indexOf(`${name}() {`)
|
|
const end = source.indexOf('\n}\n\n', start)
|
|
assert.ok(start >= 0 && end > start, `missing ${name}`)
|
|
return source.slice(start, end + 3)
|
|
}
|
|
|
|
test('preview inventory and fixed probes guard direct stop, switch, and baseline promotion in both wrappers', () => {
|
|
const direct = shellFunction('direct_replace_active_container')
|
|
const stop = direct.indexOf('docker stop --time 10')
|
|
const switchTraffic = direct.indexOf('write_proxy_config "${replacement_ip}:3000"')
|
|
const record = direct.indexOf('write_current_container "$name"')
|
|
assert.ok(direct.indexOf('prepare_material_preview_release') < stop)
|
|
assert.ok(direct.indexOf('run_material_preview_release_gate "$name" capture') < stop)
|
|
assert.ok(direct.indexOf('run_material_preview_release_gate "$name" candidate') > stop)
|
|
assert.ok(direct.indexOf('run_material_preview_release_gate "$name" candidate') < switchTraffic)
|
|
assert.ok(direct.indexOf('run_material_preview_release_gate "$name" post') > switchTraffic)
|
|
assert.ok(direct.indexOf('run_material_preview_release_gate "$name" post') < record)
|
|
assert.ok(direct.indexOf('accept_material_preview_release') > record)
|
|
const payload = extractRemoteTemplate(deploySource())
|
|
assert.match(payload, /node scripts\/material-preview-release-gate\.mjs bundle/)
|
|
assert.match(payload, /run_material_preview_release_gate "\$NEXT_CONTAINER" candidate/)
|
|
assert.match(payload, /run_material_preview_release_gate "\$NEXT_CONTAINER" post/)
|
|
assert.match(payload, /MATERIAL_PREVIEW_GATE_MODULE=\$\(printf '%q'/)
|
|
assert.doesNotMatch(payload, /SKIP_MATERIAL_PREVIEW|IGNORE_MATERIAL_PREVIEW|ALLOW_PREVIEW_FAILURE/)
|
|
})
|
|
|
|
test('real direct shell control flow stops before replacement or uses existing rollback exactly once on preview regression', () => {
|
|
const direct = shellFunction('direct_replace_active_container')
|
|
const inert = ['verify_container_uploads_storage', 'verify_image_revision_label', 'prepare_worker_runtime_directories', 'prepare_detail_generator_release',
|
|
'create_container', 'verify_container_uploads_mount_metadata', 'verify_container_upload_dir_config', 'verify_container_baked_revision',
|
|
'verify_transcode_lease_schema', 'verify_task_detail_upload_lease_schema', 'verify_detail_generator_schema', 'verify_detail_generator_dependencies',
|
|
'verify_brand_logo_defaults', 'verify_material_upload_idempotency_schema', 'verify_candidate_database_identity', 'verify_candidate_brand_and_storage_dependencies',
|
|
'wait_for_direct_worker_capabilities', 'verify_video_transcode_capability', 'verify_task_detail_upload_capability', 'verify_proxy_target', 'retire_inactive_direct_slot']
|
|
for (const phase of ['capture', 'candidate', 'post', 'none']) {
|
|
const input = `set -u
|
|
BLUE_CONTAINER=material-blue
|
|
GREEN_CONTAINER=material-green
|
|
REGISTRY_COMMIT=registry/exact-image
|
|
COMMIT_SHA=${'a'.repeat(40)}
|
|
log() { :; }
|
|
ok() { :; }
|
|
warn() { :; }
|
|
fail() { printf 'blocked:%s\n' "$*"; exit 82; }
|
|
docker() { case "$1" in inspect) printf 'sha256:${'b'.repeat(64)}\n' ;; *) printf 'docker:%s\n' "$1" ;; esac; }
|
|
${inert.map(name => `${name}() { return 0; }`).join('\n')}
|
|
prepare_material_preview_release() { printf 'prepare\n'; }
|
|
run_material_preview_release_gate() { printf 'probe:%s\n' "$2"; [ "$2" != '${phase}' ]; }
|
|
fail_direct_replace_with_restore() { printf 'existing-rollback-once\n'; exit 81; }
|
|
wait_for_container_health() { printf '172.18.0.9\n'; }
|
|
write_proxy_config() { printf 'traffic-prepared\n'; }
|
|
switch_proxy() { printf 'traffic-switched\n'; }
|
|
write_current_container() { printf 'current-recorded\n'; }
|
|
accept_material_preview_release() { printf 'baseline-promoted\n'; }
|
|
transcode_drain_file() { printf '/not-a-real-drain-file\n'; }
|
|
task_detail_drain_file() { printf '/not-a-real-upload-file\n'; }
|
|
rm() { return 0; }
|
|
${direct}
|
|
direct_replace_active_container material-blue
|
|
`
|
|
const result = spawnSync(bash, ['-s'], { input, encoding: 'utf8' })
|
|
assert.equal(result.status, phase === 'none' ? 0 : phase === 'capture' ? 82 : 81, result.stderr || result.stdout)
|
|
assert.equal((result.stdout.match(/existing-rollback-once/g) || []).length, ['candidate', 'post'].includes(phase) ? 1 : 0)
|
|
if (phase === 'capture') assert.doesNotMatch(result.stdout, /docker:stop|traffic-switched|baseline-promoted/)
|
|
if (phase === 'candidate') assert.doesNotMatch(result.stdout, /traffic-switched|baseline-promoted/)
|
|
if (phase === 'post') { assert.match(result.stdout, /traffic-switched/); assert.doesNotMatch(result.stdout, /current-recorded|baseline-promoted/) }
|
|
if (phase === 'none') assert.ok(result.stdout.indexOf('probe:post') < result.stdout.indexOf('baseline-promoted'))
|
|
}
|
|
})
|
|
|
|
test('storage deployment gate rejects broken public bucket URLs before authenticating', async () => {
|
|
const fn = shellFunction('verify_candidate_authenticated_s3')
|
|
const code = fn.slice(fn.indexOf("node -e '\n") + "node -e '\n".length, fn.lastIndexOf("\n'"))
|
|
const base = { S3_ENDPOINT: 'http://minio:9000', S3_BUCKET: 'gongxiangpan', S3_ACCESS_KEY_ID: 'fixture', S3_SECRET_ACCESS_KEY: 'fixture' }
|
|
for (const domain of ['', 'https://s3.jiansun.vip', 'https://s3.jiansun.vip/other-bucket', 'http://s3.jiansun.vip/gongxiangpan', 'https://user:pass@s3.jiansun.vip/gongxiangpan', 'https://s3.jiansun.vip/gongxiangpan?token=fixture', 'https://s3.jiansun.vip/gongxiangpan#fragment']) {
|
|
let calls = 0
|
|
assert.throws(() => runInNewContext(code, {
|
|
URL, AbortSignal,
|
|
process: { env: { ...base, S3_PUBLIC_DOMAIN: domain }, exit() { throw new Error('missing public bucket URL') } },
|
|
require() { return { S3Client: class { constructor() { calls++ } }, HeadBucketCommand: class {} } },
|
|
}), /S3_PUBLIC_DOMAIN|missing public bucket URL/)
|
|
assert.equal(calls, 0, 'invalid public URL must fail before touching storage')
|
|
}
|
|
})
|
|
|
|
test('storage deployment gate accepts and authenticates the complete public bucket URL', async () => {
|
|
const fn = shellFunction('verify_candidate_authenticated_s3')
|
|
const code = fn.slice(fn.indexOf("node -e '\n") + "node -e '\n".length, fn.lastIndexOf("\n'"))
|
|
for (const domain of ['https://s3.jiansun.vip/gongxiangpan', 'https://s3.jiansun.vip/gongxiangpan/']) {
|
|
let calls = 0, destroyed = false
|
|
await runInNewContext(code, {
|
|
URL, AbortSignal,
|
|
process: { env: { S3_ENDPOINT: 'http://minio:9000', S3_BUCKET: 'gongxiangpan', S3_PUBLIC_DOMAIN: domain, S3_ACCESS_KEY_ID: 'fixture', S3_SECRET_ACCESS_KEY: 'fixture' }, stderr: { write(message) { throw new Error(message) } }, exit() { throw new Error('unexpected exit') } },
|
|
require() { return { S3Client: class { async send(command) { assert.equal(command.input.Bucket, 'gongxiangpan'); calls++ } destroy() { destroyed = true } }, HeadBucketCommand: class { constructor(input) { this.input = input } } } },
|
|
})
|
|
assert.equal(calls, 1)
|
|
assert.equal(destroyed, true)
|
|
}
|
|
})
|
|
|
|
test('direct rollback restores an absent or partial replacement from the immutable image', () => {
|
|
const rollback = shellFunction('restore_direct_previous_container')
|
|
const image = `sha256:${'c'.repeat(64)}`
|
|
for (const scenario of [
|
|
{ present: 0, removeFails: 0, status: 0, removed: false },
|
|
{ present: 1, removeFails: 0, status: 0, removed: true },
|
|
{ present: 1, removeFails: 1, status: 1, removed: true },
|
|
]) {
|
|
const input = `set -u
|
|
present=${scenario.present}
|
|
remove_fails=${scenario.removeFails}
|
|
removed=0
|
|
docker() {
|
|
case "$1" in
|
|
container) [ "$present" = 1 ] ;;
|
|
rm) removed=1; [ "$remove_fails" = 0 ] ;;
|
|
start) return 0 ;;
|
|
*) return 90 ;;
|
|
esac
|
|
}
|
|
create_container() { printf 'created:%s:%s\\n' "$1" "$2"; }
|
|
verify_container_uploads_mount_metadata() { return 0; }
|
|
verify_container_upload_dir_config() { return 0; }
|
|
verify_container_uploads_storage() { return 0; }
|
|
wait_for_container_health() { printf '172.18.0.9\\n'; }
|
|
write_proxy_config() { printf 'proxy:%s\\n' "$1"; }
|
|
switch_proxy() { return 0; }
|
|
verify_proxy_target() { printf 'proxy-verified\\n'; }
|
|
write_current_container() { return 0; }
|
|
resume_video_transcode() { printf 'video-resumed\\n'; }
|
|
resume_task_detail_upload() { printf 'upload-resumed\\n'; }
|
|
${rollback}
|
|
restore_direct_previous_container material-blue '${image}'
|
|
status=$?
|
|
printf 'removed:%s\\n' "$removed" >&2
|
|
exit "$status"
|
|
`
|
|
const result = spawnSync(bash, ['-s'], { input, encoding: 'utf8' })
|
|
assert.equal(result.status, scenario.status, result.stderr || result.stdout)
|
|
assert.equal(result.stderr.includes('removed:1'), scenario.removed)
|
|
if (scenario.status === 0) {
|
|
assert.match(result.stdout, new RegExp(`created:material-blue:${image}`))
|
|
assert.match(result.stdout, /proxy:172\.18\.0\.9:3000/)
|
|
assert.ok(result.stdout.indexOf('proxy-verified') < result.stdout.indexOf('video-resumed'))
|
|
assert.ok(result.stdout.indexOf('proxy-verified') < result.stdout.indexOf('upload-resumed'))
|
|
} else assert.doesNotMatch(result.stdout, /created:|proxy:/)
|
|
}
|
|
const invalid = spawnSync(bash, ['-s'], { input: `docker() { echo docker-was-called; }\n${rollback}\nrestore_direct_previous_container material-blue mutable:latest\n`, encoding: 'utf8' })
|
|
assert.equal(invalid.status, 1)
|
|
assert.doesNotMatch(invalid.stdout, /docker-was-called/)
|
|
const direct = shellFunction('direct_replace_active_container')
|
|
assert.match(direct, /previous_image="\$\(docker inspect -f '\{\{\.Image\}\}'/)
|
|
assert.doesNotMatch(direct, /\.Config\.Image/)
|
|
})
|
|
|
|
function runWorkerReadiness({ afterSleep = '', video = '', upload = '', identity = '', extra = '' } = {}) {
|
|
const directory = mkdtempSync(path.join(os.tmpdir(), 'gongxiang-worker-gate-'))
|
|
const quote = value => `'${value.replace(/'/g, `'\\''`)}'`
|
|
const videoFile = path.join(directory, 'video.v1'), uploadFile = path.join(directory, 'upload.v1')
|
|
if (video) writeFileSync(videoFile, video)
|
|
if (upload) writeFileSync(uploadFile, upload)
|
|
const input = `set -u
|
|
COMMIT_SHA=${'a'.repeat(40)}
|
|
expected_instance="material-blue-$COMMIT_SHA-new-start"
|
|
docker_identity='${'b'.repeat(64)}|true'
|
|
video_file=${quote(videoFile.replaceAll('\\', '/'))}
|
|
upload_file=${quote(uploadFile.replaceAll('\\', '/'))}
|
|
sleeps=0
|
|
docker() {
|
|
if [ "$1" != inspect ]; then echo unexpected-container-mutation >&2; return 90; fi
|
|
printf '%s|%s|%s\\n' "$docker_identity" "$expected_instance" "$expected_instance"
|
|
}
|
|
timeout() { shift 3; "$@"; }
|
|
container_transcode_instance_id() { printf '%s\\n' "$expected_instance"; }
|
|
container_task_detail_instance_id() { printf '%s\\n' "$expected_instance"; }
|
|
transcode_capability_file() { printf '%s\\n' "$video_file"; }
|
|
task_detail_capability_file() { printf '%s\\n' "$upload_file"; }
|
|
sleep() { sleeps=$((sleeps+1)); SECONDS=$((SECONDS+$1)); ${afterSleep || ':'}; }
|
|
${shellFunction('verify_video_transcode_capability')}
|
|
${shellFunction('verify_task_detail_upload_capability')}
|
|
${shellFunction('direct_worker_readiness_identity')}
|
|
${shellFunction('wait_for_direct_worker_capabilities')}
|
|
publish_video() { printf 'VIDEO_TRANSCODE_DRAIN_PROTOCOL_V1\\nworkerId=material-blue\\ninstanceId=%s\\n' "$expected_instance" > "$video_file"; }
|
|
publish_upload() { printf 'TASK_DETAIL_UPLOAD_DRAIN_PROTOCOL_V1\\nworkerId=material-blue\\ninstanceId=%s\\n' "$expected_instance" > "$upload_file"; }
|
|
${identity ? `docker_identity=${quote(identity)}` : ''}
|
|
${extra}
|
|
wait_for_direct_worker_capabilities material-blue
|
|
result=$?
|
|
printf 'result=%s sleeps=%s\\n' "$result" "$sleeps"
|
|
exit "$result"
|
|
`
|
|
try { return spawnSync(bash, ['-s'], { input, encoding: 'utf8', timeout: 15000 }) }
|
|
finally {
|
|
assert.equal(path.dirname(path.resolve(directory)), path.resolve(os.tmpdir()))
|
|
assert.ok(path.basename(directory).startsWith('gongxiang-worker-gate-'))
|
|
rmSync(directory, { recursive: true })
|
|
}
|
|
}
|
|
|
|
test('direct worker readiness waits for both matching new-instance markers', () => {
|
|
const result = runWorkerReadiness({ afterSleep: 'if [ "$sleeps" = 1 ]; then publish_video; else publish_upload; fi' })
|
|
assert.equal(result.status, 0, result.stderr || result.stdout)
|
|
assert.match(result.stdout, /result=0 sleeps=2/)
|
|
})
|
|
|
|
test('direct worker readiness rejects old-instance markers until its bounded timeout', () => {
|
|
const result = runWorkerReadiness({ video: 'VIDEO_TRANSCODE_DRAIN_PROTOCOL_V1\nworkerId=material-blue\ninstanceId=old-start\n', upload: 'TASK_DETAIL_UPLOAD_DRAIN_PROTOCOL_V1\nworkerId=material-blue\ninstanceId=old-start\n' })
|
|
assert.equal(result.status, 1, result.stderr || result.stdout)
|
|
const sleeps = Number(result.stdout.match(/result=1 sleeps=(\d+)/)?.[1])
|
|
assert.ok(sleeps > 0 && sleeps <= 30, result.stdout)
|
|
assert.doesNotMatch(result.stderr, /unexpected-container-mutation/)
|
|
})
|
|
|
|
test('direct worker readiness never restarts or repeats deployment when markers stay absent', () => {
|
|
const result = runWorkerReadiness()
|
|
assert.equal(result.status, 1, result.stderr || result.stdout)
|
|
const sleeps = Number(result.stdout.match(/result=1 sleeps=(\d+)/)?.[1])
|
|
assert.ok(sleeps > 0 && sleeps <= 30, result.stdout)
|
|
assert.doesNotMatch(result.stderr, /unexpected-container-mutation/)
|
|
})
|
|
|
|
test('direct worker readiness fails immediately when the candidate exits or its container changes', () => {
|
|
for (const identity of [`${'b'.repeat(64)}|false`, `${'c'.repeat(64)}|true`]) {
|
|
const result = runWorkerReadiness({ afterSleep: `docker_identity='${identity}'` })
|
|
assert.equal(result.status, 1, result.stderr || result.stdout)
|
|
assert.match(result.stdout, /result=1 sleeps=1/)
|
|
}
|
|
const deadBeforeWait = runWorkerReadiness({ identity: `${'b'.repeat(64)}|false` })
|
|
assert.equal(deadBeforeWait.status, 1)
|
|
assert.match(deadBeforeWait.stdout, /sleeps=0/)
|
|
})
|
|
|
|
test('direct worker readiness rejects previous-commit instance identity before waiting', () => {
|
|
const result = runWorkerReadiness({ extra: 'expected_instance=material-blue-previous-commit-start' })
|
|
assert.equal(result.status, 1, result.stderr || result.stdout)
|
|
assert.match(result.stdout, /sleeps=0/)
|
|
})
|
|
|
|
test('direct worker readiness rejects duplicate candidate instance metadata', () => {
|
|
const result = runWorkerReadiness({ extra: `docker() { printf '%s|%s|%s|%s\\n' "$docker_identity" "$expected_instance" "$expected_instance" "$expected_instance"; }` })
|
|
assert.equal(result.status, 1, result.stderr || result.stdout)
|
|
assert.match(result.stdout, /sleeps=0/)
|
|
})
|
|
|
|
test('container construction replaces duplicated worker runtime keys once and preserves user configuration', () => {
|
|
const directory = mkdtempSync(path.join(os.tmpdir(), 'gongxiang-worker-gate-'))
|
|
const quote = value => `'${value.replace(/'/g, `'\\''`)}'`
|
|
const inputFile = path.join(directory, 'original.env'), sanitized = path.join(directory, 'sanitized.env'), captured = path.join(directory, 'captured.env')
|
|
const runtimeKeys = ['VIDEO_TRANSCODE_WORKER_ID', 'VIDEO_TRANSCODE_INSTANCE_ID', 'VIDEO_TRANSCODE_RUNTIME_DIR', 'TASK_DETAIL_UPLOAD_WORKER_ID', 'TASK_DETAIL_UPLOAD_INSTANCE_ID', 'TASK_DETAIL_UPLOAD_RUNTIME_DIR', 'DEPLOYMENT_VERSION']
|
|
const originals = runtimeKeys.flatMap(key => [`${key}=old-first`, `${key}=old-last`])
|
|
const preserved = ['DATABASE_URL=synthetic-private-database', 'DETAIL_GENERATOR_API_KEY=synthetic-private-model-key', 'VIDEO_TRANSCODE_MAX_CONCURRENT=1', 'CUSTOM_USER_SETTING=preserve-exact-value']
|
|
const original = [...originals, ...preserved].join('\n') + '\n'
|
|
writeFileSync(inputFile, original); writeFileSync(sanitized, '')
|
|
const script = `set -eu
|
|
ENV_FILE_PATH=${quote(inputFile.replaceAll('\\', '/'))}
|
|
sanitized_fixture=${quote(sanitized.replaceAll('\\', '/'))}
|
|
captured_fixture=${quote(captured.replaceAll('\\', '/'))}
|
|
COMMIT_SHA=${'a'.repeat(40)}
|
|
CONTAINER_NAME=material
|
|
NETWORK_NAME=network
|
|
UPLOADS_MOUNT=/synthetic/uploads
|
|
DETAIL_FEATURE_ENABLED=false
|
|
PUBLIC_APP_URL=https://example.invalid
|
|
CORS_ALLOWED_ORIGINS=https://example.invalid
|
|
DINGTALK_CALLBACK_ORIGIN=https://example.invalid
|
|
DINGTALK_INTERNAL_RETURN_ORIGINS=https://example.invalid
|
|
S3_PUBLIC_DOMAIN=https://example.invalid/bucket
|
|
DOWNLOAD_TOKEN_SECRET=''
|
|
mktemp() { printf '%s\\n' "$sanitized_fixture"; }
|
|
date() { printf '1234567890\\n'; }
|
|
docker() {
|
|
[ "$1" = create ] || return 90
|
|
shift
|
|
while [ "$#" -gt 0 ]; do
|
|
case "$1" in
|
|
--env-file) cat "$2" >> "$captured_fixture"; shift 2 ;;
|
|
-e) printf '%s\\n' "$2" >> "$captured_fixture"; shift 2 ;;
|
|
*) shift ;;
|
|
esac
|
|
done
|
|
}
|
|
${shellFunction('create_container')}
|
|
create_container material-blue local:fixture 0
|
|
`
|
|
try {
|
|
const result = spawnSync(bash, ['-s'], { input: script, encoding: 'utf8' })
|
|
assert.equal(result.status, 0, result.stderr || result.stdout)
|
|
const lines = readFileSync(captured, 'utf8').trimEnd().split('\n')
|
|
for (const key of runtimeKeys) {
|
|
const values = lines.filter(line => line.startsWith(`${key}=`))
|
|
assert.equal(values.length, 1, `${key} is duplicated`)
|
|
assert.doesNotMatch(values[0], /old-first|old-last/)
|
|
}
|
|
for (const line of preserved) assert.ok(lines.includes(line), 'a user-owned configuration was lost')
|
|
assert.equal(readFileSync(inputFile, 'utf8'), original, 'original server env must remain unchanged')
|
|
const byKey = Object.fromEntries(lines.map(line => [line.split('=', 1)[0], line.slice(line.indexOf('=') + 1)]))
|
|
assert.equal(byKey.VIDEO_TRANSCODE_INSTANCE_ID, byKey.TASK_DETAIL_UPLOAD_INSTANCE_ID)
|
|
assert.match(byKey.VIDEO_TRANSCODE_INSTANCE_ID, new RegExp(`^material-blue-${'a'.repeat(40)}-1234567890-\\d+$`))
|
|
assert.doesNotMatch(result.stdout + result.stderr, /synthetic-private/)
|
|
} finally {
|
|
assert.equal(path.dirname(path.resolve(directory)), path.resolve(os.tmpdir()))
|
|
assert.ok(path.basename(directory).startsWith('gongxiang-worker-gate-'))
|
|
rmSync(directory, { recursive: true })
|
|
}
|
|
})
|
|
|
|
test('direct failure retains diagnostics before exactly one existing rollback, even if capture fails', () => {
|
|
for (const captureStatus of [0, 1]) {
|
|
const input = `set -u
|
|
warn() { :; }
|
|
fail() { printf 'failed\\n'; exit 1; }
|
|
capture_direct_candidate_diagnostics() { printf 'diagnostics:%s\\n' "$1"; return ${captureStatus}; }
|
|
restore_direct_previous_container() { printf 'rollback:%s:%s\\n' "$1" "$2"; return 0; }
|
|
${shellFunction('fail_direct_replace_with_restore')}
|
|
fail_direct_replace_with_restore 'worker gate timeout' material-blue immutable-original
|
|
`
|
|
const result = spawnSync(bash, ['-s'], { input, encoding: 'utf8' })
|
|
assert.equal(result.status, 1, result.stderr || result.stdout)
|
|
assert.ok(result.stdout.indexOf('diagnostics:') < result.stdout.indexOf('rollback:'))
|
|
assert.equal(result.stdout.match(/rollback:/g)?.length, 1)
|
|
assert.match(result.stdout, /rollback:material-blue:immutable-original/)
|
|
}
|
|
const direct = shellFunction('direct_replace_active_container')
|
|
assert.ok(direct.indexOf('wait_for_direct_worker_capabilities') < direct.indexOf('verify_video_transcode_capability'))
|
|
assert.match(direct, /fail_direct_replace_with_restore "New direct container workers did not become ready/)
|
|
const diagnostics = shellFunction('capture_direct_candidate_diagnostics')
|
|
assert.match(diagnostics, /target\.parent\.name=='native-final-'\+commit/)
|
|
assert.match(diagnostics, /mode=0o700/)
|
|
assert.match(diagnostics, /os\.O_EXCL,0o600/)
|
|
assert.match(diagnostics, /'--tail','400'/)
|
|
assert.doesNotMatch(diagnostics, /print\(.*(?:report|value|logs)/)
|
|
})
|
|
|
|
test('private candidate diagnostics compiles and excludes unrelated credentials from retained state', () => {
|
|
const source = shellFunction('capture_direct_candidate_diagnostics').match(/<<'PY'\n([\s\S]*?)\nPY/)[1]
|
|
const python = process.platform === 'win32' ? 'python' : 'python3'
|
|
const compile = spawnSync(python, ['-c', 'import sys; compile(sys.stdin.read(), "diagnostics", "exec")'], { input: source, encoding: 'utf8' })
|
|
assert.equal(compile.status, 0, compile.stderr || compile.error?.message)
|
|
const allowed = source.match(/ allowed=(.+)\n/)[1]
|
|
const assignment = source.match(/ report\['candidate'\]=(.+)\n/)[1]
|
|
const fixture = { Id: 'candidate', Image: 'sha256:fixture', Created: 'now', Config: { Image: 'local:fixture', Env: ['VIDEO_TRANSCODE_WORKER_ID=material-blue', 'VIDEO_TRANSCODE_INSTANCE_ID=expected-instance', 'DATABASE_URL=synthetic-do-not-retain-password', 'DETAIL_GENERATOR_API_KEY=synthetic-do-not-retain-key'] }, State: { Status: 'running', Running: true, Error: 'synthetic-do-not-retain-error', StartedAt: 'now' } }
|
|
const input = `import json\nvalue=json.loads(${JSON.stringify(JSON.stringify(fixture))})\nallowed=${allowed}\nreport={'candidate':${assignment}}\nprint(json.dumps(report))\n`
|
|
const result = spawnSync(python, ['-'], { input, encoding: 'utf8' })
|
|
assert.equal(result.status, 0, result.stderr || result.error?.message)
|
|
const candidate = JSON.parse(result.stdout).candidate
|
|
assert.deepEqual(candidate.environment, { VIDEO_TRANSCODE_WORKER_ID: ['material-blue'], VIDEO_TRANSCODE_INSTANCE_ID: ['expected-instance'] })
|
|
assert.doesNotMatch(result.stdout + result.stderr, /synthetic-do-not-retain/)
|
|
})
|
|
|
|
test('detail beta configuration validates enabled key/base without exposing provider secrets', () => {
|
|
const source = shellFunction('prepare_detail_generator_release')
|
|
const code = source.match(/<<'NODE'\n([\s\S]*?)\nNODE/)[1]
|
|
const env = { ...process.env, DETAIL_GENERATOR_ENABLED: 'true', DETAIL_GENERATOR_API_KEY: 'synthetic-private-key', DETAIL_GENERATOR_BASE_URL: 'https://model.example.invalid/v1', DETAIL_GENERATOR_STORAGE_DIR: '/app/detail-generator-private' }
|
|
for (const input of [
|
|
{ env, status: 0, output: 'true' },
|
|
{ env: { ...env, DETAIL_GENERATOR_API_KEY: '' }, status: 1, output: '' },
|
|
{ env: { ...env, DETAIL_GENERATOR_BASE_URL: '' }, status: 1, output: '' },
|
|
{ env: { ...env, DETAIL_GENERATOR_BASE_URL: 'https://user:synthetic-private-key@model.example.invalid' }, status: 1, output: '' },
|
|
{ env: { ...env, DETAIL_GENERATOR_ENABLED: '', DETAIL_GENERATOR_API_KEY: '' }, status: 0, output: 'false' },
|
|
{ env: { ...env, DETAIL_GENERATOR_ENABLED: 'false', DETAIL_GENERATOR_API_KEY: '' }, status: 0, output: 'false' },
|
|
]) {
|
|
const result = spawnSync(process.execPath, ['-e', code], { env: input.env, encoding: 'utf8' })
|
|
assert.equal(result.status, input.status)
|
|
assert.equal(result.stdout, input.output)
|
|
assert.doesNotMatch(result.stdout + result.stderr, /synthetic-private-key/)
|
|
}
|
|
assert.match(source, /--env-file "\$ENV_FILE_PATH"/)
|
|
assert.doesNotMatch(source, /-e [^\n]*DETAIL_GENERATOR_API_KEY/)
|
|
})
|
|
|
|
test('fixed private storage rejects overlapping uploads before creating directories', () => {
|
|
const check = shellFunction('detail_private_mount_is_safe')
|
|
const privateRoot = '/mnt/PoolOne/apps/youyidian-3d/detail-generator-private'
|
|
for (const upload of [privateRoot, '/mnt/PoolOne/apps/youyidian-3d', `${privateRoot}/uploads`]) {
|
|
const result = spawnSync(bash, ['-s'], { input: `DETAIL_PRIVATE_MOUNT='${privateRoot}'\nUPLOADS_MOUNT='${upload}'\n${check}\ndetail_private_mount_is_safe\n`, encoding: 'utf8' })
|
|
assert.equal(result.status, 1, `overlap accepted: ${upload}`)
|
|
}
|
|
assert.match(check, /while \[ "\$ancestor" != "\/" \]/)
|
|
assert.match(check, /\[ ! -L "\$ancestor" \]/)
|
|
const prepare = shellFunction('prepare_detail_generator_release')
|
|
assert.match(prepare, /install -d -m 0700 -o 1001 -g 1001/)
|
|
assert.match(prepare, /1001:1001:700/)
|
|
})
|
|
|
|
test('enabled containers bind private persistence and verify actual Node22 PSD/PNG without model calls', () => {
|
|
const create = shellFunction('create_container')
|
|
assert.match(create, /if \[ "\$DETAIL_FEATURE_ENABLED" = "true" \]/)
|
|
assert.match(create, /--mount "type=bind,source=\$\{DETAIL_PRIVATE_MOUNT\},target=\/app\/detail-generator-private"/)
|
|
assert.match(create, /--env-file "\$sanitized_env_file"/)
|
|
assert.doesNotMatch(create, /DETAIL_GENERATOR_API_KEY/)
|
|
const schema = shellFunction('verify_detail_generator_schema')
|
|
assert.match(schema, /20261012_detail_generator/)
|
|
assert.match(schema, /completed\[0\]\.checksum !== checksum/)
|
|
for (const table of ['projects', 'assets', 'jobs', 'brand_profiles']) assert.match(schema, new RegExp(`detail_generator_${table}`))
|
|
const runtime = shellFunction('verify_detail_generator_dependencies')
|
|
assert.match(runtime, /\/opt\/detail-renderer\/bin\/node/)
|
|
assert.match(runtime, /!== 22/)
|
|
assert.match(runtime, /renderRequest/)
|
|
assert.match(runtime, /format:'psd'/)
|
|
assert.match(runtime, /format:'png'/)
|
|
assert.match(runtime, /8BPS/)
|
|
assert.match(runtime, /\.deploy-probe-/)
|
|
assert.doesNotMatch(runtime, /fetch\(|generateDetailImage|runDetailGeneratorWorker/)
|
|
})
|
|
|
|
test('AI model cutover gate checks every migration and rejects missing, mistyped or wrongly nullable verification state', async () => {
|
|
const code = shellFunction('verify_detail_generator_schema').match(/<<'NODE'\n([\s\S]*?)\nNODE/)[1]
|
|
const columns = [
|
|
...aiModelRegistryEvidence().columns,
|
|
...Object.entries({
|
|
detail_generator_projects: ['id', 'owner_id', 'document', 'revision', 'deleted_at'],
|
|
detail_generator_assets: ['id', 'project_id', 'owner_id', 'storage_key', 'byte_size'],
|
|
detail_generator_jobs: ['id', 'project_id', 'owner_id', 'status', 'idempotency_key', 'fingerprint', 'project_revision', 'payload', 'checkpoint', 'lease_token', 'lease_expires_at'],
|
|
detail_generator_brand_profiles: ['id', 'name', 'profile', 'revision'],
|
|
}).flatMap(([table_name, names]) => names.map(column_name => ({ table_name, column_name }))),
|
|
]
|
|
const checksum = createHash('sha256').update('schema fixture\n').digest('hex')
|
|
const run = async (changedColumns = columns, missingMigration = '') => {
|
|
const state = { exitCode: 0 }, checked = []
|
|
class PrismaClient {
|
|
async $queryRawUnsafe(sql, name) {
|
|
if (sql.includes('_prisma_migrations')) { checked.push(name); return name === missingMigration ? [] : [{ checksum }] }
|
|
return changedColumns
|
|
}
|
|
async $disconnect() {}
|
|
}
|
|
await runInNewContext(code, { process: state, require: name => ({ fs: { readFileSync: () => 'schema fixture\r\n' }, crypto: { createHash }, '@prisma/client': { PrismaClient } })[name] })
|
|
return { ...state, checked }
|
|
}
|
|
const accepted = await run()
|
|
assert.equal(accepted.exitCode, 0)
|
|
assert.deepEqual(accepted.checked, ['20261012_detail_generator', '20261012_ai_model_registry', '20261013_ai_model_capabilities', '20261013_ai_model_verifications', '20261020090000_desktop_client_workspace', '20261020120000_desktop_creation_tasks', '20261020150000_desktop_project_deletion'])
|
|
for (const migration of ['20261012_ai_model_registry', '20261013_ai_model_capabilities', '20261013_ai_model_verifications', '20261020120000_desktop_creation_tasks', '20261020150000_desktop_project_deletion']) assert.equal((await run(columns, migration)).exitCode, 1)
|
|
assert.equal((await run(columns.filter(row => row.column_name !== 'capability_metadata'))).exitCode, 1)
|
|
assert.equal((await run(columns.map(row => row.column_name === 'capability_metadata' ? { ...row, data_type: 'text' } : row))).exitCode, 1)
|
|
assert.equal((await run(columns.filter(row => row.table_name !== 'ai_model_verifications'))).exitCode, 1, 'missing verification table must block cutover')
|
|
for (const field of ['stage', 'context_hash', 'task_id', 'evidence']) {
|
|
const target = row => row.table_name === 'ai_model_verifications' && row.column_name === field
|
|
assert.equal((await run(columns.filter(row => !target(row)))).exitCode, 1, `missing ${field} must block cutover`)
|
|
assert.equal((await run(columns.map(row => target(row) ? { ...row, data_type: 'boolean' } : row))).exitCode, 1, `mistyped ${field} must block cutover`)
|
|
assert.equal((await run(columns.map(row => target(row) ? { ...row, is_nullable: row.is_nullable === 'YES' ? 'NO' : 'YES' } : row))).exitCode, 1, `wrong nullability for ${field} must block cutover`)
|
|
}
|
|
})
|
|
|
|
test('AI registry migration files are present in builder and immutable runner verification scope', () => {
|
|
const dockerfile = readFileSync(new URL('../Dockerfile', import.meta.url), 'utf8')
|
|
for (const migration of ['20261012_ai_model_registry', '20261013_ai_model_capabilities', '20261013_ai_model_verifications']) {
|
|
assert.ok(dockerfile.includes(`test -f "/app/prisma/migrations/${migration}/migration.sql"`))
|
|
assert.ok(dockerfile.includes(`COPY --from=builder --chown=root:root /app/prisma/migrations/${migration} ./prisma/migrations/${migration}`))
|
|
}
|
|
})
|
|
|
|
test('detail uploads keep multipart headroom without expanding all proxy routes', () => {
|
|
assert.match(deploySource(), /location ~ "\^\/api\/detail-generator\/projects\/\[0-9a-fA-F-\]\{36\}\/assets\\\$" \{\n\s+client_max_body_size 17m;\n\s+proxy_request_buffering off;/)
|
|
})
|
|
|
|
test('desktop private image sync supports the route 20MiB limit without widening other endpoints', () => {
|
|
const source=deploySource()
|
|
const marker='location ~ "^/api/desktop/projects/desktop/[A-Za-z0-9_-]{1,80}/assets/[^/]+\\$" {'
|
|
const start=source.indexOf(marker)
|
|
assert.ok(start>=0)
|
|
const location=source.slice(start,source.indexOf('\n }',start))
|
|
assert.match(location,/client_max_body_size 21m;/)
|
|
assert.match(location,/proxy_request_buffering off;/)
|
|
assert.match(location,/proxy_set_header Host \\\$http_host;/)
|
|
})
|
|
|
|
test('brand knowledge upload stays bounded in the route and stable proxy', () => {
|
|
const deploySource = readFileSync(new URL('./git-incremental-deploy.ps1', import.meta.url), 'utf8')
|
|
.replace(/\r\n/g, '\n')
|
|
const routeSource = readFileSync(
|
|
new URL('../app/api/department-dashboard/brand-knowledge/route.ts', import.meta.url),
|
|
'utf8',
|
|
)
|
|
|
|
assert.match(
|
|
deploySource,
|
|
/location = \/api\/department-dashboard\/brand-knowledge \{\n\s+client_max_body_size 53m;\n\s+proxy_request_buffering off;/,
|
|
)
|
|
|
|
const headerGuard = routeSource.indexOf('assertBoundedMultipartHeaders(')
|
|
const acquireSlot = routeSource.indexOf('boundedMultipartRequestLimiter.acquire()')
|
|
const parseBody = routeSource.indexOf('request.formData()')
|
|
const parsedFiles = routeSource.indexOf('assertMultipartFiles(', parseBody)
|
|
assert.ok(headerGuard > 0)
|
|
assert.ok(acquireSlot > headerGuard)
|
|
assert.ok(parseBody > acquireSlot)
|
|
assert.ok(parsedFiles > parseBody)
|
|
assert.match(routeSource, /BRAND_KNOWLEDGE_MAX_MAIN_FILE_BYTES/)
|
|
assert.match(routeSource, /BRAND_KNOWLEDGE_MAX_REQUEST_BYTES/)
|
|
assert.match(routeSource, /resolveOwnedMergedUpload/)
|
|
assert.match(routeSource, /publishMergedUploadByHardLink/)
|
|
assert.doesNotMatch(routeSource, /formData\.get\('file'\)/)
|
|
assert.match(routeSource, /import \{ isWebFileLike \} from '@\/lib\/web-file'/)
|
|
assert.doesNotMatch(routeSource, /instanceof File/)
|
|
assert.match(routeSource, /releaseRequestSlot\?\.\(\)/)
|
|
})
|