cangku/test/remote-access.test.mjs
yanhui 38fa3f2127 fix: 隧道断线自愈 + 修复配置负缓存永久固化
远端 RST 会把隧道反复踢掉(09-20 实测当天断 5 次,每次存活 15~25 分钟)。原先的
实现把「一次网络抖动」放大成「整个开发栈不可用」,根因有两处,本次一并修掉。

1) scripts/start-remote.mjs —— 隧道断了却拉着应用陪葬

   隧道进程 exit 时直接 stopChildren(),把后端和前端一起杀掉,于是每一次 RST 都
   升级成一次「网站打不开」,还得靠外部守护清残留、整树重启才能恢复。

   改为「退避重连隧道本身,后端/前端进程不受影响」——这正是 autossh 的语义
   (autossh 只负责重启 ssh,从不牵连被它服务的应用)。建立失败走 1s→30s 指数
   退避,建立成功即重置退避。

   顺带补齐保活参数:TCPKeepAlive=yes、ConnectTimeout=10,探活间隔 30s → 15s,
   以便更快发现已被对端 RST 的死连接。

2) backend/src/config/cloudDatabase.js —— 配置负缓存被永久固化

   系统库读取失败时把「空候选」写进缓存,而入口判断是
   `if (cachedCloudConfigCandidates)` —— [] 在 JS 里是 truthy,于是这次失败被
   永久固化:之后永不重试,直接抛「业务数据库尚未配置」500,隧道恢复了也没用,
   必须重启后端。

   改为「带 TTL 的负缓存」(默认 30s,可用 CLOUD_DB_CONFIG_NEGATIVE_TTL_MS 调整):
   入口用 `!== null` 判断,失败态到期自动重新解析,使隧道恢复后无需人工干预。
   缓存清理收敛到 invalidateCloudConfigCache() 单点,避免日后只清数组、忘了清
   TTL 时间戳而退化回「永不过期」。同时让「测试连接」入口显式失效,保证用户改完
   配置点测试能立即看到真实结果。

   顺带把请求超时默认值 600000ms → 60000ms:10 分钟 × maxAttempts=2 最坏挂
   20 分钟,用户体感就是「卡死」;冷查实测 16~42s,60s 仍有充足余量。

验证:
- 新增 backend/test/cloudDatabaseConfigCache.test.js,锁定负缓存三段语义
  (TTL 内不重复解析 / 到期必须重试 / 上游恢复后自动复原且成功态不被 TTL 淘汰);
  test/remote-access.test.mjs 补隧道保活参数、退避曲线,以及「自愈循环不得调用
  stopChildren」的防腐化断言。
- 两处均做对照组实验:去掉保护后测试如实报错(负缓存 → 报 TTL 到期未重试;
  隧道 → 把 stopChildren 加回去即被用例抓住),证明用例真能拦住回归。
- 后端全量 340 项 / 339 通过,唯一失败为 resilientSecretStore 的 Windows 权限
  断言(环境固有,与本次无关);check-syntax 通过。
- 本地实测:kill 掉隧道进程后 1 秒内自动重建(PID 36268 → 15816),后端进程
  PID 未变(29580)、/health 仍返回 200。
2026-09-22 19:32:40 +08:00

131 lines
5.3 KiB
JavaScript
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

import assert from 'node:assert/strict';
import { readFile } from 'node:fs/promises';
import test from 'node:test';
import { getKnownHostsLine, mergeEnv, validateEnrollmentResponse } from '../scripts/remote-access-common.mjs';
import { buildSshArgs, computeTunnelRetryDelay } from '../scripts/start-remote.mjs';
const enrollment = {
installation_id: 'dev-123',
ssh: {
host: '203.0.113.10',
port: 22,
user: 'tunnel',
certificate: 'ssh-ed25519-cert-v01@openssh.com AAAATEST cert',
host_key: 'ssh-ed25519 AAAATEST host',
},
database: {
name: 'system_db',
username: 'client_123',
password: 'a password with # symbols',
},
redis: {
host: 'redis.example.test',
port: 6380,
username: 'cangku_1234567890abcdef',
password: 'redis secret',
db: 0,
tls: true,
tls_server_name: 'redis.example.test',
},
application_admin: {
username: 'dev_123',
password: 'one-time-password',
},
};
test('validates complete enrollment responses', () => {
assert.equal(validateEnrollmentResponse(enrollment), enrollment);
assert.throws(
() => validateEnrollmentResponse({ ...enrollment, database: { ...enrollment.database, password: '' } }),
/database\.password/u
);
});
test('builds an SSH tunnel only for the system database', () => {
const args = buildSshArgs({
localDatabasePort: 13307,
ssh: {
host: 'example.test',
port: 50232,
user: 'cangku-tunnel',
remoteDatabaseHost: '127.0.0.1',
remoteDatabasePort: 3307,
},
});
assert.ok(args.includes('127.0.0.1:13307:127.0.0.1:3307'));
assert.equal(args.some((value) => String(value).includes(':6380')), false);
});
test('merges environment values without losing unrelated settings', () => {
const result = mergeEnv('KEEP=yes\nLOCAL_DB_HOST=old\n', {
LOCAL_DB_HOST: '127.0.0.1',
LOCAL_DB_PASSWORD: 'secret # value',
});
assert.match(result, /^KEEP=yes$/mu);
assert.match(result, /^LOCAL_DB_HOST=127\.0\.0\.1$/mu);
assert.match(result, /^LOCAL_DB_PASSWORD="secret # value"$/mu);
});
test('pins standard and non-standard SSH host targets', () => {
assert.equal(getKnownHostsLine('example.test', 22, 'ssh-ed25519 KEY'), 'example.test ssh-ed25519 KEY\n');
assert.equal(getKnownHostsLine('example.test', 2222, 'ssh-ed25519 KEY'), '[example.test]:2222 ssh-ed25519 KEY\n');
});
// ─────────────────────────────────────────────────────────────────────────────
// 隧道自愈(根治远端 RST 导致「网站打不开」)
//
// 旧行为:隧道进程 exit → stopChildren(),把后端 + 前端一起杀掉,于是每一次远端
// RST 都升级成一次整栈不可用,还得靠外部守护清残留、整树重启。
// 新行为:隧道 exit → 只重连隧道本身,应用进程不受影响。
// ─────────────────────────────────────────────────────────────────────────────
const tunnelConfig = {
localDatabasePort: 13307,
ssh: {
host: 'example.test',
port: 50232,
user: 'cangku-tunnel',
remoteDatabaseHost: '127.0.0.1',
remoteDatabasePort: 3307,
},
};
test('tunnel keepalive: 更快发现死连接,且不会在 connect 阶段无限等待', () => {
const flags = new Set(buildSshArgs(tunnelConfig));
// ExitOnForwardFailure 缺失会出现最阴险的故障:进程活着、端口却没绑上(「静默成功」)
assert.ok(flags.has('ExitOnForwardFailure=yes'), '必须保留 ExitOnForwardFailure=yes');
assert.ok(flags.has('ServerAliveInterval=15'), '探活间隔应为 15s,以便更快发现被 RST 的连接');
assert.ok(flags.has('ServerAliveCountMax=3'));
assert.ok(flags.has('TCPKeepAlive=yes'), '需要内核层保活兜底');
assert.ok(flags.has('ConnectTimeout=10'), '连不上时应快速失败,交给退避重连而不是干等');
});
test('tunnel retry backoff: 1s 起指数增长,封顶 30s 且不会溢出', () => {
assert.equal(computeTunnelRetryDelay(0), 1000);
assert.equal(computeTunnelRetryDelay(1), 2000);
assert.equal(computeTunnelRetryDelay(2), 4000);
assert.equal(computeTunnelRetryDelay(4), 16000);
assert.equal(computeTunnelRetryDelay(5), 30000);
// 长期不可用时 attempt 会一路增长,必须封顶而不是算成 Infinity / 0
assert.equal(computeTunnelRetryDelay(200), 30000);
assert.ok(Number.isFinite(computeTunnelRetryDelay(2000)));
});
test('防腐化:隧道自愈循环不得停止后端/前端进程', async () => {
const source = await readFile(new URL('../scripts/start-remote.mjs', import.meta.url), 'utf8');
const start = source.indexOf('const superviseTunnel = async () => {');
const end = source.indexOf('const startTunnelSupervisor = () => {');
assert.ok(start > 0 && end > start, '未定位到 superviseTunnel 实现');
const body = source.slice(start, end);
assert.equal(
body.includes('stopChildren()'),
false,
'隧道重连不得牵连后端/前端 —— 否则又回到「隧道一断,整个开发栈陪葬」'
);
assert.ok(
body.includes('await sleep('),
'隧道退出后必须先等待再重连,避免 RST 时疯狂重连打爆对端'
);
});