远端 RST 会把隧道反复踢掉(09-20 实测当天断 5 次,每次存活 15~25 分钟)。原先的 实现把「一次网络抖动」放大成「整个开发栈不可用」,根因有两处,本次一并修掉。 1) scripts/start-remote.mjs —— 隧道断了却拉着应用陪葬 隧道进程 exit 时直接 stopChildren(),把后端和前端一起杀掉,于是每一次 RST 都 升级成一次「网站打不开」,还得靠外部守护清残留、整树重启才能恢复。 改为「退避重连隧道本身,后端/前端进程不受影响」——这正是 autossh 的语义 (autossh 只负责重启 ssh,从不牵连被它服务的应用)。建立失败走 1s→30s 指数 退避,建立成功即重置退避。 顺带补齐保活参数:TCPKeepAlive=yes、ConnectTimeout=10,探活间隔 30s → 15s, 以便更快发现已被对端 RST 的死连接。 2) backend/src/config/cloudDatabase.js —— 配置负缓存被永久固化 系统库读取失败时把「空候选」写进缓存,而入口判断是 `if (cachedCloudConfigCandidates)` —— [] 在 JS 里是 truthy,于是这次失败被 永久固化:之后永不重试,直接抛「业务数据库尚未配置」500,隧道恢复了也没用, 必须重启后端。 改为「带 TTL 的负缓存」(默认 30s,可用 CLOUD_DB_CONFIG_NEGATIVE_TTL_MS 调整): 入口用 `!== null` 判断,失败态到期自动重新解析,使隧道恢复后无需人工干预。 缓存清理收敛到 invalidateCloudConfigCache() 单点,避免日后只清数组、忘了清 TTL 时间戳而退化回「永不过期」。同时让「测试连接」入口显式失效,保证用户改完 配置点测试能立即看到真实结果。 顺带把请求超时默认值 600000ms → 60000ms:10 分钟 × maxAttempts=2 最坏挂 20 分钟,用户体感就是「卡死」;冷查实测 16~42s,60s 仍有充足余量。 验证: - 新增 backend/test/cloudDatabaseConfigCache.test.js,锁定负缓存三段语义 (TTL 内不重复解析 / 到期必须重试 / 上游恢复后自动复原且成功态不被 TTL 淘汰); test/remote-access.test.mjs 补隧道保活参数、退避曲线,以及「自愈循环不得调用 stopChildren」的防腐化断言。 - 两处均做对照组实验:去掉保护后测试如实报错(负缓存 → 报 TTL 到期未重试; 隧道 → 把 stopChildren 加回去即被用例抓住),证明用例真能拦住回归。 - 后端全量 340 项 / 339 通过,唯一失败为 resilientSecretStore 的 Windows 权限 断言(环境固有,与本次无关);check-syntax 通过。 - 本地实测:kill 掉隧道进程后 1 秒内自动重建(PID 36268 → 15816),后端进程 PID 未变(29580)、/health 仍返回 200。
291 lines
9.7 KiB
JavaScript
291 lines
9.7 KiB
JavaScript
import fs from 'node:fs';
|
||
import net from 'node:net';
|
||
import path from 'node:path';
|
||
import process from 'node:process';
|
||
import { spawn } from 'node:child_process';
|
||
import { fileURLToPath } from 'node:url';
|
||
import { loadConnectionConfig } from './remote-access-common.mjs';
|
||
|
||
const ROOT_DIR = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..');
|
||
const ACCESS_DIR = path.join(ROOT_DIR, '.remote-access');
|
||
const CONNECTION_PATH = path.join(ACCESS_DIR, 'connection.json');
|
||
const RESTART_MARKER_PATH = path.join(ACCESS_DIR, 'restart-requested.json');
|
||
|
||
/**
|
||
* 解析后端实际监听端口。
|
||
* 优先级:CANGKU_BACKEND_PORT 环境变量 > backend/.env 的 PORT > 默认 4001。
|
||
* 前端 dev server 的 /api 代理必须指向同一端口,否则请求会被转发到空端口。
|
||
*/
|
||
function resolveBackendPort() {
|
||
const fromEnv = Number(process.env.CANGKU_BACKEND_PORT);
|
||
if (Number.isInteger(fromEnv) && fromEnv > 0 && fromEnv < 65536) {
|
||
return fromEnv;
|
||
}
|
||
|
||
const envPath = path.join(ROOT_DIR, 'backend', '.env');
|
||
if (fs.existsSync(envPath)) {
|
||
const matched = fs.readFileSync(envPath, 'utf8').match(/^PORT=(\d+)\s*$/mu);
|
||
const fromFile = Number(matched?.[1]);
|
||
if (Number.isInteger(fromFile) && fromFile > 0 && fromFile < 65536) {
|
||
return fromFile;
|
||
}
|
||
}
|
||
|
||
return 4001;
|
||
}
|
||
|
||
/**
|
||
* 隧道重连退避:1s → 2s → 4s … 封顶 30s。
|
||
* 首次失败也立即重试(等价 autossh 的 AUTOSSH_GATETIME=0),避免网络尚未就绪时白等。
|
||
*/
|
||
const TUNNEL_RETRY_BASE_MS = 1000;
|
||
const TUNNEL_RETRY_MAX_MS = 30000;
|
||
const TUNNEL_RETRY_EXPONENT_CAP = 10;
|
||
|
||
function sleep(ms) {
|
||
return new Promise((resolve) => setTimeout(resolve, ms));
|
||
}
|
||
|
||
export function computeTunnelRetryDelay(attempt) {
|
||
const exponent = Math.min(attempt, TUNNEL_RETRY_EXPONENT_CAP);
|
||
return Math.min(TUNNEL_RETRY_BASE_MS * 2 ** exponent, TUNNEL_RETRY_MAX_MS);
|
||
}
|
||
|
||
export function buildSshArgs(config) {
|
||
return [
|
||
'-N', '-T',
|
||
'-o', 'BatchMode=yes',
|
||
// 远端仍可能主动 RST(中间设备/服务端空闲超时),本地探活只能更快发现,不能阻止;
|
||
// 真正的根治是下面的退避重连循环。
|
||
'-o', 'ExitOnForwardFailure=yes',
|
||
'-o', 'ServerAliveInterval=15',
|
||
'-o', 'ServerAliveCountMax=3',
|
||
'-o', 'TCPKeepAlive=yes',
|
||
'-o', 'ConnectTimeout=10',
|
||
'-o', 'StrictHostKeyChecking=yes',
|
||
'-o', `UserKnownHostsFile=${path.join(ACCESS_DIR, 'known_hosts')}`,
|
||
'-o', `CertificateFile=${path.join(ACCESS_DIR, 'id_ed25519-cert.pub')}`,
|
||
'-i', path.join(ACCESS_DIR, 'id_ed25519'),
|
||
'-p', String(config.ssh.port),
|
||
'-L', `127.0.0.1:${config.localDatabasePort}:${config.ssh.remoteDatabaseHost}:${config.ssh.remoteDatabasePort}`,
|
||
`${config.ssh.user}@${config.ssh.host}`,
|
||
];
|
||
}
|
||
|
||
function waitForPort(port, timeoutMs = 60000) {
|
||
const started = Date.now();
|
||
return new Promise((resolve, reject) => {
|
||
const attempt = () => {
|
||
const socket = net.createConnection({ host: '127.0.0.1', port });
|
||
socket.once('connect', () => {
|
||
socket.destroy();
|
||
resolve();
|
||
});
|
||
socket.once('error', () => {
|
||
socket.destroy();
|
||
if (Date.now() - started >= timeoutMs) {
|
||
reject(new Error(`SSH 隧道未能在 ${timeoutMs / 1000} 秒内监听端口 ${port}`));
|
||
return;
|
||
}
|
||
setTimeout(attempt, 250);
|
||
});
|
||
};
|
||
attempt();
|
||
});
|
||
}
|
||
|
||
function waitForFile(filePath) {
|
||
return new Promise((resolve) => {
|
||
const timer = setInterval(() => {
|
||
if (fs.existsSync(filePath)) {
|
||
clearInterval(timer);
|
||
resolve();
|
||
}
|
||
}, 500);
|
||
});
|
||
}
|
||
|
||
function waitForExit(child, timeoutMs = 5000) {
|
||
if (child.exitCode !== null || child.signalCode !== null) return Promise.resolve();
|
||
return Promise.race([
|
||
new Promise((resolve) => child.once('exit', resolve)),
|
||
new Promise((resolve) => setTimeout(resolve, timeoutMs)),
|
||
]);
|
||
}
|
||
|
||
export async function main(mode = process.argv[2] || 'tunnel') {
|
||
if (!['tunnel', 'dev', 'backend'].includes(mode)) {
|
||
throw new Error(`未知启动模式:${mode}`);
|
||
}
|
||
|
||
const children = new Set();
|
||
let shuttingDown = false;
|
||
let restartingBackend = false;
|
||
let backend = null;
|
||
|
||
const stopChildren = () => {
|
||
shuttingDown = true;
|
||
for (const child of children) {
|
||
if (!child.killed) child.kill('SIGTERM');
|
||
}
|
||
};
|
||
for (const signal of ['SIGINT', 'SIGTERM']) {
|
||
process.once(signal, stopChildren);
|
||
}
|
||
|
||
let tunnelRetryAttempt = 0;
|
||
let resolveTunnelReady = null;
|
||
const tunnelReady = new Promise((resolve) => {
|
||
resolveTunnelReady = resolve;
|
||
});
|
||
|
||
/**
|
||
* 启动一条 SSH 隧道并等它监听就绪。
|
||
* 返回 { tunnel, exitPromise };隧道结束由 exitPromise 上报退出码,交给
|
||
* superviseTunnel 决定重连 —— 这里**不再**牵连后端/前端。
|
||
*/
|
||
const startTunnel = async () => {
|
||
const { value: config } = loadConnectionConfig(ROOT_DIR);
|
||
const tunnel = spawn('ssh', buildSshArgs(config), { cwd: ROOT_DIR, stdio: 'inherit' });
|
||
children.add(tunnel);
|
||
|
||
const exitPromise = new Promise((resolve) => {
|
||
tunnel.once('exit', (code) => {
|
||
children.delete(tunnel);
|
||
resolve(code);
|
||
});
|
||
});
|
||
|
||
// 两个分支都 resolve,避免 Promise.race 落败方变成 unhandled rejection
|
||
const outcome = await Promise.race([
|
||
waitForPort(Number(config.localDatabasePort)).then(() => ({ ready: true })),
|
||
exitPromise.then((code) => ({ ready: false, code })),
|
||
]);
|
||
|
||
if (!outcome.ready) {
|
||
if (!tunnel.killed) tunnel.kill('SIGTERM');
|
||
throw new Error(`SSH 隧道在建立前退出(${outcome.code ?? 'unknown'})`);
|
||
}
|
||
|
||
console.log(`系统数据库隧道已建立:127.0.0.1:${config.localDatabasePort}/${config.databaseName}`);
|
||
return { tunnel, exitPromise };
|
||
};
|
||
|
||
/**
|
||
* 隧道自愈循环(Windows 版 autossh 的语义)。
|
||
*
|
||
* 隧道断开只重连隧道本身,后端/前端进程保持不动。这是原先缺失的容错:
|
||
* 旧实现在隧道 exit 时直接 stopChildren(),把整个开发栈一起杀掉,于是每一次
|
||
* 远端 RST 都升级成一次「网站打不开」,还得靠外部守护清残留、整树重启。
|
||
*
|
||
* 建立失败走指数退避(1s→30s);建立成功后重置退避,断开则固定 1s 后重连。
|
||
*/
|
||
const superviseTunnel = async () => {
|
||
for (;;) {
|
||
if (shuttingDown) return;
|
||
|
||
let session;
|
||
try {
|
||
session = await startTunnel();
|
||
tunnelRetryAttempt = 0;
|
||
if (resolveTunnelReady) {
|
||
resolveTunnelReady();
|
||
resolveTunnelReady = null;
|
||
}
|
||
} catch (error) {
|
||
if (shuttingDown) return;
|
||
const delay = computeTunnelRetryDelay(tunnelRetryAttempt);
|
||
tunnelRetryAttempt += 1;
|
||
console.error(`SSH 隧道建立失败(${error.message}),${delay}ms 后重试`);
|
||
await sleep(delay);
|
||
continue;
|
||
}
|
||
|
||
const code = await session.exitPromise;
|
||
if (shuttingDown) return;
|
||
console.error(`SSH 隧道已退出(${code ?? 'unknown'}),正在重连…(后端/前端不受影响)`);
|
||
await sleep(TUNNEL_RETRY_BASE_MS);
|
||
}
|
||
};
|
||
|
||
// 守护循环本身理论上不抛错,兜底仍保留:真异常时按老路径收尾,避免留下半死进程
|
||
const startTunnelSupervisor = () => {
|
||
superviseTunnel().catch((error) => {
|
||
console.error(`隧道守护异常终止:${error.message}`);
|
||
stopChildren();
|
||
process.exitCode = 1;
|
||
});
|
||
};
|
||
|
||
const npmCommand = process.platform === 'win32' ? 'npm.cmd' : 'npm';
|
||
const startApplication = (args, kind) => {
|
||
const childEnvironment = {
|
||
...process.env,
|
||
CANGKU_REMOTE_SUPERVISOR: 'true',
|
||
};
|
||
if (kind === 'frontend' && !childEnvironment.VITE_API_PROXY_TARGET) {
|
||
childEnvironment.VITE_API_PROXY_TARGET = `http://localhost:${resolveBackendPort()}`;
|
||
}
|
||
const child = spawn(npmCommand, args, {
|
||
cwd: ROOT_DIR,
|
||
stdio: 'inherit',
|
||
env: childEnvironment,
|
||
// Windows 下 .cmd 不是可执行文件,必须经 shell 启动,否则 spawn 直接抛 EINVAL
|
||
shell: process.platform === 'win32',
|
||
});
|
||
children.add(child);
|
||
child.once('exit', (code) => {
|
||
children.delete(child);
|
||
if (kind === 'backend' && restartingBackend) return;
|
||
if (!shuttingDown) {
|
||
console.error(`${kind === 'backend' ? '后端' : '前端'}进程已退出(${code ?? 'unknown'})`);
|
||
stopChildren();
|
||
process.exitCode = code || 1;
|
||
}
|
||
});
|
||
return child;
|
||
};
|
||
|
||
const hasConnection = fs.existsSync(CONNECTION_PATH);
|
||
if (hasConnection) {
|
||
if (fs.existsSync(RESTART_MARKER_PATH)) fs.unlinkSync(RESTART_MARKER_PATH);
|
||
startTunnelSupervisor();
|
||
await tunnelReady;
|
||
} else if (mode === 'tunnel') {
|
||
throw new Error('尚未配置远程系统数据库,请先运行应用并在首次启动弹窗中输入请求地址和 API Key');
|
||
}
|
||
|
||
if (mode === 'tunnel') return;
|
||
|
||
backend = startApplication(
|
||
['--prefix', 'backend', 'run', mode === 'dev' ? 'dev' : 'start'],
|
||
'backend'
|
||
);
|
||
if (mode === 'dev') {
|
||
startApplication(['--prefix', 'frontend', 'run', 'dev'], 'frontend');
|
||
}
|
||
|
||
if (!hasConnection) {
|
||
console.log('首次启动:请在安装页面输入配置请求地址和 API Key。');
|
||
await waitForFile(RESTART_MARKER_PATH);
|
||
restartingBackend = true;
|
||
if (!backend.killed) backend.kill('SIGTERM');
|
||
await waitForExit(backend);
|
||
restartingBackend = false;
|
||
fs.unlinkSync(RESTART_MARKER_PATH);
|
||
startTunnelSupervisor();
|
||
await tunnelReady;
|
||
backend = startApplication(
|
||
['--prefix', 'backend', 'run', mode === 'dev' ? 'dev' : 'start'],
|
||
'backend'
|
||
);
|
||
}
|
||
}
|
||
|
||
if (process.argv[1] === fileURLToPath(import.meta.url)) {
|
||
main().catch((error) => {
|
||
console.error(`远程运行启动失败:${error.message}`);
|
||
process.exitCode = 1;
|
||
});
|
||
}
|