Merge pull request 'fix(wle): 钉钉推送失败后自动重试，不再「一天只有一次机会」' (#96)
This commit is contained in:
commit
2f44cdce82
@ -6,6 +6,8 @@
|
||||
* 业务口径（2026-10-22 定稿）：
|
||||
* ① 每天 09:10（核验 09:00 之后）把**当天核验批次**的异常情况发到指定钉钉群；
|
||||
* **0 条也推**（群里看到「本批 0 异常」= 任务没挂）。
|
||||
* ⚠️ 失败必须重试（2026-11-11 加）：09:10 起 30 分钟内每 5 分钟再试一次，成功即停；
|
||||
* 窗口过了不再补推（上午的数据下午才发会误导业务）。见 `resolveScheduledPushDecision`。
|
||||
* ② 不发 .xlsx 附件 —— 群机器人 webhook 发不了文件，业务也确认群里看文本就够。
|
||||
* 内容 = 汇总（发货/异常/异常率/分类型/分仓）+ 前 N 条明细 + 页面链接。
|
||||
* N 由页面配置（默认 30，上限 50）；要全部明细就点链接去页面导出 Excel。
|
||||
@ -42,6 +44,32 @@ const DEFAULT_TICK_CRON = '* * * * *';
|
||||
// 钉钉 text 上限 20000 字节；留 300 字节给「已精简」尾巴，避免临界失败
|
||||
const SAFE_CONTENT_BYTES = MAX_CONTENT_BYTES - 300;
|
||||
|
||||
// 🔥 2026-11-11 加：推送失败要重试，不能「一天只有一次机会」。
|
||||
// 实况：11-11 09:10 那次撞上业务库连不上（状态里 `Failed to connect to 218.93.9.21:1433`），
|
||||
// 老逻辑只在 `HH:mm === sendTime` 这一分钟内跑一次，失败后当天彻底不推 —— 群里一整天没消息。
|
||||
// 现在改成：从推送时刻起 `RETRY_WINDOW_MINUTES` 分钟内，每隔 `RETRY_INTERVAL_MINUTES` 分钟再试，
|
||||
// 成功即停（`lastSentDate` 落库后后续 tick 自动 skip），窗口过了就不再补（避免上午的数据下午才发）。
|
||||
// 两个值都可用环境变量覆盖，现网想调不用改前端配置、也不用发版。
|
||||
const DEFAULT_RETRY_WINDOW_MINUTES = 30;
|
||||
const DEFAULT_RETRY_INTERVAL_MINUTES = 5;
|
||||
|
||||
function readPositiveMinutes(value, fallback, max) {
|
||||
const num = Number(value);
|
||||
if (!Number.isFinite(num) || num <= 0) return fallback;
|
||||
return Math.min(Math.round(num), max);
|
||||
}
|
||||
|
||||
const RETRY_WINDOW_MINUTES = readPositiveMinutes(
|
||||
process.env.WLE_NOTIFY_RETRY_WINDOW_MINUTES,
|
||||
DEFAULT_RETRY_WINDOW_MINUTES,
|
||||
12 * 60
|
||||
);
|
||||
const RETRY_INTERVAL_MINUTES = readPositiveMinutes(
|
||||
process.env.WLE_NOTIFY_RETRY_INTERVAL_MINUTES,
|
||||
DEFAULT_RETRY_INTERVAL_MINUTES,
|
||||
RETRY_WINDOW_MINUTES
|
||||
);
|
||||
|
||||
let tickRunning = false;
|
||||
|
||||
function pad2(value) {
|
||||
@ -72,6 +100,66 @@ function chinaHourMinute(date = new Date()) {
|
||||
}).format(base);
|
||||
}
|
||||
|
||||
/** 'HH:mm' → 当天第几分钟（09:10 → 550）。非法输入返回 null，调用方按「不跑」处理 */
|
||||
function minuteOfDay(hhmm) {
|
||||
const match = /^([01]\d|2[0-3]):([0-5]\d)$/.exec(String(hhmm || '').trim());
|
||||
if (!match) return null;
|
||||
return Number(match[1]) * 60 + Number(match[2]);
|
||||
}
|
||||
|
||||
/**
|
||||
* 定时器每一分钟问一次「现在该不该推」。纯函数，便于把时间窗口/节流的边界钉死在单测里。
|
||||
*
|
||||
* 返回 `{ run, reason, pushDate, attempt }`：
|
||||
* · `already_sent_today` 今天已推成功 → 不再动（重试自然也停）
|
||||
* · `before_time` 还没到配置的推送时刻
|
||||
* · `window_closed` 已过「推送时刻 + 重试窗口」，不再补推（避免 10 点失败、下午才发出去）
|
||||
* · `wait_interval` 上一轮失败距现在不到重试间隔，等着
|
||||
* · `first_round` 到点第一次跑
|
||||
* · `retry` 上一轮失败后的第 N 次重试
|
||||
*
|
||||
* ⚠️ 窗口**不跨日**：`sendTime + window` 超过 23:59 就截到当天最后一分钟。
|
||||
* 跨日会让 `pushDate` 换天，去重键（`lastSentDate`）跟着变，容易出现同一天推两遍。
|
||||
*/
|
||||
function resolveScheduledPushDecision(options = {}) {
|
||||
const now = options.now instanceof Date ? options.now : new Date(options.now ?? Date.now());
|
||||
const pushDate = chinaDateKey(now);
|
||||
const state = options.state && typeof options.state === 'object' ? options.state : {};
|
||||
const windowMinutes = readPositiveMinutes(options.retryWindowMinutes, RETRY_WINDOW_MINUTES, 12 * 60);
|
||||
const intervalMinutes = readPositiveMinutes(options.retryIntervalMinutes, RETRY_INTERVAL_MINUTES, windowMinutes);
|
||||
|
||||
if (state.lastSentDate === pushDate) {
|
||||
return { run: false, reason: 'already_sent_today', pushDate, attempt: 0 };
|
||||
}
|
||||
|
||||
const nowMinute = minuteOfDay(chinaHourMinute(now));
|
||||
const startMinute = minuteOfDay(options.sendTime);
|
||||
if (nowMinute === null || startMinute === null) {
|
||||
return { run: false, reason: 'bad_send_time', pushDate, attempt: 0 };
|
||||
}
|
||||
if (nowMinute < startMinute) {
|
||||
return { run: false, reason: 'before_time', pushDate, attempt: 0 };
|
||||
}
|
||||
const endMinute = Math.min(startMinute + windowMinutes, 23 * 60 + 59);
|
||||
if (nowMinute > endMinute) {
|
||||
return { run: false, reason: 'window_closed', pushDate, attempt: 0 };
|
||||
}
|
||||
|
||||
// 当天的尝试次数：跨天或没有记录都归零，避免沿用昨天的计数把节流算歪
|
||||
const attempt = state.attemptDate === pushDate ? Number(state.attemptCount) || 0 : 0;
|
||||
if (attempt > 0) {
|
||||
const lastAttemptMs = Date.parse(String(state.lastAttemptAt || ''));
|
||||
if (Number.isFinite(lastAttemptMs)) {
|
||||
const gapMinutes = (now.getTime() - lastAttemptMs) / 60000;
|
||||
if (gapMinutes < intervalMinutes) {
|
||||
return { run: false, reason: 'wait_interval', pushDate, attempt };
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return { run: true, reason: attempt > 0 ? 'retry' : 'first_round', pushDate, attempt: attempt + 1 };
|
||||
}
|
||||
|
||||
/**
|
||||
* 字段清洗：换行/连续空白压成单空格。
|
||||
* ⚠️ 物流描述里常有 `\n`，直接拼进 text 消息会把一条明细劈成两行，看着像两条异常。
|
||||
@ -479,8 +567,9 @@ async function runWarehouseLogisticsExceptionNotifyJob(source = 'scheduled', opt
|
||||
}
|
||||
|
||||
/**
|
||||
* 调度：每分钟比对「北京时间 HH:mm == 配置的 sendTime」。
|
||||
* 调度：每分钟问一次 `resolveScheduledPushDecision`。
|
||||
* 之所以不用固定 cron 表达式：业务在页面上改时间后要立即生效，不能等后端重启。
|
||||
* 之所以不是「HH:mm 严格相等」：那样只给一次机会，09:10 撞上业务库连不上就整天不推（11-11 实况）。
|
||||
*/
|
||||
function startWarehouseLogisticsExceptionNotifyScheduler() {
|
||||
const disabled = String(process.env.DISABLE_WLE_NOTIFY || '').trim().toLowerCase() === 'true';
|
||||
@ -489,6 +578,7 @@ function startWarehouseLogisticsExceptionNotifyScheduler() {
|
||||
return;
|
||||
}
|
||||
const expression = String(process.env.WLE_NOTIFY_TICK_CRON || DEFAULT_TICK_CRON).trim();
|
||||
let windowClosedLoggedDate = '';
|
||||
cron.schedule(expression, () => {
|
||||
if (tickRunning) return;
|
||||
tickRunning = true;
|
||||
@ -497,8 +587,32 @@ function startWarehouseLogisticsExceptionNotifyScheduler() {
|
||||
const now = new Date();
|
||||
const config = await getWarehouseLogisticsExceptionNotifyRuntimeConfig();
|
||||
if (!config.enabled) return;
|
||||
if (chinaHourMinute(now) !== config.sendTime) return;
|
||||
await runWarehouseLogisticsExceptionNotifyJob('daily_schedule', { now });
|
||||
const state = await loadWarehouseLogisticsExceptionNotifyState();
|
||||
const decision = resolveScheduledPushDecision({ now, sendTime: config.sendTime, state });
|
||||
if (!decision.run) {
|
||||
// 窗口刚过、当天又一直没推成功 → 打一条，明确「今天不会自己来了」，别让人一直等
|
||||
if (decision.reason === 'window_closed' && windowClosedLoggedDate !== decision.pushDate) {
|
||||
windowClosedLoggedDate = decision.pushDate;
|
||||
console.error(
|
||||
`[WleNotify] 今日推送窗口已关闭（${config.sendTime} 起 ${RETRY_WINDOW_MINUTES} 分钟内重试，`
|
||||
+ '始终未成功），今天不再自动重试，请人工处理'
|
||||
);
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
// 先落「本轮尝试」：失败后下一分钟的 tick 才能按间隔节流。
|
||||
// 成功时 push 内部会覆盖 lastSentDate（去重键），此后本函数直接走 already_sent_today。
|
||||
await saveWarehouseLogisticsExceptionNotifyState({
|
||||
...state,
|
||||
attemptDate: decision.pushDate,
|
||||
attemptCount: decision.attempt,
|
||||
lastAttemptAt: now.toISOString(),
|
||||
});
|
||||
await runWarehouseLogisticsExceptionNotifyJob(
|
||||
decision.reason === 'retry' ? 'daily_retry' : 'daily_schedule',
|
||||
{ now }
|
||||
);
|
||||
} catch (error) {
|
||||
console.error('[WleNotify] 定时检查失败:', error.message || error);
|
||||
} finally {
|
||||
@ -507,10 +621,15 @@ function startWarehouseLogisticsExceptionNotifyScheduler() {
|
||||
})();
|
||||
}, { timezone: TIMEZONE });
|
||||
|
||||
console.log(`[Scheduler] 物流转运异常钉钉推送已注册: 每分钟检查，到点(默认 09:10)推送 (${TIMEZONE})`);
|
||||
console.log(
|
||||
`[Scheduler] 物流转运异常钉钉推送已注册: 每分钟检查，到点(默认 09:10)推送，`
|
||||
+ `失败后每 ${RETRY_INTERVAL_MINUTES} 分钟重试、最多 ${RETRY_WINDOW_MINUTES} 分钟内 (${TIMEZONE})`
|
||||
);
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
DEFAULT_RETRY_INTERVAL_MINUTES,
|
||||
DEFAULT_RETRY_WINDOW_MINUTES,
|
||||
DEFAULT_TICK_CRON,
|
||||
SAFE_CONTENT_BYTES,
|
||||
TIMEZONE,
|
||||
@ -521,9 +640,11 @@ module.exports = {
|
||||
chinaDateKey,
|
||||
chinaHourMinute,
|
||||
groupRowsByWarehouse,
|
||||
minuteOfDay,
|
||||
pickDetailRows,
|
||||
cleanCell,
|
||||
pushWarehouseLogisticsExceptionNotify,
|
||||
resolveScheduledPushDecision,
|
||||
runWarehouseLogisticsExceptionNotifyJob,
|
||||
startWarehouseLogisticsExceptionNotifyScheduler,
|
||||
};
|
||||
|
||||
@ -542,3 +542,137 @@ test('数据源正常但当天 0 条记录 → 说明是「库里没有记录」
|
||||
assert.ok(!built.text.includes('物流数据未同步'), '表是新的，不能谎报停更');
|
||||
assert.ok(!built.text.includes('全部正常'), '0 条不能糊成「全部正常」');
|
||||
});
|
||||
|
||||
// ── ⑥ 推送失败要重试，不能「一天只有一次机会」（2026-11-11）──────────────
|
||||
// 实况：11-11 09:10 那次撞上业务库连不上（状态里 `Failed to connect to 218.93.9.21:1433`），
|
||||
// 老逻辑用 `HH:mm === sendTime` 严格相等，只给一分钟机会，失败后当天彻底不推、群里一整天没消息。
|
||||
// 下面把窗口与节流的边界钉死，避免改成「每分钟都重跑」把 240s 的全量核验打成 DDoS。
|
||||
function beijing(isoText) {
|
||||
return new Date(isoText); // 测试用 UTC 写法，避免本机时区把断言带偏
|
||||
}
|
||||
|
||||
test('重试调度：到点第一次跑，推成功之后当天不再跑', () => {
|
||||
const sendTime = '09:10';
|
||||
|
||||
const first = job.resolveScheduledPushDecision({
|
||||
now: beijing('2026-11-11T01:10:00Z'), // 北京 09:10
|
||||
sendTime,
|
||||
state: {},
|
||||
});
|
||||
assert.equal(first.run, true);
|
||||
assert.equal(first.reason, 'first_round');
|
||||
assert.equal(first.attempt, 1);
|
||||
assert.equal(first.pushDate, '2026-11-11');
|
||||
|
||||
const sent = job.resolveScheduledPushDecision({
|
||||
now: beijing('2026-11-11T01:11:00Z'), // 北京 09:11，09:10 那轮已成功
|
||||
sendTime,
|
||||
state: { lastSentDate: '2026-11-11', lastSentAt: '2026-11-11T01:10:03Z' },
|
||||
});
|
||||
assert.equal(sent.run, false);
|
||||
assert.equal(sent.reason, 'already_sent_today');
|
||||
});
|
||||
|
||||
test('重试调度：没到点的分钟一律不跑（不会被「窗口」提前触发）', () => {
|
||||
const decision = job.resolveScheduledPushDecision({
|
||||
now: beijing('2026-11-11T01:09:00Z'), // 北京 09:09
|
||||
sendTime: '09:10',
|
||||
state: {},
|
||||
});
|
||||
assert.equal(decision.run, false);
|
||||
assert.equal(decision.reason, 'before_time');
|
||||
});
|
||||
|
||||
test('重试调度：失败后按 5 分钟间隔重试，间隔内不重复拉全量核验', () => {
|
||||
const sendTime = '09:10';
|
||||
const failedState = { lastAttemptAt: '2026-11-11T01:10:00Z', attemptDate: '2026-11-11', attemptCount: 1 };
|
||||
|
||||
const tooSoon = job.resolveScheduledPushDecision({
|
||||
now: beijing('2026-11-11T01:13:00Z'), // 距上次尝试 3 分钟
|
||||
sendTime,
|
||||
state: failedState,
|
||||
});
|
||||
assert.equal(tooSoon.run, false);
|
||||
assert.equal(tooSoon.reason, 'wait_interval');
|
||||
|
||||
const retry = job.resolveScheduledPushDecision({
|
||||
now: beijing('2026-11-11T01:15:00Z'), // 距上次尝试 5 分钟，正好到间隔
|
||||
sendTime,
|
||||
state: failedState,
|
||||
});
|
||||
assert.equal(retry.run, true);
|
||||
assert.equal(retry.reason, 'retry');
|
||||
assert.equal(retry.attempt, 2);
|
||||
});
|
||||
|
||||
test('重试调度：过了「推送时刻 + 30 分钟」窗口就不再补推', () => {
|
||||
const sendTime = '09:10';
|
||||
const failedState = { lastAttemptAt: '2026-11-11T01:35:00Z', attemptDate: '2026-11-11', attemptCount: 6 };
|
||||
|
||||
const inside = job.resolveScheduledPushDecision({
|
||||
now: beijing('2026-11-11T01:40:00Z'), // 北京 09:40，窗口最后一分钟
|
||||
sendTime,
|
||||
state: failedState,
|
||||
});
|
||||
assert.equal(inside.run, true, '窗口边界（含）仍要重试');
|
||||
|
||||
const closed = job.resolveScheduledPushDecision({
|
||||
now: beijing('2026-11-11T01:41:00Z'), // 北京 09:41
|
||||
sendTime,
|
||||
state: failedState,
|
||||
});
|
||||
assert.equal(closed.run, false);
|
||||
assert.equal(closed.reason, 'window_closed');
|
||||
});
|
||||
|
||||
test('重试调度：跨天后尝试次数归零，昨天的失败不节流今天的首轮', () => {
|
||||
const decision = job.resolveScheduledPushDecision({
|
||||
now: beijing('2026-11-12T01:10:00Z'), // 北京 11-12 09:10
|
||||
sendTime: '09:10',
|
||||
state: { lastAttemptAt: '2026-11-11T01:35:00Z', attemptDate: '2026-11-11', attemptCount: 6 },
|
||||
});
|
||||
assert.equal(decision.run, true);
|
||||
assert.equal(decision.reason, 'first_round');
|
||||
assert.equal(decision.attempt, 1);
|
||||
});
|
||||
|
||||
test('重试调度：重试窗口不跨日（23:50 推送不会被次日 00:10 当成「窗口内」）', () => {
|
||||
const decision = job.resolveScheduledPushDecision({
|
||||
now: beijing('2026-11-11T16:10:00Z'), // 北京 11-12 00:10
|
||||
sendTime: '23:50',
|
||||
state: {},
|
||||
});
|
||||
assert.equal(decision.run, false);
|
||||
assert.equal(decision.reason, 'before_time');
|
||||
});
|
||||
|
||||
test('重试调度：尝试记录残缺（有次数没时间）时不能卡死，该重试就重试', () => {
|
||||
const decision = job.resolveScheduledPushDecision({
|
||||
now: beijing('2026-11-11T01:11:00Z'),
|
||||
sendTime: '09:10',
|
||||
state: { attemptDate: '2026-11-11', attemptCount: 2, lastAttemptAt: '' },
|
||||
});
|
||||
assert.equal(decision.run, true);
|
||||
assert.equal(decision.reason, 'retry');
|
||||
assert.equal(decision.attempt, 3);
|
||||
});
|
||||
|
||||
test('重试调度：sendTime 非法时安静不跑（配置坏了不能把定时任务打崩）', () => {
|
||||
const decision = job.resolveScheduledPushDecision({
|
||||
now: beijing('2026-11-11T01:10:00Z'),
|
||||
sendTime: 'not-a-time',
|
||||
state: {},
|
||||
});
|
||||
assert.equal(decision.run, false);
|
||||
assert.equal(decision.reason, 'bad_send_time');
|
||||
});
|
||||
|
||||
test('重试参数：默认 30 分钟窗口 / 5 分钟间隔，环境变量可覆盖', () => {
|
||||
assert.equal(job.DEFAULT_RETRY_WINDOW_MINUTES, 30);
|
||||
assert.equal(job.DEFAULT_RETRY_INTERVAL_MINUTES, 5);
|
||||
assert.equal(job.minuteOfDay('09:10'), 550);
|
||||
assert.equal(job.minuteOfDay('00:00'), 0);
|
||||
assert.equal(job.minuteOfDay('23:59'), 1439);
|
||||
assert.equal(job.minuteOfDay('9:10'), null, '必须是零填充的 HH:mm');
|
||||
assert.equal(job.minuteOfDay(''), null);
|
||||
});
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user