feat: reconcile workflow product ids daily

This commit is contained in:
江林铖 2026-09-17 16:14:14 +08:00
parent 35a301a45e
commit d0cd7ab979
12 changed files with 263 additions and 62 deletions

View File

@ -30,7 +30,7 @@ web
- “进行中项目”项目明细按业务导出表展示审批标题、审批编号、需求人、需求部门、平台、品牌、类目、SPU、需求类型、标准时效、当前节点与负责人;需求人取流程发起人,需求部门按 HR 人员架构匹配。
- 美术设计流程的明细开始时间取“品牌策划主管分配任务”节点创建时间,结束时间取“品牌策划主管审核”节点完成时间;内容视频流程的明细开始时间取已确认的“内容主管分配任务”节点创建时间,结束时间按规则分支取“编导审核”或“内容主管审核”节点完成时间。内容流程中多个主管分配节点、编导制作节点及普通领导者任务均不等同于最终审核;未知 activityId 不得仅凭负责人身份、AGREE 结果或完成状态推断为审核节点。
- 美术设计申请流程的 `ding_WorkflowTask.activityId` 按已确认流程定义映射为精确中文节点名并写入本地快照;未知节点继续使用同步字段推断或明确回退,不改写上游只读数据。
- 钉钉流程快照每小时从只读报表库的 `ding_Workflow`、`ding_WorkflowTask`、`ding_WorkflowScheme` 联合增量同步,只接收已批准的美术设计申请和内容视频流程;三表复合游标写入现有同步状态并继续复用同一批生产快照数据。
- 钉钉流程快照每小时从只读报表库的 `ding_Workflow`、`ding_WorkflowTask`、`ding_WorkflowScheme` 联合增量同步,只接收已批准的美术设计申请和内容视频流程;三表复合游标写入现有同步状态,并每 24 小时全量校准流程、品牌和商品 ID,修正上游未推进游标的历史表单改动。
- “本月上新”只读取表单字段“需求类型”为“新品”的六品牌美术设计流程,不以流程标题关键词判断,也不计“优化”等其他需求类型;分子为本月整条流程已完结数量,分母为本月已完结数量与当前进行中数量之和;其中进行中流程直接复用“进行中项目”的美术设计流程状态、时间与当前品牌中心处理人条件。
- “按时交付”只统计对应主管/规则审核节点在本月完成、审批状态为已完结且审批结论未拒绝的美术设计与内容视频流程;待计算流程不进入分母。详情页可在全部流程、美术设计流程和内容视频流程之间筛选,公式、数量与明细必须共用当前筛选口径。
- “首轮通过”读取作品过稿数据,分子为 V1 版本审核通过的作品数,分母为当前已通过的作品数;详情页列出部门范围内所有含 V1 的作品,搜索只过滤明细,不改变顶部完整口径。

View File

@ -16,7 +16,11 @@ const emptyList = {
page: 1,
pageSize: 20,
hasMore: false,
lastWorkflowSyncedAt: '2026-09-17T00:00:00.000Z',
workflowSync: {
status: 'ready' as const,
lastCompletedAt: '2099-09-17T01:00:00.000Z',
lastFullReconciledAt: '2099-09-17T00:00:00.000Z',
},
}
const workflowItem = {
@ -65,6 +69,33 @@ describe('PerformanceConversionDashboard filters', () => {
expect(requestUrl).toContain('completedTo=2026-07-31')
expect(window.location.search).toContain('completedFrom=2026-07-01')
expect(window.location.search).toContain('completedTo=2026-07-31')
expect(screen.getByText('流程与商品 ID 每日增量同步')).toBeInTheDocument()
expect(screen.getByText('最近成功 2099/09/17 09:00')).toBeInTheDocument()
})
it.each([
['syncing', '正在同步流程与商品 ID'],
['error', '流程与商品 ID 同步异常'],
['pending', '等待首次流程与商品 ID 同步'],
] as const)('shows the %s workflow synchronization state', async (status, label) => {
global.fetch = jest.fn().mockResolvedValue({
ok: true,
json: async () => ({
success: true,
data: {
...emptyList,
workflowSync: {
status,
lastCompletedAt: status === 'pending' ? null : '2099-09-17T01:00:00.000Z',
lastFullReconciledAt: status === 'pending' ? null : '2099-09-17T00:00:00.000Z',
},
},
}),
} as Response)
render(<PerformanceConversionDashboard />)
expect(await screen.findByText(label)).toBeInTheDocument()
})
it('opens each product ID on a separate detail page and preserves the list context', async () => {

View File

@ -4,15 +4,19 @@ import Link from 'next/link'
import { FormEvent, useCallback, useEffect, useRef, useState } from 'react'
import {
Activity,
AlertTriangle,
CalendarDays,
CheckCircle2,
ChevronLeft,
ChevronRight,
Clock3,
RefreshCw,
Search,
} from 'lucide-react'
import DepartmentShell from './DepartmentShell'
import {
BUSINESS_TREND_BRANDS,
type PerformanceConversionWorkflowSync,
type PerformanceConversionWorkflowItem,
type PerformanceConversionWorkflowList,
isValidDateKey,
@ -43,6 +47,41 @@ function formatDate(value: string | null, includeTime = false) {
}).format(date)
}
function WorkflowSyncStatus({ sync }: { sync: PerformanceConversionWorkflowSync }) {
const latestSuccess = sync.lastCompletedAt || sync.lastFullReconciledAt
const latestSuccessLabel = latestSuccess ? formatDate(latestSuccess, true) : null
const latestReconciliation = sync.lastFullReconciledAt || sync.lastCompletedAt
const isDelayed = sync.status === 'ready'
&& latestReconciliation !== null
&& Date.now() - new Date(latestReconciliation).getTime() > 30 * 60 * 60 * 1000
const view = sync.status === 'syncing'
? { label: '正在同步流程与商品 ID', detail: latestSuccessLabel ? `上次成功 ${latestSuccessLabel}` : '首次同步进行中', tone: 'text-[#0050cb]', icon: RefreshCw, spin: true }
: sync.status === 'error'
? { label: '流程与商品 ID 同步异常', detail: latestSuccessLabel ? `暂显示 ${latestSuccessLabel} 的数据` : '暂无可用同步记录', tone: 'text-amber-700', icon: AlertTriangle, spin: false }
: sync.status === 'pending'
? { label: '等待首次流程与商品 ID 同步', detail: '后台任务尚未完成', tone: 'text-slate-500', icon: Clock3, spin: false }
: isDelayed
? { label: '流程与商品 ID 每日校准延迟', detail: `最近增量成功 ${latestSuccessLabel}`, tone: 'text-amber-700', icon: AlertTriangle, spin: false }
: { label: '流程与商品 ID 每日增量同步', detail: latestSuccessLabel ? `最近成功 ${latestSuccessLabel}` : '等待首次成功记录', tone: 'text-emerald-700', icon: CheckCircle2, spin: false }
const Icon = view.icon
return (
<div
role="status"
aria-live="polite"
className={`min-w-0 text-left text-xs leading-5 lg:text-right ${view.tone}`}
title={latestSuccess ? `最近同步成功:${latestSuccessLabel}(北京时间)` : undefined}
>
<p className="inline-flex items-center gap-1.5 font-bold">
<Icon className={`h-3.5 w-3.5 shrink-0 ${view.spin ? 'animate-spin' : ''}`} aria-hidden="true" />
<span>{view.label}</span>
</p>
<p className="text-slate-500">{view.detail}</p>
</div>
)
}
function WorkflowTable({ items, detailHref }: {
items: PerformanceConversionWorkflowItem[]
detailHref: (item: PerformanceConversionWorkflowItem) => string
@ -178,7 +217,7 @@ export default function PerformanceConversionDashboard() {
<aside aria-label="品牌导航" className="border-b border-slate-200 bg-slate-50 lg:border-b-0 lg:border-r"><div className="border-b border-slate-200 px-4 py-4"><p className="text-xs font-bold text-slate-500">美术申请流程</p><p className="mt-1 text-sm font-bold text-slate-900">六品牌</p></div><nav className="flex gap-1 overflow-x-auto p-2 lg:block lg:space-y-1 lg:overflow-visible" aria-label="选择品牌">{BUSINESS_TREND_BRANDS.map((brand) => { const count = listState.data?.brands.find((item) => item.brand === brand)?.count; return <button key={brand} type="button" aria-current={selectedBrand === brand ? 'page' : undefined} onClick={() => { const nextFilters = { ...appliedFilters, query: '' }; setSelectedBrand(brand); setQuery(''); setCompletedFrom(nextFilters.completedFrom); setCompletedTo(nextFilters.completedTo); setAppliedFilters(nextFilters); setFilterError(null); setPage(1); void loadList(brand, 1, nextFilters) }} className={`flex min-h-11 shrink-0 items-center gap-2 rounded-lg px-3 text-left text-sm font-bold focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-[#1677ff] lg:w-full ${selectedBrand === brand ? 'bg-[#1677ff] text-white' : 'text-slate-600 hover:bg-slate-200/70 hover:text-slate-950'}`}><span className="min-w-0 flex-1 truncate">{brand}</span>{count !== undefined && <span className={`tabular-nums text-xs ${selectedBrand === brand ? 'text-white/80' : 'text-slate-400'}`}>{count}</span>}</button> })}</nav></aside>
<section className="min-w-0 bg-white" aria-labelledby="performance-brand-title" aria-busy={listState.loading}>
<header className="flex flex-col gap-4 border-b border-slate-200 px-5 py-5 sm:px-7 lg:flex-row lg:items-end lg:justify-between"><div><p className="text-xs font-bold text-[#9a4d08]">当前品牌</p><h2 id="performance-brand-title" className="mt-1 text-xl font-bold tracking-normal text-slate-950">{selectedBrand}</h2><p className="mt-1 text-xs text-slate-500">{appliedFilters.completedFrom || appliedFilters.completedTo ? '区间内 ' : ''}{selectedBrandCount} 个有效流程商品 ID{listState.data?.invalidProductIdCount ? ` · ${listState.data.invalidProductIdCount} 条流程未填写有效 ID` : ''}</p></div><div className="text-xs leading-5 text-slate-500">流程同步 {formatDate(listState.data?.lastWorkflowSyncedAt || null, true)}</div></header>
<header className="flex flex-col gap-4 border-b border-slate-200 px-5 py-5 sm:px-7 lg:flex-row lg:items-end lg:justify-between"><div><p className="text-xs font-bold text-[#9a4d08]">当前品牌</p><h2 id="performance-brand-title" className="mt-1 text-xl font-bold tracking-normal text-slate-950">{selectedBrand}</h2><p className="mt-1 text-xs text-slate-500">{appliedFilters.completedFrom || appliedFilters.completedTo ? '区间内 ' : ''}{selectedBrandCount} 个有效流程商品 ID{listState.data?.invalidProductIdCount ? ` · ${listState.data.invalidProductIdCount} 条流程未填写有效 ID` : ''}</p></div>{listState.data && <WorkflowSyncStatus sync={listState.data.workflowSync} />}</header>
<form onSubmit={submitSearch} className="grid gap-3 border-b border-slate-200 bg-slate-50 px-5 py-4 sm:px-7 md:grid-cols-[minmax(0,1fr)_auto] md:items-end xl:grid-cols-[minmax(280px,0.9fr)_minmax(220px,1fr)_auto]">
<fieldset className="min-w-0 md:col-span-2 xl:col-span-1">
<legend className="mb-1 text-xs font-bold text-slate-600">完结日期</legend>

View File

@ -104,6 +104,7 @@ describe('PerformanceConversionDetail', () => {
expect.objectContaining({ cache: 'no-store' }),
)
expect(screen.getByRole('link', { name: '返回业绩转化' })).toHaveAttribute('href', returnHref)
expect(screen.getByText(/经营数据更新至/)).toBeInTheDocument()
expect(screen.getAllByText('12.50%').length).toBeGreaterThan(0)
expect(screen.getAllByText('¥1,299.50').length).toBeGreaterThan(0)
expect(screen.getByRole('img', { name: /支付金额从/ })).toBeInTheDocument()

View File

@ -161,7 +161,7 @@ function DetailContent({ data }: { data: PerformanceConversionTimeseries }) {
<dl className="grid shrink-0 gap-x-5 gap-y-2 text-left text-xs leading-5 text-slate-500 sm:grid-cols-3 lg:block lg:text-right">
<div><dt className="sr-only">审批编号</dt><dd>审批编号 {data.workflow.approvalNumber || '—'}</dd></div>
<div><dt className="sr-only">流程完结</dt><dd>流程完结 {formatDate(data.workflow.completedAt, true)}</dd></div>
<div><dt className="sr-only">数据更新</dt><dd>数据更新至 {formatDate(data.coverage.sourceLatestDate)}</dd></div>
<div><dt className="sr-only">经营数据更新</dt><dd>经营数据更新至 {formatDate(data.coverage.sourceLatestDate)}</dd></div>
</dl>
</div>
{data.product.brandMatchesWorkflow === false && (

View File

@ -1,8 +1,13 @@
import { BUSINESS_TREND_BRANDS, type BusinessTrendBrandName } from './business-trends'
export { BUSINESS_TREND_BRANDS }
export type PerformanceConversionBrand = BusinessTrendBrandName
export type PerformanceConversionWorkflowSync = {
status: 'pending' | 'syncing' | 'ready' | 'error'
lastCompletedAt: string | null
lastFullReconciledAt: string | null
}
export const PERFORMANCE_CONVERSION_METRICS = [
{ key: 'impressions', label: '展现量', group: '流量', format: 'integer' },
{ key: 'visitors', label: '商品访客数', group: '流量', format: 'integer' },
@ -38,7 +43,7 @@ export type PerformanceConversionWorkflowList = {
page: number
pageSize: number
hasMore: boolean
lastWorkflowSyncedAt: string | null
workflowSync: PerformanceConversionWorkflowSync
}
export type PerformanceConversionDailyMetric = {

View File

@ -120,12 +120,12 @@ export async function register() {
console.error('[Instrumentation] 调度组织架构每日同步任务失败:', error)
}
// 钉钉审批流程快照:启动后建立基线,之后每小时增量同步。
// 钉钉审批流程快照:每小时增量同步,并每 24 小时全量校准流程与商品 ID。
// 看板只读本地 PostgreSQL 快照,避免用户请求直接扫描上游无索引大表。
try {
const { ensureHourlyDingWorkflowSyncStarted } = await import('./lib/ding-workflow-sync')
ensureHourlyDingWorkflowSyncStarted()
console.log('[Instrumentation] 钉钉流程每小时增量同步任务已调度')
console.log('[Instrumentation] 钉钉流程增量同步与每日全量校准任务已调度')
} catch (error) {
console.error('[Instrumentation] 调度钉钉流程每小时同步任务失败:', error)
}

View File

@ -28,7 +28,9 @@ import {
parseDingWorkflowSourceCursor,
resolveDingWorkflowInitiator,
resolveDingWorkflowDetailMilestones,
selectLatestDingWorkflowSchemeValue,
serializeDingWorkflowSourceCursor,
shouldRunDingWorkflowDailyReconciliation,
sourceWorkflowTimestampToIso,
storedWorkflowTimestampToIso,
} from '@/lib/ding-workflow-sync'
@ -182,29 +184,32 @@ describe('ding workflow snapshot sync semantics', () => {
it('upgrades the legacy workflow-only cursor to a three-table cursor', () => {
expect(parseDingWorkflowSourceCursor('2026-08-26 19:42:36')).toEqual({
version: 3,
version: 4,
workflowUpdatedAt: '2026-08-26 19:42:36',
taskId: null,
schemeId: null,
lastFullReconciledAt: null,
})
})
it('round-trips the Workflow, Task and Scheme incremental cursors', () => {
it('round-trips the Workflow, Task and Scheme cursors with the daily reconciliation watermark', () => {
const cursor = {
version: 3 as const,
version: 4 as const,
workflowUpdatedAt: '2026-08-26 19:42:36',
taskId: 2248339,
schemeId: 10634556,
lastFullReconciledAt: '2026-09-16T18:00:00.000Z',
}
expect(parseDingWorkflowSourceCursor(serializeDingWorkflowSourceCursor(cursor))).toEqual(cursor)
})
it('preserves empty Task and Scheme cursors so the next run performs alignment', () => {
const cursor = {
version: 3 as const,
version: 4 as const,
workflowUpdatedAt: '2026-08-26 19:42:36',
taskId: null,
schemeId: null,
lastFullReconciledAt: null,
}
expect(parseDingWorkflowSourceCursor(serializeDingWorkflowSourceCursor(cursor))).toEqual(cursor)
})
@ -216,27 +221,46 @@ describe('ding workflow snapshot sync semantics', () => {
taskId: 2248339,
schemeId: 10634556,
}))).toEqual({
version: 3,
version: 4,
workflowUpdatedAt: '2026-08-26 19:42:36',
taskId: null,
schemeId: null,
lastFullReconciledAt: null,
})
})
it('rejects malformed version 3 cursor ids and safely requests re-alignment', () => {
it('upgrades version 3 cursors and requests the first daily reconciliation', () => {
expect(parseDingWorkflowSourceCursor(JSON.stringify({
version: 3,
workflowUpdatedAt: '2026-08-26 19:42:36',
taskId: -1,
schemeId: 'invalid',
}))).toEqual({
version: 3,
version: 4,
workflowUpdatedAt: '2026-08-26 19:42:36',
taskId: null,
schemeId: null,
lastFullReconciledAt: null,
})
})
it('runs a full reconciliation at least once every 24 hours', () => {
const now = new Date('2026-09-17T02:00:00.000Z')
expect(shouldRunDingWorkflowDailyReconciliation(null, now)).toBe(true)
expect(shouldRunDingWorkflowDailyReconciliation('2026-09-16T02:00:01.000Z', now)).toBe(false)
expect(shouldRunDingWorkflowDailyReconciliation('2026-09-16T02:00:00.000Z', now)).toBe(true)
expect(shouldRunDingWorkflowDailyReconciliation('invalid', now)).toBe(true)
})
it('selects the latest non-empty Scheme row across product ID aliases', () => {
expect(selectLatestDingWorkflowSchemeValue([
{ sourceRowId: 101, name: '商品ID', value: '100000001' },
{ sourceRowId: 109, name: '商品Id', value: '100000009' },
{ sourceRowId: 110, name: '商品id', value: ' ' },
{ sourceRowId: 108, name: '品牌', value: '亿家达' },
], ['商品ID', '商品Id', '商品id'])).toBe('100000009')
})
it('only queries ongoing workflows whose current assignee belongs to Brand Center', () => {
expect(buildBrandCenterOngoingWorkflowWhere()).toEqual({
isOngoing: true,

View File

@ -19,6 +19,11 @@ jest.mock('@/lib/prisma', () => ({
jest.mock('@/lib/ding-workflow-sync', () => ({
DING_WORKFLOW_ALLOWED_PROCESS_CODES: ['art-process'],
DING_WORKFLOW_STARTED_AT_CUTOFF_ISO: '2026-03-31T16:00:00.000Z',
getDingWorkflowSyncMetadata: jest.fn().mockResolvedValue({
status: 'ready',
lastCompletedAt: '2026-09-17T01:30:00.000Z',
lastFullReconciledAt: '2026-09-17T00:00:00.000Z',
}),
isDingWorkflowApprovedCompletion: jest.fn(() => true),
}))
@ -70,6 +75,11 @@ describe('performance conversion completion date filters', () => {
expect(result.invalidProductIdCount).toBe(1)
expect(result.brands.find((item) => item.brand === '亿家达')?.count).toBe(2)
expect(result.brands.find((item) => item.brand === '卓禾')?.count).toBe(1)
expect(result.workflowSync).toEqual({
status: 'ready',
lastCompletedAt: '2026-09-17T01:30:00.000Z',
lastFullReconciledAt: '2026-09-17T00:00:00.000Z',
})
})
it('rejects invalid or reversed date ranges before reading snapshots', async () => {

View File

@ -31,9 +31,9 @@ export type {
TeamCapacityWorkflowRow,
} from '@/lib/ding-workflow-team-capacity'
// v4 forces one full refresh so historical snapshots also receive the newly
// collected 商品ID/现商品ID fields used by the performance conversion view.
const SYNC_STATE_ID = 'ding-workflow-hourly-v4-product-id'
// Keep this persisted key stable: changing it would discard the production
// cursor and force another baseline scan of the upstream tables.
export const DING_WORKFLOW_SYNC_STATE_ID = 'ding-workflow-hourly-v4-product-id'
const TERMINAL_WORKFLOW_STATUSES = new Set(['COMPLETED', 'TERMINATED', 'CANCELED'])
const REJECTED_WORKFLOW_DECISIONS = new Set([
'REFUSE',
@ -50,6 +50,7 @@ const REJECTED_WORKFLOW_DECISIONS = new Set([
const LEASE_DURATION_MS = 20 * 60 * 1000
const DEFAULT_SYNC_INTERVAL_MS = 60 * 60 * 1000
const INITIAL_SYNC_DELAY_MS = 5_000
const DAILY_RECONCILIATION_INTERVAL_MS = 24 * 60 * 60 * 1000
const UPSERT_BATCH_SIZE = 750
export const DING_WORKFLOW_STARTED_AT_CUTOFF_LOCAL = '2026-04-01 00:00:00'
@ -235,19 +236,24 @@ type SourceSchemeAggregate = {
schemeFieldCount: unknown
}
type SourceSchemeField = {
processInstanceId: unknown
componentType: unknown
export type DingWorkflowSchemeValueRow = {
name: unknown
value: unknown
sourceRowId?: unknown
}
type SourceSchemeField = DingWorkflowSchemeValueRow & {
processInstanceId: unknown
componentType: unknown
extValue: unknown
}
type DingWorkflowSourceCursor = {
version: 3
version: 4
workflowUpdatedAt: string | null
taskId: number | null
schemeId: number | null
lastFullReconciledAt: string | null
}
type SourceDeltaState = {
@ -642,13 +648,31 @@ function extractContactAssignments(fields: SourceSchemeField[]): ContactAssignme
])).values())
}
export function selectLatestDingWorkflowSchemeValue(
fields: DingWorkflowSchemeValueRow[],
names: string[],
): string | null {
const acceptedNames = new Set(names)
const candidates = fields.flatMap((field, index) => {
const name = clean(field.name)
const value = clean(field.value)
if (!name || !acceptedNames.has(name) || !value) return []
const sourceRowId = normalizeSourceCursorId(field.sourceRowId)
return [{ value, sourceRowId, index }]
})
candidates.sort((left, right) => {
if (left.sourceRowId !== null && right.sourceRowId !== null && left.sourceRowId !== right.sourceRowId) {
return right.sourceRowId - left.sourceRowId
}
if (left.sourceRowId !== null) return -1
if (right.sourceRowId !== null) return 1
return left.index - right.index
})
return candidates[0]?.value || null
}
function firstSchemeValue(fields: SourceSchemeField[], names: string[]): string | null {
for (const name of names) {
const value = fields.find((field) => clean(field.name) === name && clean(field.value))?.value
const normalized = clean(value)
if (normalized) return normalized
}
return null
return selectLatestDingWorkflowSchemeValue(fields, names)
}
function collectSchemeDisplayValues(value: unknown): string[] {
@ -1309,43 +1333,75 @@ function normalizeSourceCursorId(value: unknown): number | null {
/**
* 兼容旧版只保存 ding_Workflow.updateAt 的游标。
*
* v3 新增交付时效所需表单字段;读取旧版 v2 时保留主表游标,但清空
* Task / Scheme 游标触发一次全量对齐,之后继续按三表复合游标增量同步。
* 后续每小时只读取主表变化和 Task / Scheme 新增版本所关联的流程。
* v4 在三表复合游标之外记录最近一次全量校准时间。小时增量继续负责
* 快速更新,24 小时校准负责修正上游原地修改但未推进任何游标的历史表单。
*/
export function parseDingWorkflowSourceCursor(raw: string | null): DingWorkflowSourceCursor {
const normalized = clean(raw)
if (!normalized) {
return { version: 3, workflowUpdatedAt: null, taskId: null, schemeId: null }
return {
version: 4,
workflowUpdatedAt: null,
taskId: null,
schemeId: null,
lastFullReconciledAt: null,
}
}
try {
const parsed = JSON.parse(normalized) as Record<string, unknown>
if (parsed.version === 3) {
if (parsed.version === 4) {
return {
version: 3,
version: 4,
workflowUpdatedAt: clean(parsed.workflowUpdatedAt),
taskId: normalizeSourceCursorId(parsed.taskId),
schemeId: normalizeSourceCursorId(parsed.schemeId),
lastFullReconciledAt: clean(parsed.lastFullReconciledAt),
}
}
if (parsed.version === 3) {
return {
version: 4,
workflowUpdatedAt: clean(parsed.workflowUpdatedAt),
taskId: normalizeSourceCursorId(parsed.taskId),
schemeId: normalizeSourceCursorId(parsed.schemeId),
lastFullReconciledAt: null,
}
}
if (parsed.version === 2) {
return {
version: 3,
version: 4,
workflowUpdatedAt: clean(parsed.workflowUpdatedAt),
taskId: null,
schemeId: null,
lastFullReconciledAt: null,
}
}
} catch {
// Legacy cursor is a plain ding_Workflow.updateAt string.
}
return { version: 3, workflowUpdatedAt: normalized, taskId: null, schemeId: null }
return {
version: 4,
workflowUpdatedAt: normalized,
taskId: null,
schemeId: null,
lastFullReconciledAt: null,
}
}
export function serializeDingWorkflowSourceCursor(cursor: DingWorkflowSourceCursor): string {
return JSON.stringify(cursor)
}
export function shouldRunDingWorkflowDailyReconciliation(
lastFullReconciledAt: string | null,
now = new Date(),
) {
if (!lastFullReconciledAt) return true
const previous = new Date(lastFullReconciledAt)
if (Number.isNaN(previous.getTime())) return true
return now.getTime() - previous.getTime() >= DAILY_RECONCILIATION_INTERVAL_MS
}
export function isReportWorkflowDbConfigured() {
return Boolean(
process.env.REPORT_DB_SERVER
@ -1605,6 +1661,7 @@ async function fetchWorkflowAggregates(pool: ConnectionPool, processInstanceIds:
)
SELECT
CONVERT(nvarchar(255), scheme.processInstanceId) AS processInstanceId,
scheme.id AS sourceRowId,
CONVERT(nvarchar(500), scheme.componentType) AS componentType,
CONVERT(nvarchar(2000), scheme.name) AS name,
CONVERT(nvarchar(max), scheme.value) AS value,
@ -1622,7 +1679,8 @@ async function fetchWorkflowAggregates(pool: ConnectionPool, processInstanceIds:
'数量', '需求数量', '拍摄数量', '摄像数量', '摄影数量',
'策划任务统计', '平面任务统计', '出图任务统计'
)
OR CONVERT(nvarchar(500), scheme.componentType) IN ('InnerContactField', 'TableField');
OR CONVERT(nvarchar(500), scheme.componentType) IN ('InnerContactField', 'TableField')
ORDER BY scheme.processInstanceId, scheme.id DESC;
`)
const taskRows = (result.recordsets?.[0] || []) as SourceTaskAggregate[]
@ -1920,13 +1978,13 @@ async function upsertSnapshotBatch(rows: WorkflowSnapshotPayload[], syncedAt: Da
async function acquireSyncLease(leaseOwner: string, startedAt: Date) {
await prisma.dingWorkflowSyncState.upsert({
where: { id: SYNC_STATE_ID },
create: { id: SYNC_STATE_ID },
where: { id: DING_WORKFLOW_SYNC_STATE_ID },
create: { id: DING_WORKFLOW_SYNC_STATE_ID },
update: {},
})
const claimed = await prisma.dingWorkflowSyncState.updateMany({
where: {
id: SYNC_STATE_ID,
id: DING_WORKFLOW_SYNC_STATE_ID,
OR: [
{ leaseExpiresAt: null },
{ leaseExpiresAt: { lt: startedAt } },
@ -1942,7 +2000,16 @@ async function acquireSyncLease(leaseOwner: string, startedAt: Date) {
},
})
if (claimed.count !== 1) return null
return prisma.dingWorkflowSyncState.findUniqueOrThrow({ where: { id: SYNC_STATE_ID } })
return prisma.dingWorkflowSyncState.findUniqueOrThrow({ where: { id: DING_WORKFLOW_SYNC_STATE_ID } })
}
async function renewSyncLease(leaseOwner: string) {
const now = new Date()
const renewed = await prisma.dingWorkflowSyncState.updateMany({
where: { id: DING_WORKFLOW_SYNC_STATE_ID, leaseOwner },
data: { leaseExpiresAt: new Date(now.getTime() + LEASE_DURATION_MS) },
})
if (renewed.count !== 1) throw new Error('钉钉流程同步租约已失效,本轮停止写入')
}
export async function syncDingWorkflowSnapshots() {
@ -1960,6 +2027,8 @@ export async function syncDingWorkflowSnapshots() {
await pool.connect()
const sourceCursor = parseDingWorkflowSourceCursor(state.sourceCursorRaw)
const sourceDelta = await fetchSourceDeltaState(pool, sourceCursor)
const performFullReconciliation = sourceDelta.requiresFullSync
|| shouldRunDingWorkflowDailyReconciliation(sourceCursor.lastFullReconciledAt, startedAt)
const knownOngoingRows = await prisma.dingWorkflowSnapshot.findMany({
where: {
isOngoing: true,
@ -1970,10 +2039,11 @@ export async function syncDingWorkflowSnapshots() {
const knownOngoingWorkflowIds = knownOngoingRows.map((row) => row.processInstanceId)
const sourceRows = await fetchSourceWorkflows(
pool,
sourceDelta.requiresFullSync ? null : sourceCursor.workflowUpdatedAt,
performFullReconciliation ? null : sourceCursor.workflowUpdatedAt,
sourceDelta.changedWorkflowIds,
knownOngoingWorkflowIds,
)
await renewSyncLease(leaseOwner)
const workflowIds = sourceRows.flatMap((row) => {
const id = clean(row.processInstanceId)
return id ? [id] : []
@ -1984,6 +2054,7 @@ export async function syncDingWorkflowSnapshots() {
schemeByWorkflow,
schemeRowsByWorkflow,
} = await fetchWorkflowAggregates(pool, workflowIds)
await renewSyncLease(leaseOwner)
const employeeDirectory = await readEmployeeDirectory(sourceRows, taskRowsByWorkflow)
const activityNodeNames = inferActivityNodeNames(sourceRows, taskRowsByWorkflow, schemeRowsByWorkflow)
const rows = buildSnapshotPayload(
@ -1999,6 +2070,7 @@ export async function syncDingWorkflowSnapshots() {
for (let index = 0; index < rows.length; index += UPSERT_BATCH_SIZE) {
await upsertSnapshotBatch(rows.slice(index, index + UPSERT_BATCH_SIZE), syncedAt)
await renewSyncLease(leaseOwner)
}
const deleted = await prisma.dingWorkflowSnapshot.deleteMany({
@ -2015,13 +2087,16 @@ export async function syncDingWorkflowSnapshots() {
where: buildBrandCenterOngoingWorkflowWhere(),
})
const sourceCursorRaw = serializeDingWorkflowSourceCursor({
version: 3,
version: 4,
workflowUpdatedAt: maxSourceCursor(rows, sourceCursor.workflowUpdatedAt),
taskId: sourceDelta.maxTaskId,
schemeId: sourceDelta.maxSchemeId,
lastFullReconciledAt: performFullReconciliation
? syncedAt.toISOString()
: sourceCursor.lastFullReconciledAt,
})
await prisma.dingWorkflowSyncState.updateMany({
where: { id: SYNC_STATE_ID, leaseOwner },
where: { id: DING_WORKFLOW_SYNC_STATE_ID, leaseOwner },
data: {
status: 'ready',
sourceCursorRaw,
@ -2037,7 +2112,7 @@ export async function syncDingWorkflowSnapshots() {
} catch (error) {
const message = error instanceof Error ? error.message : String(error)
await prisma.dingWorkflowSyncState.updateMany({
where: { id: SYNC_STATE_ID, leaseOwner },
where: { id: DING_WORKFLOW_SYNC_STATE_ID, leaseOwner },
data: {
status: 'error',
lastError: message.slice(0, 2_000),
@ -2058,13 +2133,32 @@ function normalizeSyncStatus(state: { status: string; lastCompletedAt: Date | nu
return 'ready'
}
export type DingWorkflowSyncMetadata = {
status: DingWorkflowSyncStatus
lastCompletedAt: string | null
lastFullReconciledAt: string | null
}
export async function getDingWorkflowSyncMetadata(): Promise<DingWorkflowSyncMetadata> {
const state = await prisma.dingWorkflowSyncState.findUnique({
where: { id: DING_WORKFLOW_SYNC_STATE_ID },
select: { status: true, lastCompletedAt: true, sourceCursorRaw: true },
})
const sourceCursor = parseDingWorkflowSourceCursor(state?.sourceCursorRaw || null)
return {
status: normalizeSyncStatus(state),
lastCompletedAt: state?.lastCompletedAt?.toISOString() || null,
lastFullReconciledAt: sourceCursor.lastFullReconciledAt,
}
}
async function readSummaryWithoutAccessCheck(): Promise<OngoingDingWorkflowSummary> {
const [total, state] = await prisma.$transaction([
prisma.dingWorkflowSnapshot.count({
where: buildBrandCenterOngoingWorkflowWhere(),
}),
prisma.dingWorkflowSyncState.findUnique({
where: { id: SYNC_STATE_ID },
where: { id: DING_WORKFLOW_SYNC_STATE_ID },
select: { status: true, lastCompletedAt: true },
}),
])
@ -2333,7 +2427,7 @@ async function readBusinessTrendsInsightsWithoutAccessCheck(
],
}),
prisma.dingWorkflowSyncState.findUnique({
where: { id: SYNC_STATE_ID },
where: { id: DING_WORKFLOW_SYNC_STATE_ID },
select: { status: true, lastCompletedAt: true },
}),
])
@ -2392,7 +2486,7 @@ async function readTeamCapacityInsightsWithoutAccessCheck(
],
}),
prisma.dingWorkflowSyncState.findUnique({
where: { id: SYNC_STATE_ID },
where: { id: DING_WORKFLOW_SYNC_STATE_ID },
select: { status: true, lastCompletedAt: true },
}),
])
@ -2542,7 +2636,7 @@ async function readMonthlyLaunchInsightsWithoutAccessCheck(
select: MONTHLY_LAUNCH_WORKFLOW_SELECT,
}),
prisma.dingWorkflowSyncState.findUnique({
where: { id: SYNC_STATE_ID },
where: { id: DING_WORKFLOW_SYNC_STATE_ID },
select: { status: true, lastCompletedAt: true },
}),
])
@ -2632,7 +2726,7 @@ async function readOnTimeDeliveryInsightsWithoutAccessCheck(
],
}),
prisma.dingWorkflowSyncState.findUnique({
where: { id: SYNC_STATE_ID },
where: { id: DING_WORKFLOW_SYNC_STATE_ID },
select: { status: true, lastCompletedAt: true },
}),
])
@ -2732,7 +2826,7 @@ export function ensureHourlyDingWorkflowSyncStarted() {
const configuredHours = Number(process.env.DING_WORKFLOW_SYNC_INTERVAL_HOURS || 1)
const intervalMs = Number.isFinite(configuredHours) && configuredHours > 0
? Math.max(5 * 60 * 1000, configuredHours * 60 * 60 * 1000)
? Math.min(DAILY_RECONCILIATION_INTERVAL_MS, Math.max(5 * 60 * 1000, configuredHours * 60 * 60 * 1000))
: DEFAULT_SYNC_INTERVAL_MS
const runOnce = () => {
void syncDingWorkflowSnapshots()
@ -2750,5 +2844,5 @@ export function ensureHourlyDingWorkflowSyncStarted() {
initialTimer.unref?.()
const interval = setInterval(runOnce, intervalMs)
interval.unref?.()
console.info(`[ding-workflow-sync] 已调度每小时增量同步,周期 ${Math.round(intervalMs / 60_000)} 分钟`)
console.info(`[ding-workflow-sync] 已调度流程增量同步,周期 ${Math.round(intervalMs / 60_000)} 分钟,并保证每 24 小时全量校准流程与商品 ID`)
}

View File

@ -5,6 +5,7 @@ import { prisma } from '@/lib/prisma'
import {
DING_WORKFLOW_ALLOWED_PROCESS_CODES,
DING_WORKFLOW_STARTED_AT_CUTOFF_ISO,
getDingWorkflowSyncMetadata,
isDingWorkflowApprovedCompletion,
} from '@/lib/ding-workflow-sync'
import {
@ -49,7 +50,6 @@ type SnapshotRow = {
brand: string | null
sourceFinishedAtRaw: string | null
formFields: unknown
syncedAt: Date
}
type SourceMetricRow = {
@ -138,7 +138,6 @@ async function readEligibleSnapshots(): Promise<SnapshotRow[]> {
brand: true,
sourceFinishedAtRaw: true,
formFields: true,
syncedAt: true,
},
orderBy: [
{ sourceFinishedAtRaw: 'desc' },
@ -172,7 +171,10 @@ export async function getPerformanceConversionWorkflows(
if (completedFrom && completedTo && compareDateKeys(completedFrom, completedTo) > 0) {
throw new PerformanceConversionError('完结开始日期不能晚于结束日期')
}
const snapshots = await readEligibleSnapshots()
const [snapshots, workflowSync] = await Promise.all([
readEligibleSnapshots(),
getDingWorkflowSyncMetadata(),
])
const rangeSnapshots = snapshots.filter((row) => {
const completedDate = row.sourceFinishedAtRaw ? toShanghaiDateKey(row.sourceFinishedAtRaw) : null
if (!completedDate) return false
@ -195,11 +197,6 @@ export async function getPerformanceConversionWorkflows(
item.title,
].some((value) => String(value || '').toLocaleLowerCase('zh-CN').includes(query))))
const offset = (page - 1) * pageSize
const latestSyncedAt = snapshots.reduce<Date | null>(
(latest, row) => !latest || row.syncedAt > latest ? row.syncedAt : latest,
null,
)
return {
brands: BUSINESS_TREND_BRANDS.map((item) => ({ brand: item, count: brandCounts.get(item) || 0 })),
items: filtered.slice(offset, offset + pageSize),
@ -208,7 +205,7 @@ export async function getPerformanceConversionWorkflows(
page,
pageSize,
hasMore: offset + pageSize < filtered.length,
lastWorkflowSyncedAt: latestSyncedAt?.toISOString() || null,
workflowSync,
}
}

View File

@ -1469,7 +1469,7 @@ model TaskDetailChangeNotification {
///
/// 上游 rpt.ding_Workflow / ding_WorkflowTask / ding_WorkflowScheme 均为无索引大表,
/// 部门看板不得在用户请求链路中直连扫描。后台任务首次建立全量基线,之后每小时
/// 增量同步主流程,并为全部未完结流程刷新任务与表单字段汇总。
/// 增量同步主流程,并每 24 小时全量校准流程与商品 ID 表单字段。
model DingWorkflowSnapshot {
processInstanceId String @id @map("process_instance_id")
businessId String? @map("business_id")