CaiWuXiTong/scripts/kaifa-progress-sync.mjs

440 lines
22 KiB
JavaScript

import fs from 'node:fs'
import path from 'node:path'
import { createHash } from 'node:crypto'
import { spawnSync } from 'node:child_process'
import { fileURLToPath, pathToFileURL } from 'node:url'
const ALLOWED_STATUSES = new Set(['code_complete', 'testing', 'done'])
const DEFAULT_API_BASE = 'https://kaifa.yydd.ai/api/v1'
const DEFAULT_SINCE = '30 days ago'
export const FEATURE_RULES = [
{
key: 'team-system-database', priority: 90, moduleName: '数据接入与共享镜像', featureName: '团队系统数据库与新人接入',
subject: /系统数据库|团队数据库|共享连接|新人|引导库/i,
paths: /(?:SystemDatabase|TeamSystemDatabase|ConnectTeamSystemDatabase|local-init\.sh)/i,
},
{
key: 'upstream-connections', priority: 90, moduleName: '数据接入与共享镜像', featureName: '多上游数据库连接与字段目录',
subject: /operations_replica|经营分析副本库|PostgreSQL|云数据库|多数据库|上游.*(?:连接|驱动|隧道|ANSI|目录)/i,
paths: /(?:Upstream(?:Connection|DatabaseProbe|OdbcConnection|ReadOnlyQuery|SchemaCatalog)|SystemSettings\/Databases|system_settings\/databases)/i,
},
{
key: 'business-mirror-sync', priority: 70, moduleName: '数据接入与共享镜像', featureName: '业务镜像全量与增量同步',
subject: /业务镜像|共享镜像|共享快照|快照|增量同步|business mirror|shared business data|\bmirrored?\b/i,
paths: /(?:BusinessMirror|SnapshotRefresher|UpstreamQueryCache|upstream_mirrors\.php)/i,
},
{
key: 'business-directory', priority: 60, moduleName: '基础资料与经营报表', featureName: '公司、店铺与商标基础资料',
subject: /公司店铺商标|基础资料|业务管理页面|中文表头|经营信息|business company|store lifecycle/i,
paths: /(?:BusinessInformationMirrorRepository|store-management\/store-information|company-management\/company-information|trademark-management\/trademark-information)/i,
},
{
key: 'store-reports', priority: 80, moduleName: '基础资料与经营报表', featureName: '店铺与主体维度经营报表',
subject: /店铺维度|主体维度|经营报表|store report|finance prototype business reports/i,
paths: /(?:StoreReport|store-management\/store-report|company-management\/entity-report)/i,
},
{
key: 'installation-details', priority: 100, moduleName: '基础资料与经营报表', featureName: '安装费用与明细查询',
subject: /安装明细|installation/i,
paths: /InstallationDetail/i,
},
{
key: 'order-expenses', priority: 70, moduleName: '订单、退款与店铺收款', featureName: '订单售后与商品推广费用',
subject: /订单|售后|推广费用|order mirror|promotion sku/i,
paths: /(?:OrderData|order-data\/after-sales-expenses|order-data\/promotion-expenses)/i,
},
{
key: 'refunds', priority: 100, moduleName: '订单、退款与店铺收款', featureName: '退款数据与导出',
subject: /退款|refund/i,
paths: /(?:refunds?|non-jingxi-refund)/i,
},
{
key: 'store-receivables', priority: 100, moduleName: '订单、退款与店铺收款', featureName: '店铺收款账户与支付宝账单',
subject: /收款账户|支付宝|store receiving/i,
paths: /(?:AlipayBills|StoreReceiving|store-receiving)/i,
},
{
key: 'zhongcai-reconciliation', priority: 100, moduleName: '订单、退款与店铺收款', featureName: '种菜应收对账平台页签与专用表头',
subject: /种菜应收|zhongcai reconciliation/i,
paths: /(?:zhongcai_reconciliations|ZhongcaiReconciliationPlatformTest)/i,
},
{
key: 'marketing-bills', priority: 60, moduleName: '营销账单与供应商应付', featureName: '淘系、1688、拼多多与京东营销账单',
subject: /营销|Alibaba|1688|京东|拼多多|账单.*导出|marketing bills|platform bills/i,
paths: /(?:MarketingAccountBill|AlibabaBill|JdWalletBill|PinduoduoSpreadBill|TmallTaobaoSpreadBill|marketing_.*bills)/i,
},
{
key: 'supplier-products', priority: 80, moduleName: '营销账单与供应商应付', featureName: '商品价格、普通商品与供应商赔偿',
subject: /商品价格|普通商品|商品及库存|供应商|赔偿|product price/i,
paths: /(?:ProductPrice|SupplierPayable|supplier-payables)/i,
},
{
key: 'courier-esign', priority: 90, moduleName: '快递与审批', featureName: 'E 签宝无回调状态轮询与归档',
subject: /快递|报价|签署|协议|E\s*签|esign|cooperation|courier|signing/i,
paths: /(?:Courier|Esign|agreement-signing|quote-management)/i,
},
{
key: 'ding-payables', priority: 100, moduleName: '快递与审批', featureName: '钉钉应付流程与状态同步',
subject: /钉钉|应付.*流程|ding payable/i,
paths: /(?:Ding|ding_|dingtalk)/i,
},
{
key: 'production-release', priority: 80, moduleName: '生产发布与团队协作', featureName: '生产服务器发布、回滚与存储',
subject: /发布|部署|服务器|构建|缓存|七牛|production|生产业务迁移|迁移兼容/i,
paths: /(?:deploy|deploy-from-workstation|Dockerfile|Qiniu)/i,
},
{
key: 'team-collaboration', priority: 60, moduleName: '生产发布与团队协作', featureName: '团队规则、任务预检与导航权限',
subject: /规则|预检|敏感|协作|目录|导航|权限/i,
paths: /(?:AGENTS\.md|codex-task-preflight|business_navigation|ShowInAdmin)/i,
},
]
function fail(message) {
const error = new Error(message)
error.name = 'ProgressSyncError'
throw error
}
export function parseEnv(text) {
const result = {}
for (const rawLine of text.split(/\r?\n/)) {
const line = rawLine.trim()
if (!line || line.startsWith('#')) continue
const splitAt = line.indexOf('=')
if (splitAt < 1) continue
result[line.slice(0, splitAt).trim()] = line.slice(splitAt + 1).trim()
}
return result
}
export function normalizeRepoName(value = '') {
return value.trim().replace(/\/$/, '').replace(/\.git$/i, '').split('/').at(-1)?.replace(/^.*@/, '') || ''
}
export function buildApiContext(env) {
return {
department: env.DEPARTMENT || '',
project: env.PROJECT || '',
developer: env.DEVELOPER_CONTEXT || '',
project_description: env.PROJECT_DESCRIPTION || '',
}
}
export function makeFeatureCatalogKey(repoName, definitions) {
const signature = definitions
.map((rule) => `${rule.moduleName}\u0000${rule.featureName}`)
.sort()
.join('\u0001')
const digest = createHash('sha256').update(signature).digest('hex').slice(0, 16)
return `${normalizeRepoName(repoName)}:feature-catalog:${digest}`
}
function normalizeUrl(value = '') {
return value ? value.trim().replace(/\/+$/, '').toLowerCase() : ''
}
export function selectUniqueProject(catalog, identity) {
const expected = {
projectName: identity.projectName.trim(),
repoName: normalizeRepoName(identity.repoName),
productionUrl: normalizeUrl(identity.productionUrl),
deploymentName: identity.deploymentName.trim(),
}
const candidates = catalog.filter((project) => [
expected.projectName && project.projectName === expected.projectName,
expected.repoName && normalizeRepoName(project.repoName || '') === expected.repoName,
expected.productionUrl && normalizeUrl(project.productionUrl || '') === expected.productionUrl,
expected.deploymentName && (project.deploymentName || '').trim() === expected.deploymentName,
expected.deploymentName && (project.infrastructure?.servers || []).some((server) => server.deploymentName === expected.deploymentName),
].some(Boolean))
if (candidates.length !== 1) fail(`PROJECT_MATCH_${candidates.length === 0 ? 'NOT_FOUND' : 'AMBIGUOUS'}`)
const project = candidates[0]
const conflicts = []
if (project.projectName !== expected.projectName) conflicts.push('projectName')
if (project.repoName && normalizeRepoName(project.repoName) !== expected.repoName) conflicts.push('repoName')
if (project.productionUrl && normalizeUrl(project.productionUrl) !== expected.productionUrl) conflicts.push('productionUrl')
if (project.deploymentName && project.deploymentName !== expected.deploymentName) conflicts.push('deploymentName')
if (conflicts.length) fail(`PROJECT_MATCH_CONFLICT:${conflicts.join(',')}`)
return project
}
export function classifyCommit(subject, paths) {
if (/^(?:merge|revert)\b|^(?:合并|回滚)/i.test(subject)) return { reason: 'merge_or_revert' }
const subjectMatches = FEATURE_RULES.filter((rule) => rule.subject.test(subject))
if (subjectMatches.length === 1) return { rule: subjectMatches[0] }
if (subjectMatches.length > 1) {
const highestPriority = Math.max(...subjectMatches.map((rule) => rule.priority))
const strongest = subjectMatches.filter((rule) => rule.priority === highestPriority)
if (strongest.length === 1) return { rule: strongest[0] }
fail(`FEATURE_MATCH_AMBIGUOUS:${subject}`)
}
const pathMatches = FEATURE_RULES.filter((rule) => paths.some((filePath) => rule.paths.test(filePath)))
if (pathMatches.length === 1) return { rule: pathMatches[0] }
if (pathMatches.length > 1) fail(`FEATURE_MATCH_AMBIGUOUS:${subject}`)
return { reason: 'no_clear_feature' }
}
export function makeIdempotencyKey(repoName, sha, status) {
if (!ALLOWED_STATUSES.has(status)) fail(`INVALID_STATUS:${status}`)
return `${normalizeRepoName(repoName)}:${sha}:${status}`
}
export function validateStatusEvidence(status, options) {
if (!ALLOWED_STATUSES.has(status)) fail(`INVALID_STATUS:${status}`)
if (status === 'testing' && !options.testStatus) fail('TESTING_REQUIRES_TEST_STATUS')
if (status === 'done' && (options.testStatus !== 'passed' || !options.acceptanceEvidence)) {
fail('DONE_REQUIRES_PASSED_TEST_AND_ACCEPTANCE_EVIDENCE')
}
}
function git(repoRoot, args) {
const result = spawnSync('git', args, { cwd: repoRoot, encoding: 'utf8', maxBuffer: 64 * 1024 * 1024 })
if (result.status !== 0) fail(`GIT_FAILED:${args[0]}:${result.stderr.trim()}`)
return result.stdout.trim()
}
function getChangedFiles(repoRoot, sha) {
const statuses = git(repoRoot, ['diff-tree', '--root', '--no-commit-id', '--name-status', '-r', '-M', sha]).split('\n').filter(Boolean)
const nums = git(repoRoot, ['diff-tree', '--root', '--no-commit-id', '--numstat', '-r', '-M', sha]).split('\n').filter(Boolean)
if (statuses.length !== nums.length) fail(`GIT_FILE_STAT_MISMATCH:${sha}`)
return statuses.map((line, index) => {
const fields = line.split('\t')
const status = fields[0][0]
if (!['A', 'M', 'D', 'R'].includes(status)) fail(`UNSUPPORTED_FILE_STATUS:${fields[0]}`)
const numFields = nums[index].split('\t')
return {
path: status === 'R' ? `${fields[1]} -> ${fields[2]}` : fields[1],
status,
additions: numFields[0] === '-' ? 0 : Number(numFields[0]),
deletions: numFields[1] === '-' ? 0 : Number(numFields[1]),
}
})
}
function getCommit(repoRoot, sha, branch) {
const output = git(repoRoot, ['show', '-s', '--use-mailmap', '--format=%H%x00%aN%x00%aE%x00%cI%x00%s', sha])
const [fullSha, authorName, authorEmail, committedAt, message] = output.split('\0')
const changedFiles = getChangedFiles(repoRoot, sha)
return {
sha: fullSha,
message,
author_name: authorName,
author_email: authorEmail,
committed_at: committedAt,
branch,
changed_files: changedFiles,
stats: {
additions: changedFiles.reduce((sum, file) => sum + file.additions, 0),
deletions: changedFiles.reduce((sum, file) => sum + file.deletions, 0),
files_changed: changedFiles.length,
},
}
}
function parseArgs(argv) {
const values = {}
const flags = new Set()
for (const arg of argv) {
if (!arg.startsWith('--')) continue
const splitAt = arg.indexOf('=')
if (splitAt === -1) flags.add(arg.slice(2))
else values[arg.slice(2, splitAt)] = arg.slice(splitAt + 1)
}
return { values, flags }
}
async function apiRequest(base, apiKey, endpoint, options = {}) {
for (let attempt = 0; attempt < 4; attempt += 1) {
let response
try {
response = await fetch(`${base}${endpoint}`, {
...options,
headers: { Authorization: `Bearer ${apiKey}`, Accept: 'application/json', ...(options.headers || {}) },
})
} catch (error) {
if (attempt < 3) {
await new Promise((resolve) => setTimeout(resolve, (attempt + 1) * 5000))
continue
}
fail(`API_NETWORK_ERROR:${error instanceof Error ? error.message : 'request failed'}`)
}
const payload = await response.json().catch(() => null)
if (response.ok) return payload
if (response.status === 429 && attempt < 3) {
const retryAfter = Number(response.headers.get('retry-after'))
const waitSeconds = Number.isFinite(retryAfter) && retryAfter > 0 ? retryAfter : 30
await new Promise((resolve) => setTimeout(resolve, Math.min(waitSeconds + 1, 60) * 1000))
continue
}
if ([502, 503, 504].includes(response.status) && attempt < 3) {
await new Promise((resolve) => setTimeout(resolve, (attempt + 1) * 5000))
continue
}
const code = payload?.error?.code || 'UNKNOWN'
const message = payload?.error?.message || ''
const details = payload?.error?.mismatch || payload?.error?.details || payload?.error?.issues || payload?.details || payload?.issues
fail(`API_${response.status}:${code}${message ? `:${message}` : ''}${details ? `:${JSON.stringify(details)}` : ''}`)
}
fail('API_RETRY_EXHAUSTED')
}
function readState(statePath) {
if (!fs.existsSync(statePath)) return { version: 1, synced: {} }
const state = JSON.parse(fs.readFileSync(statePath, 'utf8'))
return state?.version === 1 && state.synced && typeof state.synced === 'object' ? state : { version: 1, synced: {} }
}
function writeState(statePath, state) {
fs.mkdirSync(path.dirname(statePath), { recursive: true })
const tempPath = `${statePath}.${process.pid}.tmp`
fs.writeFileSync(tempPath, `${JSON.stringify(state, null, 2)}\n`, { encoding: 'utf8', mode: 0o600 })
fs.renameSync(tempPath, statePath)
}
export async function main(argv = process.argv.slice(2)) {
const { values, flags } = parseArgs(argv)
const scriptDir = path.dirname(fileURLToPath(import.meta.url))
const repoRoot = git(path.resolve(scriptDir, '..'), ['rev-parse', '--show-toplevel'])
const privateDir = path.join(repoRoot, '.private', 'kaifa-progress-sync')
const envPath = path.join(privateDir, '.env.local')
const statePath = path.join(privateDir, 'state.json')
if (!fs.existsSync(envPath)) fail(`CONFIG_NOT_FOUND:${envPath}`)
const env = parseEnv(fs.readFileSync(envPath, 'utf8'))
const apiBase = (env.API_BASE || DEFAULT_API_BASE).replace(/\/$/, '')
if (!env.API_KEY) fail(`API_KEY_NOT_CONFIGURED:${envPath}`)
for (const field of ['OWNER_EMAIL', 'DEVELOPER_CONTEXT', 'PROJECT_DESCRIPTION', 'DEPARTMENT', 'PROJECT', 'REPO_NAME', 'PRODUCTION_URL', 'DEPLOYMENT_NAME']) {
if (!env[field]) fail(`CONFIG_FIELD_MISSING:${field}`)
}
const apiContext = buildApiContext(env)
const status = values.status || 'code_complete'
const testStatus = values['test-status'] || (status === 'code_complete' ? 'not_run' : null)
const acceptanceEvidence = values['acceptance-evidence'] || null
validateStatusEvidence(status, { testStatus, acceptanceEvidence })
const dryRun = flags.has('dry-run')
const ref = values.ref || 'origin/main'
const since = values.since || DEFAULT_SINCE
const limit = values.limit ? Number(values.limit) : Number.POSITIVE_INFINITY
if (!(limit > 0)) fail('INVALID_LIMIT')
const remoteUrl = git(repoRoot, ['config', '--get', 'remote.origin.url'])
if (normalizeRepoName(remoteUrl) !== normalizeRepoName(env.REPO_NAME)) fail('REPOSITORY_NAME_MISMATCH')
const catalogResponse = await apiRequest(apiBase, env.API_KEY, '/my/catalog')
const project = selectUniqueProject(catalogResponse.catalog || [], {
projectName: env.PROJECT,
repoName: env.REPO_NAME,
productionUrl: env.PRODUCTION_URL,
deploymentName: env.DEPLOYMENT_NAME,
})
if (project.department !== env.DEPARTMENT) fail('PROJECT_DEPARTMENT_MISMATCH')
const state = readState(statePath)
const shas = git(repoRoot, ['log', ref, `--since=${since}`, '--no-merges', '--reverse', '--format=%H']).split('\n').filter(Boolean)
const excluded = {}
const excludedCommits = []
const excludedAuthors = {}
const candidates = []
for (const sha of shas) {
const key = makeIdempotencyKey(env.REPO_NAME, sha, status)
if (state.synced[key]) {
excluded.already_synced = (excluded.already_synced || 0) + 1
continue
}
const commit = getCommit(repoRoot, sha, 'main')
if (commit.author_email.toLowerCase() !== env.OWNER_EMAIL.toLowerCase()) {
excluded.other_developer = (excluded.other_developer || 0) + 1
const author = `${commit.author_name} <${commit.author_email}>`
excludedAuthors[author] = (excludedAuthors[author] || 0) + 1
continue
}
const classification = classifyCommit(commit.message, commit.changed_files.map((file) => file.path))
if (!classification.rule) {
excluded[classification.reason] = (excluded[classification.reason] || 0) + 1
excludedCommits.push({ sha: commit.sha, title: commit.message, reason: classification.reason })
continue
}
candidates.push({ commit, rule: classification.rule, key })
if (candidates.length >= limit) break
}
const featureSummary = Object.values(candidates.reduce((map, item) => {
const name = `${item.rule.moduleName}/${item.rule.featureName}`
map[name] ||= { module: item.rule.moduleName, feature: item.rule.featureName, commits: 0 }
map[name].commits += 1
return map
}, {}))
const authorSummary = Object.values(candidates.reduce((map, item) => {
const key = `${item.commit.author_name} <${item.commit.author_email}>`
map[key] ||= { author: key, commits: 0 }
map[key].commits += 1
return map
}, {}))
if (dryRun) {
console.log(JSON.stringify({ mode: 'dry-run', project: project.projectName, ref, since, status, candidates: candidates.length, excluded, excludedAuthors, excludedCommits, features: featureSummary, authors: authorSummary }, null, 2))
return
}
if (!candidates.length) {
console.log(JSON.stringify({ mode: 'apply', project: project.projectName, synced: 0, excluded }, null, 2))
return
}
const definitions = [...new Map(candidates.map(({ rule }) => [rule.key, rule])).values()]
const resolved = await apiRequest(apiBase, env.API_KEY, `/projects/${project.projectId}/features/resolve`, {
method: 'POST',
headers: { 'Content-Type': 'application/json', 'Idempotency-Key': makeFeatureCatalogKey(env.REPO_NAME, definitions) },
body: JSON.stringify({
context: apiContext,
project_description: env.PROJECT_DESCRIPTION,
items: definitions.map((rule) => ({ module_name: rule.moduleName, feature_name: rule.featureName, create_module: true })),
}),
})
const featureIds = new Map()
for (const item of resolved.items || []) {
const rule = definitions.find((candidate) => candidate.moduleName === item.moduleName && candidate.featureName === item.featureName)
if (!rule || !item.moduleId || !item.featureId || !item.canPush) fail(`FEATURE_NOT_PUSHABLE:${item.moduleName}/${item.featureName}`)
featureIds.set(rule.key, { moduleId: item.moduleId, featureId: item.featureId })
}
if (featureIds.size !== definitions.length) fail('FEATURE_RESOLUTION_INCOMPLETE')
let synced = 0
const syncedByFeature = {}
for (const candidate of candidates) {
const ids = featureIds.get(candidate.rule.key)
const commit = { ...candidate.commit, module_id: ids.moduleId, feature_id: ids.featureId }
const update = {
module_id: ids.moduleId,
feature_id: ids.featureId,
change_type: 'feature',
status,
summary: commit.message,
blockers: null,
next_steps: status === 'code_complete' ? '待测试' : status === 'testing' ? '待验收或发布' : '已完成验收或发布',
evidence_shas: [commit.sha],
test_status: testStatus,
...(acceptanceEvidence ? { completion_evidence: acceptanceEvidence } : {}),
}
const response = await apiRequest(apiBase, env.API_KEY, `/projects/${project.projectId}/progress-sync`, {
method: 'POST',
headers: { 'Content-Type': 'application/json', 'Idempotency-Key': candidate.key },
body: JSON.stringify({
context: apiContext,
project_description: env.PROJECT_DESCRIPTION,
commits: [commit],
updates: [update],
}),
})
if (!response.commits || !response.updates || !response.project) fail(`SYNC_RESPONSE_INCOMPLETE:${commit.sha}`)
state.synced[candidate.key] = { syncedAt: new Date().toISOString(), projectId: project.projectId, moduleId: ids.moduleId, featureId: ids.featureId }
state.lastSuccessfulAt = new Date().toISOString()
writeState(statePath, state)
synced += 1
syncedByFeature[candidate.rule.key] = (syncedByFeature[candidate.rule.key] || 0) + 1
await new Promise((resolve) => setTimeout(resolve, 1100))
}
console.log(JSON.stringify({ mode: 'apply', project: project.projectName, ref, status, synced, excluded, excludedAuthors, excludedCommits, syncedByFeature }, null, 2))
}
if (process.argv[1] && import.meta.url === pathToFileURL(path.resolve(process.argv[1])).href) {
main().catch((error) => {
const message = String(error?.message || error).replace(/ptrk_[A-Za-z0-9_-]+/g, '[REDACTED_API_KEY]')
console.error(`progress-sync stopped: ${message}`)
process.exitCode = 1
})
}