fix: optimize performance survey employee sync
This commit is contained in:
chenpei
parent
e0d3b3d475
commit
007f35df86
@ -352,6 +352,213 @@ function buildSelectSql(
|
||||
`
|
||||
}
|
||||
|
||||
function buildEmployeeListSql(employeeColumns: TableColumns) {
|
||||
const e = {
|
||||
dingtalkUserId: pickColumn(employeeColumns, FIELD_CANDIDATES.dingtalkUserId),
|
||||
email: pickColumn(employeeColumns, FIELD_CANDIDATES.email),
|
||||
jobNumber: pickColumn(employeeColumns, FIELD_CANDIDATES.jobNumber),
|
||||
name: pickColumn(employeeColumns, FIELD_CANDIDATES.name),
|
||||
title: pickColumn(employeeColumns, FIELD_CANDIDATES.title),
|
||||
workPlace: pickColumn(employeeColumns, FIELD_CANDIDATES.workPlace),
|
||||
extension: pickColumn(employeeColumns, FIELD_CANDIDATES.extension),
|
||||
active: pickColumn(employeeColumns, FIELD_CANDIDATES.active),
|
||||
managerDingtalkUserId: pickColumn(employeeColumns, FIELD_CANDIDATES.managerDingtalkUserId),
|
||||
managerJobNumber: pickColumn(employeeColumns, FIELD_CANDIDATES.managerJobNumber),
|
||||
managerName: pickColumn(employeeColumns, FIELD_CANDIDATES.managerName),
|
||||
}
|
||||
const activeFilter = e.active ? `WHERE (e.${quoteIdentifier(e.active)} = 1 OR e.${quoteIdentifier(e.active)} = '1')` : ''
|
||||
|
||||
return `
|
||||
SELECT TOP (@limit)
|
||||
${columnExpr('e', e.dingtalkUserId)} AS "dingtalkUserId",
|
||||
${columnExpr('e', e.email)} AS "dingEmail",
|
||||
${columnExpr('e', e.jobNumber)} AS "jobNumber",
|
||||
${columnExpr('e', e.name)} AS "contactName",
|
||||
${columnExpr('e', e.title)} AS "contactTitle",
|
||||
${columnExpr('e', e.workPlace)} AS "workPlace",
|
||||
${columnExpr('e', e.extension)} AS "extension",
|
||||
${columnExpr('e', e.managerDingtalkUserId)} AS "managerDingtalkUserId",
|
||||
${columnExpr('e', e.managerJobNumber)} AS "managerJobNumber",
|
||||
${columnExpr('e', e.managerName)} AS "managerName"
|
||||
FROM ${quoteIdentifier(EMPLOYEE_TABLE.schema)}.${quoteIdentifier(EMPLOYEE_TABLE.table)} e
|
||||
${activeFilter}
|
||||
ORDER BY COALESCE(${columnExpr('e', e.name)}, ${columnExpr('e', e.jobNumber)}, ${columnExpr('e', e.dingtalkUserId)}) ASC
|
||||
`
|
||||
}
|
||||
|
||||
function buildRegisterListSql(registerColumns: TableColumns) {
|
||||
const r = {
|
||||
dingtalkUserId: pickColumn(registerColumns, FIELD_CANDIDATES.dingtalkUserId),
|
||||
email: pickColumn(registerColumns, FIELD_CANDIDATES.email),
|
||||
registerName: pickColumn(registerColumns, FIELD_CANDIDATES.registerName),
|
||||
flowerName: pickColumn(registerColumns, FIELD_CANDIDATES.flowerName),
|
||||
realName: pickColumn(registerColumns, FIELD_CANDIDATES.registerRealName),
|
||||
jobNumber: pickColumn(registerColumns, FIELD_CANDIDATES.registerJobNumber),
|
||||
positionTitle: pickColumn(registerColumns, FIELD_CANDIDATES.positionTitle),
|
||||
positionLevel: pickColumn(registerColumns, FIELD_CANDIDATES.positionLevel),
|
||||
departmentName: pickColumn(registerColumns, FIELD_CANDIDATES.departmentName),
|
||||
departmentPath: pickColumn(registerColumns, FIELD_CANDIDATES.departmentPath),
|
||||
employmentStatus: pickColumn(registerColumns, FIELD_CANDIDATES.employmentStatus),
|
||||
managerDingtalkUserId: pickColumn(registerColumns, FIELD_CANDIDATES.managerDingtalkUserId),
|
||||
managerJobNumber: pickColumn(registerColumns, FIELD_CANDIDATES.managerJobNumber),
|
||||
managerName: pickColumn(registerColumns, FIELD_CANDIDATES.managerName),
|
||||
}
|
||||
|
||||
return `
|
||||
SELECT TOP (@limit)
|
||||
${columnExpr('r', r.dingtalkUserId)} AS "dingtalkUserId",
|
||||
${columnExpr('r', r.email)} AS "dingEmail",
|
||||
${columnExpr('r', r.registerName)} AS "registerName",
|
||||
${columnExpr('r', r.flowerName)} AS "flowerName",
|
||||
${columnExpr('r', r.realName)} AS "realName",
|
||||
${columnExpr('r', r.jobNumber)} AS "registerJobNumber",
|
||||
${columnExpr('r', r.positionTitle)} AS "positionTitle",
|
||||
${columnExpr('r', r.positionLevel)} AS "positionLevel",
|
||||
${columnExpr('r', r.departmentName)} AS "departmentName",
|
||||
${columnExpr('r', r.departmentPath)} AS "departmentPath",
|
||||
${columnExpr('r', r.employmentStatus)} AS "employmentStatus",
|
||||
${columnExpr('r', r.managerDingtalkUserId)} AS "managerDingtalkUserId",
|
||||
${columnExpr('r', r.managerJobNumber)} AS "managerJobNumber",
|
||||
${columnExpr('r', r.managerName)} AS "managerName"
|
||||
FROM ${quoteIdentifier(REGISTER_TABLE.schema)}.${quoteIdentifier(REGISTER_TABLE.table)} r
|
||||
ORDER BY COALESCE(${columnExpr('r', r.flowerName)}, ${columnExpr('r', r.registerName)}, ${columnExpr('r', r.jobNumber)}, ${columnExpr('r', r.dingtalkUserId)}) ASC
|
||||
`
|
||||
}
|
||||
|
||||
function buildFlowerListSql(flowerColumns: TableColumns) {
|
||||
const f = {
|
||||
jobNumber: pickColumn(flowerColumns, FIELD_CANDIDATES.flowerBookJobNumber),
|
||||
name: pickColumn(flowerColumns, FIELD_CANDIDATES.flowerBookName),
|
||||
flowerName: pickColumn(flowerColumns, FIELD_CANDIDATES.flowerBookFlowerName),
|
||||
dept: pickColumn(flowerColumns, FIELD_CANDIDATES.flowerBookDept),
|
||||
workPlace: pickColumn(flowerColumns, FIELD_CANDIDATES.flowerBookWorkPlace),
|
||||
job: pickColumn(flowerColumns, FIELD_CANDIDATES.flowerBookJob),
|
||||
jobLevel: pickColumn(flowerColumns, FIELD_CANDIDATES.flowerBookJobLevel),
|
||||
role: pickColumn(flowerColumns, FIELD_CANDIDATES.flowerBookRole),
|
||||
director: pickColumn(flowerColumns, FIELD_CANDIDATES.flowerBookDirector),
|
||||
manager: pickColumn(flowerColumns, FIELD_CANDIDATES.flowerBookManager),
|
||||
employeeStatus: pickColumn(flowerColumns, FIELD_CANDIDATES.flowerBookEmployeeStatus),
|
||||
}
|
||||
|
||||
return `
|
||||
SELECT TOP (@limit)
|
||||
${columnExpr('f', f.jobNumber)} AS "registerJobNumber",
|
||||
${columnExpr('f', f.name)} AS "flowerBookName",
|
||||
${columnExpr('f', f.flowerName)} AS "flowerBookFlowerName",
|
||||
${columnExpr('f', f.dept)} AS "flowerBookDept",
|
||||
${columnExpr('f', f.workPlace)} AS "flowerBookWorkPlace",
|
||||
${columnExpr('f', f.job)} AS "flowerBookJob",
|
||||
${columnExpr('f', f.jobLevel)} AS "flowerBookJobLevel",
|
||||
${columnExpr('f', f.role)} AS "flowerBookRole",
|
||||
${columnExpr('f', f.director)} AS "flowerBookDirector",
|
||||
${columnExpr('f', f.manager)} AS "flowerBookManager",
|
||||
${columnExpr('f', f.employeeStatus)} AS "flowerBookEmployeeStatus"
|
||||
FROM ${quoteIdentifier(FLOWER_TABLE.schema)}.${quoteIdentifier(FLOWER_TABLE.table)} f
|
||||
ORDER BY COALESCE(${columnExpr('f', f.flowerName)}, ${columnExpr('f', f.name)}, ${columnExpr('f', f.jobNumber)}) ASC
|
||||
`
|
||||
}
|
||||
|
||||
function normalizedKey(value: unknown) {
|
||||
return clean(value)?.toLowerCase() || null
|
||||
}
|
||||
|
||||
function addToIndex<T>(index: Map<string, T>, key: unknown, value: T) {
|
||||
const normalized = normalizedKey(key)
|
||||
if (normalized && !index.has(normalized)) {
|
||||
index.set(normalized, value)
|
||||
}
|
||||
}
|
||||
|
||||
function indexRows(rows: ReportEmployeeRow[], keys: Array<keyof ReportEmployeeRow>) {
|
||||
const index = new Map<string, ReportEmployeeRow>()
|
||||
for (const row of rows) {
|
||||
for (const key of keys) {
|
||||
addToIndex(index, row[key], row)
|
||||
}
|
||||
}
|
||||
return index
|
||||
}
|
||||
|
||||
function getIndexed(index: Map<string, ReportEmployeeRow>, ...keys: unknown[]) {
|
||||
for (const key of keys) {
|
||||
const normalized = normalizedKey(key)
|
||||
if (!normalized) continue
|
||||
const matched = index.get(normalized)
|
||||
if (matched) return matched
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
function mergeReportRows(...rows: Array<ReportEmployeeRow | null | undefined>): ReportEmployeeRow {
|
||||
const merged: ReportEmployeeRow = {}
|
||||
for (const row of rows) {
|
||||
if (!row) continue
|
||||
for (const [key, value] of Object.entries(row) as Array<[keyof ReportEmployeeRow, unknown]>) {
|
||||
if (clean(merged[key]) || clean(value) === null) continue
|
||||
;(merged as Record<string, unknown>)[key] = value
|
||||
}
|
||||
}
|
||||
return merged
|
||||
}
|
||||
|
||||
async function queryReportRows(
|
||||
pool: any,
|
||||
sql: typeof import('mssql'),
|
||||
query: string,
|
||||
limit: number,
|
||||
): Promise<ReportEmployeeRow[]> {
|
||||
const result = await pool.request().input('limit', sql.Int, limit).query(query)
|
||||
return (result.recordset || []) as ReportEmployeeRow[]
|
||||
}
|
||||
|
||||
async function fetchFullSyncRows(
|
||||
pool: any,
|
||||
sql: typeof import('mssql'),
|
||||
schema: Awaited<ReturnType<typeof getReportSchema>>,
|
||||
limit: number,
|
||||
) {
|
||||
const employeeRows = await queryReportRows(pool, sql, buildEmployeeListSql(schema.employeeColumns), limit)
|
||||
const registerRows = schema.registerColumns.size > 0
|
||||
? await queryReportRows(pool, sql, buildRegisterListSql(schema.registerColumns), limit)
|
||||
: []
|
||||
const flowerRows = schema.flowerColumns.size > 0
|
||||
? await queryReportRows(pool, sql, buildFlowerListSql(schema.flowerColumns), Math.max(limit, Math.min(limit * 3, 10_000)))
|
||||
: []
|
||||
|
||||
const registerByUserId = indexRows(registerRows, ['dingtalkUserId'])
|
||||
const registerByJobNumber = indexRows(registerRows, ['registerJobNumber', 'jobNumber'])
|
||||
const registerByName = indexRows(registerRows, ['flowerName', 'realName', 'registerName'])
|
||||
const flowerByJobNumber = indexRows(flowerRows, ['registerJobNumber', 'jobNumber'])
|
||||
const flowerByName = indexRows(flowerRows, ['flowerBookFlowerName', 'flowerBookName'])
|
||||
|
||||
const mergedRows: ReportEmployeeRow[] = []
|
||||
const seen = new Set<string>()
|
||||
const pushMerged = (row: ReportEmployeeRow) => {
|
||||
const normalized = normalizeEmployeeRow(row)
|
||||
const key = normalized.dingtalkUserId || normalized.jobNumber || normalized.dingEmail || normalized.displayName
|
||||
if (seen.has(key)) return
|
||||
seen.add(key)
|
||||
mergedRows.push(row)
|
||||
}
|
||||
|
||||
for (const employee of employeeRows) {
|
||||
const register = getIndexed(registerByUserId, employee.dingtalkUserId)
|
||||
|| getIndexed(registerByJobNumber, employee.jobNumber)
|
||||
|| getIndexed(registerByName, employee.contactName)
|
||||
const flower = getIndexed(flowerByJobNumber, register?.registerJobNumber, employee.jobNumber)
|
||||
|| getIndexed(flowerByName, register?.flowerName, register?.realName, register?.registerName, employee.contactName)
|
||||
pushMerged(mergeReportRows(employee, register, flower))
|
||||
}
|
||||
|
||||
for (const register of registerRows) {
|
||||
const flower = getIndexed(flowerByJobNumber, register.registerJobNumber, register.jobNumber)
|
||||
|| getIndexed(flowerByName, register.flowerName, register.realName, register.registerName)
|
||||
pushMerged(mergeReportRows(register, flower))
|
||||
}
|
||||
|
||||
return mergedRows.slice(0, limit)
|
||||
}
|
||||
|
||||
function buildIdentityWhere(
|
||||
employeeColumns: TableColumns,
|
||||
registerColumns: TableColumns,
|
||||
@ -702,22 +909,11 @@ export async function syncPerformanceSurveyEmployees(options?: { limit?: number
|
||||
|
||||
try {
|
||||
const schema = await getReportSchema(pool)
|
||||
const result = await pool
|
||||
.request()
|
||||
.input('limit', sql.Int, limit)
|
||||
.query<ReportEmployeeRow>(
|
||||
buildSelectSql(
|
||||
schema.employeeColumns,
|
||||
schema.registerColumns,
|
||||
schema.flowerColumns,
|
||||
'1 = 1',
|
||||
'@limit',
|
||||
),
|
||||
)
|
||||
const rows = await fetchFullSyncRows(pool, sql, schema, limit)
|
||||
|
||||
let upserted = 0
|
||||
let skipped = 0
|
||||
for (const row of result.recordset || []) {
|
||||
for (const row of rows) {
|
||||
const normalized = normalizeEmployeeRow(row)
|
||||
if (!normalized.dingtalkUserId && !normalized.jobNumber && !normalized.dingEmail) {
|
||||
skipped += 1
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user