flnxi-Wu/lib/field-pool.ts

/**
 * 字段池管理器
 * 
 * 负责管理模板字段与字段池之间的同步
 * 
 * @module field-pool
 * @created 2025-12-13
 * @updated 2025-12-13 - 优化选项清洗逻辑
 */

import { Prisma, UploadField } from '@prisma/client'
import { prisma } from '@/lib/prisma'
import { STANDARD_FIELDS } from './upload-template-presets'

const prismaAny = prisma as any

type AggregatedField = {
  key: string
  label: string
  type: string
  source: 'standard' | 'custom'
  templateIds: Set<string>
  options: Set<string>
}

/**
 * 字段池同步结果
 */
export type FieldPoolSyncResult = {
  success: boolean
  duration: number
  fieldsAdded: number
  fieldsUpdated: number
  fieldsRemoved: number
  optionsAdded: number
  optionsRemoved: number
  errors: string[]
}

/**
 * 字段池同步选项
 */
export type FieldPoolSyncOptions = {
  incremental?: boolean  // 是否增量同步，默认false
  retryOnError?: boolean  // 是否在错误时重试，默认true
  maxRetries?: number  // 最大重试次数，默认3
  logDetails?: boolean  // 是否记录详细日志，默认true
}

/**
 * 清洗选项值类型枚举
 */
export enum OptionCleaningIssue {
  EMPTY_INPUT = 'EMPTY_INPUT',
  INVALID_JSON = 'INVALID_JSON',
  NESTED_ARRAY = 'NESTED_ARRAY',
  SPECIAL_CHARS = 'SPECIAL_CHARS',
  EXCESSIVE_WHITESPACE = 'EXCESSIVE_WHITESPACE',
  UNICODE_ISSUES = 'UNICODE_ISSUES',
  NULL_OR_UNDEFINED = 'NULL_OR_UNDEFINED',
}

/**
 * 记录选项清洗问题
 */
function logCleaningIssue(
  originalValue: any,
  cleanedValue: string,
  issues: OptionCleaningIssue[]
): void {
  console.warn(JSON.stringify({
    timestamp: new Date().toISOString(),
    module: 'field-pool',
    action: 'cleanOption',
    originalValue: String(originalValue).substring(0, 100),
    cleanedValue: cleanedValue.substring(0, 100),
    issues,
    issueCount: issues.length,
  }))
}

function stripRedundantBrackets(value: string): string {
  let cleaned = value
  let changed = false
  let safeGuard = 0
  while (
    cleaned.startsWith('[[') &&
    cleaned.endsWith(']]') &&
    safeGuard < 5
  ) {
    const next = cleaned.slice(1, -1).trim()
    if (!next || next === cleaned) {
      break
    }
    cleaned = next
    changed = true
    safeGuard += 1
  }
  if (
    changed &&
    cleaned.startsWith('[') &&
    cleaned.endsWith(']') &&
    !/[,\n，]/.test(cleaned.slice(1, -1))
  ) {
    const inner = cleaned.slice(1, -1).trim()
    if (inner) {
      cleaned = inner
    }
  }
  return cleaned
}

/**
 * 规范化字段选项值（增强版）
 * 
 * 处理各种脏数据场景:
 * - 空值、null、undefined
 * - JSON字符串（单引号、双引号）
 * - 嵌套数组格式 [[value]]
 * - 转义字符 \", \'
 * - 特殊引号 ""、''
 * - 多余空白字符
 * - Unicode控制字符
 * 
 * @param option - 原始选项值
 * @param options - 清洗选项
 * @returns 清洗后的值
 */
export function normalizeFieldOptionValue(
  option: string,
  options: {
    logIssues?: boolean
    throwOnError?: boolean
  } = {}
): string {
  const { logIssues = false, throwOnError = false } = options
  const issues: OptionCleaningIssue[] = []
  const originalValue = option

  try {
    if (option === null || option === undefined) {
      issues.push(OptionCleaningIssue.NULL_OR_UNDEFINED)
      if (logIssues) {
        logCleaningIssue(originalValue, '', issues)
      }
      return ''
    }

    let cleaned = option?.trim?.() ?? ''
    if (!cleaned) {
      issues.push(OptionCleaningIssue.EMPTY_INPUT)
      if (logIssues) {
        logCleaningIssue(originalValue, '', issues)
      }
      return ''
    }

    const hasControlChars = /[\u0000-\u001F\u007F-\u009F]/.test(cleaned)
    if (hasControlChars) {
      cleaned = cleaned.replace(/[\u0000-\u001F\u007F-\u009F]/g, '')
      issues.push(OptionCleaningIssue.UNICODE_ISSUES)
    }

    const originalCleaned = cleaned
    cleaned = stripRedundantBrackets(cleaned)
    if (cleaned !== originalCleaned) {
      issues.push(OptionCleaningIssue.NESTED_ARRAY)
    }

    try {
      if (
        (cleaned.startsWith('"') && cleaned.endsWith('"')) ||
        (cleaned.startsWith("'") && cleaned.endsWith("'"))
      ) {
        const parsed = JSON.parse(cleaned)
        if (typeof parsed === 'string') {
          cleaned = parsed.trim()
          issues.push(OptionCleaningIssue.INVALID_JSON)
        }
      }
    } catch (jsonError) {
      issues.push(OptionCleaningIssue.INVALID_JSON)
    }

    const hasEscapeChars = /\\["']/.test(cleaned)
    if (hasEscapeChars) {
      cleaned = cleaned.replace(/\\+"/g, '"').replace(/\\'/g, "'")
      issues.push(OptionCleaningIssue.SPECIAL_CHARS)
    }

    const hasSpecialQuotes = /["'`""]/.test(cleaned)
    if (hasSpecialQuotes) {
      cleaned = cleaned.replace(/["'`""]/g, '')
      issues.push(OptionCleaningIssue.SPECIAL_CHARS)
    }

    cleaned = cleaned.trim()
    if (cleaned.startsWith('\\') && cleaned.endsWith('\\')) {
      cleaned = cleaned.slice(1, -1).trim()
      issues.push(OptionCleaningIssue.SPECIAL_CHARS)
    }

    cleaned = stripRedundantBrackets(cleaned)

    const hasExcessiveWhitespace = /\s{2,}/.test(cleaned)
    if (hasExcessiveWhitespace) {
      cleaned = cleaned.replace(/\s+/g, ' ')
      issues.push(OptionCleaningIssue.EXCESSIVE_WHITESPACE)
    }

    cleaned = cleaned.trim()

    if (logIssues && issues.length > 0) {
      logCleaningIssue(originalValue, cleaned, issues)
    }

    return cleaned
  } catch (error) {
    const errorMsg = `Failed to normalize option value: ${error instanceof Error ? error.message : String(error)}`
    console.error('[FieldPool] Option cleaning error:', {
      originalValue,
      error: errorMsg,
      issues,
    })

    if (throwOnError) {
      throw new Error(errorMsg)
    }

    if (logIssues) {
      logCleaningIssue(originalValue, '', [...issues, OptionCleaningIssue.EMPTY_INPUT])
    }
    return ''
  }
}

function extractLooseArrayOptions(option: string): string[] | null {
  let trimmed = option?.trim?.() ?? ''
  if (!trimmed.startsWith('[') || !trimmed.endsWith(']')) {
    return null
  }

  let inner = trimmed
  let safeGuard = 0
  while (inner.startsWith('[') && inner.endsWith(']') && safeGuard < 5) {
    const next = inner.slice(1, -1).trim()
    if (!next || next === inner) {
      break
    }
    inner = next
    safeGuard += 1
  }

  if (!inner) return []

  const segments = inner
    .split(/[,，]/)
    .map((seg) => normalizeFieldOptionValue(seg, { logIssues: true }))
    .filter(Boolean)

  if (segments.length > 1) {
    return segments
  }

  const single = normalizeFieldOptionValue(inner, { logIssues: true })
  return single ? [single] : null
}

export function parseOptionsStatic(
  raw: string | null | undefined,
  options: {
    logIssues?: boolean
  } = {}
): string[] {
  const { logIssues = false } = options

  if (!raw) return []
  const normalizedOptions: string[] = []
  const cleaningOptions = { logIssues }

  try {
    const parsed = JSON.parse(raw)
    if (Array.isArray(parsed)) {
      parsed.forEach((item, index) => {
        const value = String(item ?? '').trim()
        if (!value) {
          if (logIssues) {
            console.warn(`[FieldPool] Empty option at index ${index}`)
          }
          return
        }

        const nested = extractLooseArrayOptions(value)
        if (nested && nested.length) {
          nested.forEach((nestedValue) => {
            const normalized = normalizeFieldOptionValue(nestedValue, cleaningOptions)
            if (normalized) {
              normalizedOptions.push(normalized)
            }
          })
          return
        }

        const normalized = normalizeFieldOptionValue(value, cleaningOptions)
        if (normalized) {
          normalizedOptions.push(normalized)
        }
      })

      const uniqueOptions = Array.from(new Set(normalizedOptions))
      if (logIssues && uniqueOptions.length < normalizedOptions.length) {
        console.warn(`[FieldPool] Removed ${normalizedOptions.length - uniqueOptions.length} duplicate options`)
      }
      return uniqueOptions
    }
  } catch (jsonError) {
    if (logIssues) {
      console.warn('[FieldPool] Failed to parse options as JSON, falling back to newline parsing')
    }
  }

  const lines = raw.split('\n')
  const expandedOptions = lines.flatMap((line) => expandOptionValue(line, cleaningOptions))
  
  const uniqueOptions = Array.from(new Set(expandedOptions))
  if (logIssues && uniqueOptions.length < expandedOptions.length) {
    console.warn(`[FieldPool] Removed ${expandedOptions.length - uniqueOptions.length} duplicate options from newline format`)
  }
  
  return uniqueOptions
}

function aggregateUploadField(target: AggregatedField, field: UploadField): void {
  const options = parseOptionsStatic(field.optionsStatic)
  options.forEach((option) => {
    if (option) {
      target.options.add(option)
    }
  })
}

export async function syncFieldPoolFromTemplates(): Promise<void> {
  const aggregated = new Map<string, AggregatedField>()

  Object.values(STANDARD_FIELDS).forEach((field) => {
    if (field.addToCategory) {
      aggregated.set(field.key, {
        key: field.key,
        label: field.label,
        type: field.type ?? 'select',
        source: 'standard',
        templateIds: new Set<string>(),
        options: new Set(parseOptionsStatic(field.optionsStatic ?? null)),
      })
    }
  })

  const categoryFields = await prismaAny.uploadField.findMany({
    where: {
      addToCategory: true,
      type: 'select',
    },
    include: {
      template: {
        select: {
          id: true,
        },
      },
    },
    orderBy: [
      { templateId: 'asc' },
      { orderIndex: 'asc' },
    ],
  })

  categoryFields.forEach((field: UploadField) => {
    if (!field.key) return
    const key = field.key.trim()
    if (!key) return
    const existing = aggregated.get(key)
    if (existing) {
      existing.label = field.label || existing.label || key
      aggregateUploadField(existing, field)
      if (field.templateId) {
        existing.templateIds.add(field.templateId)
      }
      return
    }

    const entry: AggregatedField = {
      key,
      label: field.label || key,
      type: field.type || 'select',
      source: STANDARD_FIELDS[key] ? 'standard' : 'custom',
      templateIds: new Set(field.templateId ? [field.templateId] : []),
      options: new Set(parseOptionsStatic(field.optionsStatic)),
    }
    aggregated.set(key, entry)
  })

  await prismaAny.$transaction(async (tx: any) => {
    const existing = (await tx.fieldDefinition.findMany({
      select: { id: true, key: true },
    })) as Array<{ id: string; key: string }>
    const existingKeyMap = new Map(
      existing.map((item: { id: string; key: string }) => [item.key, item.id]),
    )
    const activeKeys = new Set(aggregated.keys())

    const removedKeys = Array.from(existingKeyMap.keys()).filter((key) => !activeKeys.has(key))
    if (removedKeys.length) {
      await tx.fieldOption.deleteMany({
        where: { field: { key: { in: removedKeys } } },
      })
      await tx.fieldDefinition.deleteMany({
        where: { key: { in: removedKeys } },
      })
    }

    const aggregatedEntries = Array.from(aggregated.values())
    for (const entry of aggregatedEntries) {
      const templateIds = Array.from(entry.templateIds)
      
      const existing = await tx.fieldDefinition.findUnique({
        where: { key: entry.key },
        select: { filterDisplayName: true },
      })
      
      const payload = {
        label: entry.label || entry.key,
        type: entry.type || 'select',
        source: entry.source,
        isStandard: entry.source === 'standard',
        sourceTemplateIds: templateIds.length ? JSON.stringify(templateIds) : null,
        filterDisplayName: existing?.filterDisplayName ?? (entry.label || entry.key),
      }

      const fieldDefinition = await tx.fieldDefinition.upsert({
        where: { key: entry.key },
        update: {
          ...payload,
          updatedAt: new Date(),
        },
        create: {
          key: entry.key,
          ...payload,
          isFilterEnabled: entry.source === 'standard',
        },
      })

      await syncFieldOptions(tx, fieldDefinition.id, Array.from(entry.options))
    }
  })
}

async function syncFieldOptions(tx: any, fieldId: string, options: string[]): Promise<void> {
  const uniqueOptions = Array.from(new Set(options.filter(Boolean)))
  const existingOptions = (await tx.fieldOption.findMany({
    where: { fieldId },
    orderBy: { orderIndex: 'asc' },
  })) as Array<{ id: string; value: string }>

  const existingMap = new Map<string, { id: string; value: string }>(
    existingOptions.map((opt: { id: string; value: string }) => [opt.value, opt]),
  )
  let orderIndex = 0

  for (const value of uniqueOptions) {
    const label = value
    const existing = existingMap.get(value)
    if (existing) {
      await tx.fieldOption.update({
        where: { id: existing.id },
        data: {
          label,
          orderIndex,
        },
      })
      orderIndex += 1
      continue
    }

    await tx.fieldOption.create({
      data: {
        fieldId,
        value,
        label,
        orderIndex,
      },
    })
    orderIndex += 1
  }

  const keepValues = uniqueOptions
  await tx.fieldOption.deleteMany({
    where: {
      fieldId,
      value: { notIn: keepValues.length ? keepValues : ['__none__'] },
    },
  })
}

export async function getFieldPoolRecords() {
  return prismaAny.fieldDefinition.findMany({
    orderBy: [
      { filterOrder: 'asc' },
      { label: 'asc' },
    ],
    include: {
      options: {
        orderBy: { orderIndex: 'asc' },
      },
    },
  })
}

export async function getEnabledFilterFields() {
  const fields = await prismaAny.fieldDefinition.findMany({
    where: { isFilterEnabled: true },
    orderBy: [
      { filterOrder: 'asc' },
      { label: 'asc' },
    ],
    include: {
      options: {
        orderBy: { orderIndex: 'asc' },
      },
    },
  })

  return fields.map((field: { key: string; label: string; filterDisplayName: string | null; options: Array<{ value: string }> }) => ({
    key: field.key,
    label: field.filterDisplayName || field.label,
    options: formatFieldOptionValues(field.options),
  }))
}

export async function updateFieldFilterState(fieldKey: string, enabled: boolean) {
  return prismaAny.fieldDefinition.update({
    where: { key: fieldKey },
    data: {
      isFilterEnabled: enabled,
      updatedAt: new Date(),
    },
  })
}

export function formatFieldOptionEntries(
  optionRecords: Array<{ value: string; label?: string | null }>
): Array<{ value: string; label: string }> {
  const entries: Array<{ value: string; label: string }> = []
  const seen = new Set<string>()
  optionRecords.forEach((option) => {
    const values = expandOptionValue(option.value)
    const labelSource =
      typeof option.label === 'string' && option.label.trim() ? option.label : option.value
    const labels = expandOptionValue(labelSource)

    values.forEach((value, index) => {
      if (!value || seen.has(value)) return
      const labelCandidate = labels[index] || labels[0] || normalizeFieldOptionValue(labelSource)
      entries.push({
        value,
        label: labelCandidate || value,
      })
      seen.add(value)
    })
  })
  return entries
}

export function formatFieldOptionValues(optionRecords: Array<{ value: string; label?: string | null }>): string[] {
  return formatFieldOptionEntries(optionRecords).map((entry) => entry.value)
}

export function expandOptionValue(
  raw: string,
  options: {
    logIssues?: boolean
  } = {}
): string[] {
  const { logIssues = false } = options

  if (!raw?.trim()) return []

  const cleaningOptions = { logIssues }
  const attempts = [raw, normalizeFieldOptionValue(raw, cleaningOptions)]

  for (const candidate of attempts) {
    const nested = extractLooseArrayOptions(candidate)
    if (nested && nested.length) {
      return nested
        .map((item) => normalizeFieldOptionValue(item, cleaningOptions))
        .filter(Boolean)
    }
  }

  const normalized = normalizeFieldOptionValue(raw, cleaningOptions)
  return normalized ? [normalized] : []
}

export async function syncFieldPoolWithRetry(
  options: FieldPoolSyncOptions = {}
): Promise<FieldPoolSyncResult> {
  const {
    incremental = false,
    retryOnError = true,
    maxRetries = 3,
    logDetails = true,
  } = options

  const startTime = Date.now()
  let attempt = 0
  let lastError: Error | null = null

  while (attempt < maxRetries) {
    try {
      attempt++

      if (logDetails) {
        console.log(JSON.stringify({
          timestamp: new Date().toISOString(),
          module: 'field-pool',
          action: 'syncStart',
          attempt,
          maxRetries,
          incremental,
        }))
      }

      const result = incremental
        ? await syncFieldPoolIncremental()
        : await syncFieldPoolFull()

      const duration = Date.now() - startTime

      if (logDetails) {
        console.log(JSON.stringify({
          timestamp: new Date().toISOString(),
          module: 'field-pool',
          action: 'syncComplete',
          success: true,
          attempt,
          duration,
          ...result,
        }))
      }

      return {
        ...result,
        success: true,
        duration,
      }
    } catch (error) {
      lastError = error as Error
      const duration = Date.now() - startTime

      console.error('[FieldPool] Sync error:', {
        attempt,
        maxRetries,
        error: lastError.message,
        stack: lastError.stack,
        duration,
      })

      if (!retryOnError || attempt >= maxRetries) {
        break
      }

      const waitTime = Math.pow(2, attempt) * 1000
      console.log(`[FieldPool] Retrying in ${waitTime}ms...`)
      await new Promise(resolve => setTimeout(resolve, waitTime))
    }
  }

  const duration = Date.now() - startTime
  return {
    success: false,
    duration,
    fieldsAdded: 0,
    fieldsUpdated: 0,
    fieldsRemoved: 0,
    optionsAdded: 0,
    optionsRemoved: 0,
    errors: [lastError?.message || '未知错误'],
  }
}

async function syncFieldPoolFull(): Promise<Omit<FieldPoolSyncResult, 'success' | 'duration'>> {
  await syncFieldPoolFromTemplates()

  return {
    fieldsAdded: 0,
    fieldsUpdated: 0,
    fieldsRemoved: 0,
    optionsAdded: 0,
    optionsRemoved: 0,
    errors: [],
  }
}

async function syncFieldPoolIncremental(): Promise<Omit<FieldPoolSyncResult, 'success' | 'duration'>> {
  let fieldsAdded = 0
  let fieldsUpdated = 0
  let fieldsRemoved = 0
  let optionsAdded = 0
  let optionsRemoved = 0
  const errors: string[] = []

  try {
    const categoryFields = await prismaAny.uploadField.findMany({
      where: {
        addToCategory: true,
        type: 'select',
      },
      include: {
        template: {
          select: {
            id: true,
            updatedAt: true,
          },
        },
      },
    })

    const fieldsByKey = new Map<string, UploadField[]>()
    for (const field of categoryFields) {
      const existing = fieldsByKey.get(field.key) || []
      existing.push(field)
      fieldsByKey.set(field.key, existing)
    }

    const existingFields: any[] = await prismaAny.fieldDefinition.findMany({
      select: {
        id: true,
        key: true,
        updatedAt: true,
      },
    })
    const existingFieldMap = new Map(existingFields.map((f) => [f.key, f]))

    for (const [key, fields] of Array.from(fieldsByKey.entries())) {
      try {
        const existing = existingFieldMap.get(key)
        const templateIds = fields.map(f => f.templateId).filter(Boolean)
        const options = new Set<string>()

        for (const field of fields) {
          const fieldOptions = parseOptionsStatic(field.optionsStatic)
          fieldOptions.forEach(opt => options.add(opt))
        }

        if (!existing) {
          await prismaAny.fieldDefinition.create({
            data: {
              key,
              label: fields[0].label,
              type: fields[0].type,
              source: STANDARD_FIELDS[key] ? 'standard' : 'custom',
              isStandard: !!STANDARD_FIELDS[key],
              sourceTemplateIds: JSON.stringify(templateIds),
              filterDisplayName: fields[0].label,
            },
          })
          fieldsAdded++

          const fieldDef: any = await prismaAny.fieldDefinition.findUnique({
            where: { key },
          })
          if (fieldDef) {
            const optionArray = Array.from(options)
            for (let i = 0; i < optionArray.length; i++) {
              await prismaAny.fieldOption.create({
                data: {
                  fieldId: fieldDef.id,
                  value: optionArray[i],
                  label: optionArray[i],
                  orderIndex: i,
                },
              })
              optionsAdded++
            }
          }
        } else {
          await prismaAny.fieldDefinition.update({
            where: { id: existing.id },
            data: {
              sourceTemplateIds: JSON.stringify(templateIds),
              updatedAt: new Date(),
            },
          })
          fieldsUpdated++

          const result = await syncFieldOptionsIncremental(existing.id, Array.from(options))
          optionsAdded += result.added
          optionsRemoved += result.removed
        }
      } catch (error) {
        const errorMsg = `Failed to sync field ${key}: ${error instanceof Error ? error.message : String(error)}`
        console.error(`[FieldPool] ${errorMsg}`)
        errors.push(errorMsg)
      }
    }

    const activeKeys = Array.from(fieldsByKey.keys())
    const orphanedFields = existingFields.filter((f) => !activeKeys.includes(f.key))

    for (const orphaned of orphanedFields) {
      try {
        await prismaAny.fieldOption.deleteMany({
          where: { fieldId: (orphaned as any).id },
        })
        await prismaAny.fieldDefinition.delete({
          where: { id: (orphaned as any).id },
        })
        fieldsRemoved++
      } catch (error) {
        const errorMsg = `Failed to remove orphaned field ${(orphaned as any).key}: ${error instanceof Error ? error.message : String(error)}`
        console.error(`[FieldPool] ${errorMsg}`)
        errors.push(errorMsg)
      }
    }
  } catch (error) {
    const errorMsg = `Incremental sync failed: ${error instanceof Error ? error.message : String(error)}`
    console.error(`[FieldPool] ${errorMsg}`)
    errors.push(errorMsg)
    throw error
  }

  return {
    fieldsAdded,
    fieldsUpdated,
    fieldsRemoved,
    optionsAdded,
    optionsRemoved,
    errors,
  }
}

async function syncFieldOptionsIncremental(
  fieldId: string,
  options: string[]
): Promise<{ added: number; removed: number }> {
  let added = 0
  let removed = 0

  const uniqueOptions = Array.from(new Set(options.filter(Boolean)))
  const existingOptions: any[] = await prismaAny.fieldOption.findMany({
    where: { fieldId },
    orderBy: { orderIndex: 'asc' },
  })

  const existingMap = new Map(
    existingOptions.map((opt) => [opt.value, opt])
  )

  let orderIndex = 0

  for (const value of uniqueOptions) {
    const existing = existingMap.get(value)
    if (existing) {
      await prismaAny.fieldOption.update({
        where: { id: existing.id },
        data: {
          label: value,
          orderIndex,
        },
      })
    } else {
      await prismaAny.fieldOption.create({
        data: {
          fieldId,
          value,
          label: value,
          orderIndex,
        },
      })
      added++
    }
    orderIndex++
  }

  const keepValues = uniqueOptions
  const toDelete = existingOptions.filter((opt) => !keepValues.includes(opt.value))

  if (toDelete.length > 0) {
    await prismaAny.fieldOption.deleteMany({
      where: {
        fieldId,
        value: { notIn: keepValues.length ? keepValues : ['__none__'] },
      },
    })
    removed = toDelete.length
  }

  return { added, removed }
}