@@ -880,26 +878,16 @@ export default function ProductPackageUploader({
disabled={uploading}
/>
{/* 上传按钮 */}
-
-
-
-
-
- 支持 JPG、PNG、WebP 等格式,可一次选择多张或拖拽上传
-
-
+
+
+ JPG / PNG / WebP,支持拖拽
{/* 详情图排序组件 */}
@@ -943,55 +931,33 @@ export default function ProductPackageUploader({
disabled={uploading}
/>
{/* 上传按钮 */}
-
- {psdFiles.length > 0 ? (
-
-
-
- {psdFiles.length}
-
-
- ) : (
-
-
+
+
+
{TASK_DETAIL_SOURCE_FILE_LABEL}
+ {psdFiles.length > 0 && (
+
+ {psdFiles.map((psdFile) => (
+
+ {psdFile.file.name}
+
+
+ ))}
)}
-
-
-
- 支持 {TASK_DETAIL_SOURCE_FILE_LABEL} 格式,可一次选择多个或拖拽上传
-
- {psdFiles.length > 0 && (
-
- {psdFiles.map((psdFile) => (
-
- {psdFile.file.name}
-
-
- ))}
-
- )}
-
diff --git a/docs/生产存储桶接入信息.md b/docs/生产存储桶接入信息.md
index 205c067..9457eaa 100644
--- a/docs/生产存储桶接入信息.md
+++ b/docs/生产存储桶接入信息.md
@@ -1,11 +1,11 @@
# 生产存储桶接入信息
-> 更新时间:2026-05-07
+> 更新时间:2026-05-08
> 本文记录当前素材平台的 MinIO / S3 / uploads 存储事实。
## 1. 当前结论
-- 新素材预览 / 展示文件:写入生产 MinIO 新桶路径,公开 URL 统一为 `https://s3.jiansun.vip/gongxiangpan/public-display/...`。
+- 新素材预览 / 展示文件:写入生产 MinIO 新桶路径,公开 URL 统一为 `https://s3.jiansun.vip/gongxiangpan/由壹点AI/public-display/...`。
- 源文件 / 下载型文件:写入生产服务器 uploads 目录:`/mnt/PoolOne/apps/youyidian-3d/uploads`。
- 旧 `1080p.hk` 桶:只用于极少数老素材兼容,不作为新素材写入目标。
- `/gongxiangpan/uploads/` 是历史混合区,不再作为新展示资源的默认路径。
@@ -26,6 +26,7 @@ S3_ENDPOINT=http://192.168.58.20:9000
S3_BUCKET=gongxiangpan
S3_REGION=us-east-1
S3_PUBLIC_DOMAIN=https://s3.jiansun.vip/gongxiangpan
+S3_FILE_PATH=由壹点AI/
```
代码固定行为:
@@ -47,16 +48,16 @@ S3_PUBLIC_DOMAIN=https://s3.jiansun.vip/gongxiangpan
正式路径:
```text
-https://s3.jiansun.vip/gongxiangpan/public-display/...
+https://s3.jiansun.vip/gongxiangpan/由壹点AI/public-display/...
```
常见前缀:
```text
-public-display/materials/...
-public-display/task-detail/...
-public-display/white-background/...
-public-display/certificate-report/...
+由壹点AI/public-display/materials/...
+由壹点AI/public-display/task-detail/...
+由壹点AI/public-display/white-background/...
+由壹点AI/public-display/certificate-report/...
```
这些资源用于页面展示、AI 平台预览、钉钉返回地址和外部接口展示直链。
@@ -69,7 +70,7 @@ public-display/certificate-report/...
由壹点AI/
```
-它主要用于旧逻辑或非 `public-display` 的对象,不代表新素材展示图默认落点。
+它是当前生产环境的默认业务前缀。新素材展示图应落在 `由壹点AI/public-display/...`,不要写入旧桶。
## 4. uploads 目录规则
@@ -101,21 +102,40 @@ public-display/certificate-report/...
不要把 uploads 理解成“少量视频临时区”。它是当前生产源文件和下载型文件的核心落点。
-## 5. 历史兼容
+## 5. 历史素材正式导入要求
-### 5.1 旧 `1080p.hk` 桶
+历史素材导入有一个特殊约束:执行机要能读取历史素材源目录,同时下载型文件必须最终落到生产 uploads。
+
+当前天蓝盘批量导入的标准做法:
+
+- Windows 维护机读取 `Z:\...` 源素材。
+- 预览图上传到新桶 `gongxiangpan` 的 `由壹点AI/public-display/...`。
+- ZIP / PSD / 源文件先在本机生成,再通过远程同步写入生产机 `/mnt/PoolOne/apps/youyidian-3d/uploads`,并校验远程文件大小。
+
+正式 `--apply` 如果在 Windows 上连接生产库,必须同时配置:
+
+```env
+HISTORY_IMPORT_REMOTE_UPLOADS_HOST=prod
+HISTORY_IMPORT_REMOTE_UPLOADS_DIR=/mnt/PoolOne/apps/youyidian-3d/uploads
+```
+
+没有远程 uploads 同步时,脚本必须阻止正式导入。否则会把数据库写到生产库,但 ZIP / PSD 留在本机 `uploads`,线上下载会 404。
+
+## 6. 历史兼容
+
+### 6.1 旧 `1080p.hk` 桶
`s3.1080p.hk/jsxiangche` 只保留给极少数老素材兼容。新素材、批量新录入素材、后续上传素材都不应继续写入旧桶。
-### 5.2 旧资源域名
+### 6.2 旧资源域名
`s3.yydd.ai` 不再作为默认对接口径,只用于识别历史链接。
-### 5.3 历史混合区
+### 6.3 历史混合区
`/gongxiangpan/uploads/` 保留历史兼容,不再作为新展示图默认路径。
-## 6. 只读核对命令
+## 7. 只读核对命令
核对应用环境变量:
diff --git a/lib/history-material-import.ts b/lib/history-material-import.ts
index 9d07c28..ce0d1c2 100644
--- a/lib/history-material-import.ts
+++ b/lib/history-material-import.ts
@@ -1,7 +1,12 @@
import { prisma } from '@/lib/prisma'
-import { completeProductPackageUpload, type ProductPackageImageInput } from '@/lib/product-package-completion'
+import {
+ completeProductPackageUpload,
+ type ProductPackageImageInput,
+ type ProductPackageSourceInput,
+} from '@/lib/product-package-completion'
import { createMaterialLog } from '@/lib/material-log'
import { isTaskDetailFullSetFromUploadGroups } from '@/lib/task-detail-full-set'
+import { normalizeTaskDetailSourceType } from '@/lib/task-detail-source-type'
import path from 'path'
import { promises as fs } from 'fs'
import crypto from 'crypto'
@@ -33,6 +38,10 @@ const IGNORED_DIR_PATTERNS = [
/废稿/i,
/字体/i,
/勿用/i,
+ /错误勿用/i,
+ /图片有误/i,
+ /作废/i,
+ /3D建模/i,
/DingTalkAppData/i,
/^PsTool$/i,
/^Cache$/i,
@@ -68,6 +77,20 @@ const STORE_DIR_IGNORES = new Set([
const execFileAsync = promisify(execFile)
const HISTORY_ZIP_TIMEOUT_MS = Number(process.env.HISTORY_IMPORT_ZIP_TIMEOUT_MS || 180000)
const HISTORY_ALL_ZIP_TIMEOUT_MS = Number(process.env.HISTORY_IMPORT_ALL_ZIP_TIMEOUT_MS || 300000)
+const BRANDED_SPU_BODY_PATTERN = '[A-Z]{2,5}-[A-Z]{1,6}\\d{3,5}(?:K\\d{1,3})?'
+const UNBRANDED_SPU_BODY_PATTERN = '[A-Z]{1,6}\\d{3,5}(?:K\\d{1,3})?'
+const COMBO_SPU_TOKEN_PATTERN = new RegExp(
+ `(?
+ skippedInvalidFiles: Array<{
+ group: 'main' | 'sku' | 'detail'
+ fileName: string
+ fileSize: number
+ localFilePath?: string
+ reason: string
+ }>
+}
+
function normalizeSpu(value: string) {
- return value.trim().toUpperCase()
+ return value.trim().toUpperCase().replace(/+/g, '+').replace(/\s*\+\s*/g, '+')
+}
+
+function expandComboSpuToken(token: string) {
+ const parts = normalizeSpu(token).split('+').filter(Boolean)
+ const first = parts[0] || ''
+ const prefix = first.includes('-') ? first.split('-')[0] : ''
+
+ if (parts.length < 2 || !prefix) {
+ return null
+ }
+
+ const expanded = parts.map((part, index) => {
+ if (index === 0 || part.includes('-')) {
+ return part
+ }
+ return `${prefix}-${part}`
+ })
+
+ return expanded.join('+')
+}
+
+function inferComboSpuFromPath(sourcePath: string) {
+ const match = sourcePath.match(COMBO_SPU_TOKEN_PATTERN)
+ if (!match) {
+ return null
+ }
+
+ return expandComboSpuToken(match[0])
}
function isSpuName(name: string) {
@@ -688,18 +758,61 @@ async function ensureCategoryAndSceneNames(categoryName: string, sceneName: stri
}
}
-async function hasImportedFingerprint(fingerprint: string) {
- const existing = await prisma.materialLog.findFirst({
+async function hasImportedHistoryRecord(record: HistoryImportRecord) {
+ const existingByFingerprint = await prisma.materialLog.findFirst({
where: {
action: 'HISTORY_IMPORT',
+ material: {
+ deletedAt: null,
+ },
changes: {
- contains: fingerprint,
+ contains: record.fingerprint,
} as any,
},
select: { id: true },
})
- return Boolean(existing)
+ if (existingByFingerprint) {
+ return true
+ }
+
+ const existingBySourcePath = await prisma.$queryRaw
>`
+ SELECT logs.id
+ FROM "material_logs" AS logs
+ INNER JOIN "materials" AS materials
+ ON materials.id = logs."materialId"
+ WHERE logs.action = 'HISTORY_IMPORT'
+ AND materials."deletedAt" IS NULL
+ AND logs.changes IS NOT NULL
+ AND logs.changes::jsonb->>'sourcePath' = ${record.sourcePath}
+ LIMIT 1
+ `
+
+ return existingBySourcePath.length > 0
+}
+
+function getBlockedReadySourceReason(record: HistoryImportRecord) {
+ const haystack = [
+ record.sourcePath,
+ record.relativeSourcePath,
+ record.skipReason || '',
+ ].join('\n')
+ const matched = BLOCKED_READY_SOURCE_PATH_PATTERNS.find(({ pattern }) => pattern.test(haystack))
+
+ if (!matched) {
+ return null
+ }
+
+ return `来源路径包含“${matched.label}”,即使台账标记 ready 也禁止正式导入`
+}
+
+function getSpuMismatchReason(record: HistoryImportRecord) {
+ const inferredComboSpu = inferComboSpuFromPath(record.sourcePath)
+ if (!inferredComboSpu || normalizeSpu(record.spu) === inferredComboSpu) {
+ return null
+ }
+
+ return `来源路径包含组合 SPU“${inferredComboSpu}”,但台账 SPU 是“${record.spu}”,禁止按单 SPU 导入`
}
async function createHistoryImportMaterial(record: HistoryImportRecord, operatorUserId: string) {
@@ -717,12 +830,103 @@ async function createHistoryImportMaterial(record: HistoryImportRecord, operator
renderer: '',
isFullSet: false,
isAiGenerated: false,
+ taskDetailSourceType: normalizeTaskDetailSourceType(record.taskDetailSourceType) || '渲染',
} as any,
})
}
+function filterNonEmptyAssets(
+ group: 'main' | 'sku' | 'detail' | 'psd',
+ assets: T[] | undefined,
+ skippedEmptyFiles: SanitizedHistoryImportRecord['skippedEmptyFiles'],
+) {
+ return (assets || []).filter((asset) => {
+ if (asset.fileSize > 0) {
+ return true
+ }
+
+ skippedEmptyFiles.push({
+ group,
+ fileName: asset.fileName,
+ localFilePath: asset.localFilePath,
+ })
+ return false
+ })
+}
+
+async function filterReadableImageAssets(
+ group: 'main' | 'sku' | 'detail',
+ assets: T[],
+ skippedInvalidFiles: SanitizedHistoryImportRecord['skippedInvalidFiles'],
+) {
+ if (assets.length === 0) {
+ return assets
+ }
+
+ const sharp = await import('sharp')
+ const readableAssets: T[] = []
+
+ for (const asset of assets) {
+ if (!asset.localFilePath) {
+ readableAssets.push(asset)
+ continue
+ }
+
+ try {
+ await sharp
+ .default(asset.localFilePath)
+ .resize(1, 1, {
+ fit: 'inside',
+ withoutEnlargement: true,
+ })
+ .webp({ quality: 1 })
+ .toBuffer()
+ readableAssets.push(asset)
+ } catch (error) {
+ skippedInvalidFiles.push({
+ group,
+ fileName: asset.fileName,
+ fileSize: asset.fileSize,
+ localFilePath: asset.localFilePath,
+ reason: error instanceof Error ? error.message : '图片无法读取',
+ })
+ }
+ }
+
+ return readableAssets
+}
+
+async function sanitizeHistoryImportRecord(record: HistoryImportRecord): Promise {
+ const skippedEmptyFiles: SanitizedHistoryImportRecord['skippedEmptyFiles'] = []
+ const skippedInvalidFiles: SanitizedHistoryImportRecord['skippedInvalidFiles'] = []
+ const nonEmptyMainImages = filterNonEmptyAssets('main', record.mainImages, skippedEmptyFiles)
+ const nonEmptySkuImages = filterNonEmptyAssets('sku', record.skuImages, skippedEmptyFiles)
+ const nonEmptyDetailImages = filterNonEmptyAssets('detail', record.detailImages, skippedEmptyFiles)
+ const sanitizedRecord: HistoryImportRecord = {
+ ...record,
+ mainImages: await filterReadableImageAssets('main', nonEmptyMainImages, skippedInvalidFiles),
+ skuImages: await filterReadableImageAssets('sku', nonEmptySkuImages, skippedInvalidFiles),
+ detailImages: await filterReadableImageAssets('detail', nonEmptyDetailImages, skippedInvalidFiles),
+ psdSources: filterNonEmptyAssets('psd', record.psdSources, skippedEmptyFiles),
+ }
+
+ if (
+ sanitizedRecord.mainImages.length === 0 ||
+ sanitizedRecord.skuImages.length === 0 ||
+ sanitizedRecord.detailImages.length === 0
+ ) {
+ throw new Error(
+ `过滤 0 字节文件后不满足导入条件:主图=${sanitizedRecord.mainImages.length} SKU=${sanitizedRecord.skuImages.length} 详情=${sanitizedRecord.detailImages.length}`,
+ )
+ }
+
+ return { record: sanitizedRecord, skippedEmptyFiles, skippedInvalidFiles }
+}
+
export async function importHistoryMaterials(options: ImportHistoryMaterialsOptions) {
- await ensureCategoryAndScene('其他', '其他')
+ if (options.apply) {
+ await ensureCategoryAndScene('其他', '其他')
+ }
const candidates = options.records
.filter((record) => {
@@ -735,7 +939,35 @@ export async function importHistoryMaterials(options: ImportHistoryMaterialsOpti
const results: ImportHistoryMaterialsResultItem[] = []
for (const record of candidates) {
- if (await hasImportedFingerprint(record.fingerprint)) {
+ const blockedReadySourceReason = getBlockedReadySourceReason(record)
+ if (blockedReadySourceReason) {
+ results.push({
+ fingerprint: record.fingerprint,
+ spu: record.spu,
+ sourcePath: record.sourcePath,
+ guessedStore: record.guessedStore,
+ materialId: null,
+ status: 'skipped',
+ reason: blockedReadySourceReason,
+ })
+ continue
+ }
+
+ const spuMismatchReason = getSpuMismatchReason(record)
+ if (spuMismatchReason) {
+ results.push({
+ fingerprint: record.fingerprint,
+ spu: record.spu,
+ sourcePath: record.sourcePath,
+ guessedStore: record.guessedStore,
+ materialId: null,
+ status: 'skipped',
+ reason: spuMismatchReason,
+ })
+ continue
+ }
+
+ if (await hasImportedHistoryRecord(record)) {
results.push({
fingerprint: record.fingerprint,
spu: record.spu,
@@ -762,16 +994,18 @@ export async function importHistoryMaterials(options: ImportHistoryMaterialsOpti
}
try {
+ const sanitized = await sanitizeHistoryImportRecord(record)
await ensureCategoryAndSceneNames(record.guessedStore || '其他', '其他')
- const material = await createHistoryImportMaterial(record, options.operator.id)
+ const material = await createHistoryImportMaterial(sanitized.record, options.operator.id)
await completeProductPackageUpload({
materialId: material.id,
taskNo: null,
operatorUserId: options.operator.id,
assets: {
- main: record.mainImages,
- sku: record.skuImages,
- detail: record.detailImages,
+ main: sanitized.record.mainImages,
+ sku: sanitized.record.skuImages,
+ detail: sanitized.record.detailImages,
+ psd: sanitized.record.psdSources || [],
},
options: {
allowSkuWithoutBarcodes: true,
@@ -789,7 +1023,11 @@ export async function importHistoryMaterials(options: ImportHistoryMaterialsOpti
sourcePath: record.sourcePath,
guessedStore: record.guessedStore,
storeCandidates: record.storeCandidates,
+ sourceCount: record.sourceCount || record.psdSources?.length || 0,
+ taskDetailSourceType: normalizeTaskDetailSourceType(record.taskDetailSourceType) || '渲染',
seenInYya: record.seenInYya,
+ skippedEmptyFiles: sanitized.skippedEmptyFiles,
+ skippedInvalidFiles: sanitized.skippedInvalidFiles,
},
})
diff --git a/lib/history-sku-csv-import.ts b/lib/history-sku-csv-import.ts
index 93559e4..779d6fd 100644
--- a/lib/history-sku-csv-import.ts
+++ b/lib/history-sku-csv-import.ts
@@ -3,7 +3,7 @@ import { promises as fs } from 'fs'
import crypto from 'crypto'
import type { ProductPackageImageInput } from '@/lib/product-package-completion'
import type { HistoryImportRecord, HistoryImportStatus } from '@/lib/history-material-import'
-import { parseSPU } from '@/lib/spu-parser'
+import { getShopPrefixMap, parseSPU } from '@/lib/spu-parser'
const IMAGE_EXTENSIONS = new Set(['.jpg', '.jpeg', '.png', '.webp'])
const SKIPPED_FILE_EXTENSIONS = new Set([
@@ -50,9 +50,19 @@ const EXCLUDED_FILE_NAME_PATTERNS = [
/mask/i,
]
+const BRANDED_SPU_TOKEN_PATTERN = /([A-Za-z]{2,5}-[A-Za-z]{1,5}\d{3,5})/g
const PRIMARY_SPU_TOKEN_PATTERN = /([A-Za-z]{2,3}\d{4})/g
const FALLBACK_SPU_TOKEN_PATTERN = /([A-Za-z]{1,5}\d{3,5})/g
const IGNORED_SPU_PREFIXES = ['TASK', 'SKU', 'IMG']
+const EXTRA_SHOP_PREFIX_MAP: Record = {
+ GBJP: '顾家',
+}
+const EXTRA_CATEGORY_CODE_MAP: Record = {
+ BI: '边几',
+ CFZWJ: '置物架',
+ SMC: '床',
+}
+const CATEGORY_CODE_PATTERN = /^([A-Z]{1,5})\d/
const AUTHOR_SEGMENT_EXCLUDES = new Set([
'亿家木',
'亿家达',
@@ -216,7 +226,35 @@ function inferSpuFromText(value: string) {
const isStandaloneToken = (matchIndex: number, token: string) => {
const prevChar = matchIndex > 0 ? text[matchIndex - 1] : ''
const nextChar = text[matchIndex + token.length] || ''
- return !/[A-Za-z]/.test(prevChar) && !/[A-Za-z0-9]/.test(nextChar)
+ if (/[A-Za-z0-9]/.test(prevChar)) {
+ return false
+ }
+ if (!/[A-Za-z0-9]/.test(nextChar)) {
+ return true
+ }
+ return /^K\d{1,2}(?![A-Za-z0-9])/i.test(text.slice(matchIndex + token.length))
+ }
+
+ BRANDED_SPU_TOKEN_PATTERN.lastIndex = 0
+ const brandedMatches = Array.from(text.matchAll(BRANDED_SPU_TOKEN_PATTERN))
+ if (brandedMatches.length > 0) {
+ const shopPrefixMap = { ...getShopPrefixMap(), ...EXTRA_SHOP_PREFIX_MAP }
+ for (let index = brandedMatches.length - 1; index >= 0; index -= 1) {
+ const currentMatch = brandedMatches[index]
+ const rawToken = currentMatch?.[1] || ''
+ const token = normalizeSpu(rawToken)
+ const [shopPrefix, skuPart] = token.split('-')
+ const matchIndex = currentMatch?.index ?? -1
+ if (
+ token &&
+ shopPrefixMap[shopPrefix] &&
+ skuPart &&
+ matchIndex >= 0 &&
+ isStandaloneToken(matchIndex, rawToken)
+ ) {
+ return token
+ }
+ }
}
PRIMARY_SPU_TOKEN_PATTERN.lastIndex = 0
@@ -291,12 +329,13 @@ function inferSpu(row: ParsedSkuCsvRow) {
if (row.spu) {
const normalizedRowSpu = normalizeSpu(row.spu)
- const rowSpuLooksReliable =
+ const rowSpuLooksUsable =
normalizedRowSpu &&
!IGNORED_SPU_PREFIXES.some((prefix) => normalizedRowSpu.startsWith(prefix)) &&
- candidates.some((candidate) => hasStandaloneSpuToken(candidate, normalizedRowSpu))
+ (/^[A-Z]{2,5}-[A-Z]{1,5}\d{3,5}$/.test(normalizedRowSpu) ||
+ /^[A-Z]{1,5}\d{3,5}$/.test(normalizedRowSpu))
- if (rowSpuLooksReliable) {
+ if (rowSpuLooksUsable) {
return normalizedRowSpu
}
}
@@ -482,7 +521,25 @@ function buildAssetInputs(spu: string, files: FileEntry[], group: 'main' | 'sku'
}
function inferStoreBySpu(spu: string, row?: ParsedSkuCsvRow) {
+ const sourceRootStore = inferStoreBySourceRoot(row)
+ if (sourceRootStore) {
+ return {
+ guessedStore: sourceRootStore,
+ storeConfidence: 1,
+ storeCandidates: [sourceRootStore],
+ }
+ }
+
const normalizedSpu = normalizeSpu(spu)
+ const spuParseResult = parseSPU(normalizedSpu)
+ if (spuParseResult.shopMatched && spuParseResult.shopName) {
+ return {
+ guessedStore: spuParseResult.shopName,
+ storeConfidence: 1,
+ storeCandidates: [spuParseResult.shopName],
+ }
+ }
+
const storeProbeText = [
normalizedSpu,
row?.relativePath || '',
@@ -493,36 +550,17 @@ function inferStoreBySpu(spu: string, row?: ParsedSkuCsvRow) {
]
.join('\n')
.toUpperCase()
+ const shopPrefixMap = { ...getShopPrefixMap(), ...EXTRA_SHOP_PREFIX_MAP }
+ const matchedPrefix = Object.keys(shopPrefixMap)
+ .sort((left, right) => right.length - left.length)
+ .find((prefix) => new RegExp(`(^|[^A-Z0-9])${prefix}-`).test(storeProbeText))
- if (storeProbeText.includes('GBGJ')) {
+ if (matchedPrefix) {
+ const storeName = shopPrefixMap[matchedPrefix]
return {
- guessedStore: '顾家',
+ guessedStore: storeName,
storeConfidence: 1,
- storeCandidates: ['顾家'],
- }
- }
-
- if (storeProbeText.includes('GBSFY')) {
- return {
- guessedStore: '索菲亚',
- storeConfidence: 1,
- storeCandidates: ['索菲亚'],
- }
- }
-
- if (normalizedSpu.startsWith('GBGJ')) {
- return {
- guessedStore: '顾家',
- storeConfidence: 1,
- storeCandidates: ['顾家'],
- }
- }
-
- if (normalizedSpu.startsWith('GBSFY')) {
- return {
- guessedStore: '索菲亚',
- storeConfidence: 1,
- storeCandidates: ['索菲亚'],
+ storeCandidates: [storeName],
}
}
@@ -533,6 +571,36 @@ function inferStoreBySpu(spu: string, row?: ParsedSkuCsvRow) {
}
}
+function inferStoreBySourceRoot(row?: ParsedSkuCsvRow) {
+ const sourceRoot = row?.sourceRoot || ''
+ const probes = [
+ ['顾家项目', '顾家'],
+ ['金牌项目', '顾家'],
+ ['朴美项目', '朴美'],
+ ['索菲亚项目', '索菲亚'],
+ ] as const
+
+ const matched = probes.find(([keyword]) => sourceRoot.includes(keyword))
+ return matched?.[1] || ''
+}
+
+function inferProductCategoryBySpu(spu: string) {
+ const spuParseResult = parseSPU(spu)
+ if (spuParseResult.categoryMatched) {
+ return spuParseResult.categoryName
+ }
+
+ const normalizedSpu = normalizeSpu(spu)
+ const skuPart = normalizedSpu.includes('-') ? normalizedSpu.split('-').pop() || '' : normalizedSpu
+ const skuPartParseResult = parseSPU(skuPart)
+ if (skuPartParseResult.categoryMatched) {
+ return skuPartParseResult.categoryName
+ }
+
+ const categoryCode = skuPart.match(CATEGORY_CODE_PATTERN)?.[1] || ''
+ return EXTRA_CATEGORY_CODE_MAP[categoryCode] || ''
+}
+
function normalizeAuthorToken(value: string) {
return value.replace(/[\s()()_\-]+/g, '').trim().toLowerCase()
}
@@ -608,6 +676,23 @@ function makeFingerprint(spu: string, sourcePath: string) {
return crypto.createHash('sha1').update(`${normalizeSpu(spu)}|${sourcePath}`).digest('hex')
}
+function normalizeDuplicatePathKey(relativeSourcePath: string) {
+ const segments = relativeSourcePath
+ .split(/[\\/]+/)
+ .map((item) => item.trim())
+ .filter(Boolean)
+
+ if (segments.length > 1 && segments[0] === segments[1]) {
+ segments.shift()
+ }
+
+ return segments.join('/').toLowerCase()
+}
+
+function makeDuplicateKey(spu: string, relativeSourcePath: string) {
+ return `${normalizeSpu(spu)}|${normalizeDuplicatePathKey(relativeSourcePath)}`
+}
+
export async function buildHistoryImportRecordsFromSkuCsv(
options: BuildHistoryImportRecordsFromSkuCsvOptions,
) {
@@ -624,9 +709,16 @@ export async function buildHistoryImportRecordsFromSkuCsv(
new Set((options.visualCenterAuthors || []).map((item) => item.trim()).filter(Boolean)),
)
const records: HistoryImportRecord[] = []
+ const seenDuplicateKeys = new Set()
for (const row of limitedRows) {
const spu = inferSpu(row)
+ const duplicateKey = makeDuplicateKey(spu || row.directoryName || 'UNKNOWN', row.relativePath)
+ if (seenDuplicateKeys.has(duplicateKey)) {
+ continue
+ }
+ seenDuplicateKeys.add(duplicateKey)
+
const detailDirName = pickPreferredDirName(row.siblingDetailDirNames, 'detail')
const mainDirName = pickPreferredDirName(row.siblingMainDirNames, 'main')
@@ -636,8 +728,7 @@ export async function buildHistoryImportRecordsFromSkuCsv(
const storeGuess = inferStoreBySpu(spu, row)
const hasFullSet = mainFiles.length > 0 && skuFiles.length > 0 && detailFiles.length > 0
- const spuParseResult = parseSPU(spu)
- const productCategoryName = spuParseResult.categoryMatched ? spuParseResult.categoryName : ''
+ const productCategoryName = inferProductCategoryBySpu(spu)
const authorName = inferAuthorFromRow(row, visualCenterAuthors)
let status: HistoryImportStatus = 'ready'
diff --git a/lib/product-package-completion.ts b/lib/product-package-completion.ts
index c519c10..535fa3c 100644
--- a/lib/product-package-completion.ts
+++ b/lib/product-package-completion.ts
@@ -7,6 +7,8 @@ import {
createZipFromZips,
saveProductPackageAssets,
} from '@/lib/task-material-processor'
+import { buildStoredUploadPath, getUploadRootDir } from '@/lib/upload-paths'
+import { isRemoteUploadSyncEnabled, syncStoredUploadFileToRemote } from '@/lib/remote-upload-sync'
import { getUploadedFile } from '@/lib/upload-sessions'
import { createMaterialLog } from '@/lib/material-log'
import { isTaskDetailFullSetFromUploadGroups } from '@/lib/task-detail-full-set'
@@ -289,8 +291,8 @@ async function movePsdFilesToPermanentDir(
return psdPermanentPaths
}
- const { rename, copyFile } = await import('fs/promises')
- const psdDir = path.join(process.cwd(), 'uploads', 'materials')
+ const { rename, copyFile, stat } = await import('fs/promises')
+ const psdDir = path.join(getUploadRootDir(), 'materials')
await mkdir(psdDir, { recursive: true })
for (const psd of assets) {
@@ -299,15 +301,36 @@ async function movePsdFilesToPermanentDir(
throw new Error(`PSD 资产参数无效: ${psd.fileName}`)
}
- const tempFilePath = await downloadToTempFile('psd', psd)
const timestamp = Date.now()
const sanitizedFileName = psd.fileName.replace(/[\/\\:*?"<>|]/g, '_')
const permanentFileName = `psd-${materialId}-${timestamp}-${sanitizedFileName}`
const permanentFilePath = path.join(psdDir, permanentFileName)
+ const relativePath = buildStoredUploadPath('materials', permanentFileName)
+
+ if (psd.localFilePath) {
+ const sourceStat = await stat(psd.localFilePath)
+ const expectedSize = psd.fileSize || sourceStat.size
+
+ if (isRemoteUploadSyncEnabled()) {
+ await syncStoredUploadFileToRemote(psd.localFilePath, relativePath, expectedSize)
+ } else {
+ await copyFile(psd.localFilePath, permanentFilePath)
+ }
+
+ psdPermanentPaths.push({
+ filePath: relativePath,
+ fileName: psd.fileName,
+ fileSize: expectedSize,
+ })
+ continue
+ }
+
+ const tempFilePath = await downloadToTempFile('psd', psd)
await copyFile(tempFilePath, permanentFilePath)
+ await syncStoredUploadFileToRemote(permanentFilePath, relativePath, psd.fileSize)
tempFiles.push(tempFilePath)
psdPermanentPaths.push({
- filePath: path.join('uploads', 'materials', permanentFileName),
+ filePath: relativePath,
fileName: psd.fileName,
fileSize: psd.fileSize,
})
@@ -323,7 +346,7 @@ async function movePsdFilesToPermanentDir(
const sanitizedFileName = psd.fileName.replace(/[\/\\:*?"<>|]/g, '_')
const permanentFileName = `psd-${materialId}-${timestamp}-${sanitizedFileName}`
const permanentFilePath = path.join(psdDir, permanentFileName)
- const relativePath = path.join('uploads', 'materials', permanentFileName)
+ const relativePath = buildStoredUploadPath('materials', permanentFileName)
try {
await rename(tempFilePath, permanentFilePath)
@@ -332,6 +355,8 @@ async function movePsdFilesToPermanentDir(
tempFiles.push(tempFilePath)
}
+ await syncStoredUploadFileToRemote(permanentFilePath, relativePath, psd.fileSize)
+
psdPermanentPaths.push({
filePath: relativePath,
fileName: psd.fileName,
@@ -468,6 +493,7 @@ export async function completeProductPackageUpload(input: CompleteProductPackage
createdZipPaths.push(zipInfos.all.filePath)
const psdPermanentPaths = await movePsdFilesToPermanentDir(materialId, normalizedAssets.psd, tempFiles)
+ createdZipPaths.push(...psdPermanentPaths.map((item) => item.filePath))
const saveResult = await saveProductPackageAssets(materialId, previewAssets, zipInfos, psdPermanentPaths)
diff --git a/lib/remote-upload-sync.ts b/lib/remote-upload-sync.ts
new file mode 100644
index 0000000..118877d
--- /dev/null
+++ b/lib/remote-upload-sync.ts
@@ -0,0 +1,214 @@
+import path from 'path'
+import { execFile } from 'child_process'
+import { stat } from 'fs/promises'
+import { promisify } from 'util'
+
+const execFileAsync = promisify(execFile)
+const REMOTE_SYNC_SSH_TIMEOUT_MS = Number(process.env.REMOTE_SYNC_SSH_TIMEOUT_MS || 30000)
+const REMOTE_SYNC_SCP_TIMEOUT_MS = Number(process.env.REMOTE_SYNC_SCP_TIMEOUT_MS || 7200000)
+const REMOTE_SYNC_RETRY_COUNT = Number(process.env.REMOTE_SYNC_RETRY_COUNT || 3)
+const SSH_NON_INTERACTIVE_ARGS = [
+ '-o',
+ 'BatchMode=yes',
+ '-o',
+ 'ConnectTimeout=15',
+ '-o',
+ 'ServerAliveInterval=15',
+ '-o',
+ 'ServerAliveCountMax=2',
+]
+
+interface RemoteUploadSyncConfig {
+ target: string
+ uploadRoot: string
+}
+
+function cleanEnv(value: string | undefined) {
+ return String(value || '').trim()
+}
+
+function shellQuote(value: string) {
+ return `'${value.replace(/'/g, `'\\''`)}'`
+}
+
+function normalizeRemotePath(value: string) {
+ return value.replace(/\\/g, '/').replace(/\/+$/, '')
+}
+
+function sleep(ms: number) {
+ return new Promise((resolve) => setTimeout(resolve, ms))
+}
+
+async function execFileWithRetry(
+ file: string,
+ args: string[],
+ options: { timeout: number },
+ label: string,
+) {
+ let lastError: unknown
+ for (let attempt = 1; attempt <= REMOTE_SYNC_RETRY_COUNT; attempt += 1) {
+ try {
+ return await execFileAsync(file, args, options)
+ } catch (error) {
+ lastError = error
+ const message = error instanceof Error ? error.message : String(error)
+ console.warn(
+ `[RemoteUploadSync] ${label} 失败 ${attempt}/${REMOTE_SYNC_RETRY_COUNT}: ${message}`,
+ )
+ if (attempt < REMOTE_SYNC_RETRY_COUNT) {
+ await sleep(1000 * attempt)
+ }
+ }
+ }
+
+ throw lastError
+}
+
+function relativeToUploadRoot(storedPath: string) {
+ const normalized = String(storedPath || '').replace(/\\/g, '/')
+ const relative = normalized.replace(/^\.?\/*uploads\//i, '')
+
+ if (!relative || relative === normalized && path.posix.isAbsolute(normalized)) {
+ throw new Error(`远程同步路径无效: ${storedPath}`)
+ }
+
+ if (relative.split('/').some((segment) => segment === '..')) {
+ throw new Error(`远程同步路径不能包含上级目录: ${storedPath}`)
+ }
+
+ return relative
+}
+
+export function getRemoteUploadSyncConfig(): RemoteUploadSyncConfig | null {
+ const target = cleanEnv(
+ process.env.HISTORY_IMPORT_REMOTE_UPLOADS_HOST || process.env.UPLOAD_REMOTE_SYNC_HOST,
+ )
+ const uploadRoot = cleanEnv(
+ process.env.HISTORY_IMPORT_REMOTE_UPLOADS_DIR || process.env.UPLOAD_REMOTE_SYNC_ROOT,
+ )
+
+ if (!target && !uploadRoot) {
+ return null
+ }
+
+ if (!target || !uploadRoot) {
+ throw new Error(
+ '远程 uploads 同步配置不完整:需要同时配置 HISTORY_IMPORT_REMOTE_UPLOADS_HOST 和 HISTORY_IMPORT_REMOTE_UPLOADS_DIR',
+ )
+ }
+
+ return {
+ target,
+ uploadRoot: normalizeRemotePath(uploadRoot),
+ }
+}
+
+export function isRemoteUploadSyncEnabled() {
+ return Boolean(getRemoteUploadSyncConfig())
+}
+
+export function getRemoteUploadPath(storedPath: string) {
+ const config = getRemoteUploadSyncConfig()
+ if (!config) {
+ return null
+ }
+
+ return path.posix.join(config.uploadRoot, relativeToUploadRoot(storedPath))
+}
+
+export async function assertRemoteUploadSyncAccess() {
+ const config = getRemoteUploadSyncConfig()
+ if (!config) {
+ return
+ }
+
+ const materialsDir = path.posix.join(config.uploadRoot, 'materials')
+ await execFileWithRetry(
+ 'ssh',
+ [
+ ...SSH_NON_INTERACTIVE_ARGS,
+ config.target,
+ `test -d ${shellQuote(materialsDir)} && test -w ${shellQuote(materialsDir)}`,
+ ],
+ { timeout: REMOTE_SYNC_SSH_TIMEOUT_MS },
+ '检查远程 uploads 权限',
+ )
+}
+
+export async function syncStoredUploadFileToRemote(
+ localFilePath: string,
+ storedPath: string,
+ expectedSize?: number,
+) {
+ const config = getRemoteUploadSyncConfig()
+ if (!config) {
+ return null
+ }
+
+ const remotePath = getRemoteUploadPath(storedPath)
+ if (!remotePath) {
+ return null
+ }
+
+ const localStat = await stat(localFilePath)
+ const expectedFileSize = expectedSize || localStat.size
+ const remoteDir = path.posix.dirname(remotePath)
+ const sizeMiB = (expectedFileSize / 1024 / 1024).toFixed(2)
+
+ console.log(
+ `[RemoteUploadSync] 准备同步 ${storedPath} -> ${config.target}:${remotePath} (${sizeMiB} MiB)`,
+ )
+ await execFileWithRetry(
+ 'ssh',
+ [...SSH_NON_INTERACTIVE_ARGS, config.target, `mkdir -p ${shellQuote(remoteDir)}`],
+ { timeout: REMOTE_SYNC_SSH_TIMEOUT_MS },
+ `创建远程目录 ${remoteDir}`,
+ )
+ console.log(`[RemoteUploadSync] 开始复制 ${storedPath} (${sizeMiB} MiB)`)
+ await execFileWithRetry(
+ 'scp',
+ ['-q', ...SSH_NON_INTERACTIVE_ARGS, localFilePath, `${config.target}:${remotePath}`],
+ { timeout: REMOTE_SYNC_SCP_TIMEOUT_MS },
+ `复制 ${storedPath}`,
+ )
+
+ const { stdout } = await execFileWithRetry(
+ 'ssh',
+ [
+ ...SSH_NON_INTERACTIVE_ARGS,
+ config.target,
+ `test -f ${shellQuote(remotePath)} && stat -c %s ${shellQuote(remotePath)}`,
+ ],
+ { timeout: REMOTE_SYNC_SSH_TIMEOUT_MS },
+ `校验远程文件 ${remotePath}`,
+ )
+ const remoteSize = Number(String(stdout || '').trim())
+
+ if (remoteSize !== expectedFileSize) {
+ throw new Error(
+ `远程 uploads 同步校验失败: ${storedPath} 本地=${expectedFileSize} 远程=${remoteSize || 'unknown'}`,
+ )
+ }
+
+ console.log(`[RemoteUploadSync] ${storedPath} -> ${config.target}:${remotePath} (${remoteSize} bytes)`)
+ return remotePath
+}
+
+export async function deleteStoredUploadFileFromRemote(storedPath: string) {
+ const config = getRemoteUploadSyncConfig()
+ if (!config) {
+ return
+ }
+
+ const remotePath = getRemoteUploadPath(storedPath)
+ if (!remotePath) {
+ return
+ }
+
+ await execFileWithRetry(
+ 'ssh',
+ [...SSH_NON_INTERACTIVE_ARGS, config.target, `rm -f -- ${shellQuote(remotePath)}`],
+ { timeout: REMOTE_SYNC_SSH_TIMEOUT_MS },
+ `删除远程文件 ${remotePath}`,
+ )
+}
diff --git a/lib/spu-parser.ts b/lib/spu-parser.ts
index 558099a..9b704d8 100644
--- a/lib/spu-parser.ts
+++ b/lib/spu-parser.ts
@@ -32,6 +32,7 @@ const SHOP_PREFIX_MAP: Record = {
'GBPM': '朴美',
'GBSFY': '索菲亚',
'GBGJ': '顾家',
+ 'GBJP': '顾家',
'XYL': '橡野里',
}
diff --git a/lib/task-material-processor.ts b/lib/task-material-processor.ts
index 93b154d..9ec604c 100644
--- a/lib/task-material-processor.ts
+++ b/lib/task-material-processor.ts
@@ -10,11 +10,15 @@ import {
uploadPublicDisplayToS3,
} from '@/lib/public-display';
import { compressImage } from '@/lib/image-processor';
+import { buildStoredUploadPath, getUploadRootDir, resolveStoredUploadPath } from '@/lib/upload-paths';
+import {
+ deleteStoredUploadFileFromRemote,
+ syncStoredUploadFileToRemote,
+} from '@/lib/remote-upload-sync';
import { mkdir, unlink, readFile, rm } from 'fs/promises';
import { createWriteStream } from 'fs';
import path from 'path';
import archiver from 'archiver';
-import { deleteUploadSession } from '@/lib/upload-sessions-redis';
// 资产类型常量
export const ASSET_TYPE = {
@@ -92,7 +96,7 @@ export async function createZipFromFiles(
}
// 创建目标目录
- const uploadsDir = path.join(process.cwd(), 'uploads', 'materials');
+ const uploadsDir = path.join(getUploadRootDir(), 'materials');
await mkdir(uploadsDir, { recursive: true });
// 生成zip文件名和路径(保留中文,只清理文件系统不安全字符)
@@ -105,14 +109,14 @@ export async function createZipFromFiles(
if (authorName && categoryName && zipType !== 'all') {
const sanitizedAuthorName = authorName.replace(/[\/\\:*?"<>|]/g, '_');
const sanitizedCategoryName = categoryName.replace(/[\/\\:*?"<>|]/g, '_');
- zipFileName = `${sanitizedProductModel}_${sanitizedAuthorName}_${sanitizedCategoryName}_${zipType}.zip`;
+ zipFileName = `${sanitizedProductModel}_${sanitizedAuthorName}_${sanitizedCategoryName}_${zipType}_${materialId}_${timestamp}.zip`;
} else {
// 保持原有命名格式(用于all.zip或其他情况)
zipFileName = `${sanitizedProductModel}_${zipType}_${materialId}_${timestamp}.zip`;
}
const zipFilePath = path.join(uploadsDir, zipFileName);
- const relativeZipPath = path.join('uploads', 'materials', zipFileName);
+ const relativeZipPath = buildStoredUploadPath('materials', zipFileName);
console.log(`[Zip] Creating ${zipType} zip file for ${materialType}:`, zipFilePath);
console.log(`[Zip] Files count:`, files.length);
@@ -153,17 +157,23 @@ export async function createZipFromFiles(
});
// 监听完成事件
- output.on('close', () => {
+ output.on('close', async () => {
clearTimeout(timeout);
const fileSize = archive.pointer();
console.log(`[Zip] ${zipType} zip file created successfully: ${fileSize} bytes`);
- cleanup();
- resolve({
- filePath: relativeZipPath,
- fileName: zipFileName,
- fileSize
- });
+ try {
+ await syncStoredUploadFileToRemote(zipFilePath, relativeZipPath, fileSize);
+ cleanup();
+ resolve({
+ filePath: relativeZipPath,
+ fileName: zipFileName,
+ fileSize
+ });
+ } catch (error) {
+ cleanup();
+ reject(error);
+ }
});
// 监听错误
@@ -230,7 +240,7 @@ export async function createZipFromZips(
const { access } = await import('fs/promises');
for (const file of zipFiles) {
try {
- const fullPath = path.isAbsolute(file.path) ? file.path : path.join(process.cwd(), file.path);
+ const fullPath = path.isAbsolute(file.path) ? file.path : resolveStoredUploadPath(file.path);
await access(fullPath);
} catch (error) {
throw new Error(`文件不存在: ${file.path}`);
@@ -238,7 +248,7 @@ export async function createZipFromZips(
}
// 创建目标目录
- const uploadsDir = path.join(process.cwd(), 'uploads', 'materials');
+ const uploadsDir = path.join(getUploadRootDir(), 'materials');
await mkdir(uploadsDir, { recursive: true });
// 生成zip文件名和路径
@@ -246,7 +256,7 @@ export async function createZipFromZips(
const sanitizedProductModel = productModel.replace(/[\/\\:*?"<>|]/g, '_');
const zipFileName = `${sanitizedProductModel}_all_${materialId}_${timestamp}.zip`;
const zipFilePath = path.join(uploadsDir, zipFileName);
- const relativeZipPath = path.join('uploads', 'materials', zipFileName);
+ const relativeZipPath = buildStoredUploadPath('materials', zipFileName);
console.log(`[Zip] Creating all.zip for ${materialType}:`, zipFilePath);
console.log(`[Zip] Zip files count:`, zipFiles.length);
@@ -285,17 +295,23 @@ export async function createZipFromZips(
zlib: { level: 9 } // 最高压缩级别
});
- output.on('close', () => {
+ output.on('close', async () => {
clearTimeout(timeout);
const fileSize = archive.pointer();
console.log(`[Zip] all.zip created successfully: ${fileSize} bytes`);
- cleanup();
- resolve({
- filePath: relativeZipPath,
- fileName: zipFileName,
- fileSize
- });
+ try {
+ await syncStoredUploadFileToRemote(zipFilePath, relativeZipPath, fileSize);
+ cleanup();
+ resolve({
+ filePath: relativeZipPath,
+ fileName: zipFileName,
+ fileSize
+ });
+ } catch (error) {
+ cleanup();
+ reject(error);
+ }
});
archive.on('error', (err: Error) => {
@@ -326,7 +342,7 @@ export async function createZipFromZips(
// 只添加zip文件,不添加PSD文件
for (const file of zipFiles) {
- const fullPath = path.isAbsolute(file.path) ? file.path : path.join(process.cwd(), file.path);
+ const fullPath = path.isAbsolute(file.path) ? file.path : resolveStoredUploadPath(file.path);
console.log(`[Zip] Adding zip file: ${file.name}`);
archive.file(fullPath, { name: file.name });
}
@@ -475,6 +491,7 @@ export async function cleanupTempFiles(
// 清理Redis上传会话(关键修复:防止孤立会话堆积)
if (uploadIds.length > 0) {
console.log(`[Cleanup] Cleaning up ${uploadIds.length} Redis upload sessions...`);
+ const { deleteUploadSession } = await import('@/lib/upload-sessions-redis');
for (const uploadId of uploadIds) {
try {
await deleteUploadSession(uploadId);
@@ -516,7 +533,13 @@ export async function cleanupOnError(
console.log(`[Cleanup] Cleaning up ${paths.length} local zip files...`);
for (const zipFilePath of paths) {
try {
- const fullPath = path.join(process.cwd(), zipFilePath);
+ await deleteStoredUploadFileFromRemote(zipFilePath);
+ } catch (cleanupError) {
+ console.error('[Cleanup] Failed to delete remote uploads file:', cleanupError);
+ }
+
+ try {
+ const fullPath = resolveStoredUploadPath(zipFilePath);
await unlink(fullPath);
console.log(`[Cleanup] Deleted local zip file: ${fullPath}`);
} catch (cleanupError) {
diff --git a/lib/upload-paths.ts b/lib/upload-paths.ts
new file mode 100644
index 0000000..37da7f0
--- /dev/null
+++ b/lib/upload-paths.ts
@@ -0,0 +1,26 @@
+import path from 'path'
+
+export function buildStoredUploadPath(...segments: string[]) {
+ return path.posix.join('uploads', ...segments.map((segment) => String(segment || '').replace(/\\/g, '/')))
+}
+
+export function getUploadRootDir(): string {
+ const configuredUploadDir = String(process.env.UPLOAD_DIR || '').trim()
+ if (!configuredUploadDir) {
+ return path.join(process.cwd(), 'uploads')
+ }
+
+ return path.isAbsolute(configuredUploadDir)
+ ? configuredUploadDir
+ : path.resolve(process.cwd(), configuredUploadDir)
+}
+
+export function resolveStoredUploadPath(storedPath: string): string {
+ const normalizedPath = String(storedPath || '').replace(/\\/g, '/')
+ if (path.isAbsolute(normalizedPath)) {
+ return normalizedPath
+ }
+
+ const relativeToUploadRoot = normalizedPath.replace(/^\.?\/*uploads\//i, '')
+ return path.join(getUploadRootDir(), relativeToUploadRoot)
+}
diff --git a/scripts/README.md b/scripts/README.md
index 3e2d1ff..49fadac 100644
--- a/scripts/README.md
+++ b/scripts/README.md
@@ -1,6 +1,6 @@
# 脚本说明
-> 更新时间:2026-05-07
+> 更新时间:2026-05-08
> 部署相关脚本以 `scripts/git-incremental-deploy.ps1` 为当前默认口径。
详细说明:
@@ -102,3 +102,67 @@
### `scripts/execute-migration.sh`
Linux / macOS 版本的一次性迁移工具,适用范围同上。
+
+## 5. 历史素材 SPU 台账与导入脚本
+
+### `scripts/scan_spu_ledger.py` ⭐ 当前历史素材新入口
+
+按 SPU / 组合 SPU / 版本工作区维度扫描真实素材目录,生成 SQLite 台账。
+
+重要口径:
+
+- 不读取旧的 SKU 锚点 CSV/SQLite 输出。
+- 组合 SPU 原样保留,例如 `GBSFY-AD0297+GBSFY-CZ0102`。
+- 目录名不是标准 SPU 时,即使文件名能反推,也只标记 `needs_review`。
+- 店铺归属按当前业务口径:`金牌项目-1` 和 `GBJP-*` 归到 `顾家`,不再使用 `金牌` 店铺。
+- `正版`、`盗版` 都允许进入候选;`错误勿用`、`废稿`、`勿用` 默认跳过。
+- ready 要求同时存在主图、SKU 图、详情图。
+
+示例:
+
+```powershell
+python scripts\scan_spu_ledger.py `
+ --roots "Z:\顾家项目|Z:\金牌项目-1|Z:\朴美项目|Z:\索菲亚项目" `
+ --db-path "output\tianlan-spu-ledger.db" `
+ --source-batch "tianlan-z" `
+ --reset
+```
+
+### `scripts/export_spu_ledger.py`
+
+从 SPU 台账导出 review CSV、每 20 个工作区一个批次 CSV、汇总 JSON,以及可供历史导入脚本使用的 ready JSON。
+
+```powershell
+python scripts\export_spu_ledger.py `
+ --db-path "output\tianlan-spu-ledger.db" `
+ --output-dir "output" `
+ --prefix "tianlan-spu-ledger"
+```
+
+### `scripts/import-history-materials.ts` / `scripts/run-history-import-queue.ts`
+
+继续作为历史素材导入执行入口。新的 SPU 台账 ready JSON 可以直接作为输入,并支持 `任务-详情` 的 PS 源文件通道。
+
+正式导入的关键边界:
+
+- Windows 维护机可以读取 `Z:\...` 源素材,但不能只靠本机 `uploads` 写生产库。
+- 连接生产库并执行 `--apply` 时,预览图必须写入新桶 `gongxiangpan`,公开地址必须是 `https://s3.jiansun.vip/gongxiangpan/由壹点AI/public-display/...`。
+- ZIP / PSD / 源文件必须最终同步到生产机 `/mnt/PoolOne/apps/youyidian-3d/uploads`,并通过大小校验。
+- 在 Windows 上正式导入生产库时,必须启用:
+
+```env
+HISTORY_IMPORT_REMOTE_UPLOADS_HOST=prod
+HISTORY_IMPORT_REMOTE_UPLOADS_DIR=/mnt/PoolOne/apps/youyidian-3d/uploads
+```
+
+没有远程 uploads 同步配置时,脚本会阻止 `--apply`,避免再次出现数据库有记录但线上下载 404。
+
+### `scripts/hard-delete-history-materials.ts`
+
+只用于已经确认的旧批次错误素材硬删除。默认只做备份/演练;正式执行必须同时满足:
+
+- 候选清单里有 `materialId` 和删除原因。
+- 候选清单提供的 `fingerprint` 或 `sourcePath` 能在该素材的 `HISTORY_IMPORT` 日志中匹配。
+- 命令包含 `--apply --confirm=HARD_DELETE_HISTORY_MATERIALS`。
+
+未确认清单前禁止执行正式硬删除。
diff --git a/scripts/build_import_records_from_package_candidates.py b/scripts/build_import_records_from_package_candidates.py
new file mode 100644
index 0000000..2eb6deb
--- /dev/null
+++ b/scripts/build_import_records_from_package_candidates.py
@@ -0,0 +1,233 @@
+#!/usr/bin/env python3
+"""
+Build HistoryImportRecord JSON from reviewed package candidates.
+
+This script is read-only. It scans selected package paths and emits the JSON
+shape consumed by scripts/import-history-materials.ts. Formal import should
+still be done through that TypeScript script so environment guards, duplicate
+checks and validation stay in one place.
+"""
+
+from __future__ import annotations
+
+import argparse
+import csv
+import json
+import os
+from pathlib import Path
+from typing import Any
+
+from scan_spu_ledger import (
+ EXCLUDED_DIR_NAMES,
+ RELEVANT_EXTENSIONS,
+ build_sku_code,
+ classify_file,
+ extract_spu_candidates,
+ fingerprint_for,
+ infer_author,
+ infer_task_detail_source_type,
+ is_compatible_spu_token,
+ load_visual_author_names,
+ normalize_spu,
+ safe_relative,
+ spu_prefix,
+)
+
+
+DEFAULT_ROOTS = [
+ r"Z:\顾家项目",
+ r"Z:\金牌项目-1",
+ r"Z:\朴美项目",
+ r"Z:\索菲亚项目",
+]
+
+KNOWN_AUTO_PREFIXES = {"GBPM", "GBSFY", "GBGJ", "GBJP"}
+
+
+def json_dumps(value: Any) -> str:
+ return json.dumps(value, ensure_ascii=False, indent=2)
+
+
+def norm_path(value: str) -> str:
+ return os.path.normcase(os.path.normpath(value))
+
+
+def find_source_root(package_path: str, roots: list[str]) -> str:
+ package_norm = norm_path(package_path)
+ matches = [root for root in roots if package_norm.startswith(norm_path(root) + os.sep.lower()) or package_norm == norm_path(root)]
+ if not matches:
+ raise ValueError(f"无法识别 source_root: {package_path}")
+ return max(matches, key=len)
+
+
+def image_input(file_path: Path, package_root: Path, group: str, spu: str, index: int) -> dict[str, Any]:
+ payload: dict[str, Any] = {
+ "localFilePath": str(file_path),
+ "fileName": file_path.name,
+ "fileSize": int(file_path.stat().st_size),
+ "sortOrder": index,
+ }
+ if group == "sku":
+ payload["skuCode"] = build_sku_code(spu, file_path.name, index)
+ payload["barcodes"] = []
+ return payload
+
+
+def source_input(file_path: Path) -> dict[str, Any]:
+ return {
+ "localFilePath": str(file_path),
+ "fileName": file_path.name,
+ "fileSize": int(file_path.stat().st_size),
+ }
+
+
+def collect_assets(package_path: str, spu: str) -> dict[str, list[dict[str, Any]]]:
+ package_root = Path(package_path)
+ grouped_files: dict[str, list[Path]] = {"main": [], "sku": [], "detail": [], "source": []}
+
+ for current, dir_names, file_names in os.walk(package_root):
+ dir_names[:] = [name for name in dir_names if name not in EXCLUDED_DIR_NAMES]
+ current_path = Path(current)
+ for file_name in file_names:
+ file_path = current_path / file_name
+ extension = file_path.suffix.lower()
+ if extension not in RELEVANT_EXTENSIONS:
+ continue
+ relative_path = safe_relative(file_path, package_root)
+ group = classify_file(relative_path, file_name, extension)
+ if group in grouped_files:
+ grouped_files[group].append(file_path)
+
+ for group in grouped_files:
+ grouped_files[group].sort(key=lambda item: safe_relative(item, package_root).lower())
+
+ return {
+ "mainImages": [image_input(path, package_root, "main", spu, index) for index, path in enumerate(grouped_files["main"])],
+ "skuImages": [image_input(path, package_root, "sku", spu, index) for index, path in enumerate(grouped_files["sku"])],
+ "detailImages": [image_input(path, package_root, "detail", spu, index) for index, path in enumerate(grouped_files["detail"])],
+ "psdSources": [source_input(path) for path in grouped_files["source"]],
+ }
+
+
+def build_record(row: dict[str, str], roots: list[str], visual_author_names: list[str]) -> dict[str, Any]:
+ package_path = row["package_path"]
+ spu = normalize_spu(row["spu"])
+ source_root = find_source_root(package_path, roots)
+ relative_source_path = safe_relative(Path(package_path), Path(source_root))
+ assets = collect_assets(package_path, spu)
+ author, _author_evidence = infer_author(relative_source_path, visual_author_names)
+ source_type, _source_type_evidence = infer_task_detail_source_type(relative_source_path)
+
+ return {
+ "fingerprint": fingerprint_for(source_root, package_path, spu),
+ "spu": spu,
+ "sourcePath": package_path,
+ "relativeSourcePath": relative_source_path,
+ "guessedStore": row["store"],
+ "productCategoryName": row["product_category"],
+ "authorName": author or "历史导入",
+ "taskDetailSourceType": source_type or "渲染",
+ "storeConfidence": 1,
+ "storeCandidates": [row["store"]] if row.get("store") else [],
+ "mainCount": len(assets["mainImages"]),
+ "skuCount": len(assets["skuImages"]),
+ "detailCount": len(assets["detailImages"]),
+ "sourceCount": len(assets["psdSources"]),
+ "status": "ready",
+ "skipReason": None,
+ **assets,
+ "seenInYya": False,
+ "ledger": {
+ "source": "needs-review-package-candidates",
+ "decision": row.get("decision"),
+ "cautionReason": row.get("caution_reason") or "",
+ },
+ }
+
+
+def safety_reason(row: dict[str, str], duplicate_paths: set[str]) -> str:
+ spu = normalize_spu(row["spu"])
+ package_path = row["package_path"]
+ prefix = spu_prefix(spu)
+ if prefix not in KNOWN_AUTO_PREFIXES:
+ return f"SPU 前缀不在自动导入白名单: {prefix or '(none)'}"
+
+ if os.path.normcase(os.path.normpath(package_path)) in duplicate_paths:
+ return "同一个 package_path 被识别为多个 SPU,需人工确认后再导入"
+
+ path_tokens = extract_spu_candidates(package_path, allow_unbranded=False)
+ incompatible_tokens = [
+ token for token in path_tokens if not is_compatible_spu_token(spu, token)
+ ]
+ if incompatible_tokens:
+ return "package_path 包含不兼容 SPU: " + ",".join(incompatible_tokens[:6])
+
+ return ""
+
+
+def main() -> int:
+ parser = argparse.ArgumentParser(description="Build import JSON from package candidates.")
+ parser.add_argument("--input-csv", required=True)
+ parser.add_argument("--output", required=True)
+ parser.add_argument("--decision", default="package_ready_review")
+ parser.add_argument("--limit", type=int, default=0)
+ parser.add_argument("--visual-authors-file", default="output/tianlan-spu-ledger/visual-authors.tsv")
+ parser.add_argument("--roots", nargs="*", default=DEFAULT_ROOTS)
+ args = parser.parse_args()
+
+ visual_author_names = (
+ load_visual_author_names(args.visual_authors_file)
+ if args.visual_authors_file and Path(args.visual_authors_file).exists()
+ else []
+ )
+
+ with Path(args.input_csv).open("r", encoding="utf-8-sig", newline="") as handle:
+ candidate_rows = [row for row in csv.DictReader(handle) if row.get("decision") == args.decision]
+
+ path_to_spus: dict[str, set[str]] = {}
+ for row in candidate_rows:
+ path_key = os.path.normcase(os.path.normpath(row["package_path"]))
+ path_to_spus.setdefault(path_key, set()).add(normalize_spu(row["spu"]))
+ duplicate_paths = {path_key for path_key, spus in path_to_spus.items() if len(spus) > 1}
+
+ skipped_rows: list[dict[str, str]] = []
+ rows: list[dict[str, str]] = []
+ for row in candidate_rows:
+ reason = safety_reason(row, duplicate_paths)
+ if reason:
+ skipped_rows.append({**row, "safety_skip_reason": reason})
+ else:
+ rows.append(row)
+
+ if args.limit > 0:
+ rows = rows[: args.limit]
+
+ records = [build_record(row, args.roots, visual_author_names) for row in rows]
+ output_path = Path(args.output)
+ output_path.parent.mkdir(parents=True, exist_ok=True)
+ output_path.write_text(json_dumps(records), encoding="utf-8")
+ skipped_path = output_path.with_suffix(".skipped-candidates.json")
+ skipped_path.write_text(json_dumps(skipped_rows), encoding="utf-8")
+
+ print(
+ json_dumps(
+ {
+ "inputCsv": args.input_csv,
+ "decision": args.decision,
+ "candidateRows": len(candidate_rows),
+ "safetySkipped": len(skipped_rows),
+ "records": len(records),
+ "output": str(output_path),
+ "skippedCandidates": str(skipped_path),
+ "totalMain": sum(record["mainCount"] for record in records),
+ "totalSku": sum(record["skuCount"] for record in records),
+ "totalDetail": sum(record["detailCount"] for record in records),
+ "totalSources": sum(record["sourceCount"] for record in records),
+ }
+ )
+ )
+ return 0
+
+
+if __name__ == "__main__":
+ raise SystemExit(main())
diff --git a/scripts/discover_needs_review_package_candidates.py b/scripts/discover_needs_review_package_candidates.py
new file mode 100644
index 0000000..140e6b1
--- /dev/null
+++ b/scripts/discover_needs_review_package_candidates.py
@@ -0,0 +1,340 @@
+#!/usr/bin/env python3
+"""
+Discover minimal package candidates inside needs_review SPU roots.
+
+This is a read-only second pass. It is stricter than grouping by SPU root: a
+candidate package is the smallest directory subtree that contains main, SKU and
+detail assets. That handles cross-level asset folders while avoiding merging
+multiple versions under the same SPU root.
+"""
+
+from __future__ import annotations
+
+import argparse
+import csv
+import json
+import os
+import re
+import sqlite3
+from collections import Counter, defaultdict
+from datetime import datetime
+from pathlib import Path
+from typing import Any
+
+from review_needs_review_spu_groups import choose_spu_root_path
+from scan_spu_ledger import (
+ EXCLUDED_DIR_NAMES,
+ RELEVANT_EXTENSIONS,
+ build_sku_code,
+ classify_file,
+ extract_file_spu_candidates,
+ has_hard_skip,
+ is_compatible_spu_token,
+ normalize_spu,
+ safe_relative,
+)
+
+
+CAUTION_RE = re.compile(
+ r"(^|[\\/])3D($|[\\/])|3D建模|3D原图|3D渲染原图|建模|建模图|建模原图|渲染原图|原图建模|"
+ r"源文件|psd源文件|PSD源文件|PS源文件|"
+ r"大图|AI合成|AI原图|图片有误|错误勿用|勿用|废稿|作废|弃用|不要",
+ re.IGNORECASE,
+)
+
+HEADERS = [
+ "decision",
+ "spu",
+ "store",
+ "product_category",
+ "package_path",
+ "relative_package_path",
+ "main_count",
+ "sku_count",
+ "detail_count",
+ "source_count",
+ "needs_review_rows_under_root",
+ "caution_reason",
+ "related_workspace_samples",
+ "main_samples",
+ "sku_samples",
+ "detail_samples",
+]
+
+
+def json_dumps(value: Any) -> str:
+ return json.dumps(value, ensure_ascii=False, indent=2)
+
+
+def latest_run_id(conn: sqlite3.Connection) -> str:
+ row = conn.execute(
+ "SELECT run_id FROM spu_ledger_runs ORDER BY started_at DESC LIMIT 1"
+ ).fetchone()
+ if not row:
+ raise SystemExit("台账数据库里没有扫描 run")
+ return str(row["run_id"])
+
+
+class Node:
+ def __init__(self, path: str, root: str) -> None:
+ self.path = path
+ self.root = root
+ self.children: list[str] = []
+ self.files: list[dict[str, Any]] = []
+ self.direct_counts: Counter[str] = Counter()
+ self.subtree_counts: Counter[str] = Counter()
+ self.samples: dict[str, list[dict[str, Any]]] = defaultdict(list)
+ self.has_caution = False
+ self.conflicts: list[dict[str, str]] = []
+
+
+def scan_tree(root_path: str, spu: str) -> dict[str, Node]:
+ root = Path(root_path)
+ nodes: dict[str, Node] = {}
+ if not root.exists():
+ return nodes
+
+ for current, dir_names, file_names in os.walk(root):
+ current_path = Path(current)
+ current_str = str(current_path)
+ node = nodes.setdefault(current_str, Node(current_str, root_path))
+ node.has_caution = node.has_caution or bool(CAUTION_RE.search(current_str))
+
+ kept_dirs: list[str] = []
+ for name in dir_names:
+ child = current_path / name
+ if name in EXCLUDED_DIR_NAMES or has_hard_skip(str(child)):
+ continue
+ kept_dirs.append(name)
+ child_str = str(child)
+ node.children.append(child_str)
+ nodes.setdefault(child_str, Node(child_str, root_path))
+ dir_names[:] = kept_dirs
+
+ for file_name in file_names:
+ file_path = current_path / file_name
+ extension = file_path.suffix.lower()
+ if extension not in RELEVANT_EXTENSIONS or has_hard_skip(str(file_path)):
+ continue
+ try:
+ file_size = file_path.stat().st_size
+ except OSError:
+ continue
+
+ relative_path = safe_relative(file_path, root)
+ group = classify_file(relative_path, file_name, extension)
+ record = {
+ "filePath": str(file_path),
+ "relativePath": relative_path,
+ "fileName": file_name,
+ "fileSize": int(file_size),
+ "group": group,
+ }
+ if group == "sku":
+ record["skuCode"] = build_sku_code(spu, file_name, node.direct_counts[group])
+ record["barcodes"] = []
+ node.files.append(record)
+ node.direct_counts[group] += 1
+ if group in {"main", "sku", "detail", "source"} and len(node.samples[group]) < 8:
+ node.samples[group].append(record)
+ if bool(CAUTION_RE.search(str(file_path))):
+ node.has_caution = True
+
+ if group in {"main", "detail"}:
+ tokens = extract_file_spu_candidates(file_name)
+ if tokens and not any(is_compatible_spu_token(spu, token) for token in tokens):
+ node.conflicts.append(
+ {
+ "group": group,
+ "file": relative_path,
+ "tokens": ",".join(tokens),
+ }
+ )
+
+ for path in sorted(nodes, key=lambda item: len(Path(item).parts), reverse=True):
+ node = nodes[path]
+ node.subtree_counts.update(node.direct_counts)
+ for group, records in node.samples.items():
+ node.samples[group] = records[:8]
+ for child_path in node.children:
+ child = nodes[child_path]
+ node.subtree_counts.update(child.subtree_counts)
+ node.has_caution = node.has_caution or child.has_caution
+ if len(node.conflicts) < 12:
+ node.conflicts.extend(child.conflicts[: max(0, 12 - len(node.conflicts))])
+ for group, records in child.samples.items():
+ existing = node.samples[group]
+ for record in records:
+ if len(existing) >= 8:
+ break
+ existing.append(record)
+ return nodes
+
+
+def has_all_three(node: Node) -> bool:
+ return (
+ node.subtree_counts["main"] > 0
+ and node.subtree_counts["sku"] > 0
+ and node.subtree_counts["detail"] > 0
+ )
+
+
+def minimal_package_nodes(nodes: dict[str, Node]) -> list[Node]:
+ candidates = [node for node in nodes.values() if has_all_three(node)]
+ candidate_paths = {node.path for node in candidates}
+ minimal: list[Node] = []
+ for node in candidates:
+ descendant_has_all_three = False
+ stack = list(node.children)
+ while stack:
+ child_path = stack.pop()
+ if child_path in candidate_paths:
+ descendant_has_all_three = True
+ break
+ child = nodes.get(child_path)
+ if child:
+ stack.extend(child.children)
+ if not descendant_has_all_three:
+ minimal.append(node)
+ minimal.sort(key=lambda item: item.path.lower())
+ return minimal
+
+
+def package_decision(node: Node) -> tuple[str, str]:
+ reasons: list[str] = []
+ if node.has_caution:
+ reasons.append("路径或文件名包含建模/源文件/大图/AI原图/错误等谨慎标识")
+ if node.conflicts:
+ tokens = sorted({conflict["tokens"] for conflict in node.conflicts})[:4]
+ reasons.append("主图/详情文件名出现不同SPU: " + ";".join(tokens))
+ if node.subtree_counts["main"] == 0 or node.subtree_counts["sku"] == 0 or node.subtree_counts["detail"] == 0:
+ reasons.append("缺少主图/SKU/详情之一")
+ if reasons:
+ return "package_caution_review", " | ".join(reasons)
+ return "package_ready_review", ""
+
+
+def row_for(node: Node, item_group: list[sqlite3.Row], spu: str, store: str, category: str) -> dict[str, Any]:
+ package = Path(node.path)
+ root = Path(node.root)
+ decision, caution_reason = package_decision(node)
+ related = [
+ str(item["workspace_path"])
+ for item in item_group
+ if str(item["workspace_path"]).lower().startswith(node.path.lower())
+ ][:8]
+ if not related:
+ related = [str(item["workspace_path"]) for item in item_group[:8]]
+
+ return {
+ "decision": decision,
+ "spu": spu,
+ "store": store,
+ "product_category": category,
+ "package_path": str(package),
+ "relative_package_path": safe_relative(package, root),
+ "main_count": node.subtree_counts["main"],
+ "sku_count": node.subtree_counts["sku"],
+ "detail_count": node.subtree_counts["detail"],
+ "source_count": node.subtree_counts["source"],
+ "needs_review_rows_under_root": len(item_group),
+ "caution_reason": caution_reason,
+ "related_workspace_samples": " || ".join(related),
+ "main_samples": " || ".join(record["relativePath"] for record in node.samples.get("main", [])),
+ "sku_samples": " || ".join(record["relativePath"] for record in node.samples.get("sku", [])),
+ "detail_samples": " || ".join(record["relativePath"] for record in node.samples.get("detail", [])),
+ }
+
+
+def write_csv(path: Path, rows: list[dict[str, Any]]) -> None:
+ path.parent.mkdir(parents=True, exist_ok=True)
+ with path.open("w", encoding="utf-8-sig", newline="") as handle:
+ writer = csv.DictWriter(handle, fieldnames=HEADERS)
+ writer.writeheader()
+ writer.writerows(rows)
+
+
+def main() -> int:
+ parser = argparse.ArgumentParser(description="Discover package candidates for needs_review rows.")
+ parser.add_argument("--db-path", default="output/tianlan-spu-ledger.db")
+ parser.add_argument("--run-id", default="")
+ parser.add_argument("--output-dir", default="")
+ args = parser.parse_args()
+
+ stamp = datetime.now().strftime("%Y%m%d-%H%M%S")
+ output_dir = Path(args.output_dir or f"output/tianlan-spu-ledger/needs-review-package-candidates-{stamp}")
+ output_dir.mkdir(parents=True, exist_ok=True)
+
+ conn = sqlite3.connect(str(Path(args.db_path)))
+ conn.row_factory = sqlite3.Row
+ run_id = args.run_id or latest_run_id(conn)
+ items = conn.execute(
+ """
+ SELECT *
+ FROM spu_ledger_items
+ WHERE run_id = ? AND status = 'needs_review'
+ ORDER BY source_root_name, relative_workspace_path, spu
+ """,
+ (run_id,),
+ ).fetchall()
+
+ groups: dict[tuple[str, str, str, str], list[sqlite3.Row]] = defaultdict(list)
+ for item in items:
+ spu = normalize_spu(item["spu"])
+ root_path = choose_spu_root_path(item)
+ key = (str(item["store"] or ""), spu, str(item["product_category"] or ""), root_path)
+ groups[key].append(item)
+
+ rows: list[dict[str, Any]] = []
+ detail_payload: list[dict[str, Any]] = []
+ roots_without_package = 0
+ for (store, spu, category, root_path), item_group in sorted(groups.items()):
+ nodes = scan_tree(root_path, spu)
+ packages = minimal_package_nodes(nodes)
+ if not packages:
+ roots_without_package += 1
+ continue
+ for node in packages:
+ row = row_for(node, item_group, spu, store, category)
+ rows.append(row)
+ detail_payload.append(
+ {
+ **row,
+ "samples": node.samples,
+ "conflicts": node.conflicts[:12],
+ "workspaces": [
+ {
+ "fingerprint": item["fingerprint"],
+ "workspacePath": item["workspace_path"],
+ "relativeWorkspacePath": item["relative_workspace_path"],
+ "versionLabel": item["version_label"],
+ "mainCount": item["main_count"],
+ "skuCount": item["sku_count"],
+ "detailCount": item["detail_count"],
+ "statusReason": item["status_reason"],
+ }
+ for item in item_group
+ ],
+ }
+ )
+
+ write_csv(output_dir / "package-candidates.csv", rows)
+ (output_dir / "package-candidates.json").write_text(json_dumps(detail_payload), encoding="utf-8")
+ summary = {
+ "runId": run_id,
+ "needsReviewItems": len(items),
+ "spuRootGroups": len(groups),
+ "rootsWithoutPackage": roots_without_package,
+ "packageCandidates": len(rows),
+ "decisionCounts": dict(Counter(row["decision"] for row in rows)),
+ "outputCsv": str(output_dir / "package-candidates.csv"),
+ "outputJson": str(output_dir / "package-candidates.json"),
+ }
+ (output_dir / "summary.json").write_text(json_dumps(summary), encoding="utf-8")
+ print(json_dumps(summary))
+ conn.close()
+ return 0
+
+
+if __name__ == "__main__":
+ raise SystemExit(main())
diff --git a/scripts/export-history-import-scan-summary.js b/scripts/export-history-import-scan-summary.js
new file mode 100644
index 0000000..2629766
--- /dev/null
+++ b/scripts/export-history-import-scan-summary.js
@@ -0,0 +1,76 @@
+const fs = require('fs')
+const path = require('path')
+
+const inputPath = process.argv[2] || path.join('output', 'tianlan-history-sku-import-scan-full.json')
+const outputDir = process.argv[3] || path.join('output')
+const baseName = process.argv[4] || 'tianlan-history-sku-import'
+
+function escapeCsvCell(value) {
+ const text = String(value ?? '')
+ if (/[",\r\n]/.test(text)) {
+ return `"${text.replace(/"/g, '""')}"`
+ }
+ return text
+}
+
+function writeCsv(filePath, rows) {
+ const headers = [
+ 'status(状态)',
+ 'spu(SPU)',
+ 'store(店铺)',
+ 'product_category(产品品类)',
+ 'author(作者)',
+ 'main_count(主图数量)',
+ 'sku_count(SKU图数量)',
+ 'detail_count(详情图数量)',
+ 'skip_reason(跳过或复核原因)',
+ 'source_path(来源完整路径)',
+ 'relative_source_path(来源相对路径)',
+ ]
+ const lines = [
+ headers.join(','),
+ ...rows.map((record) =>
+ [
+ record.status,
+ record.spu,
+ record.guessedStore,
+ record.productCategoryName,
+ record.authorName,
+ record.mainCount,
+ record.skuCount,
+ record.detailCount,
+ record.skipReason || '',
+ record.sourcePath,
+ record.relativeSourcePath,
+ ]
+ .map(escapeCsvCell)
+ .join(','),
+ ),
+ ]
+ fs.writeFileSync(filePath, `\uFEFF${lines.join('\r\n')}`, 'utf8')
+}
+
+function countBy(records, picker) {
+ return records.reduce((acc, record) => {
+ const key = picker(record) || ''
+ acc[key] = (acc[key] || 0) + 1
+ return acc
+ }, {})
+}
+
+const records = JSON.parse(fs.readFileSync(inputPath, 'utf8'))
+fs.mkdirSync(outputDir, { recursive: true })
+
+writeCsv(path.join(outputDir, `${baseName}-ready.csv`), records.filter((record) => record.status === 'ready'))
+writeCsv(path.join(outputDir, `${baseName}-needs-review.csv`), records.filter((record) => record.status === 'needs_review'))
+writeCsv(path.join(outputDir, `${baseName}-skipped.csv`), records.filter((record) => record.status === 'skipped'))
+
+const summary = {
+ total: records.length,
+ status: countBy(records, (record) => record.status),
+ store: countBy(records, (record) => record.guessedStore),
+ productCategory: countBy(records, (record) => record.productCategoryName),
+}
+
+fs.writeFileSync(path.join(outputDir, `${baseName}-summary.json`), JSON.stringify(summary, null, 2), 'utf8')
+console.log(JSON.stringify(summary, null, 2))
diff --git a/scripts/export_sku_folder_db.py b/scripts/export_sku_folder_db.py
index be53759..2df8c4d 100644
--- a/scripts/export_sku_folder_db.py
+++ b/scripts/export_sku_folder_db.py
@@ -10,6 +10,7 @@ EXPORT_HEADERS = [
("spu", "spu(SPU)"),
("directory_name", "directory_name(目录名称)"),
("relative_path", "relative_path(相对路径)"),
+ ("windows_path", "windows_path(Windows路径)"),
("sku_dir_path", "sku_dir_path(SKU目录完整路径)"),
("parent_path", "parent_path(上级目录)"),
("grandparent_path", "grandparent_path(上上级目录)"),
@@ -83,6 +84,7 @@ def main():
spu,
directory_name,
relative_path,
+ sku_dir_path AS windows_path,
sku_dir_path,
parent_path,
grandparent_path,
diff --git a/scripts/export_spu_ledger.py b/scripts/export_spu_ledger.py
new file mode 100644
index 0000000..4ff01d1
--- /dev/null
+++ b/scripts/export_spu_ledger.py
@@ -0,0 +1,259 @@
+#!/usr/bin/env python3
+"""
+Export the SPU ledger into review CSV files and ready import JSON.
+"""
+
+from __future__ import annotations
+
+import argparse
+import csv
+import json
+import sqlite3
+from pathlib import Path
+
+
+STATUS_EXPORTS = {
+ "ready": "ready",
+ "needs_review": "needs-review",
+ "skipped": "skipped",
+ "duplicate": "duplicate",
+ "delete_candidate": "delete-candidate",
+}
+
+
+def json_dumps(value) -> str:
+ return json.dumps(value, ensure_ascii=False, indent=2)
+
+
+def latest_run_id(conn: sqlite3.Connection) -> str:
+ row = conn.execute(
+ "SELECT run_id FROM spu_ledger_runs ORDER BY started_at DESC LIMIT 1"
+ ).fetchone()
+ if not row:
+ raise SystemExit("台账数据库里没有扫描 run")
+ return str(row["run_id"])
+
+
+def read_items(conn: sqlite3.Connection, run_id: str) -> list[sqlite3.Row]:
+ return conn.execute(
+ """
+ SELECT *
+ FROM spu_ledger_items
+ WHERE run_id = ?
+ ORDER BY source_root_name, relative_workspace_path, spu
+ """,
+ (run_id,),
+ ).fetchall()
+
+
+def read_files(conn: sqlite3.Connection, item_id: int, asset_group: str) -> list[sqlite3.Row]:
+ return conn.execute(
+ """
+ SELECT *
+ FROM spu_ledger_files
+ WHERE item_id = ? AND asset_group = ?
+ ORDER BY sort_order, relative_file_path
+ """,
+ (item_id, asset_group),
+ ).fetchall()
+
+
+def csv_headers() -> list[str]:
+ return [
+ "status(状态)",
+ "spu(SPU)",
+ "store(店铺)",
+ "product_category(产品品类)",
+ "author(作者)",
+ "task_detail_source_type(出图方式)",
+ "version_label(版本/工作区)",
+ "main_count(主图数量)",
+ "sku_count(SKU图数量)",
+ "detail_count(详情图数量)",
+ "source_count(PS源文件数量)",
+ "status_reason(判断原因)",
+ "spu_source(SPU来源)",
+ "store_evidence(店铺证据)",
+ "author_evidence(作者证据)",
+ "task_detail_source_type_evidence(出图方式证据)",
+ "source_batch(来源批次)",
+ "source_root(来源根目录)",
+ "workspace_path(工作区完整路径)",
+ "relative_workspace_path(工作区相对路径)",
+ "fingerprint(台账指纹)",
+ ]
+
+
+def item_to_csv_row(item: sqlite3.Row) -> dict[str, object]:
+ return {
+ "status(状态)": item["status"],
+ "spu(SPU)": item["spu"],
+ "store(店铺)": item["store"],
+ "product_category(产品品类)": item["product_category"],
+ "author(作者)": item["author"],
+ "task_detail_source_type(出图方式)": item["task_detail_source_type"],
+ "version_label(版本/工作区)": item["version_label"],
+ "main_count(主图数量)": item["main_count"],
+ "sku_count(SKU图数量)": item["sku_count"],
+ "detail_count(详情图数量)": item["detail_count"],
+ "source_count(PS源文件数量)": item["source_count"],
+ "status_reason(判断原因)": item["status_reason"],
+ "spu_source(SPU来源)": item["spu_source"],
+ "store_evidence(店铺证据)": item["store_evidence"],
+ "author_evidence(作者证据)": item["author_evidence"],
+ "task_detail_source_type_evidence(出图方式证据)": item["task_detail_source_type_evidence"],
+ "source_batch(来源批次)": item["source_batch"],
+ "source_root(来源根目录)": item["source_root"],
+ "workspace_path(工作区完整路径)": item["workspace_path"],
+ "relative_workspace_path(工作区相对路径)": item["relative_workspace_path"],
+ "fingerprint(台账指纹)": item["fingerprint"],
+ }
+
+
+def write_csv(path: Path, rows: list[dict[str, object]], headers: list[str]) -> None:
+ path.parent.mkdir(parents=True, exist_ok=True)
+ with path.open("w", encoding="utf-8-sig", newline="") as handle:
+ writer = csv.DictWriter(handle, fieldnames=headers)
+ writer.writeheader()
+ for row in rows:
+ writer.writerow(row)
+
+
+def to_image_input(file_row: sqlite3.Row) -> dict[str, object]:
+ payload = {
+ "localFilePath": file_row["file_path"],
+ "fileName": file_row["file_name"],
+ "fileSize": int(file_row["file_size"] or 0),
+ "sortOrder": int(file_row["sort_order"] or 0),
+ }
+ if file_row["asset_group"] == "sku":
+ payload["skuCode"] = file_row["sku_code"] or ""
+ payload["barcodes"] = []
+ return payload
+
+
+def to_source_input(file_row: sqlite3.Row) -> dict[str, object]:
+ return {
+ "localFilePath": file_row["file_path"],
+ "fileName": file_row["file_name"],
+ "fileSize": int(file_row["file_size"] or 0),
+ }
+
+
+def build_import_record(conn: sqlite3.Connection, item: sqlite3.Row) -> dict[str, object]:
+ main = [to_image_input(row) for row in read_files(conn, item["id"], "main")]
+ sku = [to_image_input(row) for row in read_files(conn, item["id"], "sku")]
+ detail = [to_image_input(row) for row in read_files(conn, item["id"], "detail")]
+ sources = [to_source_input(row) for row in read_files(conn, item["id"], "source")]
+
+ return {
+ "fingerprint": item["fingerprint"],
+ "spu": item["spu"],
+ "sourcePath": item["workspace_path"],
+ "relativeSourcePath": item["relative_workspace_path"],
+ "guessedStore": item["store"],
+ "productCategoryName": item["product_category"],
+ "authorName": item["author"] or "历史导入",
+ "taskDetailSourceType": item["task_detail_source_type"] or "渲染",
+ "storeConfidence": 1,
+ "storeCandidates": [item["store"]] if item["store"] else [],
+ "mainCount": item["main_count"],
+ "skuCount": item["sku_count"],
+ "detailCount": item["detail_count"],
+ "sourceCount": item["source_count"],
+ "status": item["status"],
+ "skipReason": None if item["status"] == "ready" else item["status_reason"],
+ "mainImages": main,
+ "skuImages": sku,
+ "detailImages": detail,
+ "psdSources": sources,
+ "seenInYya": False,
+ "ledger": {
+ "runId": item["run_id"],
+ "sourceBatch": item["source_batch"],
+ "spuSource": item["spu_source"],
+ "spuEvidence": json.loads(item["spu_evidence_json"] or "[]"),
+ "pathFlags": json.loads(item["path_flags_json"] or "{}"),
+ },
+ }
+
+
+def write_batch_csvs(output_dir: Path, prefix: str, items: list[sqlite3.Row], batch_size: int) -> None:
+ if batch_size <= 0:
+ return
+ headers = csv_headers()
+ pending = [item for item in items if item["status"] in ("ready", "needs_review")]
+ for start in range(0, len(pending), batch_size):
+ batch_no = start // batch_size + 1
+ batch = pending[start : start + batch_size]
+ rows = [item_to_csv_row(item) for item in batch]
+ write_csv(output_dir / f"{prefix}-batch-{batch_no:03d}.csv", rows, headers)
+
+
+def parse_args() -> argparse.Namespace:
+ parser = argparse.ArgumentParser(description="Export SPU ledger review files.")
+ parser.add_argument("--db-path", required=True)
+ parser.add_argument("--output-dir", default=str(Path.cwd() / "output"))
+ parser.add_argument("--prefix", default="spu-ledger")
+ parser.add_argument("--run-id", default="")
+ parser.add_argument("--batch-size", type=int, default=20)
+ return parser.parse_args()
+
+
+def main() -> int:
+ args = parse_args()
+ db_path = Path(args.db_path)
+ output_dir = Path(args.output_dir)
+ output_dir.mkdir(parents=True, exist_ok=True)
+
+ conn = sqlite3.connect(str(db_path))
+ conn.row_factory = sqlite3.Row
+ run_id = args.run_id or latest_run_id(conn)
+ items = read_items(conn, run_id)
+
+ headers = csv_headers()
+ status_counts: dict[str, int] = {}
+ store_counts: dict[str, int] = {}
+ category_counts: dict[str, int] = {}
+
+ for item in items:
+ status_counts[item["status"]] = status_counts.get(item["status"], 0) + 1
+ store_counts[item["store"]] = store_counts.get(item["store"], 0) + 1
+ category = item["product_category"] or "未识别"
+ category_counts[category] = category_counts.get(category, 0) + 1
+
+ for status, suffix in STATUS_EXPORTS.items():
+ rows = [item_to_csv_row(item) for item in items if item["status"] == status]
+ write_csv(output_dir / f"{args.prefix}-{suffix}.csv", rows, headers)
+
+ write_csv(output_dir / f"{args.prefix}-all.csv", [item_to_csv_row(item) for item in items], headers)
+ write_batch_csvs(output_dir, args.prefix, items, args.batch_size)
+
+ ready_records = [build_import_record(conn, item) for item in items if item["status"] == "ready"]
+ ready_json_path = output_dir / f"{args.prefix}-ready-import-records.json"
+ ready_json_path.write_text(json_dumps(ready_records), encoding="utf-8")
+
+ summary = {
+ "runId": run_id,
+ "dbPath": str(db_path),
+ "total": len(items),
+ "status": status_counts,
+ "store": store_counts,
+ "productCategory": category_counts,
+ "readyImportRecords": str(ready_json_path),
+ }
+ summary_path = output_dir / f"{args.prefix}-summary.json"
+ summary_path.write_text(json_dumps(summary), encoding="utf-8")
+
+ conn.close()
+
+ print("SPU 台账导出完成")
+ print(f"- run_id: {run_id}")
+ print(f"- total: {len(items)}")
+ print(f"- ready_import_records: {ready_json_path}")
+ print(f"- summary: {summary_path}")
+ return 0
+
+
+if __name__ == "__main__":
+ raise SystemExit(main())
diff --git a/scripts/filter_needs_review_dry_run_records.py b/scripts/filter_needs_review_dry_run_records.py
new file mode 100644
index 0000000..aab0b88
--- /dev/null
+++ b/scripts/filter_needs_review_dry_run_records.py
@@ -0,0 +1,148 @@
+#!/usr/bin/env python3
+"""
+Filter dry-run import records from needs_review package candidates.
+
+This is a conservative, read-only guard used after import-history-materials
+dry-run. It keeps only records that still look like a complete package root
+instead of an asset subfolder or ambiguous multi-SPU package.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import re
+from pathlib import Path
+from typing import Any
+
+
+MAIN_DIR_RE = re.compile(r"主图|main|^(750|800|1440|1920)$", re.IGNORECASE)
+SKU_DIR_RE = re.compile(r"sku|白底|场景", re.IGNORECASE)
+DETAIL_DIR_RE = re.compile(r"详情|detail", re.IGNORECASE)
+FORBIDDEN_RE = re.compile(r"错误|勿用|废稿|图片有误|3D建模|3D原图|建模图|建模原图|渲染原图|AI原图|大图", re.IGNORECASE)
+ASSET_SEGMENT_RE = re.compile(r"^(主图|sku|SKU|详情|详情页|白底|场景|750|800|790|1200|1440|1920|切图|800-800主图)$")
+BRANDED_SPU_RE = re.compile(r"\b(GBPM|GBSFY|GBGJ|GBJP)-[A-Z]{1,8}\d{3,5}\b", re.IGNORECASE)
+
+
+def load_json(path: Path) -> Any:
+ return json.loads(path.read_text(encoding="utf-8-sig"))
+
+
+def dump_json(path: Path, value: Any) -> None:
+ path.parent.mkdir(parents=True, exist_ok=True)
+ path.write_text(json.dumps(value, ensure_ascii=False, indent=2), encoding="utf-8")
+
+
+def norm_spu(value: str) -> str:
+ return value.replace("-", "-").replace("—", "-").replace("–", "-").upper().strip()
+
+
+def path_segments(path: str) -> list[str]:
+ normalized = path.replace("/", "\\")
+ return [part for part in normalized.split("\\") if part]
+
+
+def child_dirs(path: str) -> list[str]:
+ root = Path(path)
+ if not root.exists():
+ return []
+ return sorted([item.name for item in root.iterdir() if item.is_dir()], key=str.lower)
+
+
+def has_required_child_dirs(names: list[str]) -> bool:
+ has_main = any(MAIN_DIR_RE.search(name) for name in names)
+ has_sku = any(SKU_DIR_RE.search(name) for name in names)
+ has_detail = any(DETAIL_DIR_RE.search(name) for name in names)
+ return has_main and has_sku and has_detail
+
+
+def first_branded_token(path: str) -> str:
+ match = BRANDED_SPU_RE.search(norm_spu(path))
+ return norm_spu(match.group(0)) if match else ""
+
+
+def skip_reason(record: dict[str, Any]) -> str:
+ source_path = str(record.get("sourcePath") or "")
+ spu = norm_spu(str(record.get("spu") or ""))
+ segments = path_segments(source_path)
+ dirs = child_dirs(source_path)
+
+ if int(record.get("mainCount") or 0) <= 0:
+ return "mainCount 为 0,疑似把主图子目录误判为包根"
+ if int(record.get("skuCount") or 0) <= 0:
+ return "skuCount 为 0,不满足三类资产齐全"
+ if int(record.get("detailCount") or 0) <= 0:
+ return "detailCount 为 0,不满足三类资产齐全"
+ if FORBIDDEN_RE.search(source_path):
+ return "路径包含禁用/高风险目录词"
+ if "组合" in source_path and "+" not in spu:
+ return "组合目录未表达为组合 SPU,需人工确认"
+ if len(segments) >= 2 and ASSET_SEGMENT_RE.match(segments[-1]) and ASSET_SEGMENT_RE.match(segments[-2]):
+ return "候选路径落在资产子目录内,不是包根目录"
+ if len(segments) >= 1 and ASSET_SEGMENT_RE.match(segments[-1]) and not has_required_child_dirs(dirs):
+ return "候选路径末级像资产目录且首层缺少主图/SKU/详情"
+ if not has_required_child_dirs(dirs):
+ return "首层目录缺少明确的主图/SKU/详情三类证据"
+
+ token = first_branded_token(source_path)
+ if token and token != spu:
+ return f"路径首个品牌 SPU 为 {token},记录 SPU 为 {spu},需人工确认"
+
+ return ""
+
+
+def main() -> int:
+ parser = argparse.ArgumentParser(description="Filter dry-run import records conservatively.")
+ parser.add_argument("--records", required=True, help="Input HistoryImportRecord JSON")
+ parser.add_argument("--result", required=True, help="Dry-run result JSON")
+ parser.add_argument("--output", required=True, help="Filtered records JSON")
+ parser.add_argument("--skipped-output", required=True, help="Skipped records JSON")
+ args = parser.parse_args()
+
+ records = load_json(Path(args.records))
+ results = load_json(Path(args.result))
+ dry_fingerprints = {item.get("fingerprint") for item in results if item.get("status") == "dry_run"}
+
+ selected: list[dict[str, Any]] = []
+ skipped: list[dict[str, Any]] = []
+ for record in records:
+ if record.get("fingerprint") not in dry_fingerprints:
+ continue
+ reason = skip_reason(record)
+ if reason:
+ skipped.append(
+ {
+ "fingerprint": record.get("fingerprint"),
+ "spu": record.get("spu"),
+ "sourcePath": record.get("sourcePath"),
+ "mainCount": record.get("mainCount"),
+ "skuCount": record.get("skuCount"),
+ "detailCount": record.get("detailCount"),
+ "sourceCount": record.get("sourceCount"),
+ "skipReason": reason,
+ }
+ )
+ else:
+ selected.append(record)
+
+ dump_json(Path(args.output), selected)
+ dump_json(Path(args.skipped_output), skipped)
+
+ print(
+ json.dumps(
+ {
+ "dryRunFingerprints": len(dry_fingerprints),
+ "selected": len(selected),
+ "skipped": len(skipped),
+ "output": args.output,
+ "skippedOutput": args.skipped_output,
+ },
+ ensure_ascii=False,
+ indent=2,
+ )
+ )
+ return 0
+
+
+if __name__ == "__main__":
+ raise SystemExit(main())
diff --git a/scripts/hard-delete-history-materials.ts b/scripts/hard-delete-history-materials.ts
new file mode 100644
index 0000000..0925fc9
--- /dev/null
+++ b/scripts/hard-delete-history-materials.ts
@@ -0,0 +1,214 @@
+import path from 'path'
+import { promises as fs } from 'fs'
+import { prisma } from '@/lib/prisma'
+
+type DeleteCandidate = {
+ materialId: string
+ fingerprint?: string
+ sourcePath?: string
+ reason: string
+}
+
+function parseArgs() {
+ const args = process.argv.slice(2)
+ const getArg = (name: string) => {
+ const match = args.find((arg) => arg.startsWith(`--${name}=`))
+ return match ? match.slice(name.length + 3) : ''
+ }
+
+ const candidates = getArg('candidates')
+ const output = getArg('output')
+ const confirm = getArg('confirm')
+ const apply = args.includes('--apply')
+
+ if (!candidates) {
+ throw new Error('缺少 --candidates=删除候选清单 JSON')
+ }
+
+ return {
+ candidates,
+ output: output || path.join(path.dirname(candidates), `history-hard-delete-backup-${Date.now()}.json`),
+ apply,
+ confirm,
+ }
+}
+
+function normalizeCandidate(raw: any): DeleteCandidate {
+ return {
+ materialId: String(raw?.materialId || raw?.id || '').trim(),
+ fingerprint: String(raw?.fingerprint || '').trim() || undefined,
+ sourcePath: String(raw?.sourcePath || raw?.source_path || '').trim() || undefined,
+ reason: String(raw?.reason || raw?.deleteReason || '').trim(),
+ }
+}
+
+function assertCandidate(candidate: DeleteCandidate) {
+ if (!candidate.materialId) {
+ throw new Error('删除候选缺少 materialId')
+ }
+ if (!candidate.reason) {
+ throw new Error(`删除候选 ${candidate.materialId} 缺少 reason`)
+ }
+ if (!candidate.fingerprint && !candidate.sourcePath) {
+ throw new Error(`删除候选 ${candidate.materialId} 必须提供 fingerprint 或 sourcePath`)
+ }
+}
+
+function stringifyJson(value: unknown) {
+ return JSON.stringify(
+ value,
+ (_key, item) => (typeof item === 'bigint' ? item.toString() : item),
+ 2,
+ )
+}
+
+async function loadCandidates(candidatePath: string) {
+ const raw = await fs.readFile(candidatePath, 'utf8')
+ const parsed = JSON.parse(raw)
+ const list = Array.isArray(parsed) ? parsed : parsed.candidates
+ if (!Array.isArray(list)) {
+ throw new Error('删除候选清单必须是数组,或包含 candidates 数组')
+ }
+ const candidates = list.map(normalizeCandidate)
+ candidates.forEach(assertCandidate)
+ return candidates
+}
+
+async function loadMaterialSnapshot(materialId: string) {
+ return prisma.material.findUnique({
+ where: { id: materialId },
+ include: {
+ assets: {
+ include: {
+ barcodes: true,
+ },
+ },
+ skus: true,
+ logs: true,
+ likes: true,
+ favorites: true,
+ downloadLogs: true,
+ transfers: true,
+ },
+ })
+}
+
+function hasHistoryEvidence(material: Awaited>, candidate: DeleteCandidate) {
+ if (!material) {
+ return false
+ }
+
+ return material.logs.some((log) => {
+ if (log.action !== 'HISTORY_IMPORT') {
+ return false
+ }
+ const changes = String(log.changes || '')
+ return Boolean(
+ (candidate.fingerprint && changes.includes(candidate.fingerprint)) ||
+ (candidate.sourcePath && changes.includes(candidate.sourcePath)),
+ )
+ })
+}
+
+async function main() {
+ const options = parseArgs()
+ const candidates = await loadCandidates(options.candidates)
+ const backups: any[] = []
+ const results: any[] = []
+
+ console.log('历史错误素材硬删除预检开始')
+ console.log(`- 候选清单: ${options.candidates}`)
+ console.log(`- 模式: ${options.apply ? '正式硬删除' : '仅备份/演练'}`)
+ console.log(`- 备份输出: ${options.output}`)
+
+ if (options.apply && options.confirm !== 'HARD_DELETE_HISTORY_MATERIALS') {
+ throw new Error('正式硬删除必须追加 --confirm=HARD_DELETE_HISTORY_MATERIALS')
+ }
+
+ for (const candidate of candidates) {
+ const material = await loadMaterialSnapshot(candidate.materialId)
+ if (!material) {
+ results.push({
+ materialId: candidate.materialId,
+ status: 'skipped',
+ reason: '素材不存在',
+ })
+ continue
+ }
+
+ const evidenceMatched = hasHistoryEvidence(material, candidate)
+ backups.push({
+ candidate,
+ material,
+ backedUpAt: new Date().toISOString(),
+ evidenceMatched,
+ })
+
+ if (!evidenceMatched) {
+ results.push({
+ materialId: candidate.materialId,
+ productModel: material.productModel,
+ status: 'blocked',
+ reason: '未在 HISTORY_IMPORT 日志中匹配 fingerprint/sourcePath,拒绝删除',
+ })
+ continue
+ }
+
+ if (!options.apply) {
+ results.push({
+ materialId: candidate.materialId,
+ productModel: material.productModel,
+ status: 'dry_run',
+ reason: candidate.reason,
+ })
+ continue
+ }
+
+ await prisma.material.delete({
+ where: { id: candidate.materialId },
+ })
+
+ results.push({
+ materialId: candidate.materialId,
+ productModel: material.productModel,
+ status: 'deleted',
+ reason: candidate.reason,
+ })
+ }
+
+ await fs.mkdir(path.dirname(options.output), { recursive: true })
+ await fs.writeFile(
+ options.output,
+ stringifyJson({
+ sourceCandidates: options.candidates,
+ apply: options.apply,
+ generatedAt: new Date().toISOString(),
+ results,
+ backups,
+ }),
+ 'utf8',
+ )
+
+ const summary = results.reduce(
+ (acc, item) => {
+ acc.total += 1
+ acc[item.status] = (acc[item.status] || 0) + 1
+ return acc
+ },
+ { total: 0 } as Record,
+ )
+
+ console.log('历史错误素材硬删除预检完成')
+ console.log(summary)
+ console.log(`- 备份/结果: ${options.output}`)
+}
+
+main()
+ .catch((error) => {
+ console.error('历史错误素材硬删除失败:', error)
+ process.exitCode = 1
+ })
+ .finally(async () => {
+ await prisma.$disconnect()
+ process.exit(process.exitCode ?? 0)
+ })
diff --git a/scripts/import-history-materials.ts b/scripts/import-history-materials.ts
index 12ca424..e7b1c78 100644
--- a/scripts/import-history-materials.ts
+++ b/scripts/import-history-materials.ts
@@ -1,11 +1,106 @@
import path from 'path'
import { promises as fs } from 'fs'
+import { HeadBucketCommand, S3Client } from '@aws-sdk/client-s3'
import {
importHistoryMaterials,
resolveImportOperator,
type HistoryImportRecord,
} from '@/lib/history-material-import'
import { prisma } from '@/lib/prisma'
+import {
+ assertRemoteUploadSyncAccess,
+ getRemoteUploadSyncConfig,
+} from '@/lib/remote-upload-sync'
+
+function getDatabaseHost() {
+ const rawUrl = String(process.env.DATABASE_URL || '').trim()
+ if (!rawUrl) {
+ return ''
+ }
+
+ try {
+ return new URL(rawUrl).hostname
+ } catch {
+ return ''
+ }
+}
+
+function assertApplyEnvironment() {
+ const databaseHost = getDatabaseHost()
+ const s3Bucket = String(process.env.S3_BUCKET || '').trim()
+ const s3Endpoint = String(process.env.S3_ENDPOINT || '').trim().replace(/\/+$/, '')
+ const s3PublicDomain = String(process.env.S3_PUBLIC_DOMAIN || '').trim()
+ const s3FilePath = String(process.env.S3_FILE_PATH || '').trim().replace(/\\/g, '/')
+ const uploadDir = String(process.env.UPLOAD_DIR || '').trim()
+ const remoteUploadSyncConfig = getRemoteUploadSyncConfig()
+
+ const isProductionDatabase = databaseHost === '192.168.58.20'
+ const isExpectedEndpoint = s3Endpoint === 'http://192.168.58.20:9000'
+ const isExpectedPreviewBucket =
+ s3Bucket === 'gongxiangpan' &&
+ /^https:\/\/s3\.jiansun\.vip\/gongxiangpan\/?$/i.test(s3PublicDomain)
+ const isExpectedFilePath = s3FilePath === '由壹点AI/'
+
+ if (!isExpectedEndpoint || !isExpectedPreviewBucket || !isExpectedFilePath) {
+ throw new Error(
+ [
+ '正式导入环境不正确:预览文件必须写入新桶 gongxiangpan 的正确前缀。',
+ `当前 S3_ENDPOINT=${s3Endpoint || '(empty)'}`,
+ `当前 S3_BUCKET=${s3Bucket || '(empty)'}`,
+ `当前 S3_PUBLIC_DOMAIN=${s3PublicDomain || '(empty)'}`,
+ `当前 S3_FILE_PATH=${s3FilePath || '(empty)'}`,
+ ].join('\n'),
+ )
+ }
+
+ if (isProductionDatabase && process.platform === 'win32' && !remoteUploadSyncConfig) {
+ throw new Error(
+ [
+ '正式导入环境不正确:当前是 Windows 工作站,但 DATABASE_URL 指向生产库。',
+ '直接导入会把下载 zip / PSD 源文件写到本机 uploads,生产站点无法下载。',
+ '如确需在 Windows 读取 Z 盘源文件并写生产库,必须配置远程 uploads 同步:',
+ 'HISTORY_IMPORT_REMOTE_UPLOADS_HOST=prod',
+ 'HISTORY_IMPORT_REMOTE_UPLOADS_DIR=/mnt/PoolOne/apps/youyidian-3d/uploads',
+ `当前 UPLOAD_DIR=${uploadDir || '(empty)'}`,
+ ].join('\n'),
+ )
+ }
+}
+
+async function assertApplyStorageAccess() {
+ const s3Endpoint = String(process.env.S3_ENDPOINT || '').trim()
+ const s3Bucket = String(process.env.S3_BUCKET || '').trim()
+ const accessKeyId = String(process.env.S3_ACCESS_KEY_ID || '').trim()
+ const secretAccessKey = String(process.env.S3_SECRET_ACCESS_KEY || '').trim()
+
+ if (!accessKeyId || !secretAccessKey) {
+ throw new Error('正式导入环境不正确:缺少 S3 访问凭据,不能验证新桶写入权限。')
+ }
+
+ const client = new S3Client({
+ endpoint: s3Endpoint,
+ region: process.env.S3_REGION || 'us-east-1',
+ credentials: {
+ accessKeyId,
+ secretAccessKey,
+ },
+ forcePathStyle: String(process.env.S3_ENABLE_PATH_STYLE || '').trim() !== '0',
+ })
+
+ try {
+ await client.send(new HeadBucketCommand({ Bucket: s3Bucket }))
+ } catch (error) {
+ const statusCode = (error as { $metadata?: { httpStatusCode?: number } }).$metadata?.httpStatusCode
+ throw new Error(
+ [
+ '正式导入环境不正确:当前 S3 凭据无法访问新桶,已阻止写入数据库。',
+ `S3_BUCKET=${s3Bucket || '(empty)'}`,
+ `S3_ENDPOINT=${s3Endpoint || '(empty)'}`,
+ `HTTP_STATUS=${statusCode || 'unknown'}`,
+ ].join('\n'),
+ )
+ }
+}
function parseArgs() {
const args = process.argv.slice(2)
@@ -36,6 +131,12 @@ function parseArgs() {
async function main() {
const options = parseArgs()
+ if (options.apply) {
+ assertApplyEnvironment()
+ await assertApplyStorageAccess()
+ await assertRemoteUploadSyncAccess()
+ }
+
const raw = await fs.readFile(options.input, 'utf8')
const records = JSON.parse(raw) as HistoryImportRecord[]
const operator = await resolveImportOperator(options.operator)
diff --git a/scripts/migrate-history-preview-assets.js b/scripts/migrate-history-preview-assets.js
new file mode 100644
index 0000000..8c0c125
--- /dev/null
+++ b/scripts/migrate-history-preview-assets.js
@@ -0,0 +1,337 @@
+#!/usr/bin/env node
+
+const { PrismaClient } = require('@prisma/client')
+const { S3Client, HeadObjectCommand, PutObjectCommand } = require('@aws-sdk/client-s3')
+
+const prisma = new PrismaClient()
+
+const DEFAULT_OLD_PREFIX = 'https://s3.1080p.hk/jsxiangche/'
+
+function getArg(name, fallback = '') {
+ const prefix = `--${name}=`
+ const match = process.argv.slice(2).find((arg) => arg.startsWith(prefix))
+ return match ? match.slice(prefix.length) : fallback
+}
+
+function hasFlag(name) {
+ return process.argv.includes(`--${name}`)
+}
+
+function normalizePrefix(prefix) {
+ return String(prefix || '').trim().replace(/\/+$/, '') + '/'
+}
+
+function normalizePath(input) {
+ return String(input || '')
+ .trim()
+ .replace(/\\/g, '/')
+ .replace(/^\/+/, '')
+ .replace(/\/{2,}/g, '/')
+}
+
+function parseOldObjectKey(oldUrl, oldBucket) {
+ const parsed = new URL(oldUrl)
+ let key = decodeURIComponent(parsed.pathname).replace(/^\/+/, '')
+ if (oldBucket && key.startsWith(`${oldBucket}/`)) {
+ key = key.slice(oldBucket.length + 1)
+ }
+ return normalizePath(key)
+}
+
+function ensureDestinationKey(oldKey) {
+ const s3FilePath = normalizePath(process.env.S3_FILE_PATH || '')
+ if (!s3FilePath) {
+ return oldKey
+ }
+
+ if (oldKey.startsWith(`${s3FilePath}/`)) {
+ return oldKey
+ }
+
+ if (oldKey.startsWith('public-display/')) {
+ return `${s3FilePath}/${oldKey}`
+ }
+
+ return oldKey
+}
+
+function buildPublicUrl(key) {
+ const publicDomain = String(process.env.S3_PUBLIC_DOMAIN || '').trim().replace(/\/+$/, '')
+ if (!publicDomain) {
+ throw new Error('Missing S3_PUBLIC_DOMAIN')
+ }
+ return `${publicDomain}/${key}`
+}
+
+function guessContentType(url) {
+ const lower = url.toLowerCase()
+ if (lower.endsWith('.webp')) return 'image/webp'
+ if (lower.endsWith('.png')) return 'image/png'
+ if (lower.endsWith('.jpg') || lower.endsWith('.jpeg')) return 'image/jpeg'
+ return 'application/octet-stream'
+}
+
+function createS3Client() {
+ const endpoint = String(process.env.S3_ENDPOINT || '').split('#')[0].trim()
+ if (!endpoint) throw new Error('Missing S3_ENDPOINT')
+ if (!process.env.S3_BUCKET) throw new Error('Missing S3_BUCKET')
+ if (!process.env.S3_ACCESS_KEY_ID) throw new Error('Missing S3_ACCESS_KEY_ID')
+ if (!process.env.S3_SECRET_ACCESS_KEY) throw new Error('Missing S3_SECRET_ACCESS_KEY')
+
+ return new S3Client({
+ endpoint,
+ region: process.env.S3_REGION || 'us-east-1',
+ credentials: {
+ accessKeyId: process.env.S3_ACCESS_KEY_ID,
+ secretAccessKey: process.env.S3_SECRET_ACCESS_KEY,
+ },
+ forcePathStyle: process.env.S3_ENABLE_PATH_STYLE !== '0',
+ maxAttempts: 5,
+ })
+}
+
+async function destinationExists(client, bucket, key) {
+ try {
+ await client.send(new HeadObjectCommand({ Bucket: bucket, Key: key }))
+ return true
+ } catch (error) {
+ const statusCode = error && error.$metadata && error.$metadata.httpStatusCode
+ if (statusCode === 404 || error.name === 'NotFound') {
+ return false
+ }
+ throw error
+ }
+}
+
+async function fetchOldObject(oldUrl) {
+ const response = await fetch(oldUrl)
+ if (!response.ok) {
+ throw new Error(`Fetch failed ${response.status}`)
+ }
+
+ return {
+ body: Buffer.from(await response.arrayBuffer()),
+ contentType: response.headers.get('content-type') || guessContentType(oldUrl),
+ }
+}
+
+async function getOldPreviewUrls(oldPrefix, limit) {
+ const limitedSql = limit > 0 ? `LIMIT ${Number(limit)}` : ''
+ return prisma.$queryRawUnsafe(`
+ WITH history_materials AS (
+ SELECT DISTINCT "materialId" AS id
+ FROM material_logs
+ WHERE action = 'HISTORY_IMPORT'
+ ), active_history AS (
+ SELECT m.id, m."previewUrl"
+ FROM materials m
+ JOIN history_materials h ON h.id = m.id
+ WHERE m."deletedAt" IS NULL
+ )
+ SELECT DISTINCT old_url
+ FROM (
+ SELECT "previewUrl" AS old_url
+ FROM active_history
+ WHERE "previewUrl" LIKE $1
+ UNION
+ SELECT ma.file_url AS old_url
+ FROM material_assets ma
+ JOIN active_history h ON h.id = ma.material_id
+ WHERE ma.asset_type IN ('main-preview', 'sku-preview', 'detail-preview')
+ AND ma.file_url LIKE $1
+ ) urls
+ WHERE old_url IS NOT NULL AND old_url <> ''
+ ORDER BY old_url
+ ${limitedSql}
+ `, `${oldPrefix}%`)
+}
+
+async function updateDatabaseUrl(oldUrl, newUrl) {
+ const materialUpdates = await prisma.$executeRawUnsafe(`
+ WITH history_materials AS (
+ SELECT DISTINCT "materialId" AS id
+ FROM material_logs
+ WHERE action = 'HISTORY_IMPORT'
+ )
+ UPDATE materials m
+ SET "previewUrl" = $1,
+ "updatedAt" = NOW()
+ FROM history_materials h
+ WHERE h.id = m.id
+ AND m."deletedAt" IS NULL
+ AND m."previewUrl" = $2
+ `, newUrl, oldUrl)
+
+ const assetUpdates = await prisma.$executeRawUnsafe(`
+ WITH history_materials AS (
+ SELECT DISTINCT "materialId" AS id
+ FROM material_logs
+ WHERE action = 'HISTORY_IMPORT'
+ )
+ UPDATE material_assets ma
+ SET file_url = $1,
+ updated_at = NOW()
+ FROM history_materials h
+ JOIN materials m ON m.id = h.id
+ WHERE ma.material_id = m.id
+ AND m."deletedAt" IS NULL
+ AND ma.asset_type IN ('main-preview', 'sku-preview', 'detail-preview')
+ AND ma.file_url = $2
+ `, newUrl, oldUrl)
+
+ return { materialUpdates, assetUpdates }
+}
+
+async function bulkUpdateDatabaseUrls(oldPrefix, newPrefix) {
+ const materialUpdates = await prisma.$executeRawUnsafe(`
+ WITH history_materials AS (
+ SELECT DISTINCT "materialId" AS id
+ FROM material_logs
+ WHERE action = 'HISTORY_IMPORT'
+ )
+ UPDATE materials m
+ SET "previewUrl" = REPLACE(m."previewUrl", $1, $2),
+ "updatedAt" = NOW()
+ FROM history_materials h
+ WHERE h.id = m.id
+ AND m."deletedAt" IS NULL
+ AND m."previewUrl" LIKE $3
+ `, oldPrefix, newPrefix, `${oldPrefix}%`)
+
+ const assetUpdates = await prisma.$executeRawUnsafe(`
+ WITH history_materials AS (
+ SELECT DISTINCT "materialId" AS id
+ FROM material_logs
+ WHERE action = 'HISTORY_IMPORT'
+ )
+ UPDATE material_assets ma
+ SET file_url = REPLACE(ma.file_url, $1, $2),
+ updated_at = NOW()
+ FROM history_materials h
+ JOIN materials m ON m.id = h.id
+ WHERE ma.material_id = m.id
+ AND m."deletedAt" IS NULL
+ AND ma.asset_type IN ('main-preview', 'sku-preview', 'detail-preview')
+ AND ma.file_url LIKE $3
+ `, oldPrefix, newPrefix, `${oldPrefix}%`)
+
+ return { materialUpdates, assetUpdates }
+}
+
+async function runPool(items, concurrency, worker) {
+ let index = 0
+ const workers = Array.from({ length: concurrency }, async () => {
+ while (index < items.length) {
+ const currentIndex = index
+ index += 1
+ await worker(items[currentIndex], currentIndex)
+ }
+ })
+ await Promise.all(workers)
+}
+
+async function main() {
+ const apply = hasFlag('apply')
+ const bulkDbUpdate = hasFlag('bulk-db-update')
+ const skipDbUpdate = hasFlag('skip-db-update')
+ const oldPrefix = normalizePrefix(getArg('old-prefix', DEFAULT_OLD_PREFIX))
+ const oldBucket = getArg('old-bucket', 'jsxiangche')
+ const limit = Number(getArg('limit', '0')) || 0
+ const concurrency = Math.max(1, Number(getArg('concurrency', '8')) || 8)
+ const oldRows = await getOldPreviewUrls(oldPrefix, limit)
+ const mappings = oldRows.map((row) => {
+ const oldUrl = row.old_url
+ const key = ensureDestinationKey(parseOldObjectKey(oldUrl, oldBucket))
+ return {
+ oldUrl,
+ key,
+ newUrl: buildPublicUrl(key),
+ }
+ })
+
+ console.log(JSON.stringify({
+ mode: apply ? 'apply' : 'dry-run',
+ oldPrefix,
+ totalUrls: mappings.length,
+ concurrency,
+ bulkDbUpdate,
+ skipDbUpdate,
+ sample: mappings.slice(0, 3),
+ }, null, 2))
+
+ if (!apply) {
+ return
+ }
+
+ const s3Client = createS3Client()
+ const bucket = process.env.S3_BUCKET
+ const stats = {
+ uploaded: 0,
+ skippedExisting: 0,
+ dbMaterialUpdates: 0,
+ dbAssetUpdates: 0,
+ failed: 0,
+ }
+ const failures = []
+
+ await runPool(mappings, concurrency, async (mapping, currentIndex) => {
+ try {
+ const exists = await destinationExists(s3Client, bucket, mapping.key)
+ if (exists) {
+ stats.skippedExisting += 1
+ } else {
+ const oldObject = await fetchOldObject(mapping.oldUrl)
+ await s3Client.send(new PutObjectCommand({
+ Bucket: bucket,
+ Key: mapping.key,
+ Body: oldObject.body,
+ ContentType: oldObject.contentType,
+ ACL: process.env.S3_SET_ACL === '0' ? undefined : 'public-read',
+ }))
+ stats.uploaded += 1
+ }
+
+ if (!bulkDbUpdate && !skipDbUpdate) {
+ const dbResult = await updateDatabaseUrl(mapping.oldUrl, mapping.newUrl)
+ stats.dbMaterialUpdates += Number(dbResult.materialUpdates || 0)
+ stats.dbAssetUpdates += Number(dbResult.assetUpdates || 0)
+ }
+ } catch (error) {
+ stats.failed += 1
+ failures.push({
+ oldUrl: mapping.oldUrl,
+ message: error instanceof Error ? error.message : String(error),
+ })
+ }
+
+ if ((currentIndex + 1) % 1000 === 0 || currentIndex + 1 === mappings.length) {
+ console.log(JSON.stringify({ processed: currentIndex + 1, total: mappings.length, stats }))
+ }
+ })
+
+ if (bulkDbUpdate && stats.failed === 0) {
+ const newPrefix = mappings[0]
+ ? mappings[0].newUrl.slice(0, mappings[0].newUrl.length - mappings[0].key.length)
+ : ''
+ if (!newPrefix) {
+ throw new Error('Cannot infer new prefix for bulk database update')
+ }
+ const dbResult = await bulkUpdateDatabaseUrls(oldPrefix, newPrefix)
+ stats.dbMaterialUpdates += Number(dbResult.materialUpdates || 0)
+ stats.dbAssetUpdates += Number(dbResult.assetUpdates || 0)
+ }
+
+ console.log(JSON.stringify({ done: true, stats, failures: failures.slice(0, 20) }, null, 2))
+ if (stats.failed > 0) {
+ process.exitCode = 1
+ }
+}
+
+main()
+ .catch((error) => {
+ console.error(error)
+ process.exitCode = 1
+ })
+ .finally(async () => {
+ await prisma.$disconnect()
+ })
diff --git a/scripts/review_needs_review_spu_groups.py b/scripts/review_needs_review_spu_groups.py
new file mode 100644
index 0000000..4e61995
--- /dev/null
+++ b/scripts/review_needs_review_spu_groups.py
@@ -0,0 +1,315 @@
+#!/usr/bin/env python3
+"""
+Build a second-pass review table for needs_review SPU ledger items.
+
+The first SPU ledger intentionally keeps uncertain workspaces out of the import
+path. This script does not change the ledger or production data; it groups those
+uncertain rows by the real SPU root and rescans that root so cross-level
+main/SKU/detail folders can be reviewed together.
+"""
+
+from __future__ import annotations
+
+import argparse
+import csv
+import json
+import os
+import sqlite3
+from collections import Counter, defaultdict
+from datetime import datetime
+from pathlib import Path
+from typing import Any
+
+from scan_spu_ledger import (
+ EXCLUDED_DIR_NAMES,
+ IMAGE_EXTENSIONS,
+ OTHER_IMAGE_EXTENSIONS,
+ PS_SOURCE_EXTENSIONS,
+ RELEVANT_EXTENSIONS,
+ SOURCE_ARCHIVE_EXTENSIONS,
+ build_sku_code,
+ classify_file,
+ extract_spu_candidates,
+ has_hard_skip,
+ has_modeling_source_segment,
+ is_compatible_spu_token,
+ normalize_spu,
+ safe_relative,
+)
+
+
+REVIEW_HEADERS = [
+ "decision",
+ "spu",
+ "store",
+ "product_category",
+ "spu_root_path",
+ "needs_review_rows",
+ "direct_all_three_rows",
+ "aggregate_main_count",
+ "aggregate_sku_count",
+ "aggregate_detail_count",
+ "aggregate_source_count",
+ "hard_skip_file_count",
+ "modeling_file_count",
+ "status_reasons",
+ "workspace_samples",
+ "main_samples",
+ "sku_samples",
+ "detail_samples",
+]
+
+
+def json_dumps(value: Any) -> str:
+ return json.dumps(value, ensure_ascii=False, indent=2)
+
+
+def latest_run_id(conn: sqlite3.Connection) -> str:
+ row = conn.execute(
+ "SELECT run_id FROM spu_ledger_runs ORDER BY started_at DESC LIMIT 1"
+ ).fetchone()
+ if not row:
+ raise SystemExit("台账数据库里没有扫描 run")
+ return str(row["run_id"])
+
+
+def choose_spu_root_path(item: sqlite3.Row) -> str:
+ source_root = Path(item["source_root"])
+ workspace_path = Path(item["workspace_path"])
+ relative = Path(item["relative_workspace_path"])
+ spu = normalize_spu(item["spu"])
+
+ parts = list(relative.parts)
+ for index, part in enumerate(parts):
+ tokens = extract_spu_candidates(part, allow_unbranded=True)
+ if any(is_compatible_spu_token(spu, token) for token in tokens):
+ return str(source_root.joinpath(*parts[: index + 1]))
+
+ # File-inferred and non-standard folders can have no SPU-looking segment.
+ # In that case use the nearest useful parent instead of the leaf itself so
+ # sibling main/SKU/detail folders can still be discovered.
+ if len(parts) >= 2:
+ return str(source_root.joinpath(*parts[:-1]))
+ return str(workspace_path)
+
+
+def scan_spu_root(root_path: str, spu: str) -> dict[str, Any]:
+ root = Path(root_path)
+ counts: Counter[str] = Counter()
+ samples: dict[str, list[dict[str, Any]]] = defaultdict(list)
+ hard_skip_file_count = 0
+ modeling_file_count = 0
+
+ if not root.exists():
+ return {
+ "counts": counts,
+ "samples": samples,
+ "hard_skip_file_count": 0,
+ "modeling_file_count": 0,
+ "exists": False,
+ }
+
+ for current, dir_names, file_names in os.walk(root):
+ dir_names[:] = [
+ name
+ for name in dir_names
+ if name not in EXCLUDED_DIR_NAMES and not has_hard_skip(str(Path(current) / name))
+ ]
+
+ current_path = Path(current)
+ current_is_modeling = has_modeling_source_segment(str(current_path))
+ for file_name in file_names:
+ file_path = current_path / file_name
+ extension = file_path.suffix.lower()
+ if extension not in RELEVANT_EXTENSIONS:
+ continue
+
+ relative_path = safe_relative(file_path, root)
+ if has_hard_skip(str(file_path)):
+ hard_skip_file_count += 1
+ continue
+ if current_is_modeling:
+ modeling_file_count += 1
+
+ group = classify_file(relative_path, file_name, extension)
+ if group == "other_source_archive" and extension in SOURCE_ARCHIVE_EXTENSIONS:
+ # Keep archives visible, but do not promote generic zip files to
+ # source unless the first-pass classifier saw source evidence.
+ pass
+
+ counts[group] += 1
+ if group in {"main", "sku", "detail", "source"} and len(samples[group]) < 8:
+ record = {
+ "filePath": str(file_path),
+ "relativePath": relative_path,
+ "fileName": file_name,
+ "fileSize": file_path.stat().st_size,
+ }
+ if group == "sku":
+ record["skuCode"] = build_sku_code(spu, file_name, counts[group] - 1)
+ record["barcodes"] = []
+ samples[group].append(record)
+
+ return {
+ "counts": counts,
+ "samples": samples,
+ "hard_skip_file_count": hard_skip_file_count,
+ "modeling_file_count": modeling_file_count,
+ "exists": True,
+ }
+
+
+def decide_group(items: list[sqlite3.Row], aggregate: dict[str, Any]) -> str:
+ counts: Counter[str] = aggregate["counts"]
+ has_all_three = counts["main"] > 0 and counts["sku"] > 0 and counts["detail"] > 0
+ direct_all_three = [
+ item
+ for item in items
+ if int(item["main_count"] or 0) > 0
+ and int(item["sku_count"] or 0) > 0
+ and int(item["detail_count"] or 0) > 0
+ ]
+ reasons = " | ".join(sorted({str(item["status_reason"] or "") for item in items}))
+ has_modeling_reason = "建模" in reasons or "3D" in reasons or aggregate["modeling_file_count"] > 0
+
+ if not aggregate["exists"]:
+ return "manual_missing_root"
+ if direct_all_three and not has_modeling_reason:
+ return "direct_ready_review"
+ if has_all_three and not has_modeling_reason:
+ return "cross_level_ready_review"
+ if has_modeling_reason and not has_all_three:
+ return "skip_likely_modeling_or_partial"
+ if counts["detail"] == 0:
+ return "skip_likely_no_detail"
+ if counts["main"] == 0 or counts["sku"] == 0:
+ return "partial_needs_pairing"
+ return "manual_review"
+
+
+def write_csv(path: Path, rows: list[dict[str, Any]]) -> None:
+ path.parent.mkdir(parents=True, exist_ok=True)
+ with path.open("w", encoding="utf-8-sig", newline="") as handle:
+ writer = csv.DictWriter(handle, fieldnames=REVIEW_HEADERS)
+ writer.writeheader()
+ writer.writerows(rows)
+
+
+def main() -> int:
+ parser = argparse.ArgumentParser(description="Second-pass review for needs_review SPU groups.")
+ parser.add_argument("--db-path", default="output/tianlan-spu-ledger.db")
+ parser.add_argument("--run-id", default="")
+ parser.add_argument("--output-dir", default="")
+ args = parser.parse_args()
+
+ db_path = Path(args.db_path)
+ stamp = datetime.now().strftime("%Y%m%d-%H%M%S")
+ output_dir = Path(args.output_dir or f"output/tianlan-spu-ledger/needs-review-cross-level-{stamp}")
+ output_dir.mkdir(parents=True, exist_ok=True)
+
+ conn = sqlite3.connect(str(db_path))
+ conn.row_factory = sqlite3.Row
+ run_id = args.run_id or latest_run_id(conn)
+ items = conn.execute(
+ """
+ SELECT *
+ FROM spu_ledger_items
+ WHERE run_id = ? AND status = 'needs_review'
+ ORDER BY source_root_name, relative_workspace_path, spu
+ """,
+ (run_id,),
+ ).fetchall()
+
+ groups: dict[tuple[str, str, str, str], list[sqlite3.Row]] = defaultdict(list)
+ for item in items:
+ spu_root_path = choose_spu_root_path(item)
+ key = (
+ str(item["store"] or ""),
+ normalize_spu(item["spu"]),
+ str(item["source_root_name"] or ""),
+ spu_root_path,
+ )
+ groups[key].append(item)
+
+ review_rows: list[dict[str, Any]] = []
+ details: list[dict[str, Any]] = []
+ for (store, spu, source_root_name, spu_root_path), grouped_items in sorted(groups.items()):
+ aggregate = scan_spu_root(spu_root_path, spu)
+ counts: Counter[str] = aggregate["counts"]
+ decision = decide_group(grouped_items, aggregate)
+ status_reasons = sorted({str(item["status_reason"] or "") for item in grouped_items})
+ workspace_samples = [str(item["workspace_path"]) for item in grouped_items[:8]]
+ direct_all_three_rows = sum(
+ 1
+ for item in grouped_items
+ if int(item["main_count"] or 0) > 0
+ and int(item["sku_count"] or 0) > 0
+ and int(item["detail_count"] or 0) > 0
+ )
+
+ row = {
+ "decision": decision,
+ "spu": spu,
+ "store": store,
+ "product_category": grouped_items[0]["product_category"],
+ "spu_root_path": spu_root_path,
+ "needs_review_rows": len(grouped_items),
+ "direct_all_three_rows": direct_all_three_rows,
+ "aggregate_main_count": counts["main"],
+ "aggregate_sku_count": counts["sku"],
+ "aggregate_detail_count": counts["detail"],
+ "aggregate_source_count": counts["source"],
+ "hard_skip_file_count": aggregate["hard_skip_file_count"],
+ "modeling_file_count": aggregate["modeling_file_count"],
+ "status_reasons": " | ".join(status_reasons),
+ "workspace_samples": " || ".join(workspace_samples),
+ "main_samples": " || ".join(item["relativePath"] for item in aggregate["samples"].get("main", [])),
+ "sku_samples": " || ".join(item["relativePath"] for item in aggregate["samples"].get("sku", [])),
+ "detail_samples": " || ".join(item["relativePath"] for item in aggregate["samples"].get("detail", [])),
+ }
+ review_rows.append(row)
+ details.append(
+ {
+ **row,
+ "sourceRootName": source_root_name,
+ "workspaces": [
+ {
+ "fingerprint": item["fingerprint"],
+ "workspacePath": item["workspace_path"],
+ "relativeWorkspacePath": item["relative_workspace_path"],
+ "versionLabel": item["version_label"],
+ "mainCount": item["main_count"],
+ "skuCount": item["sku_count"],
+ "detailCount": item["detail_count"],
+ "sourceCount": item["source_count"],
+ "statusReason": item["status_reason"],
+ }
+ for item in grouped_items
+ ],
+ "samples": aggregate["samples"],
+ }
+ )
+
+ write_csv(output_dir / "needs-review-cross-level-groups.csv", review_rows)
+ (output_dir / "needs-review-cross-level-groups.json").write_text(
+ json_dumps(details), encoding="utf-8"
+ )
+
+ summary = {
+ "runId": run_id,
+ "dbPath": str(db_path),
+ "needsReviewItems": len(items),
+ "groups": len(review_rows),
+ "decisionCounts": dict(Counter(row["decision"] for row in review_rows)),
+ "outputCsv": str(output_dir / "needs-review-cross-level-groups.csv"),
+ "outputJson": str(output_dir / "needs-review-cross-level-groups.json"),
+ }
+ (output_dir / "summary.json").write_text(json_dumps(summary), encoding="utf-8")
+
+ conn.close()
+ print(json_dumps(summary))
+ return 0
+
+
+if __name__ == "__main__":
+ raise SystemExit(main())
diff --git a/scripts/run-tianlan-remaining-batches.ps1 b/scripts/run-tianlan-remaining-batches.ps1
new file mode 100644
index 0000000..3489590
--- /dev/null
+++ b/scripts/run-tianlan-remaining-batches.ps1
@@ -0,0 +1,202 @@
+param(
+ [int]$StartBatch = 2,
+ [int]$EndBatch = 15,
+ [string]$BatchDir = "D:\AI Code\gongxiangp2\output\tianlan-spu-ledger\remaining-batches-1778239500000",
+ [string]$Operator = "222"
+)
+
+$ErrorActionPreference = "Stop"
+Set-Location "D:\AI Code\gongxiangp2"
+
+function Set-ImportEnvironment {
+ $env:DATABASE_URL = ((Get-Content .env | Where-Object { $_ -match '^DATABASE_URL=' } | Select-Object -First 1) -replace '^DATABASE_URL=', '').Trim()
+ $env:S3_ENDPOINT = "http://192.168.58.20:9000"
+ $env:S3_BUCKET = "gongxiangpan"
+ $env:S3_PUBLIC_DOMAIN = "https://s3.jiansun.vip/gongxiangpan"
+ $env:S3_FILE_PATH = [System.Text.Encoding]::UTF8.GetString([System.Convert]::FromBase64String("55Sx5aO554K5QUkv"))
+ $sshArgs = @(
+ "-o", "BatchMode=yes",
+ "-o", "ConnectTimeout=15",
+ "-o", "ServerAliveInterval=15",
+ "-o", "ServerAliveCountMax=2",
+ "prod"
+ )
+ $env:S3_ACCESS_KEY_ID = Invoke-TextWithRetry "read S3_ACCESS_KEY_ID" { ssh @sshArgs "docker exec youyidian-3d-material printenv S3_ACCESS_KEY_ID" }
+ $env:S3_SECRET_ACCESS_KEY = Invoke-TextWithRetry "read S3_SECRET_ACCESS_KEY" { ssh @sshArgs "docker exec youyidian-3d-material printenv S3_SECRET_ACCESS_KEY" }
+ $env:HISTORY_IMPORT_REMOTE_UPLOADS_HOST = "prod"
+ $env:HISTORY_IMPORT_REMOTE_UPLOADS_DIR = "/mnt/PoolOne/apps/youyidian-3d/uploads"
+}
+
+function Get-NewestImportResult([datetime]$StartedAt) {
+ Get-ChildItem -LiteralPath $BatchDir -Filter "history-material-import-result-*.json" |
+ Where-Object { $_.Name -notlike "*.verification.json" -and $_.LastWriteTime -ge $StartedAt } |
+ Sort-Object LastWriteTime -Descending |
+ Select-Object -First 1
+}
+
+function Write-GoalProgress($Payload) {
+ $progressPath = Join-Path $BatchDir "goal-progress.json"
+ $Payload | ConvertTo-Json -Depth 8 | Set-Content -LiteralPath $progressPath -Encoding UTF8
+}
+
+function Invoke-NpxToLog([string[]]$Arguments, [string]$LogPath) {
+ $previousErrorActionPreference = $ErrorActionPreference
+ $ErrorActionPreference = "Continue"
+ try {
+ & npx @Arguments *> $LogPath
+ return $LASTEXITCODE
+ } finally {
+ $script:ErrorActionPreference = $previousErrorActionPreference
+ }
+}
+
+function Invoke-TextWithRetry([string]$Label, [scriptblock]$Command, [int]$Retries = 3) {
+ $lastOutput = $null
+ for ($attempt = 1; $attempt -le $Retries; $attempt += 1) {
+ $lastOutput = & $Command
+ if ($LASTEXITCODE -eq 0 -and -not [string]::IsNullOrWhiteSpace(($lastOutput | Out-String))) {
+ return ($lastOutput | Out-String).Trim()
+ }
+
+ if ($attempt -lt $Retries) {
+ Write-Host ("[Goal] {0} failed on attempt {1}/{2}; retrying..." -f $Label, $attempt, $Retries)
+ Start-Sleep -Seconds ([Math]::Min(10, 2 * $attempt))
+ }
+ }
+
+ throw ("{0} failed after {1} attempts" -f $Label, $Retries)
+}
+
+Set-ImportEnvironment
+
+$runStartedAt = Get-Date
+$completed = @()
+
+[pscustomobject]@{
+ pid = $PID
+ startBatch = $StartBatch
+ endBatch = $EndBatch
+ batchDir = $BatchDir
+ startedAt = $runStartedAt.ToString("o")
+ command = "scripts/run-tianlan-remaining-batches.ps1 -StartBatch $StartBatch -EndBatch $EndBatch"
+} | ConvertTo-Json -Depth 5 | Set-Content -LiteralPath (Join-Path $BatchDir "active-goal-process.json") -Encoding UTF8
+
+for ($batchNumber = $StartBatch; $batchNumber -le $EndBatch; $batchNumber += 1) {
+ $batchName = "batch-{0:D3}.json" -f $batchNumber
+ $batchPath = Join-Path $BatchDir $batchName
+
+ if (-not (Test-Path -LiteralPath $batchPath)) {
+ throw "Batch file not found: $batchPath"
+ }
+
+ $stamp = Get-Date -Format "yyyyMMdd-HHmmss"
+ $importLog = Join-Path $BatchDir ("batch-{0:D3}-{1}.import.log" -f $batchNumber, $stamp)
+ $verifyLog = Join-Path $BatchDir ("batch-{0:D3}-{1}.verify.log" -f $batchNumber, $stamp)
+ $startedAt = Get-Date
+
+ Write-Host ""
+ Write-Host ("[Goal] Importing {0}" -f $batchName)
+ Write-GoalProgress ([pscustomobject]@{
+ mode = "tianlan-remaining-batches"
+ runStartedAt = $runStartedAt.ToString("o")
+ updatedAt = (Get-Date).ToString("o")
+ startBatch = $StartBatch
+ endBatch = $EndBatch
+ completed = $completed
+ current = [pscustomobject]@{
+ batch = $batchName
+ phase = "importing"
+ importLog = Split-Path -Leaf $importLog
+ verifyLog = Split-Path -Leaf $verifyLog
+ startedAt = $startedAt.ToString("o")
+ }
+ })
+
+ $importExitCode = Invoke-NpxToLog @(
+ "tsx",
+ "scripts/import-history-materials.ts",
+ "--input=$batchPath",
+ "--operator=$Operator",
+ "--apply"
+ ) $importLog
+ $resultFile = Get-NewestImportResult $startedAt
+
+ if ($importExitCode -ne 0 -or -not $resultFile) {
+ Write-Host ("[Goal] Import failed for {0}; log: {1}" -f $batchName, $importLog)
+ if (Test-Path -LiteralPath $importLog) {
+ Get-Content -LiteralPath $importLog -Tail 80
+ }
+ exit 1
+ }
+
+ Write-Host ("[Goal] Result: {0}" -f $resultFile.Name)
+ Write-GoalProgress ([pscustomobject]@{
+ mode = "tianlan-remaining-batches"
+ runStartedAt = $runStartedAt.ToString("o")
+ updatedAt = (Get-Date).ToString("o")
+ startBatch = $StartBatch
+ endBatch = $EndBatch
+ completed = $completed
+ current = [pscustomobject]@{
+ batch = $batchName
+ phase = "validating"
+ result = $resultFile.Name
+ importLog = Split-Path -Leaf $importLog
+ verifyLog = Split-Path -Leaf $verifyLog
+ }
+ })
+ $verifyExitCode = Invoke-NpxToLog @(
+ "tsx",
+ "scripts/validate-history-import-result.ts",
+ "--input=$($resultFile.FullName)",
+ "--remote-host=prod",
+ "--remote-uploads-dir=/mnt/PoolOne/apps/youyidian-3d/uploads"
+ ) $verifyLog
+ $verificationPath = $resultFile.FullName -replace '\.json$', '.verification.json'
+ $verification = Get-Content -LiteralPath $verificationPath -Raw | ConvertFrom-Json
+
+ $completed += [pscustomobject]@{
+ batch = $batchName
+ result = $resultFile.Name
+ total = $verification.total
+ imported = $verification.imported
+ skipped = $verification.skipped
+ failed = $verification.failed
+ checked = $verification.checked
+ ok = $verification.ok
+ issueCount = $verification.issueCount
+ importLog = Split-Path -Leaf $importLog
+ verifyLog = Split-Path -Leaf $verifyLog
+ finishedAt = (Get-Date).ToString("o")
+ }
+
+ Write-GoalProgress ([pscustomobject]@{
+ mode = "tianlan-remaining-batches"
+ runStartedAt = $runStartedAt.ToString("o")
+ updatedAt = (Get-Date).ToString("o")
+ startBatch = $StartBatch
+ endBatch = $EndBatch
+ completed = $completed
+ current = $completed[-1]
+ })
+
+ Write-Host ("[Goal] Verified {0}: imported={1}, skipped={2}, failed={3}, ok={4}, issues={5}" -f $batchName, $verification.imported, $verification.skipped, $verification.failed, $verification.ok, $verification.issueCount)
+
+ if ($verifyExitCode -ne 0 -or $verification.failed -gt 0 -or $verification.issueCount -gt 0) {
+ Write-Host ("[Goal] Stopped on verification issue for {0}; log: {1}" -f $batchName, $verifyLog)
+ Get-Content -LiteralPath $verifyLog -Tail 120
+ exit 1
+ }
+}
+
+Write-Host ""
+Write-Host "[Goal] All requested batches completed."
+Write-GoalProgress ([pscustomobject]@{
+ mode = "tianlan-remaining-batches"
+ runStartedAt = $runStartedAt.ToString("o")
+ updatedAt = (Get-Date).ToString("o")
+ startBatch = $StartBatch
+ endBatch = $EndBatch
+ completed = $completed
+ done = $true
+})
diff --git a/scripts/scan_sku_folders_to_sqlite.py b/scripts/scan_sku_folders_to_sqlite.py
index 86cb685..c76ce4a 100644
--- a/scripts/scan_sku_folders_to_sqlite.py
+++ b/scripts/scan_sku_folders_to_sqlite.py
@@ -16,9 +16,20 @@ NON_DIRECTORY_EXTENSIONS = IMAGE_EXTENSIONS | {
}
SKU_DIR_PATTERN = re.compile(r"sku", re.IGNORECASE)
SPU_PATTERN = re.compile(r"^[A-Za-z]{1,5}\d{3,5}$")
+SPU_TOKEN_PATTERN = re.compile(r"([A-Za-z]{1,5}\d{3,5})")
+BRANDED_SPU_TOKEN_PATTERN = re.compile(r"([A-Za-z]{2,5}-[A-Za-z]{1,5}\d{3,5})")
+BRANDED_SKU_FILE_PATTERN = re.compile(r"([A-Za-z]{2,5})-([A-Za-z]{1,5})(\d{4,5})(\d{2})(?:\D|$)")
+KNOWN_SHOP_PREFIXES = {"GBPM", "GBSFY", "GBGJ", "GBJP", "XYL"}
+IGNORED_SPU_PREFIXES = {"TASK", "SKU", "IMG"}
+KNOWN_CATEGORY_CODES = {
+ "SN", "DG", "XG", "BG", "SG", "CBG", "CZ", "SZ", "DNZ", "SJZ", "ZDZ",
+ "CBZ", "CY", "AD", "SZD", "HXD", "SJ", "ZSJ", "ZWJ", "CFZW", "CFZWJ",
+ "YMJ", "BJ", "CJ", "DSG", "CTG", "CYJ", "SZT", "XJ", "LRSF", "SMC",
+}
IGNORED_PARTS = {"@eadir", "dingtalkappdata", ".recycle", "$nsec.recycle.bin"}
DETAIL_DIR_KEYWORDS = ("images", "详情页", "详情")
MAIN_DIR_KEYWORD = "主图"
+MAIN_DIMENSION_DIR_NAMES = {"750", "800"}
def utc_now() -> str:
@@ -29,11 +40,93 @@ def normalize_spu(value: str) -> str:
return value.strip().upper()
-def guess_spu(path_obj: Path) -> str:
+def looks_like_spu_token(value: str) -> bool:
+ normalized = normalize_spu(value)
+ if "-" in normalized:
+ prefix, _, sku_part = normalized.partition("-")
+ return prefix in KNOWN_SHOP_PREFIXES and bool(SPU_PATTERN.match(sku_part))
+
+ return bool(SPU_PATTERN.match(normalized)) and not any(
+ normalized.startswith(prefix) for prefix in IGNORED_SPU_PREFIXES
+ )
+
+
+def is_standalone_spu_match(text: str, start: int, end: int) -> bool:
+ prev_char = text[start - 1] if start > 0 else ""
+ next_char = text[end] if end < len(text) else ""
+ if re.match(r"[A-Za-z0-9]", prev_char or ""):
+ return False
+ if not re.match(r"[A-Za-z0-9]", next_char or ""):
+ return True
+ return bool(re.match(r"K\d{1,2}(?![A-Za-z0-9])", text[end:], re.IGNORECASE))
+
+
+def get_sku_part(spu: str) -> str:
+ normalized = normalize_spu(spu)
+ return normalized.partition("-")[2] if "-" in normalized else normalized
+
+
+def has_known_category_code(spu: str) -> bool:
+ sku_part = get_sku_part(spu)
+ for code in sorted(KNOWN_CATEGORY_CODES, key=len, reverse=True):
+ if sku_part.startswith(code) and len(sku_part) > len(code) and sku_part[len(code)].isdigit():
+ return True
+ return False
+
+
+def collect_spu_tokens(path_obj: Path):
+ tokens = []
+ seen = set()
+ for part in reversed(path_obj.parts):
+ matches = list(BRANDED_SPU_TOKEN_PATTERN.finditer(part))
+ for match in reversed(matches):
+ token = normalize_spu(match.group(1))
+ if (
+ token not in seen
+ and looks_like_spu_token(token)
+ and is_standalone_spu_match(part, match.start(), match.end())
+ ):
+ tokens.append(token)
+ seen.add(token)
+
for part in reversed(path_obj.parts):
normalized = normalize_spu(part)
- if SPU_PATTERN.match(normalized):
- return normalized
+ if normalized not in seen and looks_like_spu_token(normalized):
+ tokens.append(normalized)
+ seen.add(normalized)
+
+ matches = list(SPU_TOKEN_PATTERN.finditer(part))
+ for match in reversed(matches):
+ token = normalize_spu(match.group(1))
+ if (
+ token not in seen
+ and looks_like_spu_token(token)
+ and is_standalone_spu_match(part, match.start(), match.end())
+ ):
+ tokens.append(token)
+ seen.add(token)
+
+ return tokens
+
+
+def guess_spu(path_obj: Path) -> str:
+ tokens = collect_spu_tokens(path_obj)
+ for token in tokens:
+ if has_known_category_code(token):
+ return token
+ if tokens:
+ return tokens[0]
+ return ""
+
+
+def guess_spu_from_file_names(file_names) -> str:
+ for file_name in file_names:
+ match = BRANDED_SKU_FILE_PATTERN.search(file_name)
+ if not match:
+ continue
+ token = normalize_spu(f"{match.group(1)}-{match.group(2)}{match.group(3)}")
+ if looks_like_spu_token(token):
+ return token
return ""
@@ -63,7 +156,7 @@ def is_detail_dir_name(name: str) -> bool:
def is_main_dir_name(name: str) -> bool:
normalized = normalize_dir_name(name)
- return MAIN_DIR_KEYWORD in normalized
+ return MAIN_DIR_KEYWORD in normalized or normalized in MAIN_DIMENSION_DIR_NAMES
def list_sibling_dirs(parent_path: Path, current_dir_name: str):
@@ -161,12 +254,15 @@ def ensure_schema(conn: sqlite3.Connection) -> None:
conn.commit()
-def scan_sku_dirs(root_path: Path):
+def scan_sku_dirs(root_path: Path, skip_repeated_root_name: bool = False):
for current_root, dir_names, file_names in os.walk(root_path):
current_path = Path(current_root)
if should_ignore_path(current_path):
dir_names[:] = []
continue
+ if skip_repeated_root_name and current_path == root_path:
+ root_name = root_path.name.casefold()
+ dir_names[:] = [name for name in dir_names if name.casefold() != root_name]
for dir_name in list(dir_names):
if not matches_sku_dir(dir_name):
@@ -195,7 +291,7 @@ def scan_sku_dirs(root_path: Path):
"directory_name": sku_dir_path.name,
"parent_path": str(sku_dir_path.parent),
"grandparent_path": str(sku_dir_path.parent.parent) if sku_dir_path.parent != root_path else "",
- "spu": guess_spu(sku_dir_path),
+ "spu": guess_spu(sku_dir_path) or guess_spu_from_file_names(image_files),
"depth": len(sku_dir_path.relative_to(root_path).parts),
"file_count": len(files),
"image_count": len(image_files),
@@ -259,21 +355,32 @@ def parse_args():
default="/mnt/history-sc/大货营销策划图片",
help="Root directory to scan.",
)
+ parser.add_argument(
+ "--roots",
+ default="",
+ help="Multiple root directories separated by |. When provided, --root is ignored.",
+ )
parser.add_argument(
"--db-path",
default="/root/gongxiangp2/output/history-sku-folders.db",
help="SQLite database file path.",
)
+ parser.add_argument(
+ "--skip-repeated-root-name",
+ action="store_true",
+ help="Skip an immediate child directory whose name equals the scan root name.",
+ )
return parser.parse_args()
def main() -> int:
args = parse_args()
- root_path = Path(args.root)
+ root_paths = [Path(item.strip()) for item in args.roots.split("|") if item.strip()] if args.roots else [Path(args.root)]
db_path = Path(args.db_path)
- if not root_path.exists():
- print(f"Root path not found: {root_path}", file=sys.stderr)
+ missing_roots = [str(root_path) for root_path in root_paths if not root_path.exists()]
+ if missing_roots:
+ print(f"Root path not found: {missing_roots[0]}", file=sys.stderr)
return 1
db_path.parent.mkdir(parents=True, exist_ok=True)
@@ -281,29 +388,35 @@ def main() -> int:
conn = sqlite3.connect(str(db_path))
try:
ensure_schema(conn)
- scan_id = str(uuid.uuid4())
- started_at = utc_now()
- conn.execute(
- "INSERT INTO sku_folder_scans (scan_id, root_path, started_at) VALUES (?, ?, ?)",
- (scan_id, str(root_path), started_at),
- )
- conn.commit()
+ all_previews = []
+ grand_total = 0
+ scan_ids = []
+ for root_path in root_paths:
+ scan_id = str(uuid.uuid4())
+ scan_ids.append(scan_id)
+ started_at = utc_now()
+ conn.execute(
+ "INSERT INTO sku_folder_scans (scan_id, root_path, started_at) VALUES (?, ?, ?)",
+ (scan_id, str(root_path), started_at),
+ )
+ conn.commit()
- rows = list(scan_sku_dirs(root_path))
- total_rows = insert_rows(conn, scan_id, rows)
- finished_at = utc_now()
- conn.execute(
- "UPDATE sku_folder_scans SET finished_at = ?, total_rows = ? WHERE scan_id = ?",
- (finished_at, total_rows, scan_id),
- )
- conn.commit()
+ rows = list(scan_sku_dirs(root_path, args.skip_repeated_root_name))
+ total_rows = insert_rows(conn, scan_id, rows)
+ grand_total += total_rows
+ finished_at = utc_now()
+ conn.execute(
+ "UPDATE sku_folder_scans SET finished_at = ?, total_rows = ? WHERE scan_id = ?",
+ (finished_at, total_rows, scan_id),
+ )
+ conn.commit()
+ all_previews.extend(rows[:3])
- preview = rows[:12]
print("SKU folder scan completed")
- print(f"scan_id={scan_id}")
+ print(f"scan_ids={','.join(scan_ids)}")
print(f"db_path={db_path}")
- print(f"total_rows={total_rows}")
- print(json.dumps(preview, ensure_ascii=False, indent=2))
+ print(f"total_rows={grand_total}")
+ print(json.dumps(all_previews[:12], ensure_ascii=False, indent=2))
return 0
finally:
conn.close()
diff --git a/scripts/scan_spu_ledger.py b/scripts/scan_spu_ledger.py
new file mode 100644
index 0000000..9654b39
--- /dev/null
+++ b/scripts/scan_spu_ledger.py
@@ -0,0 +1,1282 @@
+#!/usr/bin/env python3
+"""
+Scan real material folders into an SPU-centered SQLite ledger.
+
+This scanner intentionally does not read the older SKU-anchor CSV/SQLite outputs.
+Those files can be used later as evidence for old-batch review, but the ledger
+itself is built from filesystem facts under the provided roots.
+"""
+
+from __future__ import annotations
+
+import argparse
+import hashlib
+import json
+import os
+import re
+import sqlite3
+import sys
+import uuid
+from dataclasses import dataclass, field
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Iterable
+
+
+IMAGE_EXTENSIONS = {".jpg", ".jpeg", ".png", ".webp"}
+OTHER_IMAGE_EXTENSIONS = {".tif", ".tiff", ".bmp", ".gif"}
+PS_SOURCE_EXTENSIONS = {".psd", ".psb"}
+SOURCE_ARCHIVE_EXTENSIONS = {".zip", ".rar", ".7z"}
+RELEVANT_EXTENSIONS = IMAGE_EXTENSIONS | OTHER_IMAGE_EXTENSIONS | PS_SOURCE_EXTENSIONS | SOURCE_ARCHIVE_EXTENSIONS
+
+BRANDED_SPU_BODY = r"[A-Z]{2,5}-[A-Z]{1,6}\d{3,5}(?:K\d{1,3})?"
+UNBRANDED_SPU_BODY = r"[A-Z]{1,6}\d{3,5}(?:K\d{1,3})?"
+BRANDED_SPU_RE = re.compile(rf"(? str:
+ return datetime.now(timezone.utc).replace(microsecond=0).isoformat()
+
+
+def normalize_token_text(value: str) -> str:
+ return (
+ value.replace("—", "-")
+ .replace("–", "-")
+ .replace("-", "-")
+ .replace("+", "+")
+ .upper()
+ .strip()
+ )
+
+
+def normalize_spu(value: str) -> str:
+ normalized = normalize_token_text(value)
+ normalized = re.sub(r"\s*\+\s*", "+", normalized)
+ return normalized
+
+
+def split_combo(spu: str) -> list[str]:
+ normalized = normalize_spu(spu)
+ return [part for part in normalized.split("+") if part]
+
+
+def expand_combo_spu_token(token: str) -> str:
+ parts = split_combo(token)
+ if len(parts) < 2:
+ return normalize_spu(token)
+ prefix = spu_prefix(parts[0])
+ expanded = [parts[0]]
+ for part in parts[1:]:
+ expanded.append(part if "-" in part or not prefix else f"{prefix}-{part}")
+ return "+".join(expanded)
+
+
+def spu_prefix(spu: str) -> str:
+ first_component = split_combo(spu)[0] if spu else ""
+ return first_component.split("-", 1)[0] if "-" in first_component else ""
+
+
+def spu_body(spu: str) -> str:
+ normalized = normalize_spu(spu)
+ return normalized.split("-", 1)[1] if "-" in normalized else normalized
+
+
+def has_unknown_branded_prefix(spu: str) -> bool:
+ prefix = spu_prefix(spu)
+ return bool(prefix and prefix not in KNOWN_SHOP_PREFIXES)
+
+
+def looks_like_unbranded_spu(token: str) -> bool:
+ normalized = normalize_spu(token)
+ if any(normalized.startswith(prefix) for prefix in ("IMG", "SKU", "TASK", "TALOG")):
+ return False
+ return any(re.match(rf"^{code}\d", normalized) for code in CATEGORY_CODE_MAP)
+
+
+def extract_spu_candidates(text: str, allow_unbranded: bool = True) -> list[str]:
+ normalized = normalize_token_text(text)
+ candidates: list[str] = []
+ seen = set()
+
+ for match in COMBO_SPU_RE.finditer(normalized):
+ token = expand_combo_spu_token(match.group(0))
+ if token not in seen:
+ candidates.append(token)
+ seen.add(token)
+
+ for match in BRANDED_SPU_RE.finditer(normalized):
+ token = normalize_spu(match.group(0))
+ if token not in seen:
+ candidates.append(token)
+ seen.add(token)
+
+ if allow_unbranded:
+ for match in UNBRANDED_SPU_RE.finditer(normalized):
+ token = normalize_spu(match.group(0))
+ if token in seen or "-" in token or not looks_like_unbranded_spu(token):
+ continue
+ candidates.append(token)
+ seen.add(token)
+
+ return candidates
+
+
+def extract_file_spu_candidates(file_name: str) -> list[str]:
+ normalized = normalize_token_text(file_name)
+ candidates: list[str] = []
+ seen = set()
+
+ for match in SKU_FILE_SPU_RE.finditer(normalized):
+ token = normalize_spu(f"{match.group(1)}-{match.group(2)}{match.group(3)}")
+ if token not in seen:
+ candidates.append(token)
+ seen.add(token)
+
+ for token in extract_spu_candidates(normalized, allow_unbranded=False):
+ if token not in seen:
+ candidates.append(token)
+ seen.add(token)
+
+ return candidates
+
+
+def has_hard_skip(path_text: str) -> bool:
+ return any(pattern.search(path_text) for pattern in HARD_SKIP_PATTERNS)
+
+
+def has_modeling_source_segment(path_text: str) -> bool:
+ normalized = path_text.replace("\\", "/")
+ return bool(MODELING_SOURCE_SEGMENT_RE.search(normalized))
+
+
+def safe_relative(path_obj: Path, root: Path) -> str:
+ try:
+ return str(path_obj.relative_to(root))
+ except ValueError:
+ return str(path_obj)
+
+
+def fingerprint_for(*parts: str) -> str:
+ payload = "|".join(parts)
+ return hashlib.sha1(payload.encode("utf-8", errors="ignore")).hexdigest()
+
+
+def json_dumps(value) -> str:
+ return json.dumps(value, ensure_ascii=False, separators=(",", ":"))
+
+
+def load_visual_author_names(path_text: str) -> list[str]:
+ if not path_text:
+ return []
+
+ path_obj = Path(path_text)
+ if not path_obj.exists():
+ raise SystemExit(f"视觉人员名单不存在: {path_text}")
+
+ names: list[str] = []
+ seen = set()
+
+ def add_name(value: str) -> None:
+ name = value.strip()
+ if not name or name in seen:
+ return
+ if not re.search(r"[\u4e00-\u9fff]", name):
+ return
+ if len(name) > 8:
+ return
+ seen.add(name)
+ names.append(name)
+
+ if path_obj.suffix.lower() == ".json":
+ payload = json.loads(path_obj.read_text(encoding="utf-8-sig"))
+ if isinstance(payload, list):
+ for item in payload:
+ if isinstance(item, str):
+ add_name(item)
+ elif isinstance(item, dict):
+ add_name(str(item.get("username") or ""))
+ add_name(str(item.get("realName") or ""))
+ else:
+ for line in path_obj.read_text(encoding="utf-8-sig").splitlines():
+ for cell in re.split(r"[\t,,]", line):
+ add_name(cell)
+
+ return sorted(names, key=lambda item: (-len(item), item))
+
+
+@dataclass
+class FileEntry:
+ full_path: str
+ relative_path: str
+ file_name: str
+ file_size: int
+ extension: str
+ group_hint: str
+ sku_code: str = ""
+
+
+@dataclass
+class DirInfo:
+ full_path: str
+ name: str
+ parent_path: str
+ depth: int
+ root_path: str
+ relative_path: str
+ children: list[str] = field(default_factory=list)
+ files: list[FileEntry] = field(default_factory=list)
+ subtree_files: int = 0
+ subtree_groups: dict[str, int] = field(default_factory=dict)
+ name_spus: list[str] = field(default_factory=list)
+
+
+@dataclass
+class Candidate:
+ full_path: str
+ root_path: str
+ relative_path: str
+ spu: str
+ spu_source: str
+ spu_evidence: list[str]
+
+
+def classify_file(relative_path: str, file_name: str, extension: str) -> str:
+ normalized = relative_path.replace("\\", "/")
+ lower_ext = extension.lower()
+ if lower_ext in PS_SOURCE_EXTENSIONS:
+ return "source"
+ if lower_ext in SOURCE_ARCHIVE_EXTENSIONS:
+ return "source" if SOURCE_HINT_RE.search(normalized) else "other_source_archive"
+ if lower_ext in OTHER_IMAGE_EXTENSIONS:
+ return "other_image"
+ if lower_ext not in IMAGE_EXTENSIONS:
+ return "other"
+ if EXCLUDE_PREVIEW_RE.search(normalized):
+ return "other_image"
+ if AI_SOURCE_RE.search(normalized):
+ return "other_image"
+ if SKU_HINT_RE.search(normalized):
+ return "sku"
+ if DETAIL_HINT_RE.search(normalized):
+ return "detail"
+ if re.search(r"(^|/)(790)(/|$)", normalized):
+ return "detail"
+ if MAIN_HINT_RE.search(normalized):
+ return "main"
+ if re.search(r"(^|/)(750|800|1440|1920|1440-1440|1440-1920|750-1000|800-800)(/|$)", normalized):
+ return "main"
+ return "other_image"
+
+
+def walk_tree(root: Path, max_depth: int) -> dict[str, DirInfo]:
+ root = root.absolute()
+ dirs: dict[str, DirInfo] = {}
+ nested_duplicate_root_name = root.name.lower()
+
+ def visit(current: Path, parent: Path | None, depth: int) -> None:
+ if depth > max_depth:
+ return
+
+ current_str = str(current)
+ parent_str = str(parent) if parent else ""
+ relative = safe_relative(current, root)
+ dir_info = DirInfo(
+ full_path=current_str,
+ name=current.name,
+ parent_path=parent_str,
+ depth=depth,
+ root_path=str(root),
+ relative_path="" if relative == "." else relative,
+ name_spus=extract_spu_candidates(current.name, allow_unbranded=True),
+ )
+ dirs[current_str] = dir_info
+ if parent_str and parent_str in dirs:
+ dirs[parent_str].children.append(current_str)
+
+ try:
+ entries = sorted(os.scandir(current), key=lambda item: item.name.lower())
+ except OSError:
+ return
+
+ for entry in entries:
+ full_path = Path(entry.path)
+ if entry.is_dir(follow_symlinks=False):
+ if entry.name in EXCLUDED_DIR_NAMES:
+ print(f"[skip] excluded dir: {entry.path}")
+ continue
+ if depth == 0 and entry.name.lower() == nested_duplicate_root_name:
+ print(f"[skip] nested duplicate root: {entry.path}")
+ continue
+ visit(full_path, current, depth + 1)
+ continue
+
+ if not entry.is_file(follow_symlinks=False):
+ continue
+
+ extension = full_path.suffix.lower()
+ if extension not in RELEVANT_EXTENSIONS:
+ continue
+
+ try:
+ stat = entry.stat(follow_symlinks=False)
+ except OSError:
+ continue
+
+ relative_file = safe_relative(full_path, root)
+ group = classify_file(relative_file, entry.name, extension)
+ dir_info.files.append(
+ FileEntry(
+ full_path=str(full_path),
+ relative_path=relative_file,
+ file_name=entry.name,
+ file_size=int(stat.st_size),
+ extension=extension,
+ group_hint=group,
+ )
+ )
+
+ visit(root, None, 0)
+
+ for dir_info in sorted(dirs.values(), key=lambda item: item.depth, reverse=True):
+ group_counts: dict[str, int] = {}
+ file_count = 0
+ for file in dir_info.files:
+ file_count += 1
+ group_counts[file.group_hint] = group_counts.get(file.group_hint, 0) + 1
+ for child_path in dir_info.children:
+ child = dirs[child_path]
+ file_count += child.subtree_files
+ for group, count in child.subtree_groups.items():
+ group_counts[group] = group_counts.get(group, 0) + count
+ dir_info.subtree_files = file_count
+ dir_info.subtree_groups = group_counts
+
+ return dirs
+
+
+def find_nearest_spu_ancestor(dir_info: DirInfo, dirs: dict[str, DirInfo]) -> tuple[str, str] | tuple[str, str]:
+ current_parent = dir_info.parent_path
+ while current_parent:
+ parent = dirs.get(current_parent)
+ if not parent:
+ break
+ if parent.name_spus:
+ return parent.name_spus[0], parent.relative_path
+ current_parent = parent.parent_path
+ return "", ""
+
+
+def infer_spu_from_files(dir_info: DirInfo, dirs: dict[str, DirInfo]) -> tuple[str, list[str]]:
+ token_counts: dict[str, int] = {}
+ samples: dict[str, list[str]] = {}
+ for file in collect_files_under(dir_info.full_path, dirs, set()):
+ tokens = extract_file_spu_candidates(file.file_name)
+ for token in tokens:
+ if any(token in component or component in token for component in split_combo(token)):
+ pass
+ token_counts[token] = token_counts.get(token, 0) + 1
+ samples.setdefault(token, []).append(file.relative_path)
+
+ if not token_counts:
+ return "", []
+
+ ranked = sorted(token_counts.items(), key=lambda item: (-item[1], item[0]))
+ best = ranked[0][0]
+ evidence = samples.get(best, [])[:6]
+ return best, evidence
+
+
+def has_asset_signal(dir_info: DirInfo) -> bool:
+ return any(dir_info.subtree_groups.get(group, 0) > 0 for group in ("main", "sku", "detail", "source"))
+
+
+def discover_candidates(dirs: dict[str, DirInfo], limit: int | None = None) -> list[Candidate]:
+ candidates: dict[str, Candidate] = {}
+
+ for dir_info in sorted(dirs.values(), key=lambda item: (item.root_path, item.relative_path)):
+ if not has_asset_signal(dir_info):
+ continue
+
+ if dir_info.name_spus:
+ spu = dir_info.name_spus[0]
+ candidates[dir_info.full_path] = Candidate(
+ full_path=dir_info.full_path,
+ root_path=dir_info.root_path,
+ relative_path=dir_info.relative_path,
+ spu=spu,
+ spu_source="directory_combo" if "+" in spu else "directory",
+ spu_evidence=[dir_info.name],
+ )
+ continue
+
+ ancestor_spu, ancestor_relative = find_nearest_spu_ancestor(dir_info, dirs)
+ if ancestor_spu and VERSION_HINT_RE.search(dir_info.name):
+ candidates[dir_info.full_path] = Candidate(
+ full_path=dir_info.full_path,
+ root_path=dir_info.root_path,
+ relative_path=dir_info.relative_path,
+ spu=ancestor_spu,
+ spu_source="ancestor",
+ spu_evidence=[ancestor_relative, dir_info.name],
+ )
+ continue
+
+ if not ancestor_spu and VERSION_HINT_RE.search(dir_info.name):
+ inferred_spu, evidence = infer_spu_from_files(dir_info, dirs)
+ if inferred_spu:
+ candidates[dir_info.full_path] = Candidate(
+ full_path=dir_info.full_path,
+ root_path=dir_info.root_path,
+ relative_path=dir_info.relative_path,
+ spu=inferred_spu,
+ spu_source="file_inferred",
+ spu_evidence=evidence,
+ )
+
+ ordered = list(candidates.values())
+ ordered.sort(key=lambda item: (item.root_path, item.relative_path))
+ if limit:
+ return ordered[:limit]
+ return ordered
+
+
+def collect_files_under(candidate_path: str, dirs: dict[str, DirInfo], boundary_paths: set[str]) -> list[FileEntry]:
+ result: list[FileEntry] = []
+ candidate_norm = os.path.normcase(os.path.normpath(candidate_path))
+ candidate_has_hard_skip = has_hard_skip(candidate_path)
+ boundary_norms = {
+ os.path.normcase(os.path.normpath(boundary))
+ for boundary in boundary_paths
+ if os.path.normcase(os.path.normpath(boundary)) != candidate_norm
+ }
+
+ stack = [candidate_path]
+ while stack:
+ current_path = stack.pop()
+ current_norm = os.path.normcase(os.path.normpath(current_path))
+ if current_norm != candidate_norm and current_norm in boundary_norms:
+ continue
+ if not candidate_has_hard_skip and current_norm != candidate_norm and has_hard_skip(current_path):
+ continue
+ dir_info = dirs.get(current_path)
+ if not dir_info:
+ continue
+ result.extend(dir_info.files)
+ stack.extend(reversed(dir_info.children))
+ return result
+
+
+def choose_store(root_name: str, full_path: str, spu: str) -> tuple[str, str]:
+ normalized_path = full_path.replace("\\", "/")
+ for marker, store in TRANSFER_HINTS.items():
+ if marker in normalized_path:
+ return store, marker
+ for marker, store in ROOT_STORE_HINTS.items():
+ if marker in root_name:
+ return store, f"root:{root_name}"
+ prefix = spu_prefix(spu)
+ if prefix in KNOWN_SHOP_PREFIXES:
+ return KNOWN_SHOP_PREFIXES[prefix], f"spu_prefix:{prefix}"
+ return "其他", "fallback"
+
+
+def infer_product_category(spu: str, relative_path: str) -> tuple[str, str]:
+ category_names = set(CATEGORY_CODE_MAP.values())
+ for part in re.split(r"[\\/]", relative_path):
+ cleaned = part.strip()
+ if cleaned in category_names:
+ return cleaned, f"path:{cleaned}"
+
+ primary = split_combo(spu)[0] if spu else ""
+ sku_part = primary.split("-", 1)[1] if "-" in primary else primary
+ for code in sorted(CATEGORY_CODE_MAP, key=len, reverse=True):
+ if re.match(rf"^{code}\d", sku_part, re.IGNORECASE):
+ return CATEGORY_CODE_MAP[code], f"spu_code:{code}"
+ return "", "unknown"
+
+
+def match_visual_author(relative_path: str, visual_author_names: list[str]) -> tuple[str, str]:
+ if not visual_author_names:
+ return "", ""
+
+ parts = [part for part in re.split(r"[\\/]", relative_path) if part]
+ for part in reversed(parts):
+ matches: list[tuple[int, int, str]] = []
+ for name in visual_author_names:
+ index = part.rfind(name)
+ if index >= 0:
+ matches.append((index, len(name), name))
+ if matches:
+ matches.sort(key=lambda item: (item[0], item[1]), reverse=True)
+ return matches[0][2], f"visual_user:{part}"
+ return "", ""
+
+
+def infer_author(relative_path: str, visual_author_names: list[str] | None = None) -> tuple[str, str]:
+ visual_author, visual_evidence = match_visual_author(relative_path, visual_author_names or [])
+ if visual_author:
+ return visual_author, visual_evidence
+
+ parts = [part for part in re.split(r"[\\/]", relative_path) if part]
+ for part in reversed(parts):
+ match = AUTHOR_MARK_RE.search(part)
+ if match:
+ return match.group(1), f"marker:{part}"
+
+ stop_words = {
+ "正版",
+ "盗版",
+ "正版原创",
+ "全套图",
+ "全套",
+ "视觉原图",
+ "全套视觉图",
+ "全套图(视觉原图)",
+ "主图",
+ "主图优化",
+ "功能图",
+ "详情",
+ "详情动态",
+ "详情静态",
+ "SKU",
+ "sku",
+ "美工",
+ "平面",
+ "平面全套",
+ "平面内容",
+ "3D",
+ "3D渲染原图",
+ "D渲染原图",
+ "渲染原图",
+ "AI",
+ "AI合成图",
+ "原图",
+ "源文件",
+ "psd源文件",
+ "PSD源文件",
+ "PS源文件",
+ "新品",
+ "优化",
+ "建模图",
+ "3D建模",
+ "3D原图",
+ "坐具类",
+ "书架",
+ "桌类",
+ "椅类",
+ "凳类",
+ "矮凳",
+ "架类",
+ "柜类",
+ "几类",
+ "床类",
+ "沙发类",
+ "厨房置物架",
+ "桌面收纳",
+ "升降桌",
+ "懒人沙发",
+ "客厅",
+ "卧室",
+ "书房",
+ "厨房",
+ "餐厅",
+ "方向",
+ "卧室方向",
+ "书房方向",
+ "客厅方向",
+ "图片有误",
+ "出图",
+ "精修图片",
+ "电竞",
+ "收纳",
+ "斗柜",
+ "餐椅",
+ "梳妆台",
+ "梳妆凳",
+ "实木床",
+ "折叠桌",
+ "茶室",
+ "视觉",
+ "内容",
+ "单双电机",
+ "大图",
+ "白底",
+ "白底图",
+ "转索菲亚",
+ "转顾家",
+ "转金牌",
+ "转朴美",
+ *CATEGORY_CODE_MAP.values(),
+ *ROOT_STORE_HINTS.values(),
+ }
+ descriptor_fragments = (
+ "全套图",
+ "视觉原图",
+ "全套视觉",
+ "详情",
+ "主图",
+ "SKU",
+ "sku",
+ "类",
+ "源文件",
+ "建模原图",
+ "建模",
+ "渲染",
+ "原图",
+ "出图",
+ "方向",
+ "图片有误",
+ "实拍",
+ "拍摄",
+ "摄影",
+ "优化",
+ "新品",
+ "找不到人",
+ "精修图片",
+ "电竞",
+ "收纳",
+ "斗柜",
+ "餐椅",
+ "梳妆台",
+ "梳妆凳",
+ "实木床",
+ "折叠桌",
+ "茶室",
+ "视觉",
+ "内容",
+ "单双电机",
+ "电机",
+ "谁出的图",
+ "上传的人",
+ "设计师",
+ "改款",
+ "正确",
+ "最新",
+ "客厅",
+ "卧室",
+ "书房",
+ "厨房",
+ "餐厅",
+ "平面内容",
+ "白底图",
+ "书架",
+ "桌类",
+ "椅类",
+ "凳类",
+ "矮凳",
+ "架类",
+ "柜类",
+ "几类",
+ "床类",
+ "厨房置物架",
+ "桌面收纳",
+ )
+
+ def clean_author_tokens(part: str) -> list[str]:
+ tail = part
+ for spu in extract_spu_candidates(part):
+ tail = re.sub(re.escape(spu), "", tail, flags=re.IGNORECASE)
+ tail = re.sub(COMBO_SPU_RE, "", tail)
+ tail = re.sub(BRANDED_SPU_RE, "", tail)
+ tokens = [
+ token.strip(" -_+()()0123456789.")
+ for token in re.split(r"[-_\s+]+", tail)
+ if token.strip(" -_+()()0123456789.")
+ ]
+ names: list[str] = []
+ descriptor_prefixes = ("平面", "美工", "建模", "设计", "摄影", "实拍")
+ for token in tokens:
+ cleaned = token
+ for prefix in descriptor_prefixes:
+ if cleaned.startswith(prefix) and len(cleaned) > len(prefix) + 1:
+ cleaned = cleaned[len(prefix) :]
+ cleaned = cleaned.strip(" -_+()()0123456789.")
+ break
+ for suffix in descriptor_prefixes:
+ if cleaned.endswith(suffix) and len(cleaned) > len(suffix) + 1:
+ cleaned = cleaned[: -len(suffix)]
+ cleaned = cleaned.strip(" -_+()()0123456789.")
+ break
+ if (
+ cleaned in stop_words
+ or cleaned.startswith("转")
+ or any(fragment in cleaned for fragment in descriptor_fragments)
+ or VERSION_TOKEN_RE.search(cleaned)
+ or not re.search(r"[\u4e00-\u9fff]", cleaned)
+ or len(cleaned) > 8
+ ):
+ continue
+ names.append(cleaned)
+ return names
+
+ for part in reversed(parts):
+ names = clean_author_tokens(part)
+ if names:
+ return names[-1], f"path:{part}"
+ return "历史导入", "fallback"
+
+
+def infer_task_detail_source_type(path_text: str) -> tuple[str, str]:
+ if AI_SOURCE_RE.search(path_text):
+ return "AI", "path:AI"
+ if REAL_PHOTO_SOURCE_RE.search(path_text):
+ return "实拍", "path:实拍"
+ if RENDER_SOURCE_RE.search(path_text):
+ return "渲染", "path:渲染/3D/建模"
+ return "渲染", "default"
+
+
+def is_empty_container_item(item: dict) -> bool:
+ if item["status"] != "skipped":
+ return False
+ if item["path_flags_json"] and json.loads(item["path_flags_json"]).get("has_hard_skip"):
+ return False
+ count_keys = (
+ "main_count",
+ "sku_count",
+ "detail_count",
+ "source_count",
+ "other_image_count",
+ "other_source_archive_count",
+ )
+ return all(int(item.get(key) or 0) == 0 for key in count_keys)
+
+
+def build_sku_code(spu: str, file_name: str, index: int) -> str:
+ normalized_name = normalize_token_text(file_name)
+ file_code_match = SKU_FILE_CODE_RE.search(normalized_name)
+ if file_code_match:
+ return normalize_spu(file_code_match.group(1))
+
+ escaped_spu = re.escape(normalize_spu(spu))
+ match = re.search(rf"({escaped_spu}\d{{2,3}})", normalized_name, re.IGNORECASE)
+ if match:
+ return normalize_spu(match.group(1))
+ return f"{normalize_spu(spu)}{str(index + 1).zfill(2)}"
+
+
+def is_compatible_spu_token(candidate_spu: str, token: str) -> bool:
+ candidate_components = split_combo(candidate_spu)
+ token_normalized = normalize_spu(token)
+ token_prefix = spu_prefix(token_normalized)
+ token_body = spu_body(token_normalized)
+
+ for component in candidate_components:
+ component_normalized = normalize_spu(component)
+ if token_normalized == component_normalized:
+ return True
+
+ component_prefix = spu_prefix(component_normalized)
+ component_body = spu_body(component_normalized)
+ if token_body != component_body:
+ continue
+
+ if not token_prefix or not component_prefix:
+ return True
+
+ if KNOWN_SHOP_PREFIXES.get(token_prefix) == KNOWN_SHOP_PREFIXES.get(component_prefix):
+ return True
+
+ return False
+
+
+def detect_asset_spu_conflicts(spu: str, grouped: dict[str, list[FileEntry]]) -> list[dict[str, str]]:
+ conflicts: list[dict[str, str]] = []
+ for group in ("main", "sku", "detail"):
+ for file in grouped.get(group, []):
+ tokens = extract_file_spu_candidates(file.file_name)
+ if not tokens:
+ continue
+ if any(is_compatible_spu_token(spu, token) for token in tokens):
+ continue
+ conflicts.append(
+ {
+ "group": group,
+ "file": file.relative_path,
+ "tokens": ",".join(tokens),
+ }
+ )
+ if len(conflicts) >= 12:
+ return conflicts
+ return conflicts
+
+
+def blocking_asset_spu_conflicts(conflicts: list[dict[str, str]]) -> list[dict[str, str]]:
+ # 业务上 SKU 可能天然混入其它 SPU 的组合/配套 SKU;只记录证据,不阻断 ready。
+ return [item for item in conflicts if item.get("group") in {"main", "detail"}]
+
+
+def build_item(
+ candidate: Candidate,
+ dirs: dict[str, DirInfo],
+ boundary_paths: set[str],
+ visual_author_names: list[str],
+) -> tuple[dict, dict[str, list[FileEntry]]]:
+ dir_info = dirs[candidate.full_path]
+ files = collect_files_under(candidate.full_path, dirs, boundary_paths)
+ grouped: dict[str, list[FileEntry]] = {
+ "main": [],
+ "sku": [],
+ "detail": [],
+ "source": [],
+ "other_image": [],
+ "other_source_archive": [],
+ }
+
+ for file in files:
+ grouped.setdefault(file.group_hint, []).append(file)
+
+ for group in grouped:
+ grouped[group].sort(key=lambda item: item.relative_path.lower())
+
+ for index, file in enumerate(grouped.get("sku", [])):
+ file.sku_code = build_sku_code(candidate.spu, file.file_name, index)
+
+ root_name = Path(candidate.root_path).name
+ store, store_evidence = choose_store(root_name, candidate.full_path, candidate.spu)
+ product_category, category_evidence = infer_product_category(candidate.spu, candidate.relative_path)
+ author, author_evidence = infer_author(candidate.relative_path, visual_author_names)
+ task_detail_source_type, task_detail_source_type_evidence = infer_task_detail_source_type(candidate.relative_path)
+ path_text = candidate.full_path
+ has_skip = has_hard_skip(path_text)
+ has_modeling_source = has_modeling_source_segment(path_text)
+ main_count = len(grouped.get("main", []))
+ sku_count = len(grouped.get("sku", []))
+ detail_count = len(grouped.get("detail", []))
+ source_count = len(grouped.get("source", []))
+ asset_count = main_count + sku_count + detail_count + source_count
+ asset_spu_conflicts = detect_asset_spu_conflicts(candidate.spu, grouped)
+ blocking_spu_conflicts = blocking_asset_spu_conflicts(asset_spu_conflicts)
+
+ if has_skip:
+ status = "skipped"
+ status_reason = "路径包含错误勿用/废稿/勿用等跳过标识"
+ elif has_unknown_branded_prefix(candidate.spu):
+ status = "needs_review"
+ status_reason = f"SPU 前缀 {spu_prefix(candidate.spu)} 不在已知店铺前缀表,需人工确认"
+ elif candidate.spu_source == "file_inferred":
+ status = "needs_review"
+ status_reason = "目录名不是标准 SPU,仅能从文件名反推,需人工确认"
+ elif asset_count == 0:
+ status = "skipped"
+ status_reason = "未发现可导入素材"
+ elif has_modeling_source:
+ status = "needs_review"
+ status_reason = "路径属于 3D建模/建模图/原图,按 3Dmax 渲染原图处理,不自动作为任务-详情导入"
+ elif blocking_spu_conflicts:
+ status = "needs_review"
+ conflict_tokens = sorted({item["tokens"] for item in blocking_spu_conflicts})[:4]
+ status_reason = f"主图/详情文件名出现与工作区 SPU 不一致的型号:{';'.join(conflict_tokens)},需人工确认"
+ elif main_count > 0 and sku_count > 0 and detail_count > 0:
+ status = "ready"
+ status_reason = "主图、SKU 图、详情图均存在"
+ elif asset_count > 0:
+ status = "needs_review"
+ missing = [
+ label
+ for label, count in (("主图", main_count), ("SKU图", sku_count), ("详情图", detail_count))
+ if count == 0
+ ]
+ status_reason = f"缺少{','.join(missing)},不能自动导入"
+
+ components = split_combo(candidate.spu)
+ fingerprint = fingerprint_for(candidate.root_path, candidate.full_path, candidate.spu)
+ flags = {
+ "has正版": "正版" in path_text,
+ "has盗版": "盗版" in path_text,
+ "has_transfer": any(marker in path_text for marker in TRANSFER_HINTS),
+ "has_hard_skip": has_skip,
+ "has_modeling_source_segment": has_modeling_source,
+ "asset_spu_conflicts": asset_spu_conflicts,
+ "is_combo_spu": "+" in candidate.spu,
+ }
+
+ item = {
+ "fingerprint": fingerprint,
+ "source_root": candidate.root_path,
+ "source_root_name": root_name,
+ "source_batch": "",
+ "workspace_path": candidate.full_path,
+ "relative_workspace_path": candidate.relative_path,
+ "spu": candidate.spu,
+ "spu_components_json": json_dumps(components),
+ "spu_source": candidate.spu_source,
+ "spu_evidence_json": json_dumps(candidate.spu_evidence),
+ "store": store,
+ "store_evidence": store_evidence,
+ "product_category": product_category,
+ "product_category_evidence": category_evidence,
+ "author": author,
+ "author_evidence": author_evidence,
+ "task_detail_source_type": task_detail_source_type,
+ "task_detail_source_type_evidence": task_detail_source_type_evidence,
+ "version_label": dir_info.name,
+ "path_flags_json": json_dumps(flags),
+ "status": status,
+ "status_reason": status_reason,
+ "main_count": main_count,
+ "sku_count": sku_count,
+ "detail_count": detail_count,
+ "source_count": source_count,
+ "other_image_count": len(grouped.get("other_image", [])),
+ "other_source_archive_count": len(grouped.get("other_source_archive", [])),
+ }
+ return item, grouped
+
+
+def ensure_schema(conn: sqlite3.Connection) -> None:
+ conn.executescript(
+ """
+ CREATE TABLE IF NOT EXISTS spu_ledger_runs (
+ run_id TEXT PRIMARY KEY,
+ source_batch TEXT NOT NULL,
+ roots_json TEXT NOT NULL,
+ started_at TEXT NOT NULL,
+ finished_at TEXT,
+ total_items INTEGER NOT NULL DEFAULT 0
+ );
+
+ CREATE TABLE IF NOT EXISTS spu_ledger_items (
+ id INTEGER PRIMARY KEY AUTOINCREMENT,
+ run_id TEXT NOT NULL,
+ fingerprint TEXT NOT NULL,
+ source_batch TEXT NOT NULL,
+ source_root TEXT NOT NULL,
+ source_root_name TEXT NOT NULL,
+ workspace_path TEXT NOT NULL,
+ relative_workspace_path TEXT NOT NULL,
+ spu TEXT NOT NULL,
+ spu_components_json TEXT NOT NULL,
+ spu_source TEXT NOT NULL,
+ spu_evidence_json TEXT NOT NULL,
+ store TEXT NOT NULL,
+ store_evidence TEXT NOT NULL,
+ product_category TEXT NOT NULL,
+ product_category_evidence TEXT NOT NULL,
+ author TEXT NOT NULL,
+ author_evidence TEXT NOT NULL,
+ task_detail_source_type TEXT NOT NULL DEFAULT '渲染',
+ task_detail_source_type_evidence TEXT NOT NULL DEFAULT 'default',
+ version_label TEXT NOT NULL,
+ path_flags_json TEXT NOT NULL,
+ status TEXT NOT NULL,
+ status_reason TEXT NOT NULL,
+ main_count INTEGER NOT NULL DEFAULT 0,
+ sku_count INTEGER NOT NULL DEFAULT 0,
+ detail_count INTEGER NOT NULL DEFAULT 0,
+ source_count INTEGER NOT NULL DEFAULT 0,
+ other_image_count INTEGER NOT NULL DEFAULT 0,
+ other_source_archive_count INTEGER NOT NULL DEFAULT 0,
+ created_at TEXT NOT NULL,
+ updated_at TEXT NOT NULL,
+ FOREIGN KEY (run_id) REFERENCES spu_ledger_runs(run_id)
+ );
+
+ CREATE UNIQUE INDEX IF NOT EXISTS idx_spu_ledger_items_run_fingerprint
+ ON spu_ledger_items(run_id, fingerprint);
+
+ CREATE INDEX IF NOT EXISTS idx_spu_ledger_items_status
+ ON spu_ledger_items(status);
+
+ CREATE INDEX IF NOT EXISTS idx_spu_ledger_items_spu
+ ON spu_ledger_items(spu);
+
+ CREATE TABLE IF NOT EXISTS spu_ledger_files (
+ id INTEGER PRIMARY KEY AUTOINCREMENT,
+ item_id INTEGER NOT NULL,
+ asset_group TEXT NOT NULL,
+ file_path TEXT NOT NULL,
+ relative_file_path TEXT NOT NULL,
+ file_name TEXT NOT NULL,
+ file_size INTEGER NOT NULL DEFAULT 0,
+ extension TEXT NOT NULL,
+ sort_order INTEGER NOT NULL DEFAULT 0,
+ sku_code TEXT,
+ FOREIGN KEY (item_id) REFERENCES spu_ledger_items(id) ON DELETE CASCADE
+ );
+
+ CREATE INDEX IF NOT EXISTS idx_spu_ledger_files_item_group
+ ON spu_ledger_files(item_id, asset_group);
+ """
+ )
+
+
+def reset_database(conn: sqlite3.Connection) -> None:
+ conn.executescript(
+ """
+ DROP TABLE IF EXISTS spu_ledger_files;
+ DROP TABLE IF EXISTS spu_ledger_items;
+ DROP TABLE IF EXISTS spu_ledger_runs;
+ """
+ )
+ ensure_schema(conn)
+
+
+def insert_item(conn: sqlite3.Connection, run_id: str, source_batch: str, item: dict, grouped: dict[str, list[FileEntry]]) -> None:
+ now = utc_now()
+ columns = [
+ "run_id",
+ "fingerprint",
+ "source_batch",
+ "source_root",
+ "source_root_name",
+ "workspace_path",
+ "relative_workspace_path",
+ "spu",
+ "spu_components_json",
+ "spu_source",
+ "spu_evidence_json",
+ "store",
+ "store_evidence",
+ "product_category",
+ "product_category_evidence",
+ "author",
+ "author_evidence",
+ "task_detail_source_type",
+ "task_detail_source_type_evidence",
+ "version_label",
+ "path_flags_json",
+ "status",
+ "status_reason",
+ "main_count",
+ "sku_count",
+ "detail_count",
+ "source_count",
+ "other_image_count",
+ "other_source_archive_count",
+ "created_at",
+ "updated_at",
+ ]
+ payload = {
+ **item,
+ "run_id": run_id,
+ "source_batch": source_batch,
+ "created_at": now,
+ "updated_at": now,
+ }
+ placeholders = ",".join("?" for _ in columns)
+ conn.execute(
+ f"INSERT INTO spu_ledger_items ({','.join(columns)}) VALUES ({placeholders})",
+ [payload[column] for column in columns],
+ )
+ item_id = int(conn.execute("SELECT last_insert_rowid()").fetchone()[0])
+
+ file_rows = []
+ for group in ("main", "sku", "detail", "source", "other_image", "other_source_archive"):
+ for index, file in enumerate(grouped.get(group, [])):
+ file_rows.append(
+ (
+ item_id,
+ group,
+ file.full_path,
+ file.relative_path,
+ file.file_name,
+ file.file_size,
+ file.extension,
+ index,
+ file.sku_code or None,
+ )
+ )
+ if file_rows:
+ conn.executemany(
+ """
+ INSERT INTO spu_ledger_files (
+ item_id, asset_group, file_path, relative_file_path, file_name,
+ file_size, extension, sort_order, sku_code
+ )
+ VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
+ """,
+ file_rows,
+ )
+
+
+def parse_args() -> argparse.Namespace:
+ parser = argparse.ArgumentParser(description="Scan SPU-centered material ledger.")
+ parser.add_argument("--roots", required=True, help="Root paths separated by |")
+ parser.add_argument("--db-path", default=str(Path.cwd() / "output" / "spu-ledger.db"))
+ parser.add_argument("--source-batch", default="tianlan-z")
+ parser.add_argument("--max-depth", type=int, default=14)
+ parser.add_argument("--limit", type=int, default=0, help="Limit candidate items after discovery")
+ parser.add_argument("--visual-authors-file", default="", help="Optional TSV/CSV/JSON names from 视觉中心 users")
+ parser.add_argument("--reset", action="store_true", help="Drop and recreate ledger tables before scanning")
+ return parser.parse_args()
+
+
+def main() -> int:
+ args = parse_args()
+ roots = [Path(item.strip()) for item in args.roots.split("|") if item.strip()]
+ if not roots:
+ raise SystemExit("缺少有效 --roots")
+
+ db_path = Path(args.db_path)
+ db_path.parent.mkdir(parents=True, exist_ok=True)
+ conn = sqlite3.connect(str(db_path))
+ conn.execute("PRAGMA foreign_keys = ON")
+ if args.reset:
+ reset_database(conn)
+ else:
+ ensure_schema(conn)
+
+ run_id = str(uuid.uuid4())
+ started_at = utc_now()
+ conn.execute(
+ "INSERT INTO spu_ledger_runs (run_id, source_batch, roots_json, started_at) VALUES (?, ?, ?, ?)",
+ (run_id, args.source_batch, json_dumps([str(root) for root in roots]), started_at),
+ )
+
+ total_items = 0
+ visual_author_names = load_visual_author_names(args.visual_authors_file) if args.visual_authors_file else []
+ if visual_author_names:
+ print(f"[authors] visual_author_names={len(visual_author_names)}")
+ try:
+ for root in roots:
+ if not root.exists():
+ print(f"[WARN] root missing: {root}", file=sys.stderr)
+ continue
+ print(f"[scan] {root}")
+ dirs = walk_tree(root, args.max_depth)
+ candidates = discover_candidates(dirs, args.limit if args.limit > 0 else None)
+ boundary_paths = {candidate.full_path for candidate in candidates}
+ print(f"[scan] candidates={len(candidates)}")
+ for candidate in candidates:
+ item, grouped = build_item(candidate, dirs, boundary_paths, visual_author_names)
+ if is_empty_container_item(item):
+ continue
+ insert_item(conn, run_id, args.source_batch, item, grouped)
+ total_items += 1
+ conn.execute(
+ "UPDATE spu_ledger_runs SET finished_at = ?, total_items = ? WHERE run_id = ?",
+ (utc_now(), total_items, run_id),
+ )
+ conn.commit()
+ except Exception:
+ conn.rollback()
+ raise
+ finally:
+ conn.close()
+
+ print("SPU 台账扫描完成")
+ print(f"- run_id: {run_id}")
+ print(f"- total_items: {total_items}")
+ print(f"- db_path: {db_path}")
+ return 0
+
+
+if __name__ == "__main__":
+ raise SystemExit(main())
diff --git a/scripts/validate-history-import-result.ts b/scripts/validate-history-import-result.ts
new file mode 100644
index 0000000..868cf99
--- /dev/null
+++ b/scripts/validate-history-import-result.ts
@@ -0,0 +1,274 @@
+import { execFile } from 'child_process'
+import { promises as fs } from 'fs'
+import path from 'path'
+import { promisify } from 'util'
+import { PrismaClient } from '@prisma/client'
+
+const execFileAsync = promisify(execFile)
+const SSH_TIMEOUT_MS = Number(process.env.HISTORY_IMPORT_VALIDATE_SSH_TIMEOUT_MS || 30000)
+const SSH_RETRY_COUNT = Number(process.env.HISTORY_IMPORT_VALIDATE_SSH_RETRY_COUNT || 3)
+const SSH_ARGS = [
+ '-o',
+ 'BatchMode=yes',
+ '-o',
+ 'ConnectTimeout=15',
+ '-o',
+ 'ServerAliveInterval=15',
+ '-o',
+ 'ServerAliveCountMax=2',
+]
+
+type ImportResultItem = {
+ fingerprint: string
+ spu: string
+ sourcePath: string
+ materialId: string | null
+ status: string
+ reason: string | null
+}
+
+function parseArgs() {
+ const args = process.argv.slice(2)
+ const getArg = (name: string) => {
+ const match = args.find((arg) => arg.startsWith(`--${name}=`))
+ return match ? match.slice(name.length + 3) : ''
+ }
+
+ const input = getArg('input')
+ const remoteHost = getArg('remote-host') || process.env.HISTORY_IMPORT_REMOTE_UPLOADS_HOST || 'prod'
+ const remoteUploadsDir =
+ getArg('remote-uploads-dir') ||
+ process.env.HISTORY_IMPORT_REMOTE_UPLOADS_DIR ||
+ '/mnt/PoolOne/apps/youyidian-3d/uploads'
+
+ if (!input) {
+ throw new Error('缺少 --input=导入结果 JSON')
+ }
+
+ return { input, remoteHost, remoteUploadsDir }
+}
+
+function shellQuote(value: string) {
+ return `'${value.replace(/'/g, `'\\''`)}'`
+}
+
+function storedPathToRemotePath(remoteUploadsDir: string, storedPath: string) {
+ const normalized = String(storedPath || '').replace(/\\/g, '/')
+ const relative = normalized.replace(/^\.?\/*uploads\//i, '')
+ if (!relative || relative.includes('../')) {
+ throw new Error(`下载文件路径无效: ${storedPath}`)
+ }
+ return path.posix.join(remoteUploadsDir.replace(/\/+$/, ''), relative)
+}
+
+function sleep(ms: number) {
+ return new Promise((resolve) => setTimeout(resolve, ms))
+}
+
+async function getRemoteFileSize(remoteHost: string, remotePath: string) {
+ let lastError: unknown
+ for (let attempt = 1; attempt <= SSH_RETRY_COUNT; attempt += 1) {
+ try {
+ const { stdout } = await execFileAsync(
+ 'ssh',
+ [
+ ...SSH_ARGS,
+ remoteHost,
+ `test -f ${shellQuote(remotePath)} && stat -c %s ${shellQuote(remotePath)}`,
+ ],
+ { timeout: SSH_TIMEOUT_MS },
+ )
+ return Number(String(stdout || '').trim())
+ } catch (error) {
+ lastError = error
+ if (attempt < SSH_RETRY_COUNT) {
+ await sleep(1000 * attempt)
+ }
+ }
+ }
+
+ throw lastError
+}
+
+async function main() {
+ const options = parseArgs()
+ const prisma = new PrismaClient()
+
+ try {
+ const raw = await fs.readFile(options.input, 'utf8')
+ const results = JSON.parse(raw) as ImportResultItem[]
+ const imported = results.filter((item) => item.status === 'imported' && item.materialId)
+ const failures = results.filter((item) => item.status === 'failed')
+ const skipped = results.filter((item) => item.status === 'skipped')
+ const ids = imported.map((item) => item.materialId!).filter(Boolean)
+
+ const rows = await prisma.material.findMany({
+ where: { id: { in: ids } },
+ select: {
+ id: true,
+ productModel: true,
+ previewUrl: true,
+ zipFilePath: true,
+ zipFileSize: true,
+ isFullSet: true,
+ deletedAt: true,
+ },
+ })
+ const rowById = new Map(rows.map((row) => [row.id, row]))
+ const sourceAssets = await prisma.materialAsset.findMany({
+ where: {
+ materialId: { in: ids },
+ assetType: 'psd',
+ },
+ select: {
+ materialId: true,
+ fileUrl: true,
+ fileName: true,
+ fileSize: true,
+ },
+ })
+ const sourceAssetsByMaterialId = new Map()
+ for (const asset of sourceAssets) {
+ const current = sourceAssetsByMaterialId.get(asset.materialId) || []
+ current.push(asset)
+ sourceAssetsByMaterialId.set(asset.materialId, current)
+ }
+ const checks = []
+
+ for (const item of imported) {
+ const row = rowById.get(item.materialId!)
+ if (!row) {
+ checks.push({
+ materialId: item.materialId,
+ spu: item.spu,
+ ok: false,
+ issue: '数据库记录不存在',
+ })
+ continue
+ }
+
+ const remotePath = storedPathToRemotePath(options.remoteUploadsDir, row.zipFilePath || '')
+ let remoteSize = -1
+ try {
+ remoteSize = await getRemoteFileSize(options.remoteHost, remotePath)
+ } catch {
+ remoteSize = -1
+ }
+
+ let downloadStatus = 0
+ try {
+ const response = await fetch(
+ `https://sc.jiansun.vip/api/materials/download?id=${row.id}&type=all&validate=1&share=1`,
+ )
+ downloadStatus = response.status
+ } catch {
+ downloadStatus = -1
+ }
+
+ let previewStatus = 0
+ try {
+ const response = await fetch(row.previewUrl || '', { method: 'HEAD' })
+ previewStatus = response.status
+ } catch {
+ previewStatus = -1
+ }
+
+ const expectedSize = Number(row.zipFileSize || 0)
+ const materialSourceAssets = sourceAssetsByMaterialId.get(row.id) || []
+ const sourceChecks = []
+ for (const asset of materialSourceAssets) {
+ const sourcePathOk =
+ String(asset.fileUrl || '').startsWith('uploads/materials/') &&
+ !String(asset.fileUrl || '').includes('\\')
+ const expectedSourceSize = Number(asset.fileSize || 0)
+ let remoteSourceSize = -1
+
+ if (sourcePathOk) {
+ try {
+ remoteSourceSize = await getRemoteFileSize(
+ options.remoteHost,
+ storedPathToRemotePath(options.remoteUploadsDir, asset.fileUrl || ''),
+ )
+ } catch {
+ remoteSourceSize = -1
+ }
+ }
+
+ sourceChecks.push({
+ fileName: asset.fileName,
+ sourcePathOk,
+ remoteSize: remoteSourceSize,
+ expectedSize: expectedSourceSize,
+ ok: sourcePathOk && remoteSourceSize === expectedSourceSize,
+ })
+ }
+ const sourceAssetsOk = sourceChecks.every((asset) => asset.ok)
+ const previewOk = String(row.previewUrl || '').startsWith(
+ 'https://s3.jiansun.vip/gongxiangpan/由壹点AI/public-display/',
+ )
+ const zipPathOk =
+ String(row.zipFilePath || '').startsWith('uploads/materials/') &&
+ !String(row.zipFilePath || '').includes('\\')
+ const ok =
+ !row.deletedAt &&
+ row.isFullSet &&
+ previewOk &&
+ zipPathOk &&
+ sourceAssetsOk &&
+ remoteSize === expectedSize &&
+ downloadStatus === 200 &&
+ previewStatus === 200
+
+ checks.push({
+ materialId: row.id,
+ spu: row.productModel,
+ ok,
+ previewOk,
+ zipPathOk,
+ sourceAssetCount: materialSourceAssets.length,
+ sourceAssetsOk,
+ sourceIssues: sourceChecks.filter((asset) => !asset.ok),
+ remoteSize,
+ expectedSize,
+ downloadStatus,
+ previewStatus,
+ })
+ }
+
+ const summary = {
+ input: options.input,
+ total: results.length,
+ imported: imported.length,
+ skipped: skipped.length,
+ failed: failures.length,
+ checked: checks.length,
+ ok: checks.filter((item) => item.ok).length,
+ issueCount: checks.filter((item) => !item.ok).length + failures.length,
+ issues: [
+ ...failures.map((item) => ({
+ materialId: item.materialId,
+ spu: item.spu,
+ ok: false,
+ issue: item.reason || '导入失败',
+ })),
+ ...checks.filter((item) => !item.ok),
+ ],
+ checkedAt: new Date().toISOString(),
+ }
+
+ const outputPath = options.input.replace(/\.json$/i, '.verification.json')
+ await fs.writeFile(outputPath, JSON.stringify(summary, null, 2), 'utf8')
+ console.log(JSON.stringify(summary, null, 2))
+
+ if (summary.failed > 0 || summary.issueCount > 0 || summary.imported + summary.skipped !== summary.total) {
+ process.exitCode = 1
+ }
+ } finally {
+ await prisma.$disconnect()
+ }
+}
+
+main().catch((error) => {
+ console.error('导入结果验收失败:', error)
+ process.exitCode = 1
+})