feat: 任务详情无条码开关 + 上传面板信息密度优化 + 历史导入/脚本补全

- 任务详情上传支持"本批素材无条码"开关,跳过条码必填校验
- SKU卡片/主图网格从4列改5列,图片高度压缩,去掉冗余标签和提示
- 详情图/PSD上传区去掉大占位框,改成紧凑按钮+标签
- 补充历史导入相关脚本和工具文件

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
chenpei 2026-05-11 10:01:08 +08:00
parent 340f688b7b
commit 9cd3c0f231
26 changed files with 4819 additions and 244 deletions

View File

@ -22,6 +22,11 @@ import {
} from '@/lib/download-errors';
import { isProductPackageType } from '@/lib/material-type-mapper';
import { ensureMaterialDownloadLogsTable } from '@/lib/material-download-log';
import { resolveStoredUploadPath } from '@/lib/upload-paths';
function normalizeStoredLocalPath(filePath: string) {
return String(filePath || '').replace(/\\/g, '/');
}
// GET - 下载素材
export async function GET(request: NextRequest) {
@ -156,10 +161,8 @@ export async function GET(request: NextRequest) {
);
}
let filePath = material.zipFilePath;
if (!path.isAbsolute(filePath)) {
filePath = path.join(process.cwd(), filePath);
}
let filePath = normalizeStoredLocalPath(material.zipFilePath);
filePath = path.isAbsolute(filePath) ? filePath : resolveStoredUploadPath(filePath);
return serveFile(filePath, material, downloadType || '源文件', validateOnly);
} catch (error) {
@ -201,7 +204,10 @@ async function handleProductPackageDownload(
where: {
materialId: material.id,
assetType,
fileUrl: { startsWith: 'uploads/' }, // 只查询本地 zip 文件
OR: [
{ fileUrl: { startsWith: 'uploads/' } },
{ fileUrl: { startsWith: 'uploads\\' } },
],
},
});
}
@ -214,10 +220,10 @@ async function handleProductPackageDownload(
);
}
const filePath = asset.fileUrl;
const absolutePath = path.isAbsolute(filePath)
? filePath
: path.join(process.cwd(), filePath);
const filePath = normalizeStoredLocalPath(asset.fileUrl);
const absolutePath = path.isAbsolute(filePath)
? filePath
: resolveStoredUploadPath(filePath);
console.log(`[Download] 文件路径: ${absolutePath}`);
console.log(`[Download] 文件名: ${asset.fileName}`);

View File

@ -33,6 +33,7 @@ interface PsdAsset {
interface CompleteRequestBody {
materialId: string
taskNo?: string
noBarcodes?: boolean
assets: {
main?: AssetUpload[]
sku?: AssetUpload[]
@ -79,6 +80,9 @@ export async function POST(request: NextRequest) {
taskNo: body.taskNo,
assets: body.assets,
operatorUserId: (session.user as any).id,
options: {
allowSkuWithoutBarcodes: body.noBarcodes === true,
},
})
return NextResponse.json({

View File

@ -114,6 +114,7 @@ export default function ProductPackageUploader({
const [isPsdDragging, setIsPsdDragging] = useState(false);
// 上传状态
const [noBarcodes, setNoBarcodes] = useState(false);
const [uploading, setUploading] = useState(false);
const [uploadProgress, setUploadProgress] = useState(0);
const [processingStep, setProcessingStep] = useState('');
@ -406,7 +407,7 @@ export default function ProductPackageUploader({
}
const barcodes = parseBarcodeInput(image.barcodeInput || '');
if (barcodes.length === 0) {
if (!noBarcodes && barcodes.length === 0) {
throw new Error(`请为 SKU 图 ${image.file.name} 至少填写 1 个产品条码`);
}
@ -560,6 +561,7 @@ export default function ProductPackageUploader({
body: JSON.stringify({
materialId,
taskNo, // 传递taskNo,上传完成后写入数据库
noBarcodes,
assets: {
main: mainAssets,
sku: skuAssets,
@ -698,9 +700,9 @@ export default function ProductPackageUploader({
</select>
</div>
<div className="rounded-apple border border-blue-200 bg-blue-50 px-4 py-3 text-sm text-blue-700 dark:border-blue-800 dark:bg-blue-900/20 dark:text-blue-200">
系统会按素材实际内容自动判定是否整套:同时拥有主图、SKU 图、详情图时,自动标记为整套,无需手工选择。
</div>
<p className="text-xs text-gray-400 dark:text-gray-500">
同时上传主图 + SKU 图 + 详情图时自动标记为整套
</p>
{/* 主图上传 */}
<div>
@ -729,15 +731,15 @@ export default function ProductPackageUploader({
className="hidden"
disabled={uploading}
/>
<div className="grid grid-cols-4 gap-4">
<div className="grid grid-cols-5 gap-3">
{mainImages.map((img) => (
<div key={img.id} className="relative group">
<div className="relative w-full h-32 rounded-apple overflow-hidden border-2 border-gray-200 dark:border-gray-600">
<div className="relative w-full h-24 rounded-apple overflow-hidden border-2 border-gray-200 dark:border-gray-600">
<img src={img.preview} alt="" className="w-full h-full object-cover" />
</div>
<button
onClick={() => handleRemoveImage(img.id, setMainImages)}
className="absolute -top-2 -right-2 bg-red-500 text-white rounded-full w-6 h-6 flex items-center justify-center opacity-0 group-hover:opacity-100 transition-opacity"
className="absolute -top-1.5 -right-1.5 bg-red-500 text-white rounded-full w-5 h-5 flex items-center justify-center text-xs opacity-0 group-hover:opacity-100 transition-opacity"
disabled={uploading}
type="button"
>
@ -749,25 +751,34 @@ export default function ProductPackageUploader({
type="button"
onClick={() => mainInputRef.current?.click()}
disabled={uploading}
className="group w-full h-32 rounded-apple bg-gray-100 dark:bg-gray-700 border-2 border-dashed border-gray-300 dark:border-gray-600 flex items-center justify-center hover:border-gray-400 dark:hover:border-gray-500 transition-colors disabled:opacity-50"
className="group w-full h-24 rounded-apple bg-gray-100 dark:bg-gray-700 border-2 border-dashed border-gray-300 dark:border-gray-600 flex items-center justify-center hover:border-gray-400 dark:hover:border-gray-500 transition-colors disabled:opacity-50"
>
<svg className="w-12 h-12 text-gray-400 group-hover:text-gray-500" fill="none" stroke="currentColor" viewBox="0 0 24 24">
<svg className="w-8 h-8 text-gray-400 group-hover:text-gray-500" fill="none" stroke="currentColor" viewBox="0 0 24 24">
<title>添加主图</title>
<path strokeLinecap="round" strokeLinejoin="round" strokeWidth={2} d="M12 4v16m8-8H4" />
</svg>
</button>
</div>
<p className="text-xs text-gray-500 dark:text-gray-400 mt-3">
支持 JPG、PNG、WebP 等格式,可一次选择多张或拖拽上传。每张 SKU 图都要填写 SKU 编号,并录入至少 1 个产品条码。
</p>
</div>
</div>
{/* SKU图上传 */}
<div>
<label htmlFor="sku-images-input" className="block text-sm font-medium text-gray-700 dark:text-gray-300 mb-2">
SKU图
</label>
<div className="flex items-center justify-between mb-2">
<label htmlFor="sku-images-input" className="block text-sm font-medium text-gray-700 dark:text-gray-300">
SKU图
</label>
<label className="flex items-center gap-2 text-sm text-gray-600 dark:text-gray-400 cursor-pointer select-none">
<input
type="checkbox"
checked={noBarcodes}
onChange={(e) => setNoBarcodes(e.target.checked)}
disabled={uploading}
className="rounded border-gray-300 text-blue-600 focus:ring-blue-500 dark:border-gray-600"
/>
本批素材无条码
</label>
</div>
{/* biome-ignore lint/a11y/noStaticElementInteractions: 拖拽上传区域需要使用 drag/drop 事件 */}
<div
className={`border-2 border-dashed rounded-apple p-4 transition-colors ${
@ -790,65 +801,52 @@ export default function ProductPackageUploader({
className="hidden"
disabled={uploading}
/>
<div className="grid grid-cols-4 gap-4">
<div className="grid grid-cols-5 gap-3">
{skuImages.map((img) => (
<div key={img.id} className="relative group space-y-3 rounded-apple border border-gray-200 bg-white p-3 dark:border-gray-700 dark:bg-gray-800/80">
<div className="relative w-full h-32 rounded-apple overflow-hidden border-2 border-gray-200 dark:border-gray-600">
<div key={img.id} className="relative group space-y-1.5 rounded-apple border border-gray-200 bg-white p-2 dark:border-gray-700 dark:bg-gray-800/80">
<div className="relative w-full h-20 rounded-apple overflow-hidden border border-gray-200 dark:border-gray-600">
<img src={img.preview} alt="" className="w-full h-full object-cover" />
</div>
<button
onClick={() => handleRemoveImage(img.id, setSkuImages)}
className="absolute -top-2 -right-2 bg-red-500 text-white rounded-full w-6 h-6 flex items-center justify-center opacity-0 group-hover:opacity-100 transition-opacity"
className="absolute -top-1.5 -right-1.5 bg-red-500 text-white rounded-full w-5 h-5 flex items-center justify-center text-xs opacity-0 group-hover:opacity-100 transition-opacity"
disabled={uploading}
type="button"
>
×
</button>
<div className="space-y-2">
<div>
<label className="mb-1 block text-xs font-medium text-gray-600 dark:text-gray-300">
SKU 编号
</label>
<input
type="text"
value={img.skuCode || ''}
onChange={(e) => updateSkuImageMeta(img.id, { skuCode: e.target.value })}
placeholder="例如 SKU001"
disabled={uploading}
className="w-full rounded-apple border border-gray-300 px-3 py-2 text-sm dark:border-gray-600 dark:bg-gray-700 dark:text-white"
/>
</div>
<div>
<label className="mb-1 block text-xs font-medium text-gray-600 dark:text-gray-300">
产品条码
</label>
<textarea
value={img.barcodeInput || ''}
onChange={(e) => updateSkuImageMeta(img.id, { barcodeInput: e.target.value })}
placeholder="每行一个,或用逗号分隔"
disabled={uploading}
rows={4}
className="w-full rounded-apple border border-gray-300 px-3 py-2 text-sm dark:border-gray-600 dark:bg-gray-700 dark:text-white"
/>
</div>
</div>
<input
type="text"
value={img.skuCode || ''}
onChange={(e) => updateSkuImageMeta(img.id, { skuCode: e.target.value })}
placeholder="SKU 编号"
disabled={uploading}
className="w-full rounded border border-gray-300 px-2 py-1 text-xs dark:border-gray-600 dark:bg-gray-700 dark:text-white"
/>
{!noBarcodes && (
<textarea
value={img.barcodeInput || ''}
onChange={(e) => updateSkuImageMeta(img.id, { barcodeInput: e.target.value })}
placeholder="产品条码(逗号分隔)"
disabled={uploading}
rows={2}
className="w-full rounded border border-gray-300 px-2 py-1 text-xs dark:border-gray-600 dark:bg-gray-700 dark:text-white"
/>
)}
</div>
))}
<button
type="button"
onClick={() => skuInputRef.current?.click()}
disabled={uploading}
className="group w-full h-32 rounded-apple bg-gray-100 dark:bg-gray-700 border-2 border-dashed border-gray-300 dark:border-gray-600 flex items-center justify-center hover:border-gray-400 dark:hover:border-gray-500 transition-colors disabled:opacity-50"
className="group w-full h-20 rounded-apple bg-gray-100 dark:bg-gray-700 border-2 border-dashed border-gray-300 dark:border-gray-600 flex items-center justify-center hover:border-gray-400 dark:hover:border-gray-500 transition-colors disabled:opacity-50"
>
<svg className="w-12 h-12 text-gray-400 group-hover:text-gray-500" fill="none" stroke="currentColor" viewBox="0 0 24 24">
<svg className="w-8 h-8 text-gray-400 group-hover:text-gray-500" fill="none" stroke="currentColor" viewBox="0 0 24 24">
<title>添加SKU图</title>
<path strokeLinecap="round" strokeLinejoin="round" strokeWidth={2} d="M12 4v16m8-8H4" />
</svg>
</button>
</div>
<p className="text-xs text-gray-500 dark:text-gray-400 mt-3">
支持 JPG、PNG、WebP 等格式,可一次选择多张或拖拽上传
</p>
</div>
</div>
@ -880,26 +878,16 @@ export default function ProductPackageUploader({
disabled={uploading}
/>
{/* 上传按钮 */}
<div className="flex items-center space-x-4 mb-4">
<div className="w-32 h-32 rounded-apple bg-gray-100 dark:bg-gray-700 flex items-center justify-center">
<svg className="w-12 h-12 text-gray-400" fill="none" stroke="currentColor" viewBox="0 0 24 24">
<title>添加详情图</title>
<path strokeLinecap="round" strokeLinejoin="round" strokeWidth={2} d="M12 4v16m8-8H4" />
</svg>
</div>
<div className="flex-1">
<button
onClick={() => detailInputRef.current?.click()}
className="btn-secondary"
disabled={uploading}
type="button"
>
{detailImages.length > 0 ? '继续添加图片' : '选择详情图'}
</button>
<p className="text-xs text-gray-500 dark:text-gray-400 mt-2">
支持 JPG、PNG、WebP 等格式,可一次选择多张或拖拽上传
</p>
</div>
<div className="flex items-center space-x-3 mb-3">
<button
onClick={() => detailInputRef.current?.click()}
className="btn-secondary text-sm"
disabled={uploading}
type="button"
>
{detailImages.length > 0 ? '继续添加' : '选择详情图'}
</button>
<span className="text-xs text-gray-400">JPG / PNG / WebP,支持拖拽</span>
</div>
{/* 详情图排序组件 */}
@ -943,55 +931,33 @@ export default function ProductPackageUploader({
disabled={uploading}
/>
{/* 上传按钮 */}
<div className="flex items-center space-x-4">
{psdFiles.length > 0 ? (
<div className="relative w-32 h-32 rounded-apple overflow-hidden border-2 border-gray-200 dark:border-gray-600 bg-gray-100 dark:bg-gray-700 flex items-center justify-center">
<svg className="w-12 h-12 text-gray-400" fill="none" stroke="currentColor" viewBox="0 0 24 24">
<title>PSD文件</title>
<path strokeLinecap="round" strokeLinejoin="round" strokeWidth={2} d="M7 21h10a2 2 0 002-2V9.414a1 1 0 00-.293-.707l-5.414-5.414A1 1 0 0012.586 3H7a2 2 0 00-2 2v14a2 2 0 002 2z" />
</svg>
<div className="absolute bottom-1 right-1 bg-blue-500 text-white text-xs rounded-full w-6 h-6 flex items-center justify-center">
{psdFiles.length}
</div>
</div>
) : (
<div className="w-32 h-32 rounded-apple bg-gray-100 dark:bg-gray-700 flex items-center justify-center">
<svg className="w-12 h-12 text-gray-400" fill="none" stroke="currentColor" viewBox="0 0 24 24">
<title>添加PSD文件</title>
<path strokeLinecap="round" strokeLinejoin="round" strokeWidth={2} d="M12 4v16m8-8H4" />
</svg>
<div className="flex items-center space-x-3">
<button
onClick={() => psdInputRef.current?.click()}
className="btn-secondary text-sm"
disabled={uploading}
type="button"
>
{psdFiles.length > 0 ? `继续添加 (${psdFiles.length}个)` : '选择源文件'}
</button>
<span className="text-xs text-gray-400">{TASK_DETAIL_SOURCE_FILE_LABEL}</span>
{psdFiles.length > 0 && (
<div className="flex flex-wrap gap-1.5 ml-2">
{psdFiles.map((psdFile) => (
<span key={psdFile.id} className="inline-flex items-center gap-1 px-2 py-0.5 bg-gray-100 dark:bg-gray-700 rounded text-xs text-gray-700 dark:text-gray-300">
{psdFile.file.name}
<button
onClick={() => setPsdFiles((prev) => prev.filter((f) => f.id !== psdFile.id))}
className="text-red-400 hover:text-red-600"
disabled={uploading}
type="button"
>
×
</button>
</span>
))}
</div>
)}
<div className="flex-1">
<button
onClick={() => psdInputRef.current?.click()}
className="btn-secondary"
disabled={uploading}
type="button"
>
{psdFiles.length > 0 ? '继续添加PSD文件' : '选择PSD文件'}
</button>
<p className="text-xs text-gray-500 dark:text-gray-400 mt-2">
支持 {TASK_DETAIL_SOURCE_FILE_LABEL} 格式,可一次选择多个或拖拽上传
</p>
{psdFiles.length > 0 && (
<div className="mt-2 space-y-2">
{psdFiles.map((psdFile) => (
<div key={psdFile.id} className="flex items-center justify-between p-2 bg-gray-100 dark:bg-gray-700 rounded-apple">
<span className="text-sm text-gray-700 dark:text-gray-300">{psdFile.file.name}</span>
<button
onClick={() => setPsdFiles((prev) => prev.filter((f) => f.id !== psdFile.id))}
className="text-red-500 dark:text-red-400 text-sm hover:text-red-600 dark:hover:text-red-300"
disabled={uploading}
type="button"
>
删除
</button>
</div>
))}
</div>
)}
</div>
</div>
</div>
</div>

View File

@ -1,11 +1,11 @@
# 生产存储桶接入信息
> 更新时间:2026-05-07
> 更新时间:2026-05-08
> 本文记录当前素材平台的 MinIO / S3 / uploads 存储事实。
## 1. 当前结论
- 新素材预览 / 展示文件:写入生产 MinIO 新桶路径,公开 URL 统一为 `https://s3.jiansun.vip/gongxiangpan/public-display/...`。
- 新素材预览 / 展示文件:写入生产 MinIO 新桶路径,公开 URL 统一为 `https://s3.jiansun.vip/gongxiangpan/由壹点AI/public-display/...`。
- 源文件 / 下载型文件:写入生产服务器 uploads 目录:`/mnt/PoolOne/apps/youyidian-3d/uploads`。
- 旧 `1080p.hk` 桶:只用于极少数老素材兼容,不作为新素材写入目标。
- `/gongxiangpan/uploads/` 是历史混合区,不再作为新展示资源的默认路径。
@ -26,6 +26,7 @@ S3_ENDPOINT=http://192.168.58.20:9000
S3_BUCKET=gongxiangpan
S3_REGION=us-east-1
S3_PUBLIC_DOMAIN=https://s3.jiansun.vip/gongxiangpan
S3_FILE_PATH=由壹点AI/
```
代码固定行为:
@ -47,16 +48,16 @@ S3_PUBLIC_DOMAIN=https://s3.jiansun.vip/gongxiangpan
正式路径:
```text
https://s3.jiansun.vip/gongxiangpan/public-display/...
https://s3.jiansun.vip/gongxiangpan/由壹点AI/public-display/...
```
常见前缀:
```text
public-display/materials/...
public-display/task-detail/...
public-display/white-background/...
public-display/certificate-report/...
由壹点AI/public-display/materials/...
由壹点AI/public-display/task-detail/...
由壹点AI/public-display/white-background/...
由壹点AI/public-display/certificate-report/...
```
这些资源用于页面展示、AI 平台预览、钉钉返回地址和外部接口展示直链。
@ -69,7 +70,7 @@ public-display/certificate-report/...
由壹点AI/
```
它主要用于旧逻辑或非 `public-display` 的对象,不代表新素材展示图默认落点。
它是当前生产环境的默认业务前缀。新素材展示图应落在 `由壹点AI/public-display/...`,不要写入旧桶。
## 4. uploads 目录规则
@ -101,21 +102,40 @@ public-display/certificate-report/...
不要把 uploads 理解成“少量视频临时区”。它是当前生产源文件和下载型文件的核心落点。
## 5. 历史兼容
## 5. 历史素材正式导入要求
### 5.1 旧 `1080p.hk` 桶
历史素材导入有一个特殊约束:执行机要能读取历史素材源目录,同时下载型文件必须最终落到生产 uploads。
当前天蓝盘批量导入的标准做法:
- Windows 维护机读取 `Z:\...` 源素材。
- 预览图上传到新桶 `gongxiangpan` 的 `由壹点AI/public-display/...`。
- ZIP / PSD / 源文件先在本机生成,再通过远程同步写入生产机 `/mnt/PoolOne/apps/youyidian-3d/uploads`,并校验远程文件大小。
正式 `--apply` 如果在 Windows 上连接生产库,必须同时配置:
```env
HISTORY_IMPORT_REMOTE_UPLOADS_HOST=prod
HISTORY_IMPORT_REMOTE_UPLOADS_DIR=/mnt/PoolOne/apps/youyidian-3d/uploads
```
没有远程 uploads 同步时,脚本必须阻止正式导入。否则会把数据库写到生产库,但 ZIP / PSD 留在本机 `uploads`,线上下载会 404。
## 6. 历史兼容
### 6.1 旧 `1080p.hk` 桶
`s3.1080p.hk/jsxiangche` 只保留给极少数老素材兼容。新素材、批量新录入素材、后续上传素材都不应继续写入旧桶。
### 5.2 旧资源域名
### 6.2 旧资源域名
`s3.yydd.ai` 不再作为默认对接口径,只用于识别历史链接。
### 5.3 历史混合区
### 6.3 历史混合区
`/gongxiangpan/uploads/` 保留历史兼容,不再作为新展示图默认路径。
## 6. 只读核对命令
## 7. 只读核对命令
核对应用环境变量:

View File

@ -1,7 +1,12 @@
import { prisma } from '@/lib/prisma'
import { completeProductPackageUpload, type ProductPackageImageInput } from '@/lib/product-package-completion'
import {
completeProductPackageUpload,
type ProductPackageImageInput,
type ProductPackageSourceInput,
} from '@/lib/product-package-completion'
import { createMaterialLog } from '@/lib/material-log'
import { isTaskDetailFullSetFromUploadGroups } from '@/lib/task-detail-full-set'
import { normalizeTaskDetailSourceType } from '@/lib/task-detail-source-type'
import path from 'path'
import { promises as fs } from 'fs'
import crypto from 'crypto'
@ -33,6 +38,10 @@ const IGNORED_DIR_PATTERNS = [
/废稿/i,
/字体/i,
/勿用/i,
/错误勿用/i,
/图片有误/i,
/作废/i,
/3D建模/i,
/DingTalkAppData/i,
/^PsTool$/i,
/^Cache$/i,
@ -68,6 +77,20 @@ const STORE_DIR_IGNORES = new Set([
const execFileAsync = promisify(execFile)
const HISTORY_ZIP_TIMEOUT_MS = Number(process.env.HISTORY_IMPORT_ZIP_TIMEOUT_MS || 180000)
const HISTORY_ALL_ZIP_TIMEOUT_MS = Number(process.env.HISTORY_IMPORT_ALL_ZIP_TIMEOUT_MS || 300000)
const BRANDED_SPU_BODY_PATTERN = '[A-Z]{2,5}-[A-Z]{1,6}\\d{3,5}(?:K\\d{1,3})?'
const UNBRANDED_SPU_BODY_PATTERN = '[A-Z]{1,6}\\d{3,5}(?:K\\d{1,3})?'
const COMBO_SPU_TOKEN_PATTERN = new RegExp(
`(?<![A-Z0-9])${BRANDED_SPU_BODY_PATTERN}(?:\\s*[++]\\s*(?:${BRANDED_SPU_BODY_PATTERN}|${UNBRANDED_SPU_BODY_PATTERN}))+(?![A-Z0-9])`,
'i',
)
const BLOCKED_READY_SOURCE_PATH_PATTERNS = [
{ pattern: /错误勿用/i, label: '错误勿用' },
{ pattern: /废稿/i, label: '废稿' },
{ pattern: /勿用/i, label: '勿用' },
{ pattern: /图片有误/i, label: '图片有误' },
{ pattern: /作废/i, label: '作废' },
{ pattern: /3D建模/i, label: '3D建模' },
]
export type HistoryImportStatus = 'ready' | 'needs_review' | 'skipped'
@ -79,16 +102,19 @@ export interface HistoryImportRecord {
guessedStore: string
productCategoryName: string
authorName: string
taskDetailSourceType?: string
storeConfidence: number
storeCandidates: string[]
mainCount: number
skuCount: number
detailCount: number
sourceCount?: number
status: HistoryImportStatus
skipReason: string | null
mainImages: ProductPackageImageInput[]
skuImages: ProductPackageImageInput[]
detailImages: ProductPackageImageInput[]
psdSources?: ProductPackageSourceInput[]
seenInYya: boolean
}
@ -133,8 +159,52 @@ interface FileEntry {
fileSize: number
}
interface SanitizedHistoryImportRecord {
record: HistoryImportRecord
skippedEmptyFiles: Array<{
group: 'main' | 'sku' | 'detail' | 'psd'
fileName: string
localFilePath?: string
}>
skippedInvalidFiles: Array<{
group: 'main' | 'sku' | 'detail'
fileName: string
fileSize: number
localFilePath?: string
reason: string
}>
}
function normalizeSpu(value: string) {
return value.trim().toUpperCase()
return value.trim().toUpperCase().replace(/+/g, '+').replace(/\s*\+\s*/g, '+')
}
function expandComboSpuToken(token: string) {
const parts = normalizeSpu(token).split('+').filter(Boolean)
const first = parts[0] || ''
const prefix = first.includes('-') ? first.split('-')[0] : ''
if (parts.length < 2 || !prefix) {
return null
}
const expanded = parts.map((part, index) => {
if (index === 0 || part.includes('-')) {
return part
}
return `${prefix}-${part}`
})
return expanded.join('+')
}
function inferComboSpuFromPath(sourcePath: string) {
const match = sourcePath.match(COMBO_SPU_TOKEN_PATTERN)
if (!match) {
return null
}
return expandComboSpuToken(match[0])
}
function isSpuName(name: string) {
@ -688,18 +758,61 @@ async function ensureCategoryAndSceneNames(categoryName: string, sceneName: stri
}
}
async function hasImportedFingerprint(fingerprint: string) {
const existing = await prisma.materialLog.findFirst({
async function hasImportedHistoryRecord(record: HistoryImportRecord) {
const existingByFingerprint = await prisma.materialLog.findFirst({
where: {
action: 'HISTORY_IMPORT',
material: {
deletedAt: null,
},
changes: {
contains: fingerprint,
contains: record.fingerprint,
} as any,
},
select: { id: true },
})
return Boolean(existing)
if (existingByFingerprint) {
return true
}
const existingBySourcePath = await prisma.$queryRaw<Array<{ id: string }>>`
SELECT logs.id
FROM "material_logs" AS logs
INNER JOIN "materials" AS materials
ON materials.id = logs."materialId"
WHERE logs.action = 'HISTORY_IMPORT'
AND materials."deletedAt" IS NULL
AND logs.changes IS NOT NULL
AND logs.changes::jsonb->>'sourcePath' = ${record.sourcePath}
LIMIT 1
`
return existingBySourcePath.length > 0
}
function getBlockedReadySourceReason(record: HistoryImportRecord) {
const haystack = [
record.sourcePath,
record.relativeSourcePath,
record.skipReason || '',
].join('\n')
const matched = BLOCKED_READY_SOURCE_PATH_PATTERNS.find(({ pattern }) => pattern.test(haystack))
if (!matched) {
return null
}
return `来源路径包含“${matched.label}”,即使台账标记 ready 也禁止正式导入`
}
function getSpuMismatchReason(record: HistoryImportRecord) {
const inferredComboSpu = inferComboSpuFromPath(record.sourcePath)
if (!inferredComboSpu || normalizeSpu(record.spu) === inferredComboSpu) {
return null
}
return `来源路径包含组合 SPU“${inferredComboSpu}”,但台账 SPU 是“${record.spu}”,禁止按单 SPU 导入`
}
async function createHistoryImportMaterial(record: HistoryImportRecord, operatorUserId: string) {
@ -717,12 +830,103 @@ async function createHistoryImportMaterial(record: HistoryImportRecord, operator
renderer: '',
isFullSet: false,
isAiGenerated: false,
taskDetailSourceType: normalizeTaskDetailSourceType(record.taskDetailSourceType) || '渲染',
} as any,
})
}
function filterNonEmptyAssets<T extends { fileName: string; fileSize: number; localFilePath?: string }>(
group: 'main' | 'sku' | 'detail' | 'psd',
assets: T[] | undefined,
skippedEmptyFiles: SanitizedHistoryImportRecord['skippedEmptyFiles'],
) {
return (assets || []).filter((asset) => {
if (asset.fileSize > 0) {
return true
}
skippedEmptyFiles.push({
group,
fileName: asset.fileName,
localFilePath: asset.localFilePath,
})
return false
})
}
async function filterReadableImageAssets<T extends { fileName: string; fileSize: number; localFilePath?: string }>(
group: 'main' | 'sku' | 'detail',
assets: T[],
skippedInvalidFiles: SanitizedHistoryImportRecord['skippedInvalidFiles'],
) {
if (assets.length === 0) {
return assets
}
const sharp = await import('sharp')
const readableAssets: T[] = []
for (const asset of assets) {
if (!asset.localFilePath) {
readableAssets.push(asset)
continue
}
try {
await sharp
.default(asset.localFilePath)
.resize(1, 1, {
fit: 'inside',
withoutEnlargement: true,
})
.webp({ quality: 1 })
.toBuffer()
readableAssets.push(asset)
} catch (error) {
skippedInvalidFiles.push({
group,
fileName: asset.fileName,
fileSize: asset.fileSize,
localFilePath: asset.localFilePath,
reason: error instanceof Error ? error.message : '图片无法读取',
})
}
}
return readableAssets
}
async function sanitizeHistoryImportRecord(record: HistoryImportRecord): Promise<SanitizedHistoryImportRecord> {
const skippedEmptyFiles: SanitizedHistoryImportRecord['skippedEmptyFiles'] = []
const skippedInvalidFiles: SanitizedHistoryImportRecord['skippedInvalidFiles'] = []
const nonEmptyMainImages = filterNonEmptyAssets('main', record.mainImages, skippedEmptyFiles)
const nonEmptySkuImages = filterNonEmptyAssets('sku', record.skuImages, skippedEmptyFiles)
const nonEmptyDetailImages = filterNonEmptyAssets('detail', record.detailImages, skippedEmptyFiles)
const sanitizedRecord: HistoryImportRecord = {
...record,
mainImages: await filterReadableImageAssets('main', nonEmptyMainImages, skippedInvalidFiles),
skuImages: await filterReadableImageAssets('sku', nonEmptySkuImages, skippedInvalidFiles),
detailImages: await filterReadableImageAssets('detail', nonEmptyDetailImages, skippedInvalidFiles),
psdSources: filterNonEmptyAssets('psd', record.psdSources, skippedEmptyFiles),
}
if (
sanitizedRecord.mainImages.length === 0 ||
sanitizedRecord.skuImages.length === 0 ||
sanitizedRecord.detailImages.length === 0
) {
throw new Error(
`过滤 0 字节文件后不满足导入条件:主图=${sanitizedRecord.mainImages.length} SKU=${sanitizedRecord.skuImages.length} 详情=${sanitizedRecord.detailImages.length}`,
)
}
return { record: sanitizedRecord, skippedEmptyFiles, skippedInvalidFiles }
}
export async function importHistoryMaterials(options: ImportHistoryMaterialsOptions) {
await ensureCategoryAndScene('其他', '其他')
if (options.apply) {
await ensureCategoryAndScene('其他', '其他')
}
const candidates = options.records
.filter((record) => {
@ -735,7 +939,35 @@ export async function importHistoryMaterials(options: ImportHistoryMaterialsOpti
const results: ImportHistoryMaterialsResultItem[] = []
for (const record of candidates) {
if (await hasImportedFingerprint(record.fingerprint)) {
const blockedReadySourceReason = getBlockedReadySourceReason(record)
if (blockedReadySourceReason) {
results.push({
fingerprint: record.fingerprint,
spu: record.spu,
sourcePath: record.sourcePath,
guessedStore: record.guessedStore,
materialId: null,
status: 'skipped',
reason: blockedReadySourceReason,
})
continue
}
const spuMismatchReason = getSpuMismatchReason(record)
if (spuMismatchReason) {
results.push({
fingerprint: record.fingerprint,
spu: record.spu,
sourcePath: record.sourcePath,
guessedStore: record.guessedStore,
materialId: null,
status: 'skipped',
reason: spuMismatchReason,
})
continue
}
if (await hasImportedHistoryRecord(record)) {
results.push({
fingerprint: record.fingerprint,
spu: record.spu,
@ -762,16 +994,18 @@ export async function importHistoryMaterials(options: ImportHistoryMaterialsOpti
}
try {
const sanitized = await sanitizeHistoryImportRecord(record)
await ensureCategoryAndSceneNames(record.guessedStore || '其他', '其他')
const material = await createHistoryImportMaterial(record, options.operator.id)
const material = await createHistoryImportMaterial(sanitized.record, options.operator.id)
await completeProductPackageUpload({
materialId: material.id,
taskNo: null,
operatorUserId: options.operator.id,
assets: {
main: record.mainImages,
sku: record.skuImages,
detail: record.detailImages,
main: sanitized.record.mainImages,
sku: sanitized.record.skuImages,
detail: sanitized.record.detailImages,
psd: sanitized.record.psdSources || [],
},
options: {
allowSkuWithoutBarcodes: true,
@ -789,7 +1023,11 @@ export async function importHistoryMaterials(options: ImportHistoryMaterialsOpti
sourcePath: record.sourcePath,
guessedStore: record.guessedStore,
storeCandidates: record.storeCandidates,
sourceCount: record.sourceCount || record.psdSources?.length || 0,
taskDetailSourceType: normalizeTaskDetailSourceType(record.taskDetailSourceType) || '渲染',
seenInYya: record.seenInYya,
skippedEmptyFiles: sanitized.skippedEmptyFiles,
skippedInvalidFiles: sanitized.skippedInvalidFiles,
},
})

View File

@ -3,7 +3,7 @@ import { promises as fs } from 'fs'
import crypto from 'crypto'
import type { ProductPackageImageInput } from '@/lib/product-package-completion'
import type { HistoryImportRecord, HistoryImportStatus } from '@/lib/history-material-import'
import { parseSPU } from '@/lib/spu-parser'
import { getShopPrefixMap, parseSPU } from '@/lib/spu-parser'
const IMAGE_EXTENSIONS = new Set(['.jpg', '.jpeg', '.png', '.webp'])
const SKIPPED_FILE_EXTENSIONS = new Set([
@ -50,9 +50,19 @@ const EXCLUDED_FILE_NAME_PATTERNS = [
/mask/i,
]
const BRANDED_SPU_TOKEN_PATTERN = /([A-Za-z]{2,5}-[A-Za-z]{1,5}\d{3,5})/g
const PRIMARY_SPU_TOKEN_PATTERN = /([A-Za-z]{2,3}\d{4})/g
const FALLBACK_SPU_TOKEN_PATTERN = /([A-Za-z]{1,5}\d{3,5})/g
const IGNORED_SPU_PREFIXES = ['TASK', 'SKU', 'IMG']
const EXTRA_SHOP_PREFIX_MAP: Record<string, string> = {
GBJP: '顾家',
}
const EXTRA_CATEGORY_CODE_MAP: Record<string, string> = {
BI: '边几',
CFZWJ: '置物架',
SMC: '床',
}
const CATEGORY_CODE_PATTERN = /^([A-Z]{1,5})\d/
const AUTHOR_SEGMENT_EXCLUDES = new Set([
'亿家木',
'亿家达',
@ -216,7 +226,35 @@ function inferSpuFromText(value: string) {
const isStandaloneToken = (matchIndex: number, token: string) => {
const prevChar = matchIndex > 0 ? text[matchIndex - 1] : ''
const nextChar = text[matchIndex + token.length] || ''
return !/[A-Za-z]/.test(prevChar) && !/[A-Za-z0-9]/.test(nextChar)
if (/[A-Za-z0-9]/.test(prevChar)) {
return false
}
if (!/[A-Za-z0-9]/.test(nextChar)) {
return true
}
return /^K\d{1,2}(?![A-Za-z0-9])/i.test(text.slice(matchIndex + token.length))
}
BRANDED_SPU_TOKEN_PATTERN.lastIndex = 0
const brandedMatches = Array.from(text.matchAll(BRANDED_SPU_TOKEN_PATTERN))
if (brandedMatches.length > 0) {
const shopPrefixMap = { ...getShopPrefixMap(), ...EXTRA_SHOP_PREFIX_MAP }
for (let index = brandedMatches.length - 1; index >= 0; index -= 1) {
const currentMatch = brandedMatches[index]
const rawToken = currentMatch?.[1] || ''
const token = normalizeSpu(rawToken)
const [shopPrefix, skuPart] = token.split('-')
const matchIndex = currentMatch?.index ?? -1
if (
token &&
shopPrefixMap[shopPrefix] &&
skuPart &&
matchIndex >= 0 &&
isStandaloneToken(matchIndex, rawToken)
) {
return token
}
}
}
PRIMARY_SPU_TOKEN_PATTERN.lastIndex = 0
@ -291,12 +329,13 @@ function inferSpu(row: ParsedSkuCsvRow) {
if (row.spu) {
const normalizedRowSpu = normalizeSpu(row.spu)
const rowSpuLooksReliable =
const rowSpuLooksUsable =
normalizedRowSpu &&
!IGNORED_SPU_PREFIXES.some((prefix) => normalizedRowSpu.startsWith(prefix)) &&
candidates.some((candidate) => hasStandaloneSpuToken(candidate, normalizedRowSpu))
(/^[A-Z]{2,5}-[A-Z]{1,5}\d{3,5}$/.test(normalizedRowSpu) ||
/^[A-Z]{1,5}\d{3,5}$/.test(normalizedRowSpu))
if (rowSpuLooksReliable) {
if (rowSpuLooksUsable) {
return normalizedRowSpu
}
}
@ -482,7 +521,25 @@ function buildAssetInputs(spu: string, files: FileEntry[], group: 'main' | 'sku'
}
function inferStoreBySpu(spu: string, row?: ParsedSkuCsvRow) {
const sourceRootStore = inferStoreBySourceRoot(row)
if (sourceRootStore) {
return {
guessedStore: sourceRootStore,
storeConfidence: 1,
storeCandidates: [sourceRootStore],
}
}
const normalizedSpu = normalizeSpu(spu)
const spuParseResult = parseSPU(normalizedSpu)
if (spuParseResult.shopMatched && spuParseResult.shopName) {
return {
guessedStore: spuParseResult.shopName,
storeConfidence: 1,
storeCandidates: [spuParseResult.shopName],
}
}
const storeProbeText = [
normalizedSpu,
row?.relativePath || '',
@ -493,36 +550,17 @@ function inferStoreBySpu(spu: string, row?: ParsedSkuCsvRow) {
]
.join('\n')
.toUpperCase()
const shopPrefixMap = { ...getShopPrefixMap(), ...EXTRA_SHOP_PREFIX_MAP }
const matchedPrefix = Object.keys(shopPrefixMap)
.sort((left, right) => right.length - left.length)
.find((prefix) => new RegExp(`(^|[^A-Z0-9])${prefix}-`).test(storeProbeText))
if (storeProbeText.includes('GBGJ')) {
if (matchedPrefix) {
const storeName = shopPrefixMap[matchedPrefix]
return {
guessedStore: '顾家',
guessedStore: storeName,
storeConfidence: 1,
storeCandidates: ['顾家'],
}
}
if (storeProbeText.includes('GBSFY')) {
return {
guessedStore: '索菲亚',
storeConfidence: 1,
storeCandidates: ['索菲亚'],
}
}
if (normalizedSpu.startsWith('GBGJ')) {
return {
guessedStore: '顾家',
storeConfidence: 1,
storeCandidates: ['顾家'],
}
}
if (normalizedSpu.startsWith('GBSFY')) {
return {
guessedStore: '索菲亚',
storeConfidence: 1,
storeCandidates: ['索菲亚'],
storeCandidates: [storeName],
}
}
@ -533,6 +571,36 @@ function inferStoreBySpu(spu: string, row?: ParsedSkuCsvRow) {
}
}
function inferStoreBySourceRoot(row?: ParsedSkuCsvRow) {
const sourceRoot = row?.sourceRoot || ''
const probes = [
['顾家项目', '顾家'],
['金牌项目', '顾家'],
['朴美项目', '朴美'],
['索菲亚项目', '索菲亚'],
] as const
const matched = probes.find(([keyword]) => sourceRoot.includes(keyword))
return matched?.[1] || ''
}
function inferProductCategoryBySpu(spu: string) {
const spuParseResult = parseSPU(spu)
if (spuParseResult.categoryMatched) {
return spuParseResult.categoryName
}
const normalizedSpu = normalizeSpu(spu)
const skuPart = normalizedSpu.includes('-') ? normalizedSpu.split('-').pop() || '' : normalizedSpu
const skuPartParseResult = parseSPU(skuPart)
if (skuPartParseResult.categoryMatched) {
return skuPartParseResult.categoryName
}
const categoryCode = skuPart.match(CATEGORY_CODE_PATTERN)?.[1] || ''
return EXTRA_CATEGORY_CODE_MAP[categoryCode] || ''
}
function normalizeAuthorToken(value: string) {
return value.replace(/[\s()()_\-]+/g, '').trim().toLowerCase()
}
@ -608,6 +676,23 @@ function makeFingerprint(spu: string, sourcePath: string) {
return crypto.createHash('sha1').update(`${normalizeSpu(spu)}|${sourcePath}`).digest('hex')
}
function normalizeDuplicatePathKey(relativeSourcePath: string) {
const segments = relativeSourcePath
.split(/[\\/]+/)
.map((item) => item.trim())
.filter(Boolean)
if (segments.length > 1 && segments[0] === segments[1]) {
segments.shift()
}
return segments.join('/').toLowerCase()
}
function makeDuplicateKey(spu: string, relativeSourcePath: string) {
return `${normalizeSpu(spu)}|${normalizeDuplicatePathKey(relativeSourcePath)}`
}
export async function buildHistoryImportRecordsFromSkuCsv(
options: BuildHistoryImportRecordsFromSkuCsvOptions,
) {
@ -624,9 +709,16 @@ export async function buildHistoryImportRecordsFromSkuCsv(
new Set((options.visualCenterAuthors || []).map((item) => item.trim()).filter(Boolean)),
)
const records: HistoryImportRecord[] = []
const seenDuplicateKeys = new Set<string>()
for (const row of limitedRows) {
const spu = inferSpu(row)
const duplicateKey = makeDuplicateKey(spu || row.directoryName || 'UNKNOWN', row.relativePath)
if (seenDuplicateKeys.has(duplicateKey)) {
continue
}
seenDuplicateKeys.add(duplicateKey)
const detailDirName = pickPreferredDirName(row.siblingDetailDirNames, 'detail')
const mainDirName = pickPreferredDirName(row.siblingMainDirNames, 'main')
@ -636,8 +728,7 @@ export async function buildHistoryImportRecordsFromSkuCsv(
const storeGuess = inferStoreBySpu(spu, row)
const hasFullSet = mainFiles.length > 0 && skuFiles.length > 0 && detailFiles.length > 0
const spuParseResult = parseSPU(spu)
const productCategoryName = spuParseResult.categoryMatched ? spuParseResult.categoryName : ''
const productCategoryName = inferProductCategoryBySpu(spu)
const authorName = inferAuthorFromRow(row, visualCenterAuthors)
let status: HistoryImportStatus = 'ready'

View File

@ -7,6 +7,8 @@ import {
createZipFromZips,
saveProductPackageAssets,
} from '@/lib/task-material-processor'
import { buildStoredUploadPath, getUploadRootDir } from '@/lib/upload-paths'
import { isRemoteUploadSyncEnabled, syncStoredUploadFileToRemote } from '@/lib/remote-upload-sync'
import { getUploadedFile } from '@/lib/upload-sessions'
import { createMaterialLog } from '@/lib/material-log'
import { isTaskDetailFullSetFromUploadGroups } from '@/lib/task-detail-full-set'
@ -289,8 +291,8 @@ async function movePsdFilesToPermanentDir(
return psdPermanentPaths
}
const { rename, copyFile } = await import('fs/promises')
const psdDir = path.join(process.cwd(), 'uploads', 'materials')
const { rename, copyFile, stat } = await import('fs/promises')
const psdDir = path.join(getUploadRootDir(), 'materials')
await mkdir(psdDir, { recursive: true })
for (const psd of assets) {
@ -299,15 +301,36 @@ async function movePsdFilesToPermanentDir(
throw new Error(`PSD 资产参数无效: ${psd.fileName}`)
}
const tempFilePath = await downloadToTempFile('psd', psd)
const timestamp = Date.now()
const sanitizedFileName = psd.fileName.replace(/[\/\\:*?"<>|]/g, '_')
const permanentFileName = `psd-${materialId}-${timestamp}-${sanitizedFileName}`
const permanentFilePath = path.join(psdDir, permanentFileName)
const relativePath = buildStoredUploadPath('materials', permanentFileName)
if (psd.localFilePath) {
const sourceStat = await stat(psd.localFilePath)
const expectedSize = psd.fileSize || sourceStat.size
if (isRemoteUploadSyncEnabled()) {
await syncStoredUploadFileToRemote(psd.localFilePath, relativePath, expectedSize)
} else {
await copyFile(psd.localFilePath, permanentFilePath)
}
psdPermanentPaths.push({
filePath: relativePath,
fileName: psd.fileName,
fileSize: expectedSize,
})
continue
}
const tempFilePath = await downloadToTempFile('psd', psd)
await copyFile(tempFilePath, permanentFilePath)
await syncStoredUploadFileToRemote(permanentFilePath, relativePath, psd.fileSize)
tempFiles.push(tempFilePath)
psdPermanentPaths.push({
filePath: path.join('uploads', 'materials', permanentFileName),
filePath: relativePath,
fileName: psd.fileName,
fileSize: psd.fileSize,
})
@ -323,7 +346,7 @@ async function movePsdFilesToPermanentDir(
const sanitizedFileName = psd.fileName.replace(/[\/\\:*?"<>|]/g, '_')
const permanentFileName = `psd-${materialId}-${timestamp}-${sanitizedFileName}`
const permanentFilePath = path.join(psdDir, permanentFileName)
const relativePath = path.join('uploads', 'materials', permanentFileName)
const relativePath = buildStoredUploadPath('materials', permanentFileName)
try {
await rename(tempFilePath, permanentFilePath)
@ -332,6 +355,8 @@ async function movePsdFilesToPermanentDir(
tempFiles.push(tempFilePath)
}
await syncStoredUploadFileToRemote(permanentFilePath, relativePath, psd.fileSize)
psdPermanentPaths.push({
filePath: relativePath,
fileName: psd.fileName,
@ -468,6 +493,7 @@ export async function completeProductPackageUpload(input: CompleteProductPackage
createdZipPaths.push(zipInfos.all.filePath)
const psdPermanentPaths = await movePsdFilesToPermanentDir(materialId, normalizedAssets.psd, tempFiles)
createdZipPaths.push(...psdPermanentPaths.map((item) => item.filePath))
const saveResult = await saveProductPackageAssets(materialId, previewAssets, zipInfos, psdPermanentPaths)

214
lib/remote-upload-sync.ts Normal file
View File

@ -0,0 +1,214 @@
import path from 'path'
import { execFile } from 'child_process'
import { stat } from 'fs/promises'
import { promisify } from 'util'
const execFileAsync = promisify(execFile)
const REMOTE_SYNC_SSH_TIMEOUT_MS = Number(process.env.REMOTE_SYNC_SSH_TIMEOUT_MS || 30000)
const REMOTE_SYNC_SCP_TIMEOUT_MS = Number(process.env.REMOTE_SYNC_SCP_TIMEOUT_MS || 7200000)
const REMOTE_SYNC_RETRY_COUNT = Number(process.env.REMOTE_SYNC_RETRY_COUNT || 3)
const SSH_NON_INTERACTIVE_ARGS = [
'-o',
'BatchMode=yes',
'-o',
'ConnectTimeout=15',
'-o',
'ServerAliveInterval=15',
'-o',
'ServerAliveCountMax=2',
]
interface RemoteUploadSyncConfig {
target: string
uploadRoot: string
}
function cleanEnv(value: string | undefined) {
return String(value || '').trim()
}
function shellQuote(value: string) {
return `'${value.replace(/'/g, `'\\''`)}'`
}
function normalizeRemotePath(value: string) {
return value.replace(/\\/g, '/').replace(/\/+$/, '')
}
function sleep(ms: number) {
return new Promise((resolve) => setTimeout(resolve, ms))
}
async function execFileWithRetry(
file: string,
args: string[],
options: { timeout: number },
label: string,
) {
let lastError: unknown
for (let attempt = 1; attempt <= REMOTE_SYNC_RETRY_COUNT; attempt += 1) {
try {
return await execFileAsync(file, args, options)
} catch (error) {
lastError = error
const message = error instanceof Error ? error.message : String(error)
console.warn(
`[RemoteUploadSync] ${label} 失败 ${attempt}/${REMOTE_SYNC_RETRY_COUNT}: ${message}`,
)
if (attempt < REMOTE_SYNC_RETRY_COUNT) {
await sleep(1000 * attempt)
}
}
}
throw lastError
}
function relativeToUploadRoot(storedPath: string) {
const normalized = String(storedPath || '').replace(/\\/g, '/')
const relative = normalized.replace(/^\.?\/*uploads\//i, '')
if (!relative || relative === normalized && path.posix.isAbsolute(normalized)) {
throw new Error(`远程同步路径无效: ${storedPath}`)
}
if (relative.split('/').some((segment) => segment === '..')) {
throw new Error(`远程同步路径不能包含上级目录: ${storedPath}`)
}
return relative
}
export function getRemoteUploadSyncConfig(): RemoteUploadSyncConfig | null {
const target = cleanEnv(
process.env.HISTORY_IMPORT_REMOTE_UPLOADS_HOST || process.env.UPLOAD_REMOTE_SYNC_HOST,
)
const uploadRoot = cleanEnv(
process.env.HISTORY_IMPORT_REMOTE_UPLOADS_DIR || process.env.UPLOAD_REMOTE_SYNC_ROOT,
)
if (!target && !uploadRoot) {
return null
}
if (!target || !uploadRoot) {
throw new Error(
'远程 uploads 同步配置不完整:需要同时配置 HISTORY_IMPORT_REMOTE_UPLOADS_HOST 和 HISTORY_IMPORT_REMOTE_UPLOADS_DIR',
)
}
return {
target,
uploadRoot: normalizeRemotePath(uploadRoot),
}
}
export function isRemoteUploadSyncEnabled() {
return Boolean(getRemoteUploadSyncConfig())
}
export function getRemoteUploadPath(storedPath: string) {
const config = getRemoteUploadSyncConfig()
if (!config) {
return null
}
return path.posix.join(config.uploadRoot, relativeToUploadRoot(storedPath))
}
export async function assertRemoteUploadSyncAccess() {
const config = getRemoteUploadSyncConfig()
if (!config) {
return
}
const materialsDir = path.posix.join(config.uploadRoot, 'materials')
await execFileWithRetry(
'ssh',
[
...SSH_NON_INTERACTIVE_ARGS,
config.target,
`test -d ${shellQuote(materialsDir)} && test -w ${shellQuote(materialsDir)}`,
],
{ timeout: REMOTE_SYNC_SSH_TIMEOUT_MS },
'检查远程 uploads 权限',
)
}
export async function syncStoredUploadFileToRemote(
localFilePath: string,
storedPath: string,
expectedSize?: number,
) {
const config = getRemoteUploadSyncConfig()
if (!config) {
return null
}
const remotePath = getRemoteUploadPath(storedPath)
if (!remotePath) {
return null
}
const localStat = await stat(localFilePath)
const expectedFileSize = expectedSize || localStat.size
const remoteDir = path.posix.dirname(remotePath)
const sizeMiB = (expectedFileSize / 1024 / 1024).toFixed(2)
console.log(
`[RemoteUploadSync] 准备同步 ${storedPath} -> ${config.target}:${remotePath} (${sizeMiB} MiB)`,
)
await execFileWithRetry(
'ssh',
[...SSH_NON_INTERACTIVE_ARGS, config.target, `mkdir -p ${shellQuote(remoteDir)}`],
{ timeout: REMOTE_SYNC_SSH_TIMEOUT_MS },
`创建远程目录 ${remoteDir}`,
)
console.log(`[RemoteUploadSync] 开始复制 ${storedPath} (${sizeMiB} MiB)`)
await execFileWithRetry(
'scp',
['-q', ...SSH_NON_INTERACTIVE_ARGS, localFilePath, `${config.target}:${remotePath}`],
{ timeout: REMOTE_SYNC_SCP_TIMEOUT_MS },
`复制 ${storedPath}`,
)
const { stdout } = await execFileWithRetry(
'ssh',
[
...SSH_NON_INTERACTIVE_ARGS,
config.target,
`test -f ${shellQuote(remotePath)} && stat -c %s ${shellQuote(remotePath)}`,
],
{ timeout: REMOTE_SYNC_SSH_TIMEOUT_MS },
`校验远程文件 ${remotePath}`,
)
const remoteSize = Number(String(stdout || '').trim())
if (remoteSize !== expectedFileSize) {
throw new Error(
`远程 uploads 同步校验失败: ${storedPath} 本地=${expectedFileSize} 远程=${remoteSize || 'unknown'}`,
)
}
console.log(`[RemoteUploadSync] ${storedPath} -> ${config.target}:${remotePath} (${remoteSize} bytes)`)
return remotePath
}
export async function deleteStoredUploadFileFromRemote(storedPath: string) {
const config = getRemoteUploadSyncConfig()
if (!config) {
return
}
const remotePath = getRemoteUploadPath(storedPath)
if (!remotePath) {
return
}
await execFileWithRetry(
'ssh',
[...SSH_NON_INTERACTIVE_ARGS, config.target, `rm -f -- ${shellQuote(remotePath)}`],
{ timeout: REMOTE_SYNC_SSH_TIMEOUT_MS },
`删除远程文件 ${remotePath}`,
)
}

View File

@ -32,6 +32,7 @@ const SHOP_PREFIX_MAP: Record<string, string> = {
'GBPM': '朴美',
'GBSFY': '索菲亚',
'GBGJ': '顾家',
'GBJP': '顾家',
'XYL': '橡野里',
}

View File

@ -10,11 +10,15 @@ import {
uploadPublicDisplayToS3,
} from '@/lib/public-display';
import { compressImage } from '@/lib/image-processor';
import { buildStoredUploadPath, getUploadRootDir, resolveStoredUploadPath } from '@/lib/upload-paths';
import {
deleteStoredUploadFileFromRemote,
syncStoredUploadFileToRemote,
} from '@/lib/remote-upload-sync';
import { mkdir, unlink, readFile, rm } from 'fs/promises';
import { createWriteStream } from 'fs';
import path from 'path';
import archiver from 'archiver';
import { deleteUploadSession } from '@/lib/upload-sessions-redis';
// 资产类型常量
export const ASSET_TYPE = {
@ -92,7 +96,7 @@ export async function createZipFromFiles(
}
// 创建目标目录
const uploadsDir = path.join(process.cwd(), 'uploads', 'materials');
const uploadsDir = path.join(getUploadRootDir(), 'materials');
await mkdir(uploadsDir, { recursive: true });
// 生成zip文件名和路径(保留中文,只清理文件系统不安全字符)
@ -105,14 +109,14 @@ export async function createZipFromFiles(
if (authorName && categoryName && zipType !== 'all') {
const sanitizedAuthorName = authorName.replace(/[\/\\:*?"<>|]/g, '_');
const sanitizedCategoryName = categoryName.replace(/[\/\\:*?"<>|]/g, '_');
zipFileName = `${sanitizedProductModel}_${sanitizedAuthorName}_${sanitizedCategoryName}_${zipType}.zip`;
zipFileName = `${sanitizedProductModel}_${sanitizedAuthorName}_${sanitizedCategoryName}_${zipType}_${materialId}_${timestamp}.zip`;
} else {
// 保持原有命名格式(用于all.zip或其他情况)
zipFileName = `${sanitizedProductModel}_${zipType}_${materialId}_${timestamp}.zip`;
}
const zipFilePath = path.join(uploadsDir, zipFileName);
const relativeZipPath = path.join('uploads', 'materials', zipFileName);
const relativeZipPath = buildStoredUploadPath('materials', zipFileName);
console.log(`[Zip] Creating ${zipType} zip file for ${materialType}:`, zipFilePath);
console.log(`[Zip] Files count:`, files.length);
@ -153,17 +157,23 @@ export async function createZipFromFiles(
});
// 监听完成事件
output.on('close', () => {
output.on('close', async () => {
clearTimeout(timeout);
const fileSize = archive.pointer();
console.log(`[Zip] ${zipType} zip file created successfully: ${fileSize} bytes`);
cleanup();
resolve({
filePath: relativeZipPath,
fileName: zipFileName,
fileSize
});
try {
await syncStoredUploadFileToRemote(zipFilePath, relativeZipPath, fileSize);
cleanup();
resolve({
filePath: relativeZipPath,
fileName: zipFileName,
fileSize
});
} catch (error) {
cleanup();
reject(error);
}
});
// 监听错误
@ -230,7 +240,7 @@ export async function createZipFromZips(
const { access } = await import('fs/promises');
for (const file of zipFiles) {
try {
const fullPath = path.isAbsolute(file.path) ? file.path : path.join(process.cwd(), file.path);
const fullPath = path.isAbsolute(file.path) ? file.path : resolveStoredUploadPath(file.path);
await access(fullPath);
} catch (error) {
throw new Error(`文件不存在: ${file.path}`);
@ -238,7 +248,7 @@ export async function createZipFromZips(
}
// 创建目标目录
const uploadsDir = path.join(process.cwd(), 'uploads', 'materials');
const uploadsDir = path.join(getUploadRootDir(), 'materials');
await mkdir(uploadsDir, { recursive: true });
// 生成zip文件名和路径
@ -246,7 +256,7 @@ export async function createZipFromZips(
const sanitizedProductModel = productModel.replace(/[\/\\:*?"<>|]/g, '_');
const zipFileName = `${sanitizedProductModel}_all_${materialId}_${timestamp}.zip`;
const zipFilePath = path.join(uploadsDir, zipFileName);
const relativeZipPath = path.join('uploads', 'materials', zipFileName);
const relativeZipPath = buildStoredUploadPath('materials', zipFileName);
console.log(`[Zip] Creating all.zip for ${materialType}:`, zipFilePath);
console.log(`[Zip] Zip files count:`, zipFiles.length);
@ -285,17 +295,23 @@ export async function createZipFromZips(
zlib: { level: 9 } // 最高压缩级别
});
output.on('close', () => {
output.on('close', async () => {
clearTimeout(timeout);
const fileSize = archive.pointer();
console.log(`[Zip] all.zip created successfully: ${fileSize} bytes`);
cleanup();
resolve({
filePath: relativeZipPath,
fileName: zipFileName,
fileSize
});
try {
await syncStoredUploadFileToRemote(zipFilePath, relativeZipPath, fileSize);
cleanup();
resolve({
filePath: relativeZipPath,
fileName: zipFileName,
fileSize
});
} catch (error) {
cleanup();
reject(error);
}
});
archive.on('error', (err: Error) => {
@ -326,7 +342,7 @@ export async function createZipFromZips(
// 只添加zip文件,不添加PSD文件
for (const file of zipFiles) {
const fullPath = path.isAbsolute(file.path) ? file.path : path.join(process.cwd(), file.path);
const fullPath = path.isAbsolute(file.path) ? file.path : resolveStoredUploadPath(file.path);
console.log(`[Zip] Adding zip file: ${file.name}`);
archive.file(fullPath, { name: file.name });
}
@ -475,6 +491,7 @@ export async function cleanupTempFiles(
// 清理Redis上传会话(关键修复:防止孤立会话堆积)
if (uploadIds.length > 0) {
console.log(`[Cleanup] Cleaning up ${uploadIds.length} Redis upload sessions...`);
const { deleteUploadSession } = await import('@/lib/upload-sessions-redis');
for (const uploadId of uploadIds) {
try {
await deleteUploadSession(uploadId);
@ -516,7 +533,13 @@ export async function cleanupOnError(
console.log(`[Cleanup] Cleaning up ${paths.length} local zip files...`);
for (const zipFilePath of paths) {
try {
const fullPath = path.join(process.cwd(), zipFilePath);
await deleteStoredUploadFileFromRemote(zipFilePath);
} catch (cleanupError) {
console.error('[Cleanup] Failed to delete remote uploads file:', cleanupError);
}
try {
const fullPath = resolveStoredUploadPath(zipFilePath);
await unlink(fullPath);
console.log(`[Cleanup] Deleted local zip file: ${fullPath}`);
} catch (cleanupError) {

26
lib/upload-paths.ts Normal file
View File

@ -0,0 +1,26 @@
import path from 'path'
export function buildStoredUploadPath(...segments: string[]) {
return path.posix.join('uploads', ...segments.map((segment) => String(segment || '').replace(/\\/g, '/')))
}
export function getUploadRootDir(): string {
const configuredUploadDir = String(process.env.UPLOAD_DIR || '').trim()
if (!configuredUploadDir) {
return path.join(process.cwd(), 'uploads')
}
return path.isAbsolute(configuredUploadDir)
? configuredUploadDir
: path.resolve(process.cwd(), configuredUploadDir)
}
export function resolveStoredUploadPath(storedPath: string): string {
const normalizedPath = String(storedPath || '').replace(/\\/g, '/')
if (path.isAbsolute(normalizedPath)) {
return normalizedPath
}
const relativeToUploadRoot = normalizedPath.replace(/^\.?\/*uploads\//i, '')
return path.join(getUploadRootDir(), relativeToUploadRoot)
}

View File

@ -1,6 +1,6 @@
# 脚本说明
> 更新时间:2026-05-07
> 更新时间:2026-05-08
> 部署相关脚本以 `scripts/git-incremental-deploy.ps1` 为当前默认口径。
详细说明:
@ -102,3 +102,67 @@
### `scripts/execute-migration.sh`
Linux / macOS 版本的一次性迁移工具,适用范围同上。
## 5. 历史素材 SPU 台账与导入脚本
### `scripts/scan_spu_ledger.py` ⭐ 当前历史素材新入口
按 SPU / 组合 SPU / 版本工作区维度扫描真实素材目录,生成 SQLite 台账。
重要口径:
- 不读取旧的 SKU 锚点 CSV/SQLite 输出。
- 组合 SPU 原样保留,例如 `GBSFY-AD0297+GBSFY-CZ0102`。
- 目录名不是标准 SPU 时,即使文件名能反推,也只标记 `needs_review`。
- 店铺归属按当前业务口径:`金牌项目-1` 和 `GBJP-*` 归到 `顾家`,不再使用 `金牌` 店铺。
- `正版`、`盗版` 都允许进入候选;`错误勿用`、`废稿`、`勿用` 默认跳过。
- ready 要求同时存在主图、SKU 图、详情图。
示例:
```powershell
python scripts\scan_spu_ledger.py `
--roots "Z:\顾家项目|Z:\金牌项目-1|Z:\朴美项目|Z:\索菲亚项目" `
--db-path "output\tianlan-spu-ledger.db" `
--source-batch "tianlan-z" `
--reset
```
### `scripts/export_spu_ledger.py`
从 SPU 台账导出 review CSV、每 20 个工作区一个批次 CSV、汇总 JSON,以及可供历史导入脚本使用的 ready JSON。
```powershell
python scripts\export_spu_ledger.py `
--db-path "output\tianlan-spu-ledger.db" `
--output-dir "output" `
--prefix "tianlan-spu-ledger"
```
### `scripts/import-history-materials.ts` / `scripts/run-history-import-queue.ts`
继续作为历史素材导入执行入口。新的 SPU 台账 ready JSON 可以直接作为输入,并支持 `任务-详情` 的 PS 源文件通道。
正式导入的关键边界:
- Windows 维护机可以读取 `Z:\...` 源素材,但不能只靠本机 `uploads` 写生产库。
- 连接生产库并执行 `--apply` 时,预览图必须写入新桶 `gongxiangpan`,公开地址必须是 `https://s3.jiansun.vip/gongxiangpan/由壹点AI/public-display/...`。
- ZIP / PSD / 源文件必须最终同步到生产机 `/mnt/PoolOne/apps/youyidian-3d/uploads`,并通过大小校验。
- 在 Windows 上正式导入生产库时,必须启用:
```env
HISTORY_IMPORT_REMOTE_UPLOADS_HOST=prod
HISTORY_IMPORT_REMOTE_UPLOADS_DIR=/mnt/PoolOne/apps/youyidian-3d/uploads
```
没有远程 uploads 同步配置时,脚本会阻止 `--apply`,避免再次出现数据库有记录但线上下载 404。
### `scripts/hard-delete-history-materials.ts`
只用于已经确认的旧批次错误素材硬删除。默认只做备份/演练;正式执行必须同时满足:
- 候选清单里有 `materialId` 和删除原因。
- 候选清单提供的 `fingerprint` 或 `sourcePath` 能在该素材的 `HISTORY_IMPORT` 日志中匹配。
- 命令包含 `--apply --confirm=HARD_DELETE_HISTORY_MATERIALS`。
未确认清单前禁止执行正式硬删除。

View File

@ -0,0 +1,233 @@
#!/usr/bin/env python3
"""
Build HistoryImportRecord JSON from reviewed package candidates.
This script is read-only. It scans selected package paths and emits the JSON
shape consumed by scripts/import-history-materials.ts. Formal import should
still be done through that TypeScript script so environment guards, duplicate
checks and validation stay in one place.
"""
from __future__ import annotations
import argparse
import csv
import json
import os
from pathlib import Path
from typing import Any
from scan_spu_ledger import (
EXCLUDED_DIR_NAMES,
RELEVANT_EXTENSIONS,
build_sku_code,
classify_file,
extract_spu_candidates,
fingerprint_for,
infer_author,
infer_task_detail_source_type,
is_compatible_spu_token,
load_visual_author_names,
normalize_spu,
safe_relative,
spu_prefix,
)
DEFAULT_ROOTS = [
r"Z:\顾家项目",
r"Z:\金牌项目-1",
r"Z:\朴美项目",
r"Z:\索菲亚项目",
]
KNOWN_AUTO_PREFIXES = {"GBPM", "GBSFY", "GBGJ", "GBJP"}
def json_dumps(value: Any) -> str:
return json.dumps(value, ensure_ascii=False, indent=2)
def norm_path(value: str) -> str:
return os.path.normcase(os.path.normpath(value))
def find_source_root(package_path: str, roots: list[str]) -> str:
package_norm = norm_path(package_path)
matches = [root for root in roots if package_norm.startswith(norm_path(root) + os.sep.lower()) or package_norm == norm_path(root)]
if not matches:
raise ValueError(f"无法识别 source_root: {package_path}")
return max(matches, key=len)
def image_input(file_path: Path, package_root: Path, group: str, spu: str, index: int) -> dict[str, Any]:
payload: dict[str, Any] = {
"localFilePath": str(file_path),
"fileName": file_path.name,
"fileSize": int(file_path.stat().st_size),
"sortOrder": index,
}
if group == "sku":
payload["skuCode"] = build_sku_code(spu, file_path.name, index)
payload["barcodes"] = []
return payload
def source_input(file_path: Path) -> dict[str, Any]:
return {
"localFilePath": str(file_path),
"fileName": file_path.name,
"fileSize": int(file_path.stat().st_size),
}
def collect_assets(package_path: str, spu: str) -> dict[str, list[dict[str, Any]]]:
package_root = Path(package_path)
grouped_files: dict[str, list[Path]] = {"main": [], "sku": [], "detail": [], "source": []}
for current, dir_names, file_names in os.walk(package_root):
dir_names[:] = [name for name in dir_names if name not in EXCLUDED_DIR_NAMES]
current_path = Path(current)
for file_name in file_names:
file_path = current_path / file_name
extension = file_path.suffix.lower()
if extension not in RELEVANT_EXTENSIONS:
continue
relative_path = safe_relative(file_path, package_root)
group = classify_file(relative_path, file_name, extension)
if group in grouped_files:
grouped_files[group].append(file_path)
for group in grouped_files:
grouped_files[group].sort(key=lambda item: safe_relative(item, package_root).lower())
return {
"mainImages": [image_input(path, package_root, "main", spu, index) for index, path in enumerate(grouped_files["main"])],
"skuImages": [image_input(path, package_root, "sku", spu, index) for index, path in enumerate(grouped_files["sku"])],
"detailImages": [image_input(path, package_root, "detail", spu, index) for index, path in enumerate(grouped_files["detail"])],
"psdSources": [source_input(path) for path in grouped_files["source"]],
}
def build_record(row: dict[str, str], roots: list[str], visual_author_names: list[str]) -> dict[str, Any]:
package_path = row["package_path"]
spu = normalize_spu(row["spu"])
source_root = find_source_root(package_path, roots)
relative_source_path = safe_relative(Path(package_path), Path(source_root))
assets = collect_assets(package_path, spu)
author, _author_evidence = infer_author(relative_source_path, visual_author_names)
source_type, _source_type_evidence = infer_task_detail_source_type(relative_source_path)
return {
"fingerprint": fingerprint_for(source_root, package_path, spu),
"spu": spu,
"sourcePath": package_path,
"relativeSourcePath": relative_source_path,
"guessedStore": row["store"],
"productCategoryName": row["product_category"],
"authorName": author or "历史导入",
"taskDetailSourceType": source_type or "渲染",
"storeConfidence": 1,
"storeCandidates": [row["store"]] if row.get("store") else [],
"mainCount": len(assets["mainImages"]),
"skuCount": len(assets["skuImages"]),
"detailCount": len(assets["detailImages"]),
"sourceCount": len(assets["psdSources"]),
"status": "ready",
"skipReason": None,
**assets,
"seenInYya": False,
"ledger": {
"source": "needs-review-package-candidates",
"decision": row.get("decision"),
"cautionReason": row.get("caution_reason") or "",
},
}
def safety_reason(row: dict[str, str], duplicate_paths: set[str]) -> str:
spu = normalize_spu(row["spu"])
package_path = row["package_path"]
prefix = spu_prefix(spu)
if prefix not in KNOWN_AUTO_PREFIXES:
return f"SPU 前缀不在自动导入白名单: {prefix or '(none)'}"
if os.path.normcase(os.path.normpath(package_path)) in duplicate_paths:
return "同一个 package_path 被识别为多个 SPU,需人工确认后再导入"
path_tokens = extract_spu_candidates(package_path, allow_unbranded=False)
incompatible_tokens = [
token for token in path_tokens if not is_compatible_spu_token(spu, token)
]
if incompatible_tokens:
return "package_path 包含不兼容 SPU: " + ",".join(incompatible_tokens[:6])
return ""
def main() -> int:
parser = argparse.ArgumentParser(description="Build import JSON from package candidates.")
parser.add_argument("--input-csv", required=True)
parser.add_argument("--output", required=True)
parser.add_argument("--decision", default="package_ready_review")
parser.add_argument("--limit", type=int, default=0)
parser.add_argument("--visual-authors-file", default="output/tianlan-spu-ledger/visual-authors.tsv")
parser.add_argument("--roots", nargs="*", default=DEFAULT_ROOTS)
args = parser.parse_args()
visual_author_names = (
load_visual_author_names(args.visual_authors_file)
if args.visual_authors_file and Path(args.visual_authors_file).exists()
else []
)
with Path(args.input_csv).open("r", encoding="utf-8-sig", newline="") as handle:
candidate_rows = [row for row in csv.DictReader(handle) if row.get("decision") == args.decision]
path_to_spus: dict[str, set[str]] = {}
for row in candidate_rows:
path_key = os.path.normcase(os.path.normpath(row["package_path"]))
path_to_spus.setdefault(path_key, set()).add(normalize_spu(row["spu"]))
duplicate_paths = {path_key for path_key, spus in path_to_spus.items() if len(spus) > 1}
skipped_rows: list[dict[str, str]] = []
rows: list[dict[str, str]] = []
for row in candidate_rows:
reason = safety_reason(row, duplicate_paths)
if reason:
skipped_rows.append({**row, "safety_skip_reason": reason})
else:
rows.append(row)
if args.limit > 0:
rows = rows[: args.limit]
records = [build_record(row, args.roots, visual_author_names) for row in rows]
output_path = Path(args.output)
output_path.parent.mkdir(parents=True, exist_ok=True)
output_path.write_text(json_dumps(records), encoding="utf-8")
skipped_path = output_path.with_suffix(".skipped-candidates.json")
skipped_path.write_text(json_dumps(skipped_rows), encoding="utf-8")
print(
json_dumps(
{
"inputCsv": args.input_csv,
"decision": args.decision,
"candidateRows": len(candidate_rows),
"safetySkipped": len(skipped_rows),
"records": len(records),
"output": str(output_path),
"skippedCandidates": str(skipped_path),
"totalMain": sum(record["mainCount"] for record in records),
"totalSku": sum(record["skuCount"] for record in records),
"totalDetail": sum(record["detailCount"] for record in records),
"totalSources": sum(record["sourceCount"] for record in records),
}
)
)
return 0
if __name__ == "__main__":
raise SystemExit(main())

View File

@ -0,0 +1,340 @@
#!/usr/bin/env python3
"""
Discover minimal package candidates inside needs_review SPU roots.
This is a read-only second pass. It is stricter than grouping by SPU root: a
candidate package is the smallest directory subtree that contains main, SKU and
detail assets. That handles cross-level asset folders while avoiding merging
multiple versions under the same SPU root.
"""
from __future__ import annotations
import argparse
import csv
import json
import os
import re
import sqlite3
from collections import Counter, defaultdict
from datetime import datetime
from pathlib import Path
from typing import Any
from review_needs_review_spu_groups import choose_spu_root_path
from scan_spu_ledger import (
EXCLUDED_DIR_NAMES,
RELEVANT_EXTENSIONS,
build_sku_code,
classify_file,
extract_file_spu_candidates,
has_hard_skip,
is_compatible_spu_token,
normalize_spu,
safe_relative,
)
CAUTION_RE = re.compile(
r"(^|[\\/])3D($|[\\/])|3D建模|3D原图|3D渲染原图|建模|建模图|建模原图|渲染原图|原图建模|"
r"源文件|psd源文件|PSD源文件|PS源文件|"
r"大图|AI合成|AI原图|图片有误|错误勿用|勿用|废稿|作废|弃用|不要",
re.IGNORECASE,
)
HEADERS = [
"decision",
"spu",
"store",
"product_category",
"package_path",
"relative_package_path",
"main_count",
"sku_count",
"detail_count",
"source_count",
"needs_review_rows_under_root",
"caution_reason",
"related_workspace_samples",
"main_samples",
"sku_samples",
"detail_samples",
]
def json_dumps(value: Any) -> str:
return json.dumps(value, ensure_ascii=False, indent=2)
def latest_run_id(conn: sqlite3.Connection) -> str:
row = conn.execute(
"SELECT run_id FROM spu_ledger_runs ORDER BY started_at DESC LIMIT 1"
).fetchone()
if not row:
raise SystemExit("台账数据库里没有扫描 run")
return str(row["run_id"])
class Node:
def __init__(self, path: str, root: str) -> None:
self.path = path
self.root = root
self.children: list[str] = []
self.files: list[dict[str, Any]] = []
self.direct_counts: Counter[str] = Counter()
self.subtree_counts: Counter[str] = Counter()
self.samples: dict[str, list[dict[str, Any]]] = defaultdict(list)
self.has_caution = False
self.conflicts: list[dict[str, str]] = []
def scan_tree(root_path: str, spu: str) -> dict[str, Node]:
root = Path(root_path)
nodes: dict[str, Node] = {}
if not root.exists():
return nodes
for current, dir_names, file_names in os.walk(root):
current_path = Path(current)
current_str = str(current_path)
node = nodes.setdefault(current_str, Node(current_str, root_path))
node.has_caution = node.has_caution or bool(CAUTION_RE.search(current_str))
kept_dirs: list[str] = []
for name in dir_names:
child = current_path / name
if name in EXCLUDED_DIR_NAMES or has_hard_skip(str(child)):
continue
kept_dirs.append(name)
child_str = str(child)
node.children.append(child_str)
nodes.setdefault(child_str, Node(child_str, root_path))
dir_names[:] = kept_dirs
for file_name in file_names:
file_path = current_path / file_name
extension = file_path.suffix.lower()
if extension not in RELEVANT_EXTENSIONS or has_hard_skip(str(file_path)):
continue
try:
file_size = file_path.stat().st_size
except OSError:
continue
relative_path = safe_relative(file_path, root)
group = classify_file(relative_path, file_name, extension)
record = {
"filePath": str(file_path),
"relativePath": relative_path,
"fileName": file_name,
"fileSize": int(file_size),
"group": group,
}
if group == "sku":
record["skuCode"] = build_sku_code(spu, file_name, node.direct_counts[group])
record["barcodes"] = []
node.files.append(record)
node.direct_counts[group] += 1
if group in {"main", "sku", "detail", "source"} and len(node.samples[group]) < 8:
node.samples[group].append(record)
if bool(CAUTION_RE.search(str(file_path))):
node.has_caution = True
if group in {"main", "detail"}:
tokens = extract_file_spu_candidates(file_name)
if tokens and not any(is_compatible_spu_token(spu, token) for token in tokens):
node.conflicts.append(
{
"group": group,
"file": relative_path,
"tokens": ",".join(tokens),
}
)
for path in sorted(nodes, key=lambda item: len(Path(item).parts), reverse=True):
node = nodes[path]
node.subtree_counts.update(node.direct_counts)
for group, records in node.samples.items():
node.samples[group] = records[:8]
for child_path in node.children:
child = nodes[child_path]
node.subtree_counts.update(child.subtree_counts)
node.has_caution = node.has_caution or child.has_caution
if len(node.conflicts) < 12:
node.conflicts.extend(child.conflicts[: max(0, 12 - len(node.conflicts))])
for group, records in child.samples.items():
existing = node.samples[group]
for record in records:
if len(existing) >= 8:
break
existing.append(record)
return nodes
def has_all_three(node: Node) -> bool:
return (
node.subtree_counts["main"] > 0
and node.subtree_counts["sku"] > 0
and node.subtree_counts["detail"] > 0
)
def minimal_package_nodes(nodes: dict[str, Node]) -> list[Node]:
candidates = [node for node in nodes.values() if has_all_three(node)]
candidate_paths = {node.path for node in candidates}
minimal: list[Node] = []
for node in candidates:
descendant_has_all_three = False
stack = list(node.children)
while stack:
child_path = stack.pop()
if child_path in candidate_paths:
descendant_has_all_three = True
break
child = nodes.get(child_path)
if child:
stack.extend(child.children)
if not descendant_has_all_three:
minimal.append(node)
minimal.sort(key=lambda item: item.path.lower())
return minimal
def package_decision(node: Node) -> tuple[str, str]:
reasons: list[str] = []
if node.has_caution:
reasons.append("路径或文件名包含建模/源文件/大图/AI原图/错误等谨慎标识")
if node.conflicts:
tokens = sorted({conflict["tokens"] for conflict in node.conflicts})[:4]
reasons.append("主图/详情文件名出现不同SPU: " + ";".join(tokens))
if node.subtree_counts["main"] == 0 or node.subtree_counts["sku"] == 0 or node.subtree_counts["detail"] == 0:
reasons.append("缺少主图/SKU/详情之一")
if reasons:
return "package_caution_review", " | ".join(reasons)
return "package_ready_review", ""
def row_for(node: Node, item_group: list[sqlite3.Row], spu: str, store: str, category: str) -> dict[str, Any]:
package = Path(node.path)
root = Path(node.root)
decision, caution_reason = package_decision(node)
related = [
str(item["workspace_path"])
for item in item_group
if str(item["workspace_path"]).lower().startswith(node.path.lower())
][:8]
if not related:
related = [str(item["workspace_path"]) for item in item_group[:8]]
return {
"decision": decision,
"spu": spu,
"store": store,
"product_category": category,
"package_path": str(package),
"relative_package_path": safe_relative(package, root),
"main_count": node.subtree_counts["main"],
"sku_count": node.subtree_counts["sku"],
"detail_count": node.subtree_counts["detail"],
"source_count": node.subtree_counts["source"],
"needs_review_rows_under_root": len(item_group),
"caution_reason": caution_reason,
"related_workspace_samples": " || ".join(related),
"main_samples": " || ".join(record["relativePath"] for record in node.samples.get("main", [])),
"sku_samples": " || ".join(record["relativePath"] for record in node.samples.get("sku", [])),
"detail_samples": " || ".join(record["relativePath"] for record in node.samples.get("detail", [])),
}
def write_csv(path: Path, rows: list[dict[str, Any]]) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
with path.open("w", encoding="utf-8-sig", newline="") as handle:
writer = csv.DictWriter(handle, fieldnames=HEADERS)
writer.writeheader()
writer.writerows(rows)
def main() -> int:
parser = argparse.ArgumentParser(description="Discover package candidates for needs_review rows.")
parser.add_argument("--db-path", default="output/tianlan-spu-ledger.db")
parser.add_argument("--run-id", default="")
parser.add_argument("--output-dir", default="")
args = parser.parse_args()
stamp = datetime.now().strftime("%Y%m%d-%H%M%S")
output_dir = Path(args.output_dir or f"output/tianlan-spu-ledger/needs-review-package-candidates-{stamp}")
output_dir.mkdir(parents=True, exist_ok=True)
conn = sqlite3.connect(str(Path(args.db_path)))
conn.row_factory = sqlite3.Row
run_id = args.run_id or latest_run_id(conn)
items = conn.execute(
"""
SELECT *
FROM spu_ledger_items
WHERE run_id = ? AND status = 'needs_review'
ORDER BY source_root_name, relative_workspace_path, spu
""",
(run_id,),
).fetchall()
groups: dict[tuple[str, str, str, str], list[sqlite3.Row]] = defaultdict(list)
for item in items:
spu = normalize_spu(item["spu"])
root_path = choose_spu_root_path(item)
key = (str(item["store"] or ""), spu, str(item["product_category"] or ""), root_path)
groups[key].append(item)
rows: list[dict[str, Any]] = []
detail_payload: list[dict[str, Any]] = []
roots_without_package = 0
for (store, spu, category, root_path), item_group in sorted(groups.items()):
nodes = scan_tree(root_path, spu)
packages = minimal_package_nodes(nodes)
if not packages:
roots_without_package += 1
continue
for node in packages:
row = row_for(node, item_group, spu, store, category)
rows.append(row)
detail_payload.append(
{
**row,
"samples": node.samples,
"conflicts": node.conflicts[:12],
"workspaces": [
{
"fingerprint": item["fingerprint"],
"workspacePath": item["workspace_path"],
"relativeWorkspacePath": item["relative_workspace_path"],
"versionLabel": item["version_label"],
"mainCount": item["main_count"],
"skuCount": item["sku_count"],
"detailCount": item["detail_count"],
"statusReason": item["status_reason"],
}
for item in item_group
],
}
)
write_csv(output_dir / "package-candidates.csv", rows)
(output_dir / "package-candidates.json").write_text(json_dumps(detail_payload), encoding="utf-8")
summary = {
"runId": run_id,
"needsReviewItems": len(items),
"spuRootGroups": len(groups),
"rootsWithoutPackage": roots_without_package,
"packageCandidates": len(rows),
"decisionCounts": dict(Counter(row["decision"] for row in rows)),
"outputCsv": str(output_dir / "package-candidates.csv"),
"outputJson": str(output_dir / "package-candidates.json"),
}
(output_dir / "summary.json").write_text(json_dumps(summary), encoding="utf-8")
print(json_dumps(summary))
conn.close()
return 0
if __name__ == "__main__":
raise SystemExit(main())

View File

@ -0,0 +1,76 @@
const fs = require('fs')
const path = require('path')
const inputPath = process.argv[2] || path.join('output', 'tianlan-history-sku-import-scan-full.json')
const outputDir = process.argv[3] || path.join('output')
const baseName = process.argv[4] || 'tianlan-history-sku-import'
function escapeCsvCell(value) {
const text = String(value ?? '')
if (/[",\r\n]/.test(text)) {
return `"${text.replace(/"/g, '""')}"`
}
return text
}
function writeCsv(filePath, rows) {
const headers = [
'status(状态)',
'spu(SPU)',
'store(店铺)',
'product_category(产品品类)',
'author(作者)',
'main_count(主图数量)',
'sku_count(SKU图数量)',
'detail_count(详情图数量)',
'skip_reason(跳过或复核原因)',
'source_path(来源完整路径)',
'relative_source_path(来源相对路径)',
]
const lines = [
headers.join(','),
...rows.map((record) =>
[
record.status,
record.spu,
record.guessedStore,
record.productCategoryName,
record.authorName,
record.mainCount,
record.skuCount,
record.detailCount,
record.skipReason || '',
record.sourcePath,
record.relativeSourcePath,
]
.map(escapeCsvCell)
.join(','),
),
]
fs.writeFileSync(filePath, `\uFEFF${lines.join('\r\n')}`, 'utf8')
}
function countBy(records, picker) {
return records.reduce((acc, record) => {
const key = picker(record) || '<empty>'
acc[key] = (acc[key] || 0) + 1
return acc
}, {})
}
const records = JSON.parse(fs.readFileSync(inputPath, 'utf8'))
fs.mkdirSync(outputDir, { recursive: true })
writeCsv(path.join(outputDir, `${baseName}-ready.csv`), records.filter((record) => record.status === 'ready'))
writeCsv(path.join(outputDir, `${baseName}-needs-review.csv`), records.filter((record) => record.status === 'needs_review'))
writeCsv(path.join(outputDir, `${baseName}-skipped.csv`), records.filter((record) => record.status === 'skipped'))
const summary = {
total: records.length,
status: countBy(records, (record) => record.status),
store: countBy(records, (record) => record.guessedStore),
productCategory: countBy(records, (record) => record.productCategoryName),
}
fs.writeFileSync(path.join(outputDir, `${baseName}-summary.json`), JSON.stringify(summary, null, 2), 'utf8')
console.log(JSON.stringify(summary, null, 2))

View File

@ -10,6 +10,7 @@ EXPORT_HEADERS = [
("spu", "spu(SPU)"),
("directory_name", "directory_name(目录名称)"),
("relative_path", "relative_path(相对路径)"),
("windows_path", "windows_path(Windows路径)"),
("sku_dir_path", "sku_dir_path(SKU目录完整路径)"),
("parent_path", "parent_path(上级目录)"),
("grandparent_path", "grandparent_path(上上级目录)"),
@ -83,6 +84,7 @@ def main():
spu,
directory_name,
relative_path,
sku_dir_path AS windows_path,
sku_dir_path,
parent_path,
grandparent_path,

View File

@ -0,0 +1,259 @@
#!/usr/bin/env python3
"""
Export the SPU ledger into review CSV files and ready import JSON.
"""
from __future__ import annotations
import argparse
import csv
import json
import sqlite3
from pathlib import Path
STATUS_EXPORTS = {
"ready": "ready",
"needs_review": "needs-review",
"skipped": "skipped",
"duplicate": "duplicate",
"delete_candidate": "delete-candidate",
}
def json_dumps(value) -> str:
return json.dumps(value, ensure_ascii=False, indent=2)
def latest_run_id(conn: sqlite3.Connection) -> str:
row = conn.execute(
"SELECT run_id FROM spu_ledger_runs ORDER BY started_at DESC LIMIT 1"
).fetchone()
if not row:
raise SystemExit("台账数据库里没有扫描 run")
return str(row["run_id"])
def read_items(conn: sqlite3.Connection, run_id: str) -> list[sqlite3.Row]:
return conn.execute(
"""
SELECT *
FROM spu_ledger_items
WHERE run_id = ?
ORDER BY source_root_name, relative_workspace_path, spu
""",
(run_id,),
).fetchall()
def read_files(conn: sqlite3.Connection, item_id: int, asset_group: str) -> list[sqlite3.Row]:
return conn.execute(
"""
SELECT *
FROM spu_ledger_files
WHERE item_id = ? AND asset_group = ?
ORDER BY sort_order, relative_file_path
""",
(item_id, asset_group),
).fetchall()
def csv_headers() -> list[str]:
return [
"status(状态)",
"spu(SPU)",
"store(店铺)",
"product_category(产品品类)",
"author(作者)",
"task_detail_source_type(出图方式)",
"version_label(版本/工作区)",
"main_count(主图数量)",
"sku_count(SKU图数量)",
"detail_count(详情图数量)",
"source_count(PS源文件数量)",
"status_reason(判断原因)",
"spu_source(SPU来源)",
"store_evidence(店铺证据)",
"author_evidence(作者证据)",
"task_detail_source_type_evidence(出图方式证据)",
"source_batch(来源批次)",
"source_root(来源根目录)",
"workspace_path(工作区完整路径)",
"relative_workspace_path(工作区相对路径)",
"fingerprint(台账指纹)",
]
def item_to_csv_row(item: sqlite3.Row) -> dict[str, object]:
return {
"status(状态)": item["status"],
"spu(SPU)": item["spu"],
"store(店铺)": item["store"],
"product_category(产品品类)": item["product_category"],
"author(作者)": item["author"],
"task_detail_source_type(出图方式)": item["task_detail_source_type"],
"version_label(版本/工作区)": item["version_label"],
"main_count(主图数量)": item["main_count"],
"sku_count(SKU图数量)": item["sku_count"],
"detail_count(详情图数量)": item["detail_count"],
"source_count(PS源文件数量)": item["source_count"],
"status_reason(判断原因)": item["status_reason"],
"spu_source(SPU来源)": item["spu_source"],
"store_evidence(店铺证据)": item["store_evidence"],
"author_evidence(作者证据)": item["author_evidence"],
"task_detail_source_type_evidence(出图方式证据)": item["task_detail_source_type_evidence"],
"source_batch(来源批次)": item["source_batch"],
"source_root(来源根目录)": item["source_root"],
"workspace_path(工作区完整路径)": item["workspace_path"],
"relative_workspace_path(工作区相对路径)": item["relative_workspace_path"],
"fingerprint(台账指纹)": item["fingerprint"],
}
def write_csv(path: Path, rows: list[dict[str, object]], headers: list[str]) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
with path.open("w", encoding="utf-8-sig", newline="") as handle:
writer = csv.DictWriter(handle, fieldnames=headers)
writer.writeheader()
for row in rows:
writer.writerow(row)
def to_image_input(file_row: sqlite3.Row) -> dict[str, object]:
payload = {
"localFilePath": file_row["file_path"],
"fileName": file_row["file_name"],
"fileSize": int(file_row["file_size"] or 0),
"sortOrder": int(file_row["sort_order"] or 0),
}
if file_row["asset_group"] == "sku":
payload["skuCode"] = file_row["sku_code"] or ""
payload["barcodes"] = []
return payload
def to_source_input(file_row: sqlite3.Row) -> dict[str, object]:
return {
"localFilePath": file_row["file_path"],
"fileName": file_row["file_name"],
"fileSize": int(file_row["file_size"] or 0),
}
def build_import_record(conn: sqlite3.Connection, item: sqlite3.Row) -> dict[str, object]:
main = [to_image_input(row) for row in read_files(conn, item["id"], "main")]
sku = [to_image_input(row) for row in read_files(conn, item["id"], "sku")]
detail = [to_image_input(row) for row in read_files(conn, item["id"], "detail")]
sources = [to_source_input(row) for row in read_files(conn, item["id"], "source")]
return {
"fingerprint": item["fingerprint"],
"spu": item["spu"],
"sourcePath": item["workspace_path"],
"relativeSourcePath": item["relative_workspace_path"],
"guessedStore": item["store"],
"productCategoryName": item["product_category"],
"authorName": item["author"] or "历史导入",
"taskDetailSourceType": item["task_detail_source_type"] or "渲染",
"storeConfidence": 1,
"storeCandidates": [item["store"]] if item["store"] else [],
"mainCount": item["main_count"],
"skuCount": item["sku_count"],
"detailCount": item["detail_count"],
"sourceCount": item["source_count"],
"status": item["status"],
"skipReason": None if item["status"] == "ready" else item["status_reason"],
"mainImages": main,
"skuImages": sku,
"detailImages": detail,
"psdSources": sources,
"seenInYya": False,
"ledger": {
"runId": item["run_id"],
"sourceBatch": item["source_batch"],
"spuSource": item["spu_source"],
"spuEvidence": json.loads(item["spu_evidence_json"] or "[]"),
"pathFlags": json.loads(item["path_flags_json"] or "{}"),
},
}
def write_batch_csvs(output_dir: Path, prefix: str, items: list[sqlite3.Row], batch_size: int) -> None:
if batch_size <= 0:
return
headers = csv_headers()
pending = [item for item in items if item["status"] in ("ready", "needs_review")]
for start in range(0, len(pending), batch_size):
batch_no = start // batch_size + 1
batch = pending[start : start + batch_size]
rows = [item_to_csv_row(item) for item in batch]
write_csv(output_dir / f"{prefix}-batch-{batch_no:03d}.csv", rows, headers)
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(description="Export SPU ledger review files.")
parser.add_argument("--db-path", required=True)
parser.add_argument("--output-dir", default=str(Path.cwd() / "output"))
parser.add_argument("--prefix", default="spu-ledger")
parser.add_argument("--run-id", default="")
parser.add_argument("--batch-size", type=int, default=20)
return parser.parse_args()
def main() -> int:
args = parse_args()
db_path = Path(args.db_path)
output_dir = Path(args.output_dir)
output_dir.mkdir(parents=True, exist_ok=True)
conn = sqlite3.connect(str(db_path))
conn.row_factory = sqlite3.Row
run_id = args.run_id or latest_run_id(conn)
items = read_items(conn, run_id)
headers = csv_headers()
status_counts: dict[str, int] = {}
store_counts: dict[str, int] = {}
category_counts: dict[str, int] = {}
for item in items:
status_counts[item["status"]] = status_counts.get(item["status"], 0) + 1
store_counts[item["store"]] = store_counts.get(item["store"], 0) + 1
category = item["product_category"] or "未识别"
category_counts[category] = category_counts.get(category, 0) + 1
for status, suffix in STATUS_EXPORTS.items():
rows = [item_to_csv_row(item) for item in items if item["status"] == status]
write_csv(output_dir / f"{args.prefix}-{suffix}.csv", rows, headers)
write_csv(output_dir / f"{args.prefix}-all.csv", [item_to_csv_row(item) for item in items], headers)
write_batch_csvs(output_dir, args.prefix, items, args.batch_size)
ready_records = [build_import_record(conn, item) for item in items if item["status"] == "ready"]
ready_json_path = output_dir / f"{args.prefix}-ready-import-records.json"
ready_json_path.write_text(json_dumps(ready_records), encoding="utf-8")
summary = {
"runId": run_id,
"dbPath": str(db_path),
"total": len(items),
"status": status_counts,
"store": store_counts,
"productCategory": category_counts,
"readyImportRecords": str(ready_json_path),
}
summary_path = output_dir / f"{args.prefix}-summary.json"
summary_path.write_text(json_dumps(summary), encoding="utf-8")
conn.close()
print("SPU 台账导出完成")
print(f"- run_id: {run_id}")
print(f"- total: {len(items)}")
print(f"- ready_import_records: {ready_json_path}")
print(f"- summary: {summary_path}")
return 0
if __name__ == "__main__":
raise SystemExit(main())

View File

@ -0,0 +1,148 @@
#!/usr/bin/env python3
"""
Filter dry-run import records from needs_review package candidates.
This is a conservative, read-only guard used after import-history-materials
dry-run. It keeps only records that still look like a complete package root
instead of an asset subfolder or ambiguous multi-SPU package.
"""
from __future__ import annotations
import argparse
import json
import re
from pathlib import Path
from typing import Any
MAIN_DIR_RE = re.compile(r"主图|main|^(750|800|1440|1920)$", re.IGNORECASE)
SKU_DIR_RE = re.compile(r"sku|白底|场景", re.IGNORECASE)
DETAIL_DIR_RE = re.compile(r"详情|detail", re.IGNORECASE)
FORBIDDEN_RE = re.compile(r"错误|勿用|废稿|图片有误|3D建模|3D原图|建模图|建模原图|渲染原图|AI原图|大图", re.IGNORECASE)
ASSET_SEGMENT_RE = re.compile(r"^(主图|sku|SKU|详情|详情页|白底|场景|750|800|790|1200|1440|1920|切图|800-800主图)$")
BRANDED_SPU_RE = re.compile(r"\b(GBPM|GBSFY|GBGJ|GBJP)-[A-Z]{1,8}\d{3,5}\b", re.IGNORECASE)
def load_json(path: Path) -> Any:
return json.loads(path.read_text(encoding="utf-8-sig"))
def dump_json(path: Path, value: Any) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(json.dumps(value, ensure_ascii=False, indent=2), encoding="utf-8")
def norm_spu(value: str) -> str:
return value.replace("-", "-").replace("—", "-").replace("–", "-").upper().strip()
def path_segments(path: str) -> list[str]:
normalized = path.replace("/", "\\")
return [part for part in normalized.split("\\") if part]
def child_dirs(path: str) -> list[str]:
root = Path(path)
if not root.exists():
return []
return sorted([item.name for item in root.iterdir() if item.is_dir()], key=str.lower)
def has_required_child_dirs(names: list[str]) -> bool:
has_main = any(MAIN_DIR_RE.search(name) for name in names)
has_sku = any(SKU_DIR_RE.search(name) for name in names)
has_detail = any(DETAIL_DIR_RE.search(name) for name in names)
return has_main and has_sku and has_detail
def first_branded_token(path: str) -> str:
match = BRANDED_SPU_RE.search(norm_spu(path))
return norm_spu(match.group(0)) if match else ""
def skip_reason(record: dict[str, Any]) -> str:
source_path = str(record.get("sourcePath") or "")
spu = norm_spu(str(record.get("spu") or ""))
segments = path_segments(source_path)
dirs = child_dirs(source_path)
if int(record.get("mainCount") or 0) <= 0:
return "mainCount 为 0,疑似把主图子目录误判为包根"
if int(record.get("skuCount") or 0) <= 0:
return "skuCount 为 0,不满足三类资产齐全"
if int(record.get("detailCount") or 0) <= 0:
return "detailCount 为 0,不满足三类资产齐全"
if FORBIDDEN_RE.search(source_path):
return "路径包含禁用/高风险目录词"
if "组合" in source_path and "+" not in spu:
return "组合目录未表达为组合 SPU,需人工确认"
if len(segments) >= 2 and ASSET_SEGMENT_RE.match(segments[-1]) and ASSET_SEGMENT_RE.match(segments[-2]):
return "候选路径落在资产子目录内,不是包根目录"
if len(segments) >= 1 and ASSET_SEGMENT_RE.match(segments[-1]) and not has_required_child_dirs(dirs):
return "候选路径末级像资产目录且首层缺少主图/SKU/详情"
if not has_required_child_dirs(dirs):
return "首层目录缺少明确的主图/SKU/详情三类证据"
token = first_branded_token(source_path)
if token and token != spu:
return f"路径首个品牌 SPU 为 {token},记录 SPU 为 {spu},需人工确认"
return ""
def main() -> int:
parser = argparse.ArgumentParser(description="Filter dry-run import records conservatively.")
parser.add_argument("--records", required=True, help="Input HistoryImportRecord JSON")
parser.add_argument("--result", required=True, help="Dry-run result JSON")
parser.add_argument("--output", required=True, help="Filtered records JSON")
parser.add_argument("--skipped-output", required=True, help="Skipped records JSON")
args = parser.parse_args()
records = load_json(Path(args.records))
results = load_json(Path(args.result))
dry_fingerprints = {item.get("fingerprint") for item in results if item.get("status") == "dry_run"}
selected: list[dict[str, Any]] = []
skipped: list[dict[str, Any]] = []
for record in records:
if record.get("fingerprint") not in dry_fingerprints:
continue
reason = skip_reason(record)
if reason:
skipped.append(
{
"fingerprint": record.get("fingerprint"),
"spu": record.get("spu"),
"sourcePath": record.get("sourcePath"),
"mainCount": record.get("mainCount"),
"skuCount": record.get("skuCount"),
"detailCount": record.get("detailCount"),
"sourceCount": record.get("sourceCount"),
"skipReason": reason,
}
)
else:
selected.append(record)
dump_json(Path(args.output), selected)
dump_json(Path(args.skipped_output), skipped)
print(
json.dumps(
{
"dryRunFingerprints": len(dry_fingerprints),
"selected": len(selected),
"skipped": len(skipped),
"output": args.output,
"skippedOutput": args.skipped_output,
},
ensure_ascii=False,
indent=2,
)
)
return 0
if __name__ == "__main__":
raise SystemExit(main())

View File

@ -0,0 +1,214 @@
import path from 'path'
import { promises as fs } from 'fs'
import { prisma } from '@/lib/prisma'
type DeleteCandidate = {
materialId: string
fingerprint?: string
sourcePath?: string
reason: string
}
function parseArgs() {
const args = process.argv.slice(2)
const getArg = (name: string) => {
const match = args.find((arg) => arg.startsWith(`--${name}=`))
return match ? match.slice(name.length + 3) : ''
}
const candidates = getArg('candidates')
const output = getArg('output')
const confirm = getArg('confirm')
const apply = args.includes('--apply')
if (!candidates) {
throw new Error('缺少 --candidates=删除候选清单 JSON')
}
return {
candidates,
output: output || path.join(path.dirname(candidates), `history-hard-delete-backup-${Date.now()}.json`),
apply,
confirm,
}
}
function normalizeCandidate(raw: any): DeleteCandidate {
return {
materialId: String(raw?.materialId || raw?.id || '').trim(),
fingerprint: String(raw?.fingerprint || '').trim() || undefined,
sourcePath: String(raw?.sourcePath || raw?.source_path || '').trim() || undefined,
reason: String(raw?.reason || raw?.deleteReason || '').trim(),
}
}
function assertCandidate(candidate: DeleteCandidate) {
if (!candidate.materialId) {
throw new Error('删除候选缺少 materialId')
}
if (!candidate.reason) {
throw new Error(`删除候选 ${candidate.materialId} 缺少 reason`)
}
if (!candidate.fingerprint && !candidate.sourcePath) {
throw new Error(`删除候选 ${candidate.materialId} 必须提供 fingerprint 或 sourcePath`)
}
}
function stringifyJson(value: unknown) {
return JSON.stringify(
value,
(_key, item) => (typeof item === 'bigint' ? item.toString() : item),
2,
)
}
async function loadCandidates(candidatePath: string) {
const raw = await fs.readFile(candidatePath, 'utf8')
const parsed = JSON.parse(raw)
const list = Array.isArray(parsed) ? parsed : parsed.candidates
if (!Array.isArray(list)) {
throw new Error('删除候选清单必须是数组,或包含 candidates 数组')
}
const candidates = list.map(normalizeCandidate)
candidates.forEach(assertCandidate)
return candidates
}
async function loadMaterialSnapshot(materialId: string) {
return prisma.material.findUnique({
where: { id: materialId },
include: {
assets: {
include: {
barcodes: true,
},
},
skus: true,
logs: true,
likes: true,
favorites: true,
downloadLogs: true,
transfers: true,
},
})
}
function hasHistoryEvidence(material: Awaited<ReturnType<typeof loadMaterialSnapshot>>, candidate: DeleteCandidate) {
if (!material) {
return false
}
return material.logs.some((log) => {
if (log.action !== 'HISTORY_IMPORT') {
return false
}
const changes = String(log.changes || '')
return Boolean(
(candidate.fingerprint && changes.includes(candidate.fingerprint)) ||
(candidate.sourcePath && changes.includes(candidate.sourcePath)),
)
})
}
async function main() {
const options = parseArgs()
const candidates = await loadCandidates(options.candidates)
const backups: any[] = []
const results: any[] = []
console.log('历史错误素材硬删除预检开始')
console.log(`- 候选清单: ${options.candidates}`)
console.log(`- 模式: ${options.apply ? '正式硬删除' : '仅备份/演练'}`)
console.log(`- 备份输出: ${options.output}`)
if (options.apply && options.confirm !== 'HARD_DELETE_HISTORY_MATERIALS') {
throw new Error('正式硬删除必须追加 --confirm=HARD_DELETE_HISTORY_MATERIALS')
}
for (const candidate of candidates) {
const material = await loadMaterialSnapshot(candidate.materialId)
if (!material) {
results.push({
materialId: candidate.materialId,
status: 'skipped',
reason: '素材不存在',
})
continue
}
const evidenceMatched = hasHistoryEvidence(material, candidate)
backups.push({
candidate,
material,
backedUpAt: new Date().toISOString(),
evidenceMatched,
})
if (!evidenceMatched) {
results.push({
materialId: candidate.materialId,
productModel: material.productModel,
status: 'blocked',
reason: '未在 HISTORY_IMPORT 日志中匹配 fingerprint/sourcePath,拒绝删除',
})
continue
}
if (!options.apply) {
results.push({
materialId: candidate.materialId,
productModel: material.productModel,
status: 'dry_run',
reason: candidate.reason,
})
continue
}
await prisma.material.delete({
where: { id: candidate.materialId },
})
results.push({
materialId: candidate.materialId,
productModel: material.productModel,
status: 'deleted',
reason: candidate.reason,
})
}
await fs.mkdir(path.dirname(options.output), { recursive: true })
await fs.writeFile(
options.output,
stringifyJson({
sourceCandidates: options.candidates,
apply: options.apply,
generatedAt: new Date().toISOString(),
results,
backups,
}),
'utf8',
)
const summary = results.reduce(
(acc, item) => {
acc.total += 1
acc[item.status] = (acc[item.status] || 0) + 1
return acc
},
{ total: 0 } as Record<string, number>,
)
console.log('历史错误素材硬删除预检完成')
console.log(summary)
console.log(`- 备份/结果: ${options.output}`)
}
main()
.catch((error) => {
console.error('历史错误素材硬删除失败:', error)
process.exitCode = 1
})
.finally(async () => {
await prisma.$disconnect()
process.exit(process.exitCode ?? 0)
})

View File

@ -1,11 +1,106 @@
import path from 'path'
import { promises as fs } from 'fs'
import { HeadBucketCommand, S3Client } from '@aws-sdk/client-s3'
import {
importHistoryMaterials,
resolveImportOperator,
type HistoryImportRecord,
} from '@/lib/history-material-import'
import { prisma } from '@/lib/prisma'
import {
assertRemoteUploadSyncAccess,
getRemoteUploadSyncConfig,
} from '@/lib/remote-upload-sync'
function getDatabaseHost() {
const rawUrl = String(process.env.DATABASE_URL || '').trim()
if (!rawUrl) {
return ''
}
try {
return new URL(rawUrl).hostname
} catch {
return ''
}
}
function assertApplyEnvironment() {
const databaseHost = getDatabaseHost()
const s3Bucket = String(process.env.S3_BUCKET || '').trim()
const s3Endpoint = String(process.env.S3_ENDPOINT || '').trim().replace(/\/+$/, '')
const s3PublicDomain = String(process.env.S3_PUBLIC_DOMAIN || '').trim()
const s3FilePath = String(process.env.S3_FILE_PATH || '').trim().replace(/\\/g, '/')
const uploadDir = String(process.env.UPLOAD_DIR || '').trim()
const remoteUploadSyncConfig = getRemoteUploadSyncConfig()
const isProductionDatabase = databaseHost === '192.168.58.20'
const isExpectedEndpoint = s3Endpoint === 'http://192.168.58.20:9000'
const isExpectedPreviewBucket =
s3Bucket === 'gongxiangpan' &&
/^https:\/\/s3\.jiansun\.vip\/gongxiangpan\/?$/i.test(s3PublicDomain)
const isExpectedFilePath = s3FilePath === '由壹点AI/'
if (!isExpectedEndpoint || !isExpectedPreviewBucket || !isExpectedFilePath) {
throw new Error(
[
'正式导入环境不正确:预览文件必须写入新桶 gongxiangpan 的正确前缀。',
`当前 S3_ENDPOINT=${s3Endpoint || '(empty)'}`,
`当前 S3_BUCKET=${s3Bucket || '(empty)'}`,
`当前 S3_PUBLIC_DOMAIN=${s3PublicDomain || '(empty)'}`,
`当前 S3_FILE_PATH=${s3FilePath || '(empty)'}`,
].join('\n'),
)
}
if (isProductionDatabase && process.platform === 'win32' && !remoteUploadSyncConfig) {
throw new Error(
[
'正式导入环境不正确:当前是 Windows 工作站,但 DATABASE_URL 指向生产库。',
'直接导入会把下载 zip / PSD 源文件写到本机 uploads,生产站点无法下载。',
'如确需在 Windows 读取 Z 盘源文件并写生产库,必须配置远程 uploads 同步:',
'HISTORY_IMPORT_REMOTE_UPLOADS_HOST=prod',
'HISTORY_IMPORT_REMOTE_UPLOADS_DIR=/mnt/PoolOne/apps/youyidian-3d/uploads',
`当前 UPLOAD_DIR=${uploadDir || '(empty)'}`,
].join('\n'),
)
}
}
async function assertApplyStorageAccess() {
const s3Endpoint = String(process.env.S3_ENDPOINT || '').trim()
const s3Bucket = String(process.env.S3_BUCKET || '').trim()
const accessKeyId = String(process.env.S3_ACCESS_KEY_ID || '').trim()
const secretAccessKey = String(process.env.S3_SECRET_ACCESS_KEY || '').trim()
if (!accessKeyId || !secretAccessKey) {
throw new Error('正式导入环境不正确:缺少 S3 访问凭据,不能验证新桶写入权限。')
}
const client = new S3Client({
endpoint: s3Endpoint,
region: process.env.S3_REGION || 'us-east-1',
credentials: {
accessKeyId,
secretAccessKey,
},
forcePathStyle: String(process.env.S3_ENABLE_PATH_STYLE || '').trim() !== '0',
})
try {
await client.send(new HeadBucketCommand({ Bucket: s3Bucket }))
} catch (error) {
const statusCode = (error as { $metadata?: { httpStatusCode?: number } }).$metadata?.httpStatusCode
throw new Error(
[
'正式导入环境不正确:当前 S3 凭据无法访问新桶,已阻止写入数据库。',
`S3_BUCKET=${s3Bucket || '(empty)'}`,
`S3_ENDPOINT=${s3Endpoint || '(empty)'}`,
`HTTP_STATUS=${statusCode || 'unknown'}`,
].join('\n'),
)
}
}
function parseArgs() {
const args = process.argv.slice(2)
@ -36,6 +131,12 @@ function parseArgs() {
async function main() {
const options = parseArgs()
if (options.apply) {
assertApplyEnvironment()
await assertApplyStorageAccess()
await assertRemoteUploadSyncAccess()
}
const raw = await fs.readFile(options.input, 'utf8')
const records = JSON.parse(raw) as HistoryImportRecord[]
const operator = await resolveImportOperator(options.operator)

View File

@ -0,0 +1,337 @@
#!/usr/bin/env node
const { PrismaClient } = require('@prisma/client')
const { S3Client, HeadObjectCommand, PutObjectCommand } = require('@aws-sdk/client-s3')
const prisma = new PrismaClient()
const DEFAULT_OLD_PREFIX = 'https://s3.1080p.hk/jsxiangche/'
function getArg(name, fallback = '') {
const prefix = `--${name}=`
const match = process.argv.slice(2).find((arg) => arg.startsWith(prefix))
return match ? match.slice(prefix.length) : fallback
}
function hasFlag(name) {
return process.argv.includes(`--${name}`)
}
function normalizePrefix(prefix) {
return String(prefix || '').trim().replace(/\/+$/, '') + '/'
}
function normalizePath(input) {
return String(input || '')
.trim()
.replace(/\\/g, '/')
.replace(/^\/+/, '')
.replace(/\/{2,}/g, '/')
}
function parseOldObjectKey(oldUrl, oldBucket) {
const parsed = new URL(oldUrl)
let key = decodeURIComponent(parsed.pathname).replace(/^\/+/, '')
if (oldBucket && key.startsWith(`${oldBucket}/`)) {
key = key.slice(oldBucket.length + 1)
}
return normalizePath(key)
}
function ensureDestinationKey(oldKey) {
const s3FilePath = normalizePath(process.env.S3_FILE_PATH || '')
if (!s3FilePath) {
return oldKey
}
if (oldKey.startsWith(`${s3FilePath}/`)) {
return oldKey
}
if (oldKey.startsWith('public-display/')) {
return `${s3FilePath}/${oldKey}`
}
return oldKey
}
function buildPublicUrl(key) {
const publicDomain = String(process.env.S3_PUBLIC_DOMAIN || '').trim().replace(/\/+$/, '')
if (!publicDomain) {
throw new Error('Missing S3_PUBLIC_DOMAIN')
}
return `${publicDomain}/${key}`
}
function guessContentType(url) {
const lower = url.toLowerCase()
if (lower.endsWith('.webp')) return 'image/webp'
if (lower.endsWith('.png')) return 'image/png'
if (lower.endsWith('.jpg') || lower.endsWith('.jpeg')) return 'image/jpeg'
return 'application/octet-stream'
}
function createS3Client() {
const endpoint = String(process.env.S3_ENDPOINT || '').split('#')[0].trim()
if (!endpoint) throw new Error('Missing S3_ENDPOINT')
if (!process.env.S3_BUCKET) throw new Error('Missing S3_BUCKET')
if (!process.env.S3_ACCESS_KEY_ID) throw new Error('Missing S3_ACCESS_KEY_ID')
if (!process.env.S3_SECRET_ACCESS_KEY) throw new Error('Missing S3_SECRET_ACCESS_KEY')
return new S3Client({
endpoint,
region: process.env.S3_REGION || 'us-east-1',
credentials: {
accessKeyId: process.env.S3_ACCESS_KEY_ID,
secretAccessKey: process.env.S3_SECRET_ACCESS_KEY,
},
forcePathStyle: process.env.S3_ENABLE_PATH_STYLE !== '0',
maxAttempts: 5,
})
}
async function destinationExists(client, bucket, key) {
try {
await client.send(new HeadObjectCommand({ Bucket: bucket, Key: key }))
return true
} catch (error) {
const statusCode = error && error.$metadata && error.$metadata.httpStatusCode
if (statusCode === 404 || error.name === 'NotFound') {
return false
}
throw error
}
}
async function fetchOldObject(oldUrl) {
const response = await fetch(oldUrl)
if (!response.ok) {
throw new Error(`Fetch failed ${response.status}`)
}
return {
body: Buffer.from(await response.arrayBuffer()),
contentType: response.headers.get('content-type') || guessContentType(oldUrl),
}
}
async function getOldPreviewUrls(oldPrefix, limit) {
const limitedSql = limit > 0 ? `LIMIT ${Number(limit)}` : ''
return prisma.$queryRawUnsafe(`
WITH history_materials AS (
SELECT DISTINCT "materialId" AS id
FROM material_logs
WHERE action = 'HISTORY_IMPORT'
), active_history AS (
SELECT m.id, m."previewUrl"
FROM materials m
JOIN history_materials h ON h.id = m.id
WHERE m."deletedAt" IS NULL
)
SELECT DISTINCT old_url
FROM (
SELECT "previewUrl" AS old_url
FROM active_history
WHERE "previewUrl" LIKE $1
UNION
SELECT ma.file_url AS old_url
FROM material_assets ma
JOIN active_history h ON h.id = ma.material_id
WHERE ma.asset_type IN ('main-preview', 'sku-preview', 'detail-preview')
AND ma.file_url LIKE $1
) urls
WHERE old_url IS NOT NULL AND old_url <> ''
ORDER BY old_url
${limitedSql}
`, `${oldPrefix}%`)
}
async function updateDatabaseUrl(oldUrl, newUrl) {
const materialUpdates = await prisma.$executeRawUnsafe(`
WITH history_materials AS (
SELECT DISTINCT "materialId" AS id
FROM material_logs
WHERE action = 'HISTORY_IMPORT'
)
UPDATE materials m
SET "previewUrl" = $1,
"updatedAt" = NOW()
FROM history_materials h
WHERE h.id = m.id
AND m."deletedAt" IS NULL
AND m."previewUrl" = $2
`, newUrl, oldUrl)
const assetUpdates = await prisma.$executeRawUnsafe(`
WITH history_materials AS (
SELECT DISTINCT "materialId" AS id
FROM material_logs
WHERE action = 'HISTORY_IMPORT'
)
UPDATE material_assets ma
SET file_url = $1,
updated_at = NOW()
FROM history_materials h
JOIN materials m ON m.id = h.id
WHERE ma.material_id = m.id
AND m."deletedAt" IS NULL
AND ma.asset_type IN ('main-preview', 'sku-preview', 'detail-preview')
AND ma.file_url = $2
`, newUrl, oldUrl)
return { materialUpdates, assetUpdates }
}
async function bulkUpdateDatabaseUrls(oldPrefix, newPrefix) {
const materialUpdates = await prisma.$executeRawUnsafe(`
WITH history_materials AS (
SELECT DISTINCT "materialId" AS id
FROM material_logs
WHERE action = 'HISTORY_IMPORT'
)
UPDATE materials m
SET "previewUrl" = REPLACE(m."previewUrl", $1, $2),
"updatedAt" = NOW()
FROM history_materials h
WHERE h.id = m.id
AND m."deletedAt" IS NULL
AND m."previewUrl" LIKE $3
`, oldPrefix, newPrefix, `${oldPrefix}%`)
const assetUpdates = await prisma.$executeRawUnsafe(`
WITH history_materials AS (
SELECT DISTINCT "materialId" AS id
FROM material_logs
WHERE action = 'HISTORY_IMPORT'
)
UPDATE material_assets ma
SET file_url = REPLACE(ma.file_url, $1, $2),
updated_at = NOW()
FROM history_materials h
JOIN materials m ON m.id = h.id
WHERE ma.material_id = m.id
AND m."deletedAt" IS NULL
AND ma.asset_type IN ('main-preview', 'sku-preview', 'detail-preview')
AND ma.file_url LIKE $3
`, oldPrefix, newPrefix, `${oldPrefix}%`)
return { materialUpdates, assetUpdates }
}
async function runPool(items, concurrency, worker) {
let index = 0
const workers = Array.from({ length: concurrency }, async () => {
while (index < items.length) {
const currentIndex = index
index += 1
await worker(items[currentIndex], currentIndex)
}
})
await Promise.all(workers)
}
async function main() {
const apply = hasFlag('apply')
const bulkDbUpdate = hasFlag('bulk-db-update')
const skipDbUpdate = hasFlag('skip-db-update')
const oldPrefix = normalizePrefix(getArg('old-prefix', DEFAULT_OLD_PREFIX))
const oldBucket = getArg('old-bucket', 'jsxiangche')
const limit = Number(getArg('limit', '0')) || 0
const concurrency = Math.max(1, Number(getArg('concurrency', '8')) || 8)
const oldRows = await getOldPreviewUrls(oldPrefix, limit)
const mappings = oldRows.map((row) => {
const oldUrl = row.old_url
const key = ensureDestinationKey(parseOldObjectKey(oldUrl, oldBucket))
return {
oldUrl,
key,
newUrl: buildPublicUrl(key),
}
})
console.log(JSON.stringify({
mode: apply ? 'apply' : 'dry-run',
oldPrefix,
totalUrls: mappings.length,
concurrency,
bulkDbUpdate,
skipDbUpdate,
sample: mappings.slice(0, 3),
}, null, 2))
if (!apply) {
return
}
const s3Client = createS3Client()
const bucket = process.env.S3_BUCKET
const stats = {
uploaded: 0,
skippedExisting: 0,
dbMaterialUpdates: 0,
dbAssetUpdates: 0,
failed: 0,
}
const failures = []
await runPool(mappings, concurrency, async (mapping, currentIndex) => {
try {
const exists = await destinationExists(s3Client, bucket, mapping.key)
if (exists) {
stats.skippedExisting += 1
} else {
const oldObject = await fetchOldObject(mapping.oldUrl)
await s3Client.send(new PutObjectCommand({
Bucket: bucket,
Key: mapping.key,
Body: oldObject.body,
ContentType: oldObject.contentType,
ACL: process.env.S3_SET_ACL === '0' ? undefined : 'public-read',
}))
stats.uploaded += 1
}
if (!bulkDbUpdate && !skipDbUpdate) {
const dbResult = await updateDatabaseUrl(mapping.oldUrl, mapping.newUrl)
stats.dbMaterialUpdates += Number(dbResult.materialUpdates || 0)
stats.dbAssetUpdates += Number(dbResult.assetUpdates || 0)
}
} catch (error) {
stats.failed += 1
failures.push({
oldUrl: mapping.oldUrl,
message: error instanceof Error ? error.message : String(error),
})
}
if ((currentIndex + 1) % 1000 === 0 || currentIndex + 1 === mappings.length) {
console.log(JSON.stringify({ processed: currentIndex + 1, total: mappings.length, stats }))
}
})
if (bulkDbUpdate && stats.failed === 0) {
const newPrefix = mappings[0]
? mappings[0].newUrl.slice(0, mappings[0].newUrl.length - mappings[0].key.length)
: ''
if (!newPrefix) {
throw new Error('Cannot infer new prefix for bulk database update')
}
const dbResult = await bulkUpdateDatabaseUrls(oldPrefix, newPrefix)
stats.dbMaterialUpdates += Number(dbResult.materialUpdates || 0)
stats.dbAssetUpdates += Number(dbResult.assetUpdates || 0)
}
console.log(JSON.stringify({ done: true, stats, failures: failures.slice(0, 20) }, null, 2))
if (stats.failed > 0) {
process.exitCode = 1
}
}
main()
.catch((error) => {
console.error(error)
process.exitCode = 1
})
.finally(async () => {
await prisma.$disconnect()
})

View File

@ -0,0 +1,315 @@
#!/usr/bin/env python3
"""
Build a second-pass review table for needs_review SPU ledger items.
The first SPU ledger intentionally keeps uncertain workspaces out of the import
path. This script does not change the ledger or production data; it groups those
uncertain rows by the real SPU root and rescans that root so cross-level
main/SKU/detail folders can be reviewed together.
"""
from __future__ import annotations
import argparse
import csv
import json
import os
import sqlite3
from collections import Counter, defaultdict
from datetime import datetime
from pathlib import Path
from typing import Any
from scan_spu_ledger import (
EXCLUDED_DIR_NAMES,
IMAGE_EXTENSIONS,
OTHER_IMAGE_EXTENSIONS,
PS_SOURCE_EXTENSIONS,
RELEVANT_EXTENSIONS,
SOURCE_ARCHIVE_EXTENSIONS,
build_sku_code,
classify_file,
extract_spu_candidates,
has_hard_skip,
has_modeling_source_segment,
is_compatible_spu_token,
normalize_spu,
safe_relative,
)
REVIEW_HEADERS = [
"decision",
"spu",
"store",
"product_category",
"spu_root_path",
"needs_review_rows",
"direct_all_three_rows",
"aggregate_main_count",
"aggregate_sku_count",
"aggregate_detail_count",
"aggregate_source_count",
"hard_skip_file_count",
"modeling_file_count",
"status_reasons",
"workspace_samples",
"main_samples",
"sku_samples",
"detail_samples",
]
def json_dumps(value: Any) -> str:
return json.dumps(value, ensure_ascii=False, indent=2)
def latest_run_id(conn: sqlite3.Connection) -> str:
row = conn.execute(
"SELECT run_id FROM spu_ledger_runs ORDER BY started_at DESC LIMIT 1"
).fetchone()
if not row:
raise SystemExit("台账数据库里没有扫描 run")
return str(row["run_id"])
def choose_spu_root_path(item: sqlite3.Row) -> str:
source_root = Path(item["source_root"])
workspace_path = Path(item["workspace_path"])
relative = Path(item["relative_workspace_path"])
spu = normalize_spu(item["spu"])
parts = list(relative.parts)
for index, part in enumerate(parts):
tokens = extract_spu_candidates(part, allow_unbranded=True)
if any(is_compatible_spu_token(spu, token) for token in tokens):
return str(source_root.joinpath(*parts[: index + 1]))
# File-inferred and non-standard folders can have no SPU-looking segment.
# In that case use the nearest useful parent instead of the leaf itself so
# sibling main/SKU/detail folders can still be discovered.
if len(parts) >= 2:
return str(source_root.joinpath(*parts[:-1]))
return str(workspace_path)
def scan_spu_root(root_path: str, spu: str) -> dict[str, Any]:
root = Path(root_path)
counts: Counter[str] = Counter()
samples: dict[str, list[dict[str, Any]]] = defaultdict(list)
hard_skip_file_count = 0
modeling_file_count = 0
if not root.exists():
return {
"counts": counts,
"samples": samples,
"hard_skip_file_count": 0,
"modeling_file_count": 0,
"exists": False,
}
for current, dir_names, file_names in os.walk(root):
dir_names[:] = [
name
for name in dir_names
if name not in EXCLUDED_DIR_NAMES and not has_hard_skip(str(Path(current) / name))
]
current_path = Path(current)
current_is_modeling = has_modeling_source_segment(str(current_path))
for file_name in file_names:
file_path = current_path / file_name
extension = file_path.suffix.lower()
if extension not in RELEVANT_EXTENSIONS:
continue
relative_path = safe_relative(file_path, root)
if has_hard_skip(str(file_path)):
hard_skip_file_count += 1
continue
if current_is_modeling:
modeling_file_count += 1
group = classify_file(relative_path, file_name, extension)
if group == "other_source_archive" and extension in SOURCE_ARCHIVE_EXTENSIONS:
# Keep archives visible, but do not promote generic zip files to
# source unless the first-pass classifier saw source evidence.
pass
counts[group] += 1
if group in {"main", "sku", "detail", "source"} and len(samples[group]) < 8:
record = {
"filePath": str(file_path),
"relativePath": relative_path,
"fileName": file_name,
"fileSize": file_path.stat().st_size,
}
if group == "sku":
record["skuCode"] = build_sku_code(spu, file_name, counts[group] - 1)
record["barcodes"] = []
samples[group].append(record)
return {
"counts": counts,
"samples": samples,
"hard_skip_file_count": hard_skip_file_count,
"modeling_file_count": modeling_file_count,
"exists": True,
}
def decide_group(items: list[sqlite3.Row], aggregate: dict[str, Any]) -> str:
counts: Counter[str] = aggregate["counts"]
has_all_three = counts["main"] > 0 and counts["sku"] > 0 and counts["detail"] > 0
direct_all_three = [
item
for item in items
if int(item["main_count"] or 0) > 0
and int(item["sku_count"] or 0) > 0
and int(item["detail_count"] or 0) > 0
]
reasons = " | ".join(sorted({str(item["status_reason"] or "") for item in items}))
has_modeling_reason = "建模" in reasons or "3D" in reasons or aggregate["modeling_file_count"] > 0
if not aggregate["exists"]:
return "manual_missing_root"
if direct_all_three and not has_modeling_reason:
return "direct_ready_review"
if has_all_three and not has_modeling_reason:
return "cross_level_ready_review"
if has_modeling_reason and not has_all_three:
return "skip_likely_modeling_or_partial"
if counts["detail"] == 0:
return "skip_likely_no_detail"
if counts["main"] == 0 or counts["sku"] == 0:
return "partial_needs_pairing"
return "manual_review"
def write_csv(path: Path, rows: list[dict[str, Any]]) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
with path.open("w", encoding="utf-8-sig", newline="") as handle:
writer = csv.DictWriter(handle, fieldnames=REVIEW_HEADERS)
writer.writeheader()
writer.writerows(rows)
def main() -> int:
parser = argparse.ArgumentParser(description="Second-pass review for needs_review SPU groups.")
parser.add_argument("--db-path", default="output/tianlan-spu-ledger.db")
parser.add_argument("--run-id", default="")
parser.add_argument("--output-dir", default="")
args = parser.parse_args()
db_path = Path(args.db_path)
stamp = datetime.now().strftime("%Y%m%d-%H%M%S")
output_dir = Path(args.output_dir or f"output/tianlan-spu-ledger/needs-review-cross-level-{stamp}")
output_dir.mkdir(parents=True, exist_ok=True)
conn = sqlite3.connect(str(db_path))
conn.row_factory = sqlite3.Row
run_id = args.run_id or latest_run_id(conn)
items = conn.execute(
"""
SELECT *
FROM spu_ledger_items
WHERE run_id = ? AND status = 'needs_review'
ORDER BY source_root_name, relative_workspace_path, spu
""",
(run_id,),
).fetchall()
groups: dict[tuple[str, str, str, str], list[sqlite3.Row]] = defaultdict(list)
for item in items:
spu_root_path = choose_spu_root_path(item)
key = (
str(item["store"] or ""),
normalize_spu(item["spu"]),
str(item["source_root_name"] or ""),
spu_root_path,
)
groups[key].append(item)
review_rows: list[dict[str, Any]] = []
details: list[dict[str, Any]] = []
for (store, spu, source_root_name, spu_root_path), grouped_items in sorted(groups.items()):
aggregate = scan_spu_root(spu_root_path, spu)
counts: Counter[str] = aggregate["counts"]
decision = decide_group(grouped_items, aggregate)
status_reasons = sorted({str(item["status_reason"] or "") for item in grouped_items})
workspace_samples = [str(item["workspace_path"]) for item in grouped_items[:8]]
direct_all_three_rows = sum(
1
for item in grouped_items
if int(item["main_count"] or 0) > 0
and int(item["sku_count"] or 0) > 0
and int(item["detail_count"] or 0) > 0
)
row = {
"decision": decision,
"spu": spu,
"store": store,
"product_category": grouped_items[0]["product_category"],
"spu_root_path": spu_root_path,
"needs_review_rows": len(grouped_items),
"direct_all_three_rows": direct_all_three_rows,
"aggregate_main_count": counts["main"],
"aggregate_sku_count": counts["sku"],
"aggregate_detail_count": counts["detail"],
"aggregate_source_count": counts["source"],
"hard_skip_file_count": aggregate["hard_skip_file_count"],
"modeling_file_count": aggregate["modeling_file_count"],
"status_reasons": " | ".join(status_reasons),
"workspace_samples": " || ".join(workspace_samples),
"main_samples": " || ".join(item["relativePath"] for item in aggregate["samples"].get("main", [])),
"sku_samples": " || ".join(item["relativePath"] for item in aggregate["samples"].get("sku", [])),
"detail_samples": " || ".join(item["relativePath"] for item in aggregate["samples"].get("detail", [])),
}
review_rows.append(row)
details.append(
{
**row,
"sourceRootName": source_root_name,
"workspaces": [
{
"fingerprint": item["fingerprint"],
"workspacePath": item["workspace_path"],
"relativeWorkspacePath": item["relative_workspace_path"],
"versionLabel": item["version_label"],
"mainCount": item["main_count"],
"skuCount": item["sku_count"],
"detailCount": item["detail_count"],
"sourceCount": item["source_count"],
"statusReason": item["status_reason"],
}
for item in grouped_items
],
"samples": aggregate["samples"],
}
)
write_csv(output_dir / "needs-review-cross-level-groups.csv", review_rows)
(output_dir / "needs-review-cross-level-groups.json").write_text(
json_dumps(details), encoding="utf-8"
)
summary = {
"runId": run_id,
"dbPath": str(db_path),
"needsReviewItems": len(items),
"groups": len(review_rows),
"decisionCounts": dict(Counter(row["decision"] for row in review_rows)),
"outputCsv": str(output_dir / "needs-review-cross-level-groups.csv"),
"outputJson": str(output_dir / "needs-review-cross-level-groups.json"),
}
(output_dir / "summary.json").write_text(json_dumps(summary), encoding="utf-8")
conn.close()
print(json_dumps(summary))
return 0
if __name__ == "__main__":
raise SystemExit(main())

View File

@ -0,0 +1,202 @@
param(
[int]$StartBatch = 2,
[int]$EndBatch = 15,
[string]$BatchDir = "D:\AI Code\gongxiangp2\output\tianlan-spu-ledger\remaining-batches-1778239500000",
[string]$Operator = "222"
)
$ErrorActionPreference = "Stop"
Set-Location "D:\AI Code\gongxiangp2"
function Set-ImportEnvironment {
$env:DATABASE_URL = ((Get-Content .env | Where-Object { $_ -match '^DATABASE_URL=' } | Select-Object -First 1) -replace '^DATABASE_URL=', '').Trim()
$env:S3_ENDPOINT = "http://192.168.58.20:9000"
$env:S3_BUCKET = "gongxiangpan"
$env:S3_PUBLIC_DOMAIN = "https://s3.jiansun.vip/gongxiangpan"
$env:S3_FILE_PATH = [System.Text.Encoding]::UTF8.GetString([System.Convert]::FromBase64String("55Sx5aO554K5QUkv"))
$sshArgs = @(
"-o", "BatchMode=yes",
"-o", "ConnectTimeout=15",
"-o", "ServerAliveInterval=15",
"-o", "ServerAliveCountMax=2",
"prod"
)
$env:S3_ACCESS_KEY_ID = Invoke-TextWithRetry "read S3_ACCESS_KEY_ID" { ssh @sshArgs "docker exec youyidian-3d-material printenv S3_ACCESS_KEY_ID" }
$env:S3_SECRET_ACCESS_KEY = Invoke-TextWithRetry "read S3_SECRET_ACCESS_KEY" { ssh @sshArgs "docker exec youyidian-3d-material printenv S3_SECRET_ACCESS_KEY" }
$env:HISTORY_IMPORT_REMOTE_UPLOADS_HOST = "prod"
$env:HISTORY_IMPORT_REMOTE_UPLOADS_DIR = "/mnt/PoolOne/apps/youyidian-3d/uploads"
}
function Get-NewestImportResult([datetime]$StartedAt) {
Get-ChildItem -LiteralPath $BatchDir -Filter "history-material-import-result-*.json" |
Where-Object { $_.Name -notlike "*.verification.json" -and $_.LastWriteTime -ge $StartedAt } |
Sort-Object LastWriteTime -Descending |
Select-Object -First 1
}
function Write-GoalProgress($Payload) {
$progressPath = Join-Path $BatchDir "goal-progress.json"
$Payload | ConvertTo-Json -Depth 8 | Set-Content -LiteralPath $progressPath -Encoding UTF8
}
function Invoke-NpxToLog([string[]]$Arguments, [string]$LogPath) {
$previousErrorActionPreference = $ErrorActionPreference
$ErrorActionPreference = "Continue"
try {
& npx @Arguments *> $LogPath
return $LASTEXITCODE
} finally {
$script:ErrorActionPreference = $previousErrorActionPreference
}
}
function Invoke-TextWithRetry([string]$Label, [scriptblock]$Command, [int]$Retries = 3) {
$lastOutput = $null
for ($attempt = 1; $attempt -le $Retries; $attempt += 1) {
$lastOutput = & $Command
if ($LASTEXITCODE -eq 0 -and -not [string]::IsNullOrWhiteSpace(($lastOutput | Out-String))) {
return ($lastOutput | Out-String).Trim()
}
if ($attempt -lt $Retries) {
Write-Host ("[Goal] {0} failed on attempt {1}/{2}; retrying..." -f $Label, $attempt, $Retries)
Start-Sleep -Seconds ([Math]::Min(10, 2 * $attempt))
}
}
throw ("{0} failed after {1} attempts" -f $Label, $Retries)
}
Set-ImportEnvironment
$runStartedAt = Get-Date
$completed = @()
[pscustomobject]@{
pid = $PID
startBatch = $StartBatch
endBatch = $EndBatch
batchDir = $BatchDir
startedAt = $runStartedAt.ToString("o")
command = "scripts/run-tianlan-remaining-batches.ps1 -StartBatch $StartBatch -EndBatch $EndBatch"
} | ConvertTo-Json -Depth 5 | Set-Content -LiteralPath (Join-Path $BatchDir "active-goal-process.json") -Encoding UTF8
for ($batchNumber = $StartBatch; $batchNumber -le $EndBatch; $batchNumber += 1) {
$batchName = "batch-{0:D3}.json" -f $batchNumber
$batchPath = Join-Path $BatchDir $batchName
if (-not (Test-Path -LiteralPath $batchPath)) {
throw "Batch file not found: $batchPath"
}
$stamp = Get-Date -Format "yyyyMMdd-HHmmss"
$importLog = Join-Path $BatchDir ("batch-{0:D3}-{1}.import.log" -f $batchNumber, $stamp)
$verifyLog = Join-Path $BatchDir ("batch-{0:D3}-{1}.verify.log" -f $batchNumber, $stamp)
$startedAt = Get-Date
Write-Host ""
Write-Host ("[Goal] Importing {0}" -f $batchName)
Write-GoalProgress ([pscustomobject]@{
mode = "tianlan-remaining-batches"
runStartedAt = $runStartedAt.ToString("o")
updatedAt = (Get-Date).ToString("o")
startBatch = $StartBatch
endBatch = $EndBatch
completed = $completed
current = [pscustomobject]@{
batch = $batchName
phase = "importing"
importLog = Split-Path -Leaf $importLog
verifyLog = Split-Path -Leaf $verifyLog
startedAt = $startedAt.ToString("o")
}
})
$importExitCode = Invoke-NpxToLog @(
"tsx",
"scripts/import-history-materials.ts",
"--input=$batchPath",
"--operator=$Operator",
"--apply"
) $importLog
$resultFile = Get-NewestImportResult $startedAt
if ($importExitCode -ne 0 -or -not $resultFile) {
Write-Host ("[Goal] Import failed for {0}; log: {1}" -f $batchName, $importLog)
if (Test-Path -LiteralPath $importLog) {
Get-Content -LiteralPath $importLog -Tail 80
}
exit 1
}
Write-Host ("[Goal] Result: {0}" -f $resultFile.Name)
Write-GoalProgress ([pscustomobject]@{
mode = "tianlan-remaining-batches"
runStartedAt = $runStartedAt.ToString("o")
updatedAt = (Get-Date).ToString("o")
startBatch = $StartBatch
endBatch = $EndBatch
completed = $completed
current = [pscustomobject]@{
batch = $batchName
phase = "validating"
result = $resultFile.Name
importLog = Split-Path -Leaf $importLog
verifyLog = Split-Path -Leaf $verifyLog
}
})
$verifyExitCode = Invoke-NpxToLog @(
"tsx",
"scripts/validate-history-import-result.ts",
"--input=$($resultFile.FullName)",
"--remote-host=prod",
"--remote-uploads-dir=/mnt/PoolOne/apps/youyidian-3d/uploads"
) $verifyLog
$verificationPath = $resultFile.FullName -replace '\.json$', '.verification.json'
$verification = Get-Content -LiteralPath $verificationPath -Raw | ConvertFrom-Json
$completed += [pscustomobject]@{
batch = $batchName
result = $resultFile.Name
total = $verification.total
imported = $verification.imported
skipped = $verification.skipped
failed = $verification.failed
checked = $verification.checked
ok = $verification.ok
issueCount = $verification.issueCount
importLog = Split-Path -Leaf $importLog
verifyLog = Split-Path -Leaf $verifyLog
finishedAt = (Get-Date).ToString("o")
}
Write-GoalProgress ([pscustomobject]@{
mode = "tianlan-remaining-batches"
runStartedAt = $runStartedAt.ToString("o")
updatedAt = (Get-Date).ToString("o")
startBatch = $StartBatch
endBatch = $EndBatch
completed = $completed
current = $completed[-1]
})
Write-Host ("[Goal] Verified {0}: imported={1}, skipped={2}, failed={3}, ok={4}, issues={5}" -f $batchName, $verification.imported, $verification.skipped, $verification.failed, $verification.ok, $verification.issueCount)
if ($verifyExitCode -ne 0 -or $verification.failed -gt 0 -or $verification.issueCount -gt 0) {
Write-Host ("[Goal] Stopped on verification issue for {0}; log: {1}" -f $batchName, $verifyLog)
Get-Content -LiteralPath $verifyLog -Tail 120
exit 1
}
}
Write-Host ""
Write-Host "[Goal] All requested batches completed."
Write-GoalProgress ([pscustomobject]@{
mode = "tianlan-remaining-batches"
runStartedAt = $runStartedAt.ToString("o")
updatedAt = (Get-Date).ToString("o")
startBatch = $StartBatch
endBatch = $EndBatch
completed = $completed
done = $true
})

View File

@ -16,9 +16,20 @@ NON_DIRECTORY_EXTENSIONS = IMAGE_EXTENSIONS | {
}
SKU_DIR_PATTERN = re.compile(r"sku", re.IGNORECASE)
SPU_PATTERN = re.compile(r"^[A-Za-z]{1,5}\d{3,5}$")
SPU_TOKEN_PATTERN = re.compile(r"([A-Za-z]{1,5}\d{3,5})")
BRANDED_SPU_TOKEN_PATTERN = re.compile(r"([A-Za-z]{2,5}-[A-Za-z]{1,5}\d{3,5})")
BRANDED_SKU_FILE_PATTERN = re.compile(r"([A-Za-z]{2,5})-([A-Za-z]{1,5})(\d{4,5})(\d{2})(?:\D|$)")
KNOWN_SHOP_PREFIXES = {"GBPM", "GBSFY", "GBGJ", "GBJP", "XYL"}
IGNORED_SPU_PREFIXES = {"TASK", "SKU", "IMG"}
KNOWN_CATEGORY_CODES = {
"SN", "DG", "XG", "BG", "SG", "CBG", "CZ", "SZ", "DNZ", "SJZ", "ZDZ",
"CBZ", "CY", "AD", "SZD", "HXD", "SJ", "ZSJ", "ZWJ", "CFZW", "CFZWJ",
"YMJ", "BJ", "CJ", "DSG", "CTG", "CYJ", "SZT", "XJ", "LRSF", "SMC",
}
IGNORED_PARTS = {"@eadir", "dingtalkappdata", ".recycle", "$nsec.recycle.bin"}
DETAIL_DIR_KEYWORDS = ("images", "详情页", "详情")
MAIN_DIR_KEYWORD = "主图"
MAIN_DIMENSION_DIR_NAMES = {"750", "800"}
def utc_now() -> str:
@ -29,11 +40,93 @@ def normalize_spu(value: str) -> str:
return value.strip().upper()
def guess_spu(path_obj: Path) -> str:
def looks_like_spu_token(value: str) -> bool:
normalized = normalize_spu(value)
if "-" in normalized:
prefix, _, sku_part = normalized.partition("-")
return prefix in KNOWN_SHOP_PREFIXES and bool(SPU_PATTERN.match(sku_part))
return bool(SPU_PATTERN.match(normalized)) and not any(
normalized.startswith(prefix) for prefix in IGNORED_SPU_PREFIXES
)
def is_standalone_spu_match(text: str, start: int, end: int) -> bool:
prev_char = text[start - 1] if start > 0 else ""
next_char = text[end] if end < len(text) else ""
if re.match(r"[A-Za-z0-9]", prev_char or ""):
return False
if not re.match(r"[A-Za-z0-9]", next_char or ""):
return True
return bool(re.match(r"K\d{1,2}(?![A-Za-z0-9])", text[end:], re.IGNORECASE))
def get_sku_part(spu: str) -> str:
normalized = normalize_spu(spu)
return normalized.partition("-")[2] if "-" in normalized else normalized
def has_known_category_code(spu: str) -> bool:
sku_part = get_sku_part(spu)
for code in sorted(KNOWN_CATEGORY_CODES, key=len, reverse=True):
if sku_part.startswith(code) and len(sku_part) > len(code) and sku_part[len(code)].isdigit():
return True
return False
def collect_spu_tokens(path_obj: Path):
tokens = []
seen = set()
for part in reversed(path_obj.parts):
matches = list(BRANDED_SPU_TOKEN_PATTERN.finditer(part))
for match in reversed(matches):
token = normalize_spu(match.group(1))
if (
token not in seen
and looks_like_spu_token(token)
and is_standalone_spu_match(part, match.start(), match.end())
):
tokens.append(token)
seen.add(token)
for part in reversed(path_obj.parts):
normalized = normalize_spu(part)
if SPU_PATTERN.match(normalized):
return normalized
if normalized not in seen and looks_like_spu_token(normalized):
tokens.append(normalized)
seen.add(normalized)
matches = list(SPU_TOKEN_PATTERN.finditer(part))
for match in reversed(matches):
token = normalize_spu(match.group(1))
if (
token not in seen
and looks_like_spu_token(token)
and is_standalone_spu_match(part, match.start(), match.end())
):
tokens.append(token)
seen.add(token)
return tokens
def guess_spu(path_obj: Path) -> str:
tokens = collect_spu_tokens(path_obj)
for token in tokens:
if has_known_category_code(token):
return token
if tokens:
return tokens[0]
return ""
def guess_spu_from_file_names(file_names) -> str:
for file_name in file_names:
match = BRANDED_SKU_FILE_PATTERN.search(file_name)
if not match:
continue
token = normalize_spu(f"{match.group(1)}-{match.group(2)}{match.group(3)}")
if looks_like_spu_token(token):
return token
return ""
@ -63,7 +156,7 @@ def is_detail_dir_name(name: str) -> bool:
def is_main_dir_name(name: str) -> bool:
normalized = normalize_dir_name(name)
return MAIN_DIR_KEYWORD in normalized
return MAIN_DIR_KEYWORD in normalized or normalized in MAIN_DIMENSION_DIR_NAMES
def list_sibling_dirs(parent_path: Path, current_dir_name: str):
@ -161,12 +254,15 @@ def ensure_schema(conn: sqlite3.Connection) -> None:
conn.commit()
def scan_sku_dirs(root_path: Path):
def scan_sku_dirs(root_path: Path, skip_repeated_root_name: bool = False):
for current_root, dir_names, file_names in os.walk(root_path):
current_path = Path(current_root)
if should_ignore_path(current_path):
dir_names[:] = []
continue
if skip_repeated_root_name and current_path == root_path:
root_name = root_path.name.casefold()
dir_names[:] = [name for name in dir_names if name.casefold() != root_name]
for dir_name in list(dir_names):
if not matches_sku_dir(dir_name):
@ -195,7 +291,7 @@ def scan_sku_dirs(root_path: Path):
"directory_name": sku_dir_path.name,
"parent_path": str(sku_dir_path.parent),
"grandparent_path": str(sku_dir_path.parent.parent) if sku_dir_path.parent != root_path else "",
"spu": guess_spu(sku_dir_path),
"spu": guess_spu(sku_dir_path) or guess_spu_from_file_names(image_files),
"depth": len(sku_dir_path.relative_to(root_path).parts),
"file_count": len(files),
"image_count": len(image_files),
@ -259,21 +355,32 @@ def parse_args():
default="/mnt/history-sc/大货营销策划图片",
help="Root directory to scan.",
)
parser.add_argument(
"--roots",
default="",
help="Multiple root directories separated by |. When provided, --root is ignored.",
)
parser.add_argument(
"--db-path",
default="/root/gongxiangp2/output/history-sku-folders.db",
help="SQLite database file path.",
)
parser.add_argument(
"--skip-repeated-root-name",
action="store_true",
help="Skip an immediate child directory whose name equals the scan root name.",
)
return parser.parse_args()
def main() -> int:
args = parse_args()
root_path = Path(args.root)
root_paths = [Path(item.strip()) for item in args.roots.split("|") if item.strip()] if args.roots else [Path(args.root)]
db_path = Path(args.db_path)
if not root_path.exists():
print(f"Root path not found: {root_path}", file=sys.stderr)
missing_roots = [str(root_path) for root_path in root_paths if not root_path.exists()]
if missing_roots:
print(f"Root path not found: {missing_roots[0]}", file=sys.stderr)
return 1
db_path.parent.mkdir(parents=True, exist_ok=True)
@ -281,29 +388,35 @@ def main() -> int:
conn = sqlite3.connect(str(db_path))
try:
ensure_schema(conn)
scan_id = str(uuid.uuid4())
started_at = utc_now()
conn.execute(
"INSERT INTO sku_folder_scans (scan_id, root_path, started_at) VALUES (?, ?, ?)",
(scan_id, str(root_path), started_at),
)
conn.commit()
all_previews = []
grand_total = 0
scan_ids = []
for root_path in root_paths:
scan_id = str(uuid.uuid4())
scan_ids.append(scan_id)
started_at = utc_now()
conn.execute(
"INSERT INTO sku_folder_scans (scan_id, root_path, started_at) VALUES (?, ?, ?)",
(scan_id, str(root_path), started_at),
)
conn.commit()
rows = list(scan_sku_dirs(root_path))
total_rows = insert_rows(conn, scan_id, rows)
finished_at = utc_now()
conn.execute(
"UPDATE sku_folder_scans SET finished_at = ?, total_rows = ? WHERE scan_id = ?",
(finished_at, total_rows, scan_id),
)
conn.commit()
rows = list(scan_sku_dirs(root_path, args.skip_repeated_root_name))
total_rows = insert_rows(conn, scan_id, rows)
grand_total += total_rows
finished_at = utc_now()
conn.execute(
"UPDATE sku_folder_scans SET finished_at = ?, total_rows = ? WHERE scan_id = ?",
(finished_at, total_rows, scan_id),
)
conn.commit()
all_previews.extend(rows[:3])
preview = rows[:12]
print("SKU folder scan completed")
print(f"scan_id={scan_id}")
print(f"scan_ids={','.join(scan_ids)}")
print(f"db_path={db_path}")
print(f"total_rows={total_rows}")
print(json.dumps(preview, ensure_ascii=False, indent=2))
print(f"total_rows={grand_total}")
print(json.dumps(all_previews[:12], ensure_ascii=False, indent=2))
return 0
finally:
conn.close()

1282
scripts/scan_spu_ledger.py Normal file

File diff suppressed because it is too large Load Diff

View File

@ -0,0 +1,274 @@
import { execFile } from 'child_process'
import { promises as fs } from 'fs'
import path from 'path'
import { promisify } from 'util'
import { PrismaClient } from '@prisma/client'
const execFileAsync = promisify(execFile)
const SSH_TIMEOUT_MS = Number(process.env.HISTORY_IMPORT_VALIDATE_SSH_TIMEOUT_MS || 30000)
const SSH_RETRY_COUNT = Number(process.env.HISTORY_IMPORT_VALIDATE_SSH_RETRY_COUNT || 3)
const SSH_ARGS = [
'-o',
'BatchMode=yes',
'-o',
'ConnectTimeout=15',
'-o',
'ServerAliveInterval=15',
'-o',
'ServerAliveCountMax=2',
]
type ImportResultItem = {
fingerprint: string
spu: string
sourcePath: string
materialId: string | null
status: string
reason: string | null
}
function parseArgs() {
const args = process.argv.slice(2)
const getArg = (name: string) => {
const match = args.find((arg) => arg.startsWith(`--${name}=`))
return match ? match.slice(name.length + 3) : ''
}
const input = getArg('input')
const remoteHost = getArg('remote-host') || process.env.HISTORY_IMPORT_REMOTE_UPLOADS_HOST || 'prod'
const remoteUploadsDir =
getArg('remote-uploads-dir') ||
process.env.HISTORY_IMPORT_REMOTE_UPLOADS_DIR ||
'/mnt/PoolOne/apps/youyidian-3d/uploads'
if (!input) {
throw new Error('缺少 --input=导入结果 JSON')
}
return { input, remoteHost, remoteUploadsDir }
}
function shellQuote(value: string) {
return `'${value.replace(/'/g, `'\\''`)}'`
}
function storedPathToRemotePath(remoteUploadsDir: string, storedPath: string) {
const normalized = String(storedPath || '').replace(/\\/g, '/')
const relative = normalized.replace(/^\.?\/*uploads\//i, '')
if (!relative || relative.includes('../')) {
throw new Error(`下载文件路径无效: ${storedPath}`)
}
return path.posix.join(remoteUploadsDir.replace(/\/+$/, ''), relative)
}
function sleep(ms: number) {
return new Promise((resolve) => setTimeout(resolve, ms))
}
async function getRemoteFileSize(remoteHost: string, remotePath: string) {
let lastError: unknown
for (let attempt = 1; attempt <= SSH_RETRY_COUNT; attempt += 1) {
try {
const { stdout } = await execFileAsync(
'ssh',
[
...SSH_ARGS,
remoteHost,
`test -f ${shellQuote(remotePath)} && stat -c %s ${shellQuote(remotePath)}`,
],
{ timeout: SSH_TIMEOUT_MS },
)
return Number(String(stdout || '').trim())
} catch (error) {
lastError = error
if (attempt < SSH_RETRY_COUNT) {
await sleep(1000 * attempt)
}
}
}
throw lastError
}
async function main() {
const options = parseArgs()
const prisma = new PrismaClient()
try {
const raw = await fs.readFile(options.input, 'utf8')
const results = JSON.parse(raw) as ImportResultItem[]
const imported = results.filter((item) => item.status === 'imported' && item.materialId)
const failures = results.filter((item) => item.status === 'failed')
const skipped = results.filter((item) => item.status === 'skipped')
const ids = imported.map((item) => item.materialId!).filter(Boolean)
const rows = await prisma.material.findMany({
where: { id: { in: ids } },
select: {
id: true,
productModel: true,
previewUrl: true,
zipFilePath: true,
zipFileSize: true,
isFullSet: true,
deletedAt: true,
},
})
const rowById = new Map(rows.map((row) => [row.id, row]))
const sourceAssets = await prisma.materialAsset.findMany({
where: {
materialId: { in: ids },
assetType: 'psd',
},
select: {
materialId: true,
fileUrl: true,
fileName: true,
fileSize: true,
},
})
const sourceAssetsByMaterialId = new Map<string, typeof sourceAssets>()
for (const asset of sourceAssets) {
const current = sourceAssetsByMaterialId.get(asset.materialId) || []
current.push(asset)
sourceAssetsByMaterialId.set(asset.materialId, current)
}
const checks = []
for (const item of imported) {
const row = rowById.get(item.materialId!)
if (!row) {
checks.push({
materialId: item.materialId,
spu: item.spu,
ok: false,
issue: '数据库记录不存在',
})
continue
}
const remotePath = storedPathToRemotePath(options.remoteUploadsDir, row.zipFilePath || '')
let remoteSize = -1
try {
remoteSize = await getRemoteFileSize(options.remoteHost, remotePath)
} catch {
remoteSize = -1
}
let downloadStatus = 0
try {
const response = await fetch(
`https://sc.jiansun.vip/api/materials/download?id=${row.id}&type=all&validate=1&share=1`,
)
downloadStatus = response.status
} catch {
downloadStatus = -1
}
let previewStatus = 0
try {
const response = await fetch(row.previewUrl || '', { method: 'HEAD' })
previewStatus = response.status
} catch {
previewStatus = -1
}
const expectedSize = Number(row.zipFileSize || 0)
const materialSourceAssets = sourceAssetsByMaterialId.get(row.id) || []
const sourceChecks = []
for (const asset of materialSourceAssets) {
const sourcePathOk =
String(asset.fileUrl || '').startsWith('uploads/materials/') &&
!String(asset.fileUrl || '').includes('\\')
const expectedSourceSize = Number(asset.fileSize || 0)
let remoteSourceSize = -1
if (sourcePathOk) {
try {
remoteSourceSize = await getRemoteFileSize(
options.remoteHost,
storedPathToRemotePath(options.remoteUploadsDir, asset.fileUrl || ''),
)
} catch {
remoteSourceSize = -1
}
}
sourceChecks.push({
fileName: asset.fileName,
sourcePathOk,
remoteSize: remoteSourceSize,
expectedSize: expectedSourceSize,
ok: sourcePathOk && remoteSourceSize === expectedSourceSize,
})
}
const sourceAssetsOk = sourceChecks.every((asset) => asset.ok)
const previewOk = String(row.previewUrl || '').startsWith(
'https://s3.jiansun.vip/gongxiangpan/由壹点AI/public-display/',
)
const zipPathOk =
String(row.zipFilePath || '').startsWith('uploads/materials/') &&
!String(row.zipFilePath || '').includes('\\')
const ok =
!row.deletedAt &&
row.isFullSet &&
previewOk &&
zipPathOk &&
sourceAssetsOk &&
remoteSize === expectedSize &&
downloadStatus === 200 &&
previewStatus === 200
checks.push({
materialId: row.id,
spu: row.productModel,
ok,
previewOk,
zipPathOk,
sourceAssetCount: materialSourceAssets.length,
sourceAssetsOk,
sourceIssues: sourceChecks.filter((asset) => !asset.ok),
remoteSize,
expectedSize,
downloadStatus,
previewStatus,
})
}
const summary = {
input: options.input,
total: results.length,
imported: imported.length,
skipped: skipped.length,
failed: failures.length,
checked: checks.length,
ok: checks.filter((item) => item.ok).length,
issueCount: checks.filter((item) => !item.ok).length + failures.length,
issues: [
...failures.map((item) => ({
materialId: item.materialId,
spu: item.spu,
ok: false,
issue: item.reason || '导入失败',
})),
...checks.filter((item) => !item.ok),
],
checkedAt: new Date().toISOString(),
}
const outputPath = options.input.replace(/\.json$/i, '.verification.json')
await fs.writeFile(outputPath, JSON.stringify(summary, null, 2), 'utf8')
console.log(JSON.stringify(summary, null, 2))
if (summary.failed > 0 || summary.issueCount > 0 || summary.imported + summary.skipped !== summary.total) {
process.exitCode = 1
}
} finally {
await prisma.$disconnect()
}
}
main().catch((error) => {
console.error('导入结果验收失败:', error)
process.exitCode = 1
})