feat: 任务详情无条码开关 + 上传面板信息密度优化 + 历史导入/脚本补全
- 任务详情上传支持"本批素材无条码"开关,跳过条码必填校验 - SKU卡片/主图网格从4列改5列,图片高度压缩,去掉冗余标签和提示 - 详情图/PSD上传区去掉大占位框,改成紧凑按钮+标签 - 补充历史导入相关脚本和工具文件 Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
parent
340f688b7b
commit
9cd3c0f231
@ -22,6 +22,11 @@ import {
|
||||
} from '@/lib/download-errors';
|
||||
import { isProductPackageType } from '@/lib/material-type-mapper';
|
||||
import { ensureMaterialDownloadLogsTable } from '@/lib/material-download-log';
|
||||
import { resolveStoredUploadPath } from '@/lib/upload-paths';
|
||||
|
||||
function normalizeStoredLocalPath(filePath: string) {
|
||||
return String(filePath || '').replace(/\\/g, '/');
|
||||
}
|
||||
|
||||
// GET - 下载素材
|
||||
export async function GET(request: NextRequest) {
|
||||
@ -156,10 +161,8 @@ export async function GET(request: NextRequest) {
|
||||
);
|
||||
}
|
||||
|
||||
let filePath = material.zipFilePath;
|
||||
if (!path.isAbsolute(filePath)) {
|
||||
filePath = path.join(process.cwd(), filePath);
|
||||
}
|
||||
let filePath = normalizeStoredLocalPath(material.zipFilePath);
|
||||
filePath = path.isAbsolute(filePath) ? filePath : resolveStoredUploadPath(filePath);
|
||||
|
||||
return serveFile(filePath, material, downloadType || '源文件', validateOnly);
|
||||
} catch (error) {
|
||||
@ -201,7 +204,10 @@ async function handleProductPackageDownload(
|
||||
where: {
|
||||
materialId: material.id,
|
||||
assetType,
|
||||
fileUrl: { startsWith: 'uploads/' }, // 只查询本地 zip 文件
|
||||
OR: [
|
||||
{ fileUrl: { startsWith: 'uploads/' } },
|
||||
{ fileUrl: { startsWith: 'uploads\\' } },
|
||||
],
|
||||
},
|
||||
});
|
||||
}
|
||||
@ -214,10 +220,10 @@ async function handleProductPackageDownload(
|
||||
);
|
||||
}
|
||||
|
||||
const filePath = asset.fileUrl;
|
||||
const absolutePath = path.isAbsolute(filePath)
|
||||
? filePath
|
||||
: path.join(process.cwd(), filePath);
|
||||
const filePath = normalizeStoredLocalPath(asset.fileUrl);
|
||||
const absolutePath = path.isAbsolute(filePath)
|
||||
? filePath
|
||||
: resolveStoredUploadPath(filePath);
|
||||
|
||||
console.log(`[Download] 文件路径: ${absolutePath}`);
|
||||
console.log(`[Download] 文件名: ${asset.fileName}`);
|
||||
|
||||
@ -33,6 +33,7 @@ interface PsdAsset {
|
||||
interface CompleteRequestBody {
|
||||
materialId: string
|
||||
taskNo?: string
|
||||
noBarcodes?: boolean
|
||||
assets: {
|
||||
main?: AssetUpload[]
|
||||
sku?: AssetUpload[]
|
||||
@ -79,6 +80,9 @@ export async function POST(request: NextRequest) {
|
||||
taskNo: body.taskNo,
|
||||
assets: body.assets,
|
||||
operatorUserId: (session.user as any).id,
|
||||
options: {
|
||||
allowSkuWithoutBarcodes: body.noBarcodes === true,
|
||||
},
|
||||
})
|
||||
|
||||
return NextResponse.json({
|
||||
|
||||
@ -114,6 +114,7 @@ export default function ProductPackageUploader({
|
||||
const [isPsdDragging, setIsPsdDragging] = useState(false);
|
||||
|
||||
// 上传状态
|
||||
const [noBarcodes, setNoBarcodes] = useState(false);
|
||||
const [uploading, setUploading] = useState(false);
|
||||
const [uploadProgress, setUploadProgress] = useState(0);
|
||||
const [processingStep, setProcessingStep] = useState('');
|
||||
@ -406,7 +407,7 @@ export default function ProductPackageUploader({
|
||||
}
|
||||
|
||||
const barcodes = parseBarcodeInput(image.barcodeInput || '');
|
||||
if (barcodes.length === 0) {
|
||||
if (!noBarcodes && barcodes.length === 0) {
|
||||
throw new Error(`请为 SKU 图 ${image.file.name} 至少填写 1 个产品条码`);
|
||||
}
|
||||
|
||||
@ -560,6 +561,7 @@ export default function ProductPackageUploader({
|
||||
body: JSON.stringify({
|
||||
materialId,
|
||||
taskNo, // 传递taskNo,上传完成后写入数据库
|
||||
noBarcodes,
|
||||
assets: {
|
||||
main: mainAssets,
|
||||
sku: skuAssets,
|
||||
@ -698,9 +700,9 @@ export default function ProductPackageUploader({
|
||||
</select>
|
||||
</div>
|
||||
|
||||
<div className="rounded-apple border border-blue-200 bg-blue-50 px-4 py-3 text-sm text-blue-700 dark:border-blue-800 dark:bg-blue-900/20 dark:text-blue-200">
|
||||
系统会按素材实际内容自动判定是否整套:同时拥有主图、SKU 图、详情图时,自动标记为整套,无需手工选择。
|
||||
</div>
|
||||
<p className="text-xs text-gray-400 dark:text-gray-500">
|
||||
同时上传主图 + SKU 图 + 详情图时自动标记为整套
|
||||
</p>
|
||||
|
||||
{/* 主图上传 */}
|
||||
<div>
|
||||
@ -729,15 +731,15 @@ export default function ProductPackageUploader({
|
||||
className="hidden"
|
||||
disabled={uploading}
|
||||
/>
|
||||
<div className="grid grid-cols-4 gap-4">
|
||||
<div className="grid grid-cols-5 gap-3">
|
||||
{mainImages.map((img) => (
|
||||
<div key={img.id} className="relative group">
|
||||
<div className="relative w-full h-32 rounded-apple overflow-hidden border-2 border-gray-200 dark:border-gray-600">
|
||||
<div className="relative w-full h-24 rounded-apple overflow-hidden border-2 border-gray-200 dark:border-gray-600">
|
||||
<img src={img.preview} alt="" className="w-full h-full object-cover" />
|
||||
</div>
|
||||
<button
|
||||
onClick={() => handleRemoveImage(img.id, setMainImages)}
|
||||
className="absolute -top-2 -right-2 bg-red-500 text-white rounded-full w-6 h-6 flex items-center justify-center opacity-0 group-hover:opacity-100 transition-opacity"
|
||||
className="absolute -top-1.5 -right-1.5 bg-red-500 text-white rounded-full w-5 h-5 flex items-center justify-center text-xs opacity-0 group-hover:opacity-100 transition-opacity"
|
||||
disabled={uploading}
|
||||
type="button"
|
||||
>
|
||||
@ -749,25 +751,34 @@ export default function ProductPackageUploader({
|
||||
type="button"
|
||||
onClick={() => mainInputRef.current?.click()}
|
||||
disabled={uploading}
|
||||
className="group w-full h-32 rounded-apple bg-gray-100 dark:bg-gray-700 border-2 border-dashed border-gray-300 dark:border-gray-600 flex items-center justify-center hover:border-gray-400 dark:hover:border-gray-500 transition-colors disabled:opacity-50"
|
||||
className="group w-full h-24 rounded-apple bg-gray-100 dark:bg-gray-700 border-2 border-dashed border-gray-300 dark:border-gray-600 flex items-center justify-center hover:border-gray-400 dark:hover:border-gray-500 transition-colors disabled:opacity-50"
|
||||
>
|
||||
<svg className="w-12 h-12 text-gray-400 group-hover:text-gray-500" fill="none" stroke="currentColor" viewBox="0 0 24 24">
|
||||
<svg className="w-8 h-8 text-gray-400 group-hover:text-gray-500" fill="none" stroke="currentColor" viewBox="0 0 24 24">
|
||||
<title>添加主图</title>
|
||||
<path strokeLinecap="round" strokeLinejoin="round" strokeWidth={2} d="M12 4v16m8-8H4" />
|
||||
</svg>
|
||||
</button>
|
||||
</div>
|
||||
<p className="text-xs text-gray-500 dark:text-gray-400 mt-3">
|
||||
支持 JPG、PNG、WebP 等格式,可一次选择多张或拖拽上传。每张 SKU 图都要填写 SKU 编号,并录入至少 1 个产品条码。
|
||||
</p>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* SKU图上传 */}
|
||||
<div>
|
||||
<label htmlFor="sku-images-input" className="block text-sm font-medium text-gray-700 dark:text-gray-300 mb-2">
|
||||
SKU图
|
||||
</label>
|
||||
<div className="flex items-center justify-between mb-2">
|
||||
<label htmlFor="sku-images-input" className="block text-sm font-medium text-gray-700 dark:text-gray-300">
|
||||
SKU图
|
||||
</label>
|
||||
<label className="flex items-center gap-2 text-sm text-gray-600 dark:text-gray-400 cursor-pointer select-none">
|
||||
<input
|
||||
type="checkbox"
|
||||
checked={noBarcodes}
|
||||
onChange={(e) => setNoBarcodes(e.target.checked)}
|
||||
disabled={uploading}
|
||||
className="rounded border-gray-300 text-blue-600 focus:ring-blue-500 dark:border-gray-600"
|
||||
/>
|
||||
本批素材无条码
|
||||
</label>
|
||||
</div>
|
||||
{/* biome-ignore lint/a11y/noStaticElementInteractions: 拖拽上传区域需要使用 drag/drop 事件 */}
|
||||
<div
|
||||
className={`border-2 border-dashed rounded-apple p-4 transition-colors ${
|
||||
@ -790,65 +801,52 @@ export default function ProductPackageUploader({
|
||||
className="hidden"
|
||||
disabled={uploading}
|
||||
/>
|
||||
<div className="grid grid-cols-4 gap-4">
|
||||
<div className="grid grid-cols-5 gap-3">
|
||||
{skuImages.map((img) => (
|
||||
<div key={img.id} className="relative group space-y-3 rounded-apple border border-gray-200 bg-white p-3 dark:border-gray-700 dark:bg-gray-800/80">
|
||||
<div className="relative w-full h-32 rounded-apple overflow-hidden border-2 border-gray-200 dark:border-gray-600">
|
||||
<div key={img.id} className="relative group space-y-1.5 rounded-apple border border-gray-200 bg-white p-2 dark:border-gray-700 dark:bg-gray-800/80">
|
||||
<div className="relative w-full h-20 rounded-apple overflow-hidden border border-gray-200 dark:border-gray-600">
|
||||
<img src={img.preview} alt="" className="w-full h-full object-cover" />
|
||||
</div>
|
||||
<button
|
||||
onClick={() => handleRemoveImage(img.id, setSkuImages)}
|
||||
className="absolute -top-2 -right-2 bg-red-500 text-white rounded-full w-6 h-6 flex items-center justify-center opacity-0 group-hover:opacity-100 transition-opacity"
|
||||
className="absolute -top-1.5 -right-1.5 bg-red-500 text-white rounded-full w-5 h-5 flex items-center justify-center text-xs opacity-0 group-hover:opacity-100 transition-opacity"
|
||||
disabled={uploading}
|
||||
type="button"
|
||||
>
|
||||
×
|
||||
</button>
|
||||
<div className="space-y-2">
|
||||
<div>
|
||||
<label className="mb-1 block text-xs font-medium text-gray-600 dark:text-gray-300">
|
||||
SKU 编号
|
||||
</label>
|
||||
<input
|
||||
type="text"
|
||||
value={img.skuCode || ''}
|
||||
onChange={(e) => updateSkuImageMeta(img.id, { skuCode: e.target.value })}
|
||||
placeholder="例如 SKU001"
|
||||
disabled={uploading}
|
||||
className="w-full rounded-apple border border-gray-300 px-3 py-2 text-sm dark:border-gray-600 dark:bg-gray-700 dark:text-white"
|
||||
/>
|
||||
</div>
|
||||
<div>
|
||||
<label className="mb-1 block text-xs font-medium text-gray-600 dark:text-gray-300">
|
||||
产品条码
|
||||
</label>
|
||||
<textarea
|
||||
value={img.barcodeInput || ''}
|
||||
onChange={(e) => updateSkuImageMeta(img.id, { barcodeInput: e.target.value })}
|
||||
placeholder="每行一个,或用逗号分隔"
|
||||
disabled={uploading}
|
||||
rows={4}
|
||||
className="w-full rounded-apple border border-gray-300 px-3 py-2 text-sm dark:border-gray-600 dark:bg-gray-700 dark:text-white"
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
<input
|
||||
type="text"
|
||||
value={img.skuCode || ''}
|
||||
onChange={(e) => updateSkuImageMeta(img.id, { skuCode: e.target.value })}
|
||||
placeholder="SKU 编号"
|
||||
disabled={uploading}
|
||||
className="w-full rounded border border-gray-300 px-2 py-1 text-xs dark:border-gray-600 dark:bg-gray-700 dark:text-white"
|
||||
/>
|
||||
{!noBarcodes && (
|
||||
<textarea
|
||||
value={img.barcodeInput || ''}
|
||||
onChange={(e) => updateSkuImageMeta(img.id, { barcodeInput: e.target.value })}
|
||||
placeholder="产品条码(逗号分隔)"
|
||||
disabled={uploading}
|
||||
rows={2}
|
||||
className="w-full rounded border border-gray-300 px-2 py-1 text-xs dark:border-gray-600 dark:bg-gray-700 dark:text-white"
|
||||
/>
|
||||
)}
|
||||
</div>
|
||||
))}
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => skuInputRef.current?.click()}
|
||||
disabled={uploading}
|
||||
className="group w-full h-32 rounded-apple bg-gray-100 dark:bg-gray-700 border-2 border-dashed border-gray-300 dark:border-gray-600 flex items-center justify-center hover:border-gray-400 dark:hover:border-gray-500 transition-colors disabled:opacity-50"
|
||||
className="group w-full h-20 rounded-apple bg-gray-100 dark:bg-gray-700 border-2 border-dashed border-gray-300 dark:border-gray-600 flex items-center justify-center hover:border-gray-400 dark:hover:border-gray-500 transition-colors disabled:opacity-50"
|
||||
>
|
||||
<svg className="w-12 h-12 text-gray-400 group-hover:text-gray-500" fill="none" stroke="currentColor" viewBox="0 0 24 24">
|
||||
<svg className="w-8 h-8 text-gray-400 group-hover:text-gray-500" fill="none" stroke="currentColor" viewBox="0 0 24 24">
|
||||
<title>添加SKU图</title>
|
||||
<path strokeLinecap="round" strokeLinejoin="round" strokeWidth={2} d="M12 4v16m8-8H4" />
|
||||
</svg>
|
||||
</button>
|
||||
</div>
|
||||
<p className="text-xs text-gray-500 dark:text-gray-400 mt-3">
|
||||
支持 JPG、PNG、WebP 等格式,可一次选择多张或拖拽上传
|
||||
</p>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
@ -880,26 +878,16 @@ export default function ProductPackageUploader({
|
||||
disabled={uploading}
|
||||
/>
|
||||
{/* 上传按钮 */}
|
||||
<div className="flex items-center space-x-4 mb-4">
|
||||
<div className="w-32 h-32 rounded-apple bg-gray-100 dark:bg-gray-700 flex items-center justify-center">
|
||||
<svg className="w-12 h-12 text-gray-400" fill="none" stroke="currentColor" viewBox="0 0 24 24">
|
||||
<title>添加详情图</title>
|
||||
<path strokeLinecap="round" strokeLinejoin="round" strokeWidth={2} d="M12 4v16m8-8H4" />
|
||||
</svg>
|
||||
</div>
|
||||
<div className="flex-1">
|
||||
<button
|
||||
onClick={() => detailInputRef.current?.click()}
|
||||
className="btn-secondary"
|
||||
disabled={uploading}
|
||||
type="button"
|
||||
>
|
||||
{detailImages.length > 0 ? '继续添加图片' : '选择详情图'}
|
||||
</button>
|
||||
<p className="text-xs text-gray-500 dark:text-gray-400 mt-2">
|
||||
支持 JPG、PNG、WebP 等格式,可一次选择多张或拖拽上传
|
||||
</p>
|
||||
</div>
|
||||
<div className="flex items-center space-x-3 mb-3">
|
||||
<button
|
||||
onClick={() => detailInputRef.current?.click()}
|
||||
className="btn-secondary text-sm"
|
||||
disabled={uploading}
|
||||
type="button"
|
||||
>
|
||||
{detailImages.length > 0 ? '继续添加' : '选择详情图'}
|
||||
</button>
|
||||
<span className="text-xs text-gray-400">JPG / PNG / WebP,支持拖拽</span>
|
||||
</div>
|
||||
|
||||
{/* 详情图排序组件 */}
|
||||
@ -943,55 +931,33 @@ export default function ProductPackageUploader({
|
||||
disabled={uploading}
|
||||
/>
|
||||
{/* 上传按钮 */}
|
||||
<div className="flex items-center space-x-4">
|
||||
{psdFiles.length > 0 ? (
|
||||
<div className="relative w-32 h-32 rounded-apple overflow-hidden border-2 border-gray-200 dark:border-gray-600 bg-gray-100 dark:bg-gray-700 flex items-center justify-center">
|
||||
<svg className="w-12 h-12 text-gray-400" fill="none" stroke="currentColor" viewBox="0 0 24 24">
|
||||
<title>PSD文件</title>
|
||||
<path strokeLinecap="round" strokeLinejoin="round" strokeWidth={2} d="M7 21h10a2 2 0 002-2V9.414a1 1 0 00-.293-.707l-5.414-5.414A1 1 0 0012.586 3H7a2 2 0 00-2 2v14a2 2 0 002 2z" />
|
||||
</svg>
|
||||
<div className="absolute bottom-1 right-1 bg-blue-500 text-white text-xs rounded-full w-6 h-6 flex items-center justify-center">
|
||||
{psdFiles.length}
|
||||
</div>
|
||||
</div>
|
||||
) : (
|
||||
<div className="w-32 h-32 rounded-apple bg-gray-100 dark:bg-gray-700 flex items-center justify-center">
|
||||
<svg className="w-12 h-12 text-gray-400" fill="none" stroke="currentColor" viewBox="0 0 24 24">
|
||||
<title>添加PSD文件</title>
|
||||
<path strokeLinecap="round" strokeLinejoin="round" strokeWidth={2} d="M12 4v16m8-8H4" />
|
||||
</svg>
|
||||
<div className="flex items-center space-x-3">
|
||||
<button
|
||||
onClick={() => psdInputRef.current?.click()}
|
||||
className="btn-secondary text-sm"
|
||||
disabled={uploading}
|
||||
type="button"
|
||||
>
|
||||
{psdFiles.length > 0 ? `继续添加 (${psdFiles.length}个)` : '选择源文件'}
|
||||
</button>
|
||||
<span className="text-xs text-gray-400">{TASK_DETAIL_SOURCE_FILE_LABEL}</span>
|
||||
{psdFiles.length > 0 && (
|
||||
<div className="flex flex-wrap gap-1.5 ml-2">
|
||||
{psdFiles.map((psdFile) => (
|
||||
<span key={psdFile.id} className="inline-flex items-center gap-1 px-2 py-0.5 bg-gray-100 dark:bg-gray-700 rounded text-xs text-gray-700 dark:text-gray-300">
|
||||
{psdFile.file.name}
|
||||
<button
|
||||
onClick={() => setPsdFiles((prev) => prev.filter((f) => f.id !== psdFile.id))}
|
||||
className="text-red-400 hover:text-red-600"
|
||||
disabled={uploading}
|
||||
type="button"
|
||||
>
|
||||
×
|
||||
</button>
|
||||
</span>
|
||||
))}
|
||||
</div>
|
||||
)}
|
||||
<div className="flex-1">
|
||||
<button
|
||||
onClick={() => psdInputRef.current?.click()}
|
||||
className="btn-secondary"
|
||||
disabled={uploading}
|
||||
type="button"
|
||||
>
|
||||
{psdFiles.length > 0 ? '继续添加PSD文件' : '选择PSD文件'}
|
||||
</button>
|
||||
<p className="text-xs text-gray-500 dark:text-gray-400 mt-2">
|
||||
支持 {TASK_DETAIL_SOURCE_FILE_LABEL} 格式,可一次选择多个或拖拽上传
|
||||
</p>
|
||||
{psdFiles.length > 0 && (
|
||||
<div className="mt-2 space-y-2">
|
||||
{psdFiles.map((psdFile) => (
|
||||
<div key={psdFile.id} className="flex items-center justify-between p-2 bg-gray-100 dark:bg-gray-700 rounded-apple">
|
||||
<span className="text-sm text-gray-700 dark:text-gray-300">{psdFile.file.name}</span>
|
||||
<button
|
||||
onClick={() => setPsdFiles((prev) => prev.filter((f) => f.id !== psdFile.id))}
|
||||
className="text-red-500 dark:text-red-400 text-sm hover:text-red-600 dark:hover:text-red-300"
|
||||
disabled={uploading}
|
||||
type="button"
|
||||
>
|
||||
删除
|
||||
</button>
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
@ -1,11 +1,11 @@
|
||||
# 生产存储桶接入信息
|
||||
|
||||
> 更新时间:2026-05-07
|
||||
> 更新时间:2026-05-08
|
||||
> 本文记录当前素材平台的 MinIO / S3 / uploads 存储事实。
|
||||
|
||||
## 1. 当前结论
|
||||
|
||||
- 新素材预览 / 展示文件:写入生产 MinIO 新桶路径,公开 URL 统一为 `https://s3.jiansun.vip/gongxiangpan/public-display/...`。
|
||||
- 新素材预览 / 展示文件:写入生产 MinIO 新桶路径,公开 URL 统一为 `https://s3.jiansun.vip/gongxiangpan/由壹点AI/public-display/...`。
|
||||
- 源文件 / 下载型文件:写入生产服务器 uploads 目录:`/mnt/PoolOne/apps/youyidian-3d/uploads`。
|
||||
- 旧 `1080p.hk` 桶:只用于极少数老素材兼容,不作为新素材写入目标。
|
||||
- `/gongxiangpan/uploads/` 是历史混合区,不再作为新展示资源的默认路径。
|
||||
@ -26,6 +26,7 @@ S3_ENDPOINT=http://192.168.58.20:9000
|
||||
S3_BUCKET=gongxiangpan
|
||||
S3_REGION=us-east-1
|
||||
S3_PUBLIC_DOMAIN=https://s3.jiansun.vip/gongxiangpan
|
||||
S3_FILE_PATH=由壹点AI/
|
||||
```
|
||||
|
||||
代码固定行为:
|
||||
@ -47,16 +48,16 @@ S3_PUBLIC_DOMAIN=https://s3.jiansun.vip/gongxiangpan
|
||||
正式路径:
|
||||
|
||||
```text
|
||||
https://s3.jiansun.vip/gongxiangpan/public-display/...
|
||||
https://s3.jiansun.vip/gongxiangpan/由壹点AI/public-display/...
|
||||
```
|
||||
|
||||
常见前缀:
|
||||
|
||||
```text
|
||||
public-display/materials/...
|
||||
public-display/task-detail/...
|
||||
public-display/white-background/...
|
||||
public-display/certificate-report/...
|
||||
由壹点AI/public-display/materials/...
|
||||
由壹点AI/public-display/task-detail/...
|
||||
由壹点AI/public-display/white-background/...
|
||||
由壹点AI/public-display/certificate-report/...
|
||||
```
|
||||
|
||||
这些资源用于页面展示、AI 平台预览、钉钉返回地址和外部接口展示直链。
|
||||
@ -69,7 +70,7 @@ public-display/certificate-report/...
|
||||
由壹点AI/
|
||||
```
|
||||
|
||||
它主要用于旧逻辑或非 `public-display` 的对象,不代表新素材展示图默认落点。
|
||||
它是当前生产环境的默认业务前缀。新素材展示图应落在 `由壹点AI/public-display/...`,不要写入旧桶。
|
||||
|
||||
## 4. uploads 目录规则
|
||||
|
||||
@ -101,21 +102,40 @@ public-display/certificate-report/...
|
||||
|
||||
不要把 uploads 理解成“少量视频临时区”。它是当前生产源文件和下载型文件的核心落点。
|
||||
|
||||
## 5. 历史兼容
|
||||
## 5. 历史素材正式导入要求
|
||||
|
||||
### 5.1 旧 `1080p.hk` 桶
|
||||
历史素材导入有一个特殊约束:执行机要能读取历史素材源目录,同时下载型文件必须最终落到生产 uploads。
|
||||
|
||||
当前天蓝盘批量导入的标准做法:
|
||||
|
||||
- Windows 维护机读取 `Z:\...` 源素材。
|
||||
- 预览图上传到新桶 `gongxiangpan` 的 `由壹点AI/public-display/...`。
|
||||
- ZIP / PSD / 源文件先在本机生成,再通过远程同步写入生产机 `/mnt/PoolOne/apps/youyidian-3d/uploads`,并校验远程文件大小。
|
||||
|
||||
正式 `--apply` 如果在 Windows 上连接生产库,必须同时配置:
|
||||
|
||||
```env
|
||||
HISTORY_IMPORT_REMOTE_UPLOADS_HOST=prod
|
||||
HISTORY_IMPORT_REMOTE_UPLOADS_DIR=/mnt/PoolOne/apps/youyidian-3d/uploads
|
||||
```
|
||||
|
||||
没有远程 uploads 同步时,脚本必须阻止正式导入。否则会把数据库写到生产库,但 ZIP / PSD 留在本机 `uploads`,线上下载会 404。
|
||||
|
||||
## 6. 历史兼容
|
||||
|
||||
### 6.1 旧 `1080p.hk` 桶
|
||||
|
||||
`s3.1080p.hk/jsxiangche` 只保留给极少数老素材兼容。新素材、批量新录入素材、后续上传素材都不应继续写入旧桶。
|
||||
|
||||
### 5.2 旧资源域名
|
||||
### 6.2 旧资源域名
|
||||
|
||||
`s3.yydd.ai` 不再作为默认对接口径,只用于识别历史链接。
|
||||
|
||||
### 5.3 历史混合区
|
||||
### 6.3 历史混合区
|
||||
|
||||
`/gongxiangpan/uploads/` 保留历史兼容,不再作为新展示图默认路径。
|
||||
|
||||
## 6. 只读核对命令
|
||||
## 7. 只读核对命令
|
||||
|
||||
核对应用环境变量:
|
||||
|
||||
|
||||
@ -1,7 +1,12 @@
|
||||
import { prisma } from '@/lib/prisma'
|
||||
import { completeProductPackageUpload, type ProductPackageImageInput } from '@/lib/product-package-completion'
|
||||
import {
|
||||
completeProductPackageUpload,
|
||||
type ProductPackageImageInput,
|
||||
type ProductPackageSourceInput,
|
||||
} from '@/lib/product-package-completion'
|
||||
import { createMaterialLog } from '@/lib/material-log'
|
||||
import { isTaskDetailFullSetFromUploadGroups } from '@/lib/task-detail-full-set'
|
||||
import { normalizeTaskDetailSourceType } from '@/lib/task-detail-source-type'
|
||||
import path from 'path'
|
||||
import { promises as fs } from 'fs'
|
||||
import crypto from 'crypto'
|
||||
@ -33,6 +38,10 @@ const IGNORED_DIR_PATTERNS = [
|
||||
/废稿/i,
|
||||
/字体/i,
|
||||
/勿用/i,
|
||||
/错误勿用/i,
|
||||
/图片有误/i,
|
||||
/作废/i,
|
||||
/3D建模/i,
|
||||
/DingTalkAppData/i,
|
||||
/^PsTool$/i,
|
||||
/^Cache$/i,
|
||||
@ -68,6 +77,20 @@ const STORE_DIR_IGNORES = new Set([
|
||||
const execFileAsync = promisify(execFile)
|
||||
const HISTORY_ZIP_TIMEOUT_MS = Number(process.env.HISTORY_IMPORT_ZIP_TIMEOUT_MS || 180000)
|
||||
const HISTORY_ALL_ZIP_TIMEOUT_MS = Number(process.env.HISTORY_IMPORT_ALL_ZIP_TIMEOUT_MS || 300000)
|
||||
const BRANDED_SPU_BODY_PATTERN = '[A-Z]{2,5}-[A-Z]{1,6}\\d{3,5}(?:K\\d{1,3})?'
|
||||
const UNBRANDED_SPU_BODY_PATTERN = '[A-Z]{1,6}\\d{3,5}(?:K\\d{1,3})?'
|
||||
const COMBO_SPU_TOKEN_PATTERN = new RegExp(
|
||||
`(?<![A-Z0-9])${BRANDED_SPU_BODY_PATTERN}(?:\\s*[++]\\s*(?:${BRANDED_SPU_BODY_PATTERN}|${UNBRANDED_SPU_BODY_PATTERN}))+(?![A-Z0-9])`,
|
||||
'i',
|
||||
)
|
||||
const BLOCKED_READY_SOURCE_PATH_PATTERNS = [
|
||||
{ pattern: /错误勿用/i, label: '错误勿用' },
|
||||
{ pattern: /废稿/i, label: '废稿' },
|
||||
{ pattern: /勿用/i, label: '勿用' },
|
||||
{ pattern: /图片有误/i, label: '图片有误' },
|
||||
{ pattern: /作废/i, label: '作废' },
|
||||
{ pattern: /3D建模/i, label: '3D建模' },
|
||||
]
|
||||
|
||||
export type HistoryImportStatus = 'ready' | 'needs_review' | 'skipped'
|
||||
|
||||
@ -79,16 +102,19 @@ export interface HistoryImportRecord {
|
||||
guessedStore: string
|
||||
productCategoryName: string
|
||||
authorName: string
|
||||
taskDetailSourceType?: string
|
||||
storeConfidence: number
|
||||
storeCandidates: string[]
|
||||
mainCount: number
|
||||
skuCount: number
|
||||
detailCount: number
|
||||
sourceCount?: number
|
||||
status: HistoryImportStatus
|
||||
skipReason: string | null
|
||||
mainImages: ProductPackageImageInput[]
|
||||
skuImages: ProductPackageImageInput[]
|
||||
detailImages: ProductPackageImageInput[]
|
||||
psdSources?: ProductPackageSourceInput[]
|
||||
seenInYya: boolean
|
||||
}
|
||||
|
||||
@ -133,8 +159,52 @@ interface FileEntry {
|
||||
fileSize: number
|
||||
}
|
||||
|
||||
interface SanitizedHistoryImportRecord {
|
||||
record: HistoryImportRecord
|
||||
skippedEmptyFiles: Array<{
|
||||
group: 'main' | 'sku' | 'detail' | 'psd'
|
||||
fileName: string
|
||||
localFilePath?: string
|
||||
}>
|
||||
skippedInvalidFiles: Array<{
|
||||
group: 'main' | 'sku' | 'detail'
|
||||
fileName: string
|
||||
fileSize: number
|
||||
localFilePath?: string
|
||||
reason: string
|
||||
}>
|
||||
}
|
||||
|
||||
function normalizeSpu(value: string) {
|
||||
return value.trim().toUpperCase()
|
||||
return value.trim().toUpperCase().replace(/+/g, '+').replace(/\s*\+\s*/g, '+')
|
||||
}
|
||||
|
||||
function expandComboSpuToken(token: string) {
|
||||
const parts = normalizeSpu(token).split('+').filter(Boolean)
|
||||
const first = parts[0] || ''
|
||||
const prefix = first.includes('-') ? first.split('-')[0] : ''
|
||||
|
||||
if (parts.length < 2 || !prefix) {
|
||||
return null
|
||||
}
|
||||
|
||||
const expanded = parts.map((part, index) => {
|
||||
if (index === 0 || part.includes('-')) {
|
||||
return part
|
||||
}
|
||||
return `${prefix}-${part}`
|
||||
})
|
||||
|
||||
return expanded.join('+')
|
||||
}
|
||||
|
||||
function inferComboSpuFromPath(sourcePath: string) {
|
||||
const match = sourcePath.match(COMBO_SPU_TOKEN_PATTERN)
|
||||
if (!match) {
|
||||
return null
|
||||
}
|
||||
|
||||
return expandComboSpuToken(match[0])
|
||||
}
|
||||
|
||||
function isSpuName(name: string) {
|
||||
@ -688,18 +758,61 @@ async function ensureCategoryAndSceneNames(categoryName: string, sceneName: stri
|
||||
}
|
||||
}
|
||||
|
||||
async function hasImportedFingerprint(fingerprint: string) {
|
||||
const existing = await prisma.materialLog.findFirst({
|
||||
async function hasImportedHistoryRecord(record: HistoryImportRecord) {
|
||||
const existingByFingerprint = await prisma.materialLog.findFirst({
|
||||
where: {
|
||||
action: 'HISTORY_IMPORT',
|
||||
material: {
|
||||
deletedAt: null,
|
||||
},
|
||||
changes: {
|
||||
contains: fingerprint,
|
||||
contains: record.fingerprint,
|
||||
} as any,
|
||||
},
|
||||
select: { id: true },
|
||||
})
|
||||
|
||||
return Boolean(existing)
|
||||
if (existingByFingerprint) {
|
||||
return true
|
||||
}
|
||||
|
||||
const existingBySourcePath = await prisma.$queryRaw<Array<{ id: string }>>`
|
||||
SELECT logs.id
|
||||
FROM "material_logs" AS logs
|
||||
INNER JOIN "materials" AS materials
|
||||
ON materials.id = logs."materialId"
|
||||
WHERE logs.action = 'HISTORY_IMPORT'
|
||||
AND materials."deletedAt" IS NULL
|
||||
AND logs.changes IS NOT NULL
|
||||
AND logs.changes::jsonb->>'sourcePath' = ${record.sourcePath}
|
||||
LIMIT 1
|
||||
`
|
||||
|
||||
return existingBySourcePath.length > 0
|
||||
}
|
||||
|
||||
function getBlockedReadySourceReason(record: HistoryImportRecord) {
|
||||
const haystack = [
|
||||
record.sourcePath,
|
||||
record.relativeSourcePath,
|
||||
record.skipReason || '',
|
||||
].join('\n')
|
||||
const matched = BLOCKED_READY_SOURCE_PATH_PATTERNS.find(({ pattern }) => pattern.test(haystack))
|
||||
|
||||
if (!matched) {
|
||||
return null
|
||||
}
|
||||
|
||||
return `来源路径包含“${matched.label}”,即使台账标记 ready 也禁止正式导入`
|
||||
}
|
||||
|
||||
function getSpuMismatchReason(record: HistoryImportRecord) {
|
||||
const inferredComboSpu = inferComboSpuFromPath(record.sourcePath)
|
||||
if (!inferredComboSpu || normalizeSpu(record.spu) === inferredComboSpu) {
|
||||
return null
|
||||
}
|
||||
|
||||
return `来源路径包含组合 SPU“${inferredComboSpu}”,但台账 SPU 是“${record.spu}”,禁止按单 SPU 导入`
|
||||
}
|
||||
|
||||
async function createHistoryImportMaterial(record: HistoryImportRecord, operatorUserId: string) {
|
||||
@ -717,12 +830,103 @@ async function createHistoryImportMaterial(record: HistoryImportRecord, operator
|
||||
renderer: '',
|
||||
isFullSet: false,
|
||||
isAiGenerated: false,
|
||||
taskDetailSourceType: normalizeTaskDetailSourceType(record.taskDetailSourceType) || '渲染',
|
||||
} as any,
|
||||
})
|
||||
}
|
||||
|
||||
function filterNonEmptyAssets<T extends { fileName: string; fileSize: number; localFilePath?: string }>(
|
||||
group: 'main' | 'sku' | 'detail' | 'psd',
|
||||
assets: T[] | undefined,
|
||||
skippedEmptyFiles: SanitizedHistoryImportRecord['skippedEmptyFiles'],
|
||||
) {
|
||||
return (assets || []).filter((asset) => {
|
||||
if (asset.fileSize > 0) {
|
||||
return true
|
||||
}
|
||||
|
||||
skippedEmptyFiles.push({
|
||||
group,
|
||||
fileName: asset.fileName,
|
||||
localFilePath: asset.localFilePath,
|
||||
})
|
||||
return false
|
||||
})
|
||||
}
|
||||
|
||||
async function filterReadableImageAssets<T extends { fileName: string; fileSize: number; localFilePath?: string }>(
|
||||
group: 'main' | 'sku' | 'detail',
|
||||
assets: T[],
|
||||
skippedInvalidFiles: SanitizedHistoryImportRecord['skippedInvalidFiles'],
|
||||
) {
|
||||
if (assets.length === 0) {
|
||||
return assets
|
||||
}
|
||||
|
||||
const sharp = await import('sharp')
|
||||
const readableAssets: T[] = []
|
||||
|
||||
for (const asset of assets) {
|
||||
if (!asset.localFilePath) {
|
||||
readableAssets.push(asset)
|
||||
continue
|
||||
}
|
||||
|
||||
try {
|
||||
await sharp
|
||||
.default(asset.localFilePath)
|
||||
.resize(1, 1, {
|
||||
fit: 'inside',
|
||||
withoutEnlargement: true,
|
||||
})
|
||||
.webp({ quality: 1 })
|
||||
.toBuffer()
|
||||
readableAssets.push(asset)
|
||||
} catch (error) {
|
||||
skippedInvalidFiles.push({
|
||||
group,
|
||||
fileName: asset.fileName,
|
||||
fileSize: asset.fileSize,
|
||||
localFilePath: asset.localFilePath,
|
||||
reason: error instanceof Error ? error.message : '图片无法读取',
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
return readableAssets
|
||||
}
|
||||
|
||||
async function sanitizeHistoryImportRecord(record: HistoryImportRecord): Promise<SanitizedHistoryImportRecord> {
|
||||
const skippedEmptyFiles: SanitizedHistoryImportRecord['skippedEmptyFiles'] = []
|
||||
const skippedInvalidFiles: SanitizedHistoryImportRecord['skippedInvalidFiles'] = []
|
||||
const nonEmptyMainImages = filterNonEmptyAssets('main', record.mainImages, skippedEmptyFiles)
|
||||
const nonEmptySkuImages = filterNonEmptyAssets('sku', record.skuImages, skippedEmptyFiles)
|
||||
const nonEmptyDetailImages = filterNonEmptyAssets('detail', record.detailImages, skippedEmptyFiles)
|
||||
const sanitizedRecord: HistoryImportRecord = {
|
||||
...record,
|
||||
mainImages: await filterReadableImageAssets('main', nonEmptyMainImages, skippedInvalidFiles),
|
||||
skuImages: await filterReadableImageAssets('sku', nonEmptySkuImages, skippedInvalidFiles),
|
||||
detailImages: await filterReadableImageAssets('detail', nonEmptyDetailImages, skippedInvalidFiles),
|
||||
psdSources: filterNonEmptyAssets('psd', record.psdSources, skippedEmptyFiles),
|
||||
}
|
||||
|
||||
if (
|
||||
sanitizedRecord.mainImages.length === 0 ||
|
||||
sanitizedRecord.skuImages.length === 0 ||
|
||||
sanitizedRecord.detailImages.length === 0
|
||||
) {
|
||||
throw new Error(
|
||||
`过滤 0 字节文件后不满足导入条件:主图=${sanitizedRecord.mainImages.length} SKU=${sanitizedRecord.skuImages.length} 详情=${sanitizedRecord.detailImages.length}`,
|
||||
)
|
||||
}
|
||||
|
||||
return { record: sanitizedRecord, skippedEmptyFiles, skippedInvalidFiles }
|
||||
}
|
||||
|
||||
export async function importHistoryMaterials(options: ImportHistoryMaterialsOptions) {
|
||||
await ensureCategoryAndScene('其他', '其他')
|
||||
if (options.apply) {
|
||||
await ensureCategoryAndScene('其他', '其他')
|
||||
}
|
||||
|
||||
const candidates = options.records
|
||||
.filter((record) => {
|
||||
@ -735,7 +939,35 @@ export async function importHistoryMaterials(options: ImportHistoryMaterialsOpti
|
||||
|
||||
const results: ImportHistoryMaterialsResultItem[] = []
|
||||
for (const record of candidates) {
|
||||
if (await hasImportedFingerprint(record.fingerprint)) {
|
||||
const blockedReadySourceReason = getBlockedReadySourceReason(record)
|
||||
if (blockedReadySourceReason) {
|
||||
results.push({
|
||||
fingerprint: record.fingerprint,
|
||||
spu: record.spu,
|
||||
sourcePath: record.sourcePath,
|
||||
guessedStore: record.guessedStore,
|
||||
materialId: null,
|
||||
status: 'skipped',
|
||||
reason: blockedReadySourceReason,
|
||||
})
|
||||
continue
|
||||
}
|
||||
|
||||
const spuMismatchReason = getSpuMismatchReason(record)
|
||||
if (spuMismatchReason) {
|
||||
results.push({
|
||||
fingerprint: record.fingerprint,
|
||||
spu: record.spu,
|
||||
sourcePath: record.sourcePath,
|
||||
guessedStore: record.guessedStore,
|
||||
materialId: null,
|
||||
status: 'skipped',
|
||||
reason: spuMismatchReason,
|
||||
})
|
||||
continue
|
||||
}
|
||||
|
||||
if (await hasImportedHistoryRecord(record)) {
|
||||
results.push({
|
||||
fingerprint: record.fingerprint,
|
||||
spu: record.spu,
|
||||
@ -762,16 +994,18 @@ export async function importHistoryMaterials(options: ImportHistoryMaterialsOpti
|
||||
}
|
||||
|
||||
try {
|
||||
const sanitized = await sanitizeHistoryImportRecord(record)
|
||||
await ensureCategoryAndSceneNames(record.guessedStore || '其他', '其他')
|
||||
const material = await createHistoryImportMaterial(record, options.operator.id)
|
||||
const material = await createHistoryImportMaterial(sanitized.record, options.operator.id)
|
||||
await completeProductPackageUpload({
|
||||
materialId: material.id,
|
||||
taskNo: null,
|
||||
operatorUserId: options.operator.id,
|
||||
assets: {
|
||||
main: record.mainImages,
|
||||
sku: record.skuImages,
|
||||
detail: record.detailImages,
|
||||
main: sanitized.record.mainImages,
|
||||
sku: sanitized.record.skuImages,
|
||||
detail: sanitized.record.detailImages,
|
||||
psd: sanitized.record.psdSources || [],
|
||||
},
|
||||
options: {
|
||||
allowSkuWithoutBarcodes: true,
|
||||
@ -789,7 +1023,11 @@ export async function importHistoryMaterials(options: ImportHistoryMaterialsOpti
|
||||
sourcePath: record.sourcePath,
|
||||
guessedStore: record.guessedStore,
|
||||
storeCandidates: record.storeCandidates,
|
||||
sourceCount: record.sourceCount || record.psdSources?.length || 0,
|
||||
taskDetailSourceType: normalizeTaskDetailSourceType(record.taskDetailSourceType) || '渲染',
|
||||
seenInYya: record.seenInYya,
|
||||
skippedEmptyFiles: sanitized.skippedEmptyFiles,
|
||||
skippedInvalidFiles: sanitized.skippedInvalidFiles,
|
||||
},
|
||||
})
|
||||
|
||||
|
||||
@ -3,7 +3,7 @@ import { promises as fs } from 'fs'
|
||||
import crypto from 'crypto'
|
||||
import type { ProductPackageImageInput } from '@/lib/product-package-completion'
|
||||
import type { HistoryImportRecord, HistoryImportStatus } from '@/lib/history-material-import'
|
||||
import { parseSPU } from '@/lib/spu-parser'
|
||||
import { getShopPrefixMap, parseSPU } from '@/lib/spu-parser'
|
||||
|
||||
const IMAGE_EXTENSIONS = new Set(['.jpg', '.jpeg', '.png', '.webp'])
|
||||
const SKIPPED_FILE_EXTENSIONS = new Set([
|
||||
@ -50,9 +50,19 @@ const EXCLUDED_FILE_NAME_PATTERNS = [
|
||||
/mask/i,
|
||||
]
|
||||
|
||||
const BRANDED_SPU_TOKEN_PATTERN = /([A-Za-z]{2,5}-[A-Za-z]{1,5}\d{3,5})/g
|
||||
const PRIMARY_SPU_TOKEN_PATTERN = /([A-Za-z]{2,3}\d{4})/g
|
||||
const FALLBACK_SPU_TOKEN_PATTERN = /([A-Za-z]{1,5}\d{3,5})/g
|
||||
const IGNORED_SPU_PREFIXES = ['TASK', 'SKU', 'IMG']
|
||||
const EXTRA_SHOP_PREFIX_MAP: Record<string, string> = {
|
||||
GBJP: '顾家',
|
||||
}
|
||||
const EXTRA_CATEGORY_CODE_MAP: Record<string, string> = {
|
||||
BI: '边几',
|
||||
CFZWJ: '置物架',
|
||||
SMC: '床',
|
||||
}
|
||||
const CATEGORY_CODE_PATTERN = /^([A-Z]{1,5})\d/
|
||||
const AUTHOR_SEGMENT_EXCLUDES = new Set([
|
||||
'亿家木',
|
||||
'亿家达',
|
||||
@ -216,7 +226,35 @@ function inferSpuFromText(value: string) {
|
||||
const isStandaloneToken = (matchIndex: number, token: string) => {
|
||||
const prevChar = matchIndex > 0 ? text[matchIndex - 1] : ''
|
||||
const nextChar = text[matchIndex + token.length] || ''
|
||||
return !/[A-Za-z]/.test(prevChar) && !/[A-Za-z0-9]/.test(nextChar)
|
||||
if (/[A-Za-z0-9]/.test(prevChar)) {
|
||||
return false
|
||||
}
|
||||
if (!/[A-Za-z0-9]/.test(nextChar)) {
|
||||
return true
|
||||
}
|
||||
return /^K\d{1,2}(?![A-Za-z0-9])/i.test(text.slice(matchIndex + token.length))
|
||||
}
|
||||
|
||||
BRANDED_SPU_TOKEN_PATTERN.lastIndex = 0
|
||||
const brandedMatches = Array.from(text.matchAll(BRANDED_SPU_TOKEN_PATTERN))
|
||||
if (brandedMatches.length > 0) {
|
||||
const shopPrefixMap = { ...getShopPrefixMap(), ...EXTRA_SHOP_PREFIX_MAP }
|
||||
for (let index = brandedMatches.length - 1; index >= 0; index -= 1) {
|
||||
const currentMatch = brandedMatches[index]
|
||||
const rawToken = currentMatch?.[1] || ''
|
||||
const token = normalizeSpu(rawToken)
|
||||
const [shopPrefix, skuPart] = token.split('-')
|
||||
const matchIndex = currentMatch?.index ?? -1
|
||||
if (
|
||||
token &&
|
||||
shopPrefixMap[shopPrefix] &&
|
||||
skuPart &&
|
||||
matchIndex >= 0 &&
|
||||
isStandaloneToken(matchIndex, rawToken)
|
||||
) {
|
||||
return token
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
PRIMARY_SPU_TOKEN_PATTERN.lastIndex = 0
|
||||
@ -291,12 +329,13 @@ function inferSpu(row: ParsedSkuCsvRow) {
|
||||
|
||||
if (row.spu) {
|
||||
const normalizedRowSpu = normalizeSpu(row.spu)
|
||||
const rowSpuLooksReliable =
|
||||
const rowSpuLooksUsable =
|
||||
normalizedRowSpu &&
|
||||
!IGNORED_SPU_PREFIXES.some((prefix) => normalizedRowSpu.startsWith(prefix)) &&
|
||||
candidates.some((candidate) => hasStandaloneSpuToken(candidate, normalizedRowSpu))
|
||||
(/^[A-Z]{2,5}-[A-Z]{1,5}\d{3,5}$/.test(normalizedRowSpu) ||
|
||||
/^[A-Z]{1,5}\d{3,5}$/.test(normalizedRowSpu))
|
||||
|
||||
if (rowSpuLooksReliable) {
|
||||
if (rowSpuLooksUsable) {
|
||||
return normalizedRowSpu
|
||||
}
|
||||
}
|
||||
@ -482,7 +521,25 @@ function buildAssetInputs(spu: string, files: FileEntry[], group: 'main' | 'sku'
|
||||
}
|
||||
|
||||
function inferStoreBySpu(spu: string, row?: ParsedSkuCsvRow) {
|
||||
const sourceRootStore = inferStoreBySourceRoot(row)
|
||||
if (sourceRootStore) {
|
||||
return {
|
||||
guessedStore: sourceRootStore,
|
||||
storeConfidence: 1,
|
||||
storeCandidates: [sourceRootStore],
|
||||
}
|
||||
}
|
||||
|
||||
const normalizedSpu = normalizeSpu(spu)
|
||||
const spuParseResult = parseSPU(normalizedSpu)
|
||||
if (spuParseResult.shopMatched && spuParseResult.shopName) {
|
||||
return {
|
||||
guessedStore: spuParseResult.shopName,
|
||||
storeConfidence: 1,
|
||||
storeCandidates: [spuParseResult.shopName],
|
||||
}
|
||||
}
|
||||
|
||||
const storeProbeText = [
|
||||
normalizedSpu,
|
||||
row?.relativePath || '',
|
||||
@ -493,36 +550,17 @@ function inferStoreBySpu(spu: string, row?: ParsedSkuCsvRow) {
|
||||
]
|
||||
.join('\n')
|
||||
.toUpperCase()
|
||||
const shopPrefixMap = { ...getShopPrefixMap(), ...EXTRA_SHOP_PREFIX_MAP }
|
||||
const matchedPrefix = Object.keys(shopPrefixMap)
|
||||
.sort((left, right) => right.length - left.length)
|
||||
.find((prefix) => new RegExp(`(^|[^A-Z0-9])${prefix}-`).test(storeProbeText))
|
||||
|
||||
if (storeProbeText.includes('GBGJ')) {
|
||||
if (matchedPrefix) {
|
||||
const storeName = shopPrefixMap[matchedPrefix]
|
||||
return {
|
||||
guessedStore: '顾家',
|
||||
guessedStore: storeName,
|
||||
storeConfidence: 1,
|
||||
storeCandidates: ['顾家'],
|
||||
}
|
||||
}
|
||||
|
||||
if (storeProbeText.includes('GBSFY')) {
|
||||
return {
|
||||
guessedStore: '索菲亚',
|
||||
storeConfidence: 1,
|
||||
storeCandidates: ['索菲亚'],
|
||||
}
|
||||
}
|
||||
|
||||
if (normalizedSpu.startsWith('GBGJ')) {
|
||||
return {
|
||||
guessedStore: '顾家',
|
||||
storeConfidence: 1,
|
||||
storeCandidates: ['顾家'],
|
||||
}
|
||||
}
|
||||
|
||||
if (normalizedSpu.startsWith('GBSFY')) {
|
||||
return {
|
||||
guessedStore: '索菲亚',
|
||||
storeConfidence: 1,
|
||||
storeCandidates: ['索菲亚'],
|
||||
storeCandidates: [storeName],
|
||||
}
|
||||
}
|
||||
|
||||
@ -533,6 +571,36 @@ function inferStoreBySpu(spu: string, row?: ParsedSkuCsvRow) {
|
||||
}
|
||||
}
|
||||
|
||||
function inferStoreBySourceRoot(row?: ParsedSkuCsvRow) {
|
||||
const sourceRoot = row?.sourceRoot || ''
|
||||
const probes = [
|
||||
['顾家项目', '顾家'],
|
||||
['金牌项目', '顾家'],
|
||||
['朴美项目', '朴美'],
|
||||
['索菲亚项目', '索菲亚'],
|
||||
] as const
|
||||
|
||||
const matched = probes.find(([keyword]) => sourceRoot.includes(keyword))
|
||||
return matched?.[1] || ''
|
||||
}
|
||||
|
||||
function inferProductCategoryBySpu(spu: string) {
|
||||
const spuParseResult = parseSPU(spu)
|
||||
if (spuParseResult.categoryMatched) {
|
||||
return spuParseResult.categoryName
|
||||
}
|
||||
|
||||
const normalizedSpu = normalizeSpu(spu)
|
||||
const skuPart = normalizedSpu.includes('-') ? normalizedSpu.split('-').pop() || '' : normalizedSpu
|
||||
const skuPartParseResult = parseSPU(skuPart)
|
||||
if (skuPartParseResult.categoryMatched) {
|
||||
return skuPartParseResult.categoryName
|
||||
}
|
||||
|
||||
const categoryCode = skuPart.match(CATEGORY_CODE_PATTERN)?.[1] || ''
|
||||
return EXTRA_CATEGORY_CODE_MAP[categoryCode] || ''
|
||||
}
|
||||
|
||||
function normalizeAuthorToken(value: string) {
|
||||
return value.replace(/[\s()()_\-]+/g, '').trim().toLowerCase()
|
||||
}
|
||||
@ -608,6 +676,23 @@ function makeFingerprint(spu: string, sourcePath: string) {
|
||||
return crypto.createHash('sha1').update(`${normalizeSpu(spu)}|${sourcePath}`).digest('hex')
|
||||
}
|
||||
|
||||
function normalizeDuplicatePathKey(relativeSourcePath: string) {
|
||||
const segments = relativeSourcePath
|
||||
.split(/[\\/]+/)
|
||||
.map((item) => item.trim())
|
||||
.filter(Boolean)
|
||||
|
||||
if (segments.length > 1 && segments[0] === segments[1]) {
|
||||
segments.shift()
|
||||
}
|
||||
|
||||
return segments.join('/').toLowerCase()
|
||||
}
|
||||
|
||||
function makeDuplicateKey(spu: string, relativeSourcePath: string) {
|
||||
return `${normalizeSpu(spu)}|${normalizeDuplicatePathKey(relativeSourcePath)}`
|
||||
}
|
||||
|
||||
export async function buildHistoryImportRecordsFromSkuCsv(
|
||||
options: BuildHistoryImportRecordsFromSkuCsvOptions,
|
||||
) {
|
||||
@ -624,9 +709,16 @@ export async function buildHistoryImportRecordsFromSkuCsv(
|
||||
new Set((options.visualCenterAuthors || []).map((item) => item.trim()).filter(Boolean)),
|
||||
)
|
||||
const records: HistoryImportRecord[] = []
|
||||
const seenDuplicateKeys = new Set<string>()
|
||||
|
||||
for (const row of limitedRows) {
|
||||
const spu = inferSpu(row)
|
||||
const duplicateKey = makeDuplicateKey(spu || row.directoryName || 'UNKNOWN', row.relativePath)
|
||||
if (seenDuplicateKeys.has(duplicateKey)) {
|
||||
continue
|
||||
}
|
||||
seenDuplicateKeys.add(duplicateKey)
|
||||
|
||||
const detailDirName = pickPreferredDirName(row.siblingDetailDirNames, 'detail')
|
||||
const mainDirName = pickPreferredDirName(row.siblingMainDirNames, 'main')
|
||||
|
||||
@ -636,8 +728,7 @@ export async function buildHistoryImportRecordsFromSkuCsv(
|
||||
|
||||
const storeGuess = inferStoreBySpu(spu, row)
|
||||
const hasFullSet = mainFiles.length > 0 && skuFiles.length > 0 && detailFiles.length > 0
|
||||
const spuParseResult = parseSPU(spu)
|
||||
const productCategoryName = spuParseResult.categoryMatched ? spuParseResult.categoryName : ''
|
||||
const productCategoryName = inferProductCategoryBySpu(spu)
|
||||
const authorName = inferAuthorFromRow(row, visualCenterAuthors)
|
||||
|
||||
let status: HistoryImportStatus = 'ready'
|
||||
|
||||
@ -7,6 +7,8 @@ import {
|
||||
createZipFromZips,
|
||||
saveProductPackageAssets,
|
||||
} from '@/lib/task-material-processor'
|
||||
import { buildStoredUploadPath, getUploadRootDir } from '@/lib/upload-paths'
|
||||
import { isRemoteUploadSyncEnabled, syncStoredUploadFileToRemote } from '@/lib/remote-upload-sync'
|
||||
import { getUploadedFile } from '@/lib/upload-sessions'
|
||||
import { createMaterialLog } from '@/lib/material-log'
|
||||
import { isTaskDetailFullSetFromUploadGroups } from '@/lib/task-detail-full-set'
|
||||
@ -289,8 +291,8 @@ async function movePsdFilesToPermanentDir(
|
||||
return psdPermanentPaths
|
||||
}
|
||||
|
||||
const { rename, copyFile } = await import('fs/promises')
|
||||
const psdDir = path.join(process.cwd(), 'uploads', 'materials')
|
||||
const { rename, copyFile, stat } = await import('fs/promises')
|
||||
const psdDir = path.join(getUploadRootDir(), 'materials')
|
||||
await mkdir(psdDir, { recursive: true })
|
||||
|
||||
for (const psd of assets) {
|
||||
@ -299,15 +301,36 @@ async function movePsdFilesToPermanentDir(
|
||||
throw new Error(`PSD 资产参数无效: ${psd.fileName}`)
|
||||
}
|
||||
|
||||
const tempFilePath = await downloadToTempFile('psd', psd)
|
||||
const timestamp = Date.now()
|
||||
const sanitizedFileName = psd.fileName.replace(/[\/\\:*?"<>|]/g, '_')
|
||||
const permanentFileName = `psd-${materialId}-${timestamp}-${sanitizedFileName}`
|
||||
const permanentFilePath = path.join(psdDir, permanentFileName)
|
||||
const relativePath = buildStoredUploadPath('materials', permanentFileName)
|
||||
|
||||
if (psd.localFilePath) {
|
||||
const sourceStat = await stat(psd.localFilePath)
|
||||
const expectedSize = psd.fileSize || sourceStat.size
|
||||
|
||||
if (isRemoteUploadSyncEnabled()) {
|
||||
await syncStoredUploadFileToRemote(psd.localFilePath, relativePath, expectedSize)
|
||||
} else {
|
||||
await copyFile(psd.localFilePath, permanentFilePath)
|
||||
}
|
||||
|
||||
psdPermanentPaths.push({
|
||||
filePath: relativePath,
|
||||
fileName: psd.fileName,
|
||||
fileSize: expectedSize,
|
||||
})
|
||||
continue
|
||||
}
|
||||
|
||||
const tempFilePath = await downloadToTempFile('psd', psd)
|
||||
await copyFile(tempFilePath, permanentFilePath)
|
||||
await syncStoredUploadFileToRemote(permanentFilePath, relativePath, psd.fileSize)
|
||||
tempFiles.push(tempFilePath)
|
||||
psdPermanentPaths.push({
|
||||
filePath: path.join('uploads', 'materials', permanentFileName),
|
||||
filePath: relativePath,
|
||||
fileName: psd.fileName,
|
||||
fileSize: psd.fileSize,
|
||||
})
|
||||
@ -323,7 +346,7 @@ async function movePsdFilesToPermanentDir(
|
||||
const sanitizedFileName = psd.fileName.replace(/[\/\\:*?"<>|]/g, '_')
|
||||
const permanentFileName = `psd-${materialId}-${timestamp}-${sanitizedFileName}`
|
||||
const permanentFilePath = path.join(psdDir, permanentFileName)
|
||||
const relativePath = path.join('uploads', 'materials', permanentFileName)
|
||||
const relativePath = buildStoredUploadPath('materials', permanentFileName)
|
||||
|
||||
try {
|
||||
await rename(tempFilePath, permanentFilePath)
|
||||
@ -332,6 +355,8 @@ async function movePsdFilesToPermanentDir(
|
||||
tempFiles.push(tempFilePath)
|
||||
}
|
||||
|
||||
await syncStoredUploadFileToRemote(permanentFilePath, relativePath, psd.fileSize)
|
||||
|
||||
psdPermanentPaths.push({
|
||||
filePath: relativePath,
|
||||
fileName: psd.fileName,
|
||||
@ -468,6 +493,7 @@ export async function completeProductPackageUpload(input: CompleteProductPackage
|
||||
createdZipPaths.push(zipInfos.all.filePath)
|
||||
|
||||
const psdPermanentPaths = await movePsdFilesToPermanentDir(materialId, normalizedAssets.psd, tempFiles)
|
||||
createdZipPaths.push(...psdPermanentPaths.map((item) => item.filePath))
|
||||
|
||||
const saveResult = await saveProductPackageAssets(materialId, previewAssets, zipInfos, psdPermanentPaths)
|
||||
|
||||
|
||||
214
lib/remote-upload-sync.ts
Normal file
214
lib/remote-upload-sync.ts
Normal file
@ -0,0 +1,214 @@
|
||||
import path from 'path'
|
||||
import { execFile } from 'child_process'
|
||||
import { stat } from 'fs/promises'
|
||||
import { promisify } from 'util'
|
||||
|
||||
const execFileAsync = promisify(execFile)
|
||||
const REMOTE_SYNC_SSH_TIMEOUT_MS = Number(process.env.REMOTE_SYNC_SSH_TIMEOUT_MS || 30000)
|
||||
const REMOTE_SYNC_SCP_TIMEOUT_MS = Number(process.env.REMOTE_SYNC_SCP_TIMEOUT_MS || 7200000)
|
||||
const REMOTE_SYNC_RETRY_COUNT = Number(process.env.REMOTE_SYNC_RETRY_COUNT || 3)
|
||||
const SSH_NON_INTERACTIVE_ARGS = [
|
||||
'-o',
|
||||
'BatchMode=yes',
|
||||
'-o',
|
||||
'ConnectTimeout=15',
|
||||
'-o',
|
||||
'ServerAliveInterval=15',
|
||||
'-o',
|
||||
'ServerAliveCountMax=2',
|
||||
]
|
||||
|
||||
interface RemoteUploadSyncConfig {
|
||||
target: string
|
||||
uploadRoot: string
|
||||
}
|
||||
|
||||
function cleanEnv(value: string | undefined) {
|
||||
return String(value || '').trim()
|
||||
}
|
||||
|
||||
function shellQuote(value: string) {
|
||||
return `'${value.replace(/'/g, `'\\''`)}'`
|
||||
}
|
||||
|
||||
function normalizeRemotePath(value: string) {
|
||||
return value.replace(/\\/g, '/').replace(/\/+$/, '')
|
||||
}
|
||||
|
||||
function sleep(ms: number) {
|
||||
return new Promise((resolve) => setTimeout(resolve, ms))
|
||||
}
|
||||
|
||||
async function execFileWithRetry(
|
||||
file: string,
|
||||
args: string[],
|
||||
options: { timeout: number },
|
||||
label: string,
|
||||
) {
|
||||
let lastError: unknown
|
||||
for (let attempt = 1; attempt <= REMOTE_SYNC_RETRY_COUNT; attempt += 1) {
|
||||
try {
|
||||
return await execFileAsync(file, args, options)
|
||||
} catch (error) {
|
||||
lastError = error
|
||||
const message = error instanceof Error ? error.message : String(error)
|
||||
console.warn(
|
||||
`[RemoteUploadSync] ${label} 失败 ${attempt}/${REMOTE_SYNC_RETRY_COUNT}: ${message}`,
|
||||
)
|
||||
if (attempt < REMOTE_SYNC_RETRY_COUNT) {
|
||||
await sleep(1000 * attempt)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
throw lastError
|
||||
}
|
||||
|
||||
function relativeToUploadRoot(storedPath: string) {
|
||||
const normalized = String(storedPath || '').replace(/\\/g, '/')
|
||||
const relative = normalized.replace(/^\.?\/*uploads\//i, '')
|
||||
|
||||
if (!relative || relative === normalized && path.posix.isAbsolute(normalized)) {
|
||||
throw new Error(`远程同步路径无效: ${storedPath}`)
|
||||
}
|
||||
|
||||
if (relative.split('/').some((segment) => segment === '..')) {
|
||||
throw new Error(`远程同步路径不能包含上级目录: ${storedPath}`)
|
||||
}
|
||||
|
||||
return relative
|
||||
}
|
||||
|
||||
export function getRemoteUploadSyncConfig(): RemoteUploadSyncConfig | null {
|
||||
const target = cleanEnv(
|
||||
process.env.HISTORY_IMPORT_REMOTE_UPLOADS_HOST || process.env.UPLOAD_REMOTE_SYNC_HOST,
|
||||
)
|
||||
const uploadRoot = cleanEnv(
|
||||
process.env.HISTORY_IMPORT_REMOTE_UPLOADS_DIR || process.env.UPLOAD_REMOTE_SYNC_ROOT,
|
||||
)
|
||||
|
||||
if (!target && !uploadRoot) {
|
||||
return null
|
||||
}
|
||||
|
||||
if (!target || !uploadRoot) {
|
||||
throw new Error(
|
||||
'远程 uploads 同步配置不完整:需要同时配置 HISTORY_IMPORT_REMOTE_UPLOADS_HOST 和 HISTORY_IMPORT_REMOTE_UPLOADS_DIR',
|
||||
)
|
||||
}
|
||||
|
||||
return {
|
||||
target,
|
||||
uploadRoot: normalizeRemotePath(uploadRoot),
|
||||
}
|
||||
}
|
||||
|
||||
export function isRemoteUploadSyncEnabled() {
|
||||
return Boolean(getRemoteUploadSyncConfig())
|
||||
}
|
||||
|
||||
export function getRemoteUploadPath(storedPath: string) {
|
||||
const config = getRemoteUploadSyncConfig()
|
||||
if (!config) {
|
||||
return null
|
||||
}
|
||||
|
||||
return path.posix.join(config.uploadRoot, relativeToUploadRoot(storedPath))
|
||||
}
|
||||
|
||||
export async function assertRemoteUploadSyncAccess() {
|
||||
const config = getRemoteUploadSyncConfig()
|
||||
if (!config) {
|
||||
return
|
||||
}
|
||||
|
||||
const materialsDir = path.posix.join(config.uploadRoot, 'materials')
|
||||
await execFileWithRetry(
|
||||
'ssh',
|
||||
[
|
||||
...SSH_NON_INTERACTIVE_ARGS,
|
||||
config.target,
|
||||
`test -d ${shellQuote(materialsDir)} && test -w ${shellQuote(materialsDir)}`,
|
||||
],
|
||||
{ timeout: REMOTE_SYNC_SSH_TIMEOUT_MS },
|
||||
'检查远程 uploads 权限',
|
||||
)
|
||||
}
|
||||
|
||||
export async function syncStoredUploadFileToRemote(
|
||||
localFilePath: string,
|
||||
storedPath: string,
|
||||
expectedSize?: number,
|
||||
) {
|
||||
const config = getRemoteUploadSyncConfig()
|
||||
if (!config) {
|
||||
return null
|
||||
}
|
||||
|
||||
const remotePath = getRemoteUploadPath(storedPath)
|
||||
if (!remotePath) {
|
||||
return null
|
||||
}
|
||||
|
||||
const localStat = await stat(localFilePath)
|
||||
const expectedFileSize = expectedSize || localStat.size
|
||||
const remoteDir = path.posix.dirname(remotePath)
|
||||
const sizeMiB = (expectedFileSize / 1024 / 1024).toFixed(2)
|
||||
|
||||
console.log(
|
||||
`[RemoteUploadSync] 准备同步 ${storedPath} -> ${config.target}:${remotePath} (${sizeMiB} MiB)`,
|
||||
)
|
||||
await execFileWithRetry(
|
||||
'ssh',
|
||||
[...SSH_NON_INTERACTIVE_ARGS, config.target, `mkdir -p ${shellQuote(remoteDir)}`],
|
||||
{ timeout: REMOTE_SYNC_SSH_TIMEOUT_MS },
|
||||
`创建远程目录 ${remoteDir}`,
|
||||
)
|
||||
console.log(`[RemoteUploadSync] 开始复制 ${storedPath} (${sizeMiB} MiB)`)
|
||||
await execFileWithRetry(
|
||||
'scp',
|
||||
['-q', ...SSH_NON_INTERACTIVE_ARGS, localFilePath, `${config.target}:${remotePath}`],
|
||||
{ timeout: REMOTE_SYNC_SCP_TIMEOUT_MS },
|
||||
`复制 ${storedPath}`,
|
||||
)
|
||||
|
||||
const { stdout } = await execFileWithRetry(
|
||||
'ssh',
|
||||
[
|
||||
...SSH_NON_INTERACTIVE_ARGS,
|
||||
config.target,
|
||||
`test -f ${shellQuote(remotePath)} && stat -c %s ${shellQuote(remotePath)}`,
|
||||
],
|
||||
{ timeout: REMOTE_SYNC_SSH_TIMEOUT_MS },
|
||||
`校验远程文件 ${remotePath}`,
|
||||
)
|
||||
const remoteSize = Number(String(stdout || '').trim())
|
||||
|
||||
if (remoteSize !== expectedFileSize) {
|
||||
throw new Error(
|
||||
`远程 uploads 同步校验失败: ${storedPath} 本地=${expectedFileSize} 远程=${remoteSize || 'unknown'}`,
|
||||
)
|
||||
}
|
||||
|
||||
console.log(`[RemoteUploadSync] ${storedPath} -> ${config.target}:${remotePath} (${remoteSize} bytes)`)
|
||||
return remotePath
|
||||
}
|
||||
|
||||
export async function deleteStoredUploadFileFromRemote(storedPath: string) {
|
||||
const config = getRemoteUploadSyncConfig()
|
||||
if (!config) {
|
||||
return
|
||||
}
|
||||
|
||||
const remotePath = getRemoteUploadPath(storedPath)
|
||||
if (!remotePath) {
|
||||
return
|
||||
}
|
||||
|
||||
await execFileWithRetry(
|
||||
'ssh',
|
||||
[...SSH_NON_INTERACTIVE_ARGS, config.target, `rm -f -- ${shellQuote(remotePath)}`],
|
||||
{ timeout: REMOTE_SYNC_SSH_TIMEOUT_MS },
|
||||
`删除远程文件 ${remotePath}`,
|
||||
)
|
||||
}
|
||||
@ -32,6 +32,7 @@ const SHOP_PREFIX_MAP: Record<string, string> = {
|
||||
'GBPM': '朴美',
|
||||
'GBSFY': '索菲亚',
|
||||
'GBGJ': '顾家',
|
||||
'GBJP': '顾家',
|
||||
'XYL': '橡野里',
|
||||
}
|
||||
|
||||
|
||||
@ -10,11 +10,15 @@ import {
|
||||
uploadPublicDisplayToS3,
|
||||
} from '@/lib/public-display';
|
||||
import { compressImage } from '@/lib/image-processor';
|
||||
import { buildStoredUploadPath, getUploadRootDir, resolveStoredUploadPath } from '@/lib/upload-paths';
|
||||
import {
|
||||
deleteStoredUploadFileFromRemote,
|
||||
syncStoredUploadFileToRemote,
|
||||
} from '@/lib/remote-upload-sync';
|
||||
import { mkdir, unlink, readFile, rm } from 'fs/promises';
|
||||
import { createWriteStream } from 'fs';
|
||||
import path from 'path';
|
||||
import archiver from 'archiver';
|
||||
import { deleteUploadSession } from '@/lib/upload-sessions-redis';
|
||||
|
||||
// 资产类型常量
|
||||
export const ASSET_TYPE = {
|
||||
@ -92,7 +96,7 @@ export async function createZipFromFiles(
|
||||
}
|
||||
|
||||
// 创建目标目录
|
||||
const uploadsDir = path.join(process.cwd(), 'uploads', 'materials');
|
||||
const uploadsDir = path.join(getUploadRootDir(), 'materials');
|
||||
await mkdir(uploadsDir, { recursive: true });
|
||||
|
||||
// 生成zip文件名和路径(保留中文,只清理文件系统不安全字符)
|
||||
@ -105,14 +109,14 @@ export async function createZipFromFiles(
|
||||
if (authorName && categoryName && zipType !== 'all') {
|
||||
const sanitizedAuthorName = authorName.replace(/[\/\\:*?"<>|]/g, '_');
|
||||
const sanitizedCategoryName = categoryName.replace(/[\/\\:*?"<>|]/g, '_');
|
||||
zipFileName = `${sanitizedProductModel}_${sanitizedAuthorName}_${sanitizedCategoryName}_${zipType}.zip`;
|
||||
zipFileName = `${sanitizedProductModel}_${sanitizedAuthorName}_${sanitizedCategoryName}_${zipType}_${materialId}_${timestamp}.zip`;
|
||||
} else {
|
||||
// 保持原有命名格式(用于all.zip或其他情况)
|
||||
zipFileName = `${sanitizedProductModel}_${zipType}_${materialId}_${timestamp}.zip`;
|
||||
}
|
||||
|
||||
const zipFilePath = path.join(uploadsDir, zipFileName);
|
||||
const relativeZipPath = path.join('uploads', 'materials', zipFileName);
|
||||
const relativeZipPath = buildStoredUploadPath('materials', zipFileName);
|
||||
|
||||
console.log(`[Zip] Creating ${zipType} zip file for ${materialType}:`, zipFilePath);
|
||||
console.log(`[Zip] Files count:`, files.length);
|
||||
@ -153,17 +157,23 @@ export async function createZipFromFiles(
|
||||
});
|
||||
|
||||
// 监听完成事件
|
||||
output.on('close', () => {
|
||||
output.on('close', async () => {
|
||||
clearTimeout(timeout);
|
||||
const fileSize = archive.pointer();
|
||||
console.log(`[Zip] ${zipType} zip file created successfully: ${fileSize} bytes`);
|
||||
|
||||
cleanup();
|
||||
resolve({
|
||||
filePath: relativeZipPath,
|
||||
fileName: zipFileName,
|
||||
fileSize
|
||||
});
|
||||
try {
|
||||
await syncStoredUploadFileToRemote(zipFilePath, relativeZipPath, fileSize);
|
||||
cleanup();
|
||||
resolve({
|
||||
filePath: relativeZipPath,
|
||||
fileName: zipFileName,
|
||||
fileSize
|
||||
});
|
||||
} catch (error) {
|
||||
cleanup();
|
||||
reject(error);
|
||||
}
|
||||
});
|
||||
|
||||
// 监听错误
|
||||
@ -230,7 +240,7 @@ export async function createZipFromZips(
|
||||
const { access } = await import('fs/promises');
|
||||
for (const file of zipFiles) {
|
||||
try {
|
||||
const fullPath = path.isAbsolute(file.path) ? file.path : path.join(process.cwd(), file.path);
|
||||
const fullPath = path.isAbsolute(file.path) ? file.path : resolveStoredUploadPath(file.path);
|
||||
await access(fullPath);
|
||||
} catch (error) {
|
||||
throw new Error(`文件不存在: ${file.path}`);
|
||||
@ -238,7 +248,7 @@ export async function createZipFromZips(
|
||||
}
|
||||
|
||||
// 创建目标目录
|
||||
const uploadsDir = path.join(process.cwd(), 'uploads', 'materials');
|
||||
const uploadsDir = path.join(getUploadRootDir(), 'materials');
|
||||
await mkdir(uploadsDir, { recursive: true });
|
||||
|
||||
// 生成zip文件名和路径
|
||||
@ -246,7 +256,7 @@ export async function createZipFromZips(
|
||||
const sanitizedProductModel = productModel.replace(/[\/\\:*?"<>|]/g, '_');
|
||||
const zipFileName = `${sanitizedProductModel}_all_${materialId}_${timestamp}.zip`;
|
||||
const zipFilePath = path.join(uploadsDir, zipFileName);
|
||||
const relativeZipPath = path.join('uploads', 'materials', zipFileName);
|
||||
const relativeZipPath = buildStoredUploadPath('materials', zipFileName);
|
||||
|
||||
console.log(`[Zip] Creating all.zip for ${materialType}:`, zipFilePath);
|
||||
console.log(`[Zip] Zip files count:`, zipFiles.length);
|
||||
@ -285,17 +295,23 @@ export async function createZipFromZips(
|
||||
zlib: { level: 9 } // 最高压缩级别
|
||||
});
|
||||
|
||||
output.on('close', () => {
|
||||
output.on('close', async () => {
|
||||
clearTimeout(timeout);
|
||||
const fileSize = archive.pointer();
|
||||
console.log(`[Zip] all.zip created successfully: ${fileSize} bytes`);
|
||||
|
||||
cleanup();
|
||||
resolve({
|
||||
filePath: relativeZipPath,
|
||||
fileName: zipFileName,
|
||||
fileSize
|
||||
});
|
||||
try {
|
||||
await syncStoredUploadFileToRemote(zipFilePath, relativeZipPath, fileSize);
|
||||
cleanup();
|
||||
resolve({
|
||||
filePath: relativeZipPath,
|
||||
fileName: zipFileName,
|
||||
fileSize
|
||||
});
|
||||
} catch (error) {
|
||||
cleanup();
|
||||
reject(error);
|
||||
}
|
||||
});
|
||||
|
||||
archive.on('error', (err: Error) => {
|
||||
@ -326,7 +342,7 @@ export async function createZipFromZips(
|
||||
|
||||
// 只添加zip文件,不添加PSD文件
|
||||
for (const file of zipFiles) {
|
||||
const fullPath = path.isAbsolute(file.path) ? file.path : path.join(process.cwd(), file.path);
|
||||
const fullPath = path.isAbsolute(file.path) ? file.path : resolveStoredUploadPath(file.path);
|
||||
console.log(`[Zip] Adding zip file: ${file.name}`);
|
||||
archive.file(fullPath, { name: file.name });
|
||||
}
|
||||
@ -475,6 +491,7 @@ export async function cleanupTempFiles(
|
||||
// 清理Redis上传会话(关键修复:防止孤立会话堆积)
|
||||
if (uploadIds.length > 0) {
|
||||
console.log(`[Cleanup] Cleaning up ${uploadIds.length} Redis upload sessions...`);
|
||||
const { deleteUploadSession } = await import('@/lib/upload-sessions-redis');
|
||||
for (const uploadId of uploadIds) {
|
||||
try {
|
||||
await deleteUploadSession(uploadId);
|
||||
@ -516,7 +533,13 @@ export async function cleanupOnError(
|
||||
console.log(`[Cleanup] Cleaning up ${paths.length} local zip files...`);
|
||||
for (const zipFilePath of paths) {
|
||||
try {
|
||||
const fullPath = path.join(process.cwd(), zipFilePath);
|
||||
await deleteStoredUploadFileFromRemote(zipFilePath);
|
||||
} catch (cleanupError) {
|
||||
console.error('[Cleanup] Failed to delete remote uploads file:', cleanupError);
|
||||
}
|
||||
|
||||
try {
|
||||
const fullPath = resolveStoredUploadPath(zipFilePath);
|
||||
await unlink(fullPath);
|
||||
console.log(`[Cleanup] Deleted local zip file: ${fullPath}`);
|
||||
} catch (cleanupError) {
|
||||
|
||||
26
lib/upload-paths.ts
Normal file
26
lib/upload-paths.ts
Normal file
@ -0,0 +1,26 @@
|
||||
import path from 'path'
|
||||
|
||||
export function buildStoredUploadPath(...segments: string[]) {
|
||||
return path.posix.join('uploads', ...segments.map((segment) => String(segment || '').replace(/\\/g, '/')))
|
||||
}
|
||||
|
||||
export function getUploadRootDir(): string {
|
||||
const configuredUploadDir = String(process.env.UPLOAD_DIR || '').trim()
|
||||
if (!configuredUploadDir) {
|
||||
return path.join(process.cwd(), 'uploads')
|
||||
}
|
||||
|
||||
return path.isAbsolute(configuredUploadDir)
|
||||
? configuredUploadDir
|
||||
: path.resolve(process.cwd(), configuredUploadDir)
|
||||
}
|
||||
|
||||
export function resolveStoredUploadPath(storedPath: string): string {
|
||||
const normalizedPath = String(storedPath || '').replace(/\\/g, '/')
|
||||
if (path.isAbsolute(normalizedPath)) {
|
||||
return normalizedPath
|
||||
}
|
||||
|
||||
const relativeToUploadRoot = normalizedPath.replace(/^\.?\/*uploads\//i, '')
|
||||
return path.join(getUploadRootDir(), relativeToUploadRoot)
|
||||
}
|
||||
@ -1,6 +1,6 @@
|
||||
# 脚本说明
|
||||
|
||||
> 更新时间:2026-05-07
|
||||
> 更新时间:2026-05-08
|
||||
> 部署相关脚本以 `scripts/git-incremental-deploy.ps1` 为当前默认口径。
|
||||
|
||||
详细说明:
|
||||
@ -102,3 +102,67 @@
|
||||
### `scripts/execute-migration.sh`
|
||||
|
||||
Linux / macOS 版本的一次性迁移工具,适用范围同上。
|
||||
|
||||
## 5. 历史素材 SPU 台账与导入脚本
|
||||
|
||||
### `scripts/scan_spu_ledger.py` ⭐ 当前历史素材新入口
|
||||
|
||||
按 SPU / 组合 SPU / 版本工作区维度扫描真实素材目录,生成 SQLite 台账。
|
||||
|
||||
重要口径:
|
||||
|
||||
- 不读取旧的 SKU 锚点 CSV/SQLite 输出。
|
||||
- 组合 SPU 原样保留,例如 `GBSFY-AD0297+GBSFY-CZ0102`。
|
||||
- 目录名不是标准 SPU 时,即使文件名能反推,也只标记 `needs_review`。
|
||||
- 店铺归属按当前业务口径:`金牌项目-1` 和 `GBJP-*` 归到 `顾家`,不再使用 `金牌` 店铺。
|
||||
- `正版`、`盗版` 都允许进入候选;`错误勿用`、`废稿`、`勿用` 默认跳过。
|
||||
- ready 要求同时存在主图、SKU 图、详情图。
|
||||
|
||||
示例:
|
||||
|
||||
```powershell
|
||||
python scripts\scan_spu_ledger.py `
|
||||
--roots "Z:\顾家项目|Z:\金牌项目-1|Z:\朴美项目|Z:\索菲亚项目" `
|
||||
--db-path "output\tianlan-spu-ledger.db" `
|
||||
--source-batch "tianlan-z" `
|
||||
--reset
|
||||
```
|
||||
|
||||
### `scripts/export_spu_ledger.py`
|
||||
|
||||
从 SPU 台账导出 review CSV、每 20 个工作区一个批次 CSV、汇总 JSON,以及可供历史导入脚本使用的 ready JSON。
|
||||
|
||||
```powershell
|
||||
python scripts\export_spu_ledger.py `
|
||||
--db-path "output\tianlan-spu-ledger.db" `
|
||||
--output-dir "output" `
|
||||
--prefix "tianlan-spu-ledger"
|
||||
```
|
||||
|
||||
### `scripts/import-history-materials.ts` / `scripts/run-history-import-queue.ts`
|
||||
|
||||
继续作为历史素材导入执行入口。新的 SPU 台账 ready JSON 可以直接作为输入,并支持 `任务-详情` 的 PS 源文件通道。
|
||||
|
||||
正式导入的关键边界:
|
||||
|
||||
- Windows 维护机可以读取 `Z:\...` 源素材,但不能只靠本机 `uploads` 写生产库。
|
||||
- 连接生产库并执行 `--apply` 时,预览图必须写入新桶 `gongxiangpan`,公开地址必须是 `https://s3.jiansun.vip/gongxiangpan/由壹点AI/public-display/...`。
|
||||
- ZIP / PSD / 源文件必须最终同步到生产机 `/mnt/PoolOne/apps/youyidian-3d/uploads`,并通过大小校验。
|
||||
- 在 Windows 上正式导入生产库时,必须启用:
|
||||
|
||||
```env
|
||||
HISTORY_IMPORT_REMOTE_UPLOADS_HOST=prod
|
||||
HISTORY_IMPORT_REMOTE_UPLOADS_DIR=/mnt/PoolOne/apps/youyidian-3d/uploads
|
||||
```
|
||||
|
||||
没有远程 uploads 同步配置时,脚本会阻止 `--apply`,避免再次出现数据库有记录但线上下载 404。
|
||||
|
||||
### `scripts/hard-delete-history-materials.ts`
|
||||
|
||||
只用于已经确认的旧批次错误素材硬删除。默认只做备份/演练;正式执行必须同时满足:
|
||||
|
||||
- 候选清单里有 `materialId` 和删除原因。
|
||||
- 候选清单提供的 `fingerprint` 或 `sourcePath` 能在该素材的 `HISTORY_IMPORT` 日志中匹配。
|
||||
- 命令包含 `--apply --confirm=HARD_DELETE_HISTORY_MATERIALS`。
|
||||
|
||||
未确认清单前禁止执行正式硬删除。
|
||||
|
||||
233
scripts/build_import_records_from_package_candidates.py
Normal file
233
scripts/build_import_records_from_package_candidates.py
Normal file
@ -0,0 +1,233 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Build HistoryImportRecord JSON from reviewed package candidates.
|
||||
|
||||
This script is read-only. It scans selected package paths and emits the JSON
|
||||
shape consumed by scripts/import-history-materials.ts. Formal import should
|
||||
still be done through that TypeScript script so environment guards, duplicate
|
||||
checks and validation stay in one place.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import csv
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from scan_spu_ledger import (
|
||||
EXCLUDED_DIR_NAMES,
|
||||
RELEVANT_EXTENSIONS,
|
||||
build_sku_code,
|
||||
classify_file,
|
||||
extract_spu_candidates,
|
||||
fingerprint_for,
|
||||
infer_author,
|
||||
infer_task_detail_source_type,
|
||||
is_compatible_spu_token,
|
||||
load_visual_author_names,
|
||||
normalize_spu,
|
||||
safe_relative,
|
||||
spu_prefix,
|
||||
)
|
||||
|
||||
|
||||
DEFAULT_ROOTS = [
|
||||
r"Z:\顾家项目",
|
||||
r"Z:\金牌项目-1",
|
||||
r"Z:\朴美项目",
|
||||
r"Z:\索菲亚项目",
|
||||
]
|
||||
|
||||
KNOWN_AUTO_PREFIXES = {"GBPM", "GBSFY", "GBGJ", "GBJP"}
|
||||
|
||||
|
||||
def json_dumps(value: Any) -> str:
|
||||
return json.dumps(value, ensure_ascii=False, indent=2)
|
||||
|
||||
|
||||
def norm_path(value: str) -> str:
|
||||
return os.path.normcase(os.path.normpath(value))
|
||||
|
||||
|
||||
def find_source_root(package_path: str, roots: list[str]) -> str:
|
||||
package_norm = norm_path(package_path)
|
||||
matches = [root for root in roots if package_norm.startswith(norm_path(root) + os.sep.lower()) or package_norm == norm_path(root)]
|
||||
if not matches:
|
||||
raise ValueError(f"无法识别 source_root: {package_path}")
|
||||
return max(matches, key=len)
|
||||
|
||||
|
||||
def image_input(file_path: Path, package_root: Path, group: str, spu: str, index: int) -> dict[str, Any]:
|
||||
payload: dict[str, Any] = {
|
||||
"localFilePath": str(file_path),
|
||||
"fileName": file_path.name,
|
||||
"fileSize": int(file_path.stat().st_size),
|
||||
"sortOrder": index,
|
||||
}
|
||||
if group == "sku":
|
||||
payload["skuCode"] = build_sku_code(spu, file_path.name, index)
|
||||
payload["barcodes"] = []
|
||||
return payload
|
||||
|
||||
|
||||
def source_input(file_path: Path) -> dict[str, Any]:
|
||||
return {
|
||||
"localFilePath": str(file_path),
|
||||
"fileName": file_path.name,
|
||||
"fileSize": int(file_path.stat().st_size),
|
||||
}
|
||||
|
||||
|
||||
def collect_assets(package_path: str, spu: str) -> dict[str, list[dict[str, Any]]]:
|
||||
package_root = Path(package_path)
|
||||
grouped_files: dict[str, list[Path]] = {"main": [], "sku": [], "detail": [], "source": []}
|
||||
|
||||
for current, dir_names, file_names in os.walk(package_root):
|
||||
dir_names[:] = [name for name in dir_names if name not in EXCLUDED_DIR_NAMES]
|
||||
current_path = Path(current)
|
||||
for file_name in file_names:
|
||||
file_path = current_path / file_name
|
||||
extension = file_path.suffix.lower()
|
||||
if extension not in RELEVANT_EXTENSIONS:
|
||||
continue
|
||||
relative_path = safe_relative(file_path, package_root)
|
||||
group = classify_file(relative_path, file_name, extension)
|
||||
if group in grouped_files:
|
||||
grouped_files[group].append(file_path)
|
||||
|
||||
for group in grouped_files:
|
||||
grouped_files[group].sort(key=lambda item: safe_relative(item, package_root).lower())
|
||||
|
||||
return {
|
||||
"mainImages": [image_input(path, package_root, "main", spu, index) for index, path in enumerate(grouped_files["main"])],
|
||||
"skuImages": [image_input(path, package_root, "sku", spu, index) for index, path in enumerate(grouped_files["sku"])],
|
||||
"detailImages": [image_input(path, package_root, "detail", spu, index) for index, path in enumerate(grouped_files["detail"])],
|
||||
"psdSources": [source_input(path) for path in grouped_files["source"]],
|
||||
}
|
||||
|
||||
|
||||
def build_record(row: dict[str, str], roots: list[str], visual_author_names: list[str]) -> dict[str, Any]:
|
||||
package_path = row["package_path"]
|
||||
spu = normalize_spu(row["spu"])
|
||||
source_root = find_source_root(package_path, roots)
|
||||
relative_source_path = safe_relative(Path(package_path), Path(source_root))
|
||||
assets = collect_assets(package_path, spu)
|
||||
author, _author_evidence = infer_author(relative_source_path, visual_author_names)
|
||||
source_type, _source_type_evidence = infer_task_detail_source_type(relative_source_path)
|
||||
|
||||
return {
|
||||
"fingerprint": fingerprint_for(source_root, package_path, spu),
|
||||
"spu": spu,
|
||||
"sourcePath": package_path,
|
||||
"relativeSourcePath": relative_source_path,
|
||||
"guessedStore": row["store"],
|
||||
"productCategoryName": row["product_category"],
|
||||
"authorName": author or "历史导入",
|
||||
"taskDetailSourceType": source_type or "渲染",
|
||||
"storeConfidence": 1,
|
||||
"storeCandidates": [row["store"]] if row.get("store") else [],
|
||||
"mainCount": len(assets["mainImages"]),
|
||||
"skuCount": len(assets["skuImages"]),
|
||||
"detailCount": len(assets["detailImages"]),
|
||||
"sourceCount": len(assets["psdSources"]),
|
||||
"status": "ready",
|
||||
"skipReason": None,
|
||||
**assets,
|
||||
"seenInYya": False,
|
||||
"ledger": {
|
||||
"source": "needs-review-package-candidates",
|
||||
"decision": row.get("decision"),
|
||||
"cautionReason": row.get("caution_reason") or "",
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def safety_reason(row: dict[str, str], duplicate_paths: set[str]) -> str:
|
||||
spu = normalize_spu(row["spu"])
|
||||
package_path = row["package_path"]
|
||||
prefix = spu_prefix(spu)
|
||||
if prefix not in KNOWN_AUTO_PREFIXES:
|
||||
return f"SPU 前缀不在自动导入白名单: {prefix or '(none)'}"
|
||||
|
||||
if os.path.normcase(os.path.normpath(package_path)) in duplicate_paths:
|
||||
return "同一个 package_path 被识别为多个 SPU,需人工确认后再导入"
|
||||
|
||||
path_tokens = extract_spu_candidates(package_path, allow_unbranded=False)
|
||||
incompatible_tokens = [
|
||||
token for token in path_tokens if not is_compatible_spu_token(spu, token)
|
||||
]
|
||||
if incompatible_tokens:
|
||||
return "package_path 包含不兼容 SPU: " + ",".join(incompatible_tokens[:6])
|
||||
|
||||
return ""
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="Build import JSON from package candidates.")
|
||||
parser.add_argument("--input-csv", required=True)
|
||||
parser.add_argument("--output", required=True)
|
||||
parser.add_argument("--decision", default="package_ready_review")
|
||||
parser.add_argument("--limit", type=int, default=0)
|
||||
parser.add_argument("--visual-authors-file", default="output/tianlan-spu-ledger/visual-authors.tsv")
|
||||
parser.add_argument("--roots", nargs="*", default=DEFAULT_ROOTS)
|
||||
args = parser.parse_args()
|
||||
|
||||
visual_author_names = (
|
||||
load_visual_author_names(args.visual_authors_file)
|
||||
if args.visual_authors_file and Path(args.visual_authors_file).exists()
|
||||
else []
|
||||
)
|
||||
|
||||
with Path(args.input_csv).open("r", encoding="utf-8-sig", newline="") as handle:
|
||||
candidate_rows = [row for row in csv.DictReader(handle) if row.get("decision") == args.decision]
|
||||
|
||||
path_to_spus: dict[str, set[str]] = {}
|
||||
for row in candidate_rows:
|
||||
path_key = os.path.normcase(os.path.normpath(row["package_path"]))
|
||||
path_to_spus.setdefault(path_key, set()).add(normalize_spu(row["spu"]))
|
||||
duplicate_paths = {path_key for path_key, spus in path_to_spus.items() if len(spus) > 1}
|
||||
|
||||
skipped_rows: list[dict[str, str]] = []
|
||||
rows: list[dict[str, str]] = []
|
||||
for row in candidate_rows:
|
||||
reason = safety_reason(row, duplicate_paths)
|
||||
if reason:
|
||||
skipped_rows.append({**row, "safety_skip_reason": reason})
|
||||
else:
|
||||
rows.append(row)
|
||||
|
||||
if args.limit > 0:
|
||||
rows = rows[: args.limit]
|
||||
|
||||
records = [build_record(row, args.roots, visual_author_names) for row in rows]
|
||||
output_path = Path(args.output)
|
||||
output_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
output_path.write_text(json_dumps(records), encoding="utf-8")
|
||||
skipped_path = output_path.with_suffix(".skipped-candidates.json")
|
||||
skipped_path.write_text(json_dumps(skipped_rows), encoding="utf-8")
|
||||
|
||||
print(
|
||||
json_dumps(
|
||||
{
|
||||
"inputCsv": args.input_csv,
|
||||
"decision": args.decision,
|
||||
"candidateRows": len(candidate_rows),
|
||||
"safetySkipped": len(skipped_rows),
|
||||
"records": len(records),
|
||||
"output": str(output_path),
|
||||
"skippedCandidates": str(skipped_path),
|
||||
"totalMain": sum(record["mainCount"] for record in records),
|
||||
"totalSku": sum(record["skuCount"] for record in records),
|
||||
"totalDetail": sum(record["detailCount"] for record in records),
|
||||
"totalSources": sum(record["sourceCount"] for record in records),
|
||||
}
|
||||
)
|
||||
)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
340
scripts/discover_needs_review_package_candidates.py
Normal file
340
scripts/discover_needs_review_package_candidates.py
Normal file
@ -0,0 +1,340 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Discover minimal package candidates inside needs_review SPU roots.
|
||||
|
||||
This is a read-only second pass. It is stricter than grouping by SPU root: a
|
||||
candidate package is the smallest directory subtree that contains main, SKU and
|
||||
detail assets. That handles cross-level asset folders while avoiding merging
|
||||
multiple versions under the same SPU root.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import csv
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sqlite3
|
||||
from collections import Counter, defaultdict
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from review_needs_review_spu_groups import choose_spu_root_path
|
||||
from scan_spu_ledger import (
|
||||
EXCLUDED_DIR_NAMES,
|
||||
RELEVANT_EXTENSIONS,
|
||||
build_sku_code,
|
||||
classify_file,
|
||||
extract_file_spu_candidates,
|
||||
has_hard_skip,
|
||||
is_compatible_spu_token,
|
||||
normalize_spu,
|
||||
safe_relative,
|
||||
)
|
||||
|
||||
|
||||
CAUTION_RE = re.compile(
|
||||
r"(^|[\\/])3D($|[\\/])|3D建模|3D原图|3D渲染原图|建模|建模图|建模原图|渲染原图|原图建模|"
|
||||
r"源文件|psd源文件|PSD源文件|PS源文件|"
|
||||
r"大图|AI合成|AI原图|图片有误|错误勿用|勿用|废稿|作废|弃用|不要",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
HEADERS = [
|
||||
"decision",
|
||||
"spu",
|
||||
"store",
|
||||
"product_category",
|
||||
"package_path",
|
||||
"relative_package_path",
|
||||
"main_count",
|
||||
"sku_count",
|
||||
"detail_count",
|
||||
"source_count",
|
||||
"needs_review_rows_under_root",
|
||||
"caution_reason",
|
||||
"related_workspace_samples",
|
||||
"main_samples",
|
||||
"sku_samples",
|
||||
"detail_samples",
|
||||
]
|
||||
|
||||
|
||||
def json_dumps(value: Any) -> str:
|
||||
return json.dumps(value, ensure_ascii=False, indent=2)
|
||||
|
||||
|
||||
def latest_run_id(conn: sqlite3.Connection) -> str:
|
||||
row = conn.execute(
|
||||
"SELECT run_id FROM spu_ledger_runs ORDER BY started_at DESC LIMIT 1"
|
||||
).fetchone()
|
||||
if not row:
|
||||
raise SystemExit("台账数据库里没有扫描 run")
|
||||
return str(row["run_id"])
|
||||
|
||||
|
||||
class Node:
|
||||
def __init__(self, path: str, root: str) -> None:
|
||||
self.path = path
|
||||
self.root = root
|
||||
self.children: list[str] = []
|
||||
self.files: list[dict[str, Any]] = []
|
||||
self.direct_counts: Counter[str] = Counter()
|
||||
self.subtree_counts: Counter[str] = Counter()
|
||||
self.samples: dict[str, list[dict[str, Any]]] = defaultdict(list)
|
||||
self.has_caution = False
|
||||
self.conflicts: list[dict[str, str]] = []
|
||||
|
||||
|
||||
def scan_tree(root_path: str, spu: str) -> dict[str, Node]:
|
||||
root = Path(root_path)
|
||||
nodes: dict[str, Node] = {}
|
||||
if not root.exists():
|
||||
return nodes
|
||||
|
||||
for current, dir_names, file_names in os.walk(root):
|
||||
current_path = Path(current)
|
||||
current_str = str(current_path)
|
||||
node = nodes.setdefault(current_str, Node(current_str, root_path))
|
||||
node.has_caution = node.has_caution or bool(CAUTION_RE.search(current_str))
|
||||
|
||||
kept_dirs: list[str] = []
|
||||
for name in dir_names:
|
||||
child = current_path / name
|
||||
if name in EXCLUDED_DIR_NAMES or has_hard_skip(str(child)):
|
||||
continue
|
||||
kept_dirs.append(name)
|
||||
child_str = str(child)
|
||||
node.children.append(child_str)
|
||||
nodes.setdefault(child_str, Node(child_str, root_path))
|
||||
dir_names[:] = kept_dirs
|
||||
|
||||
for file_name in file_names:
|
||||
file_path = current_path / file_name
|
||||
extension = file_path.suffix.lower()
|
||||
if extension not in RELEVANT_EXTENSIONS or has_hard_skip(str(file_path)):
|
||||
continue
|
||||
try:
|
||||
file_size = file_path.stat().st_size
|
||||
except OSError:
|
||||
continue
|
||||
|
||||
relative_path = safe_relative(file_path, root)
|
||||
group = classify_file(relative_path, file_name, extension)
|
||||
record = {
|
||||
"filePath": str(file_path),
|
||||
"relativePath": relative_path,
|
||||
"fileName": file_name,
|
||||
"fileSize": int(file_size),
|
||||
"group": group,
|
||||
}
|
||||
if group == "sku":
|
||||
record["skuCode"] = build_sku_code(spu, file_name, node.direct_counts[group])
|
||||
record["barcodes"] = []
|
||||
node.files.append(record)
|
||||
node.direct_counts[group] += 1
|
||||
if group in {"main", "sku", "detail", "source"} and len(node.samples[group]) < 8:
|
||||
node.samples[group].append(record)
|
||||
if bool(CAUTION_RE.search(str(file_path))):
|
||||
node.has_caution = True
|
||||
|
||||
if group in {"main", "detail"}:
|
||||
tokens = extract_file_spu_candidates(file_name)
|
||||
if tokens and not any(is_compatible_spu_token(spu, token) for token in tokens):
|
||||
node.conflicts.append(
|
||||
{
|
||||
"group": group,
|
||||
"file": relative_path,
|
||||
"tokens": ",".join(tokens),
|
||||
}
|
||||
)
|
||||
|
||||
for path in sorted(nodes, key=lambda item: len(Path(item).parts), reverse=True):
|
||||
node = nodes[path]
|
||||
node.subtree_counts.update(node.direct_counts)
|
||||
for group, records in node.samples.items():
|
||||
node.samples[group] = records[:8]
|
||||
for child_path in node.children:
|
||||
child = nodes[child_path]
|
||||
node.subtree_counts.update(child.subtree_counts)
|
||||
node.has_caution = node.has_caution or child.has_caution
|
||||
if len(node.conflicts) < 12:
|
||||
node.conflicts.extend(child.conflicts[: max(0, 12 - len(node.conflicts))])
|
||||
for group, records in child.samples.items():
|
||||
existing = node.samples[group]
|
||||
for record in records:
|
||||
if len(existing) >= 8:
|
||||
break
|
||||
existing.append(record)
|
||||
return nodes
|
||||
|
||||
|
||||
def has_all_three(node: Node) -> bool:
|
||||
return (
|
||||
node.subtree_counts["main"] > 0
|
||||
and node.subtree_counts["sku"] > 0
|
||||
and node.subtree_counts["detail"] > 0
|
||||
)
|
||||
|
||||
|
||||
def minimal_package_nodes(nodes: dict[str, Node]) -> list[Node]:
|
||||
candidates = [node for node in nodes.values() if has_all_three(node)]
|
||||
candidate_paths = {node.path for node in candidates}
|
||||
minimal: list[Node] = []
|
||||
for node in candidates:
|
||||
descendant_has_all_three = False
|
||||
stack = list(node.children)
|
||||
while stack:
|
||||
child_path = stack.pop()
|
||||
if child_path in candidate_paths:
|
||||
descendant_has_all_three = True
|
||||
break
|
||||
child = nodes.get(child_path)
|
||||
if child:
|
||||
stack.extend(child.children)
|
||||
if not descendant_has_all_three:
|
||||
minimal.append(node)
|
||||
minimal.sort(key=lambda item: item.path.lower())
|
||||
return minimal
|
||||
|
||||
|
||||
def package_decision(node: Node) -> tuple[str, str]:
|
||||
reasons: list[str] = []
|
||||
if node.has_caution:
|
||||
reasons.append("路径或文件名包含建模/源文件/大图/AI原图/错误等谨慎标识")
|
||||
if node.conflicts:
|
||||
tokens = sorted({conflict["tokens"] for conflict in node.conflicts})[:4]
|
||||
reasons.append("主图/详情文件名出现不同SPU: " + ";".join(tokens))
|
||||
if node.subtree_counts["main"] == 0 or node.subtree_counts["sku"] == 0 or node.subtree_counts["detail"] == 0:
|
||||
reasons.append("缺少主图/SKU/详情之一")
|
||||
if reasons:
|
||||
return "package_caution_review", " | ".join(reasons)
|
||||
return "package_ready_review", ""
|
||||
|
||||
|
||||
def row_for(node: Node, item_group: list[sqlite3.Row], spu: str, store: str, category: str) -> dict[str, Any]:
|
||||
package = Path(node.path)
|
||||
root = Path(node.root)
|
||||
decision, caution_reason = package_decision(node)
|
||||
related = [
|
||||
str(item["workspace_path"])
|
||||
for item in item_group
|
||||
if str(item["workspace_path"]).lower().startswith(node.path.lower())
|
||||
][:8]
|
||||
if not related:
|
||||
related = [str(item["workspace_path"]) for item in item_group[:8]]
|
||||
|
||||
return {
|
||||
"decision": decision,
|
||||
"spu": spu,
|
||||
"store": store,
|
||||
"product_category": category,
|
||||
"package_path": str(package),
|
||||
"relative_package_path": safe_relative(package, root),
|
||||
"main_count": node.subtree_counts["main"],
|
||||
"sku_count": node.subtree_counts["sku"],
|
||||
"detail_count": node.subtree_counts["detail"],
|
||||
"source_count": node.subtree_counts["source"],
|
||||
"needs_review_rows_under_root": len(item_group),
|
||||
"caution_reason": caution_reason,
|
||||
"related_workspace_samples": " || ".join(related),
|
||||
"main_samples": " || ".join(record["relativePath"] for record in node.samples.get("main", [])),
|
||||
"sku_samples": " || ".join(record["relativePath"] for record in node.samples.get("sku", [])),
|
||||
"detail_samples": " || ".join(record["relativePath"] for record in node.samples.get("detail", [])),
|
||||
}
|
||||
|
||||
|
||||
def write_csv(path: Path, rows: list[dict[str, Any]]) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
with path.open("w", encoding="utf-8-sig", newline="") as handle:
|
||||
writer = csv.DictWriter(handle, fieldnames=HEADERS)
|
||||
writer.writeheader()
|
||||
writer.writerows(rows)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="Discover package candidates for needs_review rows.")
|
||||
parser.add_argument("--db-path", default="output/tianlan-spu-ledger.db")
|
||||
parser.add_argument("--run-id", default="")
|
||||
parser.add_argument("--output-dir", default="")
|
||||
args = parser.parse_args()
|
||||
|
||||
stamp = datetime.now().strftime("%Y%m%d-%H%M%S")
|
||||
output_dir = Path(args.output_dir or f"output/tianlan-spu-ledger/needs-review-package-candidates-{stamp}")
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
conn = sqlite3.connect(str(Path(args.db_path)))
|
||||
conn.row_factory = sqlite3.Row
|
||||
run_id = args.run_id or latest_run_id(conn)
|
||||
items = conn.execute(
|
||||
"""
|
||||
SELECT *
|
||||
FROM spu_ledger_items
|
||||
WHERE run_id = ? AND status = 'needs_review'
|
||||
ORDER BY source_root_name, relative_workspace_path, spu
|
||||
""",
|
||||
(run_id,),
|
||||
).fetchall()
|
||||
|
||||
groups: dict[tuple[str, str, str, str], list[sqlite3.Row]] = defaultdict(list)
|
||||
for item in items:
|
||||
spu = normalize_spu(item["spu"])
|
||||
root_path = choose_spu_root_path(item)
|
||||
key = (str(item["store"] or ""), spu, str(item["product_category"] or ""), root_path)
|
||||
groups[key].append(item)
|
||||
|
||||
rows: list[dict[str, Any]] = []
|
||||
detail_payload: list[dict[str, Any]] = []
|
||||
roots_without_package = 0
|
||||
for (store, spu, category, root_path), item_group in sorted(groups.items()):
|
||||
nodes = scan_tree(root_path, spu)
|
||||
packages = minimal_package_nodes(nodes)
|
||||
if not packages:
|
||||
roots_without_package += 1
|
||||
continue
|
||||
for node in packages:
|
||||
row = row_for(node, item_group, spu, store, category)
|
||||
rows.append(row)
|
||||
detail_payload.append(
|
||||
{
|
||||
**row,
|
||||
"samples": node.samples,
|
||||
"conflicts": node.conflicts[:12],
|
||||
"workspaces": [
|
||||
{
|
||||
"fingerprint": item["fingerprint"],
|
||||
"workspacePath": item["workspace_path"],
|
||||
"relativeWorkspacePath": item["relative_workspace_path"],
|
||||
"versionLabel": item["version_label"],
|
||||
"mainCount": item["main_count"],
|
||||
"skuCount": item["sku_count"],
|
||||
"detailCount": item["detail_count"],
|
||||
"statusReason": item["status_reason"],
|
||||
}
|
||||
for item in item_group
|
||||
],
|
||||
}
|
||||
)
|
||||
|
||||
write_csv(output_dir / "package-candidates.csv", rows)
|
||||
(output_dir / "package-candidates.json").write_text(json_dumps(detail_payload), encoding="utf-8")
|
||||
summary = {
|
||||
"runId": run_id,
|
||||
"needsReviewItems": len(items),
|
||||
"spuRootGroups": len(groups),
|
||||
"rootsWithoutPackage": roots_without_package,
|
||||
"packageCandidates": len(rows),
|
||||
"decisionCounts": dict(Counter(row["decision"] for row in rows)),
|
||||
"outputCsv": str(output_dir / "package-candidates.csv"),
|
||||
"outputJson": str(output_dir / "package-candidates.json"),
|
||||
}
|
||||
(output_dir / "summary.json").write_text(json_dumps(summary), encoding="utf-8")
|
||||
print(json_dumps(summary))
|
||||
conn.close()
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
76
scripts/export-history-import-scan-summary.js
Normal file
76
scripts/export-history-import-scan-summary.js
Normal file
@ -0,0 +1,76 @@
|
||||
const fs = require('fs')
|
||||
const path = require('path')
|
||||
|
||||
const inputPath = process.argv[2] || path.join('output', 'tianlan-history-sku-import-scan-full.json')
|
||||
const outputDir = process.argv[3] || path.join('output')
|
||||
const baseName = process.argv[4] || 'tianlan-history-sku-import'
|
||||
|
||||
function escapeCsvCell(value) {
|
||||
const text = String(value ?? '')
|
||||
if (/[",\r\n]/.test(text)) {
|
||||
return `"${text.replace(/"/g, '""')}"`
|
||||
}
|
||||
return text
|
||||
}
|
||||
|
||||
function writeCsv(filePath, rows) {
|
||||
const headers = [
|
||||
'status(状态)',
|
||||
'spu(SPU)',
|
||||
'store(店铺)',
|
||||
'product_category(产品品类)',
|
||||
'author(作者)',
|
||||
'main_count(主图数量)',
|
||||
'sku_count(SKU图数量)',
|
||||
'detail_count(详情图数量)',
|
||||
'skip_reason(跳过或复核原因)',
|
||||
'source_path(来源完整路径)',
|
||||
'relative_source_path(来源相对路径)',
|
||||
]
|
||||
const lines = [
|
||||
headers.join(','),
|
||||
...rows.map((record) =>
|
||||
[
|
||||
record.status,
|
||||
record.spu,
|
||||
record.guessedStore,
|
||||
record.productCategoryName,
|
||||
record.authorName,
|
||||
record.mainCount,
|
||||
record.skuCount,
|
||||
record.detailCount,
|
||||
record.skipReason || '',
|
||||
record.sourcePath,
|
||||
record.relativeSourcePath,
|
||||
]
|
||||
.map(escapeCsvCell)
|
||||
.join(','),
|
||||
),
|
||||
]
|
||||
fs.writeFileSync(filePath, `\uFEFF${lines.join('\r\n')}`, 'utf8')
|
||||
}
|
||||
|
||||
function countBy(records, picker) {
|
||||
return records.reduce((acc, record) => {
|
||||
const key = picker(record) || '<empty>'
|
||||
acc[key] = (acc[key] || 0) + 1
|
||||
return acc
|
||||
}, {})
|
||||
}
|
||||
|
||||
const records = JSON.parse(fs.readFileSync(inputPath, 'utf8'))
|
||||
fs.mkdirSync(outputDir, { recursive: true })
|
||||
|
||||
writeCsv(path.join(outputDir, `${baseName}-ready.csv`), records.filter((record) => record.status === 'ready'))
|
||||
writeCsv(path.join(outputDir, `${baseName}-needs-review.csv`), records.filter((record) => record.status === 'needs_review'))
|
||||
writeCsv(path.join(outputDir, `${baseName}-skipped.csv`), records.filter((record) => record.status === 'skipped'))
|
||||
|
||||
const summary = {
|
||||
total: records.length,
|
||||
status: countBy(records, (record) => record.status),
|
||||
store: countBy(records, (record) => record.guessedStore),
|
||||
productCategory: countBy(records, (record) => record.productCategoryName),
|
||||
}
|
||||
|
||||
fs.writeFileSync(path.join(outputDir, `${baseName}-summary.json`), JSON.stringify(summary, null, 2), 'utf8')
|
||||
console.log(JSON.stringify(summary, null, 2))
|
||||
@ -10,6 +10,7 @@ EXPORT_HEADERS = [
|
||||
("spu", "spu(SPU)"),
|
||||
("directory_name", "directory_name(目录名称)"),
|
||||
("relative_path", "relative_path(相对路径)"),
|
||||
("windows_path", "windows_path(Windows路径)"),
|
||||
("sku_dir_path", "sku_dir_path(SKU目录完整路径)"),
|
||||
("parent_path", "parent_path(上级目录)"),
|
||||
("grandparent_path", "grandparent_path(上上级目录)"),
|
||||
@ -83,6 +84,7 @@ def main():
|
||||
spu,
|
||||
directory_name,
|
||||
relative_path,
|
||||
sku_dir_path AS windows_path,
|
||||
sku_dir_path,
|
||||
parent_path,
|
||||
grandparent_path,
|
||||
|
||||
259
scripts/export_spu_ledger.py
Normal file
259
scripts/export_spu_ledger.py
Normal file
@ -0,0 +1,259 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Export the SPU ledger into review CSV files and ready import JSON.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import csv
|
||||
import json
|
||||
import sqlite3
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
STATUS_EXPORTS = {
|
||||
"ready": "ready",
|
||||
"needs_review": "needs-review",
|
||||
"skipped": "skipped",
|
||||
"duplicate": "duplicate",
|
||||
"delete_candidate": "delete-candidate",
|
||||
}
|
||||
|
||||
|
||||
def json_dumps(value) -> str:
|
||||
return json.dumps(value, ensure_ascii=False, indent=2)
|
||||
|
||||
|
||||
def latest_run_id(conn: sqlite3.Connection) -> str:
|
||||
row = conn.execute(
|
||||
"SELECT run_id FROM spu_ledger_runs ORDER BY started_at DESC LIMIT 1"
|
||||
).fetchone()
|
||||
if not row:
|
||||
raise SystemExit("台账数据库里没有扫描 run")
|
||||
return str(row["run_id"])
|
||||
|
||||
|
||||
def read_items(conn: sqlite3.Connection, run_id: str) -> list[sqlite3.Row]:
|
||||
return conn.execute(
|
||||
"""
|
||||
SELECT *
|
||||
FROM spu_ledger_items
|
||||
WHERE run_id = ?
|
||||
ORDER BY source_root_name, relative_workspace_path, spu
|
||||
""",
|
||||
(run_id,),
|
||||
).fetchall()
|
||||
|
||||
|
||||
def read_files(conn: sqlite3.Connection, item_id: int, asset_group: str) -> list[sqlite3.Row]:
|
||||
return conn.execute(
|
||||
"""
|
||||
SELECT *
|
||||
FROM spu_ledger_files
|
||||
WHERE item_id = ? AND asset_group = ?
|
||||
ORDER BY sort_order, relative_file_path
|
||||
""",
|
||||
(item_id, asset_group),
|
||||
).fetchall()
|
||||
|
||||
|
||||
def csv_headers() -> list[str]:
|
||||
return [
|
||||
"status(状态)",
|
||||
"spu(SPU)",
|
||||
"store(店铺)",
|
||||
"product_category(产品品类)",
|
||||
"author(作者)",
|
||||
"task_detail_source_type(出图方式)",
|
||||
"version_label(版本/工作区)",
|
||||
"main_count(主图数量)",
|
||||
"sku_count(SKU图数量)",
|
||||
"detail_count(详情图数量)",
|
||||
"source_count(PS源文件数量)",
|
||||
"status_reason(判断原因)",
|
||||
"spu_source(SPU来源)",
|
||||
"store_evidence(店铺证据)",
|
||||
"author_evidence(作者证据)",
|
||||
"task_detail_source_type_evidence(出图方式证据)",
|
||||
"source_batch(来源批次)",
|
||||
"source_root(来源根目录)",
|
||||
"workspace_path(工作区完整路径)",
|
||||
"relative_workspace_path(工作区相对路径)",
|
||||
"fingerprint(台账指纹)",
|
||||
]
|
||||
|
||||
|
||||
def item_to_csv_row(item: sqlite3.Row) -> dict[str, object]:
|
||||
return {
|
||||
"status(状态)": item["status"],
|
||||
"spu(SPU)": item["spu"],
|
||||
"store(店铺)": item["store"],
|
||||
"product_category(产品品类)": item["product_category"],
|
||||
"author(作者)": item["author"],
|
||||
"task_detail_source_type(出图方式)": item["task_detail_source_type"],
|
||||
"version_label(版本/工作区)": item["version_label"],
|
||||
"main_count(主图数量)": item["main_count"],
|
||||
"sku_count(SKU图数量)": item["sku_count"],
|
||||
"detail_count(详情图数量)": item["detail_count"],
|
||||
"source_count(PS源文件数量)": item["source_count"],
|
||||
"status_reason(判断原因)": item["status_reason"],
|
||||
"spu_source(SPU来源)": item["spu_source"],
|
||||
"store_evidence(店铺证据)": item["store_evidence"],
|
||||
"author_evidence(作者证据)": item["author_evidence"],
|
||||
"task_detail_source_type_evidence(出图方式证据)": item["task_detail_source_type_evidence"],
|
||||
"source_batch(来源批次)": item["source_batch"],
|
||||
"source_root(来源根目录)": item["source_root"],
|
||||
"workspace_path(工作区完整路径)": item["workspace_path"],
|
||||
"relative_workspace_path(工作区相对路径)": item["relative_workspace_path"],
|
||||
"fingerprint(台账指纹)": item["fingerprint"],
|
||||
}
|
||||
|
||||
|
||||
def write_csv(path: Path, rows: list[dict[str, object]], headers: list[str]) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
with path.open("w", encoding="utf-8-sig", newline="") as handle:
|
||||
writer = csv.DictWriter(handle, fieldnames=headers)
|
||||
writer.writeheader()
|
||||
for row in rows:
|
||||
writer.writerow(row)
|
||||
|
||||
|
||||
def to_image_input(file_row: sqlite3.Row) -> dict[str, object]:
|
||||
payload = {
|
||||
"localFilePath": file_row["file_path"],
|
||||
"fileName": file_row["file_name"],
|
||||
"fileSize": int(file_row["file_size"] or 0),
|
||||
"sortOrder": int(file_row["sort_order"] or 0),
|
||||
}
|
||||
if file_row["asset_group"] == "sku":
|
||||
payload["skuCode"] = file_row["sku_code"] or ""
|
||||
payload["barcodes"] = []
|
||||
return payload
|
||||
|
||||
|
||||
def to_source_input(file_row: sqlite3.Row) -> dict[str, object]:
|
||||
return {
|
||||
"localFilePath": file_row["file_path"],
|
||||
"fileName": file_row["file_name"],
|
||||
"fileSize": int(file_row["file_size"] or 0),
|
||||
}
|
||||
|
||||
|
||||
def build_import_record(conn: sqlite3.Connection, item: sqlite3.Row) -> dict[str, object]:
|
||||
main = [to_image_input(row) for row in read_files(conn, item["id"], "main")]
|
||||
sku = [to_image_input(row) for row in read_files(conn, item["id"], "sku")]
|
||||
detail = [to_image_input(row) for row in read_files(conn, item["id"], "detail")]
|
||||
sources = [to_source_input(row) for row in read_files(conn, item["id"], "source")]
|
||||
|
||||
return {
|
||||
"fingerprint": item["fingerprint"],
|
||||
"spu": item["spu"],
|
||||
"sourcePath": item["workspace_path"],
|
||||
"relativeSourcePath": item["relative_workspace_path"],
|
||||
"guessedStore": item["store"],
|
||||
"productCategoryName": item["product_category"],
|
||||
"authorName": item["author"] or "历史导入",
|
||||
"taskDetailSourceType": item["task_detail_source_type"] or "渲染",
|
||||
"storeConfidence": 1,
|
||||
"storeCandidates": [item["store"]] if item["store"] else [],
|
||||
"mainCount": item["main_count"],
|
||||
"skuCount": item["sku_count"],
|
||||
"detailCount": item["detail_count"],
|
||||
"sourceCount": item["source_count"],
|
||||
"status": item["status"],
|
||||
"skipReason": None if item["status"] == "ready" else item["status_reason"],
|
||||
"mainImages": main,
|
||||
"skuImages": sku,
|
||||
"detailImages": detail,
|
||||
"psdSources": sources,
|
||||
"seenInYya": False,
|
||||
"ledger": {
|
||||
"runId": item["run_id"],
|
||||
"sourceBatch": item["source_batch"],
|
||||
"spuSource": item["spu_source"],
|
||||
"spuEvidence": json.loads(item["spu_evidence_json"] or "[]"),
|
||||
"pathFlags": json.loads(item["path_flags_json"] or "{}"),
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def write_batch_csvs(output_dir: Path, prefix: str, items: list[sqlite3.Row], batch_size: int) -> None:
|
||||
if batch_size <= 0:
|
||||
return
|
||||
headers = csv_headers()
|
||||
pending = [item for item in items if item["status"] in ("ready", "needs_review")]
|
||||
for start in range(0, len(pending), batch_size):
|
||||
batch_no = start // batch_size + 1
|
||||
batch = pending[start : start + batch_size]
|
||||
rows = [item_to_csv_row(item) for item in batch]
|
||||
write_csv(output_dir / f"{prefix}-batch-{batch_no:03d}.csv", rows, headers)
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(description="Export SPU ledger review files.")
|
||||
parser.add_argument("--db-path", required=True)
|
||||
parser.add_argument("--output-dir", default=str(Path.cwd() / "output"))
|
||||
parser.add_argument("--prefix", default="spu-ledger")
|
||||
parser.add_argument("--run-id", default="")
|
||||
parser.add_argument("--batch-size", type=int, default=20)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def main() -> int:
|
||||
args = parse_args()
|
||||
db_path = Path(args.db_path)
|
||||
output_dir = Path(args.output_dir)
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
conn = sqlite3.connect(str(db_path))
|
||||
conn.row_factory = sqlite3.Row
|
||||
run_id = args.run_id or latest_run_id(conn)
|
||||
items = read_items(conn, run_id)
|
||||
|
||||
headers = csv_headers()
|
||||
status_counts: dict[str, int] = {}
|
||||
store_counts: dict[str, int] = {}
|
||||
category_counts: dict[str, int] = {}
|
||||
|
||||
for item in items:
|
||||
status_counts[item["status"]] = status_counts.get(item["status"], 0) + 1
|
||||
store_counts[item["store"]] = store_counts.get(item["store"], 0) + 1
|
||||
category = item["product_category"] or "未识别"
|
||||
category_counts[category] = category_counts.get(category, 0) + 1
|
||||
|
||||
for status, suffix in STATUS_EXPORTS.items():
|
||||
rows = [item_to_csv_row(item) for item in items if item["status"] == status]
|
||||
write_csv(output_dir / f"{args.prefix}-{suffix}.csv", rows, headers)
|
||||
|
||||
write_csv(output_dir / f"{args.prefix}-all.csv", [item_to_csv_row(item) for item in items], headers)
|
||||
write_batch_csvs(output_dir, args.prefix, items, args.batch_size)
|
||||
|
||||
ready_records = [build_import_record(conn, item) for item in items if item["status"] == "ready"]
|
||||
ready_json_path = output_dir / f"{args.prefix}-ready-import-records.json"
|
||||
ready_json_path.write_text(json_dumps(ready_records), encoding="utf-8")
|
||||
|
||||
summary = {
|
||||
"runId": run_id,
|
||||
"dbPath": str(db_path),
|
||||
"total": len(items),
|
||||
"status": status_counts,
|
||||
"store": store_counts,
|
||||
"productCategory": category_counts,
|
||||
"readyImportRecords": str(ready_json_path),
|
||||
}
|
||||
summary_path = output_dir / f"{args.prefix}-summary.json"
|
||||
summary_path.write_text(json_dumps(summary), encoding="utf-8")
|
||||
|
||||
conn.close()
|
||||
|
||||
print("SPU 台账导出完成")
|
||||
print(f"- run_id: {run_id}")
|
||||
print(f"- total: {len(items)}")
|
||||
print(f"- ready_import_records: {ready_json_path}")
|
||||
print(f"- summary: {summary_path}")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
148
scripts/filter_needs_review_dry_run_records.py
Normal file
148
scripts/filter_needs_review_dry_run_records.py
Normal file
@ -0,0 +1,148 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Filter dry-run import records from needs_review package candidates.
|
||||
|
||||
This is a conservative, read-only guard used after import-history-materials
|
||||
dry-run. It keeps only records that still look like a complete package root
|
||||
instead of an asset subfolder or ambiguous multi-SPU package.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import re
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
|
||||
MAIN_DIR_RE = re.compile(r"主图|main|^(750|800|1440|1920)$", re.IGNORECASE)
|
||||
SKU_DIR_RE = re.compile(r"sku|白底|场景", re.IGNORECASE)
|
||||
DETAIL_DIR_RE = re.compile(r"详情|detail", re.IGNORECASE)
|
||||
FORBIDDEN_RE = re.compile(r"错误|勿用|废稿|图片有误|3D建模|3D原图|建模图|建模原图|渲染原图|AI原图|大图", re.IGNORECASE)
|
||||
ASSET_SEGMENT_RE = re.compile(r"^(主图|sku|SKU|详情|详情页|白底|场景|750|800|790|1200|1440|1920|切图|800-800主图)$")
|
||||
BRANDED_SPU_RE = re.compile(r"\b(GBPM|GBSFY|GBGJ|GBJP)-[A-Z]{1,8}\d{3,5}\b", re.IGNORECASE)
|
||||
|
||||
|
||||
def load_json(path: Path) -> Any:
|
||||
return json.loads(path.read_text(encoding="utf-8-sig"))
|
||||
|
||||
|
||||
def dump_json(path: Path, value: Any) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(json.dumps(value, ensure_ascii=False, indent=2), encoding="utf-8")
|
||||
|
||||
|
||||
def norm_spu(value: str) -> str:
|
||||
return value.replace("-", "-").replace("—", "-").replace("–", "-").upper().strip()
|
||||
|
||||
|
||||
def path_segments(path: str) -> list[str]:
|
||||
normalized = path.replace("/", "\\")
|
||||
return [part for part in normalized.split("\\") if part]
|
||||
|
||||
|
||||
def child_dirs(path: str) -> list[str]:
|
||||
root = Path(path)
|
||||
if not root.exists():
|
||||
return []
|
||||
return sorted([item.name for item in root.iterdir() if item.is_dir()], key=str.lower)
|
||||
|
||||
|
||||
def has_required_child_dirs(names: list[str]) -> bool:
|
||||
has_main = any(MAIN_DIR_RE.search(name) for name in names)
|
||||
has_sku = any(SKU_DIR_RE.search(name) for name in names)
|
||||
has_detail = any(DETAIL_DIR_RE.search(name) for name in names)
|
||||
return has_main and has_sku and has_detail
|
||||
|
||||
|
||||
def first_branded_token(path: str) -> str:
|
||||
match = BRANDED_SPU_RE.search(norm_spu(path))
|
||||
return norm_spu(match.group(0)) if match else ""
|
||||
|
||||
|
||||
def skip_reason(record: dict[str, Any]) -> str:
|
||||
source_path = str(record.get("sourcePath") or "")
|
||||
spu = norm_spu(str(record.get("spu") or ""))
|
||||
segments = path_segments(source_path)
|
||||
dirs = child_dirs(source_path)
|
||||
|
||||
if int(record.get("mainCount") or 0) <= 0:
|
||||
return "mainCount 为 0,疑似把主图子目录误判为包根"
|
||||
if int(record.get("skuCount") or 0) <= 0:
|
||||
return "skuCount 为 0,不满足三类资产齐全"
|
||||
if int(record.get("detailCount") or 0) <= 0:
|
||||
return "detailCount 为 0,不满足三类资产齐全"
|
||||
if FORBIDDEN_RE.search(source_path):
|
||||
return "路径包含禁用/高风险目录词"
|
||||
if "组合" in source_path and "+" not in spu:
|
||||
return "组合目录未表达为组合 SPU,需人工确认"
|
||||
if len(segments) >= 2 and ASSET_SEGMENT_RE.match(segments[-1]) and ASSET_SEGMENT_RE.match(segments[-2]):
|
||||
return "候选路径落在资产子目录内,不是包根目录"
|
||||
if len(segments) >= 1 and ASSET_SEGMENT_RE.match(segments[-1]) and not has_required_child_dirs(dirs):
|
||||
return "候选路径末级像资产目录且首层缺少主图/SKU/详情"
|
||||
if not has_required_child_dirs(dirs):
|
||||
return "首层目录缺少明确的主图/SKU/详情三类证据"
|
||||
|
||||
token = first_branded_token(source_path)
|
||||
if token and token != spu:
|
||||
return f"路径首个品牌 SPU 为 {token},记录 SPU 为 {spu},需人工确认"
|
||||
|
||||
return ""
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="Filter dry-run import records conservatively.")
|
||||
parser.add_argument("--records", required=True, help="Input HistoryImportRecord JSON")
|
||||
parser.add_argument("--result", required=True, help="Dry-run result JSON")
|
||||
parser.add_argument("--output", required=True, help="Filtered records JSON")
|
||||
parser.add_argument("--skipped-output", required=True, help="Skipped records JSON")
|
||||
args = parser.parse_args()
|
||||
|
||||
records = load_json(Path(args.records))
|
||||
results = load_json(Path(args.result))
|
||||
dry_fingerprints = {item.get("fingerprint") for item in results if item.get("status") == "dry_run"}
|
||||
|
||||
selected: list[dict[str, Any]] = []
|
||||
skipped: list[dict[str, Any]] = []
|
||||
for record in records:
|
||||
if record.get("fingerprint") not in dry_fingerprints:
|
||||
continue
|
||||
reason = skip_reason(record)
|
||||
if reason:
|
||||
skipped.append(
|
||||
{
|
||||
"fingerprint": record.get("fingerprint"),
|
||||
"spu": record.get("spu"),
|
||||
"sourcePath": record.get("sourcePath"),
|
||||
"mainCount": record.get("mainCount"),
|
||||
"skuCount": record.get("skuCount"),
|
||||
"detailCount": record.get("detailCount"),
|
||||
"sourceCount": record.get("sourceCount"),
|
||||
"skipReason": reason,
|
||||
}
|
||||
)
|
||||
else:
|
||||
selected.append(record)
|
||||
|
||||
dump_json(Path(args.output), selected)
|
||||
dump_json(Path(args.skipped_output), skipped)
|
||||
|
||||
print(
|
||||
json.dumps(
|
||||
{
|
||||
"dryRunFingerprints": len(dry_fingerprints),
|
||||
"selected": len(selected),
|
||||
"skipped": len(skipped),
|
||||
"output": args.output,
|
||||
"skippedOutput": args.skipped_output,
|
||||
},
|
||||
ensure_ascii=False,
|
||||
indent=2,
|
||||
)
|
||||
)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
214
scripts/hard-delete-history-materials.ts
Normal file
214
scripts/hard-delete-history-materials.ts
Normal file
@ -0,0 +1,214 @@
|
||||
import path from 'path'
|
||||
import { promises as fs } from 'fs'
|
||||
import { prisma } from '@/lib/prisma'
|
||||
|
||||
type DeleteCandidate = {
|
||||
materialId: string
|
||||
fingerprint?: string
|
||||
sourcePath?: string
|
||||
reason: string
|
||||
}
|
||||
|
||||
function parseArgs() {
|
||||
const args = process.argv.slice(2)
|
||||
const getArg = (name: string) => {
|
||||
const match = args.find((arg) => arg.startsWith(`--${name}=`))
|
||||
return match ? match.slice(name.length + 3) : ''
|
||||
}
|
||||
|
||||
const candidates = getArg('candidates')
|
||||
const output = getArg('output')
|
||||
const confirm = getArg('confirm')
|
||||
const apply = args.includes('--apply')
|
||||
|
||||
if (!candidates) {
|
||||
throw new Error('缺少 --candidates=删除候选清单 JSON')
|
||||
}
|
||||
|
||||
return {
|
||||
candidates,
|
||||
output: output || path.join(path.dirname(candidates), `history-hard-delete-backup-${Date.now()}.json`),
|
||||
apply,
|
||||
confirm,
|
||||
}
|
||||
}
|
||||
|
||||
function normalizeCandidate(raw: any): DeleteCandidate {
|
||||
return {
|
||||
materialId: String(raw?.materialId || raw?.id || '').trim(),
|
||||
fingerprint: String(raw?.fingerprint || '').trim() || undefined,
|
||||
sourcePath: String(raw?.sourcePath || raw?.source_path || '').trim() || undefined,
|
||||
reason: String(raw?.reason || raw?.deleteReason || '').trim(),
|
||||
}
|
||||
}
|
||||
|
||||
function assertCandidate(candidate: DeleteCandidate) {
|
||||
if (!candidate.materialId) {
|
||||
throw new Error('删除候选缺少 materialId')
|
||||
}
|
||||
if (!candidate.reason) {
|
||||
throw new Error(`删除候选 ${candidate.materialId} 缺少 reason`)
|
||||
}
|
||||
if (!candidate.fingerprint && !candidate.sourcePath) {
|
||||
throw new Error(`删除候选 ${candidate.materialId} 必须提供 fingerprint 或 sourcePath`)
|
||||
}
|
||||
}
|
||||
|
||||
function stringifyJson(value: unknown) {
|
||||
return JSON.stringify(
|
||||
value,
|
||||
(_key, item) => (typeof item === 'bigint' ? item.toString() : item),
|
||||
2,
|
||||
)
|
||||
}
|
||||
|
||||
async function loadCandidates(candidatePath: string) {
|
||||
const raw = await fs.readFile(candidatePath, 'utf8')
|
||||
const parsed = JSON.parse(raw)
|
||||
const list = Array.isArray(parsed) ? parsed : parsed.candidates
|
||||
if (!Array.isArray(list)) {
|
||||
throw new Error('删除候选清单必须是数组,或包含 candidates 数组')
|
||||
}
|
||||
const candidates = list.map(normalizeCandidate)
|
||||
candidates.forEach(assertCandidate)
|
||||
return candidates
|
||||
}
|
||||
|
||||
async function loadMaterialSnapshot(materialId: string) {
|
||||
return prisma.material.findUnique({
|
||||
where: { id: materialId },
|
||||
include: {
|
||||
assets: {
|
||||
include: {
|
||||
barcodes: true,
|
||||
},
|
||||
},
|
||||
skus: true,
|
||||
logs: true,
|
||||
likes: true,
|
||||
favorites: true,
|
||||
downloadLogs: true,
|
||||
transfers: true,
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
function hasHistoryEvidence(material: Awaited<ReturnType<typeof loadMaterialSnapshot>>, candidate: DeleteCandidate) {
|
||||
if (!material) {
|
||||
return false
|
||||
}
|
||||
|
||||
return material.logs.some((log) => {
|
||||
if (log.action !== 'HISTORY_IMPORT') {
|
||||
return false
|
||||
}
|
||||
const changes = String(log.changes || '')
|
||||
return Boolean(
|
||||
(candidate.fingerprint && changes.includes(candidate.fingerprint)) ||
|
||||
(candidate.sourcePath && changes.includes(candidate.sourcePath)),
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const options = parseArgs()
|
||||
const candidates = await loadCandidates(options.candidates)
|
||||
const backups: any[] = []
|
||||
const results: any[] = []
|
||||
|
||||
console.log('历史错误素材硬删除预检开始')
|
||||
console.log(`- 候选清单: ${options.candidates}`)
|
||||
console.log(`- 模式: ${options.apply ? '正式硬删除' : '仅备份/演练'}`)
|
||||
console.log(`- 备份输出: ${options.output}`)
|
||||
|
||||
if (options.apply && options.confirm !== 'HARD_DELETE_HISTORY_MATERIALS') {
|
||||
throw new Error('正式硬删除必须追加 --confirm=HARD_DELETE_HISTORY_MATERIALS')
|
||||
}
|
||||
|
||||
for (const candidate of candidates) {
|
||||
const material = await loadMaterialSnapshot(candidate.materialId)
|
||||
if (!material) {
|
||||
results.push({
|
||||
materialId: candidate.materialId,
|
||||
status: 'skipped',
|
||||
reason: '素材不存在',
|
||||
})
|
||||
continue
|
||||
}
|
||||
|
||||
const evidenceMatched = hasHistoryEvidence(material, candidate)
|
||||
backups.push({
|
||||
candidate,
|
||||
material,
|
||||
backedUpAt: new Date().toISOString(),
|
||||
evidenceMatched,
|
||||
})
|
||||
|
||||
if (!evidenceMatched) {
|
||||
results.push({
|
||||
materialId: candidate.materialId,
|
||||
productModel: material.productModel,
|
||||
status: 'blocked',
|
||||
reason: '未在 HISTORY_IMPORT 日志中匹配 fingerprint/sourcePath,拒绝删除',
|
||||
})
|
||||
continue
|
||||
}
|
||||
|
||||
if (!options.apply) {
|
||||
results.push({
|
||||
materialId: candidate.materialId,
|
||||
productModel: material.productModel,
|
||||
status: 'dry_run',
|
||||
reason: candidate.reason,
|
||||
})
|
||||
continue
|
||||
}
|
||||
|
||||
await prisma.material.delete({
|
||||
where: { id: candidate.materialId },
|
||||
})
|
||||
|
||||
results.push({
|
||||
materialId: candidate.materialId,
|
||||
productModel: material.productModel,
|
||||
status: 'deleted',
|
||||
reason: candidate.reason,
|
||||
})
|
||||
}
|
||||
|
||||
await fs.mkdir(path.dirname(options.output), { recursive: true })
|
||||
await fs.writeFile(
|
||||
options.output,
|
||||
stringifyJson({
|
||||
sourceCandidates: options.candidates,
|
||||
apply: options.apply,
|
||||
generatedAt: new Date().toISOString(),
|
||||
results,
|
||||
backups,
|
||||
}),
|
||||
'utf8',
|
||||
)
|
||||
|
||||
const summary = results.reduce(
|
||||
(acc, item) => {
|
||||
acc.total += 1
|
||||
acc[item.status] = (acc[item.status] || 0) + 1
|
||||
return acc
|
||||
},
|
||||
{ total: 0 } as Record<string, number>,
|
||||
)
|
||||
|
||||
console.log('历史错误素材硬删除预检完成')
|
||||
console.log(summary)
|
||||
console.log(`- 备份/结果: ${options.output}`)
|
||||
}
|
||||
|
||||
main()
|
||||
.catch((error) => {
|
||||
console.error('历史错误素材硬删除失败:', error)
|
||||
process.exitCode = 1
|
||||
})
|
||||
.finally(async () => {
|
||||
await prisma.$disconnect()
|
||||
process.exit(process.exitCode ?? 0)
|
||||
})
|
||||
@ -1,11 +1,106 @@
|
||||
import path from 'path'
|
||||
import { promises as fs } from 'fs'
|
||||
import { HeadBucketCommand, S3Client } from '@aws-sdk/client-s3'
|
||||
import {
|
||||
importHistoryMaterials,
|
||||
resolveImportOperator,
|
||||
type HistoryImportRecord,
|
||||
} from '@/lib/history-material-import'
|
||||
import { prisma } from '@/lib/prisma'
|
||||
import {
|
||||
assertRemoteUploadSyncAccess,
|
||||
getRemoteUploadSyncConfig,
|
||||
} from '@/lib/remote-upload-sync'
|
||||
|
||||
function getDatabaseHost() {
|
||||
const rawUrl = String(process.env.DATABASE_URL || '').trim()
|
||||
if (!rawUrl) {
|
||||
return ''
|
||||
}
|
||||
|
||||
try {
|
||||
return new URL(rawUrl).hostname
|
||||
} catch {
|
||||
return ''
|
||||
}
|
||||
}
|
||||
|
||||
function assertApplyEnvironment() {
|
||||
const databaseHost = getDatabaseHost()
|
||||
const s3Bucket = String(process.env.S3_BUCKET || '').trim()
|
||||
const s3Endpoint = String(process.env.S3_ENDPOINT || '').trim().replace(/\/+$/, '')
|
||||
const s3PublicDomain = String(process.env.S3_PUBLIC_DOMAIN || '').trim()
|
||||
const s3FilePath = String(process.env.S3_FILE_PATH || '').trim().replace(/\\/g, '/')
|
||||
const uploadDir = String(process.env.UPLOAD_DIR || '').trim()
|
||||
const remoteUploadSyncConfig = getRemoteUploadSyncConfig()
|
||||
|
||||
const isProductionDatabase = databaseHost === '192.168.58.20'
|
||||
const isExpectedEndpoint = s3Endpoint === 'http://192.168.58.20:9000'
|
||||
const isExpectedPreviewBucket =
|
||||
s3Bucket === 'gongxiangpan' &&
|
||||
/^https:\/\/s3\.jiansun\.vip\/gongxiangpan\/?$/i.test(s3PublicDomain)
|
||||
const isExpectedFilePath = s3FilePath === '由壹点AI/'
|
||||
|
||||
if (!isExpectedEndpoint || !isExpectedPreviewBucket || !isExpectedFilePath) {
|
||||
throw new Error(
|
||||
[
|
||||
'正式导入环境不正确:预览文件必须写入新桶 gongxiangpan 的正确前缀。',
|
||||
`当前 S3_ENDPOINT=${s3Endpoint || '(empty)'}`,
|
||||
`当前 S3_BUCKET=${s3Bucket || '(empty)'}`,
|
||||
`当前 S3_PUBLIC_DOMAIN=${s3PublicDomain || '(empty)'}`,
|
||||
`当前 S3_FILE_PATH=${s3FilePath || '(empty)'}`,
|
||||
].join('\n'),
|
||||
)
|
||||
}
|
||||
|
||||
if (isProductionDatabase && process.platform === 'win32' && !remoteUploadSyncConfig) {
|
||||
throw new Error(
|
||||
[
|
||||
'正式导入环境不正确:当前是 Windows 工作站,但 DATABASE_URL 指向生产库。',
|
||||
'直接导入会把下载 zip / PSD 源文件写到本机 uploads,生产站点无法下载。',
|
||||
'如确需在 Windows 读取 Z 盘源文件并写生产库,必须配置远程 uploads 同步:',
|
||||
'HISTORY_IMPORT_REMOTE_UPLOADS_HOST=prod',
|
||||
'HISTORY_IMPORT_REMOTE_UPLOADS_DIR=/mnt/PoolOne/apps/youyidian-3d/uploads',
|
||||
`当前 UPLOAD_DIR=${uploadDir || '(empty)'}`,
|
||||
].join('\n'),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
async function assertApplyStorageAccess() {
|
||||
const s3Endpoint = String(process.env.S3_ENDPOINT || '').trim()
|
||||
const s3Bucket = String(process.env.S3_BUCKET || '').trim()
|
||||
const accessKeyId = String(process.env.S3_ACCESS_KEY_ID || '').trim()
|
||||
const secretAccessKey = String(process.env.S3_SECRET_ACCESS_KEY || '').trim()
|
||||
|
||||
if (!accessKeyId || !secretAccessKey) {
|
||||
throw new Error('正式导入环境不正确:缺少 S3 访问凭据,不能验证新桶写入权限。')
|
||||
}
|
||||
|
||||
const client = new S3Client({
|
||||
endpoint: s3Endpoint,
|
||||
region: process.env.S3_REGION || 'us-east-1',
|
||||
credentials: {
|
||||
accessKeyId,
|
||||
secretAccessKey,
|
||||
},
|
||||
forcePathStyle: String(process.env.S3_ENABLE_PATH_STYLE || '').trim() !== '0',
|
||||
})
|
||||
|
||||
try {
|
||||
await client.send(new HeadBucketCommand({ Bucket: s3Bucket }))
|
||||
} catch (error) {
|
||||
const statusCode = (error as { $metadata?: { httpStatusCode?: number } }).$metadata?.httpStatusCode
|
||||
throw new Error(
|
||||
[
|
||||
'正式导入环境不正确:当前 S3 凭据无法访问新桶,已阻止写入数据库。',
|
||||
`S3_BUCKET=${s3Bucket || '(empty)'}`,
|
||||
`S3_ENDPOINT=${s3Endpoint || '(empty)'}`,
|
||||
`HTTP_STATUS=${statusCode || 'unknown'}`,
|
||||
].join('\n'),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
function parseArgs() {
|
||||
const args = process.argv.slice(2)
|
||||
@ -36,6 +131,12 @@ function parseArgs() {
|
||||
|
||||
async function main() {
|
||||
const options = parseArgs()
|
||||
if (options.apply) {
|
||||
assertApplyEnvironment()
|
||||
await assertApplyStorageAccess()
|
||||
await assertRemoteUploadSyncAccess()
|
||||
}
|
||||
|
||||
const raw = await fs.readFile(options.input, 'utf8')
|
||||
const records = JSON.parse(raw) as HistoryImportRecord[]
|
||||
const operator = await resolveImportOperator(options.operator)
|
||||
|
||||
337
scripts/migrate-history-preview-assets.js
Normal file
337
scripts/migrate-history-preview-assets.js
Normal file
@ -0,0 +1,337 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
const { PrismaClient } = require('@prisma/client')
|
||||
const { S3Client, HeadObjectCommand, PutObjectCommand } = require('@aws-sdk/client-s3')
|
||||
|
||||
const prisma = new PrismaClient()
|
||||
|
||||
const DEFAULT_OLD_PREFIX = 'https://s3.1080p.hk/jsxiangche/'
|
||||
|
||||
function getArg(name, fallback = '') {
|
||||
const prefix = `--${name}=`
|
||||
const match = process.argv.slice(2).find((arg) => arg.startsWith(prefix))
|
||||
return match ? match.slice(prefix.length) : fallback
|
||||
}
|
||||
|
||||
function hasFlag(name) {
|
||||
return process.argv.includes(`--${name}`)
|
||||
}
|
||||
|
||||
function normalizePrefix(prefix) {
|
||||
return String(prefix || '').trim().replace(/\/+$/, '') + '/'
|
||||
}
|
||||
|
||||
function normalizePath(input) {
|
||||
return String(input || '')
|
||||
.trim()
|
||||
.replace(/\\/g, '/')
|
||||
.replace(/^\/+/, '')
|
||||
.replace(/\/{2,}/g, '/')
|
||||
}
|
||||
|
||||
function parseOldObjectKey(oldUrl, oldBucket) {
|
||||
const parsed = new URL(oldUrl)
|
||||
let key = decodeURIComponent(parsed.pathname).replace(/^\/+/, '')
|
||||
if (oldBucket && key.startsWith(`${oldBucket}/`)) {
|
||||
key = key.slice(oldBucket.length + 1)
|
||||
}
|
||||
return normalizePath(key)
|
||||
}
|
||||
|
||||
function ensureDestinationKey(oldKey) {
|
||||
const s3FilePath = normalizePath(process.env.S3_FILE_PATH || '')
|
||||
if (!s3FilePath) {
|
||||
return oldKey
|
||||
}
|
||||
|
||||
if (oldKey.startsWith(`${s3FilePath}/`)) {
|
||||
return oldKey
|
||||
}
|
||||
|
||||
if (oldKey.startsWith('public-display/')) {
|
||||
return `${s3FilePath}/${oldKey}`
|
||||
}
|
||||
|
||||
return oldKey
|
||||
}
|
||||
|
||||
function buildPublicUrl(key) {
|
||||
const publicDomain = String(process.env.S3_PUBLIC_DOMAIN || '').trim().replace(/\/+$/, '')
|
||||
if (!publicDomain) {
|
||||
throw new Error('Missing S3_PUBLIC_DOMAIN')
|
||||
}
|
||||
return `${publicDomain}/${key}`
|
||||
}
|
||||
|
||||
function guessContentType(url) {
|
||||
const lower = url.toLowerCase()
|
||||
if (lower.endsWith('.webp')) return 'image/webp'
|
||||
if (lower.endsWith('.png')) return 'image/png'
|
||||
if (lower.endsWith('.jpg') || lower.endsWith('.jpeg')) return 'image/jpeg'
|
||||
return 'application/octet-stream'
|
||||
}
|
||||
|
||||
function createS3Client() {
|
||||
const endpoint = String(process.env.S3_ENDPOINT || '').split('#')[0].trim()
|
||||
if (!endpoint) throw new Error('Missing S3_ENDPOINT')
|
||||
if (!process.env.S3_BUCKET) throw new Error('Missing S3_BUCKET')
|
||||
if (!process.env.S3_ACCESS_KEY_ID) throw new Error('Missing S3_ACCESS_KEY_ID')
|
||||
if (!process.env.S3_SECRET_ACCESS_KEY) throw new Error('Missing S3_SECRET_ACCESS_KEY')
|
||||
|
||||
return new S3Client({
|
||||
endpoint,
|
||||
region: process.env.S3_REGION || 'us-east-1',
|
||||
credentials: {
|
||||
accessKeyId: process.env.S3_ACCESS_KEY_ID,
|
||||
secretAccessKey: process.env.S3_SECRET_ACCESS_KEY,
|
||||
},
|
||||
forcePathStyle: process.env.S3_ENABLE_PATH_STYLE !== '0',
|
||||
maxAttempts: 5,
|
||||
})
|
||||
}
|
||||
|
||||
async function destinationExists(client, bucket, key) {
|
||||
try {
|
||||
await client.send(new HeadObjectCommand({ Bucket: bucket, Key: key }))
|
||||
return true
|
||||
} catch (error) {
|
||||
const statusCode = error && error.$metadata && error.$metadata.httpStatusCode
|
||||
if (statusCode === 404 || error.name === 'NotFound') {
|
||||
return false
|
||||
}
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
async function fetchOldObject(oldUrl) {
|
||||
const response = await fetch(oldUrl)
|
||||
if (!response.ok) {
|
||||
throw new Error(`Fetch failed ${response.status}`)
|
||||
}
|
||||
|
||||
return {
|
||||
body: Buffer.from(await response.arrayBuffer()),
|
||||
contentType: response.headers.get('content-type') || guessContentType(oldUrl),
|
||||
}
|
||||
}
|
||||
|
||||
async function getOldPreviewUrls(oldPrefix, limit) {
|
||||
const limitedSql = limit > 0 ? `LIMIT ${Number(limit)}` : ''
|
||||
return prisma.$queryRawUnsafe(`
|
||||
WITH history_materials AS (
|
||||
SELECT DISTINCT "materialId" AS id
|
||||
FROM material_logs
|
||||
WHERE action = 'HISTORY_IMPORT'
|
||||
), active_history AS (
|
||||
SELECT m.id, m."previewUrl"
|
||||
FROM materials m
|
||||
JOIN history_materials h ON h.id = m.id
|
||||
WHERE m."deletedAt" IS NULL
|
||||
)
|
||||
SELECT DISTINCT old_url
|
||||
FROM (
|
||||
SELECT "previewUrl" AS old_url
|
||||
FROM active_history
|
||||
WHERE "previewUrl" LIKE $1
|
||||
UNION
|
||||
SELECT ma.file_url AS old_url
|
||||
FROM material_assets ma
|
||||
JOIN active_history h ON h.id = ma.material_id
|
||||
WHERE ma.asset_type IN ('main-preview', 'sku-preview', 'detail-preview')
|
||||
AND ma.file_url LIKE $1
|
||||
) urls
|
||||
WHERE old_url IS NOT NULL AND old_url <> ''
|
||||
ORDER BY old_url
|
||||
${limitedSql}
|
||||
`, `${oldPrefix}%`)
|
||||
}
|
||||
|
||||
async function updateDatabaseUrl(oldUrl, newUrl) {
|
||||
const materialUpdates = await prisma.$executeRawUnsafe(`
|
||||
WITH history_materials AS (
|
||||
SELECT DISTINCT "materialId" AS id
|
||||
FROM material_logs
|
||||
WHERE action = 'HISTORY_IMPORT'
|
||||
)
|
||||
UPDATE materials m
|
||||
SET "previewUrl" = $1,
|
||||
"updatedAt" = NOW()
|
||||
FROM history_materials h
|
||||
WHERE h.id = m.id
|
||||
AND m."deletedAt" IS NULL
|
||||
AND m."previewUrl" = $2
|
||||
`, newUrl, oldUrl)
|
||||
|
||||
const assetUpdates = await prisma.$executeRawUnsafe(`
|
||||
WITH history_materials AS (
|
||||
SELECT DISTINCT "materialId" AS id
|
||||
FROM material_logs
|
||||
WHERE action = 'HISTORY_IMPORT'
|
||||
)
|
||||
UPDATE material_assets ma
|
||||
SET file_url = $1,
|
||||
updated_at = NOW()
|
||||
FROM history_materials h
|
||||
JOIN materials m ON m.id = h.id
|
||||
WHERE ma.material_id = m.id
|
||||
AND m."deletedAt" IS NULL
|
||||
AND ma.asset_type IN ('main-preview', 'sku-preview', 'detail-preview')
|
||||
AND ma.file_url = $2
|
||||
`, newUrl, oldUrl)
|
||||
|
||||
return { materialUpdates, assetUpdates }
|
||||
}
|
||||
|
||||
async function bulkUpdateDatabaseUrls(oldPrefix, newPrefix) {
|
||||
const materialUpdates = await prisma.$executeRawUnsafe(`
|
||||
WITH history_materials AS (
|
||||
SELECT DISTINCT "materialId" AS id
|
||||
FROM material_logs
|
||||
WHERE action = 'HISTORY_IMPORT'
|
||||
)
|
||||
UPDATE materials m
|
||||
SET "previewUrl" = REPLACE(m."previewUrl", $1, $2),
|
||||
"updatedAt" = NOW()
|
||||
FROM history_materials h
|
||||
WHERE h.id = m.id
|
||||
AND m."deletedAt" IS NULL
|
||||
AND m."previewUrl" LIKE $3
|
||||
`, oldPrefix, newPrefix, `${oldPrefix}%`)
|
||||
|
||||
const assetUpdates = await prisma.$executeRawUnsafe(`
|
||||
WITH history_materials AS (
|
||||
SELECT DISTINCT "materialId" AS id
|
||||
FROM material_logs
|
||||
WHERE action = 'HISTORY_IMPORT'
|
||||
)
|
||||
UPDATE material_assets ma
|
||||
SET file_url = REPLACE(ma.file_url, $1, $2),
|
||||
updated_at = NOW()
|
||||
FROM history_materials h
|
||||
JOIN materials m ON m.id = h.id
|
||||
WHERE ma.material_id = m.id
|
||||
AND m."deletedAt" IS NULL
|
||||
AND ma.asset_type IN ('main-preview', 'sku-preview', 'detail-preview')
|
||||
AND ma.file_url LIKE $3
|
||||
`, oldPrefix, newPrefix, `${oldPrefix}%`)
|
||||
|
||||
return { materialUpdates, assetUpdates }
|
||||
}
|
||||
|
||||
async function runPool(items, concurrency, worker) {
|
||||
let index = 0
|
||||
const workers = Array.from({ length: concurrency }, async () => {
|
||||
while (index < items.length) {
|
||||
const currentIndex = index
|
||||
index += 1
|
||||
await worker(items[currentIndex], currentIndex)
|
||||
}
|
||||
})
|
||||
await Promise.all(workers)
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const apply = hasFlag('apply')
|
||||
const bulkDbUpdate = hasFlag('bulk-db-update')
|
||||
const skipDbUpdate = hasFlag('skip-db-update')
|
||||
const oldPrefix = normalizePrefix(getArg('old-prefix', DEFAULT_OLD_PREFIX))
|
||||
const oldBucket = getArg('old-bucket', 'jsxiangche')
|
||||
const limit = Number(getArg('limit', '0')) || 0
|
||||
const concurrency = Math.max(1, Number(getArg('concurrency', '8')) || 8)
|
||||
const oldRows = await getOldPreviewUrls(oldPrefix, limit)
|
||||
const mappings = oldRows.map((row) => {
|
||||
const oldUrl = row.old_url
|
||||
const key = ensureDestinationKey(parseOldObjectKey(oldUrl, oldBucket))
|
||||
return {
|
||||
oldUrl,
|
||||
key,
|
||||
newUrl: buildPublicUrl(key),
|
||||
}
|
||||
})
|
||||
|
||||
console.log(JSON.stringify({
|
||||
mode: apply ? 'apply' : 'dry-run',
|
||||
oldPrefix,
|
||||
totalUrls: mappings.length,
|
||||
concurrency,
|
||||
bulkDbUpdate,
|
||||
skipDbUpdate,
|
||||
sample: mappings.slice(0, 3),
|
||||
}, null, 2))
|
||||
|
||||
if (!apply) {
|
||||
return
|
||||
}
|
||||
|
||||
const s3Client = createS3Client()
|
||||
const bucket = process.env.S3_BUCKET
|
||||
const stats = {
|
||||
uploaded: 0,
|
||||
skippedExisting: 0,
|
||||
dbMaterialUpdates: 0,
|
||||
dbAssetUpdates: 0,
|
||||
failed: 0,
|
||||
}
|
||||
const failures = []
|
||||
|
||||
await runPool(mappings, concurrency, async (mapping, currentIndex) => {
|
||||
try {
|
||||
const exists = await destinationExists(s3Client, bucket, mapping.key)
|
||||
if (exists) {
|
||||
stats.skippedExisting += 1
|
||||
} else {
|
||||
const oldObject = await fetchOldObject(mapping.oldUrl)
|
||||
await s3Client.send(new PutObjectCommand({
|
||||
Bucket: bucket,
|
||||
Key: mapping.key,
|
||||
Body: oldObject.body,
|
||||
ContentType: oldObject.contentType,
|
||||
ACL: process.env.S3_SET_ACL === '0' ? undefined : 'public-read',
|
||||
}))
|
||||
stats.uploaded += 1
|
||||
}
|
||||
|
||||
if (!bulkDbUpdate && !skipDbUpdate) {
|
||||
const dbResult = await updateDatabaseUrl(mapping.oldUrl, mapping.newUrl)
|
||||
stats.dbMaterialUpdates += Number(dbResult.materialUpdates || 0)
|
||||
stats.dbAssetUpdates += Number(dbResult.assetUpdates || 0)
|
||||
}
|
||||
} catch (error) {
|
||||
stats.failed += 1
|
||||
failures.push({
|
||||
oldUrl: mapping.oldUrl,
|
||||
message: error instanceof Error ? error.message : String(error),
|
||||
})
|
||||
}
|
||||
|
||||
if ((currentIndex + 1) % 1000 === 0 || currentIndex + 1 === mappings.length) {
|
||||
console.log(JSON.stringify({ processed: currentIndex + 1, total: mappings.length, stats }))
|
||||
}
|
||||
})
|
||||
|
||||
if (bulkDbUpdate && stats.failed === 0) {
|
||||
const newPrefix = mappings[0]
|
||||
? mappings[0].newUrl.slice(0, mappings[0].newUrl.length - mappings[0].key.length)
|
||||
: ''
|
||||
if (!newPrefix) {
|
||||
throw new Error('Cannot infer new prefix for bulk database update')
|
||||
}
|
||||
const dbResult = await bulkUpdateDatabaseUrls(oldPrefix, newPrefix)
|
||||
stats.dbMaterialUpdates += Number(dbResult.materialUpdates || 0)
|
||||
stats.dbAssetUpdates += Number(dbResult.assetUpdates || 0)
|
||||
}
|
||||
|
||||
console.log(JSON.stringify({ done: true, stats, failures: failures.slice(0, 20) }, null, 2))
|
||||
if (stats.failed > 0) {
|
||||
process.exitCode = 1
|
||||
}
|
||||
}
|
||||
|
||||
main()
|
||||
.catch((error) => {
|
||||
console.error(error)
|
||||
process.exitCode = 1
|
||||
})
|
||||
.finally(async () => {
|
||||
await prisma.$disconnect()
|
||||
})
|
||||
315
scripts/review_needs_review_spu_groups.py
Normal file
315
scripts/review_needs_review_spu_groups.py
Normal file
@ -0,0 +1,315 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Build a second-pass review table for needs_review SPU ledger items.
|
||||
|
||||
The first SPU ledger intentionally keeps uncertain workspaces out of the import
|
||||
path. This script does not change the ledger or production data; it groups those
|
||||
uncertain rows by the real SPU root and rescans that root so cross-level
|
||||
main/SKU/detail folders can be reviewed together.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import csv
|
||||
import json
|
||||
import os
|
||||
import sqlite3
|
||||
from collections import Counter, defaultdict
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from scan_spu_ledger import (
|
||||
EXCLUDED_DIR_NAMES,
|
||||
IMAGE_EXTENSIONS,
|
||||
OTHER_IMAGE_EXTENSIONS,
|
||||
PS_SOURCE_EXTENSIONS,
|
||||
RELEVANT_EXTENSIONS,
|
||||
SOURCE_ARCHIVE_EXTENSIONS,
|
||||
build_sku_code,
|
||||
classify_file,
|
||||
extract_spu_candidates,
|
||||
has_hard_skip,
|
||||
has_modeling_source_segment,
|
||||
is_compatible_spu_token,
|
||||
normalize_spu,
|
||||
safe_relative,
|
||||
)
|
||||
|
||||
|
||||
REVIEW_HEADERS = [
|
||||
"decision",
|
||||
"spu",
|
||||
"store",
|
||||
"product_category",
|
||||
"spu_root_path",
|
||||
"needs_review_rows",
|
||||
"direct_all_three_rows",
|
||||
"aggregate_main_count",
|
||||
"aggregate_sku_count",
|
||||
"aggregate_detail_count",
|
||||
"aggregate_source_count",
|
||||
"hard_skip_file_count",
|
||||
"modeling_file_count",
|
||||
"status_reasons",
|
||||
"workspace_samples",
|
||||
"main_samples",
|
||||
"sku_samples",
|
||||
"detail_samples",
|
||||
]
|
||||
|
||||
|
||||
def json_dumps(value: Any) -> str:
|
||||
return json.dumps(value, ensure_ascii=False, indent=2)
|
||||
|
||||
|
||||
def latest_run_id(conn: sqlite3.Connection) -> str:
|
||||
row = conn.execute(
|
||||
"SELECT run_id FROM spu_ledger_runs ORDER BY started_at DESC LIMIT 1"
|
||||
).fetchone()
|
||||
if not row:
|
||||
raise SystemExit("台账数据库里没有扫描 run")
|
||||
return str(row["run_id"])
|
||||
|
||||
|
||||
def choose_spu_root_path(item: sqlite3.Row) -> str:
|
||||
source_root = Path(item["source_root"])
|
||||
workspace_path = Path(item["workspace_path"])
|
||||
relative = Path(item["relative_workspace_path"])
|
||||
spu = normalize_spu(item["spu"])
|
||||
|
||||
parts = list(relative.parts)
|
||||
for index, part in enumerate(parts):
|
||||
tokens = extract_spu_candidates(part, allow_unbranded=True)
|
||||
if any(is_compatible_spu_token(spu, token) for token in tokens):
|
||||
return str(source_root.joinpath(*parts[: index + 1]))
|
||||
|
||||
# File-inferred and non-standard folders can have no SPU-looking segment.
|
||||
# In that case use the nearest useful parent instead of the leaf itself so
|
||||
# sibling main/SKU/detail folders can still be discovered.
|
||||
if len(parts) >= 2:
|
||||
return str(source_root.joinpath(*parts[:-1]))
|
||||
return str(workspace_path)
|
||||
|
||||
|
||||
def scan_spu_root(root_path: str, spu: str) -> dict[str, Any]:
|
||||
root = Path(root_path)
|
||||
counts: Counter[str] = Counter()
|
||||
samples: dict[str, list[dict[str, Any]]] = defaultdict(list)
|
||||
hard_skip_file_count = 0
|
||||
modeling_file_count = 0
|
||||
|
||||
if not root.exists():
|
||||
return {
|
||||
"counts": counts,
|
||||
"samples": samples,
|
||||
"hard_skip_file_count": 0,
|
||||
"modeling_file_count": 0,
|
||||
"exists": False,
|
||||
}
|
||||
|
||||
for current, dir_names, file_names in os.walk(root):
|
||||
dir_names[:] = [
|
||||
name
|
||||
for name in dir_names
|
||||
if name not in EXCLUDED_DIR_NAMES and not has_hard_skip(str(Path(current) / name))
|
||||
]
|
||||
|
||||
current_path = Path(current)
|
||||
current_is_modeling = has_modeling_source_segment(str(current_path))
|
||||
for file_name in file_names:
|
||||
file_path = current_path / file_name
|
||||
extension = file_path.suffix.lower()
|
||||
if extension not in RELEVANT_EXTENSIONS:
|
||||
continue
|
||||
|
||||
relative_path = safe_relative(file_path, root)
|
||||
if has_hard_skip(str(file_path)):
|
||||
hard_skip_file_count += 1
|
||||
continue
|
||||
if current_is_modeling:
|
||||
modeling_file_count += 1
|
||||
|
||||
group = classify_file(relative_path, file_name, extension)
|
||||
if group == "other_source_archive" and extension in SOURCE_ARCHIVE_EXTENSIONS:
|
||||
# Keep archives visible, but do not promote generic zip files to
|
||||
# source unless the first-pass classifier saw source evidence.
|
||||
pass
|
||||
|
||||
counts[group] += 1
|
||||
if group in {"main", "sku", "detail", "source"} and len(samples[group]) < 8:
|
||||
record = {
|
||||
"filePath": str(file_path),
|
||||
"relativePath": relative_path,
|
||||
"fileName": file_name,
|
||||
"fileSize": file_path.stat().st_size,
|
||||
}
|
||||
if group == "sku":
|
||||
record["skuCode"] = build_sku_code(spu, file_name, counts[group] - 1)
|
||||
record["barcodes"] = []
|
||||
samples[group].append(record)
|
||||
|
||||
return {
|
||||
"counts": counts,
|
||||
"samples": samples,
|
||||
"hard_skip_file_count": hard_skip_file_count,
|
||||
"modeling_file_count": modeling_file_count,
|
||||
"exists": True,
|
||||
}
|
||||
|
||||
|
||||
def decide_group(items: list[sqlite3.Row], aggregate: dict[str, Any]) -> str:
|
||||
counts: Counter[str] = aggregate["counts"]
|
||||
has_all_three = counts["main"] > 0 and counts["sku"] > 0 and counts["detail"] > 0
|
||||
direct_all_three = [
|
||||
item
|
||||
for item in items
|
||||
if int(item["main_count"] or 0) > 0
|
||||
and int(item["sku_count"] or 0) > 0
|
||||
and int(item["detail_count"] or 0) > 0
|
||||
]
|
||||
reasons = " | ".join(sorted({str(item["status_reason"] or "") for item in items}))
|
||||
has_modeling_reason = "建模" in reasons or "3D" in reasons or aggregate["modeling_file_count"] > 0
|
||||
|
||||
if not aggregate["exists"]:
|
||||
return "manual_missing_root"
|
||||
if direct_all_three and not has_modeling_reason:
|
||||
return "direct_ready_review"
|
||||
if has_all_three and not has_modeling_reason:
|
||||
return "cross_level_ready_review"
|
||||
if has_modeling_reason and not has_all_three:
|
||||
return "skip_likely_modeling_or_partial"
|
||||
if counts["detail"] == 0:
|
||||
return "skip_likely_no_detail"
|
||||
if counts["main"] == 0 or counts["sku"] == 0:
|
||||
return "partial_needs_pairing"
|
||||
return "manual_review"
|
||||
|
||||
|
||||
def write_csv(path: Path, rows: list[dict[str, Any]]) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
with path.open("w", encoding="utf-8-sig", newline="") as handle:
|
||||
writer = csv.DictWriter(handle, fieldnames=REVIEW_HEADERS)
|
||||
writer.writeheader()
|
||||
writer.writerows(rows)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="Second-pass review for needs_review SPU groups.")
|
||||
parser.add_argument("--db-path", default="output/tianlan-spu-ledger.db")
|
||||
parser.add_argument("--run-id", default="")
|
||||
parser.add_argument("--output-dir", default="")
|
||||
args = parser.parse_args()
|
||||
|
||||
db_path = Path(args.db_path)
|
||||
stamp = datetime.now().strftime("%Y%m%d-%H%M%S")
|
||||
output_dir = Path(args.output_dir or f"output/tianlan-spu-ledger/needs-review-cross-level-{stamp}")
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
conn = sqlite3.connect(str(db_path))
|
||||
conn.row_factory = sqlite3.Row
|
||||
run_id = args.run_id or latest_run_id(conn)
|
||||
items = conn.execute(
|
||||
"""
|
||||
SELECT *
|
||||
FROM spu_ledger_items
|
||||
WHERE run_id = ? AND status = 'needs_review'
|
||||
ORDER BY source_root_name, relative_workspace_path, spu
|
||||
""",
|
||||
(run_id,),
|
||||
).fetchall()
|
||||
|
||||
groups: dict[tuple[str, str, str, str], list[sqlite3.Row]] = defaultdict(list)
|
||||
for item in items:
|
||||
spu_root_path = choose_spu_root_path(item)
|
||||
key = (
|
||||
str(item["store"] or ""),
|
||||
normalize_spu(item["spu"]),
|
||||
str(item["source_root_name"] or ""),
|
||||
spu_root_path,
|
||||
)
|
||||
groups[key].append(item)
|
||||
|
||||
review_rows: list[dict[str, Any]] = []
|
||||
details: list[dict[str, Any]] = []
|
||||
for (store, spu, source_root_name, spu_root_path), grouped_items in sorted(groups.items()):
|
||||
aggregate = scan_spu_root(spu_root_path, spu)
|
||||
counts: Counter[str] = aggregate["counts"]
|
||||
decision = decide_group(grouped_items, aggregate)
|
||||
status_reasons = sorted({str(item["status_reason"] or "") for item in grouped_items})
|
||||
workspace_samples = [str(item["workspace_path"]) for item in grouped_items[:8]]
|
||||
direct_all_three_rows = sum(
|
||||
1
|
||||
for item in grouped_items
|
||||
if int(item["main_count"] or 0) > 0
|
||||
and int(item["sku_count"] or 0) > 0
|
||||
and int(item["detail_count"] or 0) > 0
|
||||
)
|
||||
|
||||
row = {
|
||||
"decision": decision,
|
||||
"spu": spu,
|
||||
"store": store,
|
||||
"product_category": grouped_items[0]["product_category"],
|
||||
"spu_root_path": spu_root_path,
|
||||
"needs_review_rows": len(grouped_items),
|
||||
"direct_all_three_rows": direct_all_three_rows,
|
||||
"aggregate_main_count": counts["main"],
|
||||
"aggregate_sku_count": counts["sku"],
|
||||
"aggregate_detail_count": counts["detail"],
|
||||
"aggregate_source_count": counts["source"],
|
||||
"hard_skip_file_count": aggregate["hard_skip_file_count"],
|
||||
"modeling_file_count": aggregate["modeling_file_count"],
|
||||
"status_reasons": " | ".join(status_reasons),
|
||||
"workspace_samples": " || ".join(workspace_samples),
|
||||
"main_samples": " || ".join(item["relativePath"] for item in aggregate["samples"].get("main", [])),
|
||||
"sku_samples": " || ".join(item["relativePath"] for item in aggregate["samples"].get("sku", [])),
|
||||
"detail_samples": " || ".join(item["relativePath"] for item in aggregate["samples"].get("detail", [])),
|
||||
}
|
||||
review_rows.append(row)
|
||||
details.append(
|
||||
{
|
||||
**row,
|
||||
"sourceRootName": source_root_name,
|
||||
"workspaces": [
|
||||
{
|
||||
"fingerprint": item["fingerprint"],
|
||||
"workspacePath": item["workspace_path"],
|
||||
"relativeWorkspacePath": item["relative_workspace_path"],
|
||||
"versionLabel": item["version_label"],
|
||||
"mainCount": item["main_count"],
|
||||
"skuCount": item["sku_count"],
|
||||
"detailCount": item["detail_count"],
|
||||
"sourceCount": item["source_count"],
|
||||
"statusReason": item["status_reason"],
|
||||
}
|
||||
for item in grouped_items
|
||||
],
|
||||
"samples": aggregate["samples"],
|
||||
}
|
||||
)
|
||||
|
||||
write_csv(output_dir / "needs-review-cross-level-groups.csv", review_rows)
|
||||
(output_dir / "needs-review-cross-level-groups.json").write_text(
|
||||
json_dumps(details), encoding="utf-8"
|
||||
)
|
||||
|
||||
summary = {
|
||||
"runId": run_id,
|
||||
"dbPath": str(db_path),
|
||||
"needsReviewItems": len(items),
|
||||
"groups": len(review_rows),
|
||||
"decisionCounts": dict(Counter(row["decision"] for row in review_rows)),
|
||||
"outputCsv": str(output_dir / "needs-review-cross-level-groups.csv"),
|
||||
"outputJson": str(output_dir / "needs-review-cross-level-groups.json"),
|
||||
}
|
||||
(output_dir / "summary.json").write_text(json_dumps(summary), encoding="utf-8")
|
||||
|
||||
conn.close()
|
||||
print(json_dumps(summary))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
202
scripts/run-tianlan-remaining-batches.ps1
Normal file
202
scripts/run-tianlan-remaining-batches.ps1
Normal file
@ -0,0 +1,202 @@
|
||||
param(
|
||||
[int]$StartBatch = 2,
|
||||
[int]$EndBatch = 15,
|
||||
[string]$BatchDir = "D:\AI Code\gongxiangp2\output\tianlan-spu-ledger\remaining-batches-1778239500000",
|
||||
[string]$Operator = "222"
|
||||
)
|
||||
|
||||
$ErrorActionPreference = "Stop"
|
||||
Set-Location "D:\AI Code\gongxiangp2"
|
||||
|
||||
function Set-ImportEnvironment {
|
||||
$env:DATABASE_URL = ((Get-Content .env | Where-Object { $_ -match '^DATABASE_URL=' } | Select-Object -First 1) -replace '^DATABASE_URL=', '').Trim()
|
||||
$env:S3_ENDPOINT = "http://192.168.58.20:9000"
|
||||
$env:S3_BUCKET = "gongxiangpan"
|
||||
$env:S3_PUBLIC_DOMAIN = "https://s3.jiansun.vip/gongxiangpan"
|
||||
$env:S3_FILE_PATH = [System.Text.Encoding]::UTF8.GetString([System.Convert]::FromBase64String("55Sx5aO554K5QUkv"))
|
||||
$sshArgs = @(
|
||||
"-o", "BatchMode=yes",
|
||||
"-o", "ConnectTimeout=15",
|
||||
"-o", "ServerAliveInterval=15",
|
||||
"-o", "ServerAliveCountMax=2",
|
||||
"prod"
|
||||
)
|
||||
$env:S3_ACCESS_KEY_ID = Invoke-TextWithRetry "read S3_ACCESS_KEY_ID" { ssh @sshArgs "docker exec youyidian-3d-material printenv S3_ACCESS_KEY_ID" }
|
||||
$env:S3_SECRET_ACCESS_KEY = Invoke-TextWithRetry "read S3_SECRET_ACCESS_KEY" { ssh @sshArgs "docker exec youyidian-3d-material printenv S3_SECRET_ACCESS_KEY" }
|
||||
$env:HISTORY_IMPORT_REMOTE_UPLOADS_HOST = "prod"
|
||||
$env:HISTORY_IMPORT_REMOTE_UPLOADS_DIR = "/mnt/PoolOne/apps/youyidian-3d/uploads"
|
||||
}
|
||||
|
||||
function Get-NewestImportResult([datetime]$StartedAt) {
|
||||
Get-ChildItem -LiteralPath $BatchDir -Filter "history-material-import-result-*.json" |
|
||||
Where-Object { $_.Name -notlike "*.verification.json" -and $_.LastWriteTime -ge $StartedAt } |
|
||||
Sort-Object LastWriteTime -Descending |
|
||||
Select-Object -First 1
|
||||
}
|
||||
|
||||
function Write-GoalProgress($Payload) {
|
||||
$progressPath = Join-Path $BatchDir "goal-progress.json"
|
||||
$Payload | ConvertTo-Json -Depth 8 | Set-Content -LiteralPath $progressPath -Encoding UTF8
|
||||
}
|
||||
|
||||
function Invoke-NpxToLog([string[]]$Arguments, [string]$LogPath) {
|
||||
$previousErrorActionPreference = $ErrorActionPreference
|
||||
$ErrorActionPreference = "Continue"
|
||||
try {
|
||||
& npx @Arguments *> $LogPath
|
||||
return $LASTEXITCODE
|
||||
} finally {
|
||||
$script:ErrorActionPreference = $previousErrorActionPreference
|
||||
}
|
||||
}
|
||||
|
||||
function Invoke-TextWithRetry([string]$Label, [scriptblock]$Command, [int]$Retries = 3) {
|
||||
$lastOutput = $null
|
||||
for ($attempt = 1; $attempt -le $Retries; $attempt += 1) {
|
||||
$lastOutput = & $Command
|
||||
if ($LASTEXITCODE -eq 0 -and -not [string]::IsNullOrWhiteSpace(($lastOutput | Out-String))) {
|
||||
return ($lastOutput | Out-String).Trim()
|
||||
}
|
||||
|
||||
if ($attempt -lt $Retries) {
|
||||
Write-Host ("[Goal] {0} failed on attempt {1}/{2}; retrying..." -f $Label, $attempt, $Retries)
|
||||
Start-Sleep -Seconds ([Math]::Min(10, 2 * $attempt))
|
||||
}
|
||||
}
|
||||
|
||||
throw ("{0} failed after {1} attempts" -f $Label, $Retries)
|
||||
}
|
||||
|
||||
Set-ImportEnvironment
|
||||
|
||||
$runStartedAt = Get-Date
|
||||
$completed = @()
|
||||
|
||||
[pscustomobject]@{
|
||||
pid = $PID
|
||||
startBatch = $StartBatch
|
||||
endBatch = $EndBatch
|
||||
batchDir = $BatchDir
|
||||
startedAt = $runStartedAt.ToString("o")
|
||||
command = "scripts/run-tianlan-remaining-batches.ps1 -StartBatch $StartBatch -EndBatch $EndBatch"
|
||||
} | ConvertTo-Json -Depth 5 | Set-Content -LiteralPath (Join-Path $BatchDir "active-goal-process.json") -Encoding UTF8
|
||||
|
||||
for ($batchNumber = $StartBatch; $batchNumber -le $EndBatch; $batchNumber += 1) {
|
||||
$batchName = "batch-{0:D3}.json" -f $batchNumber
|
||||
$batchPath = Join-Path $BatchDir $batchName
|
||||
|
||||
if (-not (Test-Path -LiteralPath $batchPath)) {
|
||||
throw "Batch file not found: $batchPath"
|
||||
}
|
||||
|
||||
$stamp = Get-Date -Format "yyyyMMdd-HHmmss"
|
||||
$importLog = Join-Path $BatchDir ("batch-{0:D3}-{1}.import.log" -f $batchNumber, $stamp)
|
||||
$verifyLog = Join-Path $BatchDir ("batch-{0:D3}-{1}.verify.log" -f $batchNumber, $stamp)
|
||||
$startedAt = Get-Date
|
||||
|
||||
Write-Host ""
|
||||
Write-Host ("[Goal] Importing {0}" -f $batchName)
|
||||
Write-GoalProgress ([pscustomobject]@{
|
||||
mode = "tianlan-remaining-batches"
|
||||
runStartedAt = $runStartedAt.ToString("o")
|
||||
updatedAt = (Get-Date).ToString("o")
|
||||
startBatch = $StartBatch
|
||||
endBatch = $EndBatch
|
||||
completed = $completed
|
||||
current = [pscustomobject]@{
|
||||
batch = $batchName
|
||||
phase = "importing"
|
||||
importLog = Split-Path -Leaf $importLog
|
||||
verifyLog = Split-Path -Leaf $verifyLog
|
||||
startedAt = $startedAt.ToString("o")
|
||||
}
|
||||
})
|
||||
|
||||
$importExitCode = Invoke-NpxToLog @(
|
||||
"tsx",
|
||||
"scripts/import-history-materials.ts",
|
||||
"--input=$batchPath",
|
||||
"--operator=$Operator",
|
||||
"--apply"
|
||||
) $importLog
|
||||
$resultFile = Get-NewestImportResult $startedAt
|
||||
|
||||
if ($importExitCode -ne 0 -or -not $resultFile) {
|
||||
Write-Host ("[Goal] Import failed for {0}; log: {1}" -f $batchName, $importLog)
|
||||
if (Test-Path -LiteralPath $importLog) {
|
||||
Get-Content -LiteralPath $importLog -Tail 80
|
||||
}
|
||||
exit 1
|
||||
}
|
||||
|
||||
Write-Host ("[Goal] Result: {0}" -f $resultFile.Name)
|
||||
Write-GoalProgress ([pscustomobject]@{
|
||||
mode = "tianlan-remaining-batches"
|
||||
runStartedAt = $runStartedAt.ToString("o")
|
||||
updatedAt = (Get-Date).ToString("o")
|
||||
startBatch = $StartBatch
|
||||
endBatch = $EndBatch
|
||||
completed = $completed
|
||||
current = [pscustomobject]@{
|
||||
batch = $batchName
|
||||
phase = "validating"
|
||||
result = $resultFile.Name
|
||||
importLog = Split-Path -Leaf $importLog
|
||||
verifyLog = Split-Path -Leaf $verifyLog
|
||||
}
|
||||
})
|
||||
$verifyExitCode = Invoke-NpxToLog @(
|
||||
"tsx",
|
||||
"scripts/validate-history-import-result.ts",
|
||||
"--input=$($resultFile.FullName)",
|
||||
"--remote-host=prod",
|
||||
"--remote-uploads-dir=/mnt/PoolOne/apps/youyidian-3d/uploads"
|
||||
) $verifyLog
|
||||
$verificationPath = $resultFile.FullName -replace '\.json$', '.verification.json'
|
||||
$verification = Get-Content -LiteralPath $verificationPath -Raw | ConvertFrom-Json
|
||||
|
||||
$completed += [pscustomobject]@{
|
||||
batch = $batchName
|
||||
result = $resultFile.Name
|
||||
total = $verification.total
|
||||
imported = $verification.imported
|
||||
skipped = $verification.skipped
|
||||
failed = $verification.failed
|
||||
checked = $verification.checked
|
||||
ok = $verification.ok
|
||||
issueCount = $verification.issueCount
|
||||
importLog = Split-Path -Leaf $importLog
|
||||
verifyLog = Split-Path -Leaf $verifyLog
|
||||
finishedAt = (Get-Date).ToString("o")
|
||||
}
|
||||
|
||||
Write-GoalProgress ([pscustomobject]@{
|
||||
mode = "tianlan-remaining-batches"
|
||||
runStartedAt = $runStartedAt.ToString("o")
|
||||
updatedAt = (Get-Date).ToString("o")
|
||||
startBatch = $StartBatch
|
||||
endBatch = $EndBatch
|
||||
completed = $completed
|
||||
current = $completed[-1]
|
||||
})
|
||||
|
||||
Write-Host ("[Goal] Verified {0}: imported={1}, skipped={2}, failed={3}, ok={4}, issues={5}" -f $batchName, $verification.imported, $verification.skipped, $verification.failed, $verification.ok, $verification.issueCount)
|
||||
|
||||
if ($verifyExitCode -ne 0 -or $verification.failed -gt 0 -or $verification.issueCount -gt 0) {
|
||||
Write-Host ("[Goal] Stopped on verification issue for {0}; log: {1}" -f $batchName, $verifyLog)
|
||||
Get-Content -LiteralPath $verifyLog -Tail 120
|
||||
exit 1
|
||||
}
|
||||
}
|
||||
|
||||
Write-Host ""
|
||||
Write-Host "[Goal] All requested batches completed."
|
||||
Write-GoalProgress ([pscustomobject]@{
|
||||
mode = "tianlan-remaining-batches"
|
||||
runStartedAt = $runStartedAt.ToString("o")
|
||||
updatedAt = (Get-Date).ToString("o")
|
||||
startBatch = $StartBatch
|
||||
endBatch = $EndBatch
|
||||
completed = $completed
|
||||
done = $true
|
||||
})
|
||||
@ -16,9 +16,20 @@ NON_DIRECTORY_EXTENSIONS = IMAGE_EXTENSIONS | {
|
||||
}
|
||||
SKU_DIR_PATTERN = re.compile(r"sku", re.IGNORECASE)
|
||||
SPU_PATTERN = re.compile(r"^[A-Za-z]{1,5}\d{3,5}$")
|
||||
SPU_TOKEN_PATTERN = re.compile(r"([A-Za-z]{1,5}\d{3,5})")
|
||||
BRANDED_SPU_TOKEN_PATTERN = re.compile(r"([A-Za-z]{2,5}-[A-Za-z]{1,5}\d{3,5})")
|
||||
BRANDED_SKU_FILE_PATTERN = re.compile(r"([A-Za-z]{2,5})-([A-Za-z]{1,5})(\d{4,5})(\d{2})(?:\D|$)")
|
||||
KNOWN_SHOP_PREFIXES = {"GBPM", "GBSFY", "GBGJ", "GBJP", "XYL"}
|
||||
IGNORED_SPU_PREFIXES = {"TASK", "SKU", "IMG"}
|
||||
KNOWN_CATEGORY_CODES = {
|
||||
"SN", "DG", "XG", "BG", "SG", "CBG", "CZ", "SZ", "DNZ", "SJZ", "ZDZ",
|
||||
"CBZ", "CY", "AD", "SZD", "HXD", "SJ", "ZSJ", "ZWJ", "CFZW", "CFZWJ",
|
||||
"YMJ", "BJ", "CJ", "DSG", "CTG", "CYJ", "SZT", "XJ", "LRSF", "SMC",
|
||||
}
|
||||
IGNORED_PARTS = {"@eadir", "dingtalkappdata", ".recycle", "$nsec.recycle.bin"}
|
||||
DETAIL_DIR_KEYWORDS = ("images", "详情页", "详情")
|
||||
MAIN_DIR_KEYWORD = "主图"
|
||||
MAIN_DIMENSION_DIR_NAMES = {"750", "800"}
|
||||
|
||||
|
||||
def utc_now() -> str:
|
||||
@ -29,11 +40,93 @@ def normalize_spu(value: str) -> str:
|
||||
return value.strip().upper()
|
||||
|
||||
|
||||
def guess_spu(path_obj: Path) -> str:
|
||||
def looks_like_spu_token(value: str) -> bool:
|
||||
normalized = normalize_spu(value)
|
||||
if "-" in normalized:
|
||||
prefix, _, sku_part = normalized.partition("-")
|
||||
return prefix in KNOWN_SHOP_PREFIXES and bool(SPU_PATTERN.match(sku_part))
|
||||
|
||||
return bool(SPU_PATTERN.match(normalized)) and not any(
|
||||
normalized.startswith(prefix) for prefix in IGNORED_SPU_PREFIXES
|
||||
)
|
||||
|
||||
|
||||
def is_standalone_spu_match(text: str, start: int, end: int) -> bool:
|
||||
prev_char = text[start - 1] if start > 0 else ""
|
||||
next_char = text[end] if end < len(text) else ""
|
||||
if re.match(r"[A-Za-z0-9]", prev_char or ""):
|
||||
return False
|
||||
if not re.match(r"[A-Za-z0-9]", next_char or ""):
|
||||
return True
|
||||
return bool(re.match(r"K\d{1,2}(?![A-Za-z0-9])", text[end:], re.IGNORECASE))
|
||||
|
||||
|
||||
def get_sku_part(spu: str) -> str:
|
||||
normalized = normalize_spu(spu)
|
||||
return normalized.partition("-")[2] if "-" in normalized else normalized
|
||||
|
||||
|
||||
def has_known_category_code(spu: str) -> bool:
|
||||
sku_part = get_sku_part(spu)
|
||||
for code in sorted(KNOWN_CATEGORY_CODES, key=len, reverse=True):
|
||||
if sku_part.startswith(code) and len(sku_part) > len(code) and sku_part[len(code)].isdigit():
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def collect_spu_tokens(path_obj: Path):
|
||||
tokens = []
|
||||
seen = set()
|
||||
for part in reversed(path_obj.parts):
|
||||
matches = list(BRANDED_SPU_TOKEN_PATTERN.finditer(part))
|
||||
for match in reversed(matches):
|
||||
token = normalize_spu(match.group(1))
|
||||
if (
|
||||
token not in seen
|
||||
and looks_like_spu_token(token)
|
||||
and is_standalone_spu_match(part, match.start(), match.end())
|
||||
):
|
||||
tokens.append(token)
|
||||
seen.add(token)
|
||||
|
||||
for part in reversed(path_obj.parts):
|
||||
normalized = normalize_spu(part)
|
||||
if SPU_PATTERN.match(normalized):
|
||||
return normalized
|
||||
if normalized not in seen and looks_like_spu_token(normalized):
|
||||
tokens.append(normalized)
|
||||
seen.add(normalized)
|
||||
|
||||
matches = list(SPU_TOKEN_PATTERN.finditer(part))
|
||||
for match in reversed(matches):
|
||||
token = normalize_spu(match.group(1))
|
||||
if (
|
||||
token not in seen
|
||||
and looks_like_spu_token(token)
|
||||
and is_standalone_spu_match(part, match.start(), match.end())
|
||||
):
|
||||
tokens.append(token)
|
||||
seen.add(token)
|
||||
|
||||
return tokens
|
||||
|
||||
|
||||
def guess_spu(path_obj: Path) -> str:
|
||||
tokens = collect_spu_tokens(path_obj)
|
||||
for token in tokens:
|
||||
if has_known_category_code(token):
|
||||
return token
|
||||
if tokens:
|
||||
return tokens[0]
|
||||
return ""
|
||||
|
||||
|
||||
def guess_spu_from_file_names(file_names) -> str:
|
||||
for file_name in file_names:
|
||||
match = BRANDED_SKU_FILE_PATTERN.search(file_name)
|
||||
if not match:
|
||||
continue
|
||||
token = normalize_spu(f"{match.group(1)}-{match.group(2)}{match.group(3)}")
|
||||
if looks_like_spu_token(token):
|
||||
return token
|
||||
return ""
|
||||
|
||||
|
||||
@ -63,7 +156,7 @@ def is_detail_dir_name(name: str) -> bool:
|
||||
|
||||
def is_main_dir_name(name: str) -> bool:
|
||||
normalized = normalize_dir_name(name)
|
||||
return MAIN_DIR_KEYWORD in normalized
|
||||
return MAIN_DIR_KEYWORD in normalized or normalized in MAIN_DIMENSION_DIR_NAMES
|
||||
|
||||
|
||||
def list_sibling_dirs(parent_path: Path, current_dir_name: str):
|
||||
@ -161,12 +254,15 @@ def ensure_schema(conn: sqlite3.Connection) -> None:
|
||||
conn.commit()
|
||||
|
||||
|
||||
def scan_sku_dirs(root_path: Path):
|
||||
def scan_sku_dirs(root_path: Path, skip_repeated_root_name: bool = False):
|
||||
for current_root, dir_names, file_names in os.walk(root_path):
|
||||
current_path = Path(current_root)
|
||||
if should_ignore_path(current_path):
|
||||
dir_names[:] = []
|
||||
continue
|
||||
if skip_repeated_root_name and current_path == root_path:
|
||||
root_name = root_path.name.casefold()
|
||||
dir_names[:] = [name for name in dir_names if name.casefold() != root_name]
|
||||
|
||||
for dir_name in list(dir_names):
|
||||
if not matches_sku_dir(dir_name):
|
||||
@ -195,7 +291,7 @@ def scan_sku_dirs(root_path: Path):
|
||||
"directory_name": sku_dir_path.name,
|
||||
"parent_path": str(sku_dir_path.parent),
|
||||
"grandparent_path": str(sku_dir_path.parent.parent) if sku_dir_path.parent != root_path else "",
|
||||
"spu": guess_spu(sku_dir_path),
|
||||
"spu": guess_spu(sku_dir_path) or guess_spu_from_file_names(image_files),
|
||||
"depth": len(sku_dir_path.relative_to(root_path).parts),
|
||||
"file_count": len(files),
|
||||
"image_count": len(image_files),
|
||||
@ -259,21 +355,32 @@ def parse_args():
|
||||
default="/mnt/history-sc/大货营销策划图片",
|
||||
help="Root directory to scan.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--roots",
|
||||
default="",
|
||||
help="Multiple root directories separated by |. When provided, --root is ignored.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--db-path",
|
||||
default="/root/gongxiangp2/output/history-sku-folders.db",
|
||||
help="SQLite database file path.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--skip-repeated-root-name",
|
||||
action="store_true",
|
||||
help="Skip an immediate child directory whose name equals the scan root name.",
|
||||
)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def main() -> int:
|
||||
args = parse_args()
|
||||
root_path = Path(args.root)
|
||||
root_paths = [Path(item.strip()) for item in args.roots.split("|") if item.strip()] if args.roots else [Path(args.root)]
|
||||
db_path = Path(args.db_path)
|
||||
|
||||
if not root_path.exists():
|
||||
print(f"Root path not found: {root_path}", file=sys.stderr)
|
||||
missing_roots = [str(root_path) for root_path in root_paths if not root_path.exists()]
|
||||
if missing_roots:
|
||||
print(f"Root path not found: {missing_roots[0]}", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
db_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
@ -281,29 +388,35 @@ def main() -> int:
|
||||
conn = sqlite3.connect(str(db_path))
|
||||
try:
|
||||
ensure_schema(conn)
|
||||
scan_id = str(uuid.uuid4())
|
||||
started_at = utc_now()
|
||||
conn.execute(
|
||||
"INSERT INTO sku_folder_scans (scan_id, root_path, started_at) VALUES (?, ?, ?)",
|
||||
(scan_id, str(root_path), started_at),
|
||||
)
|
||||
conn.commit()
|
||||
all_previews = []
|
||||
grand_total = 0
|
||||
scan_ids = []
|
||||
for root_path in root_paths:
|
||||
scan_id = str(uuid.uuid4())
|
||||
scan_ids.append(scan_id)
|
||||
started_at = utc_now()
|
||||
conn.execute(
|
||||
"INSERT INTO sku_folder_scans (scan_id, root_path, started_at) VALUES (?, ?, ?)",
|
||||
(scan_id, str(root_path), started_at),
|
||||
)
|
||||
conn.commit()
|
||||
|
||||
rows = list(scan_sku_dirs(root_path))
|
||||
total_rows = insert_rows(conn, scan_id, rows)
|
||||
finished_at = utc_now()
|
||||
conn.execute(
|
||||
"UPDATE sku_folder_scans SET finished_at = ?, total_rows = ? WHERE scan_id = ?",
|
||||
(finished_at, total_rows, scan_id),
|
||||
)
|
||||
conn.commit()
|
||||
rows = list(scan_sku_dirs(root_path, args.skip_repeated_root_name))
|
||||
total_rows = insert_rows(conn, scan_id, rows)
|
||||
grand_total += total_rows
|
||||
finished_at = utc_now()
|
||||
conn.execute(
|
||||
"UPDATE sku_folder_scans SET finished_at = ?, total_rows = ? WHERE scan_id = ?",
|
||||
(finished_at, total_rows, scan_id),
|
||||
)
|
||||
conn.commit()
|
||||
all_previews.extend(rows[:3])
|
||||
|
||||
preview = rows[:12]
|
||||
print("SKU folder scan completed")
|
||||
print(f"scan_id={scan_id}")
|
||||
print(f"scan_ids={','.join(scan_ids)}")
|
||||
print(f"db_path={db_path}")
|
||||
print(f"total_rows={total_rows}")
|
||||
print(json.dumps(preview, ensure_ascii=False, indent=2))
|
||||
print(f"total_rows={grand_total}")
|
||||
print(json.dumps(all_previews[:12], ensure_ascii=False, indent=2))
|
||||
return 0
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
1282
scripts/scan_spu_ledger.py
Normal file
1282
scripts/scan_spu_ledger.py
Normal file
File diff suppressed because it is too large
Load Diff
274
scripts/validate-history-import-result.ts
Normal file
274
scripts/validate-history-import-result.ts
Normal file
@ -0,0 +1,274 @@
|
||||
import { execFile } from 'child_process'
|
||||
import { promises as fs } from 'fs'
|
||||
import path from 'path'
|
||||
import { promisify } from 'util'
|
||||
import { PrismaClient } from '@prisma/client'
|
||||
|
||||
const execFileAsync = promisify(execFile)
|
||||
const SSH_TIMEOUT_MS = Number(process.env.HISTORY_IMPORT_VALIDATE_SSH_TIMEOUT_MS || 30000)
|
||||
const SSH_RETRY_COUNT = Number(process.env.HISTORY_IMPORT_VALIDATE_SSH_RETRY_COUNT || 3)
|
||||
const SSH_ARGS = [
|
||||
'-o',
|
||||
'BatchMode=yes',
|
||||
'-o',
|
||||
'ConnectTimeout=15',
|
||||
'-o',
|
||||
'ServerAliveInterval=15',
|
||||
'-o',
|
||||
'ServerAliveCountMax=2',
|
||||
]
|
||||
|
||||
type ImportResultItem = {
|
||||
fingerprint: string
|
||||
spu: string
|
||||
sourcePath: string
|
||||
materialId: string | null
|
||||
status: string
|
||||
reason: string | null
|
||||
}
|
||||
|
||||
function parseArgs() {
|
||||
const args = process.argv.slice(2)
|
||||
const getArg = (name: string) => {
|
||||
const match = args.find((arg) => arg.startsWith(`--${name}=`))
|
||||
return match ? match.slice(name.length + 3) : ''
|
||||
}
|
||||
|
||||
const input = getArg('input')
|
||||
const remoteHost = getArg('remote-host') || process.env.HISTORY_IMPORT_REMOTE_UPLOADS_HOST || 'prod'
|
||||
const remoteUploadsDir =
|
||||
getArg('remote-uploads-dir') ||
|
||||
process.env.HISTORY_IMPORT_REMOTE_UPLOADS_DIR ||
|
||||
'/mnt/PoolOne/apps/youyidian-3d/uploads'
|
||||
|
||||
if (!input) {
|
||||
throw new Error('缺少 --input=导入结果 JSON')
|
||||
}
|
||||
|
||||
return { input, remoteHost, remoteUploadsDir }
|
||||
}
|
||||
|
||||
function shellQuote(value: string) {
|
||||
return `'${value.replace(/'/g, `'\\''`)}'`
|
||||
}
|
||||
|
||||
function storedPathToRemotePath(remoteUploadsDir: string, storedPath: string) {
|
||||
const normalized = String(storedPath || '').replace(/\\/g, '/')
|
||||
const relative = normalized.replace(/^\.?\/*uploads\//i, '')
|
||||
if (!relative || relative.includes('../')) {
|
||||
throw new Error(`下载文件路径无效: ${storedPath}`)
|
||||
}
|
||||
return path.posix.join(remoteUploadsDir.replace(/\/+$/, ''), relative)
|
||||
}
|
||||
|
||||
function sleep(ms: number) {
|
||||
return new Promise((resolve) => setTimeout(resolve, ms))
|
||||
}
|
||||
|
||||
async function getRemoteFileSize(remoteHost: string, remotePath: string) {
|
||||
let lastError: unknown
|
||||
for (let attempt = 1; attempt <= SSH_RETRY_COUNT; attempt += 1) {
|
||||
try {
|
||||
const { stdout } = await execFileAsync(
|
||||
'ssh',
|
||||
[
|
||||
...SSH_ARGS,
|
||||
remoteHost,
|
||||
`test -f ${shellQuote(remotePath)} && stat -c %s ${shellQuote(remotePath)}`,
|
||||
],
|
||||
{ timeout: SSH_TIMEOUT_MS },
|
||||
)
|
||||
return Number(String(stdout || '').trim())
|
||||
} catch (error) {
|
||||
lastError = error
|
||||
if (attempt < SSH_RETRY_COUNT) {
|
||||
await sleep(1000 * attempt)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
throw lastError
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const options = parseArgs()
|
||||
const prisma = new PrismaClient()
|
||||
|
||||
try {
|
||||
const raw = await fs.readFile(options.input, 'utf8')
|
||||
const results = JSON.parse(raw) as ImportResultItem[]
|
||||
const imported = results.filter((item) => item.status === 'imported' && item.materialId)
|
||||
const failures = results.filter((item) => item.status === 'failed')
|
||||
const skipped = results.filter((item) => item.status === 'skipped')
|
||||
const ids = imported.map((item) => item.materialId!).filter(Boolean)
|
||||
|
||||
const rows = await prisma.material.findMany({
|
||||
where: { id: { in: ids } },
|
||||
select: {
|
||||
id: true,
|
||||
productModel: true,
|
||||
previewUrl: true,
|
||||
zipFilePath: true,
|
||||
zipFileSize: true,
|
||||
isFullSet: true,
|
||||
deletedAt: true,
|
||||
},
|
||||
})
|
||||
const rowById = new Map(rows.map((row) => [row.id, row]))
|
||||
const sourceAssets = await prisma.materialAsset.findMany({
|
||||
where: {
|
||||
materialId: { in: ids },
|
||||
assetType: 'psd',
|
||||
},
|
||||
select: {
|
||||
materialId: true,
|
||||
fileUrl: true,
|
||||
fileName: true,
|
||||
fileSize: true,
|
||||
},
|
||||
})
|
||||
const sourceAssetsByMaterialId = new Map<string, typeof sourceAssets>()
|
||||
for (const asset of sourceAssets) {
|
||||
const current = sourceAssetsByMaterialId.get(asset.materialId) || []
|
||||
current.push(asset)
|
||||
sourceAssetsByMaterialId.set(asset.materialId, current)
|
||||
}
|
||||
const checks = []
|
||||
|
||||
for (const item of imported) {
|
||||
const row = rowById.get(item.materialId!)
|
||||
if (!row) {
|
||||
checks.push({
|
||||
materialId: item.materialId,
|
||||
spu: item.spu,
|
||||
ok: false,
|
||||
issue: '数据库记录不存在',
|
||||
})
|
||||
continue
|
||||
}
|
||||
|
||||
const remotePath = storedPathToRemotePath(options.remoteUploadsDir, row.zipFilePath || '')
|
||||
let remoteSize = -1
|
||||
try {
|
||||
remoteSize = await getRemoteFileSize(options.remoteHost, remotePath)
|
||||
} catch {
|
||||
remoteSize = -1
|
||||
}
|
||||
|
||||
let downloadStatus = 0
|
||||
try {
|
||||
const response = await fetch(
|
||||
`https://sc.jiansun.vip/api/materials/download?id=${row.id}&type=all&validate=1&share=1`,
|
||||
)
|
||||
downloadStatus = response.status
|
||||
} catch {
|
||||
downloadStatus = -1
|
||||
}
|
||||
|
||||
let previewStatus = 0
|
||||
try {
|
||||
const response = await fetch(row.previewUrl || '', { method: 'HEAD' })
|
||||
previewStatus = response.status
|
||||
} catch {
|
||||
previewStatus = -1
|
||||
}
|
||||
|
||||
const expectedSize = Number(row.zipFileSize || 0)
|
||||
const materialSourceAssets = sourceAssetsByMaterialId.get(row.id) || []
|
||||
const sourceChecks = []
|
||||
for (const asset of materialSourceAssets) {
|
||||
const sourcePathOk =
|
||||
String(asset.fileUrl || '').startsWith('uploads/materials/') &&
|
||||
!String(asset.fileUrl || '').includes('\\')
|
||||
const expectedSourceSize = Number(asset.fileSize || 0)
|
||||
let remoteSourceSize = -1
|
||||
|
||||
if (sourcePathOk) {
|
||||
try {
|
||||
remoteSourceSize = await getRemoteFileSize(
|
||||
options.remoteHost,
|
||||
storedPathToRemotePath(options.remoteUploadsDir, asset.fileUrl || ''),
|
||||
)
|
||||
} catch {
|
||||
remoteSourceSize = -1
|
||||
}
|
||||
}
|
||||
|
||||
sourceChecks.push({
|
||||
fileName: asset.fileName,
|
||||
sourcePathOk,
|
||||
remoteSize: remoteSourceSize,
|
||||
expectedSize: expectedSourceSize,
|
||||
ok: sourcePathOk && remoteSourceSize === expectedSourceSize,
|
||||
})
|
||||
}
|
||||
const sourceAssetsOk = sourceChecks.every((asset) => asset.ok)
|
||||
const previewOk = String(row.previewUrl || '').startsWith(
|
||||
'https://s3.jiansun.vip/gongxiangpan/由壹点AI/public-display/',
|
||||
)
|
||||
const zipPathOk =
|
||||
String(row.zipFilePath || '').startsWith('uploads/materials/') &&
|
||||
!String(row.zipFilePath || '').includes('\\')
|
||||
const ok =
|
||||
!row.deletedAt &&
|
||||
row.isFullSet &&
|
||||
previewOk &&
|
||||
zipPathOk &&
|
||||
sourceAssetsOk &&
|
||||
remoteSize === expectedSize &&
|
||||
downloadStatus === 200 &&
|
||||
previewStatus === 200
|
||||
|
||||
checks.push({
|
||||
materialId: row.id,
|
||||
spu: row.productModel,
|
||||
ok,
|
||||
previewOk,
|
||||
zipPathOk,
|
||||
sourceAssetCount: materialSourceAssets.length,
|
||||
sourceAssetsOk,
|
||||
sourceIssues: sourceChecks.filter((asset) => !asset.ok),
|
||||
remoteSize,
|
||||
expectedSize,
|
||||
downloadStatus,
|
||||
previewStatus,
|
||||
})
|
||||
}
|
||||
|
||||
const summary = {
|
||||
input: options.input,
|
||||
total: results.length,
|
||||
imported: imported.length,
|
||||
skipped: skipped.length,
|
||||
failed: failures.length,
|
||||
checked: checks.length,
|
||||
ok: checks.filter((item) => item.ok).length,
|
||||
issueCount: checks.filter((item) => !item.ok).length + failures.length,
|
||||
issues: [
|
||||
...failures.map((item) => ({
|
||||
materialId: item.materialId,
|
||||
spu: item.spu,
|
||||
ok: false,
|
||||
issue: item.reason || '导入失败',
|
||||
})),
|
||||
...checks.filter((item) => !item.ok),
|
||||
],
|
||||
checkedAt: new Date().toISOString(),
|
||||
}
|
||||
|
||||
const outputPath = options.input.replace(/\.json$/i, '.verification.json')
|
||||
await fs.writeFile(outputPath, JSON.stringify(summary, null, 2), 'utf8')
|
||||
console.log(JSON.stringify(summary, null, 2))
|
||||
|
||||
if (summary.failed > 0 || summary.issueCount > 0 || summary.imported + summary.skipped !== summary.total) {
|
||||
process.exitCode = 1
|
||||
}
|
||||
} finally {
|
||||
await prisma.$disconnect()
|
||||
}
|
||||
}
|
||||
|
||||
main().catch((error) => {
|
||||
console.error('导入结果验收失败:', error)
|
||||
process.exitCode = 1
|
||||
})
|
||||
Loading…
x
Reference in New Issue
Block a user