feat(storage): migrate project assets to Qiniu with verified legacy aliases
This commit is contained in:
134
parent
33d68b9ce1
commit
a9c0c8dff4
@ -786,28 +786,28 @@ const STARTER_CARDS = [
|
||||
title: "故事脚本生成",
|
||||
subtitle: "脚本到分镜",
|
||||
coverUrl:
|
||||
"https://demo-sparkai.oss-cn-shanghai.aliyuncs.com/video-canvas-demo/starter-story-script-cover.png",
|
||||
"https://scqnycdn.jiansun.vip/legacy-oss/demo-sparkai/video-canvas-demo/starter-story-script-cover.png",
|
||||
steps: ["故事脚本", "多模态文本模型Pro", "可灵 3.0", "预览"],
|
||||
},
|
||||
{
|
||||
title: "角色三视图",
|
||||
subtitle: "角色资产沉淀",
|
||||
coverUrl:
|
||||
"https://demo-sparkai.oss-cn-shanghai.aliyuncs.com/video-canvas-demo/starter-character-three-view-cover.png",
|
||||
"https://scqnycdn.jiansun.vip/legacy-oss/demo-sparkai/video-canvas-demo/starter-character-three-view-cover.png",
|
||||
steps: ["角色描述", "角色三视图", "全能图片模型V2", "保存"],
|
||||
},
|
||||
{
|
||||
title: "首帧图生视频",
|
||||
subtitle: "参考图到成片",
|
||||
coverUrl:
|
||||
"https://demo-sparkai.oss-cn-shanghai.aliyuncs.com/video-canvas-demo/starter-first-frame-video-cover.png",
|
||||
"https://scqnycdn.jiansun.vip/legacy-oss/demo-sparkai/video-canvas-demo/starter-first-frame-video-cover.png",
|
||||
steps: ["上传图片", "首帧图", "可灵 3.0", "预览"],
|
||||
},
|
||||
{
|
||||
title: "音频生视频",
|
||||
subtitle: "音频驱动画面",
|
||||
coverUrl:
|
||||
"https://demo-sparkai.oss-cn-shanghai.aliyuncs.com/video-canvas-demo/starter-audio-to-video-cover.png",
|
||||
"https://scqnycdn.jiansun.vip/legacy-oss/demo-sparkai/video-canvas-demo/starter-audio-to-video-cover.png",
|
||||
icon: "music",
|
||||
steps: ["上传音频", "音频主视频", "Wan 2.6", "预览"],
|
||||
},
|
||||
@ -822,7 +822,7 @@ const TOOLBOX_TEMPLATE_PRESETS: ToolboxTemplatePreset[] = [
|
||||
badge: "预设",
|
||||
category: "周星驰经典名场面",
|
||||
cover:
|
||||
"https://demo-sparkai.oss-cn-shanghai.aliyuncs.com/video-canvas-demo/starter-first-frame-video-cover.png",
|
||||
"https://scqnycdn.jiansun.vip/legacy-oss/demo-sparkai/video-canvas-demo/starter-first-frame-video-cover.png",
|
||||
description: "左弧滑行，镜头从人物左侧弧线推进，带轻微喜剧节奏和夸张表情停顿。",
|
||||
duration: "5s",
|
||||
id: "left-arc-slide",
|
||||
@ -837,7 +837,7 @@ const TOOLBOX_TEMPLATE_PRESETS: ToolboxTemplatePreset[] = [
|
||||
badge: "商用",
|
||||
category: "电商展示",
|
||||
cover:
|
||||
"https://demo-sparkai.oss-cn-shanghai.aliyuncs.com/video-canvas-demo/starter-character-three-view-cover.png",
|
||||
"https://scqnycdn.jiansun.vip/legacy-oss/demo-sparkai/video-canvas-demo/starter-character-three-view-cover.png",
|
||||
description: "电商手机弹出效果，产品从画面中心弹出并旋转定格，适合主图视频开场。",
|
||||
duration: "5s",
|
||||
id: "phone-pop-product",
|
||||
@ -852,7 +852,7 @@ const TOOLBOX_TEMPLATE_PRESETS: ToolboxTemplatePreset[] = [
|
||||
badge: "商用",
|
||||
category: "电商展示",
|
||||
cover:
|
||||
"https://demo-sparkai.oss-cn-shanghai.aliyuncs.com/video-canvas-demo/starter-story-script-cover.png",
|
||||
"https://scqnycdn.jiansun.vip/legacy-oss/demo-sparkai/video-canvas-demo/starter-story-script-cover.png",
|
||||
description: "咖啡杯出场，杯身从桌面滑入，蒸汽和柔光形成温暖的品牌氛围。",
|
||||
duration: "5s",
|
||||
id: "coffee-cup-entry",
|
||||
@ -867,7 +867,7 @@ const TOOLBOX_TEMPLATE_PRESETS: ToolboxTemplatePreset[] = [
|
||||
badge: "热门",
|
||||
category: "镜头运动",
|
||||
cover:
|
||||
"https://demo-sparkai.oss-cn-shanghai.aliyuncs.com/video-canvas-demo/starter-audio-to-video-cover.png",
|
||||
"https://scqnycdn.jiansun.vip/legacy-oss/demo-sparkai/video-canvas-demo/starter-audio-to-video-cover.png",
|
||||
description: "360旋转展示，围绕主体做一圈稳定环绕，适合产品、人物和空间展示。",
|
||||
duration: "6s",
|
||||
id: "orbit-360-showcase",
|
||||
@ -881,7 +881,7 @@ const TOOLBOX_TEMPLATE_PRESETS: ToolboxTemplatePreset[] = [
|
||||
author: "工业镜头",
|
||||
category: "镜头运动",
|
||||
cover:
|
||||
"https://demo-sparkai.oss-cn-shanghai.aliyuncs.com/video-canvas-demo/character-image.png",
|
||||
"https://scqnycdn.jiansun.vip/legacy-oss/demo-sparkai/video-canvas-demo/character-image.png",
|
||||
description: "机械臂视角，以第一视角穿过机械臂与产品线，增强科技感和空间纵深。",
|
||||
duration: "5s",
|
||||
id: "robot-arm-pov",
|
||||
@ -896,7 +896,7 @@ const TOOLBOX_TEMPLATE_PRESETS: ToolboxTemplatePreset[] = [
|
||||
badge: "NEW",
|
||||
category: "人物动画",
|
||||
cover:
|
||||
"https://demo-sparkai.oss-cn-shanghai.aliyuncs.com/video-canvas-demo/character-three-view.png",
|
||||
"https://scqnycdn.jiansun.vip/legacy-oss/demo-sparkai/video-canvas-demo/character-three-view.png",
|
||||
description: "Live 2D，让单张角色图产生眨眼、微笑和轻微身体摆动，适合口播开场。",
|
||||
duration: "4s",
|
||||
id: "live-2d-character",
|
||||
@ -910,7 +910,7 @@ const TOOLBOX_TEMPLATE_PRESETS: ToolboxTemplatePreset[] = [
|
||||
author: "电影运镜",
|
||||
category: "镜头运动",
|
||||
cover:
|
||||
"https://demo-sparkai.oss-cn-shanghai.aliyuncs.com/video-canvas-demo/starter-first-frame-video-cover.png",
|
||||
"https://scqnycdn.jiansun.vip/legacy-oss/demo-sparkai/video-canvas-demo/starter-first-frame-video-cover.png",
|
||||
description: "瞳孔拉近，镜头快速推向眼部并在反光里展示下一场景，适合悬疑转场。",
|
||||
duration: "5s",
|
||||
id: "pupil-zoom",
|
||||
@ -925,7 +925,7 @@ const TOOLBOX_TEMPLATE_PRESETS: ToolboxTemplatePreset[] = [
|
||||
badge: "热门",
|
||||
category: "人物动画",
|
||||
cover:
|
||||
"https://demo-sparkai.oss-cn-shanghai.aliyuncs.com/video-canvas-demo/starter-story-script-cover.png",
|
||||
"https://scqnycdn.jiansun.vip/legacy-oss/demo-sparkai/video-canvas-demo/starter-story-script-cover.png",
|
||||
description: "飞鸟解体，主体由细碎羽片散开再重组，适合视觉冲击强的短视频片头。",
|
||||
duration: "6s",
|
||||
id: "bird-dissolve",
|
||||
@ -940,7 +940,7 @@ const TOOLBOX_TEMPLATE_PRESETS: ToolboxTemplatePreset[] = [
|
||||
badge: "商用",
|
||||
category: "电商展示",
|
||||
cover:
|
||||
"https://demo-sparkai.oss-cn-shanghai.aliyuncs.com/video-canvas-demo/starter-character-three-view-cover.png",
|
||||
"https://scqnycdn.jiansun.vip/legacy-oss/demo-sparkai/video-canvas-demo/starter-character-three-view-cover.png",
|
||||
description: "商品震撼登场，产品带冲击波与局部高光进入画面，适合活动主推款。",
|
||||
duration: "5s",
|
||||
id: "product-hero-entry",
|
||||
@ -954,7 +954,7 @@ const TOOLBOX_TEMPLATE_PRESETS: ToolboxTemplatePreset[] = [
|
||||
author: "旅拍模板",
|
||||
category: "镜头运动",
|
||||
cover:
|
||||
"https://demo-sparkai.oss-cn-shanghai.aliyuncs.com/video-canvas-demo/starter-audio-to-video-cover.png",
|
||||
"https://scqnycdn.jiansun.vip/legacy-oss/demo-sparkai/video-canvas-demo/starter-audio-to-video-cover.png",
|
||||
description: "旅拍转场，前景遮挡切换到新地点，带自然手持感和轻微速度变化。",
|
||||
duration: "5s",
|
||||
id: "travel-whip-transition",
|
||||
@ -1299,19 +1299,19 @@ const NINE_GRID_CELL_VISUALS: GridCellVisual[] = [
|
||||
];
|
||||
|
||||
const TARGET_AVATAR =
|
||||
"https://demo-sparkai.oss-cn-shanghai.aliyuncs.com/video-canvas-demo/target-avatar.jpg";
|
||||
"https://scqnycdn.jiansun.vip/legacy-oss/demo-sparkai/video-canvas-demo/target-avatar.jpg";
|
||||
const CHARACTER_IMAGE_URL =
|
||||
"https://demo-sparkai.oss-cn-shanghai.aliyuncs.com/video-canvas-demo/character-image.png";
|
||||
"https://scqnycdn.jiansun.vip/legacy-oss/demo-sparkai/video-canvas-demo/character-image.png";
|
||||
const CHARACTER_THREE_VIEW_URL =
|
||||
"https://demo-sparkai.oss-cn-shanghai.aliyuncs.com/video-canvas-demo/character-three-view.png";
|
||||
"https://scqnycdn.jiansun.vip/legacy-oss/demo-sparkai/video-canvas-demo/character-three-view.png";
|
||||
const FIRST_FRAME_IMAGE_URL =
|
||||
"https://demo-sparkai.oss-cn-shanghai.aliyuncs.com/video-canvas-demo/first-frame-image.png";
|
||||
"https://scqnycdn.jiansun.vip/legacy-oss/demo-sparkai/video-canvas-demo/first-frame-image.png";
|
||||
const AUDIO_SOURCE_URL =
|
||||
"https://demo-sparkai.oss-cn-shanghai.aliyuncs.com/video-canvas-demo/audio-source.mp3";
|
||||
"https://scqnycdn.jiansun.vip/legacy-oss/demo-sparkai/video-canvas-demo/audio-source.mp3";
|
||||
const AUDIO_REFERENCE_IMAGE_URL =
|
||||
"https://demo-sparkai.oss-cn-shanghai.aliyuncs.com/video-canvas-demo/audio-reference-image.png";
|
||||
"https://scqnycdn.jiansun.vip/legacy-oss/demo-sparkai/video-canvas-demo/audio-reference-image.png";
|
||||
const PUBLISH_MODAL_BG_URL =
|
||||
"https://demo-sparkai.oss-cn-shanghai.aliyuncs.com/video-canvas-demo/publish-modal-bg.png";
|
||||
"https://scqnycdn.jiansun.vip/legacy-oss/demo-sparkai/video-canvas-demo/publish-modal-bg.png";
|
||||
const STORY_SCRIPT_PROMPT = "根据我上传的剧本生成一个完整的故事脚本";
|
||||
const AUDIO_TO_VIDEO_PROMPT =
|
||||
"根据上传的音频生成对应场景画面，镜头语言、节奏、音乐匹配情绪变化，电影级质感。";
|
||||
@ -1322,22 +1322,22 @@ const TEXT_TO_MUSIC_PROMPT =
|
||||
"生成一首现代品牌电子音乐，干净有力的低频贝斯，清晰电子鼓点，整体风格高级、未来感强。";
|
||||
const IMAGE_TO_PROMPT_PROMPT = "根据图片生成结构化中文提示词，包括主体描述、环境、光影、镜头语言、风格关键词。";
|
||||
const HD_IMAGE_SOURCE_URL =
|
||||
"https://demo-sparkai.oss-cn-shanghai.aliyuncs.com/video-canvas-demo/hd-image-source.png";
|
||||
"https://scqnycdn.jiansun.vip/legacy-oss/demo-sparkai/video-canvas-demo/hd-image-source.png";
|
||||
const VIDEO_FIRST_FRAME_SOURCE_URL =
|
||||
"https://demo-sparkai.oss-cn-shanghai.aliyuncs.com/video-canvas-demo/video-first-frame-source.png";
|
||||
"https://scqnycdn.jiansun.vip/legacy-oss/demo-sparkai/video-canvas-demo/video-first-frame-source.png";
|
||||
const VIDEO_FIRST_FRAME_PROMPT = "以当前图为首帧生成视频。";
|
||||
const VIDEO_FIRST_LAST_FIRST_URL =
|
||||
"https://demo-sparkai.oss-cn-shanghai.aliyuncs.com/video-canvas-demo/video-first-last-first.png";
|
||||
"https://scqnycdn.jiansun.vip/legacy-oss/demo-sparkai/video-canvas-demo/video-first-last-first.png";
|
||||
const VIDEO_FIRST_LAST_LAST_URL =
|
||||
"https://demo-sparkai.oss-cn-shanghai.aliyuncs.com/video-canvas-demo/video-first-last-last.png";
|
||||
"https://scqnycdn.jiansun.vip/legacy-oss/demo-sparkai/video-canvas-demo/video-first-last-last.png";
|
||||
const VIDEO_FIRST_LAST_PROMPT =
|
||||
"手从白帆布包中取出青柠气泡水罐的特写，过渡到短发女生在户外草地闭眼仰头喝同款气泡水的中景，日系清新治愈风格，明亮自然光，柔和暖调，胶片质感，动作流畅自然";
|
||||
const SCRIPT_VIDEO_REFERENCE_URL =
|
||||
"https://demo-sparkai.oss-cn-shanghai.aliyuncs.com/video-canvas-demo/script-video-reference.mp4";
|
||||
"https://scqnycdn.jiansun.vip/legacy-oss/demo-sparkai/video-canvas-demo/script-video-reference.mp4";
|
||||
const SCRIPT_CHARACTER_ONE_URL =
|
||||
"https://demo-sparkai.oss-cn-shanghai.aliyuncs.com/video-canvas-demo/script-character-one.png";
|
||||
"https://scqnycdn.jiansun.vip/legacy-oss/demo-sparkai/video-canvas-demo/script-character-one.png";
|
||||
const SCRIPT_CHARACTER_TWO_URL =
|
||||
"https://demo-sparkai.oss-cn-shanghai.aliyuncs.com/video-canvas-demo/script-character-two.png";
|
||||
"https://scqnycdn.jiansun.vip/legacy-oss/demo-sparkai/video-canvas-demo/script-character-two.png";
|
||||
const SCRIPT_CHARACTER_PROMPT =
|
||||
"根据角色参考图生成分镜脚本，围绕旧城区、机械修复、暴雨停电和被重新点亮的街道展开，突出人物关系、镜头画面和声音设计。";
|
||||
const SLASH_IMAGE_ITEMS = [
|
||||
@ -22937,8 +22937,8 @@ const STYLE_MARKET_CATEGORIES = [
|
||||
"全部",
|
||||
];
|
||||
|
||||
const STYLE_MARKET_THUMBNAIL_URL = "https://demo-sparkai.oss-cn-shanghai.aliyuncs.com/video-canvas-demo/starter-character-three-view-cover.png";
|
||||
const EFFECT_MARKET_THUMBNAIL_URL = "https://demo-sparkai.oss-cn-shanghai.aliyuncs.com/video-canvas-demo/starter-first-frame-video-cover.png";
|
||||
const STYLE_MARKET_THUMBNAIL_URL = "https://scqnycdn.jiansun.vip/legacy-oss/demo-sparkai/video-canvas-demo/starter-character-three-view-cover.png";
|
||||
const EFFECT_MARKET_THUMBNAIL_URL = "https://scqnycdn.jiansun.vip/legacy-oss/demo-sparkai/video-canvas-demo/starter-first-frame-video-cover.png";
|
||||
|
||||
const STYLE_MARKET_CARDS: AssetMarketCard[] = [
|
||||
["复古马卡龙", "算力潮汐", 177],
|
||||
|
||||
@ -526,13 +526,13 @@ export const defaultVideoCanvasMotionPresets: VideoCanvasMotionPreset[] = [
|
||||
|
||||
const defaultRoleImages: VideoCanvasRolePreset["images"] = {
|
||||
expression:
|
||||
"https://demo-sparkai.oss-cn-shanghai.aliyuncs.com/video-canvas-demo/character-image.png",
|
||||
"https://scqnycdn.jiansun.vip/legacy-oss/demo-sparkai/video-canvas-demo/character-image.png",
|
||||
face:
|
||||
"https://demo-sparkai.oss-cn-shanghai.aliyuncs.com/video-canvas-demo/character-image.png",
|
||||
"https://scqnycdn.jiansun.vip/legacy-oss/demo-sparkai/video-canvas-demo/character-image.png",
|
||||
full:
|
||||
"https://demo-sparkai.oss-cn-shanghai.aliyuncs.com/video-canvas-demo/character-image.png",
|
||||
"https://scqnycdn.jiansun.vip/legacy-oss/demo-sparkai/video-canvas-demo/character-image.png",
|
||||
threeView:
|
||||
"https://demo-sparkai.oss-cn-shanghai.aliyuncs.com/video-canvas-demo/character-three-view.png",
|
||||
"https://scqnycdn.jiansun.vip/legacy-oss/demo-sparkai/video-canvas-demo/character-three-view.png",
|
||||
};
|
||||
|
||||
function createDefaultRolePreset(
|
||||
|
||||
25
scripts/storage/QINIU_MIGRATION.md
Normal file
25
scripts/storage/QINIU_MIGRATION.md
Normal file
@ -0,0 +1,25 @@
|
||||
# 项目旧资源迁移到七牛
|
||||
|
||||
这组工具只复制应用代码和当前业务数据库引用的旧资源，不复制整桶。画布、聊天、媒体资产与业务历史记录在范围内；已删除项目的无引用文件，以及管理设置的历史审计备份，不用于扩大迁移范围。源桶保留，不删除源对象。
|
||||
|
||||
目标使用数据库中启用且默认的 `qiniu-kodo` 配置。原 OSS 资源保存在 `legacy-oss/demo-sparkai/`，PSD 的旧备用 S3 资源保存在 `legacy-s3/gongxiangpan/`。图片处理查询生成单独的 variants 对象，以保留原有裁剪、缩放结果。过期的 OSS 签名字段不保留。
|
||||
|
||||
先复制并验证文件长度及 MD5（流式备用传输验证 SHA256），再替换引用。替换前的配置和业务数据写入仅 root 可读的备份目录；业务更新比较旧值，保留迁移期间用户的并发编辑和画布内容修订时间。URL 形式的对象键与 `|||2K` 缓存后缀也按规则处理。
|
||||
|
||||
## 执行与复核
|
||||
|
||||
从应用根目录运行；密钥来自数据库和 `.env.local`，不作为命令行参数，也不写入仓库。默认工作目录为 `/var/lib/ai-chatbot/aliyun-qiniu-20261014`，可以用 `MIGRATION_RUN_DIR` 指定；`MIGRATION_APP_ROOT` 可指定应用目录。
|
||||
|
||||
1. `node scripts/storage/migrate-aliyun-to-qiniu.cjs inventory-all`：盘点业务引用。
|
||||
2. `node scripts/storage/inventory-aliyun-qiniu-source.cjs`：只读列举源桶，保存所需对象的大小及 ETag，避免每个文件单独请求源站头信息。
|
||||
3. `node scripts/storage/migrate-aliyun-to-qiniu.cjs migrate-settings`：迁移含旧 OSS 的当前设置。
|
||||
4. `node scripts/storage/migrate-aliyun-to-qiniu.cjs migrate-psd-extra`：补齐 PSD 模板和字体的旧备用 S3 依赖。
|
||||
5. `node scripts/storage/migrate-aliyun-to-qiniu.cjs migrate-all`：复制剩余业务引用，校验后替换地址。
|
||||
6. `node scripts/storage/register-aliyun-qiniu-aliases.cjs`：把已验证映射登记为现有归档系统的已完成记录。旧客户端再次保存时，可复用既有地址解析逻辑；不创建待处理归档任务，不覆盖既有任务。
|
||||
7. `node scripts/storage/audit-aliyun-qiniu-references.cjs`：复核业务引用。完成映射的来源地址是追溯信息，不是读取旧桶的依赖；审计器保留这一差别。
|
||||
|
||||
`MIGRATION_CONCURRENCY` 控制复制并发，最大 96，与模型生图并发无关。已验证对象通过 `verified.jsonl` 断点续跑。复制失败的对象保持旧地址，进程返回 2，需要核查失败记录；只有复制成功后才替换地址。`rewrite-all` 可复核已验证地址的替换，`MIGRATION_RESCAN=1` 会重新盘点业务表。`migrate-literals` 复制代码里的示例素材；代码地址修改仍需正常构建验证和部署。
|
||||
|
||||
`migrate-recovery` 从 `MIGRATION_RESOURCE_FILE` 指定的私有恢复清单复制已有备份，只接受配置中登记的存储来源。成功恢复的文件仍要校验。源文件不存在或为空且无有效备份时，保留记录，不能把不存在的数据宣称为迁移成功。
|
||||
|
||||
验证：`node --test scripts/storage/migrate-aliyun-to-qiniu.test.cjs`。实际迁移还验证了数据库并发更新保护、主图/SKU PSD 导出、两站上传与下载分段/CORS，以及旧画布保存时自动解析到七牛。与资源地址无关的模型、上传和聊天代码保持原样。
|
||||
6
scripts/storage/audit-aliyun-qiniu-references.cjs
Normal file
6
scripts/storage/audit-aliyun-qiniu-references.cjs
Normal file
@ -0,0 +1,6 @@
|
||||
/* Read-only audit of live business references; obsolete admin configuration backups stay intact. */
|
||||
const fs=require('fs'),path=require('path');const root=process.env.MIGRATION_APP_ROOT||process.cwd(),dir=process.env.MIGRATION_RUN_DIR||'/var/lib/ai-chatbot/aliyun-qiniu-20261014';const {urlsIn}=require('./migrate-aliyun-to-qiniu.cjs');require(path.join(root,'node_modules/dotenv')).config({path:path.join(root,'.env.local')});const sql=require(path.join(root,'node_modules/postgres'))(process.env.POSTGRES_URL,{max:1,prepare:false,connection:{statement_timeout:600000}});const host='demo-sparkai.oss-cn-shanghai.aliyuncs.com',quote=s=>'"'+s.replaceAll('"','""')+'"';
|
||||
(async()=>{try{const verified=new Set(fs.readFileSync(path.join(dir,'verified.jsonl'),'utf8').split('\n').filter(Boolean).map(s=>JSON.parse(s).id)),unavailable=new Set(fs.existsSync(path.join(dir,'unavailable-source.json'))?JSON.parse(fs.readFileSync(path.join(dir,'unavailable-source.json'))).map(r=>r.id):[]),remaining=new Map(),tables=new Map(),excluded=new Set(['StorageConfig','StorageMigrationTask','ImageArchiveJob','__drizzle_migrations','SystemSettingHistory','SystemSettingSnapshot']);
|
||||
const columns=await sql`select c.table_name as "tableName",c.column_name as "columnName" from information_schema.columns c join information_schema.tables t on c.table_schema=t.table_schema and c.table_name=t.table_name where c.table_schema='public' and t.table_type='BASE TABLE' and c.data_type in ('text','character varying','json','jsonb') order by c.table_name,c.ordinal_position`;
|
||||
for(const c of columns)if(!excluded.has(c.tableName)){if(!tables.has(c.tableName))tables.set(c.tableName,[]);tables.get(c.tableName).push(c.columnName);}const tableCounts={};for(const [table,cols] of tables){const where=cols.map(c=>'('+quote(c)+'::text like $1'+(table==='ImageArchiveTask'&&c==='sourceUrl'?" and coalesce(metadata->>'legacyOssMigrationRun','') <> '20261014'":'')+')').join(' or ');let matched=0;for await(const rows of sql.unsafe('select '+cols.map(quote).join(',')+' from '+quote(table)+' where '+where,['%'+host+'/%']).cursor(50))for(const row of rows){matched++;for(const [column,value]of Object.entries(row)){if(table==='ImageArchiveTask'&&column==='sourceUrl'&&row.metadata?.legacyOssMigrationRun==='20261014')continue;for(const r of urlsIn(value))if(r.sourceHost===host){if(!remaining.has(r.id))remaining.set(r.id,{...r,verified:verified.has(r.id),knownUnavailable:unavailable.has(r.id),locations:{}});const item=remaining.get(r.id),key=table+'.'+column;item.locations[key]=(item.locations[key]||0)+1;}}}if(matched){tableCounts[table]=matched;console.log('remaining_reference_rows',JSON.stringify({table,rows:matched}));}}
|
||||
const resources=[...remaining.values()],summary={auditedAt:new Date().toISOString(),remainingUniqueResources:resources.length,verifiedPendingRewrite:resources.filter(r=>r.verified).length,knownUnavailable:resources.filter(r=>r.knownUnavailable&&!r.verified).length,unclassified:resources.filter(r=>!r.knownUnavailable&&!r.verified).length,tableCounts};fs.writeFileSync(path.join(dir,'reference-audit-resources.json'),JSON.stringify(resources),{mode:0o600});fs.writeFileSync(path.join(dir,'reference-audit-summary.json'),JSON.stringify(summary,null,2),{mode:0o600});console.log('reference_audit',JSON.stringify(summary));}finally{await sql.end()}})().catch(e=>{console.error('audit_failed',e.name,String(e.message).replace(/https?:\/\/\S+/g,'[URL]').slice(0,180));process.exitCode=1});
|
||||
8
scripts/storage/inventory-aliyun-qiniu-source.cjs
Normal file
8
scripts/storage/inventory-aliyun-qiniu-source.cjs
Normal file
@ -0,0 +1,8 @@
|
||||
const fs=require('fs'),path=require('path'),root=process.env.MIGRATION_APP_ROOT||process.cwd(),dir=process.env.MIGRATION_RUN_DIR||'/var/lib/ai-chatbot/aliyun-qiniu-20261014';
|
||||
require(path.join(root,'node_modules/dotenv')).config({path:path.join(root,'.env.local')});const sql=require(path.join(root,'node_modules/postgres'))(process.env.POSTGRES_URL,{max:1,prepare:false});const {S3Client,ListObjectsV2Command}=require(path.join(root,'node_modules/@aws-sdk/client-s3'));
|
||||
(async()=>{let client;try{const complete=path.join(dir,'source-metadata.complete');if(fs.existsSync(complete)){console.log('Source metadata inventory already completed');return;}const plan=JSON.parse(fs.readFileSync(path.join(dir,'plan-all.json'))),wanted=new Set(plan.filter(r=>!r.variant).map(r=>r.sourceKey));
|
||||
const [s]=await sql`select * from "StorageConfig" where bucket='demo-sparkai' limit 1`;client=new S3Client({endpoint:s.endpoint,region:s.region,forcePathStyle:false,credentials:{accessKeyId:s.accessKeyId,secretAccessKey:s.secretAccessKey},maxAttempts:3});let marker,listed=0,matched=0;const checkpointFile=path.join(dir,'source-list-checkpoint.json'),file=path.join(dir,'source-metadata.jsonl');if(fs.existsSync(checkpointFile)){const c=JSON.parse(fs.readFileSync(checkpointFile));marker=c.marker;listed=c.listed;matched=c.matched;}
|
||||
for(;;){const r=await client.send(new ListObjectsV2Command({Bucket:s.bucket,MaxKeys:1000,ContinuationToken:marker}),{abortSignal:AbortSignal.timeout(60000)});const lines=[];for(const o of r.Contents||[]){listed++;if(wanted.has(o.Key)){lines.push(JSON.stringify({key:o.Key,size:o.Size,md5:String(o.ETag||'').replaceAll('"','').toLowerCase()}));matched++;}}
|
||||
if(lines.length)fs.appendFileSync(file,lines.join('\n')+'\n',{mode:0o600});marker=r.NextContinuationToken;fs.writeFileSync(checkpointFile,JSON.stringify({marker,listed,matched}),{mode:0o600});if(listed%25000<1000)console.log(JSON.stringify({listed,matched,wanted:wanted.size}));if(!r.IsTruncated)break;}
|
||||
fs.writeFileSync(complete,JSON.stringify({listed,matched,wanted:wanted.size,completedAt:new Date().toISOString()}),{mode:0o600});console.log('Source inventory complete',JSON.stringify({listed,matched,wanted:wanted.size}));
|
||||
}finally{client?.destroy();await sql.end()}})().catch(e=>{console.error('source_inventory_failed',e.name,String(e.message).replace(/https?:\/\/\S+/g,'[URL]').slice(0,180));process.exitCode=1});
|
||||
220
scripts/storage/migrate-aliyun-to-qiniu.cjs
Normal file
220
scripts/storage/migrate-aliyun-to-qiniu.cjs
Normal file
@ -0,0 +1,220 @@
|
||||
/* Resumable, verified migration. No source objects are deleted. */
|
||||
const fs = require('node:fs');
|
||||
const path = require('node:path');
|
||||
const crypto = require('node:crypto');
|
||||
const { Readable, Transform } = require('node:stream');
|
||||
const SOURCE_HOST = 'demo-sparkai.oss-cn-shanghai.aliyuncs.com';
|
||||
const URL_PATTERN = /https?:\/\/(?:demo-sparkai\.oss-cn-shanghai\.aliyuncs\.com\/|s3\.jiansun\.vip\/gongxiangpan\/)[^\s"'<>`\\)\]}|]+/gi;
|
||||
function resourceFor(url) {
|
||||
const u = new URL(url);
|
||||
const isPrimary=u.hostname===SOURCE_HOST,isPsdBackup=u.hostname==='s3.jiansun.vip'&&u.pathname.startsWith('/gongxiangpan/');
|
||||
if ((!isPrimary&&!isPsdBackup) || u.pathname === '/') return null;
|
||||
for (const name of ['Signature', 'OSSAccessKeyId', 'Expires', 'security-token']) u.searchParams.delete(name);
|
||||
u.protocol = 'https:'; u.hash = ''; u.searchParams.sort();
|
||||
const identity = u.toString(), sourceKey = decodeURIComponent(u.pathname.slice(isPsdBackup?'/gongxiangpan/'.length:1));
|
||||
if (!sourceKey || sourceKey.split('/').some(p => p === '..')) throw Error('Invalid object key');
|
||||
const id = crypto.createHash('sha256').update(identity).digest('hex');
|
||||
const namespace=isPrimary?'legacy-oss/demo-sparkai/':'legacy-s3/gongxiangpan/';
|
||||
const targetKey = u.search ? namespace+'variants/' + id + path.extname(sourceKey) : namespace + sourceKey;
|
||||
return { id, identity, sourceHost:u.hostname, sourceKey, targetKey, variant: Boolean(u.search) };
|
||||
}
|
||||
function urlsIn(value) {
|
||||
const found = new Map();
|
||||
function walk(v) {
|
||||
if (typeof v === 'string') for (const url of v.match(URL_PATTERN) || []) { const r = resourceFor(url); if (r) found.set(r.id, r); }
|
||||
else if (Array.isArray(v)) v.forEach(walk);
|
||||
else if (v && typeof v === 'object') Object.entries(v).forEach(([k,val])=>{walk(k);walk(val);});
|
||||
}
|
||||
walk(value); return [...found.values()];
|
||||
}
|
||||
function replaceVerified(value, verified) {
|
||||
if (typeof value === 'string') return value.replace(URL_PATTERN, url => { const r = resourceFor(url); return r && verified.get(r.id)?.targetUrl || url; });
|
||||
if (Array.isArray(value)) return value.map(v => replaceVerified(v, verified));
|
||||
if (value && typeof value === 'object') return Object.fromEntries(Object.entries(value).map(([k, v]) => [replaceVerified(k, verified), replaceVerified(v, verified)]));
|
||||
return value;
|
||||
}
|
||||
function quote(s) { return '"' + s.replaceAll('"', '""') + '"'; }
|
||||
async function rewriteBatch(sql, column, pk, pkType, rows, verified, backup) {
|
||||
const isJson=['json','jsonb'].includes(column.dataType);
|
||||
const changes=[],originals=[];
|
||||
for(const row of rows){
|
||||
const next=replaceVerified(row.value,verified);
|
||||
if(JSON.stringify(next)===JSON.stringify(row.value))continue;
|
||||
originals.push({table:column.tableName,column:column.columnName,pkColumn:pk,pk:row.pk,value:row.value});
|
||||
changes.push({pk:String(row.pk),oldvalue:isJson?JSON.stringify(row.value):row.value,newvalue:isJson?JSON.stringify(next):next});
|
||||
}
|
||||
if(!changes.length)return 0;
|
||||
backup(originals);
|
||||
const cast=isJson?'::'+column.dataType:'';
|
||||
const updated=await sql.unsafe('update '+quote(column.tableName)+' as t set '+quote(column.columnName)+'=v.newvalue'+cast+' from jsonb_to_recordset($1::jsonb) as v(pk text,oldvalue text,newvalue text) where t.'+quote(pk)+'=v.pk::'+pkType+' and t.'+quote(column.columnName)+(isJson?'::jsonb=v.oldvalue::jsonb':'=v.oldvalue')+' returning t.'+quote(pk),[sql.json(changes)]);
|
||||
return updated.length;
|
||||
}
|
||||
module.exports = { resourceFor, urlsIn, replaceVerified, rewriteBatch };
|
||||
if (require.main === module) main().catch(e => { console.error('migration_failed', e.name, String(e.message).replace(/https?:\/\/\S+/g, '[URL]').slice(0,240)); process.exitCode = 1; });
|
||||
async function main() {
|
||||
const root = process.env.MIGRATION_APP_ROOT || process.cwd();
|
||||
require(path.join(root, 'node_modules/dotenv')).config({ path: path.join(root, '.env.local') });
|
||||
const postgres = require(path.join(root, 'node_modules/postgres'));
|
||||
const { S3Client, PutObjectCommand, GetObjectCommand, HeadObjectCommand } = require(path.join(root, 'node_modules/@aws-sdk/client-s3'));
|
||||
const sql = postgres(process.env.POSTGRES_URL, { max: 2, prepare: false, connection: { statement_timeout: 600000 } });
|
||||
const dir = process.env.MIGRATION_RUN_DIR || '/var/lib/ai-chatbot/aliyun-qiniu-20261014';
|
||||
fs.mkdirSync(dir, { recursive: true, mode: 0o700 });
|
||||
const journal = path.join(dir, 'verified.jsonl'), verified = new Map();
|
||||
if (fs.existsSync(journal)) for (const line of fs.readFileSync(journal,'utf8').split('\n').filter(Boolean)) { const r = JSON.parse(line); verified.set(r.id,r); }
|
||||
let sourceClient, targetClient;
|
||||
try {
|
||||
const configs = await sql`select * from "StorageConfig"`;
|
||||
const source = configs.find(c => c.bucket === 'demo-sparkai' && String(c.endpoint).includes('aliyuncs.com'));
|
||||
const target = configs.find(c => c.type === 'qiniu-kodo' && c.isEnabled && c.isDefault);
|
||||
if (!source || !target || !target.publicUrl) throw Error('Expected source and default Qiniu configuration unavailable');
|
||||
function client(c) { return new S3Client({ endpoint: c.endpoint, region: c.region || 'auto', forcePathStyle: true, credentials: { accessKeyId:c.accessKeyId,secretAccessKey:c.secretAccessKey }, requestChecksumCalculation:'WHEN_REQUIRED',responseChecksumValidation:'WHEN_REQUIRED', maxAttempts: 2 }); }
|
||||
sourceClient = client(source); targetClient = client(target);
|
||||
const publicUrl = key => target.publicUrl.replace(/\/$/,'') + '/' + key.split('/').map(encodeURIComponent).join('/');
|
||||
const command = process.argv[2] || 'audit';
|
||||
const settings = command==='migrate-psd-extra'?await sql`select id,key,value from "SystemSettings" where key in ('agent_psd_templates','agent_psd_fonts')`:await sql`select id,key,value from "SystemSettings" where value::text like ${'%'+SOURCE_HOST+'/%'}`;
|
||||
const settingsResources = urlsIn(settings.map(s => s.value));
|
||||
console.log('settings_inventory', JSON.stringify({ rows: settings.length, keys: settings.map(s => s.key), uniqueObjects:settingsResources.length, alreadyVerified:verified.size }));
|
||||
if (command === 'audit') return;
|
||||
const resources = new Map(settingsResources.map(r => [r.id,r]));
|
||||
const scope = command.endsWith('-all') ? 'all' : command==='migrate-psd-extra'?'psd-extra':command==='migrate-recovery'?'recovery':command==='migrate-literals'?'literals':'settings';
|
||||
if(command==='migrate-recovery'){
|
||||
resources.clear();
|
||||
for(const candidate of JSON.parse(fs.readFileSync(process.env.MIGRATION_RESOURCE_FILE,'utf8'))){
|
||||
const canonical=resourceFor(candidate.identity);
|
||||
if(!canonical||canonical.id!==candidate.id)throw Error('Invalid recovery identity');
|
||||
const backup=new URL(candidate.fetchUrl);
|
||||
if(!configs.some(c=>c.publicUrl&&backup.origin===new URL(c.publicUrl).origin&&backup.pathname.startsWith(new URL(c.publicUrl).pathname.replace(/\/$/,'')+'/')))throw Error('Recovery source is not a configured storage origin');
|
||||
resources.set(canonical.id,{...canonical,fetchUrl:candidate.fetchUrl});
|
||||
}
|
||||
}
|
||||
let columns = [];
|
||||
if (scope === 'all') {
|
||||
columns = await sql`select c.table_name as "tableName",c.column_name as "columnName",c.data_type as "dataType" from information_schema.columns c join information_schema.tables t on c.table_schema=t.table_schema and c.table_name=t.table_name where c.table_schema='public' and t.table_type='BASE TABLE' and c.is_generated='NEVER' and c.is_updatable='YES' and c.data_type in ('text','character varying','json','jsonb') order by c.table_name,c.ordinal_position`;
|
||||
const excluded = new Set(['StorageConfig','StorageMigrationTask','ImageArchiveJob','__drizzle_migrations','SystemSettings','SystemSettingHistory','SystemSettingSnapshot']);
|
||||
const tables = new Map();
|
||||
const inventoryJournal=path.join(dir,'inventory.jsonl'), doneFile=path.join(dir,'inventory-completed-tables.json');
|
||||
const completedTables=new Set(fs.existsSync(doneFile)?JSON.parse(fs.readFileSync(doneFile,'utf8')):[]);
|
||||
if(fs.existsSync(inventoryJournal))for(const line of fs.readFileSync(inventoryJournal,'utf8').split('\n').filter(Boolean)){const r=JSON.parse(line);for(const canonical of urlsIn(r.identity))resources.set(canonical.id,canonical);}
|
||||
for (const c of columns) if (!excluded.has(c.tableName)) { if (!tables.has(c.tableName)) tables.set(c.tableName,[]); tables.get(c.tableName).push(c); }
|
||||
for (const [table, cols] of tables) {
|
||||
if(completedTables.has(table)&&process.env.MIGRATION_RESCAN!=='1')continue;
|
||||
console.log('scan_start',table);
|
||||
const where = cols.map(c => quote(c.columnName)+'::text like $1').join(' or ');
|
||||
let matched=0;
|
||||
for await (const rows of sql.unsafe('select '+cols.map(c=>quote(c.columnName)).join(',')+' from '+quote(table)+' where '+where,['%'+SOURCE_HOST+'/%']).cursor(50)){const added=[];for (const row of rows) { matched++; for (const r of urlsIn(row)) if(!resources.has(r.id)){resources.set(r.id,r);added.push(r);} }if(added.length)fs.appendFileSync(inventoryJournal,added.map(r=>JSON.stringify(r)).join('\n')+'\n',{mode:0o600});}
|
||||
completedTables.add(table);fs.writeFileSync(doneFile,JSON.stringify([...completedTables]),{mode:0o600});
|
||||
if (matched) console.log('database_inventory',JSON.stringify({table,rows:matched,uniqueObjects:resources.size}));
|
||||
}
|
||||
}
|
||||
if(scope==='all'||scope==='literals'){
|
||||
if(scope==='literals')resources.clear();
|
||||
for (const folder of ['app','components','lib']) {
|
||||
function walk(d) { for (const e of fs.readdirSync(d,{withFileTypes:true})) { const f=path.join(d,e.name); if(e.isDirectory())walk(f); else if(/\.(tsx?|jsx?)$/.test(f)) for(const r of urlsIn(fs.readFileSync(f,'utf8')))resources.set(r.id,r); } }
|
||||
walk(path.join(root,folder));
|
||||
}
|
||||
}
|
||||
fs.writeFileSync(path.join(dir,'plan-'+scope+'.json'),JSON.stringify([...resources.values()]),{mode:0o600});
|
||||
if (command === 'inventory-all') return;
|
||||
if (!['migrate-settings','migrate-all','migrate-psd-extra','migrate-recovery','migrate-literals','rewrite-all'].includes(command)) throw Error('Unknown command');
|
||||
if (scope === 'settings' && !fs.existsSync(path.join(dir,'settings-before.json'))) fs.writeFileSync(path.join(dir,'settings-before.json'),JSON.stringify(settings),{mode:0o600,flag:'wx'});
|
||||
const sourceMetadata=new Map(); const metadataFile=path.join(dir,'source-metadata.jsonl');
|
||||
if(fs.existsSync(metadataFile))for(const line of fs.readFileSync(metadataFile,'utf8').split('\n').filter(Boolean)){const r=JSON.parse(line);sourceMetadata.set(r.key,r);}
|
||||
const base64url = value => Buffer.from(value).toString('base64').replace(/\+/g,'-').replace(/\//g,'_');
|
||||
async function cloudCopy(r) {
|
||||
const fetchUrl=r.fetchUrl||r.identity;
|
||||
const cached=r.fetchUrl||r.variant||new URL(r.identity).hostname!==SOURCE_HOST?null:sourceMetadata.get(r.sourceKey);
|
||||
let size=cached?.size,md5=cached?.md5;
|
||||
if(!r.variant&&!cached){const head=await fetch(fetchUrl,{method:'HEAD',headers:{'Accept-Encoding':'identity'},signal:AbortSignal.timeout(45000)});if(!head.ok)throw Error('Source HTTP '+head.status);size=Number(head.headers.get('content-length'));md5=String(head.headers.get('etag')||'').replaceAll('"','').toLowerCase();}
|
||||
if(r.variant || !size || !/^[a-f0-9]{32}$/.test(md5||'')) {
|
||||
const download=await fetch(fetchUrl,{headers:{'Accept-Encoding':'identity'},signal:AbortSignal.timeout(15*60*1000)});
|
||||
if(!download.ok)throw Error('Source HTTP '+download.status);
|
||||
const hash=crypto.createHash('md5');size=0;for await(const chunk of Readable.fromWeb(download.body)){hash.update(chunk);size+=chunk.length;}md5=hash.digest('hex');
|
||||
}
|
||||
if(!size)throw Error('Empty source object');
|
||||
const requestPath='/fetch/'+base64url(fetchUrl)+'/to/'+base64url(target.bucket+':'+r.targetKey);
|
||||
const signature=base64url(crypto.createHmac('sha1',target.secretAccessKey).update(requestPath+'\n').digest());
|
||||
const fetchResult=await fetch('https://iovip.qbox.me'+requestPath,{method:'POST',headers:{Authorization:'QBox '+target.accessKeyId+':'+signature,'Content-Type':'application/x-www-form-urlencoded'},signal:AbortSignal.timeout(15*60*1000)});
|
||||
if(!fetchResult.ok)throw Error('Qiniu cloud fetch HTTP '+fetchResult.status);
|
||||
const info=await fetchResult.json();if(Number(info.fsize)!==size)throw Error('Cloud target size mismatch');
|
||||
const targetUrl=publicUrl(r.targetKey);
|
||||
const hashResponse=await fetch(targetUrl+'?qhash/md5',{signal:AbortSignal.timeout(45000)});
|
||||
if(!hashResponse.ok)throw Error('CDN hash verification HTTP '+hashResponse.status);
|
||||
const targetDigest=await hashResponse.json();
|
||||
if(targetDigest.hash!==md5 || Number(targetDigest.fsize)!==size)throw Error('Cloud target checksum mismatch');
|
||||
const record={...r,targetUrl,size,md5,transport:'qiniu-cloud-fetch',verifiedAt:new Date().toISOString()};fs.appendFileSync(journal,JSON.stringify(record)+'\n',{mode:0o600});verified.set(r.id,record);
|
||||
}
|
||||
async function copy(r) {
|
||||
if (verified.has(r.id)) return;
|
||||
let last;
|
||||
for (let attempt=1;attempt<=3;attempt++) {
|
||||
let input;
|
||||
try {
|
||||
if(process.env.MIGRATION_TRANSPORT!=='stream'){await cloudCopy(r);return;}
|
||||
const timeout=AbortSignal.timeout(15*60*1000);
|
||||
const response=await fetch(r.identity,{signal:timeout,headers:{'Accept-Encoding':'identity'}});
|
||||
let size,type,disposition;
|
||||
if (response.ok) { size=Number(response.headers.get('content-length'));type=response.headers.get('content-type');disposition=response.headers.get('content-disposition'); input=Readable.fromWeb(response.body); }
|
||||
else {
|
||||
await response.body?.cancel();
|
||||
if(r.variant)throw Error('Source variant HTTP '+response.status);
|
||||
const get=await sourceClient.send(new GetObjectCommand({Bucket:source.bucket,Key:r.sourceKey}),{abortSignal:timeout});input=get.Body;size=get.ContentLength;type=get.ContentType;disposition=get.ContentDisposition;
|
||||
}
|
||||
if (!size || size>5*1024**3)throw Error('Unsupported source size '+size);
|
||||
const hash=crypto.createHash('sha256');let readBytes=0;
|
||||
const hashStream=new Transform({transform(chunk,enc,cb){hash.update(chunk);readBytes+=chunk.length;cb(null,chunk)}});
|
||||
input.on('error',e=>hashStream.destroy(e));
|
||||
await targetClient.send(new PutObjectCommand({Bucket:target.bucket,Key:r.targetKey,Body:input.pipe(hashStream),ContentLength:size,ContentType:type||'application/octet-stream',ContentDisposition:disposition||undefined,CacheControl:'public, max-age=31536000, immutable',Metadata:{'migration-source-id':r.id}}),{abortSignal:timeout});
|
||||
if(readBytes!==size)throw Error('Source size mismatch'); const sha256=hash.digest('hex');
|
||||
const get=await targetClient.send(new GetObjectCommand({Bucket:target.bucket,Key:r.targetKey}),{abortSignal:timeout});const targetHash=crypto.createHash('sha256');let storedBytes=0;for await(const chunk of get.Body){targetHash.update(chunk);storedBytes+=chunk.length;}
|
||||
if(storedBytes!==size||targetHash.digest('hex')!==sha256)throw Error('Target checksum mismatch');
|
||||
const targetUrl=publicUrl(r.targetKey);const head=await fetch(targetUrl,{method:'HEAD',signal:AbortSignal.timeout(30000)});
|
||||
if(!head.ok||Number(head.headers.get('content-length'))!==size)throw Error('CDN verification HTTP '+head.status);
|
||||
const record={...r,targetUrl,size,sha256,verifiedAt:new Date().toISOString()};fs.appendFileSync(journal,JSON.stringify(record)+'\n',{mode:0o600});verified.set(r.id,record);return;
|
||||
} catch(e) { last=e;input?.destroy();if(attempt<3)await new Promise(resolve=>setTimeout(resolve,attempt*1000)); }
|
||||
}
|
||||
fs.appendFileSync(path.join(dir,'failures-'+scope+'.jsonl'),JSON.stringify({id:r.id,sourceKey:r.sourceKey,name:last.name,message:String(last.message).replace(/https?:\/\/\S+/g,'[URL]').slice(0,240)})+'\n',{mode:0o600});
|
||||
}
|
||||
const psdIds=new Set(urlsIn(settings.find(s=>s.key==='agent_psd_templates')?.value).map(r=>r.id));
|
||||
const list=[...resources.values()].sort((a,b)=>(psdIds.has(a.id)?0:2)-(psdIds.has(b.id)?0:2)||Number(/\.(psd|psb)$/i.test(a.sourceKey))-Number(/\.(psd|psb)$/i.test(b.sourceKey)));
|
||||
const pending=list.filter(r=>!verified.has(r.id));
|
||||
const concurrency=Math.max(1,Math.min(96,Number(process.env.MIGRATION_CONCURRENCY||64)));
|
||||
let cursor=0,done=0;const start=Date.now();
|
||||
function progress(){const p={scope,total:list.length,processed:done,verified:verified.size,remaining:list.filter(r=>!verified.has(r.id)).length,elapsedSeconds:Math.round((Date.now()-start)/1000),verifiedBytes:[...verified.values()].reduce((n,r)=>n+r.size,0)};fs.writeFileSync(path.join(dir,'progress-'+scope+'.json'),JSON.stringify(p),{mode:0o600});console.log('copy_progress',JSON.stringify(p));}
|
||||
const interval=setInterval(progress,30000);
|
||||
try { if(command!=='rewrite-all')await Promise.all(Array.from({length:concurrency},async()=>{while(cursor<pending.length){await copy(pending[cursor++]);done++;if(done%1000===0)progress();}})); } finally {clearInterval(interval);progress();}
|
||||
// Recovery copies may finish in a separate process while the primary transfer runs.
|
||||
if(fs.existsSync(journal))for(const line of fs.readFileSync(journal,'utf8').split('\n').filter(Boolean)){const r=JSON.parse(line);verified.set(r.id,r);}
|
||||
const missing=list.filter(r=>!verified.has(r.id));
|
||||
// Only verified addresses are changed; unresolved references keep their original source.
|
||||
let changedSettings=0;
|
||||
for(const setting of settings){
|
||||
for(let attempt=0;attempt<4;attempt++){
|
||||
const [current]=await sql`select value from "SystemSettings" where id=${setting.id}`;const next=replaceVerified(current.value,verified);if(JSON.stringify(next)===JSON.stringify(current.value))break;
|
||||
const updated=await sql`update "SystemSettings" set value=${sql.json(next)},"updatedAt"=now() where id=${setting.id} and value=${sql.json(current.value)} returning id`;
|
||||
if(updated.length){changedSettings++;break;}if(attempt===3)throw Error('Concurrent setting update prevented replacement');
|
||||
}
|
||||
}
|
||||
if(scope==='all'){
|
||||
const backupFile=path.join(dir,'database-before.jsonl.gz');
|
||||
for(const c of columns){
|
||||
// Administrative audit snapshots preserve their original configuration values.
|
||||
// User canvas/chat history remains in scope; obsolete configuration backups do not.
|
||||
if(['StorageConfig','StorageMigrationTask','ImageArchiveJob','__drizzle_migrations','SystemSettings','SystemSettingHistory','SystemSettingSnapshot'].includes(c.tableName))continue;
|
||||
const pks=await sql`select a.attname,a.atttypid::regtype::text as "pkType" from pg_index i join pg_attribute a on a.attrelid=i.indrelid and a.attnum=any(i.indkey) where i.indrelid=${'"'+c.tableName+'"'}::regclass and i.indisprimary`;
|
||||
if(pks.length!==1){continue;} const pk=pks[0].attname;
|
||||
let count=0;
|
||||
const aliasGuard=c.tableName==='ImageArchiveTask'&&c.columnName==='sourceUrl'?" and coalesce(metadata->>'legacyOssMigrationRun','') <> '20261014'":'';
|
||||
for await(const rows of sql.unsafe('select '+quote(pk)+' as pk,'+quote(c.columnName)+' as value from '+quote(c.tableName)+' where '+quote(c.columnName)+'::text like $1'+aliasGuard,['%'+SOURCE_HOST+'/%']).cursor(50)){
|
||||
// Flush before updating to retain the original data even on interruption.
|
||||
count+=await rewriteBatch(sql,c,pk,pks[0].pkType,rows,verified,backups=>fs.appendFileSync(backupFile,require('node:zlib').gzipSync(backups.map(r=>JSON.stringify(r)).join('\n')+'\n',{level:6}),{mode:0o600}));
|
||||
}
|
||||
if(count)console.log('references_replaced',JSON.stringify({table:c.tableName,column:c.columnName,rows:count}));
|
||||
}
|
||||
}
|
||||
const [migration]=await sql`select id,value from "SystemSettings" where key='oss_image_migration_config'`;
|
||||
if(migration){const next={...migration.value,enabled:false,preferAliyunForUploads:false,targetStorageConfigId:target.id};await sql`update "SystemSettings" set value=${sql.json(next)},"updatedAt"=now() where id=${migration.id}`;}
|
||||
await sql`update "StorageConfig" set "isEnabled"=false,"isDefault"=false where id=${source.id}`;
|
||||
const result={scope,objects:list.length,verified:list.length-missing.length,missing:missing.length,changedSettings,defaultStorage:'qiniu-kodo',sourceRetained:true,completedAt:new Date().toISOString()};fs.writeFileSync(path.join(dir,'result-'+scope+'.json'),JSON.stringify(result,null,2));console.log('migration_result',JSON.stringify(result));
|
||||
if(missing.length)process.exitCode=2;
|
||||
} finally {sourceClient?.destroy();targetClient?.destroy();await sql.end();}
|
||||
}
|
||||
10
scripts/storage/migrate-aliyun-to-qiniu.test.cjs
Normal file
10
scripts/storage/migrate-aliyun-to-qiniu.test.cjs
Normal file
@ -0,0 +1,10 @@
|
||||
const test=require('node:test'),assert=require('node:assert/strict');const {resourceFor,urlsIn,replaceVerified}=require('./migrate-aliyun-to-qiniu.cjs');
|
||||
const origin='https://demo-sparkai.oss-cn-shanghai.aliyuncs.com';
|
||||
test('migrates nested PSD layer, font and media references without consuming Markdown punctuation',()=>{const doc={layers:[{sourceUrl:origin+'/uploads/layer.png'}],font:origin+'/fonts/font.ttf',text:''};assert.equal(urlsIn(doc).length,3);assert.equal(urlsIn(doc)[2].sourceKey,'uploads/img.jpg');});
|
||||
test('expired authorization parameters do not alter identity; processing variants remain distinct',()=>{assert.equal(resourceFor(origin+'/a.png?Expires=1&Signature=old&OSSAccessKeyId=id').id,resourceFor(origin+'/a.png').id);assert.notEqual(resourceFor(origin+'/a.png?x-oss-process=image%2Fresize,w_400').id,resourceFor(origin+'/a.png').id);});
|
||||
test('only verified assets are replaced and other hosts remain intact',()=>{const a=origin+'/a.psd',b=origin+'/b.png',r=resourceFor(a),m=new Map([[r.id,{targetUrl:'https://cdn.example/legacy/a.psd'}]]);assert.deepEqual(replaceVerified({a,b,other:'https://other.example/a.psd'},m),{a:'https://cdn.example/legacy/a.psd',b,other:'https://other.example/a.psd'});});
|
||||
test('encoded object keys are safely canonicalized and variants are namespaced',()=>{const r=resourceFor(origin+'/uploads/%E5%9B%BE.png');assert.equal(r.sourceKey,'uploads/图.png');assert.equal(r.targetKey,'legacy-oss/demo-sparkai/uploads/图.png');assert.ok(resourceFor(origin+'/a.png?x-oss-process=image/resize,w_400').targetKey.includes('/variants/'));});
|
||||
|
||||
test('cache resolution suffixes are not part of the storage object key',()=>{const text=origin+'/uploads/img.png|||2K',r=urlsIn(text)[0];assert.equal(r.sourceKey,'uploads/img.png');const map=new Map([[r.id,{targetUrl:'https://cdn.example/img.png'}]]);assert.equal(replaceVerified(text,map),'https://cdn.example/img.png|||2K');});
|
||||
|
||||
test('URL keyed lookup maps stay in sync with migrated URL values',()=>{const key=origin+'/uploads/key.png|||2K',r=urlsIn(key)[0],map=new Map([[r.id,{targetUrl:'https://cdn.example/key.png'}]]);assert.equal(urlsIn({[key]:{width:1024}}).length,1);assert.deepEqual(replaceVerified({[key]:{width:1024}},map),{'https://cdn.example/key.png|||2K':{width:1024}});});
|
||||
6
scripts/storage/register-aliyun-qiniu-aliases.cjs
Normal file
6
scripts/storage/register-aliyun-qiniu-aliases.cjs
Normal file
@ -0,0 +1,6 @@
|
||||
/* Publish verified legacy aliases through the application's existing completed-archive resolver.
|
||||
* No source objects are deleted, and no pending archive jobs are created or overwritten. */
|
||||
const fs=require('fs'),path=require('path'),crypto=require('crypto');const root=process.env.MIGRATION_APP_ROOT||process.cwd(),dir=process.env.MIGRATION_RUN_DIR||'/var/lib/ai-chatbot/aliyun-qiniu-20261014';const {resourceFor}=require('./migrate-aliyun-to-qiniu.cjs');require(path.join(root,'node_modules/dotenv')).config({path:path.join(root,'.env.local')});const sql=require(path.join(root,'node_modules/postgres'))(process.env.POSTGRES_URL,{max:1,prepare:false});
|
||||
(async()=>{try{const [target]=await sql`select id,bucket,"publicUrl" from "StorageConfig" where type='qiniu-kodo' and "isDefault" and "isEnabled" limit 1`;if(!target?.publicUrl)throw Error('No default Qiniu target');const map=new Map();for(const line of fs.readFileSync(path.join(dir,'verified.jsonl'),'utf8').split('\n').filter(Boolean)){const r=JSON.parse(line),canonical=resourceFor(r.identity),expectedUrl=target.publicUrl.replace(/\/$/,'')+'/'+r.targetKey.split('/').map(encodeURIComponent).join('/');if(!canonical||canonical.id!==r.id||r.targetUrl!==expectedUrl||r.size<=0||(!r.md5&&!r.sha256))throw Error('Unverified storage alias');map.set(r.identity,r);}let inserted=0,processed=0,batch=[];
|
||||
async function flush(){if(!batch.length)return;const added=await sql.unsafe('insert into "ImageArchiveTask" (id,"sourceUrl","targetUrl","targetStorageConfigId",status,"attemptCount","archiveAfterAt","replaceAfterAt","archivedAt","replacedAt",metadata,"createdAt","updatedAt") select gen_random_uuid(),r.source,r.target,$2::uuid,\'replaced\',0,now(),now(),now(),now(),r.metadata,now(),now() from jsonb_to_recordset($1::jsonb) as r(source text,target text,metadata jsonb) on conflict ("sourceUrl") do nothing returning id',[sql.json(batch),target.id]);inserted+=added.length;processed+=batch.length;batch=[];if(processed%25000===0)console.log(JSON.stringify({processed,total:map.size,inserted}));}
|
||||
for(const r of map.values()){batch.push({source:r.identity,target:r.targetUrl,metadata:{legacyOssMigrationRun:'20261014',migrationResourceId:r.id,migrationVerifiedAt:r.verifiedAt,verifiedBytes:r.size,verifiedChecksum:r.md5||r.sha256}});if(batch.length>=500)await flush();}await flush();const result={aliasResources:map.size,inserted,existingPreserved:map.size-inserted,pendingJobsCreated:0,completedAt:new Date().toISOString()};fs.writeFileSync(path.join(dir,'alias-registration.json'),JSON.stringify(result,null,2),{mode:0o600});console.log('alias_registration',JSON.stringify(result));}finally{await sql.end()}})().catch(e=>{console.error('alias_registration_failed',e.name,String(e.message).replace(/https?:\/\/\S+/g,'[URL]').slice(0,180));process.exitCode=1});
|
||||
Loading…
x
Reference in New Issue
Block a user