diff --git a/components/canvas/LiblibVideoCanvasReplica.tsx b/components/canvas/LiblibVideoCanvasReplica.tsx index 8594cec7a..d8282bf03 100644 --- a/components/canvas/LiblibVideoCanvasReplica.tsx +++ b/components/canvas/LiblibVideoCanvasReplica.tsx @@ -786,28 +786,28 @@ const STARTER_CARDS = [ title: "故事脚本生成", subtitle: "脚本到分镜", coverUrl: - "https://demo-sparkai.oss-cn-shanghai.aliyuncs.com/video-canvas-demo/starter-story-script-cover.png", + "https://scqnycdn.jiansun.vip/legacy-oss/demo-sparkai/video-canvas-demo/starter-story-script-cover.png", steps: ["故事脚本", "多模态文本模型Pro", "可灵 3.0", "预览"], }, { title: "角色三视图", subtitle: "角色资产沉淀", coverUrl: - "https://demo-sparkai.oss-cn-shanghai.aliyuncs.com/video-canvas-demo/starter-character-three-view-cover.png", + "https://scqnycdn.jiansun.vip/legacy-oss/demo-sparkai/video-canvas-demo/starter-character-three-view-cover.png", steps: ["角色描述", "角色三视图", "全能图片模型V2", "保存"], }, { title: "首帧图生视频", subtitle: "参考图到成片", coverUrl: - "https://demo-sparkai.oss-cn-shanghai.aliyuncs.com/video-canvas-demo/starter-first-frame-video-cover.png", + "https://scqnycdn.jiansun.vip/legacy-oss/demo-sparkai/video-canvas-demo/starter-first-frame-video-cover.png", steps: ["上传图片", "首帧图", "可灵 3.0", "预览"], }, { title: "音频生视频", subtitle: "音频驱动画面", coverUrl: - "https://demo-sparkai.oss-cn-shanghai.aliyuncs.com/video-canvas-demo/starter-audio-to-video-cover.png", + "https://scqnycdn.jiansun.vip/legacy-oss/demo-sparkai/video-canvas-demo/starter-audio-to-video-cover.png", icon: "music", steps: ["上传音频", "音频主视频", "Wan 2.6", "预览"], }, @@ -822,7 +822,7 @@ const TOOLBOX_TEMPLATE_PRESETS: ToolboxTemplatePreset[] = [ badge: "预设", category: "周星驰经典名场面", cover: - "https://demo-sparkai.oss-cn-shanghai.aliyuncs.com/video-canvas-demo/starter-first-frame-video-cover.png", + "https://scqnycdn.jiansun.vip/legacy-oss/demo-sparkai/video-canvas-demo/starter-first-frame-video-cover.png", description: "左弧滑行，镜头从人物左侧弧线推进，带轻微喜剧节奏和夸张表情停顿。", duration: "5s", id: "left-arc-slide", @@ -837,7 +837,7 @@ const TOOLBOX_TEMPLATE_PRESETS: ToolboxTemplatePreset[] = [ badge: "商用", category: "电商展示", cover: - "https://demo-sparkai.oss-cn-shanghai.aliyuncs.com/video-canvas-demo/starter-character-three-view-cover.png", + "https://scqnycdn.jiansun.vip/legacy-oss/demo-sparkai/video-canvas-demo/starter-character-three-view-cover.png", description: "电商手机弹出效果，产品从画面中心弹出并旋转定格，适合主图视频开场。", duration: "5s", id: "phone-pop-product", @@ -852,7 +852,7 @@ const TOOLBOX_TEMPLATE_PRESETS: ToolboxTemplatePreset[] = [ badge: "商用", category: "电商展示", cover: - "https://demo-sparkai.oss-cn-shanghai.aliyuncs.com/video-canvas-demo/starter-story-script-cover.png", + "https://scqnycdn.jiansun.vip/legacy-oss/demo-sparkai/video-canvas-demo/starter-story-script-cover.png", description: "咖啡杯出场，杯身从桌面滑入，蒸汽和柔光形成温暖的品牌氛围。", duration: "5s", id: "coffee-cup-entry", @@ -867,7 +867,7 @@ const TOOLBOX_TEMPLATE_PRESETS: ToolboxTemplatePreset[] = [ badge: "热门", category: "镜头运动", cover: - "https://demo-sparkai.oss-cn-shanghai.aliyuncs.com/video-canvas-demo/starter-audio-to-video-cover.png", + "https://scqnycdn.jiansun.vip/legacy-oss/demo-sparkai/video-canvas-demo/starter-audio-to-video-cover.png", description: "360旋转展示，围绕主体做一圈稳定环绕，适合产品、人物和空间展示。", duration: "6s", id: "orbit-360-showcase", @@ -881,7 +881,7 @@ const TOOLBOX_TEMPLATE_PRESETS: ToolboxTemplatePreset[] = [ author: "工业镜头", category: "镜头运动", cover: - "https://demo-sparkai.oss-cn-shanghai.aliyuncs.com/video-canvas-demo/character-image.png", + "https://scqnycdn.jiansun.vip/legacy-oss/demo-sparkai/video-canvas-demo/character-image.png", description: "机械臂视角，以第一视角穿过机械臂与产品线，增强科技感和空间纵深。", duration: "5s", id: "robot-arm-pov", @@ -896,7 +896,7 @@ const TOOLBOX_TEMPLATE_PRESETS: ToolboxTemplatePreset[] = [ badge: "NEW", category: "人物动画", cover: - "https://demo-sparkai.oss-cn-shanghai.aliyuncs.com/video-canvas-demo/character-three-view.png", + "https://scqnycdn.jiansun.vip/legacy-oss/demo-sparkai/video-canvas-demo/character-three-view.png", description: "Live 2D，让单张角色图产生眨眼、微笑和轻微身体摆动，适合口播开场。", duration: "4s", id: "live-2d-character", @@ -910,7 +910,7 @@ const TOOLBOX_TEMPLATE_PRESETS: ToolboxTemplatePreset[] = [ author: "电影运镜", category: "镜头运动", cover: - "https://demo-sparkai.oss-cn-shanghai.aliyuncs.com/video-canvas-demo/starter-first-frame-video-cover.png", + "https://scqnycdn.jiansun.vip/legacy-oss/demo-sparkai/video-canvas-demo/starter-first-frame-video-cover.png", description: "瞳孔拉近，镜头快速推向眼部并在反光里展示下一场景，适合悬疑转场。", duration: "5s", id: "pupil-zoom", @@ -925,7 +925,7 @@ const TOOLBOX_TEMPLATE_PRESETS: ToolboxTemplatePreset[] = [ badge: "热门", category: "人物动画", cover: - "https://demo-sparkai.oss-cn-shanghai.aliyuncs.com/video-canvas-demo/starter-story-script-cover.png", + "https://scqnycdn.jiansun.vip/legacy-oss/demo-sparkai/video-canvas-demo/starter-story-script-cover.png", description: "飞鸟解体，主体由细碎羽片散开再重组，适合视觉冲击强的短视频片头。", duration: "6s", id: "bird-dissolve", @@ -940,7 +940,7 @@ const TOOLBOX_TEMPLATE_PRESETS: ToolboxTemplatePreset[] = [ badge: "商用", category: "电商展示", cover: - "https://demo-sparkai.oss-cn-shanghai.aliyuncs.com/video-canvas-demo/starter-character-three-view-cover.png", + "https://scqnycdn.jiansun.vip/legacy-oss/demo-sparkai/video-canvas-demo/starter-character-three-view-cover.png", description: "商品震撼登场，产品带冲击波与局部高光进入画面，适合活动主推款。", duration: "5s", id: "product-hero-entry", @@ -954,7 +954,7 @@ const TOOLBOX_TEMPLATE_PRESETS: ToolboxTemplatePreset[] = [ author: "旅拍模板", category: "镜头运动", cover: - "https://demo-sparkai.oss-cn-shanghai.aliyuncs.com/video-canvas-demo/starter-audio-to-video-cover.png", + "https://scqnycdn.jiansun.vip/legacy-oss/demo-sparkai/video-canvas-demo/starter-audio-to-video-cover.png", description: "旅拍转场，前景遮挡切换到新地点，带自然手持感和轻微速度变化。", duration: "5s", id: "travel-whip-transition", @@ -1299,19 +1299,19 @@ const NINE_GRID_CELL_VISUALS: GridCellVisual[] = [ ]; const TARGET_AVATAR = - "https://demo-sparkai.oss-cn-shanghai.aliyuncs.com/video-canvas-demo/target-avatar.jpg"; + "https://scqnycdn.jiansun.vip/legacy-oss/demo-sparkai/video-canvas-demo/target-avatar.jpg"; const CHARACTER_IMAGE_URL = - "https://demo-sparkai.oss-cn-shanghai.aliyuncs.com/video-canvas-demo/character-image.png"; + "https://scqnycdn.jiansun.vip/legacy-oss/demo-sparkai/video-canvas-demo/character-image.png"; const CHARACTER_THREE_VIEW_URL = - "https://demo-sparkai.oss-cn-shanghai.aliyuncs.com/video-canvas-demo/character-three-view.png"; + "https://scqnycdn.jiansun.vip/legacy-oss/demo-sparkai/video-canvas-demo/character-three-view.png"; const FIRST_FRAME_IMAGE_URL = - "https://demo-sparkai.oss-cn-shanghai.aliyuncs.com/video-canvas-demo/first-frame-image.png"; + "https://scqnycdn.jiansun.vip/legacy-oss/demo-sparkai/video-canvas-demo/first-frame-image.png"; const AUDIO_SOURCE_URL = - "https://demo-sparkai.oss-cn-shanghai.aliyuncs.com/video-canvas-demo/audio-source.mp3"; + "https://scqnycdn.jiansun.vip/legacy-oss/demo-sparkai/video-canvas-demo/audio-source.mp3"; const AUDIO_REFERENCE_IMAGE_URL = - "https://demo-sparkai.oss-cn-shanghai.aliyuncs.com/video-canvas-demo/audio-reference-image.png"; + "https://scqnycdn.jiansun.vip/legacy-oss/demo-sparkai/video-canvas-demo/audio-reference-image.png"; const PUBLISH_MODAL_BG_URL = - "https://demo-sparkai.oss-cn-shanghai.aliyuncs.com/video-canvas-demo/publish-modal-bg.png"; + "https://scqnycdn.jiansun.vip/legacy-oss/demo-sparkai/video-canvas-demo/publish-modal-bg.png"; const STORY_SCRIPT_PROMPT = "根据我上传的剧本生成一个完整的故事脚本"; const AUDIO_TO_VIDEO_PROMPT = "根据上传的音频生成对应场景画面，镜头语言、节奏、音乐匹配情绪变化，电影级质感。"; @@ -1322,22 +1322,22 @@ const TEXT_TO_MUSIC_PROMPT = "生成一首现代品牌电子音乐，干净有力的低频贝斯，清晰电子鼓点，整体风格高级、未来感强。"; const IMAGE_TO_PROMPT_PROMPT = "根据图片生成结构化中文提示词，包括主体描述、环境、光影、镜头语言、风格关键词。"; const HD_IMAGE_SOURCE_URL = - "https://demo-sparkai.oss-cn-shanghai.aliyuncs.com/video-canvas-demo/hd-image-source.png"; + "https://scqnycdn.jiansun.vip/legacy-oss/demo-sparkai/video-canvas-demo/hd-image-source.png"; const VIDEO_FIRST_FRAME_SOURCE_URL = - "https://demo-sparkai.oss-cn-shanghai.aliyuncs.com/video-canvas-demo/video-first-frame-source.png"; + "https://scqnycdn.jiansun.vip/legacy-oss/demo-sparkai/video-canvas-demo/video-first-frame-source.png"; const VIDEO_FIRST_FRAME_PROMPT = "以当前图为首帧生成视频。"; const VIDEO_FIRST_LAST_FIRST_URL = - "https://demo-sparkai.oss-cn-shanghai.aliyuncs.com/video-canvas-demo/video-first-last-first.png"; + "https://scqnycdn.jiansun.vip/legacy-oss/demo-sparkai/video-canvas-demo/video-first-last-first.png"; const VIDEO_FIRST_LAST_LAST_URL = - "https://demo-sparkai.oss-cn-shanghai.aliyuncs.com/video-canvas-demo/video-first-last-last.png"; + "https://scqnycdn.jiansun.vip/legacy-oss/demo-sparkai/video-canvas-demo/video-first-last-last.png"; const VIDEO_FIRST_LAST_PROMPT = "手从白帆布包中取出青柠气泡水罐的特写，过渡到短发女生在户外草地闭眼仰头喝同款气泡水的中景，日系清新治愈风格，明亮自然光，柔和暖调，胶片质感，动作流畅自然"; const SCRIPT_VIDEO_REFERENCE_URL = - "https://demo-sparkai.oss-cn-shanghai.aliyuncs.com/video-canvas-demo/script-video-reference.mp4"; + "https://scqnycdn.jiansun.vip/legacy-oss/demo-sparkai/video-canvas-demo/script-video-reference.mp4"; const SCRIPT_CHARACTER_ONE_URL = - "https://demo-sparkai.oss-cn-shanghai.aliyuncs.com/video-canvas-demo/script-character-one.png"; + "https://scqnycdn.jiansun.vip/legacy-oss/demo-sparkai/video-canvas-demo/script-character-one.png"; const SCRIPT_CHARACTER_TWO_URL = - "https://demo-sparkai.oss-cn-shanghai.aliyuncs.com/video-canvas-demo/script-character-two.png"; + "https://scqnycdn.jiansun.vip/legacy-oss/demo-sparkai/video-canvas-demo/script-character-two.png"; const SCRIPT_CHARACTER_PROMPT = "根据角色参考图生成分镜脚本，围绕旧城区、机械修复、暴雨停电和被重新点亮的街道展开，突出人物关系、镜头画面和声音设计。"; const SLASH_IMAGE_ITEMS = [ @@ -22937,8 +22937,8 @@ const STYLE_MARKET_CATEGORIES = [ "全部", ]; -const STYLE_MARKET_THUMBNAIL_URL = "https://demo-sparkai.oss-cn-shanghai.aliyuncs.com/video-canvas-demo/starter-character-three-view-cover.png"; -const EFFECT_MARKET_THUMBNAIL_URL = "https://demo-sparkai.oss-cn-shanghai.aliyuncs.com/video-canvas-demo/starter-first-frame-video-cover.png"; +const STYLE_MARKET_THUMBNAIL_URL = "https://scqnycdn.jiansun.vip/legacy-oss/demo-sparkai/video-canvas-demo/starter-character-three-view-cover.png"; +const EFFECT_MARKET_THUMBNAIL_URL = "https://scqnycdn.jiansun.vip/legacy-oss/demo-sparkai/video-canvas-demo/starter-first-frame-video-cover.png"; const STYLE_MARKET_CARDS: AssetMarketCard[] = [ ["复古马卡龙", "算力潮汐", 177], diff --git a/lib/video-canvas/config.ts b/lib/video-canvas/config.ts index c2864348b..9eadb3e7a 100644 --- a/lib/video-canvas/config.ts +++ b/lib/video-canvas/config.ts @@ -526,13 +526,13 @@ export const defaultVideoCanvasMotionPresets: VideoCanvasMotionPreset[] = [ const defaultRoleImages: VideoCanvasRolePreset["images"] = { expression: - "https://demo-sparkai.oss-cn-shanghai.aliyuncs.com/video-canvas-demo/character-image.png", + "https://scqnycdn.jiansun.vip/legacy-oss/demo-sparkai/video-canvas-demo/character-image.png", face: - "https://demo-sparkai.oss-cn-shanghai.aliyuncs.com/video-canvas-demo/character-image.png", + "https://scqnycdn.jiansun.vip/legacy-oss/demo-sparkai/video-canvas-demo/character-image.png", full: - "https://demo-sparkai.oss-cn-shanghai.aliyuncs.com/video-canvas-demo/character-image.png", + "https://scqnycdn.jiansun.vip/legacy-oss/demo-sparkai/video-canvas-demo/character-image.png", threeView: - "https://demo-sparkai.oss-cn-shanghai.aliyuncs.com/video-canvas-demo/character-three-view.png", + "https://scqnycdn.jiansun.vip/legacy-oss/demo-sparkai/video-canvas-demo/character-three-view.png", }; function createDefaultRolePreset( diff --git a/scripts/storage/QINIU_MIGRATION.md b/scripts/storage/QINIU_MIGRATION.md new file mode 100644 index 000000000..64ef776ef --- /dev/null +++ b/scripts/storage/QINIU_MIGRATION.md @@ -0,0 +1,25 @@ +# 项目旧资源迁移到七牛 + +这组工具只复制应用代码和当前业务数据库引用的旧资源，不复制整桶。画布、聊天、媒体资产与业务历史记录在范围内；已删除项目的无引用文件，以及管理设置的历史审计备份，不用于扩大迁移范围。源桶保留，不删除源对象。 + +目标使用数据库中启用且默认的 `qiniu-kodo` 配置。原 OSS 资源保存在 `legacy-oss/demo-sparkai/`，PSD 的旧备用 S3 资源保存在 `legacy-s3/gongxiangpan/`。图片处理查询生成单独的 variants 对象，以保留原有裁剪、缩放结果。过期的 OSS 签名字段不保留。 + +先复制并验证文件长度及 MD5（流式备用传输验证 SHA256），再替换引用。替换前的配置和业务数据写入仅 root 可读的备份目录；业务更新比较旧值，保留迁移期间用户的并发编辑和画布内容修订时间。URL 形式的对象键与 `|||2K` 缓存后缀也按规则处理。 + +## 执行与复核 + +从应用根目录运行；密钥来自数据库和 `.env.local`，不作为命令行参数，也不写入仓库。默认工作目录为 `/var/lib/ai-chatbot/aliyun-qiniu-20261014`，可以用 `MIGRATION_RUN_DIR` 指定；`MIGRATION_APP_ROOT` 可指定应用目录。 + +1. `node scripts/storage/migrate-aliyun-to-qiniu.cjs inventory-all`：盘点业务引用。 +2. `node scripts/storage/inventory-aliyun-qiniu-source.cjs`：只读列举源桶，保存所需对象的大小及 ETag，避免每个文件单独请求源站头信息。 +3. `node scripts/storage/migrate-aliyun-to-qiniu.cjs migrate-settings`：迁移含旧 OSS 的当前设置。 +4. `node scripts/storage/migrate-aliyun-to-qiniu.cjs migrate-psd-extra`：补齐 PSD 模板和字体的旧备用 S3 依赖。 +5. `node scripts/storage/migrate-aliyun-to-qiniu.cjs migrate-all`：复制剩余业务引用，校验后替换地址。 +6. `node scripts/storage/register-aliyun-qiniu-aliases.cjs`：把已验证映射登记为现有归档系统的已完成记录。旧客户端再次保存时，可复用既有地址解析逻辑；不创建待处理归档任务，不覆盖既有任务。 +7. `node scripts/storage/audit-aliyun-qiniu-references.cjs`：复核业务引用。完成映射的来源地址是追溯信息，不是读取旧桶的依赖；审计器保留这一差别。 + +`MIGRATION_CONCURRENCY` 控制复制并发，最大 96，与模型生图并发无关。已验证对象通过 `verified.jsonl` 断点续跑。复制失败的对象保持旧地址，进程返回 2，需要核查失败记录；只有复制成功后才替换地址。`rewrite-all` 可复核已验证地址的替换，`MIGRATION_RESCAN=1` 会重新盘点业务表。`migrate-literals` 复制代码里的示例素材；代码地址修改仍需正常构建验证和部署。 + +`migrate-recovery` 从 `MIGRATION_RESOURCE_FILE` 指定的私有恢复清单复制已有备份，只接受配置中登记的存储来源。成功恢复的文件仍要校验。源文件不存在或为空且无有效备份时，保留记录，不能把不存在的数据宣称为迁移成功。 + +验证：`node --test scripts/storage/migrate-aliyun-to-qiniu.test.cjs`。实际迁移还验证了数据库并发更新保护、主图/SKU PSD 导出、两站上传与下载分段/CORS，以及旧画布保存时自动解析到七牛。与资源地址无关的模型、上传和聊天代码保持原样。 diff --git a/scripts/storage/audit-aliyun-qiniu-references.cjs b/scripts/storage/audit-aliyun-qiniu-references.cjs new file mode 100644 index 000000000..779e7a3a2 --- /dev/null +++ b/scripts/storage/audit-aliyun-qiniu-references.cjs @@ -0,0 +1,6 @@ +/* Read-only audit of live business references; obsolete admin configuration backups stay intact. */ +const fs=require('fs'),path=require('path');const root=process.env.MIGRATION_APP_ROOT||process.cwd(),dir=process.env.MIGRATION_RUN_DIR||'/var/lib/ai-chatbot/aliyun-qiniu-20261014';const {urlsIn}=require('./migrate-aliyun-to-qiniu.cjs');require(path.join(root,'node_modules/dotenv')).config({path:path.join(root,'.env.local')});const sql=require(path.join(root,'node_modules/postgres'))(process.env.POSTGRES_URL,{max:1,prepare:false,connection:{statement_timeout:600000}});const host='demo-sparkai.oss-cn-shanghai.aliyuncs.com',quote=s=>'"'+s.replaceAll('"','""')+'"'; +(async()=>{try{const verified=new Set(fs.readFileSync(path.join(dir,'verified.jsonl'),'utf8').split('\n').filter(Boolean).map(s=>JSON.parse(s).id)),unavailable=new Set(fs.existsSync(path.join(dir,'unavailable-source.json'))?JSON.parse(fs.readFileSync(path.join(dir,'unavailable-source.json'))).map(r=>r.id):[]),remaining=new Map(),tables=new Map(),excluded=new Set(['StorageConfig','StorageMigrationTask','ImageArchiveJob','__drizzle_migrations','SystemSettingHistory','SystemSettingSnapshot']); +const columns=await sql`select c.table_name as "tableName",c.column_name as "columnName" from information_schema.columns c join information_schema.tables t on c.table_schema=t.table_schema and c.table_name=t.table_name where c.table_schema='public' and t.table_type='BASE TABLE' and c.data_type in ('text','character varying','json','jsonb') order by c.table_name,c.ordinal_position`; +for(const c of columns)if(!excluded.has(c.tableName)){if(!tables.has(c.tableName))tables.set(c.tableName,[]);tables.get(c.tableName).push(c.columnName);}const tableCounts={};for(const [table,cols] of tables){const where=cols.map(c=>'('+quote(c)+'::text like $1'+(table==='ImageArchiveTask'&&c==='sourceUrl'?" and coalesce(metadata->>'legacyOssMigrationRun','') <> '20261014'":'')+')').join(' or ');let matched=0;for await(const rows of sql.unsafe('select '+cols.map(quote).join(',')+' from '+quote(table)+' where '+where,['%'+host+'/%']).cursor(50))for(const row of rows){matched++;for(const [column,value]of Object.entries(row)){if(table==='ImageArchiveTask'&&column==='sourceUrl'&&row.metadata?.legacyOssMigrationRun==='20261014')continue;for(const r of urlsIn(value))if(r.sourceHost===host){if(!remaining.has(r.id))remaining.set(r.id,{...r,verified:verified.has(r.id),knownUnavailable:unavailable.has(r.id),locations:{}});const item=remaining.get(r.id),key=table+'.'+column;item.locations[key]=(item.locations[key]||0)+1;}}}if(matched){tableCounts[table]=matched;console.log('remaining_reference_rows',JSON.stringify({table,rows:matched}));}} +const resources=[...remaining.values()],summary={auditedAt:new Date().toISOString(),remainingUniqueResources:resources.length,verifiedPendingRewrite:resources.filter(r=>r.verified).length,knownUnavailable:resources.filter(r=>r.knownUnavailable&&!r.verified).length,unclassified:resources.filter(r=>!r.knownUnavailable&&!r.verified).length,tableCounts};fs.writeFileSync(path.join(dir,'reference-audit-resources.json'),JSON.stringify(resources),{mode:0o600});fs.writeFileSync(path.join(dir,'reference-audit-summary.json'),JSON.stringify(summary,null,2),{mode:0o600});console.log('reference_audit',JSON.stringify(summary));}finally{await sql.end()}})().catch(e=>{console.error('audit_failed',e.name,String(e.message).replace(/https?:\/\/\S+/g,'[URL]').slice(0,180));process.exitCode=1}); diff --git a/scripts/storage/inventory-aliyun-qiniu-source.cjs b/scripts/storage/inventory-aliyun-qiniu-source.cjs new file mode 100644 index 000000000..9820f8d3d --- /dev/null +++ b/scripts/storage/inventory-aliyun-qiniu-source.cjs @@ -0,0 +1,8 @@ +const fs=require('fs'),path=require('path'),root=process.env.MIGRATION_APP_ROOT||process.cwd(),dir=process.env.MIGRATION_RUN_DIR||'/var/lib/ai-chatbot/aliyun-qiniu-20261014'; +require(path.join(root,'node_modules/dotenv')).config({path:path.join(root,'.env.local')});const sql=require(path.join(root,'node_modules/postgres'))(process.env.POSTGRES_URL,{max:1,prepare:false});const {S3Client,ListObjectsV2Command}=require(path.join(root,'node_modules/@aws-sdk/client-s3')); +(async()=>{let client;try{const complete=path.join(dir,'source-metadata.complete');if(fs.existsSync(complete)){console.log('Source metadata inventory already completed');return;}const plan=JSON.parse(fs.readFileSync(path.join(dir,'plan-all.json'))),wanted=new Set(plan.filter(r=>!r.variant).map(r=>r.sourceKey)); +const [s]=await sql`select * from "StorageConfig" where bucket='demo-sparkai' limit 1`;client=new S3Client({endpoint:s.endpoint,region:s.region,forcePathStyle:false,credentials:{accessKeyId:s.accessKeyId,secretAccessKey:s.secretAccessKey},maxAttempts:3});let marker,listed=0,matched=0;const checkpointFile=path.join(dir,'source-list-checkpoint.json'),file=path.join(dir,'source-metadata.jsonl');if(fs.existsSync(checkpointFile)){const c=JSON.parse(fs.readFileSync(checkpointFile));marker=c.marker;listed=c.listed;matched=c.matched;} +for(;;){const r=await client.send(new ListObjectsV2Command({Bucket:s.bucket,MaxKeys:1000,ContinuationToken:marker}),{abortSignal:AbortSignal.timeout(60000)});const lines=[];for(const o of r.Contents||[]){listed++;if(wanted.has(o.Key)){lines.push(JSON.stringify({key:o.Key,size:o.Size,md5:String(o.ETag||'').replaceAll('"','').toLowerCase()}));matched++;}} +if(lines.length)fs.appendFileSync(file,lines.join('\n')+'\n',{mode:0o600});marker=r.NextContinuationToken;fs.writeFileSync(checkpointFile,JSON.stringify({marker,listed,matched}),{mode:0o600});if(listed%25000<1000)console.log(JSON.stringify({listed,matched,wanted:wanted.size}));if(!r.IsTruncated)break;} +fs.writeFileSync(complete,JSON.stringify({listed,matched,wanted:wanted.size,completedAt:new Date().toISOString()}),{mode:0o600});console.log('Source inventory complete',JSON.stringify({listed,matched,wanted:wanted.size})); +}finally{client?.destroy();await sql.end()}})().catch(e=>{console.error('source_inventory_failed',e.name,String(e.message).replace(/https?:\/\/\S+/g,'[URL]').slice(0,180));process.exitCode=1}); diff --git a/scripts/storage/migrate-aliyun-to-qiniu.cjs b/scripts/storage/migrate-aliyun-to-qiniu.cjs new file mode 100644 index 000000000..629ede4e8 --- /dev/null +++ b/scripts/storage/migrate-aliyun-to-qiniu.cjs @@ -0,0 +1,220 @@ +/* Resumable, verified migration. No source objects are deleted. */ +const fs = require('node:fs'); +const path = require('node:path'); +const crypto = require('node:crypto'); +const { Readable, Transform } = require('node:stream'); +const SOURCE_HOST = 'demo-sparkai.oss-cn-shanghai.aliyuncs.com'; +const URL_PATTERN = /https?:\/\/(?:demo-sparkai\.oss-cn-shanghai\.aliyuncs\.com\/|s3\.jiansun\.vip\/gongxiangpan\/)[^\s"'<>`\\)\]}|]+/gi; +function resourceFor(url) { + const u = new URL(url); + const isPrimary=u.hostname===SOURCE_HOST,isPsdBackup=u.hostname==='s3.jiansun.vip'&&u.pathname.startsWith('/gongxiangpan/'); + if ((!isPrimary&&!isPsdBackup) || u.pathname === '/') return null; + for (const name of ['Signature', 'OSSAccessKeyId', 'Expires', 'security-token']) u.searchParams.delete(name); + u.protocol = 'https:'; u.hash = ''; u.searchParams.sort(); + const identity = u.toString(), sourceKey = decodeURIComponent(u.pathname.slice(isPsdBackup?'/gongxiangpan/'.length:1)); + if (!sourceKey || sourceKey.split('/').some(p => p === '..')) throw Error('Invalid object key'); + const id = crypto.createHash('sha256').update(identity).digest('hex'); + const namespace=isPrimary?'legacy-oss/demo-sparkai/':'legacy-s3/gongxiangpan/'; + const targetKey = u.search ? namespace+'variants/' + id + path.extname(sourceKey) : namespace + sourceKey; + return { id, identity, sourceHost:u.hostname, sourceKey, targetKey, variant: Boolean(u.search) }; +} +function urlsIn(value) { + const found = new Map(); + function walk(v) { + if (typeof v === 'string') for (const url of v.match(URL_PATTERN) || []) { const r = resourceFor(url); if (r) found.set(r.id, r); } + else if (Array.isArray(v)) v.forEach(walk); + else if (v && typeof v === 'object') Object.entries(v).forEach(([k,val])=>{walk(k);walk(val);}); + } + walk(value); return [...found.values()]; +} +function replaceVerified(value, verified) { + if (typeof value === 'string') return value.replace(URL_PATTERN, url => { const r = resourceFor(url); return r && verified.get(r.id)?.targetUrl || url; }); + if (Array.isArray(value)) return value.map(v => replaceVerified(v, verified)); + if (value && typeof value === 'object') return Object.fromEntries(Object.entries(value).map(([k, v]) => [replaceVerified(k, verified), replaceVerified(v, verified)])); + return value; +} +function quote(s) { return '"' + s.replaceAll('"', '""') + '"'; } +async function rewriteBatch(sql, column, pk, pkType, rows, verified, backup) { + const isJson=['json','jsonb'].includes(column.dataType); + const changes=[],originals=[]; + for(const row of rows){ + const next=replaceVerified(row.value,verified); + if(JSON.stringify(next)===JSON.stringify(row.value))continue; + originals.push({table:column.tableName,column:column.columnName,pkColumn:pk,pk:row.pk,value:row.value}); + changes.push({pk:String(row.pk),oldvalue:isJson?JSON.stringify(row.value):row.value,newvalue:isJson?JSON.stringify(next):next}); + } + if(!changes.length)return 0; + backup(originals); + const cast=isJson?'::'+column.dataType:''; + const updated=await sql.unsafe('update '+quote(column.tableName)+' as t set '+quote(column.columnName)+'=v.newvalue'+cast+' from jsonb_to_recordset($1::jsonb) as v(pk text,oldvalue text,newvalue text) where t.'+quote(pk)+'=v.pk::'+pkType+' and t.'+quote(column.columnName)+(isJson?'::jsonb=v.oldvalue::jsonb':'=v.oldvalue')+' returning t.'+quote(pk),[sql.json(changes)]); + return updated.length; +} +module.exports = { resourceFor, urlsIn, replaceVerified, rewriteBatch }; +if (require.main === module) main().catch(e => { console.error('migration_failed', e.name, String(e.message).replace(/https?:\/\/\S+/g, '[URL]').slice(0,240)); process.exitCode = 1; }); +async function main() { + const root = process.env.MIGRATION_APP_ROOT || process.cwd(); + require(path.join(root, 'node_modules/dotenv')).config({ path: path.join(root, '.env.local') }); + const postgres = require(path.join(root, 'node_modules/postgres')); + const { S3Client, PutObjectCommand, GetObjectCommand, HeadObjectCommand } = require(path.join(root, 'node_modules/@aws-sdk/client-s3')); + const sql = postgres(process.env.POSTGRES_URL, { max: 2, prepare: false, connection: { statement_timeout: 600000 } }); + const dir = process.env.MIGRATION_RUN_DIR || '/var/lib/ai-chatbot/aliyun-qiniu-20261014'; + fs.mkdirSync(dir, { recursive: true, mode: 0o700 }); + const journal = path.join(dir, 'verified.jsonl'), verified = new Map(); + if (fs.existsSync(journal)) for (const line of fs.readFileSync(journal,'utf8').split('\n').filter(Boolean)) { const r = JSON.parse(line); verified.set(r.id,r); } + let sourceClient, targetClient; + try { + const configs = await sql`select * from "StorageConfig"`; + const source = configs.find(c => c.bucket === 'demo-sparkai' && String(c.endpoint).includes('aliyuncs.com')); + const target = configs.find(c => c.type === 'qiniu-kodo' && c.isEnabled && c.isDefault); + if (!source || !target || !target.publicUrl) throw Error('Expected source and default Qiniu configuration unavailable'); + function client(c) { return new S3Client({ endpoint: c.endpoint, region: c.region || 'auto', forcePathStyle: true, credentials: { accessKeyId:c.accessKeyId,secretAccessKey:c.secretAccessKey }, requestChecksumCalculation:'WHEN_REQUIRED',responseChecksumValidation:'WHEN_REQUIRED', maxAttempts: 2 }); } + sourceClient = client(source); targetClient = client(target); + const publicUrl = key => target.publicUrl.replace(/\/$/,'') + '/' + key.split('/').map(encodeURIComponent).join('/'); + const command = process.argv[2] || 'audit'; + const settings = command==='migrate-psd-extra'?await sql`select id,key,value from "SystemSettings" where key in ('agent_psd_templates','agent_psd_fonts')`:await sql`select id,key,value from "SystemSettings" where value::text like ${'%'+SOURCE_HOST+'/%'}`; + const settingsResources = urlsIn(settings.map(s => s.value)); + console.log('settings_inventory', JSON.stringify({ rows: settings.length, keys: settings.map(s => s.key), uniqueObjects:settingsResources.length, alreadyVerified:verified.size })); + if (command === 'audit') return; + const resources = new Map(settingsResources.map(r => [r.id,r])); + const scope = command.endsWith('-all') ? 'all' : command==='migrate-psd-extra'?'psd-extra':command==='migrate-recovery'?'recovery':command==='migrate-literals'?'literals':'settings'; + if(command==='migrate-recovery'){ + resources.clear(); + for(const candidate of JSON.parse(fs.readFileSync(process.env.MIGRATION_RESOURCE_FILE,'utf8'))){ + const canonical=resourceFor(candidate.identity); + if(!canonical||canonical.id!==candidate.id)throw Error('Invalid recovery identity'); + const backup=new URL(candidate.fetchUrl); + if(!configs.some(c=>c.publicUrl&&backup.origin===new URL(c.publicUrl).origin&&backup.pathname.startsWith(new URL(c.publicUrl).pathname.replace(/\/$/,'')+'/')))throw Error('Recovery source is not a configured storage origin'); + resources.set(canonical.id,{...canonical,fetchUrl:candidate.fetchUrl}); + } + } + let columns = []; + if (scope === 'all') { + columns = await sql`select c.table_name as "tableName",c.column_name as "columnName",c.data_type as "dataType" from information_schema.columns c join information_schema.tables t on c.table_schema=t.table_schema and c.table_name=t.table_name where c.table_schema='public' and t.table_type='BASE TABLE' and c.is_generated='NEVER' and c.is_updatable='YES' and c.data_type in ('text','character varying','json','jsonb') order by c.table_name,c.ordinal_position`; + const excluded = new Set(['StorageConfig','StorageMigrationTask','ImageArchiveJob','__drizzle_migrations','SystemSettings','SystemSettingHistory','SystemSettingSnapshot']); + const tables = new Map(); + const inventoryJournal=path.join(dir,'inventory.jsonl'), doneFile=path.join(dir,'inventory-completed-tables.json'); + const completedTables=new Set(fs.existsSync(doneFile)?JSON.parse(fs.readFileSync(doneFile,'utf8')):[]); + if(fs.existsSync(inventoryJournal))for(const line of fs.readFileSync(inventoryJournal,'utf8').split('\n').filter(Boolean)){const r=JSON.parse(line);for(const canonical of urlsIn(r.identity))resources.set(canonical.id,canonical);} + for (const c of columns) if (!excluded.has(c.tableName)) { if (!tables.has(c.tableName)) tables.set(c.tableName,[]); tables.get(c.tableName).push(c); } + for (const [table, cols] of tables) { + if(completedTables.has(table)&&process.env.MIGRATION_RESCAN!=='1')continue; + console.log('scan_start',table); + const where = cols.map(c => quote(c.columnName)+'::text like $1').join(' or '); + let matched=0; + for await (const rows of sql.unsafe('select '+cols.map(c=>quote(c.columnName)).join(',')+' from '+quote(table)+' where '+where,['%'+SOURCE_HOST+'/%']).cursor(50)){const added=[];for (const row of rows) { matched++; for (const r of urlsIn(row)) if(!resources.has(r.id)){resources.set(r.id,r);added.push(r);} }if(added.length)fs.appendFileSync(inventoryJournal,added.map(r=>JSON.stringify(r)).join('\n')+'\n',{mode:0o600});} + completedTables.add(table);fs.writeFileSync(doneFile,JSON.stringify([...completedTables]),{mode:0o600}); + if (matched) console.log('database_inventory',JSON.stringify({table,rows:matched,uniqueObjects:resources.size})); + } + } + if(scope==='all'||scope==='literals'){ + if(scope==='literals')resources.clear(); + for (const folder of ['app','components','lib']) { + function walk(d) { for (const e of fs.readdirSync(d,{withFileTypes:true})) { const f=path.join(d,e.name); if(e.isDirectory())walk(f); else if(/\.(tsx?|jsx?)$/.test(f)) for(const r of urlsIn(fs.readFileSync(f,'utf8')))resources.set(r.id,r); } } + walk(path.join(root,folder)); + } + } + fs.writeFileSync(path.join(dir,'plan-'+scope+'.json'),JSON.stringify([...resources.values()]),{mode:0o600}); + if (command === 'inventory-all') return; + if (!['migrate-settings','migrate-all','migrate-psd-extra','migrate-recovery','migrate-literals','rewrite-all'].includes(command)) throw Error('Unknown command'); + if (scope === 'settings' && !fs.existsSync(path.join(dir,'settings-before.json'))) fs.writeFileSync(path.join(dir,'settings-before.json'),JSON.stringify(settings),{mode:0o600,flag:'wx'}); + const sourceMetadata=new Map(); const metadataFile=path.join(dir,'source-metadata.jsonl'); + if(fs.existsSync(metadataFile))for(const line of fs.readFileSync(metadataFile,'utf8').split('\n').filter(Boolean)){const r=JSON.parse(line);sourceMetadata.set(r.key,r);} + const base64url = value => Buffer.from(value).toString('base64').replace(/\+/g,'-').replace(/\//g,'_'); + async function cloudCopy(r) { + const fetchUrl=r.fetchUrl||r.identity; + const cached=r.fetchUrl||r.variant||new URL(r.identity).hostname!==SOURCE_HOST?null:sourceMetadata.get(r.sourceKey); + let size=cached?.size,md5=cached?.md5; + if(!r.variant&&!cached){const head=await fetch(fetchUrl,{method:'HEAD',headers:{'Accept-Encoding':'identity'},signal:AbortSignal.timeout(45000)});if(!head.ok)throw Error('Source HTTP '+head.status);size=Number(head.headers.get('content-length'));md5=String(head.headers.get('etag')||'').replaceAll('"','').toLowerCase();} + if(r.variant || !size || !/^[a-f0-9]{32}$/.test(md5||'')) { + const download=await fetch(fetchUrl,{headers:{'Accept-Encoding':'identity'},signal:AbortSignal.timeout(15*60*1000)}); + if(!download.ok)throw Error('Source HTTP '+download.status); + const hash=crypto.createHash('md5');size=0;for await(const chunk of Readable.fromWeb(download.body)){hash.update(chunk);size+=chunk.length;}md5=hash.digest('hex'); + } + if(!size)throw Error('Empty source object'); + const requestPath='/fetch/'+base64url(fetchUrl)+'/to/'+base64url(target.bucket+':'+r.targetKey); + const signature=base64url(crypto.createHmac('sha1',target.secretAccessKey).update(requestPath+'\n').digest()); + const fetchResult=await fetch('https://iovip.qbox.me'+requestPath,{method:'POST',headers:{Authorization:'QBox '+target.accessKeyId+':'+signature,'Content-Type':'application/x-www-form-urlencoded'},signal:AbortSignal.timeout(15*60*1000)}); + if(!fetchResult.ok)throw Error('Qiniu cloud fetch HTTP '+fetchResult.status); + const info=await fetchResult.json();if(Number(info.fsize)!==size)throw Error('Cloud target size mismatch'); + const targetUrl=publicUrl(r.targetKey); + const hashResponse=await fetch(targetUrl+'?qhash/md5',{signal:AbortSignal.timeout(45000)}); + if(!hashResponse.ok)throw Error('CDN hash verification HTTP '+hashResponse.status); + const targetDigest=await hashResponse.json(); + if(targetDigest.hash!==md5 || Number(targetDigest.fsize)!==size)throw Error('Cloud target checksum mismatch'); + const record={...r,targetUrl,size,md5,transport:'qiniu-cloud-fetch',verifiedAt:new Date().toISOString()};fs.appendFileSync(journal,JSON.stringify(record)+'\n',{mode:0o600});verified.set(r.id,record); + } + async function copy(r) { + if (verified.has(r.id)) return; + let last; + for (let attempt=1;attempt<=3;attempt++) { + let input; + try { + if(process.env.MIGRATION_TRANSPORT!=='stream'){await cloudCopy(r);return;} + const timeout=AbortSignal.timeout(15*60*1000); + const response=await fetch(r.identity,{signal:timeout,headers:{'Accept-Encoding':'identity'}}); + let size,type,disposition; + if (response.ok) { size=Number(response.headers.get('content-length'));type=response.headers.get('content-type');disposition=response.headers.get('content-disposition'); input=Readable.fromWeb(response.body); } + else { + await response.body?.cancel(); + if(r.variant)throw Error('Source variant HTTP '+response.status); + const get=await sourceClient.send(new GetObjectCommand({Bucket:source.bucket,Key:r.sourceKey}),{abortSignal:timeout});input=get.Body;size=get.ContentLength;type=get.ContentType;disposition=get.ContentDisposition; + } + if (!size || size>5*1024**3)throw Error('Unsupported source size '+size); + const hash=crypto.createHash('sha256');let readBytes=0; + const hashStream=new Transform({transform(chunk,enc,cb){hash.update(chunk);readBytes+=chunk.length;cb(null,chunk)}}); + input.on('error',e=>hashStream.destroy(e)); + await targetClient.send(new PutObjectCommand({Bucket:target.bucket,Key:r.targetKey,Body:input.pipe(hashStream),ContentLength:size,ContentType:type||'application/octet-stream',ContentDisposition:disposition||undefined,CacheControl:'public, max-age=31536000, immutable',Metadata:{'migration-source-id':r.id}}),{abortSignal:timeout}); + if(readBytes!==size)throw Error('Source size mismatch'); const sha256=hash.digest('hex'); + const get=await targetClient.send(new GetObjectCommand({Bucket:target.bucket,Key:r.targetKey}),{abortSignal:timeout});const targetHash=crypto.createHash('sha256');let storedBytes=0;for await(const chunk of get.Body){targetHash.update(chunk);storedBytes+=chunk.length;} + if(storedBytes!==size||targetHash.digest('hex')!==sha256)throw Error('Target checksum mismatch'); + const targetUrl=publicUrl(r.targetKey);const head=await fetch(targetUrl,{method:'HEAD',signal:AbortSignal.timeout(30000)}); + if(!head.ok||Number(head.headers.get('content-length'))!==size)throw Error('CDN verification HTTP '+head.status); + const record={...r,targetUrl,size,sha256,verifiedAt:new Date().toISOString()};fs.appendFileSync(journal,JSON.stringify(record)+'\n',{mode:0o600});verified.set(r.id,record);return; + } catch(e) { last=e;input?.destroy();if(attempt<3)await new Promise(resolve=>setTimeout(resolve,attempt*1000)); } + } + fs.appendFileSync(path.join(dir,'failures-'+scope+'.jsonl'),JSON.stringify({id:r.id,sourceKey:r.sourceKey,name:last.name,message:String(last.message).replace(/https?:\/\/\S+/g,'[URL]').slice(0,240)})+'\n',{mode:0o600}); + } + const psdIds=new Set(urlsIn(settings.find(s=>s.key==='agent_psd_templates')?.value).map(r=>r.id)); + const list=[...resources.values()].sort((a,b)=>(psdIds.has(a.id)?0:2)-(psdIds.has(b.id)?0:2)||Number(/\.(psd|psb)$/i.test(a.sourceKey))-Number(/\.(psd|psb)$/i.test(b.sourceKey))); + const pending=list.filter(r=>!verified.has(r.id)); + const concurrency=Math.max(1,Math.min(96,Number(process.env.MIGRATION_CONCURRENCY||64))); + let cursor=0,done=0;const start=Date.now(); + function progress(){const p={scope,total:list.length,processed:done,verified:verified.size,remaining:list.filter(r=>!verified.has(r.id)).length,elapsedSeconds:Math.round((Date.now()-start)/1000),verifiedBytes:[...verified.values()].reduce((n,r)=>n+r.size,0)};fs.writeFileSync(path.join(dir,'progress-'+scope+'.json'),JSON.stringify(p),{mode:0o600});console.log('copy_progress',JSON.stringify(p));} + const interval=setInterval(progress,30000); + try { if(command!=='rewrite-all')await Promise.all(Array.from({length:concurrency},async()=>{while(cursor!verified.has(r.id)); + // Only verified addresses are changed; unresolved references keep their original source. + let changedSettings=0; + for(const setting of settings){ + for(let attempt=0;attempt<4;attempt++){ + const [current]=await sql`select value from "SystemSettings" where id=${setting.id}`;const next=replaceVerified(current.value,verified);if(JSON.stringify(next)===JSON.stringify(current.value))break; + const updated=await sql`update "SystemSettings" set value=${sql.json(next)},"updatedAt"=now() where id=${setting.id} and value=${sql.json(current.value)} returning id`; + if(updated.length){changedSettings++;break;}if(attempt===3)throw Error('Concurrent setting update prevented replacement'); + } + } + if(scope==='all'){ + const backupFile=path.join(dir,'database-before.jsonl.gz'); + for(const c of columns){ + // Administrative audit snapshots preserve their original configuration values. + // User canvas/chat history remains in scope; obsolete configuration backups do not. + if(['StorageConfig','StorageMigrationTask','ImageArchiveJob','__drizzle_migrations','SystemSettings','SystemSettingHistory','SystemSettingSnapshot'].includes(c.tableName))continue; + const pks=await sql`select a.attname,a.atttypid::regtype::text as "pkType" from pg_index i join pg_attribute a on a.attrelid=i.indrelid and a.attnum=any(i.indkey) where i.indrelid=${'"'+c.tableName+'"'}::regclass and i.indisprimary`; + if(pks.length!==1){continue;} const pk=pks[0].attname; + let count=0; + const aliasGuard=c.tableName==='ImageArchiveTask'&&c.columnName==='sourceUrl'?" and coalesce(metadata->>'legacyOssMigrationRun','') <> '20261014'":''; + for await(const rows of sql.unsafe('select '+quote(pk)+' as pk,'+quote(c.columnName)+' as value from '+quote(c.tableName)+' where '+quote(c.columnName)+'::text like $1'+aliasGuard,['%'+SOURCE_HOST+'/%']).cursor(50)){ + // Flush before updating to retain the original data even on interruption. + count+=await rewriteBatch(sql,c,pk,pks[0].pkType,rows,verified,backups=>fs.appendFileSync(backupFile,require('node:zlib').gzipSync(backups.map(r=>JSON.stringify(r)).join('\n')+'\n',{level:6}),{mode:0o600})); + } + if(count)console.log('references_replaced',JSON.stringify({table:c.tableName,column:c.columnName,rows:count})); + } + } + const [migration]=await sql`select id,value from "SystemSettings" where key='oss_image_migration_config'`; + if(migration){const next={...migration.value,enabled:false,preferAliyunForUploads:false,targetStorageConfigId:target.id};await sql`update "SystemSettings" set value=${sql.json(next)},"updatedAt"=now() where id=${migration.id}`;} + await sql`update "StorageConfig" set "isEnabled"=false,"isDefault"=false where id=${source.id}`; + const result={scope,objects:list.length,verified:list.length-missing.length,missing:missing.length,changedSettings,defaultStorage:'qiniu-kodo',sourceRetained:true,completedAt:new Date().toISOString()};fs.writeFileSync(path.join(dir,'result-'+scope+'.json'),JSON.stringify(result,null,2));console.log('migration_result',JSON.stringify(result)); + if(missing.length)process.exitCode=2; + } finally {sourceClient?.destroy();targetClient?.destroy();await sql.end();} +} diff --git a/scripts/storage/migrate-aliyun-to-qiniu.test.cjs b/scripts/storage/migrate-aliyun-to-qiniu.test.cjs new file mode 100644 index 000000000..6b94a01bd --- /dev/null +++ b/scripts/storage/migrate-aliyun-to-qiniu.test.cjs @@ -0,0 +1,10 @@ +const test=require('node:test'),assert=require('node:assert/strict');const {resourceFor,urlsIn,replaceVerified}=require('./migrate-aliyun-to-qiniu.cjs'); +const origin='https://demo-sparkai.oss-cn-shanghai.aliyuncs.com'; +test('migrates nested PSD layer, font and media references without consuming Markdown punctuation',()=>{const doc={layers:[{sourceUrl:origin+'/uploads/layer.png'}],font:origin+'/fonts/font.ttf',text:'![image]('+origin+'/uploads/img.jpg)'};assert.equal(urlsIn(doc).length,3);assert.equal(urlsIn(doc)[2].sourceKey,'uploads/img.jpg');}); +test('expired authorization parameters do not alter identity; processing variants remain distinct',()=>{assert.equal(resourceFor(origin+'/a.png?Expires=1&Signature=old&OSSAccessKeyId=id').id,resourceFor(origin+'/a.png').id);assert.notEqual(resourceFor(origin+'/a.png?x-oss-process=image%2Fresize,w_400').id,resourceFor(origin+'/a.png').id);}); +test('only verified assets are replaced and other hosts remain intact',()=>{const a=origin+'/a.psd',b=origin+'/b.png',r=resourceFor(a),m=new Map([[r.id,{targetUrl:'https://cdn.example/legacy/a.psd'}]]);assert.deepEqual(replaceVerified({a,b,other:'https://other.example/a.psd'},m),{a:'https://cdn.example/legacy/a.psd',b,other:'https://other.example/a.psd'});}); +test('encoded object keys are safely canonicalized and variants are namespaced',()=>{const r=resourceFor(origin+'/uploads/%E5%9B%BE.png');assert.equal(r.sourceKey,'uploads/图.png');assert.equal(r.targetKey,'legacy-oss/demo-sparkai/uploads/图.png');assert.ok(resourceFor(origin+'/a.png?x-oss-process=image/resize,w_400').targetKey.includes('/variants/'));}); + +test('cache resolution suffixes are not part of the storage object key',()=>{const text=origin+'/uploads/img.png|||2K',r=urlsIn(text)[0];assert.equal(r.sourceKey,'uploads/img.png');const map=new Map([[r.id,{targetUrl:'https://cdn.example/img.png'}]]);assert.equal(replaceVerified(text,map),'https://cdn.example/img.png|||2K');}); + +test('URL keyed lookup maps stay in sync with migrated URL values',()=>{const key=origin+'/uploads/key.png|||2K',r=urlsIn(key)[0],map=new Map([[r.id,{targetUrl:'https://cdn.example/key.png'}]]);assert.equal(urlsIn({[key]:{width:1024}}).length,1);assert.deepEqual(replaceVerified({[key]:{width:1024}},map),{'https://cdn.example/key.png|||2K':{width:1024}});}); diff --git a/scripts/storage/register-aliyun-qiniu-aliases.cjs b/scripts/storage/register-aliyun-qiniu-aliases.cjs new file mode 100644 index 000000000..52f1aa3f8 --- /dev/null +++ b/scripts/storage/register-aliyun-qiniu-aliases.cjs @@ -0,0 +1,6 @@ +/* Publish verified legacy aliases through the application's existing completed-archive resolver. + * No source objects are deleted, and no pending archive jobs are created or overwritten. */ +const fs=require('fs'),path=require('path'),crypto=require('crypto');const root=process.env.MIGRATION_APP_ROOT||process.cwd(),dir=process.env.MIGRATION_RUN_DIR||'/var/lib/ai-chatbot/aliyun-qiniu-20261014';const {resourceFor}=require('./migrate-aliyun-to-qiniu.cjs');require(path.join(root,'node_modules/dotenv')).config({path:path.join(root,'.env.local')});const sql=require(path.join(root,'node_modules/postgres'))(process.env.POSTGRES_URL,{max:1,prepare:false}); +(async()=>{try{const [target]=await sql`select id,bucket,"publicUrl" from "StorageConfig" where type='qiniu-kodo' and "isDefault" and "isEnabled" limit 1`;if(!target?.publicUrl)throw Error('No default Qiniu target');const map=new Map();for(const line of fs.readFileSync(path.join(dir,'verified.jsonl'),'utf8').split('\n').filter(Boolean)){const r=JSON.parse(line),canonical=resourceFor(r.identity),expectedUrl=target.publicUrl.replace(/\/$/,'')+'/'+r.targetKey.split('/').map(encodeURIComponent).join('/');if(!canonical||canonical.id!==r.id||r.targetUrl!==expectedUrl||r.size<=0||(!r.md5&&!r.sha256))throw Error('Unverified storage alias');map.set(r.identity,r);}let inserted=0,processed=0,batch=[]; +async function flush(){if(!batch.length)return;const added=await sql.unsafe('insert into "ImageArchiveTask" (id,"sourceUrl","targetUrl","targetStorageConfigId",status,"attemptCount","archiveAfterAt","replaceAfterAt","archivedAt","replacedAt",metadata,"createdAt","updatedAt") select gen_random_uuid(),r.source,r.target,$2::uuid,\'replaced\',0,now(),now(),now(),now(),r.metadata,now(),now() from jsonb_to_recordset($1::jsonb) as r(source text,target text,metadata jsonb) on conflict ("sourceUrl") do nothing returning id',[sql.json(batch),target.id]);inserted+=added.length;processed+=batch.length;batch=[];if(processed%25000===0)console.log(JSON.stringify({processed,total:map.size,inserted}));} +for(const r of map.values()){batch.push({source:r.identity,target:r.targetUrl,metadata:{legacyOssMigrationRun:'20261014',migrationResourceId:r.id,migrationVerifiedAt:r.verifiedAt,verifiedBytes:r.size,verifiedChecksum:r.md5||r.sha256}});if(batch.length>=500)await flush();}await flush();const result={aliasResources:map.size,inserted,existingPreserved:map.size-inserted,pendingJobsCreated:0,completedAt:new Date().toISOString()};fs.writeFileSync(path.join(dir,'alias-registration.json'),JSON.stringify(result,null,2),{mode:0o600});console.log('alias_registration',JSON.stringify(result));}finally{await sql.end()}})().catch(e=>{console.error('alias_registration_failed',e.name,String(e.message).replace(/https?:\/\/\S+/g,'[URL]').slice(0,180));process.exitCode=1});