From 598f5f617a10d0fc3f261f37b8619279a6f0e1c3 Mon Sep 17 00:00:00 2001 From: 134 Date: Wed, 16 Sep 2026 23:03:37 +0800 Subject: [PATCH] feat(video-canvas): add tested Aliyun Wan first-frame adapter --- docs/video-canvas-aliyun-test-2026-09-16.md | 35 +++++++++++ docs/video-canvas-followup-2026-09-16.md | 2 + lib/ai/providers/aliyun-canvas-video.ts | 70 +++++++++++++++++++++ lib/ai/tools/generate-video.ts | 9 ++- lib/video-canvas/model-matching.ts | 1 + playwright.helpers.config.ts | 1 + tests/routes/aliyun-canvas-video.test.ts | 34 ++++++++++ 7 files changed, 151 insertions(+), 1 deletion(-) create mode 100644 docs/video-canvas-aliyun-test-2026-09-16.md create mode 100644 lib/ai/providers/aliyun-canvas-video.ts create mode 100644 tests/routes/aliyun-canvas-video.test.ts diff --git a/docs/video-canvas-aliyun-test-2026-09-16.md b/docs/video-canvas-aliyun-test-2026-09-16.md new file mode 100644 index 000000000..b0f03c1c1 --- /dev/null +++ b/docs/video-canvas-aliyun-test-2026-09-16.md @@ -0,0 +1,35 @@ +# 阿里云万相视频测试（2026-09-16） + +按“先用阿里云的视频测试”执行。沿用红色咖啡杯参考图与推进镜头提示词，真实调用阿里云百炼北京线路；只测试 `wan2.6-i2v-flash` 首帧生视频，未改线上默认模型，未部署。 + +## 实际结果 + +| 环节 | 结果 | 说明 | +|---|---|---| +| 原“千问视频”凭据 | 401 / invalid_api_key | 该配置下没有模型；不是已可用的视频线路 | +| 现有“阿里云”官方凭据 | 可用 | 用现有配置进行测试，没有展示或提交密钥 | +| 阿里云直接请求 | 成功，41.888 秒 | 任务 `ba336392-6ddb-41f0-8db5-33af2ab41396`，包含 10 秒轮询间隔；不是精确服务商计算时长 | +| 本站 `/api/canvas/generate/video` | 成功，39.166 秒 | 独立新任务 `593dced8-36e0-4a9a-8b3b-ff90abd603ff`；实际使用万相，未走 Veo | +| 成片文件 | 通过解码检查 | H264、1280×720、5.000 秒、1,118,864 字节；按请求生成无音轨视频 | +| 画布自动转存、刷新播放 | 本轮未验收 | 本地数据库隧道出现 CONNECT_TIMEOUT，画布读取超时；恢复后刷新时浏览器连接关闭。不能沿用上一轮 Veo 的浏览器通过结果作为万相本轮通过证据 | +| 画面效果 | 尚不达标 | 2 秒抽帧出现大小两个杯子叠影；提示词要求保持单个杯子，不能把接口成功当成产品一致性达标 | + +## 本轮接入 + +新增独立万相画布适配器，调用官方 `video-synthesis` 创建异步任务，再用任务 ID 查询；生成成功交给已有画布转存流程。不改旧聊天视频 Provider 的行为。 + +模型筛选仅开放已接入的 `qianwen-video / wan2.6-i2v-flash`。配置采用首帧模式、720P/1080P、2–15 秒整数、随参考图比例。单图精确传入 `img_url`；显式传入分辨率、时长和音频开关，不再依赖旧 Provider 的默认参数。首尾帧、多参考、外部音频、其他万相版本不因此标为已支持。 + +本轮相关回归共 35 项通过（34 项批次通过，随后新增 1 项错误处理检查并重跑 6 项适配器测试通过），源码 TypeScript 检查通过。覆盖提交参数、只读轮询、任务失败、无产物不成功、未实现模型不出现在候选中。未运行完整生产构建。 + +官方参数依据：[阿里云万相首帧 API 文档](https://help.aliyun.com/zh/model-studio/legacy-image-to-video-api-reference/)。本轮真实请求只覆盖 5 秒、720P、无声，其他官方支持参数没有逐个付费生成。 + +## 隔离范围与证据 + +本地工作树在 `codex/video-canvas-parity-0916`。数据库专用 schema `codex_wan_20260916` 只覆盖测试用 AiProvider/AiModel，复用现有官方凭据；生产 public 模型与服务商配置未修改。使用专用 QA 账号和测试画布。 + +原始证据位于 `.qa/video-aliyun/`：`direct.jsonl`、`canvas-result.json`、`media-probe.json`、`wan-canvas.mp4`、`wan-frame.png`、`tests.log`、`typecheck.log`。临时签名地址、账号凭据和原始环境文件不提交 Git。 + +阿里云任务在当前请求内轮询，275 秒内未完成会带任务 ID 报超时；尚未增加持久化任务队列或刷新后续查。当前通过范围是官方直连与本站完整生成 API，不包含浏览器一键生成到刷新恢复的完整闭环。 + +收尾已禁用专用测试账号并撤销会话，删除本轮隔离模型 schema、临时凭据，停止本轮开发服务及数据库隧道。未改生产服务商 Key 或模型默认项。 diff --git a/docs/video-canvas-followup-2026-09-16.md b/docs/video-canvas-followup-2026-09-16.md index a11ae04d7..5b8b304a9 100644 --- a/docs/video-canvas-followup-2026-09-16.md +++ b/docs/video-canvas-followup-2026-09-16.md @@ -2,6 +2,8 @@ 接续 [上一轮记录](video-canvas-completion-2026-09-16.md)。继续在独立本地工作树测试，未部署生产，未更改官方千问 3.8 默认模型。 +后续按用户要求测试阿里云，见 [万相真实测试记录](video-canvas-aliyun-test-2026-09-16.md)。 + ## 逐项完成 | 功能 | 查明的问题 | 本轮修复与验证 | diff --git a/lib/ai/providers/aliyun-canvas-video.ts b/lib/ai/providers/aliyun-canvas-video.ts new file mode 100644 index 000000000..9f8e51785 --- /dev/null +++ b/lib/ai/providers/aliyun-canvas-video.ts @@ -0,0 +1,70 @@ +import type { ExecuteDirectVideoGenerationParams } from "@/lib/ai/tools/generate-video"; + +// This adapter deliberately exposes only the official model whose contract we support. +export const ALIYUN_CANVAS_VIDEO_MODEL = "wan2.6-i2v-flash"; + +export function buildAliyunCanvasVideoRequest(params: ExecuteDirectVideoGenerationParams, modelId: string) { + if (modelId !== ALIYUN_CANVAS_VIDEO_MODEL) throw new Error("当前阿里云视频模型尚未接入画布"); + if (params.mode !== "first-frame" || params.images?.length !== 1) throw new Error("万相首帧生视频需要一张参考图"); + if (params.audioUrls?.length) throw new Error("此画布接入暂不支持参考音频"); + const duration = params.duration ?? 5; + const resolution = params.resolution ?? "720p"; + if (!Number.isInteger(duration) || duration < 2 || duration > 15) throw new Error("万相视频时长需为 2–15 秒整数"); + if (!["720p", "1080p"].includes(resolution)) throw new Error("万相视频仅支持 720P / 1080P"); + if (params.cameraFixed || params.returnLastFrame) throw new Error("此万相接入不支持固定镜头参数或返回尾帧参数"); + return { + model: modelId, + input: { prompt: params.prompt, img_url: params.images[0] }, + parameters: { + resolution: resolution.toUpperCase(), duration, + // The upstream derives aspect ratio from the input image, not a ratio parameter. + audio: params.generateAudio ?? false, + prompt_extend: false, watermark: params.watermark ?? false, + ...(params.seed != null ? { seed: params.seed } : {}), + }, + }; +} + +export async function executeAliyunCanvasVideo(params: ExecuteDirectVideoGenerationParams & { + modelId: string; apiKey: string; baseURL: string; +}, runtime = { fetch: globalThis.fetch, sleep: (ms: number) => new Promise(resolve => setTimeout(resolve, ms)) }) { + const body = buildAliyunCanvasVideoRequest(params, params.modelId); + const base = new URL(params.baseURL); + base.pathname = "/api/v1"; + base.search = ""; + base.hash = ""; + const headers = { Authorization: `Bearer ${params.apiKey}`, "Content-Type": "application/json", "X-DashScope-Async": "enable" }; + const deadline = Date.now() + 275_000; + let taskId = ""; + const request = async (path: string, init: RequestInit = {}) => { + const remaining = deadline - Date.now(); + if (remaining <= 0) throw new Error(`阿里云视频查询超时，任务 ${taskId}，请勿重复生成`); + const response = await runtime.fetch(`${base.toString().replace(/\/$/, "")}${path}`, { + ...init, headers, signal: AbortSignal.timeout(Math.min(30_000, remaining)), + }); + const data = await response.json(); + if (!response.ok) throw new Error(`阿里云视频请求失败（${response.status}）：${data.code || data.error?.code || "UnknownError"}${taskId ? `，任务 ${taskId}` : ""}`); + return data; + }; + const submitted = await request("/services/aigc/video-generation/video-synthesis", { method: "POST", body: JSON.stringify(body) }); + taskId = submitted.output?.task_id; + if (!taskId) throw new Error("阿里云未返回视频任务 ID"); + await params.onProgress?.(`阿里云视频任务已提交：${taskId}`); + while (Date.now() < deadline) { + await runtime.sleep(Math.min(5_000, Math.max(0, deadline - Date.now()))); + const result = await request(`/tasks/${encodeURIComponent(taskId)}`); + const output = result.output || {}; + if (output.task_status === "SUCCEEDED") { + if (!/^https?:\/\//i.test(output.video_url || "")) throw new Error("阿里云任务完成但没有返回视频"); + return { url: output.video_url as string, metadata: { + provider: "aliyun", taskId, requestId: result.request_id, + duration: body.parameters.duration, resolution: params.resolution ?? "720p", + generateAudio: body.parameters.audio, + } }; + } + if (["FAILED", "CANCELED", "UNKNOWN"].includes(output.task_status)) { + throw new Error(`阿里云视频生成失败：${output.code || result.code || output.task_status}，任务 ${taskId}`); + } + } + throw new Error(`阿里云视频查询超时，任务 ${taskId}，请勿重复生成`); +} diff --git a/lib/ai/tools/generate-video.ts b/lib/ai/tools/generate-video.ts index a6f7d6ac8..229627489 100644 --- a/lib/ai/tools/generate-video.ts +++ b/lib/ai/tools/generate-video.ts @@ -3,6 +3,7 @@ * 支持火山引擎视频生成的多种模式 */ +import { executeAliyunCanvasVideo } from "@/lib/ai/providers/aliyun-canvas-video"; import { getCanvasVideoModes, isExecutableCanvasVideoModel, supportsCanvasAudioVideo, validateCanvasVideoInputs, getCanvasVideoParameters } from "@/lib/video-canvas/model-matching"; import { tool } from "ai"; import { z } from "zod"; @@ -734,7 +735,13 @@ export async function executeDirectVideoGeneration( ); let videoMetadata: Record | null = null; - if (isAgnesVideoModel(model)) { + if (model.provider.type === "qianwen-video") { + const result = await executeAliyunCanvasVideo({ + ...params, duration: generationParams.duration, + modelId: model.model.modelId, apiKey: model.provider.apiKey, baseURL: model.provider.baseUrl, + }); + videoMetadata = { ...result.metadata, temporaryVideoUrl: result.url }; + } else if (isAgnesVideoModel(model)) { const agnesResult = await executeAgnesVideoGeneration({ apiKey: model.provider.apiKey, baseUrl: model.provider.baseUrl, diff --git a/lib/video-canvas/model-matching.ts b/lib/video-canvas/model-matching.ts index 7d732bc64..df26632c8 100644 --- a/lib/video-canvas/model-matching.ts +++ b/lib/video-canvas/model-matching.ts @@ -56,6 +56,7 @@ export function getCanvasVideoModes(model: CanvasModelDescriptor): VideoGenerati // Only adapters actually handled by the canvas video endpoint are selectable here. export function isExecutableCanvasVideoModel(model: CanvasModelDescriptor) { if (!isVideoModelDescriptor(model)) return false; + if (model.providerType === "qianwen-video" && model.modelId === "wan2.6-i2v-flash") return true; if (["volcano-video", "doubao-video", "atlascloud", "vip98k"].includes(model.providerType || "")) return true; const caps = record(model.capabilities); return model.providerType === "custom" && caps.kind === "video" && diff --git a/playwright.helpers.config.ts b/playwright.helpers.config.ts index 947e00f54..f741911f5 100644 --- a/playwright.helpers.config.ts +++ b/playwright.helpers.config.ts @@ -3,6 +3,7 @@ import { defineConfig } from "@playwright/test"; export default defineConfig({ testDir: "./tests", testMatch: [ + /routes\/aliyun-canvas-video\.test\.ts/, /routes\/video-canvas-followup\.test\.ts/, /routes\/video-canvas-tools\.test\.ts/, /routes\/video-canvas-save-queue\.test\.ts/, diff --git a/tests/routes/aliyun-canvas-video.test.ts b/tests/routes/aliyun-canvas-video.test.ts new file mode 100644 index 000000000..573847b93 --- /dev/null +++ b/tests/routes/aliyun-canvas-video.test.ts @@ -0,0 +1,34 @@ +import { test, expect } from '@playwright/test'; +import { buildAliyunCanvasVideoRequest, executeAliyunCanvasVideo } from '@/lib/ai/providers/aliyun-canvas-video'; +import { filterCanvasModels } from '@/lib/video-canvas/model-matching'; +const input = { modelId:'wan2.6-i2v-flash', prompt:'red mug', mode:'first-frame' as const, images:['https://example.com/mug.png'], duration:5, resolution:'720p' as const, ratio:'adaptive' as const, apiKey:'test',baseURL:'https://dashscope.aliyuncs.com/api/v1' }; +test('maps Wan first frame to official synthesis contract, preserving audio false', () => { + expect(buildAliyunCanvasVideoRequest({...input,generateAudio:false},input.modelId)).toEqual({model:input.modelId,input:{prompt:'red mug',img_url:input.images[0]},parameters:{resolution:'720P',duration:5,audio:false,prompt_extend:false,watermark:false}}); +}); +test('unsupported modes, models and parameter combinations fail before submission', () => { + for(const patch of [{mode:'first-last-frame',images:[...input.images,...input.images]},{duration:1},{duration:5.5},{resolution:'480p'},{audioUrls:['https://example.com/audio.mp3']},{returnLastFrame:true}]) expect(()=>buildAliyunCanvasVideoRequest({...input,...patch} as any,input.modelId)).toThrow(); + expect(()=>buildAliyunCanvasVideoRequest(input,'wan3-unknown')).toThrow(); +}); +test('only the implemented Wan model is selectable, only for first frame', () => { + const model={modelId:input.modelId,providerType:'qianwen-video',capabilities:{kind:'video',supportedModes:['first-frame']}}; + expect(filterCanvasModels([model],'video','first-frame')).toEqual([model]); + expect(filterCanvasModels([model],'video','first-last-frame')).toEqual([]); + expect(filterCanvasModels([{...model,modelId:'wan3-unknown'}],'video')).toEqual([]); +}); +test('polls accepted task without resubmitting, returns upstream artifact and metadata', async () => { + const calls:Array<{url:string;init:any}>=[];let polls=0; + const fetch:typeof globalThis.fetch=async(url,init)=>{calls.push({url:String(url),init});return Response.json(init?.method==='POST'?{output:{task_id:'task-1'}}:{output:++polls===1?{task_status:'RUNNING'}:{task_status:'SUCCEEDED',video_url:'https://example.com/video.mp4'},request_id:'req-1'});}; + const result=await executeAliyunCanvasVideo(input,{fetch,sleep:async()=>{}}); + expect(calls.map(c=>[c.url,c.init.method||'GET'])).toEqual([['https://dashscope.aliyuncs.com/api/v1/services/aigc/video-generation/video-synthesis','POST'],['https://dashscope.aliyuncs.com/api/v1/tasks/task-1','GET'],['https://dashscope.aliyuncs.com/api/v1/tasks/task-1','GET']]); + expect(result).toMatchObject({url:'https://example.com/video.mp4',metadata:{taskId:'task-1',duration:5,generateAudio:false}}); +}); +test('task failure is not returned as a successful video', async()=>{ + const fetch:typeof globalThis.fetch=async(_url,init)=>Response.json(init?.method==='POST'?{output:{task_id:'task-failed'}}:{output:{task_status:'FAILED',code:'InvalidParameter'}}); + await expect(executeAliyunCanvasVideo(input,{fetch,sleep:async()=>{}})).rejects.toThrow('InvalidParameter，任务 task-failed'); +}); +test('completed task without output is rejected and API auth errors do not expose secrets', async()=>{ + const missing:typeof globalThis.fetch=async(_url,init)=>Response.json(init?.method==='POST'?{output:{task_id:'empty'}}:{output:{task_status:'SUCCEEDED'}}); + await expect(executeAliyunCanvasVideo(input,{fetch:missing,sleep:async()=>{}})).rejects.toThrow('没有返回视频'); + const denied:typeof globalThis.fetch=async()=>Response.json({code:'InvalidApiKey',message:'private upstream diagnostic'}, {status:401}); + await expect(executeAliyunCanvasVideo(input,{fetch:denied,sleep:async()=>{}})).rejects.toThrow('阿里云视频请求失败（401）：InvalidApiKey'); +});