/** * 千问视频生成 Provider(通义万相 I2V) * 支持图片转视频功能 */ import { uploadFile } from "@/lib/storage/service"; /** * 将 base64 图片转换为 HTTP URL(上传到 S3) */ async function convertBase64ToUrl(base64Data: string): Promise { try { // 检查是否是 base64 data URL if (!base64Data.startsWith("data:image/")) { // 已经是 HTTP URL,直接返回 return base64Data; } console.log("[Qianwen Video] Converting base64 to URL..."); // 提取 base64 数据 const matches = base64Data.match(/^data:image\/(\w+);base64,(.+)$/); if (!matches) { throw new Error("Invalid base64 image format"); } const imageBuffer = Buffer.from(matches[2], "base64"); const filename = `qianwen-input-${Date.now()}.${matches[1]}`; // 上传到 S3 const uploadResult = await uploadFile(filename, imageBuffer as any); console.log("[Qianwen Video] Image uploaded to:", uploadResult.url); return uploadResult.url; } catch (error) { console.error("[Qianwen Video] Failed to convert base64 to URL:", error); throw new Error("图片处理失败,请重试"); } } interface QianwenVideoConfig { apiKey: string; baseURL: string; } interface VideoTaskResponse { output: { task_id: string; task_status?: string; video_url?: string; }; request_id: string; code?: string; message?: string; } /** * 提交视频生成任务 */ async function submitVideoTask( baseURL: string, apiKey: string, prompt: string, imageUrl: string, modelId: string ): Promise { const url = `${baseURL}/services/aigc/video-generation/generation`; console.log("[Qianwen Video] Submitting task to:", url); console.log("[Qianwen Video] Model:", modelId); console.log("[Qianwen Video] Prompt:", prompt); console.log("[Qianwen Video] Image URL:", imageUrl); const requestBody = { model: modelId, input: { prompt: prompt, img_url: imageUrl, // 注意:阿里云 API 使用 img_url 而不是 image_url }, }; console.log("[Qianwen Video] 完整请求体:", JSON.stringify(requestBody, null, 2)); const response = await fetch(url, { method: "POST", headers: { "Content-Type": "application/json", "Authorization": `Bearer ${apiKey}`, "X-DashScope-Async": "enable", // 启用异步模式 }, body: JSON.stringify(requestBody), }); if (!response.ok) { const errorText = await response.text(); console.error("[Qianwen Video] Submit failed:", response.status, errorText); throw new Error( `视频任务提交失败: ${response.status} ${response.statusText}: ${errorText}` ); } const data = await response.json(); console.log("[Qianwen Video] Task submitted, task_id:", data.output?.task_id); return data; } /** * 查询任务状态 */ async function fetchTaskStatus( baseURL: string, apiKey: string, taskId: string ): Promise { const url = `${baseURL}/tasks/${taskId}`; const response = await fetch(url, { method: "GET", headers: { "Authorization": `Bearer ${apiKey}`, }, }); if (!response.ok) { const errorText = await response.text(); throw new Error(`任务查询失败: ${response.status} ${errorText}`); } return await response.json(); } /** * 等待任务完成(轮询) */ async function waitForTaskComplete( baseURL: string, apiKey: string, taskId: string, maxWaitTime: number = 600000 // 10分钟 ): Promise { const startTime = Date.now(); const pollInterval = 3000; // 3秒轮询一次 console.log("[Qianwen Video] Waiting for task:", taskId); while (Date.now() - startTime < maxWaitTime) { const status = await fetchTaskStatus(baseURL, apiKey, taskId); console.log("[Qianwen Video] Task status:", status.output?.task_status); if (status.output?.task_status === "SUCCEEDED") { if (status.output?.video_url) { console.log("[Qianwen Video] Task completed, video URL:", status.output.video_url); return status.output.video_url; } else { throw new Error("任务完成但未返回视频URL"); } } if (status.output?.task_status === "FAILED") { throw new Error(`视频生成失败: ${status.message || "未知错误"}`); } // 等待后继续轮询 await new Promise((resolve) => setTimeout(resolve, pollInterval)); } throw new Error("视频生成超时,请稍后重试"); } /** * 下载视频并上传到 S3 */ async function downloadAndUploadVideo(videoUrl: string): Promise { try { console.log("[Qianwen Video] Downloading video from:", videoUrl.substring(0, 100) + "..."); // 下载视频 const response = await fetch(videoUrl); if (!response.ok) { throw new Error(`下载视频失败: ${response.status}`); } const videoBuffer = Buffer.from(await response.arrayBuffer()); console.log("[Qianwen Video] Video downloaded, size:", videoBuffer.length, "bytes"); // 上传到 S3 const filename = `qianwen-video-${Date.now()}.mp4`; const uploadResult = await uploadFile(filename, videoBuffer as any); console.log("[Qianwen Video] Video uploaded to S3:", uploadResult.url); return uploadResult.url; } catch (error) { console.error("[Qianwen Video] Failed to download/upload video:", error); // 如果失败,返回原始 URL return videoUrl; } } /** * 异步替换视频 URL(5分钟后) * 导出供 route handler 调用 */ export async function scheduleVideoUrlReplacement( chatId: string, messageId: string, tempVideoUrl: string, prompt: string ) { // 延迟 5 分钟后执行 setTimeout(async () => { try { console.log("[Qianwen Video] Starting video URL replacement for message:", messageId); // 下载并上传到 S3 const s3VideoUrl = await downloadAndUploadVideo(tempVideoUrl); // 更新数据库中的消息内容 const newMarkdown = formatVideoResponse(s3VideoUrl, prompt, true); // 调用数据库更新 API await fetch(`${process.env.NEXT_PUBLIC_APP_URL || "http://localhost:3000"}/api/chat/replace-video-url`, { method: "POST", headers: { "Content-Type": "application/json", }, body: JSON.stringify({ chatId, messageId, newContent: newMarkdown, }), }); console.log("[Qianwen Video] ✅ Video URL replaced successfully"); } catch (error) { console.error("[Qianwen Video] Failed to replace video URL:", error); } }, 5 * 60 * 1000); // 5 分钟 } /** * 格式化视频响应为 Markdown */ function formatVideoResponse(videoUrl: string, prompt: string, isPermanent: boolean = false): string { const storageNote = isPermanent ? "" : "\n\n_(视频链接将在5分钟内自动转存至永久存储)_"; return `### 🎬 视频生成完成 **提示词**: ${prompt}${storageNote} [下载视频](${videoUrl})`; } /** * 创建千问视频生成 Provider */ export function createQianwenVideo({ apiKey, baseURL }: QianwenVideoConfig) { return { chat: (modelId: string) => ({ doGenerate: async (options: any) => { console.log("[Qianwen Video] doGenerate called!"); console.log("[Qianwen Video] Messages received:", JSON.stringify(options.messages).substring(0, 500) + "..."); try { // 提取文本提示词 const lastMessage = options.messages[options.messages.length - 1]; const prompt = lastMessage?.content ?.filter((part: any) => part.type === "text") .map((part: any) => part.text) .join(""); // 提取图片 URL(支持 file 和 image_url 两种格式) const imagePart = lastMessage?.content?.find( (part: any) => part.type === "image_url" || part.type === "file" ); const rawImageUrl = imagePart?.type === "file" ? imagePart?.data : imagePart?.image_url?.url; if (!rawImageUrl) { throw new Error("请上传一张图片作为视频生成的起始帧"); } console.log("[Qianwen Video] Prompt:", prompt); console.log("[Qianwen Video] Image URL:", rawImageUrl); console.log("[Qianwen Video] Image part type:", imagePart?.type); // 1. 提交任务 const taskResponse = await submitVideoTask( baseURL, apiKey, prompt || "生成视频", rawImageUrl, modelId ); const taskId = taskResponse.output?.task_id; if (!taskId) { throw new Error("未能获取任务ID"); } // 2. 等待任务完成 const videoUrl = await waitForTaskComplete(baseURL, apiKey, taskId); // 3. 格式化响应 const markdownText = formatVideoResponse(videoUrl, prompt || "生成视频"); return { text: markdownText, finishReason: "stop" as const, usage: { promptTokens: 0, completionTokens: markdownText.length, }, }; } catch (error) { console.error("[Qianwen Video] Generation failed:", error); const errorMessage = error instanceof Error ? `视频生成失败: ${error.message}` : "视频生成失败,请稍后重试"; return { text: errorMessage, finishReason: "error" as const, usage: { promptTokens: 0, completionTokens: 0 }, }; } }, doStream: async function* (options: any) { console.log("[Qianwen Video] doStream called!"); try { // 提取文本提示词 const lastMessage = options.messages[options.messages.length - 1]; const prompt = lastMessage?.content ?.filter((part: any) => part.type === "text") .map((part: any) => part.text) .join(""); // 提取图片 URL(支持 file 和 image_url 两种格式) const imagePart = lastMessage?.content?.find( (part: any) => part.type === "image_url" || part.type === "file" ); const rawImageUrl = imagePart?.type === "file" ? imagePart?.data : imagePart?.image_url?.url; if (!rawImageUrl) { throw new Error("请上传一张图片作为视频生成的起始帧"); } console.log("[Qianwen Video Stream] Image URL:", rawImageUrl); console.log("[Qianwen Video Stream] Image part type:", imagePart?.type); // 1. 提交任务 const taskResponse = await submitVideoTask( baseURL, apiKey, prompt || "生成视频", rawImageUrl, modelId ); const taskId = taskResponse.output?.task_id; if (!taskId) { throw new Error("未能获取任务ID"); } // 2. 等待任务完成 const videoUrl = await waitForTaskComplete(baseURL, apiKey, taskId); // 3. 一次性发送完整的 Markdown(视频 URL 不大) const markdownText = formatVideoResponse(videoUrl, prompt || "生成视频"); yield { type: "textDelta" as const, textDelta: markdownText, }; yield { type: "finish" as const, finishReason: "stop" as const, usage: { promptTokens: 0, completionTokens: markdownText.length, }, }; } catch (error) { console.error("[Qianwen Video] Stream generation failed:", error); const errorMessage = error instanceof Error ? `视频生成失败: ${error.message}` : "视频生成失败,请稍后重试"; yield { type: "textDelta" as const, textDelta: errorMessage, }; yield { type: "finish" as const, finishReason: "error" as const, usage: { promptTokens: 0, completionTokens: 0 }, }; } }, modelId, provider: "qianwen-video", specificationVersion: "v1" as const, defaultObjectGenerationMode: "json" as const, }), healthCheck: async () => { return true; }, }; }