424 lines
12 KiB
TypeScript
424 lines
12 KiB
TypeScript
/**
|
||
* 千问视频生成 Provider（通义万相 I2V）
|
||
* 支持图片转视频功能
|
||
*/
|
||
|
||
import { uploadFile } from "@/lib/storage/service";
|
||
|
||
/**
|
||
* 将 base64 图片转换为 HTTP URL（上传到 S3）
|
||
*/
|
||
async function convertBase64ToUrl(base64Data: string): Promise<string> {
|
||
try {
|
||
// 检查是否是 base64 data URL
|
||
if (!base64Data.startsWith("data:image/")) {
|
||
// 已经是 HTTP URL，直接返回
|
||
return base64Data;
|
||
}
|
||
|
||
console.log("[Qianwen Video] Converting base64 to URL...");
|
||
|
||
// 提取 base64 数据
|
||
const matches = base64Data.match(/^data:image\/(\w+);base64,(.+)$/);
|
||
if (!matches) {
|
||
throw new Error("Invalid base64 image format");
|
||
}
|
||
|
||
const imageBuffer = Buffer.from(matches[2], "base64");
|
||
const filename = `qianwen-input-${Date.now()}.${matches[1]}`;
|
||
|
||
// 上传到 S3
|
||
const uploadResult = await uploadFile(filename, imageBuffer as any);
|
||
console.log("[Qianwen Video] Image uploaded to:", uploadResult.url);
|
||
|
||
return uploadResult.url;
|
||
} catch (error) {
|
||
console.error("[Qianwen Video] Failed to convert base64 to URL:", error);
|
||
throw new Error("图片处理失败，请重试");
|
||
}
|
||
}
|
||
|
||
interface QianwenVideoConfig {
|
||
apiKey: string;
|
||
baseURL: string;
|
||
}
|
||
|
||
interface VideoTaskResponse {
|
||
output: {
|
||
task_id: string;
|
||
task_status?: string;
|
||
video_url?: string;
|
||
};
|
||
request_id: string;
|
||
code?: string;
|
||
message?: string;
|
||
}
|
||
|
||
/**
|
||
* 提交视频生成任务
|
||
*/
|
||
async function submitVideoTask(
|
||
baseURL: string,
|
||
apiKey: string,
|
||
prompt: string,
|
||
imageUrl: string,
|
||
modelId: string
|
||
): Promise<VideoTaskResponse> {
|
||
const url = `${baseURL}/services/aigc/video-generation/generation`;
|
||
|
||
console.log("[Qianwen Video] Submitting task to:", url);
|
||
console.log("[Qianwen Video] Model:", modelId);
|
||
console.log("[Qianwen Video] Prompt:", prompt);
|
||
console.log("[Qianwen Video] Image URL:", imageUrl);
|
||
|
||
const requestBody = {
|
||
model: modelId,
|
||
input: {
|
||
prompt: prompt,
|
||
img_url: imageUrl, // 注意：阿里云 API 使用 img_url 而不是 image_url
|
||
},
|
||
};
|
||
|
||
console.log("[Qianwen Video] 完整请求体:", JSON.stringify(requestBody, null, 2));
|
||
|
||
const response = await fetch(url, {
|
||
method: "POST",
|
||
headers: {
|
||
"Content-Type": "application/json",
|
||
"Authorization": `Bearer ${apiKey}`,
|
||
"X-DashScope-Async": "enable", // 启用异步模式
|
||
},
|
||
body: JSON.stringify(requestBody),
|
||
});
|
||
|
||
if (!response.ok) {
|
||
const errorText = await response.text();
|
||
console.error("[Qianwen Video] Submit failed:", response.status, errorText);
|
||
throw new Error(
|
||
`视频任务提交失败: ${response.status} ${response.statusText}: ${errorText}`
|
||
);
|
||
}
|
||
|
||
const data = await response.json();
|
||
console.log("[Qianwen Video] Task submitted, task_id:", data.output?.task_id);
|
||
|
||
return data;
|
||
}
|
||
|
||
/**
|
||
* 查询任务状态
|
||
*/
|
||
async function fetchTaskStatus(
|
||
baseURL: string,
|
||
apiKey: string,
|
||
taskId: string
|
||
): Promise<VideoTaskResponse> {
|
||
const url = `${baseURL}/tasks/${taskId}`;
|
||
|
||
const response = await fetch(url, {
|
||
method: "GET",
|
||
headers: {
|
||
"Authorization": `Bearer ${apiKey}`,
|
||
},
|
||
});
|
||
|
||
if (!response.ok) {
|
||
const errorText = await response.text();
|
||
throw new Error(`任务查询失败: ${response.status} ${errorText}`);
|
||
}
|
||
|
||
return await response.json();
|
||
}
|
||
|
||
/**
|
||
* 等待任务完成（轮询）
|
||
*/
|
||
async function waitForTaskComplete(
|
||
baseURL: string,
|
||
apiKey: string,
|
||
taskId: string,
|
||
maxWaitTime: number = 600000 // 10分钟
|
||
): Promise<string> {
|
||
const startTime = Date.now();
|
||
const pollInterval = 3000; // 3秒轮询一次
|
||
|
||
console.log("[Qianwen Video] Waiting for task:", taskId);
|
||
|
||
while (Date.now() - startTime < maxWaitTime) {
|
||
const status = await fetchTaskStatus(baseURL, apiKey, taskId);
|
||
|
||
console.log("[Qianwen Video] Task status:", status.output?.task_status);
|
||
|
||
if (status.output?.task_status === "SUCCEEDED") {
|
||
if (status.output?.video_url) {
|
||
console.log("[Qianwen Video] Task completed, video URL:", status.output.video_url);
|
||
return status.output.video_url;
|
||
} else {
|
||
throw new Error("任务完成但未返回视频URL");
|
||
}
|
||
}
|
||
|
||
if (status.output?.task_status === "FAILED") {
|
||
throw new Error(`视频生成失败: ${status.message || "未知错误"}`);
|
||
}
|
||
|
||
// 等待后继续轮询
|
||
await new Promise((resolve) => setTimeout(resolve, pollInterval));
|
||
}
|
||
|
||
throw new Error("视频生成超时，请稍后重试");
|
||
}
|
||
|
||
/**
|
||
* 下载视频并上传到 S3
|
||
*/
|
||
async function downloadAndUploadVideo(videoUrl: string): Promise<string> {
|
||
try {
|
||
console.log("[Qianwen Video] Downloading video from:", videoUrl.substring(0, 100) + "...");
|
||
|
||
// 下载视频
|
||
const response = await fetch(videoUrl);
|
||
if (!response.ok) {
|
||
throw new Error(`下载视频失败: ${response.status}`);
|
||
}
|
||
|
||
const videoBuffer = Buffer.from(await response.arrayBuffer());
|
||
console.log("[Qianwen Video] Video downloaded, size:", videoBuffer.length, "bytes");
|
||
|
||
// 上传到 S3
|
||
const filename = `qianwen-video-${Date.now()}.mp4`;
|
||
const uploadResult = await uploadFile(filename, videoBuffer as any);
|
||
|
||
console.log("[Qianwen Video] Video uploaded to S3:", uploadResult.url);
|
||
return uploadResult.url;
|
||
} catch (error) {
|
||
console.error("[Qianwen Video] Failed to download/upload video:", error);
|
||
// 如果失败，返回原始 URL
|
||
return videoUrl;
|
||
}
|
||
}
|
||
|
||
/**
|
||
* 异步替换视频 URL（5分钟后）
|
||
* 导出供 route handler 调用
|
||
*/
|
||
export async function scheduleVideoUrlReplacement(
|
||
chatId: string,
|
||
messageId: string,
|
||
tempVideoUrl: string,
|
||
prompt: string
|
||
) {
|
||
// 延迟 5 分钟后执行
|
||
setTimeout(async () => {
|
||
try {
|
||
console.log("[Qianwen Video] Starting video URL replacement for message:", messageId);
|
||
|
||
// 下载并上传到 S3
|
||
const s3VideoUrl = await downloadAndUploadVideo(tempVideoUrl);
|
||
|
||
// 更新数据库中的消息内容
|
||
const newMarkdown = formatVideoResponse(s3VideoUrl, prompt, true);
|
||
|
||
// 调用数据库更新 API
|
||
await fetch(`${process.env.NEXT_PUBLIC_APP_URL || "http://localhost:3000"}/api/chat/replace-video-url`, {
|
||
method: "POST",
|
||
headers: {
|
||
"Content-Type": "application/json",
|
||
},
|
||
body: JSON.stringify({
|
||
chatId,
|
||
messageId,
|
||
newContent: newMarkdown,
|
||
}),
|
||
});
|
||
|
||
console.log("[Qianwen Video] ✅ Video URL replaced successfully");
|
||
} catch (error) {
|
||
console.error("[Qianwen Video] Failed to replace video URL:", error);
|
||
}
|
||
}, 5 * 60 * 1000); // 5 分钟
|
||
}
|
||
|
||
/**
|
||
* 格式化视频响应为 Markdown
|
||
*/
|
||
function formatVideoResponse(videoUrl: string, prompt: string, isPermanent: boolean = false): string {
|
||
const storageNote = isPermanent ? "" : "\n\n_（视频链接将在5分钟内自动转存至永久存储）_";
|
||
|
||
return `### 🎬 视频生成完成
|
||
|
||
**提示词**: ${prompt}${storageNote}
|
||
|
||
<video src="${videoUrl}" controls style="max-width: 100%; border-radius: 8px;"></video>
|
||
|
||
[下载视频](${videoUrl})`;
|
||
}
|
||
|
||
/**
|
||
* 创建千问视频生成 Provider
|
||
*/
|
||
export function createQianwenVideo({ apiKey, baseURL }: QianwenVideoConfig) {
|
||
return {
|
||
chat: (modelId: string) => ({
|
||
doGenerate: async (options: any) => {
|
||
console.log("[Qianwen Video] doGenerate called!");
|
||
console.log("[Qianwen Video] Messages received:", JSON.stringify(options.messages).substring(0, 500) + "...");
|
||
|
||
try {
|
||
// 提取文本提示词
|
||
const lastMessage = options.messages[options.messages.length - 1];
|
||
const prompt = lastMessage?.content
|
||
?.filter((part: any) => part.type === "text")
|
||
.map((part: any) => part.text)
|
||
.join("");
|
||
|
||
// 提取图片 URL（支持 file 和 image_url 两种格式）
|
||
const imagePart = lastMessage?.content?.find(
|
||
(part: any) => part.type === "image_url" || part.type === "file"
|
||
);
|
||
const rawImageUrl = imagePart?.type === "file"
|
||
? imagePart?.data
|
||
: imagePart?.image_url?.url;
|
||
|
||
if (!rawImageUrl) {
|
||
throw new Error("请上传一张图片作为视频生成的起始帧");
|
||
}
|
||
|
||
console.log("[Qianwen Video] Prompt:", prompt);
|
||
console.log("[Qianwen Video] Image URL:", rawImageUrl);
|
||
console.log("[Qianwen Video] Image part type:", imagePart?.type);
|
||
|
||
// 1. 提交任务
|
||
const taskResponse = await submitVideoTask(
|
||
baseURL,
|
||
apiKey,
|
||
prompt || "生成视频",
|
||
rawImageUrl,
|
||
modelId
|
||
);
|
||
|
||
const taskId = taskResponse.output?.task_id;
|
||
if (!taskId) {
|
||
throw new Error("未能获取任务ID");
|
||
}
|
||
|
||
// 2. 等待任务完成
|
||
const videoUrl = await waitForTaskComplete(baseURL, apiKey, taskId);
|
||
|
||
// 3. 格式化响应
|
||
const markdownText = formatVideoResponse(videoUrl, prompt || "生成视频");
|
||
|
||
return {
|
||
text: markdownText,
|
||
finishReason: "stop" as const,
|
||
usage: {
|
||
promptTokens: 0,
|
||
completionTokens: markdownText.length,
|
||
},
|
||
};
|
||
} catch (error) {
|
||
console.error("[Qianwen Video] Generation failed:", error);
|
||
const errorMessage =
|
||
error instanceof Error
|
||
? `视频生成失败: ${error.message}`
|
||
: "视频生成失败，请稍后重试";
|
||
return {
|
||
text: errorMessage,
|
||
finishReason: "error" as const,
|
||
usage: { promptTokens: 0, completionTokens: 0 },
|
||
};
|
||
}
|
||
},
|
||
|
||
doStream: async function* (options: any) {
|
||
console.log("[Qianwen Video] doStream called!");
|
||
|
||
try {
|
||
// 提取文本提示词
|
||
const lastMessage = options.messages[options.messages.length - 1];
|
||
const prompt = lastMessage?.content
|
||
?.filter((part: any) => part.type === "text")
|
||
.map((part: any) => part.text)
|
||
.join("");
|
||
|
||
// 提取图片 URL（支持 file 和 image_url 两种格式）
|
||
const imagePart = lastMessage?.content?.find(
|
||
(part: any) => part.type === "image_url" || part.type === "file"
|
||
);
|
||
const rawImageUrl = imagePart?.type === "file"
|
||
? imagePart?.data
|
||
: imagePart?.image_url?.url;
|
||
|
||
if (!rawImageUrl) {
|
||
throw new Error("请上传一张图片作为视频生成的起始帧");
|
||
}
|
||
|
||
console.log("[Qianwen Video Stream] Image URL:", rawImageUrl);
|
||
console.log("[Qianwen Video Stream] Image part type:", imagePart?.type);
|
||
|
||
// 1. 提交任务
|
||
const taskResponse = await submitVideoTask(
|
||
baseURL,
|
||
apiKey,
|
||
prompt || "生成视频",
|
||
rawImageUrl,
|
||
modelId
|
||
);
|
||
|
||
const taskId = taskResponse.output?.task_id;
|
||
if (!taskId) {
|
||
throw new Error("未能获取任务ID");
|
||
}
|
||
|
||
// 2. 等待任务完成
|
||
const videoUrl = await waitForTaskComplete(baseURL, apiKey, taskId);
|
||
|
||
// 3. 一次性发送完整的 Markdown（视频 URL 不大）
|
||
const markdownText = formatVideoResponse(videoUrl, prompt || "生成视频");
|
||
|
||
yield {
|
||
type: "textDelta" as const,
|
||
textDelta: markdownText,
|
||
};
|
||
|
||
yield {
|
||
type: "finish" as const,
|
||
finishReason: "stop" as const,
|
||
usage: {
|
||
promptTokens: 0,
|
||
completionTokens: markdownText.length,
|
||
},
|
||
};
|
||
} catch (error) {
|
||
console.error("[Qianwen Video] Stream generation failed:", error);
|
||
|
||
const errorMessage =
|
||
error instanceof Error
|
||
? `视频生成失败: ${error.message}`
|
||
: "视频生成失败，请稍后重试";
|
||
|
||
yield {
|
||
type: "textDelta" as const,
|
||
textDelta: errorMessage,
|
||
};
|
||
|
||
yield {
|
||
type: "finish" as const,
|
||
finishReason: "error" as const,
|
||
usage: { promptTokens: 0, completionTokens: 0 },
|
||
};
|
||
}
|
||
},
|
||
|
||
modelId,
|
||
provider: "qianwen-video",
|
||
specificationVersion: "v1" as const,
|
||
defaultObjectGenerationMode: "json" as const,
|
||
}),
|
||
healthCheck: async () => {
|
||
return true;
|
||
},
|
||
};
|
||
}
|
||
|