4992 lines
187 KiB
TypeScript
4992 lines
187 KiB
TypeScript
import { geolocation } from "@vercel/functions";
|
||
import {
|
||
convertToModelMessages,
|
||
createUIMessageStream,
|
||
generateText,
|
||
JsonToSseTransformStream,
|
||
smoothStream,
|
||
stepCountIs,
|
||
streamText,
|
||
} from "ai";
|
||
import { unstable_cache as cache } from "next/cache";
|
||
import { after, NextResponse } from "next/server";
|
||
import {
|
||
createResumableStreamContext,
|
||
type ResumableStreamContext,
|
||
} from "resumable-stream";
|
||
import type { ModelCatalog } from "tokenlens/core";
|
||
import { fetchModels } from "tokenlens/fetch";
|
||
import { getUsage } from "tokenlens/helpers";
|
||
import { auth, type UserType } from "@/app/(auth)/auth";
|
||
import type { VisibilityType } from "@/components/visibility-selector";
|
||
import { entitlementsByUserType } from "@/lib/ai/entitlements";
|
||
import type { ChatModel } from "@/lib/ai/models";
|
||
import { type RequestHints, systemPrompt } from "@/lib/ai/prompts";
|
||
import { myProvider } from "@/lib/ai/providers";
|
||
import { getLanguageModelById } from "@/lib/ai/dynamic-provider";
|
||
import { createDocument } from "@/lib/ai/tools/create-document";
|
||
import { getWeather } from "@/lib/ai/tools/get-weather";
|
||
import { requestSuggestions } from "@/lib/ai/tools/request-suggestions";
|
||
import { updateDocument } from "@/lib/ai/tools/update-document";
|
||
import { generateVideo } from "@/lib/ai/tools/generate-video";
|
||
import { getEnabledTools } from "@/lib/ai/tools-registry";
|
||
import { isProductionEnvironment } from "@/lib/constants";
|
||
import {
|
||
createStreamId,
|
||
deleteChatById,
|
||
getChatById,
|
||
getPointsSystemConfig,
|
||
getUserById,
|
||
getMessageCountByUserId,
|
||
getMessagesByChatId,
|
||
saveChat,
|
||
saveMessages,
|
||
updateMessageById,
|
||
updateChatLastContextById,
|
||
getAiModelById,
|
||
getEnabledAiModelsWithProvider,
|
||
getSubModelsByParentModelId,
|
||
getAiProviderById,
|
||
checkUserImageQuota,
|
||
consumeUserPoints,
|
||
} from "@/lib/db/queries";
|
||
import { db } from "@/lib/db";
|
||
import type { DBMessage } from "@/lib/db/schema";
|
||
import { generatedImage, message as messageTable } from "@/lib/db/schema";
|
||
import { ChatSDKError } from "@/lib/errors";
|
||
import { extractImageUrlsFromText } from "@/lib/image-url";
|
||
import { videoArchiver } from "@/lib/storage/video-archiver";
|
||
import { uploadFile } from "@/lib/storage/service";
|
||
import type { ChatMessage } from "@/lib/types";
|
||
import type { AppUsage } from "@/lib/usage";
|
||
import { convertToUIMessages, generateUUID } from "@/lib/utils";
|
||
import { isAiModelAgentAuto } from "@/lib/db/ai-model-flags";
|
||
import { generateTitleFromUserMessage } from "../../actions";
|
||
import { type PostRequestBody, postRequestBodySchema } from "./schema";
|
||
|
||
/**
|
||
* 将图片 URL 转换为 base64 格式
|
||
* 用于将 S3 URL 的图片转换为 base64 发送给 Gemini API
|
||
*/
|
||
async function convertImageUrlToBase64(url: string): Promise<string> {
|
||
try {
|
||
console.log("[Image Convert] Fetching image from:", url);
|
||
const response = await fetch(url);
|
||
if (!response.ok) {
|
||
throw new Error(`Failed to fetch image: ${response.statusText}`);
|
||
}
|
||
|
||
const arrayBuffer = await response.arrayBuffer();
|
||
const buffer = Buffer.from(arrayBuffer);
|
||
const base64 = buffer.toString("base64");
|
||
|
||
// 获取 MIME 类型
|
||
const contentType = response.headers.get("content-type") || "image/jpeg";
|
||
const dataUrl = `data:${contentType};base64,${base64}`;
|
||
|
||
console.log("[Image Convert] Converted to base64, size:", base64.length);
|
||
return dataUrl;
|
||
} catch (error) {
|
||
console.error("[Image Convert] Failed:", error);
|
||
throw error;
|
||
}
|
||
}
|
||
|
||
/**
|
||
* 检测消息中是否包含用户标注过的图片（文件名以 annotated- 开头）
|
||
*/
|
||
function hasAnnotatedImage(messages: ChatMessage[]): boolean {
|
||
for (const message of messages) {
|
||
for (const part of message.parts) {
|
||
if (part.type === "file" && "name" in part && typeof part.name === "string") {
|
||
if (part.name.startsWith("annotated-")) {
|
||
return true;
|
||
}
|
||
}
|
||
}
|
||
}
|
||
return false;
|
||
}
|
||
|
||
/**
|
||
* 标注图片的系统提示词（用户不可见，后台自动添加）
|
||
*/
|
||
const ANNOTATION_SYSTEM_PROMPT = "【重要】用户提供的图片上有标注线条（如箭头、圆圈、画笔痕迹或文字说明），这些标注是用户用来指示需要修改的区域或方向。请仔细理解标注的含义，按照标注指示进行编辑，但在生成的新图片中不要保留这些标注线条，输出干净的编辑结果。";
|
||
|
||
function applyHiddenPromptToMessages(messages: ChatMessage[], hiddenPrompt?: string): ChatMessage[] {
|
||
if (!hiddenPrompt || !hiddenPrompt.trim()) return messages;
|
||
const cloned = messages.map((msg) => ({
|
||
...msg,
|
||
parts: msg.parts.map((part) => ({ ...part })),
|
||
}));
|
||
for (let i = cloned.length - 1; i >= 0; i--) {
|
||
const msg = cloned[i];
|
||
if (msg.role !== "user") continue;
|
||
const textPart = msg.parts.find((part) => part.type === "text") as { type: "text"; text: string } | undefined;
|
||
if (textPart) {
|
||
textPart.text = `${hiddenPrompt}\n\n${textPart.text}`.trim();
|
||
} else {
|
||
msg.parts.unshift({ type: "text", text: hiddenPrompt });
|
||
}
|
||
break;
|
||
}
|
||
return cloned;
|
||
}
|
||
|
||
function applyOptimizedPromptToLatestUserMessage(
|
||
messages: ChatMessage[],
|
||
optimizedPrompt?: string,
|
||
hiddenPromptPrefix = "",
|
||
): ChatMessage[] {
|
||
const prompt = optimizedPrompt?.trim();
|
||
if (!prompt) return messages;
|
||
|
||
const cloned = messages.map((msg) => ({
|
||
...msg,
|
||
parts: msg.parts.map((part) => ({ ...part })),
|
||
}));
|
||
|
||
for (let i = cloned.length - 1; i >= 0; i -= 1) {
|
||
const msg = cloned[i];
|
||
if (msg.role !== "user") continue;
|
||
|
||
const textPart = msg.parts.find(
|
||
(part): part is { type: "text"; text: string } => part.type === "text",
|
||
);
|
||
const originalText = textPart?.text?.trim() || "";
|
||
const normalizedOriginal = hiddenPromptPrefix && originalText.startsWith(hiddenPromptPrefix)
|
||
? originalText.slice(hiddenPromptPrefix.length).trim()
|
||
: originalText;
|
||
const hasOriginalDemand = normalizedOriginal.length > 0;
|
||
const hasOriginalInOptimized =
|
||
hasOriginalDemand &&
|
||
(prompt.includes(normalizedOriginal) || normalizedOriginal.includes(prompt));
|
||
const finalPrompt = hasOriginalDemand && !hasOriginalInOptimized
|
||
? [
|
||
hiddenPromptPrefix.trim(),
|
||
`【核心目标（必须优先满足）】${normalizedOriginal}`,
|
||
`【执行优化细节】${prompt}`,
|
||
]
|
||
.filter(Boolean)
|
||
.join("\n\n")
|
||
.trim()
|
||
: `${hiddenPromptPrefix}${prompt}`.trim();
|
||
console.log("[Prompt Merge] Final image prompt prepared:", {
|
||
original: normalizedOriginal.substring(0, 120),
|
||
optimized: prompt.substring(0, 120),
|
||
merged: finalPrompt.substring(0, 180),
|
||
forcePreserveOriginal: hasOriginalDemand && !hasOriginalInOptimized,
|
||
});
|
||
|
||
if (textPart) {
|
||
textPart.text = finalPrompt;
|
||
} else {
|
||
msg.parts.unshift({ type: "text", text: finalPrompt });
|
||
}
|
||
break;
|
||
}
|
||
|
||
return cloned;
|
||
}
|
||
|
||
type StreamWriter = {
|
||
write: (chunk: any) => void;
|
||
};
|
||
|
||
function safeWriteStreamChunk(writer: StreamWriter, chunk: any, label: string): boolean {
|
||
try {
|
||
writer.write(chunk);
|
||
return true;
|
||
} catch (error) {
|
||
console.warn(`[Chat API] Stream write skipped (${label})`, error);
|
||
return false;
|
||
}
|
||
}
|
||
|
||
function writeAssistantTextChunk(
|
||
writer: StreamWriter,
|
||
text: string,
|
||
label: string,
|
||
messageId = generateUUID(),
|
||
) {
|
||
safeWriteStreamChunk(
|
||
writer,
|
||
{
|
||
type: "data-appendMessage",
|
||
data: JSON.stringify({
|
||
id: messageId,
|
||
role: "assistant",
|
||
parts: [{ type: "text", text }],
|
||
}),
|
||
},
|
||
label,
|
||
);
|
||
return messageId;
|
||
}
|
||
|
||
function createSseResponse(stream: ReadableStream) {
|
||
return new Response(stream.pipeThrough(new JsonToSseTransformStream()), {
|
||
headers: {
|
||
"Content-Type": "text/event-stream; charset=utf-8",
|
||
"Cache-Control": "no-cache, no-transform",
|
||
Connection: "keep-alive",
|
||
"X-Accel-Buffering": "no",
|
||
},
|
||
});
|
||
}
|
||
|
||
function startStreamKeepalive(writer: StreamWriter, label: string, intervalMs = 15000) {
|
||
const timer = setInterval(() => {
|
||
safeWriteStreamChunk(
|
||
writer,
|
||
{
|
||
type: "data-keepalive",
|
||
data: JSON.stringify({ ts: Date.now(), label }),
|
||
},
|
||
`${label}:keepalive`,
|
||
);
|
||
}, intervalMs);
|
||
|
||
return () => clearInterval(timer);
|
||
}
|
||
|
||
type VideoModeType = "text-to-video" | "first-frame" | "first-last-frame" | "reference-images";
|
||
|
||
const VIDEO_PROVIDER_TYPES = new Set([
|
||
"volcano-video",
|
||
"doubao-video",
|
||
"qianwen-video",
|
||
"fal-video",
|
||
]);
|
||
|
||
function isVideoProviderType(providerType?: string | null): boolean {
|
||
return Boolean(providerType && VIDEO_PROVIDER_TYPES.has(providerType));
|
||
}
|
||
|
||
function inferSupportedVideoModesByModelId(modelId: string): VideoModeType[] {
|
||
const normalized = modelId.toLowerCase();
|
||
|
||
if (normalized.includes("lite-t2v")) {
|
||
return ["text-to-video"];
|
||
}
|
||
if (normalized.includes("lite-i2v")) {
|
||
return ["first-frame", "first-last-frame", "reference-images"];
|
||
}
|
||
if (normalized.includes("pro-fast")) {
|
||
return ["text-to-video", "first-frame"];
|
||
}
|
||
if (normalized.includes("-pro-") || normalized.endsWith("-pro")) {
|
||
return ["text-to-video", "first-frame", "first-last-frame"];
|
||
}
|
||
|
||
return ["text-to-video", "first-frame", "first-last-frame", "reference-images"];
|
||
}
|
||
|
||
function getSupportedVideoModesForModel(model: any): VideoModeType[] {
|
||
const capabilityModes = (((model?.capabilities as any)?.supportedModes ?? []) as string[])
|
||
.filter((mode): mode is VideoModeType =>
|
||
mode === "text-to-video" ||
|
||
mode === "first-frame" ||
|
||
mode === "first-last-frame" ||
|
||
mode === "reference-images"
|
||
);
|
||
|
||
if (capabilityModes.length > 0) {
|
||
return capabilityModes;
|
||
}
|
||
|
||
const modelId = String(model?.modelId || "");
|
||
return inferSupportedVideoModesByModelId(modelId);
|
||
}
|
||
|
||
function scoreVideoSubModelForMode(modelId: string, mode: VideoModeType): number {
|
||
const normalized = modelId.toLowerCase();
|
||
const isProFast = normalized.includes("pro-fast");
|
||
const isLiteT2V = normalized.includes("lite-t2v");
|
||
const isLiteI2V = normalized.includes("lite-i2v");
|
||
const isPro = (normalized.includes("-pro-") || normalized.endsWith("-pro")) && !isProFast;
|
||
const isV15Pro = normalized.includes("seedance-1-5-pro");
|
||
|
||
switch (mode) {
|
||
case "first-last-frame":
|
||
if (isV15Pro) return 150;
|
||
if (isPro) return 120;
|
||
if (isLiteI2V) return 90;
|
||
if (isProFast) return -1000;
|
||
if (isLiteT2V) return -1000;
|
||
return 10;
|
||
case "first-frame":
|
||
if (isV15Pro) return 140;
|
||
if (isProFast) return 130;
|
||
if (isPro) return 120;
|
||
if (isLiteI2V) return 100;
|
||
if (isLiteT2V) return 20;
|
||
return 10;
|
||
case "reference-images":
|
||
if (isV15Pro) return 150;
|
||
if (isLiteI2V) return 140;
|
||
if (isPro) return 110;
|
||
if (isProFast || isLiteT2V) return -1000;
|
||
return 10;
|
||
case "text-to-video":
|
||
default:
|
||
if (isV15Pro) return 140;
|
||
if (isLiteT2V) return 130;
|
||
if (isProFast) return 120;
|
||
if (isPro) return 100;
|
||
if (isLiteI2V) return 80;
|
||
return 10;
|
||
}
|
||
}
|
||
|
||
function selectBestVideoSubModelForMode(
|
||
subModels: Array<{ model: any }>,
|
||
mode: VideoModeType,
|
||
): { model: any } | null {
|
||
const compatible = subModels.filter(({ model }) => {
|
||
const supportedModes = getSupportedVideoModesForModel(model);
|
||
return supportedModes.includes(mode);
|
||
});
|
||
|
||
const candidates = compatible.length > 0 ? compatible : subModels;
|
||
if (candidates.length === 0) return null;
|
||
|
||
return [...candidates].sort((a, b) => {
|
||
const scoreA = scoreVideoSubModelForMode(String(a.model?.modelId || ""), mode);
|
||
const scoreB = scoreVideoSubModelForMode(String(b.model?.modelId || ""), mode);
|
||
return scoreB - scoreA;
|
||
})[0];
|
||
}
|
||
|
||
function extractVideoUrlFromText(text: string): string | null {
|
||
const normalizedText = String(text || "");
|
||
if (!normalizedText) return null;
|
||
|
||
const htmlMatch = normalizedText.match(/<video[^>]+src="([^"]+)"/i);
|
||
if (htmlMatch?.[1]) return htmlMatch[1];
|
||
|
||
const markdownMatch = normalizedText.match(
|
||
/\[[^\]]*]\((https?:\/\/[^)\s]+\.mp4(?:\?[^)\s]*)?)\)/i,
|
||
);
|
||
if (markdownMatch?.[1]) return markdownMatch[1];
|
||
|
||
const plainMatch = normalizedText.match(/(https?:\/\/[^\s"'<>]+\.mp4(?:\?[^\s"'<>]+)?)/i);
|
||
return plainMatch?.[1] || null;
|
||
}
|
||
|
||
function replaceVideoUrlInText(text: string, oldUrl: string, newUrl: string): string {
|
||
if (!text || !oldUrl || !newUrl || oldUrl === newUrl) return text;
|
||
return text.split(oldUrl).join(newUrl);
|
||
}
|
||
|
||
function isAlreadyArchivedVideoUrl(url: string): boolean {
|
||
return /\/uploads\//i.test(url);
|
||
}
|
||
|
||
function scheduleVideoArchive(params: {
|
||
chatId: string;
|
||
messageId: string;
|
||
text: string;
|
||
temporaryVideoUrl: string;
|
||
taskId?: string;
|
||
metadata?: Record<string, any> | null;
|
||
}) {
|
||
const { chatId, messageId, text, temporaryVideoUrl, taskId, metadata } = params;
|
||
if (!temporaryVideoUrl || isAlreadyArchivedVideoUrl(temporaryVideoUrl)) {
|
||
return;
|
||
}
|
||
|
||
after(async () => {
|
||
try {
|
||
const permanentVideoUrl = await videoArchiver.archiveVideo({
|
||
messageId,
|
||
temporaryUrl: temporaryVideoUrl,
|
||
taskId: taskId || messageId,
|
||
metadata: {
|
||
duration:
|
||
typeof metadata?.duration === "number" && Number.isFinite(metadata.duration)
|
||
? metadata.duration
|
||
: undefined,
|
||
resolution:
|
||
typeof metadata?.resolution === "string" ? metadata.resolution : undefined,
|
||
ratio: typeof metadata?.ratio === "string" ? metadata.ratio : undefined,
|
||
},
|
||
});
|
||
|
||
const archivedText = replaceVideoUrlInText(text, temporaryVideoUrl, permanentVideoUrl);
|
||
const nextMetadata = {
|
||
...(metadata || {}),
|
||
temporaryVideoUrl,
|
||
permanentVideoUrl,
|
||
storageStatus: "archived",
|
||
archivedAt: Date.now(),
|
||
updatedAt: Date.now(),
|
||
};
|
||
|
||
await updateMessageById({
|
||
messageId,
|
||
chatId,
|
||
parts: [{ type: "text", text: archivedText }],
|
||
videoStorageStatus: "archived",
|
||
temporaryVideoUrl,
|
||
permanentVideoUrl,
|
||
archivedAt: new Date(),
|
||
videoMetadata: nextMetadata as any,
|
||
});
|
||
|
||
console.log("[Chat API] ✅ 视频已归档到永久存储:", {
|
||
messageId,
|
||
permanentVideoUrl,
|
||
});
|
||
} catch (error) {
|
||
console.error("[Chat API] ❌ 视频归档失败（保留临时地址）:", {
|
||
messageId,
|
||
temporaryVideoUrl,
|
||
error,
|
||
});
|
||
}
|
||
});
|
||
}
|
||
|
||
async function replayAssistantMessagesForDuplicateRequest({
|
||
chatId,
|
||
userMessageId,
|
||
writer,
|
||
source,
|
||
waitMs = DUPLICATE_REQUEST_WAIT_MS,
|
||
}: {
|
||
chatId: string;
|
||
userMessageId: string;
|
||
writer: StreamWriter;
|
||
source: string;
|
||
waitMs?: number;
|
||
}) {
|
||
const startedAt = Date.now();
|
||
const seenAssistantMessageIds = new Set<string>();
|
||
|
||
while (Date.now() - startedAt < waitMs) {
|
||
try {
|
||
const dbMessages = await getMessagesByChatId({ id: chatId });
|
||
const userMessageIndex = dbMessages.findIndex((msg) => msg.id === userMessageId);
|
||
|
||
if (userMessageIndex >= 0) {
|
||
const assistantMessages = dbMessages
|
||
.slice(userMessageIndex + 1)
|
||
.filter((msg) => msg.role === "assistant");
|
||
|
||
if (assistantMessages.length > 0) {
|
||
for (const assistantMessage of assistantMessages) {
|
||
if (seenAssistantMessageIds.has(assistantMessage.id)) continue;
|
||
seenAssistantMessageIds.add(assistantMessage.id);
|
||
|
||
safeWriteStreamChunk(
|
||
writer,
|
||
{
|
||
type: "data-appendMessage",
|
||
data: JSON.stringify({
|
||
id: assistantMessage.id,
|
||
role: "assistant",
|
||
parts: assistantMessage.parts,
|
||
}),
|
||
},
|
||
`${source}:replay`,
|
||
);
|
||
}
|
||
|
||
return;
|
||
}
|
||
}
|
||
} catch (error) {
|
||
console.warn(`[Chat API] Failed duplicate replay fetch (${source})`, error);
|
||
}
|
||
|
||
safeWriteStreamChunk(
|
||
writer,
|
||
{
|
||
type: "data-keepalive",
|
||
data: JSON.stringify({ ts: Date.now(), source }),
|
||
},
|
||
`${source}:keepalive`,
|
||
);
|
||
|
||
await sleep(1000);
|
||
}
|
||
}
|
||
|
||
type ChatImageParams = NonNullable<PostRequestBody["imageParams"]>;
|
||
type AgentRecommendedImageParams = {
|
||
aspectRatio?: string;
|
||
imageSize?: string;
|
||
imageCount?: number;
|
||
negativePrompt?: string;
|
||
promptExtend?: boolean;
|
||
};
|
||
|
||
type ChatVideoParams = NonNullable<PostRequestBody["videoParams"]>;
|
||
type ChatVideoParamsDraft = Partial<ChatVideoParams>;
|
||
type AgentRecommendedVideoParams = {
|
||
mode?: "text-to-video" | "first-frame" | "first-last-frame" | "reference-images";
|
||
resolution?: "480p" | "720p" | "1080p";
|
||
ratio?: string;
|
||
duration?: number;
|
||
generateAudio?: boolean;
|
||
cameraMove?: string;
|
||
};
|
||
|
||
type AgentVideoPlan = {
|
||
script: string;
|
||
storyboard: Array<{
|
||
shot: number;
|
||
title: string;
|
||
duration: number;
|
||
description: string;
|
||
prompt: string;
|
||
camera?: string;
|
||
}>;
|
||
finalPrompt: string;
|
||
negativePrompt?: string;
|
||
recommendedVideoParams?: AgentRecommendedVideoParams;
|
||
};
|
||
|
||
const IMAGE_FAILOVER_PROVIDER_ALLOWLIST = new Set([
|
||
"alibaba-image",
|
||
"ruxa-image",
|
||
"fal-image",
|
||
"volcano-vision",
|
||
"gemini",
|
||
"huanke-gemini",
|
||
]);
|
||
|
||
const IMAGE_FAILOVER_MODEL_COOLDOWN_MS = 30 * 60 * 1000;
|
||
const imageFailoverModelCooldown = new Map<string, { until: number; reason: string }>();
|
||
|
||
function normalizeFailoverErrorMessage(error: unknown): string {
|
||
if (!error) return "";
|
||
if (typeof error === "string") return error;
|
||
if (error instanceof Error) return error.message || String(error);
|
||
try {
|
||
return JSON.stringify(error);
|
||
} catch {
|
||
return String(error);
|
||
}
|
||
}
|
||
|
||
function shouldCooldownImageFallbackModel(errorMessage: string): boolean {
|
||
const msg = errorMessage.toLowerCase();
|
||
return (
|
||
msg.includes("model not found") ||
|
||
msg.includes("model_not_found") ||
|
||
msg.includes("no available channel for model") ||
|
||
msg.includes("unknown model") ||
|
||
msg.includes("invalid model") ||
|
||
msg.includes("api key is missing") ||
|
||
msg.includes("invalid api key") ||
|
||
msg.includes("unauthorized") ||
|
||
msg.includes("cannot convert argument to a bytestring")
|
||
);
|
||
}
|
||
|
||
function isImageFallbackModelCoolingDown(modelId: string): boolean {
|
||
const snapshot = imageFailoverModelCooldown.get(modelId);
|
||
if (!snapshot) return false;
|
||
if (snapshot.until <= Date.now()) {
|
||
imageFailoverModelCooldown.delete(modelId);
|
||
return false;
|
||
}
|
||
return true;
|
||
}
|
||
|
||
function setImageFallbackModelCooldown(modelId: string, reason: string): void {
|
||
imageFailoverModelCooldown.set(modelId, {
|
||
until: Date.now() + IMAGE_FAILOVER_MODEL_COOLDOWN_MS,
|
||
reason: reason.slice(0, 500),
|
||
});
|
||
}
|
||
|
||
function getImageCrossProviderCandidates(
|
||
allModels: any[],
|
||
options: {
|
||
currentModelId: string;
|
||
currentProviderId?: string;
|
||
providerPriority: Record<string, number>;
|
||
}
|
||
) {
|
||
return allModels
|
||
.filter((m) => {
|
||
if (!m?.model?.supportsImageGen || !m?.model?.isEnabled || m?.model?.isSubModel) {
|
||
return false;
|
||
}
|
||
if (!IMAGE_FAILOVER_PROVIDER_ALLOWLIST.has(String(m?.provider?.type || ""))) {
|
||
return false;
|
||
}
|
||
if (m.model.id === options.currentModelId) return false;
|
||
if (options.currentProviderId && m.provider.id === options.currentProviderId) return false;
|
||
if (isImageFallbackModelCoolingDown(m.model.id)) {
|
||
const cooldown = imageFailoverModelCooldown.get(m.model.id);
|
||
console.log(
|
||
`[Chat API] ⏭ Skip image failover model on cooldown: ${m.model.displayName} (${m.provider.type}) - ${cooldown?.reason || "cooldown"}`
|
||
);
|
||
return false;
|
||
}
|
||
return true;
|
||
})
|
||
.sort((a, b) => {
|
||
const left = options.providerPriority[a.provider.type as string] ?? 99;
|
||
const right = options.providerPriority[b.provider.type as string] ?? 99;
|
||
if (left !== right) return left - right;
|
||
return (a.model.sortOrder ?? 0) - (b.model.sortOrder ?? 0);
|
||
});
|
||
}
|
||
|
||
function hasUserCustomizedImageParams(params?: PostRequestBody["imageParams"]): boolean {
|
||
if (!params) return false;
|
||
const customizedAspectRatio =
|
||
typeof params.aspectRatio === "string" && params.aspectRatio.trim() && params.aspectRatio !== "auto";
|
||
const customizedImageSize =
|
||
typeof params.imageSize === "string" && params.imageSize.trim() && params.imageSize.toUpperCase() !== "2K";
|
||
const customizedImageCount = typeof params.imageCount === "number" && params.imageCount !== 1;
|
||
const customizedPromptExtend = typeof params.promptExtend === "boolean";
|
||
const customizedAdvanced = Boolean(
|
||
params.size || params.seed !== undefined || params.n || params.negativePrompt || customizedPromptExtend
|
||
);
|
||
return customizedAspectRatio || customizedImageSize || customizedImageCount || customizedAdvanced;
|
||
}
|
||
|
||
function mergeImageParamsWithAgentRecommendation(params: {
|
||
incoming?: PostRequestBody["imageParams"];
|
||
recommended?: AgentRecommendedImageParams;
|
||
intent?: "image_edit" | "image_gen" | "vision" | "video_gen" | "chat";
|
||
hasUserCustomSettings: boolean;
|
||
}): PostRequestBody["imageParams"] {
|
||
const { incoming, recommended, intent, hasUserCustomSettings } = params;
|
||
const base: ChatImageParams = { ...(incoming || {}) };
|
||
if (!recommended) {
|
||
return Object.keys(base).length > 0 ? base : undefined;
|
||
}
|
||
|
||
const merged: ChatImageParams = {
|
||
...base,
|
||
negativePrompt:
|
||
typeof recommended.negativePrompt === "string" && recommended.negativePrompt.trim()
|
||
? recommended.negativePrompt.trim()
|
||
: base.negativePrompt,
|
||
promptExtend:
|
||
typeof recommended.promptExtend === "boolean"
|
||
? recommended.promptExtend
|
||
: base.promptExtend,
|
||
};
|
||
|
||
if (!hasUserCustomSettings) {
|
||
if (recommended.aspectRatio) merged.aspectRatio = recommended.aspectRatio;
|
||
if (recommended.imageSize) merged.imageSize = recommended.imageSize;
|
||
if (typeof recommended.imageCount === "number") merged.imageCount = recommended.imageCount;
|
||
}
|
||
|
||
if (intent === "image_edit") {
|
||
merged.imageCount = 1;
|
||
if (typeof base.promptExtend !== "boolean") {
|
||
merged.promptExtend = false;
|
||
}
|
||
}
|
||
|
||
return Object.keys(merged).length > 0 ? merged : undefined;
|
||
}
|
||
|
||
function hasUserCustomizedVideoParams(params?: PostRequestBody["videoParams"]): boolean {
|
||
if (!params) return false;
|
||
const defaults = {
|
||
mode: "text-to-video",
|
||
resolution: "720p",
|
||
ratio: "16:9",
|
||
duration: 5,
|
||
generateAudio: true,
|
||
cameraMove: "none",
|
||
} as const;
|
||
|
||
if (params.mode && params.mode !== defaults.mode) return true;
|
||
if (params.resolution && params.resolution !== defaults.resolution) return true;
|
||
if (params.ratio && params.ratio !== defaults.ratio) return true;
|
||
if (typeof params.duration === "number" && params.duration !== defaults.duration) return true;
|
||
if (typeof params.generateAudio === "boolean" && params.generateAudio !== defaults.generateAudio)
|
||
return true;
|
||
if (typeof params.cameraMove === "string" && params.cameraMove !== defaults.cameraMove)
|
||
return true;
|
||
if (typeof params.seed === "number") return true;
|
||
return false;
|
||
}
|
||
|
||
function mergeVideoParamsWithAgentRecommendation(params: {
|
||
incoming?: PostRequestBody["videoParams"];
|
||
recommended?: AgentRecommendedVideoParams;
|
||
hasUserCustomSettings: boolean;
|
||
}): PostRequestBody["videoParams"] {
|
||
const { incoming, recommended, hasUserCustomSettings } = params;
|
||
const base: ChatVideoParamsDraft = { ...(incoming || {}) };
|
||
if (!recommended) {
|
||
return Object.keys(base).length > 0 ? (base as ChatVideoParams) : undefined;
|
||
}
|
||
|
||
const merged: ChatVideoParamsDraft = { ...base };
|
||
if (!hasUserCustomSettings) {
|
||
if (recommended.mode) merged.mode = recommended.mode;
|
||
if (recommended.resolution) merged.resolution = recommended.resolution;
|
||
if (recommended.ratio) merged.ratio = recommended.ratio;
|
||
if (typeof recommended.duration === "number") merged.duration = recommended.duration;
|
||
if (typeof recommended.generateAudio === "boolean") {
|
||
merged.generateAudio = recommended.generateAudio;
|
||
}
|
||
if (recommended.cameraMove) merged.cameraMove = recommended.cameraMove;
|
||
}
|
||
|
||
if (!merged.mode) {
|
||
merged.mode = recommended.mode || (incoming?.mode as ChatVideoParams["mode"]) || "text-to-video";
|
||
}
|
||
if (!merged.resolution) {
|
||
merged.resolution =
|
||
recommended.resolution || (incoming?.resolution as ChatVideoParams["resolution"]) || "720p";
|
||
}
|
||
if (!merged.ratio) {
|
||
merged.ratio = recommended.ratio || incoming?.ratio || "16:9";
|
||
}
|
||
if (typeof merged.duration !== "number") {
|
||
merged.duration = recommended.duration || incoming?.duration || 5;
|
||
}
|
||
|
||
return Object.keys(merged).length > 0 ? (merged as ChatVideoParams) : undefined;
|
||
}
|
||
|
||
function sanitizeAgentOutputSettingsSection(params: {
|
||
message?: string;
|
||
userImageParams?: PostRequestBody["imageParams"];
|
||
hasUserCustomImageParams: boolean;
|
||
}): string | undefined {
|
||
const { message, userImageParams, hasUserCustomImageParams } = params;
|
||
if (!message?.trim()) return message;
|
||
|
||
const outputSectionRegex =
|
||
/(?:###\s*输出设置|【\s*输出设置\s*】)[\s\S]*?(?=\n(?:###\s*|【\s*[^】]+\s*】)|$)/;
|
||
if (!outputSectionRegex.test(message)) return message;
|
||
|
||
if (!hasUserCustomImageParams) {
|
||
return message.replace(
|
||
outputSectionRegex,
|
||
"【输出设置】\n- 参数：按用户面板当前设置执行（未自动改写）"
|
||
);
|
||
}
|
||
|
||
const lines: string[] = [];
|
||
const aspectRatio =
|
||
typeof userImageParams?.aspectRatio === "string" ? userImageParams.aspectRatio.trim() : "";
|
||
const imageSize =
|
||
typeof userImageParams?.imageSize === "string" ? userImageParams.imageSize.trim() : "";
|
||
const imageCount =
|
||
typeof userImageParams?.imageCount === "number" ? userImageParams.imageCount : undefined;
|
||
|
||
if (aspectRatio && aspectRatio !== "auto") {
|
||
lines.push(`- 宽高比：${aspectRatio}（来自用户面板设置）`);
|
||
}
|
||
if (imageSize && imageSize.toUpperCase() !== "2K") {
|
||
lines.push(`- 清晰度档位：${imageSize.toUpperCase()}（来自用户面板设置）`);
|
||
}
|
||
if (typeof imageCount === "number" && Number.isFinite(imageCount) && imageCount !== 1) {
|
||
lines.push(`- 出图数量：${Math.max(1, Math.round(imageCount))} 张（来自用户面板设置）`);
|
||
}
|
||
lines.push("- 其余参数：保持用户面板当前设置");
|
||
|
||
return message.replace(outputSectionRegex, `【输出设置】\n${lines.join("\n")}`);
|
||
}
|
||
|
||
function normalizeAgentReportHeadingLine(line: string): string {
|
||
const trimmed = line.trim();
|
||
if (!trimmed) return line;
|
||
|
||
const markdownHeading = trimmed.match(/^#{1,6}\s*(.+?)\s*$/);
|
||
const plainTitle = markdownHeading?.[1]?.trim() || trimmed;
|
||
const normalizedTitle = plainTitle
|
||
.replace(/^[^\u4e00-\u9fa5A-Za-z0-9]+/, "")
|
||
.trim();
|
||
|
||
const knownTitles = [
|
||
"需求理解",
|
||
"图片分析",
|
||
"执行计划",
|
||
"输出设置",
|
||
"脚本规划",
|
||
"分镜规划",
|
||
"执行说明",
|
||
"视频生成完成",
|
||
];
|
||
|
||
if (knownTitles.some((title) => normalizedTitle === title)) {
|
||
return `【${normalizedTitle}】`;
|
||
}
|
||
|
||
if (markdownHeading) {
|
||
return `【${normalizedTitle || plainTitle}】`;
|
||
}
|
||
return line;
|
||
}
|
||
|
||
function stripAgentFriendlyMediaEmbeds(message: string): string {
|
||
if (!message) return message;
|
||
|
||
const withoutInlineMedia = message
|
||
.replace(/!\[[^\]]*]\((https?:\/\/[^\s)]+)\)/gi, "")
|
||
.replace(/<video[\s\S]*?<\/video>/gi, "")
|
||
.replace(/<img[^>]*>/gi, "");
|
||
|
||
const cleanedLines = withoutInlineMedia
|
||
.split("\n")
|
||
.filter((line) => {
|
||
const trimmed = line.trim();
|
||
if (!trimmed) return true;
|
||
if (/^https?:\/\/\S+$/i.test(trimmed)) return false;
|
||
if (/^\[[^\]]*]\(https?:\/\/[^\s)]+\)$/i.test(trimmed)) return false;
|
||
return true;
|
||
})
|
||
.join("\n");
|
||
|
||
return cleanedLines.replace(/\n{3,}/g, "\n\n").trim();
|
||
}
|
||
|
||
function normalizeAgentReportForDisplay(message?: string): string | undefined {
|
||
if (!message?.trim()) return message;
|
||
const stripped = stripAgentFriendlyMediaEmbeds(message);
|
||
if (!stripped) return undefined;
|
||
return stripped
|
||
.split("\n")
|
||
.map((line) => normalizeAgentReportHeadingLine(line))
|
||
.join("\n");
|
||
}
|
||
|
||
function guessImageMediaType(url: string): string {
|
||
const cleanUrl = url.split("?")[0]?.toLowerCase() || "";
|
||
if (cleanUrl.endsWith(".jpg") || cleanUrl.endsWith(".jpeg")) return "image/jpeg";
|
||
if (cleanUrl.endsWith(".webp")) return "image/webp";
|
||
if (cleanUrl.endsWith(".gif")) return "image/gif";
|
||
if (cleanUrl.endsWith(".bmp")) return "image/bmp";
|
||
if (cleanUrl.endsWith(".svg")) return "image/svg+xml";
|
||
return "image/png";
|
||
}
|
||
|
||
function injectHistoryImagesToLastUserMessage(
|
||
messages: ChatMessage[],
|
||
historyImages: string[]
|
||
): ChatMessage[] {
|
||
if (!historyImages || historyImages.length === 0) return messages;
|
||
|
||
let lastUserIndex = -1;
|
||
for (let i = messages.length - 1; i >= 0; i--) {
|
||
if (messages[i]?.role === "user") {
|
||
lastUserIndex = i;
|
||
break;
|
||
}
|
||
}
|
||
|
||
if (lastUserIndex < 0) return messages;
|
||
|
||
const lastUserMessage = messages[lastUserIndex];
|
||
const hasExistingImage = lastUserMessage.parts?.some(
|
||
(p: any) => p.type === "file" || p.type === "image" || p.type === "image_url"
|
||
);
|
||
if (hasExistingImage) return messages;
|
||
|
||
const imageParts = historyImages.map((url, index) => ({
|
||
type: "file" as const,
|
||
url,
|
||
name: `history-image-${index + 1}`,
|
||
mediaType: guessImageMediaType(url),
|
||
mimeType: guessImageMediaType(url),
|
||
}));
|
||
|
||
const updatedMessage: ChatMessage = {
|
||
...lastUserMessage,
|
||
parts: [...lastUserMessage.parts, ...imageParts],
|
||
};
|
||
|
||
return messages.map((msg, idx) => (idx === lastUserIndex ? updatedMessage : msg));
|
||
}
|
||
|
||
function compactAutoDemand(text: string, maxLength = 56): string {
|
||
const normalized = text.replace(/\s+/g, " ").trim();
|
||
if (!normalized) return "按你刚才的描述执行";
|
||
if (normalized.length <= maxLength) return normalized;
|
||
return `${normalized.slice(0, maxLength)}...`;
|
||
}
|
||
|
||
function buildAutoModeFallbackMessage(params: {
|
||
text: string;
|
||
hasImage: boolean;
|
||
fromHistory: boolean;
|
||
imageSummary?: string;
|
||
}): string {
|
||
const demand = compactAutoDemand(params.text);
|
||
|
||
if (params.hasImage) {
|
||
const sourceLine = params.fromHistory
|
||
? "基于上一轮图片继续处理。"
|
||
: "基于当前上传图片处理。";
|
||
|
||
return [
|
||
"### 需求理解",
|
||
`- ${demand}`,
|
||
"### 图片分析",
|
||
params.imageSummary
|
||
? `- ${params.imageSummary.replace(/\s+/g, " ").trim()}`
|
||
: `- ${sourceLine}`,
|
||
"- 我会优先保持主体、构图和整体风格一致，只处理你指定的改动。",
|
||
"### 执行计划",
|
||
"- 先完成本轮核心改动，再检查边缘与光影一致性。",
|
||
"- 若结果偏差，我会按“保留A，修改B”的方式继续迭代。",
|
||
"### 输出设置",
|
||
"- 默认单图输出，优先稳定命中。",
|
||
].join("\n");
|
||
}
|
||
|
||
return [
|
||
"### 需求理解",
|
||
`- ${demand}`,
|
||
"### 执行计划",
|
||
"- 先锁定主体与场景，再细化风格、材质与光影。",
|
||
"### 输出设置",
|
||
"- 默认单图输出，优先稳定命中。",
|
||
].join("\n");
|
||
}
|
||
|
||
function hasStructuredAgentSections(message?: string): boolean {
|
||
if (!message) return false;
|
||
return /(?:###\s*需求理解|【\s*需求理解\s*】|###\s*图片分析|【\s*图片分析\s*】|###\s*执行计划|【\s*执行计划\s*】)/.test(
|
||
message,
|
||
);
|
||
}
|
||
|
||
function ensureStructuredAgentFriendlyMessage(params: {
|
||
message?: string;
|
||
text: string;
|
||
hasImage: boolean;
|
||
fromHistory: boolean;
|
||
}): string {
|
||
const existingMessage = String(params.message || "").trim();
|
||
if (hasStructuredAgentSections(existingMessage)) {
|
||
return existingMessage;
|
||
}
|
||
|
||
const fallback = buildAutoModeFallbackMessage({
|
||
text: params.text,
|
||
hasImage: params.hasImage,
|
||
fromHistory: params.fromHistory,
|
||
});
|
||
|
||
if (!existingMessage) {
|
||
return fallback;
|
||
}
|
||
|
||
const normalizedSupplement = existingMessage.replace(/\s+/g, " ").trim();
|
||
if (!normalizedSupplement) {
|
||
return fallback;
|
||
}
|
||
|
||
return fallback.replace(
|
||
"### 执行计划\n",
|
||
`### 执行计划\n- 补充说明：${normalizedSupplement}\n`,
|
||
);
|
||
}
|
||
|
||
function extractMessageImageUrls(parts: any[]): string[] {
|
||
return (parts || [])
|
||
.map((part: any) => {
|
||
if (!part) return null;
|
||
if (part.type === "file") return part.url || part.data || null;
|
||
if (part.type === "image") return part.url || part.image || null;
|
||
if (part.type === "image_url") return part.image_url?.url || null;
|
||
return null;
|
||
})
|
||
.filter((url: any): url is string => typeof url === "string" && /^https?:\/\//i.test(url));
|
||
}
|
||
|
||
function shouldEnrichImageAnalysisText(message?: string): boolean {
|
||
if (!message) return false;
|
||
if (!/(###\s*图片分析|【\s*图片分析\s*】)/.test(message)) return false;
|
||
return /(基于当前上传图片处理|基于你选中的原图处理|基于上一轮图片继续处理)/.test(message);
|
||
}
|
||
|
||
function injectImageAnalysisSummary(message: string, summary: string): string {
|
||
const normalizedSummary = summary.replace(/\s+/g, " ").trim();
|
||
if (!normalizedSummary) return message;
|
||
const lines = message.split("\n");
|
||
const analysisHeaderIndex = lines.findIndex((line) =>
|
||
/^(###\s*图片分析|【\s*图片分析\s*】)/.test(line.trim())
|
||
);
|
||
if (analysisHeaderIndex < 0) return message;
|
||
|
||
for (let i = analysisHeaderIndex + 1; i < lines.length; i += 1) {
|
||
const current = lines[i]?.trim() || "";
|
||
if (/^(###\s*|【\s*[^】]+\s*】)/.test(current)) break;
|
||
if (/^- /.test(current)) {
|
||
lines[i] = `- ${normalizedSummary}`;
|
||
return lines.join("\n");
|
||
}
|
||
}
|
||
|
||
const insertIndex = analysisHeaderIndex + 1;
|
||
lines.splice(insertIndex, 0, `- ${normalizedSummary}`);
|
||
return lines.join("\n");
|
||
}
|
||
|
||
function ensureImageAnalysisSummarySection(message: string, summary: string): string {
|
||
const normalizedSummary = summary.replace(/\s+/g, " ").trim();
|
||
if (!normalizedSummary) return message;
|
||
|
||
if (/(###\s*图片分析|【\s*图片分析\s*】)/.test(message)) {
|
||
return injectImageAnalysisSummary(message, normalizedSummary);
|
||
}
|
||
|
||
const trimmed = message.trim();
|
||
if (!trimmed) return `【图片分析】\n- ${normalizedSummary}`;
|
||
return `${trimmed}\n【图片分析】\n- ${normalizedSummary}`;
|
||
}
|
||
|
||
/**
|
||
* 自定义消息转换函数
|
||
* 注意：gemini-image 和 huanke-gemini 都需要 URL 格式（不是 Base64）
|
||
* 因为这些 provider 内部会自行处理图片上传到 S3
|
||
*/
|
||
async function convertMessagesWithImages(messages: ChatMessage[], providerType: string) {
|
||
if (providerType === "ruxa-image" || providerType === "fal-image") {
|
||
const convertedMessages = [];
|
||
|
||
for (const message of messages) {
|
||
const content: any[] = [];
|
||
|
||
for (const part of message.parts) {
|
||
const partAny = part as any;
|
||
|
||
if (partAny.type === "text" && partAny.text) {
|
||
content.push({
|
||
type: "text",
|
||
text: partAny.text,
|
||
});
|
||
continue;
|
||
}
|
||
|
||
const candidateUrl =
|
||
partAny.type === "file"
|
||
? (partAny.url || partAny.data)
|
||
: partAny.type === "image"
|
||
? (partAny.image || partAny.url)
|
||
: partAny.type === "image_url"
|
||
? partAny.image_url?.url
|
||
: undefined;
|
||
|
||
if (!candidateUrl || typeof candidateUrl !== "string") continue;
|
||
if (candidateUrl.startsWith("data:")) {
|
||
throw new Error("当前模型不支持 Base64 图片传输，请先上传图片后再编辑");
|
||
}
|
||
if (!candidateUrl.startsWith("http")) continue;
|
||
|
||
content.push({
|
||
type: "image_url",
|
||
image_url: {
|
||
url: candidateUrl,
|
||
},
|
||
});
|
||
}
|
||
|
||
convertedMessages.push({
|
||
role: message.role,
|
||
content: content.length === 1 && content[0].type === "text" ? content[0].text : content,
|
||
});
|
||
}
|
||
|
||
return convertedMessages;
|
||
}
|
||
|
||
// 所有 provider 都使用 URL 格式传输图片
|
||
// gemini-image 和 huanke-gemini 内部会将 Base64 转换为 S3 URL
|
||
// 这里不再进行 Base64 转换，直接返回标准格式
|
||
if (providerType !== "gemini") {
|
||
return convertToModelMessages(messages);
|
||
}
|
||
|
||
// 🆕 检测是否有标注图片
|
||
const hasAnnotation = hasAnnotatedImage(messages);
|
||
if (hasAnnotation) {
|
||
console.log("[Message Convert] 🎨 检测到标注图片，将自动添加系统提示");
|
||
}
|
||
|
||
// 对于 Gemini，需要特殊处理图片
|
||
const convertedMessages = [];
|
||
|
||
for (const message of messages) {
|
||
const content: any[] = [];
|
||
|
||
// 🆕 如果是用户消息且有标注图片，在文本前添加系统提示
|
||
let hasAddedAnnotationPrompt = false;
|
||
|
||
// 处理消息的各个部分
|
||
for (const part of message.parts) {
|
||
if (part.type === "text" && part.text) {
|
||
let textContent = part.text;
|
||
|
||
// 🆕 如果有标注图片且是用户消息，在用户文本前添加系统提示
|
||
if (hasAnnotation && message.role === "user" && !hasAddedAnnotationPrompt) {
|
||
textContent = ANNOTATION_SYSTEM_PROMPT + "\n\n用户请求：" + textContent;
|
||
hasAddedAnnotationPrompt = true;
|
||
console.log("[Message Convert] 🎨 已注入标注提示词");
|
||
}
|
||
|
||
content.push({
|
||
type: "text",
|
||
text: textContent,
|
||
});
|
||
} else if (part.type === "file" && part.url) {
|
||
// 直接使用 URL 格式（不再转换为 Base64）
|
||
// gemini-image 和 huanke-gemini provider 内部会处理图片上传
|
||
content.push({
|
||
type: "image_url",
|
||
image_url: {
|
||
url: part.url,
|
||
},
|
||
});
|
||
console.log("[Message Convert] Image kept as URL format:", part.url.substring(0, 80) + "...");
|
||
}
|
||
}
|
||
|
||
// 🆕 如果有标注图片但消息中没有文本部分，添加一个纯提示文本
|
||
if (hasAnnotation && message.role === "user" && !hasAddedAnnotationPrompt && content.length > 0) {
|
||
content.unshift({
|
||
type: "text",
|
||
text: ANNOTATION_SYSTEM_PROMPT,
|
||
});
|
||
console.log("[Message Convert] 🎨 已在消息开头注入标注提示词");
|
||
}
|
||
|
||
convertedMessages.push({
|
||
role: message.role,
|
||
content: content.length === 1 && content[0].type === "text"
|
||
? content[0].text
|
||
: content,
|
||
});
|
||
}
|
||
|
||
return convertedMessages;
|
||
}
|
||
|
||
function parseImageAltMetadata(altText: string, thumbnailUrl: string): {
|
||
originalUrl: string;
|
||
imageSize: string;
|
||
} {
|
||
let originalUrl = thumbnailUrl;
|
||
let imageSize = "2K";
|
||
|
||
// 格式1: thumb|||originalUrl|||size (新格式)
|
||
if (altText.includes("thumb|||")) {
|
||
const content = altText.substring(altText.indexOf("thumb|||") + 8);
|
||
const parts = content.split("|||");
|
||
originalUrl = parts[0] || thumbnailUrl;
|
||
imageSize = parts[1] || "2K";
|
||
}
|
||
// 格式2: thumb$$originalUrl$$size (中间格式，可能被 LaTeX 解析器破坏)
|
||
else if (altText.includes("thumb$$")) {
|
||
const content = altText.substring(altText.indexOf("thumb$$") + 7);
|
||
const parts = content.split("$$");
|
||
originalUrl = parts[0] || thumbnailUrl;
|
||
imageSize = parts[1] || "2K";
|
||
}
|
||
// 格式3: thumb|originalUrl|size (旧格式)
|
||
else if (altText.includes("thumb|")) {
|
||
const content = altText.substring(altText.indexOf("thumb|") + 6);
|
||
const parts = content.split("|");
|
||
originalUrl = parts[0] || thumbnailUrl;
|
||
imageSize = parts[1] || "2K";
|
||
}
|
||
|
||
// 回退策略：从缩略图 URL 推断原图 URL
|
||
if (originalUrl === thumbnailUrl) {
|
||
if (thumbnailUrl.includes("gemini-thumb-")) {
|
||
originalUrl = thumbnailUrl.replace("gemini-thumb-", "gemini-orig-");
|
||
} else if (thumbnailUrl.includes("openai-thumb-")) {
|
||
originalUrl = thumbnailUrl.replace("openai-thumb-", "openai-orig-");
|
||
} else if (thumbnailUrl.includes("volcano-thumb-")) {
|
||
originalUrl = thumbnailUrl.replace("volcano-thumb-", "volcano-orig-");
|
||
} else if (thumbnailUrl.includes("alibaba-image-thumb-")) {
|
||
originalUrl = thumbnailUrl.replace("alibaba-image-thumb-", "alibaba-image-orig-");
|
||
} else if (thumbnailUrl.includes("ruxa-image-thumb-")) {
|
||
originalUrl = thumbnailUrl.replace("ruxa-image-thumb-", "ruxa-image-orig-");
|
||
} else if (thumbnailUrl.includes("fal-image-thumb-")) {
|
||
originalUrl = thumbnailUrl.replace("fal-image-thumb-", "fal-image-orig-");
|
||
}
|
||
}
|
||
|
||
return { originalUrl, imageSize };
|
||
}
|
||
|
||
function extractOriginalImageUrlsFromText(text: string): string[] {
|
||
return extractImageUrlsFromText(text);
|
||
}
|
||
|
||
function mimeTypeToExtension(mimeType: string) {
|
||
const normalized = mimeType.toLowerCase();
|
||
if (normalized.includes("png")) return "png";
|
||
if (normalized.includes("webp")) return "webp";
|
||
if (normalized.includes("gif")) return "gif";
|
||
if (normalized.includes("bmp")) return "bmp";
|
||
return "jpg";
|
||
}
|
||
|
||
function getExtensionFromUrl(url: string): string {
|
||
try {
|
||
const pathname = new URL(url).pathname;
|
||
const ext = pathname.split(".").pop()?.toLowerCase();
|
||
if (!ext) return "jpg";
|
||
if (["jpg", "jpeg", "png", "webp", "gif", "bmp"].includes(ext)) return ext;
|
||
} catch {
|
||
// ignore
|
||
}
|
||
return "jpg";
|
||
}
|
||
|
||
async function uploadRemoteImageToStorage(imageUrl: string): Promise<string> {
|
||
const response = await fetch(imageUrl, {
|
||
method: "GET",
|
||
signal: AbortSignal.timeout(30_000),
|
||
});
|
||
|
||
if (!response.ok) {
|
||
throw new Error(`Remote image fetch failed: ${response.status} ${response.statusText}`);
|
||
}
|
||
|
||
const contentType = response.headers.get("content-type")?.toLowerCase() || "";
|
||
if (contentType && !contentType.startsWith("image/")) {
|
||
throw new Error(`Remote url is not an image: ${contentType}`);
|
||
}
|
||
|
||
const arrayBuffer = await response.arrayBuffer();
|
||
if (!arrayBuffer || arrayBuffer.byteLength === 0) {
|
||
throw new Error("Remote image content is empty");
|
||
}
|
||
|
||
const ext = contentType ? mimeTypeToExtension(contentType) : getExtensionFromUrl(imageUrl);
|
||
const fileName = `remote-image-${Date.now()}-${Math.random().toString(36).slice(2, 8)}.${ext}`;
|
||
const uploaded = await uploadFile(fileName, arrayBuffer);
|
||
return uploaded.url;
|
||
}
|
||
|
||
async function isRemoteImageAccessible(imageUrl: string): Promise<boolean> {
|
||
try {
|
||
const headResponse = await fetch(imageUrl, {
|
||
method: "HEAD",
|
||
signal: AbortSignal.timeout(10_000),
|
||
});
|
||
|
||
if (headResponse.ok) {
|
||
return true;
|
||
}
|
||
|
||
if (headResponse.status !== 405 && headResponse.status !== 403) {
|
||
return false;
|
||
}
|
||
|
||
const probeResponse = await fetch(imageUrl, {
|
||
method: "GET",
|
||
headers: {
|
||
Range: "bytes=0-0",
|
||
},
|
||
signal: AbortSignal.timeout(10_000),
|
||
});
|
||
return probeResponse.ok;
|
||
} catch {
|
||
return false;
|
||
}
|
||
}
|
||
|
||
async function ensureHostedImageUrl(imageUrl: string): Promise<string> {
|
||
const normalizedUrl = imageUrl.trim();
|
||
if (!normalizedUrl) {
|
||
throw new Error("Image URL is empty");
|
||
}
|
||
|
||
if (normalizedUrl.startsWith("data:image")) {
|
||
return uploadDataUrlImage(normalizedUrl);
|
||
}
|
||
|
||
if (!/^https?:\/\//i.test(normalizedUrl)) {
|
||
throw new Error(`Unsupported image URL: ${normalizedUrl}`);
|
||
}
|
||
|
||
if (/\/uploads\//i.test(normalizedUrl)) {
|
||
const accessible = await isRemoteImageAccessible(normalizedUrl);
|
||
if (accessible) {
|
||
return normalizedUrl;
|
||
}
|
||
}
|
||
|
||
return uploadRemoteImageToStorage(normalizedUrl);
|
||
}
|
||
|
||
async function uploadDataUrlImage(dataUrl: string): Promise<string> {
|
||
const match = dataUrl.match(/^data:([^;]+);base64,([\s\S]+)$/i);
|
||
if (!match) {
|
||
throw new Error("Invalid data URL image format");
|
||
}
|
||
|
||
const mimeType = match[1] || "image/jpeg";
|
||
const base64Data = (match[2] || "").replace(/\s+/g, "");
|
||
if (!base64Data) {
|
||
throw new Error("Empty base64 image content");
|
||
}
|
||
|
||
const buffer = Buffer.from(base64Data, "base64");
|
||
const fileExt = mimeTypeToExtension(mimeType);
|
||
const fileName = `inline-image-${Date.now()}-${Math.random().toString(36).slice(2, 8)}.${fileExt}`;
|
||
const arrayBuffer = buffer.buffer.slice(
|
||
buffer.byteOffset,
|
||
buffer.byteOffset + buffer.byteLength
|
||
) as ArrayBuffer;
|
||
const uploaded = await uploadFile(fileName, arrayBuffer);
|
||
return uploaded.url;
|
||
}
|
||
|
||
async function normalizeAssistantImageMarkdown(
|
||
text: string,
|
||
imageSize = "2K"
|
||
): Promise<string> {
|
||
if (!text) return text;
|
||
|
||
const markdownDataUrlRegex = /!\[([^\]]*)\]\s*\(\s*(data:image\/[a-z0-9+.-]+;base64,[^)]+)\s*\)/gim;
|
||
const markdownMatches = [...text.matchAll(markdownDataUrlRegex)];
|
||
let normalizedText = text;
|
||
const uploadedUrls: string[] = [];
|
||
|
||
if (markdownMatches.length > 0) {
|
||
let rebuilt = "";
|
||
let cursor = 0;
|
||
|
||
for (const match of markdownMatches) {
|
||
const matchText = match[0];
|
||
const dataUrl = match[2];
|
||
const start = match.index ?? -1;
|
||
if (start < 0) continue;
|
||
|
||
rebuilt += normalizedText.slice(cursor, start);
|
||
|
||
try {
|
||
const uploadedUrl = await uploadDataUrlImage(dataUrl);
|
||
uploadedUrls.push(uploadedUrl);
|
||
rebuilt += ``;
|
||
} catch (error) {
|
||
console.error("[Chat API] Failed to upload inline data URL image:", error);
|
||
rebuilt += matchText;
|
||
}
|
||
|
||
cursor = start + matchText.length;
|
||
}
|
||
|
||
rebuilt += normalizedText.slice(cursor);
|
||
normalizedText = rebuilt;
|
||
}
|
||
|
||
if (normalizedText.includes("data:image")) {
|
||
const rawDataUrlRegex = /data:image\/[a-z0-9+.-]+;base64,[a-z0-9+/=\s]+/gim;
|
||
const rawMatches = normalizedText.match(rawDataUrlRegex) ?? [];
|
||
|
||
for (const rawDataUrl of rawMatches) {
|
||
try {
|
||
const uploadedUrl = await uploadDataUrlImage(rawDataUrl);
|
||
uploadedUrls.push(uploadedUrl);
|
||
normalizedText = normalizedText.replace(rawDataUrl, uploadedUrl);
|
||
} catch (error) {
|
||
console.error("[Chat API] Failed to normalize raw data URL image:", error);
|
||
}
|
||
}
|
||
}
|
||
|
||
const markdownHttpImageRegex = /!\[([^\]]*)\]\s*\(\s*(https?:\/\/[^)\s]+)\s*\)/gim;
|
||
const markdownHttpMatches = [...normalizedText.matchAll(markdownHttpImageRegex)];
|
||
|
||
if (markdownHttpMatches.length > 0) {
|
||
let rebuilt = "";
|
||
let cursor = 0;
|
||
|
||
for (const match of markdownHttpMatches) {
|
||
const matchText = match[0];
|
||
const altText = match[1] || "";
|
||
const thumbnailUrl = (match[2] || "").trim();
|
||
const start = match.index ?? -1;
|
||
if (start < 0) continue;
|
||
|
||
rebuilt += normalizedText.slice(cursor, start);
|
||
|
||
try {
|
||
const { originalUrl, imageSize: parsedImageSize } = parseImageAltMetadata(
|
||
altText,
|
||
thumbnailUrl
|
||
);
|
||
|
||
let hostedThumbnailUrl = thumbnailUrl;
|
||
let hostedOriginalUrl =
|
||
originalUrl && originalUrl !== thumbnailUrl ? originalUrl : thumbnailUrl;
|
||
|
||
try {
|
||
hostedThumbnailUrl = await ensureHostedImageUrl(thumbnailUrl);
|
||
} catch (error) {
|
||
console.error("[Chat API] Failed to normalize remote thumbnail URL:", error);
|
||
}
|
||
|
||
if (originalUrl && originalUrl !== thumbnailUrl) {
|
||
try {
|
||
hostedOriginalUrl = await ensureHostedImageUrl(originalUrl);
|
||
} catch (error) {
|
||
console.error("[Chat API] Failed to normalize remote original URL:", error);
|
||
hostedOriginalUrl = hostedThumbnailUrl || originalUrl;
|
||
}
|
||
} else {
|
||
hostedOriginalUrl = hostedThumbnailUrl;
|
||
}
|
||
|
||
const finalThumbnailUrl = hostedThumbnailUrl || thumbnailUrl;
|
||
const finalOriginalUrl = hostedOriginalUrl || finalThumbnailUrl;
|
||
|
||
uploadedUrls.push(finalThumbnailUrl);
|
||
uploadedUrls.push(finalOriginalUrl);
|
||
rebuilt += ``;
|
||
} catch (error) {
|
||
console.error("[Chat API] Failed to parse remote image markdown metadata:", error);
|
||
rebuilt += matchText;
|
||
}
|
||
|
||
cursor = start + matchText.length;
|
||
}
|
||
|
||
rebuilt += normalizedText.slice(cursor);
|
||
normalizedText = rebuilt;
|
||
}
|
||
|
||
const hasMarkdownImage = /!\[[^\]]*\]\((https?:\/\/[^)]+)\)/i.test(normalizedText);
|
||
if (!hasMarkdownImage && uploadedUrls.length > 0) {
|
||
const uniqueUrls = [...new Set(uploadedUrls)];
|
||
console.log(
|
||
`[Chat API] Converted ${uniqueUrls.length} image link(s) to hosted link(s)`
|
||
);
|
||
return uniqueUrls
|
||
.map((url) => ``)
|
||
.join("\n");
|
||
}
|
||
|
||
return normalizedText;
|
||
}
|
||
|
||
function prependImageModelBadge(text: string, modelDisplayName?: string | null): string {
|
||
const normalizedText = String(text || "").trim();
|
||
if (!normalizedText) return normalizedText;
|
||
|
||
const normalizedModelName = String(modelDisplayName || "").trim();
|
||
if (!normalizedModelName) return normalizedText;
|
||
|
||
if (
|
||
/^\s*[◈◇◆🔹⬡✦•·]\s*[^\n]+/im.test(normalizedText) ||
|
||
/^\s*(?:Routed to|Using model|使用模型|模型)\s*[:：]/im.test(normalizedText)
|
||
) {
|
||
return normalizedText;
|
||
}
|
||
|
||
return `◈ ${normalizedModelName}\n\n${normalizedText}`;
|
||
}
|
||
|
||
function resolveModelDisplayNameByModelId(
|
||
models: Awaited<ReturnType<typeof getEnabledAiModelsWithProvider>> | undefined,
|
||
modelId: string | undefined,
|
||
): string {
|
||
if (!modelId || !models?.length) return "";
|
||
const match = models.find(
|
||
(item) => item.model.modelId === modelId || item.model.id === modelId,
|
||
);
|
||
return String(match?.model?.displayName || "");
|
||
}
|
||
|
||
/**
|
||
* 解析生图 Markdown 并保存图片记录到数据库
|
||
* 支持三种格式：thumb|||url|||size、thumb$$url$$size、thumb|url|size
|
||
*/
|
||
async function saveGeneratedImages(
|
||
imageMarkdown: string,
|
||
userId: string,
|
||
chatId: string,
|
||
messageId: string,
|
||
prompt: string,
|
||
modelId: string
|
||
): Promise<void> {
|
||
// 匹配 Markdown 图片格式: 
|
||
const imageRegex = /!\[([^\]]*)\]\(([^)]+)\)/g;
|
||
const matches = [...imageMarkdown.matchAll(imageRegex)];
|
||
|
||
let savedImageCount = 0;
|
||
|
||
for (const match of matches) {
|
||
const altText = match[1] || "";
|
||
const thumbnailUrl = match[2];
|
||
const { originalUrl, imageSize } = parseImageAltMetadata(altText, thumbnailUrl);
|
||
|
||
try {
|
||
await db.insert(generatedImage).values({
|
||
userId,
|
||
chatId,
|
||
messageId,
|
||
thumbnailUrl,
|
||
originalUrl,
|
||
imageSize,
|
||
prompt,
|
||
modelId,
|
||
});
|
||
savedImageCount += 1;
|
||
console.log(`[Gallery] Saved image: ${originalUrl.substring(0, 80)}...`);
|
||
} catch (error) {
|
||
console.error("[Gallery] Failed to save image record:", error);
|
||
}
|
||
}
|
||
|
||
if (savedImageCount <= 0) return;
|
||
|
||
try {
|
||
const pointsConfig = await getPointsSystemConfig();
|
||
if (!pointsConfig.enabled) return;
|
||
|
||
const modelConfig = await resolveBillingModelByReference(modelId);
|
||
const unitCost = getModelCreditCost(modelConfig, "image");
|
||
if (unitCost <= 0) return;
|
||
|
||
const totalCost = unitCost * savedImageCount;
|
||
const consumeResult = await consumeUserPoints({
|
||
userId,
|
||
amount: totalCost,
|
||
reason: "image_generation",
|
||
modelId: modelConfig?.id || modelId,
|
||
chatId,
|
||
messageId,
|
||
note: `模型(${modelConfig?.displayName || modelId}) 生图 ${savedImageCount} 张`,
|
||
});
|
||
|
||
if (!consumeResult.ok) {
|
||
console.warn(
|
||
`[Points] Failed to consume points after generation. user=${userId} model=${modelId} cost=${totalCost}`
|
||
);
|
||
} else {
|
||
console.log(
|
||
`[Points] Consumed ${totalCost} points (${unitCost} x ${savedImageCount}). balance=${consumeResult.balance}`
|
||
);
|
||
}
|
||
} catch (error) {
|
||
console.error("[Points] Failed to consume points after image generation:", error);
|
||
}
|
||
}
|
||
|
||
const MODEL_UUID_REGEX =
|
||
/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
|
||
|
||
function getModelCreditCost(modelConfig: any, target: "image" | "video"): number {
|
||
const config = (modelConfig as any)?.imageGenConfig || {};
|
||
const sourceValue =
|
||
target === "video"
|
||
? config.creditCostPerVideo ?? config.creditCostPerImage ?? 0
|
||
: config.creditCostPerImage ?? 0;
|
||
const normalized = Number(sourceValue);
|
||
return Number.isFinite(normalized) ? Math.max(0, Math.round(normalized)) : 0;
|
||
}
|
||
|
||
async function resolveBillingModelByReference(modelRef: string): Promise<any | undefined> {
|
||
if (!modelRef) return undefined;
|
||
|
||
if (MODEL_UUID_REGEX.test(modelRef)) {
|
||
return await getAiModelById({ id: modelRef }).catch(() => undefined);
|
||
}
|
||
|
||
const allModels = await getEnabledAiModelsWithProvider(true, true).catch(() => []);
|
||
const matched = allModels.find(
|
||
({ model }: any) => model.id === modelRef || model.modelId === modelRef
|
||
);
|
||
return matched?.model;
|
||
}
|
||
|
||
async function consumeGeneratedVideoPoints(params: {
|
||
userId: string;
|
||
chatId: string;
|
||
messageId: string;
|
||
modelRefs: Array<string | null | undefined>;
|
||
}) {
|
||
try {
|
||
const pointsConfig = await getPointsSystemConfig();
|
||
if (!pointsConfig.enabled) return;
|
||
|
||
let billingModel: any | undefined;
|
||
for (const ref of params.modelRefs) {
|
||
if (!ref) continue;
|
||
const resolved = await resolveBillingModelByReference(ref);
|
||
if (!resolved) continue;
|
||
billingModel = resolved;
|
||
if (getModelCreditCost(resolved, "video") > 0) break;
|
||
}
|
||
|
||
if (!billingModel) return;
|
||
|
||
const unitCost = getModelCreditCost(billingModel, "video");
|
||
if (unitCost <= 0) return;
|
||
|
||
const consumeResult = await consumeUserPoints({
|
||
userId: params.userId,
|
||
amount: unitCost,
|
||
reason: "video_generation",
|
||
modelId: billingModel.id,
|
||
chatId: params.chatId,
|
||
messageId: params.messageId,
|
||
note: `模型(${billingModel.displayName || billingModel.modelId || billingModel.id}) 生视频 1 条`,
|
||
});
|
||
|
||
if (!consumeResult.ok) {
|
||
console.warn(
|
||
`[Points] Failed to consume video points. user=${params.userId} model=${billingModel.id} cost=${unitCost}`
|
||
);
|
||
} else {
|
||
console.log(
|
||
`[Points] Consumed ${unitCost} points for video. balance=${consumeResult.balance}`
|
||
);
|
||
}
|
||
} catch (error) {
|
||
console.error("[Points] Failed to consume points after video generation:", error);
|
||
}
|
||
}
|
||
|
||
export const maxDuration = 600; // 10 分钟，用于支持图片编辑等耗时任务
|
||
|
||
let globalStreamContext: ResumableStreamContext | null = null;
|
||
const inFlightChatRequests = new Map<string, number>();
|
||
const IN_FLIGHT_REQUEST_TTL_MS = 5 * 60 * 1000;
|
||
const DUPLICATE_REQUEST_WAIT_MS = 2 * 60 * 1000;
|
||
const SMART_AUTO_GRAPH_TIMEOUT_MS = 120000;
|
||
const SMART_AUTO_FALLBACK_TIMEOUT_MS = 120000;
|
||
|
||
const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms));
|
||
|
||
class OperationTimeoutError extends Error {
|
||
constructor(operation: string, timeoutMs: number) {
|
||
super(`${operation} timed out after ${timeoutMs}ms`);
|
||
this.name = "OperationTimeoutError";
|
||
}
|
||
}
|
||
|
||
async function withOperationTimeout<T>(
|
||
operation: Promise<T>,
|
||
operationName: string,
|
||
timeoutMs: number,
|
||
): Promise<T> {
|
||
let timer: ReturnType<typeof setTimeout> | null = null;
|
||
try {
|
||
return await Promise.race([
|
||
operation,
|
||
new Promise<never>((_, reject) => {
|
||
timer = setTimeout(
|
||
() => reject(new OperationTimeoutError(operationName, timeoutMs)),
|
||
timeoutMs,
|
||
);
|
||
}),
|
||
]);
|
||
} finally {
|
||
if (timer) {
|
||
clearTimeout(timer);
|
||
}
|
||
}
|
||
}
|
||
|
||
const getTokenlensCatalog = cache(
|
||
async (): Promise<ModelCatalog | undefined> => {
|
||
try {
|
||
return await fetchModels();
|
||
} catch (err) {
|
||
console.warn(
|
||
"TokenLens: catalog fetch failed, using default catalog",
|
||
err
|
||
);
|
||
return; // tokenlens helpers will fall back to defaultCatalog
|
||
}
|
||
},
|
||
["tokenlens-catalog"],
|
||
{ revalidate: 24 * 60 * 60 } // 24 hours
|
||
);
|
||
|
||
export function getStreamContext() {
|
||
if (!globalStreamContext) {
|
||
try {
|
||
globalStreamContext = createResumableStreamContext({
|
||
waitUntil: after,
|
||
});
|
||
} catch (error: any) {
|
||
if (error.message.includes("REDIS_URL")) {
|
||
console.log(
|
||
" > Resumable streams are disabled due to missing REDIS_URL"
|
||
);
|
||
} else {
|
||
console.error(error);
|
||
}
|
||
}
|
||
}
|
||
|
||
return globalStreamContext;
|
||
}
|
||
|
||
export async function POST(request: Request) {
|
||
let requestBody: PostRequestBody;
|
||
let requestDedupKey: string | null = null;
|
||
let acquiredInFlightLock = false;
|
||
|
||
try {
|
||
const json = await request.json();
|
||
console.log("[Chat API] Received request body:", JSON.stringify(json, null, 2));
|
||
requestBody = postRequestBodySchema.parse(json);
|
||
} catch (error) {
|
||
console.error("[Chat API] Schema validation failed:", error);
|
||
return new ChatSDKError("bad_request:api").toResponse();
|
||
}
|
||
|
||
try {
|
||
const {
|
||
id,
|
||
message,
|
||
selectedChatModel,
|
||
selectedVisibilityType,
|
||
hiddenPrompt,
|
||
autoContext,
|
||
deepThinkingEnabled,
|
||
}: {
|
||
id: string;
|
||
message: ChatMessage;
|
||
selectedChatModel: ChatModel["id"];
|
||
selectedVisibilityType: VisibilityType;
|
||
hiddenPrompt?: string;
|
||
autoContext?: string;
|
||
deepThinkingEnabled?: boolean;
|
||
} = requestBody;
|
||
|
||
const now = Date.now();
|
||
for (const [key, startedAt] of inFlightChatRequests.entries()) {
|
||
if (now - startedAt > IN_FLIGHT_REQUEST_TTL_MS) {
|
||
inFlightChatRequests.delete(key);
|
||
}
|
||
}
|
||
|
||
requestDedupKey = `${id}:${message.id}`;
|
||
if (inFlightChatRequests.has(requestDedupKey)) {
|
||
console.warn(
|
||
`[Chat API] Duplicate in-flight request ignored (chatId=${id}, messageId=${message.id})`
|
||
);
|
||
const duplicateStream = createUIMessageStream({
|
||
execute: async ({ writer }) => {
|
||
await replayAssistantMessagesForDuplicateRequest({
|
||
chatId: id,
|
||
userMessageId: message.id,
|
||
writer,
|
||
source: "duplicate-in-flight",
|
||
});
|
||
},
|
||
});
|
||
return createSseResponse(duplicateStream);
|
||
}
|
||
inFlightChatRequests.set(requestDedupKey, now);
|
||
acquiredInFlightLock = true;
|
||
|
||
let finalModelId = selectedChatModel;
|
||
|
||
// 🆕 存储智能模式的相关信息
|
||
let smartAutoFriendlyMessage: string | undefined; // AI的自然对话回复
|
||
let smartAutoOptimizedPrompt: string | undefined; // 优化后的提示词（不修改用户消息，只用于API调用）
|
||
let smartAutoPromptFromVideoPlan = false;
|
||
let smartAutoRecommendedImageParams: AgentRecommendedImageParams | undefined;
|
||
let smartAutoVideoPlan: AgentVideoPlan | undefined;
|
||
let smartAutoRecommendedVideoParams: AgentRecommendedVideoParams | undefined;
|
||
let smartAutoDetectedIntent: "image_edit" | "vision" | "image_gen" | "video_gen" | "chat" | undefined;
|
||
let smartAutoHistoryImages: string[] = []; // 从历史对话中提取的图片URL（支持多张）
|
||
let smartAutoNeedsClarification = false; // 是否需要先追问用户
|
||
let smartAutoClarificationQuestion: string | undefined;
|
||
let preferredAgentModelId: string | undefined;
|
||
const deepThinkingOn = deepThinkingEnabled !== false;
|
||
let shouldRunAgentConversation = finalModelId === "auto";
|
||
let shouldKeepSelectedModel = false;
|
||
const requestedModelId = selectedChatModel;
|
||
let requestedModelProviderType: string | undefined;
|
||
|
||
if (!shouldRunAgentConversation && requestedModelId !== "auto") {
|
||
const selectedModelIsUUID =
|
||
/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test(requestedModelId);
|
||
let selectedModelConfig = selectedModelIsUUID
|
||
? await getAiModelById({ id: requestedModelId })
|
||
: null;
|
||
let selectedModelProviderType: string | undefined;
|
||
|
||
if (!selectedModelConfig) {
|
||
const enabledModels = await getEnabledAiModelsWithProvider(false);
|
||
const matchedModel = enabledModels.find(
|
||
({ model }) => model.id === requestedModelId || model.modelId === requestedModelId
|
||
);
|
||
if (matchedModel) {
|
||
selectedModelConfig = matchedModel.model;
|
||
selectedModelProviderType = matchedModel.provider.type as string;
|
||
}
|
||
}
|
||
|
||
if (selectedModelConfig && !selectedModelProviderType) {
|
||
const selectedProvider = await getAiProviderById({
|
||
id: selectedModelConfig.providerId,
|
||
});
|
||
selectedModelProviderType = selectedProvider?.type as string | undefined;
|
||
}
|
||
|
||
requestedModelProviderType = selectedModelProviderType;
|
||
|
||
if (selectedModelConfig) {
|
||
const fixedImageProviderTypes = new Set([
|
||
"gemini",
|
||
"huanke-gemini",
|
||
"alibaba-image",
|
||
"ruxa-image",
|
||
"fal-image",
|
||
"volcano-vision",
|
||
]);
|
||
const isFixedVisualModel =
|
||
!selectedModelConfig.isSubModel &&
|
||
(selectedModelConfig.supportsImageGen ||
|
||
(selectedModelProviderType
|
||
? isVideoProviderType(selectedModelProviderType)
|
||
: false) ||
|
||
(selectedModelProviderType
|
||
? fixedImageProviderTypes.has(selectedModelProviderType)
|
||
: false));
|
||
|
||
if (deepThinkingOn) {
|
||
if (isAiModelAgentAuto(selectedModelConfig)) {
|
||
shouldRunAgentConversation = true;
|
||
shouldKeepSelectedModel = false;
|
||
console.log(
|
||
`[Smart Auto Mode] 🧠 Agent-only model detected, force auto routing: ${selectedModelConfig.displayName}`
|
||
);
|
||
} else if (isFixedVisualModel) {
|
||
shouldRunAgentConversation = true;
|
||
shouldKeepSelectedModel = true;
|
||
preferredAgentModelId = selectedModelConfig.id;
|
||
console.log(
|
||
`[Smart Auto Mode] 🎯 Agent conversation enabled for fixed visual model: ${selectedModelConfig.displayName} (${selectedModelProviderType || "unknown"}), preferredAgentModelId=${preferredAgentModelId}`
|
||
);
|
||
}
|
||
} else if (isFixedVisualModel || isAiModelAgentAuto(selectedModelConfig)) {
|
||
console.log(
|
||
`[Smart Auto Mode] ⏭️ Deep thinking disabled, skip agent analysis for: ${selectedModelConfig.displayName}`
|
||
);
|
||
}
|
||
}
|
||
}
|
||
|
||
const allowAgentVideoGeneration =
|
||
Boolean(requestBody.videoParams) || isVideoProviderType(requestedModelProviderType);
|
||
|
||
// ========== Smart Auto Mode Logic ==========
|
||
if (shouldRunAgentConversation) {
|
||
console.log("[Smart Auto Mode] 🚀 Starting intelligent analysis...");
|
||
let autoTextForFallback = "";
|
||
let autoHasImageForFallback = false;
|
||
let autoFromHistoryForFallback = false;
|
||
|
||
try {
|
||
// 🆕 首先获取历史消息（在 Smart Auto Mode 内部单独获取，避免依赖后面的 messagesFromDb）
|
||
const smartAutoHistoryMessagesFromDb = await getMessagesByChatId({ id });
|
||
console.log("[Smart Auto Mode] Loaded", smartAutoHistoryMessagesFromDb.length, "messages from DB");
|
||
|
||
// 1. 提取用户消息内容
|
||
const lastUserMessage = message;
|
||
const textContent = lastUserMessage.parts
|
||
.filter((p): p is { type: "text"; text: string } => p.type === "text")
|
||
.map((p) => p.text)
|
||
.join(" ");
|
||
autoTextForFallback = textContent;
|
||
const autoContextText = typeof autoContext === "string" ? autoContext.trim() : "";
|
||
const hasSelectedImageHint = /用户已选中\s*\d+\s*张图片/.test(autoContextText);
|
||
const hasExplicitTargetHint = /第[一二三四五六七八九十\d]+张|左|右|上|下|中间|选中/.test(textContent);
|
||
const hasActionableEditIntent = /改|调整|优化|换|替换|去掉|抠|重绘|增强|修|处理|按照|按|保留|统一|色差|背景|清晰|风格/.test(
|
||
textContent
|
||
);
|
||
|
||
// @ts-ignore - 检查各种可能的图片类型
|
||
let hasImage = lastUserMessage.parts.some(
|
||
(p: any) => p.type === "file" || p.type === "image" || p.type === "image_url"
|
||
);
|
||
autoHasImageForFallback = hasImage;
|
||
|
||
// @ts-ignore - 提取图片URL
|
||
let imageUrls = lastUserMessage.parts
|
||
.filter((p: any) => p.type === "file" && p.url)
|
||
.map((p: any) => p.url as string);
|
||
|
||
// 🆕 历史图片上下文感知：如果用户没有发送新图片，自动使用历史中最近的图片（支持多张）
|
||
// 逻辑：只要用户在对话中没有附上新图片，就默认继续处理之前的图片
|
||
if (!hasImage && smartAutoHistoryMessagesFromDb.length > 0) {
|
||
console.log("[Smart Auto Mode] 🔍 No image in current message, auto-searching history...");
|
||
|
||
const explicitFollowLatestResult = /(上一张|上一轮|刚生成|最新生成|这个结果|该结果|按这个结果|继续在这张|基于这张结果|延续上一次结果)/i.test(
|
||
textContent
|
||
);
|
||
|
||
const latestAssistantImages: string[] = [];
|
||
const latestUserImages: string[] = [];
|
||
const pushUniqueUrl = (bucket: string[], maybeUrl?: string) => {
|
||
if (!maybeUrl || typeof maybeUrl !== "string") return;
|
||
const normalized = maybeUrl.trim();
|
||
if (!/^https?:\/\//i.test(normalized)) return;
|
||
if (!bucket.includes(normalized)) {
|
||
bucket.push(normalized);
|
||
}
|
||
};
|
||
|
||
// 从历史消息中倒序查找最近的“AI生成图”和“用户上传图”
|
||
for (let i = smartAutoHistoryMessagesFromDb.length - 1; i >= 0; i--) {
|
||
const msg = smartAutoHistoryMessagesFromDb[i];
|
||
const parts = msg.parts as any[];
|
||
|
||
if (!parts) continue;
|
||
|
||
if (msg.role === "assistant" && latestAssistantImages.length === 0) {
|
||
for (const part of parts) {
|
||
if (part.type === "text" && part.text) {
|
||
const parsedImageUrls = extractOriginalImageUrlsFromText(part.text);
|
||
for (const parsedUrl of parsedImageUrls) {
|
||
pushUniqueUrl(latestAssistantImages, parsedUrl);
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
if (msg.role === "user" && latestUserImages.length === 0) {
|
||
for (const part of parts) {
|
||
if ((part.type === "file" || part.type === "image") && part.url) {
|
||
pushUniqueUrl(latestUserImages, part.url);
|
||
}
|
||
if (part.type === "image_url" && part.image_url?.url) {
|
||
pushUniqueUrl(latestUserImages, part.image_url.url);
|
||
}
|
||
if (part.type === "text" && part.text) {
|
||
const parsedImageUrls = extractOriginalImageUrlsFromText(part.text);
|
||
for (const parsedUrl of parsedImageUrls) {
|
||
pushUniqueUrl(latestUserImages, parsedUrl);
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
if (latestAssistantImages.length > 0 && latestUserImages.length > 0) {
|
||
break;
|
||
}
|
||
}
|
||
|
||
if (explicitFollowLatestResult && latestAssistantImages.length > 0) {
|
||
smartAutoHistoryImages = latestAssistantImages.slice(0, 4);
|
||
console.log(
|
||
"[Smart Auto Mode] 🖼️ Using latest generated image(s) by explicit instruction:",
|
||
smartAutoHistoryImages.length
|
||
);
|
||
} else if (latestUserImages.length > 0) {
|
||
smartAutoHistoryImages = latestUserImages.slice(0, 4);
|
||
console.log(
|
||
"[Smart Auto Mode] 🖼️ Using latest user-uploaded image(s) to avoid iterative blur:",
|
||
smartAutoHistoryImages.length
|
||
);
|
||
} else if (latestAssistantImages.length > 0) {
|
||
smartAutoHistoryImages = latestAssistantImages.slice(0, 4);
|
||
console.log(
|
||
"[Smart Auto Mode] 🖼️ Fallback to latest generated image(s):",
|
||
smartAutoHistoryImages.length
|
||
);
|
||
}
|
||
|
||
if (smartAutoHistoryImages.length > 0) {
|
||
hasImage = true;
|
||
imageUrls = smartAutoHistoryImages;
|
||
autoHasImageForFallback = true;
|
||
autoFromHistoryForFallback = true;
|
||
console.log("[Smart Auto Mode] ✅ Auto-using", smartAutoHistoryImages.length, "history image(s) for context continuity");
|
||
} else {
|
||
console.log("[Smart Auto Mode] ℹ️ No image found in history, proceeding as pure text");
|
||
}
|
||
}
|
||
|
||
console.log("[Smart Auto Mode] Input:", { textContent: textContent.substring(0, 100), hasImage, imageCount: imageUrls.length, fromHistory: smartAutoHistoryImages.length > 0 });
|
||
|
||
// 🆕 从历史消息中提取对话文本（用于上下文理解）
|
||
const historyMessages: { role: "user" | "assistant"; content: string }[] = [];
|
||
for (const msg of smartAutoHistoryMessagesFromDb.slice(-10)) { // 最多取最近10条
|
||
const parts = msg.parts as any[];
|
||
if (parts && parts.length > 0) {
|
||
// 提取文本内容
|
||
const textParts = parts.filter((p: any) => p.type === 'text');
|
||
const content = textParts.map((p: any) => p.text || '').join(' ').trim();
|
||
if (content) {
|
||
historyMessages.push({
|
||
role: msg.role as "user" | "assistant",
|
||
content: content.substring(0, 300), // 限制长度
|
||
});
|
||
}
|
||
}
|
||
}
|
||
console.log("[Smart Auto Mode] History context:", historyMessages.length, "messages");
|
||
|
||
// 2. 使用 LangGraph 驱动的自动模式工作流
|
||
const { runAutoAgentGraph } = await import("@/lib/ai/auto-agent-graph");
|
||
const autoGraphResult = await withOperationTimeout(
|
||
runAutoAgentGraph({
|
||
text: textContent,
|
||
hasImage,
|
||
imageUrls,
|
||
imageFromHistory: smartAutoHistoryImages.length > 0,
|
||
historyMessages,
|
||
context: autoContextText || undefined,
|
||
preferredModelId: preferredAgentModelId,
|
||
}),
|
||
"runAutoAgentGraph",
|
||
SMART_AUTO_GRAPH_TIMEOUT_MS,
|
||
);
|
||
|
||
if (autoGraphResult.analysis) {
|
||
console.log("[Smart Auto Mode] Analysis result:", {
|
||
intent: autoGraphResult.analysis.intent,
|
||
subIntent: autoGraphResult.analysis.subIntent,
|
||
confidence: autoGraphResult.analysis.confidence,
|
||
needsClarification: autoGraphResult.analysis.needsClarification,
|
||
});
|
||
|
||
smartAutoDetectedIntent = autoGraphResult.analysis.intent;
|
||
smartAutoNeedsClarification = Boolean(autoGraphResult.analysis.needsClarification);
|
||
smartAutoClarificationQuestion = autoGraphResult.analysis.clarifyingQuestions?.[0];
|
||
|
||
if (
|
||
smartAutoNeedsClarification &&
|
||
hasImage &&
|
||
hasActionableEditIntent &&
|
||
(hasSelectedImageHint || hasExplicitTargetHint)
|
||
) {
|
||
smartAutoNeedsClarification = false;
|
||
smartAutoClarificationQuestion = undefined;
|
||
if (!smartAutoOptimizedPrompt && textContent.trim()) {
|
||
smartAutoOptimizedPrompt = textContent.trim();
|
||
}
|
||
if (
|
||
!smartAutoFriendlyMessage ||
|
||
/[?？]$/.test(smartAutoFriendlyMessage.trim()) ||
|
||
/哪一张|哪个|请先确认/.test(smartAutoFriendlyMessage)
|
||
) {
|
||
smartAutoFriendlyMessage = "收到，我将按你指定的目标图直接执行。";
|
||
}
|
||
console.log(
|
||
"[Smart Auto Mode] ✅ Skip clarification due to explicit target/selection context"
|
||
);
|
||
}
|
||
}
|
||
|
||
if (autoGraphResult.friendlyMessage) {
|
||
smartAutoFriendlyMessage = autoGraphResult.friendlyMessage;
|
||
console.log("[Smart Auto Mode] 💬 Conversational reply:", smartAutoFriendlyMessage);
|
||
}
|
||
|
||
if (
|
||
!smartAutoNeedsClarification &&
|
||
hasImage &&
|
||
hasActionableEditIntent &&
|
||
(hasSelectedImageHint || hasExplicitTargetHint) &&
|
||
smartAutoFriendlyMessage &&
|
||
(/[?？]$/.test(smartAutoFriendlyMessage.trim()) ||
|
||
/哪一张|哪个|请先确认/.test(smartAutoFriendlyMessage))
|
||
) {
|
||
smartAutoFriendlyMessage = "收到，我将按你指定的目标图直接执行。";
|
||
console.log("[Smart Auto Mode] ✅ Replace clarification-like friendly reply");
|
||
}
|
||
|
||
const autoIntent = autoGraphResult.analysis?.intent;
|
||
const isImageTask =
|
||
autoIntent === "image_gen" ||
|
||
autoIntent === "image_edit" ||
|
||
hasImage;
|
||
const needsRicherAutoReply =
|
||
isImageTask &&
|
||
!smartAutoNeedsClarification &&
|
||
!smartAutoFriendlyMessage;
|
||
|
||
if (needsRicherAutoReply) {
|
||
if (isImageTask) {
|
||
smartAutoFriendlyMessage =
|
||
autoGraphResult.analysis?.conversationalReply ||
|
||
"我先分析你这轮图片里的主体、风格和构图，再给你执行方案。";
|
||
} else {
|
||
smartAutoFriendlyMessage = buildAutoModeFallbackMessage({
|
||
text: autoTextForFallback,
|
||
hasImage,
|
||
fromHistory: smartAutoHistoryImages.length > 0,
|
||
});
|
||
}
|
||
console.log("[Smart Auto Mode] 🧩 Filled missing conversational reply");
|
||
}
|
||
|
||
if (autoGraphResult.optimizedPrompt) {
|
||
smartAutoOptimizedPrompt = autoGraphResult.optimizedPrompt;
|
||
console.log("[Smart Auto Mode] ✨ Prompt optimized (LLM):", {
|
||
original: textContent.substring(0, 50),
|
||
optimized: autoGraphResult.optimizedPrompt.substring(0, 100),
|
||
});
|
||
}
|
||
if (autoGraphResult.videoPlan) {
|
||
smartAutoVideoPlan = autoGraphResult.videoPlan;
|
||
smartAutoRecommendedVideoParams = autoGraphResult.videoPlan.recommendedVideoParams;
|
||
if (!smartAutoOptimizedPrompt && autoGraphResult.videoPlan.finalPrompt?.trim()) {
|
||
smartAutoOptimizedPrompt = autoGraphResult.videoPlan.finalPrompt.trim();
|
||
smartAutoPromptFromVideoPlan = true;
|
||
}
|
||
console.log("[Smart Auto Mode] 🎬 Agent video plan ready:", {
|
||
hasScript: Boolean(autoGraphResult.videoPlan.script),
|
||
storyboardCount: autoGraphResult.videoPlan.storyboard?.length || 0,
|
||
hasFinalPrompt: Boolean(autoGraphResult.videoPlan.finalPrompt),
|
||
recommendedVideoParams: smartAutoRecommendedVideoParams,
|
||
});
|
||
}
|
||
if (autoGraphResult.recommendedImageParams) {
|
||
smartAutoRecommendedImageParams = autoGraphResult.recommendedImageParams;
|
||
console.log("[Smart Auto Mode] 🎛️ Agent recommended image params:", smartAutoRecommendedImageParams);
|
||
}
|
||
|
||
if (autoGraphResult.route && !shouldKeepSelectedModel) {
|
||
const routeIsVideo = isVideoProviderType(autoGraphResult.route.providerType);
|
||
if (routeIsVideo && !allowAgentVideoGeneration) {
|
||
const fallbackIntent: "image_edit" | "image_gen" = hasImage ? "image_edit" : "image_gen";
|
||
smartAutoDetectedIntent = fallbackIntent;
|
||
smartAutoVideoPlan = undefined;
|
||
smartAutoRecommendedVideoParams = undefined;
|
||
if (smartAutoPromptFromVideoPlan) {
|
||
smartAutoOptimizedPrompt = textContent.trim() || undefined;
|
||
smartAutoPromptFromVideoPlan = false;
|
||
}
|
||
smartAutoFriendlyMessage =
|
||
smartAutoFriendlyMessage && !/视频/.test(smartAutoFriendlyMessage)
|
||
? smartAutoFriendlyMessage
|
||
: "当前 Agent 默认按图片任务执行；如需生成视频，请先手动切换到“视频”模式。";
|
||
console.log(
|
||
`[Smart Auto Mode] 🚫 Blocked auto video route (${autoGraphResult.route.displayName}), fallback to ${fallbackIntent}`,
|
||
);
|
||
try {
|
||
const { routeToModel } = await import("@/lib/ai/model-router");
|
||
const fallbackRoute = await routeToModel({
|
||
intent: fallbackIntent,
|
||
subIntent: autoGraphResult.analysis?.subIntent,
|
||
hasImage,
|
||
});
|
||
if (!isVideoProviderType(fallbackRoute.providerType)) {
|
||
finalModelId = fallbackRoute.id;
|
||
console.log(
|
||
`[Smart Auto Mode] ✅ Fallback routed to: ${fallbackRoute.displayName} (${fallbackRoute.reason})`,
|
||
);
|
||
} else {
|
||
console.warn(
|
||
`[Smart Auto Mode] ⚠️ Fallback route still video (${fallbackRoute.displayName}), keep previous model`,
|
||
);
|
||
}
|
||
} catch (fallbackRouteError) {
|
||
console.error("[Smart Auto Mode] Failed to resolve non-video fallback route:", fallbackRouteError);
|
||
}
|
||
} else {
|
||
finalModelId = autoGraphResult.route.id;
|
||
console.log(
|
||
`[Smart Auto Mode] ✅ Routed to: ${autoGraphResult.route.displayName} (${autoGraphResult.route.reason})`,
|
||
);
|
||
}
|
||
} else if (autoGraphResult.route && shouldKeepSelectedModel) {
|
||
finalModelId = requestedModelId;
|
||
console.log("[Smart Auto Mode] 🔒 Keep user-selected model, skip routing override");
|
||
}
|
||
|
||
} catch (autoError) {
|
||
// 智能分析失败时，回退到简单逻辑
|
||
console.error("[Smart Auto Mode] ❌ Error, falling back:", autoError);
|
||
const isAutoTimeout = autoError instanceof OperationTimeoutError;
|
||
if (isAutoTimeout) {
|
||
console.warn(
|
||
`[Smart Auto Mode] ⏱️ Auto-agent timed out after ${SMART_AUTO_GRAPH_TIMEOUT_MS}ms, using local fallback`,
|
||
);
|
||
}
|
||
try {
|
||
const { analyzeIntent } = await import("@/lib/ai/smart-analyzer");
|
||
const rescueAnalysis = await withOperationTimeout(
|
||
analyzeIntent({
|
||
text: autoTextForFallback,
|
||
hasImage: autoHasImageForFallback || smartAutoHistoryImages.length > 0,
|
||
imageUrls: smartAutoHistoryImages,
|
||
imageFromHistory: autoFromHistoryForFallback || smartAutoHistoryImages.length > 0,
|
||
historyMessages: [],
|
||
context: undefined,
|
||
preferredModelId: preferredAgentModelId,
|
||
}),
|
||
"fallbackAnalyzeIntent",
|
||
SMART_AUTO_FALLBACK_TIMEOUT_MS,
|
||
);
|
||
if (rescueAnalysis.conversationalReply?.trim()) {
|
||
smartAutoFriendlyMessage = rescueAnalysis.conversationalReply.trim();
|
||
}
|
||
} catch (rescueError) {
|
||
console.error("[Smart Auto Mode] Rescue analysis failed:", rescueError);
|
||
}
|
||
|
||
const allModels = await getEnabledAiModelsWithProvider();
|
||
// @ts-ignore - 检查各种可能的图片类型
|
||
const hasImage = autoHasImageForFallback || message.parts.some(
|
||
(p: any) => p.type === "file" || p.type === "image" || p.type === "image_url"
|
||
);
|
||
let fallbackVisualSummary: string | undefined;
|
||
|
||
if (isAutoTimeout && hasImage) {
|
||
try {
|
||
const extractCurrentImageUrls = (parts: any[]) =>
|
||
parts
|
||
.map((part: any) => {
|
||
if (!part) return null;
|
||
if (part.type === "file") return part.url || part.data || null;
|
||
if (part.type === "image") return part.url || part.image || null;
|
||
if (part.type === "image_url") return part.image_url?.url || null;
|
||
return null;
|
||
})
|
||
.filter((url: any): url is string => typeof url === "string" && /^https?:\/\//i.test(url));
|
||
|
||
const currentImageUrls = extractCurrentImageUrls(message.parts as any[]);
|
||
const summaryImageUrls =
|
||
currentImageUrls.length > 0 ? currentImageUrls : smartAutoHistoryImages;
|
||
|
||
if (summaryImageUrls.length > 0) {
|
||
const { summarizeImageForExecutionReport } = await import("@/lib/ai/smart-analyzer");
|
||
const summary = await withOperationTimeout(
|
||
summarizeImageForExecutionReport({
|
||
text: autoTextForFallback,
|
||
imageUrls: summaryImageUrls,
|
||
context: typeof autoContext === "string" ? autoContext : undefined,
|
||
preferredModelId: preferredAgentModelId,
|
||
}),
|
||
"fallbackVisualSummary",
|
||
SMART_AUTO_FALLBACK_TIMEOUT_MS,
|
||
);
|
||
if (summary?.trim()) {
|
||
fallbackVisualSummary = summary.trim();
|
||
}
|
||
}
|
||
} catch (summaryError) {
|
||
console.warn("[Smart Auto Mode] Fallback visual summary failed:", summaryError);
|
||
}
|
||
}
|
||
|
||
// 简单回退：有图选图片模型，无图选默认模型
|
||
let bestModel = null;
|
||
if (hasImage) {
|
||
bestModel = allModels.find(
|
||
(m) =>
|
||
m.model.supportsImageGen &&
|
||
!m.model.isSubModel &&
|
||
!isVideoProviderType(m.provider?.type),
|
||
);
|
||
if (isAutoTimeout) {
|
||
smartAutoFriendlyMessage = buildAutoModeFallbackMessage({
|
||
text: autoTextForFallback,
|
||
hasImage: true,
|
||
fromHistory: autoFromHistoryForFallback || smartAutoHistoryImages.length > 0,
|
||
imageSummary: fallbackVisualSummary,
|
||
});
|
||
} else {
|
||
smartAutoFriendlyMessage =
|
||
smartAutoFriendlyMessage ||
|
||
buildAutoModeFallbackMessage({
|
||
text: autoTextForFallback,
|
||
hasImage: true,
|
||
fromHistory: autoFromHistoryForFallback || smartAutoHistoryImages.length > 0,
|
||
imageSummary: fallbackVisualSummary,
|
||
});
|
||
}
|
||
} else {
|
||
smartAutoFriendlyMessage = smartAutoFriendlyMessage || buildAutoModeFallbackMessage({
|
||
text: autoTextForFallback,
|
||
hasImage: false,
|
||
fromHistory: false,
|
||
});
|
||
}
|
||
if (!bestModel) {
|
||
bestModel = allModels.find(
|
||
(m) => m.model.isDefault && !m.model.isSubModel && !isVideoProviderType(m.provider?.type),
|
||
);
|
||
}
|
||
if (!bestModel && allModels.length > 0) {
|
||
bestModel = allModels.find(
|
||
(m) => !m.model.isSubModel && !isVideoProviderType(m.provider?.type),
|
||
) || allModels.find((m) => !m.model.isSubModel) || allModels[0];
|
||
}
|
||
|
||
if (shouldKeepSelectedModel) {
|
||
finalModelId = requestedModelId;
|
||
console.log("[Smart Auto Mode] 🔒 Keep user-selected model in fallback mode");
|
||
} else if (bestModel) {
|
||
finalModelId = bestModel.model.id;
|
||
console.log(`[Smart Auto Mode] Fallback selected: ${bestModel.model.displayName}`);
|
||
} else {
|
||
return new ChatSDKError("bad_request:api", "No available models for auto mode").toResponse();
|
||
}
|
||
}
|
||
}
|
||
// ========== End Smart Auto Mode Logic ==========
|
||
|
||
if (shouldRunAgentConversation && !allowAgentVideoGeneration) {
|
||
const finalModelIsUuid =
|
||
/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test(finalModelId);
|
||
let finalProviderType: string | undefined;
|
||
|
||
if (finalModelIsUuid) {
|
||
const finalModelConfig = await getAiModelById({ id: finalModelId }).catch(() => null);
|
||
if (finalModelConfig) {
|
||
finalProviderType = (
|
||
await getAiProviderById({ id: finalModelConfig.providerId }).catch(() => null)
|
||
)?.type as string | undefined;
|
||
}
|
||
}
|
||
|
||
if (isVideoProviderType(finalProviderType)) {
|
||
const allModels = await getEnabledAiModelsWithProvider(false);
|
||
const fallbackModel =
|
||
allModels.find(
|
||
(m) =>
|
||
!m.model.isSubModel &&
|
||
m.model.supportsImageGen &&
|
||
!isVideoProviderType(m.provider?.type),
|
||
) ||
|
||
allModels.find(
|
||
(m) =>
|
||
!m.model.isSubModel &&
|
||
!isVideoProviderType(m.provider?.type),
|
||
);
|
||
|
||
if (fallbackModel) {
|
||
const hasImageInput = message.parts.some((part: any) =>
|
||
part?.type === "file" || part?.type === "image" || part?.type === "image_url"
|
||
);
|
||
smartAutoDetectedIntent = hasImageInput ? "image_edit" : "image_gen";
|
||
finalModelId = fallbackModel.model.id;
|
||
smartAutoVideoPlan = undefined;
|
||
smartAutoRecommendedVideoParams = undefined;
|
||
console.log(
|
||
`[Smart Auto Mode] 🚫 Safety override: replace video model (${finalProviderType}) with ${fallbackModel.model.displayName}`,
|
||
);
|
||
}
|
||
}
|
||
}
|
||
|
||
if (
|
||
shouldRunAgentConversation &&
|
||
!smartAutoNeedsClarification &&
|
||
!smartAutoFriendlyMessage?.trim()
|
||
) {
|
||
const latestUserText = message.parts
|
||
.filter((part): part is { type: "text"; text: string } => part.type === "text")
|
||
.map((part) => part.text)
|
||
.join(" ");
|
||
const hasCurrentImageInput = message.parts.some((part: any) =>
|
||
part?.type === "file" || part?.type === "image" || part?.type === "image_url"
|
||
);
|
||
const hasHistoryImageInput = smartAutoHistoryImages.length > 0;
|
||
smartAutoFriendlyMessage = buildAutoModeFallbackMessage({
|
||
text: latestUserText,
|
||
hasImage: hasCurrentImageInput || hasHistoryImageInput,
|
||
fromHistory: !hasCurrentImageInput && hasHistoryImageInput,
|
||
});
|
||
console.log("[Smart Auto Mode] 🧩 Backfilled conversational reply with fallback");
|
||
}
|
||
|
||
if (shouldRunAgentConversation && !smartAutoNeedsClarification) {
|
||
const latestUserText = message.parts
|
||
.filter((part): part is { type: "text"; text: string } => part.type === "text")
|
||
.map((part) => part.text)
|
||
.join(" ");
|
||
const hasCurrentImageInput = message.parts.some((part: any) =>
|
||
part?.type === "file" || part?.type === "image" || part?.type === "image_url"
|
||
);
|
||
const hasHistoryImageInput = smartAutoHistoryImages.length > 0;
|
||
smartAutoFriendlyMessage = ensureStructuredAgentFriendlyMessage({
|
||
message: smartAutoFriendlyMessage,
|
||
text: latestUserText,
|
||
hasImage: hasCurrentImageInput || hasHistoryImageInput,
|
||
fromHistory: !hasCurrentImageInput && hasHistoryImageInput,
|
||
});
|
||
}
|
||
|
||
const hasUserCustomImageParams = hasUserCustomizedImageParams(requestBody.imageParams);
|
||
const hasUserCustomVideoParams = hasUserCustomizedVideoParams(requestBody.videoParams);
|
||
const shouldApplyAgentImageParams =
|
||
shouldRunAgentConversation &&
|
||
!smartAutoNeedsClarification &&
|
||
(smartAutoDetectedIntent === "image_gen" || smartAutoDetectedIntent === "image_edit");
|
||
const shouldApplyAgentVideoParams =
|
||
shouldRunAgentConversation &&
|
||
!smartAutoNeedsClarification &&
|
||
allowAgentVideoGeneration &&
|
||
smartAutoDetectedIntent === "video_gen";
|
||
const effectiveImageParams = shouldApplyAgentImageParams
|
||
? mergeImageParamsWithAgentRecommendation({
|
||
incoming: requestBody.imageParams,
|
||
recommended: smartAutoRecommendedImageParams,
|
||
intent: smartAutoDetectedIntent,
|
||
hasUserCustomSettings: hasUserCustomImageParams,
|
||
})
|
||
: requestBody.imageParams;
|
||
const effectiveVideoParams = shouldApplyAgentVideoParams
|
||
? mergeVideoParamsWithAgentRecommendation({
|
||
incoming: requestBody.videoParams,
|
||
recommended: smartAutoRecommendedVideoParams,
|
||
hasUserCustomSettings: hasUserCustomVideoParams,
|
||
})
|
||
: requestBody.videoParams;
|
||
const defaultImageQualityParams = {
|
||
aspectRatio: "auto",
|
||
imageSize: smartAutoDetectedIntent === "image_edit" ? "4K" : "2K",
|
||
imageCount: 1,
|
||
} as const;
|
||
|
||
if (shouldApplyAgentImageParams) {
|
||
console.log("[Smart Auto Mode] 🎯 Effective image params:", {
|
||
incoming: requestBody.imageParams,
|
||
recommended: smartAutoRecommendedImageParams,
|
||
merged: effectiveImageParams,
|
||
hasUserCustomImageParams,
|
||
});
|
||
}
|
||
if (shouldApplyAgentVideoParams) {
|
||
console.log("[Smart Auto Mode] 🎯 Effective video params:", {
|
||
incoming: requestBody.videoParams,
|
||
recommended: smartAutoRecommendedVideoParams,
|
||
merged: effectiveVideoParams,
|
||
hasUserCustomVideoParams,
|
||
hasVideoPlan: Boolean(smartAutoVideoPlan),
|
||
});
|
||
}
|
||
|
||
if (shouldRunAgentConversation && smartAutoFriendlyMessage) {
|
||
smartAutoFriendlyMessage = sanitizeAgentOutputSettingsSection({
|
||
message: smartAutoFriendlyMessage,
|
||
userImageParams: requestBody.imageParams,
|
||
hasUserCustomImageParams,
|
||
});
|
||
}
|
||
|
||
if (shouldRunAgentConversation && !smartAutoNeedsClarification) {
|
||
const currentImageUrls = extractMessageImageUrls(message.parts as any[]);
|
||
const summaryImageUrls =
|
||
currentImageUrls.length > 0 ? currentImageUrls : smartAutoHistoryImages;
|
||
const shouldRequireImageAnalysis = summaryImageUrls.length > 0;
|
||
|
||
if (shouldRequireImageAnalysis) {
|
||
try {
|
||
const { summarizeImageForExecutionReport } = await import("@/lib/ai/smart-analyzer");
|
||
const latestUserText = message.parts
|
||
.filter((part): part is { type: "text"; text: string } => part.type === "text")
|
||
.map((part) => part.text)
|
||
.join(" ");
|
||
|
||
let summary: string | null = null;
|
||
const summaryTimeouts = [120000];
|
||
for (const timeoutMs of summaryTimeouts) {
|
||
summary = await withOperationTimeout(
|
||
summarizeImageForExecutionReport({
|
||
text: latestUserText,
|
||
imageUrls: summaryImageUrls,
|
||
context: typeof autoContext === "string" ? autoContext : undefined,
|
||
preferredModelId: preferredAgentModelId,
|
||
}),
|
||
"enrichImageAnalysisSummary",
|
||
timeoutMs,
|
||
);
|
||
if (summary?.trim()) break;
|
||
}
|
||
|
||
if (summary?.trim()) {
|
||
const baseMessage =
|
||
smartAutoFriendlyMessage ||
|
||
buildAutoModeFallbackMessage({
|
||
text: latestUserText,
|
||
hasImage: true,
|
||
fromHistory: currentImageUrls.length === 0 && summaryImageUrls.length > 0,
|
||
imageSummary: summary.trim(),
|
||
});
|
||
smartAutoFriendlyMessage = ensureImageAnalysisSummarySection(baseMessage, summary.trim());
|
||
console.log(
|
||
"[Smart Auto Mode] ✅ Image analysis summary ready:",
|
||
summary.trim().slice(0, 160),
|
||
);
|
||
} else if (smartAutoFriendlyMessage && shouldEnrichImageAnalysisText(smartAutoFriendlyMessage)) {
|
||
console.warn("[Smart Auto Mode] ⚠️ Image summary unavailable, keep original analysis text");
|
||
}
|
||
} catch (enrichError) {
|
||
console.warn("[Smart Auto Mode] Skip image analysis enrichment:", enrichError);
|
||
}
|
||
}
|
||
}
|
||
|
||
if (shouldRunAgentConversation && smartAutoFriendlyMessage) {
|
||
smartAutoFriendlyMessage = normalizeAgentReportForDisplay(smartAutoFriendlyMessage);
|
||
}
|
||
|
||
const session = await auth();
|
||
|
||
if (!session?.user) {
|
||
return new ChatSDKError("unauthorized:chat").toResponse();
|
||
}
|
||
|
||
const userType: UserType = session.user.type;
|
||
|
||
const messageCount = await getMessageCountByUserId({
|
||
id: session.user.id,
|
||
differenceInHours: 24,
|
||
});
|
||
|
||
if (messageCount > entitlementsByUserType[userType].maxMessagesPerDay) {
|
||
return new ChatSDKError("rate_limit:chat").toResponse();
|
||
}
|
||
|
||
// ========== 使用时间限制验证（后端安全检查） ==========
|
||
try {
|
||
// 从数据库获取网站设置
|
||
const { siteSettings } = await import("@/lib/db/schema");
|
||
const [settings] = await db.select().from(siteSettings).limit(1);
|
||
|
||
if (settings?.usageTimeEnabled) {
|
||
const now = new Date();
|
||
const currentTime = `${String(now.getHours()).padStart(2, "0")}:${String(now.getMinutes()).padStart(2, "0")}`;
|
||
const startTime = settings.usageTimeStart || "08:00";
|
||
const endTime = settings.usageTimeEnd || "20:00";
|
||
|
||
const isWithinUsageTime = currentTime >= startTime && currentTime <= endTime;
|
||
|
||
if (!isWithinUsageTime) {
|
||
console.log(`[Chat API] ⛔ 访问被拒绝：当前时间 ${currentTime} 不在允许范围 ${startTime}-${endTime}`);
|
||
return NextResponse.json(
|
||
{
|
||
error: "usage_time_restricted",
|
||
message: settings.usageTimeMessage || "当前不在系统使用时间段内",
|
||
currentTime,
|
||
allowedTime: `${startTime} - ${endTime}`
|
||
},
|
||
{ status: 403 }
|
||
);
|
||
}
|
||
|
||
console.log(`[Chat API] ✅ 时间验证通过：${currentTime} 在允许范围 ${startTime}-${endTime}`);
|
||
}
|
||
} catch (timeCheckError) {
|
||
console.error("[Chat API] ⚠️ 时间验证失败（继续执行）:", timeCheckError);
|
||
// 如果时间检查失败，继续执行（避免因检查逻辑错误导致系统不可用）
|
||
}
|
||
|
||
const chat = await getChatById({ id });
|
||
let messagesFromDb: DBMessage[] = [];
|
||
|
||
if (chat) {
|
||
if (chat.userId !== session.user.id) {
|
||
return new ChatSDKError("forbidden:chat").toResponse();
|
||
}
|
||
// Only fetch messages if chat already exists
|
||
messagesFromDb = await getMessagesByChatId({ id });
|
||
} else {
|
||
const title = await generateTitleFromUserMessage({
|
||
message,
|
||
});
|
||
|
||
await saveChat({
|
||
id,
|
||
userId: session.user.id,
|
||
title,
|
||
visibility: selectedVisibilityType,
|
||
});
|
||
// New chat - no need to fetch messages, it's empty
|
||
}
|
||
|
||
const uiMessages = [...convertToUIMessages(messagesFromDb), message];
|
||
const hiddenPromptText = typeof hiddenPrompt === "string" ? hiddenPrompt.trim() : "";
|
||
const hiddenPromptPrefix = hiddenPromptText ? `${hiddenPromptText}\n\n` : "";
|
||
const uiMessagesForModel = hiddenPromptText
|
||
? applyHiddenPromptToMessages(uiMessages, hiddenPromptText)
|
||
: uiMessages;
|
||
|
||
const { longitude, latitude, city, country } = geolocation(request);
|
||
|
||
const requestHints: RequestHints = {
|
||
longitude,
|
||
latitude,
|
||
city,
|
||
country,
|
||
};
|
||
|
||
const insertedUserMessage = await db
|
||
.insert(messageTable)
|
||
.values({
|
||
chatId: id,
|
||
id: message.id,
|
||
role: "user",
|
||
parts: message.parts as any,
|
||
attachments: [],
|
||
createdAt: new Date(),
|
||
videoStorageStatus: null,
|
||
temporaryVideoUrl: null,
|
||
permanentVideoUrl: null,
|
||
archivedAt: null,
|
||
videoMetadata: null,
|
||
})
|
||
.onConflictDoNothing({
|
||
target: [messageTable.id],
|
||
})
|
||
.returning({ id: messageTable.id });
|
||
|
||
if (insertedUserMessage.length === 0) {
|
||
console.warn(
|
||
`[Chat API] Duplicate user message ignored (chatId=${id}, messageId=${message.id})`
|
||
);
|
||
const duplicateStream = createUIMessageStream({
|
||
execute: async ({ writer }) => {
|
||
await replayAssistantMessagesForDuplicateRequest({
|
||
chatId: id,
|
||
userMessageId: message.id,
|
||
writer,
|
||
source: "duplicate-user-message",
|
||
});
|
||
},
|
||
});
|
||
return createSseResponse(duplicateStream);
|
||
}
|
||
|
||
const streamId = generateUUID();
|
||
await createStreamId({ streamId, chatId: id });
|
||
|
||
if (shouldRunAgentConversation && smartAutoNeedsClarification) {
|
||
const hasCurrentImageInput = message.parts.some((part: any) =>
|
||
part?.type === "file" || part?.type === "image" || part?.type === "image_url"
|
||
);
|
||
const hasHistoryImageInput = smartAutoHistoryImages.length > 0;
|
||
const latestUserText = message.parts
|
||
.filter((part): part is { type: "text"; text: string } => part.type === "text")
|
||
.map((part) => part.text)
|
||
.join(" ")
|
||
.trim();
|
||
|
||
if (hasCurrentImageInput || hasHistoryImageInput || latestUserText.length > 0) {
|
||
smartAutoNeedsClarification = false;
|
||
smartAutoClarificationQuestion = undefined;
|
||
if (
|
||
!smartAutoFriendlyMessage ||
|
||
/[?？]$/.test(smartAutoFriendlyMessage.trim()) ||
|
||
/哪一张|哪个|请先确认/.test(smartAutoFriendlyMessage)
|
||
) {
|
||
smartAutoFriendlyMessage = "收到，我将按你的当前需求直接执行。";
|
||
}
|
||
console.log("[Smart Auto Mode] ✅ Force direct execution (skip clarification)");
|
||
}
|
||
}
|
||
|
||
if (shouldRunAgentConversation && smartAutoNeedsClarification) {
|
||
const clarificationMessage =
|
||
smartAutoClarificationQuestion ||
|
||
smartAutoFriendlyMessage ||
|
||
"我先确认一下，你希望我优先处理哪一部分？";
|
||
|
||
const clarificationStream = createUIMessageStream({
|
||
execute: async ({ writer }) => {
|
||
const messageId = generateUUID();
|
||
writer.write({
|
||
type: "data-appendMessage",
|
||
data: JSON.stringify({
|
||
id: messageId,
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: clarificationMessage }],
|
||
}),
|
||
});
|
||
|
||
await saveMessages({
|
||
messages: [
|
||
{
|
||
id: messageId,
|
||
chatId: id,
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: clarificationMessage }],
|
||
attachments: [],
|
||
createdAt: new Date(),
|
||
videoStorageStatus: null,
|
||
temporaryVideoUrl: null,
|
||
permanentVideoUrl: null,
|
||
archivedAt: null,
|
||
videoMetadata: null,
|
||
},
|
||
],
|
||
});
|
||
},
|
||
});
|
||
|
||
return createSseResponse(clarificationStream);
|
||
}
|
||
|
||
let finalMergedUsage: AppUsage | undefined;
|
||
|
||
// 获取启用的工具列表
|
||
const enabledTools = await getEnabledTools();
|
||
|
||
// 检查是否是 UUID 格式（数据库中的模型 ID）
|
||
const isUUID =
|
||
/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test(
|
||
finalModelId
|
||
);
|
||
|
||
// 获取语言模型实例
|
||
let languageModel;
|
||
let actualModelId: string;
|
||
let resolvedModelSupportsImageGen = false;
|
||
let resolvedProviderType: string | undefined;
|
||
|
||
if (isUUID) {
|
||
// 从数据库获取动态配置的模型
|
||
let targetModelId = finalModelId;
|
||
|
||
// 🆕 检查是否是视频主模型，如果是，根据 videoParams.mode 选择子模型
|
||
const aiModelData = await getAiModelById({ id: finalModelId });
|
||
const aiModelProvider = aiModelData
|
||
? await getAiProviderById({ id: aiModelData.providerId }).catch(() => null)
|
||
: null;
|
||
console.log("[Chat API] Model data:", { id: finalModelId, isSubModel: aiModelData?.isSubModel });
|
||
|
||
if (aiModelData && !aiModelData.isSubModel && aiModelProvider?.type === "volcano-video") {
|
||
// 检查是否有子模型
|
||
const subModels = await getSubModelsByParentModelId(finalModelId);
|
||
console.log("[Chat API] Found sub-models:", subModels.length);
|
||
|
||
if (subModels.length > 0) {
|
||
// 这是一个主模型，需要根据 videoParams 选择子模型
|
||
const videoParams = effectiveVideoParams;
|
||
console.log("[Chat API] Video params:", videoParams);
|
||
|
||
// 如果没有 videoParams 或没有 mode，使用默认模式
|
||
const mode = (videoParams?.mode || "text-to-video") as VideoModeType;
|
||
console.log("[Chat API] Using mode:", mode);
|
||
|
||
const matchedSubModel = selectBestVideoSubModelForMode(subModels as any, mode);
|
||
|
||
if (matchedSubModel) {
|
||
console.log(
|
||
`[Chat API] 视频模式 ${mode} → 选择子模型: ${matchedSubModel.model.displayName} (${matchedSubModel.model.modelId})`
|
||
);
|
||
targetModelId = matchedSubModel.model.id;
|
||
} else {
|
||
console.warn(`[Chat API] 未找到可用子模型，继续使用主模型`);
|
||
}
|
||
}
|
||
}
|
||
|
||
const dynamicModel = await getLanguageModelById(targetModelId);
|
||
if (!dynamicModel) {
|
||
return new ChatSDKError(
|
||
"bad_request:api",
|
||
"Model not found or disabled"
|
||
).toResponse();
|
||
}
|
||
languageModel = dynamicModel;
|
||
// 对于动态模型，我们需要从数据库获取 modelId
|
||
const finalModelData = await getAiModelById({ id: targetModelId });
|
||
actualModelId = finalModelData?.modelId ?? targetModelId;
|
||
resolvedModelSupportsImageGen = Boolean(finalModelData?.supportsImageGen);
|
||
if (finalModelData) {
|
||
resolvedProviderType = (
|
||
await getAiProviderById({ id: finalModelData.providerId }).catch(() => null)
|
||
)?.type;
|
||
}
|
||
} else {
|
||
// 使用预定义的模型
|
||
languageModel = myProvider.languageModel(finalModelId);
|
||
actualModelId = languageModel.modelId;
|
||
resolvedProviderType = (languageModel as any)?.provider;
|
||
}
|
||
|
||
// ========== 图片生成配额检查 ==========
|
||
const isImageGenerationModel =
|
||
(languageModel as any).provider === "gemini-image" ||
|
||
(languageModel as any).provider === "volcano-vision" ||
|
||
(languageModel as any).provider === "alibaba-image" ||
|
||
(languageModel as any).provider === "ruxa-image" ||
|
||
(languageModel as any).provider === "fal-image" ||
|
||
(languageModel as any).provider === "huanke-gemini";
|
||
|
||
const isUuidModelId = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test(
|
||
finalModelId
|
||
);
|
||
const currentModelConfig = isUuidModelId
|
||
? await getAiModelById({ id: finalModelId }).catch(() => undefined)
|
||
: undefined;
|
||
const modelCreditCostPerImage = Number(
|
||
(currentModelConfig as any)?.imageGenConfig?.creditCostPerImage ?? 0
|
||
);
|
||
const modelCreditCostPerVideo = getModelCreditCost(currentModelConfig, "video");
|
||
|
||
if (isImageGenerationModel) {
|
||
const quota = await checkUserImageQuota(session.user.id);
|
||
|
||
if (!quota.allowed) {
|
||
console.log(`[Chat API] ⛔ 图片配额已用完: ${quota.used}/${quota.limit}`);
|
||
return NextResponse.json(
|
||
{
|
||
error: "quota_exceeded",
|
||
message: `本月图片生成配额已用完（已使用 ${quota.used}/${quota.limit} 张），配额将于下月1日重置，如需更多配额请联系管理员。`,
|
||
used: quota.used,
|
||
limit: quota.limit,
|
||
},
|
||
{ status: 429 }
|
||
);
|
||
}
|
||
|
||
console.log(`[Chat API] ✅ 图片配额检查通过: ${quota.used}/${quota.limit ?? '无限制'}`);
|
||
|
||
const pointsConfig = await getPointsSystemConfig();
|
||
const normalizedUnitCost = Number.isFinite(modelCreditCostPerImage)
|
||
? Math.max(0, Math.round(modelCreditCostPerImage))
|
||
: 0;
|
||
|
||
if (pointsConfig.enabled && normalizedUnitCost > 0 && pointsConfig.blockOnInsufficient !== false) {
|
||
const requestedImageCount = Math.max(
|
||
1,
|
||
Math.round(
|
||
Number(
|
||
effectiveImageParams?.imageCount || defaultImageQualityParams.imageCount || 1
|
||
)
|
||
)
|
||
);
|
||
const expectedCost = normalizedUnitCost * requestedImageCount;
|
||
const userData = await getUserById({ id: session.user.id });
|
||
const currentBalance = Math.max(0, Number(userData?.pointsBalance || 0));
|
||
|
||
if (currentBalance < expectedCost) {
|
||
return NextResponse.json(
|
||
{
|
||
error: "insufficient_points",
|
||
message: `积分不足，当前余额 ${currentBalance}，本次预计消耗 ${expectedCost}（${normalizedUnitCost} x ${requestedImageCount} 张）。`,
|
||
balance: currentBalance,
|
||
expectedCost,
|
||
unitCost: normalizedUnitCost,
|
||
},
|
||
{ status: 402 }
|
||
);
|
||
}
|
||
}
|
||
}
|
||
|
||
const isVideoGenerationModel = [
|
||
"qianwen-video",
|
||
"fal-video",
|
||
"volcano-video",
|
||
"doubao-video",
|
||
].includes(String(resolvedProviderType || (languageModel as any).provider || ""));
|
||
|
||
if (isVideoGenerationModel) {
|
||
const pointsConfig = await getPointsSystemConfig();
|
||
const normalizedVideoUnitCost = Number.isFinite(modelCreditCostPerVideo)
|
||
? Math.max(0, Math.round(modelCreditCostPerVideo))
|
||
: 0;
|
||
|
||
if (
|
||
pointsConfig.enabled &&
|
||
normalizedVideoUnitCost > 0 &&
|
||
pointsConfig.blockOnInsufficient !== false
|
||
) {
|
||
const expectedCost = normalizedVideoUnitCost;
|
||
const userData = await getUserById({ id: session.user.id });
|
||
const currentBalance = Math.max(0, Number(userData?.pointsBalance || 0));
|
||
|
||
if (currentBalance < expectedCost) {
|
||
return NextResponse.json(
|
||
{
|
||
error: "insufficient_points",
|
||
message: `积分不足，当前余额 ${currentBalance}，本次预计消耗 ${expectedCost}（${normalizedVideoUnitCost} x 1 条视频）。`,
|
||
balance: currentBalance,
|
||
expectedCost,
|
||
unitCost: normalizedVideoUnitCost,
|
||
},
|
||
{ status: 402 }
|
||
);
|
||
}
|
||
}
|
||
}
|
||
|
||
// ========== Gemini 图片生成独立处理 ==========
|
||
// 支持 gemini-image 和 huanke-gemini 两种图片生成 provider
|
||
const isGeminiImageProvider = ["gemini-image", "huanke-gemini"].includes((languageModel as any).provider);
|
||
if (isGeminiImageProvider && typeof (languageModel as any).doGenerate === "function") {
|
||
console.log("[Chat API] Using Gemini image generation, provider:", (languageModel as any).provider);
|
||
console.log("[Chat API] Image params:", effectiveImageParams);
|
||
|
||
// 🆕 获取备用模型配置
|
||
const currentModelConfig = await getAiModelById({ id: finalModelId });
|
||
const configuredFallbacks = [
|
||
currentModelConfig?.fallbackModelId1,
|
||
currentModelConfig?.fallbackModelId2,
|
||
].filter(Boolean) as string[];
|
||
const allModels = await getEnabledAiModelsWithProvider();
|
||
const sameProviderFallbacks = currentModelConfig
|
||
? allModels
|
||
.filter(m =>
|
||
m.provider.id === currentModelConfig.providerId &&
|
||
m.model.supportsImageGen &&
|
||
m.model.isEnabled &&
|
||
!m.model.isSubModel &&
|
||
m.model.modelId !== actualModelId
|
||
)
|
||
.map(m => m.model.modelId)
|
||
: [];
|
||
const fallbackModelIds = Array.from(new Set([...configuredFallbacks, ...sameProviderFallbacks]));
|
||
console.log("[Chat API] Fallback model IDs:", fallbackModelIds);
|
||
|
||
const geminiStream = createUIMessageStream({
|
||
execute: async ({ writer }) => {
|
||
// 🆕 如果是智能模式且有对话回复，先发送对话消息让用户看到
|
||
// 在消息末尾加上特殊标记，告诉前端"图片正在生成中"
|
||
const conversationMessage = smartAutoFriendlyMessage || null;
|
||
|
||
if (conversationMessage) {
|
||
const chatMessageId = generateUUID();
|
||
console.log("[Chat API] 💬 Gemini: Sending conversational reply first:", smartAutoFriendlyMessage);
|
||
|
||
// 发送带标记的对话消息
|
||
writeAssistantTextChunk(
|
||
writer,
|
||
conversationMessage,
|
||
"gemini:conversation",
|
||
chatMessageId,
|
||
);
|
||
|
||
// 保存对话消息到数据库（不带标记，纯对话）
|
||
await saveMessages({
|
||
messages: [
|
||
{
|
||
id: chatMessageId,
|
||
chatId: id,
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: smartAutoFriendlyMessage }],
|
||
attachments: [],
|
||
createdAt: new Date(),
|
||
videoStorageStatus: null,
|
||
temporaryVideoUrl: null,
|
||
permanentVideoUrl: null,
|
||
archivedAt: null,
|
||
videoMetadata: null,
|
||
},
|
||
],
|
||
});
|
||
}
|
||
|
||
// 🆕 尝试生成图片的函数（支持备用模型切换）
|
||
const tryGenerateImage = async (modelIdToUse: string, isRetry = false): Promise<string> => {
|
||
console.log(`[Chat API] ${isRetry ? "🔄 Retry with fallback" : "🚀 Trying"} model: ${modelIdToUse}`);
|
||
|
||
// 只传递最后一条用户消息，避免上下文历史干扰生图效果
|
||
const lastUserMessage = uiMessagesForModel.filter((m) => m.role === "user").pop();
|
||
let messagesForGemini = lastUserMessage ? [lastUserMessage] : uiMessagesForModel;
|
||
messagesForGemini = applyOptimizedPromptToLatestUserMessage(
|
||
messagesForGemini,
|
||
smartAutoOptimizedPrompt,
|
||
hiddenPromptPrefix,
|
||
);
|
||
messagesForGemini = injectHistoryImagesToLastUserMessage(messagesForGemini, smartAutoHistoryImages);
|
||
console.log("[Chat API] Gemini: Using only last user message (no history context)");
|
||
|
||
// 转换消息：gemini-image 需要 base64 格式，huanke-gemini 需要 URL 格式
|
||
const providerForConvert = (languageModel as any).provider === "huanke-gemini" ? "huanke-gemini" : "gemini";
|
||
const modelMessages = await convertMessagesWithImages(messagesForGemini, providerForConvert);
|
||
console.log("[Chat API] Message conversion provider:", providerForConvert);
|
||
|
||
// 调用 doGenerate 获取图片（如果是备用模型，传入 overrideModelId）
|
||
const result = await (languageModel as any).doGenerate({
|
||
messages: modelMessages,
|
||
imageParams:
|
||
effectiveImageParams || {
|
||
aspectRatio: defaultImageQualityParams.aspectRatio,
|
||
imageSize: defaultImageQualityParams.imageSize,
|
||
},
|
||
overrideModelId: isRetry ? modelIdToUse : undefined, // 🆕 使用备用模型 ID
|
||
});
|
||
|
||
return result.text || "图片生成失败";
|
||
};
|
||
|
||
let stopKeepalive: (() => void) | null = null;
|
||
try {
|
||
let imageMarkdown = ""; // 初始化为空字符串
|
||
let usedModelId = actualModelId;
|
||
|
||
// 🆕 图片生成真正开始时发出标记（用于前端显示占位图）
|
||
writeAssistantTextChunk(
|
||
writer,
|
||
"<!-- IMAGE_GENERATION_START -->",
|
||
"gemini:image-start",
|
||
);
|
||
stopKeepalive = startStreamKeepalive(writer, "gemini-image");
|
||
|
||
try {
|
||
// 首先尝试主模型
|
||
imageMarkdown = await tryGenerateImage(actualModelId, false);
|
||
} catch (primaryError) {
|
||
console.error("[Chat API] Primary model failed:", primaryError);
|
||
|
||
// 🆕 尝试备用模型（使用同一个 provider，但切换 modelId）
|
||
let fallbackSuccess = false;
|
||
for (const fallbackModelId of fallbackModelIds) {
|
||
try {
|
||
console.log(`[Chat API] 🔄 Trying fallback model: ${fallbackModelId}`);
|
||
imageMarkdown = await tryGenerateImage(fallbackModelId, true);
|
||
usedModelId = fallbackModelId;
|
||
fallbackSuccess = true;
|
||
console.log(`[Chat API] ✅ Fallback model ${fallbackModelId} succeeded!`);
|
||
break;
|
||
} catch (fallbackError) {
|
||
console.error(`[Chat API] ❌ Fallback model ${fallbackModelId} failed:`, fallbackError);
|
||
}
|
||
}
|
||
|
||
// 如果所有备用模型都失败，抛出原始错误触发火山引擎故障转移
|
||
if (!fallbackSuccess) {
|
||
throw primaryError;
|
||
}
|
||
}
|
||
|
||
console.log("[Chat API] Gemini generated, length:", imageMarkdown.length);
|
||
console.log("[Chat API] Gemini image markdown:", imageMarkdown.substring(0, 200));
|
||
console.log("[Chat API] Used model ID:", usedModelId);
|
||
|
||
const usedModelDisplayName =
|
||
resolveModelDisplayNameByModelId(allModels, usedModelId) ||
|
||
currentModelConfig?.displayName ||
|
||
actualModelId;
|
||
const imageMarkdownWithModel = prependImageModelBadge(
|
||
imageMarkdown,
|
||
usedModelDisplayName,
|
||
);
|
||
|
||
// 生成一个统一的消息 ID
|
||
const messageId = generateUUID();
|
||
|
||
// 写入消息到流 - 使用正确的 parts 格式
|
||
writeAssistantTextChunk(
|
||
writer,
|
||
imageMarkdownWithModel,
|
||
"gemini:image-result",
|
||
messageId,
|
||
);
|
||
|
||
// 🆕 图片生成结束标记（成功）
|
||
writeAssistantTextChunk(
|
||
writer,
|
||
"<!-- IMAGE_GENERATION_END -->",
|
||
"gemini:image-end-success",
|
||
);
|
||
|
||
// 保存消息到数据库（现在是 S3 URL，可以完整保存）
|
||
try {
|
||
await saveMessages({
|
||
messages: [
|
||
{
|
||
id: messageId,
|
||
chatId: id,
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: imageMarkdownWithModel }],
|
||
attachments: [], // 必须提供 attachments 字段（数据库 notNull）
|
||
createdAt: new Date(),
|
||
videoStorageStatus: null,
|
||
temporaryVideoUrl: null,
|
||
permanentVideoUrl: null,
|
||
archivedAt: null,
|
||
videoMetadata: null,
|
||
},
|
||
],
|
||
});
|
||
console.log("[Chat API] ✅ Gemini message saved successfully!");
|
||
|
||
// 提取用户 prompt（最后一条用户消息的文本）
|
||
const lastUserMessage = uiMessages.filter((m) => m.role === "user").pop();
|
||
const userPrompt = lastUserMessage?.parts?.find((p): p is { type: "text"; text: string } => p.type === "text")?.text || "";
|
||
|
||
// 保存图片记录到 GeneratedImage 表
|
||
await saveGeneratedImages(
|
||
imageMarkdownWithModel,
|
||
session.user.id,
|
||
id,
|
||
messageId,
|
||
userPrompt,
|
||
finalModelId
|
||
);
|
||
} catch (saveError) {
|
||
console.error("[Chat API] ❌ Failed to save Gemini message to database:", saveError);
|
||
console.error("[Chat API] Message content that failed:", imageMarkdown.substring(0, 300));
|
||
}
|
||
} catch (error) {
|
||
console.error("[Chat API] Gemini generation error:", error);
|
||
console.log("[Chat API] Attempting failover to Volcano Vision...");
|
||
|
||
// 故障转移：尝试使用火山引擎模型
|
||
try {
|
||
// 查找火山引擎模型
|
||
const allModels = await getEnabledAiModelsWithProvider();
|
||
console.log("[Chat API] Total enabled models:", allModels.length);
|
||
|
||
const volcanoModel = allModels.find(
|
||
(m: any) => m.provider.type === "volcano-vision" && m.model.isEnabled
|
||
);
|
||
|
||
if (volcanoModel) {
|
||
console.log("[Chat API] Found Volcano Vision model for failover:", volcanoModel.model.id);
|
||
|
||
// 获取火山引擎模型实例
|
||
const volcanoLanguageModel = await getLanguageModelById(volcanoModel.model.id);
|
||
|
||
if (volcanoLanguageModel && typeof (volcanoLanguageModel as any).doGenerate === "function") {
|
||
console.log("[Chat API] Executing Volcano Vision failover...");
|
||
|
||
// 转换消息（火山引擎会自动处理 Base64）
|
||
const volcanoMessages = await convertMessagesWithImages(
|
||
injectHistoryImagesToLastUserMessage(uiMessagesForModel, smartAutoHistoryImages),
|
||
"volcano"
|
||
);
|
||
|
||
// 调用火山引擎生成（故障转移时只生成 1 张，保持与 Gemini 一致）
|
||
const volcanoResult = await (volcanoLanguageModel as any).doGenerate({
|
||
messages: volcanoMessages,
|
||
maxImages: 1, // 故障转移时只生成 1 张
|
||
imageParams:
|
||
effectiveImageParams || {
|
||
aspectRatio: defaultImageQualityParams.aspectRatio,
|
||
imageSize: defaultImageQualityParams.imageSize,
|
||
imageCount: defaultImageQualityParams.imageCount,
|
||
},
|
||
});
|
||
|
||
const imageMarkdown = volcanoResult.text || "图片生成失败";
|
||
const imageMarkdownWithModel = prependImageModelBadge(
|
||
imageMarkdown,
|
||
volcanoModel.model.displayName || volcanoModel.model.modelId,
|
||
);
|
||
console.log("[Chat API] Volcano Vision failover success, length:", imageMarkdown.length);
|
||
|
||
// 生成消息 ID
|
||
const messageId = generateUUID();
|
||
|
||
// 写入消息到流
|
||
writeAssistantTextChunk(
|
||
writer,
|
||
imageMarkdownWithModel,
|
||
"gemini:volcano-failover-result",
|
||
messageId,
|
||
);
|
||
|
||
// 🆕 图片生成结束标记（故障转移成功）
|
||
writeAssistantTextChunk(
|
||
writer,
|
||
"<!-- IMAGE_GENERATION_END -->",
|
||
"gemini:volcano-failover-end",
|
||
);
|
||
|
||
// 保存消息到数据库
|
||
try {
|
||
await saveMessages({
|
||
messages: [
|
||
{
|
||
id: messageId,
|
||
chatId: id,
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: imageMarkdownWithModel }],
|
||
attachments: [],
|
||
createdAt: new Date(),
|
||
videoStorageStatus: null,
|
||
temporaryVideoUrl: null,
|
||
permanentVideoUrl: null,
|
||
archivedAt: null,
|
||
videoMetadata: null,
|
||
},
|
||
],
|
||
});
|
||
console.log("[Chat API] ✅ Volcano Vision failover message saved!");
|
||
|
||
// 提取用户 prompt
|
||
const lastUserMessage = uiMessages.filter((m) => m.role === "user").pop();
|
||
const userPrompt = lastUserMessage?.parts?.find((p): p is { type: "text"; text: string } => p.type === "text")?.text || "";
|
||
|
||
// 保存图片记录到 GeneratedImage 表
|
||
await saveGeneratedImages(
|
||
imageMarkdownWithModel,
|
||
session.user.id,
|
||
id,
|
||
messageId,
|
||
userPrompt,
|
||
volcanoModel.model.id
|
||
);
|
||
} catch (saveError) {
|
||
console.error("[Chat API] Failed to save failover message:", saveError);
|
||
}
|
||
|
||
return; // 成功，直接返回
|
||
}
|
||
}
|
||
|
||
console.log("[Chat API] No Volcano Vision model available for failover");
|
||
} catch (failoverError) {
|
||
console.error("[Chat API] Failover to Volcano Vision also failed:", failoverError);
|
||
}
|
||
|
||
// 如果故障转移也失败，返回错误消息
|
||
const errorMessageId = generateUUID();
|
||
writeAssistantTextChunk(
|
||
writer,
|
||
"图片生成失败，请稍后重试",
|
||
"gemini:error",
|
||
errorMessageId,
|
||
);
|
||
|
||
// 🆕 图片生成结束标记（失败）
|
||
writeAssistantTextChunk(
|
||
writer,
|
||
"<!-- IMAGE_GENERATION_END -->",
|
||
"gemini:image-end-failed",
|
||
);
|
||
|
||
// 尝试保存错误消息
|
||
try {
|
||
await saveMessages({
|
||
messages: [
|
||
{
|
||
id: errorMessageId,
|
||
chatId: id,
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: "图片生成失败，请稍后重试" }],
|
||
attachments: [],
|
||
createdAt: new Date(),
|
||
videoStorageStatus: null,
|
||
temporaryVideoUrl: null,
|
||
permanentVideoUrl: null,
|
||
archivedAt: null,
|
||
videoMetadata: null,
|
||
},
|
||
],
|
||
});
|
||
} catch (saveError) {
|
||
console.error("[Chat API] Failed to save error message:", saveError);
|
||
}
|
||
} finally {
|
||
stopKeepalive?.();
|
||
}
|
||
},
|
||
});
|
||
|
||
return createSseResponse(geminiStream);
|
||
}
|
||
// ========== Gemini 处理结束 ==========
|
||
|
||
// ========== 阿里云千问图片生成独立处理 ==========
|
||
const isAlibabaLikeImageProvider = ["alibaba-image", "ruxa-image", "fal-image"].includes((languageModel as any).provider);
|
||
if (isAlibabaLikeImageProvider && typeof (languageModel as any).doGenerate === "function") {
|
||
console.log("[Chat API] Using image generation provider:", (languageModel as any).provider);
|
||
console.log("[Chat API] Image params:", effectiveImageParams);
|
||
|
||
// 🆕 获取模型显示名称，用于错误提示
|
||
const alibabaModelConfig = await getAiModelById({ id: finalModelId });
|
||
const modelDisplayName = alibabaModelConfig?.displayName || actualModelId;
|
||
const configuredFallbacks = [
|
||
alibabaModelConfig?.fallbackModelId1,
|
||
alibabaModelConfig?.fallbackModelId2,
|
||
].filter(Boolean) as string[];
|
||
const allModels = await getEnabledAiModelsWithProvider();
|
||
const sameProviderFallbacks = alibabaModelConfig
|
||
? allModels
|
||
.filter(m =>
|
||
m.provider.id === alibabaModelConfig.providerId &&
|
||
m.model.supportsImageGen &&
|
||
m.model.isEnabled &&
|
||
!m.model.isSubModel &&
|
||
m.model.modelId !== actualModelId
|
||
)
|
||
.map(m => m.model.modelId)
|
||
: [];
|
||
const fallbackModelIds = Array.from(new Set([...configuredFallbacks, ...sameProviderFallbacks]));
|
||
console.log("[Chat API] Alibaba fallback model IDs:", fallbackModelIds);
|
||
|
||
const alibabaStream = createUIMessageStream({
|
||
execute: async ({ writer }) => {
|
||
let stopKeepalive: (() => void) | null = null;
|
||
try {
|
||
// 🆕 如果是智能模式且有对话回复，先发送对话消息让用户看到
|
||
const conversationMessage = smartAutoFriendlyMessage || null;
|
||
|
||
if (conversationMessage) {
|
||
const chatMessageId = generateUUID();
|
||
console.log("[Chat API] 💬 Sending conversational reply first:", smartAutoFriendlyMessage);
|
||
|
||
// 发送带标记的对话消息
|
||
writeAssistantTextChunk(
|
||
writer,
|
||
conversationMessage,
|
||
"alibaba:conversation",
|
||
chatMessageId,
|
||
);
|
||
|
||
// 保存对话消息到数据库（不带标记，纯对话）
|
||
await saveMessages({
|
||
messages: [
|
||
{
|
||
id: chatMessageId,
|
||
chatId: id,
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: smartAutoFriendlyMessage }],
|
||
attachments: [],
|
||
createdAt: new Date(),
|
||
videoStorageStatus: null,
|
||
temporaryVideoUrl: null,
|
||
permanentVideoUrl: null,
|
||
archivedAt: null,
|
||
videoMetadata: null,
|
||
},
|
||
],
|
||
});
|
||
}
|
||
|
||
// 只传递最后一条用户消息
|
||
const lastUserMessage = uiMessagesForModel.filter((m) => m.role === "user").pop();
|
||
let messagesForAlibaba = lastUserMessage ? [lastUserMessage] : uiMessagesForModel;
|
||
console.log("[Chat API] Alibaba: Using only last user message");
|
||
|
||
// 🆕 如果有智能模式优化后的提示词，替换消息中的文本（仅用于API调用）
|
||
if (smartAutoOptimizedPrompt) {
|
||
console.log("[Chat API] Using optimized prompt for API call");
|
||
messagesForAlibaba = applyOptimizedPromptToLatestUserMessage(
|
||
messagesForAlibaba,
|
||
smartAutoOptimizedPrompt,
|
||
hiddenPromptPrefix,
|
||
);
|
||
}
|
||
|
||
messagesForAlibaba = injectHistoryImagesToLastUserMessage(messagesForAlibaba, smartAutoHistoryImages);
|
||
|
||
const tryGenerateImage = async (modelIdToUse: string, isRetry: boolean) => {
|
||
// 转换消息（图片编辑模型统一使用 URL 输入）
|
||
const providerForConvert =
|
||
(languageModel as any).provider === "ruxa-image"
|
||
? "ruxa-image"
|
||
: (languageModel as any).provider === "fal-image"
|
||
? "fal-image"
|
||
: "alibaba-image";
|
||
const modelMessages = await convertMessagesWithImages(messagesForAlibaba, providerForConvert);
|
||
|
||
// 调用 doGenerate 获取图片，传递模型显示名称
|
||
const result = await (languageModel as any).doGenerate({
|
||
messages: modelMessages,
|
||
imageParams:
|
||
effectiveImageParams || {
|
||
aspectRatio: defaultImageQualityParams.aspectRatio,
|
||
imageSize: defaultImageQualityParams.imageSize,
|
||
imageCount: defaultImageQualityParams.imageCount,
|
||
},
|
||
modelDisplayName, // 🆕 传递模型显示名称
|
||
overrideModelId: isRetry ? modelIdToUse : undefined,
|
||
});
|
||
|
||
if (result?.finishReason === "error") {
|
||
throw new Error(result.text || "图片生成失败");
|
||
}
|
||
|
||
return result.text || "图片生成失败";
|
||
};
|
||
|
||
let imageMarkdown = "";
|
||
let usedModelId = actualModelId;
|
||
|
||
// 🆕 图片生成真正开始时发出标记（用于前端显示占位图）
|
||
writeAssistantTextChunk(
|
||
writer,
|
||
"<!-- IMAGE_GENERATION_START -->",
|
||
"alibaba:image-start",
|
||
);
|
||
stopKeepalive = startStreamKeepalive(writer, "alibaba-image");
|
||
|
||
try {
|
||
imageMarkdown = await tryGenerateImage(actualModelId, false);
|
||
} catch (primaryError) {
|
||
console.error("[Chat API] Alibaba primary model failed:", primaryError);
|
||
|
||
let fallbackSuccess = false;
|
||
for (const fallbackModelId of fallbackModelIds) {
|
||
try {
|
||
console.log(`[Chat API] 🔄 Alibaba fallback model: ${fallbackModelId}`);
|
||
imageMarkdown = await tryGenerateImage(fallbackModelId, true);
|
||
usedModelId = fallbackModelId;
|
||
fallbackSuccess = true;
|
||
console.log(`[Chat API] ✅ Alibaba fallback model ${fallbackModelId} succeeded!`);
|
||
break;
|
||
} catch (fallbackError) {
|
||
console.error(`[Chat API] ❌ Alibaba fallback model ${fallbackModelId} failed:`, fallbackError);
|
||
}
|
||
}
|
||
|
||
if (!fallbackSuccess) {
|
||
const providerPriority: Record<string, number> = {
|
||
"volcano-vision": 0,
|
||
"alibaba-image": 1,
|
||
"ruxa-image": 2,
|
||
"fal-image": 3,
|
||
gemini: 4,
|
||
"huanke-gemini": 5,
|
||
};
|
||
const crossProviderCandidates = getImageCrossProviderCandidates(allModels, {
|
||
currentModelId: finalModelId,
|
||
currentProviderId: alibabaModelConfig?.providerId,
|
||
providerPriority,
|
||
});
|
||
|
||
for (const candidate of crossProviderCandidates) {
|
||
try {
|
||
console.log(
|
||
`[Chat API] 🔁 Cross-provider failover attempt: ${candidate.model.displayName} (${candidate.provider.type})`
|
||
);
|
||
|
||
const candidateLanguageModel = await getLanguageModelById(candidate.model.id);
|
||
if (!candidateLanguageModel || typeof (candidateLanguageModel as any).doGenerate !== "function") {
|
||
console.log(`[Chat API] Skip candidate without doGenerate: ${candidate.model.id}`);
|
||
continue;
|
||
}
|
||
|
||
const convertProvider =
|
||
candidate.provider.type === "ruxa-image"
|
||
? "ruxa-image"
|
||
: candidate.provider.type === "fal-image"
|
||
? "fal-image"
|
||
: candidate.provider.type === "gemini"
|
||
? "gemini"
|
||
: candidate.provider.type === "huanke-gemini"
|
||
? "huanke-gemini"
|
||
: candidate.provider.type === "volcano-vision"
|
||
? "volcano"
|
||
: "alibaba-image";
|
||
|
||
const candidateMessages = await convertMessagesWithImages(
|
||
messagesForAlibaba,
|
||
convertProvider
|
||
);
|
||
|
||
const candidateResult = await (candidateLanguageModel as any).doGenerate({
|
||
messages: candidateMessages,
|
||
imageParams:
|
||
effectiveImageParams || {
|
||
aspectRatio: defaultImageQualityParams.aspectRatio,
|
||
imageSize: defaultImageQualityParams.imageSize,
|
||
imageCount: defaultImageQualityParams.imageCount,
|
||
},
|
||
modelDisplayName: candidate.model.displayName,
|
||
overrideModelId: candidate.model.modelId,
|
||
});
|
||
|
||
if (candidateResult?.finishReason === "error") {
|
||
throw new Error(candidateResult.text || "图片生成失败");
|
||
}
|
||
|
||
imageMarkdown = candidateResult.text || "图片生成失败";
|
||
usedModelId = candidate.model.modelId;
|
||
fallbackSuccess = true;
|
||
console.log(
|
||
`[Chat API] ✅ Cross-provider failover succeeded: ${candidate.model.displayName} (${candidate.provider.type})`
|
||
);
|
||
break;
|
||
} catch (crossError) {
|
||
const crossErrorMessage = normalizeFailoverErrorMessage(crossError);
|
||
if (shouldCooldownImageFallbackModel(crossErrorMessage)) {
|
||
setImageFallbackModelCooldown(candidate.model.id, crossErrorMessage);
|
||
}
|
||
console.error(
|
||
`[Chat API] ❌ Cross-provider failover failed: ${candidate.model.displayName} (${candidate.provider.type})`,
|
||
crossError
|
||
);
|
||
}
|
||
}
|
||
}
|
||
|
||
if (!fallbackSuccess) {
|
||
throw primaryError;
|
||
}
|
||
}
|
||
|
||
console.log("[Chat API] Alibaba generated, length:", imageMarkdown.length);
|
||
console.log("[Chat API] Alibaba used model:", usedModelId);
|
||
|
||
const usedModelDisplayName =
|
||
resolveModelDisplayNameByModelId(allModels, usedModelId) ||
|
||
alibabaModelConfig?.displayName ||
|
||
actualModelId;
|
||
const imageMarkdownWithModel = prependImageModelBadge(
|
||
imageMarkdown,
|
||
usedModelDisplayName,
|
||
);
|
||
|
||
// 生成消息 ID（图片消息）
|
||
const messageId = generateUUID();
|
||
|
||
// 写入图片消息到流
|
||
writeAssistantTextChunk(
|
||
writer,
|
||
imageMarkdownWithModel,
|
||
"alibaba:image-result",
|
||
messageId,
|
||
);
|
||
|
||
// 🆕 图片生成结束标记（成功）
|
||
writeAssistantTextChunk(
|
||
writer,
|
||
"<!-- IMAGE_GENERATION_END -->",
|
||
"alibaba:image-end-success",
|
||
);
|
||
|
||
// 保存图片消息到数据库
|
||
try {
|
||
await saveMessages({
|
||
messages: [
|
||
{
|
||
id: messageId,
|
||
chatId: id,
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: imageMarkdownWithModel }],
|
||
attachments: [],
|
||
createdAt: new Date(),
|
||
videoStorageStatus: null,
|
||
temporaryVideoUrl: null,
|
||
permanentVideoUrl: null,
|
||
archivedAt: null,
|
||
videoMetadata: null,
|
||
},
|
||
],
|
||
});
|
||
console.log("[Chat API] ✅ Alibaba message saved successfully!");
|
||
|
||
// 提取用户 prompt
|
||
const userPrompt = lastUserMessage?.parts?.find((p): p is { type: "text"; text: string } => p.type === "text")?.text || "";
|
||
|
||
// 保存图片记录到 GeneratedImage 表
|
||
await saveGeneratedImages(
|
||
imageMarkdownWithModel,
|
||
session.user.id,
|
||
id,
|
||
messageId,
|
||
userPrompt,
|
||
finalModelId
|
||
);
|
||
} catch (saveError) {
|
||
console.error("[Chat API] ❌ Failed to save Alibaba message:", saveError);
|
||
}
|
||
} catch (error) {
|
||
console.error("[Chat API] Alibaba generation error:", error);
|
||
|
||
// 返回错误消息
|
||
const errorMessageId = generateUUID();
|
||
const errorText = error instanceof Error ? error.message : "图片生成失败，请稍后重试";
|
||
writeAssistantTextChunk(
|
||
writer,
|
||
errorText,
|
||
"alibaba:error",
|
||
errorMessageId,
|
||
);
|
||
|
||
// 🆕 图片生成结束标记（失败）
|
||
writeAssistantTextChunk(
|
||
writer,
|
||
"<!-- IMAGE_GENERATION_END -->",
|
||
"alibaba:image-end-failed",
|
||
);
|
||
|
||
try {
|
||
await saveMessages({
|
||
messages: [
|
||
{
|
||
id: errorMessageId,
|
||
chatId: id,
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: errorText }],
|
||
attachments: [],
|
||
createdAt: new Date(),
|
||
videoStorageStatus: null,
|
||
temporaryVideoUrl: null,
|
||
permanentVideoUrl: null,
|
||
archivedAt: null,
|
||
videoMetadata: null,
|
||
},
|
||
],
|
||
});
|
||
} catch (saveError) {
|
||
console.error("[Chat API] Failed to save error message:", saveError);
|
||
}
|
||
} finally {
|
||
stopKeepalive?.();
|
||
}
|
||
},
|
||
});
|
||
|
||
return createSseResponse(alibabaStream);
|
||
}
|
||
// ========== 阿里云图片生成处理结束 ==========
|
||
|
||
// ========== 火山引擎视觉图片生成独立处理 ==========
|
||
if ((languageModel as any).provider === "volcano-vision" && typeof (languageModel as any).doGenerate === "function") {
|
||
console.log("[Chat API] Using Volcano Vision image generation");
|
||
|
||
// 🆕 检测是否有标注图片，如果有则在消息中注入系统提示
|
||
const hasAnnotation = hasAnnotatedImage(uiMessages);
|
||
let messagesForVolcano = applyOptimizedPromptToLatestUserMessage(
|
||
uiMessagesForModel,
|
||
smartAutoOptimizedPrompt,
|
||
hiddenPromptPrefix,
|
||
);
|
||
const volcanoModelConfig = await getAiModelById({ id: finalModelId });
|
||
const configuredFallbacks = [
|
||
volcanoModelConfig?.fallbackModelId1,
|
||
volcanoModelConfig?.fallbackModelId2,
|
||
].filter(Boolean) as string[];
|
||
const allModels = await getEnabledAiModelsWithProvider();
|
||
const sameProviderFallbacks = volcanoModelConfig
|
||
? allModels
|
||
.filter(m =>
|
||
m.provider.id === volcanoModelConfig.providerId &&
|
||
m.model.supportsImageGen &&
|
||
m.model.isEnabled &&
|
||
!m.model.isSubModel &&
|
||
m.model.modelId !== actualModelId
|
||
)
|
||
.map(m => m.model.modelId)
|
||
: [];
|
||
const fallbackModelIds = Array.from(new Set([...configuredFallbacks, ...sameProviderFallbacks]));
|
||
console.log("[Chat API] Volcano fallback model IDs:", fallbackModelIds);
|
||
|
||
if (hasAnnotation) {
|
||
console.log("[Chat API] 🎨 Volcano Vision: 检测到标注图片，将注入系统提示");
|
||
// 创建消息副本，修改最后一条用户消息的文本
|
||
messagesForVolcano = messagesForVolcano.map((msg, idx) => {
|
||
if (msg.role === "user" && idx === messagesForVolcano.length - 1) {
|
||
// 修改最后一条用户消息
|
||
const modifiedParts = msg.parts.map((part) => {
|
||
if (part.type === "text" && part.text) {
|
||
return {
|
||
...part,
|
||
text: ANNOTATION_SYSTEM_PROMPT + "\n\n用户请求：" + part.text,
|
||
};
|
||
}
|
||
return part;
|
||
});
|
||
return { ...msg, parts: modifiedParts };
|
||
}
|
||
return msg;
|
||
});
|
||
}
|
||
|
||
messagesForVolcano = injectHistoryImagesToLastUserMessage(messagesForVolcano, smartAutoHistoryImages);
|
||
|
||
const volcanoStream = createUIMessageStream({
|
||
execute: async ({ writer }) => {
|
||
let stopKeepalive: (() => void) | null = null;
|
||
try {
|
||
// 🆕 如果是智能模式且有对话回复，先发送对话消息让用户看到
|
||
const conversationMessage = smartAutoFriendlyMessage || null;
|
||
|
||
if (conversationMessage) {
|
||
const chatMessageId = generateUUID();
|
||
console.log("[Chat API] 💬 Volcano: Sending conversational reply first:", smartAutoFriendlyMessage);
|
||
|
||
// 发送带标记的对话消息
|
||
writeAssistantTextChunk(
|
||
writer,
|
||
conversationMessage,
|
||
"volcano:conversation",
|
||
chatMessageId,
|
||
);
|
||
|
||
// 保存对话消息到数据库（不带标记，纯对话）
|
||
await saveMessages({
|
||
messages: [
|
||
{
|
||
id: chatMessageId,
|
||
chatId: id,
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: smartAutoFriendlyMessage }],
|
||
attachments: [],
|
||
createdAt: new Date(),
|
||
videoStorageStatus: null,
|
||
temporaryVideoUrl: null,
|
||
permanentVideoUrl: null,
|
||
archivedAt: null,
|
||
videoMetadata: null,
|
||
},
|
||
],
|
||
});
|
||
}
|
||
|
||
// 转换消息，火山引擎支持 URL 和 base64
|
||
const modelMessages = await convertMessagesWithImages(messagesForVolcano, "volcano");
|
||
|
||
const tryGenerateImage = async (modelIdToUse: string, isRetry: boolean) => {
|
||
const result = await (languageModel as any).doGenerate({
|
||
messages: modelMessages,
|
||
imageParams:
|
||
effectiveImageParams || {
|
||
aspectRatio: defaultImageQualityParams.aspectRatio,
|
||
imageSize: defaultImageQualityParams.imageSize,
|
||
imageCount: defaultImageQualityParams.imageCount,
|
||
},
|
||
overrideModelId: isRetry ? modelIdToUse : undefined,
|
||
});
|
||
if (result?.finishReason === "error") {
|
||
throw new Error(result.text || "图片生成失败");
|
||
}
|
||
return result.text || "图片生成失败";
|
||
};
|
||
|
||
let imageMarkdown = "";
|
||
let usedModelId = actualModelId;
|
||
|
||
// 🆕 图片生成真正开始时发出标记（用于前端显示占位图）
|
||
writeAssistantTextChunk(
|
||
writer,
|
||
"<!-- IMAGE_GENERATION_START -->",
|
||
"volcano:image-start",
|
||
);
|
||
stopKeepalive = startStreamKeepalive(writer, "volcano-image");
|
||
try {
|
||
imageMarkdown = await tryGenerateImage(actualModelId, false);
|
||
} catch (primaryError) {
|
||
console.error("[Chat API] Volcano primary model failed:", primaryError);
|
||
let fallbackSuccess = false;
|
||
for (const fallbackModelId of fallbackModelIds) {
|
||
try {
|
||
console.log(`[Chat API] 🔄 Volcano fallback model: ${fallbackModelId}`);
|
||
imageMarkdown = await tryGenerateImage(fallbackModelId, true);
|
||
usedModelId = fallbackModelId;
|
||
fallbackSuccess = true;
|
||
console.log(`[Chat API] ✅ Volcano fallback model ${fallbackModelId} succeeded!`);
|
||
break;
|
||
} catch (fallbackError) {
|
||
console.error(`[Chat API] ❌ Volcano fallback model ${fallbackModelId} failed:`, fallbackError);
|
||
}
|
||
}
|
||
if (!fallbackSuccess) {
|
||
const providerPriority: Record<string, number> = {
|
||
"alibaba-image": 0,
|
||
"ruxa-image": 1,
|
||
"fal-image": 2,
|
||
gemini: 3,
|
||
"huanke-gemini": 4,
|
||
"volcano-vision": 5,
|
||
};
|
||
const crossProviderCandidates = getImageCrossProviderCandidates(allModels, {
|
||
currentModelId: finalModelId,
|
||
currentProviderId: volcanoModelConfig?.providerId,
|
||
providerPriority,
|
||
});
|
||
|
||
for (const candidate of crossProviderCandidates) {
|
||
try {
|
||
console.log(
|
||
`[Chat API] 🔁 Volcano cross-provider failover attempt: ${candidate.model.displayName} (${candidate.provider.type})`
|
||
);
|
||
|
||
const candidateLanguageModel = await getLanguageModelById(candidate.model.id);
|
||
if (!candidateLanguageModel || typeof (candidateLanguageModel as any).doGenerate !== "function") {
|
||
console.log(`[Chat API] Skip candidate without doGenerate: ${candidate.model.id}`);
|
||
continue;
|
||
}
|
||
|
||
const convertProvider =
|
||
candidate.provider.type === "ruxa-image"
|
||
? "ruxa-image"
|
||
: candidate.provider.type === "fal-image"
|
||
? "fal-image"
|
||
: candidate.provider.type === "gemini"
|
||
? "gemini"
|
||
: candidate.provider.type === "huanke-gemini"
|
||
? "huanke-gemini"
|
||
: candidate.provider.type === "volcano-vision"
|
||
? "volcano"
|
||
: "alibaba-image";
|
||
|
||
const candidateMessages = await convertMessagesWithImages(
|
||
messagesForVolcano,
|
||
convertProvider
|
||
);
|
||
|
||
const candidateResult = await (candidateLanguageModel as any).doGenerate({
|
||
messages: candidateMessages,
|
||
imageParams:
|
||
effectiveImageParams || {
|
||
aspectRatio: defaultImageQualityParams.aspectRatio,
|
||
imageSize: defaultImageQualityParams.imageSize,
|
||
imageCount: defaultImageQualityParams.imageCount,
|
||
},
|
||
modelDisplayName: candidate.model.displayName,
|
||
overrideModelId: candidate.model.modelId,
|
||
});
|
||
|
||
if (candidateResult?.finishReason === "error") {
|
||
throw new Error(candidateResult.text || "图片生成失败");
|
||
}
|
||
|
||
imageMarkdown = candidateResult.text || "图片生成失败";
|
||
usedModelId = candidate.model.modelId;
|
||
fallbackSuccess = true;
|
||
console.log(
|
||
`[Chat API] ✅ Volcano cross-provider failover succeeded: ${candidate.model.displayName} (${candidate.provider.type})`
|
||
);
|
||
break;
|
||
} catch (crossError) {
|
||
const crossErrorMessage = normalizeFailoverErrorMessage(crossError);
|
||
if (shouldCooldownImageFallbackModel(crossErrorMessage)) {
|
||
setImageFallbackModelCooldown(candidate.model.id, crossErrorMessage);
|
||
}
|
||
console.error(
|
||
`[Chat API] ❌ Volcano cross-provider failover failed: ${candidate.model.displayName} (${candidate.provider.type})`,
|
||
crossError
|
||
);
|
||
}
|
||
}
|
||
}
|
||
if (!fallbackSuccess) {
|
||
throw primaryError;
|
||
}
|
||
}
|
||
console.log("[Chat API] Volcano Vision generated, length:", imageMarkdown.length);
|
||
console.log("[Chat API] Volcano Vision image markdown:", imageMarkdown.substring(0, 200));
|
||
|
||
const usedModelDisplayName =
|
||
resolveModelDisplayNameByModelId(allModels, usedModelId) ||
|
||
volcanoModelConfig?.displayName ||
|
||
actualModelId;
|
||
const imageMarkdownWithModel = prependImageModelBadge(
|
||
imageMarkdown,
|
||
usedModelDisplayName,
|
||
);
|
||
|
||
// 生成消息 ID（图片消息）
|
||
const messageId = generateUUID();
|
||
|
||
// 写入图片消息到流
|
||
writeAssistantTextChunk(
|
||
writer,
|
||
imageMarkdownWithModel,
|
||
"volcano:image-result",
|
||
messageId,
|
||
);
|
||
|
||
// 🆕 图片生成结束标记（成功）
|
||
writeAssistantTextChunk(
|
||
writer,
|
||
"<!-- IMAGE_GENERATION_END -->",
|
||
"volcano:image-end-success",
|
||
);
|
||
|
||
// 保存图片消息到数据库
|
||
try {
|
||
await saveMessages({
|
||
messages: [
|
||
{
|
||
id: messageId,
|
||
chatId: id,
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: imageMarkdownWithModel }],
|
||
attachments: [], // 必须提供 attachments 字段（数据库 notNull）
|
||
createdAt: new Date(),
|
||
videoStorageStatus: null,
|
||
temporaryVideoUrl: null,
|
||
permanentVideoUrl: null,
|
||
archivedAt: null,
|
||
videoMetadata: null,
|
||
},
|
||
],
|
||
});
|
||
console.log("[Chat API] ✅ Volcano Vision message saved successfully!");
|
||
|
||
// 提取用户 prompt
|
||
const lastUserMessage = uiMessages.filter((m) => m.role === "user").pop();
|
||
const userPrompt = lastUserMessage?.parts?.find((p): p is { type: "text"; text: string } => p.type === "text")?.text || "";
|
||
|
||
// 保存图片记录到 GeneratedImage 表
|
||
await saveGeneratedImages(
|
||
imageMarkdownWithModel,
|
||
session.user.id,
|
||
id,
|
||
messageId,
|
||
userPrompt,
|
||
finalModelId
|
||
);
|
||
} catch (saveError) {
|
||
console.error("[Chat API] ❌ Failed to save Volcano Vision message to database:", saveError);
|
||
console.error("[Chat API] Message content that failed:", imageMarkdown.substring(0, 300));
|
||
}
|
||
} catch (error) {
|
||
console.error("[Chat API] Volcano Vision generation error:", error);
|
||
const errorMessageId = generateUUID();
|
||
writeAssistantTextChunk(
|
||
writer,
|
||
"图片生成失败，请稍后重试",
|
||
"volcano:error",
|
||
errorMessageId,
|
||
);
|
||
|
||
// 🆕 图片生成结束标记（失败）
|
||
writeAssistantTextChunk(
|
||
writer,
|
||
"<!-- IMAGE_GENERATION_END -->",
|
||
"volcano:image-end-failed",
|
||
);
|
||
|
||
// 尝试保存错误消息
|
||
try {
|
||
await saveMessages({
|
||
messages: [
|
||
{
|
||
id: errorMessageId,
|
||
chatId: id,
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: "图片生成失败，请稍后重试" }],
|
||
attachments: [], // 必须提供 attachments 字段
|
||
createdAt: new Date(),
|
||
videoStorageStatus: null,
|
||
temporaryVideoUrl: null,
|
||
permanentVideoUrl: null,
|
||
archivedAt: null,
|
||
videoMetadata: null,
|
||
},
|
||
],
|
||
});
|
||
} catch (saveError) {
|
||
console.error("[Chat API] Failed to save error message:", saveError);
|
||
}
|
||
} finally {
|
||
stopKeepalive?.();
|
||
}
|
||
},
|
||
});
|
||
|
||
return createSseResponse(volcanoStream);
|
||
}
|
||
// ========== 火山引擎视觉处理结束 ==========
|
||
|
||
// ========== 千问/FAL 视频生成独立处理 ==========
|
||
if (
|
||
["qianwen-video", "fal-video"].includes((languageModel as any).provider) &&
|
||
typeof (languageModel as any).doGenerate === "function"
|
||
) {
|
||
const currentVideoProvider = (languageModel as any).provider as "qianwen-video" | "fal-video";
|
||
console.log(`[Chat API] Using ${currentVideoProvider} video generation`);
|
||
|
||
const qianwenVideoStream = createUIMessageStream({
|
||
execute: async ({ writer }) => {
|
||
try {
|
||
const conversationMessage = smartAutoFriendlyMessage || null;
|
||
if (conversationMessage) {
|
||
const chatMessageId = generateUUID();
|
||
writer.write({
|
||
type: "data-appendMessage",
|
||
data: JSON.stringify({
|
||
id: chatMessageId,
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: conversationMessage }],
|
||
}),
|
||
});
|
||
|
||
await saveMessages({
|
||
messages: [
|
||
{
|
||
id: chatMessageId,
|
||
chatId: id,
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: conversationMessage }],
|
||
attachments: [],
|
||
createdAt: new Date(),
|
||
videoStorageStatus: null,
|
||
temporaryVideoUrl: null,
|
||
permanentVideoUrl: null,
|
||
archivedAt: null,
|
||
videoMetadata: null,
|
||
},
|
||
],
|
||
});
|
||
}
|
||
|
||
const messagesForVideo = injectHistoryImagesToLastUserMessage(
|
||
applyOptimizedPromptToLatestUserMessage(
|
||
uiMessagesForModel,
|
||
smartAutoOptimizedPrompt,
|
||
hiddenPromptPrefix,
|
||
),
|
||
smartAutoHistoryImages,
|
||
);
|
||
|
||
// 千问/FAL 视频直接使用原始 URL，不转换为 base64
|
||
const modelMessages = convertToModelMessages(messagesForVideo);
|
||
console.log(`[Chat API] ${currentVideoProvider} messages prepared, count:`, modelMessages.length);
|
||
|
||
// 调用 doGenerate 获取视频
|
||
const result = await (languageModel as any).doGenerate({
|
||
messages: modelMessages,
|
||
videoParams: effectiveVideoParams,
|
||
});
|
||
|
||
const videoMarkdown = result.text || "视频生成失败";
|
||
console.log(`[Chat API] ${currentVideoProvider} video generated, length:`, videoMarkdown.length);
|
||
|
||
// 提取视频 URL（后续会自动转存到永久存储）
|
||
const tempVideoUrl = extractVideoUrlFromText(videoMarkdown) || "";
|
||
|
||
// 生成一个统一的消息 ID
|
||
const messageId = generateUUID();
|
||
|
||
const videoMetadata = {
|
||
provider: currentVideoProvider,
|
||
model: finalModelId,
|
||
mode: effectiveVideoParams?.mode,
|
||
resolution: effectiveVideoParams?.resolution,
|
||
ratio: effectiveVideoParams?.ratio,
|
||
duration: effectiveVideoParams?.duration,
|
||
status: "succeeded",
|
||
updatedAt: Date.now(),
|
||
};
|
||
|
||
// 写入消息到流
|
||
writer.write({
|
||
type: "data-appendMessage",
|
||
data: JSON.stringify({
|
||
id: messageId,
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: videoMarkdown }],
|
||
attachments: [],
|
||
}),
|
||
});
|
||
|
||
// 保存消息到数据库
|
||
try {
|
||
await saveMessages({
|
||
messages: [
|
||
{
|
||
id: messageId,
|
||
chatId: id,
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: videoMarkdown }],
|
||
attachments: [],
|
||
createdAt: new Date(),
|
||
videoStorageStatus: tempVideoUrl ? "temporary" : null,
|
||
temporaryVideoUrl: tempVideoUrl || null,
|
||
permanentVideoUrl: null,
|
||
archivedAt: null,
|
||
videoMetadata: videoMetadata as any,
|
||
},
|
||
],
|
||
});
|
||
console.log(`[Chat API] ✅ ${currentVideoProvider} video message saved successfully!`);
|
||
await consumeGeneratedVideoPoints({
|
||
userId: session.user.id,
|
||
chatId: id,
|
||
messageId,
|
||
modelRefs: [finalModelId],
|
||
});
|
||
|
||
if (tempVideoUrl) {
|
||
scheduleVideoArchive({
|
||
chatId: id,
|
||
messageId,
|
||
text: videoMarkdown,
|
||
temporaryVideoUrl: tempVideoUrl,
|
||
taskId: messageId,
|
||
metadata: videoMetadata,
|
||
});
|
||
}
|
||
} catch (saveError) {
|
||
console.error(`[Chat API] ❌ Failed to save ${currentVideoProvider} video message:`, saveError);
|
||
}
|
||
} catch (error) {
|
||
console.error(`[Chat API] ${currentVideoProvider} video generation error:`, error);
|
||
const errorMessageId = generateUUID();
|
||
writer.write({
|
||
type: "data-appendMessage",
|
||
data: JSON.stringify({
|
||
id: errorMessageId,
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: "视频生成失败，请稍后重试" }],
|
||
attachments: [],
|
||
}),
|
||
});
|
||
}
|
||
},
|
||
});
|
||
|
||
return createSseResponse(qianwenVideoStream);
|
||
}
|
||
// ========== 千问/FAL 视频处理结束 ==========
|
||
|
||
const stream = createUIMessageStream({
|
||
execute: async ({ writer: dataStream }) => {
|
||
// 构建工具对象，只包含启用的工具
|
||
const tools: any = {};
|
||
const activeTools: string[] = [];
|
||
|
||
if (enabledTools.includes("getWeather")) {
|
||
tools.getWeather = getWeather;
|
||
activeTools.push("getWeather");
|
||
}
|
||
if (enabledTools.includes("createDocument")) {
|
||
tools.createDocument = createDocument({ session, dataStream });
|
||
activeTools.push("createDocument");
|
||
}
|
||
if (enabledTools.includes("updateDocument")) {
|
||
tools.updateDocument = updateDocument({ session, dataStream });
|
||
activeTools.push("updateDocument");
|
||
}
|
||
if (enabledTools.includes("requestSuggestions")) {
|
||
tools.requestSuggestions = requestSuggestions({
|
||
session,
|
||
dataStream,
|
||
});
|
||
activeTools.push("requestSuggestions");
|
||
}
|
||
const shouldEnableGenerateVideoTool =
|
||
enabledTools.includes("generateVideo") &&
|
||
(!shouldRunAgentConversation || allowAgentVideoGeneration);
|
||
if (shouldEnableGenerateVideoTool) {
|
||
tools.generateVideo = generateVideo({ dataStream });
|
||
activeTools.push("generateVideo");
|
||
} else if (enabledTools.includes("generateVideo") && shouldRunAgentConversation) {
|
||
console.log("[Smart Auto Mode] 🚫 generateVideo tool disabled (manual video mode not selected)");
|
||
}
|
||
|
||
console.log("[Chat API] Calling streamText with model:", (languageModel as any).modelId, (languageModel as any).provider); // 调试日志
|
||
console.log("[Chat API] Provider type:", (languageModel as any).provider); // 调试日志
|
||
|
||
// 转换消息，对于 Gemini 和千问视频将图片 URL 转换为 base64
|
||
const convertedMessages = await convertMessagesWithImages(
|
||
injectHistoryImagesToLastUserMessage(uiMessagesForModel, smartAutoHistoryImages),
|
||
(languageModel as any).provider || ""
|
||
);
|
||
console.log("[Chat API] Messages converted, count:", convertedMessages.length); // 调试日志
|
||
|
||
// 🆕 检测是否是视频模型
|
||
const providerType = (languageModel as any).provider;
|
||
const effectiveProviderType = resolvedProviderType || providerType;
|
||
console.log("[Chat API] Provider type for tool detection:", {
|
||
runtime: providerType,
|
||
resolved: resolvedProviderType,
|
||
effective: effectiveProviderType,
|
||
});
|
||
const isVolcanoVideoModel =
|
||
effectiveProviderType === "volcano-video" ||
|
||
effectiveProviderType === "doubao-video" ||
|
||
actualModelId.includes("seedance");
|
||
const isOpenAICompatImageModel =
|
||
resolvedModelSupportsImageGen &&
|
||
providerType === "openai.chat" &&
|
||
!isVolcanoVideoModel;
|
||
console.log("[Chat API] Is volcano video model:", isVolcanoVideoModel);
|
||
|
||
// 🆕 处理 OpenAI 兼容的生图模型输出（防止返回 data:image;base64 原文）
|
||
if (isOpenAICompatImageModel) {
|
||
console.log("[Chat API] Using generateText for OpenAI-compatible image model");
|
||
|
||
// 生成开始标记
|
||
dataStream.write({
|
||
type: "data-appendMessage",
|
||
data: JSON.stringify({
|
||
id: generateUUID(),
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: "<!-- IMAGE_GENERATION_START -->" }],
|
||
}),
|
||
});
|
||
|
||
try {
|
||
const imageRequestTimeoutMs = 180_000;
|
||
const generated = await generateText({
|
||
model: languageModel,
|
||
system: systemPrompt({ selectedChatModel: finalModelId, requestHints }),
|
||
messages: convertedMessages,
|
||
abortSignal: AbortSignal.timeout(imageRequestTimeoutMs),
|
||
});
|
||
|
||
const rawText = generated.text?.trim() || "图片生成失败，请稍后重试";
|
||
const normalizedText = await normalizeAssistantImageMarkdown(
|
||
rawText,
|
||
effectiveImageParams?.imageSize || "2K"
|
||
);
|
||
const normalizedTextWithModel = prependImageModelBadge(
|
||
normalizedText,
|
||
currentModelConfig?.displayName || actualModelId,
|
||
);
|
||
|
||
const messageId = generateUUID();
|
||
dataStream.write({
|
||
type: "data-appendMessage",
|
||
data: JSON.stringify({
|
||
id: messageId,
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: normalizedTextWithModel }],
|
||
}),
|
||
});
|
||
|
||
// 生成结束标记
|
||
dataStream.write({
|
||
type: "data-appendMessage",
|
||
data: JSON.stringify({
|
||
id: generateUUID(),
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: "<!-- IMAGE_GENERATION_END -->" }],
|
||
}),
|
||
});
|
||
|
||
await saveMessages({
|
||
messages: [
|
||
{
|
||
id: messageId,
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: normalizedTextWithModel }],
|
||
createdAt: new Date(),
|
||
attachments: [],
|
||
chatId: id,
|
||
videoStorageStatus: null,
|
||
temporaryVideoUrl: null,
|
||
permanentVideoUrl: null,
|
||
archivedAt: null,
|
||
videoMetadata: null,
|
||
},
|
||
],
|
||
});
|
||
|
||
const latestUserMessage = uiMessages.filter((m) => m.role === "user").pop();
|
||
const userPrompt =
|
||
latestUserMessage?.parts?.find(
|
||
(p): p is { type: "text"; text: string } => p.type === "text"
|
||
)?.text || "";
|
||
await saveGeneratedImages(
|
||
normalizedTextWithModel,
|
||
session.user.id,
|
||
id,
|
||
messageId,
|
||
userPrompt,
|
||
finalModelId
|
||
);
|
||
return;
|
||
} catch (error) {
|
||
console.error("[Chat API] OpenAI-compatible image model generation error:", error);
|
||
|
||
const errorMessageId = generateUUID();
|
||
const errorText =
|
||
error instanceof Error ? error.message : "图片生成失败，请稍后重试";
|
||
|
||
dataStream.write({
|
||
type: "data-appendMessage",
|
||
data: JSON.stringify({
|
||
id: errorMessageId,
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: errorText }],
|
||
}),
|
||
});
|
||
dataStream.write({
|
||
type: "data-appendMessage",
|
||
data: JSON.stringify({
|
||
id: generateUUID(),
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: "<!-- IMAGE_GENERATION_END -->" }],
|
||
}),
|
||
});
|
||
|
||
await saveMessages({
|
||
messages: [
|
||
{
|
||
id: errorMessageId,
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: errorText }],
|
||
createdAt: new Date(),
|
||
attachments: [],
|
||
chatId: id,
|
||
videoStorageStatus: null,
|
||
temporaryVideoUrl: null,
|
||
permanentVideoUrl: null,
|
||
archivedAt: null,
|
||
videoMetadata: null,
|
||
},
|
||
],
|
||
});
|
||
return;
|
||
}
|
||
}
|
||
|
||
// 🎬 视频模型使用专门的生成流程
|
||
if (isVolcanoVideoModel) {
|
||
console.log("[Chat API] Using video generation API");
|
||
|
||
// 导入视频生成函数
|
||
const { doGenerateStream } = await import("@/lib/ai/providers/volcano-video");
|
||
|
||
const conversationMessage = smartAutoFriendlyMessage || null;
|
||
if (conversationMessage) {
|
||
const chatMessageId = generateUUID();
|
||
dataStream.write({
|
||
type: "data-appendMessage",
|
||
data: JSON.stringify({
|
||
id: chatMessageId,
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: conversationMessage }],
|
||
attachments: [],
|
||
}),
|
||
});
|
||
|
||
await saveMessages({
|
||
messages: [
|
||
{
|
||
id: chatMessageId,
|
||
chatId: id,
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: conversationMessage }],
|
||
attachments: [],
|
||
createdAt: new Date(),
|
||
videoStorageStatus: null,
|
||
temporaryVideoUrl: null,
|
||
permanentVideoUrl: null,
|
||
archivedAt: null,
|
||
videoMetadata: null,
|
||
},
|
||
],
|
||
});
|
||
}
|
||
|
||
const messagesForVideo = injectHistoryImagesToLastUserMessage(
|
||
applyOptimizedPromptToLatestUserMessage(
|
||
uiMessagesForModel,
|
||
smartAutoOptimizedPrompt,
|
||
hiddenPromptPrefix,
|
||
),
|
||
smartAutoHistoryImages,
|
||
);
|
||
|
||
// 从最后一条用户消息中提取提示词和图片
|
||
const lastUserMessage = messagesForVideo.filter(m => m.role === "user").pop();
|
||
let prompt = "";
|
||
const imageUrls: string[] = [];
|
||
const summarizeImageRef = (rawUrl: string) => {
|
||
const trimmed = String(rawUrl || "").trim();
|
||
if (!trimmed) return "";
|
||
if (trimmed.startsWith("data:image/")) {
|
||
return `data:image(base64,${trimmed.length} chars)`;
|
||
}
|
||
try {
|
||
const parsed = new URL(trimmed);
|
||
return `${parsed.origin}${parsed.pathname}`;
|
||
} catch {
|
||
return trimmed.slice(0, 120);
|
||
}
|
||
};
|
||
|
||
if (lastUserMessage) {
|
||
// 🆕 处理 parts 数组（AI SDK v4 格式）
|
||
if ((lastUserMessage as any).parts && Array.isArray((lastUserMessage as any).parts)) {
|
||
console.log("[Chat API] Video input parts:", (lastUserMessage as any).parts.length);
|
||
|
||
for (const part of (lastUserMessage as any).parts) {
|
||
if (part.type === "text" && part.text) {
|
||
// 提取文本
|
||
prompt += part.text;
|
||
} else if (part.type === "file" && part.url && part.mediaType?.startsWith("image/")) {
|
||
// 提取图片
|
||
imageUrls.push(part.url);
|
||
}
|
||
}
|
||
}
|
||
// 备用方案1：content 字符串/数组
|
||
const legacyContent = (lastUserMessage as any).content;
|
||
if (typeof legacyContent === "string") {
|
||
prompt = legacyContent;
|
||
} else if (Array.isArray(legacyContent)) {
|
||
for (const part of legacyContent) {
|
||
if (typeof part === "string") {
|
||
prompt += part;
|
||
} else if (part.type === "text") {
|
||
prompt += part.text;
|
||
} else if (part.type === "image" && (part as any).image) {
|
||
imageUrls.push((part as any).image);
|
||
}
|
||
}
|
||
} else if ((lastUserMessage as any).experimental_attachments) {
|
||
// 备用方案2：experimental_attachments
|
||
for (const attachment of (lastUserMessage as any).experimental_attachments) {
|
||
if (attachment.url && attachment.contentType?.startsWith("image/")) {
|
||
imageUrls.push(attachment.url);
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
console.log("[Chat API] Video input extracted:", {
|
||
promptLength: prompt.trim().length,
|
||
imageCount: imageUrls.length,
|
||
imageRefs: imageUrls.slice(0, 4).map(summarizeImageRef),
|
||
});
|
||
|
||
const videoMessageId = generateUUID();
|
||
const videoStartAt = Date.now();
|
||
const baseVideoMetadata: Record<string, any> = {
|
||
provider: effectiveProviderType || "volcano-video",
|
||
mode: effectiveVideoParams?.mode || "text-to-video",
|
||
resolution: effectiveVideoParams?.resolution || "720p",
|
||
ratio: effectiveVideoParams?.ratio || "16:9",
|
||
duration: effectiveVideoParams?.duration || 5,
|
||
status: "queued",
|
||
startedAt: videoStartAt,
|
||
updatedAt: videoStartAt,
|
||
};
|
||
|
||
try {
|
||
console.log("[Chat API] Video generation params:", {
|
||
promptLength: prompt.trim().length,
|
||
imageCount: imageUrls.length,
|
||
videoParams: effectiveVideoParams,
|
||
});
|
||
|
||
const videoMode = (effectiveVideoParams?.mode || "text-to-video") as VideoModeType;
|
||
|
||
// 🆕 重新执行子模型选择逻辑，确定要使用的模型 UUID
|
||
let videoModelId = finalModelId; // 默认使用选择的模型 ID
|
||
|
||
// 检查是否是主模型，需要选择子模型（仅 volcano-video 支持子模型）
|
||
const selectedModelData = await getAiModelById({ id: finalModelId });
|
||
const selectedModelProvider = selectedModelData
|
||
? await getAiProviderById({ id: selectedModelData.providerId }).catch(() => null)
|
||
: null;
|
||
if (
|
||
selectedModelData &&
|
||
!selectedModelData.isSubModel &&
|
||
selectedModelProvider?.type === "volcano-video"
|
||
) {
|
||
const videoSubModels = await getSubModelsByParentModelId(finalModelId);
|
||
console.log(`[Chat API] Video: Found ${videoSubModels.length} sub-models for master model`);
|
||
|
||
if (videoSubModels.length > 0) {
|
||
const matchedModel = selectBestVideoSubModelForMode(videoSubModels as any, videoMode);
|
||
|
||
if (matchedModel) {
|
||
videoModelId = matchedModel.model.id;
|
||
console.log(
|
||
`[Chat API] Video: Selected sub-model ${matchedModel.model.displayName} (${matchedModel.model.modelId}) for mode ${videoMode}`
|
||
);
|
||
} else {
|
||
console.log(
|
||
`[Chat API] Video: No compatible sub-model found for mode ${videoMode}, keep current model`
|
||
);
|
||
}
|
||
}
|
||
}
|
||
|
||
console.log("[Chat API] Getting model config for:", videoModelId);
|
||
const aiModelData = await getAiModelById({ id: videoModelId });
|
||
if (!aiModelData) {
|
||
throw new Error("视频模型配置未找到");
|
||
}
|
||
|
||
// 若用户手动选了不兼容模式的子模型，自动回退到同父模型兼容子模型
|
||
const selectedSupportsMode = getSupportedVideoModesForModel(aiModelData as any).includes(videoMode);
|
||
if (!selectedSupportsMode) {
|
||
let fallbackModel: { model: any } | null = null;
|
||
|
||
if (aiModelData.parentModelId) {
|
||
const siblingSubModels = await getSubModelsByParentModelId(aiModelData.parentModelId);
|
||
fallbackModel = selectBestVideoSubModelForMode(siblingSubModels as any, videoMode);
|
||
} else if (selectedModelProvider?.type === "volcano-video") {
|
||
const enabledModels = await getEnabledAiModelsWithProvider();
|
||
const sameProviderModels = enabledModels
|
||
.filter(
|
||
({ model, provider }) =>
|
||
provider.type === "volcano-video" &&
|
||
model.isEnabled &&
|
||
model.id !== aiModelData.id,
|
||
)
|
||
.map(({ model }) => ({ model }));
|
||
fallbackModel = selectBestVideoSubModelForMode(sameProviderModels as any, videoMode);
|
||
}
|
||
|
||
if (fallbackModel) {
|
||
console.warn(
|
||
`[Chat API] Video: Model ${aiModelData.modelId} does not support ${videoMode}, fallback to ${fallbackModel.model.modelId}`
|
||
);
|
||
videoModelId = fallbackModel.model.id;
|
||
}
|
||
}
|
||
|
||
const finalVideoModelData = videoModelId === aiModelData.id
|
||
? aiModelData
|
||
: await getAiModelById({ id: videoModelId });
|
||
if (!finalVideoModelData) {
|
||
throw new Error("视频模型配置未找到");
|
||
}
|
||
|
||
// 如果是子模型，获取父模型的 Provider 配置
|
||
let providerModelData = finalVideoModelData;
|
||
if (finalVideoModelData.isSubModel && finalVideoModelData.parentModelId) {
|
||
console.log(
|
||
"[Chat API] This is a sub-model, getting parent model config:",
|
||
finalVideoModelData.parentModelId
|
||
);
|
||
const parentModel = await getAiModelById({ id: finalVideoModelData.parentModelId });
|
||
if (parentModel) {
|
||
providerModelData = parentModel;
|
||
}
|
||
}
|
||
|
||
console.log("[Chat API] Getting provider config for:", providerModelData.providerId);
|
||
const providerData = await getAiProviderById({ id: providerModelData.providerId });
|
||
if (!providerData) {
|
||
throw new Error("Provider 配置未找到");
|
||
}
|
||
|
||
const actualVideoModelId = finalVideoModelData.modelId;
|
||
|
||
// 构建视频生成选项
|
||
const videoOptions: any = {
|
||
model: actualVideoModelId,
|
||
prompt: prompt || "生成视频",
|
||
mode: videoMode,
|
||
resolution: effectiveVideoParams?.resolution || "720p",
|
||
ratio: effectiveVideoParams?.ratio || "16:9",
|
||
duration: effectiveVideoParams?.duration || 5,
|
||
fps: effectiveVideoParams?.fps || 24,
|
||
cameraMove: effectiveVideoParams?.cameraMove || "none",
|
||
cameraFixed: effectiveVideoParams?.cameraFixed || false,
|
||
watermark: effectiveVideoParams?.watermark || false,
|
||
seed: effectiveVideoParams?.seed,
|
||
};
|
||
|
||
// 根据模式添加图片
|
||
if (videoOptions.mode === "first-frame" && imageUrls[0]) {
|
||
videoOptions.firstFrameImage = imageUrls[0];
|
||
} else if (videoOptions.mode === "first-last-frame" && imageUrls.length >= 2) {
|
||
videoOptions.firstFrameImage = imageUrls[0];
|
||
videoOptions.lastFrameImage = imageUrls[1];
|
||
} else if (videoOptions.mode === "reference-images" && imageUrls.length > 0) {
|
||
videoOptions.referenceImages = imageUrls;
|
||
}
|
||
|
||
console.log("[Chat API] Video options constructed:", {
|
||
model: videoOptions.model,
|
||
mode: videoOptions.mode,
|
||
resolution: videoOptions.resolution,
|
||
ratio: videoOptions.ratio,
|
||
duration: videoOptions.duration,
|
||
imageCount:
|
||
videoOptions.referenceImages?.length ||
|
||
Number(Boolean(videoOptions.firstFrameImage)) +
|
||
Number(Boolean(videoOptions.lastFrameImage)),
|
||
});
|
||
|
||
// 先写入占位消息，避免刷新后消息丢失
|
||
const pendingText = "🎬 视频任务已创建，正在处理中，请稍候...";
|
||
await saveMessages({
|
||
messages: [
|
||
{
|
||
id: videoMessageId,
|
||
chatId: id,
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: pendingText }],
|
||
attachments: [],
|
||
createdAt: new Date(),
|
||
videoStorageStatus: "temporary",
|
||
temporaryVideoUrl: null,
|
||
permanentVideoUrl: null,
|
||
archivedAt: null,
|
||
videoMetadata: {
|
||
...baseVideoMetadata,
|
||
model: actualVideoModelId,
|
||
status: "running",
|
||
updatedAt: Date.now(),
|
||
} as any,
|
||
},
|
||
],
|
||
});
|
||
|
||
dataStream.write({
|
||
type: "data-appendMessage",
|
||
data: JSON.stringify({
|
||
id: videoMessageId,
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: pendingText }],
|
||
attachments: [],
|
||
}),
|
||
});
|
||
|
||
const apiKey = providerData.apiKey;
|
||
const baseURL = providerData.baseUrl || "https://ark.cn-beijing.volces.com/api/v3";
|
||
console.log("[Chat API] Provider configured:", { baseURL, hasApiKey: !!apiKey });
|
||
|
||
// 调用视频生成流
|
||
console.log("[Chat API] Starting video generation stream...");
|
||
const videoStream = doGenerateStream(apiKey, baseURL, videoOptions);
|
||
|
||
// 收集所有流输出
|
||
console.log("[Chat API] Processing video stream...");
|
||
let fullText = "";
|
||
let chunkCount = 0;
|
||
|
||
for await (const chunk of videoStream) {
|
||
chunkCount++;
|
||
fullText += chunk;
|
||
console.log(`[Chat API] Received chunk ${chunkCount}, length: ${chunk.length}`);
|
||
}
|
||
|
||
console.log(
|
||
`[Chat API] Video stream completed. Total chunks: ${chunkCount}, total text length: ${fullText.length}`
|
||
);
|
||
|
||
// 提取流中的元数据并清理输出文案
|
||
const metadataMatch = fullText.match(/<!--\s*VIDEO_METADATA:(.*?)-->/s);
|
||
let streamMetadata: Record<string, any> = {};
|
||
if (metadataMatch?.[1]) {
|
||
try {
|
||
streamMetadata = JSON.parse(metadataMatch[1]);
|
||
} catch (parseError) {
|
||
console.warn("[Chat API] Failed to parse VIDEO_METADATA from stream", parseError);
|
||
}
|
||
}
|
||
const cleanedText = fullText
|
||
.replace(/<!--\s*VIDEO_METADATA:.*?-->/gs, "")
|
||
.trim();
|
||
const videoUrlMatch = cleanedText.match(/<video[^>]+src="([^"]+)"/);
|
||
const temporaryVideoUrl =
|
||
(typeof streamMetadata.temporaryVideoUrl === "string" && streamMetadata.temporaryVideoUrl) ||
|
||
(videoUrlMatch ? videoUrlMatch[1] : null);
|
||
|
||
const successMetadata = {
|
||
...baseVideoMetadata,
|
||
...streamMetadata,
|
||
model: actualVideoModelId,
|
||
taskId:
|
||
typeof streamMetadata.taskId === "string" && streamMetadata.taskId
|
||
? streamMetadata.taskId
|
||
: undefined,
|
||
status: "succeeded",
|
||
completedAt: Date.now(),
|
||
updatedAt: Date.now(),
|
||
};
|
||
|
||
await updateMessageById({
|
||
messageId: videoMessageId,
|
||
chatId: id,
|
||
parts: [{ type: "text", text: cleanedText }],
|
||
videoStorageStatus: temporaryVideoUrl ? "temporary" : null,
|
||
temporaryVideoUrl: temporaryVideoUrl,
|
||
permanentVideoUrl: null,
|
||
archivedAt: null,
|
||
videoMetadata: successMetadata as any,
|
||
});
|
||
await consumeGeneratedVideoPoints({
|
||
userId: session.user.id,
|
||
chatId: id,
|
||
messageId: videoMessageId,
|
||
modelRefs: [finalVideoModelData.id, providerModelData.id, finalModelId],
|
||
});
|
||
|
||
dataStream.write({
|
||
type: "data-appendMessage",
|
||
data: JSON.stringify({
|
||
id: videoMessageId,
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: cleanedText }],
|
||
attachments: [],
|
||
}),
|
||
});
|
||
|
||
console.log("[Chat API] ✅ Video message updated successfully");
|
||
if (temporaryVideoUrl) {
|
||
console.log("[Chat API] 📹 临时视频URL已保存，开始异步归档到S3");
|
||
scheduleVideoArchive({
|
||
chatId: id,
|
||
messageId: videoMessageId,
|
||
text: cleanedText,
|
||
temporaryVideoUrl,
|
||
taskId:
|
||
typeof successMetadata.taskId === "string" && successMetadata.taskId
|
||
? successMetadata.taskId
|
||
: videoMessageId,
|
||
metadata: successMetadata,
|
||
});
|
||
}
|
||
|
||
console.log("[Chat API] Video generation finished successfully");
|
||
return;
|
||
} catch (error: any) {
|
||
console.error("[Chat API] ❌ Video generation error:", error);
|
||
console.error("[Chat API] Error stack:", error.stack);
|
||
|
||
const errorMessage = `❌ 视频生成失败：${error.message}\n\n请检查配置或稍后重试。`;
|
||
const failedMetadata = {
|
||
...baseVideoMetadata,
|
||
status: "failed",
|
||
error: error?.message || "未知错误",
|
||
updatedAt: Date.now(),
|
||
};
|
||
|
||
try {
|
||
await updateMessageById({
|
||
messageId: videoMessageId,
|
||
chatId: id,
|
||
parts: [{ type: "text", text: errorMessage }],
|
||
videoStorageStatus: null,
|
||
temporaryVideoUrl: null,
|
||
permanentVideoUrl: null,
|
||
archivedAt: null,
|
||
videoMetadata: failedMetadata as any,
|
||
});
|
||
|
||
dataStream.write({
|
||
type: "data-appendMessage",
|
||
data: JSON.stringify({
|
||
id: videoMessageId,
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: errorMessage }],
|
||
attachments: [],
|
||
}),
|
||
});
|
||
} catch (updateError) {
|
||
console.error("[Chat API] Failed to update placeholder video message:", updateError);
|
||
const fallbackMessageId = generateUUID();
|
||
dataStream.write({
|
||
type: "data-appendMessage",
|
||
data: JSON.stringify({
|
||
id: fallbackMessageId,
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: errorMessage }],
|
||
attachments: [],
|
||
}),
|
||
});
|
||
await saveMessages({
|
||
messages: [
|
||
{
|
||
id: fallbackMessageId,
|
||
chatId: id,
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: errorMessage }],
|
||
attachments: [],
|
||
createdAt: new Date(),
|
||
videoStorageStatus: null,
|
||
temporaryVideoUrl: null,
|
||
permanentVideoUrl: null,
|
||
archivedAt: null,
|
||
videoMetadata: failedMetadata as any,
|
||
},
|
||
],
|
||
});
|
||
}
|
||
|
||
return;
|
||
}
|
||
}
|
||
|
||
// 常规文本模型使用 streamText
|
||
const streamRequestTimeoutMs = resolvedModelSupportsImageGen ? 180_000 : 90_000;
|
||
const shouldDisableSmoothStream =
|
||
resolvedModelSupportsImageGen || isVolcanoVideoModel;
|
||
|
||
const result = streamText({
|
||
model: languageModel,
|
||
system: systemPrompt({ selectedChatModel: finalModelId, requestHints }),
|
||
messages: convertedMessages,
|
||
stopWhen: stepCountIs(5),
|
||
experimental_activeTools:
|
||
finalModelId === "chat-model-reasoning" || isVolcanoVideoModel ? [] : activeTools,
|
||
experimental_transform: shouldDisableSmoothStream
|
||
? undefined
|
||
: smoothStream({ chunking: "word" }),
|
||
tools: isVolcanoVideoModel ? {} : tools, // 🆕 视频模型禁用工具
|
||
abortSignal: AbortSignal.timeout(streamRequestTimeoutMs),
|
||
experimental_telemetry: {
|
||
isEnabled: isProductionEnvironment,
|
||
functionId: "stream-text",
|
||
},
|
||
onFinish: async ({ usage }) => {
|
||
try {
|
||
const providers = await getTokenlensCatalog();
|
||
if (!actualModelId) {
|
||
finalMergedUsage = usage;
|
||
dataStream.write({
|
||
type: "data-usage",
|
||
data: finalMergedUsage,
|
||
});
|
||
return;
|
||
}
|
||
|
||
if (!providers) {
|
||
finalMergedUsage = usage;
|
||
dataStream.write({
|
||
type: "data-usage",
|
||
data: finalMergedUsage,
|
||
});
|
||
return;
|
||
}
|
||
|
||
const summary = getUsage({
|
||
modelId: actualModelId,
|
||
usage,
|
||
providers,
|
||
});
|
||
finalMergedUsage = {
|
||
...usage,
|
||
...summary,
|
||
modelId: actualModelId,
|
||
} as AppUsage;
|
||
dataStream.write({ type: "data-usage", data: finalMergedUsage });
|
||
} catch (err) {
|
||
console.warn("TokenLens enrichment failed", err);
|
||
finalMergedUsage = usage;
|
||
dataStream.write({ type: "data-usage", data: finalMergedUsage });
|
||
}
|
||
},
|
||
});
|
||
|
||
result.consumeStream();
|
||
|
||
dataStream.merge(
|
||
result.toUIMessageStream({
|
||
sendReasoning: true,
|
||
})
|
||
);
|
||
},
|
||
generateId: generateUUID,
|
||
onFinish: async ({ messages }) => {
|
||
await saveMessages({
|
||
messages: messages.map((currentMessage) => ({
|
||
id: currentMessage.id,
|
||
role: currentMessage.role,
|
||
parts: currentMessage.parts,
|
||
createdAt: new Date(),
|
||
attachments: [],
|
||
chatId: id,
|
||
videoStorageStatus: null,
|
||
temporaryVideoUrl: null,
|
||
permanentVideoUrl: null,
|
||
archivedAt: null,
|
||
videoMetadata: null,
|
||
})),
|
||
});
|
||
|
||
// 检测助手消息中是否包含视频，如果有则立即标记并归档到永久存储
|
||
for (const msg of messages) {
|
||
if (msg.role === "assistant" && msg.parts) {
|
||
const textContent = msg.parts
|
||
.filter((part: any) => part && part.type === "text" && part.text)
|
||
.map((part: any) => part.text)
|
||
.join("");
|
||
|
||
const tempVideoUrl = extractVideoUrlFromText(textContent);
|
||
if (tempVideoUrl) {
|
||
const genericVideoMetadata = {
|
||
provider: "stream",
|
||
model: actualModelId,
|
||
status: "succeeded",
|
||
updatedAt: Date.now(),
|
||
};
|
||
|
||
await updateMessageById({
|
||
messageId: msg.id,
|
||
chatId: id,
|
||
videoStorageStatus: "temporary",
|
||
temporaryVideoUrl: tempVideoUrl,
|
||
permanentVideoUrl: null,
|
||
archivedAt: null,
|
||
videoMetadata: genericVideoMetadata as any,
|
||
});
|
||
|
||
scheduleVideoArchive({
|
||
chatId: id,
|
||
messageId: msg.id,
|
||
text: textContent,
|
||
temporaryVideoUrl: tempVideoUrl,
|
||
taskId: msg.id,
|
||
metadata: genericVideoMetadata,
|
||
});
|
||
}
|
||
}
|
||
}
|
||
|
||
if (finalMergedUsage) {
|
||
try {
|
||
await updateChatLastContextById({
|
||
chatId: id,
|
||
context: finalMergedUsage,
|
||
});
|
||
} catch (err) {
|
||
console.warn("Unable to persist last usage for chat", id, err);
|
||
}
|
||
}
|
||
},
|
||
onError: () => {
|
||
return "Oops, an error occurred!";
|
||
},
|
||
});
|
||
|
||
// const streamContext = getStreamContext();
|
||
|
||
// if (streamContext) {
|
||
// return new Response(
|
||
// await streamContext.resumableStream(streamId, () =>
|
||
// stream.pipeThrough(new JsonToSseTransformStream())
|
||
// )
|
||
// );
|
||
// }
|
||
|
||
return createSseResponse(stream);
|
||
} catch (error) {
|
||
const vercelId = request.headers.get("x-vercel-id");
|
||
|
||
if (error instanceof ChatSDKError) {
|
||
return error.toResponse();
|
||
}
|
||
|
||
// Check for Vercel AI Gateway credit card error
|
||
if (
|
||
error instanceof Error &&
|
||
error.message?.includes(
|
||
"AI Gateway requires a valid credit card on file to service requests"
|
||
)
|
||
) {
|
||
return new ChatSDKError("bad_request:activate_gateway").toResponse();
|
||
}
|
||
|
||
console.error("Unhandled error in chat API:", error, { vercelId });
|
||
return new ChatSDKError("offline:chat").toResponse();
|
||
} finally {
|
||
if (requestDedupKey && acquiredInFlightLock) {
|
||
inFlightChatRequests.delete(requestDedupKey);
|
||
}
|
||
}
|
||
}
|
||
|
||
export async function DELETE(request: Request) {
|
||
const { searchParams } = new URL(request.url);
|
||
const id = searchParams.get("id");
|
||
|
||
if (!id) {
|
||
return new ChatSDKError("bad_request:api").toResponse();
|
||
}
|
||
|
||
const session = await auth();
|
||
|
||
if (!session?.user) {
|
||
return new ChatSDKError("unauthorized:chat").toResponse();
|
||
}
|
||
|
||
const chat = await getChatById({ id });
|
||
|
||
if (chat?.userId !== session.user.id) {
|
||
return new ChatSDKError("forbidden:chat").toResponse();
|
||
}
|
||
|
||
const deletedChat = await deleteChatById({ id });
|
||
|
||
return Response.json(deletedChat, { status: 200 });
|
||
}
|