3839 lines
150 KiB
TypeScript
3839 lines
150 KiB
TypeScript
import { geolocation } from "@vercel/functions";
|
||
import {
|
||
convertToModelMessages,
|
||
createUIMessageStream,
|
||
generateText,
|
||
JsonToSseTransformStream,
|
||
smoothStream,
|
||
stepCountIs,
|
||
streamText,
|
||
} from "ai";
|
||
import { unstable_cache as cache } from "next/cache";
|
||
import { after, NextResponse } from "next/server";
|
||
import {
|
||
createResumableStreamContext,
|
||
type ResumableStreamContext,
|
||
} from "resumable-stream";
|
||
import type { ModelCatalog } from "tokenlens/core";
|
||
import { fetchModels } from "tokenlens/fetch";
|
||
import { getUsage } from "tokenlens/helpers";
|
||
import { auth, type UserType } from "@/app/(auth)/auth";
|
||
import type { VisibilityType } from "@/components/visibility-selector";
|
||
import { entitlementsByUserType } from "@/lib/ai/entitlements";
|
||
import type { ChatModel } from "@/lib/ai/models";
|
||
import { type RequestHints, systemPrompt } from "@/lib/ai/prompts";
|
||
import { myProvider } from "@/lib/ai/providers";
|
||
import { getLanguageModelById } from "@/lib/ai/dynamic-provider";
|
||
import { createDocument } from "@/lib/ai/tools/create-document";
|
||
import { getWeather } from "@/lib/ai/tools/get-weather";
|
||
import { requestSuggestions } from "@/lib/ai/tools/request-suggestions";
|
||
import { updateDocument } from "@/lib/ai/tools/update-document";
|
||
import { generateVideo } from "@/lib/ai/tools/generate-video";
|
||
import { getEnabledTools } from "@/lib/ai/tools-registry";
|
||
import { isProductionEnvironment } from "@/lib/constants";
|
||
import {
|
||
createStreamId,
|
||
deleteChatById,
|
||
getChatById,
|
||
getPointsSystemConfig,
|
||
getUserById,
|
||
getMessageCountByUserId,
|
||
getMessagesByChatId,
|
||
saveChat,
|
||
saveMessages,
|
||
updateChatLastContextById,
|
||
getAiModelById,
|
||
getEnabledAiModelsWithProvider,
|
||
getSubModelsByParentModelId,
|
||
getAiProviderById,
|
||
checkUserImageQuota,
|
||
consumeUserPoints,
|
||
} from "@/lib/db/queries";
|
||
import { db } from "@/lib/db";
|
||
import type { DBMessage } from "@/lib/db/schema";
|
||
import { generatedImage } from "@/lib/db/schema";
|
||
import { ChatSDKError } from "@/lib/errors";
|
||
import { extractImageUrlsFromText } from "@/lib/image-url";
|
||
import { uploadFile } from "@/lib/storage/service";
|
||
import type { ChatMessage } from "@/lib/types";
|
||
import type { AppUsage } from "@/lib/usage";
|
||
import { convertToUIMessages, generateUUID } from "@/lib/utils";
|
||
import { isAiModelAgentAuto } from "@/lib/db/ai-model-flags";
|
||
import { generateTitleFromUserMessage } from "../../actions";
|
||
import { type PostRequestBody, postRequestBodySchema } from "./schema";
|
||
|
||
/**
|
||
* 将图片 URL 转换为 base64 格式
|
||
* 用于将 S3 URL 的图片转换为 base64 发送给 Gemini API
|
||
*/
|
||
async function convertImageUrlToBase64(url: string): Promise<string> {
|
||
try {
|
||
console.log("[Image Convert] Fetching image from:", url);
|
||
const response = await fetch(url);
|
||
if (!response.ok) {
|
||
throw new Error(`Failed to fetch image: ${response.statusText}`);
|
||
}
|
||
|
||
const arrayBuffer = await response.arrayBuffer();
|
||
const buffer = Buffer.from(arrayBuffer);
|
||
const base64 = buffer.toString("base64");
|
||
|
||
// 获取 MIME 类型
|
||
const contentType = response.headers.get("content-type") || "image/jpeg";
|
||
const dataUrl = `data:${contentType};base64,${base64}`;
|
||
|
||
console.log("[Image Convert] Converted to base64, size:", base64.length);
|
||
return dataUrl;
|
||
} catch (error) {
|
||
console.error("[Image Convert] Failed:", error);
|
||
throw error;
|
||
}
|
||
}
|
||
|
||
/**
|
||
* 检测消息中是否包含用户标注过的图片（文件名以 annotated- 开头）
|
||
*/
|
||
function hasAnnotatedImage(messages: ChatMessage[]): boolean {
|
||
for (const message of messages) {
|
||
for (const part of message.parts) {
|
||
if (part.type === "file" && "name" in part && typeof part.name === "string") {
|
||
if (part.name.startsWith("annotated-")) {
|
||
return true;
|
||
}
|
||
}
|
||
}
|
||
}
|
||
return false;
|
||
}
|
||
|
||
/**
|
||
* 标注图片的系统提示词（用户不可见，后台自动添加）
|
||
*/
|
||
const ANNOTATION_SYSTEM_PROMPT = "【重要】用户提供的图片上有标注线条（如箭头、圆圈、画笔痕迹或文字说明），这些标注是用户用来指示需要修改的区域或方向。请仔细理解标注的含义，按照标注指示进行编辑，但在生成的新图片中不要保留这些标注线条，输出干净的编辑结果。";
|
||
|
||
function applyHiddenPromptToMessages(messages: ChatMessage[], hiddenPrompt?: string): ChatMessage[] {
|
||
if (!hiddenPrompt || !hiddenPrompt.trim()) return messages;
|
||
const cloned = messages.map((msg) => ({
|
||
...msg,
|
||
parts: msg.parts.map((part) => ({ ...part })),
|
||
}));
|
||
for (let i = cloned.length - 1; i >= 0; i--) {
|
||
const msg = cloned[i];
|
||
if (msg.role !== "user") continue;
|
||
const textPart = msg.parts.find((part) => part.type === "text") as { type: "text"; text: string } | undefined;
|
||
if (textPart) {
|
||
textPart.text = `${hiddenPrompt}\n\n${textPart.text}`.trim();
|
||
} else {
|
||
msg.parts.unshift({ type: "text", text: hiddenPrompt });
|
||
}
|
||
break;
|
||
}
|
||
return cloned;
|
||
}
|
||
|
||
function applyOptimizedPromptToLatestUserMessage(
|
||
messages: ChatMessage[],
|
||
optimizedPrompt?: string,
|
||
hiddenPromptPrefix = "",
|
||
): ChatMessage[] {
|
||
const prompt = optimizedPrompt?.trim();
|
||
if (!prompt) return messages;
|
||
|
||
const cloned = messages.map((msg) => ({
|
||
...msg,
|
||
parts: msg.parts.map((part) => ({ ...part })),
|
||
}));
|
||
|
||
for (let i = cloned.length - 1; i >= 0; i -= 1) {
|
||
const msg = cloned[i];
|
||
if (msg.role !== "user") continue;
|
||
|
||
const textPart = msg.parts.find(
|
||
(part): part is { type: "text"; text: string } => part.type === "text",
|
||
);
|
||
const originalText = textPart?.text?.trim() || "";
|
||
const normalizedOriginal = hiddenPromptPrefix && originalText.startsWith(hiddenPromptPrefix)
|
||
? originalText.slice(hiddenPromptPrefix.length).trim()
|
||
: originalText;
|
||
const hasOriginalDemand = normalizedOriginal.length > 0;
|
||
const hasOriginalInOptimized =
|
||
hasOriginalDemand &&
|
||
(prompt.includes(normalizedOriginal) || normalizedOriginal.includes(prompt));
|
||
const finalPrompt = hasOriginalDemand && !hasOriginalInOptimized
|
||
? [
|
||
hiddenPromptPrefix.trim(),
|
||
`【核心目标（必须优先满足）】${normalizedOriginal}`,
|
||
`【执行优化细节】${prompt}`,
|
||
]
|
||
.filter(Boolean)
|
||
.join("\n\n")
|
||
.trim()
|
||
: `${hiddenPromptPrefix}${prompt}`.trim();
|
||
console.log("[Prompt Merge] Final image prompt prepared:", {
|
||
original: normalizedOriginal.substring(0, 120),
|
||
optimized: prompt.substring(0, 120),
|
||
merged: finalPrompt.substring(0, 180),
|
||
forcePreserveOriginal: hasOriginalDemand && !hasOriginalInOptimized,
|
||
});
|
||
|
||
if (textPart) {
|
||
textPart.text = finalPrompt;
|
||
} else {
|
||
msg.parts.unshift({ type: "text", text: finalPrompt });
|
||
}
|
||
break;
|
||
}
|
||
|
||
return cloned;
|
||
}
|
||
|
||
type ChatImageParams = NonNullable<PostRequestBody["imageParams"]>;
|
||
type AgentRecommendedImageParams = {
|
||
aspectRatio?: string;
|
||
imageSize?: string;
|
||
imageCount?: number;
|
||
negativePrompt?: string;
|
||
promptExtend?: boolean;
|
||
};
|
||
|
||
type ChatVideoParams = NonNullable<PostRequestBody["videoParams"]>;
|
||
type ChatVideoParamsDraft = Partial<ChatVideoParams>;
|
||
type AgentRecommendedVideoParams = {
|
||
mode?: "text-to-video" | "first-frame" | "first-last-frame" | "reference-images";
|
||
resolution?: "480p" | "720p" | "1080p";
|
||
ratio?: string;
|
||
duration?: number;
|
||
generateAudio?: boolean;
|
||
cameraMove?: string;
|
||
};
|
||
|
||
type AgentVideoPlan = {
|
||
script: string;
|
||
storyboard: Array<{
|
||
shot: number;
|
||
title: string;
|
||
duration: number;
|
||
description: string;
|
||
prompt: string;
|
||
camera?: string;
|
||
}>;
|
||
finalPrompt: string;
|
||
negativePrompt?: string;
|
||
recommendedVideoParams?: AgentRecommendedVideoParams;
|
||
};
|
||
|
||
const IMAGE_FAILOVER_PROVIDER_ALLOWLIST = new Set([
|
||
"alibaba-image",
|
||
"ruxa-image",
|
||
"fal-image",
|
||
"volcano-vision",
|
||
"gemini",
|
||
"huanke-gemini",
|
||
]);
|
||
|
||
const IMAGE_FAILOVER_MODEL_COOLDOWN_MS = 30 * 60 * 1000;
|
||
const imageFailoverModelCooldown = new Map<string, { until: number; reason: string }>();
|
||
|
||
function normalizeFailoverErrorMessage(error: unknown): string {
|
||
if (!error) return "";
|
||
if (typeof error === "string") return error;
|
||
if (error instanceof Error) return error.message || String(error);
|
||
try {
|
||
return JSON.stringify(error);
|
||
} catch {
|
||
return String(error);
|
||
}
|
||
}
|
||
|
||
function shouldCooldownImageFallbackModel(errorMessage: string): boolean {
|
||
const msg = errorMessage.toLowerCase();
|
||
return (
|
||
msg.includes("model not found") ||
|
||
msg.includes("model_not_found") ||
|
||
msg.includes("no available channel for model") ||
|
||
msg.includes("unknown model") ||
|
||
msg.includes("invalid model") ||
|
||
msg.includes("api key is missing") ||
|
||
msg.includes("invalid api key") ||
|
||
msg.includes("unauthorized") ||
|
||
msg.includes("cannot convert argument to a bytestring")
|
||
);
|
||
}
|
||
|
||
function isImageFallbackModelCoolingDown(modelId: string): boolean {
|
||
const snapshot = imageFailoverModelCooldown.get(modelId);
|
||
if (!snapshot) return false;
|
||
if (snapshot.until <= Date.now()) {
|
||
imageFailoverModelCooldown.delete(modelId);
|
||
return false;
|
||
}
|
||
return true;
|
||
}
|
||
|
||
function setImageFallbackModelCooldown(modelId: string, reason: string): void {
|
||
imageFailoverModelCooldown.set(modelId, {
|
||
until: Date.now() + IMAGE_FAILOVER_MODEL_COOLDOWN_MS,
|
||
reason: reason.slice(0, 500),
|
||
});
|
||
}
|
||
|
||
function getImageCrossProviderCandidates(
|
||
allModels: any[],
|
||
options: {
|
||
currentModelId: string;
|
||
currentProviderId?: string;
|
||
providerPriority: Record<string, number>;
|
||
}
|
||
) {
|
||
return allModels
|
||
.filter((m) => {
|
||
if (!m?.model?.supportsImageGen || !m?.model?.isEnabled || m?.model?.isSubModel) {
|
||
return false;
|
||
}
|
||
if (!IMAGE_FAILOVER_PROVIDER_ALLOWLIST.has(String(m?.provider?.type || ""))) {
|
||
return false;
|
||
}
|
||
if (m.model.id === options.currentModelId) return false;
|
||
if (options.currentProviderId && m.provider.id === options.currentProviderId) return false;
|
||
if (isImageFallbackModelCoolingDown(m.model.id)) {
|
||
const cooldown = imageFailoverModelCooldown.get(m.model.id);
|
||
console.log(
|
||
`[Chat API] ⏭ Skip image failover model on cooldown: ${m.model.displayName} (${m.provider.type}) - ${cooldown?.reason || "cooldown"}`
|
||
);
|
||
return false;
|
||
}
|
||
return true;
|
||
})
|
||
.sort((a, b) => {
|
||
const left = options.providerPriority[a.provider.type as string] ?? 99;
|
||
const right = options.providerPriority[b.provider.type as string] ?? 99;
|
||
if (left !== right) return left - right;
|
||
return (a.model.sortOrder ?? 0) - (b.model.sortOrder ?? 0);
|
||
});
|
||
}
|
||
|
||
function hasUserCustomizedImageParams(params?: PostRequestBody["imageParams"]): boolean {
|
||
if (!params) return false;
|
||
const customizedAspectRatio =
|
||
typeof params.aspectRatio === "string" && params.aspectRatio.trim() && params.aspectRatio !== "auto";
|
||
const customizedImageSize =
|
||
typeof params.imageSize === "string" && params.imageSize.trim() && params.imageSize.toUpperCase() !== "2K";
|
||
const customizedImageCount = typeof params.imageCount === "number" && params.imageCount !== 1;
|
||
const customizedPromptExtend = typeof params.promptExtend === "boolean";
|
||
const customizedAdvanced = Boolean(
|
||
params.size || params.seed !== undefined || params.n || params.negativePrompt || customizedPromptExtend
|
||
);
|
||
return customizedAspectRatio || customizedImageSize || customizedImageCount || customizedAdvanced;
|
||
}
|
||
|
||
function mergeImageParamsWithAgentRecommendation(params: {
|
||
incoming?: PostRequestBody["imageParams"];
|
||
recommended?: AgentRecommendedImageParams;
|
||
intent?: "image_edit" | "image_gen" | "vision" | "video_gen" | "chat";
|
||
hasUserCustomSettings: boolean;
|
||
}): PostRequestBody["imageParams"] {
|
||
const { incoming, recommended, intent, hasUserCustomSettings } = params;
|
||
const base: ChatImageParams = { ...(incoming || {}) };
|
||
if (!recommended) {
|
||
return Object.keys(base).length > 0 ? base : undefined;
|
||
}
|
||
|
||
const merged: ChatImageParams = {
|
||
...base,
|
||
negativePrompt:
|
||
typeof recommended.negativePrompt === "string" && recommended.negativePrompt.trim()
|
||
? recommended.negativePrompt.trim()
|
||
: base.negativePrompt,
|
||
promptExtend:
|
||
typeof recommended.promptExtend === "boolean"
|
||
? recommended.promptExtend
|
||
: base.promptExtend,
|
||
};
|
||
|
||
if (!hasUserCustomSettings) {
|
||
if (recommended.aspectRatio) merged.aspectRatio = recommended.aspectRatio;
|
||
if (recommended.imageSize) merged.imageSize = recommended.imageSize;
|
||
if (typeof recommended.imageCount === "number") merged.imageCount = recommended.imageCount;
|
||
}
|
||
|
||
if (intent === "image_edit") {
|
||
merged.imageCount = 1;
|
||
if (typeof base.promptExtend !== "boolean") {
|
||
merged.promptExtend = false;
|
||
}
|
||
}
|
||
|
||
return Object.keys(merged).length > 0 ? merged : undefined;
|
||
}
|
||
|
||
function hasUserCustomizedVideoParams(params?: PostRequestBody["videoParams"]): boolean {
|
||
if (!params) return false;
|
||
const defaults = {
|
||
mode: "text-to-video",
|
||
resolution: "720p",
|
||
ratio: "16:9",
|
||
duration: 5,
|
||
generateAudio: true,
|
||
cameraMove: "none",
|
||
} as const;
|
||
|
||
if (params.mode && params.mode !== defaults.mode) return true;
|
||
if (params.resolution && params.resolution !== defaults.resolution) return true;
|
||
if (params.ratio && params.ratio !== defaults.ratio) return true;
|
||
if (typeof params.duration === "number" && params.duration !== defaults.duration) return true;
|
||
if (typeof params.generateAudio === "boolean" && params.generateAudio !== defaults.generateAudio)
|
||
return true;
|
||
if (typeof params.cameraMove === "string" && params.cameraMove !== defaults.cameraMove)
|
||
return true;
|
||
if (typeof params.seed === "number") return true;
|
||
return false;
|
||
}
|
||
|
||
function mergeVideoParamsWithAgentRecommendation(params: {
|
||
incoming?: PostRequestBody["videoParams"];
|
||
recommended?: AgentRecommendedVideoParams;
|
||
hasUserCustomSettings: boolean;
|
||
}): PostRequestBody["videoParams"] {
|
||
const { incoming, recommended, hasUserCustomSettings } = params;
|
||
const base: ChatVideoParamsDraft = { ...(incoming || {}) };
|
||
if (!recommended) {
|
||
return Object.keys(base).length > 0 ? (base as ChatVideoParams) : undefined;
|
||
}
|
||
|
||
const merged: ChatVideoParamsDraft = { ...base };
|
||
if (!hasUserCustomSettings) {
|
||
if (recommended.mode) merged.mode = recommended.mode;
|
||
if (recommended.resolution) merged.resolution = recommended.resolution;
|
||
if (recommended.ratio) merged.ratio = recommended.ratio;
|
||
if (typeof recommended.duration === "number") merged.duration = recommended.duration;
|
||
if (typeof recommended.generateAudio === "boolean") {
|
||
merged.generateAudio = recommended.generateAudio;
|
||
}
|
||
if (recommended.cameraMove) merged.cameraMove = recommended.cameraMove;
|
||
}
|
||
|
||
if (!merged.mode) {
|
||
merged.mode = recommended.mode || (incoming?.mode as ChatVideoParams["mode"]) || "text-to-video";
|
||
}
|
||
if (!merged.resolution) {
|
||
merged.resolution =
|
||
recommended.resolution || (incoming?.resolution as ChatVideoParams["resolution"]) || "720p";
|
||
}
|
||
if (!merged.ratio) {
|
||
merged.ratio = recommended.ratio || incoming?.ratio || "16:9";
|
||
}
|
||
if (typeof merged.duration !== "number") {
|
||
merged.duration = recommended.duration || incoming?.duration || 5;
|
||
}
|
||
|
||
return Object.keys(merged).length > 0 ? (merged as ChatVideoParams) : undefined;
|
||
}
|
||
|
||
function sanitizeAgentOutputSettingsSection(params: {
|
||
message?: string;
|
||
userImageParams?: PostRequestBody["imageParams"];
|
||
hasUserCustomImageParams: boolean;
|
||
}): string | undefined {
|
||
const { message, userImageParams, hasUserCustomImageParams } = params;
|
||
if (!message?.trim()) return message;
|
||
|
||
const outputSectionRegex = /###\s*输出设置[\s\S]*?(?=\n###\s*|$)/;
|
||
if (!outputSectionRegex.test(message)) return message;
|
||
|
||
if (!hasUserCustomImageParams) {
|
||
return message.replace(
|
||
outputSectionRegex,
|
||
"### 输出设置\n- 参数：按用户面板当前设置执行（未自动改写）"
|
||
);
|
||
}
|
||
|
||
const lines: string[] = [];
|
||
const aspectRatio =
|
||
typeof userImageParams?.aspectRatio === "string" ? userImageParams.aspectRatio.trim() : "";
|
||
const imageSize =
|
||
typeof userImageParams?.imageSize === "string" ? userImageParams.imageSize.trim() : "";
|
||
const imageCount =
|
||
typeof userImageParams?.imageCount === "number" ? userImageParams.imageCount : undefined;
|
||
|
||
if (aspectRatio && aspectRatio !== "auto") {
|
||
lines.push(`- 宽高比：${aspectRatio}（来自用户面板设置）`);
|
||
}
|
||
if (imageSize && imageSize.toUpperCase() !== "2K") {
|
||
lines.push(`- 清晰度档位：${imageSize.toUpperCase()}（来自用户面板设置）`);
|
||
}
|
||
if (typeof imageCount === "number" && Number.isFinite(imageCount) && imageCount !== 1) {
|
||
lines.push(`- 出图数量：${Math.max(1, Math.round(imageCount))} 张（来自用户面板设置）`);
|
||
}
|
||
lines.push("- 其余参数：保持用户面板当前设置");
|
||
|
||
return message.replace(outputSectionRegex, `### 输出设置\n${lines.join("\n")}`);
|
||
}
|
||
|
||
function guessImageMediaType(url: string): string {
|
||
const cleanUrl = url.split("?")[0]?.toLowerCase() || "";
|
||
if (cleanUrl.endsWith(".jpg") || cleanUrl.endsWith(".jpeg")) return "image/jpeg";
|
||
if (cleanUrl.endsWith(".webp")) return "image/webp";
|
||
if (cleanUrl.endsWith(".gif")) return "image/gif";
|
||
if (cleanUrl.endsWith(".bmp")) return "image/bmp";
|
||
if (cleanUrl.endsWith(".svg")) return "image/svg+xml";
|
||
return "image/png";
|
||
}
|
||
|
||
function injectHistoryImagesToLastUserMessage(
|
||
messages: ChatMessage[],
|
||
historyImages: string[]
|
||
): ChatMessage[] {
|
||
if (!historyImages || historyImages.length === 0) return messages;
|
||
|
||
let lastUserIndex = -1;
|
||
for (let i = messages.length - 1; i >= 0; i--) {
|
||
if (messages[i]?.role === "user") {
|
||
lastUserIndex = i;
|
||
break;
|
||
}
|
||
}
|
||
|
||
if (lastUserIndex < 0) return messages;
|
||
|
||
const lastUserMessage = messages[lastUserIndex];
|
||
const hasExistingImage = lastUserMessage.parts?.some(
|
||
(p: any) => p.type === "file" || p.type === "image" || p.type === "image_url"
|
||
);
|
||
if (hasExistingImage) return messages;
|
||
|
||
const imageParts = historyImages.map((url, index) => ({
|
||
type: "file" as const,
|
||
url,
|
||
name: `history-image-${index + 1}`,
|
||
mediaType: guessImageMediaType(url),
|
||
mimeType: guessImageMediaType(url),
|
||
}));
|
||
|
||
const updatedMessage: ChatMessage = {
|
||
...lastUserMessage,
|
||
parts: [...lastUserMessage.parts, ...imageParts],
|
||
};
|
||
|
||
return messages.map((msg, idx) => (idx === lastUserIndex ? updatedMessage : msg));
|
||
}
|
||
|
||
function compactAutoDemand(text: string, maxLength = 56): string {
|
||
const normalized = text.replace(/\s+/g, " ").trim();
|
||
if (!normalized) return "按你刚才的描述执行";
|
||
if (normalized.length <= maxLength) return normalized;
|
||
return `${normalized.slice(0, maxLength)}...`;
|
||
}
|
||
|
||
function buildAutoModeFallbackMessage(params: {
|
||
text: string;
|
||
hasImage: boolean;
|
||
fromHistory: boolean;
|
||
}): string {
|
||
const demand = compactAutoDemand(params.text);
|
||
|
||
if (params.hasImage) {
|
||
const sourceLine = params.fromHistory
|
||
? "我会基于你上一轮图片继续处理。"
|
||
: "我会基于你当前上传图片处理。";
|
||
|
||
return [
|
||
"收到，我先帮你梳理这次修改目标。",
|
||
`- 需求理解：${demand}`,
|
||
`- 执行基准：${sourceLine}`,
|
||
"- 执行方式：优先只改你指定区域，主体、构图和风格尽量保持一致。",
|
||
"- 继续迭代：生成后你可直接说“保留A，改B”，我会按最新要求继续优化。",
|
||
].join("\n");
|
||
}
|
||
|
||
return [
|
||
"收到，我来按自动模式帮你生成。",
|
||
`- 需求理解：${demand}`,
|
||
"- 执行方式：先锁定主体和场景，再细化风格、材质与光影。",
|
||
"- 继续迭代：你可以继续补充偏好（如构图、镜头、色调、质感）。",
|
||
].join("\n");
|
||
}
|
||
|
||
/**
|
||
* 自定义消息转换函数
|
||
* 注意：gemini-image 和 huanke-gemini 都需要 URL 格式（不是 Base64）
|
||
* 因为这些 provider 内部会自行处理图片上传到 S3
|
||
*/
|
||
async function convertMessagesWithImages(messages: ChatMessage[], providerType: string) {
|
||
if (providerType === "ruxa-image" || providerType === "fal-image") {
|
||
const convertedMessages = [];
|
||
|
||
for (const message of messages) {
|
||
const content: any[] = [];
|
||
|
||
for (const part of message.parts) {
|
||
const partAny = part as any;
|
||
|
||
if (partAny.type === "text" && partAny.text) {
|
||
content.push({
|
||
type: "text",
|
||
text: partAny.text,
|
||
});
|
||
continue;
|
||
}
|
||
|
||
const candidateUrl =
|
||
partAny.type === "file"
|
||
? (partAny.url || partAny.data)
|
||
: partAny.type === "image"
|
||
? (partAny.image || partAny.url)
|
||
: partAny.type === "image_url"
|
||
? partAny.image_url?.url
|
||
: undefined;
|
||
|
||
if (!candidateUrl || typeof candidateUrl !== "string") continue;
|
||
if (candidateUrl.startsWith("data:")) {
|
||
throw new Error("当前模型不支持 Base64 图片传输，请先上传图片后再编辑");
|
||
}
|
||
if (!candidateUrl.startsWith("http")) continue;
|
||
|
||
content.push({
|
||
type: "image_url",
|
||
image_url: {
|
||
url: candidateUrl,
|
||
},
|
||
});
|
||
}
|
||
|
||
convertedMessages.push({
|
||
role: message.role,
|
||
content: content.length === 1 && content[0].type === "text" ? content[0].text : content,
|
||
});
|
||
}
|
||
|
||
return convertedMessages;
|
||
}
|
||
|
||
// 所有 provider 都使用 URL 格式传输图片
|
||
// gemini-image 和 huanke-gemini 内部会将 Base64 转换为 S3 URL
|
||
// 这里不再进行 Base64 转换，直接返回标准格式
|
||
if (providerType !== "gemini") {
|
||
return convertToModelMessages(messages);
|
||
}
|
||
|
||
// 🆕 检测是否有标注图片
|
||
const hasAnnotation = hasAnnotatedImage(messages);
|
||
if (hasAnnotation) {
|
||
console.log("[Message Convert] 🎨 检测到标注图片，将自动添加系统提示");
|
||
}
|
||
|
||
// 对于 Gemini，需要特殊处理图片
|
||
const convertedMessages = [];
|
||
|
||
for (const message of messages) {
|
||
const content: any[] = [];
|
||
|
||
// 🆕 如果是用户消息且有标注图片，在文本前添加系统提示
|
||
let hasAddedAnnotationPrompt = false;
|
||
|
||
// 处理消息的各个部分
|
||
for (const part of message.parts) {
|
||
if (part.type === "text" && part.text) {
|
||
let textContent = part.text;
|
||
|
||
// 🆕 如果有标注图片且是用户消息，在用户文本前添加系统提示
|
||
if (hasAnnotation && message.role === "user" && !hasAddedAnnotationPrompt) {
|
||
textContent = ANNOTATION_SYSTEM_PROMPT + "\n\n用户请求：" + textContent;
|
||
hasAddedAnnotationPrompt = true;
|
||
console.log("[Message Convert] 🎨 已注入标注提示词");
|
||
}
|
||
|
||
content.push({
|
||
type: "text",
|
||
text: textContent,
|
||
});
|
||
} else if (part.type === "file" && part.url) {
|
||
// 直接使用 URL 格式（不再转换为 Base64）
|
||
// gemini-image 和 huanke-gemini provider 内部会处理图片上传
|
||
content.push({
|
||
type: "image_url",
|
||
image_url: {
|
||
url: part.url,
|
||
},
|
||
});
|
||
console.log("[Message Convert] Image kept as URL format:", part.url.substring(0, 80) + "...");
|
||
}
|
||
}
|
||
|
||
// 🆕 如果有标注图片但消息中没有文本部分，添加一个纯提示文本
|
||
if (hasAnnotation && message.role === "user" && !hasAddedAnnotationPrompt && content.length > 0) {
|
||
content.unshift({
|
||
type: "text",
|
||
text: ANNOTATION_SYSTEM_PROMPT,
|
||
});
|
||
console.log("[Message Convert] 🎨 已在消息开头注入标注提示词");
|
||
}
|
||
|
||
convertedMessages.push({
|
||
role: message.role,
|
||
content: content.length === 1 && content[0].type === "text"
|
||
? content[0].text
|
||
: content,
|
||
});
|
||
}
|
||
|
||
return convertedMessages;
|
||
}
|
||
|
||
function parseImageAltMetadata(altText: string, thumbnailUrl: string): {
|
||
originalUrl: string;
|
||
imageSize: string;
|
||
} {
|
||
let originalUrl = thumbnailUrl;
|
||
let imageSize = "2K";
|
||
|
||
// 格式1: thumb|||originalUrl|||size (新格式)
|
||
if (altText.includes("thumb|||")) {
|
||
const content = altText.substring(altText.indexOf("thumb|||") + 8);
|
||
const parts = content.split("|||");
|
||
originalUrl = parts[0] || thumbnailUrl;
|
||
imageSize = parts[1] || "2K";
|
||
}
|
||
// 格式2: thumb$$originalUrl$$size (中间格式，可能被 LaTeX 解析器破坏)
|
||
else if (altText.includes("thumb$$")) {
|
||
const content = altText.substring(altText.indexOf("thumb$$") + 7);
|
||
const parts = content.split("$$");
|
||
originalUrl = parts[0] || thumbnailUrl;
|
||
imageSize = parts[1] || "2K";
|
||
}
|
||
// 格式3: thumb|originalUrl|size (旧格式)
|
||
else if (altText.includes("thumb|")) {
|
||
const content = altText.substring(altText.indexOf("thumb|") + 6);
|
||
const parts = content.split("|");
|
||
originalUrl = parts[0] || thumbnailUrl;
|
||
imageSize = parts[1] || "2K";
|
||
}
|
||
|
||
// 回退策略：从缩略图 URL 推断原图 URL
|
||
if (originalUrl === thumbnailUrl) {
|
||
if (thumbnailUrl.includes("gemini-thumb-")) {
|
||
originalUrl = thumbnailUrl.replace("gemini-thumb-", "gemini-orig-");
|
||
} else if (thumbnailUrl.includes("openai-thumb-")) {
|
||
originalUrl = thumbnailUrl.replace("openai-thumb-", "openai-orig-");
|
||
} else if (thumbnailUrl.includes("volcano-thumb-")) {
|
||
originalUrl = thumbnailUrl.replace("volcano-thumb-", "volcano-orig-");
|
||
} else if (thumbnailUrl.includes("alibaba-image-thumb-")) {
|
||
originalUrl = thumbnailUrl.replace("alibaba-image-thumb-", "alibaba-image-orig-");
|
||
} else if (thumbnailUrl.includes("ruxa-image-thumb-")) {
|
||
originalUrl = thumbnailUrl.replace("ruxa-image-thumb-", "ruxa-image-orig-");
|
||
} else if (thumbnailUrl.includes("fal-image-thumb-")) {
|
||
originalUrl = thumbnailUrl.replace("fal-image-thumb-", "fal-image-orig-");
|
||
}
|
||
}
|
||
|
||
return { originalUrl, imageSize };
|
||
}
|
||
|
||
function extractOriginalImageUrlsFromText(text: string): string[] {
|
||
return extractImageUrlsFromText(text);
|
||
}
|
||
|
||
function mimeTypeToExtension(mimeType: string) {
|
||
const normalized = mimeType.toLowerCase();
|
||
if (normalized.includes("png")) return "png";
|
||
if (normalized.includes("webp")) return "webp";
|
||
if (normalized.includes("gif")) return "gif";
|
||
if (normalized.includes("bmp")) return "bmp";
|
||
return "jpg";
|
||
}
|
||
|
||
function getExtensionFromUrl(url: string): string {
|
||
try {
|
||
const pathname = new URL(url).pathname;
|
||
const ext = pathname.split(".").pop()?.toLowerCase();
|
||
if (!ext) return "jpg";
|
||
if (["jpg", "jpeg", "png", "webp", "gif", "bmp"].includes(ext)) return ext;
|
||
} catch {
|
||
// ignore
|
||
}
|
||
return "jpg";
|
||
}
|
||
|
||
async function uploadRemoteImageToStorage(imageUrl: string): Promise<string> {
|
||
const response = await fetch(imageUrl, {
|
||
method: "GET",
|
||
signal: AbortSignal.timeout(30_000),
|
||
});
|
||
|
||
if (!response.ok) {
|
||
throw new Error(`Remote image fetch failed: ${response.status} ${response.statusText}`);
|
||
}
|
||
|
||
const contentType = response.headers.get("content-type")?.toLowerCase() || "";
|
||
if (contentType && !contentType.startsWith("image/")) {
|
||
throw new Error(`Remote url is not an image: ${contentType}`);
|
||
}
|
||
|
||
const arrayBuffer = await response.arrayBuffer();
|
||
if (!arrayBuffer || arrayBuffer.byteLength === 0) {
|
||
throw new Error("Remote image content is empty");
|
||
}
|
||
|
||
const ext = contentType ? mimeTypeToExtension(contentType) : getExtensionFromUrl(imageUrl);
|
||
const fileName = `remote-image-${Date.now()}-${Math.random().toString(36).slice(2, 8)}.${ext}`;
|
||
const uploaded = await uploadFile(fileName, arrayBuffer);
|
||
return uploaded.url;
|
||
}
|
||
|
||
async function isRemoteImageAccessible(imageUrl: string): Promise<boolean> {
|
||
try {
|
||
const headResponse = await fetch(imageUrl, {
|
||
method: "HEAD",
|
||
signal: AbortSignal.timeout(10_000),
|
||
});
|
||
|
||
if (headResponse.ok) {
|
||
return true;
|
||
}
|
||
|
||
if (headResponse.status !== 405 && headResponse.status !== 403) {
|
||
return false;
|
||
}
|
||
|
||
const probeResponse = await fetch(imageUrl, {
|
||
method: "GET",
|
||
headers: {
|
||
Range: "bytes=0-0",
|
||
},
|
||
signal: AbortSignal.timeout(10_000),
|
||
});
|
||
return probeResponse.ok;
|
||
} catch {
|
||
return false;
|
||
}
|
||
}
|
||
|
||
async function ensureHostedImageUrl(imageUrl: string): Promise<string> {
|
||
const normalizedUrl = imageUrl.trim();
|
||
if (!normalizedUrl) {
|
||
throw new Error("Image URL is empty");
|
||
}
|
||
|
||
if (normalizedUrl.startsWith("data:image")) {
|
||
return uploadDataUrlImage(normalizedUrl);
|
||
}
|
||
|
||
if (!/^https?:\/\//i.test(normalizedUrl)) {
|
||
throw new Error(`Unsupported image URL: ${normalizedUrl}`);
|
||
}
|
||
|
||
if (/\/uploads\//i.test(normalizedUrl)) {
|
||
const accessible = await isRemoteImageAccessible(normalizedUrl);
|
||
if (accessible) {
|
||
return normalizedUrl;
|
||
}
|
||
}
|
||
|
||
return uploadRemoteImageToStorage(normalizedUrl);
|
||
}
|
||
|
||
async function uploadDataUrlImage(dataUrl: string): Promise<string> {
|
||
const match = dataUrl.match(/^data:([^;]+);base64,([\s\S]+)$/i);
|
||
if (!match) {
|
||
throw new Error("Invalid data URL image format");
|
||
}
|
||
|
||
const mimeType = match[1] || "image/jpeg";
|
||
const base64Data = (match[2] || "").replace(/\s+/g, "");
|
||
if (!base64Data) {
|
||
throw new Error("Empty base64 image content");
|
||
}
|
||
|
||
const buffer = Buffer.from(base64Data, "base64");
|
||
const fileExt = mimeTypeToExtension(mimeType);
|
||
const fileName = `inline-image-${Date.now()}-${Math.random().toString(36).slice(2, 8)}.${fileExt}`;
|
||
const arrayBuffer = buffer.buffer.slice(
|
||
buffer.byteOffset,
|
||
buffer.byteOffset + buffer.byteLength
|
||
) as ArrayBuffer;
|
||
const uploaded = await uploadFile(fileName, arrayBuffer);
|
||
return uploaded.url;
|
||
}
|
||
|
||
async function normalizeAssistantImageMarkdown(
|
||
text: string,
|
||
imageSize = "2K"
|
||
): Promise<string> {
|
||
if (!text) return text;
|
||
|
||
const markdownDataUrlRegex = /!\[([^\]]*)\]\s*\(\s*(data:image\/[a-z0-9+.-]+;base64,[^)]+)\s*\)/gim;
|
||
const markdownMatches = [...text.matchAll(markdownDataUrlRegex)];
|
||
let normalizedText = text;
|
||
const uploadedUrls: string[] = [];
|
||
|
||
if (markdownMatches.length > 0) {
|
||
let rebuilt = "";
|
||
let cursor = 0;
|
||
|
||
for (const match of markdownMatches) {
|
||
const matchText = match[0];
|
||
const dataUrl = match[2];
|
||
const start = match.index ?? -1;
|
||
if (start < 0) continue;
|
||
|
||
rebuilt += normalizedText.slice(cursor, start);
|
||
|
||
try {
|
||
const uploadedUrl = await uploadDataUrlImage(dataUrl);
|
||
uploadedUrls.push(uploadedUrl);
|
||
rebuilt += ``;
|
||
} catch (error) {
|
||
console.error("[Chat API] Failed to upload inline data URL image:", error);
|
||
rebuilt += matchText;
|
||
}
|
||
|
||
cursor = start + matchText.length;
|
||
}
|
||
|
||
rebuilt += normalizedText.slice(cursor);
|
||
normalizedText = rebuilt;
|
||
}
|
||
|
||
if (normalizedText.includes("data:image")) {
|
||
const rawDataUrlRegex = /data:image\/[a-z0-9+.-]+;base64,[a-z0-9+/=\s]+/gim;
|
||
const rawMatches = normalizedText.match(rawDataUrlRegex) ?? [];
|
||
|
||
for (const rawDataUrl of rawMatches) {
|
||
try {
|
||
const uploadedUrl = await uploadDataUrlImage(rawDataUrl);
|
||
uploadedUrls.push(uploadedUrl);
|
||
normalizedText = normalizedText.replace(rawDataUrl, uploadedUrl);
|
||
} catch (error) {
|
||
console.error("[Chat API] Failed to normalize raw data URL image:", error);
|
||
}
|
||
}
|
||
}
|
||
|
||
const markdownHttpImageRegex = /!\[([^\]]*)\]\s*\(\s*(https?:\/\/[^)\s]+)\s*\)/gim;
|
||
const markdownHttpMatches = [...normalizedText.matchAll(markdownHttpImageRegex)];
|
||
|
||
if (markdownHttpMatches.length > 0) {
|
||
let rebuilt = "";
|
||
let cursor = 0;
|
||
|
||
for (const match of markdownHttpMatches) {
|
||
const matchText = match[0];
|
||
const altText = match[1] || "";
|
||
const thumbnailUrl = (match[2] || "").trim();
|
||
const start = match.index ?? -1;
|
||
if (start < 0) continue;
|
||
|
||
rebuilt += normalizedText.slice(cursor, start);
|
||
|
||
try {
|
||
const { originalUrl, imageSize: parsedImageSize } = parseImageAltMetadata(
|
||
altText,
|
||
thumbnailUrl
|
||
);
|
||
|
||
let hostedThumbnailUrl = thumbnailUrl;
|
||
let hostedOriginalUrl =
|
||
originalUrl && originalUrl !== thumbnailUrl ? originalUrl : thumbnailUrl;
|
||
|
||
try {
|
||
hostedThumbnailUrl = await ensureHostedImageUrl(thumbnailUrl);
|
||
} catch (error) {
|
||
console.error("[Chat API] Failed to normalize remote thumbnail URL:", error);
|
||
}
|
||
|
||
if (originalUrl && originalUrl !== thumbnailUrl) {
|
||
try {
|
||
hostedOriginalUrl = await ensureHostedImageUrl(originalUrl);
|
||
} catch (error) {
|
||
console.error("[Chat API] Failed to normalize remote original URL:", error);
|
||
hostedOriginalUrl = hostedThumbnailUrl || originalUrl;
|
||
}
|
||
} else {
|
||
hostedOriginalUrl = hostedThumbnailUrl;
|
||
}
|
||
|
||
const finalThumbnailUrl = hostedThumbnailUrl || thumbnailUrl;
|
||
const finalOriginalUrl = hostedOriginalUrl || finalThumbnailUrl;
|
||
|
||
uploadedUrls.push(finalThumbnailUrl);
|
||
uploadedUrls.push(finalOriginalUrl);
|
||
rebuilt += ``;
|
||
} catch (error) {
|
||
console.error("[Chat API] Failed to parse remote image markdown metadata:", error);
|
||
rebuilt += matchText;
|
||
}
|
||
|
||
cursor = start + matchText.length;
|
||
}
|
||
|
||
rebuilt += normalizedText.slice(cursor);
|
||
normalizedText = rebuilt;
|
||
}
|
||
|
||
const hasMarkdownImage = /!\[[^\]]*\]\((https?:\/\/[^)]+)\)/i.test(normalizedText);
|
||
if (!hasMarkdownImage && uploadedUrls.length > 0) {
|
||
const uniqueUrls = [...new Set(uploadedUrls)];
|
||
console.log(
|
||
`[Chat API] Converted ${uniqueUrls.length} image link(s) to hosted link(s)`
|
||
);
|
||
return uniqueUrls
|
||
.map((url) => ``)
|
||
.join("\n");
|
||
}
|
||
|
||
return normalizedText;
|
||
}
|
||
|
||
/**
|
||
* 解析生图 Markdown 并保存图片记录到数据库
|
||
* 支持三种格式：thumb|||url|||size、thumb$$url$$size、thumb|url|size
|
||
*/
|
||
async function saveGeneratedImages(
|
||
imageMarkdown: string,
|
||
userId: string,
|
||
chatId: string,
|
||
messageId: string,
|
||
prompt: string,
|
||
modelId: string
|
||
): Promise<void> {
|
||
// 匹配 Markdown 图片格式: 
|
||
const imageRegex = /!\[([^\]]*)\]\(([^)]+)\)/g;
|
||
const matches = [...imageMarkdown.matchAll(imageRegex)];
|
||
|
||
let savedImageCount = 0;
|
||
|
||
for (const match of matches) {
|
||
const altText = match[1] || "";
|
||
const thumbnailUrl = match[2];
|
||
const { originalUrl, imageSize } = parseImageAltMetadata(altText, thumbnailUrl);
|
||
|
||
try {
|
||
await db.insert(generatedImage).values({
|
||
userId,
|
||
chatId,
|
||
messageId,
|
||
thumbnailUrl,
|
||
originalUrl,
|
||
imageSize,
|
||
prompt,
|
||
modelId,
|
||
});
|
||
savedImageCount += 1;
|
||
console.log(`[Gallery] Saved image: ${originalUrl.substring(0, 80)}...`);
|
||
} catch (error) {
|
||
console.error("[Gallery] Failed to save image record:", error);
|
||
}
|
||
}
|
||
|
||
if (savedImageCount <= 0) return;
|
||
|
||
try {
|
||
const pointsConfig = await getPointsSystemConfig();
|
||
if (!pointsConfig.enabled) return;
|
||
|
||
const isUuidModelId = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test(
|
||
modelId
|
||
);
|
||
if (!isUuidModelId) return;
|
||
|
||
const modelConfig = await getAiModelById({ id: modelId });
|
||
const rawUnitCost = Number((modelConfig as any)?.imageGenConfig?.creditCostPerImage ?? 0);
|
||
const unitCost = Number.isFinite(rawUnitCost) ? Math.max(0, Math.round(rawUnitCost)) : 0;
|
||
if (unitCost <= 0) return;
|
||
|
||
const totalCost = unitCost * savedImageCount;
|
||
const consumeResult = await consumeUserPoints({
|
||
userId,
|
||
amount: totalCost,
|
||
reason: "image_generation",
|
||
modelId,
|
||
chatId,
|
||
messageId,
|
||
note: `模型(${modelConfig?.displayName || modelId}) 生图 ${savedImageCount} 张`,
|
||
});
|
||
|
||
if (!consumeResult.ok) {
|
||
console.warn(
|
||
`[Points] Failed to consume points after generation. user=${userId} model=${modelId} cost=${totalCost}`
|
||
);
|
||
} else {
|
||
console.log(
|
||
`[Points] Consumed ${totalCost} points (${unitCost} x ${savedImageCount}). balance=${consumeResult.balance}`
|
||
);
|
||
}
|
||
} catch (error) {
|
||
console.error("[Points] Failed to consume points after image generation:", error);
|
||
}
|
||
}
|
||
|
||
export const maxDuration = 600; // 10 分钟，用于支持图片编辑等耗时任务
|
||
|
||
let globalStreamContext: ResumableStreamContext | null = null;
|
||
|
||
const getTokenlensCatalog = cache(
|
||
async (): Promise<ModelCatalog | undefined> => {
|
||
try {
|
||
return await fetchModels();
|
||
} catch (err) {
|
||
console.warn(
|
||
"TokenLens: catalog fetch failed, using default catalog",
|
||
err
|
||
);
|
||
return; // tokenlens helpers will fall back to defaultCatalog
|
||
}
|
||
},
|
||
["tokenlens-catalog"],
|
||
{ revalidate: 24 * 60 * 60 } // 24 hours
|
||
);
|
||
|
||
export function getStreamContext() {
|
||
if (!globalStreamContext) {
|
||
try {
|
||
globalStreamContext = createResumableStreamContext({
|
||
waitUntil: after,
|
||
});
|
||
} catch (error: any) {
|
||
if (error.message.includes("REDIS_URL")) {
|
||
console.log(
|
||
" > Resumable streams are disabled due to missing REDIS_URL"
|
||
);
|
||
} else {
|
||
console.error(error);
|
||
}
|
||
}
|
||
}
|
||
|
||
return globalStreamContext;
|
||
}
|
||
|
||
export async function POST(request: Request) {
|
||
let requestBody: PostRequestBody;
|
||
|
||
try {
|
||
const json = await request.json();
|
||
console.log("[Chat API] Received request body:", JSON.stringify(json, null, 2));
|
||
requestBody = postRequestBodySchema.parse(json);
|
||
} catch (error) {
|
||
console.error("[Chat API] Schema validation failed:", error);
|
||
return new ChatSDKError("bad_request:api").toResponse();
|
||
}
|
||
|
||
try {
|
||
const {
|
||
id,
|
||
message,
|
||
selectedChatModel,
|
||
selectedVisibilityType,
|
||
hiddenPrompt,
|
||
autoContext,
|
||
}: {
|
||
id: string;
|
||
message: ChatMessage;
|
||
selectedChatModel: ChatModel["id"];
|
||
selectedVisibilityType: VisibilityType;
|
||
hiddenPrompt?: string;
|
||
autoContext?: string;
|
||
} = requestBody;
|
||
|
||
let finalModelId = selectedChatModel;
|
||
|
||
// 🆕 存储智能模式的相关信息
|
||
let smartAutoFriendlyMessage: string | undefined; // AI的自然对话回复
|
||
let smartAutoOptimizedPrompt: string | undefined; // 优化后的提示词（不修改用户消息，只用于API调用）
|
||
let smartAutoRecommendedImageParams: AgentRecommendedImageParams | undefined;
|
||
let smartAutoVideoPlan: AgentVideoPlan | undefined;
|
||
let smartAutoRecommendedVideoParams: AgentRecommendedVideoParams | undefined;
|
||
let smartAutoDetectedIntent: "image_edit" | "vision" | "image_gen" | "video_gen" | "chat" | undefined;
|
||
let smartAutoHistoryImages: string[] = []; // 从历史对话中提取的图片URL（支持多张）
|
||
let smartAutoNeedsClarification = false; // 是否需要先追问用户
|
||
let smartAutoClarificationQuestion: string | undefined;
|
||
let shouldRunAgentConversation = finalModelId === "auto";
|
||
let shouldKeepSelectedModel = false;
|
||
const requestedModelId = selectedChatModel;
|
||
|
||
if (!shouldRunAgentConversation && requestedModelId !== "auto") {
|
||
const selectedModelIsUUID =
|
||
/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test(requestedModelId);
|
||
let selectedModelConfig = selectedModelIsUUID
|
||
? await getAiModelById({ id: requestedModelId })
|
||
: null;
|
||
let selectedModelProviderType: string | undefined;
|
||
|
||
if (!selectedModelConfig) {
|
||
const enabledModels = await getEnabledAiModelsWithProvider(false);
|
||
const matchedModel = enabledModels.find(
|
||
({ model }) => model.id === requestedModelId || model.modelId === requestedModelId
|
||
);
|
||
if (matchedModel) {
|
||
selectedModelConfig = matchedModel.model;
|
||
selectedModelProviderType = matchedModel.provider.type as string;
|
||
}
|
||
}
|
||
|
||
if (selectedModelConfig && !selectedModelProviderType) {
|
||
const selectedProvider = await getAiProviderById({
|
||
id: selectedModelConfig.providerId,
|
||
});
|
||
selectedModelProviderType = selectedProvider?.type as string | undefined;
|
||
}
|
||
|
||
if (selectedModelConfig) {
|
||
const fixedImageProviderTypes = new Set([
|
||
"gemini",
|
||
"huanke-gemini",
|
||
"alibaba-image",
|
||
"ruxa-image",
|
||
"fal-image",
|
||
"volcano-vision",
|
||
]);
|
||
const fixedVideoProviderTypes = new Set([
|
||
"volcano-video",
|
||
"qianwen-video",
|
||
"fal-video",
|
||
]);
|
||
const isFixedVisualModel =
|
||
!selectedModelConfig.isSubModel &&
|
||
(selectedModelConfig.supportsImageGen ||
|
||
(selectedModelProviderType
|
||
? fixedVideoProviderTypes.has(selectedModelProviderType)
|
||
: false) ||
|
||
(selectedModelProviderType
|
||
? fixedImageProviderTypes.has(selectedModelProviderType)
|
||
: false));
|
||
|
||
if (isAiModelAgentAuto(selectedModelConfig)) {
|
||
shouldRunAgentConversation = true;
|
||
shouldKeepSelectedModel = false;
|
||
console.log(
|
||
`[Smart Auto Mode] 🧠 Agent-only model detected, force auto routing: ${selectedModelConfig.displayName}`
|
||
);
|
||
} else if (isFixedVisualModel) {
|
||
shouldRunAgentConversation = true;
|
||
shouldKeepSelectedModel = true;
|
||
console.log(
|
||
`[Smart Auto Mode] 🎯 Agent conversation enabled for fixed visual model: ${selectedModelConfig.displayName} (${selectedModelProviderType || "unknown"})`
|
||
);
|
||
}
|
||
}
|
||
}
|
||
|
||
// ========== Smart Auto Mode Logic ==========
|
||
if (shouldRunAgentConversation) {
|
||
console.log("[Smart Auto Mode] 🚀 Starting intelligent analysis...");
|
||
let autoTextForFallback = "";
|
||
let autoHasImageForFallback = false;
|
||
let autoFromHistoryForFallback = false;
|
||
|
||
try {
|
||
// 🆕 首先获取历史消息（在 Smart Auto Mode 内部单独获取，避免依赖后面的 messagesFromDb）
|
||
const smartAutoHistoryMessagesFromDb = await getMessagesByChatId({ id });
|
||
console.log("[Smart Auto Mode] Loaded", smartAutoHistoryMessagesFromDb.length, "messages from DB");
|
||
|
||
// 1. 提取用户消息内容
|
||
const lastUserMessage = message;
|
||
const textContent = lastUserMessage.parts
|
||
.filter((p): p is { type: "text"; text: string } => p.type === "text")
|
||
.map((p) => p.text)
|
||
.join(" ");
|
||
autoTextForFallback = textContent;
|
||
const autoContextText = typeof autoContext === "string" ? autoContext.trim() : "";
|
||
const hasSelectedImageHint = /用户已选中\s*\d+\s*张图片/.test(autoContextText);
|
||
const hasExplicitTargetHint = /第[一二三四五六七八九十\d]+张|左|右|上|下|中间|选中/.test(textContent);
|
||
const hasActionableEditIntent = /改|调整|优化|换|替换|去掉|抠|重绘|增强|修|处理|按照|按|保留|统一|色差|背景|清晰|风格/.test(
|
||
textContent
|
||
);
|
||
|
||
// @ts-ignore - 检查各种可能的图片类型
|
||
let hasImage = lastUserMessage.parts.some(
|
||
(p: any) => p.type === "file" || p.type === "image" || p.type === "image_url"
|
||
);
|
||
autoHasImageForFallback = hasImage;
|
||
|
||
// @ts-ignore - 提取图片URL
|
||
let imageUrls = lastUserMessage.parts
|
||
.filter((p: any) => p.type === "file" && p.url)
|
||
.map((p: any) => p.url as string);
|
||
|
||
// 🆕 历史图片上下文感知：如果用户没有发送新图片，自动使用历史中最近的图片（支持多张）
|
||
// 逻辑：只要用户在对话中没有附上新图片，就默认继续处理之前的图片
|
||
if (!hasImage && smartAutoHistoryMessagesFromDb.length > 0) {
|
||
console.log("[Smart Auto Mode] 🔍 No image in current message, auto-searching history...");
|
||
|
||
const explicitFollowLatestResult = /(上一张|上一轮|刚生成|最新生成|这个结果|该结果|按这个结果|继续在这张|基于这张结果|延续上一次结果)/i.test(
|
||
textContent
|
||
);
|
||
|
||
const latestAssistantImages: string[] = [];
|
||
const latestUserImages: string[] = [];
|
||
const pushUniqueUrl = (bucket: string[], maybeUrl?: string) => {
|
||
if (!maybeUrl || typeof maybeUrl !== "string") return;
|
||
const normalized = maybeUrl.trim();
|
||
if (!/^https?:\/\//i.test(normalized)) return;
|
||
if (!bucket.includes(normalized)) {
|
||
bucket.push(normalized);
|
||
}
|
||
};
|
||
|
||
// 从历史消息中倒序查找最近的“AI生成图”和“用户上传图”
|
||
for (let i = smartAutoHistoryMessagesFromDb.length - 1; i >= 0; i--) {
|
||
const msg = smartAutoHistoryMessagesFromDb[i];
|
||
const parts = msg.parts as any[];
|
||
|
||
if (!parts) continue;
|
||
|
||
if (msg.role === "assistant" && latestAssistantImages.length === 0) {
|
||
for (const part of parts) {
|
||
if (part.type === "text" && part.text) {
|
||
const parsedImageUrls = extractOriginalImageUrlsFromText(part.text);
|
||
for (const parsedUrl of parsedImageUrls) {
|
||
pushUniqueUrl(latestAssistantImages, parsedUrl);
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
if (msg.role === "user" && latestUserImages.length === 0) {
|
||
for (const part of parts) {
|
||
if ((part.type === "file" || part.type === "image") && part.url) {
|
||
pushUniqueUrl(latestUserImages, part.url);
|
||
}
|
||
if (part.type === "image_url" && part.image_url?.url) {
|
||
pushUniqueUrl(latestUserImages, part.image_url.url);
|
||
}
|
||
if (part.type === "text" && part.text) {
|
||
const parsedImageUrls = extractOriginalImageUrlsFromText(part.text);
|
||
for (const parsedUrl of parsedImageUrls) {
|
||
pushUniqueUrl(latestUserImages, parsedUrl);
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
if (latestAssistantImages.length > 0 && latestUserImages.length > 0) {
|
||
break;
|
||
}
|
||
}
|
||
|
||
if (explicitFollowLatestResult && latestAssistantImages.length > 0) {
|
||
smartAutoHistoryImages = latestAssistantImages.slice(0, 4);
|
||
console.log(
|
||
"[Smart Auto Mode] 🖼️ Using latest generated image(s) by explicit instruction:",
|
||
smartAutoHistoryImages.length
|
||
);
|
||
} else if (latestUserImages.length > 0) {
|
||
smartAutoHistoryImages = latestUserImages.slice(0, 4);
|
||
console.log(
|
||
"[Smart Auto Mode] 🖼️ Using latest user-uploaded image(s) to avoid iterative blur:",
|
||
smartAutoHistoryImages.length
|
||
);
|
||
} else if (latestAssistantImages.length > 0) {
|
||
smartAutoHistoryImages = latestAssistantImages.slice(0, 4);
|
||
console.log(
|
||
"[Smart Auto Mode] 🖼️ Fallback to latest generated image(s):",
|
||
smartAutoHistoryImages.length
|
||
);
|
||
}
|
||
|
||
if (smartAutoHistoryImages.length > 0) {
|
||
hasImage = true;
|
||
imageUrls = smartAutoHistoryImages;
|
||
autoHasImageForFallback = true;
|
||
autoFromHistoryForFallback = true;
|
||
console.log("[Smart Auto Mode] ✅ Auto-using", smartAutoHistoryImages.length, "history image(s) for context continuity");
|
||
} else {
|
||
console.log("[Smart Auto Mode] ℹ️ No image found in history, proceeding as pure text");
|
||
}
|
||
}
|
||
|
||
console.log("[Smart Auto Mode] Input:", { textContent: textContent.substring(0, 100), hasImage, imageCount: imageUrls.length, fromHistory: smartAutoHistoryImages.length > 0 });
|
||
|
||
// 🆕 从历史消息中提取对话文本（用于上下文理解）
|
||
const historyMessages: { role: "user" | "assistant"; content: string }[] = [];
|
||
for (const msg of smartAutoHistoryMessagesFromDb.slice(-10)) { // 最多取最近10条
|
||
const parts = msg.parts as any[];
|
||
if (parts && parts.length > 0) {
|
||
// 提取文本内容
|
||
const textParts = parts.filter((p: any) => p.type === 'text');
|
||
const content = textParts.map((p: any) => p.text || '').join(' ').trim();
|
||
if (content) {
|
||
historyMessages.push({
|
||
role: msg.role as "user" | "assistant",
|
||
content: content.substring(0, 300), // 限制长度
|
||
});
|
||
}
|
||
}
|
||
}
|
||
console.log("[Smart Auto Mode] History context:", historyMessages.length, "messages");
|
||
|
||
// 2. 使用 LangGraph 驱动的自动模式工作流
|
||
const { runAutoAgentGraph } = await import("@/lib/ai/auto-agent-graph");
|
||
const autoGraphResult = await runAutoAgentGraph({
|
||
text: textContent,
|
||
hasImage,
|
||
imageUrls,
|
||
imageFromHistory: smartAutoHistoryImages.length > 0,
|
||
historyMessages,
|
||
context: autoContextText || undefined,
|
||
});
|
||
|
||
if (autoGraphResult.analysis) {
|
||
console.log("[Smart Auto Mode] Analysis result:", {
|
||
intent: autoGraphResult.analysis.intent,
|
||
subIntent: autoGraphResult.analysis.subIntent,
|
||
confidence: autoGraphResult.analysis.confidence,
|
||
needsClarification: autoGraphResult.analysis.needsClarification,
|
||
});
|
||
|
||
smartAutoDetectedIntent = autoGraphResult.analysis.intent;
|
||
smartAutoNeedsClarification = Boolean(autoGraphResult.analysis.needsClarification);
|
||
smartAutoClarificationQuestion = autoGraphResult.analysis.clarifyingQuestions?.[0];
|
||
|
||
if (
|
||
smartAutoNeedsClarification &&
|
||
hasImage &&
|
||
hasActionableEditIntent &&
|
||
(hasSelectedImageHint || hasExplicitTargetHint)
|
||
) {
|
||
smartAutoNeedsClarification = false;
|
||
smartAutoClarificationQuestion = undefined;
|
||
if (!smartAutoOptimizedPrompt && textContent.trim()) {
|
||
smartAutoOptimizedPrompt = textContent.trim();
|
||
}
|
||
if (
|
||
!smartAutoFriendlyMessage ||
|
||
/[?？]$/.test(smartAutoFriendlyMessage.trim()) ||
|
||
/哪一张|哪个|请先确认/.test(smartAutoFriendlyMessage)
|
||
) {
|
||
smartAutoFriendlyMessage = "收到，我将按你指定的目标图直接执行。";
|
||
}
|
||
console.log(
|
||
"[Smart Auto Mode] ✅ Skip clarification due to explicit target/selection context"
|
||
);
|
||
}
|
||
}
|
||
|
||
if (autoGraphResult.friendlyMessage) {
|
||
smartAutoFriendlyMessage = autoGraphResult.friendlyMessage;
|
||
console.log("[Smart Auto Mode] 💬 Conversational reply:", smartAutoFriendlyMessage);
|
||
}
|
||
|
||
if (
|
||
!smartAutoNeedsClarification &&
|
||
hasImage &&
|
||
hasActionableEditIntent &&
|
||
(hasSelectedImageHint || hasExplicitTargetHint) &&
|
||
smartAutoFriendlyMessage &&
|
||
(/[?？]$/.test(smartAutoFriendlyMessage.trim()) ||
|
||
/哪一张|哪个|请先确认/.test(smartAutoFriendlyMessage))
|
||
) {
|
||
smartAutoFriendlyMessage = "收到，我将按你指定的目标图直接执行。";
|
||
console.log("[Smart Auto Mode] ✅ Replace clarification-like friendly reply");
|
||
}
|
||
|
||
const autoIntent = autoGraphResult.analysis?.intent;
|
||
const isImageTask =
|
||
autoIntent === "image_gen" ||
|
||
autoIntent === "image_edit" ||
|
||
hasImage;
|
||
const needsRicherAutoReply =
|
||
isImageTask &&
|
||
!smartAutoNeedsClarification &&
|
||
!smartAutoFriendlyMessage;
|
||
|
||
if (needsRicherAutoReply) {
|
||
if (isImageTask) {
|
||
smartAutoFriendlyMessage =
|
||
autoGraphResult.analysis?.conversationalReply ||
|
||
"我先分析你这轮图片里的主体、风格和构图，再给你执行方案。";
|
||
} else {
|
||
smartAutoFriendlyMessage = buildAutoModeFallbackMessage({
|
||
text: autoTextForFallback,
|
||
hasImage,
|
||
fromHistory: smartAutoHistoryImages.length > 0,
|
||
});
|
||
}
|
||
console.log("[Smart Auto Mode] 🧩 Filled missing conversational reply");
|
||
}
|
||
|
||
if (autoGraphResult.optimizedPrompt) {
|
||
smartAutoOptimizedPrompt = autoGraphResult.optimizedPrompt;
|
||
console.log("[Smart Auto Mode] ✨ Prompt optimized (LLM):", {
|
||
original: textContent.substring(0, 50),
|
||
optimized: autoGraphResult.optimizedPrompt.substring(0, 100),
|
||
});
|
||
}
|
||
if (autoGraphResult.videoPlan) {
|
||
smartAutoVideoPlan = autoGraphResult.videoPlan;
|
||
smartAutoRecommendedVideoParams = autoGraphResult.videoPlan.recommendedVideoParams;
|
||
if (!smartAutoOptimizedPrompt && autoGraphResult.videoPlan.finalPrompt?.trim()) {
|
||
smartAutoOptimizedPrompt = autoGraphResult.videoPlan.finalPrompt.trim();
|
||
}
|
||
console.log("[Smart Auto Mode] 🎬 Agent video plan ready:", {
|
||
hasScript: Boolean(autoGraphResult.videoPlan.script),
|
||
storyboardCount: autoGraphResult.videoPlan.storyboard?.length || 0,
|
||
hasFinalPrompt: Boolean(autoGraphResult.videoPlan.finalPrompt),
|
||
recommendedVideoParams: smartAutoRecommendedVideoParams,
|
||
});
|
||
}
|
||
if (autoGraphResult.recommendedImageParams) {
|
||
smartAutoRecommendedImageParams = autoGraphResult.recommendedImageParams;
|
||
console.log("[Smart Auto Mode] 🎛️ Agent recommended image params:", smartAutoRecommendedImageParams);
|
||
}
|
||
|
||
if (autoGraphResult.route && !shouldKeepSelectedModel) {
|
||
finalModelId = autoGraphResult.route.id;
|
||
console.log(`[Smart Auto Mode] ✅ Routed to: ${autoGraphResult.route.displayName} (${autoGraphResult.route.reason})`);
|
||
} else if (autoGraphResult.route && shouldKeepSelectedModel) {
|
||
finalModelId = requestedModelId;
|
||
console.log("[Smart Auto Mode] 🔒 Keep user-selected model, skip routing override");
|
||
}
|
||
|
||
} catch (autoError) {
|
||
// 智能分析失败时，回退到简单逻辑
|
||
console.error("[Smart Auto Mode] ❌ Error, falling back:", autoError);
|
||
try {
|
||
const { analyzeIntent } = await import("@/lib/ai/smart-analyzer");
|
||
const rescueAnalysis = await analyzeIntent({
|
||
text: autoTextForFallback,
|
||
hasImage: autoHasImageForFallback || smartAutoHistoryImages.length > 0,
|
||
imageUrls: smartAutoHistoryImages,
|
||
imageFromHistory: autoFromHistoryForFallback || smartAutoHistoryImages.length > 0,
|
||
historyMessages: [],
|
||
context: undefined,
|
||
});
|
||
if (rescueAnalysis.conversationalReply?.trim()) {
|
||
smartAutoFriendlyMessage = rescueAnalysis.conversationalReply.trim();
|
||
}
|
||
} catch (rescueError) {
|
||
console.error("[Smart Auto Mode] Rescue analysis failed:", rescueError);
|
||
}
|
||
|
||
const allModels = await getEnabledAiModelsWithProvider();
|
||
// @ts-ignore - 检查各种可能的图片类型
|
||
const hasImage = autoHasImageForFallback || message.parts.some(
|
||
(p: any) => p.type === "file" || p.type === "image" || p.type === "image_url"
|
||
);
|
||
|
||
// 简单回退：有图选图片模型，无图选默认模型
|
||
let bestModel = null;
|
||
if (hasImage) {
|
||
bestModel = allModels.find(m => m.model.supportsImageGen && !m.model.isSubModel);
|
||
smartAutoFriendlyMessage =
|
||
smartAutoFriendlyMessage ||
|
||
"我正在先做这一轮图片分析，接着按你的要求执行修改。";
|
||
} else {
|
||
smartAutoFriendlyMessage = smartAutoFriendlyMessage || buildAutoModeFallbackMessage({
|
||
text: autoTextForFallback,
|
||
hasImage: false,
|
||
fromHistory: false,
|
||
});
|
||
}
|
||
if (!bestModel) {
|
||
bestModel = allModels.find(m => m.model.isDefault && !m.model.isSubModel);
|
||
}
|
||
if (!bestModel && allModels.length > 0) {
|
||
bestModel = allModels.find(m => !m.model.isSubModel) || allModels[0];
|
||
}
|
||
|
||
if (shouldKeepSelectedModel) {
|
||
finalModelId = requestedModelId;
|
||
console.log("[Smart Auto Mode] 🔒 Keep user-selected model in fallback mode");
|
||
} else if (bestModel) {
|
||
finalModelId = bestModel.model.id;
|
||
console.log(`[Smart Auto Mode] Fallback selected: ${bestModel.model.displayName}`);
|
||
} else {
|
||
return new ChatSDKError("bad_request:api", "No available models for auto mode").toResponse();
|
||
}
|
||
}
|
||
}
|
||
// ========== End Smart Auto Mode Logic ==========
|
||
|
||
if (
|
||
shouldRunAgentConversation &&
|
||
!smartAutoNeedsClarification &&
|
||
!smartAutoFriendlyMessage?.trim()
|
||
) {
|
||
const latestUserText = message.parts
|
||
.filter((part): part is { type: "text"; text: string } => part.type === "text")
|
||
.map((part) => part.text)
|
||
.join(" ");
|
||
const hasCurrentImageInput = message.parts.some((part: any) =>
|
||
part?.type === "file" || part?.type === "image" || part?.type === "image_url"
|
||
);
|
||
const hasHistoryImageInput = smartAutoHistoryImages.length > 0;
|
||
smartAutoFriendlyMessage = buildAutoModeFallbackMessage({
|
||
text: latestUserText,
|
||
hasImage: hasCurrentImageInput || hasHistoryImageInput,
|
||
fromHistory: !hasCurrentImageInput && hasHistoryImageInput,
|
||
});
|
||
console.log("[Smart Auto Mode] 🧩 Backfilled conversational reply with fallback");
|
||
}
|
||
|
||
const hasUserCustomImageParams = hasUserCustomizedImageParams(requestBody.imageParams);
|
||
const hasUserCustomVideoParams = hasUserCustomizedVideoParams(requestBody.videoParams);
|
||
const shouldApplyAgentImageParams =
|
||
shouldRunAgentConversation &&
|
||
!smartAutoNeedsClarification &&
|
||
(smartAutoDetectedIntent === "image_gen" || smartAutoDetectedIntent === "image_edit");
|
||
const shouldApplyAgentVideoParams =
|
||
shouldRunAgentConversation &&
|
||
!smartAutoNeedsClarification &&
|
||
smartAutoDetectedIntent === "video_gen";
|
||
const effectiveImageParams = shouldApplyAgentImageParams
|
||
? mergeImageParamsWithAgentRecommendation({
|
||
incoming: requestBody.imageParams,
|
||
recommended: smartAutoRecommendedImageParams,
|
||
intent: smartAutoDetectedIntent,
|
||
hasUserCustomSettings: hasUserCustomImageParams,
|
||
})
|
||
: requestBody.imageParams;
|
||
const effectiveVideoParams = shouldApplyAgentVideoParams
|
||
? mergeVideoParamsWithAgentRecommendation({
|
||
incoming: requestBody.videoParams,
|
||
recommended: smartAutoRecommendedVideoParams,
|
||
hasUserCustomSettings: hasUserCustomVideoParams,
|
||
})
|
||
: requestBody.videoParams;
|
||
const defaultImageQualityParams = {
|
||
aspectRatio: "auto",
|
||
imageSize: smartAutoDetectedIntent === "image_edit" ? "4K" : "2K",
|
||
imageCount: 1,
|
||
} as const;
|
||
|
||
if (shouldApplyAgentImageParams) {
|
||
console.log("[Smart Auto Mode] 🎯 Effective image params:", {
|
||
incoming: requestBody.imageParams,
|
||
recommended: smartAutoRecommendedImageParams,
|
||
merged: effectiveImageParams,
|
||
hasUserCustomImageParams,
|
||
});
|
||
}
|
||
if (shouldApplyAgentVideoParams) {
|
||
console.log("[Smart Auto Mode] 🎯 Effective video params:", {
|
||
incoming: requestBody.videoParams,
|
||
recommended: smartAutoRecommendedVideoParams,
|
||
merged: effectiveVideoParams,
|
||
hasUserCustomVideoParams,
|
||
hasVideoPlan: Boolean(smartAutoVideoPlan),
|
||
});
|
||
}
|
||
|
||
if (shouldRunAgentConversation && smartAutoFriendlyMessage) {
|
||
smartAutoFriendlyMessage = sanitizeAgentOutputSettingsSection({
|
||
message: smartAutoFriendlyMessage,
|
||
userImageParams: requestBody.imageParams,
|
||
hasUserCustomImageParams,
|
||
});
|
||
}
|
||
|
||
const session = await auth();
|
||
|
||
if (!session?.user) {
|
||
return new ChatSDKError("unauthorized:chat").toResponse();
|
||
}
|
||
|
||
const userType: UserType = session.user.type;
|
||
|
||
const messageCount = await getMessageCountByUserId({
|
||
id: session.user.id,
|
||
differenceInHours: 24,
|
||
});
|
||
|
||
if (messageCount > entitlementsByUserType[userType].maxMessagesPerDay) {
|
||
return new ChatSDKError("rate_limit:chat").toResponse();
|
||
}
|
||
|
||
// ========== 使用时间限制验证（后端安全检查） ==========
|
||
try {
|
||
// 从数据库获取网站设置
|
||
const { siteSettings } = await import("@/lib/db/schema");
|
||
const [settings] = await db.select().from(siteSettings).limit(1);
|
||
|
||
if (settings?.usageTimeEnabled) {
|
||
const now = new Date();
|
||
const currentTime = `${String(now.getHours()).padStart(2, "0")}:${String(now.getMinutes()).padStart(2, "0")}`;
|
||
const startTime = settings.usageTimeStart || "08:00";
|
||
const endTime = settings.usageTimeEnd || "20:00";
|
||
|
||
const isWithinUsageTime = currentTime >= startTime && currentTime <= endTime;
|
||
|
||
if (!isWithinUsageTime) {
|
||
console.log(`[Chat API] ⛔ 访问被拒绝：当前时间 ${currentTime} 不在允许范围 ${startTime}-${endTime}`);
|
||
return NextResponse.json(
|
||
{
|
||
error: "usage_time_restricted",
|
||
message: settings.usageTimeMessage || "当前不在系统使用时间段内",
|
||
currentTime,
|
||
allowedTime: `${startTime} - ${endTime}`
|
||
},
|
||
{ status: 403 }
|
||
);
|
||
}
|
||
|
||
console.log(`[Chat API] ✅ 时间验证通过：${currentTime} 在允许范围 ${startTime}-${endTime}`);
|
||
}
|
||
} catch (timeCheckError) {
|
||
console.error("[Chat API] ⚠️ 时间验证失败（继续执行）:", timeCheckError);
|
||
// 如果时间检查失败，继续执行（避免因检查逻辑错误导致系统不可用）
|
||
}
|
||
|
||
const chat = await getChatById({ id });
|
||
let messagesFromDb: DBMessage[] = [];
|
||
|
||
if (chat) {
|
||
if (chat.userId !== session.user.id) {
|
||
return new ChatSDKError("forbidden:chat").toResponse();
|
||
}
|
||
// Only fetch messages if chat already exists
|
||
messagesFromDb = await getMessagesByChatId({ id });
|
||
} else {
|
||
const title = await generateTitleFromUserMessage({
|
||
message,
|
||
});
|
||
|
||
await saveChat({
|
||
id,
|
||
userId: session.user.id,
|
||
title,
|
||
visibility: selectedVisibilityType,
|
||
});
|
||
// New chat - no need to fetch messages, it's empty
|
||
}
|
||
|
||
const uiMessages = [...convertToUIMessages(messagesFromDb), message];
|
||
const hiddenPromptText = typeof hiddenPrompt === "string" ? hiddenPrompt.trim() : "";
|
||
const hiddenPromptPrefix = hiddenPromptText ? `${hiddenPromptText}\n\n` : "";
|
||
const uiMessagesForModel = hiddenPromptText
|
||
? applyHiddenPromptToMessages(uiMessages, hiddenPromptText)
|
||
: uiMessages;
|
||
|
||
const { longitude, latitude, city, country } = geolocation(request);
|
||
|
||
const requestHints: RequestHints = {
|
||
longitude,
|
||
latitude,
|
||
city,
|
||
country,
|
||
};
|
||
|
||
await saveMessages({
|
||
messages: [
|
||
{
|
||
chatId: id,
|
||
id: message.id,
|
||
role: "user",
|
||
parts: message.parts,
|
||
attachments: [],
|
||
createdAt: new Date(),
|
||
videoStorageStatus: null,
|
||
temporaryVideoUrl: null,
|
||
permanentVideoUrl: null,
|
||
archivedAt: null,
|
||
videoMetadata: null,
|
||
},
|
||
],
|
||
});
|
||
|
||
const streamId = generateUUID();
|
||
await createStreamId({ streamId, chatId: id });
|
||
|
||
if (shouldRunAgentConversation && smartAutoNeedsClarification) {
|
||
const hasCurrentImageInput = message.parts.some((part: any) =>
|
||
part?.type === "file" || part?.type === "image" || part?.type === "image_url"
|
||
);
|
||
const hasHistoryImageInput = smartAutoHistoryImages.length > 0;
|
||
const latestUserText = message.parts
|
||
.filter((part): part is { type: "text"; text: string } => part.type === "text")
|
||
.map((part) => part.text)
|
||
.join(" ")
|
||
.trim();
|
||
|
||
if (hasCurrentImageInput || hasHistoryImageInput || latestUserText.length > 0) {
|
||
smartAutoNeedsClarification = false;
|
||
smartAutoClarificationQuestion = undefined;
|
||
if (
|
||
!smartAutoFriendlyMessage ||
|
||
/[?？]$/.test(smartAutoFriendlyMessage.trim()) ||
|
||
/哪一张|哪个|请先确认/.test(smartAutoFriendlyMessage)
|
||
) {
|
||
smartAutoFriendlyMessage = "收到，我将按你的当前需求直接执行。";
|
||
}
|
||
console.log("[Smart Auto Mode] ✅ Force direct execution (skip clarification)");
|
||
}
|
||
}
|
||
|
||
if (shouldRunAgentConversation && smartAutoNeedsClarification) {
|
||
const clarificationMessage =
|
||
smartAutoClarificationQuestion ||
|
||
smartAutoFriendlyMessage ||
|
||
"我先确认一下，你希望我优先处理哪一部分？";
|
||
|
||
const clarificationStream = createUIMessageStream({
|
||
execute: async ({ writer }) => {
|
||
const messageId = generateUUID();
|
||
writer.write({
|
||
type: "data-appendMessage",
|
||
data: JSON.stringify({
|
||
id: messageId,
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: clarificationMessage }],
|
||
}),
|
||
});
|
||
|
||
await saveMessages({
|
||
messages: [
|
||
{
|
||
id: messageId,
|
||
chatId: id,
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: clarificationMessage }],
|
||
attachments: [],
|
||
createdAt: new Date(),
|
||
videoStorageStatus: null,
|
||
temporaryVideoUrl: null,
|
||
permanentVideoUrl: null,
|
||
archivedAt: null,
|
||
videoMetadata: null,
|
||
},
|
||
],
|
||
});
|
||
},
|
||
});
|
||
|
||
return new Response(clarificationStream.pipeThrough(new JsonToSseTransformStream()));
|
||
}
|
||
|
||
let finalMergedUsage: AppUsage | undefined;
|
||
|
||
// 获取启用的工具列表
|
||
const enabledTools = await getEnabledTools();
|
||
|
||
// 检查是否是 UUID 格式（数据库中的模型 ID）
|
||
const isUUID =
|
||
/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test(
|
||
finalModelId
|
||
);
|
||
|
||
// 获取语言模型实例
|
||
let languageModel;
|
||
let actualModelId: string;
|
||
let resolvedModelSupportsImageGen = false;
|
||
|
||
if (isUUID) {
|
||
// 从数据库获取动态配置的模型
|
||
let targetModelId = finalModelId;
|
||
|
||
// 🆕 检查是否是视频主模型，如果是，根据 videoParams.mode 选择子模型
|
||
const aiModelData = await getAiModelById({ id: finalModelId });
|
||
console.log("[Chat API] Model data:", { id: finalModelId, isSubModel: aiModelData?.isSubModel });
|
||
|
||
if (aiModelData && !aiModelData.isSubModel) {
|
||
// 检查是否有子模型
|
||
const subModels = await getSubModelsByParentModelId(finalModelId);
|
||
console.log("[Chat API] Found sub-models:", subModels.length);
|
||
|
||
if (subModels.length > 0) {
|
||
// 这是一个主模型，需要根据 videoParams 选择子模型
|
||
const videoParams = effectiveVideoParams;
|
||
console.log("[Chat API] Video params:", videoParams);
|
||
|
||
// 如果没有 videoParams 或没有 mode，使用默认模式
|
||
const mode = videoParams?.mode || "text-to-video";
|
||
console.log("[Chat API] Using mode:", mode);
|
||
|
||
// 根据模式选择对应的子模型
|
||
const modeToModelMap: Record<string, string> = {
|
||
"text-to-video": "lite-t2v", // 文生视频 → Lite T2V
|
||
"first-last-frame": "pro", // 首尾帧 → Pro
|
||
"first-frame": "pro-fast", // 智能多帧 → Pro Fast
|
||
"reference-images": "lite-i2v", // 主体参考 → Lite I2V
|
||
};
|
||
|
||
const targetSubModelKey = modeToModelMap[mode];
|
||
if (targetSubModelKey) {
|
||
// 查找匹配的子模型（通过 modelId 中包含的关键字）
|
||
const matchedSubModel = subModels.find(({ model }) =>
|
||
model.modelId.toLowerCase().includes(targetSubModelKey)
|
||
);
|
||
|
||
if (matchedSubModel) {
|
||
console.log(`[Chat API] 视频模式 ${mode} → 选择子模型: ${matchedSubModel.model.displayName}`);
|
||
targetModelId = matchedSubModel.model.id;
|
||
} else {
|
||
console.warn(`[Chat API] 未找到匹配的子模型: ${targetSubModelKey}，将使用第一个可用子模型`);
|
||
// 如果没找到匹配的，使用第一个子模型作为后备
|
||
targetModelId = subModels[0].model.id;
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
const dynamicModel = await getLanguageModelById(targetModelId);
|
||
if (!dynamicModel) {
|
||
return new ChatSDKError(
|
||
"bad_request:api",
|
||
"Model not found or disabled"
|
||
).toResponse();
|
||
}
|
||
languageModel = dynamicModel;
|
||
// 对于动态模型，我们需要从数据库获取 modelId
|
||
const finalModelData = await getAiModelById({ id: targetModelId });
|
||
actualModelId = finalModelData?.modelId ?? targetModelId;
|
||
resolvedModelSupportsImageGen = Boolean(finalModelData?.supportsImageGen);
|
||
} else {
|
||
// 使用预定义的模型
|
||
languageModel = myProvider.languageModel(finalModelId);
|
||
actualModelId = languageModel.modelId;
|
||
}
|
||
|
||
// ========== 图片生成配额检查 ==========
|
||
const isImageGenerationModel =
|
||
(languageModel as any).provider === "gemini-image" ||
|
||
(languageModel as any).provider === "volcano-vision" ||
|
||
(languageModel as any).provider === "alibaba-image" ||
|
||
(languageModel as any).provider === "ruxa-image" ||
|
||
(languageModel as any).provider === "fal-image" ||
|
||
(languageModel as any).provider === "huanke-gemini";
|
||
|
||
const isUuidModelId = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test(
|
||
finalModelId
|
||
);
|
||
const currentModelConfig = isUuidModelId
|
||
? await getAiModelById({ id: finalModelId }).catch(() => undefined)
|
||
: undefined;
|
||
const modelCreditCostPerImage = Number(
|
||
(currentModelConfig as any)?.imageGenConfig?.creditCostPerImage ?? 0
|
||
);
|
||
|
||
if (isImageGenerationModel) {
|
||
const quota = await checkUserImageQuota(session.user.id);
|
||
|
||
if (!quota.allowed) {
|
||
console.log(`[Chat API] ⛔ 图片配额已用完: ${quota.used}/${quota.limit}`);
|
||
return NextResponse.json(
|
||
{
|
||
error: "quota_exceeded",
|
||
message: `本月图片生成配额已用完（已使用 ${quota.used}/${quota.limit} 张），配额将于下月1日重置，如需更多配额请联系管理员。`,
|
||
used: quota.used,
|
||
limit: quota.limit,
|
||
},
|
||
{ status: 429 }
|
||
);
|
||
}
|
||
|
||
console.log(`[Chat API] ✅ 图片配额检查通过: ${quota.used}/${quota.limit ?? '无限制'}`);
|
||
|
||
const pointsConfig = await getPointsSystemConfig();
|
||
const normalizedUnitCost = Number.isFinite(modelCreditCostPerImage)
|
||
? Math.max(0, Math.round(modelCreditCostPerImage))
|
||
: 0;
|
||
|
||
if (pointsConfig.enabled && normalizedUnitCost > 0 && pointsConfig.blockOnInsufficient !== false) {
|
||
const requestedImageCount = Math.max(
|
||
1,
|
||
Math.round(
|
||
Number(
|
||
effectiveImageParams?.imageCount || defaultImageQualityParams.imageCount || 1
|
||
)
|
||
)
|
||
);
|
||
const expectedCost = normalizedUnitCost * requestedImageCount;
|
||
const userData = await getUserById({ id: session.user.id });
|
||
const currentBalance = Math.max(0, Number(userData?.pointsBalance || 0));
|
||
|
||
if (currentBalance < expectedCost) {
|
||
return NextResponse.json(
|
||
{
|
||
error: "insufficient_points",
|
||
message: `积分不足，当前余额 ${currentBalance}，本次预计消耗 ${expectedCost}（${normalizedUnitCost} x ${requestedImageCount} 张）。`,
|
||
balance: currentBalance,
|
||
expectedCost,
|
||
unitCost: normalizedUnitCost,
|
||
},
|
||
{ status: 402 }
|
||
);
|
||
}
|
||
}
|
||
}
|
||
|
||
// ========== Gemini 图片生成独立处理 ==========
|
||
// 支持 gemini-image 和 huanke-gemini 两种图片生成 provider
|
||
const isGeminiImageProvider = ["gemini-image", "huanke-gemini"].includes((languageModel as any).provider);
|
||
if (isGeminiImageProvider && typeof (languageModel as any).doGenerate === "function") {
|
||
console.log("[Chat API] Using Gemini image generation, provider:", (languageModel as any).provider);
|
||
console.log("[Chat API] Image params:", effectiveImageParams);
|
||
|
||
// 🆕 获取备用模型配置
|
||
const currentModelConfig = await getAiModelById({ id: finalModelId });
|
||
const configuredFallbacks = [
|
||
currentModelConfig?.fallbackModelId1,
|
||
currentModelConfig?.fallbackModelId2,
|
||
].filter(Boolean) as string[];
|
||
const allModels = await getEnabledAiModelsWithProvider();
|
||
const sameProviderFallbacks = currentModelConfig
|
||
? allModels
|
||
.filter(m =>
|
||
m.provider.id === currentModelConfig.providerId &&
|
||
m.model.supportsImageGen &&
|
||
m.model.isEnabled &&
|
||
!m.model.isSubModel &&
|
||
m.model.modelId !== actualModelId
|
||
)
|
||
.map(m => m.model.modelId)
|
||
: [];
|
||
const fallbackModelIds = Array.from(new Set([...configuredFallbacks, ...sameProviderFallbacks]));
|
||
console.log("[Chat API] Fallback model IDs:", fallbackModelIds);
|
||
|
||
const geminiStream = createUIMessageStream({
|
||
execute: async ({ writer }) => {
|
||
// 🆕 如果是智能模式且有对话回复，先发送对话消息让用户看到
|
||
// 在消息末尾加上特殊标记，告诉前端"图片正在生成中"
|
||
const conversationMessage = smartAutoFriendlyMessage || null;
|
||
|
||
if (conversationMessage) {
|
||
const chatMessageId = generateUUID();
|
||
console.log("[Chat API] 💬 Gemini: Sending conversational reply first:", smartAutoFriendlyMessage);
|
||
|
||
// 发送带标记的对话消息
|
||
writer.write({
|
||
type: "data-appendMessage",
|
||
data: JSON.stringify({
|
||
id: chatMessageId,
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: conversationMessage }],
|
||
}),
|
||
});
|
||
|
||
// 保存对话消息到数据库（不带标记，纯对话）
|
||
await saveMessages({
|
||
messages: [
|
||
{
|
||
id: chatMessageId,
|
||
chatId: id,
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: smartAutoFriendlyMessage }],
|
||
attachments: [],
|
||
createdAt: new Date(),
|
||
videoStorageStatus: null,
|
||
temporaryVideoUrl: null,
|
||
permanentVideoUrl: null,
|
||
archivedAt: null,
|
||
videoMetadata: null,
|
||
},
|
||
],
|
||
});
|
||
}
|
||
|
||
// 🆕 尝试生成图片的函数（支持备用模型切换）
|
||
const tryGenerateImage = async (modelIdToUse: string, isRetry = false): Promise<string> => {
|
||
console.log(`[Chat API] ${isRetry ? "🔄 Retry with fallback" : "🚀 Trying"} model: ${modelIdToUse}`);
|
||
|
||
// 只传递最后一条用户消息，避免上下文历史干扰生图效果
|
||
const lastUserMessage = uiMessagesForModel.filter((m) => m.role === "user").pop();
|
||
let messagesForGemini = lastUserMessage ? [lastUserMessage] : uiMessagesForModel;
|
||
messagesForGemini = applyOptimizedPromptToLatestUserMessage(
|
||
messagesForGemini,
|
||
smartAutoOptimizedPrompt,
|
||
hiddenPromptPrefix,
|
||
);
|
||
messagesForGemini = injectHistoryImagesToLastUserMessage(messagesForGemini, smartAutoHistoryImages);
|
||
console.log("[Chat API] Gemini: Using only last user message (no history context)");
|
||
|
||
// 转换消息：gemini-image 需要 base64 格式，huanke-gemini 需要 URL 格式
|
||
const providerForConvert = (languageModel as any).provider === "huanke-gemini" ? "huanke-gemini" : "gemini";
|
||
const modelMessages = await convertMessagesWithImages(messagesForGemini, providerForConvert);
|
||
console.log("[Chat API] Message conversion provider:", providerForConvert);
|
||
|
||
// 调用 doGenerate 获取图片（如果是备用模型，传入 overrideModelId）
|
||
const result = await (languageModel as any).doGenerate({
|
||
messages: modelMessages,
|
||
imageParams:
|
||
effectiveImageParams || {
|
||
aspectRatio: defaultImageQualityParams.aspectRatio,
|
||
imageSize: defaultImageQualityParams.imageSize,
|
||
},
|
||
overrideModelId: isRetry ? modelIdToUse : undefined, // 🆕 使用备用模型 ID
|
||
});
|
||
|
||
return result.text || "图片生成失败";
|
||
};
|
||
|
||
try {
|
||
let imageMarkdown = ""; // 初始化为空字符串
|
||
let usedModelId = actualModelId;
|
||
|
||
// 🆕 图片生成真正开始时发出标记（用于前端显示占位图）
|
||
writer.write({
|
||
type: "data-appendMessage",
|
||
data: JSON.stringify({
|
||
id: generateUUID(),
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: "<!-- IMAGE_GENERATION_START -->" }],
|
||
}),
|
||
});
|
||
|
||
try {
|
||
// 首先尝试主模型
|
||
imageMarkdown = await tryGenerateImage(actualModelId, false);
|
||
} catch (primaryError) {
|
||
console.error("[Chat API] Primary model failed:", primaryError);
|
||
|
||
// 🆕 尝试备用模型（使用同一个 provider，但切换 modelId）
|
||
let fallbackSuccess = false;
|
||
for (const fallbackModelId of fallbackModelIds) {
|
||
try {
|
||
console.log(`[Chat API] 🔄 Trying fallback model: ${fallbackModelId}`);
|
||
imageMarkdown = await tryGenerateImage(fallbackModelId, true);
|
||
usedModelId = fallbackModelId;
|
||
fallbackSuccess = true;
|
||
console.log(`[Chat API] ✅ Fallback model ${fallbackModelId} succeeded!`);
|
||
break;
|
||
} catch (fallbackError) {
|
||
console.error(`[Chat API] ❌ Fallback model ${fallbackModelId} failed:`, fallbackError);
|
||
}
|
||
}
|
||
|
||
// 如果所有备用模型都失败，抛出原始错误触发火山引擎故障转移
|
||
if (!fallbackSuccess) {
|
||
throw primaryError;
|
||
}
|
||
}
|
||
|
||
console.log("[Chat API] Gemini generated, length:", imageMarkdown.length);
|
||
console.log("[Chat API] Gemini image markdown:", imageMarkdown.substring(0, 200));
|
||
console.log("[Chat API] Used model ID:", usedModelId);
|
||
|
||
// 生成一个统一的消息 ID
|
||
const messageId = generateUUID();
|
||
|
||
// 写入消息到流 - 使用正确的 parts 格式
|
||
writer.write({
|
||
type: "data-appendMessage",
|
||
data: JSON.stringify({
|
||
id: messageId,
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: imageMarkdown }],
|
||
}),
|
||
});
|
||
|
||
// 🆕 图片生成结束标记（成功）
|
||
writer.write({
|
||
type: "data-appendMessage",
|
||
data: JSON.stringify({
|
||
id: generateUUID(),
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: "<!-- IMAGE_GENERATION_END -->" }],
|
||
}),
|
||
});
|
||
|
||
// 保存消息到数据库（现在是 S3 URL，可以完整保存）
|
||
try {
|
||
await saveMessages({
|
||
messages: [
|
||
{
|
||
id: messageId,
|
||
chatId: id,
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: imageMarkdown }],
|
||
attachments: [], // 必须提供 attachments 字段（数据库 notNull）
|
||
createdAt: new Date(),
|
||
videoStorageStatus: null,
|
||
temporaryVideoUrl: null,
|
||
permanentVideoUrl: null,
|
||
archivedAt: null,
|
||
videoMetadata: null,
|
||
},
|
||
],
|
||
});
|
||
console.log("[Chat API] ✅ Gemini message saved successfully!");
|
||
|
||
// 提取用户 prompt（最后一条用户消息的文本）
|
||
const lastUserMessage = uiMessages.filter((m) => m.role === "user").pop();
|
||
const userPrompt = lastUserMessage?.parts?.find((p): p is { type: "text"; text: string } => p.type === "text")?.text || "";
|
||
|
||
// 保存图片记录到 GeneratedImage 表
|
||
await saveGeneratedImages(
|
||
imageMarkdown,
|
||
session.user.id,
|
||
id,
|
||
messageId,
|
||
userPrompt,
|
||
finalModelId
|
||
);
|
||
} catch (saveError) {
|
||
console.error("[Chat API] ❌ Failed to save Gemini message to database:", saveError);
|
||
console.error("[Chat API] Message content that failed:", imageMarkdown.substring(0, 300));
|
||
}
|
||
} catch (error) {
|
||
console.error("[Chat API] Gemini generation error:", error);
|
||
console.log("[Chat API] Attempting failover to Volcano Vision...");
|
||
|
||
// 故障转移：尝试使用火山引擎模型
|
||
try {
|
||
// 查找火山引擎模型
|
||
const allModels = await getEnabledAiModelsWithProvider();
|
||
console.log("[Chat API] Total enabled models:", allModels.length);
|
||
|
||
const volcanoModel = allModels.find(
|
||
(m: any) => m.provider.type === "volcano-vision" && m.model.isEnabled
|
||
);
|
||
|
||
if (volcanoModel) {
|
||
console.log("[Chat API] Found Volcano Vision model for failover:", volcanoModel.model.id);
|
||
|
||
// 获取火山引擎模型实例
|
||
const volcanoLanguageModel = await getLanguageModelById(volcanoModel.model.id);
|
||
|
||
if (volcanoLanguageModel && typeof (volcanoLanguageModel as any).doGenerate === "function") {
|
||
console.log("[Chat API] Executing Volcano Vision failover...");
|
||
|
||
// 转换消息（火山引擎会自动处理 Base64）
|
||
const volcanoMessages = await convertMessagesWithImages(
|
||
injectHistoryImagesToLastUserMessage(uiMessagesForModel, smartAutoHistoryImages),
|
||
"volcano"
|
||
);
|
||
|
||
// 调用火山引擎生成（故障转移时只生成 1 张，保持与 Gemini 一致）
|
||
const volcanoResult = await (volcanoLanguageModel as any).doGenerate({
|
||
messages: volcanoMessages,
|
||
maxImages: 1, // 故障转移时只生成 1 张
|
||
});
|
||
|
||
const imageMarkdown = volcanoResult.text || "图片生成失败";
|
||
console.log("[Chat API] Volcano Vision failover success, length:", imageMarkdown.length);
|
||
|
||
// 生成消息 ID
|
||
const messageId = generateUUID();
|
||
|
||
// 写入消息到流
|
||
writer.write({
|
||
type: "data-appendMessage",
|
||
data: JSON.stringify({
|
||
id: messageId,
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: imageMarkdown }],
|
||
}),
|
||
});
|
||
|
||
// 🆕 图片生成结束标记（故障转移成功）
|
||
writer.write({
|
||
type: "data-appendMessage",
|
||
data: JSON.stringify({
|
||
id: generateUUID(),
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: "<!-- IMAGE_GENERATION_END -->" }],
|
||
}),
|
||
});
|
||
|
||
// 保存消息到数据库
|
||
try {
|
||
await saveMessages({
|
||
messages: [
|
||
{
|
||
id: messageId,
|
||
chatId: id,
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: imageMarkdown }],
|
||
attachments: [],
|
||
createdAt: new Date(),
|
||
videoStorageStatus: null,
|
||
temporaryVideoUrl: null,
|
||
permanentVideoUrl: null,
|
||
archivedAt: null,
|
||
videoMetadata: null,
|
||
},
|
||
],
|
||
});
|
||
console.log("[Chat API] ✅ Volcano Vision failover message saved!");
|
||
|
||
// 提取用户 prompt
|
||
const lastUserMessage = uiMessages.filter((m) => m.role === "user").pop();
|
||
const userPrompt = lastUserMessage?.parts?.find((p): p is { type: "text"; text: string } => p.type === "text")?.text || "";
|
||
|
||
// 保存图片记录到 GeneratedImage 表
|
||
await saveGeneratedImages(
|
||
imageMarkdown,
|
||
session.user.id,
|
||
id,
|
||
messageId,
|
||
userPrompt,
|
||
volcanoModel.model.id
|
||
);
|
||
} catch (saveError) {
|
||
console.error("[Chat API] Failed to save failover message:", saveError);
|
||
}
|
||
|
||
return; // 成功，直接返回
|
||
}
|
||
}
|
||
|
||
console.log("[Chat API] No Volcano Vision model available for failover");
|
||
} catch (failoverError) {
|
||
console.error("[Chat API] Failover to Volcano Vision also failed:", failoverError);
|
||
}
|
||
|
||
// 如果故障转移也失败，返回错误消息
|
||
const errorMessageId = generateUUID();
|
||
writer.write({
|
||
type: "data-appendMessage",
|
||
data: JSON.stringify({
|
||
id: errorMessageId,
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: "图片生成失败，请稍后重试" }],
|
||
}),
|
||
});
|
||
|
||
// 🆕 图片生成结束标记（失败）
|
||
writer.write({
|
||
type: "data-appendMessage",
|
||
data: JSON.stringify({
|
||
id: generateUUID(),
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: "<!-- IMAGE_GENERATION_END -->" }],
|
||
}),
|
||
});
|
||
|
||
// 尝试保存错误消息
|
||
try {
|
||
await saveMessages({
|
||
messages: [
|
||
{
|
||
id: errorMessageId,
|
||
chatId: id,
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: "图片生成失败，请稍后重试" }],
|
||
attachments: [],
|
||
createdAt: new Date(),
|
||
videoStorageStatus: null,
|
||
temporaryVideoUrl: null,
|
||
permanentVideoUrl: null,
|
||
archivedAt: null,
|
||
videoMetadata: null,
|
||
},
|
||
],
|
||
});
|
||
} catch (saveError) {
|
||
console.error("[Chat API] Failed to save error message:", saveError);
|
||
}
|
||
}
|
||
},
|
||
});
|
||
|
||
return new Response(geminiStream.pipeThrough(new JsonToSseTransformStream()));
|
||
}
|
||
// ========== Gemini 处理结束 ==========
|
||
|
||
// ========== 阿里云千问图片生成独立处理 ==========
|
||
const isAlibabaLikeImageProvider = ["alibaba-image", "ruxa-image", "fal-image"].includes((languageModel as any).provider);
|
||
if (isAlibabaLikeImageProvider && typeof (languageModel as any).doGenerate === "function") {
|
||
console.log("[Chat API] Using image generation provider:", (languageModel as any).provider);
|
||
console.log("[Chat API] Image params:", effectiveImageParams);
|
||
|
||
// 🆕 获取模型显示名称，用于错误提示
|
||
const alibabaModelConfig = await getAiModelById({ id: finalModelId });
|
||
const modelDisplayName = alibabaModelConfig?.displayName || actualModelId;
|
||
const configuredFallbacks = [
|
||
alibabaModelConfig?.fallbackModelId1,
|
||
alibabaModelConfig?.fallbackModelId2,
|
||
].filter(Boolean) as string[];
|
||
const allModels = await getEnabledAiModelsWithProvider();
|
||
const sameProviderFallbacks = alibabaModelConfig
|
||
? allModels
|
||
.filter(m =>
|
||
m.provider.id === alibabaModelConfig.providerId &&
|
||
m.model.supportsImageGen &&
|
||
m.model.isEnabled &&
|
||
!m.model.isSubModel &&
|
||
m.model.modelId !== actualModelId
|
||
)
|
||
.map(m => m.model.modelId)
|
||
: [];
|
||
const fallbackModelIds = Array.from(new Set([...configuredFallbacks, ...sameProviderFallbacks]));
|
||
console.log("[Chat API] Alibaba fallback model IDs:", fallbackModelIds);
|
||
|
||
const alibabaStream = createUIMessageStream({
|
||
execute: async ({ writer }) => {
|
||
try {
|
||
// 🆕 如果是智能模式且有对话回复，先发送对话消息让用户看到
|
||
const conversationMessage = smartAutoFriendlyMessage || null;
|
||
|
||
if (conversationMessage) {
|
||
const chatMessageId = generateUUID();
|
||
console.log("[Chat API] 💬 Sending conversational reply first:", smartAutoFriendlyMessage);
|
||
|
||
// 发送带标记的对话消息
|
||
writer.write({
|
||
type: "data-appendMessage",
|
||
data: JSON.stringify({
|
||
id: chatMessageId,
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: conversationMessage }],
|
||
}),
|
||
});
|
||
|
||
// 保存对话消息到数据库（不带标记，纯对话）
|
||
await saveMessages({
|
||
messages: [
|
||
{
|
||
id: chatMessageId,
|
||
chatId: id,
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: smartAutoFriendlyMessage }],
|
||
attachments: [],
|
||
createdAt: new Date(),
|
||
videoStorageStatus: null,
|
||
temporaryVideoUrl: null,
|
||
permanentVideoUrl: null,
|
||
archivedAt: null,
|
||
videoMetadata: null,
|
||
},
|
||
],
|
||
});
|
||
}
|
||
|
||
// 只传递最后一条用户消息
|
||
const lastUserMessage = uiMessagesForModel.filter((m) => m.role === "user").pop();
|
||
let messagesForAlibaba = lastUserMessage ? [lastUserMessage] : uiMessagesForModel;
|
||
console.log("[Chat API] Alibaba: Using only last user message");
|
||
|
||
// 🆕 如果有智能模式优化后的提示词，替换消息中的文本（仅用于API调用）
|
||
if (smartAutoOptimizedPrompt) {
|
||
console.log("[Chat API] Using optimized prompt for API call");
|
||
messagesForAlibaba = applyOptimizedPromptToLatestUserMessage(
|
||
messagesForAlibaba,
|
||
smartAutoOptimizedPrompt,
|
||
hiddenPromptPrefix,
|
||
);
|
||
}
|
||
|
||
messagesForAlibaba = injectHistoryImagesToLastUserMessage(messagesForAlibaba, smartAutoHistoryImages);
|
||
|
||
const tryGenerateImage = async (modelIdToUse: string, isRetry: boolean) => {
|
||
// 转换消息（图片编辑模型统一使用 URL 输入）
|
||
const providerForConvert =
|
||
(languageModel as any).provider === "ruxa-image"
|
||
? "ruxa-image"
|
||
: (languageModel as any).provider === "fal-image"
|
||
? "fal-image"
|
||
: "alibaba-image";
|
||
const modelMessages = await convertMessagesWithImages(messagesForAlibaba, providerForConvert);
|
||
|
||
// 调用 doGenerate 获取图片，传递模型显示名称
|
||
const result = await (languageModel as any).doGenerate({
|
||
messages: modelMessages,
|
||
imageParams:
|
||
effectiveImageParams || {
|
||
aspectRatio: defaultImageQualityParams.aspectRatio,
|
||
imageSize: defaultImageQualityParams.imageSize,
|
||
imageCount: defaultImageQualityParams.imageCount,
|
||
},
|
||
modelDisplayName, // 🆕 传递模型显示名称
|
||
overrideModelId: isRetry ? modelIdToUse : undefined,
|
||
});
|
||
|
||
if (result?.finishReason === "error") {
|
||
throw new Error(result.text || "图片生成失败");
|
||
}
|
||
|
||
return result.text || "图片生成失败";
|
||
};
|
||
|
||
let imageMarkdown = "";
|
||
let usedModelId = actualModelId;
|
||
|
||
// 🆕 图片生成真正开始时发出标记（用于前端显示占位图）
|
||
writer.write({
|
||
type: "data-appendMessage",
|
||
data: JSON.stringify({
|
||
id: generateUUID(),
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: "<!-- IMAGE_GENERATION_START -->" }],
|
||
}),
|
||
});
|
||
|
||
try {
|
||
imageMarkdown = await tryGenerateImage(actualModelId, false);
|
||
} catch (primaryError) {
|
||
console.error("[Chat API] Alibaba primary model failed:", primaryError);
|
||
|
||
let fallbackSuccess = false;
|
||
for (const fallbackModelId of fallbackModelIds) {
|
||
try {
|
||
console.log(`[Chat API] 🔄 Alibaba fallback model: ${fallbackModelId}`);
|
||
imageMarkdown = await tryGenerateImage(fallbackModelId, true);
|
||
usedModelId = fallbackModelId;
|
||
fallbackSuccess = true;
|
||
console.log(`[Chat API] ✅ Alibaba fallback model ${fallbackModelId} succeeded!`);
|
||
break;
|
||
} catch (fallbackError) {
|
||
console.error(`[Chat API] ❌ Alibaba fallback model ${fallbackModelId} failed:`, fallbackError);
|
||
}
|
||
}
|
||
|
||
if (!fallbackSuccess) {
|
||
const providerPriority: Record<string, number> = {
|
||
"volcano-vision": 0,
|
||
"alibaba-image": 1,
|
||
"ruxa-image": 2,
|
||
"fal-image": 3,
|
||
gemini: 4,
|
||
"huanke-gemini": 5,
|
||
};
|
||
const crossProviderCandidates = getImageCrossProviderCandidates(allModels, {
|
||
currentModelId: finalModelId,
|
||
currentProviderId: alibabaModelConfig?.providerId,
|
||
providerPriority,
|
||
});
|
||
|
||
for (const candidate of crossProviderCandidates) {
|
||
try {
|
||
console.log(
|
||
`[Chat API] 🔁 Cross-provider failover attempt: ${candidate.model.displayName} (${candidate.provider.type})`
|
||
);
|
||
|
||
const candidateLanguageModel = await getLanguageModelById(candidate.model.id);
|
||
if (!candidateLanguageModel || typeof (candidateLanguageModel as any).doGenerate !== "function") {
|
||
console.log(`[Chat API] Skip candidate without doGenerate: ${candidate.model.id}`);
|
||
continue;
|
||
}
|
||
|
||
const convertProvider =
|
||
candidate.provider.type === "ruxa-image"
|
||
? "ruxa-image"
|
||
: candidate.provider.type === "fal-image"
|
||
? "fal-image"
|
||
: candidate.provider.type === "gemini"
|
||
? "gemini"
|
||
: candidate.provider.type === "huanke-gemini"
|
||
? "huanke-gemini"
|
||
: candidate.provider.type === "volcano-vision"
|
||
? "volcano"
|
||
: "alibaba-image";
|
||
|
||
const candidateMessages = await convertMessagesWithImages(
|
||
messagesForAlibaba,
|
||
convertProvider
|
||
);
|
||
|
||
const candidateResult = await (candidateLanguageModel as any).doGenerate({
|
||
messages: candidateMessages,
|
||
imageParams:
|
||
effectiveImageParams || {
|
||
aspectRatio: defaultImageQualityParams.aspectRatio,
|
||
imageSize: defaultImageQualityParams.imageSize,
|
||
imageCount: defaultImageQualityParams.imageCount,
|
||
},
|
||
modelDisplayName: candidate.model.displayName,
|
||
overrideModelId: candidate.model.modelId,
|
||
});
|
||
|
||
if (candidateResult?.finishReason === "error") {
|
||
throw new Error(candidateResult.text || "图片生成失败");
|
||
}
|
||
|
||
imageMarkdown = candidateResult.text || "图片生成失败";
|
||
usedModelId = candidate.model.modelId;
|
||
fallbackSuccess = true;
|
||
console.log(
|
||
`[Chat API] ✅ Cross-provider failover succeeded: ${candidate.model.displayName} (${candidate.provider.type})`
|
||
);
|
||
break;
|
||
} catch (crossError) {
|
||
const crossErrorMessage = normalizeFailoverErrorMessage(crossError);
|
||
if (shouldCooldownImageFallbackModel(crossErrorMessage)) {
|
||
setImageFallbackModelCooldown(candidate.model.id, crossErrorMessage);
|
||
}
|
||
console.error(
|
||
`[Chat API] ❌ Cross-provider failover failed: ${candidate.model.displayName} (${candidate.provider.type})`,
|
||
crossError
|
||
);
|
||
}
|
||
}
|
||
}
|
||
|
||
if (!fallbackSuccess) {
|
||
throw primaryError;
|
||
}
|
||
}
|
||
|
||
console.log("[Chat API] Alibaba generated, length:", imageMarkdown.length);
|
||
console.log("[Chat API] Alibaba used model:", usedModelId);
|
||
|
||
// 生成消息 ID（图片消息）
|
||
const messageId = generateUUID();
|
||
|
||
// 写入图片消息到流
|
||
writer.write({
|
||
type: "data-appendMessage",
|
||
data: JSON.stringify({
|
||
id: messageId,
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: imageMarkdown }],
|
||
}),
|
||
});
|
||
|
||
// 🆕 图片生成结束标记（成功）
|
||
writer.write({
|
||
type: "data-appendMessage",
|
||
data: JSON.stringify({
|
||
id: generateUUID(),
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: "<!-- IMAGE_GENERATION_END -->" }],
|
||
}),
|
||
});
|
||
|
||
// 保存图片消息到数据库
|
||
try {
|
||
await saveMessages({
|
||
messages: [
|
||
{
|
||
id: messageId,
|
||
chatId: id,
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: imageMarkdown }],
|
||
attachments: [],
|
||
createdAt: new Date(),
|
||
videoStorageStatus: null,
|
||
temporaryVideoUrl: null,
|
||
permanentVideoUrl: null,
|
||
archivedAt: null,
|
||
videoMetadata: null,
|
||
},
|
||
],
|
||
});
|
||
console.log("[Chat API] ✅ Alibaba message saved successfully!");
|
||
|
||
// 提取用户 prompt
|
||
const userPrompt = lastUserMessage?.parts?.find((p): p is { type: "text"; text: string } => p.type === "text")?.text || "";
|
||
|
||
// 保存图片记录到 GeneratedImage 表
|
||
await saveGeneratedImages(
|
||
imageMarkdown,
|
||
session.user.id,
|
||
id,
|
||
messageId,
|
||
userPrompt,
|
||
finalModelId
|
||
);
|
||
} catch (saveError) {
|
||
console.error("[Chat API] ❌ Failed to save Alibaba message:", saveError);
|
||
}
|
||
} catch (error) {
|
||
console.error("[Chat API] Alibaba generation error:", error);
|
||
|
||
// 返回错误消息
|
||
const errorMessageId = generateUUID();
|
||
const errorText = error instanceof Error ? error.message : "图片生成失败，请稍后重试";
|
||
writer.write({
|
||
type: "data-appendMessage",
|
||
data: JSON.stringify({
|
||
id: errorMessageId,
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: errorText }],
|
||
}),
|
||
});
|
||
|
||
// 🆕 图片生成结束标记（失败）
|
||
writer.write({
|
||
type: "data-appendMessage",
|
||
data: JSON.stringify({
|
||
id: generateUUID(),
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: "<!-- IMAGE_GENERATION_END -->" }],
|
||
}),
|
||
});
|
||
|
||
try {
|
||
await saveMessages({
|
||
messages: [
|
||
{
|
||
id: errorMessageId,
|
||
chatId: id,
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: errorText }],
|
||
attachments: [],
|
||
createdAt: new Date(),
|
||
videoStorageStatus: null,
|
||
temporaryVideoUrl: null,
|
||
permanentVideoUrl: null,
|
||
archivedAt: null,
|
||
videoMetadata: null,
|
||
},
|
||
],
|
||
});
|
||
} catch (saveError) {
|
||
console.error("[Chat API] Failed to save error message:", saveError);
|
||
}
|
||
}
|
||
},
|
||
});
|
||
|
||
return new Response(alibabaStream.pipeThrough(new JsonToSseTransformStream()));
|
||
}
|
||
// ========== 阿里云图片生成处理结束 ==========
|
||
|
||
// ========== 火山引擎视觉图片生成独立处理 ==========
|
||
if ((languageModel as any).provider === "volcano-vision" && typeof (languageModel as any).doGenerate === "function") {
|
||
console.log("[Chat API] Using Volcano Vision image generation");
|
||
|
||
// 🆕 检测是否有标注图片，如果有则在消息中注入系统提示
|
||
const hasAnnotation = hasAnnotatedImage(uiMessages);
|
||
let messagesForVolcano = applyOptimizedPromptToLatestUserMessage(
|
||
uiMessagesForModel,
|
||
smartAutoOptimizedPrompt,
|
||
hiddenPromptPrefix,
|
||
);
|
||
const volcanoModelConfig = await getAiModelById({ id: finalModelId });
|
||
const configuredFallbacks = [
|
||
volcanoModelConfig?.fallbackModelId1,
|
||
volcanoModelConfig?.fallbackModelId2,
|
||
].filter(Boolean) as string[];
|
||
const allModels = await getEnabledAiModelsWithProvider();
|
||
const sameProviderFallbacks = volcanoModelConfig
|
||
? allModels
|
||
.filter(m =>
|
||
m.provider.id === volcanoModelConfig.providerId &&
|
||
m.model.supportsImageGen &&
|
||
m.model.isEnabled &&
|
||
!m.model.isSubModel &&
|
||
m.model.modelId !== actualModelId
|
||
)
|
||
.map(m => m.model.modelId)
|
||
: [];
|
||
const fallbackModelIds = Array.from(new Set([...configuredFallbacks, ...sameProviderFallbacks]));
|
||
console.log("[Chat API] Volcano fallback model IDs:", fallbackModelIds);
|
||
|
||
if (hasAnnotation) {
|
||
console.log("[Chat API] 🎨 Volcano Vision: 检测到标注图片，将注入系统提示");
|
||
// 创建消息副本，修改最后一条用户消息的文本
|
||
messagesForVolcano = messagesForVolcano.map((msg, idx) => {
|
||
if (msg.role === "user" && idx === messagesForVolcano.length - 1) {
|
||
// 修改最后一条用户消息
|
||
const modifiedParts = msg.parts.map((part) => {
|
||
if (part.type === "text" && part.text) {
|
||
return {
|
||
...part,
|
||
text: ANNOTATION_SYSTEM_PROMPT + "\n\n用户请求：" + part.text,
|
||
};
|
||
}
|
||
return part;
|
||
});
|
||
return { ...msg, parts: modifiedParts };
|
||
}
|
||
return msg;
|
||
});
|
||
}
|
||
|
||
messagesForVolcano = injectHistoryImagesToLastUserMessage(messagesForVolcano, smartAutoHistoryImages);
|
||
|
||
const volcanoStream = createUIMessageStream({
|
||
execute: async ({ writer }) => {
|
||
try {
|
||
// 🆕 如果是智能模式且有对话回复，先发送对话消息让用户看到
|
||
const conversationMessage = smartAutoFriendlyMessage || null;
|
||
|
||
if (conversationMessage) {
|
||
const chatMessageId = generateUUID();
|
||
console.log("[Chat API] 💬 Volcano: Sending conversational reply first:", smartAutoFriendlyMessage);
|
||
|
||
// 发送带标记的对话消息
|
||
writer.write({
|
||
type: "data-appendMessage",
|
||
data: JSON.stringify({
|
||
id: chatMessageId,
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: conversationMessage }],
|
||
}),
|
||
});
|
||
|
||
// 保存对话消息到数据库（不带标记，纯对话）
|
||
await saveMessages({
|
||
messages: [
|
||
{
|
||
id: chatMessageId,
|
||
chatId: id,
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: smartAutoFriendlyMessage }],
|
||
attachments: [],
|
||
createdAt: new Date(),
|
||
videoStorageStatus: null,
|
||
temporaryVideoUrl: null,
|
||
permanentVideoUrl: null,
|
||
archivedAt: null,
|
||
videoMetadata: null,
|
||
},
|
||
],
|
||
});
|
||
}
|
||
|
||
// 转换消息，火山引擎支持 URL 和 base64
|
||
const modelMessages = await convertMessagesWithImages(messagesForVolcano, "volcano");
|
||
|
||
const tryGenerateImage = async (modelIdToUse: string, isRetry: boolean) => {
|
||
const result = await (languageModel as any).doGenerate({
|
||
messages: modelMessages,
|
||
imageParams:
|
||
effectiveImageParams || {
|
||
aspectRatio: defaultImageQualityParams.aspectRatio,
|
||
imageSize: defaultImageQualityParams.imageSize,
|
||
imageCount: defaultImageQualityParams.imageCount,
|
||
},
|
||
overrideModelId: isRetry ? modelIdToUse : undefined,
|
||
});
|
||
if (result?.finishReason === "error") {
|
||
throw new Error(result.text || "图片生成失败");
|
||
}
|
||
return result.text || "图片生成失败";
|
||
};
|
||
|
||
let imageMarkdown = "";
|
||
let usedModelId = actualModelId;
|
||
|
||
// 🆕 图片生成真正开始时发出标记（用于前端显示占位图）
|
||
writer.write({
|
||
type: "data-appendMessage",
|
||
data: JSON.stringify({
|
||
id: generateUUID(),
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: "<!-- IMAGE_GENERATION_START -->" }],
|
||
}),
|
||
});
|
||
try {
|
||
imageMarkdown = await tryGenerateImage(actualModelId, false);
|
||
} catch (primaryError) {
|
||
console.error("[Chat API] Volcano primary model failed:", primaryError);
|
||
let fallbackSuccess = false;
|
||
for (const fallbackModelId of fallbackModelIds) {
|
||
try {
|
||
console.log(`[Chat API] 🔄 Volcano fallback model: ${fallbackModelId}`);
|
||
imageMarkdown = await tryGenerateImage(fallbackModelId, true);
|
||
usedModelId = fallbackModelId;
|
||
fallbackSuccess = true;
|
||
console.log(`[Chat API] ✅ Volcano fallback model ${fallbackModelId} succeeded!`);
|
||
break;
|
||
} catch (fallbackError) {
|
||
console.error(`[Chat API] ❌ Volcano fallback model ${fallbackModelId} failed:`, fallbackError);
|
||
}
|
||
}
|
||
if (!fallbackSuccess) {
|
||
const providerPriority: Record<string, number> = {
|
||
"alibaba-image": 0,
|
||
"ruxa-image": 1,
|
||
"fal-image": 2,
|
||
gemini: 3,
|
||
"huanke-gemini": 4,
|
||
"volcano-vision": 5,
|
||
};
|
||
const crossProviderCandidates = getImageCrossProviderCandidates(allModels, {
|
||
currentModelId: finalModelId,
|
||
currentProviderId: volcanoModelConfig?.providerId,
|
||
providerPriority,
|
||
});
|
||
|
||
for (const candidate of crossProviderCandidates) {
|
||
try {
|
||
console.log(
|
||
`[Chat API] 🔁 Volcano cross-provider failover attempt: ${candidate.model.displayName} (${candidate.provider.type})`
|
||
);
|
||
|
||
const candidateLanguageModel = await getLanguageModelById(candidate.model.id);
|
||
if (!candidateLanguageModel || typeof (candidateLanguageModel as any).doGenerate !== "function") {
|
||
console.log(`[Chat API] Skip candidate without doGenerate: ${candidate.model.id}`);
|
||
continue;
|
||
}
|
||
|
||
const convertProvider =
|
||
candidate.provider.type === "ruxa-image"
|
||
? "ruxa-image"
|
||
: candidate.provider.type === "fal-image"
|
||
? "fal-image"
|
||
: candidate.provider.type === "gemini"
|
||
? "gemini"
|
||
: candidate.provider.type === "huanke-gemini"
|
||
? "huanke-gemini"
|
||
: candidate.provider.type === "volcano-vision"
|
||
? "volcano"
|
||
: "alibaba-image";
|
||
|
||
const candidateMessages = await convertMessagesWithImages(
|
||
messagesForVolcano,
|
||
convertProvider
|
||
);
|
||
|
||
const candidateResult = await (candidateLanguageModel as any).doGenerate({
|
||
messages: candidateMessages,
|
||
imageParams:
|
||
effectiveImageParams || {
|
||
aspectRatio: defaultImageQualityParams.aspectRatio,
|
||
imageSize: defaultImageQualityParams.imageSize,
|
||
imageCount: defaultImageQualityParams.imageCount,
|
||
},
|
||
modelDisplayName: candidate.model.displayName,
|
||
overrideModelId: candidate.model.modelId,
|
||
});
|
||
|
||
if (candidateResult?.finishReason === "error") {
|
||
throw new Error(candidateResult.text || "图片生成失败");
|
||
}
|
||
|
||
imageMarkdown = candidateResult.text || "图片生成失败";
|
||
usedModelId = candidate.model.modelId;
|
||
fallbackSuccess = true;
|
||
console.log(
|
||
`[Chat API] ✅ Volcano cross-provider failover succeeded: ${candidate.model.displayName} (${candidate.provider.type})`
|
||
);
|
||
break;
|
||
} catch (crossError) {
|
||
const crossErrorMessage = normalizeFailoverErrorMessage(crossError);
|
||
if (shouldCooldownImageFallbackModel(crossErrorMessage)) {
|
||
setImageFallbackModelCooldown(candidate.model.id, crossErrorMessage);
|
||
}
|
||
console.error(
|
||
`[Chat API] ❌ Volcano cross-provider failover failed: ${candidate.model.displayName} (${candidate.provider.type})`,
|
||
crossError
|
||
);
|
||
}
|
||
}
|
||
}
|
||
if (!fallbackSuccess) {
|
||
throw primaryError;
|
||
}
|
||
}
|
||
console.log("[Chat API] Volcano Vision generated, length:", imageMarkdown.length);
|
||
console.log("[Chat API] Volcano Vision image markdown:", imageMarkdown.substring(0, 200));
|
||
|
||
// 生成消息 ID（图片消息）
|
||
const messageId = generateUUID();
|
||
|
||
// 写入图片消息到流
|
||
writer.write({
|
||
type: "data-appendMessage",
|
||
data: JSON.stringify({
|
||
id: messageId,
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: imageMarkdown }],
|
||
}),
|
||
});
|
||
|
||
// 🆕 图片生成结束标记（成功）
|
||
writer.write({
|
||
type: "data-appendMessage",
|
||
data: JSON.stringify({
|
||
id: generateUUID(),
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: "<!-- IMAGE_GENERATION_END -->" }],
|
||
}),
|
||
});
|
||
|
||
// 保存图片消息到数据库
|
||
try {
|
||
await saveMessages({
|
||
messages: [
|
||
{
|
||
id: messageId,
|
||
chatId: id,
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: imageMarkdown }],
|
||
attachments: [], // 必须提供 attachments 字段（数据库 notNull）
|
||
createdAt: new Date(),
|
||
videoStorageStatus: null,
|
||
temporaryVideoUrl: null,
|
||
permanentVideoUrl: null,
|
||
archivedAt: null,
|
||
videoMetadata: null,
|
||
},
|
||
],
|
||
});
|
||
console.log("[Chat API] ✅ Volcano Vision message saved successfully!");
|
||
|
||
// 提取用户 prompt
|
||
const lastUserMessage = uiMessages.filter((m) => m.role === "user").pop();
|
||
const userPrompt = lastUserMessage?.parts?.find((p): p is { type: "text"; text: string } => p.type === "text")?.text || "";
|
||
|
||
// 保存图片记录到 GeneratedImage 表
|
||
await saveGeneratedImages(
|
||
imageMarkdown,
|
||
session.user.id,
|
||
id,
|
||
messageId,
|
||
userPrompt,
|
||
finalModelId
|
||
);
|
||
} catch (saveError) {
|
||
console.error("[Chat API] ❌ Failed to save Volcano Vision message to database:", saveError);
|
||
console.error("[Chat API] Message content that failed:", imageMarkdown.substring(0, 300));
|
||
}
|
||
} catch (error) {
|
||
console.error("[Chat API] Volcano Vision generation error:", error);
|
||
const errorMessageId = generateUUID();
|
||
writer.write({
|
||
type: "data-appendMessage",
|
||
data: JSON.stringify({
|
||
id: errorMessageId,
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: "图片生成失败，请稍后重试" }],
|
||
}),
|
||
});
|
||
|
||
// 🆕 图片生成结束标记（失败）
|
||
writer.write({
|
||
type: "data-appendMessage",
|
||
data: JSON.stringify({
|
||
id: generateUUID(),
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: "<!-- IMAGE_GENERATION_END -->" }],
|
||
}),
|
||
});
|
||
|
||
// 尝试保存错误消息
|
||
try {
|
||
await saveMessages({
|
||
messages: [
|
||
{
|
||
id: errorMessageId,
|
||
chatId: id,
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: "图片生成失败，请稍后重试" }],
|
||
attachments: [], // 必须提供 attachments 字段
|
||
createdAt: new Date(),
|
||
videoStorageStatus: null,
|
||
temporaryVideoUrl: null,
|
||
permanentVideoUrl: null,
|
||
archivedAt: null,
|
||
videoMetadata: null,
|
||
},
|
||
],
|
||
});
|
||
} catch (saveError) {
|
||
console.error("[Chat API] Failed to save error message:", saveError);
|
||
}
|
||
}
|
||
},
|
||
});
|
||
|
||
return new Response(volcanoStream.pipeThrough(new JsonToSseTransformStream()));
|
||
}
|
||
// ========== 火山引擎视觉处理结束 ==========
|
||
|
||
// ========== 千问/FAL 视频生成独立处理 ==========
|
||
if (
|
||
["qianwen-video", "fal-video"].includes((languageModel as any).provider) &&
|
||
typeof (languageModel as any).doGenerate === "function"
|
||
) {
|
||
const currentVideoProvider = (languageModel as any).provider as "qianwen-video" | "fal-video";
|
||
console.log(`[Chat API] Using ${currentVideoProvider} video generation`);
|
||
const needsAsyncReplacement = currentVideoProvider === "qianwen-video";
|
||
const scheduleVideoUrlReplacement = needsAsyncReplacement
|
||
? (await import("@/lib/ai/providers/qianwen-video")).scheduleVideoUrlReplacement
|
||
: null;
|
||
|
||
const qianwenVideoStream = createUIMessageStream({
|
||
execute: async ({ writer }) => {
|
||
try {
|
||
const conversationMessage = smartAutoFriendlyMessage || null;
|
||
if (conversationMessage) {
|
||
const chatMessageId = generateUUID();
|
||
writer.write({
|
||
type: "data-appendMessage",
|
||
data: JSON.stringify({
|
||
id: chatMessageId,
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: conversationMessage }],
|
||
}),
|
||
});
|
||
|
||
await saveMessages({
|
||
messages: [
|
||
{
|
||
id: chatMessageId,
|
||
chatId: id,
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: conversationMessage }],
|
||
attachments: [],
|
||
createdAt: new Date(),
|
||
videoStorageStatus: null,
|
||
temporaryVideoUrl: null,
|
||
permanentVideoUrl: null,
|
||
archivedAt: null,
|
||
videoMetadata: null,
|
||
},
|
||
],
|
||
});
|
||
}
|
||
|
||
const messagesForVideo = injectHistoryImagesToLastUserMessage(
|
||
applyOptimizedPromptToLatestUserMessage(
|
||
uiMessagesForModel,
|
||
smartAutoOptimizedPrompt,
|
||
hiddenPromptPrefix,
|
||
),
|
||
smartAutoHistoryImages,
|
||
);
|
||
|
||
// 千问/FAL 视频直接使用原始 URL，不转换为 base64
|
||
const modelMessages = convertToModelMessages(messagesForVideo);
|
||
console.log(`[Chat API] ${currentVideoProvider} messages prepared, count:`, modelMessages.length);
|
||
|
||
// 调用 doGenerate 获取视频
|
||
const result = await (languageModel as any).doGenerate({
|
||
messages: modelMessages,
|
||
videoParams: effectiveVideoParams,
|
||
});
|
||
|
||
const videoMarkdown = result.text || "视频生成失败";
|
||
console.log(`[Chat API] ${currentVideoProvider} video generated, length:`, videoMarkdown.length);
|
||
|
||
// 提取临时视频 URL 和提示词（用于后续替换）
|
||
const videoUrlMatch = videoMarkdown.match(/<video\s+src="([^"]+)"/);
|
||
const tempVideoUrl = videoUrlMatch ? videoUrlMatch[1] : "";
|
||
const promptMatch = videoMarkdown.match(/\*\*提示词\*\*:\s*(.+)/);
|
||
const prompt = promptMatch ? promptMatch[1].trim() : "生成视频";
|
||
|
||
// 生成一个统一的消息 ID
|
||
const messageId = generateUUID();
|
||
|
||
// 写入消息到流
|
||
writer.write({
|
||
type: "data-appendMessage",
|
||
data: JSON.stringify({
|
||
id: messageId,
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: videoMarkdown }],
|
||
attachments: [],
|
||
}),
|
||
});
|
||
|
||
// 保存消息到数据库
|
||
try {
|
||
await saveMessages({
|
||
messages: [
|
||
{
|
||
id: messageId,
|
||
chatId: id,
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: videoMarkdown }],
|
||
attachments: [],
|
||
createdAt: new Date(),
|
||
videoStorageStatus: null,
|
||
temporaryVideoUrl: null,
|
||
permanentVideoUrl: null,
|
||
archivedAt: null,
|
||
videoMetadata: effectiveVideoParams
|
||
? {
|
||
provider: currentVideoProvider,
|
||
mode: effectiveVideoParams.mode,
|
||
resolution: effectiveVideoParams.resolution,
|
||
ratio: effectiveVideoParams.ratio,
|
||
duration: effectiveVideoParams.duration,
|
||
createdAt: Date.now(),
|
||
}
|
||
: null,
|
||
},
|
||
],
|
||
});
|
||
console.log(`[Chat API] ✅ ${currentVideoProvider} video message saved successfully!`);
|
||
|
||
// 启动异步视频替换任务（5分钟后执行）
|
||
if (needsAsyncReplacement && tempVideoUrl && scheduleVideoUrlReplacement) {
|
||
console.log("[Chat API] 🕐 Scheduling video URL replacement in 5 minutes...");
|
||
scheduleVideoUrlReplacement(id, messageId, tempVideoUrl, prompt);
|
||
}
|
||
} catch (saveError) {
|
||
console.error(`[Chat API] ❌ Failed to save ${currentVideoProvider} video message:`, saveError);
|
||
}
|
||
} catch (error) {
|
||
console.error(`[Chat API] ${currentVideoProvider} video generation error:`, error);
|
||
const errorMessageId = generateUUID();
|
||
writer.write({
|
||
type: "data-appendMessage",
|
||
data: JSON.stringify({
|
||
id: errorMessageId,
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: "视频生成失败，请稍后重试" }],
|
||
attachments: [],
|
||
}),
|
||
});
|
||
}
|
||
},
|
||
});
|
||
|
||
return new Response(qianwenVideoStream.pipeThrough(new JsonToSseTransformStream()));
|
||
}
|
||
// ========== 千问/FAL 视频处理结束 ==========
|
||
|
||
const stream = createUIMessageStream({
|
||
execute: async ({ writer: dataStream }) => {
|
||
// 构建工具对象，只包含启用的工具
|
||
const tools: any = {};
|
||
const activeTools: string[] = [];
|
||
|
||
if (enabledTools.includes("getWeather")) {
|
||
tools.getWeather = getWeather;
|
||
activeTools.push("getWeather");
|
||
}
|
||
if (enabledTools.includes("createDocument")) {
|
||
tools.createDocument = createDocument({ session, dataStream });
|
||
activeTools.push("createDocument");
|
||
}
|
||
if (enabledTools.includes("updateDocument")) {
|
||
tools.updateDocument = updateDocument({ session, dataStream });
|
||
activeTools.push("updateDocument");
|
||
}
|
||
if (enabledTools.includes("requestSuggestions")) {
|
||
tools.requestSuggestions = requestSuggestions({
|
||
session,
|
||
dataStream,
|
||
});
|
||
activeTools.push("requestSuggestions");
|
||
}
|
||
if (enabledTools.includes("generateVideo")) {
|
||
tools.generateVideo = generateVideo({ dataStream });
|
||
activeTools.push("generateVideo");
|
||
}
|
||
|
||
console.log("[Chat API] Calling streamText with model:", (languageModel as any).modelId, (languageModel as any).provider); // 调试日志
|
||
console.log("[Chat API] Provider type:", (languageModel as any).provider); // 调试日志
|
||
|
||
// 转换消息，对于 Gemini 和千问视频将图片 URL 转换为 base64
|
||
const convertedMessages = await convertMessagesWithImages(
|
||
injectHistoryImagesToLastUserMessage(uiMessagesForModel, smartAutoHistoryImages),
|
||
(languageModel as any).provider || ""
|
||
);
|
||
console.log("[Chat API] Messages converted, count:", convertedMessages.length); // 调试日志
|
||
|
||
// 🆕 检测是否是视频模型
|
||
const providerType = (languageModel as any).provider;
|
||
console.log("[Chat API] Provider type for tool detection:", providerType);
|
||
const isVolcanoVideoModel = providerType === "volcano-video" || actualModelId.includes("seedance");
|
||
const isOpenAICompatImageModel =
|
||
resolvedModelSupportsImageGen && providerType === "openai.chat";
|
||
console.log("[Chat API] Is volcano video model:", isVolcanoVideoModel);
|
||
|
||
// 🆕 处理 OpenAI 兼容的生图模型输出（防止返回 data:image;base64 原文）
|
||
if (isOpenAICompatImageModel) {
|
||
console.log("[Chat API] Using generateText for OpenAI-compatible image model");
|
||
|
||
// 生成开始标记
|
||
dataStream.write({
|
||
type: "data-appendMessage",
|
||
data: JSON.stringify({
|
||
id: generateUUID(),
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: "<!-- IMAGE_GENERATION_START -->" }],
|
||
}),
|
||
});
|
||
|
||
try {
|
||
const imageRequestTimeoutMs = 180_000;
|
||
const generated = await generateText({
|
||
model: languageModel,
|
||
system: systemPrompt({ selectedChatModel: finalModelId, requestHints }),
|
||
messages: convertedMessages,
|
||
abortSignal: AbortSignal.timeout(imageRequestTimeoutMs),
|
||
});
|
||
|
||
const rawText = generated.text?.trim() || "图片生成失败，请稍后重试";
|
||
const normalizedText = await normalizeAssistantImageMarkdown(
|
||
rawText,
|
||
effectiveImageParams?.imageSize || "2K"
|
||
);
|
||
|
||
const messageId = generateUUID();
|
||
dataStream.write({
|
||
type: "data-appendMessage",
|
||
data: JSON.stringify({
|
||
id: messageId,
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: normalizedText }],
|
||
}),
|
||
});
|
||
|
||
// 生成结束标记
|
||
dataStream.write({
|
||
type: "data-appendMessage",
|
||
data: JSON.stringify({
|
||
id: generateUUID(),
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: "<!-- IMAGE_GENERATION_END -->" }],
|
||
}),
|
||
});
|
||
|
||
await saveMessages({
|
||
messages: [
|
||
{
|
||
id: messageId,
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: normalizedText }],
|
||
createdAt: new Date(),
|
||
attachments: [],
|
||
chatId: id,
|
||
videoStorageStatus: null,
|
||
temporaryVideoUrl: null,
|
||
permanentVideoUrl: null,
|
||
archivedAt: null,
|
||
videoMetadata: null,
|
||
},
|
||
],
|
||
});
|
||
|
||
const latestUserMessage = uiMessages.filter((m) => m.role === "user").pop();
|
||
const userPrompt =
|
||
latestUserMessage?.parts?.find(
|
||
(p): p is { type: "text"; text: string } => p.type === "text"
|
||
)?.text || "";
|
||
await saveGeneratedImages(
|
||
normalizedText,
|
||
session.user.id,
|
||
id,
|
||
messageId,
|
||
userPrompt,
|
||
finalModelId
|
||
);
|
||
return;
|
||
} catch (error) {
|
||
console.error("[Chat API] OpenAI-compatible image model generation error:", error);
|
||
|
||
const errorMessageId = generateUUID();
|
||
const errorText =
|
||
error instanceof Error ? error.message : "图片生成失败，请稍后重试";
|
||
|
||
dataStream.write({
|
||
type: "data-appendMessage",
|
||
data: JSON.stringify({
|
||
id: errorMessageId,
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: errorText }],
|
||
}),
|
||
});
|
||
dataStream.write({
|
||
type: "data-appendMessage",
|
||
data: JSON.stringify({
|
||
id: generateUUID(),
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: "<!-- IMAGE_GENERATION_END -->" }],
|
||
}),
|
||
});
|
||
|
||
await saveMessages({
|
||
messages: [
|
||
{
|
||
id: errorMessageId,
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: errorText }],
|
||
createdAt: new Date(),
|
||
attachments: [],
|
||
chatId: id,
|
||
videoStorageStatus: null,
|
||
temporaryVideoUrl: null,
|
||
permanentVideoUrl: null,
|
||
archivedAt: null,
|
||
videoMetadata: null,
|
||
},
|
||
],
|
||
});
|
||
return;
|
||
}
|
||
}
|
||
|
||
// 🎬 视频模型使用专门的生成流程
|
||
if (isVolcanoVideoModel) {
|
||
console.log("[Chat API] Using video generation API");
|
||
|
||
// 导入视频生成函数
|
||
const { doGenerateStream } = await import("@/lib/ai/providers/volcano-video");
|
||
|
||
const conversationMessage = smartAutoFriendlyMessage || null;
|
||
if (conversationMessage) {
|
||
const chatMessageId = generateUUID();
|
||
dataStream.write({
|
||
type: "data-appendMessage",
|
||
data: JSON.stringify({
|
||
id: chatMessageId,
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: conversationMessage }],
|
||
attachments: [],
|
||
}),
|
||
});
|
||
|
||
await saveMessages({
|
||
messages: [
|
||
{
|
||
id: chatMessageId,
|
||
chatId: id,
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: conversationMessage }],
|
||
attachments: [],
|
||
createdAt: new Date(),
|
||
videoStorageStatus: null,
|
||
temporaryVideoUrl: null,
|
||
permanentVideoUrl: null,
|
||
archivedAt: null,
|
||
videoMetadata: null,
|
||
},
|
||
],
|
||
});
|
||
}
|
||
|
||
const messagesForVideo = injectHistoryImagesToLastUserMessage(
|
||
applyOptimizedPromptToLatestUserMessage(
|
||
uiMessagesForModel,
|
||
smartAutoOptimizedPrompt,
|
||
hiddenPromptPrefix,
|
||
),
|
||
smartAutoHistoryImages,
|
||
);
|
||
|
||
// 从最后一条用户消息中提取提示词和图片
|
||
const lastUserMessage = messagesForVideo.filter(m => m.role === "user").pop();
|
||
let prompt = "";
|
||
const imageUrls: string[] = [];
|
||
|
||
// 🐛 调试：打印原始消息结构
|
||
console.log("[Chat API] 🔍 原始 lastUserMessage:", JSON.stringify(lastUserMessage, null, 2));
|
||
|
||
if (lastUserMessage) {
|
||
// 🆕 处理 parts 数组（AI SDK v4 格式）
|
||
if ((lastUserMessage as any).parts && Array.isArray((lastUserMessage as any).parts)) {
|
||
console.log("[Chat API] 🔍 Found parts array, length:", (lastUserMessage as any).parts.length);
|
||
|
||
for (const part of (lastUserMessage as any).parts) {
|
||
console.log("[Chat API] 🔍 Processing part:", JSON.stringify(part, null, 2));
|
||
|
||
if (part.type === "text" && part.text) {
|
||
// 提取文本
|
||
prompt += part.text;
|
||
console.log("[Chat API] 🔍 ✅ Extracted text:", part.text);
|
||
} else if (part.type === "file" && part.url && part.mediaType?.startsWith("image/")) {
|
||
// 提取图片
|
||
imageUrls.push(part.url);
|
||
console.log("[Chat API] 🔍 ✅ Extracted image URL:", part.url);
|
||
}
|
||
}
|
||
}
|
||
// 备用方案1：content 字符串/数组
|
||
const legacyContent = (lastUserMessage as any).content;
|
||
if (typeof legacyContent === "string") {
|
||
prompt = legacyContent;
|
||
} else if (Array.isArray(legacyContent)) {
|
||
for (const part of legacyContent) {
|
||
if (typeof part === "string") {
|
||
prompt += part;
|
||
} else if (part.type === "text") {
|
||
prompt += part.text;
|
||
} else if (part.type === "image" && (part as any).image) {
|
||
imageUrls.push((part as any).image);
|
||
}
|
||
}
|
||
} else if ((lastUserMessage as any).experimental_attachments) {
|
||
// 备用方案2：experimental_attachments
|
||
for (const attachment of (lastUserMessage as any).experimental_attachments) {
|
||
if (attachment.url && attachment.contentType?.startsWith("image/")) {
|
||
imageUrls.push(attachment.url);
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
console.log("[Chat API] 🔍 最终提取结果 - prompt:", prompt, "imageCount:", imageUrls.length);
|
||
console.log("[Chat API] 🔍 imageUrls:", imageUrls);
|
||
|
||
try {
|
||
console.log("[Chat API] Video generation params:", {
|
||
prompt,
|
||
imageCount: imageUrls.length,
|
||
videoParams: effectiveVideoParams,
|
||
});
|
||
|
||
// 🆕 重新执行子模型选择逻辑，确定要使用的模型 UUID
|
||
let videoModelId = finalModelId; // 默认使用选择的模型 ID
|
||
|
||
// 检查是否是主模型，需要选择子模型
|
||
const selectedModelData = await getAiModelById({ id: finalModelId });
|
||
if (selectedModelData && !selectedModelData.isSubModel) {
|
||
// 是主模型，根据 mode 选择子模型
|
||
const videoSubModels = await getSubModelsByParentModelId(finalModelId);
|
||
console.log(`[Chat API] Video: Found ${videoSubModels.length} sub-models for master model`);
|
||
|
||
if (videoSubModels.length > 0) {
|
||
const videoMode = effectiveVideoParams?.mode || "text-to-video";
|
||
const modeToModelMap: Record<string, string> = {
|
||
"text-to-video": "lite-t2v",
|
||
"first-last-frame": "pro",
|
||
"first-frame": "pro-fast",
|
||
"reference-images": "lite-i2v",
|
||
};
|
||
|
||
const targetKey = modeToModelMap[videoMode];
|
||
const matchedModel = videoSubModels.find(({ model }) =>
|
||
model.modelId.toLowerCase().includes(targetKey)
|
||
);
|
||
|
||
if (matchedModel) {
|
||
videoModelId = matchedModel.model.id;
|
||
console.log(`[Chat API] Video: Selected sub-model ${matchedModel.model.displayName} for mode ${videoMode}`);
|
||
} else {
|
||
videoModelId = videoSubModels[0].model.id;
|
||
console.log(`[Chat API] Video: Using fallback sub-model ${videoSubModels[0].model.displayName}`);
|
||
}
|
||
}
|
||
}
|
||
|
||
// 构建视频生成选项
|
||
const videoOptions: any = {
|
||
model: actualModelId,
|
||
prompt: prompt || "生成视频",
|
||
mode: effectiveVideoParams?.mode || "text-to-video",
|
||
resolution: effectiveVideoParams?.resolution || "720p",
|
||
ratio: effectiveVideoParams?.ratio || "16:9",
|
||
duration: effectiveVideoParams?.duration || 5,
|
||
fps: effectiveVideoParams?.fps || 24,
|
||
cameraMove: effectiveVideoParams?.cameraMove || "none",
|
||
cameraFixed: effectiveVideoParams?.cameraFixed || false,
|
||
watermark: effectiveVideoParams?.watermark || false,
|
||
seed: effectiveVideoParams?.seed,
|
||
};
|
||
|
||
console.log("[Chat API] Video options constructed:", videoOptions);
|
||
|
||
// 根据模式添加图片
|
||
if (videoOptions.mode === "first-frame" && imageUrls[0]) {
|
||
videoOptions.firstFrameImage = imageUrls[0];
|
||
} else if (videoOptions.mode === "first-last-frame" && imageUrls.length >= 2) {
|
||
videoOptions.firstFrameImage = imageUrls[0];
|
||
videoOptions.lastFrameImage = imageUrls[1];
|
||
} else if (videoOptions.mode === "reference-images" && imageUrls.length > 0) {
|
||
videoOptions.referenceImages = imageUrls;
|
||
}
|
||
|
||
// 获取 provider 配置（使用确定的视频模型 UUID）
|
||
console.log("[Chat API] Getting model config for:", videoModelId);
|
||
const aiModelData = await getAiModelById({ id: videoModelId });
|
||
if (!aiModelData) {
|
||
throw new Error("视频模型配置未找到");
|
||
}
|
||
|
||
// 如果是子模型，获取父模型的 Provider 配置
|
||
let providerModelData = aiModelData;
|
||
if (aiModelData.isSubModel && aiModelData.parentModelId) {
|
||
console.log("[Chat API] This is a sub-model, getting parent model config:", aiModelData.parentModelId);
|
||
const parentModel = await getAiModelById({ id: aiModelData.parentModelId });
|
||
if (parentModel) {
|
||
providerModelData = parentModel;
|
||
}
|
||
}
|
||
|
||
console.log("[Chat API] Getting provider config for:", providerModelData.providerId);
|
||
const providerData = await getAiProviderById({ id: providerModelData.providerId });
|
||
if (!providerData) {
|
||
throw new Error("Provider 配置未找到");
|
||
}
|
||
|
||
const apiKey = providerData.apiKey;
|
||
const baseURL = providerData.baseUrl || "https://ark.cn-beijing.volces.com/api/v3";
|
||
|
||
console.log("[Chat API] Provider configured:", { baseURL, hasApiKey: !!apiKey });
|
||
|
||
// 调用视频生成流
|
||
console.log("[Chat API] Starting video generation stream...");
|
||
const videoStream = doGenerateStream(apiKey, baseURL, videoOptions);
|
||
|
||
// 收集所有流输出
|
||
console.log("[Chat API] Processing video stream...");
|
||
let fullText = "";
|
||
let chunkCount = 0;
|
||
|
||
for await (const chunk of videoStream) {
|
||
chunkCount++;
|
||
fullText += chunk;
|
||
console.log(`[Chat API] Received chunk ${chunkCount}, length: ${chunk.length}`);
|
||
}
|
||
|
||
console.log(`[Chat API] Video stream completed. Total chunks: ${chunkCount}, total text length: ${fullText.length}`);
|
||
|
||
// 生成消息 ID
|
||
const messageId = generateUUID();
|
||
|
||
// 一次性写入完整消息（与千问视频相同）
|
||
dataStream.write({
|
||
type: "data-appendMessage",
|
||
data: JSON.stringify({
|
||
id: messageId,
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: fullText }],
|
||
attachments: [],
|
||
}),
|
||
});
|
||
|
||
// 保存消息到数据库（提取视频URL用于归档）
|
||
try {
|
||
// 从 fullText 中提取视频URL
|
||
const videoUrlMatch = fullText.match(/<video[^>]+src="([^"]+)"/);
|
||
const temporaryVideoUrl = videoUrlMatch ? videoUrlMatch[1] : null;
|
||
|
||
// 构建视频元数据
|
||
const videoMetadata = temporaryVideoUrl ? {
|
||
provider: "volcano-video",
|
||
mode: effectiveVideoParams?.mode,
|
||
resolution: effectiveVideoParams?.resolution,
|
||
ratio: effectiveVideoParams?.ratio,
|
||
duration: effectiveVideoParams?.duration,
|
||
createdAt: Date.now(),
|
||
} : null;
|
||
|
||
await saveMessages({
|
||
messages: [{
|
||
id: messageId,
|
||
chatId: id,
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: fullText }],
|
||
attachments: [],
|
||
createdAt: new Date(),
|
||
videoStorageStatus: temporaryVideoUrl ? "temporary" : null,
|
||
temporaryVideoUrl: temporaryVideoUrl,
|
||
permanentVideoUrl: null,
|
||
archivedAt: null,
|
||
videoMetadata: videoMetadata,
|
||
}],
|
||
});
|
||
console.log("[Chat API] ✅ Video message saved successfully!");
|
||
if (temporaryVideoUrl) {
|
||
console.log("[Chat API] 📹 临时视频URL已保存，将在8小时后自动归档到S3");
|
||
}
|
||
} catch (saveError) {
|
||
console.error("[Chat API] ❌ Failed to save video message:", saveError);
|
||
}
|
||
|
||
console.log("[Chat API] Video generation finished successfully");
|
||
return;
|
||
} catch (error: any) {
|
||
console.error("[Chat API] ❌ Video generation error:", error);
|
||
console.error("[Chat API] Error stack:", error.stack);
|
||
|
||
// 生成错误消息 ID
|
||
const errorMessageId = generateUUID();
|
||
const errorMessage = `❌ 视频生成失败：${error.message}\n\n请检查配置或稍后重试。`;
|
||
|
||
// 使用正确的格式发送错误消息
|
||
dataStream.write({
|
||
type: "data-appendMessage",
|
||
data: JSON.stringify({
|
||
id: errorMessageId,
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: errorMessage }],
|
||
attachments: [],
|
||
}),
|
||
});
|
||
|
||
// 保存错误消息到数据库
|
||
try {
|
||
await saveMessages({
|
||
messages: [{
|
||
id: errorMessageId,
|
||
chatId: id,
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: errorMessage }],
|
||
attachments: [],
|
||
createdAt: new Date(),
|
||
videoStorageStatus: null,
|
||
temporaryVideoUrl: null,
|
||
permanentVideoUrl: null,
|
||
archivedAt: null,
|
||
videoMetadata: null,
|
||
}],
|
||
});
|
||
} catch (saveError) {
|
||
console.error("[Chat API] Failed to save error message:", saveError);
|
||
}
|
||
|
||
return;
|
||
}
|
||
}
|
||
|
||
// 常规文本模型使用 streamText
|
||
const streamRequestTimeoutMs = resolvedModelSupportsImageGen ? 180_000 : 90_000;
|
||
const shouldDisableSmoothStream =
|
||
resolvedModelSupportsImageGen || isVolcanoVideoModel;
|
||
|
||
const result = streamText({
|
||
model: languageModel,
|
||
system: systemPrompt({ selectedChatModel: finalModelId, requestHints }),
|
||
messages: convertedMessages,
|
||
stopWhen: stepCountIs(5),
|
||
experimental_activeTools:
|
||
finalModelId === "chat-model-reasoning" || isVolcanoVideoModel ? [] : activeTools,
|
||
experimental_transform: shouldDisableSmoothStream
|
||
? undefined
|
||
: smoothStream({ chunking: "word" }),
|
||
tools: isVolcanoVideoModel ? {} : tools, // 🆕 视频模型禁用工具
|
||
abortSignal: AbortSignal.timeout(streamRequestTimeoutMs),
|
||
experimental_telemetry: {
|
||
isEnabled: isProductionEnvironment,
|
||
functionId: "stream-text",
|
||
},
|
||
onFinish: async ({ usage }) => {
|
||
try {
|
||
const providers = await getTokenlensCatalog();
|
||
if (!actualModelId) {
|
||
finalMergedUsage = usage;
|
||
dataStream.write({
|
||
type: "data-usage",
|
||
data: finalMergedUsage,
|
||
});
|
||
return;
|
||
}
|
||
|
||
if (!providers) {
|
||
finalMergedUsage = usage;
|
||
dataStream.write({
|
||
type: "data-usage",
|
||
data: finalMergedUsage,
|
||
});
|
||
return;
|
||
}
|
||
|
||
const summary = getUsage({
|
||
modelId: actualModelId,
|
||
usage,
|
||
providers,
|
||
});
|
||
finalMergedUsage = {
|
||
...usage,
|
||
...summary,
|
||
modelId: actualModelId,
|
||
} as AppUsage;
|
||
dataStream.write({ type: "data-usage", data: finalMergedUsage });
|
||
} catch (err) {
|
||
console.warn("TokenLens enrichment failed", err);
|
||
finalMergedUsage = usage;
|
||
dataStream.write({ type: "data-usage", data: finalMergedUsage });
|
||
}
|
||
},
|
||
});
|
||
|
||
result.consumeStream();
|
||
|
||
dataStream.merge(
|
||
result.toUIMessageStream({
|
||
sendReasoning: true,
|
||
})
|
||
);
|
||
},
|
||
generateId: generateUUID,
|
||
onFinish: async ({ messages }) => {
|
||
await saveMessages({
|
||
messages: messages.map((currentMessage) => ({
|
||
id: currentMessage.id,
|
||
role: currentMessage.role,
|
||
parts: currentMessage.parts,
|
||
createdAt: new Date(),
|
||
attachments: [],
|
||
chatId: id,
|
||
videoStorageStatus: null,
|
||
temporaryVideoUrl: null,
|
||
permanentVideoUrl: null,
|
||
archivedAt: null,
|
||
videoMetadata: null,
|
||
})),
|
||
});
|
||
|
||
// 检测助手消息中是否包含视频，如果有则启动异步替换任务
|
||
for (const msg of messages) {
|
||
if (msg.role === "assistant" && msg.parts) {
|
||
const textContent = msg.parts
|
||
.filter((part: any) => part && part.type === "text" && part.text)
|
||
.map((part: any) => part.text)
|
||
.join("");
|
||
|
||
// 检测是否包含视频标签
|
||
const videoUrlMatch = textContent?.match(/<video[^>]+src="([^"]+)"/);
|
||
if (videoUrlMatch) {
|
||
const tempVideoUrl = videoUrlMatch[1];
|
||
const promptMatch = textContent?.match(/\*\*提示词\*\*:\s*(.+)/);
|
||
const prompt = promptMatch ? promptMatch[1].trim() : "生成视频";
|
||
|
||
console.log("[Chat API] 检测到视频消息，启动URL替换任务");
|
||
console.log("[Chat API] 临时视频URL:", tempVideoUrl);
|
||
|
||
// 动态导入并启动替换任务（5分钟后执行）
|
||
import("@/lib/ai/providers/qianwen-video").then(({ scheduleVideoUrlReplacement }) => {
|
||
scheduleVideoUrlReplacement(id, msg.id, tempVideoUrl, prompt);
|
||
}).catch((err) => {
|
||
console.error("[Chat API] 启动视频替换任务失败:", err);
|
||
});
|
||
}
|
||
}
|
||
}
|
||
|
||
if (finalMergedUsage) {
|
||
try {
|
||
await updateChatLastContextById({
|
||
chatId: id,
|
||
context: finalMergedUsage,
|
||
});
|
||
} catch (err) {
|
||
console.warn("Unable to persist last usage for chat", id, err);
|
||
}
|
||
}
|
||
},
|
||
onError: () => {
|
||
return "Oops, an error occurred!";
|
||
},
|
||
});
|
||
|
||
// const streamContext = getStreamContext();
|
||
|
||
// if (streamContext) {
|
||
// return new Response(
|
||
// await streamContext.resumableStream(streamId, () =>
|
||
// stream.pipeThrough(new JsonToSseTransformStream())
|
||
// )
|
||
// );
|
||
// }
|
||
|
||
return new Response(stream.pipeThrough(new JsonToSseTransformStream()));
|
||
} catch (error) {
|
||
const vercelId = request.headers.get("x-vercel-id");
|
||
|
||
if (error instanceof ChatSDKError) {
|
||
return error.toResponse();
|
||
}
|
||
|
||
// Check for Vercel AI Gateway credit card error
|
||
if (
|
||
error instanceof Error &&
|
||
error.message?.includes(
|
||
"AI Gateway requires a valid credit card on file to service requests"
|
||
)
|
||
) {
|
||
return new ChatSDKError("bad_request:activate_gateway").toResponse();
|
||
}
|
||
|
||
console.error("Unhandled error in chat API:", error, { vercelId });
|
||
return new ChatSDKError("offline:chat").toResponse();
|
||
}
|
||
}
|
||
|
||
export async function DELETE(request: Request) {
|
||
const { searchParams } = new URL(request.url);
|
||
const id = searchParams.get("id");
|
||
|
||
if (!id) {
|
||
return new ChatSDKError("bad_request:api").toResponse();
|
||
}
|
||
|
||
const session = await auth();
|
||
|
||
if (!session?.user) {
|
||
return new ChatSDKError("unauthorized:chat").toResponse();
|
||
}
|
||
|
||
const chat = await getChatById({ id });
|
||
|
||
if (chat?.userId !== session.user.id) {
|
||
return new ChatSDKError("forbidden:chat").toResponse();
|
||
}
|
||
|
||
const deletedChat = await deleteChatById({ id });
|
||
|
||
return Response.json(deletedChat, { status: 200 });
|
||
}
|