import { geolocation } from "@vercel/functions"; import { convertToModelMessages, createUIMessageStream, generateText, JsonToSseTransformStream, smoothStream, stepCountIs, streamText, } from "ai"; import { eq, sql } from "drizzle-orm"; import { unstable_cache as cache } from "next/cache"; import { after, NextResponse } from "next/server"; import { createResumableStreamContext, type ResumableStreamContext, } from "resumable-stream"; import type { ModelCatalog } from "tokenlens/core"; import { fetchModels } from "tokenlens/fetch"; import { getUsage } from "tokenlens/helpers"; import { auth, type UserType } from "@/app/(auth)/auth"; import type { VisibilityType } from "@/components/visibility-selector"; import { entitlementsByUserType } from "@/lib/ai/entitlements"; import { BoxCutoutServiceError, executeBoxCutoutJob, } from "@/lib/ai/box-cutout/service"; import { executeExtendJob, ExtendServiceError, inferExtendOptionsFromText, } from "@/lib/ai/extend/service"; import type { ChatModel } from "@/lib/ai/models"; import { type RequestHints, systemPrompt } from "@/lib/ai/prompts"; import { myProvider } from "@/lib/ai/providers"; import { executeResizeJob, inferResizeOptionsFromText, ResizeServiceError, } from "@/lib/ai/resize/service"; import { executeRmbgJob, fetchImageBufferFromUrl, RmbgServiceError, } from "@/lib/ai/rmbg/service"; import { getLanguageModelById } from "@/lib/ai/dynamic-provider"; import { createDocument } from "@/lib/ai/tools/create-document"; import { getWeather } from "@/lib/ai/tools/get-weather"; import { requestSuggestions } from "@/lib/ai/tools/request-suggestions"; import { updateDocument } from "@/lib/ai/tools/update-document"; import { executeDirectVideoGeneration, generateVideo, } from "@/lib/ai/tools/generate-video"; import { getAgentRequiredUploadCount, normalizeAgentPresetModeConfig, } from "@/lib/agents/preset-mode-config"; import { getEnabledTools } from "@/lib/ai/tools-registry"; import { isProductionEnvironment } from "@/lib/constants"; import { createStreamId, deleteChatById, getChatById, getPointsSystemConfig, getUserById, getMessageCountByUserId, getMessagesByChatId, saveChat, saveMessages, updateMessageById, updateChatLastContextById, getAgentAiModelWithProvider, getAiModelById, getEnabledAiModelsWithProvider, getSubModelsByParentModelId, getAiProviderById, checkUserImageQuota, consumeUserPoints, createAgentPresetExecutionSnapshot, createMessageTargetResolution, createMediaAssetRelation, getAccessibleAgentPresetById, getConversationWorkingStateByChatId, getMediaAssetsByUserAndUrls, upsertMediaAsset, } from "@/lib/db/queries"; import { db } from "@/lib/db"; import type { DBMessage } from "@/lib/db/schema"; import { agentPreset as agentPresetTable, generatedImage, message as messageTable, } from "@/lib/db/schema"; import { generateCanvasImage } from "@/lib/canvas/generate-image"; import { detectRequestedStoryboardGrid, isGridConfirmationReply, type PendingGridConfirmation, } from "@/lib/ai/conversation-action-resolver"; import { isImageGenerationModelDescriptor, isVideoModelDescriptor, } from "@/lib/ai/model-classification"; import { bootstrapConversationStateProtocol, composeConversationStateProtocol, persistAgentPresetTurn, persistClarificationTurn, setConversationStateProtocolEventObserver, persistResolvedTargetTurn, persistImageWorkflowTurn, persistPromptOnlyTurn, resolveGridConfirmationRequest, resolveTargetExecutionOverride, persistStoryboardConfirmedTurn, persistStoryboardExecutionCompletedTurn, persistStoryboardExecutionStartedTurn, persistStoryboardRevisionTurn, persistStoryboardAssistantTurn, persistVideoGenerationCompletedTurn, persistVideoGenerationFailedTurn, persistVideoGenerationStartedTurn, persistWorkflowCancelledTurn, } from "@/lib/ai/conversation-state-protocol"; import { resolveChatMediaInput } from "@/lib/ai/context-memory/media-input"; import { resolveTargetAssets } from "@/lib/ai/context-memory/target-resolution"; import type { ResolvedChatMediaInput, ResolvedTargetAssets, } from "@/lib/ai/context-memory/types"; import { ChatSDKError } from "@/lib/errors"; import { extractImageUrlsFromText } from "@/lib/image-url"; import { resolveClarificationBypass, resolveClarificationMessage, } from "@/lib/ai/runtime/clarification-interruption"; import { pruneTaskBoundaryContext, pruneTaskBoundaryHistoryMessages, } from "@/lib/ai/runtime/task-boundary-context-pruner"; import { resolveAgentModelSlotDescriptor, upsertAgentModelSlot, type AgentModelSlots, } from "@/lib/ai/runtime/agent-model-slots"; import { isPureGreetingTurnRequest } from "@/lib/ai/runtime/task-boundary-classifier"; import { getProviderHealthSnapshot, recordProviderHealthFailure, recordProviderHealthSuccess, shouldAvoidProvider, } from "@/lib/ai/runtime/provider-health"; import { arbitrateAgentIntent } from "@/lib/ai/runtime/intent-arbiter"; import { buildAgentRoleIdentityInstruction } from "@/lib/ai/runtime/agent-role-profile"; import { uploadFile } from "@/lib/storage/service"; import { getVideoStoryboardConfig } from "@/lib/video-agent/config"; import { runRuxaNineGridPipeline } from "@/lib/video-agent/pipeline/ruxa-nine-grid-pipeline"; import { buildStoryboardConfirmedAck, executeStoryboardConfirmedWorkflow, handleStoryboardWorkflowFollowup, } from "@/lib/video-agent/storyboard-chat-followup"; import { buildStoryboardPreferenceMemoryUpdate, } from "@/lib/video-agent/storyboard-preference-memory"; import { attachVideoStoryboardTaskMessage, getLatestActiveVideoStoryboardTaskByChatId, markVideoStoryboardAwaitingConfirmation, markVideoStoryboardConfirmed, markVideoStoryboardDraftForRevision, cancelVideoStoryboardTaskByUser, } from "@/lib/video-agent/task-store"; import type { StoryboardModelCandidate } from "@/lib/video-agent/types"; import type { ChatMessage } from "@/lib/types"; import type { AppUsage } from "@/lib/usage"; import { convertToUIMessages, generateUUID } from "@/lib/utils"; import { isAiModelAgentAuto } from "@/lib/db/ai-model-flags"; import { generateTitleFromUserMessage } from "../../actions"; import { type PostRequestBody, postRequestBodySchema } from "./schema"; import { isAiModelMultimodal } from "@/lib/db/ai-model-flags"; /** * 将图片 URL 转换为 base64 格式 * 用于将 S3 URL 的图片转换为 base64 发送给 Gemini API */ async function convertImageUrlToBase64(url: string): Promise { try { console.log("[Image Convert] Fetching image from:", url); const response = await fetch(url); if (!response.ok) { throw new Error(`Failed to fetch image: ${response.statusText}`); } const arrayBuffer = await response.arrayBuffer(); const buffer = Buffer.from(arrayBuffer); const base64 = buffer.toString("base64"); // 获取 MIME 类型 const contentType = response.headers.get("content-type") || "image/jpeg"; const dataUrl = `data:${contentType};base64,${base64}`; console.log("[Image Convert] Converted to base64, size:", base64.length); return dataUrl; } catch (error) { console.error("[Image Convert] Failed:", error); throw error; } } /** * 检测消息中是否包含用户标注过的图片（文件名以 annotated- 开头） */ function hasAnnotatedImage(messages: ChatMessage[]): boolean { for (const message of messages) { for (const part of message.parts) { if (part.type === "file" && "name" in part && typeof part.name === "string") { if (part.name.startsWith("annotated-")) { return true; } } } } return false; } /** * 标注图片的系统提示词（用户不可见，后台自动添加） */ const ANNOTATION_SYSTEM_PROMPT = "【重要】用户提供的图片上有标注线条（如箭头、圆圈、画笔痕迹或文字说明），这些标注是用户用来指示需要修改的区域或方向。请仔细理解标注的含义，按照标注指示进行编辑，但在生成的新图片中不要保留这些标注线条，输出干净的编辑结果。"; function applyHiddenPromptToMessages(messages: ChatMessage[], hiddenPrompt?: string): ChatMessage[] { if (!hiddenPrompt || !hiddenPrompt.trim()) return messages; const cloned = messages.map((msg) => ({ ...msg, parts: msg.parts.map((part) => ({ ...part })), })); for (let i = cloned.length - 1; i >= 0; i--) { const msg = cloned[i]; if (msg.role !== "user") continue; const textPart = msg.parts.find((part) => part.type === "text") as { type: "text"; text: string } | undefined; if (textPart) { textPart.text = `${hiddenPrompt}\n\n${textPart.text}`.trim(); } else { msg.parts.unshift({ type: "text", text: hiddenPrompt }); } break; } return cloned; } function applyOptimizedPromptToLatestUserMessage( messages: ChatMessage[], optimizedPrompt?: string, hiddenPromptPrefix = "", ): ChatMessage[] { const prompt = optimizedPrompt?.trim(); if (!prompt) return messages; const cloned = messages.map((msg) => ({ ...msg, parts: msg.parts.map((part) => ({ ...part })), })); for (let i = cloned.length - 1; i >= 0; i -= 1) { const msg = cloned[i]; if (msg.role !== "user") continue; const textPart = msg.parts.find( (part): part is { type: "text"; text: string } => part.type === "text", ); const originalText = textPart?.text?.trim() || ""; const { normalizedOriginal, hasOriginalDemand, hasOriginalInOptimized } = analyzeOptimizedPromptMerge(originalText, prompt, hiddenPromptPrefix); const finalPrompt = hasOriginalDemand && !hasOriginalInOptimized ? [ hiddenPromptPrefix.trim(), `【核心目标（必须优先满足）】${normalizedOriginal}`, `【执行优化细节】${prompt}`, ] .filter(Boolean) .join("\n\n") .trim() : `${hiddenPromptPrefix}${prompt}`.trim(); console.log("[Prompt Merge] Final image prompt prepared:", { original: normalizedOriginal.substring(0, 120), optimized: prompt.substring(0, 120), merged: finalPrompt.substring(0, 180), forcePreserveOriginal: hasOriginalDemand && !hasOriginalInOptimized, }); if (textPart) { textPart.text = finalPrompt; } else { msg.parts.unshift({ type: "text", text: finalPrompt }); } break; } return cloned; } function analyzeOptimizedPromptMerge( originalText: string, optimizedPrompt?: string, hiddenPromptPrefix = "", ) { const prompt = optimizedPrompt?.trim() || ""; const trimmedOriginal = String(originalText || "").trim(); const normalizedOriginal = hiddenPromptPrefix && trimmedOriginal.startsWith(hiddenPromptPrefix) ? trimmedOriginal.slice(hiddenPromptPrefix.length).trim() : trimmedOriginal; const hasOriginalDemand = normalizedOriginal.length > 0; const hasOriginalInOptimized = hasOriginalDemand && (prompt.includes(normalizedOriginal) || normalizedOriginal.includes(prompt)); return { prompt, normalizedOriginal, hasOriginalDemand, hasOriginalInOptimized, }; } function buildOptimizedPromptPreview(params: { originalText: string; optimizedPrompt?: string; hiddenPromptPrefix?: string; }): string | undefined { const { prompt, normalizedOriginal, hasOriginalDemand, hasOriginalInOptimized } = analyzeOptimizedPromptMerge( params.originalText, params.optimizedPrompt, params.hiddenPromptPrefix || "", ); if (!prompt) return undefined; const normalizedPrompt = prompt.replace(/\s+/g, " ").trim(); if (!normalizedPrompt) return undefined; if (hasOriginalDemand && !hasOriginalInOptimized) { return [ `- 核心目标：${normalizedOriginal.replace(/\s+/g, " ").trim()}`, `- 优化提示词：${normalizedPrompt}`, ].join("\n"); } return `- ${normalizedPrompt}`; } function injectOptimizedPromptSection(params: { message?: string; originalText: string; optimizedPrompt?: string; hiddenPromptPrefix?: string; }): string | undefined { const { message, originalText, optimizedPrompt, hiddenPromptPrefix } = params; if (!message?.trim()) return message; const preview = buildOptimizedPromptPreview({ originalText, optimizedPrompt, hiddenPromptPrefix, }); if (!preview) return message; const optimizedPromptSection = `【优化后提示词】\n${preview}`; const optimizedSectionRegex = /(?:###\s*优化后提示词|【\s*优化后提示词\s*】)[\s\S]*?(?=\n(?:###\s*|【\s*[^】]+\s*】)|$)/; if (optimizedSectionRegex.test(message)) { return message.replace(optimizedSectionRegex, optimizedPromptSection); } const executionSectionRegex = /(?:###\s*执行计划|【\s*执行计划\s*】)[\s\S]*?(?=\n(?:###\s*|【\s*[^】]+\s*】)|$)/; if (executionSectionRegex.test(message)) { return message.replace(executionSectionRegex, optimizedPromptSection); } const outputSectionRegex = /(?:###\s*输出设置|【\s*输出设置\s*】)/; if (outputSectionRegex.test(message)) { return message.replace(outputSectionRegex, `${optimizedPromptSection}\n$&`); } return `${message.trim()}\n\n${optimizedPromptSection}`; } type StreamWriter = { write: (chunk: any) => void; }; function safeWriteStreamChunk(writer: StreamWriter, chunk: any, label: string): boolean { try { writer.write(chunk); return true; } catch (error) { console.warn(`[Chat API] Stream write skipped (${label})`, error); return false; } } function writeAssistantTextChunk( writer: StreamWriter, text: string, label: string, messageId = generateUUID(), ) { safeWriteStreamChunk( writer, { type: "data-appendMessage", data: JSON.stringify({ id: messageId, role: "assistant", parts: [{ type: "text", text }], }), }, label, ); return messageId; } function writeStructuredDataChunk( writer: StreamWriter, type: string, data: unknown, label: string, ) { safeWriteStreamChunk( writer, { type, data: JSON.stringify(data), }, label, ); } function writeProtocolDebugChunk( writer: StreamWriter, enabled: boolean, data: unknown, label: string, ) { if (!enabled) return; writeStructuredDataChunk(writer, "data-protocolDebug", data, label); } function normalizeImageLoadingProviderType(providerType?: string | null): string { const normalized = String(providerType || "").trim().toLowerCase(); if (normalized === "gemini-image" || normalized === "huanke-gemini") { return "gemini"; } if ( normalized === "alibaba-image" || normalized === "bfl-image" || normalized === "ruxa-image" || normalized === "fal-image" || normalized === "openai-image" || normalized === "midjourney" || normalized === "runway" || normalized === "volcano-vision" || normalized === "gemini" ) { return normalized; } return "alibaba-image"; } function writeImageGenerationStatusChunk( writer: StreamWriter, params: { phase: "queued" | "processing" | "finalizing" | "failed" | "completed"; providerType?: string | null; modelName?: string | null; imageCount?: number | null; }, label: string, ) { writeStructuredDataChunk( writer, "data-imageGenerationStatus", { phase: params.phase, providerType: normalizeImageLoadingProviderType(params.providerType), modelName: String(params.modelName || "").trim() || undefined, imageCount: Number.isFinite(Number(params.imageCount)) && Number(params.imageCount) > 0 ? Number(params.imageCount) : undefined, }, label, ); } function writeCanvasMediaChunk( writer: StreamWriter, params: { messageId: string; text?: string; imageUrls?: string[]; videoUrls?: string[]; prompt?: string | null; sourceImageUrls?: string[] | null; agentPresetId?: string | null; agentPresetName?: string | null; agentCategoryId?: string | null; agentCategoryName?: string | null; agentExecutionSnapshotId?: string | null; }, label: string, ) { const imageUrls = Array.isArray(params.imageUrls) ? params.imageUrls : extractImageUrlsFromText(String(params.text || "")); const videoUrls = Array.isArray(params.videoUrls) ? params.videoUrls : []; const normalizedImageUrls = Array.from( new Set( imageUrls .map((url) => String(url || "").trim()) .filter((url) => /^https?:\/\//i.test(url)), ), ); const normalizedVideoUrls = Array.from( new Set( videoUrls .map((url) => String(url || "").trim()) .filter((url) => /^https?:\/\//i.test(url)), ), ); if (normalizedImageUrls.length === 0 && normalizedVideoUrls.length === 0) { return; } writeStructuredDataChunk( writer, "data-canvasMedia", { messageId: params.messageId, imageUrls: normalizedImageUrls, videoUrls: normalizedVideoUrls, prompt: String(params.prompt || "").trim() || undefined, sourceImageUrls: Array.isArray(params.sourceImageUrls) ? params.sourceImageUrls .map((url) => String(url || "").trim()) .filter((url) => /^https?:\/\//i.test(url)) : [], agentPresetId: String(params.agentPresetId || "").trim() || undefined, agentPresetName: String(params.agentPresetName || "").trim() || undefined, agentCategoryId: String(params.agentCategoryId || "").trim() || undefined, agentCategoryName: String(params.agentCategoryName || "").trim() || undefined, agentExecutionSnapshotId: String(params.agentExecutionSnapshotId || "").trim() || undefined, }, label, ); } function createSseResponse(stream: ReadableStream) { return new Response(stream.pipeThrough(new JsonToSseTransformStream()), { headers: { "Content-Type": "text/event-stream; charset=utf-8", "Cache-Control": "no-cache, no-transform", Connection: "keep-alive", "X-Accel-Buffering": "no", }, }); } function startStreamKeepalive(writer: StreamWriter, label: string, intervalMs = 15000) { const timer = setInterval(() => { safeWriteStreamChunk( writer, { type: "data-keepalive", data: JSON.stringify({ ts: Date.now(), label }), }, `${label}:keepalive`, ); }, intervalMs); return () => clearInterval(timer); } type VideoModeType = "text-to-video" | "first-frame" | "first-last-frame" | "reference-images"; function isVideoProviderType( providerType?: string | null, model?: { modelId?: string | null; capabilities?: unknown; supportsImageGen?: boolean | null; } ): boolean { return isVideoModelDescriptor({ providerType, modelId: model?.modelId, capabilities: model?.capabilities, supportsImageGen: model?.supportsImageGen, }); } function isImageGenerationModelType( providerType?: string | null, model?: { modelId?: string | null; capabilities?: unknown; supportsImageGen?: boolean | null; } ): boolean { return isImageGenerationModelDescriptor({ providerType, modelId: model?.modelId, capabilities: model?.capabilities, supportsImageGen: model?.supportsImageGen, }); } function inferSupportedVideoModesByModelId(modelId: string): VideoModeType[] { const normalized = modelId.toLowerCase(); if ( normalized === "gen4.5" || normalized === "gen4_turbo" || normalized === "gen4_aleph" || normalized === "gen3a_turbo" || normalized === "veo3" || normalized === "veo3.1" || normalized === "veo3.1_fast" ) { return ["text-to-video", "first-frame"]; } if (normalized.includes("lite-t2v")) { return ["text-to-video"]; } if (normalized.includes("lite-i2v")) { return ["first-frame", "first-last-frame", "reference-images"]; } if (normalized.includes("pro-fast")) { return ["text-to-video", "first-frame"]; } if (normalized.includes("-pro-") || normalized.endsWith("-pro")) { return ["text-to-video", "first-frame", "first-last-frame"]; } return ["text-to-video", "first-frame", "first-last-frame", "reference-images"]; } function getSupportedVideoModesForModel(model: any): VideoModeType[] { const capabilityModes = (((model?.capabilities as any)?.supportedModes ?? []) as string[]) .filter((mode): mode is VideoModeType => mode === "text-to-video" || mode === "first-frame" || mode === "first-last-frame" || mode === "reference-images" ); if (capabilityModes.length > 0) { return capabilityModes; } const modelId = String(model?.modelId || ""); return inferSupportedVideoModesByModelId(modelId); } function scoreVideoSubModelForMode(modelId: string, mode: VideoModeType): number { const normalized = modelId.toLowerCase(); const isProFast = normalized.includes("pro-fast"); const isLiteT2V = normalized.includes("lite-t2v"); const isLiteI2V = normalized.includes("lite-i2v"); const isPro = (normalized.includes("-pro-") || normalized.endsWith("-pro")) && !isProFast; const isV15Pro = normalized.includes("seedance-1-5-pro"); switch (mode) { case "first-last-frame": if (isV15Pro) return 150; if (isPro) return 120; if (isLiteI2V) return 90; if (isProFast) return -1000; if (isLiteT2V) return -1000; return 10; case "first-frame": if (isV15Pro) return 140; if (isProFast) return 130; if (isPro) return 120; if (isLiteI2V) return 100; if (isLiteT2V) return 20; return 10; case "reference-images": if (isV15Pro) return 150; if (isLiteI2V) return 140; if (isPro) return 110; if (isProFast || isLiteT2V) return -1000; return 10; case "text-to-video": default: if (isV15Pro) return 140; if (isLiteT2V) return 130; if (isProFast) return 120; if (isPro) return 100; if (isLiteI2V) return 80; return 10; } } function selectBestVideoSubModelForMode( subModels: Array<{ model: any }>, mode: VideoModeType, ): { model: any } | null { const compatible = subModels.filter(({ model }) => { const supportedModes = getSupportedVideoModesForModel(model); return supportedModes.includes(mode); }); const candidates = compatible.length > 0 ? compatible : subModels; if (candidates.length === 0) return null; return [...candidates].sort((a, b) => { const scoreA = scoreVideoSubModelForMode(String(a.model?.modelId || ""), mode); const scoreB = scoreVideoSubModelForMode(String(b.model?.modelId || ""), mode); return scoreB - scoreA; })[0]; } async function resolveStoryboardModelCandidates( selectedModelRef: string ): Promise { const selectedModel = await getAiModelById({ id: selectedModelRef }).catch(() => undefined); if (!selectedModel) return []; const configuredFallbackIds = [ selectedModel.fallbackModelId1, selectedModel.fallbackModelId2, ].filter((item): item is string => Boolean(String(item || "").trim())); const allModels = await getEnabledAiModelsWithProvider(true, true).catch(() => []); const sameProvider = allModels .filter( ({ model, provider }: any) => provider?.id === selectedModel.providerId && model?.id !== selectedModel.id && model?.isEnabled ) .map(({ model }: any) => String(model?.id || "").trim()) .filter(Boolean); const candidateIds = Array.from( new Set([selectedModel.id, ...configuredFallbackIds, ...sameProvider]) ); const byId = new Map( allModels .map(({ model }: any) => model) .filter(Boolean) .map((model: any) => [String(model.id), model] as const) ); return candidateIds .map((id) => byId.get(id)) .filter(Boolean) .map((model: any) => ({ id: String(model.id), modelId: String(model.modelId || "").trim(), displayName: String(model.displayName || model.modelId || "").trim(), })) .filter((item) => item.modelId); } function extractVideoUrlFromText(text: string): string | null { const normalizedText = String(text || ""); if (!normalizedText) return null; const htmlMatch = normalizedText.match(/]+src="([^"]+)"/i); if (htmlMatch?.[1]) return htmlMatch[1]; const markdownMatch = normalizedText.match( /\[[^\]]*]\((https?:\/\/[^)\s]+\.mp4(?:\?[^)\s]*)?)\)/i, ); if (markdownMatch?.[1]) return markdownMatch[1]; const plainMatch = normalizedText.match(/(https?:\/\/[^\s"'<>]+\.mp4(?:\?[^\s"'<>]+)?)/i); return plainMatch?.[1] || null; } function isAlreadyArchivedVideoUrl(url: string): boolean { return /\/uploads\//i.test(url); } function scheduleVideoArchive(params: { chatId: string; messageId: string; text: string; temporaryVideoUrl: string; taskId?: string; metadata?: Record | null; }) { const { chatId, messageId, temporaryVideoUrl, taskId, metadata } = params; if (!temporaryVideoUrl || isAlreadyArchivedVideoUrl(temporaryVideoUrl)) { return; } const delayHoursRaw = Number(process.env.VIDEO_ARCHIVE_DELAY_HOURS || 48); const delayHours = Number.isFinite(delayHoursRaw) ? Math.max(1, Math.min(24 * 30, Math.round(delayHoursRaw))) : 48; const archiveAfterAtMs = Date.now() + delayHours * 60 * 60 * 1000; after(async () => { try { const nextMetadata = { ...(metadata || {}), temporaryVideoUrl, storageStatus: "temporary", archiveMode: "delayed_job", archiveAfterAt: archiveAfterAtMs, archiveDelayHours: delayHours, archiveTaskId: taskId || messageId, updatedAt: Date.now(), }; await updateMessageById({ messageId, chatId, videoStorageStatus: "temporary", temporaryVideoUrl, permanentVideoUrl: null, archivedAt: null, videoMetadata: nextMetadata as any, }); console.log("[Chat API] 🕒 已登记延迟归档视频:", { messageId, archiveAfterAt: new Date(archiveAfterAtMs).toISOString(), }); } catch (error) { console.error("[Chat API] ❌ 登记视频延迟归档失败（保留临时地址）:", { messageId, temporaryVideoUrl, error, }); } }); } async function replayAssistantMessagesForDuplicateRequest({ chatId, userMessageId, writer, source, waitMs = DUPLICATE_REQUEST_WAIT_MS, }: { chatId: string; userMessageId: string; writer: StreamWriter; source: string; waitMs?: number; }) { const startedAt = Date.now(); const seenAssistantMessageIds = new Set(); while (Date.now() - startedAt < waitMs) { try { const dbMessages = await getMessagesByChatId({ id: chatId }); const userMessageIndex = dbMessages.findIndex((msg) => msg.id === userMessageId); if (userMessageIndex >= 0) { const assistantMessages = dbMessages .slice(userMessageIndex + 1) .filter((msg) => msg.role === "assistant"); if (assistantMessages.length > 0) { for (const assistantMessage of assistantMessages) { if (seenAssistantMessageIds.has(assistantMessage.id)) continue; seenAssistantMessageIds.add(assistantMessage.id); safeWriteStreamChunk( writer, { type: "data-appendMessage", data: JSON.stringify({ id: assistantMessage.id, role: "assistant", parts: assistantMessage.parts, }), }, `${source}:replay`, ); } return; } } } catch (error) { console.warn(`[Chat API] Failed duplicate replay fetch (${source})`, error); } safeWriteStreamChunk( writer, { type: "data-keepalive", data: JSON.stringify({ ts: Date.now(), source }), }, `${source}:keepalive`, ); await sleep(1000); } } type ChatImageParams = NonNullable; type AgentRecommendedImageParams = { aspectRatio?: string; imageSize?: string; imageCount?: number; negativePrompt?: string; promptExtend?: boolean; }; type ChatVideoParams = NonNullable; type ChatVideoParamsDraft = Partial; type AgentRecommendedVideoParams = { mode?: "text-to-video" | "first-frame" | "first-last-frame" | "reference-images"; resolution?: "480p" | "720p" | "1080p"; ratio?: string; duration?: number; generateAudio?: boolean; cameraMove?: string; }; type AgentVideoPlan = { script: string; storyboard: Array<{ shot: number; title: string; duration: number; description: string; prompt: string; camera?: string; }>; finalPrompt: string; negativePrompt?: string; recommendedVideoParams?: AgentRecommendedVideoParams; }; type AgentPlanNextAction = | "respond" | "clarify" | "analyze_image" | "generate_image" | "edit_image" | "generate_video"; type AgentPlannedToolCall = { name: string; reason?: string; args?: Record; }; type AgentExecutionPlan = { goal: string; nextAction: AgentPlanNextAction; intent: "image_edit" | "vision" | "image_gen" | "video_gen" | "chat"; subIntent?: string; targetSource: | "explicit_selection" | "current_upload" | "text_reference" | "working_memory" | "latest_result" | "clarification" | "none"; preferredCapability: | "respond" | "analyze" | "generic_edit" | "remove_background" | "enhance" | "extend" | "resize" | "box_cutout" | "generate_image" | "generate_video" | "publish_canvas"; responseMode: "direct_answer" | "tool_then_answer" | "tool_then_execute" | "clarify"; shouldUseHistoryImage: boolean; shouldUseStructuredReply: boolean; confidence: number; toolCalls: AgentPlannedToolCall[]; }; type AgentPreparedExecution = { mode: AgentPlanNextAction; routeIntent: "image_edit" | "vision" | "image_gen" | "video_gen" | "chat"; routeSubIntent?: string; targetSource: AgentExecutionPlan["targetSource"]; preferredCapability: AgentExecutionPlan["preferredCapability"]; shouldUseStructuredReply: boolean; shouldOptimizePrompt: boolean; shouldApplyImagePipeline: boolean; shouldApplyVideoPipeline: boolean; shouldStayConversational: boolean; shouldPublishToCanvas: boolean; toolCalls: AgentPlannedToolCall[]; availableTools: string[]; }; type DirectAgentToolCapability = | "remove_background" | "resize" | "extend" | "box_cutout" | "generate_image" | "generate_video"; type DirectAgentToolName = | "removeBackground" | "resizeImage" | "extendImage" | "boxCutout" | "generateImage" | "generateVideo"; type DirectAgentSourceToolName = | "useSelectedCanvasImage" | "useLatestResultImage" | "useOriginalUploadImage"; type ResolvedDirectAgentToolExecution = { toolName: DirectAgentToolName; capability: DirectAgentToolCapability; sourceTools: DirectAgentSourceToolName[]; publishToCanvas: boolean; }; function collectAgentToolCalls( execution?: AgentPreparedExecution, plan?: AgentExecutionPlan, ): AgentPlannedToolCall[] { const merged = [...(execution?.toolCalls || []), ...(plan?.toolCalls || [])]; const seen = new Set(); const result: AgentPlannedToolCall[] = []; for (const tool of merged) { const name = String(tool?.name || "").trim(); if (!name || seen.has(name)) continue; seen.add(name); result.push(tool); } return result; } function mapDirectToolNameToCapability( toolName: DirectAgentToolName, ): DirectAgentToolCapability { switch (toolName) { case "removeBackground": return "remove_background"; case "resizeImage": return "resize"; case "extendImage": return "extend"; case "boxCutout": return "box_cutout"; case "generateImage": return "generate_image"; case "generateVideo": return "generate_video"; default: return "generate_image"; } } function resolveDirectAgentToolExecution( execution?: AgentPreparedExecution, plan?: AgentExecutionPlan, ): ResolvedDirectAgentToolExecution | null { const mergedToolCalls = collectAgentToolCalls(execution, plan); const toolNames = mergedToolCalls.map((tool) => tool.name); const sourceTools = toolNames.filter((toolName): toolName is DirectAgentSourceToolName => toolName === "useSelectedCanvasImage" || toolName === "useLatestResultImage" || toolName === "useOriginalUploadImage", ); const directToolName = toolNames.find((toolName): toolName is DirectAgentToolName => toolName === "removeBackground" || toolName === "resizeImage" || toolName === "extendImage" || toolName === "boxCutout" || toolName === "generateImage" || toolName === "generateVideo", ) || (() => { if ( execution?.preferredCapability === "remove_background" || execution?.routeSubIntent === "remove_background" ) { return "removeBackground"; } if ( execution?.preferredCapability === "resize" || execution?.routeSubIntent === "resize" ) { return "resizeImage"; } if ( execution?.preferredCapability === "extend" || execution?.routeSubIntent === "extend" ) { return "extendImage"; } if ( execution?.preferredCapability === "box_cutout" || execution?.routeSubIntent === "box_cutout" ) { return "boxCutout"; } if (execution?.mode === "generate_image") { return "generateImage"; } if (execution?.mode === "generate_video") { return "generateVideo"; } return null; })(); if (!directToolName) { return null; } return { toolName: directToolName, capability: mapDirectToolNameToCapability(directToolName), sourceTools, publishToCanvas: execution?.shouldPublishToCanvas === true || toolNames.includes("publishToCanvas"), }; } function getDirectAgentToolLabel(capability: DirectAgentToolCapability): string { switch (capability) { case "remove_background": return "Agent工具：抠图"; case "resize": return "Agent工具：尺寸调整"; case "extend": return "Agent工具：图像拓展"; case "box_cutout": return "Agent工具：框选抠图"; case "generate_image": return "Agent工具：图片生成"; case "generate_video": return "Agent工具：视频生成"; default: return "Agent工具"; } } function findAgentToolCallArgs( toolName: AgentPlannedToolCall["name"], execution?: AgentPreparedExecution, plan?: AgentExecutionPlan, ): Record | undefined { const matchedTool = execution?.toolCalls?.find((tool) => tool.name === toolName) || plan?.toolCalls?.find((tool) => tool.name === toolName); if (!matchedTool?.args || typeof matchedTool.args !== "object") { return undefined; } return matchedTool.args; } function toFiniteNumber(value: unknown): number | undefined { const normalized = Number(value); return Number.isFinite(normalized) ? normalized : undefined; } function getDirectAgentToolSuccessMessage(capability: DirectAgentToolCapability): string { switch (capability) { case "remove_background": return "已完成抠图处理，结果如下："; case "resize": return "已完成尺寸调整，结果如下："; case "extend": return "已完成图像拓展，结果如下："; case "box_cutout": return "已完成框选抠图，结果如下："; case "generate_image": return "已完成图片生成，结果如下："; case "generate_video": return "已完成视频生成，结果如下："; default: return "已完成处理，结果如下："; } } function getDirectAgentToolTargetMissingMessage( capability: DirectAgentToolCapability, ): string { switch (capability) { case "remove_background": return "请先上传或选中一张图片，我才能继续帮你执行抠图。"; case "resize": return "请先上传或选中一张图片，我才能继续帮你调整尺寸。"; case "extend": return "请先上传或选中一张图片，我才能继续帮你做图像拓展。"; case "box_cutout": return "请先选中一张图片并提供框选区域，我才能继续帮你做框选抠图。"; case "generate_image": return "请先告诉我你想生成什么，或者提供参考图。"; case "generate_video": return "请先告诉我你想生成什么视频，必要时再提供参考图。"; default: return "请先提供要处理的图片。"; } } function getDirectAgentToolErrorMessage( capability: DirectAgentToolCapability, error: unknown, ): string { const errorMessage = error instanceof Error ? String(error.message || "").trim() : String(error || "").trim(); const errorDetail = error && typeof error === "object" && "detail" in error ? String((error as { detail?: unknown }).detail || "").trim() : ""; const detail = errorDetail || errorMessage || "请稍后重试。"; switch (capability) { case "remove_background": return `抠图暂时失败：${detail}`; case "resize": return `尺寸调整暂时失败：${detail}`; case "extend": return `图像拓展暂时失败：${detail}`; case "box_cutout": return `框选抠图暂时失败：${detail}`; case "generate_image": return `图片生成暂时失败：${detail}`; case "generate_video": return `视频生成暂时失败：${detail}`; default: return `处理暂时失败：${detail}`; } } function getChatStreamErrorMessage(error: unknown): string { const rawMessage = error instanceof Error ? String(error.message || "").trim() : String(error || "").trim(); const normalized = rawMessage.toLowerCase(); if ( normalized.includes("token quota is not enough") || normalized.includes("pre_consume_token_quota_failed") || normalized.includes("quota is not enough") ) { return "当前对话模型额度不足，请切换其他模型，或联系管理员补充该模型额度。"; } if ( normalized.includes("timeout") || normalized.includes("timed out") || normalized.includes("body timeout") ) { return "当前对话模型响应超时，请重试，或切换到其他可用模型。"; } if (rawMessage) { return `当前对话暂时失败：${rawMessage}`; } return "当前对话暂时失败，请重试或切换其他模型。"; } function normalizeAssistantPartsForPersistence(parts: Array> | undefined) { const safeParts = Array.isArray(parts) ? parts.filter(Boolean) : []; return safeParts.filter((part) => { const partType = String(part?.type || "").trim(); if (!partType) return false; if (partType === "data-keepalive" || partType === "step-start" || partType === "step-finish") { return false; } return true; }); } function buildAgentToolImageMarkdown(params: { originalUrl: string; thumbnailUrl?: string | null; imageSize?: string; }): string { const originalUrl = String(params.originalUrl || "").trim(); const thumbnailUrl = String(params.thumbnailUrl || originalUrl).trim() || originalUrl; const imageSize = String(params.imageSize || "2K").trim() || "2K"; return `![thumb|||${originalUrl}|||${imageSize}](${thumbnailUrl})`; } function buildAgentToolImagesMarkdown( items: Array<{ originalUrl: string; thumbnailUrl?: string | null; imageSize?: string }>, ) { return items .map((item) => buildAgentToolImageMarkdown({ originalUrl: item.originalUrl, thumbnailUrl: item.thumbnailUrl, imageSize: item.imageSize, }), ) .join("\n"); } function buildAgentToolVideoMarkdown(videoUrl: string) { const normalizedUrl = String(videoUrl || "").trim(); if (!normalizedUrl) return ""; return [ ``, `[下载视频](${normalizedUrl})`, ].join("\n\n"); } function pickAgentSourceImageUrls(params: { sourceTools: DirectAgentSourceToolName[]; requestResolvedMediaInput?: ResolvedChatMediaInput | null; requestTargetResolution?: ResolvedTargetAssets | null; requestWorkingState?: { lastResolvedTargetImageUrls?: string[] | null; lastGeneratedImageUrls?: string[] | null; lastUploadedImageUrls?: string[] | null; } | null; }): string[] { const preferredBuckets: string[][] = []; const sourceTools = params.sourceTools || []; if (sourceTools.includes("useSelectedCanvasImage")) { preferredBuckets.push(params.requestResolvedMediaInput?.selectedImageUrls || []); } if (sourceTools.includes("useLatestResultImage")) { preferredBuckets.push(params.requestWorkingState?.lastGeneratedImageUrls || []); preferredBuckets.push(params.requestResolvedMediaInput?.historyAssistantImageUrls || []); } if (sourceTools.includes("useOriginalUploadImage")) { preferredBuckets.push(params.requestResolvedMediaInput?.currentMessageImageUrls || []); preferredBuckets.push(params.requestWorkingState?.lastUploadedImageUrls || []); preferredBuckets.push(params.requestResolvedMediaInput?.historyUserImageUrls || []); } preferredBuckets.push( params.requestTargetResolution?.targetImageUrls || [], params.requestResolvedMediaInput?.selectedImageUrls || [], params.requestResolvedMediaInput?.currentMessageImageUrls || [], params.requestWorkingState?.lastResolvedTargetImageUrls || [], params.requestWorkingState?.lastGeneratedImageUrls || [], params.requestResolvedMediaInput?.historyAssistantImageUrls || [], params.requestResolvedMediaInput?.historyUserImageUrls || [], ); return Array.from( new Set( preferredBuckets .flat() .map((url) => String(url || "").trim()) .filter((url) => /^https?:\/\//i.test(url)), ), ); } const IMAGE_FAILOVER_PROVIDER_ALLOWLIST = new Set([ "alibaba-image", "bfl-image", "ruxa-image", "fal-image", "openai-image", "runway", "volcano-vision", "gemini", "huanke-gemini", ]); const IMAGE_FAILOVER_MODEL_COOLDOWN_MS = 30 * 60 * 1000; const imageFailoverModelCooldown = new Map(); function normalizeFailoverErrorMessage(error: unknown): string { if (!error) return ""; if (typeof error === "string") return error; if (error instanceof Error) return error.message || String(error); try { return JSON.stringify(error); } catch { return String(error); } } function shouldCooldownImageFallbackModel(errorMessage: string): boolean { const msg = errorMessage.toLowerCase(); return ( msg.includes("model not found") || msg.includes("model_not_found") || msg.includes("no available channel for model") || msg.includes("unknown model") || msg.includes("invalid model") || msg.includes("api key is missing") || msg.includes("invalid api key") || msg.includes("unauthorized") || msg.includes("cannot convert argument to a bytestring") ); } function isImageFallbackModelCoolingDown(modelId: string): boolean { const snapshot = imageFailoverModelCooldown.get(modelId); if (!snapshot) return false; if (snapshot.until <= Date.now()) { imageFailoverModelCooldown.delete(modelId); return false; } return true; } function setImageFallbackModelCooldown(modelId: string, reason: string): void { imageFailoverModelCooldown.set(modelId, { until: Date.now() + IMAGE_FAILOVER_MODEL_COOLDOWN_MS, reason: reason.slice(0, 500), }); } function isImageFailoverExcludedModel(model: { excludeFromFailover?: boolean } | null | undefined) { return Boolean(model?.excludeFromFailover); } function buildImageFallbackModelIds( allModels: Array<{ model: any; provider: any }>, options: { currentProviderId?: string; currentModelId?: string; currentModelKey?: string; configuredFallbackIds?: string[]; } ) { const { currentProviderId, currentModelId, currentModelKey, configuredFallbackIds = [] } = options; const eligibleFallbackModelIds = new Set( allModels .filter( (item) => item.model.supportsImageGen && item.model.isEnabled && !item.model.isSubModel && !isImageFailoverExcludedModel(item.model) && item.model.modelId !== currentModelKey && item.model.id !== currentModelId ) .map((item) => String(item.model.modelId || "")) .filter(Boolean) ); const sameProviderFallbacks = currentProviderId ? allModels .filter( (item) => item.provider.id === currentProviderId && item.model.supportsImageGen && item.model.isEnabled && !item.model.isSubModel && !isImageFailoverExcludedModel(item.model) && item.model.modelId !== currentModelKey && item.model.id !== currentModelId ) .map((item) => item.model.modelId) : []; return Array.from(new Set([...configuredFallbackIds, ...sameProviderFallbacks])).filter( (modelId) => Boolean(modelId) && eligibleFallbackModelIds.has(String(modelId)) ); } function getImageCrossProviderCandidates( allModels: any[], options: { currentModelId: string; currentProviderId?: string; providerPriority: Record; } ) { return allModels .filter((m) => { if ( !m?.model?.supportsImageGen || !m?.model?.isEnabled || m?.model?.isSubModel || isImageFailoverExcludedModel(m?.model) ) { return false; } if (!IMAGE_FAILOVER_PROVIDER_ALLOWLIST.has(String(m?.provider?.type || ""))) { return false; } if (m.model.id === options.currentModelId) return false; if (options.currentProviderId && m.provider.id === options.currentProviderId) return false; if (isImageFallbackModelCoolingDown(m.model.id)) { const cooldown = imageFailoverModelCooldown.get(m.model.id); console.log( `[Chat API] ⏭ Skip image failover model on cooldown: ${m.model.displayName} (${m.provider.type}) - ${cooldown?.reason || "cooldown"}` ); return false; } return true; }) .sort((a, b) => { const left = options.providerPriority[a.provider.type as string] ?? 99; const right = options.providerPriority[b.provider.type as string] ?? 99; if (left !== right) return left - right; return (a.model.sortOrder ?? 0) - (b.model.sortOrder ?? 0); }); } function hasUserCustomizedImageParams(params?: PostRequestBody["imageParams"]): boolean { if (!params) return false; const customizedAspectRatio = typeof params.aspectRatio === "string" && params.aspectRatio.trim() && params.aspectRatio !== "auto"; const customizedImageSize = typeof params.imageSize === "string" && params.imageSize.trim() && params.imageSize.toUpperCase() !== "2K"; const customizedImageCount = typeof params.imageCount === "number" && params.imageCount !== 1; const customizedPromptExtend = typeof params.promptExtend === "boolean"; const customizedAdvanced = Boolean( params.size || params.seed !== undefined || params.n || params.negativePrompt || customizedPromptExtend ); return customizedAspectRatio || customizedImageSize || customizedImageCount || customizedAdvanced; } function mergeImageParamsWithAgentRecommendation(params: { incoming?: PostRequestBody["imageParams"]; recommended?: AgentRecommendedImageParams; intent?: "image_edit" | "image_gen" | "vision" | "video_gen" | "chat"; hasUserCustomSettings: boolean; }): PostRequestBody["imageParams"] { const { incoming, recommended, intent, hasUserCustomSettings } = params; const base: ChatImageParams = { ...(incoming || {}) }; if (!recommended) { return Object.keys(base).length > 0 ? base : undefined; } const merged: ChatImageParams = { ...base, negativePrompt: typeof recommended.negativePrompt === "string" && recommended.negativePrompt.trim() ? recommended.negativePrompt.trim() : base.negativePrompt, promptExtend: typeof recommended.promptExtend === "boolean" ? recommended.promptExtend : base.promptExtend, }; if (!hasUserCustomSettings) { if (recommended.aspectRatio) merged.aspectRatio = recommended.aspectRatio; if (recommended.imageSize) merged.imageSize = recommended.imageSize; if (typeof recommended.imageCount === "number") merged.imageCount = recommended.imageCount; } if (intent === "image_edit") { merged.imageCount = 1; if (typeof base.promptExtend !== "boolean") { merged.promptExtend = false; } } return Object.keys(merged).length > 0 ? merged : undefined; } function hasUserCustomizedVideoParams(params?: PostRequestBody["videoParams"]): boolean { if (!params) return false; const defaults = { mode: "text-to-video", resolution: "720p", ratio: "16:9", duration: 5, generateAudio: true, cameraMove: "none", } as const; if (params.mode && params.mode !== defaults.mode) return true; if (params.resolution && params.resolution !== defaults.resolution) return true; if (params.ratio && params.ratio !== defaults.ratio) return true; if (typeof params.duration === "number" && params.duration !== defaults.duration) return true; if (typeof params.generateAudio === "boolean" && params.generateAudio !== defaults.generateAudio) return true; if (typeof params.cameraMove === "string" && params.cameraMove !== defaults.cameraMove) return true; if (typeof params.seed === "number") return true; return false; } function mergeVideoParamsWithAgentRecommendation(params: { incoming?: PostRequestBody["videoParams"]; recommended?: AgentRecommendedVideoParams; hasUserCustomSettings: boolean; }): PostRequestBody["videoParams"] { const { incoming, recommended, hasUserCustomSettings } = params; const base: ChatVideoParamsDraft = { ...(incoming || {}) }; if (!recommended) { return Object.keys(base).length > 0 ? (base as ChatVideoParams) : undefined; } const merged: ChatVideoParamsDraft = { ...base }; if (!hasUserCustomSettings) { if (recommended.mode) merged.mode = recommended.mode; if (recommended.resolution) merged.resolution = recommended.resolution; if (recommended.ratio) merged.ratio = recommended.ratio; if (typeof recommended.duration === "number") merged.duration = recommended.duration; if (typeof recommended.generateAudio === "boolean") { merged.generateAudio = recommended.generateAudio; } if (recommended.cameraMove) merged.cameraMove = recommended.cameraMove; } if (!merged.mode) { merged.mode = recommended.mode || (incoming?.mode as ChatVideoParams["mode"]) || "text-to-video"; } if (!merged.resolution) { merged.resolution = recommended.resolution || (incoming?.resolution as ChatVideoParams["resolution"]) || "720p"; } if (!merged.ratio) { merged.ratio = recommended.ratio || incoming?.ratio || "16:9"; } if (typeof merged.duration !== "number") { merged.duration = recommended.duration || incoming?.duration || 5; } return Object.keys(merged).length > 0 ? (merged as ChatVideoParams) : undefined; } function sanitizeAgentOutputSettingsSection(params: { message?: string; userImageParams?: PostRequestBody["imageParams"]; hasUserCustomImageParams: boolean; }): string | undefined { const { message, userImageParams, hasUserCustomImageParams } = params; if (!message?.trim()) return message; const outputSectionRegex = /(?:###\s*输出设置|【\s*输出设置\s*】)[\s\S]*?(?=\n(?:###\s*|【\s*[^】]+\s*】)|$)/; if (!outputSectionRegex.test(message)) return message; if (!hasUserCustomImageParams) { return message.replace( outputSectionRegex, "【输出设置】\n- 参数：按用户面板当前设置执行（未自动改写）" ); } const lines: string[] = []; const aspectRatio = typeof userImageParams?.aspectRatio === "string" ? userImageParams.aspectRatio.trim() : ""; const imageSize = typeof userImageParams?.imageSize === "string" ? userImageParams.imageSize.trim() : ""; const imageCount = typeof userImageParams?.imageCount === "number" ? userImageParams.imageCount : undefined; if (aspectRatio && aspectRatio !== "auto") { lines.push(`- 宽高比：${aspectRatio}（来自用户面板设置）`); } if (imageSize && imageSize.toUpperCase() !== "2K") { lines.push(`- 清晰度档位：${imageSize.toUpperCase()}（来自用户面板设置）`); } if (typeof imageCount === "number" && Number.isFinite(imageCount) && imageCount !== 1) { lines.push(`- 出图数量：${Math.max(1, Math.round(imageCount))} 张（来自用户面板设置）`); } lines.push("- 其余参数：保持用户面板当前设置"); return message.replace(outputSectionRegex, `【输出设置】\n${lines.join("\n")}`); } function normalizeAgentReportHeadingLine(line: string): string { const trimmed = line.trim(); if (!trimmed) return line; const markdownHeading = trimmed.match(/^#{1,6}\s*(.+?)\s*$/); const plainTitle = markdownHeading?.[1]?.trim() || trimmed; const normalizedTitle = plainTitle .replace(/^[^\u4e00-\u9fa5A-Za-z0-9]+/, "") .trim(); const knownTitles = [ "需求理解", "图片分析", "执行计划", "优化后提示词", "输出设置", "脚本规划", "分镜规划", "执行说明", "视频生成完成", ]; if (knownTitles.some((title) => normalizedTitle === title)) { return `【${normalizedTitle}】`; } if (markdownHeading) { return `【${normalizedTitle || plainTitle}】`; } return line; } function stripAgentFriendlyMediaEmbeds(message: string): string { if (!message) return message; const withoutInlineMedia = message .replace(/!\[[^\]]*]\((https?:\/\/[^\s)]+)\)/gi, "") .replace(//gi, "") .replace(/]*>/gi, ""); const cleanedLines = withoutInlineMedia .split("\n") .filter((line) => { const trimmed = line.trim(); if (!trimmed) return true; if (/^https?:\/\/\S+$/i.test(trimmed)) return false; if (/^\[[^\]]*]\(https?:\/\/[^\s)]+\)$/i.test(trimmed)) return false; return true; }) .join("\n"); return cleanedLines.replace(/\n{3,}/g, "\n\n").trim(); } function normalizeAgentReportForDisplay(message?: string): string | undefined { if (!message?.trim()) return message; const stripped = stripAgentFriendlyMediaEmbeds(message); if (!stripped) return undefined; return stripped .split("\n") .map((line) => normalizeAgentReportHeadingLine(line)) .join("\n"); } function guessImageMediaType(url: string): string { const cleanUrl = url.split("?")[0]?.toLowerCase() || ""; if (cleanUrl.endsWith(".jpg") || cleanUrl.endsWith(".jpeg")) return "image/jpeg"; if (cleanUrl.endsWith(".webp")) return "image/webp"; if (cleanUrl.endsWith(".gif")) return "image/gif"; if (cleanUrl.endsWith(".bmp")) return "image/bmp"; if (cleanUrl.endsWith(".svg")) return "image/svg+xml"; return "image/png"; } function injectHistoryImagesToLastUserMessage( messages: ChatMessage[], historyImages: string[] ): ChatMessage[] { if (!historyImages || historyImages.length === 0) return messages; let lastUserIndex = -1; for (let i = messages.length - 1; i >= 0; i--) { if (messages[i]?.role === "user") { lastUserIndex = i; break; } } if (lastUserIndex < 0) return messages; const lastUserMessage = messages[lastUserIndex]; const hasExistingImage = lastUserMessage.parts?.some( (p: any) => p.type === "file" || p.type === "image" || p.type === "image_url" ); if (hasExistingImage) return messages; const imageParts = historyImages.map((url, index) => ({ type: "file" as const, url, name: `history-image-${index + 1}`, mediaType: guessImageMediaType(url), mimeType: guessImageMediaType(url), })); const updatedMessage: ChatMessage = { ...lastUserMessage, parts: [...lastUserMessage.parts, ...imageParts], }; return messages.map((msg, idx) => (idx === lastUserIndex ? updatedMessage : msg)); } function compactAutoDemand(text: string, maxLength = 56): string { const normalized = text.replace(/\s+/g, " ").trim(); if (!normalized) return "按你刚才的描述执行"; if (normalized.length <= maxLength) return normalized; return `${normalized.slice(0, maxLength)}...`; } function buildAutoModeFallbackMessage(params: { text: string; hasImage: boolean; fromHistory: boolean; imageSummary?: string; }): string { const demand = compactAutoDemand(params.text); if (params.hasImage) { const sourceLine = params.fromHistory ? "基于上一轮图片继续处理。" : "基于当前上传图片处理。"; return [ "### 需求理解", `- ${demand}`, "### 图片分析", params.imageSummary ? `- ${params.imageSummary.replace(/\s+/g, " ").trim()}` : `- ${sourceLine}`, "- 我会优先保持主体、构图和整体风格一致，只处理你指定的改动。", "### 执行计划", "- 先完成本轮核心改动，再检查边缘与光影一致性。", "- 若结果偏差，我会按“保留A，修改B”的方式继续迭代。", "### 输出设置", "- 默认单图输出，优先稳定命中。", ].join("\n"); } return [ "### 需求理解", `- ${demand}`, "### 执行计划", "- 先锁定主体与场景，再细化风格、材质与光影。", "### 输出设置", "- 默认单图输出，优先稳定命中。", ].join("\n"); } function sanitizePromptOnlyUserIntent(text: string): string { return String(text || "") .replace(/先?分析(?:一下)?(?:这张)?(?:图片|图像|这张图)?/g, "") .replace(/帮我|给我|先给我|只给我|仅给我/g, "") .replace(/优化(?:一下)?提示词|提示词优化|提示词|prompt/gi, "") .replace(/不要生图|先别生图|不需要生图|不要生成|别生成|不直接生成/g, "") .replace(/[，。；：,.!！?？]/g, " ") .replace(/\s+/g, " ") .trim(); } function buildPromptOnlyOptimizationSeed(params: { text: string; imageSummary?: string; }): string { const cleanedIntent = sanitizePromptOnlyUserIntent(params.text); const summary = String(params.imageSummary || "").replace(/\s+/g, " ").trim(); if (cleanedIntent && summary) { return `${cleanedIntent}。参考图关键特征：${summary}`; } if (summary) { return `基于参考图生成一段可直接复现核心主体、场景、风格与光线特征的中文提示词。参考图关键特征：${summary}`; } if (cleanedIntent) { return cleanedIntent; } return "基于当前参考图整理一段可直接使用的中文提示词。"; } function buildPromptOnlyReply(params: { userText: string; imageSummary?: string; optimizedPrompt: string; }): string { const demand = sanitizePromptOnlyUserIntent(params.userText) || "你希望我先分析这张图并整理可直接使用的提示词"; const summaryLines = String(params.imageSummary || "") .split(/\r?\n/) .map((line) => line.replace(/\s+/g, " ").trim()) .filter(Boolean); const formattedSummary = summaryLines.length > 0 ? summaryLines.map((line) => `- ${line}`).join("\n") : "- 我已基于当前图片梳理主体、场景、风格与光线特征。"; return [ "【需求理解】", `- ${demand}`, "【图片分析】", formattedSummary, "【优化后提示词】", `- ${params.optimizedPrompt.replace(/\s+/g, " ").trim()}`, "【使用建议】", "- 本轮仅输出优化后的提示词，不直接执行生图。", ].join("\n"); } function hasStructuredAgentSections(message?: string): boolean { if (!message) return false; return /(?:###\s*需求理解|【\s*需求理解\s*】|###\s*图片分析|【\s*图片分析\s*】|###\s*执行计划|【\s*执行计划\s*】|###\s*优化后提示词|【\s*优化后提示词\s*】)/.test( message, ); } function ensureStructuredAgentFriendlyMessage(params: { message?: string; text: string; hasImage: boolean; fromHistory: boolean; }): string { const existingMessage = String(params.message || "").trim(); if (hasStructuredAgentSections(existingMessage)) { return existingMessage; } const fallback = buildAutoModeFallbackMessage({ text: params.text, hasImage: params.hasImage, fromHistory: params.fromHistory, }); if (!existingMessage) { return fallback; } const normalizedSupplement = existingMessage.replace(/\s+/g, " ").trim(); if (!normalizedSupplement) { return fallback; } return fallback.replace( "### 执行计划\n", `### 执行计划\n- 补充说明：${normalizedSupplement}\n`, ); } function extractMessageImageUrls(parts: any[]): string[] { return (parts || []) .map((part: any) => { if (!part) return null; if (part.type === "file") return part.url || part.data || null; if (part.type === "image") return part.url || part.image || null; if (part.type === "image_url") return part.image_url?.url || null; return null; }) .filter((url: any): url is string => typeof url === "string" && /^https?:\/\//i.test(url)); } function extractMessageImageAssetIds(parts: any[]): string[] { return Array.from( new Set( (parts || []) .map((part: any) => { if (!part || typeof part !== "object") return null; return typeof part.assetId === "string" && part.assetId.trim() ? part.assetId.trim() : null; }) .filter((assetId: any): assetId is string => typeof assetId === "string" && assetId.length > 0), ), ); } function shouldEnrichImageAnalysisText(message?: string): boolean { if (!message) return false; if (!/(###\s*图片分析|【\s*图片分析\s*】)/.test(message)) return false; return /(基于当前上传图片处理|基于你选中的原图处理|基于上一轮图片继续处理)/.test(message); } function injectImageAnalysisSummary(message: string, summary: string): string { const normalizedSummaryLines = summary .split(/\r?\n/) .map((line) => line.replace(/\s+/g, " ").trim()) .filter(Boolean); if (normalizedSummaryLines.length === 0) return message; const lines = message.split("\n"); const analysisHeaderIndex = lines.findIndex((line) => /^(###\s*图片分析|【\s*图片分析\s*】)/.test(line.trim()) ); if (analysisHeaderIndex < 0) return message; for (let i = analysisHeaderIndex + 1; i < lines.length; i += 1) { const current = lines[i]?.trim() || ""; if (/^(###\s*|【\s*[^】]+\s*】)/.test(current)) break; if (/^- /.test(current)) { lines.splice(i, 1, ...normalizedSummaryLines.map((line) => `- ${line}`)); return lines.join("\n"); } } const insertIndex = analysisHeaderIndex + 1; lines.splice(insertIndex, 0, ...normalizedSummaryLines.map((line) => `- ${line}`)); return lines.join("\n"); } function ensureImageAnalysisSummarySection(message: string, summary: string): string { const normalizedSummaryLines = summary .split(/\r?\n/) .map((line) => line.replace(/\s+/g, " ").trim()) .filter(Boolean); if (normalizedSummaryLines.length === 0) return message; if (/(###\s*图片分析|【\s*图片分析\s*】)/.test(message)) { return injectImageAnalysisSummary(message, normalizedSummaryLines.join("\n")); } const trimmed = message.trim(); const summaryBlock = normalizedSummaryLines.map((line) => `- ${line}`).join("\n"); if (!trimmed) return `【图片分析】\n${summaryBlock}`; return `${trimmed}\n【图片分析】\n${summaryBlock}`; } /** * 自定义消息转换函数 * 注意：gemini-image 和 huanke-gemini 都需要 URL 格式（不是 Base64） * 因为这些 provider 内部会自行处理图片上传到 S3 */ async function convertMessagesWithImages(messages: ChatMessage[], providerType: string) { if ( providerType === "bfl-image" || providerType === "ruxa-image" || providerType === "fal-image" || providerType === "openai-image" || providerType === "midjourney" || providerType === "runway" ) { const convertedMessages = []; for (const message of messages) { const content: any[] = []; for (const part of message.parts) { const partAny = part as any; if (partAny.type === "text" && partAny.text) { content.push({ type: "text", text: partAny.text, }); continue; } const candidateUrl = partAny.type === "file" ? (partAny.url || partAny.data) : partAny.type === "image" ? (partAny.image || partAny.url) : partAny.type === "image_url" ? partAny.image_url?.url : undefined; if (!candidateUrl || typeof candidateUrl !== "string") continue; if (candidateUrl.startsWith("data:")) { throw new Error("当前模型不支持 Base64 图片传输，请先上传图片后再编辑"); } if (!candidateUrl.startsWith("http")) continue; content.push({ type: "image_url", image_url: { url: candidateUrl, }, }); } convertedMessages.push({ role: message.role, content: content.length === 1 && content[0].type === "text" ? content[0].text : content, }); } return convertedMessages; } // 所有 provider 都使用 URL 格式传输图片 // gemini-image 和 huanke-gemini 内部会将 Base64 转换为 S3 URL // 这里不再进行 Base64 转换，直接返回标准格式 if (providerType !== "gemini") { return convertToModelMessages(messages); } // 🆕 检测是否有标注图片 const hasAnnotation = hasAnnotatedImage(messages); if (hasAnnotation) { console.log("[Message Convert] 🎨 检测到标注图片，将自动添加系统提示"); } // 对于 Gemini，需要特殊处理图片 const convertedMessages = []; for (const message of messages) { const content: any[] = []; // 🆕 如果是用户消息且有标注图片，在文本前添加系统提示 let hasAddedAnnotationPrompt = false; // 处理消息的各个部分 for (const part of message.parts) { if (part.type === "text" && part.text) { let textContent = part.text; // 🆕 如果有标注图片且是用户消息，在用户文本前添加系统提示 if (hasAnnotation && message.role === "user" && !hasAddedAnnotationPrompt) { textContent = ANNOTATION_SYSTEM_PROMPT + "\n\n用户请求：" + textContent; hasAddedAnnotationPrompt = true; console.log("[Message Convert] 🎨 已注入标注提示词"); } content.push({ type: "text", text: textContent, }); } else if (part.type === "file" && part.url) { // 直接使用 URL 格式（不再转换为 Base64） // gemini-image 和 huanke-gemini provider 内部会处理图片上传 content.push({ type: "image_url", image_url: { url: part.url, }, }); console.log("[Message Convert] Image kept as URL format:", part.url.substring(0, 80) + "..."); } } // 🆕 如果有标注图片但消息中没有文本部分，添加一个纯提示文本 if (hasAnnotation && message.role === "user" && !hasAddedAnnotationPrompt && content.length > 0) { content.unshift({ type: "text", text: ANNOTATION_SYSTEM_PROMPT, }); console.log("[Message Convert] 🎨 已在消息开头注入标注提示词"); } convertedMessages.push({ role: message.role, content: content.length === 1 && content[0].type === "text" ? content[0].text : content, }); } return convertedMessages; } function parseImageAltMetadata(altText: string, thumbnailUrl: string): { originalUrl: string; imageSize: string; } { let originalUrl = thumbnailUrl; let imageSize = "2K"; // 格式1: thumb|||originalUrl|||size (新格式) if (altText.includes("thumb|||")) { const content = altText.substring(altText.indexOf("thumb|||") + 8); const parts = content.split("|||"); originalUrl = parts[0] || thumbnailUrl; imageSize = parts[1] || "2K"; } // 格式2: thumb$$originalUrl$$size (中间格式，可能被 LaTeX 解析器破坏) else if (altText.includes("thumb$$")) { const content = altText.substring(altText.indexOf("thumb$$") + 7); const parts = content.split("$$"); originalUrl = parts[0] || thumbnailUrl; imageSize = parts[1] || "2K"; } // 格式3: thumb|originalUrl|size (旧格式) else if (altText.includes("thumb|")) { const content = altText.substring(altText.indexOf("thumb|") + 6); const parts = content.split("|"); originalUrl = parts[0] || thumbnailUrl; imageSize = parts[1] || "2K"; } // 回退策略：从缩略图 URL 推断原图 URL if (originalUrl === thumbnailUrl) { if (thumbnailUrl.includes("gemini-thumb-")) { originalUrl = thumbnailUrl.replace("gemini-thumb-", "gemini-orig-"); } else if (thumbnailUrl.includes("openai-thumb-")) { originalUrl = thumbnailUrl.replace("openai-thumb-", "openai-orig-"); } else if (thumbnailUrl.includes("volcano-thumb-")) { originalUrl = thumbnailUrl.replace("volcano-thumb-", "volcano-orig-"); } else if (thumbnailUrl.includes("bfl-image-thumb-")) { originalUrl = thumbnailUrl.replace("bfl-image-thumb-", "bfl-image-orig-"); } else if (thumbnailUrl.includes("alibaba-image-thumb-")) { originalUrl = thumbnailUrl.replace("alibaba-image-thumb-", "alibaba-image-orig-"); } else if (thumbnailUrl.includes("ruxa-image-thumb-")) { originalUrl = thumbnailUrl.replace("ruxa-image-thumb-", "ruxa-image-orig-"); } else if (thumbnailUrl.includes("fal-image-thumb-")) { originalUrl = thumbnailUrl.replace("fal-image-thumb-", "fal-image-orig-"); } } return { originalUrl, imageSize }; } function extractOriginalImageUrlsFromText(text: string): string[] { return extractImageUrlsFromText(text); } function normalizeImageInputRef(rawValue: unknown, requestOrigin?: string): string | null { if (typeof rawValue !== "string") return null; const trimmed = rawValue.trim(); if (!trimmed) return null; if (/^data:image\/[a-z0-9+.-]+;base64,/i.test(trimmed)) { return trimmed.replace(/\s+/g, ""); } if (/^https?:\/\//i.test(trimmed)) { return trimmed; } if (requestOrigin && trimmed.startsWith("/")) { try { return new URL(trimmed, requestOrigin).toString(); } catch { return null; } } return null; } function collectImageInputsFromMessageParts( parts: Array> | undefined, requestOrigin?: string, ): string[] { if (!Array.isArray(parts) || parts.length === 0) return []; const result: string[] = []; const pushInput = (rawValue: unknown) => { const normalized = normalizeImageInputRef(rawValue, requestOrigin); if (!normalized) return; if (!result.includes(normalized)) { result.push(normalized); } }; for (const part of parts) { if (!part) continue; if (part.type === "file") { pushInput(part.url); pushInput(part.data); continue; } if (part.type === "image") { pushInput(part.url); pushInput(part.image); pushInput(part.data); continue; } if (part.type === "image_url") { pushInput(part.image_url?.url); pushInput(part.url); continue; } if (part.type === "text" && typeof part.text === "string") { const parsedUrls = extractOriginalImageUrlsFromText(part.text); for (const parsedUrl of parsedUrls) { pushInput(parsedUrl); } } } return result; } function mimeTypeToExtension(mimeType: string) { const normalized = mimeType.toLowerCase(); if (normalized.includes("png")) return "png"; if (normalized.includes("webp")) return "webp"; if (normalized.includes("gif")) return "gif"; if (normalized.includes("bmp")) return "bmp"; return "jpg"; } function getExtensionFromUrl(url: string): string { try { const pathname = new URL(url).pathname; const ext = pathname.split(".").pop()?.toLowerCase(); if (!ext) return "jpg"; if (["jpg", "jpeg", "png", "webp", "gif", "bmp"].includes(ext)) return ext; } catch { // ignore } return "jpg"; } async function uploadRemoteImageToStorage(imageUrl: string): Promise { const response = await fetch(imageUrl, { method: "GET", signal: AbortSignal.timeout(30_000), }); if (!response.ok) { throw new Error(`Remote image fetch failed: ${response.status} ${response.statusText}`); } const contentType = response.headers.get("content-type")?.toLowerCase() || ""; if (contentType && !contentType.startsWith("image/")) { throw new Error(`Remote url is not an image: ${contentType}`); } const arrayBuffer = await response.arrayBuffer(); if (!arrayBuffer || arrayBuffer.byteLength === 0) { throw new Error("Remote image content is empty"); } const ext = contentType ? mimeTypeToExtension(contentType) : getExtensionFromUrl(imageUrl); const fileName = `remote-image-${Date.now()}-${Math.random().toString(36).slice(2, 8)}.${ext}`; const uploaded = await uploadFile(fileName, arrayBuffer); return uploaded.url; } async function isRemoteImageAccessible(imageUrl: string): Promise { try { const headResponse = await fetch(imageUrl, { method: "HEAD", signal: AbortSignal.timeout(10_000), }); if (headResponse.ok) { return true; } if (headResponse.status !== 405 && headResponse.status !== 403) { return false; } const probeResponse = await fetch(imageUrl, { method: "GET", headers: { Range: "bytes=0-0", }, signal: AbortSignal.timeout(10_000), }); return probeResponse.ok; } catch { return false; } } async function ensureHostedImageUrl(imageUrl: string): Promise { const normalizedUrl = imageUrl.trim(); if (!normalizedUrl) { throw new Error("Image URL is empty"); } if (normalizedUrl.startsWith("data:image")) { return uploadDataUrlImage(normalizedUrl); } if (!/^https?:\/\//i.test(normalizedUrl)) { throw new Error(`Unsupported image URL: ${normalizedUrl}`); } if (/\/uploads\//i.test(normalizedUrl)) { const accessible = await isRemoteImageAccessible(normalizedUrl); if (accessible) { return normalizedUrl; } } return uploadRemoteImageToStorage(normalizedUrl); } async function uploadDataUrlImage(dataUrl: string): Promise { const match = dataUrl.match(/^data:([^;]+);base64,([\s\S]+)$/i); if (!match) { throw new Error("Invalid data URL image format"); } const mimeType = match[1] || "image/jpeg"; const base64Data = (match[2] || "").replace(/\s+/g, ""); if (!base64Data) { throw new Error("Empty base64 image content"); } const buffer = Buffer.from(base64Data, "base64"); const fileExt = mimeTypeToExtension(mimeType); const fileName = `inline-image-${Date.now()}-${Math.random().toString(36).slice(2, 8)}.${fileExt}`; const arrayBuffer = buffer.buffer.slice( buffer.byteOffset, buffer.byteOffset + buffer.byteLength ) as ArrayBuffer; const uploaded = await uploadFile(fileName, arrayBuffer); return uploaded.url; } async function normalizeAssistantImageMarkdown( text: string, imageSize = "2K" ): Promise { if (!text) return text; const markdownDataUrlRegex = /!\[([^\]]*)\]\s*\(\s*(data:image\/[a-z0-9+.-]+;base64,[^)]+)\s*\)/gim; const markdownMatches = [...text.matchAll(markdownDataUrlRegex)]; let normalizedText = text; const uploadedUrls: string[] = []; if (markdownMatches.length > 0) { let rebuilt = ""; let cursor = 0; for (const match of markdownMatches) { const matchText = match[0]; const dataUrl = match[2]; const start = match.index ?? -1; if (start < 0) continue; rebuilt += normalizedText.slice(cursor, start); try { const uploadedUrl = await uploadDataUrlImage(dataUrl); uploadedUrls.push(uploadedUrl); rebuilt += `![thumb|||${uploadedUrl}|||${imageSize}](${uploadedUrl})`; } catch (error) { console.error("[Chat API] Failed to upload inline data URL image:", error); rebuilt += matchText; } cursor = start + matchText.length; } rebuilt += normalizedText.slice(cursor); normalizedText = rebuilt; } if (normalizedText.includes("data:image")) { const rawDataUrlRegex = /data:image\/[a-z0-9+.-]+;base64,[a-z0-9+/=\s]+/gim; const rawMatches = normalizedText.match(rawDataUrlRegex) ?? []; for (const rawDataUrl of rawMatches) { try { const uploadedUrl = await uploadDataUrlImage(rawDataUrl); uploadedUrls.push(uploadedUrl); normalizedText = normalizedText.replace(rawDataUrl, uploadedUrl); } catch (error) { console.error("[Chat API] Failed to normalize raw data URL image:", error); } } } const markdownHttpImageRegex = /!\[([^\]]*)\]\s*\(\s*(https?:\/\/[^)\s]+)\s*\)/gim; const markdownHttpMatches = [...normalizedText.matchAll(markdownHttpImageRegex)]; if (markdownHttpMatches.length > 0) { let rebuilt = ""; let cursor = 0; for (const match of markdownHttpMatches) { const matchText = match[0]; const altText = match[1] || ""; const thumbnailUrl = (match[2] || "").trim(); const start = match.index ?? -1; if (start < 0) continue; rebuilt += normalizedText.slice(cursor, start); try { const { originalUrl, imageSize: parsedImageSize } = parseImageAltMetadata( altText, thumbnailUrl ); let hostedThumbnailUrl = thumbnailUrl; let hostedOriginalUrl = originalUrl && originalUrl !== thumbnailUrl ? originalUrl : thumbnailUrl; try { hostedThumbnailUrl = await ensureHostedImageUrl(thumbnailUrl); } catch (error) { console.error("[Chat API] Failed to normalize remote thumbnail URL:", error); } if (originalUrl && originalUrl !== thumbnailUrl) { try { hostedOriginalUrl = await ensureHostedImageUrl(originalUrl); } catch (error) { console.error("[Chat API] Failed to normalize remote original URL:", error); hostedOriginalUrl = hostedThumbnailUrl || originalUrl; } } else { hostedOriginalUrl = hostedThumbnailUrl; } const finalThumbnailUrl = hostedThumbnailUrl || thumbnailUrl; const finalOriginalUrl = hostedOriginalUrl || finalThumbnailUrl; uploadedUrls.push(finalThumbnailUrl); uploadedUrls.push(finalOriginalUrl); rebuilt += `![thumb|||${finalOriginalUrl}|||${parsedImageSize || imageSize}](${finalThumbnailUrl})`; } catch (error) { console.error("[Chat API] Failed to parse remote image markdown metadata:", error); rebuilt += matchText; } cursor = start + matchText.length; } rebuilt += normalizedText.slice(cursor); normalizedText = rebuilt; } const hasMarkdownImage = /!\[[^\]]*\]\((https?:\/\/[^)]+)\)/i.test(normalizedText); if (!hasMarkdownImage && uploadedUrls.length > 0) { const uniqueUrls = [...new Set(uploadedUrls)]; console.log( `[Chat API] Converted ${uniqueUrls.length} image link(s) to hosted link(s)` ); return uniqueUrls .map((url) => `![thumb|||${url}|||${imageSize}](${url})`) .join("\n"); } return normalizedText; } function prependImageModelBadge(text: string, modelDisplayName?: string | null): string { const normalizedText = String(text || "").trim(); if (!normalizedText) return normalizedText; const normalizedModelName = String(modelDisplayName || "").trim(); if (!normalizedModelName) return normalizedText; if ( /^\s*[◈◇◆🔹⬡✦•·]\s*[^\n]+/im.test(normalizedText) || /^\s*(?:Routed to|Using model|使用模型|模型)\s*[:：]/im.test(normalizedText) ) { return normalizedText; } return `◈ ${normalizedModelName}\n\n${normalizedText}`; } function resolveModelDisplayNameByModelId( models: Awaited> | undefined, modelId: string | undefined, ): string { if (!modelId || !models?.length) return ""; const match = models.find( (item) => item.model.modelId === modelId || item.model.id === modelId, ); return String(match?.model?.displayName || ""); } /** * 解析生图 Markdown 并保存图片记录到数据库 * 支持三种格式：thumb|||url|||size、thumb$$url$$size、thumb|url|size */ async function saveGeneratedImages( imageMarkdown: string, userId: string, chatId: string, messageId: string, prompt: string, modelId: string, options?: { sourceAssetIds?: string[]; sourceImageUrls?: string[]; relationType?: "generated_from" | "edited_from"; sourceType?: string; projectId?: string | null; userPrompt?: string; optimizedPrompt?: string; requestedImageCount?: number; aspectRatio?: string; imageSize?: string; }, ): Promise { // 匹配 Markdown 图片格式: ![alt](url) const imageRegex = /!\[([^\]]*)\]\(([^)]+)\)/g; const matches = [...imageMarkdown.matchAll(imageRegex)]; let savedImageCount = 0; const savedAssetIds: string[] = []; const savedOriginalUrls: string[] = []; let relationSourceAssetIds = Array.isArray(options?.sourceAssetIds) ? options.sourceAssetIds.filter((assetId): assetId is string => Boolean(assetId)) : []; if ( relationSourceAssetIds.length === 0 && Array.isArray(options?.sourceImageUrls) && options.sourceImageUrls.length > 0 ) { try { const sourceAssets = await getMediaAssetsByUserAndUrls({ userId, urls: options.sourceImageUrls, }); relationSourceAssetIds = sourceAssets.map((asset) => asset.id); } catch (sourceLookupError) { console.error("[Gallery] Failed to resolve source assets by URL:", sourceLookupError); } } for (const match of matches) { const altText = match[1] || ""; const thumbnailUrl = match[2]; const { originalUrl, imageSize } = parseImageAltMetadata(altText, thumbnailUrl); try { const [savedImage] = await db.insert(generatedImage).values({ userId, chatId, messageId, thumbnailUrl, originalUrl, imageSize, prompt, modelId, }).returning(); savedImageCount += 1; if (!savedOriginalUrls.includes(originalUrl)) { savedOriginalUrls.push(originalUrl); } const media = await upsertMediaAsset({ userId, chatId, projectId: options?.projectId ?? null, mediaType: "image", sourceType: options?.sourceType || "generated", status: "ready", originalUrl, previewUrl: originalUrl, thumbnailUrl, displayName: prompt?.slice(0, 80) || "AI 生成图片", prompt, modelId, sourceMessageId: messageId, sourceTable: "GeneratedImage", sourceRowId: savedImage?.id || null, primaryParentAssetId: relationSourceAssetIds[0] || null, metadata: { imageSize, }, }); if (media?.id && !savedAssetIds.includes(media.id)) { savedAssetIds.push(media.id); } if (media && relationSourceAssetIds.length > 0) { for (const sourceAssetId of relationSourceAssetIds) { if (!sourceAssetId) continue; await createMediaAssetRelation({ userId, chatId, projectId: options?.projectId ?? null, fromAssetId: sourceAssetId, toAssetId: media.id, relationType: options?.relationType || "generated_from", sourceMessageId: messageId, metadata: { modelId, prompt, sourceImageUrls: options?.sourceImageUrls || [], }, }); } } console.log(`[Gallery] Saved image: ${originalUrl.substring(0, 80)}...`); } catch (error) { console.error("[Gallery] Failed to save image record:", error); } } if (savedOriginalUrls.length > 0) { try { const resultAssetIds = savedAssetIds.length > 0 ? savedAssetIds : null; await persistImageWorkflowTurn({ userId, chatId, projectId: options?.projectId || null, currentIntent: options?.relationType === "edited_from" ? "image_edit" : "image_gen", currentFocusAssetIds: resultAssetIds, currentFocusImageUrls: savedOriginalUrls, lastGeneratedAssetIds: resultAssetIds, lastGeneratedImageUrls: savedOriginalUrls, lastResolvedTargetAssetIds: resultAssetIds, lastResolvedTargetImageUrls: savedOriginalUrls, lastMessageId: messageId, lastUserPrompt: String(options?.userPrompt || prompt || "").trim() || prompt, lastOptimizedPrompt: String( options?.optimizedPrompt || (options?.relationType !== "edited_from" ? prompt : "") || "", ).trim() || undefined, lastRequestedImageCount: options?.requestedImageCount || savedOriginalUrls.length, lastAspectRatio: options?.aspectRatio, lastImageSize: options?.imageSize, lastSourceImageUrls: options?.sourceImageUrls || null, lastResultImageUrls: savedOriginalUrls, }); } catch (memoryError) { console.warn("[Gallery] Failed to update working state with generated images:", memoryError); } } if (savedImageCount <= 0) return; try { const pointsConfig = await getPointsSystemConfig(); if (!pointsConfig.enabled) return; const userData = await getUserById({ id: userId }).catch(() => undefined); if (userData?.isHighVip) { console.log(`[Points] Skip image points for VIP user=${userId} model=${modelId}`); return; } const modelConfig = await resolveBillingModelByReference(modelId); const unitCost = getModelCreditCost(modelConfig, "image"); if (unitCost <= 0) return; const totalCost = unitCost * savedImageCount; const consumeResult = await consumeUserPoints({ userId, amount: totalCost, reason: "image_generation", modelId: modelConfig?.id || modelId, chatId, messageId, note: `模型(${modelConfig?.displayName || modelId}) 生图 ${savedImageCount} 张`, }); if (!consumeResult.ok) { console.warn( `[Points] Failed to consume points after generation. user=${userId} model=${modelId} cost=${totalCost}` ); } else if (consumeResult.skipped) { console.log( `[Points] Skipped ${totalCost} image points for VIP user=${userId} model=${modelId}` ); } else { console.log( `[Points] Consumed ${totalCost} points (${unitCost} x ${savedImageCount}). balance=${consumeResult.balance}` ); } } catch (error) { console.error("[Points] Failed to consume points after image generation:", error); } } type AgentPresetMessageImagePart = { type: "file"; url: string; name: string; mediaType: string; assetId?: string; }; function extractCurrentMessageImageParts( parts: Array>, ): AgentPresetMessageImagePart[] { return parts .filter( (part): part is AgentPresetMessageImagePart => part?.type === "file" && typeof part?.url === "string" && /^https?:\/\//i.test(part.url) && typeof part?.mediaType === "string" && part.mediaType.startsWith("image/") ) .map((part) => ({ type: "file", url: String(part.url).trim(), name: String(part.name || "图片").trim() || "图片", mediaType: String(part.mediaType || "image/jpeg").trim() || "image/jpeg", assetId: typeof part.assetId === "string" && part.assetId.trim() ? part.assetId.trim() : undefined, })); } function normalizeAgentSupplementPrompt(text: string, presetName?: string | null) { const normalized = String(text || "").trim(); if (!normalized) return ""; const escapedPresetName = String(presetName || "").replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); const genericPattern = escapedPresetName ? new RegExp(`^请按「${escapedPresetName}」智能体配置顺序处理这\\d+张图片。?$`) : /^请按「.+」智能体配置顺序处理这\d+张图片。?$/; if (genericPattern.test(normalized)) { return ""; } return normalized; } function normalizeAgentWorkflowPrompt(text: string) { return String(text || "").trim(); } function buildAgentSlotPrompt(params: { workflowPrompt?: string | null; slotPrompt: string; slotTitle?: string | null; userSupplementPrompt?: string | null; }) { const lines: string[] = []; const workflowPrompt = normalizeAgentWorkflowPrompt(params.workflowPrompt || ""); if (workflowPrompt) { lines.push(workflowPrompt); } const slotTitle = String(params.slotTitle || "").trim(); if (slotTitle) { lines.push(`【槽位】${slotTitle}`); } const slotPrompt = String(params.slotPrompt || "").trim(); if (slotPrompt) { lines.push(slotPrompt); } const supplement = String(params.userSupplementPrompt || "").trim(); if (supplement) { lines.push(`【补充要求】${supplement}`); } return lines.join("\n\n").trim(); } type AgentReplacementModePayload = NonNullable; function normalizeAgentModeReferences( payload?: PostRequestBody["agentModePayload"], ): AgentReplacementModePayload["references"] { if (!payload?.references?.length) return []; return payload.references .map((item) => ({ id: String(item.id || "").trim(), label: String(item.label || "").trim() || "参考图", imageUrl: String(item.imageUrl || "").trim(), candidateId: String(item.candidateId || "").trim() || null, sortOrder: Number.isFinite(Number(item.sortOrder)) ? Math.max(0, Math.min(999, Math.round(Number(item.sortOrder)))) : 0, assignedProductIndex: Number.isFinite(Number(item.assignedProductIndex)) ? Math.max(0, Math.min(9, Math.round(Number(item.assignedProductIndex)))) : null, })) .filter((item) => item.id && /^https?:\/\//i.test(item.imageUrl)) .sort((left, right) => left.sortOrder - right.sortOrder); } function buildAgentReplacementPrompt(params: { basePrompt: string; modeType: "detail_replacement" | "sku_replacement"; referenceLabel: string; referenceIndex: number; totalReferences: number; hasModelImage: boolean; pairedProductIndex?: number; }) { const instructionLines = params.modeType === "detail_replacement" ? [ "【执行模式】详情替换", `当前要处理第 ${params.referenceIndex + 1} / ${params.totalReferences} 张详情参考图：${params.referenceLabel}`, "请严格保留最后一张参考图的版式结构、镜头关系、构图、光影氛围、留白位置和信息层级，只替换其中的商品主体。", "前面的输入商品图用于提供商品主体、材质、颜色和细节特征；如有模特图，请在需要人物展示时优先参考模特图。", "不要改动参考图的整体布局，不要新增无关主体，不要删除关键版式元素。", ] : [ "【执行模式】SKU替换", `当前要处理第 ${params.referenceIndex + 1} / ${params.totalReferences} 张 SKU 参考图：${params.referenceLabel}`, params.pairedProductIndex !== undefined ? `请优先使用第 ${params.pairedProductIndex + 1} 张输入商品图作为本张参考图的一对一替换主体。` : "请使用输入商品图作为本张参考图的一对一替换主体。", "请严格保留最后一张参考图的场景、机位、透视、光影和构图关系，只替换商品主体本身。", "输出要自然融合，不要改变参考图原有风格和景别。", ]; if (params.hasModelImage) { instructionLines.push("如有模特图，请在需要人物展示时保持模特体态、穿搭和出镜关系一致。"); } return [params.basePrompt, instructionLines.join("\n")] .filter(Boolean) .join("\n\n") .trim(); } const MODEL_UUID_REGEX = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i; function getModelCreditCost(modelConfig: any, target: "image" | "video"): number { const config = (modelConfig as any)?.imageGenConfig || {}; const sourceValue = target === "video" ? config.creditCostPerVideo ?? config.creditCostPerImage ?? 0 : config.creditCostPerImage ?? 0; const normalized = Number(sourceValue); return Number.isFinite(normalized) ? Math.max(0, Math.round(normalized)) : 0; } async function resolveBillingModelByReference(modelRef: string): Promise { if (!modelRef) return undefined; if (MODEL_UUID_REGEX.test(modelRef)) { return await getAiModelById({ id: modelRef }).catch(() => undefined); } const allModels = await getEnabledAiModelsWithProvider(true, true).catch(() => []); const matched = allModels.find( ({ model }: any) => model.id === modelRef || model.modelId === modelRef ); return matched?.model; } async function consumeGeneratedVideoPoints(params: { userId: string; chatId: string; messageId: string; modelRefs: Array; }) { try { const pointsConfig = await getPointsSystemConfig(); if (!pointsConfig.enabled) return; const userData = await getUserById({ id: params.userId }).catch(() => undefined); if (userData?.isHighVip) { console.log(`[Points] Skip video points for VIP user=${params.userId}`); return; } let billingModel: any | undefined; for (const ref of params.modelRefs) { if (!ref) continue; const resolved = await resolveBillingModelByReference(ref); if (!resolved) continue; billingModel = resolved; if (getModelCreditCost(resolved, "video") > 0) break; } if (!billingModel) return; const unitCost = getModelCreditCost(billingModel, "video"); if (unitCost <= 0) return; const consumeResult = await consumeUserPoints({ userId: params.userId, amount: unitCost, reason: "video_generation", modelId: billingModel.id, chatId: params.chatId, messageId: params.messageId, note: `模型(${billingModel.displayName || billingModel.modelId || billingModel.id}) 生视频 1 条`, }); if (!consumeResult.ok) { console.warn( `[Points] Failed to consume video points. user=${params.userId} model=${billingModel.id} cost=${unitCost}` ); } else if (consumeResult.skipped) { console.log( `[Points] Skipped ${unitCost} video points for VIP user=${params.userId} model=${billingModel.id}` ); } else { console.log( `[Points] Consumed ${unitCost} points for video. balance=${consumeResult.balance}` ); } } catch (error) { console.error("[Points] Failed to consume points after video generation:", error); } } export const maxDuration = 600; // 10 分钟，用于支持图片编辑等耗时任务 let globalStreamContext: ResumableStreamContext | null = null; const inFlightChatRequests = new Map(); const IN_FLIGHT_REQUEST_TTL_MS = 5 * 60 * 1000; const DUPLICATE_REQUEST_WAIT_MS = 2 * 60 * 1000; const CHAT_PRE_RESPONSE_SOFT_TIMEOUT_MS = 55_000; const SMART_AUTO_GRAPH_TIMEOUT_MS = 28_000; const SMART_AUTO_FALLBACK_TIMEOUT_MS = 20_000; const SMART_AUTO_IMAGE_SUMMARY_TIMEOUT_MS = 18_000; const DEFAULT_STREAM_KEEPALIVE_INTERVAL_MS = 12_000; const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); class OperationTimeoutError extends Error { constructor(operation: string, timeoutMs: number) { super(`${operation} timed out after ${timeoutMs}ms`); this.name = "OperationTimeoutError"; } } async function withOperationTimeout( operation: Promise, operationName: string, timeoutMs: number, ): Promise { let timer: ReturnType | null = null; try { return await Promise.race([ operation, new Promise((_, reject) => { timer = setTimeout( () => reject(new OperationTimeoutError(operationName, timeoutMs)), timeoutMs, ); }), ]); } finally { if (timer) { clearTimeout(timer); } } } function getRemainingBudgetMs(startedAt: number, budgetMs: number) { return Math.max(0, budgetMs - (Date.now() - startedAt)); } function getBudgetedTimeout(params: { startedAt: number; preferredMs: number; minimumMs?: number; }) { const minimumMs = params.minimumMs ?? 6_000; const remainingMs = getRemainingBudgetMs( params.startedAt, CHAT_PRE_RESPONSE_SOFT_TIMEOUT_MS ); if (remainingMs < minimumMs) { return null; } return Math.min(params.preferredMs, remainingMs); } const getTokenlensCatalog = cache( async (): Promise => { try { return await fetchModels(); } catch (err) { console.warn( "TokenLens: catalog fetch failed, using default catalog", err ); return; // tokenlens helpers will fall back to defaultCatalog } }, ["tokenlens-catalog"], { revalidate: 24 * 60 * 60 } // 24 hours ); export function getStreamContext() { if (!globalStreamContext) { try { globalStreamContext = createResumableStreamContext({ waitUntil: after, }); } catch (error: any) { if (error.message.includes("REDIS_URL")) { console.log( " > Resumable streams are disabled due to missing REDIS_URL" ); } else { console.error(error); } } } return globalStreamContext; } export async function POST(request: Request) { let requestBody: PostRequestBody; let requestDedupKey: string | null = null; let acquiredInFlightLock = false; try { const json = await request.json(); console.log("[Chat API] Received request body:", JSON.stringify(json, null, 2)); requestBody = postRequestBodySchema.parse(json); } catch (error) { console.error("[Chat API] Schema validation failed:", error); return new ChatSDKError("bad_request:api").toResponse(); } try { const { id, message, agentPresetId, selectedChatModel, selectedVisibilityType, composerMode, hiddenPrompt, autoContext, contextPayload, agentModePayload, deepThinkingEnabled, debugProtocol, }: { id: string; message: ChatMessage; agentPresetId?: string; selectedChatModel: ChatModel["id"]; selectedVisibilityType: VisibilityType; composerMode?: "agent" | "image" | "video" | "detail"; hiddenPrompt?: string; autoContext?: string; contextPayload?: PostRequestBody["contextPayload"]; agentModePayload?: PostRequestBody["agentModePayload"]; deepThinkingEnabled?: boolean; debugProtocol?: boolean; } = requestBody; const requestStartedAt = Date.now(); const requestOrigin = (() => { try { return new URL(request.url).origin; } catch { return ""; } })(); const requestTextContent = message.parts .filter((part): part is { type: "text"; text: string } => part.type === "text") .map((part) => part.text) .join(" ") .trim(); const autoContextText = typeof autoContext === "string" ? autoContext.trim() : ""; const now = Date.now(); for (const [key, startedAt] of inFlightChatRequests.entries()) { if (now - startedAt > IN_FLIGHT_REQUEST_TTL_MS) { inFlightChatRequests.delete(key); } } requestDedupKey = `${id}:${message.id}`; if (inFlightChatRequests.has(requestDedupKey)) { console.warn( `[Chat API] Duplicate in-flight request ignored (chatId=${id}, messageId=${message.id})` ); const duplicateStream = createUIMessageStream({ execute: async ({ writer }) => { await replayAssistantMessagesForDuplicateRequest({ chatId: id, userMessageId: message.id, writer, source: "duplicate-in-flight", }); }, }); return createSseResponse(duplicateStream); } inFlightChatRequests.set(requestDedupKey, now); acquiredInFlightLock = true; let finalModelId = selectedChatModel; // 🆕 存储智能模式的相关信息 let smartAutoFriendlyMessage: string | undefined; // AI的自然对话回复 let smartAutoOptimizedPrompt: string | undefined; // 优化后的提示词（不修改用户消息，只用于API调用） let smartAutoPromptFromVideoPlan = false; let smartAutoRecommendedImageParams: AgentRecommendedImageParams | undefined; let smartAutoVideoPlan: AgentVideoPlan | undefined; let smartAutoRecommendedVideoParams: AgentRecommendedVideoParams | undefined; let smartAutoPlan: AgentExecutionPlan | undefined; let smartAutoExecution: AgentPreparedExecution | undefined; let smartAutoDetectedIntent: "image_edit" | "vision" | "image_gen" | "video_gen" | "chat" | undefined; let smartAutoHistoryImages: string[] = []; // 从历史对话中提取的图片URL（支持多张） let smartAutoNeedsClarification = false; // 是否需要先追问用户 let smartAutoClarificationQuestion: string | undefined; let requestResolvedMediaInput: ResolvedChatMediaInput | null = null; let requestTargetResolution: ResolvedTargetAssets | null = null; let requestWorkingState: | Awaited> | null = null; let storyboardPreferenceMemory = null; let imageWorkflowMemory = null; let reusablePromptMemory = null; let confirmedPromptForConversationState: string | undefined; let smartAutoHistoryMessagesFromDb: DBMessage[] = []; let smartAutoMediaHistoryMessagesFromDb: DBMessage[] = []; let preferredAgentModelId: string | undefined; let configuredAgentConversationModelId: string | undefined; let agentModelSlots: AgentModelSlots = {}; let pendingGridConfirmation: | PendingGridConfirmation | null = null; let shouldAskGridConfirmation = false; const deepThinkingOn = deepThinkingEnabled !== false; const shouldRespectDirectComposerMode = composerMode === "image" || composerMode === "video" || composerMode === "detail"; const shouldIsolateDirectComposerContext = shouldRespectDirectComposerMode; let shouldRunAgentConversation = finalModelId === "auto"; let shouldKeepSelectedModel = false; if (shouldRespectDirectComposerMode && shouldRunAgentConversation) { shouldRunAgentConversation = false; console.log( `[Smart Auto Mode] ⛳ Direct composer mode (${composerMode}) disables initial auto-agent routing`, ); } const pendingProtocolDebugEvents: Array<{ phase: string; payload: Record }> = []; const requestedModelId = selectedChatModel; let requestedModelProviderType: string | undefined; let requestedModelIsFixedVisual = false; let requestedModelIsConversationMultimodal = false; let requestedModelConfig: Awaited> | null = null; if (!shouldRunAgentConversation && requestedModelId !== "auto") { const selectedModelIsUUID = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test(requestedModelId); let selectedModelConfig = selectedModelIsUUID ? await getAiModelById({ id: requestedModelId }) : null; let selectedModelProviderType: string | undefined; if (!selectedModelConfig) { const enabledModels = await getEnabledAiModelsWithProvider(false); const matchedModel = enabledModels.find( ({ model }) => model.id === requestedModelId || model.modelId === requestedModelId ); if (matchedModel) { selectedModelConfig = matchedModel.model; selectedModelProviderType = matchedModel.provider.type as string; } } if (selectedModelConfig && !selectedModelProviderType) { const selectedProvider = await getAiProviderById({ id: selectedModelConfig.providerId, }); selectedModelProviderType = selectedProvider?.type as string | undefined; } requestedModelConfig = selectedModelConfig; requestedModelProviderType = selectedModelProviderType; if (selectedModelConfig) { const isFixedVisualModel = !selectedModelConfig.isSubModel && ( isImageGenerationModelType(selectedModelProviderType, selectedModelConfig) || isVideoProviderType(selectedModelProviderType, selectedModelConfig) ); const isConversationMultimodalModel = !selectedModelConfig.isSubModel && isAiModelMultimodal(selectedModelConfig) && !isImageGenerationModelType(selectedModelProviderType, selectedModelConfig) && !isVideoProviderType(selectedModelProviderType, selectedModelConfig); requestedModelIsFixedVisual = isFixedVisualModel; requestedModelIsConversationMultimodal = isConversationMultimodalModel; if (deepThinkingOn) { if (isAiModelAgentAuto(selectedModelConfig)) { if (shouldRespectDirectComposerMode) { console.log( `[Smart Auto Mode] ⛳ Respect direct composer mode (${composerMode}), skip forcing agent-only routing for: ${selectedModelConfig.displayName}`, ); } else { shouldRunAgentConversation = true; shouldKeepSelectedModel = false; preferredAgentModelId = selectedModelConfig.id; configuredAgentConversationModelId = selectedModelConfig.id; console.log( `[Smart Auto Mode] 🧠 Agent-only model detected, force auto routing: ${selectedModelConfig.displayName}` ); } } else if (isFixedVisualModel || isConversationMultimodalModel) { if (shouldRespectDirectComposerMode) { console.log( `[Smart Auto Mode] ⛳ Respect direct composer mode (${composerMode}), keep direct execution model: ${selectedModelConfig.displayName}`, ); } else { shouldRunAgentConversation = true; shouldKeepSelectedModel = true; preferredAgentModelId = selectedModelConfig.id; console.log( `[Smart Auto Mode] 🎯 Agent conversation enabled for selected model: ${selectedModelConfig.displayName} (${selectedModelProviderType || "unknown"}), preferredAgentModelId=${preferredAgentModelId}` ); } } } else if ( isFixedVisualModel || isConversationMultimodalModel || isAiModelAgentAuto(selectedModelConfig) ) { console.log( `[Smart Auto Mode] ⏭️ Deep thinking disabled, skip agent analysis for: ${selectedModelConfig.displayName}` ); } } } if (shouldRunAgentConversation && !preferredAgentModelId) { const configuredAgentModel = await getAgentAiModelWithProvider().catch((agentModelError) => { console.warn("[Chat API] Failed to load configured agent model:", agentModelError); return null; }); if (configuredAgentModel?.model?.id) { preferredAgentModelId = configuredAgentModel.model.id; configuredAgentConversationModelId = configuredAgentModel.model.id; agentModelSlots = upsertAgentModelSlot(agentModelSlots, "planner", { modelId: configuredAgentModel.model.id, displayName: configuredAgentModel.model.displayName || configuredAgentModel.model.modelId, providerId: configuredAgentModel.model.providerId, providerType: configuredAgentModel.provider.type, providerKey: configuredAgentModel.model.providerId, source: "configured_agent_default", reason: "agent_default_model", }); agentModelSlots = upsertAgentModelSlot(agentModelSlots, "analyzer", { modelId: configuredAgentModel.model.id, displayName: configuredAgentModel.model.displayName || configuredAgentModel.model.modelId, providerId: configuredAgentModel.model.providerId, providerType: configuredAgentModel.provider.type, providerKey: configuredAgentModel.model.providerId, source: "configured_agent_default", reason: "agent_default_model", }); console.log( `[Smart Auto Mode] 🧠 Loaded configured agent conversation model: ${configuredAgentModel.model.displayName}`, ); } } const hasCurrentMessageMediaInput = message.parts.some((part: any) => part?.type === "file" || part?.type === "image" || part?.type === "image_url" ); if ( shouldRunAgentConversation && isPureGreetingTurnRequest(requestTextContent) && !hasCurrentMessageMediaInput ) { const session = await auth(); if (!session?.user) { return new ChatSDKError("unauthorized:chat").toResponse(); } const greetingMessage = "你好，我在。你可以直接告诉我想做什么：生图、修图、生成视频、分析图片都可以。"; const greetingStream = createUIMessageStream({ execute: async ({ writer }) => { const messageId = generateUUID(); writeAssistantTextChunk(writer, greetingMessage, "greeting-fast-path", messageId); await saveMessages({ messages: [ { id: messageId, chatId: id, role: "assistant", parts: [{ type: "text", text: greetingMessage }], attachments: [], createdAt: new Date(), videoStorageStatus: null, temporaryVideoUrl: null, permanentVideoUrl: null, archivedAt: null, videoMetadata: null, }, ], }); }, }); return createSseResponse(greetingStream); } const allowAgentVideoGeneration = Boolean(requestBody.videoParams) || isVideoProviderType(requestedModelProviderType, requestedModelConfig || undefined); let smartAutoWorkflowDecision: | { action?: string; workflowId?: string; workflowType?: string; allowWorkflowExecution?: boolean; } | undefined; const activeStoryboardTaskForChat = shouldIsolateDirectComposerContext ? null : await getLatestActiveVideoStoryboardTaskByChatId({ chatId: id, }).catch((storyboardLookupError) => { console.warn( "[Chat API] Failed to preload active storyboard task before smart auto mode:", storyboardLookupError ); return null; }); requestWorkingState = shouldIsolateDirectComposerContext ? null : await getConversationWorkingStateByChatId({ chatId: id, }).catch((workingStatePreloadError) => { console.warn( "[Chat API] Failed to preload conversation working state before smart auto mode:", workingStatePreloadError ); return null; }); const hasImageInputForConversationAction = message.parts.some((part: any) => part?.type === "file" || part?.type === "image" || part?.type === "image_url" ); const conversationStateBootstrap = bootstrapConversationStateProtocol({ text: requestTextContent, workingState: requestWorkingState, activeStoryboardTask: activeStoryboardTaskForChat, hasImageInputForPromptCarryForward: hasImageInputForConversationAction, hasCanvasSelection: Boolean(contextPayload?.selectedNodeIds?.length), }); console.log("[ConversationStateProtocol] Bootstrap:", { action: conversationStateBootstrap.actionResolution.kind, taskType: conversationStateBootstrap.taskBoundaryDecision.taskType, taskMode: conversationStateBootstrap.taskBoundaryDecision.taskMode, hasPromptMemory: Boolean(conversationStateBootstrap.promptMemory?.optimizedPrompt), hasImageWorkflowMemory: Boolean(conversationStateBootstrap.imageWorkflowMemory), hasStoryboardPreferenceMemory: Boolean(conversationStateBootstrap.storyboardPreferenceMemory), hasPendingGridConfirmation: Boolean(conversationStateBootstrap.pendingGridConfirmation), hasActiveStoryboardTask: Boolean(conversationStateBootstrap.activeStoryboardTask?.task?.id), }); pendingProtocolDebugEvents.push({ phase: "bootstrap", payload: { action: conversationStateBootstrap.actionResolution.kind, taskBoundary: conversationStateBootstrap.taskBoundaryDecision, pendingGridConfirmation: conversationStateBootstrap.pendingGridConfirmation, hasPromptMemory: Boolean(conversationStateBootstrap.promptMemory?.optimizedPrompt), hasImageWorkflowMemory: Boolean(conversationStateBootstrap.imageWorkflowMemory), hasStoryboardPreferenceMemory: Boolean(conversationStateBootstrap.storyboardPreferenceMemory), hasActiveStoryboardTask: Boolean(conversationStateBootstrap.activeStoryboardTask?.task?.id), }, }); storyboardPreferenceMemory = conversationStateBootstrap.storyboardPreferenceMemory; imageWorkflowMemory = conversationStateBootstrap.imageWorkflowMemory; reusablePromptMemory = conversationStateBootstrap.promptMemory; pendingGridConfirmation = conversationStateBootstrap.pendingGridConfirmation; confirmedPromptForConversationState = conversationStateBootstrap.confirmedPrompt; const resolvedConversationAction = conversationStateBootstrap.actionResolution; const taskBoundaryDecision = conversationStateBootstrap.taskBoundaryDecision; const taskBoundaryRequiresAgentConversation = taskBoundaryDecision.taskType === "prompt_only" || taskBoundaryDecision.taskType === "analysis_only" || taskBoundaryDecision.taskType === "execution_image" || taskBoundaryDecision.taskType === "execution_video" || taskBoundaryDecision.taskType === "workflow_followup"; const taskBoundaryOverridesLegacyWorkflow = taskBoundaryDecision.taskType === "greeting" || taskBoundaryDecision.taskType === "chat" || taskBoundaryDecision.taskType === "analysis_only" || taskBoundaryDecision.taskType === "prompt_only"; const prunedTaskBoundaryContext = pruneTaskBoundaryContext({ decision: taskBoundaryDecision, contextPayload: contextPayload || undefined, autoContextText: autoContextText || undefined, workingState: requestWorkingState || undefined, hasCurrentImageInput: hasImageInputForConversationAction, }); const effectiveContextPayload = prunedTaskBoundaryContext.contextPayload; const effectiveAutoContextText = prunedTaskBoundaryContext.autoContextText || ""; const effectiveWorkingState = prunedTaskBoundaryContext.workingState; if (taskBoundaryRequiresAgentConversation && !shouldRunAgentConversation) { if (shouldRespectDirectComposerMode) { console.log( `[Smart Auto Mode] ⛳ Respect direct composer mode (${composerMode}), skip task-boundary agent force: ${taskBoundaryDecision.taskType}/${taskBoundaryDecision.taskMode}`, ); } else { shouldRunAgentConversation = true; shouldKeepSelectedModel = false; console.log( `[Smart Auto Mode] 🧭 Force agent conversation due to task boundary: ${taskBoundaryDecision.taskType}/${taskBoundaryDecision.taskMode}`, ); } } if (taskBoundaryOverridesLegacyWorkflow && shouldKeepSelectedModel) { shouldKeepSelectedModel = false; console.log( `[Smart Auto Mode] 🧼 Release selected-model lock due to task boundary override: ${taskBoundaryDecision.taskType}`, ); } if (resolvedConversationAction.forceAgentConversation && !shouldRunAgentConversation) { if (shouldRespectDirectComposerMode) { console.log( `[Smart Auto Mode] ⛳ Respect direct composer mode (${composerMode}), skip legacy workflow force: ${resolvedConversationAction.kind}`, ); } else if (taskBoundaryOverridesLegacyWorkflow) { console.log( `[Smart Auto Mode] 🪫 Ignore legacy workflow follow-up due to task boundary override: ${resolvedConversationAction.kind} -> ${taskBoundaryDecision.taskType}`, ); } else { shouldRunAgentConversation = true; shouldKeepSelectedModel = resolvedConversationAction.shouldKeepSelectedModel; if (resolvedConversationAction.kind === "grid_confirmation_followup" && pendingGridConfirmation) { console.log( `[Smart Auto Mode] 🧩 Force agent conversation due to pending grid confirmation: ${pendingGridConfirmation.label}` ); } else if ( resolvedConversationAction.kind === "active_storyboard_followup" && activeStoryboardTaskForChat?.task?.id ) { console.log( `[Smart Auto Mode] 🔐 Force agent conversation due to active storyboard task: ${activeStoryboardTaskForChat.task.id}` ); } else if (resolvedConversationAction.kind === "image_generation_followup") { console.log("[Smart Auto Mode] 🪄 Force agent conversation due to image generation follow-up workflow"); } else if (resolvedConversationAction.kind === "image_edit_followup") { console.log("[Smart Auto Mode] 🖼️ Force agent conversation due to image edit follow-up workflow"); } } } const shouldFastTrackPromptOnly = shouldRunAgentConversation && taskBoundaryDecision.taskType === "prompt_only"; if (shouldFastTrackPromptOnly) { if (!requestResolvedMediaInput) { requestResolvedMediaInput = resolveChatMediaInput({ messageParts: message.parts as any[], historyMessages: [], contextPayload: effectiveContextPayload, requestOrigin, maxImages: 4, }); } if (!requestTargetResolution) { requestTargetResolution = resolveTargetAssets({ text: requestTextContent, mediaInput: requestResolvedMediaInput, contextPayload: effectiveContextPayload, workingState: effectiveWorkingState || undefined, }); } smartAutoDetectedIntent = "vision"; smartAutoNeedsClarification = false; smartAutoFriendlyMessage = "收到，我先分析这张图并整理可直接使用的提示词，不会直接生图。"; console.log("[Smart Auto Mode] ✍️ Fast-path prompt-only request, skip heavy auto-agent workflow"); } // ========== Smart Auto Mode Logic ========== if (shouldRunAgentConversation && !shouldFastTrackPromptOnly) { console.log("[Smart Auto Mode] 🚀 Starting intelligent analysis..."); let autoTextForFallback = ""; let autoFromHistoryForFallback = false; let autoCurrentImageUrlsForFallback: string[] = []; let confirmedPromptForGeneration: string | undefined; let forcePromptDrivenImageGeneration = false; let shouldPreferArbitratedFriendlyMessage = false; const smartAutoInputText = resolvedConversationAction.effectiveInputText || requestTextContent; try { // 🆕 首先获取历史消息（在 Smart Auto Mode 内部单独获取，避免依赖后面的 messagesFromDb） const loadedHistoryMessages = await getMessagesByChatId({ id }); smartAutoHistoryMessagesFromDb = pruneTaskBoundaryHistoryMessages({ decision: taskBoundaryDecision, messages: loadedHistoryMessages, purpose: "model", hasCurrentImageInput: hasImageInputForConversationAction, }); smartAutoMediaHistoryMessagesFromDb = pruneTaskBoundaryHistoryMessages({ decision: taskBoundaryDecision, messages: loadedHistoryMessages, purpose: "media", hasCurrentImageInput: hasImageInputForConversationAction, }); console.log("[Smart Auto Mode] Loaded", loadedHistoryMessages.length, "messages from DB"); console.log("[Smart Auto Mode] Pruned history:", { modelMessages: smartAutoHistoryMessagesFromDb.length, mediaMessages: smartAutoMediaHistoryMessagesFromDb.length, taskType: taskBoundaryDecision.taskType, taskMode: taskBoundaryDecision.taskMode, }); autoTextForFallback = smartAutoInputText; if (!requestWorkingState && !shouldIsolateDirectComposerContext) { requestWorkingState = await getConversationWorkingStateByChatId({ chatId: id, }); } confirmedPromptForGeneration = confirmedPromptForConversationState; const isPromptCarryForwardExecution = resolvedConversationAction.kind === "prompt_carry_forward"; forcePromptDrivenImageGeneration = resolvedConversationAction.usePromptCarryForward && Boolean(confirmedPromptForGeneration); if (!requestResolvedMediaInput) { requestResolvedMediaInput = resolveChatMediaInput({ messageParts: message.parts as any[], historyMessages: smartAutoMediaHistoryMessagesFromDb, contextPayload: effectiveContextPayload, requestOrigin, maxImages: 4, }); } if (!requestTargetResolution) { requestTargetResolution = resolveTargetAssets({ text: requestTextContent, mediaInput: requestResolvedMediaInput, contextPayload: effectiveContextPayload, workingState: effectiveWorkingState || undefined, }); } const hasSelectedImageHint = Boolean(effectiveContextPayload?.selectedNodeIds?.length) || /用户已选中\s*\d+\s*张图片/.test(effectiveAutoContextText); const hasExplicitTargetHint = requestTargetResolution.resolutionMode === "text_reference" || /第[一二三四五六七八九十\d]+张|左边|左侧|左上|左下|右边|右侧|右上|右下|上面|上方|下面|下方|中间|居中|中央|选中/.test(requestTextContent); const hasActionableEditIntent = /改|调整|优化|换|替换|去掉|抠|重绘|增强|修|处理|按照|按|保留|统一|色差|背景|清晰|风格/.test( requestTextContent ); const currentMessageImageInputs = requestResolvedMediaInput.currentMessageImageUrls.slice(0, 4); let imageUrls = requestTargetResolution.targetImageUrls.length > 0 ? requestTargetResolution.targetImageUrls.slice(0, 4) : requestResolvedMediaInput.resolvedImageUrls.slice(0, 4); let hasImage = imageUrls.length > 0; autoCurrentImageUrlsForFallback = currentMessageImageInputs.length > 0 ? currentMessageImageInputs.slice() : imageUrls.slice(); smartAutoHistoryImages = currentMessageImageInputs.length === 0 ? imageUrls.slice(0, 4) : []; autoFromHistoryForFallback = currentMessageImageInputs.length === 0 && imageUrls.length > 0; if (forcePromptDrivenImageGeneration) { imageUrls = []; hasImage = false; autoCurrentImageUrlsForFallback = []; smartAutoHistoryImages = []; autoFromHistoryForFallback = false; autoTextForFallback = confirmedPromptForGeneration || requestTextContent; smartAutoOptimizedPrompt = confirmedPromptForGeneration; console.log("[Smart Auto Mode] 🎯 Using confirmed prompt-only result for image generation"); } console.log("[Smart Auto Mode] Input:", { textContent: requestTextContent.substring(0, 100), hasImage, imageCount: imageUrls.length, fromHistory: smartAutoHistoryImages.length > 0, resolutionMode: requestTargetResolution.resolutionMode, resolutionConfidence: requestTargetResolution.confidence, }); // 🆕 从历史消息中提取对话文本（用于上下文理解） const historyMessages: { role: "user" | "assistant"; content: string }[] = []; for (const msg of smartAutoHistoryMessagesFromDb.slice(-10)) { // 最多取最近10条 const parts = msg.parts as any[]; if (parts && parts.length > 0) { // 提取文本内容 const textParts = parts.filter((p: any) => p.type === 'text'); const content = textParts.map((p: any) => p.text || '').join(' ').trim(); if (content) { historyMessages.push({ role: msg.role as "user" | "assistant", content: content.substring(0, 300), // 限制长度 }); } } } console.log("[Smart Auto Mode] History context:", historyMessages.length, "messages"); // 2. 使用 LangGraph 驱动的自动模式工作流 const { runAutoAgentGraph } = await import("@/lib/ai/auto-agent-graph"); const autoGraphTimeoutMs = getBudgetedTimeout({ startedAt: requestStartedAt, preferredMs: SMART_AUTO_GRAPH_TIMEOUT_MS, minimumMs: 8_000, }); if (!autoGraphTimeoutMs) { throw new OperationTimeoutError( "runAutoAgentGraph.preResponseBudget", CHAT_PRE_RESPONSE_SOFT_TIMEOUT_MS ); } const composedConversationState = composeConversationStateProtocol({ bootstrap: conversationStateBootstrap, autoContextText: effectiveAutoContextText || undefined, resolvedMediaInput: requestResolvedMediaInput, targetResolution: requestTargetResolution, }); console.log("[ConversationStateProtocol] Composed:", { workflowType: composedConversationState.threadState.workflowState?.type || null, workflowStatus: composedConversationState.threadState.workflowState?.status || null, memoryHintsCount: composedConversationState.threadState.memoryHints.length, contextSummaryPreview: composedConversationState.threadState.contextSummary?.slice(0, 180), }); pendingProtocolDebugEvents.push({ phase: "composed", payload: { workflowState: composedConversationState.threadState.workflowState, memoryHints: composedConversationState.threadState.memoryHints, contextSummary: composedConversationState.threadState.contextSummary, }, }); const autoGraphResult = await withOperationTimeout( runAutoAgentGraph({ text: smartAutoInputText, hasImage, imageUrls, assetIds: Array.from( new Set([ ...extractMessageImageAssetIds(message.parts as any[]), ...((effectiveContextPayload?.selectedAssetIds || []).filter(Boolean) as string[]), ]), ), imageFromHistory: smartAutoHistoryImages.length > 0, targetResolutionMode: requestTargetResolution.resolutionMode, mediaSource: requestResolvedMediaInput.source, hasCanvasSelection: Boolean(effectiveContextPayload?.selectedNodeIds?.length) || requestResolvedMediaInput.selectedImageUrls.length > 0, selectedImageCount: requestResolvedMediaInput.selectedImageUrls.length, historyImageCount: requestResolvedMediaInput.historyAssistantImageUrls.length + requestResolvedMediaInput.historyUserImageUrls.length, currentUploadImageCount: requestResolvedMediaInput.currentMessageImageUrls.length, historyMessages, context: composedConversationState.composedConversationMemory.contextSummary || effectiveAutoContextText || undefined, preferredModelId: preferredAgentModelId, threadState: composedConversationState.threadState, }), "runAutoAgentGraph", autoGraphTimeoutMs, ); const requestedStoryboardGrid = detectRequestedStoryboardGrid( pendingGridConfirmation && isGridConfirmationReply(requestTextContent) ? pendingGridConfirmation.originalPrompt : requestTextContent ); if (requestedStoryboardGrid) { storyboardPreferenceMemory = buildStoryboardPreferenceMemoryUpdate({ existing: storyboardPreferenceMemory, preferredGridType: requestedStoryboardGrid.gridType, preferredShotCount: requestedStoryboardGrid.shotCount, prefersConfirmation: true, lastUsedGridType: requestedStoryboardGrid.gridType, lastUsedShotCount: requestedStoryboardGrid.shotCount, }); } smartAutoPlan = autoGraphResult.plan; smartAutoExecution = autoGraphResult.execution; if (autoGraphResult.workflowDecision) { smartAutoWorkflowDecision = autoGraphResult.workflowDecision; console.log("[Smart Auto Mode] Workflow decision:", { action: autoGraphResult.workflowDecision.action, intent: autoGraphResult.workflowDecision.confirmationIntent, workflowId: autoGraphResult.workflowDecision.workflowId, workflowType: autoGraphResult.workflowDecision.workflowType, nextWorkflowStatus: autoGraphResult.workflowDecision.nextWorkflowStatus, allowWorkflowExecution: autoGraphResult.workflowDecision.allowWorkflowExecution, }); if ( autoGraphResult.workflowDecision.workflowType === "storyboard_video" && autoGraphResult.workflowDecision.workflowId ) { const storyboardTaskId = autoGraphResult.workflowDecision.workflowId; try { switch (autoGraphResult.workflowDecision.action) { case "awaiting_confirmation": await markVideoStoryboardAwaitingConfirmation({ taskId: storyboardTaskId, }); break; case "confirmed": await markVideoStoryboardConfirmed({ taskId: storyboardTaskId, messageId: message.id, }); break; case "revision_requested": await markVideoStoryboardDraftForRevision({ taskId: storyboardTaskId, }); break; case "cancel_requested": await cancelVideoStoryboardTaskByUser({ taskId: storyboardTaskId, error: "user_cancelled_via_chat", }); break; case "pass": default: break; } } catch (workflowPersistError) { console.warn( "[Smart Auto Mode] Failed to persist storyboard workflow decision:", workflowPersistError ); } } } if (forcePromptDrivenImageGeneration && confirmedPromptForGeneration) { smartAutoDetectedIntent = "image_gen"; smartAutoOptimizedPrompt = confirmedPromptForGeneration; smartAutoExecution = { mode: "generate_image", routeIntent: "image_gen", routeSubIntent: undefined, targetSource: "none", preferredCapability: "generate_image", shouldUseStructuredReply: true, shouldOptimizePrompt: true, shouldApplyImagePipeline: true, shouldApplyVideoPipeline: false, shouldStayConversational: false, shouldPublishToCanvas: false, toolCalls: [], availableTools: [], }; smartAutoFriendlyMessage = smartAutoFriendlyMessage || (isPromptCarryForwardExecution ? "收到，我会使用上一轮优化后的提示词，结合你这轮图片继续执行。" : "收到，我会按你刚确认的提示词直接生图。"); } if (smartAutoPlan) { console.log("[Smart Auto Mode] Plan result:", { goal: smartAutoPlan.goal, nextAction: smartAutoPlan.nextAction, intent: smartAutoPlan.intent, targetSource: smartAutoPlan.targetSource, preferredCapability: smartAutoPlan.preferredCapability, confidence: smartAutoPlan.confidence, toolCalls: smartAutoPlan.toolCalls?.map((tool) => tool.name) || [], }); } if (autoGraphResult.analysis) { console.log("[Smart Auto Mode] Analysis result:", { intent: autoGraphResult.analysis.intent, subIntent: autoGraphResult.analysis.subIntent, confidence: autoGraphResult.analysis.confidence, needsClarification: autoGraphResult.analysis.needsClarification, }); const arbitratedIntent = arbitrateAgentIntent({ text: requestTextContent, hasImage, hasHistoryImage: smartAutoHistoryImages.length > 0, actionResolution: resolvedConversationAction, taskBoundaryDecision, analysis: autoGraphResult.analysis, execution: smartAutoExecution, targetSource: requestTargetResolution.resolutionMode, }); if (arbitratedIntent.overridden) { console.log("[IntentArbiter] Override auto-agent intent:", { fromIntent: autoGraphResult.analysis.intent, fromMode: autoGraphResult.execution?.mode, toIntent: arbitratedIntent.analysis.intent, toMode: arbitratedIntent.execution.mode, reason: arbitratedIntent.reason, }); } smartAutoExecution = arbitratedIntent.execution as AgentPreparedExecution; smartAutoDetectedIntent = arbitratedIntent.analysis.intent; smartAutoNeedsClarification = Boolean(arbitratedIntent.analysis.needsClarification) || (!arbitratedIntent.overridden && smartAutoPlan?.nextAction === "clarify"); smartAutoClarificationQuestion = arbitratedIntent.analysis.clarifyingQuestions?.[0]; if (arbitratedIntent.friendlyMessage) { smartAutoFriendlyMessage = arbitratedIntent.friendlyMessage; shouldPreferArbitratedFriendlyMessage = true; } const targetExecutionOverride = resolveTargetExecutionOverride({ hasImage, hasSelectedImageHint, hasExplicitTargetHint, hasActionableEditIntent, executionMode: smartAutoExecution?.mode, friendlyMessage: smartAutoFriendlyMessage, optimizedPrompt: smartAutoOptimizedPrompt, requestText: requestTextContent, }); if ( smartAutoNeedsClarification && targetExecutionOverride.shouldBypassClarification ) { smartAutoNeedsClarification = false; smartAutoClarificationQuestion = undefined; smartAutoOptimizedPrompt = targetExecutionOverride.nextOptimizedPrompt; smartAutoFriendlyMessage = targetExecutionOverride.nextFriendlyMessage; console.log( "[Smart Auto Mode] ✅ Skip clarification due to explicit target/selection context" ); } } const requestedGrid = !pendingGridConfirmation || smartAutoInputText === requestTextContent ? detectRequestedStoryboardGrid(requestTextContent) : pendingGridConfirmation; const gridConfirmationResolution = resolveGridConfirmationRequest({ requestedGrid, hasActiveStoryboardTask: Boolean(activeStoryboardTaskForChat?.task?.id), executionMode: smartAutoExecution?.mode, detectedIntent: smartAutoDetectedIntent, requestText: requestTextContent, effectiveInputText: smartAutoInputText, }); if (gridConfirmationResolution.shouldRequestConfirmation) { shouldAskGridConfirmation = true; smartAutoNeedsClarification = true; smartAutoClarificationQuestion = gridConfirmationResolution.clarificationMessage; smartAutoFriendlyMessage = gridConfirmationResolution.clarificationMessage; pendingGridConfirmation = gridConfirmationResolution.pendingGridConfirmation; console.log("[Smart Auto Mode] 🧱 Require grid confirmation before generation:", { gridType: gridConfirmationResolution.pendingGridConfirmation?.gridType, shotCount: gridConfirmationResolution.pendingGridConfirmation?.shotCount, }); } if (autoGraphResult.friendlyMessage && !shouldPreferArbitratedFriendlyMessage) { smartAutoFriendlyMessage = autoGraphResult.friendlyMessage; console.log("[Smart Auto Mode] 💬 Conversational reply:", smartAutoFriendlyMessage); } if (shouldAskGridConfirmation && smartAutoClarificationQuestion) { smartAutoFriendlyMessage = smartAutoClarificationQuestion; } if ( !smartAutoNeedsClarification && hasImage && hasActionableEditIntent && (hasSelectedImageHint || hasExplicitTargetHint) && smartAutoFriendlyMessage && (/[?？]$/.test(smartAutoFriendlyMessage.trim()) || /哪一张|哪个|请先确认/.test(smartAutoFriendlyMessage)) ) { smartAutoFriendlyMessage = "收到，我将按你指定的目标图直接执行。"; console.log("[Smart Auto Mode] ✅ Replace clarification-like friendly reply"); } const autoIntent = smartAutoDetectedIntent || autoGraphResult.analysis?.intent; const autoExecutionMode = smartAutoExecution?.mode; const isImageTask = autoExecutionMode === "generate_image" || autoExecutionMode === "edit_image"; const needsRicherAutoReply = !smartAutoNeedsClarification && !smartAutoFriendlyMessage; if (needsRicherAutoReply) { if (isImageTask) { smartAutoFriendlyMessage = autoGraphResult.analysis?.conversationalReply || "我先分析你这轮图片里的主体、风格和构图，再给你执行方案。"; } else if (autoIntent === "vision") { smartAutoFriendlyMessage = autoGraphResult.analysis?.conversationalReply || "收到，我先看图分析，再直接回答你。"; } else if (autoIntent === "chat") { smartAutoFriendlyMessage = autoGraphResult.analysis?.conversationalReply || "收到，我们先继续聊你的这个需求。"; } else { smartAutoFriendlyMessage = smartAutoExecution?.shouldUseStructuredReply ? buildAutoModeFallbackMessage({ text: autoTextForFallback, hasImage, fromHistory: smartAutoHistoryImages.length > 0, }) : "收到，我先理解你的当前目标，再继续处理。"; } console.log("[Smart Auto Mode] 🧩 Filled missing conversational reply"); } if (autoGraphResult.optimizedPrompt) { smartAutoOptimizedPrompt = autoGraphResult.optimizedPrompt; console.log("[Smart Auto Mode] ✨ Prompt optimized (LLM):", { original: requestTextContent.substring(0, 50), optimized: autoGraphResult.optimizedPrompt.substring(0, 100), }); } if (autoGraphResult.videoPlan) { smartAutoVideoPlan = autoGraphResult.videoPlan; smartAutoRecommendedVideoParams = autoGraphResult.videoPlan.recommendedVideoParams; if (!smartAutoOptimizedPrompt && autoGraphResult.videoPlan.finalPrompt?.trim()) { smartAutoOptimizedPrompt = autoGraphResult.videoPlan.finalPrompt.trim(); smartAutoPromptFromVideoPlan = true; } console.log("[Smart Auto Mode] 🎬 Agent video plan ready:", { hasScript: Boolean(autoGraphResult.videoPlan.script), storyboardCount: autoGraphResult.videoPlan.storyboard?.length || 0, hasFinalPrompt: Boolean(autoGraphResult.videoPlan.finalPrompt), recommendedVideoParams: smartAutoRecommendedVideoParams, }); } if (autoGraphResult.recommendedImageParams) { smartAutoRecommendedImageParams = autoGraphResult.recommendedImageParams; console.log("[Smart Auto Mode] 🎛️ Agent recommended image params:", smartAutoRecommendedImageParams); } const shouldForceConversationalRouteForVisualModel = shouldKeepSelectedModel && requestedModelIsFixedVisual && !requestedModelIsConversationMultimodal && Boolean(smartAutoExecution?.shouldStayConversational); if (autoGraphResult.route && (!shouldKeepSelectedModel || shouldForceConversationalRouteForVisualModel)) { const routeIsVideo = isVideoProviderType(autoGraphResult.route.providerType, { modelId: autoGraphResult.route.modelId, }); if (routeIsVideo && !allowAgentVideoGeneration) { const fallbackIntent: "image_edit" | "image_gen" = hasImage ? "image_edit" : "image_gen"; smartAutoDetectedIntent = fallbackIntent; smartAutoExecution = smartAutoExecution ? { ...smartAutoExecution, mode: hasImage ? "edit_image" : "generate_image", routeIntent: fallbackIntent, shouldApplyImagePipeline: true, shouldApplyVideoPipeline: false, shouldOptimizePrompt: true, shouldStayConversational: false, shouldUseStructuredReply: true, } : smartAutoExecution; smartAutoVideoPlan = undefined; smartAutoRecommendedVideoParams = undefined; if (smartAutoPromptFromVideoPlan) { smartAutoOptimizedPrompt = requestTextContent.trim() || undefined; smartAutoPromptFromVideoPlan = false; } smartAutoFriendlyMessage = smartAutoFriendlyMessage && !/视频/.test(smartAutoFriendlyMessage) ? smartAutoFriendlyMessage : "当前 Agent 默认按图片任务执行；如需生成视频，请先手动切换到“视频”模式。"; console.log( `[Smart Auto Mode] 🚫 Blocked auto video route (${autoGraphResult.route.displayName}), fallback to ${fallbackIntent}`, ); try { const { routeToModel } = await import("@/lib/ai/model-router"); const fallbackRoute = await routeToModel({ intent: fallbackIntent, subIntent: autoGraphResult.analysis?.subIntent, hasImage, }); if (!isVideoProviderType(fallbackRoute.providerType, { modelId: fallbackRoute.modelId })) { finalModelId = fallbackRoute.id; console.log( `[Smart Auto Mode] ✅ Fallback routed to: ${fallbackRoute.displayName} (${fallbackRoute.reason})`, ); } else { console.warn( `[Smart Auto Mode] ⚠️ Fallback route still video (${fallbackRoute.displayName}), keep previous model`, ); } } catch (fallbackRouteError) { console.error("[Smart Auto Mode] Failed to resolve non-video fallback route:", fallbackRouteError); } } else { finalModelId = autoGraphResult.route.id; console.log( shouldForceConversationalRouteForVisualModel ? `[Smart Auto Mode] ✅ Switched visual model to conversational route: ${autoGraphResult.route.displayName} (${autoGraphResult.route.reason})` : `[Smart Auto Mode] ✅ Routed to: ${autoGraphResult.route.displayName} (${autoGraphResult.route.reason})`, ); } } else if (autoGraphResult.route && shouldKeepSelectedModel) { finalModelId = requestedModelId; console.log("[Smart Auto Mode] 🔒 Keep user-selected model, skip routing override"); } } catch (autoError) { // 智能分析失败时，回退到简单逻辑 console.error("[Smart Auto Mode] ❌ Error, falling back:", autoError); const isAutoTimeout = autoError instanceof OperationTimeoutError; if (isAutoTimeout) { console.warn( `[Smart Auto Mode] ⏱️ Auto-agent timed out or exceeded pre-response budget (${CHAT_PRE_RESPONSE_SOFT_TIMEOUT_MS}ms), using local fallback`, ); } try { const { analyzeIntent } = await import("@/lib/ai/smart-analyzer"); const rescueImageUrls = autoCurrentImageUrlsForFallback.length > 0 ? autoCurrentImageUrlsForFallback : smartAutoHistoryImages; const rescueHasImage = rescueImageUrls.length > 0; const rescueFromHistory = rescueHasImage && autoCurrentImageUrlsForFallback.length === 0 && smartAutoHistoryImages.length > 0; const rescueAssetIds = Array.from( new Set([ ...extractMessageImageAssetIds(message.parts as any[]), ...((effectiveContextPayload?.selectedAssetIds || []).filter(Boolean) as string[]), ]), ); const rescueTimeoutMs = getBudgetedTimeout({ startedAt: requestStartedAt, preferredMs: SMART_AUTO_FALLBACK_TIMEOUT_MS, minimumMs: 5_000, }); if (rescueTimeoutMs) { const rescueAnalysis = await withOperationTimeout( analyzeIntent({ text: autoTextForFallback, hasImage: rescueHasImage, imageUrls: rescueImageUrls, assetIds: rescueAssetIds, imageFromHistory: rescueFromHistory || autoFromHistoryForFallback, historyMessages: [], context: undefined, preferredModelId: preferredAgentModelId, }), "fallbackAnalyzeIntent", rescueTimeoutMs, ); smartAutoDetectedIntent = rescueAnalysis.intent; const arbitratedIntent = arbitrateAgentIntent({ text: autoTextForFallback, hasImage: rescueHasImage, hasHistoryImage: rescueFromHistory || autoFromHistoryForFallback, actionResolution: resolvedConversationAction, taskBoundaryDecision, analysis: rescueAnalysis, execution: undefined, targetSource: requestTargetResolution?.resolutionMode || "none", }); if (arbitratedIntent.overridden) { console.log("[IntentArbiter] Override rescue intent:", { fromIntent: rescueAnalysis.intent, toIntent: arbitratedIntent.analysis.intent, toMode: arbitratedIntent.execution.mode, reason: arbitratedIntent.reason, }); } smartAutoDetectedIntent = arbitratedIntent.analysis.intent; smartAutoNeedsClarification = Boolean(arbitratedIntent.analysis.needsClarification); smartAutoClarificationQuestion = arbitratedIntent.analysis.clarifyingQuestions?.[0]; smartAutoExecution = arbitratedIntent.execution as AgentPreparedExecution; if (arbitratedIntent.friendlyMessage || rescueAnalysis.conversationalReply?.trim()) { smartAutoFriendlyMessage = ( arbitratedIntent.friendlyMessage || rescueAnalysis.conversationalReply || "" ).trim(); } } else { console.warn("[Smart Auto Mode] Skip rescue analysis due to pre-response budget"); } } catch (rescueError) { console.error("[Smart Auto Mode] Rescue analysis failed:", rescueError); try { const { fallbackKeywordAnalysis } = await import("@/lib/ai/smart-analyzer"); const keywordFallback = fallbackKeywordAnalysis( autoTextForFallback, autoCurrentImageUrlsForFallback.length > 0 || smartAutoHistoryImages.length > 0, ); const arbitratedIntent = arbitrateAgentIntent({ text: autoTextForFallback, hasImage: autoCurrentImageUrlsForFallback.length > 0 || smartAutoHistoryImages.length > 0, hasHistoryImage: smartAutoHistoryImages.length > 0 || autoFromHistoryForFallback, actionResolution: resolvedConversationAction, taskBoundaryDecision, analysis: keywordFallback, execution: undefined, targetSource: requestTargetResolution?.resolutionMode || "none", }); if (arbitratedIntent.overridden) { console.log("[IntentArbiter] Override keyword-fallback intent:", { fromIntent: keywordFallback.intent, toIntent: arbitratedIntent.analysis.intent, toMode: arbitratedIntent.execution.mode, reason: arbitratedIntent.reason, }); } smartAutoDetectedIntent = arbitratedIntent.analysis.intent; smartAutoNeedsClarification = Boolean(arbitratedIntent.analysis.needsClarification); smartAutoClarificationQuestion = arbitratedIntent.analysis.clarifyingQuestions?.[0]; smartAutoExecution = arbitratedIntent.execution as AgentPreparedExecution; if (arbitratedIntent.friendlyMessage || keywordFallback.conversationalReply?.trim()) { smartAutoFriendlyMessage = ( arbitratedIntent.friendlyMessage || keywordFallback.conversationalReply || "" ).trim(); console.warn("[Smart Auto Mode] Applied keyword fallback after rescue failure"); } } catch (keywordFallbackError) { console.warn( "[Smart Auto Mode] Keyword fallback after rescue failure also failed:", keywordFallbackError, ); } } if ( !smartAutoOptimizedPrompt && smartAutoExecution?.shouldOptimizePrompt && (smartAutoDetectedIntent === "image_edit" || smartAutoDetectedIntent === "image_gen") ) { try { const { optimizePrompt } = await import("@/lib/ai/prompt-optimizer"); const optimizedFallback = optimizePrompt({ originalPrompt: autoTextForFallback, intent: smartAutoDetectedIntent, subIntent: smartAutoExecution.routeSubIntent, }); if (optimizedFallback.optimizedPrompt?.trim()) { smartAutoOptimizedPrompt = optimizedFallback.optimizedPrompt.trim(); console.log("[Smart Auto Mode] ✨ Prompt optimized (fallback heuristic):", { original: autoTextForFallback.substring(0, 80), optimized: smartAutoOptimizedPrompt.substring(0, 140), intent: smartAutoDetectedIntent, subIntent: smartAutoExecution.routeSubIntent || null, }); } if (!smartAutoRecommendedImageParams && optimizedFallback.recommendedImageParams) { smartAutoRecommendedImageParams = optimizedFallback.recommendedImageParams; } } catch (fallbackOptimizeError) { console.warn("[Smart Auto Mode] Fallback prompt optimization failed:", fallbackOptimizeError); } } const allModels = await getEnabledAiModelsWithProvider(); const hasImage = autoCurrentImageUrlsForFallback.length > 0 || smartAutoHistoryImages.length > 0; let fallbackVisualSummary: string | undefined; if (isAutoTimeout && hasImage) { try { const currentImageUrls = autoCurrentImageUrlsForFallback.length > 0 ? autoCurrentImageUrlsForFallback : collectImageInputsFromMessageParts(message.parts as any[], requestOrigin); const summaryImageUrls = currentImageUrls.length > 0 ? currentImageUrls : smartAutoHistoryImages; if (summaryImageUrls.length > 0) { const { summarizeImageForExecutionReport } = await import("@/lib/ai/smart-analyzer"); const summaryAssetIds = Array.from( new Set([ ...extractMessageImageAssetIds(message.parts as any[]), ...((effectiveContextPayload?.selectedAssetIds || []).filter(Boolean) as string[]), ]), ); const fallbackSummaryTimeoutMs = getBudgetedTimeout({ startedAt: requestStartedAt, preferredMs: SMART_AUTO_IMAGE_SUMMARY_TIMEOUT_MS, minimumMs: 5_000, }); if (fallbackSummaryTimeoutMs) { const summary = await withOperationTimeout( summarizeImageForExecutionReport({ text: autoTextForFallback, imageUrls: summaryImageUrls, context: effectiveAutoContextText || undefined, preferredModelId: preferredAgentModelId, assetIds: summaryAssetIds, }), "fallbackVisualSummary", fallbackSummaryTimeoutMs, ); if (summary?.trim()) { fallbackVisualSummary = summary.trim(); } } else { console.warn("[Smart Auto Mode] Skip fallback visual summary due to pre-response budget"); } } } catch (summaryError) { console.warn("[Smart Auto Mode] Fallback visual summary failed:", summaryError); } } // 回退路由：优先按分析出的意图选模型，避免“有图就生图” let bestModel = null; try { const fallbackIntent = smartAutoDetectedIntent || (hasImage ? "vision" : "chat"); const { routeToModel } = await import("@/lib/ai/model-router"); const fallbackRoute = await routeToModel({ intent: fallbackIntent, subIntent: smartAutoExecution?.routeSubIntent, hasImage, }); bestModel = allModels.find( (item) => item.model.id === fallbackRoute.id || item.model.modelId === fallbackRoute.modelId, ) || null; if (bestModel) { console.log( `[Smart Auto Mode] Fallback routed by intent: ${fallbackRoute.displayName} (${fallbackRoute.reason})`, ); } } catch (fallbackRouteError) { console.warn("[Smart Auto Mode] Fallback routeByIntent failed, use local heuristic:", fallbackRouteError); } if (!bestModel && hasImage) { bestModel = allModels.find( (m) => m.model.supportsImageGen && !m.model.isSubModel && !isVideoProviderType(m.provider?.type, m.model), ); if (isAutoTimeout) { smartAutoFriendlyMessage = buildAutoModeFallbackMessage({ text: autoTextForFallback, hasImage: true, fromHistory: autoFromHistoryForFallback || smartAutoHistoryImages.length > 0, imageSummary: fallbackVisualSummary, }); } else { smartAutoFriendlyMessage = smartAutoFriendlyMessage || buildAutoModeFallbackMessage({ text: autoTextForFallback, hasImage: true, fromHistory: autoFromHistoryForFallback || smartAutoHistoryImages.length > 0, imageSummary: fallbackVisualSummary, }); } } else { smartAutoFriendlyMessage = smartAutoFriendlyMessage || buildAutoModeFallbackMessage({ text: autoTextForFallback, hasImage: false, fromHistory: false, }); } if (!bestModel) { bestModel = allModels.find( (m) => m.model.isDefault && !m.model.isSubModel && !isVideoProviderType(m.provider?.type, m.model), ); } if (!bestModel && allModels.length > 0) { bestModel = allModels.find( (m) => !m.model.isSubModel && !isVideoProviderType(m.provider?.type, m.model), ) || allModels.find((m) => !m.model.isSubModel) || allModels[0]; } if (shouldKeepSelectedModel) { finalModelId = requestedModelId; console.log("[Smart Auto Mode] 🔒 Keep user-selected model in fallback mode"); } else if (bestModel) { finalModelId = bestModel.model.id; console.log(`[Smart Auto Mode] Fallback selected: ${bestModel.model.displayName}`); } else { return new ChatSDKError("bad_request:api", "No available models for auto mode").toResponse(); } } } // ========== End Smart Auto Mode Logic ========== if (shouldRunAgentConversation && !allowAgentVideoGeneration) { const finalModelIsUuid = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test(finalModelId); let finalProviderType: string | undefined; let finalModelConfig: Awaited> | null = null; if (finalModelIsUuid) { finalModelConfig = await getAiModelById({ id: finalModelId }).catch(() => null); if (finalModelConfig) { finalProviderType = ( await getAiProviderById({ id: finalModelConfig.providerId }).catch(() => null) )?.type as string | undefined; } } if (isVideoProviderType(finalProviderType, finalModelConfig || undefined)) { const allModels = await getEnabledAiModelsWithProvider(false); const fallbackModel = allModels.find( (m) => !m.model.isSubModel && m.model.supportsImageGen && !isVideoProviderType(m.provider?.type, m.model), ) || allModels.find( (m) => !m.model.isSubModel && !isVideoProviderType(m.provider?.type, m.model), ); if (fallbackModel) { const hasImageInput = message.parts.some((part: any) => part?.type === "file" || part?.type === "image" || part?.type === "image_url" ); smartAutoDetectedIntent = hasImageInput ? "image_edit" : "image_gen"; smartAutoExecution = smartAutoExecution ? { ...smartAutoExecution, mode: hasImageInput ? "edit_image" : "generate_image", routeIntent: hasImageInput ? "image_edit" : "image_gen", shouldApplyImagePipeline: true, shouldApplyVideoPipeline: false, shouldOptimizePrompt: true, shouldStayConversational: false, shouldUseStructuredReply: true, } : smartAutoExecution; finalModelId = fallbackModel.model.id; smartAutoVideoPlan = undefined; smartAutoRecommendedVideoParams = undefined; console.log( `[Smart Auto Mode] 🚫 Safety override: replace video model (${finalProviderType}) with ${fallbackModel.model.displayName}`, ); } } } if ( shouldRunAgentConversation && !smartAutoNeedsClarification && !smartAutoFriendlyMessage?.trim() ) { const latestUserText = message.parts .filter((part): part is { type: "text"; text: string } => part.type === "text") .map((part) => part.text) .join(" "); const hasCurrentImageInput = message.parts.some((part: any) => part?.type === "file" || part?.type === "image" || part?.type === "image_url" ); const hasHistoryImageInput = smartAutoHistoryImages.length > 0; smartAutoFriendlyMessage = smartAutoExecution?.shouldUseStructuredReply ? buildAutoModeFallbackMessage({ text: latestUserText, hasImage: hasCurrentImageInput || hasHistoryImageInput, fromHistory: !hasCurrentImageInput && hasHistoryImageInput, }) : "收到，我先理解你的当前目标，再继续处理。"; console.log("[Smart Auto Mode] 🧩 Backfilled conversational reply with fallback"); } if ( shouldRunAgentConversation && !smartAutoNeedsClarification && smartAutoExecution?.shouldUseStructuredReply ) { const latestUserText = message.parts .filter((part): part is { type: "text"; text: string } => part.type === "text") .map((part) => part.text) .join(" "); const hasCurrentImageInput = message.parts.some((part: any) => part?.type === "file" || part?.type === "image" || part?.type === "image_url" ); const hasHistoryImageInput = smartAutoHistoryImages.length > 0; smartAutoFriendlyMessage = ensureStructuredAgentFriendlyMessage({ message: smartAutoFriendlyMessage, text: latestUserText, hasImage: hasCurrentImageInput || hasHistoryImageInput, fromHistory: !hasCurrentImageInput && hasHistoryImageInput, }); } const hasUserCustomImageParams = hasUserCustomizedImageParams(requestBody.imageParams); const hasUserCustomVideoParams = hasUserCustomizedVideoParams(requestBody.videoParams); const shouldApplyAgentImageParams = shouldRunAgentConversation && !smartAutoNeedsClarification && Boolean(smartAutoExecution?.shouldApplyImagePipeline); const shouldApplyAgentVideoParams = shouldRunAgentConversation && !smartAutoNeedsClarification && allowAgentVideoGeneration && Boolean(smartAutoExecution?.shouldApplyVideoPipeline); const effectiveImageParams = shouldApplyAgentImageParams ? mergeImageParamsWithAgentRecommendation({ incoming: requestBody.imageParams, recommended: smartAutoRecommendedImageParams, intent: smartAutoDetectedIntent, hasUserCustomSettings: hasUserCustomImageParams, }) : requestBody.imageParams; const effectiveVideoParams = shouldApplyAgentVideoParams ? mergeVideoParamsWithAgentRecommendation({ incoming: requestBody.videoParams, recommended: smartAutoRecommendedVideoParams, hasUserCustomSettings: hasUserCustomVideoParams, }) : requestBody.videoParams; const defaultImageQualityParams = { aspectRatio: "auto", imageSize: smartAutoDetectedIntent === "image_edit" ? "4K" : "2K", imageCount: 1, } as const; if (shouldApplyAgentImageParams) { console.log("[Smart Auto Mode] 🎯 Effective image params:", { incoming: requestBody.imageParams, recommended: smartAutoRecommendedImageParams, merged: effectiveImageParams, hasUserCustomImageParams, }); } if (shouldApplyAgentVideoParams) { console.log("[Smart Auto Mode] 🎯 Effective video params:", { incoming: requestBody.videoParams, recommended: smartAutoRecommendedVideoParams, merged: effectiveVideoParams, hasUserCustomVideoParams, hasVideoPlan: Boolean(smartAutoVideoPlan), }); } if ( shouldRunAgentConversation && smartAutoFriendlyMessage && smartAutoExecution?.shouldUseStructuredReply ) { smartAutoFriendlyMessage = sanitizeAgentOutputSettingsSection({ message: smartAutoFriendlyMessage, userImageParams: requestBody.imageParams, hasUserCustomImageParams, }); smartAutoFriendlyMessage = injectOptimizedPromptSection({ message: smartAutoFriendlyMessage, originalText: requestTextContent, optimizedPrompt: smartAutoOptimizedPrompt, }); } if (shouldRunAgentConversation) { const currentImageUrls = extractMessageImageUrls(message.parts as any[]); const summaryImageUrls = currentImageUrls.length > 0 ? currentImageUrls : smartAutoHistoryImages; const shouldRequireImageAnalysis = summaryImageUrls.length > 0; if (shouldRequireImageAnalysis) { try { const { summarizeImageForExecutionReport } = await import("@/lib/ai/smart-analyzer"); const latestUserText = message.parts .filter((part): part is { type: "text"; text: string } => part.type === "text") .map((part) => part.text) .join(" "); const summaryAssetIds = Array.from( new Set([ ...extractMessageImageAssetIds(message.parts as any[]), ...((effectiveContextPayload?.selectedAssetIds || []).filter(Boolean) as string[]), ]), ); let summary: string | null = null; const summaryTimeoutMs = getBudgetedTimeout({ startedAt: requestStartedAt, preferredMs: SMART_AUTO_IMAGE_SUMMARY_TIMEOUT_MS, minimumMs: 6_000, }); if (summaryTimeoutMs) { summary = await withOperationTimeout( summarizeImageForExecutionReport({ text: latestUserText, imageUrls: summaryImageUrls, context: effectiveAutoContextText || undefined, preferredModelId: preferredAgentModelId, assetIds: summaryAssetIds, }), "enrichImageAnalysisSummary", summaryTimeoutMs, ); } else { console.warn("[Smart Auto Mode] Skip image summary enrichment due to pre-response budget"); } if (summary?.trim() && smartAutoExecution?.shouldUseStructuredReply) { const baseMessage = ensureStructuredAgentFriendlyMessage({ message: smartAutoFriendlyMessage, text: latestUserText, hasImage: true, fromHistory: currentImageUrls.length === 0 && summaryImageUrls.length > 0, }); smartAutoFriendlyMessage = ensureImageAnalysisSummarySection( baseMessage || buildAutoModeFallbackMessage({ text: latestUserText, hasImage: true, fromHistory: currentImageUrls.length === 0 && summaryImageUrls.length > 0, imageSummary: summary.trim(), }), summary.trim(), ); console.log( "[Smart Auto Mode] ✅ Image analysis summary ready:", summary.trim().slice(0, 160), ); } else if ( summary?.trim() && !smartAutoExecution?.shouldUseStructuredReply && !smartAutoFriendlyMessage ) { smartAutoFriendlyMessage = "收到，我会结合这张图的内容直接回答你。"; } else if (smartAutoFriendlyMessage && shouldEnrichImageAnalysisText(smartAutoFriendlyMessage)) { console.warn("[Smart Auto Mode] ⚠️ Image summary unavailable, keep original analysis text"); } } catch (enrichError) { console.warn("[Smart Auto Mode] Skip image analysis enrichment:", enrichError); } } } if ( shouldRunAgentConversation && smartAutoFriendlyMessage && smartAutoExecution?.shouldUseStructuredReply ) { smartAutoFriendlyMessage = normalizeAgentReportForDisplay(smartAutoFriendlyMessage); } if ( shouldRunAgentConversation && !shouldKeepSelectedModel && smartAutoExecution?.shouldStayConversational && preferredAgentModelId ) { if (finalModelId !== preferredAgentModelId) { finalModelId = preferredAgentModelId; console.log( `[Smart Auto Mode] 🧠 Conversational reply follows agent default model: ${configuredAgentConversationModelId || preferredAgentModelId}`, ); } } if (shouldRunAgentConversation && smartAutoExecution?.shouldStayConversational) { const candidateReplyDescriptor = await resolveAgentModelSlotDescriptor(finalModelId).catch( () => null, ); if (candidateReplyDescriptor) { agentModelSlots = upsertAgentModelSlot(agentModelSlots, "reply", { ...candidateReplyDescriptor, source: "candidate_reply_model", reason: "post_agent_routing", locked: shouldKeepSelectedModel, }); const providerHealth = getProviderHealthSnapshot(candidateReplyDescriptor.providerKey); if ( !shouldKeepSelectedModel && shouldAvoidProvider(providerHealth) ) { let fallbackReplyModelId: string | undefined; if ( preferredAgentModelId && preferredAgentModelId !== finalModelId ) { const preferredDescriptor = await resolveAgentModelSlotDescriptor( preferredAgentModelId, ).catch(() => null); if ( preferredDescriptor && !shouldAvoidProvider(getProviderHealthSnapshot(preferredDescriptor.providerKey)) ) { fallbackReplyModelId = preferredAgentModelId; } } if (!fallbackReplyModelId) { try { const { routeToModel } = await import("@/lib/ai/model-router"); const replyFallbackRoute = await routeToModel({ intent: "chat", subIntent: undefined, hasImage: false, }); const fallbackDescriptor = await resolveAgentModelSlotDescriptor( replyFallbackRoute.id, ).catch(() => null); if ( fallbackDescriptor && !shouldAvoidProvider(getProviderHealthSnapshot(fallbackDescriptor.providerKey)) ) { fallbackReplyModelId = replyFallbackRoute.id; } } catch (replyRouteError) { console.warn( "[Smart Auto Mode] Failed to resolve reply fallback route after provider degradation:", replyRouteError, ); } } if (fallbackReplyModelId && fallbackReplyModelId !== finalModelId) { finalModelId = fallbackReplyModelId; const fallbackDescriptor = await resolveAgentModelSlotDescriptor(finalModelId).catch( () => null, ); if (fallbackDescriptor) { agentModelSlots = upsertAgentModelSlot(agentModelSlots, "reply", { ...fallbackDescriptor, source: "provider_health_fallback", reason: providerHealth?.lastEventType || "degraded_provider", locked: false, }); } console.warn( `[Smart Auto Mode] ♻️ Switched reply model due provider health: ${candidateReplyDescriptor.displayName || candidateReplyDescriptor.modelId} -> ${fallbackDescriptor?.displayName || fallbackReplyModelId}`, ); } } } } const session = await auth(); if (!session?.user) { return new ChatSDKError("unauthorized:chat").toResponse(); } const userType: UserType = session.user.type; const messageCount = await getMessageCountByUserId({ id: session.user.id, differenceInHours: 24, }); if (messageCount > entitlementsByUserType[userType].maxMessagesPerDay) { return new ChatSDKError("rate_limit:chat").toResponse(); } // ========== 使用时间限制验证（后端安全检查） ========== try { // 从数据库获取网站设置 const { siteSettings } = await import("@/lib/db/schema"); const [settings] = await db.select().from(siteSettings).limit(1); if (settings?.usageTimeEnabled) { const now = new Date(); const currentTime = `${String(now.getHours()).padStart(2, "0")}:${String(now.getMinutes()).padStart(2, "0")}`; const startTime = settings.usageTimeStart || "08:00"; const endTime = settings.usageTimeEnd || "20:00"; const isWithinUsageTime = currentTime >= startTime && currentTime <= endTime; if (!isWithinUsageTime) { console.log(`[Chat API] ⛔ 访问被拒绝：当前时间 ${currentTime} 不在允许范围 ${startTime}-${endTime}`); return NextResponse.json( { error: "usage_time_restricted", message: settings.usageTimeMessage || "当前不在系统使用时间段内", currentTime, allowedTime: `${startTime} - ${endTime}` }, { status: 403 } ); } console.log(`[Chat API] ✅ 时间验证通过：${currentTime} 在允许范围 ${startTime}-${endTime}`); } } catch (timeCheckError) { console.error("[Chat API] ⚠️ 时间验证失败（继续执行）:", timeCheckError); // 如果时间检查失败，继续执行（避免因检查逻辑错误导致系统不可用） } let chat = await getChatById({ id }); let messagesFromDb: DBMessage[] = []; let messagesForMediaInput: DBMessage[] = []; if (chat) { if (chat.userId !== session.user.id) { return new ChatSDKError("forbidden:chat").toResponse(); } // Only fetch messages if chat already exists messagesFromDb = await getMessagesByChatId({ id }); } else { const title = await generateTitleFromUserMessage({ message, }); try { await saveChat({ id, userId: session.user.id, title, visibility: selectedVisibilityType, }); } catch (error) { const racedChat = await getChatById({ id }); if (!racedChat) { throw error; } if (racedChat.userId !== session.user.id) { return new ChatSDKError("forbidden:chat").toResponse(); } console.warn("[Chat API] saveChat conflicted, reused existing chat:", { chatId: id, userId: session.user.id, }); } chat = await getChatById({ id }); if (!chat) { throw new ChatSDKError("bad_request:database", "Failed to save chat"); } if (chat.userId !== session.user.id) { return new ChatSDKError("forbidden:chat").toResponse(); } // In a race scenario another request may have already written messages. messagesFromDb = await getMessagesByChatId({ id }); } const loadedMessagesForBoundary = shouldIsolateDirectComposerContext ? [] : messagesFromDb; messagesFromDb = shouldIsolateDirectComposerContext ? [] : pruneTaskBoundaryHistoryMessages({ decision: taskBoundaryDecision, messages: loadedMessagesForBoundary, purpose: "model", hasCurrentImageInput: hasImageInputForConversationAction, }); messagesForMediaInput = shouldIsolateDirectComposerContext ? [] : pruneTaskBoundaryHistoryMessages({ decision: taskBoundaryDecision, messages: loadedMessagesForBoundary, purpose: "media", hasCurrentImageInput: hasImageInputForConversationAction, }); if (!requestWorkingState && !shouldIsolateDirectComposerContext) { requestWorkingState = await getConversationWorkingStateByChatId({ chatId: id, }); } if (!requestResolvedMediaInput) { requestResolvedMediaInput = resolveChatMediaInput({ messageParts: message.parts as any[], historyMessages: messagesForMediaInput, contextPayload: effectiveContextPayload, requestOrigin, maxImages: 4, }); } if (!requestTargetResolution) { requestTargetResolution = resolveTargetAssets({ text: requestTextContent, mediaInput: requestResolvedMediaInput, contextPayload: effectiveContextPayload, workingState: effectiveWorkingState || undefined, }); } if ( requestResolvedMediaInput.currentMessageImageUrls.length === 0 && smartAutoHistoryImages.length === 0 && requestTargetResolution.targetImageUrls.length > 0 ) { smartAutoHistoryImages = requestTargetResolution.targetImageUrls.slice(0, 4); } const uiMessages = [...convertToUIMessages(messagesFromDb), message]; const hiddenPromptText = typeof hiddenPrompt === "string" ? hiddenPrompt.trim() : ""; const hiddenPromptPrefix = hiddenPromptText ? `${hiddenPromptText}\n\n` : ""; const uiMessagesForModel = hiddenPromptText ? applyHiddenPromptToMessages(uiMessages, hiddenPromptText) : uiMessages; const { longitude, latitude, city, country } = geolocation(request); const requestHints: RequestHints = { longitude, latitude, city, country, }; const insertedUserMessage = await db .insert(messageTable) .values({ chatId: id, id: message.id, role: "user", parts: message.parts as any, attachments: [], createdAt: new Date(), videoStorageStatus: null, temporaryVideoUrl: null, permanentVideoUrl: null, archivedAt: null, videoMetadata: null, }) .onConflictDoNothing({ target: [messageTable.id], }) .returning({ id: messageTable.id }); if (insertedUserMessage.length === 0) { console.warn( `[Chat API] Duplicate user message ignored (chatId=${id}, messageId=${message.id})` ); const duplicateStream = createUIMessageStream({ execute: async ({ writer }) => { await replayAssistantMessagesForDuplicateRequest({ chatId: id, userMessageId: message.id, writer, source: "duplicate-user-message", }); }, }); return createSseResponse(duplicateStream); } if (requestResolvedMediaInput && requestTargetResolution) { try { await createMessageTargetResolution({ userId: session.user.id, chatId: id, messageId: message.id, projectId: contextPayload?.projectId || null, rawText: requestTextContent, uploadedAssetIds: requestResolvedMediaInput.uploadedAssetIds, uploadedImageUrls: requestResolvedMediaInput.currentMessageImageUrls, selectedAssetIds: requestResolvedMediaInput.selectedAssetIds, selectedImageUrls: requestResolvedMediaInput.selectedImageUrls, candidateAssetIds: requestResolvedMediaInput.candidateAssetIds, candidateImageUrls: requestResolvedMediaInput.imageRefs.map((ref) => ref.url), resolvedAssetIds: requestTargetResolution.targetAssetIds, resolvedImageUrls: requestTargetResolution.targetImageUrls, resolutionMode: requestTargetResolution.resolutionMode, confidence: requestTargetResolution.confidence, needsClarification: requestTargetResolution.needsClarification, reasoning: { reason: requestTargetResolution.reason, fromHistory: requestTargetResolution.fromHistory, focusPolicy: contextPayload?.focusPolicy || null, contextSource: contextPayload?.source || null, }, }); await persistResolvedTargetTurn({ userId: session.user.id, chatId: id, projectId: contextPayload?.projectId || null, currentIntent: smartAutoExecution?.routeIntent || smartAutoDetectedIntent || (requestResolvedMediaInput.hasImage ? "vision" : requestBody.videoParams ? "video_gen" : null), currentFocusAssetIds: requestTargetResolution.targetAssetIds, currentFocusImageUrls: requestTargetResolution.targetImageUrls, selectedAssetIds: requestResolvedMediaInput.selectedAssetIds, selectedImageUrls: requestResolvedMediaInput.selectedImageUrls, lastUploadedAssetIds: requestResolvedMediaInput.uploadedAssetIds, lastUploadedImageUrls: requestResolvedMediaInput.currentMessageImageUrls, lastReferencedAssetIds: requestTargetResolution.candidateAssetIds, lastReferencedImageUrls: requestTargetResolution.candidateImageUrls, lastResolvedTargetAssetIds: requestTargetResolution.targetAssetIds, lastResolvedTargetImageUrls: requestTargetResolution.targetImageUrls, activeCanvasNodeIds: contextPayload?.selectedNodeIds || [], lastCanvasSnapshot: Array.isArray(contextPayload?.canvasImageNodes) && contextPayload?.canvasImageNodes.length > 0 ? { source: contextPayload?.source || "canvas", selectedNodeIds: contextPayload?.selectedNodeIds || [], canvasImageNodes: contextPayload?.canvasImageNodes, } : null, lastMessageId: message.id, text: requestTextContent, resolutionMode: requestTargetResolution.resolutionMode, confidence: requestTargetResolution.confidence, }); } catch (memoryError) { console.warn("[Chat API] Failed to persist context-memory state:", memoryError); } } const streamId = generateUUID(); await createStreamId({ streamId, chatId: id }); const promptOnlySourceImageUrls = requestResolvedMediaInput.currentMessageImageUrls.length > 0 ? requestResolvedMediaInput.currentMessageImageUrls.slice(0, 4) : requestTargetResolution?.targetImageUrls.slice(0, 4) || smartAutoHistoryImages.slice(0, 4); const promptOnlySourceAssetIds = requestTargetResolution?.targetAssetIds?.length ? requestTargetResolution.targetAssetIds : requestResolvedMediaInput.uploadedAssetIds; const wantsPromptOnlyFromImageContext = shouldRunAgentConversation && !smartAutoNeedsClarification && promptOnlySourceImageUrls.length > 0 && taskBoundaryDecision.taskType === "prompt_only"; if (wantsPromptOnlyFromImageContext) { let promptOnlyImageSummary = ""; try { const { summarizeImageForExecutionReport } = await import("@/lib/ai/smart-analyzer"); const summaryTimeoutMs = getBudgetedTimeout({ startedAt: requestStartedAt, preferredMs: SMART_AUTO_IMAGE_SUMMARY_TIMEOUT_MS, minimumMs: 6_000, }); if (summaryTimeoutMs) { const summaryAssetIds = Array.from( new Set([ ...extractMessageImageAssetIds(message.parts as any[]), ...((effectiveContextPayload?.selectedAssetIds || []).filter(Boolean) as string[]), ]), ); const summary = await withOperationTimeout( summarizeImageForExecutionReport({ text: requestTextContent, imageUrls: promptOnlySourceImageUrls, context: effectiveAutoContextText || undefined, preferredModelId: preferredAgentModelId, assetIds: promptOnlySourceAssetIds.length > 0 ? promptOnlySourceAssetIds : summaryAssetIds, }), "promptOnlyImageSummary", summaryTimeoutMs, ); if (summary?.trim()) { promptOnlyImageSummary = summary.trim(); } } } catch (promptOnlySummaryError) { console.warn("[Chat API] Prompt-only image summary failed:", promptOnlySummaryError); } let promptOnlyOptimizedPrompt = smartAutoOptimizedPrompt?.trim() || ""; if (!promptOnlyOptimizedPrompt) { try { const { optimizePromptWithLLM, optimizePrompt } = await import("@/lib/ai/prompt-optimizer"); const promptSeed = buildPromptOnlyOptimizationSeed({ text: requestTextContent, imageSummary: promptOnlyImageSummary, }); const optimizedResult = await optimizePromptWithLLM({ originalPrompt: promptSeed, intent: "image_gen", historyMessages: smartAutoHistoryMessagesFromDb .map((msg) => { const parts = Array.isArray(msg.parts) ? (msg.parts as any[]) : []; const content = parts .filter((part: any) => part?.type === "text") .map((part: any) => String(part?.text || "")) .join(" ") .trim(); if (!content) return null; return { role: msg.role as "user" | "assistant", content: content.slice(0, 300), }; }) .filter(Boolean) as Array<{ role: "user" | "assistant"; content: string }>, context: effectiveAutoContextText || undefined, }).catch(() => optimizePrompt({ originalPrompt: promptSeed, intent: "image_gen", }), ); promptOnlyOptimizedPrompt = String(optimizedResult?.optimizedPrompt || "").trim(); } catch (promptOnlyOptimizeError) { console.warn("[Chat API] Prompt-only optimization failed:", promptOnlyOptimizeError); } } if (!promptOnlyOptimizedPrompt) { promptOnlyOptimizedPrompt = buildPromptOnlyOptimizationSeed({ text: requestTextContent, imageSummary: promptOnlyImageSummary, }); } const promptOnlyMessage = buildPromptOnlyReply({ userText: requestTextContent, imageSummary: promptOnlyImageSummary, optimizedPrompt: promptOnlyOptimizedPrompt, }); const promptOnlyStream = createUIMessageStream({ execute: async ({ writer }) => { const messageId = generateUUID(); writeAssistantTextChunk(writer, promptOnlyMessage, "prompt-only:reply", messageId); await saveMessages({ messages: [ { id: messageId, chatId: id, role: "assistant", parts: [{ type: "text", text: promptOnlyMessage }], attachments: [], createdAt: new Date(), videoStorageStatus: null, temporaryVideoUrl: null, permanentVideoUrl: null, archivedAt: null, videoMetadata: null, }, ], }); try { await persistPromptOnlyTurn({ userId: session.user.id, chatId: id, projectId: contextPayload?.projectId || null, messageId, text: requestTextContent, optimizedPrompt: promptOnlyOptimizedPrompt, imageSummary: promptOnlyImageSummary, sourceImageUrls: promptOnlySourceImageUrls, }); } catch (promptOnlyMemoryError) { console.warn("[Chat API] Failed to persist prompt-only response state:", promptOnlyMemoryError); } }, }); return createSseResponse(promptOnlyStream); } if (shouldRunAgentConversation && smartAutoNeedsClarification) { const clarificationBypass = resolveClarificationBypass({ text: message.parts .filter((part): part is { type: "text"; text: string } => part.type === "text") .map((part) => part.text) .join(" ") .trim(), hasCurrentImageInput: message.parts.some((part: any) => part?.type === "file" || part?.type === "image" || part?.type === "image_url" ), hasHistoryImageInput: smartAutoHistoryImages.length > 0, hasCanvasSelection: Boolean(effectiveContextPayload?.selectedNodeIds?.length), targetResolutionMode: requestTargetResolution?.resolutionMode, executionMode: smartAutoExecution?.mode, friendlyMessage: smartAutoFriendlyMessage, }); if (clarificationBypass.shouldBypass) { smartAutoNeedsClarification = false; smartAutoClarificationQuestion = undefined; smartAutoFriendlyMessage = clarificationBypass.nextFriendlyMessage; console.log("[Smart Auto Mode] ✅ Force direct execution (skip clarification)"); } } if (shouldRunAgentConversation && smartAutoNeedsClarification) { const clarificationMessage = resolveClarificationMessage({ clarificationQuestion: smartAutoClarificationQuestion, friendlyMessage: smartAutoFriendlyMessage, }); const clarificationStream = createUIMessageStream({ execute: async ({ writer }) => { const messageId = generateUUID(); writer.write({ type: "data-appendMessage", data: JSON.stringify({ id: messageId, role: "assistant", parts: [{ type: "text", text: clarificationMessage }], }), }); await saveMessages({ messages: [ { id: messageId, chatId: id, role: "assistant", parts: [{ type: "text", text: clarificationMessage }], attachments: [], createdAt: new Date(), videoStorageStatus: null, temporaryVideoUrl: null, permanentVideoUrl: null, archivedAt: null, videoMetadata: null, }, ], }); try { await persistClarificationTurn({ userId: session.user.id, chatId: id, projectId: contextPayload?.projectId || null, messageId, text: requestTextContent, clarificationMessage, currentIntent: smartAutoDetectedIntent || null, pendingGridConfirmation: shouldAskGridConfirmation && pendingGridConfirmation ? pendingGridConfirmation : null, }); } catch (clarificationPersistError) { console.warn( "[Chat API] Failed to persist clarification state:", clarificationPersistError ); } }, }); return createSseResponse(clarificationStream); } if ( shouldRunAgentConversation && smartAutoWorkflowDecision?.workflowType === "storyboard_video" && (smartAutoWorkflowDecision?.action === "revision_requested" || smartAutoWorkflowDecision?.action === "cancel_requested") ) { const workflowReplyStream = createUIMessageStream({ execute: async ({ writer }) => { const messageId = generateUUID(); const workflowFollowupResult = await handleStoryboardWorkflowFollowup({ action: smartAutoWorkflowDecision.action as "revision_requested" | "cancel_requested", workflowId: smartAutoWorkflowDecision.workflowId, text: requestTextContent, friendlyMessage: smartAutoFriendlyMessage, onProgress: (event) => { safeWriteStreamChunk( writer, { type: "data-videoStoryboard", data: JSON.stringify(event), }, `storyboard-revision:${event.stage}`, ); }, }); const replyText = workflowFollowupResult.replyText; writer.write({ type: "data-appendMessage", data: JSON.stringify({ id: messageId, role: "assistant", parts: [{ type: "text", text: replyText }], attachments: [], }), }); await saveMessages({ messages: [ { id: messageId, chatId: id, role: "assistant", parts: [{ type: "text", text: replyText }], attachments: [], createdAt: new Date(), videoStorageStatus: null, temporaryVideoUrl: null, permanentVideoUrl: null, archivedAt: null, videoMetadata: workflowFollowupResult.videoMetadata, }, ], }); try { if ( smartAutoWorkflowDecision?.action === "revision_requested" && smartAutoWorkflowDecision.workflowId ) { await persistStoryboardRevisionTurn({ userId: session.user.id, chatId: id, projectId: contextPayload?.projectId || null, messageId, text: requestTextContent, taskId: smartAutoWorkflowDecision.workflowId, status: workflowFollowupResult.nextWorkflowStatus, }); } else if (smartAutoWorkflowDecision?.action === "cancel_requested") { await persistWorkflowCancelledTurn({ userId: session.user.id, chatId: id, projectId: contextPayload?.projectId || null, messageId, text: requestTextContent, workflowType: "storyboard_video", workflowId: smartAutoWorkflowDecision?.workflowId, status: workflowFollowupResult.nextWorkflowStatus, }); } } catch (workflowPersistError) { console.warn( "[Chat API] Failed to persist storyboard workflow interruption state:", workflowPersistError ); } }, }); return createSseResponse(workflowReplyStream); } if ( shouldRunAgentConversation && smartAutoWorkflowDecision?.workflowType === "storyboard_video" && smartAutoWorkflowDecision?.action === "confirmed" && smartAutoWorkflowDecision.workflowId ) { const confirmedTaskId = smartAutoWorkflowDecision.workflowId; const confirmedStoryboardStream = createUIMessageStream({ execute: async ({ writer }) => { const acknowledgeMessageId = generateUUID(); const { acknowledgeMessage, acknowledgeMetadata } = buildStoryboardConfirmedAck(confirmedTaskId); writer.write({ type: "data-appendMessage", data: JSON.stringify({ id: acknowledgeMessageId, role: "assistant", parts: [{ type: "text", text: acknowledgeMessage }], attachments: [], }), }); await saveMessages({ messages: [ { id: acknowledgeMessageId, chatId: id, role: "assistant", parts: [{ type: "text", text: acknowledgeMessage }], attachments: [], createdAt: new Date(), videoStorageStatus: null, temporaryVideoUrl: null, permanentVideoUrl: null, archivedAt: null, videoMetadata: acknowledgeMetadata, }, ], }); try { await persistStoryboardConfirmedTurn({ userId: session.user.id, chatId: id, projectId: contextPayload?.projectId || null, messageId: acknowledgeMessageId, text: requestTextContent, taskId: confirmedTaskId, status: "confirmed", }); } catch (storyboardConfirmMemoryError) { console.warn( "[Chat API] Failed to persist storyboard confirmation memory:", storyboardConfirmMemoryError ); } try { await persistStoryboardExecutionStartedTurn({ userId: session.user.id, chatId: id, projectId: contextPayload?.projectId || null, messageId: acknowledgeMessageId, text: requestTextContent, taskId: confirmedTaskId, status: "running", }); } catch (storyboardExecutionStartError) { console.warn( "[Chat API] Failed to persist storyboard execution start state:", storyboardExecutionStartError ); } const { executionResult, firstVideoUrl, finalVideoMetadata } = await executeStoryboardConfirmedWorkflow({ taskId: confirmedTaskId, onProgress: (event) => { safeWriteStreamChunk( writer, { type: "data-videoStoryboard", data: JSON.stringify(event), }, `storyboard-confirmed:${event.stage}`, ); }, }); const finalMessageId = generateUUID(); writer.write({ type: "data-appendMessage", data: JSON.stringify({ id: finalMessageId, role: "assistant", parts: [{ type: "text", text: executionResult.markdown }], attachments: [], }), }); await saveMessages({ messages: [ { id: finalMessageId, chatId: id, role: "assistant", parts: [{ type: "text", text: executionResult.markdown }], attachments: [], createdAt: new Date(), videoStorageStatus: firstVideoUrl ? "temporary" : null, temporaryVideoUrl: firstVideoUrl || null, permanentVideoUrl: null, archivedAt: null, videoMetadata: finalVideoMetadata, }, ], }); await attachVideoStoryboardTaskMessage({ taskId: executionResult.taskId, messageId: finalMessageId, }).catch((taskAttachError) => { console.warn( "[Chat API] Failed to attach confirmed storyboard task with message:", taskAttachError ); }); try { await persistStoryboardExecutionCompletedTurn({ userId: session.user.id, chatId: id, projectId: contextPayload?.projectId || null, messageId: finalMessageId, text: requestTextContent, taskId: executionResult.taskId, status: executionResult.status, successCount: executionResult.successCount, failedCount: executionResult.failedCount, firstVideoUrl: firstVideoUrl || undefined, }); } catch (storyboardExecutionCompletedError) { console.warn( "[Chat API] Failed to persist storyboard execution completion state:", storyboardExecutionCompletedError ); } if (firstVideoUrl) { await consumeGeneratedVideoPoints({ userId: session.user.id, chatId: id, messageId: finalMessageId, modelRefs: executionResult.modelRefs, }); scheduleVideoArchive({ chatId: id, messageId: finalMessageId, text: executionResult.markdown, temporaryVideoUrl: firstVideoUrl, taskId: finalMessageId, metadata: { storyboard: { enabled: true, taskId: executionResult.taskId, successCount: executionResult.successCount, failedCount: executionResult.failedCount, }, status: executionResult.status, updatedAt: Date.now(), }, }); } }, }); return createSseResponse(confirmedStoryboardStream); } if (agentPresetId) { const selectedAgentPreset = await getAccessibleAgentPresetById({ id: agentPresetId, userId: session.user.id, }); if (!selectedAgentPreset) { return NextResponse.json( { code: "not_found:chat", message: "智能体不存在或无权使用", }, { status: 404 }, ); } if (selectedAgentPreset.type === "text") { // 生文智能体走常规多模态对话链路： // - 模型已由前端切到智能体预设模型 // - 提示词通过 hiddenPrompt 注入 // - 图片输入沿用普通多模态附件 } else { const orderedSlots = [...(selectedAgentPreset.slots || [])].sort( (left, right) => left.slotIndex - right.slotIndex, ); const presetModeConfig = normalizeAgentPresetModeConfig(selectedAgentPreset.defaultParams); const isSceneGenerationPreset = selectedAgentPreset.type === "image" && presetModeConfig.modeType === "scene_generation"; const isDetailReplacementPreset = selectedAgentPreset.type === "image" && presetModeConfig.modeType === "detail_replacement"; const isSkuReplacementPreset = selectedAgentPreset.type === "image" && presetModeConfig.modeType === "sku_replacement"; const replacementProductSlotCount = isDetailReplacementPreset ? presetModeConfig.detail.productImageSlotCount : isSkuReplacementPreset ? presetModeConfig.sku.productImageSlotCount : 0; const replacementModelSlotEnabled = isDetailReplacementPreset ? presetModeConfig.detail.includeModelSlot : isSkuReplacementPreset ? presetModeConfig.sku.includeModelSlot : false; const replacementReferences = normalizeAgentModeReferences(agentModePayload); const isMatchingReplacementPayload = (isDetailReplacementPreset && agentModePayload?.modeType === "detail_replacement") || (isSkuReplacementPreset && agentModePayload?.modeType === "sku_replacement"); if (orderedSlots.length === 0) { return NextResponse.json( { code: "bad_request:api", message: "当前智能体未配置可执行槽位", }, { status: 400 }, ); } if ((isDetailReplacementPreset || isSkuReplacementPreset) && !isMatchingReplacementPayload) { return NextResponse.json( { code: "bad_request:api", message: "当前模式缺少执行上下文，请返回第二步重新选择参考图后再试", }, { status: 400 }, ); } if ((isDetailReplacementPreset || isSkuReplacementPreset) && replacementReferences.length === 0) { return NextResponse.json( { code: "bad_request:api", message: isDetailReplacementPreset ? "详情替换模式至少需要 1 张详情参考图" : "SKU替换模式至少需要 1 张 SKU 参考图", }, { status: 400 }, ); } const uploadedImageParts = extractCurrentMessageImageParts(message.parts as any[]); const requiredUploadCount = getAgentRequiredUploadCount({ presetType: selectedAgentPreset.type, defaultParams: selectedAgentPreset.defaultParams || null, fallback: 3, }); if (uploadedImageParts.length < requiredUploadCount) { return NextResponse.json( { code: "bad_request:api", message: `当前智能体需要上传 ${requiredUploadCount} 张图片`, }, { status: 400 }, ); } const presetModelId = String(selectedAgentPreset.defaultModelId || "").trim(); if (!presetModelId) { return NextResponse.json( { code: "bad_request:api", message: "当前智能体未配置默认模型", }, { status: 400 }, ); } const sanitizedUserSupplementPrompt = normalizeAgentSupplementPrompt( requestTextContent, selectedAgentPreset.name, ); const presetSnapshot = await createAgentPresetExecutionSnapshot({ presetId: selectedAgentPreset.id, userId: session.user.id, projectId: contextPayload?.projectId || null, chatId: id, messageId: message.id, presetName: selectedAgentPreset.name, presetType: selectedAgentPreset.type, visibility: selectedAgentPreset.visibility, categoryId: selectedAgentPreset.category?.id || selectedAgentPreset.categoryId || null, categoryName: selectedAgentPreset.category?.name || null, promptTemplate: selectedAgentPreset.promptTemplate, defaultModelId: selectedAgentPreset.defaultModelId || null, resolvedModelId: presetModelId, defaultParams: selectedAgentPreset.defaultParams || null, slotCount: orderedSlots.length, requiredUploadCount, slots: orderedSlots.map((slot) => ({ slotIndex: slot.slotIndex, title: slot.title || null, promptTemplate: slot.promptTemplate, defaultParams: slot.defaultParams || null, requireUpload: slot.requireUpload !== false, uploadLabel: slot.uploadLabel || null, referenceAssetIds: slot.referenceAssetIds || [], referenceImageUrls: (slot.referenceAssets || []) .map((asset) => String(asset.originalUrl || "").trim()) .filter(Boolean), })), referenceAssetIds: orderedSlots.flatMap((slot) => slot.referenceAssetIds || []), referenceImageUrls: orderedSlots.flatMap((slot) => (slot.referenceAssets || []) .map((asset) => String(asset.originalUrl || "").trim()) .filter(Boolean), ), userInputText: sanitizedUserSupplementPrompt || null, requestPayload: { selectedChatModel, agentPresetId, requiredUploadCount, uploadedImageUrls: uploadedImageParts.map((part) => part.url), agentModePayload: isDetailReplacementPreset || isSkuReplacementPreset ? { modeType: agentModePayload?.modeType || null, query: agentModePayload?.query || null, queryType: agentModePayload?.queryType || null, selectedCandidateId: agentModePayload?.selectedCandidateId || null, referenceCount: replacementReferences.length, } : null, }, }); try { await db .update(agentPresetTable) .set({ usageCount: sql`${agentPresetTable.usageCount} + 1`, lastUsedAt: new Date(), updatedAt: new Date(), }) .where(eq(agentPresetTable.id, selectedAgentPreset.id)); } catch (agentUsageError) { console.warn("[Chat API] Failed to update agent preset usage stats:", agentUsageError); } const agentSequenceStream = createUIMessageStream({ execute: async ({ writer }) => { if (selectedAgentPreset.type === "image") { writeAssistantTextChunk( writer, "", "agent-preset:image:start", ); if (contextPayload?.source === "canvas") { writeImageGenerationStatusChunk( writer, { phase: "processing", modelName: selectedAgentPreset.defaultModel?.displayName || selectedAgentPreset.name, imageCount: isSceneGenerationPreset ? Math.max(1, effectiveImageParams?.imageCount || 1) : isDetailReplacementPreset || isSkuReplacementPreset ? replacementReferences.length : orderedSlots.length, }, "agent-preset:image:status-processing", ); } } let successCount = 0; const sharedUploadedImageUrls = uploadedImageParts .map((part) => part.url) .filter((url) => /^https?:\/\//i.test(url)); const sharedUploadedAssetIds = uploadedImageParts .map((part) => part.assetId) .filter((assetId): assetId is string => Boolean(assetId)); if ( selectedAgentPreset.type === "image" && (isDetailReplacementPreset || isSkuReplacementPreset) ) { const replacementSlot = orderedSlots[0]; const replacementAssistantMessageId = generateUUID(); const replacementTemplateReferenceImageUrls = Array.from( new Set( (replacementSlot?.referenceAssets || []) .map((asset) => String(asset.originalUrl || "").trim()) .filter(Boolean), ), ); const productInputImageUrls = sharedUploadedImageUrls.slice( 0, Math.max(1, replacementProductSlotCount) ); const modelInputImageUrl = replacementModelSlotEnabled ? sharedUploadedImageUrls[Math.max(0, replacementProductSlotCount)] || null : null; const sourceAssetIds = Array.from( new Set( [ ...sharedUploadedAssetIds, ...(replacementSlot?.referenceAssetIds || []), ].filter(Boolean), ), ); const baseReplacementPrompt = buildAgentSlotPrompt({ workflowPrompt: hiddenPromptText || null, slotPrompt: replacementSlot?.promptTemplate || selectedAgentPreset.promptTemplate, slotTitle: selectedAgentPreset.name, userSupplementPrompt: sanitizedUserSupplementPrompt || null, }); if (productInputImageUrls.length === 0) { throw new Error("替换模式至少需要 1 张输入商品图"); } const mergedImageParams = { ...(selectedAgentPreset.defaultParams || {}), ...(replacementSlot?.defaultParams || {}), } as Record; const resolvedModelRef = String(selectedChatModel || "").trim() && String(selectedChatModel || "").trim() !== "auto" ? String(selectedChatModel || "").trim() : presetModelId; const resolvedAspectRatio = typeof mergedImageParams.aspectRatio === "string" ? mergedImageParams.aspectRatio : effectiveImageParams?.aspectRatio || defaultImageQualityParams.aspectRatio; const resolvedImageSize = typeof mergedImageParams.imageSize === "string" ? mergedImageParams.imageSize : effectiveImageParams?.imageSize || defaultImageQualityParams.imageSize; const replacementSections: string[] = []; const replacementResultImages: Array<{ originalUrl: string; thumbnailUrl?: string | null; }> = []; let resolvedModelDisplayName = selectedAgentPreset.defaultModel?.displayName || selectedAgentPreset.name; const stopKeepalive = startStreamKeepalive( writer, "agent-preset:replacement", 12_000, ); try { for (const [referenceIndex, reference] of replacementReferences.entries()) { const pairedProductIndex = isSkuReplacementPreset && productInputImageUrls.length > 0 ? Math.min( typeof reference.assignedProductIndex === "number" ? reference.assignedProductIndex : referenceIndex, productInputImageUrls.length - 1 ) : undefined; const currentProductImageUrls = pairedProductIndex !== undefined ? [productInputImageUrls[pairedProductIndex]!] : productInputImageUrls; const sourceImageUrls = Array.from( new Set( [ ...currentProductImageUrls, ...(modelInputImageUrl ? [modelInputImageUrl] : []), ...replacementTemplateReferenceImageUrls, reference.imageUrl, ].filter(Boolean), ), ); const replacementPrompt = buildAgentReplacementPrompt({ basePrompt: baseReplacementPrompt || (isDetailReplacementPreset ? "执行详情替换" : "执行 SKU 替换"), modeType: isDetailReplacementPreset ? "detail_replacement" : "sku_replacement", referenceLabel: reference.label, referenceIndex, totalReferences: replacementReferences.length, hasModelImage: Boolean(modelInputImageUrl), pairedProductIndex, }); try { const result = await generateCanvasImage({ prompt: replacementPrompt, negativePrompt: typeof mergedImageParams.negativePrompt === "string" ? mergedImageParams.negativePrompt : undefined, model: resolvedModelRef, aspectRatio: resolvedAspectRatio, imageSize: resolvedImageSize, imageCount: 1, referenceImages: sourceImageUrls, }); resolvedModelDisplayName = result.model || resolvedModelDisplayName; const firstImage = result.images[0]; if (!firstImage) { throw new Error("未返回结果图片"); } replacementResultImages.push(firstImage); replacementSections.push( `### ${reference.label}\n\n${buildAgentToolImagesMarkdown([ { originalUrl: firstImage.originalUrl, thumbnailUrl: firstImage.thumbnailUrl, imageSize: resolvedImageSize, }, ])}`, ); successCount += 1; } catch (replacementItemError) { console.error("[Chat API] Agent replacement item execution failed:", { referenceIndex, referenceLabel: reference.label, error: replacementItemError, }); replacementSections.push( `### ${reference.label}\n\n执行失败：${ replacementItemError instanceof Error ? replacementItemError.message : "请稍后重试" }`, ); } } } finally { stopKeepalive(); } const assistantText = prependImageModelBadge( [ `### ${selectedAgentPreset.name}`, ...replacementSections, ] .filter(Boolean) .join("\n\n"), resolvedModelDisplayName, ); await saveMessages({ messages: [ { id: replacementAssistantMessageId, chatId: id, role: "assistant", parts: [{ type: "text", text: assistantText }], attachments: [], createdAt: new Date(), videoStorageStatus: null, temporaryVideoUrl: null, permanentVideoUrl: null, archivedAt: null, videoMetadata: null, }, ], }); if (replacementResultImages.length > 0) { await saveGeneratedImages( assistantText, session.user.id, id, replacementAssistantMessageId, baseReplacementPrompt || selectedAgentPreset.name, resolvedModelRef, { sourceAssetIds, sourceImageUrls: Array.from( new Set([ ...sharedUploadedImageUrls, ...replacementTemplateReferenceImageUrls, ...replacementReferences.map((item) => item.imageUrl), ]), ), relationType: "edited_from", sourceType: isDetailReplacementPreset ? "detail_replacement" : "sku_replacement", projectId: contextPayload?.projectId || null, userPrompt: requestTextContent, optimizedPrompt: baseReplacementPrompt || undefined, requestedImageCount: replacementResultImages.length, aspectRatio: resolvedAspectRatio, imageSize: resolvedImageSize, }, ); } writeAssistantTextChunk( writer, assistantText, isDetailReplacementPreset ? "agent-preset:image:detail-replacement" : "agent-preset:image:sku-replacement", replacementAssistantMessageId, ); if (contextPayload?.source === "canvas" && replacementResultImages.length > 0) { writeCanvasMediaChunk( writer, { messageId: replacementAssistantMessageId, imageUrls: replacementResultImages.map((image) => image.originalUrl), prompt: baseReplacementPrompt || selectedAgentPreset.name, sourceImageUrls: Array.from( new Set([ ...sharedUploadedImageUrls, ...replacementTemplateReferenceImageUrls, ...replacementReferences.map((item) => item.imageUrl), ]), ), agentPresetId: selectedAgentPreset.id, agentPresetName: selectedAgentPreset.name, agentCategoryId: selectedAgentPreset.category?.id || selectedAgentPreset.categoryId || null, agentCategoryName: selectedAgentPreset.category?.name || null, agentExecutionSnapshotId: presetSnapshot.id, }, isDetailReplacementPreset ? "agent-preset:image:detail-replacement:canvas-media" : "agent-preset:image:sku-replacement:canvas-media", ); } } else if (selectedAgentPreset.type === "image" && isSceneGenerationPreset) { const sceneSlot = orderedSlots[0]; const sceneAssistantMessageId = generateUUID(); const sceneReferenceImageUrls = Array.from( new Set( (sceneSlot?.referenceAssets || []) .map((asset) => String(asset.originalUrl || "").trim()) .filter(Boolean), ), ); const sourceImageUrls = Array.from( new Set([...sharedUploadedImageUrls, ...sceneReferenceImageUrls].filter(Boolean)), ); const sourceAssetIds = Array.from( new Set( [...sharedUploadedAssetIds, ...(sceneSlot?.referenceAssetIds || [])].filter(Boolean), ), ); const scenePrompt = buildAgentSlotPrompt({ workflowPrompt: hiddenPromptText || null, slotPrompt: sceneSlot?.promptTemplate || selectedAgentPreset.promptTemplate, slotTitle: selectedAgentPreset.name, userSupplementPrompt: sanitizedUserSupplementPrompt || null, }); try { const mergedImageParams = { ...(selectedAgentPreset.defaultParams || {}), ...(sceneSlot?.defaultParams || {}), } as Record; const resolvedModelRef = presetModeConfig.scene.allowModelOverride && String(selectedChatModel || "").trim() && String(selectedChatModel || "").trim() !== "auto" ? String(selectedChatModel || "").trim() : presetModelId; const resolvedImageCount = Math.max(1, effectiveImageParams?.imageCount || 1); const result = await generateCanvasImage({ prompt: scenePrompt || `生成${selectedAgentPreset.name}`, negativePrompt: typeof mergedImageParams.negativePrompt === "string" ? mergedImageParams.negativePrompt : undefined, model: resolvedModelRef, aspectRatio: typeof mergedImageParams.aspectRatio === "string" ? mergedImageParams.aspectRatio : effectiveImageParams?.aspectRatio || defaultImageQualityParams.aspectRatio, imageSize: typeof mergedImageParams.imageSize === "string" ? mergedImageParams.imageSize : effectiveImageParams?.imageSize || defaultImageQualityParams.imageSize, imageCount: resolvedImageCount, referenceImages: sourceImageUrls, }); const imageMarkdown = buildAgentToolImagesMarkdown( result.images.map((image) => ({ originalUrl: image.originalUrl, thumbnailUrl: image.thumbnailUrl, imageSize: typeof mergedImageParams.imageSize === "string" ? mergedImageParams.imageSize : effectiveImageParams?.imageSize || defaultImageQualityParams.imageSize, })), ); const assistantText = prependImageModelBadge( [`### ${selectedAgentPreset.name}`, imageMarkdown].filter(Boolean).join("\n\n"), result.model, ); await saveMessages({ messages: [ { id: sceneAssistantMessageId, chatId: id, role: "assistant", parts: [{ type: "text", text: assistantText }], attachments: [], createdAt: new Date(), videoStorageStatus: null, temporaryVideoUrl: null, permanentVideoUrl: null, archivedAt: null, videoMetadata: null, }, ], }); await saveGeneratedImages( assistantText, session.user.id, id, sceneAssistantMessageId, scenePrompt || selectedAgentPreset.name, result.modelRef || resolvedModelRef, { sourceAssetIds, sourceImageUrls, relationType: "generated_from", sourceType: sourceImageUrls.length > 0 ? "reference_generate" : "generated", projectId: contextPayload?.projectId || null, userPrompt: requestTextContent, optimizedPrompt: scenePrompt || undefined, requestedImageCount: resolvedImageCount, }, ); writeAssistantTextChunk( writer, assistantText, "agent-preset:image:scene-generation", sceneAssistantMessageId, ); if (contextPayload?.source === "canvas") { writeCanvasMediaChunk( writer, { messageId: sceneAssistantMessageId, imageUrls: result.images.map((image) => image.originalUrl), prompt: scenePrompt || selectedAgentPreset.name, sourceImageUrls, agentPresetId: selectedAgentPreset.id, agentPresetName: selectedAgentPreset.name, agentCategoryId: selectedAgentPreset.category?.id || selectedAgentPreset.categoryId || null, agentCategoryName: selectedAgentPreset.category?.name || null, agentExecutionSnapshotId: presetSnapshot.id, }, "agent-preset:image:scene-generation:canvas-media", ); } successCount = result.images.length; } catch (sceneError) { console.error("[Chat API] Agent preset scene generation failed:", sceneError); const errorMessage = `### ${selectedAgentPreset.name}\n\n执行失败：${ sceneError instanceof Error ? sceneError.message : "请稍后重试" }`; await saveMessages({ messages: [ { id: sceneAssistantMessageId, chatId: id, role: "assistant", parts: [{ type: "text", text: errorMessage }], attachments: [], createdAt: new Date(), videoStorageStatus: null, temporaryVideoUrl: null, permanentVideoUrl: null, archivedAt: null, videoMetadata: null, }, ], }); writeAssistantTextChunk( writer, errorMessage, "agent-preset:error:scene-generation", sceneAssistantMessageId, ); } } else { for (const [slotOrder, slot] of orderedSlots.entries()) { const slotLabel = String(slot.title || slot.uploadLabel || "").trim() || `第${slotOrder + 1}张`; const slotReferenceImageUrls = Array.from( new Set( (slot.referenceAssets || []) .map((asset) => String(asset.originalUrl || "").trim()) .filter(Boolean), ), ); const sourceImageUrls = Array.from( new Set( [ ...sharedUploadedImageUrls, ...slotReferenceImageUrls, ].filter(Boolean), ), ); const sourceAssetIds = Array.from( new Set( [ ...sharedUploadedAssetIds, ...(slot.referenceAssetIds || []), ].filter(Boolean), ), ); const slotPrompt = buildAgentSlotPrompt({ workflowPrompt: hiddenPromptText || null, slotPrompt: slot.promptTemplate, slotTitle: slot.title || slot.uploadLabel || null, userSupplementPrompt: sanitizedUserSupplementPrompt || null, }); const assistantMessageId = generateUUID(); try { if (selectedAgentPreset.type === "image") { const mergedImageParams = { ...(selectedAgentPreset.defaultParams || {}), ...(slot.defaultParams || {}), } as Record; const result = await generateCanvasImage({ prompt: slotPrompt || `生成${slotLabel}`, negativePrompt: typeof mergedImageParams.negativePrompt === "string" ? mergedImageParams.negativePrompt : undefined, model: presetModelId, aspectRatio: typeof mergedImageParams.aspectRatio === "string" ? mergedImageParams.aspectRatio : effectiveImageParams?.aspectRatio || defaultImageQualityParams.aspectRatio, imageSize: typeof mergedImageParams.imageSize === "string" ? mergedImageParams.imageSize : effectiveImageParams?.imageSize || defaultImageQualityParams.imageSize, imageCount: 1, referenceImages: sourceImageUrls, }); const imageMarkdown = buildAgentToolImagesMarkdown( result.images.map((image) => ({ originalUrl: image.originalUrl, thumbnailUrl: image.thumbnailUrl, imageSize: typeof mergedImageParams.imageSize === "string" ? mergedImageParams.imageSize : effectiveImageParams?.imageSize || defaultImageQualityParams.imageSize, })), ); const assistantText = prependImageModelBadge( [`### ${slotLabel}`, imageMarkdown].filter(Boolean).join("\n\n"), result.model, ); await saveMessages({ messages: [ { id: assistantMessageId, chatId: id, role: "assistant", parts: [{ type: "text", text: assistantText }], attachments: [], createdAt: new Date(), videoStorageStatus: null, temporaryVideoUrl: null, permanentVideoUrl: null, archivedAt: null, videoMetadata: null, }, ], }); await saveGeneratedImages( assistantText, session.user.id, id, assistantMessageId, slotPrompt || slotLabel, result.modelRef || presetModelId, { sourceAssetIds, sourceImageUrls, relationType: "generated_from", sourceType: sourceImageUrls.length > 0 ? "reference_generate" : "generated", projectId: contextPayload?.projectId || null, userPrompt: requestTextContent, optimizedPrompt: slotPrompt || undefined, requestedImageCount: 1, }, ); writeAssistantTextChunk( writer, assistantText, `agent-preset:image:slot-${slotOrder + 1}`, assistantMessageId, ); if (contextPayload?.source === "canvas") { writeCanvasMediaChunk( writer, { messageId: assistantMessageId, imageUrls: result.images.map((image) => image.originalUrl), prompt: slotPrompt || slotLabel, sourceImageUrls, agentPresetId: selectedAgentPreset.id, agentPresetName: selectedAgentPreset.name, agentCategoryId: selectedAgentPreset.category?.id || selectedAgentPreset.categoryId || null, agentCategoryName: selectedAgentPreset.category?.name || null, agentExecutionSnapshotId: presetSnapshot.id, }, `agent-preset:image:slot-${slotOrder + 1}:canvas-media`, ); } } else { const mergedVideoParams = { ...(selectedAgentPreset.defaultParams || {}), ...(slot.defaultParams || {}), } as Record; const requestedMode = String(mergedVideoParams.mode || "").trim() as | "text-to-video" | "first-frame" | "first-last-frame" | "reference-images"; const mode = requestedMode || (sourceImageUrls.length >= 2 ? "first-last-frame" : sourceImageUrls.length >= 1 ? "first-frame" : "text-to-video"); const directVideoRatio = mergedVideoParams.ratio === "16:9" || mergedVideoParams.ratio === "4:3" || mergedVideoParams.ratio === "1:1" || mergedVideoParams.ratio === "3:4" || mergedVideoParams.ratio === "9:16" || mergedVideoParams.ratio === "21:9" || mergedVideoParams.ratio === "adaptive" ? mergedVideoParams.ratio : undefined; const images = mode === "first-last-frame" ? sourceImageUrls.slice(0, 2) : mode === "first-frame" ? sourceImageUrls.slice(0, 1) : mode === "reference-images" ? sourceImageUrls.slice(0, 4) : []; const result = await executeDirectVideoGeneration({ prompt: slotPrompt || `生成${slotLabel}`, mode, images, resolution: mergedVideoParams.resolution, ratio: directVideoRatio, duration: mergedVideoParams.duration, preferredModelRef: presetModelId, }); const videoUrl = String(result.videoUrl || "").trim(); if (!videoUrl) { throw new Error("视频生成完成但未返回视频地址"); } const assistantText = prependImageModelBadge( [ `### ${slotLabel}`, buildAgentToolVideoMarkdown(videoUrl), ] .filter(Boolean) .join("\n\n"), result.modelDisplayName, ); await saveMessages({ messages: [ { id: assistantMessageId, chatId: id, role: "assistant", parts: [{ type: "text", text: assistantText }], attachments: [], createdAt: new Date(), videoStorageStatus: "temporary", temporaryVideoUrl: videoUrl, permanentVideoUrl: null, archivedAt: null, videoMetadata: (result.metadata || {}) as any, }, ], }); try { await persistVideoGenerationCompletedTurn({ userId: session.user.id, chatId: id, projectId: contextPayload?.projectId || null, messageId: assistantMessageId, text: requestTextContent, source: "agent_preset_direct_video", providerType: "direct-video", modelRef: result.modelRef || presetModelId, mode, status: "completed", temporaryVideoUrl: videoUrl, }); } catch (memoryError) { console.warn( "[Chat API] Failed to update working state after agent preset video generation:", memoryError, ); } await consumeGeneratedVideoPoints({ userId: session.user.id, chatId: id, messageId: assistantMessageId, modelRefs: [result.modelRef || presetModelId], }); scheduleVideoArchive({ chatId: id, messageId: assistantMessageId, text: assistantText, temporaryVideoUrl: videoUrl, taskId: assistantMessageId, metadata: result.metadata, }); writeAssistantTextChunk( writer, assistantText, `agent-preset:video:slot-${slotOrder + 1}`, assistantMessageId, ); if (contextPayload?.source === "canvas") { writeCanvasMediaChunk( writer, { messageId: assistantMessageId, videoUrls: [videoUrl], prompt: slotPrompt || slotLabel, sourceImageUrls, agentPresetId: selectedAgentPreset.id, agentPresetName: selectedAgentPreset.name, agentCategoryId: selectedAgentPreset.category?.id || selectedAgentPreset.categoryId || null, agentCategoryName: selectedAgentPreset.category?.name || null, agentExecutionSnapshotId: presetSnapshot.id, }, `agent-preset:video:slot-${slotOrder + 1}:canvas-media`, ); } } successCount += 1; } catch (slotError) { console.error("[Chat API] Agent preset slot execution failed:", { slotIndex: slot.slotIndex, slotLabel, error: slotError, }); const errorMessage = `### ${slotLabel}\n\n执行失败：${ slotError instanceof Error ? slotError.message : "请稍后重试" }`; await saveMessages({ messages: [ { id: assistantMessageId, chatId: id, role: "assistant", parts: [{ type: "text", text: errorMessage }], attachments: [], createdAt: new Date(), videoStorageStatus: null, temporaryVideoUrl: null, permanentVideoUrl: null, archivedAt: null, videoMetadata: null, }, ], }); if (selectedAgentPreset.type === "video") { try { await persistVideoGenerationFailedTurn({ userId: session.user.id, chatId: id, projectId: contextPayload?.projectId || null, messageId: assistantMessageId, text: requestTextContent, source: "agent_preset_direct_video", providerType: "direct-video", modelRef: presetModelId, status: "failed", error: slotError instanceof Error ? slotError.message : String(slotError), }); } catch (videoMemoryError) { console.warn( "[Chat API] Failed to persist agent preset video failure state:", videoMemoryError, ); } } writeAssistantTextChunk( writer, errorMessage, `agent-preset:error:slot-${slotOrder + 1}`, assistantMessageId, ); } } } try { await persistAgentPresetTurn({ userId: session.user.id, chatId: id, projectId: contextPayload?.projectId || null, currentIntent: selectedAgentPreset.type === "video" ? "video_gen" : "image_gen", lastMessageId: message.id, text: requestTextContent, agentPresetId: selectedAgentPreset.id, agentPresetName: selectedAgentPreset.name, slotCount: orderedSlots.length, successCount, }); } catch (agentPresetMemoryError) { console.warn("[Chat API] Failed to persist agent preset working state:", agentPresetMemoryError); } if (selectedAgentPreset.type === "image") { if (contextPayload?.source === "canvas") { writeImageGenerationStatusChunk( writer, { phase: successCount > 0 ? "completed" : "failed", modelName: selectedAgentPreset.defaultModel?.displayName || selectedAgentPreset.name, imageCount: successCount, }, "agent-preset:image:status-finished", ); } writeAssistantTextChunk( writer, "", "agent-preset:image:end", ); } }, }); return createSseResponse(agentSequenceStream); } } const directAgentExecution = shouldRunAgentConversation ? resolveDirectAgentToolExecution(smartAutoExecution, smartAutoPlan) : null; const directAgentToolTargetImageUrls = directAgentExecution ? pickAgentSourceImageUrls({ sourceTools: directAgentExecution.sourceTools, requestResolvedMediaInput, requestTargetResolution, requestWorkingState, }) : []; const directAgentToolTargetImageUrl = directAgentToolTargetImageUrls[0] || null; if ( shouldRunAgentConversation && !smartAutoNeedsClarification && directAgentExecution?.toolName === "generateImage" ) { const generationPrompt = String(smartAutoOptimizedPrompt || requestTextContent || "").trim() || "生成图片"; const directImageStream = createUIMessageStream({ execute: async ({ writer }) => { writeAssistantTextChunk( writer, "", "agent:generate_image:start", ); if (contextPayload?.source === "canvas") { writeImageGenerationStatusChunk( writer, { phase: "processing", providerType: requestedModelProviderType, modelName: finalModelId, imageCount: effectiveImageParams?.imageCount || defaultImageQualityParams.imageCount || 1, }, "agent:generate_image:status-processing", ); } const assistantMessageId = generateUUID(); try { const result = await generateCanvasImage({ prompt: generationPrompt, negativePrompt: effectiveImageParams?.negativePrompt, model: finalModelId, aspectRatio: effectiveImageParams?.aspectRatio || defaultImageQualityParams.aspectRatio, imageSize: effectiveImageParams?.imageSize || defaultImageQualityParams.imageSize, imageCount: effectiveImageParams?.imageCount || defaultImageQualityParams.imageCount, referenceImages: directAgentToolTargetImageUrls, }); const imageMarkdown = buildAgentToolImagesMarkdown( result.images.map((image) => ({ originalUrl: image.originalUrl, thumbnailUrl: image.thumbnailUrl, imageSize: effectiveImageParams?.imageSize || defaultImageQualityParams.imageSize, })), ); const assistantText = prependImageModelBadge( [ getDirectAgentToolSuccessMessage("generate_image"), imageMarkdown, ] .filter(Boolean) .join("\n\n"), result.model, ); await saveMessages({ messages: [ { id: assistantMessageId, chatId: id, role: "assistant", parts: [{ type: "text", text: assistantText }], attachments: [], createdAt: new Date(), videoStorageStatus: null, temporaryVideoUrl: null, permanentVideoUrl: null, archivedAt: null, videoMetadata: null, }, ], }); await saveGeneratedImages( assistantText, session.user.id, id, assistantMessageId, generationPrompt, result.modelRef || finalModelId, { sourceImageUrls: directAgentToolTargetImageUrls, relationType: "generated_from", sourceType: directAgentToolTargetImageUrls.length > 0 ? "reference_generate" : "generated", projectId: contextPayload?.projectId || null, userPrompt: requestTextContent, optimizedPrompt: generationPrompt, requestedImageCount: effectiveImageParams?.imageCount || defaultImageQualityParams.imageCount || 1, aspectRatio: effectiveImageParams?.aspectRatio || defaultImageQualityParams.aspectRatio, imageSize: effectiveImageParams?.imageSize || defaultImageQualityParams.imageSize, }, ); writeAssistantTextChunk( writer, assistantText, "agent:generate_image:result", assistantMessageId, ); if (contextPayload?.source === "canvas") { writeCanvasMediaChunk( writer, { messageId: assistantMessageId, imageUrls: result.images.map((image) => image.originalUrl), prompt: generationPrompt, sourceImageUrls: directAgentToolTargetImageUrls, }, "agent:generate_image:canvas-media", ); writeImageGenerationStatusChunk( writer, { phase: "completed", providerType: requestedModelProviderType, modelName: result.model, imageCount: result.images.length, }, "agent:generate_image:status-completed", ); } writeAssistantTextChunk( writer, "", "agent:generate_image:end", ); } catch (directToolError) { console.error("[Chat API] Direct agent image generation failed:", directToolError); const errorMessage = getDirectAgentToolErrorMessage( "generate_image", directToolError, ); await saveMessages({ messages: [ { id: assistantMessageId, chatId: id, role: "assistant", parts: [{ type: "text", text: errorMessage }], attachments: [], createdAt: new Date(), videoStorageStatus: null, temporaryVideoUrl: null, permanentVideoUrl: null, archivedAt: null, videoMetadata: null, }, ], }); writeAssistantTextChunk( writer, errorMessage, "agent:generate_image:error", assistantMessageId, ); if (contextPayload?.source === "canvas") { writeImageGenerationStatusChunk( writer, { phase: "failed", providerType: requestedModelProviderType, modelName: finalModelId, }, "agent:generate_image:status-failed", ); } writeAssistantTextChunk( writer, "", "agent:generate_image:end-error", ); } }, }); return createSseResponse(directImageStream); } if ( shouldRunAgentConversation && !smartAutoNeedsClarification && directAgentExecution?.toolName === "generateVideo" ) { const generationPrompt = String(smartAutoOptimizedPrompt || requestTextContent || "").trim() || "生成视频"; const directVideoStream = createUIMessageStream({ execute: async ({ writer }) => { const assistantMessageId = generateUUID(); try { const plannedArgs = findAgentToolCallArgs( "generateVideo", smartAutoExecution, smartAutoPlan, ); const requestedMode = String( plannedArgs?.mode || effectiveVideoParams?.mode || "", ).trim() as | "text-to-video" | "first-frame" | "first-last-frame" | "reference-images"; const mode = requestedMode || (directAgentToolTargetImageUrls.length >= 2 ? "first-last-frame" : directAgentToolTargetImageUrls.length >= 1 ? "first-frame" : "text-to-video"); const images = mode === "first-last-frame" ? directAgentToolTargetImageUrls.slice(0, 2) : mode === "first-frame" ? directAgentToolTargetImageUrls.slice(0, 1) : mode === "reference-images" ? directAgentToolTargetImageUrls.slice(0, 4) : []; const directVideoRatio = effectiveVideoParams?.ratio === "16:9" || effectiveVideoParams?.ratio === "4:3" || effectiveVideoParams?.ratio === "1:1" || effectiveVideoParams?.ratio === "3:4" || effectiveVideoParams?.ratio === "9:16" || effectiveVideoParams?.ratio === "21:9" || effectiveVideoParams?.ratio === "adaptive" ? effectiveVideoParams.ratio : undefined; const result = await executeDirectVideoGeneration({ prompt: generationPrompt, mode, images, resolution: effectiveVideoParams?.resolution, ratio: directVideoRatio, duration: effectiveVideoParams?.duration, preferredModelRef: finalModelId, }); const videoUrl = String(result.videoUrl || "").trim(); if (!videoUrl) { throw new Error("视频生成完成但未返回视频地址"); } const assistantText = prependImageModelBadge( [ getDirectAgentToolSuccessMessage("generate_video"), buildAgentToolVideoMarkdown(videoUrl), ] .filter(Boolean) .join("\n\n"), result.modelDisplayName, ); await saveMessages({ messages: [ { id: assistantMessageId, chatId: id, role: "assistant", parts: [{ type: "text", text: assistantText }], attachments: [], createdAt: new Date(), videoStorageStatus: "temporary", temporaryVideoUrl: videoUrl, permanentVideoUrl: null, archivedAt: null, videoMetadata: (result.metadata || {}) as any, }, ], }); try { await persistVideoGenerationCompletedTurn({ userId: session.user.id, chatId: id, projectId: contextPayload?.projectId || null, messageId: assistantMessageId, text: requestTextContent, source: "agent_direct_generate_video", providerType: "direct-video", modelRef: result.modelRef || finalModelId, mode, status: "completed", temporaryVideoUrl: videoUrl, }); } catch (memoryError) { console.warn( "[Chat API] Failed to update working state after direct video generation:", memoryError, ); } await consumeGeneratedVideoPoints({ userId: session.user.id, chatId: id, messageId: assistantMessageId, modelRefs: [result.modelRef || finalModelId], }); if (videoUrl) { scheduleVideoArchive({ chatId: id, messageId: assistantMessageId, text: assistantText, temporaryVideoUrl: videoUrl, taskId: assistantMessageId, metadata: result.metadata, }); } writeAssistantTextChunk( writer, assistantText, "agent:generate_video:result", assistantMessageId, ); if (contextPayload?.source === "canvas") { writeCanvasMediaChunk( writer, { messageId: assistantMessageId, videoUrls: [videoUrl], prompt: generationPrompt, sourceImageUrls: directAgentToolTargetImageUrls, }, "agent:generate_video:canvas-media", ); } } catch (directToolError) { console.error("[Chat API] Direct agent video generation failed:", directToolError); const errorMessage = getDirectAgentToolErrorMessage( "generate_video", directToolError, ); await saveMessages({ messages: [ { id: assistantMessageId, chatId: id, role: "assistant", parts: [{ type: "text", text: errorMessage }], attachments: [], createdAt: new Date(), videoStorageStatus: null, temporaryVideoUrl: null, permanentVideoUrl: null, archivedAt: null, videoMetadata: null, }, ], }); try { await persistVideoGenerationFailedTurn({ userId: session.user.id, chatId: id, projectId: contextPayload?.projectId || null, messageId: assistantMessageId, text: requestTextContent, source: "agent_direct_generate_video", providerType: "direct-video", modelRef: finalModelId, status: "failed", error: directToolError instanceof Error ? directToolError.message : String(directToolError), }); } catch (videoMemoryError) { console.warn( "[Chat API] Failed to persist direct video failure state:", videoMemoryError, ); } writeAssistantTextChunk( writer, errorMessage, "agent:generate_video:error", assistantMessageId, ); } }, }); return createSseResponse(directVideoStream); } if ( shouldRunAgentConversation && !smartAutoNeedsClarification && directAgentExecution && (directAgentExecution.toolName === "removeBackground" || directAgentExecution.toolName === "resizeImage" || directAgentExecution.toolName === "extendImage" || directAgentExecution.toolName === "boxCutout") ) { const directAgentToolCapability = directAgentExecution.capability; if (!directAgentToolTargetImageUrl) { const missingTargetMessage = getDirectAgentToolTargetMissingMessage( directAgentToolCapability, ); const missingTargetStream = createUIMessageStream({ execute: async ({ writer }) => { const messageId = generateUUID(); writeAssistantTextChunk( writer, missingTargetMessage, `agent:${directAgentToolCapability}:missing-target`, messageId, ); await saveMessages({ messages: [ { id: messageId, chatId: id, role: "assistant", parts: [{ type: "text", text: missingTargetMessage }], attachments: [], createdAt: new Date(), videoStorageStatus: null, temporaryVideoUrl: null, permanentVideoUrl: null, archivedAt: null, videoMetadata: null, }, ], }); }, }); return createSseResponse(missingTargetStream); } const assistantMessageId = generateUUID(); await saveMessages({ messages: [ { id: assistantMessageId, chatId: id, role: "assistant", parts: [{ type: "text", text: "处理中..." }], attachments: [], createdAt: new Date(), videoStorageStatus: null, temporaryVideoUrl: null, permanentVideoUrl: null, archivedAt: null, videoMetadata: null, }, ], }); try { let resultOriginalUrl = ""; let resultThumbnailUrl: string | null = null; let resultImageSize = "2K"; let resultAssetId: string | null = null; if (directAgentToolCapability === "remove_background") { const sourceBuffer = await fetchImageBufferFromUrl(directAgentToolTargetImageUrl); const result = await executeRmbgJob({ userId: session.user.id, imageBuffer: sourceBuffer, prompt: requestTextContent || "抠图处理", chatId: id, messageId: assistantMessageId, sourceImageUrl: directAgentToolTargetImageUrl, }); resultOriginalUrl = result.original_url; resultThumbnailUrl = result.thumbnail_url; resultImageSize = "2K"; resultAssetId = result.mediaAssetId || null; } else if (directAgentToolCapability === "resize") { const plannedArgs = findAgentToolCallArgs( "resizeImage", smartAutoExecution, smartAutoPlan, ); const inferredOptions = inferResizeOptionsFromText(requestTextContent); const canvasWidth = toFiniteNumber( plannedArgs?.canvasWidth ?? plannedArgs?.width ?? inferredOptions.canvasWidth, ); const canvasHeight = toFiniteNumber( plannedArgs?.canvasHeight ?? plannedArgs?.height ?? inferredOptions.canvasHeight, ); if (!canvasWidth || !canvasHeight) { throw new ResizeServiceError( "请告诉我目标尺寸或比例，例如 1024x1024、4:5、16:9。", 400, ); } const transparentBackground = plannedArgs?.transparentBackground === true || plannedArgs?.transparentBackground === "true" || inferredOptions.transparentBackground === true; const backgroundColor = typeof plannedArgs?.backgroundColor === "string" ? plannedArgs.backgroundColor : inferredOptions.backgroundColor; const mimeType = typeof plannedArgs?.mimeType === "string" ? plannedArgs.mimeType : undefined; const result = await executeResizeJob({ userId: session.user.id, imageUrl: directAgentToolTargetImageUrl, prompt: requestTextContent || "尺寸调整处理", chatId: id, messageId: assistantMessageId, sourceImageUrl: directAgentToolTargetImageUrl, canvasWidth, canvasHeight, transparentBackground, backgroundColor, mimeType, }); resultOriginalUrl = result.original_url; resultThumbnailUrl = result.thumbnail_url; resultImageSize = `${result.canvas_width}x${result.canvas_height}`; resultAssetId = result.mediaAssetId || null; } else if (directAgentToolCapability === "extend") { const plannedArgs = findAgentToolCallArgs( "extendImage", smartAutoExecution, smartAutoPlan, ); const inferredOptions = inferExtendOptionsFromText(requestTextContent); const xScale = toFiniteNumber(plannedArgs?.xScale ?? inferredOptions.xScale); const yScale = toFiniteNumber(plannedArgs?.yScale ?? inferredOptions.yScale); const result = await executeExtendJob({ userId: session.user.id, imageUrl: directAgentToolTargetImageUrl, prompt: requestTextContent || "图像拓展处理", chatId: id, messageId: assistantMessageId, sourceImageUrl: directAgentToolTargetImageUrl, ...(xScale ? { xScale } : {}), ...(yScale ? { yScale } : {}), }); resultOriginalUrl = result.original_url; resultThumbnailUrl = result.thumbnail_url; resultImageSize = "2K"; resultAssetId = result.mediaAssetId || null; } else if (directAgentToolCapability === "box_cutout") { const plannedArgs = findAgentToolCallArgs( "boxCutout", smartAutoExecution, smartAutoPlan, ); const cropRect = plannedArgs?.cropRect && typeof plannedArgs.cropRect === "object" ? (plannedArgs.cropRect as Record) : undefined; if (!cropRect) { throw new BoxCutoutServiceError( "缺少框选区域，暂时无法执行框选抠图。", 400, ); } const result = await executeBoxCutoutJob({ userId: session.user.id, imageUrl: directAgentToolTargetImageUrl, cropRect: { x: toFiniteNumber(cropRect.x), y: toFiniteNumber(cropRect.y), width: toFiniteNumber(cropRect.width), height: toFiniteNumber(cropRect.height), }, paddingRatio: toFiniteNumber(plannedArgs?.paddingRatio), prompt: requestTextContent || "框选抠图处理", chatId: id, messageId: assistantMessageId, sourceNodeId: typeof plannedArgs?.sourceNodeId === "string" ? plannedArgs.sourceNodeId : undefined, }); resultOriginalUrl = result.original_url; resultThumbnailUrl = result.thumbnail_url || null; resultImageSize = "2K"; resultAssetId = result.mediaAssetId || null; } const assistantText = prependImageModelBadge( [ getDirectAgentToolSuccessMessage(directAgentToolCapability), buildAgentToolImageMarkdown({ originalUrl: resultOriginalUrl, thumbnailUrl: resultThumbnailUrl, imageSize: resultImageSize, }), ] .filter(Boolean) .join("\n\n"), getDirectAgentToolLabel(directAgentToolCapability), ); await updateMessageById({ messageId: assistantMessageId, chatId: id, parts: [{ type: "text", text: assistantText }], }); try { const resultAssetIds = resultAssetId ? [resultAssetId] : null; const resultImageUrls = resultOriginalUrl ? [resultOriginalUrl] : null; await persistImageWorkflowTurn({ userId: session.user.id, chatId: id, projectId: contextPayload?.projectId || null, currentIntent: "image_edit", currentSubIntent: directAgentToolCapability, currentFocusAssetIds: resultAssetIds, currentFocusImageUrls: resultImageUrls, lastGeneratedAssetIds: resultAssetIds, lastGeneratedImageUrls: resultImageUrls, lastResolvedTargetAssetIds: resultAssetIds, lastResolvedTargetImageUrls: resultImageUrls, lastMessageId: assistantMessageId, lastUserPrompt: requestTextContent, lastOptimizedPrompt: smartAutoOptimizedPrompt || undefined, lastRequestedImageCount: resultImageUrls?.length || 1, lastImageSize: resultImageSize || undefined, lastSourceImageUrls: directAgentToolTargetImageUrls, lastResultImageUrls: resultImageUrls, }); } catch (memoryError) { console.warn( `[Chat API] Failed to update working state after ${directAgentToolCapability}:`, memoryError, ); } if (directAgentExecution.publishToCanvas && contextPayload?.source === "canvas") { console.log( `[Chat API] Agent publishToCanvas satisfied by assistant media auto-sync (${directAgentToolCapability})`, ); } const directToolStream = createUIMessageStream({ execute: async ({ writer }) => { writeAssistantTextChunk( writer, assistantText, `agent:${directAgentToolCapability}:success`, assistantMessageId, ); if (contextPayload?.source === "canvas") { writeCanvasMediaChunk( writer, { messageId: assistantMessageId, imageUrls: resultOriginalUrl ? [resultOriginalUrl] : [], prompt: requestTextContent || getDirectAgentToolLabel(directAgentToolCapability), sourceImageUrls: directAgentToolTargetImageUrls, }, `agent:${directAgentToolCapability}:canvas-media`, ); } writeAssistantTextChunk( writer, "", `agent:${directAgentToolCapability}:image-end`, ); }, }); return createSseResponse(directToolStream); } catch (directToolError) { console.error( `[Chat API] Direct agent tool ${directAgentToolCapability} failed:`, directToolError, ); if ( !(directToolError instanceof RmbgServiceError) && !(directToolError instanceof ResizeServiceError) && !(directToolError instanceof ExtendServiceError) && !(directToolError instanceof BoxCutoutServiceError) ) { console.warn( `[Chat API] Direct agent tool ${directAgentToolCapability} threw unexpected error type`, ); } const errorMessage = getDirectAgentToolErrorMessage( directAgentToolCapability, directToolError, ); try { await updateMessageById({ messageId: assistantMessageId, chatId: id, parts: [{ type: "text", text: errorMessage }], }); } catch (updateError) { console.warn( `[Chat API] Failed to persist direct-tool error message (${directAgentToolCapability}):`, updateError, ); } const directToolErrorStream = createUIMessageStream({ execute: async ({ writer }) => { writeAssistantTextChunk( writer, errorMessage, `agent:${directAgentToolCapability}:error`, assistantMessageId, ); writeAssistantTextChunk( writer, "", `agent:${directAgentToolCapability}:image-end-error`, ); }, }); return createSseResponse(directToolErrorStream); } } let finalMergedUsage: AppUsage | undefined; // 获取启用的工具列表 const enabledTools = await getEnabledTools(); // 检查是否是 UUID 格式（数据库中的模型 ID） const isUUID = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test( finalModelId ); // 获取语言模型实例 let languageModel; let actualModelId: string; let resolvedModelSupportsImageGen = false; let resolvedProviderType: string | undefined; let resolvedRuntimeModelLookupId = finalModelId; if (isUUID) { // 从数据库获取动态配置的模型 let targetModelId = finalModelId; // 🆕 检查是否是视频主模型，如果是，根据 videoParams.mode 选择子模型 const aiModelData = await getAiModelById({ id: finalModelId }); const aiModelProvider = aiModelData ? await getAiProviderById({ id: aiModelData.providerId }).catch(() => null) : null; console.log("[Chat API] Model data:", { id: finalModelId, isSubModel: aiModelData?.isSubModel }); if (aiModelData && !aiModelData.isSubModel && aiModelProvider?.type === "volcano-video") { // 检查是否有子模型 const subModels = await getSubModelsByParentModelId(finalModelId); console.log("[Chat API] Found sub-models:", subModels.length); if (subModels.length > 0) { // 这是一个主模型，需要根据 videoParams 选择子模型 const videoParams = effectiveVideoParams; console.log("[Chat API] Video params:", videoParams); // 如果没有 videoParams 或没有 mode，使用默认模式 const mode = (videoParams?.mode || "text-to-video") as VideoModeType; console.log("[Chat API] Using mode:", mode); const matchedSubModel = selectBestVideoSubModelForMode(subModels as any, mode); if (matchedSubModel) { console.log( `[Chat API] 视频模式 ${mode} → 选择子模型: ${matchedSubModel.model.displayName} (${matchedSubModel.model.modelId})` ); targetModelId = matchedSubModel.model.id; } else { console.warn(`[Chat API] 未找到可用子模型，继续使用主模型`); } } } const dynamicModel = await getLanguageModelById(targetModelId); if (!dynamicModel) { return new ChatSDKError( "bad_request:api", "Model not found or disabled" ).toResponse(); } languageModel = dynamicModel; resolvedRuntimeModelLookupId = targetModelId; // 对于动态模型，我们需要从数据库获取 modelId const finalModelData = await getAiModelById({ id: targetModelId }); actualModelId = finalModelData?.modelId ?? targetModelId; resolvedModelSupportsImageGen = Boolean(finalModelData?.supportsImageGen); if (finalModelData) { resolvedProviderType = ( await getAiProviderById({ id: finalModelData.providerId }).catch(() => null) )?.type; } } else { // 使用预定义的模型 languageModel = myProvider.languageModel(finalModelId); actualModelId = languageModel.modelId; resolvedProviderType = (languageModel as any)?.provider; resolvedRuntimeModelLookupId = actualModelId; } const effectiveConversationSystemPrompt = shouldRunAgentConversation ? `${await buildAgentRoleIdentityInstruction({ allowVideoGeneration: allowAgentVideoGeneration, hasCanvasContext: Boolean( effectiveContextPayload?.source === "canvas" || effectiveContextPayload?.selectedNodeIds?.length || effectiveContextPayload?.canvasImageNodes?.length, ), })}\n\n${systemPrompt({ selectedChatModel: finalModelId, requestHints })}` : systemPrompt({ selectedChatModel: finalModelId, requestHints }); const activeRuntimeModelDescriptor = await resolveAgentModelSlotDescriptor( resolvedRuntimeModelLookupId, ).catch(() => null); if (shouldRunAgentConversation && activeRuntimeModelDescriptor) { const slotName = smartAutoExecution?.shouldApplyImagePipeline || smartAutoExecution?.shouldApplyVideoPipeline ? "execution" : "reply"; agentModelSlots = upsertAgentModelSlot(agentModelSlots, slotName, { ...activeRuntimeModelDescriptor, source: "runtime_selected_model", reason: slotName === "reply" ? smartAutoExecution?.shouldStayConversational ? "conversational_reply_runtime" : "default_reply_runtime" : "execution_runtime_model", locked: slotName === "reply" ? shouldKeepSelectedModel : false, }); pendingProtocolDebugEvents.push({ phase: "modelSlots", payload: { modelSlots: agentModelSlots, providerHealth: activeRuntimeModelDescriptor.providerKey ? getProviderHealthSnapshot(activeRuntimeModelDescriptor.providerKey) : null, }, }); } // ========== 图片生成配额检查 ========== const isUuidModelId = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test( finalModelId ); const currentModelConfig = isUuidModelId ? await getAiModelById({ id: finalModelId }).catch(() => undefined) : undefined; const runtimeProviderType = String( resolvedProviderType || (languageModel as any).provider || "" ); const normalizedRuntimeProviderType = runtimeProviderType === "gemini-image" ? "gemini" : runtimeProviderType === "huanke-gemini" ? "huanke-gemini" : runtimeProviderType; const isImageGenerationModel = isImageGenerationModelType( normalizedRuntimeProviderType, currentModelConfig ? { modelId: currentModelConfig.modelId, capabilities: currentModelConfig.capabilities, supportsImageGen: currentModelConfig.supportsImageGen, } : undefined ); const modelCreditCostPerImage = Number( (currentModelConfig as any)?.imageGenConfig?.creditCostPerImage ?? 0 ); const modelCreditCostPerVideo = getModelCreditCost(currentModelConfig, "video"); if (isImageGenerationModel) { const quota = await checkUserImageQuota(session.user.id); if (!quota.allowed) { console.log(`[Chat API] ⛔ 图片配额已用完: ${quota.used}/${quota.limit}`); return NextResponse.json( { error: "quota_exceeded", message: `本月图片生成配额已用完（已使用 ${quota.used}/${quota.limit} 张），配额将于下月1日重置，如需更多配额请联系管理员。`, used: quota.used, limit: quota.limit, }, { status: 429 } ); } console.log(`[Chat API] ✅ 图片配额检查通过: ${quota.used}/${quota.limit ?? '无限制'}`); const pointsConfig = await getPointsSystemConfig(); const normalizedUnitCost = Number.isFinite(modelCreditCostPerImage) ? Math.max(0, Math.round(modelCreditCostPerImage)) : 0; if (pointsConfig.enabled && normalizedUnitCost > 0 && pointsConfig.blockOnInsufficient !== false) { const requestedImageCount = Math.max( 1, Math.round( Number( effectiveImageParams?.imageCount || defaultImageQualityParams.imageCount || 1 ) ) ); const expectedCost = normalizedUnitCost * requestedImageCount; const userData = await getUserById({ id: session.user.id }); if (!userData?.isHighVip) { const currentBalance = Math.max(0, Number(userData?.pointsBalance || 0)); if (currentBalance < expectedCost) { return NextResponse.json( { error: "insufficient_points", message: `积分不足，当前余额 ${currentBalance}，本次预计消耗 ${expectedCost}（${normalizedUnitCost} x ${requestedImageCount} 张）。`, balance: currentBalance, expectedCost, unitCost: normalizedUnitCost, }, { status: 402 } ); } } } } const isVideoGenerationModel = isVideoProviderType( normalizedRuntimeProviderType, currentModelConfig ? { modelId: currentModelConfig.modelId, capabilities: currentModelConfig.capabilities, supportsImageGen: currentModelConfig.supportsImageGen, } : undefined ); if (isVideoGenerationModel) { const pointsConfig = await getPointsSystemConfig(); const normalizedVideoUnitCost = Number.isFinite(modelCreditCostPerVideo) ? Math.max(0, Math.round(modelCreditCostPerVideo)) : 0; if ( pointsConfig.enabled && normalizedVideoUnitCost > 0 && pointsConfig.blockOnInsufficient !== false ) { const expectedCost = normalizedVideoUnitCost; const userData = await getUserById({ id: session.user.id }); if (!userData?.isHighVip) { const currentBalance = Math.max(0, Number(userData?.pointsBalance || 0)); if (currentBalance < expectedCost) { return NextResponse.json( { error: "insufficient_points", message: `积分不足，当前余额 ${currentBalance}，本次预计消耗 ${expectedCost}（${normalizedVideoUnitCost} x 1 条视频）。`, balance: currentBalance, expectedCost, unitCost: normalizedVideoUnitCost, }, { status: 402 } ); } } } } // ========== Gemini 图片生成独立处理 ========== // 支持 gemini-image 和 huanke-gemini 两种图片生成 provider const isGeminiImageProvider = ["gemini-image", "huanke-gemini"].includes((languageModel as any).provider); if (isGeminiImageProvider && typeof (languageModel as any).doGenerate === "function") { console.log("[Chat API] Using Gemini image generation, provider:", (languageModel as any).provider); console.log("[Chat API] Image params:", effectiveImageParams); // 🆕 获取备用模型配置 const currentModelConfig = await getAiModelById({ id: finalModelId }); const configuredFallbacks = [ currentModelConfig?.fallbackModelId1, currentModelConfig?.fallbackModelId2, ].filter(Boolean) as string[]; const allModels = await getEnabledAiModelsWithProvider(); const fallbackModelIds = buildImageFallbackModelIds(allModels, { currentProviderId: currentModelConfig?.providerId, currentModelId: currentModelConfig?.id, currentModelKey: actualModelId, configuredFallbackIds: configuredFallbacks, }); console.log("[Chat API] Fallback model IDs:", fallbackModelIds); const geminiStream = createUIMessageStream({ execute: async ({ writer }) => { // 🆕 如果是智能模式且有对话回复，先发送对话消息让用户看到 // 在消息末尾加上特殊标记，告诉前端"图片正在生成中" const conversationMessage = smartAutoFriendlyMessage || null; if (conversationMessage) { const chatMessageId = generateUUID(); console.log("[Chat API] 💬 Gemini: Sending conversational reply first:", smartAutoFriendlyMessage); // 发送带标记的对话消息 writeAssistantTextChunk( writer, conversationMessage, "gemini:conversation", chatMessageId, ); // 保存对话消息到数据库（不带标记，纯对话） await saveMessages({ messages: [ { id: chatMessageId, chatId: id, role: "assistant", parts: [{ type: "text", text: smartAutoFriendlyMessage }], attachments: [], createdAt: new Date(), videoStorageStatus: null, temporaryVideoUrl: null, permanentVideoUrl: null, archivedAt: null, videoMetadata: null, }, ], }); } // 🆕 尝试生成图片的函数（支持备用模型切换） const tryGenerateImage = async (modelIdToUse: string, isRetry = false): Promise => { console.log(`[Chat API] ${isRetry ? "🔄 Retry with fallback" : "🚀 Trying"} model: ${modelIdToUse}`); // 只传递最后一条用户消息，避免上下文历史干扰生图效果 const lastUserMessage = uiMessagesForModel.filter((m) => m.role === "user").pop(); let messagesForGemini = lastUserMessage ? [lastUserMessage] : uiMessagesForModel; messagesForGemini = applyOptimizedPromptToLatestUserMessage( messagesForGemini, smartAutoOptimizedPrompt, hiddenPromptPrefix, ); messagesForGemini = injectHistoryImagesToLastUserMessage(messagesForGemini, smartAutoHistoryImages); console.log("[Chat API] Gemini: Using only last user message (no history context)"); // 转换消息：gemini-image 需要 base64 格式，huanke-gemini 需要 URL 格式 const providerForConvert = (languageModel as any).provider === "huanke-gemini" ? "huanke-gemini" : "gemini"; const modelMessages = await convertMessagesWithImages(messagesForGemini, providerForConvert); console.log("[Chat API] Message conversion provider:", providerForConvert); // 调用 doGenerate 获取图片（如果是备用模型，传入 overrideModelId） const result = await (languageModel as any).doGenerate({ messages: modelMessages, imageParams: effectiveImageParams || { aspectRatio: defaultImageQualityParams.aspectRatio, imageSize: defaultImageQualityParams.imageSize, }, overrideModelId: isRetry ? modelIdToUse : undefined, // 🆕 使用备用模型 ID }); return result.text || "图片生成失败"; }; let stopKeepalive: (() => void) | null = null; try { let imageMarkdown = ""; // 初始化为空字符串 let usedModelId = actualModelId; // 🆕 图片生成真正开始时发出标记（用于前端显示占位图） writeAssistantTextChunk( writer, "", "gemini:image-start", ); if (contextPayload?.source === "canvas") { writeImageGenerationStatusChunk( writer, { phase: "processing", providerType: (languageModel as any).provider, modelName: currentModelConfig?.displayName || actualModelId, imageCount: effectiveImageParams?.imageCount || defaultImageQualityParams.imageCount || 1, }, "gemini:image-status-start", ); } stopKeepalive = startStreamKeepalive(writer, "gemini-image"); try { // 首先尝试主模型 imageMarkdown = await tryGenerateImage(actualModelId, false); } catch (primaryError) { console.error("[Chat API] Primary model failed:", primaryError); // 🆕 尝试备用模型（使用同一个 provider，但切换 modelId） let fallbackSuccess = false; for (const fallbackModelId of fallbackModelIds) { try { console.log(`[Chat API] 🔄 Trying fallback model: ${fallbackModelId}`); imageMarkdown = await tryGenerateImage(fallbackModelId, true); usedModelId = fallbackModelId; fallbackSuccess = true; console.log(`[Chat API] ✅ Fallback model ${fallbackModelId} succeeded!`); break; } catch (fallbackError) { console.error(`[Chat API] ❌ Fallback model ${fallbackModelId} failed:`, fallbackError); } } // 如果所有备用模型都失败，抛出原始错误触发火山引擎故障转移 if (!fallbackSuccess) { throw primaryError; } } console.log("[Chat API] Gemini generated, length:", imageMarkdown.length); console.log("[Chat API] Gemini image markdown:", imageMarkdown.substring(0, 200)); console.log("[Chat API] Used model ID:", usedModelId); const usedModelDisplayName = resolveModelDisplayNameByModelId(allModels, usedModelId) || currentModelConfig?.displayName || actualModelId; const imageMarkdownWithModel = prependImageModelBadge( imageMarkdown, usedModelDisplayName, ); // 生成一个统一的消息 ID const messageId = generateUUID(); // 写入消息到流 - 使用正确的 parts 格式 writeAssistantTextChunk( writer, imageMarkdownWithModel, "gemini:image-result", messageId, ); if (contextPayload?.source === "canvas") { writeCanvasMediaChunk( writer, { messageId, text: imageMarkdownWithModel, prompt: requestTextContent, sourceImageUrls: requestTargetResolution?.targetImageUrls || [], }, "gemini:image-canvas-media", ); } // 🆕 图片生成结束标记（成功） writeAssistantTextChunk( writer, "", "gemini:image-end-success", ); if (contextPayload?.source === "canvas") { writeImageGenerationStatusChunk( writer, { phase: "completed", providerType: (languageModel as any).provider, modelName: usedModelDisplayName, imageCount: effectiveImageParams?.imageCount || defaultImageQualityParams.imageCount || 1, }, "gemini:image-status-completed", ); } // 保存消息到数据库（现在是 S3 URL，可以完整保存） try { await saveMessages({ messages: [ { id: messageId, chatId: id, role: "assistant", parts: [{ type: "text", text: imageMarkdownWithModel }], attachments: [], // 必须提供 attachments 字段（数据库 notNull） createdAt: new Date(), videoStorageStatus: null, temporaryVideoUrl: null, permanentVideoUrl: null, archivedAt: null, videoMetadata: null, }, ], }); console.log("[Chat API] ✅ Gemini message saved successfully!"); // 提取用户 prompt（最后一条用户消息的文本） const lastUserMessage = uiMessages.filter((m) => m.role === "user").pop(); const userPrompt = lastUserMessage?.parts?.find((p): p is { type: "text"; text: string } => p.type === "text")?.text || ""; // 保存图片记录到 GeneratedImage 表 await saveGeneratedImages( imageMarkdownWithModel, session.user.id, id, messageId, userPrompt, finalModelId, { sourceAssetIds: requestTargetResolution?.targetAssetIds, sourceImageUrls: requestTargetResolution?.targetImageUrls, relationType: requestResolvedMediaInput?.hasImage ? "edited_from" : "generated_from", sourceType: requestResolvedMediaInput?.hasImage ? "generated" : "generated", projectId: contextPayload?.projectId || null, optimizedPrompt: smartAutoOptimizedPrompt || userPrompt, requestedImageCount: effectiveImageParams?.imageCount, aspectRatio: effectiveImageParams?.aspectRatio, imageSize: effectiveImageParams?.imageSize, }, ); } catch (saveError) { console.error("[Chat API] ❌ Failed to save Gemini message to database:", saveError); console.error("[Chat API] Message content that failed:", imageMarkdown.substring(0, 300)); } } catch (error) { console.error("[Chat API] Gemini generation error:", error); console.log("[Chat API] Attempting failover to Volcano Vision..."); // 故障转移：尝试使用火山引擎模型 try { // 查找火山引擎模型 const allModels = await getEnabledAiModelsWithProvider(); console.log("[Chat API] Total enabled models:", allModels.length); const volcanoModel = allModels.find( (m: any) => m.provider.type === "volcano-vision" && m.model.isEnabled ); if (volcanoModel) { console.log("[Chat API] Found Volcano Vision model for failover:", volcanoModel.model.id); // 获取火山引擎模型实例 const volcanoLanguageModel = await getLanguageModelById(volcanoModel.model.id); if (volcanoLanguageModel && typeof (volcanoLanguageModel as any).doGenerate === "function") { console.log("[Chat API] Executing Volcano Vision failover..."); // 转换消息（火山引擎会自动处理 Base64） const volcanoMessages = await convertMessagesWithImages( injectHistoryImagesToLastUserMessage(uiMessagesForModel, smartAutoHistoryImages), "volcano" ); // 调用火山引擎生成（故障转移时只生成 1 张，保持与 Gemini 一致） const volcanoResult = await (volcanoLanguageModel as any).doGenerate({ messages: volcanoMessages, maxImages: 1, // 故障转移时只生成 1 张 imageParams: effectiveImageParams || { aspectRatio: defaultImageQualityParams.aspectRatio, imageSize: defaultImageQualityParams.imageSize, imageCount: defaultImageQualityParams.imageCount, }, }); const imageMarkdown = volcanoResult.text || "图片生成失败"; const imageMarkdownWithModel = prependImageModelBadge( imageMarkdown, volcanoModel.model.displayName || volcanoModel.model.modelId, ); console.log("[Chat API] Volcano Vision failover success, length:", imageMarkdown.length); // 生成消息 ID const messageId = generateUUID(); // 写入消息到流 writeAssistantTextChunk( writer, imageMarkdownWithModel, "gemini:volcano-failover-result", messageId, ); if (contextPayload?.source === "canvas") { writeCanvasMediaChunk( writer, { messageId, text: imageMarkdownWithModel, prompt: requestTextContent, sourceImageUrls: requestTargetResolution?.targetImageUrls || [], }, "gemini:volcano-failover-canvas-media", ); } // 🆕 图片生成结束标记（故障转移成功） writeAssistantTextChunk( writer, "", "gemini:volcano-failover-end", ); if (contextPayload?.source === "canvas") { writeImageGenerationStatusChunk( writer, { phase: "completed", providerType: "volcano-vision", modelName: volcanoModel.model.displayName || volcanoModel.model.modelId, imageCount: 1, }, "gemini:volcano-failover-status-completed", ); } // 保存消息到数据库 try { await saveMessages({ messages: [ { id: messageId, chatId: id, role: "assistant", parts: [{ type: "text", text: imageMarkdownWithModel }], attachments: [], createdAt: new Date(), videoStorageStatus: null, temporaryVideoUrl: null, permanentVideoUrl: null, archivedAt: null, videoMetadata: null, }, ], }); console.log("[Chat API] ✅ Volcano Vision failover message saved!"); // 提取用户 prompt const lastUserMessage = uiMessages.filter((m) => m.role === "user").pop(); const userPrompt = lastUserMessage?.parts?.find((p): p is { type: "text"; text: string } => p.type === "text")?.text || ""; // 保存图片记录到 GeneratedImage 表 await saveGeneratedImages( imageMarkdownWithModel, session.user.id, id, messageId, userPrompt, volcanoModel.model.id, { sourceAssetIds: requestTargetResolution?.targetAssetIds, sourceImageUrls: requestTargetResolution?.targetImageUrls, relationType: requestResolvedMediaInput?.hasImage ? "edited_from" : "generated_from", sourceType: "generated", projectId: contextPayload?.projectId || null, optimizedPrompt: smartAutoOptimizedPrompt || userPrompt, requestedImageCount: effectiveImageParams?.imageCount, aspectRatio: effectiveImageParams?.aspectRatio, imageSize: effectiveImageParams?.imageSize, }, ); } catch (saveError) { console.error("[Chat API] Failed to save failover message:", saveError); } return; // 成功，直接返回 } } console.log("[Chat API] No Volcano Vision model available for failover"); } catch (failoverError) { console.error("[Chat API] Failover to Volcano Vision also failed:", failoverError); } // 如果故障转移也失败，返回错误消息 const errorMessageId = generateUUID(); writeAssistantTextChunk( writer, "图片生成失败，请稍后重试", "gemini:error", errorMessageId, ); // 🆕 图片生成结束标记（失败） writeAssistantTextChunk( writer, "", "gemini:image-end-failed", ); // 尝试保存错误消息 try { await saveMessages({ messages: [ { id: errorMessageId, chatId: id, role: "assistant", parts: [{ type: "text", text: "图片生成失败，请稍后重试" }], attachments: [], createdAt: new Date(), videoStorageStatus: null, temporaryVideoUrl: null, permanentVideoUrl: null, archivedAt: null, videoMetadata: null, }, ], }); } catch (saveError) { console.error("[Chat API] Failed to save error message:", saveError); } } finally { stopKeepalive?.(); } }, }); return createSseResponse(geminiStream); } // ========== Gemini 处理结束 ========== // ========== 阿里云千问图片生成独立处理 ========== const isAlibabaLikeImageProvider = isImageGenerationModel && [ "alibaba-image", "bfl-image", "ruxa-image", "fal-image", "openai-image", "midjourney", "runway", ].includes((languageModel as any).provider); if (isAlibabaLikeImageProvider && typeof (languageModel as any).doGenerate === "function") { console.log("[Chat API] Using image generation provider:", (languageModel as any).provider); console.log("[Chat API] Image params:", effectiveImageParams); // 🆕 获取模型显示名称，用于错误提示 const alibabaModelConfig = await getAiModelById({ id: finalModelId }); const modelDisplayName = alibabaModelConfig?.displayName || actualModelId; const configuredFallbacks = [ alibabaModelConfig?.fallbackModelId1, alibabaModelConfig?.fallbackModelId2, ].filter(Boolean) as string[]; const allModels = await getEnabledAiModelsWithProvider(); const fallbackModelIds = buildImageFallbackModelIds(allModels, { currentProviderId: alibabaModelConfig?.providerId, currentModelId: alibabaModelConfig?.id, currentModelKey: actualModelId, configuredFallbackIds: configuredFallbacks, }); console.log("[Chat API] Alibaba fallback model IDs:", fallbackModelIds); const alibabaStream = createUIMessageStream({ execute: async ({ writer }) => { let stopKeepalive: (() => void) | null = null; try { // 🆕 如果是智能模式且有对话回复，先发送对话消息让用户看到 const conversationMessage = smartAutoFriendlyMessage || null; if (conversationMessage) { const chatMessageId = generateUUID(); console.log("[Chat API] 💬 Sending conversational reply first:", smartAutoFriendlyMessage); // 发送带标记的对话消息 writeAssistantTextChunk( writer, conversationMessage, "alibaba:conversation", chatMessageId, ); // 保存对话消息到数据库（不带标记，纯对话） await saveMessages({ messages: [ { id: chatMessageId, chatId: id, role: "assistant", parts: [{ type: "text", text: smartAutoFriendlyMessage }], attachments: [], createdAt: new Date(), videoStorageStatus: null, temporaryVideoUrl: null, permanentVideoUrl: null, archivedAt: null, videoMetadata: null, }, ], }); } // 只传递最后一条用户消息 const lastUserMessage = uiMessagesForModel.filter((m) => m.role === "user").pop(); let messagesForAlibaba = lastUserMessage ? [lastUserMessage] : uiMessagesForModel; console.log("[Chat API] Alibaba: Using only last user message"); // 🆕 如果有智能模式优化后的提示词，替换消息中的文本（仅用于API调用） if (smartAutoOptimizedPrompt) { console.log("[Chat API] Using optimized prompt for API call"); messagesForAlibaba = applyOptimizedPromptToLatestUserMessage( messagesForAlibaba, smartAutoOptimizedPrompt, hiddenPromptPrefix, ); } messagesForAlibaba = injectHistoryImagesToLastUserMessage(messagesForAlibaba, smartAutoHistoryImages); const tryGenerateImage = async (modelIdToUse: string, isRetry: boolean) => { // 转换消息（图片编辑模型统一使用 URL 输入） const providerForConvert = (languageModel as any).provider === "bfl-image" ? "bfl-image" : (languageModel as any).provider === "ruxa-image" ? "ruxa-image" : (languageModel as any).provider === "fal-image" ? "fal-image" : (languageModel as any).provider === "openai-image" ? "openai-image" : (languageModel as any).provider === "midjourney" ? "midjourney" : (languageModel as any).provider === "runway" ? "runway" : "alibaba-image"; const modelMessages = await convertMessagesWithImages(messagesForAlibaba, providerForConvert); // 调用 doGenerate 获取图片，传递模型显示名称 const result = await (languageModel as any).doGenerate({ messages: modelMessages, imageParams: effectiveImageParams || { aspectRatio: defaultImageQualityParams.aspectRatio, imageSize: defaultImageQualityParams.imageSize, imageCount: defaultImageQualityParams.imageCount, }, modelDisplayName, // 🆕 传递模型显示名称 overrideModelId: isRetry ? modelIdToUse : undefined, }); if (result?.finishReason === "error") { throw new Error(result.text || "图片生成失败"); } return result.text || "图片生成失败"; }; let imageMarkdown = ""; let usedModelId = actualModelId; // 🆕 图片生成真正开始时发出标记（用于前端显示占位图） writeAssistantTextChunk( writer, "", "alibaba:image-start", ); if (contextPayload?.source === "canvas") { writeImageGenerationStatusChunk( writer, { phase: "processing", providerType: (languageModel as any).provider, modelName: alibabaModelConfig?.displayName || actualModelId, imageCount: effectiveImageParams?.imageCount || defaultImageQualityParams.imageCount || 1, }, "alibaba:image-status-start", ); } stopKeepalive = startStreamKeepalive(writer, "alibaba-image"); try { imageMarkdown = await tryGenerateImage(actualModelId, false); } catch (primaryError) { console.error("[Chat API] Alibaba primary model failed:", primaryError); let fallbackSuccess = false; for (const fallbackModelId of fallbackModelIds) { try { console.log(`[Chat API] 🔄 Alibaba fallback model: ${fallbackModelId}`); imageMarkdown = await tryGenerateImage(fallbackModelId, true); usedModelId = fallbackModelId; fallbackSuccess = true; console.log(`[Chat API] ✅ Alibaba fallback model ${fallbackModelId} succeeded!`); break; } catch (fallbackError) { console.error(`[Chat API] ❌ Alibaba fallback model ${fallbackModelId} failed:`, fallbackError); } } if (!fallbackSuccess) { const providerPriority: Record = { "volcano-vision": 0, "alibaba-image": 1, "bfl-image": 2, "ruxa-image": 3, "fal-image": 4, "openai-image": 5, runway: 6, gemini: 6, "huanke-gemini": 7, }; const crossProviderCandidates = getImageCrossProviderCandidates(allModels, { currentModelId: finalModelId, currentProviderId: alibabaModelConfig?.providerId, providerPriority, }); for (const candidate of crossProviderCandidates) { try { console.log( `[Chat API] 🔁 Cross-provider failover attempt: ${candidate.model.displayName} (${candidate.provider.type})` ); const candidateLanguageModel = await getLanguageModelById(candidate.model.id); if (!candidateLanguageModel || typeof (candidateLanguageModel as any).doGenerate !== "function") { console.log(`[Chat API] Skip candidate without doGenerate: ${candidate.model.id}`); continue; } const convertProvider = candidate.provider.type === "bfl-image" ? "bfl-image" : candidate.provider.type === "ruxa-image" ? "ruxa-image" : candidate.provider.type === "fal-image" ? "fal-image" : candidate.provider.type === "openai-image" ? "openai-image" : candidate.provider.type === "midjourney" ? "midjourney" : candidate.provider.type === "runway" ? "runway" : candidate.provider.type === "gemini" ? "gemini" : candidate.provider.type === "huanke-gemini" ? "huanke-gemini" : candidate.provider.type === "volcano-vision" ? "volcano" : "alibaba-image"; const candidateMessages = await convertMessagesWithImages( messagesForAlibaba, convertProvider ); const candidateResult = await (candidateLanguageModel as any).doGenerate({ messages: candidateMessages, imageParams: effectiveImageParams || { aspectRatio: defaultImageQualityParams.aspectRatio, imageSize: defaultImageQualityParams.imageSize, imageCount: defaultImageQualityParams.imageCount, }, modelDisplayName: candidate.model.displayName, overrideModelId: candidate.model.modelId, }); if (candidateResult?.finishReason === "error") { throw new Error(candidateResult.text || "图片生成失败"); } imageMarkdown = candidateResult.text || "图片生成失败"; usedModelId = candidate.model.modelId; fallbackSuccess = true; console.log( `[Chat API] ✅ Cross-provider failover succeeded: ${candidate.model.displayName} (${candidate.provider.type})` ); break; } catch (crossError) { const crossErrorMessage = normalizeFailoverErrorMessage(crossError); if (shouldCooldownImageFallbackModel(crossErrorMessage)) { setImageFallbackModelCooldown(candidate.model.id, crossErrorMessage); } console.error( `[Chat API] ❌ Cross-provider failover failed: ${candidate.model.displayName} (${candidate.provider.type})`, crossError ); } } } if (!fallbackSuccess) { throw primaryError; } } console.log("[Chat API] Alibaba generated, length:", imageMarkdown.length); console.log("[Chat API] Alibaba used model:", usedModelId); const usedModelDisplayName = resolveModelDisplayNameByModelId(allModels, usedModelId) || alibabaModelConfig?.displayName || actualModelId; const imageMarkdownWithModel = prependImageModelBadge( imageMarkdown, usedModelDisplayName, ); // 生成消息 ID（图片消息） const messageId = generateUUID(); // 写入图片消息到流 writeAssistantTextChunk( writer, imageMarkdownWithModel, "alibaba:image-result", messageId, ); if (contextPayload?.source === "canvas") { writeCanvasMediaChunk( writer, { messageId, text: imageMarkdownWithModel, prompt: requestTextContent, sourceImageUrls: requestTargetResolution?.targetImageUrls || [], }, "alibaba:image-canvas-media", ); } // 🆕 图片生成结束标记（成功） writeAssistantTextChunk( writer, "", "alibaba:image-end-success", ); if (contextPayload?.source === "canvas") { writeImageGenerationStatusChunk( writer, { phase: "completed", providerType: (languageModel as any).provider, modelName: usedModelDisplayName, imageCount: effectiveImageParams?.imageCount || defaultImageQualityParams.imageCount || 1, }, "alibaba:image-status-completed", ); } // 保存图片消息到数据库 try { await saveMessages({ messages: [ { id: messageId, chatId: id, role: "assistant", parts: [{ type: "text", text: imageMarkdownWithModel }], attachments: [], createdAt: new Date(), videoStorageStatus: null, temporaryVideoUrl: null, permanentVideoUrl: null, archivedAt: null, videoMetadata: null, }, ], }); console.log("[Chat API] ✅ Alibaba message saved successfully!"); // 提取用户 prompt const userPrompt = lastUserMessage?.parts?.find((p): p is { type: "text"; text: string } => p.type === "text")?.text || ""; // 保存图片记录到 GeneratedImage 表 await saveGeneratedImages( imageMarkdownWithModel, session.user.id, id, messageId, userPrompt, finalModelId, { sourceAssetIds: requestTargetResolution?.targetAssetIds, sourceImageUrls: requestTargetResolution?.targetImageUrls, relationType: requestResolvedMediaInput?.hasImage ? "edited_from" : "generated_from", sourceType: "generated", projectId: contextPayload?.projectId || null, optimizedPrompt: smartAutoOptimizedPrompt || userPrompt, requestedImageCount: effectiveImageParams?.imageCount, aspectRatio: effectiveImageParams?.aspectRatio, imageSize: effectiveImageParams?.imageSize, }, ); } catch (saveError) { console.error("[Chat API] ❌ Failed to save Alibaba message:", saveError); } } catch (error) { console.error("[Chat API] Alibaba generation error:", error); // 返回错误消息 const errorMessageId = generateUUID(); const errorText = error instanceof Error ? error.message : "图片生成失败，请稍后重试"; writeAssistantTextChunk( writer, errorText, "alibaba:error", errorMessageId, ); // 🆕 图片生成结束标记（失败） writeAssistantTextChunk( writer, "", "alibaba:image-end-failed", ); try { await saveMessages({ messages: [ { id: errorMessageId, chatId: id, role: "assistant", parts: [{ type: "text", text: errorText }], attachments: [], createdAt: new Date(), videoStorageStatus: null, temporaryVideoUrl: null, permanentVideoUrl: null, archivedAt: null, videoMetadata: null, }, ], }); } catch (saveError) { console.error("[Chat API] Failed to save error message:", saveError); } } finally { stopKeepalive?.(); } }, }); return createSseResponse(alibabaStream); } // ========== 阿里云图片生成处理结束 ========== // ========== 火山引擎视觉图片生成独立处理 ========== if ((languageModel as any).provider === "volcano-vision" && typeof (languageModel as any).doGenerate === "function") { console.log("[Chat API] Using Volcano Vision image generation"); // 🆕 检测是否有标注图片，如果有则在消息中注入系统提示 const hasAnnotation = hasAnnotatedImage(uiMessages); let messagesForVolcano = applyOptimizedPromptToLatestUserMessage( uiMessagesForModel, smartAutoOptimizedPrompt, hiddenPromptPrefix, ); const volcanoModelConfig = await getAiModelById({ id: finalModelId }); const configuredFallbacks = [ volcanoModelConfig?.fallbackModelId1, volcanoModelConfig?.fallbackModelId2, ].filter(Boolean) as string[]; const allModels = await getEnabledAiModelsWithProvider(); const fallbackModelIds = buildImageFallbackModelIds(allModels, { currentProviderId: volcanoModelConfig?.providerId, currentModelId: volcanoModelConfig?.id, currentModelKey: actualModelId, configuredFallbackIds: configuredFallbacks, }); console.log("[Chat API] Volcano fallback model IDs:", fallbackModelIds); if (hasAnnotation) { console.log("[Chat API] 🎨 Volcano Vision: 检测到标注图片，将注入系统提示"); // 创建消息副本，修改最后一条用户消息的文本 messagesForVolcano = messagesForVolcano.map((msg, idx) => { if (msg.role === "user" && idx === messagesForVolcano.length - 1) { // 修改最后一条用户消息 const modifiedParts = msg.parts.map((part) => { if (part.type === "text" && part.text) { return { ...part, text: ANNOTATION_SYSTEM_PROMPT + "\n\n用户请求：" + part.text, }; } return part; }); return { ...msg, parts: modifiedParts }; } return msg; }); } messagesForVolcano = injectHistoryImagesToLastUserMessage(messagesForVolcano, smartAutoHistoryImages); const volcanoStream = createUIMessageStream({ execute: async ({ writer }) => { let stopKeepalive: (() => void) | null = null; try { // 🆕 如果是智能模式且有对话回复，先发送对话消息让用户看到 const conversationMessage = smartAutoFriendlyMessage || null; if (conversationMessage) { const chatMessageId = generateUUID(); console.log("[Chat API] 💬 Volcano: Sending conversational reply first:", smartAutoFriendlyMessage); // 发送带标记的对话消息 writeAssistantTextChunk( writer, conversationMessage, "volcano:conversation", chatMessageId, ); // 保存对话消息到数据库（不带标记，纯对话） await saveMessages({ messages: [ { id: chatMessageId, chatId: id, role: "assistant", parts: [{ type: "text", text: smartAutoFriendlyMessage }], attachments: [], createdAt: new Date(), videoStorageStatus: null, temporaryVideoUrl: null, permanentVideoUrl: null, archivedAt: null, videoMetadata: null, }, ], }); } // 转换消息，火山引擎支持 URL 和 base64 const modelMessages = await convertMessagesWithImages(messagesForVolcano, "volcano"); const tryGenerateImage = async (modelIdToUse: string, isRetry: boolean) => { const result = await (languageModel as any).doGenerate({ messages: modelMessages, imageParams: effectiveImageParams || { aspectRatio: defaultImageQualityParams.aspectRatio, imageSize: defaultImageQualityParams.imageSize, imageCount: defaultImageQualityParams.imageCount, }, overrideModelId: isRetry ? modelIdToUse : undefined, }); if (result?.finishReason === "error") { throw new Error(result.text || "图片生成失败"); } return result.text || "图片生成失败"; }; let imageMarkdown = ""; let usedModelId = actualModelId; // 🆕 图片生成真正开始时发出标记（用于前端显示占位图） writeAssistantTextChunk( writer, "", "volcano:image-start", ); if (contextPayload?.source === "canvas") { writeImageGenerationStatusChunk( writer, { phase: "processing", providerType: "volcano-vision", modelName: volcanoModelConfig?.displayName || actualModelId, imageCount: effectiveImageParams?.imageCount || defaultImageQualityParams.imageCount || 1, }, "volcano:image-status-start", ); } stopKeepalive = startStreamKeepalive(writer, "volcano-image"); try { imageMarkdown = await tryGenerateImage(actualModelId, false); } catch (primaryError) { console.error("[Chat API] Volcano primary model failed:", primaryError); let fallbackSuccess = false; for (const fallbackModelId of fallbackModelIds) { try { console.log(`[Chat API] 🔄 Volcano fallback model: ${fallbackModelId}`); imageMarkdown = await tryGenerateImage(fallbackModelId, true); usedModelId = fallbackModelId; fallbackSuccess = true; console.log(`[Chat API] ✅ Volcano fallback model ${fallbackModelId} succeeded!`); break; } catch (fallbackError) { console.error(`[Chat API] ❌ Volcano fallback model ${fallbackModelId} failed:`, fallbackError); } } if (!fallbackSuccess) { const providerPriority: Record = { "alibaba-image": 0, "bfl-image": 1, "ruxa-image": 2, "fal-image": 3, "openai-image": 4, runway: 5, gemini: 5, "huanke-gemini": 6, "volcano-vision": 7, }; const crossProviderCandidates = getImageCrossProviderCandidates(allModels, { currentModelId: finalModelId, currentProviderId: volcanoModelConfig?.providerId, providerPriority, }); for (const candidate of crossProviderCandidates) { try { console.log( `[Chat API] 🔁 Volcano cross-provider failover attempt: ${candidate.model.displayName} (${candidate.provider.type})` ); const candidateLanguageModel = await getLanguageModelById(candidate.model.id); if (!candidateLanguageModel || typeof (candidateLanguageModel as any).doGenerate !== "function") { console.log(`[Chat API] Skip candidate without doGenerate: ${candidate.model.id}`); continue; } const convertProvider = candidate.provider.type === "bfl-image" ? "bfl-image" : candidate.provider.type === "ruxa-image" ? "ruxa-image" : candidate.provider.type === "fal-image" ? "fal-image" : candidate.provider.type === "openai-image" ? "openai-image" : candidate.provider.type === "midjourney" ? "midjourney" : candidate.provider.type === "runway" ? "runway" : candidate.provider.type === "gemini" ? "gemini" : candidate.provider.type === "huanke-gemini" ? "huanke-gemini" : candidate.provider.type === "volcano-vision" ? "volcano" : "alibaba-image"; const candidateMessages = await convertMessagesWithImages( messagesForVolcano, convertProvider ); const candidateResult = await (candidateLanguageModel as any).doGenerate({ messages: candidateMessages, imageParams: effectiveImageParams || { aspectRatio: defaultImageQualityParams.aspectRatio, imageSize: defaultImageQualityParams.imageSize, imageCount: defaultImageQualityParams.imageCount, }, modelDisplayName: candidate.model.displayName, overrideModelId: candidate.model.modelId, }); if (candidateResult?.finishReason === "error") { throw new Error(candidateResult.text || "图片生成失败"); } imageMarkdown = candidateResult.text || "图片生成失败"; usedModelId = candidate.model.modelId; fallbackSuccess = true; console.log( `[Chat API] ✅ Volcano cross-provider failover succeeded: ${candidate.model.displayName} (${candidate.provider.type})` ); break; } catch (crossError) { const crossErrorMessage = normalizeFailoverErrorMessage(crossError); if (shouldCooldownImageFallbackModel(crossErrorMessage)) { setImageFallbackModelCooldown(candidate.model.id, crossErrorMessage); } console.error( `[Chat API] ❌ Volcano cross-provider failover failed: ${candidate.model.displayName} (${candidate.provider.type})`, crossError ); } } } if (!fallbackSuccess) { throw primaryError; } } console.log("[Chat API] Volcano Vision generated, length:", imageMarkdown.length); console.log("[Chat API] Volcano Vision image markdown:", imageMarkdown.substring(0, 200)); const usedModelDisplayName = resolveModelDisplayNameByModelId(allModels, usedModelId) || volcanoModelConfig?.displayName || actualModelId; const imageMarkdownWithModel = prependImageModelBadge( imageMarkdown, usedModelDisplayName, ); // 生成消息 ID（图片消息） const messageId = generateUUID(); // 写入图片消息到流 writeAssistantTextChunk( writer, imageMarkdownWithModel, "volcano:image-result", messageId, ); if (contextPayload?.source === "canvas") { writeCanvasMediaChunk( writer, { messageId, text: imageMarkdownWithModel, prompt: requestTextContent, sourceImageUrls: requestTargetResolution?.targetImageUrls || [], }, "volcano:image-canvas-media", ); } // 🆕 图片生成结束标记（成功） writeAssistantTextChunk( writer, "", "volcano:image-end-success", ); if (contextPayload?.source === "canvas") { writeImageGenerationStatusChunk( writer, { phase: "completed", providerType: "volcano-vision", modelName: usedModelDisplayName, imageCount: effectiveImageParams?.imageCount || defaultImageQualityParams.imageCount || 1, }, "volcano:image-status-completed", ); } // 保存图片消息到数据库 try { await saveMessages({ messages: [ { id: messageId, chatId: id, role: "assistant", parts: [{ type: "text", text: imageMarkdownWithModel }], attachments: [], // 必须提供 attachments 字段（数据库 notNull） createdAt: new Date(), videoStorageStatus: null, temporaryVideoUrl: null, permanentVideoUrl: null, archivedAt: null, videoMetadata: null, }, ], }); console.log("[Chat API] ✅ Volcano Vision message saved successfully!"); // 提取用户 prompt const lastUserMessage = uiMessages.filter((m) => m.role === "user").pop(); const userPrompt = lastUserMessage?.parts?.find((p): p is { type: "text"; text: string } => p.type === "text")?.text || ""; // 保存图片记录到 GeneratedImage 表 await saveGeneratedImages( imageMarkdownWithModel, session.user.id, id, messageId, userPrompt, finalModelId, { sourceAssetIds: requestTargetResolution?.targetAssetIds, sourceImageUrls: requestTargetResolution?.targetImageUrls, relationType: requestResolvedMediaInput?.hasImage ? "edited_from" : "generated_from", sourceType: "generated", projectId: contextPayload?.projectId || null, optimizedPrompt: smartAutoOptimizedPrompt || userPrompt, requestedImageCount: effectiveImageParams?.imageCount, aspectRatio: effectiveImageParams?.aspectRatio, imageSize: effectiveImageParams?.imageSize, }, ); } catch (saveError) { console.error("[Chat API] ❌ Failed to save Volcano Vision message to database:", saveError); console.error("[Chat API] Message content that failed:", imageMarkdown.substring(0, 300)); } } catch (error) { console.error("[Chat API] Volcano Vision generation error:", error); const errorMessageId = generateUUID(); writeAssistantTextChunk( writer, "图片生成失败，请稍后重试", "volcano:error", errorMessageId, ); // 🆕 图片生成结束标记（失败） writeAssistantTextChunk( writer, "", "volcano:image-end-failed", ); // 尝试保存错误消息 try { await saveMessages({ messages: [ { id: errorMessageId, chatId: id, role: "assistant", parts: [{ type: "text", text: "图片生成失败，请稍后重试" }], attachments: [], // 必须提供 attachments 字段 createdAt: new Date(), videoStorageStatus: null, temporaryVideoUrl: null, permanentVideoUrl: null, archivedAt: null, videoMetadata: null, }, ], }); } catch (saveError) { console.error("[Chat API] Failed to save error message:", saveError); } } finally { stopKeepalive?.(); } }, }); return createSseResponse(volcanoStream); } // ========== 火山引擎视觉处理结束 ========== // ========== 千问/FAL/RUXA 视频生成独立处理 ========== if ( ["qianwen-video", "fal-video", "ruxa-video", "runway"].includes((languageModel as any).provider) && typeof (languageModel as any).doGenerate === "function" ) { const currentVideoProvider = (languageModel as any).provider as | "qianwen-video" | "fal-video" | "ruxa-video" | "runway"; console.log(`[Chat API] Using ${currentVideoProvider} video generation`); const videoGenerationStream = createUIMessageStream({ execute: async ({ writer }) => { try { const conversationMessage = smartAutoFriendlyMessage || null; if (conversationMessage) { const chatMessageId = generateUUID(); writer.write({ type: "data-appendMessage", data: JSON.stringify({ id: chatMessageId, role: "assistant", parts: [{ type: "text", text: conversationMessage }], }), }); await saveMessages({ messages: [ { id: chatMessageId, chatId: id, role: "assistant", parts: [{ type: "text", text: conversationMessage }], attachments: [], createdAt: new Date(), videoStorageStatus: null, temporaryVideoUrl: null, permanentVideoUrl: null, archivedAt: null, videoMetadata: null, }, ], }); } const messagesForVideo = injectHistoryImagesToLastUserMessage( applyOptimizedPromptToLatestUserMessage( uiMessagesForModel, smartAutoOptimizedPrompt, hiddenPromptPrefix, ), smartAutoHistoryImages, ); // 千问/FAL 视频直接使用原始 URL，不转换为 base64 const modelMessages = convertToModelMessages(messagesForVideo); console.log(`[Chat API] ${currentVideoProvider} messages prepared, count:`, modelMessages.length); let videoMarkdown = "视频生成失败"; let tempVideoUrl = ""; let billedModelRefs: string[] = [finalModelId]; let videoMetadata: Record = { provider: currentVideoProvider, model: finalModelId, mode: effectiveVideoParams?.mode, resolution: effectiveVideoParams?.resolution, ratio: effectiveVideoParams?.ratio, duration: effectiveVideoParams?.duration, status: "succeeded", updatedAt: Date.now(), }; let usedStoryboardPipeline = false; if (currentVideoProvider === "ruxa-video") { const storyboardConfig = await getVideoStoryboardConfig().catch(() => null); if (storyboardConfig?.enabled) { const storyboardModelCandidates = await resolveStoryboardModelCandidates(finalModelId); const runtimeModelId = String((languageModel as any).modelId || "").trim(); const selectedModelId = runtimeModelId || storyboardModelCandidates.find((item) => item.id === finalModelId)?.modelId || ""; const storyboardResult = await runRuxaNineGridPipeline({ userId: session.user.id, chatId: id, modelMessages: modelMessages as any, selectedModelId, selectedModelRef: finalModelId, modelCandidates: storyboardModelCandidates, videoParams: effectiveVideoParams as any, config: storyboardConfig, doGenerate: (input) => (languageModel as any).doGenerate({ messages: input.messages, videoParams: input.videoParams, overrideModelId: input.overrideModelId, }), onProgress: (event) => { safeWriteStreamChunk( writer, { type: "data-videoStoryboard", data: JSON.stringify(event), }, `ruxa-video:storyboard:${event.stage}`, ); }, }); if (storyboardResult.triggered) { usedStoryboardPipeline = true; videoMarkdown = storyboardResult.markdown || "视频生成失败，请稍后重试"; tempVideoUrl = storyboardResult.videoUrls?.[0] || ""; const isAwaitingConfirmation = storyboardResult.reason === "awaiting_user_confirmation"; billedModelRefs = storyboardResult.modelRefs && storyboardResult.modelRefs.length > 0 ? storyboardResult.modelRefs : [finalModelId]; const successCount = storyboardResult.videoUrls?.length || 0; videoMetadata = { ...videoMetadata, ...(storyboardResult.metadata || {}), status: isAwaitingConfirmation ? "awaiting_user_confirmation" : successCount > 0 ? "succeeded" : "failed", storyboard: { enabled: true, taskId: storyboardResult.taskId, confidence: storyboardResult.confidence, successCount, awaitingConfirmation: isAwaitingConfirmation, }, updatedAt: Date.now(), }; } } } if (!usedStoryboardPipeline) { const result = await (languageModel as any).doGenerate({ messages: modelMessages, videoParams: effectiveVideoParams, }); if (result?.finishReason === "error") { throw new Error(result.text || "视频生成失败"); } videoMarkdown = result.text || "视频生成失败"; tempVideoUrl = extractVideoUrlFromText(videoMarkdown) || ""; } console.log( `[Chat API] ${currentVideoProvider} video generated, length:`, videoMarkdown.length ); // 生成一个统一的消息 ID const messageId = generateUUID(); // 写入消息到流 writer.write({ type: "data-appendMessage", data: JSON.stringify({ id: messageId, role: "assistant", parts: [{ type: "text", text: videoMarkdown }], attachments: [], }), }); // 保存消息到数据库 try { await saveMessages({ messages: [ { id: messageId, chatId: id, role: "assistant", parts: [{ type: "text", text: videoMarkdown }], attachments: [], createdAt: new Date(), videoStorageStatus: tempVideoUrl ? "temporary" : null, temporaryVideoUrl: tempVideoUrl || null, permanentVideoUrl: null, archivedAt: null, videoMetadata: videoMetadata as any, }, ], }); const storyboardTaskId = String((videoMetadata as any)?.taskId || "").trim(); if (storyboardTaskId) { await attachVideoStoryboardTaskMessage({ taskId: storyboardTaskId, messageId, }).catch((taskAttachError) => { console.warn( "[Chat API] Failed to attach storyboard task with message:", taskAttachError ); }); } console.log(`[Chat API] ✅ ${currentVideoProvider} video message saved successfully!`); if ((videoMetadata as any)?.storyboard?.enabled) { try { const storyboardGridType = String((videoMetadata as any)?.gridType || "").trim(); const storyboardShotCount = Number((videoMetadata as any)?.shotCount || 0); await persistStoryboardAssistantTurn({ userId: session.user.id, chatId: id, projectId: contextPayload?.projectId || null, lastMessageId: messageId, text: requestTextContent, storyboardGridType: storyboardGridType === "grid_2x2" || storyboardGridType === "grid_3x2" || storyboardGridType === "grid_3x3" ? (storyboardGridType as "grid_2x2" | "grid_3x2" | "grid_3x3") : undefined, storyboardShotCount: storyboardShotCount === 4 || storyboardShotCount === 6 || storyboardShotCount === 9 ? storyboardShotCount : undefined, }); } catch (storyboardMemoryPersistError) { console.warn( "[Chat API] Failed to persist storyboard memory after assistant response:", storyboardMemoryPersistError ); } } else { try { await persistVideoGenerationCompletedTurn({ userId: session.user.id, chatId: id, projectId: contextPayload?.projectId || null, messageId, text: requestTextContent, source: "native_video_provider", providerType: currentVideoProvider, modelRef: finalModelId, mode: typeof effectiveVideoParams?.mode === "string" ? effectiveVideoParams.mode : undefined, taskId: storyboardTaskId || undefined, status: typeof (videoMetadata as any)?.status === "string" ? (videoMetadata as any).status : tempVideoUrl ? "completed" : undefined, temporaryVideoUrl: tempVideoUrl || undefined, }); } catch (videoWorkflowPersistError) { console.warn( "[Chat API] Failed to persist native video workflow state:", videoWorkflowPersistError, ); } } const shouldConsumeVideoPoints = tempVideoUrl && (videoMetadata as any)?.status !== "awaiting_user_confirmation"; if (shouldConsumeVideoPoints) { await consumeGeneratedVideoPoints({ userId: session.user.id, chatId: id, messageId, modelRefs: billedModelRefs, }); } if (tempVideoUrl) { scheduleVideoArchive({ chatId: id, messageId, text: videoMarkdown, temporaryVideoUrl: tempVideoUrl, taskId: messageId, metadata: videoMetadata, }); } } catch (saveError) { console.error(`[Chat API] ❌ Failed to save ${currentVideoProvider} video message:`, saveError); } } catch (error) { console.error(`[Chat API] ${currentVideoProvider} video generation error:`, error); const errorMessageId = generateUUID(); const errorText = "视频生成失败，请稍后重试"; writer.write({ type: "data-appendMessage", data: JSON.stringify({ id: errorMessageId, role: "assistant", parts: [{ type: "text", text: errorText }], attachments: [], }), }); try { await saveMessages({ messages: [ { id: errorMessageId, chatId: id, role: "assistant", parts: [{ type: "text", text: errorText }], attachments: [], createdAt: new Date(), videoStorageStatus: null, temporaryVideoUrl: null, permanentVideoUrl: null, archivedAt: null, videoMetadata: null, }, ], }); await persistVideoGenerationFailedTurn({ userId: session.user.id, chatId: id, projectId: contextPayload?.projectId || null, messageId: errorMessageId, text: requestTextContent, source: "native_video_provider", providerType: currentVideoProvider, modelRef: finalModelId, mode: typeof effectiveVideoParams?.mode === "string" ? effectiveVideoParams.mode : undefined, status: "failed", error: error instanceof Error ? error.message : String(error), }); } catch (videoFailurePersistError) { console.warn( "[Chat API] Failed to persist native video failure state:", videoFailurePersistError, ); } } }, }); return createSseResponse(videoGenerationStream); } // ========== 千问/FAL/RUXA 视频处理结束 ========== let stopStreamKeepalive: (() => void) | null = null; const stream = createUIMessageStream({ execute: async ({ writer: dataStream }) => { const stopDefaultKeepalive = startStreamKeepalive( dataStream, "chat-default", DEFAULT_STREAM_KEEPALIVE_INTERVAL_MS ); stopStreamKeepalive = stopDefaultKeepalive; setConversationStateProtocolEventObserver((event) => { writeProtocolDebugChunk( dataStream, Boolean(debugProtocol), { phase: "event", type: event.type, payload: event.payload, }, `protocol:event:${event.type}` ); }); safeWriteStreamChunk( dataStream, { type: "data-keepalive", data: JSON.stringify({ ts: Date.now(), label: "chat-default:init" }), }, "chat-default:init" ); writeProtocolDebugChunk( dataStream, Boolean(debugProtocol), { phase: "request", chatId: id, selectedChatModel, hasHiddenPrompt: Boolean(hiddenPrompt), hasAutoContext: Boolean(autoContext), }, "protocol:request" ); for (const event of pendingProtocolDebugEvents) { writeProtocolDebugChunk( dataStream, Boolean(debugProtocol), { phase: event.phase, ...event.payload, }, `protocol:${event.phase}` ); } try { // 构建工具对象，只包含启用的工具 const tools: any = {}; const activeTools: string[] = []; if (enabledTools.includes("getWeather")) { tools.getWeather = getWeather; activeTools.push("getWeather"); } if (enabledTools.includes("createDocument")) { tools.createDocument = createDocument({ session, dataStream }); activeTools.push("createDocument"); } if (enabledTools.includes("updateDocument")) { tools.updateDocument = updateDocument({ session, dataStream }); activeTools.push("updateDocument"); } if (enabledTools.includes("requestSuggestions")) { tools.requestSuggestions = requestSuggestions({ session, dataStream, }); activeTools.push("requestSuggestions"); } const shouldEnableGenerateVideoTool = enabledTools.includes("generateVideo") && (!shouldRunAgentConversation || allowAgentVideoGeneration); if (shouldEnableGenerateVideoTool) { tools.generateVideo = generateVideo({ dataStream }); activeTools.push("generateVideo"); } else if (enabledTools.includes("generateVideo") && shouldRunAgentConversation) { console.log("[Smart Auto Mode] 🚫 generateVideo tool disabled (manual video mode not selected)"); } console.log("[Chat API] Calling streamText with model:", (languageModel as any).modelId, (languageModel as any).provider); // 调试日志 console.log("[Chat API] Provider type:", (languageModel as any).provider); // 调试日志 // 转换消息，对于 Gemini 和千问视频将图片 URL 转换为 base64 const convertedMessages = await convertMessagesWithImages( injectHistoryImagesToLastUserMessage(uiMessagesForModel, smartAutoHistoryImages), (languageModel as any).provider || "" ); console.log("[Chat API] Messages converted, count:", convertedMessages.length); // 调试日志 // 🆕 检测是否是视频模型 const providerType = (languageModel as any).provider; const effectiveProviderType = resolvedProviderType || providerType; console.log("[Chat API] Provider type for tool detection:", { runtime: providerType, resolved: resolvedProviderType, effective: effectiveProviderType, }); const isVolcanoVideoModel = effectiveProviderType === "volcano-video" || effectiveProviderType === "doubao-video" || actualModelId.includes("seedance"); const isOpenAICompatImageModel = resolvedModelSupportsImageGen && providerType === "openai.chat" && !isVolcanoVideoModel; console.log("[Chat API] Is volcano video model:", isVolcanoVideoModel); // 🆕 处理 OpenAI 兼容的生图模型输出（防止返回 data:image;base64 原文） if (isOpenAICompatImageModel) { console.log("[Chat API] Using generateText for OpenAI-compatible image model"); const configuredFallbacks = [ currentModelConfig?.fallbackModelId1, currentModelConfig?.fallbackModelId2, ].filter(Boolean) as string[]; const allModels = await getEnabledAiModelsWithProvider(); const fallbackModelIds = buildImageFallbackModelIds(allModels, { currentProviderId: currentModelConfig?.providerId, currentModelId: currentModelConfig?.id, currentModelKey: actualModelId, configuredFallbackIds: configuredFallbacks, }); console.log("[Chat API] OpenAI-compatible fallback model IDs:", fallbackModelIds); const lastUserMessage = uiMessagesForModel.filter((m) => m.role === "user").pop(); let messagesForCrossProvider = lastUserMessage ? [lastUserMessage] : uiMessagesForModel; messagesForCrossProvider = applyOptimizedPromptToLatestUserMessage( messagesForCrossProvider, smartAutoOptimizedPrompt, hiddenPromptPrefix, ); messagesForCrossProvider = injectHistoryImagesToLastUserMessage( messagesForCrossProvider, smartAutoHistoryImages ); // 生成开始标记 dataStream.write({ type: "data-appendMessage", data: JSON.stringify({ id: generateUUID(), role: "assistant", parts: [{ type: "text", text: "" }], }), }); if (contextPayload?.source === "canvas") { writeImageGenerationStatusChunk( dataStream as unknown as StreamWriter, { phase: "processing", providerType: currentModelConfig?.providerId ? resolvedProviderType || providerType : providerType, modelName: currentModelConfig?.displayName || actualModelId, imageCount: effectiveImageParams?.imageCount || 1, }, "openai-compat:image-status-start", ); } try { const tryGenerateWithOpenAICompatModel = async (modelToUse: any) => { const imageRequestTimeoutMs = 180_000; const generated = await generateText({ model: modelToUse, system: systemPrompt({ selectedChatModel: finalModelId, requestHints }), messages: convertedMessages, abortSignal: AbortSignal.timeout(imageRequestTimeoutMs), }); const rawText = generated.text?.trim() || "图片生成失败，请稍后重试"; return normalizeAssistantImageMarkdown( rawText, effectiveImageParams?.imageSize || "2K" ); }; let normalizedText = ""; let usedModelId = actualModelId; try { normalizedText = await tryGenerateWithOpenAICompatModel(languageModel); } catch (primaryError) { console.error("[Chat API] OpenAI-compatible primary model failed:", primaryError); let fallbackSuccess = false; for (const fallbackModelId of fallbackModelIds) { try { console.log( `[Chat API] 🔄 OpenAI-compatible fallback model: ${fallbackModelId}` ); const fallbackLanguageModel = await getLanguageModelById( finalModelId, fallbackModelId ); if (!fallbackLanguageModel) { console.log( `[Chat API] Skip unavailable OpenAI-compatible fallback model: ${fallbackModelId}` ); continue; } normalizedText = await tryGenerateWithOpenAICompatModel(fallbackLanguageModel); usedModelId = fallbackModelId; fallbackSuccess = true; console.log( `[Chat API] ✅ OpenAI-compatible fallback model ${fallbackModelId} succeeded!` ); break; } catch (fallbackError) { console.error( `[Chat API] ❌ OpenAI-compatible fallback model ${fallbackModelId} failed:`, fallbackError ); } } if (!fallbackSuccess) { const providerPriority: Record = { "volcano-vision": 0, "alibaba-image": 1, "bfl-image": 2, "ruxa-image": 3, "fal-image": 4, "openai-image": 5, runway: 6, gemini: 6, "huanke-gemini": 7, }; const crossProviderCandidates = getImageCrossProviderCandidates(allModels, { currentModelId: finalModelId, currentProviderId: currentModelConfig?.providerId, providerPriority, }); for (const candidate of crossProviderCandidates) { try { console.log( `[Chat API] 🔁 OpenAI-compatible cross-provider failover: ${candidate.model.displayName} (${candidate.provider.type})` ); const candidateLanguageModel = await getLanguageModelById(candidate.model.id); if ( !candidateLanguageModel || typeof (candidateLanguageModel as any).doGenerate !== "function" ) { console.log( `[Chat API] Skip cross-provider candidate without doGenerate: ${candidate.model.id}` ); continue; } const convertProvider = candidate.provider.type === "bfl-image" ? "bfl-image" : candidate.provider.type === "ruxa-image" ? "ruxa-image" : candidate.provider.type === "fal-image" ? "fal-image" : candidate.provider.type === "openai-image" ? "openai-image" : candidate.provider.type === "midjourney" ? "midjourney" : candidate.provider.type === "runway" ? "runway" : candidate.provider.type === "gemini" ? "gemini" : candidate.provider.type === "huanke-gemini" ? "huanke-gemini" : candidate.provider.type === "volcano-vision" ? "volcano" : "alibaba-image"; const candidateMessages = await convertMessagesWithImages( messagesForCrossProvider, convertProvider ); const candidateResult = await (candidateLanguageModel as any).doGenerate({ messages: candidateMessages, imageParams: effectiveImageParams || { aspectRatio: defaultImageQualityParams.aspectRatio, imageSize: defaultImageQualityParams.imageSize, imageCount: defaultImageQualityParams.imageCount, }, modelDisplayName: candidate.model.displayName, overrideModelId: candidate.model.modelId, }); if (candidateResult?.finishReason === "error") { throw new Error(candidateResult.text || "图片生成失败"); } normalizedText = candidateResult.text || "图片生成失败"; usedModelId = candidate.model.modelId; fallbackSuccess = true; console.log( `[Chat API] ✅ OpenAI-compatible cross-provider failover succeeded: ${candidate.model.displayName} (${candidate.provider.type})` ); break; } catch (crossError) { const crossErrorMessage = normalizeFailoverErrorMessage(crossError); if (shouldCooldownImageFallbackModel(crossErrorMessage)) { setImageFallbackModelCooldown(candidate.model.id, crossErrorMessage); } console.error( `[Chat API] ❌ OpenAI-compatible cross-provider failover failed: ${candidate.model.displayName} (${candidate.provider.type})`, crossError ); } } } if (!fallbackSuccess) { throw primaryError; } } const usedModelDisplayName = resolveModelDisplayNameByModelId(allModels, usedModelId) || currentModelConfig?.displayName || actualModelId; const normalizedTextWithModel = prependImageModelBadge( normalizedText, usedModelDisplayName, ); const messageId = generateUUID(); dataStream.write({ type: "data-appendMessage", data: JSON.stringify({ id: messageId, role: "assistant", parts: [{ type: "text", text: normalizedTextWithModel }], }), }); if (contextPayload?.source === "canvas") { writeCanvasMediaChunk( dataStream as unknown as StreamWriter, { messageId, text: normalizedTextWithModel, prompt: requestTextContent, sourceImageUrls: requestTargetResolution?.targetImageUrls || [], }, "openai-compat:image-canvas-media", ); } // 生成结束标记 dataStream.write({ type: "data-appendMessage", data: JSON.stringify({ id: generateUUID(), role: "assistant", parts: [{ type: "text", text: "" }], }), }); if (contextPayload?.source === "canvas") { writeImageGenerationStatusChunk( dataStream as unknown as StreamWriter, { phase: "completed", providerType: resolvedProviderType || providerType, modelName: usedModelDisplayName, imageCount: effectiveImageParams?.imageCount || 1, }, "openai-compat:image-status-completed", ); } await saveMessages({ messages: [ { id: messageId, role: "assistant", parts: [{ type: "text", text: normalizedTextWithModel }], createdAt: new Date(), attachments: [], chatId: id, videoStorageStatus: null, temporaryVideoUrl: null, permanentVideoUrl: null, archivedAt: null, videoMetadata: null, }, ], }); const latestUserMessage = uiMessages.filter((m) => m.role === "user").pop(); const userPrompt = latestUserMessage?.parts?.find( (p): p is { type: "text"; text: string } => p.type === "text" )?.text || ""; await saveGeneratedImages( normalizedTextWithModel, session.user.id, id, messageId, userPrompt, finalModelId, { sourceAssetIds: requestTargetResolution?.targetAssetIds, sourceImageUrls: requestTargetResolution?.targetImageUrls, relationType: requestResolvedMediaInput?.hasImage ? "edited_from" : "generated_from", sourceType: "generated", projectId: contextPayload?.projectId || null, optimizedPrompt: smartAutoOptimizedPrompt || userPrompt, requestedImageCount: effectiveImageParams?.imageCount, aspectRatio: effectiveImageParams?.aspectRatio, imageSize: effectiveImageParams?.imageSize, }, ); return; } catch (error) { console.error("[Chat API] OpenAI-compatible image model generation error:", error); const errorMessageId = generateUUID(); const errorText = error instanceof Error ? error.message : "图片生成失败，请稍后重试"; dataStream.write({ type: "data-appendMessage", data: JSON.stringify({ id: errorMessageId, role: "assistant", parts: [{ type: "text", text: errorText }], }), }); dataStream.write({ type: "data-appendMessage", data: JSON.stringify({ id: generateUUID(), role: "assistant", parts: [{ type: "text", text: "" }], }), }); if (contextPayload?.source === "canvas") { writeImageGenerationStatusChunk( dataStream as unknown as StreamWriter, { phase: "failed", providerType: resolvedProviderType || providerType, modelName: currentModelConfig?.displayName || actualModelId, imageCount: effectiveImageParams?.imageCount || 1, }, "openai-compat:image-status-failed", ); } await saveMessages({ messages: [ { id: errorMessageId, role: "assistant", parts: [{ type: "text", text: errorText }], createdAt: new Date(), attachments: [], chatId: id, videoStorageStatus: null, temporaryVideoUrl: null, permanentVideoUrl: null, archivedAt: null, videoMetadata: null, }, ], }); return; } } // 🎬 视频模型使用专门的生成流程 if (isVolcanoVideoModel) { console.log("[Chat API] Using video generation API"); // 导入视频生成函数 const { doGenerateStream } = await import("@/lib/ai/providers/volcano-video"); const conversationMessage = smartAutoFriendlyMessage || null; if (conversationMessage) { const chatMessageId = generateUUID(); dataStream.write({ type: "data-appendMessage", data: JSON.stringify({ id: chatMessageId, role: "assistant", parts: [{ type: "text", text: conversationMessage }], attachments: [], }), }); await saveMessages({ messages: [ { id: chatMessageId, chatId: id, role: "assistant", parts: [{ type: "text", text: conversationMessage }], attachments: [], createdAt: new Date(), videoStorageStatus: null, temporaryVideoUrl: null, permanentVideoUrl: null, archivedAt: null, videoMetadata: null, }, ], }); } const messagesForVideo = injectHistoryImagesToLastUserMessage( applyOptimizedPromptToLatestUserMessage( uiMessagesForModel, smartAutoOptimizedPrompt, hiddenPromptPrefix, ), smartAutoHistoryImages, ); // 从最后一条用户消息中提取提示词和图片 const lastUserMessage = messagesForVideo.filter(m => m.role === "user").pop(); let prompt = ""; const imageUrls: string[] = []; const summarizeImageRef = (rawUrl: string) => { const trimmed = String(rawUrl || "").trim(); if (!trimmed) return ""; if (trimmed.startsWith("data:image/")) { return `data:image(base64,${trimmed.length} chars)`; } try { const parsed = new URL(trimmed); return `${parsed.origin}${parsed.pathname}`; } catch { return trimmed.slice(0, 120); } }; if (lastUserMessage) { // 🆕 处理 parts 数组（AI SDK v4 格式） if ((lastUserMessage as any).parts && Array.isArray((lastUserMessage as any).parts)) { console.log("[Chat API] Video input parts:", (lastUserMessage as any).parts.length); for (const part of (lastUserMessage as any).parts) { if (part.type === "text" && part.text) { // 提取文本 prompt += part.text; } else if (part.type === "file" && part.url && part.mediaType?.startsWith("image/")) { // 提取图片 imageUrls.push(part.url); } } } // 备用方案1：content 字符串/数组 const legacyContent = (lastUserMessage as any).content; if (typeof legacyContent === "string") { prompt = legacyContent; } else if (Array.isArray(legacyContent)) { for (const part of legacyContent) { if (typeof part === "string") { prompt += part; } else if (part.type === "text") { prompt += part.text; } else if (part.type === "image" && (part as any).image) { imageUrls.push((part as any).image); } } } else if ((lastUserMessage as any).experimental_attachments) { // 备用方案2：experimental_attachments for (const attachment of (lastUserMessage as any).experimental_attachments) { if (attachment.url && attachment.contentType?.startsWith("image/")) { imageUrls.push(attachment.url); } } } } console.log("[Chat API] Video input extracted:", { promptLength: prompt.trim().length, imageCount: imageUrls.length, imageRefs: imageUrls.slice(0, 4).map(summarizeImageRef), }); const videoMessageId = generateUUID(); const videoStartAt = Date.now(); const baseVideoMetadata: Record = { provider: effectiveProviderType || "volcano-video", mode: effectiveVideoParams?.mode || "text-to-video", resolution: effectiveVideoParams?.resolution || "720p", ratio: effectiveVideoParams?.ratio || "16:9", duration: effectiveVideoParams?.duration || 5, status: "queued", startedAt: videoStartAt, updatedAt: videoStartAt, }; try { console.log("[Chat API] Video generation params:", { promptLength: prompt.trim().length, imageCount: imageUrls.length, videoParams: effectiveVideoParams, }); const videoMode = (effectiveVideoParams?.mode || "text-to-video") as VideoModeType; // 🆕 重新执行子模型选择逻辑，确定要使用的模型 UUID let videoModelId = finalModelId; // 默认使用选择的模型 ID // 检查是否是主模型，需要选择子模型（仅 volcano-video 支持子模型） const selectedModelData = await getAiModelById({ id: finalModelId }); const selectedModelProvider = selectedModelData ? await getAiProviderById({ id: selectedModelData.providerId }).catch(() => null) : null; if ( selectedModelData && !selectedModelData.isSubModel && selectedModelProvider?.type === "volcano-video" ) { const videoSubModels = await getSubModelsByParentModelId(finalModelId); console.log(`[Chat API] Video: Found ${videoSubModels.length} sub-models for master model`); if (videoSubModels.length > 0) { const matchedModel = selectBestVideoSubModelForMode(videoSubModels as any, videoMode); if (matchedModel) { videoModelId = matchedModel.model.id; console.log( `[Chat API] Video: Selected sub-model ${matchedModel.model.displayName} (${matchedModel.model.modelId}) for mode ${videoMode}` ); } else { console.log( `[Chat API] Video: No compatible sub-model found for mode ${videoMode}, keep current model` ); } } } console.log("[Chat API] Getting model config for:", videoModelId); const aiModelData = await getAiModelById({ id: videoModelId }); if (!aiModelData) { throw new Error("视频模型配置未找到"); } // 若用户手动选了不兼容模式的子模型，自动回退到同父模型兼容子模型 const selectedSupportsMode = getSupportedVideoModesForModel(aiModelData as any).includes(videoMode); if (!selectedSupportsMode) { let fallbackModel: { model: any } | null = null; if (aiModelData.parentModelId) { const siblingSubModels = await getSubModelsByParentModelId(aiModelData.parentModelId); fallbackModel = selectBestVideoSubModelForMode(siblingSubModels as any, videoMode); } else if (selectedModelProvider?.type === "volcano-video") { const enabledModels = await getEnabledAiModelsWithProvider(); const sameProviderModels = enabledModels .filter( ({ model, provider }) => provider.type === "volcano-video" && model.isEnabled && model.id !== aiModelData.id, ) .map(({ model }) => ({ model })); fallbackModel = selectBestVideoSubModelForMode(sameProviderModels as any, videoMode); } if (fallbackModel) { console.warn( `[Chat API] Video: Model ${aiModelData.modelId} does not support ${videoMode}, fallback to ${fallbackModel.model.modelId}` ); videoModelId = fallbackModel.model.id; } } const finalVideoModelData = videoModelId === aiModelData.id ? aiModelData : await getAiModelById({ id: videoModelId }); if (!finalVideoModelData) { throw new Error("视频模型配置未找到"); } // 如果是子模型，获取父模型的 Provider 配置 let providerModelData = finalVideoModelData; if (finalVideoModelData.isSubModel && finalVideoModelData.parentModelId) { console.log( "[Chat API] This is a sub-model, getting parent model config:", finalVideoModelData.parentModelId ); const parentModel = await getAiModelById({ id: finalVideoModelData.parentModelId }); if (parentModel) { providerModelData = parentModel; } } console.log("[Chat API] Getting provider config for:", providerModelData.providerId); const providerData = await getAiProviderById({ id: providerModelData.providerId }); if (!providerData) { throw new Error("Provider 配置未找到"); } const actualVideoModelId = finalVideoModelData.modelId; // 构建视频生成选项 const videoOptions: any = { model: actualVideoModelId, prompt: prompt || "生成视频", mode: videoMode, resolution: effectiveVideoParams?.resolution || "720p", ratio: effectiveVideoParams?.ratio || "16:9", duration: effectiveVideoParams?.duration || 5, fps: effectiveVideoParams?.fps || 24, cameraMove: effectiveVideoParams?.cameraMove || "none", cameraFixed: effectiveVideoParams?.cameraFixed || false, watermark: effectiveVideoParams?.watermark || false, seed: effectiveVideoParams?.seed, }; // 根据模式添加图片 if (videoOptions.mode === "first-frame" && imageUrls[0]) { videoOptions.firstFrameImage = imageUrls[0]; } else if (videoOptions.mode === "first-last-frame" && imageUrls.length >= 2) { videoOptions.firstFrameImage = imageUrls[0]; videoOptions.lastFrameImage = imageUrls[1]; } else if (videoOptions.mode === "reference-images" && imageUrls.length > 0) { videoOptions.referenceImages = imageUrls; } console.log("[Chat API] Video options constructed:", { model: videoOptions.model, mode: videoOptions.mode, resolution: videoOptions.resolution, ratio: videoOptions.ratio, duration: videoOptions.duration, imageCount: videoOptions.referenceImages?.length || Number(Boolean(videoOptions.firstFrameImage)) + Number(Boolean(videoOptions.lastFrameImage)), }); // 先写入占位消息，避免刷新后消息丢失 const pendingText = "🎬 视频任务已创建，正在处理中，请稍候..."; await saveMessages({ messages: [ { id: videoMessageId, chatId: id, role: "assistant", parts: [{ type: "text", text: pendingText }], attachments: [], createdAt: new Date(), videoStorageStatus: "temporary", temporaryVideoUrl: null, permanentVideoUrl: null, archivedAt: null, videoMetadata: { ...baseVideoMetadata, model: actualVideoModelId, status: "running", updatedAt: Date.now(), } as any, }, ], }); try { await persistVideoGenerationStartedTurn({ userId: session.user.id, chatId: id, projectId: contextPayload?.projectId || null, messageId: videoMessageId, text: requestTextContent, source: "stream_video_provider", providerType: String(providerData.type || "stream"), modelRef: finalVideoModelData.id || finalModelId, mode: videoOptions.mode, status: "running", }); } catch (videoWorkflowStartError) { console.warn( "[Chat API] Failed to persist stream video start state:", videoWorkflowStartError, ); } dataStream.write({ type: "data-appendMessage", data: JSON.stringify({ id: videoMessageId, role: "assistant", parts: [{ type: "text", text: pendingText }], attachments: [], }), }); const apiKey = providerData.apiKey; const baseURL = providerData.baseUrl || "https://ark.cn-beijing.volces.com/api/v3"; console.log("[Chat API] Provider configured:", { baseURL, hasApiKey: !!apiKey }); // 调用视频生成流 console.log("[Chat API] Starting video generation stream..."); const videoStream = doGenerateStream(apiKey, baseURL, videoOptions); // 收集所有流输出 console.log("[Chat API] Processing video stream..."); let fullText = ""; let chunkCount = 0; for await (const chunk of videoStream) { chunkCount++; fullText += chunk; console.log(`[Chat API] Received chunk ${chunkCount}, length: ${chunk.length}`); } console.log( `[Chat API] Video stream completed. Total chunks: ${chunkCount}, total text length: ${fullText.length}` ); // 提取流中的元数据并清理输出文案 const metadataMatch = fullText.match(//s); let streamMetadata: Record = {}; if (metadataMatch?.[1]) { try { streamMetadata = JSON.parse(metadataMatch[1]); } catch (parseError) { console.warn("[Chat API] Failed to parse VIDEO_METADATA from stream", parseError); } } const cleanedText = fullText .replace(//gs, "") .trim(); const videoUrlMatch = cleanedText.match(/]+src="([^"]+)"/); const temporaryVideoUrl = (typeof streamMetadata.temporaryVideoUrl === "string" && streamMetadata.temporaryVideoUrl) || (videoUrlMatch ? videoUrlMatch[1] : null); const successMetadata = { ...baseVideoMetadata, ...streamMetadata, model: actualVideoModelId, taskId: typeof streamMetadata.taskId === "string" && streamMetadata.taskId ? streamMetadata.taskId : undefined, status: "succeeded", completedAt: Date.now(), updatedAt: Date.now(), }; await updateMessageById({ messageId: videoMessageId, chatId: id, parts: [{ type: "text", text: cleanedText }], videoStorageStatus: temporaryVideoUrl ? "temporary" : null, temporaryVideoUrl: temporaryVideoUrl, permanentVideoUrl: null, archivedAt: null, videoMetadata: successMetadata as any, }); try { await persistVideoGenerationCompletedTurn({ userId: session.user.id, chatId: id, projectId: contextPayload?.projectId || null, messageId: videoMessageId, text: requestTextContent, source: "stream_video_provider", providerType: String(providerData.type || "stream"), modelRef: finalVideoModelData.id || finalModelId, mode: videoOptions.mode, taskId: typeof successMetadata.taskId === "string" && successMetadata.taskId ? successMetadata.taskId : undefined, status: "completed", temporaryVideoUrl: temporaryVideoUrl || undefined, }); } catch (videoWorkflowCompletedError) { console.warn( "[Chat API] Failed to persist stream video completion state:", videoWorkflowCompletedError, ); } await consumeGeneratedVideoPoints({ userId: session.user.id, chatId: id, messageId: videoMessageId, modelRefs: [finalVideoModelData.id, providerModelData.id, finalModelId], }); dataStream.write({ type: "data-appendMessage", data: JSON.stringify({ id: videoMessageId, role: "assistant", parts: [{ type: "text", text: cleanedText }], attachments: [], }), }); console.log("[Chat API] ✅ Video message updated successfully"); if (temporaryVideoUrl) { console.log("[Chat API] 📹 临时视频URL已保存，开始异步归档到S3"); scheduleVideoArchive({ chatId: id, messageId: videoMessageId, text: cleanedText, temporaryVideoUrl, taskId: typeof successMetadata.taskId === "string" && successMetadata.taskId ? successMetadata.taskId : videoMessageId, metadata: successMetadata, }); } console.log("[Chat API] Video generation finished successfully"); return; } catch (error: any) { console.error("[Chat API] ❌ Video generation error:", error); console.error("[Chat API] Error stack:", error.stack); const errorMessage = `❌ 视频生成失败：${error.message}\n\n请检查配置或稍后重试。`; const failedMetadata = { ...baseVideoMetadata, status: "failed", error: error?.message || "未知错误", updatedAt: Date.now(), }; try { await updateMessageById({ messageId: videoMessageId, chatId: id, parts: [{ type: "text", text: errorMessage }], videoStorageStatus: null, temporaryVideoUrl: null, permanentVideoUrl: null, archivedAt: null, videoMetadata: failedMetadata as any, }); try { await persistVideoGenerationFailedTurn({ userId: session.user.id, chatId: id, projectId: contextPayload?.projectId || null, messageId: videoMessageId, text: requestTextContent, source: "stream_video_provider", providerType: "stream", modelRef: finalModelId, mode: typeof effectiveVideoParams?.mode === "string" ? effectiveVideoParams.mode : undefined, status: "failed", error: error?.message || "未知错误", }); } catch (videoWorkflowFailedError) { console.warn( "[Chat API] Failed to persist stream video failure state:", videoWorkflowFailedError, ); } dataStream.write({ type: "data-appendMessage", data: JSON.stringify({ id: videoMessageId, role: "assistant", parts: [{ type: "text", text: errorMessage }], attachments: [], }), }); } catch (updateError) { console.error("[Chat API] Failed to update placeholder video message:", updateError); const fallbackMessageId = generateUUID(); dataStream.write({ type: "data-appendMessage", data: JSON.stringify({ id: fallbackMessageId, role: "assistant", parts: [{ type: "text", text: errorMessage }], attachments: [], }), }); await saveMessages({ messages: [ { id: fallbackMessageId, chatId: id, role: "assistant", parts: [{ type: "text", text: errorMessage }], attachments: [], createdAt: new Date(), videoStorageStatus: null, temporaryVideoUrl: null, permanentVideoUrl: null, archivedAt: null, videoMetadata: failedMetadata as any, }, ], }); try { await persistVideoGenerationFailedTurn({ userId: session.user.id, chatId: id, projectId: contextPayload?.projectId || null, messageId: fallbackMessageId, text: requestTextContent, source: "stream_video_provider", providerType: "stream", modelRef: finalModelId, mode: typeof effectiveVideoParams?.mode === "string" ? effectiveVideoParams.mode : undefined, status: "failed", error: error?.message || "未知错误", }); } catch (videoWorkflowFailedError) { console.warn( "[Chat API] Failed to persist fallback stream video failure state:", videoWorkflowFailedError, ); } } return; } } // 常规文本模型使用 streamText const streamRequestTimeoutMs = resolvedModelSupportsImageGen ? 180_000 : 90_000; const shouldDisableSmoothStream = resolvedModelSupportsImageGen || isVolcanoVideoModel; const result = streamText({ model: languageModel, system: effectiveConversationSystemPrompt, messages: convertedMessages, stopWhen: stepCountIs(5), experimental_activeTools: finalModelId === "chat-model-reasoning" || isVolcanoVideoModel ? [] : activeTools, experimental_transform: shouldDisableSmoothStream ? undefined : smoothStream({ chunking: "word" }), tools: isVolcanoVideoModel ? {} : tools, // 🆕 视频模型禁用工具 abortSignal: AbortSignal.timeout(streamRequestTimeoutMs), experimental_telemetry: { isEnabled: isProductionEnvironment, functionId: "stream-text", }, onFinish: async ({ usage }) => { if (activeRuntimeModelDescriptor?.providerKey) { recordProviderHealthSuccess({ providerKey: activeRuntimeModelDescriptor.providerKey, providerType: activeRuntimeModelDescriptor.providerType, }); } try { const providers = await getTokenlensCatalog(); if (!actualModelId) { finalMergedUsage = usage; dataStream.write({ type: "data-usage", data: finalMergedUsage, }); return; } if (!providers) { finalMergedUsage = usage; dataStream.write({ type: "data-usage", data: finalMergedUsage, }); return; } const summary = getUsage({ modelId: actualModelId, usage, providers, }); finalMergedUsage = { ...usage, ...summary, modelId: actualModelId, } as AppUsage; dataStream.write({ type: "data-usage", data: finalMergedUsage }); } catch (err) { console.warn("TokenLens enrichment failed", err); finalMergedUsage = usage; dataStream.write({ type: "data-usage", data: finalMergedUsage }); } }, }); result.consumeStream({ onError: (error) => { console.error("[Chat API] streamText consume error:", error); if (activeRuntimeModelDescriptor?.providerKey) { recordProviderHealthFailure({ providerKey: activeRuntimeModelDescriptor.providerKey, providerType: activeRuntimeModelDescriptor.providerType, error, }); } }, }); dataStream.merge( result.toUIMessageStream({ sendReasoning: true, onError: (error) => { console.error("[Chat API] streamText ui stream error:", error); if (activeRuntimeModelDescriptor?.providerKey) { recordProviderHealthFailure({ providerKey: activeRuntimeModelDescriptor.providerKey, providerType: activeRuntimeModelDescriptor.providerType, error, }); } return getChatStreamErrorMessage(error); }, }) ); } finally { setConversationStateProtocolEventObserver(null); stopStreamKeepalive = stopDefaultKeepalive; } }, generateId: generateUUID, onFinish: async ({ messages }) => { stopStreamKeepalive?.(); stopStreamKeepalive = null; const messagesToPersist: Array<{ id: string; role: string; parts: unknown; createdAt: Date; attachments: unknown; chatId: string; videoStorageStatus: null; temporaryVideoUrl: null; permanentVideoUrl: null; archivedAt: null; videoMetadata: null; }> = []; for (const currentMessage of messages) { const normalizedParts = currentMessage.role === "assistant" ? normalizeAssistantPartsForPersistence( currentMessage.parts as Array> | undefined, ) : currentMessage.parts; if ( currentMessage.role === "assistant" && (!Array.isArray(normalizedParts) || normalizedParts.length === 0) ) { continue; } messagesToPersist.push({ id: currentMessage.id, role: currentMessage.role, parts: normalizedParts, createdAt: new Date(), attachments: [], chatId: id, videoStorageStatus: null, temporaryVideoUrl: null, permanentVideoUrl: null, archivedAt: null, videoMetadata: null, }); } if (messagesToPersist.length > 0) { await saveMessages({ messages: messagesToPersist, }); } // 检测助手消息中是否包含视频，如果有则立即标记并归档到永久存储 for (const msg of messages) { if (msg.role === "assistant" && msg.parts) { const textContent = msg.parts .filter((part: any) => part && part.type === "text" && part.text) .map((part: any) => part.text) .join(""); const tempVideoUrl = extractVideoUrlFromText(textContent); if (tempVideoUrl) { const genericVideoMetadata = { provider: "stream", model: actualModelId, status: "succeeded", updatedAt: Date.now(), }; await updateMessageById({ messageId: msg.id, chatId: id, videoStorageStatus: "temporary", temporaryVideoUrl: tempVideoUrl, permanentVideoUrl: null, archivedAt: null, videoMetadata: genericVideoMetadata as any, }); try { await persistVideoGenerationCompletedTurn({ userId: session.user.id, chatId: id, projectId: contextPayload?.projectId || null, messageId: msg.id, text: requestTextContent, source: "chat_stream_output", providerType: "stream", modelRef: actualModelId, status: "completed", temporaryVideoUrl: tempVideoUrl, }); } catch (videoWorkflowCompletedError) { console.warn( "[Chat API] Failed to persist streamed assistant video state:", videoWorkflowCompletedError, ); } scheduleVideoArchive({ chatId: id, messageId: msg.id, text: textContent, temporaryVideoUrl: tempVideoUrl, taskId: msg.id, metadata: genericVideoMetadata, }); } } } if (finalMergedUsage) { try { await updateChatLastContextById({ chatId: id, context: finalMergedUsage, }); } catch (err) { console.warn("Unable to persist last usage for chat", id, err); } } }, onError: () => { stopStreamKeepalive?.(); stopStreamKeepalive = null; return "Oops, an error occurred!"; }, }); // const streamContext = getStreamContext(); // if (streamContext) { // return new Response( // await streamContext.resumableStream(streamId, () => // stream.pipeThrough(new JsonToSseTransformStream()) // ) // ); // } return createSseResponse(stream); } catch (error) { const vercelId = request.headers.get("x-vercel-id"); if (error instanceof ChatSDKError) { return error.toResponse(); } // Check for Vercel AI Gateway credit card error if ( error instanceof Error && error.message?.includes( "AI Gateway requires a valid credit card on file to service requests" ) ) { return new ChatSDKError("bad_request:activate_gateway").toResponse(); } console.error("Unhandled error in chat API:", error, { vercelId }); return new ChatSDKError("offline:chat").toResponse(); } finally { if (requestDedupKey && acquiredInFlightLock) { inFlightChatRequests.delete(requestDedupKey); } } } export async function DELETE(request: Request) { const { searchParams } = new URL(request.url); const id = searchParams.get("id"); if (!id) { return new ChatSDKError("bad_request:api").toResponse(); } const session = await auth(); if (!session?.user) { return new ChatSDKError("unauthorized:chat").toResponse(); } const chat = await getChatById({ id }); if (chat?.userId !== session.user.id) { return new ChatSDKError("forbidden:chat").toResponse(); } const deletedChat = await deleteChatById({ id }); return Response.json(deletedChat, { status: 200 }); }