diff --git a/app/(chat)/api/chat/route.ts b/app/(chat)/api/chat/route.ts index 6b9c673e8..2161501e5 100644 --- a/app/(chat)/api/chat/route.ts +++ b/app/(chat)/api/chat/route.ts @@ -20,9 +20,24 @@ import { getUsage } from "tokenlens/helpers"; import { auth, type UserType } from "@/app/(auth)/auth"; import type { VisibilityType } from "@/components/visibility-selector"; import { entitlementsByUserType } from "@/lib/ai/entitlements"; +import { + executeExtendJob, + ExtendServiceError, + inferExtendOptionsFromText, +} from "@/lib/ai/extend/service"; import type { ChatModel } from "@/lib/ai/models"; import { type RequestHints, systemPrompt } from "@/lib/ai/prompts"; import { myProvider } from "@/lib/ai/providers"; +import { + executeResizeJob, + inferResizeOptionsFromText, + ResizeServiceError, +} from "@/lib/ai/resize/service"; +import { + executeRmbgJob, + fetchImageBufferFromUrl, + RmbgServiceError, +} from "@/lib/ai/rmbg/service"; import { getLanguageModelById } from "@/lib/ai/dynamic-provider"; import { createDocument } from "@/lib/ai/tools/create-document"; import { getWeather } from "@/lib/ai/tools/get-weather"; @@ -239,6 +254,119 @@ function writeAssistantTextChunk( return messageId; } +function writeStructuredDataChunk( + writer: StreamWriter, + type: string, + data: unknown, + label: string, +) { + safeWriteStreamChunk( + writer, + { + type, + data: JSON.stringify(data), + }, + label, + ); +} + +function normalizeImageLoadingProviderType(providerType?: string | null): string { + const normalized = String(providerType || "").trim().toLowerCase(); + if (normalized === "gemini-image" || normalized === "huanke-gemini") { + return "gemini"; + } + if ( + normalized === "alibaba-image" || + normalized === "ruxa-image" || + normalized === "fal-image" || + normalized === "openai-image" || + normalized === "volcano-vision" || + normalized === "gemini" + ) { + return normalized; + } + return "alibaba-image"; +} + +function writeImageGenerationStatusChunk( + writer: StreamWriter, + params: { + phase: "queued" | "processing" | "finalizing" | "failed" | "completed"; + providerType?: string | null; + modelName?: string | null; + imageCount?: number | null; + }, + label: string, +) { + writeStructuredDataChunk( + writer, + "data-imageGenerationStatus", + { + phase: params.phase, + providerType: normalizeImageLoadingProviderType(params.providerType), + modelName: String(params.modelName || "").trim() || undefined, + imageCount: + Number.isFinite(Number(params.imageCount)) && Number(params.imageCount) > 0 + ? Number(params.imageCount) + : undefined, + }, + label, + ); +} + +function writeCanvasMediaChunk( + writer: StreamWriter, + params: { + messageId: string; + text?: string; + imageUrls?: string[]; + videoUrls?: string[]; + prompt?: string | null; + sourceImageUrls?: string[] | null; + }, + label: string, +) { + const imageUrls = Array.isArray(params.imageUrls) + ? params.imageUrls + : extractImageUrlsFromText(String(params.text || "")); + const videoUrls = Array.isArray(params.videoUrls) ? params.videoUrls : []; + const normalizedImageUrls = Array.from( + new Set( + imageUrls + .map((url) => String(url || "").trim()) + .filter((url) => /^https?:\/\//i.test(url)), + ), + ); + const normalizedVideoUrls = Array.from( + new Set( + videoUrls + .map((url) => String(url || "").trim()) + .filter((url) => /^https?:\/\//i.test(url)), + ), + ); + + if (normalizedImageUrls.length === 0 && normalizedVideoUrls.length === 0) { + return; + } + + writeStructuredDataChunk( + writer, + "data-canvasMedia", + { + messageId: params.messageId, + imageUrls: normalizedImageUrls, + videoUrls: normalizedVideoUrls, + prompt: String(params.prompt || "").trim() || undefined, + sourceImageUrls: Array.isArray(params.sourceImageUrls) + ? params.sourceImageUrls + .map((url) => String(url || "").trim()) + .filter((url) => /^https?:\/\//i.test(url)) + : [], + }, + label, + ); +} + function createSseResponse(stream: ReadableStream) { return new Response(stream.pipeThrough(new JsonToSseTransformStream()), { headers: { @@ -593,6 +721,203 @@ type AgentVideoPlan = { recommendedVideoParams?: AgentRecommendedVideoParams; }; +type AgentPlanNextAction = + | "respond" + | "clarify" + | "analyze_image" + | "generate_image" + | "edit_image" + | "generate_video"; + +type AgentPlannedToolCall = { + name: string; + reason?: string; + args?: Record; +}; + +type AgentExecutionPlan = { + goal: string; + nextAction: AgentPlanNextAction; + intent: "image_edit" | "vision" | "image_gen" | "video_gen" | "chat"; + subIntent?: string; + targetSource: + | "explicit_selection" + | "current_upload" + | "text_reference" + | "working_memory" + | "latest_result" + | "clarification" + | "none"; + preferredCapability: + | "respond" + | "analyze" + | "generic_edit" + | "remove_background" + | "enhance" + | "extend" + | "resize" + | "box_cutout" + | "generate_image" + | "generate_video" + | "publish_canvas"; + responseMode: "direct_answer" | "tool_then_answer" | "tool_then_execute" | "clarify"; + shouldUseHistoryImage: boolean; + shouldUseStructuredReply: boolean; + confidence: number; + toolCalls: AgentPlannedToolCall[]; +}; + +type AgentPreparedExecution = { + mode: AgentPlanNextAction; + routeIntent: "image_edit" | "vision" | "image_gen" | "video_gen" | "chat"; + routeSubIntent?: string; + targetSource: AgentExecutionPlan["targetSource"]; + preferredCapability: AgentExecutionPlan["preferredCapability"]; + shouldUseStructuredReply: boolean; + shouldOptimizePrompt: boolean; + shouldApplyImagePipeline: boolean; + shouldApplyVideoPipeline: boolean; + shouldStayConversational: boolean; + shouldPublishToCanvas: boolean; + toolCalls: AgentPlannedToolCall[]; + availableTools: string[]; +}; + +type DirectAgentToolCapability = "remove_background" | "resize" | "extend"; + +function resolveDirectAgentToolCapability( + execution?: AgentPreparedExecution, + plan?: AgentExecutionPlan, +): DirectAgentToolCapability | null { + const toolNames = new Set([ + ...(execution?.toolCalls || []).map((tool) => tool.name), + ...(plan?.toolCalls || []).map((tool) => tool.name), + ]); + + if ( + execution?.preferredCapability === "remove_background" || + execution?.routeSubIntent === "remove_background" || + toolNames.has("removeBackground") + ) { + return "remove_background"; + } + + if ( + execution?.preferredCapability === "resize" || + execution?.routeSubIntent === "resize" || + toolNames.has("resizeImage") + ) { + return "resize"; + } + + if ( + execution?.preferredCapability === "extend" || + execution?.routeSubIntent === "extend" || + toolNames.has("extendImage") + ) { + return "extend"; + } + + return null; +} + +function getDirectAgentToolLabel(capability: DirectAgentToolCapability): string { + switch (capability) { + case "remove_background": + return "Agent工具：抠图"; + case "resize": + return "Agent工具：尺寸调整"; + case "extend": + return "Agent工具：图像拓展"; + default: + return "Agent工具"; + } +} + +function findAgentToolCallArgs( + toolName: AgentPlannedToolCall["name"], + execution?: AgentPreparedExecution, + plan?: AgentExecutionPlan, +): Record | undefined { + const matchedTool = + execution?.toolCalls?.find((tool) => tool.name === toolName) || + plan?.toolCalls?.find((tool) => tool.name === toolName); + + if (!matchedTool?.args || typeof matchedTool.args !== "object") { + return undefined; + } + + return matchedTool.args; +} + +function toFiniteNumber(value: unknown): number | undefined { + const normalized = Number(value); + return Number.isFinite(normalized) ? normalized : undefined; +} + +function getDirectAgentToolSuccessMessage(capability: DirectAgentToolCapability): string { + switch (capability) { + case "remove_background": + return "已完成抠图处理，结果如下："; + case "resize": + return "已完成尺寸调整，结果如下："; + case "extend": + return "已完成图像拓展，结果如下："; + default: + return "已完成处理，结果如下："; + } +} + +function getDirectAgentToolTargetMissingMessage( + capability: DirectAgentToolCapability, +): string { + switch (capability) { + case "remove_background": + return "请先上传或选中一张图片，我才能继续帮你执行抠图。"; + case "resize": + return "请先上传或选中一张图片，我才能继续帮你调整尺寸。"; + case "extend": + return "请先上传或选中一张图片，我才能继续帮你做图像拓展。"; + default: + return "请先提供要处理的图片。"; + } +} + +function getDirectAgentToolErrorMessage( + capability: DirectAgentToolCapability, + error: unknown, +): string { + const errorMessage = + error instanceof Error ? String(error.message || "").trim() : String(error || "").trim(); + const errorDetail = + error && typeof error === "object" && "detail" in error + ? String((error as { detail?: unknown }).detail || "").trim() + : ""; + const detail = errorDetail || errorMessage || "请稍后重试。"; + + switch (capability) { + case "remove_background": + return `抠图暂时失败：${detail}`; + case "resize": + return `尺寸调整暂时失败：${detail}`; + case "extend": + return `图像拓展暂时失败：${detail}`; + default: + return `处理暂时失败：${detail}`; + } +} + +function buildAgentToolImageMarkdown(params: { + originalUrl: string; + thumbnailUrl?: string | null; + imageSize?: string; +}): string { + const originalUrl = String(params.originalUrl || "").trim(); + const thumbnailUrl = String(params.thumbnailUrl || originalUrl).trim() || originalUrl; + const imageSize = String(params.imageSize || "2K").trim() || "2K"; + return `![thumb|||${originalUrl}|||${imageSize}](${thumbnailUrl})`; +} + const IMAGE_FAILOVER_PROVIDER_ALLOWLIST = new Set([ "alibaba-image", "ruxa-image", @@ -2065,6 +2390,8 @@ export async function POST(request: Request) { let smartAutoRecommendedImageParams: AgentRecommendedImageParams | undefined; let smartAutoVideoPlan: AgentVideoPlan | undefined; let smartAutoRecommendedVideoParams: AgentRecommendedVideoParams | undefined; + let smartAutoPlan: AgentExecutionPlan | undefined; + let smartAutoExecution: AgentPreparedExecution | undefined; let smartAutoDetectedIntent: "image_edit" | "vision" | "image_gen" | "video_gen" | "chat" | undefined; let smartAutoHistoryImages: string[] = []; // 从历史对话中提取的图片URL（支持多张） let smartAutoNeedsClarification = false; // 是否需要先追问用户 @@ -2128,6 +2455,11 @@ export async function POST(request: Request) { (selectedModelProviderType ? fixedImageProviderTypes.has(selectedModelProviderType) : false)); + const isConversationMultimodalModel = + !selectedModelConfig.isSubModel && + selectedModelConfig.supportsVision && + !selectedModelConfig.supportsImageGen && + !isVideoProviderType(selectedModelProviderType); if (deepThinkingOn) { if (isAiModelAgentAuto(selectedModelConfig)) { @@ -2136,15 +2468,19 @@ export async function POST(request: Request) { console.log( `[Smart Auto Mode] 🧠 Agent-only model detected, force auto routing: ${selectedModelConfig.displayName}` ); - } else if (isFixedVisualModel) { + } else if (isFixedVisualModel || isConversationMultimodalModel) { shouldRunAgentConversation = true; shouldKeepSelectedModel = true; preferredAgentModelId = selectedModelConfig.id; console.log( - `[Smart Auto Mode] 🎯 Agent conversation enabled for fixed visual model: ${selectedModelConfig.displayName} (${selectedModelProviderType || "unknown"}), preferredAgentModelId=${preferredAgentModelId}` + `[Smart Auto Mode] 🎯 Agent conversation enabled for selected model: ${selectedModelConfig.displayName} (${selectedModelProviderType || "unknown"}), preferredAgentModelId=${preferredAgentModelId}` ); } - } else if (isFixedVisualModel || isAiModelAgentAuto(selectedModelConfig)) { + } else if ( + isFixedVisualModel || + isConversationMultimodalModel || + isAiModelAgentAuto(selectedModelConfig) + ) { console.log( `[Smart Auto Mode] ⏭️ Deep thinking disabled, skip agent analysis for: ${selectedModelConfig.displayName}` ); @@ -2266,6 +2602,16 @@ export async function POST(request: Request) { ]), ), imageFromHistory: smartAutoHistoryImages.length > 0, + targetResolutionMode: requestTargetResolution.resolutionMode, + mediaSource: requestResolvedMediaInput.source, + hasCanvasSelection: + Boolean(contextPayload?.selectedNodeIds?.length) || + requestResolvedMediaInput.selectedImageUrls.length > 0, + selectedImageCount: requestResolvedMediaInput.selectedImageUrls.length, + historyImageCount: + requestResolvedMediaInput.historyAssistantImageUrls.length + + requestResolvedMediaInput.historyUserImageUrls.length, + currentUploadImageCount: requestResolvedMediaInput.currentMessageImageUrls.length, historyMessages, context: autoContextText || undefined, preferredModelId: preferredAgentModelId, @@ -2274,6 +2620,21 @@ export async function POST(request: Request) { autoGraphTimeoutMs, ); + smartAutoPlan = autoGraphResult.plan; + smartAutoExecution = autoGraphResult.execution; + + if (smartAutoPlan) { + console.log("[Smart Auto Mode] Plan result:", { + goal: smartAutoPlan.goal, + nextAction: smartAutoPlan.nextAction, + intent: smartAutoPlan.intent, + targetSource: smartAutoPlan.targetSource, + preferredCapability: smartAutoPlan.preferredCapability, + confidence: smartAutoPlan.confidence, + toolCalls: smartAutoPlan.toolCalls?.map((tool) => tool.name) || [], + }); + } + if (autoGraphResult.analysis) { console.log("[Smart Auto Mode] Analysis result:", { intent: autoGraphResult.analysis.intent, @@ -2282,15 +2643,20 @@ export async function POST(request: Request) { needsClarification: autoGraphResult.analysis.needsClarification, }); - smartAutoDetectedIntent = autoGraphResult.analysis.intent; - smartAutoNeedsClarification = Boolean(autoGraphResult.analysis.needsClarification); + smartAutoDetectedIntent = + autoGraphResult.execution?.routeIntent || autoGraphResult.analysis.intent; + smartAutoNeedsClarification = + smartAutoPlan?.nextAction === "clarify" || + Boolean(autoGraphResult.analysis.needsClarification); smartAutoClarificationQuestion = autoGraphResult.analysis.clarifyingQuestions?.[0]; if ( smartAutoNeedsClarification && hasImage && hasActionableEditIntent && - (hasSelectedImageHint || hasExplicitTargetHint) + (hasSelectedImageHint || hasExplicitTargetHint) && + (smartAutoExecution?.mode === "edit_image" || + smartAutoExecution?.mode === "generate_image") ) { smartAutoNeedsClarification = false; smartAutoClarificationQuestion = undefined; @@ -2329,12 +2695,11 @@ export async function POST(request: Request) { } const autoIntent = autoGraphResult.analysis?.intent; + const autoExecutionMode = smartAutoExecution?.mode; const isImageTask = - autoIntent === "image_gen" || - autoIntent === "image_edit" || - hasImage; + autoExecutionMode === "generate_image" || + autoExecutionMode === "edit_image"; const needsRicherAutoReply = - isImageTask && !smartAutoNeedsClarification && !smartAutoFriendlyMessage; @@ -2343,12 +2708,23 @@ export async function POST(request: Request) { smartAutoFriendlyMessage = autoGraphResult.analysis?.conversationalReply || "我先分析你这轮图片里的主体、风格和构图，再给你执行方案。"; + } else if (autoIntent === "vision") { + smartAutoFriendlyMessage = + autoGraphResult.analysis?.conversationalReply || + "收到，我先看图分析，再直接回答你。"; + } else if (autoIntent === "chat") { + smartAutoFriendlyMessage = + autoGraphResult.analysis?.conversationalReply || + "收到，我们先继续聊你的这个需求。"; } else { - smartAutoFriendlyMessage = buildAutoModeFallbackMessage({ - text: autoTextForFallback, - hasImage, - fromHistory: smartAutoHistoryImages.length > 0, - }); + smartAutoFriendlyMessage = + smartAutoExecution?.shouldUseStructuredReply + ? buildAutoModeFallbackMessage({ + text: autoTextForFallback, + hasImage, + fromHistory: smartAutoHistoryImages.length > 0, + }) + : "收到，我先理解你的当前目标，再继续处理。"; } console.log("[Smart Auto Mode] 🧩 Filled missing conversational reply"); } @@ -2384,6 +2760,18 @@ export async function POST(request: Request) { if (routeIsVideo && !allowAgentVideoGeneration) { const fallbackIntent: "image_edit" | "image_gen" = hasImage ? "image_edit" : "image_gen"; smartAutoDetectedIntent = fallbackIntent; + smartAutoExecution = smartAutoExecution + ? { + ...smartAutoExecution, + mode: hasImage ? "edit_image" : "generate_image", + routeIntent: fallbackIntent, + shouldApplyImagePipeline: true, + shouldApplyVideoPipeline: false, + shouldOptimizePrompt: true, + shouldStayConversational: false, + shouldUseStructuredReply: true, + } + : smartAutoExecution; smartAutoVideoPlan = undefined; smartAutoRecommendedVideoParams = undefined; if (smartAutoPromptFromVideoPlan) { @@ -2474,6 +2862,61 @@ export async function POST(request: Request) { "fallbackAnalyzeIntent", rescueTimeoutMs, ); + smartAutoDetectedIntent = rescueAnalysis.intent; + smartAutoNeedsClarification = Boolean(rescueAnalysis.needsClarification); + smartAutoClarificationQuestion = rescueAnalysis.clarifyingQuestions?.[0]; + smartAutoExecution = { + mode: + rescueAnalysis.intent === "video_gen" + ? "generate_video" + : rescueAnalysis.intent === "image_gen" + ? "generate_image" + : rescueAnalysis.intent === "image_edit" + ? "edit_image" + : rescueAnalysis.intent === "vision" + ? "analyze_image" + : rescueAnalysis.needsClarification + ? "clarify" + : "respond", + routeIntent: rescueAnalysis.intent, + routeSubIntent: rescueAnalysis.subIntent, + targetSource: requestTargetResolution?.resolutionMode || "none", + preferredCapability: + rescueAnalysis.intent === "video_gen" + ? "generate_video" + : rescueAnalysis.intent === "image_gen" + ? "generate_image" + : rescueAnalysis.intent === "image_edit" + ? rescueAnalysis.subIntent === "remove_background" + ? "remove_background" + : rescueAnalysis.subIntent === "enhance" + ? "enhance" + : rescueAnalysis.subIntent === "extend" + ? "extend" + : rescueAnalysis.subIntent === "resize" + ? "resize" + : "generic_edit" + : rescueAnalysis.intent === "vision" + ? "analyze" + : "respond", + shouldUseStructuredReply: + rescueAnalysis.intent === "video_gen" || + rescueAnalysis.intent === "image_gen" || + rescueAnalysis.intent === "image_edit", + shouldOptimizePrompt: + rescueAnalysis.intent === "video_gen" || + rescueAnalysis.intent === "image_gen" || + rescueAnalysis.intent === "image_edit", + shouldApplyImagePipeline: + rescueAnalysis.intent === "image_gen" || + rescueAnalysis.intent === "image_edit", + shouldApplyVideoPipeline: rescueAnalysis.intent === "video_gen", + shouldStayConversational: + rescueAnalysis.intent === "chat" || rescueAnalysis.intent === "vision", + shouldPublishToCanvas: false, + toolCalls: [], + availableTools: [], + }; if (rescueAnalysis.conversationalReply?.trim()) { smartAutoFriendlyMessage = rescueAnalysis.conversationalReply.trim(); } @@ -2488,6 +2931,59 @@ export async function POST(request: Request) { autoTextForFallback, autoCurrentImageUrlsForFallback.length > 0 || smartAutoHistoryImages.length > 0, ); + smartAutoDetectedIntent = keywordFallback.intent; + smartAutoExecution = { + mode: + keywordFallback.intent === "video_gen" + ? "generate_video" + : keywordFallback.intent === "image_gen" + ? "generate_image" + : keywordFallback.intent === "image_edit" + ? "edit_image" + : keywordFallback.intent === "vision" + ? "analyze_image" + : keywordFallback.needsClarification + ? "clarify" + : "respond", + routeIntent: keywordFallback.intent, + routeSubIntent: keywordFallback.subIntent, + targetSource: requestTargetResolution?.resolutionMode || "none", + preferredCapability: + keywordFallback.intent === "video_gen" + ? "generate_video" + : keywordFallback.intent === "image_gen" + ? "generate_image" + : keywordFallback.intent === "image_edit" + ? keywordFallback.subIntent === "remove_background" + ? "remove_background" + : keywordFallback.subIntent === "enhance" + ? "enhance" + : keywordFallback.subIntent === "extend" + ? "extend" + : keywordFallback.subIntent === "resize" + ? "resize" + : "generic_edit" + : keywordFallback.intent === "vision" + ? "analyze" + : "respond", + shouldUseStructuredReply: + keywordFallback.intent === "video_gen" || + keywordFallback.intent === "image_gen" || + keywordFallback.intent === "image_edit", + shouldOptimizePrompt: + keywordFallback.intent === "video_gen" || + keywordFallback.intent === "image_gen" || + keywordFallback.intent === "image_edit", + shouldApplyImagePipeline: + keywordFallback.intent === "image_gen" || + keywordFallback.intent === "image_edit", + shouldApplyVideoPipeline: keywordFallback.intent === "video_gen", + shouldStayConversational: + keywordFallback.intent === "chat" || keywordFallback.intent === "vision", + shouldPublishToCanvas: false, + toolCalls: [], + availableTools: [], + }; if (keywordFallback.conversationalReply?.trim()) { smartAutoFriendlyMessage = keywordFallback.conversationalReply.trim(); console.warn("[Smart Auto Mode] Applied keyword fallback after rescue failure"); @@ -2551,9 +3047,30 @@ export async function POST(request: Request) { } } - // 简单回退：有图选图片模型，无图选默认模型 + // 回退路由：优先按分析出的意图选模型，避免“有图就生图” let bestModel = null; - if (hasImage) { + try { + const fallbackIntent = + smartAutoDetectedIntent || (hasImage ? "vision" : "chat"); + const { routeToModel } = await import("@/lib/ai/model-router"); + const fallbackRoute = await routeToModel({ + intent: fallbackIntent, + subIntent: smartAutoExecution?.routeSubIntent, + hasImage, + }); + bestModel = allModels.find( + (item) => item.model.id === fallbackRoute.id || item.model.modelId === fallbackRoute.modelId, + ) || null; + if (bestModel) { + console.log( + `[Smart Auto Mode] Fallback routed by intent: ${fallbackRoute.displayName} (${fallbackRoute.reason})`, + ); + } + } catch (fallbackRouteError) { + console.warn("[Smart Auto Mode] Fallback routeByIntent failed, use local heuristic:", fallbackRouteError); + } + + if (!bestModel && hasImage) { bestModel = allModels.find( (m) => m.model.supportsImageGen && @@ -2642,6 +3159,18 @@ export async function POST(request: Request) { part?.type === "file" || part?.type === "image" || part?.type === "image_url" ); smartAutoDetectedIntent = hasImageInput ? "image_edit" : "image_gen"; + smartAutoExecution = smartAutoExecution + ? { + ...smartAutoExecution, + mode: hasImageInput ? "edit_image" : "generate_image", + routeIntent: hasImageInput ? "image_edit" : "image_gen", + shouldApplyImagePipeline: true, + shouldApplyVideoPipeline: false, + shouldOptimizePrompt: true, + shouldStayConversational: false, + shouldUseStructuredReply: true, + } + : smartAutoExecution; finalModelId = fallbackModel.model.id; smartAutoVideoPlan = undefined; smartAutoRecommendedVideoParams = undefined; @@ -2665,15 +3194,22 @@ export async function POST(request: Request) { part?.type === "file" || part?.type === "image" || part?.type === "image_url" ); const hasHistoryImageInput = smartAutoHistoryImages.length > 0; - smartAutoFriendlyMessage = buildAutoModeFallbackMessage({ - text: latestUserText, - hasImage: hasCurrentImageInput || hasHistoryImageInput, - fromHistory: !hasCurrentImageInput && hasHistoryImageInput, - }); + smartAutoFriendlyMessage = + smartAutoExecution?.shouldUseStructuredReply + ? buildAutoModeFallbackMessage({ + text: latestUserText, + hasImage: hasCurrentImageInput || hasHistoryImageInput, + fromHistory: !hasCurrentImageInput && hasHistoryImageInput, + }) + : "收到，我先理解你的当前目标，再继续处理。"; console.log("[Smart Auto Mode] 🧩 Backfilled conversational reply with fallback"); } - if (shouldRunAgentConversation && !smartAutoNeedsClarification) { + if ( + shouldRunAgentConversation && + !smartAutoNeedsClarification && + smartAutoExecution?.shouldUseStructuredReply + ) { const latestUserText = message.parts .filter((part): part is { type: "text"; text: string } => part.type === "text") .map((part) => part.text) @@ -2695,12 +3231,12 @@ export async function POST(request: Request) { const shouldApplyAgentImageParams = shouldRunAgentConversation && !smartAutoNeedsClarification && - (smartAutoDetectedIntent === "image_gen" || smartAutoDetectedIntent === "image_edit"); + Boolean(smartAutoExecution?.shouldApplyImagePipeline); const shouldApplyAgentVideoParams = shouldRunAgentConversation && !smartAutoNeedsClarification && allowAgentVideoGeneration && - smartAutoDetectedIntent === "video_gen"; + Boolean(smartAutoExecution?.shouldApplyVideoPipeline); const effectiveImageParams = shouldApplyAgentImageParams ? mergeImageParamsWithAgentRecommendation({ incoming: requestBody.imageParams, @@ -2740,7 +3276,11 @@ export async function POST(request: Request) { }); } - if (shouldRunAgentConversation && smartAutoFriendlyMessage) { + if ( + shouldRunAgentConversation && + smartAutoFriendlyMessage && + smartAutoExecution?.shouldUseStructuredReply + ) { smartAutoFriendlyMessage = sanitizeAgentOutputSettingsSection({ message: smartAutoFriendlyMessage, userImageParams: requestBody.imageParams, @@ -2790,7 +3330,7 @@ export async function POST(request: Request) { console.warn("[Smart Auto Mode] Skip image summary enrichment due to pre-response budget"); } - if (summary?.trim()) { + if (summary?.trim() && smartAutoExecution?.shouldUseStructuredReply) { const baseMessage = ensureStructuredAgentFriendlyMessage({ message: smartAutoFriendlyMessage, text: latestUserText, @@ -2811,6 +3351,12 @@ export async function POST(request: Request) { "[Smart Auto Mode] ✅ Image analysis summary ready:", summary.trim().slice(0, 160), ); + } else if ( + summary?.trim() && + !smartAutoExecution?.shouldUseStructuredReply && + !smartAutoFriendlyMessage + ) { + smartAutoFriendlyMessage = "收到，我会结合这张图的内容直接回答你。"; } else if (smartAutoFriendlyMessage && shouldEnrichImageAnalysisText(smartAutoFriendlyMessage)) { console.warn("[Smart Auto Mode] ⚠️ Image summary unavailable, keep original analysis text"); } @@ -2820,7 +3366,11 @@ export async function POST(request: Request) { } } - if (shouldRunAgentConversation && smartAutoFriendlyMessage) { + if ( + shouldRunAgentConversation && + smartAutoFriendlyMessage && + smartAutoExecution?.shouldUseStructuredReply + ) { smartAutoFriendlyMessage = normalizeAgentReportForDisplay(smartAutoFriendlyMessage); } @@ -3040,9 +3590,10 @@ export async function POST(request: Request) { chatId: id, projectId: contextPayload?.projectId || null, currentIntent: + smartAutoExecution?.routeIntent || smartAutoDetectedIntent || (requestResolvedMediaInput.hasImage - ? "image_edit" + ? "vision" : requestBody.videoParams ? "video_gen" : null), @@ -3093,16 +3644,32 @@ export async function POST(request: Request) { .trim(); if (hasCurrentImageInput || hasHistoryImageInput || latestUserText.length > 0) { - smartAutoNeedsClarification = false; - smartAutoClarificationQuestion = undefined; - if ( - !smartAutoFriendlyMessage || - /[?？]$/.test(smartAutoFriendlyMessage.trim()) || - /哪一张|哪个|请先确认/.test(smartAutoFriendlyMessage) - ) { - smartAutoFriendlyMessage = "收到，我将按你的当前需求直接执行。"; + const canForceByIntent = /改|调整|优化|换|替换|去掉|抠|重绘|增强|修|处理|按照|按|保留|统一|色差|背景|清晰|风格/.test( + latestUserText + ); + const hasForceTargetHint = + Boolean(contextPayload?.selectedNodeIds?.length) || + requestTargetResolution?.resolutionMode === "text_reference" || + /第[一二三四五六七八九十\d]+张|左|右|上|下|中间|选中/.test(latestUserText); + const canForceExecution = + (smartAutoExecution?.mode === "edit_image" || + smartAutoExecution?.mode === "generate_image" || + smartAutoExecution?.mode === "generate_video") && + canForceByIntent && + hasForceTargetHint; + + if (canForceExecution) { + smartAutoNeedsClarification = false; + smartAutoClarificationQuestion = undefined; + if ( + !smartAutoFriendlyMessage || + /[?？]$/.test(smartAutoFriendlyMessage.trim()) || + /哪一张|哪个|请先确认/.test(smartAutoFriendlyMessage) + ) { + smartAutoFriendlyMessage = "收到，我将按你的当前需求直接执行。"; + } + console.log("[Smart Auto Mode] ✅ Force direct execution (skip clarification)"); } - console.log("[Smart Auto Mode] ✅ Force direct execution (skip clarification)"); } } @@ -3147,6 +3714,307 @@ export async function POST(request: Request) { return createSseResponse(clarificationStream); } + const directAgentToolCapability = shouldRunAgentConversation + ? resolveDirectAgentToolCapability(smartAutoExecution, smartAutoPlan) + : null; + const directAgentToolTargetImageUrl = + requestTargetResolution?.targetImageUrls?.[0] || + requestResolvedMediaInput?.selectedImageUrls?.[0] || + requestResolvedMediaInput?.currentMessageImageUrls?.[0] || + requestWorkingState?.lastResolvedTargetImageUrls?.[0] || + requestWorkingState?.lastGeneratedImageUrls?.[0] || + null; + + if ( + shouldRunAgentConversation && + !smartAutoNeedsClarification && + directAgentToolCapability + ) { + if (!directAgentToolTargetImageUrl) { + const missingTargetMessage = getDirectAgentToolTargetMissingMessage( + directAgentToolCapability, + ); + const missingTargetStream = createUIMessageStream({ + execute: async ({ writer }) => { + const messageId = generateUUID(); + writeAssistantTextChunk( + writer, + missingTargetMessage, + `agent:${directAgentToolCapability}:missing-target`, + messageId, + ); + + await saveMessages({ + messages: [ + { + id: messageId, + chatId: id, + role: "assistant", + parts: [{ type: "text", text: missingTargetMessage }], + attachments: [], + createdAt: new Date(), + videoStorageStatus: null, + temporaryVideoUrl: null, + permanentVideoUrl: null, + archivedAt: null, + videoMetadata: null, + }, + ], + }); + }, + }); + + return createSseResponse(missingTargetStream); + } + + const assistantMessageId = generateUUID(); + await saveMessages({ + messages: [ + { + id: assistantMessageId, + chatId: id, + role: "assistant", + parts: [{ type: "text", text: "处理中..." }], + attachments: [], + createdAt: new Date(), + videoStorageStatus: null, + temporaryVideoUrl: null, + permanentVideoUrl: null, + archivedAt: null, + videoMetadata: null, + }, + ], + }); + + try { + let resultOriginalUrl = ""; + let resultThumbnailUrl: string | null = null; + let resultImageSize = "2K"; + let resultAssetId: string | null = null; + + if (directAgentToolCapability === "remove_background") { + const sourceBuffer = await fetchImageBufferFromUrl(directAgentToolTargetImageUrl); + const result = await executeRmbgJob({ + userId: session.user.id, + imageBuffer: sourceBuffer, + prompt: requestTextContent || "抠图处理", + chatId: id, + messageId: assistantMessageId, + sourceImageUrl: directAgentToolTargetImageUrl, + }); + + resultOriginalUrl = result.original_url; + resultThumbnailUrl = result.thumbnail_url; + resultImageSize = "2K"; + resultAssetId = result.mediaAssetId || null; + } else if (directAgentToolCapability === "resize") { + const plannedArgs = findAgentToolCallArgs( + "resizeImage", + smartAutoExecution, + smartAutoPlan, + ); + const inferredOptions = inferResizeOptionsFromText(requestTextContent); + const canvasWidth = toFiniteNumber( + plannedArgs?.canvasWidth ?? plannedArgs?.width ?? inferredOptions.canvasWidth, + ); + const canvasHeight = toFiniteNumber( + plannedArgs?.canvasHeight ?? plannedArgs?.height ?? inferredOptions.canvasHeight, + ); + + if (!canvasWidth || !canvasHeight) { + throw new ResizeServiceError( + "请告诉我目标尺寸或比例，例如 1024x1024、4:5、16:9。", + 400, + ); + } + + const transparentBackground = + plannedArgs?.transparentBackground === true || + plannedArgs?.transparentBackground === "true" || + inferredOptions.transparentBackground === true; + const backgroundColor = + typeof plannedArgs?.backgroundColor === "string" + ? plannedArgs.backgroundColor + : inferredOptions.backgroundColor; + const mimeType = + typeof plannedArgs?.mimeType === "string" ? plannedArgs.mimeType : undefined; + const result = await executeResizeJob({ + userId: session.user.id, + imageUrl: directAgentToolTargetImageUrl, + prompt: requestTextContent || "尺寸调整处理", + chatId: id, + messageId: assistantMessageId, + sourceImageUrl: directAgentToolTargetImageUrl, + canvasWidth, + canvasHeight, + transparentBackground, + backgroundColor, + mimeType, + }); + + resultOriginalUrl = result.original_url; + resultThumbnailUrl = result.thumbnail_url; + resultImageSize = `${result.canvas_width}x${result.canvas_height}`; + resultAssetId = result.mediaAssetId || null; + } else if (directAgentToolCapability === "extend") { + const plannedArgs = findAgentToolCallArgs( + "extendImage", + smartAutoExecution, + smartAutoPlan, + ); + const inferredOptions = inferExtendOptionsFromText(requestTextContent); + const xScale = toFiniteNumber(plannedArgs?.xScale ?? inferredOptions.xScale); + const yScale = toFiniteNumber(plannedArgs?.yScale ?? inferredOptions.yScale); + const result = await executeExtendJob({ + userId: session.user.id, + imageUrl: directAgentToolTargetImageUrl, + prompt: requestTextContent || "图像拓展处理", + chatId: id, + messageId: assistantMessageId, + sourceImageUrl: directAgentToolTargetImageUrl, + ...(xScale ? { xScale } : {}), + ...(yScale ? { yScale } : {}), + }); + + resultOriginalUrl = result.original_url; + resultThumbnailUrl = result.thumbnail_url; + resultImageSize = "2K"; + resultAssetId = result.mediaAssetId || null; + } + + const assistantText = prependImageModelBadge( + [ + getDirectAgentToolSuccessMessage(directAgentToolCapability), + buildAgentToolImageMarkdown({ + originalUrl: resultOriginalUrl, + thumbnailUrl: resultThumbnailUrl, + imageSize: resultImageSize, + }), + ] + .filter(Boolean) + .join("\n\n"), + getDirectAgentToolLabel(directAgentToolCapability), + ); + + await updateMessageById({ + messageId: assistantMessageId, + chatId: id, + parts: [{ type: "text", text: assistantText }], + }); + + try { + const resultAssetIds = resultAssetId ? [resultAssetId] : null; + const resultImageUrls = resultOriginalUrl ? [resultOriginalUrl] : null; + await upsertConversationWorkingState({ + userId: session.user.id, + chatId: id, + projectId: contextPayload?.projectId || null, + currentIntent: "image_edit", + currentSubIntent: directAgentToolCapability, + currentFocusAssetIds: resultAssetIds, + currentFocusImageUrls: resultImageUrls, + lastGeneratedAssetIds: resultAssetIds, + lastGeneratedImageUrls: resultImageUrls, + lastResolvedTargetAssetIds: resultAssetIds, + lastResolvedTargetImageUrls: resultImageUrls, + lastMessageId: assistantMessageId, + }); + } catch (memoryError) { + console.warn( + `[Chat API] Failed to update working state after ${directAgentToolCapability}:`, + memoryError, + ); + } + + if (smartAutoExecution?.shouldPublishToCanvas && contextPayload?.source === "canvas") { + console.log( + `[Chat API] Agent publishToCanvas satisfied by assistant media auto-sync (${directAgentToolCapability})`, + ); + } + + const directToolStream = createUIMessageStream({ + execute: async ({ writer }) => { + writeAssistantTextChunk( + writer, + assistantText, + `agent:${directAgentToolCapability}:success`, + assistantMessageId, + ); + if (contextPayload?.source === "canvas") { + writeCanvasMediaChunk( + writer, + { + messageId: assistantMessageId, + imageUrls: resultOriginalUrl ? [resultOriginalUrl] : [], + prompt: requestTextContent || getDirectAgentToolLabel(directAgentToolCapability), + sourceImageUrls: requestTargetResolution?.targetImageUrls || [], + }, + `agent:${directAgentToolCapability}:canvas-media`, + ); + } + writeAssistantTextChunk( + writer, + "", + `agent:${directAgentToolCapability}:image-end`, + ); + }, + }); + + return createSseResponse(directToolStream); + } catch (directToolError) { + console.error( + `[Chat API] Direct agent tool ${directAgentToolCapability} failed:`, + directToolError, + ); + + if ( + !(directToolError instanceof RmbgServiceError) && + !(directToolError instanceof ResizeServiceError) && + !(directToolError instanceof ExtendServiceError) + ) { + console.warn( + `[Chat API] Direct agent tool ${directAgentToolCapability} threw unexpected error type`, + ); + } + + const errorMessage = getDirectAgentToolErrorMessage( + directAgentToolCapability, + directToolError, + ); + + try { + await updateMessageById({ + messageId: assistantMessageId, + chatId: id, + parts: [{ type: "text", text: errorMessage }], + }); + } catch (updateError) { + console.warn( + `[Chat API] Failed to persist direct-tool error message (${directAgentToolCapability}):`, + updateError, + ); + } + + const directToolErrorStream = createUIMessageStream({ + execute: async ({ writer }) => { + writeAssistantTextChunk( + writer, + errorMessage, + `agent:${directAgentToolCapability}:error`, + assistantMessageId, + ); + writeAssistantTextChunk( + writer, + "", + `agent:${directAgentToolCapability}:image-end-error`, + ); + }, + }); + + return createSseResponse(directToolErrorStream); + } + } + let finalMergedUsage: AppUsage | undefined; // 获取启用的工具列表 @@ -3441,13 +4309,25 @@ export async function POST(request: Request) { let imageMarkdown = ""; // 初始化为空字符串 let usedModelId = actualModelId; - // 🆕 图片生成真正开始时发出标记（用于前端显示占位图） - writeAssistantTextChunk( - writer, - "", - "gemini:image-start", - ); - stopKeepalive = startStreamKeepalive(writer, "gemini-image"); + // 🆕 图片生成真正开始时发出标记（用于前端显示占位图） + writeAssistantTextChunk( + writer, + "", + "gemini:image-start", + ); + if (contextPayload?.source === "canvas") { + writeImageGenerationStatusChunk( + writer, + { + phase: "processing", + providerType: (languageModel as any).provider, + modelName: currentModelConfig?.displayName || actualModelId, + imageCount: effectiveImageParams?.imageCount || defaultImageQualityParams.imageCount || 1, + }, + "gemini:image-status-start", + ); + } + stopKeepalive = startStreamKeepalive(writer, "gemini-image"); try { // 首先尝试主模型 @@ -3493,19 +4373,43 @@ export async function POST(request: Request) { const messageId = generateUUID(); // 写入消息到流 - 使用正确的 parts 格式 - writeAssistantTextChunk( - writer, - imageMarkdownWithModel, - "gemini:image-result", - messageId, - ); - - // 🆕 图片生成结束标记（成功） - writeAssistantTextChunk( - writer, - "", - "gemini:image-end-success", - ); + writeAssistantTextChunk( + writer, + imageMarkdownWithModel, + "gemini:image-result", + messageId, + ); + if (contextPayload?.source === "canvas") { + writeCanvasMediaChunk( + writer, + { + messageId, + text: imageMarkdownWithModel, + prompt: requestTextContent, + sourceImageUrls: requestTargetResolution?.targetImageUrls || [], + }, + "gemini:image-canvas-media", + ); + } + + // 🆕 图片生成结束标记（成功） + writeAssistantTextChunk( + writer, + "", + "gemini:image-end-success", + ); + if (contextPayload?.source === "canvas") { + writeImageGenerationStatusChunk( + writer, + { + phase: "completed", + providerType: (languageModel as any).provider, + modelName: usedModelDisplayName, + imageCount: effectiveImageParams?.imageCount || defaultImageQualityParams.imageCount || 1, + }, + "gemini:image-status-completed", + ); + } // 保存消息到数据库（现在是 S3 URL，可以完整保存） try { @@ -3604,19 +4508,43 @@ export async function POST(request: Request) { const messageId = generateUUID(); // 写入消息到流 - writeAssistantTextChunk( - writer, - imageMarkdownWithModel, - "gemini:volcano-failover-result", - messageId, - ); - - // 🆕 图片生成结束标记（故障转移成功） - writeAssistantTextChunk( - writer, - "", - "gemini:volcano-failover-end", - ); + writeAssistantTextChunk( + writer, + imageMarkdownWithModel, + "gemini:volcano-failover-result", + messageId, + ); + if (contextPayload?.source === "canvas") { + writeCanvasMediaChunk( + writer, + { + messageId, + text: imageMarkdownWithModel, + prompt: requestTextContent, + sourceImageUrls: requestTargetResolution?.targetImageUrls || [], + }, + "gemini:volcano-failover-canvas-media", + ); + } + + // 🆕 图片生成结束标记（故障转移成功） + writeAssistantTextChunk( + writer, + "", + "gemini:volcano-failover-end", + ); + if (contextPayload?.source === "canvas") { + writeImageGenerationStatusChunk( + writer, + { + phase: "completed", + providerType: "volcano-vision", + modelName: volcanoModel.model.displayName || volcanoModel.model.modelId, + imageCount: 1, + }, + "gemini:volcano-failover-status-completed", + ); + } // 保存消息到数据库 try { @@ -3840,12 +4768,24 @@ export async function POST(request: Request) { let usedModelId = actualModelId; // 🆕 图片生成真正开始时发出标记（用于前端显示占位图） - writeAssistantTextChunk( - writer, - "", - "alibaba:image-start", - ); - stopKeepalive = startStreamKeepalive(writer, "alibaba-image"); + writeAssistantTextChunk( + writer, + "", + "alibaba:image-start", + ); + if (contextPayload?.source === "canvas") { + writeImageGenerationStatusChunk( + writer, + { + phase: "processing", + providerType: (languageModel as any).provider, + modelName: alibabaModelConfig?.displayName || actualModelId, + imageCount: effectiveImageParams?.imageCount || defaultImageQualityParams.imageCount || 1, + }, + "alibaba:image-status-start", + ); + } + stopKeepalive = startStreamKeepalive(writer, "alibaba-image"); try { imageMarkdown = await tryGenerateImage(actualModelId, false); @@ -3971,19 +4911,43 @@ export async function POST(request: Request) { const messageId = generateUUID(); // 写入图片消息到流 - writeAssistantTextChunk( - writer, - imageMarkdownWithModel, - "alibaba:image-result", - messageId, - ); - - // 🆕 图片生成结束标记（成功） - writeAssistantTextChunk( - writer, - "", - "alibaba:image-end-success", - ); + writeAssistantTextChunk( + writer, + imageMarkdownWithModel, + "alibaba:image-result", + messageId, + ); + if (contextPayload?.source === "canvas") { + writeCanvasMediaChunk( + writer, + { + messageId, + text: imageMarkdownWithModel, + prompt: requestTextContent, + sourceImageUrls: requestTargetResolution?.targetImageUrls || [], + }, + "alibaba:image-canvas-media", + ); + } + + // 🆕 图片生成结束标记（成功） + writeAssistantTextChunk( + writer, + "", + "alibaba:image-end-success", + ); + if (contextPayload?.source === "canvas") { + writeImageGenerationStatusChunk( + writer, + { + phase: "completed", + providerType: (languageModel as any).provider, + modelName: usedModelDisplayName, + imageCount: effectiveImageParams?.imageCount || defaultImageQualityParams.imageCount || 1, + }, + "alibaba:image-status-completed", + ); + } // 保存图片消息到数据库 try { @@ -4196,12 +5160,24 @@ export async function POST(request: Request) { let usedModelId = actualModelId; // 🆕 图片生成真正开始时发出标记（用于前端显示占位图） - writeAssistantTextChunk( - writer, - "", - "volcano:image-start", - ); - stopKeepalive = startStreamKeepalive(writer, "volcano-image"); + writeAssistantTextChunk( + writer, + "", + "volcano:image-start", + ); + if (contextPayload?.source === "canvas") { + writeImageGenerationStatusChunk( + writer, + { + phase: "processing", + providerType: "volcano-vision", + modelName: volcanoModelConfig?.displayName || actualModelId, + imageCount: effectiveImageParams?.imageCount || defaultImageQualityParams.imageCount || 1, + }, + "volcano:image-status-start", + ); + } + stopKeepalive = startStreamKeepalive(writer, "volcano-image"); try { imageMarkdown = await tryGenerateImage(actualModelId, false); } catch (primaryError) { @@ -4322,19 +5298,43 @@ export async function POST(request: Request) { const messageId = generateUUID(); // 写入图片消息到流 - writeAssistantTextChunk( - writer, - imageMarkdownWithModel, - "volcano:image-result", - messageId, - ); - - // 🆕 图片生成结束标记（成功） - writeAssistantTextChunk( - writer, - "", - "volcano:image-end-success", - ); + writeAssistantTextChunk( + writer, + imageMarkdownWithModel, + "volcano:image-result", + messageId, + ); + if (contextPayload?.source === "canvas") { + writeCanvasMediaChunk( + writer, + { + messageId, + text: imageMarkdownWithModel, + prompt: requestTextContent, + sourceImageUrls: requestTargetResolution?.targetImageUrls || [], + }, + "volcano:image-canvas-media", + ); + } + + // 🆕 图片生成结束标记（成功） + writeAssistantTextChunk( + writer, + "", + "volcano:image-end-success", + ); + if (contextPayload?.source === "canvas") { + writeImageGenerationStatusChunk( + writer, + { + phase: "completed", + providerType: "volcano-vision", + modelName: usedModelDisplayName, + imageCount: effectiveImageParams?.imageCount || defaultImageQualityParams.imageCount || 1, + }, + "volcano:image-status-completed", + ); + } // 保存图片消息到数据库 try { @@ -4783,15 +5783,27 @@ export async function POST(request: Request) { smartAutoHistoryImages ); - // 生成开始标记 - dataStream.write({ - type: "data-appendMessage", - data: JSON.stringify({ - id: generateUUID(), - role: "assistant", - parts: [{ type: "text", text: "" }], - }), - }); + // 生成开始标记 + dataStream.write({ + type: "data-appendMessage", + data: JSON.stringify({ + id: generateUUID(), + role: "assistant", + parts: [{ type: "text", text: "" }], + }), + }); + if (contextPayload?.source === "canvas") { + writeImageGenerationStatusChunk( + dataStream as unknown as StreamWriter, + { + phase: "processing", + providerType: currentModelConfig?.providerId ? resolvedProviderType || providerType : providerType, + modelName: currentModelConfig?.displayName || actualModelId, + imageCount: effectiveImageParams?.imageCount || 1, + }, + "openai-compat:image-status-start", + ); + } try { const tryGenerateWithOpenAICompatModel = async (modelToUse: any) => { @@ -4953,24 +5965,48 @@ export async function POST(request: Request) { ); const messageId = generateUUID(); - dataStream.write({ - type: "data-appendMessage", - data: JSON.stringify({ - id: messageId, - role: "assistant", - parts: [{ type: "text", text: normalizedTextWithModel }], - }), - }); + dataStream.write({ + type: "data-appendMessage", + data: JSON.stringify({ + id: messageId, + role: "assistant", + parts: [{ type: "text", text: normalizedTextWithModel }], + }), + }); + if (contextPayload?.source === "canvas") { + writeCanvasMediaChunk( + dataStream as unknown as StreamWriter, + { + messageId, + text: normalizedTextWithModel, + prompt: requestTextContent, + sourceImageUrls: requestTargetResolution?.targetImageUrls || [], + }, + "openai-compat:image-canvas-media", + ); + } - // 生成结束标记 - dataStream.write({ - type: "data-appendMessage", + // 生成结束标记 + dataStream.write({ + type: "data-appendMessage", data: JSON.stringify({ id: generateUUID(), role: "assistant", - parts: [{ type: "text", text: "" }], - }), - }); + parts: [{ type: "text", text: "" }], + }), + }); + if (contextPayload?.source === "canvas") { + writeImageGenerationStatusChunk( + dataStream as unknown as StreamWriter, + { + phase: "completed", + providerType: resolvedProviderType || providerType, + modelName: usedModelDisplayName, + imageCount: effectiveImageParams?.imageCount || 1, + }, + "openai-compat:image-status-completed", + ); + } await saveMessages({ messages: [ @@ -5026,14 +6062,26 @@ export async function POST(request: Request) { parts: [{ type: "text", text: errorText }], }), }); - dataStream.write({ - type: "data-appendMessage", - data: JSON.stringify({ - id: generateUUID(), - role: "assistant", - parts: [{ type: "text", text: "" }], - }), - }); + dataStream.write({ + type: "data-appendMessage", + data: JSON.stringify({ + id: generateUUID(), + role: "assistant", + parts: [{ type: "text", text: "" }], + }), + }); + if (contextPayload?.source === "canvas") { + writeImageGenerationStatusChunk( + dataStream as unknown as StreamWriter, + { + phase: "failed", + providerType: resolvedProviderType || providerType, + modelName: currentModelConfig?.displayName || actualModelId, + imageCount: effectiveImageParams?.imageCount || 1, + }, + "openai-compat:image-status-failed", + ); + } await saveMessages({ messages: [ diff --git a/app/api/extend/route.ts b/app/api/extend/route.ts index ff80a7c16..61676c735 100644 --- a/app/api/extend/route.ts +++ b/app/api/extend/route.ts @@ -1,465 +1,110 @@ import { NextRequest, NextResponse } from "next/server"; -import { getSystemSetting } from "@/lib/db/queries"; import { auth } from "@/app/(auth)/auth"; -import { uploadFile } from "@/lib/storage/service"; -import { db } from "@/lib/db"; -import { generatedImage } from "@/lib/db/schema"; +import { + executeExtendJob, + ExtendServiceError, +} from "@/lib/ai/extend/service"; -/** - * 压缩图片生成缩略图（使用 sharp） - */ -async function createThumbnail(buffer: Buffer, mimeType: string): Promise { - try { - const sharp = await import("sharp"); - - // 创建缩略图（宽度 500px，高度按比例，质量 80%） - const thumbnailBuffer = await sharp.default(buffer) - .resize(500, null, { - fit: "inside", - withoutEnlargement: true, - }) - .jpeg({ quality: 80 }) // JPEG格式 - .toBuffer(); - - console.log(`[Extend] Thumbnail created, size: ${thumbnailBuffer.length} bytes (original: ${buffer.length} bytes)`); - return thumbnailBuffer; - } catch (error) { - console.error("[Extend] Failed to create thumbnail:", error); - // 如果失败，返回原图 - return buffer; - } -} - -/** - * 将 base64 图片上传到 S3，返回缩略图和原图的 URL - */ -async function uploadBase64ToS3(base64Data: string, mimeType: string = "image/jpeg"): Promise<{ thumbnail: string; original: string }> { - try { - // 解析 base64 数据 - const buffer = Buffer.from(base64Data, "base64"); - - // 生成文件名 - const ext = mimeType.split("/")[1] || "jpg"; - const timestamp = Date.now(); - const randomStr = Math.random().toString(36).slice(2, 8); - const thumbnailFilename = `extend-thumb-${timestamp}-${randomStr}.${ext}`; - const originalFilename = `extend-orig-${timestamp}-${randomStr}.${ext}`; - - console.log(`[Extend] Original size: ${buffer.length} bytes`); - - // 生成缩略图 - const thumbnailBuffer = await createThumbnail(buffer, mimeType); - - // 转换为 ArrayBuffer - const thumbnailArrayBuffer = thumbnailBuffer.buffer.slice( - thumbnailBuffer.byteOffset, - thumbnailBuffer.byteOffset + thumbnailBuffer.byteLength - ) as ArrayBuffer; - const originalArrayBuffer = buffer.buffer.slice( - buffer.byteOffset, - buffer.byteOffset + buffer.byteLength - ) as ArrayBuffer; - - // 并行上传缩略图和原图到 S3 - console.log(`[Extend] Uploading thumbnail and original to S3...`); - const [thumbnailResult, originalResult] = await Promise.all([ - uploadFile(thumbnailFilename, thumbnailArrayBuffer), - uploadFile(originalFilename, originalArrayBuffer), - ]); - - console.log(`[Extend] Uploaded to S3:`); - console.log(` - Thumbnail: ${thumbnailResult.url}`); - console.log(` - Original: ${originalResult.url}`); - - return { - thumbnail: thumbnailResult.url, - original: originalResult.url, - }; - } catch (error) { - console.error("[Extend] Failed to upload to S3:", error); - throw new Error(`图片上传到存储桶失败: ${error instanceof Error ? error.message : "未知错误"}。请检查系统设置中的存储配置。`); - } -} - -// 设置API路由最大执行时间为3分钟（180秒） export const maxDuration = 180; -/** - * 将图片上传到S3并返回公网URL（用于API调用） - */ -async function uploadImageToS3ForAPI(imageBase64: string, mimeType: string = "image/jpeg"): Promise { - try { - // 解析 base64 数据 - const buffer = Buffer.from(imageBase64, "base64"); - - // 生成文件名 - const ext = mimeType.split("/")[1] || "jpg"; - const timestamp = Date.now(); - const randomStr = Math.random().toString(36).slice(2, 8); - const filename = `extend-input-${timestamp}-${randomStr}.${ext}`; - - console.log(`[Extend API] 上传原图到S3，大小: ${(buffer.length / 1024).toFixed(2)}KB`); - - // 转换为 ArrayBuffer - const arrayBuffer = buffer.buffer.slice( - buffer.byteOffset, - buffer.byteOffset + buffer.byteLength - ) as ArrayBuffer; - - // 上传到 S3 - const result = await uploadFile(filename, arrayBuffer); - - console.log(`[Extend API] ✅ 原图已上传到S3: ${result.url}`); - - return result.url; - } catch (error) { - console.error("[Extend API] 上传到S3失败:", error); - throw new Error(`图片上传到存储桶失败: ${error instanceof Error ? error.message : "未知错误"}。请检查系统设置中的存储配置。`); - } -} - -/** - * 图像拓展核心实现（使用URL方式，避免Base64长度限制） - * @param imageUrl - 图片的公网URL - * @param apiKey - Dashscope API Key - * @param xScale - X轴放大倍数（默认1.5） - * @param yScale - Y轴放大倍数（默认1.5） - * @returns 拓展后的图片URL - */ -async function extendImage( - imageUrl: string, - apiKey: string, - xScale: number = 1.5, - yScale: number = 1.5 -): Promise { - console.log("[Extend API] 开始图像拓展..."); - console.log("[Extend API] 使用图片URL:", imageUrl); - - // 1. 提交拓展任务（使用URL方式，避免Base64长度限制） - const requestBody = { - model: "image-out-painting", - input: { - image_url: imageUrl, - }, - parameters: { - x_scale: xScale, - y_scale: yScale, - best_quality: false, - limit_image_size: true, - add_watermark: false, - }, - }; - - console.log("[Extend API] 提交拓展任务..."); - const response = await fetch( - "https://dashscope.aliyuncs.com/api/v1/services/aigc/image2image/out-painting", - { - method: "POST", - headers: { - "Content-Type": "application/json", - Authorization: `Bearer ${apiKey}`, - "X-DashScope-Async": "enable", - }, - body: JSON.stringify(requestBody), - } - ); - - const initialResult = await response.json(); - - if (initialResult.code) { - console.error("[Extend API] API调用失败:", initialResult); - throw new Error( - `图像拓展API错误: ${initialResult.message || JSON.stringify(initialResult)}` - ); - } - - // 2. 获取任务ID - const taskId = initialResult.output.task_id; - const taskStatusUrl = `https://dashscope.aliyuncs.com/api/v1/tasks/${taskId}`; - console.log(`[Extend API] 任务已提交，任务ID: ${taskId}`); - - // 3. 等待3秒后开始轮询 - console.log("[Extend API] 等待API初始化..."); - await new Promise((resolve) => setTimeout(resolve, 3000)); - - // 4. 轮询获取结果 - let attempts = 0; - const maxAttempts = 60; // 最多轮询60次 - const pollInterval = 2000; // 每2秒轮询一次 - - console.log("[Extend API] 开始轮询任务状态..."); - while (attempts < maxAttempts) { - const statusResponse = await fetch(taskStatusUrl, { - headers: { - Authorization: `Bearer ${apiKey}`, - }, - }); - - const result = await statusResponse.json(); - - if (attempts === 0 || attempts % 5 === 0) { - const elapsedTime = attempts * 2 + 3; - console.log(`[Extend API] [${elapsedTime}秒] 任务状态: ${result.output?.task_status || "UNKNOWN"}`); - } - - if (result.output?.task_status === "SUCCEEDED") { - const elapsedTime = attempts * 2 + 3; - console.log(`[Extend API] ✅ 任务完成（总用时 ${elapsedTime}秒）`); - return result.output.output_image_url; - } else if (result.output?.task_status === "FAILED") { - console.error("[Extend API] 任务失败:", result.output); - throw new Error( - `图像拓展任务失败: ${result.output.message || result.output.code || "未知错误"}` - ); - } - - attempts++; - if (attempts < maxAttempts) { - await new Promise((resolve) => setTimeout(resolve, pollInterval)); - } - } - - throw new Error(`图像拓展任务超时（等待${maxAttempts * 2 + 3}秒）`); -} - - -/** - * 图像拓展API路由 - * 从数据库读取图像拓展API配置，调用Dashscope API进行图像拓展处理 - * 将处理后的图片上传到S3并保存到数据库 - */ export async function POST(request: NextRequest) { - console.log("[Extend API] ===== 收到请求 ====="); try { - // 0. 获取用户会话 - console.log("[Extend API] 开始获取用户会话..."); const session = await auth(); - console.log("[Extend API] 用户会话获取完成:", session?.user?.id ? "已登录" : "未登录"); if (!session?.user?.id) { - return NextResponse.json( - { error: "未授权，请先登录" }, - { status: 401 } - ); + return NextResponse.json({ error: "未授权，请先登录" }, { status: 401 }); } - const userId = session.user.id; - console.log("[Extend API] 用户ID:", userId); - - // 1. 从数据库获取图像拓展API配置 - console.log("[Extend API] 开始获取配置..."); - const extendConfig = await getSystemSetting({ key: "extend_api_config" }); - console.log("[Extend API] 配置获取完成:", extendConfig ? "已找到" : "未找到"); - - if (!extendConfig || !extendConfig.value) { - return NextResponse.json( - { error: "图像拓展API配置未设置，请在系统设置中配置" }, - { status: 400 } - ); - } - - const config = extendConfig.value as { - apiKey?: string; - xScale?: number; - yScale?: number; - }; - - if (!config.apiKey) { - return NextResponse.json( - { error: "图像拓展API密钥未配置" }, - { status: 400 } - ); - } - - // 2. 获取请求体（Base64图片或FormData） - console.log("[Extend API] 开始获取请求体..."); const contentType = request.headers.get("content-type") || ""; - console.log("[Extend API] Content-Type:", contentType); - - let imageBase64: string; - let prompt = ""; // 用户提示词（可选） - let chatId: string | null = null; // 聊天ID（可选） + + let imageUrl: string | null = null; + let imageBase64: string | null = null; + let prompt = "图像拓展处理"; + let chatId: string | null = null; let messageId: string | null = null; if (contentType.includes("application/json")) { - // JSON格式：支持Base64图片或图片URL const body = await request.json(); - const imageUrl = body.image_url; // 优先使用URL - imageBase64 = body.image_base64; // 如果没有URL，使用Base64 - prompt = body.prompt || "图像拓展处理"; + imageUrl = body.image_url ? String(body.image_url).trim() : null; + imageBase64 = body.image_base64 ? String(body.image_base64).trim() : null; + prompt = body.prompt || prompt; chatId = body.chatId || null; messageId = body.messageId || null; - - // 如果提供了URL，直接使用；否则需要Base64 - if (imageUrl) { - console.log("[Extend API] 使用图片URL:", imageUrl); - // 直接使用URL，跳过Base64转换和上传步骤 - const startTime = Date.now(); - const xScale = config.xScale || 1.5; - const yScale = config.yScale || 1.5; - - console.log(`[Extend API] 拓展参数: X=${xScale}, Y=${yScale}`); - - const extendedImageUrl = await extendImage(imageUrl, config.apiKey, xScale, yScale); - console.log("[Extend API] 拓展后图片URL:", extendedImageUrl); - // 下载拓展后的图片 - console.log("[Extend API] 下载拓展后的图片..."); - const imageResponse = await fetch(extendedImageUrl); - if (!imageResponse.ok) { - throw new Error(`下载拓展图片失败: ${imageResponse.status} ${imageResponse.statusText}`); - } - - const arrayBuffer = await imageResponse.arrayBuffer(); - const buffer = Buffer.from(arrayBuffer); - const extendedImageBase64 = buffer.toString("base64"); - - // 上传到S3 - console.log("[Extend API] 上传到S3..."); - const { thumbnail, original } = await uploadBase64ToS3(extendedImageBase64, "image/jpeg"); - - // 保存到数据库（相册和生图记录） - console.log("[Extend API] 保存到数据库..."); - let savedImage; - try { - [savedImage] = await db - .insert(generatedImage) - .values({ - userId, - chatId: chatId || null, - messageId: messageId || null, - thumbnailUrl: thumbnail, - originalUrl: original, - prompt: prompt || "图像拓展处理", - modelId: "extend-1.0", - imageSize: "2K", - }) - .returning(); - console.log(`[Extend API] ✅ 图片已保存到相册和生图记录，ID: ${savedImage.id}`); - } catch (dbError) { - console.error("[Extend API] ❌ 保存到数据库失败:", dbError); - } - - const processingTime = ((Date.now() - startTime) / 1000).toFixed(2); - console.log(`[Extend API] ✅ 图像拓展完成，处理时间: ${processingTime}秒`); - - // 返回结果 - return NextResponse.json({ - success: true, - original_url: original, - thumbnail_url: thumbnail, - base64: extendedImageBase64, - processing_time: parseFloat(processingTime), - }); - } - - if (!imageBase64) { + if (!imageUrl && !imageBase64) { return NextResponse.json( { error: "缺少image_base64或image_url参数" }, - { status: 400 } - ); - } - } else if (contentType.includes("multipart/form-data")) { - // FormData格式：文件上传 - const formData = await request.formData(); - const file = formData.get("file") as File; - - if (!file) { - return NextResponse.json( - { error: "缺少file参数" }, - { status: 400 } + { status: 400 }, ); } - // 将文件转换为Base64 + const result = await executeExtendJob({ + userId: session.user.id, + imageUrl, + imageBase64, + prompt, + chatId, + messageId, + xScale: body.xScale ? Number(body.xScale) : undefined, + yScale: body.yScale ? Number(body.yScale) : undefined, + }); + + return NextResponse.json({ + success: true, + original_url: result.original_url, + thumbnail_url: result.thumbnail_url, + base64: result.image_base64, + processing_time: result.processing_time, + }); + } + + if (contentType.includes("multipart/form-data")) { + const formData = await request.formData(); + const file = formData.get("file") as File | null; + if (!file) { + return NextResponse.json({ error: "缺少file参数" }, { status: 400 }); + } + const arrayBuffer = await file.arrayBuffer(); const buffer = Buffer.from(arrayBuffer); - imageBase64 = buffer.toString("base64"); - prompt = (formData.get("prompt") as string) || "图像拓展处理"; - chatId = (formData.get("chatId") as string) || null; - messageId = (formData.get("messageId") as string) || null; - } else { + const result = await executeExtendJob({ + userId: session.user.id, + imageBase64: buffer.toString("base64"), + prompt: String(formData.get("prompt") || prompt), + chatId: String(formData.get("chatId") || "") || null, + messageId: String(formData.get("messageId") || "") || null, + }); + + return NextResponse.json({ + success: true, + original_url: result.original_url, + thumbnail_url: result.thumbnail_url, + base64: result.image_base64, + processing_time: result.processing_time, + }); + } + + return NextResponse.json( + { error: "不支持的Content-Type，请使用application/json或multipart/form-data" }, + { status: 400 }, + ); + } catch (error) { + if (error instanceof ExtendServiceError) { return NextResponse.json( - { error: "不支持的Content-Type，请使用application/json或multipart/form-data" }, - { status: 400 } + { + error: error.message, + detail: error.detail || error.message, + }, + { status: error.status }, ); } - console.log(`[Extend API] 图片Base64长度: ${imageBase64.length} 字符`); - - // 3. 先将图片上传到S3获取公网URL（使用URL方式避免Base64长度限制） - console.log("[Extend API] 上传原图到S3获取公网URL..."); - const inputImageUrl = await uploadImageToS3ForAPI(imageBase64, "image/jpeg"); - console.log("[Extend API] 原图URL:", inputImageUrl); - - // 4. 调用图像拓展API（使用URL方式） - const startTime = Date.now(); - const xScale = config.xScale || 1.5; - const yScale = config.yScale || 1.5; - - console.log(`[Extend API] 拓展参数: X=${xScale}, Y=${yScale}`); - - const extendedImageUrl = await extendImage(inputImageUrl, config.apiKey, xScale, yScale); - console.log("[Extend API] 拓展后图片URL:", extendedImageUrl); - - // 4. 下载拓展后的图片 - console.log("[Extend API] 下载拓展后的图片..."); - const imageResponse = await fetch(extendedImageUrl); - if (!imageResponse.ok) { - throw new Error(`下载拓展图片失败: ${imageResponse.status} ${imageResponse.statusText}`); - } - - const arrayBuffer = await imageResponse.arrayBuffer(); - const buffer = Buffer.from(arrayBuffer); - const extendedImageBase64 = buffer.toString("base64"); - - // 5. 上传到S3 - console.log("[Extend API] 上传到S3..."); - const { thumbnail, original } = await uploadBase64ToS3(extendedImageBase64, "image/jpeg"); - - // 6. 保存到数据库（相册和生图记录） - console.log("[Extend API] 保存到数据库..."); - let savedImage; - try { - [savedImage] = await db - .insert(generatedImage) - .values({ - userId, - chatId: chatId || null, - messageId: messageId || null, - thumbnailUrl: thumbnail, - originalUrl: original, - prompt: prompt || "图像拓展处理", - modelId: "extend-1.0", - imageSize: "2K", - }) - .returning(); - console.log(`[Extend API] ✅ 图片已保存到相册和生图记录，ID: ${savedImage.id}`); - } catch (dbError) { - console.error("[Extend API] ❌ 保存到数据库失败:", dbError); - // 数据库保存失败时仍然返回成功，但记录错误（图片已上传到S3，用户可以访问） - } - - const processingTime = ((Date.now() - startTime) / 1000).toFixed(2); - console.log(`[Extend API] ✅ 图像拓展完成，处理时间: ${processingTime}秒`); - - // 7. 返回结果 - return NextResponse.json({ - success: true, - original_url: original, - thumbnail_url: thumbnail, - base64: extendedImageBase64, - processing_time: parseFloat(processingTime), - }); - } catch (error) { console.error("[Extend API] 错误:", error); return NextResponse.json( { error: "图像拓展失败", detail: error instanceof Error ? error.message : String(error), }, - { status: 500 } + { status: 500 }, ); } } - - diff --git a/app/api/resize/route.ts b/app/api/resize/route.ts index a9fc95782..dfb0aaac1 100644 --- a/app/api/resize/route.ts +++ b/app/api/resize/route.ts @@ -1,225 +1,66 @@ import { NextRequest, NextResponse } from "next/server"; import { auth } from "@/app/(auth)/auth"; -import { uploadFile } from "@/lib/storage/service"; -import { db } from "@/lib/db"; -import { generatedImage } from "@/lib/db/schema"; -import { upsertMediaAsset } from "@/lib/db/queries"; +import { + executeResizeJob, + ResizeServiceError, +} from "@/lib/ai/resize/service"; -/** - * 压缩图片生成缩略图（使用 sharp） - */ -async function createThumbnail(buffer: Buffer, mimeType: string): Promise { - try { - const sharp = await import("sharp"); - - // 创建缩略图（宽度 500px，高度按比例，质量 80%） - const thumbnailBuffer = await sharp.default(buffer) - .resize(500, null, { - fit: "inside", - withoutEnlargement: true, - }) - .png({ quality: 80 }) // PNG格式保持透明背景 - .toBuffer(); - - console.log(`[Resize] Thumbnail created, size: ${thumbnailBuffer.length} bytes (original: ${buffer.length} bytes)`); - return thumbnailBuffer; - } catch (error) { - console.error("[Resize] Failed to create thumbnail:", error); - // 如果失败，返回原图 - return buffer; - } -} - -/** - * 将 base64 图片上传到 S3，返回缩略图和原图的 URL - */ -async function uploadBase64ToS3(base64Data: string, mimeType: string = "image/png"): Promise<{ thumbnail: string; original: string }> { - try { - // 解析 base64 数据 - const buffer = Buffer.from(base64Data, "base64"); - - // 生成文件名 - const ext = mimeType.split("/")[1] || "png"; - const timestamp = Date.now(); - const randomStr = Math.random().toString(36).slice(2, 8); - const thumbnailFilename = `resize-thumb-${timestamp}-${randomStr}.${ext}`; - const originalFilename = `resize-orig-${timestamp}-${randomStr}.${ext}`; - - console.log(`[Resize] Original size: ${buffer.length} bytes`); - - // 生成缩略图 - const thumbnailBuffer = await createThumbnail(buffer, mimeType); - - // 转换为 ArrayBuffer - const thumbnailArrayBuffer = thumbnailBuffer.buffer.slice( - thumbnailBuffer.byteOffset, - thumbnailBuffer.byteOffset + thumbnailBuffer.byteLength - ) as ArrayBuffer; - const originalArrayBuffer = buffer.buffer.slice( - buffer.byteOffset, - buffer.byteOffset + buffer.byteLength - ) as ArrayBuffer; - - // 并行上传缩略图和原图到 S3 - console.log(`[Resize] Uploading thumbnail and original to S3...`); - const [thumbnailResult, originalResult] = await Promise.all([ - uploadFile(thumbnailFilename, thumbnailArrayBuffer), - uploadFile(originalFilename, originalArrayBuffer), - ]); - - console.log(`[Resize] Uploaded to S3:`); - console.log(` - Thumbnail: ${thumbnailResult.url}`); - console.log(` - Original: ${originalResult.url}`); - - return { - thumbnail: thumbnailResult.url, - original: originalResult.url, - }; - } catch (error) { - console.error("[Resize] Failed to upload to S3:", error); - throw new Error(`图片上传到存储桶失败: ${error instanceof Error ? error.message : "未知错误"}。请检查系统设置中的存储配置。`); - } -} - -/** - * 尺寸调整API路由 - * 接收前端处理后的Base64图片，上传到S3并保存到数据库 - */ export async function POST(request: NextRequest) { try { - // 0. 获取用户会话 const session = await auth(); if (!session?.user?.id) { - return NextResponse.json( - { error: "未授权，请先登录" }, - { status: 401 } - ); + return NextResponse.json({ error: "未授权，请先登录" }, { status: 401 }); } - const userId = session.user.id; - - // 1. 获取请求体 const contentType = request.headers.get("content-type") || ""; - - let imageBase64: string; - let prompt = "尺寸调整处理"; - let chatId: string | null = null; - let messageId: string | null = null; - let canvasWidth: number; - let canvasHeight: number; - let mimeType = "image/png"; - - if (contentType.includes("application/json")) { - const body = await request.json(); - imageBase64 = body.image_base64; - prompt = body.prompt || "尺寸调整处理"; - chatId = body.chatId || null; - messageId = body.messageId || null; - canvasWidth = body.canvasWidth; - canvasHeight = body.canvasHeight; - mimeType = body.mimeType || "image/png"; - - if (!imageBase64) { - return NextResponse.json( - { error: "缺少image_base64参数" }, - { status: 400 } - ); - } - - if (!canvasWidth || !canvasHeight) { - return NextResponse.json( - { error: "缺少canvasWidth或canvasHeight参数" }, - { status: 400 } - ); - } - } else { + if (!contentType.includes("application/json")) { return NextResponse.json( { error: "不支持的Content-Type，请使用application/json" }, - { status: 400 } + { status: 400 }, ); } - console.log(`[Resize API] 图片Base64长度: ${imageBase64.length} 字符`); - console.log(`[Resize API] 画布尺寸: ${canvasWidth}x${canvasHeight}`); + const body = await request.json(); + const imageBase64 = String(body.image_base64 || "").trim(); + const canvasWidth = Number(body.canvasWidth); + const canvasHeight = Number(body.canvasHeight); - // 2. 上传处理后的图片到S3存储桶 - let thumbnailUrl: string; - let originalUrl: string; - - try { - const uploadResult = await uploadBase64ToS3(imageBase64, mimeType); - thumbnailUrl = uploadResult.thumbnail; - originalUrl = uploadResult.original; - console.log(`[Resize] ✅ Images uploaded to S3 successfully`); - } catch (uploadError) { - console.error("[Resize] ❌ Failed to upload to S3:", uploadError); - return NextResponse.json( - { - error: "图片上传到存储桶失败", - detail: uploadError instanceof Error ? uploadError.message : "未知错误" - }, - { status: 500 } - ); + if (!imageBase64) { + return NextResponse.json({ error: "缺少image_base64参数" }, { status: 400 }); } - // 3. 保存图片记录到数据库（相册和生图记录） - try { - const [savedImage] = await db.insert(generatedImage).values({ - userId, - chatId: chatId || null, - messageId: messageId || null, - thumbnailUrl, - originalUrl, - imageSize: `${canvasWidth}x${canvasHeight}`, // 使用实际尺寸 - prompt: prompt || "尺寸调整处理", - modelId: "resize-1.0", // 标记为尺寸调整处理 - }).returning(); - console.log(`[Resize] ✅ 图片已保存到相册和生图记录，ID: ${savedImage.id}`); + const result = await executeResizeJob({ + userId: session.user.id, + imageBuffer: Buffer.from(imageBase64, "base64"), + prompt: body.prompt || "尺寸调整处理", + chatId: body.chatId || null, + messageId: body.messageId || null, + canvasWidth, + canvasHeight, + mimeType: body.mimeType || "image/png", + transparentBackground: body.transparentBackground === true, + backgroundColor: body.backgroundColor || undefined, + }); - try { - await upsertMediaAsset({ - userId, - chatId, - mediaType: "image", - sourceType: "edited", - status: "ready", - originalUrl, - previewUrl: originalUrl, - thumbnailUrl, - displayName: (prompt || "尺寸调整处理").slice(0, 80), - prompt: prompt || "尺寸调整处理", - modelId: "resize-1.0", - sourceMessageId: messageId, - sourceTable: "GeneratedImage", - sourceRowId: savedImage.id, - metadata: { - imageSize: `${canvasWidth}x${canvasHeight}`, - canvasWidth, - canvasHeight, - }, - }); - } catch (assetError) { - console.error("[Resize] ⚠️ MediaAsset 注册失败:", assetError); - } - } catch (dbError) { - console.error("[Resize] ❌ 保存到数据库失败:", dbError); - // 数据库保存失败时仍然返回成功，但记录错误（图片已上传到S3，用户可以访问） - } - - // 4. 返回结果（包含S3 URL和Base64，前端可以选择使用） return NextResponse.json({ success: true, - image_base64: imageBase64, // 保留Base64用于前端立即显示 - thumbnail_url: thumbnailUrl, // S3缩略图URL - original_url: originalUrl, // S3原图URL - canvas_width: canvasWidth, - canvas_height: canvasHeight, + ...result, }); } catch (error) { + if (error instanceof ResizeServiceError) { + return NextResponse.json( + { + error: error.message, + detail: error.detail || error.message, + }, + { status: error.status }, + ); + } + console.error("[Resize API] 处理错误:", error); return NextResponse.json( { error: "服务器内部错误", detail: error instanceof Error ? error.message : "未知错误" }, - { status: 500 } + { status: 500 }, ); } } diff --git a/components/canvas/CanvasChatPanel.tsx b/components/canvas/CanvasChatPanel.tsx index 269a2ae26..b35b01f51 100644 --- a/components/canvas/CanvasChatPanel.tsx +++ b/components/canvas/CanvasChatPanel.tsx @@ -82,6 +82,21 @@ interface TagPreviewInfo { }; } +type CanvasMediaStreamPayload = { + messageId?: string; + imageUrls?: string[]; + videoUrls?: string[]; + prompt?: string; + sourceImageUrls?: string[]; +}; + +type ImageGenerationStatusStreamPayload = { + phase?: "queued" | "processing" | "finalizing" | "failed" | "completed"; + providerType?: string; + modelName?: string; + imageCount?: number; +}; + interface CanvasChatPanelProps { chatId: string; initialMessages?: ChatMessage[]; @@ -1665,6 +1680,8 @@ export function CanvasChatPanel({ chatId, initialMessages = [], onClose, onOpenF unitCost: number; imageCount: number; } | null>(null); + const [streamImageGenerationState, setStreamImageGenerationState] = + useState(null); const imageParamsRef = useRef(imageParams); const videoParamsRef = useRef(videoParams); const deepThinkingEnabledRef = useRef(deepThinkingEnabled); @@ -2723,22 +2740,27 @@ export function CanvasChatPanel({ chatId, initialMessages = [], onClose, onOpenF currentModel.isMultimodal && !VIDEO_PROVIDER_TYPES.has(currentModel.providerType) ); + const isAgentModel = !isDetailGenerationMode && (selectedModelId === "auto" || isConversationModel); + const agentSelectableModels = useMemo( + () => [SMART_AUTO_MODEL, ...conversationOnlyModels], + [SMART_AUTO_MODEL, conversationOnlyModels] + ); const selectableModels = useMemo(() => { if (isDetailGenerationMode) { return [SMART_AUTO_MODEL]; } - if (isConversationModel) { - return conversationOnlyModels; - } if (isVideoModel) { return videoOnlyModels; } - return [SMART_AUTO_MODEL, ...imageOnlyModels]; + if (isAgentModel) { + return agentSelectableModels; + } + return imageOnlyModels; }, [ SMART_AUTO_MODEL, - conversationOnlyModels, + agentSelectableModels, imageOnlyModels, - isConversationModel, + isAgentModel, isDetailGenerationMode, isVideoModel, videoOnlyModels, @@ -2751,8 +2773,8 @@ export function CanvasChatPanel({ chatId, initialMessages = [], onClose, onOpenF currentModel || SMART_AUTO_MODEL; const agentDisplayName = - modelsData?.agentModel?.displayName || selectDisplayModel?.displayName || "Agent模式"; - const agentAvatarUrl = modelsData?.agentModel?.avatarUrl || selectDisplayModel?.avatarUrl || null; + currentModel?.displayName || selectDisplayModel?.displayName || "Agent模式"; + const agentAvatarUrl = currentModel?.avatarUrl || selectDisplayModel?.avatarUrl || null; const videoFrameSlots = useMemo(() => { if (!isVideoModel || !videoParams) return [] as string[]; if (videoParams.mode === "first-last-frame") return ["首帧", "尾帧"] as string[]; @@ -2901,28 +2923,21 @@ export function CanvasChatPanel({ chatId, initialMessages = [], onClose, onOpenF () => imageOnlyModels[0]?.id || null, [imageOnlyModels] ); - const firstConversationModelId = useMemo( - () => conversationOnlyModels[0]?.id || null, - [conversationOnlyModels] - ); const firstVideoModelId = useMemo( () => videoOnlyModels[0]?.id || null, [videoOnlyModels] ); - const activeComposerMode = useMemo<"agent" | "chat" | "image" | "video" | "detail">(() => { + const activeComposerMode = useMemo<"agent" | "image" | "video" | "detail">(() => { if (isDetailGenerationMode) return "detail"; - if (selectedModelId === "auto") return "agent"; if (isVideoModel) return "video"; - if (isConversationModel) return "chat"; + if (isAgentModel) return "agent"; return "image"; - }, [isConversationModel, isDetailGenerationMode, isVideoModel, selectedModelId]); + }, [isAgentModel, isDetailGenerationMode, isVideoModel]); const activeModeLabel = activeComposerMode === "video" ? "视频" : activeComposerMode === "detail" ? "AI详情" - : activeComposerMode === "chat" - ? "对话" : activeComposerMode === "image" ? "生图" : "Agent"; @@ -3025,10 +3040,17 @@ export function CanvasChatPanel({ chatId, initialMessages = [], onClose, onOpenF }, [currentModel, isConversationModel, isDetailGenerationMode, isVideoModel]); const handleSelectGenerationMode = useCallback( - (mode: "agent" | "chat" | "image" | "video" | "detail") => { + (mode: "agent" | "image" | "video" | "detail") => { if (mode === "agent") { - currentModelIdRef.current = "auto"; - setSelectedModelId("auto"); + const preferredAgentModelId = + allModels.find( + (model) => + model.id === lastConversationModelId && + model.isMultimodal && + !VIDEO_PROVIDER_TYPES.has(model.providerType) + )?.id || "auto"; + currentModelIdRef.current = preferredAgentModelId; + setSelectedModelId(preferredAgentModelId); setIsDetailGenerationMode(false); return; } @@ -3045,25 +3067,6 @@ export function CanvasChatPanel({ chatId, initialMessages = [], onClose, onOpenF return; } - if (mode === "chat") { - const preferredConversationModelId = - allModels.find( - (model) => - model.id === lastConversationModelId && - model.isMultimodal && - !VIDEO_PROVIDER_TYPES.has(model.providerType) - )?.id || firstConversationModelId; - - if (!preferredConversationModelId) { - toast.error("当前没有可用的多模态模型"); - return; - } - currentModelIdRef.current = preferredConversationModelId; - setSelectedModelId(preferredConversationModelId); - setIsDetailGenerationMode(false); - return; - } - if (mode === "image") { const preferredImageModelId = allModels.find( @@ -3100,7 +3103,6 @@ export function CanvasChatPanel({ chatId, initialMessages = [], onClose, onOpenF [ aiDetailGenerationEnabled, allModels, - firstConversationModelId, firstImageModelId, firstVideoModelId, lastConversationModelId, @@ -3264,6 +3266,159 @@ export function CanvasChatPanel({ chatId, initialMessages = [], onClose, onOpenF // 合并 initialMessages 和历史消息 const mergedInitialMessages = historyData?.messages ?? (activeChatId === chatId ? initialMessages : []); + + const appendStreamMediaToCanvas = useCallback( + async (payload: CanvasMediaStreamPayload) => { + const imageUrls = Array.isArray(payload.imageUrls) + ? payload.imageUrls.filter((url): url is string => typeof url === "string" && /^https?:\/\//i.test(url)) + : []; + const videoUrls = Array.isArray(payload.videoUrls) + ? payload.videoUrls.filter((url): url is string => typeof url === "string" && /^https?:\/\//i.test(url)) + : []; + if (imageUrls.length === 0 && videoUrls.length === 0) { + return; + } + + const latestNodes = useCanvasStore.getState().nodes; + const existingImageUrlKeys = new Set( + latestNodes + .filter((node) => node.type === "image" && (node.data as any)?.src) + .map((node) => normalizeImageIdentityUrl(String((node.data as any)?.src || ""))) + .filter(Boolean) + ); + const existingMessageImageKeys = new Set( + latestNodes + .filter((node) => node.type === "image") + .map((node) => { + const data = ((node.data as any) || {}) as Record; + const metadata = ((data.metadata as any) || {}) as Record; + const sourceMessageId = String(metadata.sourceMessageId || "").trim(); + const sourceImageKey = normalizeImageIdentityUrl( + String(metadata.sourceImageKey || data.src || "").trim() + ); + if (!sourceMessageId || !sourceImageKey) return ""; + return `${sourceMessageId}::${sourceImageKey}`; + }) + .filter(Boolean) + ); + const existingVideoUrlKeys = new Set( + latestNodes + .filter((node) => node.type === "video" && (node.data as any)?.src) + .map((node) => normalizeMediaUrl(String((node.data as any)?.src || ""))) + .filter(Boolean) + ); + + const sourceImageUrls = Array.isArray(payload.sourceImageUrls) + ? payload.sourceImageUrls.filter((url): url is string => typeof url === "string" && /^https?:\/\//i.test(url)) + : []; + const messageId = String(payload.messageId || "").trim(); + const generationPrompt = String(payload.prompt || "").trim() || "对话生成"; + let addedImageCount = 0; + let addedVideoCount = 0; + + for (let index = 0; index < imageUrls.length; index += 1) { + const url = imageUrls[index]!; + const sourceImageKey = normalizeImageIdentityUrl(url); + if (!sourceImageKey || existingImageUrlKeys.has(sourceImageKey)) { + continue; + } + const messageImageKey = messageId ? `${messageId}::${sourceImageKey}` : ""; + if (messageImageKey && existingMessageImageKeys.has(messageImageKey)) { + continue; + } + + existingImageUrlKeys.add(sourceImageKey); + if (messageImageKey) { + existingMessageImageKeys.add(messageImageKey); + } + + const dimensions = await getImageDimensions(url); + const currentNodes = useCanvasStore.getState().nodes; + const existingNodes = currentNodes.length + addedImageCount + addedVideoCount; + const spacing = 20; + const x = 100 + (existingNodes % 3) * (dimensions.width + spacing); + const y = 100 + Math.floor(existingNodes / 3) * (dimensions.height + spacing); + const compareSourceUrlRaw = + sourceImageUrls[Math.min(index, Math.max(0, sourceImageUrls.length - 1))] || + sourceImageUrls[0] || + ""; + const compareSourceUrl = String(compareSourceUrlRaw || "").trim(); + const normalizedCompareSource = normalizeImageIdentityUrl(compareSourceUrl); + const canUseCompareSource = + Boolean(normalizedCompareSource) && + normalizedCompareSource !== sourceImageKey; + const sourceNodeId = canUseCompareSource + ? useCanvasStore + .getState() + .nodes.find((node) => { + if (node.type !== "image") return false; + const src = String(((node.data as any) || {}).src || "").trim(); + return normalizeImageIdentityUrl(src) === normalizedCompareSource; + })?.id || "" + : ""; + + const nodeId = addNode({ + type: "image", + position: { x, y }, + size: { width: dimensions.width, height: dimensions.height }, + data: { + src: url, + status: "completed", + title: "图片", + prompt: generationPrompt, + originalSrc: canUseCompareSource ? compareSourceUrl : undefined, + metadata: { + originalWidth: dimensions.originalWidth, + originalHeight: dimensions.originalHeight, + generationPrompt, + sourceMessageId: messageId || undefined, + sourceImageKey, + compareSourceUrl: canUseCompareSource ? compareSourceUrl : undefined, + }, + }, + connections: sourceNodeId ? [sourceNodeId] : [], + }); + void autoNameGeneratedImageNode(nodeId, url); + addedImageCount += 1; + } + + for (let index = 0; index < videoUrls.length; index += 1) { + const url = videoUrls[index]!; + const key = normalizeMediaUrl(url); + if (!key || existingVideoUrlKeys.has(key)) { + continue; + } + existingVideoUrlKeys.add(key); + const currentNodes = useCanvasStore.getState().nodes; + const existingNodes = currentNodes.length + addedImageCount + addedVideoCount; + const x = 120 + (existingNodes % 2) * 420; + const y = 120 + Math.floor(existingNodes / 2) * 260; + addNode({ + type: "video", + position: { x, y }, + size: { width: 380, height: 220 }, + data: { + src: url, + status: "completed", + prompt: generationPrompt, + }, + connections: [], + }); + addedVideoCount += 1; + } + + if (addedImageCount > 0 || addedVideoCount > 0) { + const notice = [ + addedImageCount > 0 ? `${addedImageCount} 张图片` : "", + addedVideoCount > 0 ? `${addedVideoCount} 条视频` : "", + ] + .filter(Boolean) + .join("、"); + toast.success(`已将 ${notice} 添加到画布`); + } + }, + [addNode, autoNameGeneratedImageNode] + ); const { messages, @@ -3332,12 +3487,79 @@ export function CanvasChatPanel({ chatId, initialMessages = [], onClose, onOpenF }, }), onData: (dataPart: any) => { + if (dataPart.type === "data-imageGenerationStatus") { + try { + const payload: ImageGenerationStatusStreamPayload = + typeof dataPart.data === "string" ? JSON.parse(dataPart.data) : dataPart.data; + if (!payload || typeof payload !== "object") return; + if (payload.phase === "completed") { + setStreamImageGenerationState(null); + return; + } + setStreamImageGenerationState({ + phase: payload.phase || "processing", + providerType: payload.providerType, + modelName: payload.modelName, + imageCount: Number(payload.imageCount || 0) || undefined, + }); + } catch (e) { + console.error("[CanvasChat] Failed to parse data-imageGenerationStatus:", e); + } + return; + } + + if (dataPart.type === "data-canvasMedia") { + try { + const payload: CanvasMediaStreamPayload = + typeof dataPart.data === "string" ? JSON.parse(dataPart.data) : dataPart.data; + if (!payload || typeof payload !== "object") return; + void appendStreamMediaToCanvas(payload); + } catch (e) { + console.error("[CanvasChat] Failed to parse data-canvasMedia:", e); + } + return; + } + // 处理后端返回的自定义消息 if (dataPart.type === "data-appendMessage") { try { const messageData = typeof dataPart.data === 'string' ? JSON.parse(dataPart.data) : dataPart.data; + const hasImageGenStartMarker = Array.isArray(messageData.parts) + ? messageData.parts.some( + (part: any) => + part?.type === "text" && + typeof part?.text === "string" && + part.text.includes("") + ) + : false; + const hasImageGenEndMarker = Array.isArray(messageData.parts) + ? messageData.parts.some( + (part: any) => + part?.type === "text" && + typeof part?.text === "string" && + part.text.includes("") + ) + : false; + if (hasImageGenStartMarker) { + setStreamImageGenerationState((prev) => ({ + phase: "processing", + providerType: + prev?.providerType || + (activeComposerMode === "agent" ? "alibaba-image" : currentModel?.providerType), + modelName: + prev?.modelName || + (activeComposerMode === "agent" + ? "智能处理" + : currentModel?.displayName || undefined), + imageCount: + prev?.imageCount || Math.max(1, Number(imageParamsRef.current?.imageCount || 1)), + })); + } + if (hasImageGenEndMarker) { + setStreamImageGenerationState(null); + } const normalizedParts = Array.isArray(messageData.parts) ? messageData.parts : []; const normalizedAttachments = Array.isArray(messageData.attachments) ? messageData.attachments @@ -5950,7 +6172,7 @@ export function CanvasChatPanel({ chatId, initialMessages = [], onClose, onOpenF ]); const ensureImagePointsBeforeSend = useCallback(() => { - if (isVideoModel || activeComposerMode === "chat") return true; + if (isVideoModel || activeComposerMode === "agent") return true; const pointsEnabled = membershipSummary?.pointsSystemConfig?.enabled === true; const blockOnInsufficient = @@ -6169,14 +6391,15 @@ export function CanvasChatPanel({ chatId, initialMessages = [], onClose, onOpenF return; } - sendGuardRef.current = { - inFlight: true, - signature: sendSignature, - timestamp: Date.now(), - }; + sendGuardRef.current = { + inFlight: true, + signature: sendSignature, + timestamp: Date.now(), + }; + setStreamImageGenerationState(null); - // 使用与 multimodal-input 相同的格式发送消息 - try { + // 使用与 multimodal-input 相同的格式发送消息 + try { sendMessage({ role: "user", parts: parts, @@ -6633,10 +6856,27 @@ export function CanvasChatPanel({ chatId, initialMessages = [], onClose, onOpenF (part: any) => part.type === "text" && part.text?.includes("") ) ); - const shouldShowImageLoading = hasGenerationStart && !hasGenerationEnd && !!currentModel; + const shouldShowImageLoading = + ((hasGenerationStart && !hasGenerationEnd) || Boolean(streamImageGenerationState)) && + !!currentModel; const hasActiveTypewriter = Object.keys(activeTypingSections).length > 0; const shouldRenderImageLoading = shouldShowImageLoading && !hasActiveTypewriter; - const loadingImageCount = imageParams.imageCount || 1; + const loadingImageCount = streamImageGenerationState?.imageCount || imageParams.imageCount || 1; + const loadingProviderType = + streamImageGenerationState?.providerType || + (activeComposerMode === "agent" ? "alibaba-image" : currentModel?.providerType); + const loadingModelName = + streamImageGenerationState?.modelName || + (activeComposerMode === "agent" + ? "智能处理" + : currentModel?.displayName || undefined); + const loadingPhase = + streamImageGenerationState?.phase === "queued" || + streamImageGenerationState?.phase === "processing" || + streamImageGenerationState?.phase === "finalizing" || + streamImageGenerationState?.phase === "failed" + ? streamImageGenerationState.phase + : "processing"; const renderableVisibleMessages = useMemo(() => { return visibleMessages.reduce((acc, message) => { const rawParts = (message.parts || []) as any[]; @@ -7474,8 +7714,9 @@ export function CanvasChatPanel({ chatId, initialMessages = [], onClose, onOpenF )} @@ -8498,8 +8739,6 @@ export function CanvasChatPanel({ chatId, initialMessages = [], onClose, onOpenF ? "border border-blue-500 bg-blue-50 text-blue-600 hover:bg-blue-100" : activeComposerMode === "detail" ? "border border-violet-500 bg-violet-50 text-violet-700 hover:bg-violet-100" - : activeComposerMode === "chat" - ? "border border-sky-500 bg-sky-50 text-sky-700 hover:bg-sky-100" : activeComposerMode === "image" ? "border border-purple-500 bg-purple-50 text-purple-700 hover:bg-purple-100" : "border border-[#9eb3ff] bg-white text-zinc-700 hover:bg-zinc-50" @@ -8510,8 +8749,6 @@ export function CanvasChatPanel({ chatId, initialMessages = [], onClose, onOpenF