9513 lines
400 KiB
TypeScript
9513 lines
400 KiB
TypeScript
import { createLogger } from "@/lib/observability/logger";
|
||
import { geolocation } from "@vercel/functions";
|
||
import {
|
||
convertToModelMessages,
|
||
createUIMessageStream,
|
||
generateText,
|
||
smoothStream,
|
||
stepCountIs,
|
||
streamText,
|
||
} from "ai";
|
||
import { eq, sql } from "drizzle-orm";
|
||
import { unstable_cache as cache } from "next/cache";
|
||
import { after, NextResponse } from "next/server";
|
||
import {
|
||
createResumableStreamContext,
|
||
type ResumableStreamContext,
|
||
} from "resumable-stream";
|
||
import type { ModelCatalog } from "tokenlens/core";
|
||
import { fetchModels } from "tokenlens/fetch";
|
||
import { getUsage } from "tokenlens/helpers";
|
||
import { auth, type UserType } from "@/app/(auth)/auth";
|
||
import type { VisibilityType } from "@/components/visibility-selector";
|
||
import { entitlementsByUserType } from "@/lib/ai/entitlements";
|
||
import type { ChatModel } from "@/lib/ai/models";
|
||
import { type RequestHints, systemPrompt } from "@/lib/ai/prompts";
|
||
import { myProvider } from "@/lib/ai/providers";
|
||
import {
|
||
isFalGptImage2EditModelId,
|
||
submitFalImageTask,
|
||
} from "@/lib/ai/providers/fal-image";
|
||
import { getLanguageModelById } from "@/lib/ai/dynamic-provider";
|
||
import { shouldForceInlineVisionInputs } from "@/lib/ai/model-compat";
|
||
import { createDocument } from "@/lib/ai/tools/create-document";
|
||
import { getWeather } from "@/lib/ai/tools/get-weather";
|
||
import { requestSuggestions } from "@/lib/ai/tools/request-suggestions";
|
||
import { updateDocument } from "@/lib/ai/tools/update-document";
|
||
import {
|
||
executeDirectVideoGeneration,
|
||
generateVideo,
|
||
} from "@/lib/ai/tools/generate-video";
|
||
import {
|
||
getAgentRequiredUploadCount,
|
||
normalizeAgentPresetModeConfig,
|
||
} from "@/lib/agents/preset-mode-config";
|
||
import { buildAgentSceneImageRolePrompt } from "@/lib/agents/scene-image-role-prompt";
|
||
import { optimizeAgentSceneWorkflowPrompt } from "@/lib/agents/scene-workflow-prompt-optimizer";
|
||
import { getEnabledTools } from "@/lib/ai/tools-registry";
|
||
import { isProductionEnvironment } from "@/lib/constants";
|
||
import {
|
||
createStreamId,
|
||
deleteChatById,
|
||
getChatById,
|
||
getPointsSystemConfig,
|
||
getUserById,
|
||
getMessageCountByUserId,
|
||
getMessagesByChatId,
|
||
saveChat,
|
||
saveMessages,
|
||
updateMessageById,
|
||
updateChatLastContextById,
|
||
getAgentAiModelWithProvider,
|
||
getAiModelById,
|
||
getEnabledAiModelsWithProvider,
|
||
getSubModelsByParentModelId,
|
||
getAiProviderById,
|
||
checkUserImageQuota,
|
||
consumeUserPoints,
|
||
createAgentPresetExecutionSnapshot,
|
||
createMessageTargetResolution,
|
||
createMediaAssetRelation,
|
||
getAccessibleAgentPresetById,
|
||
getConversationWorkingStateByChatId,
|
||
getMediaAssetsByUserAndUrls,
|
||
getSiteSettings,
|
||
upsertMediaAsset,
|
||
} from "@/lib/db/queries";
|
||
import { db } from "@/lib/db";
|
||
import type { DBMessage } from "@/lib/db/schema";
|
||
import {
|
||
agentPreset as agentPresetTable,
|
||
generatedImage,
|
||
message as messageTable,
|
||
} from "@/lib/db/schema";
|
||
import { generateCanvasImage } from "@/lib/canvas/generate-image";
|
||
import {
|
||
detectRequestedStoryboardGrid,
|
||
isGridConfirmationReply,
|
||
type PendingGridConfirmation,
|
||
} from "@/lib/ai/conversation-action-resolver";
|
||
import {
|
||
bootstrapConversationStateProtocol,
|
||
composeConversationStateProtocol,
|
||
persistAgentPresetTurn,
|
||
persistClarificationTurn,
|
||
setConversationStateProtocolEventObserver,
|
||
persistResolvedTargetTurn,
|
||
persistImageWorkflowTurn,
|
||
persistPromptOnlyTurn,
|
||
resolveGridConfirmationRequest,
|
||
resolveTargetExecutionOverride,
|
||
persistStoryboardConfirmedTurn,
|
||
persistStoryboardExecutionCompletedTurn,
|
||
persistStoryboardExecutionStartedTurn,
|
||
persistStoryboardRevisionTurn,
|
||
persistStoryboardAssistantTurn,
|
||
persistVideoGenerationCompletedTurn,
|
||
persistVideoGenerationFailedTurn,
|
||
persistVideoGenerationStartedTurn,
|
||
persistWorkflowCancelledTurn,
|
||
} from "@/lib/ai/conversation-state-protocol";
|
||
import { resolveChatMediaInput } from "@/lib/ai/context-memory/media-input";
|
||
import { resolveTargetAssets } from "@/lib/ai/context-memory/target-resolution";
|
||
import type {
|
||
ResolvedChatMediaInput,
|
||
ResolvedTargetAssets,
|
||
} from "@/lib/ai/context-memory/types";
|
||
import { ChatSDKError } from "@/lib/errors";
|
||
import {
|
||
getFriendlyRateLimitMessage,
|
||
getRequestErrorMessage,
|
||
isRateLimitError,
|
||
} from "@/lib/request-error";
|
||
import {
|
||
resolveClarificationBypass,
|
||
resolveClarificationMessage,
|
||
} from "@/lib/ai/runtime/clarification-interruption";
|
||
import {
|
||
buildAgentToolImagesMarkdown,
|
||
buildAgentToolVideoMarkdown,
|
||
prependImageModelBadge,
|
||
} from "@/lib/ai/runtime/agent-media-markdown";
|
||
import { saveGeneratedImages } from "@/lib/ai/runtime/generated-image-persistence";
|
||
import {
|
||
recordAiModelHealthFailure,
|
||
recordAiModelHealthSuccess,
|
||
} from "@/lib/ai/runtime/model-health";
|
||
import type { FalImageEditTaskMetadata } from "@/lib/ai/runtime/fal-image-edit-task";
|
||
import { saveAssistantTextOnlyMessage } from "@/lib/ai/runtime/assistant-message-persistence";
|
||
import { resolveCanvasAgentClarificationGate } from "@/lib/ai/runtime/middlewares/canvas-agent-clarification-gate";
|
||
import {
|
||
executeDirectAgentImageEditTool,
|
||
isDirectAgentImageEditServiceError,
|
||
} from "@/lib/ai/runtime/direct-agent-image-edit-executor";
|
||
import {
|
||
executeDirectAgentImageAnalysis,
|
||
executeDirectAgentImageGeneration,
|
||
executeDirectAgentVideoGeneration,
|
||
} from "@/lib/ai/runtime/direct-agent-media-executors";
|
||
import {
|
||
buildDirectAgentAnalysisAssistantText,
|
||
buildDirectAgentImageEditAssistantText,
|
||
buildDirectAgentImageGenerationAssistantText,
|
||
buildDirectAgentVideoGenerationAssistantText,
|
||
} from "@/lib/ai/runtime/direct-agent-response-builders";
|
||
import {
|
||
buildDirectAgentAnalyzeCompletedProgress,
|
||
buildDirectAgentAnalyzeFailedProgress,
|
||
buildDirectAgentAnalyzePlanningProgress,
|
||
buildDirectAgentImageContextProgress,
|
||
buildDirectAgentImageEditCompletedProgress,
|
||
buildDirectAgentImageEditExecutingProgress,
|
||
buildDirectAgentImageEditFailedProgress,
|
||
buildDirectAgentImageEditSelectProgress,
|
||
buildDirectAgentImageGenerateCompletedProgress,
|
||
buildDirectAgentImageGenerateExecutingProgress,
|
||
buildDirectAgentImageGenerateFailedProgress,
|
||
buildDirectAgentImageGenerateSelectProgress,
|
||
buildDirectAgentVideoGenerateCompletedProgress,
|
||
buildDirectAgentVideoGenerateExecutingProgress,
|
||
buildDirectAgentVideoGenerateFailedProgress,
|
||
buildDirectAgentVideoGenerateSelectProgress,
|
||
} from "@/lib/ai/runtime/direct-agent-progress";
|
||
import {
|
||
safePersistDirectAgentAnalyzeTurn,
|
||
safePersistDirectAgentImageEditTurn,
|
||
safePersistDirectAgentVideoCompletedTurn,
|
||
safePersistDirectAgentVideoFailedTurn,
|
||
} from "@/lib/ai/runtime/direct-agent-working-state";
|
||
import {
|
||
getDirectAgentToolErrorMessage,
|
||
getDirectAgentToolLabel,
|
||
getDirectAgentToolTargetMissingMessage,
|
||
isDirectAgentImageEditToolName,
|
||
resolveDirectAgentToolContext,
|
||
} from "@/lib/ai/runtime/direct-agent-execution";
|
||
import {
|
||
pruneTaskBoundaryContext,
|
||
pruneTaskBoundaryHistoryMessages,
|
||
} from "@/lib/ai/runtime/task-boundary-context-pruner";
|
||
import {
|
||
resolveAgentModelSlotDescriptor,
|
||
upsertAgentModelSlot,
|
||
type AgentModelSlots,
|
||
} from "@/lib/ai/runtime/agent-model-slots";
|
||
import { isPureGreetingTurnRequest } from "@/lib/ai/runtime/task-boundary-classifier";
|
||
import {
|
||
getProviderHealthSnapshot,
|
||
recordProviderHealthFailure,
|
||
recordProviderHealthSuccess,
|
||
shouldAvoidProvider,
|
||
} from "@/lib/ai/runtime/provider-health";
|
||
import { arbitrateAgentIntent } from "@/lib/ai/runtime/intent-arbiter";
|
||
import { buildAgentRoleIdentityInstruction } from "@/lib/ai/runtime/agent-role-profile";
|
||
import { getVideoStoryboardConfig } from "@/lib/video-agent/config";
|
||
import { runRuxaNineGridPipeline } from "@/lib/video-agent/pipeline/ruxa-nine-grid-pipeline";
|
||
import {
|
||
buildStoryboardConfirmedAck,
|
||
executeStoryboardConfirmedWorkflow,
|
||
handleStoryboardWorkflowFollowup,
|
||
} from "@/lib/video-agent/storyboard-chat-followup";
|
||
import {
|
||
buildStoryboardPreferenceMemoryUpdate,
|
||
} from "@/lib/video-agent/storyboard-preference-memory";
|
||
import {
|
||
attachVideoStoryboardTaskMessage,
|
||
getLatestActiveVideoStoryboardTaskByChatId,
|
||
markVideoStoryboardAwaitingConfirmation,
|
||
markVideoStoryboardConfirmed,
|
||
markVideoStoryboardDraftForRevision,
|
||
cancelVideoStoryboardTaskByUser,
|
||
} from "@/lib/video-agent/task-store";
|
||
import type { StoryboardModelCandidate } from "@/lib/video-agent/types";
|
||
import type { ChatMessage } from "@/lib/types";
|
||
import type { AppUsage } from "@/lib/usage";
|
||
import { convertToUIMessages, generateUUID } from "@/lib/utils";
|
||
import { isAiModelAgentAuto } from "@/lib/db/ai-model-flags";
|
||
import type {
|
||
AgentExecutionPlan,
|
||
AgentPreparedExecution,
|
||
} from "@/lib/ai/auto-agent-graph";
|
||
import { generateTitleFromUserMessage } from "../../actions";
|
||
import { type PostRequestBody, postRequestBodySchema } from "./schema";
|
||
import { isAiModelMultimodal } from "@/lib/db/ai-model-flags";
|
||
import {
|
||
createChatApiFailureResponse,
|
||
getBudgetedTimeout,
|
||
getChatStreamErrorMessage,
|
||
OperationTimeoutError,
|
||
withOperationTimeout,
|
||
} from "./_lib/errors";
|
||
import {
|
||
applyResolvedTargetImagesToLastUserMessage,
|
||
buildAutoModeFallbackMessage,
|
||
buildPromptOnlyOptimizationSeed,
|
||
buildPromptOnlyReply,
|
||
collectImageInputsFromMessageParts,
|
||
convertMessagesWithImages,
|
||
ensureImageAnalysisSummarySection,
|
||
ensureStructuredAgentFriendlyMessage,
|
||
extractMessageImageAssetIds,
|
||
extractMessageImageUrls,
|
||
isRemoteImageAccessible,
|
||
injectHistoryImagesToLastUserMessage,
|
||
normalizeAssistantImageMarkdown,
|
||
normalizeResolvedTargetImageUrls,
|
||
shouldEnrichImageAnalysisText,
|
||
summarizeTargetImageUrl,
|
||
} from "./_lib/image-media";
|
||
import {
|
||
buildImageFallbackCandidates,
|
||
generateImageWithFailoverCandidate,
|
||
getImageCrossProviderCandidates,
|
||
IMAGE_FAILOVER_PROVIDER_ALLOWLIST,
|
||
normalizeFailoverErrorMessage,
|
||
setImageFallbackModelCooldown,
|
||
shouldCooldownImageFallbackModel,
|
||
} from "./_lib/image-failover";
|
||
import {
|
||
buildAgentSceneWorkflowPrompt,
|
||
buildAgentSlotPrompt,
|
||
buildOptimizedAgentSceneWorkflowPrompt,
|
||
dedupeAgentSlotPromptAgainstWorkflow,
|
||
extractCurrentMessageImageParts,
|
||
getSceneUploadedImageBreakdown,
|
||
normalizeAgentSupplementPrompt,
|
||
resolveScenePromptOptimizationImageUrls,
|
||
type AgentPresetMessageImagePart,
|
||
} from "./_lib/agent-scene-workflow";
|
||
import {
|
||
analyzeReplacementExecutionPlan,
|
||
buildAgentReplacementPrompt,
|
||
isMarketplaceProductDetailReferenceId,
|
||
normalizeAgentModeDetailReplacementStrategy,
|
||
normalizeAgentModeReferences,
|
||
normalizeAgentModeSourceImages,
|
||
normalizeReplacementResultImage,
|
||
resolveDirectReplacementModelRef,
|
||
resolveImagePartsByReferenceExclusion,
|
||
resolveReplacementReferenceGeometry,
|
||
resolveReplacementScenePlanningRuntime,
|
||
} from "./_lib/detail-replacement";
|
||
import {
|
||
normalizeAgentReportForDisplay,
|
||
sanitizeAgentOutputSettingsSection,
|
||
} from "./_lib/agent-output";
|
||
import {
|
||
consumeGeneratedVideoPoints,
|
||
getModelCreditCost,
|
||
} from "./_lib/billing";
|
||
import {
|
||
ANNOTATION_SYSTEM_PROMPT,
|
||
applyHiddenPromptToMessages,
|
||
applyOptimizedPromptToLatestUserMessage,
|
||
convertImageUrlToBase64,
|
||
hasAnnotatedImage,
|
||
injectOptimizedPromptSection,
|
||
} from "./_lib/message-preparation";
|
||
import {
|
||
resolveModelDisplayNameByModelId,
|
||
resolveModelEntryByRuntimeModelId,
|
||
resolveModelRefByRuntimeModelId,
|
||
} from "./_lib/model-resolver";
|
||
import { normalizeAssistantPartsForPersistence } from "./_lib/persistence";
|
||
import {
|
||
buildAgentTargetLockPayload,
|
||
createSseResponse,
|
||
runBoundedParallel,
|
||
safeWriteStreamChunk,
|
||
startStreamKeepalive,
|
||
type StreamWriter,
|
||
writeAgentProgressChunk,
|
||
writeAgentTargetLockChunk,
|
||
writeAssistantTextChunk,
|
||
writeCanvasMediaChunk,
|
||
writeImageGenerationStatusChunk,
|
||
writeProtocolDebugChunk,
|
||
} from "./_lib/stream";
|
||
import {
|
||
extractVideoUrlFromText,
|
||
getSupportedVideoModesForModel,
|
||
isImageGenerationModelType,
|
||
isVideoProviderType,
|
||
resolveStoryboardModelCandidates,
|
||
scheduleVideoArchive,
|
||
selectBestVideoSubModelForMode,
|
||
type VideoModeType,
|
||
} from "./_lib/video-models";
|
||
|
||
const chatApiLogger = createLogger("chat.route");
|
||
|
||
|
||
const AGENT_REPLACEMENT_PARALLEL_WORKER_COUNT = Math.max(
|
||
1,
|
||
Math.min(8, Number(process.env.AGENT_REPLACEMENT_PARALLEL_WORKER_COUNT || 6) || 6),
|
||
);
|
||
|
||
async function replayAssistantMessagesForDuplicateRequest({
|
||
chatId,
|
||
userMessageId,
|
||
writer,
|
||
source,
|
||
waitMs = DUPLICATE_REQUEST_WAIT_MS,
|
||
}: {
|
||
chatId: string;
|
||
userMessageId: string;
|
||
writer: StreamWriter;
|
||
source: string;
|
||
waitMs?: number;
|
||
}) {
|
||
const startedAt = Date.now();
|
||
const seenAssistantMessageIds = new Set<string>();
|
||
|
||
while (Date.now() - startedAt < waitMs) {
|
||
try {
|
||
const dbMessages = await getMessagesByChatId({ id: chatId });
|
||
const userMessageIndex = dbMessages.findIndex((msg) => msg.id === userMessageId);
|
||
|
||
if (userMessageIndex >= 0) {
|
||
const assistantMessages = dbMessages
|
||
.slice(userMessageIndex + 1)
|
||
.filter((msg) => msg.role === "assistant");
|
||
|
||
if (assistantMessages.length > 0) {
|
||
for (const assistantMessage of assistantMessages) {
|
||
if (seenAssistantMessageIds.has(assistantMessage.id)) continue;
|
||
seenAssistantMessageIds.add(assistantMessage.id);
|
||
|
||
safeWriteStreamChunk(
|
||
writer,
|
||
{
|
||
type: "data-appendMessage",
|
||
data: JSON.stringify({
|
||
id: assistantMessage.id,
|
||
role: "assistant",
|
||
parts: assistantMessage.parts,
|
||
}),
|
||
},
|
||
`${source}:replay`,
|
||
);
|
||
}
|
||
|
||
return;
|
||
}
|
||
}
|
||
} catch (error) {
|
||
chatApiLogger.warn(`[Chat API] Failed duplicate replay fetch (${source})`, error);
|
||
}
|
||
|
||
safeWriteStreamChunk(
|
||
writer,
|
||
{
|
||
type: "data-keepalive",
|
||
data: JSON.stringify({ ts: Date.now(), source }),
|
||
},
|
||
`${source}:keepalive`,
|
||
);
|
||
|
||
await sleep(1000);
|
||
}
|
||
}
|
||
|
||
type ChatImageParams = NonNullable<PostRequestBody["imageParams"]>;
|
||
type AgentRecommendedImageParams = {
|
||
aspectRatio?: string;
|
||
imageSize?: string;
|
||
imageCount?: number;
|
||
negativePrompt?: string;
|
||
promptExtend?: boolean;
|
||
};
|
||
|
||
type ChatVideoParams = NonNullable<PostRequestBody["videoParams"]>;
|
||
type ChatVideoParamsDraft = Partial<ChatVideoParams>;
|
||
type AgentRecommendedVideoParams = {
|
||
mode?: "text-to-video" | "first-frame" | "first-last-frame" | "reference-images";
|
||
resolution?: "480p" | "720p" | "1080p";
|
||
ratio?: string;
|
||
duration?: number;
|
||
generateAudio?: boolean;
|
||
cameraMove?: string;
|
||
};
|
||
|
||
type AgentVideoPlan = {
|
||
script: string;
|
||
storyboard: Array<{
|
||
shot: number;
|
||
title: string;
|
||
duration: number;
|
||
description: string;
|
||
prompt: string;
|
||
camera?: string;
|
||
}>;
|
||
finalPrompt: string;
|
||
negativePrompt?: string;
|
||
recommendedVideoParams?: AgentRecommendedVideoParams;
|
||
};
|
||
|
||
function hasUserCustomizedImageParams(params?: PostRequestBody["imageParams"]): boolean {
|
||
if (!params) return false;
|
||
const customizedAspectRatio =
|
||
typeof params.aspectRatio === "string" && params.aspectRatio.trim() && params.aspectRatio !== "auto";
|
||
const customizedImageSize =
|
||
typeof params.imageSize === "string" && params.imageSize.trim() && params.imageSize.toUpperCase() !== "2K";
|
||
const customizedImageCount = typeof params.imageCount === "number" && params.imageCount !== 1;
|
||
const customizedPromptExtend = typeof params.promptExtend === "boolean";
|
||
const customizedAdvanced = Boolean(
|
||
params.size || params.seed !== undefined || params.n || params.negativePrompt || customizedPromptExtend
|
||
);
|
||
return customizedAspectRatio || customizedImageSize || customizedImageCount || customizedAdvanced;
|
||
}
|
||
|
||
function mergeImageParamsWithAgentRecommendation(params: {
|
||
incoming?: PostRequestBody["imageParams"];
|
||
recommended?: AgentRecommendedImageParams;
|
||
intent?: "image_edit" | "image_gen" | "vision" | "video_gen" | "chat";
|
||
hasUserCustomSettings: boolean;
|
||
}): PostRequestBody["imageParams"] {
|
||
const { incoming, recommended, intent, hasUserCustomSettings } = params;
|
||
const base: ChatImageParams = { ...(incoming || {}) };
|
||
if (!recommended) {
|
||
return Object.keys(base).length > 0 ? base : undefined;
|
||
}
|
||
|
||
const merged: ChatImageParams = {
|
||
...base,
|
||
negativePrompt:
|
||
typeof recommended.negativePrompt === "string" && recommended.negativePrompt.trim()
|
||
? recommended.negativePrompt.trim()
|
||
: base.negativePrompt,
|
||
promptExtend:
|
||
typeof recommended.promptExtend === "boolean"
|
||
? recommended.promptExtend
|
||
: base.promptExtend,
|
||
};
|
||
|
||
if (!hasUserCustomSettings) {
|
||
if (recommended.aspectRatio) merged.aspectRatio = recommended.aspectRatio;
|
||
if (recommended.imageSize) merged.imageSize = recommended.imageSize;
|
||
if (typeof recommended.imageCount === "number") merged.imageCount = recommended.imageCount;
|
||
}
|
||
|
||
if (intent === "image_edit") {
|
||
merged.imageCount = 1;
|
||
if (typeof base.promptExtend !== "boolean") {
|
||
merged.promptExtend = false;
|
||
}
|
||
}
|
||
|
||
return Object.keys(merged).length > 0 ? merged : undefined;
|
||
}
|
||
|
||
function hasUserCustomizedVideoParams(params?: PostRequestBody["videoParams"]): boolean {
|
||
if (!params) return false;
|
||
const defaults = {
|
||
mode: "text-to-video",
|
||
resolution: "720p",
|
||
ratio: "16:9",
|
||
duration: 5,
|
||
generateAudio: true,
|
||
cameraMove: "none",
|
||
} as const;
|
||
|
||
if (params.mode && params.mode !== defaults.mode) return true;
|
||
if (params.resolution && params.resolution !== defaults.resolution) return true;
|
||
if (params.ratio && params.ratio !== defaults.ratio) return true;
|
||
if (typeof params.duration === "number" && params.duration !== defaults.duration) return true;
|
||
if (typeof params.generateAudio === "boolean" && params.generateAudio !== defaults.generateAudio)
|
||
return true;
|
||
if (typeof params.cameraMove === "string" && params.cameraMove !== defaults.cameraMove)
|
||
return true;
|
||
if (typeof params.seed === "number") return true;
|
||
return false;
|
||
}
|
||
|
||
function mergeVideoParamsWithAgentRecommendation(params: {
|
||
incoming?: PostRequestBody["videoParams"];
|
||
recommended?: AgentRecommendedVideoParams;
|
||
hasUserCustomSettings: boolean;
|
||
}): PostRequestBody["videoParams"] {
|
||
const { incoming, recommended, hasUserCustomSettings } = params;
|
||
const base: ChatVideoParamsDraft = { ...(incoming || {}) };
|
||
if (!recommended) {
|
||
return Object.keys(base).length > 0 ? (base as ChatVideoParams) : undefined;
|
||
}
|
||
|
||
const merged: ChatVideoParamsDraft = { ...base };
|
||
if (!hasUserCustomSettings) {
|
||
if (recommended.mode) merged.mode = recommended.mode;
|
||
if (recommended.resolution) merged.resolution = recommended.resolution;
|
||
if (recommended.ratio) merged.ratio = recommended.ratio;
|
||
if (typeof recommended.duration === "number") merged.duration = recommended.duration;
|
||
if (typeof recommended.generateAudio === "boolean") {
|
||
merged.generateAudio = recommended.generateAudio;
|
||
}
|
||
if (recommended.cameraMove) merged.cameraMove = recommended.cameraMove;
|
||
}
|
||
|
||
if (!merged.mode) {
|
||
merged.mode = recommended.mode || (incoming?.mode as ChatVideoParams["mode"]) || "text-to-video";
|
||
}
|
||
if (!merged.resolution) {
|
||
merged.resolution =
|
||
recommended.resolution || (incoming?.resolution as ChatVideoParams["resolution"]) || "720p";
|
||
}
|
||
if (!merged.ratio) {
|
||
merged.ratio = recommended.ratio || incoming?.ratio || "16:9";
|
||
}
|
||
if (typeof merged.duration !== "number") {
|
||
merged.duration = recommended.duration || incoming?.duration || 5;
|
||
}
|
||
|
||
return Object.keys(merged).length > 0 ? (merged as ChatVideoParams) : undefined;
|
||
}
|
||
|
||
export const maxDuration = 600; // 10 分钟，用于支持图片编辑等耗时任务
|
||
|
||
let globalStreamContext: ResumableStreamContext | null = null;
|
||
const inFlightChatRequests = new Map<string, number>();
|
||
const IN_FLIGHT_REQUEST_TTL_MS = 5 * 60 * 1000;
|
||
const DUPLICATE_REQUEST_WAIT_MS = 2 * 60 * 1000;
|
||
const CHAT_PRE_RESPONSE_SOFT_TIMEOUT_MS = 55_000;
|
||
const SMART_AUTO_GRAPH_TIMEOUT_MS = 28_000;
|
||
const SMART_AUTO_FALLBACK_TIMEOUT_MS = 20_000;
|
||
const SMART_AUTO_IMAGE_SUMMARY_TIMEOUT_MS = 18_000;
|
||
const DEFAULT_STREAM_KEEPALIVE_INTERVAL_MS = 12_000;
|
||
const CHAT_CREATION_LOOKUP_RETRY_DELAYS_MS = [60, 180, 360] as const;
|
||
|
||
const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms));
|
||
|
||
async function findChatWithRetry(chatId: string) {
|
||
let currentChat = await getChatById({ id: chatId });
|
||
if (currentChat) {
|
||
return currentChat;
|
||
}
|
||
|
||
for (const delayMs of CHAT_CREATION_LOOKUP_RETRY_DELAYS_MS) {
|
||
await sleep(delayMs);
|
||
currentChat = await getChatById({ id: chatId });
|
||
if (currentChat) {
|
||
return currentChat;
|
||
}
|
||
}
|
||
|
||
return null;
|
||
}
|
||
|
||
const getTokenlensCatalog = cache(
|
||
async (): Promise<ModelCatalog | undefined> => {
|
||
try {
|
||
return await fetchModels();
|
||
} catch (err) {
|
||
chatApiLogger.warn("TokenLens: catalog fetch failed, using default catalog", err);
|
||
return; // tokenlens helpers will fall back to defaultCatalog
|
||
}
|
||
},
|
||
["tokenlens-catalog"],
|
||
{ revalidate: 24 * 60 * 60 } // 24 hours
|
||
);
|
||
|
||
export function getStreamContext() {
|
||
if (!globalStreamContext) {
|
||
try {
|
||
globalStreamContext = createResumableStreamContext({
|
||
waitUntil: after,
|
||
});
|
||
} catch (error: any) {
|
||
if (error.message.includes("REDIS_URL")) {
|
||
chatApiLogger.info(" > Resumable streams are disabled due to missing REDIS_URL");
|
||
} else {
|
||
chatApiLogger.error("console_error", error);
|
||
}
|
||
}
|
||
}
|
||
|
||
return globalStreamContext;
|
||
}
|
||
|
||
export async function POST(request: Request) {
|
||
let requestBody: PostRequestBody;
|
||
let requestDedupKey: string | null = null;
|
||
let acquiredInFlightLock = false;
|
||
|
||
try {
|
||
const json = await request.json();
|
||
chatApiLogger.info("[Chat API] Received request body:", JSON.stringify(json, null, 2));
|
||
requestBody = postRequestBodySchema.parse(json);
|
||
} catch (error) {
|
||
chatApiLogger.error("[Chat API] Schema validation failed:", error);
|
||
return new ChatSDKError("bad_request:api").toResponse();
|
||
}
|
||
|
||
try {
|
||
const {
|
||
id,
|
||
message,
|
||
agentPresetId,
|
||
selectedChatModel,
|
||
selectedVisibilityType,
|
||
composerMode,
|
||
hiddenPrompt,
|
||
workflowPromptOptimizationEnabled,
|
||
autoContext,
|
||
contextPayload,
|
||
agentModePayload,
|
||
deepThinkingEnabled,
|
||
debugProtocol,
|
||
}: {
|
||
id: string;
|
||
message: ChatMessage;
|
||
agentPresetId?: string;
|
||
selectedChatModel: ChatModel["id"];
|
||
selectedVisibilityType: VisibilityType;
|
||
composerMode?: "agent" | "image" | "video" | "detail";
|
||
hiddenPrompt?: string;
|
||
workflowPromptOptimizationEnabled?: boolean;
|
||
autoContext?: string;
|
||
contextPayload?: PostRequestBody["contextPayload"];
|
||
agentModePayload?: PostRequestBody["agentModePayload"];
|
||
deepThinkingEnabled?: boolean;
|
||
debugProtocol?: boolean;
|
||
} = requestBody;
|
||
const requestStartedAt = Date.now();
|
||
const requestOrigin = (() => {
|
||
try {
|
||
return new URL(request.url).origin;
|
||
} catch {
|
||
return "";
|
||
}
|
||
})();
|
||
const requestTextContent = message.parts
|
||
.filter((part): part is { type: "text"; text: string } => part.type === "text")
|
||
.map((part) => part.text)
|
||
.join(" ")
|
||
.trim();
|
||
const autoContextText = typeof autoContext === "string" ? autoContext.trim() : "";
|
||
|
||
const now = Date.now();
|
||
for (const [key, startedAt] of inFlightChatRequests.entries()) {
|
||
if (now - startedAt > IN_FLIGHT_REQUEST_TTL_MS) {
|
||
inFlightChatRequests.delete(key);
|
||
}
|
||
}
|
||
|
||
requestDedupKey = `${id}:${message.id}`;
|
||
if (inFlightChatRequests.has(requestDedupKey)) {
|
||
chatApiLogger.warn(`[Chat API] Duplicate in-flight request ignored (chatId=${id}, messageId=${message.id})`);
|
||
const duplicateStream = createUIMessageStream({
|
||
execute: async ({ writer }) => {
|
||
await replayAssistantMessagesForDuplicateRequest({
|
||
chatId: id,
|
||
userMessageId: message.id,
|
||
writer,
|
||
source: "duplicate-in-flight",
|
||
});
|
||
},
|
||
});
|
||
return createSseResponse(duplicateStream);
|
||
}
|
||
inFlightChatRequests.set(requestDedupKey, now);
|
||
acquiredInFlightLock = true;
|
||
|
||
let finalModelId = selectedChatModel;
|
||
|
||
// 🆕 存储智能模式的相关信息
|
||
let smartAutoFriendlyMessage: string | undefined; // AI的自然对话回复
|
||
let smartAutoOptimizedPrompt: string | undefined; // 优化后的提示词（不修改用户消息，只用于API调用）
|
||
let smartAutoPromptFromVideoPlan = false;
|
||
let smartAutoRecommendedImageParams: AgentRecommendedImageParams | undefined;
|
||
let smartAutoVideoPlan: AgentVideoPlan | undefined;
|
||
let smartAutoRecommendedVideoParams: AgentRecommendedVideoParams | undefined;
|
||
let smartAutoPlan: AgentExecutionPlan | undefined;
|
||
let smartAutoExecution: AgentPreparedExecution | undefined;
|
||
let smartAutoDetectedIntent: "image_edit" | "vision" | "image_gen" | "video_gen" | "chat" | undefined;
|
||
let smartAutoHistoryImages: string[] = []; // 从历史对话中提取的图片URL（支持多张）
|
||
let smartAutoNeedsClarification = false; // 是否需要先追问用户
|
||
let smartAutoClarificationQuestion: string | undefined;
|
||
let requestResolvedMediaInput: ResolvedChatMediaInput | null = null;
|
||
let requestTargetResolution: ResolvedTargetAssets | null = null;
|
||
let requestWorkingState:
|
||
| Awaited<ReturnType<typeof getConversationWorkingStateByChatId>>
|
||
| null = null;
|
||
let storyboardPreferenceMemory = null;
|
||
let imageWorkflowMemory = null;
|
||
let reusablePromptMemory = null;
|
||
let confirmedPromptForConversationState: string | undefined;
|
||
let smartAutoHistoryMessagesFromDb: DBMessage[] = [];
|
||
let smartAutoMediaHistoryMessagesFromDb: DBMessage[] = [];
|
||
let preferredAgentModelId: string | undefined;
|
||
let configuredAgentConversationModelId: string | undefined;
|
||
let agentModelSlots: AgentModelSlots = {};
|
||
let pendingGridConfirmation:
|
||
| PendingGridConfirmation
|
||
| null = null;
|
||
let shouldAskGridConfirmation = false;
|
||
const deepThinkingOn = deepThinkingEnabled !== false;
|
||
const shouldRespectDirectComposerMode =
|
||
composerMode === "image" || composerMode === "video" || composerMode === "detail";
|
||
const shouldIsolateDirectComposerContext = shouldRespectDirectComposerMode;
|
||
let shouldRunAgentConversation = finalModelId === "auto";
|
||
let shouldKeepSelectedModel = false;
|
||
if (shouldRespectDirectComposerMode && shouldRunAgentConversation) {
|
||
shouldRunAgentConversation = false;
|
||
chatApiLogger.info(`[Smart Auto Mode] ⛳ Direct composer mode (${composerMode}) disables initial auto-agent routing`);
|
||
}
|
||
const pendingProtocolDebugEvents: Array<{ phase: string; payload: Record<string, unknown> }> = [];
|
||
const requestedModelId = selectedChatModel;
|
||
let requestedModelProviderType: string | undefined;
|
||
let requestedModelIsFixedVisual = false;
|
||
let requestedModelIsConversationMultimodal = false;
|
||
let requestedModelConfig: Awaited<ReturnType<typeof getAiModelById>> | null = null;
|
||
|
||
if (!shouldRunAgentConversation && requestedModelId !== "auto") {
|
||
const selectedModelIsUUID =
|
||
/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test(requestedModelId);
|
||
let selectedModelConfig = selectedModelIsUUID
|
||
? await getAiModelById({ id: requestedModelId })
|
||
: null;
|
||
let selectedModelProviderType: string | undefined;
|
||
|
||
if (!selectedModelConfig) {
|
||
const enabledModels = await getEnabledAiModelsWithProvider(false);
|
||
const matchedModel = enabledModels.find(
|
||
({ model }) => model.id === requestedModelId || model.modelId === requestedModelId
|
||
);
|
||
if (matchedModel) {
|
||
selectedModelConfig = matchedModel.model;
|
||
selectedModelProviderType = matchedModel.provider.type as string;
|
||
}
|
||
}
|
||
|
||
if (selectedModelConfig && !selectedModelProviderType) {
|
||
const selectedProvider = await getAiProviderById({
|
||
id: selectedModelConfig.providerId,
|
||
});
|
||
selectedModelProviderType = selectedProvider?.type as string | undefined;
|
||
}
|
||
|
||
requestedModelConfig = selectedModelConfig;
|
||
requestedModelProviderType = selectedModelProviderType;
|
||
|
||
if (selectedModelConfig) {
|
||
const isFixedVisualModel =
|
||
!selectedModelConfig.isSubModel &&
|
||
(
|
||
isImageGenerationModelType(selectedModelProviderType, selectedModelConfig) ||
|
||
isVideoProviderType(selectedModelProviderType, selectedModelConfig)
|
||
);
|
||
const isConversationMultimodalModel =
|
||
!selectedModelConfig.isSubModel &&
|
||
isAiModelMultimodal(selectedModelConfig) &&
|
||
!isImageGenerationModelType(selectedModelProviderType, selectedModelConfig) &&
|
||
!isVideoProviderType(selectedModelProviderType, selectedModelConfig);
|
||
requestedModelIsFixedVisual = isFixedVisualModel;
|
||
requestedModelIsConversationMultimodal = isConversationMultimodalModel;
|
||
|
||
if (deepThinkingOn) {
|
||
if (isAiModelAgentAuto(selectedModelConfig)) {
|
||
if (shouldRespectDirectComposerMode) {
|
||
chatApiLogger.info(`[Smart Auto Mode] ⛳ Respect direct composer mode (${composerMode}), skip forcing agent-only routing for: ${selectedModelConfig.displayName}`);
|
||
} else {
|
||
shouldRunAgentConversation = true;
|
||
shouldKeepSelectedModel = false;
|
||
preferredAgentModelId = selectedModelConfig.id;
|
||
configuredAgentConversationModelId = selectedModelConfig.id;
|
||
chatApiLogger.info(`[Smart Auto Mode] 🧠 Agent-only model detected, force auto routing: ${selectedModelConfig.displayName}`);
|
||
}
|
||
} else if (isFixedVisualModel || isConversationMultimodalModel) {
|
||
if (shouldRespectDirectComposerMode) {
|
||
chatApiLogger.info(`[Smart Auto Mode] ⛳ Respect direct composer mode (${composerMode}), keep direct execution model: ${selectedModelConfig.displayName}`);
|
||
} else {
|
||
shouldRunAgentConversation = true;
|
||
shouldKeepSelectedModel = true;
|
||
preferredAgentModelId = selectedModelConfig.id;
|
||
chatApiLogger.info(`[Smart Auto Mode] 🎯 Agent conversation enabled for selected model: ${selectedModelConfig.displayName} (${selectedModelProviderType || "unknown"}), preferredAgentModelId=${preferredAgentModelId}`);
|
||
}
|
||
}
|
||
} else if (
|
||
isFixedVisualModel ||
|
||
isConversationMultimodalModel ||
|
||
isAiModelAgentAuto(selectedModelConfig)
|
||
) {
|
||
chatApiLogger.info(`[Smart Auto Mode] ⏭️ Deep thinking disabled, skip agent analysis for: ${selectedModelConfig.displayName}`);
|
||
}
|
||
}
|
||
}
|
||
|
||
if (shouldRunAgentConversation && !preferredAgentModelId) {
|
||
const configuredAgentModel = await getAgentAiModelWithProvider().catch((agentModelError) => {
|
||
chatApiLogger.warn("[Chat API] Failed to load configured agent model:", agentModelError);
|
||
return null;
|
||
});
|
||
|
||
if (configuredAgentModel?.model?.id) {
|
||
preferredAgentModelId = configuredAgentModel.model.id;
|
||
configuredAgentConversationModelId = configuredAgentModel.model.id;
|
||
agentModelSlots = upsertAgentModelSlot(agentModelSlots, "planner", {
|
||
modelId: configuredAgentModel.model.id,
|
||
displayName: configuredAgentModel.model.displayName || configuredAgentModel.model.modelId,
|
||
providerId: configuredAgentModel.model.providerId,
|
||
providerType: configuredAgentModel.provider.type,
|
||
providerKey: configuredAgentModel.model.providerId,
|
||
source: "configured_agent_default",
|
||
reason: "agent_default_model",
|
||
});
|
||
agentModelSlots = upsertAgentModelSlot(agentModelSlots, "analyzer", {
|
||
modelId: configuredAgentModel.model.id,
|
||
displayName: configuredAgentModel.model.displayName || configuredAgentModel.model.modelId,
|
||
providerId: configuredAgentModel.model.providerId,
|
||
providerType: configuredAgentModel.provider.type,
|
||
providerKey: configuredAgentModel.model.providerId,
|
||
source: "configured_agent_default",
|
||
reason: "agent_default_model",
|
||
});
|
||
chatApiLogger.info(`[Smart Auto Mode] 🧠 Loaded configured agent conversation model: ${configuredAgentModel.model.displayName}`);
|
||
}
|
||
}
|
||
|
||
const hasCurrentMessageMediaInput = message.parts.some((part: any) =>
|
||
part?.type === "file" || part?.type === "image" || part?.type === "image_url"
|
||
);
|
||
if (
|
||
shouldRunAgentConversation &&
|
||
isPureGreetingTurnRequest(requestTextContent) &&
|
||
!hasCurrentMessageMediaInput
|
||
) {
|
||
const session = await auth();
|
||
|
||
if (!session?.user) {
|
||
return new ChatSDKError("unauthorized:chat").toResponse();
|
||
}
|
||
|
||
const greetingMessage = "你好，我在。你可以直接告诉我想做什么：生图、修图、生成视频、分析图片都可以。";
|
||
const fastPathTitle = requestTextContent.trim().slice(0, 80) || "新对话";
|
||
await saveChat({
|
||
id,
|
||
userId: session.user.id,
|
||
title: fastPathTitle,
|
||
visibility: selectedVisibilityType,
|
||
});
|
||
|
||
const greetingChat = await findChatWithRetry(id);
|
||
if (!greetingChat) {
|
||
throw new ChatSDKError("bad_request:database", "Failed to save chat");
|
||
}
|
||
|
||
if (greetingChat.userId !== session.user.id) {
|
||
return new ChatSDKError("forbidden:chat").toResponse();
|
||
}
|
||
|
||
const greetingStream = createUIMessageStream({
|
||
execute: async ({ writer }) => {
|
||
const messageId = generateUUID();
|
||
writeAssistantTextChunk(writer, greetingMessage, "greeting-fast-path", messageId);
|
||
|
||
await saveMessages({
|
||
messages: [
|
||
{
|
||
id: message.id,
|
||
chatId: id,
|
||
role: "user",
|
||
parts: message.parts as any,
|
||
attachments: [],
|
||
createdAt: new Date(),
|
||
videoStorageStatus: null,
|
||
temporaryVideoUrl: null,
|
||
permanentVideoUrl: null,
|
||
archivedAt: null,
|
||
videoMetadata: null,
|
||
},
|
||
{
|
||
id: messageId,
|
||
chatId: id,
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: greetingMessage }],
|
||
attachments: [],
|
||
createdAt: new Date(),
|
||
videoStorageStatus: null,
|
||
temporaryVideoUrl: null,
|
||
permanentVideoUrl: null,
|
||
archivedAt: null,
|
||
videoMetadata: null,
|
||
},
|
||
],
|
||
});
|
||
},
|
||
});
|
||
|
||
return createSseResponse(greetingStream);
|
||
}
|
||
|
||
const allowAgentVideoGeneration =
|
||
Boolean(requestBody.videoParams) ||
|
||
isVideoProviderType(requestedModelProviderType, requestedModelConfig || undefined);
|
||
let smartAutoWorkflowDecision:
|
||
| {
|
||
action?: string;
|
||
workflowId?: string;
|
||
workflowType?: string;
|
||
allowWorkflowExecution?: boolean;
|
||
}
|
||
| undefined;
|
||
const activeStoryboardTaskForChat = shouldIsolateDirectComposerContext
|
||
? null
|
||
: await getLatestActiveVideoStoryboardTaskByChatId({
|
||
chatId: id,
|
||
}).catch((storyboardLookupError) => {
|
||
chatApiLogger.warn("[Chat API] Failed to preload active storyboard task before smart auto mode:", storyboardLookupError);
|
||
return null;
|
||
});
|
||
|
||
requestWorkingState = shouldIsolateDirectComposerContext
|
||
? null
|
||
: await getConversationWorkingStateByChatId({
|
||
chatId: id,
|
||
}).catch((workingStatePreloadError) => {
|
||
chatApiLogger.warn("[Chat API] Failed to preload conversation working state before smart auto mode:", workingStatePreloadError);
|
||
return null;
|
||
});
|
||
const hasImageInputForConversationAction = message.parts.some((part: any) =>
|
||
part?.type === "file" || part?.type === "image" || part?.type === "image_url"
|
||
);
|
||
const conversationStateBootstrap = bootstrapConversationStateProtocol({
|
||
text: requestTextContent,
|
||
workingState: requestWorkingState,
|
||
activeStoryboardTask: activeStoryboardTaskForChat,
|
||
hasImageInputForPromptCarryForward: hasImageInputForConversationAction,
|
||
hasCanvasSelection: Boolean(contextPayload?.selectedNodeIds?.length),
|
||
});
|
||
chatApiLogger.info("[ConversationStateProtocol] Bootstrap:", {
|
||
action: conversationStateBootstrap.actionResolution.kind,
|
||
taskType: conversationStateBootstrap.taskBoundaryDecision.taskType,
|
||
taskMode: conversationStateBootstrap.taskBoundaryDecision.taskMode,
|
||
hasPromptMemory: Boolean(conversationStateBootstrap.promptMemory?.optimizedPrompt),
|
||
hasImageWorkflowMemory: Boolean(conversationStateBootstrap.imageWorkflowMemory),
|
||
hasStoryboardPreferenceMemory: Boolean(conversationStateBootstrap.storyboardPreferenceMemory),
|
||
hasPendingGridConfirmation: Boolean(conversationStateBootstrap.pendingGridConfirmation),
|
||
hasActiveStoryboardTask: Boolean(conversationStateBootstrap.activeStoryboardTask?.task?.id),
|
||
});
|
||
pendingProtocolDebugEvents.push({
|
||
phase: "bootstrap",
|
||
payload: {
|
||
action: conversationStateBootstrap.actionResolution.kind,
|
||
taskBoundary: conversationStateBootstrap.taskBoundaryDecision,
|
||
pendingGridConfirmation: conversationStateBootstrap.pendingGridConfirmation,
|
||
hasPromptMemory: Boolean(conversationStateBootstrap.promptMemory?.optimizedPrompt),
|
||
hasImageWorkflowMemory: Boolean(conversationStateBootstrap.imageWorkflowMemory),
|
||
hasStoryboardPreferenceMemory: Boolean(conversationStateBootstrap.storyboardPreferenceMemory),
|
||
hasActiveStoryboardTask: Boolean(conversationStateBootstrap.activeStoryboardTask?.task?.id),
|
||
},
|
||
});
|
||
storyboardPreferenceMemory = conversationStateBootstrap.storyboardPreferenceMemory;
|
||
imageWorkflowMemory = conversationStateBootstrap.imageWorkflowMemory;
|
||
reusablePromptMemory = conversationStateBootstrap.promptMemory;
|
||
pendingGridConfirmation = conversationStateBootstrap.pendingGridConfirmation;
|
||
confirmedPromptForConversationState = conversationStateBootstrap.confirmedPrompt;
|
||
const resolvedConversationAction = conversationStateBootstrap.actionResolution;
|
||
const taskBoundaryDecision = conversationStateBootstrap.taskBoundaryDecision;
|
||
const taskBoundaryRequiresAgentConversation =
|
||
taskBoundaryDecision.taskType === "prompt_only" ||
|
||
taskBoundaryDecision.taskType === "analysis_only" ||
|
||
taskBoundaryDecision.taskType === "execution_image" ||
|
||
taskBoundaryDecision.taskType === "execution_video" ||
|
||
taskBoundaryDecision.taskType === "workflow_followup";
|
||
const taskBoundaryOverridesLegacyWorkflow =
|
||
taskBoundaryDecision.taskType === "greeting" ||
|
||
taskBoundaryDecision.taskType === "chat" ||
|
||
taskBoundaryDecision.taskType === "analysis_only" ||
|
||
taskBoundaryDecision.taskType === "prompt_only";
|
||
const prunedTaskBoundaryContext = pruneTaskBoundaryContext({
|
||
decision: taskBoundaryDecision,
|
||
contextPayload: contextPayload || undefined,
|
||
autoContextText: autoContextText || undefined,
|
||
workingState: requestWorkingState || undefined,
|
||
hasCurrentImageInput: hasImageInputForConversationAction,
|
||
});
|
||
const effectiveContextPayload = prunedTaskBoundaryContext.contextPayload;
|
||
const effectiveAutoContextText = prunedTaskBoundaryContext.autoContextText || "";
|
||
const effectiveWorkingState = prunedTaskBoundaryContext.workingState;
|
||
|
||
if (taskBoundaryRequiresAgentConversation && !shouldRunAgentConversation) {
|
||
if (shouldRespectDirectComposerMode) {
|
||
chatApiLogger.info(`[Smart Auto Mode] ⛳ Respect direct composer mode (${composerMode}), skip task-boundary agent force: ${taskBoundaryDecision.taskType}/${taskBoundaryDecision.taskMode}`);
|
||
} else {
|
||
shouldRunAgentConversation = true;
|
||
shouldKeepSelectedModel = false;
|
||
chatApiLogger.info(`[Smart Auto Mode] 🧭 Force agent conversation due to task boundary: ${taskBoundaryDecision.taskType}/${taskBoundaryDecision.taskMode}`);
|
||
}
|
||
}
|
||
|
||
if (taskBoundaryOverridesLegacyWorkflow && shouldKeepSelectedModel) {
|
||
shouldKeepSelectedModel = false;
|
||
chatApiLogger.info(`[Smart Auto Mode] 🧼 Release selected-model lock due to task boundary override: ${taskBoundaryDecision.taskType}`);
|
||
}
|
||
|
||
if (resolvedConversationAction.forceAgentConversation && !shouldRunAgentConversation) {
|
||
if (shouldRespectDirectComposerMode) {
|
||
chatApiLogger.info(`[Smart Auto Mode] ⛳ Respect direct composer mode (${composerMode}), skip legacy workflow force: ${resolvedConversationAction.kind}`);
|
||
} else if (taskBoundaryOverridesLegacyWorkflow) {
|
||
chatApiLogger.info(`[Smart Auto Mode] 🪫 Ignore legacy workflow follow-up due to task boundary override: ${resolvedConversationAction.kind} -> ${taskBoundaryDecision.taskType}`);
|
||
} else {
|
||
shouldRunAgentConversation = true;
|
||
shouldKeepSelectedModel = resolvedConversationAction.shouldKeepSelectedModel;
|
||
if (resolvedConversationAction.kind === "grid_confirmation_followup" && pendingGridConfirmation) {
|
||
chatApiLogger.info(`[Smart Auto Mode] 🧩 Force agent conversation due to pending grid confirmation: ${pendingGridConfirmation.label}`);
|
||
} else if (
|
||
resolvedConversationAction.kind === "active_storyboard_followup" &&
|
||
activeStoryboardTaskForChat?.task?.id
|
||
) {
|
||
chatApiLogger.info(`[Smart Auto Mode] 🔐 Force agent conversation due to active storyboard task: ${activeStoryboardTaskForChat.task.id}`);
|
||
} else if (resolvedConversationAction.kind === "image_generation_followup") {
|
||
chatApiLogger.info("[Smart Auto Mode] 🪄 Force agent conversation due to image generation follow-up workflow");
|
||
} else if (resolvedConversationAction.kind === "image_edit_followup") {
|
||
chatApiLogger.info("[Smart Auto Mode] 🖼️ Force agent conversation due to image edit follow-up workflow");
|
||
}
|
||
}
|
||
}
|
||
|
||
const shouldFastTrackPromptOnly =
|
||
shouldRunAgentConversation && taskBoundaryDecision.taskType === "prompt_only";
|
||
if (shouldFastTrackPromptOnly) {
|
||
if (!requestResolvedMediaInput) {
|
||
requestResolvedMediaInput = resolveChatMediaInput({
|
||
messageParts: message.parts as any[],
|
||
historyMessages: [],
|
||
contextPayload: effectiveContextPayload,
|
||
requestOrigin,
|
||
maxImages: 4,
|
||
});
|
||
}
|
||
|
||
if (!requestTargetResolution) {
|
||
requestTargetResolution = resolveTargetAssets({
|
||
text: requestTextContent,
|
||
mediaInput: requestResolvedMediaInput,
|
||
contextPayload: effectiveContextPayload,
|
||
workingState: effectiveWorkingState || undefined,
|
||
});
|
||
}
|
||
|
||
smartAutoDetectedIntent = "vision";
|
||
smartAutoNeedsClarification = false;
|
||
smartAutoFriendlyMessage =
|
||
"收到，我先分析这张图并整理可直接使用的提示词，不会直接生图。";
|
||
chatApiLogger.info("[Smart Auto Mode] ✍️ Fast-path prompt-only request, skip heavy auto-agent workflow");
|
||
}
|
||
|
||
// ========== Smart Auto Mode Logic ==========
|
||
if (shouldRunAgentConversation && !shouldFastTrackPromptOnly) {
|
||
chatApiLogger.info("[Smart Auto Mode] 🚀 Starting intelligent analysis...");
|
||
let autoTextForFallback = "";
|
||
let autoFromHistoryForFallback = false;
|
||
let autoCurrentImageUrlsForFallback: string[] = [];
|
||
let confirmedPromptForGeneration: string | undefined;
|
||
let forcePromptDrivenImageGeneration = false;
|
||
let shouldPreferArbitratedFriendlyMessage = false;
|
||
const smartAutoInputText =
|
||
resolvedConversationAction.effectiveInputText || requestTextContent;
|
||
|
||
try {
|
||
// 🆕 首先获取历史消息（在 Smart Auto Mode 内部单独获取，避免依赖后面的 messagesFromDb）
|
||
const loadedHistoryMessages = await getMessagesByChatId({ id });
|
||
smartAutoHistoryMessagesFromDb = pruneTaskBoundaryHistoryMessages({
|
||
decision: taskBoundaryDecision,
|
||
messages: loadedHistoryMessages,
|
||
purpose: "model",
|
||
hasCurrentImageInput: hasImageInputForConversationAction,
|
||
});
|
||
smartAutoMediaHistoryMessagesFromDb = pruneTaskBoundaryHistoryMessages({
|
||
decision: taskBoundaryDecision,
|
||
messages: loadedHistoryMessages,
|
||
purpose: "media",
|
||
hasCurrentImageInput: hasImageInputForConversationAction,
|
||
});
|
||
chatApiLogger.info("[Smart Auto Mode] Loaded", loadedHistoryMessages.length, "messages from DB");
|
||
chatApiLogger.info("[Smart Auto Mode] Pruned history:", {
|
||
modelMessages: smartAutoHistoryMessagesFromDb.length,
|
||
mediaMessages: smartAutoMediaHistoryMessagesFromDb.length,
|
||
taskType: taskBoundaryDecision.taskType,
|
||
taskMode: taskBoundaryDecision.taskMode,
|
||
});
|
||
|
||
autoTextForFallback = smartAutoInputText;
|
||
if (!requestWorkingState && !shouldIsolateDirectComposerContext) {
|
||
requestWorkingState = await getConversationWorkingStateByChatId({
|
||
chatId: id,
|
||
});
|
||
}
|
||
confirmedPromptForGeneration = confirmedPromptForConversationState;
|
||
const isPromptCarryForwardExecution =
|
||
resolvedConversationAction.kind === "prompt_carry_forward";
|
||
forcePromptDrivenImageGeneration =
|
||
resolvedConversationAction.usePromptCarryForward &&
|
||
Boolean(confirmedPromptForGeneration);
|
||
if (!requestResolvedMediaInput) {
|
||
requestResolvedMediaInput = resolveChatMediaInput({
|
||
messageParts: message.parts as any[],
|
||
historyMessages: smartAutoMediaHistoryMessagesFromDb,
|
||
contextPayload: effectiveContextPayload,
|
||
requestOrigin,
|
||
maxImages: 4,
|
||
});
|
||
}
|
||
if (!requestTargetResolution) {
|
||
requestTargetResolution = resolveTargetAssets({
|
||
text: requestTextContent,
|
||
mediaInput: requestResolvedMediaInput,
|
||
contextPayload: effectiveContextPayload,
|
||
workingState: effectiveWorkingState || undefined,
|
||
});
|
||
}
|
||
|
||
const hasSelectedImageHint =
|
||
Boolean(effectiveContextPayload?.selectedNodeIds?.length) ||
|
||
/用户已选中\s*\d+\s*张图片/.test(effectiveAutoContextText);
|
||
const hasExplicitTargetHint =
|
||
requestTargetResolution.resolutionMode === "text_reference" ||
|
||
/第[一二三四五六七八九十\d]+张|左边|左侧|左上|左下|右边|右侧|右上|右下|上面|上方|下面|下方|中间|居中|中央|选中/.test(requestTextContent);
|
||
const hasActionableEditIntent = /改|调整|优化|换|替换|去掉|抠|重绘|增强|修|处理|按照|按|保留|统一|色差|背景|清晰|风格/.test(
|
||
requestTextContent
|
||
);
|
||
|
||
const currentMessageImageInputs = requestResolvedMediaInput.currentMessageImageUrls.slice(0, 4);
|
||
let imageUrls =
|
||
requestTargetResolution.targetImageUrls.length > 0
|
||
? requestTargetResolution.targetImageUrls.slice(0, 4)
|
||
: requestResolvedMediaInput.resolvedImageUrls.slice(0, 4);
|
||
let hasImage = imageUrls.length > 0;
|
||
autoCurrentImageUrlsForFallback =
|
||
currentMessageImageInputs.length > 0 ? currentMessageImageInputs.slice() : imageUrls.slice();
|
||
smartAutoHistoryImages =
|
||
currentMessageImageInputs.length === 0 ? imageUrls.slice(0, 4) : [];
|
||
autoFromHistoryForFallback =
|
||
currentMessageImageInputs.length === 0 && imageUrls.length > 0;
|
||
|
||
if (forcePromptDrivenImageGeneration) {
|
||
imageUrls = [];
|
||
hasImage = false;
|
||
autoCurrentImageUrlsForFallback = [];
|
||
smartAutoHistoryImages = [];
|
||
autoFromHistoryForFallback = false;
|
||
autoTextForFallback = confirmedPromptForGeneration || requestTextContent;
|
||
smartAutoOptimizedPrompt = confirmedPromptForGeneration;
|
||
chatApiLogger.info("[Smart Auto Mode] 🎯 Using confirmed prompt-only result for image generation");
|
||
}
|
||
|
||
chatApiLogger.info("[Smart Auto Mode] Input:", {
|
||
textContent: requestTextContent.substring(0, 100),
|
||
hasImage,
|
||
imageCount: imageUrls.length,
|
||
fromHistory: smartAutoHistoryImages.length > 0,
|
||
resolutionMode: requestTargetResolution.resolutionMode,
|
||
resolutionConfidence: requestTargetResolution.confidence,
|
||
});
|
||
|
||
// 🆕 从历史消息中提取对话文本（用于上下文理解）
|
||
const historyMessages: { role: "user" | "assistant"; content: string }[] = [];
|
||
for (const msg of smartAutoHistoryMessagesFromDb.slice(-10)) { // 最多取最近10条
|
||
const parts = msg.parts as any[];
|
||
if (parts && parts.length > 0) {
|
||
// 提取文本内容
|
||
const textParts = parts.filter((p: any) => p.type === 'text');
|
||
const content = textParts.map((p: any) => p.text || '').join(' ').trim();
|
||
if (content) {
|
||
historyMessages.push({
|
||
role: msg.role as "user" | "assistant",
|
||
content: content.substring(0, 300), // 限制长度
|
||
});
|
||
}
|
||
}
|
||
}
|
||
chatApiLogger.info("[Smart Auto Mode] History context:", historyMessages.length, "messages");
|
||
|
||
// 2. 使用 LangGraph 驱动的自动模式工作流
|
||
const { runAutoAgentGraph } = await import("@/lib/ai/auto-agent-graph");
|
||
const autoGraphTimeoutMs = getBudgetedTimeout({
|
||
startedAt: requestStartedAt,
|
||
preferredMs: SMART_AUTO_GRAPH_TIMEOUT_MS,
|
||
minimumMs: 8_000,
|
||
});
|
||
if (!autoGraphTimeoutMs) {
|
||
throw new OperationTimeoutError(
|
||
"runAutoAgentGraph.preResponseBudget",
|
||
CHAT_PRE_RESPONSE_SOFT_TIMEOUT_MS
|
||
);
|
||
}
|
||
const composedConversationState = composeConversationStateProtocol({
|
||
bootstrap: conversationStateBootstrap,
|
||
autoContextText: effectiveAutoContextText || undefined,
|
||
resolvedMediaInput: requestResolvedMediaInput,
|
||
targetResolution: requestTargetResolution,
|
||
});
|
||
chatApiLogger.info("[ConversationStateProtocol] Composed:", {
|
||
workflowType: composedConversationState.threadState.workflowState?.type || null,
|
||
workflowStatus: composedConversationState.threadState.workflowState?.status || null,
|
||
memoryHintsCount: composedConversationState.threadState.memoryHints.length,
|
||
contextSummaryPreview: composedConversationState.threadState.contextSummary?.slice(0, 180),
|
||
});
|
||
pendingProtocolDebugEvents.push({
|
||
phase: "composed",
|
||
payload: {
|
||
workflowState: composedConversationState.threadState.workflowState,
|
||
memoryHints: composedConversationState.threadState.memoryHints,
|
||
contextSummary: composedConversationState.threadState.contextSummary,
|
||
},
|
||
});
|
||
const autoGraphResult = await withOperationTimeout(
|
||
runAutoAgentGraph({
|
||
text: smartAutoInputText,
|
||
hasImage,
|
||
imageUrls,
|
||
assetIds: Array.from(
|
||
new Set([
|
||
...extractMessageImageAssetIds(message.parts as any[]),
|
||
...((effectiveContextPayload?.selectedAssetIds || []).filter(Boolean) as string[]),
|
||
]),
|
||
),
|
||
imageFromHistory: smartAutoHistoryImages.length > 0,
|
||
targetResolutionMode: requestTargetResolution.resolutionMode,
|
||
mediaSource: requestResolvedMediaInput.source,
|
||
hasCanvasSelection:
|
||
Boolean(effectiveContextPayload?.selectedNodeIds?.length) ||
|
||
requestResolvedMediaInput.selectedImageUrls.length > 0,
|
||
selectedImageCount: requestResolvedMediaInput.selectedImageUrls.length,
|
||
historyImageCount:
|
||
requestResolvedMediaInput.historyAssistantImageUrls.length +
|
||
requestResolvedMediaInput.historyUserImageUrls.length,
|
||
currentUploadImageCount: requestResolvedMediaInput.currentMessageImageUrls.length,
|
||
historyMessages,
|
||
context:
|
||
composedConversationState.composedConversationMemory.contextSummary ||
|
||
effectiveAutoContextText ||
|
||
undefined,
|
||
preferredModelId: preferredAgentModelId,
|
||
threadState: composedConversationState.threadState,
|
||
}),
|
||
"runAutoAgentGraph",
|
||
autoGraphTimeoutMs,
|
||
);
|
||
|
||
const requestedStoryboardGrid = detectRequestedStoryboardGrid(
|
||
pendingGridConfirmation && isGridConfirmationReply(requestTextContent)
|
||
? pendingGridConfirmation.originalPrompt
|
||
: requestTextContent
|
||
);
|
||
if (requestedStoryboardGrid) {
|
||
storyboardPreferenceMemory = buildStoryboardPreferenceMemoryUpdate({
|
||
existing: storyboardPreferenceMemory,
|
||
preferredGridType: requestedStoryboardGrid.gridType,
|
||
preferredShotCount: requestedStoryboardGrid.shotCount,
|
||
prefersConfirmation: true,
|
||
lastUsedGridType: requestedStoryboardGrid.gridType,
|
||
lastUsedShotCount: requestedStoryboardGrid.shotCount,
|
||
});
|
||
}
|
||
|
||
smartAutoPlan = autoGraphResult.plan;
|
||
smartAutoExecution = autoGraphResult.execution;
|
||
|
||
if (autoGraphResult.workflowDecision) {
|
||
smartAutoWorkflowDecision = autoGraphResult.workflowDecision;
|
||
chatApiLogger.info("[Smart Auto Mode] Workflow decision:", {
|
||
action: autoGraphResult.workflowDecision.action,
|
||
intent: autoGraphResult.workflowDecision.confirmationIntent,
|
||
workflowId: autoGraphResult.workflowDecision.workflowId,
|
||
workflowType: autoGraphResult.workflowDecision.workflowType,
|
||
nextWorkflowStatus: autoGraphResult.workflowDecision.nextWorkflowStatus,
|
||
allowWorkflowExecution: autoGraphResult.workflowDecision.allowWorkflowExecution,
|
||
});
|
||
|
||
if (
|
||
autoGraphResult.workflowDecision.workflowType === "storyboard_video" &&
|
||
autoGraphResult.workflowDecision.workflowId
|
||
) {
|
||
const storyboardTaskId = autoGraphResult.workflowDecision.workflowId;
|
||
try {
|
||
switch (autoGraphResult.workflowDecision.action) {
|
||
case "awaiting_confirmation":
|
||
await markVideoStoryboardAwaitingConfirmation({
|
||
taskId: storyboardTaskId,
|
||
});
|
||
break;
|
||
case "confirmed":
|
||
await markVideoStoryboardConfirmed({
|
||
taskId: storyboardTaskId,
|
||
messageId: message.id,
|
||
});
|
||
break;
|
||
case "revision_requested":
|
||
await markVideoStoryboardDraftForRevision({
|
||
taskId: storyboardTaskId,
|
||
});
|
||
break;
|
||
case "cancel_requested":
|
||
await cancelVideoStoryboardTaskByUser({
|
||
taskId: storyboardTaskId,
|
||
error: "user_cancelled_via_chat",
|
||
});
|
||
break;
|
||
case "pass":
|
||
default:
|
||
break;
|
||
}
|
||
} catch (workflowPersistError) {
|
||
chatApiLogger.warn("[Smart Auto Mode] Failed to persist storyboard workflow decision:", workflowPersistError);
|
||
}
|
||
}
|
||
}
|
||
|
||
if (forcePromptDrivenImageGeneration && confirmedPromptForGeneration) {
|
||
smartAutoDetectedIntent = "image_gen";
|
||
smartAutoOptimizedPrompt = confirmedPromptForGeneration;
|
||
smartAutoExecution = {
|
||
mode: "generate_image",
|
||
routeIntent: "image_gen",
|
||
routeSubIntent: undefined,
|
||
targetSource: "none",
|
||
preferredCapability: "generate_image",
|
||
shouldUseStructuredReply: true,
|
||
shouldOptimizePrompt: true,
|
||
shouldApplyImagePipeline: true,
|
||
shouldApplyVideoPipeline: false,
|
||
shouldStayConversational: false,
|
||
shouldPublishToCanvas: false,
|
||
toolCalls: [],
|
||
availableTools: [],
|
||
};
|
||
smartAutoFriendlyMessage =
|
||
smartAutoFriendlyMessage ||
|
||
(isPromptCarryForwardExecution
|
||
? "收到，我会使用上一轮优化后的提示词，结合你这轮图片继续执行。"
|
||
: "收到，我会按你刚确认的提示词直接生图。");
|
||
}
|
||
|
||
if (smartAutoPlan) {
|
||
chatApiLogger.info("[Smart Auto Mode] Plan result:", {
|
||
goal: smartAutoPlan.goal,
|
||
nextAction: smartAutoPlan.nextAction,
|
||
intent: smartAutoPlan.intent,
|
||
targetSource: smartAutoPlan.targetSource,
|
||
preferredCapability: smartAutoPlan.preferredCapability,
|
||
confidence: smartAutoPlan.confidence,
|
||
toolCalls: smartAutoPlan.toolCalls?.map((tool) => tool.name) || [],
|
||
});
|
||
}
|
||
|
||
if (autoGraphResult.analysis) {
|
||
chatApiLogger.info("[Smart Auto Mode] Analysis result:", {
|
||
intent: autoGraphResult.analysis.intent,
|
||
subIntent: autoGraphResult.analysis.subIntent,
|
||
confidence: autoGraphResult.analysis.confidence,
|
||
needsClarification: autoGraphResult.analysis.needsClarification,
|
||
});
|
||
|
||
const arbitratedIntent = arbitrateAgentIntent({
|
||
text: requestTextContent,
|
||
hasImage,
|
||
hasHistoryImage: smartAutoHistoryImages.length > 0,
|
||
actionResolution: resolvedConversationAction,
|
||
taskBoundaryDecision,
|
||
analysis: autoGraphResult.analysis,
|
||
execution: smartAutoExecution,
|
||
targetSource: requestTargetResolution.resolutionMode,
|
||
});
|
||
if (arbitratedIntent.overridden) {
|
||
chatApiLogger.info("[IntentArbiter] Override auto-agent intent:", {
|
||
fromIntent: autoGraphResult.analysis.intent,
|
||
fromMode: autoGraphResult.execution?.mode,
|
||
toIntent: arbitratedIntent.analysis.intent,
|
||
toMode: arbitratedIntent.execution.mode,
|
||
reason: arbitratedIntent.reason,
|
||
});
|
||
}
|
||
|
||
smartAutoExecution = arbitratedIntent.execution as AgentPreparedExecution;
|
||
smartAutoDetectedIntent = arbitratedIntent.analysis.intent;
|
||
smartAutoNeedsClarification =
|
||
Boolean(arbitratedIntent.analysis.needsClarification) ||
|
||
(!arbitratedIntent.overridden && smartAutoPlan?.nextAction === "clarify");
|
||
smartAutoClarificationQuestion = arbitratedIntent.analysis.clarifyingQuestions?.[0];
|
||
if (arbitratedIntent.friendlyMessage) {
|
||
smartAutoFriendlyMessage = arbitratedIntent.friendlyMessage;
|
||
shouldPreferArbitratedFriendlyMessage = true;
|
||
}
|
||
|
||
const targetExecutionOverride = resolveTargetExecutionOverride({
|
||
hasImage,
|
||
hasSelectedImageHint,
|
||
hasExplicitTargetHint,
|
||
hasActionableEditIntent,
|
||
executionMode: smartAutoExecution?.mode,
|
||
friendlyMessage: smartAutoFriendlyMessage,
|
||
optimizedPrompt: smartAutoOptimizedPrompt,
|
||
requestText: requestTextContent,
|
||
});
|
||
|
||
if (
|
||
smartAutoNeedsClarification &&
|
||
targetExecutionOverride.shouldBypassClarification
|
||
) {
|
||
smartAutoNeedsClarification = false;
|
||
smartAutoClarificationQuestion = undefined;
|
||
smartAutoOptimizedPrompt = targetExecutionOverride.nextOptimizedPrompt;
|
||
smartAutoFriendlyMessage = targetExecutionOverride.nextFriendlyMessage;
|
||
chatApiLogger.info("[Smart Auto Mode] ✅ Skip clarification due to explicit target/selection context");
|
||
}
|
||
}
|
||
|
||
const requestedGrid =
|
||
!pendingGridConfirmation || smartAutoInputText === requestTextContent
|
||
? detectRequestedStoryboardGrid(requestTextContent)
|
||
: pendingGridConfirmation;
|
||
const gridConfirmationResolution = resolveGridConfirmationRequest({
|
||
requestedGrid,
|
||
hasActiveStoryboardTask: Boolean(activeStoryboardTaskForChat?.task?.id),
|
||
executionMode: smartAutoExecution?.mode,
|
||
detectedIntent: smartAutoDetectedIntent,
|
||
requestText: requestTextContent,
|
||
effectiveInputText: smartAutoInputText,
|
||
});
|
||
if (gridConfirmationResolution.shouldRequestConfirmation) {
|
||
shouldAskGridConfirmation = true;
|
||
smartAutoNeedsClarification = true;
|
||
smartAutoClarificationQuestion = gridConfirmationResolution.clarificationMessage;
|
||
smartAutoFriendlyMessage = gridConfirmationResolution.clarificationMessage;
|
||
pendingGridConfirmation = gridConfirmationResolution.pendingGridConfirmation;
|
||
chatApiLogger.info("[Smart Auto Mode] 🧱 Require grid confirmation before generation:", {
|
||
gridType: gridConfirmationResolution.pendingGridConfirmation?.gridType,
|
||
shotCount: gridConfirmationResolution.pendingGridConfirmation?.shotCount,
|
||
});
|
||
}
|
||
|
||
if (autoGraphResult.friendlyMessage && !shouldPreferArbitratedFriendlyMessage) {
|
||
smartAutoFriendlyMessage = autoGraphResult.friendlyMessage;
|
||
chatApiLogger.info("[Smart Auto Mode] 💬 Conversational reply:", smartAutoFriendlyMessage);
|
||
}
|
||
|
||
if (shouldAskGridConfirmation && smartAutoClarificationQuestion) {
|
||
smartAutoFriendlyMessage = smartAutoClarificationQuestion;
|
||
}
|
||
|
||
if (
|
||
!smartAutoNeedsClarification &&
|
||
hasImage &&
|
||
hasActionableEditIntent &&
|
||
(hasSelectedImageHint || hasExplicitTargetHint) &&
|
||
smartAutoFriendlyMessage &&
|
||
(/[?？]$/.test(smartAutoFriendlyMessage.trim()) ||
|
||
/哪一张|哪个|请先确认/.test(smartAutoFriendlyMessage))
|
||
) {
|
||
smartAutoFriendlyMessage = "收到，我将按你指定的目标图直接执行。";
|
||
chatApiLogger.info("[Smart Auto Mode] ✅ Replace clarification-like friendly reply");
|
||
}
|
||
|
||
const autoIntent = smartAutoDetectedIntent || autoGraphResult.analysis?.intent;
|
||
const autoExecutionMode = smartAutoExecution?.mode;
|
||
const isImageTask =
|
||
autoExecutionMode === "generate_image" ||
|
||
autoExecutionMode === "edit_image";
|
||
const needsRicherAutoReply =
|
||
!smartAutoNeedsClarification &&
|
||
!smartAutoFriendlyMessage;
|
||
|
||
if (needsRicherAutoReply) {
|
||
if (isImageTask) {
|
||
smartAutoFriendlyMessage =
|
||
autoGraphResult.analysis?.conversationalReply ||
|
||
"我先分析你这轮图片里的主体、风格和构图，再给你执行方案。";
|
||
} else if (autoIntent === "vision") {
|
||
smartAutoFriendlyMessage =
|
||
autoGraphResult.analysis?.conversationalReply ||
|
||
"收到，我先看图分析，再直接回答你。";
|
||
} else if (autoIntent === "chat") {
|
||
smartAutoFriendlyMessage =
|
||
autoGraphResult.analysis?.conversationalReply ||
|
||
"收到，我们先继续聊你的这个需求。";
|
||
} else {
|
||
smartAutoFriendlyMessage =
|
||
smartAutoExecution?.shouldUseStructuredReply
|
||
? buildAutoModeFallbackMessage({
|
||
text: autoTextForFallback,
|
||
hasImage,
|
||
fromHistory: smartAutoHistoryImages.length > 0,
|
||
})
|
||
: "收到，我先理解你的当前目标，再继续处理。";
|
||
}
|
||
chatApiLogger.info("[Smart Auto Mode] 🧩 Filled missing conversational reply");
|
||
}
|
||
|
||
if (autoGraphResult.optimizedPrompt) {
|
||
smartAutoOptimizedPrompt = autoGraphResult.optimizedPrompt;
|
||
chatApiLogger.info("[Smart Auto Mode] ✨ Prompt optimized (LLM):", {
|
||
original: requestTextContent.substring(0, 50),
|
||
optimized: autoGraphResult.optimizedPrompt.substring(0, 100),
|
||
});
|
||
}
|
||
if (autoGraphResult.videoPlan) {
|
||
smartAutoVideoPlan = autoGraphResult.videoPlan;
|
||
smartAutoRecommendedVideoParams = autoGraphResult.videoPlan.recommendedVideoParams;
|
||
if (!smartAutoOptimizedPrompt && autoGraphResult.videoPlan.finalPrompt?.trim()) {
|
||
smartAutoOptimizedPrompt = autoGraphResult.videoPlan.finalPrompt.trim();
|
||
smartAutoPromptFromVideoPlan = true;
|
||
}
|
||
chatApiLogger.info("[Smart Auto Mode] 🎬 Agent video plan ready:", {
|
||
hasScript: Boolean(autoGraphResult.videoPlan.script),
|
||
storyboardCount: autoGraphResult.videoPlan.storyboard?.length || 0,
|
||
hasFinalPrompt: Boolean(autoGraphResult.videoPlan.finalPrompt),
|
||
recommendedVideoParams: smartAutoRecommendedVideoParams,
|
||
});
|
||
}
|
||
if (autoGraphResult.recommendedImageParams) {
|
||
smartAutoRecommendedImageParams = autoGraphResult.recommendedImageParams;
|
||
chatApiLogger.info("[Smart Auto Mode] 🎛️ Agent recommended image params:", smartAutoRecommendedImageParams);
|
||
}
|
||
|
||
const shouldForceConversationalRouteForVisualModel =
|
||
shouldKeepSelectedModel &&
|
||
requestedModelIsFixedVisual &&
|
||
!requestedModelIsConversationMultimodal &&
|
||
Boolean(smartAutoExecution?.shouldStayConversational);
|
||
|
||
if (autoGraphResult.route && (!shouldKeepSelectedModel || shouldForceConversationalRouteForVisualModel)) {
|
||
const routeIsVideo = isVideoProviderType(autoGraphResult.route.providerType, {
|
||
modelId: autoGraphResult.route.modelId,
|
||
});
|
||
if (routeIsVideo && !allowAgentVideoGeneration) {
|
||
const fallbackIntent: "image_edit" | "image_gen" = hasImage ? "image_edit" : "image_gen";
|
||
smartAutoDetectedIntent = fallbackIntent;
|
||
smartAutoExecution = smartAutoExecution
|
||
? {
|
||
...smartAutoExecution,
|
||
mode: hasImage ? "edit_image" : "generate_image",
|
||
routeIntent: fallbackIntent,
|
||
shouldApplyImagePipeline: true,
|
||
shouldApplyVideoPipeline: false,
|
||
shouldOptimizePrompt: true,
|
||
shouldStayConversational: false,
|
||
shouldUseStructuredReply: true,
|
||
}
|
||
: smartAutoExecution;
|
||
smartAutoVideoPlan = undefined;
|
||
smartAutoRecommendedVideoParams = undefined;
|
||
if (smartAutoPromptFromVideoPlan) {
|
||
smartAutoOptimizedPrompt = requestTextContent.trim() || undefined;
|
||
smartAutoPromptFromVideoPlan = false;
|
||
}
|
||
smartAutoFriendlyMessage =
|
||
smartAutoFriendlyMessage && !/视频/.test(smartAutoFriendlyMessage)
|
||
? smartAutoFriendlyMessage
|
||
: "当前 Agent 默认按图片任务执行；如需生成视频，请先手动切换到“视频”模式。";
|
||
chatApiLogger.info(`[Smart Auto Mode] 🚫 Blocked auto video route (${autoGraphResult.route.displayName}), fallback to ${fallbackIntent}`);
|
||
try {
|
||
const { routeToModel } = await import("@/lib/ai/model-router");
|
||
const fallbackRoute = await routeToModel({
|
||
intent: fallbackIntent,
|
||
subIntent: autoGraphResult.analysis?.subIntent,
|
||
hasImage,
|
||
});
|
||
if (!isVideoProviderType(fallbackRoute.providerType, { modelId: fallbackRoute.modelId })) {
|
||
finalModelId = fallbackRoute.id;
|
||
chatApiLogger.info(`[Smart Auto Mode] ✅ Fallback routed to: ${fallbackRoute.displayName} (${fallbackRoute.reason})`);
|
||
} else {
|
||
chatApiLogger.warn(`[Smart Auto Mode] ⚠️ Fallback route still video (${fallbackRoute.displayName}), keep previous model`);
|
||
}
|
||
} catch (fallbackRouteError) {
|
||
chatApiLogger.error("[Smart Auto Mode] Failed to resolve non-video fallback route:", fallbackRouteError);
|
||
}
|
||
} else {
|
||
finalModelId = autoGraphResult.route.id;
|
||
chatApiLogger.info("console_log", shouldForceConversationalRouteForVisualModel
|
||
? `[Smart Auto Mode] ✅ Switched visual model to conversational route: ${autoGraphResult.route.displayName} (${autoGraphResult.route.reason})`
|
||
: `[Smart Auto Mode] ✅ Routed to: ${autoGraphResult.route.displayName} (${autoGraphResult.route.reason})`);
|
||
}
|
||
} else if (autoGraphResult.route && shouldKeepSelectedModel) {
|
||
finalModelId = requestedModelId;
|
||
chatApiLogger.info("[Smart Auto Mode] 🔒 Keep user-selected model, skip routing override");
|
||
}
|
||
|
||
} catch (autoError) {
|
||
// 智能分析失败时，回退到简单逻辑
|
||
chatApiLogger.error("[Smart Auto Mode] ❌ Error, falling back:", autoError);
|
||
const isAutoTimeout = autoError instanceof OperationTimeoutError;
|
||
if (isAutoTimeout) {
|
||
chatApiLogger.warn(`[Smart Auto Mode] ⏱️ Auto-agent timed out or exceeded pre-response budget (${CHAT_PRE_RESPONSE_SOFT_TIMEOUT_MS}ms), using local fallback`);
|
||
}
|
||
try {
|
||
const { analyzeIntent } = await import("@/lib/ai/smart-analyzer");
|
||
const rescueImageUrls =
|
||
autoCurrentImageUrlsForFallback.length > 0
|
||
? autoCurrentImageUrlsForFallback
|
||
: smartAutoHistoryImages;
|
||
const rescueHasImage = rescueImageUrls.length > 0;
|
||
const rescueFromHistory =
|
||
rescueHasImage &&
|
||
autoCurrentImageUrlsForFallback.length === 0 &&
|
||
smartAutoHistoryImages.length > 0;
|
||
const rescueAssetIds = Array.from(
|
||
new Set([
|
||
...extractMessageImageAssetIds(message.parts as any[]),
|
||
...((effectiveContextPayload?.selectedAssetIds || []).filter(Boolean) as string[]),
|
||
]),
|
||
);
|
||
const rescueTimeoutMs = getBudgetedTimeout({
|
||
startedAt: requestStartedAt,
|
||
preferredMs: SMART_AUTO_FALLBACK_TIMEOUT_MS,
|
||
minimumMs: 5_000,
|
||
});
|
||
if (rescueTimeoutMs) {
|
||
const rescueAnalysis = await withOperationTimeout(
|
||
analyzeIntent({
|
||
text: autoTextForFallback,
|
||
hasImage: rescueHasImage,
|
||
imageUrls: rescueImageUrls,
|
||
assetIds: rescueAssetIds,
|
||
imageFromHistory: rescueFromHistory || autoFromHistoryForFallback,
|
||
historyMessages: [],
|
||
context: undefined,
|
||
preferredModelId: preferredAgentModelId,
|
||
}),
|
||
"fallbackAnalyzeIntent",
|
||
rescueTimeoutMs,
|
||
);
|
||
smartAutoDetectedIntent = rescueAnalysis.intent;
|
||
const arbitratedIntent = arbitrateAgentIntent({
|
||
text: autoTextForFallback,
|
||
hasImage: rescueHasImage,
|
||
hasHistoryImage: rescueFromHistory || autoFromHistoryForFallback,
|
||
actionResolution: resolvedConversationAction,
|
||
taskBoundaryDecision,
|
||
analysis: rescueAnalysis,
|
||
execution: undefined,
|
||
targetSource: requestTargetResolution?.resolutionMode || "none",
|
||
});
|
||
if (arbitratedIntent.overridden) {
|
||
chatApiLogger.info("[IntentArbiter] Override rescue intent:", {
|
||
fromIntent: rescueAnalysis.intent,
|
||
toIntent: arbitratedIntent.analysis.intent,
|
||
toMode: arbitratedIntent.execution.mode,
|
||
reason: arbitratedIntent.reason,
|
||
});
|
||
}
|
||
smartAutoDetectedIntent = arbitratedIntent.analysis.intent;
|
||
smartAutoNeedsClarification = Boolean(arbitratedIntent.analysis.needsClarification);
|
||
smartAutoClarificationQuestion = arbitratedIntent.analysis.clarifyingQuestions?.[0];
|
||
smartAutoExecution = arbitratedIntent.execution as AgentPreparedExecution;
|
||
if (arbitratedIntent.friendlyMessage || rescueAnalysis.conversationalReply?.trim()) {
|
||
smartAutoFriendlyMessage = (
|
||
arbitratedIntent.friendlyMessage ||
|
||
rescueAnalysis.conversationalReply ||
|
||
""
|
||
).trim();
|
||
}
|
||
} else {
|
||
chatApiLogger.warn("[Smart Auto Mode] Skip rescue analysis due to pre-response budget");
|
||
}
|
||
} catch (rescueError) {
|
||
chatApiLogger.error("[Smart Auto Mode] Rescue analysis failed:", rescueError);
|
||
try {
|
||
const { fallbackKeywordAnalysis } = await import("@/lib/ai/smart-analyzer");
|
||
const keywordFallback = fallbackKeywordAnalysis(
|
||
autoTextForFallback,
|
||
autoCurrentImageUrlsForFallback.length > 0 || smartAutoHistoryImages.length > 0,
|
||
);
|
||
const arbitratedIntent = arbitrateAgentIntent({
|
||
text: autoTextForFallback,
|
||
hasImage:
|
||
autoCurrentImageUrlsForFallback.length > 0 || smartAutoHistoryImages.length > 0,
|
||
hasHistoryImage: smartAutoHistoryImages.length > 0 || autoFromHistoryForFallback,
|
||
actionResolution: resolvedConversationAction,
|
||
taskBoundaryDecision,
|
||
analysis: keywordFallback,
|
||
execution: undefined,
|
||
targetSource: requestTargetResolution?.resolutionMode || "none",
|
||
});
|
||
if (arbitratedIntent.overridden) {
|
||
chatApiLogger.info("[IntentArbiter] Override keyword-fallback intent:", {
|
||
fromIntent: keywordFallback.intent,
|
||
toIntent: arbitratedIntent.analysis.intent,
|
||
toMode: arbitratedIntent.execution.mode,
|
||
reason: arbitratedIntent.reason,
|
||
});
|
||
}
|
||
smartAutoDetectedIntent = arbitratedIntent.analysis.intent;
|
||
smartAutoNeedsClarification = Boolean(arbitratedIntent.analysis.needsClarification);
|
||
smartAutoClarificationQuestion = arbitratedIntent.analysis.clarifyingQuestions?.[0];
|
||
smartAutoExecution = arbitratedIntent.execution as AgentPreparedExecution;
|
||
if (arbitratedIntent.friendlyMessage || keywordFallback.conversationalReply?.trim()) {
|
||
smartAutoFriendlyMessage = (
|
||
arbitratedIntent.friendlyMessage ||
|
||
keywordFallback.conversationalReply ||
|
||
""
|
||
).trim();
|
||
chatApiLogger.warn("[Smart Auto Mode] Applied keyword fallback after rescue failure");
|
||
}
|
||
} catch (keywordFallbackError) {
|
||
chatApiLogger.warn("[Smart Auto Mode] Keyword fallback after rescue failure also failed:", keywordFallbackError);
|
||
}
|
||
}
|
||
|
||
if (
|
||
!smartAutoOptimizedPrompt &&
|
||
smartAutoExecution?.shouldOptimizePrompt &&
|
||
(smartAutoDetectedIntent === "image_edit" || smartAutoDetectedIntent === "image_gen")
|
||
) {
|
||
try {
|
||
const { optimizePrompt } = await import("@/lib/ai/prompt-optimizer");
|
||
const optimizedFallback = optimizePrompt({
|
||
originalPrompt: autoTextForFallback,
|
||
intent: smartAutoDetectedIntent,
|
||
subIntent: smartAutoExecution.routeSubIntent,
|
||
});
|
||
if (optimizedFallback.optimizedPrompt?.trim()) {
|
||
smartAutoOptimizedPrompt = optimizedFallback.optimizedPrompt.trim();
|
||
chatApiLogger.info("[Smart Auto Mode] ✨ Prompt optimized (fallback heuristic):", {
|
||
original: autoTextForFallback.substring(0, 80),
|
||
optimized: smartAutoOptimizedPrompt.substring(0, 140),
|
||
intent: smartAutoDetectedIntent,
|
||
subIntent: smartAutoExecution.routeSubIntent || null,
|
||
});
|
||
}
|
||
if (!smartAutoRecommendedImageParams && optimizedFallback.recommendedImageParams) {
|
||
smartAutoRecommendedImageParams = optimizedFallback.recommendedImageParams;
|
||
}
|
||
} catch (fallbackOptimizeError) {
|
||
chatApiLogger.warn("[Smart Auto Mode] Fallback prompt optimization failed:", fallbackOptimizeError);
|
||
}
|
||
}
|
||
|
||
const allModels = await getEnabledAiModelsWithProvider();
|
||
const hasImage =
|
||
autoCurrentImageUrlsForFallback.length > 0 || smartAutoHistoryImages.length > 0;
|
||
let fallbackVisualSummary: string | undefined;
|
||
|
||
if (isAutoTimeout && hasImage) {
|
||
try {
|
||
const currentImageUrls =
|
||
autoCurrentImageUrlsForFallback.length > 0
|
||
? autoCurrentImageUrlsForFallback
|
||
: collectImageInputsFromMessageParts(message.parts as any[], requestOrigin);
|
||
const summaryImageUrls =
|
||
currentImageUrls.length > 0 ? currentImageUrls : smartAutoHistoryImages;
|
||
|
||
if (summaryImageUrls.length > 0) {
|
||
const { summarizeImageForExecutionReport } = await import("@/lib/ai/smart-analyzer");
|
||
const summaryAssetIds = Array.from(
|
||
new Set([
|
||
...extractMessageImageAssetIds(message.parts as any[]),
|
||
...((effectiveContextPayload?.selectedAssetIds || []).filter(Boolean) as string[]),
|
||
]),
|
||
);
|
||
const fallbackSummaryTimeoutMs = getBudgetedTimeout({
|
||
startedAt: requestStartedAt,
|
||
preferredMs: SMART_AUTO_IMAGE_SUMMARY_TIMEOUT_MS,
|
||
minimumMs: 5_000,
|
||
});
|
||
if (fallbackSummaryTimeoutMs) {
|
||
const summary = await withOperationTimeout(
|
||
summarizeImageForExecutionReport({
|
||
text: autoTextForFallback,
|
||
imageUrls: summaryImageUrls,
|
||
context: effectiveAutoContextText || undefined,
|
||
preferredModelId: preferredAgentModelId,
|
||
assetIds: summaryAssetIds,
|
||
timeoutMs: fallbackSummaryTimeoutMs,
|
||
}),
|
||
"fallbackVisualSummary",
|
||
fallbackSummaryTimeoutMs,
|
||
);
|
||
if (summary?.trim()) {
|
||
fallbackVisualSummary = summary.trim();
|
||
}
|
||
} else {
|
||
chatApiLogger.warn("[Smart Auto Mode] Skip fallback visual summary due to pre-response budget");
|
||
}
|
||
}
|
||
} catch (summaryError) {
|
||
chatApiLogger.warn("[Smart Auto Mode] Fallback visual summary failed:", summaryError);
|
||
}
|
||
}
|
||
|
||
// 回退路由：优先按分析出的意图选模型，避免“有图就生图”
|
||
let bestModel = null;
|
||
try {
|
||
const fallbackIntent =
|
||
smartAutoDetectedIntent || (hasImage ? "vision" : "chat");
|
||
const { routeToModel } = await import("@/lib/ai/model-router");
|
||
const fallbackRoute = await routeToModel({
|
||
intent: fallbackIntent,
|
||
subIntent: smartAutoExecution?.routeSubIntent,
|
||
hasImage,
|
||
});
|
||
bestModel = allModels.find(
|
||
(item) => item.model.id === fallbackRoute.id || item.model.modelId === fallbackRoute.modelId,
|
||
) || null;
|
||
if (bestModel) {
|
||
chatApiLogger.info(`[Smart Auto Mode] Fallback routed by intent: ${fallbackRoute.displayName} (${fallbackRoute.reason})`);
|
||
}
|
||
} catch (fallbackRouteError) {
|
||
chatApiLogger.warn("[Smart Auto Mode] Fallback routeByIntent failed, use local heuristic:", fallbackRouteError);
|
||
}
|
||
|
||
if (!bestModel && hasImage) {
|
||
bestModel = allModels.find(
|
||
(m) =>
|
||
m.model.supportsImageGen &&
|
||
!m.model.isSubModel &&
|
||
!isVideoProviderType(m.provider?.type, m.model),
|
||
);
|
||
if (isAutoTimeout) {
|
||
smartAutoFriendlyMessage = buildAutoModeFallbackMessage({
|
||
text: autoTextForFallback,
|
||
hasImage: true,
|
||
fromHistory: autoFromHistoryForFallback || smartAutoHistoryImages.length > 0,
|
||
imageSummary: fallbackVisualSummary,
|
||
});
|
||
} else {
|
||
smartAutoFriendlyMessage =
|
||
smartAutoFriendlyMessage ||
|
||
buildAutoModeFallbackMessage({
|
||
text: autoTextForFallback,
|
||
hasImage: true,
|
||
fromHistory: autoFromHistoryForFallback || smartAutoHistoryImages.length > 0,
|
||
imageSummary: fallbackVisualSummary,
|
||
});
|
||
}
|
||
} else {
|
||
smartAutoFriendlyMessage = smartAutoFriendlyMessage || buildAutoModeFallbackMessage({
|
||
text: autoTextForFallback,
|
||
hasImage: false,
|
||
fromHistory: false,
|
||
});
|
||
}
|
||
if (!bestModel) {
|
||
bestModel = allModels.find(
|
||
(m) =>
|
||
m.model.isDefault &&
|
||
!m.model.isSubModel &&
|
||
!isVideoProviderType(m.provider?.type, m.model),
|
||
);
|
||
}
|
||
if (!bestModel && allModels.length > 0) {
|
||
bestModel = allModels.find(
|
||
(m) => !m.model.isSubModel && !isVideoProviderType(m.provider?.type, m.model),
|
||
) || allModels.find((m) => !m.model.isSubModel) || allModels[0];
|
||
}
|
||
|
||
if (shouldKeepSelectedModel) {
|
||
finalModelId = requestedModelId;
|
||
chatApiLogger.info("[Smart Auto Mode] 🔒 Keep user-selected model in fallback mode");
|
||
} else if (bestModel) {
|
||
finalModelId = bestModel.model.id;
|
||
chatApiLogger.info(`[Smart Auto Mode] Fallback selected: ${bestModel.model.displayName}`);
|
||
} else {
|
||
return new ChatSDKError("bad_request:api", "No available models for auto mode").toResponse();
|
||
}
|
||
}
|
||
}
|
||
// ========== End Smart Auto Mode Logic ==========
|
||
|
||
if (shouldRunAgentConversation && !allowAgentVideoGeneration) {
|
||
const finalModelIsUuid =
|
||
/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test(finalModelId);
|
||
let finalProviderType: string | undefined;
|
||
let finalModelConfig: Awaited<ReturnType<typeof getAiModelById>> | null = null;
|
||
|
||
if (finalModelIsUuid) {
|
||
finalModelConfig = await getAiModelById({ id: finalModelId }).catch(() => null);
|
||
if (finalModelConfig) {
|
||
finalProviderType = (
|
||
await getAiProviderById({ id: finalModelConfig.providerId }).catch(() => null)
|
||
)?.type as string | undefined;
|
||
}
|
||
}
|
||
|
||
if (isVideoProviderType(finalProviderType, finalModelConfig || undefined)) {
|
||
const allModels = await getEnabledAiModelsWithProvider(false);
|
||
const fallbackModel =
|
||
allModels.find(
|
||
(m) =>
|
||
!m.model.isSubModel &&
|
||
m.model.supportsImageGen &&
|
||
!isVideoProviderType(m.provider?.type, m.model),
|
||
) ||
|
||
allModels.find(
|
||
(m) =>
|
||
!m.model.isSubModel &&
|
||
!isVideoProviderType(m.provider?.type, m.model),
|
||
);
|
||
|
||
if (fallbackModel) {
|
||
const hasImageInput = message.parts.some((part: any) =>
|
||
part?.type === "file" || part?.type === "image" || part?.type === "image_url"
|
||
);
|
||
smartAutoDetectedIntent = hasImageInput ? "image_edit" : "image_gen";
|
||
smartAutoExecution = smartAutoExecution
|
||
? {
|
||
...smartAutoExecution,
|
||
mode: hasImageInput ? "edit_image" : "generate_image",
|
||
routeIntent: hasImageInput ? "image_edit" : "image_gen",
|
||
shouldApplyImagePipeline: true,
|
||
shouldApplyVideoPipeline: false,
|
||
shouldOptimizePrompt: true,
|
||
shouldStayConversational: false,
|
||
shouldUseStructuredReply: true,
|
||
}
|
||
: smartAutoExecution;
|
||
finalModelId = fallbackModel.model.id;
|
||
smartAutoVideoPlan = undefined;
|
||
smartAutoRecommendedVideoParams = undefined;
|
||
chatApiLogger.info(`[Smart Auto Mode] 🚫 Safety override: replace video model (${finalProviderType}) with ${fallbackModel.model.displayName}`);
|
||
}
|
||
}
|
||
}
|
||
|
||
if (
|
||
shouldRunAgentConversation &&
|
||
!smartAutoNeedsClarification &&
|
||
!smartAutoFriendlyMessage?.trim()
|
||
) {
|
||
const latestUserText = message.parts
|
||
.filter((part): part is { type: "text"; text: string } => part.type === "text")
|
||
.map((part) => part.text)
|
||
.join(" ");
|
||
const hasCurrentImageInput = message.parts.some((part: any) =>
|
||
part?.type === "file" || part?.type === "image" || part?.type === "image_url"
|
||
);
|
||
const hasHistoryImageInput = smartAutoHistoryImages.length > 0;
|
||
smartAutoFriendlyMessage =
|
||
smartAutoExecution?.shouldUseStructuredReply
|
||
? buildAutoModeFallbackMessage({
|
||
text: latestUserText,
|
||
hasImage: hasCurrentImageInput || hasHistoryImageInput,
|
||
fromHistory: !hasCurrentImageInput && hasHistoryImageInput,
|
||
})
|
||
: "收到，我先理解你的当前目标，再继续处理。";
|
||
chatApiLogger.info("[Smart Auto Mode] 🧩 Backfilled conversational reply with fallback");
|
||
}
|
||
|
||
if (
|
||
shouldRunAgentConversation &&
|
||
!smartAutoNeedsClarification &&
|
||
smartAutoExecution?.shouldUseStructuredReply
|
||
) {
|
||
const latestUserText = message.parts
|
||
.filter((part): part is { type: "text"; text: string } => part.type === "text")
|
||
.map((part) => part.text)
|
||
.join(" ");
|
||
const hasCurrentImageInput = message.parts.some((part: any) =>
|
||
part?.type === "file" || part?.type === "image" || part?.type === "image_url"
|
||
);
|
||
const hasHistoryImageInput = smartAutoHistoryImages.length > 0;
|
||
smartAutoFriendlyMessage = ensureStructuredAgentFriendlyMessage({
|
||
message: smartAutoFriendlyMessage,
|
||
text: latestUserText,
|
||
hasImage: hasCurrentImageInput || hasHistoryImageInput,
|
||
fromHistory: !hasCurrentImageInput && hasHistoryImageInput,
|
||
});
|
||
}
|
||
|
||
const hasUserCustomImageParams = hasUserCustomizedImageParams(requestBody.imageParams);
|
||
const hasUserCustomVideoParams = hasUserCustomizedVideoParams(requestBody.videoParams);
|
||
const shouldApplyAgentImageParams =
|
||
shouldRunAgentConversation &&
|
||
!smartAutoNeedsClarification &&
|
||
Boolean(smartAutoExecution?.shouldApplyImagePipeline);
|
||
const shouldApplyAgentVideoParams =
|
||
shouldRunAgentConversation &&
|
||
!smartAutoNeedsClarification &&
|
||
allowAgentVideoGeneration &&
|
||
Boolean(smartAutoExecution?.shouldApplyVideoPipeline);
|
||
const effectiveImageParams = shouldApplyAgentImageParams
|
||
? mergeImageParamsWithAgentRecommendation({
|
||
incoming: requestBody.imageParams,
|
||
recommended: smartAutoRecommendedImageParams,
|
||
intent: smartAutoDetectedIntent,
|
||
hasUserCustomSettings: hasUserCustomImageParams,
|
||
})
|
||
: requestBody.imageParams;
|
||
const effectiveVideoParams = shouldApplyAgentVideoParams
|
||
? mergeVideoParamsWithAgentRecommendation({
|
||
incoming: requestBody.videoParams,
|
||
recommended: smartAutoRecommendedVideoParams,
|
||
hasUserCustomSettings: hasUserCustomVideoParams,
|
||
})
|
||
: requestBody.videoParams;
|
||
const defaultImageQualityParams = {
|
||
aspectRatio: "auto",
|
||
imageSize: smartAutoDetectedIntent === "image_edit" ? "4K" : "2K",
|
||
imageCount: 1,
|
||
} as const;
|
||
|
||
if (shouldApplyAgentImageParams) {
|
||
chatApiLogger.info("[Smart Auto Mode] 🎯 Effective image params:", {
|
||
incoming: requestBody.imageParams,
|
||
recommended: smartAutoRecommendedImageParams,
|
||
merged: effectiveImageParams,
|
||
hasUserCustomImageParams,
|
||
});
|
||
}
|
||
if (shouldApplyAgentVideoParams) {
|
||
chatApiLogger.info("[Smart Auto Mode] 🎯 Effective video params:", {
|
||
incoming: requestBody.videoParams,
|
||
recommended: smartAutoRecommendedVideoParams,
|
||
merged: effectiveVideoParams,
|
||
hasUserCustomVideoParams,
|
||
hasVideoPlan: Boolean(smartAutoVideoPlan),
|
||
});
|
||
}
|
||
|
||
if (
|
||
shouldRunAgentConversation &&
|
||
smartAutoFriendlyMessage &&
|
||
smartAutoExecution?.shouldUseStructuredReply
|
||
) {
|
||
smartAutoFriendlyMessage = sanitizeAgentOutputSettingsSection({
|
||
message: smartAutoFriendlyMessage,
|
||
userImageParams: requestBody.imageParams,
|
||
hasUserCustomImageParams,
|
||
});
|
||
smartAutoFriendlyMessage = injectOptimizedPromptSection({
|
||
message: smartAutoFriendlyMessage,
|
||
originalText: requestTextContent,
|
||
optimizedPrompt: smartAutoOptimizedPrompt,
|
||
});
|
||
}
|
||
|
||
if (shouldRunAgentConversation) {
|
||
const currentImageUrls = extractMessageImageUrls(message.parts as any[]);
|
||
const summaryImageUrls =
|
||
currentImageUrls.length > 0 ? currentImageUrls : smartAutoHistoryImages;
|
||
const shouldRequireImageAnalysis = summaryImageUrls.length > 0;
|
||
|
||
if (shouldRequireImageAnalysis) {
|
||
try {
|
||
const { summarizeImageForExecutionReport } = await import("@/lib/ai/smart-analyzer");
|
||
const latestUserText = message.parts
|
||
.filter((part): part is { type: "text"; text: string } => part.type === "text")
|
||
.map((part) => part.text)
|
||
.join(" ");
|
||
const summaryAssetIds = Array.from(
|
||
new Set([
|
||
...extractMessageImageAssetIds(message.parts as any[]),
|
||
...((effectiveContextPayload?.selectedAssetIds || []).filter(Boolean) as string[]),
|
||
]),
|
||
);
|
||
|
||
let summary: string | null = null;
|
||
const summaryTimeoutMs = getBudgetedTimeout({
|
||
startedAt: requestStartedAt,
|
||
preferredMs: SMART_AUTO_IMAGE_SUMMARY_TIMEOUT_MS,
|
||
minimumMs: 6_000,
|
||
});
|
||
if (summaryTimeoutMs) {
|
||
summary = await withOperationTimeout(
|
||
summarizeImageForExecutionReport({
|
||
text: latestUserText,
|
||
imageUrls: summaryImageUrls,
|
||
context: effectiveAutoContextText || undefined,
|
||
preferredModelId: preferredAgentModelId,
|
||
assetIds: summaryAssetIds,
|
||
timeoutMs: summaryTimeoutMs,
|
||
}),
|
||
"enrichImageAnalysisSummary",
|
||
summaryTimeoutMs,
|
||
);
|
||
} else {
|
||
chatApiLogger.warn("[Smart Auto Mode] Skip image summary enrichment due to pre-response budget");
|
||
}
|
||
|
||
if (summary?.trim() && smartAutoExecution?.shouldUseStructuredReply) {
|
||
const baseMessage = ensureStructuredAgentFriendlyMessage({
|
||
message: smartAutoFriendlyMessage,
|
||
text: latestUserText,
|
||
hasImage: true,
|
||
fromHistory: currentImageUrls.length === 0 && summaryImageUrls.length > 0,
|
||
});
|
||
smartAutoFriendlyMessage = ensureImageAnalysisSummarySection(
|
||
baseMessage ||
|
||
buildAutoModeFallbackMessage({
|
||
text: latestUserText,
|
||
hasImage: true,
|
||
fromHistory: currentImageUrls.length === 0 && summaryImageUrls.length > 0,
|
||
imageSummary: summary.trim(),
|
||
}),
|
||
summary.trim(),
|
||
);
|
||
chatApiLogger.info("[Smart Auto Mode] ✅ Image analysis summary ready:", summary.trim().slice(0, 160));
|
||
} else if (
|
||
summary?.trim() &&
|
||
!smartAutoExecution?.shouldUseStructuredReply &&
|
||
!smartAutoFriendlyMessage
|
||
) {
|
||
smartAutoFriendlyMessage = "收到，我会结合这张图的内容直接回答你。";
|
||
} else if (smartAutoFriendlyMessage && shouldEnrichImageAnalysisText(smartAutoFriendlyMessage)) {
|
||
chatApiLogger.warn("[Smart Auto Mode] ⚠️ Image summary unavailable, keep original analysis text");
|
||
}
|
||
} catch (enrichError) {
|
||
chatApiLogger.warn("[Smart Auto Mode] Skip image analysis enrichment:", enrichError);
|
||
}
|
||
}
|
||
}
|
||
|
||
if (
|
||
shouldRunAgentConversation &&
|
||
smartAutoFriendlyMessage &&
|
||
smartAutoExecution?.shouldUseStructuredReply
|
||
) {
|
||
smartAutoFriendlyMessage = normalizeAgentReportForDisplay(smartAutoFriendlyMessage);
|
||
}
|
||
|
||
if (
|
||
shouldRunAgentConversation &&
|
||
!shouldKeepSelectedModel &&
|
||
smartAutoExecution?.shouldStayConversational &&
|
||
preferredAgentModelId
|
||
) {
|
||
if (finalModelId !== preferredAgentModelId) {
|
||
finalModelId = preferredAgentModelId;
|
||
chatApiLogger.info(`[Smart Auto Mode] 🧠 Conversational reply follows agent default model: ${configuredAgentConversationModelId || preferredAgentModelId}`);
|
||
}
|
||
}
|
||
|
||
if (shouldRunAgentConversation && smartAutoExecution?.shouldStayConversational) {
|
||
const candidateReplyDescriptor = await resolveAgentModelSlotDescriptor(finalModelId).catch(
|
||
() => null,
|
||
);
|
||
if (candidateReplyDescriptor) {
|
||
agentModelSlots = upsertAgentModelSlot(agentModelSlots, "reply", {
|
||
...candidateReplyDescriptor,
|
||
source: "candidate_reply_model",
|
||
reason: "post_agent_routing",
|
||
locked: shouldKeepSelectedModel,
|
||
});
|
||
|
||
const providerHealth = getProviderHealthSnapshot(candidateReplyDescriptor.providerKey);
|
||
if (
|
||
!shouldKeepSelectedModel &&
|
||
shouldAvoidProvider(providerHealth)
|
||
) {
|
||
let fallbackReplyModelId: string | undefined;
|
||
|
||
if (
|
||
preferredAgentModelId &&
|
||
preferredAgentModelId !== finalModelId
|
||
) {
|
||
const preferredDescriptor = await resolveAgentModelSlotDescriptor(
|
||
preferredAgentModelId,
|
||
).catch(() => null);
|
||
if (
|
||
preferredDescriptor &&
|
||
!shouldAvoidProvider(getProviderHealthSnapshot(preferredDescriptor.providerKey))
|
||
) {
|
||
fallbackReplyModelId = preferredAgentModelId;
|
||
}
|
||
}
|
||
|
||
if (!fallbackReplyModelId) {
|
||
try {
|
||
const { routeToModel } = await import("@/lib/ai/model-router");
|
||
const replyFallbackRoute = await routeToModel({
|
||
intent: "chat",
|
||
subIntent: undefined,
|
||
hasImage: false,
|
||
});
|
||
const fallbackDescriptor = await resolveAgentModelSlotDescriptor(
|
||
replyFallbackRoute.id,
|
||
).catch(() => null);
|
||
if (
|
||
fallbackDescriptor &&
|
||
!shouldAvoidProvider(getProviderHealthSnapshot(fallbackDescriptor.providerKey))
|
||
) {
|
||
fallbackReplyModelId = replyFallbackRoute.id;
|
||
}
|
||
} catch (replyRouteError) {
|
||
chatApiLogger.warn("[Smart Auto Mode] Failed to resolve reply fallback route after provider degradation:", replyRouteError);
|
||
}
|
||
}
|
||
|
||
if (fallbackReplyModelId && fallbackReplyModelId !== finalModelId) {
|
||
finalModelId = fallbackReplyModelId;
|
||
const fallbackDescriptor = await resolveAgentModelSlotDescriptor(finalModelId).catch(
|
||
() => null,
|
||
);
|
||
if (fallbackDescriptor) {
|
||
agentModelSlots = upsertAgentModelSlot(agentModelSlots, "reply", {
|
||
...fallbackDescriptor,
|
||
source: "provider_health_fallback",
|
||
reason: providerHealth?.lastEventType || "degraded_provider",
|
||
locked: false,
|
||
});
|
||
}
|
||
chatApiLogger.warn(`[Smart Auto Mode] ♻️ Switched reply model due provider health: ${candidateReplyDescriptor.displayName || candidateReplyDescriptor.modelId} -> ${fallbackDescriptor?.displayName || fallbackReplyModelId}`);
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
const session = await auth();
|
||
|
||
if (!session?.user) {
|
||
return new ChatSDKError("unauthorized:chat").toResponse();
|
||
}
|
||
|
||
const userType: UserType = session.user.type;
|
||
|
||
const messageCount = await getMessageCountByUserId({
|
||
id: session.user.id,
|
||
differenceInHours: 24,
|
||
});
|
||
|
||
if (messageCount > entitlementsByUserType[userType].maxMessagesPerDay) {
|
||
return new ChatSDKError("rate_limit:chat").toResponse();
|
||
}
|
||
|
||
// ========== 使用时间限制验证（后端安全检查） ==========
|
||
try {
|
||
// 从数据库获取网站设置
|
||
const { siteSettings } = await import("@/lib/db/schema");
|
||
const [settings] = await db.select().from(siteSettings).limit(1);
|
||
|
||
if (settings?.usageTimeEnabled) {
|
||
const now = new Date();
|
||
const currentTime = `${String(now.getHours()).padStart(2, "0")}:${String(now.getMinutes()).padStart(2, "0")}`;
|
||
const startTime = settings.usageTimeStart || "08:00";
|
||
const endTime = settings.usageTimeEnd || "20:00";
|
||
|
||
const isWithinUsageTime = currentTime >= startTime && currentTime <= endTime;
|
||
|
||
if (!isWithinUsageTime) {
|
||
chatApiLogger.info(`[Chat API] ⛔ 访问被拒绝：当前时间 ${currentTime} 不在允许范围 ${startTime}-${endTime}`);
|
||
return NextResponse.json(
|
||
{
|
||
error: "usage_time_restricted",
|
||
message: settings.usageTimeMessage || "当前不在系统使用时间段内",
|
||
currentTime,
|
||
allowedTime: `${startTime} - ${endTime}`
|
||
},
|
||
{ status: 403 }
|
||
);
|
||
}
|
||
|
||
chatApiLogger.info(`[Chat API] ✅ 时间验证通过：${currentTime} 在允许范围 ${startTime}-${endTime}`);
|
||
}
|
||
} catch (timeCheckError) {
|
||
chatApiLogger.error("[Chat API] ⚠️ 时间验证失败（继续执行）:", timeCheckError);
|
||
// 如果时间检查失败，继续执行（避免因检查逻辑错误导致系统不可用）
|
||
}
|
||
|
||
let chat = await getChatById({ id });
|
||
let messagesFromDb: DBMessage[] = [];
|
||
let messagesForMediaInput: DBMessage[] = [];
|
||
|
||
if (chat) {
|
||
if (chat.userId !== session.user.id) {
|
||
return new ChatSDKError("forbidden:chat").toResponse();
|
||
}
|
||
// Only fetch messages if chat already exists
|
||
messagesFromDb = await getMessagesByChatId({ id });
|
||
} else {
|
||
const title = await generateTitleFromUserMessage({
|
||
message,
|
||
});
|
||
|
||
try {
|
||
await saveChat({
|
||
id,
|
||
userId: session.user.id,
|
||
title,
|
||
visibility: selectedVisibilityType,
|
||
});
|
||
} catch (error) {
|
||
const racedChat = await findChatWithRetry(id);
|
||
if (!racedChat) {
|
||
throw error;
|
||
}
|
||
|
||
if (racedChat.userId !== session.user.id) {
|
||
return new ChatSDKError("forbidden:chat").toResponse();
|
||
}
|
||
|
||
chatApiLogger.warn("[Chat API] saveChat conflicted, reused existing chat:", {
|
||
chatId: id,
|
||
userId: session.user.id,
|
||
});
|
||
}
|
||
|
||
chat = await findChatWithRetry(id);
|
||
if (!chat) {
|
||
throw new ChatSDKError("bad_request:database", "Failed to save chat");
|
||
}
|
||
|
||
if (chat.userId !== session.user.id) {
|
||
return new ChatSDKError("forbidden:chat").toResponse();
|
||
}
|
||
|
||
// In a race scenario another request may have already written messages.
|
||
messagesFromDb = await getMessagesByChatId({ id });
|
||
}
|
||
|
||
const loadedMessagesForBoundary = shouldIsolateDirectComposerContext ? [] : messagesFromDb;
|
||
messagesFromDb = shouldIsolateDirectComposerContext
|
||
? []
|
||
: pruneTaskBoundaryHistoryMessages({
|
||
decision: taskBoundaryDecision,
|
||
messages: loadedMessagesForBoundary,
|
||
purpose: "model",
|
||
hasCurrentImageInput: hasImageInputForConversationAction,
|
||
});
|
||
messagesForMediaInput = shouldIsolateDirectComposerContext
|
||
? []
|
||
: pruneTaskBoundaryHistoryMessages({
|
||
decision: taskBoundaryDecision,
|
||
messages: loadedMessagesForBoundary,
|
||
purpose: "media",
|
||
hasCurrentImageInput: hasImageInputForConversationAction,
|
||
});
|
||
|
||
if (!requestWorkingState && !shouldIsolateDirectComposerContext) {
|
||
requestWorkingState = await getConversationWorkingStateByChatId({
|
||
chatId: id,
|
||
});
|
||
}
|
||
if (!requestResolvedMediaInput) {
|
||
requestResolvedMediaInput = resolveChatMediaInput({
|
||
messageParts: message.parts as any[],
|
||
historyMessages: messagesForMediaInput,
|
||
contextPayload: effectiveContextPayload,
|
||
requestOrigin,
|
||
maxImages: 4,
|
||
});
|
||
}
|
||
if (!requestTargetResolution) {
|
||
requestTargetResolution = resolveTargetAssets({
|
||
text: requestTextContent,
|
||
mediaInput: requestResolvedMediaInput,
|
||
contextPayload: effectiveContextPayload,
|
||
workingState: effectiveWorkingState || undefined,
|
||
});
|
||
}
|
||
if (
|
||
requestResolvedMediaInput.currentMessageImageUrls.length === 0 &&
|
||
smartAutoHistoryImages.length === 0 &&
|
||
requestTargetResolution.targetImageUrls.length > 0
|
||
) {
|
||
smartAutoHistoryImages = requestTargetResolution.targetImageUrls.slice(0, 4);
|
||
}
|
||
|
||
const uiMessages = [...convertToUIMessages(messagesFromDb), message];
|
||
const hiddenPromptText = typeof hiddenPrompt === "string" ? hiddenPrompt.trim() : "";
|
||
const hiddenPromptPrefix = hiddenPromptText ? `${hiddenPromptText}\n\n` : "";
|
||
const uiMessagesForModel = hiddenPromptText
|
||
? applyHiddenPromptToMessages(uiMessages, hiddenPromptText)
|
||
: uiMessages;
|
||
|
||
if (hiddenPromptText) {
|
||
const hiddenPromptLines = hiddenPromptText
|
||
.split("\n")
|
||
.map((line) => line.trim())
|
||
.filter(Boolean);
|
||
chatApiLogger.info("[Chat API] Hidden workflow prompt summary:", {
|
||
lineCount: hiddenPromptLines.length,
|
||
preview: hiddenPromptLines.slice(0, 12),
|
||
});
|
||
}
|
||
|
||
const { longitude, latitude, city, country } = geolocation(request);
|
||
|
||
const requestHints: RequestHints = {
|
||
longitude,
|
||
latitude,
|
||
city,
|
||
country,
|
||
};
|
||
|
||
const insertedUserMessage = await db
|
||
.insert(messageTable)
|
||
.values({
|
||
chatId: id,
|
||
id: message.id,
|
||
role: "user",
|
||
parts: message.parts as any,
|
||
attachments: [],
|
||
createdAt: new Date(),
|
||
videoStorageStatus: null,
|
||
temporaryVideoUrl: null,
|
||
permanentVideoUrl: null,
|
||
archivedAt: null,
|
||
videoMetadata: null,
|
||
})
|
||
.onConflictDoNothing({
|
||
target: [messageTable.id],
|
||
})
|
||
.returning({ id: messageTable.id });
|
||
|
||
if (insertedUserMessage.length === 0) {
|
||
chatApiLogger.warn(`[Chat API] Duplicate user message ignored (chatId=${id}, messageId=${message.id})`);
|
||
const duplicateStream = createUIMessageStream({
|
||
execute: async ({ writer }) => {
|
||
await replayAssistantMessagesForDuplicateRequest({
|
||
chatId: id,
|
||
userMessageId: message.id,
|
||
writer,
|
||
source: "duplicate-user-message",
|
||
});
|
||
},
|
||
});
|
||
return createSseResponse(duplicateStream);
|
||
}
|
||
|
||
if (requestResolvedMediaInput && requestTargetResolution) {
|
||
try {
|
||
await createMessageTargetResolution({
|
||
userId: session.user.id,
|
||
chatId: id,
|
||
messageId: message.id,
|
||
projectId: contextPayload?.projectId || null,
|
||
rawText: requestTextContent,
|
||
uploadedAssetIds: requestResolvedMediaInput.uploadedAssetIds,
|
||
uploadedImageUrls: requestResolvedMediaInput.currentMessageImageUrls,
|
||
selectedAssetIds: requestResolvedMediaInput.selectedAssetIds,
|
||
selectedImageUrls: requestResolvedMediaInput.selectedImageUrls,
|
||
candidateAssetIds: requestResolvedMediaInput.candidateAssetIds,
|
||
candidateImageUrls: requestResolvedMediaInput.imageRefs.map((ref) => ref.url),
|
||
resolvedAssetIds: requestTargetResolution.targetAssetIds,
|
||
resolvedImageUrls: requestTargetResolution.targetImageUrls,
|
||
resolutionMode: requestTargetResolution.resolutionMode,
|
||
confidence: requestTargetResolution.confidence,
|
||
needsClarification: requestTargetResolution.needsClarification,
|
||
reasoning: {
|
||
reason: requestTargetResolution.reason,
|
||
fromHistory: requestTargetResolution.fromHistory,
|
||
focusPolicy: contextPayload?.focusPolicy || null,
|
||
contextSource: contextPayload?.source || null,
|
||
},
|
||
});
|
||
|
||
await persistResolvedTargetTurn({
|
||
userId: session.user.id,
|
||
chatId: id,
|
||
projectId: contextPayload?.projectId || null,
|
||
currentIntent:
|
||
smartAutoExecution?.routeIntent ||
|
||
smartAutoDetectedIntent ||
|
||
(requestResolvedMediaInput.hasImage
|
||
? "vision"
|
||
: requestBody.videoParams
|
||
? "video_gen"
|
||
: null),
|
||
currentFocusAssetIds: requestTargetResolution.targetAssetIds,
|
||
currentFocusImageUrls: requestTargetResolution.targetImageUrls,
|
||
selectedAssetIds: requestResolvedMediaInput.selectedAssetIds,
|
||
selectedImageUrls: requestResolvedMediaInput.selectedImageUrls,
|
||
lastUploadedAssetIds: requestResolvedMediaInput.uploadedAssetIds,
|
||
lastUploadedImageUrls: requestResolvedMediaInput.currentMessageImageUrls,
|
||
lastReferencedAssetIds: requestTargetResolution.candidateAssetIds,
|
||
lastReferencedImageUrls: requestTargetResolution.candidateImageUrls,
|
||
lastResolvedTargetAssetIds: requestTargetResolution.targetAssetIds,
|
||
lastResolvedTargetImageUrls: requestTargetResolution.targetImageUrls,
|
||
activeCanvasNodeIds: contextPayload?.selectedNodeIds || [],
|
||
lastCanvasSnapshot:
|
||
Array.isArray(contextPayload?.canvasImageNodes) && contextPayload?.canvasImageNodes.length > 0
|
||
? {
|
||
source: contextPayload?.source || "canvas",
|
||
selectedNodeIds: contextPayload?.selectedNodeIds || [],
|
||
canvasImageNodes: contextPayload?.canvasImageNodes,
|
||
}
|
||
: null,
|
||
lastMessageId: message.id,
|
||
text: requestTextContent,
|
||
resolutionMode: requestTargetResolution.resolutionMode,
|
||
confidence: requestTargetResolution.confidence,
|
||
});
|
||
} catch (memoryError) {
|
||
chatApiLogger.warn("[Chat API] Failed to persist context-memory state:", memoryError);
|
||
}
|
||
}
|
||
|
||
const buildCurrentAgentTargetLockPayload = () => {
|
||
if (!shouldRunAgentConversation) return null;
|
||
if (composerMode !== "agent") return null;
|
||
if (effectiveContextPayload?.source !== "canvas") return null;
|
||
if (!requestResolvedMediaInput || !requestTargetResolution) return null;
|
||
|
||
return buildAgentTargetLockPayload({
|
||
requestTargetResolution,
|
||
requestResolvedMediaInput,
|
||
needsClarification: smartAutoNeedsClarification,
|
||
clarificationMessage:
|
||
smartAutoClarificationQuestion || smartAutoFriendlyMessage || undefined,
|
||
});
|
||
};
|
||
|
||
const writeCurrentAgentTargetLockChunk = (
|
||
writer: StreamWriter,
|
||
label: string,
|
||
) => {
|
||
const payload = buildCurrentAgentTargetLockPayload();
|
||
if (!payload) return;
|
||
writeAgentTargetLockChunk(writer, payload, label);
|
||
};
|
||
|
||
const streamId = generateUUID();
|
||
await createStreamId({ streamId, chatId: id });
|
||
|
||
const promptOnlySourceImageUrls =
|
||
requestResolvedMediaInput.currentMessageImageUrls.length > 0
|
||
? requestResolvedMediaInput.currentMessageImageUrls.slice(0, 4)
|
||
: requestTargetResolution?.targetImageUrls.slice(0, 4) || smartAutoHistoryImages.slice(0, 4);
|
||
const promptOnlySourceAssetIds =
|
||
requestTargetResolution?.targetAssetIds?.length
|
||
? requestTargetResolution.targetAssetIds
|
||
: requestResolvedMediaInput.uploadedAssetIds;
|
||
const wantsPromptOnlyFromImageContext =
|
||
shouldRunAgentConversation &&
|
||
!smartAutoNeedsClarification &&
|
||
promptOnlySourceImageUrls.length > 0 &&
|
||
taskBoundaryDecision.taskType === "prompt_only";
|
||
|
||
if (wantsPromptOnlyFromImageContext) {
|
||
let promptOnlyImageSummary = "";
|
||
try {
|
||
const { summarizeImageForExecutionReport } = await import("@/lib/ai/smart-analyzer");
|
||
const summaryTimeoutMs = getBudgetedTimeout({
|
||
startedAt: requestStartedAt,
|
||
preferredMs: SMART_AUTO_IMAGE_SUMMARY_TIMEOUT_MS,
|
||
minimumMs: 6_000,
|
||
});
|
||
if (summaryTimeoutMs) {
|
||
const summaryAssetIds = Array.from(
|
||
new Set([
|
||
...extractMessageImageAssetIds(message.parts as any[]),
|
||
...((effectiveContextPayload?.selectedAssetIds || []).filter(Boolean) as string[]),
|
||
]),
|
||
);
|
||
const summary = await withOperationTimeout(
|
||
summarizeImageForExecutionReport({
|
||
text: requestTextContent,
|
||
imageUrls: promptOnlySourceImageUrls,
|
||
context: effectiveAutoContextText || undefined,
|
||
preferredModelId: preferredAgentModelId,
|
||
assetIds:
|
||
promptOnlySourceAssetIds.length > 0 ? promptOnlySourceAssetIds : summaryAssetIds,
|
||
timeoutMs: summaryTimeoutMs,
|
||
}),
|
||
"promptOnlyImageSummary",
|
||
summaryTimeoutMs,
|
||
);
|
||
if (summary?.trim()) {
|
||
promptOnlyImageSummary = summary.trim();
|
||
}
|
||
}
|
||
} catch (promptOnlySummaryError) {
|
||
chatApiLogger.warn("[Chat API] Prompt-only image summary failed:", promptOnlySummaryError);
|
||
}
|
||
|
||
let promptOnlyOptimizedPrompt = smartAutoOptimizedPrompt?.trim() || "";
|
||
if (!promptOnlyOptimizedPrompt) {
|
||
try {
|
||
const { optimizePromptWithLLM, optimizePrompt } = await import("@/lib/ai/prompt-optimizer");
|
||
const promptSeed = buildPromptOnlyOptimizationSeed({
|
||
text: requestTextContent,
|
||
imageSummary: promptOnlyImageSummary,
|
||
});
|
||
const optimizedResult = await optimizePromptWithLLM({
|
||
originalPrompt: promptSeed,
|
||
intent: "image_gen",
|
||
historyMessages: smartAutoHistoryMessagesFromDb
|
||
.map((msg) => {
|
||
const parts = Array.isArray(msg.parts) ? (msg.parts as any[]) : [];
|
||
const content = parts
|
||
.filter((part: any) => part?.type === "text")
|
||
.map((part: any) => String(part?.text || ""))
|
||
.join(" ")
|
||
.trim();
|
||
if (!content) return null;
|
||
return {
|
||
role: msg.role as "user" | "assistant",
|
||
content: content.slice(0, 300),
|
||
};
|
||
})
|
||
.filter(Boolean) as Array<{ role: "user" | "assistant"; content: string }>,
|
||
context: effectiveAutoContextText || undefined,
|
||
preferredModelId: preferredAgentModelId,
|
||
}).catch(() =>
|
||
optimizePrompt({
|
||
originalPrompt: promptSeed,
|
||
intent: "image_gen",
|
||
}),
|
||
);
|
||
promptOnlyOptimizedPrompt = String(optimizedResult?.optimizedPrompt || "").trim();
|
||
} catch (promptOnlyOptimizeError) {
|
||
chatApiLogger.warn("[Chat API] Prompt-only optimization failed:", promptOnlyOptimizeError);
|
||
}
|
||
}
|
||
|
||
if (!promptOnlyOptimizedPrompt) {
|
||
promptOnlyOptimizedPrompt = buildPromptOnlyOptimizationSeed({
|
||
text: requestTextContent,
|
||
imageSummary: promptOnlyImageSummary,
|
||
});
|
||
}
|
||
|
||
const promptOnlyMessage = buildPromptOnlyReply({
|
||
userText: requestTextContent,
|
||
imageSummary: promptOnlyImageSummary,
|
||
optimizedPrompt: promptOnlyOptimizedPrompt,
|
||
});
|
||
|
||
const promptOnlyStream = createUIMessageStream({
|
||
execute: async ({ writer }) => {
|
||
const messageId = generateUUID();
|
||
writeAgentProgressChunk(
|
||
writer,
|
||
{
|
||
stage: "planning",
|
||
currentStep: "analyze",
|
||
title: "正在提炼可执行提示词",
|
||
message: "我在结合当前目标图和上下文整理更稳定的提示词。",
|
||
toolName: "promptOnly",
|
||
capability: "prompt_only",
|
||
},
|
||
"prompt-only:progress-planning",
|
||
);
|
||
writeCurrentAgentTargetLockChunk(writer, "prompt-only:target-lock");
|
||
writeAssistantTextChunk(writer, promptOnlyMessage, "prompt-only:reply", messageId);
|
||
|
||
await saveMessages({
|
||
messages: [
|
||
{
|
||
id: messageId,
|
||
chatId: id,
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: promptOnlyMessage }],
|
||
attachments: [],
|
||
createdAt: new Date(),
|
||
videoStorageStatus: null,
|
||
temporaryVideoUrl: null,
|
||
permanentVideoUrl: null,
|
||
archivedAt: null,
|
||
videoMetadata: null,
|
||
},
|
||
],
|
||
});
|
||
|
||
try {
|
||
await persistPromptOnlyTurn({
|
||
userId: session.user.id,
|
||
chatId: id,
|
||
projectId: contextPayload?.projectId || null,
|
||
messageId,
|
||
text: requestTextContent,
|
||
optimizedPrompt: promptOnlyOptimizedPrompt,
|
||
imageSummary: promptOnlyImageSummary,
|
||
sourceImageUrls: promptOnlySourceImageUrls,
|
||
});
|
||
} catch (promptOnlyMemoryError) {
|
||
chatApiLogger.warn("[Chat API] Failed to persist prompt-only response state:", promptOnlyMemoryError);
|
||
}
|
||
|
||
writeAgentProgressChunk(
|
||
writer,
|
||
{
|
||
stage: "completed",
|
||
currentStep: "finalize",
|
||
title: "提示词整理完成",
|
||
message: "已经给出可直接复用的优化提示词。",
|
||
toolName: "promptOnly",
|
||
capability: "prompt_only",
|
||
},
|
||
"prompt-only:progress-completed",
|
||
);
|
||
},
|
||
});
|
||
|
||
return createSseResponse(promptOnlyStream);
|
||
}
|
||
|
||
if (shouldRunAgentConversation && smartAutoNeedsClarification) {
|
||
const clarificationBypass = resolveClarificationBypass({
|
||
text: message.parts
|
||
.filter((part): part is { type: "text"; text: string } => part.type === "text")
|
||
.map((part) => part.text)
|
||
.join(" ")
|
||
.trim(),
|
||
hasCurrentImageInput: message.parts.some((part: any) =>
|
||
part?.type === "file" || part?.type === "image" || part?.type === "image_url"
|
||
),
|
||
hasHistoryImageInput: smartAutoHistoryImages.length > 0,
|
||
hasCanvasSelection: Boolean(effectiveContextPayload?.selectedNodeIds?.length),
|
||
targetResolutionMode: requestTargetResolution?.resolutionMode,
|
||
executionMode: smartAutoExecution?.mode,
|
||
friendlyMessage: smartAutoFriendlyMessage,
|
||
});
|
||
|
||
if (clarificationBypass.shouldBypass) {
|
||
smartAutoNeedsClarification = false;
|
||
smartAutoClarificationQuestion = undefined;
|
||
smartAutoFriendlyMessage = clarificationBypass.nextFriendlyMessage;
|
||
chatApiLogger.info("[Smart Auto Mode] ✅ Force direct execution (skip clarification)");
|
||
}
|
||
}
|
||
|
||
if (shouldRunAgentConversation) {
|
||
const canvasClarificationGate = resolveCanvasAgentClarificationGate({
|
||
text: requestTextContent,
|
||
composerMode,
|
||
contextPayload: effectiveContextPayload,
|
||
mediaInput: requestResolvedMediaInput,
|
||
targetResolution: requestTargetResolution,
|
||
taskBoundaryDecision,
|
||
executionMode: smartAutoExecution?.mode,
|
||
actionKind: resolvedConversationAction.kind,
|
||
});
|
||
|
||
if (canvasClarificationGate.shouldClarify) {
|
||
smartAutoNeedsClarification = true;
|
||
smartAutoClarificationQuestion =
|
||
canvasClarificationGate.clarificationMessage || smartAutoClarificationQuestion;
|
||
smartAutoFriendlyMessage =
|
||
canvasClarificationGate.clarificationMessage || smartAutoFriendlyMessage;
|
||
chatApiLogger.info("[Smart Auto Mode] 🧭 Canvas clarification gate triggered:", {
|
||
reason: canvasClarificationGate.reason,
|
||
executionMode: smartAutoExecution?.mode,
|
||
resolutionMode: requestTargetResolution?.resolutionMode,
|
||
selectedCount: requestResolvedMediaInput?.selectedImageRefs.length || 0,
|
||
candidateCount: requestTargetResolution?.candidateRefs.length || 0,
|
||
});
|
||
}
|
||
}
|
||
|
||
if (shouldRunAgentConversation && smartAutoNeedsClarification) {
|
||
const clarificationMessage = resolveClarificationMessage({
|
||
clarificationQuestion: smartAutoClarificationQuestion,
|
||
friendlyMessage: smartAutoFriendlyMessage,
|
||
});
|
||
|
||
const clarificationStream = createUIMessageStream({
|
||
execute: async ({ writer }) => {
|
||
const messageId = generateUUID();
|
||
writeAgentProgressChunk(
|
||
writer,
|
||
{
|
||
stage: "clarify",
|
||
currentStep: "lock_target",
|
||
title: "等待你确认关键信息",
|
||
message:
|
||
smartAutoClarificationQuestion ||
|
||
smartAutoFriendlyMessage ||
|
||
"我需要先确认目标图或执行范围，避免误处理。",
|
||
toolName: "clarification",
|
||
capability: smartAutoExecution?.preferredCapability || undefined,
|
||
},
|
||
"clarification:progress",
|
||
);
|
||
writeCurrentAgentTargetLockChunk(writer, "clarification:target-lock");
|
||
writer.write({
|
||
type: "data-appendMessage",
|
||
data: JSON.stringify({
|
||
id: messageId,
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: clarificationMessage }],
|
||
}),
|
||
});
|
||
|
||
await saveMessages({
|
||
messages: [
|
||
{
|
||
id: messageId,
|
||
chatId: id,
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: clarificationMessage }],
|
||
attachments: [],
|
||
createdAt: new Date(),
|
||
videoStorageStatus: null,
|
||
temporaryVideoUrl: null,
|
||
permanentVideoUrl: null,
|
||
archivedAt: null,
|
||
videoMetadata: null,
|
||
},
|
||
],
|
||
});
|
||
|
||
try {
|
||
await persistClarificationTurn({
|
||
userId: session.user.id,
|
||
chatId: id,
|
||
projectId: contextPayload?.projectId || null,
|
||
messageId,
|
||
text: requestTextContent,
|
||
clarificationMessage,
|
||
currentIntent: smartAutoDetectedIntent || null,
|
||
pendingGridConfirmation:
|
||
shouldAskGridConfirmation && pendingGridConfirmation
|
||
? pendingGridConfirmation
|
||
: null,
|
||
});
|
||
} catch (clarificationPersistError) {
|
||
chatApiLogger.warn("[Chat API] Failed to persist clarification state:", clarificationPersistError);
|
||
}
|
||
},
|
||
});
|
||
|
||
return createSseResponse(clarificationStream);
|
||
}
|
||
|
||
if (
|
||
shouldRunAgentConversation &&
|
||
smartAutoWorkflowDecision?.workflowType === "storyboard_video" &&
|
||
(smartAutoWorkflowDecision?.action === "revision_requested" ||
|
||
smartAutoWorkflowDecision?.action === "cancel_requested")
|
||
) {
|
||
const workflowReplyStream = createUIMessageStream({
|
||
execute: async ({ writer }) => {
|
||
const messageId = generateUUID();
|
||
const workflowFollowupResult = await handleStoryboardWorkflowFollowup({
|
||
action: smartAutoWorkflowDecision.action as "revision_requested" | "cancel_requested",
|
||
workflowId: smartAutoWorkflowDecision.workflowId,
|
||
text: requestTextContent,
|
||
friendlyMessage: smartAutoFriendlyMessage,
|
||
onProgress: (event) => {
|
||
safeWriteStreamChunk(
|
||
writer,
|
||
{
|
||
type: "data-videoStoryboard",
|
||
data: JSON.stringify(event),
|
||
},
|
||
`storyboard-revision:${event.stage}`,
|
||
);
|
||
},
|
||
});
|
||
const replyText = workflowFollowupResult.replyText;
|
||
|
||
writer.write({
|
||
type: "data-appendMessage",
|
||
data: JSON.stringify({
|
||
id: messageId,
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: replyText }],
|
||
attachments: [],
|
||
}),
|
||
});
|
||
|
||
await saveMessages({
|
||
messages: [
|
||
{
|
||
id: messageId,
|
||
chatId: id,
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: replyText }],
|
||
attachments: [],
|
||
createdAt: new Date(),
|
||
videoStorageStatus: null,
|
||
temporaryVideoUrl: null,
|
||
permanentVideoUrl: null,
|
||
archivedAt: null,
|
||
videoMetadata: workflowFollowupResult.videoMetadata,
|
||
},
|
||
],
|
||
});
|
||
|
||
try {
|
||
if (
|
||
smartAutoWorkflowDecision?.action === "revision_requested" &&
|
||
smartAutoWorkflowDecision.workflowId
|
||
) {
|
||
await persistStoryboardRevisionTurn({
|
||
userId: session.user.id,
|
||
chatId: id,
|
||
projectId: contextPayload?.projectId || null,
|
||
messageId,
|
||
text: requestTextContent,
|
||
taskId: smartAutoWorkflowDecision.workflowId,
|
||
status: workflowFollowupResult.nextWorkflowStatus,
|
||
});
|
||
} else if (smartAutoWorkflowDecision?.action === "cancel_requested") {
|
||
await persistWorkflowCancelledTurn({
|
||
userId: session.user.id,
|
||
chatId: id,
|
||
projectId: contextPayload?.projectId || null,
|
||
messageId,
|
||
text: requestTextContent,
|
||
workflowType: "storyboard_video",
|
||
workflowId: smartAutoWorkflowDecision?.workflowId,
|
||
status: workflowFollowupResult.nextWorkflowStatus,
|
||
});
|
||
}
|
||
} catch (workflowPersistError) {
|
||
chatApiLogger.warn("[Chat API] Failed to persist storyboard workflow interruption state:", workflowPersistError);
|
||
}
|
||
},
|
||
});
|
||
|
||
return createSseResponse(workflowReplyStream);
|
||
}
|
||
|
||
if (
|
||
shouldRunAgentConversation &&
|
||
smartAutoWorkflowDecision?.workflowType === "storyboard_video" &&
|
||
smartAutoWorkflowDecision?.action === "confirmed" &&
|
||
smartAutoWorkflowDecision.workflowId
|
||
) {
|
||
const confirmedTaskId = smartAutoWorkflowDecision.workflowId;
|
||
const confirmedStoryboardStream = createUIMessageStream({
|
||
execute: async ({ writer }) => {
|
||
const acknowledgeMessageId = generateUUID();
|
||
const { acknowledgeMessage, acknowledgeMetadata } =
|
||
buildStoryboardConfirmedAck(confirmedTaskId);
|
||
|
||
writer.write({
|
||
type: "data-appendMessage",
|
||
data: JSON.stringify({
|
||
id: acknowledgeMessageId,
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: acknowledgeMessage }],
|
||
attachments: [],
|
||
}),
|
||
});
|
||
|
||
await saveMessages({
|
||
messages: [
|
||
{
|
||
id: acknowledgeMessageId,
|
||
chatId: id,
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: acknowledgeMessage }],
|
||
attachments: [],
|
||
createdAt: new Date(),
|
||
videoStorageStatus: null,
|
||
temporaryVideoUrl: null,
|
||
permanentVideoUrl: null,
|
||
archivedAt: null,
|
||
videoMetadata: acknowledgeMetadata,
|
||
},
|
||
],
|
||
});
|
||
|
||
try {
|
||
await persistStoryboardConfirmedTurn({
|
||
userId: session.user.id,
|
||
chatId: id,
|
||
projectId: contextPayload?.projectId || null,
|
||
messageId: acknowledgeMessageId,
|
||
text: requestTextContent,
|
||
taskId: confirmedTaskId,
|
||
status: "confirmed",
|
||
});
|
||
} catch (storyboardConfirmMemoryError) {
|
||
chatApiLogger.warn("[Chat API] Failed to persist storyboard confirmation memory:", storyboardConfirmMemoryError);
|
||
}
|
||
|
||
try {
|
||
await persistStoryboardExecutionStartedTurn({
|
||
userId: session.user.id,
|
||
chatId: id,
|
||
projectId: contextPayload?.projectId || null,
|
||
messageId: acknowledgeMessageId,
|
||
text: requestTextContent,
|
||
taskId: confirmedTaskId,
|
||
status: "running",
|
||
});
|
||
} catch (storyboardExecutionStartError) {
|
||
chatApiLogger.warn("[Chat API] Failed to persist storyboard execution start state:", storyboardExecutionStartError);
|
||
}
|
||
|
||
const { executionResult, firstVideoUrl, finalVideoMetadata } =
|
||
await executeStoryboardConfirmedWorkflow({
|
||
taskId: confirmedTaskId,
|
||
onProgress: (event) => {
|
||
safeWriteStreamChunk(
|
||
writer,
|
||
{
|
||
type: "data-videoStoryboard",
|
||
data: JSON.stringify(event),
|
||
},
|
||
`storyboard-confirmed:${event.stage}`,
|
||
);
|
||
},
|
||
});
|
||
|
||
const finalMessageId = generateUUID();
|
||
writer.write({
|
||
type: "data-appendMessage",
|
||
data: JSON.stringify({
|
||
id: finalMessageId,
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: executionResult.markdown }],
|
||
attachments: [],
|
||
}),
|
||
});
|
||
|
||
await saveMessages({
|
||
messages: [
|
||
{
|
||
id: finalMessageId,
|
||
chatId: id,
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: executionResult.markdown }],
|
||
attachments: [],
|
||
createdAt: new Date(),
|
||
videoStorageStatus: firstVideoUrl ? "temporary" : null,
|
||
temporaryVideoUrl: firstVideoUrl || null,
|
||
permanentVideoUrl: null,
|
||
archivedAt: null,
|
||
videoMetadata: finalVideoMetadata,
|
||
},
|
||
],
|
||
});
|
||
|
||
await attachVideoStoryboardTaskMessage({
|
||
taskId: executionResult.taskId,
|
||
messageId: finalMessageId,
|
||
}).catch((taskAttachError) => {
|
||
chatApiLogger.warn("[Chat API] Failed to attach confirmed storyboard task with message:", taskAttachError);
|
||
});
|
||
|
||
try {
|
||
await persistStoryboardExecutionCompletedTurn({
|
||
userId: session.user.id,
|
||
chatId: id,
|
||
projectId: contextPayload?.projectId || null,
|
||
messageId: finalMessageId,
|
||
text: requestTextContent,
|
||
taskId: executionResult.taskId,
|
||
status: executionResult.status,
|
||
successCount: executionResult.successCount,
|
||
failedCount: executionResult.failedCount,
|
||
firstVideoUrl: firstVideoUrl || undefined,
|
||
});
|
||
} catch (storyboardExecutionCompletedError) {
|
||
chatApiLogger.warn("[Chat API] Failed to persist storyboard execution completion state:", storyboardExecutionCompletedError);
|
||
}
|
||
|
||
if (firstVideoUrl) {
|
||
await consumeGeneratedVideoPoints({
|
||
userId: session.user.id,
|
||
chatId: id,
|
||
messageId: finalMessageId,
|
||
modelRefs: executionResult.modelRefs,
|
||
duration: Number((finalVideoMetadata as any)?.duration) || undefined,
|
||
resolution:
|
||
String((finalVideoMetadata as any)?.resolution || "").trim() ||
|
||
undefined,
|
||
});
|
||
|
||
scheduleVideoArchive({
|
||
chatId: id,
|
||
messageId: finalMessageId,
|
||
text: executionResult.markdown,
|
||
temporaryVideoUrl: firstVideoUrl,
|
||
taskId: finalMessageId,
|
||
metadata: {
|
||
storyboard: {
|
||
enabled: true,
|
||
taskId: executionResult.taskId,
|
||
successCount: executionResult.successCount,
|
||
failedCount: executionResult.failedCount,
|
||
},
|
||
status: executionResult.status,
|
||
updatedAt: Date.now(),
|
||
},
|
||
});
|
||
}
|
||
},
|
||
});
|
||
|
||
return createSseResponse(confirmedStoryboardStream);
|
||
}
|
||
|
||
if (!agentPresetId && agentModePayload?.modeType === "detail_replacement") {
|
||
const replacementReferences = normalizeAgentModeReferences(agentModePayload);
|
||
const detailReplacementStrategy =
|
||
normalizeAgentModeDetailReplacementStrategy(agentModePayload);
|
||
if (replacementReferences.length === 0) {
|
||
return NextResponse.json(
|
||
{
|
||
code: "bad_request:api",
|
||
message: "详情参考图替换至少需要 1 张详情参考图",
|
||
},
|
||
{ status: 400 },
|
||
);
|
||
}
|
||
|
||
const uploadedImageParts = extractCurrentMessageImageParts(message.parts as any[]);
|
||
const sourceImages = normalizeAgentModeSourceImages(agentModePayload);
|
||
const inferredImageParts = resolveImagePartsByReferenceExclusion({
|
||
uploadedImageParts,
|
||
referenceImageUrls: replacementReferences.map((item) => item.imageUrl),
|
||
});
|
||
const productSourceImages =
|
||
sourceImages.productImages.length > 0
|
||
? sourceImages.productImages
|
||
: inferredImageParts.productParts.map((part) => ({
|
||
url: part.url,
|
||
label: part.name || "产品图",
|
||
name: part.name || "产品图",
|
||
assetId: part.assetId || null,
|
||
}));
|
||
const modelSourceImage =
|
||
sourceImages.modelImage ||
|
||
(inferredImageParts.modelPart
|
||
? {
|
||
url: inferredImageParts.modelPart.url,
|
||
label: inferredImageParts.modelPart.name || "模特图",
|
||
name: inferredImageParts.modelPart.name || "模特图",
|
||
assetId: inferredImageParts.modelPart.assetId || null,
|
||
}
|
||
: null);
|
||
const replacementReferenceUrlSet = new Set(
|
||
replacementReferences.map((item) => item.imageUrl),
|
||
);
|
||
const productInputImageUrls = Array.from(
|
||
new Set(
|
||
productSourceImages
|
||
.map((item) => item.url)
|
||
.filter((url) => /^https?:\/\//i.test(url) && !replacementReferenceUrlSet.has(url)),
|
||
),
|
||
).slice(0, 16);
|
||
const modelInputImageUrl =
|
||
modelSourceImage?.url && !replacementReferenceUrlSet.has(modelSourceImage.url)
|
||
? modelSourceImage.url
|
||
: null;
|
||
|
||
const directReplacementModels = await getEnabledAiModelsWithProvider().catch((error) => {
|
||
chatApiLogger.warn("[Chat API] Failed to load models for direct detail replacement:", error);
|
||
return undefined;
|
||
});
|
||
const resolvedModelRef = await resolveDirectReplacementModelRef({
|
||
selectedChatModel,
|
||
finalModelId,
|
||
models: directReplacementModels,
|
||
});
|
||
if (!resolvedModelRef || resolvedModelRef === "auto") {
|
||
return NextResponse.json(
|
||
{
|
||
code: "bad_request:api",
|
||
message: "当前风格化未配置可用的生图模型",
|
||
},
|
||
{ status: 400 },
|
||
);
|
||
}
|
||
|
||
const sanitizedUserSupplementPrompt = normalizeAgentSupplementPrompt(
|
||
requestTextContent,
|
||
"详情参考图替换",
|
||
);
|
||
const baseReplacementPrompt =
|
||
detailReplacementStrategy === "product_only"
|
||
? "仅替换参考详情图中的产品主体"
|
||
: buildAgentSlotPrompt({
|
||
workflowPrompt: hiddenPromptText || null,
|
||
slotPrompt: "",
|
||
userSupplementPrompt: sanitizedUserSupplementPrompt || null,
|
||
includeSlotTitle: false,
|
||
modeType: "detail_replacement",
|
||
}) ||
|
||
hiddenPromptText ||
|
||
sanitizedUserSupplementPrompt ||
|
||
"执行详情参考图背景替换";
|
||
const resolvedAspectRatio = "auto";
|
||
const resolvedImageSize =
|
||
effectiveImageParams?.imageSize || defaultImageQualityParams.imageSize;
|
||
const sourceAssetIds = Array.from(
|
||
new Set(
|
||
[
|
||
...productSourceImages.map((item) => item.assetId),
|
||
modelSourceImage?.assetId,
|
||
...uploadedImageParts.map((part) => part.assetId),
|
||
].filter((assetId): assetId is string => Boolean(assetId)),
|
||
),
|
||
);
|
||
|
||
const directDetailReplacementStream = createUIMessageStream({
|
||
execute: async ({ writer }) => {
|
||
writeAssistantTextChunk(
|
||
writer,
|
||
"<!-- IMAGE_GENERATION_START -->",
|
||
"agent-mode:detail-replacement:start",
|
||
);
|
||
if (contextPayload?.source === "canvas") {
|
||
writeImageGenerationStatusChunk(
|
||
writer,
|
||
{
|
||
phase: "processing",
|
||
modelName: resolvedModelRef,
|
||
imageCount: replacementReferences.length,
|
||
},
|
||
"agent-mode:detail-replacement:status-processing",
|
||
);
|
||
}
|
||
|
||
let successCount = 0;
|
||
let resolvedModelDisplayName = resolvedModelRef;
|
||
const replacementVisionModels =
|
||
directReplacementModels ||
|
||
(await getEnabledAiModelsWithProvider().catch((error) => {
|
||
chatApiLogger.warn("[Chat API] Failed to load configured vision models for direct detail replacement:", error);
|
||
return undefined;
|
||
}));
|
||
const replacementScenePlanningRuntime =
|
||
await resolveReplacementScenePlanningRuntime({
|
||
models: replacementVisionModels,
|
||
});
|
||
const stopKeepalive = startStreamKeepalive(
|
||
writer,
|
||
"agent-mode:detail-replacement",
|
||
12_000,
|
||
);
|
||
|
||
const emitReplacementResultMessage = async (params: {
|
||
reference: (typeof replacementReferences)[number];
|
||
bodyText?: string | null;
|
||
image: {
|
||
originalUrl: string;
|
||
thumbnailUrl: string;
|
||
};
|
||
imageSizeLabel?: string | null;
|
||
modelDisplayName?: string | null;
|
||
sourceImageUrls: string[];
|
||
streamLabel: string;
|
||
promptForPersistence?: string | null;
|
||
}) => {
|
||
const persistedImageSize =
|
||
String(params.imageSizeLabel || "").trim() || resolvedImageSize;
|
||
const assistantText = prependImageModelBadge(
|
||
[
|
||
`<!-- AGENT_REPLACEMENT_CARD_ID:${String(params.reference.id || "").trim()} -->`,
|
||
`### ${params.reference.label}`,
|
||
String(params.bodyText || "").trim(),
|
||
buildAgentToolImagesMarkdown([
|
||
{
|
||
originalUrl: params.image.originalUrl,
|
||
thumbnailUrl: params.image.thumbnailUrl,
|
||
imageSize: persistedImageSize,
|
||
},
|
||
]),
|
||
]
|
||
.filter(Boolean)
|
||
.join("\n\n"),
|
||
params.modelDisplayName || null,
|
||
);
|
||
const assistantMessageId = generateUUID();
|
||
|
||
await saveMessages({
|
||
messages: [
|
||
{
|
||
id: assistantMessageId,
|
||
chatId: id,
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: assistantText }],
|
||
attachments: [],
|
||
createdAt: new Date(),
|
||
videoStorageStatus: null,
|
||
temporaryVideoUrl: null,
|
||
permanentVideoUrl: null,
|
||
archivedAt: null,
|
||
videoMetadata: null,
|
||
},
|
||
],
|
||
});
|
||
|
||
await saveGeneratedImages(
|
||
assistantText,
|
||
session.user.id,
|
||
id,
|
||
assistantMessageId,
|
||
params.promptForPersistence || baseReplacementPrompt,
|
||
resolvedModelRef,
|
||
{
|
||
sourceAssetIds,
|
||
sourceImageUrls: params.sourceImageUrls,
|
||
relationType: "edited_from",
|
||
sourceType: "detail_replacement",
|
||
projectId: contextPayload?.projectId || null,
|
||
userPrompt: requestTextContent,
|
||
optimizedPrompt: params.promptForPersistence || baseReplacementPrompt,
|
||
requestedImageCount: 1,
|
||
aspectRatio: resolvedAspectRatio,
|
||
imageSize: persistedImageSize,
|
||
},
|
||
);
|
||
|
||
writeAssistantTextChunk(
|
||
writer,
|
||
assistantText,
|
||
params.streamLabel,
|
||
assistantMessageId,
|
||
);
|
||
|
||
if (contextPayload?.source === "canvas") {
|
||
writeCanvasMediaChunk(
|
||
writer,
|
||
{
|
||
messageId: assistantMessageId,
|
||
imageUrls: [params.image.originalUrl],
|
||
prompt: params.promptForPersistence || baseReplacementPrompt,
|
||
sourceImageUrls: params.sourceImageUrls,
|
||
},
|
||
`${params.streamLabel}:canvas-media`,
|
||
);
|
||
}
|
||
};
|
||
|
||
try {
|
||
await runBoundedParallel(
|
||
Array.from(replacementReferences.entries()),
|
||
AGENT_REPLACEMENT_PARALLEL_WORKER_COUNT,
|
||
async ([referenceIndex, reference]) => {
|
||
if (reference.passthrough === true) {
|
||
const referenceSourceImageUrls = Array.from(
|
||
new Set([
|
||
...productInputImageUrls,
|
||
...(modelInputImageUrl ? [modelInputImageUrl] : []),
|
||
reference.imageUrl,
|
||
]),
|
||
);
|
||
await emitReplacementResultMessage({
|
||
reference,
|
||
bodyText: "已按人工标记沿用参考图，直接应用当前切片，不参与继续生成。",
|
||
image: {
|
||
originalUrl: reference.imageUrl,
|
||
thumbnailUrl: reference.imageUrl,
|
||
},
|
||
modelDisplayName: null,
|
||
sourceImageUrls: referenceSourceImageUrls,
|
||
streamLabel: "agent-mode:image:detail-replacement:item-passthrough",
|
||
});
|
||
successCount += 1;
|
||
return;
|
||
}
|
||
|
||
const replacementExecutionPlan = await analyzeReplacementExecutionPlan({
|
||
modeType: "detail_replacement",
|
||
replacementStrategy: detailReplacementStrategy,
|
||
referenceImageUrl: reference.imageUrl,
|
||
referenceLabel: reference.label,
|
||
productImageUrls: productInputImageUrls,
|
||
modelImageUrl: modelInputImageUrl,
|
||
workflowPrompt:
|
||
detailReplacementStrategy === "product_only"
|
||
? null
|
||
: baseReplacementPrompt || hiddenPromptText || null,
|
||
planningRuntime: replacementScenePlanningRuntime,
|
||
models: replacementVisionModels,
|
||
});
|
||
chatApiLogger.info("[Chat API] Direct detail replacement reference decision:", {
|
||
referenceIndex: referenceIndex + 1,
|
||
totalReferences: replacementReferences.length,
|
||
referenceLabel: reference.label,
|
||
shouldReplace: replacementExecutionPlan.shouldReplace,
|
||
reason: replacementExecutionPlan.reason,
|
||
detectionSource: replacementExecutionPlan.detectionSource,
|
||
scenePlanSource: replacementExecutionPlan.scenePlan.analysisSource,
|
||
pageKind: replacementExecutionPlan.scenePlan.pageKind,
|
||
generationRoute: replacementExecutionPlan.scenePlan.generationRoute,
|
||
promptPackPolicy: replacementExecutionPlan.scenePlan.promptPackPolicy,
|
||
packPolicy: replacementExecutionPlan.scenePlan.promptPackPolicy,
|
||
textPolicy: replacementExecutionPlan.scenePlan.textPolicy,
|
||
peoplePolicy: replacementExecutionPlan.scenePlan.peoplePolicy,
|
||
whiteLayout: replacementExecutionPlan.scenePlan.whiteBackgroundLikely,
|
||
copyCoupled: replacementExecutionPlan.scenePlan.copyImageCoupled,
|
||
useModelImage: replacementExecutionPlan.scenePlan.useModelImage,
|
||
sourceProductImages: productInputImageUrls.length,
|
||
sourceModelImage: Boolean(modelInputImageUrl),
|
||
});
|
||
const referenceSourceImageUrls = Array.from(
|
||
new Set([
|
||
...productInputImageUrls,
|
||
...(modelInputImageUrl ? [modelInputImageUrl] : []),
|
||
reference.imageUrl,
|
||
]),
|
||
);
|
||
|
||
if (!replacementExecutionPlan.shouldReplace) {
|
||
chatApiLogger.warn("[Chat API] Direct detail replacement plan returned shouldReplace=false; forcing generation:", {
|
||
referenceIndex: referenceIndex + 1,
|
||
totalReferences: replacementReferences.length,
|
||
referenceLabel: reference.label,
|
||
reason: replacementExecutionPlan.reason,
|
||
});
|
||
}
|
||
|
||
const referenceGeometry = await resolveReplacementReferenceGeometry(reference.imageUrl);
|
||
const generationAspectRatio = referenceGeometry?.aspectRatio || resolvedAspectRatio;
|
||
const selectedProductIndexes = replacementExecutionPlan.scenePlan
|
||
.selectedProductImageIndexes.map((index) => index - 1)
|
||
.filter((index) => index >= 0 && index < productInputImageUrls.length);
|
||
const selectedProductContextImageUrls =
|
||
selectedProductIndexes.length > 0
|
||
? selectedProductIndexes.map((index) => productInputImageUrls[index]!)
|
||
: productInputImageUrls.slice(0, Math.min(productInputImageUrls.length, 2));
|
||
const shouldIncludeModelImage =
|
||
detailReplacementStrategy !== "product_only" &&
|
||
Boolean(modelInputImageUrl) &&
|
||
(replacementExecutionPlan.scenePlan.useModelImage ||
|
||
selectedProductContextImageUrls.length === 0);
|
||
const sourceImageUrls = Array.from(
|
||
new Set([
|
||
...selectedProductContextImageUrls,
|
||
...(shouldIncludeModelImage && modelInputImageUrl
|
||
? [modelInputImageUrl]
|
||
: []),
|
||
reference.imageUrl,
|
||
]),
|
||
);
|
||
chatApiLogger.info("[Chat API] Direct detail replacement image generation started:", {
|
||
referenceIndex: referenceIndex + 1,
|
||
totalReferences: replacementReferences.length,
|
||
referenceLabel: reference.label,
|
||
pageKind: replacementExecutionPlan.scenePlan.pageKind,
|
||
generationRoute: replacementExecutionPlan.scenePlan.generationRoute,
|
||
packPolicy: replacementExecutionPlan.scenePlan.promptPackPolicy,
|
||
selectedProductImages: selectedProductContextImageUrls.length,
|
||
sourceProductImages: productInputImageUrls.length,
|
||
includeModelImage: shouldIncludeModelImage,
|
||
sourceImageCount: sourceImageUrls.length,
|
||
whiteLayout: replacementExecutionPlan.scenePlan.whiteBackgroundLikely,
|
||
copyCoupled: replacementExecutionPlan.scenePlan.copyImageCoupled,
|
||
});
|
||
const replacementPrompt = buildAgentReplacementPrompt({
|
||
basePrompt: baseReplacementPrompt,
|
||
modeType: "detail_replacement",
|
||
replacementStrategy: detailReplacementStrategy,
|
||
marketplaceProductDetailReference:
|
||
isMarketplaceProductDetailReferenceId(reference.id) ||
|
||
isMarketplaceProductDetailReferenceId(reference.candidateId),
|
||
referenceLabel: reference.label,
|
||
referenceIndex,
|
||
totalReferences: replacementReferences.length,
|
||
hasModelImage: shouldIncludeModelImage,
|
||
productImageCount: selectedProductContextImageUrls.length,
|
||
originalProductImageCount: productInputImageUrls.length,
|
||
templateReferenceCount: 0,
|
||
scenePlan: replacementExecutionPlan.scenePlan,
|
||
referenceGeometry,
|
||
});
|
||
chatApiLogger.info("[Chat API] Direct detail replacement prompt assembled:", {
|
||
referenceIndex: referenceIndex + 1,
|
||
totalReferences: replacementReferences.length,
|
||
referenceLabel: reference.label,
|
||
pageKind: replacementExecutionPlan.scenePlan.pageKind,
|
||
generationRoute: replacementExecutionPlan.scenePlan.generationRoute,
|
||
promptPackPolicy: replacementExecutionPlan.scenePlan.promptPackPolicy,
|
||
basePromptChars: baseReplacementPrompt.length,
|
||
promptChars: replacementPrompt.length,
|
||
hasPromptPackBlock: replacementPrompt.includes("【必须生效的前置场景词包】"),
|
||
});
|
||
|
||
try {
|
||
const result = await generateCanvasImage({
|
||
prompt: replacementPrompt,
|
||
negativePrompt: effectiveImageParams?.negativePrompt,
|
||
model: resolvedModelRef,
|
||
ownerKey: session.user.id,
|
||
aspectRatio: generationAspectRatio,
|
||
imageSize: resolvedImageSize,
|
||
imageCount: 1,
|
||
referenceImages: sourceImageUrls,
|
||
debugLabel: "agent-detail-replacement",
|
||
debugMeta: {
|
||
referenceLabel: reference.label,
|
||
referenceWidth: referenceGeometry?.width,
|
||
referenceHeight: referenceGeometry?.height,
|
||
referenceAspectRatio: referenceGeometry?.aspectRatio,
|
||
},
|
||
onQueueStatusChange:
|
||
contextPayload?.source === "canvas"
|
||
? async (queueStatus) => {
|
||
writeImageGenerationStatusChunk(
|
||
writer,
|
||
{
|
||
phase:
|
||
queueStatus.status === "running"
|
||
? "processing"
|
||
: "queued",
|
||
modelName:
|
||
queueStatus.modelName || resolvedModelDisplayName || resolvedModelRef,
|
||
imageCount: 1,
|
||
queuePosition: queueStatus.queuePosition,
|
||
sameModelRunningCount: queueStatus.sameModelRunningCount,
|
||
concurrencyLimit: queueStatus.concurrencyLimit,
|
||
estimatedWaitMs: queueStatus.estimatedWaitMs,
|
||
},
|
||
`agent-mode:detail-replacement:queue:${reference.id}`,
|
||
);
|
||
}
|
||
: undefined,
|
||
});
|
||
resolvedModelDisplayName = result.model || resolvedModelDisplayName;
|
||
const firstImage = result.images[0];
|
||
if (!firstImage) {
|
||
throw new Error("未返回结果图片");
|
||
}
|
||
|
||
let finalImage = firstImage;
|
||
let finalImageSizeLabel = resolvedImageSize;
|
||
try {
|
||
const normalizedImage = await normalizeReplacementResultImage({
|
||
generatedImageUrl: firstImage.originalUrl,
|
||
generatedThumbnailUrl: firstImage.thumbnailUrl,
|
||
referenceImageUrl: reference.imageUrl,
|
||
referenceGeometry,
|
||
scenePlan: replacementExecutionPlan.scenePlan,
|
||
uploadPrefix: "agent-detail-replacement",
|
||
});
|
||
finalImage = {
|
||
originalUrl: normalizedImage.originalUrl,
|
||
thumbnailUrl: normalizedImage.thumbnailUrl || normalizedImage.originalUrl,
|
||
};
|
||
finalImageSizeLabel = normalizedImage.imageSize;
|
||
} catch (normalizeError) {
|
||
chatApiLogger.warn("[Chat API] Failed to normalize direct detail replacement output:", {
|
||
referenceLabel: reference.label,
|
||
referenceImageUrl: reference.imageUrl,
|
||
resultImageUrl: firstImage.originalUrl,
|
||
error:
|
||
normalizeError instanceof Error
|
||
? normalizeError.message
|
||
: "unknown",
|
||
});
|
||
const rawResultAccessible = await isRemoteImageAccessible(
|
||
firstImage.originalUrl
|
||
).catch(() => false);
|
||
if (!rawResultAccessible) {
|
||
throw normalizeError;
|
||
}
|
||
}
|
||
|
||
await emitReplacementResultMessage({
|
||
reference,
|
||
image: finalImage,
|
||
imageSizeLabel: finalImageSizeLabel,
|
||
modelDisplayName: resolvedModelDisplayName,
|
||
sourceImageUrls,
|
||
streamLabel: "agent-mode:image:detail-replacement:item",
|
||
promptForPersistence: replacementPrompt,
|
||
});
|
||
successCount += 1;
|
||
} catch (replacementItemError) {
|
||
chatApiLogger.error("[Chat API] Direct detail replacement item failed:", {
|
||
referenceIndex,
|
||
referenceLabel: reference.label,
|
||
error: replacementItemError,
|
||
});
|
||
await emitReplacementResultMessage({
|
||
reference,
|
||
bodyText: `生成失败，当前先沿用参考图：${
|
||
replacementItemError instanceof Error
|
||
? replacementItemError.message
|
||
: "请稍后重试"
|
||
}`,
|
||
image: {
|
||
originalUrl: reference.imageUrl,
|
||
thumbnailUrl: reference.imageUrl,
|
||
},
|
||
modelDisplayName: null,
|
||
sourceImageUrls: referenceSourceImageUrls,
|
||
streamLabel: "agent-mode:image:detail-replacement:item-fallback",
|
||
});
|
||
successCount += 1;
|
||
}
|
||
});
|
||
} finally {
|
||
stopKeepalive();
|
||
}
|
||
|
||
if (contextPayload?.source === "canvas") {
|
||
writeImageGenerationStatusChunk(
|
||
writer,
|
||
{
|
||
phase: successCount > 0 ? "completed" : "failed",
|
||
modelName: resolvedModelDisplayName,
|
||
imageCount: successCount,
|
||
},
|
||
"agent-mode:detail-replacement:status-finished",
|
||
);
|
||
}
|
||
writeAssistantTextChunk(
|
||
writer,
|
||
"<!-- IMAGE_GENERATION_END -->",
|
||
"agent-mode:detail-replacement:end",
|
||
);
|
||
},
|
||
});
|
||
|
||
return createSseResponse(directDetailReplacementStream);
|
||
}
|
||
|
||
if (agentPresetId) {
|
||
const selectedAgentPreset = await getAccessibleAgentPresetById({
|
||
id: agentPresetId,
|
||
userId: session.user.id,
|
||
});
|
||
|
||
if (!selectedAgentPreset) {
|
||
return NextResponse.json(
|
||
{
|
||
code: "not_found:chat",
|
||
message: "风格化不存在或无权使用",
|
||
},
|
||
{ status: 404 },
|
||
);
|
||
}
|
||
|
||
if (selectedAgentPreset.type === "text") {
|
||
// 生文风格化走常规多模态对话链路：
|
||
// - 模型已由前端切到风格化预设模型
|
||
// - 提示词通过 hiddenPrompt 注入
|
||
// - 图片输入沿用普通多模态附件
|
||
} else {
|
||
|
||
const orderedSlots = [...(selectedAgentPreset.slots || [])].sort(
|
||
(left, right) => left.slotIndex - right.slotIndex,
|
||
);
|
||
const presetModeConfig = normalizeAgentPresetModeConfig(selectedAgentPreset.defaultParams);
|
||
const isSceneGenerationPreset =
|
||
selectedAgentPreset.type === "image" &&
|
||
presetModeConfig.modeType === "scene_generation";
|
||
const isDetailReplacementPreset =
|
||
selectedAgentPreset.type === "image" &&
|
||
presetModeConfig.modeType === "detail_replacement";
|
||
const isSkuReplacementPreset =
|
||
selectedAgentPreset.type === "image" &&
|
||
presetModeConfig.modeType === "sku_replacement";
|
||
const sceneProductSlotCount = isSceneGenerationPreset
|
||
? Math.max(1, presetModeConfig.scene.productImageSlotCount || 1)
|
||
: 0;
|
||
const sceneModelSlotEnabled = isSceneGenerationPreset
|
||
? presetModeConfig.scene.includeModelSlot
|
||
: false;
|
||
const replacementProductSlotCount = isDetailReplacementPreset
|
||
? presetModeConfig.detail.productImageSlotCount
|
||
: isSkuReplacementPreset
|
||
? presetModeConfig.sku.productImageSlotCount
|
||
: 0;
|
||
const replacementModelSlotEnabled = isDetailReplacementPreset
|
||
? presetModeConfig.detail.includeModelSlot
|
||
: isSkuReplacementPreset
|
||
? presetModeConfig.sku.includeModelSlot
|
||
: false;
|
||
const replacementReferences = normalizeAgentModeReferences(agentModePayload);
|
||
const detailReplacementStrategy =
|
||
normalizeAgentModeDetailReplacementStrategy(agentModePayload);
|
||
const isMatchingReplacementPayload =
|
||
(isDetailReplacementPreset && agentModePayload?.modeType === "detail_replacement") ||
|
||
(isSkuReplacementPreset && agentModePayload?.modeType === "sku_replacement");
|
||
if (orderedSlots.length === 0) {
|
||
return NextResponse.json(
|
||
{
|
||
code: "bad_request:api",
|
||
message: "当前风格化未配置可执行槽位",
|
||
},
|
||
{ status: 400 },
|
||
);
|
||
}
|
||
if ((isDetailReplacementPreset || isSkuReplacementPreset) && !isMatchingReplacementPayload) {
|
||
return NextResponse.json(
|
||
{
|
||
code: "bad_request:api",
|
||
message: "当前模式缺少执行上下文，请返回第二步重新选择参考图后再试",
|
||
},
|
||
{ status: 400 },
|
||
);
|
||
}
|
||
if ((isDetailReplacementPreset || isSkuReplacementPreset) && replacementReferences.length === 0) {
|
||
return NextResponse.json(
|
||
{
|
||
code: "bad_request:api",
|
||
message: isDetailReplacementPreset
|
||
? "详情替换模式至少需要 1 张详情参考图"
|
||
: "SKU替换模式至少需要 1 张 SKU 参考图",
|
||
},
|
||
{ status: 400 },
|
||
);
|
||
}
|
||
|
||
const uploadedImageParts = extractCurrentMessageImageParts(message.parts as any[]);
|
||
const requiredUploadCount = getAgentRequiredUploadCount({
|
||
presetType: selectedAgentPreset.type,
|
||
defaultParams: selectedAgentPreset.defaultParams || null,
|
||
fallback: 3,
|
||
});
|
||
|
||
if (uploadedImageParts.length < requiredUploadCount) {
|
||
return NextResponse.json(
|
||
{
|
||
code: "bad_request:api",
|
||
message: `当前风格化需要上传 ${requiredUploadCount} 张图片`,
|
||
},
|
||
{ status: 400 },
|
||
);
|
||
}
|
||
|
||
const presetModelId = String(selectedAgentPreset.defaultModelId || "").trim();
|
||
if (!presetModelId) {
|
||
return NextResponse.json(
|
||
{
|
||
code: "bad_request:api",
|
||
message: "当前风格化未配置默认模型",
|
||
},
|
||
{ status: 400 },
|
||
);
|
||
}
|
||
|
||
const sanitizedUserSupplementPrompt = normalizeAgentSupplementPrompt(
|
||
requestTextContent,
|
||
selectedAgentPreset.name,
|
||
);
|
||
|
||
const presetSnapshot = await createAgentPresetExecutionSnapshot({
|
||
presetId: selectedAgentPreset.id,
|
||
userId: session.user.id,
|
||
projectId: contextPayload?.projectId || null,
|
||
chatId: id,
|
||
messageId: message.id,
|
||
presetName: selectedAgentPreset.name,
|
||
presetType: selectedAgentPreset.type,
|
||
visibility: selectedAgentPreset.visibility,
|
||
categoryId: selectedAgentPreset.category?.id || selectedAgentPreset.categoryId || null,
|
||
categoryName: selectedAgentPreset.category?.name || null,
|
||
promptTemplate: selectedAgentPreset.promptTemplate,
|
||
defaultModelId: selectedAgentPreset.defaultModelId || null,
|
||
resolvedModelId: presetModelId,
|
||
defaultParams: selectedAgentPreset.defaultParams || null,
|
||
slotCount: orderedSlots.length,
|
||
requiredUploadCount,
|
||
slots: orderedSlots.map((slot) => ({
|
||
slotIndex: slot.slotIndex,
|
||
title: slot.title || null,
|
||
promptTemplate: slot.promptTemplate,
|
||
defaultParams: slot.defaultParams || null,
|
||
requireUpload: slot.requireUpload !== false,
|
||
uploadLabel: slot.uploadLabel || null,
|
||
referenceAssetIds: slot.referenceAssetIds || [],
|
||
referenceImageUrls: (slot.referenceAssets || [])
|
||
.map((asset) => String(asset.originalUrl || "").trim())
|
||
.filter(Boolean),
|
||
})),
|
||
referenceAssetIds: orderedSlots.flatMap((slot) => slot.referenceAssetIds || []),
|
||
referenceImageUrls: orderedSlots.flatMap((slot) =>
|
||
(slot.referenceAssets || [])
|
||
.map((asset) => String(asset.originalUrl || "").trim())
|
||
.filter(Boolean),
|
||
),
|
||
userInputText: sanitizedUserSupplementPrompt || null,
|
||
requestPayload: {
|
||
selectedChatModel,
|
||
agentPresetId,
|
||
requiredUploadCount,
|
||
uploadedImageUrls: uploadedImageParts.map((part) => part.url),
|
||
agentModePayload:
|
||
isDetailReplacementPreset || isSkuReplacementPreset
|
||
? {
|
||
modeType: agentModePayload?.modeType || null,
|
||
replacementStrategy: agentModePayload?.replacementStrategy || null,
|
||
query: agentModePayload?.query || null,
|
||
queryType: agentModePayload?.queryType || null,
|
||
selectedCandidateId: agentModePayload?.selectedCandidateId || null,
|
||
referenceCount: replacementReferences.length,
|
||
}
|
||
: null,
|
||
},
|
||
});
|
||
|
||
try {
|
||
await db
|
||
.update(agentPresetTable)
|
||
.set({
|
||
usageCount: sql`${agentPresetTable.usageCount} + 1`,
|
||
lastUsedAt: new Date(),
|
||
updatedAt: new Date(),
|
||
})
|
||
.where(eq(agentPresetTable.id, selectedAgentPreset.id));
|
||
} catch (agentUsageError) {
|
||
chatApiLogger.warn("[Chat API] Failed to update agent preset usage stats:", agentUsageError);
|
||
}
|
||
|
||
const agentSequenceStream = createUIMessageStream({
|
||
execute: async ({ writer }) => {
|
||
if (selectedAgentPreset.type === "image") {
|
||
writeAssistantTextChunk(
|
||
writer,
|
||
"<!-- IMAGE_GENERATION_START -->",
|
||
"agent-preset:image:start",
|
||
);
|
||
if (contextPayload?.source === "canvas") {
|
||
writeImageGenerationStatusChunk(
|
||
writer,
|
||
{
|
||
phase: "processing",
|
||
modelName:
|
||
selectedAgentPreset.defaultModel?.displayName ||
|
||
selectedAgentPreset.name,
|
||
imageCount: isSceneGenerationPreset
|
||
? Math.max(1, effectiveImageParams?.imageCount || 1)
|
||
: isDetailReplacementPreset || isSkuReplacementPreset
|
||
? replacementReferences.length
|
||
: orderedSlots.length,
|
||
},
|
||
"agent-preset:image:status-processing",
|
||
);
|
||
}
|
||
}
|
||
|
||
let successCount = 0;
|
||
const sharedUploadedImageUrls = uploadedImageParts
|
||
.map((part) => part.url)
|
||
.filter((url) => /^https?:\/\//i.test(url));
|
||
const sharedUploadedAssetIds = uploadedImageParts
|
||
.map((part) => part.assetId)
|
||
.filter((assetId): assetId is string => Boolean(assetId));
|
||
|
||
if (
|
||
selectedAgentPreset.type === "image" &&
|
||
(isDetailReplacementPreset || isSkuReplacementPreset)
|
||
) {
|
||
const replacementSlot = orderedSlots[0];
|
||
const replacementTemplateReferenceImageUrls = Array.from(
|
||
new Set(
|
||
(replacementSlot?.referenceAssets || [])
|
||
.map((asset) => String(asset.originalUrl || "").trim())
|
||
.filter(Boolean),
|
||
),
|
||
);
|
||
const modelInputImageUrl = replacementModelSlotEnabled
|
||
? sharedUploadedImageUrls[Math.max(0, replacementProductSlotCount)] || null
|
||
: null;
|
||
const productInputImageUrls = isDetailReplacementPreset
|
||
? sharedUploadedImageUrls.filter((url) => url !== modelInputImageUrl)
|
||
: sharedUploadedImageUrls.slice(
|
||
0,
|
||
Math.max(1, replacementProductSlotCount)
|
||
);
|
||
const sourceAssetIds = Array.from(
|
||
new Set(
|
||
[
|
||
...sharedUploadedAssetIds,
|
||
...(replacementSlot?.referenceAssetIds || []),
|
||
].filter(Boolean),
|
||
),
|
||
);
|
||
const baseReplacementPrompt = buildAgentSlotPrompt({
|
||
workflowPrompt:
|
||
isDetailReplacementPreset &&
|
||
detailReplacementStrategy === "product_only"
|
||
? null
|
||
: hiddenPromptText || null,
|
||
slotPrompt:
|
||
isDetailReplacementPreset &&
|
||
detailReplacementStrategy === "product_only"
|
||
? ""
|
||
: replacementSlot?.promptTemplate || selectedAgentPreset.promptTemplate,
|
||
slotTitle: selectedAgentPreset.name,
|
||
userSupplementPrompt:
|
||
isDetailReplacementPreset &&
|
||
detailReplacementStrategy === "product_only"
|
||
? null
|
||
: sanitizedUserSupplementPrompt || null,
|
||
includeSlotTitle: false,
|
||
modeType: isDetailReplacementPreset
|
||
? "detail_replacement"
|
||
: isSkuReplacementPreset
|
||
? "sku_replacement"
|
||
: "generic",
|
||
});
|
||
|
||
if (!isDetailReplacementPreset && productInputImageUrls.length === 0) {
|
||
throw new Error("替换模式至少需要 1 张输入商品图");
|
||
}
|
||
|
||
const mergedImageParams = {
|
||
...(selectedAgentPreset.defaultParams || {}),
|
||
...(replacementSlot?.defaultParams || {}),
|
||
} as Record<string, any>;
|
||
const resolvedModelRef =
|
||
String(selectedChatModel || "").trim() &&
|
||
String(selectedChatModel || "").trim() !== "auto"
|
||
? String(selectedChatModel || "").trim()
|
||
: presetModelId;
|
||
const configuredReplacementAspectRatio =
|
||
typeof mergedImageParams.aspectRatio === "string"
|
||
? mergedImageParams.aspectRatio
|
||
: effectiveImageParams?.aspectRatio || defaultImageQualityParams.aspectRatio;
|
||
const resolvedAspectRatio = "auto";
|
||
const resolvedImageSize =
|
||
typeof mergedImageParams.imageSize === "string"
|
||
? mergedImageParams.imageSize
|
||
: effectiveImageParams?.imageSize || defaultImageQualityParams.imageSize;
|
||
let resolvedModelDisplayName =
|
||
selectedAgentPreset.defaultModel?.displayName || selectedAgentPreset.name;
|
||
if (
|
||
configuredReplacementAspectRatio &&
|
||
configuredReplacementAspectRatio !== "auto"
|
||
) {
|
||
chatApiLogger.info("[Chat API] Replacement mode forced aspect ratio to auto to preserve reference canvas:", {
|
||
configuredAspectRatio: configuredReplacementAspectRatio,
|
||
});
|
||
}
|
||
const replacementVisionModels = await getEnabledAiModelsWithProvider().catch(
|
||
(error) => {
|
||
chatApiLogger.warn("[Chat API] Failed to load configured vision models for replacement analysis:", error);
|
||
return undefined;
|
||
},
|
||
);
|
||
const replacementScenePlanningRuntime =
|
||
await resolveReplacementScenePlanningRuntime({
|
||
models: replacementVisionModels,
|
||
});
|
||
const replacementScenePlanningTimeoutMs =
|
||
replacementScenePlanningRuntime?.kind === "local_moonshot"
|
||
? 90_000
|
||
: 45_000;
|
||
chatApiLogger.info("[Chat API] Replacement scene planning runtime:", {
|
||
kind: replacementScenePlanningRuntime?.kind || "none",
|
||
modelName: replacementScenePlanningRuntime?.modelName || null,
|
||
timeoutMs: replacementScenePlanningTimeoutMs,
|
||
});
|
||
const stopKeepalive = startStreamKeepalive(
|
||
writer,
|
||
"agent-preset:replacement",
|
||
12_000,
|
||
);
|
||
const emitReplacementResultMessage = async (params: {
|
||
reference: (typeof replacementReferences)[number];
|
||
bodyText?: string | null;
|
||
image: {
|
||
originalUrl: string;
|
||
thumbnailUrl?: string | null;
|
||
};
|
||
imageSizeLabel?: string | null;
|
||
modelDisplayName?: string | null;
|
||
sourceImageUrls: string[];
|
||
sourceType: "detail_replacement" | "sku_replacement";
|
||
streamLabel: string;
|
||
}) => {
|
||
const persistedImageSize =
|
||
String(params.imageSizeLabel || "").trim() || resolvedImageSize;
|
||
const assistantText = prependImageModelBadge(
|
||
[
|
||
`<!-- AGENT_REPLACEMENT_CARD_ID:${String(params.reference.id || "").trim()} -->`,
|
||
`### ${params.reference.label}`,
|
||
String(params.bodyText || "").trim(),
|
||
buildAgentToolImagesMarkdown([
|
||
{
|
||
originalUrl: params.image.originalUrl,
|
||
thumbnailUrl: params.image.thumbnailUrl,
|
||
imageSize: persistedImageSize,
|
||
},
|
||
]),
|
||
]
|
||
.filter(Boolean)
|
||
.join("\n\n"),
|
||
params.modelDisplayName || null,
|
||
);
|
||
const assistantMessageId = generateUUID();
|
||
|
||
await saveMessages({
|
||
messages: [
|
||
{
|
||
id: assistantMessageId,
|
||
chatId: id,
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: assistantText }],
|
||
attachments: [],
|
||
createdAt: new Date(),
|
||
videoStorageStatus: null,
|
||
temporaryVideoUrl: null,
|
||
permanentVideoUrl: null,
|
||
archivedAt: null,
|
||
videoMetadata: null,
|
||
},
|
||
],
|
||
});
|
||
|
||
await saveGeneratedImages(
|
||
assistantText,
|
||
session.user.id,
|
||
id,
|
||
assistantMessageId,
|
||
baseReplacementPrompt || selectedAgentPreset.name,
|
||
resolvedModelRef,
|
||
{
|
||
sourceAssetIds,
|
||
sourceImageUrls: params.sourceImageUrls,
|
||
relationType: "edited_from",
|
||
sourceType: params.sourceType,
|
||
projectId: contextPayload?.projectId || null,
|
||
userPrompt: requestTextContent,
|
||
optimizedPrompt: baseReplacementPrompt || undefined,
|
||
requestedImageCount: 1,
|
||
aspectRatio: resolvedAspectRatio,
|
||
imageSize: persistedImageSize,
|
||
},
|
||
);
|
||
|
||
writeAssistantTextChunk(
|
||
writer,
|
||
assistantText,
|
||
params.streamLabel,
|
||
assistantMessageId,
|
||
);
|
||
|
||
if (contextPayload?.source === "canvas") {
|
||
writeCanvasMediaChunk(
|
||
writer,
|
||
{
|
||
messageId: assistantMessageId,
|
||
imageUrls: [params.image.originalUrl],
|
||
prompt: baseReplacementPrompt || selectedAgentPreset.name,
|
||
sourceImageUrls: params.sourceImageUrls,
|
||
agentPresetId: selectedAgentPreset.id,
|
||
agentPresetName: selectedAgentPreset.name,
|
||
agentCategoryId:
|
||
selectedAgentPreset.category?.id ||
|
||
selectedAgentPreset.categoryId ||
|
||
null,
|
||
agentCategoryName: selectedAgentPreset.category?.name || null,
|
||
agentExecutionSnapshotId: presetSnapshot.id,
|
||
},
|
||
`${params.streamLabel}:canvas-media`,
|
||
);
|
||
}
|
||
};
|
||
|
||
try {
|
||
await runBoundedParallel(
|
||
Array.from(replacementReferences.entries()),
|
||
AGENT_REPLACEMENT_PARALLEL_WORKER_COUNT,
|
||
async ([referenceIndex, reference]) => {
|
||
if (reference.passthrough === true) {
|
||
const replacementSourceType = isDetailReplacementPreset
|
||
? "detail_replacement"
|
||
: "sku_replacement";
|
||
const referenceSourceImageUrls = Array.from(
|
||
new Set([
|
||
...sharedUploadedImageUrls,
|
||
...replacementTemplateReferenceImageUrls,
|
||
reference.imageUrl,
|
||
]),
|
||
);
|
||
await emitReplacementResultMessage({
|
||
reference,
|
||
bodyText: "已按人工标记沿用参考图，直接应用当前切片，不参与继续生成。",
|
||
image: {
|
||
originalUrl: reference.imageUrl,
|
||
thumbnailUrl: reference.imageUrl,
|
||
},
|
||
modelDisplayName: null,
|
||
sourceImageUrls: referenceSourceImageUrls,
|
||
sourceType: replacementSourceType,
|
||
streamLabel: isDetailReplacementPreset
|
||
? "agent-preset:image:detail-replacement:item-passthrough"
|
||
: "agent-preset:image:sku-replacement:item-passthrough",
|
||
});
|
||
successCount += 1;
|
||
return;
|
||
}
|
||
|
||
const pairedProductIndex =
|
||
isSkuReplacementPreset && productInputImageUrls.length > 0
|
||
? Math.min(
|
||
typeof reference.assignedProductIndex === "number"
|
||
? reference.assignedProductIndex
|
||
: referenceIndex,
|
||
productInputImageUrls.length - 1
|
||
)
|
||
: undefined;
|
||
const orderedProductContextImageUrls =
|
||
pairedProductIndex !== undefined
|
||
? [
|
||
productInputImageUrls[pairedProductIndex]!,
|
||
...productInputImageUrls.filter(
|
||
(_, productIndex) => productIndex !== pairedProductIndex
|
||
),
|
||
]
|
||
: productInputImageUrls;
|
||
const replacementExecutionPlan = await analyzeReplacementExecutionPlan({
|
||
modeType: isDetailReplacementPreset ? "detail_replacement" : "sku_replacement",
|
||
replacementStrategy:
|
||
isDetailReplacementPreset ? detailReplacementStrategy : undefined,
|
||
referenceImageUrl: reference.imageUrl,
|
||
referenceLabel: reference.label,
|
||
productImageUrls: orderedProductContextImageUrls,
|
||
modelImageUrl: modelInputImageUrl,
|
||
workflowPrompt:
|
||
isDetailReplacementPreset &&
|
||
detailReplacementStrategy === "product_only"
|
||
? null
|
||
: baseReplacementPrompt || hiddenPromptText || null,
|
||
planningRuntime: replacementScenePlanningRuntime,
|
||
models: replacementVisionModels,
|
||
});
|
||
const replacementSourceType = isDetailReplacementPreset
|
||
? "detail_replacement"
|
||
: "sku_replacement";
|
||
chatApiLogger.info("[Chat API] Replacement reference decision:", {
|
||
mode: replacementSourceType,
|
||
referenceIndex: referenceIndex + 1,
|
||
totalReferences: replacementReferences.length,
|
||
referenceLabel: reference.label,
|
||
referenceImageUrl: reference.imageUrl,
|
||
pairedProductIndex:
|
||
typeof pairedProductIndex === "number" ? pairedProductIndex + 1 : null,
|
||
shouldReplace: replacementExecutionPlan.shouldReplace,
|
||
reason: replacementExecutionPlan.reason,
|
||
detectionModelName: replacementExecutionPlan.detectionModelName,
|
||
detectionSource: replacementExecutionPlan.detectionSource,
|
||
scenePlanSource: replacementExecutionPlan.scenePlan.analysisSource,
|
||
generationRoute: replacementExecutionPlan.scenePlan.generationRoute,
|
||
promptPackPolicy: replacementExecutionPlan.scenePlan.promptPackPolicy,
|
||
textPolicy: replacementExecutionPlan.scenePlan.textPolicy,
|
||
peoplePolicy: replacementExecutionPlan.scenePlan.peoplePolicy,
|
||
selectedProductImageIndexes:
|
||
replacementExecutionPlan.scenePlan.selectedProductImageIndexes,
|
||
useModelImage: replacementExecutionPlan.scenePlan.useModelImage,
|
||
});
|
||
chatApiLogger.info("[Chat API] Replacement scene plan summary:", {
|
||
mode: replacementSourceType,
|
||
referenceIndex: referenceIndex + 1,
|
||
referenceLabel: reference.label,
|
||
analysisModelName: replacementExecutionPlan.scenePlan.analysisModelName,
|
||
analysisSource: replacementExecutionPlan.scenePlan.analysisSource,
|
||
selectedProductImageIndexes:
|
||
replacementExecutionPlan.scenePlan.selectedProductImageIndexes,
|
||
productSelectionReason:
|
||
replacementExecutionPlan.scenePlan.productSelectionReason,
|
||
useModelImage: replacementExecutionPlan.scenePlan.useModelImage,
|
||
modelImageReason: replacementExecutionPlan.scenePlan.modelImageReason,
|
||
pageKind: replacementExecutionPlan.scenePlan.pageKind,
|
||
generationRoute: replacementExecutionPlan.scenePlan.generationRoute,
|
||
promptPackPolicy: replacementExecutionPlan.scenePlan.promptPackPolicy,
|
||
routeReason: replacementExecutionPlan.scenePlan.routeReason,
|
||
});
|
||
const referenceSourceImageUrls = Array.from(
|
||
new Set([
|
||
...sharedUploadedImageUrls,
|
||
...replacementTemplateReferenceImageUrls,
|
||
reference.imageUrl,
|
||
]),
|
||
);
|
||
|
||
if (!isDetailReplacementPreset && !replacementExecutionPlan.shouldReplace) {
|
||
chatApiLogger.info("[Chat API] Replacement reference skipped and reused:", {
|
||
mode: replacementSourceType,
|
||
referenceIndex: referenceIndex + 1,
|
||
totalReferences: replacementReferences.length,
|
||
referenceLabel: reference.label,
|
||
reason: replacementExecutionPlan.reason,
|
||
});
|
||
await emitReplacementResultMessage({
|
||
reference,
|
||
bodyText: replacementExecutionPlan.reason,
|
||
image: {
|
||
originalUrl: reference.imageUrl,
|
||
thumbnailUrl: reference.imageUrl,
|
||
},
|
||
modelDisplayName: null,
|
||
sourceImageUrls: referenceSourceImageUrls,
|
||
sourceType: replacementSourceType,
|
||
streamLabel: isDetailReplacementPreset
|
||
? "agent-preset:image:detail-replacement:item"
|
||
: "agent-preset:image:sku-replacement:item",
|
||
});
|
||
successCount += 1;
|
||
return;
|
||
}
|
||
if (isDetailReplacementPreset && !replacementExecutionPlan.shouldReplace) {
|
||
chatApiLogger.warn("[Chat API] Detail replacement preset plan returned shouldReplace=false; forcing generation:", {
|
||
referenceIndex: referenceIndex + 1,
|
||
totalReferences: replacementReferences.length,
|
||
referenceLabel: reference.label,
|
||
reason: replacementExecutionPlan.reason,
|
||
});
|
||
}
|
||
|
||
const referenceGeometry = await resolveReplacementReferenceGeometry(reference.imageUrl);
|
||
const generationAspectRatio = referenceGeometry?.aspectRatio || resolvedAspectRatio;
|
||
const selectedProductIndexes = replacementExecutionPlan.scenePlan
|
||
.selectedProductImageIndexes.map((index) => index - 1)
|
||
.filter(
|
||
(index) =>
|
||
index >= 0 && index < orderedProductContextImageUrls.length,
|
||
);
|
||
const selectedProductContextImageUrls =
|
||
selectedProductIndexes.length > 0
|
||
? selectedProductIndexes.map(
|
||
(index) => orderedProductContextImageUrls[index]!,
|
||
)
|
||
: orderedProductContextImageUrls.slice(
|
||
0,
|
||
Math.min(orderedProductContextImageUrls.length, 2),
|
||
);
|
||
const shouldIncludeModelImage =
|
||
!(isDetailReplacementPreset &&
|
||
detailReplacementStrategy === "product_only") &&
|
||
Boolean(modelInputImageUrl) &&
|
||
(replacementExecutionPlan.scenePlan.useModelImage ||
|
||
(isDetailReplacementPreset &&
|
||
selectedProductContextImageUrls.length === 0));
|
||
const sourceImageUrls = Array.from(
|
||
new Set(
|
||
[
|
||
...selectedProductContextImageUrls,
|
||
...(shouldIncludeModelImage && modelInputImageUrl
|
||
? [modelInputImageUrl]
|
||
: []),
|
||
...replacementTemplateReferenceImageUrls,
|
||
reference.imageUrl,
|
||
].filter(Boolean),
|
||
),
|
||
);
|
||
const replacementPrompt = buildAgentReplacementPrompt({
|
||
basePrompt:
|
||
baseReplacementPrompt ||
|
||
(isDetailReplacementPreset ? "执行详情替换" : "执行 SKU 替换"),
|
||
modeType: isDetailReplacementPreset
|
||
? "detail_replacement"
|
||
: "sku_replacement",
|
||
replacementStrategy:
|
||
isDetailReplacementPreset ? detailReplacementStrategy : undefined,
|
||
marketplaceProductDetailReference:
|
||
isDetailReplacementPreset &&
|
||
(isMarketplaceProductDetailReferenceId(reference.id) ||
|
||
isMarketplaceProductDetailReferenceId(reference.candidateId)),
|
||
referenceLabel: reference.label,
|
||
referenceIndex,
|
||
totalReferences: replacementReferences.length,
|
||
hasModelImage: shouldIncludeModelImage,
|
||
productImageCount: selectedProductContextImageUrls.length,
|
||
originalProductImageCount: productInputImageUrls.length,
|
||
templateReferenceCount: replacementTemplateReferenceImageUrls.length,
|
||
pairedProductIndex,
|
||
scenePlan: replacementExecutionPlan.scenePlan,
|
||
referenceGeometry,
|
||
});
|
||
chatApiLogger.info("[Chat API] Replacement image generation started:", {
|
||
mode: replacementSourceType,
|
||
referenceIndex: referenceIndex + 1,
|
||
totalReferences: replacementReferences.length,
|
||
referenceLabel: reference.label,
|
||
pairedProductIndex:
|
||
typeof pairedProductIndex === "number" ? pairedProductIndex + 1 : null,
|
||
pageKind: replacementExecutionPlan.scenePlan.pageKind,
|
||
generationRoute: replacementExecutionPlan.scenePlan.generationRoute,
|
||
promptPackPolicy: replacementExecutionPlan.scenePlan.promptPackPolicy,
|
||
basePromptChars: baseReplacementPrompt.length,
|
||
promptChars: replacementPrompt.length,
|
||
hasPromptPackBlock: replacementPrompt.includes("【必须生效的前置场景词包】"),
|
||
sourceImageCount: sourceImageUrls.length,
|
||
generationAspectRatio,
|
||
imageOrder: {
|
||
productImages: selectedProductContextImageUrls.length,
|
||
originalProductImages: productInputImageUrls.length,
|
||
hasModelImage: shouldIncludeModelImage,
|
||
templateReferenceImages: replacementTemplateReferenceImageUrls.length,
|
||
finalReferenceImageIndex: sourceImageUrls.length,
|
||
},
|
||
});
|
||
|
||
try {
|
||
const result = await generateCanvasImage({
|
||
prompt: replacementPrompt,
|
||
negativePrompt:
|
||
typeof mergedImageParams.negativePrompt === "string"
|
||
? mergedImageParams.negativePrompt
|
||
: undefined,
|
||
model: resolvedModelRef,
|
||
ownerKey: session.user.id,
|
||
aspectRatio: generationAspectRatio,
|
||
imageSize: resolvedImageSize,
|
||
imageCount: 1,
|
||
referenceImages: sourceImageUrls,
|
||
debugLabel: `agent-${replacementSourceType}`,
|
||
debugMeta: {
|
||
referenceLabel: reference.label,
|
||
referenceWidth: referenceGeometry?.width,
|
||
referenceHeight: referenceGeometry?.height,
|
||
referenceAspectRatio: referenceGeometry?.aspectRatio,
|
||
},
|
||
onQueueStatusChange:
|
||
contextPayload?.source === "canvas"
|
||
? async (queueStatus) => {
|
||
writeImageGenerationStatusChunk(
|
||
writer,
|
||
{
|
||
phase:
|
||
queueStatus.status === "running"
|
||
? "processing"
|
||
: "queued",
|
||
modelName:
|
||
queueStatus.modelName ||
|
||
resolvedModelDisplayName ||
|
||
resolvedModelRef,
|
||
imageCount: 1,
|
||
queuePosition: queueStatus.queuePosition,
|
||
sameModelRunningCount: queueStatus.sameModelRunningCount,
|
||
concurrencyLimit: queueStatus.concurrencyLimit,
|
||
estimatedWaitMs: queueStatus.estimatedWaitMs,
|
||
},
|
||
`agent-preset:replacement:queue:${reference.id}`,
|
||
);
|
||
}
|
||
: undefined,
|
||
});
|
||
resolvedModelDisplayName = result.model || resolvedModelDisplayName;
|
||
const firstImage = result.images[0];
|
||
if (!firstImage) {
|
||
throw new Error("未返回结果图片");
|
||
}
|
||
let finalImage = firstImage;
|
||
let finalImageSizeLabel = resolvedImageSize;
|
||
try {
|
||
const normalizedImage = await normalizeReplacementResultImage({
|
||
generatedImageUrl: firstImage.originalUrl,
|
||
generatedThumbnailUrl: firstImage.thumbnailUrl,
|
||
referenceImageUrl: reference.imageUrl,
|
||
referenceGeometry,
|
||
scenePlan: replacementExecutionPlan.scenePlan,
|
||
uploadPrefix: isDetailReplacementPreset
|
||
? "agent-detail-replacement"
|
||
: "agent-sku-replacement",
|
||
});
|
||
finalImage = {
|
||
originalUrl: normalizedImage.originalUrl,
|
||
thumbnailUrl: normalizedImage.thumbnailUrl,
|
||
};
|
||
finalImageSizeLabel = normalizedImage.imageSize;
|
||
} catch (normalizeError) {
|
||
chatApiLogger.warn("[Chat API] Failed to normalize replacement output to reference size, using original model output:", {
|
||
referenceLabel: reference.label,
|
||
referenceImageUrl: reference.imageUrl,
|
||
resultImageUrl: firstImage.originalUrl,
|
||
error:
|
||
normalizeError instanceof Error
|
||
? normalizeError.message
|
||
: "unknown",
|
||
});
|
||
const rawResultAccessible = await isRemoteImageAccessible(
|
||
firstImage.originalUrl
|
||
).catch(() => false);
|
||
if (!rawResultAccessible) {
|
||
throw normalizeError;
|
||
}
|
||
}
|
||
chatApiLogger.info("[Chat API] Replacement image generation completed:", {
|
||
mode: replacementSourceType,
|
||
referenceIndex: referenceIndex + 1,
|
||
totalReferences: replacementReferences.length,
|
||
referenceLabel: reference.label,
|
||
modelDisplayName: resolvedModelDisplayName,
|
||
resultImageUrl: finalImage.originalUrl,
|
||
resultImageSize: finalImageSizeLabel,
|
||
});
|
||
await emitReplacementResultMessage({
|
||
reference,
|
||
image: finalImage,
|
||
imageSizeLabel: finalImageSizeLabel,
|
||
modelDisplayName: resolvedModelDisplayName,
|
||
sourceImageUrls: Array.from(
|
||
new Set([
|
||
...sharedUploadedImageUrls,
|
||
...replacementTemplateReferenceImageUrls,
|
||
...replacementReferences.map((item) => item.imageUrl),
|
||
]),
|
||
),
|
||
sourceType: replacementSourceType,
|
||
streamLabel: isDetailReplacementPreset
|
||
? "agent-preset:image:detail-replacement:item"
|
||
: "agent-preset:image:sku-replacement:item",
|
||
});
|
||
successCount += 1;
|
||
} catch (replacementItemError) {
|
||
chatApiLogger.error("[Chat API] Agent replacement item execution failed:", {
|
||
referenceIndex,
|
||
referenceLabel: reference.label,
|
||
error: replacementItemError,
|
||
});
|
||
await emitReplacementResultMessage({
|
||
reference,
|
||
bodyText: `生成失败，当前先沿用参考图：${
|
||
replacementItemError instanceof Error
|
||
? replacementItemError.message
|
||
: "请稍后重试"
|
||
}`,
|
||
image: {
|
||
originalUrl: reference.imageUrl,
|
||
thumbnailUrl: reference.imageUrl,
|
||
},
|
||
modelDisplayName: null,
|
||
sourceImageUrls: referenceSourceImageUrls,
|
||
sourceType: replacementSourceType,
|
||
streamLabel: isDetailReplacementPreset
|
||
? "agent-preset:image:detail-replacement:item-fallback"
|
||
: "agent-preset:image:sku-replacement:item-fallback",
|
||
});
|
||
successCount += 1;
|
||
}
|
||
});
|
||
} finally {
|
||
stopKeepalive();
|
||
}
|
||
} else if (selectedAgentPreset.type === "image" && isSceneGenerationPreset) {
|
||
const sceneSlot = orderedSlots[0];
|
||
const sceneAssistantMessageId = generateUUID();
|
||
const sceneReferenceImageUrls = Array.from(
|
||
new Set(
|
||
(sceneSlot?.referenceAssets || [])
|
||
.map((asset) => String(asset.originalUrl || "").trim())
|
||
.filter(Boolean),
|
||
),
|
||
);
|
||
const sourceImageUrls = Array.from(
|
||
new Set([...sharedUploadedImageUrls, ...sceneReferenceImageUrls].filter(Boolean)),
|
||
);
|
||
const sourceAssetIds = Array.from(
|
||
new Set(
|
||
[...sharedUploadedAssetIds, ...(sceneSlot?.referenceAssetIds || [])].filter(Boolean),
|
||
),
|
||
);
|
||
const sceneUploadedImageBreakdown = getSceneUploadedImageBreakdown({
|
||
uploadedImageParts,
|
||
fallbackPrimaryCount: sceneProductSlotCount,
|
||
fallbackIncludeModelImage: sceneModelSlotEnabled,
|
||
});
|
||
const sceneImageRolePrompt = buildAgentSceneImageRolePrompt({
|
||
primaryProductImageCount: sceneUploadedImageBreakdown.primaryProductImageCount,
|
||
bundleProductImageCount: sceneUploadedImageBreakdown.bundleProductImageCount,
|
||
includeModelImage: sceneUploadedImageBreakdown.includeModelImage,
|
||
});
|
||
const sceneOptimizationImageUrls = resolveScenePromptOptimizationImageUrls({
|
||
uploadedImageParts,
|
||
primaryProductImageCount: sceneUploadedImageBreakdown.primaryProductImageCount,
|
||
bundleProductImageCount: sceneUploadedImageBreakdown.bundleProductImageCount,
|
||
includeModelImage: sceneUploadedImageBreakdown.includeModelImage,
|
||
});
|
||
const shouldAppendSceneSlotPrompt = !hiddenPromptText;
|
||
const rawSceneSlotPrompt = shouldAppendSceneSlotPrompt
|
||
? sceneSlot?.promptTemplate || selectedAgentPreset.promptTemplate || ""
|
||
: "";
|
||
const dedupedSceneSlotPrompt = rawSceneSlotPrompt
|
||
? dedupeAgentSlotPromptAgainstWorkflow({
|
||
workflowPrompt: hiddenPromptText || null,
|
||
slotPrompt: rawSceneSlotPrompt,
|
||
modeType: "scene_generation",
|
||
})
|
||
: "";
|
||
const optimizedSceneWorkflowResult =
|
||
workflowPromptOptimizationEnabled !== false &&
|
||
hiddenPromptText &&
|
||
sceneOptimizationImageUrls.primaryImageUrl
|
||
? await optimizeAgentSceneWorkflowPrompt({
|
||
workflowPrompt: hiddenPromptText,
|
||
slotPrompt: dedupedSceneSlotPrompt || null,
|
||
primaryImageUrl: sceneOptimizationImageUrls.primaryImageUrl,
|
||
modelImageUrl: sceneOptimizationImageUrls.modelImageUrl,
|
||
}).catch((error) => {
|
||
chatApiLogger.warn("[Chat API] Agent scene workflow prompt optimization failed:", error);
|
||
return null;
|
||
})
|
||
: null;
|
||
const optimizedSceneWorkflowPrompt = hiddenPromptText
|
||
? buildOptimizedAgentSceneWorkflowPrompt({
|
||
workflowPrompt: hiddenPromptText,
|
||
optimizationResult: optimizedSceneWorkflowResult,
|
||
})
|
||
: "";
|
||
if (optimizedSceneWorkflowResult) {
|
||
chatApiLogger.info("[Chat API] Agent scene workflow prompt optimized:", {
|
||
chatId: id,
|
||
agentPresetId: selectedAgentPreset.id,
|
||
optimizationModelName: optimizedSceneWorkflowResult.optimizationModelName || null,
|
||
optimizationSource: optimizedSceneWorkflowResult.optimizationSource || null,
|
||
sceneGoal: optimizedSceneWorkflowResult.sceneGoal,
|
||
environmentSummary: optimizedSceneWorkflowResult.environmentSummary,
|
||
cameraLightingSummary: optimizedSceneWorkflowResult.cameraLightingSummary,
|
||
characterSummary: optimizedSceneWorkflowResult.characterSummary,
|
||
negativeConstraints: optimizedSceneWorkflowResult.negativeConstraints,
|
||
conflicts: optimizedSceneWorkflowResult.conflicts,
|
||
});
|
||
} else if (hiddenPromptText && workflowPromptOptimizationEnabled === false) {
|
||
chatApiLogger.info("[Chat API] Agent scene workflow prompt optimization skipped:", {
|
||
chatId: id,
|
||
agentPresetId: selectedAgentPreset.id,
|
||
reason: "workflow_prompt_optimization_disabled",
|
||
});
|
||
}
|
||
const scenePrompt =
|
||
buildAgentSceneWorkflowPrompt({
|
||
workflowPrompt: optimizedSceneWorkflowPrompt || hiddenPromptText || null,
|
||
slotPrompt: optimizedSceneWorkflowPrompt ? null : dedupedSceneSlotPrompt || null,
|
||
imageRolePrompt: sceneImageRolePrompt,
|
||
userSupplementPrompt: sanitizedUserSupplementPrompt || null,
|
||
}) ||
|
||
buildAgentSlotPrompt({
|
||
workflowPrompt: hiddenPromptText || null,
|
||
slotPrompt: sceneSlot?.promptTemplate || selectedAgentPreset.promptTemplate,
|
||
userSupplementPrompt: sanitizedUserSupplementPrompt || null,
|
||
includeSlotTitle: false,
|
||
modeType: "scene_generation",
|
||
});
|
||
const sceneGenerationStartedAt = Date.now();
|
||
chatApiLogger.info("[Chat API] Agent scene prompt guardrails:", {
|
||
chatId: id,
|
||
agentPresetId: selectedAgentPreset.id,
|
||
agentPresetName: selectedAgentPreset.name,
|
||
agentExecutionSnapshotId: presetSnapshot.id,
|
||
includesWorkflowPrompt: Boolean(hiddenPromptText),
|
||
includesUserSupplementPrompt: Boolean(sanitizedUserSupplementPrompt),
|
||
includesModelImage: sceneUploadedImageBreakdown.includeModelImage,
|
||
shouldAppendSceneSlotPrompt,
|
||
includesSceneSlotPrompt: Boolean(dedupedSceneSlotPrompt),
|
||
productImageCount: sceneUploadedImageBreakdown.primaryProductImageCount,
|
||
bundleImageCount: sceneUploadedImageBreakdown.bundleProductImageCount,
|
||
uploadedImageCount: sharedUploadedImageUrls.length,
|
||
hasProductIdentityLock: sceneImageRolePrompt.includes("【产品锁定约束】"),
|
||
hasModelIdentityLock: sceneImageRolePrompt.includes("【模特锁定约束】"),
|
||
hasModelClothingOverrideRule: sceneImageRolePrompt.includes(
|
||
"只有当工作流设定或补充要求里明确要求修改模特衣服"
|
||
),
|
||
sceneSlotPromptPreview: dedupedSceneSlotPrompt
|
||
.split("\n")
|
||
.map((line) => line.trim())
|
||
.filter(Boolean)
|
||
.slice(0, 10),
|
||
imageRolePreview: sceneImageRolePrompt
|
||
.split("\n")
|
||
.map((line) => line.trim())
|
||
.filter(Boolean)
|
||
.slice(0, 12),
|
||
});
|
||
|
||
try {
|
||
const mergedImageParams = {
|
||
...(selectedAgentPreset.defaultParams || {}),
|
||
...(sceneSlot?.defaultParams || {}),
|
||
} as Record<string, any>;
|
||
const resolvedModelRef =
|
||
presetModeConfig.scene.allowModelOverride &&
|
||
String(selectedChatModel || "").trim() &&
|
||
String(selectedChatModel || "").trim() !== "auto"
|
||
? String(selectedChatModel || "").trim()
|
||
: presetModelId;
|
||
const resolvedImageCount = Math.max(1, effectiveImageParams?.imageCount || 1);
|
||
chatApiLogger.info("[Chat API] Agent scene generation started:", {
|
||
chatId: id,
|
||
agentPresetId: selectedAgentPreset.id,
|
||
agentPresetName: selectedAgentPreset.name,
|
||
agentExecutionSnapshotId: presetSnapshot.id,
|
||
projectId: contextPayload?.projectId || null,
|
||
selectedChatModel: String(selectedChatModel || "").trim() || "auto",
|
||
resolvedModelRef,
|
||
resolvedImageCount,
|
||
sourceImageCount: sourceImageUrls.length,
|
||
sourceAssetCount: sourceAssetIds.length,
|
||
promptStrategy: hiddenPromptText
|
||
? "workflow_plus_image_lock"
|
||
: rawSceneSlotPrompt
|
||
? "legacy_template_fallback"
|
||
: sanitizedUserSupplementPrompt
|
||
? "workflow_plus_user_supplement"
|
||
: "workflow_only",
|
||
promptLength: String(scenePrompt || `生成${selectedAgentPreset.name}`).length,
|
||
aspectRatio:
|
||
typeof mergedImageParams.aspectRatio === "string"
|
||
? mergedImageParams.aspectRatio
|
||
: effectiveImageParams?.aspectRatio || defaultImageQualityParams.aspectRatio,
|
||
imageSize:
|
||
typeof mergedImageParams.imageSize === "string"
|
||
? mergedImageParams.imageSize
|
||
: effectiveImageParams?.imageSize || defaultImageQualityParams.imageSize,
|
||
});
|
||
const result = await generateCanvasImage({
|
||
prompt: scenePrompt || `生成${selectedAgentPreset.name}`,
|
||
negativePrompt:
|
||
typeof mergedImageParams.negativePrompt === "string"
|
||
? mergedImageParams.negativePrompt
|
||
: undefined,
|
||
model: resolvedModelRef,
|
||
ownerKey: session.user.id,
|
||
aspectRatio:
|
||
typeof mergedImageParams.aspectRatio === "string"
|
||
? mergedImageParams.aspectRatio
|
||
: effectiveImageParams?.aspectRatio || defaultImageQualityParams.aspectRatio,
|
||
imageSize:
|
||
typeof mergedImageParams.imageSize === "string"
|
||
? mergedImageParams.imageSize
|
||
: effectiveImageParams?.imageSize || defaultImageQualityParams.imageSize,
|
||
imageCount: resolvedImageCount,
|
||
referenceImages: sourceImageUrls,
|
||
debugLabel: "agent-scene-generation",
|
||
debugMeta: {
|
||
chatId: id,
|
||
agentPresetId: selectedAgentPreset.id,
|
||
agentPresetName: selectedAgentPreset.name,
|
||
agentExecutionSnapshotId: presetSnapshot.id,
|
||
projectId: contextPayload?.projectId || null,
|
||
},
|
||
onQueueStatusChange:
|
||
contextPayload?.source === "canvas"
|
||
? async (queueStatus) => {
|
||
writeImageGenerationStatusChunk(
|
||
writer,
|
||
{
|
||
phase:
|
||
queueStatus.status === "running"
|
||
? "processing"
|
||
: "queued",
|
||
modelName:
|
||
queueStatus.modelName || selectedAgentPreset.name,
|
||
imageCount: resolvedImageCount,
|
||
queuePosition: queueStatus.queuePosition,
|
||
sameModelRunningCount: queueStatus.sameModelRunningCount,
|
||
concurrencyLimit: queueStatus.concurrencyLimit,
|
||
estimatedWaitMs: queueStatus.estimatedWaitMs,
|
||
},
|
||
"agent-preset:image:scene-generation:queue",
|
||
);
|
||
}
|
||
: undefined,
|
||
});
|
||
chatApiLogger.info("[Chat API] Agent scene generation model completed:", {
|
||
chatId: id,
|
||
agentPresetId: selectedAgentPreset.id,
|
||
agentPresetName: selectedAgentPreset.name,
|
||
agentExecutionSnapshotId: presetSnapshot.id,
|
||
resultImageCount: result.images.length,
|
||
resultModelName: result.model,
|
||
resultModelRef: result.modelRef,
|
||
elapsedMs: Date.now() - sceneGenerationStartedAt,
|
||
});
|
||
|
||
const imageMarkdown = buildAgentToolImagesMarkdown(
|
||
result.images.map((image) => ({
|
||
originalUrl: image.originalUrl,
|
||
thumbnailUrl: image.thumbnailUrl,
|
||
imageSize:
|
||
typeof mergedImageParams.imageSize === "string"
|
||
? mergedImageParams.imageSize
|
||
: effectiveImageParams?.imageSize || defaultImageQualityParams.imageSize,
|
||
})),
|
||
);
|
||
const assistantText = prependImageModelBadge(
|
||
[`### ${selectedAgentPreset.name}`, imageMarkdown].filter(Boolean).join("\n\n"),
|
||
result.model,
|
||
);
|
||
|
||
await saveMessages({
|
||
messages: [
|
||
{
|
||
id: sceneAssistantMessageId,
|
||
chatId: id,
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: assistantText }],
|
||
attachments: [],
|
||
createdAt: new Date(),
|
||
videoStorageStatus: null,
|
||
temporaryVideoUrl: null,
|
||
permanentVideoUrl: null,
|
||
archivedAt: null,
|
||
videoMetadata: null,
|
||
},
|
||
],
|
||
});
|
||
|
||
await saveGeneratedImages(
|
||
assistantText,
|
||
session.user.id,
|
||
id,
|
||
sceneAssistantMessageId,
|
||
scenePrompt || selectedAgentPreset.name,
|
||
result.modelRef || resolvedModelRef,
|
||
{
|
||
sourceAssetIds,
|
||
sourceImageUrls,
|
||
relationType: "generated_from",
|
||
sourceType:
|
||
sourceImageUrls.length > 0 ? "reference_generate" : "generated",
|
||
projectId: contextPayload?.projectId || null,
|
||
userPrompt: requestTextContent,
|
||
optimizedPrompt: scenePrompt || undefined,
|
||
requestedImageCount: resolvedImageCount,
|
||
},
|
||
);
|
||
chatApiLogger.info("[Chat API] Agent scene generation persisted:", {
|
||
chatId: id,
|
||
agentPresetId: selectedAgentPreset.id,
|
||
agentPresetName: selectedAgentPreset.name,
|
||
agentExecutionSnapshotId: presetSnapshot.id,
|
||
savedImageCount: result.images.length,
|
||
elapsedMs: Date.now() - sceneGenerationStartedAt,
|
||
});
|
||
|
||
writeAssistantTextChunk(
|
||
writer,
|
||
assistantText,
|
||
"agent-preset:image:scene-generation",
|
||
sceneAssistantMessageId,
|
||
);
|
||
if (contextPayload?.source === "canvas") {
|
||
writeCanvasMediaChunk(
|
||
writer,
|
||
{
|
||
messageId: sceneAssistantMessageId,
|
||
imageUrls: result.images.map((image) => image.originalUrl),
|
||
prompt: scenePrompt || selectedAgentPreset.name,
|
||
sourceImageUrls,
|
||
agentPresetId: selectedAgentPreset.id,
|
||
agentPresetName: selectedAgentPreset.name,
|
||
agentCategoryId:
|
||
selectedAgentPreset.category?.id ||
|
||
selectedAgentPreset.categoryId ||
|
||
null,
|
||
agentCategoryName: selectedAgentPreset.category?.name || null,
|
||
agentExecutionSnapshotId: presetSnapshot.id,
|
||
},
|
||
"agent-preset:image:scene-generation:canvas-media",
|
||
);
|
||
}
|
||
chatApiLogger.info("[Chat API] Agent scene generation response streamed:", {
|
||
chatId: id,
|
||
agentPresetId: selectedAgentPreset.id,
|
||
agentPresetName: selectedAgentPreset.name,
|
||
agentExecutionSnapshotId: presetSnapshot.id,
|
||
resultImageCount: result.images.length,
|
||
elapsedMs: Date.now() - sceneGenerationStartedAt,
|
||
});
|
||
|
||
successCount = result.images.length;
|
||
} catch (sceneError) {
|
||
chatApiLogger.error("[Chat API] Agent preset scene generation failed:", sceneError);
|
||
chatApiLogger.error("[Chat API] Agent scene generation failed detail:", {
|
||
chatId: id,
|
||
agentPresetId: selectedAgentPreset.id,
|
||
agentPresetName: selectedAgentPreset.name,
|
||
agentExecutionSnapshotId: presetSnapshot.id,
|
||
elapsedMs: Date.now() - sceneGenerationStartedAt,
|
||
error: sceneError instanceof Error ? sceneError.message : String(sceneError),
|
||
});
|
||
const errorMessage = `### ${selectedAgentPreset.name}\n\n执行失败：${
|
||
sceneError instanceof Error ? sceneError.message : "请稍后重试"
|
||
}`;
|
||
await saveMessages({
|
||
messages: [
|
||
{
|
||
id: sceneAssistantMessageId,
|
||
chatId: id,
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: errorMessage }],
|
||
attachments: [],
|
||
createdAt: new Date(),
|
||
videoStorageStatus: null,
|
||
temporaryVideoUrl: null,
|
||
permanentVideoUrl: null,
|
||
archivedAt: null,
|
||
videoMetadata: null,
|
||
},
|
||
],
|
||
});
|
||
writeAssistantTextChunk(
|
||
writer,
|
||
errorMessage,
|
||
"agent-preset:error:scene-generation",
|
||
sceneAssistantMessageId,
|
||
);
|
||
}
|
||
} else {
|
||
for (const [slotOrder, slot] of orderedSlots.entries()) {
|
||
const slotLabel =
|
||
String(slot.title || slot.uploadLabel || "").trim() || `第${slotOrder + 1}张`;
|
||
const slotReferenceImageUrls = Array.from(
|
||
new Set(
|
||
(slot.referenceAssets || [])
|
||
.map((asset) => String(asset.originalUrl || "").trim())
|
||
.filter(Boolean),
|
||
),
|
||
);
|
||
const sourceImageUrls = Array.from(
|
||
new Set(
|
||
[
|
||
...sharedUploadedImageUrls,
|
||
...slotReferenceImageUrls,
|
||
].filter(Boolean),
|
||
),
|
||
);
|
||
const sourceAssetIds = Array.from(
|
||
new Set(
|
||
[
|
||
...sharedUploadedAssetIds,
|
||
...(slot.referenceAssetIds || []),
|
||
].filter(Boolean),
|
||
),
|
||
);
|
||
const slotPrompt = buildAgentSlotPrompt({
|
||
workflowPrompt: hiddenPromptText || null,
|
||
slotPrompt: slot.promptTemplate,
|
||
slotTitle: slot.title || slot.uploadLabel || null,
|
||
userSupplementPrompt: sanitizedUserSupplementPrompt || null,
|
||
modeType: "generic",
|
||
});
|
||
const assistantMessageId = generateUUID();
|
||
|
||
try {
|
||
if (selectedAgentPreset.type === "image") {
|
||
const mergedImageParams = {
|
||
...(selectedAgentPreset.defaultParams || {}),
|
||
...(slot.defaultParams || {}),
|
||
} as Record<string, any>;
|
||
const result = await generateCanvasImage({
|
||
prompt: slotPrompt || `生成${slotLabel}`,
|
||
negativePrompt:
|
||
typeof mergedImageParams.negativePrompt === "string"
|
||
? mergedImageParams.negativePrompt
|
||
: undefined,
|
||
model: presetModelId,
|
||
ownerKey: session.user.id,
|
||
aspectRatio:
|
||
typeof mergedImageParams.aspectRatio === "string"
|
||
? mergedImageParams.aspectRatio
|
||
: effectiveImageParams?.aspectRatio || defaultImageQualityParams.aspectRatio,
|
||
imageSize:
|
||
typeof mergedImageParams.imageSize === "string"
|
||
? mergedImageParams.imageSize
|
||
: effectiveImageParams?.imageSize || defaultImageQualityParams.imageSize,
|
||
imageCount: 1,
|
||
referenceImages: sourceImageUrls,
|
||
onQueueStatusChange:
|
||
contextPayload?.source === "canvas"
|
||
? async (queueStatus) => {
|
||
writeImageGenerationStatusChunk(
|
||
writer,
|
||
{
|
||
phase:
|
||
queueStatus.status === "running"
|
||
? "processing"
|
||
: "queued",
|
||
modelName: queueStatus.modelName || slotLabel,
|
||
imageCount: 1,
|
||
queuePosition: queueStatus.queuePosition,
|
||
sameModelRunningCount: queueStatus.sameModelRunningCount,
|
||
concurrencyLimit: queueStatus.concurrencyLimit,
|
||
estimatedWaitMs: queueStatus.estimatedWaitMs,
|
||
},
|
||
`agent-preset:image:slot-queue:${slot.slotIndex}`,
|
||
);
|
||
}
|
||
: undefined,
|
||
});
|
||
|
||
const imageMarkdown = buildAgentToolImagesMarkdown(
|
||
result.images.map((image) => ({
|
||
originalUrl: image.originalUrl,
|
||
thumbnailUrl: image.thumbnailUrl,
|
||
imageSize:
|
||
typeof mergedImageParams.imageSize === "string"
|
||
? mergedImageParams.imageSize
|
||
: effectiveImageParams?.imageSize || defaultImageQualityParams.imageSize,
|
||
})),
|
||
);
|
||
const assistantText = prependImageModelBadge(
|
||
[`### ${slotLabel}`, imageMarkdown].filter(Boolean).join("\n\n"),
|
||
result.model,
|
||
);
|
||
|
||
await saveMessages({
|
||
messages: [
|
||
{
|
||
id: assistantMessageId,
|
||
chatId: id,
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: assistantText }],
|
||
attachments: [],
|
||
createdAt: new Date(),
|
||
videoStorageStatus: null,
|
||
temporaryVideoUrl: null,
|
||
permanentVideoUrl: null,
|
||
archivedAt: null,
|
||
videoMetadata: null,
|
||
},
|
||
],
|
||
});
|
||
|
||
await saveGeneratedImages(
|
||
assistantText,
|
||
session.user.id,
|
||
id,
|
||
assistantMessageId,
|
||
slotPrompt || slotLabel,
|
||
result.modelRef || presetModelId,
|
||
{
|
||
sourceAssetIds,
|
||
sourceImageUrls,
|
||
relationType: "generated_from",
|
||
sourceType:
|
||
sourceImageUrls.length > 0 ? "reference_generate" : "generated",
|
||
projectId: contextPayload?.projectId || null,
|
||
userPrompt: requestTextContent,
|
||
optimizedPrompt: slotPrompt || undefined,
|
||
requestedImageCount: 1,
|
||
},
|
||
);
|
||
|
||
writeAssistantTextChunk(
|
||
writer,
|
||
assistantText,
|
||
`agent-preset:image:slot-${slotOrder + 1}`,
|
||
assistantMessageId,
|
||
);
|
||
if (contextPayload?.source === "canvas") {
|
||
writeCanvasMediaChunk(
|
||
writer,
|
||
{
|
||
messageId: assistantMessageId,
|
||
imageUrls: result.images.map((image) => image.originalUrl),
|
||
prompt: slotPrompt || slotLabel,
|
||
sourceImageUrls,
|
||
agentPresetId: selectedAgentPreset.id,
|
||
agentPresetName: selectedAgentPreset.name,
|
||
agentCategoryId:
|
||
selectedAgentPreset.category?.id ||
|
||
selectedAgentPreset.categoryId ||
|
||
null,
|
||
agentCategoryName: selectedAgentPreset.category?.name || null,
|
||
agentExecutionSnapshotId: presetSnapshot.id,
|
||
},
|
||
`agent-preset:image:slot-${slotOrder + 1}:canvas-media`,
|
||
);
|
||
}
|
||
} else {
|
||
const mergedVideoParams = {
|
||
...(selectedAgentPreset.defaultParams || {}),
|
||
...(slot.defaultParams || {}),
|
||
} as Record<string, any>;
|
||
const requestedMode = String(mergedVideoParams.mode || "").trim() as
|
||
| "text-to-video"
|
||
| "first-frame"
|
||
| "first-last-frame"
|
||
| "reference-images";
|
||
const mode =
|
||
requestedMode ||
|
||
(sourceImageUrls.length >= 2
|
||
? "first-last-frame"
|
||
: sourceImageUrls.length >= 1
|
||
? "first-frame"
|
||
: "text-to-video");
|
||
const directVideoRatio =
|
||
mergedVideoParams.ratio === "16:9" ||
|
||
mergedVideoParams.ratio === "4:3" ||
|
||
mergedVideoParams.ratio === "1:1" ||
|
||
mergedVideoParams.ratio === "3:4" ||
|
||
mergedVideoParams.ratio === "9:16" ||
|
||
mergedVideoParams.ratio === "21:9" ||
|
||
mergedVideoParams.ratio === "adaptive"
|
||
? mergedVideoParams.ratio
|
||
: undefined;
|
||
const images =
|
||
mode === "first-last-frame"
|
||
? sourceImageUrls.slice(0, 2)
|
||
: mode === "first-frame"
|
||
? sourceImageUrls.slice(0, 1)
|
||
: mode === "reference-images"
|
||
? sourceImageUrls.slice(0, 4)
|
||
: [];
|
||
|
||
const result = await executeDirectVideoGeneration({
|
||
prompt: slotPrompt || `生成${slotLabel}`,
|
||
mode,
|
||
images,
|
||
resolution: mergedVideoParams.resolution,
|
||
ratio: directVideoRatio,
|
||
duration: mergedVideoParams.duration,
|
||
preferredModelRef: presetModelId,
|
||
});
|
||
|
||
const videoUrl = String(result.videoUrl || "").trim();
|
||
if (!videoUrl) {
|
||
throw new Error("视频生成完成但未返回视频地址");
|
||
}
|
||
|
||
const assistantText = prependImageModelBadge(
|
||
[
|
||
`### ${slotLabel}`,
|
||
buildAgentToolVideoMarkdown(videoUrl),
|
||
]
|
||
.filter(Boolean)
|
||
.join("\n\n"),
|
||
result.modelDisplayName,
|
||
);
|
||
|
||
await saveMessages({
|
||
messages: [
|
||
{
|
||
id: assistantMessageId,
|
||
chatId: id,
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: assistantText }],
|
||
attachments: [],
|
||
createdAt: new Date(),
|
||
videoStorageStatus: "temporary",
|
||
temporaryVideoUrl: videoUrl,
|
||
permanentVideoUrl: null,
|
||
archivedAt: null,
|
||
videoMetadata: (result.metadata || {}) as any,
|
||
},
|
||
],
|
||
});
|
||
|
||
try {
|
||
await persistVideoGenerationCompletedTurn({
|
||
userId: session.user.id,
|
||
chatId: id,
|
||
projectId: contextPayload?.projectId || null,
|
||
messageId: assistantMessageId,
|
||
text: requestTextContent,
|
||
source: "agent_preset_direct_video",
|
||
providerType: "direct-video",
|
||
modelRef: result.modelRef || presetModelId,
|
||
mode,
|
||
status: "completed",
|
||
temporaryVideoUrl: videoUrl,
|
||
});
|
||
} catch (memoryError) {
|
||
chatApiLogger.warn("[Chat API] Failed to update working state after agent preset video generation:", memoryError);
|
||
}
|
||
|
||
await consumeGeneratedVideoPoints({
|
||
userId: session.user.id,
|
||
chatId: id,
|
||
messageId: assistantMessageId,
|
||
modelRefs: [result.modelRef || presetModelId],
|
||
duration: Number((result.metadata as any)?.duration) || undefined,
|
||
resolution:
|
||
String((result.metadata as any)?.resolution || "").trim() ||
|
||
undefined,
|
||
});
|
||
|
||
scheduleVideoArchive({
|
||
chatId: id,
|
||
messageId: assistantMessageId,
|
||
text: assistantText,
|
||
temporaryVideoUrl: videoUrl,
|
||
taskId: assistantMessageId,
|
||
metadata: result.metadata,
|
||
});
|
||
|
||
writeAssistantTextChunk(
|
||
writer,
|
||
assistantText,
|
||
`agent-preset:video:slot-${slotOrder + 1}`,
|
||
assistantMessageId,
|
||
);
|
||
if (contextPayload?.source === "canvas") {
|
||
writeCanvasMediaChunk(
|
||
writer,
|
||
{
|
||
messageId: assistantMessageId,
|
||
videoUrls: [videoUrl],
|
||
prompt: slotPrompt || slotLabel,
|
||
sourceImageUrls,
|
||
agentPresetId: selectedAgentPreset.id,
|
||
agentPresetName: selectedAgentPreset.name,
|
||
agentCategoryId:
|
||
selectedAgentPreset.category?.id ||
|
||
selectedAgentPreset.categoryId ||
|
||
null,
|
||
agentCategoryName: selectedAgentPreset.category?.name || null,
|
||
agentExecutionSnapshotId: presetSnapshot.id,
|
||
},
|
||
`agent-preset:video:slot-${slotOrder + 1}:canvas-media`,
|
||
);
|
||
}
|
||
}
|
||
|
||
successCount += 1;
|
||
} catch (slotError) {
|
||
chatApiLogger.error("[Chat API] Agent preset slot execution failed:", {
|
||
slotIndex: slot.slotIndex,
|
||
slotLabel,
|
||
error: slotError,
|
||
});
|
||
const errorMessage = `### ${slotLabel}\n\n执行失败：${
|
||
slotError instanceof Error ? slotError.message : "请稍后重试"
|
||
}`;
|
||
await saveMessages({
|
||
messages: [
|
||
{
|
||
id: assistantMessageId,
|
||
chatId: id,
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: errorMessage }],
|
||
attachments: [],
|
||
createdAt: new Date(),
|
||
videoStorageStatus: null,
|
||
temporaryVideoUrl: null,
|
||
permanentVideoUrl: null,
|
||
archivedAt: null,
|
||
videoMetadata: null,
|
||
},
|
||
],
|
||
});
|
||
if (selectedAgentPreset.type === "video") {
|
||
try {
|
||
await persistVideoGenerationFailedTurn({
|
||
userId: session.user.id,
|
||
chatId: id,
|
||
projectId: contextPayload?.projectId || null,
|
||
messageId: assistantMessageId,
|
||
text: requestTextContent,
|
||
source: "agent_preset_direct_video",
|
||
providerType: "direct-video",
|
||
modelRef: presetModelId,
|
||
status: "failed",
|
||
error: slotError instanceof Error ? slotError.message : String(slotError),
|
||
});
|
||
} catch (videoMemoryError) {
|
||
chatApiLogger.warn("[Chat API] Failed to persist agent preset video failure state:", videoMemoryError);
|
||
}
|
||
}
|
||
writeAssistantTextChunk(
|
||
writer,
|
||
errorMessage,
|
||
`agent-preset:error:slot-${slotOrder + 1}`,
|
||
assistantMessageId,
|
||
);
|
||
}
|
||
}
|
||
}
|
||
|
||
try {
|
||
await persistAgentPresetTurn({
|
||
userId: session.user.id,
|
||
chatId: id,
|
||
projectId: contextPayload?.projectId || null,
|
||
currentIntent: selectedAgentPreset.type === "video" ? "video_gen" : "image_gen",
|
||
lastMessageId: message.id,
|
||
text: requestTextContent,
|
||
agentPresetId: selectedAgentPreset.id,
|
||
agentPresetName: selectedAgentPreset.name,
|
||
slotCount: orderedSlots.length,
|
||
successCount,
|
||
});
|
||
} catch (agentPresetMemoryError) {
|
||
chatApiLogger.warn("[Chat API] Failed to persist agent preset working state:", agentPresetMemoryError);
|
||
}
|
||
|
||
if (selectedAgentPreset.type === "image") {
|
||
if (contextPayload?.source === "canvas") {
|
||
writeImageGenerationStatusChunk(
|
||
writer,
|
||
{
|
||
phase: successCount > 0 ? "completed" : "failed",
|
||
modelName:
|
||
selectedAgentPreset.defaultModel?.displayName ||
|
||
selectedAgentPreset.name,
|
||
imageCount: successCount,
|
||
},
|
||
"agent-preset:image:status-finished",
|
||
);
|
||
}
|
||
writeAssistantTextChunk(
|
||
writer,
|
||
"<!-- IMAGE_GENERATION_END -->",
|
||
"agent-preset:image:end",
|
||
);
|
||
}
|
||
},
|
||
});
|
||
|
||
return createSseResponse(agentSequenceStream);
|
||
}
|
||
}
|
||
|
||
const directAgentToolContext = shouldRunAgentConversation
|
||
? resolveDirectAgentToolContext({
|
||
execution: smartAutoExecution,
|
||
plan: smartAutoPlan,
|
||
requestResolvedMediaInput,
|
||
requestTargetResolution,
|
||
requestWorkingState,
|
||
})
|
||
: null;
|
||
const directAgentExecution = directAgentToolContext?.execution || null;
|
||
const directAgentToolTargetImageUrls = directAgentToolContext?.targetImageUrls || [];
|
||
const directAgentToolTargetImageUrl = directAgentToolContext?.targetImageUrl || null;
|
||
|
||
if (
|
||
shouldRunAgentConversation &&
|
||
!smartAutoNeedsClarification &&
|
||
directAgentExecution?.toolName === "analyzeImage"
|
||
) {
|
||
if (directAgentToolTargetImageUrls.length <= 0) {
|
||
const missingTargetMessage = getDirectAgentToolTargetMissingMessage("analyze");
|
||
const missingTargetStream = createUIMessageStream({
|
||
execute: async ({ writer }) => {
|
||
const messageId = generateUUID();
|
||
writeCurrentAgentTargetLockChunk(writer, "agent:analyze:missing-target-lock");
|
||
writeAssistantTextChunk(
|
||
writer,
|
||
missingTargetMessage,
|
||
"agent:analyze:missing-target",
|
||
messageId,
|
||
);
|
||
|
||
await saveAssistantTextOnlyMessage({
|
||
chatId: id,
|
||
messageId,
|
||
text: missingTargetMessage,
|
||
});
|
||
},
|
||
});
|
||
|
||
return createSseResponse(missingTargetStream);
|
||
}
|
||
|
||
const analysisAssistantMessageId = generateUUID();
|
||
const analysisSummaryAssetIds = Array.from(
|
||
new Set([
|
||
...(requestTargetResolution?.targetAssetIds || []),
|
||
...(requestResolvedMediaInput.selectedAssetIds || []),
|
||
...(requestResolvedMediaInput.uploadedAssetIds || []),
|
||
...extractMessageImageAssetIds(message.parts as any[]),
|
||
]),
|
||
);
|
||
|
||
const directAnalyzeStream = createUIMessageStream({
|
||
execute: async ({ writer }) => {
|
||
writeAgentProgressChunk(
|
||
writer,
|
||
buildDirectAgentAnalyzePlanningProgress(),
|
||
"agent:analyze:progress-planning",
|
||
);
|
||
writeCurrentAgentTargetLockChunk(writer, "agent:analyze:target-lock");
|
||
|
||
try {
|
||
const analysisResult = await executeDirectAgentImageAnalysis({
|
||
requestTextContent,
|
||
imageUrls: directAgentToolTargetImageUrls,
|
||
context: effectiveAutoContextText || undefined,
|
||
preferredModelId: preferredAgentModelId || finalModelId,
|
||
answerModelId: finalModelId || preferredAgentModelId,
|
||
assetIds: analysisSummaryAssetIds,
|
||
});
|
||
const assistantText = buildDirectAgentAnalysisAssistantText({
|
||
summary: analysisResult.summary,
|
||
answer: analysisResult.answer,
|
||
});
|
||
|
||
await saveAssistantTextOnlyMessage({
|
||
chatId: id,
|
||
messageId: analysisAssistantMessageId,
|
||
text: assistantText,
|
||
});
|
||
|
||
await safePersistDirectAgentAnalyzeTurn({
|
||
userId: session.user.id,
|
||
chatId: id,
|
||
projectId: contextPayload?.projectId || null,
|
||
currentSubIntent: smartAutoExecution?.routeSubIntent || "analyze",
|
||
targetAssetIds: requestTargetResolution?.targetAssetIds || null,
|
||
targetImageUrls: directAgentToolTargetImageUrls,
|
||
messageId: analysisAssistantMessageId,
|
||
requestTextContent,
|
||
});
|
||
|
||
writeAssistantTextChunk(
|
||
writer,
|
||
assistantText,
|
||
"agent:analyze:success",
|
||
analysisAssistantMessageId,
|
||
);
|
||
writeAgentProgressChunk(
|
||
writer,
|
||
buildDirectAgentAnalyzeCompletedProgress(),
|
||
"agent:analyze:progress-completed",
|
||
);
|
||
} catch (directAnalyzeError) {
|
||
chatApiLogger.error("[Chat API] Direct agent image analysis failed:", directAnalyzeError);
|
||
const errorMessage = getDirectAgentToolErrorMessage("analyze", directAnalyzeError);
|
||
|
||
await saveAssistantTextOnlyMessage({
|
||
chatId: id,
|
||
messageId: analysisAssistantMessageId,
|
||
text: errorMessage,
|
||
});
|
||
|
||
writeAssistantTextChunk(
|
||
writer,
|
||
errorMessage,
|
||
"agent:analyze:error",
|
||
analysisAssistantMessageId,
|
||
);
|
||
writeAgentProgressChunk(
|
||
writer,
|
||
buildDirectAgentAnalyzeFailedProgress(errorMessage),
|
||
"agent:analyze:progress-failed",
|
||
);
|
||
}
|
||
},
|
||
});
|
||
|
||
return createSseResponse(directAnalyzeStream);
|
||
}
|
||
|
||
if (
|
||
shouldRunAgentConversation &&
|
||
!smartAutoNeedsClarification &&
|
||
directAgentExecution?.toolName === "generateImage"
|
||
) {
|
||
const directImageStream = createUIMessageStream({
|
||
execute: async ({ writer }) => {
|
||
writeAgentProgressChunk(
|
||
writer,
|
||
buildDirectAgentImageContextProgress({
|
||
imageCount: directAgentToolTargetImageUrls.length,
|
||
mode: "generate",
|
||
}),
|
||
"agent:generate_image:progress-image-context",
|
||
);
|
||
writeAgentProgressChunk(
|
||
writer,
|
||
buildDirectAgentImageGenerateSelectProgress(),
|
||
"agent:generate_image:progress-select-tool",
|
||
);
|
||
writeCurrentAgentTargetLockChunk(writer, "agent:generate_image:target-lock");
|
||
writeAssistantTextChunk(
|
||
writer,
|
||
"<!-- IMAGE_GENERATION_START -->",
|
||
"agent:generate_image:start",
|
||
);
|
||
writeAgentProgressChunk(
|
||
writer,
|
||
buildDirectAgentImageGenerateExecutingProgress(),
|
||
"agent:generate_image:progress-executing",
|
||
);
|
||
if (contextPayload?.source === "canvas") {
|
||
writeImageGenerationStatusChunk(
|
||
writer,
|
||
{
|
||
phase: "processing",
|
||
providerType: requestedModelProviderType,
|
||
modelName: finalModelId,
|
||
imageCount:
|
||
effectiveImageParams?.imageCount ||
|
||
defaultImageQualityParams.imageCount ||
|
||
1,
|
||
},
|
||
"agent:generate_image:status-processing",
|
||
);
|
||
}
|
||
|
||
const assistantMessageId = generateUUID();
|
||
try {
|
||
const directImageResult = await executeDirectAgentImageGeneration({
|
||
requestTextContent,
|
||
optimizedPrompt: smartAutoOptimizedPrompt,
|
||
userId: session.user.id,
|
||
finalModelId,
|
||
imageParams: effectiveImageParams,
|
||
defaultImageQualityParams,
|
||
referenceImageUrls: directAgentToolTargetImageUrls,
|
||
});
|
||
const assistantText = buildDirectAgentImageGenerationAssistantText({
|
||
imageMarkdown: directImageResult.imageMarkdown,
|
||
modelDisplayName: directImageResult.model,
|
||
});
|
||
|
||
await saveAssistantTextOnlyMessage({
|
||
chatId: id,
|
||
messageId: assistantMessageId,
|
||
text: assistantText,
|
||
});
|
||
|
||
await saveGeneratedImages(
|
||
assistantText,
|
||
session.user.id,
|
||
id,
|
||
assistantMessageId,
|
||
directImageResult.prompt,
|
||
directImageResult.modelRef || finalModelId,
|
||
{
|
||
sourceImageUrls: directAgentToolTargetImageUrls,
|
||
relationType: "generated_from",
|
||
sourceType:
|
||
directAgentToolTargetImageUrls.length > 0
|
||
? "reference_generate"
|
||
: "generated",
|
||
projectId: contextPayload?.projectId || null,
|
||
userPrompt: requestTextContent,
|
||
optimizedPrompt: directImageResult.prompt,
|
||
requestedImageCount:
|
||
effectiveImageParams?.imageCount ||
|
||
defaultImageQualityParams.imageCount ||
|
||
1,
|
||
aspectRatio:
|
||
effectiveImageParams?.aspectRatio ||
|
||
defaultImageQualityParams.aspectRatio,
|
||
imageSize:
|
||
effectiveImageParams?.imageSize ||
|
||
defaultImageQualityParams.imageSize,
|
||
},
|
||
);
|
||
|
||
writeAssistantTextChunk(
|
||
writer,
|
||
assistantText,
|
||
"agent:generate_image:result",
|
||
assistantMessageId,
|
||
);
|
||
if (contextPayload?.source === "canvas") {
|
||
writeCanvasMediaChunk(
|
||
writer,
|
||
{
|
||
messageId: assistantMessageId,
|
||
imageUrls: directImageResult.images.map((image) => image.originalUrl),
|
||
prompt: directImageResult.prompt,
|
||
sourceImageUrls: directAgentToolTargetImageUrls,
|
||
},
|
||
"agent:generate_image:canvas-media",
|
||
);
|
||
writeImageGenerationStatusChunk(
|
||
writer,
|
||
{
|
||
phase: "completed",
|
||
providerType: requestedModelProviderType,
|
||
modelName: directImageResult.model,
|
||
imageCount: directImageResult.imageCount,
|
||
},
|
||
"agent:generate_image:status-completed",
|
||
);
|
||
}
|
||
writeAgentProgressChunk(
|
||
writer,
|
||
buildDirectAgentImageGenerateCompletedProgress(),
|
||
"agent:generate_image:progress-completed",
|
||
);
|
||
writeAssistantTextChunk(
|
||
writer,
|
||
"<!-- IMAGE_GENERATION_END -->",
|
||
"agent:generate_image:end",
|
||
);
|
||
} catch (directToolError) {
|
||
chatApiLogger.error("[Chat API] Direct agent image generation failed:", directToolError);
|
||
const errorMessage = getDirectAgentToolErrorMessage(
|
||
"generate_image",
|
||
directToolError,
|
||
);
|
||
await saveAssistantTextOnlyMessage({
|
||
chatId: id,
|
||
messageId: assistantMessageId,
|
||
text: errorMessage,
|
||
});
|
||
writeAssistantTextChunk(
|
||
writer,
|
||
errorMessage,
|
||
"agent:generate_image:error",
|
||
assistantMessageId,
|
||
);
|
||
if (contextPayload?.source === "canvas") {
|
||
writeImageGenerationStatusChunk(
|
||
writer,
|
||
{
|
||
phase: "failed",
|
||
providerType: requestedModelProviderType,
|
||
modelName: finalModelId,
|
||
},
|
||
"agent:generate_image:status-failed",
|
||
);
|
||
}
|
||
writeAgentProgressChunk(
|
||
writer,
|
||
buildDirectAgentImageGenerateFailedProgress(errorMessage),
|
||
"agent:generate_image:progress-failed",
|
||
);
|
||
writeAssistantTextChunk(
|
||
writer,
|
||
"<!-- IMAGE_GENERATION_END -->",
|
||
"agent:generate_image:end-error",
|
||
);
|
||
}
|
||
},
|
||
});
|
||
|
||
return createSseResponse(directImageStream);
|
||
}
|
||
|
||
if (
|
||
shouldRunAgentConversation &&
|
||
!smartAutoNeedsClarification &&
|
||
directAgentExecution?.toolName === "generateVideo"
|
||
) {
|
||
const directVideoStream = createUIMessageStream({
|
||
execute: async ({ writer }) => {
|
||
const assistantMessageId = generateUUID();
|
||
writeAgentProgressChunk(
|
||
writer,
|
||
buildDirectAgentImageContextProgress({
|
||
imageCount: directAgentToolTargetImageUrls.length,
|
||
mode: "video",
|
||
}),
|
||
"agent:generate_video:progress-image-context",
|
||
);
|
||
writeAgentProgressChunk(
|
||
writer,
|
||
buildDirectAgentVideoGenerateSelectProgress(),
|
||
"agent:generate_video:progress-select-tool",
|
||
);
|
||
writeCurrentAgentTargetLockChunk(writer, "agent:generate_video:target-lock");
|
||
try {
|
||
writeAgentProgressChunk(
|
||
writer,
|
||
buildDirectAgentVideoGenerateExecutingProgress(),
|
||
"agent:generate_video:progress-executing",
|
||
);
|
||
const directVideoResult = await executeDirectAgentVideoGeneration({
|
||
requestTextContent,
|
||
optimizedPrompt: smartAutoOptimizedPrompt,
|
||
finalModelId,
|
||
sourceImageUrls: directAgentToolTargetImageUrls,
|
||
videoParams: effectiveVideoParams,
|
||
smartAutoExecution,
|
||
smartAutoPlan,
|
||
});
|
||
const { mode, result, videoUrl } = directVideoResult;
|
||
|
||
const assistantText = buildDirectAgentVideoGenerationAssistantText({
|
||
videoUrl,
|
||
modelDisplayName: result.modelDisplayName,
|
||
});
|
||
|
||
await saveAssistantTextOnlyMessage({
|
||
chatId: id,
|
||
messageId: assistantMessageId,
|
||
text: assistantText,
|
||
videoStorageStatus: "temporary",
|
||
temporaryVideoUrl: videoUrl,
|
||
videoMetadata: (result.metadata || {}) as any,
|
||
});
|
||
|
||
await safePersistDirectAgentVideoCompletedTurn({
|
||
userId: session.user.id,
|
||
chatId: id,
|
||
projectId: contextPayload?.projectId || null,
|
||
messageId: assistantMessageId,
|
||
requestTextContent,
|
||
modelRef: result.modelRef || finalModelId,
|
||
mode,
|
||
temporaryVideoUrl: videoUrl,
|
||
});
|
||
|
||
await consumeGeneratedVideoPoints({
|
||
userId: session.user.id,
|
||
chatId: id,
|
||
messageId: assistantMessageId,
|
||
modelRefs: [result.modelRef || finalModelId],
|
||
duration:
|
||
Number((result.metadata as any)?.duration) ||
|
||
Number(effectiveVideoParams?.duration) ||
|
||
undefined,
|
||
resolution:
|
||
String(
|
||
(result.metadata as any)?.resolution ||
|
||
effectiveVideoParams?.resolution ||
|
||
""
|
||
).trim() || undefined,
|
||
});
|
||
|
||
if (videoUrl) {
|
||
scheduleVideoArchive({
|
||
chatId: id,
|
||
messageId: assistantMessageId,
|
||
text: assistantText,
|
||
temporaryVideoUrl: videoUrl,
|
||
taskId: assistantMessageId,
|
||
metadata: result.metadata,
|
||
});
|
||
}
|
||
|
||
writeAssistantTextChunk(
|
||
writer,
|
||
assistantText,
|
||
"agent:generate_video:result",
|
||
assistantMessageId,
|
||
);
|
||
if (contextPayload?.source === "canvas") {
|
||
writeCanvasMediaChunk(
|
||
writer,
|
||
{
|
||
messageId: assistantMessageId,
|
||
videoUrls: [videoUrl],
|
||
prompt: directVideoResult.prompt,
|
||
sourceImageUrls: directAgentToolTargetImageUrls,
|
||
},
|
||
"agent:generate_video:canvas-media",
|
||
);
|
||
}
|
||
writeAgentProgressChunk(
|
||
writer,
|
||
buildDirectAgentVideoGenerateCompletedProgress(),
|
||
"agent:generate_video:progress-completed",
|
||
);
|
||
} catch (directToolError) {
|
||
chatApiLogger.error("[Chat API] Direct agent video generation failed:", directToolError);
|
||
const errorMessage = getDirectAgentToolErrorMessage(
|
||
"generate_video",
|
||
directToolError,
|
||
);
|
||
await saveAssistantTextOnlyMessage({
|
||
chatId: id,
|
||
messageId: assistantMessageId,
|
||
text: errorMessage,
|
||
});
|
||
await safePersistDirectAgentVideoFailedTurn({
|
||
userId: session.user.id,
|
||
chatId: id,
|
||
projectId: contextPayload?.projectId || null,
|
||
messageId: assistantMessageId,
|
||
requestTextContent,
|
||
modelRef: finalModelId,
|
||
error: directToolError,
|
||
});
|
||
writeAssistantTextChunk(
|
||
writer,
|
||
errorMessage,
|
||
"agent:generate_video:error",
|
||
assistantMessageId,
|
||
);
|
||
writeAgentProgressChunk(
|
||
writer,
|
||
buildDirectAgentVideoGenerateFailedProgress(errorMessage),
|
||
"agent:generate_video:progress-failed",
|
||
);
|
||
}
|
||
},
|
||
});
|
||
|
||
return createSseResponse(directVideoStream);
|
||
}
|
||
|
||
if (
|
||
shouldRunAgentConversation &&
|
||
!smartAutoNeedsClarification &&
|
||
directAgentExecution &&
|
||
isDirectAgentImageEditToolName(directAgentExecution.toolName)
|
||
) {
|
||
const directAgentToolCapability = directAgentExecution.capability;
|
||
if (!directAgentToolTargetImageUrl) {
|
||
const missingTargetMessage = getDirectAgentToolTargetMissingMessage(
|
||
directAgentToolCapability,
|
||
);
|
||
const missingTargetStream = createUIMessageStream({
|
||
execute: async ({ writer }) => {
|
||
const messageId = generateUUID();
|
||
writeCurrentAgentTargetLockChunk(
|
||
writer,
|
||
`agent:${directAgentToolCapability}:missing-target-lock`,
|
||
);
|
||
writeAssistantTextChunk(
|
||
writer,
|
||
missingTargetMessage,
|
||
`agent:${directAgentToolCapability}:missing-target`,
|
||
messageId,
|
||
);
|
||
|
||
await saveAssistantTextOnlyMessage({
|
||
chatId: id,
|
||
messageId,
|
||
text: missingTargetMessage,
|
||
});
|
||
},
|
||
});
|
||
|
||
return createSseResponse(missingTargetStream);
|
||
}
|
||
|
||
const assistantMessageId = generateUUID();
|
||
const directToolLabel = getDirectAgentToolLabel(directAgentToolCapability);
|
||
const directToolStream = createUIMessageStream({
|
||
execute: async ({ writer }) => {
|
||
writeAgentProgressChunk(
|
||
writer,
|
||
buildDirectAgentImageContextProgress({
|
||
imageCount: directAgentToolTargetImageUrls.length,
|
||
mode: "edit",
|
||
}),
|
||
`agent:${directAgentToolCapability}:progress-image-context`,
|
||
);
|
||
writeAgentProgressChunk(
|
||
writer,
|
||
buildDirectAgentImageEditSelectProgress({
|
||
toolName: directAgentExecution.toolName,
|
||
capability: directAgentToolCapability,
|
||
label: directToolLabel,
|
||
}),
|
||
`agent:${directAgentToolCapability}:progress-select-tool`,
|
||
);
|
||
writeCurrentAgentTargetLockChunk(
|
||
writer,
|
||
`agent:${directAgentToolCapability}:target-lock`,
|
||
);
|
||
writeAgentProgressChunk(
|
||
writer,
|
||
buildDirectAgentImageEditExecutingProgress({
|
||
toolName: directAgentExecution.toolName,
|
||
capability: directAgentToolCapability,
|
||
label: directToolLabel,
|
||
}),
|
||
`agent:${directAgentToolCapability}:progress-executing`,
|
||
);
|
||
|
||
try {
|
||
const directImageEditResult = await executeDirectAgentImageEditTool({
|
||
capability: directAgentToolCapability,
|
||
requestUrl: request.url,
|
||
requestHeaders: request.headers,
|
||
userId: session.user.id,
|
||
chatId: id,
|
||
messageId: assistantMessageId,
|
||
projectId: contextPayload?.projectId || null,
|
||
targetAssetIds: requestTargetResolution?.targetAssetIds || null,
|
||
sourceImageUrl: directAgentToolTargetImageUrl,
|
||
sourceImageUrls: directAgentToolTargetImageUrls,
|
||
requestTextContent,
|
||
optimizedPrompt: smartAutoOptimizedPrompt,
|
||
hiddenPromptPrefix,
|
||
finalModelId,
|
||
imageParams: effectiveImageParams,
|
||
defaultImageQualityParams,
|
||
smartAutoExecution,
|
||
smartAutoPlan,
|
||
saveGeneratedImages,
|
||
});
|
||
const resultOriginalUrl = directImageEditResult.originalUrl;
|
||
const resultThumbnailUrl = directImageEditResult.thumbnailUrl;
|
||
const resultImageSize = directImageEditResult.imageSize;
|
||
const resultAssetId = directImageEditResult.mediaAssetId;
|
||
|
||
const assistantText = buildDirectAgentImageEditAssistantText({
|
||
capability: directAgentToolCapability,
|
||
originalUrl: resultOriginalUrl,
|
||
thumbnailUrl: resultThumbnailUrl,
|
||
imageSize: resultImageSize,
|
||
});
|
||
|
||
await saveAssistantTextOnlyMessage({
|
||
chatId: id,
|
||
messageId: assistantMessageId,
|
||
text: assistantText,
|
||
});
|
||
|
||
await safePersistDirectAgentImageEditTurn({
|
||
userId: session.user.id,
|
||
chatId: id,
|
||
projectId: contextPayload?.projectId || null,
|
||
capability: directAgentToolCapability,
|
||
currentSubIntent: smartAutoExecution?.routeSubIntent || directAgentToolCapability,
|
||
resultAssetId,
|
||
resultOriginalUrl,
|
||
resultImageSize,
|
||
sourceImageUrls: directAgentToolTargetImageUrls,
|
||
messageId: assistantMessageId,
|
||
requestTextContent,
|
||
optimizedPrompt: smartAutoOptimizedPrompt || undefined,
|
||
});
|
||
|
||
if (directAgentExecution.publishToCanvas && contextPayload?.source === "canvas") {
|
||
chatApiLogger.info(`[Chat API] Agent publishToCanvas satisfied by assistant media auto-sync (${directAgentToolCapability})`);
|
||
}
|
||
|
||
writeAssistantTextChunk(
|
||
writer,
|
||
assistantText,
|
||
`agent:${directAgentToolCapability}:success`,
|
||
assistantMessageId,
|
||
);
|
||
if (contextPayload?.source === "canvas") {
|
||
writeCanvasMediaChunk(
|
||
writer,
|
||
{
|
||
messageId: assistantMessageId,
|
||
imageUrls: resultOriginalUrl ? [resultOriginalUrl] : [],
|
||
prompt: requestTextContent || directToolLabel,
|
||
sourceImageUrls: directAgentToolTargetImageUrls,
|
||
},
|
||
`agent:${directAgentToolCapability}:canvas-media`,
|
||
);
|
||
}
|
||
writeAgentProgressChunk(
|
||
writer,
|
||
buildDirectAgentImageEditCompletedProgress({
|
||
toolName: directAgentExecution.toolName,
|
||
capability: directAgentToolCapability,
|
||
label: directToolLabel,
|
||
}),
|
||
`agent:${directAgentToolCapability}:progress-completed`,
|
||
);
|
||
} catch (directToolError) {
|
||
chatApiLogger.error(`[Chat API] Direct agent tool ${directAgentToolCapability} failed:`, directToolError);
|
||
|
||
if (!isDirectAgentImageEditServiceError(directToolError)) {
|
||
chatApiLogger.warn(`[Chat API] Direct agent tool ${directAgentToolCapability} threw unexpected error type`);
|
||
}
|
||
|
||
const errorMessage = getDirectAgentToolErrorMessage(
|
||
directAgentToolCapability,
|
||
directToolError,
|
||
);
|
||
|
||
await saveAssistantTextOnlyMessage({
|
||
chatId: id,
|
||
messageId: assistantMessageId,
|
||
text: errorMessage,
|
||
});
|
||
|
||
writeAssistantTextChunk(
|
||
writer,
|
||
errorMessage,
|
||
`agent:${directAgentToolCapability}:error`,
|
||
assistantMessageId,
|
||
);
|
||
writeAgentProgressChunk(
|
||
writer,
|
||
buildDirectAgentImageEditFailedProgress({
|
||
toolName: directAgentExecution.toolName,
|
||
capability: directAgentToolCapability,
|
||
label: directToolLabel,
|
||
errorMessage,
|
||
}),
|
||
`agent:${directAgentToolCapability}:progress-failed`,
|
||
);
|
||
}
|
||
},
|
||
});
|
||
|
||
return createSseResponse(directToolStream);
|
||
}
|
||
|
||
let finalMergedUsage: AppUsage | undefined;
|
||
|
||
// 获取启用的工具列表
|
||
const enabledTools = await getEnabledTools();
|
||
|
||
if (String(finalModelId || "").trim() === "auto") {
|
||
const hasImageInputForAutoFallback =
|
||
message.parts.some((part: any) =>
|
||
part?.type === "file" || part?.type === "image" || part?.type === "image_url"
|
||
) || smartAutoHistoryImages.length > 0;
|
||
const autoFallbackIntent =
|
||
smartAutoExecution?.shouldApplyVideoPipeline || composerMode === "video"
|
||
? "video_gen"
|
||
: smartAutoExecution?.shouldApplyImagePipeline ||
|
||
composerMode === "image" ||
|
||
composerMode === "detail"
|
||
? hasImageInputForAutoFallback
|
||
? "image_edit"
|
||
: "image_gen"
|
||
: hasImageInputForAutoFallback
|
||
? "vision"
|
||
: "chat";
|
||
|
||
try {
|
||
const { routeToModel } = await import("@/lib/ai/model-router");
|
||
const fallbackRoute = await routeToModel({
|
||
intent: autoFallbackIntent,
|
||
hasImage: hasImageInputForAutoFallback,
|
||
});
|
||
finalModelId = fallbackRoute.id;
|
||
chatApiLogger.warn("[Chat API] Auto model was unresolved, routed to fallback model:", {
|
||
intent: autoFallbackIntent,
|
||
modelId: fallbackRoute.id,
|
||
displayName: fallbackRoute.displayName,
|
||
reason: fallbackRoute.reason,
|
||
});
|
||
} catch (autoModelRouteError) {
|
||
chatApiLogger.error("[Chat API] Failed to route unresolved auto model:", autoModelRouteError);
|
||
const enabledModels = await getEnabledAiModelsWithProvider(false);
|
||
const nonAgentModels = enabledModels.filter(({ model }) => !isAiModelAgentAuto(model));
|
||
const candidates = nonAgentModels.length > 0 ? nonAgentModels : enabledModels;
|
||
const fallbackModel =
|
||
candidates.find(({ model, provider }) => {
|
||
if (!model.isEnabled || model.isSubModel) return false;
|
||
const providerType = String(provider?.type || "");
|
||
if (autoFallbackIntent === "video_gen") {
|
||
return isVideoProviderType(providerType, model);
|
||
}
|
||
if (autoFallbackIntent === "image_edit" || autoFallbackIntent === "image_gen") {
|
||
return isImageGenerationModelType(providerType, model);
|
||
}
|
||
return !isVideoProviderType(providerType, model) && !model.supportsImageGen;
|
||
}) ||
|
||
candidates.find(
|
||
({ model, provider }) =>
|
||
model.isEnabled &&
|
||
!model.isSubModel &&
|
||
!isVideoProviderType(String(provider?.type || ""), model)
|
||
) ||
|
||
candidates.find(({ model }) => model.isEnabled && !model.isSubModel);
|
||
|
||
if (!fallbackModel) {
|
||
return new ChatSDKError("bad_request:api", "No available models for auto mode").toResponse();
|
||
}
|
||
|
||
finalModelId = fallbackModel.model.id;
|
||
chatApiLogger.warn("[Chat API] Auto model fallback used first available model:", {
|
||
intent: autoFallbackIntent,
|
||
modelId: fallbackModel.model.id,
|
||
displayName: fallbackModel.model.displayName,
|
||
providerType: fallbackModel.provider?.type,
|
||
});
|
||
}
|
||
}
|
||
|
||
// 检查是否是 UUID 格式（数据库中的模型 ID）
|
||
const isUUID =
|
||
/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test(
|
||
finalModelId
|
||
);
|
||
|
||
// 获取语言模型实例
|
||
let languageModel;
|
||
let actualModelId: string;
|
||
let resolvedModelSupportsImageGen = false;
|
||
let resolvedProviderType: string | undefined;
|
||
let resolvedRuntimeModelLookupId = finalModelId;
|
||
|
||
if (isUUID) {
|
||
// 从数据库获取动态配置的模型
|
||
let targetModelId = finalModelId;
|
||
|
||
// 🆕 检查是否是视频主模型，如果是，根据 videoParams.mode 选择子模型
|
||
const aiModelData = await getAiModelById({ id: finalModelId });
|
||
const aiModelProvider = aiModelData
|
||
? await getAiProviderById({ id: aiModelData.providerId }).catch(() => null)
|
||
: null;
|
||
chatApiLogger.info("[Chat API] Model data:", { id: finalModelId, isSubModel: aiModelData?.isSubModel });
|
||
|
||
if (aiModelData && !aiModelData.isSubModel && aiModelProvider?.type === "volcano-video") {
|
||
// 检查是否有子模型
|
||
const subModels = await getSubModelsByParentModelId(finalModelId);
|
||
chatApiLogger.info("[Chat API] Found sub-models:", subModels.length);
|
||
|
||
if (subModels.length > 0) {
|
||
// 这是一个主模型，需要根据 videoParams 选择子模型
|
||
const videoParams = effectiveVideoParams;
|
||
chatApiLogger.info("[Chat API] Video params:", videoParams);
|
||
|
||
// 如果没有 videoParams 或没有 mode，使用默认模式
|
||
const mode = (videoParams?.mode || "text-to-video") as VideoModeType;
|
||
chatApiLogger.info("[Chat API] Using mode:", mode);
|
||
|
||
const matchedSubModel = selectBestVideoSubModelForMode(subModels as any, mode);
|
||
|
||
if (matchedSubModel) {
|
||
chatApiLogger.info(`[Chat API] 视频模式 ${mode} → 选择子模型: ${matchedSubModel.model.displayName} (${matchedSubModel.model.modelId})`);
|
||
targetModelId = matchedSubModel.model.id;
|
||
} else {
|
||
chatApiLogger.warn(`[Chat API] 未找到可用子模型，继续使用主模型`);
|
||
}
|
||
}
|
||
}
|
||
|
||
const dynamicModel = await getLanguageModelById(targetModelId);
|
||
if (!dynamicModel) {
|
||
return new ChatSDKError(
|
||
"bad_request:api",
|
||
"Model not found or disabled"
|
||
).toResponse();
|
||
}
|
||
languageModel = dynamicModel;
|
||
resolvedRuntimeModelLookupId = targetModelId;
|
||
// 对于动态模型，我们需要从数据库获取 modelId
|
||
const finalModelData = await getAiModelById({ id: targetModelId });
|
||
actualModelId = finalModelData?.modelId ?? targetModelId;
|
||
resolvedModelSupportsImageGen = Boolean(finalModelData?.supportsImageGen);
|
||
if (finalModelData) {
|
||
resolvedProviderType = (
|
||
await getAiProviderById({ id: finalModelData.providerId }).catch(() => null)
|
||
)?.type;
|
||
}
|
||
} else {
|
||
// 使用预定义的模型
|
||
languageModel = myProvider.languageModel(finalModelId);
|
||
actualModelId = languageModel.modelId;
|
||
resolvedProviderType = (languageModel as any)?.provider;
|
||
resolvedRuntimeModelLookupId = actualModelId;
|
||
}
|
||
|
||
const effectiveConversationSystemPrompt = shouldRunAgentConversation
|
||
? `${await buildAgentRoleIdentityInstruction({
|
||
allowVideoGeneration: allowAgentVideoGeneration,
|
||
hasCanvasContext: Boolean(
|
||
effectiveContextPayload?.source === "canvas" ||
|
||
effectiveContextPayload?.selectedNodeIds?.length ||
|
||
effectiveContextPayload?.canvasImageNodes?.length,
|
||
),
|
||
})}\n\n${systemPrompt({ selectedChatModel: finalModelId, requestHints })}`
|
||
: systemPrompt({ selectedChatModel: finalModelId, requestHints });
|
||
|
||
const activeRuntimeModelDescriptor = await resolveAgentModelSlotDescriptor(
|
||
resolvedRuntimeModelLookupId,
|
||
).catch(() => null);
|
||
if (shouldRunAgentConversation && activeRuntimeModelDescriptor) {
|
||
const slotName =
|
||
smartAutoExecution?.shouldApplyImagePipeline ||
|
||
smartAutoExecution?.shouldApplyVideoPipeline
|
||
? "execution"
|
||
: "reply";
|
||
agentModelSlots = upsertAgentModelSlot(agentModelSlots, slotName, {
|
||
...activeRuntimeModelDescriptor,
|
||
source: "runtime_selected_model",
|
||
reason:
|
||
slotName === "reply"
|
||
? smartAutoExecution?.shouldStayConversational
|
||
? "conversational_reply_runtime"
|
||
: "default_reply_runtime"
|
||
: "execution_runtime_model",
|
||
locked: slotName === "reply" ? shouldKeepSelectedModel : false,
|
||
});
|
||
pendingProtocolDebugEvents.push({
|
||
phase: "modelSlots",
|
||
payload: {
|
||
modelSlots: agentModelSlots,
|
||
providerHealth: activeRuntimeModelDescriptor.providerKey
|
||
? getProviderHealthSnapshot(activeRuntimeModelDescriptor.providerKey)
|
||
: null,
|
||
},
|
||
});
|
||
}
|
||
|
||
// ========== 图片生成配额检查 ==========
|
||
const isUuidModelId = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test(
|
||
finalModelId
|
||
);
|
||
const currentModelConfig = isUuidModelId
|
||
? await getAiModelById({ id: finalModelId }).catch(() => undefined)
|
||
: undefined;
|
||
const currentProviderConfig =
|
||
currentModelConfig?.providerId
|
||
? await getAiProviderById({ id: currentModelConfig.providerId }).catch(() => null)
|
||
: null;
|
||
const runtimeProviderType = String(
|
||
resolvedProviderType || (languageModel as any).provider || ""
|
||
);
|
||
const normalizedRuntimeProviderType =
|
||
runtimeProviderType === "gemini-image"
|
||
? "gemini"
|
||
: runtimeProviderType === "huanke-gemini"
|
||
? "huanke-gemini"
|
||
: runtimeProviderType;
|
||
const isImageGenerationModel = isImageGenerationModelType(
|
||
normalizedRuntimeProviderType,
|
||
currentModelConfig
|
||
? {
|
||
modelId: currentModelConfig.modelId,
|
||
capabilities: currentModelConfig.capabilities,
|
||
supportsImageGen: currentModelConfig.supportsImageGen,
|
||
}
|
||
: undefined
|
||
);
|
||
const modelCreditCostPerImage = getModelCreditCost(currentModelConfig, "image", {
|
||
imageSize:
|
||
typeof effectiveImageParams?.imageSize === "string"
|
||
? effectiveImageParams.imageSize
|
||
: defaultImageQualityParams.imageSize,
|
||
});
|
||
const modelCreditCostPerVideo = getModelCreditCost(currentModelConfig, "video", {
|
||
duration: Number(effectiveVideoParams?.duration) || undefined,
|
||
resolution: String(effectiveVideoParams?.resolution || "").trim() || undefined,
|
||
});
|
||
|
||
if (isImageGenerationModel) {
|
||
const quota = await checkUserImageQuota(session.user.id);
|
||
|
||
if (!quota.allowed) {
|
||
chatApiLogger.info(`[Chat API] ⛔ 图片配额已用完: ${quota.used}/${quota.limit}`);
|
||
return NextResponse.json(
|
||
{
|
||
error: "quota_exceeded",
|
||
message: `本月图片生成配额已用完（已使用 ${quota.used}/${quota.limit} 张），配额将于下月1日重置，如需更多配额请联系管理员。`,
|
||
used: quota.used,
|
||
limit: quota.limit,
|
||
},
|
||
{ status: 429 }
|
||
);
|
||
}
|
||
|
||
chatApiLogger.info(`[Chat API] ✅ 图片配额检查通过: ${quota.used}/${quota.limit ?? '无限制'}`);
|
||
|
||
const pointsConfig = await getPointsSystemConfig();
|
||
const normalizedUnitCost = Number.isFinite(modelCreditCostPerImage)
|
||
? Math.max(0, Math.round(modelCreditCostPerImage))
|
||
: 0;
|
||
|
||
if (pointsConfig.enabled && normalizedUnitCost > 0 && pointsConfig.blockOnInsufficient !== false) {
|
||
const requestedImageCount = Math.max(
|
||
1,
|
||
Math.round(
|
||
Number(
|
||
effectiveImageParams?.imageCount || defaultImageQualityParams.imageCount || 1
|
||
)
|
||
)
|
||
);
|
||
const expectedCost = normalizedUnitCost * requestedImageCount;
|
||
const userData = await getUserById({ id: session.user.id });
|
||
if (!userData?.isHighVip) {
|
||
const currentBalance = Math.max(0, Number(userData?.pointsBalance || 0));
|
||
|
||
if (currentBalance < expectedCost) {
|
||
return NextResponse.json(
|
||
{
|
||
error: "insufficient_points",
|
||
message: `积分不足，当前余额 ${currentBalance}，本次预计消耗 ${expectedCost}（${normalizedUnitCost} x ${requestedImageCount} 张）。`,
|
||
balance: currentBalance,
|
||
expectedCost,
|
||
unitCost: normalizedUnitCost,
|
||
},
|
||
{ status: 402 }
|
||
);
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
const isVideoGenerationModel = isVideoProviderType(
|
||
normalizedRuntimeProviderType,
|
||
currentModelConfig
|
||
? {
|
||
modelId: currentModelConfig.modelId,
|
||
capabilities: currentModelConfig.capabilities,
|
||
supportsImageGen: currentModelConfig.supportsImageGen,
|
||
}
|
||
: undefined
|
||
);
|
||
|
||
if (isVideoGenerationModel) {
|
||
const pointsConfig = await getPointsSystemConfig();
|
||
const normalizedVideoUnitCost = Number.isFinite(modelCreditCostPerVideo)
|
||
? Math.max(0, Math.round(modelCreditCostPerVideo))
|
||
: 0;
|
||
|
||
if (
|
||
pointsConfig.enabled &&
|
||
normalizedVideoUnitCost > 0 &&
|
||
pointsConfig.blockOnInsufficient !== false
|
||
) {
|
||
const requestedVideoDuration = Math.max(
|
||
1,
|
||
Math.round(Number(effectiveVideoParams?.duration || 0)) || 0
|
||
);
|
||
const expectedCost = normalizedVideoUnitCost;
|
||
const userData = await getUserById({ id: session.user.id });
|
||
if (!userData?.isHighVip) {
|
||
const currentBalance = Math.max(0, Number(userData?.pointsBalance || 0));
|
||
|
||
if (currentBalance < expectedCost) {
|
||
return NextResponse.json(
|
||
{
|
||
error: "insufficient_points",
|
||
message: requestedVideoDuration
|
||
? `积分不足，当前余额 ${currentBalance}，本次预计消耗 ${expectedCost}（${requestedVideoDuration} 秒视频）。`
|
||
: `积分不足，当前余额 ${currentBalance}，本次预计消耗 ${expectedCost}（1 条视频）。`,
|
||
balance: currentBalance,
|
||
expectedCost,
|
||
unitCost: normalizedVideoUnitCost,
|
||
duration: requestedVideoDuration || undefined,
|
||
},
|
||
{ status: 402 }
|
||
);
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
// ========== Gemini 图片生成独立处理 ==========
|
||
// 支持 gemini-image 和 huanke-gemini 两种图片生成 provider
|
||
const isGeminiImageProvider = ["gemini-image", "huanke-gemini"].includes((languageModel as any).provider);
|
||
if (isGeminiImageProvider && typeof (languageModel as any).doGenerate === "function") {
|
||
chatApiLogger.info("[Chat API] Using Gemini image generation, provider:", (languageModel as any).provider);
|
||
chatApiLogger.info("[Chat API] Image params:", effectiveImageParams);
|
||
|
||
// 🆕 获取备用模型配置
|
||
const currentModelConfig = await getAiModelById({ id: finalModelId });
|
||
const configuredFallbacks = [
|
||
currentModelConfig?.fallbackModelId1,
|
||
currentModelConfig?.fallbackModelId2,
|
||
].filter(Boolean) as string[];
|
||
const allModels = await getEnabledAiModelsWithProvider();
|
||
const currentModelEntry = resolveModelEntryByRuntimeModelId(
|
||
allModels,
|
||
actualModelId,
|
||
finalModelId,
|
||
);
|
||
const fallbackCandidates = buildImageFallbackCandidates(allModels, {
|
||
currentProviderId: currentModelConfig?.providerId,
|
||
currentModelId: currentModelConfig?.id,
|
||
currentModelKey: actualModelId,
|
||
configuredFallbackIds: configuredFallbacks,
|
||
});
|
||
chatApiLogger.info("[Chat API] Gemini fallback candidates:", fallbackCandidates.map((candidate) => `${candidate.model.displayName}(${candidate.provider.type})`));
|
||
|
||
const geminiStream = createUIMessageStream({
|
||
execute: async ({ writer }) => {
|
||
// 🆕 如果是智能模式且有对话回复，先发送对话消息让用户看到
|
||
// 在消息末尾加上特殊标记，告诉前端"图片正在生成中"
|
||
const conversationMessage = smartAutoFriendlyMessage || null;
|
||
|
||
if (conversationMessage) {
|
||
const chatMessageId = generateUUID();
|
||
chatApiLogger.info("[Chat API] 💬 Gemini: Sending conversational reply first:", smartAutoFriendlyMessage);
|
||
|
||
// 发送带标记的对话消息
|
||
writeAssistantTextChunk(
|
||
writer,
|
||
conversationMessage,
|
||
"gemini:conversation",
|
||
chatMessageId,
|
||
);
|
||
|
||
// 保存对话消息到数据库（不带标记，纯对话）
|
||
await saveMessages({
|
||
messages: [
|
||
{
|
||
id: chatMessageId,
|
||
chatId: id,
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: smartAutoFriendlyMessage }],
|
||
attachments: [],
|
||
createdAt: new Date(),
|
||
videoStorageStatus: null,
|
||
temporaryVideoUrl: null,
|
||
permanentVideoUrl: null,
|
||
archivedAt: null,
|
||
videoMetadata: null,
|
||
},
|
||
],
|
||
});
|
||
}
|
||
|
||
const buildMessagesForGemini = () => {
|
||
const lastUserMessage = uiMessagesForModel.filter((m) => m.role === "user").pop();
|
||
let messagesForGemini = lastUserMessage ? [lastUserMessage] : uiMessagesForModel;
|
||
messagesForGemini = applyOptimizedPromptToLatestUserMessage(
|
||
messagesForGemini,
|
||
smartAutoOptimizedPrompt,
|
||
hiddenPromptPrefix,
|
||
);
|
||
return injectHistoryImagesToLastUserMessage(messagesForGemini, smartAutoHistoryImages);
|
||
};
|
||
|
||
// 🆕 尝试生成图片的函数（支持备用模型切换）
|
||
const tryGenerateImage = async (modelIdToUse: string, isRetry = false): Promise<string> => {
|
||
chatApiLogger.info(`[Chat API] ${isRetry ? "🔄 Retry with fallback" : "🚀 Trying"} model: ${modelIdToUse}`);
|
||
|
||
// 只传递最后一条用户消息，避免上下文历史干扰生图效果
|
||
const messagesForGemini = buildMessagesForGemini();
|
||
chatApiLogger.info("[Chat API] Gemini: Using only last user message (no history context)");
|
||
|
||
// 转换消息：gemini-image 需要 base64 格式，huanke-gemini 需要 URL 格式
|
||
const providerForConvert = (languageModel as any).provider === "huanke-gemini" ? "huanke-gemini" : "gemini";
|
||
const modelMessages = await convertMessagesWithImages(messagesForGemini, providerForConvert);
|
||
chatApiLogger.info("[Chat API] Message conversion provider:", providerForConvert);
|
||
|
||
// 调用 doGenerate 获取图片（如果是备用模型，传入 overrideModelId）
|
||
const result = await (languageModel as any).doGenerate({
|
||
messages: modelMessages,
|
||
imageParams:
|
||
effectiveImageParams || {
|
||
aspectRatio: defaultImageQualityParams.aspectRatio,
|
||
imageSize: defaultImageQualityParams.imageSize,
|
||
},
|
||
overrideModelId: isRetry ? modelIdToUse : undefined, // 🆕 使用备用模型 ID
|
||
});
|
||
|
||
return result.text || "图片生成失败";
|
||
};
|
||
|
||
let stopKeepalive: (() => void) | null = null;
|
||
try {
|
||
let imageMarkdown = ""; // 初始化为空字符串
|
||
let usedModelId = actualModelId;
|
||
const generationStartedAt = Date.now();
|
||
|
||
// 🆕 图片生成真正开始时发出标记（用于前端显示占位图）
|
||
writeAssistantTextChunk(
|
||
writer,
|
||
"<!-- IMAGE_GENERATION_START -->",
|
||
"gemini:image-start",
|
||
);
|
||
if (contextPayload?.source === "canvas") {
|
||
writeImageGenerationStatusChunk(
|
||
writer,
|
||
{
|
||
phase: "processing",
|
||
providerType: (languageModel as any).provider,
|
||
modelName: currentModelConfig?.displayName || actualModelId,
|
||
imageCount: effectiveImageParams?.imageCount || defaultImageQualityParams.imageCount || 1,
|
||
},
|
||
"gemini:image-status-start",
|
||
);
|
||
}
|
||
stopKeepalive = startStreamKeepalive(writer, "gemini-image");
|
||
|
||
try {
|
||
// 首先尝试主模型
|
||
imageMarkdown = await tryGenerateImage(actualModelId, false);
|
||
} catch (primaryError) {
|
||
chatApiLogger.error("[Chat API] Primary model failed:", primaryError);
|
||
await recordAiModelHealthFailure({
|
||
model: currentModelEntry?.model || currentModelConfig,
|
||
provider: currentModelEntry?.provider,
|
||
error: primaryError,
|
||
context: "chat_gemini_primary",
|
||
elapsedMs: Date.now() - generationStartedAt,
|
||
metadata: {
|
||
runtimeModelId: actualModelId,
|
||
finalModelId,
|
||
providerType: (languageModel as any).provider,
|
||
aspectRatio: effectiveImageParams?.aspectRatio,
|
||
imageSize: effectiveImageParams?.imageSize,
|
||
imageCount:
|
||
effectiveImageParams?.imageCount ||
|
||
defaultImageQualityParams.imageCount ||
|
||
1,
|
||
},
|
||
}).catch((healthError) => {
|
||
chatApiLogger.warn("[Chat API] Record Gemini primary health failure failed:", healthError);
|
||
});
|
||
|
||
// 🆕 尝试备用模型（支持跨服务商切换）
|
||
let fallbackSuccess = false;
|
||
const triedFallbackModelIds = new Set<string>();
|
||
for (const candidate of fallbackCandidates) {
|
||
try {
|
||
triedFallbackModelIds.add(String(candidate.model.id || ""));
|
||
chatApiLogger.info(`[Chat API] 🔄 Trying fallback model: ${candidate.model.displayName} (${candidate.provider.type})`);
|
||
const fallbackResult = await generateImageWithFailoverCandidate({
|
||
candidate,
|
||
messages: buildMessagesForGemini(),
|
||
imageParams: effectiveImageParams,
|
||
fallbackImageParams: {
|
||
aspectRatio: defaultImageQualityParams.aspectRatio,
|
||
imageSize: defaultImageQualityParams.imageSize,
|
||
},
|
||
});
|
||
imageMarkdown = fallbackResult.text;
|
||
usedModelId = fallbackResult.usedModelId;
|
||
fallbackSuccess = true;
|
||
chatApiLogger.info(`[Chat API] ✅ Fallback model ${fallbackResult.displayName} succeeded!`);
|
||
break;
|
||
} catch (fallbackError) {
|
||
const fallbackErrorMessage = normalizeFailoverErrorMessage(fallbackError);
|
||
if (shouldCooldownImageFallbackModel(fallbackErrorMessage)) {
|
||
setImageFallbackModelCooldown(candidate.model.id, fallbackErrorMessage);
|
||
}
|
||
await recordAiModelHealthFailure({
|
||
model: candidate.model,
|
||
provider: candidate.provider,
|
||
error: fallbackError,
|
||
context: "chat_gemini_fallback",
|
||
elapsedMs: Date.now() - generationStartedAt,
|
||
metadata: {
|
||
primaryModelId: actualModelId,
|
||
fallbackModelId: candidate.model.modelId,
|
||
providerType: candidate.provider.type,
|
||
aspectRatio: effectiveImageParams?.aspectRatio,
|
||
imageSize: effectiveImageParams?.imageSize,
|
||
},
|
||
}).catch((healthError) => {
|
||
chatApiLogger.warn("[Chat API] Record Gemini fallback health failure failed:", healthError);
|
||
});
|
||
chatApiLogger.error(`[Chat API] ❌ Fallback model ${candidate.model.displayName} (${candidate.provider.type}) failed:`, fallbackError);
|
||
}
|
||
}
|
||
|
||
if (!fallbackSuccess) {
|
||
const providerPriority: Record<string, number> = {
|
||
"volcano-vision": 0,
|
||
"alibaba-image": 1,
|
||
"bfl-image": 2,
|
||
"ruxa-image": 3,
|
||
"fal-image": 4,
|
||
"openai-image": 5,
|
||
"stepfun-image": 6,
|
||
"grsai-gpt-image": 7,
|
||
runway: 8,
|
||
krea: 9,
|
||
gemini: 9,
|
||
"huanke-gemini": 10,
|
||
};
|
||
const crossProviderCandidates = getImageCrossProviderCandidates(allModels, {
|
||
currentModelId: finalModelId,
|
||
currentProviderId: currentModelConfig?.providerId,
|
||
providerPriority,
|
||
}).filter((candidate) => !triedFallbackModelIds.has(String(candidate.model.id || "")));
|
||
|
||
for (const candidate of crossProviderCandidates) {
|
||
try {
|
||
triedFallbackModelIds.add(String(candidate.model.id || ""));
|
||
chatApiLogger.info(`[Chat API] 🔁 Gemini cross-provider failover: ${candidate.model.displayName} (${candidate.provider.type})`);
|
||
const fallbackResult = await generateImageWithFailoverCandidate({
|
||
candidate,
|
||
messages: buildMessagesForGemini(),
|
||
imageParams: effectiveImageParams,
|
||
fallbackImageParams: {
|
||
aspectRatio: defaultImageQualityParams.aspectRatio,
|
||
imageSize: defaultImageQualityParams.imageSize,
|
||
imageCount: defaultImageQualityParams.imageCount,
|
||
},
|
||
});
|
||
imageMarkdown = fallbackResult.text;
|
||
usedModelId = fallbackResult.usedModelId;
|
||
fallbackSuccess = true;
|
||
chatApiLogger.info(`[Chat API] ✅ Gemini cross-provider fallback model ${fallbackResult.displayName} succeeded!`);
|
||
break;
|
||
} catch (fallbackError) {
|
||
const fallbackErrorMessage = normalizeFailoverErrorMessage(fallbackError);
|
||
if (shouldCooldownImageFallbackModel(fallbackErrorMessage)) {
|
||
setImageFallbackModelCooldown(candidate.model.id, fallbackErrorMessage);
|
||
}
|
||
await recordAiModelHealthFailure({
|
||
model: candidate.model,
|
||
provider: candidate.provider,
|
||
error: fallbackError,
|
||
context: "chat_gemini_cross_fallback",
|
||
elapsedMs: Date.now() - generationStartedAt,
|
||
metadata: {
|
||
primaryModelId: actualModelId,
|
||
fallbackModelId: candidate.model.modelId,
|
||
providerType: candidate.provider.type,
|
||
aspectRatio: effectiveImageParams?.aspectRatio,
|
||
imageSize: effectiveImageParams?.imageSize,
|
||
},
|
||
}).catch((healthError) => {
|
||
chatApiLogger.warn("[Chat API] Record Gemini cross-provider health failure failed:", healthError);
|
||
});
|
||
chatApiLogger.error(`[Chat API] ❌ Gemini cross-provider fallback model ${candidate.model.displayName} (${candidate.provider.type}) failed:`, fallbackError);
|
||
}
|
||
}
|
||
}
|
||
|
||
// 如果所有备用模型都失败，抛出原始错误触发火山引擎故障转移
|
||
if (!fallbackSuccess) {
|
||
throw primaryError;
|
||
}
|
||
}
|
||
|
||
chatApiLogger.info("[Chat API] Gemini generated, length:", imageMarkdown.length);
|
||
chatApiLogger.info("[Chat API] Gemini image markdown:", imageMarkdown.substring(0, 200));
|
||
chatApiLogger.info("[Chat API] Used model ID:", usedModelId);
|
||
|
||
const usedModelEntry = resolveModelEntryByRuntimeModelId(
|
||
allModels,
|
||
usedModelId,
|
||
usedModelId === actualModelId ? finalModelId : undefined,
|
||
);
|
||
await recordAiModelHealthSuccess({
|
||
model: usedModelEntry?.model || (usedModelId === actualModelId ? currentModelConfig : null),
|
||
}).catch((healthError) => {
|
||
chatApiLogger.warn("[Chat API] Record Gemini health success failed:", healthError);
|
||
});
|
||
|
||
const usedModelDisplayName =
|
||
resolveModelDisplayNameByModelId(allModels, usedModelId) ||
|
||
currentModelConfig?.displayName ||
|
||
actualModelId;
|
||
const imageMarkdownWithModel = prependImageModelBadge(
|
||
imageMarkdown,
|
||
usedModelDisplayName,
|
||
);
|
||
|
||
// 生成一个统一的消息 ID
|
||
const messageId = generateUUID();
|
||
|
||
// 写入消息到流 - 使用正确的 parts 格式
|
||
writeAssistantTextChunk(
|
||
writer,
|
||
imageMarkdownWithModel,
|
||
"gemini:image-result",
|
||
messageId,
|
||
);
|
||
if (contextPayload?.source === "canvas") {
|
||
writeCanvasMediaChunk(
|
||
writer,
|
||
{
|
||
messageId,
|
||
text: imageMarkdownWithModel,
|
||
prompt: requestTextContent,
|
||
sourceImageUrls: requestTargetResolution?.targetImageUrls || [],
|
||
},
|
||
"gemini:image-canvas-media",
|
||
);
|
||
}
|
||
|
||
// 🆕 图片生成结束标记（成功）
|
||
writeAssistantTextChunk(
|
||
writer,
|
||
"<!-- IMAGE_GENERATION_END -->",
|
||
"gemini:image-end-success",
|
||
);
|
||
if (contextPayload?.source === "canvas") {
|
||
writeImageGenerationStatusChunk(
|
||
writer,
|
||
{
|
||
phase: "completed",
|
||
providerType: (languageModel as any).provider,
|
||
modelName: usedModelDisplayName,
|
||
imageCount: effectiveImageParams?.imageCount || defaultImageQualityParams.imageCount || 1,
|
||
},
|
||
"gemini:image-status-completed",
|
||
);
|
||
}
|
||
|
||
// 保存消息到数据库（现在是 S3 URL，可以完整保存）
|
||
try {
|
||
await saveMessages({
|
||
messages: [
|
||
{
|
||
id: messageId,
|
||
chatId: id,
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: imageMarkdownWithModel }],
|
||
attachments: [], // 必须提供 attachments 字段（数据库 notNull）
|
||
createdAt: new Date(),
|
||
videoStorageStatus: null,
|
||
temporaryVideoUrl: null,
|
||
permanentVideoUrl: null,
|
||
archivedAt: null,
|
||
videoMetadata: null,
|
||
},
|
||
],
|
||
});
|
||
chatApiLogger.info("[Chat API] ✅ Gemini message saved successfully!");
|
||
|
||
// 提取用户 prompt（最后一条用户消息的文本）
|
||
const lastUserMessage = uiMessages.filter((m) => m.role === "user").pop();
|
||
const userPrompt = lastUserMessage?.parts?.find((p): p is { type: "text"; text: string } => p.type === "text")?.text || "";
|
||
|
||
// 保存图片记录到 GeneratedImage 表
|
||
await saveGeneratedImages(
|
||
imageMarkdownWithModel,
|
||
session.user.id,
|
||
id,
|
||
messageId,
|
||
userPrompt,
|
||
resolveModelRefByRuntimeModelId(
|
||
allModels,
|
||
usedModelId,
|
||
usedModelId === actualModelId ? finalModelId : undefined,
|
||
),
|
||
{
|
||
sourceAssetIds: requestTargetResolution?.targetAssetIds,
|
||
sourceImageUrls: requestTargetResolution?.targetImageUrls,
|
||
relationType: requestResolvedMediaInput?.hasImage ? "edited_from" : "generated_from",
|
||
sourceType: requestResolvedMediaInput?.hasImage ? "generated" : "generated",
|
||
projectId: contextPayload?.projectId || null,
|
||
optimizedPrompt: smartAutoOptimizedPrompt || userPrompt,
|
||
requestedImageCount: effectiveImageParams?.imageCount,
|
||
aspectRatio: effectiveImageParams?.aspectRatio,
|
||
imageSize: effectiveImageParams?.imageSize,
|
||
},
|
||
);
|
||
} catch (saveError) {
|
||
chatApiLogger.error("[Chat API] ❌ Failed to save Gemini message to database:", saveError);
|
||
chatApiLogger.error("[Chat API] Message content that failed:", imageMarkdown.substring(0, 300));
|
||
}
|
||
} catch (error) {
|
||
chatApiLogger.error("[Chat API] Gemini generation error:", error);
|
||
chatApiLogger.info("[Chat API] Attempting failover to Volcano Vision...");
|
||
|
||
// 故障转移：尝试使用火山引擎模型
|
||
try {
|
||
// 查找火山引擎模型
|
||
const allModels = await getEnabledAiModelsWithProvider();
|
||
chatApiLogger.info("[Chat API] Total enabled models:", allModels.length);
|
||
|
||
const volcanoModel = allModels.find(
|
||
(m: any) => m.provider.type === "volcano-vision" && m.model.isEnabled
|
||
);
|
||
|
||
if (volcanoModel) {
|
||
chatApiLogger.info("[Chat API] Found Volcano Vision model for failover:", volcanoModel.model.id);
|
||
|
||
// 获取火山引擎模型实例
|
||
const volcanoLanguageModel = await getLanguageModelById(volcanoModel.model.id);
|
||
|
||
if (volcanoLanguageModel && typeof (volcanoLanguageModel as any).doGenerate === "function") {
|
||
chatApiLogger.info("[Chat API] Executing Volcano Vision failover...");
|
||
|
||
// 转换消息（火山引擎会自动处理 Base64）
|
||
const volcanoMessages = await convertMessagesWithImages(
|
||
injectHistoryImagesToLastUserMessage(uiMessagesForModel, smartAutoHistoryImages),
|
||
"volcano"
|
||
);
|
||
|
||
// 调用火山引擎生成（故障转移时只生成 1 张，保持与 Gemini 一致）
|
||
const volcanoResult = await (volcanoLanguageModel as any).doGenerate({
|
||
messages: volcanoMessages,
|
||
maxImages: 1, // 故障转移时只生成 1 张
|
||
imageParams:
|
||
effectiveImageParams || {
|
||
aspectRatio: defaultImageQualityParams.aspectRatio,
|
||
imageSize: defaultImageQualityParams.imageSize,
|
||
imageCount: defaultImageQualityParams.imageCount,
|
||
},
|
||
});
|
||
|
||
const imageMarkdown = volcanoResult.text || "图片生成失败";
|
||
const imageMarkdownWithModel = prependImageModelBadge(
|
||
imageMarkdown,
|
||
volcanoModel.model.displayName || volcanoModel.model.modelId,
|
||
);
|
||
chatApiLogger.info("[Chat API] Volcano Vision failover success, length:", imageMarkdown.length);
|
||
|
||
// 生成消息 ID
|
||
const messageId = generateUUID();
|
||
|
||
// 写入消息到流
|
||
writeAssistantTextChunk(
|
||
writer,
|
||
imageMarkdownWithModel,
|
||
"gemini:volcano-failover-result",
|
||
messageId,
|
||
);
|
||
if (contextPayload?.source === "canvas") {
|
||
writeCanvasMediaChunk(
|
||
writer,
|
||
{
|
||
messageId,
|
||
text: imageMarkdownWithModel,
|
||
prompt: requestTextContent,
|
||
sourceImageUrls: requestTargetResolution?.targetImageUrls || [],
|
||
},
|
||
"gemini:volcano-failover-canvas-media",
|
||
);
|
||
}
|
||
|
||
// 🆕 图片生成结束标记（故障转移成功）
|
||
writeAssistantTextChunk(
|
||
writer,
|
||
"<!-- IMAGE_GENERATION_END -->",
|
||
"gemini:volcano-failover-end",
|
||
);
|
||
if (contextPayload?.source === "canvas") {
|
||
writeImageGenerationStatusChunk(
|
||
writer,
|
||
{
|
||
phase: "completed",
|
||
providerType: "volcano-vision",
|
||
modelName: volcanoModel.model.displayName || volcanoModel.model.modelId,
|
||
imageCount: 1,
|
||
},
|
||
"gemini:volcano-failover-status-completed",
|
||
);
|
||
}
|
||
|
||
// 保存消息到数据库
|
||
try {
|
||
await saveMessages({
|
||
messages: [
|
||
{
|
||
id: messageId,
|
||
chatId: id,
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: imageMarkdownWithModel }],
|
||
attachments: [],
|
||
createdAt: new Date(),
|
||
videoStorageStatus: null,
|
||
temporaryVideoUrl: null,
|
||
permanentVideoUrl: null,
|
||
archivedAt: null,
|
||
videoMetadata: null,
|
||
},
|
||
],
|
||
});
|
||
chatApiLogger.info("[Chat API] ✅ Volcano Vision failover message saved!");
|
||
|
||
// 提取用户 prompt
|
||
const lastUserMessage = uiMessages.filter((m) => m.role === "user").pop();
|
||
const userPrompt = lastUserMessage?.parts?.find((p): p is { type: "text"; text: string } => p.type === "text")?.text || "";
|
||
|
||
// 保存图片记录到 GeneratedImage 表
|
||
await saveGeneratedImages(
|
||
imageMarkdownWithModel,
|
||
session.user.id,
|
||
id,
|
||
messageId,
|
||
userPrompt,
|
||
volcanoModel.model.id,
|
||
{
|
||
sourceAssetIds: requestTargetResolution?.targetAssetIds,
|
||
sourceImageUrls: requestTargetResolution?.targetImageUrls,
|
||
relationType: requestResolvedMediaInput?.hasImage ? "edited_from" : "generated_from",
|
||
sourceType: "generated",
|
||
projectId: contextPayload?.projectId || null,
|
||
optimizedPrompt: smartAutoOptimizedPrompt || userPrompt,
|
||
requestedImageCount: effectiveImageParams?.imageCount,
|
||
aspectRatio: effectiveImageParams?.aspectRatio,
|
||
imageSize: effectiveImageParams?.imageSize,
|
||
},
|
||
);
|
||
} catch (saveError) {
|
||
chatApiLogger.error("[Chat API] Failed to save failover message:", saveError);
|
||
}
|
||
|
||
return; // 成功，直接返回
|
||
}
|
||
}
|
||
|
||
chatApiLogger.info("[Chat API] No Volcano Vision model available for failover");
|
||
} catch (failoverError) {
|
||
chatApiLogger.error("[Chat API] Failover to Volcano Vision also failed:", failoverError);
|
||
}
|
||
|
||
// 如果故障转移也失败，返回错误消息
|
||
const errorMessageId = generateUUID();
|
||
writeAssistantTextChunk(
|
||
writer,
|
||
"图片生成失败，请稍后重试",
|
||
"gemini:error",
|
||
errorMessageId,
|
||
);
|
||
|
||
// 🆕 图片生成结束标记（失败）
|
||
writeAssistantTextChunk(
|
||
writer,
|
||
"<!-- IMAGE_GENERATION_END -->",
|
||
"gemini:image-end-failed",
|
||
);
|
||
|
||
// 尝试保存错误消息
|
||
try {
|
||
await saveMessages({
|
||
messages: [
|
||
{
|
||
id: errorMessageId,
|
||
chatId: id,
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: "图片生成失败，请稍后重试" }],
|
||
attachments: [],
|
||
createdAt: new Date(),
|
||
videoStorageStatus: null,
|
||
temporaryVideoUrl: null,
|
||
permanentVideoUrl: null,
|
||
archivedAt: null,
|
||
videoMetadata: null,
|
||
},
|
||
],
|
||
});
|
||
} catch (saveError) {
|
||
chatApiLogger.error("[Chat API] Failed to save error message:", saveError);
|
||
}
|
||
} finally {
|
||
stopKeepalive?.();
|
||
}
|
||
},
|
||
});
|
||
|
||
return createSseResponse(geminiStream);
|
||
}
|
||
// ========== Gemini 处理结束 ==========
|
||
|
||
// ========== 阿里云千问图片生成独立处理 ==========
|
||
const isAlibabaLikeImageProvider =
|
||
isImageGenerationModel &&
|
||
[
|
||
"alibaba-image",
|
||
"bfl-image",
|
||
"ruxa-image",
|
||
"fal-image",
|
||
"openai-image",
|
||
"stepfun-image",
|
||
"midjourney",
|
||
"runway",
|
||
"krea",
|
||
].includes((languageModel as any).provider);
|
||
if (isAlibabaLikeImageProvider && typeof (languageModel as any).doGenerate === "function") {
|
||
chatApiLogger.info("[Chat API] Using image generation provider:", (languageModel as any).provider);
|
||
chatApiLogger.info("[Chat API] Image params:", effectiveImageParams);
|
||
|
||
// 🆕 获取模型显示名称，用于错误提示
|
||
const alibabaModelConfig = await getAiModelById({ id: finalModelId });
|
||
const modelDisplayName = alibabaModelConfig?.displayName || actualModelId;
|
||
const configuredFallbacks = [
|
||
alibabaModelConfig?.fallbackModelId1,
|
||
alibabaModelConfig?.fallbackModelId2,
|
||
].filter(Boolean) as string[];
|
||
const allModels = await getEnabledAiModelsWithProvider();
|
||
const fallbackCandidates = buildImageFallbackCandidates(allModels, {
|
||
currentProviderId: alibabaModelConfig?.providerId,
|
||
currentModelId: alibabaModelConfig?.id,
|
||
currentModelKey: actualModelId,
|
||
configuredFallbackIds: configuredFallbacks,
|
||
});
|
||
chatApiLogger.info("[Chat API] Alibaba fallback candidates:", fallbackCandidates.map((candidate) => `${candidate.model.displayName}(${candidate.provider.type})`));
|
||
|
||
const alibabaStream = createUIMessageStream({
|
||
execute: async ({ writer }) => {
|
||
let stopKeepalive: (() => void) | null = null;
|
||
try {
|
||
// 🆕 如果是智能模式且有对话回复，先发送对话消息让用户看到
|
||
const conversationMessage = smartAutoFriendlyMessage || null;
|
||
|
||
if (conversationMessage) {
|
||
const chatMessageId = generateUUID();
|
||
chatApiLogger.info("[Chat API] 💬 Sending conversational reply first:", smartAutoFriendlyMessage);
|
||
|
||
// 发送带标记的对话消息
|
||
writeAssistantTextChunk(
|
||
writer,
|
||
conversationMessage,
|
||
"alibaba:conversation",
|
||
chatMessageId,
|
||
);
|
||
|
||
// 保存对话消息到数据库（不带标记，纯对话）
|
||
await saveMessages({
|
||
messages: [
|
||
{
|
||
id: chatMessageId,
|
||
chatId: id,
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: smartAutoFriendlyMessage }],
|
||
attachments: [],
|
||
createdAt: new Date(),
|
||
videoStorageStatus: null,
|
||
temporaryVideoUrl: null,
|
||
permanentVideoUrl: null,
|
||
archivedAt: null,
|
||
videoMetadata: null,
|
||
},
|
||
],
|
||
});
|
||
}
|
||
|
||
// 只传递最后一条用户消息
|
||
const lastUserMessage = uiMessagesForModel.filter((m) => m.role === "user").pop();
|
||
let messagesForAlibaba = lastUserMessage ? [lastUserMessage] : uiMessagesForModel;
|
||
chatApiLogger.info("[Chat API] Alibaba: Using only last user message");
|
||
|
||
// 🆕 如果有智能模式优化后的提示词，替换消息中的文本（仅用于API调用）
|
||
if (smartAutoOptimizedPrompt) {
|
||
chatApiLogger.info("[Chat API] Using optimized prompt for API call");
|
||
messagesForAlibaba = applyOptimizedPromptToLatestUserMessage(
|
||
messagesForAlibaba,
|
||
smartAutoOptimizedPrompt,
|
||
hiddenPromptPrefix,
|
||
);
|
||
}
|
||
|
||
const resolvedTargetImageUrlsForProvider = normalizeResolvedTargetImageUrls(
|
||
requestTargetResolution?.targetImageUrls
|
||
);
|
||
if (resolvedTargetImageUrlsForProvider.length > 0) {
|
||
messagesForAlibaba = applyResolvedTargetImagesToLastUserMessage(
|
||
messagesForAlibaba,
|
||
resolvedTargetImageUrlsForProvider
|
||
);
|
||
chatApiLogger.info("[Chat API] Image provider target images:", {
|
||
resolutionMode: requestTargetResolution?.resolutionMode || null,
|
||
imageCount: resolvedTargetImageUrlsForProvider.length,
|
||
firstImageUrl: summarizeTargetImageUrl(resolvedTargetImageUrlsForProvider[0]),
|
||
});
|
||
}
|
||
|
||
messagesForAlibaba = injectHistoryImagesToLastUserMessage(messagesForAlibaba, smartAutoHistoryImages);
|
||
const isFalAsyncImageEdit =
|
||
(languageModel as any).provider === "fal-image" &&
|
||
isFalGptImage2EditModelId(actualModelId);
|
||
|
||
if (isFalAsyncImageEdit) {
|
||
const falProvider = alibabaModelConfig?.providerId
|
||
? await getAiProviderById({ id: alibabaModelConfig.providerId }).catch(() => undefined)
|
||
: undefined;
|
||
const falApiKey = String(falProvider?.apiKey || "").trim();
|
||
if (!falApiKey) {
|
||
throw new Error("FAL 图片编辑模型未配置可用的 API Key");
|
||
}
|
||
|
||
const falMessages = await convertMessagesWithImages(messagesForAlibaba, "fal-image");
|
||
const queuedTask = await submitFalImageTask({
|
||
apiKey: falApiKey,
|
||
modelId: actualModelId,
|
||
messages: falMessages,
|
||
imageParams:
|
||
effectiveImageParams || {
|
||
aspectRatio: defaultImageQualityParams.aspectRatio,
|
||
imageSize: defaultImageQualityParams.imageSize,
|
||
imageCount: defaultImageQualityParams.imageCount,
|
||
},
|
||
});
|
||
|
||
const placeholderMessageId = generateUUID();
|
||
const userPrompt =
|
||
lastUserMessage?.parts?.find(
|
||
(p): p is { type: "text"; text: string } => p.type === "text",
|
||
)?.text || "";
|
||
const modelRef = resolveModelRefByRuntimeModelId(
|
||
allModels,
|
||
queuedTask.modelId,
|
||
finalModelId,
|
||
);
|
||
const placeholderText = `已提交 ${modelDisplayName} 编辑任务，预计 2-4 分钟完成，结果会自动回填到这里。`;
|
||
const queuedMetadata: FalImageEditTaskMetadata = {
|
||
taskKind: "fal_gpt_image_2_edit",
|
||
provider: "fal-image",
|
||
providerId: String(alibabaModelConfig?.providerId || ""),
|
||
endpointId: queuedTask.endpointId,
|
||
requestId: queuedTask.requestId,
|
||
modelId: queuedTask.modelId,
|
||
modelRef,
|
||
modelDisplayName,
|
||
prompt: queuedTask.prompt,
|
||
optimizedPrompt: smartAutoOptimizedPrompt || userPrompt || queuedTask.prompt,
|
||
userPrompt: userPrompt || queuedTask.prompt,
|
||
requestedImageCount:
|
||
effectiveImageParams?.imageCount || defaultImageQualityParams.imageCount || 1,
|
||
aspectRatio: effectiveImageParams?.aspectRatio,
|
||
imageSize: effectiveImageParams?.imageSize,
|
||
outputSizeLabel: queuedTask.outputSizeLabel,
|
||
sourceImageUrls: requestTargetResolution?.targetImageUrls || queuedTask.sourceImageUrls,
|
||
sourceAssetIds: requestTargetResolution?.targetAssetIds || [],
|
||
relationType: requestResolvedMediaInput?.hasImage ? "edited_from" : "generated_from",
|
||
sourceType: "generated",
|
||
projectId: contextPayload?.projectId || null,
|
||
contextSource: contextPayload?.source || null,
|
||
status: "queued",
|
||
startedAt: Date.now(),
|
||
updatedAt: Date.now(),
|
||
queuePosition: null,
|
||
lastLogs: [],
|
||
error: null,
|
||
};
|
||
|
||
writeAssistantTextChunk(
|
||
writer,
|
||
"<!-- IMAGE_GENERATION_START -->",
|
||
"alibaba:image-start",
|
||
);
|
||
writeImageGenerationStatusChunk(
|
||
writer,
|
||
{
|
||
phase: "queued",
|
||
providerType: "fal-image",
|
||
modelName: modelDisplayName,
|
||
imageCount:
|
||
effectiveImageParams?.imageCount || defaultImageQualityParams.imageCount || 1,
|
||
},
|
||
"alibaba:image-status-queued",
|
||
);
|
||
writeAssistantTextChunk(
|
||
writer,
|
||
placeholderText,
|
||
"alibaba:image-queued",
|
||
placeholderMessageId,
|
||
);
|
||
|
||
await saveMessages({
|
||
messages: [
|
||
{
|
||
id: placeholderMessageId,
|
||
chatId: id,
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: placeholderText }],
|
||
attachments: [],
|
||
createdAt: new Date(),
|
||
videoStorageStatus: null,
|
||
temporaryVideoUrl: null,
|
||
permanentVideoUrl: null,
|
||
archivedAt: null,
|
||
videoMetadata: queuedMetadata as any,
|
||
},
|
||
],
|
||
});
|
||
|
||
chatApiLogger.info("[Chat API] ✅ Queued FAL GPT Image 2 Edit task", {
|
||
requestId: queuedTask.requestId,
|
||
endpointId: queuedTask.endpointId,
|
||
messageId: placeholderMessageId,
|
||
});
|
||
return;
|
||
}
|
||
|
||
const tryGenerateImage = async (modelIdToUse: string, isRetry: boolean) => {
|
||
// 转换消息（图片编辑模型统一使用 URL 输入）
|
||
const providerForConvert =
|
||
(languageModel as any).provider === "bfl-image"
|
||
? "bfl-image"
|
||
: (languageModel as any).provider === "ruxa-image"
|
||
? "ruxa-image"
|
||
: (languageModel as any).provider === "fal-image"
|
||
? "fal-image"
|
||
: (languageModel as any).provider === "openai-image"
|
||
? "openai-image"
|
||
: (languageModel as any).provider === "stepfun-image"
|
||
? "stepfun-image"
|
||
: (languageModel as any).provider === "midjourney"
|
||
? "midjourney"
|
||
: (languageModel as any).provider === "runway"
|
||
? "runway"
|
||
: (languageModel as any).provider === "krea"
|
||
? "krea"
|
||
: "alibaba-image";
|
||
const modelMessages = await convertMessagesWithImages(messagesForAlibaba, providerForConvert);
|
||
|
||
// 调用 doGenerate 获取图片，传递模型显示名称
|
||
const result = await (languageModel as any).doGenerate({
|
||
messages: modelMessages,
|
||
imageParams:
|
||
effectiveImageParams || {
|
||
aspectRatio: defaultImageQualityParams.aspectRatio,
|
||
imageSize: defaultImageQualityParams.imageSize,
|
||
imageCount: defaultImageQualityParams.imageCount,
|
||
},
|
||
modelDisplayName, // 🆕 传递模型显示名称
|
||
overrideModelId: isRetry ? modelIdToUse : undefined,
|
||
});
|
||
|
||
if (result?.finishReason === "error") {
|
||
throw new Error(result.text || "图片生成失败");
|
||
}
|
||
|
||
return result.text || "图片生成失败";
|
||
};
|
||
|
||
let imageMarkdown = "";
|
||
let usedModelId = actualModelId;
|
||
|
||
// 🆕 图片生成真正开始时发出标记（用于前端显示占位图）
|
||
writeAssistantTextChunk(
|
||
writer,
|
||
"<!-- IMAGE_GENERATION_START -->",
|
||
"alibaba:image-start",
|
||
);
|
||
if (contextPayload?.source === "canvas") {
|
||
writeImageGenerationStatusChunk(
|
||
writer,
|
||
{
|
||
phase: "processing",
|
||
providerType: (languageModel as any).provider,
|
||
modelName: alibabaModelConfig?.displayName || actualModelId,
|
||
imageCount: effectiveImageParams?.imageCount || defaultImageQualityParams.imageCount || 1,
|
||
},
|
||
"alibaba:image-status-start",
|
||
);
|
||
}
|
||
stopKeepalive = startStreamKeepalive(writer, "alibaba-image");
|
||
|
||
try {
|
||
imageMarkdown = await tryGenerateImage(actualModelId, false);
|
||
} catch (primaryError) {
|
||
chatApiLogger.error("[Chat API] Alibaba primary model failed:", primaryError);
|
||
|
||
let fallbackSuccess = false;
|
||
for (const candidate of fallbackCandidates) {
|
||
try {
|
||
chatApiLogger.info(`[Chat API] 🔄 Alibaba fallback model: ${candidate.model.displayName} (${candidate.provider.type})`);
|
||
const fallbackResult = await generateImageWithFailoverCandidate({
|
||
candidate,
|
||
messages: messagesForAlibaba,
|
||
imageParams: effectiveImageParams,
|
||
fallbackImageParams: {
|
||
aspectRatio: defaultImageQualityParams.aspectRatio,
|
||
imageSize: defaultImageQualityParams.imageSize,
|
||
imageCount: defaultImageQualityParams.imageCount,
|
||
},
|
||
});
|
||
imageMarkdown = fallbackResult.text;
|
||
usedModelId = fallbackResult.usedModelId;
|
||
fallbackSuccess = true;
|
||
chatApiLogger.info(`[Chat API] ✅ Alibaba fallback model ${fallbackResult.displayName} succeeded!`);
|
||
break;
|
||
} catch (fallbackError) {
|
||
const fallbackErrorMessage = normalizeFailoverErrorMessage(fallbackError);
|
||
if (shouldCooldownImageFallbackModel(fallbackErrorMessage)) {
|
||
setImageFallbackModelCooldown(candidate.model.id, fallbackErrorMessage);
|
||
}
|
||
chatApiLogger.error(`[Chat API] ❌ Alibaba fallback model ${candidate.model.displayName} (${candidate.provider.type}) failed:`, fallbackError);
|
||
}
|
||
}
|
||
|
||
if (!fallbackSuccess) {
|
||
const providerPriority: Record<string, number> = {
|
||
"volcano-vision": 0,
|
||
"alibaba-image": 1,
|
||
"bfl-image": 2,
|
||
"ruxa-image": 3,
|
||
"fal-image": 4,
|
||
"openai-image": 5,
|
||
"stepfun-image": 6,
|
||
runway: 7,
|
||
krea: 8,
|
||
gemini: 8,
|
||
"huanke-gemini": 9,
|
||
};
|
||
const crossProviderCandidates = getImageCrossProviderCandidates(allModels, {
|
||
currentModelId: finalModelId,
|
||
currentProviderId: alibabaModelConfig?.providerId,
|
||
providerPriority,
|
||
});
|
||
|
||
for (const candidate of crossProviderCandidates) {
|
||
try {
|
||
chatApiLogger.info(`[Chat API] 🔁 Cross-provider failover attempt: ${candidate.model.displayName} (${candidate.provider.type})`);
|
||
if (
|
||
candidate.provider.type === "fal-image" &&
|
||
isFalGptImage2EditModelId(
|
||
String(candidate.model?.modelId || candidate.model?.id || "")
|
||
)
|
||
) {
|
||
chatApiLogger.info(`[Chat API] Skip async-only failover candidate: ${candidate.model.displayName}`);
|
||
continue;
|
||
}
|
||
|
||
const candidateLanguageModel = await getLanguageModelById(candidate.model.id);
|
||
if (!candidateLanguageModel || typeof (candidateLanguageModel as any).doGenerate !== "function") {
|
||
chatApiLogger.info(`[Chat API] Skip candidate without doGenerate: ${candidate.model.id}`);
|
||
continue;
|
||
}
|
||
|
||
const convertProvider =
|
||
candidate.provider.type === "bfl-image"
|
||
? "bfl-image"
|
||
: candidate.provider.type === "ruxa-image"
|
||
? "ruxa-image"
|
||
: candidate.provider.type === "fal-image"
|
||
? "fal-image"
|
||
: candidate.provider.type === "openai-image"
|
||
? "openai-image"
|
||
: candidate.provider.type === "stepfun-image"
|
||
? "stepfun-image"
|
||
: candidate.provider.type === "midjourney"
|
||
? "midjourney"
|
||
: candidate.provider.type === "runway"
|
||
? "runway"
|
||
: candidate.provider.type === "krea"
|
||
? "krea"
|
||
: candidate.provider.type === "gemini"
|
||
? "gemini"
|
||
: candidate.provider.type === "huanke-gemini"
|
||
? "huanke-gemini"
|
||
: candidate.provider.type === "volcano-vision"
|
||
? "volcano"
|
||
: "alibaba-image";
|
||
|
||
const candidateMessages = await convertMessagesWithImages(
|
||
messagesForAlibaba,
|
||
convertProvider
|
||
);
|
||
|
||
const candidateResult = await (candidateLanguageModel as any).doGenerate({
|
||
messages: candidateMessages,
|
||
imageParams:
|
||
effectiveImageParams || {
|
||
aspectRatio: defaultImageQualityParams.aspectRatio,
|
||
imageSize: defaultImageQualityParams.imageSize,
|
||
imageCount: defaultImageQualityParams.imageCount,
|
||
},
|
||
modelDisplayName: candidate.model.displayName,
|
||
overrideModelId: candidate.model.modelId,
|
||
});
|
||
|
||
if (candidateResult?.finishReason === "error") {
|
||
throw new Error(candidateResult.text || "图片生成失败");
|
||
}
|
||
|
||
imageMarkdown = candidateResult.text || "图片生成失败";
|
||
usedModelId = candidate.model.modelId;
|
||
fallbackSuccess = true;
|
||
chatApiLogger.info(`[Chat API] ✅ Cross-provider failover succeeded: ${candidate.model.displayName} (${candidate.provider.type})`);
|
||
break;
|
||
} catch (crossError) {
|
||
const crossErrorMessage = normalizeFailoverErrorMessage(crossError);
|
||
if (shouldCooldownImageFallbackModel(crossErrorMessage)) {
|
||
setImageFallbackModelCooldown(candidate.model.id, crossErrorMessage);
|
||
}
|
||
chatApiLogger.error(`[Chat API] ❌ Cross-provider failover failed: ${candidate.model.displayName} (${candidate.provider.type})`, crossError);
|
||
}
|
||
}
|
||
}
|
||
|
||
if (!fallbackSuccess) {
|
||
throw primaryError;
|
||
}
|
||
}
|
||
|
||
chatApiLogger.info("[Chat API] Alibaba generated, length:", imageMarkdown.length);
|
||
chatApiLogger.info("[Chat API] Alibaba used model:", usedModelId);
|
||
|
||
const usedModelDisplayName =
|
||
resolveModelDisplayNameByModelId(allModels, usedModelId) ||
|
||
alibabaModelConfig?.displayName ||
|
||
actualModelId;
|
||
const imageMarkdownWithModel = prependImageModelBadge(
|
||
imageMarkdown,
|
||
usedModelDisplayName,
|
||
);
|
||
|
||
// 生成消息 ID（图片消息）
|
||
const messageId = generateUUID();
|
||
|
||
// 写入图片消息到流
|
||
writeAssistantTextChunk(
|
||
writer,
|
||
imageMarkdownWithModel,
|
||
"alibaba:image-result",
|
||
messageId,
|
||
);
|
||
if (contextPayload?.source === "canvas") {
|
||
writeCanvasMediaChunk(
|
||
writer,
|
||
{
|
||
messageId,
|
||
text: imageMarkdownWithModel,
|
||
prompt: requestTextContent,
|
||
sourceImageUrls: requestTargetResolution?.targetImageUrls || [],
|
||
},
|
||
"alibaba:image-canvas-media",
|
||
);
|
||
}
|
||
|
||
// 🆕 图片生成结束标记（成功）
|
||
writeAssistantTextChunk(
|
||
writer,
|
||
"<!-- IMAGE_GENERATION_END -->",
|
||
"alibaba:image-end-success",
|
||
);
|
||
if (contextPayload?.source === "canvas") {
|
||
writeImageGenerationStatusChunk(
|
||
writer,
|
||
{
|
||
phase: "completed",
|
||
providerType: (languageModel as any).provider,
|
||
modelName: usedModelDisplayName,
|
||
imageCount: effectiveImageParams?.imageCount || defaultImageQualityParams.imageCount || 1,
|
||
},
|
||
"alibaba:image-status-completed",
|
||
);
|
||
}
|
||
|
||
// 保存图片消息到数据库
|
||
try {
|
||
await saveMessages({
|
||
messages: [
|
||
{
|
||
id: messageId,
|
||
chatId: id,
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: imageMarkdownWithModel }],
|
||
attachments: [],
|
||
createdAt: new Date(),
|
||
videoStorageStatus: null,
|
||
temporaryVideoUrl: null,
|
||
permanentVideoUrl: null,
|
||
archivedAt: null,
|
||
videoMetadata: null,
|
||
},
|
||
],
|
||
});
|
||
chatApiLogger.info("[Chat API] ✅ Alibaba message saved successfully!");
|
||
|
||
// 提取用户 prompt
|
||
const userPrompt = lastUserMessage?.parts?.find((p): p is { type: "text"; text: string } => p.type === "text")?.text || "";
|
||
|
||
// 保存图片记录到 GeneratedImage 表
|
||
await saveGeneratedImages(
|
||
imageMarkdownWithModel,
|
||
session.user.id,
|
||
id,
|
||
messageId,
|
||
userPrompt,
|
||
resolveModelRefByRuntimeModelId(
|
||
allModels,
|
||
usedModelId,
|
||
usedModelId === actualModelId ? finalModelId : undefined,
|
||
),
|
||
{
|
||
sourceAssetIds: requestTargetResolution?.targetAssetIds,
|
||
sourceImageUrls: requestTargetResolution?.targetImageUrls,
|
||
relationType: requestResolvedMediaInput?.hasImage ? "edited_from" : "generated_from",
|
||
sourceType: "generated",
|
||
projectId: contextPayload?.projectId || null,
|
||
optimizedPrompt: smartAutoOptimizedPrompt || userPrompt,
|
||
requestedImageCount: effectiveImageParams?.imageCount,
|
||
aspectRatio: effectiveImageParams?.aspectRatio,
|
||
imageSize: effectiveImageParams?.imageSize,
|
||
},
|
||
);
|
||
} catch (saveError) {
|
||
chatApiLogger.error("[Chat API] ❌ Failed to save Alibaba message:", saveError);
|
||
}
|
||
} catch (error) {
|
||
chatApiLogger.error("[Chat API] Alibaba generation error:", error);
|
||
|
||
// 返回错误消息
|
||
const errorMessageId = generateUUID();
|
||
const errorText = error instanceof Error ? error.message : "图片生成失败，请稍后重试";
|
||
writeAssistantTextChunk(
|
||
writer,
|
||
errorText,
|
||
"alibaba:error",
|
||
errorMessageId,
|
||
);
|
||
|
||
// 🆕 图片生成结束标记（失败）
|
||
writeAssistantTextChunk(
|
||
writer,
|
||
"<!-- IMAGE_GENERATION_END -->",
|
||
"alibaba:image-end-failed",
|
||
);
|
||
|
||
try {
|
||
await saveMessages({
|
||
messages: [
|
||
{
|
||
id: errorMessageId,
|
||
chatId: id,
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: errorText }],
|
||
attachments: [],
|
||
createdAt: new Date(),
|
||
videoStorageStatus: null,
|
||
temporaryVideoUrl: null,
|
||
permanentVideoUrl: null,
|
||
archivedAt: null,
|
||
videoMetadata: null,
|
||
},
|
||
],
|
||
});
|
||
} catch (saveError) {
|
||
chatApiLogger.error("[Chat API] Failed to save error message:", saveError);
|
||
}
|
||
} finally {
|
||
stopKeepalive?.();
|
||
}
|
||
},
|
||
});
|
||
|
||
return createSseResponse(alibabaStream);
|
||
}
|
||
// ========== 阿里云图片生成处理结束 ==========
|
||
|
||
// ========== 火山引擎视觉图片生成独立处理 ==========
|
||
if ((languageModel as any).provider === "volcano-vision" && typeof (languageModel as any).doGenerate === "function") {
|
||
chatApiLogger.info("[Chat API] Using Volcano Vision image generation");
|
||
|
||
// 🆕 检测是否有标注图片，如果有则在消息中注入系统提示
|
||
const hasAnnotation = hasAnnotatedImage(uiMessages);
|
||
let messagesForVolcano = applyOptimizedPromptToLatestUserMessage(
|
||
uiMessagesForModel,
|
||
smartAutoOptimizedPrompt,
|
||
hiddenPromptPrefix,
|
||
);
|
||
const volcanoModelConfig = await getAiModelById({ id: finalModelId });
|
||
const configuredFallbacks = [
|
||
volcanoModelConfig?.fallbackModelId1,
|
||
volcanoModelConfig?.fallbackModelId2,
|
||
].filter(Boolean) as string[];
|
||
const allModels = await getEnabledAiModelsWithProvider();
|
||
const fallbackCandidates = buildImageFallbackCandidates(allModels, {
|
||
currentProviderId: volcanoModelConfig?.providerId,
|
||
currentModelId: volcanoModelConfig?.id,
|
||
currentModelKey: actualModelId,
|
||
configuredFallbackIds: configuredFallbacks,
|
||
});
|
||
chatApiLogger.info("[Chat API] Volcano fallback candidates:", fallbackCandidates.map((candidate) => `${candidate.model.displayName}(${candidate.provider.type})`));
|
||
|
||
if (hasAnnotation) {
|
||
chatApiLogger.info("[Chat API] 🎨 Volcano Vision: 检测到标注图片，将注入系统提示");
|
||
// 创建消息副本，修改最后一条用户消息的文本
|
||
messagesForVolcano = messagesForVolcano.map((msg, idx) => {
|
||
if (msg.role === "user" && idx === messagesForVolcano.length - 1) {
|
||
// 修改最后一条用户消息
|
||
const modifiedParts = msg.parts.map((part) => {
|
||
if (part.type === "text" && part.text) {
|
||
return {
|
||
...part,
|
||
text: ANNOTATION_SYSTEM_PROMPT + "\n\n用户请求：" + part.text,
|
||
};
|
||
}
|
||
return part;
|
||
});
|
||
return { ...msg, parts: modifiedParts };
|
||
}
|
||
return msg;
|
||
});
|
||
}
|
||
|
||
messagesForVolcano = injectHistoryImagesToLastUserMessage(messagesForVolcano, smartAutoHistoryImages);
|
||
|
||
const volcanoStream = createUIMessageStream({
|
||
execute: async ({ writer }) => {
|
||
let stopKeepalive: (() => void) | null = null;
|
||
try {
|
||
// 🆕 如果是智能模式且有对话回复，先发送对话消息让用户看到
|
||
const conversationMessage = smartAutoFriendlyMessage || null;
|
||
|
||
if (conversationMessage) {
|
||
const chatMessageId = generateUUID();
|
||
chatApiLogger.info("[Chat API] 💬 Volcano: Sending conversational reply first:", smartAutoFriendlyMessage);
|
||
|
||
// 发送带标记的对话消息
|
||
writeAssistantTextChunk(
|
||
writer,
|
||
conversationMessage,
|
||
"volcano:conversation",
|
||
chatMessageId,
|
||
);
|
||
|
||
// 保存对话消息到数据库（不带标记，纯对话）
|
||
await saveMessages({
|
||
messages: [
|
||
{
|
||
id: chatMessageId,
|
||
chatId: id,
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: smartAutoFriendlyMessage }],
|
||
attachments: [],
|
||
createdAt: new Date(),
|
||
videoStorageStatus: null,
|
||
temporaryVideoUrl: null,
|
||
permanentVideoUrl: null,
|
||
archivedAt: null,
|
||
videoMetadata: null,
|
||
},
|
||
],
|
||
});
|
||
}
|
||
|
||
// 转换消息，火山引擎支持 URL 和 base64
|
||
const modelMessages = await convertMessagesWithImages(messagesForVolcano, "volcano");
|
||
|
||
const tryGenerateImage = async (modelIdToUse: string, isRetry: boolean) => {
|
||
const result = await (languageModel as any).doGenerate({
|
||
messages: modelMessages,
|
||
imageParams:
|
||
effectiveImageParams || {
|
||
aspectRatio: defaultImageQualityParams.aspectRatio,
|
||
imageSize: defaultImageQualityParams.imageSize,
|
||
imageCount: defaultImageQualityParams.imageCount,
|
||
},
|
||
overrideModelId: isRetry ? modelIdToUse : undefined,
|
||
});
|
||
if (result?.finishReason === "error") {
|
||
throw new Error(result.text || "图片生成失败");
|
||
}
|
||
return result.text || "图片生成失败";
|
||
};
|
||
|
||
let imageMarkdown = "";
|
||
let usedModelId = actualModelId;
|
||
|
||
// 🆕 图片生成真正开始时发出标记（用于前端显示占位图）
|
||
writeAssistantTextChunk(
|
||
writer,
|
||
"<!-- IMAGE_GENERATION_START -->",
|
||
"volcano:image-start",
|
||
);
|
||
if (contextPayload?.source === "canvas") {
|
||
writeImageGenerationStatusChunk(
|
||
writer,
|
||
{
|
||
phase: "processing",
|
||
providerType: "volcano-vision",
|
||
modelName: volcanoModelConfig?.displayName || actualModelId,
|
||
imageCount: effectiveImageParams?.imageCount || defaultImageQualityParams.imageCount || 1,
|
||
},
|
||
"volcano:image-status-start",
|
||
);
|
||
}
|
||
stopKeepalive = startStreamKeepalive(writer, "volcano-image");
|
||
try {
|
||
imageMarkdown = await tryGenerateImage(actualModelId, false);
|
||
} catch (primaryError) {
|
||
chatApiLogger.error("[Chat API] Volcano primary model failed:", primaryError);
|
||
let fallbackSuccess = false;
|
||
for (const candidate of fallbackCandidates) {
|
||
try {
|
||
chatApiLogger.info(`[Chat API] 🔄 Volcano fallback model: ${candidate.model.displayName} (${candidate.provider.type})`);
|
||
const fallbackResult = await generateImageWithFailoverCandidate({
|
||
candidate,
|
||
messages: messagesForVolcano,
|
||
imageParams: effectiveImageParams,
|
||
fallbackImageParams: {
|
||
aspectRatio: defaultImageQualityParams.aspectRatio,
|
||
imageSize: defaultImageQualityParams.imageSize,
|
||
imageCount: defaultImageQualityParams.imageCount,
|
||
},
|
||
});
|
||
imageMarkdown = fallbackResult.text;
|
||
usedModelId = fallbackResult.usedModelId;
|
||
fallbackSuccess = true;
|
||
chatApiLogger.info(`[Chat API] ✅ Volcano fallback model ${fallbackResult.displayName} succeeded!`);
|
||
break;
|
||
} catch (fallbackError) {
|
||
const fallbackErrorMessage = normalizeFailoverErrorMessage(fallbackError);
|
||
if (shouldCooldownImageFallbackModel(fallbackErrorMessage)) {
|
||
setImageFallbackModelCooldown(candidate.model.id, fallbackErrorMessage);
|
||
}
|
||
chatApiLogger.error(`[Chat API] ❌ Volcano fallback model ${candidate.model.displayName} (${candidate.provider.type}) failed:`, fallbackError);
|
||
}
|
||
}
|
||
if (!fallbackSuccess) {
|
||
const providerPriority: Record<string, number> = {
|
||
"alibaba-image": 0,
|
||
"bfl-image": 1,
|
||
"ruxa-image": 2,
|
||
"fal-image": 3,
|
||
"openai-image": 4,
|
||
"stepfun-image": 5,
|
||
runway: 6,
|
||
krea: 7,
|
||
gemini: 7,
|
||
"huanke-gemini": 8,
|
||
"volcano-vision": 9,
|
||
};
|
||
const crossProviderCandidates = getImageCrossProviderCandidates(allModels, {
|
||
currentModelId: finalModelId,
|
||
currentProviderId: volcanoModelConfig?.providerId,
|
||
providerPriority,
|
||
});
|
||
|
||
for (const candidate of crossProviderCandidates) {
|
||
try {
|
||
chatApiLogger.info(`[Chat API] 🔁 Volcano cross-provider failover attempt: ${candidate.model.displayName} (${candidate.provider.type})`);
|
||
if (
|
||
candidate.provider.type === "fal-image" &&
|
||
isFalGptImage2EditModelId(
|
||
String(candidate.model?.modelId || candidate.model?.id || "")
|
||
)
|
||
) {
|
||
chatApiLogger.info(`[Chat API] Skip async-only failover candidate: ${candidate.model.displayName}`);
|
||
continue;
|
||
}
|
||
|
||
const candidateLanguageModel = await getLanguageModelById(candidate.model.id);
|
||
if (!candidateLanguageModel || typeof (candidateLanguageModel as any).doGenerate !== "function") {
|
||
chatApiLogger.info(`[Chat API] Skip candidate without doGenerate: ${candidate.model.id}`);
|
||
continue;
|
||
}
|
||
|
||
const convertProvider =
|
||
candidate.provider.type === "bfl-image"
|
||
? "bfl-image"
|
||
: candidate.provider.type === "ruxa-image"
|
||
? "ruxa-image"
|
||
: candidate.provider.type === "fal-image"
|
||
? "fal-image"
|
||
: candidate.provider.type === "openai-image"
|
||
? "openai-image"
|
||
: candidate.provider.type === "stepfun-image"
|
||
? "stepfun-image"
|
||
: candidate.provider.type === "midjourney"
|
||
? "midjourney"
|
||
: candidate.provider.type === "runway"
|
||
? "runway"
|
||
: candidate.provider.type === "krea"
|
||
? "krea"
|
||
: candidate.provider.type === "gemini"
|
||
? "gemini"
|
||
: candidate.provider.type === "huanke-gemini"
|
||
? "huanke-gemini"
|
||
: candidate.provider.type === "volcano-vision"
|
||
? "volcano"
|
||
: "alibaba-image";
|
||
|
||
const candidateMessages = await convertMessagesWithImages(
|
||
messagesForVolcano,
|
||
convertProvider
|
||
);
|
||
|
||
const candidateResult = await (candidateLanguageModel as any).doGenerate({
|
||
messages: candidateMessages,
|
||
imageParams:
|
||
effectiveImageParams || {
|
||
aspectRatio: defaultImageQualityParams.aspectRatio,
|
||
imageSize: defaultImageQualityParams.imageSize,
|
||
imageCount: defaultImageQualityParams.imageCount,
|
||
},
|
||
modelDisplayName: candidate.model.displayName,
|
||
overrideModelId: candidate.model.modelId,
|
||
});
|
||
|
||
if (candidateResult?.finishReason === "error") {
|
||
throw new Error(candidateResult.text || "图片生成失败");
|
||
}
|
||
|
||
imageMarkdown = candidateResult.text || "图片生成失败";
|
||
usedModelId = candidate.model.modelId;
|
||
fallbackSuccess = true;
|
||
chatApiLogger.info(`[Chat API] ✅ Volcano cross-provider failover succeeded: ${candidate.model.displayName} (${candidate.provider.type})`);
|
||
break;
|
||
} catch (crossError) {
|
||
const crossErrorMessage = normalizeFailoverErrorMessage(crossError);
|
||
if (shouldCooldownImageFallbackModel(crossErrorMessage)) {
|
||
setImageFallbackModelCooldown(candidate.model.id, crossErrorMessage);
|
||
}
|
||
chatApiLogger.error(`[Chat API] ❌ Volcano cross-provider failover failed: ${candidate.model.displayName} (${candidate.provider.type})`, crossError);
|
||
}
|
||
}
|
||
}
|
||
if (!fallbackSuccess) {
|
||
throw primaryError;
|
||
}
|
||
}
|
||
chatApiLogger.info("[Chat API] Volcano Vision generated, length:", imageMarkdown.length);
|
||
chatApiLogger.info("[Chat API] Volcano Vision image markdown:", imageMarkdown.substring(0, 200));
|
||
|
||
const usedModelDisplayName =
|
||
resolveModelDisplayNameByModelId(allModels, usedModelId) ||
|
||
volcanoModelConfig?.displayName ||
|
||
actualModelId;
|
||
const imageMarkdownWithModel = prependImageModelBadge(
|
||
imageMarkdown,
|
||
usedModelDisplayName,
|
||
);
|
||
|
||
// 生成消息 ID（图片消息）
|
||
const messageId = generateUUID();
|
||
|
||
// 写入图片消息到流
|
||
writeAssistantTextChunk(
|
||
writer,
|
||
imageMarkdownWithModel,
|
||
"volcano:image-result",
|
||
messageId,
|
||
);
|
||
if (contextPayload?.source === "canvas") {
|
||
writeCanvasMediaChunk(
|
||
writer,
|
||
{
|
||
messageId,
|
||
text: imageMarkdownWithModel,
|
||
prompt: requestTextContent,
|
||
sourceImageUrls: requestTargetResolution?.targetImageUrls || [],
|
||
},
|
||
"volcano:image-canvas-media",
|
||
);
|
||
}
|
||
|
||
// 🆕 图片生成结束标记（成功）
|
||
writeAssistantTextChunk(
|
||
writer,
|
||
"<!-- IMAGE_GENERATION_END -->",
|
||
"volcano:image-end-success",
|
||
);
|
||
if (contextPayload?.source === "canvas") {
|
||
writeImageGenerationStatusChunk(
|
||
writer,
|
||
{
|
||
phase: "completed",
|
||
providerType: "volcano-vision",
|
||
modelName: usedModelDisplayName,
|
||
imageCount: effectiveImageParams?.imageCount || defaultImageQualityParams.imageCount || 1,
|
||
},
|
||
"volcano:image-status-completed",
|
||
);
|
||
}
|
||
|
||
// 保存图片消息到数据库
|
||
try {
|
||
await saveMessages({
|
||
messages: [
|
||
{
|
||
id: messageId,
|
||
chatId: id,
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: imageMarkdownWithModel }],
|
||
attachments: [], // 必须提供 attachments 字段（数据库 notNull）
|
||
createdAt: new Date(),
|
||
videoStorageStatus: null,
|
||
temporaryVideoUrl: null,
|
||
permanentVideoUrl: null,
|
||
archivedAt: null,
|
||
videoMetadata: null,
|
||
},
|
||
],
|
||
});
|
||
chatApiLogger.info("[Chat API] ✅ Volcano Vision message saved successfully!");
|
||
|
||
// 提取用户 prompt
|
||
const lastUserMessage = uiMessages.filter((m) => m.role === "user").pop();
|
||
const userPrompt = lastUserMessage?.parts?.find((p): p is { type: "text"; text: string } => p.type === "text")?.text || "";
|
||
|
||
// 保存图片记录到 GeneratedImage 表
|
||
await saveGeneratedImages(
|
||
imageMarkdownWithModel,
|
||
session.user.id,
|
||
id,
|
||
messageId,
|
||
userPrompt,
|
||
resolveModelRefByRuntimeModelId(
|
||
allModels,
|
||
usedModelId,
|
||
usedModelId === actualModelId ? finalModelId : undefined,
|
||
),
|
||
{
|
||
sourceAssetIds: requestTargetResolution?.targetAssetIds,
|
||
sourceImageUrls: requestTargetResolution?.targetImageUrls,
|
||
relationType: requestResolvedMediaInput?.hasImage ? "edited_from" : "generated_from",
|
||
sourceType: "generated",
|
||
projectId: contextPayload?.projectId || null,
|
||
optimizedPrompt: smartAutoOptimizedPrompt || userPrompt,
|
||
requestedImageCount: effectiveImageParams?.imageCount,
|
||
aspectRatio: effectiveImageParams?.aspectRatio,
|
||
imageSize: effectiveImageParams?.imageSize,
|
||
},
|
||
);
|
||
} catch (saveError) {
|
||
chatApiLogger.error("[Chat API] ❌ Failed to save Volcano Vision message to database:", saveError);
|
||
chatApiLogger.error("[Chat API] Message content that failed:", imageMarkdown.substring(0, 300));
|
||
}
|
||
} catch (error) {
|
||
chatApiLogger.error("[Chat API] Volcano Vision generation error:", error);
|
||
const errorMessageId = generateUUID();
|
||
writeAssistantTextChunk(
|
||
writer,
|
||
"图片生成失败，请稍后重试",
|
||
"volcano:error",
|
||
errorMessageId,
|
||
);
|
||
|
||
// 🆕 图片生成结束标记（失败）
|
||
writeAssistantTextChunk(
|
||
writer,
|
||
"<!-- IMAGE_GENERATION_END -->",
|
||
"volcano:image-end-failed",
|
||
);
|
||
|
||
// 尝试保存错误消息
|
||
try {
|
||
await saveMessages({
|
||
messages: [
|
||
{
|
||
id: errorMessageId,
|
||
chatId: id,
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: "图片生成失败，请稍后重试" }],
|
||
attachments: [], // 必须提供 attachments 字段
|
||
createdAt: new Date(),
|
||
videoStorageStatus: null,
|
||
temporaryVideoUrl: null,
|
||
permanentVideoUrl: null,
|
||
archivedAt: null,
|
||
videoMetadata: null,
|
||
},
|
||
],
|
||
});
|
||
} catch (saveError) {
|
||
chatApiLogger.error("[Chat API] Failed to save error message:", saveError);
|
||
}
|
||
} finally {
|
||
stopKeepalive?.();
|
||
}
|
||
},
|
||
});
|
||
|
||
return createSseResponse(volcanoStream);
|
||
}
|
||
// ========== 火山引擎视觉处理结束 ==========
|
||
|
||
// ========== 千问/FAL/RUXA 视频生成独立处理 ==========
|
||
if (
|
||
["qianwen-video", "fal-video", "ruxa-video", "runway", "krea", "bltcy-video"].includes((languageModel as any).provider) &&
|
||
typeof (languageModel as any).doGenerate === "function"
|
||
) {
|
||
const currentVideoProvider = (languageModel as any).provider as
|
||
| "qianwen-video"
|
||
| "fal-video"
|
||
| "ruxa-video"
|
||
| "runway"
|
||
| "krea"
|
||
| "bltcy-video";
|
||
chatApiLogger.info(`[Chat API] Using ${currentVideoProvider} video generation`);
|
||
|
||
const videoGenerationStream = createUIMessageStream({
|
||
execute: async ({ writer }) => {
|
||
try {
|
||
const conversationMessage = smartAutoFriendlyMessage || null;
|
||
if (conversationMessage) {
|
||
const chatMessageId = generateUUID();
|
||
writer.write({
|
||
type: "data-appendMessage",
|
||
data: JSON.stringify({
|
||
id: chatMessageId,
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: conversationMessage }],
|
||
}),
|
||
});
|
||
|
||
await saveMessages({
|
||
messages: [
|
||
{
|
||
id: chatMessageId,
|
||
chatId: id,
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: conversationMessage }],
|
||
attachments: [],
|
||
createdAt: new Date(),
|
||
videoStorageStatus: null,
|
||
temporaryVideoUrl: null,
|
||
permanentVideoUrl: null,
|
||
archivedAt: null,
|
||
videoMetadata: null,
|
||
},
|
||
],
|
||
});
|
||
}
|
||
|
||
const messagesForVideo = injectHistoryImagesToLastUserMessage(
|
||
applyOptimizedPromptToLatestUserMessage(
|
||
uiMessagesForModel,
|
||
smartAutoOptimizedPrompt,
|
||
hiddenPromptPrefix,
|
||
),
|
||
smartAutoHistoryImages,
|
||
);
|
||
|
||
// 千问/FAL 视频直接使用原始 URL，不转换为 base64
|
||
const modelMessages = convertToModelMessages(messagesForVideo);
|
||
chatApiLogger.info(`[Chat API] ${currentVideoProvider} messages prepared, count:`, modelMessages.length);
|
||
|
||
let videoMarkdown = "视频生成失败";
|
||
let tempVideoUrl = "";
|
||
let billedModelRefs: string[] = [finalModelId];
|
||
let videoMetadata: Record<string, any> = {
|
||
provider: currentVideoProvider,
|
||
model: finalModelId,
|
||
mode: effectiveVideoParams?.mode,
|
||
resolution: effectiveVideoParams?.resolution,
|
||
ratio: effectiveVideoParams?.ratio,
|
||
duration: effectiveVideoParams?.duration,
|
||
status: "succeeded",
|
||
updatedAt: Date.now(),
|
||
};
|
||
|
||
let usedStoryboardPipeline = false;
|
||
if (currentVideoProvider === "ruxa-video") {
|
||
const storyboardConfig = await getVideoStoryboardConfig().catch(() => null);
|
||
if (storyboardConfig?.enabled) {
|
||
const storyboardModelCandidates = await resolveStoryboardModelCandidates(finalModelId);
|
||
const runtimeModelId = String((languageModel as any).modelId || "").trim();
|
||
const selectedModelId =
|
||
runtimeModelId ||
|
||
storyboardModelCandidates.find((item) => item.id === finalModelId)?.modelId ||
|
||
"";
|
||
|
||
const storyboardResult = await runRuxaNineGridPipeline({
|
||
userId: session.user.id,
|
||
chatId: id,
|
||
modelMessages: modelMessages as any,
|
||
selectedModelId,
|
||
selectedModelRef: finalModelId,
|
||
modelCandidates: storyboardModelCandidates,
|
||
videoParams: effectiveVideoParams as any,
|
||
config: storyboardConfig,
|
||
doGenerate: (input) =>
|
||
(languageModel as any).doGenerate({
|
||
messages: input.messages,
|
||
videoParams: input.videoParams,
|
||
overrideModelId: input.overrideModelId,
|
||
}),
|
||
onProgress: (event) => {
|
||
safeWriteStreamChunk(
|
||
writer,
|
||
{
|
||
type: "data-videoStoryboard",
|
||
data: JSON.stringify(event),
|
||
},
|
||
`ruxa-video:storyboard:${event.stage}`,
|
||
);
|
||
},
|
||
});
|
||
|
||
if (storyboardResult.triggered) {
|
||
usedStoryboardPipeline = true;
|
||
videoMarkdown = storyboardResult.markdown || "视频生成失败，请稍后重试";
|
||
tempVideoUrl = storyboardResult.videoUrls?.[0] || "";
|
||
const isAwaitingConfirmation =
|
||
storyboardResult.reason === "awaiting_user_confirmation";
|
||
billedModelRefs =
|
||
storyboardResult.modelRefs && storyboardResult.modelRefs.length > 0
|
||
? storyboardResult.modelRefs
|
||
: [finalModelId];
|
||
|
||
const successCount = storyboardResult.videoUrls?.length || 0;
|
||
videoMetadata = {
|
||
...videoMetadata,
|
||
...(storyboardResult.metadata || {}),
|
||
status: isAwaitingConfirmation
|
||
? "awaiting_user_confirmation"
|
||
: successCount > 0
|
||
? "succeeded"
|
||
: "failed",
|
||
storyboard: {
|
||
enabled: true,
|
||
taskId: storyboardResult.taskId,
|
||
confidence: storyboardResult.confidence,
|
||
successCount,
|
||
awaitingConfirmation: isAwaitingConfirmation,
|
||
},
|
||
updatedAt: Date.now(),
|
||
};
|
||
}
|
||
}
|
||
}
|
||
|
||
if (!usedStoryboardPipeline) {
|
||
const result = await (languageModel as any).doGenerate({
|
||
messages: modelMessages,
|
||
videoParams: effectiveVideoParams,
|
||
});
|
||
|
||
if (result?.finishReason === "error") {
|
||
throw new Error(result.text || "视频生成失败");
|
||
}
|
||
|
||
videoMarkdown = result.text || "视频生成失败";
|
||
tempVideoUrl = extractVideoUrlFromText(videoMarkdown) || "";
|
||
}
|
||
|
||
chatApiLogger.info(`[Chat API] ${currentVideoProvider} video generated, length:`, videoMarkdown.length);
|
||
|
||
// 生成一个统一的消息 ID
|
||
const messageId = generateUUID();
|
||
|
||
// 写入消息到流
|
||
writer.write({
|
||
type: "data-appendMessage",
|
||
data: JSON.stringify({
|
||
id: messageId,
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: videoMarkdown }],
|
||
attachments: [],
|
||
}),
|
||
});
|
||
|
||
// 保存消息到数据库
|
||
try {
|
||
await saveMessages({
|
||
messages: [
|
||
{
|
||
id: messageId,
|
||
chatId: id,
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: videoMarkdown }],
|
||
attachments: [],
|
||
createdAt: new Date(),
|
||
videoStorageStatus: tempVideoUrl ? "temporary" : null,
|
||
temporaryVideoUrl: tempVideoUrl || null,
|
||
permanentVideoUrl: null,
|
||
archivedAt: null,
|
||
videoMetadata: videoMetadata as any,
|
||
},
|
||
],
|
||
});
|
||
const storyboardTaskId = String((videoMetadata as any)?.taskId || "").trim();
|
||
if (storyboardTaskId) {
|
||
await attachVideoStoryboardTaskMessage({
|
||
taskId: storyboardTaskId,
|
||
messageId,
|
||
}).catch((taskAttachError) => {
|
||
chatApiLogger.warn("[Chat API] Failed to attach storyboard task with message:", taskAttachError);
|
||
});
|
||
}
|
||
chatApiLogger.info(`[Chat API] ✅ ${currentVideoProvider} video message saved successfully!`);
|
||
|
||
if ((videoMetadata as any)?.storyboard?.enabled) {
|
||
try {
|
||
const storyboardGridType = String((videoMetadata as any)?.gridType || "").trim();
|
||
const storyboardShotCount = Number((videoMetadata as any)?.shotCount || 0);
|
||
await persistStoryboardAssistantTurn({
|
||
userId: session.user.id,
|
||
chatId: id,
|
||
projectId: contextPayload?.projectId || null,
|
||
lastMessageId: messageId,
|
||
text: requestTextContent,
|
||
storyboardGridType:
|
||
storyboardGridType === "grid_2x2" ||
|
||
storyboardGridType === "grid_3x2" ||
|
||
storyboardGridType === "grid_3x3"
|
||
? (storyboardGridType as "grid_2x2" | "grid_3x2" | "grid_3x3")
|
||
: undefined,
|
||
storyboardShotCount:
|
||
storyboardShotCount === 4 ||
|
||
storyboardShotCount === 6 ||
|
||
storyboardShotCount === 9
|
||
? storyboardShotCount
|
||
: undefined,
|
||
});
|
||
} catch (storyboardMemoryPersistError) {
|
||
chatApiLogger.warn("[Chat API] Failed to persist storyboard memory after assistant response:", storyboardMemoryPersistError);
|
||
}
|
||
} else {
|
||
try {
|
||
await persistVideoGenerationCompletedTurn({
|
||
userId: session.user.id,
|
||
chatId: id,
|
||
projectId: contextPayload?.projectId || null,
|
||
messageId,
|
||
text: requestTextContent,
|
||
source: "native_video_provider",
|
||
providerType: currentVideoProvider,
|
||
modelRef: finalModelId,
|
||
mode:
|
||
typeof effectiveVideoParams?.mode === "string"
|
||
? effectiveVideoParams.mode
|
||
: undefined,
|
||
taskId: storyboardTaskId || undefined,
|
||
status:
|
||
typeof (videoMetadata as any)?.status === "string"
|
||
? (videoMetadata as any).status
|
||
: tempVideoUrl
|
||
? "completed"
|
||
: undefined,
|
||
temporaryVideoUrl: tempVideoUrl || undefined,
|
||
});
|
||
} catch (videoWorkflowPersistError) {
|
||
chatApiLogger.warn("[Chat API] Failed to persist native video workflow state:", videoWorkflowPersistError);
|
||
}
|
||
}
|
||
const shouldConsumeVideoPoints =
|
||
tempVideoUrl &&
|
||
(videoMetadata as any)?.status !== "awaiting_user_confirmation";
|
||
if (shouldConsumeVideoPoints) {
|
||
await consumeGeneratedVideoPoints({
|
||
userId: session.user.id,
|
||
chatId: id,
|
||
messageId,
|
||
modelRefs: billedModelRefs,
|
||
duration: Number(effectiveVideoParams?.duration) || undefined,
|
||
resolution:
|
||
String(effectiveVideoParams?.resolution || "").trim() ||
|
||
undefined,
|
||
});
|
||
}
|
||
|
||
if (tempVideoUrl) {
|
||
scheduleVideoArchive({
|
||
chatId: id,
|
||
messageId,
|
||
text: videoMarkdown,
|
||
temporaryVideoUrl: tempVideoUrl,
|
||
taskId: messageId,
|
||
metadata: videoMetadata,
|
||
});
|
||
}
|
||
} catch (saveError) {
|
||
chatApiLogger.error(`[Chat API] ❌ Failed to save ${currentVideoProvider} video message:`, saveError);
|
||
}
|
||
} catch (error) {
|
||
chatApiLogger.error(`[Chat API] ${currentVideoProvider} video generation error:`, error);
|
||
const errorMessageId = generateUUID();
|
||
const errorText = "视频生成失败，请稍后重试";
|
||
writer.write({
|
||
type: "data-appendMessage",
|
||
data: JSON.stringify({
|
||
id: errorMessageId,
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: errorText }],
|
||
attachments: [],
|
||
}),
|
||
});
|
||
try {
|
||
await saveMessages({
|
||
messages: [
|
||
{
|
||
id: errorMessageId,
|
||
chatId: id,
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: errorText }],
|
||
attachments: [],
|
||
createdAt: new Date(),
|
||
videoStorageStatus: null,
|
||
temporaryVideoUrl: null,
|
||
permanentVideoUrl: null,
|
||
archivedAt: null,
|
||
videoMetadata: null,
|
||
},
|
||
],
|
||
});
|
||
await persistVideoGenerationFailedTurn({
|
||
userId: session.user.id,
|
||
chatId: id,
|
||
projectId: contextPayload?.projectId || null,
|
||
messageId: errorMessageId,
|
||
text: requestTextContent,
|
||
source: "native_video_provider",
|
||
providerType: currentVideoProvider,
|
||
modelRef: finalModelId,
|
||
mode:
|
||
typeof effectiveVideoParams?.mode === "string"
|
||
? effectiveVideoParams.mode
|
||
: undefined,
|
||
status: "failed",
|
||
error: error instanceof Error ? error.message : String(error),
|
||
});
|
||
} catch (videoFailurePersistError) {
|
||
chatApiLogger.warn("[Chat API] Failed to persist native video failure state:", videoFailurePersistError);
|
||
}
|
||
}
|
||
},
|
||
});
|
||
|
||
return createSseResponse(videoGenerationStream);
|
||
}
|
||
// ========== 千问/FAL/RUXA 视频处理结束 ==========
|
||
|
||
let stopStreamKeepalive: (() => void) | null = null;
|
||
const stream = createUIMessageStream({
|
||
execute: async ({ writer: dataStream }) => {
|
||
const stopDefaultKeepalive = startStreamKeepalive(
|
||
dataStream,
|
||
"chat-default",
|
||
DEFAULT_STREAM_KEEPALIVE_INTERVAL_MS
|
||
);
|
||
stopStreamKeepalive = stopDefaultKeepalive;
|
||
setConversationStateProtocolEventObserver((event) => {
|
||
writeProtocolDebugChunk(
|
||
dataStream,
|
||
Boolean(debugProtocol),
|
||
{
|
||
phase: "event",
|
||
type: event.type,
|
||
payload: event.payload,
|
||
},
|
||
`protocol:event:${event.type}`
|
||
);
|
||
});
|
||
safeWriteStreamChunk(
|
||
dataStream,
|
||
{
|
||
type: "data-keepalive",
|
||
data: JSON.stringify({ ts: Date.now(), label: "chat-default:init" }),
|
||
},
|
||
"chat-default:init"
|
||
);
|
||
writeProtocolDebugChunk(
|
||
dataStream,
|
||
Boolean(debugProtocol),
|
||
{
|
||
phase: "request",
|
||
chatId: id,
|
||
selectedChatModel,
|
||
hasHiddenPrompt: Boolean(hiddenPrompt),
|
||
hasAutoContext: Boolean(autoContext),
|
||
},
|
||
"protocol:request"
|
||
);
|
||
writeCurrentAgentTargetLockChunk(dataStream, "chat-default:target-lock");
|
||
for (const event of pendingProtocolDebugEvents) {
|
||
writeProtocolDebugChunk(
|
||
dataStream,
|
||
Boolean(debugProtocol),
|
||
{
|
||
phase: event.phase,
|
||
...event.payload,
|
||
},
|
||
`protocol:${event.phase}`
|
||
);
|
||
}
|
||
try {
|
||
// 构建工具对象，只包含启用的工具
|
||
const tools: any = {};
|
||
const activeTools: string[] = [];
|
||
|
||
if (enabledTools.includes("getWeather")) {
|
||
tools.getWeather = getWeather;
|
||
activeTools.push("getWeather");
|
||
}
|
||
if (enabledTools.includes("createDocument")) {
|
||
tools.createDocument = createDocument({ session, dataStream });
|
||
activeTools.push("createDocument");
|
||
}
|
||
if (enabledTools.includes("updateDocument")) {
|
||
tools.updateDocument = updateDocument({ session, dataStream });
|
||
activeTools.push("updateDocument");
|
||
}
|
||
if (enabledTools.includes("requestSuggestions")) {
|
||
tools.requestSuggestions = requestSuggestions({
|
||
session,
|
||
dataStream,
|
||
});
|
||
activeTools.push("requestSuggestions");
|
||
}
|
||
const shouldEnableGenerateVideoTool =
|
||
enabledTools.includes("generateVideo") &&
|
||
(!shouldRunAgentConversation || allowAgentVideoGeneration);
|
||
if (shouldEnableGenerateVideoTool) {
|
||
tools.generateVideo = generateVideo({ dataStream });
|
||
activeTools.push("generateVideo");
|
||
} else if (enabledTools.includes("generateVideo") && shouldRunAgentConversation) {
|
||
chatApiLogger.info("[Smart Auto Mode] 🚫 generateVideo tool disabled (manual video mode not selected)");
|
||
}
|
||
|
||
chatApiLogger.info("[Chat API] Calling streamText with model:", (languageModel as any).modelId, (languageModel as any).provider); // 调试日志
|
||
chatApiLogger.info("[Chat API] Provider type:", (languageModel as any).provider); // 调试日志
|
||
|
||
// 转换消息，对于 Gemini 和千问视频将图片 URL 转换为 base64
|
||
const convertedMessages = await convertMessagesWithImages(
|
||
injectHistoryImagesToLastUserMessage(uiMessagesForModel, smartAutoHistoryImages),
|
||
{
|
||
providerType: (languageModel as any).provider || "",
|
||
providerBaseUrl: currentProviderConfig?.baseUrl || null,
|
||
modelId: currentModelConfig?.modelId || actualModelId,
|
||
displayName: currentModelConfig?.displayName || actualModelId,
|
||
forceInlineImageUrls: shouldForceInlineVisionInputs({
|
||
providerType: resolvedProviderType || currentProviderConfig?.type || runtimeProviderType,
|
||
providerBaseUrl: currentProviderConfig?.baseUrl || null,
|
||
modelId: currentModelConfig?.modelId || actualModelId,
|
||
displayName: currentModelConfig?.displayName || actualModelId,
|
||
}),
|
||
loggerPrefix: "[Chat API] Stream text vision inline",
|
||
}
|
||
);
|
||
chatApiLogger.info("[Chat API] Messages converted, count:", convertedMessages.length); // 调试日志
|
||
|
||
// 🆕 检测是否是视频模型
|
||
const providerType = (languageModel as any).provider;
|
||
const effectiveProviderType = resolvedProviderType || providerType;
|
||
chatApiLogger.info("[Chat API] Provider type for tool detection:", {
|
||
runtime: providerType,
|
||
resolved: resolvedProviderType,
|
||
effective: effectiveProviderType,
|
||
});
|
||
const isVolcanoVideoModel =
|
||
effectiveProviderType === "volcano-video" ||
|
||
effectiveProviderType === "doubao-video" ||
|
||
actualModelId.includes("seedance");
|
||
const isOpenAICompatImageModel =
|
||
resolvedModelSupportsImageGen &&
|
||
providerType === "openai.chat" &&
|
||
!isVolcanoVideoModel;
|
||
chatApiLogger.info("[Chat API] Is volcano video model:", isVolcanoVideoModel);
|
||
|
||
// 🆕 处理 OpenAI 兼容的生图模型输出（防止返回 data:image;base64 原文）
|
||
if (isOpenAICompatImageModel) {
|
||
chatApiLogger.info("[Chat API] Using generateText for OpenAI-compatible image model");
|
||
const configuredFallbacks = [
|
||
currentModelConfig?.fallbackModelId1,
|
||
currentModelConfig?.fallbackModelId2,
|
||
].filter(Boolean) as string[];
|
||
const allModels = await getEnabledAiModelsWithProvider();
|
||
const fallbackCandidates = buildImageFallbackCandidates(allModels, {
|
||
currentProviderId: currentModelConfig?.providerId,
|
||
currentModelId: currentModelConfig?.id,
|
||
currentModelKey: actualModelId,
|
||
configuredFallbackIds: configuredFallbacks,
|
||
});
|
||
chatApiLogger.info("[Chat API] OpenAI-compatible fallback candidates:", fallbackCandidates.map((candidate) => `${candidate.model.displayName}(${candidate.provider.type})`));
|
||
|
||
const lastUserMessage = uiMessagesForModel.filter((m) => m.role === "user").pop();
|
||
let messagesForCrossProvider = lastUserMessage ? [lastUserMessage] : uiMessagesForModel;
|
||
messagesForCrossProvider = applyOptimizedPromptToLatestUserMessage(
|
||
messagesForCrossProvider,
|
||
smartAutoOptimizedPrompt,
|
||
hiddenPromptPrefix,
|
||
);
|
||
messagesForCrossProvider = injectHistoryImagesToLastUserMessage(
|
||
messagesForCrossProvider,
|
||
smartAutoHistoryImages
|
||
);
|
||
|
||
// 生成开始标记
|
||
dataStream.write({
|
||
type: "data-appendMessage",
|
||
data: JSON.stringify({
|
||
id: generateUUID(),
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: "<!-- IMAGE_GENERATION_START -->" }],
|
||
}),
|
||
});
|
||
if (contextPayload?.source === "canvas") {
|
||
writeImageGenerationStatusChunk(
|
||
dataStream as unknown as StreamWriter,
|
||
{
|
||
phase: "processing",
|
||
providerType: currentModelConfig?.providerId ? resolvedProviderType || providerType : providerType,
|
||
modelName: currentModelConfig?.displayName || actualModelId,
|
||
imageCount: effectiveImageParams?.imageCount || 1,
|
||
},
|
||
"openai-compat:image-status-start",
|
||
);
|
||
}
|
||
|
||
try {
|
||
const tryGenerateWithOpenAICompatModel = async (modelToUse: any) => {
|
||
const imageRequestTimeoutMs = 180_000;
|
||
const generated = await generateText({
|
||
model: modelToUse,
|
||
system: systemPrompt({ selectedChatModel: finalModelId, requestHints }),
|
||
messages: convertedMessages,
|
||
abortSignal: AbortSignal.timeout(imageRequestTimeoutMs),
|
||
});
|
||
|
||
const rawText = generated.text?.trim() || "图片生成失败，请稍后重试";
|
||
return normalizeAssistantImageMarkdown(
|
||
rawText,
|
||
effectiveImageParams?.imageSize || "2K"
|
||
);
|
||
};
|
||
|
||
let normalizedText = "";
|
||
let usedModelId = actualModelId;
|
||
|
||
try {
|
||
normalizedText = await tryGenerateWithOpenAICompatModel(languageModel);
|
||
} catch (primaryError) {
|
||
chatApiLogger.error("[Chat API] OpenAI-compatible primary model failed:", primaryError);
|
||
|
||
let fallbackSuccess = false;
|
||
for (const candidate of fallbackCandidates) {
|
||
try {
|
||
chatApiLogger.info(`[Chat API] 🔄 OpenAI-compatible fallback model: ${candidate.model.displayName} (${candidate.provider.type})`);
|
||
const shouldUseOpenAICompatFallback =
|
||
candidate.sameProvider &&
|
||
!IMAGE_FAILOVER_PROVIDER_ALLOWLIST.has(String(candidate.provider?.type || ""));
|
||
|
||
if (shouldUseOpenAICompatFallback) {
|
||
const fallbackRuntimeModelId = String(
|
||
candidate.model?.modelId || candidate.model?.id || ""
|
||
);
|
||
const fallbackLanguageModel = await getLanguageModelById(
|
||
finalModelId,
|
||
fallbackRuntimeModelId
|
||
);
|
||
if (!fallbackLanguageModel) {
|
||
chatApiLogger.info(`[Chat API] Skip unavailable OpenAI-compatible fallback model: ${fallbackRuntimeModelId}`);
|
||
continue;
|
||
}
|
||
normalizedText = await tryGenerateWithOpenAICompatModel(fallbackLanguageModel);
|
||
usedModelId = fallbackRuntimeModelId;
|
||
} else {
|
||
const fallbackResult = await generateImageWithFailoverCandidate({
|
||
candidate,
|
||
messages: messagesForCrossProvider,
|
||
imageParams: effectiveImageParams,
|
||
fallbackImageParams: {
|
||
aspectRatio: defaultImageQualityParams.aspectRatio,
|
||
imageSize: defaultImageQualityParams.imageSize,
|
||
imageCount: defaultImageQualityParams.imageCount,
|
||
},
|
||
});
|
||
normalizedText = fallbackResult.text;
|
||
usedModelId = fallbackResult.usedModelId;
|
||
}
|
||
fallbackSuccess = true;
|
||
chatApiLogger.info(`[Chat API] ✅ OpenAI-compatible fallback model ${candidate.model.displayName} succeeded!`);
|
||
break;
|
||
} catch (fallbackError) {
|
||
const fallbackErrorMessage = normalizeFailoverErrorMessage(fallbackError);
|
||
if (shouldCooldownImageFallbackModel(fallbackErrorMessage)) {
|
||
setImageFallbackModelCooldown(candidate.model.id, fallbackErrorMessage);
|
||
}
|
||
chatApiLogger.error(`[Chat API] ❌ OpenAI-compatible fallback model ${candidate.model.displayName} (${candidate.provider.type}) failed:`, fallbackError);
|
||
}
|
||
}
|
||
|
||
if (!fallbackSuccess) {
|
||
const providerPriority: Record<string, number> = {
|
||
"volcano-vision": 0,
|
||
"alibaba-image": 1,
|
||
"bfl-image": 2,
|
||
"ruxa-image": 3,
|
||
"fal-image": 4,
|
||
"openai-image": 5,
|
||
"stepfun-image": 6,
|
||
runway: 7,
|
||
krea: 8,
|
||
gemini: 8,
|
||
"huanke-gemini": 9,
|
||
};
|
||
const crossProviderCandidates = getImageCrossProviderCandidates(allModels, {
|
||
currentModelId: finalModelId,
|
||
currentProviderId: currentModelConfig?.providerId,
|
||
providerPriority,
|
||
});
|
||
|
||
for (const candidate of crossProviderCandidates) {
|
||
try {
|
||
chatApiLogger.info(`[Chat API] 🔁 OpenAI-compatible cross-provider failover: ${candidate.model.displayName} (${candidate.provider.type})`);
|
||
if (
|
||
candidate.provider.type === "fal-image" &&
|
||
isFalGptImage2EditModelId(
|
||
String(candidate.model?.modelId || candidate.model?.id || "")
|
||
)
|
||
) {
|
||
chatApiLogger.info(`[Chat API] Skip async-only failover candidate: ${candidate.model.displayName}`);
|
||
continue;
|
||
}
|
||
|
||
const candidateLanguageModel = await getLanguageModelById(candidate.model.id);
|
||
if (
|
||
!candidateLanguageModel ||
|
||
typeof (candidateLanguageModel as any).doGenerate !== "function"
|
||
) {
|
||
chatApiLogger.info(`[Chat API] Skip cross-provider candidate without doGenerate: ${candidate.model.id}`);
|
||
continue;
|
||
}
|
||
|
||
const convertProvider =
|
||
candidate.provider.type === "bfl-image"
|
||
? "bfl-image"
|
||
: candidate.provider.type === "ruxa-image"
|
||
? "ruxa-image"
|
||
: candidate.provider.type === "fal-image"
|
||
? "fal-image"
|
||
: candidate.provider.type === "openai-image"
|
||
? "openai-image"
|
||
: candidate.provider.type === "stepfun-image"
|
||
? "stepfun-image"
|
||
: candidate.provider.type === "midjourney"
|
||
? "midjourney"
|
||
: candidate.provider.type === "runway"
|
||
? "runway"
|
||
: candidate.provider.type === "krea"
|
||
? "krea"
|
||
: candidate.provider.type === "gemini"
|
||
? "gemini"
|
||
: candidate.provider.type === "huanke-gemini"
|
||
? "huanke-gemini"
|
||
: candidate.provider.type === "volcano-vision"
|
||
? "volcano"
|
||
: "alibaba-image";
|
||
|
||
const candidateMessages = await convertMessagesWithImages(
|
||
messagesForCrossProvider,
|
||
convertProvider
|
||
);
|
||
|
||
const candidateResult = await (candidateLanguageModel as any).doGenerate({
|
||
messages: candidateMessages,
|
||
imageParams:
|
||
effectiveImageParams || {
|
||
aspectRatio: defaultImageQualityParams.aspectRatio,
|
||
imageSize: defaultImageQualityParams.imageSize,
|
||
imageCount: defaultImageQualityParams.imageCount,
|
||
},
|
||
modelDisplayName: candidate.model.displayName,
|
||
overrideModelId: candidate.model.modelId,
|
||
});
|
||
|
||
if (candidateResult?.finishReason === "error") {
|
||
throw new Error(candidateResult.text || "图片生成失败");
|
||
}
|
||
|
||
normalizedText = candidateResult.text || "图片生成失败";
|
||
usedModelId = candidate.model.modelId;
|
||
fallbackSuccess = true;
|
||
chatApiLogger.info(`[Chat API] ✅ OpenAI-compatible cross-provider failover succeeded: ${candidate.model.displayName} (${candidate.provider.type})`);
|
||
break;
|
||
} catch (crossError) {
|
||
const crossErrorMessage = normalizeFailoverErrorMessage(crossError);
|
||
if (shouldCooldownImageFallbackModel(crossErrorMessage)) {
|
||
setImageFallbackModelCooldown(candidate.model.id, crossErrorMessage);
|
||
}
|
||
chatApiLogger.error(`[Chat API] ❌ OpenAI-compatible cross-provider failover failed: ${candidate.model.displayName} (${candidate.provider.type})`, crossError);
|
||
}
|
||
}
|
||
}
|
||
|
||
if (!fallbackSuccess) {
|
||
throw primaryError;
|
||
}
|
||
}
|
||
|
||
const usedModelDisplayName =
|
||
resolveModelDisplayNameByModelId(allModels, usedModelId) ||
|
||
currentModelConfig?.displayName ||
|
||
actualModelId;
|
||
const normalizedTextWithModel = prependImageModelBadge(
|
||
normalizedText,
|
||
usedModelDisplayName,
|
||
);
|
||
|
||
const messageId = generateUUID();
|
||
dataStream.write({
|
||
type: "data-appendMessage",
|
||
data: JSON.stringify({
|
||
id: messageId,
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: normalizedTextWithModel }],
|
||
}),
|
||
});
|
||
if (contextPayload?.source === "canvas") {
|
||
writeCanvasMediaChunk(
|
||
dataStream as unknown as StreamWriter,
|
||
{
|
||
messageId,
|
||
text: normalizedTextWithModel,
|
||
prompt: requestTextContent,
|
||
sourceImageUrls: requestTargetResolution?.targetImageUrls || [],
|
||
},
|
||
"openai-compat:image-canvas-media",
|
||
);
|
||
}
|
||
|
||
// 生成结束标记
|
||
dataStream.write({
|
||
type: "data-appendMessage",
|
||
data: JSON.stringify({
|
||
id: generateUUID(),
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: "<!-- IMAGE_GENERATION_END -->" }],
|
||
}),
|
||
});
|
||
if (contextPayload?.source === "canvas") {
|
||
writeImageGenerationStatusChunk(
|
||
dataStream as unknown as StreamWriter,
|
||
{
|
||
phase: "completed",
|
||
providerType: resolvedProviderType || providerType,
|
||
modelName: usedModelDisplayName,
|
||
imageCount: effectiveImageParams?.imageCount || 1,
|
||
},
|
||
"openai-compat:image-status-completed",
|
||
);
|
||
}
|
||
|
||
await saveMessages({
|
||
messages: [
|
||
{
|
||
id: messageId,
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: normalizedTextWithModel }],
|
||
createdAt: new Date(),
|
||
attachments: [],
|
||
chatId: id,
|
||
videoStorageStatus: null,
|
||
temporaryVideoUrl: null,
|
||
permanentVideoUrl: null,
|
||
archivedAt: null,
|
||
videoMetadata: null,
|
||
},
|
||
],
|
||
});
|
||
|
||
const latestUserMessage = uiMessages.filter((m) => m.role === "user").pop();
|
||
const userPrompt =
|
||
latestUserMessage?.parts?.find(
|
||
(p): p is { type: "text"; text: string } => p.type === "text"
|
||
)?.text || "";
|
||
await saveGeneratedImages(
|
||
normalizedTextWithModel,
|
||
session.user.id,
|
||
id,
|
||
messageId,
|
||
userPrompt,
|
||
resolveModelRefByRuntimeModelId(
|
||
allModels,
|
||
usedModelId,
|
||
usedModelId === actualModelId ? finalModelId : undefined,
|
||
),
|
||
{
|
||
sourceAssetIds: requestTargetResolution?.targetAssetIds,
|
||
sourceImageUrls: requestTargetResolution?.targetImageUrls,
|
||
relationType: requestResolvedMediaInput?.hasImage ? "edited_from" : "generated_from",
|
||
sourceType: "generated",
|
||
projectId: contextPayload?.projectId || null,
|
||
optimizedPrompt: smartAutoOptimizedPrompt || userPrompt,
|
||
requestedImageCount: effectiveImageParams?.imageCount,
|
||
aspectRatio: effectiveImageParams?.aspectRatio,
|
||
imageSize: effectiveImageParams?.imageSize,
|
||
},
|
||
);
|
||
return;
|
||
} catch (error) {
|
||
chatApiLogger.error("[Chat API] OpenAI-compatible image model generation error:", error);
|
||
|
||
const errorMessageId = generateUUID();
|
||
const errorText =
|
||
error instanceof Error ? error.message : "图片生成失败，请稍后重试";
|
||
|
||
dataStream.write({
|
||
type: "data-appendMessage",
|
||
data: JSON.stringify({
|
||
id: errorMessageId,
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: errorText }],
|
||
}),
|
||
});
|
||
dataStream.write({
|
||
type: "data-appendMessage",
|
||
data: JSON.stringify({
|
||
id: generateUUID(),
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: "<!-- IMAGE_GENERATION_END -->" }],
|
||
}),
|
||
});
|
||
if (contextPayload?.source === "canvas") {
|
||
writeImageGenerationStatusChunk(
|
||
dataStream as unknown as StreamWriter,
|
||
{
|
||
phase: "failed",
|
||
providerType: resolvedProviderType || providerType,
|
||
modelName: currentModelConfig?.displayName || actualModelId,
|
||
imageCount: effectiveImageParams?.imageCount || 1,
|
||
},
|
||
"openai-compat:image-status-failed",
|
||
);
|
||
}
|
||
|
||
await saveMessages({
|
||
messages: [
|
||
{
|
||
id: errorMessageId,
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: errorText }],
|
||
createdAt: new Date(),
|
||
attachments: [],
|
||
chatId: id,
|
||
videoStorageStatus: null,
|
||
temporaryVideoUrl: null,
|
||
permanentVideoUrl: null,
|
||
archivedAt: null,
|
||
videoMetadata: null,
|
||
},
|
||
],
|
||
});
|
||
return;
|
||
}
|
||
}
|
||
|
||
// 🎬 视频模型使用专门的生成流程
|
||
if (isVolcanoVideoModel) {
|
||
chatApiLogger.info("[Chat API] Using video generation API");
|
||
|
||
// 导入视频生成函数
|
||
const { doGenerateStream } = await import("@/lib/ai/providers/volcano-video");
|
||
|
||
const conversationMessage = smartAutoFriendlyMessage || null;
|
||
if (conversationMessage) {
|
||
const chatMessageId = generateUUID();
|
||
dataStream.write({
|
||
type: "data-appendMessage",
|
||
data: JSON.stringify({
|
||
id: chatMessageId,
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: conversationMessage }],
|
||
attachments: [],
|
||
}),
|
||
});
|
||
|
||
await saveMessages({
|
||
messages: [
|
||
{
|
||
id: chatMessageId,
|
||
chatId: id,
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: conversationMessage }],
|
||
attachments: [],
|
||
createdAt: new Date(),
|
||
videoStorageStatus: null,
|
||
temporaryVideoUrl: null,
|
||
permanentVideoUrl: null,
|
||
archivedAt: null,
|
||
videoMetadata: null,
|
||
},
|
||
],
|
||
});
|
||
}
|
||
|
||
const messagesForVideo = injectHistoryImagesToLastUserMessage(
|
||
applyOptimizedPromptToLatestUserMessage(
|
||
uiMessagesForModel,
|
||
smartAutoOptimizedPrompt,
|
||
hiddenPromptPrefix,
|
||
),
|
||
smartAutoHistoryImages,
|
||
);
|
||
|
||
// 从最后一条用户消息中提取提示词和图片
|
||
const lastUserMessage = messagesForVideo.filter(m => m.role === "user").pop();
|
||
let prompt = "";
|
||
const imageUrls: string[] = [];
|
||
const summarizeImageRef = (rawUrl: string) => {
|
||
const trimmed = String(rawUrl || "").trim();
|
||
if (!trimmed) return "";
|
||
if (trimmed.startsWith("data:image/")) {
|
||
return `data:image(base64,${trimmed.length} chars)`;
|
||
}
|
||
try {
|
||
const parsed = new URL(trimmed);
|
||
return `${parsed.origin}${parsed.pathname}`;
|
||
} catch {
|
||
return trimmed.slice(0, 120);
|
||
}
|
||
};
|
||
|
||
if (lastUserMessage) {
|
||
// 🆕 处理 parts 数组（AI SDK v4 格式）
|
||
if ((lastUserMessage as any).parts && Array.isArray((lastUserMessage as any).parts)) {
|
||
chatApiLogger.info("[Chat API] Video input parts:", (lastUserMessage as any).parts.length);
|
||
|
||
for (const part of (lastUserMessage as any).parts) {
|
||
if (part.type === "text" && part.text) {
|
||
// 提取文本
|
||
prompt += part.text;
|
||
} else if (part.type === "file" && part.url && part.mediaType?.startsWith("image/")) {
|
||
// 提取图片
|
||
imageUrls.push(part.url);
|
||
}
|
||
}
|
||
}
|
||
// 备用方案1：content 字符串/数组
|
||
const legacyContent = (lastUserMessage as any).content;
|
||
if (typeof legacyContent === "string") {
|
||
prompt = legacyContent;
|
||
} else if (Array.isArray(legacyContent)) {
|
||
for (const part of legacyContent) {
|
||
if (typeof part === "string") {
|
||
prompt += part;
|
||
} else if (part.type === "text") {
|
||
prompt += part.text;
|
||
} else if (part.type === "image" && (part as any).image) {
|
||
imageUrls.push((part as any).image);
|
||
}
|
||
}
|
||
} else if ((lastUserMessage as any).experimental_attachments) {
|
||
// 备用方案2：experimental_attachments
|
||
for (const attachment of (lastUserMessage as any).experimental_attachments) {
|
||
if (attachment.url && attachment.contentType?.startsWith("image/")) {
|
||
imageUrls.push(attachment.url);
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
chatApiLogger.info("[Chat API] Video input extracted:", {
|
||
promptLength: prompt.trim().length,
|
||
imageCount: imageUrls.length,
|
||
imageRefs: imageUrls.slice(0, 4).map(summarizeImageRef),
|
||
});
|
||
|
||
const videoMessageId = generateUUID();
|
||
const videoStartAt = Date.now();
|
||
const baseVideoMetadata: Record<string, any> = {
|
||
provider: effectiveProviderType || "volcano-video",
|
||
mode: effectiveVideoParams?.mode || "text-to-video",
|
||
resolution: effectiveVideoParams?.resolution || "720p",
|
||
ratio: effectiveVideoParams?.ratio || "16:9",
|
||
duration: effectiveVideoParams?.duration || 5,
|
||
status: "queued",
|
||
startedAt: videoStartAt,
|
||
updatedAt: videoStartAt,
|
||
};
|
||
|
||
try {
|
||
chatApiLogger.info("[Chat API] Video generation params:", {
|
||
promptLength: prompt.trim().length,
|
||
imageCount: imageUrls.length,
|
||
videoParams: effectiveVideoParams,
|
||
});
|
||
|
||
const videoMode = (effectiveVideoParams?.mode || "text-to-video") as VideoModeType;
|
||
|
||
// 🆕 重新执行子模型选择逻辑，确定要使用的模型 UUID
|
||
let videoModelId = finalModelId; // 默认使用选择的模型 ID
|
||
|
||
// 检查是否是主模型，需要选择子模型（仅 volcano-video 支持子模型）
|
||
const selectedModelData = await getAiModelById({ id: finalModelId });
|
||
const selectedModelProvider = selectedModelData
|
||
? await getAiProviderById({ id: selectedModelData.providerId }).catch(() => null)
|
||
: null;
|
||
if (
|
||
selectedModelData &&
|
||
!selectedModelData.isSubModel &&
|
||
selectedModelProvider?.type === "volcano-video"
|
||
) {
|
||
const videoSubModels = await getSubModelsByParentModelId(finalModelId);
|
||
chatApiLogger.info(`[Chat API] Video: Found ${videoSubModels.length} sub-models for master model`);
|
||
|
||
if (videoSubModels.length > 0) {
|
||
const matchedModel = selectBestVideoSubModelForMode(videoSubModels as any, videoMode);
|
||
|
||
if (matchedModel) {
|
||
videoModelId = matchedModel.model.id;
|
||
chatApiLogger.info(`[Chat API] Video: Selected sub-model ${matchedModel.model.displayName} (${matchedModel.model.modelId}) for mode ${videoMode}`);
|
||
} else {
|
||
chatApiLogger.info(`[Chat API] Video: No compatible sub-model found for mode ${videoMode}, keep current model`);
|
||
}
|
||
}
|
||
}
|
||
|
||
chatApiLogger.info("[Chat API] Getting model config for:", videoModelId);
|
||
const aiModelData = await getAiModelById({ id: videoModelId });
|
||
if (!aiModelData) {
|
||
throw new Error("视频模型配置未找到");
|
||
}
|
||
|
||
// 若用户手动选了不兼容模式的子模型，自动回退到同父模型兼容子模型
|
||
const selectedSupportsMode = getSupportedVideoModesForModel(aiModelData as any).includes(videoMode);
|
||
if (!selectedSupportsMode) {
|
||
let fallbackModel: { model: any } | null = null;
|
||
|
||
if (aiModelData.parentModelId) {
|
||
const siblingSubModels = await getSubModelsByParentModelId(aiModelData.parentModelId);
|
||
fallbackModel = selectBestVideoSubModelForMode(siblingSubModels as any, videoMode);
|
||
} else if (selectedModelProvider?.type === "volcano-video") {
|
||
const enabledModels = await getEnabledAiModelsWithProvider();
|
||
const sameProviderModels = enabledModels
|
||
.filter(
|
||
({ model, provider }) =>
|
||
provider.type === "volcano-video" &&
|
||
model.isEnabled &&
|
||
model.id !== aiModelData.id,
|
||
)
|
||
.map(({ model }) => ({ model }));
|
||
fallbackModel = selectBestVideoSubModelForMode(sameProviderModels as any, videoMode);
|
||
}
|
||
|
||
if (fallbackModel) {
|
||
chatApiLogger.warn(`[Chat API] Video: Model ${aiModelData.modelId} does not support ${videoMode}, fallback to ${fallbackModel.model.modelId}`);
|
||
videoModelId = fallbackModel.model.id;
|
||
}
|
||
}
|
||
|
||
const finalVideoModelData = videoModelId === aiModelData.id
|
||
? aiModelData
|
||
: await getAiModelById({ id: videoModelId });
|
||
if (!finalVideoModelData) {
|
||
throw new Error("视频模型配置未找到");
|
||
}
|
||
|
||
// 如果是子模型，获取父模型的 Provider 配置
|
||
let providerModelData = finalVideoModelData;
|
||
if (finalVideoModelData.isSubModel && finalVideoModelData.parentModelId) {
|
||
chatApiLogger.info("[Chat API] This is a sub-model, getting parent model config:", finalVideoModelData.parentModelId);
|
||
const parentModel = await getAiModelById({ id: finalVideoModelData.parentModelId });
|
||
if (parentModel) {
|
||
providerModelData = parentModel;
|
||
}
|
||
}
|
||
|
||
chatApiLogger.info("[Chat API] Getting provider config for:", providerModelData.providerId);
|
||
const providerData = await getAiProviderById({ id: providerModelData.providerId });
|
||
if (!providerData) {
|
||
throw new Error("Provider 配置未找到");
|
||
}
|
||
|
||
const actualVideoModelId = finalVideoModelData.modelId;
|
||
|
||
// 构建视频生成选项
|
||
const videoOptions: any = {
|
||
model: actualVideoModelId,
|
||
prompt: prompt || "生成视频",
|
||
mode: videoMode,
|
||
resolution: effectiveVideoParams?.resolution || "720p",
|
||
ratio: effectiveVideoParams?.ratio || "16:9",
|
||
duration: effectiveVideoParams?.duration || 5,
|
||
fps: effectiveVideoParams?.fps || 24,
|
||
cameraMove: effectiveVideoParams?.cameraMove || "none",
|
||
cameraFixed: effectiveVideoParams?.cameraFixed || false,
|
||
watermark: effectiveVideoParams?.watermark || false,
|
||
seed: effectiveVideoParams?.seed,
|
||
};
|
||
|
||
// 根据模式添加图片
|
||
if (videoOptions.mode === "first-frame" && imageUrls[0]) {
|
||
videoOptions.firstFrameImage = imageUrls[0];
|
||
} else if (videoOptions.mode === "first-last-frame" && imageUrls.length >= 2) {
|
||
videoOptions.firstFrameImage = imageUrls[0];
|
||
videoOptions.lastFrameImage = imageUrls[1];
|
||
} else if (videoOptions.mode === "reference-images" && imageUrls.length > 0) {
|
||
videoOptions.referenceImages = imageUrls;
|
||
}
|
||
|
||
chatApiLogger.info("[Chat API] Video options constructed:", {
|
||
model: videoOptions.model,
|
||
mode: videoOptions.mode,
|
||
resolution: videoOptions.resolution,
|
||
ratio: videoOptions.ratio,
|
||
duration: videoOptions.duration,
|
||
imageCount:
|
||
videoOptions.referenceImages?.length ||
|
||
Number(Boolean(videoOptions.firstFrameImage)) +
|
||
Number(Boolean(videoOptions.lastFrameImage)),
|
||
});
|
||
|
||
// 先写入占位消息，避免刷新后消息丢失
|
||
const pendingText = "🎬 视频任务已创建，正在处理中，请稍候...";
|
||
await saveMessages({
|
||
messages: [
|
||
{
|
||
id: videoMessageId,
|
||
chatId: id,
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: pendingText }],
|
||
attachments: [],
|
||
createdAt: new Date(),
|
||
videoStorageStatus: "temporary",
|
||
temporaryVideoUrl: null,
|
||
permanentVideoUrl: null,
|
||
archivedAt: null,
|
||
videoMetadata: {
|
||
...baseVideoMetadata,
|
||
model: actualVideoModelId,
|
||
status: "running",
|
||
updatedAt: Date.now(),
|
||
} as any,
|
||
},
|
||
],
|
||
});
|
||
try {
|
||
await persistVideoGenerationStartedTurn({
|
||
userId: session.user.id,
|
||
chatId: id,
|
||
projectId: contextPayload?.projectId || null,
|
||
messageId: videoMessageId,
|
||
text: requestTextContent,
|
||
source: "stream_video_provider",
|
||
providerType: String(providerData.type || "stream"),
|
||
modelRef: finalVideoModelData.id || finalModelId,
|
||
mode: videoOptions.mode,
|
||
status: "running",
|
||
});
|
||
} catch (videoWorkflowStartError) {
|
||
chatApiLogger.warn("[Chat API] Failed to persist stream video start state:", videoWorkflowStartError);
|
||
}
|
||
|
||
dataStream.write({
|
||
type: "data-appendMessage",
|
||
data: JSON.stringify({
|
||
id: videoMessageId,
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: pendingText }],
|
||
attachments: [],
|
||
}),
|
||
});
|
||
|
||
const apiKey = providerData.apiKey;
|
||
const baseURL = providerData.baseUrl || "https://ark.cn-beijing.volces.com/api/v3";
|
||
chatApiLogger.info("[Chat API] Provider configured:", { baseURL, hasApiKey: !!apiKey });
|
||
|
||
// 调用视频生成流
|
||
chatApiLogger.info("[Chat API] Starting video generation stream...");
|
||
const videoStream = doGenerateStream(apiKey, baseURL, videoOptions);
|
||
|
||
// 收集所有流输出
|
||
chatApiLogger.info("[Chat API] Processing video stream...");
|
||
let fullText = "";
|
||
let chunkCount = 0;
|
||
|
||
for await (const chunk of videoStream) {
|
||
chunkCount++;
|
||
fullText += chunk;
|
||
chatApiLogger.info(`[Chat API] Received chunk ${chunkCount}, length: ${chunk.length}`);
|
||
}
|
||
|
||
chatApiLogger.info(`[Chat API] Video stream completed. Total chunks: ${chunkCount}, total text length: ${fullText.length}`);
|
||
|
||
// 提取流中的元数据并清理输出文案
|
||
const metadataMatch = fullText.match(/<!--\s*VIDEO_METADATA:(.*?)-->/s);
|
||
let streamMetadata: Record<string, any> = {};
|
||
if (metadataMatch?.[1]) {
|
||
try {
|
||
streamMetadata = JSON.parse(metadataMatch[1]);
|
||
} catch (parseError) {
|
||
chatApiLogger.warn("[Chat API] Failed to parse VIDEO_METADATA from stream", parseError);
|
||
}
|
||
}
|
||
const cleanedText = fullText
|
||
.replace(/<!--\s*VIDEO_METADATA:.*?-->/gs, "")
|
||
.trim();
|
||
const videoUrlMatch = cleanedText.match(/<video[^>]+src="([^"]+)"/);
|
||
const temporaryVideoUrl =
|
||
(typeof streamMetadata.temporaryVideoUrl === "string" && streamMetadata.temporaryVideoUrl) ||
|
||
(videoUrlMatch ? videoUrlMatch[1] : null);
|
||
|
||
const successMetadata = {
|
||
...baseVideoMetadata,
|
||
...streamMetadata,
|
||
model: actualVideoModelId,
|
||
taskId:
|
||
typeof streamMetadata.taskId === "string" && streamMetadata.taskId
|
||
? streamMetadata.taskId
|
||
: undefined,
|
||
status: "succeeded",
|
||
completedAt: Date.now(),
|
||
updatedAt: Date.now(),
|
||
};
|
||
|
||
await updateMessageById({
|
||
messageId: videoMessageId,
|
||
chatId: id,
|
||
parts: [{ type: "text", text: cleanedText }],
|
||
videoStorageStatus: temporaryVideoUrl ? "temporary" : null,
|
||
temporaryVideoUrl: temporaryVideoUrl,
|
||
permanentVideoUrl: null,
|
||
archivedAt: null,
|
||
videoMetadata: successMetadata as any,
|
||
});
|
||
try {
|
||
await persistVideoGenerationCompletedTurn({
|
||
userId: session.user.id,
|
||
chatId: id,
|
||
projectId: contextPayload?.projectId || null,
|
||
messageId: videoMessageId,
|
||
text: requestTextContent,
|
||
source: "stream_video_provider",
|
||
providerType: String(providerData.type || "stream"),
|
||
modelRef: finalVideoModelData.id || finalModelId,
|
||
mode: videoOptions.mode,
|
||
taskId:
|
||
typeof successMetadata.taskId === "string" && successMetadata.taskId
|
||
? successMetadata.taskId
|
||
: undefined,
|
||
status: "completed",
|
||
temporaryVideoUrl: temporaryVideoUrl || undefined,
|
||
});
|
||
} catch (videoWorkflowCompletedError) {
|
||
chatApiLogger.warn("[Chat API] Failed to persist stream video completion state:", videoWorkflowCompletedError);
|
||
}
|
||
await consumeGeneratedVideoPoints({
|
||
userId: session.user.id,
|
||
chatId: id,
|
||
messageId: videoMessageId,
|
||
modelRefs: [finalVideoModelData.id, providerModelData.id, finalModelId],
|
||
duration: Number(videoOptions.duration) || undefined,
|
||
resolution: String(videoOptions.resolution || "").trim() || undefined,
|
||
});
|
||
|
||
dataStream.write({
|
||
type: "data-appendMessage",
|
||
data: JSON.stringify({
|
||
id: videoMessageId,
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: cleanedText }],
|
||
attachments: [],
|
||
}),
|
||
});
|
||
|
||
chatApiLogger.info("[Chat API] ✅ Video message updated successfully");
|
||
if (temporaryVideoUrl) {
|
||
chatApiLogger.info("[Chat API] 📹 临时视频URL已保存，开始异步归档到S3");
|
||
scheduleVideoArchive({
|
||
chatId: id,
|
||
messageId: videoMessageId,
|
||
text: cleanedText,
|
||
temporaryVideoUrl,
|
||
taskId:
|
||
typeof successMetadata.taskId === "string" && successMetadata.taskId
|
||
? successMetadata.taskId
|
||
: videoMessageId,
|
||
metadata: successMetadata,
|
||
});
|
||
}
|
||
|
||
chatApiLogger.info("[Chat API] Video generation finished successfully");
|
||
return;
|
||
} catch (error: any) {
|
||
chatApiLogger.error("[Chat API] ❌ Video generation error:", error);
|
||
chatApiLogger.error("[Chat API] Error stack:", error.stack);
|
||
|
||
const errorMessage = `❌ 视频生成失败：${error.message}\n\n请检查配置或稍后重试。`;
|
||
const failedMetadata = {
|
||
...baseVideoMetadata,
|
||
status: "failed",
|
||
error: error?.message || "未知错误",
|
||
updatedAt: Date.now(),
|
||
};
|
||
|
||
try {
|
||
await updateMessageById({
|
||
messageId: videoMessageId,
|
||
chatId: id,
|
||
parts: [{ type: "text", text: errorMessage }],
|
||
videoStorageStatus: null,
|
||
temporaryVideoUrl: null,
|
||
permanentVideoUrl: null,
|
||
archivedAt: null,
|
||
videoMetadata: failedMetadata as any,
|
||
});
|
||
try {
|
||
await persistVideoGenerationFailedTurn({
|
||
userId: session.user.id,
|
||
chatId: id,
|
||
projectId: contextPayload?.projectId || null,
|
||
messageId: videoMessageId,
|
||
text: requestTextContent,
|
||
source: "stream_video_provider",
|
||
providerType: "stream",
|
||
modelRef: finalModelId,
|
||
mode:
|
||
typeof effectiveVideoParams?.mode === "string"
|
||
? effectiveVideoParams.mode
|
||
: undefined,
|
||
status: "failed",
|
||
error: error?.message || "未知错误",
|
||
});
|
||
} catch (videoWorkflowFailedError) {
|
||
chatApiLogger.warn("[Chat API] Failed to persist stream video failure state:", videoWorkflowFailedError);
|
||
}
|
||
|
||
dataStream.write({
|
||
type: "data-appendMessage",
|
||
data: JSON.stringify({
|
||
id: videoMessageId,
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: errorMessage }],
|
||
attachments: [],
|
||
}),
|
||
});
|
||
} catch (updateError) {
|
||
chatApiLogger.error("[Chat API] Failed to update placeholder video message:", updateError);
|
||
const fallbackMessageId = generateUUID();
|
||
dataStream.write({
|
||
type: "data-appendMessage",
|
||
data: JSON.stringify({
|
||
id: fallbackMessageId,
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: errorMessage }],
|
||
attachments: [],
|
||
}),
|
||
});
|
||
await saveMessages({
|
||
messages: [
|
||
{
|
||
id: fallbackMessageId,
|
||
chatId: id,
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: errorMessage }],
|
||
attachments: [],
|
||
createdAt: new Date(),
|
||
videoStorageStatus: null,
|
||
temporaryVideoUrl: null,
|
||
permanentVideoUrl: null,
|
||
archivedAt: null,
|
||
videoMetadata: failedMetadata as any,
|
||
},
|
||
],
|
||
});
|
||
try {
|
||
await persistVideoGenerationFailedTurn({
|
||
userId: session.user.id,
|
||
chatId: id,
|
||
projectId: contextPayload?.projectId || null,
|
||
messageId: fallbackMessageId,
|
||
text: requestTextContent,
|
||
source: "stream_video_provider",
|
||
providerType: "stream",
|
||
modelRef: finalModelId,
|
||
mode:
|
||
typeof effectiveVideoParams?.mode === "string"
|
||
? effectiveVideoParams.mode
|
||
: undefined,
|
||
status: "failed",
|
||
error: error?.message || "未知错误",
|
||
});
|
||
} catch (videoWorkflowFailedError) {
|
||
chatApiLogger.warn("[Chat API] Failed to persist fallback stream video failure state:", videoWorkflowFailedError);
|
||
}
|
||
}
|
||
|
||
return;
|
||
}
|
||
}
|
||
|
||
// 常规文本模型使用 streamText
|
||
const streamRequestTimeoutMs = resolvedModelSupportsImageGen ? 180_000 : 90_000;
|
||
const shouldDisableSmoothStream =
|
||
resolvedModelSupportsImageGen || isVolcanoVideoModel;
|
||
|
||
const result = streamText({
|
||
model: languageModel,
|
||
system: effectiveConversationSystemPrompt,
|
||
messages: convertedMessages,
|
||
stopWhen: stepCountIs(5),
|
||
experimental_activeTools:
|
||
finalModelId === "chat-model-reasoning" || isVolcanoVideoModel ? [] : activeTools,
|
||
experimental_transform: shouldDisableSmoothStream
|
||
? undefined
|
||
: smoothStream({ chunking: "word" }),
|
||
tools: isVolcanoVideoModel ? {} : tools, // 🆕 视频模型禁用工具
|
||
abortSignal: AbortSignal.timeout(streamRequestTimeoutMs),
|
||
experimental_telemetry: {
|
||
isEnabled: isProductionEnvironment,
|
||
functionId: "stream-text",
|
||
},
|
||
onFinish: async ({ usage }) => {
|
||
if (activeRuntimeModelDescriptor?.providerKey) {
|
||
recordProviderHealthSuccess({
|
||
providerKey: activeRuntimeModelDescriptor.providerKey,
|
||
providerType: activeRuntimeModelDescriptor.providerType,
|
||
});
|
||
}
|
||
try {
|
||
const providers = await getTokenlensCatalog();
|
||
if (!actualModelId) {
|
||
finalMergedUsage = usage;
|
||
dataStream.write({
|
||
type: "data-usage",
|
||
data: finalMergedUsage,
|
||
});
|
||
return;
|
||
}
|
||
|
||
if (!providers) {
|
||
finalMergedUsage = usage;
|
||
dataStream.write({
|
||
type: "data-usage",
|
||
data: finalMergedUsage,
|
||
});
|
||
return;
|
||
}
|
||
|
||
const summary = getUsage({
|
||
modelId: actualModelId,
|
||
usage,
|
||
providers,
|
||
});
|
||
finalMergedUsage = {
|
||
...usage,
|
||
...summary,
|
||
modelId: actualModelId,
|
||
} as AppUsage;
|
||
dataStream.write({ type: "data-usage", data: finalMergedUsage });
|
||
} catch (err) {
|
||
chatApiLogger.warn("TokenLens enrichment failed", err);
|
||
finalMergedUsage = usage;
|
||
dataStream.write({ type: "data-usage", data: finalMergedUsage });
|
||
}
|
||
},
|
||
});
|
||
|
||
result.consumeStream({
|
||
onError: (error) => {
|
||
chatApiLogger.error("[Chat API] streamText consume error:", error);
|
||
if (activeRuntimeModelDescriptor?.providerKey) {
|
||
recordProviderHealthFailure({
|
||
providerKey: activeRuntimeModelDescriptor.providerKey,
|
||
providerType: activeRuntimeModelDescriptor.providerType,
|
||
error,
|
||
});
|
||
}
|
||
},
|
||
});
|
||
|
||
dataStream.merge(
|
||
result.toUIMessageStream({
|
||
sendReasoning: true,
|
||
onError: (error) => {
|
||
chatApiLogger.error("[Chat API] streamText ui stream error:", error);
|
||
if (activeRuntimeModelDescriptor?.providerKey) {
|
||
recordProviderHealthFailure({
|
||
providerKey: activeRuntimeModelDescriptor.providerKey,
|
||
providerType: activeRuntimeModelDescriptor.providerType,
|
||
error,
|
||
});
|
||
}
|
||
return getChatStreamErrorMessage(error);
|
||
},
|
||
})
|
||
);
|
||
} finally {
|
||
setConversationStateProtocolEventObserver(null);
|
||
stopStreamKeepalive = stopDefaultKeepalive;
|
||
}
|
||
},
|
||
generateId: generateUUID,
|
||
onFinish: async ({ messages }) => {
|
||
stopStreamKeepalive?.();
|
||
stopStreamKeepalive = null;
|
||
const messagesToPersist: Array<{
|
||
id: string;
|
||
role: string;
|
||
parts: unknown;
|
||
createdAt: Date;
|
||
attachments: unknown;
|
||
chatId: string;
|
||
videoStorageStatus: null;
|
||
temporaryVideoUrl: null;
|
||
permanentVideoUrl: null;
|
||
archivedAt: null;
|
||
videoMetadata: null;
|
||
}> = [];
|
||
|
||
for (const currentMessage of messages) {
|
||
const normalizedParts =
|
||
currentMessage.role === "assistant"
|
||
? normalizeAssistantPartsForPersistence(
|
||
currentMessage.parts as Array<Record<string, any>> | undefined,
|
||
)
|
||
: currentMessage.parts;
|
||
|
||
if (
|
||
currentMessage.role === "assistant" &&
|
||
(!Array.isArray(normalizedParts) || normalizedParts.length === 0)
|
||
) {
|
||
continue;
|
||
}
|
||
|
||
messagesToPersist.push({
|
||
id: currentMessage.id,
|
||
role: currentMessage.role,
|
||
parts: normalizedParts,
|
||
createdAt: new Date(),
|
||
attachments: [],
|
||
chatId: id,
|
||
videoStorageStatus: null,
|
||
temporaryVideoUrl: null,
|
||
permanentVideoUrl: null,
|
||
archivedAt: null,
|
||
videoMetadata: null,
|
||
});
|
||
}
|
||
|
||
if (messagesToPersist.length > 0) {
|
||
await saveMessages({
|
||
messages: messagesToPersist,
|
||
});
|
||
}
|
||
|
||
// 检测助手消息中是否包含视频，如果有则立即标记并归档到永久存储
|
||
for (const msg of messages) {
|
||
if (msg.role === "assistant" && msg.parts) {
|
||
const textContent = msg.parts
|
||
.filter((part: any) => part && part.type === "text" && part.text)
|
||
.map((part: any) => part.text)
|
||
.join("");
|
||
|
||
const tempVideoUrl = extractVideoUrlFromText(textContent);
|
||
if (tempVideoUrl) {
|
||
const genericVideoMetadata = {
|
||
provider: "stream",
|
||
model: actualModelId,
|
||
status: "succeeded",
|
||
updatedAt: Date.now(),
|
||
};
|
||
|
||
await updateMessageById({
|
||
messageId: msg.id,
|
||
chatId: id,
|
||
videoStorageStatus: "temporary",
|
||
temporaryVideoUrl: tempVideoUrl,
|
||
permanentVideoUrl: null,
|
||
archivedAt: null,
|
||
videoMetadata: genericVideoMetadata as any,
|
||
});
|
||
try {
|
||
await persistVideoGenerationCompletedTurn({
|
||
userId: session.user.id,
|
||
chatId: id,
|
||
projectId: contextPayload?.projectId || null,
|
||
messageId: msg.id,
|
||
text: requestTextContent,
|
||
source: "chat_stream_output",
|
||
providerType: "stream",
|
||
modelRef: actualModelId,
|
||
status: "completed",
|
||
temporaryVideoUrl: tempVideoUrl,
|
||
});
|
||
} catch (videoWorkflowCompletedError) {
|
||
chatApiLogger.warn("[Chat API] Failed to persist streamed assistant video state:", videoWorkflowCompletedError);
|
||
}
|
||
|
||
scheduleVideoArchive({
|
||
chatId: id,
|
||
messageId: msg.id,
|
||
text: textContent,
|
||
temporaryVideoUrl: tempVideoUrl,
|
||
taskId: msg.id,
|
||
metadata: genericVideoMetadata,
|
||
});
|
||
}
|
||
}
|
||
}
|
||
|
||
if (finalMergedUsage) {
|
||
try {
|
||
await updateChatLastContextById({
|
||
chatId: id,
|
||
context: finalMergedUsage,
|
||
});
|
||
} catch (err) {
|
||
chatApiLogger.warn("Unable to persist last usage for chat", id, err);
|
||
}
|
||
}
|
||
},
|
||
onError: () => {
|
||
stopStreamKeepalive?.();
|
||
stopStreamKeepalive = null;
|
||
return "Oops, an error occurred!";
|
||
},
|
||
});
|
||
|
||
// const streamContext = getStreamContext();
|
||
|
||
// if (streamContext) {
|
||
// return new Response(
|
||
// await streamContext.resumableStream(streamId, () =>
|
||
// stream.pipeThrough(new JsonToSseTransformStream())
|
||
// )
|
||
// );
|
||
// }
|
||
|
||
return createSseResponse(stream);
|
||
} catch (error) {
|
||
const vercelId = request.headers.get("x-vercel-id");
|
||
|
||
if (error instanceof ChatSDKError) {
|
||
return error.toResponse();
|
||
}
|
||
|
||
if (isRateLimitError(error)) {
|
||
const detailMessage = getRequestErrorMessage(error);
|
||
chatApiLogger.warn("[Chat API] Upstream rate limited request:", {
|
||
vercelId,
|
||
error: detailMessage,
|
||
});
|
||
return NextResponse.json(
|
||
{
|
||
code: "rate_limit:chat",
|
||
message: getFriendlyRateLimitMessage(),
|
||
cause: detailMessage || "upstream_rate_limited",
|
||
retryAfter: 10,
|
||
},
|
||
{
|
||
status: 429,
|
||
headers: {
|
||
"Retry-After": "10",
|
||
},
|
||
}
|
||
);
|
||
}
|
||
|
||
// Check for Vercel AI Gateway credit card error
|
||
if (
|
||
error instanceof Error &&
|
||
error.message?.includes(
|
||
"AI Gateway requires a valid credit card on file to service requests"
|
||
)
|
||
) {
|
||
return new ChatSDKError("bad_request:activate_gateway").toResponse();
|
||
}
|
||
|
||
return createChatApiFailureResponse(error, vercelId);
|
||
} finally {
|
||
if (requestDedupKey && acquiredInFlightLock) {
|
||
inFlightChatRequests.delete(requestDedupKey);
|
||
}
|
||
}
|
||
}
|
||
|
||
export async function DELETE(request: Request) {
|
||
const { searchParams } = new URL(request.url);
|
||
const id = searchParams.get("id");
|
||
|
||
if (!id) {
|
||
return new ChatSDKError("bad_request:api").toResponse();
|
||
}
|
||
|
||
const session = await auth();
|
||
|
||
if (!session?.user) {
|
||
return new ChatSDKError("unauthorized:chat").toResponse();
|
||
}
|
||
|
||
const chat = await getChatById({ id });
|
||
|
||
if (chat?.userId !== session.user.id) {
|
||
return new ChatSDKError("forbidden:chat").toResponse();
|
||
}
|
||
|
||
const deletedChat = await deleteChatById({ id });
|
||
|
||
return Response.json(deletedChat, { status: 200 });
|
||
}
|