12683 lines
487 KiB
TypeScript
12683 lines
487 KiB
TypeScript
import { geolocation } from "@vercel/functions";
|
||
import {
|
||
convertToModelMessages,
|
||
createUIMessageStream,
|
||
generateText,
|
||
JsonToSseTransformStream,
|
||
smoothStream,
|
||
stepCountIs,
|
||
streamText,
|
||
} from "ai";
|
||
import { eq, sql } from "drizzle-orm";
|
||
import { unstable_cache as cache } from "next/cache";
|
||
import { after, NextResponse } from "next/server";
|
||
import {
|
||
createResumableStreamContext,
|
||
type ResumableStreamContext,
|
||
} from "resumable-stream";
|
||
import type { ModelCatalog } from "tokenlens/core";
|
||
import { fetchModels } from "tokenlens/fetch";
|
||
import { getUsage } from "tokenlens/helpers";
|
||
import { auth, type UserType } from "@/app/(auth)/auth";
|
||
import type { VisibilityType } from "@/components/visibility-selector";
|
||
import { entitlementsByUserType } from "@/lib/ai/entitlements";
|
||
import {
|
||
BoxCutoutServiceError,
|
||
executeBoxCutoutJob,
|
||
} from "@/lib/ai/box-cutout/service";
|
||
import {
|
||
executeExtendJob,
|
||
ExtendServiceError,
|
||
inferExtendOptionsFromText,
|
||
} from "@/lib/ai/extend/service";
|
||
import type { ChatModel } from "@/lib/ai/models";
|
||
import { type RequestHints, systemPrompt } from "@/lib/ai/prompts";
|
||
import { myProvider } from "@/lib/ai/providers";
|
||
import {
|
||
executeResizeJob,
|
||
inferResizeOptionsFromText,
|
||
ResizeServiceError,
|
||
} from "@/lib/ai/resize/service";
|
||
import {
|
||
executeRmbgJob,
|
||
fetchImageBufferFromUrl,
|
||
RmbgServiceError,
|
||
} from "@/lib/ai/rmbg/service";
|
||
import { getLanguageModelById } from "@/lib/ai/dynamic-provider";
|
||
import { resolveDetailAgentLocalMoonshotModel } from "@/lib/ai/detail-agent-model-runtime";
|
||
import { shouldForceInlineVisionInputs } from "@/lib/ai/model-compat";
|
||
import { inlineVisionImages } from "@/lib/ai/vision-inline";
|
||
import { requestLongTaskResult } from "@/lib/client/long-task";
|
||
import { createDocument } from "@/lib/ai/tools/create-document";
|
||
import { getWeather } from "@/lib/ai/tools/get-weather";
|
||
import { requestSuggestions } from "@/lib/ai/tools/request-suggestions";
|
||
import { updateDocument } from "@/lib/ai/tools/update-document";
|
||
import {
|
||
executeDirectVideoGeneration,
|
||
generateVideo,
|
||
} from "@/lib/ai/tools/generate-video";
|
||
import {
|
||
getAgentRequiredUploadCount,
|
||
normalizeAgentPresetModeConfig,
|
||
} from "@/lib/agents/preset-mode-config";
|
||
import { buildAgentSceneImageRolePrompt } from "@/lib/agents/scene-image-role-prompt";
|
||
import { optimizeAgentSceneWorkflowPrompt } from "@/lib/agents/scene-workflow-prompt-optimizer";
|
||
import { getEnabledTools } from "@/lib/ai/tools-registry";
|
||
import { isProductionEnvironment } from "@/lib/constants";
|
||
import {
|
||
createStreamId,
|
||
deleteChatById,
|
||
getChatById,
|
||
getPointsSystemConfig,
|
||
getUserById,
|
||
getMessageCountByUserId,
|
||
getMessagesByChatId,
|
||
saveChat,
|
||
saveMessages,
|
||
updateMessageById,
|
||
updateChatLastContextById,
|
||
getAgentAiModelWithProvider,
|
||
getAiModelById,
|
||
getEnabledAiModelsWithProvider,
|
||
getSubModelsByParentModelId,
|
||
getAiProviderById,
|
||
checkUserImageQuota,
|
||
consumeUserPoints,
|
||
createAgentPresetExecutionSnapshot,
|
||
createMessageTargetResolution,
|
||
createMediaAssetRelation,
|
||
getAccessibleAgentPresetById,
|
||
getConversationWorkingStateByChatId,
|
||
getMediaAssetsByUserAndUrls,
|
||
getSiteSettings,
|
||
upsertMediaAsset,
|
||
} from "@/lib/db/queries";
|
||
import { db } from "@/lib/db";
|
||
import type { DBMessage } from "@/lib/db/schema";
|
||
import {
|
||
agentPreset as agentPresetTable,
|
||
generatedImage,
|
||
message as messageTable,
|
||
} from "@/lib/db/schema";
|
||
import { generateCanvasImage } from "@/lib/canvas/generate-image";
|
||
import {
|
||
detectRequestedStoryboardGrid,
|
||
isGridConfirmationReply,
|
||
type PendingGridConfirmation,
|
||
} from "@/lib/ai/conversation-action-resolver";
|
||
import {
|
||
isImageGenerationModelDescriptor,
|
||
isVideoModelDescriptor,
|
||
} from "@/lib/ai/model-classification";
|
||
import { getVideoCreditCostFromConfig } from "@/lib/ai/video-model-config";
|
||
import {
|
||
bootstrapConversationStateProtocol,
|
||
composeConversationStateProtocol,
|
||
persistAgentPresetTurn,
|
||
persistClarificationTurn,
|
||
setConversationStateProtocolEventObserver,
|
||
persistResolvedTargetTurn,
|
||
persistImageWorkflowTurn,
|
||
persistPromptOnlyTurn,
|
||
resolveGridConfirmationRequest,
|
||
resolveTargetExecutionOverride,
|
||
persistStoryboardConfirmedTurn,
|
||
persistStoryboardExecutionCompletedTurn,
|
||
persistStoryboardExecutionStartedTurn,
|
||
persistStoryboardRevisionTurn,
|
||
persistStoryboardAssistantTurn,
|
||
persistVideoGenerationCompletedTurn,
|
||
persistVideoGenerationFailedTurn,
|
||
persistVideoGenerationStartedTurn,
|
||
persistWorkflowCancelledTurn,
|
||
} from "@/lib/ai/conversation-state-protocol";
|
||
import { resolveChatMediaInput } from "@/lib/ai/context-memory/media-input";
|
||
import { resolveTargetAssets } from "@/lib/ai/context-memory/target-resolution";
|
||
import type {
|
||
ResolvedChatMediaInput,
|
||
ResolvedImageRef,
|
||
ResolvedTargetAssets,
|
||
} from "@/lib/ai/context-memory/types";
|
||
import { ChatSDKError } from "@/lib/errors";
|
||
import { extractImageUrlsFromText } from "@/lib/image-url";
|
||
import {
|
||
resolveClarificationBypass,
|
||
resolveClarificationMessage,
|
||
} from "@/lib/ai/runtime/clarification-interruption";
|
||
import { resolveCanvasAgentClarificationGate } from "@/lib/ai/runtime/middlewares/canvas-agent-clarification-gate";
|
||
import { buildDirectAgentImageAnalysisAnswer } from "@/lib/ai/runtime/direct-agent-analysis-answer";
|
||
import {
|
||
analyzeOptimizedPromptMerge,
|
||
buildDirectAgentExecutionPrompt,
|
||
buildOptimizedPromptPreview,
|
||
} from "@/lib/ai/runtime/optimized-prompt";
|
||
import {
|
||
findAgentToolCallArgs,
|
||
getDirectAgentToolErrorMessage,
|
||
getDirectAgentToolLabel,
|
||
getDirectAgentToolSuccessMessage,
|
||
getDirectAgentToolTargetMissingMessage,
|
||
pickAgentSourceImageUrls,
|
||
resolveDirectAgentToolExecution,
|
||
type DirectAgentToolCapability,
|
||
} from "@/lib/ai/runtime/direct-agent-execution";
|
||
import {
|
||
pruneTaskBoundaryContext,
|
||
pruneTaskBoundaryHistoryMessages,
|
||
} from "@/lib/ai/runtime/task-boundary-context-pruner";
|
||
import {
|
||
resolveAgentModelSlotDescriptor,
|
||
upsertAgentModelSlot,
|
||
type AgentModelSlots,
|
||
} from "@/lib/ai/runtime/agent-model-slots";
|
||
import { isPureGreetingTurnRequest } from "@/lib/ai/runtime/task-boundary-classifier";
|
||
import {
|
||
getProviderHealthSnapshot,
|
||
recordProviderHealthFailure,
|
||
recordProviderHealthSuccess,
|
||
shouldAvoidProvider,
|
||
} from "@/lib/ai/runtime/provider-health";
|
||
import { arbitrateAgentIntent } from "@/lib/ai/runtime/intent-arbiter";
|
||
import { buildAgentRoleIdentityInstruction } from "@/lib/ai/runtime/agent-role-profile";
|
||
import { uploadFile } from "@/lib/storage/service";
|
||
import { getVideoStoryboardConfig } from "@/lib/video-agent/config";
|
||
import { runRuxaNineGridPipeline } from "@/lib/video-agent/pipeline/ruxa-nine-grid-pipeline";
|
||
import {
|
||
buildStoryboardConfirmedAck,
|
||
executeStoryboardConfirmedWorkflow,
|
||
handleStoryboardWorkflowFollowup,
|
||
} from "@/lib/video-agent/storyboard-chat-followup";
|
||
import {
|
||
buildStoryboardPreferenceMemoryUpdate,
|
||
} from "@/lib/video-agent/storyboard-preference-memory";
|
||
import {
|
||
attachVideoStoryboardTaskMessage,
|
||
getLatestActiveVideoStoryboardTaskByChatId,
|
||
markVideoStoryboardAwaitingConfirmation,
|
||
markVideoStoryboardConfirmed,
|
||
markVideoStoryboardDraftForRevision,
|
||
cancelVideoStoryboardTaskByUser,
|
||
} from "@/lib/video-agent/task-store";
|
||
import type { StoryboardModelCandidate } from "@/lib/video-agent/types";
|
||
import type { ChatMessage } from "@/lib/types";
|
||
import type { AppUsage } from "@/lib/usage";
|
||
import { convertToUIMessages, generateUUID } from "@/lib/utils";
|
||
import { isAiModelAgentAuto } from "@/lib/db/ai-model-flags";
|
||
import type {
|
||
AgentExecutionPlan,
|
||
AgentPreparedExecution,
|
||
} from "@/lib/ai/auto-agent-graph";
|
||
import { generateTitleFromUserMessage } from "../../actions";
|
||
import { type PostRequestBody, postRequestBodySchema } from "./schema";
|
||
import { isAiModelMultimodal } from "@/lib/db/ai-model-flags";
|
||
|
||
/**
|
||
* 将图片 URL 转换为 base64 格式
|
||
* 用于将 S3 URL 的图片转换为 base64 发送给 Gemini API
|
||
*/
|
||
async function convertImageUrlToBase64(url: string): Promise<string> {
|
||
try {
|
||
console.log("[Image Convert] Fetching image from:", url);
|
||
const response = await fetch(url);
|
||
if (!response.ok) {
|
||
throw new Error(`Failed to fetch image: ${response.statusText}`);
|
||
}
|
||
|
||
const arrayBuffer = await response.arrayBuffer();
|
||
const buffer = Buffer.from(arrayBuffer);
|
||
const base64 = buffer.toString("base64");
|
||
|
||
// 获取 MIME 类型
|
||
const contentType = response.headers.get("content-type") || "image/jpeg";
|
||
const dataUrl = `data:${contentType};base64,${base64}`;
|
||
|
||
console.log("[Image Convert] Converted to base64, size:", base64.length);
|
||
return dataUrl;
|
||
} catch (error) {
|
||
console.error("[Image Convert] Failed:", error);
|
||
throw error;
|
||
}
|
||
}
|
||
|
||
/**
|
||
* 检测消息中是否包含用户标注过的图片（文件名以 annotated- 开头）
|
||
*/
|
||
function hasAnnotatedImage(messages: ChatMessage[]): boolean {
|
||
for (const message of messages) {
|
||
for (const part of message.parts) {
|
||
if (part.type === "file" && "name" in part && typeof part.name === "string") {
|
||
if (part.name.startsWith("annotated-")) {
|
||
return true;
|
||
}
|
||
}
|
||
}
|
||
}
|
||
return false;
|
||
}
|
||
|
||
/**
|
||
* 标注图片的系统提示词（用户不可见，后台自动添加）
|
||
*/
|
||
const ANNOTATION_SYSTEM_PROMPT = "【重要】用户提供的图片上有标注线条（如箭头、圆圈、画笔痕迹或文字说明），这些标注是用户用来指示需要修改的区域或方向。请仔细理解标注的含义，按照标注指示进行编辑，但在生成的新图片中不要保留这些标注线条，输出干净的编辑结果。";
|
||
|
||
function applyHiddenPromptToMessages(messages: ChatMessage[], hiddenPrompt?: string): ChatMessage[] {
|
||
if (!hiddenPrompt || !hiddenPrompt.trim()) return messages;
|
||
const cloned = messages.map((msg) => ({
|
||
...msg,
|
||
parts: msg.parts.map((part) => ({ ...part })),
|
||
}));
|
||
for (let i = cloned.length - 1; i >= 0; i--) {
|
||
const msg = cloned[i];
|
||
if (msg.role !== "user") continue;
|
||
const textPart = msg.parts.find((part) => part.type === "text") as { type: "text"; text: string } | undefined;
|
||
if (textPart) {
|
||
textPart.text = `${hiddenPrompt}\n\n${textPart.text}`.trim();
|
||
} else {
|
||
msg.parts.unshift({ type: "text", text: hiddenPrompt });
|
||
}
|
||
break;
|
||
}
|
||
return cloned;
|
||
}
|
||
|
||
function applyOptimizedPromptToLatestUserMessage(
|
||
messages: ChatMessage[],
|
||
optimizedPrompt?: string,
|
||
hiddenPromptPrefix = "",
|
||
): ChatMessage[] {
|
||
const prompt = optimizedPrompt?.trim();
|
||
if (!prompt) return messages;
|
||
|
||
const cloned = messages.map((msg) => ({
|
||
...msg,
|
||
parts: msg.parts.map((part) => ({ ...part })),
|
||
}));
|
||
|
||
for (let i = cloned.length - 1; i >= 0; i -= 1) {
|
||
const msg = cloned[i];
|
||
if (msg.role !== "user") continue;
|
||
|
||
const textPart = msg.parts.find(
|
||
(part): part is { type: "text"; text: string } => part.type === "text",
|
||
);
|
||
const originalText = textPart?.text?.trim() || "";
|
||
const { normalizedOriginal, hasOriginalDemand, hasOriginalInOptimized } =
|
||
analyzeOptimizedPromptMerge(originalText, prompt, hiddenPromptPrefix);
|
||
const finalPrompt = hasOriginalDemand && !hasOriginalInOptimized
|
||
? [
|
||
hiddenPromptPrefix.trim(),
|
||
`【核心目标（必须优先满足）】${normalizedOriginal}`,
|
||
`【执行优化细节】${prompt}`,
|
||
]
|
||
.filter(Boolean)
|
||
.join("\n\n")
|
||
.trim()
|
||
: `${hiddenPromptPrefix}${prompt}`.trim();
|
||
console.log("[Prompt Merge] Final image prompt prepared:", {
|
||
original: normalizedOriginal.substring(0, 120),
|
||
optimized: prompt.substring(0, 120),
|
||
merged: finalPrompt.substring(0, 180),
|
||
forcePreserveOriginal: hasOriginalDemand && !hasOriginalInOptimized,
|
||
});
|
||
|
||
if (textPart) {
|
||
textPart.text = finalPrompt;
|
||
} else {
|
||
msg.parts.unshift({ type: "text", text: finalPrompt });
|
||
}
|
||
break;
|
||
}
|
||
|
||
return cloned;
|
||
}
|
||
|
||
function injectOptimizedPromptSection(params: {
|
||
message?: string;
|
||
originalText: string;
|
||
optimizedPrompt?: string;
|
||
hiddenPromptPrefix?: string;
|
||
}): string | undefined {
|
||
const { message, originalText, optimizedPrompt, hiddenPromptPrefix } = params;
|
||
if (!message?.trim()) return message;
|
||
|
||
const preview = buildOptimizedPromptPreview({
|
||
originalText,
|
||
optimizedPrompt,
|
||
hiddenPromptPrefix,
|
||
});
|
||
if (!preview) return message;
|
||
|
||
const optimizedPromptSection = `【优化后提示词】\n${preview}`;
|
||
const optimizedSectionRegex =
|
||
/(?:###\s*优化后提示词|【\s*优化后提示词\s*】)[\s\S]*?(?=\n(?:###\s*|【\s*[^】]+\s*】)|$)/;
|
||
if (optimizedSectionRegex.test(message)) {
|
||
return message.replace(optimizedSectionRegex, optimizedPromptSection);
|
||
}
|
||
|
||
const executionSectionRegex =
|
||
/(?:###\s*执行计划|【\s*执行计划\s*】)[\s\S]*?(?=\n(?:###\s*|【\s*[^】]+\s*】)|$)/;
|
||
if (executionSectionRegex.test(message)) {
|
||
return message.replace(executionSectionRegex, optimizedPromptSection);
|
||
}
|
||
|
||
const outputSectionRegex = /(?:###\s*输出设置|【\s*输出设置\s*】)/;
|
||
if (outputSectionRegex.test(message)) {
|
||
return message.replace(outputSectionRegex, `${optimizedPromptSection}\n$&`);
|
||
}
|
||
|
||
return `${message.trim()}\n\n${optimizedPromptSection}`;
|
||
}
|
||
|
||
type StreamWriter = {
|
||
write: (chunk: any) => void;
|
||
};
|
||
|
||
function safeWriteStreamChunk(writer: StreamWriter, chunk: any, label: string): boolean {
|
||
try {
|
||
writer.write(chunk);
|
||
return true;
|
||
} catch (error) {
|
||
console.warn(`[Chat API] Stream write skipped (${label})`, error);
|
||
return false;
|
||
}
|
||
}
|
||
|
||
function writeAssistantTextChunk(
|
||
writer: StreamWriter,
|
||
text: string,
|
||
label: string,
|
||
messageId = generateUUID(),
|
||
) {
|
||
safeWriteStreamChunk(
|
||
writer,
|
||
{
|
||
type: "data-appendMessage",
|
||
data: JSON.stringify({
|
||
id: messageId,
|
||
role: "assistant",
|
||
parts: [{ type: "text", text }],
|
||
}),
|
||
},
|
||
label,
|
||
);
|
||
return messageId;
|
||
}
|
||
|
||
function writeStructuredDataChunk(
|
||
writer: StreamWriter,
|
||
type: string,
|
||
data: unknown,
|
||
label: string,
|
||
) {
|
||
safeWriteStreamChunk(
|
||
writer,
|
||
{
|
||
type,
|
||
data: JSON.stringify(data),
|
||
},
|
||
label,
|
||
);
|
||
}
|
||
|
||
function writeProtocolDebugChunk(
|
||
writer: StreamWriter,
|
||
enabled: boolean,
|
||
data: unknown,
|
||
label: string,
|
||
) {
|
||
if (!enabled) return;
|
||
writeStructuredDataChunk(writer, "data-protocolDebug", data, label);
|
||
}
|
||
|
||
function normalizeImageLoadingProviderType(providerType?: string | null): string {
|
||
const normalized = String(providerType || "").trim().toLowerCase();
|
||
if (normalized === "gemini-image" || normalized === "huanke-gemini") {
|
||
return "gemini";
|
||
}
|
||
if (
|
||
normalized === "alibaba-image" ||
|
||
normalized === "bfl-image" ||
|
||
normalized === "ruxa-image" ||
|
||
normalized === "fal-image" ||
|
||
normalized === "openai-image" ||
|
||
normalized === "midjourney" ||
|
||
normalized === "runway" ||
|
||
normalized === "volcano-vision" ||
|
||
normalized === "gemini"
|
||
) {
|
||
return normalized;
|
||
}
|
||
return "alibaba-image";
|
||
}
|
||
|
||
function writeImageGenerationStatusChunk(
|
||
writer: StreamWriter,
|
||
params: {
|
||
phase: "queued" | "processing" | "finalizing" | "failed" | "completed";
|
||
providerType?: string | null;
|
||
modelName?: string | null;
|
||
imageCount?: number | null;
|
||
},
|
||
label: string,
|
||
) {
|
||
writeStructuredDataChunk(
|
||
writer,
|
||
"data-imageGenerationStatus",
|
||
{
|
||
phase: params.phase,
|
||
providerType: normalizeImageLoadingProviderType(params.providerType),
|
||
modelName: String(params.modelName || "").trim() || undefined,
|
||
imageCount:
|
||
Number.isFinite(Number(params.imageCount)) && Number(params.imageCount) > 0
|
||
? Number(params.imageCount)
|
||
: undefined,
|
||
},
|
||
label,
|
||
);
|
||
}
|
||
|
||
type AgentProgressStreamPayload = {
|
||
stage: "planning" | "clarify" | "executing" | "completed" | "failed";
|
||
currentStep:
|
||
| "lock_target"
|
||
| "analyze"
|
||
| "select_tool"
|
||
| "execute"
|
||
| "finalize";
|
||
title?: string;
|
||
message?: string;
|
||
toolName?: string;
|
||
capability?: string;
|
||
};
|
||
|
||
function writeAgentProgressChunk(
|
||
writer: StreamWriter,
|
||
payload: AgentProgressStreamPayload,
|
||
label: string,
|
||
) {
|
||
writeStructuredDataChunk(
|
||
writer,
|
||
"data-agentProgress",
|
||
{
|
||
stage: payload.stage,
|
||
currentStep: payload.currentStep,
|
||
title: String(payload.title || "").trim() || undefined,
|
||
message: String(payload.message || "").trim() || undefined,
|
||
toolName: String(payload.toolName || "").trim() || undefined,
|
||
capability: String(payload.capability || "").trim() || undefined,
|
||
},
|
||
label,
|
||
);
|
||
}
|
||
|
||
function writeCanvasMediaChunk(
|
||
writer: StreamWriter,
|
||
params: {
|
||
messageId: string;
|
||
text?: string;
|
||
imageUrls?: string[];
|
||
videoUrls?: string[];
|
||
prompt?: string | null;
|
||
sourceImageUrls?: string[] | null;
|
||
agentPresetId?: string | null;
|
||
agentPresetName?: string | null;
|
||
agentCategoryId?: string | null;
|
||
agentCategoryName?: string | null;
|
||
agentExecutionSnapshotId?: string | null;
|
||
},
|
||
label: string,
|
||
) {
|
||
const imageUrls = Array.isArray(params.imageUrls)
|
||
? params.imageUrls
|
||
: extractImageUrlsFromText(String(params.text || ""));
|
||
const videoUrls = Array.isArray(params.videoUrls) ? params.videoUrls : [];
|
||
const normalizedImageUrls = Array.from(
|
||
new Set(
|
||
imageUrls
|
||
.map((url) => String(url || "").trim())
|
||
.filter((url) => /^https?:\/\//i.test(url)),
|
||
),
|
||
);
|
||
const normalizedVideoUrls = Array.from(
|
||
new Set(
|
||
videoUrls
|
||
.map((url) => String(url || "").trim())
|
||
.filter((url) => /^https?:\/\//i.test(url)),
|
||
),
|
||
);
|
||
|
||
if (normalizedImageUrls.length === 0 && normalizedVideoUrls.length === 0) {
|
||
return;
|
||
}
|
||
|
||
writeStructuredDataChunk(
|
||
writer,
|
||
"data-canvasMedia",
|
||
{
|
||
messageId: params.messageId,
|
||
imageUrls: normalizedImageUrls,
|
||
videoUrls: normalizedVideoUrls,
|
||
prompt: String(params.prompt || "").trim() || undefined,
|
||
sourceImageUrls: Array.isArray(params.sourceImageUrls)
|
||
? params.sourceImageUrls
|
||
.map((url) => String(url || "").trim())
|
||
.filter((url) => /^https?:\/\//i.test(url))
|
||
: [],
|
||
agentPresetId: String(params.agentPresetId || "").trim() || undefined,
|
||
agentPresetName: String(params.agentPresetName || "").trim() || undefined,
|
||
agentCategoryId: String(params.agentCategoryId || "").trim() || undefined,
|
||
agentCategoryName: String(params.agentCategoryName || "").trim() || undefined,
|
||
agentExecutionSnapshotId:
|
||
String(params.agentExecutionSnapshotId || "").trim() || undefined,
|
||
},
|
||
label,
|
||
);
|
||
}
|
||
|
||
type AgentTargetLockRefPreview = {
|
||
url: string;
|
||
title?: string;
|
||
nodeId?: string;
|
||
assetId?: string;
|
||
source: ResolvedImageRef["source"];
|
||
role: ResolvedImageRef["role"];
|
||
isSelected?: boolean;
|
||
positionLabel?: string;
|
||
};
|
||
|
||
type AgentTargetLockStreamPayload = {
|
||
stage: "locked" | "clarify" | "unresolved";
|
||
resolutionMode: ResolvedTargetAssets["resolutionMode"];
|
||
confidence: number;
|
||
reason: string;
|
||
needsClarification: boolean;
|
||
clarificationMessage?: string;
|
||
targetCount: number;
|
||
candidateCount: number;
|
||
targets: AgentTargetLockRefPreview[];
|
||
candidates: AgentTargetLockRefPreview[];
|
||
};
|
||
|
||
function sortTargetLockRefsForCanvasOrdering(refs: ResolvedImageRef[]) {
|
||
return [...refs].sort((left, right) => {
|
||
const leftY = Number(left.y ?? 0);
|
||
const rightY = Number(right.y ?? 0);
|
||
if (Math.abs(leftY - rightY) > 12) {
|
||
return leftY - rightY;
|
||
}
|
||
return Number(left.x ?? 0) - Number(right.x ?? 0);
|
||
});
|
||
}
|
||
|
||
function getTargetLockPositionLabel(ref: ResolvedImageRef, refs: ResolvedImageRef[]): string {
|
||
if (refs.length <= 1) return "中间";
|
||
const xs = refs.map((item) => Number(item.x ?? 0));
|
||
const ys = refs.map((item) => Number(item.y ?? 0));
|
||
const minX = Math.min(...xs);
|
||
const maxX = Math.max(...xs);
|
||
const minY = Math.min(...ys);
|
||
const maxY = Math.max(...ys);
|
||
const spanX = Math.max(1, maxX - minX);
|
||
const spanY = Math.max(1, maxY - minY);
|
||
const relX = (Number(ref.x ?? 0) - minX) / spanX;
|
||
const relY = (Number(ref.y ?? 0) - minY) / spanY;
|
||
|
||
const horizontal = relX <= 0.33 ? "左" : relX >= 0.66 ? "右" : "中";
|
||
const vertical = relY <= 0.33 ? "上" : relY >= 0.66 ? "下" : "中";
|
||
|
||
if (horizontal === "中" && vertical === "中") return "中间";
|
||
return `${vertical}${horizontal}`;
|
||
}
|
||
|
||
function serializeAgentTargetLockRefs(refs: ResolvedImageRef[]) {
|
||
const orderedRefs = sortTargetLockRefsForCanvasOrdering(refs).slice(0, 4);
|
||
return orderedRefs.map((ref) => ({
|
||
url: ref.url,
|
||
title: String(ref.title || "").trim() || undefined,
|
||
nodeId: ref.nodeId,
|
||
assetId: ref.assetId,
|
||
source: ref.source,
|
||
role: ref.role,
|
||
isSelected: ref.isSelected === true,
|
||
positionLabel: getTargetLockPositionLabel(ref, orderedRefs),
|
||
})) satisfies AgentTargetLockRefPreview[];
|
||
}
|
||
|
||
function buildAgentTargetLockPayload(params: {
|
||
requestTargetResolution: ResolvedTargetAssets;
|
||
requestResolvedMediaInput: ResolvedChatMediaInput;
|
||
needsClarification: boolean;
|
||
clarificationMessage?: string;
|
||
}): AgentTargetLockStreamPayload {
|
||
const stage =
|
||
params.needsClarification
|
||
? "clarify"
|
||
: params.requestTargetResolution.targetRefs.length > 0
|
||
? "locked"
|
||
: "unresolved";
|
||
|
||
return {
|
||
stage,
|
||
resolutionMode: params.requestTargetResolution.resolutionMode,
|
||
confidence: Number(params.requestTargetResolution.confidence || 0),
|
||
reason: params.requestTargetResolution.reason,
|
||
needsClarification: params.needsClarification,
|
||
clarificationMessage: params.clarificationMessage?.trim() || undefined,
|
||
targetCount: params.requestTargetResolution.targetRefs.length,
|
||
candidateCount: params.requestTargetResolution.candidateRefs.length,
|
||
targets: serializeAgentTargetLockRefs(params.requestTargetResolution.targetRefs),
|
||
candidates: serializeAgentTargetLockRefs(
|
||
params.requestTargetResolution.candidateRefs.length > 0
|
||
? params.requestTargetResolution.candidateRefs
|
||
: params.requestResolvedMediaInput.imageRefs,
|
||
),
|
||
};
|
||
}
|
||
|
||
function writeAgentTargetLockChunk(
|
||
writer: StreamWriter,
|
||
payload: AgentTargetLockStreamPayload,
|
||
label: string,
|
||
) {
|
||
writeStructuredDataChunk(writer, "data-agentTargetLock", payload, label);
|
||
}
|
||
|
||
function createSseResponse(stream: ReadableStream) {
|
||
return new Response(stream.pipeThrough(new JsonToSseTransformStream()), {
|
||
headers: {
|
||
"Content-Type": "text/event-stream; charset=utf-8",
|
||
"Cache-Control": "no-cache, no-transform",
|
||
Connection: "keep-alive",
|
||
"X-Accel-Buffering": "no",
|
||
},
|
||
});
|
||
}
|
||
|
||
function startStreamKeepalive(writer: StreamWriter, label: string, intervalMs = 15000) {
|
||
const timer = setInterval(() => {
|
||
safeWriteStreamChunk(
|
||
writer,
|
||
{
|
||
type: "data-keepalive",
|
||
data: JSON.stringify({ ts: Date.now(), label }),
|
||
},
|
||
`${label}:keepalive`,
|
||
);
|
||
}, intervalMs);
|
||
|
||
return () => clearInterval(timer);
|
||
}
|
||
|
||
type VideoModeType = "text-to-video" | "first-frame" | "first-last-frame" | "reference-images";
|
||
|
||
function isVideoProviderType(
|
||
providerType?: string | null,
|
||
model?: {
|
||
modelId?: string | null;
|
||
capabilities?: unknown;
|
||
supportsImageGen?: boolean | null;
|
||
}
|
||
): boolean {
|
||
return isVideoModelDescriptor({
|
||
providerType,
|
||
modelId: model?.modelId,
|
||
capabilities: model?.capabilities,
|
||
supportsImageGen: model?.supportsImageGen,
|
||
});
|
||
}
|
||
|
||
function isImageGenerationModelType(
|
||
providerType?: string | null,
|
||
model?: {
|
||
modelId?: string | null;
|
||
capabilities?: unknown;
|
||
supportsImageGen?: boolean | null;
|
||
}
|
||
): boolean {
|
||
return isImageGenerationModelDescriptor({
|
||
providerType,
|
||
modelId: model?.modelId,
|
||
capabilities: model?.capabilities,
|
||
supportsImageGen: model?.supportsImageGen,
|
||
});
|
||
}
|
||
|
||
function inferSupportedVideoModesByModelId(modelId: string): VideoModeType[] {
|
||
const normalized = modelId.toLowerCase();
|
||
|
||
if (
|
||
normalized === "gen4.5" ||
|
||
normalized === "gen4_turbo" ||
|
||
normalized === "gen4_aleph" ||
|
||
normalized === "gen3a_turbo" ||
|
||
normalized === "veo3" ||
|
||
normalized === "veo3.1" ||
|
||
normalized === "veo3.1_fast"
|
||
) {
|
||
return ["text-to-video", "first-frame"];
|
||
}
|
||
|
||
if (normalized.includes("lite-t2v")) {
|
||
return ["text-to-video"];
|
||
}
|
||
if (normalized.includes("lite-i2v")) {
|
||
return ["first-frame", "first-last-frame", "reference-images"];
|
||
}
|
||
if (normalized.includes("pro-fast")) {
|
||
return ["text-to-video", "first-frame"];
|
||
}
|
||
if (normalized.includes("-pro-") || normalized.endsWith("-pro")) {
|
||
return ["text-to-video", "first-frame", "first-last-frame"];
|
||
}
|
||
|
||
return ["text-to-video", "first-frame", "first-last-frame", "reference-images"];
|
||
}
|
||
|
||
function getSupportedVideoModesForModel(model: any): VideoModeType[] {
|
||
const capabilityModes = (((model?.capabilities as any)?.supportedModes ?? []) as string[])
|
||
.filter((mode): mode is VideoModeType =>
|
||
mode === "text-to-video" ||
|
||
mode === "first-frame" ||
|
||
mode === "first-last-frame" ||
|
||
mode === "reference-images"
|
||
);
|
||
|
||
if (capabilityModes.length > 0) {
|
||
return capabilityModes;
|
||
}
|
||
|
||
const modelId = String(model?.modelId || "");
|
||
return inferSupportedVideoModesByModelId(modelId);
|
||
}
|
||
|
||
function scoreVideoSubModelForMode(modelId: string, mode: VideoModeType): number {
|
||
const normalized = modelId.toLowerCase();
|
||
const isProFast = normalized.includes("pro-fast");
|
||
const isLiteT2V = normalized.includes("lite-t2v");
|
||
const isLiteI2V = normalized.includes("lite-i2v");
|
||
const isPro = (normalized.includes("-pro-") || normalized.endsWith("-pro")) && !isProFast;
|
||
const isV15Pro = normalized.includes("seedance-1-5-pro");
|
||
|
||
switch (mode) {
|
||
case "first-last-frame":
|
||
if (isV15Pro) return 150;
|
||
if (isPro) return 120;
|
||
if (isLiteI2V) return 90;
|
||
if (isProFast) return -1000;
|
||
if (isLiteT2V) return -1000;
|
||
return 10;
|
||
case "first-frame":
|
||
if (isV15Pro) return 140;
|
||
if (isProFast) return 130;
|
||
if (isPro) return 120;
|
||
if (isLiteI2V) return 100;
|
||
if (isLiteT2V) return 20;
|
||
return 10;
|
||
case "reference-images":
|
||
if (isV15Pro) return 150;
|
||
if (isLiteI2V) return 140;
|
||
if (isPro) return 110;
|
||
if (isProFast || isLiteT2V) return -1000;
|
||
return 10;
|
||
case "text-to-video":
|
||
default:
|
||
if (isV15Pro) return 140;
|
||
if (isLiteT2V) return 130;
|
||
if (isProFast) return 120;
|
||
if (isPro) return 100;
|
||
if (isLiteI2V) return 80;
|
||
return 10;
|
||
}
|
||
}
|
||
|
||
function selectBestVideoSubModelForMode(
|
||
subModels: Array<{ model: any }>,
|
||
mode: VideoModeType,
|
||
): { model: any } | null {
|
||
const compatible = subModels.filter(({ model }) => {
|
||
const supportedModes = getSupportedVideoModesForModel(model);
|
||
return supportedModes.includes(mode);
|
||
});
|
||
|
||
const candidates = compatible.length > 0 ? compatible : subModels;
|
||
if (candidates.length === 0) return null;
|
||
|
||
return [...candidates].sort((a, b) => {
|
||
const scoreA = scoreVideoSubModelForMode(String(a.model?.modelId || ""), mode);
|
||
const scoreB = scoreVideoSubModelForMode(String(b.model?.modelId || ""), mode);
|
||
return scoreB - scoreA;
|
||
})[0];
|
||
}
|
||
|
||
async function resolveStoryboardModelCandidates(
|
||
selectedModelRef: string
|
||
): Promise<StoryboardModelCandidate[]> {
|
||
const selectedModel = await getAiModelById({ id: selectedModelRef }).catch(() => undefined);
|
||
if (!selectedModel) return [];
|
||
|
||
const configuredFallbackIds = [
|
||
selectedModel.fallbackModelId1,
|
||
selectedModel.fallbackModelId2,
|
||
].filter((item): item is string => Boolean(String(item || "").trim()));
|
||
|
||
const allModels = await getEnabledAiModelsWithProvider(true, true).catch(() => []);
|
||
const sameProvider = allModels
|
||
.filter(
|
||
({ model, provider }: any) =>
|
||
provider?.id === selectedModel.providerId &&
|
||
model?.id !== selectedModel.id &&
|
||
model?.isEnabled
|
||
)
|
||
.map(({ model }: any) => String(model?.id || "").trim())
|
||
.filter(Boolean);
|
||
|
||
const candidateIds = Array.from(
|
||
new Set([selectedModel.id, ...configuredFallbackIds, ...sameProvider])
|
||
);
|
||
const byId = new Map(
|
||
allModels
|
||
.map(({ model }: any) => model)
|
||
.filter(Boolean)
|
||
.map((model: any) => [String(model.id), model] as const)
|
||
);
|
||
|
||
return candidateIds
|
||
.map((id) => byId.get(id))
|
||
.filter(Boolean)
|
||
.map((model: any) => ({
|
||
id: String(model.id),
|
||
modelId: String(model.modelId || "").trim(),
|
||
displayName: String(model.displayName || model.modelId || "").trim(),
|
||
}))
|
||
.filter((item) => item.modelId);
|
||
}
|
||
|
||
function extractVideoUrlFromText(text: string): string | null {
|
||
const normalizedText = String(text || "");
|
||
if (!normalizedText) return null;
|
||
|
||
const htmlMatch = normalizedText.match(/<video[^>]+src="([^"]+)"/i);
|
||
if (htmlMatch?.[1]) return htmlMatch[1];
|
||
|
||
const markdownMatch = normalizedText.match(
|
||
/\[[^\]]*]\((https?:\/\/[^)\s]+\.mp4(?:\?[^)\s]*)?)\)/i,
|
||
);
|
||
if (markdownMatch?.[1]) return markdownMatch[1];
|
||
|
||
const plainMatch = normalizedText.match(/(https?:\/\/[^\s"'<>]+\.mp4(?:\?[^\s"'<>]+)?)/i);
|
||
return plainMatch?.[1] || null;
|
||
}
|
||
|
||
function isAlreadyArchivedVideoUrl(url: string): boolean {
|
||
return /\/uploads\//i.test(url);
|
||
}
|
||
|
||
function scheduleVideoArchive(params: {
|
||
chatId: string;
|
||
messageId: string;
|
||
text: string;
|
||
temporaryVideoUrl: string;
|
||
taskId?: string;
|
||
metadata?: Record<string, any> | null;
|
||
}) {
|
||
const { chatId, messageId, temporaryVideoUrl, taskId, metadata } = params;
|
||
if (!temporaryVideoUrl || isAlreadyArchivedVideoUrl(temporaryVideoUrl)) {
|
||
return;
|
||
}
|
||
|
||
const delayHoursRaw = Number(process.env.VIDEO_ARCHIVE_DELAY_HOURS || 48);
|
||
const delayHours = Number.isFinite(delayHoursRaw)
|
||
? Math.max(1, Math.min(24 * 30, Math.round(delayHoursRaw)))
|
||
: 48;
|
||
const archiveAfterAtMs = Date.now() + delayHours * 60 * 60 * 1000;
|
||
|
||
after(async () => {
|
||
try {
|
||
const nextMetadata = {
|
||
...(metadata || {}),
|
||
temporaryVideoUrl,
|
||
storageStatus: "temporary",
|
||
archiveMode: "delayed_job",
|
||
archiveAfterAt: archiveAfterAtMs,
|
||
archiveDelayHours: delayHours,
|
||
archiveTaskId: taskId || messageId,
|
||
updatedAt: Date.now(),
|
||
};
|
||
|
||
await updateMessageById({
|
||
messageId,
|
||
chatId,
|
||
videoStorageStatus: "temporary",
|
||
temporaryVideoUrl,
|
||
permanentVideoUrl: null,
|
||
archivedAt: null,
|
||
videoMetadata: nextMetadata as any,
|
||
});
|
||
|
||
console.log("[Chat API] 🕒 已登记延迟归档视频:", {
|
||
messageId,
|
||
archiveAfterAt: new Date(archiveAfterAtMs).toISOString(),
|
||
});
|
||
} catch (error) {
|
||
console.error("[Chat API] ❌ 登记视频延迟归档失败（保留临时地址）:", {
|
||
messageId,
|
||
temporaryVideoUrl,
|
||
error,
|
||
});
|
||
}
|
||
});
|
||
}
|
||
|
||
async function replayAssistantMessagesForDuplicateRequest({
|
||
chatId,
|
||
userMessageId,
|
||
writer,
|
||
source,
|
||
waitMs = DUPLICATE_REQUEST_WAIT_MS,
|
||
}: {
|
||
chatId: string;
|
||
userMessageId: string;
|
||
writer: StreamWriter;
|
||
source: string;
|
||
waitMs?: number;
|
||
}) {
|
||
const startedAt = Date.now();
|
||
const seenAssistantMessageIds = new Set<string>();
|
||
|
||
while (Date.now() - startedAt < waitMs) {
|
||
try {
|
||
const dbMessages = await getMessagesByChatId({ id: chatId });
|
||
const userMessageIndex = dbMessages.findIndex((msg) => msg.id === userMessageId);
|
||
|
||
if (userMessageIndex >= 0) {
|
||
const assistantMessages = dbMessages
|
||
.slice(userMessageIndex + 1)
|
||
.filter((msg) => msg.role === "assistant");
|
||
|
||
if (assistantMessages.length > 0) {
|
||
for (const assistantMessage of assistantMessages) {
|
||
if (seenAssistantMessageIds.has(assistantMessage.id)) continue;
|
||
seenAssistantMessageIds.add(assistantMessage.id);
|
||
|
||
safeWriteStreamChunk(
|
||
writer,
|
||
{
|
||
type: "data-appendMessage",
|
||
data: JSON.stringify({
|
||
id: assistantMessage.id,
|
||
role: "assistant",
|
||
parts: assistantMessage.parts,
|
||
}),
|
||
},
|
||
`${source}:replay`,
|
||
);
|
||
}
|
||
|
||
return;
|
||
}
|
||
}
|
||
} catch (error) {
|
||
console.warn(`[Chat API] Failed duplicate replay fetch (${source})`, error);
|
||
}
|
||
|
||
safeWriteStreamChunk(
|
||
writer,
|
||
{
|
||
type: "data-keepalive",
|
||
data: JSON.stringify({ ts: Date.now(), source }),
|
||
},
|
||
`${source}:keepalive`,
|
||
);
|
||
|
||
await sleep(1000);
|
||
}
|
||
}
|
||
|
||
type ChatImageParams = NonNullable<PostRequestBody["imageParams"]>;
|
||
type AgentRecommendedImageParams = {
|
||
aspectRatio?: string;
|
||
imageSize?: string;
|
||
imageCount?: number;
|
||
negativePrompt?: string;
|
||
promptExtend?: boolean;
|
||
};
|
||
|
||
type ChatVideoParams = NonNullable<PostRequestBody["videoParams"]>;
|
||
type ChatVideoParamsDraft = Partial<ChatVideoParams>;
|
||
type AgentRecommendedVideoParams = {
|
||
mode?: "text-to-video" | "first-frame" | "first-last-frame" | "reference-images";
|
||
resolution?: "480p" | "720p" | "1080p";
|
||
ratio?: string;
|
||
duration?: number;
|
||
generateAudio?: boolean;
|
||
cameraMove?: string;
|
||
};
|
||
|
||
type AgentVideoPlan = {
|
||
script: string;
|
||
storyboard: Array<{
|
||
shot: number;
|
||
title: string;
|
||
duration: number;
|
||
description: string;
|
||
prompt: string;
|
||
camera?: string;
|
||
}>;
|
||
finalPrompt: string;
|
||
negativePrompt?: string;
|
||
recommendedVideoParams?: AgentRecommendedVideoParams;
|
||
};
|
||
|
||
function toFiniteNumber(value: unknown): number | undefined {
|
||
const normalized = Number(value);
|
||
return Number.isFinite(normalized) ? normalized : undefined;
|
||
}
|
||
|
||
async function executeDirectAgentEnhanceJob(params: {
|
||
requestUrl: string;
|
||
requestHeaders: Headers;
|
||
imageUrl: string;
|
||
prompt: string;
|
||
chatId: string;
|
||
messageId: string;
|
||
}) {
|
||
const endpoint = new URL("/api/enhance", params.requestUrl).toString();
|
||
const cookieHeader = params.requestHeaders.get("cookie") || "";
|
||
const result = await requestLongTaskResult({
|
||
endpoint,
|
||
payload: {
|
||
imageUrl: params.imageUrl,
|
||
prompt: params.prompt,
|
||
chatId: params.chatId,
|
||
messageId: params.messageId,
|
||
},
|
||
pollingParams: {
|
||
prompt: params.prompt,
|
||
chatId: params.chatId,
|
||
messageId: params.messageId,
|
||
sourceImageUrl: params.imageUrl,
|
||
},
|
||
timeoutMs: 8 * 60 * 1000,
|
||
headers: cookieHeader
|
||
? {
|
||
cookie: cookieHeader,
|
||
}
|
||
: undefined,
|
||
});
|
||
|
||
if (!result?.success || !result?.original_url) {
|
||
throw new Error(String(result?.error || result?.detail || "图像增强失败"));
|
||
}
|
||
|
||
return {
|
||
originalUrl: String(result.original_url),
|
||
thumbnailUrl: String(result.thumbnail_url || result.original_url),
|
||
imageSize: "2K",
|
||
};
|
||
}
|
||
|
||
function getChatStreamErrorMessage(error: unknown): string {
|
||
const rawMessage =
|
||
error instanceof Error ? String(error.message || "").trim() : String(error || "").trim();
|
||
const normalized = rawMessage.toLowerCase();
|
||
|
||
if (
|
||
normalized.includes("token quota is not enough") ||
|
||
normalized.includes("pre_consume_token_quota_failed") ||
|
||
normalized.includes("quota is not enough")
|
||
) {
|
||
return "当前对话模型额度不足，请切换其他模型，或联系管理员补充该模型额度。";
|
||
}
|
||
|
||
if (
|
||
normalized.includes("timeout") ||
|
||
normalized.includes("timed out") ||
|
||
normalized.includes("body timeout")
|
||
) {
|
||
return "当前对话模型响应超时，请重试，或切换到其他可用模型。";
|
||
}
|
||
|
||
if (rawMessage) {
|
||
return `当前对话暂时失败：${rawMessage}`;
|
||
}
|
||
|
||
return "当前对话暂时失败，请重试或切换其他模型。";
|
||
}
|
||
|
||
function normalizeAssistantPartsForPersistence(parts: Array<Record<string, any>> | undefined) {
|
||
const safeParts = Array.isArray(parts) ? parts.filter(Boolean) : [];
|
||
return safeParts.filter((part) => {
|
||
const partType = String(part?.type || "").trim();
|
||
if (!partType) return false;
|
||
if (partType === "data-keepalive" || partType === "step-start" || partType === "step-finish") {
|
||
return false;
|
||
}
|
||
return true;
|
||
});
|
||
}
|
||
|
||
function buildAgentToolImageMarkdown(params: {
|
||
originalUrl: string;
|
||
thumbnailUrl?: string | null;
|
||
imageSize?: string;
|
||
}): string {
|
||
const originalUrl = String(params.originalUrl || "").trim();
|
||
const thumbnailUrl = String(params.thumbnailUrl || originalUrl).trim() || originalUrl;
|
||
const imageSize = String(params.imageSize || "2K").trim() || "2K";
|
||
return ``;
|
||
}
|
||
|
||
function buildAgentToolImagesMarkdown(
|
||
items: Array<{ originalUrl: string; thumbnailUrl?: string | null; imageSize?: string }>,
|
||
) {
|
||
return items
|
||
.map((item) =>
|
||
buildAgentToolImageMarkdown({
|
||
originalUrl: item.originalUrl,
|
||
thumbnailUrl: item.thumbnailUrl,
|
||
imageSize: item.imageSize,
|
||
}),
|
||
)
|
||
.join("\n");
|
||
}
|
||
|
||
function bufferToArrayBuffer(buffer: Buffer): ArrayBuffer {
|
||
return buffer.buffer.slice(
|
||
buffer.byteOffset,
|
||
buffer.byteOffset + buffer.byteLength,
|
||
) as ArrayBuffer;
|
||
}
|
||
|
||
async function createAgentImageThumbnailBuffer(buffer: Buffer): Promise<Buffer> {
|
||
const sharp = (await import("sharp")).default;
|
||
return sharp(buffer, { failOn: "none" })
|
||
.rotate()
|
||
.resize(500, null, {
|
||
fit: "inside",
|
||
withoutEnlargement: true,
|
||
})
|
||
.jpeg({ quality: 82 })
|
||
.toBuffer();
|
||
}
|
||
|
||
async function normalizeReplacementResultImage(params: {
|
||
generatedImageUrl: string;
|
||
referenceImageUrl: string;
|
||
uploadPrefix: string;
|
||
}): Promise<{
|
||
originalUrl: string;
|
||
thumbnailUrl: string;
|
||
imageSize: string;
|
||
}> {
|
||
const sharp = (await import("sharp")).default;
|
||
const [generatedBuffer, referenceBuffer] = await Promise.all([
|
||
fetchImageBufferFromUrl(params.generatedImageUrl),
|
||
fetchImageBufferFromUrl(params.referenceImageUrl),
|
||
]);
|
||
|
||
const referenceMetadata = await sharp(referenceBuffer, { failOn: "none" })
|
||
.rotate()
|
||
.metadata();
|
||
const targetWidth = referenceMetadata.width;
|
||
const targetHeight = referenceMetadata.height;
|
||
|
||
if (!targetWidth || !targetHeight) {
|
||
throw new Error("无法读取参考图尺寸");
|
||
}
|
||
|
||
const normalizedBuffer = await sharp(generatedBuffer, { failOn: "none" })
|
||
.rotate()
|
||
.resize(targetWidth, targetHeight, {
|
||
fit: "fill",
|
||
withoutEnlargement: false,
|
||
})
|
||
.png()
|
||
.toBuffer();
|
||
const thumbnailBuffer = await createAgentImageThumbnailBuffer(normalizedBuffer);
|
||
|
||
const uniqueSuffix = `${Date.now()}-${Math.random().toString(36).slice(2, 8)}`;
|
||
const originalFilename = `${params.uploadPrefix}-orig-${uniqueSuffix}.png`;
|
||
const thumbnailFilename = `${params.uploadPrefix}-thumb-${uniqueSuffix}.jpg`;
|
||
const [originalUpload, thumbnailUpload] = await Promise.all([
|
||
uploadFile(originalFilename, bufferToArrayBuffer(normalizedBuffer)),
|
||
uploadFile(thumbnailFilename, bufferToArrayBuffer(thumbnailBuffer)),
|
||
]);
|
||
|
||
return {
|
||
originalUrl: originalUpload.url,
|
||
thumbnailUrl: thumbnailUpload.url,
|
||
imageSize: `${targetWidth}x${targetHeight}`,
|
||
};
|
||
}
|
||
|
||
function buildAgentToolVideoMarkdown(videoUrl: string) {
|
||
const normalizedUrl = String(videoUrl || "").trim();
|
||
if (!normalizedUrl) return "";
|
||
return [
|
||
`<video src="${normalizedUrl}" controls style="max-width: 100%; border-radius: 8px;"></video>`,
|
||
`[下载视频](${normalizedUrl})`,
|
||
].join("\n\n");
|
||
}
|
||
|
||
const IMAGE_FAILOVER_PROVIDER_ALLOWLIST = new Set([
|
||
"alibaba-image",
|
||
"bfl-image",
|
||
"ruxa-image",
|
||
"fal-image",
|
||
"openai-image",
|
||
"runway",
|
||
"volcano-vision",
|
||
"gemini",
|
||
"huanke-gemini",
|
||
]);
|
||
|
||
const IMAGE_FAILOVER_MODEL_COOLDOWN_MS = 30 * 60 * 1000;
|
||
const imageFailoverModelCooldown = new Map<string, { until: number; reason: string }>();
|
||
|
||
function normalizeFailoverErrorMessage(error: unknown): string {
|
||
if (!error) return "";
|
||
if (typeof error === "string") return error;
|
||
if (error instanceof Error) return error.message || String(error);
|
||
try {
|
||
return JSON.stringify(error);
|
||
} catch {
|
||
return String(error);
|
||
}
|
||
}
|
||
|
||
function shouldCooldownImageFallbackModel(errorMessage: string): boolean {
|
||
const msg = errorMessage.toLowerCase();
|
||
return (
|
||
msg.includes("model not found") ||
|
||
msg.includes("model_not_found") ||
|
||
msg.includes("no available channel for model") ||
|
||
msg.includes("unknown model") ||
|
||
msg.includes("invalid model") ||
|
||
msg.includes("api key is missing") ||
|
||
msg.includes("invalid api key") ||
|
||
msg.includes("unauthorized") ||
|
||
msg.includes("cannot convert argument to a bytestring")
|
||
);
|
||
}
|
||
|
||
function isImageFallbackModelCoolingDown(modelId: string): boolean {
|
||
const snapshot = imageFailoverModelCooldown.get(modelId);
|
||
if (!snapshot) return false;
|
||
if (snapshot.until <= Date.now()) {
|
||
imageFailoverModelCooldown.delete(modelId);
|
||
return false;
|
||
}
|
||
return true;
|
||
}
|
||
|
||
function setImageFallbackModelCooldown(modelId: string, reason: string): void {
|
||
imageFailoverModelCooldown.set(modelId, {
|
||
until: Date.now() + IMAGE_FAILOVER_MODEL_COOLDOWN_MS,
|
||
reason: reason.slice(0, 500),
|
||
});
|
||
}
|
||
|
||
function isImageFailoverExcludedModel(model: { excludeFromFailover?: boolean } | null | undefined) {
|
||
return Boolean(model?.excludeFromFailover);
|
||
}
|
||
|
||
function buildImageFallbackModelIds(
|
||
allModels: Array<{ model: any; provider: any }>,
|
||
options: {
|
||
currentProviderId?: string;
|
||
currentModelId?: string;
|
||
currentModelKey?: string;
|
||
configuredFallbackIds?: string[];
|
||
}
|
||
) {
|
||
const { currentProviderId, currentModelId, currentModelKey, configuredFallbackIds = [] } = options;
|
||
|
||
const eligibleFallbackModelIds = new Set(
|
||
allModels
|
||
.filter(
|
||
(item) =>
|
||
item.model.supportsImageGen &&
|
||
item.model.isEnabled &&
|
||
!item.model.isSubModel &&
|
||
!isImageFailoverExcludedModel(item.model) &&
|
||
item.model.modelId !== currentModelKey &&
|
||
item.model.id !== currentModelId
|
||
)
|
||
.map((item) => String(item.model.modelId || ""))
|
||
.filter(Boolean)
|
||
);
|
||
|
||
const sameProviderFallbacks = currentProviderId
|
||
? allModels
|
||
.filter(
|
||
(item) =>
|
||
item.provider.id === currentProviderId &&
|
||
item.model.supportsImageGen &&
|
||
item.model.isEnabled &&
|
||
!item.model.isSubModel &&
|
||
!isImageFailoverExcludedModel(item.model) &&
|
||
item.model.modelId !== currentModelKey &&
|
||
item.model.id !== currentModelId
|
||
)
|
||
.map((item) => item.model.modelId)
|
||
: [];
|
||
|
||
return Array.from(new Set([...configuredFallbackIds, ...sameProviderFallbacks])).filter(
|
||
(modelId) => Boolean(modelId) && eligibleFallbackModelIds.has(String(modelId))
|
||
);
|
||
}
|
||
|
||
function getImageCrossProviderCandidates(
|
||
allModels: any[],
|
||
options: {
|
||
currentModelId: string;
|
||
currentProviderId?: string;
|
||
providerPriority: Record<string, number>;
|
||
}
|
||
) {
|
||
return allModels
|
||
.filter((m) => {
|
||
if (
|
||
!m?.model?.supportsImageGen ||
|
||
!m?.model?.isEnabled ||
|
||
m?.model?.isSubModel ||
|
||
isImageFailoverExcludedModel(m?.model)
|
||
) {
|
||
return false;
|
||
}
|
||
if (!IMAGE_FAILOVER_PROVIDER_ALLOWLIST.has(String(m?.provider?.type || ""))) {
|
||
return false;
|
||
}
|
||
if (m.model.id === options.currentModelId) return false;
|
||
if (options.currentProviderId && m.provider.id === options.currentProviderId) return false;
|
||
if (isImageFallbackModelCoolingDown(m.model.id)) {
|
||
const cooldown = imageFailoverModelCooldown.get(m.model.id);
|
||
console.log(
|
||
`[Chat API] ⏭ Skip image failover model on cooldown: ${m.model.displayName} (${m.provider.type}) - ${cooldown?.reason || "cooldown"}`
|
||
);
|
||
return false;
|
||
}
|
||
return true;
|
||
})
|
||
.sort((a, b) => {
|
||
const left = options.providerPriority[a.provider.type as string] ?? 99;
|
||
const right = options.providerPriority[b.provider.type as string] ?? 99;
|
||
if (left !== right) return left - right;
|
||
return (a.model.sortOrder ?? 0) - (b.model.sortOrder ?? 0);
|
||
});
|
||
}
|
||
|
||
function hasUserCustomizedImageParams(params?: PostRequestBody["imageParams"]): boolean {
|
||
if (!params) return false;
|
||
const customizedAspectRatio =
|
||
typeof params.aspectRatio === "string" && params.aspectRatio.trim() && params.aspectRatio !== "auto";
|
||
const customizedImageSize =
|
||
typeof params.imageSize === "string" && params.imageSize.trim() && params.imageSize.toUpperCase() !== "2K";
|
||
const customizedImageCount = typeof params.imageCount === "number" && params.imageCount !== 1;
|
||
const customizedPromptExtend = typeof params.promptExtend === "boolean";
|
||
const customizedAdvanced = Boolean(
|
||
params.size || params.seed !== undefined || params.n || params.negativePrompt || customizedPromptExtend
|
||
);
|
||
return customizedAspectRatio || customizedImageSize || customizedImageCount || customizedAdvanced;
|
||
}
|
||
|
||
function mergeImageParamsWithAgentRecommendation(params: {
|
||
incoming?: PostRequestBody["imageParams"];
|
||
recommended?: AgentRecommendedImageParams;
|
||
intent?: "image_edit" | "image_gen" | "vision" | "video_gen" | "chat";
|
||
hasUserCustomSettings: boolean;
|
||
}): PostRequestBody["imageParams"] {
|
||
const { incoming, recommended, intent, hasUserCustomSettings } = params;
|
||
const base: ChatImageParams = { ...(incoming || {}) };
|
||
if (!recommended) {
|
||
return Object.keys(base).length > 0 ? base : undefined;
|
||
}
|
||
|
||
const merged: ChatImageParams = {
|
||
...base,
|
||
negativePrompt:
|
||
typeof recommended.negativePrompt === "string" && recommended.negativePrompt.trim()
|
||
? recommended.negativePrompt.trim()
|
||
: base.negativePrompt,
|
||
promptExtend:
|
||
typeof recommended.promptExtend === "boolean"
|
||
? recommended.promptExtend
|
||
: base.promptExtend,
|
||
};
|
||
|
||
if (!hasUserCustomSettings) {
|
||
if (recommended.aspectRatio) merged.aspectRatio = recommended.aspectRatio;
|
||
if (recommended.imageSize) merged.imageSize = recommended.imageSize;
|
||
if (typeof recommended.imageCount === "number") merged.imageCount = recommended.imageCount;
|
||
}
|
||
|
||
if (intent === "image_edit") {
|
||
merged.imageCount = 1;
|
||
if (typeof base.promptExtend !== "boolean") {
|
||
merged.promptExtend = false;
|
||
}
|
||
}
|
||
|
||
return Object.keys(merged).length > 0 ? merged : undefined;
|
||
}
|
||
|
||
function hasUserCustomizedVideoParams(params?: PostRequestBody["videoParams"]): boolean {
|
||
if (!params) return false;
|
||
const defaults = {
|
||
mode: "text-to-video",
|
||
resolution: "720p",
|
||
ratio: "16:9",
|
||
duration: 5,
|
||
generateAudio: true,
|
||
cameraMove: "none",
|
||
} as const;
|
||
|
||
if (params.mode && params.mode !== defaults.mode) return true;
|
||
if (params.resolution && params.resolution !== defaults.resolution) return true;
|
||
if (params.ratio && params.ratio !== defaults.ratio) return true;
|
||
if (typeof params.duration === "number" && params.duration !== defaults.duration) return true;
|
||
if (typeof params.generateAudio === "boolean" && params.generateAudio !== defaults.generateAudio)
|
||
return true;
|
||
if (typeof params.cameraMove === "string" && params.cameraMove !== defaults.cameraMove)
|
||
return true;
|
||
if (typeof params.seed === "number") return true;
|
||
return false;
|
||
}
|
||
|
||
function mergeVideoParamsWithAgentRecommendation(params: {
|
||
incoming?: PostRequestBody["videoParams"];
|
||
recommended?: AgentRecommendedVideoParams;
|
||
hasUserCustomSettings: boolean;
|
||
}): PostRequestBody["videoParams"] {
|
||
const { incoming, recommended, hasUserCustomSettings } = params;
|
||
const base: ChatVideoParamsDraft = { ...(incoming || {}) };
|
||
if (!recommended) {
|
||
return Object.keys(base).length > 0 ? (base as ChatVideoParams) : undefined;
|
||
}
|
||
|
||
const merged: ChatVideoParamsDraft = { ...base };
|
||
if (!hasUserCustomSettings) {
|
||
if (recommended.mode) merged.mode = recommended.mode;
|
||
if (recommended.resolution) merged.resolution = recommended.resolution;
|
||
if (recommended.ratio) merged.ratio = recommended.ratio;
|
||
if (typeof recommended.duration === "number") merged.duration = recommended.duration;
|
||
if (typeof recommended.generateAudio === "boolean") {
|
||
merged.generateAudio = recommended.generateAudio;
|
||
}
|
||
if (recommended.cameraMove) merged.cameraMove = recommended.cameraMove;
|
||
}
|
||
|
||
if (!merged.mode) {
|
||
merged.mode = recommended.mode || (incoming?.mode as ChatVideoParams["mode"]) || "text-to-video";
|
||
}
|
||
if (!merged.resolution) {
|
||
merged.resolution =
|
||
recommended.resolution || (incoming?.resolution as ChatVideoParams["resolution"]) || "720p";
|
||
}
|
||
if (!merged.ratio) {
|
||
merged.ratio = recommended.ratio || incoming?.ratio || "16:9";
|
||
}
|
||
if (typeof merged.duration !== "number") {
|
||
merged.duration = recommended.duration || incoming?.duration || 5;
|
||
}
|
||
|
||
return Object.keys(merged).length > 0 ? (merged as ChatVideoParams) : undefined;
|
||
}
|
||
|
||
function sanitizeAgentOutputSettingsSection(params: {
|
||
message?: string;
|
||
userImageParams?: PostRequestBody["imageParams"];
|
||
hasUserCustomImageParams: boolean;
|
||
}): string | undefined {
|
||
const { message, userImageParams, hasUserCustomImageParams } = params;
|
||
if (!message?.trim()) return message;
|
||
|
||
const outputSectionRegex =
|
||
/(?:###\s*输出设置|【\s*输出设置\s*】)[\s\S]*?(?=\n(?:###\s*|【\s*[^】]+\s*】)|$)/;
|
||
if (!outputSectionRegex.test(message)) return message;
|
||
|
||
if (!hasUserCustomImageParams) {
|
||
return message.replace(
|
||
outputSectionRegex,
|
||
"【输出设置】\n- 参数：按用户面板当前设置执行（未自动改写）"
|
||
);
|
||
}
|
||
|
||
const lines: string[] = [];
|
||
const aspectRatio =
|
||
typeof userImageParams?.aspectRatio === "string" ? userImageParams.aspectRatio.trim() : "";
|
||
const imageSize =
|
||
typeof userImageParams?.imageSize === "string" ? userImageParams.imageSize.trim() : "";
|
||
const imageCount =
|
||
typeof userImageParams?.imageCount === "number" ? userImageParams.imageCount : undefined;
|
||
|
||
if (aspectRatio && aspectRatio !== "auto") {
|
||
lines.push(`- 宽高比：${aspectRatio}（来自用户面板设置）`);
|
||
}
|
||
if (imageSize && imageSize.toUpperCase() !== "2K") {
|
||
lines.push(`- 清晰度档位：${imageSize.toUpperCase()}（来自用户面板设置）`);
|
||
}
|
||
if (typeof imageCount === "number" && Number.isFinite(imageCount) && imageCount !== 1) {
|
||
lines.push(`- 出图数量：${Math.max(1, Math.round(imageCount))} 张（来自用户面板设置）`);
|
||
}
|
||
lines.push("- 其余参数：保持用户面板当前设置");
|
||
|
||
return message.replace(outputSectionRegex, `【输出设置】\n${lines.join("\n")}`);
|
||
}
|
||
|
||
function normalizeAgentReportHeadingLine(line: string): string {
|
||
const trimmed = line.trim();
|
||
if (!trimmed) return line;
|
||
|
||
const markdownHeading = trimmed.match(/^#{1,6}\s*(.+?)\s*$/);
|
||
const plainTitle = markdownHeading?.[1]?.trim() || trimmed;
|
||
const normalizedTitle = plainTitle
|
||
.replace(/^[^\u4e00-\u9fa5A-Za-z0-9]+/, "")
|
||
.trim();
|
||
|
||
const knownTitles = [
|
||
"需求理解",
|
||
"图片分析",
|
||
"执行计划",
|
||
"优化后提示词",
|
||
"输出设置",
|
||
"脚本规划",
|
||
"分镜规划",
|
||
"执行说明",
|
||
"视频生成完成",
|
||
];
|
||
|
||
if (knownTitles.some((title) => normalizedTitle === title)) {
|
||
return `【${normalizedTitle}】`;
|
||
}
|
||
|
||
if (markdownHeading) {
|
||
return `【${normalizedTitle || plainTitle}】`;
|
||
}
|
||
return line;
|
||
}
|
||
|
||
function stripAgentFriendlyMediaEmbeds(message: string): string {
|
||
if (!message) return message;
|
||
|
||
const withoutInlineMedia = message
|
||
.replace(/!\[[^\]]*]\((https?:\/\/[^\s)]+)\)/gi, "")
|
||
.replace(/<video[\s\S]*?<\/video>/gi, "")
|
||
.replace(/<img[^>]*>/gi, "");
|
||
|
||
const cleanedLines = withoutInlineMedia
|
||
.split("\n")
|
||
.filter((line) => {
|
||
const trimmed = line.trim();
|
||
if (!trimmed) return true;
|
||
if (/^https?:\/\/\S+$/i.test(trimmed)) return false;
|
||
if (/^\[[^\]]*]\(https?:\/\/[^\s)]+\)$/i.test(trimmed)) return false;
|
||
return true;
|
||
})
|
||
.join("\n");
|
||
|
||
return cleanedLines.replace(/\n{3,}/g, "\n\n").trim();
|
||
}
|
||
|
||
function normalizeAgentReportForDisplay(message?: string): string | undefined {
|
||
if (!message?.trim()) return message;
|
||
const stripped = stripAgentFriendlyMediaEmbeds(message);
|
||
if (!stripped) return undefined;
|
||
return stripped
|
||
.split("\n")
|
||
.map((line) => normalizeAgentReportHeadingLine(line))
|
||
.join("\n");
|
||
}
|
||
|
||
function guessImageMediaType(url: string): string {
|
||
const cleanUrl = url.split("?")[0]?.toLowerCase() || "";
|
||
if (cleanUrl.endsWith(".jpg") || cleanUrl.endsWith(".jpeg")) return "image/jpeg";
|
||
if (cleanUrl.endsWith(".webp")) return "image/webp";
|
||
if (cleanUrl.endsWith(".gif")) return "image/gif";
|
||
if (cleanUrl.endsWith(".bmp")) return "image/bmp";
|
||
if (cleanUrl.endsWith(".svg")) return "image/svg+xml";
|
||
return "image/png";
|
||
}
|
||
|
||
function injectHistoryImagesToLastUserMessage(
|
||
messages: ChatMessage[],
|
||
historyImages: string[]
|
||
): ChatMessage[] {
|
||
if (!historyImages || historyImages.length === 0) return messages;
|
||
|
||
let lastUserIndex = -1;
|
||
for (let i = messages.length - 1; i >= 0; i--) {
|
||
if (messages[i]?.role === "user") {
|
||
lastUserIndex = i;
|
||
break;
|
||
}
|
||
}
|
||
|
||
if (lastUserIndex < 0) return messages;
|
||
|
||
const lastUserMessage = messages[lastUserIndex];
|
||
const hasExistingImage = lastUserMessage.parts?.some(
|
||
(p: any) => p.type === "file" || p.type === "image" || p.type === "image_url"
|
||
);
|
||
if (hasExistingImage) return messages;
|
||
|
||
const imageParts = historyImages.map((url, index) => ({
|
||
type: "file" as const,
|
||
url,
|
||
name: `history-image-${index + 1}`,
|
||
mediaType: guessImageMediaType(url),
|
||
mimeType: guessImageMediaType(url),
|
||
}));
|
||
|
||
const updatedMessage: ChatMessage = {
|
||
...lastUserMessage,
|
||
parts: [...lastUserMessage.parts, ...imageParts],
|
||
};
|
||
|
||
return messages.map((msg, idx) => (idx === lastUserIndex ? updatedMessage : msg));
|
||
}
|
||
|
||
function compactAutoDemand(text: string, maxLength = 56): string {
|
||
const normalized = text.replace(/\s+/g, " ").trim();
|
||
if (!normalized) return "按你刚才的描述执行";
|
||
if (normalized.length <= maxLength) return normalized;
|
||
return `${normalized.slice(0, maxLength)}...`;
|
||
}
|
||
|
||
function buildAutoModeFallbackMessage(params: {
|
||
text: string;
|
||
hasImage: boolean;
|
||
fromHistory: boolean;
|
||
imageSummary?: string;
|
||
}): string {
|
||
const demand = compactAutoDemand(params.text);
|
||
|
||
if (params.hasImage) {
|
||
const sourceLine = params.fromHistory
|
||
? "基于上一轮图片继续处理。"
|
||
: "基于当前上传图片处理。";
|
||
|
||
return [
|
||
"### 需求理解",
|
||
`- ${demand}`,
|
||
"### 图片分析",
|
||
params.imageSummary
|
||
? `- ${params.imageSummary.replace(/\s+/g, " ").trim()}`
|
||
: `- ${sourceLine}`,
|
||
"- 我会优先保持主体、构图和整体风格一致，只处理你指定的改动。",
|
||
"### 执行计划",
|
||
"- 先完成本轮核心改动，再检查边缘与光影一致性。",
|
||
"- 若结果偏差，我会按“保留A，修改B”的方式继续迭代。",
|
||
"### 输出设置",
|
||
"- 默认单图输出，优先稳定命中。",
|
||
].join("\n");
|
||
}
|
||
|
||
return [
|
||
"### 需求理解",
|
||
`- ${demand}`,
|
||
"### 执行计划",
|
||
"- 先锁定主体与场景，再细化风格、材质与光影。",
|
||
"### 输出设置",
|
||
"- 默认单图输出，优先稳定命中。",
|
||
].join("\n");
|
||
}
|
||
|
||
function sanitizePromptOnlyUserIntent(text: string): string {
|
||
return String(text || "")
|
||
.replace(/先?分析(?:一下)?(?:这张)?(?:图片|图像|这张图)?/g, "")
|
||
.replace(/帮我|给我|先给我|只给我|仅给我/g, "")
|
||
.replace(/优化(?:一下)?提示词|提示词优化|提示词|prompt/gi, "")
|
||
.replace(/不要生图|先别生图|不需要生图|不要生成|别生成|不直接生成/g, "")
|
||
.replace(/[，。；：,.!！?？]/g, " ")
|
||
.replace(/\s+/g, " ")
|
||
.trim();
|
||
}
|
||
|
||
function buildPromptOnlyOptimizationSeed(params: {
|
||
text: string;
|
||
imageSummary?: string;
|
||
}): string {
|
||
const cleanedIntent = sanitizePromptOnlyUserIntent(params.text);
|
||
const summary = String(params.imageSummary || "").replace(/\s+/g, " ").trim();
|
||
|
||
if (cleanedIntent && summary) {
|
||
return `${cleanedIntent}。参考图关键特征：${summary}`;
|
||
}
|
||
if (summary) {
|
||
return `基于参考图生成一段可直接复现核心主体、场景、风格与光线特征的中文提示词。参考图关键特征：${summary}`;
|
||
}
|
||
if (cleanedIntent) {
|
||
return cleanedIntent;
|
||
}
|
||
return "基于当前参考图整理一段可直接使用的中文提示词。";
|
||
}
|
||
|
||
function buildPromptOnlyReply(params: {
|
||
userText: string;
|
||
imageSummary?: string;
|
||
optimizedPrompt: string;
|
||
}): string {
|
||
const demand = sanitizePromptOnlyUserIntent(params.userText) || "你希望我先分析这张图并整理可直接使用的提示词";
|
||
const summaryLines = String(params.imageSummary || "")
|
||
.split(/\r?\n/)
|
||
.map((line) => line.replace(/\s+/g, " ").trim())
|
||
.filter(Boolean);
|
||
const formattedSummary =
|
||
summaryLines.length > 0
|
||
? summaryLines.map((line) => `- ${line}`).join("\n")
|
||
: "- 我已基于当前图片梳理主体、场景、风格与光线特征。";
|
||
|
||
return [
|
||
"【需求理解】",
|
||
`- ${demand}`,
|
||
"【图片分析】",
|
||
formattedSummary,
|
||
"【优化后提示词】",
|
||
`- ${params.optimizedPrompt.replace(/\s+/g, " ").trim()}`,
|
||
"【使用建议】",
|
||
"- 本轮仅输出优化后的提示词，不直接执行生图。",
|
||
].join("\n");
|
||
}
|
||
|
||
function hasStructuredAgentSections(message?: string): boolean {
|
||
if (!message) return false;
|
||
return /(?:###\s*需求理解|【\s*需求理解\s*】|###\s*图片分析|【\s*图片分析\s*】|###\s*执行计划|【\s*执行计划\s*】|###\s*优化后提示词|【\s*优化后提示词\s*】)/.test(
|
||
message,
|
||
);
|
||
}
|
||
|
||
function ensureStructuredAgentFriendlyMessage(params: {
|
||
message?: string;
|
||
text: string;
|
||
hasImage: boolean;
|
||
fromHistory: boolean;
|
||
}): string {
|
||
const existingMessage = String(params.message || "").trim();
|
||
if (hasStructuredAgentSections(existingMessage)) {
|
||
return existingMessage;
|
||
}
|
||
|
||
const fallback = buildAutoModeFallbackMessage({
|
||
text: params.text,
|
||
hasImage: params.hasImage,
|
||
fromHistory: params.fromHistory,
|
||
});
|
||
|
||
if (!existingMessage) {
|
||
return fallback;
|
||
}
|
||
|
||
const normalizedSupplement = existingMessage.replace(/\s+/g, " ").trim();
|
||
if (!normalizedSupplement) {
|
||
return fallback;
|
||
}
|
||
|
||
return fallback.replace(
|
||
"### 执行计划\n",
|
||
`### 执行计划\n- 补充说明：${normalizedSupplement}\n`,
|
||
);
|
||
}
|
||
|
||
function extractMessageImageUrls(parts: any[]): string[] {
|
||
return (parts || [])
|
||
.map((part: any) => {
|
||
if (!part) return null;
|
||
if (part.type === "file") return part.url || part.data || null;
|
||
if (part.type === "image") return part.url || part.image || null;
|
||
if (part.type === "image_url") return part.image_url?.url || null;
|
||
return null;
|
||
})
|
||
.filter((url: any): url is string => typeof url === "string" && /^https?:\/\//i.test(url));
|
||
}
|
||
|
||
function extractMessageImageAssetIds(parts: any[]): string[] {
|
||
return Array.from(
|
||
new Set(
|
||
(parts || [])
|
||
.map((part: any) => {
|
||
if (!part || typeof part !== "object") return null;
|
||
return typeof part.assetId === "string" && part.assetId.trim()
|
||
? part.assetId.trim()
|
||
: null;
|
||
})
|
||
.filter((assetId: any): assetId is string => typeof assetId === "string" && assetId.length > 0),
|
||
),
|
||
);
|
||
}
|
||
|
||
function shouldEnrichImageAnalysisText(message?: string): boolean {
|
||
if (!message) return false;
|
||
if (!/(###\s*图片分析|【\s*图片分析\s*】)/.test(message)) return false;
|
||
return /(基于当前上传图片处理|基于你选中的原图处理|基于上一轮图片继续处理)/.test(message);
|
||
}
|
||
|
||
function injectImageAnalysisSummary(message: string, summary: string): string {
|
||
const normalizedSummaryLines = summary
|
||
.split(/\r?\n/)
|
||
.map((line) => line.replace(/\s+/g, " ").trim())
|
||
.filter(Boolean);
|
||
if (normalizedSummaryLines.length === 0) return message;
|
||
const lines = message.split("\n");
|
||
const analysisHeaderIndex = lines.findIndex((line) =>
|
||
/^(###\s*图片分析|【\s*图片分析\s*】)/.test(line.trim())
|
||
);
|
||
if (analysisHeaderIndex < 0) return message;
|
||
|
||
for (let i = analysisHeaderIndex + 1; i < lines.length; i += 1) {
|
||
const current = lines[i]?.trim() || "";
|
||
if (/^(###\s*|【\s*[^】]+\s*】)/.test(current)) break;
|
||
if (/^- /.test(current)) {
|
||
lines.splice(i, 1, ...normalizedSummaryLines.map((line) => `- ${line}`));
|
||
return lines.join("\n");
|
||
}
|
||
}
|
||
|
||
const insertIndex = analysisHeaderIndex + 1;
|
||
lines.splice(insertIndex, 0, ...normalizedSummaryLines.map((line) => `- ${line}`));
|
||
return lines.join("\n");
|
||
}
|
||
|
||
function ensureImageAnalysisSummarySection(message: string, summary: string): string {
|
||
const normalizedSummaryLines = summary
|
||
.split(/\r?\n/)
|
||
.map((line) => line.replace(/\s+/g, " ").trim())
|
||
.filter(Boolean);
|
||
if (normalizedSummaryLines.length === 0) return message;
|
||
|
||
if (/(###\s*图片分析|【\s*图片分析\s*】)/.test(message)) {
|
||
return injectImageAnalysisSummary(message, normalizedSummaryLines.join("\n"));
|
||
}
|
||
|
||
const trimmed = message.trim();
|
||
const summaryBlock = normalizedSummaryLines.map((line) => `- ${line}`).join("\n");
|
||
if (!trimmed) return `【图片分析】\n${summaryBlock}`;
|
||
return `${trimmed}\n【图片分析】\n${summaryBlock}`;
|
||
}
|
||
|
||
/**
|
||
* 自定义消息转换函数
|
||
* 注意：gemini-image 和 huanke-gemini 都需要 URL 格式（不是 Base64）
|
||
* 因为这些 provider 内部会自行处理图片上传到 S3
|
||
*/
|
||
async function convertMessagesWithImages(
|
||
messages: ChatMessage[],
|
||
providerType:
|
||
| string
|
||
| {
|
||
providerType?: string;
|
||
providerBaseUrl?: string | null;
|
||
modelId?: string | null;
|
||
displayName?: string | null;
|
||
forceInlineImageUrls?: boolean;
|
||
loggerPrefix?: string;
|
||
},
|
||
) {
|
||
type ImageConversionTarget = {
|
||
providerType?: string;
|
||
providerBaseUrl?: string | null;
|
||
modelId?: string | null;
|
||
displayName?: string | null;
|
||
forceInlineImageUrls?: boolean;
|
||
loggerPrefix?: string;
|
||
};
|
||
|
||
const normalizeTarget = (
|
||
target: string | ImageConversionTarget,
|
||
): Required<Pick<ImageConversionTarget, "providerType" | "forceInlineImageUrls" | "loggerPrefix">> &
|
||
Omit<ImageConversionTarget, "providerType" | "forceInlineImageUrls" | "loggerPrefix"> => {
|
||
if (typeof target === "string") {
|
||
return {
|
||
providerType: target,
|
||
providerBaseUrl: null,
|
||
modelId: null,
|
||
displayName: null,
|
||
forceInlineImageUrls: false,
|
||
loggerPrefix: "[Chat API] Vision inline",
|
||
};
|
||
}
|
||
|
||
return {
|
||
providerType: String(target.providerType || ""),
|
||
providerBaseUrl: target.providerBaseUrl || null,
|
||
modelId: target.modelId || null,
|
||
displayName: target.displayName || null,
|
||
forceInlineImageUrls: target.forceInlineImageUrls === true,
|
||
loggerPrefix: target.loggerPrefix || "[Chat API] Vision inline",
|
||
};
|
||
};
|
||
|
||
const extractImagePartUrl = (part: unknown): string | null => {
|
||
const partAny = part as any;
|
||
const candidateUrl =
|
||
partAny?.type === "file"
|
||
? partAny.url || partAny.data
|
||
: partAny?.type === "image"
|
||
? partAny.image || partAny.url
|
||
: partAny?.type === "image_url"
|
||
? partAny.image_url?.url
|
||
: undefined;
|
||
|
||
if (typeof candidateUrl !== "string") return null;
|
||
const normalized = candidateUrl.trim();
|
||
if (!/^https?:\/\//i.test(normalized)) return null;
|
||
return normalized;
|
||
};
|
||
|
||
const replaceImagePartUrl = (part: unknown, nextUrl: string): unknown => {
|
||
const partAny = part as any;
|
||
if (partAny?.type === "file") {
|
||
return {
|
||
...partAny,
|
||
url: nextUrl,
|
||
data: nextUrl,
|
||
mediaType: String(partAny.mediaType || "").startsWith("image/")
|
||
? partAny.mediaType
|
||
: "image/jpeg",
|
||
mimeType: String(partAny.mimeType || "").startsWith("image/")
|
||
? partAny.mimeType
|
||
: "image/jpeg",
|
||
};
|
||
}
|
||
if (partAny?.type === "image") {
|
||
return {
|
||
...partAny,
|
||
image: nextUrl,
|
||
url: nextUrl,
|
||
};
|
||
}
|
||
if (partAny?.type === "image_url") {
|
||
return {
|
||
...partAny,
|
||
image_url: {
|
||
...(partAny.image_url || {}),
|
||
url: nextUrl,
|
||
},
|
||
};
|
||
}
|
||
return part;
|
||
};
|
||
|
||
const inlineImageUrlsInMessages = async (
|
||
sourceMessages: ChatMessage[],
|
||
target: ReturnType<typeof normalizeTarget>,
|
||
) => {
|
||
const uniqueUrls = Array.from(
|
||
new Set(
|
||
sourceMessages
|
||
.flatMap((message) => message.parts || [])
|
||
.map((part) => extractImagePartUrl(part))
|
||
.filter((url): url is string => Boolean(url)),
|
||
),
|
||
);
|
||
|
||
if (uniqueUrls.length === 0) {
|
||
return sourceMessages;
|
||
}
|
||
|
||
const { inlineDataUrls } = await inlineVisionImages(uniqueUrls, {
|
||
maxImages: Math.min(8, uniqueUrls.length),
|
||
maxSide: 1200,
|
||
timeoutMs: 12000,
|
||
retries: 2,
|
||
loggerPrefix: target.loggerPrefix,
|
||
}).catch((error) => {
|
||
console.warn(`${target.loggerPrefix} failed to inline message images:`, error);
|
||
return { inlineDataUrls: [], skipped: [] };
|
||
});
|
||
|
||
if (inlineDataUrls.length === 0) {
|
||
return sourceMessages;
|
||
}
|
||
|
||
const inlineMap = new Map<string, string>();
|
||
for (let index = 0; index < inlineDataUrls.length; index += 1) {
|
||
const originalUrl = uniqueUrls[index];
|
||
const inlineUrl = inlineDataUrls[index];
|
||
if (originalUrl && inlineUrl) {
|
||
inlineMap.set(originalUrl, inlineUrl);
|
||
}
|
||
}
|
||
|
||
if (inlineMap.size === 0) {
|
||
return sourceMessages;
|
||
}
|
||
|
||
console.log(`${target.loggerPrefix} inlined chat message images:`, {
|
||
total: uniqueUrls.length,
|
||
inlined: inlineMap.size,
|
||
providerType: target.providerType,
|
||
modelId: target.modelId,
|
||
displayName: target.displayName,
|
||
});
|
||
|
||
return sourceMessages.map((message) => ({
|
||
...message,
|
||
parts: (message.parts || []).map((part) => {
|
||
const originalUrl = extractImagePartUrl(part);
|
||
if (!originalUrl) return part;
|
||
const inlineUrl = inlineMap.get(originalUrl);
|
||
return inlineUrl ? replaceImagePartUrl(part, inlineUrl) : part;
|
||
}) as typeof message.parts,
|
||
})) as ChatMessage[];
|
||
};
|
||
|
||
const target = normalizeTarget(providerType);
|
||
let preparedMessages = messages;
|
||
|
||
if (target.forceInlineImageUrls) {
|
||
preparedMessages = await inlineImageUrlsInMessages(preparedMessages, target);
|
||
}
|
||
|
||
const normalizedProviderType = target.providerType;
|
||
|
||
if (
|
||
normalizedProviderType === "bfl-image" ||
|
||
normalizedProviderType === "ruxa-image" ||
|
||
normalizedProviderType === "fal-image" ||
|
||
normalizedProviderType === "openai-image" ||
|
||
normalizedProviderType === "midjourney" ||
|
||
normalizedProviderType === "runway"
|
||
) {
|
||
const convertedMessages = [];
|
||
|
||
for (const message of preparedMessages) {
|
||
const content: any[] = [];
|
||
|
||
for (const part of message.parts) {
|
||
const partAny = part as any;
|
||
|
||
if (partAny.type === "text" && partAny.text) {
|
||
content.push({
|
||
type: "text",
|
||
text: partAny.text,
|
||
});
|
||
continue;
|
||
}
|
||
|
||
const candidateUrl =
|
||
partAny.type === "file"
|
||
? (partAny.url || partAny.data)
|
||
: partAny.type === "image"
|
||
? (partAny.image || partAny.url)
|
||
: partAny.type === "image_url"
|
||
? partAny.image_url?.url
|
||
: undefined;
|
||
|
||
if (!candidateUrl || typeof candidateUrl !== "string") continue;
|
||
if (candidateUrl.startsWith("data:")) {
|
||
throw new Error("当前模型不支持 Base64 图片传输，请先上传图片后再编辑");
|
||
}
|
||
if (!candidateUrl.startsWith("http")) continue;
|
||
|
||
content.push({
|
||
type: "image_url",
|
||
image_url: {
|
||
url: candidateUrl,
|
||
},
|
||
});
|
||
}
|
||
|
||
convertedMessages.push({
|
||
role: message.role,
|
||
content: content.length === 1 && content[0].type === "text" ? content[0].text : content,
|
||
});
|
||
}
|
||
|
||
return convertedMessages;
|
||
}
|
||
|
||
// 所有 provider 都使用 URL 格式传输图片
|
||
// gemini-image 和 huanke-gemini 内部会将 Base64 转换为 S3 URL
|
||
// 这里不再进行 Base64 转换，直接返回标准格式
|
||
if (normalizedProviderType !== "gemini") {
|
||
return convertToModelMessages(preparedMessages);
|
||
}
|
||
|
||
// 🆕 检测是否有标注图片
|
||
const hasAnnotation = hasAnnotatedImage(messages);
|
||
if (hasAnnotation) {
|
||
console.log("[Message Convert] 🎨 检测到标注图片，将自动添加系统提示");
|
||
}
|
||
|
||
// 对于 Gemini，需要特殊处理图片
|
||
const convertedMessages = [];
|
||
|
||
for (const message of preparedMessages) {
|
||
const content: any[] = [];
|
||
|
||
// 🆕 如果是用户消息且有标注图片，在文本前添加系统提示
|
||
let hasAddedAnnotationPrompt = false;
|
||
|
||
// 处理消息的各个部分
|
||
for (const part of message.parts) {
|
||
if (part.type === "text" && part.text) {
|
||
let textContent = part.text;
|
||
|
||
// 🆕 如果有标注图片且是用户消息，在用户文本前添加系统提示
|
||
if (hasAnnotation && message.role === "user" && !hasAddedAnnotationPrompt) {
|
||
textContent = ANNOTATION_SYSTEM_PROMPT + "\n\n用户请求：" + textContent;
|
||
hasAddedAnnotationPrompt = true;
|
||
console.log("[Message Convert] 🎨 已注入标注提示词");
|
||
}
|
||
|
||
content.push({
|
||
type: "text",
|
||
text: textContent,
|
||
});
|
||
} else if (part.type === "file" && part.url) {
|
||
// 直接使用 URL 格式（不再转换为 Base64）
|
||
// gemini-image 和 huanke-gemini provider 内部会处理图片上传
|
||
content.push({
|
||
type: "image_url",
|
||
image_url: {
|
||
url: part.url,
|
||
},
|
||
});
|
||
console.log("[Message Convert] Image kept as URL format:", part.url.substring(0, 80) + "...");
|
||
}
|
||
}
|
||
|
||
// 🆕 如果有标注图片但消息中没有文本部分，添加一个纯提示文本
|
||
if (hasAnnotation && message.role === "user" && !hasAddedAnnotationPrompt && content.length > 0) {
|
||
content.unshift({
|
||
type: "text",
|
||
text: ANNOTATION_SYSTEM_PROMPT,
|
||
});
|
||
console.log("[Message Convert] 🎨 已在消息开头注入标注提示词");
|
||
}
|
||
|
||
convertedMessages.push({
|
||
role: message.role,
|
||
content: content.length === 1 && content[0].type === "text"
|
||
? content[0].text
|
||
: content,
|
||
});
|
||
}
|
||
|
||
return convertedMessages;
|
||
}
|
||
|
||
function parseImageAltMetadata(altText: string, thumbnailUrl: string): {
|
||
originalUrl: string;
|
||
imageSize: string;
|
||
} {
|
||
let originalUrl = thumbnailUrl;
|
||
let imageSize = "2K";
|
||
|
||
// 格式1: thumb|||originalUrl|||size (新格式)
|
||
if (altText.includes("thumb|||")) {
|
||
const content = altText.substring(altText.indexOf("thumb|||") + 8);
|
||
const parts = content.split("|||");
|
||
originalUrl = parts[0] || thumbnailUrl;
|
||
imageSize = parts[1] || "2K";
|
||
}
|
||
// 格式2: thumb$$originalUrl$$size (中间格式，可能被 LaTeX 解析器破坏)
|
||
else if (altText.includes("thumb$$")) {
|
||
const content = altText.substring(altText.indexOf("thumb$$") + 7);
|
||
const parts = content.split("$$");
|
||
originalUrl = parts[0] || thumbnailUrl;
|
||
imageSize = parts[1] || "2K";
|
||
}
|
||
// 格式3: thumb|originalUrl|size (旧格式)
|
||
else if (altText.includes("thumb|")) {
|
||
const content = altText.substring(altText.indexOf("thumb|") + 6);
|
||
const parts = content.split("|");
|
||
originalUrl = parts[0] || thumbnailUrl;
|
||
imageSize = parts[1] || "2K";
|
||
}
|
||
|
||
// 回退策略：从缩略图 URL 推断原图 URL
|
||
if (originalUrl === thumbnailUrl) {
|
||
if (thumbnailUrl.includes("gemini-thumb-")) {
|
||
originalUrl = thumbnailUrl.replace("gemini-thumb-", "gemini-orig-");
|
||
} else if (thumbnailUrl.includes("openai-thumb-")) {
|
||
originalUrl = thumbnailUrl.replace("openai-thumb-", "openai-orig-");
|
||
} else if (thumbnailUrl.includes("volcano-thumb-")) {
|
||
originalUrl = thumbnailUrl.replace("volcano-thumb-", "volcano-orig-");
|
||
} else if (thumbnailUrl.includes("bfl-image-thumb-")) {
|
||
originalUrl = thumbnailUrl.replace("bfl-image-thumb-", "bfl-image-orig-");
|
||
} else if (thumbnailUrl.includes("alibaba-image-thumb-")) {
|
||
originalUrl = thumbnailUrl.replace("alibaba-image-thumb-", "alibaba-image-orig-");
|
||
} else if (thumbnailUrl.includes("ruxa-image-thumb-")) {
|
||
originalUrl = thumbnailUrl.replace("ruxa-image-thumb-", "ruxa-image-orig-");
|
||
} else if (thumbnailUrl.includes("fal-image-thumb-")) {
|
||
originalUrl = thumbnailUrl.replace("fal-image-thumb-", "fal-image-orig-");
|
||
}
|
||
}
|
||
|
||
return { originalUrl, imageSize };
|
||
}
|
||
|
||
function extractOriginalImageUrlsFromText(text: string): string[] {
|
||
return extractImageUrlsFromText(text);
|
||
}
|
||
|
||
function normalizeImageInputRef(rawValue: unknown, requestOrigin?: string): string | null {
|
||
if (typeof rawValue !== "string") return null;
|
||
const trimmed = rawValue.trim();
|
||
if (!trimmed) return null;
|
||
|
||
if (/^data:image\/[a-z0-9+.-]+;base64,/i.test(trimmed)) {
|
||
return trimmed.replace(/\s+/g, "");
|
||
}
|
||
|
||
if (/^https?:\/\//i.test(trimmed)) {
|
||
return trimmed;
|
||
}
|
||
|
||
if (requestOrigin && trimmed.startsWith("/")) {
|
||
try {
|
||
return new URL(trimmed, requestOrigin).toString();
|
||
} catch {
|
||
return null;
|
||
}
|
||
}
|
||
|
||
return null;
|
||
}
|
||
|
||
function collectImageInputsFromMessageParts(
|
||
parts: Array<Record<string, any>> | undefined,
|
||
requestOrigin?: string,
|
||
): string[] {
|
||
if (!Array.isArray(parts) || parts.length === 0) return [];
|
||
|
||
const result: string[] = [];
|
||
const pushInput = (rawValue: unknown) => {
|
||
const normalized = normalizeImageInputRef(rawValue, requestOrigin);
|
||
if (!normalized) return;
|
||
if (!result.includes(normalized)) {
|
||
result.push(normalized);
|
||
}
|
||
};
|
||
|
||
for (const part of parts) {
|
||
if (!part) continue;
|
||
|
||
if (part.type === "file") {
|
||
pushInput(part.url);
|
||
pushInput(part.data);
|
||
continue;
|
||
}
|
||
|
||
if (part.type === "image") {
|
||
pushInput(part.url);
|
||
pushInput(part.image);
|
||
pushInput(part.data);
|
||
continue;
|
||
}
|
||
|
||
if (part.type === "image_url") {
|
||
pushInput(part.image_url?.url);
|
||
pushInput(part.url);
|
||
continue;
|
||
}
|
||
|
||
if (part.type === "text" && typeof part.text === "string") {
|
||
const parsedUrls = extractOriginalImageUrlsFromText(part.text);
|
||
for (const parsedUrl of parsedUrls) {
|
||
pushInput(parsedUrl);
|
||
}
|
||
}
|
||
}
|
||
|
||
return result;
|
||
}
|
||
|
||
function mimeTypeToExtension(mimeType: string) {
|
||
const normalized = mimeType.toLowerCase();
|
||
if (normalized.includes("png")) return "png";
|
||
if (normalized.includes("webp")) return "webp";
|
||
if (normalized.includes("gif")) return "gif";
|
||
if (normalized.includes("bmp")) return "bmp";
|
||
return "jpg";
|
||
}
|
||
|
||
function getExtensionFromUrl(url: string): string {
|
||
try {
|
||
const pathname = new URL(url).pathname;
|
||
const ext = pathname.split(".").pop()?.toLowerCase();
|
||
if (!ext) return "jpg";
|
||
if (["jpg", "jpeg", "png", "webp", "gif", "bmp"].includes(ext)) return ext;
|
||
} catch {
|
||
// ignore
|
||
}
|
||
return "jpg";
|
||
}
|
||
|
||
async function uploadRemoteImageToStorage(imageUrl: string): Promise<string> {
|
||
const response = await fetch(imageUrl, {
|
||
method: "GET",
|
||
signal: AbortSignal.timeout(30_000),
|
||
});
|
||
|
||
if (!response.ok) {
|
||
throw new Error(`Remote image fetch failed: ${response.status} ${response.statusText}`);
|
||
}
|
||
|
||
const contentType = response.headers.get("content-type")?.toLowerCase() || "";
|
||
if (contentType && !contentType.startsWith("image/")) {
|
||
throw new Error(`Remote url is not an image: ${contentType}`);
|
||
}
|
||
|
||
const arrayBuffer = await response.arrayBuffer();
|
||
if (!arrayBuffer || arrayBuffer.byteLength === 0) {
|
||
throw new Error("Remote image content is empty");
|
||
}
|
||
|
||
const ext = contentType ? mimeTypeToExtension(contentType) : getExtensionFromUrl(imageUrl);
|
||
const fileName = `remote-image-${Date.now()}-${Math.random().toString(36).slice(2, 8)}.${ext}`;
|
||
const uploaded = await uploadFile(fileName, arrayBuffer);
|
||
return uploaded.url;
|
||
}
|
||
|
||
async function isRemoteImageAccessible(imageUrl: string): Promise<boolean> {
|
||
try {
|
||
const headResponse = await fetch(imageUrl, {
|
||
method: "HEAD",
|
||
signal: AbortSignal.timeout(10_000),
|
||
});
|
||
|
||
if (headResponse.ok) {
|
||
return true;
|
||
}
|
||
|
||
if (headResponse.status !== 405 && headResponse.status !== 403) {
|
||
return false;
|
||
}
|
||
|
||
const probeResponse = await fetch(imageUrl, {
|
||
method: "GET",
|
||
headers: {
|
||
Range: "bytes=0-0",
|
||
},
|
||
signal: AbortSignal.timeout(10_000),
|
||
});
|
||
return probeResponse.ok;
|
||
} catch {
|
||
return false;
|
||
}
|
||
}
|
||
|
||
async function ensureHostedImageUrl(imageUrl: string): Promise<string> {
|
||
const normalizedUrl = imageUrl.trim();
|
||
if (!normalizedUrl) {
|
||
throw new Error("Image URL is empty");
|
||
}
|
||
|
||
if (normalizedUrl.startsWith("data:image")) {
|
||
return uploadDataUrlImage(normalizedUrl);
|
||
}
|
||
|
||
if (!/^https?:\/\//i.test(normalizedUrl)) {
|
||
throw new Error(`Unsupported image URL: ${normalizedUrl}`);
|
||
}
|
||
|
||
if (/\/uploads\//i.test(normalizedUrl)) {
|
||
const accessible = await isRemoteImageAccessible(normalizedUrl);
|
||
if (accessible) {
|
||
return normalizedUrl;
|
||
}
|
||
}
|
||
|
||
return uploadRemoteImageToStorage(normalizedUrl);
|
||
}
|
||
|
||
async function uploadDataUrlImage(dataUrl: string): Promise<string> {
|
||
const match = dataUrl.match(/^data:([^;]+);base64,([\s\S]+)$/i);
|
||
if (!match) {
|
||
throw new Error("Invalid data URL image format");
|
||
}
|
||
|
||
const mimeType = match[1] || "image/jpeg";
|
||
const base64Data = (match[2] || "").replace(/\s+/g, "");
|
||
if (!base64Data) {
|
||
throw new Error("Empty base64 image content");
|
||
}
|
||
|
||
const buffer = Buffer.from(base64Data, "base64");
|
||
const fileExt = mimeTypeToExtension(mimeType);
|
||
const fileName = `inline-image-${Date.now()}-${Math.random().toString(36).slice(2, 8)}.${fileExt}`;
|
||
const arrayBuffer = buffer.buffer.slice(
|
||
buffer.byteOffset,
|
||
buffer.byteOffset + buffer.byteLength
|
||
) as ArrayBuffer;
|
||
const uploaded = await uploadFile(fileName, arrayBuffer);
|
||
return uploaded.url;
|
||
}
|
||
|
||
async function normalizeAssistantImageMarkdown(
|
||
text: string,
|
||
imageSize = "2K"
|
||
): Promise<string> {
|
||
if (!text) return text;
|
||
|
||
const markdownDataUrlRegex = /!\[([^\]]*)\]\s*\(\s*(data:image\/[a-z0-9+.-]+;base64,[^)]+)\s*\)/gim;
|
||
const markdownMatches = [...text.matchAll(markdownDataUrlRegex)];
|
||
let normalizedText = text;
|
||
const uploadedUrls: string[] = [];
|
||
|
||
if (markdownMatches.length > 0) {
|
||
let rebuilt = "";
|
||
let cursor = 0;
|
||
|
||
for (const match of markdownMatches) {
|
||
const matchText = match[0];
|
||
const dataUrl = match[2];
|
||
const start = match.index ?? -1;
|
||
if (start < 0) continue;
|
||
|
||
rebuilt += normalizedText.slice(cursor, start);
|
||
|
||
try {
|
||
const uploadedUrl = await uploadDataUrlImage(dataUrl);
|
||
uploadedUrls.push(uploadedUrl);
|
||
rebuilt += ``;
|
||
} catch (error) {
|
||
console.error("[Chat API] Failed to upload inline data URL image:", error);
|
||
rebuilt += matchText;
|
||
}
|
||
|
||
cursor = start + matchText.length;
|
||
}
|
||
|
||
rebuilt += normalizedText.slice(cursor);
|
||
normalizedText = rebuilt;
|
||
}
|
||
|
||
if (normalizedText.includes("data:image")) {
|
||
const rawDataUrlRegex = /data:image\/[a-z0-9+.-]+;base64,[a-z0-9+/=\s]+/gim;
|
||
const rawMatches = normalizedText.match(rawDataUrlRegex) ?? [];
|
||
|
||
for (const rawDataUrl of rawMatches) {
|
||
try {
|
||
const uploadedUrl = await uploadDataUrlImage(rawDataUrl);
|
||
uploadedUrls.push(uploadedUrl);
|
||
normalizedText = normalizedText.replace(rawDataUrl, uploadedUrl);
|
||
} catch (error) {
|
||
console.error("[Chat API] Failed to normalize raw data URL image:", error);
|
||
}
|
||
}
|
||
}
|
||
|
||
const markdownHttpImageRegex = /!\[([^\]]*)\]\s*\(\s*(https?:\/\/[^)\s]+)\s*\)/gim;
|
||
const markdownHttpMatches = [...normalizedText.matchAll(markdownHttpImageRegex)];
|
||
|
||
if (markdownHttpMatches.length > 0) {
|
||
let rebuilt = "";
|
||
let cursor = 0;
|
||
|
||
for (const match of markdownHttpMatches) {
|
||
const matchText = match[0];
|
||
const altText = match[1] || "";
|
||
const thumbnailUrl = (match[2] || "").trim();
|
||
const start = match.index ?? -1;
|
||
if (start < 0) continue;
|
||
|
||
rebuilt += normalizedText.slice(cursor, start);
|
||
|
||
try {
|
||
const { originalUrl, imageSize: parsedImageSize } = parseImageAltMetadata(
|
||
altText,
|
||
thumbnailUrl
|
||
);
|
||
|
||
let hostedThumbnailUrl = thumbnailUrl;
|
||
let hostedOriginalUrl =
|
||
originalUrl && originalUrl !== thumbnailUrl ? originalUrl : thumbnailUrl;
|
||
|
||
try {
|
||
hostedThumbnailUrl = await ensureHostedImageUrl(thumbnailUrl);
|
||
} catch (error) {
|
||
console.error("[Chat API] Failed to normalize remote thumbnail URL:", error);
|
||
}
|
||
|
||
if (originalUrl && originalUrl !== thumbnailUrl) {
|
||
try {
|
||
hostedOriginalUrl = await ensureHostedImageUrl(originalUrl);
|
||
} catch (error) {
|
||
console.error("[Chat API] Failed to normalize remote original URL:", error);
|
||
hostedOriginalUrl = hostedThumbnailUrl || originalUrl;
|
||
}
|
||
} else {
|
||
hostedOriginalUrl = hostedThumbnailUrl;
|
||
}
|
||
|
||
const finalThumbnailUrl = hostedThumbnailUrl || thumbnailUrl;
|
||
const finalOriginalUrl = hostedOriginalUrl || finalThumbnailUrl;
|
||
|
||
uploadedUrls.push(finalThumbnailUrl);
|
||
uploadedUrls.push(finalOriginalUrl);
|
||
rebuilt += ``;
|
||
} catch (error) {
|
||
console.error("[Chat API] Failed to parse remote image markdown metadata:", error);
|
||
rebuilt += matchText;
|
||
}
|
||
|
||
cursor = start + matchText.length;
|
||
}
|
||
|
||
rebuilt += normalizedText.slice(cursor);
|
||
normalizedText = rebuilt;
|
||
}
|
||
|
||
const hasMarkdownImage = /!\[[^\]]*\]\((https?:\/\/[^)]+)\)/i.test(normalizedText);
|
||
if (!hasMarkdownImage && uploadedUrls.length > 0) {
|
||
const uniqueUrls = [...new Set(uploadedUrls)];
|
||
console.log(
|
||
`[Chat API] Converted ${uniqueUrls.length} image link(s) to hosted link(s)`
|
||
);
|
||
return uniqueUrls
|
||
.map((url) => ``)
|
||
.join("\n");
|
||
}
|
||
|
||
return normalizedText;
|
||
}
|
||
|
||
function prependImageModelBadge(text: string, modelDisplayName?: string | null): string {
|
||
const normalizedText = String(text || "").trim();
|
||
if (!normalizedText) return normalizedText;
|
||
|
||
const normalizedModelName = String(modelDisplayName || "").trim();
|
||
if (!normalizedModelName) return normalizedText;
|
||
|
||
if (
|
||
/^\s*[◈◇◆🔹⬡✦•·]\s*[^\n]+/im.test(normalizedText) ||
|
||
/^\s*(?:Routed to|Using model|使用模型|模型)\s*[:：]/im.test(normalizedText)
|
||
) {
|
||
return normalizedText;
|
||
}
|
||
|
||
return `◈ ${normalizedModelName}\n\n${normalizedText}`;
|
||
}
|
||
|
||
function resolveModelDisplayNameByModelId(
|
||
models: Awaited<ReturnType<typeof getEnabledAiModelsWithProvider>> | undefined,
|
||
modelId: string | undefined,
|
||
): string {
|
||
if (!modelId || !models?.length) return "";
|
||
const match = models.find(
|
||
(item) => item.model.modelId === modelId || item.model.id === modelId,
|
||
);
|
||
return String(match?.model?.displayName || "");
|
||
}
|
||
|
||
/**
|
||
* 解析生图 Markdown 并保存图片记录到数据库
|
||
* 支持三种格式：thumb|||url|||size、thumb$$url$$size、thumb|url|size
|
||
*/
|
||
async function saveGeneratedImages(
|
||
imageMarkdown: string,
|
||
userId: string,
|
||
chatId: string,
|
||
messageId: string,
|
||
prompt: string,
|
||
modelId: string,
|
||
options?: {
|
||
sourceAssetIds?: string[];
|
||
sourceImageUrls?: string[];
|
||
relationType?: "generated_from" | "edited_from";
|
||
sourceType?: string;
|
||
projectId?: string | null;
|
||
userPrompt?: string;
|
||
optimizedPrompt?: string;
|
||
requestedImageCount?: number;
|
||
aspectRatio?: string;
|
||
imageSize?: string;
|
||
},
|
||
): Promise<void> {
|
||
// 匹配 Markdown 图片格式: 
|
||
const imageRegex = /!\[([^\]]*)\]\(([^)]+)\)/g;
|
||
const matches = [...imageMarkdown.matchAll(imageRegex)];
|
||
|
||
let savedImageCount = 0;
|
||
const savedAssetIds: string[] = [];
|
||
const savedOriginalUrls: string[] = [];
|
||
let relationSourceAssetIds = Array.isArray(options?.sourceAssetIds)
|
||
? options.sourceAssetIds.filter((assetId): assetId is string => Boolean(assetId))
|
||
: [];
|
||
|
||
if (
|
||
relationSourceAssetIds.length === 0 &&
|
||
Array.isArray(options?.sourceImageUrls) &&
|
||
options.sourceImageUrls.length > 0
|
||
) {
|
||
try {
|
||
const sourceAssets = await getMediaAssetsByUserAndUrls({
|
||
userId,
|
||
urls: options.sourceImageUrls,
|
||
});
|
||
relationSourceAssetIds = sourceAssets.map((asset) => asset.id);
|
||
} catch (sourceLookupError) {
|
||
console.error("[Gallery] Failed to resolve source assets by URL:", sourceLookupError);
|
||
}
|
||
}
|
||
|
||
for (const match of matches) {
|
||
const altText = match[1] || "";
|
||
const thumbnailUrl = match[2];
|
||
const { originalUrl, imageSize } = parseImageAltMetadata(altText, thumbnailUrl);
|
||
|
||
try {
|
||
const [savedImage] = await db.insert(generatedImage).values({
|
||
userId,
|
||
chatId,
|
||
messageId,
|
||
thumbnailUrl,
|
||
originalUrl,
|
||
imageSize,
|
||
prompt,
|
||
modelId,
|
||
}).returning();
|
||
savedImageCount += 1;
|
||
if (!savedOriginalUrls.includes(originalUrl)) {
|
||
savedOriginalUrls.push(originalUrl);
|
||
}
|
||
const media = await upsertMediaAsset({
|
||
userId,
|
||
chatId,
|
||
projectId: options?.projectId ?? null,
|
||
mediaType: "image",
|
||
sourceType: options?.sourceType || "generated",
|
||
status: "ready",
|
||
originalUrl,
|
||
previewUrl: originalUrl,
|
||
thumbnailUrl,
|
||
displayName: prompt?.slice(0, 80) || "AI 生成图片",
|
||
prompt,
|
||
modelId,
|
||
sourceMessageId: messageId,
|
||
sourceTable: "GeneratedImage",
|
||
sourceRowId: savedImage?.id || null,
|
||
primaryParentAssetId: relationSourceAssetIds[0] || null,
|
||
metadata: {
|
||
imageSize,
|
||
},
|
||
});
|
||
if (media?.id && !savedAssetIds.includes(media.id)) {
|
||
savedAssetIds.push(media.id);
|
||
}
|
||
if (media && relationSourceAssetIds.length > 0) {
|
||
for (const sourceAssetId of relationSourceAssetIds) {
|
||
if (!sourceAssetId) continue;
|
||
await createMediaAssetRelation({
|
||
userId,
|
||
chatId,
|
||
projectId: options?.projectId ?? null,
|
||
fromAssetId: sourceAssetId,
|
||
toAssetId: media.id,
|
||
relationType: options?.relationType || "generated_from",
|
||
sourceMessageId: messageId,
|
||
metadata: {
|
||
modelId,
|
||
prompt,
|
||
sourceImageUrls: options?.sourceImageUrls || [],
|
||
},
|
||
});
|
||
}
|
||
}
|
||
console.log(`[Gallery] Saved image: ${originalUrl.substring(0, 80)}...`);
|
||
} catch (error) {
|
||
console.error("[Gallery] Failed to save image record:", error);
|
||
}
|
||
}
|
||
|
||
if (savedOriginalUrls.length > 0) {
|
||
try {
|
||
const resultAssetIds = savedAssetIds.length > 0 ? savedAssetIds : null;
|
||
await persistImageWorkflowTurn({
|
||
userId,
|
||
chatId,
|
||
projectId: options?.projectId || null,
|
||
currentIntent: options?.relationType === "edited_from" ? "image_edit" : "image_gen",
|
||
currentFocusAssetIds: resultAssetIds,
|
||
currentFocusImageUrls: savedOriginalUrls,
|
||
lastGeneratedAssetIds: resultAssetIds,
|
||
lastGeneratedImageUrls: savedOriginalUrls,
|
||
lastResolvedTargetAssetIds: resultAssetIds,
|
||
lastResolvedTargetImageUrls: savedOriginalUrls,
|
||
lastMessageId: messageId,
|
||
lastUserPrompt: String(options?.userPrompt || prompt || "").trim() || prompt,
|
||
lastOptimizedPrompt:
|
||
String(
|
||
options?.optimizedPrompt ||
|
||
(options?.relationType !== "edited_from" ? prompt : "") ||
|
||
"",
|
||
).trim() || undefined,
|
||
lastRequestedImageCount: options?.requestedImageCount || savedOriginalUrls.length,
|
||
lastAspectRatio: options?.aspectRatio,
|
||
lastImageSize: options?.imageSize,
|
||
lastSourceImageUrls: options?.sourceImageUrls || null,
|
||
lastResultImageUrls: savedOriginalUrls,
|
||
});
|
||
} catch (memoryError) {
|
||
console.warn("[Gallery] Failed to update working state with generated images:", memoryError);
|
||
}
|
||
}
|
||
|
||
if (savedImageCount <= 0) return;
|
||
|
||
try {
|
||
const pointsConfig = await getPointsSystemConfig();
|
||
if (!pointsConfig.enabled) return;
|
||
|
||
const userData = await getUserById({ id: userId }).catch(() => undefined);
|
||
if (userData?.isHighVip) {
|
||
console.log(`[Points] Skip image points for VIP user=${userId} model=${modelId}`);
|
||
return;
|
||
}
|
||
|
||
const modelConfig = await resolveBillingModelByReference(modelId);
|
||
const unitCost = getModelCreditCost(modelConfig, "image");
|
||
if (unitCost <= 0) return;
|
||
|
||
const totalCost = unitCost * savedImageCount;
|
||
const consumeResult = await consumeUserPoints({
|
||
userId,
|
||
amount: totalCost,
|
||
reason: "image_generation",
|
||
modelId: modelConfig?.id || modelId,
|
||
chatId,
|
||
messageId,
|
||
note: `模型(${modelConfig?.displayName || modelId}) 生图 ${savedImageCount} 张`,
|
||
});
|
||
|
||
if (!consumeResult.ok) {
|
||
console.warn(
|
||
`[Points] Failed to consume points after generation. user=${userId} model=${modelId} cost=${totalCost}`
|
||
);
|
||
} else if (consumeResult.skipped) {
|
||
console.log(
|
||
`[Points] Skipped ${totalCost} image points for VIP user=${userId} model=${modelId}`
|
||
);
|
||
} else {
|
||
console.log(
|
||
`[Points] Consumed ${totalCost} points (${unitCost} x ${savedImageCount}). balance=${consumeResult.balance}`
|
||
);
|
||
}
|
||
} catch (error) {
|
||
console.error("[Points] Failed to consume points after image generation:", error);
|
||
}
|
||
}
|
||
|
||
type AgentPresetMessageImagePart = {
|
||
type: "file";
|
||
url: string;
|
||
name: string;
|
||
mediaType: string;
|
||
assetId?: string;
|
||
};
|
||
|
||
function extractCurrentMessageImageParts(
|
||
parts: Array<Record<string, any>>,
|
||
): AgentPresetMessageImagePart[] {
|
||
return parts
|
||
.filter(
|
||
(part): part is AgentPresetMessageImagePart =>
|
||
part?.type === "file" &&
|
||
typeof part?.url === "string" &&
|
||
/^https?:\/\//i.test(part.url) &&
|
||
typeof part?.mediaType === "string" &&
|
||
part.mediaType.startsWith("image/")
|
||
)
|
||
.map((part) => ({
|
||
type: "file",
|
||
url: String(part.url).trim(),
|
||
name: String(part.name || "图片").trim() || "图片",
|
||
mediaType: String(part.mediaType || "image/jpeg").trim() || "image/jpeg",
|
||
assetId:
|
||
typeof part.assetId === "string" && part.assetId.trim() ? part.assetId.trim() : undefined,
|
||
}));
|
||
}
|
||
|
||
function getSceneUploadedImageBreakdown(params: {
|
||
uploadedImageParts: AgentPresetMessageImagePart[];
|
||
fallbackPrimaryCount: number;
|
||
fallbackIncludeModelImage: boolean;
|
||
}) {
|
||
const normalizedNames = params.uploadedImageParts.map((part) => String(part.name || "").trim());
|
||
const primaryCountFromNames = normalizedNames.filter((name) => name.startsWith("主SKU参考图")).length;
|
||
const secondaryCountFromNames = normalizedNames.filter((name) => name.startsWith("搭配SKU参考图")).length;
|
||
const hasModelFromNames = normalizedNames.some((name) => name.startsWith("品牌模特图"));
|
||
const hasNamedBreakdown =
|
||
primaryCountFromNames > 0 || secondaryCountFromNames > 0 || hasModelFromNames;
|
||
|
||
if (hasNamedBreakdown) {
|
||
return {
|
||
primaryProductImageCount: primaryCountFromNames,
|
||
bundleProductImageCount: secondaryCountFromNames,
|
||
includeModelImage: hasModelFromNames,
|
||
};
|
||
}
|
||
|
||
const uploadedImageCount = params.uploadedImageParts.length;
|
||
const includeModelImage =
|
||
params.fallbackIncludeModelImage && uploadedImageCount > params.fallbackPrimaryCount;
|
||
const bundleProductImageCount = Math.max(
|
||
0,
|
||
uploadedImageCount - params.fallbackPrimaryCount - (includeModelImage ? 1 : 0)
|
||
);
|
||
const primaryProductImageCount = Math.max(
|
||
0,
|
||
Math.min(
|
||
params.fallbackPrimaryCount,
|
||
uploadedImageCount - bundleProductImageCount - (includeModelImage ? 1 : 0)
|
||
)
|
||
);
|
||
|
||
return {
|
||
primaryProductImageCount,
|
||
bundleProductImageCount,
|
||
includeModelImage,
|
||
};
|
||
}
|
||
|
||
function resolveScenePromptOptimizationImageUrls(params: {
|
||
uploadedImageParts: AgentPresetMessageImagePart[];
|
||
primaryProductImageCount: number;
|
||
bundleProductImageCount: number;
|
||
includeModelImage: boolean;
|
||
}) {
|
||
const namedPrimaryParts = params.uploadedImageParts.filter((part) =>
|
||
String(part.name || "").trim().startsWith("主SKU参考图")
|
||
);
|
||
const namedModelPart = params.uploadedImageParts.find((part) =>
|
||
String(part.name || "").trim().startsWith("品牌模特图")
|
||
);
|
||
|
||
if (namedPrimaryParts.length > 0 || namedModelPart) {
|
||
return {
|
||
primaryImageUrl: namedPrimaryParts[0]?.url || "",
|
||
modelImageUrl: namedModelPart?.url || null,
|
||
};
|
||
}
|
||
|
||
const primaryImageUrl =
|
||
params.uploadedImageParts.slice(0, Math.max(0, params.primaryProductImageCount))[0]?.url || "";
|
||
const modelIndex = params.primaryProductImageCount + params.bundleProductImageCount;
|
||
|
||
return {
|
||
primaryImageUrl,
|
||
modelImageUrl:
|
||
params.includeModelImage && modelIndex < params.uploadedImageParts.length
|
||
? params.uploadedImageParts[modelIndex]?.url || null
|
||
: null,
|
||
};
|
||
}
|
||
|
||
function normalizeAgentSupplementPrompt(text: string, presetName?: string | null) {
|
||
const normalized = String(text || "").trim();
|
||
if (!normalized) return "";
|
||
const escapedPresetName = String(presetName || "").replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
||
const genericPattern = escapedPresetName
|
||
? new RegExp(`^请按「${escapedPresetName}」智能体配置顺序处理这\\d+张图片。?$`)
|
||
: /^请按「.+」智能体配置顺序处理这\d+张图片。?$/;
|
||
if (genericPattern.test(normalized)) {
|
||
return "";
|
||
}
|
||
return normalized;
|
||
}
|
||
|
||
function normalizeAgentWorkflowPrompt(text: string) {
|
||
return String(text || "")
|
||
.split(/\n+/)
|
||
.map((line) => line.replace(/\s+/g, " ").trim())
|
||
.filter(Boolean)
|
||
.join("\n")
|
||
.trim();
|
||
}
|
||
|
||
function parseAgentWorkflowPromptSections(text: string) {
|
||
const sectionMap = new Map<string, string[]>();
|
||
let activeSection = "__root__";
|
||
const normalizedPrompt = normalizeAgentWorkflowPrompt(text);
|
||
if (!normalizedPrompt) {
|
||
return sectionMap;
|
||
}
|
||
|
||
normalizedPrompt.split("\n").forEach((line) => {
|
||
const trimmedLine = line.trim();
|
||
if (!trimmedLine) {
|
||
return;
|
||
}
|
||
const headerMatch = trimmedLine.match(/^【([^】]{1,30})】$/);
|
||
if (headerMatch) {
|
||
activeSection = headerMatch[1].trim();
|
||
if (!sectionMap.has(activeSection)) {
|
||
sectionMap.set(activeSection, []);
|
||
}
|
||
return;
|
||
}
|
||
|
||
const currentLines = sectionMap.get(activeSection) || [];
|
||
currentLines.push(trimmedLine);
|
||
sectionMap.set(activeSection, currentLines);
|
||
});
|
||
|
||
return sectionMap;
|
||
}
|
||
|
||
function buildAgentPromptSectionText(
|
||
title: string,
|
||
lines: string[],
|
||
options?: { bullet?: boolean }
|
||
) {
|
||
const normalizedLines = lines
|
||
.map((line) => String(line || "").trim())
|
||
.filter(Boolean);
|
||
if (normalizedLines.length === 0) {
|
||
return "";
|
||
}
|
||
|
||
const sectionBody = options?.bullet
|
||
? normalizedLines.map((line) => (line.startsWith("- ") ? line : `- ${line}`)).join("\n")
|
||
: normalizedLines.join("\n");
|
||
|
||
return `【${title}】\n${sectionBody}`;
|
||
}
|
||
|
||
function buildOptimizedAgentSceneWorkflowPrompt(params: {
|
||
workflowPrompt: string;
|
||
optimizationResult?: {
|
||
sceneGoal: string;
|
||
environmentSummary: string;
|
||
propSummary: string;
|
||
cameraLightingSummary: string;
|
||
characterSummary: string;
|
||
negativeConstraints: string[];
|
||
conflicts: string[];
|
||
} | null;
|
||
}) {
|
||
const normalizedWorkflowPrompt = normalizeAgentWorkflowPrompt(params.workflowPrompt);
|
||
if (!normalizedWorkflowPrompt) {
|
||
return "";
|
||
}
|
||
|
||
const optimizationResult = params.optimizationResult || null;
|
||
if (!optimizationResult) {
|
||
return normalizedWorkflowPrompt;
|
||
}
|
||
|
||
const sectionMap = parseAgentWorkflowPromptSections(normalizedWorkflowPrompt);
|
||
const sceneGoalLines = [optimizationResult.sceneGoal, optimizationResult.environmentSummary].filter(
|
||
Boolean
|
||
);
|
||
const propLines = [optimizationResult.propSummary].filter(Boolean);
|
||
const cameraLightingLines = [optimizationResult.cameraLightingSummary].filter(Boolean);
|
||
const characterLines = [optimizationResult.characterSummary].filter(Boolean);
|
||
const negativeLines = Array.from(
|
||
new Set([
|
||
...optimizationResult.negativeConstraints,
|
||
...optimizationResult.conflicts.map((item) => `冲突校正：${item}`),
|
||
])
|
||
);
|
||
|
||
const optimizedSections = [
|
||
buildAgentPromptSectionText("项目背景", sectionMap.get("项目背景") || sectionMap.get("__root__") || []),
|
||
buildAgentPromptSectionText("主SKU锁定", sectionMap.get("主SKU锁定") || []),
|
||
buildAgentPromptSectionText("搭配SKU锁定", sectionMap.get("搭配SKU锁定") || []),
|
||
buildAgentPromptSectionText("参考图", sectionMap.get("参考图") || []),
|
||
buildAgentPromptSectionText("场景创作总纲", sceneGoalLines),
|
||
buildAgentPromptSectionText("陈列与道具", propLines),
|
||
buildAgentPromptSectionText("机位与光线", cameraLightingLines),
|
||
buildAgentPromptSectionText("人物规则", characterLines),
|
||
buildAgentPromptSectionText("负面约束", negativeLines, { bullet: true }),
|
||
].filter(Boolean);
|
||
|
||
return optimizedSections.join("\n\n").trim();
|
||
}
|
||
|
||
const AGENT_PROMPT_STRUCTURAL_LABELS = new Set([
|
||
"角色",
|
||
"技能",
|
||
"目标",
|
||
"约束条件",
|
||
"流程",
|
||
"输出",
|
||
]);
|
||
|
||
const AGENT_PROMPT_LABEL_SEMANTIC_ALIASES = new Map<string, string>([
|
||
["角色", "role"],
|
||
["技能", "skills"],
|
||
["目标", "goal"],
|
||
["约束条件", "constraints"],
|
||
["流程", "process"],
|
||
["输出", "output"],
|
||
["人群选择", "audience"],
|
||
["人物", "audience"],
|
||
["主题风格", "style"],
|
||
["整体风格", "style"],
|
||
["空间选择", "space"],
|
||
["空间布局", "space"],
|
||
["主空间", "space"],
|
||
["子空间选择", "subspace"],
|
||
["子空间", "subspace"],
|
||
["硬装方案", "hardDecoration"],
|
||
["硬装", "hardDecoration"],
|
||
["软装方案", "softDecoration"],
|
||
["软装", "softDecoration"],
|
||
["光氛选择", "lighting"],
|
||
["光源布置", "lighting"],
|
||
["灯具", "lighting"],
|
||
["机位选择", "camera"],
|
||
["拍摄角度", "camera"],
|
||
["配饰陈列", "accessories"],
|
||
["放在产品内的小件配饰", "accessories"],
|
||
["前景", "sceneLayout"],
|
||
["中景", "sceneLayout"],
|
||
["背景", "sceneLayout"],
|
||
]);
|
||
|
||
const AGENT_SCENE_WORKFLOW_OWNED_SEMANTICS = new Set([
|
||
"role",
|
||
"skills",
|
||
"goal",
|
||
"constraints",
|
||
"process",
|
||
"output",
|
||
"audience",
|
||
"style",
|
||
"space",
|
||
"subspace",
|
||
"hardDecoration",
|
||
"softDecoration",
|
||
"lighting",
|
||
"camera",
|
||
"accessories",
|
||
"sceneLayout",
|
||
]);
|
||
|
||
function normalizePromptLineForCompare(line: string) {
|
||
return String(line || "")
|
||
.replace(/\s+/g, " ")
|
||
.replace(/[：:]/g, ":")
|
||
.replace(/[。；;，,、!！?？]+$/g, "")
|
||
.trim()
|
||
.toLowerCase();
|
||
}
|
||
|
||
function parsePromptLineLabel(line: string) {
|
||
const normalized = String(line || "").replace(/\s+/g, " ").trim();
|
||
if (!normalized) return { label: "", value: "" };
|
||
|
||
const headerOnlyMatch = normalized.match(/^([^:：]{1,20})[:：]\s*$/);
|
||
if (headerOnlyMatch) {
|
||
return {
|
||
label: headerOnlyMatch[1].trim(),
|
||
value: "",
|
||
};
|
||
}
|
||
|
||
const colonMatch = normalized.match(/^([^:：]{1,20})[:：]\s*(.+)$/);
|
||
if (colonMatch) {
|
||
return {
|
||
label: colonMatch[1].trim(),
|
||
value: colonMatch[2].trim(),
|
||
};
|
||
}
|
||
|
||
const whitespaceMatch = normalized.match(/^([^\s]{1,20})\s+(.+)$/);
|
||
if (whitespaceMatch) {
|
||
return {
|
||
label: whitespaceMatch[1].trim(),
|
||
value: whitespaceMatch[2].trim(),
|
||
};
|
||
}
|
||
|
||
return { label: "", value: normalized };
|
||
}
|
||
|
||
function normalizePromptLabel(label: string) {
|
||
return String(label || "")
|
||
.replace(/\s+/g, "")
|
||
.replace(/[：:]/g, "")
|
||
.trim();
|
||
}
|
||
|
||
function getPromptSemanticLabel(label: string) {
|
||
const normalized = normalizePromptLabel(label);
|
||
return AGENT_PROMPT_LABEL_SEMANTIC_ALIASES.get(normalized) || normalized;
|
||
}
|
||
|
||
function collectPromptSemanticLabelsFromText(text: string) {
|
||
const semanticLabels = new Set<string>();
|
||
const normalizedText = String(text || "").replace(/\s+/g, " ").trim();
|
||
if (!normalizedText) {
|
||
return semanticLabels;
|
||
}
|
||
|
||
for (const [label, semantic] of AGENT_PROMPT_LABEL_SEMANTIC_ALIASES.entries()) {
|
||
if (!label) {
|
||
continue;
|
||
}
|
||
if (normalizedText.includes(`${label}：`) || normalizedText.includes(`${label}:`)) {
|
||
semanticLabels.add(semantic);
|
||
continue;
|
||
}
|
||
if (normalizedText.includes(`${label} `) || normalizedText.includes(label)) {
|
||
semanticLabels.add(semantic);
|
||
}
|
||
}
|
||
|
||
return semanticLabels;
|
||
}
|
||
|
||
function shouldDropSceneSlotSemanticLine(
|
||
slotSemanticLabel: string,
|
||
workflowSemanticLabels: Set<string>,
|
||
) {
|
||
if (!slotSemanticLabel || !AGENT_SCENE_WORKFLOW_OWNED_SEMANTICS.has(slotSemanticLabel)) {
|
||
return false;
|
||
}
|
||
|
||
if (slotSemanticLabel === "sceneLayout") {
|
||
return [
|
||
"style",
|
||
"space",
|
||
"subspace",
|
||
"hardDecoration",
|
||
"softDecoration",
|
||
"accessories",
|
||
"camera",
|
||
"lighting",
|
||
].some((semantic) => workflowSemanticLabels.has(semantic));
|
||
}
|
||
|
||
return workflowSemanticLabels.has(slotSemanticLabel);
|
||
}
|
||
|
||
function dedupeAgentSlotPromptAgainstWorkflow(params: {
|
||
workflowPrompt?: string | null;
|
||
slotPrompt?: string | null;
|
||
modeType?: "scene_generation" | "detail_replacement" | "sku_replacement" | "generic";
|
||
}) {
|
||
const dedupedSlotLines: string[] = [];
|
||
const seenExactLines = new Set<string>();
|
||
const workflowStructuralValues = new Map<string, string>();
|
||
const workflowSemanticLabels = new Set<string>();
|
||
|
||
const rememberLine = (line: string) => {
|
||
const normalizedLine = normalizePromptLineForCompare(line);
|
||
if (!normalizedLine) {
|
||
return;
|
||
}
|
||
seenExactLines.add(normalizedLine);
|
||
};
|
||
|
||
const normalizedWorkflowLines = normalizeAgentWorkflowPrompt(params.workflowPrompt || "")
|
||
.split("\n")
|
||
.map((line) => line.trim())
|
||
.filter(Boolean);
|
||
normalizedWorkflowLines.forEach((line) => {
|
||
const parsed = parsePromptLineLabel(line);
|
||
if (parsed.label) {
|
||
workflowSemanticLabels.add(getPromptSemanticLabel(parsed.label));
|
||
}
|
||
collectPromptSemanticLabelsFromText(line).forEach((semantic) => {
|
||
workflowSemanticLabels.add(semantic);
|
||
});
|
||
collectPromptSemanticLabelsFromText(parsed.value || "").forEach((semantic) => {
|
||
workflowSemanticLabels.add(semantic);
|
||
});
|
||
if (parsed.label && AGENT_PROMPT_STRUCTURAL_LABELS.has(parsed.label)) {
|
||
workflowStructuralValues.set(
|
||
parsed.label,
|
||
normalizePromptLineForCompare(parsed.value || "")
|
||
);
|
||
}
|
||
rememberLine(line);
|
||
});
|
||
|
||
const normalizedSlotLines = String(params.slotPrompt || "")
|
||
.split(/\n+/)
|
||
.map((line) => line.replace(/\s+/g, " ").trim())
|
||
.filter(Boolean);
|
||
let activeSlotSemanticLabel = "";
|
||
normalizedSlotLines.forEach((line) => {
|
||
const normalizedLine = normalizePromptLineForCompare(line);
|
||
if (!normalizedLine || seenExactLines.has(normalizedLine)) {
|
||
return;
|
||
}
|
||
const parsed = parsePromptLineLabel(line);
|
||
const slotSemanticLabel = parsed.label ? getPromptSemanticLabel(parsed.label) : "";
|
||
if (slotSemanticLabel) {
|
||
activeSlotSemanticLabel = slotSemanticLabel;
|
||
}
|
||
if (
|
||
params.modeType === "scene_generation" &&
|
||
shouldDropSceneSlotSemanticLine(
|
||
slotSemanticLabel || activeSlotSemanticLabel,
|
||
workflowSemanticLabels,
|
||
)
|
||
) {
|
||
return;
|
||
}
|
||
if (parsed.label && AGENT_PROMPT_STRUCTURAL_LABELS.has(parsed.label)) {
|
||
const normalizedValue = normalizePromptLineForCompare(parsed.value || "");
|
||
const workflowValue = workflowStructuralValues.get(parsed.label);
|
||
if (workflowValue && workflowValue === normalizedValue) {
|
||
return;
|
||
}
|
||
}
|
||
seenExactLines.add(normalizedLine);
|
||
dedupedSlotLines.push(line);
|
||
});
|
||
|
||
return dedupedSlotLines.join("\n").trim();
|
||
}
|
||
|
||
function buildAgentSlotPrompt(params: {
|
||
workflowPrompt?: string | null;
|
||
slotPrompt: string;
|
||
slotTitle?: string | null;
|
||
userSupplementPrompt?: string | null;
|
||
includeSlotTitle?: boolean;
|
||
modeType?: "scene_generation" | "detail_replacement" | "sku_replacement" | "generic";
|
||
}) {
|
||
const lines: string[] = [];
|
||
const workflowPrompt = normalizeAgentWorkflowPrompt(params.workflowPrompt || "");
|
||
if (workflowPrompt) {
|
||
lines.push(workflowPrompt);
|
||
}
|
||
const slotTitle = String(params.slotTitle || "").trim();
|
||
if (params.includeSlotTitle !== false && slotTitle) {
|
||
lines.push(`【槽位】${slotTitle}`);
|
||
}
|
||
const dedupedSlotPrompt = dedupeAgentSlotPromptAgainstWorkflow({
|
||
workflowPrompt,
|
||
slotPrompt: params.slotPrompt,
|
||
modeType: params.modeType || "generic",
|
||
});
|
||
if (dedupedSlotPrompt) {
|
||
lines.push(dedupedSlotPrompt);
|
||
}
|
||
const supplement = String(params.userSupplementPrompt || "").trim();
|
||
if (supplement) {
|
||
lines.push(`【补充要求】${supplement}`);
|
||
}
|
||
return lines.join("\n\n").trim();
|
||
}
|
||
|
||
function buildAgentSceneWorkflowPrompt(params: {
|
||
workflowPrompt?: string | null;
|
||
slotPrompt?: string | null;
|
||
userSupplementPrompt?: string | null;
|
||
imageRolePrompt?: string | null;
|
||
}) {
|
||
const lines: string[] = [];
|
||
const workflowPrompt = normalizeAgentWorkflowPrompt(params.workflowPrompt || "");
|
||
if (workflowPrompt) {
|
||
lines.push(workflowPrompt);
|
||
}
|
||
const slotPrompt = String(params.slotPrompt || "")
|
||
.split(/\n+/)
|
||
.map((line) => line.replace(/\s+/g, " ").trim())
|
||
.filter(Boolean)
|
||
.join("\n")
|
||
.trim();
|
||
if (slotPrompt) {
|
||
lines.push(slotPrompt);
|
||
}
|
||
let imageRolePrompt = String(params.imageRolePrompt || "").trim();
|
||
if (workflowPrompt.includes("品牌模特图：已选") || workflowPrompt.includes("固定模特图")) {
|
||
imageRolePrompt = imageRolePrompt
|
||
.replace(
|
||
/^如当前请求未附带固定模特图，仅在工作流明确要求人物时再安排辅助人物，且人物不能喧宾夺主。$/m,
|
||
""
|
||
)
|
||
.replace(/^当前没有固定模特图，不要凭空生成强人物主体。$/m, "")
|
||
.split(/\n+/)
|
||
.map((line) => line.trim())
|
||
.filter(Boolean)
|
||
.join("\n");
|
||
}
|
||
if (imageRolePrompt) {
|
||
lines.push(imageRolePrompt);
|
||
}
|
||
const supplement = String(params.userSupplementPrompt || "").trim();
|
||
if (supplement) {
|
||
lines.push(`【补充要求】${supplement}`);
|
||
}
|
||
return lines.join("\n\n").trim();
|
||
}
|
||
|
||
type AgentReplacementModePayload = NonNullable<PostRequestBody["agentModePayload"]>;
|
||
|
||
type ReplacementReferenceDecision = {
|
||
shouldReplace: boolean;
|
||
containsFurnitureProduct: boolean;
|
||
reason: string;
|
||
detectionModelName?: string | null;
|
||
detectionSource:
|
||
| "local_moonshot"
|
||
| "configured_vision"
|
||
| "configured_qwen3_vl"
|
||
| "fallback_qwen3_vl"
|
||
| "safe_passthrough";
|
||
};
|
||
|
||
type ReplacementScenePlan = {
|
||
useModelImage: boolean;
|
||
modelImageReason: string;
|
||
sceneSummary: string;
|
||
cameraSummary: string;
|
||
placementSummary: string;
|
||
preserveSummary: string;
|
||
styleSummary: string;
|
||
avoidSummary: string;
|
||
generationPrompt: string;
|
||
selectedProductImageIndexes: number[];
|
||
productSelectionReason: string;
|
||
analysisModelName?: string | null;
|
||
analysisSource: "local_moonshot" | "configured_vision" | "heuristic";
|
||
};
|
||
|
||
type ReplacementExecutionPlan = ReplacementReferenceDecision & {
|
||
scenePlan: ReplacementScenePlan;
|
||
};
|
||
|
||
type ReplacementScenePlanningRuntime =
|
||
| {
|
||
kind: "local_moonshot";
|
||
model: any;
|
||
modelName: string;
|
||
}
|
||
| {
|
||
kind: "configured_vision";
|
||
model: any;
|
||
modelName: string;
|
||
}
|
||
| null;
|
||
|
||
const QWEN_VL_MODEL_MATCHER = /(qwen|千问|通义).*(vl|vision)|qwen3-vl|qwen-vl/i;
|
||
|
||
function truncateReplacementAnalysisText(value: unknown, maxLength = 240) {
|
||
return String(value || "").trim().replace(/\s+/g, " ").slice(0, maxLength);
|
||
}
|
||
|
||
function parseReplacementScenePlan(
|
||
text: string,
|
||
params: {
|
||
productImageCount: number;
|
||
hasModelImage: boolean;
|
||
},
|
||
): ReplacementScenePlan | null {
|
||
const jsonText = extractJsonObjectText(text);
|
||
if (!jsonText) {
|
||
return null;
|
||
}
|
||
|
||
try {
|
||
const parsed = JSON.parse(jsonText) as Record<string, unknown>;
|
||
const useModelImageRaw =
|
||
normalizeBooleanLike(parsed.useModelImage) ??
|
||
normalizeBooleanLike(parsed.needModelImage) ??
|
||
normalizeBooleanLike(parsed.referenceNeedsModel) ??
|
||
false;
|
||
|
||
const generationPrompt = truncateReplacementAnalysisText(
|
||
parsed.generationPrompt || parsed.recommendedScenePrompt || parsed.scenePrompt,
|
||
1200,
|
||
);
|
||
|
||
const scenePlan: ReplacementScenePlan = {
|
||
useModelImage: params.hasModelImage ? Boolean(useModelImageRaw) : false,
|
||
modelImageReason: truncateReplacementAnalysisText(
|
||
parsed.modelImageReason || parsed.modelReason,
|
||
160,
|
||
),
|
||
sceneSummary: truncateReplacementAnalysisText(
|
||
parsed.sceneSummary || parsed.sceneDescription,
|
||
220,
|
||
),
|
||
cameraSummary: truncateReplacementAnalysisText(
|
||
parsed.cameraSummary || parsed.cameraDescription,
|
||
180,
|
||
),
|
||
placementSummary: truncateReplacementAnalysisText(
|
||
parsed.placementSummary || parsed.productPlacement,
|
||
180,
|
||
),
|
||
preserveSummary: truncateReplacementAnalysisText(
|
||
parsed.preserveSummary || parsed.mustKeep || parsed.preserveInstruction,
|
||
220,
|
||
),
|
||
styleSummary: truncateReplacementAnalysisText(
|
||
parsed.styleSummary || parsed.styleInstruction || parsed.styleKeywords,
|
||
220,
|
||
),
|
||
avoidSummary: truncateReplacementAnalysisText(
|
||
parsed.avoidSummary || parsed.doNotInclude || parsed.avoidInstruction,
|
||
220,
|
||
),
|
||
generationPrompt,
|
||
selectedProductImageIndexes: Array.from(
|
||
{ length: Math.min(Math.max(params.productImageCount, 1), 2) },
|
||
(_, index) => index + 1,
|
||
),
|
||
productSelectionReason:
|
||
params.productImageCount > 1
|
||
? "默认优先使用前两张产品图作为主视角与补充细节，其余角度不强制同时融入。"
|
||
: "默认使用当前唯一产品图作为主体参考。",
|
||
analysisModelName: null,
|
||
analysisSource: "heuristic",
|
||
};
|
||
|
||
if (!scenePlan.generationPrompt) {
|
||
scenePlan.generationPrompt = [
|
||
scenePlan.sceneSummary,
|
||
scenePlan.cameraSummary,
|
||
scenePlan.placementSummary,
|
||
scenePlan.styleSummary,
|
||
]
|
||
.filter(Boolean)
|
||
.join("；")
|
||
.slice(0, 1200);
|
||
}
|
||
|
||
return scenePlan;
|
||
} catch (error) {
|
||
console.warn("[Chat API] Failed to parse replacement scene plan JSON:", error);
|
||
return null;
|
||
}
|
||
}
|
||
|
||
function buildHeuristicReplacementScenePlan(params: {
|
||
productImageCount: number;
|
||
hasModelImage: boolean;
|
||
shouldReplace: boolean;
|
||
reason: string;
|
||
}): ReplacementScenePlan {
|
||
return {
|
||
selectedProductImageIndexes: Array.from(
|
||
{ length: Math.min(Math.max(params.productImageCount, 1), 2) },
|
||
(_, index) => index + 1,
|
||
),
|
||
productSelectionReason:
|
||
params.productImageCount > 1
|
||
? "默认优先使用前两张产品图作为主视角与补充细节，其余角度不强制同时融入。"
|
||
: "默认使用当前唯一产品图作为主体参考。",
|
||
useModelImage: false,
|
||
modelImageReason: params.hasModelImage
|
||
? "前置分析未明确当前参考图存在人物展示需求，兜底阶段默认不使用模特图。"
|
||
: "当前未提供模特图。",
|
||
sceneSummary: params.shouldReplace
|
||
? "保留参考图版式骨架、构图和主体位置，但把旧场景视觉内容按工作流目标风格重建。"
|
||
: "当前参考图不满足替换条件。",
|
||
cameraSummary: "保持参考图原有机位、构图和景别，不做整体重画。",
|
||
placementSummary: "产品应落在参考图当前主体所在区域，并围绕该区域同步重建新的场景内容。",
|
||
preserveSummary: "文字排版、标题区、分割线、留白和信息层级必须保留。",
|
||
styleSummary: "除版式骨架外，墙面、地面、背景家具、软装、灯光、摆件和空间氛围都要按工作流风格重建。",
|
||
avoidSummary:
|
||
"不要只换产品不换场景，不要把参考图原有背景物件和空间氛围原样保留下来，不要把所有输入产品角度图同时强行出现在一张图中。",
|
||
generationPrompt: params.shouldReplace
|
||
? `以工作流目标风格重建当前这一屏的场景，只保留版式、机位、文字区与留白结构。${params.reason}`
|
||
: params.reason,
|
||
analysisModelName: null,
|
||
analysisSource: "heuristic",
|
||
};
|
||
}
|
||
|
||
function normalizeAgentModeReferences(
|
||
payload?: PostRequestBody["agentModePayload"],
|
||
): AgentReplacementModePayload["references"] {
|
||
if (!payload?.references?.length) return [];
|
||
return payload.references
|
||
.map((item) => ({
|
||
id: String(item.id || "").trim(),
|
||
label: String(item.label || "").trim() || "参考图",
|
||
imageUrl: String(item.imageUrl || "").trim(),
|
||
candidateId: String(item.candidateId || "").trim() || null,
|
||
sortOrder: Number.isFinite(Number(item.sortOrder))
|
||
? Math.max(0, Math.min(999, Math.round(Number(item.sortOrder))))
|
||
: 0,
|
||
assignedProductIndex: Number.isFinite(Number(item.assignedProductIndex))
|
||
? Math.max(0, Math.min(9, Math.round(Number(item.assignedProductIndex))))
|
||
: null,
|
||
}))
|
||
.filter((item) => item.id && /^https?:\/\//i.test(item.imageUrl))
|
||
.sort((left, right) => left.sortOrder - right.sortOrder);
|
||
}
|
||
|
||
function normalizeBooleanLike(value: unknown): boolean | null {
|
||
if (typeof value === "boolean") {
|
||
return value;
|
||
}
|
||
|
||
const normalized = String(value || "")
|
||
.trim()
|
||
.toLowerCase();
|
||
if (!normalized) {
|
||
return null;
|
||
}
|
||
|
||
if (
|
||
normalized === "true" ||
|
||
normalized === "yes" ||
|
||
normalized === "y" ||
|
||
normalized === "1" ||
|
||
normalized === "是"
|
||
) {
|
||
return true;
|
||
}
|
||
|
||
if (
|
||
normalized === "false" ||
|
||
normalized === "no" ||
|
||
normalized === "n" ||
|
||
normalized === "0" ||
|
||
normalized === "否"
|
||
) {
|
||
return false;
|
||
}
|
||
|
||
return null;
|
||
}
|
||
|
||
function extractJsonObjectText(text: string): string | null {
|
||
const normalized = String(text || "").trim();
|
||
if (!normalized) {
|
||
return null;
|
||
}
|
||
|
||
const fencedMatch = normalized.match(/```(?:json)?\s*([\s\S]*?)```/i);
|
||
if (fencedMatch?.[1]) {
|
||
const fencedContent = fencedMatch[1].trim();
|
||
if (fencedContent.startsWith("{") && fencedContent.endsWith("}")) {
|
||
return fencedContent;
|
||
}
|
||
}
|
||
|
||
if (normalized.startsWith("{") && normalized.endsWith("}")) {
|
||
return normalized;
|
||
}
|
||
|
||
const firstBraceIndex = normalized.indexOf("{");
|
||
const lastBraceIndex = normalized.lastIndexOf("}");
|
||
if (firstBraceIndex >= 0 && lastBraceIndex > firstBraceIndex) {
|
||
return normalized.slice(firstBraceIndex, lastBraceIndex + 1);
|
||
}
|
||
|
||
return null;
|
||
}
|
||
|
||
function parseReplacementReferenceDecision(
|
||
text: string,
|
||
): Omit<ReplacementReferenceDecision, "detectionModelName" | "detectionSource"> | null {
|
||
const jsonText = extractJsonObjectText(text);
|
||
if (jsonText) {
|
||
try {
|
||
const parsed = JSON.parse(jsonText) as Record<string, unknown>;
|
||
const shouldReplace =
|
||
normalizeBooleanLike(parsed.shouldReplace) ??
|
||
normalizeBooleanLike(parsed.replace) ??
|
||
normalizeBooleanLike(parsed.isReplaceable);
|
||
const containsFurnitureProduct =
|
||
normalizeBooleanLike(parsed.containsFurnitureProduct) ??
|
||
normalizeBooleanLike(parsed.hasFurnitureProduct) ??
|
||
shouldReplace;
|
||
const reason = String(parsed.reason || parsed.summary || parsed.note || "")
|
||
.trim()
|
||
.slice(0, 80);
|
||
|
||
if (typeof shouldReplace === "boolean") {
|
||
return {
|
||
shouldReplace,
|
||
containsFurnitureProduct: Boolean(containsFurnitureProduct),
|
||
reason:
|
||
reason ||
|
||
(shouldReplace
|
||
? "检测到可替换的家具主体，继续执行替换"
|
||
: "未检测到可替换的家具主体，直接沿用参考图"),
|
||
};
|
||
}
|
||
} catch (error) {
|
||
console.warn("[Chat API] Failed to parse replacement reference decision JSON:", error);
|
||
}
|
||
}
|
||
|
||
const normalized = String(text || "")
|
||
.replace(/\s+/g, " ")
|
||
.trim();
|
||
if (!normalized) {
|
||
return null;
|
||
}
|
||
|
||
const shouldReplace = /shouldreplace["']?\s*[:=]\s*true|可替换|应该替换|存在可替换|检测到.*家具主体|yes/i.test(
|
||
normalized,
|
||
);
|
||
const shouldKeep = /shouldreplace["']?\s*[:=]\s*false|不应替换|不要替换|沿用参考图|未检测到.*家具主体|无可替换|not replace|no/i.test(
|
||
normalized,
|
||
);
|
||
|
||
if (!shouldReplace && !shouldKeep) {
|
||
return null;
|
||
}
|
||
|
||
return {
|
||
shouldReplace,
|
||
containsFurnitureProduct: shouldReplace,
|
||
reason: normalized.slice(0, 80),
|
||
};
|
||
}
|
||
|
||
async function analyzeReplacementReferenceImage(params: {
|
||
imageUrl: string;
|
||
models?: Awaited<ReturnType<typeof getEnabledAiModelsWithProvider>>;
|
||
}): Promise<ReplacementReferenceDecision> {
|
||
const detectionPrompt = [
|
||
"你是家具商品参考图替换判定器。",
|
||
"请判断这张图片里，是否存在清晰、主要、适合被产品替换流程替换的家具商品主体。",
|
||
"家具主体包含：床、床垫、沙发、椅子、餐桌、茶几、柜子、电视柜、斗柜、床头柜、书桌等。",
|
||
"只有当家具商品是主要视觉主体、轮廓清晰、不是单纯环境陪衬时，shouldReplace 才能为 true。",
|
||
"以下情况必须判定 shouldReplace=false：纯文字页、参数说明页、品牌介绍页、材质/五金局部特写、只有模特没有家具、只有空间环境没有明确家具主体、家具很小只是背景、主体过多无法明确替换对象。",
|
||
'只输出 JSON，不要输出额外解释。格式：{"shouldReplace":true,"containsFurnitureProduct":true,"reason":"不超过30字"}',
|
||
].join("\n");
|
||
|
||
const configuredModels = params.models || (await getEnabledAiModelsWithProvider());
|
||
const configuredQwenVisionModel = configuredModels.find(
|
||
({ model }) =>
|
||
model.supportsVision &&
|
||
!model.isSubModel &&
|
||
QWEN_VL_MODEL_MATCHER.test(`${model.displayName} ${model.modelId}`),
|
||
);
|
||
|
||
if (configuredQwenVisionModel) {
|
||
try {
|
||
const languageModel = await getLanguageModelById(configuredQwenVisionModel.model.id);
|
||
if (languageModel) {
|
||
const { text } = await generateText({
|
||
model: languageModel,
|
||
messages: [
|
||
{
|
||
role: "user",
|
||
content: [
|
||
{
|
||
type: "text",
|
||
text: detectionPrompt,
|
||
},
|
||
{
|
||
type: "image",
|
||
image: params.imageUrl,
|
||
},
|
||
],
|
||
},
|
||
],
|
||
temperature: 0,
|
||
maxTokens: 160,
|
||
abortSignal: AbortSignal.timeout(45_000),
|
||
} as any);
|
||
|
||
const parsed = parseReplacementReferenceDecision(text);
|
||
if (parsed) {
|
||
return {
|
||
...parsed,
|
||
detectionModelName:
|
||
configuredQwenVisionModel.model.displayName ||
|
||
configuredQwenVisionModel.model.modelId,
|
||
detectionSource: "configured_qwen3_vl",
|
||
};
|
||
}
|
||
}
|
||
} catch (error) {
|
||
console.warn("[Chat API] Configured Qwen-VL reference analysis failed:", error);
|
||
}
|
||
}
|
||
|
||
try {
|
||
const { callQwen3Vl } = await import("@/lib/ai/qwen3-vl");
|
||
const fallbackResult = await callQwen3Vl({
|
||
imageUrl: params.imageUrl,
|
||
prompt: detectionPrompt,
|
||
temperature: 0,
|
||
maxTokens: 160,
|
||
timeoutMs: 45_000,
|
||
retries: 1,
|
||
});
|
||
const parsed = parseReplacementReferenceDecision(fallbackResult.text);
|
||
if (parsed) {
|
||
return {
|
||
...parsed,
|
||
detectionModelName: fallbackResult.model,
|
||
detectionSource: "fallback_qwen3_vl",
|
||
};
|
||
}
|
||
} catch (error) {
|
||
console.warn("[Chat API] Fallback Qwen3-VL reference analysis failed:", error);
|
||
}
|
||
|
||
return {
|
||
shouldReplace: false,
|
||
containsFurnitureProduct: false,
|
||
reason: "视觉判断失败，为避免误替换，直接沿用参考图",
|
||
detectionModelName: null,
|
||
detectionSource: "safe_passthrough",
|
||
};
|
||
}
|
||
|
||
async function resolveReplacementScenePlanningRuntime(params: {
|
||
models?: Awaited<ReturnType<typeof getEnabledAiModelsWithProvider>>;
|
||
}): Promise<ReplacementScenePlanningRuntime> {
|
||
try {
|
||
const siteSettings = await getSiteSettings().catch(() => null);
|
||
const localDetailAgentResolved = await resolveDetailAgentLocalMoonshotModel({
|
||
siteSettingModelId: siteSettings?.canvasDetailAgentModelId,
|
||
});
|
||
if (localDetailAgentResolved) {
|
||
return {
|
||
kind: "local_moonshot",
|
||
model: localDetailAgentResolved.model,
|
||
modelName:
|
||
localDetailAgentResolved.modelRef.displayName ||
|
||
localDetailAgentResolved.modelRef.modelId,
|
||
};
|
||
}
|
||
} catch (error) {
|
||
console.warn("[Chat API] Failed to resolve local Moonshot scene planning model:", error);
|
||
}
|
||
|
||
const configuredModels = params.models || (await getEnabledAiModelsWithProvider());
|
||
const configuredVisionModel =
|
||
configuredModels.find(
|
||
({ model }) =>
|
||
model.supportsVision &&
|
||
!model.isSubModel &&
|
||
QWEN_VL_MODEL_MATCHER.test(`${model.displayName} ${model.modelId}`),
|
||
) ||
|
||
configuredModels.find(({ model }) => model.supportsVision && !model.isSubModel);
|
||
|
||
if (!configuredVisionModel) {
|
||
return null;
|
||
}
|
||
|
||
try {
|
||
const languageModel = await getLanguageModelById(configuredVisionModel.model.id);
|
||
if (!languageModel) {
|
||
return null;
|
||
}
|
||
return {
|
||
kind: "configured_vision",
|
||
model: languageModel,
|
||
modelName:
|
||
configuredVisionModel.model.displayName ||
|
||
configuredVisionModel.model.modelId,
|
||
};
|
||
} catch (error) {
|
||
console.warn("[Chat API] Failed to resolve configured replacement scene planning model:", {
|
||
modelId: configuredVisionModel.model.id,
|
||
error,
|
||
});
|
||
return null;
|
||
}
|
||
}
|
||
|
||
async function analyzeReplacementExecutionPlan(params: {
|
||
referenceImageUrl: string;
|
||
referenceLabel: string;
|
||
productImageUrls: string[];
|
||
modelImageUrl?: string | null;
|
||
workflowPrompt?: string | null;
|
||
planningRuntime: ReplacementScenePlanningRuntime;
|
||
models?: Awaited<ReturnType<typeof getEnabledAiModelsWithProvider>>;
|
||
}): Promise<ReplacementExecutionPlan> {
|
||
if (params.planningRuntime) {
|
||
try {
|
||
const replacementScenePlanningTimeoutMs =
|
||
params.planningRuntime.kind === "local_moonshot" ? 90_000 : 45_000;
|
||
const analysisImageUrls = [params.referenceImageUrl];
|
||
const { inlineDataUrls } = await inlineVisionImages(analysisImageUrls, {
|
||
maxImages: 1,
|
||
maxSide: 900,
|
||
timeoutMs: 12000,
|
||
retries: 2,
|
||
loggerPrefix: "[Chat API] Replacement scene plan",
|
||
}).catch((error) => {
|
||
console.warn("[Chat API] Failed to inline replacement scene planning images:", error);
|
||
return { inlineDataUrls: [], skipped: [] };
|
||
});
|
||
const effectiveVisionImages =
|
||
inlineDataUrls.length > 0 ? inlineDataUrls : analysisImageUrls;
|
||
const workflowPromptExcerpt = String(params.workflowPrompt || "")
|
||
.trim()
|
||
.replace(/\s+/g, " ")
|
||
.slice(0, 2200);
|
||
const analysisPrompt = [
|
||
"你是电商详情替换流程中的“场景理解规划器”。",
|
||
`当前要分析的参考图：${params.referenceLabel}`,
|
||
"你现在只分析这一张参考图本身，不分析产品图，不分析模特图。",
|
||
"产品图和模特图会在后续生图阶段单独提供给生图模型使用，你这里不要尝试理解它们的具体外观，也不要要求必须使用所有输入图。",
|
||
params.modelImageUrl
|
||
? "你只需要根据当前参考图判断：这一屏是否存在真实的人物展示需求；如果没有人物需求，useModelImage 必须为 false。"
|
||
: "本轮没有模特图，useModelImage 必须为 false。",
|
||
"这张图片是唯一的详情参考图底板，它决定当前这一屏的版式、文字排版、机位、构图、留白和信息层级。",
|
||
"这张参考图不是要求原样保留的场景照片。原图里的沙发、床、柜体、墙面、地面、窗帘、灯具、摆件、桌面道具、背景家具和环境光氛围，都可以在后续生图阶段按前置工作流风格重建。",
|
||
"请完成以下任务：",
|
||
"1. 判断 shouldReplace：当前参考图里是否存在清晰、主要、适合被替换的家具商品主体。",
|
||
"2. 判断 useModelImage，并说明原因。",
|
||
"3. 提炼 sceneSummary、cameraSummary、placementSummary、preserveSummary、styleSummary、avoidSummary。",
|
||
"4. 输出 generationPrompt：这是一段给后续生图模型的本屏生成指令，描述这一屏需要的场景、机位、主体位置、周边环境重建方向以及必须保留的内容。",
|
||
"5. sceneSummary、styleSummary 和 generationPrompt 必须优先描述“按前置工作流重建后的目标场景”，而不是只复述参考图当前已有场景。",
|
||
"6. 如果前置工作流风格与当前参考图原场景冲突，必须保留版式骨架，但放弃原场景物件本身，以前置工作流指定的空间风格、软装、灯光和氛围为准。",
|
||
"7. avoidSummary 里要明确避免“只替换产品、不重建场景”这类结果。",
|
||
workflowPromptExcerpt
|
||
? `当前风格与场景限制（来自前置工作流与模式模板）:\n${workflowPromptExcerpt}`
|
||
: "",
|
||
"只返回严格 JSON，不要任何额外解释。",
|
||
`JSON 格式：
|
||
{"shouldReplace":true,"containsFurnitureProduct":true,"reason":"不超过30字","useModelImage":false,"modelImageReason":"不超过40字","sceneSummary":"不超过80字","cameraSummary":"不超过60字","placementSummary":"不超过60字","preserveSummary":"不超过80字","styleSummary":"不超过80字","avoidSummary":"不超过80字","generationPrompt":"不超过300字"}`,
|
||
]
|
||
.filter(Boolean)
|
||
.join("\n");
|
||
|
||
const { text } = await generateText({
|
||
model: params.planningRuntime.model,
|
||
messages: [
|
||
{
|
||
role: "user",
|
||
content: [
|
||
{
|
||
type: "text",
|
||
text: analysisPrompt,
|
||
},
|
||
...effectiveVisionImages.map((image) => ({
|
||
type: "image" as const,
|
||
image,
|
||
})),
|
||
],
|
||
},
|
||
],
|
||
system:
|
||
"你是高精度商品详情图场景规划器。你只分析当前参考图本身，负责理解这一屏的场景、机位、主体位置和人物需求，不要臆测或依赖其他输入图片。",
|
||
temperature: params.planningRuntime.kind === "local_moonshot" ? 1 : 0.1,
|
||
maxTokens: 1200,
|
||
abortSignal: AbortSignal.timeout(replacementScenePlanningTimeoutMs),
|
||
} as any);
|
||
|
||
const parsedDecision = parseReplacementReferenceDecision(text);
|
||
const parsedScenePlan = parseReplacementScenePlan(text, {
|
||
productImageCount: params.productImageUrls.length,
|
||
hasModelImage: Boolean(params.modelImageUrl),
|
||
});
|
||
|
||
if (parsedDecision && parsedScenePlan) {
|
||
return {
|
||
...parsedDecision,
|
||
detectionModelName: params.planningRuntime.modelName,
|
||
detectionSource:
|
||
params.planningRuntime.kind === "local_moonshot"
|
||
? "local_moonshot"
|
||
: "configured_vision",
|
||
scenePlan: {
|
||
...parsedScenePlan,
|
||
analysisModelName: params.planningRuntime.modelName,
|
||
analysisSource:
|
||
params.planningRuntime.kind === "local_moonshot"
|
||
? "local_moonshot"
|
||
: "configured_vision",
|
||
},
|
||
};
|
||
}
|
||
} catch (error) {
|
||
console.warn("[Chat API] Replacement scene planning analysis failed:", {
|
||
referenceLabel: params.referenceLabel,
|
||
referenceImageUrl: params.referenceImageUrl,
|
||
error,
|
||
});
|
||
}
|
||
}
|
||
|
||
const fallbackDecision = await analyzeReplacementReferenceImage({
|
||
imageUrl: params.referenceImageUrl,
|
||
models: params.models,
|
||
});
|
||
return {
|
||
...fallbackDecision,
|
||
scenePlan: buildHeuristicReplacementScenePlan({
|
||
productImageCount: params.productImageUrls.length,
|
||
hasModelImage: Boolean(params.modelImageUrl),
|
||
shouldReplace: fallbackDecision.shouldReplace,
|
||
reason: fallbackDecision.reason,
|
||
}),
|
||
};
|
||
}
|
||
|
||
function buildAgentReplacementPrompt(params: {
|
||
basePrompt: string;
|
||
modeType: "detail_replacement" | "sku_replacement";
|
||
referenceLabel: string;
|
||
referenceIndex: number;
|
||
totalReferences: number;
|
||
hasModelImage: boolean;
|
||
productImageCount: number;
|
||
originalProductImageCount: number;
|
||
templateReferenceCount: number;
|
||
pairedProductIndex?: number;
|
||
scenePlan?: ReplacementScenePlan | null;
|
||
}) {
|
||
const imageOrderLines: string[] = [];
|
||
let nextImageIndex = 1;
|
||
|
||
if (params.productImageCount > 0) {
|
||
const productRangeLabel =
|
||
params.productImageCount === 1
|
||
? `图片${nextImageIndex}`
|
||
: `图片${nextImageIndex}-${nextImageIndex + params.productImageCount - 1}`;
|
||
imageOrderLines.push(
|
||
`${productRangeLabel}：同一产品的白底图/细节图，必须综合理解所有产品图，不可只看第一张。`
|
||
);
|
||
nextImageIndex += params.productImageCount;
|
||
}
|
||
|
||
if (params.hasModelImage) {
|
||
imageOrderLines.push(
|
||
`图片${nextImageIndex}：模特图，仅在当前参考图本身包含人物展示时才允许参考。`
|
||
);
|
||
nextImageIndex += 1;
|
||
}
|
||
|
||
if (params.templateReferenceCount > 0) {
|
||
const templateRangeLabel =
|
||
params.templateReferenceCount === 1
|
||
? `图片${nextImageIndex}`
|
||
: `图片${nextImageIndex}-${nextImageIndex + params.templateReferenceCount - 1}`;
|
||
imageOrderLines.push(
|
||
`${templateRangeLabel}：模板补充参考图，只能补充风格/材质氛围，不能覆盖最终参考图版式。`
|
||
);
|
||
nextImageIndex += params.templateReferenceCount;
|
||
}
|
||
|
||
imageOrderLines.push(
|
||
`图片${nextImageIndex}：当前参考图底板（唯一版式基准图），必须保留它的文字排版、信息区、分割线、留白、机位和整体结构。`
|
||
);
|
||
|
||
const scenePlanLines = params.scenePlan
|
||
? [
|
||
"【本屏场景预分析】",
|
||
params.scenePlan.sceneSummary
|
||
? `场景理解：${params.scenePlan.sceneSummary}`
|
||
: "",
|
||
params.scenePlan.cameraSummary
|
||
? `机位构图：${params.scenePlan.cameraSummary}`
|
||
: "",
|
||
params.scenePlan.placementSummary
|
||
? `主体位置：${params.scenePlan.placementSummary}`
|
||
: "",
|
||
params.scenePlan.preserveSummary
|
||
? `必须保留：${params.scenePlan.preserveSummary}`
|
||
: "",
|
||
params.scenePlan.styleSummary
|
||
? `风格约束：${params.scenePlan.styleSummary}`
|
||
: "",
|
||
params.scenePlan.avoidSummary
|
||
? `禁止事项：${params.scenePlan.avoidSummary}`
|
||
: "",
|
||
params.originalProductImageCount > params.productImageCount
|
||
? `原始共有 ${params.originalProductImageCount} 张产品角度图，本屏已筛选出最相关的 ${params.productImageCount} 张输入图；其余未入选产品图只是补充理解，不需要也不允许强行全部融入同一张结果图。`
|
||
: "",
|
||
params.scenePlan.productSelectionReason
|
||
? `产品图筛选理由：${params.scenePlan.productSelectionReason}`
|
||
: "",
|
||
params.scenePlan.useModelImage
|
||
? params.scenePlan.modelImageReason
|
||
? `本屏允许使用模特图：${params.scenePlan.modelImageReason}`
|
||
: "本屏允许使用模特图，但仅在参考图当前这一屏确实存在人物展示需求时使用。"
|
||
: params.hasModelImage
|
||
? params.scenePlan.modelImageReason
|
||
? `本屏禁止使用模特图：${params.scenePlan.modelImageReason}`
|
||
: "本屏不需要模特图，不允许把模特图强行融入结果。"
|
||
: "",
|
||
params.scenePlan.generationPrompt
|
||
? `本屏生成方向：${params.scenePlan.generationPrompt}`
|
||
: "",
|
||
].filter(Boolean)
|
||
: [];
|
||
|
||
const instructionLines =
|
||
params.modeType === "detail_replacement"
|
||
? [
|
||
"【执行模式】详情替换",
|
||
`当前要处理第 ${params.referenceIndex + 1} / ${params.totalReferences} 张详情参考图：${params.referenceLabel}`,
|
||
"【图片顺序规则】",
|
||
...imageOrderLines,
|
||
...scenePlanLines,
|
||
"你现在做的不是重新生成一张新图，而是编辑最后一张参考图本身。",
|
||
"要把最后一张参考图理解为“页面版式底板 + 构图参考”，不是“旧场景照片必须原样保留”。",
|
||
"前面“工作流设定”里的场景组合、空间风格、硬装方案、软装方案、灯光氛围、人群设定，必须作为当前这一屏的强制场景改造要求生效。",
|
||
"执行时要基于这些场景组合要求，重建产品主体周边乃至整屏视觉中的场景元素、材质氛围、软装陈列、灯光与色调，使当前这一屏呈现出与第一步已选风格一致的场景效果。",
|
||
"允许改造的不只是产品主体周边的一小块区域；除文字排版、标题区、信息区、分割线、留白和版式结构外，原参考图里的旧场景视觉内容都可以按目标风格整体重建。",
|
||
"如果前面商品图、模特图与最后一张参考图底板存在冲突，始终以最后一张参考图底板的版式、机位、构图和信息层级为最高优先级。",
|
||
"最后一张参考图是唯一底板，必须保留这张图的文字排版、品牌字样、标题区、卖点区、分割线、留白、机位、构图和信息层级。",
|
||
"前面的商品图和模特图用于替换参考图中的产品主体，并按工作流设定去改造与产品直接相关的场景内容，但不能改变参考图的版式设计。",
|
||
"可被替换重建的内容包括但不限于：背景墙面、地面材质、沙发、床、背景柜、窗帘、灯具、地毯、挂画、摆件、桌面道具、空间色调、环境光氛围。",
|
||
params.productImageCount > 1
|
||
? `前面的 ${params.productImageCount} 张商品输入图都属于同一产品的多角度/多细节白底图，请综合这些图片理解产品结构、材质、颜色、比例、边角、开孔、五金与细节，不可只参考第一张。`
|
||
: "前面的商品输入图就是当前要替换进去的产品白底图，请准确保持该产品的结构、材质、颜色、比例和细节。",
|
||
"请先识别最后一张参考图中需要被替换的产品主体所在区域，并围绕这个区域在同一版式骨架下重建新的场景画面；不要只做局部抠图式换产品。",
|
||
"请严格保留最后一张参考图的版式结构、镜头关系、构图、文字排版、留白位置和信息层级；但画面中的旧场景家具、背景物件、材质、灯光氛围、软装小物、空间质感需要按第一步工作流设定进行明显重建。",
|
||
"如果第一步工作流风格与参考图原场景冲突，必须保留版式骨架，但放弃原场景物件本身，以第一步工作流风格为准。",
|
||
"前面的输入商品图用于提供商品主体、材质、颜色和细节特征；如有模特图，请只在参考图本身包含人物展示时再参考模特图。",
|
||
"绝对不要忽略最后一张参考图去生成一张无关的新场景图。",
|
||
"如果生成结果仍然保留了参考图原有的大部分场景物件，只是把产品换掉，则视为执行失败；你必须让场景风格发生清晰可见的变化。",
|
||
"绝对不要移动、删除、改写、重排参考图里的任何文字与排版元素。",
|
||
"输出结果必须保持最后一张参考图底板的原始纵横比与画布尺寸，不允许擅自改成长图、横图或裁掉任何版面区域。",
|
||
"如果参考图中没有可替换的产品主体，应直接沿用参考图，不要硬生成。",
|
||
"不要改动参考图的整体布局，不要新增无关主体，不要删除关键版式元素。",
|
||
]
|
||
: [
|
||
"【执行模式】SKU替换",
|
||
`当前要处理第 ${params.referenceIndex + 1} / ${params.totalReferences} 张 SKU 参考图：${params.referenceLabel}`,
|
||
"【图片顺序规则】",
|
||
...imageOrderLines,
|
||
...scenePlanLines,
|
||
params.pairedProductIndex !== undefined
|
||
? `请优先使用第 ${params.pairedProductIndex + 1} 张输入商品图作为本张参考图的一对一替换主体。`
|
||
: "请使用输入商品图作为本张参考图的一对一替换主体。",
|
||
params.productImageCount > 1
|
||
? "其余商品输入图同样是这个产品的补充角度/细节参考，请一并综合理解产品信息，不可忽略。"
|
||
: "输入商品图就是当前产品主体，请保持产品信息准确一致。",
|
||
"如果前面商品图与最后一张参考图存在冲突，始终以最后一张参考图的机位、场景、光影和构图为准。",
|
||
"请严格保留最后一张参考图的场景、机位、透视、光影和构图关系，只替换商品主体本身。",
|
||
"输出结果必须保持最后一张参考图的原始纵横比与画布尺寸。",
|
||
"输出要自然融合，不要改变参考图原有风格和景别。",
|
||
];
|
||
|
||
if (params.hasModelImage) {
|
||
instructionLines.push("如有模特图，请在需要人物展示时保持模特体态、穿搭和出镜关系一致。");
|
||
}
|
||
|
||
return [params.basePrompt, instructionLines.join("\n")]
|
||
.filter(Boolean)
|
||
.join("\n\n")
|
||
.trim();
|
||
}
|
||
|
||
const MODEL_UUID_REGEX =
|
||
/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
|
||
|
||
function getModelCreditCost(
|
||
modelConfig: any,
|
||
target: "image" | "video",
|
||
options?: { duration?: number | null }
|
||
): number {
|
||
const config = (modelConfig as any)?.imageGenConfig || {};
|
||
const sourceValue =
|
||
target === "video"
|
||
? (() => {
|
||
const resolvedVideoCost = getVideoCreditCostFromConfig(config, options?.duration);
|
||
if (resolvedVideoCost > 0) return resolvedVideoCost;
|
||
return config.creditCostPerVideo ?? config.creditCostPerImage ?? 0;
|
||
})()
|
||
: config.creditCostPerImage ?? 0;
|
||
const normalized = Number(sourceValue);
|
||
return Number.isFinite(normalized) ? Math.max(0, Math.round(normalized)) : 0;
|
||
}
|
||
|
||
async function resolveBillingModelByReference(modelRef: string): Promise<any | undefined> {
|
||
if (!modelRef) return undefined;
|
||
|
||
if (MODEL_UUID_REGEX.test(modelRef)) {
|
||
return await getAiModelById({ id: modelRef }).catch(() => undefined);
|
||
}
|
||
|
||
const allModels = await getEnabledAiModelsWithProvider(true, true).catch(() => []);
|
||
const matched = allModels.find(
|
||
({ model }: any) => model.id === modelRef || model.modelId === modelRef
|
||
);
|
||
return matched?.model;
|
||
}
|
||
|
||
async function consumeGeneratedVideoPoints(params: {
|
||
userId: string;
|
||
chatId: string;
|
||
messageId: string;
|
||
modelRefs: Array<string | null | undefined>;
|
||
duration?: number | null;
|
||
}) {
|
||
try {
|
||
const pointsConfig = await getPointsSystemConfig();
|
||
if (!pointsConfig.enabled) return;
|
||
|
||
const userData = await getUserById({ id: params.userId }).catch(() => undefined);
|
||
if (userData?.isHighVip) {
|
||
console.log(`[Points] Skip video points for VIP user=${params.userId}`);
|
||
return;
|
||
}
|
||
|
||
let billingModel: any | undefined;
|
||
for (const ref of params.modelRefs) {
|
||
if (!ref) continue;
|
||
const resolved = await resolveBillingModelByReference(ref);
|
||
if (!resolved) continue;
|
||
billingModel = resolved;
|
||
if (getModelCreditCost(resolved, "video", { duration: params.duration }) > 0) break;
|
||
}
|
||
|
||
if (!billingModel) return;
|
||
|
||
const unitCost = getModelCreditCost(billingModel, "video", {
|
||
duration: params.duration,
|
||
});
|
||
if (unitCost <= 0) return;
|
||
|
||
const consumeResult = await consumeUserPoints({
|
||
userId: params.userId,
|
||
amount: unitCost,
|
||
reason: "video_generation",
|
||
modelId: billingModel.id,
|
||
chatId: params.chatId,
|
||
messageId: params.messageId,
|
||
note: `模型(${billingModel.displayName || billingModel.modelId || billingModel.id}) 生视频 1 条${params.duration ? `（${params.duration} 秒）` : ""}`,
|
||
});
|
||
|
||
if (!consumeResult.ok) {
|
||
console.warn(
|
||
`[Points] Failed to consume video points. user=${params.userId} model=${billingModel.id} cost=${unitCost}`
|
||
);
|
||
} else if (consumeResult.skipped) {
|
||
console.log(
|
||
`[Points] Skipped ${unitCost} video points for VIP user=${params.userId} model=${billingModel.id}`
|
||
);
|
||
} else {
|
||
console.log(
|
||
`[Points] Consumed ${unitCost} points for video. balance=${consumeResult.balance}`
|
||
);
|
||
}
|
||
} catch (error) {
|
||
console.error("[Points] Failed to consume points after video generation:", error);
|
||
}
|
||
}
|
||
|
||
export const maxDuration = 600; // 10 分钟，用于支持图片编辑等耗时任务
|
||
|
||
let globalStreamContext: ResumableStreamContext | null = null;
|
||
const inFlightChatRequests = new Map<string, number>();
|
||
const IN_FLIGHT_REQUEST_TTL_MS = 5 * 60 * 1000;
|
||
const DUPLICATE_REQUEST_WAIT_MS = 2 * 60 * 1000;
|
||
const CHAT_PRE_RESPONSE_SOFT_TIMEOUT_MS = 55_000;
|
||
const SMART_AUTO_GRAPH_TIMEOUT_MS = 28_000;
|
||
const SMART_AUTO_FALLBACK_TIMEOUT_MS = 20_000;
|
||
const SMART_AUTO_IMAGE_SUMMARY_TIMEOUT_MS = 18_000;
|
||
const DEFAULT_STREAM_KEEPALIVE_INTERVAL_MS = 12_000;
|
||
|
||
const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms));
|
||
|
||
class OperationTimeoutError extends Error {
|
||
constructor(operation: string, timeoutMs: number) {
|
||
super(`${operation} timed out after ${timeoutMs}ms`);
|
||
this.name = "OperationTimeoutError";
|
||
}
|
||
}
|
||
|
||
async function withOperationTimeout<T>(
|
||
operation: Promise<T>,
|
||
operationName: string,
|
||
timeoutMs: number,
|
||
): Promise<T> {
|
||
let timer: ReturnType<typeof setTimeout> | null = null;
|
||
try {
|
||
return await Promise.race([
|
||
operation,
|
||
new Promise<never>((_, reject) => {
|
||
timer = setTimeout(
|
||
() => reject(new OperationTimeoutError(operationName, timeoutMs)),
|
||
timeoutMs,
|
||
);
|
||
}),
|
||
]);
|
||
} finally {
|
||
if (timer) {
|
||
clearTimeout(timer);
|
||
}
|
||
}
|
||
}
|
||
|
||
function getRemainingBudgetMs(startedAt: number, budgetMs: number) {
|
||
return Math.max(0, budgetMs - (Date.now() - startedAt));
|
||
}
|
||
|
||
function getBudgetedTimeout(params: {
|
||
startedAt: number;
|
||
preferredMs: number;
|
||
minimumMs?: number;
|
||
}) {
|
||
const minimumMs = params.minimumMs ?? 6_000;
|
||
const remainingMs = getRemainingBudgetMs(
|
||
params.startedAt,
|
||
CHAT_PRE_RESPONSE_SOFT_TIMEOUT_MS
|
||
);
|
||
if (remainingMs < minimumMs) {
|
||
return null;
|
||
}
|
||
return Math.min(params.preferredMs, remainingMs);
|
||
}
|
||
|
||
const getTokenlensCatalog = cache(
|
||
async (): Promise<ModelCatalog | undefined> => {
|
||
try {
|
||
return await fetchModels();
|
||
} catch (err) {
|
||
console.warn(
|
||
"TokenLens: catalog fetch failed, using default catalog",
|
||
err
|
||
);
|
||
return; // tokenlens helpers will fall back to defaultCatalog
|
||
}
|
||
},
|
||
["tokenlens-catalog"],
|
||
{ revalidate: 24 * 60 * 60 } // 24 hours
|
||
);
|
||
|
||
export function getStreamContext() {
|
||
if (!globalStreamContext) {
|
||
try {
|
||
globalStreamContext = createResumableStreamContext({
|
||
waitUntil: after,
|
||
});
|
||
} catch (error: any) {
|
||
if (error.message.includes("REDIS_URL")) {
|
||
console.log(
|
||
" > Resumable streams are disabled due to missing REDIS_URL"
|
||
);
|
||
} else {
|
||
console.error(error);
|
||
}
|
||
}
|
||
}
|
||
|
||
return globalStreamContext;
|
||
}
|
||
|
||
export async function POST(request: Request) {
|
||
let requestBody: PostRequestBody;
|
||
let requestDedupKey: string | null = null;
|
||
let acquiredInFlightLock = false;
|
||
|
||
try {
|
||
const json = await request.json();
|
||
console.log("[Chat API] Received request body:", JSON.stringify(json, null, 2));
|
||
requestBody = postRequestBodySchema.parse(json);
|
||
} catch (error) {
|
||
console.error("[Chat API] Schema validation failed:", error);
|
||
return new ChatSDKError("bad_request:api").toResponse();
|
||
}
|
||
|
||
try {
|
||
const {
|
||
id,
|
||
message,
|
||
agentPresetId,
|
||
selectedChatModel,
|
||
selectedVisibilityType,
|
||
composerMode,
|
||
hiddenPrompt,
|
||
workflowPromptOptimizationEnabled,
|
||
autoContext,
|
||
contextPayload,
|
||
agentModePayload,
|
||
deepThinkingEnabled,
|
||
debugProtocol,
|
||
}: {
|
||
id: string;
|
||
message: ChatMessage;
|
||
agentPresetId?: string;
|
||
selectedChatModel: ChatModel["id"];
|
||
selectedVisibilityType: VisibilityType;
|
||
composerMode?: "agent" | "image" | "video" | "detail";
|
||
hiddenPrompt?: string;
|
||
workflowPromptOptimizationEnabled?: boolean;
|
||
autoContext?: string;
|
||
contextPayload?: PostRequestBody["contextPayload"];
|
||
agentModePayload?: PostRequestBody["agentModePayload"];
|
||
deepThinkingEnabled?: boolean;
|
||
debugProtocol?: boolean;
|
||
} = requestBody;
|
||
const requestStartedAt = Date.now();
|
||
const requestOrigin = (() => {
|
||
try {
|
||
return new URL(request.url).origin;
|
||
} catch {
|
||
return "";
|
||
}
|
||
})();
|
||
const requestTextContent = message.parts
|
||
.filter((part): part is { type: "text"; text: string } => part.type === "text")
|
||
.map((part) => part.text)
|
||
.join(" ")
|
||
.trim();
|
||
const autoContextText = typeof autoContext === "string" ? autoContext.trim() : "";
|
||
|
||
const now = Date.now();
|
||
for (const [key, startedAt] of inFlightChatRequests.entries()) {
|
||
if (now - startedAt > IN_FLIGHT_REQUEST_TTL_MS) {
|
||
inFlightChatRequests.delete(key);
|
||
}
|
||
}
|
||
|
||
requestDedupKey = `${id}:${message.id}`;
|
||
if (inFlightChatRequests.has(requestDedupKey)) {
|
||
console.warn(
|
||
`[Chat API] Duplicate in-flight request ignored (chatId=${id}, messageId=${message.id})`
|
||
);
|
||
const duplicateStream = createUIMessageStream({
|
||
execute: async ({ writer }) => {
|
||
await replayAssistantMessagesForDuplicateRequest({
|
||
chatId: id,
|
||
userMessageId: message.id,
|
||
writer,
|
||
source: "duplicate-in-flight",
|
||
});
|
||
},
|
||
});
|
||
return createSseResponse(duplicateStream);
|
||
}
|
||
inFlightChatRequests.set(requestDedupKey, now);
|
||
acquiredInFlightLock = true;
|
||
|
||
let finalModelId = selectedChatModel;
|
||
|
||
// 🆕 存储智能模式的相关信息
|
||
let smartAutoFriendlyMessage: string | undefined; // AI的自然对话回复
|
||
let smartAutoOptimizedPrompt: string | undefined; // 优化后的提示词（不修改用户消息，只用于API调用）
|
||
let smartAutoPromptFromVideoPlan = false;
|
||
let smartAutoRecommendedImageParams: AgentRecommendedImageParams | undefined;
|
||
let smartAutoVideoPlan: AgentVideoPlan | undefined;
|
||
let smartAutoRecommendedVideoParams: AgentRecommendedVideoParams | undefined;
|
||
let smartAutoPlan: AgentExecutionPlan | undefined;
|
||
let smartAutoExecution: AgentPreparedExecution | undefined;
|
||
let smartAutoDetectedIntent: "image_edit" | "vision" | "image_gen" | "video_gen" | "chat" | undefined;
|
||
let smartAutoHistoryImages: string[] = []; // 从历史对话中提取的图片URL（支持多张）
|
||
let smartAutoNeedsClarification = false; // 是否需要先追问用户
|
||
let smartAutoClarificationQuestion: string | undefined;
|
||
let requestResolvedMediaInput: ResolvedChatMediaInput | null = null;
|
||
let requestTargetResolution: ResolvedTargetAssets | null = null;
|
||
let requestWorkingState:
|
||
| Awaited<ReturnType<typeof getConversationWorkingStateByChatId>>
|
||
| null = null;
|
||
let storyboardPreferenceMemory = null;
|
||
let imageWorkflowMemory = null;
|
||
let reusablePromptMemory = null;
|
||
let confirmedPromptForConversationState: string | undefined;
|
||
let smartAutoHistoryMessagesFromDb: DBMessage[] = [];
|
||
let smartAutoMediaHistoryMessagesFromDb: DBMessage[] = [];
|
||
let preferredAgentModelId: string | undefined;
|
||
let configuredAgentConversationModelId: string | undefined;
|
||
let agentModelSlots: AgentModelSlots = {};
|
||
let pendingGridConfirmation:
|
||
| PendingGridConfirmation
|
||
| null = null;
|
||
let shouldAskGridConfirmation = false;
|
||
const deepThinkingOn = deepThinkingEnabled !== false;
|
||
const shouldRespectDirectComposerMode =
|
||
composerMode === "image" || composerMode === "video" || composerMode === "detail";
|
||
const shouldIsolateDirectComposerContext = shouldRespectDirectComposerMode;
|
||
let shouldRunAgentConversation = finalModelId === "auto";
|
||
let shouldKeepSelectedModel = false;
|
||
if (shouldRespectDirectComposerMode && shouldRunAgentConversation) {
|
||
shouldRunAgentConversation = false;
|
||
console.log(
|
||
`[Smart Auto Mode] ⛳ Direct composer mode (${composerMode}) disables initial auto-agent routing`,
|
||
);
|
||
}
|
||
const pendingProtocolDebugEvents: Array<{ phase: string; payload: Record<string, unknown> }> = [];
|
||
const requestedModelId = selectedChatModel;
|
||
let requestedModelProviderType: string | undefined;
|
||
let requestedModelIsFixedVisual = false;
|
||
let requestedModelIsConversationMultimodal = false;
|
||
let requestedModelConfig: Awaited<ReturnType<typeof getAiModelById>> | null = null;
|
||
|
||
if (!shouldRunAgentConversation && requestedModelId !== "auto") {
|
||
const selectedModelIsUUID =
|
||
/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test(requestedModelId);
|
||
let selectedModelConfig = selectedModelIsUUID
|
||
? await getAiModelById({ id: requestedModelId })
|
||
: null;
|
||
let selectedModelProviderType: string | undefined;
|
||
|
||
if (!selectedModelConfig) {
|
||
const enabledModels = await getEnabledAiModelsWithProvider(false);
|
||
const matchedModel = enabledModels.find(
|
||
({ model }) => model.id === requestedModelId || model.modelId === requestedModelId
|
||
);
|
||
if (matchedModel) {
|
||
selectedModelConfig = matchedModel.model;
|
||
selectedModelProviderType = matchedModel.provider.type as string;
|
||
}
|
||
}
|
||
|
||
if (selectedModelConfig && !selectedModelProviderType) {
|
||
const selectedProvider = await getAiProviderById({
|
||
id: selectedModelConfig.providerId,
|
||
});
|
||
selectedModelProviderType = selectedProvider?.type as string | undefined;
|
||
}
|
||
|
||
requestedModelConfig = selectedModelConfig;
|
||
requestedModelProviderType = selectedModelProviderType;
|
||
|
||
if (selectedModelConfig) {
|
||
const isFixedVisualModel =
|
||
!selectedModelConfig.isSubModel &&
|
||
(
|
||
isImageGenerationModelType(selectedModelProviderType, selectedModelConfig) ||
|
||
isVideoProviderType(selectedModelProviderType, selectedModelConfig)
|
||
);
|
||
const isConversationMultimodalModel =
|
||
!selectedModelConfig.isSubModel &&
|
||
isAiModelMultimodal(selectedModelConfig) &&
|
||
!isImageGenerationModelType(selectedModelProviderType, selectedModelConfig) &&
|
||
!isVideoProviderType(selectedModelProviderType, selectedModelConfig);
|
||
requestedModelIsFixedVisual = isFixedVisualModel;
|
||
requestedModelIsConversationMultimodal = isConversationMultimodalModel;
|
||
|
||
if (deepThinkingOn) {
|
||
if (isAiModelAgentAuto(selectedModelConfig)) {
|
||
if (shouldRespectDirectComposerMode) {
|
||
console.log(
|
||
`[Smart Auto Mode] ⛳ Respect direct composer mode (${composerMode}), skip forcing agent-only routing for: ${selectedModelConfig.displayName}`,
|
||
);
|
||
} else {
|
||
shouldRunAgentConversation = true;
|
||
shouldKeepSelectedModel = false;
|
||
preferredAgentModelId = selectedModelConfig.id;
|
||
configuredAgentConversationModelId = selectedModelConfig.id;
|
||
console.log(
|
||
`[Smart Auto Mode] 🧠 Agent-only model detected, force auto routing: ${selectedModelConfig.displayName}`
|
||
);
|
||
}
|
||
} else if (isFixedVisualModel || isConversationMultimodalModel) {
|
||
if (shouldRespectDirectComposerMode) {
|
||
console.log(
|
||
`[Smart Auto Mode] ⛳ Respect direct composer mode (${composerMode}), keep direct execution model: ${selectedModelConfig.displayName}`,
|
||
);
|
||
} else {
|
||
shouldRunAgentConversation = true;
|
||
shouldKeepSelectedModel = true;
|
||
preferredAgentModelId = selectedModelConfig.id;
|
||
console.log(
|
||
`[Smart Auto Mode] 🎯 Agent conversation enabled for selected model: ${selectedModelConfig.displayName} (${selectedModelProviderType || "unknown"}), preferredAgentModelId=${preferredAgentModelId}`
|
||
);
|
||
}
|
||
}
|
||
} else if (
|
||
isFixedVisualModel ||
|
||
isConversationMultimodalModel ||
|
||
isAiModelAgentAuto(selectedModelConfig)
|
||
) {
|
||
console.log(
|
||
`[Smart Auto Mode] ⏭️ Deep thinking disabled, skip agent analysis for: ${selectedModelConfig.displayName}`
|
||
);
|
||
}
|
||
}
|
||
}
|
||
|
||
if (shouldRunAgentConversation && !preferredAgentModelId) {
|
||
const configuredAgentModel = await getAgentAiModelWithProvider().catch((agentModelError) => {
|
||
console.warn("[Chat API] Failed to load configured agent model:", agentModelError);
|
||
return null;
|
||
});
|
||
|
||
if (configuredAgentModel?.model?.id) {
|
||
preferredAgentModelId = configuredAgentModel.model.id;
|
||
configuredAgentConversationModelId = configuredAgentModel.model.id;
|
||
agentModelSlots = upsertAgentModelSlot(agentModelSlots, "planner", {
|
||
modelId: configuredAgentModel.model.id,
|
||
displayName: configuredAgentModel.model.displayName || configuredAgentModel.model.modelId,
|
||
providerId: configuredAgentModel.model.providerId,
|
||
providerType: configuredAgentModel.provider.type,
|
||
providerKey: configuredAgentModel.model.providerId,
|
||
source: "configured_agent_default",
|
||
reason: "agent_default_model",
|
||
});
|
||
agentModelSlots = upsertAgentModelSlot(agentModelSlots, "analyzer", {
|
||
modelId: configuredAgentModel.model.id,
|
||
displayName: configuredAgentModel.model.displayName || configuredAgentModel.model.modelId,
|
||
providerId: configuredAgentModel.model.providerId,
|
||
providerType: configuredAgentModel.provider.type,
|
||
providerKey: configuredAgentModel.model.providerId,
|
||
source: "configured_agent_default",
|
||
reason: "agent_default_model",
|
||
});
|
||
console.log(
|
||
`[Smart Auto Mode] 🧠 Loaded configured agent conversation model: ${configuredAgentModel.model.displayName}`,
|
||
);
|
||
}
|
||
}
|
||
|
||
const hasCurrentMessageMediaInput = message.parts.some((part: any) =>
|
||
part?.type === "file" || part?.type === "image" || part?.type === "image_url"
|
||
);
|
||
if (
|
||
shouldRunAgentConversation &&
|
||
isPureGreetingTurnRequest(requestTextContent) &&
|
||
!hasCurrentMessageMediaInput
|
||
) {
|
||
const session = await auth();
|
||
|
||
if (!session?.user) {
|
||
return new ChatSDKError("unauthorized:chat").toResponse();
|
||
}
|
||
|
||
const greetingMessage = "你好，我在。你可以直接告诉我想做什么：生图、修图、生成视频、分析图片都可以。";
|
||
const greetingStream = createUIMessageStream({
|
||
execute: async ({ writer }) => {
|
||
const messageId = generateUUID();
|
||
writeAssistantTextChunk(writer, greetingMessage, "greeting-fast-path", messageId);
|
||
|
||
await saveMessages({
|
||
messages: [
|
||
{
|
||
id: messageId,
|
||
chatId: id,
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: greetingMessage }],
|
||
attachments: [],
|
||
createdAt: new Date(),
|
||
videoStorageStatus: null,
|
||
temporaryVideoUrl: null,
|
||
permanentVideoUrl: null,
|
||
archivedAt: null,
|
||
videoMetadata: null,
|
||
},
|
||
],
|
||
});
|
||
},
|
||
});
|
||
|
||
return createSseResponse(greetingStream);
|
||
}
|
||
|
||
const allowAgentVideoGeneration =
|
||
Boolean(requestBody.videoParams) ||
|
||
isVideoProviderType(requestedModelProviderType, requestedModelConfig || undefined);
|
||
let smartAutoWorkflowDecision:
|
||
| {
|
||
action?: string;
|
||
workflowId?: string;
|
||
workflowType?: string;
|
||
allowWorkflowExecution?: boolean;
|
||
}
|
||
| undefined;
|
||
const activeStoryboardTaskForChat = shouldIsolateDirectComposerContext
|
||
? null
|
||
: await getLatestActiveVideoStoryboardTaskByChatId({
|
||
chatId: id,
|
||
}).catch((storyboardLookupError) => {
|
||
console.warn(
|
||
"[Chat API] Failed to preload active storyboard task before smart auto mode:",
|
||
storyboardLookupError
|
||
);
|
||
return null;
|
||
});
|
||
|
||
requestWorkingState = shouldIsolateDirectComposerContext
|
||
? null
|
||
: await getConversationWorkingStateByChatId({
|
||
chatId: id,
|
||
}).catch((workingStatePreloadError) => {
|
||
console.warn(
|
||
"[Chat API] Failed to preload conversation working state before smart auto mode:",
|
||
workingStatePreloadError
|
||
);
|
||
return null;
|
||
});
|
||
const hasImageInputForConversationAction = message.parts.some((part: any) =>
|
||
part?.type === "file" || part?.type === "image" || part?.type === "image_url"
|
||
);
|
||
const conversationStateBootstrap = bootstrapConversationStateProtocol({
|
||
text: requestTextContent,
|
||
workingState: requestWorkingState,
|
||
activeStoryboardTask: activeStoryboardTaskForChat,
|
||
hasImageInputForPromptCarryForward: hasImageInputForConversationAction,
|
||
hasCanvasSelection: Boolean(contextPayload?.selectedNodeIds?.length),
|
||
});
|
||
console.log("[ConversationStateProtocol] Bootstrap:", {
|
||
action: conversationStateBootstrap.actionResolution.kind,
|
||
taskType: conversationStateBootstrap.taskBoundaryDecision.taskType,
|
||
taskMode: conversationStateBootstrap.taskBoundaryDecision.taskMode,
|
||
hasPromptMemory: Boolean(conversationStateBootstrap.promptMemory?.optimizedPrompt),
|
||
hasImageWorkflowMemory: Boolean(conversationStateBootstrap.imageWorkflowMemory),
|
||
hasStoryboardPreferenceMemory: Boolean(conversationStateBootstrap.storyboardPreferenceMemory),
|
||
hasPendingGridConfirmation: Boolean(conversationStateBootstrap.pendingGridConfirmation),
|
||
hasActiveStoryboardTask: Boolean(conversationStateBootstrap.activeStoryboardTask?.task?.id),
|
||
});
|
||
pendingProtocolDebugEvents.push({
|
||
phase: "bootstrap",
|
||
payload: {
|
||
action: conversationStateBootstrap.actionResolution.kind,
|
||
taskBoundary: conversationStateBootstrap.taskBoundaryDecision,
|
||
pendingGridConfirmation: conversationStateBootstrap.pendingGridConfirmation,
|
||
hasPromptMemory: Boolean(conversationStateBootstrap.promptMemory?.optimizedPrompt),
|
||
hasImageWorkflowMemory: Boolean(conversationStateBootstrap.imageWorkflowMemory),
|
||
hasStoryboardPreferenceMemory: Boolean(conversationStateBootstrap.storyboardPreferenceMemory),
|
||
hasActiveStoryboardTask: Boolean(conversationStateBootstrap.activeStoryboardTask?.task?.id),
|
||
},
|
||
});
|
||
storyboardPreferenceMemory = conversationStateBootstrap.storyboardPreferenceMemory;
|
||
imageWorkflowMemory = conversationStateBootstrap.imageWorkflowMemory;
|
||
reusablePromptMemory = conversationStateBootstrap.promptMemory;
|
||
pendingGridConfirmation = conversationStateBootstrap.pendingGridConfirmation;
|
||
confirmedPromptForConversationState = conversationStateBootstrap.confirmedPrompt;
|
||
const resolvedConversationAction = conversationStateBootstrap.actionResolution;
|
||
const taskBoundaryDecision = conversationStateBootstrap.taskBoundaryDecision;
|
||
const taskBoundaryRequiresAgentConversation =
|
||
taskBoundaryDecision.taskType === "prompt_only" ||
|
||
taskBoundaryDecision.taskType === "analysis_only" ||
|
||
taskBoundaryDecision.taskType === "execution_image" ||
|
||
taskBoundaryDecision.taskType === "execution_video" ||
|
||
taskBoundaryDecision.taskType === "workflow_followup";
|
||
const taskBoundaryOverridesLegacyWorkflow =
|
||
taskBoundaryDecision.taskType === "greeting" ||
|
||
taskBoundaryDecision.taskType === "chat" ||
|
||
taskBoundaryDecision.taskType === "analysis_only" ||
|
||
taskBoundaryDecision.taskType === "prompt_only";
|
||
const prunedTaskBoundaryContext = pruneTaskBoundaryContext({
|
||
decision: taskBoundaryDecision,
|
||
contextPayload: contextPayload || undefined,
|
||
autoContextText: autoContextText || undefined,
|
||
workingState: requestWorkingState || undefined,
|
||
hasCurrentImageInput: hasImageInputForConversationAction,
|
||
});
|
||
const effectiveContextPayload = prunedTaskBoundaryContext.contextPayload;
|
||
const effectiveAutoContextText = prunedTaskBoundaryContext.autoContextText || "";
|
||
const effectiveWorkingState = prunedTaskBoundaryContext.workingState;
|
||
|
||
if (taskBoundaryRequiresAgentConversation && !shouldRunAgentConversation) {
|
||
if (shouldRespectDirectComposerMode) {
|
||
console.log(
|
||
`[Smart Auto Mode] ⛳ Respect direct composer mode (${composerMode}), skip task-boundary agent force: ${taskBoundaryDecision.taskType}/${taskBoundaryDecision.taskMode}`,
|
||
);
|
||
} else {
|
||
shouldRunAgentConversation = true;
|
||
shouldKeepSelectedModel = false;
|
||
console.log(
|
||
`[Smart Auto Mode] 🧭 Force agent conversation due to task boundary: ${taskBoundaryDecision.taskType}/${taskBoundaryDecision.taskMode}`,
|
||
);
|
||
}
|
||
}
|
||
|
||
if (taskBoundaryOverridesLegacyWorkflow && shouldKeepSelectedModel) {
|
||
shouldKeepSelectedModel = false;
|
||
console.log(
|
||
`[Smart Auto Mode] 🧼 Release selected-model lock due to task boundary override: ${taskBoundaryDecision.taskType}`,
|
||
);
|
||
}
|
||
|
||
if (resolvedConversationAction.forceAgentConversation && !shouldRunAgentConversation) {
|
||
if (shouldRespectDirectComposerMode) {
|
||
console.log(
|
||
`[Smart Auto Mode] ⛳ Respect direct composer mode (${composerMode}), skip legacy workflow force: ${resolvedConversationAction.kind}`,
|
||
);
|
||
} else if (taskBoundaryOverridesLegacyWorkflow) {
|
||
console.log(
|
||
`[Smart Auto Mode] 🪫 Ignore legacy workflow follow-up due to task boundary override: ${resolvedConversationAction.kind} -> ${taskBoundaryDecision.taskType}`,
|
||
);
|
||
} else {
|
||
shouldRunAgentConversation = true;
|
||
shouldKeepSelectedModel = resolvedConversationAction.shouldKeepSelectedModel;
|
||
if (resolvedConversationAction.kind === "grid_confirmation_followup" && pendingGridConfirmation) {
|
||
console.log(
|
||
`[Smart Auto Mode] 🧩 Force agent conversation due to pending grid confirmation: ${pendingGridConfirmation.label}`
|
||
);
|
||
} else if (
|
||
resolvedConversationAction.kind === "active_storyboard_followup" &&
|
||
activeStoryboardTaskForChat?.task?.id
|
||
) {
|
||
console.log(
|
||
`[Smart Auto Mode] 🔐 Force agent conversation due to active storyboard task: ${activeStoryboardTaskForChat.task.id}`
|
||
);
|
||
} else if (resolvedConversationAction.kind === "image_generation_followup") {
|
||
console.log("[Smart Auto Mode] 🪄 Force agent conversation due to image generation follow-up workflow");
|
||
} else if (resolvedConversationAction.kind === "image_edit_followup") {
|
||
console.log("[Smart Auto Mode] 🖼️ Force agent conversation due to image edit follow-up workflow");
|
||
}
|
||
}
|
||
}
|
||
|
||
const shouldFastTrackPromptOnly =
|
||
shouldRunAgentConversation && taskBoundaryDecision.taskType === "prompt_only";
|
||
if (shouldFastTrackPromptOnly) {
|
||
if (!requestResolvedMediaInput) {
|
||
requestResolvedMediaInput = resolveChatMediaInput({
|
||
messageParts: message.parts as any[],
|
||
historyMessages: [],
|
||
contextPayload: effectiveContextPayload,
|
||
requestOrigin,
|
||
maxImages: 4,
|
||
});
|
||
}
|
||
|
||
if (!requestTargetResolution) {
|
||
requestTargetResolution = resolveTargetAssets({
|
||
text: requestTextContent,
|
||
mediaInput: requestResolvedMediaInput,
|
||
contextPayload: effectiveContextPayload,
|
||
workingState: effectiveWorkingState || undefined,
|
||
});
|
||
}
|
||
|
||
smartAutoDetectedIntent = "vision";
|
||
smartAutoNeedsClarification = false;
|
||
smartAutoFriendlyMessage =
|
||
"收到，我先分析这张图并整理可直接使用的提示词，不会直接生图。";
|
||
console.log("[Smart Auto Mode] ✍️ Fast-path prompt-only request, skip heavy auto-agent workflow");
|
||
}
|
||
|
||
// ========== Smart Auto Mode Logic ==========
|
||
if (shouldRunAgentConversation && !shouldFastTrackPromptOnly) {
|
||
console.log("[Smart Auto Mode] 🚀 Starting intelligent analysis...");
|
||
let autoTextForFallback = "";
|
||
let autoFromHistoryForFallback = false;
|
||
let autoCurrentImageUrlsForFallback: string[] = [];
|
||
let confirmedPromptForGeneration: string | undefined;
|
||
let forcePromptDrivenImageGeneration = false;
|
||
let shouldPreferArbitratedFriendlyMessage = false;
|
||
const smartAutoInputText =
|
||
resolvedConversationAction.effectiveInputText || requestTextContent;
|
||
|
||
try {
|
||
// 🆕 首先获取历史消息（在 Smart Auto Mode 内部单独获取，避免依赖后面的 messagesFromDb）
|
||
const loadedHistoryMessages = await getMessagesByChatId({ id });
|
||
smartAutoHistoryMessagesFromDb = pruneTaskBoundaryHistoryMessages({
|
||
decision: taskBoundaryDecision,
|
||
messages: loadedHistoryMessages,
|
||
purpose: "model",
|
||
hasCurrentImageInput: hasImageInputForConversationAction,
|
||
});
|
||
smartAutoMediaHistoryMessagesFromDb = pruneTaskBoundaryHistoryMessages({
|
||
decision: taskBoundaryDecision,
|
||
messages: loadedHistoryMessages,
|
||
purpose: "media",
|
||
hasCurrentImageInput: hasImageInputForConversationAction,
|
||
});
|
||
console.log("[Smart Auto Mode] Loaded", loadedHistoryMessages.length, "messages from DB");
|
||
console.log("[Smart Auto Mode] Pruned history:", {
|
||
modelMessages: smartAutoHistoryMessagesFromDb.length,
|
||
mediaMessages: smartAutoMediaHistoryMessagesFromDb.length,
|
||
taskType: taskBoundaryDecision.taskType,
|
||
taskMode: taskBoundaryDecision.taskMode,
|
||
});
|
||
|
||
autoTextForFallback = smartAutoInputText;
|
||
if (!requestWorkingState && !shouldIsolateDirectComposerContext) {
|
||
requestWorkingState = await getConversationWorkingStateByChatId({
|
||
chatId: id,
|
||
});
|
||
}
|
||
confirmedPromptForGeneration = confirmedPromptForConversationState;
|
||
const isPromptCarryForwardExecution =
|
||
resolvedConversationAction.kind === "prompt_carry_forward";
|
||
forcePromptDrivenImageGeneration =
|
||
resolvedConversationAction.usePromptCarryForward &&
|
||
Boolean(confirmedPromptForGeneration);
|
||
if (!requestResolvedMediaInput) {
|
||
requestResolvedMediaInput = resolveChatMediaInput({
|
||
messageParts: message.parts as any[],
|
||
historyMessages: smartAutoMediaHistoryMessagesFromDb,
|
||
contextPayload: effectiveContextPayload,
|
||
requestOrigin,
|
||
maxImages: 4,
|
||
});
|
||
}
|
||
if (!requestTargetResolution) {
|
||
requestTargetResolution = resolveTargetAssets({
|
||
text: requestTextContent,
|
||
mediaInput: requestResolvedMediaInput,
|
||
contextPayload: effectiveContextPayload,
|
||
workingState: effectiveWorkingState || undefined,
|
||
});
|
||
}
|
||
|
||
const hasSelectedImageHint =
|
||
Boolean(effectiveContextPayload?.selectedNodeIds?.length) ||
|
||
/用户已选中\s*\d+\s*张图片/.test(effectiveAutoContextText);
|
||
const hasExplicitTargetHint =
|
||
requestTargetResolution.resolutionMode === "text_reference" ||
|
||
/第[一二三四五六七八九十\d]+张|左边|左侧|左上|左下|右边|右侧|右上|右下|上面|上方|下面|下方|中间|居中|中央|选中/.test(requestTextContent);
|
||
const hasActionableEditIntent = /改|调整|优化|换|替换|去掉|抠|重绘|增强|修|处理|按照|按|保留|统一|色差|背景|清晰|风格/.test(
|
||
requestTextContent
|
||
);
|
||
|
||
const currentMessageImageInputs = requestResolvedMediaInput.currentMessageImageUrls.slice(0, 4);
|
||
let imageUrls =
|
||
requestTargetResolution.targetImageUrls.length > 0
|
||
? requestTargetResolution.targetImageUrls.slice(0, 4)
|
||
: requestResolvedMediaInput.resolvedImageUrls.slice(0, 4);
|
||
let hasImage = imageUrls.length > 0;
|
||
autoCurrentImageUrlsForFallback =
|
||
currentMessageImageInputs.length > 0 ? currentMessageImageInputs.slice() : imageUrls.slice();
|
||
smartAutoHistoryImages =
|
||
currentMessageImageInputs.length === 0 ? imageUrls.slice(0, 4) : [];
|
||
autoFromHistoryForFallback =
|
||
currentMessageImageInputs.length === 0 && imageUrls.length > 0;
|
||
|
||
if (forcePromptDrivenImageGeneration) {
|
||
imageUrls = [];
|
||
hasImage = false;
|
||
autoCurrentImageUrlsForFallback = [];
|
||
smartAutoHistoryImages = [];
|
||
autoFromHistoryForFallback = false;
|
||
autoTextForFallback = confirmedPromptForGeneration || requestTextContent;
|
||
smartAutoOptimizedPrompt = confirmedPromptForGeneration;
|
||
console.log("[Smart Auto Mode] 🎯 Using confirmed prompt-only result for image generation");
|
||
}
|
||
|
||
console.log("[Smart Auto Mode] Input:", {
|
||
textContent: requestTextContent.substring(0, 100),
|
||
hasImage,
|
||
imageCount: imageUrls.length,
|
||
fromHistory: smartAutoHistoryImages.length > 0,
|
||
resolutionMode: requestTargetResolution.resolutionMode,
|
||
resolutionConfidence: requestTargetResolution.confidence,
|
||
});
|
||
|
||
// 🆕 从历史消息中提取对话文本（用于上下文理解）
|
||
const historyMessages: { role: "user" | "assistant"; content: string }[] = [];
|
||
for (const msg of smartAutoHistoryMessagesFromDb.slice(-10)) { // 最多取最近10条
|
||
const parts = msg.parts as any[];
|
||
if (parts && parts.length > 0) {
|
||
// 提取文本内容
|
||
const textParts = parts.filter((p: any) => p.type === 'text');
|
||
const content = textParts.map((p: any) => p.text || '').join(' ').trim();
|
||
if (content) {
|
||
historyMessages.push({
|
||
role: msg.role as "user" | "assistant",
|
||
content: content.substring(0, 300), // 限制长度
|
||
});
|
||
}
|
||
}
|
||
}
|
||
console.log("[Smart Auto Mode] History context:", historyMessages.length, "messages");
|
||
|
||
// 2. 使用 LangGraph 驱动的自动模式工作流
|
||
const { runAutoAgentGraph } = await import("@/lib/ai/auto-agent-graph");
|
||
const autoGraphTimeoutMs = getBudgetedTimeout({
|
||
startedAt: requestStartedAt,
|
||
preferredMs: SMART_AUTO_GRAPH_TIMEOUT_MS,
|
||
minimumMs: 8_000,
|
||
});
|
||
if (!autoGraphTimeoutMs) {
|
||
throw new OperationTimeoutError(
|
||
"runAutoAgentGraph.preResponseBudget",
|
||
CHAT_PRE_RESPONSE_SOFT_TIMEOUT_MS
|
||
);
|
||
}
|
||
const composedConversationState = composeConversationStateProtocol({
|
||
bootstrap: conversationStateBootstrap,
|
||
autoContextText: effectiveAutoContextText || undefined,
|
||
resolvedMediaInput: requestResolvedMediaInput,
|
||
targetResolution: requestTargetResolution,
|
||
});
|
||
console.log("[ConversationStateProtocol] Composed:", {
|
||
workflowType: composedConversationState.threadState.workflowState?.type || null,
|
||
workflowStatus: composedConversationState.threadState.workflowState?.status || null,
|
||
memoryHintsCount: composedConversationState.threadState.memoryHints.length,
|
||
contextSummaryPreview: composedConversationState.threadState.contextSummary?.slice(0, 180),
|
||
});
|
||
pendingProtocolDebugEvents.push({
|
||
phase: "composed",
|
||
payload: {
|
||
workflowState: composedConversationState.threadState.workflowState,
|
||
memoryHints: composedConversationState.threadState.memoryHints,
|
||
contextSummary: composedConversationState.threadState.contextSummary,
|
||
},
|
||
});
|
||
const autoGraphResult = await withOperationTimeout(
|
||
runAutoAgentGraph({
|
||
text: smartAutoInputText,
|
||
hasImage,
|
||
imageUrls,
|
||
assetIds: Array.from(
|
||
new Set([
|
||
...extractMessageImageAssetIds(message.parts as any[]),
|
||
...((effectiveContextPayload?.selectedAssetIds || []).filter(Boolean) as string[]),
|
||
]),
|
||
),
|
||
imageFromHistory: smartAutoHistoryImages.length > 0,
|
||
targetResolutionMode: requestTargetResolution.resolutionMode,
|
||
mediaSource: requestResolvedMediaInput.source,
|
||
hasCanvasSelection:
|
||
Boolean(effectiveContextPayload?.selectedNodeIds?.length) ||
|
||
requestResolvedMediaInput.selectedImageUrls.length > 0,
|
||
selectedImageCount: requestResolvedMediaInput.selectedImageUrls.length,
|
||
historyImageCount:
|
||
requestResolvedMediaInput.historyAssistantImageUrls.length +
|
||
requestResolvedMediaInput.historyUserImageUrls.length,
|
||
currentUploadImageCount: requestResolvedMediaInput.currentMessageImageUrls.length,
|
||
historyMessages,
|
||
context:
|
||
composedConversationState.composedConversationMemory.contextSummary ||
|
||
effectiveAutoContextText ||
|
||
undefined,
|
||
preferredModelId: preferredAgentModelId,
|
||
threadState: composedConversationState.threadState,
|
||
}),
|
||
"runAutoAgentGraph",
|
||
autoGraphTimeoutMs,
|
||
);
|
||
|
||
const requestedStoryboardGrid = detectRequestedStoryboardGrid(
|
||
pendingGridConfirmation && isGridConfirmationReply(requestTextContent)
|
||
? pendingGridConfirmation.originalPrompt
|
||
: requestTextContent
|
||
);
|
||
if (requestedStoryboardGrid) {
|
||
storyboardPreferenceMemory = buildStoryboardPreferenceMemoryUpdate({
|
||
existing: storyboardPreferenceMemory,
|
||
preferredGridType: requestedStoryboardGrid.gridType,
|
||
preferredShotCount: requestedStoryboardGrid.shotCount,
|
||
prefersConfirmation: true,
|
||
lastUsedGridType: requestedStoryboardGrid.gridType,
|
||
lastUsedShotCount: requestedStoryboardGrid.shotCount,
|
||
});
|
||
}
|
||
|
||
smartAutoPlan = autoGraphResult.plan;
|
||
smartAutoExecution = autoGraphResult.execution;
|
||
|
||
if (autoGraphResult.workflowDecision) {
|
||
smartAutoWorkflowDecision = autoGraphResult.workflowDecision;
|
||
console.log("[Smart Auto Mode] Workflow decision:", {
|
||
action: autoGraphResult.workflowDecision.action,
|
||
intent: autoGraphResult.workflowDecision.confirmationIntent,
|
||
workflowId: autoGraphResult.workflowDecision.workflowId,
|
||
workflowType: autoGraphResult.workflowDecision.workflowType,
|
||
nextWorkflowStatus: autoGraphResult.workflowDecision.nextWorkflowStatus,
|
||
allowWorkflowExecution: autoGraphResult.workflowDecision.allowWorkflowExecution,
|
||
});
|
||
|
||
if (
|
||
autoGraphResult.workflowDecision.workflowType === "storyboard_video" &&
|
||
autoGraphResult.workflowDecision.workflowId
|
||
) {
|
||
const storyboardTaskId = autoGraphResult.workflowDecision.workflowId;
|
||
try {
|
||
switch (autoGraphResult.workflowDecision.action) {
|
||
case "awaiting_confirmation":
|
||
await markVideoStoryboardAwaitingConfirmation({
|
||
taskId: storyboardTaskId,
|
||
});
|
||
break;
|
||
case "confirmed":
|
||
await markVideoStoryboardConfirmed({
|
||
taskId: storyboardTaskId,
|
||
messageId: message.id,
|
||
});
|
||
break;
|
||
case "revision_requested":
|
||
await markVideoStoryboardDraftForRevision({
|
||
taskId: storyboardTaskId,
|
||
});
|
||
break;
|
||
case "cancel_requested":
|
||
await cancelVideoStoryboardTaskByUser({
|
||
taskId: storyboardTaskId,
|
||
error: "user_cancelled_via_chat",
|
||
});
|
||
break;
|
||
case "pass":
|
||
default:
|
||
break;
|
||
}
|
||
} catch (workflowPersistError) {
|
||
console.warn(
|
||
"[Smart Auto Mode] Failed to persist storyboard workflow decision:",
|
||
workflowPersistError
|
||
);
|
||
}
|
||
}
|
||
}
|
||
|
||
if (forcePromptDrivenImageGeneration && confirmedPromptForGeneration) {
|
||
smartAutoDetectedIntent = "image_gen";
|
||
smartAutoOptimizedPrompt = confirmedPromptForGeneration;
|
||
smartAutoExecution = {
|
||
mode: "generate_image",
|
||
routeIntent: "image_gen",
|
||
routeSubIntent: undefined,
|
||
targetSource: "none",
|
||
preferredCapability: "generate_image",
|
||
shouldUseStructuredReply: true,
|
||
shouldOptimizePrompt: true,
|
||
shouldApplyImagePipeline: true,
|
||
shouldApplyVideoPipeline: false,
|
||
shouldStayConversational: false,
|
||
shouldPublishToCanvas: false,
|
||
toolCalls: [],
|
||
availableTools: [],
|
||
};
|
||
smartAutoFriendlyMessage =
|
||
smartAutoFriendlyMessage ||
|
||
(isPromptCarryForwardExecution
|
||
? "收到，我会使用上一轮优化后的提示词，结合你这轮图片继续执行。"
|
||
: "收到，我会按你刚确认的提示词直接生图。");
|
||
}
|
||
|
||
if (smartAutoPlan) {
|
||
console.log("[Smart Auto Mode] Plan result:", {
|
||
goal: smartAutoPlan.goal,
|
||
nextAction: smartAutoPlan.nextAction,
|
||
intent: smartAutoPlan.intent,
|
||
targetSource: smartAutoPlan.targetSource,
|
||
preferredCapability: smartAutoPlan.preferredCapability,
|
||
confidence: smartAutoPlan.confidence,
|
||
toolCalls: smartAutoPlan.toolCalls?.map((tool) => tool.name) || [],
|
||
});
|
||
}
|
||
|
||
if (autoGraphResult.analysis) {
|
||
console.log("[Smart Auto Mode] Analysis result:", {
|
||
intent: autoGraphResult.analysis.intent,
|
||
subIntent: autoGraphResult.analysis.subIntent,
|
||
confidence: autoGraphResult.analysis.confidence,
|
||
needsClarification: autoGraphResult.analysis.needsClarification,
|
||
});
|
||
|
||
const arbitratedIntent = arbitrateAgentIntent({
|
||
text: requestTextContent,
|
||
hasImage,
|
||
hasHistoryImage: smartAutoHistoryImages.length > 0,
|
||
actionResolution: resolvedConversationAction,
|
||
taskBoundaryDecision,
|
||
analysis: autoGraphResult.analysis,
|
||
execution: smartAutoExecution,
|
||
targetSource: requestTargetResolution.resolutionMode,
|
||
});
|
||
if (arbitratedIntent.overridden) {
|
||
console.log("[IntentArbiter] Override auto-agent intent:", {
|
||
fromIntent: autoGraphResult.analysis.intent,
|
||
fromMode: autoGraphResult.execution?.mode,
|
||
toIntent: arbitratedIntent.analysis.intent,
|
||
toMode: arbitratedIntent.execution.mode,
|
||
reason: arbitratedIntent.reason,
|
||
});
|
||
}
|
||
|
||
smartAutoExecution = arbitratedIntent.execution as AgentPreparedExecution;
|
||
smartAutoDetectedIntent = arbitratedIntent.analysis.intent;
|
||
smartAutoNeedsClarification =
|
||
Boolean(arbitratedIntent.analysis.needsClarification) ||
|
||
(!arbitratedIntent.overridden && smartAutoPlan?.nextAction === "clarify");
|
||
smartAutoClarificationQuestion = arbitratedIntent.analysis.clarifyingQuestions?.[0];
|
||
if (arbitratedIntent.friendlyMessage) {
|
||
smartAutoFriendlyMessage = arbitratedIntent.friendlyMessage;
|
||
shouldPreferArbitratedFriendlyMessage = true;
|
||
}
|
||
|
||
const targetExecutionOverride = resolveTargetExecutionOverride({
|
||
hasImage,
|
||
hasSelectedImageHint,
|
||
hasExplicitTargetHint,
|
||
hasActionableEditIntent,
|
||
executionMode: smartAutoExecution?.mode,
|
||
friendlyMessage: smartAutoFriendlyMessage,
|
||
optimizedPrompt: smartAutoOptimizedPrompt,
|
||
requestText: requestTextContent,
|
||
});
|
||
|
||
if (
|
||
smartAutoNeedsClarification &&
|
||
targetExecutionOverride.shouldBypassClarification
|
||
) {
|
||
smartAutoNeedsClarification = false;
|
||
smartAutoClarificationQuestion = undefined;
|
||
smartAutoOptimizedPrompt = targetExecutionOverride.nextOptimizedPrompt;
|
||
smartAutoFriendlyMessage = targetExecutionOverride.nextFriendlyMessage;
|
||
console.log(
|
||
"[Smart Auto Mode] ✅ Skip clarification due to explicit target/selection context"
|
||
);
|
||
}
|
||
}
|
||
|
||
const requestedGrid =
|
||
!pendingGridConfirmation || smartAutoInputText === requestTextContent
|
||
? detectRequestedStoryboardGrid(requestTextContent)
|
||
: pendingGridConfirmation;
|
||
const gridConfirmationResolution = resolveGridConfirmationRequest({
|
||
requestedGrid,
|
||
hasActiveStoryboardTask: Boolean(activeStoryboardTaskForChat?.task?.id),
|
||
executionMode: smartAutoExecution?.mode,
|
||
detectedIntent: smartAutoDetectedIntent,
|
||
requestText: requestTextContent,
|
||
effectiveInputText: smartAutoInputText,
|
||
});
|
||
if (gridConfirmationResolution.shouldRequestConfirmation) {
|
||
shouldAskGridConfirmation = true;
|
||
smartAutoNeedsClarification = true;
|
||
smartAutoClarificationQuestion = gridConfirmationResolution.clarificationMessage;
|
||
smartAutoFriendlyMessage = gridConfirmationResolution.clarificationMessage;
|
||
pendingGridConfirmation = gridConfirmationResolution.pendingGridConfirmation;
|
||
console.log("[Smart Auto Mode] 🧱 Require grid confirmation before generation:", {
|
||
gridType: gridConfirmationResolution.pendingGridConfirmation?.gridType,
|
||
shotCount: gridConfirmationResolution.pendingGridConfirmation?.shotCount,
|
||
});
|
||
}
|
||
|
||
if (autoGraphResult.friendlyMessage && !shouldPreferArbitratedFriendlyMessage) {
|
||
smartAutoFriendlyMessage = autoGraphResult.friendlyMessage;
|
||
console.log("[Smart Auto Mode] 💬 Conversational reply:", smartAutoFriendlyMessage);
|
||
}
|
||
|
||
if (shouldAskGridConfirmation && smartAutoClarificationQuestion) {
|
||
smartAutoFriendlyMessage = smartAutoClarificationQuestion;
|
||
}
|
||
|
||
if (
|
||
!smartAutoNeedsClarification &&
|
||
hasImage &&
|
||
hasActionableEditIntent &&
|
||
(hasSelectedImageHint || hasExplicitTargetHint) &&
|
||
smartAutoFriendlyMessage &&
|
||
(/[?？]$/.test(smartAutoFriendlyMessage.trim()) ||
|
||
/哪一张|哪个|请先确认/.test(smartAutoFriendlyMessage))
|
||
) {
|
||
smartAutoFriendlyMessage = "收到，我将按你指定的目标图直接执行。";
|
||
console.log("[Smart Auto Mode] ✅ Replace clarification-like friendly reply");
|
||
}
|
||
|
||
const autoIntent = smartAutoDetectedIntent || autoGraphResult.analysis?.intent;
|
||
const autoExecutionMode = smartAutoExecution?.mode;
|
||
const isImageTask =
|
||
autoExecutionMode === "generate_image" ||
|
||
autoExecutionMode === "edit_image";
|
||
const needsRicherAutoReply =
|
||
!smartAutoNeedsClarification &&
|
||
!smartAutoFriendlyMessage;
|
||
|
||
if (needsRicherAutoReply) {
|
||
if (isImageTask) {
|
||
smartAutoFriendlyMessage =
|
||
autoGraphResult.analysis?.conversationalReply ||
|
||
"我先分析你这轮图片里的主体、风格和构图，再给你执行方案。";
|
||
} else if (autoIntent === "vision") {
|
||
smartAutoFriendlyMessage =
|
||
autoGraphResult.analysis?.conversationalReply ||
|
||
"收到，我先看图分析，再直接回答你。";
|
||
} else if (autoIntent === "chat") {
|
||
smartAutoFriendlyMessage =
|
||
autoGraphResult.analysis?.conversationalReply ||
|
||
"收到，我们先继续聊你的这个需求。";
|
||
} else {
|
||
smartAutoFriendlyMessage =
|
||
smartAutoExecution?.shouldUseStructuredReply
|
||
? buildAutoModeFallbackMessage({
|
||
text: autoTextForFallback,
|
||
hasImage,
|
||
fromHistory: smartAutoHistoryImages.length > 0,
|
||
})
|
||
: "收到，我先理解你的当前目标，再继续处理。";
|
||
}
|
||
console.log("[Smart Auto Mode] 🧩 Filled missing conversational reply");
|
||
}
|
||
|
||
if (autoGraphResult.optimizedPrompt) {
|
||
smartAutoOptimizedPrompt = autoGraphResult.optimizedPrompt;
|
||
console.log("[Smart Auto Mode] ✨ Prompt optimized (LLM):", {
|
||
original: requestTextContent.substring(0, 50),
|
||
optimized: autoGraphResult.optimizedPrompt.substring(0, 100),
|
||
});
|
||
}
|
||
if (autoGraphResult.videoPlan) {
|
||
smartAutoVideoPlan = autoGraphResult.videoPlan;
|
||
smartAutoRecommendedVideoParams = autoGraphResult.videoPlan.recommendedVideoParams;
|
||
if (!smartAutoOptimizedPrompt && autoGraphResult.videoPlan.finalPrompt?.trim()) {
|
||
smartAutoOptimizedPrompt = autoGraphResult.videoPlan.finalPrompt.trim();
|
||
smartAutoPromptFromVideoPlan = true;
|
||
}
|
||
console.log("[Smart Auto Mode] 🎬 Agent video plan ready:", {
|
||
hasScript: Boolean(autoGraphResult.videoPlan.script),
|
||
storyboardCount: autoGraphResult.videoPlan.storyboard?.length || 0,
|
||
hasFinalPrompt: Boolean(autoGraphResult.videoPlan.finalPrompt),
|
||
recommendedVideoParams: smartAutoRecommendedVideoParams,
|
||
});
|
||
}
|
||
if (autoGraphResult.recommendedImageParams) {
|
||
smartAutoRecommendedImageParams = autoGraphResult.recommendedImageParams;
|
||
console.log("[Smart Auto Mode] 🎛️ Agent recommended image params:", smartAutoRecommendedImageParams);
|
||
}
|
||
|
||
const shouldForceConversationalRouteForVisualModel =
|
||
shouldKeepSelectedModel &&
|
||
requestedModelIsFixedVisual &&
|
||
!requestedModelIsConversationMultimodal &&
|
||
Boolean(smartAutoExecution?.shouldStayConversational);
|
||
|
||
if (autoGraphResult.route && (!shouldKeepSelectedModel || shouldForceConversationalRouteForVisualModel)) {
|
||
const routeIsVideo = isVideoProviderType(autoGraphResult.route.providerType, {
|
||
modelId: autoGraphResult.route.modelId,
|
||
});
|
||
if (routeIsVideo && !allowAgentVideoGeneration) {
|
||
const fallbackIntent: "image_edit" | "image_gen" = hasImage ? "image_edit" : "image_gen";
|
||
smartAutoDetectedIntent = fallbackIntent;
|
||
smartAutoExecution = smartAutoExecution
|
||
? {
|
||
...smartAutoExecution,
|
||
mode: hasImage ? "edit_image" : "generate_image",
|
||
routeIntent: fallbackIntent,
|
||
shouldApplyImagePipeline: true,
|
||
shouldApplyVideoPipeline: false,
|
||
shouldOptimizePrompt: true,
|
||
shouldStayConversational: false,
|
||
shouldUseStructuredReply: true,
|
||
}
|
||
: smartAutoExecution;
|
||
smartAutoVideoPlan = undefined;
|
||
smartAutoRecommendedVideoParams = undefined;
|
||
if (smartAutoPromptFromVideoPlan) {
|
||
smartAutoOptimizedPrompt = requestTextContent.trim() || undefined;
|
||
smartAutoPromptFromVideoPlan = false;
|
||
}
|
||
smartAutoFriendlyMessage =
|
||
smartAutoFriendlyMessage && !/视频/.test(smartAutoFriendlyMessage)
|
||
? smartAutoFriendlyMessage
|
||
: "当前 Agent 默认按图片任务执行；如需生成视频，请先手动切换到“视频”模式。";
|
||
console.log(
|
||
`[Smart Auto Mode] 🚫 Blocked auto video route (${autoGraphResult.route.displayName}), fallback to ${fallbackIntent}`,
|
||
);
|
||
try {
|
||
const { routeToModel } = await import("@/lib/ai/model-router");
|
||
const fallbackRoute = await routeToModel({
|
||
intent: fallbackIntent,
|
||
subIntent: autoGraphResult.analysis?.subIntent,
|
||
hasImage,
|
||
});
|
||
if (!isVideoProviderType(fallbackRoute.providerType, { modelId: fallbackRoute.modelId })) {
|
||
finalModelId = fallbackRoute.id;
|
||
console.log(
|
||
`[Smart Auto Mode] ✅ Fallback routed to: ${fallbackRoute.displayName} (${fallbackRoute.reason})`,
|
||
);
|
||
} else {
|
||
console.warn(
|
||
`[Smart Auto Mode] ⚠️ Fallback route still video (${fallbackRoute.displayName}), keep previous model`,
|
||
);
|
||
}
|
||
} catch (fallbackRouteError) {
|
||
console.error("[Smart Auto Mode] Failed to resolve non-video fallback route:", fallbackRouteError);
|
||
}
|
||
} else {
|
||
finalModelId = autoGraphResult.route.id;
|
||
console.log(
|
||
shouldForceConversationalRouteForVisualModel
|
||
? `[Smart Auto Mode] ✅ Switched visual model to conversational route: ${autoGraphResult.route.displayName} (${autoGraphResult.route.reason})`
|
||
: `[Smart Auto Mode] ✅ Routed to: ${autoGraphResult.route.displayName} (${autoGraphResult.route.reason})`,
|
||
);
|
||
}
|
||
} else if (autoGraphResult.route && shouldKeepSelectedModel) {
|
||
finalModelId = requestedModelId;
|
||
console.log("[Smart Auto Mode] 🔒 Keep user-selected model, skip routing override");
|
||
}
|
||
|
||
} catch (autoError) {
|
||
// 智能分析失败时，回退到简单逻辑
|
||
console.error("[Smart Auto Mode] ❌ Error, falling back:", autoError);
|
||
const isAutoTimeout = autoError instanceof OperationTimeoutError;
|
||
if (isAutoTimeout) {
|
||
console.warn(
|
||
`[Smart Auto Mode] ⏱️ Auto-agent timed out or exceeded pre-response budget (${CHAT_PRE_RESPONSE_SOFT_TIMEOUT_MS}ms), using local fallback`,
|
||
);
|
||
}
|
||
try {
|
||
const { analyzeIntent } = await import("@/lib/ai/smart-analyzer");
|
||
const rescueImageUrls =
|
||
autoCurrentImageUrlsForFallback.length > 0
|
||
? autoCurrentImageUrlsForFallback
|
||
: smartAutoHistoryImages;
|
||
const rescueHasImage = rescueImageUrls.length > 0;
|
||
const rescueFromHistory =
|
||
rescueHasImage &&
|
||
autoCurrentImageUrlsForFallback.length === 0 &&
|
||
smartAutoHistoryImages.length > 0;
|
||
const rescueAssetIds = Array.from(
|
||
new Set([
|
||
...extractMessageImageAssetIds(message.parts as any[]),
|
||
...((effectiveContextPayload?.selectedAssetIds || []).filter(Boolean) as string[]),
|
||
]),
|
||
);
|
||
const rescueTimeoutMs = getBudgetedTimeout({
|
||
startedAt: requestStartedAt,
|
||
preferredMs: SMART_AUTO_FALLBACK_TIMEOUT_MS,
|
||
minimumMs: 5_000,
|
||
});
|
||
if (rescueTimeoutMs) {
|
||
const rescueAnalysis = await withOperationTimeout(
|
||
analyzeIntent({
|
||
text: autoTextForFallback,
|
||
hasImage: rescueHasImage,
|
||
imageUrls: rescueImageUrls,
|
||
assetIds: rescueAssetIds,
|
||
imageFromHistory: rescueFromHistory || autoFromHistoryForFallback,
|
||
historyMessages: [],
|
||
context: undefined,
|
||
preferredModelId: preferredAgentModelId,
|
||
}),
|
||
"fallbackAnalyzeIntent",
|
||
rescueTimeoutMs,
|
||
);
|
||
smartAutoDetectedIntent = rescueAnalysis.intent;
|
||
const arbitratedIntent = arbitrateAgentIntent({
|
||
text: autoTextForFallback,
|
||
hasImage: rescueHasImage,
|
||
hasHistoryImage: rescueFromHistory || autoFromHistoryForFallback,
|
||
actionResolution: resolvedConversationAction,
|
||
taskBoundaryDecision,
|
||
analysis: rescueAnalysis,
|
||
execution: undefined,
|
||
targetSource: requestTargetResolution?.resolutionMode || "none",
|
||
});
|
||
if (arbitratedIntent.overridden) {
|
||
console.log("[IntentArbiter] Override rescue intent:", {
|
||
fromIntent: rescueAnalysis.intent,
|
||
toIntent: arbitratedIntent.analysis.intent,
|
||
toMode: arbitratedIntent.execution.mode,
|
||
reason: arbitratedIntent.reason,
|
||
});
|
||
}
|
||
smartAutoDetectedIntent = arbitratedIntent.analysis.intent;
|
||
smartAutoNeedsClarification = Boolean(arbitratedIntent.analysis.needsClarification);
|
||
smartAutoClarificationQuestion = arbitratedIntent.analysis.clarifyingQuestions?.[0];
|
||
smartAutoExecution = arbitratedIntent.execution as AgentPreparedExecution;
|
||
if (arbitratedIntent.friendlyMessage || rescueAnalysis.conversationalReply?.trim()) {
|
||
smartAutoFriendlyMessage = (
|
||
arbitratedIntent.friendlyMessage ||
|
||
rescueAnalysis.conversationalReply ||
|
||
""
|
||
).trim();
|
||
}
|
||
} else {
|
||
console.warn("[Smart Auto Mode] Skip rescue analysis due to pre-response budget");
|
||
}
|
||
} catch (rescueError) {
|
||
console.error("[Smart Auto Mode] Rescue analysis failed:", rescueError);
|
||
try {
|
||
const { fallbackKeywordAnalysis } = await import("@/lib/ai/smart-analyzer");
|
||
const keywordFallback = fallbackKeywordAnalysis(
|
||
autoTextForFallback,
|
||
autoCurrentImageUrlsForFallback.length > 0 || smartAutoHistoryImages.length > 0,
|
||
);
|
||
const arbitratedIntent = arbitrateAgentIntent({
|
||
text: autoTextForFallback,
|
||
hasImage:
|
||
autoCurrentImageUrlsForFallback.length > 0 || smartAutoHistoryImages.length > 0,
|
||
hasHistoryImage: smartAutoHistoryImages.length > 0 || autoFromHistoryForFallback,
|
||
actionResolution: resolvedConversationAction,
|
||
taskBoundaryDecision,
|
||
analysis: keywordFallback,
|
||
execution: undefined,
|
||
targetSource: requestTargetResolution?.resolutionMode || "none",
|
||
});
|
||
if (arbitratedIntent.overridden) {
|
||
console.log("[IntentArbiter] Override keyword-fallback intent:", {
|
||
fromIntent: keywordFallback.intent,
|
||
toIntent: arbitratedIntent.analysis.intent,
|
||
toMode: arbitratedIntent.execution.mode,
|
||
reason: arbitratedIntent.reason,
|
||
});
|
||
}
|
||
smartAutoDetectedIntent = arbitratedIntent.analysis.intent;
|
||
smartAutoNeedsClarification = Boolean(arbitratedIntent.analysis.needsClarification);
|
||
smartAutoClarificationQuestion = arbitratedIntent.analysis.clarifyingQuestions?.[0];
|
||
smartAutoExecution = arbitratedIntent.execution as AgentPreparedExecution;
|
||
if (arbitratedIntent.friendlyMessage || keywordFallback.conversationalReply?.trim()) {
|
||
smartAutoFriendlyMessage = (
|
||
arbitratedIntent.friendlyMessage ||
|
||
keywordFallback.conversationalReply ||
|
||
""
|
||
).trim();
|
||
console.warn("[Smart Auto Mode] Applied keyword fallback after rescue failure");
|
||
}
|
||
} catch (keywordFallbackError) {
|
||
console.warn(
|
||
"[Smart Auto Mode] Keyword fallback after rescue failure also failed:",
|
||
keywordFallbackError,
|
||
);
|
||
}
|
||
}
|
||
|
||
if (
|
||
!smartAutoOptimizedPrompt &&
|
||
smartAutoExecution?.shouldOptimizePrompt &&
|
||
(smartAutoDetectedIntent === "image_edit" || smartAutoDetectedIntent === "image_gen")
|
||
) {
|
||
try {
|
||
const { optimizePrompt } = await import("@/lib/ai/prompt-optimizer");
|
||
const optimizedFallback = optimizePrompt({
|
||
originalPrompt: autoTextForFallback,
|
||
intent: smartAutoDetectedIntent,
|
||
subIntent: smartAutoExecution.routeSubIntent,
|
||
});
|
||
if (optimizedFallback.optimizedPrompt?.trim()) {
|
||
smartAutoOptimizedPrompt = optimizedFallback.optimizedPrompt.trim();
|
||
console.log("[Smart Auto Mode] ✨ Prompt optimized (fallback heuristic):", {
|
||
original: autoTextForFallback.substring(0, 80),
|
||
optimized: smartAutoOptimizedPrompt.substring(0, 140),
|
||
intent: smartAutoDetectedIntent,
|
||
subIntent: smartAutoExecution.routeSubIntent || null,
|
||
});
|
||
}
|
||
if (!smartAutoRecommendedImageParams && optimizedFallback.recommendedImageParams) {
|
||
smartAutoRecommendedImageParams = optimizedFallback.recommendedImageParams;
|
||
}
|
||
} catch (fallbackOptimizeError) {
|
||
console.warn("[Smart Auto Mode] Fallback prompt optimization failed:", fallbackOptimizeError);
|
||
}
|
||
}
|
||
|
||
const allModels = await getEnabledAiModelsWithProvider();
|
||
const hasImage =
|
||
autoCurrentImageUrlsForFallback.length > 0 || smartAutoHistoryImages.length > 0;
|
||
let fallbackVisualSummary: string | undefined;
|
||
|
||
if (isAutoTimeout && hasImage) {
|
||
try {
|
||
const currentImageUrls =
|
||
autoCurrentImageUrlsForFallback.length > 0
|
||
? autoCurrentImageUrlsForFallback
|
||
: collectImageInputsFromMessageParts(message.parts as any[], requestOrigin);
|
||
const summaryImageUrls =
|
||
currentImageUrls.length > 0 ? currentImageUrls : smartAutoHistoryImages;
|
||
|
||
if (summaryImageUrls.length > 0) {
|
||
const { summarizeImageForExecutionReport } = await import("@/lib/ai/smart-analyzer");
|
||
const summaryAssetIds = Array.from(
|
||
new Set([
|
||
...extractMessageImageAssetIds(message.parts as any[]),
|
||
...((effectiveContextPayload?.selectedAssetIds || []).filter(Boolean) as string[]),
|
||
]),
|
||
);
|
||
const fallbackSummaryTimeoutMs = getBudgetedTimeout({
|
||
startedAt: requestStartedAt,
|
||
preferredMs: SMART_AUTO_IMAGE_SUMMARY_TIMEOUT_MS,
|
||
minimumMs: 5_000,
|
||
});
|
||
if (fallbackSummaryTimeoutMs) {
|
||
const summary = await withOperationTimeout(
|
||
summarizeImageForExecutionReport({
|
||
text: autoTextForFallback,
|
||
imageUrls: summaryImageUrls,
|
||
context: effectiveAutoContextText || undefined,
|
||
preferredModelId: preferredAgentModelId,
|
||
assetIds: summaryAssetIds,
|
||
timeoutMs: fallbackSummaryTimeoutMs,
|
||
}),
|
||
"fallbackVisualSummary",
|
||
fallbackSummaryTimeoutMs,
|
||
);
|
||
if (summary?.trim()) {
|
||
fallbackVisualSummary = summary.trim();
|
||
}
|
||
} else {
|
||
console.warn("[Smart Auto Mode] Skip fallback visual summary due to pre-response budget");
|
||
}
|
||
}
|
||
} catch (summaryError) {
|
||
console.warn("[Smart Auto Mode] Fallback visual summary failed:", summaryError);
|
||
}
|
||
}
|
||
|
||
// 回退路由：优先按分析出的意图选模型，避免“有图就生图”
|
||
let bestModel = null;
|
||
try {
|
||
const fallbackIntent =
|
||
smartAutoDetectedIntent || (hasImage ? "vision" : "chat");
|
||
const { routeToModel } = await import("@/lib/ai/model-router");
|
||
const fallbackRoute = await routeToModel({
|
||
intent: fallbackIntent,
|
||
subIntent: smartAutoExecution?.routeSubIntent,
|
||
hasImage,
|
||
});
|
||
bestModel = allModels.find(
|
||
(item) => item.model.id === fallbackRoute.id || item.model.modelId === fallbackRoute.modelId,
|
||
) || null;
|
||
if (bestModel) {
|
||
console.log(
|
||
`[Smart Auto Mode] Fallback routed by intent: ${fallbackRoute.displayName} (${fallbackRoute.reason})`,
|
||
);
|
||
}
|
||
} catch (fallbackRouteError) {
|
||
console.warn("[Smart Auto Mode] Fallback routeByIntent failed, use local heuristic:", fallbackRouteError);
|
||
}
|
||
|
||
if (!bestModel && hasImage) {
|
||
bestModel = allModels.find(
|
||
(m) =>
|
||
m.model.supportsImageGen &&
|
||
!m.model.isSubModel &&
|
||
!isVideoProviderType(m.provider?.type, m.model),
|
||
);
|
||
if (isAutoTimeout) {
|
||
smartAutoFriendlyMessage = buildAutoModeFallbackMessage({
|
||
text: autoTextForFallback,
|
||
hasImage: true,
|
||
fromHistory: autoFromHistoryForFallback || smartAutoHistoryImages.length > 0,
|
||
imageSummary: fallbackVisualSummary,
|
||
});
|
||
} else {
|
||
smartAutoFriendlyMessage =
|
||
smartAutoFriendlyMessage ||
|
||
buildAutoModeFallbackMessage({
|
||
text: autoTextForFallback,
|
||
hasImage: true,
|
||
fromHistory: autoFromHistoryForFallback || smartAutoHistoryImages.length > 0,
|
||
imageSummary: fallbackVisualSummary,
|
||
});
|
||
}
|
||
} else {
|
||
smartAutoFriendlyMessage = smartAutoFriendlyMessage || buildAutoModeFallbackMessage({
|
||
text: autoTextForFallback,
|
||
hasImage: false,
|
||
fromHistory: false,
|
||
});
|
||
}
|
||
if (!bestModel) {
|
||
bestModel = allModels.find(
|
||
(m) =>
|
||
m.model.isDefault &&
|
||
!m.model.isSubModel &&
|
||
!isVideoProviderType(m.provider?.type, m.model),
|
||
);
|
||
}
|
||
if (!bestModel && allModels.length > 0) {
|
||
bestModel = allModels.find(
|
||
(m) => !m.model.isSubModel && !isVideoProviderType(m.provider?.type, m.model),
|
||
) || allModels.find((m) => !m.model.isSubModel) || allModels[0];
|
||
}
|
||
|
||
if (shouldKeepSelectedModel) {
|
||
finalModelId = requestedModelId;
|
||
console.log("[Smart Auto Mode] 🔒 Keep user-selected model in fallback mode");
|
||
} else if (bestModel) {
|
||
finalModelId = bestModel.model.id;
|
||
console.log(`[Smart Auto Mode] Fallback selected: ${bestModel.model.displayName}`);
|
||
} else {
|
||
return new ChatSDKError("bad_request:api", "No available models for auto mode").toResponse();
|
||
}
|
||
}
|
||
}
|
||
// ========== End Smart Auto Mode Logic ==========
|
||
|
||
if (shouldRunAgentConversation && !allowAgentVideoGeneration) {
|
||
const finalModelIsUuid =
|
||
/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test(finalModelId);
|
||
let finalProviderType: string | undefined;
|
||
let finalModelConfig: Awaited<ReturnType<typeof getAiModelById>> | null = null;
|
||
|
||
if (finalModelIsUuid) {
|
||
finalModelConfig = await getAiModelById({ id: finalModelId }).catch(() => null);
|
||
if (finalModelConfig) {
|
||
finalProviderType = (
|
||
await getAiProviderById({ id: finalModelConfig.providerId }).catch(() => null)
|
||
)?.type as string | undefined;
|
||
}
|
||
}
|
||
|
||
if (isVideoProviderType(finalProviderType, finalModelConfig || undefined)) {
|
||
const allModels = await getEnabledAiModelsWithProvider(false);
|
||
const fallbackModel =
|
||
allModels.find(
|
||
(m) =>
|
||
!m.model.isSubModel &&
|
||
m.model.supportsImageGen &&
|
||
!isVideoProviderType(m.provider?.type, m.model),
|
||
) ||
|
||
allModels.find(
|
||
(m) =>
|
||
!m.model.isSubModel &&
|
||
!isVideoProviderType(m.provider?.type, m.model),
|
||
);
|
||
|
||
if (fallbackModel) {
|
||
const hasImageInput = message.parts.some((part: any) =>
|
||
part?.type === "file" || part?.type === "image" || part?.type === "image_url"
|
||
);
|
||
smartAutoDetectedIntent = hasImageInput ? "image_edit" : "image_gen";
|
||
smartAutoExecution = smartAutoExecution
|
||
? {
|
||
...smartAutoExecution,
|
||
mode: hasImageInput ? "edit_image" : "generate_image",
|
||
routeIntent: hasImageInput ? "image_edit" : "image_gen",
|
||
shouldApplyImagePipeline: true,
|
||
shouldApplyVideoPipeline: false,
|
||
shouldOptimizePrompt: true,
|
||
shouldStayConversational: false,
|
||
shouldUseStructuredReply: true,
|
||
}
|
||
: smartAutoExecution;
|
||
finalModelId = fallbackModel.model.id;
|
||
smartAutoVideoPlan = undefined;
|
||
smartAutoRecommendedVideoParams = undefined;
|
||
console.log(
|
||
`[Smart Auto Mode] 🚫 Safety override: replace video model (${finalProviderType}) with ${fallbackModel.model.displayName}`,
|
||
);
|
||
}
|
||
}
|
||
}
|
||
|
||
if (
|
||
shouldRunAgentConversation &&
|
||
!smartAutoNeedsClarification &&
|
||
!smartAutoFriendlyMessage?.trim()
|
||
) {
|
||
const latestUserText = message.parts
|
||
.filter((part): part is { type: "text"; text: string } => part.type === "text")
|
||
.map((part) => part.text)
|
||
.join(" ");
|
||
const hasCurrentImageInput = message.parts.some((part: any) =>
|
||
part?.type === "file" || part?.type === "image" || part?.type === "image_url"
|
||
);
|
||
const hasHistoryImageInput = smartAutoHistoryImages.length > 0;
|
||
smartAutoFriendlyMessage =
|
||
smartAutoExecution?.shouldUseStructuredReply
|
||
? buildAutoModeFallbackMessage({
|
||
text: latestUserText,
|
||
hasImage: hasCurrentImageInput || hasHistoryImageInput,
|
||
fromHistory: !hasCurrentImageInput && hasHistoryImageInput,
|
||
})
|
||
: "收到，我先理解你的当前目标，再继续处理。";
|
||
console.log("[Smart Auto Mode] 🧩 Backfilled conversational reply with fallback");
|
||
}
|
||
|
||
if (
|
||
shouldRunAgentConversation &&
|
||
!smartAutoNeedsClarification &&
|
||
smartAutoExecution?.shouldUseStructuredReply
|
||
) {
|
||
const latestUserText = message.parts
|
||
.filter((part): part is { type: "text"; text: string } => part.type === "text")
|
||
.map((part) => part.text)
|
||
.join(" ");
|
||
const hasCurrentImageInput = message.parts.some((part: any) =>
|
||
part?.type === "file" || part?.type === "image" || part?.type === "image_url"
|
||
);
|
||
const hasHistoryImageInput = smartAutoHistoryImages.length > 0;
|
||
smartAutoFriendlyMessage = ensureStructuredAgentFriendlyMessage({
|
||
message: smartAutoFriendlyMessage,
|
||
text: latestUserText,
|
||
hasImage: hasCurrentImageInput || hasHistoryImageInput,
|
||
fromHistory: !hasCurrentImageInput && hasHistoryImageInput,
|
||
});
|
||
}
|
||
|
||
const hasUserCustomImageParams = hasUserCustomizedImageParams(requestBody.imageParams);
|
||
const hasUserCustomVideoParams = hasUserCustomizedVideoParams(requestBody.videoParams);
|
||
const shouldApplyAgentImageParams =
|
||
shouldRunAgentConversation &&
|
||
!smartAutoNeedsClarification &&
|
||
Boolean(smartAutoExecution?.shouldApplyImagePipeline);
|
||
const shouldApplyAgentVideoParams =
|
||
shouldRunAgentConversation &&
|
||
!smartAutoNeedsClarification &&
|
||
allowAgentVideoGeneration &&
|
||
Boolean(smartAutoExecution?.shouldApplyVideoPipeline);
|
||
const effectiveImageParams = shouldApplyAgentImageParams
|
||
? mergeImageParamsWithAgentRecommendation({
|
||
incoming: requestBody.imageParams,
|
||
recommended: smartAutoRecommendedImageParams,
|
||
intent: smartAutoDetectedIntent,
|
||
hasUserCustomSettings: hasUserCustomImageParams,
|
||
})
|
||
: requestBody.imageParams;
|
||
const effectiveVideoParams = shouldApplyAgentVideoParams
|
||
? mergeVideoParamsWithAgentRecommendation({
|
||
incoming: requestBody.videoParams,
|
||
recommended: smartAutoRecommendedVideoParams,
|
||
hasUserCustomSettings: hasUserCustomVideoParams,
|
||
})
|
||
: requestBody.videoParams;
|
||
const defaultImageQualityParams = {
|
||
aspectRatio: "auto",
|
||
imageSize: smartAutoDetectedIntent === "image_edit" ? "4K" : "2K",
|
||
imageCount: 1,
|
||
} as const;
|
||
|
||
if (shouldApplyAgentImageParams) {
|
||
console.log("[Smart Auto Mode] 🎯 Effective image params:", {
|
||
incoming: requestBody.imageParams,
|
||
recommended: smartAutoRecommendedImageParams,
|
||
merged: effectiveImageParams,
|
||
hasUserCustomImageParams,
|
||
});
|
||
}
|
||
if (shouldApplyAgentVideoParams) {
|
||
console.log("[Smart Auto Mode] 🎯 Effective video params:", {
|
||
incoming: requestBody.videoParams,
|
||
recommended: smartAutoRecommendedVideoParams,
|
||
merged: effectiveVideoParams,
|
||
hasUserCustomVideoParams,
|
||
hasVideoPlan: Boolean(smartAutoVideoPlan),
|
||
});
|
||
}
|
||
|
||
if (
|
||
shouldRunAgentConversation &&
|
||
smartAutoFriendlyMessage &&
|
||
smartAutoExecution?.shouldUseStructuredReply
|
||
) {
|
||
smartAutoFriendlyMessage = sanitizeAgentOutputSettingsSection({
|
||
message: smartAutoFriendlyMessage,
|
||
userImageParams: requestBody.imageParams,
|
||
hasUserCustomImageParams,
|
||
});
|
||
smartAutoFriendlyMessage = injectOptimizedPromptSection({
|
||
message: smartAutoFriendlyMessage,
|
||
originalText: requestTextContent,
|
||
optimizedPrompt: smartAutoOptimizedPrompt,
|
||
});
|
||
}
|
||
|
||
if (shouldRunAgentConversation) {
|
||
const currentImageUrls = extractMessageImageUrls(message.parts as any[]);
|
||
const summaryImageUrls =
|
||
currentImageUrls.length > 0 ? currentImageUrls : smartAutoHistoryImages;
|
||
const shouldRequireImageAnalysis = summaryImageUrls.length > 0;
|
||
|
||
if (shouldRequireImageAnalysis) {
|
||
try {
|
||
const { summarizeImageForExecutionReport } = await import("@/lib/ai/smart-analyzer");
|
||
const latestUserText = message.parts
|
||
.filter((part): part is { type: "text"; text: string } => part.type === "text")
|
||
.map((part) => part.text)
|
||
.join(" ");
|
||
const summaryAssetIds = Array.from(
|
||
new Set([
|
||
...extractMessageImageAssetIds(message.parts as any[]),
|
||
...((effectiveContextPayload?.selectedAssetIds || []).filter(Boolean) as string[]),
|
||
]),
|
||
);
|
||
|
||
let summary: string | null = null;
|
||
const summaryTimeoutMs = getBudgetedTimeout({
|
||
startedAt: requestStartedAt,
|
||
preferredMs: SMART_AUTO_IMAGE_SUMMARY_TIMEOUT_MS,
|
||
minimumMs: 6_000,
|
||
});
|
||
if (summaryTimeoutMs) {
|
||
summary = await withOperationTimeout(
|
||
summarizeImageForExecutionReport({
|
||
text: latestUserText,
|
||
imageUrls: summaryImageUrls,
|
||
context: effectiveAutoContextText || undefined,
|
||
preferredModelId: preferredAgentModelId,
|
||
assetIds: summaryAssetIds,
|
||
timeoutMs: summaryTimeoutMs,
|
||
}),
|
||
"enrichImageAnalysisSummary",
|
||
summaryTimeoutMs,
|
||
);
|
||
} else {
|
||
console.warn("[Smart Auto Mode] Skip image summary enrichment due to pre-response budget");
|
||
}
|
||
|
||
if (summary?.trim() && smartAutoExecution?.shouldUseStructuredReply) {
|
||
const baseMessage = ensureStructuredAgentFriendlyMessage({
|
||
message: smartAutoFriendlyMessage,
|
||
text: latestUserText,
|
||
hasImage: true,
|
||
fromHistory: currentImageUrls.length === 0 && summaryImageUrls.length > 0,
|
||
});
|
||
smartAutoFriendlyMessage = ensureImageAnalysisSummarySection(
|
||
baseMessage ||
|
||
buildAutoModeFallbackMessage({
|
||
text: latestUserText,
|
||
hasImage: true,
|
||
fromHistory: currentImageUrls.length === 0 && summaryImageUrls.length > 0,
|
||
imageSummary: summary.trim(),
|
||
}),
|
||
summary.trim(),
|
||
);
|
||
console.log(
|
||
"[Smart Auto Mode] ✅ Image analysis summary ready:",
|
||
summary.trim().slice(0, 160),
|
||
);
|
||
} else if (
|
||
summary?.trim() &&
|
||
!smartAutoExecution?.shouldUseStructuredReply &&
|
||
!smartAutoFriendlyMessage
|
||
) {
|
||
smartAutoFriendlyMessage = "收到，我会结合这张图的内容直接回答你。";
|
||
} else if (smartAutoFriendlyMessage && shouldEnrichImageAnalysisText(smartAutoFriendlyMessage)) {
|
||
console.warn("[Smart Auto Mode] ⚠️ Image summary unavailable, keep original analysis text");
|
||
}
|
||
} catch (enrichError) {
|
||
console.warn("[Smart Auto Mode] Skip image analysis enrichment:", enrichError);
|
||
}
|
||
}
|
||
}
|
||
|
||
if (
|
||
shouldRunAgentConversation &&
|
||
smartAutoFriendlyMessage &&
|
||
smartAutoExecution?.shouldUseStructuredReply
|
||
) {
|
||
smartAutoFriendlyMessage = normalizeAgentReportForDisplay(smartAutoFriendlyMessage);
|
||
}
|
||
|
||
if (
|
||
shouldRunAgentConversation &&
|
||
!shouldKeepSelectedModel &&
|
||
smartAutoExecution?.shouldStayConversational &&
|
||
preferredAgentModelId
|
||
) {
|
||
if (finalModelId !== preferredAgentModelId) {
|
||
finalModelId = preferredAgentModelId;
|
||
console.log(
|
||
`[Smart Auto Mode] 🧠 Conversational reply follows agent default model: ${configuredAgentConversationModelId || preferredAgentModelId}`,
|
||
);
|
||
}
|
||
}
|
||
|
||
if (shouldRunAgentConversation && smartAutoExecution?.shouldStayConversational) {
|
||
const candidateReplyDescriptor = await resolveAgentModelSlotDescriptor(finalModelId).catch(
|
||
() => null,
|
||
);
|
||
if (candidateReplyDescriptor) {
|
||
agentModelSlots = upsertAgentModelSlot(agentModelSlots, "reply", {
|
||
...candidateReplyDescriptor,
|
||
source: "candidate_reply_model",
|
||
reason: "post_agent_routing",
|
||
locked: shouldKeepSelectedModel,
|
||
});
|
||
|
||
const providerHealth = getProviderHealthSnapshot(candidateReplyDescriptor.providerKey);
|
||
if (
|
||
!shouldKeepSelectedModel &&
|
||
shouldAvoidProvider(providerHealth)
|
||
) {
|
||
let fallbackReplyModelId: string | undefined;
|
||
|
||
if (
|
||
preferredAgentModelId &&
|
||
preferredAgentModelId !== finalModelId
|
||
) {
|
||
const preferredDescriptor = await resolveAgentModelSlotDescriptor(
|
||
preferredAgentModelId,
|
||
).catch(() => null);
|
||
if (
|
||
preferredDescriptor &&
|
||
!shouldAvoidProvider(getProviderHealthSnapshot(preferredDescriptor.providerKey))
|
||
) {
|
||
fallbackReplyModelId = preferredAgentModelId;
|
||
}
|
||
}
|
||
|
||
if (!fallbackReplyModelId) {
|
||
try {
|
||
const { routeToModel } = await import("@/lib/ai/model-router");
|
||
const replyFallbackRoute = await routeToModel({
|
||
intent: "chat",
|
||
subIntent: undefined,
|
||
hasImage: false,
|
||
});
|
||
const fallbackDescriptor = await resolveAgentModelSlotDescriptor(
|
||
replyFallbackRoute.id,
|
||
).catch(() => null);
|
||
if (
|
||
fallbackDescriptor &&
|
||
!shouldAvoidProvider(getProviderHealthSnapshot(fallbackDescriptor.providerKey))
|
||
) {
|
||
fallbackReplyModelId = replyFallbackRoute.id;
|
||
}
|
||
} catch (replyRouteError) {
|
||
console.warn(
|
||
"[Smart Auto Mode] Failed to resolve reply fallback route after provider degradation:",
|
||
replyRouteError,
|
||
);
|
||
}
|
||
}
|
||
|
||
if (fallbackReplyModelId && fallbackReplyModelId !== finalModelId) {
|
||
finalModelId = fallbackReplyModelId;
|
||
const fallbackDescriptor = await resolveAgentModelSlotDescriptor(finalModelId).catch(
|
||
() => null,
|
||
);
|
||
if (fallbackDescriptor) {
|
||
agentModelSlots = upsertAgentModelSlot(agentModelSlots, "reply", {
|
||
...fallbackDescriptor,
|
||
source: "provider_health_fallback",
|
||
reason: providerHealth?.lastEventType || "degraded_provider",
|
||
locked: false,
|
||
});
|
||
}
|
||
console.warn(
|
||
`[Smart Auto Mode] ♻️ Switched reply model due provider health: ${candidateReplyDescriptor.displayName || candidateReplyDescriptor.modelId} -> ${fallbackDescriptor?.displayName || fallbackReplyModelId}`,
|
||
);
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
const session = await auth();
|
||
|
||
if (!session?.user) {
|
||
return new ChatSDKError("unauthorized:chat").toResponse();
|
||
}
|
||
|
||
const userType: UserType = session.user.type;
|
||
|
||
const messageCount = await getMessageCountByUserId({
|
||
id: session.user.id,
|
||
differenceInHours: 24,
|
||
});
|
||
|
||
if (messageCount > entitlementsByUserType[userType].maxMessagesPerDay) {
|
||
return new ChatSDKError("rate_limit:chat").toResponse();
|
||
}
|
||
|
||
// ========== 使用时间限制验证（后端安全检查） ==========
|
||
try {
|
||
// 从数据库获取网站设置
|
||
const { siteSettings } = await import("@/lib/db/schema");
|
||
const [settings] = await db.select().from(siteSettings).limit(1);
|
||
|
||
if (settings?.usageTimeEnabled) {
|
||
const now = new Date();
|
||
const currentTime = `${String(now.getHours()).padStart(2, "0")}:${String(now.getMinutes()).padStart(2, "0")}`;
|
||
const startTime = settings.usageTimeStart || "08:00";
|
||
const endTime = settings.usageTimeEnd || "20:00";
|
||
|
||
const isWithinUsageTime = currentTime >= startTime && currentTime <= endTime;
|
||
|
||
if (!isWithinUsageTime) {
|
||
console.log(`[Chat API] ⛔ 访问被拒绝：当前时间 ${currentTime} 不在允许范围 ${startTime}-${endTime}`);
|
||
return NextResponse.json(
|
||
{
|
||
error: "usage_time_restricted",
|
||
message: settings.usageTimeMessage || "当前不在系统使用时间段内",
|
||
currentTime,
|
||
allowedTime: `${startTime} - ${endTime}`
|
||
},
|
||
{ status: 403 }
|
||
);
|
||
}
|
||
|
||
console.log(`[Chat API] ✅ 时间验证通过：${currentTime} 在允许范围 ${startTime}-${endTime}`);
|
||
}
|
||
} catch (timeCheckError) {
|
||
console.error("[Chat API] ⚠️ 时间验证失败（继续执行）:", timeCheckError);
|
||
// 如果时间检查失败，继续执行（避免因检查逻辑错误导致系统不可用）
|
||
}
|
||
|
||
let chat = await getChatById({ id });
|
||
let messagesFromDb: DBMessage[] = [];
|
||
let messagesForMediaInput: DBMessage[] = [];
|
||
|
||
if (chat) {
|
||
if (chat.userId !== session.user.id) {
|
||
return new ChatSDKError("forbidden:chat").toResponse();
|
||
}
|
||
// Only fetch messages if chat already exists
|
||
messagesFromDb = await getMessagesByChatId({ id });
|
||
} else {
|
||
const title = await generateTitleFromUserMessage({
|
||
message,
|
||
});
|
||
|
||
try {
|
||
await saveChat({
|
||
id,
|
||
userId: session.user.id,
|
||
title,
|
||
visibility: selectedVisibilityType,
|
||
});
|
||
} catch (error) {
|
||
const racedChat = await getChatById({ id });
|
||
if (!racedChat) {
|
||
throw error;
|
||
}
|
||
|
||
if (racedChat.userId !== session.user.id) {
|
||
return new ChatSDKError("forbidden:chat").toResponse();
|
||
}
|
||
|
||
console.warn("[Chat API] saveChat conflicted, reused existing chat:", {
|
||
chatId: id,
|
||
userId: session.user.id,
|
||
});
|
||
}
|
||
|
||
chat = await getChatById({ id });
|
||
if (!chat) {
|
||
throw new ChatSDKError("bad_request:database", "Failed to save chat");
|
||
}
|
||
|
||
if (chat.userId !== session.user.id) {
|
||
return new ChatSDKError("forbidden:chat").toResponse();
|
||
}
|
||
|
||
// In a race scenario another request may have already written messages.
|
||
messagesFromDb = await getMessagesByChatId({ id });
|
||
}
|
||
|
||
const loadedMessagesForBoundary = shouldIsolateDirectComposerContext ? [] : messagesFromDb;
|
||
messagesFromDb = shouldIsolateDirectComposerContext
|
||
? []
|
||
: pruneTaskBoundaryHistoryMessages({
|
||
decision: taskBoundaryDecision,
|
||
messages: loadedMessagesForBoundary,
|
||
purpose: "model",
|
||
hasCurrentImageInput: hasImageInputForConversationAction,
|
||
});
|
||
messagesForMediaInput = shouldIsolateDirectComposerContext
|
||
? []
|
||
: pruneTaskBoundaryHistoryMessages({
|
||
decision: taskBoundaryDecision,
|
||
messages: loadedMessagesForBoundary,
|
||
purpose: "media",
|
||
hasCurrentImageInput: hasImageInputForConversationAction,
|
||
});
|
||
|
||
if (!requestWorkingState && !shouldIsolateDirectComposerContext) {
|
||
requestWorkingState = await getConversationWorkingStateByChatId({
|
||
chatId: id,
|
||
});
|
||
}
|
||
if (!requestResolvedMediaInput) {
|
||
requestResolvedMediaInput = resolveChatMediaInput({
|
||
messageParts: message.parts as any[],
|
||
historyMessages: messagesForMediaInput,
|
||
contextPayload: effectiveContextPayload,
|
||
requestOrigin,
|
||
maxImages: 4,
|
||
});
|
||
}
|
||
if (!requestTargetResolution) {
|
||
requestTargetResolution = resolveTargetAssets({
|
||
text: requestTextContent,
|
||
mediaInput: requestResolvedMediaInput,
|
||
contextPayload: effectiveContextPayload,
|
||
workingState: effectiveWorkingState || undefined,
|
||
});
|
||
}
|
||
if (
|
||
requestResolvedMediaInput.currentMessageImageUrls.length === 0 &&
|
||
smartAutoHistoryImages.length === 0 &&
|
||
requestTargetResolution.targetImageUrls.length > 0
|
||
) {
|
||
smartAutoHistoryImages = requestTargetResolution.targetImageUrls.slice(0, 4);
|
||
}
|
||
|
||
const uiMessages = [...convertToUIMessages(messagesFromDb), message];
|
||
const hiddenPromptText = typeof hiddenPrompt === "string" ? hiddenPrompt.trim() : "";
|
||
const hiddenPromptPrefix = hiddenPromptText ? `${hiddenPromptText}\n\n` : "";
|
||
const uiMessagesForModel = hiddenPromptText
|
||
? applyHiddenPromptToMessages(uiMessages, hiddenPromptText)
|
||
: uiMessages;
|
||
|
||
if (hiddenPromptText) {
|
||
const hiddenPromptLines = hiddenPromptText
|
||
.split("\n")
|
||
.map((line) => line.trim())
|
||
.filter(Boolean);
|
||
console.info("[Chat API] Hidden workflow prompt summary:", {
|
||
lineCount: hiddenPromptLines.length,
|
||
preview: hiddenPromptLines.slice(0, 12),
|
||
});
|
||
}
|
||
|
||
const { longitude, latitude, city, country } = geolocation(request);
|
||
|
||
const requestHints: RequestHints = {
|
||
longitude,
|
||
latitude,
|
||
city,
|
||
country,
|
||
};
|
||
|
||
const insertedUserMessage = await db
|
||
.insert(messageTable)
|
||
.values({
|
||
chatId: id,
|
||
id: message.id,
|
||
role: "user",
|
||
parts: message.parts as any,
|
||
attachments: [],
|
||
createdAt: new Date(),
|
||
videoStorageStatus: null,
|
||
temporaryVideoUrl: null,
|
||
permanentVideoUrl: null,
|
||
archivedAt: null,
|
||
videoMetadata: null,
|
||
})
|
||
.onConflictDoNothing({
|
||
target: [messageTable.id],
|
||
})
|
||
.returning({ id: messageTable.id });
|
||
|
||
if (insertedUserMessage.length === 0) {
|
||
console.warn(
|
||
`[Chat API] Duplicate user message ignored (chatId=${id}, messageId=${message.id})`
|
||
);
|
||
const duplicateStream = createUIMessageStream({
|
||
execute: async ({ writer }) => {
|
||
await replayAssistantMessagesForDuplicateRequest({
|
||
chatId: id,
|
||
userMessageId: message.id,
|
||
writer,
|
||
source: "duplicate-user-message",
|
||
});
|
||
},
|
||
});
|
||
return createSseResponse(duplicateStream);
|
||
}
|
||
|
||
if (requestResolvedMediaInput && requestTargetResolution) {
|
||
try {
|
||
await createMessageTargetResolution({
|
||
userId: session.user.id,
|
||
chatId: id,
|
||
messageId: message.id,
|
||
projectId: contextPayload?.projectId || null,
|
||
rawText: requestTextContent,
|
||
uploadedAssetIds: requestResolvedMediaInput.uploadedAssetIds,
|
||
uploadedImageUrls: requestResolvedMediaInput.currentMessageImageUrls,
|
||
selectedAssetIds: requestResolvedMediaInput.selectedAssetIds,
|
||
selectedImageUrls: requestResolvedMediaInput.selectedImageUrls,
|
||
candidateAssetIds: requestResolvedMediaInput.candidateAssetIds,
|
||
candidateImageUrls: requestResolvedMediaInput.imageRefs.map((ref) => ref.url),
|
||
resolvedAssetIds: requestTargetResolution.targetAssetIds,
|
||
resolvedImageUrls: requestTargetResolution.targetImageUrls,
|
||
resolutionMode: requestTargetResolution.resolutionMode,
|
||
confidence: requestTargetResolution.confidence,
|
||
needsClarification: requestTargetResolution.needsClarification,
|
||
reasoning: {
|
||
reason: requestTargetResolution.reason,
|
||
fromHistory: requestTargetResolution.fromHistory,
|
||
focusPolicy: contextPayload?.focusPolicy || null,
|
||
contextSource: contextPayload?.source || null,
|
||
},
|
||
});
|
||
|
||
await persistResolvedTargetTurn({
|
||
userId: session.user.id,
|
||
chatId: id,
|
||
projectId: contextPayload?.projectId || null,
|
||
currentIntent:
|
||
smartAutoExecution?.routeIntent ||
|
||
smartAutoDetectedIntent ||
|
||
(requestResolvedMediaInput.hasImage
|
||
? "vision"
|
||
: requestBody.videoParams
|
||
? "video_gen"
|
||
: null),
|
||
currentFocusAssetIds: requestTargetResolution.targetAssetIds,
|
||
currentFocusImageUrls: requestTargetResolution.targetImageUrls,
|
||
selectedAssetIds: requestResolvedMediaInput.selectedAssetIds,
|
||
selectedImageUrls: requestResolvedMediaInput.selectedImageUrls,
|
||
lastUploadedAssetIds: requestResolvedMediaInput.uploadedAssetIds,
|
||
lastUploadedImageUrls: requestResolvedMediaInput.currentMessageImageUrls,
|
||
lastReferencedAssetIds: requestTargetResolution.candidateAssetIds,
|
||
lastReferencedImageUrls: requestTargetResolution.candidateImageUrls,
|
||
lastResolvedTargetAssetIds: requestTargetResolution.targetAssetIds,
|
||
lastResolvedTargetImageUrls: requestTargetResolution.targetImageUrls,
|
||
activeCanvasNodeIds: contextPayload?.selectedNodeIds || [],
|
||
lastCanvasSnapshot:
|
||
Array.isArray(contextPayload?.canvasImageNodes) && contextPayload?.canvasImageNodes.length > 0
|
||
? {
|
||
source: contextPayload?.source || "canvas",
|
||
selectedNodeIds: contextPayload?.selectedNodeIds || [],
|
||
canvasImageNodes: contextPayload?.canvasImageNodes,
|
||
}
|
||
: null,
|
||
lastMessageId: message.id,
|
||
text: requestTextContent,
|
||
resolutionMode: requestTargetResolution.resolutionMode,
|
||
confidence: requestTargetResolution.confidence,
|
||
});
|
||
} catch (memoryError) {
|
||
console.warn("[Chat API] Failed to persist context-memory state:", memoryError);
|
||
}
|
||
}
|
||
|
||
const buildCurrentAgentTargetLockPayload = () => {
|
||
if (!shouldRunAgentConversation) return null;
|
||
if (composerMode !== "agent") return null;
|
||
if (effectiveContextPayload?.source !== "canvas") return null;
|
||
if (!requestResolvedMediaInput || !requestTargetResolution) return null;
|
||
|
||
return buildAgentTargetLockPayload({
|
||
requestTargetResolution,
|
||
requestResolvedMediaInput,
|
||
needsClarification: smartAutoNeedsClarification,
|
||
clarificationMessage:
|
||
smartAutoClarificationQuestion || smartAutoFriendlyMessage || undefined,
|
||
});
|
||
};
|
||
|
||
const writeCurrentAgentTargetLockChunk = (
|
||
writer: StreamWriter,
|
||
label: string,
|
||
) => {
|
||
const payload = buildCurrentAgentTargetLockPayload();
|
||
if (!payload) return;
|
||
writeAgentTargetLockChunk(writer, payload, label);
|
||
};
|
||
|
||
const streamId = generateUUID();
|
||
await createStreamId({ streamId, chatId: id });
|
||
|
||
const promptOnlySourceImageUrls =
|
||
requestResolvedMediaInput.currentMessageImageUrls.length > 0
|
||
? requestResolvedMediaInput.currentMessageImageUrls.slice(0, 4)
|
||
: requestTargetResolution?.targetImageUrls.slice(0, 4) || smartAutoHistoryImages.slice(0, 4);
|
||
const promptOnlySourceAssetIds =
|
||
requestTargetResolution?.targetAssetIds?.length
|
||
? requestTargetResolution.targetAssetIds
|
||
: requestResolvedMediaInput.uploadedAssetIds;
|
||
const wantsPromptOnlyFromImageContext =
|
||
shouldRunAgentConversation &&
|
||
!smartAutoNeedsClarification &&
|
||
promptOnlySourceImageUrls.length > 0 &&
|
||
taskBoundaryDecision.taskType === "prompt_only";
|
||
|
||
if (wantsPromptOnlyFromImageContext) {
|
||
let promptOnlyImageSummary = "";
|
||
try {
|
||
const { summarizeImageForExecutionReport } = await import("@/lib/ai/smart-analyzer");
|
||
const summaryTimeoutMs = getBudgetedTimeout({
|
||
startedAt: requestStartedAt,
|
||
preferredMs: SMART_AUTO_IMAGE_SUMMARY_TIMEOUT_MS,
|
||
minimumMs: 6_000,
|
||
});
|
||
if (summaryTimeoutMs) {
|
||
const summaryAssetIds = Array.from(
|
||
new Set([
|
||
...extractMessageImageAssetIds(message.parts as any[]),
|
||
...((effectiveContextPayload?.selectedAssetIds || []).filter(Boolean) as string[]),
|
||
]),
|
||
);
|
||
const summary = await withOperationTimeout(
|
||
summarizeImageForExecutionReport({
|
||
text: requestTextContent,
|
||
imageUrls: promptOnlySourceImageUrls,
|
||
context: effectiveAutoContextText || undefined,
|
||
preferredModelId: preferredAgentModelId,
|
||
assetIds:
|
||
promptOnlySourceAssetIds.length > 0 ? promptOnlySourceAssetIds : summaryAssetIds,
|
||
timeoutMs: summaryTimeoutMs,
|
||
}),
|
||
"promptOnlyImageSummary",
|
||
summaryTimeoutMs,
|
||
);
|
||
if (summary?.trim()) {
|
||
promptOnlyImageSummary = summary.trim();
|
||
}
|
||
}
|
||
} catch (promptOnlySummaryError) {
|
||
console.warn("[Chat API] Prompt-only image summary failed:", promptOnlySummaryError);
|
||
}
|
||
|
||
let promptOnlyOptimizedPrompt = smartAutoOptimizedPrompt?.trim() || "";
|
||
if (!promptOnlyOptimizedPrompt) {
|
||
try {
|
||
const { optimizePromptWithLLM, optimizePrompt } = await import("@/lib/ai/prompt-optimizer");
|
||
const promptSeed = buildPromptOnlyOptimizationSeed({
|
||
text: requestTextContent,
|
||
imageSummary: promptOnlyImageSummary,
|
||
});
|
||
const optimizedResult = await optimizePromptWithLLM({
|
||
originalPrompt: promptSeed,
|
||
intent: "image_gen",
|
||
historyMessages: smartAutoHistoryMessagesFromDb
|
||
.map((msg) => {
|
||
const parts = Array.isArray(msg.parts) ? (msg.parts as any[]) : [];
|
||
const content = parts
|
||
.filter((part: any) => part?.type === "text")
|
||
.map((part: any) => String(part?.text || ""))
|
||
.join(" ")
|
||
.trim();
|
||
if (!content) return null;
|
||
return {
|
||
role: msg.role as "user" | "assistant",
|
||
content: content.slice(0, 300),
|
||
};
|
||
})
|
||
.filter(Boolean) as Array<{ role: "user" | "assistant"; content: string }>,
|
||
context: effectiveAutoContextText || undefined,
|
||
}).catch(() =>
|
||
optimizePrompt({
|
||
originalPrompt: promptSeed,
|
||
intent: "image_gen",
|
||
}),
|
||
);
|
||
promptOnlyOptimizedPrompt = String(optimizedResult?.optimizedPrompt || "").trim();
|
||
} catch (promptOnlyOptimizeError) {
|
||
console.warn("[Chat API] Prompt-only optimization failed:", promptOnlyOptimizeError);
|
||
}
|
||
}
|
||
|
||
if (!promptOnlyOptimizedPrompt) {
|
||
promptOnlyOptimizedPrompt = buildPromptOnlyOptimizationSeed({
|
||
text: requestTextContent,
|
||
imageSummary: promptOnlyImageSummary,
|
||
});
|
||
}
|
||
|
||
const promptOnlyMessage = buildPromptOnlyReply({
|
||
userText: requestTextContent,
|
||
imageSummary: promptOnlyImageSummary,
|
||
optimizedPrompt: promptOnlyOptimizedPrompt,
|
||
});
|
||
|
||
const promptOnlyStream = createUIMessageStream({
|
||
execute: async ({ writer }) => {
|
||
const messageId = generateUUID();
|
||
writeAgentProgressChunk(
|
||
writer,
|
||
{
|
||
stage: "planning",
|
||
currentStep: "analyze",
|
||
title: "正在提炼可执行提示词",
|
||
message: "我在结合当前目标图和上下文整理更稳定的提示词。",
|
||
toolName: "promptOnly",
|
||
capability: "prompt_only",
|
||
},
|
||
"prompt-only:progress-planning",
|
||
);
|
||
writeCurrentAgentTargetLockChunk(writer, "prompt-only:target-lock");
|
||
writeAssistantTextChunk(writer, promptOnlyMessage, "prompt-only:reply", messageId);
|
||
|
||
await saveMessages({
|
||
messages: [
|
||
{
|
||
id: messageId,
|
||
chatId: id,
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: promptOnlyMessage }],
|
||
attachments: [],
|
||
createdAt: new Date(),
|
||
videoStorageStatus: null,
|
||
temporaryVideoUrl: null,
|
||
permanentVideoUrl: null,
|
||
archivedAt: null,
|
||
videoMetadata: null,
|
||
},
|
||
],
|
||
});
|
||
|
||
try {
|
||
await persistPromptOnlyTurn({
|
||
userId: session.user.id,
|
||
chatId: id,
|
||
projectId: contextPayload?.projectId || null,
|
||
messageId,
|
||
text: requestTextContent,
|
||
optimizedPrompt: promptOnlyOptimizedPrompt,
|
||
imageSummary: promptOnlyImageSummary,
|
||
sourceImageUrls: promptOnlySourceImageUrls,
|
||
});
|
||
} catch (promptOnlyMemoryError) {
|
||
console.warn("[Chat API] Failed to persist prompt-only response state:", promptOnlyMemoryError);
|
||
}
|
||
|
||
writeAgentProgressChunk(
|
||
writer,
|
||
{
|
||
stage: "completed",
|
||
currentStep: "finalize",
|
||
title: "提示词整理完成",
|
||
message: "已经给出可直接复用的优化提示词。",
|
||
toolName: "promptOnly",
|
||
capability: "prompt_only",
|
||
},
|
||
"prompt-only:progress-completed",
|
||
);
|
||
},
|
||
});
|
||
|
||
return createSseResponse(promptOnlyStream);
|
||
}
|
||
|
||
if (shouldRunAgentConversation && smartAutoNeedsClarification) {
|
||
const clarificationBypass = resolveClarificationBypass({
|
||
text: message.parts
|
||
.filter((part): part is { type: "text"; text: string } => part.type === "text")
|
||
.map((part) => part.text)
|
||
.join(" ")
|
||
.trim(),
|
||
hasCurrentImageInput: message.parts.some((part: any) =>
|
||
part?.type === "file" || part?.type === "image" || part?.type === "image_url"
|
||
),
|
||
hasHistoryImageInput: smartAutoHistoryImages.length > 0,
|
||
hasCanvasSelection: Boolean(effectiveContextPayload?.selectedNodeIds?.length),
|
||
targetResolutionMode: requestTargetResolution?.resolutionMode,
|
||
executionMode: smartAutoExecution?.mode,
|
||
friendlyMessage: smartAutoFriendlyMessage,
|
||
});
|
||
|
||
if (clarificationBypass.shouldBypass) {
|
||
smartAutoNeedsClarification = false;
|
||
smartAutoClarificationQuestion = undefined;
|
||
smartAutoFriendlyMessage = clarificationBypass.nextFriendlyMessage;
|
||
console.log("[Smart Auto Mode] ✅ Force direct execution (skip clarification)");
|
||
}
|
||
}
|
||
|
||
if (shouldRunAgentConversation) {
|
||
const canvasClarificationGate = resolveCanvasAgentClarificationGate({
|
||
text: requestTextContent,
|
||
composerMode,
|
||
contextPayload: effectiveContextPayload,
|
||
mediaInput: requestResolvedMediaInput,
|
||
targetResolution: requestTargetResolution,
|
||
taskBoundaryDecision,
|
||
executionMode: smartAutoExecution?.mode,
|
||
actionKind: resolvedConversationAction.kind,
|
||
});
|
||
|
||
if (canvasClarificationGate.shouldClarify) {
|
||
smartAutoNeedsClarification = true;
|
||
smartAutoClarificationQuestion =
|
||
canvasClarificationGate.clarificationMessage || smartAutoClarificationQuestion;
|
||
smartAutoFriendlyMessage =
|
||
canvasClarificationGate.clarificationMessage || smartAutoFriendlyMessage;
|
||
console.log("[Smart Auto Mode] 🧭 Canvas clarification gate triggered:", {
|
||
reason: canvasClarificationGate.reason,
|
||
executionMode: smartAutoExecution?.mode,
|
||
resolutionMode: requestTargetResolution?.resolutionMode,
|
||
selectedCount: requestResolvedMediaInput?.selectedImageRefs.length || 0,
|
||
candidateCount: requestTargetResolution?.candidateRefs.length || 0,
|
||
});
|
||
}
|
||
}
|
||
|
||
if (shouldRunAgentConversation && smartAutoNeedsClarification) {
|
||
const clarificationMessage = resolveClarificationMessage({
|
||
clarificationQuestion: smartAutoClarificationQuestion,
|
||
friendlyMessage: smartAutoFriendlyMessage,
|
||
});
|
||
|
||
const clarificationStream = createUIMessageStream({
|
||
execute: async ({ writer }) => {
|
||
const messageId = generateUUID();
|
||
writeAgentProgressChunk(
|
||
writer,
|
||
{
|
||
stage: "clarify",
|
||
currentStep: "lock_target",
|
||
title: "等待你确认关键信息",
|
||
message:
|
||
smartAutoClarificationQuestion ||
|
||
smartAutoFriendlyMessage ||
|
||
"我需要先确认目标图或执行范围，避免误处理。",
|
||
toolName: "clarification",
|
||
capability: smartAutoExecution?.preferredCapability || undefined,
|
||
},
|
||
"clarification:progress",
|
||
);
|
||
writeCurrentAgentTargetLockChunk(writer, "clarification:target-lock");
|
||
writer.write({
|
||
type: "data-appendMessage",
|
||
data: JSON.stringify({
|
||
id: messageId,
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: clarificationMessage }],
|
||
}),
|
||
});
|
||
|
||
await saveMessages({
|
||
messages: [
|
||
{
|
||
id: messageId,
|
||
chatId: id,
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: clarificationMessage }],
|
||
attachments: [],
|
||
createdAt: new Date(),
|
||
videoStorageStatus: null,
|
||
temporaryVideoUrl: null,
|
||
permanentVideoUrl: null,
|
||
archivedAt: null,
|
||
videoMetadata: null,
|
||
},
|
||
],
|
||
});
|
||
|
||
try {
|
||
await persistClarificationTurn({
|
||
userId: session.user.id,
|
||
chatId: id,
|
||
projectId: contextPayload?.projectId || null,
|
||
messageId,
|
||
text: requestTextContent,
|
||
clarificationMessage,
|
||
currentIntent: smartAutoDetectedIntent || null,
|
||
pendingGridConfirmation:
|
||
shouldAskGridConfirmation && pendingGridConfirmation
|
||
? pendingGridConfirmation
|
||
: null,
|
||
});
|
||
} catch (clarificationPersistError) {
|
||
console.warn(
|
||
"[Chat API] Failed to persist clarification state:",
|
||
clarificationPersistError
|
||
);
|
||
}
|
||
},
|
||
});
|
||
|
||
return createSseResponse(clarificationStream);
|
||
}
|
||
|
||
if (
|
||
shouldRunAgentConversation &&
|
||
smartAutoWorkflowDecision?.workflowType === "storyboard_video" &&
|
||
(smartAutoWorkflowDecision?.action === "revision_requested" ||
|
||
smartAutoWorkflowDecision?.action === "cancel_requested")
|
||
) {
|
||
const workflowReplyStream = createUIMessageStream({
|
||
execute: async ({ writer }) => {
|
||
const messageId = generateUUID();
|
||
const workflowFollowupResult = await handleStoryboardWorkflowFollowup({
|
||
action: smartAutoWorkflowDecision.action as "revision_requested" | "cancel_requested",
|
||
workflowId: smartAutoWorkflowDecision.workflowId,
|
||
text: requestTextContent,
|
||
friendlyMessage: smartAutoFriendlyMessage,
|
||
onProgress: (event) => {
|
||
safeWriteStreamChunk(
|
||
writer,
|
||
{
|
||
type: "data-videoStoryboard",
|
||
data: JSON.stringify(event),
|
||
},
|
||
`storyboard-revision:${event.stage}`,
|
||
);
|
||
},
|
||
});
|
||
const replyText = workflowFollowupResult.replyText;
|
||
|
||
writer.write({
|
||
type: "data-appendMessage",
|
||
data: JSON.stringify({
|
||
id: messageId,
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: replyText }],
|
||
attachments: [],
|
||
}),
|
||
});
|
||
|
||
await saveMessages({
|
||
messages: [
|
||
{
|
||
id: messageId,
|
||
chatId: id,
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: replyText }],
|
||
attachments: [],
|
||
createdAt: new Date(),
|
||
videoStorageStatus: null,
|
||
temporaryVideoUrl: null,
|
||
permanentVideoUrl: null,
|
||
archivedAt: null,
|
||
videoMetadata: workflowFollowupResult.videoMetadata,
|
||
},
|
||
],
|
||
});
|
||
|
||
try {
|
||
if (
|
||
smartAutoWorkflowDecision?.action === "revision_requested" &&
|
||
smartAutoWorkflowDecision.workflowId
|
||
) {
|
||
await persistStoryboardRevisionTurn({
|
||
userId: session.user.id,
|
||
chatId: id,
|
||
projectId: contextPayload?.projectId || null,
|
||
messageId,
|
||
text: requestTextContent,
|
||
taskId: smartAutoWorkflowDecision.workflowId,
|
||
status: workflowFollowupResult.nextWorkflowStatus,
|
||
});
|
||
} else if (smartAutoWorkflowDecision?.action === "cancel_requested") {
|
||
await persistWorkflowCancelledTurn({
|
||
userId: session.user.id,
|
||
chatId: id,
|
||
projectId: contextPayload?.projectId || null,
|
||
messageId,
|
||
text: requestTextContent,
|
||
workflowType: "storyboard_video",
|
||
workflowId: smartAutoWorkflowDecision?.workflowId,
|
||
status: workflowFollowupResult.nextWorkflowStatus,
|
||
});
|
||
}
|
||
} catch (workflowPersistError) {
|
||
console.warn(
|
||
"[Chat API] Failed to persist storyboard workflow interruption state:",
|
||
workflowPersistError
|
||
);
|
||
}
|
||
},
|
||
});
|
||
|
||
return createSseResponse(workflowReplyStream);
|
||
}
|
||
|
||
if (
|
||
shouldRunAgentConversation &&
|
||
smartAutoWorkflowDecision?.workflowType === "storyboard_video" &&
|
||
smartAutoWorkflowDecision?.action === "confirmed" &&
|
||
smartAutoWorkflowDecision.workflowId
|
||
) {
|
||
const confirmedTaskId = smartAutoWorkflowDecision.workflowId;
|
||
const confirmedStoryboardStream = createUIMessageStream({
|
||
execute: async ({ writer }) => {
|
||
const acknowledgeMessageId = generateUUID();
|
||
const { acknowledgeMessage, acknowledgeMetadata } =
|
||
buildStoryboardConfirmedAck(confirmedTaskId);
|
||
|
||
writer.write({
|
||
type: "data-appendMessage",
|
||
data: JSON.stringify({
|
||
id: acknowledgeMessageId,
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: acknowledgeMessage }],
|
||
attachments: [],
|
||
}),
|
||
});
|
||
|
||
await saveMessages({
|
||
messages: [
|
||
{
|
||
id: acknowledgeMessageId,
|
||
chatId: id,
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: acknowledgeMessage }],
|
||
attachments: [],
|
||
createdAt: new Date(),
|
||
videoStorageStatus: null,
|
||
temporaryVideoUrl: null,
|
||
permanentVideoUrl: null,
|
||
archivedAt: null,
|
||
videoMetadata: acknowledgeMetadata,
|
||
},
|
||
],
|
||
});
|
||
|
||
try {
|
||
await persistStoryboardConfirmedTurn({
|
||
userId: session.user.id,
|
||
chatId: id,
|
||
projectId: contextPayload?.projectId || null,
|
||
messageId: acknowledgeMessageId,
|
||
text: requestTextContent,
|
||
taskId: confirmedTaskId,
|
||
status: "confirmed",
|
||
});
|
||
} catch (storyboardConfirmMemoryError) {
|
||
console.warn(
|
||
"[Chat API] Failed to persist storyboard confirmation memory:",
|
||
storyboardConfirmMemoryError
|
||
);
|
||
}
|
||
|
||
try {
|
||
await persistStoryboardExecutionStartedTurn({
|
||
userId: session.user.id,
|
||
chatId: id,
|
||
projectId: contextPayload?.projectId || null,
|
||
messageId: acknowledgeMessageId,
|
||
text: requestTextContent,
|
||
taskId: confirmedTaskId,
|
||
status: "running",
|
||
});
|
||
} catch (storyboardExecutionStartError) {
|
||
console.warn(
|
||
"[Chat API] Failed to persist storyboard execution start state:",
|
||
storyboardExecutionStartError
|
||
);
|
||
}
|
||
|
||
const { executionResult, firstVideoUrl, finalVideoMetadata } =
|
||
await executeStoryboardConfirmedWorkflow({
|
||
taskId: confirmedTaskId,
|
||
onProgress: (event) => {
|
||
safeWriteStreamChunk(
|
||
writer,
|
||
{
|
||
type: "data-videoStoryboard",
|
||
data: JSON.stringify(event),
|
||
},
|
||
`storyboard-confirmed:${event.stage}`,
|
||
);
|
||
},
|
||
});
|
||
|
||
const finalMessageId = generateUUID();
|
||
writer.write({
|
||
type: "data-appendMessage",
|
||
data: JSON.stringify({
|
||
id: finalMessageId,
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: executionResult.markdown }],
|
||
attachments: [],
|
||
}),
|
||
});
|
||
|
||
await saveMessages({
|
||
messages: [
|
||
{
|
||
id: finalMessageId,
|
||
chatId: id,
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: executionResult.markdown }],
|
||
attachments: [],
|
||
createdAt: new Date(),
|
||
videoStorageStatus: firstVideoUrl ? "temporary" : null,
|
||
temporaryVideoUrl: firstVideoUrl || null,
|
||
permanentVideoUrl: null,
|
||
archivedAt: null,
|
||
videoMetadata: finalVideoMetadata,
|
||
},
|
||
],
|
||
});
|
||
|
||
await attachVideoStoryboardTaskMessage({
|
||
taskId: executionResult.taskId,
|
||
messageId: finalMessageId,
|
||
}).catch((taskAttachError) => {
|
||
console.warn(
|
||
"[Chat API] Failed to attach confirmed storyboard task with message:",
|
||
taskAttachError
|
||
);
|
||
});
|
||
|
||
try {
|
||
await persistStoryboardExecutionCompletedTurn({
|
||
userId: session.user.id,
|
||
chatId: id,
|
||
projectId: contextPayload?.projectId || null,
|
||
messageId: finalMessageId,
|
||
text: requestTextContent,
|
||
taskId: executionResult.taskId,
|
||
status: executionResult.status,
|
||
successCount: executionResult.successCount,
|
||
failedCount: executionResult.failedCount,
|
||
firstVideoUrl: firstVideoUrl || undefined,
|
||
});
|
||
} catch (storyboardExecutionCompletedError) {
|
||
console.warn(
|
||
"[Chat API] Failed to persist storyboard execution completion state:",
|
||
storyboardExecutionCompletedError
|
||
);
|
||
}
|
||
|
||
if (firstVideoUrl) {
|
||
await consumeGeneratedVideoPoints({
|
||
userId: session.user.id,
|
||
chatId: id,
|
||
messageId: finalMessageId,
|
||
modelRefs: executionResult.modelRefs,
|
||
duration: Number((finalVideoMetadata as any)?.duration) || undefined,
|
||
});
|
||
|
||
scheduleVideoArchive({
|
||
chatId: id,
|
||
messageId: finalMessageId,
|
||
text: executionResult.markdown,
|
||
temporaryVideoUrl: firstVideoUrl,
|
||
taskId: finalMessageId,
|
||
metadata: {
|
||
storyboard: {
|
||
enabled: true,
|
||
taskId: executionResult.taskId,
|
||
successCount: executionResult.successCount,
|
||
failedCount: executionResult.failedCount,
|
||
},
|
||
status: executionResult.status,
|
||
updatedAt: Date.now(),
|
||
},
|
||
});
|
||
}
|
||
},
|
||
});
|
||
|
||
return createSseResponse(confirmedStoryboardStream);
|
||
}
|
||
|
||
if (agentPresetId) {
|
||
const selectedAgentPreset = await getAccessibleAgentPresetById({
|
||
id: agentPresetId,
|
||
userId: session.user.id,
|
||
});
|
||
|
||
if (!selectedAgentPreset) {
|
||
return NextResponse.json(
|
||
{
|
||
code: "not_found:chat",
|
||
message: "智能体不存在或无权使用",
|
||
},
|
||
{ status: 404 },
|
||
);
|
||
}
|
||
|
||
if (selectedAgentPreset.type === "text") {
|
||
// 生文智能体走常规多模态对话链路：
|
||
// - 模型已由前端切到智能体预设模型
|
||
// - 提示词通过 hiddenPrompt 注入
|
||
// - 图片输入沿用普通多模态附件
|
||
} else {
|
||
|
||
const orderedSlots = [...(selectedAgentPreset.slots || [])].sort(
|
||
(left, right) => left.slotIndex - right.slotIndex,
|
||
);
|
||
const presetModeConfig = normalizeAgentPresetModeConfig(selectedAgentPreset.defaultParams);
|
||
const isSceneGenerationPreset =
|
||
selectedAgentPreset.type === "image" &&
|
||
presetModeConfig.modeType === "scene_generation";
|
||
const isDetailReplacementPreset =
|
||
selectedAgentPreset.type === "image" &&
|
||
presetModeConfig.modeType === "detail_replacement";
|
||
const isSkuReplacementPreset =
|
||
selectedAgentPreset.type === "image" &&
|
||
presetModeConfig.modeType === "sku_replacement";
|
||
const sceneProductSlotCount = isSceneGenerationPreset
|
||
? Math.max(1, presetModeConfig.scene.productImageSlotCount || 1)
|
||
: 0;
|
||
const sceneModelSlotEnabled = isSceneGenerationPreset
|
||
? presetModeConfig.scene.includeModelSlot
|
||
: false;
|
||
const replacementProductSlotCount = isDetailReplacementPreset
|
||
? presetModeConfig.detail.productImageSlotCount
|
||
: isSkuReplacementPreset
|
||
? presetModeConfig.sku.productImageSlotCount
|
||
: 0;
|
||
const replacementModelSlotEnabled = isDetailReplacementPreset
|
||
? presetModeConfig.detail.includeModelSlot
|
||
: isSkuReplacementPreset
|
||
? presetModeConfig.sku.includeModelSlot
|
||
: false;
|
||
const replacementReferences = normalizeAgentModeReferences(agentModePayload);
|
||
const isMatchingReplacementPayload =
|
||
(isDetailReplacementPreset && agentModePayload?.modeType === "detail_replacement") ||
|
||
(isSkuReplacementPreset && agentModePayload?.modeType === "sku_replacement");
|
||
if (orderedSlots.length === 0) {
|
||
return NextResponse.json(
|
||
{
|
||
code: "bad_request:api",
|
||
message: "当前智能体未配置可执行槽位",
|
||
},
|
||
{ status: 400 },
|
||
);
|
||
}
|
||
if ((isDetailReplacementPreset || isSkuReplacementPreset) && !isMatchingReplacementPayload) {
|
||
return NextResponse.json(
|
||
{
|
||
code: "bad_request:api",
|
||
message: "当前模式缺少执行上下文，请返回第二步重新选择参考图后再试",
|
||
},
|
||
{ status: 400 },
|
||
);
|
||
}
|
||
if ((isDetailReplacementPreset || isSkuReplacementPreset) && replacementReferences.length === 0) {
|
||
return NextResponse.json(
|
||
{
|
||
code: "bad_request:api",
|
||
message: isDetailReplacementPreset
|
||
? "详情替换模式至少需要 1 张详情参考图"
|
||
: "SKU替换模式至少需要 1 张 SKU 参考图",
|
||
},
|
||
{ status: 400 },
|
||
);
|
||
}
|
||
|
||
const uploadedImageParts = extractCurrentMessageImageParts(message.parts as any[]);
|
||
const requiredUploadCount = getAgentRequiredUploadCount({
|
||
presetType: selectedAgentPreset.type,
|
||
defaultParams: selectedAgentPreset.defaultParams || null,
|
||
fallback: 3,
|
||
});
|
||
|
||
if (uploadedImageParts.length < requiredUploadCount) {
|
||
return NextResponse.json(
|
||
{
|
||
code: "bad_request:api",
|
||
message: `当前智能体需要上传 ${requiredUploadCount} 张图片`,
|
||
},
|
||
{ status: 400 },
|
||
);
|
||
}
|
||
|
||
const presetModelId = String(selectedAgentPreset.defaultModelId || "").trim();
|
||
if (!presetModelId) {
|
||
return NextResponse.json(
|
||
{
|
||
code: "bad_request:api",
|
||
message: "当前智能体未配置默认模型",
|
||
},
|
||
{ status: 400 },
|
||
);
|
||
}
|
||
|
||
const sanitizedUserSupplementPrompt = normalizeAgentSupplementPrompt(
|
||
requestTextContent,
|
||
selectedAgentPreset.name,
|
||
);
|
||
|
||
const presetSnapshot = await createAgentPresetExecutionSnapshot({
|
||
presetId: selectedAgentPreset.id,
|
||
userId: session.user.id,
|
||
projectId: contextPayload?.projectId || null,
|
||
chatId: id,
|
||
messageId: message.id,
|
||
presetName: selectedAgentPreset.name,
|
||
presetType: selectedAgentPreset.type,
|
||
visibility: selectedAgentPreset.visibility,
|
||
categoryId: selectedAgentPreset.category?.id || selectedAgentPreset.categoryId || null,
|
||
categoryName: selectedAgentPreset.category?.name || null,
|
||
promptTemplate: selectedAgentPreset.promptTemplate,
|
||
defaultModelId: selectedAgentPreset.defaultModelId || null,
|
||
resolvedModelId: presetModelId,
|
||
defaultParams: selectedAgentPreset.defaultParams || null,
|
||
slotCount: orderedSlots.length,
|
||
requiredUploadCount,
|
||
slots: orderedSlots.map((slot) => ({
|
||
slotIndex: slot.slotIndex,
|
||
title: slot.title || null,
|
||
promptTemplate: slot.promptTemplate,
|
||
defaultParams: slot.defaultParams || null,
|
||
requireUpload: slot.requireUpload !== false,
|
||
uploadLabel: slot.uploadLabel || null,
|
||
referenceAssetIds: slot.referenceAssetIds || [],
|
||
referenceImageUrls: (slot.referenceAssets || [])
|
||
.map((asset) => String(asset.originalUrl || "").trim())
|
||
.filter(Boolean),
|
||
})),
|
||
referenceAssetIds: orderedSlots.flatMap((slot) => slot.referenceAssetIds || []),
|
||
referenceImageUrls: orderedSlots.flatMap((slot) =>
|
||
(slot.referenceAssets || [])
|
||
.map((asset) => String(asset.originalUrl || "").trim())
|
||
.filter(Boolean),
|
||
),
|
||
userInputText: sanitizedUserSupplementPrompt || null,
|
||
requestPayload: {
|
||
selectedChatModel,
|
||
agentPresetId,
|
||
requiredUploadCount,
|
||
uploadedImageUrls: uploadedImageParts.map((part) => part.url),
|
||
agentModePayload:
|
||
isDetailReplacementPreset || isSkuReplacementPreset
|
||
? {
|
||
modeType: agentModePayload?.modeType || null,
|
||
query: agentModePayload?.query || null,
|
||
queryType: agentModePayload?.queryType || null,
|
||
selectedCandidateId: agentModePayload?.selectedCandidateId || null,
|
||
referenceCount: replacementReferences.length,
|
||
}
|
||
: null,
|
||
},
|
||
});
|
||
|
||
try {
|
||
await db
|
||
.update(agentPresetTable)
|
||
.set({
|
||
usageCount: sql`${agentPresetTable.usageCount} + 1`,
|
||
lastUsedAt: new Date(),
|
||
updatedAt: new Date(),
|
||
})
|
||
.where(eq(agentPresetTable.id, selectedAgentPreset.id));
|
||
} catch (agentUsageError) {
|
||
console.warn("[Chat API] Failed to update agent preset usage stats:", agentUsageError);
|
||
}
|
||
|
||
const agentSequenceStream = createUIMessageStream({
|
||
execute: async ({ writer }) => {
|
||
if (selectedAgentPreset.type === "image") {
|
||
writeAssistantTextChunk(
|
||
writer,
|
||
"<!-- IMAGE_GENERATION_START -->",
|
||
"agent-preset:image:start",
|
||
);
|
||
if (contextPayload?.source === "canvas") {
|
||
writeImageGenerationStatusChunk(
|
||
writer,
|
||
{
|
||
phase: "processing",
|
||
modelName:
|
||
selectedAgentPreset.defaultModel?.displayName ||
|
||
selectedAgentPreset.name,
|
||
imageCount: isSceneGenerationPreset
|
||
? Math.max(1, effectiveImageParams?.imageCount || 1)
|
||
: isDetailReplacementPreset || isSkuReplacementPreset
|
||
? replacementReferences.length
|
||
: orderedSlots.length,
|
||
},
|
||
"agent-preset:image:status-processing",
|
||
);
|
||
}
|
||
}
|
||
|
||
let successCount = 0;
|
||
const sharedUploadedImageUrls = uploadedImageParts
|
||
.map((part) => part.url)
|
||
.filter((url) => /^https?:\/\//i.test(url));
|
||
const sharedUploadedAssetIds = uploadedImageParts
|
||
.map((part) => part.assetId)
|
||
.filter((assetId): assetId is string => Boolean(assetId));
|
||
|
||
if (
|
||
selectedAgentPreset.type === "image" &&
|
||
(isDetailReplacementPreset || isSkuReplacementPreset)
|
||
) {
|
||
const replacementSlot = orderedSlots[0];
|
||
const replacementTemplateReferenceImageUrls = Array.from(
|
||
new Set(
|
||
(replacementSlot?.referenceAssets || [])
|
||
.map((asset) => String(asset.originalUrl || "").trim())
|
||
.filter(Boolean),
|
||
),
|
||
);
|
||
const productInputImageUrls = sharedUploadedImageUrls.slice(
|
||
0,
|
||
Math.max(1, replacementProductSlotCount)
|
||
);
|
||
const modelInputImageUrl = replacementModelSlotEnabled
|
||
? sharedUploadedImageUrls[Math.max(0, replacementProductSlotCount)] || null
|
||
: null;
|
||
const sourceAssetIds = Array.from(
|
||
new Set(
|
||
[
|
||
...sharedUploadedAssetIds,
|
||
...(replacementSlot?.referenceAssetIds || []),
|
||
].filter(Boolean),
|
||
),
|
||
);
|
||
const baseReplacementPrompt = buildAgentSlotPrompt({
|
||
workflowPrompt: hiddenPromptText || null,
|
||
slotPrompt:
|
||
replacementSlot?.promptTemplate || selectedAgentPreset.promptTemplate,
|
||
slotTitle: selectedAgentPreset.name,
|
||
userSupplementPrompt: sanitizedUserSupplementPrompt || null,
|
||
includeSlotTitle: false,
|
||
modeType: isDetailReplacementPreset
|
||
? "detail_replacement"
|
||
: isSkuReplacementPreset
|
||
? "sku_replacement"
|
||
: "generic",
|
||
});
|
||
|
||
if (productInputImageUrls.length === 0) {
|
||
throw new Error("替换模式至少需要 1 张输入商品图");
|
||
}
|
||
|
||
const mergedImageParams = {
|
||
...(selectedAgentPreset.defaultParams || {}),
|
||
...(replacementSlot?.defaultParams || {}),
|
||
} as Record<string, any>;
|
||
const resolvedModelRef =
|
||
String(selectedChatModel || "").trim() &&
|
||
String(selectedChatModel || "").trim() !== "auto"
|
||
? String(selectedChatModel || "").trim()
|
||
: presetModelId;
|
||
const configuredReplacementAspectRatio =
|
||
typeof mergedImageParams.aspectRatio === "string"
|
||
? mergedImageParams.aspectRatio
|
||
: effectiveImageParams?.aspectRatio || defaultImageQualityParams.aspectRatio;
|
||
const resolvedAspectRatio = "auto";
|
||
const resolvedImageSize =
|
||
typeof mergedImageParams.imageSize === "string"
|
||
? mergedImageParams.imageSize
|
||
: effectiveImageParams?.imageSize || defaultImageQualityParams.imageSize;
|
||
let resolvedModelDisplayName =
|
||
selectedAgentPreset.defaultModel?.displayName || selectedAgentPreset.name;
|
||
if (
|
||
configuredReplacementAspectRatio &&
|
||
configuredReplacementAspectRatio !== "auto"
|
||
) {
|
||
console.info(
|
||
"[Chat API] Replacement mode forced aspect ratio to auto to preserve reference canvas:",
|
||
{
|
||
configuredAspectRatio: configuredReplacementAspectRatio,
|
||
},
|
||
);
|
||
}
|
||
const replacementVisionModels = await getEnabledAiModelsWithProvider().catch(
|
||
(error) => {
|
||
console.warn(
|
||
"[Chat API] Failed to load configured vision models for replacement analysis:",
|
||
error,
|
||
);
|
||
return undefined;
|
||
},
|
||
);
|
||
const replacementScenePlanningRuntime =
|
||
await resolveReplacementScenePlanningRuntime({
|
||
models: replacementVisionModels,
|
||
});
|
||
const replacementScenePlanningTimeoutMs =
|
||
replacementScenePlanningRuntime?.kind === "local_moonshot"
|
||
? 90_000
|
||
: 45_000;
|
||
console.info("[Chat API] Replacement scene planning runtime:", {
|
||
kind: replacementScenePlanningRuntime?.kind || "none",
|
||
modelName: replacementScenePlanningRuntime?.modelName || null,
|
||
timeoutMs: replacementScenePlanningTimeoutMs,
|
||
});
|
||
const stopKeepalive = startStreamKeepalive(
|
||
writer,
|
||
"agent-preset:replacement",
|
||
12_000,
|
||
);
|
||
const emitReplacementResultMessage = async (params: {
|
||
reference: (typeof replacementReferences)[number];
|
||
bodyText?: string | null;
|
||
image: {
|
||
originalUrl: string;
|
||
thumbnailUrl?: string | null;
|
||
};
|
||
imageSizeLabel?: string | null;
|
||
modelDisplayName?: string | null;
|
||
sourceImageUrls: string[];
|
||
sourceType: "detail_replacement" | "sku_replacement";
|
||
streamLabel: string;
|
||
}) => {
|
||
const persistedImageSize =
|
||
String(params.imageSizeLabel || "").trim() || resolvedImageSize;
|
||
const assistantText = prependImageModelBadge(
|
||
[
|
||
`### ${params.reference.label}`,
|
||
String(params.bodyText || "").trim(),
|
||
buildAgentToolImagesMarkdown([
|
||
{
|
||
originalUrl: params.image.originalUrl,
|
||
thumbnailUrl: params.image.thumbnailUrl,
|
||
imageSize: persistedImageSize,
|
||
},
|
||
]),
|
||
]
|
||
.filter(Boolean)
|
||
.join("\n\n"),
|
||
params.modelDisplayName || null,
|
||
);
|
||
const assistantMessageId = generateUUID();
|
||
|
||
await saveMessages({
|
||
messages: [
|
||
{
|
||
id: assistantMessageId,
|
||
chatId: id,
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: assistantText }],
|
||
attachments: [],
|
||
createdAt: new Date(),
|
||
videoStorageStatus: null,
|
||
temporaryVideoUrl: null,
|
||
permanentVideoUrl: null,
|
||
archivedAt: null,
|
||
videoMetadata: null,
|
||
},
|
||
],
|
||
});
|
||
|
||
await saveGeneratedImages(
|
||
assistantText,
|
||
session.user.id,
|
||
id,
|
||
assistantMessageId,
|
||
baseReplacementPrompt || selectedAgentPreset.name,
|
||
resolvedModelRef,
|
||
{
|
||
sourceAssetIds,
|
||
sourceImageUrls: params.sourceImageUrls,
|
||
relationType: "edited_from",
|
||
sourceType: params.sourceType,
|
||
projectId: contextPayload?.projectId || null,
|
||
userPrompt: requestTextContent,
|
||
optimizedPrompt: baseReplacementPrompt || undefined,
|
||
requestedImageCount: 1,
|
||
aspectRatio: resolvedAspectRatio,
|
||
imageSize: persistedImageSize,
|
||
},
|
||
);
|
||
|
||
writeAssistantTextChunk(
|
||
writer,
|
||
assistantText,
|
||
params.streamLabel,
|
||
assistantMessageId,
|
||
);
|
||
|
||
if (contextPayload?.source === "canvas") {
|
||
writeCanvasMediaChunk(
|
||
writer,
|
||
{
|
||
messageId: assistantMessageId,
|
||
imageUrls: [params.image.originalUrl],
|
||
prompt: baseReplacementPrompt || selectedAgentPreset.name,
|
||
sourceImageUrls: params.sourceImageUrls,
|
||
agentPresetId: selectedAgentPreset.id,
|
||
agentPresetName: selectedAgentPreset.name,
|
||
agentCategoryId:
|
||
selectedAgentPreset.category?.id ||
|
||
selectedAgentPreset.categoryId ||
|
||
null,
|
||
agentCategoryName: selectedAgentPreset.category?.name || null,
|
||
agentExecutionSnapshotId: presetSnapshot.id,
|
||
},
|
||
`${params.streamLabel}:canvas-media`,
|
||
);
|
||
}
|
||
};
|
||
|
||
try {
|
||
for (const [referenceIndex, reference] of replacementReferences.entries()) {
|
||
const pairedProductIndex =
|
||
isSkuReplacementPreset && productInputImageUrls.length > 0
|
||
? Math.min(
|
||
typeof reference.assignedProductIndex === "number"
|
||
? reference.assignedProductIndex
|
||
: referenceIndex,
|
||
productInputImageUrls.length - 1
|
||
)
|
||
: undefined;
|
||
const orderedProductContextImageUrls =
|
||
pairedProductIndex !== undefined
|
||
? [
|
||
productInputImageUrls[pairedProductIndex]!,
|
||
...productInputImageUrls.filter(
|
||
(_, productIndex) => productIndex !== pairedProductIndex
|
||
),
|
||
]
|
||
: productInputImageUrls;
|
||
const replacementExecutionPlan = await analyzeReplacementExecutionPlan({
|
||
referenceImageUrl: reference.imageUrl,
|
||
referenceLabel: reference.label,
|
||
productImageUrls: orderedProductContextImageUrls,
|
||
modelImageUrl: modelInputImageUrl,
|
||
workflowPrompt: baseReplacementPrompt || hiddenPromptText || null,
|
||
planningRuntime: replacementScenePlanningRuntime,
|
||
models: replacementVisionModels,
|
||
});
|
||
const replacementSourceType = isDetailReplacementPreset
|
||
? "detail_replacement"
|
||
: "sku_replacement";
|
||
console.info("[Chat API] Replacement reference decision:", {
|
||
mode: replacementSourceType,
|
||
referenceIndex: referenceIndex + 1,
|
||
totalReferences: replacementReferences.length,
|
||
referenceLabel: reference.label,
|
||
referenceImageUrl: reference.imageUrl,
|
||
pairedProductIndex:
|
||
typeof pairedProductIndex === "number" ? pairedProductIndex + 1 : null,
|
||
shouldReplace: replacementExecutionPlan.shouldReplace,
|
||
reason: replacementExecutionPlan.reason,
|
||
detectionModelName: replacementExecutionPlan.detectionModelName,
|
||
detectionSource: replacementExecutionPlan.detectionSource,
|
||
scenePlanSource: replacementExecutionPlan.scenePlan.analysisSource,
|
||
selectedProductImageIndexes:
|
||
replacementExecutionPlan.scenePlan.selectedProductImageIndexes,
|
||
useModelImage: replacementExecutionPlan.scenePlan.useModelImage,
|
||
});
|
||
console.info("[Chat API] Replacement scene plan summary:", {
|
||
mode: replacementSourceType,
|
||
referenceIndex: referenceIndex + 1,
|
||
referenceLabel: reference.label,
|
||
analysisModelName: replacementExecutionPlan.scenePlan.analysisModelName,
|
||
analysisSource: replacementExecutionPlan.scenePlan.analysisSource,
|
||
selectedProductImageIndexes:
|
||
replacementExecutionPlan.scenePlan.selectedProductImageIndexes,
|
||
productSelectionReason:
|
||
replacementExecutionPlan.scenePlan.productSelectionReason,
|
||
useModelImage: replacementExecutionPlan.scenePlan.useModelImage,
|
||
modelImageReason: replacementExecutionPlan.scenePlan.modelImageReason,
|
||
});
|
||
const referenceSourceImageUrls = Array.from(
|
||
new Set([
|
||
...sharedUploadedImageUrls,
|
||
...replacementTemplateReferenceImageUrls,
|
||
reference.imageUrl,
|
||
]),
|
||
);
|
||
|
||
if (!replacementExecutionPlan.shouldReplace) {
|
||
console.info("[Chat API] Replacement reference skipped and reused:", {
|
||
mode: replacementSourceType,
|
||
referenceIndex: referenceIndex + 1,
|
||
totalReferences: replacementReferences.length,
|
||
referenceLabel: reference.label,
|
||
reason: replacementExecutionPlan.reason,
|
||
});
|
||
await emitReplacementResultMessage({
|
||
reference,
|
||
bodyText: replacementExecutionPlan.reason,
|
||
image: {
|
||
originalUrl: reference.imageUrl,
|
||
thumbnailUrl: reference.imageUrl,
|
||
},
|
||
modelDisplayName: null,
|
||
sourceImageUrls: referenceSourceImageUrls,
|
||
sourceType: replacementSourceType,
|
||
streamLabel: isDetailReplacementPreset
|
||
? "agent-preset:image:detail-replacement:item"
|
||
: "agent-preset:image:sku-replacement:item",
|
||
});
|
||
successCount += 1;
|
||
continue;
|
||
}
|
||
|
||
const selectedProductIndexes = replacementExecutionPlan.scenePlan
|
||
.selectedProductImageIndexes.map((index) => index - 1)
|
||
.filter(
|
||
(index) =>
|
||
index >= 0 && index < orderedProductContextImageUrls.length,
|
||
);
|
||
const selectedProductContextImageUrls =
|
||
selectedProductIndexes.length > 0
|
||
? selectedProductIndexes.map(
|
||
(index) => orderedProductContextImageUrls[index]!,
|
||
)
|
||
: orderedProductContextImageUrls.slice(
|
||
0,
|
||
Math.min(orderedProductContextImageUrls.length, 2),
|
||
);
|
||
const sourceImageUrls = Array.from(
|
||
new Set(
|
||
[
|
||
...selectedProductContextImageUrls,
|
||
...(replacementExecutionPlan.scenePlan.useModelImage &&
|
||
modelInputImageUrl
|
||
? [modelInputImageUrl]
|
||
: []),
|
||
...replacementTemplateReferenceImageUrls,
|
||
reference.imageUrl,
|
||
].filter(Boolean),
|
||
),
|
||
);
|
||
const replacementPrompt = buildAgentReplacementPrompt({
|
||
basePrompt:
|
||
baseReplacementPrompt ||
|
||
(isDetailReplacementPreset ? "执行详情替换" : "执行 SKU 替换"),
|
||
modeType: isDetailReplacementPreset
|
||
? "detail_replacement"
|
||
: "sku_replacement",
|
||
referenceLabel: reference.label,
|
||
referenceIndex,
|
||
totalReferences: replacementReferences.length,
|
||
hasModelImage:
|
||
replacementExecutionPlan.scenePlan.useModelImage &&
|
||
Boolean(modelInputImageUrl),
|
||
productImageCount: selectedProductContextImageUrls.length,
|
||
originalProductImageCount: productInputImageUrls.length,
|
||
templateReferenceCount: replacementTemplateReferenceImageUrls.length,
|
||
pairedProductIndex,
|
||
scenePlan: replacementExecutionPlan.scenePlan,
|
||
});
|
||
console.info("[Chat API] Replacement image generation started:", {
|
||
mode: replacementSourceType,
|
||
referenceIndex: referenceIndex + 1,
|
||
totalReferences: replacementReferences.length,
|
||
referenceLabel: reference.label,
|
||
pairedProductIndex:
|
||
typeof pairedProductIndex === "number" ? pairedProductIndex + 1 : null,
|
||
sourceImageCount: sourceImageUrls.length,
|
||
imageOrder: {
|
||
productImages: selectedProductContextImageUrls.length,
|
||
originalProductImages: productInputImageUrls.length,
|
||
hasModelImage:
|
||
replacementExecutionPlan.scenePlan.useModelImage &&
|
||
Boolean(modelInputImageUrl),
|
||
templateReferenceImages: replacementTemplateReferenceImageUrls.length,
|
||
finalReferenceImageIndex: sourceImageUrls.length,
|
||
},
|
||
});
|
||
|
||
try {
|
||
const result = await generateCanvasImage({
|
||
prompt: replacementPrompt,
|
||
negativePrompt:
|
||
typeof mergedImageParams.negativePrompt === "string"
|
||
? mergedImageParams.negativePrompt
|
||
: undefined,
|
||
model: resolvedModelRef,
|
||
aspectRatio: resolvedAspectRatio,
|
||
imageSize: resolvedImageSize,
|
||
imageCount: 1,
|
||
referenceImages: sourceImageUrls,
|
||
});
|
||
resolvedModelDisplayName = result.model || resolvedModelDisplayName;
|
||
const firstImage = result.images[0];
|
||
if (!firstImage) {
|
||
throw new Error("未返回结果图片");
|
||
}
|
||
let finalImage = firstImage;
|
||
let finalImageSizeLabel = resolvedImageSize;
|
||
try {
|
||
const normalizedImage = await normalizeReplacementResultImage({
|
||
generatedImageUrl: firstImage.originalUrl,
|
||
referenceImageUrl: reference.imageUrl,
|
||
uploadPrefix: isDetailReplacementPreset
|
||
? "agent-detail-replacement"
|
||
: "agent-sku-replacement",
|
||
});
|
||
finalImage = {
|
||
originalUrl: normalizedImage.originalUrl,
|
||
thumbnailUrl: normalizedImage.thumbnailUrl,
|
||
};
|
||
finalImageSizeLabel = normalizedImage.imageSize;
|
||
} catch (normalizeError) {
|
||
console.warn(
|
||
"[Chat API] Failed to normalize replacement output to reference size, using original model output:",
|
||
{
|
||
referenceLabel: reference.label,
|
||
referenceImageUrl: reference.imageUrl,
|
||
resultImageUrl: firstImage.originalUrl,
|
||
error:
|
||
normalizeError instanceof Error
|
||
? normalizeError.message
|
||
: "unknown",
|
||
},
|
||
);
|
||
}
|
||
console.info("[Chat API] Replacement image generation completed:", {
|
||
mode: replacementSourceType,
|
||
referenceIndex: referenceIndex + 1,
|
||
totalReferences: replacementReferences.length,
|
||
referenceLabel: reference.label,
|
||
modelDisplayName: resolvedModelDisplayName,
|
||
resultImageUrl: finalImage.originalUrl,
|
||
resultImageSize: finalImageSizeLabel,
|
||
});
|
||
await emitReplacementResultMessage({
|
||
reference,
|
||
image: finalImage,
|
||
imageSizeLabel: finalImageSizeLabel,
|
||
modelDisplayName: resolvedModelDisplayName,
|
||
sourceImageUrls: Array.from(
|
||
new Set([
|
||
...sharedUploadedImageUrls,
|
||
...replacementTemplateReferenceImageUrls,
|
||
...replacementReferences.map((item) => item.imageUrl),
|
||
]),
|
||
),
|
||
sourceType: replacementSourceType,
|
||
streamLabel: isDetailReplacementPreset
|
||
? "agent-preset:image:detail-replacement:item"
|
||
: "agent-preset:image:sku-replacement:item",
|
||
});
|
||
successCount += 1;
|
||
} catch (replacementItemError) {
|
||
console.error("[Chat API] Agent replacement item execution failed:", {
|
||
referenceIndex,
|
||
referenceLabel: reference.label,
|
||
error: replacementItemError,
|
||
});
|
||
await emitReplacementResultMessage({
|
||
reference,
|
||
bodyText: `生成失败，当前先沿用参考图：${
|
||
replacementItemError instanceof Error
|
||
? replacementItemError.message
|
||
: "请稍后重试"
|
||
}`,
|
||
image: {
|
||
originalUrl: reference.imageUrl,
|
||
thumbnailUrl: reference.imageUrl,
|
||
},
|
||
modelDisplayName: null,
|
||
sourceImageUrls: referenceSourceImageUrls,
|
||
sourceType: replacementSourceType,
|
||
streamLabel: isDetailReplacementPreset
|
||
? "agent-preset:image:detail-replacement:item-fallback"
|
||
: "agent-preset:image:sku-replacement:item-fallback",
|
||
});
|
||
successCount += 1;
|
||
}
|
||
}
|
||
} finally {
|
||
stopKeepalive();
|
||
}
|
||
} else if (selectedAgentPreset.type === "image" && isSceneGenerationPreset) {
|
||
const sceneSlot = orderedSlots[0];
|
||
const sceneAssistantMessageId = generateUUID();
|
||
const sceneReferenceImageUrls = Array.from(
|
||
new Set(
|
||
(sceneSlot?.referenceAssets || [])
|
||
.map((asset) => String(asset.originalUrl || "").trim())
|
||
.filter(Boolean),
|
||
),
|
||
);
|
||
const sourceImageUrls = Array.from(
|
||
new Set([...sharedUploadedImageUrls, ...sceneReferenceImageUrls].filter(Boolean)),
|
||
);
|
||
const sourceAssetIds = Array.from(
|
||
new Set(
|
||
[...sharedUploadedAssetIds, ...(sceneSlot?.referenceAssetIds || [])].filter(Boolean),
|
||
),
|
||
);
|
||
const sceneUploadedImageBreakdown = getSceneUploadedImageBreakdown({
|
||
uploadedImageParts,
|
||
fallbackPrimaryCount: sceneProductSlotCount,
|
||
fallbackIncludeModelImage: sceneModelSlotEnabled,
|
||
});
|
||
const sceneImageRolePrompt = buildAgentSceneImageRolePrompt({
|
||
primaryProductImageCount: sceneUploadedImageBreakdown.primaryProductImageCount,
|
||
bundleProductImageCount: sceneUploadedImageBreakdown.bundleProductImageCount,
|
||
includeModelImage: sceneUploadedImageBreakdown.includeModelImage,
|
||
});
|
||
const sceneOptimizationImageUrls = resolveScenePromptOptimizationImageUrls({
|
||
uploadedImageParts,
|
||
primaryProductImageCount: sceneUploadedImageBreakdown.primaryProductImageCount,
|
||
bundleProductImageCount: sceneUploadedImageBreakdown.bundleProductImageCount,
|
||
includeModelImage: sceneUploadedImageBreakdown.includeModelImage,
|
||
});
|
||
const shouldAppendSceneSlotPrompt = !hiddenPromptText;
|
||
const rawSceneSlotPrompt = shouldAppendSceneSlotPrompt
|
||
? sceneSlot?.promptTemplate || selectedAgentPreset.promptTemplate || ""
|
||
: "";
|
||
const dedupedSceneSlotPrompt = rawSceneSlotPrompt
|
||
? dedupeAgentSlotPromptAgainstWorkflow({
|
||
workflowPrompt: hiddenPromptText || null,
|
||
slotPrompt: rawSceneSlotPrompt,
|
||
modeType: "scene_generation",
|
||
})
|
||
: "";
|
||
const optimizedSceneWorkflowResult =
|
||
workflowPromptOptimizationEnabled !== false &&
|
||
hiddenPromptText &&
|
||
sceneOptimizationImageUrls.primaryImageUrl
|
||
? await optimizeAgentSceneWorkflowPrompt({
|
||
workflowPrompt: hiddenPromptText,
|
||
slotPrompt: dedupedSceneSlotPrompt || null,
|
||
primaryImageUrl: sceneOptimizationImageUrls.primaryImageUrl,
|
||
modelImageUrl: sceneOptimizationImageUrls.modelImageUrl,
|
||
}).catch((error) => {
|
||
console.warn("[Chat API] Agent scene workflow prompt optimization failed:", error);
|
||
return null;
|
||
})
|
||
: null;
|
||
const optimizedSceneWorkflowPrompt = hiddenPromptText
|
||
? buildOptimizedAgentSceneWorkflowPrompt({
|
||
workflowPrompt: hiddenPromptText,
|
||
optimizationResult: optimizedSceneWorkflowResult,
|
||
})
|
||
: "";
|
||
if (optimizedSceneWorkflowResult) {
|
||
console.info("[Chat API] Agent scene workflow prompt optimized:", {
|
||
chatId: id,
|
||
agentPresetId: selectedAgentPreset.id,
|
||
optimizationModelName: optimizedSceneWorkflowResult.optimizationModelName || null,
|
||
optimizationSource: optimizedSceneWorkflowResult.optimizationSource || null,
|
||
sceneGoal: optimizedSceneWorkflowResult.sceneGoal,
|
||
environmentSummary: optimizedSceneWorkflowResult.environmentSummary,
|
||
cameraLightingSummary: optimizedSceneWorkflowResult.cameraLightingSummary,
|
||
characterSummary: optimizedSceneWorkflowResult.characterSummary,
|
||
negativeConstraints: optimizedSceneWorkflowResult.negativeConstraints,
|
||
conflicts: optimizedSceneWorkflowResult.conflicts,
|
||
});
|
||
} else if (hiddenPromptText && workflowPromptOptimizationEnabled === false) {
|
||
console.info("[Chat API] Agent scene workflow prompt optimization skipped:", {
|
||
chatId: id,
|
||
agentPresetId: selectedAgentPreset.id,
|
||
reason: "workflow_prompt_optimization_disabled",
|
||
});
|
||
}
|
||
const scenePrompt =
|
||
buildAgentSceneWorkflowPrompt({
|
||
workflowPrompt: optimizedSceneWorkflowPrompt || hiddenPromptText || null,
|
||
slotPrompt: optimizedSceneWorkflowPrompt ? null : dedupedSceneSlotPrompt || null,
|
||
imageRolePrompt: sceneImageRolePrompt,
|
||
userSupplementPrompt: sanitizedUserSupplementPrompt || null,
|
||
}) ||
|
||
buildAgentSlotPrompt({
|
||
workflowPrompt: hiddenPromptText || null,
|
||
slotPrompt: sceneSlot?.promptTemplate || selectedAgentPreset.promptTemplate,
|
||
userSupplementPrompt: sanitizedUserSupplementPrompt || null,
|
||
includeSlotTitle: false,
|
||
modeType: "scene_generation",
|
||
});
|
||
const sceneGenerationStartedAt = Date.now();
|
||
console.info("[Chat API] Agent scene prompt guardrails:", {
|
||
chatId: id,
|
||
agentPresetId: selectedAgentPreset.id,
|
||
agentPresetName: selectedAgentPreset.name,
|
||
agentExecutionSnapshotId: presetSnapshot.id,
|
||
includesWorkflowPrompt: Boolean(hiddenPromptText),
|
||
includesUserSupplementPrompt: Boolean(sanitizedUserSupplementPrompt),
|
||
includesModelImage: sceneUploadedImageBreakdown.includeModelImage,
|
||
shouldAppendSceneSlotPrompt,
|
||
includesSceneSlotPrompt: Boolean(dedupedSceneSlotPrompt),
|
||
productImageCount: sceneUploadedImageBreakdown.primaryProductImageCount,
|
||
bundleImageCount: sceneUploadedImageBreakdown.bundleProductImageCount,
|
||
uploadedImageCount: sharedUploadedImageUrls.length,
|
||
hasProductIdentityLock: sceneImageRolePrompt.includes("【产品锁定约束】"),
|
||
hasModelIdentityLock: sceneImageRolePrompt.includes("【模特锁定约束】"),
|
||
hasModelClothingOverrideRule: sceneImageRolePrompt.includes(
|
||
"只有当工作流设定或补充要求里明确要求修改模特衣服"
|
||
),
|
||
sceneSlotPromptPreview: dedupedSceneSlotPrompt
|
||
.split("\n")
|
||
.map((line) => line.trim())
|
||
.filter(Boolean)
|
||
.slice(0, 10),
|
||
imageRolePreview: sceneImageRolePrompt
|
||
.split("\n")
|
||
.map((line) => line.trim())
|
||
.filter(Boolean)
|
||
.slice(0, 12),
|
||
});
|
||
|
||
try {
|
||
const mergedImageParams = {
|
||
...(selectedAgentPreset.defaultParams || {}),
|
||
...(sceneSlot?.defaultParams || {}),
|
||
} as Record<string, any>;
|
||
const resolvedModelRef =
|
||
presetModeConfig.scene.allowModelOverride &&
|
||
String(selectedChatModel || "").trim() &&
|
||
String(selectedChatModel || "").trim() !== "auto"
|
||
? String(selectedChatModel || "").trim()
|
||
: presetModelId;
|
||
const resolvedImageCount = Math.max(1, effectiveImageParams?.imageCount || 1);
|
||
console.info("[Chat API] Agent scene generation started:", {
|
||
chatId: id,
|
||
agentPresetId: selectedAgentPreset.id,
|
||
agentPresetName: selectedAgentPreset.name,
|
||
agentExecutionSnapshotId: presetSnapshot.id,
|
||
projectId: contextPayload?.projectId || null,
|
||
selectedChatModel: String(selectedChatModel || "").trim() || "auto",
|
||
resolvedModelRef,
|
||
resolvedImageCount,
|
||
sourceImageCount: sourceImageUrls.length,
|
||
sourceAssetCount: sourceAssetIds.length,
|
||
promptStrategy: hiddenPromptText
|
||
? "workflow_plus_image_lock"
|
||
: rawSceneSlotPrompt
|
||
? "legacy_template_fallback"
|
||
: sanitizedUserSupplementPrompt
|
||
? "workflow_plus_user_supplement"
|
||
: "workflow_only",
|
||
promptLength: String(scenePrompt || `生成${selectedAgentPreset.name}`).length,
|
||
aspectRatio:
|
||
typeof mergedImageParams.aspectRatio === "string"
|
||
? mergedImageParams.aspectRatio
|
||
: effectiveImageParams?.aspectRatio || defaultImageQualityParams.aspectRatio,
|
||
imageSize:
|
||
typeof mergedImageParams.imageSize === "string"
|
||
? mergedImageParams.imageSize
|
||
: effectiveImageParams?.imageSize || defaultImageQualityParams.imageSize,
|
||
});
|
||
const result = await generateCanvasImage({
|
||
prompt: scenePrompt || `生成${selectedAgentPreset.name}`,
|
||
negativePrompt:
|
||
typeof mergedImageParams.negativePrompt === "string"
|
||
? mergedImageParams.negativePrompt
|
||
: undefined,
|
||
model: resolvedModelRef,
|
||
aspectRatio:
|
||
typeof mergedImageParams.aspectRatio === "string"
|
||
? mergedImageParams.aspectRatio
|
||
: effectiveImageParams?.aspectRatio || defaultImageQualityParams.aspectRatio,
|
||
imageSize:
|
||
typeof mergedImageParams.imageSize === "string"
|
||
? mergedImageParams.imageSize
|
||
: effectiveImageParams?.imageSize || defaultImageQualityParams.imageSize,
|
||
imageCount: resolvedImageCount,
|
||
referenceImages: sourceImageUrls,
|
||
debugLabel: "agent-scene-generation",
|
||
debugMeta: {
|
||
chatId: id,
|
||
agentPresetId: selectedAgentPreset.id,
|
||
agentPresetName: selectedAgentPreset.name,
|
||
agentExecutionSnapshotId: presetSnapshot.id,
|
||
projectId: contextPayload?.projectId || null,
|
||
},
|
||
});
|
||
console.info("[Chat API] Agent scene generation model completed:", {
|
||
chatId: id,
|
||
agentPresetId: selectedAgentPreset.id,
|
||
agentPresetName: selectedAgentPreset.name,
|
||
agentExecutionSnapshotId: presetSnapshot.id,
|
||
resultImageCount: result.images.length,
|
||
resultModelName: result.model,
|
||
resultModelRef: result.modelRef,
|
||
elapsedMs: Date.now() - sceneGenerationStartedAt,
|
||
});
|
||
|
||
const imageMarkdown = buildAgentToolImagesMarkdown(
|
||
result.images.map((image) => ({
|
||
originalUrl: image.originalUrl,
|
||
thumbnailUrl: image.thumbnailUrl,
|
||
imageSize:
|
||
typeof mergedImageParams.imageSize === "string"
|
||
? mergedImageParams.imageSize
|
||
: effectiveImageParams?.imageSize || defaultImageQualityParams.imageSize,
|
||
})),
|
||
);
|
||
const assistantText = prependImageModelBadge(
|
||
[`### ${selectedAgentPreset.name}`, imageMarkdown].filter(Boolean).join("\n\n"),
|
||
result.model,
|
||
);
|
||
|
||
await saveMessages({
|
||
messages: [
|
||
{
|
||
id: sceneAssistantMessageId,
|
||
chatId: id,
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: assistantText }],
|
||
attachments: [],
|
||
createdAt: new Date(),
|
||
videoStorageStatus: null,
|
||
temporaryVideoUrl: null,
|
||
permanentVideoUrl: null,
|
||
archivedAt: null,
|
||
videoMetadata: null,
|
||
},
|
||
],
|
||
});
|
||
|
||
await saveGeneratedImages(
|
||
assistantText,
|
||
session.user.id,
|
||
id,
|
||
sceneAssistantMessageId,
|
||
scenePrompt || selectedAgentPreset.name,
|
||
result.modelRef || resolvedModelRef,
|
||
{
|
||
sourceAssetIds,
|
||
sourceImageUrls,
|
||
relationType: "generated_from",
|
||
sourceType:
|
||
sourceImageUrls.length > 0 ? "reference_generate" : "generated",
|
||
projectId: contextPayload?.projectId || null,
|
||
userPrompt: requestTextContent,
|
||
optimizedPrompt: scenePrompt || undefined,
|
||
requestedImageCount: resolvedImageCount,
|
||
},
|
||
);
|
||
console.info("[Chat API] Agent scene generation persisted:", {
|
||
chatId: id,
|
||
agentPresetId: selectedAgentPreset.id,
|
||
agentPresetName: selectedAgentPreset.name,
|
||
agentExecutionSnapshotId: presetSnapshot.id,
|
||
savedImageCount: result.images.length,
|
||
elapsedMs: Date.now() - sceneGenerationStartedAt,
|
||
});
|
||
|
||
writeAssistantTextChunk(
|
||
writer,
|
||
assistantText,
|
||
"agent-preset:image:scene-generation",
|
||
sceneAssistantMessageId,
|
||
);
|
||
if (contextPayload?.source === "canvas") {
|
||
writeCanvasMediaChunk(
|
||
writer,
|
||
{
|
||
messageId: sceneAssistantMessageId,
|
||
imageUrls: result.images.map((image) => image.originalUrl),
|
||
prompt: scenePrompt || selectedAgentPreset.name,
|
||
sourceImageUrls,
|
||
agentPresetId: selectedAgentPreset.id,
|
||
agentPresetName: selectedAgentPreset.name,
|
||
agentCategoryId:
|
||
selectedAgentPreset.category?.id ||
|
||
selectedAgentPreset.categoryId ||
|
||
null,
|
||
agentCategoryName: selectedAgentPreset.category?.name || null,
|
||
agentExecutionSnapshotId: presetSnapshot.id,
|
||
},
|
||
"agent-preset:image:scene-generation:canvas-media",
|
||
);
|
||
}
|
||
console.info("[Chat API] Agent scene generation response streamed:", {
|
||
chatId: id,
|
||
agentPresetId: selectedAgentPreset.id,
|
||
agentPresetName: selectedAgentPreset.name,
|
||
agentExecutionSnapshotId: presetSnapshot.id,
|
||
resultImageCount: result.images.length,
|
||
elapsedMs: Date.now() - sceneGenerationStartedAt,
|
||
});
|
||
|
||
successCount = result.images.length;
|
||
} catch (sceneError) {
|
||
console.error("[Chat API] Agent preset scene generation failed:", sceneError);
|
||
console.error("[Chat API] Agent scene generation failed detail:", {
|
||
chatId: id,
|
||
agentPresetId: selectedAgentPreset.id,
|
||
agentPresetName: selectedAgentPreset.name,
|
||
agentExecutionSnapshotId: presetSnapshot.id,
|
||
elapsedMs: Date.now() - sceneGenerationStartedAt,
|
||
error: sceneError instanceof Error ? sceneError.message : String(sceneError),
|
||
});
|
||
const errorMessage = `### ${selectedAgentPreset.name}\n\n执行失败：${
|
||
sceneError instanceof Error ? sceneError.message : "请稍后重试"
|
||
}`;
|
||
await saveMessages({
|
||
messages: [
|
||
{
|
||
id: sceneAssistantMessageId,
|
||
chatId: id,
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: errorMessage }],
|
||
attachments: [],
|
||
createdAt: new Date(),
|
||
videoStorageStatus: null,
|
||
temporaryVideoUrl: null,
|
||
permanentVideoUrl: null,
|
||
archivedAt: null,
|
||
videoMetadata: null,
|
||
},
|
||
],
|
||
});
|
||
writeAssistantTextChunk(
|
||
writer,
|
||
errorMessage,
|
||
"agent-preset:error:scene-generation",
|
||
sceneAssistantMessageId,
|
||
);
|
||
}
|
||
} else {
|
||
for (const [slotOrder, slot] of orderedSlots.entries()) {
|
||
const slotLabel =
|
||
String(slot.title || slot.uploadLabel || "").trim() || `第${slotOrder + 1}张`;
|
||
const slotReferenceImageUrls = Array.from(
|
||
new Set(
|
||
(slot.referenceAssets || [])
|
||
.map((asset) => String(asset.originalUrl || "").trim())
|
||
.filter(Boolean),
|
||
),
|
||
);
|
||
const sourceImageUrls = Array.from(
|
||
new Set(
|
||
[
|
||
...sharedUploadedImageUrls,
|
||
...slotReferenceImageUrls,
|
||
].filter(Boolean),
|
||
),
|
||
);
|
||
const sourceAssetIds = Array.from(
|
||
new Set(
|
||
[
|
||
...sharedUploadedAssetIds,
|
||
...(slot.referenceAssetIds || []),
|
||
].filter(Boolean),
|
||
),
|
||
);
|
||
const slotPrompt = buildAgentSlotPrompt({
|
||
workflowPrompt: hiddenPromptText || null,
|
||
slotPrompt: slot.promptTemplate,
|
||
slotTitle: slot.title || slot.uploadLabel || null,
|
||
userSupplementPrompt: sanitizedUserSupplementPrompt || null,
|
||
modeType: "generic",
|
||
});
|
||
const assistantMessageId = generateUUID();
|
||
|
||
try {
|
||
if (selectedAgentPreset.type === "image") {
|
||
const mergedImageParams = {
|
||
...(selectedAgentPreset.defaultParams || {}),
|
||
...(slot.defaultParams || {}),
|
||
} as Record<string, any>;
|
||
const result = await generateCanvasImage({
|
||
prompt: slotPrompt || `生成${slotLabel}`,
|
||
negativePrompt:
|
||
typeof mergedImageParams.negativePrompt === "string"
|
||
? mergedImageParams.negativePrompt
|
||
: undefined,
|
||
model: presetModelId,
|
||
aspectRatio:
|
||
typeof mergedImageParams.aspectRatio === "string"
|
||
? mergedImageParams.aspectRatio
|
||
: effectiveImageParams?.aspectRatio || defaultImageQualityParams.aspectRatio,
|
||
imageSize:
|
||
typeof mergedImageParams.imageSize === "string"
|
||
? mergedImageParams.imageSize
|
||
: effectiveImageParams?.imageSize || defaultImageQualityParams.imageSize,
|
||
imageCount: 1,
|
||
referenceImages: sourceImageUrls,
|
||
});
|
||
|
||
const imageMarkdown = buildAgentToolImagesMarkdown(
|
||
result.images.map((image) => ({
|
||
originalUrl: image.originalUrl,
|
||
thumbnailUrl: image.thumbnailUrl,
|
||
imageSize:
|
||
typeof mergedImageParams.imageSize === "string"
|
||
? mergedImageParams.imageSize
|
||
: effectiveImageParams?.imageSize || defaultImageQualityParams.imageSize,
|
||
})),
|
||
);
|
||
const assistantText = prependImageModelBadge(
|
||
[`### ${slotLabel}`, imageMarkdown].filter(Boolean).join("\n\n"),
|
||
result.model,
|
||
);
|
||
|
||
await saveMessages({
|
||
messages: [
|
||
{
|
||
id: assistantMessageId,
|
||
chatId: id,
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: assistantText }],
|
||
attachments: [],
|
||
createdAt: new Date(),
|
||
videoStorageStatus: null,
|
||
temporaryVideoUrl: null,
|
||
permanentVideoUrl: null,
|
||
archivedAt: null,
|
||
videoMetadata: null,
|
||
},
|
||
],
|
||
});
|
||
|
||
await saveGeneratedImages(
|
||
assistantText,
|
||
session.user.id,
|
||
id,
|
||
assistantMessageId,
|
||
slotPrompt || slotLabel,
|
||
result.modelRef || presetModelId,
|
||
{
|
||
sourceAssetIds,
|
||
sourceImageUrls,
|
||
relationType: "generated_from",
|
||
sourceType:
|
||
sourceImageUrls.length > 0 ? "reference_generate" : "generated",
|
||
projectId: contextPayload?.projectId || null,
|
||
userPrompt: requestTextContent,
|
||
optimizedPrompt: slotPrompt || undefined,
|
||
requestedImageCount: 1,
|
||
},
|
||
);
|
||
|
||
writeAssistantTextChunk(
|
||
writer,
|
||
assistantText,
|
||
`agent-preset:image:slot-${slotOrder + 1}`,
|
||
assistantMessageId,
|
||
);
|
||
if (contextPayload?.source === "canvas") {
|
||
writeCanvasMediaChunk(
|
||
writer,
|
||
{
|
||
messageId: assistantMessageId,
|
||
imageUrls: result.images.map((image) => image.originalUrl),
|
||
prompt: slotPrompt || slotLabel,
|
||
sourceImageUrls,
|
||
agentPresetId: selectedAgentPreset.id,
|
||
agentPresetName: selectedAgentPreset.name,
|
||
agentCategoryId:
|
||
selectedAgentPreset.category?.id ||
|
||
selectedAgentPreset.categoryId ||
|
||
null,
|
||
agentCategoryName: selectedAgentPreset.category?.name || null,
|
||
agentExecutionSnapshotId: presetSnapshot.id,
|
||
},
|
||
`agent-preset:image:slot-${slotOrder + 1}:canvas-media`,
|
||
);
|
||
}
|
||
} else {
|
||
const mergedVideoParams = {
|
||
...(selectedAgentPreset.defaultParams || {}),
|
||
...(slot.defaultParams || {}),
|
||
} as Record<string, any>;
|
||
const requestedMode = String(mergedVideoParams.mode || "").trim() as
|
||
| "text-to-video"
|
||
| "first-frame"
|
||
| "first-last-frame"
|
||
| "reference-images";
|
||
const mode =
|
||
requestedMode ||
|
||
(sourceImageUrls.length >= 2
|
||
? "first-last-frame"
|
||
: sourceImageUrls.length >= 1
|
||
? "first-frame"
|
||
: "text-to-video");
|
||
const directVideoRatio =
|
||
mergedVideoParams.ratio === "16:9" ||
|
||
mergedVideoParams.ratio === "4:3" ||
|
||
mergedVideoParams.ratio === "1:1" ||
|
||
mergedVideoParams.ratio === "3:4" ||
|
||
mergedVideoParams.ratio === "9:16" ||
|
||
mergedVideoParams.ratio === "21:9" ||
|
||
mergedVideoParams.ratio === "adaptive"
|
||
? mergedVideoParams.ratio
|
||
: undefined;
|
||
const images =
|
||
mode === "first-last-frame"
|
||
? sourceImageUrls.slice(0, 2)
|
||
: mode === "first-frame"
|
||
? sourceImageUrls.slice(0, 1)
|
||
: mode === "reference-images"
|
||
? sourceImageUrls.slice(0, 4)
|
||
: [];
|
||
|
||
const result = await executeDirectVideoGeneration({
|
||
prompt: slotPrompt || `生成${slotLabel}`,
|
||
mode,
|
||
images,
|
||
resolution: mergedVideoParams.resolution,
|
||
ratio: directVideoRatio,
|
||
duration: mergedVideoParams.duration,
|
||
preferredModelRef: presetModelId,
|
||
});
|
||
|
||
const videoUrl = String(result.videoUrl || "").trim();
|
||
if (!videoUrl) {
|
||
throw new Error("视频生成完成但未返回视频地址");
|
||
}
|
||
|
||
const assistantText = prependImageModelBadge(
|
||
[
|
||
`### ${slotLabel}`,
|
||
buildAgentToolVideoMarkdown(videoUrl),
|
||
]
|
||
.filter(Boolean)
|
||
.join("\n\n"),
|
||
result.modelDisplayName,
|
||
);
|
||
|
||
await saveMessages({
|
||
messages: [
|
||
{
|
||
id: assistantMessageId,
|
||
chatId: id,
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: assistantText }],
|
||
attachments: [],
|
||
createdAt: new Date(),
|
||
videoStorageStatus: "temporary",
|
||
temporaryVideoUrl: videoUrl,
|
||
permanentVideoUrl: null,
|
||
archivedAt: null,
|
||
videoMetadata: (result.metadata || {}) as any,
|
||
},
|
||
],
|
||
});
|
||
|
||
try {
|
||
await persistVideoGenerationCompletedTurn({
|
||
userId: session.user.id,
|
||
chatId: id,
|
||
projectId: contextPayload?.projectId || null,
|
||
messageId: assistantMessageId,
|
||
text: requestTextContent,
|
||
source: "agent_preset_direct_video",
|
||
providerType: "direct-video",
|
||
modelRef: result.modelRef || presetModelId,
|
||
mode,
|
||
status: "completed",
|
||
temporaryVideoUrl: videoUrl,
|
||
});
|
||
} catch (memoryError) {
|
||
console.warn(
|
||
"[Chat API] Failed to update working state after agent preset video generation:",
|
||
memoryError,
|
||
);
|
||
}
|
||
|
||
await consumeGeneratedVideoPoints({
|
||
userId: session.user.id,
|
||
chatId: id,
|
||
messageId: assistantMessageId,
|
||
modelRefs: [result.modelRef || presetModelId],
|
||
duration: Number((result.metadata as any)?.duration) || undefined,
|
||
});
|
||
|
||
scheduleVideoArchive({
|
||
chatId: id,
|
||
messageId: assistantMessageId,
|
||
text: assistantText,
|
||
temporaryVideoUrl: videoUrl,
|
||
taskId: assistantMessageId,
|
||
metadata: result.metadata,
|
||
});
|
||
|
||
writeAssistantTextChunk(
|
||
writer,
|
||
assistantText,
|
||
`agent-preset:video:slot-${slotOrder + 1}`,
|
||
assistantMessageId,
|
||
);
|
||
if (contextPayload?.source === "canvas") {
|
||
writeCanvasMediaChunk(
|
||
writer,
|
||
{
|
||
messageId: assistantMessageId,
|
||
videoUrls: [videoUrl],
|
||
prompt: slotPrompt || slotLabel,
|
||
sourceImageUrls,
|
||
agentPresetId: selectedAgentPreset.id,
|
||
agentPresetName: selectedAgentPreset.name,
|
||
agentCategoryId:
|
||
selectedAgentPreset.category?.id ||
|
||
selectedAgentPreset.categoryId ||
|
||
null,
|
||
agentCategoryName: selectedAgentPreset.category?.name || null,
|
||
agentExecutionSnapshotId: presetSnapshot.id,
|
||
},
|
||
`agent-preset:video:slot-${slotOrder + 1}:canvas-media`,
|
||
);
|
||
}
|
||
}
|
||
|
||
successCount += 1;
|
||
} catch (slotError) {
|
||
console.error("[Chat API] Agent preset slot execution failed:", {
|
||
slotIndex: slot.slotIndex,
|
||
slotLabel,
|
||
error: slotError,
|
||
});
|
||
const errorMessage = `### ${slotLabel}\n\n执行失败：${
|
||
slotError instanceof Error ? slotError.message : "请稍后重试"
|
||
}`;
|
||
await saveMessages({
|
||
messages: [
|
||
{
|
||
id: assistantMessageId,
|
||
chatId: id,
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: errorMessage }],
|
||
attachments: [],
|
||
createdAt: new Date(),
|
||
videoStorageStatus: null,
|
||
temporaryVideoUrl: null,
|
||
permanentVideoUrl: null,
|
||
archivedAt: null,
|
||
videoMetadata: null,
|
||
},
|
||
],
|
||
});
|
||
if (selectedAgentPreset.type === "video") {
|
||
try {
|
||
await persistVideoGenerationFailedTurn({
|
||
userId: session.user.id,
|
||
chatId: id,
|
||
projectId: contextPayload?.projectId || null,
|
||
messageId: assistantMessageId,
|
||
text: requestTextContent,
|
||
source: "agent_preset_direct_video",
|
||
providerType: "direct-video",
|
||
modelRef: presetModelId,
|
||
status: "failed",
|
||
error: slotError instanceof Error ? slotError.message : String(slotError),
|
||
});
|
||
} catch (videoMemoryError) {
|
||
console.warn(
|
||
"[Chat API] Failed to persist agent preset video failure state:",
|
||
videoMemoryError,
|
||
);
|
||
}
|
||
}
|
||
writeAssistantTextChunk(
|
||
writer,
|
||
errorMessage,
|
||
`agent-preset:error:slot-${slotOrder + 1}`,
|
||
assistantMessageId,
|
||
);
|
||
}
|
||
}
|
||
}
|
||
|
||
try {
|
||
await persistAgentPresetTurn({
|
||
userId: session.user.id,
|
||
chatId: id,
|
||
projectId: contextPayload?.projectId || null,
|
||
currentIntent: selectedAgentPreset.type === "video" ? "video_gen" : "image_gen",
|
||
lastMessageId: message.id,
|
||
text: requestTextContent,
|
||
agentPresetId: selectedAgentPreset.id,
|
||
agentPresetName: selectedAgentPreset.name,
|
||
slotCount: orderedSlots.length,
|
||
successCount,
|
||
});
|
||
} catch (agentPresetMemoryError) {
|
||
console.warn("[Chat API] Failed to persist agent preset working state:", agentPresetMemoryError);
|
||
}
|
||
|
||
if (selectedAgentPreset.type === "image") {
|
||
if (contextPayload?.source === "canvas") {
|
||
writeImageGenerationStatusChunk(
|
||
writer,
|
||
{
|
||
phase: successCount > 0 ? "completed" : "failed",
|
||
modelName:
|
||
selectedAgentPreset.defaultModel?.displayName ||
|
||
selectedAgentPreset.name,
|
||
imageCount: successCount,
|
||
},
|
||
"agent-preset:image:status-finished",
|
||
);
|
||
}
|
||
writeAssistantTextChunk(
|
||
writer,
|
||
"<!-- IMAGE_GENERATION_END -->",
|
||
"agent-preset:image:end",
|
||
);
|
||
}
|
||
},
|
||
});
|
||
|
||
return createSseResponse(agentSequenceStream);
|
||
}
|
||
}
|
||
|
||
const directAgentExecution = shouldRunAgentConversation
|
||
? resolveDirectAgentToolExecution(smartAutoExecution, smartAutoPlan)
|
||
: null;
|
||
const directAgentToolTargetImageUrls = directAgentExecution
|
||
? pickAgentSourceImageUrls({
|
||
sourcePreferences: directAgentExecution.sourcePreferences,
|
||
requestResolvedMediaInput,
|
||
requestTargetResolution,
|
||
requestWorkingState,
|
||
})
|
||
: [];
|
||
const directAgentToolTargetImageUrl = directAgentToolTargetImageUrls[0] || null;
|
||
|
||
if (
|
||
shouldRunAgentConversation &&
|
||
!smartAutoNeedsClarification &&
|
||
directAgentExecution?.toolName === "analyzeImage"
|
||
) {
|
||
if (directAgentToolTargetImageUrls.length <= 0) {
|
||
const missingTargetMessage = getDirectAgentToolTargetMissingMessage("analyze");
|
||
const missingTargetStream = createUIMessageStream({
|
||
execute: async ({ writer }) => {
|
||
const messageId = generateUUID();
|
||
writeCurrentAgentTargetLockChunk(writer, "agent:analyze:missing-target-lock");
|
||
writeAssistantTextChunk(
|
||
writer,
|
||
missingTargetMessage,
|
||
"agent:analyze:missing-target",
|
||
messageId,
|
||
);
|
||
|
||
await saveMessages({
|
||
messages: [
|
||
{
|
||
id: messageId,
|
||
chatId: id,
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: missingTargetMessage }],
|
||
attachments: [],
|
||
createdAt: new Date(),
|
||
videoStorageStatus: null,
|
||
temporaryVideoUrl: null,
|
||
permanentVideoUrl: null,
|
||
archivedAt: null,
|
||
videoMetadata: null,
|
||
},
|
||
],
|
||
});
|
||
},
|
||
});
|
||
|
||
return createSseResponse(missingTargetStream);
|
||
}
|
||
|
||
const analysisAssistantMessageId = generateUUID();
|
||
const analysisSummaryAssetIds = Array.from(
|
||
new Set([
|
||
...(requestTargetResolution?.targetAssetIds || []),
|
||
...(requestResolvedMediaInput.selectedAssetIds || []),
|
||
...(requestResolvedMediaInput.uploadedAssetIds || []),
|
||
...extractMessageImageAssetIds(message.parts as any[]),
|
||
]),
|
||
);
|
||
|
||
const directAnalyzeStream = createUIMessageStream({
|
||
execute: async ({ writer }) => {
|
||
writeAgentProgressChunk(
|
||
writer,
|
||
{
|
||
stage: "planning",
|
||
currentStep: "analyze",
|
||
title: "正在分析目标图片",
|
||
message: "我会先提取稳定的视觉摘要，再整理成更直接的分析结论。",
|
||
toolName: "analyzeImage",
|
||
capability: "analyze",
|
||
},
|
||
"agent:analyze:progress-planning",
|
||
);
|
||
writeCurrentAgentTargetLockChunk(writer, "agent:analyze:target-lock");
|
||
|
||
try {
|
||
const { summarizeImageForExecutionReport } = await import("@/lib/ai/smart-analyzer");
|
||
const analysisSummary = await summarizeImageForExecutionReport({
|
||
text: requestTextContent,
|
||
imageUrls: directAgentToolTargetImageUrls,
|
||
context: effectiveAutoContextText || undefined,
|
||
preferredModelId: preferredAgentModelId || finalModelId,
|
||
assetIds: analysisSummaryAssetIds,
|
||
});
|
||
|
||
if (!analysisSummary?.trim()) {
|
||
throw new Error("当前图片暂时无法提取稳定的视觉摘要，请稍后再试。");
|
||
}
|
||
|
||
const analysisAnswer = await buildDirectAgentImageAnalysisAnswer({
|
||
question: requestTextContent,
|
||
summary: analysisSummary.trim(),
|
||
modelId: finalModelId || preferredAgentModelId,
|
||
});
|
||
const assistantText = prependImageModelBadge(
|
||
[
|
||
getDirectAgentToolSuccessMessage("analyze"),
|
||
`图像观察：\n${analysisSummary.trim()}`,
|
||
analysisAnswer.trim() ? `分析结论：\n${analysisAnswer.trim()}` : "",
|
||
]
|
||
.filter(Boolean)
|
||
.join("\n\n"),
|
||
getDirectAgentToolLabel("analyze"),
|
||
);
|
||
|
||
await saveMessages({
|
||
messages: [
|
||
{
|
||
id: analysisAssistantMessageId,
|
||
chatId: id,
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: assistantText }],
|
||
attachments: [],
|
||
createdAt: new Date(),
|
||
videoStorageStatus: null,
|
||
temporaryVideoUrl: null,
|
||
permanentVideoUrl: null,
|
||
archivedAt: null,
|
||
videoMetadata: null,
|
||
},
|
||
],
|
||
});
|
||
|
||
try {
|
||
await persistImageWorkflowTurn({
|
||
userId: session.user.id,
|
||
chatId: id,
|
||
projectId: contextPayload?.projectId || null,
|
||
currentIntent: "vision",
|
||
currentSubIntent: smartAutoExecution?.routeSubIntent || "analyze",
|
||
currentFocusAssetIds: requestTargetResolution?.targetAssetIds || null,
|
||
currentFocusImageUrls:
|
||
directAgentToolTargetImageUrls.length > 0 ? directAgentToolTargetImageUrls : null,
|
||
lastResolvedTargetAssetIds: requestTargetResolution?.targetAssetIds || null,
|
||
lastResolvedTargetImageUrls:
|
||
directAgentToolTargetImageUrls.length > 0 ? directAgentToolTargetImageUrls : null,
|
||
lastMessageId: analysisAssistantMessageId,
|
||
lastUserPrompt: requestTextContent,
|
||
lastSourceImageUrls:
|
||
directAgentToolTargetImageUrls.length > 0 ? directAgentToolTargetImageUrls : null,
|
||
});
|
||
} catch (memoryError) {
|
||
console.warn("[Chat API] Failed to persist direct analyze working state:", memoryError);
|
||
}
|
||
|
||
writeAssistantTextChunk(
|
||
writer,
|
||
assistantText,
|
||
"agent:analyze:success",
|
||
analysisAssistantMessageId,
|
||
);
|
||
writeAgentProgressChunk(
|
||
writer,
|
||
{
|
||
stage: "completed",
|
||
currentStep: "finalize",
|
||
title: "图片分析完成",
|
||
message: "我已经整理好可直接阅读的分析结论。",
|
||
toolName: "analyzeImage",
|
||
capability: "analyze",
|
||
},
|
||
"agent:analyze:progress-completed",
|
||
);
|
||
} catch (directAnalyzeError) {
|
||
console.error("[Chat API] Direct agent image analysis failed:", directAnalyzeError);
|
||
const errorMessage = getDirectAgentToolErrorMessage("analyze", directAnalyzeError);
|
||
|
||
await saveMessages({
|
||
messages: [
|
||
{
|
||
id: analysisAssistantMessageId,
|
||
chatId: id,
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: errorMessage }],
|
||
attachments: [],
|
||
createdAt: new Date(),
|
||
videoStorageStatus: null,
|
||
temporaryVideoUrl: null,
|
||
permanentVideoUrl: null,
|
||
archivedAt: null,
|
||
videoMetadata: null,
|
||
},
|
||
],
|
||
});
|
||
|
||
writeAssistantTextChunk(
|
||
writer,
|
||
errorMessage,
|
||
"agent:analyze:error",
|
||
analysisAssistantMessageId,
|
||
);
|
||
writeAgentProgressChunk(
|
||
writer,
|
||
{
|
||
stage: "failed",
|
||
currentStep: "analyze",
|
||
title: "图片分析失败",
|
||
message: errorMessage,
|
||
toolName: "analyzeImage",
|
||
capability: "analyze",
|
||
},
|
||
"agent:analyze:progress-failed",
|
||
);
|
||
}
|
||
},
|
||
});
|
||
|
||
return createSseResponse(directAnalyzeStream);
|
||
}
|
||
|
||
if (
|
||
shouldRunAgentConversation &&
|
||
!smartAutoNeedsClarification &&
|
||
directAgentExecution?.toolName === "generateImage"
|
||
) {
|
||
const generationPrompt =
|
||
String(smartAutoOptimizedPrompt || requestTextContent || "").trim() || "生成图片";
|
||
const directImageStream = createUIMessageStream({
|
||
execute: async ({ writer }) => {
|
||
writeAgentProgressChunk(
|
||
writer,
|
||
{
|
||
stage: "planning",
|
||
currentStep: "select_tool",
|
||
title: "已选中图片生成工具",
|
||
message: "我正在整理参考图和生图参数，准备开始生成。",
|
||
toolName: "generateImage",
|
||
capability: "generate_image",
|
||
},
|
||
"agent:generate_image:progress-select-tool",
|
||
);
|
||
writeCurrentAgentTargetLockChunk(writer, "agent:generate_image:target-lock");
|
||
writeAssistantTextChunk(
|
||
writer,
|
||
"<!-- IMAGE_GENERATION_START -->",
|
||
"agent:generate_image:start",
|
||
);
|
||
writeAgentProgressChunk(
|
||
writer,
|
||
{
|
||
stage: "executing",
|
||
currentStep: "execute",
|
||
title: "正在生成图片",
|
||
message: "我会结合当前提示词和参考图生成新的画面。",
|
||
toolName: "generateImage",
|
||
capability: "generate_image",
|
||
},
|
||
"agent:generate_image:progress-executing",
|
||
);
|
||
if (contextPayload?.source === "canvas") {
|
||
writeImageGenerationStatusChunk(
|
||
writer,
|
||
{
|
||
phase: "processing",
|
||
providerType: requestedModelProviderType,
|
||
modelName: finalModelId,
|
||
imageCount:
|
||
effectiveImageParams?.imageCount ||
|
||
defaultImageQualityParams.imageCount ||
|
||
1,
|
||
},
|
||
"agent:generate_image:status-processing",
|
||
);
|
||
}
|
||
|
||
const assistantMessageId = generateUUID();
|
||
try {
|
||
const result = await generateCanvasImage({
|
||
prompt: generationPrompt,
|
||
negativePrompt: effectiveImageParams?.negativePrompt,
|
||
model: finalModelId,
|
||
aspectRatio:
|
||
effectiveImageParams?.aspectRatio ||
|
||
defaultImageQualityParams.aspectRatio,
|
||
imageSize:
|
||
effectiveImageParams?.imageSize ||
|
||
defaultImageQualityParams.imageSize,
|
||
imageCount:
|
||
effectiveImageParams?.imageCount ||
|
||
defaultImageQualityParams.imageCount,
|
||
referenceImages: directAgentToolTargetImageUrls,
|
||
});
|
||
|
||
const imageMarkdown = buildAgentToolImagesMarkdown(
|
||
result.images.map((image) => ({
|
||
originalUrl: image.originalUrl,
|
||
thumbnailUrl: image.thumbnailUrl,
|
||
imageSize:
|
||
effectiveImageParams?.imageSize ||
|
||
defaultImageQualityParams.imageSize,
|
||
})),
|
||
);
|
||
const assistantText = prependImageModelBadge(
|
||
[
|
||
getDirectAgentToolSuccessMessage("generate_image"),
|
||
imageMarkdown,
|
||
]
|
||
.filter(Boolean)
|
||
.join("\n\n"),
|
||
result.model,
|
||
);
|
||
|
||
await saveMessages({
|
||
messages: [
|
||
{
|
||
id: assistantMessageId,
|
||
chatId: id,
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: assistantText }],
|
||
attachments: [],
|
||
createdAt: new Date(),
|
||
videoStorageStatus: null,
|
||
temporaryVideoUrl: null,
|
||
permanentVideoUrl: null,
|
||
archivedAt: null,
|
||
videoMetadata: null,
|
||
},
|
||
],
|
||
});
|
||
|
||
await saveGeneratedImages(
|
||
assistantText,
|
||
session.user.id,
|
||
id,
|
||
assistantMessageId,
|
||
generationPrompt,
|
||
result.modelRef || finalModelId,
|
||
{
|
||
sourceImageUrls: directAgentToolTargetImageUrls,
|
||
relationType: "generated_from",
|
||
sourceType:
|
||
directAgentToolTargetImageUrls.length > 0
|
||
? "reference_generate"
|
||
: "generated",
|
||
projectId: contextPayload?.projectId || null,
|
||
userPrompt: requestTextContent,
|
||
optimizedPrompt: generationPrompt,
|
||
requestedImageCount:
|
||
effectiveImageParams?.imageCount ||
|
||
defaultImageQualityParams.imageCount ||
|
||
1,
|
||
aspectRatio:
|
||
effectiveImageParams?.aspectRatio ||
|
||
defaultImageQualityParams.aspectRatio,
|
||
imageSize:
|
||
effectiveImageParams?.imageSize ||
|
||
defaultImageQualityParams.imageSize,
|
||
},
|
||
);
|
||
|
||
writeAssistantTextChunk(
|
||
writer,
|
||
assistantText,
|
||
"agent:generate_image:result",
|
||
assistantMessageId,
|
||
);
|
||
if (contextPayload?.source === "canvas") {
|
||
writeCanvasMediaChunk(
|
||
writer,
|
||
{
|
||
messageId: assistantMessageId,
|
||
imageUrls: result.images.map((image) => image.originalUrl),
|
||
prompt: generationPrompt,
|
||
sourceImageUrls: directAgentToolTargetImageUrls,
|
||
},
|
||
"agent:generate_image:canvas-media",
|
||
);
|
||
writeImageGenerationStatusChunk(
|
||
writer,
|
||
{
|
||
phase: "completed",
|
||
providerType: requestedModelProviderType,
|
||
modelName: result.model,
|
||
imageCount: result.images.length,
|
||
},
|
||
"agent:generate_image:status-completed",
|
||
);
|
||
}
|
||
writeAgentProgressChunk(
|
||
writer,
|
||
{
|
||
stage: "completed",
|
||
currentStep: "finalize",
|
||
title: "图片生成完成",
|
||
message: "结果已经回写到对话和画布。",
|
||
toolName: "generateImage",
|
||
capability: "generate_image",
|
||
},
|
||
"agent:generate_image:progress-completed",
|
||
);
|
||
writeAssistantTextChunk(
|
||
writer,
|
||
"<!-- IMAGE_GENERATION_END -->",
|
||
"agent:generate_image:end",
|
||
);
|
||
} catch (directToolError) {
|
||
console.error("[Chat API] Direct agent image generation failed:", directToolError);
|
||
const errorMessage = getDirectAgentToolErrorMessage(
|
||
"generate_image",
|
||
directToolError,
|
||
);
|
||
await saveMessages({
|
||
messages: [
|
||
{
|
||
id: assistantMessageId,
|
||
chatId: id,
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: errorMessage }],
|
||
attachments: [],
|
||
createdAt: new Date(),
|
||
videoStorageStatus: null,
|
||
temporaryVideoUrl: null,
|
||
permanentVideoUrl: null,
|
||
archivedAt: null,
|
||
videoMetadata: null,
|
||
},
|
||
],
|
||
});
|
||
writeAssistantTextChunk(
|
||
writer,
|
||
errorMessage,
|
||
"agent:generate_image:error",
|
||
assistantMessageId,
|
||
);
|
||
if (contextPayload?.source === "canvas") {
|
||
writeImageGenerationStatusChunk(
|
||
writer,
|
||
{
|
||
phase: "failed",
|
||
providerType: requestedModelProviderType,
|
||
modelName: finalModelId,
|
||
},
|
||
"agent:generate_image:status-failed",
|
||
);
|
||
}
|
||
writeAgentProgressChunk(
|
||
writer,
|
||
{
|
||
stage: "failed",
|
||
currentStep: "execute",
|
||
title: "图片生成失败",
|
||
message: errorMessage,
|
||
toolName: "generateImage",
|
||
capability: "generate_image",
|
||
},
|
||
"agent:generate_image:progress-failed",
|
||
);
|
||
writeAssistantTextChunk(
|
||
writer,
|
||
"<!-- IMAGE_GENERATION_END -->",
|
||
"agent:generate_image:end-error",
|
||
);
|
||
}
|
||
},
|
||
});
|
||
|
||
return createSseResponse(directImageStream);
|
||
}
|
||
|
||
if (
|
||
shouldRunAgentConversation &&
|
||
!smartAutoNeedsClarification &&
|
||
directAgentExecution?.toolName === "generateVideo"
|
||
) {
|
||
const generationPrompt =
|
||
String(smartAutoOptimizedPrompt || requestTextContent || "").trim() || "生成视频";
|
||
const directVideoStream = createUIMessageStream({
|
||
execute: async ({ writer }) => {
|
||
const assistantMessageId = generateUUID();
|
||
writeAgentProgressChunk(
|
||
writer,
|
||
{
|
||
stage: "planning",
|
||
currentStep: "select_tool",
|
||
title: "已选中视频生成工具",
|
||
message: "我正在整理模式、参考图和视频参数。",
|
||
toolName: "generateVideo",
|
||
capability: "generate_video",
|
||
},
|
||
"agent:generate_video:progress-select-tool",
|
||
);
|
||
writeCurrentAgentTargetLockChunk(writer, "agent:generate_video:target-lock");
|
||
try {
|
||
const plannedArgs = findAgentToolCallArgs(
|
||
"generateVideo",
|
||
smartAutoExecution,
|
||
smartAutoPlan,
|
||
);
|
||
const requestedMode = String(
|
||
plannedArgs?.mode || effectiveVideoParams?.mode || "",
|
||
).trim() as
|
||
| "text-to-video"
|
||
| "first-frame"
|
||
| "first-last-frame"
|
||
| "reference-images";
|
||
const mode =
|
||
requestedMode ||
|
||
(directAgentToolTargetImageUrls.length >= 2
|
||
? "first-last-frame"
|
||
: directAgentToolTargetImageUrls.length >= 1
|
||
? "first-frame"
|
||
: "text-to-video");
|
||
const images =
|
||
mode === "first-last-frame"
|
||
? directAgentToolTargetImageUrls.slice(0, 2)
|
||
: mode === "first-frame"
|
||
? directAgentToolTargetImageUrls.slice(0, 1)
|
||
: mode === "reference-images"
|
||
? directAgentToolTargetImageUrls.slice(0, 4)
|
||
: [];
|
||
const directVideoRatio =
|
||
effectiveVideoParams?.ratio === "16:9" ||
|
||
effectiveVideoParams?.ratio === "4:3" ||
|
||
effectiveVideoParams?.ratio === "1:1" ||
|
||
effectiveVideoParams?.ratio === "3:4" ||
|
||
effectiveVideoParams?.ratio === "9:16" ||
|
||
effectiveVideoParams?.ratio === "21:9" ||
|
||
effectiveVideoParams?.ratio === "adaptive"
|
||
? effectiveVideoParams.ratio
|
||
: undefined;
|
||
writeAgentProgressChunk(
|
||
writer,
|
||
{
|
||
stage: "executing",
|
||
currentStep: "execute",
|
||
title: "正在生成视频",
|
||
message: "我正在按当前模式执行视频生成。",
|
||
toolName: "generateVideo",
|
||
capability: "generate_video",
|
||
},
|
||
"agent:generate_video:progress-executing",
|
||
);
|
||
const result = await executeDirectVideoGeneration({
|
||
prompt: generationPrompt,
|
||
mode,
|
||
images,
|
||
resolution: effectiveVideoParams?.resolution,
|
||
ratio: directVideoRatio,
|
||
duration: effectiveVideoParams?.duration,
|
||
preferredModelRef: finalModelId,
|
||
});
|
||
|
||
const videoUrl = String(result.videoUrl || "").trim();
|
||
if (!videoUrl) {
|
||
throw new Error("视频生成完成但未返回视频地址");
|
||
}
|
||
|
||
const assistantText = prependImageModelBadge(
|
||
[
|
||
getDirectAgentToolSuccessMessage("generate_video"),
|
||
buildAgentToolVideoMarkdown(videoUrl),
|
||
]
|
||
.filter(Boolean)
|
||
.join("\n\n"),
|
||
result.modelDisplayName,
|
||
);
|
||
|
||
await saveMessages({
|
||
messages: [
|
||
{
|
||
id: assistantMessageId,
|
||
chatId: id,
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: assistantText }],
|
||
attachments: [],
|
||
createdAt: new Date(),
|
||
videoStorageStatus: "temporary",
|
||
temporaryVideoUrl: videoUrl,
|
||
permanentVideoUrl: null,
|
||
archivedAt: null,
|
||
videoMetadata: (result.metadata || {}) as any,
|
||
},
|
||
],
|
||
});
|
||
|
||
try {
|
||
await persistVideoGenerationCompletedTurn({
|
||
userId: session.user.id,
|
||
chatId: id,
|
||
projectId: contextPayload?.projectId || null,
|
||
messageId: assistantMessageId,
|
||
text: requestTextContent,
|
||
source: "agent_direct_generate_video",
|
||
providerType: "direct-video",
|
||
modelRef: result.modelRef || finalModelId,
|
||
mode,
|
||
status: "completed",
|
||
temporaryVideoUrl: videoUrl,
|
||
});
|
||
} catch (memoryError) {
|
||
console.warn(
|
||
"[Chat API] Failed to update working state after direct video generation:",
|
||
memoryError,
|
||
);
|
||
}
|
||
|
||
await consumeGeneratedVideoPoints({
|
||
userId: session.user.id,
|
||
chatId: id,
|
||
messageId: assistantMessageId,
|
||
modelRefs: [result.modelRef || finalModelId],
|
||
duration:
|
||
Number((result.metadata as any)?.duration) ||
|
||
Number(effectiveVideoParams?.duration) ||
|
||
undefined,
|
||
});
|
||
|
||
if (videoUrl) {
|
||
scheduleVideoArchive({
|
||
chatId: id,
|
||
messageId: assistantMessageId,
|
||
text: assistantText,
|
||
temporaryVideoUrl: videoUrl,
|
||
taskId: assistantMessageId,
|
||
metadata: result.metadata,
|
||
});
|
||
}
|
||
|
||
writeAssistantTextChunk(
|
||
writer,
|
||
assistantText,
|
||
"agent:generate_video:result",
|
||
assistantMessageId,
|
||
);
|
||
if (contextPayload?.source === "canvas") {
|
||
writeCanvasMediaChunk(
|
||
writer,
|
||
{
|
||
messageId: assistantMessageId,
|
||
videoUrls: [videoUrl],
|
||
prompt: generationPrompt,
|
||
sourceImageUrls: directAgentToolTargetImageUrls,
|
||
},
|
||
"agent:generate_video:canvas-media",
|
||
);
|
||
}
|
||
writeAgentProgressChunk(
|
||
writer,
|
||
{
|
||
stage: "completed",
|
||
currentStep: "finalize",
|
||
title: "视频生成完成",
|
||
message: "结果已经写回对话和画布。",
|
||
toolName: "generateVideo",
|
||
capability: "generate_video",
|
||
},
|
||
"agent:generate_video:progress-completed",
|
||
);
|
||
} catch (directToolError) {
|
||
console.error("[Chat API] Direct agent video generation failed:", directToolError);
|
||
const errorMessage = getDirectAgentToolErrorMessage(
|
||
"generate_video",
|
||
directToolError,
|
||
);
|
||
await saveMessages({
|
||
messages: [
|
||
{
|
||
id: assistantMessageId,
|
||
chatId: id,
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: errorMessage }],
|
||
attachments: [],
|
||
createdAt: new Date(),
|
||
videoStorageStatus: null,
|
||
temporaryVideoUrl: null,
|
||
permanentVideoUrl: null,
|
||
archivedAt: null,
|
||
videoMetadata: null,
|
||
},
|
||
],
|
||
});
|
||
try {
|
||
await persistVideoGenerationFailedTurn({
|
||
userId: session.user.id,
|
||
chatId: id,
|
||
projectId: contextPayload?.projectId || null,
|
||
messageId: assistantMessageId,
|
||
text: requestTextContent,
|
||
source: "agent_direct_generate_video",
|
||
providerType: "direct-video",
|
||
modelRef: finalModelId,
|
||
status: "failed",
|
||
error:
|
||
directToolError instanceof Error
|
||
? directToolError.message
|
||
: String(directToolError),
|
||
});
|
||
} catch (videoMemoryError) {
|
||
console.warn(
|
||
"[Chat API] Failed to persist direct video failure state:",
|
||
videoMemoryError,
|
||
);
|
||
}
|
||
writeAssistantTextChunk(
|
||
writer,
|
||
errorMessage,
|
||
"agent:generate_video:error",
|
||
assistantMessageId,
|
||
);
|
||
writeAgentProgressChunk(
|
||
writer,
|
||
{
|
||
stage: "failed",
|
||
currentStep: "execute",
|
||
title: "视频生成失败",
|
||
message: errorMessage,
|
||
toolName: "generateVideo",
|
||
capability: "generate_video",
|
||
},
|
||
"agent:generate_video:progress-failed",
|
||
);
|
||
}
|
||
},
|
||
});
|
||
|
||
return createSseResponse(directVideoStream);
|
||
}
|
||
|
||
if (
|
||
shouldRunAgentConversation &&
|
||
!smartAutoNeedsClarification &&
|
||
directAgentExecution &&
|
||
(directAgentExecution.toolName === "editImage" ||
|
||
directAgentExecution.toolName === "enhanceImage" ||
|
||
directAgentExecution.toolName === "removeBackground" ||
|
||
directAgentExecution.toolName === "resizeImage" ||
|
||
directAgentExecution.toolName === "extendImage" ||
|
||
directAgentExecution.toolName === "boxCutout")
|
||
) {
|
||
const directAgentToolCapability = directAgentExecution.capability;
|
||
if (!directAgentToolTargetImageUrl) {
|
||
const missingTargetMessage = getDirectAgentToolTargetMissingMessage(
|
||
directAgentToolCapability,
|
||
);
|
||
const missingTargetStream = createUIMessageStream({
|
||
execute: async ({ writer }) => {
|
||
const messageId = generateUUID();
|
||
writeCurrentAgentTargetLockChunk(
|
||
writer,
|
||
`agent:${directAgentToolCapability}:missing-target-lock`,
|
||
);
|
||
writeAssistantTextChunk(
|
||
writer,
|
||
missingTargetMessage,
|
||
`agent:${directAgentToolCapability}:missing-target`,
|
||
messageId,
|
||
);
|
||
|
||
await saveMessages({
|
||
messages: [
|
||
{
|
||
id: messageId,
|
||
chatId: id,
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: missingTargetMessage }],
|
||
attachments: [],
|
||
createdAt: new Date(),
|
||
videoStorageStatus: null,
|
||
temporaryVideoUrl: null,
|
||
permanentVideoUrl: null,
|
||
archivedAt: null,
|
||
videoMetadata: null,
|
||
},
|
||
],
|
||
});
|
||
},
|
||
});
|
||
|
||
return createSseResponse(missingTargetStream);
|
||
}
|
||
|
||
const assistantMessageId = generateUUID();
|
||
const directToolLabel = getDirectAgentToolLabel(directAgentToolCapability);
|
||
const directToolStream = createUIMessageStream({
|
||
execute: async ({ writer }) => {
|
||
writeAgentProgressChunk(
|
||
writer,
|
||
{
|
||
stage: "planning",
|
||
currentStep: "select_tool",
|
||
title: `已选中${directToolLabel}`,
|
||
message: "我正在整理目标图和当前参数，准备开始执行。",
|
||
toolName: directAgentExecution.toolName,
|
||
capability: directAgentToolCapability,
|
||
},
|
||
`agent:${directAgentToolCapability}:progress-select-tool`,
|
||
);
|
||
writeCurrentAgentTargetLockChunk(
|
||
writer,
|
||
`agent:${directAgentToolCapability}:target-lock`,
|
||
);
|
||
writeAgentProgressChunk(
|
||
writer,
|
||
{
|
||
stage: "executing",
|
||
currentStep: "execute",
|
||
title: `正在执行${directToolLabel}`,
|
||
message: "执行完成后我会把结果直接回写到当前对话和画布。",
|
||
toolName: directAgentExecution.toolName,
|
||
capability: directAgentToolCapability,
|
||
},
|
||
`agent:${directAgentToolCapability}:progress-executing`,
|
||
);
|
||
|
||
try {
|
||
let resultOriginalUrl = "";
|
||
let resultThumbnailUrl: string | null = null;
|
||
let resultImageSize = "2K";
|
||
let resultAssetId: string | null = null;
|
||
|
||
if (directAgentToolCapability === "generic_edit") {
|
||
const directEditPrompt =
|
||
buildDirectAgentExecutionPrompt({
|
||
originalText: requestTextContent || "",
|
||
optimizedPrompt: smartAutoOptimizedPrompt,
|
||
hiddenPromptPrefix,
|
||
fallback: "在保持主体和整体构图稳定的前提下完成本轮图片编辑。",
|
||
}) || "在保持主体和整体构图稳定的前提下完成本轮图片编辑。";
|
||
const result = await generateCanvasImage({
|
||
prompt: directEditPrompt,
|
||
negativePrompt: effectiveImageParams?.negativePrompt,
|
||
model: finalModelId,
|
||
aspectRatio:
|
||
effectiveImageParams?.aspectRatio || defaultImageQualityParams.aspectRatio,
|
||
imageSize:
|
||
effectiveImageParams?.imageSize || defaultImageQualityParams.imageSize,
|
||
imageCount: 1,
|
||
referenceImages: directAgentToolTargetImageUrls,
|
||
debugLabel: "agent-direct-edit",
|
||
debugMeta: {
|
||
chatId: id,
|
||
projectId: contextPayload?.projectId || null,
|
||
subIntent: smartAutoExecution?.routeSubIntent || null,
|
||
},
|
||
});
|
||
|
||
const firstImage = result.images[0];
|
||
if (!firstImage?.originalUrl) {
|
||
throw new Error("图片编辑完成但未返回结果图");
|
||
}
|
||
|
||
resultOriginalUrl = firstImage.originalUrl;
|
||
resultThumbnailUrl = firstImage.thumbnailUrl || null;
|
||
resultImageSize =
|
||
effectiveImageParams?.imageSize || defaultImageQualityParams.imageSize || "2K";
|
||
|
||
const editResultMarkdown = buildAgentToolImagesMarkdown([
|
||
{
|
||
originalUrl: resultOriginalUrl,
|
||
thumbnailUrl: resultThumbnailUrl,
|
||
imageSize: resultImageSize,
|
||
},
|
||
]);
|
||
await saveGeneratedImages(
|
||
editResultMarkdown,
|
||
session.user.id,
|
||
id,
|
||
assistantMessageId,
|
||
directEditPrompt,
|
||
result.modelRef || finalModelId,
|
||
{
|
||
sourceAssetIds: requestTargetResolution?.targetAssetIds || undefined,
|
||
sourceImageUrls: directAgentToolTargetImageUrls,
|
||
relationType: "edited_from",
|
||
sourceType: "edited",
|
||
projectId: contextPayload?.projectId || null,
|
||
userPrompt: requestTextContent,
|
||
optimizedPrompt: directEditPrompt,
|
||
requestedImageCount: 1,
|
||
aspectRatio:
|
||
effectiveImageParams?.aspectRatio || defaultImageQualityParams.aspectRatio,
|
||
imageSize: resultImageSize,
|
||
},
|
||
);
|
||
} else if (directAgentToolCapability === "enhance") {
|
||
const result = await executeDirectAgentEnhanceJob({
|
||
requestUrl: request.url,
|
||
requestHeaders: request.headers,
|
||
imageUrl: directAgentToolTargetImageUrl,
|
||
prompt: requestTextContent || "图像增强处理",
|
||
chatId: id,
|
||
messageId: assistantMessageId,
|
||
});
|
||
|
||
resultOriginalUrl = result.originalUrl;
|
||
resultThumbnailUrl = result.thumbnailUrl;
|
||
resultImageSize = result.imageSize;
|
||
} else if (directAgentToolCapability === "remove_background") {
|
||
const sourceBuffer = await fetchImageBufferFromUrl(directAgentToolTargetImageUrl);
|
||
const result = await executeRmbgJob({
|
||
userId: session.user.id,
|
||
imageBuffer: sourceBuffer,
|
||
prompt: requestTextContent || "抠图处理",
|
||
chatId: id,
|
||
messageId: assistantMessageId,
|
||
sourceImageUrl: directAgentToolTargetImageUrl,
|
||
});
|
||
|
||
resultOriginalUrl = result.original_url;
|
||
resultThumbnailUrl = result.thumbnail_url;
|
||
resultImageSize = "2K";
|
||
resultAssetId = result.mediaAssetId || null;
|
||
} else if (directAgentToolCapability === "resize") {
|
||
const plannedArgs = findAgentToolCallArgs(
|
||
"resizeImage",
|
||
smartAutoExecution,
|
||
smartAutoPlan,
|
||
);
|
||
const inferredOptions = inferResizeOptionsFromText(requestTextContent);
|
||
const canvasWidth = toFiniteNumber(
|
||
plannedArgs?.canvasWidth ?? plannedArgs?.width ?? inferredOptions.canvasWidth,
|
||
);
|
||
const canvasHeight = toFiniteNumber(
|
||
plannedArgs?.canvasHeight ?? plannedArgs?.height ?? inferredOptions.canvasHeight,
|
||
);
|
||
|
||
if (!canvasWidth || !canvasHeight) {
|
||
throw new ResizeServiceError(
|
||
"请告诉我目标尺寸或比例，例如 1024x1024、4:5、16:9。",
|
||
400,
|
||
);
|
||
}
|
||
|
||
const transparentBackground =
|
||
plannedArgs?.transparentBackground === true ||
|
||
plannedArgs?.transparentBackground === "true" ||
|
||
inferredOptions.transparentBackground === true;
|
||
const backgroundColor =
|
||
typeof plannedArgs?.backgroundColor === "string"
|
||
? plannedArgs.backgroundColor
|
||
: inferredOptions.backgroundColor;
|
||
const mimeType =
|
||
typeof plannedArgs?.mimeType === "string" ? plannedArgs.mimeType : undefined;
|
||
const result = await executeResizeJob({
|
||
userId: session.user.id,
|
||
imageUrl: directAgentToolTargetImageUrl,
|
||
prompt: requestTextContent || "尺寸调整处理",
|
||
chatId: id,
|
||
messageId: assistantMessageId,
|
||
sourceImageUrl: directAgentToolTargetImageUrl,
|
||
canvasWidth,
|
||
canvasHeight,
|
||
transparentBackground,
|
||
backgroundColor,
|
||
mimeType,
|
||
});
|
||
|
||
resultOriginalUrl = result.original_url;
|
||
resultThumbnailUrl = result.thumbnail_url;
|
||
resultImageSize = `${result.canvas_width}x${result.canvas_height}`;
|
||
resultAssetId = result.mediaAssetId || null;
|
||
} else if (directAgentToolCapability === "extend") {
|
||
const plannedArgs = findAgentToolCallArgs(
|
||
"extendImage",
|
||
smartAutoExecution,
|
||
smartAutoPlan,
|
||
);
|
||
const inferredOptions = inferExtendOptionsFromText(requestTextContent);
|
||
const xScale = toFiniteNumber(plannedArgs?.xScale ?? inferredOptions.xScale);
|
||
const yScale = toFiniteNumber(plannedArgs?.yScale ?? inferredOptions.yScale);
|
||
const result = await executeExtendJob({
|
||
userId: session.user.id,
|
||
imageUrl: directAgentToolTargetImageUrl,
|
||
prompt: requestTextContent || "图像拓展处理",
|
||
chatId: id,
|
||
messageId: assistantMessageId,
|
||
sourceImageUrl: directAgentToolTargetImageUrl,
|
||
...(xScale ? { xScale } : {}),
|
||
...(yScale ? { yScale } : {}),
|
||
});
|
||
|
||
resultOriginalUrl = result.original_url;
|
||
resultThumbnailUrl = result.thumbnail_url;
|
||
resultImageSize = "2K";
|
||
resultAssetId = result.mediaAssetId || null;
|
||
} else if (directAgentToolCapability === "box_cutout") {
|
||
const plannedArgs = findAgentToolCallArgs(
|
||
"boxCutout",
|
||
smartAutoExecution,
|
||
smartAutoPlan,
|
||
);
|
||
const cropRect =
|
||
plannedArgs?.cropRect && typeof plannedArgs.cropRect === "object"
|
||
? (plannedArgs.cropRect as Record<string, unknown>)
|
||
: undefined;
|
||
|
||
if (!cropRect) {
|
||
throw new BoxCutoutServiceError(
|
||
"缺少框选区域，暂时无法执行框选抠图。",
|
||
400,
|
||
);
|
||
}
|
||
|
||
const result = await executeBoxCutoutJob({
|
||
userId: session.user.id,
|
||
imageUrl: directAgentToolTargetImageUrl,
|
||
cropRect: {
|
||
x: toFiniteNumber(cropRect.x),
|
||
y: toFiniteNumber(cropRect.y),
|
||
width: toFiniteNumber(cropRect.width),
|
||
height: toFiniteNumber(cropRect.height),
|
||
},
|
||
paddingRatio: toFiniteNumber(plannedArgs?.paddingRatio),
|
||
prompt: requestTextContent || "框选抠图处理",
|
||
chatId: id,
|
||
messageId: assistantMessageId,
|
||
sourceNodeId:
|
||
typeof plannedArgs?.sourceNodeId === "string"
|
||
? plannedArgs.sourceNodeId
|
||
: undefined,
|
||
});
|
||
|
||
resultOriginalUrl = result.original_url;
|
||
resultThumbnailUrl = result.thumbnail_url || null;
|
||
resultImageSize = "2K";
|
||
resultAssetId = result.mediaAssetId || null;
|
||
}
|
||
|
||
if (!resultAssetId && resultOriginalUrl) {
|
||
try {
|
||
const [resultAsset] = await getMediaAssetsByUserAndUrls({
|
||
userId: session.user.id,
|
||
urls: [resultOriginalUrl],
|
||
});
|
||
resultAssetId = resultAsset?.id || null;
|
||
} catch (assetLookupError) {
|
||
console.warn(
|
||
`[Chat API] Failed to lookup result asset after ${directAgentToolCapability}:`,
|
||
assetLookupError,
|
||
);
|
||
}
|
||
}
|
||
|
||
const assistantText = prependImageModelBadge(
|
||
[
|
||
getDirectAgentToolSuccessMessage(directAgentToolCapability),
|
||
buildAgentToolImageMarkdown({
|
||
originalUrl: resultOriginalUrl,
|
||
thumbnailUrl: resultThumbnailUrl,
|
||
imageSize: resultImageSize,
|
||
}),
|
||
]
|
||
.filter(Boolean)
|
||
.join("\n\n"),
|
||
directToolLabel,
|
||
);
|
||
|
||
await saveMessages({
|
||
messages: [
|
||
{
|
||
id: assistantMessageId,
|
||
chatId: id,
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: assistantText }],
|
||
attachments: [],
|
||
createdAt: new Date(),
|
||
videoStorageStatus: null,
|
||
temporaryVideoUrl: null,
|
||
permanentVideoUrl: null,
|
||
archivedAt: null,
|
||
videoMetadata: null,
|
||
},
|
||
],
|
||
});
|
||
|
||
try {
|
||
const resultAssetIds = resultAssetId ? [resultAssetId] : null;
|
||
const resultImageUrls = resultOriginalUrl ? [resultOriginalUrl] : null;
|
||
await persistImageWorkflowTurn({
|
||
userId: session.user.id,
|
||
chatId: id,
|
||
projectId: contextPayload?.projectId || null,
|
||
currentIntent: "image_edit",
|
||
currentSubIntent:
|
||
smartAutoExecution?.routeSubIntent || directAgentToolCapability,
|
||
currentFocusAssetIds: resultAssetIds,
|
||
currentFocusImageUrls: resultImageUrls,
|
||
lastGeneratedAssetIds: resultAssetIds,
|
||
lastGeneratedImageUrls: resultImageUrls,
|
||
lastResolvedTargetAssetIds: resultAssetIds,
|
||
lastResolvedTargetImageUrls: resultImageUrls,
|
||
lastMessageId: assistantMessageId,
|
||
lastUserPrompt: requestTextContent,
|
||
lastOptimizedPrompt: smartAutoOptimizedPrompt || undefined,
|
||
lastRequestedImageCount: resultImageUrls?.length || 1,
|
||
lastImageSize: resultImageSize || undefined,
|
||
lastSourceImageUrls: directAgentToolTargetImageUrls,
|
||
lastResultImageUrls: resultImageUrls,
|
||
});
|
||
} catch (memoryError) {
|
||
console.warn(
|
||
`[Chat API] Failed to update working state after ${directAgentToolCapability}:`,
|
||
memoryError,
|
||
);
|
||
}
|
||
|
||
if (directAgentExecution.publishToCanvas && contextPayload?.source === "canvas") {
|
||
console.log(
|
||
`[Chat API] Agent publishToCanvas satisfied by assistant media auto-sync (${directAgentToolCapability})`,
|
||
);
|
||
}
|
||
|
||
writeAssistantTextChunk(
|
||
writer,
|
||
assistantText,
|
||
`agent:${directAgentToolCapability}:success`,
|
||
assistantMessageId,
|
||
);
|
||
if (contextPayload?.source === "canvas") {
|
||
writeCanvasMediaChunk(
|
||
writer,
|
||
{
|
||
messageId: assistantMessageId,
|
||
imageUrls: resultOriginalUrl ? [resultOriginalUrl] : [],
|
||
prompt: requestTextContent || directToolLabel,
|
||
sourceImageUrls: directAgentToolTargetImageUrls,
|
||
},
|
||
`agent:${directAgentToolCapability}:canvas-media`,
|
||
);
|
||
}
|
||
writeAgentProgressChunk(
|
||
writer,
|
||
{
|
||
stage: "completed",
|
||
currentStep: "finalize",
|
||
title: `${directToolLabel}完成`,
|
||
message: "结果已经回写到当前对话和画布。",
|
||
toolName: directAgentExecution.toolName,
|
||
capability: directAgentToolCapability,
|
||
},
|
||
`agent:${directAgentToolCapability}:progress-completed`,
|
||
);
|
||
} catch (directToolError) {
|
||
console.error(
|
||
`[Chat API] Direct agent tool ${directAgentToolCapability} failed:`,
|
||
directToolError,
|
||
);
|
||
|
||
if (
|
||
!(directToolError instanceof RmbgServiceError) &&
|
||
!(directToolError instanceof ResizeServiceError) &&
|
||
!(directToolError instanceof ExtendServiceError) &&
|
||
!(directToolError instanceof BoxCutoutServiceError)
|
||
) {
|
||
console.warn(
|
||
`[Chat API] Direct agent tool ${directAgentToolCapability} threw unexpected error type`,
|
||
);
|
||
}
|
||
|
||
const errorMessage = getDirectAgentToolErrorMessage(
|
||
directAgentToolCapability,
|
||
directToolError,
|
||
);
|
||
|
||
await saveMessages({
|
||
messages: [
|
||
{
|
||
id: assistantMessageId,
|
||
chatId: id,
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: errorMessage }],
|
||
attachments: [],
|
||
createdAt: new Date(),
|
||
videoStorageStatus: null,
|
||
temporaryVideoUrl: null,
|
||
permanentVideoUrl: null,
|
||
archivedAt: null,
|
||
videoMetadata: null,
|
||
},
|
||
],
|
||
});
|
||
|
||
writeAssistantTextChunk(
|
||
writer,
|
||
errorMessage,
|
||
`agent:${directAgentToolCapability}:error`,
|
||
assistantMessageId,
|
||
);
|
||
writeAgentProgressChunk(
|
||
writer,
|
||
{
|
||
stage: "failed",
|
||
currentStep: "execute",
|
||
title: `${directToolLabel}失败`,
|
||
message: errorMessage,
|
||
toolName: directAgentExecution.toolName,
|
||
capability: directAgentToolCapability,
|
||
},
|
||
`agent:${directAgentToolCapability}:progress-failed`,
|
||
);
|
||
}
|
||
},
|
||
});
|
||
|
||
return createSseResponse(directToolStream);
|
||
}
|
||
|
||
let finalMergedUsage: AppUsage | undefined;
|
||
|
||
// 获取启用的工具列表
|
||
const enabledTools = await getEnabledTools();
|
||
|
||
// 检查是否是 UUID 格式（数据库中的模型 ID）
|
||
const isUUID =
|
||
/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test(
|
||
finalModelId
|
||
);
|
||
|
||
// 获取语言模型实例
|
||
let languageModel;
|
||
let actualModelId: string;
|
||
let resolvedModelSupportsImageGen = false;
|
||
let resolvedProviderType: string | undefined;
|
||
let resolvedRuntimeModelLookupId = finalModelId;
|
||
|
||
if (isUUID) {
|
||
// 从数据库获取动态配置的模型
|
||
let targetModelId = finalModelId;
|
||
|
||
// 🆕 检查是否是视频主模型，如果是，根据 videoParams.mode 选择子模型
|
||
const aiModelData = await getAiModelById({ id: finalModelId });
|
||
const aiModelProvider = aiModelData
|
||
? await getAiProviderById({ id: aiModelData.providerId }).catch(() => null)
|
||
: null;
|
||
console.log("[Chat API] Model data:", { id: finalModelId, isSubModel: aiModelData?.isSubModel });
|
||
|
||
if (aiModelData && !aiModelData.isSubModel && aiModelProvider?.type === "volcano-video") {
|
||
// 检查是否有子模型
|
||
const subModels = await getSubModelsByParentModelId(finalModelId);
|
||
console.log("[Chat API] Found sub-models:", subModels.length);
|
||
|
||
if (subModels.length > 0) {
|
||
// 这是一个主模型，需要根据 videoParams 选择子模型
|
||
const videoParams = effectiveVideoParams;
|
||
console.log("[Chat API] Video params:", videoParams);
|
||
|
||
// 如果没有 videoParams 或没有 mode，使用默认模式
|
||
const mode = (videoParams?.mode || "text-to-video") as VideoModeType;
|
||
console.log("[Chat API] Using mode:", mode);
|
||
|
||
const matchedSubModel = selectBestVideoSubModelForMode(subModels as any, mode);
|
||
|
||
if (matchedSubModel) {
|
||
console.log(
|
||
`[Chat API] 视频模式 ${mode} → 选择子模型: ${matchedSubModel.model.displayName} (${matchedSubModel.model.modelId})`
|
||
);
|
||
targetModelId = matchedSubModel.model.id;
|
||
} else {
|
||
console.warn(`[Chat API] 未找到可用子模型，继续使用主模型`);
|
||
}
|
||
}
|
||
}
|
||
|
||
const dynamicModel = await getLanguageModelById(targetModelId);
|
||
if (!dynamicModel) {
|
||
return new ChatSDKError(
|
||
"bad_request:api",
|
||
"Model not found or disabled"
|
||
).toResponse();
|
||
}
|
||
languageModel = dynamicModel;
|
||
resolvedRuntimeModelLookupId = targetModelId;
|
||
// 对于动态模型，我们需要从数据库获取 modelId
|
||
const finalModelData = await getAiModelById({ id: targetModelId });
|
||
actualModelId = finalModelData?.modelId ?? targetModelId;
|
||
resolvedModelSupportsImageGen = Boolean(finalModelData?.supportsImageGen);
|
||
if (finalModelData) {
|
||
resolvedProviderType = (
|
||
await getAiProviderById({ id: finalModelData.providerId }).catch(() => null)
|
||
)?.type;
|
||
}
|
||
} else {
|
||
// 使用预定义的模型
|
||
languageModel = myProvider.languageModel(finalModelId);
|
||
actualModelId = languageModel.modelId;
|
||
resolvedProviderType = (languageModel as any)?.provider;
|
||
resolvedRuntimeModelLookupId = actualModelId;
|
||
}
|
||
|
||
const effectiveConversationSystemPrompt = shouldRunAgentConversation
|
||
? `${await buildAgentRoleIdentityInstruction({
|
||
allowVideoGeneration: allowAgentVideoGeneration,
|
||
hasCanvasContext: Boolean(
|
||
effectiveContextPayload?.source === "canvas" ||
|
||
effectiveContextPayload?.selectedNodeIds?.length ||
|
||
effectiveContextPayload?.canvasImageNodes?.length,
|
||
),
|
||
})}\n\n${systemPrompt({ selectedChatModel: finalModelId, requestHints })}`
|
||
: systemPrompt({ selectedChatModel: finalModelId, requestHints });
|
||
|
||
const activeRuntimeModelDescriptor = await resolveAgentModelSlotDescriptor(
|
||
resolvedRuntimeModelLookupId,
|
||
).catch(() => null);
|
||
if (shouldRunAgentConversation && activeRuntimeModelDescriptor) {
|
||
const slotName =
|
||
smartAutoExecution?.shouldApplyImagePipeline ||
|
||
smartAutoExecution?.shouldApplyVideoPipeline
|
||
? "execution"
|
||
: "reply";
|
||
agentModelSlots = upsertAgentModelSlot(agentModelSlots, slotName, {
|
||
...activeRuntimeModelDescriptor,
|
||
source: "runtime_selected_model",
|
||
reason:
|
||
slotName === "reply"
|
||
? smartAutoExecution?.shouldStayConversational
|
||
? "conversational_reply_runtime"
|
||
: "default_reply_runtime"
|
||
: "execution_runtime_model",
|
||
locked: slotName === "reply" ? shouldKeepSelectedModel : false,
|
||
});
|
||
pendingProtocolDebugEvents.push({
|
||
phase: "modelSlots",
|
||
payload: {
|
||
modelSlots: agentModelSlots,
|
||
providerHealth: activeRuntimeModelDescriptor.providerKey
|
||
? getProviderHealthSnapshot(activeRuntimeModelDescriptor.providerKey)
|
||
: null,
|
||
},
|
||
});
|
||
}
|
||
|
||
// ========== 图片生成配额检查 ==========
|
||
const isUuidModelId = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test(
|
||
finalModelId
|
||
);
|
||
const currentModelConfig = isUuidModelId
|
||
? await getAiModelById({ id: finalModelId }).catch(() => undefined)
|
||
: undefined;
|
||
const currentProviderConfig =
|
||
currentModelConfig?.providerId
|
||
? await getAiProviderById({ id: currentModelConfig.providerId }).catch(() => null)
|
||
: null;
|
||
const runtimeProviderType = String(
|
||
resolvedProviderType || (languageModel as any).provider || ""
|
||
);
|
||
const normalizedRuntimeProviderType =
|
||
runtimeProviderType === "gemini-image"
|
||
? "gemini"
|
||
: runtimeProviderType === "huanke-gemini"
|
||
? "huanke-gemini"
|
||
: runtimeProviderType;
|
||
const isImageGenerationModel = isImageGenerationModelType(
|
||
normalizedRuntimeProviderType,
|
||
currentModelConfig
|
||
? {
|
||
modelId: currentModelConfig.modelId,
|
||
capabilities: currentModelConfig.capabilities,
|
||
supportsImageGen: currentModelConfig.supportsImageGen,
|
||
}
|
||
: undefined
|
||
);
|
||
const modelCreditCostPerImage = Number(
|
||
(currentModelConfig as any)?.imageGenConfig?.creditCostPerImage ?? 0
|
||
);
|
||
const modelCreditCostPerVideo = getModelCreditCost(currentModelConfig, "video", {
|
||
duration: Number(effectiveVideoParams?.duration) || undefined,
|
||
});
|
||
|
||
if (isImageGenerationModel) {
|
||
const quota = await checkUserImageQuota(session.user.id);
|
||
|
||
if (!quota.allowed) {
|
||
console.log(`[Chat API] ⛔ 图片配额已用完: ${quota.used}/${quota.limit}`);
|
||
return NextResponse.json(
|
||
{
|
||
error: "quota_exceeded",
|
||
message: `本月图片生成配额已用完（已使用 ${quota.used}/${quota.limit} 张），配额将于下月1日重置，如需更多配额请联系管理员。`,
|
||
used: quota.used,
|
||
limit: quota.limit,
|
||
},
|
||
{ status: 429 }
|
||
);
|
||
}
|
||
|
||
console.log(`[Chat API] ✅ 图片配额检查通过: ${quota.used}/${quota.limit ?? '无限制'}`);
|
||
|
||
const pointsConfig = await getPointsSystemConfig();
|
||
const normalizedUnitCost = Number.isFinite(modelCreditCostPerImage)
|
||
? Math.max(0, Math.round(modelCreditCostPerImage))
|
||
: 0;
|
||
|
||
if (pointsConfig.enabled && normalizedUnitCost > 0 && pointsConfig.blockOnInsufficient !== false) {
|
||
const requestedImageCount = Math.max(
|
||
1,
|
||
Math.round(
|
||
Number(
|
||
effectiveImageParams?.imageCount || defaultImageQualityParams.imageCount || 1
|
||
)
|
||
)
|
||
);
|
||
const expectedCost = normalizedUnitCost * requestedImageCount;
|
||
const userData = await getUserById({ id: session.user.id });
|
||
if (!userData?.isHighVip) {
|
||
const currentBalance = Math.max(0, Number(userData?.pointsBalance || 0));
|
||
|
||
if (currentBalance < expectedCost) {
|
||
return NextResponse.json(
|
||
{
|
||
error: "insufficient_points",
|
||
message: `积分不足，当前余额 ${currentBalance}，本次预计消耗 ${expectedCost}（${normalizedUnitCost} x ${requestedImageCount} 张）。`,
|
||
balance: currentBalance,
|
||
expectedCost,
|
||
unitCost: normalizedUnitCost,
|
||
},
|
||
{ status: 402 }
|
||
);
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
const isVideoGenerationModel = isVideoProviderType(
|
||
normalizedRuntimeProviderType,
|
||
currentModelConfig
|
||
? {
|
||
modelId: currentModelConfig.modelId,
|
||
capabilities: currentModelConfig.capabilities,
|
||
supportsImageGen: currentModelConfig.supportsImageGen,
|
||
}
|
||
: undefined
|
||
);
|
||
|
||
if (isVideoGenerationModel) {
|
||
const pointsConfig = await getPointsSystemConfig();
|
||
const normalizedVideoUnitCost = Number.isFinite(modelCreditCostPerVideo)
|
||
? Math.max(0, Math.round(modelCreditCostPerVideo))
|
||
: 0;
|
||
|
||
if (
|
||
pointsConfig.enabled &&
|
||
normalizedVideoUnitCost > 0 &&
|
||
pointsConfig.blockOnInsufficient !== false
|
||
) {
|
||
const requestedVideoDuration = Math.max(
|
||
1,
|
||
Math.round(Number(effectiveVideoParams?.duration || 0)) || 0
|
||
);
|
||
const expectedCost = normalizedVideoUnitCost;
|
||
const userData = await getUserById({ id: session.user.id });
|
||
if (!userData?.isHighVip) {
|
||
const currentBalance = Math.max(0, Number(userData?.pointsBalance || 0));
|
||
|
||
if (currentBalance < expectedCost) {
|
||
return NextResponse.json(
|
||
{
|
||
error: "insufficient_points",
|
||
message: requestedVideoDuration
|
||
? `积分不足，当前余额 ${currentBalance}，本次预计消耗 ${expectedCost}（${requestedVideoDuration} 秒视频）。`
|
||
: `积分不足，当前余额 ${currentBalance}，本次预计消耗 ${expectedCost}（1 条视频）。`,
|
||
balance: currentBalance,
|
||
expectedCost,
|
||
unitCost: normalizedVideoUnitCost,
|
||
duration: requestedVideoDuration || undefined,
|
||
},
|
||
{ status: 402 }
|
||
);
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
// ========== Gemini 图片生成独立处理 ==========
|
||
// 支持 gemini-image 和 huanke-gemini 两种图片生成 provider
|
||
const isGeminiImageProvider = ["gemini-image", "huanke-gemini"].includes((languageModel as any).provider);
|
||
if (isGeminiImageProvider && typeof (languageModel as any).doGenerate === "function") {
|
||
console.log("[Chat API] Using Gemini image generation, provider:", (languageModel as any).provider);
|
||
console.log("[Chat API] Image params:", effectiveImageParams);
|
||
|
||
// 🆕 获取备用模型配置
|
||
const currentModelConfig = await getAiModelById({ id: finalModelId });
|
||
const configuredFallbacks = [
|
||
currentModelConfig?.fallbackModelId1,
|
||
currentModelConfig?.fallbackModelId2,
|
||
].filter(Boolean) as string[];
|
||
const allModels = await getEnabledAiModelsWithProvider();
|
||
const fallbackModelIds = buildImageFallbackModelIds(allModels, {
|
||
currentProviderId: currentModelConfig?.providerId,
|
||
currentModelId: currentModelConfig?.id,
|
||
currentModelKey: actualModelId,
|
||
configuredFallbackIds: configuredFallbacks,
|
||
});
|
||
console.log("[Chat API] Fallback model IDs:", fallbackModelIds);
|
||
|
||
const geminiStream = createUIMessageStream({
|
||
execute: async ({ writer }) => {
|
||
// 🆕 如果是智能模式且有对话回复，先发送对话消息让用户看到
|
||
// 在消息末尾加上特殊标记，告诉前端"图片正在生成中"
|
||
const conversationMessage = smartAutoFriendlyMessage || null;
|
||
|
||
if (conversationMessage) {
|
||
const chatMessageId = generateUUID();
|
||
console.log("[Chat API] 💬 Gemini: Sending conversational reply first:", smartAutoFriendlyMessage);
|
||
|
||
// 发送带标记的对话消息
|
||
writeAssistantTextChunk(
|
||
writer,
|
||
conversationMessage,
|
||
"gemini:conversation",
|
||
chatMessageId,
|
||
);
|
||
|
||
// 保存对话消息到数据库（不带标记，纯对话）
|
||
await saveMessages({
|
||
messages: [
|
||
{
|
||
id: chatMessageId,
|
||
chatId: id,
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: smartAutoFriendlyMessage }],
|
||
attachments: [],
|
||
createdAt: new Date(),
|
||
videoStorageStatus: null,
|
||
temporaryVideoUrl: null,
|
||
permanentVideoUrl: null,
|
||
archivedAt: null,
|
||
videoMetadata: null,
|
||
},
|
||
],
|
||
});
|
||
}
|
||
|
||
// 🆕 尝试生成图片的函数（支持备用模型切换）
|
||
const tryGenerateImage = async (modelIdToUse: string, isRetry = false): Promise<string> => {
|
||
console.log(`[Chat API] ${isRetry ? "🔄 Retry with fallback" : "🚀 Trying"} model: ${modelIdToUse}`);
|
||
|
||
// 只传递最后一条用户消息，避免上下文历史干扰生图效果
|
||
const lastUserMessage = uiMessagesForModel.filter((m) => m.role === "user").pop();
|
||
let messagesForGemini = lastUserMessage ? [lastUserMessage] : uiMessagesForModel;
|
||
messagesForGemini = applyOptimizedPromptToLatestUserMessage(
|
||
messagesForGemini,
|
||
smartAutoOptimizedPrompt,
|
||
hiddenPromptPrefix,
|
||
);
|
||
messagesForGemini = injectHistoryImagesToLastUserMessage(messagesForGemini, smartAutoHistoryImages);
|
||
console.log("[Chat API] Gemini: Using only last user message (no history context)");
|
||
|
||
// 转换消息：gemini-image 需要 base64 格式，huanke-gemini 需要 URL 格式
|
||
const providerForConvert = (languageModel as any).provider === "huanke-gemini" ? "huanke-gemini" : "gemini";
|
||
const modelMessages = await convertMessagesWithImages(messagesForGemini, providerForConvert);
|
||
console.log("[Chat API] Message conversion provider:", providerForConvert);
|
||
|
||
// 调用 doGenerate 获取图片（如果是备用模型，传入 overrideModelId）
|
||
const result = await (languageModel as any).doGenerate({
|
||
messages: modelMessages,
|
||
imageParams:
|
||
effectiveImageParams || {
|
||
aspectRatio: defaultImageQualityParams.aspectRatio,
|
||
imageSize: defaultImageQualityParams.imageSize,
|
||
},
|
||
overrideModelId: isRetry ? modelIdToUse : undefined, // 🆕 使用备用模型 ID
|
||
});
|
||
|
||
return result.text || "图片生成失败";
|
||
};
|
||
|
||
let stopKeepalive: (() => void) | null = null;
|
||
try {
|
||
let imageMarkdown = ""; // 初始化为空字符串
|
||
let usedModelId = actualModelId;
|
||
|
||
// 🆕 图片生成真正开始时发出标记（用于前端显示占位图）
|
||
writeAssistantTextChunk(
|
||
writer,
|
||
"<!-- IMAGE_GENERATION_START -->",
|
||
"gemini:image-start",
|
||
);
|
||
if (contextPayload?.source === "canvas") {
|
||
writeImageGenerationStatusChunk(
|
||
writer,
|
||
{
|
||
phase: "processing",
|
||
providerType: (languageModel as any).provider,
|
||
modelName: currentModelConfig?.displayName || actualModelId,
|
||
imageCount: effectiveImageParams?.imageCount || defaultImageQualityParams.imageCount || 1,
|
||
},
|
||
"gemini:image-status-start",
|
||
);
|
||
}
|
||
stopKeepalive = startStreamKeepalive(writer, "gemini-image");
|
||
|
||
try {
|
||
// 首先尝试主模型
|
||
imageMarkdown = await tryGenerateImage(actualModelId, false);
|
||
} catch (primaryError) {
|
||
console.error("[Chat API] Primary model failed:", primaryError);
|
||
|
||
// 🆕 尝试备用模型（使用同一个 provider，但切换 modelId）
|
||
let fallbackSuccess = false;
|
||
for (const fallbackModelId of fallbackModelIds) {
|
||
try {
|
||
console.log(`[Chat API] 🔄 Trying fallback model: ${fallbackModelId}`);
|
||
imageMarkdown = await tryGenerateImage(fallbackModelId, true);
|
||
usedModelId = fallbackModelId;
|
||
fallbackSuccess = true;
|
||
console.log(`[Chat API] ✅ Fallback model ${fallbackModelId} succeeded!`);
|
||
break;
|
||
} catch (fallbackError) {
|
||
console.error(`[Chat API] ❌ Fallback model ${fallbackModelId} failed:`, fallbackError);
|
||
}
|
||
}
|
||
|
||
// 如果所有备用模型都失败，抛出原始错误触发火山引擎故障转移
|
||
if (!fallbackSuccess) {
|
||
throw primaryError;
|
||
}
|
||
}
|
||
|
||
console.log("[Chat API] Gemini generated, length:", imageMarkdown.length);
|
||
console.log("[Chat API] Gemini image markdown:", imageMarkdown.substring(0, 200));
|
||
console.log("[Chat API] Used model ID:", usedModelId);
|
||
|
||
const usedModelDisplayName =
|
||
resolveModelDisplayNameByModelId(allModels, usedModelId) ||
|
||
currentModelConfig?.displayName ||
|
||
actualModelId;
|
||
const imageMarkdownWithModel = prependImageModelBadge(
|
||
imageMarkdown,
|
||
usedModelDisplayName,
|
||
);
|
||
|
||
// 生成一个统一的消息 ID
|
||
const messageId = generateUUID();
|
||
|
||
// 写入消息到流 - 使用正确的 parts 格式
|
||
writeAssistantTextChunk(
|
||
writer,
|
||
imageMarkdownWithModel,
|
||
"gemini:image-result",
|
||
messageId,
|
||
);
|
||
if (contextPayload?.source === "canvas") {
|
||
writeCanvasMediaChunk(
|
||
writer,
|
||
{
|
||
messageId,
|
||
text: imageMarkdownWithModel,
|
||
prompt: requestTextContent,
|
||
sourceImageUrls: requestTargetResolution?.targetImageUrls || [],
|
||
},
|
||
"gemini:image-canvas-media",
|
||
);
|
||
}
|
||
|
||
// 🆕 图片生成结束标记（成功）
|
||
writeAssistantTextChunk(
|
||
writer,
|
||
"<!-- IMAGE_GENERATION_END -->",
|
||
"gemini:image-end-success",
|
||
);
|
||
if (contextPayload?.source === "canvas") {
|
||
writeImageGenerationStatusChunk(
|
||
writer,
|
||
{
|
||
phase: "completed",
|
||
providerType: (languageModel as any).provider,
|
||
modelName: usedModelDisplayName,
|
||
imageCount: effectiveImageParams?.imageCount || defaultImageQualityParams.imageCount || 1,
|
||
},
|
||
"gemini:image-status-completed",
|
||
);
|
||
}
|
||
|
||
// 保存消息到数据库（现在是 S3 URL，可以完整保存）
|
||
try {
|
||
await saveMessages({
|
||
messages: [
|
||
{
|
||
id: messageId,
|
||
chatId: id,
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: imageMarkdownWithModel }],
|
||
attachments: [], // 必须提供 attachments 字段（数据库 notNull）
|
||
createdAt: new Date(),
|
||
videoStorageStatus: null,
|
||
temporaryVideoUrl: null,
|
||
permanentVideoUrl: null,
|
||
archivedAt: null,
|
||
videoMetadata: null,
|
||
},
|
||
],
|
||
});
|
||
console.log("[Chat API] ✅ Gemini message saved successfully!");
|
||
|
||
// 提取用户 prompt（最后一条用户消息的文本）
|
||
const lastUserMessage = uiMessages.filter((m) => m.role === "user").pop();
|
||
const userPrompt = lastUserMessage?.parts?.find((p): p is { type: "text"; text: string } => p.type === "text")?.text || "";
|
||
|
||
// 保存图片记录到 GeneratedImage 表
|
||
await saveGeneratedImages(
|
||
imageMarkdownWithModel,
|
||
session.user.id,
|
||
id,
|
||
messageId,
|
||
userPrompt,
|
||
finalModelId,
|
||
{
|
||
sourceAssetIds: requestTargetResolution?.targetAssetIds,
|
||
sourceImageUrls: requestTargetResolution?.targetImageUrls,
|
||
relationType: requestResolvedMediaInput?.hasImage ? "edited_from" : "generated_from",
|
||
sourceType: requestResolvedMediaInput?.hasImage ? "generated" : "generated",
|
||
projectId: contextPayload?.projectId || null,
|
||
optimizedPrompt: smartAutoOptimizedPrompt || userPrompt,
|
||
requestedImageCount: effectiveImageParams?.imageCount,
|
||
aspectRatio: effectiveImageParams?.aspectRatio,
|
||
imageSize: effectiveImageParams?.imageSize,
|
||
},
|
||
);
|
||
} catch (saveError) {
|
||
console.error("[Chat API] ❌ Failed to save Gemini message to database:", saveError);
|
||
console.error("[Chat API] Message content that failed:", imageMarkdown.substring(0, 300));
|
||
}
|
||
} catch (error) {
|
||
console.error("[Chat API] Gemini generation error:", error);
|
||
console.log("[Chat API] Attempting failover to Volcano Vision...");
|
||
|
||
// 故障转移：尝试使用火山引擎模型
|
||
try {
|
||
// 查找火山引擎模型
|
||
const allModels = await getEnabledAiModelsWithProvider();
|
||
console.log("[Chat API] Total enabled models:", allModels.length);
|
||
|
||
const volcanoModel = allModels.find(
|
||
(m: any) => m.provider.type === "volcano-vision" && m.model.isEnabled
|
||
);
|
||
|
||
if (volcanoModel) {
|
||
console.log("[Chat API] Found Volcano Vision model for failover:", volcanoModel.model.id);
|
||
|
||
// 获取火山引擎模型实例
|
||
const volcanoLanguageModel = await getLanguageModelById(volcanoModel.model.id);
|
||
|
||
if (volcanoLanguageModel && typeof (volcanoLanguageModel as any).doGenerate === "function") {
|
||
console.log("[Chat API] Executing Volcano Vision failover...");
|
||
|
||
// 转换消息（火山引擎会自动处理 Base64）
|
||
const volcanoMessages = await convertMessagesWithImages(
|
||
injectHistoryImagesToLastUserMessage(uiMessagesForModel, smartAutoHistoryImages),
|
||
"volcano"
|
||
);
|
||
|
||
// 调用火山引擎生成（故障转移时只生成 1 张，保持与 Gemini 一致）
|
||
const volcanoResult = await (volcanoLanguageModel as any).doGenerate({
|
||
messages: volcanoMessages,
|
||
maxImages: 1, // 故障转移时只生成 1 张
|
||
imageParams:
|
||
effectiveImageParams || {
|
||
aspectRatio: defaultImageQualityParams.aspectRatio,
|
||
imageSize: defaultImageQualityParams.imageSize,
|
||
imageCount: defaultImageQualityParams.imageCount,
|
||
},
|
||
});
|
||
|
||
const imageMarkdown = volcanoResult.text || "图片生成失败";
|
||
const imageMarkdownWithModel = prependImageModelBadge(
|
||
imageMarkdown,
|
||
volcanoModel.model.displayName || volcanoModel.model.modelId,
|
||
);
|
||
console.log("[Chat API] Volcano Vision failover success, length:", imageMarkdown.length);
|
||
|
||
// 生成消息 ID
|
||
const messageId = generateUUID();
|
||
|
||
// 写入消息到流
|
||
writeAssistantTextChunk(
|
||
writer,
|
||
imageMarkdownWithModel,
|
||
"gemini:volcano-failover-result",
|
||
messageId,
|
||
);
|
||
if (contextPayload?.source === "canvas") {
|
||
writeCanvasMediaChunk(
|
||
writer,
|
||
{
|
||
messageId,
|
||
text: imageMarkdownWithModel,
|
||
prompt: requestTextContent,
|
||
sourceImageUrls: requestTargetResolution?.targetImageUrls || [],
|
||
},
|
||
"gemini:volcano-failover-canvas-media",
|
||
);
|
||
}
|
||
|
||
// 🆕 图片生成结束标记（故障转移成功）
|
||
writeAssistantTextChunk(
|
||
writer,
|
||
"<!-- IMAGE_GENERATION_END -->",
|
||
"gemini:volcano-failover-end",
|
||
);
|
||
if (contextPayload?.source === "canvas") {
|
||
writeImageGenerationStatusChunk(
|
||
writer,
|
||
{
|
||
phase: "completed",
|
||
providerType: "volcano-vision",
|
||
modelName: volcanoModel.model.displayName || volcanoModel.model.modelId,
|
||
imageCount: 1,
|
||
},
|
||
"gemini:volcano-failover-status-completed",
|
||
);
|
||
}
|
||
|
||
// 保存消息到数据库
|
||
try {
|
||
await saveMessages({
|
||
messages: [
|
||
{
|
||
id: messageId,
|
||
chatId: id,
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: imageMarkdownWithModel }],
|
||
attachments: [],
|
||
createdAt: new Date(),
|
||
videoStorageStatus: null,
|
||
temporaryVideoUrl: null,
|
||
permanentVideoUrl: null,
|
||
archivedAt: null,
|
||
videoMetadata: null,
|
||
},
|
||
],
|
||
});
|
||
console.log("[Chat API] ✅ Volcano Vision failover message saved!");
|
||
|
||
// 提取用户 prompt
|
||
const lastUserMessage = uiMessages.filter((m) => m.role === "user").pop();
|
||
const userPrompt = lastUserMessage?.parts?.find((p): p is { type: "text"; text: string } => p.type === "text")?.text || "";
|
||
|
||
// 保存图片记录到 GeneratedImage 表
|
||
await saveGeneratedImages(
|
||
imageMarkdownWithModel,
|
||
session.user.id,
|
||
id,
|
||
messageId,
|
||
userPrompt,
|
||
volcanoModel.model.id,
|
||
{
|
||
sourceAssetIds: requestTargetResolution?.targetAssetIds,
|
||
sourceImageUrls: requestTargetResolution?.targetImageUrls,
|
||
relationType: requestResolvedMediaInput?.hasImage ? "edited_from" : "generated_from",
|
||
sourceType: "generated",
|
||
projectId: contextPayload?.projectId || null,
|
||
optimizedPrompt: smartAutoOptimizedPrompt || userPrompt,
|
||
requestedImageCount: effectiveImageParams?.imageCount,
|
||
aspectRatio: effectiveImageParams?.aspectRatio,
|
||
imageSize: effectiveImageParams?.imageSize,
|
||
},
|
||
);
|
||
} catch (saveError) {
|
||
console.error("[Chat API] Failed to save failover message:", saveError);
|
||
}
|
||
|
||
return; // 成功，直接返回
|
||
}
|
||
}
|
||
|
||
console.log("[Chat API] No Volcano Vision model available for failover");
|
||
} catch (failoverError) {
|
||
console.error("[Chat API] Failover to Volcano Vision also failed:", failoverError);
|
||
}
|
||
|
||
// 如果故障转移也失败，返回错误消息
|
||
const errorMessageId = generateUUID();
|
||
writeAssistantTextChunk(
|
||
writer,
|
||
"图片生成失败，请稍后重试",
|
||
"gemini:error",
|
||
errorMessageId,
|
||
);
|
||
|
||
// 🆕 图片生成结束标记（失败）
|
||
writeAssistantTextChunk(
|
||
writer,
|
||
"<!-- IMAGE_GENERATION_END -->",
|
||
"gemini:image-end-failed",
|
||
);
|
||
|
||
// 尝试保存错误消息
|
||
try {
|
||
await saveMessages({
|
||
messages: [
|
||
{
|
||
id: errorMessageId,
|
||
chatId: id,
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: "图片生成失败，请稍后重试" }],
|
||
attachments: [],
|
||
createdAt: new Date(),
|
||
videoStorageStatus: null,
|
||
temporaryVideoUrl: null,
|
||
permanentVideoUrl: null,
|
||
archivedAt: null,
|
||
videoMetadata: null,
|
||
},
|
||
],
|
||
});
|
||
} catch (saveError) {
|
||
console.error("[Chat API] Failed to save error message:", saveError);
|
||
}
|
||
} finally {
|
||
stopKeepalive?.();
|
||
}
|
||
},
|
||
});
|
||
|
||
return createSseResponse(geminiStream);
|
||
}
|
||
// ========== Gemini 处理结束 ==========
|
||
|
||
// ========== 阿里云千问图片生成独立处理 ==========
|
||
const isAlibabaLikeImageProvider =
|
||
isImageGenerationModel &&
|
||
[
|
||
"alibaba-image",
|
||
"bfl-image",
|
||
"ruxa-image",
|
||
"fal-image",
|
||
"openai-image",
|
||
"midjourney",
|
||
"runway",
|
||
].includes((languageModel as any).provider);
|
||
if (isAlibabaLikeImageProvider && typeof (languageModel as any).doGenerate === "function") {
|
||
console.log("[Chat API] Using image generation provider:", (languageModel as any).provider);
|
||
console.log("[Chat API] Image params:", effectiveImageParams);
|
||
|
||
// 🆕 获取模型显示名称，用于错误提示
|
||
const alibabaModelConfig = await getAiModelById({ id: finalModelId });
|
||
const modelDisplayName = alibabaModelConfig?.displayName || actualModelId;
|
||
const configuredFallbacks = [
|
||
alibabaModelConfig?.fallbackModelId1,
|
||
alibabaModelConfig?.fallbackModelId2,
|
||
].filter(Boolean) as string[];
|
||
const allModels = await getEnabledAiModelsWithProvider();
|
||
const fallbackModelIds = buildImageFallbackModelIds(allModels, {
|
||
currentProviderId: alibabaModelConfig?.providerId,
|
||
currentModelId: alibabaModelConfig?.id,
|
||
currentModelKey: actualModelId,
|
||
configuredFallbackIds: configuredFallbacks,
|
||
});
|
||
console.log("[Chat API] Alibaba fallback model IDs:", fallbackModelIds);
|
||
|
||
const alibabaStream = createUIMessageStream({
|
||
execute: async ({ writer }) => {
|
||
let stopKeepalive: (() => void) | null = null;
|
||
try {
|
||
// 🆕 如果是智能模式且有对话回复，先发送对话消息让用户看到
|
||
const conversationMessage = smartAutoFriendlyMessage || null;
|
||
|
||
if (conversationMessage) {
|
||
const chatMessageId = generateUUID();
|
||
console.log("[Chat API] 💬 Sending conversational reply first:", smartAutoFriendlyMessage);
|
||
|
||
// 发送带标记的对话消息
|
||
writeAssistantTextChunk(
|
||
writer,
|
||
conversationMessage,
|
||
"alibaba:conversation",
|
||
chatMessageId,
|
||
);
|
||
|
||
// 保存对话消息到数据库（不带标记，纯对话）
|
||
await saveMessages({
|
||
messages: [
|
||
{
|
||
id: chatMessageId,
|
||
chatId: id,
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: smartAutoFriendlyMessage }],
|
||
attachments: [],
|
||
createdAt: new Date(),
|
||
videoStorageStatus: null,
|
||
temporaryVideoUrl: null,
|
||
permanentVideoUrl: null,
|
||
archivedAt: null,
|
||
videoMetadata: null,
|
||
},
|
||
],
|
||
});
|
||
}
|
||
|
||
// 只传递最后一条用户消息
|
||
const lastUserMessage = uiMessagesForModel.filter((m) => m.role === "user").pop();
|
||
let messagesForAlibaba = lastUserMessage ? [lastUserMessage] : uiMessagesForModel;
|
||
console.log("[Chat API] Alibaba: Using only last user message");
|
||
|
||
// 🆕 如果有智能模式优化后的提示词，替换消息中的文本（仅用于API调用）
|
||
if (smartAutoOptimizedPrompt) {
|
||
console.log("[Chat API] Using optimized prompt for API call");
|
||
messagesForAlibaba = applyOptimizedPromptToLatestUserMessage(
|
||
messagesForAlibaba,
|
||
smartAutoOptimizedPrompt,
|
||
hiddenPromptPrefix,
|
||
);
|
||
}
|
||
|
||
messagesForAlibaba = injectHistoryImagesToLastUserMessage(messagesForAlibaba, smartAutoHistoryImages);
|
||
|
||
const tryGenerateImage = async (modelIdToUse: string, isRetry: boolean) => {
|
||
// 转换消息（图片编辑模型统一使用 URL 输入）
|
||
const providerForConvert =
|
||
(languageModel as any).provider === "bfl-image"
|
||
? "bfl-image"
|
||
: (languageModel as any).provider === "ruxa-image"
|
||
? "ruxa-image"
|
||
: (languageModel as any).provider === "fal-image"
|
||
? "fal-image"
|
||
: (languageModel as any).provider === "openai-image"
|
||
? "openai-image"
|
||
: (languageModel as any).provider === "midjourney"
|
||
? "midjourney"
|
||
: (languageModel as any).provider === "runway"
|
||
? "runway"
|
||
: "alibaba-image";
|
||
const modelMessages = await convertMessagesWithImages(messagesForAlibaba, providerForConvert);
|
||
|
||
// 调用 doGenerate 获取图片，传递模型显示名称
|
||
const result = await (languageModel as any).doGenerate({
|
||
messages: modelMessages,
|
||
imageParams:
|
||
effectiveImageParams || {
|
||
aspectRatio: defaultImageQualityParams.aspectRatio,
|
||
imageSize: defaultImageQualityParams.imageSize,
|
||
imageCount: defaultImageQualityParams.imageCount,
|
||
},
|
||
modelDisplayName, // 🆕 传递模型显示名称
|
||
overrideModelId: isRetry ? modelIdToUse : undefined,
|
||
});
|
||
|
||
if (result?.finishReason === "error") {
|
||
throw new Error(result.text || "图片生成失败");
|
||
}
|
||
|
||
return result.text || "图片生成失败";
|
||
};
|
||
|
||
let imageMarkdown = "";
|
||
let usedModelId = actualModelId;
|
||
|
||
// 🆕 图片生成真正开始时发出标记（用于前端显示占位图）
|
||
writeAssistantTextChunk(
|
||
writer,
|
||
"<!-- IMAGE_GENERATION_START -->",
|
||
"alibaba:image-start",
|
||
);
|
||
if (contextPayload?.source === "canvas") {
|
||
writeImageGenerationStatusChunk(
|
||
writer,
|
||
{
|
||
phase: "processing",
|
||
providerType: (languageModel as any).provider,
|
||
modelName: alibabaModelConfig?.displayName || actualModelId,
|
||
imageCount: effectiveImageParams?.imageCount || defaultImageQualityParams.imageCount || 1,
|
||
},
|
||
"alibaba:image-status-start",
|
||
);
|
||
}
|
||
stopKeepalive = startStreamKeepalive(writer, "alibaba-image");
|
||
|
||
try {
|
||
imageMarkdown = await tryGenerateImage(actualModelId, false);
|
||
} catch (primaryError) {
|
||
console.error("[Chat API] Alibaba primary model failed:", primaryError);
|
||
|
||
let fallbackSuccess = false;
|
||
for (const fallbackModelId of fallbackModelIds) {
|
||
try {
|
||
console.log(`[Chat API] 🔄 Alibaba fallback model: ${fallbackModelId}`);
|
||
imageMarkdown = await tryGenerateImage(fallbackModelId, true);
|
||
usedModelId = fallbackModelId;
|
||
fallbackSuccess = true;
|
||
console.log(`[Chat API] ✅ Alibaba fallback model ${fallbackModelId} succeeded!`);
|
||
break;
|
||
} catch (fallbackError) {
|
||
console.error(`[Chat API] ❌ Alibaba fallback model ${fallbackModelId} failed:`, fallbackError);
|
||
}
|
||
}
|
||
|
||
if (!fallbackSuccess) {
|
||
const providerPriority: Record<string, number> = {
|
||
"volcano-vision": 0,
|
||
"alibaba-image": 1,
|
||
"bfl-image": 2,
|
||
"ruxa-image": 3,
|
||
"fal-image": 4,
|
||
"openai-image": 5,
|
||
runway: 6,
|
||
gemini: 6,
|
||
"huanke-gemini": 7,
|
||
};
|
||
const crossProviderCandidates = getImageCrossProviderCandidates(allModels, {
|
||
currentModelId: finalModelId,
|
||
currentProviderId: alibabaModelConfig?.providerId,
|
||
providerPriority,
|
||
});
|
||
|
||
for (const candidate of crossProviderCandidates) {
|
||
try {
|
||
console.log(
|
||
`[Chat API] 🔁 Cross-provider failover attempt: ${candidate.model.displayName} (${candidate.provider.type})`
|
||
);
|
||
|
||
const candidateLanguageModel = await getLanguageModelById(candidate.model.id);
|
||
if (!candidateLanguageModel || typeof (candidateLanguageModel as any).doGenerate !== "function") {
|
||
console.log(`[Chat API] Skip candidate without doGenerate: ${candidate.model.id}`);
|
||
continue;
|
||
}
|
||
|
||
const convertProvider =
|
||
candidate.provider.type === "bfl-image"
|
||
? "bfl-image"
|
||
: candidate.provider.type === "ruxa-image"
|
||
? "ruxa-image"
|
||
: candidate.provider.type === "fal-image"
|
||
? "fal-image"
|
||
: candidate.provider.type === "openai-image"
|
||
? "openai-image"
|
||
: candidate.provider.type === "midjourney"
|
||
? "midjourney"
|
||
: candidate.provider.type === "runway"
|
||
? "runway"
|
||
: candidate.provider.type === "gemini"
|
||
? "gemini"
|
||
: candidate.provider.type === "huanke-gemini"
|
||
? "huanke-gemini"
|
||
: candidate.provider.type === "volcano-vision"
|
||
? "volcano"
|
||
: "alibaba-image";
|
||
|
||
const candidateMessages = await convertMessagesWithImages(
|
||
messagesForAlibaba,
|
||
convertProvider
|
||
);
|
||
|
||
const candidateResult = await (candidateLanguageModel as any).doGenerate({
|
||
messages: candidateMessages,
|
||
imageParams:
|
||
effectiveImageParams || {
|
||
aspectRatio: defaultImageQualityParams.aspectRatio,
|
||
imageSize: defaultImageQualityParams.imageSize,
|
||
imageCount: defaultImageQualityParams.imageCount,
|
||
},
|
||
modelDisplayName: candidate.model.displayName,
|
||
overrideModelId: candidate.model.modelId,
|
||
});
|
||
|
||
if (candidateResult?.finishReason === "error") {
|
||
throw new Error(candidateResult.text || "图片生成失败");
|
||
}
|
||
|
||
imageMarkdown = candidateResult.text || "图片生成失败";
|
||
usedModelId = candidate.model.modelId;
|
||
fallbackSuccess = true;
|
||
console.log(
|
||
`[Chat API] ✅ Cross-provider failover succeeded: ${candidate.model.displayName} (${candidate.provider.type})`
|
||
);
|
||
break;
|
||
} catch (crossError) {
|
||
const crossErrorMessage = normalizeFailoverErrorMessage(crossError);
|
||
if (shouldCooldownImageFallbackModel(crossErrorMessage)) {
|
||
setImageFallbackModelCooldown(candidate.model.id, crossErrorMessage);
|
||
}
|
||
console.error(
|
||
`[Chat API] ❌ Cross-provider failover failed: ${candidate.model.displayName} (${candidate.provider.type})`,
|
||
crossError
|
||
);
|
||
}
|
||
}
|
||
}
|
||
|
||
if (!fallbackSuccess) {
|
||
throw primaryError;
|
||
}
|
||
}
|
||
|
||
console.log("[Chat API] Alibaba generated, length:", imageMarkdown.length);
|
||
console.log("[Chat API] Alibaba used model:", usedModelId);
|
||
|
||
const usedModelDisplayName =
|
||
resolveModelDisplayNameByModelId(allModels, usedModelId) ||
|
||
alibabaModelConfig?.displayName ||
|
||
actualModelId;
|
||
const imageMarkdownWithModel = prependImageModelBadge(
|
||
imageMarkdown,
|
||
usedModelDisplayName,
|
||
);
|
||
|
||
// 生成消息 ID（图片消息）
|
||
const messageId = generateUUID();
|
||
|
||
// 写入图片消息到流
|
||
writeAssistantTextChunk(
|
||
writer,
|
||
imageMarkdownWithModel,
|
||
"alibaba:image-result",
|
||
messageId,
|
||
);
|
||
if (contextPayload?.source === "canvas") {
|
||
writeCanvasMediaChunk(
|
||
writer,
|
||
{
|
||
messageId,
|
||
text: imageMarkdownWithModel,
|
||
prompt: requestTextContent,
|
||
sourceImageUrls: requestTargetResolution?.targetImageUrls || [],
|
||
},
|
||
"alibaba:image-canvas-media",
|
||
);
|
||
}
|
||
|
||
// 🆕 图片生成结束标记（成功）
|
||
writeAssistantTextChunk(
|
||
writer,
|
||
"<!-- IMAGE_GENERATION_END -->",
|
||
"alibaba:image-end-success",
|
||
);
|
||
if (contextPayload?.source === "canvas") {
|
||
writeImageGenerationStatusChunk(
|
||
writer,
|
||
{
|
||
phase: "completed",
|
||
providerType: (languageModel as any).provider,
|
||
modelName: usedModelDisplayName,
|
||
imageCount: effectiveImageParams?.imageCount || defaultImageQualityParams.imageCount || 1,
|
||
},
|
||
"alibaba:image-status-completed",
|
||
);
|
||
}
|
||
|
||
// 保存图片消息到数据库
|
||
try {
|
||
await saveMessages({
|
||
messages: [
|
||
{
|
||
id: messageId,
|
||
chatId: id,
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: imageMarkdownWithModel }],
|
||
attachments: [],
|
||
createdAt: new Date(),
|
||
videoStorageStatus: null,
|
||
temporaryVideoUrl: null,
|
||
permanentVideoUrl: null,
|
||
archivedAt: null,
|
||
videoMetadata: null,
|
||
},
|
||
],
|
||
});
|
||
console.log("[Chat API] ✅ Alibaba message saved successfully!");
|
||
|
||
// 提取用户 prompt
|
||
const userPrompt = lastUserMessage?.parts?.find((p): p is { type: "text"; text: string } => p.type === "text")?.text || "";
|
||
|
||
// 保存图片记录到 GeneratedImage 表
|
||
await saveGeneratedImages(
|
||
imageMarkdownWithModel,
|
||
session.user.id,
|
||
id,
|
||
messageId,
|
||
userPrompt,
|
||
finalModelId,
|
||
{
|
||
sourceAssetIds: requestTargetResolution?.targetAssetIds,
|
||
sourceImageUrls: requestTargetResolution?.targetImageUrls,
|
||
relationType: requestResolvedMediaInput?.hasImage ? "edited_from" : "generated_from",
|
||
sourceType: "generated",
|
||
projectId: contextPayload?.projectId || null,
|
||
optimizedPrompt: smartAutoOptimizedPrompt || userPrompt,
|
||
requestedImageCount: effectiveImageParams?.imageCount,
|
||
aspectRatio: effectiveImageParams?.aspectRatio,
|
||
imageSize: effectiveImageParams?.imageSize,
|
||
},
|
||
);
|
||
} catch (saveError) {
|
||
console.error("[Chat API] ❌ Failed to save Alibaba message:", saveError);
|
||
}
|
||
} catch (error) {
|
||
console.error("[Chat API] Alibaba generation error:", error);
|
||
|
||
// 返回错误消息
|
||
const errorMessageId = generateUUID();
|
||
const errorText = error instanceof Error ? error.message : "图片生成失败，请稍后重试";
|
||
writeAssistantTextChunk(
|
||
writer,
|
||
errorText,
|
||
"alibaba:error",
|
||
errorMessageId,
|
||
);
|
||
|
||
// 🆕 图片生成结束标记（失败）
|
||
writeAssistantTextChunk(
|
||
writer,
|
||
"<!-- IMAGE_GENERATION_END -->",
|
||
"alibaba:image-end-failed",
|
||
);
|
||
|
||
try {
|
||
await saveMessages({
|
||
messages: [
|
||
{
|
||
id: errorMessageId,
|
||
chatId: id,
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: errorText }],
|
||
attachments: [],
|
||
createdAt: new Date(),
|
||
videoStorageStatus: null,
|
||
temporaryVideoUrl: null,
|
||
permanentVideoUrl: null,
|
||
archivedAt: null,
|
||
videoMetadata: null,
|
||
},
|
||
],
|
||
});
|
||
} catch (saveError) {
|
||
console.error("[Chat API] Failed to save error message:", saveError);
|
||
}
|
||
} finally {
|
||
stopKeepalive?.();
|
||
}
|
||
},
|
||
});
|
||
|
||
return createSseResponse(alibabaStream);
|
||
}
|
||
// ========== 阿里云图片生成处理结束 ==========
|
||
|
||
// ========== 火山引擎视觉图片生成独立处理 ==========
|
||
if ((languageModel as any).provider === "volcano-vision" && typeof (languageModel as any).doGenerate === "function") {
|
||
console.log("[Chat API] Using Volcano Vision image generation");
|
||
|
||
// 🆕 检测是否有标注图片，如果有则在消息中注入系统提示
|
||
const hasAnnotation = hasAnnotatedImage(uiMessages);
|
||
let messagesForVolcano = applyOptimizedPromptToLatestUserMessage(
|
||
uiMessagesForModel,
|
||
smartAutoOptimizedPrompt,
|
||
hiddenPromptPrefix,
|
||
);
|
||
const volcanoModelConfig = await getAiModelById({ id: finalModelId });
|
||
const configuredFallbacks = [
|
||
volcanoModelConfig?.fallbackModelId1,
|
||
volcanoModelConfig?.fallbackModelId2,
|
||
].filter(Boolean) as string[];
|
||
const allModels = await getEnabledAiModelsWithProvider();
|
||
const fallbackModelIds = buildImageFallbackModelIds(allModels, {
|
||
currentProviderId: volcanoModelConfig?.providerId,
|
||
currentModelId: volcanoModelConfig?.id,
|
||
currentModelKey: actualModelId,
|
||
configuredFallbackIds: configuredFallbacks,
|
||
});
|
||
console.log("[Chat API] Volcano fallback model IDs:", fallbackModelIds);
|
||
|
||
if (hasAnnotation) {
|
||
console.log("[Chat API] 🎨 Volcano Vision: 检测到标注图片，将注入系统提示");
|
||
// 创建消息副本，修改最后一条用户消息的文本
|
||
messagesForVolcano = messagesForVolcano.map((msg, idx) => {
|
||
if (msg.role === "user" && idx === messagesForVolcano.length - 1) {
|
||
// 修改最后一条用户消息
|
||
const modifiedParts = msg.parts.map((part) => {
|
||
if (part.type === "text" && part.text) {
|
||
return {
|
||
...part,
|
||
text: ANNOTATION_SYSTEM_PROMPT + "\n\n用户请求：" + part.text,
|
||
};
|
||
}
|
||
return part;
|
||
});
|
||
return { ...msg, parts: modifiedParts };
|
||
}
|
||
return msg;
|
||
});
|
||
}
|
||
|
||
messagesForVolcano = injectHistoryImagesToLastUserMessage(messagesForVolcano, smartAutoHistoryImages);
|
||
|
||
const volcanoStream = createUIMessageStream({
|
||
execute: async ({ writer }) => {
|
||
let stopKeepalive: (() => void) | null = null;
|
||
try {
|
||
// 🆕 如果是智能模式且有对话回复，先发送对话消息让用户看到
|
||
const conversationMessage = smartAutoFriendlyMessage || null;
|
||
|
||
if (conversationMessage) {
|
||
const chatMessageId = generateUUID();
|
||
console.log("[Chat API] 💬 Volcano: Sending conversational reply first:", smartAutoFriendlyMessage);
|
||
|
||
// 发送带标记的对话消息
|
||
writeAssistantTextChunk(
|
||
writer,
|
||
conversationMessage,
|
||
"volcano:conversation",
|
||
chatMessageId,
|
||
);
|
||
|
||
// 保存对话消息到数据库（不带标记，纯对话）
|
||
await saveMessages({
|
||
messages: [
|
||
{
|
||
id: chatMessageId,
|
||
chatId: id,
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: smartAutoFriendlyMessage }],
|
||
attachments: [],
|
||
createdAt: new Date(),
|
||
videoStorageStatus: null,
|
||
temporaryVideoUrl: null,
|
||
permanentVideoUrl: null,
|
||
archivedAt: null,
|
||
videoMetadata: null,
|
||
},
|
||
],
|
||
});
|
||
}
|
||
|
||
// 转换消息，火山引擎支持 URL 和 base64
|
||
const modelMessages = await convertMessagesWithImages(messagesForVolcano, "volcano");
|
||
|
||
const tryGenerateImage = async (modelIdToUse: string, isRetry: boolean) => {
|
||
const result = await (languageModel as any).doGenerate({
|
||
messages: modelMessages,
|
||
imageParams:
|
||
effectiveImageParams || {
|
||
aspectRatio: defaultImageQualityParams.aspectRatio,
|
||
imageSize: defaultImageQualityParams.imageSize,
|
||
imageCount: defaultImageQualityParams.imageCount,
|
||
},
|
||
overrideModelId: isRetry ? modelIdToUse : undefined,
|
||
});
|
||
if (result?.finishReason === "error") {
|
||
throw new Error(result.text || "图片生成失败");
|
||
}
|
||
return result.text || "图片生成失败";
|
||
};
|
||
|
||
let imageMarkdown = "";
|
||
let usedModelId = actualModelId;
|
||
|
||
// 🆕 图片生成真正开始时发出标记（用于前端显示占位图）
|
||
writeAssistantTextChunk(
|
||
writer,
|
||
"<!-- IMAGE_GENERATION_START -->",
|
||
"volcano:image-start",
|
||
);
|
||
if (contextPayload?.source === "canvas") {
|
||
writeImageGenerationStatusChunk(
|
||
writer,
|
||
{
|
||
phase: "processing",
|
||
providerType: "volcano-vision",
|
||
modelName: volcanoModelConfig?.displayName || actualModelId,
|
||
imageCount: effectiveImageParams?.imageCount || defaultImageQualityParams.imageCount || 1,
|
||
},
|
||
"volcano:image-status-start",
|
||
);
|
||
}
|
||
stopKeepalive = startStreamKeepalive(writer, "volcano-image");
|
||
try {
|
||
imageMarkdown = await tryGenerateImage(actualModelId, false);
|
||
} catch (primaryError) {
|
||
console.error("[Chat API] Volcano primary model failed:", primaryError);
|
||
let fallbackSuccess = false;
|
||
for (const fallbackModelId of fallbackModelIds) {
|
||
try {
|
||
console.log(`[Chat API] 🔄 Volcano fallback model: ${fallbackModelId}`);
|
||
imageMarkdown = await tryGenerateImage(fallbackModelId, true);
|
||
usedModelId = fallbackModelId;
|
||
fallbackSuccess = true;
|
||
console.log(`[Chat API] ✅ Volcano fallback model ${fallbackModelId} succeeded!`);
|
||
break;
|
||
} catch (fallbackError) {
|
||
console.error(`[Chat API] ❌ Volcano fallback model ${fallbackModelId} failed:`, fallbackError);
|
||
}
|
||
}
|
||
if (!fallbackSuccess) {
|
||
const providerPriority: Record<string, number> = {
|
||
"alibaba-image": 0,
|
||
"bfl-image": 1,
|
||
"ruxa-image": 2,
|
||
"fal-image": 3,
|
||
"openai-image": 4,
|
||
runway: 5,
|
||
gemini: 5,
|
||
"huanke-gemini": 6,
|
||
"volcano-vision": 7,
|
||
};
|
||
const crossProviderCandidates = getImageCrossProviderCandidates(allModels, {
|
||
currentModelId: finalModelId,
|
||
currentProviderId: volcanoModelConfig?.providerId,
|
||
providerPriority,
|
||
});
|
||
|
||
for (const candidate of crossProviderCandidates) {
|
||
try {
|
||
console.log(
|
||
`[Chat API] 🔁 Volcano cross-provider failover attempt: ${candidate.model.displayName} (${candidate.provider.type})`
|
||
);
|
||
|
||
const candidateLanguageModel = await getLanguageModelById(candidate.model.id);
|
||
if (!candidateLanguageModel || typeof (candidateLanguageModel as any).doGenerate !== "function") {
|
||
console.log(`[Chat API] Skip candidate without doGenerate: ${candidate.model.id}`);
|
||
continue;
|
||
}
|
||
|
||
const convertProvider =
|
||
candidate.provider.type === "bfl-image"
|
||
? "bfl-image"
|
||
: candidate.provider.type === "ruxa-image"
|
||
? "ruxa-image"
|
||
: candidate.provider.type === "fal-image"
|
||
? "fal-image"
|
||
: candidate.provider.type === "openai-image"
|
||
? "openai-image"
|
||
: candidate.provider.type === "midjourney"
|
||
? "midjourney"
|
||
: candidate.provider.type === "runway"
|
||
? "runway"
|
||
: candidate.provider.type === "gemini"
|
||
? "gemini"
|
||
: candidate.provider.type === "huanke-gemini"
|
||
? "huanke-gemini"
|
||
: candidate.provider.type === "volcano-vision"
|
||
? "volcano"
|
||
: "alibaba-image";
|
||
|
||
const candidateMessages = await convertMessagesWithImages(
|
||
messagesForVolcano,
|
||
convertProvider
|
||
);
|
||
|
||
const candidateResult = await (candidateLanguageModel as any).doGenerate({
|
||
messages: candidateMessages,
|
||
imageParams:
|
||
effectiveImageParams || {
|
||
aspectRatio: defaultImageQualityParams.aspectRatio,
|
||
imageSize: defaultImageQualityParams.imageSize,
|
||
imageCount: defaultImageQualityParams.imageCount,
|
||
},
|
||
modelDisplayName: candidate.model.displayName,
|
||
overrideModelId: candidate.model.modelId,
|
||
});
|
||
|
||
if (candidateResult?.finishReason === "error") {
|
||
throw new Error(candidateResult.text || "图片生成失败");
|
||
}
|
||
|
||
imageMarkdown = candidateResult.text || "图片生成失败";
|
||
usedModelId = candidate.model.modelId;
|
||
fallbackSuccess = true;
|
||
console.log(
|
||
`[Chat API] ✅ Volcano cross-provider failover succeeded: ${candidate.model.displayName} (${candidate.provider.type})`
|
||
);
|
||
break;
|
||
} catch (crossError) {
|
||
const crossErrorMessage = normalizeFailoverErrorMessage(crossError);
|
||
if (shouldCooldownImageFallbackModel(crossErrorMessage)) {
|
||
setImageFallbackModelCooldown(candidate.model.id, crossErrorMessage);
|
||
}
|
||
console.error(
|
||
`[Chat API] ❌ Volcano cross-provider failover failed: ${candidate.model.displayName} (${candidate.provider.type})`,
|
||
crossError
|
||
);
|
||
}
|
||
}
|
||
}
|
||
if (!fallbackSuccess) {
|
||
throw primaryError;
|
||
}
|
||
}
|
||
console.log("[Chat API] Volcano Vision generated, length:", imageMarkdown.length);
|
||
console.log("[Chat API] Volcano Vision image markdown:", imageMarkdown.substring(0, 200));
|
||
|
||
const usedModelDisplayName =
|
||
resolveModelDisplayNameByModelId(allModels, usedModelId) ||
|
||
volcanoModelConfig?.displayName ||
|
||
actualModelId;
|
||
const imageMarkdownWithModel = prependImageModelBadge(
|
||
imageMarkdown,
|
||
usedModelDisplayName,
|
||
);
|
||
|
||
// 生成消息 ID（图片消息）
|
||
const messageId = generateUUID();
|
||
|
||
// 写入图片消息到流
|
||
writeAssistantTextChunk(
|
||
writer,
|
||
imageMarkdownWithModel,
|
||
"volcano:image-result",
|
||
messageId,
|
||
);
|
||
if (contextPayload?.source === "canvas") {
|
||
writeCanvasMediaChunk(
|
||
writer,
|
||
{
|
||
messageId,
|
||
text: imageMarkdownWithModel,
|
||
prompt: requestTextContent,
|
||
sourceImageUrls: requestTargetResolution?.targetImageUrls || [],
|
||
},
|
||
"volcano:image-canvas-media",
|
||
);
|
||
}
|
||
|
||
// 🆕 图片生成结束标记（成功）
|
||
writeAssistantTextChunk(
|
||
writer,
|
||
"<!-- IMAGE_GENERATION_END -->",
|
||
"volcano:image-end-success",
|
||
);
|
||
if (contextPayload?.source === "canvas") {
|
||
writeImageGenerationStatusChunk(
|
||
writer,
|
||
{
|
||
phase: "completed",
|
||
providerType: "volcano-vision",
|
||
modelName: usedModelDisplayName,
|
||
imageCount: effectiveImageParams?.imageCount || defaultImageQualityParams.imageCount || 1,
|
||
},
|
||
"volcano:image-status-completed",
|
||
);
|
||
}
|
||
|
||
// 保存图片消息到数据库
|
||
try {
|
||
await saveMessages({
|
||
messages: [
|
||
{
|
||
id: messageId,
|
||
chatId: id,
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: imageMarkdownWithModel }],
|
||
attachments: [], // 必须提供 attachments 字段（数据库 notNull）
|
||
createdAt: new Date(),
|
||
videoStorageStatus: null,
|
||
temporaryVideoUrl: null,
|
||
permanentVideoUrl: null,
|
||
archivedAt: null,
|
||
videoMetadata: null,
|
||
},
|
||
],
|
||
});
|
||
console.log("[Chat API] ✅ Volcano Vision message saved successfully!");
|
||
|
||
// 提取用户 prompt
|
||
const lastUserMessage = uiMessages.filter((m) => m.role === "user").pop();
|
||
const userPrompt = lastUserMessage?.parts?.find((p): p is { type: "text"; text: string } => p.type === "text")?.text || "";
|
||
|
||
// 保存图片记录到 GeneratedImage 表
|
||
await saveGeneratedImages(
|
||
imageMarkdownWithModel,
|
||
session.user.id,
|
||
id,
|
||
messageId,
|
||
userPrompt,
|
||
finalModelId,
|
||
{
|
||
sourceAssetIds: requestTargetResolution?.targetAssetIds,
|
||
sourceImageUrls: requestTargetResolution?.targetImageUrls,
|
||
relationType: requestResolvedMediaInput?.hasImage ? "edited_from" : "generated_from",
|
||
sourceType: "generated",
|
||
projectId: contextPayload?.projectId || null,
|
||
optimizedPrompt: smartAutoOptimizedPrompt || userPrompt,
|
||
requestedImageCount: effectiveImageParams?.imageCount,
|
||
aspectRatio: effectiveImageParams?.aspectRatio,
|
||
imageSize: effectiveImageParams?.imageSize,
|
||
},
|
||
);
|
||
} catch (saveError) {
|
||
console.error("[Chat API] ❌ Failed to save Volcano Vision message to database:", saveError);
|
||
console.error("[Chat API] Message content that failed:", imageMarkdown.substring(0, 300));
|
||
}
|
||
} catch (error) {
|
||
console.error("[Chat API] Volcano Vision generation error:", error);
|
||
const errorMessageId = generateUUID();
|
||
writeAssistantTextChunk(
|
||
writer,
|
||
"图片生成失败，请稍后重试",
|
||
"volcano:error",
|
||
errorMessageId,
|
||
);
|
||
|
||
// 🆕 图片生成结束标记（失败）
|
||
writeAssistantTextChunk(
|
||
writer,
|
||
"<!-- IMAGE_GENERATION_END -->",
|
||
"volcano:image-end-failed",
|
||
);
|
||
|
||
// 尝试保存错误消息
|
||
try {
|
||
await saveMessages({
|
||
messages: [
|
||
{
|
||
id: errorMessageId,
|
||
chatId: id,
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: "图片生成失败，请稍后重试" }],
|
||
attachments: [], // 必须提供 attachments 字段
|
||
createdAt: new Date(),
|
||
videoStorageStatus: null,
|
||
temporaryVideoUrl: null,
|
||
permanentVideoUrl: null,
|
||
archivedAt: null,
|
||
videoMetadata: null,
|
||
},
|
||
],
|
||
});
|
||
} catch (saveError) {
|
||
console.error("[Chat API] Failed to save error message:", saveError);
|
||
}
|
||
} finally {
|
||
stopKeepalive?.();
|
||
}
|
||
},
|
||
});
|
||
|
||
return createSseResponse(volcanoStream);
|
||
}
|
||
// ========== 火山引擎视觉处理结束 ==========
|
||
|
||
// ========== 千问/FAL/RUXA 视频生成独立处理 ==========
|
||
if (
|
||
["qianwen-video", "fal-video", "ruxa-video", "runway"].includes((languageModel as any).provider) &&
|
||
typeof (languageModel as any).doGenerate === "function"
|
||
) {
|
||
const currentVideoProvider = (languageModel as any).provider as
|
||
| "qianwen-video"
|
||
| "fal-video"
|
||
| "ruxa-video"
|
||
| "runway";
|
||
console.log(`[Chat API] Using ${currentVideoProvider} video generation`);
|
||
|
||
const videoGenerationStream = createUIMessageStream({
|
||
execute: async ({ writer }) => {
|
||
try {
|
||
const conversationMessage = smartAutoFriendlyMessage || null;
|
||
if (conversationMessage) {
|
||
const chatMessageId = generateUUID();
|
||
writer.write({
|
||
type: "data-appendMessage",
|
||
data: JSON.stringify({
|
||
id: chatMessageId,
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: conversationMessage }],
|
||
}),
|
||
});
|
||
|
||
await saveMessages({
|
||
messages: [
|
||
{
|
||
id: chatMessageId,
|
||
chatId: id,
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: conversationMessage }],
|
||
attachments: [],
|
||
createdAt: new Date(),
|
||
videoStorageStatus: null,
|
||
temporaryVideoUrl: null,
|
||
permanentVideoUrl: null,
|
||
archivedAt: null,
|
||
videoMetadata: null,
|
||
},
|
||
],
|
||
});
|
||
}
|
||
|
||
const messagesForVideo = injectHistoryImagesToLastUserMessage(
|
||
applyOptimizedPromptToLatestUserMessage(
|
||
uiMessagesForModel,
|
||
smartAutoOptimizedPrompt,
|
||
hiddenPromptPrefix,
|
||
),
|
||
smartAutoHistoryImages,
|
||
);
|
||
|
||
// 千问/FAL 视频直接使用原始 URL，不转换为 base64
|
||
const modelMessages = convertToModelMessages(messagesForVideo);
|
||
console.log(`[Chat API] ${currentVideoProvider} messages prepared, count:`, modelMessages.length);
|
||
|
||
let videoMarkdown = "视频生成失败";
|
||
let tempVideoUrl = "";
|
||
let billedModelRefs: string[] = [finalModelId];
|
||
let videoMetadata: Record<string, any> = {
|
||
provider: currentVideoProvider,
|
||
model: finalModelId,
|
||
mode: effectiveVideoParams?.mode,
|
||
resolution: effectiveVideoParams?.resolution,
|
||
ratio: effectiveVideoParams?.ratio,
|
||
duration: effectiveVideoParams?.duration,
|
||
status: "succeeded",
|
||
updatedAt: Date.now(),
|
||
};
|
||
|
||
let usedStoryboardPipeline = false;
|
||
if (currentVideoProvider === "ruxa-video") {
|
||
const storyboardConfig = await getVideoStoryboardConfig().catch(() => null);
|
||
if (storyboardConfig?.enabled) {
|
||
const storyboardModelCandidates = await resolveStoryboardModelCandidates(finalModelId);
|
||
const runtimeModelId = String((languageModel as any).modelId || "").trim();
|
||
const selectedModelId =
|
||
runtimeModelId ||
|
||
storyboardModelCandidates.find((item) => item.id === finalModelId)?.modelId ||
|
||
"";
|
||
|
||
const storyboardResult = await runRuxaNineGridPipeline({
|
||
userId: session.user.id,
|
||
chatId: id,
|
||
modelMessages: modelMessages as any,
|
||
selectedModelId,
|
||
selectedModelRef: finalModelId,
|
||
modelCandidates: storyboardModelCandidates,
|
||
videoParams: effectiveVideoParams as any,
|
||
config: storyboardConfig,
|
||
doGenerate: (input) =>
|
||
(languageModel as any).doGenerate({
|
||
messages: input.messages,
|
||
videoParams: input.videoParams,
|
||
overrideModelId: input.overrideModelId,
|
||
}),
|
||
onProgress: (event) => {
|
||
safeWriteStreamChunk(
|
||
writer,
|
||
{
|
||
type: "data-videoStoryboard",
|
||
data: JSON.stringify(event),
|
||
},
|
||
`ruxa-video:storyboard:${event.stage}`,
|
||
);
|
||
},
|
||
});
|
||
|
||
if (storyboardResult.triggered) {
|
||
usedStoryboardPipeline = true;
|
||
videoMarkdown = storyboardResult.markdown || "视频生成失败，请稍后重试";
|
||
tempVideoUrl = storyboardResult.videoUrls?.[0] || "";
|
||
const isAwaitingConfirmation =
|
||
storyboardResult.reason === "awaiting_user_confirmation";
|
||
billedModelRefs =
|
||
storyboardResult.modelRefs && storyboardResult.modelRefs.length > 0
|
||
? storyboardResult.modelRefs
|
||
: [finalModelId];
|
||
|
||
const successCount = storyboardResult.videoUrls?.length || 0;
|
||
videoMetadata = {
|
||
...videoMetadata,
|
||
...(storyboardResult.metadata || {}),
|
||
status: isAwaitingConfirmation
|
||
? "awaiting_user_confirmation"
|
||
: successCount > 0
|
||
? "succeeded"
|
||
: "failed",
|
||
storyboard: {
|
||
enabled: true,
|
||
taskId: storyboardResult.taskId,
|
||
confidence: storyboardResult.confidence,
|
||
successCount,
|
||
awaitingConfirmation: isAwaitingConfirmation,
|
||
},
|
||
updatedAt: Date.now(),
|
||
};
|
||
}
|
||
}
|
||
}
|
||
|
||
if (!usedStoryboardPipeline) {
|
||
const result = await (languageModel as any).doGenerate({
|
||
messages: modelMessages,
|
||
videoParams: effectiveVideoParams,
|
||
});
|
||
|
||
if (result?.finishReason === "error") {
|
||
throw new Error(result.text || "视频生成失败");
|
||
}
|
||
|
||
videoMarkdown = result.text || "视频生成失败";
|
||
tempVideoUrl = extractVideoUrlFromText(videoMarkdown) || "";
|
||
}
|
||
|
||
console.log(
|
||
`[Chat API] ${currentVideoProvider} video generated, length:`,
|
||
videoMarkdown.length
|
||
);
|
||
|
||
// 生成一个统一的消息 ID
|
||
const messageId = generateUUID();
|
||
|
||
// 写入消息到流
|
||
writer.write({
|
||
type: "data-appendMessage",
|
||
data: JSON.stringify({
|
||
id: messageId,
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: videoMarkdown }],
|
||
attachments: [],
|
||
}),
|
||
});
|
||
|
||
// 保存消息到数据库
|
||
try {
|
||
await saveMessages({
|
||
messages: [
|
||
{
|
||
id: messageId,
|
||
chatId: id,
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: videoMarkdown }],
|
||
attachments: [],
|
||
createdAt: new Date(),
|
||
videoStorageStatus: tempVideoUrl ? "temporary" : null,
|
||
temporaryVideoUrl: tempVideoUrl || null,
|
||
permanentVideoUrl: null,
|
||
archivedAt: null,
|
||
videoMetadata: videoMetadata as any,
|
||
},
|
||
],
|
||
});
|
||
const storyboardTaskId = String((videoMetadata as any)?.taskId || "").trim();
|
||
if (storyboardTaskId) {
|
||
await attachVideoStoryboardTaskMessage({
|
||
taskId: storyboardTaskId,
|
||
messageId,
|
||
}).catch((taskAttachError) => {
|
||
console.warn(
|
||
"[Chat API] Failed to attach storyboard task with message:",
|
||
taskAttachError
|
||
);
|
||
});
|
||
}
|
||
console.log(`[Chat API] ✅ ${currentVideoProvider} video message saved successfully!`);
|
||
|
||
if ((videoMetadata as any)?.storyboard?.enabled) {
|
||
try {
|
||
const storyboardGridType = String((videoMetadata as any)?.gridType || "").trim();
|
||
const storyboardShotCount = Number((videoMetadata as any)?.shotCount || 0);
|
||
await persistStoryboardAssistantTurn({
|
||
userId: session.user.id,
|
||
chatId: id,
|
||
projectId: contextPayload?.projectId || null,
|
||
lastMessageId: messageId,
|
||
text: requestTextContent,
|
||
storyboardGridType:
|
||
storyboardGridType === "grid_2x2" ||
|
||
storyboardGridType === "grid_3x2" ||
|
||
storyboardGridType === "grid_3x3"
|
||
? (storyboardGridType as "grid_2x2" | "grid_3x2" | "grid_3x3")
|
||
: undefined,
|
||
storyboardShotCount:
|
||
storyboardShotCount === 4 ||
|
||
storyboardShotCount === 6 ||
|
||
storyboardShotCount === 9
|
||
? storyboardShotCount
|
||
: undefined,
|
||
});
|
||
} catch (storyboardMemoryPersistError) {
|
||
console.warn(
|
||
"[Chat API] Failed to persist storyboard memory after assistant response:",
|
||
storyboardMemoryPersistError
|
||
);
|
||
}
|
||
} else {
|
||
try {
|
||
await persistVideoGenerationCompletedTurn({
|
||
userId: session.user.id,
|
||
chatId: id,
|
||
projectId: contextPayload?.projectId || null,
|
||
messageId,
|
||
text: requestTextContent,
|
||
source: "native_video_provider",
|
||
providerType: currentVideoProvider,
|
||
modelRef: finalModelId,
|
||
mode:
|
||
typeof effectiveVideoParams?.mode === "string"
|
||
? effectiveVideoParams.mode
|
||
: undefined,
|
||
taskId: storyboardTaskId || undefined,
|
||
status:
|
||
typeof (videoMetadata as any)?.status === "string"
|
||
? (videoMetadata as any).status
|
||
: tempVideoUrl
|
||
? "completed"
|
||
: undefined,
|
||
temporaryVideoUrl: tempVideoUrl || undefined,
|
||
});
|
||
} catch (videoWorkflowPersistError) {
|
||
console.warn(
|
||
"[Chat API] Failed to persist native video workflow state:",
|
||
videoWorkflowPersistError,
|
||
);
|
||
}
|
||
}
|
||
const shouldConsumeVideoPoints =
|
||
tempVideoUrl &&
|
||
(videoMetadata as any)?.status !== "awaiting_user_confirmation";
|
||
if (shouldConsumeVideoPoints) {
|
||
await consumeGeneratedVideoPoints({
|
||
userId: session.user.id,
|
||
chatId: id,
|
||
messageId,
|
||
modelRefs: billedModelRefs,
|
||
duration: Number(effectiveVideoParams?.duration) || undefined,
|
||
});
|
||
}
|
||
|
||
if (tempVideoUrl) {
|
||
scheduleVideoArchive({
|
||
chatId: id,
|
||
messageId,
|
||
text: videoMarkdown,
|
||
temporaryVideoUrl: tempVideoUrl,
|
||
taskId: messageId,
|
||
metadata: videoMetadata,
|
||
});
|
||
}
|
||
} catch (saveError) {
|
||
console.error(`[Chat API] ❌ Failed to save ${currentVideoProvider} video message:`, saveError);
|
||
}
|
||
} catch (error) {
|
||
console.error(`[Chat API] ${currentVideoProvider} video generation error:`, error);
|
||
const errorMessageId = generateUUID();
|
||
const errorText = "视频生成失败，请稍后重试";
|
||
writer.write({
|
||
type: "data-appendMessage",
|
||
data: JSON.stringify({
|
||
id: errorMessageId,
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: errorText }],
|
||
attachments: [],
|
||
}),
|
||
});
|
||
try {
|
||
await saveMessages({
|
||
messages: [
|
||
{
|
||
id: errorMessageId,
|
||
chatId: id,
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: errorText }],
|
||
attachments: [],
|
||
createdAt: new Date(),
|
||
videoStorageStatus: null,
|
||
temporaryVideoUrl: null,
|
||
permanentVideoUrl: null,
|
||
archivedAt: null,
|
||
videoMetadata: null,
|
||
},
|
||
],
|
||
});
|
||
await persistVideoGenerationFailedTurn({
|
||
userId: session.user.id,
|
||
chatId: id,
|
||
projectId: contextPayload?.projectId || null,
|
||
messageId: errorMessageId,
|
||
text: requestTextContent,
|
||
source: "native_video_provider",
|
||
providerType: currentVideoProvider,
|
||
modelRef: finalModelId,
|
||
mode:
|
||
typeof effectiveVideoParams?.mode === "string"
|
||
? effectiveVideoParams.mode
|
||
: undefined,
|
||
status: "failed",
|
||
error: error instanceof Error ? error.message : String(error),
|
||
});
|
||
} catch (videoFailurePersistError) {
|
||
console.warn(
|
||
"[Chat API] Failed to persist native video failure state:",
|
||
videoFailurePersistError,
|
||
);
|
||
}
|
||
}
|
||
},
|
||
});
|
||
|
||
return createSseResponse(videoGenerationStream);
|
||
}
|
||
// ========== 千问/FAL/RUXA 视频处理结束 ==========
|
||
|
||
let stopStreamKeepalive: (() => void) | null = null;
|
||
const stream = createUIMessageStream({
|
||
execute: async ({ writer: dataStream }) => {
|
||
const stopDefaultKeepalive = startStreamKeepalive(
|
||
dataStream,
|
||
"chat-default",
|
||
DEFAULT_STREAM_KEEPALIVE_INTERVAL_MS
|
||
);
|
||
stopStreamKeepalive = stopDefaultKeepalive;
|
||
setConversationStateProtocolEventObserver((event) => {
|
||
writeProtocolDebugChunk(
|
||
dataStream,
|
||
Boolean(debugProtocol),
|
||
{
|
||
phase: "event",
|
||
type: event.type,
|
||
payload: event.payload,
|
||
},
|
||
`protocol:event:${event.type}`
|
||
);
|
||
});
|
||
safeWriteStreamChunk(
|
||
dataStream,
|
||
{
|
||
type: "data-keepalive",
|
||
data: JSON.stringify({ ts: Date.now(), label: "chat-default:init" }),
|
||
},
|
||
"chat-default:init"
|
||
);
|
||
writeProtocolDebugChunk(
|
||
dataStream,
|
||
Boolean(debugProtocol),
|
||
{
|
||
phase: "request",
|
||
chatId: id,
|
||
selectedChatModel,
|
||
hasHiddenPrompt: Boolean(hiddenPrompt),
|
||
hasAutoContext: Boolean(autoContext),
|
||
},
|
||
"protocol:request"
|
||
);
|
||
writeAgentProgressChunk(
|
||
dataStream,
|
||
{
|
||
stage: "planning",
|
||
currentStep: "analyze",
|
||
title: "正在理解你的需求",
|
||
message: "我会先结合当前图、历史结果和画布上下文整理执行方案。",
|
||
toolName: "agentConversation",
|
||
capability: smartAutoExecution?.preferredCapability || undefined,
|
||
},
|
||
"chat-default:progress-planning",
|
||
);
|
||
writeCurrentAgentTargetLockChunk(dataStream, "chat-default:target-lock");
|
||
for (const event of pendingProtocolDebugEvents) {
|
||
writeProtocolDebugChunk(
|
||
dataStream,
|
||
Boolean(debugProtocol),
|
||
{
|
||
phase: event.phase,
|
||
...event.payload,
|
||
},
|
||
`protocol:${event.phase}`
|
||
);
|
||
}
|
||
try {
|
||
// 构建工具对象，只包含启用的工具
|
||
const tools: any = {};
|
||
const activeTools: string[] = [];
|
||
|
||
if (enabledTools.includes("getWeather")) {
|
||
tools.getWeather = getWeather;
|
||
activeTools.push("getWeather");
|
||
}
|
||
if (enabledTools.includes("createDocument")) {
|
||
tools.createDocument = createDocument({ session, dataStream });
|
||
activeTools.push("createDocument");
|
||
}
|
||
if (enabledTools.includes("updateDocument")) {
|
||
tools.updateDocument = updateDocument({ session, dataStream });
|
||
activeTools.push("updateDocument");
|
||
}
|
||
if (enabledTools.includes("requestSuggestions")) {
|
||
tools.requestSuggestions = requestSuggestions({
|
||
session,
|
||
dataStream,
|
||
});
|
||
activeTools.push("requestSuggestions");
|
||
}
|
||
const shouldEnableGenerateVideoTool =
|
||
enabledTools.includes("generateVideo") &&
|
||
(!shouldRunAgentConversation || allowAgentVideoGeneration);
|
||
if (shouldEnableGenerateVideoTool) {
|
||
tools.generateVideo = generateVideo({ dataStream });
|
||
activeTools.push("generateVideo");
|
||
} else if (enabledTools.includes("generateVideo") && shouldRunAgentConversation) {
|
||
console.log("[Smart Auto Mode] 🚫 generateVideo tool disabled (manual video mode not selected)");
|
||
}
|
||
|
||
console.log("[Chat API] Calling streamText with model:", (languageModel as any).modelId, (languageModel as any).provider); // 调试日志
|
||
console.log("[Chat API] Provider type:", (languageModel as any).provider); // 调试日志
|
||
|
||
// 转换消息，对于 Gemini 和千问视频将图片 URL 转换为 base64
|
||
const convertedMessages = await convertMessagesWithImages(
|
||
injectHistoryImagesToLastUserMessage(uiMessagesForModel, smartAutoHistoryImages),
|
||
{
|
||
providerType: (languageModel as any).provider || "",
|
||
providerBaseUrl: currentProviderConfig?.baseUrl || null,
|
||
modelId: currentModelConfig?.modelId || actualModelId,
|
||
displayName: currentModelConfig?.displayName || actualModelId,
|
||
forceInlineImageUrls: shouldForceInlineVisionInputs({
|
||
providerType: resolvedProviderType || currentProviderConfig?.type || runtimeProviderType,
|
||
providerBaseUrl: currentProviderConfig?.baseUrl || null,
|
||
modelId: currentModelConfig?.modelId || actualModelId,
|
||
displayName: currentModelConfig?.displayName || actualModelId,
|
||
}),
|
||
loggerPrefix: "[Chat API] Stream text vision inline",
|
||
}
|
||
);
|
||
console.log("[Chat API] Messages converted, count:", convertedMessages.length); // 调试日志
|
||
|
||
// 🆕 检测是否是视频模型
|
||
const providerType = (languageModel as any).provider;
|
||
const effectiveProviderType = resolvedProviderType || providerType;
|
||
console.log("[Chat API] Provider type for tool detection:", {
|
||
runtime: providerType,
|
||
resolved: resolvedProviderType,
|
||
effective: effectiveProviderType,
|
||
});
|
||
const isVolcanoVideoModel =
|
||
effectiveProviderType === "volcano-video" ||
|
||
effectiveProviderType === "doubao-video" ||
|
||
actualModelId.includes("seedance");
|
||
const isOpenAICompatImageModel =
|
||
resolvedModelSupportsImageGen &&
|
||
providerType === "openai.chat" &&
|
||
!isVolcanoVideoModel;
|
||
console.log("[Chat API] Is volcano video model:", isVolcanoVideoModel);
|
||
|
||
// 🆕 处理 OpenAI 兼容的生图模型输出（防止返回 data:image;base64 原文）
|
||
if (isOpenAICompatImageModel) {
|
||
console.log("[Chat API] Using generateText for OpenAI-compatible image model");
|
||
const configuredFallbacks = [
|
||
currentModelConfig?.fallbackModelId1,
|
||
currentModelConfig?.fallbackModelId2,
|
||
].filter(Boolean) as string[];
|
||
const allModels = await getEnabledAiModelsWithProvider();
|
||
const fallbackModelIds = buildImageFallbackModelIds(allModels, {
|
||
currentProviderId: currentModelConfig?.providerId,
|
||
currentModelId: currentModelConfig?.id,
|
||
currentModelKey: actualModelId,
|
||
configuredFallbackIds: configuredFallbacks,
|
||
});
|
||
console.log("[Chat API] OpenAI-compatible fallback model IDs:", fallbackModelIds);
|
||
|
||
const lastUserMessage = uiMessagesForModel.filter((m) => m.role === "user").pop();
|
||
let messagesForCrossProvider = lastUserMessage ? [lastUserMessage] : uiMessagesForModel;
|
||
messagesForCrossProvider = applyOptimizedPromptToLatestUserMessage(
|
||
messagesForCrossProvider,
|
||
smartAutoOptimizedPrompt,
|
||
hiddenPromptPrefix,
|
||
);
|
||
messagesForCrossProvider = injectHistoryImagesToLastUserMessage(
|
||
messagesForCrossProvider,
|
||
smartAutoHistoryImages
|
||
);
|
||
|
||
// 生成开始标记
|
||
dataStream.write({
|
||
type: "data-appendMessage",
|
||
data: JSON.stringify({
|
||
id: generateUUID(),
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: "<!-- IMAGE_GENERATION_START -->" }],
|
||
}),
|
||
});
|
||
if (contextPayload?.source === "canvas") {
|
||
writeImageGenerationStatusChunk(
|
||
dataStream as unknown as StreamWriter,
|
||
{
|
||
phase: "processing",
|
||
providerType: currentModelConfig?.providerId ? resolvedProviderType || providerType : providerType,
|
||
modelName: currentModelConfig?.displayName || actualModelId,
|
||
imageCount: effectiveImageParams?.imageCount || 1,
|
||
},
|
||
"openai-compat:image-status-start",
|
||
);
|
||
}
|
||
|
||
try {
|
||
const tryGenerateWithOpenAICompatModel = async (modelToUse: any) => {
|
||
const imageRequestTimeoutMs = 180_000;
|
||
const generated = await generateText({
|
||
model: modelToUse,
|
||
system: systemPrompt({ selectedChatModel: finalModelId, requestHints }),
|
||
messages: convertedMessages,
|
||
abortSignal: AbortSignal.timeout(imageRequestTimeoutMs),
|
||
});
|
||
|
||
const rawText = generated.text?.trim() || "图片生成失败，请稍后重试";
|
||
return normalizeAssistantImageMarkdown(
|
||
rawText,
|
||
effectiveImageParams?.imageSize || "2K"
|
||
);
|
||
};
|
||
|
||
let normalizedText = "";
|
||
let usedModelId = actualModelId;
|
||
|
||
try {
|
||
normalizedText = await tryGenerateWithOpenAICompatModel(languageModel);
|
||
} catch (primaryError) {
|
||
console.error("[Chat API] OpenAI-compatible primary model failed:", primaryError);
|
||
|
||
let fallbackSuccess = false;
|
||
for (const fallbackModelId of fallbackModelIds) {
|
||
try {
|
||
console.log(
|
||
`[Chat API] 🔄 OpenAI-compatible fallback model: ${fallbackModelId}`
|
||
);
|
||
const fallbackLanguageModel = await getLanguageModelById(
|
||
finalModelId,
|
||
fallbackModelId
|
||
);
|
||
if (!fallbackLanguageModel) {
|
||
console.log(
|
||
`[Chat API] Skip unavailable OpenAI-compatible fallback model: ${fallbackModelId}`
|
||
);
|
||
continue;
|
||
}
|
||
normalizedText = await tryGenerateWithOpenAICompatModel(fallbackLanguageModel);
|
||
usedModelId = fallbackModelId;
|
||
fallbackSuccess = true;
|
||
console.log(
|
||
`[Chat API] ✅ OpenAI-compatible fallback model ${fallbackModelId} succeeded!`
|
||
);
|
||
break;
|
||
} catch (fallbackError) {
|
||
console.error(
|
||
`[Chat API] ❌ OpenAI-compatible fallback model ${fallbackModelId} failed:`,
|
||
fallbackError
|
||
);
|
||
}
|
||
}
|
||
|
||
if (!fallbackSuccess) {
|
||
const providerPriority: Record<string, number> = {
|
||
"volcano-vision": 0,
|
||
"alibaba-image": 1,
|
||
"bfl-image": 2,
|
||
"ruxa-image": 3,
|
||
"fal-image": 4,
|
||
"openai-image": 5,
|
||
runway: 6,
|
||
gemini: 6,
|
||
"huanke-gemini": 7,
|
||
};
|
||
const crossProviderCandidates = getImageCrossProviderCandidates(allModels, {
|
||
currentModelId: finalModelId,
|
||
currentProviderId: currentModelConfig?.providerId,
|
||
providerPriority,
|
||
});
|
||
|
||
for (const candidate of crossProviderCandidates) {
|
||
try {
|
||
console.log(
|
||
`[Chat API] 🔁 OpenAI-compatible cross-provider failover: ${candidate.model.displayName} (${candidate.provider.type})`
|
||
);
|
||
|
||
const candidateLanguageModel = await getLanguageModelById(candidate.model.id);
|
||
if (
|
||
!candidateLanguageModel ||
|
||
typeof (candidateLanguageModel as any).doGenerate !== "function"
|
||
) {
|
||
console.log(
|
||
`[Chat API] Skip cross-provider candidate without doGenerate: ${candidate.model.id}`
|
||
);
|
||
continue;
|
||
}
|
||
|
||
const convertProvider =
|
||
candidate.provider.type === "bfl-image"
|
||
? "bfl-image"
|
||
: candidate.provider.type === "ruxa-image"
|
||
? "ruxa-image"
|
||
: candidate.provider.type === "fal-image"
|
||
? "fal-image"
|
||
: candidate.provider.type === "openai-image"
|
||
? "openai-image"
|
||
: candidate.provider.type === "midjourney"
|
||
? "midjourney"
|
||
: candidate.provider.type === "runway"
|
||
? "runway"
|
||
: candidate.provider.type === "gemini"
|
||
? "gemini"
|
||
: candidate.provider.type === "huanke-gemini"
|
||
? "huanke-gemini"
|
||
: candidate.provider.type === "volcano-vision"
|
||
? "volcano"
|
||
: "alibaba-image";
|
||
|
||
const candidateMessages = await convertMessagesWithImages(
|
||
messagesForCrossProvider,
|
||
convertProvider
|
||
);
|
||
|
||
const candidateResult = await (candidateLanguageModel as any).doGenerate({
|
||
messages: candidateMessages,
|
||
imageParams:
|
||
effectiveImageParams || {
|
||
aspectRatio: defaultImageQualityParams.aspectRatio,
|
||
imageSize: defaultImageQualityParams.imageSize,
|
||
imageCount: defaultImageQualityParams.imageCount,
|
||
},
|
||
modelDisplayName: candidate.model.displayName,
|
||
overrideModelId: candidate.model.modelId,
|
||
});
|
||
|
||
if (candidateResult?.finishReason === "error") {
|
||
throw new Error(candidateResult.text || "图片生成失败");
|
||
}
|
||
|
||
normalizedText = candidateResult.text || "图片生成失败";
|
||
usedModelId = candidate.model.modelId;
|
||
fallbackSuccess = true;
|
||
console.log(
|
||
`[Chat API] ✅ OpenAI-compatible cross-provider failover succeeded: ${candidate.model.displayName} (${candidate.provider.type})`
|
||
);
|
||
break;
|
||
} catch (crossError) {
|
||
const crossErrorMessage = normalizeFailoverErrorMessage(crossError);
|
||
if (shouldCooldownImageFallbackModel(crossErrorMessage)) {
|
||
setImageFallbackModelCooldown(candidate.model.id, crossErrorMessage);
|
||
}
|
||
console.error(
|
||
`[Chat API] ❌ OpenAI-compatible cross-provider failover failed: ${candidate.model.displayName} (${candidate.provider.type})`,
|
||
crossError
|
||
);
|
||
}
|
||
}
|
||
}
|
||
|
||
if (!fallbackSuccess) {
|
||
throw primaryError;
|
||
}
|
||
}
|
||
|
||
const usedModelDisplayName =
|
||
resolveModelDisplayNameByModelId(allModels, usedModelId) ||
|
||
currentModelConfig?.displayName ||
|
||
actualModelId;
|
||
const normalizedTextWithModel = prependImageModelBadge(
|
||
normalizedText,
|
||
usedModelDisplayName,
|
||
);
|
||
|
||
const messageId = generateUUID();
|
||
dataStream.write({
|
||
type: "data-appendMessage",
|
||
data: JSON.stringify({
|
||
id: messageId,
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: normalizedTextWithModel }],
|
||
}),
|
||
});
|
||
if (contextPayload?.source === "canvas") {
|
||
writeCanvasMediaChunk(
|
||
dataStream as unknown as StreamWriter,
|
||
{
|
||
messageId,
|
||
text: normalizedTextWithModel,
|
||
prompt: requestTextContent,
|
||
sourceImageUrls: requestTargetResolution?.targetImageUrls || [],
|
||
},
|
||
"openai-compat:image-canvas-media",
|
||
);
|
||
}
|
||
|
||
// 生成结束标记
|
||
dataStream.write({
|
||
type: "data-appendMessage",
|
||
data: JSON.stringify({
|
||
id: generateUUID(),
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: "<!-- IMAGE_GENERATION_END -->" }],
|
||
}),
|
||
});
|
||
if (contextPayload?.source === "canvas") {
|
||
writeImageGenerationStatusChunk(
|
||
dataStream as unknown as StreamWriter,
|
||
{
|
||
phase: "completed",
|
||
providerType: resolvedProviderType || providerType,
|
||
modelName: usedModelDisplayName,
|
||
imageCount: effectiveImageParams?.imageCount || 1,
|
||
},
|
||
"openai-compat:image-status-completed",
|
||
);
|
||
}
|
||
|
||
await saveMessages({
|
||
messages: [
|
||
{
|
||
id: messageId,
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: normalizedTextWithModel }],
|
||
createdAt: new Date(),
|
||
attachments: [],
|
||
chatId: id,
|
||
videoStorageStatus: null,
|
||
temporaryVideoUrl: null,
|
||
permanentVideoUrl: null,
|
||
archivedAt: null,
|
||
videoMetadata: null,
|
||
},
|
||
],
|
||
});
|
||
|
||
const latestUserMessage = uiMessages.filter((m) => m.role === "user").pop();
|
||
const userPrompt =
|
||
latestUserMessage?.parts?.find(
|
||
(p): p is { type: "text"; text: string } => p.type === "text"
|
||
)?.text || "";
|
||
await saveGeneratedImages(
|
||
normalizedTextWithModel,
|
||
session.user.id,
|
||
id,
|
||
messageId,
|
||
userPrompt,
|
||
finalModelId,
|
||
{
|
||
sourceAssetIds: requestTargetResolution?.targetAssetIds,
|
||
sourceImageUrls: requestTargetResolution?.targetImageUrls,
|
||
relationType: requestResolvedMediaInput?.hasImage ? "edited_from" : "generated_from",
|
||
sourceType: "generated",
|
||
projectId: contextPayload?.projectId || null,
|
||
optimizedPrompt: smartAutoOptimizedPrompt || userPrompt,
|
||
requestedImageCount: effectiveImageParams?.imageCount,
|
||
aspectRatio: effectiveImageParams?.aspectRatio,
|
||
imageSize: effectiveImageParams?.imageSize,
|
||
},
|
||
);
|
||
return;
|
||
} catch (error) {
|
||
console.error("[Chat API] OpenAI-compatible image model generation error:", error);
|
||
|
||
const errorMessageId = generateUUID();
|
||
const errorText =
|
||
error instanceof Error ? error.message : "图片生成失败，请稍后重试";
|
||
|
||
dataStream.write({
|
||
type: "data-appendMessage",
|
||
data: JSON.stringify({
|
||
id: errorMessageId,
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: errorText }],
|
||
}),
|
||
});
|
||
dataStream.write({
|
||
type: "data-appendMessage",
|
||
data: JSON.stringify({
|
||
id: generateUUID(),
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: "<!-- IMAGE_GENERATION_END -->" }],
|
||
}),
|
||
});
|
||
if (contextPayload?.source === "canvas") {
|
||
writeImageGenerationStatusChunk(
|
||
dataStream as unknown as StreamWriter,
|
||
{
|
||
phase: "failed",
|
||
providerType: resolvedProviderType || providerType,
|
||
modelName: currentModelConfig?.displayName || actualModelId,
|
||
imageCount: effectiveImageParams?.imageCount || 1,
|
||
},
|
||
"openai-compat:image-status-failed",
|
||
);
|
||
}
|
||
|
||
await saveMessages({
|
||
messages: [
|
||
{
|
||
id: errorMessageId,
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: errorText }],
|
||
createdAt: new Date(),
|
||
attachments: [],
|
||
chatId: id,
|
||
videoStorageStatus: null,
|
||
temporaryVideoUrl: null,
|
||
permanentVideoUrl: null,
|
||
archivedAt: null,
|
||
videoMetadata: null,
|
||
},
|
||
],
|
||
});
|
||
return;
|
||
}
|
||
}
|
||
|
||
// 🎬 视频模型使用专门的生成流程
|
||
if (isVolcanoVideoModel) {
|
||
console.log("[Chat API] Using video generation API");
|
||
|
||
// 导入视频生成函数
|
||
const { doGenerateStream } = await import("@/lib/ai/providers/volcano-video");
|
||
|
||
const conversationMessage = smartAutoFriendlyMessage || null;
|
||
if (conversationMessage) {
|
||
const chatMessageId = generateUUID();
|
||
dataStream.write({
|
||
type: "data-appendMessage",
|
||
data: JSON.stringify({
|
||
id: chatMessageId,
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: conversationMessage }],
|
||
attachments: [],
|
||
}),
|
||
});
|
||
|
||
await saveMessages({
|
||
messages: [
|
||
{
|
||
id: chatMessageId,
|
||
chatId: id,
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: conversationMessage }],
|
||
attachments: [],
|
||
createdAt: new Date(),
|
||
videoStorageStatus: null,
|
||
temporaryVideoUrl: null,
|
||
permanentVideoUrl: null,
|
||
archivedAt: null,
|
||
videoMetadata: null,
|
||
},
|
||
],
|
||
});
|
||
}
|
||
|
||
const messagesForVideo = injectHistoryImagesToLastUserMessage(
|
||
applyOptimizedPromptToLatestUserMessage(
|
||
uiMessagesForModel,
|
||
smartAutoOptimizedPrompt,
|
||
hiddenPromptPrefix,
|
||
),
|
||
smartAutoHistoryImages,
|
||
);
|
||
|
||
// 从最后一条用户消息中提取提示词和图片
|
||
const lastUserMessage = messagesForVideo.filter(m => m.role === "user").pop();
|
||
let prompt = "";
|
||
const imageUrls: string[] = [];
|
||
const summarizeImageRef = (rawUrl: string) => {
|
||
const trimmed = String(rawUrl || "").trim();
|
||
if (!trimmed) return "";
|
||
if (trimmed.startsWith("data:image/")) {
|
||
return `data:image(base64,${trimmed.length} chars)`;
|
||
}
|
||
try {
|
||
const parsed = new URL(trimmed);
|
||
return `${parsed.origin}${parsed.pathname}`;
|
||
} catch {
|
||
return trimmed.slice(0, 120);
|
||
}
|
||
};
|
||
|
||
if (lastUserMessage) {
|
||
// 🆕 处理 parts 数组（AI SDK v4 格式）
|
||
if ((lastUserMessage as any).parts && Array.isArray((lastUserMessage as any).parts)) {
|
||
console.log("[Chat API] Video input parts:", (lastUserMessage as any).parts.length);
|
||
|
||
for (const part of (lastUserMessage as any).parts) {
|
||
if (part.type === "text" && part.text) {
|
||
// 提取文本
|
||
prompt += part.text;
|
||
} else if (part.type === "file" && part.url && part.mediaType?.startsWith("image/")) {
|
||
// 提取图片
|
||
imageUrls.push(part.url);
|
||
}
|
||
}
|
||
}
|
||
// 备用方案1：content 字符串/数组
|
||
const legacyContent = (lastUserMessage as any).content;
|
||
if (typeof legacyContent === "string") {
|
||
prompt = legacyContent;
|
||
} else if (Array.isArray(legacyContent)) {
|
||
for (const part of legacyContent) {
|
||
if (typeof part === "string") {
|
||
prompt += part;
|
||
} else if (part.type === "text") {
|
||
prompt += part.text;
|
||
} else if (part.type === "image" && (part as any).image) {
|
||
imageUrls.push((part as any).image);
|
||
}
|
||
}
|
||
} else if ((lastUserMessage as any).experimental_attachments) {
|
||
// 备用方案2：experimental_attachments
|
||
for (const attachment of (lastUserMessage as any).experimental_attachments) {
|
||
if (attachment.url && attachment.contentType?.startsWith("image/")) {
|
||
imageUrls.push(attachment.url);
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
console.log("[Chat API] Video input extracted:", {
|
||
promptLength: prompt.trim().length,
|
||
imageCount: imageUrls.length,
|
||
imageRefs: imageUrls.slice(0, 4).map(summarizeImageRef),
|
||
});
|
||
|
||
const videoMessageId = generateUUID();
|
||
const videoStartAt = Date.now();
|
||
const baseVideoMetadata: Record<string, any> = {
|
||
provider: effectiveProviderType || "volcano-video",
|
||
mode: effectiveVideoParams?.mode || "text-to-video",
|
||
resolution: effectiveVideoParams?.resolution || "720p",
|
||
ratio: effectiveVideoParams?.ratio || "16:9",
|
||
duration: effectiveVideoParams?.duration || 5,
|
||
status: "queued",
|
||
startedAt: videoStartAt,
|
||
updatedAt: videoStartAt,
|
||
};
|
||
|
||
try {
|
||
console.log("[Chat API] Video generation params:", {
|
||
promptLength: prompt.trim().length,
|
||
imageCount: imageUrls.length,
|
||
videoParams: effectiveVideoParams,
|
||
});
|
||
|
||
const videoMode = (effectiveVideoParams?.mode || "text-to-video") as VideoModeType;
|
||
|
||
// 🆕 重新执行子模型选择逻辑，确定要使用的模型 UUID
|
||
let videoModelId = finalModelId; // 默认使用选择的模型 ID
|
||
|
||
// 检查是否是主模型，需要选择子模型（仅 volcano-video 支持子模型）
|
||
const selectedModelData = await getAiModelById({ id: finalModelId });
|
||
const selectedModelProvider = selectedModelData
|
||
? await getAiProviderById({ id: selectedModelData.providerId }).catch(() => null)
|
||
: null;
|
||
if (
|
||
selectedModelData &&
|
||
!selectedModelData.isSubModel &&
|
||
selectedModelProvider?.type === "volcano-video"
|
||
) {
|
||
const videoSubModels = await getSubModelsByParentModelId(finalModelId);
|
||
console.log(`[Chat API] Video: Found ${videoSubModels.length} sub-models for master model`);
|
||
|
||
if (videoSubModels.length > 0) {
|
||
const matchedModel = selectBestVideoSubModelForMode(videoSubModels as any, videoMode);
|
||
|
||
if (matchedModel) {
|
||
videoModelId = matchedModel.model.id;
|
||
console.log(
|
||
`[Chat API] Video: Selected sub-model ${matchedModel.model.displayName} (${matchedModel.model.modelId}) for mode ${videoMode}`
|
||
);
|
||
} else {
|
||
console.log(
|
||
`[Chat API] Video: No compatible sub-model found for mode ${videoMode}, keep current model`
|
||
);
|
||
}
|
||
}
|
||
}
|
||
|
||
console.log("[Chat API] Getting model config for:", videoModelId);
|
||
const aiModelData = await getAiModelById({ id: videoModelId });
|
||
if (!aiModelData) {
|
||
throw new Error("视频模型配置未找到");
|
||
}
|
||
|
||
// 若用户手动选了不兼容模式的子模型，自动回退到同父模型兼容子模型
|
||
const selectedSupportsMode = getSupportedVideoModesForModel(aiModelData as any).includes(videoMode);
|
||
if (!selectedSupportsMode) {
|
||
let fallbackModel: { model: any } | null = null;
|
||
|
||
if (aiModelData.parentModelId) {
|
||
const siblingSubModels = await getSubModelsByParentModelId(aiModelData.parentModelId);
|
||
fallbackModel = selectBestVideoSubModelForMode(siblingSubModels as any, videoMode);
|
||
} else if (selectedModelProvider?.type === "volcano-video") {
|
||
const enabledModels = await getEnabledAiModelsWithProvider();
|
||
const sameProviderModels = enabledModels
|
||
.filter(
|
||
({ model, provider }) =>
|
||
provider.type === "volcano-video" &&
|
||
model.isEnabled &&
|
||
model.id !== aiModelData.id,
|
||
)
|
||
.map(({ model }) => ({ model }));
|
||
fallbackModel = selectBestVideoSubModelForMode(sameProviderModels as any, videoMode);
|
||
}
|
||
|
||
if (fallbackModel) {
|
||
console.warn(
|
||
`[Chat API] Video: Model ${aiModelData.modelId} does not support ${videoMode}, fallback to ${fallbackModel.model.modelId}`
|
||
);
|
||
videoModelId = fallbackModel.model.id;
|
||
}
|
||
}
|
||
|
||
const finalVideoModelData = videoModelId === aiModelData.id
|
||
? aiModelData
|
||
: await getAiModelById({ id: videoModelId });
|
||
if (!finalVideoModelData) {
|
||
throw new Error("视频模型配置未找到");
|
||
}
|
||
|
||
// 如果是子模型，获取父模型的 Provider 配置
|
||
let providerModelData = finalVideoModelData;
|
||
if (finalVideoModelData.isSubModel && finalVideoModelData.parentModelId) {
|
||
console.log(
|
||
"[Chat API] This is a sub-model, getting parent model config:",
|
||
finalVideoModelData.parentModelId
|
||
);
|
||
const parentModel = await getAiModelById({ id: finalVideoModelData.parentModelId });
|
||
if (parentModel) {
|
||
providerModelData = parentModel;
|
||
}
|
||
}
|
||
|
||
console.log("[Chat API] Getting provider config for:", providerModelData.providerId);
|
||
const providerData = await getAiProviderById({ id: providerModelData.providerId });
|
||
if (!providerData) {
|
||
throw new Error("Provider 配置未找到");
|
||
}
|
||
|
||
const actualVideoModelId = finalVideoModelData.modelId;
|
||
|
||
// 构建视频生成选项
|
||
const videoOptions: any = {
|
||
model: actualVideoModelId,
|
||
prompt: prompt || "生成视频",
|
||
mode: videoMode,
|
||
resolution: effectiveVideoParams?.resolution || "720p",
|
||
ratio: effectiveVideoParams?.ratio || "16:9",
|
||
duration: effectiveVideoParams?.duration || 5,
|
||
fps: effectiveVideoParams?.fps || 24,
|
||
cameraMove: effectiveVideoParams?.cameraMove || "none",
|
||
cameraFixed: effectiveVideoParams?.cameraFixed || false,
|
||
watermark: effectiveVideoParams?.watermark || false,
|
||
seed: effectiveVideoParams?.seed,
|
||
};
|
||
|
||
// 根据模式添加图片
|
||
if (videoOptions.mode === "first-frame" && imageUrls[0]) {
|
||
videoOptions.firstFrameImage = imageUrls[0];
|
||
} else if (videoOptions.mode === "first-last-frame" && imageUrls.length >= 2) {
|
||
videoOptions.firstFrameImage = imageUrls[0];
|
||
videoOptions.lastFrameImage = imageUrls[1];
|
||
} else if (videoOptions.mode === "reference-images" && imageUrls.length > 0) {
|
||
videoOptions.referenceImages = imageUrls;
|
||
}
|
||
|
||
console.log("[Chat API] Video options constructed:", {
|
||
model: videoOptions.model,
|
||
mode: videoOptions.mode,
|
||
resolution: videoOptions.resolution,
|
||
ratio: videoOptions.ratio,
|
||
duration: videoOptions.duration,
|
||
imageCount:
|
||
videoOptions.referenceImages?.length ||
|
||
Number(Boolean(videoOptions.firstFrameImage)) +
|
||
Number(Boolean(videoOptions.lastFrameImage)),
|
||
});
|
||
|
||
// 先写入占位消息，避免刷新后消息丢失
|
||
const pendingText = "🎬 视频任务已创建，正在处理中，请稍候...";
|
||
await saveMessages({
|
||
messages: [
|
||
{
|
||
id: videoMessageId,
|
||
chatId: id,
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: pendingText }],
|
||
attachments: [],
|
||
createdAt: new Date(),
|
||
videoStorageStatus: "temporary",
|
||
temporaryVideoUrl: null,
|
||
permanentVideoUrl: null,
|
||
archivedAt: null,
|
||
videoMetadata: {
|
||
...baseVideoMetadata,
|
||
model: actualVideoModelId,
|
||
status: "running",
|
||
updatedAt: Date.now(),
|
||
} as any,
|
||
},
|
||
],
|
||
});
|
||
try {
|
||
await persistVideoGenerationStartedTurn({
|
||
userId: session.user.id,
|
||
chatId: id,
|
||
projectId: contextPayload?.projectId || null,
|
||
messageId: videoMessageId,
|
||
text: requestTextContent,
|
||
source: "stream_video_provider",
|
||
providerType: String(providerData.type || "stream"),
|
||
modelRef: finalVideoModelData.id || finalModelId,
|
||
mode: videoOptions.mode,
|
||
status: "running",
|
||
});
|
||
} catch (videoWorkflowStartError) {
|
||
console.warn(
|
||
"[Chat API] Failed to persist stream video start state:",
|
||
videoWorkflowStartError,
|
||
);
|
||
}
|
||
|
||
dataStream.write({
|
||
type: "data-appendMessage",
|
||
data: JSON.stringify({
|
||
id: videoMessageId,
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: pendingText }],
|
||
attachments: [],
|
||
}),
|
||
});
|
||
|
||
const apiKey = providerData.apiKey;
|
||
const baseURL = providerData.baseUrl || "https://ark.cn-beijing.volces.com/api/v3";
|
||
console.log("[Chat API] Provider configured:", { baseURL, hasApiKey: !!apiKey });
|
||
|
||
// 调用视频生成流
|
||
console.log("[Chat API] Starting video generation stream...");
|
||
const videoStream = doGenerateStream(apiKey, baseURL, videoOptions);
|
||
|
||
// 收集所有流输出
|
||
console.log("[Chat API] Processing video stream...");
|
||
let fullText = "";
|
||
let chunkCount = 0;
|
||
|
||
for await (const chunk of videoStream) {
|
||
chunkCount++;
|
||
fullText += chunk;
|
||
console.log(`[Chat API] Received chunk ${chunkCount}, length: ${chunk.length}`);
|
||
}
|
||
|
||
console.log(
|
||
`[Chat API] Video stream completed. Total chunks: ${chunkCount}, total text length: ${fullText.length}`
|
||
);
|
||
|
||
// 提取流中的元数据并清理输出文案
|
||
const metadataMatch = fullText.match(/<!--\s*VIDEO_METADATA:(.*?)-->/s);
|
||
let streamMetadata: Record<string, any> = {};
|
||
if (metadataMatch?.[1]) {
|
||
try {
|
||
streamMetadata = JSON.parse(metadataMatch[1]);
|
||
} catch (parseError) {
|
||
console.warn("[Chat API] Failed to parse VIDEO_METADATA from stream", parseError);
|
||
}
|
||
}
|
||
const cleanedText = fullText
|
||
.replace(/<!--\s*VIDEO_METADATA:.*?-->/gs, "")
|
||
.trim();
|
||
const videoUrlMatch = cleanedText.match(/<video[^>]+src="([^"]+)"/);
|
||
const temporaryVideoUrl =
|
||
(typeof streamMetadata.temporaryVideoUrl === "string" && streamMetadata.temporaryVideoUrl) ||
|
||
(videoUrlMatch ? videoUrlMatch[1] : null);
|
||
|
||
const successMetadata = {
|
||
...baseVideoMetadata,
|
||
...streamMetadata,
|
||
model: actualVideoModelId,
|
||
taskId:
|
||
typeof streamMetadata.taskId === "string" && streamMetadata.taskId
|
||
? streamMetadata.taskId
|
||
: undefined,
|
||
status: "succeeded",
|
||
completedAt: Date.now(),
|
||
updatedAt: Date.now(),
|
||
};
|
||
|
||
await updateMessageById({
|
||
messageId: videoMessageId,
|
||
chatId: id,
|
||
parts: [{ type: "text", text: cleanedText }],
|
||
videoStorageStatus: temporaryVideoUrl ? "temporary" : null,
|
||
temporaryVideoUrl: temporaryVideoUrl,
|
||
permanentVideoUrl: null,
|
||
archivedAt: null,
|
||
videoMetadata: successMetadata as any,
|
||
});
|
||
try {
|
||
await persistVideoGenerationCompletedTurn({
|
||
userId: session.user.id,
|
||
chatId: id,
|
||
projectId: contextPayload?.projectId || null,
|
||
messageId: videoMessageId,
|
||
text: requestTextContent,
|
||
source: "stream_video_provider",
|
||
providerType: String(providerData.type || "stream"),
|
||
modelRef: finalVideoModelData.id || finalModelId,
|
||
mode: videoOptions.mode,
|
||
taskId:
|
||
typeof successMetadata.taskId === "string" && successMetadata.taskId
|
||
? successMetadata.taskId
|
||
: undefined,
|
||
status: "completed",
|
||
temporaryVideoUrl: temporaryVideoUrl || undefined,
|
||
});
|
||
} catch (videoWorkflowCompletedError) {
|
||
console.warn(
|
||
"[Chat API] Failed to persist stream video completion state:",
|
||
videoWorkflowCompletedError,
|
||
);
|
||
}
|
||
await consumeGeneratedVideoPoints({
|
||
userId: session.user.id,
|
||
chatId: id,
|
||
messageId: videoMessageId,
|
||
modelRefs: [finalVideoModelData.id, providerModelData.id, finalModelId],
|
||
duration: Number(videoOptions.duration) || undefined,
|
||
});
|
||
|
||
dataStream.write({
|
||
type: "data-appendMessage",
|
||
data: JSON.stringify({
|
||
id: videoMessageId,
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: cleanedText }],
|
||
attachments: [],
|
||
}),
|
||
});
|
||
|
||
console.log("[Chat API] ✅ Video message updated successfully");
|
||
if (temporaryVideoUrl) {
|
||
console.log("[Chat API] 📹 临时视频URL已保存，开始异步归档到S3");
|
||
scheduleVideoArchive({
|
||
chatId: id,
|
||
messageId: videoMessageId,
|
||
text: cleanedText,
|
||
temporaryVideoUrl,
|
||
taskId:
|
||
typeof successMetadata.taskId === "string" && successMetadata.taskId
|
||
? successMetadata.taskId
|
||
: videoMessageId,
|
||
metadata: successMetadata,
|
||
});
|
||
}
|
||
|
||
console.log("[Chat API] Video generation finished successfully");
|
||
return;
|
||
} catch (error: any) {
|
||
console.error("[Chat API] ❌ Video generation error:", error);
|
||
console.error("[Chat API] Error stack:", error.stack);
|
||
|
||
const errorMessage = `❌ 视频生成失败：${error.message}\n\n请检查配置或稍后重试。`;
|
||
const failedMetadata = {
|
||
...baseVideoMetadata,
|
||
status: "failed",
|
||
error: error?.message || "未知错误",
|
||
updatedAt: Date.now(),
|
||
};
|
||
|
||
try {
|
||
await updateMessageById({
|
||
messageId: videoMessageId,
|
||
chatId: id,
|
||
parts: [{ type: "text", text: errorMessage }],
|
||
videoStorageStatus: null,
|
||
temporaryVideoUrl: null,
|
||
permanentVideoUrl: null,
|
||
archivedAt: null,
|
||
videoMetadata: failedMetadata as any,
|
||
});
|
||
try {
|
||
await persistVideoGenerationFailedTurn({
|
||
userId: session.user.id,
|
||
chatId: id,
|
||
projectId: contextPayload?.projectId || null,
|
||
messageId: videoMessageId,
|
||
text: requestTextContent,
|
||
source: "stream_video_provider",
|
||
providerType: "stream",
|
||
modelRef: finalModelId,
|
||
mode:
|
||
typeof effectiveVideoParams?.mode === "string"
|
||
? effectiveVideoParams.mode
|
||
: undefined,
|
||
status: "failed",
|
||
error: error?.message || "未知错误",
|
||
});
|
||
} catch (videoWorkflowFailedError) {
|
||
console.warn(
|
||
"[Chat API] Failed to persist stream video failure state:",
|
||
videoWorkflowFailedError,
|
||
);
|
||
}
|
||
|
||
dataStream.write({
|
||
type: "data-appendMessage",
|
||
data: JSON.stringify({
|
||
id: videoMessageId,
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: errorMessage }],
|
||
attachments: [],
|
||
}),
|
||
});
|
||
} catch (updateError) {
|
||
console.error("[Chat API] Failed to update placeholder video message:", updateError);
|
||
const fallbackMessageId = generateUUID();
|
||
dataStream.write({
|
||
type: "data-appendMessage",
|
||
data: JSON.stringify({
|
||
id: fallbackMessageId,
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: errorMessage }],
|
||
attachments: [],
|
||
}),
|
||
});
|
||
await saveMessages({
|
||
messages: [
|
||
{
|
||
id: fallbackMessageId,
|
||
chatId: id,
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: errorMessage }],
|
||
attachments: [],
|
||
createdAt: new Date(),
|
||
videoStorageStatus: null,
|
||
temporaryVideoUrl: null,
|
||
permanentVideoUrl: null,
|
||
archivedAt: null,
|
||
videoMetadata: failedMetadata as any,
|
||
},
|
||
],
|
||
});
|
||
try {
|
||
await persistVideoGenerationFailedTurn({
|
||
userId: session.user.id,
|
||
chatId: id,
|
||
projectId: contextPayload?.projectId || null,
|
||
messageId: fallbackMessageId,
|
||
text: requestTextContent,
|
||
source: "stream_video_provider",
|
||
providerType: "stream",
|
||
modelRef: finalModelId,
|
||
mode:
|
||
typeof effectiveVideoParams?.mode === "string"
|
||
? effectiveVideoParams.mode
|
||
: undefined,
|
||
status: "failed",
|
||
error: error?.message || "未知错误",
|
||
});
|
||
} catch (videoWorkflowFailedError) {
|
||
console.warn(
|
||
"[Chat API] Failed to persist fallback stream video failure state:",
|
||
videoWorkflowFailedError,
|
||
);
|
||
}
|
||
}
|
||
|
||
return;
|
||
}
|
||
}
|
||
|
||
// 常规文本模型使用 streamText
|
||
const streamRequestTimeoutMs = resolvedModelSupportsImageGen ? 180_000 : 90_000;
|
||
const shouldDisableSmoothStream =
|
||
resolvedModelSupportsImageGen || isVolcanoVideoModel;
|
||
|
||
const result = streamText({
|
||
model: languageModel,
|
||
system: effectiveConversationSystemPrompt,
|
||
messages: convertedMessages,
|
||
stopWhen: stepCountIs(5),
|
||
experimental_activeTools:
|
||
finalModelId === "chat-model-reasoning" || isVolcanoVideoModel ? [] : activeTools,
|
||
experimental_transform: shouldDisableSmoothStream
|
||
? undefined
|
||
: smoothStream({ chunking: "word" }),
|
||
tools: isVolcanoVideoModel ? {} : tools, // 🆕 视频模型禁用工具
|
||
abortSignal: AbortSignal.timeout(streamRequestTimeoutMs),
|
||
experimental_telemetry: {
|
||
isEnabled: isProductionEnvironment,
|
||
functionId: "stream-text",
|
||
},
|
||
onFinish: async ({ usage }) => {
|
||
if (activeRuntimeModelDescriptor?.providerKey) {
|
||
recordProviderHealthSuccess({
|
||
providerKey: activeRuntimeModelDescriptor.providerKey,
|
||
providerType: activeRuntimeModelDescriptor.providerType,
|
||
});
|
||
}
|
||
try {
|
||
const providers = await getTokenlensCatalog();
|
||
if (!actualModelId) {
|
||
finalMergedUsage = usage;
|
||
dataStream.write({
|
||
type: "data-usage",
|
||
data: finalMergedUsage,
|
||
});
|
||
return;
|
||
}
|
||
|
||
if (!providers) {
|
||
finalMergedUsage = usage;
|
||
dataStream.write({
|
||
type: "data-usage",
|
||
data: finalMergedUsage,
|
||
});
|
||
return;
|
||
}
|
||
|
||
const summary = getUsage({
|
||
modelId: actualModelId,
|
||
usage,
|
||
providers,
|
||
});
|
||
finalMergedUsage = {
|
||
...usage,
|
||
...summary,
|
||
modelId: actualModelId,
|
||
} as AppUsage;
|
||
dataStream.write({ type: "data-usage", data: finalMergedUsage });
|
||
} catch (err) {
|
||
console.warn("TokenLens enrichment failed", err);
|
||
finalMergedUsage = usage;
|
||
dataStream.write({ type: "data-usage", data: finalMergedUsage });
|
||
}
|
||
},
|
||
});
|
||
|
||
result.consumeStream({
|
||
onError: (error) => {
|
||
console.error("[Chat API] streamText consume error:", error);
|
||
if (activeRuntimeModelDescriptor?.providerKey) {
|
||
recordProviderHealthFailure({
|
||
providerKey: activeRuntimeModelDescriptor.providerKey,
|
||
providerType: activeRuntimeModelDescriptor.providerType,
|
||
error,
|
||
});
|
||
}
|
||
},
|
||
});
|
||
|
||
dataStream.merge(
|
||
result.toUIMessageStream({
|
||
sendReasoning: true,
|
||
onError: (error) => {
|
||
console.error("[Chat API] streamText ui stream error:", error);
|
||
if (activeRuntimeModelDescriptor?.providerKey) {
|
||
recordProviderHealthFailure({
|
||
providerKey: activeRuntimeModelDescriptor.providerKey,
|
||
providerType: activeRuntimeModelDescriptor.providerType,
|
||
error,
|
||
});
|
||
}
|
||
return getChatStreamErrorMessage(error);
|
||
},
|
||
})
|
||
);
|
||
} finally {
|
||
setConversationStateProtocolEventObserver(null);
|
||
stopStreamKeepalive = stopDefaultKeepalive;
|
||
}
|
||
},
|
||
generateId: generateUUID,
|
||
onFinish: async ({ messages }) => {
|
||
stopStreamKeepalive?.();
|
||
stopStreamKeepalive = null;
|
||
const messagesToPersist: Array<{
|
||
id: string;
|
||
role: string;
|
||
parts: unknown;
|
||
createdAt: Date;
|
||
attachments: unknown;
|
||
chatId: string;
|
||
videoStorageStatus: null;
|
||
temporaryVideoUrl: null;
|
||
permanentVideoUrl: null;
|
||
archivedAt: null;
|
||
videoMetadata: null;
|
||
}> = [];
|
||
|
||
for (const currentMessage of messages) {
|
||
const normalizedParts =
|
||
currentMessage.role === "assistant"
|
||
? normalizeAssistantPartsForPersistence(
|
||
currentMessage.parts as Array<Record<string, any>> | undefined,
|
||
)
|
||
: currentMessage.parts;
|
||
|
||
if (
|
||
currentMessage.role === "assistant" &&
|
||
(!Array.isArray(normalizedParts) || normalizedParts.length === 0)
|
||
) {
|
||
continue;
|
||
}
|
||
|
||
messagesToPersist.push({
|
||
id: currentMessage.id,
|
||
role: currentMessage.role,
|
||
parts: normalizedParts,
|
||
createdAt: new Date(),
|
||
attachments: [],
|
||
chatId: id,
|
||
videoStorageStatus: null,
|
||
temporaryVideoUrl: null,
|
||
permanentVideoUrl: null,
|
||
archivedAt: null,
|
||
videoMetadata: null,
|
||
});
|
||
}
|
||
|
||
if (messagesToPersist.length > 0) {
|
||
await saveMessages({
|
||
messages: messagesToPersist,
|
||
});
|
||
}
|
||
|
||
// 检测助手消息中是否包含视频，如果有则立即标记并归档到永久存储
|
||
for (const msg of messages) {
|
||
if (msg.role === "assistant" && msg.parts) {
|
||
const textContent = msg.parts
|
||
.filter((part: any) => part && part.type === "text" && part.text)
|
||
.map((part: any) => part.text)
|
||
.join("");
|
||
|
||
const tempVideoUrl = extractVideoUrlFromText(textContent);
|
||
if (tempVideoUrl) {
|
||
const genericVideoMetadata = {
|
||
provider: "stream",
|
||
model: actualModelId,
|
||
status: "succeeded",
|
||
updatedAt: Date.now(),
|
||
};
|
||
|
||
await updateMessageById({
|
||
messageId: msg.id,
|
||
chatId: id,
|
||
videoStorageStatus: "temporary",
|
||
temporaryVideoUrl: tempVideoUrl,
|
||
permanentVideoUrl: null,
|
||
archivedAt: null,
|
||
videoMetadata: genericVideoMetadata as any,
|
||
});
|
||
try {
|
||
await persistVideoGenerationCompletedTurn({
|
||
userId: session.user.id,
|
||
chatId: id,
|
||
projectId: contextPayload?.projectId || null,
|
||
messageId: msg.id,
|
||
text: requestTextContent,
|
||
source: "chat_stream_output",
|
||
providerType: "stream",
|
||
modelRef: actualModelId,
|
||
status: "completed",
|
||
temporaryVideoUrl: tempVideoUrl,
|
||
});
|
||
} catch (videoWorkflowCompletedError) {
|
||
console.warn(
|
||
"[Chat API] Failed to persist streamed assistant video state:",
|
||
videoWorkflowCompletedError,
|
||
);
|
||
}
|
||
|
||
scheduleVideoArchive({
|
||
chatId: id,
|
||
messageId: msg.id,
|
||
text: textContent,
|
||
temporaryVideoUrl: tempVideoUrl,
|
||
taskId: msg.id,
|
||
metadata: genericVideoMetadata,
|
||
});
|
||
}
|
||
}
|
||
}
|
||
|
||
if (finalMergedUsage) {
|
||
try {
|
||
await updateChatLastContextById({
|
||
chatId: id,
|
||
context: finalMergedUsage,
|
||
});
|
||
} catch (err) {
|
||
console.warn("Unable to persist last usage for chat", id, err);
|
||
}
|
||
}
|
||
},
|
||
onError: () => {
|
||
stopStreamKeepalive?.();
|
||
stopStreamKeepalive = null;
|
||
return "Oops, an error occurred!";
|
||
},
|
||
});
|
||
|
||
// const streamContext = getStreamContext();
|
||
|
||
// if (streamContext) {
|
||
// return new Response(
|
||
// await streamContext.resumableStream(streamId, () =>
|
||
// stream.pipeThrough(new JsonToSseTransformStream())
|
||
// )
|
||
// );
|
||
// }
|
||
|
||
return createSseResponse(stream);
|
||
} catch (error) {
|
||
const vercelId = request.headers.get("x-vercel-id");
|
||
|
||
if (error instanceof ChatSDKError) {
|
||
return error.toResponse();
|
||
}
|
||
|
||
// Check for Vercel AI Gateway credit card error
|
||
if (
|
||
error instanceof Error &&
|
||
error.message?.includes(
|
||
"AI Gateway requires a valid credit card on file to service requests"
|
||
)
|
||
) {
|
||
return new ChatSDKError("bad_request:activate_gateway").toResponse();
|
||
}
|
||
|
||
console.error("Unhandled error in chat API:", error, { vercelId });
|
||
return new ChatSDKError("offline:chat").toResponse();
|
||
} finally {
|
||
if (requestDedupKey && acquiredInFlightLock) {
|
||
inFlightChatRequests.delete(requestDedupKey);
|
||
}
|
||
}
|
||
}
|
||
|
||
export async function DELETE(request: Request) {
|
||
const { searchParams } = new URL(request.url);
|
||
const id = searchParams.get("id");
|
||
|
||
if (!id) {
|
||
return new ChatSDKError("bad_request:api").toResponse();
|
||
}
|
||
|
||
const session = await auth();
|
||
|
||
if (!session?.user) {
|
||
return new ChatSDKError("unauthorized:chat").toResponse();
|
||
}
|
||
|
||
const chat = await getChatById({ id });
|
||
|
||
if (chat?.userId !== session.user.id) {
|
||
return new ChatSDKError("forbidden:chat").toResponse();
|
||
}
|
||
|
||
const deletedChat = await deleteChatById({ id });
|
||
|
||
return Response.json(deletedChat, { status: 200 });
|
||
}
|