增强Agent分析链路并兼容手动模型触发
This commit is contained in:
Your Name
parent
faf4a67b92
commit
c9c0356fe3
@ -183,6 +183,63 @@ function applyOptimizedPromptToLatestUserMessage(
|
||||
return cloned;
|
||||
}
|
||||
|
||||
type ChatImageParams = NonNullable<PostRequestBody["imageParams"]>;
|
||||
type AgentRecommendedImageParams = {
|
||||
aspectRatio?: string;
|
||||
imageSize?: string;
|
||||
imageCount?: number;
|
||||
negativePrompt?: string;
|
||||
promptExtend?: boolean;
|
||||
};
|
||||
|
||||
function hasUserCustomizedImageParams(params?: PostRequestBody["imageParams"]): boolean {
|
||||
if (!params) return false;
|
||||
const customizedAspectRatio =
|
||||
typeof params.aspectRatio === "string" && params.aspectRatio.trim() && params.aspectRatio !== "auto";
|
||||
const customizedImageSize =
|
||||
typeof params.imageSize === "string" && params.imageSize.trim() && params.imageSize.toUpperCase() !== "2K";
|
||||
const customizedImageCount = typeof params.imageCount === "number" && params.imageCount !== 1;
|
||||
const customizedAdvanced = Boolean(params.size || params.seed !== undefined || params.n || params.negativePrompt);
|
||||
return customizedAspectRatio || customizedImageSize || customizedImageCount || customizedAdvanced;
|
||||
}
|
||||
|
||||
function mergeImageParamsWithAgentRecommendation(params: {
|
||||
incoming?: PostRequestBody["imageParams"];
|
||||
recommended?: AgentRecommendedImageParams;
|
||||
intent?: "image_edit" | "image_gen" | "vision" | "video_gen" | "chat";
|
||||
hasUserCustomSettings: boolean;
|
||||
}): PostRequestBody["imageParams"] {
|
||||
const { incoming, recommended, intent, hasUserCustomSettings } = params;
|
||||
const base: ChatImageParams = { ...(incoming || {}) };
|
||||
if (!recommended) {
|
||||
return Object.keys(base).length > 0 ? base : undefined;
|
||||
}
|
||||
|
||||
const merged: ChatImageParams = {
|
||||
...base,
|
||||
negativePrompt:
|
||||
typeof recommended.negativePrompt === "string" && recommended.negativePrompt.trim()
|
||||
? recommended.negativePrompt.trim()
|
||||
: base.negativePrompt,
|
||||
promptExtend:
|
||||
typeof recommended.promptExtend === "boolean"
|
||||
? recommended.promptExtend
|
||||
: base.promptExtend,
|
||||
};
|
||||
|
||||
if (!hasUserCustomSettings) {
|
||||
if (recommended.aspectRatio) merged.aspectRatio = recommended.aspectRatio;
|
||||
if (recommended.imageSize) merged.imageSize = recommended.imageSize;
|
||||
if (typeof recommended.imageCount === "number") merged.imageCount = recommended.imageCount;
|
||||
}
|
||||
|
||||
if (intent === "image_edit") {
|
||||
merged.imageCount = 1;
|
||||
}
|
||||
|
||||
return Object.keys(merged).length > 0 ? merged : undefined;
|
||||
}
|
||||
|
||||
function guessImageMediaType(url: string): string {
|
||||
const cleanUrl = url.split("?")[0]?.toLowerCase() || "";
|
||||
if (cleanUrl.endsWith(".jpg") || cleanUrl.endsWith(".jpeg")) return "image/jpeg";
|
||||
@ -818,6 +875,8 @@ export async function POST(request: Request) {
|
||||
// 🆕 存储智能模式的相关信息
|
||||
let smartAutoFriendlyMessage: string | undefined; // AI的自然对话回复
|
||||
let smartAutoOptimizedPrompt: string | undefined; // 优化后的提示词（不修改用户消息，只用于API调用）
|
||||
let smartAutoRecommendedImageParams: AgentRecommendedImageParams | undefined;
|
||||
let smartAutoDetectedIntent: "image_edit" | "vision" | "image_gen" | "video_gen" | "chat" | undefined;
|
||||
let smartAutoHistoryImages: string[] = []; // 从历史对话中提取的图片URL（支持多张）
|
||||
let smartAutoNeedsClarification = false; // 是否需要先追问用户
|
||||
let smartAutoClarificationQuestion: string | undefined;
|
||||
@ -827,23 +886,56 @@ export async function POST(request: Request) {
|
||||
|
||||
if (!shouldRunAgentConversation && requestedModelId !== "auto") {
|
||||
const selectedModelIsUUID =
|
||||
/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test(
|
||||
requestedModelId
|
||||
);
|
||||
/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test(requestedModelId);
|
||||
let selectedModelConfig = selectedModelIsUUID
|
||||
? await getAiModelById({ id: requestedModelId })
|
||||
: null;
|
||||
let selectedModelProviderType: string | undefined;
|
||||
|
||||
if (selectedModelIsUUID) {
|
||||
const selectedModelConfig = await getAiModelById({ id: requestedModelId });
|
||||
if (selectedModelConfig && isAiModelAgentAuto(selectedModelConfig)) {
|
||||
if (!selectedModelConfig) {
|
||||
const enabledModels = await getEnabledAiModelsWithProvider(false);
|
||||
const matchedModel = enabledModels.find(
|
||||
({ model }) => model.id === requestedModelId || model.modelId === requestedModelId
|
||||
);
|
||||
if (matchedModel) {
|
||||
selectedModelConfig = matchedModel.model;
|
||||
selectedModelProviderType = matchedModel.provider.type as string;
|
||||
}
|
||||
}
|
||||
|
||||
if (selectedModelConfig && !selectedModelProviderType) {
|
||||
const selectedProvider = await getAiProviderById({
|
||||
id: selectedModelConfig.providerId,
|
||||
});
|
||||
selectedModelProviderType = selectedProvider?.type as string | undefined;
|
||||
}
|
||||
|
||||
if (selectedModelConfig) {
|
||||
const fixedImageProviderTypes = new Set([
|
||||
"gemini",
|
||||
"huanke-gemini",
|
||||
"alibaba-image",
|
||||
"ruxa-image",
|
||||
"volcano-vision",
|
||||
]);
|
||||
const isFixedImageModel =
|
||||
!selectedModelConfig.isSubModel &&
|
||||
(selectedModelConfig.supportsImageGen ||
|
||||
(selectedModelProviderType
|
||||
? fixedImageProviderTypes.has(selectedModelProviderType)
|
||||
: false));
|
||||
|
||||
if (isAiModelAgentAuto(selectedModelConfig)) {
|
||||
shouldRunAgentConversation = true;
|
||||
shouldKeepSelectedModel = false;
|
||||
console.log(
|
||||
`[Smart Auto Mode] 🧠 Agent-only model detected, force auto routing: ${selectedModelConfig.displayName}`
|
||||
);
|
||||
} else if (selectedModelConfig?.supportsImageGen && !selectedModelConfig?.isSubModel) {
|
||||
} else if (isFixedImageModel) {
|
||||
shouldRunAgentConversation = true;
|
||||
shouldKeepSelectedModel = true;
|
||||
console.log(
|
||||
`[Smart Auto Mode] 🎯 Agent conversation enabled for fixed image model: ${selectedModelConfig.displayName}`
|
||||
`[Smart Auto Mode] 🎯 Agent conversation enabled for fixed image model: ${selectedModelConfig.displayName} (${selectedModelProviderType || "unknown"})`
|
||||
);
|
||||
}
|
||||
}
|
||||
@ -982,6 +1074,7 @@ export async function POST(request: Request) {
|
||||
needsClarification: autoGraphResult.analysis.needsClarification,
|
||||
});
|
||||
|
||||
smartAutoDetectedIntent = autoGraphResult.analysis.intent;
|
||||
smartAutoNeedsClarification = Boolean(autoGraphResult.analysis.needsClarification);
|
||||
smartAutoClarificationQuestion = autoGraphResult.analysis.clarifyingQuestions?.[0];
|
||||
}
|
||||
@ -1023,6 +1116,10 @@ export async function POST(request: Request) {
|
||||
optimized: autoGraphResult.optimizedPrompt.substring(0, 100),
|
||||
});
|
||||
}
|
||||
if (autoGraphResult.recommendedImageParams) {
|
||||
smartAutoRecommendedImageParams = autoGraphResult.recommendedImageParams;
|
||||
console.log("[Smart Auto Mode] 🎛️ Agent recommended image params:", smartAutoRecommendedImageParams);
|
||||
}
|
||||
|
||||
if (autoGraphResult.route && !shouldKeepSelectedModel) {
|
||||
finalModelId = autoGraphResult.route.id;
|
||||
@ -1092,6 +1189,50 @@ export async function POST(request: Request) {
|
||||
}
|
||||
// ========== End Smart Auto Mode Logic ==========
|
||||
|
||||
if (
|
||||
shouldRunAgentConversation &&
|
||||
!smartAutoNeedsClarification &&
|
||||
!smartAutoFriendlyMessage?.trim()
|
||||
) {
|
||||
const latestUserText = message.parts
|
||||
.filter((part): part is { type: "text"; text: string } => part.type === "text")
|
||||
.map((part) => part.text)
|
||||
.join(" ");
|
||||
const hasCurrentImageInput = message.parts.some((part: any) =>
|
||||
part?.type === "file" || part?.type === "image" || part?.type === "image_url"
|
||||
);
|
||||
const hasHistoryImageInput = smartAutoHistoryImages.length > 0;
|
||||
smartAutoFriendlyMessage = buildAutoModeFallbackMessage({
|
||||
text: latestUserText,
|
||||
hasImage: hasCurrentImageInput || hasHistoryImageInput,
|
||||
fromHistory: !hasCurrentImageInput && hasHistoryImageInput,
|
||||
});
|
||||
console.log("[Smart Auto Mode] 🧩 Backfilled conversational reply with fallback");
|
||||
}
|
||||
|
||||
const hasUserCustomImageParams = hasUserCustomizedImageParams(requestBody.imageParams);
|
||||
const shouldApplyAgentImageParams =
|
||||
shouldRunAgentConversation &&
|
||||
!smartAutoNeedsClarification &&
|
||||
(smartAutoDetectedIntent === "image_gen" || smartAutoDetectedIntent === "image_edit");
|
||||
const effectiveImageParams = shouldApplyAgentImageParams
|
||||
? mergeImageParamsWithAgentRecommendation({
|
||||
incoming: requestBody.imageParams,
|
||||
recommended: smartAutoRecommendedImageParams,
|
||||
intent: smartAutoDetectedIntent,
|
||||
hasUserCustomSettings: hasUserCustomImageParams,
|
||||
})
|
||||
: requestBody.imageParams;
|
||||
|
||||
if (shouldApplyAgentImageParams) {
|
||||
console.log("[Smart Auto Mode] 🎯 Effective image params:", {
|
||||
incoming: requestBody.imageParams,
|
||||
recommended: smartAutoRecommendedImageParams,
|
||||
merged: effectiveImageParams,
|
||||
hasUserCustomImageParams,
|
||||
});
|
||||
}
|
||||
|
||||
const session = await auth();
|
||||
|
||||
if (!session?.user) {
|
||||
@ -1359,7 +1500,7 @@ export async function POST(request: Request) {
|
||||
const isGeminiImageProvider = ["gemini-image", "huanke-gemini"].includes((languageModel as any).provider);
|
||||
if (isGeminiImageProvider && typeof (languageModel as any).doGenerate === "function") {
|
||||
console.log("[Chat API] Using Gemini image generation, provider:", (languageModel as any).provider);
|
||||
console.log("[Chat API] Image params:", requestBody.imageParams);
|
||||
console.log("[Chat API] Image params:", effectiveImageParams);
|
||||
|
||||
// 🆕 获取备用模型配置
|
||||
const currentModelConfig = await getAiModelById({ id: finalModelId });
|
||||
@ -1445,7 +1586,7 @@ export async function POST(request: Request) {
|
||||
// 调用 doGenerate 获取图片（如果是备用模型，传入 overrideModelId）
|
||||
const result = await (languageModel as any).doGenerate({
|
||||
messages: modelMessages,
|
||||
imageParams: requestBody.imageParams || { aspectRatio: "auto", imageSize: "2K" },
|
||||
imageParams: effectiveImageParams || { aspectRatio: "auto", imageSize: "2K" },
|
||||
overrideModelId: isRetry ? modelIdToUse : undefined, // 🆕 使用备用模型 ID
|
||||
});
|
||||
|
||||
@ -1721,7 +1862,7 @@ export async function POST(request: Request) {
|
||||
const isAlibabaLikeImageProvider = ["alibaba-image", "ruxa-image"].includes((languageModel as any).provider);
|
||||
if (isAlibabaLikeImageProvider && typeof (languageModel as any).doGenerate === "function") {
|
||||
console.log("[Chat API] Using image generation provider:", (languageModel as any).provider);
|
||||
console.log("[Chat API] Image params:", requestBody.imageParams);
|
||||
console.log("[Chat API] Image params:", effectiveImageParams);
|
||||
|
||||
// 🆕 获取模型显示名称，用于错误提示
|
||||
const alibabaModelConfig = await getAiModelById({ id: finalModelId });
|
||||
@ -1811,7 +1952,7 @@ export async function POST(request: Request) {
|
||||
// 调用 doGenerate 获取图片，传递模型显示名称
|
||||
const result = await (languageModel as any).doGenerate({
|
||||
messages: modelMessages,
|
||||
imageParams: requestBody.imageParams || { aspectRatio: "auto", imageSize: "2K", imageCount: 1 },
|
||||
imageParams: effectiveImageParams || { aspectRatio: "auto", imageSize: "2K", imageCount: 1 },
|
||||
modelDisplayName, // 🆕 传递模型显示名称
|
||||
overrideModelId: isRetry ? modelIdToUse : undefined,
|
||||
});
|
||||
@ -1909,7 +2050,7 @@ export async function POST(request: Request) {
|
||||
|
||||
const candidateResult = await (candidateLanguageModel as any).doGenerate({
|
||||
messages: candidateMessages,
|
||||
imageParams: requestBody.imageParams || { aspectRatio: "auto", imageSize: "2K", imageCount: 1 },
|
||||
imageParams: effectiveImageParams || { aspectRatio: "auto", imageSize: "2K", imageCount: 1 },
|
||||
modelDisplayName: candidate.model.displayName,
|
||||
overrideModelId: candidate.model.modelId,
|
||||
});
|
||||
@ -2476,7 +2617,7 @@ export async function POST(request: Request) {
|
||||
const rawText = generated.text?.trim() || "图片生成失败，请稍后重试";
|
||||
const normalizedText = await normalizeAssistantImageMarkdown(
|
||||
rawText,
|
||||
requestBody.imageParams?.imageSize || "2K"
|
||||
effectiveImageParams?.imageSize || "2K"
|
||||
);
|
||||
|
||||
const messageId = generateUUID();
|
||||
|
||||
@ -360,10 +360,7 @@ function PureMessages({
|
||||
|
||||
// 🆕 智能自动模式：确保文字可见后再显示生图占位图
|
||||
useEffect(() => {
|
||||
const lastUserMessage = [...messages].reverse().find(m => m.role === "user");
|
||||
const isAutoModeWithImage = selectedModelId === "auto" &&
|
||||
lastUserMessage?.parts?.some((p: any) => p.type === "file" || p.type === "image");
|
||||
|
||||
const isAutoMode = selectedModelId === "auto";
|
||||
const lastUserIndex = messages.findLastIndex((m) => m.role === "user");
|
||||
const assistantMessagesAfterUser = lastUserIndex >= 0
|
||||
? messages.slice(lastUserIndex + 1).filter((m) => m.role === "assistant")
|
||||
@ -375,18 +372,27 @@ function PureMessages({
|
||||
})
|
||||
);
|
||||
|
||||
let hasImageContent = false;
|
||||
if (isAutoModeWithImage) {
|
||||
hasImageContent = assistantMessagesAfterUser.some(msg =>
|
||||
msg.parts?.some((part: any) => part.type === "text" && part.text && (
|
||||
part.text.includes("![thumb|") ||
|
||||
part.text.includes("![thumb|||") ||
|
||||
part.text.includes("![Generated Image")
|
||||
))
|
||||
);
|
||||
}
|
||||
const hasImageContent = assistantMessagesAfterUser.some((msg) =>
|
||||
msg.parts?.some(
|
||||
(part: any) =>
|
||||
part.type === "text" &&
|
||||
part.text &&
|
||||
(part.text.includes("![thumb|") ||
|
||||
part.text.includes("![thumb|||") ||
|
||||
part.text.includes("![Generated Image"))
|
||||
)
|
||||
);
|
||||
const hasGenerationStart = assistantMessagesAfterUser.some((msg) =>
|
||||
msg.parts?.some(
|
||||
(part: any) =>
|
||||
part.type === "text" &&
|
||||
part.text?.includes("<!-- IMAGE_GENERATION_START -->")
|
||||
)
|
||||
);
|
||||
const isAutoImageFlow =
|
||||
isAutoMode && (isGeneratingImageFlag || hasGenerationStart || hasImageContent);
|
||||
|
||||
if (!isAutoModeWithImage || !isGeneratingImageFlag || !hasAssistantTextReplyWithContent || hasImageContent) {
|
||||
if (!isAutoImageFlow || !isGeneratingImageFlag || !hasAssistantTextReplyWithContent || hasImageContent) {
|
||||
if (autoImagePlaceholderTimerRef.current) {
|
||||
clearTimeout(autoImagePlaceholderTimerRef.current);
|
||||
autoImagePlaceholderTimerRef.current = null;
|
||||
@ -577,10 +583,7 @@ function PureMessages({
|
||||
const showAlibabaImage = modelType === "alibaba-image" || (!modelType && isAlibabaImageModel);
|
||||
const showVideo = modelType === "volcano-video" || modelType === "qianwen-video" || (!modelType && (isQianwenVideoModel || isVideoModel));
|
||||
|
||||
// 🆕 智能自动模式：检查用户是否发送了图片
|
||||
const isAutoMode = selectedModelId === "auto";
|
||||
const lastUserMsg = [...messages].reverse().find(m => m.role === "user");
|
||||
const userMessageHasImage = lastUserMsg?.parts?.some((p: any) => p.type === "file" || p.type === "image");
|
||||
|
||||
// 🆕 检查是否收到了 GENERATING_IMAGE 标记（表示对话已发送，开始生图）
|
||||
const lastUserIndexForMarkers = messages.findLastIndex(m => m.role === "user");
|
||||
@ -598,6 +601,16 @@ function PureMessages({
|
||||
const assistantMessagesAfterUser = lastUserIndexForMarkers >= 0
|
||||
? messages.slice(lastUserIndexForMarkers + 1).filter((msg) => msg.role === "assistant")
|
||||
: messages.filter((msg) => msg.role === "assistant");
|
||||
const hasImageContent = assistantMessagesAfterUser.some((msg) =>
|
||||
msg.parts?.some(
|
||||
(part: any) =>
|
||||
part.type === "text" &&
|
||||
part.text &&
|
||||
(part.text.includes("![thumb|") ||
|
||||
part.text.includes("![thumb|||") ||
|
||||
part.text.includes("![Generated Image"))
|
||||
)
|
||||
);
|
||||
const hasAssistantTextReplyWithContent = assistantMessagesAfterUser.some((msg) =>
|
||||
msg.parts?.some((part: any) => {
|
||||
if (part.type !== "text" || !part.text) return false;
|
||||
@ -605,9 +618,11 @@ function PureMessages({
|
||||
})
|
||||
);
|
||||
const shouldWaitForAgentReplyBeforeImagePlaceholder =
|
||||
userMessageHasImage &&
|
||||
(isAutoMode || showGemini || showVolcano || showAlibabaImage) &&
|
||||
!hasAssistantTextReplyWithContent;
|
||||
const isAutoImageFlow =
|
||||
isAutoMode &&
|
||||
(isGeneratingImageFlag || hasGeneratingMarker || hasGenerationEnd || hasImageContent);
|
||||
|
||||
if (showVideo) return <QianwenVideoLoading />;
|
||||
if (shouldWaitForAgentReplyBeforeImagePlaceholder) {
|
||||
@ -633,7 +648,7 @@ function PureMessages({
|
||||
}
|
||||
|
||||
// 🆕 智能自动模式占位图
|
||||
if (isAutoMode && userMessageHasImage) {
|
||||
if (isAutoImageFlow) {
|
||||
// 🆕 只用 isGeneratingImageFlag 来控制是否显示生图占位图
|
||||
// isGeneratingImageFlag 由 chat.tsx 控制：延迟 3 秒后设置为 true，收到图片后设置为 false
|
||||
if (
|
||||
|
||||
@ -8,6 +8,14 @@ interface HistoryMessage {
|
||||
content: string;
|
||||
}
|
||||
|
||||
export interface AgentRecommendedImageParams {
|
||||
aspectRatio?: string;
|
||||
imageSize?: string;
|
||||
imageCount?: number;
|
||||
negativePrompt?: string;
|
||||
promptExtend?: boolean;
|
||||
}
|
||||
|
||||
export interface AutoAgentInput {
|
||||
text: string;
|
||||
hasImage: boolean;
|
||||
@ -21,6 +29,7 @@ export interface AutoAgentResult {
|
||||
analysis?: AnalysisResult;
|
||||
friendlyMessage?: string;
|
||||
optimizedPrompt?: string;
|
||||
recommendedImageParams?: AgentRecommendedImageParams;
|
||||
route?: RouteResult;
|
||||
}
|
||||
|
||||
@ -34,6 +43,7 @@ const AutoAgentState = Annotation.Root({
|
||||
analysis: Annotation<AnalysisResult | null>,
|
||||
friendlyMessage: Annotation<string | null>,
|
||||
optimizedPrompt: Annotation<string | null>,
|
||||
recommendedImageParams: Annotation<AgentRecommendedImageParams | null>,
|
||||
route: Annotation<RouteResult | null>,
|
||||
});
|
||||
|
||||
@ -45,6 +55,98 @@ function compactText(text: string, max = 48) {
|
||||
return `${normalized.slice(0, max)}...`;
|
||||
}
|
||||
|
||||
function normalizeAspectRatio(value?: string): string | undefined {
|
||||
if (!value) return undefined;
|
||||
const normalized = value.trim().toLowerCase();
|
||||
if (normalized === "auto") return "auto";
|
||||
const raw = value.trim();
|
||||
const allowed = new Set([
|
||||
"1:1",
|
||||
"2:3",
|
||||
"3:2",
|
||||
"3:4",
|
||||
"4:3",
|
||||
"4:5",
|
||||
"5:4",
|
||||
"9:16",
|
||||
"16:9",
|
||||
"21:9",
|
||||
]);
|
||||
return allowed.has(raw) ? raw : undefined;
|
||||
}
|
||||
|
||||
function normalizeImageSize(value?: string): string | undefined {
|
||||
if (!value) return undefined;
|
||||
const normalized = value.trim().toUpperCase();
|
||||
if (normalized === "1K" || normalized === "2K" || normalized === "4K") {
|
||||
return normalized;
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function sanitizeRecommendedImageParams(
|
||||
params: AgentRecommendedImageParams | undefined,
|
||||
intent?: AnalysisResult["intent"],
|
||||
): AgentRecommendedImageParams | undefined {
|
||||
if (!params) return undefined;
|
||||
const safe: AgentRecommendedImageParams = {};
|
||||
const normalizedAspectRatio = normalizeAspectRatio(params.aspectRatio);
|
||||
const normalizedImageSize = normalizeImageSize(params.imageSize);
|
||||
if (normalizedAspectRatio) safe.aspectRatio = normalizedAspectRatio;
|
||||
if (normalizedImageSize) safe.imageSize = normalizedImageSize;
|
||||
|
||||
if (typeof params.imageCount === "number" && Number.isFinite(params.imageCount)) {
|
||||
const min = 1;
|
||||
const max = intent === "image_edit" ? 1 : 4;
|
||||
safe.imageCount = Math.min(Math.max(Math.round(params.imageCount), min), max);
|
||||
}
|
||||
if (intent === "image_edit") {
|
||||
safe.imageCount = 1;
|
||||
}
|
||||
|
||||
if (typeof params.negativePrompt === "string" && params.negativePrompt.trim()) {
|
||||
safe.negativePrompt = params.negativePrompt.trim().slice(0, 500);
|
||||
}
|
||||
if (typeof params.promptExtend === "boolean") {
|
||||
safe.promptExtend = params.promptExtend;
|
||||
}
|
||||
|
||||
return Object.keys(safe).length > 0 ? safe : undefined;
|
||||
}
|
||||
|
||||
function heuristicImageParams(
|
||||
text: string,
|
||||
analysis: AnalysisResult,
|
||||
): AgentRecommendedImageParams | undefined {
|
||||
const normalized = text.toLowerCase();
|
||||
const params: AgentRecommendedImageParams = {
|
||||
imageSize: "2K",
|
||||
promptExtend: true,
|
||||
};
|
||||
|
||||
if (analysis.intent === "image_edit") {
|
||||
params.imageCount = 1;
|
||||
} else if (/多张|四张|4张|三张|3张/.test(normalized)) {
|
||||
params.imageCount = /四张|4张/.test(normalized) ? 4 : 3;
|
||||
} else if (/两张|2张|双图/.test(normalized)) {
|
||||
params.imageCount = 2;
|
||||
} else {
|
||||
params.imageCount = 1;
|
||||
}
|
||||
|
||||
if (/横图|横版|宽屏|封面|banner|海报/.test(normalized)) {
|
||||
params.aspectRatio = "16:9";
|
||||
} else if (/竖图|竖版|手机壁纸|全身|海报竖版/.test(normalized)) {
|
||||
params.aspectRatio = "9:16";
|
||||
} else if (/方图|方形|头像|logo|电商主图/.test(normalized)) {
|
||||
params.aspectRatio = "1:1";
|
||||
} else {
|
||||
params.aspectRatio = "auto";
|
||||
}
|
||||
|
||||
return sanitizeRecommendedImageParams(params, analysis.intent);
|
||||
}
|
||||
|
||||
function buildStrategicFriendlyMessage(
|
||||
analysis: AnalysisResult,
|
||||
state: AutoAgentStateType,
|
||||
@ -60,6 +162,58 @@ function buildStrategicFriendlyMessage(
|
||||
return generateAgentConversation(analysis, state);
|
||||
}
|
||||
|
||||
type EnabledModelWithProvider = Awaited<
|
||||
ReturnType<typeof import("@/lib/db/queries")["getEnabledAiModelsWithProvider"]>
|
||||
>[number];
|
||||
|
||||
function pickQwenVisionModel(models: EnabledModelWithProvider[]) {
|
||||
const visionModels = models.filter((item) => item.model.supportsVision && item.model.isEnabled);
|
||||
const qwenByName = visionModels.find(
|
||||
(item) =>
|
||||
item.provider.type === "alibaba" &&
|
||||
/(qwen|通义|vl)/i.test(`${item.model.displayName} ${item.model.modelId}`)
|
||||
);
|
||||
if (qwenByName) return qwenByName;
|
||||
|
||||
const alibabaVision = visionModels.find((item) => item.provider.type === "alibaba");
|
||||
if (alibabaVision) return alibabaVision;
|
||||
return visionModels[0];
|
||||
}
|
||||
|
||||
function pickAgentReasoningModel(
|
||||
agentModel: EnabledModelWithProvider | undefined,
|
||||
models: EnabledModelWithProvider[],
|
||||
) {
|
||||
const isReasoningCompatibleProvider = (providerType: string) =>
|
||||
["zhipu", "alibaba", "openai", "custom", "local"].includes(providerType);
|
||||
|
||||
if (agentModel && isReasoningCompatibleProvider(agentModel.provider.type)) return agentModel;
|
||||
const zhipuModel = models.find(
|
||||
(item) =>
|
||||
item.provider.type === "zhipu" &&
|
||||
item.model.isEnabled &&
|
||||
!item.model.isSubModel &&
|
||||
isReasoningCompatibleProvider(item.provider.type)
|
||||
);
|
||||
if (zhipuModel) return zhipuModel;
|
||||
return models.find(
|
||||
(item) =>
|
||||
item.model.isEnabled &&
|
||||
!item.model.isSubModel &&
|
||||
isReasoningCompatibleProvider(item.provider.type)
|
||||
);
|
||||
}
|
||||
|
||||
function parseJsonBlock<T = Record<string, unknown>>(text: string): T | null {
|
||||
const match = text.match(/\{[\s\S]*\}/);
|
||||
if (!match) return null;
|
||||
try {
|
||||
return JSON.parse(match[0]) as T;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
async function generateAgentConversation(
|
||||
analysis: AnalysisResult,
|
||||
state: AutoAgentStateType
|
||||
@ -72,25 +226,12 @@ async function generateAgentConversation(
|
||||
const { getLanguageModelById } = await import("./dynamic-provider");
|
||||
const allModels = await getEnabledAiModelsWithProvider();
|
||||
const agentModel = await getAgentAiModelWithProvider();
|
||||
const canUseAgentModel = Boolean(
|
||||
agentModel && (!state.hasImage || agentModel.model.supportsVision)
|
||||
);
|
||||
|
||||
const preferredVisionModel = allModels.find(
|
||||
(m) => m.provider.type === "alibaba" && m.model.supportsVision && m.model.isEnabled
|
||||
);
|
||||
const candidateModel =
|
||||
(canUseAgentModel ? agentModel : undefined) ||
|
||||
(state.hasImage
|
||||
? preferredVisionModel || allModels.find((m) => m.model.supportsVision && m.model.isEnabled)
|
||||
: allModels.find((m) => m.model.isEnabled && !m.model.isSubModel));
|
||||
|
||||
if (!candidateModel) {
|
||||
const reasoningModelConfig = pickAgentReasoningModel(agentModel, allModels);
|
||||
if (!reasoningModelConfig) {
|
||||
return analysis.conversationalReply || null;
|
||||
}
|
||||
|
||||
const model = await getLanguageModelById(candidateModel.model.id);
|
||||
if (!model) {
|
||||
const reasoningModel = await getLanguageModelById(reasoningModelConfig.model.id);
|
||||
if (!reasoningModel) {
|
||||
return analysis.conversationalReply || null;
|
||||
}
|
||||
|
||||
@ -110,6 +251,15 @@ async function generateAgentConversation(
|
||||
let visualAnalysis = "";
|
||||
if (state.hasImage && validImageUrls.length > 0) {
|
||||
try {
|
||||
const visionModelConfig = pickQwenVisionModel(allModels);
|
||||
if (!visionModelConfig) {
|
||||
throw new Error("missing_vision_model");
|
||||
}
|
||||
const visionModel = await getLanguageModelById(visionModelConfig.model.id);
|
||||
if (!visionModel) {
|
||||
throw new Error("missing_vision_runtime");
|
||||
}
|
||||
|
||||
const visionPrompt = `请分析以下参考图片，给出可用于生图/修图执行前决策的结果。
|
||||
要求：
|
||||
1) 每张图输出 1 行，格式：图X：主体...；风格...；构图...；光影与色调...；材质/氛围...
|
||||
@ -118,7 +268,7 @@ async function generateAgentConversation(
|
||||
|
||||
const visionInputs = validImageUrls.map((url) => ({ type: "image" as const, image: url }));
|
||||
const { text: visionText } = await generateText({
|
||||
model,
|
||||
model: visionModel,
|
||||
messages: [
|
||||
{
|
||||
role: "user" as const,
|
||||
@ -130,26 +280,34 @@ async function generateAgentConversation(
|
||||
} as any);
|
||||
|
||||
visualAnalysis = visionText.trim();
|
||||
console.log(
|
||||
`[Auto Agent] Vision analysis model: ${visionModelConfig.model.displayName} (${visionModelConfig.provider.type})`
|
||||
);
|
||||
} catch (error) {
|
||||
console.warn("[Auto Agent] Vision analysis failed, continue without visual summary");
|
||||
visualAnalysis = "";
|
||||
}
|
||||
}
|
||||
|
||||
const systemPrompt = `你是一个真实执行中的 AI 生图/修图 Agent。
|
||||
请根据用户需求和本轮输入图片，给出“执行前”的分析汇报。
|
||||
const systemPrompt = `你是一个真实执行中的 AI 生图/修图 Agent。你的核心目标是提升用户一次生成成功率。
|
||||
|
||||
要求：
|
||||
1) 不要说“我看不到图片”或“无法访问图片”，必须基于给定图片做真实分析。
|
||||
2) 语气自然、专业，不要模板腔，不要空话。
|
||||
3) 不暴露内部实现、模型路由、算法细节。
|
||||
4) 输出 Markdown，控制在 8-14 行。
|
||||
5) 有图片时，必须按顺序输出以下 Markdown H3 标题（逐字一致）：
|
||||
- ### 需求理解
|
||||
- ### 图片分析
|
||||
- ### 执行计划
|
||||
- ### 输出设置
|
||||
6) “图片分析”要结合每张参考图差异，给出风格/构图/材质/光影等具体观察。
|
||||
7) 只有明确拿到“原图分辨率”时，才能写具体像素尺寸；否则只写宽高比/清晰度目标，不要编造 335x500 这类预览尺寸。`;
|
||||
你必须按以下 AGENT SKILLS 执行：
|
||||
1) 需求拆解：把用户目标拆成可执行指令，优先满足用户明确要求。
|
||||
2) 风险预判：主动识别可能导致失败的歧义（主体数量、视角、风格冲突、文字内容）。
|
||||
3) 提示词策略：给出可落地的正向约束和必要的负向约束，避免空泛描述。
|
||||
4) 参数策略：在输出设置中明确建议宽高比、清晰度档位、出图数量（若用户未指定）。
|
||||
|
||||
输出要求：
|
||||
- 不要说“看不到图片/无法访问图片”。
|
||||
- 不暴露内部实现、模型路由、算法细节。
|
||||
- 输出 Markdown，控制在 8-14 行。
|
||||
- 有图片时必须按顺序输出以下 Markdown H3 标题（逐字一致）：
|
||||
### 需求理解
|
||||
### 图片分析
|
||||
### 执行计划
|
||||
### 输出设置
|
||||
- “图片分析”必须基于视觉分析结果写具体观察，不允许写泛化空话。
|
||||
- 只有明确拿到原图分辨率时，才能写具体像素尺寸；否则只写宽高比/清晰度目标。`;
|
||||
|
||||
const prompt = [
|
||||
`【用户原始需求】${state.text || "（无文本）"}`,
|
||||
@ -165,44 +323,167 @@ async function generateAgentConversation(
|
||||
.filter(Boolean)
|
||||
.join("\n\n");
|
||||
|
||||
const imageInputs = validImageUrls.map((url) => ({ type: "image" as const, image: url }));
|
||||
|
||||
const requestPayload = state.hasImage && imageInputs.length > 0
|
||||
? {
|
||||
model,
|
||||
system: systemPrompt,
|
||||
messages: [
|
||||
{
|
||||
role: "user" as const,
|
||||
content: [
|
||||
{ type: "text" as const, text: prompt },
|
||||
...imageInputs,
|
||||
],
|
||||
},
|
||||
],
|
||||
temperature: 0.35,
|
||||
maxTokens: 520,
|
||||
}
|
||||
: {
|
||||
model,
|
||||
system: systemPrompt,
|
||||
prompt,
|
||||
temperature: 0.35,
|
||||
maxTokens: 420,
|
||||
};
|
||||
|
||||
const { text } = await generateText(requestPayload as any);
|
||||
const { text } = await generateText({
|
||||
model: reasoningModel,
|
||||
system: systemPrompt,
|
||||
prompt,
|
||||
temperature: 0.35,
|
||||
maxTokens: 520,
|
||||
} as any);
|
||||
|
||||
const cleaned = text.trim();
|
||||
if (cleaned.length < 10) {
|
||||
return analysis.conversationalReply || null;
|
||||
}
|
||||
console.log(
|
||||
`[Auto Agent] Conversation model: ${reasoningModelConfig.model.displayName} (${reasoningModelConfig.provider.type})`
|
||||
);
|
||||
return cleaned;
|
||||
} catch (error) {
|
||||
return analysis.conversationalReply || null;
|
||||
}
|
||||
}
|
||||
|
||||
async function recommendImageParamsWithAgent(
|
||||
analysis: AnalysisResult,
|
||||
state: AutoAgentStateType,
|
||||
): Promise<AgentRecommendedImageParams | undefined> {
|
||||
if (analysis.intent !== "image_edit" && analysis.intent !== "image_gen") {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
try {
|
||||
const {
|
||||
getEnabledAiModelsWithProvider,
|
||||
getAgentAiModelWithProvider,
|
||||
} = await import("@/lib/db/queries");
|
||||
const { getLanguageModelById } = await import("./dynamic-provider");
|
||||
const allModels = await getEnabledAiModelsWithProvider();
|
||||
const agentModel = await getAgentAiModelWithProvider();
|
||||
const plannerModelConfig = pickAgentReasoningModel(agentModel, allModels);
|
||||
if (!plannerModelConfig) {
|
||||
return heuristicImageParams(state.text, analysis);
|
||||
}
|
||||
const plannerModel = await getLanguageModelById(plannerModelConfig.model.id);
|
||||
if (!plannerModel) {
|
||||
return heuristicImageParams(state.text, analysis);
|
||||
}
|
||||
|
||||
const history = state.historyMessages
|
||||
.slice(-4)
|
||||
.map((msg) => `${msg.role === "user" ? "用户" : "助手"}: ${compactText(msg.content, 120)}`)
|
||||
.join("\n");
|
||||
|
||||
const requestText = [
|
||||
`用户请求：${state.text || "（无文本）"}`,
|
||||
`识别意图：${analysis.intent}/${analysis.subIntent || "none"}`,
|
||||
`是否有参考图：${state.hasImage ? "是" : "否"}`,
|
||||
history ? `最近对话：\n${history}` : "",
|
||||
"请给出本次最稳妥的生图参数建议。",
|
||||
]
|
||||
.filter(Boolean)
|
||||
.join("\n\n");
|
||||
|
||||
const { text } = await generateText({
|
||||
model: plannerModel,
|
||||
system: `你是生图参数规划 Agent。目标是提升首图命中率和稳定性。
|
||||
只输出 JSON，不要输出解释。格式如下：
|
||||
{
|
||||
"aspectRatio": "1:1|2:3|3:2|3:4|4:3|4:5|5:4|9:16|16:9|21:9|auto",
|
||||
"imageSize": "1K|2K|4K",
|
||||
"imageCount": 1,
|
||||
"negativePrompt": "可选，简洁",
|
||||
"promptExtend": true
|
||||
}
|
||||
规则：
|
||||
- image_edit 场景强制 imageCount=1。
|
||||
- 若用户未指定比例，优先 auto。
|
||||
- negativePrompt 仅在明显需要避坑时填写。`,
|
||||
prompt: requestText,
|
||||
temperature: 0.15,
|
||||
maxTokens: 220,
|
||||
} as any);
|
||||
|
||||
const parsed = parseJsonBlock<AgentRecommendedImageParams>(text);
|
||||
const sanitized = sanitizeRecommendedImageParams(parsed || undefined, analysis.intent);
|
||||
if (sanitized) {
|
||||
console.log(
|
||||
`[Auto Agent] Parameter planner model: ${plannerModelConfig.model.displayName} (${plannerModelConfig.provider.type})`
|
||||
);
|
||||
return sanitized;
|
||||
}
|
||||
return heuristicImageParams(state.text, analysis);
|
||||
} catch {
|
||||
return heuristicImageParams(state.text, analysis);
|
||||
}
|
||||
}
|
||||
|
||||
async function refineRouteWithAgent(
|
||||
state: AutoAgentStateType,
|
||||
baseRoute: RouteResult,
|
||||
): Promise<RouteResult> {
|
||||
if (!state.analysis || (state.analysis.intent !== "image_edit" && state.analysis.intent !== "image_gen")) {
|
||||
return baseRoute;
|
||||
}
|
||||
|
||||
try {
|
||||
const {
|
||||
getEnabledAiModelsWithProvider,
|
||||
getAgentAiModelWithProvider,
|
||||
} = await import("@/lib/db/queries");
|
||||
const { getLanguageModelById } = await import("./dynamic-provider");
|
||||
const { isAiModelAgentAuto } = await import("@/lib/db/ai-model-flags");
|
||||
const allModelsRaw = await getEnabledAiModelsWithProvider(false);
|
||||
const allModels = allModelsRaw.filter((item) => !isAiModelAgentAuto(item.model));
|
||||
const imageCandidates = allModels
|
||||
.filter((item) => item.model.supportsImageGen && item.model.isEnabled && !item.model.isSubModel)
|
||||
.slice(0, 12);
|
||||
if (imageCandidates.length <= 1) return baseRoute;
|
||||
|
||||
const agentModel = await getAgentAiModelWithProvider();
|
||||
const plannerModelConfig = pickAgentReasoningModel(agentModel, allModelsRaw);
|
||||
if (!plannerModelConfig) return baseRoute;
|
||||
const plannerModel = await getLanguageModelById(plannerModelConfig.model.id);
|
||||
if (!plannerModel) return baseRoute;
|
||||
|
||||
const candidatesText = imageCandidates
|
||||
.map((item, index) => `${index + 1}. id=${item.model.id}; 名称=${item.model.displayName}; provider=${item.provider.type}`)
|
||||
.join("\n");
|
||||
|
||||
const { text } = await generateText({
|
||||
model: plannerModel,
|
||||
system: `你是生图模型选择 Agent。目标是优先提高一次成功率。
|
||||
只输出 JSON：
|
||||
{
|
||||
"modelId": "候选里的id",
|
||||
"reason": "一句话中文理由，不超过24字"
|
||||
}
|
||||
如果无法判断，也必须给出候选中的 modelId。`,
|
||||
prompt: `用户请求：${state.text || "（无文本）"}
|
||||
意图：${state.analysis.intent}/${state.analysis.subIntent || "none"}
|
||||
候选模型：
|
||||
${candidatesText}`,
|
||||
temperature: 0.1,
|
||||
maxTokens: 120,
|
||||
} as any);
|
||||
|
||||
const parsed = parseJsonBlock<{ modelId?: string; reason?: string }>(text);
|
||||
if (!parsed?.modelId) return baseRoute;
|
||||
const chosen = imageCandidates.find((item) => item.model.id === parsed.modelId);
|
||||
if (!chosen) return baseRoute;
|
||||
|
||||
return {
|
||||
id: chosen.model.id,
|
||||
displayName: chosen.model.displayName,
|
||||
providerType: chosen.provider.type,
|
||||
modelId: chosen.model.modelId,
|
||||
reason: `Agent规划: ${(parsed.reason || chosen.model.displayName).slice(0, 40)}`,
|
||||
};
|
||||
} catch {
|
||||
return baseRoute;
|
||||
}
|
||||
}
|
||||
|
||||
const analyzeNode = async (state: AutoAgentStateType) => {
|
||||
const { analyzeIntent } = await import("./smart-analyzer");
|
||||
const analysis = await analyzeIntent({
|
||||
@ -243,7 +524,8 @@ const optimizeNode = async (state: AutoAgentStateType) => {
|
||||
return {};
|
||||
}
|
||||
if (state.analysis.intent === "image_edit" && !state.analysis.subIntent) {
|
||||
return {};
|
||||
const recommendedImageParams = await recommendImageParamsWithAgent(state.analysis, state);
|
||||
return { recommendedImageParams };
|
||||
}
|
||||
if (state.analysis.intent !== "image_edit" && state.analysis.intent !== "image_gen") {
|
||||
return {};
|
||||
@ -256,16 +538,31 @@ const optimizeNode = async (state: AutoAgentStateType) => {
|
||||
historyMessages: state.historyMessages,
|
||||
context: state.context ?? undefined,
|
||||
});
|
||||
return { optimizedPrompt: optimized.optimizedPrompt };
|
||||
const mergedParams = sanitizeRecommendedImageParams(
|
||||
{
|
||||
...optimized.recommendedImageParams,
|
||||
negativePrompt:
|
||||
optimized.negativePrompt && optimized.negativePrompt.trim().length > 0
|
||||
? optimized.negativePrompt
|
||||
: optimized.recommendedImageParams?.negativePrompt,
|
||||
},
|
||||
state.analysis.intent,
|
||||
) ?? (await recommendImageParamsWithAgent(state.analysis, state));
|
||||
|
||||
return {
|
||||
optimizedPrompt: optimized.optimizedPrompt,
|
||||
recommendedImageParams: mergedParams,
|
||||
};
|
||||
};
|
||||
|
||||
const routeNode = async (state: AutoAgentStateType) => {
|
||||
const { routeToModel } = await import("./model-router");
|
||||
const route = await routeToModel({
|
||||
const baseRoute = await routeToModel({
|
||||
intent: state.analysis?.intent || "chat",
|
||||
subIntent: state.analysis?.subIntent,
|
||||
hasImage: state.hasImage,
|
||||
});
|
||||
const route = await refineRouteWithAgent(state, baseRoute);
|
||||
return { route };
|
||||
};
|
||||
|
||||
@ -305,6 +602,7 @@ export async function runAutoAgentGraph(params: AutoAgentInput): Promise<AutoAge
|
||||
analysis: null,
|
||||
friendlyMessage: null,
|
||||
optimizedPrompt: null,
|
||||
recommendedImageParams: null,
|
||||
route: null,
|
||||
});
|
||||
|
||||
@ -312,6 +610,7 @@ export async function runAutoAgentGraph(params: AutoAgentInput): Promise<AutoAge
|
||||
analysis: result.analysis ?? undefined,
|
||||
friendlyMessage: result.friendlyMessage ?? undefined,
|
||||
optimizedPrompt: result.optimizedPrompt ?? undefined,
|
||||
recommendedImageParams: result.recommendedImageParams ?? undefined,
|
||||
route: result.route ?? undefined,
|
||||
};
|
||||
}
|
||||
|
||||
@ -10,6 +10,13 @@ export interface OptimizeResult {
|
||||
originalPrompt: string;
|
||||
optimizedPrompt: string;
|
||||
negativePrompt: string;
|
||||
recommendedImageParams?: {
|
||||
aspectRatio?: string;
|
||||
imageSize?: string;
|
||||
imageCount?: number;
|
||||
promptExtend?: boolean;
|
||||
negativePrompt?: string;
|
||||
};
|
||||
friendlyMessage?: string; // 友好的对话消息
|
||||
}
|
||||
|
||||
@ -229,7 +236,13 @@ const IMAGE_GEN_OPTIMIZER_SYSTEM_PROMPT = `你是一个专业的图片生成提
|
||||
## 输出格式（严格 JSON）
|
||||
{
|
||||
"optimizedPrompt": "适合生图模型的中文提示词",
|
||||
"negativePrompt": "需要避免的效果"
|
||||
"negativePrompt": "需要避免的效果",
|
||||
"suggestedParams": {
|
||||
"aspectRatio": "1:1|2:3|3:2|3:4|4:3|4:5|5:4|9:16|16:9|21:9|auto",
|
||||
"imageSize": "1K|2K|4K",
|
||||
"imageCount": 1,
|
||||
"promptExtend": true
|
||||
}
|
||||
}
|
||||
|
||||
## 优化原则
|
||||
@ -238,9 +251,19 @@ const IMAGE_GEN_OPTIMIZER_SYSTEM_PROMPT = `你是一个专业的图片生成提
|
||||
3. 对电商/家居场景优先强调：真实材质、空间比例合理、商业可用。
|
||||
4. negativePrompt 要包含：畸形、模糊、水印、文字错乱、额外主体、比例错误等。
|
||||
5. 输出简洁可执行，不要解释过程。
|
||||
6. 参数建议必须服务于“首图命中率”：人物全身/竖构图偏 9:16，横向场景偏 16:9，未明确时可用 auto。
|
||||
`;
|
||||
|
||||
function parseOptimizerJson(responseText: string): { optimizedPrompt?: string; negativePrompt?: string } | null {
|
||||
function parseOptimizerJson(responseText: string): {
|
||||
optimizedPrompt?: string;
|
||||
negativePrompt?: string;
|
||||
suggestedParams?: {
|
||||
aspectRatio?: string;
|
||||
imageSize?: string;
|
||||
imageCount?: number;
|
||||
promptExtend?: boolean;
|
||||
};
|
||||
} | null {
|
||||
const jsonMatch = responseText.match(/\{[\s\S]*\}/);
|
||||
if (!jsonMatch) return null;
|
||||
try {
|
||||
@ -250,6 +273,49 @@ function parseOptimizerJson(responseText: string): { optimizedPrompt?: string; n
|
||||
}
|
||||
}
|
||||
|
||||
function sanitizeSuggestedParams(input?: {
|
||||
aspectRatio?: string;
|
||||
imageSize?: string;
|
||||
imageCount?: number;
|
||||
promptExtend?: boolean;
|
||||
}) {
|
||||
if (!input) return undefined;
|
||||
const allowedAspectRatios = new Set([
|
||||
"1:1",
|
||||
"2:3",
|
||||
"3:2",
|
||||
"3:4",
|
||||
"4:3",
|
||||
"4:5",
|
||||
"5:4",
|
||||
"9:16",
|
||||
"16:9",
|
||||
"21:9",
|
||||
"auto",
|
||||
]);
|
||||
const allowedSizes = new Set(["1K", "2K", "4K"]);
|
||||
const normalizedAspectRatio = input.aspectRatio?.trim() || undefined;
|
||||
const normalizedSize = input.imageSize?.trim().toUpperCase() || undefined;
|
||||
const normalizedCount =
|
||||
typeof input.imageCount === "number" && Number.isFinite(input.imageCount)
|
||||
? Math.min(Math.max(Math.round(input.imageCount), 1), 4)
|
||||
: undefined;
|
||||
const output: NonNullable<OptimizeResult["recommendedImageParams"]> = {};
|
||||
if (normalizedAspectRatio && allowedAspectRatios.has(normalizedAspectRatio)) {
|
||||
output.aspectRatio = normalizedAspectRatio;
|
||||
}
|
||||
if (normalizedSize && allowedSizes.has(normalizedSize)) {
|
||||
output.imageSize = normalizedSize;
|
||||
}
|
||||
if (normalizedCount) {
|
||||
output.imageCount = normalizedCount;
|
||||
}
|
||||
if (typeof input.promptExtend === "boolean") {
|
||||
output.promptExtend = input.promptExtend;
|
||||
}
|
||||
return Object.keys(output).length > 0 ? output : undefined;
|
||||
}
|
||||
|
||||
function fallbackImageGenPrompt(originalPrompt: string): OptimizeResult {
|
||||
const normalized = originalPrompt.replace(/\s+/g, " ").trim();
|
||||
const optimizedPrompt = [
|
||||
@ -262,6 +328,12 @@ function fallbackImageGenPrompt(originalPrompt: string): OptimizeResult {
|
||||
originalPrompt,
|
||||
optimizedPrompt,
|
||||
negativePrompt: "低清晰度，构图混乱，比例畸形，重复主体，手部异常，文字错乱，水印，噪点，过曝，过度锐化",
|
||||
recommendedImageParams: {
|
||||
aspectRatio: "auto",
|
||||
imageSize: "2K",
|
||||
imageCount: 1,
|
||||
promptExtend: true,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
@ -300,16 +372,25 @@ export async function optimizePromptWithLLM(params: {
|
||||
} = await import("@/lib/db/queries");
|
||||
const { getLanguageModelById } = await import("./dynamic-provider");
|
||||
|
||||
// 获取千问3 VL 模型
|
||||
// 提示词优化优先使用 Agent 模型（建议智普），失败时回退
|
||||
const allModels = await getEnabledAiModelsWithProvider();
|
||||
const agentModel = await getAgentAiModelWithProvider();
|
||||
const isReasoningCompatibleProvider = (providerType: string) =>
|
||||
["zhipu", "alibaba", "openai", "custom", "local"].includes(providerType);
|
||||
const usableAgentModel =
|
||||
agentModel && isReasoningCompatibleProvider(agentModel.provider.type)
|
||||
? agentModel
|
||||
: undefined;
|
||||
const zhipuModel = allModels.find(
|
||||
(m) => m.provider.type === "zhipu" && m.model.isEnabled && !m.model.isSubModel
|
||||
);
|
||||
const qwenModel = allModels.find(
|
||||
(m) => m.provider.type === "alibaba" && m.model.supportsVision
|
||||
);
|
||||
const selectedOptimizerModel = agentModel || qwenModel;
|
||||
const selectedOptimizerModel = usableAgentModel || zhipuModel || qwenModel;
|
||||
|
||||
if (!selectedOptimizerModel) {
|
||||
console.log("[Prompt Optimizer LLM] No Qwen3 VL model, falling back to template");
|
||||
console.log("[Prompt Optimizer LLM] No optimizer model available, falling back to template");
|
||||
return optimizePrompt({ originalPrompt, intent, subIntent });
|
||||
}
|
||||
|
||||
@ -340,7 +421,11 @@ export async function optimizePromptWithLLM(params: {
|
||||
|
||||
const finalUserMessage = `${contextMessage}## 当前用户请求：\n"${originalPrompt}"\n\n## 识别的意图：${intentLabel}\n\n请生成优化后的提示词：`;
|
||||
|
||||
console.log("[Prompt Optimizer LLM] Calling Qwen3 VL for prompt optimization");
|
||||
console.log(
|
||||
"[Prompt Optimizer LLM] Calling optimizer model:",
|
||||
selectedOptimizerModel.model.displayName,
|
||||
`(${selectedOptimizerModel.provider.type})`
|
||||
);
|
||||
|
||||
const { text: responseText } = await generateText({
|
||||
model,
|
||||
@ -358,6 +443,7 @@ export async function optimizePromptWithLLM(params: {
|
||||
originalPrompt,
|
||||
optimizedPrompt: parsedResult.optimizedPrompt || originalPrompt,
|
||||
negativePrompt: parsedResult.negativePrompt || "模糊，低质量，变形，失真",
|
||||
recommendedImageParams: sanitizeSuggestedParams(parsedResult.suggestedParams),
|
||||
friendlyMessage: intent === "image_edit" ? FRIENDLY_MESSAGES[subIntent || ""]?.template : undefined,
|
||||
};
|
||||
}
|
||||
|
||||
@ -212,19 +212,32 @@ async function analyzWithLLM(
|
||||
} = await import("@/lib/db/queries");
|
||||
const { getLanguageModelById } = await import("./dynamic-provider");
|
||||
|
||||
// 获取千问3 VL 模型
|
||||
// 图片识别固定走千问多模态；纯文本理解优先走 Agent 模型（建议智普）
|
||||
const allModels = await getEnabledAiModelsWithProvider();
|
||||
const agentModel = await getAgentAiModelWithProvider();
|
||||
const qwenModel = allModels.find(
|
||||
(m) => m.provider.type === "alibaba" && m.model.supportsVision
|
||||
const isReasoningCompatibleProvider = (providerType: string) =>
|
||||
["zhipu", "alibaba", "openai", "custom", "local"].includes(providerType);
|
||||
const qwenVisionModel = allModels.find(
|
||||
(m) =>
|
||||
m.provider.type === "alibaba" &&
|
||||
m.model.supportsVision &&
|
||||
/(qwen|通义|vl)/i.test(`${m.model.displayName} ${m.model.modelId}`)
|
||||
);
|
||||
const selectedAnalyzerModel =
|
||||
(agentModel && (!hasImage || agentModel.model.supportsVision)
|
||||
const fallbackVisionModel =
|
||||
qwenVisionModel ||
|
||||
allModels.find((m) => m.provider.type === "alibaba" && m.model.supportsVision);
|
||||
const usableAgentModel =
|
||||
agentModel && isReasoningCompatibleProvider(agentModel.provider.type)
|
||||
? agentModel
|
||||
: undefined) || qwenModel;
|
||||
: undefined;
|
||||
const selectedAnalyzerModel = hasImage
|
||||
? fallbackVisionModel
|
||||
: usableAgentModel ||
|
||||
allModels.find((m) => m.provider.type === "zhipu" && !m.model.isSubModel) ||
|
||||
fallbackVisionModel;
|
||||
|
||||
if (!selectedAnalyzerModel) {
|
||||
console.log("[Smart Analyzer] No Qwen3 VL model found");
|
||||
console.log("[Smart Analyzer] No available analyzer model found");
|
||||
return null;
|
||||
}
|
||||
|
||||
@ -261,7 +274,13 @@ ${historyText}
|
||||
? `${contextMessage}用户上传了图片，请求内容："${text}"`
|
||||
: `${contextMessage}用户请求（无图片）："${text}"`;
|
||||
|
||||
console.log("[Smart Analyzer] Calling Qwen3 VL for analysis with", historyMessages?.length || 0, "history messages");
|
||||
console.log(
|
||||
"[Smart Analyzer] Calling analyzer model:",
|
||||
selectedAnalyzerModel.model.displayName,
|
||||
`(${selectedAnalyzerModel.provider.type})`,
|
||||
"history=",
|
||||
historyMessages?.length || 0
|
||||
);
|
||||
|
||||
const visionImages = (imageUrls || [])
|
||||
.filter((url) => /^https?:\/\//i.test(url))
|
||||
|
||||
@ -10,6 +10,10 @@ function toRecord(value: unknown): Record<string, unknown> | null {
|
||||
export function isAiModelAgentAuto(input: {
|
||||
defaultParams?: unknown;
|
||||
capabilities?: unknown;
|
||||
modelId?: string | null;
|
||||
displayName?: string | null;
|
||||
supportsVision?: boolean | null;
|
||||
supportsImageGen?: boolean | null;
|
||||
}): boolean {
|
||||
const defaultParams = toRecord(input.defaultParams);
|
||||
if (typeof defaultParams?.[AGENT_AUTO_MODEL_KEY] === "boolean") {
|
||||
@ -21,6 +25,24 @@ export function isAiModelAgentAuto(input: {
|
||||
return capabilities[AGENT_AUTO_MODEL_KEY] as boolean;
|
||||
}
|
||||
|
||||
const supportsVision = Boolean(input.supportsVision);
|
||||
const supportsImageGen = Boolean(input.supportsImageGen);
|
||||
if (!supportsVision || supportsImageGen) {
|
||||
return false;
|
||||
}
|
||||
|
||||
const modelText = `${input.modelId || ""} ${input.displayName || ""}`.toLowerCase();
|
||||
const isQwenLikeModel =
|
||||
modelText.includes("qwen") || modelText.includes("通义");
|
||||
const isVisionLikeModel =
|
||||
modelText.includes("vl") ||
|
||||
modelText.includes("vision") ||
|
||||
modelText.includes("多模态");
|
||||
|
||||
if (isQwenLikeModel && isVisionLikeModel) {
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user