refactor: modularize chat route helpers
This commit is contained in:
Your Name
parent
33bfcadf97
commit
f2d4f9e698
106
app/(chat)/api/chat/_lib/agent-output.ts
Normal file
106
app/(chat)/api/chat/_lib/agent-output.ts
Normal file
@ -0,0 +1,106 @@
|
||||
import type { PostRequestBody } from "../schema";
|
||||
|
||||
export function sanitizeAgentOutputSettingsSection(params: {
|
||||
message?: string;
|
||||
userImageParams?: PostRequestBody["imageParams"];
|
||||
hasUserCustomImageParams: boolean;
|
||||
}): string | undefined {
|
||||
const { message, userImageParams, hasUserCustomImageParams } = params;
|
||||
if (!message?.trim()) return message;
|
||||
|
||||
const outputSectionRegex =
|
||||
/(?:###\s*输出设置|【\s*输出设置\s*】)[\s\S]*?(?=\n(?:###\s*|【\s*[^】]+\s*】)|$)/;
|
||||
if (!outputSectionRegex.test(message)) return message;
|
||||
|
||||
if (!hasUserCustomImageParams) {
|
||||
return message.replace(
|
||||
outputSectionRegex,
|
||||
"【输出设置】\n- 参数：按用户面板当前设置执行（未自动改写）"
|
||||
);
|
||||
}
|
||||
|
||||
const lines: string[] = [];
|
||||
const aspectRatio =
|
||||
typeof userImageParams?.aspectRatio === "string" ? userImageParams.aspectRatio.trim() : "";
|
||||
const imageSize =
|
||||
typeof userImageParams?.imageSize === "string" ? userImageParams.imageSize.trim() : "";
|
||||
const imageCount =
|
||||
typeof userImageParams?.imageCount === "number" ? userImageParams.imageCount : undefined;
|
||||
|
||||
if (aspectRatio && aspectRatio !== "auto") {
|
||||
lines.push(`- 宽高比：${aspectRatio}（来自用户面板设置）`);
|
||||
}
|
||||
if (imageSize && imageSize.toUpperCase() !== "2K") {
|
||||
lines.push(`- 清晰度档位：${imageSize.toUpperCase()}（来自用户面板设置）`);
|
||||
}
|
||||
if (typeof imageCount === "number" && Number.isFinite(imageCount) && imageCount !== 1) {
|
||||
lines.push(`- 出图数量：${Math.max(1, Math.round(imageCount))} 张（来自用户面板设置）`);
|
||||
}
|
||||
lines.push("- 其余参数：保持用户面板当前设置");
|
||||
|
||||
return message.replace(outputSectionRegex, `【输出设置】\n${lines.join("\n")}`);
|
||||
}
|
||||
|
||||
export function normalizeAgentReportHeadingLine(line: string): string {
|
||||
const trimmed = line.trim();
|
||||
if (!trimmed) return line;
|
||||
|
||||
const markdownHeading = trimmed.match(/^#{1,6}\s*(.+?)\s*$/);
|
||||
const plainTitle = markdownHeading?.[1]?.trim() || trimmed;
|
||||
const normalizedTitle = plainTitle
|
||||
.replace(/^[^\u4e00-\u9fa5A-Za-z0-9]+/, "")
|
||||
.trim();
|
||||
|
||||
const knownTitles = [
|
||||
"需求理解",
|
||||
"图片分析",
|
||||
"执行计划",
|
||||
"优化后提示词",
|
||||
"输出设置",
|
||||
"脚本规划",
|
||||
"分镜规划",
|
||||
"执行说明",
|
||||
"视频生成完成",
|
||||
];
|
||||
|
||||
if (knownTitles.some((title) => normalizedTitle === title)) {
|
||||
return `【${normalizedTitle}】`;
|
||||
}
|
||||
|
||||
if (markdownHeading) {
|
||||
return `【${normalizedTitle || plainTitle}】`;
|
||||
}
|
||||
return line;
|
||||
}
|
||||
|
||||
export function stripAgentFriendlyMediaEmbeds(message: string): string {
|
||||
if (!message) return message;
|
||||
|
||||
const withoutInlineMedia = message
|
||||
.replace(/!\[[^\]]*]\((https?:\/\/[^\s)]+)\)/gi, "")
|
||||
.replace(/<video[\s\S]*?<\/video>/gi, "")
|
||||
.replace(/<img[^>]*>/gi, "");
|
||||
|
||||
const cleanedLines = withoutInlineMedia
|
||||
.split("\n")
|
||||
.filter((line) => {
|
||||
const trimmed = line.trim();
|
||||
if (!trimmed) return true;
|
||||
if (/^https?:\/\/\S+$/i.test(trimmed)) return false;
|
||||
if (/^\[[^\]]*]\(https?:\/\/[^\s)]+\)$/i.test(trimmed)) return false;
|
||||
return true;
|
||||
})
|
||||
.join("\n");
|
||||
|
||||
return cleanedLines.replace(/\n{3,}/g, "\n\n").trim();
|
||||
}
|
||||
|
||||
export function normalizeAgentReportForDisplay(message?: string): string | undefined {
|
||||
if (!message?.trim()) return message;
|
||||
const stripped = stripAgentFriendlyMediaEmbeds(message);
|
||||
if (!stripped) return undefined;
|
||||
return stripped
|
||||
.split("\n")
|
||||
.map((line) => normalizeAgentReportHeadingLine(line))
|
||||
.join("\n");
|
||||
}
|
||||
540
app/(chat)/api/chat/_lib/agent-scene-workflow.ts
Normal file
540
app/(chat)/api/chat/_lib/agent-scene-workflow.ts
Normal file
@ -0,0 +1,540 @@
|
||||
export type AgentPresetMessageImagePart = {
|
||||
type: "file";
|
||||
url: string;
|
||||
name: string;
|
||||
mediaType: string;
|
||||
assetId?: string;
|
||||
};
|
||||
|
||||
export function extractCurrentMessageImageParts(
|
||||
parts: Array<Record<string, any>>,
|
||||
): AgentPresetMessageImagePart[] {
|
||||
return parts
|
||||
.filter(
|
||||
(part): part is AgentPresetMessageImagePart =>
|
||||
part?.type === "file" &&
|
||||
typeof part?.url === "string" &&
|
||||
/^https?:\/\//i.test(part.url) &&
|
||||
typeof part?.mediaType === "string" &&
|
||||
part.mediaType.startsWith("image/")
|
||||
)
|
||||
.map((part) => ({
|
||||
type: "file",
|
||||
url: String(part.url).trim(),
|
||||
name: String(part.name || "图片").trim() || "图片",
|
||||
mediaType: String(part.mediaType || "image/jpeg").trim() || "image/jpeg",
|
||||
assetId:
|
||||
typeof part.assetId === "string" && part.assetId.trim() ? part.assetId.trim() : undefined,
|
||||
}));
|
||||
}
|
||||
|
||||
export function getSceneUploadedImageBreakdown(params: {
|
||||
uploadedImageParts: AgentPresetMessageImagePart[];
|
||||
fallbackPrimaryCount: number;
|
||||
fallbackIncludeModelImage: boolean;
|
||||
}) {
|
||||
const normalizedNames = params.uploadedImageParts.map((part) => String(part.name || "").trim());
|
||||
const primaryCountFromNames = normalizedNames.filter((name) => name.startsWith("主SKU参考图")).length;
|
||||
const secondaryCountFromNames = normalizedNames.filter((name) => name.startsWith("搭配SKU参考图")).length;
|
||||
const hasModelFromNames = normalizedNames.some((name) => name.startsWith("品牌模特图"));
|
||||
const hasNamedBreakdown =
|
||||
primaryCountFromNames > 0 || secondaryCountFromNames > 0 || hasModelFromNames;
|
||||
|
||||
if (hasNamedBreakdown) {
|
||||
return {
|
||||
primaryProductImageCount: primaryCountFromNames,
|
||||
bundleProductImageCount: secondaryCountFromNames,
|
||||
includeModelImage: hasModelFromNames,
|
||||
};
|
||||
}
|
||||
|
||||
const uploadedImageCount = params.uploadedImageParts.length;
|
||||
const includeModelImage =
|
||||
params.fallbackIncludeModelImage && uploadedImageCount > params.fallbackPrimaryCount;
|
||||
const bundleProductImageCount = Math.max(
|
||||
0,
|
||||
uploadedImageCount - params.fallbackPrimaryCount - (includeModelImage ? 1 : 0)
|
||||
);
|
||||
const primaryProductImageCount = Math.max(
|
||||
0,
|
||||
Math.min(
|
||||
params.fallbackPrimaryCount,
|
||||
uploadedImageCount - bundleProductImageCount - (includeModelImage ? 1 : 0)
|
||||
)
|
||||
);
|
||||
|
||||
return {
|
||||
primaryProductImageCount,
|
||||
bundleProductImageCount,
|
||||
includeModelImage,
|
||||
};
|
||||
}
|
||||
|
||||
export function resolveScenePromptOptimizationImageUrls(params: {
|
||||
uploadedImageParts: AgentPresetMessageImagePart[];
|
||||
primaryProductImageCount: number;
|
||||
bundleProductImageCount: number;
|
||||
includeModelImage: boolean;
|
||||
}) {
|
||||
const namedPrimaryParts = params.uploadedImageParts.filter((part) =>
|
||||
String(part.name || "").trim().startsWith("主SKU参考图")
|
||||
);
|
||||
const namedModelPart = params.uploadedImageParts.find((part) =>
|
||||
String(part.name || "").trim().startsWith("品牌模特图")
|
||||
);
|
||||
|
||||
if (namedPrimaryParts.length > 0 || namedModelPart) {
|
||||
return {
|
||||
primaryImageUrl: namedPrimaryParts[0]?.url || "",
|
||||
modelImageUrl: namedModelPart?.url || null,
|
||||
};
|
||||
}
|
||||
|
||||
const primaryImageUrl =
|
||||
params.uploadedImageParts.slice(0, Math.max(0, params.primaryProductImageCount))[0]?.url || "";
|
||||
const modelIndex = params.primaryProductImageCount + params.bundleProductImageCount;
|
||||
|
||||
return {
|
||||
primaryImageUrl,
|
||||
modelImageUrl:
|
||||
params.includeModelImage && modelIndex < params.uploadedImageParts.length
|
||||
? params.uploadedImageParts[modelIndex]?.url || null
|
||||
: null,
|
||||
};
|
||||
}
|
||||
|
||||
export function normalizeAgentSupplementPrompt(text: string, presetName?: string | null) {
|
||||
const normalized = String(text || "").trim();
|
||||
if (!normalized) return "";
|
||||
const escapedPresetName = String(presetName || "").replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
||||
const genericPattern = escapedPresetName
|
||||
? new RegExp(`^请按「${escapedPresetName}」风格化配置顺序处理这\\d+张图片。?$`)
|
||||
: /^请按「.+」风格化配置顺序处理这\d+张图片。?$/;
|
||||
if (genericPattern.test(normalized)) {
|
||||
return "";
|
||||
}
|
||||
return normalized;
|
||||
}
|
||||
|
||||
export function normalizeAgentWorkflowPrompt(text: string) {
|
||||
return String(text || "")
|
||||
.split(/\n+/)
|
||||
.map((line) => line.replace(/\s+/g, " ").trim())
|
||||
.filter(Boolean)
|
||||
.join("\n")
|
||||
.trim();
|
||||
}
|
||||
|
||||
export function parseAgentWorkflowPromptSections(text: string) {
|
||||
const sectionMap = new Map<string, string[]>();
|
||||
let activeSection = "__root__";
|
||||
const normalizedPrompt = normalizeAgentWorkflowPrompt(text);
|
||||
if (!normalizedPrompt) {
|
||||
return sectionMap;
|
||||
}
|
||||
|
||||
normalizedPrompt.split("\n").forEach((line) => {
|
||||
const trimmedLine = line.trim();
|
||||
if (!trimmedLine) {
|
||||
return;
|
||||
}
|
||||
const headerMatch = trimmedLine.match(/^【([^】]{1,30})】$/);
|
||||
if (headerMatch) {
|
||||
activeSection = headerMatch[1].trim();
|
||||
if (!sectionMap.has(activeSection)) {
|
||||
sectionMap.set(activeSection, []);
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
const currentLines = sectionMap.get(activeSection) || [];
|
||||
currentLines.push(trimmedLine);
|
||||
sectionMap.set(activeSection, currentLines);
|
||||
});
|
||||
|
||||
return sectionMap;
|
||||
}
|
||||
|
||||
export function buildAgentPromptSectionText(
|
||||
title: string,
|
||||
lines: string[],
|
||||
options?: { bullet?: boolean }
|
||||
) {
|
||||
const normalizedLines = lines
|
||||
.map((line) => String(line || "").trim())
|
||||
.filter(Boolean);
|
||||
if (normalizedLines.length === 0) {
|
||||
return "";
|
||||
}
|
||||
|
||||
const sectionBody = options?.bullet
|
||||
? normalizedLines.map((line) => (line.startsWith("- ") ? line : `- ${line}`)).join("\n")
|
||||
: normalizedLines.join("\n");
|
||||
|
||||
return `【${title}】\n${sectionBody}`;
|
||||
}
|
||||
|
||||
export function buildOptimizedAgentSceneWorkflowPrompt(params: {
|
||||
workflowPrompt: string;
|
||||
optimizationResult?: {
|
||||
sceneGoal: string;
|
||||
environmentSummary: string;
|
||||
propSummary: string;
|
||||
cameraLightingSummary: string;
|
||||
characterSummary: string;
|
||||
negativeConstraints: string[];
|
||||
conflicts: string[];
|
||||
} | null;
|
||||
}) {
|
||||
const normalizedWorkflowPrompt = normalizeAgentWorkflowPrompt(params.workflowPrompt);
|
||||
if (!normalizedWorkflowPrompt) {
|
||||
return "";
|
||||
}
|
||||
|
||||
const optimizationResult = params.optimizationResult || null;
|
||||
if (!optimizationResult) {
|
||||
return normalizedWorkflowPrompt;
|
||||
}
|
||||
|
||||
const sectionMap = parseAgentWorkflowPromptSections(normalizedWorkflowPrompt);
|
||||
const sceneGoalLines = [optimizationResult.sceneGoal, optimizationResult.environmentSummary].filter(
|
||||
Boolean
|
||||
);
|
||||
const propLines = [optimizationResult.propSummary].filter(Boolean);
|
||||
const cameraLightingLines = [optimizationResult.cameraLightingSummary].filter(Boolean);
|
||||
const characterLines = [optimizationResult.characterSummary].filter(Boolean);
|
||||
const negativeLines = Array.from(
|
||||
new Set([
|
||||
...optimizationResult.negativeConstraints,
|
||||
...optimizationResult.conflicts.map((item) => `冲突校正：${item}`),
|
||||
])
|
||||
);
|
||||
|
||||
const optimizedSections = [
|
||||
buildAgentPromptSectionText("项目背景", sectionMap.get("项目背景") || sectionMap.get("__root__") || []),
|
||||
buildAgentPromptSectionText("主SKU锁定", sectionMap.get("主SKU锁定") || []),
|
||||
buildAgentPromptSectionText("搭配SKU锁定", sectionMap.get("搭配SKU锁定") || []),
|
||||
buildAgentPromptSectionText("参考图", sectionMap.get("参考图") || []),
|
||||
buildAgentPromptSectionText("场景创作总纲", sceneGoalLines),
|
||||
buildAgentPromptSectionText("陈列与道具", propLines),
|
||||
buildAgentPromptSectionText("机位与光线", cameraLightingLines),
|
||||
buildAgentPromptSectionText("人物规则", characterLines),
|
||||
buildAgentPromptSectionText("负面约束", negativeLines, { bullet: true }),
|
||||
].filter(Boolean);
|
||||
|
||||
return optimizedSections.join("\n\n").trim();
|
||||
}
|
||||
|
||||
const AGENT_PROMPT_STRUCTURAL_LABELS = new Set([
|
||||
"角色",
|
||||
"技能",
|
||||
"目标",
|
||||
"约束条件",
|
||||
"流程",
|
||||
"输出",
|
||||
]);
|
||||
|
||||
const AGENT_PROMPT_LABEL_SEMANTIC_ALIASES = new Map<string, string>([
|
||||
["角色", "role"],
|
||||
["技能", "skills"],
|
||||
["目标", "goal"],
|
||||
["约束条件", "constraints"],
|
||||
["流程", "process"],
|
||||
["输出", "output"],
|
||||
["人群选择", "audience"],
|
||||
["人物", "audience"],
|
||||
["主题风格", "style"],
|
||||
["整体风格", "style"],
|
||||
["空间选择", "space"],
|
||||
["空间布局", "space"],
|
||||
["主空间", "space"],
|
||||
["子空间选择", "subspace"],
|
||||
["子空间", "subspace"],
|
||||
["硬装方案", "hardDecoration"],
|
||||
["硬装", "hardDecoration"],
|
||||
["软装方案", "softDecoration"],
|
||||
["软装", "softDecoration"],
|
||||
["光氛选择", "lighting"],
|
||||
["光源布置", "lighting"],
|
||||
["灯具", "lighting"],
|
||||
["机位选择", "camera"],
|
||||
["拍摄角度", "camera"],
|
||||
["配饰陈列", "accessories"],
|
||||
["放在产品内的小件配饰", "accessories"],
|
||||
["前景", "sceneLayout"],
|
||||
["中景", "sceneLayout"],
|
||||
["背景", "sceneLayout"],
|
||||
]);
|
||||
|
||||
const AGENT_SCENE_WORKFLOW_OWNED_SEMANTICS = new Set([
|
||||
"role",
|
||||
"skills",
|
||||
"goal",
|
||||
"constraints",
|
||||
"process",
|
||||
"output",
|
||||
"audience",
|
||||
"style",
|
||||
"space",
|
||||
"subspace",
|
||||
"hardDecoration",
|
||||
"softDecoration",
|
||||
"lighting",
|
||||
"camera",
|
||||
"accessories",
|
||||
"sceneLayout",
|
||||
]);
|
||||
|
||||
export function normalizePromptLineForCompare(line: string) {
|
||||
return String(line || "")
|
||||
.replace(/\s+/g, " ")
|
||||
.replace(/[：:]/g, ":")
|
||||
.replace(/[。；;，,、!！?？]+$/g, "")
|
||||
.trim()
|
||||
.toLowerCase();
|
||||
}
|
||||
|
||||
export function parsePromptLineLabel(line: string) {
|
||||
const normalized = String(line || "").replace(/\s+/g, " ").trim();
|
||||
if (!normalized) return { label: "", value: "" };
|
||||
|
||||
const headerOnlyMatch = normalized.match(/^([^:：]{1,20})[:：]\s*$/);
|
||||
if (headerOnlyMatch) {
|
||||
return {
|
||||
label: headerOnlyMatch[1].trim(),
|
||||
value: "",
|
||||
};
|
||||
}
|
||||
|
||||
const colonMatch = normalized.match(/^([^:：]{1,20})[:：]\s*(.+)$/);
|
||||
if (colonMatch) {
|
||||
return {
|
||||
label: colonMatch[1].trim(),
|
||||
value: colonMatch[2].trim(),
|
||||
};
|
||||
}
|
||||
|
||||
const whitespaceMatch = normalized.match(/^([^\s]{1,20})\s+(.+)$/);
|
||||
if (whitespaceMatch) {
|
||||
return {
|
||||
label: whitespaceMatch[1].trim(),
|
||||
value: whitespaceMatch[2].trim(),
|
||||
};
|
||||
}
|
||||
|
||||
return { label: "", value: normalized };
|
||||
}
|
||||
|
||||
export function normalizePromptLabel(label: string) {
|
||||
return String(label || "")
|
||||
.replace(/\s+/g, "")
|
||||
.replace(/[：:]/g, "")
|
||||
.trim();
|
||||
}
|
||||
|
||||
export function getPromptSemanticLabel(label: string) {
|
||||
const normalized = normalizePromptLabel(label);
|
||||
return AGENT_PROMPT_LABEL_SEMANTIC_ALIASES.get(normalized) || normalized;
|
||||
}
|
||||
|
||||
export function collectPromptSemanticLabelsFromText(text: string) {
|
||||
const semanticLabels = new Set<string>();
|
||||
const normalizedText = String(text || "").replace(/\s+/g, " ").trim();
|
||||
if (!normalizedText) {
|
||||
return semanticLabels;
|
||||
}
|
||||
|
||||
for (const [label, semantic] of AGENT_PROMPT_LABEL_SEMANTIC_ALIASES.entries()) {
|
||||
if (!label) {
|
||||
continue;
|
||||
}
|
||||
if (normalizedText.includes(`${label}：`) || normalizedText.includes(`${label}:`)) {
|
||||
semanticLabels.add(semantic);
|
||||
continue;
|
||||
}
|
||||
if (normalizedText.includes(`${label} `) || normalizedText.includes(label)) {
|
||||
semanticLabels.add(semantic);
|
||||
}
|
||||
}
|
||||
|
||||
return semanticLabels;
|
||||
}
|
||||
|
||||
export function shouldDropSceneSlotSemanticLine(
|
||||
slotSemanticLabel: string,
|
||||
workflowSemanticLabels: Set<string>,
|
||||
) {
|
||||
if (!slotSemanticLabel || !AGENT_SCENE_WORKFLOW_OWNED_SEMANTICS.has(slotSemanticLabel)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
if (slotSemanticLabel === "sceneLayout") {
|
||||
return [
|
||||
"style",
|
||||
"space",
|
||||
"subspace",
|
||||
"hardDecoration",
|
||||
"softDecoration",
|
||||
"accessories",
|
||||
"camera",
|
||||
"lighting",
|
||||
].some((semantic) => workflowSemanticLabels.has(semantic));
|
||||
}
|
||||
|
||||
return workflowSemanticLabels.has(slotSemanticLabel);
|
||||
}
|
||||
|
||||
export function dedupeAgentSlotPromptAgainstWorkflow(params: {
|
||||
workflowPrompt?: string | null;
|
||||
slotPrompt?: string | null;
|
||||
modeType?: "scene_generation" | "detail_replacement" | "sku_replacement" | "generic";
|
||||
}) {
|
||||
const dedupedSlotLines: string[] = [];
|
||||
const seenExactLines = new Set<string>();
|
||||
const workflowStructuralValues = new Map<string, string>();
|
||||
const workflowSemanticLabels = new Set<string>();
|
||||
|
||||
const rememberLine = (line: string) => {
|
||||
const normalizedLine = normalizePromptLineForCompare(line);
|
||||
if (!normalizedLine) {
|
||||
return;
|
||||
}
|
||||
seenExactLines.add(normalizedLine);
|
||||
};
|
||||
|
||||
const normalizedWorkflowLines = normalizeAgentWorkflowPrompt(params.workflowPrompt || "")
|
||||
.split("\n")
|
||||
.map((line) => line.trim())
|
||||
.filter(Boolean);
|
||||
normalizedWorkflowLines.forEach((line) => {
|
||||
const parsed = parsePromptLineLabel(line);
|
||||
if (parsed.label) {
|
||||
workflowSemanticLabels.add(getPromptSemanticLabel(parsed.label));
|
||||
}
|
||||
collectPromptSemanticLabelsFromText(line).forEach((semantic) => {
|
||||
workflowSemanticLabels.add(semantic);
|
||||
});
|
||||
collectPromptSemanticLabelsFromText(parsed.value || "").forEach((semantic) => {
|
||||
workflowSemanticLabels.add(semantic);
|
||||
});
|
||||
if (parsed.label && AGENT_PROMPT_STRUCTURAL_LABELS.has(parsed.label)) {
|
||||
workflowStructuralValues.set(
|
||||
parsed.label,
|
||||
normalizePromptLineForCompare(parsed.value || "")
|
||||
);
|
||||
}
|
||||
rememberLine(line);
|
||||
});
|
||||
|
||||
const normalizedSlotLines = String(params.slotPrompt || "")
|
||||
.split(/\n+/)
|
||||
.map((line) => line.replace(/\s+/g, " ").trim())
|
||||
.filter(Boolean);
|
||||
let activeSlotSemanticLabel = "";
|
||||
normalizedSlotLines.forEach((line) => {
|
||||
const normalizedLine = normalizePromptLineForCompare(line);
|
||||
if (!normalizedLine || seenExactLines.has(normalizedLine)) {
|
||||
return;
|
||||
}
|
||||
const parsed = parsePromptLineLabel(line);
|
||||
const slotSemanticLabel = parsed.label ? getPromptSemanticLabel(parsed.label) : "";
|
||||
if (slotSemanticLabel) {
|
||||
activeSlotSemanticLabel = slotSemanticLabel;
|
||||
}
|
||||
if (
|
||||
params.modeType === "scene_generation" &&
|
||||
shouldDropSceneSlotSemanticLine(
|
||||
slotSemanticLabel || activeSlotSemanticLabel,
|
||||
workflowSemanticLabels,
|
||||
)
|
||||
) {
|
||||
return;
|
||||
}
|
||||
if (parsed.label && AGENT_PROMPT_STRUCTURAL_LABELS.has(parsed.label)) {
|
||||
const normalizedValue = normalizePromptLineForCompare(parsed.value || "");
|
||||
const workflowValue = workflowStructuralValues.get(parsed.label);
|
||||
if (workflowValue && workflowValue === normalizedValue) {
|
||||
return;
|
||||
}
|
||||
}
|
||||
seenExactLines.add(normalizedLine);
|
||||
dedupedSlotLines.push(line);
|
||||
});
|
||||
|
||||
return dedupedSlotLines.join("\n").trim();
|
||||
}
|
||||
|
||||
export function buildAgentSlotPrompt(params: {
|
||||
workflowPrompt?: string | null;
|
||||
slotPrompt: string;
|
||||
slotTitle?: string | null;
|
||||
userSupplementPrompt?: string | null;
|
||||
includeSlotTitle?: boolean;
|
||||
modeType?: "scene_generation" | "detail_replacement" | "sku_replacement" | "generic";
|
||||
}) {
|
||||
const lines: string[] = [];
|
||||
const workflowPrompt = normalizeAgentWorkflowPrompt(params.workflowPrompt || "");
|
||||
if (workflowPrompt) {
|
||||
lines.push(workflowPrompt);
|
||||
}
|
||||
const slotTitle = String(params.slotTitle || "").trim();
|
||||
if (params.includeSlotTitle !== false && slotTitle) {
|
||||
lines.push(`【槽位】${slotTitle}`);
|
||||
}
|
||||
const dedupedSlotPrompt = dedupeAgentSlotPromptAgainstWorkflow({
|
||||
workflowPrompt,
|
||||
slotPrompt: params.slotPrompt,
|
||||
modeType: params.modeType || "generic",
|
||||
});
|
||||
if (dedupedSlotPrompt) {
|
||||
lines.push(dedupedSlotPrompt);
|
||||
}
|
||||
const supplement = String(params.userSupplementPrompt || "").trim();
|
||||
if (supplement) {
|
||||
lines.push(`【补充要求】${supplement}`);
|
||||
}
|
||||
return lines.join("\n\n").trim();
|
||||
}
|
||||
|
||||
export function buildAgentSceneWorkflowPrompt(params: {
|
||||
workflowPrompt?: string | null;
|
||||
slotPrompt?: string | null;
|
||||
userSupplementPrompt?: string | null;
|
||||
imageRolePrompt?: string | null;
|
||||
}) {
|
||||
const lines: string[] = [];
|
||||
const workflowPrompt = normalizeAgentWorkflowPrompt(params.workflowPrompt || "");
|
||||
if (workflowPrompt) {
|
||||
lines.push(workflowPrompt);
|
||||
}
|
||||
const slotPrompt = String(params.slotPrompt || "")
|
||||
.split(/\n+/)
|
||||
.map((line) => line.replace(/\s+/g, " ").trim())
|
||||
.filter(Boolean)
|
||||
.join("\n")
|
||||
.trim();
|
||||
if (slotPrompt) {
|
||||
lines.push(slotPrompt);
|
||||
}
|
||||
let imageRolePrompt = String(params.imageRolePrompt || "").trim();
|
||||
if (workflowPrompt.includes("品牌模特图：已选") || workflowPrompt.includes("固定模特图")) {
|
||||
imageRolePrompt = imageRolePrompt
|
||||
.replace(
|
||||
/^如当前请求未附带固定模特图，仅在工作流明确要求人物时再安排辅助人物，且人物不能喧宾夺主。$/m,
|
||||
""
|
||||
)
|
||||
.replace(/^当前没有固定模特图，不要凭空生成强人物主体。$/m, "")
|
||||
.split(/\n+/)
|
||||
.map((line) => line.trim())
|
||||
.filter(Boolean)
|
||||
.join("\n");
|
||||
}
|
||||
if (imageRolePrompt) {
|
||||
lines.push(imageRolePrompt);
|
||||
}
|
||||
const supplement = String(params.userSupplementPrompt || "").trim();
|
||||
if (supplement) {
|
||||
lines.push(`【补充要求】${supplement}`);
|
||||
}
|
||||
return lines.join("\n\n").trim();
|
||||
}
|
||||
107
app/(chat)/api/chat/_lib/billing.ts
Normal file
107
app/(chat)/api/chat/_lib/billing.ts
Normal file
@ -0,0 +1,107 @@
|
||||
import { getModelGenerationCreditCost } from "@/lib/ai/image-model-pricing";
|
||||
import {
|
||||
consumeUserPoints,
|
||||
getAiModelById,
|
||||
getEnabledAiModelsWithProvider,
|
||||
getPointsSystemConfig,
|
||||
getUserById,
|
||||
} from "@/lib/db/queries";
|
||||
|
||||
const MODEL_UUID_REGEX =
|
||||
/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
|
||||
|
||||
export function getModelCreditCost(
|
||||
modelConfig: any,
|
||||
target: "image" | "video",
|
||||
options?: {
|
||||
duration?: number | null;
|
||||
imageSize?: string | null;
|
||||
resolution?: string | null;
|
||||
},
|
||||
): number {
|
||||
return getModelGenerationCreditCost(modelConfig, target, options);
|
||||
}
|
||||
|
||||
export async function resolveBillingModelByReference(modelRef: string): Promise<any | undefined> {
|
||||
if (!modelRef) return undefined;
|
||||
|
||||
if (MODEL_UUID_REGEX.test(modelRef)) {
|
||||
return await getAiModelById({ id: modelRef }).catch(() => undefined);
|
||||
}
|
||||
|
||||
const allModels = await getEnabledAiModelsWithProvider(true, true).catch(() => []);
|
||||
const matched = allModels.find(
|
||||
({ model }: any) => model.id === modelRef || model.modelId === modelRef,
|
||||
);
|
||||
return matched?.model;
|
||||
}
|
||||
|
||||
export async function consumeGeneratedVideoPoints(params: {
|
||||
userId: string;
|
||||
chatId: string;
|
||||
messageId: string;
|
||||
modelRefs: Array<string | null | undefined>;
|
||||
duration?: number | null;
|
||||
resolution?: string | null;
|
||||
}) {
|
||||
try {
|
||||
const pointsConfig = await getPointsSystemConfig();
|
||||
if (!pointsConfig.enabled) return;
|
||||
|
||||
const userData = await getUserById({ id: params.userId }).catch(() => undefined);
|
||||
if (userData?.isHighVip) {
|
||||
console.log(`[Points] Skip video points for VIP user=${params.userId}`);
|
||||
return;
|
||||
}
|
||||
|
||||
let billingModel: any | undefined;
|
||||
for (const ref of params.modelRefs) {
|
||||
if (!ref) continue;
|
||||
const resolved = await resolveBillingModelByReference(ref);
|
||||
if (!resolved) continue;
|
||||
billingModel = resolved;
|
||||
if (
|
||||
getModelCreditCost(resolved, "video", {
|
||||
duration: params.duration,
|
||||
resolution: params.resolution,
|
||||
}) > 0
|
||||
) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (!billingModel) return;
|
||||
|
||||
const unitCost = getModelCreditCost(billingModel, "video", {
|
||||
duration: params.duration,
|
||||
resolution: params.resolution,
|
||||
});
|
||||
if (unitCost <= 0) return;
|
||||
|
||||
const consumeResult = await consumeUserPoints({
|
||||
userId: params.userId,
|
||||
amount: unitCost,
|
||||
reason: "video_generation",
|
||||
modelId: billingModel.id,
|
||||
chatId: params.chatId,
|
||||
messageId: params.messageId,
|
||||
note: `模型(${billingModel.displayName || billingModel.modelId || billingModel.id}) 生视频 1 条${params.duration ? `（${params.duration} 秒）` : ""}${params.resolution ? `（${params.resolution}）` : ""}`,
|
||||
});
|
||||
|
||||
if (!consumeResult.ok) {
|
||||
console.warn(
|
||||
`[Points] Failed to consume video points. user=${params.userId} model=${billingModel.id} cost=${unitCost}`,
|
||||
);
|
||||
} else if (consumeResult.skipped) {
|
||||
console.log(
|
||||
`[Points] Skipped ${unitCost} video points for VIP user=${params.userId} model=${billingModel.id}`,
|
||||
);
|
||||
} else {
|
||||
console.log(
|
||||
`[Points] Consumed ${unitCost} points for video. balance=${consumeResult.balance}`,
|
||||
);
|
||||
}
|
||||
} catch (error) {
|
||||
console.error("[Points] Failed to consume points after video generation:", error);
|
||||
}
|
||||
}
|
||||
1150
app/(chat)/api/chat/_lib/detail-replacement.ts
Normal file
1150
app/(chat)/api/chat/_lib/detail-replacement.ts
Normal file
File diff suppressed because it is too large
Load Diff
259
app/(chat)/api/chat/_lib/image-failover.ts
Normal file
259
app/(chat)/api/chat/_lib/image-failover.ts
Normal file
@ -0,0 +1,259 @@
|
||||
import { isFalGptImage2EditModelId } from "@/lib/ai/providers/fal-image";
|
||||
import { getLanguageModelById } from "@/lib/ai/dynamic-provider";
|
||||
import type { ChatMessage } from "@/lib/types";
|
||||
import { convertMessagesWithImages } from "./image-media";
|
||||
|
||||
export const IMAGE_FAILOVER_PROVIDER_ALLOWLIST = new Set([
|
||||
"alibaba-image",
|
||||
"bfl-image",
|
||||
"ruxa-image",
|
||||
"fal-image",
|
||||
"openai-image",
|
||||
"stepfun-image",
|
||||
"runway",
|
||||
"krea",
|
||||
"grsai-gpt-image",
|
||||
"volcano-vision",
|
||||
"gemini",
|
||||
"huanke-gemini",
|
||||
]);
|
||||
|
||||
const IMAGE_FAILOVER_MODEL_COOLDOWN_MS = 30 * 60 * 1000;
|
||||
const imageFailoverModelCooldown = new Map<string, { until: number; reason: string }>();
|
||||
|
||||
export function normalizeFailoverErrorMessage(error: unknown): string {
|
||||
if (!error) return "";
|
||||
if (typeof error === "string") return error;
|
||||
if (error instanceof Error) return error.message || String(error);
|
||||
try {
|
||||
return JSON.stringify(error);
|
||||
} catch {
|
||||
return String(error);
|
||||
}
|
||||
}
|
||||
|
||||
export function shouldCooldownImageFallbackModel(errorMessage: string): boolean {
|
||||
const msg = errorMessage.toLowerCase();
|
||||
return (
|
||||
msg.includes("model not found") ||
|
||||
msg.includes("model_not_found") ||
|
||||
msg.includes("no available channel for model") ||
|
||||
msg.includes("unknown model") ||
|
||||
msg.includes("invalid model") ||
|
||||
msg.includes("api key is missing") ||
|
||||
msg.includes("invalid api key") ||
|
||||
msg.includes("unauthorized") ||
|
||||
msg.includes("cannot convert argument to a bytestring")
|
||||
);
|
||||
}
|
||||
|
||||
export function isImageFallbackModelCoolingDown(modelId: string): boolean {
|
||||
const snapshot = imageFailoverModelCooldown.get(modelId);
|
||||
if (!snapshot) return false;
|
||||
if (snapshot.until <= Date.now()) {
|
||||
imageFailoverModelCooldown.delete(modelId);
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
export function setImageFallbackModelCooldown(modelId: string, reason: string): void {
|
||||
imageFailoverModelCooldown.set(modelId, {
|
||||
until: Date.now() + IMAGE_FAILOVER_MODEL_COOLDOWN_MS,
|
||||
reason: reason.slice(0, 500),
|
||||
});
|
||||
}
|
||||
|
||||
export function isImageFailoverExcludedModel(model: { excludeFromFailover?: boolean } | null | undefined) {
|
||||
return Boolean(model?.excludeFromFailover);
|
||||
}
|
||||
|
||||
export type ImageFallbackCandidate = {
|
||||
model: any;
|
||||
provider: any;
|
||||
configured: boolean;
|
||||
sameProvider: boolean;
|
||||
};
|
||||
|
||||
export function getImageFailoverConvertProvider(providerType?: string | null) {
|
||||
switch (providerType) {
|
||||
case "bfl-image":
|
||||
case "ruxa-image":
|
||||
case "fal-image":
|
||||
case "openai-image":
|
||||
case "stepfun-image":
|
||||
case "midjourney":
|
||||
case "runway":
|
||||
case "krea":
|
||||
case "gemini":
|
||||
case "huanke-gemini":
|
||||
return providerType;
|
||||
case "grsai-gpt-image":
|
||||
return "gemini";
|
||||
case "volcano-vision":
|
||||
return "volcano";
|
||||
default:
|
||||
return "alibaba-image";
|
||||
}
|
||||
}
|
||||
|
||||
export function buildImageFallbackCandidates(
|
||||
allModels: Array<{ model: any; provider: any }>,
|
||||
options: {
|
||||
currentProviderId?: string;
|
||||
currentModelId?: string;
|
||||
currentModelKey?: string;
|
||||
configuredFallbackIds?: string[];
|
||||
}
|
||||
) {
|
||||
const { currentProviderId, currentModelId, currentModelKey, configuredFallbackIds = [] } = options;
|
||||
const configuredRefs = configuredFallbackIds
|
||||
.map((item) => String(item || "").trim())
|
||||
.filter(Boolean);
|
||||
const byModelRef = new Map<string, { model: any; provider: any }>();
|
||||
|
||||
const isEligible = (item: { model: any; provider: any }, sameProvider: boolean) => {
|
||||
if (
|
||||
!item?.model?.supportsImageGen ||
|
||||
!item?.model?.isEnabled ||
|
||||
item?.model?.isSubModel ||
|
||||
isImageFailoverExcludedModel(item?.model) ||
|
||||
item?.model?.modelId === currentModelKey ||
|
||||
item?.model?.id === currentModelId
|
||||
) {
|
||||
return false;
|
||||
}
|
||||
|
||||
const providerType = String(item?.provider?.type || "");
|
||||
if (!sameProvider && !IMAGE_FAILOVER_PROVIDER_ALLOWLIST.has(providerType)) {
|
||||
return false;
|
||||
}
|
||||
if (isImageFallbackModelCoolingDown(item.model.id)) {
|
||||
const cooldown = imageFailoverModelCooldown.get(item.model.id);
|
||||
console.log(
|
||||
`[Chat API] ⏭ Skip configured image fallback on cooldown: ${item.model.displayName} (${providerType}) - ${cooldown?.reason || "cooldown"}`
|
||||
);
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
};
|
||||
|
||||
allModels.forEach((item) => {
|
||||
if (!item?.model) return;
|
||||
byModelRef.set(String(item.model.id), item);
|
||||
if (item.model.modelId) {
|
||||
byModelRef.set(String(item.model.modelId), item);
|
||||
}
|
||||
});
|
||||
|
||||
const candidates: ImageFallbackCandidate[] = [];
|
||||
const seenModelIds = new Set<string>();
|
||||
const pushCandidate = (item: { model: any; provider: any } | undefined, configured: boolean) => {
|
||||
if (!item?.model) return;
|
||||
const sameProvider = Boolean(currentProviderId && item.provider?.id === currentProviderId);
|
||||
if (!isEligible(item, sameProvider)) return;
|
||||
const modelId = String(item.model.id || "");
|
||||
if (!modelId || seenModelIds.has(modelId)) return;
|
||||
seenModelIds.add(modelId);
|
||||
candidates.push({
|
||||
model: item.model,
|
||||
provider: item.provider,
|
||||
configured,
|
||||
sameProvider,
|
||||
});
|
||||
};
|
||||
|
||||
configuredRefs.forEach((modelRef) => {
|
||||
pushCandidate(byModelRef.get(modelRef), true);
|
||||
});
|
||||
|
||||
if (currentProviderId) {
|
||||
allModels
|
||||
.filter((item) => item.provider?.id === currentProviderId)
|
||||
.forEach((item) => pushCandidate(item, false));
|
||||
}
|
||||
|
||||
return candidates;
|
||||
}
|
||||
|
||||
export async function generateImageWithFailoverCandidate(params: {
|
||||
candidate: ImageFallbackCandidate;
|
||||
messages: ChatMessage[];
|
||||
imageParams: any;
|
||||
fallbackImageParams: any;
|
||||
}) {
|
||||
const { candidate, messages, imageParams, fallbackImageParams } = params;
|
||||
const providerType = String(candidate.provider?.type || "");
|
||||
const candidateRuntimeModelId = String(candidate.model?.modelId || candidate.model?.id || "");
|
||||
if (
|
||||
providerType === "fal-image" &&
|
||||
isFalGptImage2EditModelId(candidateRuntimeModelId)
|
||||
) {
|
||||
throw new Error("FAL GPT Image 2 Edit 仅支持异步队列调用，已跳过同步故障迁移");
|
||||
}
|
||||
const candidateLanguageModel = await getLanguageModelById(candidate.model.id);
|
||||
if (!candidateLanguageModel || typeof (candidateLanguageModel as any).doGenerate !== "function") {
|
||||
throw new Error(`备用模型不支持当前生图故障迁移调用：${candidate.model.displayName || candidate.model.id}`);
|
||||
}
|
||||
|
||||
const candidateMessages = await convertMessagesWithImages(
|
||||
messages,
|
||||
getImageFailoverConvertProvider(providerType)
|
||||
);
|
||||
const candidateResult = await (candidateLanguageModel as any).doGenerate({
|
||||
messages: candidateMessages,
|
||||
imageParams: imageParams || fallbackImageParams,
|
||||
modelDisplayName: candidate.model.displayName,
|
||||
overrideModelId: candidate.model.modelId,
|
||||
});
|
||||
|
||||
if (candidateResult?.finishReason === "error") {
|
||||
throw new Error(candidateResult.text || "图片生成失败");
|
||||
}
|
||||
|
||||
return {
|
||||
text: candidateResult.text || "图片生成失败",
|
||||
usedModelId: String(candidate.model.modelId || candidate.model.id || ""),
|
||||
displayName: String(candidate.model.displayName || candidate.model.modelId || candidate.model.id || ""),
|
||||
};
|
||||
}
|
||||
|
||||
export function getImageCrossProviderCandidates(
|
||||
allModels: any[],
|
||||
options: {
|
||||
currentModelId: string;
|
||||
currentProviderId?: string;
|
||||
providerPriority: Record<string, number>;
|
||||
}
|
||||
) {
|
||||
return allModels
|
||||
.filter((m) => {
|
||||
if (
|
||||
!m?.model?.supportsImageGen ||
|
||||
!m?.model?.isEnabled ||
|
||||
m?.model?.isSubModel ||
|
||||
isImageFailoverExcludedModel(m?.model)
|
||||
) {
|
||||
return false;
|
||||
}
|
||||
if (!IMAGE_FAILOVER_PROVIDER_ALLOWLIST.has(String(m?.provider?.type || ""))) {
|
||||
return false;
|
||||
}
|
||||
if (m.model.id === options.currentModelId) return false;
|
||||
if (options.currentProviderId && m.provider.id === options.currentProviderId) return false;
|
||||
if (isImageFallbackModelCoolingDown(m.model.id)) {
|
||||
const cooldown = imageFailoverModelCooldown.get(m.model.id);
|
||||
console.log(
|
||||
`[Chat API] ⏭ Skip image failover model on cooldown: ${m.model.displayName} (${m.provider.type}) - ${cooldown?.reason || "cooldown"}`
|
||||
);
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
})
|
||||
.sort((a, b) => {
|
||||
const left = options.providerPriority[a.provider.type as string] ?? 99;
|
||||
const right = options.providerPriority[b.provider.type as string] ?? 99;
|
||||
if (left !== right) return left - right;
|
||||
return (a.model.sortOrder ?? 0) - (b.model.sortOrder ?? 0);
|
||||
});
|
||||
}
|
||||
964
app/(chat)/api/chat/_lib/image-media.ts
Normal file
964
app/(chat)/api/chat/_lib/image-media.ts
Normal file
@ -0,0 +1,964 @@
|
||||
import { convertToModelMessages } from "ai";
|
||||
import { inlineVisionImages } from "@/lib/ai/vision-inline";
|
||||
import { extractImageUrlsFromText } from "@/lib/image-url";
|
||||
import { parseImageAltMetadata } from "@/lib/ai/runtime/generated-image-persistence";
|
||||
import { uploadFile } from "@/lib/storage/service";
|
||||
import type { ChatMessage } from "@/lib/types";
|
||||
import { ANNOTATION_SYSTEM_PROMPT, hasAnnotatedImage } from "./message-preparation";
|
||||
|
||||
export function guessImageMediaType(url: string): string {
|
||||
const cleanUrl = url.split("?")[0]?.toLowerCase() || "";
|
||||
if (cleanUrl.endsWith(".jpg") || cleanUrl.endsWith(".jpeg")) return "image/jpeg";
|
||||
if (cleanUrl.endsWith(".webp")) return "image/webp";
|
||||
if (cleanUrl.endsWith(".gif")) return "image/gif";
|
||||
if (cleanUrl.endsWith(".bmp")) return "image/bmp";
|
||||
if (cleanUrl.endsWith(".svg")) return "image/svg+xml";
|
||||
return "image/png";
|
||||
}
|
||||
|
||||
export function injectHistoryImagesToLastUserMessage(
|
||||
messages: ChatMessage[],
|
||||
historyImages: string[]
|
||||
): ChatMessage[] {
|
||||
if (!historyImages || historyImages.length === 0) return messages;
|
||||
|
||||
let lastUserIndex = -1;
|
||||
for (let i = messages.length - 1; i >= 0; i--) {
|
||||
if (messages[i]?.role === "user") {
|
||||
lastUserIndex = i;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (lastUserIndex < 0) return messages;
|
||||
|
||||
const lastUserMessage = messages[lastUserIndex];
|
||||
const hasExistingImage = lastUserMessage.parts?.some(
|
||||
(p: any) => p.type === "file" || p.type === "image" || p.type === "image_url"
|
||||
);
|
||||
if (hasExistingImage) return messages;
|
||||
|
||||
const imageParts = historyImages.map((url, index) => ({
|
||||
type: "file" as const,
|
||||
url,
|
||||
name: `history-image-${index + 1}`,
|
||||
mediaType: guessImageMediaType(url),
|
||||
mimeType: guessImageMediaType(url),
|
||||
}));
|
||||
|
||||
const updatedMessage: ChatMessage = {
|
||||
...lastUserMessage,
|
||||
parts: [...lastUserMessage.parts, ...imageParts],
|
||||
};
|
||||
|
||||
return messages.map((msg, idx) => (idx === lastUserIndex ? updatedMessage : msg));
|
||||
}
|
||||
|
||||
export function isImageInputPart(part: any) {
|
||||
return part?.type === "file" || part?.type === "image" || part?.type === "image_url";
|
||||
}
|
||||
|
||||
export function normalizeResolvedTargetImageUrls(imageUrls?: string[] | null) {
|
||||
return Array.from(
|
||||
new Set(
|
||||
(imageUrls || [])
|
||||
.map((url) => String(url || "").trim())
|
||||
.filter((url) => /^https?:\/\//i.test(url))
|
||||
)
|
||||
).slice(0, 8);
|
||||
}
|
||||
|
||||
export function summarizeTargetImageUrl(url?: string) {
|
||||
const normalized = String(url || "").trim();
|
||||
if (!normalized) return undefined;
|
||||
try {
|
||||
const parsed = new URL(normalized);
|
||||
return `${parsed.origin}${parsed.pathname}`.slice(0, 160);
|
||||
} catch {
|
||||
return normalized.slice(0, 160);
|
||||
}
|
||||
}
|
||||
|
||||
export function applyResolvedTargetImagesToLastUserMessage(
|
||||
messages: ChatMessage[],
|
||||
targetImageUrls: string[]
|
||||
): ChatMessage[] {
|
||||
const normalizedTargetImageUrls = normalizeResolvedTargetImageUrls(targetImageUrls);
|
||||
if (normalizedTargetImageUrls.length === 0) return messages;
|
||||
|
||||
let lastUserIndex = -1;
|
||||
for (let i = messages.length - 1; i >= 0; i--) {
|
||||
if (messages[i]?.role === "user") {
|
||||
lastUserIndex = i;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (lastUserIndex < 0) return messages;
|
||||
const lastUserMessage = messages[lastUserIndex];
|
||||
const nonImageParts = (lastUserMessage.parts || []).filter((part: any) => !isImageInputPart(part));
|
||||
const imageParts = normalizedTargetImageUrls.map((url, index) => ({
|
||||
type: "file" as const,
|
||||
url,
|
||||
name: `target-image-${index + 1}`,
|
||||
mediaType: guessImageMediaType(url),
|
||||
mimeType: guessImageMediaType(url),
|
||||
}));
|
||||
|
||||
const updatedMessage: ChatMessage = {
|
||||
...lastUserMessage,
|
||||
parts: [...imageParts, ...nonImageParts],
|
||||
};
|
||||
|
||||
return messages.map((msg, idx) => (idx === lastUserIndex ? updatedMessage : msg));
|
||||
}
|
||||
|
||||
export function compactAutoDemand(text: string, maxLength = 56): string {
|
||||
const normalized = text.replace(/\s+/g, " ").trim();
|
||||
if (!normalized) return "按你刚才的描述执行";
|
||||
if (normalized.length <= maxLength) return normalized;
|
||||
return `${normalized.slice(0, maxLength)}...`;
|
||||
}
|
||||
|
||||
export function buildAutoModeFallbackMessage(params: {
|
||||
text: string;
|
||||
hasImage: boolean;
|
||||
fromHistory: boolean;
|
||||
imageSummary?: string;
|
||||
}): string {
|
||||
const demand = compactAutoDemand(params.text);
|
||||
|
||||
if (params.hasImage) {
|
||||
const sourceLine = params.fromHistory
|
||||
? "基于上一轮图片继续处理。"
|
||||
: "基于当前上传图片处理。";
|
||||
|
||||
return [
|
||||
"### 需求理解",
|
||||
`- ${demand}`,
|
||||
"### 图片分析",
|
||||
params.imageSummary
|
||||
? `- ${params.imageSummary.replace(/\s+/g, " ").trim()}`
|
||||
: `- ${sourceLine}`,
|
||||
"- 我会优先保持主体、构图和整体风格一致，只处理你指定的改动。",
|
||||
"### 执行计划",
|
||||
"- 先完成本轮核心改动，再检查边缘与光影一致性。",
|
||||
"- 若结果偏差，我会按“保留A，修改B”的方式继续迭代。",
|
||||
"### 输出设置",
|
||||
"- 默认单图输出，优先稳定命中。",
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
return [
|
||||
"### 需求理解",
|
||||
`- ${demand}`,
|
||||
"### 执行计划",
|
||||
"- 先锁定主体与场景，再细化风格、材质与光影。",
|
||||
"### 输出设置",
|
||||
"- 默认单图输出，优先稳定命中。",
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
export function sanitizePromptOnlyUserIntent(text: string): string {
|
||||
return String(text || "")
|
||||
.replace(/先?分析(?:一下)?(?:这张)?(?:图片|图像|这张图)?/g, "")
|
||||
.replace(/帮我|给我|先给我|只给我|仅给我/g, "")
|
||||
.replace(/优化(?:一下)?提示词|提示词优化|提示词|prompt/gi, "")
|
||||
.replace(/不要生图|先别生图|不需要生图|不要生成|别生成|不直接生成/g, "")
|
||||
.replace(/[，。；：,.!！?？]/g, " ")
|
||||
.replace(/\s+/g, " ")
|
||||
.trim();
|
||||
}
|
||||
|
||||
export function buildPromptOnlyOptimizationSeed(params: {
|
||||
text: string;
|
||||
imageSummary?: string;
|
||||
}): string {
|
||||
const cleanedIntent = sanitizePromptOnlyUserIntent(params.text);
|
||||
const summary = String(params.imageSummary || "").replace(/\s+/g, " ").trim();
|
||||
|
||||
if (cleanedIntent && summary) {
|
||||
return `${cleanedIntent}。参考图关键特征：${summary}`;
|
||||
}
|
||||
if (summary) {
|
||||
return `基于参考图生成一段可直接复现核心主体、场景、风格与光线特征的中文提示词。参考图关键特征：${summary}`;
|
||||
}
|
||||
if (cleanedIntent) {
|
||||
return cleanedIntent;
|
||||
}
|
||||
return "基于当前参考图整理一段可直接使用的中文提示词。";
|
||||
}
|
||||
|
||||
export function buildPromptOnlyReply(params: {
|
||||
userText: string;
|
||||
imageSummary?: string;
|
||||
optimizedPrompt: string;
|
||||
}): string {
|
||||
const demand = sanitizePromptOnlyUserIntent(params.userText) || "你希望我先分析这张图并整理可直接使用的提示词";
|
||||
const summaryLines = String(params.imageSummary || "")
|
||||
.split(/\r?\n/)
|
||||
.map((line) => line.replace(/\s+/g, " ").trim())
|
||||
.filter(Boolean);
|
||||
const formattedSummary =
|
||||
summaryLines.length > 0
|
||||
? summaryLines.map((line) => `- ${line}`).join("\n")
|
||||
: "- 我已基于当前图片梳理主体、场景、风格与光线特征。";
|
||||
|
||||
return [
|
||||
"【需求理解】",
|
||||
`- ${demand}`,
|
||||
"【图片分析】",
|
||||
formattedSummary,
|
||||
"【优化后提示词】",
|
||||
`- ${params.optimizedPrompt.replace(/\s+/g, " ").trim()}`,
|
||||
"【使用建议】",
|
||||
"- 本轮仅输出优化后的提示词，不直接执行生图。",
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
export function hasStructuredAgentSections(message?: string): boolean {
|
||||
if (!message) return false;
|
||||
return /(?:###\s*需求理解|【\s*需求理解\s*】|###\s*图片分析|【\s*图片分析\s*】|###\s*执行计划|【\s*执行计划\s*】|###\s*优化后提示词|【\s*优化后提示词\s*】)/.test(
|
||||
message,
|
||||
);
|
||||
}
|
||||
|
||||
export function ensureStructuredAgentFriendlyMessage(params: {
|
||||
message?: string;
|
||||
text: string;
|
||||
hasImage: boolean;
|
||||
fromHistory: boolean;
|
||||
}): string {
|
||||
const existingMessage = String(params.message || "").trim();
|
||||
if (hasStructuredAgentSections(existingMessage)) {
|
||||
return existingMessage;
|
||||
}
|
||||
|
||||
const fallback = buildAutoModeFallbackMessage({
|
||||
text: params.text,
|
||||
hasImage: params.hasImage,
|
||||
fromHistory: params.fromHistory,
|
||||
});
|
||||
|
||||
if (!existingMessage) {
|
||||
return fallback;
|
||||
}
|
||||
|
||||
const normalizedSupplement = existingMessage.replace(/\s+/g, " ").trim();
|
||||
if (!normalizedSupplement) {
|
||||
return fallback;
|
||||
}
|
||||
|
||||
return fallback.replace(
|
||||
"### 执行计划\n",
|
||||
`### 执行计划\n- 补充说明：${normalizedSupplement}\n`,
|
||||
);
|
||||
}
|
||||
|
||||
export function extractMessageImageUrls(parts: any[]): string[] {
|
||||
return (parts || [])
|
||||
.map((part: any) => {
|
||||
if (!part) return null;
|
||||
if (part.type === "file") return part.url || part.data || null;
|
||||
if (part.type === "image") return part.url || part.image || null;
|
||||
if (part.type === "image_url") return part.image_url?.url || null;
|
||||
return null;
|
||||
})
|
||||
.filter((url: any): url is string => typeof url === "string" && /^https?:\/\//i.test(url));
|
||||
}
|
||||
|
||||
export function extractMessageImageAssetIds(parts: any[]): string[] {
|
||||
return Array.from(
|
||||
new Set(
|
||||
(parts || [])
|
||||
.map((part: any) => {
|
||||
if (!part || typeof part !== "object") return null;
|
||||
return typeof part.assetId === "string" && part.assetId.trim()
|
||||
? part.assetId.trim()
|
||||
: null;
|
||||
})
|
||||
.filter((assetId: any): assetId is string => typeof assetId === "string" && assetId.length > 0),
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
export function shouldEnrichImageAnalysisText(message?: string): boolean {
|
||||
if (!message) return false;
|
||||
if (!/(###\s*图片分析|【\s*图片分析\s*】)/.test(message)) return false;
|
||||
return /(基于当前上传图片处理|基于你选中的原图处理|基于上一轮图片继续处理)/.test(message);
|
||||
}
|
||||
|
||||
export function injectImageAnalysisSummary(message: string, summary: string): string {
|
||||
const normalizedSummaryLines = summary
|
||||
.split(/\r?\n/)
|
||||
.map((line) => line.replace(/\s+/g, " ").trim())
|
||||
.filter(Boolean);
|
||||
if (normalizedSummaryLines.length === 0) return message;
|
||||
const lines = message.split("\n");
|
||||
const analysisHeaderIndex = lines.findIndex((line) =>
|
||||
/^(###\s*图片分析|【\s*图片分析\s*】)/.test(line.trim())
|
||||
);
|
||||
if (analysisHeaderIndex < 0) return message;
|
||||
|
||||
for (let i = analysisHeaderIndex + 1; i < lines.length; i += 1) {
|
||||
const current = lines[i]?.trim() || "";
|
||||
if (/^(###\s*|【\s*[^】]+\s*】)/.test(current)) break;
|
||||
if (/^- /.test(current)) {
|
||||
lines.splice(i, 1, ...normalizedSummaryLines.map((line) => `- ${line}`));
|
||||
return lines.join("\n");
|
||||
}
|
||||
}
|
||||
|
||||
const insertIndex = analysisHeaderIndex + 1;
|
||||
lines.splice(insertIndex, 0, ...normalizedSummaryLines.map((line) => `- ${line}`));
|
||||
return lines.join("\n");
|
||||
}
|
||||
|
||||
export function ensureImageAnalysisSummarySection(message: string, summary: string): string {
|
||||
const normalizedSummaryLines = summary
|
||||
.split(/\r?\n/)
|
||||
.map((line) => line.replace(/\s+/g, " ").trim())
|
||||
.filter(Boolean);
|
||||
if (normalizedSummaryLines.length === 0) return message;
|
||||
|
||||
if (/(###\s*图片分析|【\s*图片分析\s*】)/.test(message)) {
|
||||
return injectImageAnalysisSummary(message, normalizedSummaryLines.join("\n"));
|
||||
}
|
||||
|
||||
const trimmed = message.trim();
|
||||
const summaryBlock = normalizedSummaryLines.map((line) => `- ${line}`).join("\n");
|
||||
if (!trimmed) return `【图片分析】\n${summaryBlock}`;
|
||||
return `${trimmed}\n【图片分析】\n${summaryBlock}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* 自定义消息转换函数
|
||||
* 注意：gemini-image 和 huanke-gemini 都需要 URL 格式（不是 Base64）
|
||||
* 因为这些 provider 内部会自行处理图片上传到 S3
|
||||
*/
|
||||
export async function convertMessagesWithImages(
|
||||
messages: ChatMessage[],
|
||||
providerType:
|
||||
| string
|
||||
| {
|
||||
providerType?: string;
|
||||
providerBaseUrl?: string | null;
|
||||
modelId?: string | null;
|
||||
displayName?: string | null;
|
||||
forceInlineImageUrls?: boolean;
|
||||
loggerPrefix?: string;
|
||||
},
|
||||
) {
|
||||
type ImageConversionTarget = {
|
||||
providerType?: string;
|
||||
providerBaseUrl?: string | null;
|
||||
modelId?: string | null;
|
||||
displayName?: string | null;
|
||||
forceInlineImageUrls?: boolean;
|
||||
loggerPrefix?: string;
|
||||
};
|
||||
|
||||
const normalizeTarget = (
|
||||
target: string | ImageConversionTarget,
|
||||
): Required<Pick<ImageConversionTarget, "providerType" | "forceInlineImageUrls" | "loggerPrefix">> &
|
||||
Omit<ImageConversionTarget, "providerType" | "forceInlineImageUrls" | "loggerPrefix"> => {
|
||||
if (typeof target === "string") {
|
||||
return {
|
||||
providerType: target,
|
||||
providerBaseUrl: null,
|
||||
modelId: null,
|
||||
displayName: null,
|
||||
forceInlineImageUrls: false,
|
||||
loggerPrefix: "[Chat API] Vision inline",
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
providerType: String(target.providerType || ""),
|
||||
providerBaseUrl: target.providerBaseUrl || null,
|
||||
modelId: target.modelId || null,
|
||||
displayName: target.displayName || null,
|
||||
forceInlineImageUrls: target.forceInlineImageUrls === true,
|
||||
loggerPrefix: target.loggerPrefix || "[Chat API] Vision inline",
|
||||
};
|
||||
};
|
||||
|
||||
const extractImagePartUrl = (part: unknown): string | null => {
|
||||
const partAny = part as any;
|
||||
const candidateUrl =
|
||||
partAny?.type === "file"
|
||||
? partAny.url || partAny.data
|
||||
: partAny?.type === "image"
|
||||
? partAny.image || partAny.url
|
||||
: partAny?.type === "image_url"
|
||||
? partAny.image_url?.url
|
||||
: undefined;
|
||||
|
||||
if (typeof candidateUrl !== "string") return null;
|
||||
const normalized = candidateUrl.trim();
|
||||
if (!/^https?:\/\//i.test(normalized)) return null;
|
||||
return normalized;
|
||||
};
|
||||
|
||||
const replaceImagePartUrl = (part: unknown, nextUrl: string): unknown => {
|
||||
const partAny = part as any;
|
||||
if (partAny?.type === "file") {
|
||||
return {
|
||||
...partAny,
|
||||
url: nextUrl,
|
||||
data: nextUrl,
|
||||
mediaType: String(partAny.mediaType || "").startsWith("image/")
|
||||
? partAny.mediaType
|
||||
: "image/jpeg",
|
||||
mimeType: String(partAny.mimeType || "").startsWith("image/")
|
||||
? partAny.mimeType
|
||||
: "image/jpeg",
|
||||
};
|
||||
}
|
||||
if (partAny?.type === "image") {
|
||||
return {
|
||||
...partAny,
|
||||
image: nextUrl,
|
||||
url: nextUrl,
|
||||
};
|
||||
}
|
||||
if (partAny?.type === "image_url") {
|
||||
return {
|
||||
...partAny,
|
||||
image_url: {
|
||||
...(partAny.image_url || {}),
|
||||
url: nextUrl,
|
||||
},
|
||||
};
|
||||
}
|
||||
return part;
|
||||
};
|
||||
|
||||
const inlineImageUrlsInMessages = async (
|
||||
sourceMessages: ChatMessage[],
|
||||
target: ReturnType<typeof normalizeTarget>,
|
||||
) => {
|
||||
const uniqueUrls = Array.from(
|
||||
new Set(
|
||||
sourceMessages
|
||||
.flatMap((message) => message.parts || [])
|
||||
.map((part) => extractImagePartUrl(part))
|
||||
.filter((url): url is string => Boolean(url)),
|
||||
),
|
||||
);
|
||||
|
||||
if (uniqueUrls.length === 0) {
|
||||
return sourceMessages;
|
||||
}
|
||||
|
||||
const { inlineDataUrls } = await inlineVisionImages(uniqueUrls, {
|
||||
maxImages: Math.min(8, uniqueUrls.length),
|
||||
maxSide: 1200,
|
||||
timeoutMs: 12000,
|
||||
retries: 2,
|
||||
loggerPrefix: target.loggerPrefix,
|
||||
}).catch((error) => {
|
||||
console.warn(`${target.loggerPrefix} failed to inline message images:`, error);
|
||||
return { inlineDataUrls: [], skipped: [] };
|
||||
});
|
||||
|
||||
if (inlineDataUrls.length === 0) {
|
||||
return sourceMessages;
|
||||
}
|
||||
|
||||
const inlineMap = new Map<string, string>();
|
||||
for (let index = 0; index < inlineDataUrls.length; index += 1) {
|
||||
const originalUrl = uniqueUrls[index];
|
||||
const inlineUrl = inlineDataUrls[index];
|
||||
if (originalUrl && inlineUrl) {
|
||||
inlineMap.set(originalUrl, inlineUrl);
|
||||
}
|
||||
}
|
||||
|
||||
if (inlineMap.size === 0) {
|
||||
return sourceMessages;
|
||||
}
|
||||
|
||||
console.log(`${target.loggerPrefix} inlined chat message images:`, {
|
||||
total: uniqueUrls.length,
|
||||
inlined: inlineMap.size,
|
||||
providerType: target.providerType,
|
||||
modelId: target.modelId,
|
||||
displayName: target.displayName,
|
||||
});
|
||||
|
||||
return sourceMessages.map((message) => ({
|
||||
...message,
|
||||
parts: (message.parts || []).map((part) => {
|
||||
const originalUrl = extractImagePartUrl(part);
|
||||
if (!originalUrl) return part;
|
||||
const inlineUrl = inlineMap.get(originalUrl);
|
||||
return inlineUrl ? replaceImagePartUrl(part, inlineUrl) : part;
|
||||
}) as typeof message.parts,
|
||||
})) as ChatMessage[];
|
||||
};
|
||||
|
||||
const target = normalizeTarget(providerType);
|
||||
let preparedMessages = messages;
|
||||
|
||||
if (target.forceInlineImageUrls) {
|
||||
preparedMessages = await inlineImageUrlsInMessages(preparedMessages, target);
|
||||
}
|
||||
|
||||
const normalizedProviderType = target.providerType;
|
||||
|
||||
if (
|
||||
normalizedProviderType === "bfl-image" ||
|
||||
normalizedProviderType === "ruxa-image" ||
|
||||
normalizedProviderType === "fal-image" ||
|
||||
normalizedProviderType === "openai-image" ||
|
||||
normalizedProviderType === "stepfun-image" ||
|
||||
normalizedProviderType === "midjourney" ||
|
||||
normalizedProviderType === "runway" ||
|
||||
normalizedProviderType === "krea"
|
||||
) {
|
||||
const convertedMessages = [];
|
||||
|
||||
for (const message of preparedMessages) {
|
||||
const content: any[] = [];
|
||||
|
||||
for (const part of message.parts) {
|
||||
const partAny = part as any;
|
||||
|
||||
if (partAny.type === "text" && partAny.text) {
|
||||
content.push({
|
||||
type: "text",
|
||||
text: partAny.text,
|
||||
});
|
||||
continue;
|
||||
}
|
||||
|
||||
const candidateUrl =
|
||||
partAny.type === "file"
|
||||
? (partAny.url || partAny.data)
|
||||
: partAny.type === "image"
|
||||
? (partAny.image || partAny.url)
|
||||
: partAny.type === "image_url"
|
||||
? partAny.image_url?.url
|
||||
: undefined;
|
||||
|
||||
if (!candidateUrl || typeof candidateUrl !== "string") continue;
|
||||
if (candidateUrl.startsWith("data:")) {
|
||||
throw new Error("当前模型不支持 Base64 图片传输，请先上传图片后再编辑");
|
||||
}
|
||||
if (!candidateUrl.startsWith("http")) continue;
|
||||
|
||||
content.push({
|
||||
type: "image_url",
|
||||
image_url: {
|
||||
url: candidateUrl,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
convertedMessages.push({
|
||||
role: message.role,
|
||||
content: content.length === 1 && content[0].type === "text" ? content[0].text : content,
|
||||
});
|
||||
}
|
||||
|
||||
return convertedMessages;
|
||||
}
|
||||
|
||||
// 所有 provider 都使用 URL 格式传输图片
|
||||
// gemini-image 和 huanke-gemini 内部会将 Base64 转换为 S3 URL
|
||||
// 这里不再进行 Base64 转换，直接返回标准格式
|
||||
if (normalizedProviderType !== "gemini") {
|
||||
return convertToModelMessages(preparedMessages);
|
||||
}
|
||||
|
||||
// 🆕 检测是否有标注图片
|
||||
const hasAnnotation = hasAnnotatedImage(messages);
|
||||
if (hasAnnotation) {
|
||||
console.log("[Message Convert] 🎨 检测到标注图片，将自动添加系统提示");
|
||||
}
|
||||
|
||||
// 对于 Gemini，需要特殊处理图片
|
||||
const convertedMessages = [];
|
||||
|
||||
for (const message of preparedMessages) {
|
||||
const content: any[] = [];
|
||||
|
||||
// 🆕 如果是用户消息且有标注图片，在文本前添加系统提示
|
||||
let hasAddedAnnotationPrompt = false;
|
||||
|
||||
// 处理消息的各个部分
|
||||
for (const part of message.parts) {
|
||||
if (part.type === "text" && part.text) {
|
||||
let textContent = part.text;
|
||||
|
||||
// 🆕 如果有标注图片且是用户消息，在用户文本前添加系统提示
|
||||
if (hasAnnotation && message.role === "user" && !hasAddedAnnotationPrompt) {
|
||||
textContent = ANNOTATION_SYSTEM_PROMPT + "\n\n用户请求：" + textContent;
|
||||
hasAddedAnnotationPrompt = true;
|
||||
console.log("[Message Convert] 🎨 已注入标注提示词");
|
||||
}
|
||||
|
||||
content.push({
|
||||
type: "text",
|
||||
text: textContent,
|
||||
});
|
||||
} else if (part.type === "file" && part.url) {
|
||||
// 直接使用 URL 格式（不再转换为 Base64）
|
||||
// gemini-image 和 huanke-gemini provider 内部会处理图片上传
|
||||
content.push({
|
||||
type: "image_url",
|
||||
image_url: {
|
||||
url: part.url,
|
||||
},
|
||||
});
|
||||
console.log("[Message Convert] Image kept as URL format:", part.url.substring(0, 80) + "...");
|
||||
}
|
||||
}
|
||||
|
||||
// 🆕 如果有标注图片但消息中没有文本部分，添加一个纯提示文本
|
||||
if (hasAnnotation && message.role === "user" && !hasAddedAnnotationPrompt && content.length > 0) {
|
||||
content.unshift({
|
||||
type: "text",
|
||||
text: ANNOTATION_SYSTEM_PROMPT,
|
||||
});
|
||||
console.log("[Message Convert] 🎨 已在消息开头注入标注提示词");
|
||||
}
|
||||
|
||||
convertedMessages.push({
|
||||
role: message.role,
|
||||
content: content.length === 1 && content[0].type === "text"
|
||||
? content[0].text
|
||||
: content,
|
||||
});
|
||||
}
|
||||
|
||||
return convertedMessages;
|
||||
}
|
||||
|
||||
export function extractOriginalImageUrlsFromText(text: string): string[] {
|
||||
return extractImageUrlsFromText(text);
|
||||
}
|
||||
|
||||
export function normalizeImageInputRef(rawValue: unknown, requestOrigin?: string): string | null {
|
||||
if (typeof rawValue !== "string") return null;
|
||||
const trimmed = rawValue.trim();
|
||||
if (!trimmed) return null;
|
||||
|
||||
if (/^data:image\/[a-z0-9+.-]+;base64,/i.test(trimmed)) {
|
||||
return trimmed.replace(/\s+/g, "");
|
||||
}
|
||||
|
||||
if (/^https?:\/\//i.test(trimmed)) {
|
||||
return trimmed;
|
||||
}
|
||||
|
||||
if (requestOrigin && trimmed.startsWith("/")) {
|
||||
try {
|
||||
return new URL(trimmed, requestOrigin).toString();
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
export function collectImageInputsFromMessageParts(
|
||||
parts: Array<Record<string, any>> | undefined,
|
||||
requestOrigin?: string,
|
||||
): string[] {
|
||||
if (!Array.isArray(parts) || parts.length === 0) return [];
|
||||
|
||||
const result: string[] = [];
|
||||
const pushInput = (rawValue: unknown) => {
|
||||
const normalized = normalizeImageInputRef(rawValue, requestOrigin);
|
||||
if (!normalized) return;
|
||||
if (!result.includes(normalized)) {
|
||||
result.push(normalized);
|
||||
}
|
||||
};
|
||||
|
||||
for (const part of parts) {
|
||||
if (!part) continue;
|
||||
|
||||
if (part.type === "file") {
|
||||
pushInput(part.url);
|
||||
pushInput(part.data);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (part.type === "image") {
|
||||
pushInput(part.url);
|
||||
pushInput(part.image);
|
||||
pushInput(part.data);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (part.type === "image_url") {
|
||||
pushInput(part.image_url?.url);
|
||||
pushInput(part.url);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (part.type === "text" && typeof part.text === "string") {
|
||||
const parsedUrls = extractOriginalImageUrlsFromText(part.text);
|
||||
for (const parsedUrl of parsedUrls) {
|
||||
pushInput(parsedUrl);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
export function mimeTypeToExtension(mimeType: string) {
|
||||
const normalized = mimeType.toLowerCase();
|
||||
if (normalized.includes("png")) return "png";
|
||||
if (normalized.includes("webp")) return "webp";
|
||||
if (normalized.includes("gif")) return "gif";
|
||||
if (normalized.includes("bmp")) return "bmp";
|
||||
return "jpg";
|
||||
}
|
||||
|
||||
export function getExtensionFromUrl(url: string): string {
|
||||
try {
|
||||
const pathname = new URL(url).pathname;
|
||||
const ext = pathname.split(".").pop()?.toLowerCase();
|
||||
if (!ext) return "jpg";
|
||||
if (["jpg", "jpeg", "png", "webp", "gif", "bmp"].includes(ext)) return ext;
|
||||
} catch {
|
||||
// ignore
|
||||
}
|
||||
return "jpg";
|
||||
}
|
||||
|
||||
export async function uploadRemoteImageToStorage(imageUrl: string): Promise<string> {
|
||||
const response = await fetch(imageUrl, {
|
||||
method: "GET",
|
||||
signal: AbortSignal.timeout(30_000),
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
throw new Error(`Remote image fetch failed: ${response.status} ${response.statusText}`);
|
||||
}
|
||||
|
||||
const contentType = response.headers.get("content-type")?.toLowerCase() || "";
|
||||
if (contentType && !contentType.startsWith("image/")) {
|
||||
throw new Error(`Remote url is not an image: ${contentType}`);
|
||||
}
|
||||
|
||||
const arrayBuffer = await response.arrayBuffer();
|
||||
if (!arrayBuffer || arrayBuffer.byteLength === 0) {
|
||||
throw new Error("Remote image content is empty");
|
||||
}
|
||||
|
||||
const ext = contentType ? mimeTypeToExtension(contentType) : getExtensionFromUrl(imageUrl);
|
||||
const fileName = `remote-image-${Date.now()}-${Math.random().toString(36).slice(2, 8)}.${ext}`;
|
||||
const uploaded = await uploadFile(fileName, arrayBuffer);
|
||||
return uploaded.url;
|
||||
}
|
||||
|
||||
export async function isRemoteImageAccessible(imageUrl: string): Promise<boolean> {
|
||||
try {
|
||||
const headResponse = await fetch(imageUrl, {
|
||||
method: "HEAD",
|
||||
signal: AbortSignal.timeout(10_000),
|
||||
});
|
||||
|
||||
if (headResponse.ok) {
|
||||
return true;
|
||||
}
|
||||
|
||||
if (headResponse.status !== 405 && headResponse.status !== 403) {
|
||||
return false;
|
||||
}
|
||||
|
||||
const probeResponse = await fetch(imageUrl, {
|
||||
method: "GET",
|
||||
headers: {
|
||||
Range: "bytes=0-0",
|
||||
},
|
||||
signal: AbortSignal.timeout(10_000),
|
||||
});
|
||||
return probeResponse.ok;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
export async function ensureHostedImageUrl(imageUrl: string): Promise<string> {
|
||||
const normalizedUrl = imageUrl.trim();
|
||||
if (!normalizedUrl) {
|
||||
throw new Error("Image URL is empty");
|
||||
}
|
||||
|
||||
if (normalizedUrl.startsWith("data:image")) {
|
||||
return uploadDataUrlImage(normalizedUrl);
|
||||
}
|
||||
|
||||
if (!/^https?:\/\//i.test(normalizedUrl)) {
|
||||
throw new Error(`Unsupported image URL: ${normalizedUrl}`);
|
||||
}
|
||||
|
||||
if (/\/uploads\//i.test(normalizedUrl)) {
|
||||
const accessible = await isRemoteImageAccessible(normalizedUrl);
|
||||
if (accessible) {
|
||||
return normalizedUrl;
|
||||
}
|
||||
}
|
||||
|
||||
return uploadRemoteImageToStorage(normalizedUrl);
|
||||
}
|
||||
|
||||
export async function uploadDataUrlImage(dataUrl: string): Promise<string> {
|
||||
const match = dataUrl.match(/^data:([^;]+);base64,([\s\S]+)$/i);
|
||||
if (!match) {
|
||||
throw new Error("Invalid data URL image format");
|
||||
}
|
||||
|
||||
const mimeType = match[1] || "image/jpeg";
|
||||
const base64Data = (match[2] || "").replace(/\s+/g, "");
|
||||
if (!base64Data) {
|
||||
throw new Error("Empty base64 image content");
|
||||
}
|
||||
|
||||
const buffer = Buffer.from(base64Data, "base64");
|
||||
const fileExt = mimeTypeToExtension(mimeType);
|
||||
const fileName = `inline-image-${Date.now()}-${Math.random().toString(36).slice(2, 8)}.${fileExt}`;
|
||||
const arrayBuffer = buffer.buffer.slice(
|
||||
buffer.byteOffset,
|
||||
buffer.byteOffset + buffer.byteLength
|
||||
) as ArrayBuffer;
|
||||
const uploaded = await uploadFile(fileName, arrayBuffer);
|
||||
return uploaded.url;
|
||||
}
|
||||
|
||||
export async function normalizeAssistantImageMarkdown(
|
||||
text: string,
|
||||
imageSize = "2K"
|
||||
): Promise<string> {
|
||||
if (!text) return text;
|
||||
|
||||
const markdownDataUrlRegex = /!\[([^\]]*)\]\s*\(\s*(data:image\/[a-z0-9+.-]+;base64,[^)]+)\s*\)/gim;
|
||||
const markdownMatches = [...text.matchAll(markdownDataUrlRegex)];
|
||||
let normalizedText = text;
|
||||
const uploadedUrls: string[] = [];
|
||||
|
||||
if (markdownMatches.length > 0) {
|
||||
let rebuilt = "";
|
||||
let cursor = 0;
|
||||
|
||||
for (const match of markdownMatches) {
|
||||
const matchText = match[0];
|
||||
const dataUrl = match[2];
|
||||
const start = match.index ?? -1;
|
||||
if (start < 0) continue;
|
||||
|
||||
rebuilt += normalizedText.slice(cursor, start);
|
||||
|
||||
try {
|
||||
const uploadedUrl = await uploadDataUrlImage(dataUrl);
|
||||
uploadedUrls.push(uploadedUrl);
|
||||
rebuilt += ``;
|
||||
} catch (error) {
|
||||
console.error("[Chat API] Failed to upload inline data URL image:", error);
|
||||
rebuilt += matchText;
|
||||
}
|
||||
|
||||
cursor = start + matchText.length;
|
||||
}
|
||||
|
||||
rebuilt += normalizedText.slice(cursor);
|
||||
normalizedText = rebuilt;
|
||||
}
|
||||
|
||||
if (normalizedText.includes("data:image")) {
|
||||
const rawDataUrlRegex = /data:image\/[a-z0-9+.-]+;base64,[a-z0-9+/=\s]+/gim;
|
||||
const rawMatches = normalizedText.match(rawDataUrlRegex) ?? [];
|
||||
|
||||
for (const rawDataUrl of rawMatches) {
|
||||
try {
|
||||
const uploadedUrl = await uploadDataUrlImage(rawDataUrl);
|
||||
uploadedUrls.push(uploadedUrl);
|
||||
normalizedText = normalizedText.replace(rawDataUrl, uploadedUrl);
|
||||
} catch (error) {
|
||||
console.error("[Chat API] Failed to normalize raw data URL image:", error);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const markdownHttpImageRegex = /!\[([^\]]*)\]\s*\(\s*(https?:\/\/[^)\s]+)\s*\)/gim;
|
||||
const markdownHttpMatches = [...normalizedText.matchAll(markdownHttpImageRegex)];
|
||||
|
||||
if (markdownHttpMatches.length > 0) {
|
||||
let rebuilt = "";
|
||||
let cursor = 0;
|
||||
|
||||
for (const match of markdownHttpMatches) {
|
||||
const matchText = match[0];
|
||||
const altText = match[1] || "";
|
||||
const thumbnailUrl = (match[2] || "").trim();
|
||||
const start = match.index ?? -1;
|
||||
if (start < 0) continue;
|
||||
|
||||
rebuilt += normalizedText.slice(cursor, start);
|
||||
|
||||
try {
|
||||
const { originalUrl, imageSize: parsedImageSize } = parseImageAltMetadata(
|
||||
altText,
|
||||
thumbnailUrl
|
||||
);
|
||||
|
||||
let hostedThumbnailUrl = thumbnailUrl;
|
||||
let hostedOriginalUrl =
|
||||
originalUrl && originalUrl !== thumbnailUrl ? originalUrl : thumbnailUrl;
|
||||
|
||||
try {
|
||||
hostedThumbnailUrl = await ensureHostedImageUrl(thumbnailUrl);
|
||||
} catch (error) {
|
||||
console.error("[Chat API] Failed to normalize remote thumbnail URL:", error);
|
||||
}
|
||||
|
||||
if (originalUrl && originalUrl !== thumbnailUrl) {
|
||||
try {
|
||||
hostedOriginalUrl = await ensureHostedImageUrl(originalUrl);
|
||||
} catch (error) {
|
||||
console.error("[Chat API] Failed to normalize remote original URL:", error);
|
||||
hostedOriginalUrl = hostedThumbnailUrl || originalUrl;
|
||||
}
|
||||
} else {
|
||||
hostedOriginalUrl = hostedThumbnailUrl;
|
||||
}
|
||||
|
||||
const finalThumbnailUrl = hostedThumbnailUrl || thumbnailUrl;
|
||||
const finalOriginalUrl = hostedOriginalUrl || finalThumbnailUrl;
|
||||
|
||||
uploadedUrls.push(finalThumbnailUrl);
|
||||
uploadedUrls.push(finalOriginalUrl);
|
||||
rebuilt += ``;
|
||||
} catch (error) {
|
||||
console.error("[Chat API] Failed to parse remote image markdown metadata:", error);
|
||||
rebuilt += matchText;
|
||||
}
|
||||
|
||||
cursor = start + matchText.length;
|
||||
}
|
||||
|
||||
rebuilt += normalizedText.slice(cursor);
|
||||
normalizedText = rebuilt;
|
||||
}
|
||||
|
||||
const hasMarkdownImage = /!\[[^\]]*\]\((https?:\/\/[^)]+)\)/i.test(normalizedText);
|
||||
if (!hasMarkdownImage && uploadedUrls.length > 0) {
|
||||
const uniqueUrls = [...new Set(uploadedUrls)];
|
||||
console.log(
|
||||
`[Chat API] Converted ${uniqueUrls.length} image link(s) to hosted link(s)`
|
||||
);
|
||||
return uniqueUrls
|
||||
.map((url) => ``)
|
||||
.join("\n");
|
||||
}
|
||||
|
||||
return normalizedText;
|
||||
}
|
||||
169
app/(chat)/api/chat/_lib/message-preparation.ts
Normal file
169
app/(chat)/api/chat/_lib/message-preparation.ts
Normal file
@ -0,0 +1,169 @@
|
||||
import {
|
||||
analyzeOptimizedPromptMerge,
|
||||
buildOptimizedPromptPreview,
|
||||
} from "@/lib/ai/runtime/optimized-prompt";
|
||||
import type { ChatMessage } from "@/lib/types";
|
||||
|
||||
/**
|
||||
* 将图片 URL 转换为 base64 格式
|
||||
* 用于将 S3 URL 的图片转换为 base64 发送给 Gemini API
|
||||
*/
|
||||
export async function convertImageUrlToBase64(url: string): Promise<string> {
|
||||
try {
|
||||
console.log("[Image Convert] Fetching image from:", url);
|
||||
const response = await fetch(url);
|
||||
if (!response.ok) {
|
||||
throw new Error(`Failed to fetch image: ${response.statusText}`);
|
||||
}
|
||||
|
||||
const arrayBuffer = await response.arrayBuffer();
|
||||
const buffer = Buffer.from(arrayBuffer);
|
||||
const base64 = buffer.toString("base64");
|
||||
|
||||
// 获取 MIME 类型
|
||||
const contentType = response.headers.get("content-type") || "image/jpeg";
|
||||
const dataUrl = `data:${contentType};base64,${base64}`;
|
||||
|
||||
console.log("[Image Convert] Converted to base64, size:", base64.length);
|
||||
return dataUrl;
|
||||
} catch (error) {
|
||||
console.error("[Image Convert] Failed:", error);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 检测消息中是否包含用户标注过的图片（文件名以 annotated- 开头）
|
||||
*/
|
||||
export function hasAnnotatedImage(messages: ChatMessage[]): boolean {
|
||||
for (const message of messages) {
|
||||
for (const part of message.parts) {
|
||||
if (part.type === "file" && "name" in part && typeof part.name === "string") {
|
||||
if (part.name.startsWith("annotated-")) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* 标注图片的系统提示词（用户不可见，后台自动添加）
|
||||
*/
|
||||
export const ANNOTATION_SYSTEM_PROMPT =
|
||||
"【重要】用户提供的图片上有标注线条（如箭头、圆圈、画笔痕迹或文字说明），这些标注是用户用来指示需要修改的区域或方向。请仔细理解标注的含义，按照标注指示进行编辑，但在生成的新图片中不要保留这些标注线条，输出干净的编辑结果。";
|
||||
|
||||
export function applyHiddenPromptToMessages(
|
||||
messages: ChatMessage[],
|
||||
hiddenPrompt?: string,
|
||||
): ChatMessage[] {
|
||||
if (!hiddenPrompt || !hiddenPrompt.trim()) return messages;
|
||||
const cloned = messages.map((msg) => ({
|
||||
...msg,
|
||||
parts: msg.parts.map((part) => ({ ...part })),
|
||||
}));
|
||||
for (let i = cloned.length - 1; i >= 0; i--) {
|
||||
const msg = cloned[i];
|
||||
if (msg.role !== "user") continue;
|
||||
const textPart = msg.parts.find((part) => part.type === "text") as
|
||||
| { type: "text"; text: string }
|
||||
| undefined;
|
||||
if (textPart) {
|
||||
textPart.text = `${hiddenPrompt}\n\n${textPart.text}`.trim();
|
||||
} else {
|
||||
msg.parts.unshift({ type: "text", text: hiddenPrompt });
|
||||
}
|
||||
break;
|
||||
}
|
||||
return cloned;
|
||||
}
|
||||
|
||||
export function applyOptimizedPromptToLatestUserMessage(
|
||||
messages: ChatMessage[],
|
||||
optimizedPrompt?: string,
|
||||
hiddenPromptPrefix = "",
|
||||
): ChatMessage[] {
|
||||
const prompt = optimizedPrompt?.trim();
|
||||
if (!prompt) return messages;
|
||||
|
||||
const cloned = messages.map((msg) => ({
|
||||
...msg,
|
||||
parts: msg.parts.map((part) => ({ ...part })),
|
||||
}));
|
||||
|
||||
for (let i = cloned.length - 1; i >= 0; i -= 1) {
|
||||
const msg = cloned[i];
|
||||
if (msg.role !== "user") continue;
|
||||
|
||||
const textPart = msg.parts.find(
|
||||
(part): part is { type: "text"; text: string } => part.type === "text",
|
||||
);
|
||||
const originalText = textPart?.text?.trim() || "";
|
||||
const { normalizedOriginal, hasOriginalDemand, hasOriginalInOptimized } =
|
||||
analyzeOptimizedPromptMerge(originalText, prompt, hiddenPromptPrefix);
|
||||
const finalPrompt =
|
||||
hasOriginalDemand && !hasOriginalInOptimized
|
||||
? [
|
||||
hiddenPromptPrefix.trim(),
|
||||
`【核心目标（必须优先满足）】${normalizedOriginal}`,
|
||||
`【执行优化细节】${prompt}`,
|
||||
]
|
||||
.filter(Boolean)
|
||||
.join("\n\n")
|
||||
.trim()
|
||||
: `${hiddenPromptPrefix}${prompt}`.trim();
|
||||
console.log("[Prompt Merge] Final image prompt prepared:", {
|
||||
original: normalizedOriginal.substring(0, 120),
|
||||
optimized: prompt.substring(0, 120),
|
||||
merged: finalPrompt.substring(0, 180),
|
||||
forcePreserveOriginal: hasOriginalDemand && !hasOriginalInOptimized,
|
||||
});
|
||||
|
||||
if (textPart) {
|
||||
textPart.text = finalPrompt;
|
||||
} else {
|
||||
msg.parts.unshift({ type: "text", text: finalPrompt });
|
||||
}
|
||||
break;
|
||||
}
|
||||
|
||||
return cloned;
|
||||
}
|
||||
|
||||
export function injectOptimizedPromptSection(params: {
|
||||
message?: string;
|
||||
originalText: string;
|
||||
optimizedPrompt?: string;
|
||||
hiddenPromptPrefix?: string;
|
||||
}): string | undefined {
|
||||
const { message, originalText, optimizedPrompt, hiddenPromptPrefix } = params;
|
||||
if (!message?.trim()) return message;
|
||||
|
||||
const preview = buildOptimizedPromptPreview({
|
||||
originalText,
|
||||
optimizedPrompt,
|
||||
hiddenPromptPrefix,
|
||||
});
|
||||
if (!preview) return message;
|
||||
|
||||
const optimizedPromptSection = `【优化后提示词】\n${preview}`;
|
||||
const optimizedSectionRegex =
|
||||
/(?:###\s*优化后提示词|【\s*优化后提示词\s*】)[\s\S]*?(?=\n(?:###\s*|【\s*[^】]+\s*】)|$)/;
|
||||
if (optimizedSectionRegex.test(message)) {
|
||||
return message.replace(optimizedSectionRegex, optimizedPromptSection);
|
||||
}
|
||||
|
||||
const executionSectionRegex =
|
||||
/(?:###\s*执行计划|【\s*执行计划\s*】)[\s\S]*?(?=\n(?:###\s*|【\s*[^】]+\s*】)|$)/;
|
||||
if (executionSectionRegex.test(message)) {
|
||||
return message.replace(executionSectionRegex, optimizedPromptSection);
|
||||
}
|
||||
|
||||
const outputSectionRegex = /(?:###\s*输出设置|【\s*输出设置\s*】)/;
|
||||
if (outputSectionRegex.test(message)) {
|
||||
return message.replace(outputSectionRegex, `${optimizedPromptSection}\n$&`);
|
||||
}
|
||||
|
||||
return `${message.trim()}\n\n${optimizedPromptSection}`;
|
||||
}
|
||||
43
app/(chat)/api/chat/_lib/model-resolver.ts
Normal file
43
app/(chat)/api/chat/_lib/model-resolver.ts
Normal file
@ -0,0 +1,43 @@
|
||||
export function resolveModelDisplayNameByModelId(
|
||||
models: Array<{ model: any; provider?: any }> | undefined,
|
||||
modelId: string | undefined,
|
||||
): string {
|
||||
if (!modelId || !models?.length) return "";
|
||||
const match = models.find(
|
||||
(item) => item.model.modelId === modelId || item.model.id === modelId,
|
||||
);
|
||||
return String(match?.model?.displayName || "");
|
||||
}
|
||||
|
||||
export function resolveModelEntryByRuntimeModelId(
|
||||
models: Array<{ model: any; provider?: any }> | undefined,
|
||||
modelId: string | undefined,
|
||||
fallbackRef?: string,
|
||||
) {
|
||||
const candidates = [modelId, fallbackRef]
|
||||
.map((item) => String(item || "").trim())
|
||||
.filter(Boolean);
|
||||
if (!candidates.length || !models?.length) return null;
|
||||
|
||||
return (
|
||||
models.find((item) =>
|
||||
candidates.some(
|
||||
(candidate) => item.model.modelId === candidate || item.model.id === candidate,
|
||||
),
|
||||
) || null
|
||||
);
|
||||
}
|
||||
|
||||
export function resolveModelRefByRuntimeModelId(
|
||||
models: Array<{ model: any; provider?: any }> | undefined,
|
||||
modelId: string | undefined,
|
||||
fallbackRef?: string,
|
||||
): string {
|
||||
const normalizedModelId = String(modelId || "").trim();
|
||||
if (!normalizedModelId) return fallbackRef || "";
|
||||
|
||||
const match = models?.find(
|
||||
(item) => item.model.modelId === normalizedModelId || item.model.id === normalizedModelId,
|
||||
);
|
||||
return String(match?.model?.id || fallbackRef || normalizedModelId || "");
|
||||
}
|
||||
275
app/(chat)/api/chat/_lib/video-models.ts
Normal file
275
app/(chat)/api/chat/_lib/video-models.ts
Normal file
@ -0,0 +1,275 @@
|
||||
import { after } from "next/server";
|
||||
import {
|
||||
getAiModelById,
|
||||
getEnabledAiModelsWithProvider,
|
||||
updateMessageById,
|
||||
} from "@/lib/db/queries";
|
||||
import {
|
||||
isImageGenerationModelDescriptor,
|
||||
isVideoModelDescriptor,
|
||||
} from "@/lib/ai/model-classification";
|
||||
import type { StoryboardModelCandidate } from "@/lib/video-agent/types";
|
||||
|
||||
export type VideoModeType =
|
||||
| "text-to-video"
|
||||
| "first-frame"
|
||||
| "first-last-frame"
|
||||
| "reference-images";
|
||||
|
||||
export function isVideoProviderType(
|
||||
providerType?: string | null,
|
||||
model?: {
|
||||
modelId?: string | null;
|
||||
capabilities?: unknown;
|
||||
supportsImageGen?: boolean | null;
|
||||
},
|
||||
): boolean {
|
||||
return isVideoModelDescriptor({
|
||||
providerType,
|
||||
modelId: model?.modelId,
|
||||
capabilities: model?.capabilities,
|
||||
supportsImageGen: model?.supportsImageGen,
|
||||
});
|
||||
}
|
||||
|
||||
export function isImageGenerationModelType(
|
||||
providerType?: string | null,
|
||||
model?: {
|
||||
modelId?: string | null;
|
||||
capabilities?: unknown;
|
||||
supportsImageGen?: boolean | null;
|
||||
},
|
||||
): boolean {
|
||||
return isImageGenerationModelDescriptor({
|
||||
providerType,
|
||||
modelId: model?.modelId,
|
||||
capabilities: model?.capabilities,
|
||||
supportsImageGen: model?.supportsImageGen,
|
||||
});
|
||||
}
|
||||
|
||||
export function inferSupportedVideoModesByModelId(modelId: string): VideoModeType[] {
|
||||
const normalized = modelId.toLowerCase();
|
||||
|
||||
if (
|
||||
normalized === "gen4.5" ||
|
||||
normalized === "gen4_turbo" ||
|
||||
normalized === "gen4_aleph" ||
|
||||
normalized === "gen3a_turbo" ||
|
||||
normalized === "veo3" ||
|
||||
normalized === "veo3.1" ||
|
||||
normalized === "veo3.1_fast"
|
||||
) {
|
||||
return ["text-to-video", "first-frame"];
|
||||
}
|
||||
|
||||
if (normalized.includes("lite-t2v")) {
|
||||
return ["text-to-video"];
|
||||
}
|
||||
if (normalized.includes("lite-i2v")) {
|
||||
return ["first-frame", "first-last-frame", "reference-images"];
|
||||
}
|
||||
if (normalized.includes("pro-fast")) {
|
||||
return ["text-to-video", "first-frame"];
|
||||
}
|
||||
if (normalized.includes("-pro-") || normalized.endsWith("-pro")) {
|
||||
return ["text-to-video", "first-frame", "first-last-frame"];
|
||||
}
|
||||
|
||||
return ["text-to-video", "first-frame", "first-last-frame", "reference-images"];
|
||||
}
|
||||
|
||||
export function getSupportedVideoModesForModel(model: any): VideoModeType[] {
|
||||
const capabilityModes = (((model?.capabilities as any)?.supportedModes ?? []) as string[]).filter(
|
||||
(mode): mode is VideoModeType =>
|
||||
mode === "text-to-video" ||
|
||||
mode === "first-frame" ||
|
||||
mode === "first-last-frame" ||
|
||||
mode === "reference-images",
|
||||
);
|
||||
|
||||
if (capabilityModes.length > 0) {
|
||||
return capabilityModes;
|
||||
}
|
||||
|
||||
const modelId = String(model?.modelId || "");
|
||||
return inferSupportedVideoModesByModelId(modelId);
|
||||
}
|
||||
|
||||
export function scoreVideoSubModelForMode(modelId: string, mode: VideoModeType): number {
|
||||
const normalized = modelId.toLowerCase();
|
||||
const isProFast = normalized.includes("pro-fast");
|
||||
const isLiteT2V = normalized.includes("lite-t2v");
|
||||
const isLiteI2V = normalized.includes("lite-i2v");
|
||||
const isPro = (normalized.includes("-pro-") || normalized.endsWith("-pro")) && !isProFast;
|
||||
const isV15Pro = normalized.includes("seedance-1-5-pro");
|
||||
|
||||
switch (mode) {
|
||||
case "first-last-frame":
|
||||
if (isV15Pro) return 150;
|
||||
if (isPro) return 120;
|
||||
if (isLiteI2V) return 90;
|
||||
if (isProFast) return -1000;
|
||||
if (isLiteT2V) return -1000;
|
||||
return 10;
|
||||
case "first-frame":
|
||||
if (isV15Pro) return 140;
|
||||
if (isProFast) return 130;
|
||||
if (isPro) return 120;
|
||||
if (isLiteI2V) return 100;
|
||||
if (isLiteT2V) return 20;
|
||||
return 10;
|
||||
case "reference-images":
|
||||
if (isV15Pro) return 150;
|
||||
if (isLiteI2V) return 140;
|
||||
if (isPro) return 110;
|
||||
if (isProFast || isLiteT2V) return -1000;
|
||||
return 10;
|
||||
case "text-to-video":
|
||||
default:
|
||||
if (isV15Pro) return 140;
|
||||
if (isLiteT2V) return 130;
|
||||
if (isProFast) return 120;
|
||||
if (isPro) return 100;
|
||||
if (isLiteI2V) return 80;
|
||||
return 10;
|
||||
}
|
||||
}
|
||||
|
||||
export function selectBestVideoSubModelForMode(
|
||||
subModels: Array<{ model: any }>,
|
||||
mode: VideoModeType,
|
||||
): { model: any } | null {
|
||||
const compatible = subModels.filter(({ model }) => {
|
||||
const supportedModes = getSupportedVideoModesForModel(model);
|
||||
return supportedModes.includes(mode);
|
||||
});
|
||||
|
||||
const candidates = compatible.length > 0 ? compatible : subModels;
|
||||
if (candidates.length === 0) return null;
|
||||
|
||||
return [...candidates].sort((a, b) => {
|
||||
const scoreA = scoreVideoSubModelForMode(String(a.model?.modelId || ""), mode);
|
||||
const scoreB = scoreVideoSubModelForMode(String(b.model?.modelId || ""), mode);
|
||||
return scoreB - scoreA;
|
||||
})[0];
|
||||
}
|
||||
|
||||
export async function resolveStoryboardModelCandidates(
|
||||
selectedModelRef: string,
|
||||
): Promise<StoryboardModelCandidate[]> {
|
||||
const selectedModel = await getAiModelById({ id: selectedModelRef }).catch(() => undefined);
|
||||
if (!selectedModel) return [];
|
||||
|
||||
const configuredFallbackIds = [
|
||||
selectedModel.fallbackModelId1,
|
||||
selectedModel.fallbackModelId2,
|
||||
].filter((item): item is string => Boolean(String(item || "").trim()));
|
||||
|
||||
const allModels = await getEnabledAiModelsWithProvider(true, true).catch(() => []);
|
||||
const sameProvider = allModels
|
||||
.filter(
|
||||
({ model, provider }: any) =>
|
||||
provider?.id === selectedModel.providerId && model?.id !== selectedModel.id && model?.isEnabled,
|
||||
)
|
||||
.map(({ model }: any) => String(model?.id || "").trim())
|
||||
.filter(Boolean);
|
||||
|
||||
const candidateIds = Array.from(new Set([selectedModel.id, ...configuredFallbackIds, ...sameProvider]));
|
||||
const byModelRef = new Map<string, any>();
|
||||
allModels
|
||||
.map(({ model }: any) => model)
|
||||
.filter(Boolean)
|
||||
.forEach((model: any) => {
|
||||
byModelRef.set(String(model.id), model);
|
||||
if (model.modelId) {
|
||||
byModelRef.set(String(model.modelId), model);
|
||||
}
|
||||
});
|
||||
|
||||
return candidateIds
|
||||
.map((id) => byModelRef.get(id))
|
||||
.filter(Boolean)
|
||||
.map((model: any) => ({
|
||||
id: String(model.id),
|
||||
modelId: String(model.modelId || "").trim(),
|
||||
displayName: String(model.displayName || model.modelId || "").trim(),
|
||||
}))
|
||||
.filter((item) => item.modelId);
|
||||
}
|
||||
|
||||
export function extractVideoUrlFromText(text: string): string | null {
|
||||
const normalizedText = String(text || "");
|
||||
if (!normalizedText) return null;
|
||||
|
||||
const htmlMatch = normalizedText.match(/<video[^>]+src="([^"]+)"/i);
|
||||
if (htmlMatch?.[1]) return htmlMatch[1];
|
||||
|
||||
const markdownMatch = normalizedText.match(
|
||||
/\[[^\]]*]\((https?:\/\/[^)\s]+\.mp4(?:\?[^)\s]*)?)\)/i,
|
||||
);
|
||||
if (markdownMatch?.[1]) return markdownMatch[1];
|
||||
|
||||
const plainMatch = normalizedText.match(/(https?:\/\/[^\s"'<>]+\.mp4(?:\?[^\s"'<>]+)?)/i);
|
||||
return plainMatch?.[1] || null;
|
||||
}
|
||||
|
||||
export function isAlreadyArchivedVideoUrl(url: string): boolean {
|
||||
return /\/uploads\//i.test(url);
|
||||
}
|
||||
|
||||
export function scheduleVideoArchive(params: {
|
||||
chatId: string;
|
||||
messageId: string;
|
||||
text: string;
|
||||
temporaryVideoUrl: string;
|
||||
taskId?: string;
|
||||
metadata?: Record<string, any> | null;
|
||||
}) {
|
||||
const { chatId, messageId, temporaryVideoUrl, taskId, metadata } = params;
|
||||
if (!temporaryVideoUrl || isAlreadyArchivedVideoUrl(temporaryVideoUrl)) {
|
||||
return;
|
||||
}
|
||||
|
||||
const delayHoursRaw = Number(process.env.VIDEO_ARCHIVE_DELAY_HOURS || 48);
|
||||
const delayHours = Number.isFinite(delayHoursRaw)
|
||||
? Math.max(1, Math.min(24 * 30, Math.round(delayHoursRaw)))
|
||||
: 48;
|
||||
const archiveAfterAtMs = Date.now() + delayHours * 60 * 60 * 1000;
|
||||
|
||||
after(async () => {
|
||||
try {
|
||||
const nextMetadata = {
|
||||
...(metadata || {}),
|
||||
temporaryVideoUrl,
|
||||
storageStatus: "temporary",
|
||||
archiveMode: "delayed_job",
|
||||
archiveAfterAt: archiveAfterAtMs,
|
||||
archiveDelayHours: delayHours,
|
||||
archiveTaskId: taskId || messageId,
|
||||
updatedAt: Date.now(),
|
||||
};
|
||||
|
||||
await updateMessageById({
|
||||
messageId,
|
||||
chatId,
|
||||
videoStorageStatus: "temporary",
|
||||
temporaryVideoUrl,
|
||||
permanentVideoUrl: null,
|
||||
archivedAt: null,
|
||||
videoMetadata: nextMetadata as any,
|
||||
});
|
||||
|
||||
console.log("[Chat API] 🕒 已登记延迟归档视频:", {
|
||||
messageId,
|
||||
archiveAfterAt: new Date(archiveAfterAtMs).toISOString(),
|
||||
});
|
||||
} catch (error) {
|
||||
console.error("[Chat API] ❌ 登记视频延迟归档失败（保留临时地址）:", {
|
||||
messageId,
|
||||
temporaryVideoUrl,
|
||||
error,
|
||||
});
|
||||
}
|
||||
});
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
Loading…
x
Reference in New Issue
Block a user