827 lines
29 KiB
TypeScript
827 lines
29 KiB
TypeScript
/**
|
||
* 提示词优化器
|
||
* 将用户简单输入转换为专业的图片编辑提示词
|
||
*
|
||
* 主要应用场景：图片编辑（抠图、换背景、增强等）
|
||
*/
|
||
|
||
// 优化结果类型
|
||
export interface OptimizeResult {
|
||
originalPrompt: string;
|
||
optimizedPrompt: string;
|
||
negativePrompt: string;
|
||
recommendedImageParams?: {
|
||
aspectRatio?: string;
|
||
imageSize?: string;
|
||
imageCount?: number;
|
||
promptExtend?: boolean;
|
||
negativePrompt?: string;
|
||
};
|
||
friendlyMessage?: string; // 友好的对话消息
|
||
}
|
||
|
||
// 子意图的友好名称和回复模板
|
||
const FRIENDLY_MESSAGES: Record<string, { name: string; template: string }> = {
|
||
remove_background: {
|
||
name: "抠图",
|
||
template: "好的，我来帮你抠图，去除背景保留主体~",
|
||
},
|
||
replace_background: {
|
||
name: "换背景",
|
||
template: "收到，我来帮你把背景换成{background}~",
|
||
},
|
||
enhance: {
|
||
name: "图片增强",
|
||
template: "明白，我来帮你提升图片清晰度~",
|
||
},
|
||
extend: {
|
||
name: "图像拓展",
|
||
template: "好的，我来帮你扩展图片边界~",
|
||
},
|
||
inpaint: {
|
||
name: "局部修改",
|
||
template: "收到，我来帮你{action}{target}~",
|
||
},
|
||
style_transfer: {
|
||
name: "风格迁移",
|
||
template: "好的，我来帮你转换成{style}风格~",
|
||
},
|
||
resize: {
|
||
name: "尺寸调整",
|
||
template: "明白，我来帮你调整图片尺寸~",
|
||
},
|
||
color_adjust: {
|
||
name: "调色",
|
||
template: "收到，我来帮你调整图片颜色~",
|
||
},
|
||
};
|
||
|
||
// 图片编辑子意图的提示词模板
|
||
const EDIT_TEMPLATES: Record<
|
||
string,
|
||
{
|
||
template: string;
|
||
negative: string;
|
||
extractors: Record<string, RegExp>;
|
||
defaults: Record<string, string>;
|
||
useOriginalPrompt?: boolean; // 🆕 是否使用原始提示词
|
||
}
|
||
> = {
|
||
remove_background: {
|
||
template:
|
||
"精准抠图：去除图片背景，保留主体完整清晰，输出透明背景PNG格式。边缘处理要求：平滑自然无锯齿，保留精细细节（如发丝、毛发、边缘纹理）。主体保持原有清晰度和色彩。",
|
||
negative: "残留背景色块，边缘毛糙，锯齿明显，主体不完整，边缘模糊，细节丢失，颜色溢出",
|
||
extractors: {},
|
||
defaults: {},
|
||
},
|
||
|
||
replace_background: {
|
||
template:
|
||
"背景替换：将图片背景替换为{background}。要求：保持主体完整清晰不变形，边缘过渡自然平滑，光影效果与新背景协调一致。{extra}",
|
||
negative: "残留原背景，边缘不自然，透视错误，主体变形，光影不协调，明显P图痕迹，边缘光晕",
|
||
extractors: {
|
||
background: /(?:换成|改成|替换为|变成|换为|改为)(.+?)(?:背景|底|$)/,
|
||
},
|
||
defaults: {
|
||
background: "纯白色(#FFFFFF)",
|
||
extra: "保留主体自然阴影效果",
|
||
},
|
||
},
|
||
|
||
enhance: {
|
||
template:
|
||
"图片增强：提升图片整体清晰度和画质。要求：保持自然色彩不失真，智能去除噪点，增强细节层次，适度锐化边缘。{extra}",
|
||
negative: "过度锐化，色彩失真，噪点增加，伪影，画质下降，不自然的对比度，颜色过饱和",
|
||
extractors: {},
|
||
defaults: {
|
||
extra: "整体效果自然真实",
|
||
},
|
||
},
|
||
|
||
extend: {
|
||
template:
|
||
"图像拓展(Outpainting)：向{direction}方向扩展图片约{amount}。要求：扩展区域的内容、风格、色调与原图保持高度一致，光影过渡自然流畅，无明显接缝痕迹，透视关系正确。",
|
||
negative: "风格不一致，明显接缝，透视错误，内容突变，色调不匹配，光影断裂",
|
||
extractors: {
|
||
direction: /向?(上|下|左|右|四周|周围)/,
|
||
amount: /(\d+%?|一倍|两倍|50%|100%)/,
|
||
},
|
||
defaults: {
|
||
direction: "四周",
|
||
amount: "50%",
|
||
},
|
||
},
|
||
|
||
inpaint: {
|
||
template:
|
||
"局部修改(Inpainting)：{action}图片中的{target}。要求：使用周围环境自然填充修改区域，保持整体风格、色调、光影一致，无任何修复痕迹，过渡自然。",
|
||
negative: "明显修复痕迹，风格不一致，内容错误，边缘不自然，色差明显，纹理断裂",
|
||
extractors: {
|
||
target: /(?:去除|消除|擦掉|删除|移除|P掉|p掉)(.+?)(?:，|。|$| )/,
|
||
action: /(去除|消除|擦掉|删除|移除|修改)/,
|
||
},
|
||
defaults: {
|
||
target: "指定区域内的元素",
|
||
action: "去除",
|
||
},
|
||
},
|
||
|
||
style_transfer: {
|
||
template:
|
||
"风格迁移：将图片转换为{style}风格。要求：保持主体可识别性和基本特征，色彩风格协调统一，整体效果自然美观。",
|
||
negative: "主体变形严重，特征丢失，风格不统一，色彩异常，人物不可识别",
|
||
extractors: {
|
||
style: /(?:转换为|转化为|变成|变为|改成|改为|转成)(.+?)(?:风格|样式|效果|$)/,
|
||
},
|
||
defaults: {
|
||
style: "目标风格",
|
||
},
|
||
},
|
||
|
||
resize: {
|
||
template:
|
||
"尺寸调整：将图片{action}至{size}。要求：保持画质清晰，比例协调，细节不丢失。",
|
||
negative: "画质损失严重，比例失调，模糊，变形",
|
||
extractors: {
|
||
size: /(\d+[x×]\d+|\d+%)/,
|
||
action: /(放大|缩小|调整|裁剪)/,
|
||
},
|
||
defaults: {
|
||
size: "目标尺寸",
|
||
action: "调整",
|
||
},
|
||
},
|
||
|
||
color_adjust: {
|
||
template:
|
||
"颜色调整：{original_request}。要求：精准修改指定部分的颜色，保持其他区域不变，色彩过渡自然协调。",
|
||
negative: "色彩失真，过度调整，不自然，偏色严重，其他区域被误改",
|
||
extractors: {
|
||
// 匹配各种颜色修改表达方式
|
||
target_color: /(?:改成|换成|变成|改为|换为|变为|调成|调整为)(.+?)(?:色|$)/,
|
||
target_object: /(?:把|将)?(.+?)(?:的)?(?:颜色|色)/,
|
||
},
|
||
defaults: {
|
||
original_request: "进行色彩优化处理",
|
||
},
|
||
// 特殊标记：使用原始请求
|
||
useOriginalPrompt: true,
|
||
},
|
||
};
|
||
|
||
import { generateText } from "ai";
|
||
|
||
function inferStyleTransferTarget(originalPrompt: string, extracted?: string): string {
|
||
const text = (originalPrompt || "").trim();
|
||
const normalized = text.toLowerCase();
|
||
|
||
const cleanCandidate = (value?: string) =>
|
||
String(value || "")
|
||
.replace(/[\n\r]/g, " ")
|
||
.replace(/[，。！？；,.!?]/g, " ")
|
||
.replace(/\s+/g, " ")
|
||
.trim();
|
||
|
||
const explicitMatch = text.match(
|
||
/(?:转换为|转化为|变成|变为|改成|改为|转成)\s*([^，。！？；,.!?]+)/i
|
||
);
|
||
const explicitValue = cleanCandidate(explicitMatch?.[1]);
|
||
if (explicitValue) return explicitValue;
|
||
|
||
const extractedValue = cleanCandidate(extracted);
|
||
if (extractedValue) return extractedValue;
|
||
|
||
if (/写实|真实|实拍|照片|摄影|photoreal|realistic/.test(normalized)) {
|
||
return "真实写实照片";
|
||
}
|
||
if (/卡通|动漫|二次元|插画|漫画|anime|cartoon/.test(normalized)) {
|
||
return "卡通动漫";
|
||
}
|
||
if (/电影|cinematic|cinema/.test(normalized)) {
|
||
return "电影感";
|
||
}
|
||
if (/油画|watercolor|水彩|素描|国风|古风/.test(normalized)) {
|
||
return "艺术风格";
|
||
}
|
||
return "目标风格";
|
||
}
|
||
|
||
// 历史消息类型
|
||
interface HistoryMessage {
|
||
role: "user" | "assistant";
|
||
content: string;
|
||
}
|
||
|
||
// LLM 优化提示词的系统提示 - 专注于精准局部编辑
|
||
const LLM_OPTIMIZER_SYSTEM_PROMPT = `你是一个专业的AI图片编辑提示词优化专家。本系统主要用于产品图片编辑，需要精准控制编辑范围。
|
||
|
||
## 核心原则：精准局部编辑
|
||
1. **只修改用户指定的部分**：明确指出要修改的对象/区域
|
||
2. **保护其他元素**：强调保持其他所有元素完全不变
|
||
3. **细节要求**：包含颜色、材质、光影、位置等具体参数
|
||
4. **自然过渡**：修改区域与周围环境自然融合
|
||
|
||
## 图片标注处理（重要！）
|
||
如果用户消息中提到"图片中有标注"或"标注区域"，说明用户已经在图片上画了标记（箭头、圆圈、文字等）来指示需要修改的区域：
|
||
- **优先处理标注区域**：将标注标记的区域作为主要编辑目标
|
||
- **理解标注意图**：箭头指向的位置、圆圈圈住的区域、文字说明的内容都是用户想要修改的部分
|
||
- **强调标注**：在提示词中明确提到"按照图片中的标注/标记指示进行修改"
|
||
- **保护非标注区域**：标注以外的区域应保持完全不变
|
||
|
||
## 输出格式（严格JSON）
|
||
{
|
||
"optimizedPrompt": "精准的图片编辑指令",
|
||
"negativePrompt": "需要避免的效果"
|
||
}
|
||
|
||
## Nano Banana Pro 三段式模板（官方最佳实践）
|
||
optimizedPrompt 必须按以下结构输出：
|
||
目标：一句话说明最终要生成/修改的结果
|
||
约束：一句话说明必须保持不变和禁止项
|
||
细节：一句话补充构图、光线、材质、风格等关键细节
|
||
|
||
## 示例
|
||
|
||
用户请求："把椅子颜色改成绿色"
|
||
{
|
||
"optimizedPrompt": "【精准编辑】将图片中椅子的颜色改为绿色。具体要求：1.只修改椅子颜色，保持椅子的形状、材质纹理、光影反射完全不变；2.背景、地面、其他物品保持原样不做任何改动；3.绿色需自然协调，与原有光照环境匹配",
|
||
"negativePrompt": "背景改变，其他物品变色，形状变形，材质改变，光影不自然，整体色调偏移"
|
||
}
|
||
|
||
用户请求："把背景换成白色"
|
||
{
|
||
"optimizedPrompt": "【精准编辑】将图片背景替换为纯白色(#FFFFFF)。具体要求：1.只替换背景区域，主体产品保持完全不变；2.主体边缘精准抠取，无残留背景色；3.保留主体自然投影效果",
|
||
"negativePrompt": "主体被裁切，边缘锯齿，主体颜色改变，产品变形，细节丢失"
|
||
}
|
||
|
||
用户请求："把产品放大一点"
|
||
{
|
||
"optimizedPrompt": "【精准编辑】将图片中的产品主体等比例放大约20%。具体要求：1.保持产品清晰度和细节不丢失；2.保持产品比例不变形；3.背景自然填充延伸",
|
||
"negativePrompt": "产品模糊，比例变形，画质下降，背景突兀"
|
||
}
|
||
|
||
用户历史说"抠图"，现在说"再清晰一点"
|
||
{
|
||
"optimizedPrompt": "【精准编辑】提升图片清晰度。具体要求：1.增强主体产品的锐度和细节；2.保持透明背景不变；3.不改变产品颜色和形状",
|
||
"negativePrompt": "过度锐化，噪点增加，颜色失真，边缘光晕"
|
||
}
|
||
|
||
## 关键要求
|
||
- 必须使用【精准编辑】开头，明确这是局部编辑任务
|
||
- 必须列出"具体要求"说明要改什么、不改什么
|
||
- negativePrompt 必须包含"其他元素改变"相关描述
|
||
- 结合对话历史理解用户的连续编辑意图`;
|
||
|
||
const IMAGE_GEN_OPTIMIZER_SYSTEM_PROMPT = `你是一个专业的图片生成提示词优化专家，目标是把用户自然语言改写成更稳定、高质量、可迭代的生图提示词。
|
||
|
||
## 输出格式（严格 JSON）
|
||
{
|
||
"optimizedPrompt": "适合生图模型的中文提示词",
|
||
"negativePrompt": "需要避免的效果",
|
||
"suggestedParams": {
|
||
"aspectRatio": "1:1|2:3|3:2|3:4|4:3|4:5|5:4|9:16|16:9|21:9|auto",
|
||
"imageSize": "1K|2K|4K",
|
||
"imageCount": 1,
|
||
"promptExtend": true
|
||
}
|
||
}
|
||
|
||
## 优化原则
|
||
1. 保留用户核心诉求，不擅自改题。
|
||
2. 自动补齐：主体、场景、构图、材质、光线、风格、镜头感。
|
||
3. 对电商/家居场景优先强调：真实材质、空间比例合理、商业可用。
|
||
4. negativePrompt 要包含：畸形、模糊、水印、文字错乱、额外主体、比例错误等。
|
||
5. 输出简洁可执行，不要解释过程。
|
||
6. 参数建议必须服务于“首图命中率”：人物全身/竖构图偏 9:16，横向场景偏 16:9，未明确时可用 auto。
|
||
7. 按 Nano Banana Pro 官方三段式写法组织提示词：
|
||
- 目标：一句话写清最终画面
|
||
- 约束：一句话写清必须保持不变与禁止项
|
||
- 细节：一句话补充风格/材质/光线/构图
|
||
`;
|
||
|
||
function parseOptimizerJsonCandidate(raw: string): {
|
||
optimizedPrompt?: string;
|
||
negativePrompt?: string;
|
||
suggestedParams?: {
|
||
aspectRatio?: string;
|
||
imageSize?: string;
|
||
imageCount?: number;
|
||
promptExtend?: boolean;
|
||
};
|
||
} | null {
|
||
const normalized = raw
|
||
.trim()
|
||
.replace(/\uFEFF/g, "")
|
||
.replace(/[“”]/g, "\"")
|
||
.replace(/[‘’]/g, "'")
|
||
.replace(/,\s*([}\]])/g, "$1");
|
||
if (!normalized) return null;
|
||
try {
|
||
return JSON.parse(normalized);
|
||
} catch {
|
||
return null;
|
||
}
|
||
}
|
||
|
||
function parseOptimizerJson(responseText: string): {
|
||
optimizedPrompt?: string;
|
||
negativePrompt?: string;
|
||
suggestedParams?: {
|
||
aspectRatio?: string;
|
||
imageSize?: string;
|
||
imageCount?: number;
|
||
promptExtend?: boolean;
|
||
};
|
||
} | null {
|
||
if (!responseText || typeof responseText !== "string") return null;
|
||
|
||
const fencedBlocks = Array.from(
|
||
responseText.matchAll(/```(?:json)?\s*([\s\S]*?)```/gi)
|
||
).map((match) => match[1]?.trim() || "");
|
||
|
||
for (const block of fencedBlocks) {
|
||
const parsed = parseOptimizerJsonCandidate(block);
|
||
if (parsed) return parsed;
|
||
}
|
||
|
||
const text = responseText.trim();
|
||
const starts: number[] = [];
|
||
for (let i = 0; i < text.length; i++) {
|
||
if (text[i] === "{") starts.push(i);
|
||
}
|
||
|
||
for (const start of starts) {
|
||
let depth = 0;
|
||
for (let end = start; end < text.length; end++) {
|
||
if (text[end] === "{") depth += 1;
|
||
if (text[end] === "}") {
|
||
depth -= 1;
|
||
if (depth === 0) {
|
||
const candidate = text.slice(start, end + 1);
|
||
const parsed = parseOptimizerJsonCandidate(candidate);
|
||
if (parsed) return parsed;
|
||
break;
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
return null;
|
||
}
|
||
|
||
function sanitizeSuggestedParams(input?: {
|
||
aspectRatio?: string;
|
||
imageSize?: string;
|
||
imageCount?: number;
|
||
promptExtend?: boolean;
|
||
}) {
|
||
if (!input) return undefined;
|
||
const allowedAspectRatios = new Set([
|
||
"1:1",
|
||
"2:3",
|
||
"3:2",
|
||
"3:4",
|
||
"4:3",
|
||
"4:5",
|
||
"5:4",
|
||
"9:16",
|
||
"16:9",
|
||
"21:9",
|
||
"auto",
|
||
]);
|
||
const allowedSizes = new Set(["1K", "2K", "4K"]);
|
||
const normalizedAspectRatio = input.aspectRatio?.trim() || undefined;
|
||
const normalizedSize = input.imageSize?.trim().toUpperCase() || undefined;
|
||
const normalizedCount =
|
||
typeof input.imageCount === "number" && Number.isFinite(input.imageCount)
|
||
? Math.min(Math.max(Math.round(input.imageCount), 1), 4)
|
||
: undefined;
|
||
const output: NonNullable<OptimizeResult["recommendedImageParams"]> = {};
|
||
if (normalizedAspectRatio && allowedAspectRatios.has(normalizedAspectRatio)) {
|
||
output.aspectRatio = normalizedAspectRatio;
|
||
}
|
||
if (normalizedSize && allowedSizes.has(normalizedSize)) {
|
||
output.imageSize = normalizedSize;
|
||
}
|
||
if (normalizedCount) {
|
||
output.imageCount = normalizedCount;
|
||
}
|
||
if (typeof input.promptExtend === "boolean") {
|
||
output.promptExtend = input.promptExtend;
|
||
}
|
||
return Object.keys(output).length > 0 ? output : undefined;
|
||
}
|
||
|
||
function ensureNanoBananaPromptStructure(prompt: string, intent: string): string {
|
||
const clean = prompt.replace(/\s+/g, " ").trim();
|
||
if (!clean) return prompt;
|
||
if (/目标[:：].+约束[:：].+细节[:：]/s.test(clean)) {
|
||
return clean
|
||
.replace(/\s*目标[:：]\s*/g, "目标：")
|
||
.replace(/\s*约束[:：]\s*/g, "\n约束：")
|
||
.replace(/\s*细节[:：]\s*/g, "\n细节：");
|
||
}
|
||
|
||
const fragments = clean
|
||
.split(/[。！？!?]/)
|
||
.map((item) => item.trim())
|
||
.filter(Boolean);
|
||
const goal = fragments[0] || clean;
|
||
const defaultConstraint =
|
||
intent === "image_edit"
|
||
? "仅修改用户要求的部分，保持构图、主体比例、背景和光影关系稳定"
|
||
: "保持主体比例与构图稳定，避免多余元素、变形和文字错误";
|
||
const constraint = fragments[1] || defaultConstraint;
|
||
const detail =
|
||
fragments.slice(2).join("，") ||
|
||
"材质真实、边缘清晰、光线自然、色彩准确，输出商业可用效果";
|
||
|
||
return `目标：${goal}\n约束：${constraint}\n细节：${detail}`;
|
||
}
|
||
|
||
function shouldKeepBackgroundAsIs(originalPrompt: string, subIntent?: string): boolean {
|
||
if (subIntent === "replace_background" || subIntent === "remove_background") {
|
||
return false;
|
||
}
|
||
return !/(换背景|改背景|替换背景|背景改成|背景换成|白底|黑底|透明背景|场景换成|背景变成)/i.test(
|
||
originalPrompt
|
||
);
|
||
}
|
||
|
||
function shouldKeepCameraAsIs(originalPrompt: string): boolean {
|
||
return !/(角度|视角|机位|镜头|构图|俯拍|仰拍|侧拍|旋转|翻转|远景|近景|特写)/i.test(
|
||
originalPrompt
|
||
);
|
||
}
|
||
|
||
function enforceImageEditStability(params: {
|
||
optimizedPrompt: string;
|
||
originalPrompt: string;
|
||
subIntent?: string;
|
||
negativePrompt?: string;
|
||
}) {
|
||
const keepBackground = shouldKeepBackgroundAsIs(params.originalPrompt, params.subIntent);
|
||
const keepCamera = shouldKeepCameraAsIs(params.originalPrompt);
|
||
|
||
const stabilityRules = [
|
||
"只修改用户明确要求的内容，其他元素保持不变",
|
||
keepCamera ? "保持原图拍摄角度、主体朝向、构图和透视关系不变" : "",
|
||
keepBackground
|
||
? "保持原图背景颜色与明暗不变；若原图是白底，继续保持纯白背景(#FFFFFF)"
|
||
: "",
|
||
"保持主体形状比例、材质纹理和光影一致，禁止出现黑底/灰底污染",
|
||
].filter(Boolean);
|
||
|
||
const mergedNegativePrompt = [
|
||
params.negativePrompt || "",
|
||
keepCamera ? "角度变化，构图漂移，主体旋转" : "",
|
||
keepBackground ? "背景变黑，背景变灰，背景误改，白底丢失" : "",
|
||
"主体变形，比例错误，材质改变，边缘脏污",
|
||
]
|
||
.filter(Boolean)
|
||
.join("，");
|
||
|
||
const promptWithStability = `${params.optimizedPrompt.trim()}\n\n保真约束：${stabilityRules.join("；")}。`;
|
||
|
||
return {
|
||
optimizedPrompt: promptWithStability,
|
||
negativePrompt: mergedNegativePrompt,
|
||
};
|
||
}
|
||
|
||
function fallbackImageGenPrompt(originalPrompt: string): OptimizeResult {
|
||
const normalized = originalPrompt.replace(/\s+/g, " ").trim();
|
||
const basePrompt = [
|
||
"高质量图像生成：",
|
||
normalized,
|
||
"要求：主体清晰，构图平衡，光影自然，材质真实，细节丰富，整体风格统一。",
|
||
].join(" ");
|
||
const optimizedPrompt = ensureNanoBananaPromptStructure(basePrompt, "image_gen");
|
||
|
||
return {
|
||
originalPrompt,
|
||
optimizedPrompt,
|
||
negativePrompt: "低清晰度，构图混乱，比例畸形，重复主体，手部异常，文字错乱，水印，噪点，过曝，过度锐化",
|
||
recommendedImageParams: {
|
||
aspectRatio: "auto",
|
||
imageSize: "2K",
|
||
imageCount: 1,
|
||
promptExtend: true,
|
||
},
|
||
};
|
||
}
|
||
|
||
/**
|
||
* 🆕 使用 LLM 深度优化提示词（基于历史对话上下文）
|
||
*/
|
||
export async function optimizePromptWithLLM(params: {
|
||
originalPrompt: string;
|
||
intent: string;
|
||
subIntent?: string;
|
||
historyMessages?: HistoryMessage[];
|
||
context?: string;
|
||
}): Promise<OptimizeResult> {
|
||
const { originalPrompt, intent, subIntent, historyMessages, context } = params;
|
||
|
||
console.log("[Prompt Optimizer LLM] Input:", {
|
||
originalPrompt,
|
||
intent,
|
||
subIntent,
|
||
historyCount: historyMessages?.length || 0
|
||
});
|
||
|
||
if (intent !== "image_edit" && intent !== "image_gen") {
|
||
return {
|
||
originalPrompt,
|
||
optimizedPrompt: originalPrompt,
|
||
negativePrompt: "模糊，低质量，变形，水印，失真",
|
||
};
|
||
}
|
||
|
||
try {
|
||
// 动态导入避免循环依赖
|
||
const {
|
||
getEnabledAiModelsWithProvider,
|
||
getAgentAiModelWithProvider,
|
||
} = await import("@/lib/db/queries");
|
||
const { getLanguageModelById } = await import("./dynamic-provider");
|
||
|
||
// 提示词优化优先使用 Agent 模型（建议智普），失败时回退
|
||
const allModels = await getEnabledAiModelsWithProvider();
|
||
const agentModel = await getAgentAiModelWithProvider();
|
||
const isReasoningCompatibleProvider = (providerType: string) =>
|
||
["zhipu", "alibaba", "openai", "custom", "local", "volcano-multimodal", "volcano-vision"].includes(providerType);
|
||
const usableAgentModel =
|
||
agentModel && isReasoningCompatibleProvider(agentModel.provider.type)
|
||
? agentModel
|
||
: undefined;
|
||
const zhipuModel = allModels.find(
|
||
(m) => m.provider.type === "zhipu" && m.model.isEnabled && !m.model.isSubModel
|
||
);
|
||
const qwenModel = allModels.find(
|
||
(m) => m.provider.type === "alibaba" && m.model.supportsVision
|
||
);
|
||
const selectedOptimizerModel = usableAgentModel || zhipuModel || qwenModel;
|
||
|
||
if (!selectedOptimizerModel) {
|
||
console.log("[Prompt Optimizer LLM] No optimizer model available, falling back to template");
|
||
return optimizePrompt({ originalPrompt, intent, subIntent });
|
||
}
|
||
|
||
const model = await getLanguageModelById(selectedOptimizerModel.model.id);
|
||
if (!model) {
|
||
console.log("[Prompt Optimizer LLM] Failed to get model, falling back to template");
|
||
return optimizePrompt({ originalPrompt, intent, subIntent });
|
||
}
|
||
|
||
// 构建包含上下文的消息
|
||
let contextMessage = "";
|
||
if (context && context.trim().length > 0) {
|
||
contextMessage = `## 画布上下文（仅用于理解，不需要在最终提示词中复述坐标）：\n${context.trim()}\n\n`;
|
||
}
|
||
if (historyMessages && historyMessages.length > 0) {
|
||
const recentHistory = historyMessages.slice(-6); // 最近3轮对话
|
||
const historyText = recentHistory.map(msg =>
|
||
`${msg.role === 'user' ? '用户' : 'AI'}: ${msg.content.substring(0, 150)}`
|
||
).join('\n');
|
||
contextMessage += `## 对话历史：\n${historyText}\n\n`;
|
||
}
|
||
|
||
const optimizerSystemPrompt =
|
||
intent === "image_gen" ? IMAGE_GEN_OPTIMIZER_SYSTEM_PROMPT : LLM_OPTIMIZER_SYSTEM_PROMPT;
|
||
|
||
const intentLabel =
|
||
intent === "image_gen" ? "图片生成" : (subIntent || "图片编辑");
|
||
|
||
const finalUserMessage = `${contextMessage}## 当前用户请求：\n"${originalPrompt}"\n\n## 识别的意图：${intentLabel}\n\n请生成优化后的提示词：`;
|
||
|
||
console.log(
|
||
"[Prompt Optimizer LLM] Calling optimizer model:",
|
||
selectedOptimizerModel.model.displayName,
|
||
`(${selectedOptimizerModel.provider.type})`
|
||
);
|
||
|
||
const { text: responseText } = await generateText({
|
||
model,
|
||
system: optimizerSystemPrompt,
|
||
prompt: finalUserMessage,
|
||
temperature: 0.2,
|
||
maxTokens: 200,
|
||
} as any);
|
||
|
||
// 解析 JSON 响应
|
||
const parsedResult = parseOptimizerJson(responseText);
|
||
if (parsedResult) {
|
||
console.log("[Prompt Optimizer LLM] Result parsed");
|
||
const stabilized =
|
||
intent === "image_edit"
|
||
? enforceImageEditStability({
|
||
optimizedPrompt: ensureNanoBananaPromptStructure(
|
||
parsedResult.optimizedPrompt || originalPrompt,
|
||
intent
|
||
),
|
||
originalPrompt,
|
||
subIntent,
|
||
negativePrompt: parsedResult.negativePrompt || "模糊，低质量，变形，失真",
|
||
})
|
||
: {
|
||
optimizedPrompt: ensureNanoBananaPromptStructure(
|
||
parsedResult.optimizedPrompt || originalPrompt,
|
||
intent
|
||
),
|
||
negativePrompt: parsedResult.negativePrompt || "模糊，低质量，变形，失真",
|
||
};
|
||
return {
|
||
originalPrompt,
|
||
optimizedPrompt: stabilized.optimizedPrompt,
|
||
negativePrompt: stabilized.negativePrompt,
|
||
recommendedImageParams: sanitizeSuggestedParams(parsedResult.suggestedParams),
|
||
friendlyMessage: intent === "image_edit" ? FRIENDLY_MESSAGES[subIntent || ""]?.template : undefined,
|
||
};
|
||
}
|
||
|
||
// 解析失败，回退到模板优化
|
||
console.log(
|
||
"[Prompt Optimizer LLM] Parse failed, falling back to template. raw=",
|
||
responseText.slice(0, 320)
|
||
);
|
||
return optimizePrompt({ originalPrompt, intent, subIntent });
|
||
|
||
} catch (error) {
|
||
console.error("[Prompt Optimizer LLM] Error:", error);
|
||
// 出错时回退到模板优化
|
||
return optimizePrompt({ originalPrompt, intent, subIntent });
|
||
}
|
||
}
|
||
|
||
/**
|
||
* 优化提示词（模板方式 - 作为回退）
|
||
*/
|
||
export function optimizePrompt(params: {
|
||
originalPrompt: string;
|
||
intent: string;
|
||
subIntent?: string;
|
||
}): OptimizeResult {
|
||
const { originalPrompt, intent, subIntent } = params;
|
||
|
||
console.log("[Prompt Optimizer] Input:", { originalPrompt, intent, subIntent });
|
||
|
||
if (intent === "image_gen") {
|
||
return fallbackImageGenPrompt(originalPrompt);
|
||
}
|
||
|
||
// 只对图片编辑类意图进行优化
|
||
if (intent !== "image_edit") {
|
||
return {
|
||
originalPrompt,
|
||
optimizedPrompt: originalPrompt,
|
||
negativePrompt: "模糊，低质量，变形，水印，失真",
|
||
};
|
||
}
|
||
|
||
// 如果没有子意图，返回通用优化
|
||
if (!subIntent) {
|
||
const basePrompt = `图片编辑：${originalPrompt}。要求：效果自然，画质清晰，细节完整。`;
|
||
return {
|
||
originalPrompt,
|
||
optimizedPrompt: ensureNanoBananaPromptStructure(basePrompt, intent),
|
||
negativePrompt: "模糊，低质量，变形，水印，失真，不自然",
|
||
};
|
||
}
|
||
|
||
const template = EDIT_TEMPLATES[subIntent] as any;
|
||
if (!template) {
|
||
console.log("[Prompt Optimizer] No template for subIntent:", subIntent);
|
||
return {
|
||
originalPrompt,
|
||
optimizedPrompt: originalPrompt,
|
||
negativePrompt: "模糊，低质量，变形，水印，失真",
|
||
};
|
||
}
|
||
|
||
// 🆕 对于某些子意图，直接使用原始提示词（因为用户的描述更精准）
|
||
// 例如：color_adjust - "把椅子颜色改成绿色" 比模板更精准
|
||
if (template.useOriginalPrompt) {
|
||
console.log("[Prompt Optimizer] Using original prompt for", subIntent);
|
||
const optimizedPrompt = `${originalPrompt}。要求：精准执行用户要求，保持其他区域不变，效果自然协调。`;
|
||
return {
|
||
originalPrompt,
|
||
optimizedPrompt,
|
||
negativePrompt: template.negative,
|
||
friendlyMessage: FRIENDLY_MESSAGES[subIntent]?.template,
|
||
};
|
||
}
|
||
|
||
// 使用模板生成优化后的提示词
|
||
let optimizedPrompt = template.template;
|
||
|
||
// 从用户输入中提取参数并填充模板
|
||
for (const [key, regex] of Object.entries(template.extractors)) {
|
||
const match = originalPrompt.match(regex as RegExp);
|
||
let value = match ? match[1].trim() : template.defaults[key] || "";
|
||
|
||
if (subIntent === "style_transfer" && key === "style") {
|
||
value = inferStyleTransferTarget(originalPrompt, value);
|
||
}
|
||
|
||
// 清理提取的值
|
||
value = value.replace(/[，。！？,.!?]/g, "").trim();
|
||
|
||
if (!value) {
|
||
value = template.defaults[key] || "";
|
||
}
|
||
|
||
optimizedPrompt = optimizedPrompt.replace(`{${key}}`, value);
|
||
}
|
||
|
||
// 填充剩余的默认值
|
||
for (const [key, defaultValue] of Object.entries(template.defaults)) {
|
||
optimizedPrompt = optimizedPrompt.replace(`{${key}}`, defaultValue as string);
|
||
}
|
||
|
||
// 附加用户原始意图（如果优化后的提示词没有完全覆盖用户需求）
|
||
if (originalPrompt.length > 10 && !optimizedPrompt.includes(originalPrompt)) {
|
||
optimizedPrompt = `${optimizedPrompt}\n\n用户补充要求：${originalPrompt}`;
|
||
}
|
||
optimizedPrompt = ensureNanoBananaPromptStructure(optimizedPrompt, intent);
|
||
|
||
// 生成友好的对话消息
|
||
let friendlyMessage = "";
|
||
const friendlyTemplate = FRIENDLY_MESSAGES[subIntent];
|
||
if (friendlyTemplate) {
|
||
friendlyMessage = friendlyTemplate.template;
|
||
|
||
// 填充友好消息中的变量
|
||
for (const [key, regex] of Object.entries(template.extractors) as [string, RegExp][]) {
|
||
const match = originalPrompt.match(regex);
|
||
let value = match ? match[1].trim() : template.defaults[key] || "";
|
||
if (subIntent === "style_transfer" && key === "style") {
|
||
value = inferStyleTransferTarget(originalPrompt, value);
|
||
}
|
||
value = value.replace(/[，。！？,.!?]/g, "").trim();
|
||
if (!value) {
|
||
value = template.defaults[key] || "";
|
||
}
|
||
friendlyMessage = friendlyMessage.replace(`{${key}}`, value);
|
||
}
|
||
|
||
// 填充剩余的默认值
|
||
for (const [key, defaultValue] of Object.entries(template.defaults) as [string, string][]) {
|
||
friendlyMessage = friendlyMessage.replace(`{${key}}`, defaultValue);
|
||
}
|
||
}
|
||
|
||
const result: OptimizeResult = {
|
||
originalPrompt,
|
||
optimizedPrompt,
|
||
negativePrompt: template.negative,
|
||
friendlyMessage: friendlyMessage || `好的，我来帮你处理这张图片~`,
|
||
};
|
||
|
||
if (intent === "image_edit") {
|
||
const stabilized = enforceImageEditStability({
|
||
optimizedPrompt: result.optimizedPrompt,
|
||
originalPrompt,
|
||
subIntent,
|
||
negativePrompt: result.negativePrompt,
|
||
});
|
||
result.optimizedPrompt = stabilized.optimizedPrompt;
|
||
result.negativePrompt = stabilized.negativePrompt;
|
||
}
|
||
|
||
console.log("[Prompt Optimizer] Output:", result);
|
||
return result;
|
||
}
|
||
|
||
/**
|
||
* 提取背景描述（辅助函数）
|
||
*/
|
||
export function extractBackgroundDescription(text: string): string {
|
||
// 尝试提取用户描述的背景
|
||
const patterns = [
|
||
/(?:换成|改成|替换为|变成|换为|改为)(.+?)(?:背景|底)/,
|
||
/(.+?)(?:背景|底)$/,
|
||
/背景(?:换成|改成|替换为|变成)(.+)/,
|
||
];
|
||
|
||
for (const pattern of patterns) {
|
||
const match = text.match(pattern);
|
||
if (match && match[1]) {
|
||
return match[1].trim();
|
||
}
|
||
}
|
||
|
||
return "纯白色";
|
||
}
|
||
|
||
/**
|
||
* 检测是否需要保留阴影
|
||
*/
|
||
export function shouldPreserveShadow(text: string): boolean {
|
||
// 产品图、商品图通常需要保留阴影
|
||
if (/产品|商品|电商|白底|纯白/.test(text)) {
|
||
return true;
|
||
}
|
||
return false;
|
||
}
|