feat: optimize scene workflow prompt composition
This commit is contained in:
parent
3af9990311
commit
1cf1ba6bb6
@ -59,6 +59,7 @@ import {
|
||||
normalizeAgentPresetModeConfig,
|
||||
} from "@/lib/agents/preset-mode-config";
|
||||
import { buildAgentSceneImageRolePrompt } from "@/lib/agents/scene-image-role-prompt";
|
||||
import { optimizeAgentSceneWorkflowPrompt } from "@/lib/agents/scene-workflow-prompt-optimizer";
|
||||
import { getEnabledTools } from "@/lib/ai/tools-registry";
|
||||
import { isProductionEnvironment } from "@/lib/constants";
|
||||
import {
|
||||
@ -2892,6 +2893,39 @@ function getSceneUploadedImageBreakdown(params: {
|
||||
};
|
||||
}
|
||||
|
||||
function resolveScenePromptOptimizationImageUrls(params: {
|
||||
uploadedImageParts: AgentPresetMessageImagePart[];
|
||||
primaryProductImageCount: number;
|
||||
bundleProductImageCount: number;
|
||||
includeModelImage: boolean;
|
||||
}) {
|
||||
const namedPrimaryParts = params.uploadedImageParts.filter((part) =>
|
||||
String(part.name || "").trim().startsWith("主SKU参考图")
|
||||
);
|
||||
const namedModelPart = params.uploadedImageParts.find((part) =>
|
||||
String(part.name || "").trim().startsWith("品牌模特图")
|
||||
);
|
||||
|
||||
if (namedPrimaryParts.length > 0 || namedModelPart) {
|
||||
return {
|
||||
primaryImageUrl: namedPrimaryParts[0]?.url || "",
|
||||
modelImageUrl: namedModelPart?.url || null,
|
||||
};
|
||||
}
|
||||
|
||||
const primaryImageUrl =
|
||||
params.uploadedImageParts.slice(0, Math.max(0, params.primaryProductImageCount))[0]?.url || "";
|
||||
const modelIndex = params.primaryProductImageCount + params.bundleProductImageCount;
|
||||
|
||||
return {
|
||||
primaryImageUrl,
|
||||
modelImageUrl:
|
||||
params.includeModelImage && modelIndex < params.uploadedImageParts.length
|
||||
? params.uploadedImageParts[modelIndex]?.url || null
|
||||
: null,
|
||||
};
|
||||
}
|
||||
|
||||
function normalizeAgentSupplementPrompt(text: string, presetName?: string | null) {
|
||||
const normalized = String(text || "").trim();
|
||||
if (!normalized) return "";
|
||||
@ -2914,6 +2948,106 @@ function normalizeAgentWorkflowPrompt(text: string) {
|
||||
.trim();
|
||||
}
|
||||
|
||||
function parseAgentWorkflowPromptSections(text: string) {
|
||||
const sectionMap = new Map<string, string[]>();
|
||||
let activeSection = "__root__";
|
||||
const normalizedPrompt = normalizeAgentWorkflowPrompt(text);
|
||||
if (!normalizedPrompt) {
|
||||
return sectionMap;
|
||||
}
|
||||
|
||||
normalizedPrompt.split("\n").forEach((line) => {
|
||||
const trimmedLine = line.trim();
|
||||
if (!trimmedLine) {
|
||||
return;
|
||||
}
|
||||
const headerMatch = trimmedLine.match(/^【([^】]{1,30})】$/);
|
||||
if (headerMatch) {
|
||||
activeSection = headerMatch[1].trim();
|
||||
if (!sectionMap.has(activeSection)) {
|
||||
sectionMap.set(activeSection, []);
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
const currentLines = sectionMap.get(activeSection) || [];
|
||||
currentLines.push(trimmedLine);
|
||||
sectionMap.set(activeSection, currentLines);
|
||||
});
|
||||
|
||||
return sectionMap;
|
||||
}
|
||||
|
||||
function buildAgentPromptSectionText(
|
||||
title: string,
|
||||
lines: string[],
|
||||
options?: { bullet?: boolean }
|
||||
) {
|
||||
const normalizedLines = lines
|
||||
.map((line) => String(line || "").trim())
|
||||
.filter(Boolean);
|
||||
if (normalizedLines.length === 0) {
|
||||
return "";
|
||||
}
|
||||
|
||||
const sectionBody = options?.bullet
|
||||
? normalizedLines.map((line) => (line.startsWith("- ") ? line : `- ${line}`)).join("\n")
|
||||
: normalizedLines.join("\n");
|
||||
|
||||
return `【${title}】\n${sectionBody}`;
|
||||
}
|
||||
|
||||
function buildOptimizedAgentSceneWorkflowPrompt(params: {
|
||||
workflowPrompt: string;
|
||||
optimizationResult?: {
|
||||
sceneGoal: string;
|
||||
environmentSummary: string;
|
||||
propSummary: string;
|
||||
cameraLightingSummary: string;
|
||||
characterSummary: string;
|
||||
negativeConstraints: string[];
|
||||
conflicts: string[];
|
||||
} | null;
|
||||
}) {
|
||||
const normalizedWorkflowPrompt = normalizeAgentWorkflowPrompt(params.workflowPrompt);
|
||||
if (!normalizedWorkflowPrompt) {
|
||||
return "";
|
||||
}
|
||||
|
||||
const optimizationResult = params.optimizationResult || null;
|
||||
if (!optimizationResult) {
|
||||
return normalizedWorkflowPrompt;
|
||||
}
|
||||
|
||||
const sectionMap = parseAgentWorkflowPromptSections(normalizedWorkflowPrompt);
|
||||
const sceneGoalLines = [optimizationResult.sceneGoal, optimizationResult.environmentSummary].filter(
|
||||
Boolean
|
||||
);
|
||||
const propLines = [optimizationResult.propSummary].filter(Boolean);
|
||||
const cameraLightingLines = [optimizationResult.cameraLightingSummary].filter(Boolean);
|
||||
const characterLines = [optimizationResult.characterSummary].filter(Boolean);
|
||||
const negativeLines = Array.from(
|
||||
new Set([
|
||||
...optimizationResult.negativeConstraints,
|
||||
...optimizationResult.conflicts.map((item) => `冲突校正:${item}`),
|
||||
])
|
||||
);
|
||||
|
||||
const optimizedSections = [
|
||||
buildAgentPromptSectionText("项目背景", sectionMap.get("项目背景") || sectionMap.get("__root__") || []),
|
||||
buildAgentPromptSectionText("主SKU锁定", sectionMap.get("主SKU锁定") || []),
|
||||
buildAgentPromptSectionText("搭配SKU锁定", sectionMap.get("搭配SKU锁定") || []),
|
||||
buildAgentPromptSectionText("参考图", sectionMap.get("参考图") || []),
|
||||
buildAgentPromptSectionText("场景创作总纲", sceneGoalLines),
|
||||
buildAgentPromptSectionText("陈列与道具", propLines),
|
||||
buildAgentPromptSectionText("机位与光线", cameraLightingLines),
|
||||
buildAgentPromptSectionText("人物规则", characterLines),
|
||||
buildAgentPromptSectionText("负面约束", negativeLines, { bullet: true }),
|
||||
].filter(Boolean);
|
||||
|
||||
return optimizedSections.join("\n\n").trim();
|
||||
}
|
||||
|
||||
const AGENT_PROMPT_STRUCTURAL_LABELS = new Set([
|
||||
"角色",
|
||||
"技能",
|
||||
@ -3192,15 +3326,32 @@ function buildAgentSceneWorkflowPrompt(params: {
|
||||
imageRolePrompt?: string | null;
|
||||
}) {
|
||||
const lines: string[] = [];
|
||||
const workflowPrompt = String(params.workflowPrompt || "").trim();
|
||||
const workflowPrompt = normalizeAgentWorkflowPrompt(params.workflowPrompt || "");
|
||||
if (workflowPrompt) {
|
||||
lines.push(workflowPrompt);
|
||||
}
|
||||
const slotPrompt = String(params.slotPrompt || "").trim();
|
||||
const slotPrompt = String(params.slotPrompt || "")
|
||||
.split(/\n+/)
|
||||
.map((line) => line.replace(/\s+/g, " ").trim())
|
||||
.filter(Boolean)
|
||||
.join("\n")
|
||||
.trim();
|
||||
if (slotPrompt) {
|
||||
lines.push(slotPrompt);
|
||||
}
|
||||
const imageRolePrompt = String(params.imageRolePrompt || "").trim();
|
||||
let imageRolePrompt = String(params.imageRolePrompt || "").trim();
|
||||
if (workflowPrompt.includes("品牌模特图:已选") || workflowPrompt.includes("固定模特图")) {
|
||||
imageRolePrompt = imageRolePrompt
|
||||
.replace(
|
||||
/^如当前请求未附带固定模特图,仅在工作流明确要求人物时再安排辅助人物,且人物不能喧宾夺主。$/m,
|
||||
""
|
||||
)
|
||||
.replace(/^当前没有固定模特图,不要凭空生成强人物主体。$/m, "")
|
||||
.split(/\n+/)
|
||||
.map((line) => line.trim())
|
||||
.filter(Boolean)
|
||||
.join("\n");
|
||||
}
|
||||
if (imageRolePrompt) {
|
||||
lines.push(imageRolePrompt);
|
||||
}
|
||||
@ -7309,6 +7460,12 @@ export async function POST(request: Request) {
|
||||
bundleProductImageCount: sceneUploadedImageBreakdown.bundleProductImageCount,
|
||||
includeModelImage: sceneUploadedImageBreakdown.includeModelImage,
|
||||
});
|
||||
const sceneOptimizationImageUrls = resolveScenePromptOptimizationImageUrls({
|
||||
uploadedImageParts,
|
||||
primaryProductImageCount: sceneUploadedImageBreakdown.primaryProductImageCount,
|
||||
bundleProductImageCount: sceneUploadedImageBreakdown.bundleProductImageCount,
|
||||
includeModelImage: sceneUploadedImageBreakdown.includeModelImage,
|
||||
});
|
||||
const shouldAppendSceneSlotPrompt = !hiddenPromptText;
|
||||
const rawSceneSlotPrompt = shouldAppendSceneSlotPrompt
|
||||
? sceneSlot?.promptTemplate || selectedAgentPreset.promptTemplate || ""
|
||||
@ -7320,10 +7477,42 @@ export async function POST(request: Request) {
|
||||
modeType: "scene_generation",
|
||||
})
|
||||
: "";
|
||||
const optimizedSceneWorkflowResult =
|
||||
hiddenPromptText && sceneOptimizationImageUrls.primaryImageUrl
|
||||
? await optimizeAgentSceneWorkflowPrompt({
|
||||
workflowPrompt: hiddenPromptText,
|
||||
slotPrompt: dedupedSceneSlotPrompt || null,
|
||||
primaryImageUrl: sceneOptimizationImageUrls.primaryImageUrl,
|
||||
modelImageUrl: sceneOptimizationImageUrls.modelImageUrl,
|
||||
}).catch((error) => {
|
||||
console.warn("[Chat API] Agent scene workflow prompt optimization failed:", error);
|
||||
return null;
|
||||
})
|
||||
: null;
|
||||
const optimizedSceneWorkflowPrompt = hiddenPromptText
|
||||
? buildOptimizedAgentSceneWorkflowPrompt({
|
||||
workflowPrompt: hiddenPromptText,
|
||||
optimizationResult: optimizedSceneWorkflowResult,
|
||||
})
|
||||
: "";
|
||||
if (optimizedSceneWorkflowResult) {
|
||||
console.info("[Chat API] Agent scene workflow prompt optimized:", {
|
||||
chatId: id,
|
||||
agentPresetId: selectedAgentPreset.id,
|
||||
optimizationModelName: optimizedSceneWorkflowResult.optimizationModelName || null,
|
||||
optimizationSource: optimizedSceneWorkflowResult.optimizationSource || null,
|
||||
sceneGoal: optimizedSceneWorkflowResult.sceneGoal,
|
||||
environmentSummary: optimizedSceneWorkflowResult.environmentSummary,
|
||||
cameraLightingSummary: optimizedSceneWorkflowResult.cameraLightingSummary,
|
||||
characterSummary: optimizedSceneWorkflowResult.characterSummary,
|
||||
negativeConstraints: optimizedSceneWorkflowResult.negativeConstraints,
|
||||
conflicts: optimizedSceneWorkflowResult.conflicts,
|
||||
});
|
||||
}
|
||||
const scenePrompt =
|
||||
buildAgentSceneWorkflowPrompt({
|
||||
workflowPrompt: hiddenPromptText || null,
|
||||
slotPrompt: dedupedSceneSlotPrompt || null,
|
||||
workflowPrompt: optimizedSceneWorkflowPrompt || hiddenPromptText || null,
|
||||
slotPrompt: optimizedSceneWorkflowPrompt ? null : dedupedSceneSlotPrompt || null,
|
||||
imageRolePrompt: sceneImageRolePrompt,
|
||||
userSupplementPrompt: sanitizedUserSupplementPrompt || null,
|
||||
}) ||
|
||||
|
||||
@ -26,83 +26,54 @@ export function buildAgentSceneImageRolePrompt(params: {
|
||||
const bundleStartIndex = primaryProductImageCount;
|
||||
const modelImageIndex = primaryProductImageCount + bundleProductImageCount;
|
||||
|
||||
for (let index = 0; index < uploadedImageCount; index += 1) {
|
||||
if (index < primaryProductImageCount) {
|
||||
lines.push(
|
||||
`图片${index + 1}:主SKU白底参考图,用于锁定主产品的外观、比例、材质、颜色、结构和细节。`
|
||||
);
|
||||
continue;
|
||||
const formatRangeLabel = (startIndex: number, count: number) => {
|
||||
if (count <= 0) {
|
||||
return "";
|
||||
}
|
||||
|
||||
if (index >= bundleStartIndex && index < bundleStartIndex + bundleProductImageCount) {
|
||||
lines.push(
|
||||
`图片${index + 1}:搭配SKU白底参考图,用于锁定搭配产品本身的外观与结构。它是独立的第二产品,只能用于组合展示、场景搭配或补充关联销售,不能替代主SKU,也不能和主SKU融合成一件产品。`
|
||||
);
|
||||
continue;
|
||||
if (count === 1) {
|
||||
return `图片${startIndex + 1}`;
|
||||
}
|
||||
|
||||
if (hasModelImage && index === modelImageIndex) {
|
||||
lines.push(
|
||||
`图片${index + 1}:固定模特图。若画面需要人物,必须复用这张图中的同一模特(脸型、发型、体型、肤色、穿搭风格保持一致),不得替换成其他人物。`
|
||||
);
|
||||
continue;
|
||||
}
|
||||
return `图片${startIndex + 1}到图片${startIndex + count}`;
|
||||
};
|
||||
|
||||
if (hasPrimaryProductImage) {
|
||||
lines.push(
|
||||
`图片${index + 1}:补充参考图,只用于补充产品角度、细节或辅助信息,不可替代主产品外观。`
|
||||
`${formatRangeLabel(0, primaryProductImageCount)}:主SKU白底参考图,用于锁定主产品身份、外观、比例、材质、颜色、结构和细节。`
|
||||
);
|
||||
}
|
||||
|
||||
if (hasBundleProductImage) {
|
||||
lines.push(
|
||||
`${formatRangeLabel(bundleStartIndex, bundleProductImageCount)}:搭配SKU白底参考图,用于锁定搭配产品本身的外观与结构。搭配SKU是独立第二产品,只能用于组合展示,不能替代或融合主SKU。`
|
||||
);
|
||||
}
|
||||
|
||||
if (hasModelImage) {
|
||||
lines.push(
|
||||
`图片${modelImageIndex + 1}:固定模特图。若画面需要人物,必须复用同一模特,不得替换成其他人物。`
|
||||
);
|
||||
}
|
||||
|
||||
if (hasPrimaryProductImage) {
|
||||
lines.push(
|
||||
`必须以图片1${
|
||||
primaryProductImageCount > 1 ? `到图片${primaryProductImageCount}` : ""
|
||||
}中的主SKU信息为最高优先级,严格保持主产品身份不变。`
|
||||
);
|
||||
lines.push(
|
||||
"【产品锁定约束】生成结果中的主SKU必须与主SKU白底参考图保持同一产品身份,禁止改成另一款产品。"
|
||||
);
|
||||
lines.push(
|
||||
"严禁改变主SKU的主体结构、外轮廓、长宽高比例、层板数量与位置、抽屉数量与位置、柜门/开孔/隔断布局、弧度造型、柜腿样式、把手/五金位置。"
|
||||
);
|
||||
lines.push(
|
||||
"严禁改变主SKU主色、木纹方向、材质类型、表面纹理、明暗层次和细节做工;不得把深色木纹改成浅色、白色、金属色或其他颜色体系。"
|
||||
);
|
||||
lines.push(
|
||||
"如果场景风格与主SKU外观产生冲突,必须优先保留主产品本体不变,只允许调整周边场景、软装、灯光、背景和道具,不允许为了适配风格而修改主产品。"
|
||||
);
|
||||
lines.push(
|
||||
"主SKU白底图是主产品身份锁定图,不是灵感参考图。输出时必须让人一眼看出是同一个主产品,只能换场景,不能换主产品本体。"
|
||||
"主SKU白底图优先级最高。只能换场景,不可换产品本体;必须保持同一产品身份、结构、比例、材质与主色。"
|
||||
);
|
||||
}
|
||||
if (hasBundleProductImage) {
|
||||
lines.push(
|
||||
"【搭配产品约束】搭配SKU是独立的第二产品。需要搭配展示时,结果中必须同时出现主SKU与搭配SKU,清楚表现组合关系与使用场景。"
|
||||
);
|
||||
lines.push(
|
||||
"严禁把搭配SKU当成主SKU的另一视角、局部细节或替代品;也严禁把主SKU与搭配SKU融合、拼接、嫁接成单一产品。"
|
||||
);
|
||||
lines.push(
|
||||
"如果当前场景不适合展示搭配SKU,可以弱化其占比,但不得因此篡改、替换或覆盖主SKU。"
|
||||
"如需展示搭配SKU,结果中必须同时保留主SKU与搭配SKU的独立身份,不得把搭配SKU当成主SKU的另一视角或替代品。"
|
||||
);
|
||||
}
|
||||
lines.push(
|
||||
hasModelImage
|
||||
? `如果当前场景有人物展示需求,只允许使用图片${modelImageIndex + 1}里的模特;如果不需要人物,则不要强行加入人物。`
|
||||
: "当前没有固定模特图,不要凭空生成强人物主体。"
|
||||
? `如当前场景需要人物,只允许使用图片${modelImageIndex + 1}里的模特;如果不需要人物,则不要强行加入人物。`
|
||||
: "如当前请求未附带固定模特图,仅在工作流明确要求人物时再安排辅助人物,且人物不能喧宾夺主。"
|
||||
);
|
||||
if (hasModelImage) {
|
||||
lines.push(
|
||||
"【模特锁定约束】固定模特图是人物身份锁定图。结果中出现的人物必须与模特图为同一人,不得换脸、换人、换性别、换年龄段,不得改变五官特征、脸型、发际线、发型主体、肤色、身材和辨识度。"
|
||||
);
|
||||
lines.push(
|
||||
"允许根据场景构图调整模特的站姿、坐姿、手势、视线和轻微表情,但必须让人一眼看出是同一个模特。"
|
||||
);
|
||||
lines.push(
|
||||
"只有当工作流设定或补充要求里明确要求修改模特衣服、穿搭或服装颜色时,才允许更换或调整服饰;如果没有明确要求,默认保持模特原有服装风格和穿搭关系一致。"
|
||||
);
|
||||
lines.push(
|
||||
"如果模特衣服被允许调整,也只能改衣服,不得借机改变模特本人;人物身份稳定性始终高于服装变化。"
|
||||
"固定模特图是人物身份锁定图。允许根据构图调整姿态与视线,但不得换人、换脸或改变人物主体辨识度。"
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
266
lib/agents/scene-workflow-prompt-optimizer.ts
Normal file
266
lib/agents/scene-workflow-prompt-optimizer.ts
Normal file
@ -0,0 +1,266 @@
|
||||
import "server-only";
|
||||
|
||||
import { generateText } from "ai";
|
||||
import { getLanguageModelById } from "@/lib/ai/dynamic-provider";
|
||||
import { callQwen3Vl } from "@/lib/ai/qwen3-vl";
|
||||
import { getEnabledAiModelsWithProvider } from "@/lib/db/queries";
|
||||
|
||||
const QWEN_VL_MODEL_MATCHER =
|
||||
/(qwen|千问|通义).*(vl|vision)|qwen3-vl|qwen-vl/i;
|
||||
|
||||
export interface AgentSceneWorkflowOptimizationResult {
|
||||
sceneGoal: string;
|
||||
environmentSummary: string;
|
||||
propSummary: string;
|
||||
cameraLightingSummary: string;
|
||||
characterSummary: string;
|
||||
negativeConstraints: string[];
|
||||
conflicts: string[];
|
||||
optimizationModelName?: string | null;
|
||||
optimizationSource?: "configured_qwen3_vl" | "fallback_qwen3_vl" | "disabled";
|
||||
}
|
||||
|
||||
function sanitizeShortText(value: unknown, maxLength: number) {
|
||||
return String(value || "").replace(/\s+/g, " ").trim().slice(0, maxLength);
|
||||
}
|
||||
|
||||
function sanitizeStringList(value: unknown, maxItems: number, maxLength: number) {
|
||||
const items = Array.isArray(value) ? value : [];
|
||||
return Array.from(
|
||||
new Set(
|
||||
items
|
||||
.map((item) => sanitizeShortText(item, maxLength))
|
||||
.filter(Boolean)
|
||||
.slice(0, maxItems)
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
function extractJsonObjectText(text: string): string | null {
|
||||
const normalized = String(text || "").trim();
|
||||
if (!normalized) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const fencedMatch = normalized.match(/```(?:json)?\s*([\s\S]*?)```/i);
|
||||
if (fencedMatch?.[1]) {
|
||||
const fencedText = fencedMatch[1].trim();
|
||||
if (fencedText.startsWith("{") && fencedText.endsWith("}")) {
|
||||
return fencedText;
|
||||
}
|
||||
}
|
||||
|
||||
if (normalized.startsWith("{") && normalized.endsWith("}")) {
|
||||
return normalized;
|
||||
}
|
||||
|
||||
const firstBraceIndex = normalized.indexOf("{");
|
||||
const lastBraceIndex = normalized.lastIndexOf("}");
|
||||
if (firstBraceIndex >= 0 && lastBraceIndex > firstBraceIndex) {
|
||||
return normalized.slice(firstBraceIndex, lastBraceIndex + 1);
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
function parseOptimizationResult(
|
||||
text: string
|
||||
): Omit<AgentSceneWorkflowOptimizationResult, "optimizationModelName" | "optimizationSource"> | null {
|
||||
const jsonText = extractJsonObjectText(text);
|
||||
if (!jsonText) {
|
||||
return null;
|
||||
}
|
||||
|
||||
try {
|
||||
const parsed = JSON.parse(jsonText) as Record<string, unknown>;
|
||||
const result = {
|
||||
sceneGoal: sanitizeShortText(parsed.sceneGoal, 60),
|
||||
environmentSummary: sanitizeShortText(parsed.environmentSummary, 120),
|
||||
propSummary: sanitizeShortText(parsed.propSummary, 120),
|
||||
cameraLightingSummary: sanitizeShortText(parsed.cameraLightingSummary, 80),
|
||||
characterSummary: sanitizeShortText(parsed.characterSummary, 80),
|
||||
negativeConstraints: sanitizeStringList(parsed.negativeConstraints, 6, 26),
|
||||
conflicts: sanitizeStringList(parsed.conflicts, 4, 50),
|
||||
};
|
||||
|
||||
if (
|
||||
!result.sceneGoal &&
|
||||
!result.environmentSummary &&
|
||||
!result.propSummary &&
|
||||
!result.cameraLightingSummary &&
|
||||
!result.characterSummary &&
|
||||
result.negativeConstraints.length === 0 &&
|
||||
result.conflicts.length === 0
|
||||
) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return result;
|
||||
} catch (error) {
|
||||
console.warn(
|
||||
"[Agent Scene Workflow Prompt Optimizer] Failed to parse JSON response:",
|
||||
error
|
||||
);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function buildOptimizerPrompt(params: {
|
||||
workflowPrompt: string;
|
||||
slotPrompt?: string | null;
|
||||
}) {
|
||||
return [
|
||||
"你是电商家具场景生图提示词整理器。",
|
||||
"你的任务是把冗长的工作流选项提示词,压缩成更适合 Gemini / Imagen 生图模型执行的结构化摘要。",
|
||||
"你只能做压缩、去重、冲突发现,不能改写主SKU产品身份,不能新增工作流里没有的产品结构、颜色、材质或空间元素。",
|
||||
"不要重复角色、技能、流程、请按选项执行这类管理话术。",
|
||||
"如果有人物要求,只总结人物身份、站位和作用;人物不能压过产品主体。如果没有明确人物要求,characterSummary 返回空字符串。",
|
||||
"negativeConstraints 只保留真正影响出图稳定性的禁止项,例如变形、换色、喧宾夺主、额外主体、结构错误。",
|
||||
"conflicts 只写真正的提示冲突;没有冲突返回空数组。",
|
||||
'只输出 JSON:{"sceneGoal":"","environmentSummary":"","propSummary":"","cameraLightingSummary":"","characterSummary":"","negativeConstraints":[],"conflicts":[]}',
|
||||
"字段要求:sceneGoal 不超过60字;environmentSummary 不超过120字;propSummary 不超过120字;cameraLightingSummary 不超过80字;characterSummary 不超过80字;negativeConstraints 最多6条,每条不超过26字;conflicts 最多4条,每条不超过50字。",
|
||||
`工作流设定:\n${params.workflowPrompt}`,
|
||||
params.slotPrompt ? `模式模板补充:\n${params.slotPrompt}` : "",
|
||||
]
|
||||
.filter(Boolean)
|
||||
.join("\n\n");
|
||||
}
|
||||
|
||||
async function optimizeWithConfiguredQwenVisionModel(params: {
|
||||
prompt: string;
|
||||
primaryImageUrl: string;
|
||||
modelImageUrl?: string | null;
|
||||
}) {
|
||||
const configuredModels = await getEnabledAiModelsWithProvider().catch((error) => {
|
||||
console.warn(
|
||||
"[Agent Scene Workflow Prompt Optimizer] Failed to load configured models:",
|
||||
error
|
||||
);
|
||||
return [];
|
||||
});
|
||||
|
||||
const configuredQwenVisionModel = configuredModels.find(
|
||||
({ model }) =>
|
||||
model.supportsVision &&
|
||||
!model.isSubModel &&
|
||||
QWEN_VL_MODEL_MATCHER.test(`${model.displayName} ${model.modelId}`)
|
||||
);
|
||||
|
||||
if (!configuredQwenVisionModel) {
|
||||
return null;
|
||||
}
|
||||
|
||||
try {
|
||||
const languageModel = await getLanguageModelById(
|
||||
configuredQwenVisionModel.model.id
|
||||
);
|
||||
if (!languageModel) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const content: Array<{ type: "text" | "image"; text?: string; image?: string }> = [
|
||||
{
|
||||
type: "text",
|
||||
text: params.prompt,
|
||||
},
|
||||
{
|
||||
type: "image",
|
||||
image: params.primaryImageUrl,
|
||||
},
|
||||
];
|
||||
|
||||
if (params.modelImageUrl) {
|
||||
content.push({
|
||||
type: "image",
|
||||
image: params.modelImageUrl,
|
||||
});
|
||||
}
|
||||
|
||||
const { text } = await generateText({
|
||||
model: languageModel,
|
||||
messages: [
|
||||
{
|
||||
role: "user",
|
||||
content,
|
||||
},
|
||||
],
|
||||
temperature: 0,
|
||||
maxTokens: 320,
|
||||
abortSignal: AbortSignal.timeout(30_000),
|
||||
} as any);
|
||||
|
||||
const parsed = parseOptimizationResult(text);
|
||||
if (!parsed) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return {
|
||||
...parsed,
|
||||
optimizationModelName:
|
||||
configuredQwenVisionModel.model.displayName ||
|
||||
configuredQwenVisionModel.model.modelId,
|
||||
optimizationSource: "configured_qwen3_vl" as const,
|
||||
};
|
||||
} catch (error) {
|
||||
console.warn(
|
||||
"[Agent Scene Workflow Prompt Optimizer] Configured Qwen-VL optimization failed:",
|
||||
error
|
||||
);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
export async function optimizeAgentSceneWorkflowPrompt(params: {
|
||||
workflowPrompt: string;
|
||||
slotPrompt?: string | null;
|
||||
primaryImageUrl: string;
|
||||
modelImageUrl?: string | null;
|
||||
}): Promise<AgentSceneWorkflowOptimizationResult | null> {
|
||||
const workflowPrompt = String(params.workflowPrompt || "").trim();
|
||||
const primaryImageUrl = String(params.primaryImageUrl || "").trim();
|
||||
|
||||
if (!workflowPrompt || !/^https?:\/\//i.test(primaryImageUrl)) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const prompt = buildOptimizerPrompt({
|
||||
workflowPrompt,
|
||||
slotPrompt: params.slotPrompt || null,
|
||||
});
|
||||
|
||||
const configuredResult = await optimizeWithConfiguredQwenVisionModel({
|
||||
prompt,
|
||||
primaryImageUrl,
|
||||
modelImageUrl: params.modelImageUrl || null,
|
||||
});
|
||||
if (configuredResult) {
|
||||
return configuredResult;
|
||||
}
|
||||
|
||||
try {
|
||||
const fallbackResult = await callQwen3Vl({
|
||||
imageUrl: primaryImageUrl,
|
||||
prompt,
|
||||
temperature: 0,
|
||||
maxTokens: 320,
|
||||
timeoutMs: 30_000,
|
||||
retries: 1,
|
||||
});
|
||||
const parsed = parseOptimizationResult(fallbackResult.text);
|
||||
if (!parsed) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return {
|
||||
...parsed,
|
||||
optimizationModelName: fallbackResult.model,
|
||||
optimizationSource: "fallback_qwen3_vl",
|
||||
};
|
||||
} catch (error) {
|
||||
console.warn(
|
||||
"[Agent Scene Workflow Prompt Optimizer] Fallback Qwen3-VL optimization failed:",
|
||||
error
|
||||
);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
@ -26,6 +26,21 @@ export interface AgentWorkflowInputsValue {
|
||||
priceBand: string;
|
||||
}
|
||||
|
||||
const WORKFLOW_FRAGMENT_INLINE_LABELS = [
|
||||
"角色",
|
||||
"技能",
|
||||
"目标",
|
||||
"约束条件",
|
||||
"流程",
|
||||
"输出",
|
||||
"前景",
|
||||
"中景",
|
||||
"背景",
|
||||
"主产品",
|
||||
"台面摆件",
|
||||
"模特",
|
||||
];
|
||||
|
||||
function buildGoodsInfoPromptLine(params: {
|
||||
label: string;
|
||||
result?: AgentWorkflowMaterialSelections["primary"]["result"] | null;
|
||||
@ -39,8 +54,6 @@ function buildGoodsInfoPromptLine(params: {
|
||||
goodsInfo.productName ? `商品名称:${goodsInfo.productName}` : "",
|
||||
goodsInfo.sizeInfo ? `尺寸信息:${goodsInfo.sizeInfo}` : "",
|
||||
goodsInfo.materialInfo ? `材质信息:${goodsInfo.materialInfo}` : "",
|
||||
goodsInfo.designer ? `设计师:${goodsInfo.designer}` : "",
|
||||
goodsInfo.buyer ? `采购负责人:${goodsInfo.buyer}` : "",
|
||||
goodsInfo.boardType ? `板材类型:${goodsInfo.boardType}` : "",
|
||||
].filter(Boolean);
|
||||
|
||||
@ -61,7 +74,7 @@ function buildGoodsInfoSizeConstraintLine(params: {
|
||||
return "";
|
||||
}
|
||||
|
||||
return `${params.label}尺寸比例约束:尺寸信息为 ${sizeInfo}。生成场景时必须依据该尺寸信息判断产品在空间中的真实体量、长宽高关系和与周边家具/人物/通道的比例,不得随意放大、缩小或改变产品结构比例。`;
|
||||
return `${params.label}尺寸比例:${sizeInfo}。场景中必须保持真实体量、长宽高关系与产品结构比例,不得随意拉伸、压缩或失真。`;
|
||||
}
|
||||
|
||||
function buildGoodsInfoAppearanceConstraintLine(params: {
|
||||
@ -79,7 +92,7 @@ function buildGoodsInfoAppearanceConstraintLine(params: {
|
||||
return "";
|
||||
}
|
||||
|
||||
return `${params.label}外观结构与颜色约束:首张SKU图识别结果为“${appearanceSummary}”。后续生成必须严格保持该产品的外观轮廓、主体结构、层板/格口/门板/抽屉/支撑件等数量关系和主副色,不得擅自改款、增减结构件、改变开合方式或替换主体颜色。`;
|
||||
return `${params.label}外观锁定:${appearanceSummary}。必须保持同一产品身份,不得改款、增减结构件、改变开合方式或替换主体颜色。`;
|
||||
}
|
||||
|
||||
function buildGoodsInfoSummaryLine(params: {
|
||||
@ -114,7 +127,7 @@ function buildBundleUsagePromptLine(params: {
|
||||
return "";
|
||||
}
|
||||
|
||||
return "搭配SKU使用规则:搭配SKU是独立的第二产品,不是主SKU的另一角度或替代图。需要做搭配展示时,必须同场景同时展示主SKU与搭配SKU(例如桌子+椅子),保持两者各自的外观、比例和结构,不得把两件产品融合成一件,也不得用搭配SKU替换主SKU。";
|
||||
return "搭配SKU使用规则:搭配SKU是独立的第二产品,不是主SKU的另一角度或替代图。需要做搭配展示时,必须同场景同时呈现两者,且不得融合、替换或混淆产品身份。";
|
||||
}
|
||||
|
||||
function normalizeWorkflowText(value: string) {
|
||||
@ -123,6 +136,48 @@ function normalizeWorkflowText(value: string) {
|
||||
.trim();
|
||||
}
|
||||
|
||||
function sanitizeWorkflowPromptFragment(value: string) {
|
||||
return normalizeWorkflowText(value)
|
||||
.replace(/请按选项[\d一二三四五六七八九十]+执行[。.]?/g, "")
|
||||
.replace(/流程\s*严格遵循下列流程/gi, "")
|
||||
.replace(
|
||||
/输出\s*([0-9]+\s*[kK])\s*([0-9]+\s*[::]\s*[0-9]+)/g,
|
||||
(_, quality: string, ratio: string) =>
|
||||
`输出:${String(quality || "").toUpperCase()},${String(ratio || "")
|
||||
.replace(/\s+/g, "")
|
||||
.replace(":", ":")}`
|
||||
)
|
||||
.replace(/\s+/g, " ")
|
||||
.replace(/^[;;、,,]+|[;;、,,]+$/g, "")
|
||||
.trim();
|
||||
}
|
||||
|
||||
function splitWorkflowPromptFragment(value: string) {
|
||||
const sanitized = sanitizeWorkflowPromptFragment(value);
|
||||
if (!sanitized) {
|
||||
return [] as string[];
|
||||
}
|
||||
|
||||
let normalized = sanitized;
|
||||
WORKFLOW_FRAGMENT_INLINE_LABELS.forEach((label) => {
|
||||
normalized = normalized.replace(
|
||||
new RegExp(`(?<!^)(?:${label})[::]`, "g"),
|
||||
`\n${label}:`
|
||||
);
|
||||
});
|
||||
|
||||
return normalized
|
||||
.split(/\n+/)
|
||||
.flatMap((line) =>
|
||||
line
|
||||
.split(/;+/)
|
||||
.map((item) => item.trim())
|
||||
.filter(Boolean)
|
||||
)
|
||||
.map((item) => item.replace(/\s+/g, " ").trim())
|
||||
.filter(Boolean);
|
||||
}
|
||||
|
||||
function buildGroupedSelectionLines(params: {
|
||||
selections: AgentWorkflowComposerSelection[];
|
||||
valueKey: "optionName" | "promptFragment";
|
||||
@ -184,6 +239,64 @@ function buildGroupedSelectionLines(params: {
|
||||
.map((group) => `${group.stepTitle}:${group.values.join(params.separator)}`);
|
||||
}
|
||||
|
||||
function buildStructuredSelectionPromptSections(params: {
|
||||
selections: AgentWorkflowComposerSelection[];
|
||||
}) {
|
||||
const groups = new Map<
|
||||
string,
|
||||
{
|
||||
title: string;
|
||||
values: string[];
|
||||
valueSet: Set<string>;
|
||||
}
|
||||
>();
|
||||
const globalValueSet = new Set<string>();
|
||||
|
||||
params.selections.forEach((selection) => {
|
||||
const fragments = splitWorkflowPromptFragment(
|
||||
selection.promptFragment || selection.optionName || ""
|
||||
);
|
||||
if (fragments.length === 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
const currentGroup = groups.get(selection.stepKey) || {
|
||||
title: selection.stepTitle,
|
||||
values: [],
|
||||
valueSet: new Set<string>(),
|
||||
};
|
||||
|
||||
fragments.forEach((fragment) => {
|
||||
const dedupeKey = fragment.toLowerCase();
|
||||
if (currentGroup.valueSet.has(dedupeKey) || globalValueSet.has(dedupeKey)) {
|
||||
return;
|
||||
}
|
||||
currentGroup.valueSet.add(dedupeKey);
|
||||
globalValueSet.add(dedupeKey);
|
||||
currentGroup.values.push(fragment);
|
||||
});
|
||||
|
||||
if (currentGroup.values.length > 0) {
|
||||
groups.set(selection.stepKey, currentGroup);
|
||||
}
|
||||
});
|
||||
|
||||
return Array.from(groups.values())
|
||||
.filter((group) => group.values.length > 0)
|
||||
.map((group) => `【${group.title}】\n${group.values.map((value) => `- ${value}`).join("\n")}`);
|
||||
}
|
||||
|
||||
function buildPromptSection(title: string, lines: string[]) {
|
||||
const normalizedLines = lines
|
||||
.map((line) => String(line || "").trim())
|
||||
.filter(Boolean);
|
||||
if (normalizedLines.length === 0) {
|
||||
return "";
|
||||
}
|
||||
|
||||
return `【${title}】\n${normalizedLines.join("\n")}`;
|
||||
}
|
||||
|
||||
export function buildAgentWorkflowComposerContext(params: {
|
||||
config?: AgentWorkflowUiConfig | null;
|
||||
inputs: AgentWorkflowInputsValue;
|
||||
@ -286,80 +399,86 @@ export function buildAgentWorkflowComposerContext(params: {
|
||||
return null;
|
||||
}
|
||||
|
||||
const promptLines: string[] = [];
|
||||
const basicContextLines: string[] = [];
|
||||
if (params.inputs.primarySku.trim()) {
|
||||
promptLines.push(`主SKU:${params.inputs.primarySku.trim()}`);
|
||||
basicContextLines.push(`主SKU:${params.inputs.primarySku.trim()}`);
|
||||
}
|
||||
if (params.inputs.secondarySku.trim()) {
|
||||
promptLines.push(`搭配SKU:${params.inputs.secondarySku.trim()}`);
|
||||
basicContextLines.push(`搭配SKU:${params.inputs.secondarySku.trim()}`);
|
||||
}
|
||||
if (params.inputs.brand.trim()) {
|
||||
promptLines.push(`品牌:${params.inputs.brand.trim()}`);
|
||||
basicContextLines.push(`品牌:${params.inputs.brand.trim()}`);
|
||||
}
|
||||
if (params.inputs.store.trim()) {
|
||||
promptLines.push(`店铺:${params.inputs.store.trim()}`);
|
||||
basicContextLines.push(`店铺:${params.inputs.store.trim()}`);
|
||||
}
|
||||
if (params.inputs.thirdCategory.trim()) {
|
||||
promptLines.push(`三级分类:${params.inputs.thirdCategory.trim()}`);
|
||||
basicContextLines.push(`三级分类:${params.inputs.thirdCategory.trim()}`);
|
||||
}
|
||||
if (params.inputs.fourthCategory.trim()) {
|
||||
promptLines.push(`四级分类:${params.inputs.fourthCategory.trim()}`);
|
||||
basicContextLines.push(`四级分类:${params.inputs.fourthCategory.trim()}`);
|
||||
}
|
||||
if (params.inputs.fifthCategory.trim()) {
|
||||
promptLines.push(`五级分类:${params.inputs.fifthCategory.trim()}`);
|
||||
basicContextLines.push(`五级分类:${params.inputs.fifthCategory.trim()}`);
|
||||
}
|
||||
if (params.inputs.priceBand.trim()) {
|
||||
promptLines.push(`价格带:${params.inputs.priceBand.trim()}`);
|
||||
basicContextLines.push(`价格带:${params.inputs.priceBand.trim()}`);
|
||||
}
|
||||
|
||||
const primaryLockLines: string[] = [];
|
||||
const primaryGoodsInfoPrompt = buildGoodsInfoPromptLine({
|
||||
label: "主SKU商品信息",
|
||||
result: params.materials?.primary?.result || null,
|
||||
});
|
||||
if (primaryGoodsInfoPrompt) {
|
||||
promptLines.push(primaryGoodsInfoPrompt);
|
||||
primaryLockLines.push(primaryGoodsInfoPrompt);
|
||||
}
|
||||
const primaryGoodsInfoSizeConstraint = buildGoodsInfoSizeConstraintLine({
|
||||
label: "主SKU",
|
||||
result: params.materials?.primary?.result || null,
|
||||
});
|
||||
if (primaryGoodsInfoSizeConstraint) {
|
||||
promptLines.push(primaryGoodsInfoSizeConstraint);
|
||||
primaryLockLines.push(primaryGoodsInfoSizeConstraint);
|
||||
}
|
||||
const primaryGoodsInfoAppearanceConstraint = buildGoodsInfoAppearanceConstraintLine({
|
||||
label: "主SKU",
|
||||
result: params.materials?.primary?.result || null,
|
||||
});
|
||||
if (primaryGoodsInfoAppearanceConstraint) {
|
||||
promptLines.push(primaryGoodsInfoAppearanceConstraint);
|
||||
primaryLockLines.push(primaryGoodsInfoAppearanceConstraint);
|
||||
}
|
||||
|
||||
const secondaryLockLines: string[] = [];
|
||||
const secondaryGoodsInfoPrompt = buildGoodsInfoPromptLine({
|
||||
label: "搭配SKU商品信息",
|
||||
result: params.materials?.secondary?.result || null,
|
||||
});
|
||||
if (secondaryGoodsInfoPrompt) {
|
||||
promptLines.push(secondaryGoodsInfoPrompt);
|
||||
secondaryLockLines.push(secondaryGoodsInfoPrompt);
|
||||
}
|
||||
const secondaryGoodsInfoSizeConstraint = buildGoodsInfoSizeConstraintLine({
|
||||
label: "搭配SKU",
|
||||
result: params.materials?.secondary?.result || null,
|
||||
});
|
||||
if (secondaryGoodsInfoSizeConstraint) {
|
||||
promptLines.push(secondaryGoodsInfoSizeConstraint);
|
||||
secondaryLockLines.push(secondaryGoodsInfoSizeConstraint);
|
||||
}
|
||||
const secondaryGoodsInfoAppearanceConstraint = buildGoodsInfoAppearanceConstraintLine({
|
||||
label: "搭配SKU",
|
||||
result: params.materials?.secondary?.result || null,
|
||||
});
|
||||
if (secondaryGoodsInfoAppearanceConstraint) {
|
||||
promptLines.push(secondaryGoodsInfoAppearanceConstraint);
|
||||
secondaryLockLines.push(secondaryGoodsInfoAppearanceConstraint);
|
||||
}
|
||||
|
||||
const referenceLines: string[] = [];
|
||||
if (referenceImages.some((item) => item.source === "primary")) {
|
||||
const labels = referenceImages
|
||||
.filter((item) => item.source === "primary")
|
||||
.map((item) => item.label)
|
||||
.filter(Boolean)
|
||||
.slice(0, 8);
|
||||
promptLines.push(
|
||||
referenceLines.push(
|
||||
`主SPU参考图:已选${referenceImages.filter((item) => item.source === "primary").length}张${
|
||||
labels.length > 0 ? `(${labels.join("、")})` : ""
|
||||
}`
|
||||
@ -371,7 +490,7 @@ export function buildAgentWorkflowComposerContext(params: {
|
||||
.map((item) => item.label)
|
||||
.filter(Boolean)
|
||||
.slice(0, 8);
|
||||
promptLines.push(
|
||||
referenceLines.push(
|
||||
`搭配SKU参考图:已选${referenceImages.filter((item) => item.source === "secondary").length}张${
|
||||
labels.length > 0 ? `(${labels.join("、")})` : ""
|
||||
}`
|
||||
@ -383,7 +502,7 @@ export function buildAgentWorkflowComposerContext(params: {
|
||||
.map((item) => item.label)
|
||||
.filter(Boolean)
|
||||
.slice(0, 8);
|
||||
promptLines.push(
|
||||
referenceLines.push(
|
||||
`品牌模特图:已选${referenceImages.filter((item) => item.source === "brandModel").length}张${
|
||||
labels.length > 0 ? `(${labels.join("、")})` : ""
|
||||
}`
|
||||
@ -391,16 +510,16 @@ export function buildAgentWorkflowComposerContext(params: {
|
||||
}
|
||||
const bundleUsagePrompt = buildBundleUsagePromptLine({ referenceImages });
|
||||
if (bundleUsagePrompt) {
|
||||
promptLines.push(bundleUsagePrompt);
|
||||
secondaryLockLines.push(bundleUsagePrompt);
|
||||
}
|
||||
promptLines.push(
|
||||
...buildGroupedSelectionLines({
|
||||
selections,
|
||||
valueKey: "promptFragment",
|
||||
separator: ";",
|
||||
dedupeAcrossSteps: true,
|
||||
})
|
||||
);
|
||||
|
||||
const promptSections = [
|
||||
buildPromptSection("项目背景", basicContextLines),
|
||||
buildPromptSection("主SKU锁定", primaryLockLines),
|
||||
buildPromptSection("搭配SKU锁定", secondaryLockLines),
|
||||
buildPromptSection("参考图", referenceLines),
|
||||
...buildStructuredSelectionPromptSections({ selections }),
|
||||
].filter(Boolean);
|
||||
|
||||
return {
|
||||
primarySku: params.inputs.primarySku.trim() || undefined,
|
||||
@ -414,7 +533,7 @@ export function buildAgentWorkflowComposerContext(params: {
|
||||
selections,
|
||||
referenceImages,
|
||||
summaryText: summaryParts.join(" / "),
|
||||
promptText: promptLines.join("\n"),
|
||||
promptText: promptSections.join("\n\n"),
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user