267 lines
8.0 KiB
TypeScript
267 lines
8.0 KiB
TypeScript
import "server-only";
|
||
|
||
import { generateText } from "ai";
|
||
import { getLanguageModelById } from "@/lib/ai/dynamic-provider";
|
||
import { callQwen3Vl } from "@/lib/ai/qwen3-vl";
|
||
import { getEnabledAiModelsWithProvider } from "@/lib/db/queries";
|
||
|
||
const QWEN_VL_MODEL_MATCHER =
|
||
/(qwen|千问|通义).*(vl|vision)|qwen3-vl|qwen-vl/i;
|
||
|
||
export interface AgentSceneWorkflowOptimizationResult {
|
||
sceneGoal: string;
|
||
environmentSummary: string;
|
||
propSummary: string;
|
||
cameraLightingSummary: string;
|
||
characterSummary: string;
|
||
negativeConstraints: string[];
|
||
conflicts: string[];
|
||
optimizationModelName?: string | null;
|
||
optimizationSource?: "configured_qwen3_vl" | "fallback_qwen3_vl" | "disabled";
|
||
}
|
||
|
||
function sanitizeShortText(value: unknown, maxLength: number) {
|
||
return String(value || "").replace(/\s+/g, " ").trim().slice(0, maxLength);
|
||
}
|
||
|
||
function sanitizeStringList(value: unknown, maxItems: number, maxLength: number) {
|
||
const items = Array.isArray(value) ? value : [];
|
||
return Array.from(
|
||
new Set(
|
||
items
|
||
.map((item) => sanitizeShortText(item, maxLength))
|
||
.filter(Boolean)
|
||
.slice(0, maxItems)
|
||
)
|
||
);
|
||
}
|
||
|
||
function extractJsonObjectText(text: string): string | null {
|
||
const normalized = String(text || "").trim();
|
||
if (!normalized) {
|
||
return null;
|
||
}
|
||
|
||
const fencedMatch = normalized.match(/```(?:json)?\s*([\s\S]*?)```/i);
|
||
if (fencedMatch?.[1]) {
|
||
const fencedText = fencedMatch[1].trim();
|
||
if (fencedText.startsWith("{") && fencedText.endsWith("}")) {
|
||
return fencedText;
|
||
}
|
||
}
|
||
|
||
if (normalized.startsWith("{") && normalized.endsWith("}")) {
|
||
return normalized;
|
||
}
|
||
|
||
const firstBraceIndex = normalized.indexOf("{");
|
||
const lastBraceIndex = normalized.lastIndexOf("}");
|
||
if (firstBraceIndex >= 0 && lastBraceIndex > firstBraceIndex) {
|
||
return normalized.slice(firstBraceIndex, lastBraceIndex + 1);
|
||
}
|
||
|
||
return null;
|
||
}
|
||
|
||
function parseOptimizationResult(
|
||
text: string
|
||
): Omit<AgentSceneWorkflowOptimizationResult, "optimizationModelName" | "optimizationSource"> | null {
|
||
const jsonText = extractJsonObjectText(text);
|
||
if (!jsonText) {
|
||
return null;
|
||
}
|
||
|
||
try {
|
||
const parsed = JSON.parse(jsonText) as Record<string, unknown>;
|
||
const result = {
|
||
sceneGoal: sanitizeShortText(parsed.sceneGoal, 60),
|
||
environmentSummary: sanitizeShortText(parsed.environmentSummary, 120),
|
||
propSummary: sanitizeShortText(parsed.propSummary, 120),
|
||
cameraLightingSummary: sanitizeShortText(parsed.cameraLightingSummary, 80),
|
||
characterSummary: sanitizeShortText(parsed.characterSummary, 80),
|
||
negativeConstraints: sanitizeStringList(parsed.negativeConstraints, 6, 26),
|
||
conflicts: sanitizeStringList(parsed.conflicts, 4, 50),
|
||
};
|
||
|
||
if (
|
||
!result.sceneGoal &&
|
||
!result.environmentSummary &&
|
||
!result.propSummary &&
|
||
!result.cameraLightingSummary &&
|
||
!result.characterSummary &&
|
||
result.negativeConstraints.length === 0 &&
|
||
result.conflicts.length === 0
|
||
) {
|
||
return null;
|
||
}
|
||
|
||
return result;
|
||
} catch (error) {
|
||
console.warn(
|
||
"[Agent Scene Workflow Prompt Optimizer] Failed to parse JSON response:",
|
||
error
|
||
);
|
||
return null;
|
||
}
|
||
}
|
||
|
||
function buildOptimizerPrompt(params: {
|
||
workflowPrompt: string;
|
||
slotPrompt?: string | null;
|
||
}) {
|
||
return [
|
||
"你是电商家具场景生图提示词整理器。",
|
||
"你的任务是把冗长的工作流选项提示词，压缩成更适合 Gemini / Imagen 生图模型执行的结构化摘要。",
|
||
"你只能做压缩、去重、冲突发现，不能改写主SKU产品身份，不能新增工作流里没有的产品结构、颜色、材质或空间元素。",
|
||
"不要重复角色、技能、流程、请按选项执行这类管理话术。",
|
||
"如果有人物要求，只总结人物身份、站位和作用；人物不能压过产品主体。如果没有明确人物要求，characterSummary 返回空字符串。",
|
||
"negativeConstraints 只保留真正影响出图稳定性的禁止项，例如变形、换色、喧宾夺主、额外主体、结构错误。",
|
||
"conflicts 只写真正的提示冲突；没有冲突返回空数组。",
|
||
'只输出 JSON：{"sceneGoal":"","environmentSummary":"","propSummary":"","cameraLightingSummary":"","characterSummary":"","negativeConstraints":[],"conflicts":[]}',
|
||
"字段要求：sceneGoal 不超过60字；environmentSummary 不超过120字；propSummary 不超过120字；cameraLightingSummary 不超过80字；characterSummary 不超过80字；negativeConstraints 最多6条，每条不超过26字；conflicts 最多4条，每条不超过50字。",
|
||
`工作流设定：\n${params.workflowPrompt}`,
|
||
params.slotPrompt ? `模式模板补充：\n${params.slotPrompt}` : "",
|
||
]
|
||
.filter(Boolean)
|
||
.join("\n\n");
|
||
}
|
||
|
||
async function optimizeWithConfiguredQwenVisionModel(params: {
|
||
prompt: string;
|
||
primaryImageUrl: string;
|
||
modelImageUrl?: string | null;
|
||
}) {
|
||
const configuredModels = await getEnabledAiModelsWithProvider().catch((error) => {
|
||
console.warn(
|
||
"[Agent Scene Workflow Prompt Optimizer] Failed to load configured models:",
|
||
error
|
||
);
|
||
return [];
|
||
});
|
||
|
||
const configuredQwenVisionModel = configuredModels.find(
|
||
({ model }) =>
|
||
model.supportsVision &&
|
||
!model.isSubModel &&
|
||
QWEN_VL_MODEL_MATCHER.test(`${model.displayName} ${model.modelId}`)
|
||
);
|
||
|
||
if (!configuredQwenVisionModel) {
|
||
return null;
|
||
}
|
||
|
||
try {
|
||
const languageModel = await getLanguageModelById(
|
||
configuredQwenVisionModel.model.id
|
||
);
|
||
if (!languageModel) {
|
||
return null;
|
||
}
|
||
|
||
const content: Array<{ type: "text" | "image"; text?: string; image?: string }> = [
|
||
{
|
||
type: "text",
|
||
text: params.prompt,
|
||
},
|
||
{
|
||
type: "image",
|
||
image: params.primaryImageUrl,
|
||
},
|
||
];
|
||
|
||
if (params.modelImageUrl) {
|
||
content.push({
|
||
type: "image",
|
||
image: params.modelImageUrl,
|
||
});
|
||
}
|
||
|
||
const { text } = await generateText({
|
||
model: languageModel,
|
||
messages: [
|
||
{
|
||
role: "user",
|
||
content,
|
||
},
|
||
],
|
||
temperature: 0,
|
||
maxTokens: 320,
|
||
abortSignal: AbortSignal.timeout(30_000),
|
||
} as any);
|
||
|
||
const parsed = parseOptimizationResult(text);
|
||
if (!parsed) {
|
||
return null;
|
||
}
|
||
|
||
return {
|
||
...parsed,
|
||
optimizationModelName:
|
||
configuredQwenVisionModel.model.displayName ||
|
||
configuredQwenVisionModel.model.modelId,
|
||
optimizationSource: "configured_qwen3_vl" as const,
|
||
};
|
||
} catch (error) {
|
||
console.warn(
|
||
"[Agent Scene Workflow Prompt Optimizer] Configured Qwen-VL optimization failed:",
|
||
error
|
||
);
|
||
return null;
|
||
}
|
||
}
|
||
|
||
export async function optimizeAgentSceneWorkflowPrompt(params: {
|
||
workflowPrompt: string;
|
||
slotPrompt?: string | null;
|
||
primaryImageUrl: string;
|
||
modelImageUrl?: string | null;
|
||
}): Promise<AgentSceneWorkflowOptimizationResult | null> {
|
||
const workflowPrompt = String(params.workflowPrompt || "").trim();
|
||
const primaryImageUrl = String(params.primaryImageUrl || "").trim();
|
||
|
||
if (!workflowPrompt || !/^https?:\/\//i.test(primaryImageUrl)) {
|
||
return null;
|
||
}
|
||
|
||
const prompt = buildOptimizerPrompt({
|
||
workflowPrompt,
|
||
slotPrompt: params.slotPrompt || null,
|
||
});
|
||
|
||
const configuredResult = await optimizeWithConfiguredQwenVisionModel({
|
||
prompt,
|
||
primaryImageUrl,
|
||
modelImageUrl: params.modelImageUrl || null,
|
||
});
|
||
if (configuredResult) {
|
||
return configuredResult;
|
||
}
|
||
|
||
try {
|
||
const fallbackResult = await callQwen3Vl({
|
||
imageUrl: primaryImageUrl,
|
||
prompt,
|
||
temperature: 0,
|
||
maxTokens: 320,
|
||
timeoutMs: 30_000,
|
||
retries: 1,
|
||
});
|
||
const parsed = parseOptimizationResult(fallbackResult.text);
|
||
if (!parsed) {
|
||
return null;
|
||
}
|
||
|
||
return {
|
||
...parsed,
|
||
optimizationModelName: fallbackResult.model,
|
||
optimizationSource: "fallback_qwen3_vl",
|
||
};
|
||
} catch (error) {
|
||
console.warn(
|
||
"[Agent Scene Workflow Prompt Optimizer] Fallback Qwen3-VL optimization failed:",
|
||
error
|
||
);
|
||
return null;
|
||
}
|
||
}
|