Update conversation memory and canvas persistence
This commit is contained in:
Your Name
parent
67d834f0dd
commit
e7812060b8
@ -90,6 +90,16 @@ import {
|
||||
message as messageTable,
|
||||
} from "@/lib/db/schema";
|
||||
import { generateCanvasImage } from "@/lib/canvas/generate-image";
|
||||
import {
|
||||
buildGridConfirmationMessage,
|
||||
detectRequestedStoryboardGrid,
|
||||
extractPendingGridConfirmationFromWorkingState,
|
||||
isGridConfirmationReply,
|
||||
type PendingGridConfirmation,
|
||||
resolveConversationAction,
|
||||
} from "@/lib/ai/conversation-action-resolver";
|
||||
import { composeConversationMemory } from "@/lib/ai/conversation-memory-composer";
|
||||
import { buildConversationThreadState } from "@/lib/ai/conversation-thread-state-builder";
|
||||
import { resolveChatMediaInput } from "@/lib/ai/context-memory/media-input";
|
||||
import { resolveTargetAssets } from "@/lib/ai/context-memory/target-resolution";
|
||||
import type {
|
||||
@ -98,10 +108,13 @@ import type {
|
||||
} from "@/lib/ai/context-memory/types";
|
||||
import { ChatSDKError } from "@/lib/errors";
|
||||
import { extractImageUrlsFromText } from "@/lib/image-url";
|
||||
import {
|
||||
buildImageWorkflowMemoryUpdate,
|
||||
extractImageWorkflowMemory,
|
||||
} from "@/lib/ai/image-workflow-memory";
|
||||
import {
|
||||
buildPromptMemoryUpdate,
|
||||
extractPromptMemoryFromWorkingState,
|
||||
isPromptCarryForwardRequest,
|
||||
} from "@/lib/ai/prompt-memory";
|
||||
import { uploadFile } from "@/lib/storage/service";
|
||||
import { getVideoStoryboardConfig } from "@/lib/video-agent/config";
|
||||
@ -123,7 +136,6 @@ import {
|
||||
markVideoStoryboardDraftForRevision,
|
||||
cancelVideoStoryboardTaskByUser,
|
||||
} from "@/lib/video-agent/task-store";
|
||||
import { buildAgentThreadStateFromStoryboardTask } from "@/lib/video-agent/agent-thread-state";
|
||||
import type { StoryboardModelCandidate } from "@/lib/video-agent/types";
|
||||
import type { ChatMessage } from "@/lib/types";
|
||||
import type { AppUsage } from "@/lib/usage";
|
||||
@ -1792,102 +1804,6 @@ function extractConfirmedPromptFromWorkingState(
|
||||
return optimizedPrompt;
|
||||
}
|
||||
|
||||
type PendingGridConfirmation = {
|
||||
gridType: "grid_2x2" | "grid_3x2" | "grid_3x3";
|
||||
shotCount: 4 | 6 | 9;
|
||||
label: string;
|
||||
originalPrompt: string;
|
||||
};
|
||||
|
||||
function detectRequestedStoryboardGrid(text: string): PendingGridConfirmation | null {
|
||||
const normalized = String(text || "").toLowerCase().replace(/\s+/g, "");
|
||||
if (!normalized) return null;
|
||||
|
||||
if (/4宫格|四宫格|2x2|2×2/.test(normalized)) {
|
||||
return {
|
||||
gridType: "grid_2x2",
|
||||
shotCount: 4,
|
||||
label: "4宫格",
|
||||
originalPrompt: String(text || "").trim(),
|
||||
};
|
||||
}
|
||||
|
||||
if (/6宫格|六宫格|3x2|3×2|2x3|2×3/.test(normalized)) {
|
||||
return {
|
||||
gridType: "grid_3x2",
|
||||
shotCount: 6,
|
||||
label: "6宫格",
|
||||
originalPrompt: String(text || "").trim(),
|
||||
};
|
||||
}
|
||||
|
||||
if (/9宫格|九宫格|3x3|3×3/.test(normalized)) {
|
||||
return {
|
||||
gridType: "grid_3x3",
|
||||
shotCount: 9,
|
||||
label: "9宫格",
|
||||
originalPrompt: String(text || "").trim(),
|
||||
};
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
function isGridConfirmationReply(text: string): boolean {
|
||||
const normalized = String(text || "").toLowerCase().replace(/\s+/g, "");
|
||||
if (!normalized) return false;
|
||||
return /确认|就按这个|开始生成|可以开始了|没问题|对的|就这样/.test(normalized);
|
||||
}
|
||||
|
||||
function buildGridConfirmationMessage(grid: PendingGridConfirmation): string {
|
||||
return [
|
||||
`我看到你希望按 ${grid.label} 来生成分镜图。`,
|
||||
"为了确保后续自动切片稳定，我会按标准网格、统一格线、固定顺序来出图。",
|
||||
`请确认是否按 ${grid.label} 标准版式开始生成。`,
|
||||
"",
|
||||
"如果确认，请回复：",
|
||||
`- 确认按${grid.label}生成`,
|
||||
"",
|
||||
"如果要修改，也可以直接回复：",
|
||||
"- 改成 9 宫格",
|
||||
"- 改成 6 宫格",
|
||||
"- 改成 4 宫格",
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
function extractPendingGridConfirmationFromWorkingState(
|
||||
workingState?: {
|
||||
lastTurnSummary?: unknown;
|
||||
} | null,
|
||||
): PendingGridConfirmation | null {
|
||||
const summary = workingState?.lastTurnSummary;
|
||||
if (!summary || typeof summary !== "object") return null;
|
||||
const summaryRecord = summary as Record<string, unknown>;
|
||||
const pending = summaryRecord.pendingGridConfirmation;
|
||||
if (!pending || typeof pending !== "object") return null;
|
||||
const record = pending as Record<string, unknown>;
|
||||
const gridType = String(record.gridType || "").trim();
|
||||
const shotCount = Number(record.shotCount || 0);
|
||||
const label = String(record.label || "").trim();
|
||||
const originalPrompt = String(record.originalPrompt || "").trim();
|
||||
|
||||
if (
|
||||
!originalPrompt ||
|
||||
!label ||
|
||||
!["grid_2x2", "grid_3x2", "grid_3x3"].includes(gridType) ||
|
||||
![4, 6, 9].includes(shotCount)
|
||||
) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return {
|
||||
gridType: gridType as PendingGridConfirmation["gridType"],
|
||||
shotCount: shotCount as PendingGridConfirmation["shotCount"],
|
||||
label,
|
||||
originalPrompt,
|
||||
};
|
||||
}
|
||||
|
||||
function hasStructuredAgentSections(message?: string): boolean {
|
||||
if (!message) return false;
|
||||
return /(?:###\s*需求理解|【\s*需求理解\s*】|###\s*图片分析|【\s*图片分析\s*】|###\s*执行计划|【\s*执行计划\s*】|###\s*优化后提示词|【\s*优化后提示词\s*】)/.test(
|
||||
@ -2655,6 +2571,15 @@ async function saveGeneratedImages(
|
||||
|
||||
if (savedOriginalUrls.length > 0) {
|
||||
try {
|
||||
const existingWorkingState = await getConversationWorkingStateByChatId({
|
||||
chatId,
|
||||
}).catch(() => null);
|
||||
const existingSummary =
|
||||
existingWorkingState?.lastTurnSummary &&
|
||||
typeof existingWorkingState.lastTurnSummary === "object"
|
||||
? (existingWorkingState.lastTurnSummary as Record<string, unknown>)
|
||||
: {};
|
||||
const existingImageWorkflowMemory = extractImageWorkflowMemory(existingWorkingState);
|
||||
await upsertConversationWorkingState({
|
||||
userId,
|
||||
chatId,
|
||||
@ -2665,6 +2590,17 @@ async function saveGeneratedImages(
|
||||
lastResolvedTargetAssetIds: savedAssetIds,
|
||||
lastResolvedTargetImageUrls: savedOriginalUrls,
|
||||
lastMessageId: messageId,
|
||||
lastTurnSummary: {
|
||||
...existingSummary,
|
||||
imageWorkflowMemory: buildImageWorkflowMemoryUpdate({
|
||||
existing: existingImageWorkflowMemory,
|
||||
lastIntent: options?.relationType === "edited_from" ? "image_edit" : "image_gen",
|
||||
lastUserPrompt: prompt,
|
||||
lastRequestedImageCount: savedOriginalUrls.length,
|
||||
lastSourceImageUrls: options?.sourceImageUrls || null,
|
||||
lastResultImageUrls: savedOriginalUrls,
|
||||
}),
|
||||
},
|
||||
});
|
||||
} catch (memoryError) {
|
||||
console.warn("[Gallery] Failed to update working state with generated images:", memoryError);
|
||||
@ -3064,6 +3000,7 @@ export async function POST(request: Request) {
|
||||
| Awaited<ReturnType<typeof getConversationWorkingStateByChatId>>
|
||||
| null = null;
|
||||
let storyboardPreferenceMemory = null;
|
||||
let imageWorkflowMemory = null;
|
||||
let reusablePromptMemory = null;
|
||||
let smartAutoHistoryMessagesFromDb: DBMessage[] = [];
|
||||
let preferredAgentModelId: string | undefined;
|
||||
@ -3192,29 +3129,42 @@ export async function POST(request: Request) {
|
||||
return null;
|
||||
});
|
||||
storyboardPreferenceMemory = extractStoryboardPreferenceMemory(requestWorkingState);
|
||||
imageWorkflowMemory = extractImageWorkflowMemory(requestWorkingState);
|
||||
reusablePromptMemory = extractPromptMemoryFromWorkingState(requestWorkingState);
|
||||
pendingGridConfirmation =
|
||||
extractPendingGridConfirmationFromWorkingState(requestWorkingState);
|
||||
const hasImageInputForConversationAction = message.parts.some((part: any) =>
|
||||
part?.type === "file" || part?.type === "image" || part?.type === "image_url"
|
||||
);
|
||||
const preloadedConfirmedPrompt =
|
||||
extractConfirmedPromptFromWorkingState(requestWorkingState) ||
|
||||
reusablePromptMemory?.optimizedPrompt;
|
||||
const resolvedConversationAction = resolveConversationAction({
|
||||
text: requestTextContent,
|
||||
pendingGridConfirmation,
|
||||
hasActiveStoryboardTask: Boolean(activeStoryboardTaskForChat?.task?.id),
|
||||
imageWorkflowMemory,
|
||||
confirmedPrompt: preloadedConfirmedPrompt,
|
||||
hasImageInputForPromptCarryForward: hasImageInputForConversationAction,
|
||||
});
|
||||
|
||||
if (
|
||||
pendingGridConfirmation &&
|
||||
(isGridConfirmationReply(requestTextContent) ||
|
||||
detectRequestedStoryboardGrid(requestTextContent) !== null) &&
|
||||
!shouldRunAgentConversation
|
||||
) {
|
||||
if (resolvedConversationAction.forceAgentConversation && !shouldRunAgentConversation) {
|
||||
shouldRunAgentConversation = true;
|
||||
shouldKeepSelectedModel = true;
|
||||
console.log(
|
||||
`[Smart Auto Mode] 🧩 Force agent conversation due to pending grid confirmation: ${pendingGridConfirmation.label}`
|
||||
);
|
||||
}
|
||||
|
||||
if (activeStoryboardTaskForChat?.task?.id && !shouldRunAgentConversation) {
|
||||
shouldRunAgentConversation = true;
|
||||
shouldKeepSelectedModel = true;
|
||||
console.log(
|
||||
`[Smart Auto Mode] 🔐 Force agent conversation due to active storyboard task: ${activeStoryboardTaskForChat.task.id}`
|
||||
);
|
||||
shouldKeepSelectedModel = resolvedConversationAction.shouldKeepSelectedModel;
|
||||
if (resolvedConversationAction.kind === "grid_confirmation_followup" && pendingGridConfirmation) {
|
||||
console.log(
|
||||
`[Smart Auto Mode] 🧩 Force agent conversation due to pending grid confirmation: ${pendingGridConfirmation.label}`
|
||||
);
|
||||
} else if (
|
||||
resolvedConversationAction.kind === "active_storyboard_followup" &&
|
||||
activeStoryboardTaskForChat?.task?.id
|
||||
) {
|
||||
console.log(
|
||||
`[Smart Auto Mode] 🔐 Force agent conversation due to active storyboard task: ${activeStoryboardTaskForChat.task.id}`
|
||||
);
|
||||
} else if (resolvedConversationAction.kind === "image_edit_followup") {
|
||||
console.log("[Smart Auto Mode] 🖼️ Force agent conversation due to image edit follow-up workflow");
|
||||
}
|
||||
}
|
||||
|
||||
// ========== Smart Auto Mode Logic ==========
|
||||
@ -3226,9 +3176,7 @@ export async function POST(request: Request) {
|
||||
let confirmedPromptForGeneration: string | undefined;
|
||||
let forcePromptDrivenImageGeneration = false;
|
||||
const smartAutoInputText =
|
||||
pendingGridConfirmation && isGridConfirmationReply(requestTextContent)
|
||||
? pendingGridConfirmation.originalPrompt
|
||||
: requestTextContent;
|
||||
resolvedConversationAction.effectiveInputText || requestTextContent;
|
||||
|
||||
try {
|
||||
// 🆕 首先获取历史消息（在 Smart Auto Mode 内部单独获取，避免依赖后面的 messagesFromDb）
|
||||
@ -3241,22 +3189,11 @@ export async function POST(request: Request) {
|
||||
chatId: id,
|
||||
});
|
||||
}
|
||||
confirmedPromptForGeneration =
|
||||
extractConfirmedPromptFromWorkingState(requestWorkingState) ||
|
||||
reusablePromptMemory?.optimizedPrompt;
|
||||
const hasImageInputForPromptCarryForward =
|
||||
message.parts.some((part: any) =>
|
||||
part?.type === "file" || part?.type === "image" || part?.type === "image_url"
|
||||
);
|
||||
confirmedPromptForGeneration = preloadedConfirmedPrompt;
|
||||
const isPromptCarryForwardExecution =
|
||||
Boolean(confirmedPromptForGeneration) &&
|
||||
isPromptCarryForwardRequest({
|
||||
text: requestTextContent,
|
||||
hasImage: hasImageInputForPromptCarryForward,
|
||||
});
|
||||
resolvedConversationAction.kind === "prompt_carry_forward";
|
||||
forcePromptDrivenImageGeneration =
|
||||
(isConfirmedPromptGenerationRequest(requestTextContent) ||
|
||||
isPromptCarryForwardExecution) &&
|
||||
resolvedConversationAction.usePromptCarryForward &&
|
||||
Boolean(confirmedPromptForGeneration);
|
||||
if (!requestResolvedMediaInput) {
|
||||
requestResolvedMediaInput = resolveChatMediaInput({
|
||||
@ -3350,6 +3287,37 @@ export async function POST(request: Request) {
|
||||
CHAT_PRE_RESPONSE_SOFT_TIMEOUT_MS
|
||||
);
|
||||
}
|
||||
const baseThreadState = buildConversationThreadState({
|
||||
activeStoryboardTask: activeStoryboardTaskForChat,
|
||||
storyboardPreferenceMemory,
|
||||
promptMemory: reusablePromptMemory,
|
||||
imageWorkflowMemory,
|
||||
});
|
||||
const composedConversationMemory = composeConversationMemory({
|
||||
autoContextText: autoContextText || undefined,
|
||||
resolvedMediaInput: requestResolvedMediaInput,
|
||||
targetResolution: requestTargetResolution,
|
||||
promptMemory: reusablePromptMemory,
|
||||
imageWorkflowMemory,
|
||||
storyboardPreferenceMemory,
|
||||
threadState: baseThreadState,
|
||||
});
|
||||
const composedThreadState = {
|
||||
workflowState: baseThreadState?.workflowState || null,
|
||||
memoryHints: Array.from(
|
||||
new Set([
|
||||
...(baseThreadState?.memoryHints || []),
|
||||
...composedConversationMemory.memoryHints,
|
||||
])
|
||||
),
|
||||
contextSummary:
|
||||
composedConversationMemory.contextSummary ||
|
||||
baseThreadState?.contextSummary,
|
||||
metadata: {
|
||||
...(baseThreadState?.metadata || {}),
|
||||
...composedConversationMemory.metadata,
|
||||
},
|
||||
};
|
||||
const autoGraphResult = await withOperationTimeout(
|
||||
runAutoAgentGraph({
|
||||
text: smartAutoInputText,
|
||||
@ -3373,14 +3341,10 @@ export async function POST(request: Request) {
|
||||
requestResolvedMediaInput.historyUserImageUrls.length,
|
||||
currentUploadImageCount: requestResolvedMediaInput.currentMessageImageUrls.length,
|
||||
historyMessages,
|
||||
context: autoContextText || undefined,
|
||||
context:
|
||||
composedConversationMemory.contextSummary || autoContextText || undefined,
|
||||
preferredModelId: preferredAgentModelId,
|
||||
threadState: buildAgentThreadStateFromStoryboardTask(
|
||||
activeStoryboardTaskForChat,
|
||||
{
|
||||
storyboardPreferenceMemory,
|
||||
}
|
||||
),
|
||||
threadState: composedThreadState,
|
||||
}),
|
||||
"runAutoAgentGraph",
|
||||
autoGraphTimeoutMs,
|
||||
@ -4718,6 +4682,14 @@ export async function POST(request: Request) {
|
||||
sourceText: requestTextContent.slice(0, 500),
|
||||
reusable: true,
|
||||
}),
|
||||
imageWorkflowMemory: buildImageWorkflowMemoryUpdate({
|
||||
existing: imageWorkflowMemory,
|
||||
lastIntent: "vision",
|
||||
lastOptimizedPrompt: promptOnlyOptimizedPrompt.slice(0, 1000),
|
||||
lastUserPrompt: requestTextContent.slice(0, 500),
|
||||
lastRequestedImageCount: 1,
|
||||
lastSourceImageUrls: promptOnlySourceImageUrls,
|
||||
}),
|
||||
imageSummary: promptOnlyImageSummary.slice(0, 1200),
|
||||
sourceImageUrls: promptOnlySourceImageUrls,
|
||||
},
|
||||
@ -6214,6 +6186,12 @@ export async function POST(request: Request) {
|
||||
try {
|
||||
const resultAssetIds = resultAssetId ? [resultAssetId] : null;
|
||||
const resultImageUrls = resultOriginalUrl ? [resultOriginalUrl] : null;
|
||||
const existingImageWorkflowMemory = extractImageWorkflowMemory(requestWorkingState);
|
||||
const existingSummary =
|
||||
requestWorkingState?.lastTurnSummary &&
|
||||
typeof requestWorkingState.lastTurnSummary === "object"
|
||||
? (requestWorkingState.lastTurnSummary as Record<string, unknown>)
|
||||
: {};
|
||||
await upsertConversationWorkingState({
|
||||
userId: session.user.id,
|
||||
chatId: id,
|
||||
@ -6227,6 +6205,17 @@ export async function POST(request: Request) {
|
||||
lastResolvedTargetAssetIds: resultAssetIds,
|
||||
lastResolvedTargetImageUrls: resultImageUrls,
|
||||
lastMessageId: assistantMessageId,
|
||||
lastTurnSummary: {
|
||||
...existingSummary,
|
||||
imageWorkflowMemory: buildImageWorkflowMemoryUpdate({
|
||||
existing: existingImageWorkflowMemory,
|
||||
lastIntent: "image_edit",
|
||||
lastUserPrompt: requestTextContent.slice(0, 500),
|
||||
lastRequestedImageCount: resultImageUrls?.length || 1,
|
||||
lastSourceImageUrls: directAgentToolTargetImageUrls,
|
||||
lastResultImageUrls: resultImageUrls,
|
||||
}),
|
||||
},
|
||||
});
|
||||
} catch (memoryError) {
|
||||
console.warn(
|
||||
|
||||
@ -3523,6 +3523,11 @@ export function CanvasChatPanel({
|
||||
}
|
||||
|
||||
const latestNodes = useCanvasStore.getState().nodes;
|
||||
const suppressedMediaKeys = new Set(
|
||||
(useCanvasStore.getState().getCanvasState().suppressedMediaKeys || [])
|
||||
.map((item) => String(item || "").trim())
|
||||
.filter(Boolean)
|
||||
);
|
||||
const existingImageUrlKeys = new Set(
|
||||
latestNodes
|
||||
.filter((node) => node.type === "image" && (node.data as any)?.src)
|
||||
@ -3572,6 +3577,12 @@ export function CanvasChatPanel({
|
||||
continue;
|
||||
}
|
||||
const messageImageKey = messageId ? `${messageId}::${sourceImageKey}` : "";
|
||||
if (
|
||||
suppressedMediaKeys.has(`image:${messageImageKey}`) ||
|
||||
suppressedMediaKeys.has(`image-url:${url}`)
|
||||
) {
|
||||
continue;
|
||||
}
|
||||
if (messageImageKey && existingMessageImageKeys.has(messageImageKey)) {
|
||||
continue;
|
||||
}
|
||||
@ -3642,6 +3653,9 @@ export function CanvasChatPanel({
|
||||
if (!key || existingVideoUrlKeys.has(key)) {
|
||||
continue;
|
||||
}
|
||||
if (suppressedMediaKeys.has(`video:${url}`)) {
|
||||
continue;
|
||||
}
|
||||
existingVideoUrlKeys.add(key);
|
||||
const currentNodes = useCanvasStore.getState().nodes;
|
||||
const existingNodes = currentNodes.length + addedImageCount + addedVideoCount;
|
||||
@ -4344,6 +4358,11 @@ export function CanvasChatPanel({
|
||||
let cancelled = false;
|
||||
|
||||
const processNewAssistantMessages = async () => {
|
||||
const suppressedMediaKeys = new Set(
|
||||
(useCanvasStore.getState().getCanvasState().suppressedMediaKeys || [])
|
||||
.map((item) => String(item || "").trim())
|
||||
.filter(Boolean)
|
||||
);
|
||||
const existingImageUrlKeys = new Set(
|
||||
nodes
|
||||
.filter((n) => n.type === "image" && (n.data as any)?.src)
|
||||
@ -4465,6 +4484,12 @@ export function CanvasChatPanel({
|
||||
return acc;
|
||||
}
|
||||
const messageImageKey = `${message.id}::${sourceImageKey}`;
|
||||
if (
|
||||
suppressedMediaKeys.has(`image:${messageImageKey}`) ||
|
||||
suppressedMediaKeys.has(`image-url:${url}`)
|
||||
) {
|
||||
return acc;
|
||||
}
|
||||
if (existingMessageImageKeys.has(messageImageKey)) {
|
||||
return acc;
|
||||
}
|
||||
@ -4556,6 +4581,7 @@ export function CanvasChatPanel({
|
||||
const newVideoUrls = videoUrls.filter((url) => {
|
||||
const key = normalizeMediaUrl(url);
|
||||
if (!key || existingVideoUrlKeys.has(key)) return false;
|
||||
if (suppressedMediaKeys.has(`video:${url}`)) return false;
|
||||
existingVideoUrlKeys.add(key);
|
||||
return true;
|
||||
});
|
||||
|
||||
@ -861,7 +861,7 @@ function sanitizeRecommendedImageParams(
|
||||
|
||||
if (typeof params.imageCount === "number" && Number.isFinite(params.imageCount)) {
|
||||
const min = 1;
|
||||
const max = intent === "image_edit" ? 1 : 4;
|
||||
const max = intent === "image_edit" ? 1 : 12;
|
||||
safe.imageCount = Math.min(Math.max(Math.round(params.imageCount), min), max);
|
||||
}
|
||||
if (intent === "image_edit") {
|
||||
@ -894,12 +894,18 @@ function heuristicImageParams(
|
||||
|
||||
if (analysis.intent === "image_edit") {
|
||||
params.imageCount = 1;
|
||||
} else if (/多张|四张|4张|三张|3张/.test(normalized)) {
|
||||
params.imageCount = /四张|4张/.test(normalized) ? 4 : 3;
|
||||
} else if (/两张|2张|双图/.test(normalized)) {
|
||||
params.imageCount = 2;
|
||||
} else {
|
||||
params.imageCount = 1;
|
||||
const explicitCountMatch = normalized.match(/([1-9]\d?)张/);
|
||||
const explicitCount = explicitCountMatch ? Number(explicitCountMatch[1]) : 0;
|
||||
if (explicitCount >= 1) {
|
||||
params.imageCount = Math.min(explicitCount, 12);
|
||||
} else if (/多张|四张|4张|三张|3张/.test(normalized)) {
|
||||
params.imageCount = /四张|4张/.test(normalized) ? 4 : 3;
|
||||
} else if (/两张|2张|双图/.test(normalized)) {
|
||||
params.imageCount = 2;
|
||||
} else {
|
||||
params.imageCount = 1;
|
||||
}
|
||||
}
|
||||
|
||||
if (/横图|横版|宽屏|封面|banner|海报/.test(normalized)) {
|
||||
|
||||
201
lib/ai/conversation-action-resolver.ts
Normal file
201
lib/ai/conversation-action-resolver.ts
Normal file
@ -0,0 +1,201 @@
|
||||
import "server-only";
|
||||
|
||||
import type { ImageWorkflowMemory } from "@/lib/ai/image-workflow-memory";
|
||||
import { mentionsImageEditFollowup } from "@/lib/ai/image-edit-subagent";
|
||||
import { isPromptCarryForwardRequest } from "@/lib/ai/prompt-memory";
|
||||
|
||||
export interface PendingGridConfirmation {
|
||||
gridType: "grid_2x2" | "grid_3x2" | "grid_3x3";
|
||||
shotCount: 4 | 6 | 9;
|
||||
label: string;
|
||||
originalPrompt: string;
|
||||
}
|
||||
|
||||
export type ConversationActionKind =
|
||||
| "default"
|
||||
| "grid_confirmation_followup"
|
||||
| "active_storyboard_followup"
|
||||
| "image_edit_followup"
|
||||
| "prompt_carry_forward";
|
||||
|
||||
export interface ConversationActionResolution {
|
||||
kind: ConversationActionKind;
|
||||
forceAgentConversation: boolean;
|
||||
shouldKeepSelectedModel: boolean;
|
||||
effectiveInputText?: string;
|
||||
usePromptCarryForward: boolean;
|
||||
}
|
||||
|
||||
export function isConfirmedPromptGenerationRequest(text: string): boolean {
|
||||
const normalized = String(text || "").toLowerCase().replace(/\s+/g, "");
|
||||
if (!normalized) return false;
|
||||
|
||||
const refersPrompt =
|
||||
/根据这个提示词|按这个提示词|就按这个提示词|按刚才的提示词|按上面的提示词|用这个提示词|用刚才的提示词|按该提示词/.test(
|
||||
normalized,
|
||||
);
|
||||
const wantsImageGeneration =
|
||||
/生图|生成图片|出图|画出来|生成一张|来一张|开始生成|按此生成|按这个生成/.test(normalized);
|
||||
|
||||
return refersPrompt && wantsImageGeneration;
|
||||
}
|
||||
|
||||
export function detectRequestedStoryboardGrid(text: string): PendingGridConfirmation | null {
|
||||
const normalized = String(text || "").toLowerCase().replace(/\s+/g, "");
|
||||
if (!normalized) return null;
|
||||
|
||||
if (/4宫格|四宫格|2x2|2×2/.test(normalized)) {
|
||||
return {
|
||||
gridType: "grid_2x2",
|
||||
shotCount: 4,
|
||||
label: "4宫格",
|
||||
originalPrompt: String(text || "").trim(),
|
||||
};
|
||||
}
|
||||
|
||||
if (/6宫格|六宫格|3x2|3×2|2x3|2×3/.test(normalized)) {
|
||||
return {
|
||||
gridType: "grid_3x2",
|
||||
shotCount: 6,
|
||||
label: "6宫格",
|
||||
originalPrompt: String(text || "").trim(),
|
||||
};
|
||||
}
|
||||
|
||||
if (/9宫格|九宫格|3x3|3×3/.test(normalized)) {
|
||||
return {
|
||||
gridType: "grid_3x3",
|
||||
shotCount: 9,
|
||||
label: "9宫格",
|
||||
originalPrompt: String(text || "").trim(),
|
||||
};
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
export function isGridConfirmationReply(text: string): boolean {
|
||||
const normalized = String(text || "").toLowerCase().replace(/\s+/g, "");
|
||||
if (!normalized) return false;
|
||||
return /确认|就按这个|开始生成|可以开始了|没问题|对的|就这样/.test(normalized);
|
||||
}
|
||||
|
||||
export function extractPendingGridConfirmationFromWorkingState(
|
||||
workingState?: {
|
||||
lastTurnSummary?: unknown;
|
||||
} | null,
|
||||
): PendingGridConfirmation | null {
|
||||
const summary = workingState?.lastTurnSummary;
|
||||
if (!summary || typeof summary !== "object") return null;
|
||||
const summaryRecord = summary as Record<string, unknown>;
|
||||
const pending = summaryRecord.pendingGridConfirmation;
|
||||
if (!pending || typeof pending !== "object") return null;
|
||||
const record = pending as Record<string, unknown>;
|
||||
const gridType = String(record.gridType || "").trim();
|
||||
const shotCount = Number(record.shotCount || 0);
|
||||
const label = String(record.label || "").trim();
|
||||
const originalPrompt = String(record.originalPrompt || "").trim();
|
||||
|
||||
if (
|
||||
!originalPrompt ||
|
||||
!label ||
|
||||
!["grid_2x2", "grid_3x2", "grid_3x3"].includes(gridType) ||
|
||||
![4, 6, 9].includes(shotCount)
|
||||
) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return {
|
||||
gridType: gridType as PendingGridConfirmation["gridType"],
|
||||
shotCount: shotCount as PendingGridConfirmation["shotCount"],
|
||||
label,
|
||||
originalPrompt,
|
||||
};
|
||||
}
|
||||
|
||||
export function buildGridConfirmationMessage(grid: PendingGridConfirmation): string {
|
||||
return [
|
||||
`我看到你希望按 ${grid.label} 来生成分镜图。`,
|
||||
"为了确保后续自动切片稳定，我会按标准网格、统一格线、固定顺序来出图。",
|
||||
`请确认是否按 ${grid.label} 标准版式开始生成。`,
|
||||
"",
|
||||
"如果确认，请回复：",
|
||||
`- 确认按${grid.label}生成`,
|
||||
"",
|
||||
"如果要修改，也可以直接回复：",
|
||||
"- 改成 9 宫格",
|
||||
"- 改成 6 宫格",
|
||||
"- 改成 4 宫格",
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
export function resolveConversationAction(params: {
|
||||
text: string;
|
||||
pendingGridConfirmation?: PendingGridConfirmation | null;
|
||||
hasActiveStoryboardTask: boolean;
|
||||
imageWorkflowMemory?: ImageWorkflowMemory | null;
|
||||
confirmedPrompt?: string;
|
||||
hasImageInputForPromptCarryForward: boolean;
|
||||
}) {
|
||||
const requestedGrid = detectRequestedStoryboardGrid(params.text);
|
||||
const shouldContinuePendingGrid =
|
||||
Boolean(params.pendingGridConfirmation) &&
|
||||
(isGridConfirmationReply(params.text) || requestedGrid !== null);
|
||||
|
||||
if (shouldContinuePendingGrid) {
|
||||
return {
|
||||
kind: "grid_confirmation_followup",
|
||||
forceAgentConversation: true,
|
||||
shouldKeepSelectedModel: true,
|
||||
effectiveInputText: params.pendingGridConfirmation?.originalPrompt,
|
||||
usePromptCarryForward: false,
|
||||
} satisfies ConversationActionResolution;
|
||||
}
|
||||
|
||||
if (params.hasActiveStoryboardTask) {
|
||||
return {
|
||||
kind: "active_storyboard_followup",
|
||||
forceAgentConversation: true,
|
||||
shouldKeepSelectedModel: true,
|
||||
usePromptCarryForward: false,
|
||||
} satisfies ConversationActionResolution;
|
||||
}
|
||||
|
||||
const hasImageEditFollowup =
|
||||
params.imageWorkflowMemory?.lastIntent === "image_edit" &&
|
||||
(mentionsImageEditFollowup(params.text) ||
|
||||
Boolean(params.imageWorkflowMemory?.lastResultImageUrls?.length));
|
||||
|
||||
if (hasImageEditFollowup) {
|
||||
return {
|
||||
kind: "image_edit_followup",
|
||||
forceAgentConversation: true,
|
||||
shouldKeepSelectedModel: true,
|
||||
usePromptCarryForward: false,
|
||||
} satisfies ConversationActionResolution;
|
||||
}
|
||||
|
||||
const usePromptCarryForward =
|
||||
Boolean(params.confirmedPrompt) &&
|
||||
(isConfirmedPromptGenerationRequest(params.text) ||
|
||||
isPromptCarryForwardRequest({
|
||||
text: params.text,
|
||||
hasImage: params.hasImageInputForPromptCarryForward,
|
||||
}));
|
||||
|
||||
if (usePromptCarryForward) {
|
||||
return {
|
||||
kind: "prompt_carry_forward",
|
||||
forceAgentConversation: false,
|
||||
shouldKeepSelectedModel: false,
|
||||
usePromptCarryForward: true,
|
||||
} satisfies ConversationActionResolution;
|
||||
}
|
||||
|
||||
return {
|
||||
kind: "default",
|
||||
forceAgentConversation: false,
|
||||
shouldKeepSelectedModel: false,
|
||||
usePromptCarryForward: false,
|
||||
} satisfies ConversationActionResolution;
|
||||
}
|
||||
204
lib/ai/conversation-memory-composer.ts
Normal file
204
lib/ai/conversation-memory-composer.ts
Normal file
@ -0,0 +1,204 @@
|
||||
import "server-only";
|
||||
|
||||
import type { AgentThreadState } from "@/lib/ai/runtime/agent-thread-state";
|
||||
import type { ImageWorkflowMemory } from "@/lib/ai/image-workflow-memory";
|
||||
import type { PromptMemoryEntry } from "@/lib/ai/prompt-memory";
|
||||
import type { StoryboardPreferenceMemory } from "@/lib/video-agent/storyboard-preference-memory";
|
||||
import type {
|
||||
ResolvedChatMediaInput,
|
||||
ResolvedTargetAssets,
|
||||
} from "@/lib/ai/context-memory/types";
|
||||
|
||||
export interface ComposedConversationMemory {
|
||||
contextSummary: string;
|
||||
memoryHints: string[];
|
||||
metadata: Record<string, unknown>;
|
||||
}
|
||||
|
||||
function normalizeText(value: unknown, max = 400): string | undefined {
|
||||
const normalized = String(value || "").trim();
|
||||
if (!normalized) return undefined;
|
||||
return normalized.slice(0, max);
|
||||
}
|
||||
|
||||
function uniqueStrings(values: Array<string | undefined>) {
|
||||
return Array.from(
|
||||
new Set(
|
||||
values
|
||||
.map((item) => normalizeText(item, 240))
|
||||
.filter((item): item is string => Boolean(item))
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
function summarizeTargetResolution(target?: ResolvedTargetAssets | null) {
|
||||
if (!target) return { lines: [] as string[], metadata: {} as Record<string, unknown> };
|
||||
const lines: string[] = [];
|
||||
lines.push(
|
||||
`当前目标解析：mode=${target.resolutionMode}; confidence=${target.confidence}; fromHistory=${
|
||||
target.fromHistory ? "yes" : "no"
|
||||
}`
|
||||
);
|
||||
if (target.targetImageUrls.length > 0) {
|
||||
lines.push(`当前命中的目标图片数：${target.targetImageUrls.length}`);
|
||||
}
|
||||
if (target.reason) {
|
||||
lines.push(`目标解析原因：${target.reason}`);
|
||||
}
|
||||
return {
|
||||
lines,
|
||||
metadata: {
|
||||
targetResolutionMode: target.resolutionMode,
|
||||
targetResolutionConfidence: target.confidence,
|
||||
targetImageCount: target.targetImageUrls.length,
|
||||
targetReason: target.reason,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function summarizeResolvedMedia(media?: ResolvedChatMediaInput | null) {
|
||||
if (!media) return { lines: [] as string[], metadata: {} as Record<string, unknown> };
|
||||
return {
|
||||
lines: [
|
||||
`当前媒体输入：source=${media.source}; hasImage=${media.hasImage ? "yes" : "no"}`,
|
||||
media.currentMessageImageUrls.length > 0
|
||||
? `本轮上传图片数：${media.currentMessageImageUrls.length}`
|
||||
: "",
|
||||
media.selectedImageUrls.length > 0
|
||||
? `当前选中图片数：${media.selectedImageUrls.length}`
|
||||
: "",
|
||||
media.historyAssistantImageUrls.length + media.historyUserImageUrls.length > 0
|
||||
? `历史可参考图片数：${
|
||||
media.historyAssistantImageUrls.length + media.historyUserImageUrls.length
|
||||
}`
|
||||
: "",
|
||||
].filter(Boolean),
|
||||
metadata: {
|
||||
mediaSource: media.source,
|
||||
hasImage: media.hasImage,
|
||||
currentImageCount: media.currentMessageImageUrls.length,
|
||||
selectedImageCount: media.selectedImageUrls.length,
|
||||
historyImageCount:
|
||||
media.historyAssistantImageUrls.length + media.historyUserImageUrls.length,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function summarizePromptMemory(promptMemory?: PromptMemoryEntry | null) {
|
||||
if (!promptMemory?.optimizedPrompt) {
|
||||
return { lines: [] as string[], hints: [] as string[], metadata: {} as Record<string, unknown> };
|
||||
}
|
||||
const preview = promptMemory.optimizedPrompt.replace(/\s+/g, " ").trim().slice(0, 220);
|
||||
return {
|
||||
lines: [`上一轮可复用提示词：${preview}`],
|
||||
hints: [
|
||||
"如果用户明确要求沿用上一轮提示词继续做图，优先把这段提示词作为本轮生成基础。",
|
||||
],
|
||||
metadata: {
|
||||
promptMemory,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function summarizeImageWorkflow(memory?: ImageWorkflowMemory | null) {
|
||||
if (!memory) {
|
||||
return { lines: [] as string[], hints: [] as string[], metadata: {} as Record<string, unknown> };
|
||||
}
|
||||
const lines = [
|
||||
memory.lastOptimizedPrompt
|
||||
? `上一轮优化提示词摘要：${memory.lastOptimizedPrompt.replace(/\s+/g, " ").slice(0, 180)}`
|
||||
: "",
|
||||
memory.lastRequestedImageCount
|
||||
? `上一轮目标出图数：${memory.lastRequestedImageCount} 张`
|
||||
: "",
|
||||
memory.lastAspectRatio ? `上一轮画幅：${memory.lastAspectRatio}` : "",
|
||||
memory.lastImageSize ? `上一轮清晰度：${memory.lastImageSize}` : "",
|
||||
memory.lastSourceImageUrls?.length
|
||||
? `上一轮参考/目标图数量：${memory.lastSourceImageUrls.length}`
|
||||
: "",
|
||||
memory.lastResultImageUrls?.length
|
||||
? `上一轮生成结果数量：${memory.lastResultImageUrls.length}`
|
||||
: "",
|
||||
].filter(Boolean);
|
||||
const hints = memory.lastOptimizedPrompt
|
||||
? ["如果用户要求沿用上一轮提示词继续做图，应优先复用这段提示词。"]
|
||||
: [];
|
||||
return {
|
||||
lines,
|
||||
hints,
|
||||
metadata: {
|
||||
imageWorkflowMemory: memory,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function summarizeStoryboardPreference(memory?: StoryboardPreferenceMemory | null) {
|
||||
if (!memory) {
|
||||
return { lines: [] as string[], hints: [] as string[], metadata: {} as Record<string, unknown> };
|
||||
}
|
||||
const lines = [
|
||||
memory.preferredGridType ? `用户偏好宫格：${memory.preferredGridType}` : "",
|
||||
memory.preferredShotCount ? `用户偏好镜头数：${memory.preferredShotCount}` : "",
|
||||
memory.prefersConfirmation === true ? "用户偏好先确认再执行" : "",
|
||||
memory.lastUsedGridType && memory.lastUsedShotCount
|
||||
? `最近一次分镜任务：${memory.lastUsedGridType} / ${memory.lastUsedShotCount} 段`
|
||||
: "",
|
||||
].filter(Boolean);
|
||||
const hints = memory.prefersConfirmation
|
||||
? ["如果当前任务涉及分镜或高成本执行，优先先确认再推进。"]
|
||||
: [];
|
||||
return {
|
||||
lines,
|
||||
hints,
|
||||
metadata: {
|
||||
storyboardPreferenceMemory: memory,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export function composeConversationMemory(params: {
|
||||
autoContextText?: string;
|
||||
resolvedMediaInput?: ResolvedChatMediaInput | null;
|
||||
targetResolution?: ResolvedTargetAssets | null;
|
||||
promptMemory?: PromptMemoryEntry | null;
|
||||
imageWorkflowMemory?: ImageWorkflowMemory | null;
|
||||
storyboardPreferenceMemory?: StoryboardPreferenceMemory | null;
|
||||
threadState?: AgentThreadState | null;
|
||||
}) {
|
||||
const targetSummary = summarizeTargetResolution(params.targetResolution);
|
||||
const mediaSummary = summarizeResolvedMedia(params.resolvedMediaInput);
|
||||
const promptSummary = summarizePromptMemory(params.promptMemory);
|
||||
const imageWorkflowSummary = summarizeImageWorkflow(params.imageWorkflowMemory);
|
||||
const storyboardSummary = summarizeStoryboardPreference(params.storyboardPreferenceMemory);
|
||||
|
||||
const contextSummary = uniqueStrings([
|
||||
params.autoContextText,
|
||||
...mediaSummary.lines,
|
||||
...targetSummary.lines,
|
||||
...promptSummary.lines,
|
||||
...imageWorkflowSummary.lines,
|
||||
...storyboardSummary.lines,
|
||||
]).join("\n");
|
||||
|
||||
const memoryHints = uniqueStrings([
|
||||
...(params.threadState?.memoryHints || []),
|
||||
...promptSummary.hints,
|
||||
...imageWorkflowSummary.hints,
|
||||
...storyboardSummary.hints,
|
||||
]);
|
||||
|
||||
const metadata: Record<string, unknown> = {
|
||||
...(params.threadState?.metadata || {}),
|
||||
...mediaSummary.metadata,
|
||||
...targetSummary.metadata,
|
||||
...promptSummary.metadata,
|
||||
...imageWorkflowSummary.metadata,
|
||||
...storyboardSummary.metadata,
|
||||
};
|
||||
|
||||
return {
|
||||
contextSummary,
|
||||
memoryHints,
|
||||
metadata,
|
||||
} satisfies ComposedConversationMemory;
|
||||
}
|
||||
133
lib/ai/conversation-thread-state-builder.ts
Normal file
133
lib/ai/conversation-thread-state-builder.ts
Normal file
@ -0,0 +1,133 @@
|
||||
import "server-only";
|
||||
|
||||
import type { AgentThreadState } from "@/lib/ai/runtime/agent-thread-state";
|
||||
import type { PromptMemoryEntry } from "@/lib/ai/prompt-memory";
|
||||
import type { ImageWorkflowMemory } from "@/lib/ai/image-workflow-memory";
|
||||
import type { StoryboardPreferenceMemory } from "@/lib/video-agent/storyboard-preference-memory";
|
||||
import { buildAgentThreadStateFromStoryboardTask } from "@/lib/video-agent/agent-thread-state";
|
||||
|
||||
type StoryboardTaskDetail = Parameters<typeof buildAgentThreadStateFromStoryboardTask>[0];
|
||||
|
||||
function normalizeText(value: unknown, max = 240): string | undefined {
|
||||
const normalized = String(value || "").trim();
|
||||
if (!normalized) return undefined;
|
||||
return normalized.slice(0, max);
|
||||
}
|
||||
|
||||
export function buildConversationThreadState(params: {
|
||||
activeStoryboardTask?: StoryboardTaskDetail | null;
|
||||
storyboardPreferenceMemory?: StoryboardPreferenceMemory | null;
|
||||
promptMemory?: PromptMemoryEntry | null;
|
||||
imageWorkflowMemory?: ImageWorkflowMemory | null;
|
||||
}): AgentThreadState | null {
|
||||
const storyboardState = buildAgentThreadStateFromStoryboardTask(
|
||||
params.activeStoryboardTask,
|
||||
{
|
||||
storyboardPreferenceMemory: params.storyboardPreferenceMemory,
|
||||
promptMemory: params.promptMemory,
|
||||
},
|
||||
);
|
||||
|
||||
if (params.activeStoryboardTask?.task?.id) {
|
||||
return {
|
||||
workflowState: storyboardState?.workflowState || null,
|
||||
memoryHints: [
|
||||
...(storyboardState?.memoryHints || []),
|
||||
params.imageWorkflowMemory?.lastOptimizedPrompt
|
||||
? "当前会话里也保留了最近一次图片工作流记忆，可辅助理解用户是在继续哪一轮生图。"
|
||||
: "",
|
||||
].filter(Boolean),
|
||||
contextSummary:
|
||||
storyboardState?.contextSummary ||
|
||||
normalizeText(params.imageWorkflowMemory?.lastUserPrompt, 320),
|
||||
metadata: {
|
||||
...(storyboardState?.metadata || {}),
|
||||
imageWorkflowMemory: params.imageWorkflowMemory || undefined,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
const hasImageFollowup =
|
||||
Boolean(params.promptMemory?.optimizedPrompt) ||
|
||||
Boolean(params.imageWorkflowMemory?.lastOptimizedPrompt) ||
|
||||
Boolean(params.imageWorkflowMemory?.lastResultImageUrls?.length) ||
|
||||
Boolean(params.imageWorkflowMemory?.lastSourceImageUrls?.length);
|
||||
|
||||
if (!hasImageFollowup && !params.storyboardPreferenceMemory) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const preferredCount = params.imageWorkflowMemory?.lastRequestedImageCount;
|
||||
const preferredRatio = params.imageWorkflowMemory?.lastAspectRatio;
|
||||
const preferredSize = params.imageWorkflowMemory?.lastImageSize;
|
||||
const sourceCount = params.imageWorkflowMemory?.lastSourceImageUrls?.length || 0;
|
||||
const resultCount = params.imageWorkflowMemory?.lastResultImageUrls?.length || 0;
|
||||
const imageFollowupType =
|
||||
params.imageWorkflowMemory?.lastIntent === "image_edit"
|
||||
? "image_edit_followup"
|
||||
: "image_generation_followup";
|
||||
const imageFollowupTitle =
|
||||
imageFollowupType === "image_edit_followup"
|
||||
? "连续修图工作流"
|
||||
: "连续生图工作流";
|
||||
|
||||
return {
|
||||
workflowState: hasImageFollowup
|
||||
? {
|
||||
workflowId: "image-generation-followup",
|
||||
type: imageFollowupType,
|
||||
status: "draft",
|
||||
title: imageFollowupTitle,
|
||||
summary: [
|
||||
params.promptMemory?.optimizedPrompt || params.imageWorkflowMemory?.lastOptimizedPrompt
|
||||
? "当前会话中存在上一轮可复用提示词。"
|
||||
: "",
|
||||
preferredCount ? `上一轮目标出图数：${preferredCount} 张。` : "",
|
||||
preferredRatio ? `上一轮画幅：${preferredRatio}。` : "",
|
||||
preferredSize ? `上一轮清晰度：${preferredSize}。` : "",
|
||||
sourceCount > 0 ? `上一轮参考图数：${sourceCount}。` : "",
|
||||
resultCount > 0 ? `上一轮结果图数：${resultCount}。` : "",
|
||||
]
|
||||
.filter(Boolean)
|
||||
.join(" "),
|
||||
confirmPhrases: ["按上一个提示词生成", "继续生成", "就按这个来"],
|
||||
revisionHints: ["换一版", "保持提示词不变再来一组", "沿用上一个提示词"],
|
||||
pendingItems: [],
|
||||
metadata: {
|
||||
promptMemory: params.promptMemory || undefined,
|
||||
imageWorkflowMemory: params.imageWorkflowMemory || undefined,
|
||||
storyboardPreferenceMemory: params.storyboardPreferenceMemory || undefined,
|
||||
},
|
||||
}
|
||||
: null,
|
||||
memoryHints: [
|
||||
params.promptMemory?.optimizedPrompt
|
||||
? "当前会话存在上一轮可复用提示词；如果用户明确要求沿用它继续生成，应优先复用。"
|
||||
: "",
|
||||
params.imageWorkflowMemory?.lastResultImageUrls?.length
|
||||
? "当前会话保留了上一轮生成结果图，可用于理解“继续做这张/沿用上一版”的指代。"
|
||||
: "",
|
||||
params.imageWorkflowMemory?.lastSourceImageUrls?.length
|
||||
? "当前会话保留了上一轮参考图/目标图，可用于连续图生图理解。"
|
||||
: "",
|
||||
params.storyboardPreferenceMemory
|
||||
? "当前会话也存在 storyboard 偏好记忆，后续如果转入分镜任务可继续复用。"
|
||||
: "",
|
||||
].filter(Boolean),
|
||||
contextSummary: [
|
||||
params.promptMemory?.optimizedPrompt
|
||||
? `上一轮提示词摘要：${normalizeText(params.promptMemory.optimizedPrompt, 180)}`
|
||||
: "",
|
||||
params.imageWorkflowMemory?.lastUserPrompt
|
||||
? `上一轮用户原始需求：${normalizeText(params.imageWorkflowMemory.lastUserPrompt, 180)}`
|
||||
: "",
|
||||
]
|
||||
.filter(Boolean)
|
||||
.join("\n"),
|
||||
metadata: {
|
||||
promptMemory: params.promptMemory || undefined,
|
||||
imageWorkflowMemory: params.imageWorkflowMemory || undefined,
|
||||
storyboardPreferenceMemory: params.storyboardPreferenceMemory || undefined,
|
||||
},
|
||||
};
|
||||
}
|
||||
142
lib/ai/image-edit-subagent.ts
Normal file
142
lib/ai/image-edit-subagent.ts
Normal file
@ -0,0 +1,142 @@
|
||||
import "server-only";
|
||||
|
||||
import type { AgentThreadState } from "@/lib/ai/runtime/agent-thread-state";
|
||||
import type { AgentExecutionPlan, AgentPlannedToolCall } from "@/lib/ai/auto-agent-graph";
|
||||
import type { AgentSpecialistRoute } from "@/lib/ai/specialists";
|
||||
import type { AgentProjectToolDefinition } from "@/lib/ai/tools-registry";
|
||||
import { getImageWorkflowPlanningHints } from "@/lib/ai/image-workflow-memory";
|
||||
|
||||
function normalizeCompactText(text: string) {
|
||||
return String(text || "").toLowerCase().replace(/\s+/g, "").trim();
|
||||
}
|
||||
|
||||
export function mentionsImageEditFollowup(text: string): boolean {
|
||||
return /继续改|继续调整|再改一下|再调一下|上一版|上一张结果|这张继续改|保留.*不变|只改|继续优化这张图|继续修这张图/.test(
|
||||
normalizeCompactText(text)
|
||||
);
|
||||
}
|
||||
|
||||
function hasImageEditWorkflow(threadState?: AgentThreadState | null) {
|
||||
return threadState?.workflowState?.type === "image_edit_followup";
|
||||
}
|
||||
|
||||
export function buildImageEditSpecialistRoute(params: {
|
||||
text: string;
|
||||
threadState?: AgentThreadState | null;
|
||||
hasImage: boolean;
|
||||
}): AgentSpecialistRoute | null {
|
||||
const normalized = normalizeCompactText(params.text);
|
||||
const workflowHints = getImageWorkflowPlanningHints(params.threadState);
|
||||
const hasWorkflow = hasImageEditWorkflow(params.threadState);
|
||||
const mentionsFollowupEdit = mentionsImageEditFollowup(params.text);
|
||||
|
||||
if (!hasWorkflow && !mentionsFollowupEdit && workflowHints.length === 0) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const planningHints = [
|
||||
hasWorkflow
|
||||
? "当前回合处于连续修图 workflow，应优先理解为对上一轮结果图或目标图继续编辑。"
|
||||
: "",
|
||||
mentionsFollowupEdit
|
||||
? "用户当前表达的是继续修图或局部重调，不要误解成重新生图。"
|
||||
: "",
|
||||
params.hasImage
|
||||
? "如果本轮带了图片，应优先把它理解为当前修图输入或替换后的参考图。"
|
||||
: "",
|
||||
...workflowHints,
|
||||
].filter(Boolean);
|
||||
|
||||
return {
|
||||
domain: "image_edit_memory",
|
||||
specialistId: "image-edit-subagent",
|
||||
reason: hasWorkflow ? "当前回合命中连续修图工作流" : "当前回合命中连续修图语义",
|
||||
planningHints,
|
||||
toolAllowlist: [
|
||||
"respondDirect",
|
||||
"editImage",
|
||||
"removeBackground",
|
||||
"enhanceImage",
|
||||
"extendImage",
|
||||
"resizeImage",
|
||||
"boxCutout",
|
||||
"useSelectedCanvasImage",
|
||||
"useLatestResultImage",
|
||||
"useOriginalUploadImage",
|
||||
"publishToCanvas",
|
||||
],
|
||||
};
|
||||
}
|
||||
|
||||
function hasTool(
|
||||
availableTools: AgentProjectToolDefinition[],
|
||||
name: AgentPlannedToolCall["name"],
|
||||
) {
|
||||
return availableTools.some((tool) => tool.name === name);
|
||||
}
|
||||
|
||||
function upsertTool(
|
||||
toolCalls: AgentPlannedToolCall[],
|
||||
name: AgentPlannedToolCall["name"],
|
||||
reason: string,
|
||||
options?: { prepend?: boolean },
|
||||
) {
|
||||
const existing = toolCalls.find((tool) => tool.name === name);
|
||||
if (existing) return toolCalls;
|
||||
const nextTool = { name, reason } satisfies AgentPlannedToolCall;
|
||||
return options?.prepend ? [nextTool, ...toolCalls] : [...toolCalls, nextTool];
|
||||
}
|
||||
|
||||
export function refinePlanWithImageEditSubagent(params: {
|
||||
text: string;
|
||||
plan: AgentExecutionPlan;
|
||||
availableTools: AgentProjectToolDefinition[];
|
||||
threadState?: AgentThreadState | null;
|
||||
hasImage: boolean;
|
||||
}): AgentExecutionPlan {
|
||||
const hasWorkflow = hasImageEditWorkflow(params.threadState);
|
||||
const hasImageWorkflowHints = getImageWorkflowPlanningHints(params.threadState).length > 0;
|
||||
const mentionsFollowupEdit = mentionsImageEditFollowup(params.text);
|
||||
|
||||
if (!hasWorkflow && !hasImageWorkflowHints && !mentionsFollowupEdit) {
|
||||
return params.plan;
|
||||
}
|
||||
|
||||
let toolCalls = [...params.plan.toolCalls];
|
||||
if (hasTool(params.availableTools, "useLatestResultImage")) {
|
||||
toolCalls = upsertTool(
|
||||
toolCalls,
|
||||
"useLatestResultImage",
|
||||
"image-edit 子代理优先绑定上一轮结果图作为连续修图目标",
|
||||
{ prepend: true }
|
||||
);
|
||||
}
|
||||
if (hasTool(params.availableTools, "editImage")) {
|
||||
toolCalls = upsertTool(
|
||||
toolCalls,
|
||||
"editImage",
|
||||
"image-edit 子代理要求优先走连续修图执行链路",
|
||||
{ prepend: true }
|
||||
);
|
||||
}
|
||||
if (hasTool(params.availableTools, "publishToCanvas")) {
|
||||
toolCalls = upsertTool(
|
||||
toolCalls,
|
||||
"publishToCanvas",
|
||||
"image-edit 子代理建议把修图结果同步回画布"
|
||||
);
|
||||
}
|
||||
|
||||
return {
|
||||
...params.plan,
|
||||
nextAction: "edit_image",
|
||||
intent: "image_edit",
|
||||
responseMode: "tool_then_execute",
|
||||
preferredCapability:
|
||||
params.plan.preferredCapability === "respond"
|
||||
? "generic_edit"
|
||||
: params.plan.preferredCapability,
|
||||
shouldUseStructuredReply: true,
|
||||
toolCalls,
|
||||
};
|
||||
}
|
||||
131
lib/ai/image-generation-subagent.ts
Normal file
131
lib/ai/image-generation-subagent.ts
Normal file
@ -0,0 +1,131 @@
|
||||
import "server-only";
|
||||
|
||||
import type { AgentThreadState } from "@/lib/ai/runtime/agent-thread-state";
|
||||
import type { AgentExecutionPlan, AgentPlannedToolCall } from "@/lib/ai/auto-agent-graph";
|
||||
import type { AgentSpecialistRoute } from "@/lib/ai/specialists";
|
||||
import type { AgentProjectToolDefinition } from "@/lib/ai/tools-registry";
|
||||
import {
|
||||
getImageWorkflowPlanningHints,
|
||||
} from "@/lib/ai/image-workflow-memory";
|
||||
import {
|
||||
getPromptMemoryPlanningHints,
|
||||
isPromptCarryForwardRequest,
|
||||
} from "@/lib/ai/prompt-memory";
|
||||
|
||||
function normalizeCompactText(text: string) {
|
||||
return String(text || "").toLowerCase().replace(/\s+/g, "").trim();
|
||||
}
|
||||
|
||||
export function buildImageGenerationSpecialistRoute(params: {
|
||||
text: string;
|
||||
threadState?: AgentThreadState | null;
|
||||
hasImage: boolean;
|
||||
}): AgentSpecialistRoute | null {
|
||||
const normalized = normalizeCompactText(params.text);
|
||||
const promptHints = getPromptMemoryPlanningHints(params.threadState);
|
||||
const workflowHints = getImageWorkflowPlanningHints(params.threadState);
|
||||
const hasPromptMemory = promptHints.length > 0;
|
||||
const hasImageWorkflowMemory = workflowHints.length > 0;
|
||||
const carryForward = isPromptCarryForwardRequest({
|
||||
text: params.text,
|
||||
hasImage: params.hasImage,
|
||||
});
|
||||
const mentionsPromptExecution =
|
||||
/提示词|prompt|关键词|咒语/.test(normalized) &&
|
||||
/生成|生图|出图|继续做|继续生成|执行/.test(normalized);
|
||||
|
||||
if (!hasPromptMemory && !hasImageWorkflowMemory && !carryForward && !mentionsPromptExecution) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const planningHints = [
|
||||
carryForward
|
||||
? "用户当前要求沿用上一轮优化提示词继续执行，应优先理解为连续生图执行。"
|
||||
: "",
|
||||
hasPromptMemory
|
||||
? "当前会话里存在上一轮可复用提示词，如果用户在说“上一个提示词/刚才那个 prompt”，优先复用这段提示词。"
|
||||
: "",
|
||||
hasImageWorkflowMemory
|
||||
? "当前会话里存在图片工作流记忆，优先参考上一轮参考图、结果图、数量和画幅偏好。"
|
||||
: "",
|
||||
params.hasImage
|
||||
? "如果当前回合上传了新图片，并要求沿用上一轮提示词，应理解为“上一轮提示词 + 当前图片条件”的联合执行。"
|
||||
: "",
|
||||
...promptHints,
|
||||
...workflowHints,
|
||||
].filter(Boolean);
|
||||
|
||||
return {
|
||||
domain: "image_generation_memory",
|
||||
specialistId: "image-generation-subagent",
|
||||
reason: carryForward ? "当前回合命中提示词接力生图" : "当前回合命中提示词相关的连续生图任务",
|
||||
planningHints,
|
||||
toolAllowlist: [
|
||||
"respondDirect",
|
||||
"generateImage",
|
||||
"useSelectedCanvasImage",
|
||||
"useLatestResultImage",
|
||||
"useOriginalUploadImage",
|
||||
"publishToCanvas",
|
||||
],
|
||||
};
|
||||
}
|
||||
|
||||
function hasTool(
|
||||
availableTools: AgentProjectToolDefinition[],
|
||||
name: AgentPlannedToolCall["name"],
|
||||
) {
|
||||
return availableTools.some((tool) => tool.name === name);
|
||||
}
|
||||
|
||||
function upsertTool(
|
||||
toolCalls: AgentPlannedToolCall[],
|
||||
name: AgentPlannedToolCall["name"],
|
||||
reason: string,
|
||||
options?: { prepend?: boolean },
|
||||
) {
|
||||
const existing = toolCalls.find((tool) => tool.name === name);
|
||||
if (existing) return toolCalls;
|
||||
const nextTool = { name, reason } satisfies AgentPlannedToolCall;
|
||||
return options?.prepend ? [nextTool, ...toolCalls] : [...toolCalls, nextTool];
|
||||
}
|
||||
|
||||
export function refinePlanWithImageGenerationSubagent(params: {
|
||||
text: string;
|
||||
plan: AgentExecutionPlan;
|
||||
availableTools: AgentProjectToolDefinition[];
|
||||
threadState?: AgentThreadState | null;
|
||||
hasImage: boolean;
|
||||
}): AgentExecutionPlan {
|
||||
const carryForward = isPromptCarryForwardRequest({
|
||||
text: params.text,
|
||||
hasImage: params.hasImage,
|
||||
});
|
||||
const hasPromptMemory = getPromptMemoryPlanningHints(params.threadState).length > 0;
|
||||
const hasImageWorkflowMemory = getImageWorkflowPlanningHints(params.threadState).length > 0;
|
||||
if (!carryForward && !hasPromptMemory && !hasImageWorkflowMemory) {
|
||||
return params.plan;
|
||||
}
|
||||
|
||||
let toolCalls = [...params.plan.toolCalls];
|
||||
if (hasTool(params.availableTools, "generateImage")) {
|
||||
toolCalls = [{ name: "generateImage", reason: "image-generation 子代理要求优先走连续生图执行链路" }];
|
||||
}
|
||||
if (hasTool(params.availableTools, "publishToCanvas")) {
|
||||
toolCalls = upsertTool(
|
||||
toolCalls,
|
||||
"publishToCanvas",
|
||||
"image-generation 子代理建议把连续生图结果同步到画布"
|
||||
);
|
||||
}
|
||||
|
||||
return {
|
||||
...params.plan,
|
||||
nextAction: "generate_image",
|
||||
intent: "image_gen",
|
||||
responseMode: "tool_then_execute",
|
||||
preferredCapability: "generate_image",
|
||||
shouldUseStructuredReply: true,
|
||||
toolCalls,
|
||||
};
|
||||
}
|
||||
159
lib/ai/image-workflow-memory.ts
Normal file
159
lib/ai/image-workflow-memory.ts
Normal file
@ -0,0 +1,159 @@
|
||||
import "server-only";
|
||||
|
||||
import type { AgentThreadState } from "@/lib/ai/runtime/agent-thread-state";
|
||||
|
||||
export interface ImageWorkflowMemory {
|
||||
lastIntent?: "image_gen" | "image_edit" | "vision";
|
||||
lastOptimizedPrompt?: string;
|
||||
lastUserPrompt?: string;
|
||||
lastRequestedImageCount?: number;
|
||||
lastAspectRatio?: string;
|
||||
lastImageSize?: string;
|
||||
lastSourceImageUrls?: string[];
|
||||
lastResultImageUrls?: string[];
|
||||
}
|
||||
|
||||
function normalizeText(value: unknown, max = 2000): string | undefined {
|
||||
const normalized = String(value || "").trim();
|
||||
if (!normalized) return undefined;
|
||||
return normalized.slice(0, max);
|
||||
}
|
||||
|
||||
function normalizeImageUrls(value: unknown): string[] | undefined {
|
||||
if (!Array.isArray(value)) return undefined;
|
||||
const urls = Array.from(
|
||||
new Set(
|
||||
value
|
||||
.map((item) => String(item || "").trim())
|
||||
.filter((url) => /^https?:\/\//i.test(url))
|
||||
)
|
||||
);
|
||||
return urls.length > 0 ? urls : undefined;
|
||||
}
|
||||
|
||||
function normalizeImageCount(value: unknown): number | undefined {
|
||||
const parsed = Number(value);
|
||||
if (!Number.isFinite(parsed)) return undefined;
|
||||
const rounded = Math.round(parsed);
|
||||
if (rounded < 1 || rounded > 12) return undefined;
|
||||
return rounded;
|
||||
}
|
||||
|
||||
export function extractImageWorkflowMemory(
|
||||
workingState?: {
|
||||
lastTurnSummary?: unknown;
|
||||
lastResolvedTargetImageUrls?: string[] | null;
|
||||
lastGeneratedImageUrls?: string[] | null;
|
||||
} | null,
|
||||
): ImageWorkflowMemory | null {
|
||||
const summary = workingState?.lastTurnSummary;
|
||||
let memory: ImageWorkflowMemory | null = null;
|
||||
|
||||
if (summary && typeof summary === "object") {
|
||||
const summaryRecord = summary as Record<string, unknown>;
|
||||
const imageWorkflowMemory = summaryRecord.imageWorkflowMemory;
|
||||
if (imageWorkflowMemory && typeof imageWorkflowMemory === "object") {
|
||||
const record = imageWorkflowMemory as Record<string, unknown>;
|
||||
memory = {
|
||||
lastIntent:
|
||||
record.lastIntent === "image_gen" ||
|
||||
record.lastIntent === "image_edit" ||
|
||||
record.lastIntent === "vision"
|
||||
? record.lastIntent
|
||||
: undefined,
|
||||
lastOptimizedPrompt: normalizeText(record.lastOptimizedPrompt, 4000),
|
||||
lastUserPrompt: normalizeText(record.lastUserPrompt, 1000),
|
||||
lastRequestedImageCount: normalizeImageCount(record.lastRequestedImageCount),
|
||||
lastAspectRatio: normalizeText(record.lastAspectRatio, 32),
|
||||
lastImageSize: normalizeText(record.lastImageSize, 32),
|
||||
lastSourceImageUrls: normalizeImageUrls(record.lastSourceImageUrls),
|
||||
lastResultImageUrls: normalizeImageUrls(record.lastResultImageUrls),
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
const fallbackTargetUrls = normalizeImageUrls(workingState?.lastResolvedTargetImageUrls);
|
||||
const fallbackGeneratedUrls = normalizeImageUrls(workingState?.lastGeneratedImageUrls);
|
||||
if (!memory) {
|
||||
if (!fallbackTargetUrls && !fallbackGeneratedUrls) return null;
|
||||
memory = {};
|
||||
}
|
||||
|
||||
if (!memory.lastSourceImageUrls && fallbackTargetUrls) {
|
||||
memory.lastSourceImageUrls = fallbackTargetUrls;
|
||||
}
|
||||
if (!memory.lastResultImageUrls && fallbackGeneratedUrls) {
|
||||
memory.lastResultImageUrls = fallbackGeneratedUrls;
|
||||
}
|
||||
|
||||
return memory;
|
||||
}
|
||||
|
||||
export function buildImageWorkflowMemoryUpdate(params: {
|
||||
existing?: ImageWorkflowMemory | null;
|
||||
lastIntent?: "image_gen" | "image_edit" | "vision";
|
||||
lastOptimizedPrompt?: string;
|
||||
lastUserPrompt?: string;
|
||||
lastRequestedImageCount?: number;
|
||||
lastAspectRatio?: string;
|
||||
lastImageSize?: string;
|
||||
lastSourceImageUrls?: string[] | null;
|
||||
lastResultImageUrls?: string[] | null;
|
||||
}) {
|
||||
return {
|
||||
lastIntent: params.lastIntent ?? params.existing?.lastIntent,
|
||||
lastOptimizedPrompt:
|
||||
normalizeText(params.lastOptimizedPrompt, 4000) ??
|
||||
params.existing?.lastOptimizedPrompt,
|
||||
lastUserPrompt:
|
||||
normalizeText(params.lastUserPrompt, 1000) ?? params.existing?.lastUserPrompt,
|
||||
lastRequestedImageCount:
|
||||
normalizeImageCount(params.lastRequestedImageCount) ??
|
||||
params.existing?.lastRequestedImageCount,
|
||||
lastAspectRatio:
|
||||
normalizeText(params.lastAspectRatio, 32) ?? params.existing?.lastAspectRatio,
|
||||
lastImageSize:
|
||||
normalizeText(params.lastImageSize, 32) ?? params.existing?.lastImageSize,
|
||||
lastSourceImageUrls:
|
||||
normalizeImageUrls(params.lastSourceImageUrls) ??
|
||||
params.existing?.lastSourceImageUrls,
|
||||
lastResultImageUrls:
|
||||
normalizeImageUrls(params.lastResultImageUrls) ??
|
||||
params.existing?.lastResultImageUrls,
|
||||
} satisfies ImageWorkflowMemory;
|
||||
}
|
||||
|
||||
export function getImageWorkflowPlanningHints(
|
||||
threadState?: AgentThreadState | null,
|
||||
): string[] {
|
||||
const metadata = threadState?.metadata;
|
||||
const memory =
|
||||
metadata && typeof metadata === "object" && !Array.isArray(metadata)
|
||||
? ((metadata.imageWorkflowMemory as ImageWorkflowMemory | undefined) || undefined)
|
||||
: undefined;
|
||||
if (!memory) return [];
|
||||
|
||||
const hints: string[] = [];
|
||||
if (memory.lastOptimizedPrompt) {
|
||||
hints.push(
|
||||
`上一轮可复用提示词摘要：${memory.lastOptimizedPrompt.replace(/\s+/g, " ").slice(0, 180)}`
|
||||
);
|
||||
}
|
||||
if (memory.lastRequestedImageCount) {
|
||||
hints.push(`用户上一轮期望出图数量：${memory.lastRequestedImageCount} 张。`);
|
||||
}
|
||||
if (memory.lastAspectRatio) {
|
||||
hints.push(`用户上一轮常用画幅：${memory.lastAspectRatio}。`);
|
||||
}
|
||||
if (memory.lastImageSize) {
|
||||
hints.push(`用户上一轮常用清晰度：${memory.lastImageSize}。`);
|
||||
}
|
||||
if (memory.lastSourceImageUrls?.length) {
|
||||
hints.push(`上一轮存在 ${memory.lastSourceImageUrls.length} 张参考/目标图。`);
|
||||
}
|
||||
if (memory.lastResultImageUrls?.length) {
|
||||
hints.push(`上一轮生成结果数：${memory.lastResultImageUrls.length} 张。`);
|
||||
}
|
||||
return hints;
|
||||
}
|
||||
|
||||
@ -1,5 +1,7 @@
|
||||
import "server-only";
|
||||
|
||||
import type { AgentThreadState } from "@/lib/ai/runtime/agent-thread-state";
|
||||
|
||||
export interface PromptMemoryEntry {
|
||||
optimizedPrompt: string;
|
||||
sourceIntent?: string;
|
||||
@ -48,6 +50,24 @@ export function extractPromptMemoryFromWorkingState(
|
||||
return null;
|
||||
}
|
||||
|
||||
export function extractConfirmedPromptFromWorkingState(
|
||||
workingState?: {
|
||||
lastTurnSummary?: unknown;
|
||||
} | null,
|
||||
): string | undefined {
|
||||
const summary = workingState?.lastTurnSummary;
|
||||
if (!summary || typeof summary !== "object") return undefined;
|
||||
const summaryRecord = summary as Record<string, unknown>;
|
||||
|
||||
const resolutionMode =
|
||||
typeof summaryRecord.resolutionMode === "string" ? summaryRecord.resolutionMode.trim() : "";
|
||||
const optimizedPrompt =
|
||||
typeof summaryRecord.optimizedPrompt === "string" ? summaryRecord.optimizedPrompt.trim() : "";
|
||||
|
||||
if (resolutionMode !== "prompt_only" || !optimizedPrompt) return undefined;
|
||||
return optimizedPrompt;
|
||||
}
|
||||
|
||||
export function buildPromptMemoryUpdate(params: {
|
||||
optimizedPrompt: string;
|
||||
sourceIntent?: string;
|
||||
@ -90,3 +110,24 @@ export function isPromptCarryForwardRequest(params: {
|
||||
return false;
|
||||
}
|
||||
|
||||
export function getPromptMemoryPlanningHints(
|
||||
threadState?: AgentThreadState | null,
|
||||
): string[] {
|
||||
const metadata = threadState?.metadata;
|
||||
const memory =
|
||||
metadata && typeof metadata === "object" && !Array.isArray(metadata)
|
||||
? ((metadata.promptMemory as PromptMemoryEntry | undefined) || undefined)
|
||||
: undefined;
|
||||
|
||||
if (!memory?.optimizedPrompt) return [];
|
||||
|
||||
const preview = memory.optimizedPrompt.replace(/\s+/g, " ").trim().slice(0, 180);
|
||||
const hints = [
|
||||
"当前会话中存在上一轮可复用的优化提示词；如果用户明确要求沿用、根据上一个提示词继续做图，应优先复用这段提示词。",
|
||||
`上一轮优化提示词摘要：${preview}`,
|
||||
];
|
||||
if (memory.sourceText) {
|
||||
hints.push(`该提示词来源于用户上一轮需求：${memory.sourceText}`);
|
||||
}
|
||||
return hints;
|
||||
}
|
||||
|
||||
@ -394,7 +394,7 @@ function sanitizeSuggestedParams(input?: {
|
||||
const normalizedSize = input.imageSize?.trim().toUpperCase() || undefined;
|
||||
const normalizedCount =
|
||||
typeof input.imageCount === "number" && Number.isFinite(input.imageCount)
|
||||
? Math.min(Math.max(Math.round(input.imageCount), 1), 4)
|
||||
? Math.min(Math.max(Math.round(input.imageCount), 1), 12)
|
||||
: undefined;
|
||||
const output: NonNullable<OptimizeResult["recommendedImageParams"]> = {};
|
||||
if (normalizedAspectRatio && allowedAspectRatios.has(normalizedAspectRatio)) {
|
||||
|
||||
@ -1,5 +1,7 @@
|
||||
export type AgentWorkflowType =
|
||||
| "storyboard_video"
|
||||
| "image_generation_followup"
|
||||
| "image_edit_followup"
|
||||
| "high_cost_video"
|
||||
| "image_batch"
|
||||
| "custom";
|
||||
@ -111,4 +113,3 @@ export function isAwaitingUserConfirmationWorkflow(
|
||||
): boolean {
|
||||
return workflowState?.status === "awaiting_user_confirmation";
|
||||
}
|
||||
|
||||
|
||||
@ -6,6 +6,8 @@ import type {
|
||||
AgentSpecialistContext,
|
||||
AgentSpecialistRoute,
|
||||
} from "@/lib/ai/specialists";
|
||||
import { buildImageEditSpecialistRoute } from "@/lib/ai/image-edit-subagent";
|
||||
import { buildImageGenerationSpecialistRoute } from "@/lib/ai/image-generation-subagent";
|
||||
import { buildStoryboardSpecialistRoute } from "@/lib/video-agent/storyboard-subagent";
|
||||
|
||||
export interface AgentSkillRouteResult {
|
||||
@ -215,9 +217,22 @@ export function routeAgentSkill(
|
||||
text: context.text,
|
||||
threadState: context.threadState,
|
||||
});
|
||||
const mergedOverlays = storyboardOverlay
|
||||
? [...overlays, storyboardOverlay]
|
||||
: overlays;
|
||||
const imageEditOverlay = buildImageEditSpecialistRoute({
|
||||
text: context.text,
|
||||
threadState: context.threadState,
|
||||
hasImage: context.hasImage,
|
||||
});
|
||||
const imageGenerationOverlay = buildImageGenerationSpecialistRoute({
|
||||
text: context.text,
|
||||
threadState: context.threadState,
|
||||
hasImage: context.hasImage,
|
||||
});
|
||||
const mergedOverlays = [
|
||||
...overlays,
|
||||
...(imageEditOverlay ? [imageEditOverlay] : []),
|
||||
...(imageGenerationOverlay ? [imageGenerationOverlay] : []),
|
||||
...(storyboardOverlay ? [storyboardOverlay] : []),
|
||||
];
|
||||
|
||||
const toolAllowlist = dedupeToolNames([
|
||||
...(primary.toolAllowlist || []),
|
||||
|
||||
@ -1,5 +1,7 @@
|
||||
import type { AnalysisResult } from "@/lib/ai/smart-analyzer";
|
||||
import type { AgentThreadState } from "@/lib/ai/runtime/agent-thread-state";
|
||||
import { refinePlanWithImageEditSubagent } from "@/lib/ai/image-edit-subagent";
|
||||
import { refinePlanWithImageGenerationSubagent } from "@/lib/ai/image-generation-subagent";
|
||||
import { refinePlanWithStoryboardSubagent } from "@/lib/video-agent/storyboard-subagent";
|
||||
import type {
|
||||
AgentExecutionPlan,
|
||||
@ -18,7 +20,9 @@ export type AgentSpecialistDomain =
|
||||
| "conversation"
|
||||
| "vision"
|
||||
| "image_edit"
|
||||
| "image_edit_memory"
|
||||
| "image_generate"
|
||||
| "image_generation_memory"
|
||||
| "video_generate"
|
||||
| "storyboard"
|
||||
| "canvas";
|
||||
@ -27,7 +31,9 @@ export type AgentSpecialistId =
|
||||
| "conversation-coordinator"
|
||||
| "vision-analyst"
|
||||
| "image-edit-specialist"
|
||||
| "image-edit-subagent"
|
||||
| "image-generate-specialist"
|
||||
| "image-generation-subagent"
|
||||
| "video-specialist"
|
||||
| "storyboard-subagent"
|
||||
| "canvas-specialist";
|
||||
@ -235,6 +241,16 @@ export function refinePlanWithSpecialist(
|
||||
});
|
||||
}
|
||||
|
||||
if (specialist.domain === "image_edit_memory") {
|
||||
return refinePlanWithImageEditSubagent({
|
||||
text,
|
||||
plan,
|
||||
availableTools,
|
||||
threadState: input.threadState,
|
||||
hasImage: input.hasImage,
|
||||
});
|
||||
}
|
||||
|
||||
if (specialist.domain === "image_generate") {
|
||||
if (hasTool(availableTools, "generateImage")) {
|
||||
toolCalls = [{ name: "generateImage", reason: "生图专家执行图片生成" }];
|
||||
@ -250,6 +266,16 @@ export function refinePlanWithSpecialist(
|
||||
});
|
||||
}
|
||||
|
||||
if (specialist.domain === "image_generation_memory") {
|
||||
return refinePlanWithImageGenerationSubagent({
|
||||
text,
|
||||
plan,
|
||||
availableTools,
|
||||
threadState: input.threadState,
|
||||
hasImage: input.hasImage,
|
||||
});
|
||||
}
|
||||
|
||||
if (specialist.domain === "video_generate") {
|
||||
if (hasTool(availableTools, "generateVideo")) {
|
||||
toolCalls = [{ name: "generateVideo", reason: "视频专家执行视频生成" }];
|
||||
|
||||
@ -71,6 +71,22 @@ function parseImagesFromMarkdown(markdown: string): ParsedImage[] {
|
||||
return parsed;
|
||||
}
|
||||
|
||||
function getProviderBatchLimit(providerType: string) {
|
||||
switch (providerType) {
|
||||
case "ruxa-image":
|
||||
case "openai-image":
|
||||
case "midjourney":
|
||||
return 1;
|
||||
case "gemini":
|
||||
return 2;
|
||||
case "alibaba-image":
|
||||
case "fal-image":
|
||||
return 6;
|
||||
default:
|
||||
return 2;
|
||||
}
|
||||
}
|
||||
|
||||
export async function generateCanvasImage(
|
||||
request: CanvasGenerateImageRequest
|
||||
): Promise<CanvasGenerateImageResult> {
|
||||
@ -92,7 +108,7 @@ export async function generateCanvasImage(
|
||||
: "2K";
|
||||
const imageCount =
|
||||
typeof request.imageCount === "number" && Number.isFinite(request.imageCount)
|
||||
? Math.max(1, Math.min(8, Math.floor(request.imageCount)))
|
||||
? Math.max(1, Math.min(12, Math.floor(request.imageCount)))
|
||||
: 1;
|
||||
const referenceImage =
|
||||
typeof request.referenceImage === "string" ? request.referenceImage : undefined;
|
||||
@ -191,11 +207,13 @@ export async function generateCanvasImage(
|
||||
const imageParams = {
|
||||
aspectRatio,
|
||||
imageSize,
|
||||
imageCount,
|
||||
imageCount: 1,
|
||||
negativePrompt,
|
||||
};
|
||||
|
||||
let imageMarkdown = "";
|
||||
let executeGenerateBatch:
|
||||
| ((batchImageCount: number) => Promise<string>)
|
||||
| null = null;
|
||||
|
||||
if (provider.type === "gemini") {
|
||||
const geminiProvider = createGeminiImage({
|
||||
@ -203,61 +221,128 @@ export async function generateCanvasImage(
|
||||
baseURL: provider.baseUrl,
|
||||
});
|
||||
const modelInstance = geminiProvider.chat(selectedModel.model.modelId);
|
||||
const result = await modelInstance.doGenerate({ messages, imageParams });
|
||||
imageMarkdown = result.text || "";
|
||||
executeGenerateBatch = async (batchImageCount) => {
|
||||
const result = await modelInstance.doGenerate({
|
||||
messages,
|
||||
imageParams: {
|
||||
...imageParams,
|
||||
imageCount: batchImageCount,
|
||||
},
|
||||
});
|
||||
return result.text || "";
|
||||
};
|
||||
} else if (provider.type === "alibaba-image") {
|
||||
const alibabaProvider = createAlibabaImage({
|
||||
apiKey: provider.apiKey,
|
||||
baseURL: provider.baseUrl,
|
||||
});
|
||||
const modelInstance = alibabaProvider.chat(selectedModel.model.modelId);
|
||||
const result = await modelInstance.doGenerate({ messages, imageParams });
|
||||
imageMarkdown = result.text || "";
|
||||
executeGenerateBatch = async (batchImageCount) => {
|
||||
const result = await modelInstance.doGenerate({
|
||||
messages,
|
||||
imageParams: {
|
||||
...imageParams,
|
||||
imageCount: batchImageCount,
|
||||
},
|
||||
});
|
||||
return result.text || "";
|
||||
};
|
||||
} else if (provider.type === "ruxa-image") {
|
||||
const ruxaProvider = createRuxaImage({
|
||||
apiKey: provider.apiKey,
|
||||
baseURL: provider.baseUrl,
|
||||
});
|
||||
const modelInstance = ruxaProvider.chat(selectedModel.model.modelId);
|
||||
const result = await modelInstance.doGenerate({ messages, imageParams });
|
||||
imageMarkdown = result.text || "";
|
||||
executeGenerateBatch = async (batchImageCount) => {
|
||||
const result = await modelInstance.doGenerate({
|
||||
messages,
|
||||
imageParams: {
|
||||
...imageParams,
|
||||
imageCount: batchImageCount,
|
||||
},
|
||||
});
|
||||
return result.text || "";
|
||||
};
|
||||
} else if (provider.type === "fal-image") {
|
||||
const falProvider = createFalImage({
|
||||
apiKey: provider.apiKey,
|
||||
baseURL: provider.baseUrl,
|
||||
});
|
||||
const modelInstance = falProvider.chat(selectedModel.model.modelId);
|
||||
const result = await modelInstance.doGenerate({ messages, imageParams });
|
||||
imageMarkdown = result.text || "";
|
||||
executeGenerateBatch = async (batchImageCount) => {
|
||||
const result = await modelInstance.doGenerate({
|
||||
messages,
|
||||
imageParams: {
|
||||
...imageParams,
|
||||
imageCount: batchImageCount,
|
||||
},
|
||||
});
|
||||
return result.text || "";
|
||||
};
|
||||
} else if (provider.type === "openai-image") {
|
||||
const openAIImageProvider = createOpenAIImage({
|
||||
apiKey: provider.apiKey,
|
||||
baseURL: provider.baseUrl,
|
||||
});
|
||||
const modelInstance = openAIImageProvider.chat(selectedModel.model.modelId);
|
||||
const result = await modelInstance.doGenerate({ messages, imageParams });
|
||||
imageMarkdown = result.text || "";
|
||||
executeGenerateBatch = async (batchImageCount) => {
|
||||
const result = await modelInstance.doGenerate({
|
||||
messages,
|
||||
imageParams: {
|
||||
...imageParams,
|
||||
imageCount: batchImageCount,
|
||||
},
|
||||
});
|
||||
return result.text || "";
|
||||
};
|
||||
} else if (provider.type === "midjourney") {
|
||||
const midjourneyProvider = createMidjourney({
|
||||
apiKey: provider.apiKey,
|
||||
baseURL: provider.baseUrl,
|
||||
});
|
||||
const modelInstance = midjourneyProvider.chat(selectedModel.model.modelId);
|
||||
const result = await modelInstance.doGenerate({ messages, imageParams });
|
||||
imageMarkdown = result.text || "";
|
||||
executeGenerateBatch = async (batchImageCount) => {
|
||||
const result = await modelInstance.doGenerate({
|
||||
messages,
|
||||
imageParams: {
|
||||
...imageParams,
|
||||
imageCount: batchImageCount,
|
||||
},
|
||||
});
|
||||
return result.text || "";
|
||||
};
|
||||
} else {
|
||||
throw new Error(
|
||||
`不支持的图片生成模型类型: ${provider.type || "unknown"}，请改用 Gemini/阿里/RUXA/FAL/OpenAI/Midjourney 图片模型`
|
||||
);
|
||||
}
|
||||
|
||||
const parsedImages = parseImagesFromMarkdown(imageMarkdown);
|
||||
if (parsedImages.length === 0) {
|
||||
const providerBatchLimit = getProviderBatchLimit(provider.type);
|
||||
const targetImageCount = Math.max(1, imageCount);
|
||||
const collectedImages: ParsedImage[] = [];
|
||||
const maxBatches = Math.max(1, Math.ceil(targetImageCount / providerBatchLimit));
|
||||
|
||||
for (let batchIndex = 0; batchIndex < maxBatches; batchIndex += 1) {
|
||||
const remaining = targetImageCount - collectedImages.length;
|
||||
if (remaining <= 0) break;
|
||||
const batchImageCount = Math.min(providerBatchLimit, remaining);
|
||||
const imageMarkdown = await executeGenerateBatch(batchImageCount);
|
||||
const parsedImages = parseImagesFromMarkdown(imageMarkdown);
|
||||
if (parsedImages.length === 0) {
|
||||
if (collectedImages.length > 0) {
|
||||
break;
|
||||
}
|
||||
throw new Error("图片生成失败");
|
||||
}
|
||||
collectedImages.push(...parsedImages.slice(0, batchImageCount));
|
||||
}
|
||||
|
||||
if (collectedImages.length === 0) {
|
||||
throw new Error("图片生成失败");
|
||||
}
|
||||
|
||||
return {
|
||||
images: parsedImages,
|
||||
images: collectedImages.slice(0, targetImageCount),
|
||||
model: selectedModel.model.displayName,
|
||||
modelRef: selectedModel.model.id,
|
||||
prompt,
|
||||
|
||||
@ -596,8 +596,11 @@ export const canvasProject = pgTable("CanvasProject", {
|
||||
// 存储完整画布状态（节点、边、视口等）
|
||||
canvasData: jsonb("canvasData").$type<{
|
||||
nodes: any[];
|
||||
edges: any[];
|
||||
viewport: { x: number; y: number; zoom: number };
|
||||
edges?: any[];
|
||||
selectedNodeIds?: string[];
|
||||
viewport?: { x: number; y: number; zoom: number };
|
||||
tool?: string;
|
||||
suppressedMediaKeys?: string[];
|
||||
}>(),
|
||||
createdAt: timestamp("createdAt").notNull().defaultNow(),
|
||||
updatedAt: timestamp("updatedAt").notNull().defaultNow(),
|
||||
|
||||
@ -1,5 +1,6 @@
|
||||
import type { AgentThreadState } from "@/lib/ai/runtime/agent-thread-state";
|
||||
import type { AgentWorkflowStatus } from "@/lib/ai/runtime/agent-workflow-state";
|
||||
import type { PromptMemoryEntry } from "@/lib/ai/prompt-memory";
|
||||
import type { StoryboardPreferenceMemory } from "@/lib/video-agent/storyboard-preference-memory";
|
||||
|
||||
type StoryboardTaskDetail = {
|
||||
@ -69,18 +70,25 @@ export function buildAgentThreadStateFromStoryboardTask(
|
||||
detail: StoryboardTaskDetail | null | undefined,
|
||||
options?: {
|
||||
storyboardPreferenceMemory?: StoryboardPreferenceMemory | null;
|
||||
promptMemory?: PromptMemoryEntry | null;
|
||||
},
|
||||
): AgentThreadState | null {
|
||||
if (!detail?.task?.id) {
|
||||
if (!options?.storyboardPreferenceMemory) return null;
|
||||
if (!options?.storyboardPreferenceMemory && !options?.promptMemory) return null;
|
||||
return {
|
||||
workflowState: null,
|
||||
memoryHints: [
|
||||
"当前会话里存在 storyboard 偏好记忆，可在分镜/宫格任务里优先参考用户常用版式和确认习惯。",
|
||||
],
|
||||
options?.storyboardPreferenceMemory
|
||||
? "当前会话里存在 storyboard 偏好记忆，可在分镜/宫格任务里优先参考用户常用版式和确认习惯。"
|
||||
: "",
|
||||
options?.promptMemory?.optimizedPrompt
|
||||
? "当前会话里存在上一轮可复用提示词；如果用户明确要求沿用上一个提示词继续做图，应优先把它作为本轮生图提示词。"
|
||||
: "",
|
||||
].filter(Boolean),
|
||||
metadata: {
|
||||
workflowSource: "storyboard_preference_memory",
|
||||
storyboardPreferenceMemory: options.storyboardPreferenceMemory,
|
||||
promptMemory: options?.promptMemory || undefined,
|
||||
},
|
||||
};
|
||||
}
|
||||
@ -147,6 +155,7 @@ export function buildAgentThreadStateFromStoryboardTask(
|
||||
taskId: detail.task.id,
|
||||
shotCount: totalShots,
|
||||
storyboardPreferenceMemory: options?.storyboardPreferenceMemory || undefined,
|
||||
promptMemory: options?.promptMemory || undefined,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
@ -73,6 +73,7 @@ interface CanvasStore {
|
||||
selectedNodeIds: string[];
|
||||
viewport: Viewport;
|
||||
tool: ToolType;
|
||||
suppressedMediaKeys: string[];
|
||||
|
||||
// 历史记录（用于撤销/重做）
|
||||
history: CanvasState[];
|
||||
@ -105,6 +106,7 @@ interface CanvasStore {
|
||||
updateNode: (id: string, updates: Partial<CanvasNode>) => void;
|
||||
deleteNode: (id: string) => void;
|
||||
deleteSelectedNodes: () => void;
|
||||
addSuppressedMediaKeys: (keys: string[]) => void;
|
||||
|
||||
// 选择操作
|
||||
selectNode: (id: string, multi?: boolean) => void;
|
||||
@ -179,6 +181,7 @@ export const useCanvasStore = create<CanvasStore>()(
|
||||
selectedNodeIds: [],
|
||||
viewport: defaultViewport,
|
||||
tool: 'select',
|
||||
suppressedMediaKeys: [],
|
||||
history: [],
|
||||
historyIndex: -1,
|
||||
isPanelOpen: true,
|
||||
@ -287,8 +290,36 @@ export const useCanvasStore = create<CanvasStore>()(
|
||||
}
|
||||
}),
|
||||
|
||||
addSuppressedMediaKeys: (keys) => set((state) => {
|
||||
const normalized = Array.from(
|
||||
new Set((keys || []).map((item) => String(item || "").trim()).filter(Boolean))
|
||||
);
|
||||
if (normalized.length === 0) return;
|
||||
const existing = new Set(state.suppressedMediaKeys);
|
||||
normalized.forEach((key) => existing.add(key));
|
||||
state.suppressedMediaKeys = Array.from(existing);
|
||||
}),
|
||||
|
||||
deleteNode: (id) => {
|
||||
set((state) => {
|
||||
const targetNode = state.nodes.find(n => n.id === id);
|
||||
if (targetNode && (targetNode.type === 'image' || targetNode.type === 'video')) {
|
||||
const data = ((targetNode.data as any) || {}) as Record<string, any>;
|
||||
const metadata = ((data.metadata as any) || {}) as Record<string, any>;
|
||||
const sourceMessageId = String(metadata.sourceMessageId || "").trim();
|
||||
const sourceImageKey = String(metadata.sourceImageKey || "").trim();
|
||||
const sourceMediaUrl = String(data.src || "").trim();
|
||||
const suppressionKeys = [
|
||||
sourceMessageId && sourceImageKey ? `image:${sourceMessageId}::${sourceImageKey}` : "",
|
||||
targetNode.type === 'video' && sourceMediaUrl ? `video:${sourceMediaUrl}` : "",
|
||||
targetNode.type === 'image' && sourceMediaUrl ? `image-url:${sourceMediaUrl}` : "",
|
||||
].filter(Boolean);
|
||||
if (suppressionKeys.length > 0) {
|
||||
const existing = new Set(state.suppressedMediaKeys);
|
||||
suppressionKeys.forEach((key) => existing.add(key));
|
||||
state.suppressedMediaKeys = Array.from(existing);
|
||||
}
|
||||
}
|
||||
state.nodes = state.nodes.filter(n => n.id !== id);
|
||||
state.edges = state.edges.filter(e => e.source !== id && e.target !== id);
|
||||
state.selectedNodeIds = state.selectedNodeIds.filter(nid => nid !== id);
|
||||
@ -299,6 +330,23 @@ export const useCanvasStore = create<CanvasStore>()(
|
||||
deleteSelectedNodes: () => {
|
||||
const { selectedNodeIds } = get();
|
||||
set((state) => {
|
||||
const selectedNodes = state.nodes.filter(n => selectedNodeIds.includes(n.id));
|
||||
const existing = new Set(state.suppressedMediaKeys);
|
||||
selectedNodes.forEach((targetNode) => {
|
||||
if (targetNode.type !== 'image' && targetNode.type !== 'video') return;
|
||||
const data = ((targetNode.data as any) || {}) as Record<string, any>;
|
||||
const metadata = ((data.metadata as any) || {}) as Record<string, any>;
|
||||
const sourceMessageId = String(metadata.sourceMessageId || "").trim();
|
||||
const sourceImageKey = String(metadata.sourceImageKey || "").trim();
|
||||
const sourceMediaUrl = String(data.src || "").trim();
|
||||
const suppressionKeys = [
|
||||
sourceMessageId && sourceImageKey ? `image:${sourceMessageId}::${sourceImageKey}` : "",
|
||||
targetNode.type === 'video' && sourceMediaUrl ? `video:${sourceMediaUrl}` : "",
|
||||
targetNode.type === 'image' && sourceMediaUrl ? `image-url:${sourceMediaUrl}` : "",
|
||||
].filter(Boolean);
|
||||
suppressionKeys.forEach((key) => existing.add(key));
|
||||
});
|
||||
state.suppressedMediaKeys = Array.from(existing);
|
||||
state.nodes = state.nodes.filter(n => !selectedNodeIds.includes(n.id));
|
||||
state.edges = state.edges.filter(
|
||||
e => !selectedNodeIds.includes(e.source) && !selectedNodeIds.includes(e.target)
|
||||
@ -451,6 +499,9 @@ export const useCanvasStore = create<CanvasStore>()(
|
||||
if (canvasState.selectedNodeIds) state.selectedNodeIds = canvasState.selectedNodeIds;
|
||||
if (canvasState.viewport) state.viewport = canvasState.viewport;
|
||||
if (canvasState.tool) state.tool = canvasState.tool;
|
||||
state.suppressedMediaKeys = Array.isArray(canvasState.suppressedMediaKeys)
|
||||
? canvasState.suppressedMediaKeys.map((item) => String(item || "").trim()).filter(Boolean)
|
||||
: [];
|
||||
}),
|
||||
|
||||
clearCanvas: () => set((state) => {
|
||||
@ -458,6 +509,7 @@ export const useCanvasStore = create<CanvasStore>()(
|
||||
state.edges = [];
|
||||
state.selectedNodeIds = [];
|
||||
state.viewport = defaultViewport;
|
||||
state.suppressedMediaKeys = [];
|
||||
state.history = [];
|
||||
state.historyIndex = -1;
|
||||
}),
|
||||
@ -470,6 +522,7 @@ export const useCanvasStore = create<CanvasStore>()(
|
||||
selectedNodeIds: state.selectedNodeIds,
|
||||
viewport: state.viewport,
|
||||
tool: state.tool,
|
||||
suppressedMediaKeys: state.suppressedMediaKeys,
|
||||
};
|
||||
},
|
||||
|
||||
|
||||
@ -137,6 +137,7 @@ export interface CanvasState {
|
||||
selectedNodeIds: string[];
|
||||
viewport: Viewport;
|
||||
tool: ToolType;
|
||||
suppressedMediaKeys?: string[];
|
||||
}
|
||||
|
||||
// 画布项目
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user