feat(canvas): unify image composer with native Agent and honor generation controls
This commit is contained in:
parent
85aa711633
commit
4ee3e369c6
@ -1,3 +1,4 @@
|
|||||||
|
import { getStyleProImageModelEntries } from "@/lib/style-pro/model-options";
|
||||||
import { registerProjectAgentTools } from "@/lib/ai/canvas-agent-runtime/project-tools";
|
import { registerProjectAgentTools } from "@/lib/ai/canvas-agent-runtime/project-tools";
|
||||||
import { isZcodeWorkspaceAuthorized } from "@/lib/ai/canvas-agent-runtime/authorization";
|
import { isZcodeWorkspaceAuthorized } from "@/lib/ai/canvas-agent-runtime/authorization";
|
||||||
import {
|
import {
|
||||||
@ -67,6 +68,7 @@ export const runtime = "nodejs";
|
|||||||
export const maxDuration = 1200;
|
export const maxDuration = 1200;
|
||||||
const requestSchema = postRequestBodySchema.extend({
|
const requestSchema = postRequestBodySchema.extend({
|
||||||
resumeRunId: z.string().uuid().optional(),
|
resumeRunId: z.string().uuid().optional(),
|
||||||
|
selectedImageModel: z.string().min(1).max(180).optional(),
|
||||||
workspaceVariant: z.enum(["canvas", "video"]).default("canvas"),
|
workspaceVariant: z.enum(["canvas", "video"]).default("canvas"),
|
||||||
videoAgentSettings: videoAgentSettingsSchema.optional(),
|
videoAgentSettings: videoAgentSettingsSchema.optional(),
|
||||||
});
|
});
|
||||||
@ -161,6 +163,12 @@ export async function POST(request: Request) {
|
|||||||
? configured
|
? configured
|
||||||
: (await getMultimodalAiModelsWithProvider()).find((entry) => entry.model.supportsVision);
|
: (await getMultimodalAiModelsWithProvider()).find((entry) => entry.model.supportsVision);
|
||||||
const visionModel = visionEntry ? await getLanguageModelById(visionEntry.model.id) : null;
|
const visionModel = visionEntry ? await getLanguageModelById(visionEntry.model.id) : null;
|
||||||
|
// Validate the user's image choice independently of the Agent planning model.
|
||||||
|
if (body.selectedImageModel) {
|
||||||
|
const images = await getStyleProImageModelEntries();
|
||||||
|
if (!images.some(({ model }) => model.id === body.selectedImageModel && model.healthStatus !== "auto_disabled"))
|
||||||
|
throw new AgentRunError("你选择的生图模型当前不可用,请重新选择;不会自动换模型生成。", 400);
|
||||||
|
}
|
||||||
const assistantMessageId = crypto.randomUUID();
|
const assistantMessageId = crypto.randomUUID();
|
||||||
const previousVideoMessages =
|
const previousVideoMessages =
|
||||||
existingChat
|
existingChat
|
||||||
@ -205,6 +213,12 @@ export async function POST(request: Request) {
|
|||||||
snapshot: initialSnapshot,
|
snapshot: initialSnapshot,
|
||||||
});
|
});
|
||||||
const snapshot = run.snapshot;
|
const snapshot = run.snapshot;
|
||||||
|
snapshot.imagePreferences = {
|
||||||
|
...snapshot.imagePreferences,
|
||||||
|
// Each submitted turn captures the current controls, including switching back to Auto.
|
||||||
|
modelId: body.selectedImageModel,
|
||||||
|
parameters: body.imageParams || snapshot.imagePreferences?.parameters,
|
||||||
|
};
|
||||||
const taskSourceNodeIds = snapshot.sourceNodeIds ?? selectedNodeIds;
|
const taskSourceNodeIds = snapshot.sourceNodeIds ?? selectedNodeIds;
|
||||||
const context = {
|
const context = {
|
||||||
recentConversation: previousVideoMessages,
|
recentConversation: previousVideoMessages,
|
||||||
@ -221,7 +235,7 @@ export async function POST(request: Request) {
|
|||||||
title: node.data.title,
|
title: node.data.title,
|
||||||
})),
|
})),
|
||||||
skillInstructions: body.hiddenPrompt || undefined,
|
skillInstructions: body.hiddenPrompt || undefined,
|
||||||
imageParameters: body.imageParams,
|
imagePreferences: snapshot.imagePreferences,
|
||||||
targetReason: conversation.targetReason,
|
targetReason: conversation.targetReason,
|
||||||
previousConversationContext: conversation.previousContext,
|
previousConversationContext: conversation.previousContext,
|
||||||
};
|
};
|
||||||
@ -302,8 +316,9 @@ export async function POST(request: Request) {
|
|||||||
request,
|
request,
|
||||||
model: visionModel || model,
|
model: visionModel || model,
|
||||||
supportsVision: Boolean(visionModel),
|
supportsVision: Boolean(visionModel),
|
||||||
imageSize: body.imageParams?.imageSize,
|
imageSize: snapshot.imagePreferences?.parameters?.imageSize,
|
||||||
preferredModelId: preset?.type === "image" ? preset.defaultModelId || undefined : undefined,
|
imagePreferences: snapshot.imagePreferences,
|
||||||
|
preferredModelId: snapshot.imagePreferences?.modelId || (preset?.type === "image" ? preset.defaultModelId || undefined : undefined),
|
||||||
operationSelectedNodeIds: selectedNodeIds,
|
operationSelectedNodeIds: selectedNodeIds,
|
||||||
objective: [snapshot.objective, ...snapshot.inputs.map((input) => input.text)].join(
|
objective: [snapshot.objective, ...snapshot.inputs.map((input) => input.text)].join(
|
||||||
"\n",
|
"\n",
|
||||||
@ -311,7 +326,8 @@ export async function POST(request: Request) {
|
|||||||
emit,
|
emit,
|
||||||
});
|
});
|
||||||
await registerProjectAgentTools({ session, projectId, request, objective: snapshot.objective,
|
await registerProjectAgentTools({ session, projectId, request, objective: snapshot.objective,
|
||||||
definitions: tools.definitions, executable: tools.executable, emit });
|
definitions: tools.definitions, executable: tools.executable, emit,
|
||||||
|
imagePreferences: snapshot.imagePreferences });
|
||||||
const gateway = await createZcodeModelGateway(configured);
|
const gateway = await createZcodeModelGateway(configured);
|
||||||
activeModelController = new AbortController();
|
activeModelController = new AbortController();
|
||||||
const status = await runZcodeCanvasTask({
|
const status = await runZcodeCanvasTask({
|
||||||
|
|||||||
@ -11077,6 +11077,7 @@ export default function CanvasEditorLegacyPage({
|
|||||||
onClose={() => setIsPanelOpen(false)}
|
onClose={() => setIsPanelOpen(false)}
|
||||||
onOpenFileList={() => setPanelMode("fileList")}
|
onOpenFileList={() => setPanelMode("fileList")}
|
||||||
onRequireLogin={openLoginDialog}
|
onRequireLogin={openLoginDialog}
|
||||||
|
onBeforeAgentRequest={() => persistCanvasState("autosave")}
|
||||||
selectedAgentPresetId={
|
selectedAgentPresetId={
|
||||||
isCanvasAgentModeEnabled ? selectedAgentPresetId : null
|
isCanvasAgentModeEnabled ? selectedAgentPresetId : null
|
||||||
}
|
}
|
||||||
|
|||||||
@ -422,6 +422,7 @@ interface CanvasChatPanelProps {
|
|||||||
canvasToolbarConfig?: CanvasToolbarConfig;
|
canvasToolbarConfig?: CanvasToolbarConfig;
|
||||||
canvasAgentFeatureSettings?: CanvasAgentFeatureSettings;
|
canvasAgentFeatureSettings?: CanvasAgentFeatureSettings;
|
||||||
onRequireLogin?: (message?: string) => void;
|
onRequireLogin?: (message?: string) => void;
|
||||||
|
onBeforeAgentRequest?: () => Promise<boolean>;
|
||||||
}
|
}
|
||||||
|
|
||||||
type CanvasComposerMode = "agent" | "image" | "video" | "detail" | "clawd";
|
type CanvasComposerMode = "agent" | "image" | "video" | "detail" | "clawd";
|
||||||
@ -466,16 +467,16 @@ function getCanvasComposerModeSessionLabel(mode: CanvasComposerMode) {
|
|||||||
if (mode === "clawd") return CLAW_MODE_LABEL;
|
if (mode === "clawd") return CLAW_MODE_LABEL;
|
||||||
if (mode === "video") return "视频";
|
if (mode === "video") return "视频";
|
||||||
if (mode === "detail") return "AI详情";
|
if (mode === "detail") return "AI详情";
|
||||||
if (mode === "image") return "图像";
|
if (mode === "image") return "Agent";
|
||||||
return "Agent";
|
return "Agent";
|
||||||
}
|
}
|
||||||
|
|
||||||
function normalizeCanvasComposerSessionMode(
|
function normalizeCanvasComposerSessionMode(
|
||||||
value: unknown,
|
value: unknown,
|
||||||
fallback: CanvasComposerMode = "image"
|
fallback: CanvasComposerMode = "agent"
|
||||||
): CanvasComposerMode {
|
): CanvasComposerMode {
|
||||||
|
if (value === "image") return "agent"; // Merge historical image conversations into Agent.
|
||||||
return value === "agent" ||
|
return value === "agent" ||
|
||||||
value === "image" ||
|
|
||||||
value === "video" ||
|
value === "video" ||
|
||||||
value === "detail" ||
|
value === "detail" ||
|
||||||
value === "clawd"
|
value === "clawd"
|
||||||
@ -485,7 +486,7 @@ function normalizeCanvasComposerSessionMode(
|
|||||||
|
|
||||||
function getCanvasChatSessionMode(
|
function getCanvasChatSessionMode(
|
||||||
session: Pick<CanvasChatSession, "mode" | "clawdSkillTagId"> | null | undefined,
|
session: Pick<CanvasChatSession, "mode" | "clawdSkillTagId"> | null | undefined,
|
||||||
fallback: CanvasComposerMode = "image"
|
fallback: CanvasComposerMode = "agent"
|
||||||
) {
|
) {
|
||||||
if (!session) return fallback;
|
if (!session) return fallback;
|
||||||
if (session.mode) {
|
if (session.mode) {
|
||||||
@ -3227,6 +3228,7 @@ export function CanvasChatPanel({
|
|||||||
canvasToolbarConfig: rawCanvasToolbarConfig,
|
canvasToolbarConfig: rawCanvasToolbarConfig,
|
||||||
canvasAgentFeatureSettings: propCanvasAgentFeatureSettings,
|
canvasAgentFeatureSettings: propCanvasAgentFeatureSettings,
|
||||||
onRequireLogin,
|
onRequireLogin,
|
||||||
|
onBeforeAgentRequest,
|
||||||
}: CanvasChatPanelProps) {
|
}: CanvasChatPanelProps) {
|
||||||
const isWorkspacePresentation = presentation === "workspace";
|
const isWorkspacePresentation = presentation === "workspace";
|
||||||
const isVideoWorkspace = workspaceVariant === "video";
|
const isVideoWorkspace = workspaceVariant === "video";
|
||||||
@ -5987,7 +5989,7 @@ export function CanvasChatPanel({
|
|||||||
[isCanvasImageModel, isCanvasVideoModel]
|
[isCanvasImageModel, isCanvasVideoModel]
|
||||||
);
|
);
|
||||||
const isVideoModel = Boolean(
|
const isVideoModel = Boolean(
|
||||||
currentModel && isCanvasVideoModel(currentModel)
|
isVideoWorkspace && currentModel && isCanvasVideoModel(currentModel)
|
||||||
);
|
);
|
||||||
const isConversationModel = Boolean(
|
const isConversationModel = Boolean(
|
||||||
currentModel && isCanvasConversationOnlyModel(currentModel)
|
currentModel && isCanvasConversationOnlyModel(currentModel)
|
||||||
@ -6006,11 +6008,15 @@ export function CanvasChatPanel({
|
|||||||
if (isDetailGenerationMode) return "detail";
|
if (isDetailGenerationMode) return "detail";
|
||||||
if (isVideoModel) return "video";
|
if (isVideoModel) return "video";
|
||||||
if (isAgentModel) return "agent";
|
if (isAgentModel) return "agent";
|
||||||
return "image";
|
return isVideoWorkspace || isWorkspacePresentation ? "image" : "agent";
|
||||||
}, [isAgentModel, isDetailGenerationMode, isVideoModel]);
|
}, [isAgentModel, isDetailGenerationMode, isVideoModel, isVideoWorkspace, isWorkspacePresentation]);
|
||||||
const [composerModeOverride, setComposerModeOverride] =
|
const [composerModeOverride, setComposerModeOverride] =
|
||||||
useState<CanvasComposerMode | null>(null);
|
useState<CanvasComposerMode | null>(null);
|
||||||
const activeComposerMode = composerModeOverride || computedComposerMode;
|
const unifiedImageAgent = !isVideoWorkspace && !isWorkspacePresentation;
|
||||||
|
const rawComposerMode = composerModeOverride || computedComposerMode;
|
||||||
|
const activeComposerMode = unifiedImageAgent && (rawComposerMode === "image" || rawComposerMode === "video")
|
||||||
|
? "agent" : rawComposerMode;
|
||||||
|
const isImageAgentComposer = unifiedImageAgent && activeComposerMode === "agent";
|
||||||
const isVideoToolbarDark = isVideoWorkspace || activeComposerMode === "video";
|
const isVideoToolbarDark = isVideoWorkspace || activeComposerMode === "video";
|
||||||
const isClawdMode = activeComposerMode === "clawd" && isClawdModeEnabled;
|
const isClawdMode = activeComposerMode === "clawd" && isClawdModeEnabled;
|
||||||
const activeChatSessionMode = getCanvasChatSessionMode(
|
const activeChatSessionMode = getCanvasChatSessionMode(
|
||||||
@ -6244,13 +6250,14 @@ export function CanvasChatPanel({
|
|||||||
return videoOnlyModels;
|
return videoOnlyModels;
|
||||||
}
|
}
|
||||||
if (activeComposerMode === "agent") {
|
if (activeComposerMode === "agent") {
|
||||||
return agentSelectableModels;
|
return unifiedImageAgent ? [SMART_AUTO_MODEL, ...imageOnlyModels] : agentSelectableModels;
|
||||||
}
|
}
|
||||||
return imageOnlyModels;
|
return imageOnlyModels;
|
||||||
}, [
|
}, [
|
||||||
SMART_AUTO_MODEL,
|
SMART_AUTO_MODEL,
|
||||||
activeComposerMode,
|
activeComposerMode,
|
||||||
agentSelectableModels,
|
agentSelectableModels,
|
||||||
|
unifiedImageAgent,
|
||||||
imageOnlyModels,
|
imageOnlyModels,
|
||||||
videoOnlyModels,
|
videoOnlyModels,
|
||||||
]);
|
]);
|
||||||
@ -6269,7 +6276,7 @@ export function CanvasChatPanel({
|
|||||||
activeComposerMode === "agent"
|
activeComposerMode === "agent"
|
||||||
? isPresetAgentSelection
|
? isPresetAgentSelection
|
||||||
? selectedAgentPreset?.name || "预设Agent"
|
? selectedAgentPreset?.name || "预设Agent"
|
||||||
: "Auto Agent"
|
: "Agent"
|
||||||
: currentModel?.displayName || selectDisplayModel?.displayName || "Agent模式";
|
: currentModel?.displayName || selectDisplayModel?.displayName || "Agent模式";
|
||||||
const videoFrameSlots = useMemo(() => {
|
const videoFrameSlots = useMemo(() => {
|
||||||
if (!isVideoModel || !videoParams) return [] as string[];
|
if (!isVideoModel || !videoParams) return [] as string[];
|
||||||
@ -6644,8 +6651,8 @@ export function CanvasChatPanel({
|
|||||||
? "视频"
|
? "视频"
|
||||||
: activeComposerMode === "detail"
|
: activeComposerMode === "detail"
|
||||||
? getToolbarItem("chatModeDetailActive").label
|
? getToolbarItem("chatModeDetailActive").label
|
||||||
: activeComposerMode === "image"
|
: activeComposerMode === "image" || isImageAgentComposer
|
||||||
? "图像"
|
? "Agent"
|
||||||
: isPresetAgentSelection
|
: isPresetAgentSelection
|
||||||
? getToolbarItem("chatModePresetAgent").label
|
? getToolbarItem("chatModePresetAgent").label
|
||||||
: "Agent";
|
: "Agent";
|
||||||
@ -6742,7 +6749,7 @@ export function CanvasChatPanel({
|
|||||||
|
|
||||||
useEffect(() => {
|
useEffect(() => {
|
||||||
if (!effectiveAgentEnabled) return;
|
if (!effectiveAgentEnabled) return;
|
||||||
if (activeComposerMode !== "agent") return;
|
if (activeComposerMode !== "agent" || unifiedImageAgent) return;
|
||||||
if (!isPresetAgentSelection) return;
|
if (!isPresetAgentSelection) return;
|
||||||
if (!selectedAgentPresetId) return;
|
if (!selectedAgentPresetId) return;
|
||||||
if (!selectedAgentPreset?.defaultModelId) return;
|
if (!selectedAgentPreset?.defaultModelId) return;
|
||||||
@ -6755,6 +6762,7 @@ export function CanvasChatPanel({
|
|||||||
}
|
}
|
||||||
}, [
|
}, [
|
||||||
activeComposerMode,
|
activeComposerMode,
|
||||||
|
unifiedImageAgent,
|
||||||
effectiveAgentEnabled,
|
effectiveAgentEnabled,
|
||||||
isPresetAgentSelection,
|
isPresetAgentSelection,
|
||||||
isDetailGenerationMode,
|
isDetailGenerationMode,
|
||||||
@ -6965,6 +6973,7 @@ export function CanvasChatPanel({
|
|||||||
|
|
||||||
const handleSelectGenerationMode = useCallback(
|
const handleSelectGenerationMode = useCallback(
|
||||||
(mode: CanvasComposerMode): string | null => {
|
(mode: CanvasComposerMode): string | null => {
|
||||||
|
if (unifiedImageAgent && (mode === "video" || mode === "image")) return null;
|
||||||
hiddenPromptRef.current = null;
|
hiddenPromptRef.current = null;
|
||||||
autoContextRef.current = null;
|
autoContextRef.current = null;
|
||||||
if (mode === "clawd" && !isClawdModeEnabled) {
|
if (mode === "clawd" && !isClawdModeEnabled) {
|
||||||
@ -7066,8 +7075,10 @@ export function CanvasChatPanel({
|
|||||||
}
|
}
|
||||||
|
|
||||||
if (mode === "agent") {
|
if (mode === "agent") {
|
||||||
currentModelIdRef.current = "auto";
|
// Keep the image model selected by the user when returning from Claw.
|
||||||
setSelectedModelId("auto");
|
const imageModelId = isCanvasImageModel(currentModel) ? currentModel!.id : lastImageModelId || "auto";
|
||||||
|
currentModelIdRef.current = unifiedImageAgent ? imageModelId : "auto";
|
||||||
|
setSelectedModelId(currentModelIdRef.current);
|
||||||
setIsDetailGenerationMode(false);
|
setIsDetailGenerationMode(false);
|
||||||
return targetChatSessionId;
|
return targetChatSessionId;
|
||||||
}
|
}
|
||||||
@ -7099,6 +7110,8 @@ export function CanvasChatPanel({
|
|||||||
canUseDetailGeneration,
|
canUseDetailGeneration,
|
||||||
effectiveAgentEnabled,
|
effectiveAgentEnabled,
|
||||||
isClawdModeEnabled,
|
isClawdModeEnabled,
|
||||||
|
unifiedImageAgent,
|
||||||
|
currentModel,
|
||||||
allModels,
|
allModels,
|
||||||
activeChatId,
|
activeChatId,
|
||||||
activeComposerMode,
|
activeComposerMode,
|
||||||
@ -7903,7 +7916,7 @@ export function CanvasChatPanel({
|
|||||||
transport: new DefaultChatTransport({
|
transport: new DefaultChatTransport({
|
||||||
api: "/api/chat",
|
api: "/api/chat",
|
||||||
fetch: fetchWithErrorHandlers,
|
fetch: fetchWithErrorHandlers,
|
||||||
prepareSendMessagesRequest(request) {
|
async prepareSendMessagesRequest(request) {
|
||||||
// 从 canvas-{uuid} 格式中提取纯 UUID,后端 schema 要求纯 UUID
|
// 从 canvas-{uuid} 格式中提取纯 UUID,后端 schema 要求纯 UUID
|
||||||
const rawId = request.id.startsWith('canvas-')
|
const rawId = request.id.startsWith('canvas-')
|
||||||
? request.id.replace('canvas-', '')
|
? request.id.replace('canvas-', '')
|
||||||
@ -7951,6 +7964,8 @@ export function CanvasChatPanel({
|
|||||||
? selectedAgentPreset.id
|
? selectedAgentPreset.id
|
||||||
: undefined,
|
: undefined,
|
||||||
selectedChatModel: effectiveSelectedChatModel,
|
selectedChatModel: effectiveSelectedChatModel,
|
||||||
|
selectedImageModel: effectiveComposerMode === "agent" && isCanvasImageModel(allModels.find(model => model.id === currentModelIdRef.current))
|
||||||
|
? currentModelIdRef.current : undefined,
|
||||||
composerMode: effectiveComposerMode,
|
composerMode: effectiveComposerMode,
|
||||||
workspaceVariant: isVideoWorkspace ? "video" : "canvas",
|
workspaceVariant: isVideoWorkspace ? "video" : "canvas",
|
||||||
selectedVisibilityType: "private",
|
selectedVisibilityType: "private",
|
||||||
@ -7964,7 +7979,7 @@ export function CanvasChatPanel({
|
|||||||
: deepThinkingEnabledRef.current,
|
: deepThinkingEnabledRef.current,
|
||||||
videoParams: requestIsVideoModel ? videoParamsRef.current || undefined : undefined,
|
videoParams: requestIsVideoModel ? videoParamsRef.current || undefined : undefined,
|
||||||
imageParams:
|
imageParams:
|
||||||
requestIsVideoModel || requestIsConversationModel
|
effectiveComposerMode !== "agent" && (requestIsVideoModel || requestIsConversationModel)
|
||||||
? undefined
|
? undefined
|
||||||
: imageParamsRef.current || undefined,
|
: imageParamsRef.current || undefined,
|
||||||
hiddenPrompt,
|
hiddenPrompt,
|
||||||
@ -7973,8 +7988,14 @@ export function CanvasChatPanel({
|
|||||||
...(runtimeResumeRunId ? { resumeRunId: runtimeResumeRunId } : {}),
|
...(runtimeResumeRunId ? { resumeRunId: runtimeResumeRunId } : {}),
|
||||||
...request.body,
|
...request.body,
|
||||||
};
|
};
|
||||||
|
const useAgent = shouldUseCanvasAgentRuntime(preparedBody);
|
||||||
|
// Persist references before the isolated Agent reads the owned canvas.
|
||||||
|
// Transport admission is already guarded against duplicate sends.
|
||||||
|
if (useAgent && onBeforeAgentRequest && !(await onBeforeAgentRequest())) {
|
||||||
|
throw new Error("画布保存未完成,未开始 Agent 任务,请稍后重试。");
|
||||||
|
}
|
||||||
return {
|
return {
|
||||||
api: shouldUseCanvasAgentRuntime(preparedBody) ? "/api/canvas/agent-chat" : "/api/chat",
|
api: useAgent ? "/api/canvas/agent-chat" : "/api/chat",
|
||||||
body: preparedBody,
|
body: preparedBody,
|
||||||
};
|
};
|
||||||
},
|
},
|
||||||
@ -14791,6 +14812,8 @@ export function CanvasChatPanel({
|
|||||||
? "可选补充:风格、情绪氛围、禁用元素、期望生成张数(如:生成8张)"
|
? "可选补充:风格、情绪氛围、禁用元素、期望生成张数(如:生成8张)"
|
||||||
: isVideoWorkspace
|
: isVideoWorkspace
|
||||||
? "描述镜头、主体和运动方式,也可以先切到首帧或首尾帧模式"
|
? "描述镜头、主体和运动方式,也可以先切到首帧或首尾帧模式"
|
||||||
|
: isImageAgentComposer
|
||||||
|
? "描述需求,或选择画布图片继续编辑"
|
||||||
: activeComposerMode === "agent" && !isPresetAgentSelection
|
: activeComposerMode === "agent" && !isPresetAgentSelection
|
||||||
? ""
|
? ""
|
||||||
: hasAgentUploadSlots
|
: hasAgentUploadSlots
|
||||||
@ -14807,6 +14830,8 @@ export function CanvasChatPanel({
|
|||||||
? isVideoModel && videoFrameSlots.length > 0
|
? isVideoModel && videoFrameSlots.length > 0
|
||||||
? `描述镜头、主体和运动方式;可从画布填充${videoFrameSlots.join(" / ")}`
|
? `描述镜头、主体和运动方式;可从画布填充${videoFrameSlots.join(" / ")}`
|
||||||
: "描述视频镜头、主体、运动方式与时长,也可以切换到首帧或首尾帧模式"
|
: "描述视频镜头、主体、运动方式与时长,也可以切换到首帧或首尾帧模式"
|
||||||
|
: isImageAgentComposer
|
||||||
|
? "描述需求,或选择画布图片继续编辑"
|
||||||
: activeComposerMode === "agent" && !isPresetAgentSelection
|
: activeComposerMode === "agent" && !isPresetAgentSelection
|
||||||
? ""
|
? ""
|
||||||
: activeComposerMode === "agent" && selectedAgentPreset?.type === "text"
|
: activeComposerMode === "agent" && selectedAgentPreset?.type === "text"
|
||||||
@ -15189,7 +15214,7 @@ export function CanvasChatPanel({
|
|||||||
isClawdMode
|
isClawdMode
|
||||||
? `开始 ${CLAW_MODE_LABEL} 对话`
|
? `开始 ${CLAW_MODE_LABEL} 对话`
|
||||||
: activeComposerMode === "agent" && !isPresetAgentSelection
|
: activeComposerMode === "agent" && !isPresetAgentSelection
|
||||||
? "开始 Auto Agent 对话"
|
? "开始 Agent 对话"
|
||||||
: hasAgentUploadSlots && attachments.length < agentRequiredUploadCount
|
: hasAgentUploadSlots && attachments.length < agentRequiredUploadCount
|
||||||
? `请先补齐图片槽位(${attachments.length}/${agentRequiredUploadCount})`
|
? `请先补齐图片槽位(${attachments.length}/${agentRequiredUploadCount})`
|
||||||
: "准备开始生成";
|
: "准备开始生成";
|
||||||
@ -17912,12 +17937,12 @@ export function CanvasChatPanel({
|
|||||||
<div
|
<div
|
||||||
className={cn(
|
className={cn(
|
||||||
"relative pb-0",
|
"relative pb-0",
|
||||||
!isWorkspacePresentation && !isDetailGenerationMode && !isVideoWorkspace && activeComposerMode === "image"
|
!isWorkspacePresentation && !isDetailGenerationMode && !isVideoWorkspace && (activeComposerMode === "image" || isImageAgentComposer)
|
||||||
? "px-0 pt-0"
|
? "px-0 pt-0"
|
||||||
: "px-3 pt-2.5",
|
: "px-3 pt-2.5",
|
||||||
isDetailGenerationMode
|
isDetailGenerationMode
|
||||||
? "min-h-[96px]"
|
? "min-h-[96px]"
|
||||||
: activeComposerMode === "image"
|
: (activeComposerMode === "image" || isImageAgentComposer)
|
||||||
? "min-h-[92px]"
|
? "min-h-[92px]"
|
||||||
: "min-h-[104px]"
|
: "min-h-[104px]"
|
||||||
)}
|
)}
|
||||||
@ -18066,7 +18091,7 @@ export function CanvasChatPanel({
|
|||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
)}
|
)}
|
||||||
{!isDetailGenerationMode && activeComposerMode === "agent" && (
|
{!unifiedImageAgent && !isDetailGenerationMode && activeComposerMode === "agent" && (
|
||||||
<div className="mb-2 flex flex-wrap items-center gap-2">
|
<div className="mb-2 flex flex-wrap items-center gap-2">
|
||||||
{hasSelectedAgentPreset ? (
|
{hasSelectedAgentPreset ? (
|
||||||
<>
|
<>
|
||||||
@ -18292,7 +18317,7 @@ export function CanvasChatPanel({
|
|||||||
) : null}
|
) : null}
|
||||||
</div>
|
</div>
|
||||||
) : null}
|
) : null}
|
||||||
{!isWorkspacePresentation && !isDetailGenerationMode && !isVideoWorkspace && activeComposerMode === "image" ? (
|
{!isWorkspacePresentation && !isDetailGenerationMode && !isVideoWorkspace && (activeComposerMode === "image" || isImageAgentComposer) ? (
|
||||||
<div className="lovart-reference-upload mx-[8px] mt-[8px] flex items-center">
|
<div className="lovart-reference-upload mx-[8px] mt-[8px] flex items-center">
|
||||||
<button
|
<button
|
||||||
type="button"
|
type="button"
|
||||||
@ -18384,7 +18409,7 @@ export function CanvasChatPanel({
|
|||||||
"max-h-[128px] overflow-y-auto outline-none empty:before:content-[attr(data-placeholder)]",
|
"max-h-[128px] overflow-y-auto outline-none empty:before:content-[attr(data-placeholder)]",
|
||||||
isWorkspacePresentation || isDetailGenerationMode || isVideoWorkspace
|
isWorkspacePresentation || isDetailGenerationMode || isVideoWorkspace
|
||||||
? "min-h-[36px] text-[16px] leading-6 md:text-[14px]"
|
? "min-h-[36px] text-[16px] leading-6 md:text-[14px]"
|
||||||
: activeComposerMode === "image"
|
: (activeComposerMode === "image" || isImageAgentComposer)
|
||||||
? "min-h-[38px] text-[16px] leading-6 md:text-[14px]"
|
? "min-h-[38px] text-[16px] leading-6 md:text-[14px]"
|
||||||
: "min-h-[50px] text-[16px] leading-6 md:text-[14px]",
|
: "min-h-[50px] text-[16px] leading-6 md:text-[14px]",
|
||||||
isVideoWorkspace
|
isVideoWorkspace
|
||||||
@ -18798,7 +18823,7 @@ export function CanvasChatPanel({
|
|||||||
: "dark:border-white/10 dark:bg-zinc-900/95 dark:text-zinc-100"
|
: "dark:border-white/10 dark:bg-zinc-900/95 dark:text-zinc-100"
|
||||||
)}
|
)}
|
||||||
>
|
>
|
||||||
{effectiveAgentEnabled ? (
|
{(effectiveAgentEnabled || unifiedImageAgent) ? (
|
||||||
<DropdownMenuItem
|
<DropdownMenuItem
|
||||||
onClick={() => handleSelectGenerationMode("agent")}
|
onClick={() => handleSelectGenerationMode("agent")}
|
||||||
className={getComposerModeMenuItemClass("agent")}
|
className={getComposerModeMenuItemClass("agent")}
|
||||||
@ -18828,7 +18853,7 @@ export function CanvasChatPanel({
|
|||||||
) : null}
|
) : null}
|
||||||
</DropdownMenuItem>
|
</DropdownMenuItem>
|
||||||
) : null}
|
) : null}
|
||||||
{canUseDetailGeneration ? (
|
{!unifiedImageAgent && canUseDetailGeneration ? (
|
||||||
<DropdownMenuItem
|
<DropdownMenuItem
|
||||||
onClick={() => handleSelectGenerationMode("detail")}
|
onClick={() => handleSelectGenerationMode("detail")}
|
||||||
className={getComposerModeMenuItemClass("detail")}
|
className={getComposerModeMenuItemClass("detail")}
|
||||||
@ -18846,6 +18871,7 @@ export function CanvasChatPanel({
|
|||||||
) : null}
|
) : null}
|
||||||
</DropdownMenuItem>
|
</DropdownMenuItem>
|
||||||
) : null}
|
) : null}
|
||||||
|
{!unifiedImageAgent ? <>
|
||||||
<DropdownMenuItem
|
<DropdownMenuItem
|
||||||
onClick={() => handleSelectGenerationMode("image")}
|
onClick={() => handleSelectGenerationMode("image")}
|
||||||
className={getComposerModeMenuItemClass("image")}
|
className={getComposerModeMenuItemClass("image")}
|
||||||
@ -18858,7 +18884,7 @@ export function CanvasChatPanel({
|
|||||||
<span className="flex-1">
|
<span className="flex-1">
|
||||||
图像
|
图像
|
||||||
</span>
|
</span>
|
||||||
{activeComposerMode === "image" ? (
|
{(activeComposerMode === "image" || isImageAgentComposer) ? (
|
||||||
<Check className={cn("h-4 w-4", isVideoToolbarDark ? "text-white" : "text-[#1f2937]")} />
|
<Check className={cn("h-4 w-4", isVideoToolbarDark ? "text-white" : "text-[#1f2937]")} />
|
||||||
) : null}
|
) : null}
|
||||||
</DropdownMenuItem>
|
</DropdownMenuItem>
|
||||||
@ -18878,6 +18904,7 @@ export function CanvasChatPanel({
|
|||||||
<Check className={cn("h-4 w-4", isVideoToolbarDark ? "text-white" : "text-[#1f2937]")} />
|
<Check className={cn("h-4 w-4", isVideoToolbarDark ? "text-white" : "text-[#1f2937]")} />
|
||||||
) : null}
|
) : null}
|
||||||
</DropdownMenuItem>
|
</DropdownMenuItem>
|
||||||
|
</> : null}
|
||||||
</DropdownMenuContent>
|
</DropdownMenuContent>
|
||||||
</DropdownMenu>
|
</DropdownMenu>
|
||||||
) : null}
|
) : null}
|
||||||
@ -18900,7 +18927,7 @@ export function CanvasChatPanel({
|
|||||||
</div>
|
</div>
|
||||||
|
|
||||||
<div className="flex shrink-0 items-center gap-1">
|
<div className="flex shrink-0 items-center gap-1">
|
||||||
{!isVideoModel && !isWorkspacePresentation && activeComposerMode === "image" ? (
|
{!isVideoModel && !isWorkspacePresentation && (activeComposerMode === "image" || isImageAgentComposer) ? (
|
||||||
<ImageParamsBar
|
<ImageParamsBar
|
||||||
modelName={currentModel?.displayName}
|
modelName={currentModel?.displayName}
|
||||||
initialParams={imageParams}
|
initialParams={imageParams}
|
||||||
@ -18928,7 +18955,7 @@ export function CanvasChatPanel({
|
|||||||
const nextIsConversationModel = isCanvasConversationOnlyModel(nextModel);
|
const nextIsConversationModel = isCanvasConversationOnlyModel(nextModel);
|
||||||
currentModelIdRef.current = value;
|
currentModelIdRef.current = value;
|
||||||
setSelectedModelId(value);
|
setSelectedModelId(value);
|
||||||
if (value === "auto" || nextIsConversationModel) {
|
if (unifiedImageAgent || value === "auto" || nextIsConversationModel) {
|
||||||
setComposerModeOverride("agent");
|
setComposerModeOverride("agent");
|
||||||
} else if (nextIsVideoModel) {
|
} else if (nextIsVideoModel) {
|
||||||
setComposerModeOverride("video");
|
setComposerModeOverride("video");
|
||||||
@ -18942,7 +18969,7 @@ export function CanvasChatPanel({
|
|||||||
>
|
>
|
||||||
<SelectTrigger
|
<SelectTrigger
|
||||||
className={cn(
|
className={cn(
|
||||||
cn(!isVideoWorkspace && activeComposerMode === "image" && "lovart-model-selector"),
|
cn(!isVideoWorkspace && (activeComposerMode === "image" || isImageAgentComposer) && "lovart-model-selector"),
|
||||||
"h-8 w-8 shrink-0 justify-center rounded-full p-0 outline-none focus:outline-none focus:ring-0 focus:ring-offset-0 focus-visible:outline-none focus-visible:ring-0 focus-visible:ring-offset-0 [&>svg]:hidden",
|
"h-8 w-8 shrink-0 justify-center rounded-full p-0 outline-none focus:outline-none focus:ring-0 focus:ring-offset-0 focus-visible:outline-none focus-visible:ring-0 focus-visible:ring-offset-0 [&>svg]:hidden",
|
||||||
isVideoToolbarDark
|
isVideoToolbarDark
|
||||||
? "border border-white/12 bg-[#242529] text-zinc-100 hover:bg-[#2c2d33]"
|
? "border border-white/12 bg-[#242529] text-zinc-100 hover:bg-[#2c2d33]"
|
||||||
@ -18950,7 +18977,7 @@ export function CanvasChatPanel({
|
|||||||
)}
|
)}
|
||||||
title={getToolbarItem("chatModelSelector").label}
|
title={getToolbarItem("chatModelSelector").label}
|
||||||
>
|
>
|
||||||
{!isVideoWorkspace && activeComposerMode === "image" ? (
|
{!isVideoWorkspace && (activeComposerMode === "image" || isImageAgentComposer) ? (
|
||||||
<>
|
<>
|
||||||
<ModelIdentityIcon name={selectDisplayModel?.displayName || "自动模型"} identity={`${selectDisplayModel?.id || ""} ${selectDisplayModel?.providerType || ""}`} avatarUrl={selectDisplayModel?.avatarUrl ? buildPreviewImageUrl(selectDisplayModel.avatarUrl, 64) : undefined} size={18} />
|
<ModelIdentityIcon name={selectDisplayModel?.displayName || "自动模型"} identity={`${selectDisplayModel?.id || ""} ${selectDisplayModel?.providerType || ""}`} avatarUrl={selectDisplayModel?.avatarUrl ? buildPreviewImageUrl(selectDisplayModel.avatarUrl, 64) : undefined} size={18} />
|
||||||
<span className="min-w-0 flex-1 truncate">{selectDisplayModel?.displayName || "自动模型"}</span>
|
<span className="min-w-0 flex-1 truncate">{selectDisplayModel?.displayName || "自动模型"}</span>
|
||||||
@ -19063,7 +19090,7 @@ export function CanvasChatPanel({
|
|||||||
? "停止处理并重新编辑"
|
? "停止处理并重新编辑"
|
||||||
: isWorkspacePresentation || isDetailGenerationMode
|
: isWorkspacePresentation || isDetailGenerationMode
|
||||||
? getToolbarItem("chatStartGeneration").label
|
? getToolbarItem("chatStartGeneration").label
|
||||||
: !isVideoWorkspace && activeComposerMode === "image" && estimatedPointsCost
|
: !isVideoWorkspace && (activeComposerMode === "image" || isImageAgentComposer) && estimatedPointsCost
|
||||||
? `${getToolbarItem("chatSend").label} · ${estimatedPointsCost.title}`
|
? `${getToolbarItem("chatSend").label} · ${estimatedPointsCost.title}`
|
||||||
: getToolbarItem("chatSend").label
|
: getToolbarItem("chatSend").label
|
||||||
}
|
}
|
||||||
@ -19098,7 +19125,7 @@ export function CanvasChatPanel({
|
|||||||
</span>
|
</span>
|
||||||
) : null}
|
) : null}
|
||||||
</>
|
</>
|
||||||
) : !isWorkspacePresentation && !isVideoWorkspace && activeComposerMode === "image" && estimatedPointsCost ? (
|
) : !isWorkspacePresentation && !isVideoWorkspace && (activeComposerMode === "image" || isImageAgentComposer) && estimatedPointsCost ? (
|
||||||
<>
|
<>
|
||||||
<CanvasToolbarIcon fallback={Sparkles} item={getToolbarItem("chatPoints")} referenceName="points" referenceSize={14} />
|
<CanvasToolbarIcon fallback={Sparkles} item={getToolbarItem("chatPoints")} referenceName="points" referenceSize={14} />
|
||||||
<span>{estimatedPointsCost.cost}</span>
|
<span>{estimatedPointsCost.cost}</span>
|
||||||
|
|||||||
36
lib/ai/canvas-agent-runtime/image-preferences.ts
Normal file
36
lib/ai/canvas-agent-runtime/image-preferences.ts
Normal file
@ -0,0 +1,36 @@
|
|||||||
|
/** User image controls are execution constraints, separate from the planning model. */
|
||||||
|
export type AgentImageParameters = {
|
||||||
|
aspectRatio?: string;
|
||||||
|
imageSize?: string;
|
||||||
|
imageCount?: number;
|
||||||
|
n?: number;
|
||||||
|
quality?: "auto" | "low" | "medium" | "high";
|
||||||
|
size?: string;
|
||||||
|
negativePrompt?: string;
|
||||||
|
promptExtend?: boolean;
|
||||||
|
watermark?: boolean;
|
||||||
|
seed?: number;
|
||||||
|
};
|
||||||
|
export type AgentImagePreferences = { modelId?: string; parameters?: AgentImageParameters };
|
||||||
|
|
||||||
|
export function applyAgentImagePreferences<T extends Record<string, unknown>>(
|
||||||
|
input: T,
|
||||||
|
preferences?: AgentImagePreferences,
|
||||||
|
): T & AgentImageParameters & { modelId?: string } {
|
||||||
|
const parameters = Object.fromEntries(Object.entries(preferences?.parameters || {})
|
||||||
|
.filter(([, value]) => value !== undefined));
|
||||||
|
const count = preferences?.parameters?.imageCount ?? preferences?.parameters?.n;
|
||||||
|
return {
|
||||||
|
...input,
|
||||||
|
...parameters,
|
||||||
|
...(count !== undefined ? { imageCount: count } : {}),
|
||||||
|
...(preferences?.modelId ? { modelId: preferences.modelId } : {}),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
export function agentProviderImageParameters(parameters: AgentImageParameters) {
|
||||||
|
return Object.fromEntries(["quality", "size", "watermark", "seed"].flatMap(key => {
|
||||||
|
const value = parameters[key as keyof AgentImageParameters];
|
||||||
|
return value !== undefined ? [[key, value]] : [];
|
||||||
|
}));
|
||||||
|
}
|
||||||
@ -1,3 +1,4 @@
|
|||||||
|
import type { AgentImagePreferences } from "./image-preferences";
|
||||||
import "server-only";
|
import "server-only";
|
||||||
import { AgentToolRejectedError } from "./protocol";
|
import { AgentToolRejectedError } from "./protocol";
|
||||||
import { tool, type ToolSet } from "ai";
|
import { tool, type ToolSet } from "ai";
|
||||||
@ -16,6 +17,7 @@ import { isExplicitDestructiveIntent } from "./capability-access";
|
|||||||
import type { CanvasAgentTool } from "./loop";
|
import type { CanvasAgentTool } from "./loop";
|
||||||
|
|
||||||
export async function registerProjectAgentTools(params: {
|
export async function registerProjectAgentTools(params: {
|
||||||
|
imagePreferences?: AgentImagePreferences;
|
||||||
session: Session; projectId: string; request: Request; objective: string;
|
session: Session; projectId: string; request: Request; objective: string;
|
||||||
definitions: ToolSet; executable: Record<string, CanvasAgentTool>;
|
definitions: ToolSet; executable: Record<string, CanvasAgentTool>;
|
||||||
emit: (type: string, data: unknown) => void;
|
emit: (type: string, data: unknown) => void;
|
||||||
@ -35,6 +37,8 @@ export async function registerProjectAgentTools(params: {
|
|||||||
const args = invokeSchema.parse(input);
|
const args = invokeSchema.parse(input);
|
||||||
const operation = PROJECT_AGENT_OPERATIONS.find(item => item.id === args.operationId);
|
const operation = PROJECT_AGENT_OPERATIONS.find(item => item.id === args.operationId);
|
||||||
if (!operation || operation.readOnly !== readOnly) throw new Error("请使用清单中对应读写类别的固定业务工具");
|
if (!operation || operation.readOnly !== readOnly) throw new Error("请使用清单中对应读写类别的固定业务工具");
|
||||||
|
if (params.imagePreferences?.modelId && ["style-generate.post", "wash-generate.post"].includes(operation.id))
|
||||||
|
throw new AgentToolRejectedError("用户已指定生图模型,请用 generateCanvasImages 执行并保留当前图片参数;不能通过另一条生成接口换用默认模型。");
|
||||||
if (!projectOperationAuthorized(operation)) throw new Error("用户暂未授权视频画布和分镜工具");
|
if (!projectOperationAuthorized(operation)) throw new Error("用户暂未授权视频画布和分镜工具");
|
||||||
if (JSON.stringify(args.body || {}).length > 2 * 1024 * 1024) throw new Error("业务工具输入超过大小限制");
|
if (JSON.stringify(args.body || {}).length > 2 * 1024 * 1024) throw new Error("业务工具输入超过大小限制");
|
||||||
if (args.body?.workspaceType === "video" || args.query.workspaceType === "video" || args.body?.type === "video") throw new Error("视频画布暂未授权");
|
if (args.body?.workspaceType === "video" || args.query.workspaceType === "video" || args.body?.type === "video") throw new Error("视频画布暂未授权");
|
||||||
|
|||||||
@ -29,6 +29,7 @@ export type AgentRunSnapshot = {
|
|||||||
traceId?: string;
|
traceId?: string;
|
||||||
};
|
};
|
||||||
workspace?: "canvas" | "video";
|
workspace?: "canvas" | "video";
|
||||||
|
imagePreferences?: import("./image-preferences").AgentImagePreferences;
|
||||||
videoSettings?: import("./video-protocol").VideoAgentSettings;
|
videoSettings?: import("./video-protocol").VideoAgentSettings;
|
||||||
videoRequirements?: import("./video-protocol").VideoTaskRequirements;
|
videoRequirements?: import("./video-protocol").VideoTaskRequirements;
|
||||||
objective: string;
|
objective: string;
|
||||||
|
|||||||
@ -1,3 +1,4 @@
|
|||||||
|
import { applyAgentImagePreferences, agentProviderImageParameters, type AgentImagePreferences } from "./image-preferences";
|
||||||
import { tool, type LanguageModel, type ToolSet } from "ai";
|
import { tool, type LanguageModel, type ToolSet } from "ai";
|
||||||
import { z } from "zod";
|
import { z } from "zod";
|
||||||
import { createCanvasCapabilityTools } from "@/lib/canvas/capabilities/ai-sdk-tools";
|
import { createCanvasCapabilityTools } from "@/lib/canvas/capabilities/ai-sdk-tools";
|
||||||
@ -33,6 +34,7 @@ export async function buildCanvasRuntimeTools(params: {
|
|||||||
supportsVision: boolean;
|
supportsVision: boolean;
|
||||||
imageSize?: string;
|
imageSize?: string;
|
||||||
preferredModelId?: string;
|
preferredModelId?: string;
|
||||||
|
imagePreferences?: AgentImagePreferences;
|
||||||
emit: (type: string, data: unknown) => void;
|
emit: (type: string, data: unknown) => void;
|
||||||
objective?: string;
|
objective?: string;
|
||||||
operationSelectedNodeIds?: string[];
|
operationSelectedNodeIds?: string[];
|
||||||
@ -45,6 +47,8 @@ export async function buildCanvasRuntimeTools(params: {
|
|||||||
userId: params.userId,
|
userId: params.userId,
|
||||||
selectedNodeIds: defaults,
|
selectedNodeIds: defaults,
|
||||||
allowExplicitTargets: true,
|
allowExplicitTargets: true,
|
||||||
|
defaultModelId: params.preferredModelId,
|
||||||
|
imagePreferences: params.imagePreferences,
|
||||||
authorizeDestructive: (ids) =>
|
authorizeDestructive: (ids) =>
|
||||||
isExplicitDestructiveIntent(params.objective || "") &&
|
isExplicitDestructiveIntent(params.objective || "") &&
|
||||||
ids.length > 0 &&
|
ids.length > 0 &&
|
||||||
@ -78,6 +82,14 @@ export async function buildCanvasRuntimeTools(params: {
|
|||||||
execute: async (input, callId) => {
|
execute: async (input, callId) => {
|
||||||
if (!defaults.length && !Array.isArray(input.nodeIds))
|
if (!defaults.length && !Array.isArray(input.nodeIds))
|
||||||
throw new Error("请明确提供目标图片节点,不能猜测第一张图。");
|
throw new Error("请明确提供目标图片节点,不能猜测第一张图。");
|
||||||
|
// Generic redraw is the same constrained generator, including batch count,
|
||||||
|
// provider parameters and the no-silent-model-failover policy.
|
||||||
|
if (capability.id === "image.prompt_regenerate") {
|
||||||
|
const args = (input.args || {}) as Record<string, unknown>;
|
||||||
|
return executable.generateCanvasImages.execute({
|
||||||
|
...args, referenceNodeIds: input.nodeIds || defaults,
|
||||||
|
}, callId);
|
||||||
|
}
|
||||||
const result = await entry.execute!(input as never, { toolCallId: callId, messages: [] });
|
const result = await entry.execute!(input as never, { toolCallId: callId, messages: [] });
|
||||||
return result as Record<string, unknown>;
|
return result as Record<string, unknown>;
|
||||||
},
|
},
|
||||||
@ -113,7 +125,12 @@ export async function buildCanvasRuntimeTools(params: {
|
|||||||
aspectRatio: z.string().max(24).optional(),
|
aspectRatio: z.string().max(24).optional(),
|
||||||
visualCriteria: z.array(z.string().max(200)).max(10),
|
visualCriteria: z.array(z.string().max(200)).max(10),
|
||||||
}),
|
}),
|
||||||
async (input) => ({ ...input }),
|
async (input) => ({ ...input,
|
||||||
|
...(input.expectedImageCount !== undefined && params.imagePreferences?.parameters?.imageCount !== undefined
|
||||||
|
? { expectedImageCount: params.imagePreferences.parameters.imageCount } : {}),
|
||||||
|
...(params.imagePreferences?.parameters?.aspectRatio && params.imagePreferences.parameters.aspectRatio !== "auto"
|
||||||
|
? { aspectRatio: params.imagePreferences.parameters.aspectRatio } : {}),
|
||||||
|
}),
|
||||||
);
|
);
|
||||||
register(
|
register(
|
||||||
"readCanvas",
|
"readCanvas",
|
||||||
@ -165,10 +182,11 @@ export async function buildCanvasRuntimeTools(params: {
|
|||||||
referenceNodeIds: z.array(z.string().min(1).max(180)).max(8).default([]),
|
referenceNodeIds: z.array(z.string().min(1).max(180)).max(8).default([]),
|
||||||
imageCount: z.number().int().min(1).max(10).default(1),
|
imageCount: z.number().int().min(1).max(10).default(1),
|
||||||
aspectRatio: z.string().max(24).default("auto"),
|
aspectRatio: z.string().max(24).default("auto"),
|
||||||
imageSize: z.enum(["1K", "2K", "4K"]).optional(),
|
imageSize: z.enum(["0.5K", "1K", "2K", "4K", "8K", "10K", "12K"]).optional(),
|
||||||
modelId: z.string().max(180).optional().describe("指定已配置的图片模型 ID;省略使用预设或系统模型"),
|
modelId: z.string().max(180).optional().describe("指定已配置的图片模型 ID;省略使用预设或系统模型"),
|
||||||
}),
|
}),
|
||||||
async (input, callId) => {
|
async (toolInput, callId) => {
|
||||||
|
const input = applyAgentImagePreferences(toolInput, params.imagePreferences);
|
||||||
// Bound per-task spend without introducing any global/model concurrency queue.
|
// Bound per-task spend without introducing any global/model concurrency queue.
|
||||||
if (generatedCount + input.imageCount > 24)
|
if (generatedCount + input.imageCount > 24)
|
||||||
throw new Error("本次任务已达到 24 张生成预算,请检查结果后继续。");
|
throw new Error("本次任务已达到 24 张生成预算,请检查结果后继续。");
|
||||||
@ -182,7 +200,7 @@ export async function buildCanvasRuntimeTools(params: {
|
|||||||
intent: sources.length ? "image_edit" : "image_gen",
|
intent: sources.length ? "image_edit" : "image_gen",
|
||||||
hasImage: sources.length > 0,
|
hasImage: sources.length > 0,
|
||||||
}).catch(() => null);
|
}).catch(() => null);
|
||||||
const requestedModel = input.modelId || params.preferredModelId;
|
const requestedModel = params.imagePreferences?.modelId || input.modelId || params.preferredModelId;
|
||||||
const requestedEntry = requestedModel ? models.find(({ model }) => model.healthStatus !== "auto_disabled" && (model.id === requestedModel || model.modelId === requestedModel)) : null;
|
const requestedEntry = requestedModel ? models.find(({ model }) => model.healthStatus !== "auto_disabled" && (model.id === requestedModel || model.modelId === requestedModel)) : null;
|
||||||
if (requestedModel && !requestedEntry) throw new Error("指定的图片模型不可用,未切换到其他模型收费生成");
|
if (requestedModel && !requestedEntry) throw new Error("指定的图片模型不可用,未切换到其他模型收费生成");
|
||||||
const chosen = requestedEntry || models.find(
|
const chosen = requestedEntry || models.find(
|
||||||
@ -217,6 +235,10 @@ export async function buildCanvasRuntimeTools(params: {
|
|||||||
imageCount: input.imageCount,
|
imageCount: input.imageCount,
|
||||||
imageSize,
|
imageSize,
|
||||||
aspectRatio: input.aspectRatio,
|
aspectRatio: input.aspectRatio,
|
||||||
|
negativePrompt: input.negativePrompt,
|
||||||
|
promptExtend: input.promptExtend,
|
||||||
|
providerImageParams: agentProviderImageParameters(input),
|
||||||
|
allowFailover: !params.imagePreferences?.modelId,
|
||||||
ownerKey: params.userId,
|
ownerKey: params.userId,
|
||||||
referenceImages: sources.map((node) => node.data.src || node.data.url!),
|
referenceImages: sources.map((node) => node.data.src || node.data.url!),
|
||||||
debugLabel: "canvas-agent-runtime",
|
debugLabel: "canvas-agent-runtime",
|
||||||
@ -278,6 +300,7 @@ export async function buildCanvasRuntimeTools(params: {
|
|||||||
);
|
);
|
||||||
return {
|
return {
|
||||||
status: "succeeded",
|
status: "succeeded",
|
||||||
|
appliedImagePreferences: { modelId: chosen.model.id, imageSize, aspectRatio: input.aspectRatio, imageCount: input.imageCount },
|
||||||
createdNodeIds: operation.createdNodeIds,
|
createdNodeIds: operation.createdNodeIds,
|
||||||
outputs: result.images.map((image) => ({ url: image.originalUrl })),
|
outputs: result.images.map((image) => ({ url: image.originalUrl })),
|
||||||
model: result.modelRef,
|
model: result.modelRef,
|
||||||
|
|||||||
@ -49,13 +49,14 @@ export async function runZcodeCanvasTask(ports: Ports): Promise<CanvasAgentRunSt
|
|||||||
"你正在独立沙箱内运行原生 ZCode 核心。本站已授权的图片、节点、界面及项目业务能力通过 canvas MCP 提供;视频画布暂未授权,不得绕过限制;先读实际画布,不猜测节点 ID。",
|
"你正在独立沙箱内运行原生 ZCode 核心。本站已授权的图片、节点、界面及项目业务能力通过 canvas MCP 提供;视频画布暂未授权,不得绕过限制;先读实际画布,不猜测节点 ID。",
|
||||||
`可用业务工具:${tools.join("、")}。工具名称带 mcp__canvas__ 前缀。`,
|
`可用业务工具:${tools.join("、")}。工具名称带 mcp__canvas__ 前缀。`,
|
||||||
"原生文件、Bash、记忆及子任务能力仅作用于此任务的私有工作区,不能通过 shell 直接访问模型服务商或本站后台;业务操作使用 MCP。",
|
"原生文件、Bash、记忆及子任务能力仅作用于此任务的私有工作区,不能通过 shell 直接访问模型服务商或本站后台;业务操作使用 MCP。",
|
||||||
|
"imagePreferences 是用户在输入框选择的生图模型和图片参数,与当前规划模型不同。生成和编辑图片优先用 generateCanvasImages;用户选定的模型不得被技能默认值、你选的其他模型或重试覆盖。比例、尺寸、数量、反向提示词等控件必须传入;不要通过其他生成接口绕过用户选择。分析、问答、排版等不需要出图的任务无需生成图片。",
|
||||||
"查询项目级能力先用 listProjectServices;技能先 loadProjectSkill。项目工具的任务受理、文件选择与后台缺失不等于处理完成。",
|
"查询项目级能力先用 listProjectServices;技能先 loadProjectSkill。项目工具的任务受理、文件选择与后台缺失不等于处理完成。",
|
||||||
"需要澄清时调用 requestCanvasUserInput;不要使用终端交互。工具返回的 userUpdates 是用户的新补充,必须从下一步纳入。",
|
"需要澄清时调用 requestCanvasUserInput;不要使用终端交互。工具返回的 userUpdates 是用户的新补充,必须从下一步纳入。",
|
||||||
"工具错误涉及结果未确认时停止收费动作,不通过更换调用 ID 或另一个工具重复生成。用户明确要求修改的目标才能删除或覆盖。",
|
"工具错误涉及结果未确认时停止收费动作,不通过更换调用 ID 或另一个工具重复生成。用户明确要求修改的目标才能删除或覆盖。",
|
||||||
"只有真实工具返回成功才可声称完成;打开面板、等待用户选文件、后台缺少处理服务都不能声称媒体已处理。",
|
"只有真实工具返回成功才可声称完成;打开面板、等待用户选文件、后台缺少处理服务都不能声称媒体已处理。",
|
||||||
].join("\n");
|
].join("\n");
|
||||||
let prompt = snapshot.nativeRuntime.sessionId
|
let prompt = snapshot.nativeRuntime.sessionId
|
||||||
? ["继续当前任务,保留所有已完成结果。", ...snapshot.inputs.filter(input => input.applied).slice(-4).map(input => input.text)].join("\n")
|
? ["继续当前任务,保留所有已完成结果。", `本轮用户的图片设置:${JSON.stringify(snapshot.imagePreferences || {})},后续生图以此为准。`, ...snapshot.inputs.filter(input => input.applied).slice(-4).map(input => input.text)].join("\n")
|
||||||
: `${snapshot.objective}\n\n当前画布与历史对话上下文:${JSON.stringify(ports.context)}`;
|
: `${snapshot.objective}\n\n当前画布与历史对话上下文:${JSON.stringify(ports.context)}`;
|
||||||
while (true) {
|
while (true) {
|
||||||
const control = await exclusive(() => ports.consumeInputs(snapshot));
|
const control = await exclusive(() => ports.consumeInputs(snapshot));
|
||||||
|
|||||||
@ -1,3 +1,4 @@
|
|||||||
|
import { applyAgentImagePreferences, type AgentImagePreferences } from "@/lib/ai/canvas-agent-runtime/image-preferences";
|
||||||
import { tool, type ToolSet } from "ai";
|
import { tool, type ToolSet } from "ai";
|
||||||
import { z } from "zod";
|
import { z } from "zod";
|
||||||
|
|
||||||
@ -48,6 +49,7 @@ function buildToolDescription(capability: CanvasCapabilityDefinition) {
|
|||||||
export async function createCanvasCapabilityTools(params: {
|
export async function createCanvasCapabilityTools(params: {
|
||||||
callbacks?: CanvasCapabilityToolCallbacks;
|
callbacks?: CanvasCapabilityToolCallbacks;
|
||||||
defaultModelId?: string;
|
defaultModelId?: string;
|
||||||
|
imagePreferences?: AgentImagePreferences;
|
||||||
destructiveConfirmed?: boolean;
|
destructiveConfirmed?: boolean;
|
||||||
projectId: string;
|
projectId: string;
|
||||||
requestHeaders: Headers;
|
requestHeaders: Headers;
|
||||||
@ -94,11 +96,13 @@ export async function createCanvasCapabilityTools(params: {
|
|||||||
input.args && typeof input.args === "object" && !Array.isArray(input.args)
|
input.args && typeof input.args === "object" && !Array.isArray(input.args)
|
||||||
? input.args
|
? input.args
|
||||||
: {};
|
: {};
|
||||||
|
const inheritedArgs = capability.modelPolicy === "agent-selected"
|
||||||
|
? applyAgentImagePreferences(rawArgs, params.imagePreferences) : rawArgs;
|
||||||
const args = {
|
const args = {
|
||||||
...rawArgs,
|
...inheritedArgs,
|
||||||
...(capability.modelPolicy === "agent-selected" &&
|
...(capability.modelPolicy === "agent-selected" &&
|
||||||
params.defaultModelId &&
|
params.defaultModelId &&
|
||||||
!("modelId" in rawArgs)
|
!("modelId" in inheritedArgs)
|
||||||
? { modelId: params.defaultModelId }
|
? { modelId: params.defaultModelId }
|
||||||
: {}),
|
: {}),
|
||||||
};
|
};
|
||||||
|
|||||||
@ -2,7 +2,7 @@ import { defineConfig } from "@playwright/test";
|
|||||||
|
|
||||||
export default defineConfig({
|
export default defineConfig({
|
||||||
testDir: "./tests/routes",
|
testDir: "./tests/routes",
|
||||||
testMatch: /(?:canvas-agent-runtime(?:-store|-review|-video)?|canvas-agent-project-tools|agent-capability-access)\.test\.ts/,
|
testMatch: /(?:canvas-agent-runtime(?:-store|-review|-video|-image-preferences)?|canvas-agent-project-tools|agent-capability-access)\.test\.ts/,
|
||||||
fullyParallel: true,
|
fullyParallel: true,
|
||||||
retries: 0,
|
retries: 0,
|
||||||
workers: 2,
|
workers: 2,
|
||||||
|
|||||||
@ -72,6 +72,19 @@ let checks = 0;
|
|||||||
await options.execute('edit', { nodeIds: ['a'] }, 'corrected', options.signal!); return result; };
|
await options.execute('edit', { nodeIds: ['a'] }, 'corrected', options.signal!); return result; };
|
||||||
assert.equal(await runZcodeCanvasTask(h.ports), 'completed'); assert.equal(h.snapshot.calls.find(c=>c.id==='rejected')?.uncertain,false); checks++;
|
assert.equal(await runZcodeCanvasTask(h.ports), 'completed'); assert.equal(h.snapshot.calls.find(c=>c.id==='rejected')?.uncertain,false); checks++;
|
||||||
}
|
}
|
||||||
|
{
|
||||||
|
const h = fixture('画布里有什么');
|
||||||
|
h.snapshot.nativeRuntime = { engine: 'zcode', sessionId: 'previous-native-session' };
|
||||||
|
h.snapshot.imagePreferences = { modelId: 'user-selected-image-model', parameters: { imageSize: '4K', aspectRatio: '3:4', imageCount: 3, seed: 0, watermark: false } };
|
||||||
|
h.ports.native = async options => {
|
||||||
|
assert.equal(options.sessionId, 'previous-native-session');
|
||||||
|
assert.ok(options.prompt.includes('user-selected-image-model'));
|
||||||
|
assert.ok(options.prompt.includes('4K') && options.prompt.includes('3:4'));
|
||||||
|
assert.ok(options.instructions.includes('不得被技能默认值'));
|
||||||
|
return { ...result, response: '已读取本轮图片设置' };
|
||||||
|
};
|
||||||
|
assert.equal(await runZcodeCanvasTask(h.ports), 'completed'); checks++;
|
||||||
|
}
|
||||||
console.log('NATIVE_TASK_ADAPTER_PASSED', { checks });
|
console.log('NATIVE_TASK_ADAPTER_PASSED', { checks });
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|||||||
75
tests/routes/canvas-agent-runtime-image-preferences.test.ts
Normal file
75
tests/routes/canvas-agent-runtime-image-preferences.test.ts
Normal file
@ -0,0 +1,75 @@
|
|||||||
|
import { expect, test } from "@playwright/test";
|
||||||
|
import { serverLoader } from "../helpers/canvas-download-server-loader";
|
||||||
|
import { applyAgentImagePreferences, agentProviderImageParameters } from "@/lib/ai/canvas-agent-runtime/image-preferences";
|
||||||
|
|
||||||
|
const preferences = {
|
||||||
|
modelId: "user-image-model",
|
||||||
|
parameters: { imageCount: 3, aspectRatio: "3:4", imageSize: "4K", negativePrompt: "不要文字",
|
||||||
|
quality: "high" as const, size: "1536*2048", promptExtend: false, watermark: false, seed: 0 },
|
||||||
|
};
|
||||||
|
function harness() {
|
||||||
|
const generated: any[] = [];
|
||||||
|
const operations: any[] = [];
|
||||||
|
const mock = serverLoader({
|
||||||
|
"@/lib/canvas/capabilities/ai-sdk-tools": { createCanvasCapabilityTools: async () => ({ editTargetImage: {} }) },
|
||||||
|
"@/lib/canvas/capabilities/registry": { CANVAS_CAPABILITY_REGISTRY: [{ id: "image.prompt_regenerate", title: "重绘", risk: "external", agent: { toolName: "editTargetImage" } }] },
|
||||||
|
"@/lib/canvas/agent-mutations": { applyCanvasOperationPlan: async (value: any) => { operations.push(value); return { createdNodeIds: ["new-image"] }; } },
|
||||||
|
"@/lib/canvas/generate-image": { generateCanvasImage: async (value: any) => { generated.push(value); return { modelRef: value.model, images: [{ originalUrl: "https://test.example/result.png", thumbnailUrl: "https://test.example/thumb.png" }] }; } },
|
||||||
|
"@/lib/ai/runtime/generated-image-persistence": { saveGeneratedImages: async () => {} },
|
||||||
|
"@/lib/ai/model-router": { routeToModel: async () => ({ id: "system-default" }) },
|
||||||
|
"@/lib/db/queries": { checkUserImageQuota: async () => ({ allowed: true, remaining: 10 }), getPointsSystemConfig: async () => ({ enabled: false }), getUserById: async () => ({}) },
|
||||||
|
"@/lib/style-pro/model-options": {
|
||||||
|
getStyleProImageModelEntries: async () => ["system-default", "user-image-model"].map(id => ({ model: { id, modelId: id, isDefault: id === "system-default", imageGenConfig: {} } })),
|
||||||
|
sortStyleProImageModelEntries: (entries: any) => entries,
|
||||||
|
},
|
||||||
|
"./canvas": { readOwnedCanvas: async () => ({ nodes: [{ id: "reference", type: "image" }] }), readCanvasImages: async () => [{ id: "reference", data: { src: "https://test.example/reference.png" } }] },
|
||||||
|
"./review": {},
|
||||||
|
"./image-access-tools": { registerImageCanvasAccessTools() {} },
|
||||||
|
});
|
||||||
|
const build = async (imagePreferences: any = preferences) => mock("lib/ai/canvas-agent-runtime/tools.ts").buildCanvasRuntimeTools({
|
||||||
|
userId: "u", chatId: "c", projectId: "p", runId: "r", assistantMessageId: "a", selectedNodeIds: ["reference"],
|
||||||
|
request: new Request("https://test.example/api/canvas/agent-chat"), model: {}, supportsVision: true,
|
||||||
|
preferredModelId: "system-default", imagePreferences, emit() {},
|
||||||
|
});
|
||||||
|
return { build, generated, operations };
|
||||||
|
}
|
||||||
|
test("explicit controls override conflicting planner settings without losing false or zero values", () => {
|
||||||
|
const resolved = applyAgentImagePreferences({ modelId: "planner-choice", imageCount: 1, imageSize: "1K", aspectRatio: "1:1", prompt: "产品图" }, preferences);
|
||||||
|
expect(resolved).toMatchObject({ ...preferences.parameters, modelId: preferences.modelId, prompt: "产品图" });
|
||||||
|
expect(agentProviderImageParameters(resolved)).toEqual({ quality: "high", size: "1536*2048", watermark: false, seed: 0 });
|
||||||
|
});
|
||||||
|
test("real generation tool honors the chosen image model and all parameters despite another default and planner choice", async () => {
|
||||||
|
const h = harness(); const tools = await h.build();
|
||||||
|
const output = await tools.executable.generateCanvasImages.execute({ prompt: "生成产品图", modelId: "planner-choice", imageCount: 1, imageSize: "1K", aspectRatio: "1:1" }, "call-1");
|
||||||
|
expect(h.generated).toHaveLength(1);
|
||||||
|
expect(h.generated[0]).toMatchObject({ prompt: "生成产品图", model: "user-image-model", imageCount: 3, imageSize: "4K", aspectRatio: "3:4", negativePrompt: "不要文字", promptExtend: false, allowFailover: false,
|
||||||
|
providerImageParams: { quality: "high", size: "1536*2048", watermark: false, seed: 0 } });
|
||||||
|
expect(output.appliedImagePreferences).toEqual({ modelId: "user-image-model", imageCount: 3, imageSize: "4K", aspectRatio: "3:4" });
|
||||||
|
});
|
||||||
|
test("redraw and a corrective retry inherit the same user model, batch and image controls", async () => {
|
||||||
|
const h = harness(); const tools = await h.build();
|
||||||
|
await tools.executable.editTargetImage.execute({ nodeIds: ["reference"], args: { prompt: "换背景", modelId: "system-default" } }, "edit-1");
|
||||||
|
await tools.executable.generateCanvasImages.execute({ prompt: "修正背景", modelId: "system-default" }, "retry-1");
|
||||||
|
expect(h.generated).toHaveLength(2);
|
||||||
|
for (const invocation of h.generated) expect(invocation).toMatchObject({ model: preferences.modelId, imageSize: "4K", imageCount: 3, aspectRatio: "3:4", allowFailover: false });
|
||||||
|
});
|
||||||
|
test("an unavailable explicit choice never silently generates or charges through the default", async () => {
|
||||||
|
const h = harness(); const tools = await h.build({ ...preferences, modelId: "disabled-or-missing" });
|
||||||
|
await expect(tools.executable.generateCanvasImages.execute({ prompt: "画图" }, "bad")).rejects.toThrow("指定的图片模型不可用");
|
||||||
|
expect(h.generated).toHaveLength(0); expect(h.operations).toHaveLength(0);
|
||||||
|
});
|
||||||
|
test("Auto selection retains routing and the n count alias without overriding a supplied planner ratio", async () => {
|
||||||
|
const h = harness(); const tools = await h.build({ parameters: { n: 2, imageSize: "8K" } });
|
||||||
|
await tools.executable.generateCanvasImages.execute({ prompt: "画图", aspectRatio: "16:9" }, "auto");
|
||||||
|
expect(h.generated[0]).toMatchObject({ model: "system-default", imageCount: 2, imageSize: "8K", aspectRatio: "16:9", allowFailover: true });
|
||||||
|
});
|
||||||
|
|
||||||
|
test("planning uses the image count for generation tasks but does not turn analysis into an image task", async () => {
|
||||||
|
const h = harness(); const tools = await h.build();
|
||||||
|
const plan = { steps: ["查看画布"], visualCriteria: [] };
|
||||||
|
const analysis = await tools.executable.planCanvasTask.execute(plan, "analysis-plan");
|
||||||
|
expect(analysis.expectedImageCount).toBeUndefined();
|
||||||
|
const generation = await tools.executable.planCanvasTask.execute({ ...plan, expectedImageCount: 1, aspectRatio: "1:1" }, "generation-plan");
|
||||||
|
expect(generation).toMatchObject({ expectedImageCount: 3, aspectRatio: "3:4" });
|
||||||
|
expect(h.generated).toHaveLength(0);
|
||||||
|
});
|
||||||
Loading…
x
Reference in New Issue
Block a user