diff --git a/app/api/canvas/agent-chat/route.ts b/app/api/canvas/agent-chat/route.ts index 69fd9ba5c..916c2d3bd 100644 --- a/app/api/canvas/agent-chat/route.ts +++ b/app/api/canvas/agent-chat/route.ts @@ -1,3 +1,4 @@ +import { getStyleProImageModelEntries } from "@/lib/style-pro/model-options"; import { registerProjectAgentTools } from "@/lib/ai/canvas-agent-runtime/project-tools"; import { isZcodeWorkspaceAuthorized } from "@/lib/ai/canvas-agent-runtime/authorization"; import { @@ -67,6 +68,7 @@ export const runtime = "nodejs"; export const maxDuration = 1200; const requestSchema = postRequestBodySchema.extend({ resumeRunId: z.string().uuid().optional(), + selectedImageModel: z.string().min(1).max(180).optional(), workspaceVariant: z.enum(["canvas", "video"]).default("canvas"), videoAgentSettings: videoAgentSettingsSchema.optional(), }); @@ -161,6 +163,12 @@ export async function POST(request: Request) { ? configured : (await getMultimodalAiModelsWithProvider()).find((entry) => entry.model.supportsVision); const visionModel = visionEntry ? await getLanguageModelById(visionEntry.model.id) : null; + // Validate the user's image choice independently of the Agent planning model. + if (body.selectedImageModel) { + const images = await getStyleProImageModelEntries(); + if (!images.some(({ model }) => model.id === body.selectedImageModel && model.healthStatus !== "auto_disabled")) + throw new AgentRunError("你选择的生图模型当前不可用，请重新选择；不会自动换模型生成。", 400); + } const assistantMessageId = crypto.randomUUID(); const previousVideoMessages = existingChat @@ -205,6 +213,12 @@ export async function POST(request: Request) { snapshot: initialSnapshot, }); const snapshot = run.snapshot; + snapshot.imagePreferences = { + ...snapshot.imagePreferences, + // Each submitted turn captures the current controls, including switching back to Auto. + modelId: body.selectedImageModel, + parameters: body.imageParams || snapshot.imagePreferences?.parameters, + }; const taskSourceNodeIds = snapshot.sourceNodeIds ?? selectedNodeIds; const context = { recentConversation: previousVideoMessages, @@ -221,7 +235,7 @@ export async function POST(request: Request) { title: node.data.title, })), skillInstructions: body.hiddenPrompt || undefined, - imageParameters: body.imageParams, + imagePreferences: snapshot.imagePreferences, targetReason: conversation.targetReason, previousConversationContext: conversation.previousContext, }; @@ -302,8 +316,9 @@ export async function POST(request: Request) { request, model: visionModel || model, supportsVision: Boolean(visionModel), - imageSize: body.imageParams?.imageSize, - preferredModelId: preset?.type === "image" ? preset.defaultModelId || undefined : undefined, + imageSize: snapshot.imagePreferences?.parameters?.imageSize, + imagePreferences: snapshot.imagePreferences, + preferredModelId: snapshot.imagePreferences?.modelId || (preset?.type === "image" ? preset.defaultModelId || undefined : undefined), operationSelectedNodeIds: selectedNodeIds, objective: [snapshot.objective, ...snapshot.inputs.map((input) => input.text)].join( "\n", @@ -311,7 +326,8 @@ export async function POST(request: Request) { emit, }); await registerProjectAgentTools({ session, projectId, request, objective: snapshot.objective, - definitions: tools.definitions, executable: tools.executable, emit }); + definitions: tools.definitions, executable: tools.executable, emit, + imagePreferences: snapshot.imagePreferences }); const gateway = await createZcodeModelGateway(configured); activeModelController = new AbortController(); const status = await runZcodeCanvasTask({ diff --git a/app/canvas/[id]/CanvasEditorLegacyPageClient.tsx b/app/canvas/[id]/CanvasEditorLegacyPageClient.tsx index 907016c7e..8c9828a62 100644 --- a/app/canvas/[id]/CanvasEditorLegacyPageClient.tsx +++ b/app/canvas/[id]/CanvasEditorLegacyPageClient.tsx @@ -11077,6 +11077,7 @@ export default function CanvasEditorLegacyPage({ onClose={() => setIsPanelOpen(false)} onOpenFileList={() => setPanelMode("fileList")} onRequireLogin={openLoginDialog} + onBeforeAgentRequest={() => persistCanvasState("autosave")} selectedAgentPresetId={ isCanvasAgentModeEnabled ? selectedAgentPresetId : null } diff --git a/components/canvas/CanvasChatPanel.tsx b/components/canvas/CanvasChatPanel.tsx index d49dbbca9..9e9017592 100644 --- a/components/canvas/CanvasChatPanel.tsx +++ b/components/canvas/CanvasChatPanel.tsx @@ -422,6 +422,7 @@ interface CanvasChatPanelProps { canvasToolbarConfig?: CanvasToolbarConfig; canvasAgentFeatureSettings?: CanvasAgentFeatureSettings; onRequireLogin?: (message?: string) => void; + onBeforeAgentRequest?: () => Promise; } type CanvasComposerMode = "agent" | "image" | "video" | "detail" | "clawd"; @@ -466,16 +467,16 @@ function getCanvasComposerModeSessionLabel(mode: CanvasComposerMode) { if (mode === "clawd") return CLAW_MODE_LABEL; if (mode === "video") return "视频"; if (mode === "detail") return "AI详情"; - if (mode === "image") return "图像"; + if (mode === "image") return "Agent"; return "Agent"; } function normalizeCanvasComposerSessionMode( value: unknown, - fallback: CanvasComposerMode = "image" + fallback: CanvasComposerMode = "agent" ): CanvasComposerMode { + if (value === "image") return "agent"; // Merge historical image conversations into Agent. return value === "agent" || - value === "image" || value === "video" || value === "detail" || value === "clawd" @@ -485,7 +486,7 @@ function normalizeCanvasComposerSessionMode( function getCanvasChatSessionMode( session: Pick | null | undefined, - fallback: CanvasComposerMode = "image" + fallback: CanvasComposerMode = "agent" ) { if (!session) return fallback; if (session.mode) { @@ -3227,6 +3228,7 @@ export function CanvasChatPanel({ canvasToolbarConfig: rawCanvasToolbarConfig, canvasAgentFeatureSettings: propCanvasAgentFeatureSettings, onRequireLogin, + onBeforeAgentRequest, }: CanvasChatPanelProps) { const isWorkspacePresentation = presentation === "workspace"; const isVideoWorkspace = workspaceVariant === "video"; @@ -5987,7 +5989,7 @@ export function CanvasChatPanel({ [isCanvasImageModel, isCanvasVideoModel] ); const isVideoModel = Boolean( - currentModel && isCanvasVideoModel(currentModel) + isVideoWorkspace && currentModel && isCanvasVideoModel(currentModel) ); const isConversationModel = Boolean( currentModel && isCanvasConversationOnlyModel(currentModel) @@ -6006,11 +6008,15 @@ export function CanvasChatPanel({ if (isDetailGenerationMode) return "detail"; if (isVideoModel) return "video"; if (isAgentModel) return "agent"; - return "image"; - }, [isAgentModel, isDetailGenerationMode, isVideoModel]); + return isVideoWorkspace || isWorkspacePresentation ? "image" : "agent"; + }, [isAgentModel, isDetailGenerationMode, isVideoModel, isVideoWorkspace, isWorkspacePresentation]); const [composerModeOverride, setComposerModeOverride] = useState(null); - const activeComposerMode = composerModeOverride || computedComposerMode; + const unifiedImageAgent = !isVideoWorkspace && !isWorkspacePresentation; + const rawComposerMode = composerModeOverride || computedComposerMode; + const activeComposerMode = unifiedImageAgent && (rawComposerMode === "image" || rawComposerMode === "video") + ? "agent" : rawComposerMode; + const isImageAgentComposer = unifiedImageAgent && activeComposerMode === "agent"; const isVideoToolbarDark = isVideoWorkspace || activeComposerMode === "video"; const isClawdMode = activeComposerMode === "clawd" && isClawdModeEnabled; const activeChatSessionMode = getCanvasChatSessionMode( @@ -6244,13 +6250,14 @@ export function CanvasChatPanel({ return videoOnlyModels; } if (activeComposerMode === "agent") { - return agentSelectableModels; + return unifiedImageAgent ? [SMART_AUTO_MODEL, ...imageOnlyModels] : agentSelectableModels; } return imageOnlyModels; }, [ SMART_AUTO_MODEL, activeComposerMode, agentSelectableModels, + unifiedImageAgent, imageOnlyModels, videoOnlyModels, ]); @@ -6269,7 +6276,7 @@ export function CanvasChatPanel({ activeComposerMode === "agent" ? isPresetAgentSelection ? selectedAgentPreset?.name || "预设Agent" - : "Auto Agent" + : "Agent" : currentModel?.displayName || selectDisplayModel?.displayName || "Agent模式"; const videoFrameSlots = useMemo(() => { if (!isVideoModel || !videoParams) return [] as string[]; @@ -6644,8 +6651,8 @@ export function CanvasChatPanel({ ? "视频" : activeComposerMode === "detail" ? getToolbarItem("chatModeDetailActive").label - : activeComposerMode === "image" - ? "图像" + : activeComposerMode === "image" || isImageAgentComposer + ? "Agent" : isPresetAgentSelection ? getToolbarItem("chatModePresetAgent").label : "Agent"; @@ -6742,7 +6749,7 @@ export function CanvasChatPanel({ useEffect(() => { if (!effectiveAgentEnabled) return; - if (activeComposerMode !== "agent") return; + if (activeComposerMode !== "agent" || unifiedImageAgent) return; if (!isPresetAgentSelection) return; if (!selectedAgentPresetId) return; if (!selectedAgentPreset?.defaultModelId) return; @@ -6755,6 +6762,7 @@ export function CanvasChatPanel({ } }, [ activeComposerMode, + unifiedImageAgent, effectiveAgentEnabled, isPresetAgentSelection, isDetailGenerationMode, @@ -6965,6 +6973,7 @@ export function CanvasChatPanel({ const handleSelectGenerationMode = useCallback( (mode: CanvasComposerMode): string | null => { + if (unifiedImageAgent && (mode === "video" || mode === "image")) return null; hiddenPromptRef.current = null; autoContextRef.current = null; if (mode === "clawd" && !isClawdModeEnabled) { @@ -7066,8 +7075,10 @@ export function CanvasChatPanel({ } if (mode === "agent") { - currentModelIdRef.current = "auto"; - setSelectedModelId("auto"); + // Keep the image model selected by the user when returning from Claw. + const imageModelId = isCanvasImageModel(currentModel) ? currentModel!.id : lastImageModelId || "auto"; + currentModelIdRef.current = unifiedImageAgent ? imageModelId : "auto"; + setSelectedModelId(currentModelIdRef.current); setIsDetailGenerationMode(false); return targetChatSessionId; } @@ -7099,6 +7110,8 @@ export function CanvasChatPanel({ canUseDetailGeneration, effectiveAgentEnabled, isClawdModeEnabled, + unifiedImageAgent, + currentModel, allModels, activeChatId, activeComposerMode, @@ -7903,7 +7916,7 @@ export function CanvasChatPanel({ transport: new DefaultChatTransport({ api: "/api/chat", fetch: fetchWithErrorHandlers, - prepareSendMessagesRequest(request) { + async prepareSendMessagesRequest(request) { // 从 canvas-{uuid} 格式中提取纯 UUID，后端 schema 要求纯 UUID const rawId = request.id.startsWith('canvas-') ? request.id.replace('canvas-', '') @@ -7951,6 +7964,8 @@ export function CanvasChatPanel({ ? selectedAgentPreset.id : undefined, selectedChatModel: effectiveSelectedChatModel, + selectedImageModel: effectiveComposerMode === "agent" && isCanvasImageModel(allModels.find(model => model.id === currentModelIdRef.current)) + ? currentModelIdRef.current : undefined, composerMode: effectiveComposerMode, workspaceVariant: isVideoWorkspace ? "video" : "canvas", selectedVisibilityType: "private", @@ -7964,7 +7979,7 @@ export function CanvasChatPanel({ : deepThinkingEnabledRef.current, videoParams: requestIsVideoModel ? videoParamsRef.current || undefined : undefined, imageParams: - requestIsVideoModel || requestIsConversationModel + effectiveComposerMode !== "agent" && (requestIsVideoModel || requestIsConversationModel) ? undefined : imageParamsRef.current || undefined, hiddenPrompt, @@ -7973,8 +7988,14 @@ export function CanvasChatPanel({ ...(runtimeResumeRunId ? { resumeRunId: runtimeResumeRunId } : {}), ...request.body, }; + const useAgent = shouldUseCanvasAgentRuntime(preparedBody); + // Persist references before the isolated Agent reads the owned canvas. + // Transport admission is already guarded against duplicate sends. + if (useAgent && onBeforeAgentRequest && !(await onBeforeAgentRequest())) { + throw new Error("画布保存未完成，未开始 Agent 任务，请稍后重试。"); + } return { - api: shouldUseCanvasAgentRuntime(preparedBody) ? "/api/canvas/agent-chat" : "/api/chat", + api: useAgent ? "/api/canvas/agent-chat" : "/api/chat", body: preparedBody, }; }, @@ -14791,6 +14812,8 @@ export function CanvasChatPanel({ ? "可选补充：风格、情绪氛围、禁用元素、期望生成张数（如：生成8张）" : isVideoWorkspace ? "描述镜头、主体和运动方式，也可以先切到首帧或首尾帧模式" + : isImageAgentComposer + ? "描述需求，或选择画布图片继续编辑" : activeComposerMode === "agent" && !isPresetAgentSelection ? "" : hasAgentUploadSlots @@ -14807,6 +14830,8 @@ export function CanvasChatPanel({ ? isVideoModel && videoFrameSlots.length > 0 ? `描述镜头、主体和运动方式；可从画布填充${videoFrameSlots.join(" / ")}` : "描述视频镜头、主体、运动方式与时长，也可以切换到首帧或首尾帧模式" + : isImageAgentComposer + ? "描述需求，或选择画布图片继续编辑" : activeComposerMode === "agent" && !isPresetAgentSelection ? "" : activeComposerMode === "agent" && selectedAgentPreset?.type === "text" @@ -15189,7 +15214,7 @@ export function CanvasChatPanel({ isClawdMode ? `开始 ${CLAW_MODE_LABEL} 对话` : activeComposerMode === "agent" && !isPresetAgentSelection - ? "开始 Auto Agent 对话" + ? "开始 Agent 对话" : hasAgentUploadSlots && attachments.length < agentRequiredUploadCount ? `请先补齐图片槽位（${attachments.length}/${agentRequiredUploadCount}）` : "准备开始生成"; @@ -17912,12 +17937,12 @@ export function CanvasChatPanel({
)} - {!isDetailGenerationMode && activeComposerMode === "agent" && ( + {!unifiedImageAgent && !isDetailGenerationMode && activeComposerMode === "agent" && (
{hasSelectedAgentPreset ? ( <> @@ -18292,7 +18317,7 @@ export function CanvasChatPanel({ ) : null}
) : null} - {!isWorkspacePresentation && !isDetailGenerationMode && !isVideoWorkspace && activeComposerMode === "image" ? ( + {!isWorkspacePresentation && !isDetailGenerationMode && !isVideoWorkspace && (activeComposerMode === "image" || isImageAgentComposer) ? (
- {!isVideoModel && !isWorkspacePresentation && activeComposerMode === "image" ? ( + {!isVideoModel && !isWorkspacePresentation && (activeComposerMode === "image" || isImageAgentComposer) ? ( svg]:hidden", isVideoToolbarDark ? "border border-white/12 bg-[#242529] text-zinc-100 hover:bg-[#2c2d33]" @@ -18950,7 +18977,7 @@ export function CanvasChatPanel({ )} title={getToolbarItem("chatModelSelector").label} > - {!isVideoWorkspace && activeComposerMode === "image" ? ( + {!isVideoWorkspace && (activeComposerMode === "image" || isImageAgentComposer) ? ( <> {selectDisplayModel?.displayName || "自动模型"} @@ -19063,7 +19090,7 @@ export function CanvasChatPanel({ ? "停止处理并重新编辑" : isWorkspacePresentation || isDetailGenerationMode ? getToolbarItem("chatStartGeneration").label - : !isVideoWorkspace && activeComposerMode === "image" && estimatedPointsCost + : !isVideoWorkspace && (activeComposerMode === "image" || isImageAgentComposer) && estimatedPointsCost ? `${getToolbarItem("chatSend").label} · ${estimatedPointsCost.title}` : getToolbarItem("chatSend").label } @@ -19098,7 +19125,7 @@ export function CanvasChatPanel({ ) : null} - ) : !isWorkspacePresentation && !isVideoWorkspace && activeComposerMode === "image" && estimatedPointsCost ? ( + ) : !isWorkspacePresentation && !isVideoWorkspace && (activeComposerMode === "image" || isImageAgentComposer) && estimatedPointsCost ? ( <> {estimatedPointsCost.cost} diff --git a/lib/ai/canvas-agent-runtime/image-preferences.ts b/lib/ai/canvas-agent-runtime/image-preferences.ts new file mode 100644 index 000000000..8a1adcc59 --- /dev/null +++ b/lib/ai/canvas-agent-runtime/image-preferences.ts @@ -0,0 +1,36 @@ +/** User image controls are execution constraints, separate from the planning model. */ +export type AgentImageParameters = { + aspectRatio?: string; + imageSize?: string; + imageCount?: number; + n?: number; + quality?: "auto" | "low" | "medium" | "high"; + size?: string; + negativePrompt?: string; + promptExtend?: boolean; + watermark?: boolean; + seed?: number; +}; +export type AgentImagePreferences = { modelId?: string; parameters?: AgentImageParameters }; + +export function applyAgentImagePreferences>( + input: T, + preferences?: AgentImagePreferences, +): T & AgentImageParameters & { modelId?: string } { + const parameters = Object.fromEntries(Object.entries(preferences?.parameters || {}) + .filter(([, value]) => value !== undefined)); + const count = preferences?.parameters?.imageCount ?? preferences?.parameters?.n; + return { + ...input, + ...parameters, + ...(count !== undefined ? { imageCount: count } : {}), + ...(preferences?.modelId ? { modelId: preferences.modelId } : {}), + }; +} + +export function agentProviderImageParameters(parameters: AgentImageParameters) { + return Object.fromEntries(["quality", "size", "watermark", "seed"].flatMap(key => { + const value = parameters[key as keyof AgentImageParameters]; + return value !== undefined ? [[key, value]] : []; + })); +} diff --git a/lib/ai/canvas-agent-runtime/project-tools.ts b/lib/ai/canvas-agent-runtime/project-tools.ts index eb2bfd3a9..9f91a28b3 100644 --- a/lib/ai/canvas-agent-runtime/project-tools.ts +++ b/lib/ai/canvas-agent-runtime/project-tools.ts @@ -1,3 +1,4 @@ +import type { AgentImagePreferences } from "./image-preferences"; import "server-only"; import { AgentToolRejectedError } from "./protocol"; import { tool, type ToolSet } from "ai"; @@ -16,6 +17,7 @@ import { isExplicitDestructiveIntent } from "./capability-access"; import type { CanvasAgentTool } from "./loop"; export async function registerProjectAgentTools(params: { + imagePreferences?: AgentImagePreferences; session: Session; projectId: string; request: Request; objective: string; definitions: ToolSet; executable: Record; emit: (type: string, data: unknown) => void; @@ -35,6 +37,8 @@ export async function registerProjectAgentTools(params: { const args = invokeSchema.parse(input); const operation = PROJECT_AGENT_OPERATIONS.find(item => item.id === args.operationId); if (!operation || operation.readOnly !== readOnly) throw new Error("请使用清单中对应读写类别的固定业务工具"); + if (params.imagePreferences?.modelId && ["style-generate.post", "wash-generate.post"].includes(operation.id)) + throw new AgentToolRejectedError("用户已指定生图模型，请用 generateCanvasImages 执行并保留当前图片参数；不能通过另一条生成接口换用默认模型。"); if (!projectOperationAuthorized(operation)) throw new Error("用户暂未授权视频画布和分镜工具"); if (JSON.stringify(args.body || {}).length > 2 * 1024 * 1024) throw new Error("业务工具输入超过大小限制"); if (args.body?.workspaceType === "video" || args.query.workspaceType === "video" || args.body?.type === "video") throw new Error("视频画布暂未授权"); diff --git a/lib/ai/canvas-agent-runtime/protocol.ts b/lib/ai/canvas-agent-runtime/protocol.ts index 3fd7f6723..6bc5c516e 100644 --- a/lib/ai/canvas-agent-runtime/protocol.ts +++ b/lib/ai/canvas-agent-runtime/protocol.ts @@ -29,6 +29,7 @@ export type AgentRunSnapshot = { traceId?: string; }; workspace?: "canvas" | "video"; + imagePreferences?: import("./image-preferences").AgentImagePreferences; videoSettings?: import("./video-protocol").VideoAgentSettings; videoRequirements?: import("./video-protocol").VideoTaskRequirements; objective: string; diff --git a/lib/ai/canvas-agent-runtime/tools.ts b/lib/ai/canvas-agent-runtime/tools.ts index ba0dfe2ab..b2212328e 100644 --- a/lib/ai/canvas-agent-runtime/tools.ts +++ b/lib/ai/canvas-agent-runtime/tools.ts @@ -1,3 +1,4 @@ +import { applyAgentImagePreferences, agentProviderImageParameters, type AgentImagePreferences } from "./image-preferences"; import { tool, type LanguageModel, type ToolSet } from "ai"; import { z } from "zod"; import { createCanvasCapabilityTools } from "@/lib/canvas/capabilities/ai-sdk-tools"; @@ -33,6 +34,7 @@ export async function buildCanvasRuntimeTools(params: { supportsVision: boolean; imageSize?: string; preferredModelId?: string; + imagePreferences?: AgentImagePreferences; emit: (type: string, data: unknown) => void; objective?: string; operationSelectedNodeIds?: string[]; @@ -45,6 +47,8 @@ export async function buildCanvasRuntimeTools(params: { userId: params.userId, selectedNodeIds: defaults, allowExplicitTargets: true, + defaultModelId: params.preferredModelId, + imagePreferences: params.imagePreferences, authorizeDestructive: (ids) => isExplicitDestructiveIntent(params.objective || "") && ids.length > 0 && @@ -78,6 +82,14 @@ export async function buildCanvasRuntimeTools(params: { execute: async (input, callId) => { if (!defaults.length && !Array.isArray(input.nodeIds)) throw new Error("请明确提供目标图片节点，不能猜测第一张图。"); + // Generic redraw is the same constrained generator, including batch count, + // provider parameters and the no-silent-model-failover policy. + if (capability.id === "image.prompt_regenerate") { + const args = (input.args || {}) as Record; + return executable.generateCanvasImages.execute({ + ...args, referenceNodeIds: input.nodeIds || defaults, + }, callId); + } const result = await entry.execute!(input as never, { toolCallId: callId, messages: [] }); return result as Record; }, @@ -113,7 +125,12 @@ export async function buildCanvasRuntimeTools(params: { aspectRatio: z.string().max(24).optional(), visualCriteria: z.array(z.string().max(200)).max(10), }), - async (input) => ({ ...input }), + async (input) => ({ ...input, + ...(input.expectedImageCount !== undefined && params.imagePreferences?.parameters?.imageCount !== undefined + ? { expectedImageCount: params.imagePreferences.parameters.imageCount } : {}), + ...(params.imagePreferences?.parameters?.aspectRatio && params.imagePreferences.parameters.aspectRatio !== "auto" + ? { aspectRatio: params.imagePreferences.parameters.aspectRatio } : {}), + }), ); register( "readCanvas", @@ -165,10 +182,11 @@ export async function buildCanvasRuntimeTools(params: { referenceNodeIds: z.array(z.string().min(1).max(180)).max(8).default([]), imageCount: z.number().int().min(1).max(10).default(1), aspectRatio: z.string().max(24).default("auto"), - imageSize: z.enum(["1K", "2K", "4K"]).optional(), + imageSize: z.enum(["0.5K", "1K", "2K", "4K", "8K", "10K", "12K"]).optional(), modelId: z.string().max(180).optional().describe("指定已配置的图片模型 ID；省略使用预设或系统模型"), }), - async (input, callId) => { + async (toolInput, callId) => { + const input = applyAgentImagePreferences(toolInput, params.imagePreferences); // Bound per-task spend without introducing any global/model concurrency queue. if (generatedCount + input.imageCount > 24) throw new Error("本次任务已达到 24 张生成预算，请检查结果后继续。"); @@ -182,7 +200,7 @@ export async function buildCanvasRuntimeTools(params: { intent: sources.length ? "image_edit" : "image_gen", hasImage: sources.length > 0, }).catch(() => null); - const requestedModel = input.modelId || params.preferredModelId; + const requestedModel = params.imagePreferences?.modelId || input.modelId || params.preferredModelId; const requestedEntry = requestedModel ? models.find(({ model }) => model.healthStatus !== "auto_disabled" && (model.id === requestedModel || model.modelId === requestedModel)) : null; if (requestedModel && !requestedEntry) throw new Error("指定的图片模型不可用，未切换到其他模型收费生成"); const chosen = requestedEntry || models.find( @@ -217,6 +235,10 @@ export async function buildCanvasRuntimeTools(params: { imageCount: input.imageCount, imageSize, aspectRatio: input.aspectRatio, + negativePrompt: input.negativePrompt, + promptExtend: input.promptExtend, + providerImageParams: agentProviderImageParameters(input), + allowFailover: !params.imagePreferences?.modelId, ownerKey: params.userId, referenceImages: sources.map((node) => node.data.src || node.data.url!), debugLabel: "canvas-agent-runtime", @@ -278,6 +300,7 @@ export async function buildCanvasRuntimeTools(params: { ); return { status: "succeeded", + appliedImagePreferences: { modelId: chosen.model.id, imageSize, aspectRatio: input.aspectRatio, imageCount: input.imageCount }, createdNodeIds: operation.createdNodeIds, outputs: result.images.map((image) => ({ url: image.originalUrl })), model: result.modelRef, diff --git a/lib/ai/zcode-agent/task.ts b/lib/ai/zcode-agent/task.ts index 70df3e760..b21ce8747 100644 --- a/lib/ai/zcode-agent/task.ts +++ b/lib/ai/zcode-agent/task.ts @@ -49,13 +49,14 @@ export async function runZcodeCanvasTask(ports: Ports): Promise input.applied).slice(-4).map(input => input.text)].join("\n") + ? ["继续当前任务，保留所有已完成结果。", `本轮用户的图片设置：${JSON.stringify(snapshot.imagePreferences || {})}，后续生图以此为准。`, ...snapshot.inputs.filter(input => input.applied).slice(-4).map(input => input.text)].join("\n") : `${snapshot.objective}\n\n当前画布与历史对话上下文：${JSON.stringify(ports.context)}`; while (true) { const control = await exclusive(() => ports.consumeInputs(snapshot)); diff --git a/lib/canvas/capabilities/ai-sdk-tools.ts b/lib/canvas/capabilities/ai-sdk-tools.ts index 5f548ae0e..c633a35cf 100644 --- a/lib/canvas/capabilities/ai-sdk-tools.ts +++ b/lib/canvas/capabilities/ai-sdk-tools.ts @@ -1,3 +1,4 @@ +import { applyAgentImagePreferences, type AgentImagePreferences } from "@/lib/ai/canvas-agent-runtime/image-preferences"; import { tool, type ToolSet } from "ai"; import { z } from "zod"; @@ -48,6 +49,7 @@ function buildToolDescription(capability: CanvasCapabilityDefinition) { export async function createCanvasCapabilityTools(params: { callbacks?: CanvasCapabilityToolCallbacks; defaultModelId?: string; + imagePreferences?: AgentImagePreferences; destructiveConfirmed?: boolean; projectId: string; requestHeaders: Headers; @@ -94,11 +96,13 @@ export async function createCanvasCapabilityTools(params: { input.args && typeof input.args === "object" && !Array.isArray(input.args) ? input.args : {}; + const inheritedArgs = capability.modelPolicy === "agent-selected" + ? applyAgentImagePreferences(rawArgs, params.imagePreferences) : rawArgs; const args = { - ...rawArgs, + ...inheritedArgs, ...(capability.modelPolicy === "agent-selected" && params.defaultModelId && - !("modelId" in rawArgs) + !("modelId" in inheritedArgs) ? { modelId: params.defaultModelId } : {}), }; diff --git a/playwright.canvas-agent.config.ts b/playwright.canvas-agent.config.ts index 4b8066e4b..9fc663ea8 100644 --- a/playwright.canvas-agent.config.ts +++ b/playwright.canvas-agent.config.ts @@ -2,7 +2,7 @@ import { defineConfig } from "@playwright/test"; export default defineConfig({ testDir: "./tests/routes", - testMatch: /(?:canvas-agent-runtime(?:-store|-review|-video)?|canvas-agent-project-tools|agent-capability-access)\.test\.ts/, + testMatch: /(?:canvas-agent-runtime(?:-store|-review|-video|-image-preferences)?|canvas-agent-project-tools|agent-capability-access)\.test\.ts/, fullyParallel: true, retries: 0, workers: 2, diff --git a/scripts/zcode-agent/task-fixture.ts b/scripts/zcode-agent/task-fixture.ts index 968e58606..4e4ab8985 100644 --- a/scripts/zcode-agent/task-fixture.ts +++ b/scripts/zcode-agent/task-fixture.ts @@ -72,6 +72,19 @@ let checks = 0; await options.execute('edit', { nodeIds: ['a'] }, 'corrected', options.signal!); return result; }; assert.equal(await runZcodeCanvasTask(h.ports), 'completed'); assert.equal(h.snapshot.calls.find(c=>c.id==='rejected')?.uncertain,false); checks++; } +{ + const h = fixture('画布里有什么'); + h.snapshot.nativeRuntime = { engine: 'zcode', sessionId: 'previous-native-session' }; + h.snapshot.imagePreferences = { modelId: 'user-selected-image-model', parameters: { imageSize: '4K', aspectRatio: '3:4', imageCount: 3, seed: 0, watermark: false } }; + h.ports.native = async options => { + assert.equal(options.sessionId, 'previous-native-session'); + assert.ok(options.prompt.includes('user-selected-image-model')); + assert.ok(options.prompt.includes('4K') && options.prompt.includes('3:4')); + assert.ok(options.instructions.includes('不得被技能默认值')); + return { ...result, response: '已读取本轮图片设置' }; + }; + assert.equal(await runZcodeCanvasTask(h.ports), 'completed'); checks++; +} console.log('NATIVE_TASK_ADAPTER_PASSED', { checks }); } diff --git a/tests/routes/canvas-agent-runtime-image-preferences.test.ts b/tests/routes/canvas-agent-runtime-image-preferences.test.ts new file mode 100644 index 000000000..1c4f260aa --- /dev/null +++ b/tests/routes/canvas-agent-runtime-image-preferences.test.ts @@ -0,0 +1,75 @@ +import { expect, test } from "@playwright/test"; +import { serverLoader } from "../helpers/canvas-download-server-loader"; +import { applyAgentImagePreferences, agentProviderImageParameters } from "@/lib/ai/canvas-agent-runtime/image-preferences"; + +const preferences = { + modelId: "user-image-model", + parameters: { imageCount: 3, aspectRatio: "3:4", imageSize: "4K", negativePrompt: "不要文字", + quality: "high" as const, size: "1536*2048", promptExtend: false, watermark: false, seed: 0 }, +}; +function harness() { + const generated: any[] = []; + const operations: any[] = []; + const mock = serverLoader({ + "@/lib/canvas/capabilities/ai-sdk-tools": { createCanvasCapabilityTools: async () => ({ editTargetImage: {} }) }, + "@/lib/canvas/capabilities/registry": { CANVAS_CAPABILITY_REGISTRY: [{ id: "image.prompt_regenerate", title: "重绘", risk: "external", agent: { toolName: "editTargetImage" } }] }, + "@/lib/canvas/agent-mutations": { applyCanvasOperationPlan: async (value: any) => { operations.push(value); return { createdNodeIds: ["new-image"] }; } }, + "@/lib/canvas/generate-image": { generateCanvasImage: async (value: any) => { generated.push(value); return { modelRef: value.model, images: [{ originalUrl: "https://test.example/result.png", thumbnailUrl: "https://test.example/thumb.png" }] }; } }, + "@/lib/ai/runtime/generated-image-persistence": { saveGeneratedImages: async () => {} }, + "@/lib/ai/model-router": { routeToModel: async () => ({ id: "system-default" }) }, + "@/lib/db/queries": { checkUserImageQuota: async () => ({ allowed: true, remaining: 10 }), getPointsSystemConfig: async () => ({ enabled: false }), getUserById: async () => ({}) }, + "@/lib/style-pro/model-options": { + getStyleProImageModelEntries: async () => ["system-default", "user-image-model"].map(id => ({ model: { id, modelId: id, isDefault: id === "system-default", imageGenConfig: {} } })), + sortStyleProImageModelEntries: (entries: any) => entries, + }, + "./canvas": { readOwnedCanvas: async () => ({ nodes: [{ id: "reference", type: "image" }] }), readCanvasImages: async () => [{ id: "reference", data: { src: "https://test.example/reference.png" } }] }, + "./review": {}, + "./image-access-tools": { registerImageCanvasAccessTools() {} }, + }); + const build = async (imagePreferences: any = preferences) => mock("lib/ai/canvas-agent-runtime/tools.ts").buildCanvasRuntimeTools({ + userId: "u", chatId: "c", projectId: "p", runId: "r", assistantMessageId: "a", selectedNodeIds: ["reference"], + request: new Request("https://test.example/api/canvas/agent-chat"), model: {}, supportsVision: true, + preferredModelId: "system-default", imagePreferences, emit() {}, + }); + return { build, generated, operations }; +} +test("explicit controls override conflicting planner settings without losing false or zero values", () => { + const resolved = applyAgentImagePreferences({ modelId: "planner-choice", imageCount: 1, imageSize: "1K", aspectRatio: "1:1", prompt: "产品图" }, preferences); + expect(resolved).toMatchObject({ ...preferences.parameters, modelId: preferences.modelId, prompt: "产品图" }); + expect(agentProviderImageParameters(resolved)).toEqual({ quality: "high", size: "1536*2048", watermark: false, seed: 0 }); +}); +test("real generation tool honors the chosen image model and all parameters despite another default and planner choice", async () => { + const h = harness(); const tools = await h.build(); + const output = await tools.executable.generateCanvasImages.execute({ prompt: "生成产品图", modelId: "planner-choice", imageCount: 1, imageSize: "1K", aspectRatio: "1:1" }, "call-1"); + expect(h.generated).toHaveLength(1); + expect(h.generated[0]).toMatchObject({ prompt: "生成产品图", model: "user-image-model", imageCount: 3, imageSize: "4K", aspectRatio: "3:4", negativePrompt: "不要文字", promptExtend: false, allowFailover: false, + providerImageParams: { quality: "high", size: "1536*2048", watermark: false, seed: 0 } }); + expect(output.appliedImagePreferences).toEqual({ modelId: "user-image-model", imageCount: 3, imageSize: "4K", aspectRatio: "3:4" }); +}); +test("redraw and a corrective retry inherit the same user model, batch and image controls", async () => { + const h = harness(); const tools = await h.build(); + await tools.executable.editTargetImage.execute({ nodeIds: ["reference"], args: { prompt: "换背景", modelId: "system-default" } }, "edit-1"); + await tools.executable.generateCanvasImages.execute({ prompt: "修正背景", modelId: "system-default" }, "retry-1"); + expect(h.generated).toHaveLength(2); + for (const invocation of h.generated) expect(invocation).toMatchObject({ model: preferences.modelId, imageSize: "4K", imageCount: 3, aspectRatio: "3:4", allowFailover: false }); +}); +test("an unavailable explicit choice never silently generates or charges through the default", async () => { + const h = harness(); const tools = await h.build({ ...preferences, modelId: "disabled-or-missing" }); + await expect(tools.executable.generateCanvasImages.execute({ prompt: "画图" }, "bad")).rejects.toThrow("指定的图片模型不可用"); + expect(h.generated).toHaveLength(0); expect(h.operations).toHaveLength(0); +}); +test("Auto selection retains routing and the n count alias without overriding a supplied planner ratio", async () => { + const h = harness(); const tools = await h.build({ parameters: { n: 2, imageSize: "8K" } }); + await tools.executable.generateCanvasImages.execute({ prompt: "画图", aspectRatio: "16:9" }, "auto"); + expect(h.generated[0]).toMatchObject({ model: "system-default", imageCount: 2, imageSize: "8K", aspectRatio: "16:9", allowFailover: true }); +}); + +test("planning uses the image count for generation tasks but does not turn analysis into an image task", async () => { + const h = harness(); const tools = await h.build(); + const plan = { steps: ["查看画布"], visualCriteria: [] }; + const analysis = await tools.executable.planCanvasTask.execute(plan, "analysis-plan"); + expect(analysis.expectedImageCount).toBeUndefined(); + const generation = await tools.executable.planCanvasTask.execute({ ...plan, expectedImageCount: 1, aspectRatio: "1:1" }, "generation-plan"); + expect(generation).toMatchObject({ expectedImageCount: 3, aspectRatio: "3:4" }); + expect(h.generated).toHaveLength(0); +});