import { BoxCutoutServiceError, executeBoxCutoutJob, } from "@/lib/ai/box-cutout/service"; import { executeExtendJob, ExtendServiceError, inferExtendOptionsFromText, } from "@/lib/ai/extend/service"; import { executeResizeJob, inferResizeOptionsFromText, ResizeServiceError, } from "@/lib/ai/resize/service"; import { executeRmbgJob, fetchImageBufferFromUrl, RmbgServiceError, } from "@/lib/ai/rmbg/service"; import { generateCanvasImage } from "@/lib/canvas/generate-image"; import { requestLongTaskResult } from "@/lib/client/long-task"; import { getMediaAssetsByUserAndUrls } from "@/lib/db/queries"; import type { AgentExecutionPlan, AgentPreparedExecution, } from "@/lib/ai/auto-agent-graph"; import { findAgentToolCallArgs, type DirectAgentToolCapability, } from "@/lib/ai/runtime/direct-agent-execution"; import { buildDirectAgentExecutionPrompt } from "@/lib/ai/runtime/optimized-prompt"; import { buildAgentToolImagesMarkdown } from "@/lib/ai/runtime/agent-media-markdown"; import { clampAngleEditNumber, inferAngleEditOptionsFromText, type DirectAgentAngleEditOptions, } from "@/lib/ai/runtime/angle-edit-options"; import { DirectAgentLocalEditServiceError, resolveDirectAgentLocalEditRegions, type DirectAgentLocalEditRegion, } from "@/lib/ai/runtime/local-edit-options"; import type { CanvasAgentContext } from "@/types/canvas/agent-protocol"; interface DirectAgentImageParamsSnapshot { negativePrompt?: string; aspectRatio?: string; imageSize?: string; } type SaveGeneratedImages = ( imageMarkdown: string, userId: string, chatId: string | null, messageId: string | null, prompt: string, modelId: string, options?: { sourceAssetIds?: string[]; sourceImageUrls?: string[]; relationType?: "generated_from" | "edited_from"; sourceType?: string; projectId?: string | null; userPrompt?: string; optimizedPrompt?: string; requestedImageCount?: number; aspectRatio?: string; imageSize?: string; skipBilling?: boolean; }, ) => Promise<{ savedImageCount: number; savedImageIds: string[]; savedAssetIds: string[]; savedOriginalUrls: string[]; }>; export interface DirectAgentImageEditExecutionResult { originalUrl: string; thumbnailUrl: string | null; imageSize: string; mediaAssetId: string | null; } export interface DirectAgentBatchImageEditExecutionResult extends DirectAgentImageEditExecutionResult { sourceImageUrl: string; sourceIndex: number; } export interface ExecuteDirectAgentImageEditToolParams { capability: DirectAgentToolCapability; requestUrl: string; requestHeaders: Headers; userId: string; chatId: string; messageId: string; projectId?: string | null; targetAssetIds?: string[] | null; sourceImageUrl: string; sourceImageUrls: string[]; requestTextContent: string; optimizedPrompt?: string; hiddenPromptPrefix?: string; finalModelId: string; imageParams?: DirectAgentImageParamsSnapshot | null; defaultImageQualityParams?: DirectAgentImageParamsSnapshot | null; smartAutoExecution?: AgentPreparedExecution; smartAutoPlan?: AgentExecutionPlan; canvasAgentContext?: CanvasAgentContext | null; targetNodeIds?: string[] | null; saveGeneratedImages: SaveGeneratedImages; } function toFiniteNumber(value: unknown): number | undefined { const normalized = Number(value); return Number.isFinite(normalized) ? normalized : undefined; } function sleep(ms: number) { return new Promise((resolve) => setTimeout(resolve, ms)); } async function executeDirectAgentEnhanceJob(params: { requestUrl: string; requestHeaders: Headers; imageUrl: string; prompt: string; chatId: string; messageId: string; }) { const endpoint = new URL("/api/enhance", params.requestUrl).toString(); const cookieHeader = params.requestHeaders.get("cookie") || ""; const result = await requestLongTaskResult({ endpoint, payload: { imageUrl: params.imageUrl, prompt: params.prompt, chatId: params.chatId, messageId: params.messageId, }, pollingParams: { prompt: params.prompt, chatId: params.chatId, messageId: params.messageId, sourceImageUrl: params.imageUrl, }, timeoutMs: 10 * 60 * 1000, headers: cookieHeader ? { cookie: cookieHeader, } : undefined, }); if (!result?.success || !result?.original_url) { throw new Error(String(result?.error || result?.detail || "图像增强失败")); } return { originalUrl: String(result.original_url), thumbnailUrl: String(result.thumbnail_url || result.original_url), imageSize: "2K", }; } async function executeDirectAgentWatermarkRemoveJob(params: { requestUrl: string; requestHeaders: Headers; imageUrl: string; }) { const endpoint = new URL("/api/watermark-remove", params.requestUrl).toString(); const cookieHeader = params.requestHeaders.get("cookie") || ""; const result = await requestLongTaskResult({ endpoint, payload: { image_url: params.imageUrl, }, timeoutMs: 20 * 60 * 1000, headers: cookieHeader ? { cookie: cookieHeader, } : undefined, }); if (!result?.success || !result?.original_url) { throw new Error(String(result?.error || result?.detail || "去水印失败")); } return { originalUrl: String(result.original_url), thumbnailUrl: String(result.thumbnail_url || result.original_url), imageSize: "2K", }; } async function executeDirectAgentAngleEditJob(params: { requestUrl: string; requestHeaders: Headers; imageUrl: string; options: DirectAgentAngleEditOptions; }) { const endpoint = new URL("/api/angle-edit", params.requestUrl).toString(); const cookieHeader = params.requestHeaders.get("cookie") || ""; const result = await requestLongTaskResult({ endpoint, payload: { image_url: params.imageUrl, horizontal_angle: params.options.horizontalAngle, vertical_angle: params.options.verticalAngle, zoom: params.options.zoom, ...(params.options.loraScale !== undefined ? { lora_scale: params.options.loraScale } : {}), }, timeoutMs: 20 * 60 * 1000, headers: cookieHeader ? { cookie: cookieHeader, } : undefined, }); if (!result?.success || !result?.image_url) { throw new Error(String(result?.error || result?.detail || "多视角生成失败")); } return { originalUrl: String(result.image_url), thumbnailUrl: String(result.thumbnail_url || result.image_url), imageSize: "2K", }; } async function safeParseJsonResponse(response: Response) { try { return await response.json(); } catch { return null; } } async function pollDirectAgentCanvasImageTaskResult(params: { requestUrl: string; requestHeaders: Headers; taskId: string; timeoutMs?: number; }) { const cookieHeader = params.requestHeaders.get("cookie") || ""; const endpoint = new URL("/api/canvas/generate/image", params.requestUrl); endpoint.searchParams.set("taskId", params.taskId); const timeoutMs = Math.max(10_000, params.timeoutMs ?? 20 * 60 * 1000); const startedAt = Date.now(); let pollAfterMs = 1500; while (Date.now() - startedAt < timeoutMs) { await sleep(pollAfterMs); const response = await fetch(endpoint.toString(), { method: "GET", cache: "no-store", headers: cookieHeader ? { cookie: cookieHeader } : undefined, }); const payload = await safeParseJsonResponse(response); if (!response.ok || payload?.success === false) { const detail = String(payload?.error || payload?.detail || "").trim(); throw new DirectAgentLocalEditServiceError( detail || `查询局部编辑任务失败(${response.status})`, response.status || 500, ); } const task = payload?.task || {}; const status = String(task.status || "").toLowerCase(); if (status === "succeeded") { const result = task.result || {}; const images = Array.isArray(result.images) ? result.images : []; const firstImage = images[0]; const originalUrl = String(firstImage?.originalUrl || firstImage?.original_url || "").trim(); const thumbnailUrl = String( firstImage?.thumbnailUrl || firstImage?.thumbnail_url || firstImage?.thumbnail || originalUrl ).trim(); if (!originalUrl) { throw new DirectAgentLocalEditServiceError("局部编辑任务完成但未返回图片", 500); } return { originalUrl, thumbnailUrl: thumbnailUrl || originalUrl, imageSize: "2K", modelRef: String(result.modelRef || result.model || "").trim(), }; } if (status === "failed") { throw new DirectAgentLocalEditServiceError( String(task.error || "局部编辑任务失败"), 500, ); } const estimatedWaitMs = Number(task.queueStatus?.estimatedWaitMs || 0); pollAfterMs = estimatedWaitMs > 0 ? Math.min(4000, Math.max(1200, estimatedWaitMs)) : 1500; } throw new DirectAgentLocalEditServiceError("局部编辑任务处理超时,请稍后再试", 504); } async function executeDirectAgentLocalEditJob(params: { requestUrl: string; requestHeaders: Headers; imageUrl: string; sourceNodeId?: string; regions: DirectAgentLocalEditRegion[]; model?: string; imageSize?: string; aspectRatio?: string; }) { const endpoint = new URL("/api/canvas/local-edit", params.requestUrl).toString(); const cookieHeader = params.requestHeaders.get("cookie") || ""; const response = await fetch(endpoint, { method: "POST", headers: { "Content-Type": "application/json", ...(cookieHeader ? { cookie: cookieHeader } : {}), }, body: JSON.stringify({ sourceImageUrl: params.imageUrl, sourceNodeId: params.sourceNodeId, regions: params.regions, operation: "local-edit", model: params.model, imageSize: params.imageSize || "2K", aspectRatio: params.aspectRatio || "auto", }), }); const payload = await safeParseJsonResponse(response); if (!response.ok || payload?.success === false) { throw new DirectAgentLocalEditServiceError( String(payload?.error || payload?.detail || "局部编辑提交失败"), response.status || 500, ); } const taskId = String(payload?.taskId || "").trim(); if (!taskId) { throw new DirectAgentLocalEditServiceError("局部编辑提交成功但未返回任务ID", 500); } return pollDirectAgentCanvasImageTaskResult({ requestUrl: params.requestUrl, requestHeaders: params.requestHeaders, taskId, }); } export function isDirectAgentImageEditServiceError(error: unknown) { return ( error instanceof RmbgServiceError || error instanceof ResizeServiceError || error instanceof ExtendServiceError || error instanceof BoxCutoutServiceError || error instanceof DirectAgentLocalEditServiceError ); } async function resolveDirectAgentResultAssetId(params: { userId: string; originalUrl: string; mediaAssetId?: string | null; capability: DirectAgentToolCapability; }) { if (params.mediaAssetId || !params.originalUrl) { return params.mediaAssetId || null; } try { const [resultAsset] = await getMediaAssetsByUserAndUrls({ userId: params.userId, urls: [params.originalUrl], }); return resultAsset?.id || null; } catch (assetLookupError) { console.warn( `[Direct Agent Image Edit] Failed to lookup result asset after ${params.capability}:`, assetLookupError, ); return null; } } async function finalizeDirectAgentImageEditResult(params: { userId: string; capability: DirectAgentToolCapability; originalUrl: string; thumbnailUrl?: string | null; imageSize: string; mediaAssetId?: string | null; }): Promise { return { originalUrl: params.originalUrl, thumbnailUrl: params.thumbnailUrl || null, imageSize: params.imageSize, mediaAssetId: await resolveDirectAgentResultAssetId({ userId: params.userId, originalUrl: params.originalUrl, mediaAssetId: params.mediaAssetId, capability: params.capability, }), }; } export async function executeDirectAgentImageEditTool( params: ExecuteDirectAgentImageEditToolParams, ): Promise { if (params.capability === "generic_edit") { const directEditPrompt = buildDirectAgentExecutionPrompt({ originalText: params.requestTextContent || "", optimizedPrompt: params.optimizedPrompt, hiddenPromptPrefix: params.hiddenPromptPrefix, fallback: "在保持主体和整体构图稳定的前提下完成本轮图片编辑。", }) || "在保持主体和整体构图稳定的前提下完成本轮图片编辑。"; const aspectRatio = params.imageParams?.aspectRatio || params.defaultImageQualityParams?.aspectRatio; const imageSize = params.imageParams?.imageSize || params.defaultImageQualityParams?.imageSize || "2K"; const result = await generateCanvasImage({ prompt: directEditPrompt, negativePrompt: params.imageParams?.negativePrompt, model: params.finalModelId, ownerKey: params.userId, aspectRatio, imageSize, imageCount: 1, referenceImages: params.sourceImageUrls, debugLabel: "agent-direct-edit", debugMeta: { chatId: params.chatId, projectId: params.projectId || null, subIntent: params.smartAutoExecution?.routeSubIntent || null, }, }); const firstImage = result.images[0]; if (!firstImage?.originalUrl) { throw new Error("图片编辑完成但未返回结果图"); } const originalUrl = firstImage.originalUrl; const thumbnailUrl = firstImage.thumbnailUrl || null; const editResultMarkdown = buildAgentToolImagesMarkdown([ { originalUrl, thumbnailUrl, imageSize, }, ]); await params.saveGeneratedImages( editResultMarkdown, params.userId, params.chatId, params.messageId, directEditPrompt, result.modelRef || params.finalModelId, { sourceAssetIds: params.targetAssetIds || undefined, sourceImageUrls: params.sourceImageUrls, relationType: "edited_from", sourceType: "edited", projectId: params.projectId || null, userPrompt: params.requestTextContent, optimizedPrompt: directEditPrompt, requestedImageCount: 1, aspectRatio, imageSize, }, ); return finalizeDirectAgentImageEditResult({ userId: params.userId, capability: params.capability, originalUrl, thumbnailUrl, imageSize, mediaAssetId: null, }); } if (params.capability === "enhance") { const result = await executeDirectAgentEnhanceJob({ requestUrl: params.requestUrl, requestHeaders: params.requestHeaders, imageUrl: params.sourceImageUrl, prompt: params.requestTextContent || "图像增强处理", chatId: params.chatId, messageId: params.messageId, }); return finalizeDirectAgentImageEditResult({ userId: params.userId, capability: params.capability, originalUrl: result.originalUrl, thumbnailUrl: result.thumbnailUrl, imageSize: result.imageSize, mediaAssetId: null, }); } if (params.capability === "remove_background") { const sourceBuffer = await fetchImageBufferFromUrl(params.sourceImageUrl); const result = await executeRmbgJob({ userId: params.userId, imageBuffer: sourceBuffer, prompt: params.requestTextContent || "抠图处理", chatId: params.chatId, messageId: params.messageId, sourceImageUrl: params.sourceImageUrl, }); return finalizeDirectAgentImageEditResult({ userId: params.userId, capability: params.capability, originalUrl: result.original_url, thumbnailUrl: result.thumbnail_url, imageSize: "2K", mediaAssetId: result.mediaAssetId || null, }); } if (params.capability === "remove_watermark") { const result = await executeDirectAgentWatermarkRemoveJob({ requestUrl: params.requestUrl, requestHeaders: params.requestHeaders, imageUrl: params.sourceImageUrl, }); return finalizeDirectAgentImageEditResult({ userId: params.userId, capability: params.capability, originalUrl: result.originalUrl, thumbnailUrl: result.thumbnailUrl, imageSize: result.imageSize, mediaAssetId: null, }); } if (params.capability === "resize") { const plannedArgs = findAgentToolCallArgs( "resizeImage", params.smartAutoExecution, params.smartAutoPlan, ); const inferredOptions = inferResizeOptionsFromText(params.requestTextContent); const canvasWidth = toFiniteNumber( plannedArgs?.canvasWidth ?? plannedArgs?.width ?? inferredOptions.canvasWidth, ); const canvasHeight = toFiniteNumber( plannedArgs?.canvasHeight ?? plannedArgs?.height ?? inferredOptions.canvasHeight, ); if (!canvasWidth || !canvasHeight) { throw new ResizeServiceError( "请告诉我目标尺寸或比例,例如 1024x1024、4:5、16:9。", 400, ); } const transparentBackground = plannedArgs?.transparentBackground === true || plannedArgs?.transparentBackground === "true" || inferredOptions.transparentBackground === true; const backgroundColor = typeof plannedArgs?.backgroundColor === "string" ? plannedArgs.backgroundColor : inferredOptions.backgroundColor; const mimeType = typeof plannedArgs?.mimeType === "string" ? plannedArgs.mimeType : undefined; const result = await executeResizeJob({ userId: params.userId, imageUrl: params.sourceImageUrl, prompt: params.requestTextContent || "尺寸调整处理", chatId: params.chatId, messageId: params.messageId, sourceImageUrl: params.sourceImageUrl, canvasWidth, canvasHeight, transparentBackground, backgroundColor, mimeType, }); return finalizeDirectAgentImageEditResult({ userId: params.userId, capability: params.capability, originalUrl: result.original_url, thumbnailUrl: result.thumbnail_url, imageSize: `${result.canvas_width}x${result.canvas_height}`, mediaAssetId: result.mediaAssetId || null, }); } if (params.capability === "extend") { const plannedArgs = findAgentToolCallArgs( "extendImage", params.smartAutoExecution, params.smartAutoPlan, ); const inferredOptions = inferExtendOptionsFromText(params.requestTextContent); const xScale = toFiniteNumber(plannedArgs?.xScale ?? inferredOptions.xScale); const yScale = toFiniteNumber(plannedArgs?.yScale ?? inferredOptions.yScale); const result = await executeExtendJob({ userId: params.userId, imageUrl: params.sourceImageUrl, prompt: params.requestTextContent || "图像拓展处理", chatId: params.chatId, messageId: params.messageId, sourceImageUrl: params.sourceImageUrl, ...(xScale ? { xScale } : {}), ...(yScale ? { yScale } : {}), }); return finalizeDirectAgentImageEditResult({ userId: params.userId, capability: params.capability, originalUrl: result.original_url, thumbnailUrl: result.thumbnail_url, imageSize: "2K", mediaAssetId: result.mediaAssetId || null, }); } if (params.capability === "angle_variant") { const plannedArgs = findAgentToolCallArgs( "angleEditImage", params.smartAutoExecution, params.smartAutoPlan, ); const inferredOptions = inferAngleEditOptionsFromText(params.requestTextContent); const options: DirectAgentAngleEditOptions = { horizontalAngle: clampAngleEditNumber( toFiniteNumber( plannedArgs?.horizontal_angle ?? plannedArgs?.horizontalAngle ?? plannedArgs?.angle ?? inferredOptions.horizontalAngle, ), 0, 360, inferredOptions.horizontalAngle, ), verticalAngle: clampAngleEditNumber( toFiniteNumber( plannedArgs?.vertical_angle ?? plannedArgs?.verticalAngle ?? plannedArgs?.pitch ?? inferredOptions.verticalAngle, ), -30, 90, inferredOptions.verticalAngle, ), zoom: clampAngleEditNumber( toFiniteNumber(plannedArgs?.zoom ?? inferredOptions.zoom), 0, 10, inferredOptions.zoom, ), loraScale: toFiniteNumber(plannedArgs?.lora_scale ?? plannedArgs?.loraScale), }; const result = await executeDirectAgentAngleEditJob({ requestUrl: params.requestUrl, requestHeaders: params.requestHeaders, imageUrl: params.sourceImageUrl, options, }); return finalizeDirectAgentImageEditResult({ userId: params.userId, capability: params.capability, originalUrl: result.originalUrl, thumbnailUrl: result.thumbnailUrl, imageSize: result.imageSize, mediaAssetId: null, }); } if (params.capability === "local_edit") { const plannedArgs = findAgentToolCallArgs( "localEditImage", params.smartAutoExecution, params.smartAutoPlan, ); const regionResolution = resolveDirectAgentLocalEditRegions({ canvasAgentContext: params.canvasAgentContext, plannedArgs, requestTextContent: params.requestTextContent, sourceImageUrl: params.sourceImageUrl, targetNodeIds: params.targetNodeIds, }); const aspectRatio = params.imageParams?.aspectRatio || params.defaultImageQualityParams?.aspectRatio || "auto"; const imageSize = params.imageParams?.imageSize || params.defaultImageQualityParams?.imageSize || "2K"; const result = await executeDirectAgentLocalEditJob({ requestUrl: params.requestUrl, requestHeaders: params.requestHeaders, imageUrl: params.sourceImageUrl, sourceNodeId: regionResolution.sourceNodeId, regions: regionResolution.regions, model: params.finalModelId, imageSize, aspectRatio, }); const editResultMarkdown = buildAgentToolImagesMarkdown([ { originalUrl: result.originalUrl, thumbnailUrl: result.thumbnailUrl, imageSize, }, ]); await params.saveGeneratedImages( editResultMarkdown, params.userId, params.chatId, params.messageId, params.requestTextContent || "局部编辑处理", result.modelRef || params.finalModelId, { sourceAssetIds: params.targetAssetIds || undefined, sourceImageUrls: [params.sourceImageUrl], relationType: "edited_from", sourceType: "canvas-local-edit", projectId: params.projectId || null, userPrompt: params.requestTextContent, optimizedPrompt: regionResolution.regions .map((region, index) => `区域 ${index + 1}:${region.prompt}`) .join("\n") || params.requestTextContent, requestedImageCount: 1, aspectRatio, imageSize, }, ); return finalizeDirectAgentImageEditResult({ userId: params.userId, capability: params.capability, originalUrl: result.originalUrl, thumbnailUrl: result.thumbnailUrl, imageSize, mediaAssetId: null, }); } if (params.capability === "box_cutout") { const plannedArgs = findAgentToolCallArgs( "boxCutout", params.smartAutoExecution, params.smartAutoPlan, ); const cropRect = plannedArgs?.cropRect && typeof plannedArgs.cropRect === "object" ? (plannedArgs.cropRect as Record) : undefined; if (!cropRect) { throw new BoxCutoutServiceError( "缺少框选区域,暂时无法执行框选抠图。", 400, ); } const result = await executeBoxCutoutJob({ userId: params.userId, imageUrl: params.sourceImageUrl, cropRect: { x: toFiniteNumber(cropRect.x), y: toFiniteNumber(cropRect.y), width: toFiniteNumber(cropRect.width), height: toFiniteNumber(cropRect.height), }, paddingRatio: toFiniteNumber(plannedArgs?.paddingRatio), prompt: params.requestTextContent || "框选抠图处理", chatId: params.chatId, messageId: params.messageId, sourceNodeId: typeof plannedArgs?.sourceNodeId === "string" ? plannedArgs.sourceNodeId : undefined, }); return finalizeDirectAgentImageEditResult({ userId: params.userId, capability: params.capability, originalUrl: result.original_url, thumbnailUrl: result.thumbnail_url || null, imageSize: "2K", mediaAssetId: result.mediaAssetId || null, }); } throw new Error(`不支持的图片编辑工具能力:${params.capability}`); } export async function executeDirectAgentBatchImageEditTool( params: ExecuteDirectAgentImageEditToolParams, ): Promise { const sourceImageUrls = params.sourceImageUrls .map((url) => String(url || "").trim()) .filter(Boolean); const results: DirectAgentBatchImageEditExecutionResult[] = []; for (const [sourceIndex, sourceImageUrl] of sourceImageUrls.entries()) { const targetAssetIds = params.targetAssetIds?.length ? [params.targetAssetIds[sourceIndex] || params.targetAssetIds[0]].filter(Boolean) : params.targetAssetIds; const result = await executeDirectAgentImageEditTool({ ...params, targetAssetIds, sourceImageUrl, sourceImageUrls: [sourceImageUrl], }); results.push({ ...result, sourceImageUrl, sourceIndex, }); } if (results.length <= 0) { throw new Error("批量图片编辑缺少可处理的源图"); } return results; }