diff --git a/lib/ai/canvas-agent-runtime/tools.ts b/lib/ai/canvas-agent-runtime/tools.ts index e3487c586..d20462791 100644 --- a/lib/ai/canvas-agent-runtime/tools.ts +++ b/lib/ai/canvas-agent-runtime/tools.ts @@ -173,6 +173,8 @@ export async function buildCanvasRuntimeTools(params: { }; }, ); + const inspectedImages = new Map(); + let inspectedImageBytes = 0; register( "analyzeCanvasImages", "检查图片", @@ -181,7 +183,23 @@ export async function buildCanvasRuntimeTools(params: { nodeIds: z.array(z.string().min(1).max(180)).min(1).max(4), question: z.string().min(1).max(2000), }), - async (input) => analyzeRuntimeImages({ ...params, ...input }), + async (input) => analyzeRuntimeImages({ + ...params, + ...input, + reviewPorts: { + readImage: async (url) => { + const cached = inspectedImages.get(url); + if (cached) return cached; + const { fetchImageBufferFromUrl } = await import("@/lib/ai/rmbg/service"); + const original = await fetchImageBufferFromUrl(url); + if (!inspectedImages.has(url) && inspectedImageBytes + original.length <= 32 * 1024 * 1024) { + inspectedImages.set(url, original); + inspectedImageBytes += original.length; + } + return original; + }, + }, + }), ); let submittedCount = 0; @@ -266,6 +284,7 @@ export async function buildCanvasRuntimeTools(params: { ? Math.max(0, user?.pointsBalance ?? 0) / input.imageCount : undefined, ownerKey: params.userId, referenceImages: sources.map((node) => node.data.src || node.data.url!), + referenceImageBuffers: inspectedImages, debugLabel: "canvas-agent-runtime", debugMeta: { runId: params.runId, callId }, }); diff --git a/lib/ai/providers/alibaba-image.ts b/lib/ai/providers/alibaba-image.ts index 1b25a4eea..71ca6017e 100644 --- a/lib/ai/providers/alibaba-image.ts +++ b/lib/ai/providers/alibaba-image.ts @@ -12,6 +12,7 @@ import sharp from "sharp"; import { constrainImageDimensions } from "@/lib/ai/image-size-constraints"; import { isAlibabaImageTextToImageModelId } from "@/lib/ai/model-classification"; import { buildReferenceImageOrderPrompt } from "@/lib/ai/providers/reference-image-order"; +import { ImageReferenceReadTimeoutError } from "@/lib/ai/runtime/image-generation-error"; import { createDeferredUploadedImage } from "@/lib/storage/image-archive"; import { uploadFile } from "@/lib/storage/service"; import { normalizeLegacyAliyunStorageUrl } from "@/lib/storage/retired-storage"; @@ -165,7 +166,7 @@ async function fetchImageBufferWithRetry( } } - throw lastError instanceof Error ? lastError : new Error(String(lastError || "下载图片失败")); + throw lastError ?? new Error("下载图片失败"); } function calculateAlibabaSizeByAspect(aspectValue: number, baseSize: number): string { @@ -289,12 +290,20 @@ async function prepareAlibabaInputImageUrl( imageUrl: string, modelId: string, inputRole?: AlibabaImageInputRole, + original?: Buffer, ): Promise { const normalizedUrl = String(imageUrl || "").trim(); if (!/^https?:\/\//i.test(normalizedUrl)) return normalizedUrl; const maxInputBytes = getAlibabaMaxInputBytes(modelId); - const { buffer, contentType } = await fetchImageBufferWithRetry(normalizedUrl); + const { buffer, contentType } = original + ? { buffer: original, contentType: null } + : await fetchImageBufferWithRetry(normalizedUrl).catch((error) => { + if (["AbortError", "TimeoutError"].includes(error?.name)) { + throw new ImageReferenceReadTimeoutError(); + } + throw error; + }); let metadata: sharp.Metadata; try { metadata = await sharp(buffer, { failOn: "none" }).metadata(); @@ -561,7 +570,7 @@ async function sendAlibabaImageAsyncRequest(params: { url: params.requestURL, }); const taskId = String(submitted.output?.task_id || "").trim(); - if (!taskId) throw new Error("阿里图片异步任务未返回 task_id"); + if (!taskId) throw new Error("阿里图片异步任务未返回 task_id，结果尚未确认"); const taskURL = resolveAlibabaImageTaskEndpoint(params.requestURL, taskId); const startedAt = Date.now(); @@ -585,7 +594,7 @@ async function sendAlibabaImageAsyncRequest(params: { console.log("[Alibaba Image] Async task completed:", taskId); return task; } - if (["FAILED", "CANCELED", "UNKNOWN"].includes(taskStatus)) { + if (["FAILED", "CANCELED"].includes(taskStatus)) { const reason = task.output?.message || task.message || `阿里图片任务${taskStatus || "失败"}`; const code = task.output?.code || task.code; throw new AlibabaImageTerminalTaskError( @@ -601,7 +610,7 @@ async function sendAlibabaImageAsyncRequest(params: { error.status < 500 && ![408, 425, 429].includes(error.status) ) { - throw error; + throw new Error(`阿里图片任务已提交，结果尚未确认（task_id: ${taskId}）`, { cause: error }); } lastPollError = error; console.warn("[Alibaba Image] Async task poll failed; retrying:", { @@ -723,6 +732,7 @@ export function createAlibabaImage({ apiKey, baseURL }: AlibabaImageConfig) { doGenerate: async (options: { messages: Array<{ role: string; content: string | Array<{ type: string; text?: string; image?: string }> }>; imageParams?: AlibabaImageParams & { aspectRatio?: string; imageSize?: string; imageCount?: number }; + referenceImageBuffers?: ReadonlyMap; modelDisplayName?: string; // 🆕 模型显示名称，用于错误提示 overrideModelId?: string; // 🆕 备用模型 ID }) => { @@ -798,6 +808,7 @@ export function createAlibabaImage({ apiKey, baseURL }: AlibabaImageConfig) { imageUrl, activeModelId, options.imageParams?.inputImageRoleByUrl?.[imageUrl], + options.referenceImageBuffers?.get(imageUrl), ), ), ); @@ -938,7 +949,9 @@ export function createAlibabaImage({ apiKey, baseURL }: AlibabaImageConfig) { ); // 处理响应 - const images = await processResponse(response); + const images = await processResponse(response).catch((error) => { + throw new Error("阿里图片任务已提交，结果尚未确认。", { cause: error }); + }); const markdownText = formatImageResponse(images, params.size, imageSize); console.log(`[Alibaba Image] Generated ${images.length} images successfully`); diff --git a/lib/ai/runtime/image-generation-error.ts b/lib/ai/runtime/image-generation-error.ts index b110ced04..547757359 100644 --- a/lib/ai/runtime/image-generation-error.ts +++ b/lib/ai/runtime/image-generation-error.ts @@ -8,6 +8,12 @@ export class AgentImageGenerationError extends Error { constructor(message: string, readonly confirmedNoResult: boolean) { super(message); } } +export class ImageReferenceReadTimeoutError extends AgentImageGenerationError { + constructor() { + super("提交生图前读取参考图超时，任务尚未提交，可以稍后重试。", true); + } +} + /** Preserve outcome evidence when replacing upstream diagnostics with safe UI text. */ export function agentImageGenerationError(error: unknown, mode: "auto" | "pinned") { const rejected = error instanceof ImageSubmissionRejectedError; diff --git a/lib/canvas/generate-image.ts b/lib/canvas/generate-image.ts index 50291970c..bd0565d36 100644 --- a/lib/canvas/generate-image.ts +++ b/lib/canvas/generate-image.ts @@ -5,7 +5,7 @@ import sharp from "sharp"; import { agentImageCircuitStore } from "@/lib/ai/runtime/agent-model-circuit-store"; import { getModelGenerationCreditCost, type DiscountableImageGenConfigLike } from "@/lib/ai/image-model-pricing"; import { agentImageFailurePolicy } from "@/lib/ai/runtime/agent-image-failover"; -import { agentImageGenerationError } from "@/lib/ai/runtime/image-generation-error"; +import { agentImageGenerationError, ImageReferenceReadTimeoutError } from "@/lib/ai/runtime/image-generation-error"; import { getAiModelById, getEnabledAiModelsWithProvider } from "@/lib/db/queries"; import { fetchImageBufferFromUrl } from "@/lib/ai/rmbg/service"; import { createGeminiImage } from "@/lib/ai/providers/gemini-image"; @@ -181,6 +181,7 @@ export interface CanvasGenerateImageRequest { providerImageParams?: Record; referenceImage?: string; referenceImages?: string[]; + referenceImageBuffers?: ReadonlyMap; billingFeature?: "multi-view" | "repaint"; requiredReferenceImageCount?: number; ownerKey?: string; @@ -5121,8 +5122,11 @@ export async function generateCanvasImage( }); } - const { kept: reachableReferenceImages, dropped: unreachableReferenceImages } = - await filterReachableRemoteUrls(normalizedReferenceImages, { timeoutMs: 3000 }); + const referenceImageBuffers = provider.type === "alibaba-image" ? request.referenceImageBuffers : undefined; + const referencesToCheck = normalizedReferenceImages.filter(url => !referenceImageBuffers?.has(url)); + const { kept, dropped: unreachableReferenceImages } = + await filterReachableRemoteUrls(referencesToCheck, { timeoutMs: 3000 }); + const reachableReferenceImages = normalizedReferenceImages.filter(url => referenceImageBuffers?.has(url) || kept.includes(url)); const effectiveReferenceImages = selectCanvasReferenceImages({ requested: normalizedReferenceImages, reachable: reachableReferenceImages, requiredCount: request.requiredReferenceImageCount, @@ -5233,6 +5237,7 @@ export async function generateCanvasImage( executeGenerateBatch = async (batchImageCount) => { const result = await modelInstance.doGenerate({ messages, + referenceImageBuffers, imageParams: { ...imageParams, imageCount: batchImageCount, @@ -5662,6 +5667,11 @@ export async function generateCanvasImage( await slot.release(); } + if (caughtError instanceof ImageReferenceReadTimeoutError) { + if (!collectedImages.length) throw caughtError; + return { images: collectedImages, model: selectedModel.model.displayName, modelRef: selectedModel.model.id, prompt: resultPrompt }; + } + // An accepted or uncertain request must not start another paid model/variant. if (caughtError && ( caughtError instanceof ProviderTaskUncertainError || diff --git a/lib/request-error.ts b/lib/request-error.ts index 8adadffbe..d9b44422c 100644 --- a/lib/request-error.ts +++ b/lib/request-error.ts @@ -293,6 +293,9 @@ export function getRequestFailureDetails( if (/AOS结果下载超时/.test(text)) { return detail("RMBG_RESULT_DOWNLOAD_TIMEOUT", "抠图已完成，但结果图片下载超时。", "请联系管理员按请求编号取回原任务结果，避免重复抠图。", "unknown"); } + if (/提交生图前读取参考图超时/.test(text)) { + return detail("IMAGE_REFERENCE_READ_TIMEOUT", "提交生图前读取参考图超时，任务尚未提交。", "可以稍后重试。"); + } const errorName = isRecord(error) ? String(error.name || "") : ""; if (["AbortError", "TimeoutError"].includes(errorName) || /operation was aborted/i.test(text)) { return detail("RESULT_UNCONFIRMED", errorName === "TimeoutError" ? "请求等待超时，任务结果尚未确认。" : "本次请求连接已中止，任务结果尚未确认。", "请先核对原任务或历史记录；持续没有结果时联系管理员，避免重复生成。", "unknown"); diff --git a/tests/routes/alibaba-task-errors.test.ts b/tests/routes/alibaba-task-errors.test.ts index ed4b909b1..fb2534493 100644 --- a/tests/routes/alibaba-task-errors.test.ts +++ b/tests/routes/alibaba-task-errors.test.ts @@ -32,6 +32,7 @@ test("Qwen 3 sends reference bytes directly with the right MIME and keeps auto r const jpeg = await sharp(png).jpeg().toBuffer(); const webp = await sharp(png).webp().toBuffer(); const originals = [png, jpeg, webp]; + const downloads: number[] = []; let submitted: any; let posts = 0; const loader = serverLoader({ @@ -41,14 +42,16 @@ test("Qwen 3 sends reference bytes directly with the right MIME and keeps auto r if (init.method === "POST") { posts++; submitted = JSON.parse(String(init.body)); return Response.json({ output: { task_id: "inline-task", task_status: "PENDING" } }); } if (url.includes("/tasks/")) return Response.json({ output: { task_status: "FAILED", message: "transport-inspection-complete" } }); const index = Number(new URL(url).pathname.slice(1)); + downloads.push(index); return new Response(originals[index], { headers: { "content-type": ["image/png", "image/jpeg", "image/webp"][index] } }); }, }); const provider = loader("lib/ai/providers/alibaba-image.ts").createAlibabaImage({ apiKey: "fixture-key", baseURL: "https://dashscope.example" }); await expect(provider.chat("qwen-image-3.0").doGenerate({ messages: [{ role: "user", content: [ { type: "text", text: "fixture" }, ...originals.map((_bytes, index) => ({ type: "image", image: `https://source.example/${index}` })), - ] }], imageParams: { aspectRatio: "auto", imageSize: "2K" } })).rejects.toThrow("transport-inspection-complete"); + ] }], imageParams: { aspectRatio: "auto", imageSize: "2K" }, referenceImageBuffers: new Map([["https://source.example/0", png]]) })).rejects.toThrow("transport-inspection-complete"); expect(posts).toBe(1); + expect(downloads).toEqual([1, 2]); expect(submitted.parameters.size).toBe("1024*2048"); const images = submitted.input.messages[0].content.filter((part: any) => part.image).map((part: any) => part.image); expect(images).toHaveLength(3); @@ -57,3 +60,43 @@ test("Qwen 3 sends reference bytes directly with the right MIME and keeps auto r expect(Buffer.from(value.split(",")[1], "base64").equals(originals[index])).toBe(true); }); }); + +test("a timed out reference read is safe to retry because no generation was submitted", async () => { + const methods: string[] = []; + const loader = serverLoader({ + "@/lib/storage/image-archive": {}, "@/lib/storage/service": {}, + setTimeout: (callback: () => void, ms: number) => setTimeout(callback, ms === 500 ? 0 : ms), + fetch: async (_url: string, init: RequestInit = {}) => { + methods.push(init.method || "GET"); + throw new DOMException("This operation was aborted", "AbortError"); + }, + }); + const provider = loader("lib/ai/providers/alibaba-image.ts").createAlibabaImage({ apiKey: "fixture-key", baseURL: "https://dashscope.example" }); + const error = await provider.chat("qwen-image-3.0").doGenerate({ messages: [{ role: "user", content: [ + { type: "text", text: "fixture" }, { type: "image", image: "https://source.example/slow.png" }, + ] }] }).catch((error: unknown) => error); + expect(methods).toEqual(["GET", "GET"]); + expect(error).toMatchObject({ confirmedNoResult: true }); + expect(loader("lib/request-error.ts").getRequestFailureDetails(error)).toMatchObject({ + code: "IMAGE_REFERENCE_READ_TIMEOUT", outcome: "failed", recommendation: "可以稍后重试。", + }); +}); + +test("a failed status lookup after Alibaba accepts the request keeps its outcome unknown", async () => { + const methods: string[] = []; + const loader = serverLoader({ + "@/lib/storage/image-archive": {}, "@/lib/storage/service": {}, + setTimeout: (callback: () => void, ms: number) => setTimeout(callback, ms === 5000 ? 0 : ms), + fetch: async (_url: string, init: RequestInit = {}) => { + methods.push(init.method || "GET"); + return init.method === "POST" + ? Response.json({ output: { task_id: "accepted-task", task_status: "PENDING" } }) + : Response.json({ code: "NotFound", message: "Task lookup failed" }, { status: 404 }); + }, + }); + const provider = loader("lib/ai/providers/alibaba-image.ts").createAlibabaImage({ apiKey: "fixture-key", baseURL: "https://dashscope.example" }); + const error = await provider.chat("qwen-image-3.0").doGenerate({ messages: [{ role: "user", content: "fixture" }] }).catch((error: unknown) => error); + expect(methods).toEqual(["POST", "GET"]); + expect(String(error)).toContain("accepted-task"); + expect(loader("lib/request-error.ts").getRequestFailureDetails(error)).toMatchObject({ code: "RESULT_UNCONFIRMED", outcome: "unknown" }); +}); diff --git a/tests/routes/canvas-agent-runtime-image-preferences.test.ts b/tests/routes/canvas-agent-runtime-image-preferences.test.ts index dbdc84908..48fbbf743 100644 --- a/tests/routes/canvas-agent-runtime-image-preferences.test.ts +++ b/tests/routes/canvas-agent-runtime-image-preferences.test.ts @@ -12,6 +12,8 @@ function harness(results?: any[] | ((count: number) => any[])) { const saved: any[] = []; const generated: any[] = []; const operations: any[] = []; + const referenceUrl = "https://test.example/reference.png"; + const original = Buffer.from("original reference bytes"); const mock = serverLoader({ "@/lib/canvas/capabilities/ai-sdk-tools": { createCanvasCapabilityTools: async () => ({ editTargetImage: {} }) }, "@/lib/canvas/capabilities/registry": { CANVAS_CAPABILITY_REGISTRY: [{ id: "image.prompt_regenerate", title: "重绘", risk: "external", agent: { toolName: "editTargetImage" } }] }, @@ -38,8 +40,9 @@ function harness(results?: any[] | ((count: number) => any[])) { getStyleProImageModelEntries: async () => ["system-default", "user-image-model"].map(id => ({ model: { id, modelId: id, isDefault: id === "system-default", imageGenConfig: {} } })), sortStyleProImageModelEntries: (entries: any) => entries, }, - "./canvas": { readOwnedCanvas: async () => ({ nodes: [{ id: "reference", type: "image" }] }), readCanvasImages: async () => [{ id: "reference", data: { src: "https://test.example/reference.png" } }] }, - "./review": {}, + "./canvas": { readOwnedCanvas: async () => ({ nodes: [{ id: "reference", type: "image" }] }), readCanvasImages: async () => [{ id: "reference", data: { src: referenceUrl } }] }, + "./review": { analyzeRuntimeImages: async ({ reviewPorts }: { reviewPorts: { readImage(url: string): Promise } }) => ({ bytes: (await reviewPorts.readImage(referenceUrl)).length }) }, + "@/lib/ai/rmbg/service": { fetchImageBufferFromUrl: async () => original }, "./image-access-tools": { registerImageCanvasAccessTools() {} }, }); const build = async (imagePreferences: any = preferences, priorCalls?: AgentCallRecord[]) => mock("lib/ai/canvas-agent-runtime/tools.ts").buildCanvasRuntimeTools({ @@ -47,8 +50,20 @@ function harness(results?: any[] | ((count: number) => any[])) { request: new Request("https://test.example/api/canvas/agent-chat"), model: {}, supportsVision: true, preferredModelId: "system-default", imagePreferences, priorCalls, emit() {}, }); - return { build, generated, operations, saved }; + return { build, generated, operations, saved, referenceUrl, original }; } + +test("the generation tool can reuse original bytes read by inspection within the same task", async () => { + const h = harness(); + const selection = { parameters: { imageCount: 1 } }; + const tools = await h.build(selection); + await tools.executable.analyzeCanvasImages.execute({ nodeIds: ["reference"], question: "识别素材" }, "inspect"); + await tools.executable.generateCanvasImages.execute({ prompt: "修改参考图", referenceNodeIds: ["reference"] }, "generate"); + expect(h.generated[0].referenceImageBuffers.get(h.referenceUrl)).toBe(h.original); + const nextTask = await h.build(selection); + await nextTask.executable.generateCanvasImages.execute({ prompt: "修改参考图", referenceNodeIds: ["reference"] }, "next"); + expect(h.generated[1].referenceImageBuffers.has(h.referenceUrl)).toBe(false); +}); test("explicit controls override conflicting planner settings without losing false or zero values", () => { const resolved = applyAgentImagePreferences({ modelId: "planner-choice", imageCount: 1, imageSize: "1K", aspectRatio: "1:1", prompt: "产品图" }, preferences); expect(resolved).toMatchObject({ ...preferences.parameters, modelId: preferences.modelId, prompt: "产品图" }); diff --git a/tests/routes/zcode-agent-task.test.ts b/tests/routes/zcode-agent-task.test.ts index 8bab1daeb..73dd9718e 100644 --- a/tests/routes/zcode-agent-task.test.ts +++ b/tests/routes/zcode-agent-task.test.ts @@ -25,7 +25,7 @@ function fixture() { consumeInputs: async () => ({ cancelled: false }), isCancelled: async () => false, verify: async () => ({ passed: true, issues: [] }), }; - return { ports, run, snapshot }; + return { ports, run, snapshot, load }; } const result = { type: "result", sessionId: "test-session", response: "已完成" }; @@ -81,6 +81,26 @@ test("an unconfirmed paid tool stops the planner and immediately preserves the r expect(submissions).toBe(1); }); +test("a pre-submission reference timeout offers a safe retry without an automatic resubmission", async () => { + const { ports, run, snapshot, load } = fixture(); + const { ImageReferenceReadTimeoutError } = load("lib/ai/runtime/image-generation-error.ts"); + let attempts = 0; + ports.tools.generateCanvasImages.execute = async () => { + attempts++; + throw new ImageReferenceReadTimeoutError(); + }; + ports.native = async options => { + await options.execute("planCanvasTask", {}, "plan", options.signal!); + await options.execute("generateCanvasImages", { referenceNodeIds: ["original"] }, "timeout", options.signal!); + return result; + }; + expect(await run(ports)).toBe("needs_review"); + expect(attempts).toBe(1); + expect(snapshot.calls.find(call => call.id === "timeout")).toMatchObject({ confirmedNoResult: true, uncertain: false }); + expect(snapshot.message).toContain("任务尚未提交，可以稍后重试"); + expect(snapshot.message).toContain(ports.runId); +}); + test("repairing a failed result permits original images as references without admitting another passed target", async () => { const { ports, run, snapshot } = fixture(); let submissions = 0;