From 290f36b44ef069d356319a9c7ebead644aba7de5 Mon Sep 17 00:00:00 2001 From: 134 Date: Sat, 4 Jul 2026 21:48:09 +0800 Subject: [PATCH] fix: timeout stalled canvas scale providers --- lib/ai/providers/snapgen-image.ts | 95 ++++++++++++++++++++++--- lib/canvas/generate-image.ts | 112 +++++++++++++++++++++++++++++- lib/canvas/image-task-queue.ts | 42 ++++++++++- 3 files changed, 234 insertions(+), 15 deletions(-) diff --git a/lib/ai/providers/snapgen-image.ts b/lib/ai/providers/snapgen-image.ts index c07ed33cb..d6b86fcf8 100644 --- a/lib/ai/providers/snapgen-image.ts +++ b/lib/ai/providers/snapgen-image.ts @@ -20,6 +20,10 @@ interface SnapgenImageParams { imageCount?: number; outputFormat?: "png" | "jpg" | "jpeg" | "webp"; quality?: string; + requestTimeoutMs?: number; + pollTimeoutMs?: number; + historyRequestTimeoutMs?: number; + imageDownloadTimeoutMs?: number; } interface SnapgenImageModelConfig { @@ -45,6 +49,9 @@ type SnapgenApiFormat = const DEFAULT_OUTPUT_FORMAT = "png"; const POLL_INTERVAL_MS = 2000; const POLL_MAX_ATTEMPTS = 240; +const REQUEST_TIMEOUT_MS = 120_000; +const HISTORY_REQUEST_TIMEOUT_MS = 30_000; +const IMAGE_DOWNLOAD_TIMEOUT_MS = 120_000; const REFERENCE_ASPECT_DOWNLOAD_TIMEOUT_MS = 30_000; const REFERENCE_FILE_DOWNLOAD_TIMEOUT_MS = 90_000; const REFERENCE_FILE_DOWNLOAD_RETRIES = 2; @@ -105,6 +112,29 @@ function normalizeOutputFormat(outputFormat?: string): "png" | "jpeg" { return "png"; } +function normalizeTimeoutMs(value: unknown, fallback: number, min = 1000) { + const numeric = Number(value); + if (!Number.isFinite(numeric) || numeric <= 0) return fallback; + return Math.max(min, Math.floor(numeric)); +} + +async function fetchWithTimeout( + url: string, + init: RequestInit, + timeoutMs: number +) { + const controller = new AbortController(); + const timeout = setTimeout(() => controller.abort(), timeoutMs); + try { + return await fetch(url, { + ...init, + signal: controller.signal, + }); + } finally { + clearTimeout(timeout); + } +} + function normalizeOrientation(aspectRatio: string): "landscape" | "portrait" | "square" { if (aspectRatio === "1:1") return "square"; if (["9:16", "3:4", "2:3"].includes(aspectRatio)) return "portrait"; @@ -656,8 +686,11 @@ async function uploadImageBuffer(buffer: Buffer, extension: string): Promise { - const response = await fetch(imageUrl); +async function downloadAndUploadImage( + imageUrl: string, + timeoutMs = IMAGE_DOWNLOAD_TIMEOUT_MS +): Promise { + const response = await fetchWithTimeout(imageUrl, {}, timeoutMs); if (!response.ok) { throw new Error(`Failed to download SnapGen image: ${response.status}`); } @@ -734,17 +767,19 @@ async function requestSnapgenMultipart({ apiKey, path, formData, + timeoutMs, }: { baseURL: string; apiKey: string; path: string; formData: FormData; + timeoutMs: number; }) { - const response = await fetch(joinApiPath(baseURL, path), { + const response = await fetchWithTimeout(joinApiPath(baseURL, path), { method: "POST", headers: buildHeaders(apiKey), body: formData, - }); + }, timeoutMs); const payload = await parseJsonResponse(response); if (!response.ok) { throw new Error(`${extractErrorMessage(payload)} (HTTP ${response.status})`); @@ -858,17 +893,27 @@ async function waitForHistoryResult( apiKey: string, uuid: string, sourceUrls: string[], - context: { apiFormat: SnapgenApiFormat; model: string } + context: { + apiFormat: SnapgenApiFormat; + model: string; + pollTimeoutMs: number; + requestTimeoutMs: number; + } ) { - for (let attempt = 1; attempt <= POLL_MAX_ATTEMPTS; attempt++) { - const response = await fetch( + const maxAttempts = Math.max( + 1, + Math.ceil(context.pollTimeoutMs / POLL_INTERVAL_MS) + ); + for (let attempt = 1; attempt <= maxAttempts; attempt++) { + const response = await fetchWithTimeout( joinApiPath(baseURL, `/uapi/v1/history/${encodeURIComponent(uuid)}`), - { headers: buildHeaders(apiKey) } + { headers: buildHeaders(apiKey) }, + context.requestTimeoutMs ); const payload = await parseJsonResponse(response); if (!response.ok) { - if (attempt === POLL_MAX_ATTEMPTS) { + if (attempt === maxAttempts) { const errorMessage = extractErrorMessage(payload); console.error("[SnapGen Image] History request failed:", { uuid, @@ -920,7 +965,9 @@ async function waitForHistoryResult( await new Promise((resolve) => setTimeout(resolve, POLL_INTERVAL_MS)); } - throw new Error("SnapGen 任务超时，请稍后重试"); + throw new Error( + `SnapGen 任务超时（${Math.round(context.pollTimeoutMs / 1000)}秒），请稍后重试` + ); } function formatMarkdown(images: UploadedImage[], resolution: string) { @@ -954,6 +1001,23 @@ export function createSnapgenImage({ apiKey, baseURL }: SnapgenImageConfig) { const resolution = normalizeResolution(options.imageParams?.imageSize, apiFormat); const resolutionParam = toSnapgenResolutionParam(resolution); const outputFormat = normalizeOutputFormat(options.imageParams?.outputFormat); + const requestTimeoutMs = normalizeTimeoutMs( + options.imageParams?.requestTimeoutMs, + REQUEST_TIMEOUT_MS + ); + const pollTimeoutMs = normalizeTimeoutMs( + options.imageParams?.pollTimeoutMs, + POLL_INTERVAL_MS * POLL_MAX_ATTEMPTS, + POLL_INTERVAL_MS + ); + const historyRequestTimeoutMs = normalizeTimeoutMs( + options.imageParams?.historyRequestTimeoutMs, + HISTORY_REQUEST_TIMEOUT_MS + ); + const imageDownloadTimeoutMs = normalizeTimeoutMs( + options.imageParams?.imageDownloadTimeoutMs, + IMAGE_DOWNLOAD_TIMEOUT_MS + ); const maxImages = options.imageGenConfig?.maxImages || (apiFormat === "snapgen-grok-image" ? 8 : apiFormat === "snapgen-meta-ai-image" ? 4 : 1); const imageCount = clampImageCount(options.imageParams?.imageCount, maxImages); const promptForRequest = buildSnapgenReferenceOrderPrompt(prompt, imageUrls); @@ -990,6 +1054,10 @@ export function createSnapgenImage({ apiKey, baseURL }: SnapgenImageConfig) { ? "file_urls" : "files", referenceOrderPrompt: promptForRequest !== prompt, + requestTimeoutMs, + pollTimeoutMs, + historyRequestTimeoutMs, + imageDownloadTimeoutMs, }); const createPayload = await requestSnapgenMultipart({ @@ -997,6 +1065,7 @@ export function createSnapgenImage({ apiKey, baseURL }: SnapgenImageConfig) { apiKey, path: getEndpointPath(apiFormat), formData, + timeoutMs: requestTimeoutMs, }); const statusCode = extractStatusCode(createPayload); @@ -1029,6 +1098,8 @@ export function createSnapgenImage({ apiKey, baseURL }: SnapgenImageConfig) { const historyResult = await waitForHistoryResult(baseURL, apiKey, uuid, imageUrls, { apiFormat, model: runtimeModelId, + pollTimeoutMs, + requestTimeoutMs: historyRequestTimeoutMs, }); outputUrls = historyResult.imageUrls; base64Images = historyResult.base64Images; @@ -1036,7 +1107,9 @@ export function createSnapgenImage({ apiKey, baseURL }: SnapgenImageConfig) { const uploadedImages: UploadedImage[] = []; for (const outputUrl of outputUrls.slice(0, imageCount)) { - uploadedImages.push(await downloadAndUploadImage(outputUrl)); + uploadedImages.push( + await downloadAndUploadImage(outputUrl, imageDownloadTimeoutMs) + ); } if (uploadedImages.length < imageCount) { for (const image of base64Images.slice(0, imageCount - uploadedImages.length)) { diff --git a/lib/canvas/generate-image.ts b/lib/canvas/generate-image.ts index 07c47506c..0c7d4f018 100644 --- a/lib/canvas/generate-image.ts +++ b/lib/canvas/generate-image.ts @@ -213,6 +213,13 @@ const SCALE_ADJUST_RECTANGULAR_SEAM_STRONG_RUN_RATIO = 0.1; const SCALE_ADJUST_RECTANGULAR_SEAM_STRONG_TRANSITION_RATIO = 0.26; const SCALE_ADJUST_RECTANGULAR_SEAM_VERY_STRONG_RUN_RATIO = 0.55; const SCALE_ADJUST_RECTANGULAR_SEAM_VERY_STRONG_TRANSITION_RATIO = 0.5; +const CANVAS_DEFAULT_PROVIDER_BATCH_TIMEOUT_MS = 10 * 60 * 1000; +const CANVAS_SCALE_ADJUST_PROVIDER_BATCH_TIMEOUT_MS = 3 * 60 * 1000; +const CANVAS_SCALE_ADJUST_FINAL_PROVIDER_BATCH_TIMEOUT_MS = 5 * 60 * 1000; +const CANVAS_SCALE_ADJUST_SNAPGEN_REQUEST_TIMEOUT_MS = 90 * 1000; +const CANVAS_SCALE_ADJUST_SNAPGEN_POLL_TIMEOUT_MS = 150 * 1000; +const CANVAS_SCALE_ADJUST_SNAPGEN_HISTORY_TIMEOUT_MS = 20 * 1000; +const CANVAS_SCALE_ADJUST_SNAPGEN_DOWNLOAD_TIMEOUT_MS = 60 * 1000; const SUPPORTED_CANVAS_PROVIDER_TYPES = new Set([ "gemini", @@ -230,6 +237,90 @@ const SUPPORTED_CANVAS_PROVIDER_TYPES = new Set([ "volcano-vision", ]); +function isCanvasScaleAdjustDebugRequest( + debugLabel: string, + debugMeta?: Record +) { + return ( + debugLabel === "canvas-scale-adjust" || + String(debugMeta?.operation || "") === "scale-adjust" || + Boolean(debugMeta?.scaleAdjustPatchWorkflowApplied) || + Boolean(debugMeta?.scaleAdjustPatchForcedForShrink) || + Boolean(debugMeta?.scaleAdjustPatchStructureLockedDraftApplied) || + Boolean(debugMeta?.scaleAdjustPatchCompositeClipMaskUrl) + ); +} + +function getCanvasProviderBatchTimeoutMs(params: { + debugLabel: string; + debugMeta?: Record; + providerType: string; +}) { + if (isCanvasScaleAdjustDebugRequest(params.debugLabel, params.debugMeta)) { + return params.providerType === "fal-image" + ? CANVAS_SCALE_ADJUST_FINAL_PROVIDER_BATCH_TIMEOUT_MS + : CANVAS_SCALE_ADJUST_PROVIDER_BATCH_TIMEOUT_MS; + } + + return CANVAS_DEFAULT_PROVIDER_BATCH_TIMEOUT_MS; +} + +function getCanvasProviderRuntimeParams(params: { + debugLabel: string; + debugMeta?: Record; + providerType: string; +}) { + if ( + params.providerType === "snapgen-image" && + isCanvasScaleAdjustDebugRequest(params.debugLabel, params.debugMeta) + ) { + return { + requestTimeoutMs: CANVAS_SCALE_ADJUST_SNAPGEN_REQUEST_TIMEOUT_MS, + pollTimeoutMs: CANVAS_SCALE_ADJUST_SNAPGEN_POLL_TIMEOUT_MS, + historyRequestTimeoutMs: CANVAS_SCALE_ADJUST_SNAPGEN_HISTORY_TIMEOUT_MS, + imageDownloadTimeoutMs: CANVAS_SCALE_ADJUST_SNAPGEN_DOWNLOAD_TIMEOUT_MS, + }; + } + + return {}; +} + +async function withCanvasProviderBatchTimeout( + operation: Promise, + params: { + debugLabel: string; + modelId: string; + providerType: string; + timeoutMs: number; + } +): Promise { + if (!Number.isFinite(params.timeoutMs) || params.timeoutMs <= 0) { + return operation; + } + + let timeoutId: ReturnType | null = null; + try { + return await Promise.race([ + operation, + new Promise((_, reject) => { + timeoutId = setTimeout(() => { + reject( + new Error( + `${params.providerType} 图片生成超时（${Math.round( + params.timeoutMs / 1000 + )}秒），将自动切换备用模型` + ) + ); + }, params.timeoutMs); + }), + ]); + } finally { + if (timeoutId) { + clearTimeout(timeoutId); + } + } +} + function summarizeReferenceImageHosts(urls: string[]) { return Array.from( new Set( @@ -3806,8 +3897,14 @@ export async function generateCanvasImage( }, ]; + const providerRuntimeParams = getCanvasProviderRuntimeParams({ + debugLabel, + debugMeta, + providerType: provider.type, + }); const imageParams = { ...(providerImageParams || {}), + ...providerRuntimeParams, aspectRatio, safetyTolerance: request.falSafetyTolerance, imageSize, @@ -4020,6 +4117,11 @@ export async function generateCanvasImage( } const providerBatchLimit = getProviderBatchLimit(provider.type); + const providerBatchTimeoutMs = getCanvasProviderBatchTimeoutMs({ + debugLabel, + debugMeta, + providerType: provider.type, + }); const targetImageCount = Math.max(1, imageCount); const collectedImages: ParsedImage[] = []; const maxBatches = Math.max(1, Math.ceil(targetImageCount / providerBatchLimit)); @@ -4079,7 +4181,15 @@ export async function generateCanvasImage( selectedModelId: selectedModel.model.modelId, ...debugMeta, }); - const imageMarkdown = await executeGenerateBatch(batchImageCount); + const imageMarkdown = await withCanvasProviderBatchTimeout( + executeGenerateBatch(batchImageCount), + { + debugLabel, + modelId: selectedModel.model.modelId, + providerType: provider.type, + timeoutMs: providerBatchTimeoutMs, + } + ); const parsedImages = parseImagesFromMarkdown(imageMarkdown); console.info("[Canvas Generate Image] Batch completed:", { debugLabel, diff --git a/lib/canvas/image-task-queue.ts b/lib/canvas/image-task-queue.ts index d8bf5d916..21fbd0b0d 100644 --- a/lib/canvas/image-task-queue.ts +++ b/lib/canvas/image-task-queue.ts @@ -123,6 +123,36 @@ function getCanvasImageRunningTimeoutMs(request: unknown) { : DEFAULT_RUNNING_TIMEOUT_MS; } +async function withCanvasImageTaskExecutionTimeout( + operation: Promise, + timeoutMs: number, + taskId: string +): Promise { + if (!Number.isFinite(timeoutMs) || timeoutMs <= 0) { + return operation; + } + + let timeoutId: ReturnType | null = null; + try { + return await Promise.race([ + operation, + new Promise((_, reject) => { + timeoutId = setTimeout(() => { + reject( + new Error( + `图片生成任务执行超时（${Math.round(timeoutMs / 1000)}秒，${taskId}）` + ) + ); + }, timeoutMs); + }), + ]); + } finally { + if (timeoutId) { + clearTimeout(timeoutId); + } + } +} + function isStyleProTaskRequest(request: unknown): request is StyleProTaskRequest { return Boolean( request && @@ -387,15 +417,21 @@ async function runSingleTask(entry: PendingTaskEntry) { } const request = task.request as CanvasImageTaskRequest; - const result = isStyleProTaskRequest(request) - ? await executeStyleProTask({ + const runningTimeoutMs = getCanvasImageRunningTimeoutMs(request); + const execution: Promise = isStyleProTaskRequest(request) + ? executeStyleProTask({ request, userId: String(task.userId || entry.userId), }) - : await generateCanvasImage({ + : generateCanvasImage({ ...(request as CanvasGenerateImageRequest), ownerKey: task.userId, }); + const result = await withCanvasImageTaskExecutionTimeout( + execution, + runningTimeoutMs, + entry.taskId + ); await db .update(canvasImageTask) .set({