From 598cdfd7369b8d9e5b1dbcb67df82bc7980447f4 Mon Sep 17 00:00:00 2001 From: Your Name Date: Tue, 24 Feb 2026 12:00:51 +0800 Subject: [PATCH] fix: async canvas image generation queue and stabilize runtime --- app/(chat)/api/chat/route.ts | 43 ++- app/api/canvas/generate/image/route.ts | 270 ++++++------------ app/api/fal/proxy/route.ts | 25 +- app/api/realtime/config/route.ts | 22 +- components/canvas/CanvasChatPanel.tsx | 160 +++++++---- components/canvas/RealtimeCanvasDialog.tsx | 55 +++- ecosystem.config.js | 3 +- lib/canvas/generate-image.ts | 187 ++++++++++++ lib/canvas/image-task-queue.ts | 238 +++++++++++++++ .../0045_add_canvas_image_tasks.sql | 19 ++ lib/db/migrations/meta/_journal.json | 7 + lib/db/schema.ts | 21 ++ 12 files changed, 793 insertions(+), 257 deletions(-) create mode 100644 lib/canvas/generate-image.ts create mode 100644 lib/canvas/image-task-queue.ts create mode 100644 lib/db/migrations/0045_add_canvas_image_tasks.sql diff --git a/app/(chat)/api/chat/route.ts b/app/(chat)/api/chat/route.ts index e372072ea..b0beb115c 100644 --- a/app/(chat)/api/chat/route.ts +++ b/app/(chat)/api/chat/route.ts @@ -2584,7 +2584,7 @@ export async function POST(request: Request) { // 如果时间检查失败，继续执行（避免因检查逻辑错误导致系统不可用） } - const chat = await getChatById({ id }); + let chat = await getChatById({ id }); let messagesFromDb: DBMessage[] = []; if (chat) { @@ -2598,13 +2598,40 @@ export async function POST(request: Request) { message, }); - await saveChat({ - id, - userId: session.user.id, - title, - visibility: selectedVisibilityType, - }); - // New chat - no need to fetch messages, it's empty + try { + await saveChat({ + id, + userId: session.user.id, + title, + visibility: selectedVisibilityType, + }); + } catch (error) { + const racedChat = await getChatById({ id }); + if (!racedChat) { + throw error; + } + + if (racedChat.userId !== session.user.id) { + return new ChatSDKError("forbidden:chat").toResponse(); + } + + console.warn("[Chat API] saveChat conflicted, reused existing chat:", { + chatId: id, + userId: session.user.id, + }); + } + + chat = await getChatById({ id }); + if (!chat) { + throw new ChatSDKError("bad_request:database", "Failed to save chat"); + } + + if (chat.userId !== session.user.id) { + return new ChatSDKError("forbidden:chat").toResponse(); + } + + // In a race scenario another request may have already written messages. + messagesFromDb = await getMessagesByChatId({ id }); } const uiMessages = [...convertToUIMessages(messagesFromDb), message]; diff --git a/app/api/canvas/generate/image/route.ts b/app/api/canvas/generate/image/route.ts index 07c23b025..9b19e28c0 100644 --- a/app/api/canvas/generate/image/route.ts +++ b/app/api/canvas/generate/image/route.ts @@ -1,56 +1,87 @@ import { NextResponse } from "next/server"; import { auth } from "@/app/(auth)/auth"; -import { getEnabledAiModelsWithProvider, getAiProviderById } from "@/lib/db/queries"; -import { createGeminiImage } from "@/lib/ai/providers/gemini-image"; -import { createAlibabaImage } from "@/lib/ai/providers/alibaba-image"; -import { createRuxaImage } from "@/lib/ai/providers/ruxa-image"; -import { createFalImage } from "@/lib/ai/providers/fal-image"; +import { + createCanvasImageTask, + enqueueCanvasImageTask, + getCanvasImageTaskForUser, +} from "@/lib/canvas/image-task-queue"; +import { + generateCanvasImage, + type CanvasGenerateImageRequest, +} from "@/lib/canvas/generate-image"; export const maxDuration = 120; -interface GenerateImageRequest { - prompt: string; - negativePrompt?: string; - model?: string; // 模型 ID - aspectRatio?: string; // 宽高比 "1:1", "16:9" 等 - imageCount?: number; // 生成数量 - referenceImage?: string; // 参考图片（用于图生图） - referenceImages?: string[]; // 多参考图（用于多角度/细节一致性） +interface GenerateImageRequest extends CanvasGenerateImageRequest { + async?: boolean; } -interface ParsedImage { - thumbnailUrl: string; - originalUrl: string; +function toGenerateImagePayload(body: GenerateImageRequest): CanvasGenerateImageRequest { + return { + prompt: String(body.prompt || "").trim(), + negativePrompt: + typeof body.negativePrompt === "string" ? body.negativePrompt : undefined, + model: typeof body.model === "string" ? body.model : undefined, + aspectRatio: + typeof body.aspectRatio === "string" && body.aspectRatio.trim() + ? body.aspectRatio.trim() + : "1:1", + imageCount: + typeof body.imageCount === "number" && Number.isFinite(body.imageCount) + ? body.imageCount + : 1, + referenceImage: + typeof body.referenceImage === "string" ? body.referenceImage : undefined, + referenceImages: Array.isArray(body.referenceImages) + ? body.referenceImages.map((item) => String(item || "").trim()).filter(Boolean) + : [], + }; } -function parseImagesFromMarkdown(markdown: string): ParsedImage[] { - const imageRegex = /!\[([^\]]*)\]\(([^)]+)\)/g; - const matches = [...markdown.matchAll(imageRegex)]; - const parsed: ParsedImage[] = []; - - for (const match of matches) { - const altText = match[1] || ""; - const thumbnailUrl = match[2]; - let originalUrl = thumbnailUrl; - - if (altText.includes("thumb|||")) { - const content = altText.substring(altText.indexOf("thumb|||") + 8); - const parts = content.split("|||"); - originalUrl = parts[0] || thumbnailUrl; - } else if (altText.includes("thumb$$")) { - const content = altText.substring(altText.indexOf("thumb$$") + 7); - const parts = content.split("$$"); - originalUrl = parts[0] || thumbnailUrl; - } else if (altText.includes("thumb|")) { - const content = altText.substring(altText.indexOf("thumb|") + 6); - const parts = content.split("|"); - originalUrl = parts[0] || thumbnailUrl; +export async function GET(request: Request) { + try { + const session = await auth(); + if (!session?.user?.id) { + return NextResponse.json({ success: false, error: "未登录" }, { status: 401 }); } - parsed.push({ thumbnailUrl, originalUrl }); - } + const { searchParams } = new URL(request.url); + const taskId = String(searchParams.get("taskId") || "").trim(); + if (!taskId) { + return NextResponse.json( + { success: false, error: "缺少 taskId 参数" }, + { status: 400 } + ); + } - return parsed; + const task = await getCanvasImageTaskForUser({ taskId, userId: session.user.id }); + if (!task) { + return NextResponse.json({ success: false, error: "任务不存在" }, { status: 404 }); + } + + return NextResponse.json({ + success: true, + task: { + id: task.id, + status: task.status, + attempts: task.attempts, + error: task.error || null, + result: task.result || null, + createdAt: task.createdAt, + startedAt: task.startedAt, + completedAt: task.completedAt, + }, + }); + } catch (error) { + console.error("[Canvas Generate Image][TaskStatus] Error:", error); + return NextResponse.json( + { + success: false, + error: error instanceof Error ? error.message : "查询任务失败", + }, + { status: 500 } + ); + } } export async function POST(request: Request) { @@ -64,160 +95,33 @@ export async function POST(request: Request) { } const body: GenerateImageRequest = await request.json(); - const { - prompt, - negativePrompt, - model, - aspectRatio = "1:1", - imageCount = 1, - referenceImage, - referenceImages, - } = body; - - if (!prompt) { + const payload = toGenerateImagePayload(body); + if (!payload.prompt) { return NextResponse.json( { success: false, error: "请输入提示词" }, { status: 400 } ); } - // 获取可用的图片生成模型（筛选 supportsImageGen 的模型） - const allModels = await getEnabledAiModelsWithProvider(); - const imageModels = allModels.filter(m => m.model.supportsImageGen); - - if (!imageModels || imageModels.length === 0) { - return NextResponse.json( - { success: false, error: "暂无可用的图片生成模型" }, - { status: 400 } - ); - } - - // 选择模型（如果指定了 model，使用指定的；否则使用默认的） - let selectedModel = model - ? imageModels.find(m => m.model.modelId === model || m.model.id === model) - : imageModels.find(m => m.model.isDefault) || imageModels[0]; - - if (!selectedModel) { - selectedModel = imageModels[0]; - } - - const provider = await getAiProviderById({ id: selectedModel.model.providerId }); - if (!provider) { - return NextResponse.json( - { success: false, error: "模型提供商配置错误" }, - { status: 500 } - ); - } - - const normalizedReferenceImages = Array.from( - new Set( - [ - ...(Array.isArray(referenceImages) ? referenceImages : []), - ...(referenceImage ? [referenceImage] : []), - ] - .map((item) => String(item || "").trim()) - .filter((url) => /^https?:\/\//i.test(url)) - ) - ).slice(0, 12); - - // 构建消息 - const messages = [{ - role: "user" as const, - content: normalizedReferenceImages.length > 0 - ? [ - { type: "text" as const, text: prompt }, - ...normalizedReferenceImages.map((url) => ({ - type: "image_url" as const, - image_url: { url }, - })), - ] - : prompt, - }]; - - // 图片参数 - const imageParams = { - aspectRatio, - imageSize: "2K", - imageCount, - negativePrompt, - }; - - let imageMarkdown = ""; - - // 根据提供商类型调用不同的生成器 - if (provider.type === "gemini") { - const geminiProvider = createGeminiImage({ - apiKey: provider.apiKey, - baseURL: provider.baseUrl, + const asyncMode = body.async === true; + if (asyncMode) { + const task = await createCanvasImageTask({ + userId: session.user.id, + request: payload, }); - - const model = geminiProvider.chat(selectedModel.model.modelId); - const result = await model.doGenerate({ - messages, - imageParams, - }); - imageMarkdown = result.text || ""; - } else if (provider.type === "alibaba-image") { - const alibabaProvider = createAlibabaImage({ - apiKey: provider.apiKey, - baseURL: provider.baseUrl, - }); - - const model = alibabaProvider.chat(selectedModel.model.modelId); - const result = await model.doGenerate({ - messages, - imageParams, - }); - imageMarkdown = result.text || ""; - } else if (provider.type === "ruxa-image") { - const ruxaProvider = createRuxaImage({ - apiKey: provider.apiKey, - baseURL: provider.baseUrl, - }); - - const model = ruxaProvider.chat(selectedModel.model.modelId); - const result = await model.doGenerate({ - messages, - imageParams, - }); - imageMarkdown = result.text || ""; - } else if (provider.type === "fal-image") { - const falProvider = createFalImage({ - apiKey: provider.apiKey, - baseURL: provider.baseUrl, - }); - - const model = falProvider.chat(selectedModel.model.modelId); - const result = await model.doGenerate({ - messages, - imageParams, - }); - imageMarkdown = result.text || ""; - } else { - return NextResponse.json( - { success: false, error: "不支持的图片生成模型类型" }, - { status: 400 } - ); - } - - const parsedImages = parseImagesFromMarkdown(imageMarkdown); - - if (parsedImages.length > 0) { + await enqueueCanvasImageTask(task.id); return NextResponse.json({ success: true, - data: { - images: parsedImages, - model: selectedModel.model.displayName, - prompt, - } + taskId: task.id, + status: task.status, }); - } else { - return NextResponse.json( - { success: false, error: "图片生成失败" }, - { status: 500 } - ); } + const result = await generateCanvasImage(payload); + return NextResponse.json({ + success: true, + data: result, + }); } catch (error) { console.error("[Canvas Generate Image] Error:", error); return NextResponse.json( diff --git a/app/api/fal/proxy/route.ts b/app/api/fal/proxy/route.ts index d2151ad94..04fc8303e 100644 --- a/app/api/fal/proxy/route.ts +++ b/app/api/fal/proxy/route.ts @@ -6,13 +6,30 @@ const ALLOWED_FAL_HOSTS = new Set(["rest.alpha.fal.ai", "rest.fal.ai"]); type FalRealtimeConfig = { enabled?: boolean; falApiKey?: string; + apiKey?: string; +}; + +type AngleEditConfig = { + falApiKey?: string; }; async function getRealtimeCanvasConfig() { - const realtimeSetting = await getSystemSetting({ key: "realtime_canvas_config" }).catch( - () => null - ); - return (realtimeSetting?.value || {}) as FalRealtimeConfig; + const [realtimeSetting, angleEditSetting] = await Promise.all([ + getSystemSetting({ key: "realtime_canvas_config" }).catch(() => null), + getSystemSetting({ key: "angle_edit_config" }).catch(() => null), + ]); + + const realtimeConfig = (realtimeSetting?.value || {}) as FalRealtimeConfig; + const angleEditConfig = (angleEditSetting?.value || {}) as AngleEditConfig; + const fallbackKey = String( + angleEditConfig.falApiKey || process.env.FAL_KEY || process.env.FAL_API_KEY || "" + ).trim(); + + return { + ...realtimeConfig, + falApiKey: + String(realtimeConfig.falApiKey || realtimeConfig.apiKey || "").trim() || fallbackKey, + } as FalRealtimeConfig; } async function handleProxy(request: Request) { diff --git a/app/api/realtime/config/route.ts b/app/api/realtime/config/route.ts index c61127051..bef77b7c5 100644 --- a/app/api/realtime/config/route.ts +++ b/app/api/realtime/config/route.ts @@ -4,6 +4,7 @@ import { getSystemSetting } from "@/lib/db/queries"; type RealtimeCanvasConfig = { enabled?: boolean; falApiKey?: string; + apiKey?: string; throttleInterval?: number; maxBuffering?: number; maxInputResolution?: number; @@ -16,6 +17,10 @@ type RealtimeCanvasConfig = { }; }; +type AngleEditConfig = { + falApiKey?: string; +}; + const defaultConfig = { session: { idleTimeoutMs: 120000, @@ -57,8 +62,12 @@ export async function GET() { return Response.json({ code: "UNAUTHORIZED", message: "请先登录" }, { status: 401 }); } - const setting = await getSystemSetting({ key: "realtime_canvas_config" }).catch(() => null); + const [setting, angleEditSetting] = await Promise.all([ + getSystemSetting({ key: "realtime_canvas_config" }).catch(() => null), + getSystemSetting({ key: "angle_edit_config" }).catch(() => null), + ]); const config = (setting?.value || {}) as RealtimeCanvasConfig; + const angleEditConfig = (angleEditSetting?.value || {}) as AngleEditConfig; if (config.enabled === false) { return Response.json( @@ -67,7 +76,16 @@ export async function GET() { ); } - if (!String(config.falApiKey || "").trim()) { + const falApiKey = String( + config.falApiKey || + config.apiKey || + angleEditConfig.falApiKey || + process.env.FAL_KEY || + process.env.FAL_API_KEY || + "" + ).trim(); + + if (!falApiKey) { return Response.json( { code: "FAL_KEY_MISSING", diff --git a/components/canvas/CanvasChatPanel.tsx b/components/canvas/CanvasChatPanel.tsx index 6fedc5b9c..92dc0e0c8 100644 --- a/components/canvas/CanvasChatPanel.tsx +++ b/components/canvas/CanvasChatPanel.tsx @@ -336,6 +336,85 @@ const CANVAS_AGENT_SECTION_TITLES = new Set([ "执行计划", "输出设置", ]); +const CANVAS_IMAGE_TASK_POLL_INTERVAL_MS = 1600; +const CANVAS_IMAGE_TASK_TIMEOUT_MS = 4 * 60 * 1000; + +type CanvasImageGenerationRequest = { + prompt: string; + model?: string; + aspectRatio?: string; + imageCount?: number; + negativePrompt?: string; + referenceImage?: string; + referenceImages?: string[]; +}; + +function sleep(ms: number) { + return new Promise((resolve) => window.setTimeout(resolve, ms)); +} + +async function requestCanvasImageGeneration( + payload: CanvasImageGenerationRequest +): Promise<{ images: Array<{ originalUrl?: string; thumbnailUrl?: string }> }> { + const createResponse = await fetch("/api/canvas/generate/image", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + ...payload, + async: true, + }), + }); + + const createResult = await createResponse.json().catch(() => ({})); + if (!createResponse.ok || !createResult?.success) { + throw new Error(createResult?.error || "创建生图任务失败"); + } + + if (Array.isArray(createResult?.data?.images) && createResult.data.images.length > 0) { + return { images: createResult.data.images }; + } + + const taskId = String(createResult?.taskId || "").trim(); + if (!taskId) { + throw new Error("创建生图任务失败：未返回任务ID"); + } + + const deadline = Date.now() + CANVAS_IMAGE_TASK_TIMEOUT_MS; + let lastError = ""; + + while (Date.now() < deadline) { + await sleep(CANVAS_IMAGE_TASK_POLL_INTERVAL_MS); + + const statusResponse = await fetch( + `/api/canvas/generate/image?taskId=${encodeURIComponent(taskId)}`, + { cache: "no-store" } + ); + const statusResult = await statusResponse.json().catch(() => ({})); + + if (!statusResponse.ok || !statusResult?.success) { + lastError = statusResult?.error || `查询任务失败(${statusResponse.status})`; + continue; + } + + const task = statusResult?.task; + const status = String(task?.status || "").trim(); + + if (status === "succeeded") { + const result = task?.result || {}; + const images = Array.isArray(result?.images) ? result.images : []; + if (images.length === 0) { + throw new Error("任务完成但未返回图片"); + } + return { images }; + } + + if (status === "failed") { + throw new Error(String(task?.error || "图片生成失败")); + } + } + + throw new Error(lastError || "图片生成超时，请稍后重试"); +} type CanvasAnalysisSection = { before: string; @@ -3897,24 +3976,19 @@ export function CanvasChatPanel({ chatId, initialMessages = [], onClose, onOpenF ...generationReferenceImagesBase, ...extraReferenceImages, ].filter(Boolean); - const response = await fetch("/api/canvas/generate/image", { - method: "POST", - headers: { "Content-Type": "application/json" }, - body: JSON.stringify({ - prompt: promptOverride, - model: modelForGeneration, - aspectRatio: "9:16", - imageCount: 1, - negativePrompt: imageParams.negativePrompt, - referenceImage: productAttachments[0]?.url, - referenceImages: generationReferenceImages, - }), + const result = await requestCanvasImageGeneration({ + prompt: promptOverride, + model: modelForGeneration, + aspectRatio: "9:16", + imageCount: 1, + negativePrompt: imageParams.negativePrompt, + referenceImage: productAttachments[0]?.url, + referenceImages: generationReferenceImages, }); - const result = await response.json(); - if (!response.ok || !result?.success || !Array.isArray(result?.data?.images) || result.data.images.length === 0) { - throw new Error(result?.error || `第 ${index + 1} 张生成失败`); + if (!Array.isArray(result?.images) || result.images.length === 0) { + throw new Error(`第 ${index + 1} 张生成失败`); } - const generatedImage = result.data.images[0]; + const generatedImage = result.images[0]; return { originalUrl: String(generatedImage?.originalUrl || generatedImage?.thumbnailUrl || "").trim(), thumbnailUrl: String(generatedImage?.thumbnailUrl || generatedImage?.originalUrl || "").trim(), @@ -4148,24 +4222,17 @@ export function CanvasChatPanel({ chatId, initialMessages = [], onClose, onOpenF ...(bundleAttachments.length ? bundleAttachments.map((item) => item.url) : []), ].filter(Boolean); - const reviewResponse = await fetch("/api/canvas/generate/image", { - method: "POST", - headers: { "Content-Type": "application/json" }, - body: JSON.stringify({ - prompt: reviewPrompt, - model: modelForGeneration, - aspectRatio: "1:1", - imageCount: DETAIL_REVIEW_IMAGE_COUNT, - negativePrompt: imageParams.negativePrompt, - referenceImage: productAttachments[0]?.url, - referenceImages: reviewReferenceImages, - }), + const reviewResult = await requestCanvasImageGeneration({ + prompt: reviewPrompt, + model: modelForGeneration, + aspectRatio: "1:1", + imageCount: DETAIL_REVIEW_IMAGE_COUNT, + negativePrompt: imageParams.negativePrompt, + referenceImage: productAttachments[0]?.url, + referenceImages: reviewReferenceImages, }); - const reviewResult = await reviewResponse.json(); - const reviewImagesRaw = Array.isArray(reviewResult?.data?.images) - ? reviewResult.data.images - : []; - if (!reviewResponse.ok || !reviewResult?.success || reviewImagesRaw.length === 0) { + const reviewImagesRaw = Array.isArray(reviewResult?.images) ? reviewResult.images : []; + if (reviewImagesRaw.length === 0) { reviewFailures.push(`评价${reviewIndex + 1}生成失败`); continue; } @@ -4501,24 +4568,17 @@ export function CanvasChatPanel({ chatId, initialMessages = [], onClose, onOpenF ...(bundleAttachments.length ? bundleAttachments.map((item) => item.url) : []), ].filter(Boolean); - const reviewResponse = await fetch("/api/canvas/generate/image", { - method: "POST", - headers: { "Content-Type": "application/json" }, - body: JSON.stringify({ - prompt: reviewPrompt, - model: modelForGeneration, - aspectRatio: "1:1", - imageCount: DETAIL_REVIEW_IMAGE_COUNT, - negativePrompt: imageParams.negativePrompt, - referenceImage: productAttachments[0]?.url, - referenceImages: reviewReferenceImages, - }), + const reviewResult = await requestCanvasImageGeneration({ + prompt: reviewPrompt, + model: modelForGeneration, + aspectRatio: "1:1", + imageCount: DETAIL_REVIEW_IMAGE_COUNT, + negativePrompt: imageParams.negativePrompt, + referenceImage: productAttachments[0]?.url, + referenceImages: reviewReferenceImages, }); - const reviewResult = await reviewResponse.json(); - const reviewImagesRaw = Array.isArray(reviewResult?.data?.images) - ? reviewResult.data.images - : []; - if (!reviewResponse.ok || !reviewResult?.success || reviewImagesRaw.length === 0) { + const reviewImagesRaw = Array.isArray(reviewResult?.images) ? reviewResult.images : []; + if (reviewImagesRaw.length === 0) { reviewFailures.push(`第${reviewIndex + 1}条生成失败`); continue; } diff --git a/components/canvas/RealtimeCanvasDialog.tsx b/components/canvas/RealtimeCanvasDialog.tsx index 8355e4783..ccaf1e0ac 100644 --- a/components/canvas/RealtimeCanvasDialog.tsx +++ b/components/canvas/RealtimeCanvasDialog.tsx @@ -35,6 +35,7 @@ import { Download, Undo2, Redo2, + Send, } from "lucide-react"; import { toast } from "sonner"; @@ -101,9 +102,15 @@ const defaultRealtimeConfig: RealtimeConfig = { function buildImageDataUrl(image: any) { if (!image) return ""; + if (typeof image === "string" && image.trim()) { + return image; + } if (typeof image.url === "string" && image.url.trim()) { return image.url; } + if (typeof image.image_url === "string" && image.image_url.trim()) { + return image.image_url; + } if (typeof image.content === "string" && image.content.trim()) { if (image.content.startsWith("data:")) { return image.content; @@ -162,6 +169,7 @@ export function RealtimeCanvasDialog({ open, onOpenChange, initialImageUrl, + onApplyResult, }: RealtimeCanvasDialogProps) { const canvasRef = useRef(null); const previewCanvasRef = useRef(null); @@ -402,11 +410,11 @@ export function RealtimeCanvasDialog({ }, [saveHistory]); const drawInitialImage = useCallback(async () => { - if (!open || !initialImageUrl || hasLoadedInitialImageRef.current) return; + if (!open || !initialImageUrl || hasLoadedInitialImageRef.current) return false; const canvas = canvasRef.current; - if (!canvas) return; + if (!canvas) return false; const ctx = canvas.getContext("2d"); - if (!ctx) return; + if (!ctx) return false; const image = new Image(); image.crossOrigin = "anonymous"; @@ -426,6 +434,7 @@ export function RealtimeCanvasDialog({ hasLoadedInitialImageRef.current = true; dirtyRef.current = true; saveHistory(); // Save initial image state + return true; }, [initialImageUrl, open, saveHistory]); const initializeCanvas = useCallback(() => { @@ -666,8 +675,10 @@ export function RealtimeCanvasDialog({ useEffect(() => { if (!open) return; initializeCanvas(); - void drawInitialImage().then(() => { - scheduleFlush(true, false); + void drawInitialImage().then((hasInitialImage) => { + if (hasInitialImage) { + scheduleFlush(true, false); + } }); const onResize = () => initializeCanvas(); window.addEventListener("resize", onResize); @@ -760,9 +771,14 @@ export function RealtimeCanvasDialog({ return; } - const latestImage = Array.isArray(result?.images) - ? result.images[result.images.length - 1] - : null; + const imageGroups = [result?.images, result?.output?.images, result?.data?.images]; + const firstImageGroup = imageGroups.find((group) => Array.isArray(group)) as any[] | undefined; + const latestImage = + (firstImageGroup && firstImageGroup[firstImageGroup.length - 1]) || + result?.image || + result?.output?.image || + result?.data?.image || + null; const nextPreviewUrl = buildImageDataUrl(latestImage); if (!nextPreviewUrl) return; @@ -947,6 +963,19 @@ export function RealtimeCanvasDialog({ } }, []); + const handleApplyToCanvas = useCallback(() => { + if (!previewUrl) { + toast.error("请先生成预览结果"); + return; + } + + onApplyResult({ + imageUrl: previewUrl, + prompt: prompt.trim() || FALLBACK_PROMPT, + }); + onOpenChange(false); + }, [onApplyResult, onOpenChange, previewUrl, prompt]); + return ( @@ -1259,6 +1288,16 @@ export function RealtimeCanvasDialog({ > {status === 'connecting' ? : 'Generate'} +
diff --git a/ecosystem.config.js b/ecosystem.config.js index 8a04d0e63..0a71aa591 100644 --- a/ecosystem.config.js +++ b/ecosystem.config.js @@ -16,10 +16,9 @@ module.exports = { log_date_format: "YYYY-MM-DD HH:mm:ss Z", merge_logs: true, autorestart: true, - max_memory_restart: "1G", + max_memory_restart: "3G", watch: false, ignore_watch: ["node_modules", ".next", ".git"], }, ], }; - diff --git a/lib/canvas/generate-image.ts b/lib/canvas/generate-image.ts new file mode 100644 index 000000000..b21c0283f --- /dev/null +++ b/lib/canvas/generate-image.ts @@ -0,0 +1,187 @@ +import "server-only"; + +import { getEnabledAiModelsWithProvider, getAiProviderById } from "@/lib/db/queries"; +import { createGeminiImage } from "@/lib/ai/providers/gemini-image"; +import { createAlibabaImage } from "@/lib/ai/providers/alibaba-image"; +import { createRuxaImage } from "@/lib/ai/providers/ruxa-image"; +import { createFalImage } from "@/lib/ai/providers/fal-image"; + +export interface CanvasGenerateImageRequest { + prompt: string; + negativePrompt?: string; + model?: string; + aspectRatio?: string; + imageCount?: number; + referenceImage?: string; + referenceImages?: string[]; +} + +export interface ParsedImage { + thumbnailUrl: string; + originalUrl: string; +} + +export interface CanvasGenerateImageResult { + images: ParsedImage[]; + model: string; + prompt: string; +} + +function parseImagesFromMarkdown(markdown: string): ParsedImage[] { + const imageRegex = /!\[([^\]]*)\]\(([^)]+)\)/g; + const matches = [...markdown.matchAll(imageRegex)]; + const parsed: ParsedImage[] = []; + + for (const match of matches) { + const altText = match[1] || ""; + const thumbnailUrl = match[2]; + let originalUrl = thumbnailUrl; + + if (altText.includes("thumb|||")) { + const content = altText.substring(altText.indexOf("thumb|||") + 8); + const parts = content.split("|||"); + originalUrl = parts[0] || thumbnailUrl; + } else if (altText.includes("thumb$$")) { + const content = altText.substring(altText.indexOf("thumb$$") + 7); + const parts = content.split("$$"); + originalUrl = parts[0] || thumbnailUrl; + } else if (altText.includes("thumb|")) { + const content = altText.substring(altText.indexOf("thumb|") + 6); + const parts = content.split("|"); + originalUrl = parts[0] || thumbnailUrl; + } + + parsed.push({ thumbnailUrl, originalUrl }); + } + + return parsed; +} + +export async function generateCanvasImage( + request: CanvasGenerateImageRequest +): Promise { + const prompt = String(request.prompt || "").trim(); + if (!prompt) { + throw new Error("请输入提示词"); + } + + const negativePrompt = + typeof request.negativePrompt === "string" ? request.negativePrompt : undefined; + const model = typeof request.model === "string" ? request.model : undefined; + const aspectRatio = + typeof request.aspectRatio === "string" && request.aspectRatio.trim() + ? request.aspectRatio.trim() + : "1:1"; + const imageCount = + typeof request.imageCount === "number" && Number.isFinite(request.imageCount) + ? Math.max(1, Math.min(8, Math.floor(request.imageCount))) + : 1; + const referenceImage = + typeof request.referenceImage === "string" ? request.referenceImage : undefined; + const referenceImages = Array.isArray(request.referenceImages) + ? request.referenceImages + : []; + + const allModels = await getEnabledAiModelsWithProvider(); + const imageModels = allModels.filter((m) => m.model.supportsImageGen); + + if (!imageModels || imageModels.length === 0) { + throw new Error("暂无可用的图片生成模型"); + } + + let selectedModel = model + ? imageModels.find((m) => m.model.modelId === model || m.model.id === model) + : imageModels.find((m) => m.model.isDefault) || imageModels[0]; + + if (!selectedModel) { + selectedModel = imageModels[0]; + } + + const provider = await getAiProviderById({ id: selectedModel.model.providerId }); + if (!provider) { + throw new Error("模型提供商配置错误"); + } + + const normalizedReferenceImages = Array.from( + new Set( + [ + ...referenceImages, + ...(referenceImage ? [referenceImage] : []), + ] + .map((item) => String(item || "").trim()) + .filter((url) => /^https?:\/\//i.test(url)) + ) + ).slice(0, 12); + + const messages = [ + { + role: "user" as const, + content: + normalizedReferenceImages.length > 0 + ? [ + { type: "text" as const, text: prompt }, + ...normalizedReferenceImages.map((url) => ({ + type: "image_url" as const, + image_url: { url }, + })), + ] + : prompt, + }, + ]; + + const imageParams = { + aspectRatio, + imageSize: "2K", + imageCount, + negativePrompt, + }; + + let imageMarkdown = ""; + + if (provider.type === "gemini") { + const geminiProvider = createGeminiImage({ + apiKey: provider.apiKey, + baseURL: provider.baseUrl, + }); + const modelInstance = geminiProvider.chat(selectedModel.model.modelId); + const result = await modelInstance.doGenerate({ messages, imageParams }); + imageMarkdown = result.text || ""; + } else if (provider.type === "alibaba-image") { + const alibabaProvider = createAlibabaImage({ + apiKey: provider.apiKey, + baseURL: provider.baseUrl, + }); + const modelInstance = alibabaProvider.chat(selectedModel.model.modelId); + const result = await modelInstance.doGenerate({ messages, imageParams }); + imageMarkdown = result.text || ""; + } else if (provider.type === "ruxa-image") { + const ruxaProvider = createRuxaImage({ + apiKey: provider.apiKey, + baseURL: provider.baseUrl, + }); + const modelInstance = ruxaProvider.chat(selectedModel.model.modelId); + const result = await modelInstance.doGenerate({ messages, imageParams }); + imageMarkdown = result.text || ""; + } else if (provider.type === "fal-image") { + const falProvider = createFalImage({ + apiKey: provider.apiKey, + baseURL: provider.baseUrl, + }); + const modelInstance = falProvider.chat(selectedModel.model.modelId); + const result = await modelInstance.doGenerate({ messages, imageParams }); + imageMarkdown = result.text || ""; + } else { + throw new Error("不支持的图片生成模型类型"); + } + + const parsedImages = parseImagesFromMarkdown(imageMarkdown); + if (parsedImages.length === 0) { + throw new Error("图片生成失败"); + } + + return { + images: parsedImages, + model: selectedModel.model.displayName, + prompt, + }; +} diff --git a/lib/canvas/image-task-queue.ts b/lib/canvas/image-task-queue.ts new file mode 100644 index 000000000..51c7988d6 --- /dev/null +++ b/lib/canvas/image-task-queue.ts @@ -0,0 +1,238 @@ +import "server-only"; + +import { and, asc, eq, lt, notInArray, sql } from "drizzle-orm"; +import { db } from "@/lib/db"; +import { canvasImageTask } from "@/lib/db/schema"; +import { + generateCanvasImage, + type CanvasGenerateImageRequest, + type CanvasGenerateImageResult, +} from "./generate-image"; + +type CanvasImageTaskStatus = "pending" | "running" | "succeeded" | "failed"; + +interface QueueState { + pending: string[]; + pendingSet: Set; + running: Set; + draining: boolean; +} + +const DEFAULT_CONCURRENCY = 1; +const STALE_RUNNING_MS = 5 * 60 * 1000; +const queueStateKey = "__canvas_image_task_queue_state__"; + +function getQueueConcurrency(): number { + const raw = Number.parseInt(process.env.CANVAS_IMAGE_TASK_CONCURRENCY || "", 10); + if (Number.isFinite(raw) && raw > 0) { + return Math.min(raw, 4); + } + return DEFAULT_CONCURRENCY; +} + +function getQueueState(): QueueState { + const globalObj = globalThis as typeof globalThis & { + [queueStateKey]?: QueueState; + }; + if (!globalObj[queueStateKey]) { + globalObj[queueStateKey] = { + pending: [], + pendingSet: new Set(), + running: new Set(), + draining: false, + }; + } + return globalObj[queueStateKey] as QueueState; +} + +function enqueueLocalTask(taskId: string) { + const state = getQueueState(); + if (!taskId || state.running.has(taskId) || state.pendingSet.has(taskId)) { + return; + } + state.pending.push(taskId); + state.pendingSet.add(taskId); +} + +async function claimNextPendingTask(): Promise { + const state = getQueueState(); + while (state.pending.length > 0) { + const taskId = state.pending.shift(); + if (!taskId) continue; + state.pendingSet.delete(taskId); + if (!state.running.has(taskId)) { + return taskId; + } + } + + const runningIds = Array.from(state.running); + const whereCondition = + runningIds.length > 0 + ? and(eq(canvasImageTask.status, "pending"), notInArray(canvasImageTask.id, runningIds)) + : eq(canvasImageTask.status, "pending"); + + const rows = await db + .select({ id: canvasImageTask.id }) + .from(canvasImageTask) + .where(whereCondition) + .orderBy(asc(canvasImageTask.createdAt)) + .limit(1); + + return rows[0]?.id ?? null; +} + +async function runSingleTask(taskId: string) { + const state = getQueueState(); + state.running.add(taskId); + + try { + const [task] = await db + .update(canvasImageTask) + .set({ + status: "running", + startedAt: new Date(), + updatedAt: new Date(), + completedAt: null, + error: null, + attempts: sql`${canvasImageTask.attempts} + 1`, + }) + .where(and(eq(canvasImageTask.id, taskId), eq(canvasImageTask.status, "pending"))) + .returning({ + id: canvasImageTask.id, + request: canvasImageTask.request, + }); + + if (!task) { + return; + } + + const result = await generateCanvasImage(task.request as CanvasGenerateImageRequest); + await db + .update(canvasImageTask) + .set({ + status: "succeeded", + result: result as CanvasGenerateImageResult, + error: null, + completedAt: new Date(), + updatedAt: new Date(), + }) + .where(eq(canvasImageTask.id, taskId)); + } catch (error) { + await db + .update(canvasImageTask) + .set({ + status: "failed", + error: error instanceof Error ? error.message : String(error), + completedAt: new Date(), + updatedAt: new Date(), + }) + .where(eq(canvasImageTask.id, taskId)); + } finally { + state.running.delete(taskId); + void drainCanvasImageTaskQueue(); + } +} + +export async function drainCanvasImageTaskQueue() { + const state = getQueueState(); + if (state.draining) return; + + state.draining = true; + try { + const maxConcurrent = getQueueConcurrency(); + while (state.running.size < maxConcurrent) { + const taskId = await claimNextPendingTask(); + if (!taskId) break; + if (state.running.has(taskId)) continue; + void runSingleTask(taskId); + } + } finally { + state.draining = false; + } +} + +export async function enqueueCanvasImageTask(taskId: string) { + enqueueLocalTask(taskId); + await drainCanvasImageTaskQueue(); +} + +export async function createCanvasImageTask(params: { + userId: string; + request: CanvasGenerateImageRequest; +}) { + const [task] = await db + .insert(canvasImageTask) + .values({ + userId: params.userId, + status: "pending", + request: params.request as unknown as Record, + attempts: 0, + createdAt: new Date(), + updatedAt: new Date(), + }) + .returning({ + id: canvasImageTask.id, + status: canvasImageTask.status, + createdAt: canvasImageTask.createdAt, + }); + + return task; +} + +export async function getCanvasImageTaskForUser(params: { + taskId: string; + userId: string; +}) { + const rows = await db + .select({ + id: canvasImageTask.id, + status: canvasImageTask.status, + request: canvasImageTask.request, + result: canvasImageTask.result, + error: canvasImageTask.error, + attempts: canvasImageTask.attempts, + startedAt: canvasImageTask.startedAt, + completedAt: canvasImageTask.completedAt, + createdAt: canvasImageTask.createdAt, + updatedAt: canvasImageTask.updatedAt, + }) + .from(canvasImageTask) + .where(and(eq(canvasImageTask.id, params.taskId), eq(canvasImageTask.userId, params.userId))) + .limit(1); + + const task = rows[0]; + if (!task) return null; + + if (task.status === "running" && task.startedAt) { + const staleBefore = new Date(Date.now() - STALE_RUNNING_MS); + if (task.startedAt < staleBefore) { + await db + .update(canvasImageTask) + .set({ + status: "pending", + startedAt: null, + completedAt: null, + updatedAt: new Date(), + }) + .where( + and( + eq(canvasImageTask.id, task.id), + eq(canvasImageTask.status, "running"), + lt(canvasImageTask.startedAt, staleBefore) + ) + ); + enqueueLocalTask(task.id); + await drainCanvasImageTaskQueue(); + task.status = "pending" as CanvasImageTaskStatus; + task.startedAt = null; + task.completedAt = null; + } + } + + if (task.status === "pending") { + enqueueLocalTask(task.id); + await drainCanvasImageTaskQueue(); + } + + return task; +} diff --git a/lib/db/migrations/0045_add_canvas_image_tasks.sql b/lib/db/migrations/0045_add_canvas_image_tasks.sql new file mode 100644 index 000000000..e5fc5b3f5 --- /dev/null +++ b/lib/db/migrations/0045_add_canvas_image_tasks.sql @@ -0,0 +1,19 @@ +CREATE TABLE IF NOT EXISTS "CanvasImageTask" ( + "id" uuid PRIMARY KEY DEFAULT gen_random_uuid() NOT NULL, + "userId" uuid NOT NULL REFERENCES "User"("id") ON DELETE cascade, + "status" varchar(16) NOT NULL DEFAULT 'pending', + "request" jsonb NOT NULL, + "result" jsonb, + "error" text, + "attempts" integer NOT NULL DEFAULT 0, + "startedAt" timestamp, + "completedAt" timestamp, + "createdAt" timestamp NOT NULL DEFAULT now(), + "updatedAt" timestamp NOT NULL DEFAULT now() +); + +CREATE INDEX IF NOT EXISTS "CanvasImageTask_user_status_created_idx" + ON "CanvasImageTask"("userId", "status", "createdAt"); + +CREATE INDEX IF NOT EXISTS "CanvasImageTask_status_created_idx" + ON "CanvasImageTask"("status", "createdAt"); diff --git a/lib/db/migrations/meta/_journal.json b/lib/db/migrations/meta/_journal.json index bfe3cec41..a7020adb9 100644 --- a/lib/db/migrations/meta/_journal.json +++ b/lib/db/migrations/meta/_journal.json @@ -218,6 +218,13 @@ "when": 1772366460000, "tag": "0044_add_resend_config", "breakpoints": true + }, + { + "idx": 31, + "version": "7", + "when": 1772500000000, + "tag": "0045_add_canvas_image_tasks", + "breakpoints": true } ] } diff --git a/lib/db/schema.ts b/lib/db/schema.ts index 7c77f6096..998e4c8e3 100644 --- a/lib/db/schema.ts +++ b/lib/db/schema.ts @@ -521,6 +521,27 @@ export const canvasDetailArchive = pgTable("CanvasDetailArchive", { export type CanvasDetailArchive = InferSelectModel; +// 画布图片生成任务表（异步队列） +export const canvasImageTask = pgTable("CanvasImageTask", { + id: uuid("id").primaryKey().notNull().defaultRandom(), + userId: uuid("userId") + .notNull() + .references(() => user.id, { onDelete: "cascade" }), + status: varchar("status", { length: 16 }) + .notNull() + .default("pending"), // pending | running | succeeded | failed + request: jsonb("request").notNull(), + result: jsonb("result"), + error: text("error"), + attempts: integer("attempts").notNull().default(0), + startedAt: timestamp("startedAt"), + completedAt: timestamp("completedAt"), + createdAt: timestamp("createdAt").notNull().defaultNow(), + updatedAt: timestamp("updatedAt").notNull().defaultNow(), +}); + +export type CanvasImageTask = InferSelectModel; + // 画布节点表（可选，用于更细粒度的查询） export const canvasNode = pgTable("CanvasNode", { id: uuid("id").primaryKey().notNull().defaultRandom(),