fix: run fal gpt image edit asynchronously
This commit is contained in:
Your Name
parent
ce03c85811
commit
44dee6314a
@ -25,6 +25,10 @@ import { fetchImageBufferFromUrl } from "@/lib/ai/rmbg/service";
|
||||
import type { ChatModel } from "@/lib/ai/models";
|
||||
import { type RequestHints, systemPrompt } from "@/lib/ai/prompts";
|
||||
import { myProvider } from "@/lib/ai/providers";
|
||||
import {
|
||||
isFalGptImage2EditModelId,
|
||||
submitFalImageTask,
|
||||
} from "@/lib/ai/providers/fal-image";
|
||||
import { getLanguageModelById } from "@/lib/ai/dynamic-provider";
|
||||
import { resolveDetailAgentLocalMoonshotModel } from "@/lib/ai/detail-agent-model-runtime";
|
||||
import { shouldForceInlineVisionInputs } from "@/lib/ai/model-compat";
|
||||
@ -135,6 +139,11 @@ import {
|
||||
buildAgentToolVideoMarkdown,
|
||||
prependImageModelBadge,
|
||||
} from "@/lib/ai/runtime/agent-media-markdown";
|
||||
import {
|
||||
parseImageAltMetadata,
|
||||
saveGeneratedImages,
|
||||
} from "@/lib/ai/runtime/generated-image-persistence";
|
||||
import type { FalImageEditTaskMetadata } from "@/lib/ai/runtime/fal-image-edit-task";
|
||||
import { saveAssistantTextOnlyMessage } from "@/lib/ai/runtime/assistant-message-persistence";
|
||||
import { resolveCanvasAgentClarificationGate } from "@/lib/ai/runtime/middlewares/canvas-agent-clarification-gate";
|
||||
import {
|
||||
@ -1463,6 +1472,13 @@ async function generateImageWithFailoverCandidate(params: {
|
||||
}) {
|
||||
const { candidate, messages, imageParams, fallbackImageParams } = params;
|
||||
const providerType = String(candidate.provider?.type || "");
|
||||
const candidateRuntimeModelId = String(candidate.model?.modelId || candidate.model?.id || "");
|
||||
if (
|
||||
providerType === "fal-image" &&
|
||||
isFalGptImage2EditModelId(candidateRuntimeModelId)
|
||||
) {
|
||||
throw new Error("FAL GPT Image 2 Edit 仅支持异步队列调用，已跳过同步故障迁移");
|
||||
}
|
||||
const candidateLanguageModel = await getLanguageModelById(candidate.model.id);
|
||||
if (!candidateLanguageModel || typeof (candidateLanguageModel as any).doGenerate !== "function") {
|
||||
throw new Error(`备用模型不支持当前生图故障迁移调用：${candidate.model.displayName || candidate.model.id}`);
|
||||
@ -2381,57 +2397,6 @@ async function convertMessagesWithImages(
|
||||
return convertedMessages;
|
||||
}
|
||||
|
||||
function parseImageAltMetadata(altText: string, thumbnailUrl: string): {
|
||||
originalUrl: string;
|
||||
imageSize: string;
|
||||
} {
|
||||
let originalUrl = thumbnailUrl;
|
||||
let imageSize = "2K";
|
||||
|
||||
// 格式1: thumb|||originalUrl|||size (新格式)
|
||||
if (altText.includes("thumb|||")) {
|
||||
const content = altText.substring(altText.indexOf("thumb|||") + 8);
|
||||
const parts = content.split("|||");
|
||||
originalUrl = parts[0] || thumbnailUrl;
|
||||
imageSize = parts[1] || "2K";
|
||||
}
|
||||
// 格式2: thumb$$originalUrl$$size (中间格式，可能被 LaTeX 解析器破坏)
|
||||
else if (altText.includes("thumb$$")) {
|
||||
const content = altText.substring(altText.indexOf("thumb$$") + 7);
|
||||
const parts = content.split("$$");
|
||||
originalUrl = parts[0] || thumbnailUrl;
|
||||
imageSize = parts[1] || "2K";
|
||||
}
|
||||
// 格式3: thumb|originalUrl|size (旧格式)
|
||||
else if (altText.includes("thumb|")) {
|
||||
const content = altText.substring(altText.indexOf("thumb|") + 6);
|
||||
const parts = content.split("|");
|
||||
originalUrl = parts[0] || thumbnailUrl;
|
||||
imageSize = parts[1] || "2K";
|
||||
}
|
||||
|
||||
// 回退策略：从缩略图 URL 推断原图 URL
|
||||
if (originalUrl === thumbnailUrl) {
|
||||
if (thumbnailUrl.includes("gemini-thumb-")) {
|
||||
originalUrl = thumbnailUrl.replace("gemini-thumb-", "gemini-orig-");
|
||||
} else if (thumbnailUrl.includes("openai-thumb-")) {
|
||||
originalUrl = thumbnailUrl.replace("openai-thumb-", "openai-orig-");
|
||||
} else if (thumbnailUrl.includes("volcano-thumb-")) {
|
||||
originalUrl = thumbnailUrl.replace("volcano-thumb-", "volcano-orig-");
|
||||
} else if (thumbnailUrl.includes("bfl-image-thumb-")) {
|
||||
originalUrl = thumbnailUrl.replace("bfl-image-thumb-", "bfl-image-orig-");
|
||||
} else if (thumbnailUrl.includes("alibaba-image-thumb-")) {
|
||||
originalUrl = thumbnailUrl.replace("alibaba-image-thumb-", "alibaba-image-orig-");
|
||||
} else if (thumbnailUrl.includes("ruxa-image-thumb-")) {
|
||||
originalUrl = thumbnailUrl.replace("ruxa-image-thumb-", "ruxa-image-orig-");
|
||||
} else if (thumbnailUrl.includes("fal-image-thumb-")) {
|
||||
originalUrl = thumbnailUrl.replace("fal-image-thumb-", "fal-image-orig-");
|
||||
}
|
||||
}
|
||||
|
||||
return { originalUrl, imageSize };
|
||||
}
|
||||
|
||||
function extractOriginalImageUrlsFromText(text: string): string[] {
|
||||
return extractImageUrlsFromText(text);
|
||||
}
|
||||
@ -2784,204 +2749,6 @@ function resolveModelRefByRuntimeModelId(
|
||||
return String(match?.model?.id || fallbackRef || normalizedModelId || "");
|
||||
}
|
||||
|
||||
/**
|
||||
* 解析生图 Markdown 并保存图片记录到数据库
|
||||
* 支持三种格式：thumb|||url|||size、thumb$$url$$size、thumb|url|size
|
||||
*/
|
||||
async function saveGeneratedImages(
|
||||
imageMarkdown: string,
|
||||
userId: string,
|
||||
chatId: string,
|
||||
messageId: string,
|
||||
prompt: string,
|
||||
modelId: string,
|
||||
options?: {
|
||||
sourceAssetIds?: string[];
|
||||
sourceImageUrls?: string[];
|
||||
relationType?: "generated_from" | "edited_from";
|
||||
sourceType?: string;
|
||||
projectId?: string | null;
|
||||
userPrompt?: string;
|
||||
optimizedPrompt?: string;
|
||||
requestedImageCount?: number;
|
||||
aspectRatio?: string;
|
||||
imageSize?: string;
|
||||
},
|
||||
): Promise<void> {
|
||||
// 匹配 Markdown 图片格式: 
|
||||
const imageRegex = /!\[([^\]]*)\]\(([^)]+)\)/g;
|
||||
const matches = [...imageMarkdown.matchAll(imageRegex)];
|
||||
|
||||
let savedImageCount = 0;
|
||||
const savedAssetIds: string[] = [];
|
||||
const savedOriginalUrls: string[] = [];
|
||||
let relationSourceAssetIds = Array.isArray(options?.sourceAssetIds)
|
||||
? options.sourceAssetIds.filter((assetId): assetId is string => Boolean(assetId))
|
||||
: [];
|
||||
|
||||
if (
|
||||
relationSourceAssetIds.length === 0 &&
|
||||
Array.isArray(options?.sourceImageUrls) &&
|
||||
options.sourceImageUrls.length > 0
|
||||
) {
|
||||
try {
|
||||
const sourceAssets = await getMediaAssetsByUserAndUrls({
|
||||
userId,
|
||||
urls: options.sourceImageUrls,
|
||||
});
|
||||
relationSourceAssetIds = sourceAssets.map((asset) => asset.id);
|
||||
} catch (sourceLookupError) {
|
||||
console.error("[Gallery] Failed to resolve source assets by URL:", sourceLookupError);
|
||||
}
|
||||
}
|
||||
|
||||
for (const match of matches) {
|
||||
const altText = match[1] || "";
|
||||
const thumbnailUrl = match[2];
|
||||
const { originalUrl, imageSize } = parseImageAltMetadata(altText, thumbnailUrl);
|
||||
|
||||
try {
|
||||
const [savedImage] = await db.insert(generatedImage).values({
|
||||
userId,
|
||||
chatId,
|
||||
messageId,
|
||||
thumbnailUrl,
|
||||
originalUrl,
|
||||
imageSize,
|
||||
prompt,
|
||||
modelId,
|
||||
}).returning();
|
||||
savedImageCount += 1;
|
||||
if (!savedOriginalUrls.includes(originalUrl)) {
|
||||
savedOriginalUrls.push(originalUrl);
|
||||
}
|
||||
const media = await upsertMediaAsset({
|
||||
userId,
|
||||
chatId,
|
||||
projectId: options?.projectId ?? null,
|
||||
mediaType: "image",
|
||||
sourceType: options?.sourceType || "generated",
|
||||
status: "ready",
|
||||
originalUrl,
|
||||
previewUrl: originalUrl,
|
||||
thumbnailUrl,
|
||||
displayName: prompt?.slice(0, 80) || "AI 生成图片",
|
||||
prompt,
|
||||
modelId,
|
||||
sourceMessageId: messageId,
|
||||
sourceTable: "GeneratedImage",
|
||||
sourceRowId: savedImage?.id || null,
|
||||
primaryParentAssetId: relationSourceAssetIds[0] || null,
|
||||
metadata: {
|
||||
imageSize,
|
||||
},
|
||||
});
|
||||
if (media?.id && !savedAssetIds.includes(media.id)) {
|
||||
savedAssetIds.push(media.id);
|
||||
}
|
||||
if (media && relationSourceAssetIds.length > 0) {
|
||||
for (const sourceAssetId of relationSourceAssetIds) {
|
||||
if (!sourceAssetId) continue;
|
||||
await createMediaAssetRelation({
|
||||
userId,
|
||||
chatId,
|
||||
projectId: options?.projectId ?? null,
|
||||
fromAssetId: sourceAssetId,
|
||||
toAssetId: media.id,
|
||||
relationType: options?.relationType || "generated_from",
|
||||
sourceMessageId: messageId,
|
||||
metadata: {
|
||||
modelId,
|
||||
prompt,
|
||||
sourceImageUrls: options?.sourceImageUrls || [],
|
||||
},
|
||||
});
|
||||
}
|
||||
}
|
||||
console.log(`[Gallery] Saved image: ${originalUrl.substring(0, 80)}...`);
|
||||
} catch (error) {
|
||||
console.error("[Gallery] Failed to save image record:", error);
|
||||
}
|
||||
}
|
||||
|
||||
if (savedOriginalUrls.length > 0) {
|
||||
try {
|
||||
const resultAssetIds = savedAssetIds.length > 0 ? savedAssetIds : null;
|
||||
await persistImageWorkflowTurn({
|
||||
userId,
|
||||
chatId,
|
||||
projectId: options?.projectId || null,
|
||||
currentIntent: options?.relationType === "edited_from" ? "image_edit" : "image_gen",
|
||||
currentFocusAssetIds: resultAssetIds,
|
||||
currentFocusImageUrls: savedOriginalUrls,
|
||||
lastGeneratedAssetIds: resultAssetIds,
|
||||
lastGeneratedImageUrls: savedOriginalUrls,
|
||||
lastResolvedTargetAssetIds: resultAssetIds,
|
||||
lastResolvedTargetImageUrls: savedOriginalUrls,
|
||||
lastMessageId: messageId,
|
||||
lastUserPrompt: String(options?.userPrompt || prompt || "").trim() || prompt,
|
||||
lastOptimizedPrompt:
|
||||
String(
|
||||
options?.optimizedPrompt ||
|
||||
(options?.relationType !== "edited_from" ? prompt : "") ||
|
||||
"",
|
||||
).trim() || undefined,
|
||||
lastRequestedImageCount: options?.requestedImageCount || savedOriginalUrls.length,
|
||||
lastAspectRatio: options?.aspectRatio,
|
||||
lastImageSize: options?.imageSize,
|
||||
lastSourceImageUrls: options?.sourceImageUrls || null,
|
||||
lastResultImageUrls: savedOriginalUrls,
|
||||
});
|
||||
} catch (memoryError) {
|
||||
console.warn("[Gallery] Failed to update working state with generated images:", memoryError);
|
||||
}
|
||||
}
|
||||
|
||||
if (savedImageCount <= 0) return;
|
||||
|
||||
try {
|
||||
const pointsConfig = await getPointsSystemConfig();
|
||||
if (!pointsConfig.enabled) return;
|
||||
|
||||
const userData = await getUserById({ id: userId }).catch(() => undefined);
|
||||
if (userData?.isHighVip) {
|
||||
console.log(`[Points] Skip image points for VIP user=${userId} model=${modelId}`);
|
||||
return;
|
||||
}
|
||||
|
||||
const modelConfig = await resolveBillingModelByReference(modelId);
|
||||
const unitCost = getModelCreditCost(modelConfig, "image");
|
||||
if (unitCost <= 0) return;
|
||||
|
||||
const totalCost = unitCost * savedImageCount;
|
||||
const consumeResult = await consumeUserPoints({
|
||||
userId,
|
||||
amount: totalCost,
|
||||
reason: "image_generation",
|
||||
modelId: modelConfig?.id || modelId,
|
||||
chatId,
|
||||
messageId,
|
||||
note: `模型(${modelConfig?.displayName || modelId}) 生图 ${savedImageCount} 张`,
|
||||
});
|
||||
|
||||
if (!consumeResult.ok) {
|
||||
console.warn(
|
||||
`[Points] Failed to consume points after generation. user=${userId} model=${modelId} cost=${totalCost}`
|
||||
);
|
||||
} else if (consumeResult.skipped) {
|
||||
console.log(
|
||||
`[Points] Skipped ${totalCost} image points for VIP user=${userId} model=${modelId}`
|
||||
);
|
||||
} else {
|
||||
console.log(
|
||||
`[Points] Consumed ${totalCost} points (${unitCost} x ${savedImageCount}). balance=${consumeResult.balance}`
|
||||
);
|
||||
}
|
||||
} catch (error) {
|
||||
console.error("[Points] Failed to consume points after image generation:", error);
|
||||
}
|
||||
}
|
||||
|
||||
type AgentPresetMessageImagePart = {
|
||||
type: "file";
|
||||
url: string;
|
||||
@ -10570,6 +10337,122 @@ export async function POST(request: Request) {
|
||||
}
|
||||
|
||||
messagesForAlibaba = injectHistoryImagesToLastUserMessage(messagesForAlibaba, smartAutoHistoryImages);
|
||||
const isFalAsyncImageEdit =
|
||||
(languageModel as any).provider === "fal-image" &&
|
||||
isFalGptImage2EditModelId(actualModelId);
|
||||
|
||||
if (isFalAsyncImageEdit) {
|
||||
const falProvider = alibabaModelConfig?.providerId
|
||||
? await getAiProviderById({ id: alibabaModelConfig.providerId }).catch(() => undefined)
|
||||
: undefined;
|
||||
const falApiKey = String(falProvider?.apiKey || "").trim();
|
||||
if (!falApiKey) {
|
||||
throw new Error("FAL 图片编辑模型未配置可用的 API Key");
|
||||
}
|
||||
|
||||
const falMessages = await convertMessagesWithImages(messagesForAlibaba, "fal-image");
|
||||
const queuedTask = await submitFalImageTask({
|
||||
apiKey: falApiKey,
|
||||
modelId: actualModelId,
|
||||
messages: falMessages,
|
||||
imageParams:
|
||||
effectiveImageParams || {
|
||||
aspectRatio: defaultImageQualityParams.aspectRatio,
|
||||
imageSize: defaultImageQualityParams.imageSize,
|
||||
imageCount: defaultImageQualityParams.imageCount,
|
||||
},
|
||||
});
|
||||
|
||||
const placeholderMessageId = generateUUID();
|
||||
const userPrompt =
|
||||
lastUserMessage?.parts?.find(
|
||||
(p): p is { type: "text"; text: string } => p.type === "text",
|
||||
)?.text || "";
|
||||
const modelRef = resolveModelRefByRuntimeModelId(
|
||||
allModels,
|
||||
queuedTask.modelId,
|
||||
finalModelId,
|
||||
);
|
||||
const placeholderText = `已提交 ${modelDisplayName} 编辑任务，预计 2-4 分钟完成，结果会自动回填到这里。`;
|
||||
const queuedMetadata: FalImageEditTaskMetadata = {
|
||||
taskKind: "fal_gpt_image_2_edit",
|
||||
provider: "fal-image",
|
||||
providerId: String(alibabaModelConfig?.providerId || ""),
|
||||
endpointId: queuedTask.endpointId,
|
||||
requestId: queuedTask.requestId,
|
||||
modelId: queuedTask.modelId,
|
||||
modelRef,
|
||||
modelDisplayName,
|
||||
prompt: queuedTask.prompt,
|
||||
optimizedPrompt: smartAutoOptimizedPrompt || userPrompt || queuedTask.prompt,
|
||||
userPrompt: userPrompt || queuedTask.prompt,
|
||||
requestedImageCount:
|
||||
effectiveImageParams?.imageCount || defaultImageQualityParams.imageCount || 1,
|
||||
aspectRatio: effectiveImageParams?.aspectRatio,
|
||||
imageSize: effectiveImageParams?.imageSize,
|
||||
outputSizeLabel: queuedTask.outputSizeLabel,
|
||||
sourceImageUrls: requestTargetResolution?.targetImageUrls || queuedTask.sourceImageUrls,
|
||||
sourceAssetIds: requestTargetResolution?.targetAssetIds || [],
|
||||
relationType: requestResolvedMediaInput?.hasImage ? "edited_from" : "generated_from",
|
||||
sourceType: "generated",
|
||||
projectId: contextPayload?.projectId || null,
|
||||
contextSource: contextPayload?.source || null,
|
||||
status: "queued",
|
||||
startedAt: Date.now(),
|
||||
updatedAt: Date.now(),
|
||||
queuePosition: null,
|
||||
lastLogs: [],
|
||||
error: null,
|
||||
};
|
||||
|
||||
writeAssistantTextChunk(
|
||||
writer,
|
||||
"<!-- IMAGE_GENERATION_START -->",
|
||||
"alibaba:image-start",
|
||||
);
|
||||
writeImageGenerationStatusChunk(
|
||||
writer,
|
||||
{
|
||||
phase: "queued",
|
||||
providerType: "fal-image",
|
||||
modelName: modelDisplayName,
|
||||
imageCount:
|
||||
effectiveImageParams?.imageCount || defaultImageQualityParams.imageCount || 1,
|
||||
},
|
||||
"alibaba:image-status-queued",
|
||||
);
|
||||
writeAssistantTextChunk(
|
||||
writer,
|
||||
placeholderText,
|
||||
"alibaba:image-queued",
|
||||
placeholderMessageId,
|
||||
);
|
||||
|
||||
await saveMessages({
|
||||
messages: [
|
||||
{
|
||||
id: placeholderMessageId,
|
||||
chatId: id,
|
||||
role: "assistant",
|
||||
parts: [{ type: "text", text: placeholderText }],
|
||||
attachments: [],
|
||||
createdAt: new Date(),
|
||||
videoStorageStatus: null,
|
||||
temporaryVideoUrl: null,
|
||||
permanentVideoUrl: null,
|
||||
archivedAt: null,
|
||||
videoMetadata: queuedMetadata as any,
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
console.log("[Chat API] ✅ Queued FAL GPT Image 2 Edit task", {
|
||||
requestId: queuedTask.requestId,
|
||||
endpointId: queuedTask.endpointId,
|
||||
messageId: placeholderMessageId,
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
const tryGenerateImage = async (modelIdToUse: string, isRetry: boolean) => {
|
||||
// 转换消息（图片编辑模型统一使用 URL 输入）
|
||||
@ -10698,6 +10581,17 @@ export async function POST(request: Request) {
|
||||
console.log(
|
||||
`[Chat API] 🔁 Cross-provider failover attempt: ${candidate.model.displayName} (${candidate.provider.type})`
|
||||
);
|
||||
if (
|
||||
candidate.provider.type === "fal-image" &&
|
||||
isFalGptImage2EditModelId(
|
||||
String(candidate.model?.modelId || candidate.model?.id || "")
|
||||
)
|
||||
) {
|
||||
console.log(
|
||||
`[Chat API] Skip async-only failover candidate: ${candidate.model.displayName}`
|
||||
);
|
||||
continue;
|
||||
}
|
||||
|
||||
const candidateLanguageModel = await getLanguageModelById(candidate.model.id);
|
||||
if (!candidateLanguageModel || typeof (candidateLanguageModel as any).doGenerate !== "function") {
|
||||
@ -11124,6 +11018,17 @@ export async function POST(request: Request) {
|
||||
console.log(
|
||||
`[Chat API] 🔁 Volcano cross-provider failover attempt: ${candidate.model.displayName} (${candidate.provider.type})`
|
||||
);
|
||||
if (
|
||||
candidate.provider.type === "fal-image" &&
|
||||
isFalGptImage2EditModelId(
|
||||
String(candidate.model?.modelId || candidate.model?.id || "")
|
||||
)
|
||||
) {
|
||||
console.log(
|
||||
`[Chat API] Skip async-only failover candidate: ${candidate.model.displayName}`
|
||||
);
|
||||
continue;
|
||||
}
|
||||
|
||||
const candidateLanguageModel = await getLanguageModelById(candidate.model.id);
|
||||
if (!candidateLanguageModel || typeof (candidateLanguageModel as any).doGenerate !== "function") {
|
||||
@ -12014,6 +11919,17 @@ export async function POST(request: Request) {
|
||||
console.log(
|
||||
`[Chat API] 🔁 OpenAI-compatible cross-provider failover: ${candidate.model.displayName} (${candidate.provider.type})`
|
||||
);
|
||||
if (
|
||||
candidate.provider.type === "fal-image" &&
|
||||
isFalGptImage2EditModelId(
|
||||
String(candidate.model?.modelId || candidate.model?.id || "")
|
||||
)
|
||||
) {
|
||||
console.log(
|
||||
`[Chat API] Skip async-only failover candidate: ${candidate.model.displayName}`
|
||||
);
|
||||
continue;
|
||||
}
|
||||
|
||||
const candidateLanguageModel = await getLanguageModelById(candidate.model.id);
|
||||
if (
|
||||
|
||||
@ -1,11 +1,195 @@
|
||||
import { NextRequest, NextResponse } from "next/server";
|
||||
import { auth } from "@/app/(auth)/auth";
|
||||
import { saveMessages, getMessagesByChatId } from "@/lib/db/queries";
|
||||
import { prependImageModelBadge } from "@/lib/ai/runtime/agent-media-markdown";
|
||||
import {
|
||||
isPendingFalImageEditTaskMetadata,
|
||||
type FalImageEditTaskMetadata,
|
||||
} from "@/lib/ai/runtime/fal-image-edit-task";
|
||||
import { saveGeneratedImages } from "@/lib/ai/runtime/generated-image-persistence";
|
||||
import {
|
||||
getChatById,
|
||||
getMessagesByChatId,
|
||||
getAiProviderById,
|
||||
saveMessages,
|
||||
updateMessageById,
|
||||
} from "@/lib/db/queries";
|
||||
import type { DBMessage } from "@/lib/db/schema";
|
||||
import {
|
||||
getFalImageTaskStatus,
|
||||
resolveFalImageTaskToMarkdown,
|
||||
} from "@/lib/ai/providers/fal-image";
|
||||
|
||||
export const dynamic = "force-dynamic";
|
||||
export const revalidate = 0;
|
||||
|
||||
const FAL_IMAGE_TASK_POLL_INTERVAL_MS = 2_000;
|
||||
const falTaskLastPolledAt = new Map<string, number>();
|
||||
const falTaskSyncInFlight = new Set<string>();
|
||||
|
||||
async function markFalTaskFailed(params: {
|
||||
message: DBMessage;
|
||||
metadata: FalImageEditTaskMetadata;
|
||||
errorText: string;
|
||||
}) {
|
||||
await updateMessageById({
|
||||
messageId: params.message.id,
|
||||
chatId: params.message.chatId,
|
||||
parts: [{ type: "text", text: params.errorText }],
|
||||
videoMetadata: {
|
||||
...params.metadata,
|
||||
status: "failed",
|
||||
updatedAt: Date.now(),
|
||||
error: params.errorText,
|
||||
} as any,
|
||||
});
|
||||
}
|
||||
|
||||
async function syncPendingFalImageTask(params: {
|
||||
userId: string;
|
||||
message: DBMessage;
|
||||
}): Promise<boolean> {
|
||||
const metadata = params.message.videoMetadata;
|
||||
if (!isPendingFalImageEditTaskMetadata(metadata)) return false;
|
||||
|
||||
const taskKey = `${params.message.id}:${metadata.requestId}`;
|
||||
const now = Date.now();
|
||||
const lastPolledAt = falTaskLastPolledAt.get(taskKey) || 0;
|
||||
if (now - lastPolledAt < FAL_IMAGE_TASK_POLL_INTERVAL_MS) {
|
||||
return false;
|
||||
}
|
||||
if (falTaskSyncInFlight.has(taskKey)) {
|
||||
return false;
|
||||
}
|
||||
falTaskSyncInFlight.add(taskKey);
|
||||
falTaskLastPolledAt.set(taskKey, now);
|
||||
|
||||
try {
|
||||
const provider = await getAiProviderById({ id: metadata.providerId }).catch(() => undefined);
|
||||
const apiKey = String(provider?.apiKey || "").trim();
|
||||
if (!apiKey) {
|
||||
await markFalTaskFailed({
|
||||
message: params.message,
|
||||
metadata,
|
||||
errorText: "FAL 图片编辑失败：未找到可用的服务商配置",
|
||||
});
|
||||
falTaskLastPolledAt.delete(taskKey);
|
||||
return true;
|
||||
}
|
||||
|
||||
let queueStatus: Awaited<ReturnType<typeof getFalImageTaskStatus>>;
|
||||
try {
|
||||
queueStatus = await getFalImageTaskStatus({
|
||||
apiKey,
|
||||
endpointId: metadata.endpointId,
|
||||
requestId: metadata.requestId,
|
||||
});
|
||||
} catch (error) {
|
||||
console.error("[Messages API] Failed to poll FAL task status:", {
|
||||
messageId: params.message.id,
|
||||
requestId: metadata.requestId,
|
||||
error,
|
||||
});
|
||||
return false;
|
||||
}
|
||||
|
||||
if (queueStatus.status === "COMPLETED") {
|
||||
try {
|
||||
const imageMarkdown = await resolveFalImageTaskToMarkdown({
|
||||
apiKey,
|
||||
endpointId: metadata.endpointId,
|
||||
requestId: metadata.requestId,
|
||||
imageCount: metadata.requestedImageCount,
|
||||
outputSizeLabel: metadata.outputSizeLabel,
|
||||
sourceImageUrls: metadata.sourceImageUrls || [],
|
||||
});
|
||||
const finalText = prependImageModelBadge(
|
||||
imageMarkdown,
|
||||
metadata.modelDisplayName || metadata.modelId,
|
||||
);
|
||||
|
||||
await updateMessageById({
|
||||
messageId: params.message.id,
|
||||
chatId: params.message.chatId,
|
||||
parts: [{ type: "text", text: finalText }],
|
||||
videoMetadata: {
|
||||
...metadata,
|
||||
status: "completed",
|
||||
updatedAt: Date.now(),
|
||||
queuePosition: null,
|
||||
lastLogs: Array.isArray((queueStatus as any)?.logs) ? (queueStatus as any).logs : [],
|
||||
error: null,
|
||||
} as any,
|
||||
});
|
||||
|
||||
await saveGeneratedImages(
|
||||
finalText,
|
||||
params.userId,
|
||||
params.message.chatId,
|
||||
params.message.id,
|
||||
metadata.userPrompt || metadata.prompt,
|
||||
metadata.modelRef || metadata.modelId,
|
||||
{
|
||||
sourceAssetIds: metadata.sourceAssetIds || [],
|
||||
sourceImageUrls: metadata.sourceImageUrls || [],
|
||||
relationType: metadata.relationType || "edited_from",
|
||||
sourceType: metadata.sourceType || "generated",
|
||||
projectId: metadata.projectId || null,
|
||||
optimizedPrompt: metadata.optimizedPrompt || metadata.prompt,
|
||||
userPrompt: metadata.userPrompt || metadata.prompt,
|
||||
requestedImageCount: metadata.requestedImageCount,
|
||||
aspectRatio: metadata.aspectRatio,
|
||||
imageSize: metadata.imageSize,
|
||||
},
|
||||
);
|
||||
|
||||
falTaskLastPolledAt.delete(taskKey);
|
||||
return true;
|
||||
} catch (error) {
|
||||
console.error("[Messages API] Failed to finalize completed FAL task:", {
|
||||
messageId: params.message.id,
|
||||
requestId: metadata.requestId,
|
||||
error,
|
||||
});
|
||||
await markFalTaskFailed({
|
||||
message: params.message,
|
||||
metadata,
|
||||
errorText: "FAL 图片编辑结果处理失败，请重新提交任务",
|
||||
});
|
||||
falTaskLastPolledAt.delete(taskKey);
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
const nextStatus = queueStatus.status === "IN_QUEUE" ? "queued" : "running";
|
||||
const nextQueuePosition =
|
||||
queueStatus.status === "IN_QUEUE" && typeof (queueStatus as any).queue_position === "number"
|
||||
? (queueStatus as any).queue_position
|
||||
: null;
|
||||
const hasMeaningfulChange =
|
||||
metadata.status !== nextStatus ||
|
||||
(metadata.queuePosition ?? null) !== nextQueuePosition;
|
||||
|
||||
if (hasMeaningfulChange) {
|
||||
await updateMessageById({
|
||||
messageId: params.message.id,
|
||||
chatId: params.message.chatId,
|
||||
videoMetadata: {
|
||||
...metadata,
|
||||
status: nextStatus,
|
||||
updatedAt: Date.now(),
|
||||
queuePosition: nextQueuePosition,
|
||||
lastLogs: Array.isArray((queueStatus as any)?.logs) ? (queueStatus as any).logs : [],
|
||||
} as any,
|
||||
});
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
} finally {
|
||||
falTaskSyncInFlight.delete(taskKey);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取聊天消息
|
||||
* 用于从数据库重新加载消息
|
||||
@ -27,8 +211,26 @@ export async function GET(request: NextRequest) {
|
||||
);
|
||||
}
|
||||
|
||||
// 从数据库获取消息
|
||||
const messages = await getMessagesByChatId({ id: chatId });
|
||||
const selectedChat = await getChatById({ id: chatId });
|
||||
if (!selectedChat || selectedChat.userId !== session.user.id) {
|
||||
return NextResponse.json({ error: "Forbidden" }, { status: 403 });
|
||||
}
|
||||
|
||||
let messages = await getMessagesByChatId({ id: chatId });
|
||||
let mutated = false;
|
||||
for (const message of messages) {
|
||||
if (message.role !== "assistant") continue;
|
||||
if (!isPendingFalImageEditTaskMetadata(message.videoMetadata)) continue;
|
||||
const changed = await syncPendingFalImageTask({
|
||||
userId: session.user.id,
|
||||
message,
|
||||
});
|
||||
mutated = mutated || changed;
|
||||
}
|
||||
|
||||
if (mutated) {
|
||||
messages = await getMessagesByChatId({ id: chatId });
|
||||
}
|
||||
|
||||
// 转换为前端格式
|
||||
const formattedMessages = messages.map((msg: DBMessage) => {
|
||||
@ -41,6 +243,7 @@ export async function GET(request: NextRequest) {
|
||||
createdAt: msg.createdAt,
|
||||
parts: parts,
|
||||
attachments: msg.attachments || [],
|
||||
videoMetadata: msg.videoMetadata || null,
|
||||
};
|
||||
});
|
||||
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
385
components/agents/AgentImageTextEditDialog.tsx
Normal file
385
components/agents/AgentImageTextEditDialog.tsx
Normal file
@ -0,0 +1,385 @@
|
||||
"use client";
|
||||
|
||||
import { useCallback, useEffect, useMemo, useState } from "react";
|
||||
import { Loader2, RefreshCw, Trash2, Type, X } from "lucide-react";
|
||||
import { toast } from "sonner";
|
||||
import {
|
||||
Dialog,
|
||||
DialogContent,
|
||||
DialogDescription,
|
||||
DialogHeader,
|
||||
DialogTitle,
|
||||
} from "@/components/ui/dialog";
|
||||
import { Button } from "@/components/ui/button";
|
||||
import { Input } from "@/components/ui/input";
|
||||
import { imageGenerationService, type CanvasTextEditPayloadItem } from "@/services/canvas";
|
||||
|
||||
interface AgentImageTextEditDialogProps {
|
||||
open: boolean;
|
||||
imageUrl?: string | null;
|
||||
title?: string | null;
|
||||
onOpenChange: (open: boolean) => void;
|
||||
onApplySuccess?: (result: { originalUrl: string; thumbnailUrl?: string | null; model?: string | null }) => void;
|
||||
}
|
||||
|
||||
interface AgentImageTextEditItem {
|
||||
id: string;
|
||||
originalText: string;
|
||||
editedText: string;
|
||||
confidence?: number;
|
||||
}
|
||||
|
||||
interface DeletedTextItem {
|
||||
id: string;
|
||||
originalText: string;
|
||||
}
|
||||
|
||||
function isHttpImageUrl(value?: string | null) {
|
||||
if (!value) return false;
|
||||
try {
|
||||
const parsed = new URL(String(value).trim());
|
||||
return parsed.protocol === "http:" || parsed.protocol === "https:";
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
export function AgentImageTextEditDialog({
|
||||
open,
|
||||
imageUrl,
|
||||
title,
|
||||
onOpenChange,
|
||||
onApplySuccess,
|
||||
}: AgentImageTextEditDialogProps) {
|
||||
const normalizedImageUrl = useMemo(() => String(imageUrl || "").trim(), [imageUrl]);
|
||||
const [textEditItems, setTextEditItems] = useState<AgentImageTextEditItem[]>([]);
|
||||
const [deletedTextItems, setDeletedTextItems] = useState<DeletedTextItem[]>([]);
|
||||
const [isDetectingText, setIsDetectingText] = useState(false);
|
||||
const [isApplyingTextEdit, setIsApplyingTextEdit] = useState(false);
|
||||
const [textEditError, setTextEditError] = useState<string | null>(null);
|
||||
const [detectedSourceUrl, setDetectedSourceUrl] = useState<string | null>(null);
|
||||
|
||||
const detectTextItems = useCallback(
|
||||
async (forceRefresh = false) => {
|
||||
if (!normalizedImageUrl) {
|
||||
setTextEditItems([]);
|
||||
setDeletedTextItems([]);
|
||||
setTextEditError("当前图片地址无效");
|
||||
return;
|
||||
}
|
||||
|
||||
if (!isHttpImageUrl(normalizedImageUrl)) {
|
||||
setTextEditItems([]);
|
||||
setDeletedTextItems([]);
|
||||
setTextEditError("仅支持 URL 图片，不允许 base64 传输");
|
||||
return;
|
||||
}
|
||||
|
||||
if (isDetectingText) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (!forceRefresh && detectedSourceUrl === normalizedImageUrl && textEditItems.length > 0) {
|
||||
setDeletedTextItems([]);
|
||||
setTextEditError(null);
|
||||
return;
|
||||
}
|
||||
|
||||
setIsDetectingText(true);
|
||||
setTextEditError(null);
|
||||
|
||||
try {
|
||||
const result = await imageGenerationService.detectCanvasText(normalizedImageUrl);
|
||||
if (!result.success) {
|
||||
throw new Error(result.error || "文字识别失败");
|
||||
}
|
||||
|
||||
const detectedItems: AgentImageTextEditItem[] = [];
|
||||
(result.data?.items || []).forEach((item, index) => {
|
||||
const text = String(item.text || "").trim();
|
||||
if (!text) {
|
||||
return;
|
||||
}
|
||||
detectedItems.push({
|
||||
id: item.id || String(index + 1),
|
||||
originalText: text,
|
||||
editedText: text,
|
||||
confidence: item.confidence,
|
||||
});
|
||||
});
|
||||
|
||||
setTextEditItems(detectedItems);
|
||||
setDeletedTextItems([]);
|
||||
setDetectedSourceUrl(normalizedImageUrl);
|
||||
|
||||
if (detectedItems.length === 0) {
|
||||
toast.info("未识别到可编辑文字");
|
||||
} else {
|
||||
toast.success(`已识别 ${detectedItems.length} 段文字`);
|
||||
}
|
||||
} catch (error) {
|
||||
const message = error instanceof Error ? error.message : "文字识别失败";
|
||||
setTextEditError(message);
|
||||
toast.error(message);
|
||||
} finally {
|
||||
setIsDetectingText(false);
|
||||
}
|
||||
},
|
||||
[detectedSourceUrl, isDetectingText, normalizedImageUrl, textEditItems.length]
|
||||
);
|
||||
|
||||
useEffect(() => {
|
||||
if (!open) {
|
||||
return;
|
||||
}
|
||||
|
||||
setTextEditError(null);
|
||||
|
||||
if (detectedSourceUrl === normalizedImageUrl && textEditItems.length > 0) {
|
||||
setDeletedTextItems([]);
|
||||
return;
|
||||
}
|
||||
|
||||
setTextEditItems([]);
|
||||
setDeletedTextItems([]);
|
||||
void detectTextItems(true);
|
||||
}, [detectTextItems, detectedSourceUrl, normalizedImageUrl, open, textEditItems.length]);
|
||||
|
||||
const handleTextItemChange = useCallback((id: string, value: string) => {
|
||||
setTextEditItems((prev) =>
|
||||
prev.map((item) => (item.id === id ? { ...item, editedText: value } : item))
|
||||
);
|
||||
}, []);
|
||||
|
||||
const handleRemoveTextItem = useCallback((id: string) => {
|
||||
setTextEditItems((prev) => {
|
||||
const target = prev.find((item) => item.id === id);
|
||||
if (!target?.originalText) {
|
||||
return prev.filter((item) => item.id !== id);
|
||||
}
|
||||
setDeletedTextItems((deletedPrev) => {
|
||||
if (deletedPrev.some((item) => item.id === id)) {
|
||||
return deletedPrev;
|
||||
}
|
||||
return [...deletedPrev, { id, originalText: target.originalText }];
|
||||
});
|
||||
return prev.filter((item) => item.id !== id);
|
||||
});
|
||||
}, []);
|
||||
|
||||
const handleApplyTextEdit = useCallback(async () => {
|
||||
if (!normalizedImageUrl) {
|
||||
toast.error("当前图片地址无效");
|
||||
return;
|
||||
}
|
||||
|
||||
if (!isHttpImageUrl(normalizedImageUrl)) {
|
||||
toast.error("仅支持 URL 图片，不允许 base64 传输");
|
||||
return;
|
||||
}
|
||||
|
||||
if (isApplyingTextEdit) {
|
||||
return;
|
||||
}
|
||||
|
||||
const changedItems: CanvasTextEditPayloadItem[] = textEditItems
|
||||
.map((item) => {
|
||||
const editedText = item.editedText.trim();
|
||||
const action: "replace" | "delete" = editedText.length === 0 ? "delete" : "replace";
|
||||
return {
|
||||
id: item.id,
|
||||
originalText: item.originalText.trim(),
|
||||
editedText,
|
||||
action,
|
||||
};
|
||||
})
|
||||
.filter(
|
||||
(item) =>
|
||||
item.originalText.length > 0 &&
|
||||
(item.action === "delete" ||
|
||||
(item.editedText &&
|
||||
item.editedText.length > 0 &&
|
||||
item.originalText !== item.editedText))
|
||||
);
|
||||
|
||||
const removedItems: CanvasTextEditPayloadItem[] = deletedTextItems
|
||||
.map((item) => ({
|
||||
id: item.id,
|
||||
originalText: item.originalText.trim(),
|
||||
action: "delete" as const,
|
||||
}))
|
||||
.filter((item) => item.originalText.length > 0);
|
||||
|
||||
const requestItems = [...changedItems, ...removedItems];
|
||||
if (requestItems.length === 0) {
|
||||
toast.info("请先修改或删除至少一段文字");
|
||||
return;
|
||||
}
|
||||
|
||||
setIsApplyingTextEdit(true);
|
||||
setTextEditError(null);
|
||||
|
||||
try {
|
||||
const result = await imageGenerationService.editCanvasText({
|
||||
imageUrl: normalizedImageUrl,
|
||||
items: requestItems,
|
||||
});
|
||||
|
||||
if (!result.success || !result.data?.originalUrl) {
|
||||
throw new Error(result.error || "文字替换失败");
|
||||
}
|
||||
|
||||
toast.success("文字替换已完成");
|
||||
setDeletedTextItems([]);
|
||||
setDetectedSourceUrl(result.data.originalUrl);
|
||||
onApplySuccess?.({
|
||||
originalUrl: result.data.originalUrl,
|
||||
thumbnailUrl: result.data.thumbnailUrl,
|
||||
model: result.data.model || null,
|
||||
});
|
||||
onOpenChange(false);
|
||||
} catch (error) {
|
||||
const message = error instanceof Error ? error.message : "文字替换失败";
|
||||
setTextEditError(message);
|
||||
toast.error(message);
|
||||
} finally {
|
||||
setIsApplyingTextEdit(false);
|
||||
}
|
||||
}, [
|
||||
deletedTextItems,
|
||||
isApplyingTextEdit,
|
||||
normalizedImageUrl,
|
||||
onApplySuccess,
|
||||
onOpenChange,
|
||||
textEditItems,
|
||||
]);
|
||||
|
||||
return (
|
||||
<Dialog open={open} onOpenChange={onOpenChange}>
|
||||
<DialogContent className="max-h-[88vh] overflow-hidden border-slate-200 bg-white p-0 sm:max-w-[620px] dark:border-zinc-800 dark:bg-zinc-950">
|
||||
<DialogHeader className="border-b border-slate-200 px-6 py-4 dark:border-zinc-800">
|
||||
<DialogTitle className="flex items-center gap-2 text-left text-lg">
|
||||
<Type className="h-5 w-5" />
|
||||
<span>编辑文字</span>
|
||||
</DialogTitle>
|
||||
<DialogDescription className="text-left text-sm leading-6 text-slate-500 dark:text-zinc-400">
|
||||
识别图片文字后可逐条修改或删除，应用后会在保持画面风格的前提下完成文字更新。
|
||||
{title ? ` 当前处理：${title}` : ""}
|
||||
</DialogDescription>
|
||||
</DialogHeader>
|
||||
|
||||
<div className="agent-dark-scrollbar max-h-[calc(88vh-156px)] space-y-4 overflow-y-auto px-6 py-5">
|
||||
{normalizedImageUrl ? (
|
||||
<div className="overflow-hidden rounded-2xl border border-slate-200 bg-slate-50 dark:border-zinc-800 dark:bg-zinc-900">
|
||||
<img
|
||||
src={normalizedImageUrl}
|
||||
alt={title || "待编辑结果图"}
|
||||
className="block max-h-[240px] w-full object-contain"
|
||||
/>
|
||||
</div>
|
||||
) : null}
|
||||
|
||||
{isDetectingText ? (
|
||||
<div className="flex items-center gap-2 rounded-xl border border-slate-200 bg-slate-50 px-4 py-3 text-sm text-slate-600 dark:border-zinc-800 dark:bg-zinc-900 dark:text-zinc-300">
|
||||
<Loader2 className="h-4 w-4 animate-spin" />
|
||||
正在识别图片文字...
|
||||
</div>
|
||||
) : null}
|
||||
|
||||
{textEditError && !isDetectingText ? (
|
||||
<div className="rounded-xl bg-red-500/10 px-4 py-3 text-sm text-red-500">
|
||||
{textEditError}
|
||||
</div>
|
||||
) : null}
|
||||
|
||||
{!isDetectingText && textEditItems.length === 0 && !textEditError ? (
|
||||
<div className="rounded-xl border border-dashed border-slate-200 bg-slate-50 px-4 py-4 text-sm leading-6 text-slate-500 dark:border-zinc-800 dark:bg-zinc-900 dark:text-zinc-400">
|
||||
未识别到可编辑文字，可点击“重新识别”再试一次。
|
||||
</div>
|
||||
) : null}
|
||||
|
||||
{textEditItems.map((item, index) => (
|
||||
<div
|
||||
key={item.id}
|
||||
className="space-y-3 rounded-2xl border border-slate-200 p-4 dark:border-zinc-800"
|
||||
>
|
||||
<div className="flex items-center justify-between gap-3">
|
||||
<div className="text-xs text-slate-500 dark:text-zinc-400">
|
||||
文本 {index + 1}
|
||||
{typeof item.confidence === "number"
|
||||
? ` · 置信度 ${(item.confidence * 100).toFixed(0)}%`
|
||||
: ""}
|
||||
</div>
|
||||
<Button
|
||||
type="button"
|
||||
size="icon"
|
||||
variant="ghost"
|
||||
className="h-8 w-8 text-slate-500 hover:text-red-500 dark:text-zinc-400"
|
||||
onClick={() => handleRemoveTextItem(item.id)}
|
||||
title="删除该条文字"
|
||||
>
|
||||
<Trash2 className="h-4 w-4" />
|
||||
</Button>
|
||||
</div>
|
||||
<div className="text-sm text-slate-500 dark:text-zinc-400">原文：{item.originalText}</div>
|
||||
<Input
|
||||
value={item.editedText}
|
||||
onChange={(event) => handleTextItemChange(item.id, event.target.value)}
|
||||
placeholder="输入替换后的文字"
|
||||
/>
|
||||
</div>
|
||||
))}
|
||||
|
||||
{deletedTextItems.length > 0 ? (
|
||||
<div className="rounded-xl border border-orange-300 border-dashed bg-orange-50 px-4 py-3 text-sm text-orange-700 dark:border-orange-900 dark:bg-orange-950/40 dark:text-orange-200">
|
||||
已标记删除 {deletedTextItems.length} 条文字，应用时会明确告诉生图模型移除这些文字并修复背景。
|
||||
</div>
|
||||
) : null}
|
||||
</div>
|
||||
|
||||
<div className="flex items-center justify-between gap-3 border-t border-slate-200 px-6 py-4 dark:border-zinc-800">
|
||||
<Button
|
||||
type="button"
|
||||
variant="outline"
|
||||
disabled={isDetectingText || isApplyingTextEdit}
|
||||
onClick={() => void detectTextItems(true)}
|
||||
>
|
||||
{isDetectingText ? (
|
||||
<Loader2 className="mr-2 h-4 w-4 animate-spin" />
|
||||
) : (
|
||||
<RefreshCw className="mr-2 h-4 w-4" />
|
||||
)}
|
||||
重新识别
|
||||
</Button>
|
||||
<div className="flex items-center gap-3">
|
||||
<Button
|
||||
type="button"
|
||||
variant="ghost"
|
||||
disabled={isApplyingTextEdit}
|
||||
onClick={() => onOpenChange(false)}
|
||||
>
|
||||
<X className="mr-2 h-4 w-4" />
|
||||
取消
|
||||
</Button>
|
||||
<Button
|
||||
type="button"
|
||||
disabled={
|
||||
isDetectingText ||
|
||||
isApplyingTextEdit ||
|
||||
(textEditItems.length === 0 && deletedTextItems.length === 0)
|
||||
}
|
||||
onClick={() => void handleApplyTextEdit()}
|
||||
>
|
||||
{isApplyingTextEdit ? (
|
||||
<Loader2 className="mr-2 h-4 w-4 animate-spin" />
|
||||
) : (
|
||||
<Type className="mr-2 h-4 w-4" />
|
||||
)}
|
||||
确认替换
|
||||
</Button>
|
||||
</div>
|
||||
</div>
|
||||
</DialogContent>
|
||||
</Dialog>
|
||||
);
|
||||
}
|
||||
@ -2489,7 +2489,7 @@ export function CanvasChatPanel({
|
||||
}, []);
|
||||
|
||||
// 从 API 加载历史消息
|
||||
const { data: historyData } = useSWR<{
|
||||
const { data: historyData, mutate: mutateHistoryData } = useSWR<{
|
||||
messages: ChatMessage[];
|
||||
}>(rawChatId ? `/api/messages?chatId=${rawChatId}` : null, fetcher, {
|
||||
revalidateOnFocus: false,
|
||||
@ -5171,6 +5171,63 @@ export function CanvasChatPanel({
|
||||
};
|
||||
}, [historyData, rawChatId, setMessages, status]);
|
||||
|
||||
useEffect(() => {
|
||||
const streamState = streamImageGenerationStateByChatId[activeChatId];
|
||||
if (!rawChatId || !streamState) return;
|
||||
|
||||
let cancelled = false;
|
||||
const poll = async () => {
|
||||
if (cancelled) return;
|
||||
try {
|
||||
await mutateHistoryData();
|
||||
} catch (error) {
|
||||
console.error("[CanvasChat] Failed to poll message history:", error);
|
||||
}
|
||||
};
|
||||
|
||||
void poll();
|
||||
const timer = window.setInterval(poll, 1500);
|
||||
return () => {
|
||||
cancelled = true;
|
||||
window.clearInterval(timer);
|
||||
};
|
||||
}, [activeChatId, rawChatId, streamImageGenerationStateByChatId, mutateHistoryData]);
|
||||
|
||||
useEffect(() => {
|
||||
const streamState = streamImageGenerationStateByChatId[activeChatId];
|
||||
if (!streamState || messages.length === 0) return;
|
||||
|
||||
const lastUserIndex = [...messages]
|
||||
.map((message) => message.role)
|
||||
.lastIndexOf("user");
|
||||
const assistantMessages =
|
||||
lastUserIndex >= 0
|
||||
? messages.slice(lastUserIndex + 1).filter((message) => message.role === "assistant")
|
||||
: messages.filter((message) => message.role === "assistant");
|
||||
|
||||
const hasCompletedMedia = assistantMessages.some((message) => {
|
||||
const parts = (message.parts || []) as any[];
|
||||
const textContent = parts
|
||||
.filter((part) => part?.type === "text" && typeof part?.text === "string")
|
||||
.map((part) => String(part.text))
|
||||
.join("\n");
|
||||
return (
|
||||
extractAssistantImageUrls(textContent).length > 0 ||
|
||||
extractVideoUrls(textContent).length > 0 ||
|
||||
/失败|超时|重试|error|failed/i.test(textContent)
|
||||
);
|
||||
});
|
||||
|
||||
if (hasCompletedMedia) {
|
||||
clearStreamImageGenerationStateForChat(activeChatId);
|
||||
}
|
||||
}, [
|
||||
activeChatId,
|
||||
messages,
|
||||
streamImageGenerationStateByChatId,
|
||||
clearStreamImageGenerationStateForChat,
|
||||
]);
|
||||
|
||||
// 监听消息变化，提取图片并添加到画布
|
||||
// 仅处理历史同步后的新消息，避免刷新后把历史图片重复回灌到画布
|
||||
const processedMessageIds = useRef<Set<string>>(new Set());
|
||||
|
||||
@ -1002,7 +1002,7 @@ export function Chat({
|
||||
if (!pendingGeneration || !isImagePending) return;
|
||||
|
||||
let cancelled = false;
|
||||
const maxDurationMs = 10 * 60 * 1000;
|
||||
const maxDurationMs = 20 * 60 * 1000;
|
||||
|
||||
const mergeMessagesFromDatabase = (dbMessages: ChatMessage[]) => {
|
||||
setMessages((prev) => {
|
||||
|
||||
@ -14,7 +14,7 @@ export interface PendingGeneration {
|
||||
}
|
||||
|
||||
const STORAGE_KEY = "ai-chatbot-pending-generations";
|
||||
const EXPIRY_TIME = 5 * 60 * 1000; // 5 分钟超时
|
||||
const EXPIRY_TIME = 20 * 60 * 1000; // 20 分钟超时
|
||||
|
||||
/**
|
||||
* 获取所有未完成的生成任务
|
||||
|
||||
@ -181,7 +181,7 @@ export const FAL_IMAGE_MODEL_PRESETS: FalImageModelPreset[] = [
|
||||
"quality",
|
||||
"num_images",
|
||||
"output_format",
|
||||
"mask_image_url",
|
||||
"mask_url",
|
||||
"openai_api_key",
|
||||
],
|
||||
supportsMultiImageReference: true,
|
||||
|
||||
@ -28,6 +28,16 @@ interface UploadedImage {
|
||||
original: string;
|
||||
}
|
||||
|
||||
export interface FalSubmittedImageTask {
|
||||
requestId: string;
|
||||
endpointId: string;
|
||||
modelId: string;
|
||||
imageCount: number;
|
||||
outputSizeLabel: string;
|
||||
prompt: string;
|
||||
sourceImageUrls: string[];
|
||||
}
|
||||
|
||||
const DEFAULT_ASPECT_RATIO = "auto";
|
||||
const DEFAULT_RESOLUTION = "1K";
|
||||
const DEFAULT_OUTPUT_FORMAT = "png";
|
||||
@ -203,7 +213,7 @@ function normalizeSeed(value?: string | number) {
|
||||
return seed;
|
||||
}
|
||||
|
||||
function isFalGptImage2EditModel(modelId: string) {
|
||||
export function isFalGptImage2EditModelId(modelId: string) {
|
||||
return modelId.trim().toLowerCase() === FAL_GPT_IMAGE_2_EDIT_MODEL_ID;
|
||||
}
|
||||
|
||||
@ -443,6 +453,208 @@ function summarizeFalError(error: any) {
|
||||
};
|
||||
}
|
||||
|
||||
function createFalClientFromApiKey(apiKey: string) {
|
||||
return import("@fal-ai/client").then(({ createFalClient }) =>
|
||||
createFalClient({
|
||||
credentials: normalizeFalCredentials(apiKey),
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
function buildFalRequestPayload(params: {
|
||||
modelId: string;
|
||||
overrideModelId?: string;
|
||||
messages: Array<{ role: string; content: any }>;
|
||||
imageParams?: FalImageParams;
|
||||
}) {
|
||||
const { prompt, imageUrls } = extractPromptAndImages(params.messages);
|
||||
const aspectRatio = normalizeAspectRatio(params.imageParams?.aspectRatio);
|
||||
const resolution = normalizeResolution(params.imageParams?.imageSize);
|
||||
const outputFormat = normalizeOutputFormat(params.imageParams?.outputFormat);
|
||||
const safetyTolerance = normalizeSafetyTolerance(params.imageParams?.safetyTolerance);
|
||||
const seed = normalizeSeed(params.imageParams?.seed);
|
||||
const imageCount = Math.max(1, Math.min(6, Number(params.imageParams?.imageCount || 1)));
|
||||
const activeModelId = params.overrideModelId || params.modelId;
|
||||
const isGptImage2Edit = isFalGptImage2EditModelId(activeModelId);
|
||||
const falEndpointId = isGptImage2Edit
|
||||
? activeModelId
|
||||
: normalizeFalEndpointId(activeModelId, imageUrls.length > 0);
|
||||
|
||||
if (!isGptImage2Edit && falEndpointId.toLowerCase().endsWith("/edit") && imageUrls.length === 0) {
|
||||
throw new Error("FAL 图片编辑模型需要至少一张可访问的参考图片 URL");
|
||||
}
|
||||
|
||||
const gptImage2EditSize = normalizeGptImage2EditImageSize(
|
||||
params.imageParams?.imageSize,
|
||||
params.imageParams?.aspectRatio,
|
||||
);
|
||||
const outputSizeLabel = isGptImage2Edit ? getFalImageSizeLabel(gptImage2EditSize) : resolution;
|
||||
|
||||
let input: Record<string, any>;
|
||||
if (isGptImage2Edit) {
|
||||
if (imageUrls.length === 0) {
|
||||
throw new Error("FAL GPT Image 2 Edit 需要至少一张可访问的参考图片 URL");
|
||||
}
|
||||
|
||||
input = {
|
||||
prompt,
|
||||
image_urls: imageUrls,
|
||||
image_size: gptImage2EditSize,
|
||||
quality: normalizeGptImage2EditQuality(params.imageParams?.quality),
|
||||
num_images: imageCount,
|
||||
output_format: outputFormat,
|
||||
};
|
||||
|
||||
const maskImageUrl = normalizeOptionalHttpUrl(params.imageParams?.maskImageUrl);
|
||||
if (maskImageUrl) {
|
||||
input.mask_url = maskImageUrl;
|
||||
}
|
||||
|
||||
if (seed !== undefined) {
|
||||
input.seed = seed;
|
||||
}
|
||||
|
||||
const openaiApiKey = String(params.imageParams?.openaiApiKey || "").trim();
|
||||
if (openaiApiKey) {
|
||||
input.openai_api_key = openaiApiKey;
|
||||
}
|
||||
} else {
|
||||
input = {
|
||||
prompt,
|
||||
num_images: imageCount,
|
||||
aspect_ratio: aspectRatio,
|
||||
resolution,
|
||||
output_format: outputFormat,
|
||||
safety_tolerance: safetyTolerance,
|
||||
limit_generations: true,
|
||||
};
|
||||
|
||||
if (imageUrls.length > 0) {
|
||||
input.image_urls = imageUrls;
|
||||
}
|
||||
|
||||
if (seed !== undefined) {
|
||||
input.seed = seed;
|
||||
}
|
||||
|
||||
if (typeof params.imageParams?.enableWebSearch === "boolean") {
|
||||
input.enable_web_search = params.imageParams.enableWebSearch;
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
prompt,
|
||||
imageUrls,
|
||||
aspectRatio,
|
||||
resolution,
|
||||
outputFormat,
|
||||
safetyTolerance,
|
||||
seed,
|
||||
imageCount,
|
||||
activeModelId,
|
||||
isGptImage2Edit,
|
||||
falEndpointId,
|
||||
gptImage2EditSize,
|
||||
outputSizeLabel,
|
||||
input,
|
||||
};
|
||||
}
|
||||
|
||||
async function normalizeFalResultToMarkdown(params: {
|
||||
payload: any;
|
||||
sourceImageUrls: string[];
|
||||
imageCount: number;
|
||||
outputSizeLabel: string;
|
||||
}) {
|
||||
let outputUrls = filterSourceImages(collectImageUrls(params.payload), params.sourceImageUrls);
|
||||
outputUrls = await filterSourceImagesByContent(outputUrls, params.sourceImageUrls);
|
||||
|
||||
if (outputUrls.length === 0 && params.payload?.data) {
|
||||
outputUrls = filterSourceImages(collectImageUrls(params.payload.data), params.sourceImageUrls);
|
||||
outputUrls = await filterSourceImagesByContent(outputUrls, params.sourceImageUrls);
|
||||
}
|
||||
|
||||
if (outputUrls.length === 0) {
|
||||
throw new Error("FAL 未返回可用图片");
|
||||
}
|
||||
|
||||
const uploadedImages: UploadedImage[] = [];
|
||||
for (const outputUrl of outputUrls.slice(0, params.imageCount)) {
|
||||
uploadedImages.push(await downloadAndUploadImage(outputUrl));
|
||||
}
|
||||
|
||||
if (uploadedImages.length === 0) {
|
||||
throw new Error("FAL 未返回可用图片");
|
||||
}
|
||||
|
||||
return formatMarkdown(uploadedImages, params.outputSizeLabel);
|
||||
}
|
||||
|
||||
export async function submitFalImageTask(params: {
|
||||
apiKey: string;
|
||||
modelId: string;
|
||||
messages: Array<{ role: string; content: any }>;
|
||||
imageParams?: FalImageParams;
|
||||
overrideModelId?: string;
|
||||
}): Promise<FalSubmittedImageTask> {
|
||||
const request = buildFalRequestPayload(params);
|
||||
const fal = await createFalClientFromApiKey(params.apiKey);
|
||||
|
||||
console.log("[Fal Image] Queue submit", {
|
||||
model: request.activeModelId,
|
||||
endpoint: request.falEndpointId,
|
||||
imageCount: request.imageCount,
|
||||
hasInputImages: request.imageUrls.length > 0,
|
||||
resolution: request.outputSizeLabel,
|
||||
});
|
||||
|
||||
const enqueueResult = await fal.queue.submit(request.falEndpointId, {
|
||||
input: request.input,
|
||||
});
|
||||
|
||||
return {
|
||||
requestId: String(enqueueResult.request_id),
|
||||
endpointId: request.falEndpointId,
|
||||
modelId: request.activeModelId,
|
||||
imageCount: request.imageCount,
|
||||
outputSizeLabel: request.outputSizeLabel,
|
||||
prompt: request.prompt,
|
||||
sourceImageUrls: request.imageUrls,
|
||||
};
|
||||
}
|
||||
|
||||
export async function getFalImageTaskStatus(params: {
|
||||
apiKey: string;
|
||||
endpointId: string;
|
||||
requestId: string;
|
||||
}) {
|
||||
const fal = await createFalClientFromApiKey(params.apiKey);
|
||||
return fal.queue.status(params.endpointId, {
|
||||
requestId: params.requestId,
|
||||
logs: true,
|
||||
});
|
||||
}
|
||||
|
||||
export async function resolveFalImageTaskToMarkdown(params: {
|
||||
apiKey: string;
|
||||
endpointId: string;
|
||||
requestId: string;
|
||||
imageCount: number;
|
||||
outputSizeLabel: string;
|
||||
sourceImageUrls: string[];
|
||||
}) {
|
||||
const fal = await createFalClientFromApiKey(params.apiKey);
|
||||
const result = await fal.queue.result<any>(params.endpointId, {
|
||||
requestId: params.requestId,
|
||||
});
|
||||
return normalizeFalResultToMarkdown({
|
||||
payload: result,
|
||||
sourceImageUrls: params.sourceImageUrls,
|
||||
imageCount: params.imageCount,
|
||||
outputSizeLabel: params.outputSizeLabel,
|
||||
});
|
||||
}
|
||||
|
||||
export function createFalImage({ apiKey, baseURL }: FalImageConfig) {
|
||||
return {
|
||||
provider: "fal-image",
|
||||
@ -453,111 +665,36 @@ export function createFalImage({ apiKey, baseURL }: FalImageConfig) {
|
||||
imageParams?: FalImageParams;
|
||||
overrideModelId?: string;
|
||||
}) => {
|
||||
const { prompt, imageUrls } = extractPromptAndImages(options.messages);
|
||||
const aspectRatio = normalizeAspectRatio(options.imageParams?.aspectRatio);
|
||||
const resolution = normalizeResolution(options.imageParams?.imageSize);
|
||||
const outputFormat = normalizeOutputFormat(options.imageParams?.outputFormat);
|
||||
const safetyTolerance = normalizeSafetyTolerance(options.imageParams?.safetyTolerance);
|
||||
const seed = normalizeSeed(options.imageParams?.seed);
|
||||
const imageCount = Math.max(1, Math.min(6, Number(options.imageParams?.imageCount || 1)));
|
||||
const activeModelId = options.overrideModelId || modelId;
|
||||
const isGptImage2Edit = isFalGptImage2EditModel(activeModelId);
|
||||
const falEndpointId = isGptImage2Edit
|
||||
? activeModelId
|
||||
: normalizeFalEndpointId(activeModelId, imageUrls.length > 0);
|
||||
if (!isGptImage2Edit && falEndpointId.toLowerCase().endsWith("/edit") && imageUrls.length === 0) {
|
||||
throw new Error("FAL 图片编辑模型需要至少一张可访问的参考图片 URL");
|
||||
}
|
||||
const gptImage2EditSize = normalizeGptImage2EditImageSize(
|
||||
options.imageParams?.imageSize,
|
||||
options.imageParams?.aspectRatio
|
||||
);
|
||||
const outputSizeLabel = isGptImage2Edit
|
||||
? getFalImageSizeLabel(gptImage2EditSize)
|
||||
: resolution;
|
||||
const request = buildFalRequestPayload({
|
||||
modelId,
|
||||
overrideModelId: options.overrideModelId,
|
||||
messages: options.messages,
|
||||
imageParams: options.imageParams,
|
||||
});
|
||||
|
||||
console.log("[Fal Image] Submitting request", {
|
||||
model: activeModelId,
|
||||
endpoint: falEndpointId,
|
||||
model: request.activeModelId,
|
||||
endpoint: request.falEndpointId,
|
||||
baseURL,
|
||||
imageCount,
|
||||
aspectRatio,
|
||||
resolution: outputSizeLabel,
|
||||
hasInputImages: imageUrls.length > 0,
|
||||
imageCount: request.imageCount,
|
||||
aspectRatio: request.aspectRatio,
|
||||
resolution: request.outputSizeLabel,
|
||||
hasInputImages: request.imageUrls.length > 0,
|
||||
});
|
||||
|
||||
const { createFalClient } = await import("@fal-ai/client");
|
||||
const fal = createFalClient({
|
||||
credentials: normalizeFalCredentials(apiKey),
|
||||
});
|
||||
|
||||
let input: Record<string, any>;
|
||||
if (isGptImage2Edit) {
|
||||
if (imageUrls.length === 0) {
|
||||
throw new Error("FAL GPT Image 2 Edit 需要至少一张可访问的参考图片 URL");
|
||||
}
|
||||
|
||||
input = {
|
||||
prompt,
|
||||
image_urls: imageUrls,
|
||||
image_size: gptImage2EditSize,
|
||||
quality: normalizeGptImage2EditQuality(options.imageParams?.quality),
|
||||
num_images: imageCount,
|
||||
output_format: outputFormat,
|
||||
};
|
||||
|
||||
const maskImageUrl = normalizeOptionalHttpUrl(options.imageParams?.maskImageUrl);
|
||||
if (maskImageUrl) {
|
||||
input.mask_image_url = maskImageUrl;
|
||||
}
|
||||
|
||||
if (typeof options.imageParams?.syncMode === "boolean") {
|
||||
input.sync_mode = options.imageParams.syncMode;
|
||||
}
|
||||
|
||||
if (seed !== undefined) {
|
||||
input.seed = seed;
|
||||
}
|
||||
|
||||
const openaiApiKey = String(options.imageParams?.openaiApiKey || "").trim();
|
||||
if (openaiApiKey) {
|
||||
input.openai_api_key = openaiApiKey;
|
||||
}
|
||||
} else {
|
||||
input = {
|
||||
prompt,
|
||||
num_images: imageCount,
|
||||
aspect_ratio: aspectRatio,
|
||||
resolution,
|
||||
output_format: outputFormat,
|
||||
safety_tolerance: safetyTolerance,
|
||||
limit_generations: true,
|
||||
};
|
||||
|
||||
if (imageUrls.length > 0) {
|
||||
input.image_urls = imageUrls;
|
||||
}
|
||||
|
||||
if (seed !== undefined) {
|
||||
input.seed = seed;
|
||||
}
|
||||
|
||||
if (typeof options.imageParams?.enableWebSearch === "boolean") {
|
||||
input.enable_web_search = options.imageParams.enableWebSearch;
|
||||
}
|
||||
}
|
||||
const fal = await createFalClientFromApiKey(apiKey);
|
||||
|
||||
let result: any;
|
||||
try {
|
||||
result = await fal.subscribe(falEndpointId, {
|
||||
input,
|
||||
result = await fal.subscribe(request.falEndpointId, {
|
||||
input: request.input,
|
||||
logs: true,
|
||||
timeout: DEFAULT_TIMEOUT_MS,
|
||||
onQueueUpdate: (update: any) => {
|
||||
if (update?.status === "IN_PROGRESS" && Array.isArray(update.logs)) {
|
||||
for (const log of update.logs) {
|
||||
if (log?.message) {
|
||||
console.log(`[Fal Image][${falEndpointId}]`, log.message);
|
||||
console.log(`[Fal Image][${request.falEndpointId}]`, log.message);
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -568,28 +705,12 @@ export function createFalImage({ apiKey, baseURL }: FalImageConfig) {
|
||||
throw error;
|
||||
}
|
||||
|
||||
let outputUrls = filterSourceImages(collectImageUrls(result), imageUrls);
|
||||
outputUrls = await filterSourceImagesByContent(outputUrls, imageUrls);
|
||||
|
||||
if (outputUrls.length === 0 && result?.data) {
|
||||
outputUrls = filterSourceImages(collectImageUrls(result.data), imageUrls);
|
||||
outputUrls = await filterSourceImagesByContent(outputUrls, imageUrls);
|
||||
}
|
||||
|
||||
if (outputUrls.length === 0) {
|
||||
throw new Error("FAL 未返回可用图片");
|
||||
}
|
||||
|
||||
const uploadedImages: UploadedImage[] = [];
|
||||
for (const outputUrl of outputUrls.slice(0, imageCount)) {
|
||||
uploadedImages.push(await downloadAndUploadImage(outputUrl));
|
||||
}
|
||||
|
||||
if (uploadedImages.length === 0) {
|
||||
throw new Error("FAL 未返回可用图片");
|
||||
}
|
||||
|
||||
const text = formatMarkdown(uploadedImages, outputSizeLabel);
|
||||
const text = await normalizeFalResultToMarkdown({
|
||||
payload: result,
|
||||
sourceImageUrls: request.imageUrls,
|
||||
imageCount: request.imageCount,
|
||||
outputSizeLabel: request.outputSizeLabel,
|
||||
});
|
||||
return {
|
||||
text,
|
||||
finishReason: "stop" as const,
|
||||
|
||||
58
lib/ai/runtime/fal-image-edit-task.ts
Normal file
58
lib/ai/runtime/fal-image-edit-task.ts
Normal file
@ -0,0 +1,58 @@
|
||||
export type FalImageEditTaskStatus = "queued" | "running" | "completed" | "failed";
|
||||
|
||||
export interface FalImageEditTaskMetadata {
|
||||
taskKind: "fal_gpt_image_2_edit";
|
||||
provider: "fal-image";
|
||||
providerId: string;
|
||||
endpointId: string;
|
||||
requestId: string;
|
||||
modelId: string;
|
||||
modelRef: string;
|
||||
modelDisplayName: string;
|
||||
prompt: string;
|
||||
optimizedPrompt?: string;
|
||||
userPrompt?: string;
|
||||
requestedImageCount: number;
|
||||
aspectRatio?: string;
|
||||
imageSize?: string;
|
||||
outputSizeLabel: string;
|
||||
sourceImageUrls: string[];
|
||||
sourceAssetIds?: string[];
|
||||
relationType?: "generated_from" | "edited_from";
|
||||
sourceType?: string;
|
||||
projectId?: string | null;
|
||||
contextSource?: string | null;
|
||||
status: FalImageEditTaskStatus;
|
||||
startedAt: number;
|
||||
updatedAt: number;
|
||||
queuePosition?: number | null;
|
||||
lastLogs?: Array<{
|
||||
message?: string;
|
||||
level?: string;
|
||||
timestamp?: string;
|
||||
}>;
|
||||
error?: string | null;
|
||||
}
|
||||
|
||||
export function isFalImageEditTaskMetadata(
|
||||
value: unknown,
|
||||
): value is FalImageEditTaskMetadata {
|
||||
if (!value || typeof value !== "object") return false;
|
||||
const maybe = value as Record<string, unknown>;
|
||||
return (
|
||||
maybe.taskKind === "fal_gpt_image_2_edit" &&
|
||||
maybe.provider === "fal-image" &&
|
||||
typeof maybe.providerId === "string" &&
|
||||
typeof maybe.endpointId === "string" &&
|
||||
typeof maybe.requestId === "string"
|
||||
);
|
||||
}
|
||||
|
||||
export function isPendingFalImageEditTaskMetadata(
|
||||
value: unknown,
|
||||
): value is FalImageEditTaskMetadata {
|
||||
return (
|
||||
isFalImageEditTaskMetadata(value) &&
|
||||
(value.status === "queued" || value.status === "running")
|
||||
);
|
||||
}
|
||||
300
lib/ai/runtime/generated-image-persistence.ts
Normal file
300
lib/ai/runtime/generated-image-persistence.ts
Normal file
@ -0,0 +1,300 @@
|
||||
import { db } from "@/lib/db";
|
||||
import { generatedImage } from "@/lib/db/schema";
|
||||
import {
|
||||
consumeUserPoints,
|
||||
createMediaAssetRelation,
|
||||
getAiModelById,
|
||||
getEnabledAiModelsWithProvider,
|
||||
getMediaAssetsByUserAndUrls,
|
||||
getPointsSystemConfig,
|
||||
getUserById,
|
||||
upsertMediaAsset,
|
||||
} from "@/lib/db/queries";
|
||||
import { persistImageWorkflowTurn } from "@/lib/ai/conversation-state-protocol";
|
||||
import { getVideoCreditCostFromConfig } from "@/lib/ai/video-model-config";
|
||||
|
||||
const MODEL_UUID_REGEX =
|
||||
/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
|
||||
|
||||
export function parseImageAltMetadata(altText: string, thumbnailUrl: string): {
|
||||
originalUrl: string;
|
||||
imageSize: string;
|
||||
} {
|
||||
let originalUrl = thumbnailUrl;
|
||||
let imageSize = "2K";
|
||||
|
||||
if (altText.includes("thumb|||")) {
|
||||
const content = altText.substring(altText.indexOf("thumb|||") + 8);
|
||||
const parts = content.split("|||");
|
||||
originalUrl = parts[0] || thumbnailUrl;
|
||||
imageSize = parts[1] || "2K";
|
||||
} else if (altText.includes("thumb$$")) {
|
||||
const content = altText.substring(altText.indexOf("thumb$$") + 7);
|
||||
const parts = content.split("$$");
|
||||
originalUrl = parts[0] || thumbnailUrl;
|
||||
imageSize = parts[1] || "2K";
|
||||
} else if (altText.includes("thumb|")) {
|
||||
const content = altText.substring(altText.indexOf("thumb|") + 6);
|
||||
const parts = content.split("|");
|
||||
originalUrl = parts[0] || thumbnailUrl;
|
||||
imageSize = parts[1] || "2K";
|
||||
}
|
||||
|
||||
if (originalUrl === thumbnailUrl) {
|
||||
if (thumbnailUrl.includes("gemini-thumb-")) {
|
||||
originalUrl = thumbnailUrl.replace("gemini-thumb-", "gemini-orig-");
|
||||
} else if (thumbnailUrl.includes("openai-thumb-")) {
|
||||
originalUrl = thumbnailUrl.replace("openai-thumb-", "openai-orig-");
|
||||
} else if (thumbnailUrl.includes("volcano-thumb-")) {
|
||||
originalUrl = thumbnailUrl.replace("volcano-thumb-", "volcano-orig-");
|
||||
} else if (thumbnailUrl.includes("bfl-image-thumb-")) {
|
||||
originalUrl = thumbnailUrl.replace("bfl-image-thumb-", "bfl-image-orig-");
|
||||
} else if (thumbnailUrl.includes("alibaba-image-thumb-")) {
|
||||
originalUrl = thumbnailUrl.replace("alibaba-image-thumb-", "alibaba-image-orig-");
|
||||
} else if (thumbnailUrl.includes("ruxa-image-thumb-")) {
|
||||
originalUrl = thumbnailUrl.replace("ruxa-image-thumb-", "ruxa-image-orig-");
|
||||
} else if (thumbnailUrl.includes("fal-image-thumb-")) {
|
||||
originalUrl = thumbnailUrl.replace("fal-image-thumb-", "fal-image-orig-");
|
||||
}
|
||||
}
|
||||
|
||||
return { originalUrl, imageSize };
|
||||
}
|
||||
|
||||
function getModelCreditCost(
|
||||
modelConfig: any,
|
||||
target: "image" | "video",
|
||||
options?: { duration?: number | null; resolution?: string | null },
|
||||
): number {
|
||||
const config = (modelConfig as any)?.imageGenConfig || {};
|
||||
const sourceValue =
|
||||
target === "video"
|
||||
? (() => {
|
||||
const resolvedVideoCost = getVideoCreditCostFromConfig(
|
||||
config,
|
||||
options?.duration,
|
||||
options?.resolution,
|
||||
);
|
||||
if (resolvedVideoCost > 0) return resolvedVideoCost;
|
||||
return config.creditCostPerVideo ?? config.creditCostPerImage ?? 0;
|
||||
})()
|
||||
: config.creditCostPerImage ?? 0;
|
||||
|
||||
const normalized = Number(sourceValue);
|
||||
return Number.isFinite(normalized) ? Math.max(0, Math.round(normalized)) : 0;
|
||||
}
|
||||
|
||||
async function resolveBillingModelByReference(modelRef: string): Promise<any | undefined> {
|
||||
if (!modelRef) return undefined;
|
||||
|
||||
if (MODEL_UUID_REGEX.test(modelRef)) {
|
||||
return await getAiModelById({ id: modelRef }).catch(() => undefined);
|
||||
}
|
||||
|
||||
const allModels = await getEnabledAiModelsWithProvider(true, true).catch(() => []);
|
||||
const matched = allModels.find(
|
||||
({ model }: any) => model.id === modelRef || model.modelId === modelRef,
|
||||
);
|
||||
return matched?.model;
|
||||
}
|
||||
|
||||
export async function saveGeneratedImages(
|
||||
imageMarkdown: string,
|
||||
userId: string,
|
||||
chatId: string,
|
||||
messageId: string,
|
||||
prompt: string,
|
||||
modelId: string,
|
||||
options?: {
|
||||
sourceAssetIds?: string[];
|
||||
sourceImageUrls?: string[];
|
||||
relationType?: "generated_from" | "edited_from";
|
||||
sourceType?: string;
|
||||
projectId?: string | null;
|
||||
userPrompt?: string;
|
||||
optimizedPrompt?: string;
|
||||
requestedImageCount?: number;
|
||||
aspectRatio?: string;
|
||||
imageSize?: string;
|
||||
},
|
||||
): Promise<void> {
|
||||
const imageRegex = /!\[([^\]]*)\]\(([^)]+)\)/g;
|
||||
const matches = [...imageMarkdown.matchAll(imageRegex)];
|
||||
|
||||
let savedImageCount = 0;
|
||||
const savedAssetIds: string[] = [];
|
||||
const savedOriginalUrls: string[] = [];
|
||||
let relationSourceAssetIds = Array.isArray(options?.sourceAssetIds)
|
||||
? options.sourceAssetIds.filter((assetId): assetId is string => Boolean(assetId))
|
||||
: [];
|
||||
|
||||
if (
|
||||
relationSourceAssetIds.length === 0 &&
|
||||
Array.isArray(options?.sourceImageUrls) &&
|
||||
options.sourceImageUrls.length > 0
|
||||
) {
|
||||
try {
|
||||
const sourceAssets = await getMediaAssetsByUserAndUrls({
|
||||
userId,
|
||||
urls: options.sourceImageUrls,
|
||||
});
|
||||
relationSourceAssetIds = sourceAssets.map((asset) => asset.id);
|
||||
} catch (sourceLookupError) {
|
||||
console.error("[Gallery] Failed to resolve source assets by URL:", sourceLookupError);
|
||||
}
|
||||
}
|
||||
|
||||
for (const match of matches) {
|
||||
const altText = match[1] || "";
|
||||
const thumbnailUrl = match[2];
|
||||
const { originalUrl, imageSize } = parseImageAltMetadata(altText, thumbnailUrl);
|
||||
|
||||
try {
|
||||
const [savedImage] = await db
|
||||
.insert(generatedImage)
|
||||
.values({
|
||||
userId,
|
||||
chatId,
|
||||
messageId,
|
||||
thumbnailUrl,
|
||||
originalUrl,
|
||||
imageSize,
|
||||
prompt,
|
||||
modelId,
|
||||
})
|
||||
.returning();
|
||||
|
||||
savedImageCount += 1;
|
||||
if (!savedOriginalUrls.includes(originalUrl)) {
|
||||
savedOriginalUrls.push(originalUrl);
|
||||
}
|
||||
|
||||
const media = await upsertMediaAsset({
|
||||
userId,
|
||||
chatId,
|
||||
projectId: options?.projectId ?? null,
|
||||
mediaType: "image",
|
||||
sourceType: options?.sourceType || "generated",
|
||||
status: "ready",
|
||||
originalUrl,
|
||||
previewUrl: originalUrl,
|
||||
thumbnailUrl,
|
||||
displayName: prompt?.slice(0, 80) || "AI 生成图片",
|
||||
prompt,
|
||||
modelId,
|
||||
sourceMessageId: messageId,
|
||||
sourceTable: "GeneratedImage",
|
||||
sourceRowId: savedImage?.id || null,
|
||||
primaryParentAssetId: relationSourceAssetIds[0] || null,
|
||||
metadata: {
|
||||
imageSize,
|
||||
},
|
||||
});
|
||||
|
||||
if (media?.id && !savedAssetIds.includes(media.id)) {
|
||||
savedAssetIds.push(media.id);
|
||||
}
|
||||
|
||||
if (media && relationSourceAssetIds.length > 0) {
|
||||
for (const sourceAssetId of relationSourceAssetIds) {
|
||||
if (!sourceAssetId) continue;
|
||||
await createMediaAssetRelation({
|
||||
userId,
|
||||
chatId,
|
||||
projectId: options?.projectId ?? null,
|
||||
fromAssetId: sourceAssetId,
|
||||
toAssetId: media.id,
|
||||
relationType: options?.relationType || "generated_from",
|
||||
sourceMessageId: messageId,
|
||||
metadata: {
|
||||
modelId,
|
||||
prompt,
|
||||
sourceImageUrls: options?.sourceImageUrls || [],
|
||||
},
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
console.log(`[Gallery] Saved image: ${originalUrl.substring(0, 80)}...`);
|
||||
} catch (error) {
|
||||
console.error("[Gallery] Failed to save image record:", error);
|
||||
}
|
||||
}
|
||||
|
||||
if (savedOriginalUrls.length > 0) {
|
||||
try {
|
||||
const resultAssetIds = savedAssetIds.length > 0 ? savedAssetIds : null;
|
||||
await persistImageWorkflowTurn({
|
||||
userId,
|
||||
chatId,
|
||||
projectId: options?.projectId || null,
|
||||
currentIntent: options?.relationType === "edited_from" ? "image_edit" : "image_gen",
|
||||
currentFocusAssetIds: resultAssetIds,
|
||||
currentFocusImageUrls: savedOriginalUrls,
|
||||
lastGeneratedAssetIds: resultAssetIds,
|
||||
lastGeneratedImageUrls: savedOriginalUrls,
|
||||
lastResolvedTargetAssetIds: resultAssetIds,
|
||||
lastResolvedTargetImageUrls: savedOriginalUrls,
|
||||
lastMessageId: messageId,
|
||||
lastUserPrompt: String(options?.userPrompt || prompt || "").trim() || prompt,
|
||||
lastOptimizedPrompt:
|
||||
String(
|
||||
options?.optimizedPrompt ||
|
||||
(options?.relationType !== "edited_from" ? prompt : "") ||
|
||||
"",
|
||||
).trim() || undefined,
|
||||
lastRequestedImageCount: options?.requestedImageCount || savedOriginalUrls.length,
|
||||
lastAspectRatio: options?.aspectRatio,
|
||||
lastImageSize: options?.imageSize,
|
||||
lastSourceImageUrls: options?.sourceImageUrls || null,
|
||||
lastResultImageUrls: savedOriginalUrls,
|
||||
});
|
||||
} catch (memoryError) {
|
||||
console.warn("[Gallery] Failed to update working state with generated images:", memoryError);
|
||||
}
|
||||
}
|
||||
|
||||
if (savedImageCount <= 0) return;
|
||||
|
||||
try {
|
||||
const pointsConfig = await getPointsSystemConfig();
|
||||
if (!pointsConfig.enabled) return;
|
||||
|
||||
const userData = await getUserById({ id: userId }).catch(() => undefined);
|
||||
if (userData?.isHighVip) {
|
||||
console.log(`[Points] Skip image points for VIP user=${userId} model=${modelId}`);
|
||||
return;
|
||||
}
|
||||
|
||||
const modelConfig = await resolveBillingModelByReference(modelId);
|
||||
const unitCost = getModelCreditCost(modelConfig, "image");
|
||||
if (unitCost <= 0) return;
|
||||
|
||||
const totalCost = unitCost * savedImageCount;
|
||||
const consumeResult = await consumeUserPoints({
|
||||
userId,
|
||||
amount: totalCost,
|
||||
reason: "image_generation",
|
||||
modelId: modelConfig?.id || modelId,
|
||||
chatId,
|
||||
messageId,
|
||||
note: `模型(${modelConfig?.displayName || modelId}) 生图 ${savedImageCount} 张`,
|
||||
});
|
||||
|
||||
if (!consumeResult.ok) {
|
||||
console.warn(
|
||||
`[Points] Failed to consume points after generation. user=${userId} model=${modelId} cost=${totalCost}`,
|
||||
);
|
||||
} else if (consumeResult.skipped) {
|
||||
console.log(
|
||||
`[Points] Skipped ${totalCost} image points for VIP user=${userId} model=${modelId}`,
|
||||
);
|
||||
} else {
|
||||
console.log(
|
||||
`[Points] Consumed ${totalCost} points (${unitCost} x ${savedImageCount}). balance=${consumeResult.balance}`,
|
||||
);
|
||||
}
|
||||
} catch (error) {
|
||||
console.error("[Points] Failed to consume points after image generation:", error);
|
||||
}
|
||||
}
|
||||
@ -416,7 +416,8 @@ export async function getPendingVideoMessagesByChatId({
|
||||
eq(message.chatId, chatId),
|
||||
eq(message.role, "assistant"),
|
||||
sql`${message.videoMetadata} is not null`,
|
||||
sql`coalesce(${message.videoMetadata} ->> 'status', '') in ('queued', 'running')`
|
||||
sql`coalesce(${message.videoMetadata} ->> 'status', '') in ('queued', 'running')`,
|
||||
sql`coalesce(${message.videoMetadata} ->> 'taskKind', '') <> 'fal_gpt_image_2_edit'`
|
||||
)
|
||||
)
|
||||
.orderBy(desc(message.createdAt));
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user