import { generateText } from "ai"; import { and, eq, inArray } from "drizzle-orm"; import { NextResponse } from "next/server"; import { z } from "zod"; import { auth } from "@/app/(auth)/auth"; import { getLanguageModelById } from "@/lib/ai/dynamic-provider"; import { getCompatibleTextGenerationTemperature } from "@/lib/ai/model-compat"; import { isImageGenerationModelDescriptor, isVideoModelDescriptor, } from "@/lib/ai/model-classification"; import { inlineVisionImages } from "@/lib/ai/vision-inline"; import { db } from "@/lib/db"; import { getAiModelAgentCodingModelId, isAiModelMultimodal, } from "@/lib/db/ai-model-flags"; import { getAgentAiModelWithProvider, getEnabledAiModelsWithProvider, getPointsSystemConfig, getSystemSetting, } from "@/lib/db/queries"; import { mediaAsset } from "@/lib/db/schema"; import { createLogger } from "@/lib/observability/logger"; import { VIDEO_CANVAS_FUNCTION_CONFIG_KEY, defaultVideoCanvasFunctionConfig, normalizeVideoCanvasFunctionConfig, } from "@/lib/video-canvas/config"; import { normalizeGeneratedSubjectDescription, resolveVideoCanvasSubjectCategory, } from "@/lib/video-canvas/subjects"; import { prepareFeaturePointsCharge } from "@/lib/points/feature-charge"; export const runtime = "nodejs"; export const maxDuration = 60; const logger = createLogger("video-canvas-subject-description"); const requestSchema = z.object({ assetIds: z.array(z.string().uuid()).min(1).max(4), category: z.string().trim().max(20).default("无标签"), frameDataUrl: z .string() .max(12_000_000) .refine( (value) => !value || /^data:image\/(?:jpeg|png|webp);base64,/i.test(value), "视频预览帧无效", ) .optional(), }); type ModelCandidate = Awaited< ReturnType >[number]; function isDescriptionModelCandidate(candidate: ModelCandidate) { const descriptor = { capabilities: candidate.model.capabilities, modelId: candidate.model.modelId, providerType: candidate.provider.type, supportsImageGen: candidate.model.supportsImageGen, }; return ( candidate.model.isEnabled && candidate.provider.isEnabled && !candidate.model.isSubModel && candidate.model.healthStatus !== "auto_disabled" && !isImageGenerationModelDescriptor(descriptor) && !isVideoModelDescriptor(descriptor) && (Boolean(candidate.model.supportsVision) || isAiModelMultimodal({ capabilities: candidate.model.capabilities, defaultParams: candidate.model.defaultParams, })) ); } function normalizeModelRef(value: unknown) { const normalized = String(value || "").trim(); return normalized && normalized !== "auto" ? normalized : ""; } export async function POST(request: Request) { try { const session = await auth(); if (!session?.user?.id) { return NextResponse.json({ success: false, error: "未登录" }, { status: 401 }); } const parsed = requestSchema.safeParse(await request.json()); if (!parsed.success) { return NextResponse.json( { success: false, error: parsed.error.issues[0]?.message || "请求参数无效" }, { status: 400 }, ); } const setting = await getSystemSetting({ key: VIDEO_CANVAS_FUNCTION_CONFIG_KEY, }).catch(() => null); const config = setting?.value ? normalizeVideoCanvasFunctionConfig(setting.value) : defaultVideoCanvasFunctionConfig; const subjectConfig = config.subjectLibrary; if (!subjectConfig.enabled || !subjectConfig.smartDescription.enabled) { return NextResponse.json( { success: false, error: "智能生成已停用" }, { status: 403 }, ); } const assets = await db .select({ id: mediaAsset.id, mediaType: mediaAsset.mediaType, originalUrl: mediaAsset.originalUrl, }) .from(mediaAsset) .where( and( eq(mediaAsset.userId, session.user.id), eq(mediaAsset.status, "ready"), inArray(mediaAsset.id, parsed.data.assetIds), ), ); const assetById = new Map(assets.map((asset) => [asset.id, asset])); const orderedAssets = parsed.data.assetIds.flatMap((id) => { const asset = assetById.get(id); return asset ? [asset] : []; }); if (orderedAssets.length !== parsed.data.assetIds.length) { return NextResponse.json( { success: false, error: "主体素材不存在或已失效" }, { status: 400 }, ); } const imageUrls = orderedAssets .filter((asset) => asset.mediaType === "image") .map((asset) => new URL(asset.originalUrl, request.url).toString()); const inlinedImages = imageUrls.length ? ( await inlineVisionImages(imageUrls, { loggerPrefix: "[Video Canvas Subject]", maxImages: subjectConfig.imageMaxCount, maxSide: 1024, retries: 1, timeoutMs: 12_000, }) ).inlineDataUrls : []; const references = [ ...inlinedImages, ...(parsed.data.frameDataUrl ? [parsed.data.frameDataUrl] : []), ].slice(0, subjectConfig.imageMaxCount); if (references.length === 0) { return NextResponse.json( { success: false, error: "未能读取主体画面" }, { status: 400 }, ); } const [allModels, agentModel] = await Promise.all([ getEnabledAiModelsWithProvider(false), getAgentAiModelWithProvider(), ]); const candidates = allModels.filter(isDescriptionModelCandidate); const preferredRef = normalizeModelRef(config.defaultMultimodalModel); const preferredModel = preferredRef ? candidates.find( (candidate) => candidate.model.id === preferredRef || candidate.model.modelId === preferredRef, ) : null; const orderedCandidates = [ preferredModel, agentModel && isDescriptionModelCandidate(agentModel) ? agentModel : null, ...candidates, ].filter( (candidate, index, items): candidate is ModelCandidate => Boolean(candidate) && items.findIndex((item) => item?.model.id === candidate?.model.id) === index, ); if (orderedCandidates.length === 0) { return NextResponse.json( { success: false, error: "当前站点没有可用的多模态模型" }, { status: 500 }, ); } const category = resolveVideoCanvasSubjectCategory( parsed.data.category, subjectConfig, ); const pointsConfig = await getPointsSystemConfig(); const pointsCharge = await prepareFeaturePointsCharge({ modelId: "video-canvas-subject-description", note: "主体描述智能生成消耗积分", pointsConfig, reason: "video_canvas_subject_description", unitCost: subjectConfig.smartDescription.cost, userId: session.user.id, }); if (!pointsCharge.ok) { return NextResponse.json(pointsCharge.payload, { status: pointsCharge.status }); } for (const candidate of orderedCandidates.slice(0, 3)) { try { const model = await getLanguageModelById( candidate.model.id, getAiModelAgentCodingModelId(candidate.model), ); if (!model) continue; const { text } = await generateText({ abortSignal: AbortSignal.timeout(35_000), maxTokens: subjectConfig.smartDescription.maxTokens, messages: [ { role: "user", content: [ { type: "text", text: `主体标签:${category}\n参考画面数量:${references.length}\n请根据系统规则生成主体描述。`, }, ...references.map((image) => ({ type: "image" as const, image })), ], }, ], model, system: subjectConfig.smartDescription.prompt, temperature: getCompatibleTextGenerationTemperature( candidate, subjectConfig.smartDescription.temperature, ), } as any); const description = normalizeGeneratedSubjectDescription( text, subjectConfig.descriptionMaxLength, ); if (!description) continue; await pointsCharge.consume({ modelId: candidate.model.id }).catch((error) => { logger.warn("subject description points consumption failed", { error }); }); return NextResponse.json({ success: true, data: { cost: pointsCharge.unitCost, description, model: candidate.model.displayName, }, }); } catch (error) { logger.warn("subject description model failed", { error, modelId: candidate.model.id, }); } } return NextResponse.json( { success: false, error: "智能描述模型暂时不可用,请稍后重试" }, { status: 502 }, ); } catch (error) { logger.error("subject description generation failed", { error }); return NextResponse.json( { success: false, error: "主体描述生成失败" }, { status: 500 }, ); } }