ai-chatbot/app/api/video-canvas/subjects/generate-description/route.ts

import { generateText } from "ai";
import { and, eq, inArray } from "drizzle-orm";
import { NextResponse } from "next/server";
import { z } from "zod";
import { auth } from "@/app/(auth)/auth";
import { getLanguageModelById } from "@/lib/ai/dynamic-provider";
import { getCompatibleTextGenerationTemperature } from "@/lib/ai/model-compat";
import {
  isImageGenerationModelDescriptor,
  isVideoModelDescriptor,
} from "@/lib/ai/model-classification";
import { inlineVisionImages } from "@/lib/ai/vision-inline";
import { db } from "@/lib/db";
import {
  getAiModelAgentCodingModelId,
  isAiModelMultimodal,
} from "@/lib/db/ai-model-flags";
import {
  getAgentAiModelWithProvider,
  getEnabledAiModelsWithProvider,
  getPointsSystemConfig,
  getSystemSetting,
} from "@/lib/db/queries";
import { mediaAsset } from "@/lib/db/schema";
import { createLogger } from "@/lib/observability/logger";
import {
  VIDEO_CANVAS_FUNCTION_CONFIG_KEY,
  defaultVideoCanvasFunctionConfig,
  normalizeVideoCanvasFunctionConfig,
} from "@/lib/video-canvas/config";
import {
  normalizeGeneratedSubjectDescription,
  resolveVideoCanvasSubjectCategory,
} from "@/lib/video-canvas/subjects";
import { prepareFeaturePointsCharge } from "@/lib/points/feature-charge";

export const runtime = "nodejs";
export const maxDuration = 60;

const logger = createLogger("video-canvas-subject-description");

const requestSchema = z.object({
  assetIds: z.array(z.string().uuid()).min(1).max(4),
  category: z.string().trim().max(20).default("无标签"),
  frameDataUrl: z
    .string()
    .max(12_000_000)
    .refine(
      (value) => !value || /^data:image\/(?:jpeg|png|webp);base64,/i.test(value),
      "视频预览帧无效",
    )
    .optional(),
});

type ModelCandidate = Awaited<
  ReturnType<typeof getEnabledAiModelsWithProvider>
>[number];

function isDescriptionModelCandidate(candidate: ModelCandidate) {
  const descriptor = {
    capabilities: candidate.model.capabilities,
    modelId: candidate.model.modelId,
    providerType: candidate.provider.type,
    supportsImageGen: candidate.model.supportsImageGen,
  };
  return (
    candidate.model.isEnabled &&
    candidate.provider.isEnabled &&
    !candidate.model.isSubModel &&
    candidate.model.healthStatus !== "auto_disabled" &&
    !isImageGenerationModelDescriptor(descriptor) &&
    !isVideoModelDescriptor(descriptor) &&
    (Boolean(candidate.model.supportsVision) ||
      isAiModelMultimodal({
        capabilities: candidate.model.capabilities,
        defaultParams: candidate.model.defaultParams,
      }))
  );
}

function normalizeModelRef(value: unknown) {
  const normalized = String(value || "").trim();
  return normalized && normalized !== "auto" ? normalized : "";
}

export async function POST(request: Request) {
  try {
    const session = await auth();
    if (!session?.user?.id) {
      return NextResponse.json({ success: false, error: "未登录" }, { status: 401 });
    }
    const parsed = requestSchema.safeParse(await request.json());
    if (!parsed.success) {
      return NextResponse.json(
        { success: false, error: parsed.error.issues[0]?.message || "请求参数无效" },
        { status: 400 },
      );
    }

    const setting = await getSystemSetting({
      key: VIDEO_CANVAS_FUNCTION_CONFIG_KEY,
    }).catch(() => null);
    const config = setting?.value
      ? normalizeVideoCanvasFunctionConfig(setting.value)
      : defaultVideoCanvasFunctionConfig;
    const subjectConfig = config.subjectLibrary;
    if (!subjectConfig.enabled || !subjectConfig.smartDescription.enabled) {
      return NextResponse.json(
        { success: false, error: "智能生成已停用" },
        { status: 403 },
      );
    }

    const assets = await db
      .select({
        id: mediaAsset.id,
        mediaType: mediaAsset.mediaType,
        originalUrl: mediaAsset.originalUrl,
      })
      .from(mediaAsset)
      .where(
        and(
          eq(mediaAsset.userId, session.user.id),
          eq(mediaAsset.status, "ready"),
          inArray(mediaAsset.id, parsed.data.assetIds),
        ),
      );
    const assetById = new Map(assets.map((asset) => [asset.id, asset]));
    const orderedAssets = parsed.data.assetIds.flatMap((id) => {
      const asset = assetById.get(id);
      return asset ? [asset] : [];
    });
    if (orderedAssets.length !== parsed.data.assetIds.length) {
      return NextResponse.json(
        { success: false, error: "主体素材不存在或已失效" },
        { status: 400 },
      );
    }

    const imageUrls = orderedAssets
      .filter((asset) => asset.mediaType === "image")
      .map((asset) => new URL(asset.originalUrl, request.url).toString());
    const inlinedImages = imageUrls.length
      ? (
          await inlineVisionImages(imageUrls, {
            loggerPrefix: "[Video Canvas Subject]",
            maxImages: subjectConfig.imageMaxCount,
            maxSide: 1024,
            retries: 1,
            timeoutMs: 12_000,
          })
        ).inlineDataUrls
      : [];
    const references = [
      ...inlinedImages,
      ...(parsed.data.frameDataUrl ? [parsed.data.frameDataUrl] : []),
    ].slice(0, subjectConfig.imageMaxCount);
    if (references.length === 0) {
      return NextResponse.json(
        { success: false, error: "未能读取主体画面" },
        { status: 400 },
      );
    }

    const [allModels, agentModel] = await Promise.all([
      getEnabledAiModelsWithProvider(false),
      getAgentAiModelWithProvider(),
    ]);
    const candidates = allModels.filter(isDescriptionModelCandidate);
    const preferredRef = normalizeModelRef(config.defaultMultimodalModel);
    const preferredModel = preferredRef
      ? candidates.find(
          (candidate) =>
            candidate.model.id === preferredRef ||
            candidate.model.modelId === preferredRef,
        )
      : null;
    const orderedCandidates = [
      preferredModel,
      agentModel && isDescriptionModelCandidate(agentModel) ? agentModel : null,
      ...candidates,
    ].filter(
      (candidate, index, items): candidate is ModelCandidate =>
        Boolean(candidate) &&
        items.findIndex((item) => item?.model.id === candidate?.model.id) === index,
    );
    if (orderedCandidates.length === 0) {
      return NextResponse.json(
        { success: false, error: "当前站点没有可用的多模态模型" },
        { status: 500 },
      );
    }

    const category = resolveVideoCanvasSubjectCategory(
      parsed.data.category,
      subjectConfig,
    );
    const pointsConfig = await getPointsSystemConfig();
    const pointsCharge = await prepareFeaturePointsCharge({
      modelId: "video-canvas-subject-description",
      note: "主体描述智能生成消耗积分",
      pointsConfig,
      reason: "video_canvas_subject_description",
      unitCost: subjectConfig.smartDescription.cost,
      userId: session.user.id,
    });
    if (!pointsCharge.ok) {
      return NextResponse.json(pointsCharge.payload, { status: pointsCharge.status });
    }
    for (const candidate of orderedCandidates.slice(0, 3)) {
      try {
        const model = await getLanguageModelById(
          candidate.model.id,
          getAiModelAgentCodingModelId(candidate.model),
        );
        if (!model) continue;
        const { text } = await generateText({
          abortSignal: AbortSignal.timeout(35_000),
          maxTokens: subjectConfig.smartDescription.maxTokens,
          messages: [
            {
              role: "user",
              content: [
                {
                  type: "text",
                  text: `主体标签：${category}\n参考画面数量：${references.length}\n请根据系统规则生成主体描述。`,
                },
                ...references.map((image) => ({ type: "image" as const, image })),
              ],
            },
          ],
          model,
          system: subjectConfig.smartDescription.prompt,
          temperature: getCompatibleTextGenerationTemperature(
            candidate,
            subjectConfig.smartDescription.temperature,
          ),
        } as any);
        const description = normalizeGeneratedSubjectDescription(
          text,
          subjectConfig.descriptionMaxLength,
        );
        if (!description) continue;
        await pointsCharge.consume({ modelId: candidate.model.id }).catch((error) => {
          logger.warn("subject description points consumption failed", { error });
        });
        return NextResponse.json({
          success: true,
          data: {
            cost: pointsCharge.unitCost,
            description,
            model: candidate.model.displayName,
          },
        });
      } catch (error) {
        logger.warn("subject description model failed", {
          error,
          modelId: candidate.model.id,
        });
      }
    }

    return NextResponse.json(
      { success: false, error: "智能描述模型暂时不可用，请稍后重试" },
      { status: 502 },
    );
  } catch (error) {
    logger.error("subject description generation failed", { error });
    return NextResponse.json(
      { success: false, error: "主体描述生成失败" },
      { status: 500 },
    );
  }
}