import { generateText } from "ai"; import { NextResponse } from "next/server"; import { z } from "zod"; import { auth } from "@/app/(auth)/auth"; import { getLanguageModelById } from "@/lib/ai/dynamic-provider"; import { inlineVisionImages } from "@/lib/ai/vision-inline"; import { getAiModelAgentCodingModelId, isAiModelMultimodal, } from "@/lib/db/ai-model-flags"; import { getAgentAiModelWithProvider, getEnabledAiModelsWithProvider, getSystemSetting, } from "@/lib/db/queries"; import { createLogger } from "@/lib/observability/logger"; import { prepareFeaturePointsCharge } from "@/lib/points/feature-charge"; import { VIDEO_CANVAS_FUNCTION_CONFIG_KEY, defaultVideoCanvasFunctionConfig, normalizeVideoCanvasFunctionConfig, } from "@/lib/video-canvas/config"; export const runtime = "nodejs"; export const maxDuration = 90; const logger = createLogger("video-canvas-director-ai-import"); const requestSchema = z.object({ imageUrl: z.string().trim().min(1).max(8_000_000), mode: z.enum(["insert", "overwrite"]), }); type ModelCandidate = Awaited< ReturnType >[number]; const BODY_TYPES = new Set([ "mannequin", "female", "broad", "muscular", "slim", "teen", "child", "chibi", ]); const GEOMETRY_TYPES = new Set([ "box", "sphere", "cylinder", "torus", "cone", "pyramid", ]); function supportsVision(candidate: ModelCandidate) { return ( candidate.model.supportsVision === true || isAiModelMultimodal({ capabilities: candidate.model.capabilities, defaultParams: candidate.model.defaultParams, }) ); } function normalizeModelRef(value: unknown) { const candidate = String(value || "").trim(); return candidate && candidate !== "auto" ? candidate : ""; } function extractJson(value: unknown) { const match = String(value || "").match(/\{[\s\S]*\}/); if (!match) return null; try { const parsed = JSON.parse(match[0]); return parsed && typeof parsed === "object" && !Array.isArray(parsed) ? (parsed as Record) : null; } catch { return null; } } function clampNumber(value: unknown, fallback: number, min: number, max: number) { const parsed = Number(value); return Number.isFinite(parsed) ? Number(Math.min(max, Math.max(min, parsed)).toFixed(4)) : fallback; } function normalizeVector( value: unknown, fallback: [number, number, number], min = -20, max = 20, ): [number, number, number] { if (!Array.isArray(value) || value.length < 3) return fallback; return [ clampNumber(value[0], fallback[0], min, max), clampNumber(value[1], fallback[1], min, max), clampNumber(value[2], fallback[2], min, max), ]; } function normalizeColor(value: unknown, fallback: string) { const candidate = String(value || "").trim(); return /^#[0-9a-f]{6}$/i.test(candidate) ? candidate : fallback; } function normalizeName(value: unknown, fallback: string) { return String(value || fallback).trim().slice(0, 80) || fallback; } function normalizeResult(value: Record) { const rawCharacters = Array.isArray(value.characters) ? value.characters : []; const rawProps = Array.isArray(value.props) ? value.props : []; const rawCameras = Array.isArray(value.cameras) ? value.cameras : []; return { summary: String(value.summary || "").trim().slice(0, 600), characters: rawCharacters.slice(0, 8).flatMap((item, index) => { if (!item || typeof item !== "object" || Array.isArray(item)) return []; const record = item as Record; const bodyType = String(record.bodyType || "mannequin"); return [{ bodyType: BODY_TYPES.has(bodyType) ? bodyType : "mannequin", color: normalizeColor(record.color, ["#7AA7FF", "#E56C5B", "#6CDB7A"][index % 3]!), name: normalizeName(record.name, `角色${String.fromCharCode(65 + index)}`), position: normalizeVector(record.position, [0, 0, index * 1.2]), }]; }), props: rawProps.slice(0, 12).flatMap((item, index) => { if (!item || typeof item !== "object" || Array.isArray(item)) return []; const record = item as Record; const geometryType = String(record.geometryType || "box"); return [{ color: normalizeColor(record.color, "#d7e7ff"), geometryType: GEOMETRY_TYPES.has(geometryType) ? geometryType : "box", name: normalizeName(record.name, `道具${index + 1}`), position: normalizeVector(record.position, [0, 0, 0]), scale: normalizeVector(record.scale, [1, 1, 1], 0.05, 20), }]; }), cameras: rawCameras.slice(0, 4).flatMap((item, index) => { if (!item || typeof item !== "object" || Array.isArray(item)) return []; const record = item as Record; return [{ fov: clampNumber(record.fov, 50, 10, 120), name: normalizeName(record.name, `机位${index + 1}`), position: normalizeVector(record.position, [0, 2.2, 8 + index * 1.5], -50, 50), target: normalizeVector(record.target, [0, 1.2, 0], -50, 50), }]; }), }; } export async function POST(request: Request) { try { const session = await auth(); if (!session?.user?.id) { return NextResponse.json({ success: false, error: "未登录" }, { status: 401 }); } const parsed = requestSchema.safeParse(await request.json()); if (!parsed.success) { return NextResponse.json( { success: false, error: parsed.error.issues[0]?.message || "请求参数无效" }, { status: 400 }, ); } let imageUrl = parsed.data.imageUrl; if (!imageUrl.startsWith("data:image/")) { try { const resolved = new URL(imageUrl, request.url); imageUrl = resolved.protocol === "http:" || resolved.protocol === "https:" ? resolved.toString() : ""; } catch { imageUrl = ""; } } if (!imageUrl) { return NextResponse.json({ success: false, error: "图片地址无效" }, { status: 400 }); } const setting = await getSystemSetting({ key: VIDEO_CANVAS_FUNCTION_CONFIG_KEY }).catch(() => null); const config = setting?.value ? normalizeVideoCanvasFunctionConfig(setting.value) : defaultVideoCanvasFunctionConfig; const aiImportConfig = config.directorDesk.aiImport; const definition = config.functions[aiImportConfig.functionKey]; if (!config.enabled || !aiImportConfig.enabled || !definition?.enabled) { return NextResponse.json({ success: false, error: "AI 识图导入已停用" }, { status: 403 }); } const pointsCharge = await prepareFeaturePointsCharge({ userId: session.user.id, unitCost: aiImportConfig.cost, reason: "director-ai-import", note: "3D导演台 AI 识图导入", }); if (!pointsCharge.ok) { return NextResponse.json(pointsCharge.payload, { status: pointsCharge.status }); } const [models, agentModel] = await Promise.all([ getEnabledAiModelsWithProvider(false), getAgentAiModelWithProvider(), ]); const compatible = models.filter( (candidate) => candidate.model.isEnabled && candidate.provider.isEnabled && !candidate.model.isSubModel && candidate.model.healthStatus !== "auto_disabled" && supportsVision(candidate), ); const preferredRef = normalizeModelRef(definition.model) || normalizeModelRef(config.defaultMultimodalModel); const preferred = preferredRef ? compatible.find( (candidate) => candidate.model.id === preferredRef || candidate.model.modelId === preferredRef, ) : null; const ordered = [ preferred, agentModel && supportsVision(agentModel) ? agentModel : null, ...compatible, ].filter( (candidate, index, items): candidate is ModelCandidate => Boolean(candidate) && items.findIndex((item) => item?.model.id === candidate?.model.id) === index, ); if (ordered.length === 0) { return NextResponse.json({ success: false, error: "当前站点没有可用的多模态模型" }, { status: 500 }); } const referenceImages = imageUrl.startsWith("data:image/") ? [imageUrl] : ( await inlineVisionImages([imageUrl], { maxImages: 1, maxSide: 1280, timeoutMs: 15_000, retries: 1, loggerPrefix: "[Director AI Import]", }) ).inlineDataUrls; if (referenceImages.length === 0) { return NextResponse.json({ success: false, error: "识图素材读取失败" }, { status: 400 }); } for (const candidate of ordered.slice(0, 3)) { try { const model = await getLanguageModelById( candidate.model.id, getAiModelAgentCodingModelId(candidate.model), ); if (!model) continue; const { text } = await generateText({ abortSignal: AbortSignal.timeout(45_000), maxTokens: 2600, messages: [{ role: "user", content: [ { type: "text", text: `${definition.prompt}\n\n导入模式:${parsed.data.mode === "overwrite" ? "覆盖当前导演台" : "插入当前导演台"}\n只输出 JSON,不要 Markdown。`, }, { type: "image", image: referenceImages[0]! }, ], }], model, temperature: 0.15, } as any); const json = extractJson(text); if (!json) continue; const result = normalizeResult(json); await pointsCharge.consume({ modelId: candidate.model.modelId }); return NextResponse.json({ success: true, data: { ...result, model: candidate.model.displayName, }, }); } catch (error) { logger.warn("director AI import model failed", { error, modelId: candidate.model.id, }); } } return NextResponse.json( { success: false, error: "识图模型暂时不可用,请稍后重试" }, { status: 502 }, ); } catch (error) { logger.error("director AI import failed", { error }); return NextResponse.json( { success: false, error: error instanceof Error ? error.message : "识图导入失败" }, { status: 500 }, ); } }