310 lines
10 KiB
TypeScript
310 lines
10 KiB
TypeScript
import { generateText } from "ai";
|
||
import { NextResponse } from "next/server";
|
||
import { z } from "zod";
|
||
import { auth } from "@/app/(auth)/auth";
|
||
import { getLanguageModelById } from "@/lib/ai/dynamic-provider";
|
||
import { inlineVisionImages } from "@/lib/ai/vision-inline";
|
||
import {
|
||
getAiModelAgentCodingModelId,
|
||
isAiModelMultimodal,
|
||
} from "@/lib/db/ai-model-flags";
|
||
import {
|
||
getAgentAiModelWithProvider,
|
||
getEnabledAiModelsWithProvider,
|
||
getSystemSetting,
|
||
} from "@/lib/db/queries";
|
||
import { createLogger } from "@/lib/observability/logger";
|
||
import { prepareFeaturePointsCharge } from "@/lib/points/feature-charge";
|
||
import {
|
||
VIDEO_CANVAS_FUNCTION_CONFIG_KEY,
|
||
defaultVideoCanvasFunctionConfig,
|
||
normalizeVideoCanvasFunctionConfig,
|
||
} from "@/lib/video-canvas/config";
|
||
|
||
export const runtime = "nodejs";
|
||
export const maxDuration = 90;
|
||
|
||
const logger = createLogger("video-canvas-director-ai-import");
|
||
|
||
const requestSchema = z.object({
|
||
imageUrl: z.string().trim().min(1).max(8_000_000),
|
||
mode: z.enum(["insert", "overwrite"]),
|
||
});
|
||
|
||
type ModelCandidate = Awaited<
|
||
ReturnType<typeof getEnabledAiModelsWithProvider>
|
||
>[number];
|
||
|
||
const BODY_TYPES = new Set([
|
||
"mannequin",
|
||
"female",
|
||
"broad",
|
||
"muscular",
|
||
"slim",
|
||
"teen",
|
||
"child",
|
||
"chibi",
|
||
]);
|
||
const GEOMETRY_TYPES = new Set([
|
||
"box",
|
||
"sphere",
|
||
"cylinder",
|
||
"torus",
|
||
"cone",
|
||
"pyramid",
|
||
]);
|
||
|
||
function supportsVision(candidate: ModelCandidate) {
|
||
return (
|
||
candidate.model.supportsVision === true ||
|
||
isAiModelMultimodal({
|
||
capabilities: candidate.model.capabilities,
|
||
defaultParams: candidate.model.defaultParams,
|
||
})
|
||
);
|
||
}
|
||
|
||
function normalizeModelRef(value: unknown) {
|
||
const candidate = String(value || "").trim();
|
||
return candidate && candidate !== "auto" ? candidate : "";
|
||
}
|
||
|
||
function extractJson(value: unknown) {
|
||
const match = String(value || "").match(/\{[\s\S]*\}/);
|
||
if (!match) return null;
|
||
try {
|
||
const parsed = JSON.parse(match[0]);
|
||
return parsed && typeof parsed === "object" && !Array.isArray(parsed)
|
||
? (parsed as Record<string, unknown>)
|
||
: null;
|
||
} catch {
|
||
return null;
|
||
}
|
||
}
|
||
|
||
function clampNumber(value: unknown, fallback: number, min: number, max: number) {
|
||
const parsed = Number(value);
|
||
return Number.isFinite(parsed)
|
||
? Number(Math.min(max, Math.max(min, parsed)).toFixed(4))
|
||
: fallback;
|
||
}
|
||
|
||
function normalizeVector(
|
||
value: unknown,
|
||
fallback: [number, number, number],
|
||
min = -20,
|
||
max = 20,
|
||
): [number, number, number] {
|
||
if (!Array.isArray(value) || value.length < 3) return fallback;
|
||
return [
|
||
clampNumber(value[0], fallback[0], min, max),
|
||
clampNumber(value[1], fallback[1], min, max),
|
||
clampNumber(value[2], fallback[2], min, max),
|
||
];
|
||
}
|
||
|
||
function normalizeColor(value: unknown, fallback: string) {
|
||
const candidate = String(value || "").trim();
|
||
return /^#[0-9a-f]{6}$/i.test(candidate) ? candidate : fallback;
|
||
}
|
||
|
||
function normalizeName(value: unknown, fallback: string) {
|
||
return String(value || fallback).trim().slice(0, 80) || fallback;
|
||
}
|
||
|
||
function normalizeResult(value: Record<string, unknown>) {
|
||
const rawCharacters = Array.isArray(value.characters) ? value.characters : [];
|
||
const rawProps = Array.isArray(value.props) ? value.props : [];
|
||
const rawCameras = Array.isArray(value.cameras) ? value.cameras : [];
|
||
|
||
return {
|
||
summary: String(value.summary || "").trim().slice(0, 600),
|
||
characters: rawCharacters.slice(0, 8).flatMap((item, index) => {
|
||
if (!item || typeof item !== "object" || Array.isArray(item)) return [];
|
||
const record = item as Record<string, unknown>;
|
||
const bodyType = String(record.bodyType || "mannequin");
|
||
return [{
|
||
bodyType: BODY_TYPES.has(bodyType) ? bodyType : "mannequin",
|
||
color: normalizeColor(record.color, ["#7AA7FF", "#E56C5B", "#6CDB7A"][index % 3]!),
|
||
name: normalizeName(record.name, `角色${String.fromCharCode(65 + index)}`),
|
||
position: normalizeVector(record.position, [0, 0, index * 1.2]),
|
||
}];
|
||
}),
|
||
props: rawProps.slice(0, 12).flatMap((item, index) => {
|
||
if (!item || typeof item !== "object" || Array.isArray(item)) return [];
|
||
const record = item as Record<string, unknown>;
|
||
const geometryType = String(record.geometryType || "box");
|
||
return [{
|
||
color: normalizeColor(record.color, "#d7e7ff"),
|
||
geometryType: GEOMETRY_TYPES.has(geometryType) ? geometryType : "box",
|
||
name: normalizeName(record.name, `道具${index + 1}`),
|
||
position: normalizeVector(record.position, [0, 0, 0]),
|
||
scale: normalizeVector(record.scale, [1, 1, 1], 0.05, 20),
|
||
}];
|
||
}),
|
||
cameras: rawCameras.slice(0, 4).flatMap((item, index) => {
|
||
if (!item || typeof item !== "object" || Array.isArray(item)) return [];
|
||
const record = item as Record<string, unknown>;
|
||
return [{
|
||
fov: clampNumber(record.fov, 50, 10, 120),
|
||
name: normalizeName(record.name, `机位${index + 1}`),
|
||
position: normalizeVector(record.position, [0, 2.2, 8 + index * 1.5], -50, 50),
|
||
target: normalizeVector(record.target, [0, 1.2, 0], -50, 50),
|
||
}];
|
||
}),
|
||
};
|
||
}
|
||
|
||
export async function POST(request: Request) {
|
||
try {
|
||
const session = await auth();
|
||
if (!session?.user?.id) {
|
||
return NextResponse.json({ success: false, error: "未登录" }, { status: 401 });
|
||
}
|
||
|
||
const parsed = requestSchema.safeParse(await request.json());
|
||
if (!parsed.success) {
|
||
return NextResponse.json(
|
||
{ success: false, error: parsed.error.issues[0]?.message || "请求参数无效" },
|
||
{ status: 400 },
|
||
);
|
||
}
|
||
let imageUrl = parsed.data.imageUrl;
|
||
if (!imageUrl.startsWith("data:image/")) {
|
||
try {
|
||
const resolved = new URL(imageUrl, request.url);
|
||
imageUrl = resolved.protocol === "http:" || resolved.protocol === "https:"
|
||
? resolved.toString()
|
||
: "";
|
||
} catch {
|
||
imageUrl = "";
|
||
}
|
||
}
|
||
if (!imageUrl) {
|
||
return NextResponse.json({ success: false, error: "图片地址无效" }, { status: 400 });
|
||
}
|
||
|
||
const setting = await getSystemSetting({ key: VIDEO_CANVAS_FUNCTION_CONFIG_KEY }).catch(() => null);
|
||
const config = setting?.value
|
||
? normalizeVideoCanvasFunctionConfig(setting.value)
|
||
: defaultVideoCanvasFunctionConfig;
|
||
const aiImportConfig = config.directorDesk.aiImport;
|
||
const definition = config.functions[aiImportConfig.functionKey];
|
||
if (!config.enabled || !aiImportConfig.enabled || !definition?.enabled) {
|
||
return NextResponse.json({ success: false, error: "AI 识图导入已停用" }, { status: 403 });
|
||
}
|
||
|
||
const pointsCharge = await prepareFeaturePointsCharge({
|
||
userId: session.user.id,
|
||
unitCost: aiImportConfig.cost,
|
||
reason: "director-ai-import",
|
||
note: "3D导演台 AI 识图导入",
|
||
});
|
||
if (!pointsCharge.ok) {
|
||
return NextResponse.json(pointsCharge.payload, { status: pointsCharge.status });
|
||
}
|
||
|
||
const [models, agentModel] = await Promise.all([
|
||
getEnabledAiModelsWithProvider(false),
|
||
getAgentAiModelWithProvider(),
|
||
]);
|
||
const compatible = models.filter(
|
||
(candidate) =>
|
||
candidate.model.isEnabled &&
|
||
candidate.provider.isEnabled &&
|
||
!candidate.model.isSubModel &&
|
||
candidate.model.healthStatus !== "auto_disabled" &&
|
||
supportsVision(candidate),
|
||
);
|
||
const preferredRef =
|
||
normalizeModelRef(definition.model) || normalizeModelRef(config.defaultMultimodalModel);
|
||
const preferred = preferredRef
|
||
? compatible.find(
|
||
(candidate) =>
|
||
candidate.model.id === preferredRef ||
|
||
candidate.model.modelId === preferredRef,
|
||
)
|
||
: null;
|
||
const ordered = [
|
||
preferred,
|
||
agentModel && supportsVision(agentModel) ? agentModel : null,
|
||
...compatible,
|
||
].filter(
|
||
(candidate, index, items): candidate is ModelCandidate =>
|
||
Boolean(candidate) &&
|
||
items.findIndex((item) => item?.model.id === candidate?.model.id) === index,
|
||
);
|
||
if (ordered.length === 0) {
|
||
return NextResponse.json({ success: false, error: "当前站点没有可用的多模态模型" }, { status: 500 });
|
||
}
|
||
|
||
const referenceImages = imageUrl.startsWith("data:image/")
|
||
? [imageUrl]
|
||
: (
|
||
await inlineVisionImages([imageUrl], {
|
||
maxImages: 1,
|
||
maxSide: 1280,
|
||
timeoutMs: 15_000,
|
||
retries: 1,
|
||
loggerPrefix: "[Director AI Import]",
|
||
})
|
||
).inlineDataUrls;
|
||
if (referenceImages.length === 0) {
|
||
return NextResponse.json({ success: false, error: "识图素材读取失败" }, { status: 400 });
|
||
}
|
||
|
||
for (const candidate of ordered.slice(0, 3)) {
|
||
try {
|
||
const model = await getLanguageModelById(
|
||
candidate.model.id,
|
||
getAiModelAgentCodingModelId(candidate.model),
|
||
);
|
||
if (!model) continue;
|
||
const { text } = await generateText({
|
||
abortSignal: AbortSignal.timeout(45_000),
|
||
maxTokens: 2600,
|
||
messages: [{
|
||
role: "user",
|
||
content: [
|
||
{
|
||
type: "text",
|
||
text: `${definition.prompt}\n\n导入模式：${parsed.data.mode === "overwrite" ? "覆盖当前导演台" : "插入当前导演台"}\n只输出 JSON，不要 Markdown。`,
|
||
},
|
||
{ type: "image", image: referenceImages[0]! },
|
||
],
|
||
}],
|
||
model,
|
||
temperature: 0.15,
|
||
} as any);
|
||
const json = extractJson(text);
|
||
if (!json) continue;
|
||
const result = normalizeResult(json);
|
||
await pointsCharge.consume({ modelId: candidate.model.modelId });
|
||
return NextResponse.json({
|
||
success: true,
|
||
data: {
|
||
...result,
|
||
model: candidate.model.displayName,
|
||
},
|
||
});
|
||
} catch (error) {
|
||
logger.warn("director AI import model failed", {
|
||
error,
|
||
modelId: candidate.model.id,
|
||
});
|
||
}
|
||
}
|
||
|
||
return NextResponse.json(
|
||
{ success: false, error: "识图模型暂时不可用，请稍后重试" },
|
||
{ status: 502 },
|
||
);
|
||
} catch (error) {
|
||
logger.error("director AI import failed", { error });
|
||
return NextResponse.json(
|
||
{ success: false, error: error instanceof Error ? error.message : "识图导入失败" },
|
||
{ status: 500 },
|
||
);
|
||
}
|
||
}
|