421 lines
12 KiB
TypeScript
421 lines
12 KiB
TypeScript
import "server-only";
|
||
|
||
import { generateCanvasImage } from "@/lib/canvas/generate-image";
|
||
import { generateUUID } from "@/lib/utils";
|
||
import {
|
||
createVideoStoryboardShotTasks,
|
||
createVideoStoryboardTask,
|
||
markVideoStoryboardAwaitingConfirmation,
|
||
markVideoStoryboardHdCompleted,
|
||
markVideoStoryboardHdFailed,
|
||
markVideoStoryboardHdRunning,
|
||
syncVideoStoryboardHdStageStatus,
|
||
} from "@/lib/video-agent/task-store";
|
||
import { runNineGridDetectorSkill } from "@/lib/video-agent/skills/nine-grid-detector";
|
||
import { runScriptNormalizerSkill } from "@/lib/video-agent/skills/script-normalizer";
|
||
import { runStoryboardMapperSkill } from "@/lib/video-agent/skills/storyboard-mapper";
|
||
import { runTileUpscalerSkill } from "@/lib/video-agent/skills/tile-upscaler";
|
||
import type {
|
||
RuxaNineGridPipelineParams,
|
||
RuxaNineGridPipelineResult,
|
||
StoryboardGridType,
|
||
StoryboardModelCandidate,
|
||
} from "@/lib/video-agent/types";
|
||
|
||
function normalizeText(value: unknown) {
|
||
return String(value || "").trim();
|
||
}
|
||
|
||
function extractPromptAndImages(messages: Array<{ role: string; content: any }>) {
|
||
const lastUserMessage = messages.filter((message) => message.role === "user").pop();
|
||
if (!lastUserMessage) {
|
||
return { prompt: "", imageUrls: [] as string[] };
|
||
}
|
||
|
||
let prompt = "";
|
||
const imageUrls: string[] = [];
|
||
|
||
const pushImage = (raw?: string) => {
|
||
const url = normalizeText(raw);
|
||
if (!url || !/^https?:\/\//i.test(url)) return;
|
||
if (!imageUrls.includes(url)) imageUrls.push(url);
|
||
};
|
||
|
||
if (typeof lastUserMessage.content === "string") {
|
||
prompt = normalizeText(lastUserMessage.content);
|
||
} else if (Array.isArray(lastUserMessage.content)) {
|
||
for (const part of lastUserMessage.content) {
|
||
if (part?.type === "text" && typeof part.text === "string") {
|
||
prompt = `${prompt} ${part.text}`.trim();
|
||
} else if (part?.type === "image_url" && part.image_url?.url) {
|
||
pushImage(part.image_url.url);
|
||
} else if (part?.type === "file") {
|
||
pushImage(part.url || part.data);
|
||
} else if (part?.type === "image" && typeof part.image === "string") {
|
||
pushImage(part.image);
|
||
}
|
||
}
|
||
}
|
||
|
||
if (!prompt) {
|
||
prompt = "让画面动起来";
|
||
}
|
||
|
||
return { prompt, imageUrls };
|
||
}
|
||
|
||
function normalizeModelCandidates(params: {
|
||
selectedModelId: string;
|
||
selectedModelRef: string;
|
||
modelCandidates: StoryboardModelCandidate[];
|
||
}) {
|
||
const selected: StoryboardModelCandidate = {
|
||
id: params.selectedModelRef || params.selectedModelId,
|
||
modelId: params.selectedModelId,
|
||
displayName: "",
|
||
};
|
||
const candidates = [selected, ...params.modelCandidates]
|
||
.filter((item) => normalizeText(item.modelId))
|
||
.map((item) => ({
|
||
id: normalizeText(item.id) || normalizeText(item.modelId),
|
||
modelId: normalizeText(item.modelId),
|
||
displayName: normalizeText(item.displayName),
|
||
}));
|
||
|
||
const seen = new Set<string>();
|
||
return candidates.filter((item) => {
|
||
if (seen.has(item.modelId)) return false;
|
||
seen.add(item.modelId);
|
||
return true;
|
||
});
|
||
}
|
||
|
||
function buildAwaitingConfirmationMarkdown(params: {
|
||
shots: Array<{
|
||
shotIndex: number;
|
||
prompt: string;
|
||
}>;
|
||
}) {
|
||
const lines = [
|
||
"### 分镜草案已准备完成",
|
||
"我已经根据当前宫格切片生成了每段镜头的预览图，请先确认镜头内容。",
|
||
"",
|
||
...params.shots.map(
|
||
(shot) => `- 镜头 ${shot.shotIndex}：${normalizeText(shot.prompt) || "待补充分镜文案"}`
|
||
),
|
||
"",
|
||
"如果需要修改，请直接回复：",
|
||
"- 修改第3镜头：……",
|
||
"- 调整第5镜头动作：……",
|
||
"- 重生第2张高清图",
|
||
"",
|
||
"如果确认无误，请回复：",
|
||
"- 确认开始生成视频",
|
||
];
|
||
return lines.join("\n").trim();
|
||
}
|
||
|
||
function inferPreviewAspectRatio(width?: number, height?: number): string {
|
||
const safeWidth = Math.max(1, Number(width || 0));
|
||
const safeHeight = Math.max(1, Number(height || 0));
|
||
const ratio = safeWidth / safeHeight;
|
||
if (ratio > 1.35) return "16:9";
|
||
if (ratio < 0.8) return "9:16";
|
||
return "1:1";
|
||
}
|
||
|
||
function getGridLabel(gridType: StoryboardGridType | null, shotCount: number) {
|
||
switch (gridType) {
|
||
case "grid_2x2":
|
||
return "4宫格";
|
||
case "grid_3x2":
|
||
return "6宫格";
|
||
case "grid_3x3":
|
||
return "9宫格";
|
||
default:
|
||
return `${shotCount}宫格`;
|
||
}
|
||
}
|
||
|
||
export async function runRuxaNineGridPipeline(
|
||
params: RuxaNineGridPipelineParams
|
||
): Promise<RuxaNineGridPipelineResult> {
|
||
const {
|
||
userId,
|
||
chatId,
|
||
modelMessages,
|
||
selectedModelId,
|
||
selectedModelRef,
|
||
modelCandidates,
|
||
videoParams,
|
||
config,
|
||
onProgress,
|
||
} = params;
|
||
|
||
if (!config.enabled) {
|
||
return { triggered: false, reason: "disabled" };
|
||
}
|
||
|
||
const providerAllowed = config.providerWhitelist.some(
|
||
(item) => normalizeText(item).toLowerCase() === "ruxa-video"
|
||
);
|
||
if (!providerAllowed) {
|
||
return { triggered: false, reason: "provider_not_allowed" };
|
||
}
|
||
|
||
onProgress?.({
|
||
stage: "detecting",
|
||
provider: "ruxa-video",
|
||
message: "正在识别分镜拼图与解析分镜...",
|
||
});
|
||
|
||
const { prompt, imageUrls } = extractPromptAndImages(modelMessages);
|
||
if (!imageUrls[0]) {
|
||
return { triggered: false, reason: "missing_image" };
|
||
}
|
||
|
||
const detection = await runNineGridDetectorSkill({
|
||
imageUrl: imageUrls[0],
|
||
minConfidence: config.minGridConfidence,
|
||
});
|
||
|
||
if (!detection.isStoryboardGrid || detection.tiles.length !== detection.shotCount) {
|
||
return {
|
||
triggered: false,
|
||
reason: detection.isStoryboardGrid ? "slice_incomplete" : "not_storyboard_grid",
|
||
confidence: detection.confidence,
|
||
};
|
||
}
|
||
|
||
const normalizedScript = runScriptNormalizerSkill({
|
||
scriptText: prompt,
|
||
maxShots: detection.shotCount,
|
||
totalDurationSec: Number(videoParams?.duration || 10),
|
||
});
|
||
|
||
const mappedShots = runStoryboardMapperSkill({
|
||
script: normalizedScript,
|
||
tiles: detection.tiles,
|
||
});
|
||
|
||
const preparedShots = await runTileUpscalerSkill({
|
||
shots: mappedShots,
|
||
enableTileUpscale: config.enableTileUpscale,
|
||
sliceUpscaleMinEdge: config.sliceUpscaleMinEdge,
|
||
});
|
||
|
||
const tileSummary = detection.tiles.map((tile) => ({
|
||
index: tile.index,
|
||
imageUrl: tile.imageUrl,
|
||
width: tile.width,
|
||
height: tile.height,
|
||
}));
|
||
const tileMetaByIndex = new Map(
|
||
detection.tiles.map((tile) => [
|
||
tile.index,
|
||
{
|
||
width: tile.width,
|
||
height: tile.height,
|
||
},
|
||
])
|
||
);
|
||
|
||
const shotPlanSummary = preparedShots.map((shot) => ({
|
||
shotIndex: shot.shotIndex,
|
||
tileIndex: shot.tileIndex,
|
||
tileImageUrl: shot.inputImageUrl,
|
||
prompt: shot.prompt,
|
||
action: shot.action,
|
||
camera: shot.camera,
|
||
mood: shot.mood,
|
||
durationSec: shot.durationSec,
|
||
}));
|
||
|
||
const resolvedModelCandidates = normalizeModelCandidates({
|
||
selectedModelId,
|
||
selectedModelRef,
|
||
modelCandidates,
|
||
});
|
||
const fallbackChain = resolvedModelCandidates
|
||
.map((item) => normalizeText(item.modelId))
|
||
.filter(Boolean);
|
||
|
||
const createdTask = await createVideoStoryboardTask({
|
||
userId,
|
||
chatId,
|
||
providerType: "ruxa-video",
|
||
modelRef: selectedModelRef,
|
||
gridType: detection.gridType,
|
||
gridRows: detection.rows,
|
||
gridCols: detection.cols,
|
||
sourceImageUrl: imageUrls[0],
|
||
isNineGrid: detection.gridType === "grid_3x3",
|
||
detectionConfidence: detection.confidence,
|
||
normalizedScript: normalizedScript as any,
|
||
totalShots: preparedShots.length,
|
||
status: "running",
|
||
});
|
||
|
||
const taskId = createdTask.id || `ruxa-storyboard-${generateUUID()}`;
|
||
|
||
await createVideoStoryboardShotTasks({
|
||
parentTaskId: taskId,
|
||
shots: preparedShots,
|
||
fallbackChain,
|
||
});
|
||
|
||
await onProgress?.({
|
||
stage: "start",
|
||
provider: "ruxa-video",
|
||
taskId,
|
||
totalShots: preparedShots.length,
|
||
gridType: detection.gridType || undefined,
|
||
rows: detection.rows,
|
||
cols: detection.cols,
|
||
confidence: detection.confidence,
|
||
sourceImageUrl: imageUrls[0],
|
||
shots: preparedShots.map((shot) => ({
|
||
shotIndex: shot.shotIndex,
|
||
tileImageUrl: shot.inputImageUrl,
|
||
prompt: shot.prompt,
|
||
})),
|
||
});
|
||
|
||
const previewShots: Array<{
|
||
shotIndex: number;
|
||
tileImageUrl: string;
|
||
previewImageUrl: string;
|
||
prompt: string;
|
||
fallbackUsed: boolean;
|
||
}> = [];
|
||
|
||
for (const shot of preparedShots) {
|
||
await markVideoStoryboardHdRunning({
|
||
taskId,
|
||
shotIndex: shot.shotIndex,
|
||
}).catch(() => undefined);
|
||
|
||
try {
|
||
const tileMeta = tileMetaByIndex.get(shot.tileIndex);
|
||
const hdResult = await generateCanvasImage({
|
||
prompt: shot.prompt,
|
||
referenceImage: shot.inputImageUrl,
|
||
imageCount: 1,
|
||
imageSize: "2K",
|
||
aspectRatio: inferPreviewAspectRatio(tileMeta?.width, tileMeta?.height),
|
||
});
|
||
|
||
const previewImageUrl =
|
||
hdResult.images[0]?.originalUrl || hdResult.images[0]?.thumbnailUrl || shot.inputImageUrl;
|
||
|
||
await markVideoStoryboardHdCompleted({
|
||
taskId,
|
||
shotIndex: shot.shotIndex,
|
||
hdImageUrl: previewImageUrl,
|
||
hdImagePrompt: shot.prompt,
|
||
}).catch(() => undefined);
|
||
|
||
previewShots.push({
|
||
shotIndex: shot.shotIndex,
|
||
tileImageUrl: shot.tileImageUrl,
|
||
previewImageUrl,
|
||
prompt: shot.prompt,
|
||
fallbackUsed: false,
|
||
});
|
||
|
||
await onProgress?.({
|
||
stage: "hd_shot_completed",
|
||
provider: "ruxa-video",
|
||
taskId,
|
||
shotIndex: shot.shotIndex,
|
||
imageUrl: previewImageUrl,
|
||
prompt: shot.prompt,
|
||
});
|
||
} catch (error) {
|
||
const errorMessage =
|
||
normalizeText(error instanceof Error ? error.message : String(error)) ||
|
||
"hd_image_generation_failed";
|
||
|
||
await markVideoStoryboardHdFailed({
|
||
taskId,
|
||
shotIndex: shot.shotIndex,
|
||
errorMessage,
|
||
}).catch(() => undefined);
|
||
|
||
previewShots.push({
|
||
shotIndex: shot.shotIndex,
|
||
tileImageUrl: shot.tileImageUrl,
|
||
previewImageUrl: shot.inputImageUrl,
|
||
prompt: shot.prompt,
|
||
fallbackUsed: true,
|
||
});
|
||
|
||
await onProgress?.({
|
||
stage: "hd_shot_failed",
|
||
provider: "ruxa-video",
|
||
taskId,
|
||
shotIndex: shot.shotIndex,
|
||
error: errorMessage,
|
||
});
|
||
}
|
||
}
|
||
|
||
await syncVideoStoryboardHdStageStatus(taskId).catch(() => undefined);
|
||
await markVideoStoryboardAwaitingConfirmation({
|
||
taskId,
|
||
}).catch(() => undefined);
|
||
|
||
const awaitingConfirmationMessage = buildAwaitingConfirmationMarkdown({
|
||
shots: previewShots.map((shot) => ({
|
||
shotIndex: shot.shotIndex,
|
||
prompt: shot.prompt,
|
||
})),
|
||
}).replace(
|
||
"我已经根据当前宫格切片生成了每段镜头的预览图，请先确认镜头内容。",
|
||
`我已经根据当前${getGridLabel(detection.gridType, detection.shotCount)}切片生成了每段镜头的预览图，请先确认镜头内容。`
|
||
);
|
||
|
||
await onProgress?.({
|
||
stage: "awaiting_confirmation",
|
||
provider: "ruxa-video",
|
||
taskId,
|
||
totalShots: previewShots.length,
|
||
gridType: detection.gridType || undefined,
|
||
rows: detection.rows,
|
||
cols: detection.cols,
|
||
message: awaitingConfirmationMessage,
|
||
shots: previewShots.map((shot) => ({
|
||
shotIndex: shot.shotIndex,
|
||
tileImageUrl: shot.tileImageUrl,
|
||
previewImageUrl: shot.previewImageUrl,
|
||
prompt: shot.prompt,
|
||
})),
|
||
});
|
||
|
||
return {
|
||
triggered: true,
|
||
taskId,
|
||
confidence: detection.confidence,
|
||
markdown: awaitingConfirmationMessage,
|
||
videoUrls: [],
|
||
previewImageUrls: previewShots.map((shot) => shot.previewImageUrl),
|
||
modelRefs: [],
|
||
metadata: {
|
||
taskId,
|
||
confidence: detection.confidence,
|
||
qualityScore: detection.qualityScore,
|
||
sourceImageUrl: detection.sourceImageUrl,
|
||
gridType: detection.gridType,
|
||
rows: detection.rows,
|
||
cols: detection.cols,
|
||
shotCount: detection.shotCount,
|
||
tiles: tileSummary,
|
||
shotPlan: shotPlanSummary,
|
||
normalizedScript,
|
||
awaitingConfirmation: true,
|
||
previewShots,
|
||
},
|
||
reason: "awaiting_user_confirmation",
|
||
};
|
||
}
|