ai-chatbot/lib/video-agent/storyboard-subagent.ts

import "server-only";

import type { AgentThreadState } from "@/lib/ai/runtime/agent-thread-state";
import type { AgentExecutionPlan, AgentPlannedToolCall } from "@/lib/ai/auto-agent-graph";
import type { AgentSpecialistRoute } from "@/lib/ai/specialists";
import type { AgentProjectToolDefinition } from "@/lib/ai/tools-registry";
import { getStoryboardPreferencePlanningHints } from "@/lib/video-agent/storyboard-preference-memory";

function normalizeCompactText(text: string) {
  return String(text || "").toLowerCase().replace(/\s+/g, "").trim();
}

export function mentionsStoryboardConcept(text: string): boolean {
  const normalized = normalizeCompactText(text);
  if (!normalized) return false;
  return /分镜|镜头图|镜头草案|镜头预览|storyboard|宫格|四宫格|六宫格|九宫格|拼图视频|分镜视频|高清图/.test(
    normalized
  );
}

export function hasStoryboardWorkflow(threadState?: AgentThreadState | null): boolean {
  return threadState?.workflowState?.type === "storyboard_video";
}

export function buildStoryboardSpecialistRoute(params: {
  text: string;
  threadState?: AgentThreadState | null;
}): AgentSpecialistRoute | null {
  const hasWorkflow = hasStoryboardWorkflow(params.threadState);
  const mentionsStoryboard = mentionsStoryboardConcept(params.text);
  if (!hasWorkflow && !mentionsStoryboard) return null;

  const workflowSummary = params.threadState?.workflowState?.summary;
  const workflowStatus = params.threadState?.workflowState?.status;
  const hasPendingConfirmation = workflowStatus === "awaiting_user_confirmation";
  const preferenceHints = getStoryboardPreferencePlanningHints(params.threadState);

  const planningHints = [
    hasWorkflow
      ? "当前回合与一个 storyboard workflow 强相关，优先理解为分镜流程里的确认、修改、重生或继续执行。"
      : "当前回合明显在处理分镜/宫格任务，不要按普通单图生图或普通视频问答理解。",
    "如果用户提到第N镜头、最后一格、倒数第二张、高清图重生，优先解释为对 storyboard 的局部修改。",
    hasPendingConfirmation
      ? "当前任务仍处于待确认阶段，除非用户明确确认，否则不要推进到视频执行。"
      : "如果用户目标是先做分镜图或宫格图，优先保证输出适合后续切片与连续镜头生成。",
    ...preferenceHints,
    workflowSummary ? `当前 workflow 摘要：${workflowSummary}` : "",
  ].filter(Boolean);

  return {
    domain: "storyboard",
    specialistId: "storyboard-subagent",
    reason: hasWorkflow ? "当前回合命中 storyboard 工作流" : "当前回合命中分镜/宫格任务",
    planningHints,
    toolAllowlist: [
      "respondDirect",
      "generateImage",
      "generateVideo",
      "useSelectedCanvasImage",
      "useLatestResultImage",
      "useOriginalUploadImage",
      "publishToCanvas",
    ],
  };
}

function hasTool(
  availableTools: AgentProjectToolDefinition[],
  name: AgentPlannedToolCall["name"],
) {
  return availableTools.some((tool) => tool.name === name);
}

function upsertTool(
  toolCalls: AgentPlannedToolCall[],
  name: AgentPlannedToolCall["name"],
  reason: string,
  options?: { prepend?: boolean },
) {
  const existing = toolCalls.find((tool) => tool.name === name);
  if (existing) return toolCalls;
  const nextTool = { name, reason } satisfies AgentPlannedToolCall;
  return options?.prepend ? [nextTool, ...toolCalls] : [...toolCalls, nextTool];
}

export function refinePlanWithStoryboardSubagent(params: {
  text: string;
  plan: AgentExecutionPlan;
  availableTools: AgentProjectToolDefinition[];
  threadState?: AgentThreadState | null;
}): AgentExecutionPlan {
  const { text, plan, availableTools, threadState } = params;
  const normalized = normalizeCompactText(text);
  const hasWorkflow = hasStoryboardWorkflow(threadState);
  const status = threadState?.workflowState?.status;
  let toolCalls = [...plan.toolCalls];

  const wantsGridImage =
    /四宫格|六宫格|九宫格|宫格图|分镜图|镜头图|storyboard/.test(normalized);
  const wantsStoryboardVideo =
    /分镜视频|按分镜生成视频|按这个分镜生成|图生视频|镜头视频/.test(normalized);

  if (hasWorkflow && status === "awaiting_user_confirmation") {
    if (hasTool(availableTools, "respondDirect")) {
      toolCalls = [{ name: "respondDirect", reason: "storyboard 子代理要求先处理确认/修改，不直接切换普通工具链" }];
    }
    return {
      ...plan,
      nextAction: "respond",
      intent: "chat",
      responseMode: "direct_answer",
      preferredCapability: "respond",
      shouldUseStructuredReply: false,
      toolCalls,
    };
  }

  if (wantsGridImage && plan.nextAction === "generate_image") {
    if (hasTool(availableTools, "publishToCanvas")) {
      toolCalls = upsertTool(
        toolCalls,
        "publishToCanvas",
        "storyboard 子代理要求把分镜图结果同步到画布，便于后续切片和确认"
      );
    }
    return {
      ...plan,
      toolCalls,
      shouldUseStructuredReply: true,
      preferredCapability: "generate_image",
    };
  }

  if ((wantsStoryboardVideo || hasWorkflow) && plan.nextAction === "generate_video") {
    if (hasTool(availableTools, "useLatestResultImage")) {
      toolCalls = upsertTool(
        toolCalls,
        "useLatestResultImage",
        "storyboard 子代理优先绑定最近的分镜图/镜头预览作为视频输入",
        { prepend: true }
      );
    }
    return {
      ...plan,
      toolCalls,
      shouldUseStructuredReply: true,
      preferredCapability: "generate_video",
    };
  }

  return plan;
}