ai-chatbot/lib/ai/runtime/direct-agent-media-executors.ts

import { generateCanvasImage } from "@/lib/canvas/generate-image";
import {
  executeDirectVideoGeneration,
  type ExecuteDirectVideoGenerationParams,
  type ExecuteDirectVideoGenerationResult,
} from "@/lib/ai/tools/generate-video";
import type {
  AgentExecutionPlan,
  AgentPreparedExecution,
} from "@/lib/ai/auto-agent-graph";
import { findAgentToolCallArgs } from "@/lib/ai/runtime/direct-agent-execution";
import { buildDirectAgentImageAnalysisAnswer } from "@/lib/ai/runtime/direct-agent-analysis-answer";
import { buildAgentToolImagesMarkdown } from "@/lib/ai/runtime/agent-media-markdown";

interface DirectAgentImageParamsSnapshot {
  negativePrompt?: string;
  aspectRatio?: string;
  imageSize?: string;
  imageCount?: number;
}

type DirectAgentVideoMode = ExecuteDirectVideoGenerationParams["mode"];
type DirectAgentVideoRatio = NonNullable<ExecuteDirectVideoGenerationParams["ratio"]>;

export interface DirectAgentImageAnalysisResult {
  summary: string;
  answer: string;
}

export interface ExecuteDirectAgentImageAnalysisParams {
  requestTextContent: string;
  imageUrls: string[];
  context?: string;
  preferredModelId?: string;
  answerModelId?: string;
  assetIds?: string[];
}

export interface DirectAgentImageGenerationResult {
  prompt: string;
  imageMarkdown: string;
  images: Array<{
    originalUrl: string;
    thumbnailUrl?: string | null;
  }>;
  model: string;
  modelRef?: string | null;
  imageSize?: string;
  imageCount: number;
}

export interface ExecuteDirectAgentImageGenerationParams {
  requestTextContent: string;
  optimizedPrompt?: string;
  userId?: string;
  finalModelId: string;
  imageParams?: DirectAgentImageParamsSnapshot | null;
  defaultImageQualityParams?: DirectAgentImageParamsSnapshot | null;
  referenceImageUrls: string[];
}

export interface DirectAgentVideoGenerationResult {
  prompt: string;
  mode: DirectAgentVideoMode;
  images: string[];
  videoUrl: string;
  result: ExecuteDirectVideoGenerationResult;
}

export interface ExecuteDirectAgentVideoGenerationParams {
  requestTextContent: string;
  optimizedPrompt?: string;
  finalModelId: string;
  sourceImageUrls: string[];
  videoParams?: {
    mode?: string;
    resolution?: ExecuteDirectVideoGenerationParams["resolution"];
    ratio?: string;
    duration?: number;
  } | null;
  smartAutoExecution?: AgentPreparedExecution;
  smartAutoPlan?: AgentExecutionPlan;
}

function normalizeDirectAgentVideoMode(value: unknown): DirectAgentVideoMode | undefined {
  const normalized = String(value || "").trim();
  switch (normalized) {
    case "text-to-video":
    case "first-frame":
    case "first-last-frame":
    case "reference-images":
      return normalized;
    default:
      return undefined;
  }
}

function normalizeDirectAgentVideoRatio(value: unknown): DirectAgentVideoRatio | undefined {
  const normalized = String(value || "").trim();
  switch (normalized) {
    case "16:9":
    case "4:3":
    case "1:1":
    case "3:4":
    case "9:16":
    case "21:9":
    case "adaptive":
      return normalized;
    default:
      return undefined;
  }
}

function inferDirectAgentVideoMode(imageUrls: string[]): DirectAgentVideoMode {
  if (imageUrls.length >= 2) return "first-last-frame";
  if (imageUrls.length >= 1) return "first-frame";
  return "text-to-video";
}

function pickDirectAgentVideoImages(mode: DirectAgentVideoMode, imageUrls: string[]): string[] {
  switch (mode) {
    case "first-last-frame":
      return imageUrls.slice(0, 2);
    case "first-frame":
      return imageUrls.slice(0, 1);
    case "reference-images":
      return imageUrls.slice(0, 4);
    case "text-to-video":
    default:
      return [];
  }
}

export async function executeDirectAgentImageAnalysis(
  params: ExecuteDirectAgentImageAnalysisParams,
): Promise<DirectAgentImageAnalysisResult> {
  const { summarizeImageForExecutionReport } = await import("@/lib/ai/smart-analyzer");
  const summary = await summarizeImageForExecutionReport({
    text: params.requestTextContent,
    imageUrls: params.imageUrls,
    context: params.context,
    preferredModelId: params.preferredModelId,
    assetIds: params.assetIds,
  });

  if (!summary?.trim()) {
    throw new Error("当前图片暂时无法提取稳定的视觉摘要，请稍后再试。");
  }

  const normalizedSummary = summary.trim();
  const answer = await buildDirectAgentImageAnalysisAnswer({
    question: params.requestTextContent,
    summary: normalizedSummary,
    modelId: params.answerModelId,
  });

  return {
    summary: normalizedSummary,
    answer,
  };
}

export async function executeDirectAgentImageGeneration(
  params: ExecuteDirectAgentImageGenerationParams,
): Promise<DirectAgentImageGenerationResult> {
  const prompt =
    String(params.optimizedPrompt || params.requestTextContent || "").trim() || "生成图片";
  const imageSize =
    params.imageParams?.imageSize || params.defaultImageQualityParams?.imageSize;
  const imageCount =
    params.imageParams?.imageCount || params.defaultImageQualityParams?.imageCount;
  const shouldIgnoreModelMaxImages =
    typeof imageCount === "number" && Number.isFinite(imageCount) && imageCount > 1;
  const result = await generateCanvasImage({
    prompt,
    negativePrompt: params.imageParams?.negativePrompt,
    model: params.finalModelId,
    ownerKey: params.userId,
    debugLabel: "direct-agent-image-generation",
    debugMeta: {
      agentIgnoreModelMaxImages: shouldIgnoreModelMaxImages,
      requestedAgentImageCount: imageCount ?? null,
    },
    aspectRatio:
      params.imageParams?.aspectRatio || params.defaultImageQualityParams?.aspectRatio,
    imageSize,
    imageCount,
    ignoreModelMaxImages: shouldIgnoreModelMaxImages,
    referenceImages: params.referenceImageUrls,
  });

  const imageMarkdown = buildAgentToolImagesMarkdown(
    result.images.map((image) => ({
      originalUrl: image.originalUrl,
      thumbnailUrl: image.thumbnailUrl,
      imageSize,
    })),
  );

  return {
    prompt,
    imageMarkdown,
    images: result.images,
    model: result.model,
    modelRef: result.modelRef || null,
    imageSize,
    imageCount: result.images.length,
  };
}

export async function executeDirectAgentVideoGeneration(
  params: ExecuteDirectAgentVideoGenerationParams,
): Promise<DirectAgentVideoGenerationResult> {
  const prompt =
    String(params.optimizedPrompt || params.requestTextContent || "").trim() || "生成视频";
  const plannedArgs = findAgentToolCallArgs(
    "generateVideo",
    params.smartAutoExecution,
    params.smartAutoPlan,
  );
  const mode =
    normalizeDirectAgentVideoMode(plannedArgs?.mode) ||
    normalizeDirectAgentVideoMode(params.videoParams?.mode) ||
    inferDirectAgentVideoMode(params.sourceImageUrls);
  const images = pickDirectAgentVideoImages(mode, params.sourceImageUrls);
  const result = await executeDirectVideoGeneration({
    prompt,
    mode,
    images,
    resolution: params.videoParams?.resolution,
    ratio: normalizeDirectAgentVideoRatio(params.videoParams?.ratio),
    duration: params.videoParams?.duration,
    preferredModelRef: params.finalModelId,
  });

  const videoUrl = String(result.videoUrl || "").trim();
  if (!videoUrl) {
    throw new Error("视频生成完成但未返回视频地址");
  }

  return {
    prompt,
    mode,
    images,
    videoUrl,
    result,
  };
}