ai-chatbot/lib/video-canvas/subtitle-erase.ts

export type SubtitleEraseMode = "smart" | "region";

export type SubtitleEraseRegion = {
  height: number;
  id: string;
  width: number;
  x: number;
  y: number;
};

export type SubtitleEraseRegionHandle = "nw" | "ne" | "se" | "sw";

export type VideoCanvasSubtitleEraseConfig = {
  blurRadius: number;
  edgeThreshold: number;
  enabled: boolean;
  frameRate: number;
  minBandHeightRatio: number;
  model: string;
  paddingRatio: number;
  smartSearchEndRatio: number;
  smartSearchStartRatio: number;
};

export const defaultVideoCanvasSubtitleEraseConfig: VideoCanvasSubtitleEraseConfig = {
  blurRadius: 10,
  edgeThreshold: 44,
  enabled: true,
  frameRate: 30,
  minBandHeightRatio: 0.018,
  model: "volcano-subtitle-eraser",
  paddingRatio: 0.012,
  smartSearchEndRatio: 0.96,
  smartSearchStartRatio: 0.5,
};

function clamp(value: number, min: number, max: number) {
  return Math.min(max, Math.max(min, value));
}

function finiteNumber(value: unknown, fallback: number, min: number, max: number) {
  const parsed = Number(value);
  return Number.isFinite(parsed) ? clamp(parsed, min, max) : fallback;
}

export function normalizeVideoCanvasSubtitleEraseConfig(
  value: unknown,
): VideoCanvasSubtitleEraseConfig {
  const incoming = value && typeof value === "object" && !Array.isArray(value)
    ? value as Record<string, unknown>
    : {};
  const start = finiteNumber(
    incoming.smartSearchStartRatio,
    defaultVideoCanvasSubtitleEraseConfig.smartSearchStartRatio,
    0,
    0.94,
  );
  const end = finiteNumber(
    incoming.smartSearchEndRatio,
    defaultVideoCanvasSubtitleEraseConfig.smartSearchEndRatio,
    start + 0.04,
    1,
  );
  return {
    blurRadius: finiteNumber(
      incoming.blurRadius,
      defaultVideoCanvasSubtitleEraseConfig.blurRadius,
      0,
      40,
    ),
    edgeThreshold: finiteNumber(
      incoming.edgeThreshold,
      defaultVideoCanvasSubtitleEraseConfig.edgeThreshold,
      1,
      255,
    ),
    enabled:
      typeof incoming.enabled === "boolean"
        ? incoming.enabled
        : defaultVideoCanvasSubtitleEraseConfig.enabled,
    frameRate: Math.round(finiteNumber(
      incoming.frameRate,
      defaultVideoCanvasSubtitleEraseConfig.frameRate,
      12,
      60,
    )),
    minBandHeightRatio: finiteNumber(
      incoming.minBandHeightRatio,
      defaultVideoCanvasSubtitleEraseConfig.minBandHeightRatio,
      0.005,
      0.2,
    ),
    model:
      typeof incoming.model === "string" && incoming.model.trim()
        ? incoming.model.trim().slice(0, 100)
        : defaultVideoCanvasSubtitleEraseConfig.model,
    paddingRatio: finiteNumber(
      incoming.paddingRatio,
      defaultVideoCanvasSubtitleEraseConfig.paddingRatio,
      0,
      0.1,
    ),
    smartSearchEndRatio: Number(Math.max(start + 0.04, end).toFixed(4)),
    smartSearchStartRatio: start,
  };
}

export function normalizeSubtitleEraseRegion(
  region: SubtitleEraseRegion,
): SubtitleEraseRegion {
  const x = clamp(Number(region.x) || 0, 0, 1);
  const y = clamp(Number(region.y) || 0, 0, 1);
  const normalizedWidth = clamp(Number(region.width) || 0, 0, 1 - x);
  const normalizedHeight = clamp(Number(region.height) || 0, 0, 1 - y);
  return {
    height: Number(normalizedHeight.toFixed(6)),
    id: region.id,
    width: Number(normalizedWidth.toFixed(6)),
    x: Number(x.toFixed(6)),
    y: Number(y.toFixed(6)),
  };
}

export function moveSubtitleEraseRegion(
  region: SubtitleEraseRegion,
  deltaX: number,
  deltaY: number,
) {
  const normalized = normalizeSubtitleEraseRegion(region);
  return {
    ...normalized,
    x: clamp(normalized.x + deltaX, 0, 1 - normalized.width),
    y: clamp(normalized.y + deltaY, 0, 1 - normalized.height),
  };
}

export function resizeSubtitleEraseRegion(
  region: SubtitleEraseRegion,
  handle: SubtitleEraseRegionHandle,
  pointX: number,
  pointY: number,
  minimumSize = 0.015,
) {
  const normalized = normalizeSubtitleEraseRegion(region);
  let left = normalized.x;
  let top = normalized.y;
  let right = normalized.x + normalized.width;
  let bottom = normalized.y + normalized.height;
  const x = clamp(pointX, 0, 1);
  const y = clamp(pointY, 0, 1);

  if (handle.includes("w")) left = Math.min(x, right - minimumSize);
  if (handle.includes("e")) right = Math.max(x, left + minimumSize);
  if (handle.includes("n")) top = Math.min(y, bottom - minimumSize);
  if (handle.includes("s")) bottom = Math.max(y, top + minimumSize);

  return normalizeSubtitleEraseRegion({
    height: bottom - top,
    id: normalized.id,
    width: right - left,
    x: left,
    y: top,
  });
}

function pixelLuma(data: Uint8ClampedArray, index: number) {
  return data[index] * 0.299 + data[index + 1] * 0.587 + data[index + 2] * 0.114;
}

export function detectSubtitleEraseRegions(
  imageData: ImageData,
  configValue: VideoCanvasSubtitleEraseConfig,
): SubtitleEraseRegion[] {
  const config = normalizeVideoCanvasSubtitleEraseConfig(configValue);
  const { data, height, width } = imageData;
  if (width < 8 || height < 8) return [];
  const startRow = Math.floor(height * config.smartSearchStartRatio);
  const endRow = Math.max(startRow + 1, Math.ceil(height * config.smartSearchEndRatio));
  const rowScores = Array.from({ length: endRow - startRow }, () => 0);
  const stride = width > 900 ? 3 : 2;

  for (let y = startRow; y < Math.min(height - 1, endRow); y += 1) {
    let edges = 0;
    let samples = 0;
    for (let x = 1; x < width - 1; x += stride) {
      const index = (y * width + x) * 4;
      const horizontal = Math.abs(pixelLuma(data, index - 4) - pixelLuma(data, index + 4));
      const vertical = Math.abs(pixelLuma(data, index) - pixelLuma(data, index + width * 4));
      if (Math.max(horizontal, vertical) >= config.edgeThreshold) edges += 1;
      samples += 1;
    }
    rowScores[y - startRow] = samples > 0 ? edges / samples : 0;
  }

  const smoothed = rowScores.map((_, index) => {
    let total = 0;
    let count = 0;
    for (let offset = -2; offset <= 2; offset += 1) {
      const score = rowScores[index + offset];
      if (score === undefined) continue;
      total += score;
      count += 1;
    }
    return count > 0 ? total / count : 0;
  });
  const mean = smoothed.reduce((total, score) => total + score, 0) / smoothed.length;
  const variance = smoothed.reduce((total, score) => total + (score - mean) ** 2, 0) / smoothed.length;
  const activeThreshold = Math.max(0.025, mean + Math.sqrt(variance) * 0.55);
  const minimumRows = Math.max(2, Math.round(height * config.minBandHeightRatio));
  const groups: Array<{ end: number; peak: number; start: number }> = [];
  let group: { end: number; peak: number; start: number } | null = null;

  for (let index = 0; index < smoothed.length; index += 1) {
    const score = smoothed[index];
    if (score >= activeThreshold) {
      if (!group) group = { end: index, peak: score, start: index };
      group.end = index;
      group.peak = Math.max(group.peak, score);
    } else if (group) {
      if (group.end - group.start + 1 >= minimumRows) groups.push(group);
      group = null;
    }
  }
  if (group && group.end - group.start + 1 >= minimumRows) groups.push(group);

  return groups
    .sort((left, right) => right.peak - left.peak)
    .slice(0, 3)
    .sort((left, right) => left.start - right.start)
    .map((band, index) => {
      const padding = Math.round(height * config.paddingRatio);
      const top = clamp(startRow + band.start - padding, 0, height - 1);
      const bottom = clamp(startRow + band.end + padding + 1, top + 1, height);
      return {
        height: (bottom - top) / height,
        id: `smart-subtitle-${index + 1}`,
        width: 0.94,
        x: 0.03,
        y: top / height,
      };
    });
}