import "server-only"; import sharp from "sharp"; import { uploadFile } from "@/lib/storage/service"; import type { StoryboardCropBox, StoryboardDetectionResult, StoryboardGridType, StoryboardTile, } from "@/lib/video-agent/types"; type ProjectionPeak = { index: number; value: number; }; function clamp01(value: number) { if (!Number.isFinite(value)) return 0; return Math.max(0, Math.min(1, value)); } function sampleStddevFromUint8(data: Uint8Array, limit = 20000) { const total = Math.min(data.length, Math.max(128, limit)); if (total <= 0) return 0; const stride = Math.max(1, Math.floor(data.length / total)); const sampled: number[] = []; for (let index = 0; index < data.length && sampled.length < total; index += stride) { sampled.push(data[index] || 0); } return stddev(sampled); } function toArrayBuffer(buffer: Buffer): ArrayBuffer { return buffer.buffer.slice( buffer.byteOffset, buffer.byteOffset + buffer.byteLength ) as ArrayBuffer; } function movingAverage(values: number[], radius = 3): number[] { if (values.length === 0) return values; const result = new Array(values.length).fill(0); for (let index = 0; index < values.length; index += 1) { let sum = 0; let count = 0; const start = Math.max(0, index - radius); const end = Math.min(values.length - 1, index + radius); for (let cursor = start; cursor <= end; cursor += 1) { sum += values[cursor] || 0; count += 1; } result[index] = count > 0 ? sum / count : 0; } return result; } function mean(values: number[]) { if (values.length === 0) return 0; return values.reduce((total, item) => total + item, 0) / values.length; } function stddev(values: number[]) { if (values.length === 0) return 0; const avg = mean(values); const variance = values.reduce((total, item) => total + Math.pow(item - avg, 2), 0) / values.length; return Math.sqrt(Math.max(variance, 0)); } function findPeakNear( scores: number[], targetIndex: number, windowRadius: number ): ProjectionPeak { const start = Math.max(0, Math.round(targetIndex - windowRadius)); const end = Math.min(scores.length - 1, Math.round(targetIndex + windowRadius)); let bestIndex = Math.round(targetIndex); let bestValue = -Infinity; for (let index = start; index <= end; index += 1) { const value = scores[index] || 0; if (value > bestValue) { bestValue = value; bestIndex = index; } } return { index: bestIndex, value: Number.isFinite(bestValue) ? bestValue : 0 }; } function scoreGridAxis(scores: number[], targets: number[]) { if (targets.length === 0) { return { score: 0, peaks: [] as ProjectionPeak[], }; } const radius = Math.max(8, Math.round(scores.length * 0.12)); const peaks = targets.map((target) => findPeakNear(scores, target, radius)); const avg = mean(scores); const sigma = stddev(scores); const dynamicBase = avg + sigma; const denom = Math.max(dynamicBase, 1); const closenessScores = peaks.map((peak, index) => clamp01(1 - Math.abs(peak.index - targets[index]!) / radius) ); const strengthScores = peaks.map((peak) => clamp01(peak.value / (denom * 2.2)) ); const closeness = mean(closenessScores); const strength = mean(strengthScores); return { score: clamp01(closeness * 0.55 + strength * 0.45), peaks, }; } function buildStoryboardCropBoxes( width: number, height: number, rows: number, cols: number ): StoryboardCropBox[] { const baseW = Math.floor(width / cols); const baseH = Math.floor(height / rows); const boxes: StoryboardCropBox[] = []; let index = 1; for (let row = 0; row < rows; row += 1) { for (let col = 0; col < cols; col += 1) { const x = col * baseW; const y = row * baseH; const w = col === cols - 1 ? width - x : baseW; const h = row === rows - 1 ? height - y : baseH; boxes.push({ index, row: row + 1, col: col + 1, x, y, w, h }); index += 1; } } return boxes; } async function fetchImageBuffer(imageUrl: string) { const response = await fetch(imageUrl); if (!response.ok) { throw new Error(`图片下载失败 (HTTP ${response.status})`); } const contentType = response.headers.get("content-type") || ""; if (!/^image\//i.test(contentType)) { throw new Error(`输入不是图片类型: ${contentType || "unknown"}`); } return Buffer.from(await response.arrayBuffer()); } async function sliceAndUploadTiles(params: { sourceBuffer: Buffer; cropBoxes: StoryboardCropBox[]; }): Promise { const { sourceBuffer, cropBoxes } = params; const timestamp = Date.now(); const tiles: StoryboardTile[] = []; for (const cropBox of cropBoxes) { const tileBuffer = await sharp(sourceBuffer) .extract({ left: cropBox.x, top: cropBox.y, width: cropBox.w, height: cropBox.h, }) .jpeg({ quality: 92 }) .toBuffer(); const filename = `video-storyboard-tile-${timestamp}-${cropBox.index}.jpg`; const uploaded = await uploadFile(filename, toArrayBuffer(tileBuffer)); tiles.push({ index: cropBox.index, row: cropBox.row, col: cropBox.col, imageUrl: uploaded.url, width: cropBox.w, height: cropBox.h, }); } return tiles; } type GridCandidate = { gridType: StoryboardGridType; rows: number; cols: number; expectedRatio: number; }; const GRID_CANDIDATES: GridCandidate[] = [ { gridType: "grid_2x2", rows: 2, cols: 2, expectedRatio: 1 }, { gridType: "grid_3x2", rows: 2, cols: 3, expectedRatio: 1.5 }, { gridType: "grid_3x3", rows: 3, cols: 3, expectedRatio: 1 }, ]; function buildAxisTargets(length: number, splitCount: number) { const targets: number[] = []; for (let index = 1; index < splitCount; index += 1) { targets.push((length * index) / splitCount); } return targets; } function scoreAspectRatio(actualRatio: number, expectedRatio: number) { const tolerance = expectedRatio === 1 ? 0.45 : 0.6; return clamp01(1 - Math.abs(actualRatio - expectedRatio) / tolerance); } export async function runNineGridDetectorSkill(params: { imageUrl: string; minConfidence: number; }): Promise { const sourceImageUrl = String(params.imageUrl || "").trim(); if (!sourceImageUrl) { throw new Error("缺少分镜拼图输入图片"); } const sourceBuffer = await fetchImageBuffer(sourceImageUrl); const baseImage = sharp(sourceBuffer, { failOn: "none" }).rotate(); const metadata = await baseImage.metadata(); const width = Number(metadata.width || 0); const height = Number(metadata.height || 0); if (width < 240 || height < 240) { return { isStoryboardGrid: false, isNineGrid: false, gridType: null, rows: 0, cols: 0, shotCount: 0, confidence: 0, qualityScore: 0, cropBoxes: [], width, height, sourceImageUrl, tiles: [], }; } const grayscale = await baseImage.greyscale().raw().toBuffer({ resolveWithObject: true, }); const raw = grayscale.data; const channels = grayscale.info.channels || 1; const imgW = grayscale.info.width; const imgH = grayscale.info.height; const step = Math.max(1, Math.round(Math.min(imgW, imgH) / 900)); const colScores = new Array(imgW).fill(0); const rowScores = new Array(imgH).fill(0); for (let y = 0; y < imgH; y += step) { const rowOffset = y * imgW * channels; for (let x = step; x < imgW; x += step) { const current = raw[rowOffset + x * channels] || 0; const prev = raw[rowOffset + (x - step) * channels] || 0; colScores[x] += Math.abs(current - prev); } } for (let x = 0; x < imgW; x += step) { for (let y = step; y < imgH; y += step) { const current = raw[(y * imgW + x) * channels] || 0; const prev = raw[((y - step) * imgW + x) * channels] || 0; rowScores[y] += Math.abs(current - prev); } } const smoothCols = movingAverage(colScores, 5); const smoothRows = movingAverage(rowScores, 5); const ratio = width / Math.max(height, 1); const qualityScore = clamp01((sampleStddevFromUint8(raw, 20000) || 0) / 72); const scoredCandidates = GRID_CANDIDATES.map((candidate) => { const colTargets = buildAxisTargets(imgW, candidate.cols); const rowTargets = buildAxisTargets(imgH, candidate.rows); const colAxis = scoreGridAxis(smoothCols, colTargets); const rowAxis = scoreGridAxis(smoothRows, rowTargets); const axisScores = [colAxis.score, rowAxis.score]; const gridScore = clamp01(mean(axisScores)); const aspectScore = scoreAspectRatio(ratio, candidate.expectedRatio); const confidence = clamp01(aspectScore * 0.34 + gridScore * 0.66); return { ...candidate, confidence, aspectScore, gridScore, }; }).sort((left, right) => right.confidence - left.confidence); const bestCandidate = scoredCandidates[0]; const isStoryboardGrid = Boolean(bestCandidate) && bestCandidate.confidence >= params.minConfidence && bestCandidate.aspectScore >= 0.55; const rows = isStoryboardGrid ? bestCandidate.rows : 0; const cols = isStoryboardGrid ? bestCandidate.cols : 0; const shotCount = rows * cols; const gridType = isStoryboardGrid ? bestCandidate.gridType : null; const isNineGrid = gridType === "grid_3x3"; const cropBoxes = isStoryboardGrid ? buildStoryboardCropBoxes(width, height, rows, cols) : []; const tiles = isStoryboardGrid ? await sliceAndUploadTiles({ sourceBuffer: await baseImage.toBuffer(), cropBoxes }) : []; return { isStoryboardGrid, isNineGrid, gridType, rows, cols, shotCount, confidence: isStoryboardGrid ? bestCandidate.confidence : 0, qualityScore, cropBoxes, width, height, sourceImageUrl, tiles, }; }