330 lines
9.7 KiB
TypeScript
330 lines
9.7 KiB
TypeScript
import "server-only";
|
|
|
|
import sharp from "sharp";
|
|
import { uploadFile } from "@/lib/storage/service";
|
|
import type {
|
|
StoryboardCropBox,
|
|
StoryboardDetectionResult,
|
|
StoryboardGridType,
|
|
StoryboardTile,
|
|
} from "@/lib/video-agent/types";
|
|
|
|
type ProjectionPeak = {
|
|
index: number;
|
|
value: number;
|
|
};
|
|
|
|
function clamp01(value: number) {
|
|
if (!Number.isFinite(value)) return 0;
|
|
return Math.max(0, Math.min(1, value));
|
|
}
|
|
|
|
function sampleStddevFromUint8(data: Uint8Array, limit = 20000) {
|
|
const total = Math.min(data.length, Math.max(128, limit));
|
|
if (total <= 0) return 0;
|
|
const stride = Math.max(1, Math.floor(data.length / total));
|
|
const sampled: number[] = [];
|
|
for (let index = 0; index < data.length && sampled.length < total; index += stride) {
|
|
sampled.push(data[index] || 0);
|
|
}
|
|
return stddev(sampled);
|
|
}
|
|
|
|
function toArrayBuffer(buffer: Buffer): ArrayBuffer {
|
|
return buffer.buffer.slice(
|
|
buffer.byteOffset,
|
|
buffer.byteOffset + buffer.byteLength
|
|
) as ArrayBuffer;
|
|
}
|
|
|
|
function movingAverage(values: number[], radius = 3): number[] {
|
|
if (values.length === 0) return values;
|
|
const result = new Array(values.length).fill(0);
|
|
for (let index = 0; index < values.length; index += 1) {
|
|
let sum = 0;
|
|
let count = 0;
|
|
const start = Math.max(0, index - radius);
|
|
const end = Math.min(values.length - 1, index + radius);
|
|
for (let cursor = start; cursor <= end; cursor += 1) {
|
|
sum += values[cursor] || 0;
|
|
count += 1;
|
|
}
|
|
result[index] = count > 0 ? sum / count : 0;
|
|
}
|
|
return result;
|
|
}
|
|
|
|
function mean(values: number[]) {
|
|
if (values.length === 0) return 0;
|
|
return values.reduce((total, item) => total + item, 0) / values.length;
|
|
}
|
|
|
|
function stddev(values: number[]) {
|
|
if (values.length === 0) return 0;
|
|
const avg = mean(values);
|
|
const variance =
|
|
values.reduce((total, item) => total + Math.pow(item - avg, 2), 0) / values.length;
|
|
return Math.sqrt(Math.max(variance, 0));
|
|
}
|
|
|
|
function findPeakNear(
|
|
scores: number[],
|
|
targetIndex: number,
|
|
windowRadius: number
|
|
): ProjectionPeak {
|
|
const start = Math.max(0, Math.round(targetIndex - windowRadius));
|
|
const end = Math.min(scores.length - 1, Math.round(targetIndex + windowRadius));
|
|
let bestIndex = Math.round(targetIndex);
|
|
let bestValue = -Infinity;
|
|
for (let index = start; index <= end; index += 1) {
|
|
const value = scores[index] || 0;
|
|
if (value > bestValue) {
|
|
bestValue = value;
|
|
bestIndex = index;
|
|
}
|
|
}
|
|
return { index: bestIndex, value: Number.isFinite(bestValue) ? bestValue : 0 };
|
|
}
|
|
|
|
function scoreGridAxis(scores: number[], targets: number[]) {
|
|
if (targets.length === 0) {
|
|
return {
|
|
score: 0,
|
|
peaks: [] as ProjectionPeak[],
|
|
};
|
|
}
|
|
|
|
const radius = Math.max(8, Math.round(scores.length * 0.12));
|
|
const peaks = targets.map((target) => findPeakNear(scores, target, radius));
|
|
const avg = mean(scores);
|
|
const sigma = stddev(scores);
|
|
const dynamicBase = avg + sigma;
|
|
|
|
const denom = Math.max(dynamicBase, 1);
|
|
const closenessScores = peaks.map((peak, index) =>
|
|
clamp01(1 - Math.abs(peak.index - targets[index]!) / radius)
|
|
);
|
|
const strengthScores = peaks.map((peak) =>
|
|
clamp01(peak.value / (denom * 2.2))
|
|
);
|
|
const closeness = mean(closenessScores);
|
|
const strength = mean(strengthScores);
|
|
|
|
return {
|
|
score: clamp01(closeness * 0.55 + strength * 0.45),
|
|
peaks,
|
|
};
|
|
}
|
|
|
|
function buildStoryboardCropBoxes(
|
|
width: number,
|
|
height: number,
|
|
rows: number,
|
|
cols: number
|
|
): StoryboardCropBox[] {
|
|
const baseW = Math.floor(width / cols);
|
|
const baseH = Math.floor(height / rows);
|
|
const boxes: StoryboardCropBox[] = [];
|
|
let index = 1;
|
|
for (let row = 0; row < rows; row += 1) {
|
|
for (let col = 0; col < cols; col += 1) {
|
|
const x = col * baseW;
|
|
const y = row * baseH;
|
|
const w = col === cols - 1 ? width - x : baseW;
|
|
const h = row === rows - 1 ? height - y : baseH;
|
|
boxes.push({ index, row: row + 1, col: col + 1, x, y, w, h });
|
|
index += 1;
|
|
}
|
|
}
|
|
return boxes;
|
|
}
|
|
|
|
async function fetchImageBuffer(imageUrl: string) {
|
|
const response = await fetch(imageUrl);
|
|
if (!response.ok) {
|
|
throw new Error(`图片下载失败 (HTTP ${response.status})`);
|
|
}
|
|
const contentType = response.headers.get("content-type") || "";
|
|
if (!/^image\//i.test(contentType)) {
|
|
throw new Error(`输入不是图片类型: ${contentType || "unknown"}`);
|
|
}
|
|
return Buffer.from(await response.arrayBuffer());
|
|
}
|
|
|
|
async function sliceAndUploadTiles(params: {
|
|
sourceBuffer: Buffer;
|
|
cropBoxes: StoryboardCropBox[];
|
|
}): Promise<StoryboardTile[]> {
|
|
const { sourceBuffer, cropBoxes } = params;
|
|
const timestamp = Date.now();
|
|
const tiles: StoryboardTile[] = [];
|
|
for (const cropBox of cropBoxes) {
|
|
const tileBuffer = await sharp(sourceBuffer)
|
|
.extract({
|
|
left: cropBox.x,
|
|
top: cropBox.y,
|
|
width: cropBox.w,
|
|
height: cropBox.h,
|
|
})
|
|
.jpeg({ quality: 92 })
|
|
.toBuffer();
|
|
|
|
const filename = `video-storyboard-tile-${timestamp}-${cropBox.index}.jpg`;
|
|
const uploaded = await uploadFile(filename, toArrayBuffer(tileBuffer));
|
|
tiles.push({
|
|
index: cropBox.index,
|
|
row: cropBox.row,
|
|
col: cropBox.col,
|
|
imageUrl: uploaded.url,
|
|
width: cropBox.w,
|
|
height: cropBox.h,
|
|
});
|
|
}
|
|
return tiles;
|
|
}
|
|
|
|
type GridCandidate = {
|
|
gridType: StoryboardGridType;
|
|
rows: number;
|
|
cols: number;
|
|
expectedRatio: number;
|
|
};
|
|
|
|
const GRID_CANDIDATES: GridCandidate[] = [
|
|
{ gridType: "grid_2x2", rows: 2, cols: 2, expectedRatio: 1 },
|
|
{ gridType: "grid_3x2", rows: 2, cols: 3, expectedRatio: 1.5 },
|
|
{ gridType: "grid_3x3", rows: 3, cols: 3, expectedRatio: 1 },
|
|
];
|
|
|
|
function buildAxisTargets(length: number, splitCount: number) {
|
|
const targets: number[] = [];
|
|
for (let index = 1; index < splitCount; index += 1) {
|
|
targets.push((length * index) / splitCount);
|
|
}
|
|
return targets;
|
|
}
|
|
|
|
function scoreAspectRatio(actualRatio: number, expectedRatio: number) {
|
|
const tolerance = expectedRatio === 1 ? 0.45 : 0.6;
|
|
return clamp01(1 - Math.abs(actualRatio - expectedRatio) / tolerance);
|
|
}
|
|
|
|
export async function runNineGridDetectorSkill(params: {
|
|
imageUrl: string;
|
|
minConfidence: number;
|
|
}): Promise<StoryboardDetectionResult> {
|
|
const sourceImageUrl = String(params.imageUrl || "").trim();
|
|
if (!sourceImageUrl) {
|
|
throw new Error("缺少分镜拼图输入图片");
|
|
}
|
|
|
|
const sourceBuffer = await fetchImageBuffer(sourceImageUrl);
|
|
const baseImage = sharp(sourceBuffer, { failOn: "none" }).rotate();
|
|
const metadata = await baseImage.metadata();
|
|
const width = Number(metadata.width || 0);
|
|
const height = Number(metadata.height || 0);
|
|
|
|
if (width < 240 || height < 240) {
|
|
return {
|
|
isStoryboardGrid: false,
|
|
isNineGrid: false,
|
|
gridType: null,
|
|
rows: 0,
|
|
cols: 0,
|
|
shotCount: 0,
|
|
confidence: 0,
|
|
qualityScore: 0,
|
|
cropBoxes: [],
|
|
width,
|
|
height,
|
|
sourceImageUrl,
|
|
tiles: [],
|
|
};
|
|
}
|
|
|
|
const grayscale = await baseImage.greyscale().raw().toBuffer({
|
|
resolveWithObject: true,
|
|
});
|
|
const raw = grayscale.data;
|
|
const channels = grayscale.info.channels || 1;
|
|
const imgW = grayscale.info.width;
|
|
const imgH = grayscale.info.height;
|
|
const step = Math.max(1, Math.round(Math.min(imgW, imgH) / 900));
|
|
|
|
const colScores = new Array(imgW).fill(0);
|
|
const rowScores = new Array(imgH).fill(0);
|
|
|
|
for (let y = 0; y < imgH; y += step) {
|
|
const rowOffset = y * imgW * channels;
|
|
for (let x = step; x < imgW; x += step) {
|
|
const current = raw[rowOffset + x * channels] || 0;
|
|
const prev = raw[rowOffset + (x - step) * channels] || 0;
|
|
colScores[x] += Math.abs(current - prev);
|
|
}
|
|
}
|
|
|
|
for (let x = 0; x < imgW; x += step) {
|
|
for (let y = step; y < imgH; y += step) {
|
|
const current = raw[(y * imgW + x) * channels] || 0;
|
|
const prev = raw[((y - step) * imgW + x) * channels] || 0;
|
|
rowScores[y] += Math.abs(current - prev);
|
|
}
|
|
}
|
|
|
|
const smoothCols = movingAverage(colScores, 5);
|
|
const smoothRows = movingAverage(rowScores, 5);
|
|
const ratio = width / Math.max(height, 1);
|
|
const qualityScore = clamp01((sampleStddevFromUint8(raw, 20000) || 0) / 72);
|
|
|
|
const scoredCandidates = GRID_CANDIDATES.map((candidate) => {
|
|
const colTargets = buildAxisTargets(imgW, candidate.cols);
|
|
const rowTargets = buildAxisTargets(imgH, candidate.rows);
|
|
const colAxis = scoreGridAxis(smoothCols, colTargets);
|
|
const rowAxis = scoreGridAxis(smoothRows, rowTargets);
|
|
const axisScores = [colAxis.score, rowAxis.score];
|
|
const gridScore = clamp01(mean(axisScores));
|
|
const aspectScore = scoreAspectRatio(ratio, candidate.expectedRatio);
|
|
const confidence = clamp01(aspectScore * 0.34 + gridScore * 0.66);
|
|
return {
|
|
...candidate,
|
|
confidence,
|
|
aspectScore,
|
|
gridScore,
|
|
};
|
|
}).sort((left, right) => right.confidence - left.confidence);
|
|
|
|
const bestCandidate = scoredCandidates[0];
|
|
const isStoryboardGrid =
|
|
Boolean(bestCandidate) &&
|
|
bestCandidate.confidence >= params.minConfidence &&
|
|
bestCandidate.aspectScore >= 0.55;
|
|
|
|
const rows = isStoryboardGrid ? bestCandidate.rows : 0;
|
|
const cols = isStoryboardGrid ? bestCandidate.cols : 0;
|
|
const shotCount = rows * cols;
|
|
const gridType = isStoryboardGrid ? bestCandidate.gridType : null;
|
|
const isNineGrid = gridType === "grid_3x3";
|
|
const cropBoxes = isStoryboardGrid
|
|
? buildStoryboardCropBoxes(width, height, rows, cols)
|
|
: [];
|
|
const tiles = isStoryboardGrid
|
|
? await sliceAndUploadTiles({ sourceBuffer: await baseImage.toBuffer(), cropBoxes })
|
|
: [];
|
|
|
|
return {
|
|
isStoryboardGrid,
|
|
isNineGrid,
|
|
gridType,
|
|
rows,
|
|
cols,
|
|
shotCount,
|
|
confidence: isStoryboardGrid ? bestCandidate.confidence : 0,
|
|
qualityScore,
|
|
cropBoxes,
|
|
width,
|
|
height,
|
|
sourceImageUrl,
|
|
tiles,
|
|
};
|
|
}
|